From 3bccce2eae52f5f746600c31f83d89f71af80a67 Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Sat, 11 Nov 2023 19:37:30 +0100
Subject: [PATCH 0001/1488] some preparement to support RocksDB as alternative
 keeper storage

---
 programs/keeper-converter/KeeperConverter.cpp |   7 +-
 src/Coordination/KeeperDispatcher.cpp         |  30 +-
 src/Coordination/KeeperDispatcher.h           |  10 +-
 src/Coordination/KeeperServer.cpp             |  24 +-
 src/Coordination/KeeperServer.h               |  10 +-
 src/Coordination/KeeperSnapshotManager.cpp    | 101 ++-
 src/Coordination/KeeperSnapshotManager.h      |  36 +-
 src/Coordination/KeeperStateMachine.cpp       | 172 +++--
 src/Coordination/KeeperStateMachine.h         | 201 ++++--
 src/Coordination/KeeperStorage.cpp            | 672 ++++++++++--------
 src/Coordination/KeeperStorage.h              | 174 ++---
 src/Coordination/SnapshotableHashTable.h      |   1 +
 src/Coordination/ZooKeeperDataReader.cpp      |  33 +-
 src/Coordination/ZooKeeperDataReader.h        |  12 +-
 src/Coordination/tests/gtest_coordination.cpp |  86 +--
 15 files changed, 900 insertions(+), 669 deletions(-)

diff --git a/programs/keeper-converter/KeeperConverter.cpp b/programs/keeper-converter/KeeperConverter.cpp
index 20448aafa2f..a426f67e429 100644
--- a/programs/keeper-converter/KeeperConverter.cpp
+++ b/programs/keeper-converter/KeeperConverter.cpp
@@ -44,16 +44,17 @@ int mainEntryClickHouseKeeperConverter(int argc, char ** argv)
         keeper_context->setDigestEnabled(true);
         keeper_context->setSnapshotDisk(std::make_shared<DiskLocal>("Keeper-snapshots", options["output-dir"].as<std::string>()));
 
-        DB::KeeperStorage storage(/* tick_time_ms */ 500, /* superdigest */ "", keeper_context, /* initialize_system_nodes */ false);
+        /// TODO(hanfei): support rocksdb here
+        DB::KeeperMemoryStorage storage(/* tick_time_ms */ 500, /* superdigest */ "", keeper_context, /* initialize_system_nodes */ false);
 
         DB::deserializeKeeperStorageFromSnapshotsDir(storage, options["zookeeper-snapshots-dir"].as<std::string>(), logger);
         storage.initializeSystemNodes();
 
         DB::deserializeLogsAndApplyToStorage(storage, options["zookeeper-logs-dir"].as<std::string>(), logger);
         DB::SnapshotMetadataPtr snapshot_meta = std::make_shared<DB::SnapshotMetadata>(storage.getZXID(), 1, std::make_shared<nuraft::cluster_config>());
-        DB::KeeperStorageSnapshot snapshot(&storage, snapshot_meta);
+        DB::KeeperStorageSnapshot<DB::KeeperMemoryStorage> snapshot(&storage, snapshot_meta);
 
-        DB::KeeperSnapshotManager manager(1, keeper_context);
+        DB::KeeperSnapshotManager<DB::KeeperMemoryStorage> manager(1, keeper_context);
         auto snp = manager.serializeSnapshotToBuffer(snapshot);
         auto file_info = manager.serializeSnapshotBufferToDisk(*snp, storage.getZXID());
         std::cout << "Snapshot serialized to path:" << fs::path(file_info.disk->getPath()) / file_info.path << std::endl;
diff --git a/src/Coordination/KeeperDispatcher.cpp b/src/Coordination/KeeperDispatcher.cpp
index 9e0cdbd6cd3..f97fc448532 100644
--- a/src/Coordination/KeeperDispatcher.cpp
+++ b/src/Coordination/KeeperDispatcher.cpp
@@ -66,13 +66,13 @@ void KeeperDispatcher::requestThread()
     RaftAppendResult prev_result = nullptr;
     /// Requests from previous iteration. We store them to be able
     /// to send errors to the client.
-    KeeperStorage::RequestsForSessions prev_batch;
+    KeeperStorageBase::RequestsForSessions prev_batch;
 
     auto & shutdown_called = keeper_context->shutdown_called;
 
     while (!shutdown_called)
     {
-        KeeperStorage::RequestForSession request;
+        KeeperStorageBase::RequestForSession request;
 
         auto coordination_settings = configuration_and_settings->coordination_settings;
         uint64_t max_wait = coordination_settings->operation_timeout_ms.totalMilliseconds();
@@ -93,7 +93,7 @@ void KeeperDispatcher::requestThread()
                 if (shutdown_called)
                     break;
 
-                KeeperStorage::RequestsForSessions current_batch;
+                KeeperStorageBase::RequestsForSessions current_batch;
                 size_t current_batch_bytes_size = 0;
 
                 bool has_read_request = false;
@@ -244,7 +244,7 @@ void KeeperDispatcher::responseThread()
     auto & shutdown_called = keeper_context->shutdown_called;
     while (!shutdown_called)
     {
-        KeeperStorage::ResponseForSession response_for_session;
+        KeeperStorageBase::ResponseForSession response_for_session;
 
         uint64_t max_wait = configuration_and_settings->coordination_settings->operation_timeout_ms.totalMilliseconds();
 
@@ -340,7 +340,7 @@ bool KeeperDispatcher::putRequest(const Coordination::ZooKeeperRequestPtr & requ
             return false;
     }
 
-    KeeperStorage::RequestForSession request_info;
+    KeeperStorageBase::RequestForSession request_info;
     request_info.request = request;
     using namespace std::chrono;
     request_info.time = duration_cast<milliseconds>(system_clock::now().time_since_epoch()).count();
@@ -386,7 +386,7 @@ void KeeperDispatcher::initialize(const Poco::Util::AbstractConfiguration & conf
         snapshots_queue,
         keeper_context,
         snapshot_s3,
-        [this](uint64_t log_idx, const KeeperStorage::RequestForSession & request_for_session)
+        [this](uint64_t log_idx, const KeeperStorageBase::RequestForSession & request_for_session)
         {
             {
                 /// check if we have queue of read requests depending on this request to be committed
@@ -486,7 +486,7 @@ void KeeperDispatcher::shutdown()
                 update_configuration_thread.join();
         }
 
-        KeeperStorage::RequestForSession request_for_session;
+        KeeperStorageBase::RequestForSession request_for_session;
 
         /// Set session expired for all pending requests
         while (requests_queue && requests_queue->tryPop(request_for_session))
@@ -497,7 +497,7 @@ void KeeperDispatcher::shutdown()
             setResponse(request_for_session.session_id, response);
         }
 
-        KeeperStorage::RequestsForSessions close_requests;
+        KeeperStorageBase::RequestsForSessions close_requests;
         {
             /// Clear all registered sessions
             std::lock_guard lock(session_to_response_callback_mutex);
@@ -511,7 +511,7 @@ void KeeperDispatcher::shutdown()
                     auto request = Coordination::ZooKeeperRequestFactory::instance().get(Coordination::OpNum::Close);
                     request->xid = Coordination::CLOSE_XID;
                     using namespace std::chrono;
-                    KeeperStorage::RequestForSession request_info
+                    KeeperStorageBase::RequestForSession request_info
                     {
                         .session_id = session,
                         .time = duration_cast<milliseconds>(system_clock::now().time_since_epoch()).count(),
@@ -609,7 +609,7 @@ void KeeperDispatcher::sessionCleanerTask()
                     auto request = Coordination::ZooKeeperRequestFactory::instance().get(Coordination::OpNum::Close);
                     request->xid = Coordination::CLOSE_XID;
                     using namespace std::chrono;
-                    KeeperStorage::RequestForSession request_info
+                    KeeperStorageBase::RequestForSession request_info
                     {
                         .session_id = dead_session,
                         .time = duration_cast<milliseconds>(system_clock::now().time_since_epoch()).count(),
@@ -657,16 +657,16 @@ void KeeperDispatcher::finishSession(int64_t session_id)
     }
 }
 
-void KeeperDispatcher::addErrorResponses(const KeeperStorage::RequestsForSessions & requests_for_sessions, Coordination::Error error)
+void KeeperDispatcher::addErrorResponses(const KeeperStorageBase::RequestsForSessions & requests_for_sessions, Coordination::Error error)
 {
     for (const auto & request_for_session : requests_for_sessions)
     {
-        KeeperStorage::ResponsesForSessions responses;
+        KeeperStorageBase::ResponsesForSessions responses;
         auto response = request_for_session.request->makeResponse();
         response->xid = request_for_session.request->xid;
         response->zxid = 0;
         response->error = error;
-        if (!responses_queue.push(DB::KeeperStorage::ResponseForSession{request_for_session.session_id, response}))
+        if (!responses_queue.push(DB::KeeperStorageBase::ResponseForSession{request_for_session.session_id, response}))
             throw Exception(ErrorCodes::SYSTEM_ERROR,
                 "Could not push error response xid {} zxid {} error message {} to responses queue",
                 response->xid,
@@ -675,7 +675,7 @@ void KeeperDispatcher::addErrorResponses(const KeeperStorage::RequestsForSession
     }
 }
 
-nuraft::ptr<nuraft::buffer> KeeperDispatcher::forceWaitAndProcessResult(RaftAppendResult & result, KeeperStorage::RequestsForSessions & requests_for_sessions)
+nuraft::ptr<nuraft::buffer> KeeperDispatcher::forceWaitAndProcessResult(RaftAppendResult & result, KeeperStorageBase::RequestsForSessions & requests_for_sessions)
 {
     if (!result->has_result())
         result->get();
@@ -697,7 +697,7 @@ int64_t KeeperDispatcher::getSessionID(int64_t session_timeout_ms)
 {
     /// New session id allocation is a special request, because we cannot process it in normal
     /// way: get request -> put to raft -> set response for registered callback.
-    KeeperStorage::RequestForSession request_info;
+    KeeperStorageBase::RequestForSession request_info;
     std::shared_ptr<Coordination::ZooKeeperSessionIDRequest> request = std::make_shared<Coordination::ZooKeeperSessionIDRequest>();
     /// Internal session id. It's a temporary number which is unique for each client on this server
     /// but can be same on different servers.
diff --git a/src/Coordination/KeeperDispatcher.h b/src/Coordination/KeeperDispatcher.h
index 39941f55d5e..f8dc8503380 100644
--- a/src/Coordination/KeeperDispatcher.h
+++ b/src/Coordination/KeeperDispatcher.h
@@ -27,7 +27,7 @@ using ZooKeeperResponseCallback = std::function<void(const Coordination::ZooKeep
 class KeeperDispatcher
 {
 private:
-    using RequestsQueue = ConcurrentBoundedQueue<KeeperStorage::RequestForSession>;
+    using RequestsQueue = ConcurrentBoundedQueue<KeeperStorageBase::RequestForSession>;
     using SessionToResponseCallback = std::unordered_map<int64_t, ZooKeeperResponseCallback>;
     using ClusterUpdateQueue = ConcurrentBoundedQueue<ClusterUpdateAction>;
 
@@ -96,11 +96,11 @@ private:
 
     /// Add error responses for requests to responses queue.
     /// Clears requests.
-    void addErrorResponses(const KeeperStorage::RequestsForSessions & requests_for_sessions, Coordination::Error error);
+    void addErrorResponses(const KeeperStorageBase::RequestsForSessions & requests_for_sessions, Coordination::Error error);
 
     /// Forcefully wait for result and sets errors if something when wrong.
     /// Clears both arguments
-    nuraft::ptr<nuraft::buffer> forceWaitAndProcessResult(RaftAppendResult & result, KeeperStorage::RequestsForSessions & requests_for_sessions);
+    nuraft::ptr<nuraft::buffer> forceWaitAndProcessResult(RaftAppendResult & result, KeeperStorageBase::RequestsForSessions & requests_for_sessions);
 
 public:
     std::mutex read_request_queue_mutex;
@@ -108,7 +108,7 @@ public:
     std::atomic<uint64_t> our_last_committed_log_idx = 0;
 
     /// queue of read requests that can be processed after a request with specific session ID and XID is committed
-    std::unordered_map<int64_t, std::unordered_map<Coordination::XID, KeeperStorage::RequestsForSessions>> read_request_queue;
+    std::unordered_map<int64_t, std::unordered_map<Coordination::XID, KeeperStorageBase::RequestsForSessions>> read_request_queue;
 
     /// Just allocate some objects, real initialization is done by `intialize method`
     KeeperDispatcher();
@@ -189,7 +189,7 @@ public:
 
     Keeper4LWInfo getKeeper4LWInfo() const;
 
-    const KeeperStateMachine & getStateMachine() const
+    const IKeeperStateMachine & getStateMachine() const
     {
         return *server->getKeeperStateMachine();
     }
diff --git a/src/Coordination/KeeperServer.cpp b/src/Coordination/KeeperServer.cpp
index fd3db6f1032..b25d5e85431 100644
--- a/src/Coordination/KeeperServer.cpp
+++ b/src/Coordination/KeeperServer.cpp
@@ -116,7 +116,7 @@ KeeperServer::KeeperServer(
     SnapshotsQueue & snapshots_queue_,
     KeeperContextPtr keeper_context_,
     KeeperSnapshotManagerS3 & snapshot_manager_s3,
-    KeeperStateMachine::CommitCallback commit_callback)
+    IKeeperStateMachine::CommitCallback commit_callback)
     : server_id(configuration_and_settings_->server_id)
     , coordination_settings(configuration_and_settings_->coordination_settings)
     , log(&Poco::Logger::get("KeeperServer"))
@@ -128,7 +128,7 @@ KeeperServer::KeeperServer(
     if (coordination_settings->quorum_reads)
         LOG_WARNING(log, "Quorum reads enabled, Keeper will work slower.");
 
-    state_machine = nuraft::cs_new<KeeperStateMachine>(
+    state_machine = nuraft::cs_new<KeeperStateMachine<KeeperMemoryStorage>>(
         responses_queue_,
         snapshots_queue_,
         coordination_settings,
@@ -478,7 +478,7 @@ namespace
 {
 
 // Serialize the request for the log entry
-nuraft::ptr<nuraft::buffer> getZooKeeperLogEntry(const KeeperStorage::RequestForSession & request_for_session)
+nuraft::ptr<nuraft::buffer> getZooKeeperLogEntry(const KeeperStorageBase::RequestForSession & request_for_session)
 {
     DB::WriteBufferFromNuraftBuffer write_buf;
     DB::writeIntBinary(request_for_session.session_id, write_buf);
@@ -486,7 +486,7 @@ nuraft::ptr<nuraft::buffer> getZooKeeperLogEntry(const KeeperStorage::RequestFor
     DB::writeIntBinary(request_for_session.time, write_buf);
     /// we fill with dummy values to eliminate unnecessary copy later on when we will write correct values
     DB::writeIntBinary(static_cast<int64_t>(0), write_buf); /// zxid
-    DB::writeIntBinary(KeeperStorage::DigestVersion::NO_DIGEST, write_buf); /// digest version or NO_DIGEST flag
+    DB::writeIntBinary(KeeperStorageBase::DigestVersion::NO_DIGEST, write_buf); /// digest version or NO_DIGEST flag
     DB::writeIntBinary(static_cast<uint64_t>(0), write_buf); /// digest value
     /// if new fields are added, update KeeperStateMachine::ZooKeeperLogSerializationVersion along with parseRequest function and PreAppendLog callback handler
     return write_buf.getBuffer();
@@ -494,7 +494,7 @@ nuraft::ptr<nuraft::buffer> getZooKeeperLogEntry(const KeeperStorage::RequestFor
 
 }
 
-void KeeperServer::putLocalReadRequest(const KeeperStorage::RequestForSession & request_for_session)
+void KeeperServer::putLocalReadRequest(const KeeperStorageBase::RequestForSession & request_for_session)
 {
     if (!request_for_session.request->isReadRequest())
         throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot process non-read request locally");
@@ -502,7 +502,7 @@ void KeeperServer::putLocalReadRequest(const KeeperStorage::RequestForSession &
     state_machine->processReadRequest(request_for_session);
 }
 
-RaftAppendResult KeeperServer::putRequestBatch(const KeeperStorage::RequestsForSessions & requests_for_sessions)
+RaftAppendResult KeeperServer::putRequestBatch(const KeeperStorageBase::RequestsForSessions & requests_for_sessions)
 {
     std::vector<nuraft::ptr<nuraft::buffer>> entries;
     for (const auto & request_for_session : requests_for_sessions)
@@ -725,7 +725,7 @@ nuraft::cb_func::ReturnCode KeeperServer::callbackFunc(nuraft::cb_func::Type typ
 
                 auto entry_buf = entry->get_buf_ptr();
 
-                KeeperStateMachine::ZooKeeperLogSerializationVersion serialization_version;
+                IKeeperStateMachine::ZooKeeperLogSerializationVersion serialization_version;
                 auto request_for_session = state_machine->parseRequest(*entry_buf, /*final=*/false, &serialization_version);
                 request_for_session->zxid = next_zxid;
                 if (!state_machine->preprocess(*request_for_session))
@@ -735,10 +735,10 @@ nuraft::cb_func::ReturnCode KeeperServer::callbackFunc(nuraft::cb_func::Type typ
 
                 /// older versions of Keeper can send logs that are missing some fields
                 size_t bytes_missing = 0;
-                if (serialization_version < KeeperStateMachine::ZooKeeperLogSerializationVersion::WITH_TIME)
+                if (serialization_version < IKeeperStateMachine::ZooKeeperLogSerializationVersion::WITH_TIME)
                     bytes_missing += sizeof(request_for_session->time);
 
-                if (serialization_version < KeeperStateMachine::ZooKeeperLogSerializationVersion::WITH_ZXID_DIGEST)
+                if (serialization_version < IKeeperStateMachine::ZooKeeperLogSerializationVersion::WITH_ZXID_DIGEST)
                     bytes_missing += sizeof(request_for_session->zxid) + sizeof(request_for_session->digest->version) + sizeof(request_for_session->digest->value);
 
                 if (bytes_missing != 0)
@@ -752,19 +752,19 @@ nuraft::cb_func::ReturnCode KeeperServer::callbackFunc(nuraft::cb_func::Type typ
                 size_t write_buffer_header_size
                     = sizeof(request_for_session->zxid) + sizeof(request_for_session->digest->version) + sizeof(request_for_session->digest->value);
 
-                if (serialization_version < KeeperStateMachine::ZooKeeperLogSerializationVersion::WITH_TIME)
+                if (serialization_version < IKeeperStateMachine::ZooKeeperLogSerializationVersion::WITH_TIME)
                     write_buffer_header_size += sizeof(request_for_session->time);
 
                 auto * buffer_start = reinterpret_cast<BufferBase::Position>(entry_buf->data_begin() + entry_buf->size() - write_buffer_header_size);
 
                 WriteBufferFromPointer write_buf(buffer_start, write_buffer_header_size);
 
-                if (serialization_version < KeeperStateMachine::ZooKeeperLogSerializationVersion::WITH_TIME)
+                if (serialization_version < IKeeperStateMachine::ZooKeeperLogSerializationVersion::WITH_TIME)
                     writeIntBinary(request_for_session->time, write_buf);
 
                 writeIntBinary(request_for_session->zxid, write_buf);
                 writeIntBinary(request_for_session->digest->version, write_buf);
-                if (request_for_session->digest->version != KeeperStorage::NO_DIGEST)
+                if (request_for_session->digest->version != KeeperStorageBase::NO_DIGEST)
                     writeIntBinary(request_for_session->digest->value, write_buf);
 
                 write_buf.finalize();
diff --git a/src/Coordination/KeeperServer.h b/src/Coordination/KeeperServer.h
index ed58418fe5f..402dcb38742 100644
--- a/src/Coordination/KeeperServer.h
+++ b/src/Coordination/KeeperServer.h
@@ -24,7 +24,7 @@ private:
 
     CoordinationSettingsPtr coordination_settings;
 
-    nuraft::ptr<KeeperStateMachine> state_machine;
+    nuraft::ptr<IKeeperStateMachine> state_machine;
 
     nuraft::ptr<KeeperStateManager> state_manager;
 
@@ -79,26 +79,26 @@ public:
         SnapshotsQueue & snapshots_queue_,
         KeeperContextPtr keeper_context_,
         KeeperSnapshotManagerS3 & snapshot_manager_s3,
-        KeeperStateMachine::CommitCallback commit_callback);
+        IKeeperStateMachine::CommitCallback commit_callback);
 
     /// Load state machine from the latest snapshot and load log storage. Start NuRaft with required settings.
     void startup(const Poco::Util::AbstractConfiguration & config, bool enable_ipv6 = true);
 
     /// Put local read request and execute in state machine directly and response into
     /// responses queue
-    void putLocalReadRequest(const KeeperStorage::RequestForSession & request);
+    void putLocalReadRequest(const KeeperStorageBase::RequestForSession & request);
 
     bool isRecovering() const { return is_recovering; }
     bool reconfigEnabled() const { return enable_reconfiguration; }
 
     /// Put batch of requests into Raft and get result of put. Responses will be set separately into
     /// responses_queue.
-    RaftAppendResult putRequestBatch(const KeeperStorage::RequestsForSessions & requests);
+    RaftAppendResult putRequestBatch(const KeeperStorageBase::RequestsForSessions & requests);
 
     /// Return set of the non-active sessions
     std::vector<int64_t> getDeadSessions();
 
-    nuraft::ptr<KeeperStateMachine> getKeeperStateMachine() const { return state_machine; }
+    nuraft::ptr<IKeeperStateMachine> getKeeperStateMachine() const { return state_machine; }
 
     void forceRecovery();
 
diff --git a/src/Coordination/KeeperSnapshotManager.cpp b/src/Coordination/KeeperSnapshotManager.cpp
index a216272a9e1..739a31d89e7 100644
--- a/src/Coordination/KeeperSnapshotManager.cpp
+++ b/src/Coordination/KeeperSnapshotManager.cpp
@@ -68,7 +68,8 @@ namespace
         return base;
     }
 
-    void writeNode(const KeeperStorage::Node & node, SnapshotVersion version, WriteBuffer & out)
+    template<typename Node>
+    void writeNode(const Node & node, SnapshotVersion version, WriteBuffer & out)
     {
         writeBinary(node.getData(), out);
 
@@ -96,7 +97,8 @@ namespace
         }
     }
 
-    void readNode(KeeperStorage::Node & node, ReadBuffer & in, SnapshotVersion version, ACLMap & acl_map)
+    template<typename Node>
+    void readNode(Node & node, ReadBuffer & in, SnapshotVersion version, ACLMap & acl_map)
     {
         String new_data;
         readBinary(new_data, in);
@@ -169,7 +171,8 @@ namespace
     }
 }
 
-void KeeperStorageSnapshot::serialize(const KeeperStorageSnapshot & snapshot, WriteBuffer & out, KeeperContextPtr keeper_context)
+template<typename Storage>
+void KeeperStorageSnapshot<Storage>::serialize(const KeeperStorageSnapshot<Storage> & snapshot, WriteBuffer & out, KeeperContextPtr keeper_context)
 {
     writeBinary(static_cast<uint8_t>(snapshot.version), out);
     serializeSnapshotMetadata(snapshot.snapshot_meta, out);
@@ -179,11 +182,11 @@ void KeeperStorageSnapshot::serialize(const KeeperStorageSnapshot & snapshot, Wr
         writeBinary(snapshot.zxid, out);
         if (keeper_context->digestEnabled())
         {
-            writeBinary(static_cast<uint8_t>(KeeperStorage::CURRENT_DIGEST_VERSION), out);
+            writeBinary(static_cast<uint8_t>(Storage::CURRENT_DIGEST_VERSION), out);
             writeBinary(snapshot.nodes_digest, out);
         }
         else
-            writeBinary(static_cast<uint8_t>(KeeperStorage::NO_DIGEST), out);
+            writeBinary(static_cast<uint8_t>(Storage::NO_DIGEST), out);
     }
 
     writeBinary(snapshot.session_id, out);
@@ -256,7 +259,7 @@ void KeeperStorageSnapshot::serialize(const KeeperStorageSnapshot & snapshot, Wr
         writeBinary(session_id, out);
         writeBinary(timeout, out);
 
-        KeeperStorage::AuthIDs ids;
+        KeeperStorageBase::AuthIDs ids;
         if (snapshot.session_and_auth.contains(session_id))
             ids = snapshot.session_and_auth.at(session_id);
 
@@ -277,7 +280,8 @@ void KeeperStorageSnapshot::serialize(const KeeperStorageSnapshot & snapshot, Wr
     }
 }
 
-void KeeperStorageSnapshot::deserialize(SnapshotDeserializationResult & deserialization_result, ReadBuffer & in, KeeperContextPtr keeper_context)
+template<typename Storage>
+void KeeperStorageSnapshot<Storage>::deserialize(SnapshotDeserializationResult<Storage> & deserialization_result, ReadBuffer & in, KeeperContextPtr keeper_context)
 {
     uint8_t version;
     readBinary(version, in);
@@ -286,7 +290,7 @@ void KeeperStorageSnapshot::deserialize(SnapshotDeserializationResult & deserial
         throw Exception(ErrorCodes::UNKNOWN_FORMAT_VERSION, "Unsupported snapshot version {}", version);
 
     deserialization_result.snapshot_meta = deserializeSnapshotMetadata(in);
-    KeeperStorage & storage = *deserialization_result.storage;
+    Storage & storage = *deserialization_result.storage;
 
     bool recalculate_digest = keeper_context->digestEnabled();
     if (version >= SnapshotVersion::V5)
@@ -294,11 +298,11 @@ void KeeperStorageSnapshot::deserialize(SnapshotDeserializationResult & deserial
         readBinary(storage.zxid, in);
         uint8_t digest_version;
         readBinary(digest_version, in);
-        if (digest_version != KeeperStorage::DigestVersion::NO_DIGEST)
+        if (digest_version != Storage::DigestVersion::NO_DIGEST)
         {
             uint64_t nodes_digest;
             readBinary(nodes_digest, in);
-            if (digest_version == KeeperStorage::CURRENT_DIGEST_VERSION)
+            if (digest_version == Storage::CURRENT_DIGEST_VERSION)
             {
                 storage.nodes_digest = nodes_digest;
                 recalculate_digest = false;
@@ -361,7 +365,7 @@ void KeeperStorageSnapshot::deserialize(SnapshotDeserializationResult & deserial
     {
         std::string path;
         readBinary(path, in);
-        KeeperStorage::Node node{};
+        typename Storage::Node node{};
         readNode(node, in, current_version, storage.acl_map);
 
         using enum Coordination::PathMatchResult;
@@ -389,7 +393,7 @@ void KeeperStorageSnapshot::deserialize(SnapshotDeserializationResult & deserial
                 if (keeper_context->ignoreSystemPathOnStartup() || keeper_context->getServerState() != KeeperContext::Phase::INIT)
                 {
                     LOG_ERROR(&Poco::Logger::get("KeeperSnapshotManager"), "{}. Ignoring it", error_msg);
-                    node = KeeperStorage::Node{};
+                    node = typename Storage::Node{};
                 }
                 else
                     throw Exception(
@@ -417,7 +421,7 @@ void KeeperStorageSnapshot::deserialize(SnapshotDeserializationResult & deserial
         {
             auto parent_path = parentNodePath(itr.key);
             storage.container.updateValue(
-                parent_path, [version, path = itr.key](KeeperStorage::Node & value) { value.addChild(getBaseNodeName(path), /*update_size*/ version < SnapshotVersion::V4); });
+                parent_path, [version, path = itr.key](typename Storage::Node & value) { value.addChild(getBaseNodeName(path), /*update_size*/ version < SnapshotVersion::V4); });
         }
     }
 
@@ -457,14 +461,14 @@ void KeeperStorageSnapshot::deserialize(SnapshotDeserializationResult & deserial
             size_t session_auths_size;
             readBinary(session_auths_size, in);
 
-            KeeperStorage::AuthIDs ids;
+            typename Storage::AuthIDs ids;
             size_t session_auth_counter = 0;
             while (session_auth_counter < session_auths_size)
             {
                 String scheme, id;
                 readBinary(scheme, in);
                 readBinary(id, in);
-                ids.emplace_back(KeeperStorage::AuthID{scheme, id});
+                ids.emplace_back(typename Storage::AuthID{scheme, id});
 
                 session_auth_counter++;
             }
@@ -487,7 +491,8 @@ void KeeperStorageSnapshot::deserialize(SnapshotDeserializationResult & deserial
     }
 }
 
-KeeperStorageSnapshot::KeeperStorageSnapshot(KeeperStorage * storage_, uint64_t up_to_log_idx_, const ClusterConfigPtr & cluster_config_)
+template<typename Storage>
+KeeperStorageSnapshot<Storage>::KeeperStorageSnapshot(Storage * storage_, uint64_t up_to_log_idx_, const ClusterConfigPtr & cluster_config_)
     : storage(storage_)
     , snapshot_meta(std::make_shared<SnapshotMetadata>(up_to_log_idx_, 0, std::make_shared<nuraft::cluster_config>()))
     , session_id(storage->session_id_counter)
@@ -504,8 +509,9 @@ KeeperStorageSnapshot::KeeperStorageSnapshot(KeeperStorage * storage_, uint64_t
     session_and_auth = storage->session_and_auth;
 }
 
-KeeperStorageSnapshot::KeeperStorageSnapshot(
-    KeeperStorage * storage_, const SnapshotMetadataPtr & snapshot_meta_, const ClusterConfigPtr & cluster_config_)
+template<typename Storage>
+KeeperStorageSnapshot<Storage>::KeeperStorageSnapshot(
+    Storage * storage_, const SnapshotMetadataPtr & snapshot_meta_, const ClusterConfigPtr & cluster_config_)
     : storage(storage_)
     , snapshot_meta(snapshot_meta_)
     , session_id(storage->session_id_counter)
@@ -522,12 +528,14 @@ KeeperStorageSnapshot::KeeperStorageSnapshot(
     session_and_auth = storage->session_and_auth;
 }
 
-KeeperStorageSnapshot::~KeeperStorageSnapshot()
+template<typename Storage>
+KeeperStorageSnapshot<Storage>::~KeeperStorageSnapshot()
 {
     storage->disableSnapshotMode();
 }
 
-KeeperSnapshotManager::KeeperSnapshotManager(
+template<typename Storage>
+KeeperSnapshotManager<Storage>::KeeperSnapshotManager(
     size_t snapshots_to_keep_,
     const KeeperContextPtr & keeper_context_,
     bool compress_snapshots_zstd_,
@@ -606,7 +614,8 @@ KeeperSnapshotManager::KeeperSnapshotManager(
     moveSnapshotsIfNeeded();
 }
 
-SnapshotFileInfo KeeperSnapshotManager::serializeSnapshotBufferToDisk(nuraft::buffer & buffer, uint64_t up_to_log_idx)
+template<typename Storage>
+SnapshotFileInfo KeeperSnapshotManager<Storage>::serializeSnapshotBufferToDisk(nuraft::buffer & buffer, uint64_t up_to_log_idx)
 {
     ReadBufferFromNuraftBuffer reader(buffer);
 
@@ -634,7 +643,8 @@ SnapshotFileInfo KeeperSnapshotManager::serializeSnapshotBufferToDisk(nuraft::bu
     return {snapshot_file_name, disk};
 }
 
-nuraft::ptr<nuraft::buffer> KeeperSnapshotManager::deserializeLatestSnapshotBufferFromDisk()
+template<typename Storage>
+nuraft::ptr<nuraft::buffer> KeeperSnapshotManager<Storage>::deserializeLatestSnapshotBufferFromDisk()
 {
     while (!existing_snapshots.empty())
     {
@@ -655,7 +665,8 @@ nuraft::ptr<nuraft::buffer> KeeperSnapshotManager::deserializeLatestSnapshotBuff
     return nullptr;
 }
 
-nuraft::ptr<nuraft::buffer> KeeperSnapshotManager::deserializeSnapshotBufferFromDisk(uint64_t up_to_log_idx) const
+template<typename Storage>
+nuraft::ptr<nuraft::buffer> KeeperSnapshotManager<Storage>::deserializeSnapshotBufferFromDisk(uint64_t up_to_log_idx) const
 {
     const auto & [snapshot_path, snapshot_disk] = existing_snapshots.at(up_to_log_idx);
     WriteBufferFromNuraftBuffer writer;
@@ -664,7 +675,8 @@ nuraft::ptr<nuraft::buffer> KeeperSnapshotManager::deserializeSnapshotBufferFrom
     return writer.getBuffer();
 }
 
-nuraft::ptr<nuraft::buffer> KeeperSnapshotManager::serializeSnapshotToBuffer(const KeeperStorageSnapshot & snapshot) const
+template<typename Storage>
+nuraft::ptr<nuraft::buffer> KeeperSnapshotManager<Storage>::serializeSnapshotToBuffer(const KeeperStorageSnapshot<Storage> & snapshot) const
 {
     std::unique_ptr<WriteBufferFromNuraftBuffer> writer = std::make_unique<WriteBufferFromNuraftBuffer>();
     auto * buffer_raw_ptr = writer.get();
@@ -674,13 +686,13 @@ nuraft::ptr<nuraft::buffer> KeeperSnapshotManager::serializeSnapshotToBuffer(con
     else
         compressed_writer = std::make_unique<CompressedWriteBuffer>(*writer);
 
-    KeeperStorageSnapshot::serialize(snapshot, *compressed_writer, keeper_context);
+    KeeperStorageSnapshot<Storage>::serialize(snapshot, *compressed_writer, keeper_context);
     compressed_writer->finalize();
     return buffer_raw_ptr->getBuffer();
 }
 
-
-bool KeeperSnapshotManager::isZstdCompressed(nuraft::ptr<nuraft::buffer> buffer)
+template<typename Storage>
+bool KeeperSnapshotManager<Storage>::isZstdCompressed(nuraft::ptr<nuraft::buffer> buffer)
 {
     static constexpr unsigned char ZSTD_COMPRESSED_MAGIC[4] = {0x28, 0xB5, 0x2F, 0xFD};
 
@@ -691,7 +703,8 @@ bool KeeperSnapshotManager::isZstdCompressed(nuraft::ptr<nuraft::buffer> buffer)
     return memcmp(magic_from_buffer, ZSTD_COMPRESSED_MAGIC, 4) == 0;
 }
 
-SnapshotDeserializationResult KeeperSnapshotManager::deserializeSnapshotFromBuffer(nuraft::ptr<nuraft::buffer> buffer) const
+template<typename Storage>
+SnapshotDeserializationResult<Storage> KeeperSnapshotManager<Storage>::deserializeSnapshotFromBuffer(nuraft::ptr<nuraft::buffer> buffer) const
 {
     bool is_zstd_compressed = isZstdCompressed(buffer);
 
@@ -703,14 +716,15 @@ SnapshotDeserializationResult KeeperSnapshotManager::deserializeSnapshotFromBuff
     else
         compressed_reader = std::make_unique<CompressedReadBuffer>(*reader);
 
-    SnapshotDeserializationResult result;
-    result.storage = std::make_unique<KeeperStorage>(storage_tick_time, superdigest, keeper_context, /* initialize_system_nodes */ false);
-    KeeperStorageSnapshot::deserialize(result, *compressed_reader, keeper_context);
+    SnapshotDeserializationResult<Storage> result;
+    result.storage = std::make_unique<Storage>(storage_tick_time, superdigest, keeper_context, /* initialize_system_nodes */ false);
+    KeeperStorageSnapshot<Storage>::deserialize(result, *compressed_reader, keeper_context);
     result.storage->initializeSystemNodes();
     return result;
 }
 
-SnapshotDeserializationResult KeeperSnapshotManager::restoreFromLatestSnapshot()
+template<typename Storage>
+SnapshotDeserializationResult<Storage> KeeperSnapshotManager<Storage>::restoreFromLatestSnapshot()
 {
     if (existing_snapshots.empty())
         return {};
@@ -721,23 +735,27 @@ SnapshotDeserializationResult KeeperSnapshotManager::restoreFromLatestSnapshot()
     return deserializeSnapshotFromBuffer(buffer);
 }
 
-DiskPtr KeeperSnapshotManager::getDisk() const
+template<typename Storage>
+DiskPtr KeeperSnapshotManager<Storage>::getDisk() const
 {
     return keeper_context->getSnapshotDisk();
 }
 
-DiskPtr KeeperSnapshotManager::getLatestSnapshotDisk() const
+template<typename Storage>
+DiskPtr KeeperSnapshotManager<Storage>::getLatestSnapshotDisk() const
 {
     return keeper_context->getLatestSnapshotDisk();
 }
 
-void KeeperSnapshotManager::removeOutdatedSnapshotsIfNeeded()
+template<typename Storage>
+void KeeperSnapshotManager<Storage>::removeOutdatedSnapshotsIfNeeded()
 {
     while (existing_snapshots.size() > snapshots_to_keep)
         removeSnapshot(existing_snapshots.begin()->first);
 }
 
-void KeeperSnapshotManager::moveSnapshotsIfNeeded()
+template<typename Storage>
+void KeeperSnapshotManager<Storage>::moveSnapshotsIfNeeded()
 {
     /// move snapshots to correct disks
 
@@ -767,7 +785,8 @@ void KeeperSnapshotManager::moveSnapshotsIfNeeded()
 
 }
 
-void KeeperSnapshotManager::removeSnapshot(uint64_t log_idx)
+template<typename Storage>
+void KeeperSnapshotManager<Storage>::removeSnapshot(uint64_t log_idx)
 {
     auto itr = existing_snapshots.find(log_idx);
     if (itr == existing_snapshots.end())
@@ -777,7 +796,8 @@ void KeeperSnapshotManager::removeSnapshot(uint64_t log_idx)
     existing_snapshots.erase(itr);
 }
 
-SnapshotFileInfo KeeperSnapshotManager::serializeSnapshotToDisk(const KeeperStorageSnapshot & snapshot)
+template<typename Storage>
+SnapshotFileInfo KeeperSnapshotManager<Storage>::serializeSnapshotToDisk(const KeeperStorageSnapshot<Storage> & snapshot)
 {
     auto up_to_log_idx = snapshot.snapshot_meta->get_last_log_idx();
     auto snapshot_file_name = getSnapshotFileName(up_to_log_idx, compress_snapshots_zstd);
@@ -796,7 +816,7 @@ SnapshotFileInfo KeeperSnapshotManager::serializeSnapshotToDisk(const KeeperStor
     else
         compressed_writer = std::make_unique<CompressedWriteBuffer>(*writer);
 
-    KeeperStorageSnapshot::serialize(snapshot, *compressed_writer, keeper_context);
+    KeeperStorageSnapshot<Storage>::serialize(snapshot, *compressed_writer, keeper_context);
     compressed_writer->finalize();
     compressed_writer->sync();
 
@@ -809,4 +829,7 @@ SnapshotFileInfo KeeperSnapshotManager::serializeSnapshotToDisk(const KeeperStor
     return {snapshot_file_name, disk};
 }
 
+template struct KeeperStorageSnapshot<KeeperMemoryStorage>;
+template class KeeperSnapshotManager<KeeperMemoryStorage>;
+
 }
diff --git a/src/Coordination/KeeperSnapshotManager.h b/src/Coordination/KeeperSnapshotManager.h
index 9bb287b9276..b0a9425881f 100644
--- a/src/Coordination/KeeperSnapshotManager.h
+++ b/src/Coordination/KeeperSnapshotManager.h
@@ -29,16 +29,19 @@ enum SnapshotVersion : uint8_t
 static constexpr auto CURRENT_SNAPSHOT_VERSION = SnapshotVersion::V5;
 
 /// What is stored in binary snapshot
+template<typename Storage>
 struct SnapshotDeserializationResult
 {
     /// Storage
-    KeeperStoragePtr storage;
+    std::unique_ptr<Storage> storage;
     /// Snapshot metadata (up_to_log_idx and so on)
     SnapshotMetadataPtr snapshot_meta;
     /// Cluster config
     ClusterConfigPtr cluster_config;
 };
 
+struct IKeeperStorageSnapshot {};
+
 /// In memory keeper snapshot. Keeper Storage based on a hash map which can be
 /// turned into snapshot mode. This operation is fast and KeeperStorageSnapshot
 /// class do it in constructor. It also copies iterators from storage hash table
@@ -47,21 +50,22 @@ struct SnapshotDeserializationResult
 ///
 /// This representation of snapshot have to be serialized into NuRaft
 /// buffer and send over network or saved to file.
-struct KeeperStorageSnapshot
+template<typename Storage>
+struct KeeperStorageSnapshot : IKeeperStorageSnapshot
 {
 public:
-    KeeperStorageSnapshot(KeeperStorage * storage_, uint64_t up_to_log_idx_, const ClusterConfigPtr & cluster_config_ = nullptr);
+    KeeperStorageSnapshot(Storage * storage_, uint64_t up_to_log_idx_, const ClusterConfigPtr & cluster_config_ = nullptr);
 
     KeeperStorageSnapshot(
-        KeeperStorage * storage_, const SnapshotMetadataPtr & snapshot_meta_, const ClusterConfigPtr & cluster_config_ = nullptr);
+        Storage * storage_, const SnapshotMetadataPtr & snapshot_meta_, const ClusterConfigPtr & cluster_config_ = nullptr);
 
     ~KeeperStorageSnapshot();
 
-    static void serialize(const KeeperStorageSnapshot & snapshot, WriteBuffer & out, KeeperContextPtr keeper_context);
+    static void serialize(const KeeperStorageSnapshot<Storage> & snapshot, WriteBuffer & out, KeeperContextPtr keeper_context);
 
-    static void deserialize(SnapshotDeserializationResult & deserialization_result, ReadBuffer & in, KeeperContextPtr keeper_context);
+    static void deserialize(SnapshotDeserializationResult<Storage> & deserialization_result, ReadBuffer & in, KeeperContextPtr keeper_context);
 
-    KeeperStorage * storage;
+    Storage * storage;
 
     SnapshotVersion version = CURRENT_SNAPSHOT_VERSION;
     /// Snapshot metadata
@@ -72,11 +76,11 @@ public:
     /// so we have for loop for (i = 0; i < snapshot_container_size; ++i) { doSmth(begin + i); }
     size_t snapshot_container_size;
     /// Iterator to the start of the storage
-    KeeperStorage::Container::const_iterator begin;
+    Storage::Container::const_iterator begin;
     /// Active sessions and their timeouts
     SessionAndTimeout session_and_timeout;
     /// Sessions credentials
-    KeeperStorage::SessionAndAuth session_and_auth;
+    Storage::SessionAndAuth session_and_auth;
     /// ACLs cache for better performance. Without we cannot deserialize storage.
     std::unordered_map<uint64_t, Coordination::ACLs> acl_map;
     /// Cluster config from snapshot, can be empty
@@ -93,14 +97,12 @@ struct SnapshotFileInfo
     DiskPtr disk;
 };
 
-using KeeperStorageSnapshotPtr = std::shared_ptr<KeeperStorageSnapshot>;
+using KeeperStorageSnapshotPtr = std::shared_ptr<IKeeperStorageSnapshot>;
 using CreateSnapshotCallback = std::function<SnapshotFileInfo(KeeperStorageSnapshotPtr &&)>;
 
-
-using SnapshotMetaAndStorage = std::pair<SnapshotMetadataPtr, KeeperStoragePtr>;
-
 /// Class responsible for snapshots serialization and deserialization. Each snapshot
 /// has it's path on disk and log index.
+template<typename Storage>
 class KeeperSnapshotManager
 {
 public:
@@ -112,18 +114,18 @@ public:
         size_t storage_tick_time_ = 500);
 
     /// Restore storage from latest available snapshot
-    SnapshotDeserializationResult restoreFromLatestSnapshot();
+    SnapshotDeserializationResult<Storage> restoreFromLatestSnapshot();
 
     /// Compress snapshot and serialize it to buffer
-    nuraft::ptr<nuraft::buffer> serializeSnapshotToBuffer(const KeeperStorageSnapshot & snapshot) const;
+    nuraft::ptr<nuraft::buffer> serializeSnapshotToBuffer(const KeeperStorageSnapshot<Storage> & snapshot) const;
 
     /// Serialize already compressed snapshot to disk (return path)
     SnapshotFileInfo serializeSnapshotBufferToDisk(nuraft::buffer & buffer, uint64_t up_to_log_idx);
 
     /// Serialize snapshot directly to disk
-    SnapshotFileInfo serializeSnapshotToDisk(const KeeperStorageSnapshot & snapshot);
+    SnapshotFileInfo serializeSnapshotToDisk(const KeeperStorageSnapshot<Storage> & snapshot);
 
-    SnapshotDeserializationResult deserializeSnapshotFromBuffer(nuraft::ptr<nuraft::buffer> buffer) const;
+    SnapshotDeserializationResult<Storage> deserializeSnapshotFromBuffer(nuraft::ptr<nuraft::buffer> buffer) const;
 
     /// Deserialize snapshot with log index up_to_log_idx from disk into compressed nuraft buffer.
     nuraft::ptr<nuraft::buffer> deserializeSnapshotBufferFromDisk(uint64_t up_to_log_idx) const;
diff --git a/src/Coordination/KeeperStateMachine.cpp b/src/Coordination/KeeperStateMachine.cpp
index b6185fa2e36..a3c9378a173 100644
--- a/src/Coordination/KeeperStateMachine.cpp
+++ b/src/Coordination/KeeperStateMachine.cpp
@@ -40,7 +40,7 @@ namespace ErrorCodes
     extern const int LOGICAL_ERROR;
 }
 
-KeeperStateMachine::KeeperStateMachine(
+IKeeperStateMachine::IKeeperStateMachine(
     ResponsesQueue & responses_queue_,
     SnapshotsQueue & snapshots_queue_,
     const CoordinationSettingsPtr & coordination_settings_,
@@ -50,12 +50,6 @@ KeeperStateMachine::KeeperStateMachine(
     const std::string & superdigest_)
     : commit_callback(commit_callback_)
     , coordination_settings(coordination_settings_)
-    , snapshot_manager(
-          coordination_settings->snapshots_to_keep,
-          keeper_context_,
-          coordination_settings->compress_snapshots_with_zstd_format,
-          superdigest_,
-          coordination_settings->dead_session_check_period_ms.totalMilliseconds())
     , responses_queue(responses_queue_)
     , snapshots_queue(snapshots_queue_)
     , min_request_size_to_cache(coordination_settings_->min_request_size_for_cache)
@@ -67,6 +61,32 @@ KeeperStateMachine::KeeperStateMachine(
 {
 }
 
+template<typename Storage>
+KeeperStateMachine<Storage>::KeeperStateMachine(
+    ResponsesQueue & responses_queue_,
+    SnapshotsQueue & snapshots_queue_,
+    const CoordinationSettingsPtr & coordination_settings_,
+    const KeeperContextPtr & keeper_context_,
+    KeeperSnapshotManagerS3 * snapshot_manager_s3_,
+    IKeeperStateMachine::CommitCallback commit_callback_,
+    const std::string & superdigest_)
+    : IKeeperStateMachine(
+        responses_queue_,
+        snapshots_queue_,
+        coordination_settings_,
+        keeper_context_,
+        snapshot_manager_s3_,
+        commit_callback_,
+        superdigest_),
+        snapshot_manager(
+          coordination_settings->snapshots_to_keep,
+          keeper_context_,
+          coordination_settings->compress_snapshots_with_zstd_format,
+          superdigest_,
+          coordination_settings->dead_session_check_period_ms.totalMilliseconds())
+{
+}
+
 namespace
 {
 
@@ -77,7 +97,8 @@ bool isLocalDisk(const IDisk & disk)
 
 }
 
-void KeeperStateMachine::init()
+template<typename Storage>
+void KeeperStateMachine<Storage>::init()
 {
     /// Do everything without mutexes, no other threads exist.
     LOG_DEBUG(log, "Totally have {} snapshots", snapshot_manager.totalSnapshots());
@@ -129,7 +150,7 @@ void KeeperStateMachine::init()
     }
 
     if (!storage)
-        storage = std::make_unique<KeeperStorage>(
+        storage = std::make_unique<Storage>(
             coordination_settings->dead_session_check_period_ms.totalMilliseconds(), superdigest, keeper_context);
 }
 
@@ -137,12 +158,12 @@ namespace
 {
 
 void assertDigest(
-    const KeeperStorage::Digest & first,
-    const KeeperStorage::Digest & second,
+    const KeeperStorageBase::Digest & first,
+    const KeeperStorageBase::Digest & second,
     const Coordination::ZooKeeperRequest & request,
     bool committing)
 {
-    if (!KeeperStorage::checkDigest(first, second))
+    if (!KeeperStorageBase::checkDigest(first, second))
     {
         LOG_FATAL(
             &Poco::Logger::get("KeeperStateMachine"),
@@ -160,7 +181,8 @@ void assertDigest(
 
 }
 
-nuraft::ptr<nuraft::buffer> KeeperStateMachine::pre_commit(uint64_t log_idx, nuraft::buffer & data)
+template<typename Storage>
+nuraft::ptr<nuraft::buffer> KeeperStateMachine<Storage>::pre_commit(uint64_t log_idx, nuraft::buffer & data)
 {
     auto request_for_session = parseRequest(data, /*final=*/false);
     if (!request_for_session->zxid)
@@ -175,10 +197,10 @@ nuraft::ptr<nuraft::buffer> KeeperStateMachine::pre_commit(uint64_t log_idx, nur
     return result;
 }
 
-std::shared_ptr<KeeperStorage::RequestForSession> KeeperStateMachine::parseRequest(nuraft::buffer & data, bool final, ZooKeeperLogSerializationVersion * serialization_version)
+std::shared_ptr<KeeperStorageBase::RequestForSession> IKeeperStateMachine::parseRequest(nuraft::buffer & data, bool final, ZooKeeperLogSerializationVersion * serialization_version)
 {
     ReadBufferFromNuraftBuffer buffer(data);
-    auto request_for_session = std::make_shared<KeeperStorage::RequestForSession>();
+    auto request_for_session = std::make_shared<KeeperStorageBase::RequestForSession>();
     readIntBinary(request_for_session->session_id, buffer);
 
     int32_t length;
@@ -251,7 +273,7 @@ std::shared_ptr<KeeperStorage::RequestForSession> KeeperStateMachine::parseReque
 
         request_for_session->digest.emplace();
         readIntBinary(request_for_session->digest->version, buffer);
-        if (request_for_session->digest->version != KeeperStorage::DigestVersion::NO_DIGEST || !buffer.eof())
+        if (request_for_session->digest->version != KeeperStorageBase::DigestVersion::NO_DIGEST || !buffer.eof())
             readIntBinary(request_for_session->digest->value, buffer);
     }
 
@@ -267,7 +289,8 @@ std::shared_ptr<KeeperStorage::RequestForSession> KeeperStateMachine::parseReque
     return request_for_session;
 }
 
-bool KeeperStateMachine::preprocess(const KeeperStorage::RequestForSession & request_for_session)
+template<typename Storage>
+bool KeeperStateMachine<Storage>::preprocess(const KeeperStorageBase::RequestForSession & request_for_session)
 {
     const auto op_num = request_for_session.request->getOpNum();
     if (op_num == Coordination::OpNum::SessionID || op_num == Coordination::OpNum::Reconfig)
@@ -301,10 +324,11 @@ bool KeeperStateMachine::preprocess(const KeeperStorage::RequestForSession & req
     return true;
 }
 
-void KeeperStateMachine::reconfigure(const KeeperStorage::RequestForSession& request_for_session)
+template<typename Storage>
+void KeeperStateMachine<Storage>::reconfigure(const KeeperStorageBase::RequestForSession& request_for_session)
 {
     std::lock_guard _(storage_and_responses_lock);
-    KeeperStorage::ResponseForSession response = processReconfiguration(request_for_session);
+    KeeperStorageBase::ResponseForSession response = processReconfiguration(request_for_session);
     if (!responses_queue.push(response))
     {
         ProfileEvents::increment(ProfileEvents::KeeperCommitsFailed);
@@ -314,8 +338,9 @@ void KeeperStateMachine::reconfigure(const KeeperStorage::RequestForSession& req
     }
 }
 
-KeeperStorage::ResponseForSession KeeperStateMachine::processReconfiguration(
-    const KeeperStorage::RequestForSession & request_for_session)
+template<typename Storage>
+KeeperStorageBase::ResponseForSession KeeperStateMachine<Storage>::processReconfiguration(
+    const KeeperStorageBase::RequestForSession & request_for_session)
 {
     ProfileEvents::increment(ProfileEvents::KeeperReconfigRequest);
 
@@ -324,7 +349,7 @@ KeeperStorage::ResponseForSession KeeperStateMachine::processReconfiguration(
     const int64_t zxid = request_for_session.zxid;
 
     using enum Coordination::Error;
-    auto bad_request = [&](Coordination::Error code = ZBADARGUMENTS) -> KeeperStorage::ResponseForSession
+    auto bad_request = [&](Coordination::Error code = ZBADARGUMENTS) -> KeeperStorageBase::ResponseForSession
     {
         auto res = std::make_shared<Coordination::ZooKeeperReconfigResponse>();
         res->xid = request.xid;
@@ -381,7 +406,8 @@ KeeperStorage::ResponseForSession KeeperStateMachine::processReconfiguration(
     return { session_id, std::move(response) };
 }
 
-nuraft::ptr<nuraft::buffer> KeeperStateMachine::commit(const uint64_t log_idx, nuraft::buffer & data)
+template<typename Storage>
+nuraft::ptr<nuraft::buffer> KeeperStateMachine<Storage>::commit(const uint64_t log_idx, nuraft::buffer & data)
 {
     auto request_for_session = parseRequest(data, true);
     if (!request_for_session->zxid)
@@ -392,7 +418,7 @@ nuraft::ptr<nuraft::buffer> KeeperStateMachine::commit(const uint64_t log_idx, n
     if (!keeper_context->local_logs_preprocessed)
         preprocess(*request_for_session);
 
-    auto try_push = [this](const KeeperStorage::ResponseForSession& response)
+    auto try_push = [this](const KeeperStorageBase::ResponseForSession& response)
     {
         if (!responses_queue.push(response))
         {
@@ -412,7 +438,7 @@ nuraft::ptr<nuraft::buffer> KeeperStateMachine::commit(const uint64_t log_idx, n
         std::shared_ptr<Coordination::ZooKeeperSessionIDResponse> response = std::make_shared<Coordination::ZooKeeperSessionIDResponse>();
         response->internal_id = session_id_request.internal_id;
         response->server_id = session_id_request.server_id;
-        KeeperStorage::ResponseForSession response_for_session;
+        KeeperStorageBase::ResponseForSession response_for_session;
         response_for_session.session_id = -1;
         response_for_session.response = response;
 
@@ -431,7 +457,7 @@ nuraft::ptr<nuraft::buffer> KeeperStateMachine::commit(const uint64_t log_idx, n
         }
 
         std::lock_guard lock(storage_and_responses_lock);
-        KeeperStorage::ResponsesForSessions responses_for_sessions
+        KeeperStorageBase::ResponsesForSessions responses_for_sessions
             = storage->processRequest(request_for_session->request, request_for_session->session_id, request_for_session->zxid);
         for (auto & response_for_session : responses_for_sessions)
             try_push(response_for_session);
@@ -448,7 +474,8 @@ nuraft::ptr<nuraft::buffer> KeeperStateMachine::commit(const uint64_t log_idx, n
     return nullptr;
 }
 
-bool KeeperStateMachine::apply_snapshot(nuraft::snapshot & s)
+template<typename Storage>
+bool KeeperStateMachine<Storage>::apply_snapshot(nuraft::snapshot & s)
 {
     LOG_DEBUG(log, "Applying snapshot {}", s.get_last_log_idx());
     nuraft::ptr<nuraft::buffer> latest_snapshot_ptr;
@@ -475,7 +502,7 @@ bool KeeperStateMachine::apply_snapshot(nuraft::snapshot & s)
     { /// deserialize and apply snapshot to storage
         std::lock_guard lock(storage_and_responses_lock);
 
-        SnapshotDeserializationResult snapshot_deserialization_result;
+        SnapshotDeserializationResult<Storage> snapshot_deserialization_result;
         if (latest_snapshot_ptr)
             snapshot_deserialization_result = snapshot_manager.deserializeSnapshotFromBuffer(latest_snapshot_ptr);
         else
@@ -496,7 +523,7 @@ bool KeeperStateMachine::apply_snapshot(nuraft::snapshot & s)
 }
 
 
-void KeeperStateMachine::commit_config(const uint64_t log_idx, nuraft::ptr<nuraft::cluster_config> & new_conf)
+void IKeeperStateMachine::commit_config(const uint64_t log_idx, nuraft::ptr<nuraft::cluster_config> & new_conf)
 {
     std::lock_guard lock(cluster_config_lock);
     auto tmp = new_conf->serialize();
@@ -504,7 +531,7 @@ void KeeperStateMachine::commit_config(const uint64_t log_idx, nuraft::ptr<nuraf
     last_committed_idx = log_idx;
 }
 
-void KeeperStateMachine::rollback(uint64_t log_idx, nuraft::buffer & data)
+void IKeeperStateMachine::rollback(uint64_t log_idx, nuraft::buffer & data)
 {
     auto request_for_session = parseRequest(data, true);
     // If we received a log from an older node, use the log_idx as the zxid
@@ -516,7 +543,8 @@ void KeeperStateMachine::rollback(uint64_t log_idx, nuraft::buffer & data)
     rollbackRequest(*request_for_session, false);
 }
 
-void KeeperStateMachine::rollbackRequest(const KeeperStorage::RequestForSession & request_for_session, bool allow_missing)
+template<typename Storage>
+void KeeperStateMachine<Storage>::rollbackRequest(const KeeperStorageBase::RequestForSession & request_for_session, bool allow_missing)
 {
     if (request_for_session.request->getOpNum() == Coordination::OpNum::SessionID)
         return;
@@ -525,7 +553,8 @@ void KeeperStateMachine::rollbackRequest(const KeeperStorage::RequestForSession
     storage->rollbackRequest(request_for_session.zxid, allow_missing);
 }
 
-void KeeperStateMachine::rollbackRequestNoLock(const KeeperStorage::RequestForSession & request_for_session, bool allow_missing)
+template<typename Storage>
+void KeeperStateMachine<Storage>::rollbackRequestNoLock(const KeeperStorageBase::RequestForSession & request_for_session, bool allow_missing)
 {
     if (request_for_session.request->getOpNum() == Coordination::OpNum::SessionID)
         return;
@@ -533,14 +562,15 @@ void KeeperStateMachine::rollbackRequestNoLock(const KeeperStorage::RequestForSe
     storage->rollbackRequest(request_for_session.zxid, allow_missing);
 }
 
-nuraft::ptr<nuraft::snapshot> KeeperStateMachine::last_snapshot()
+nuraft::ptr<nuraft::snapshot> IKeeperStateMachine::last_snapshot()
 {
     /// Just return the latest snapshot.
     std::lock_guard lock(snapshots_lock);
     return latest_snapshot_meta;
 }
 
-void KeeperStateMachine::create_snapshot(nuraft::snapshot & s, nuraft::async_result<bool>::handler_type & when_done)
+template<typename Storage>
+void KeeperStateMachine<Storage>::create_snapshot(nuraft::snapshot & s, nuraft::async_result<bool>::handler_type & when_done)
 {
     LOG_DEBUG(log, "Creating snapshot {}", s.get_last_log_idx());
 
@@ -549,12 +579,13 @@ void KeeperStateMachine::create_snapshot(nuraft::snapshot & s, nuraft::async_res
     CreateSnapshotTask snapshot_task;
     { /// lock storage for a short period time to turn on "snapshot mode". After that we can read consistent storage state without locking.
         std::lock_guard lock(storage_and_responses_lock);
-        snapshot_task.snapshot = std::make_shared<KeeperStorageSnapshot>(storage.get(), snapshot_meta_copy, getClusterConfig());
+        snapshot_task.snapshot = std::make_shared<KeeperStorageSnapshot<Storage>>(storage.get(), snapshot_meta_copy, getClusterConfig());
     }
 
     /// create snapshot task for background execution (in snapshot thread)
-    snapshot_task.create_snapshot = [this, when_done](KeeperStorageSnapshotPtr && snapshot)
+    snapshot_task.create_snapshot = [this, when_done](KeeperStorageSnapshotPtr && snapshot_)
     {
+        auto * snapshot = typeid_cast<KeeperStorageSnapshot<Storage> *>(snapshot_.get());
         nuraft::ptr<std::exception> exception(nullptr);
         bool ret = true;
         try
@@ -601,7 +632,7 @@ void KeeperStateMachine::create_snapshot(nuraft::snapshot & s, nuraft::async_res
                 /// Turn off "snapshot mode" and clear outdate part of storage state
                 storage->clearGarbageAfterSnapshot();
                 LOG_TRACE(log, "Cleared garbage after snapshot");
-                snapshot.reset();
+                snapshot_.reset();
             }
         }
         catch (...)
@@ -638,7 +669,8 @@ void KeeperStateMachine::create_snapshot(nuraft::snapshot & s, nuraft::async_res
         LOG_WARNING(log, "Cannot push snapshot task into queue");
 }
 
-void KeeperStateMachine::save_logical_snp_obj(
+template<typename Storage>
+void KeeperStateMachine<Storage>::save_logical_snp_obj(
     nuraft::snapshot & s, uint64_t & obj_id, nuraft::buffer & data, bool /*is_first_obj*/, bool /*is_last_obj*/)
 {
     LOG_DEBUG(log, "Saving snapshot {} obj_id {}", s.get_last_log_idx(), obj_id);
@@ -703,7 +735,7 @@ static int bufferFromFile(Poco::Logger * log, const std::string & path, nuraft::
     return 0;
 }
 
-int KeeperStateMachine::read_logical_snp_obj(
+int IKeeperStateMachine::read_logical_snp_obj(
     nuraft::snapshot & s, void *& /*user_snp_ctx*/, uint64_t obj_id, nuraft::ptr<nuraft::buffer> & data_out, bool & is_last_obj)
 {
     LOG_DEBUG(log, "Reading snapshot {} obj_id {}", s.get_last_log_idx(), obj_id);
@@ -743,7 +775,8 @@ int KeeperStateMachine::read_logical_snp_obj(
     return 1;
 }
 
-void KeeperStateMachine::processReadRequest(const KeeperStorage::RequestForSession & request_for_session)
+template<typename Storage>
+void KeeperStateMachine<Storage>::processReadRequest(const KeeperStorageBase::RequestForSession & request_for_session)
 {
     /// Pure local request, just process it with storage
     std::lock_guard lock(storage_and_responses_lock);
@@ -754,103 +787,120 @@ void KeeperStateMachine::processReadRequest(const KeeperStorage::RequestForSessi
             LOG_WARNING(log, "Failed to push response with session id {} to the queue, probably because of shutdown", response.session_id);
 }
 
-void KeeperStateMachine::shutdownStorage()
+template<typename Storage>
+void KeeperStateMachine<Storage>::shutdownStorage()
 {
     std::lock_guard lock(storage_and_responses_lock);
     storage->finalize();
 }
 
-std::vector<int64_t> KeeperStateMachine::getDeadSessions()
+template<typename Storage>
+std::vector<int64_t> KeeperStateMachine<Storage>::getDeadSessions()
 {
     std::lock_guard lock(storage_and_responses_lock);
     return storage->getDeadSessions();
 }
 
-int64_t KeeperStateMachine::getNextZxid() const
+template<typename Storage>
+int64_t KeeperStateMachine<Storage>::getNextZxid() const
 {
     std::lock_guard lock(storage_and_responses_lock);
     return storage->getNextZXID();
 }
 
-KeeperStorage::Digest KeeperStateMachine::getNodesDigest() const
+template<typename Storage>
+KeeperStorageBase::Digest KeeperStateMachine<Storage>::getNodesDigest() const
 {
     std::lock_guard lock(storage_and_responses_lock);
     return storage->getNodesDigest(false);
 }
 
-uint64_t KeeperStateMachine::getLastProcessedZxid() const
+template<typename Storage>
+uint64_t KeeperStateMachine<Storage>::getLastProcessedZxid() const
 {
     std::lock_guard lock(storage_and_responses_lock);
     return storage->getZXID();
 }
 
-uint64_t KeeperStateMachine::getNodesCount() const
+template<typename Storage>
+uint64_t KeeperStateMachine<Storage>::getNodesCount() const
 {
     std::lock_guard lock(storage_and_responses_lock);
     return storage->getNodesCount();
 }
 
-uint64_t KeeperStateMachine::getTotalWatchesCount() const
+template<typename Storage>
+uint64_t KeeperStateMachine<Storage>::getTotalWatchesCount() const
 {
     std::lock_guard lock(storage_and_responses_lock);
     return storage->getTotalWatchesCount();
 }
 
-uint64_t KeeperStateMachine::getWatchedPathsCount() const
+template<typename Storage>
+uint64_t KeeperStateMachine<Storage>::getWatchedPathsCount() const
 {
     std::lock_guard lock(storage_and_responses_lock);
     return storage->getWatchedPathsCount();
 }
 
-uint64_t KeeperStateMachine::getSessionsWithWatchesCount() const
+template<typename Storage>
+uint64_t KeeperStateMachine<Storage>::getSessionsWithWatchesCount() const
 {
     std::lock_guard lock(storage_and_responses_lock);
     return storage->getSessionsWithWatchesCount();
 }
 
-uint64_t KeeperStateMachine::getTotalEphemeralNodesCount() const
+template<typename Storage>
+uint64_t KeeperStateMachine<Storage>::getTotalEphemeralNodesCount() const
 {
     std::lock_guard lock(storage_and_responses_lock);
     return storage->getTotalEphemeralNodesCount();
 }
 
-uint64_t KeeperStateMachine::getSessionWithEphemeralNodesCount() const
+template<typename Storage>
+uint64_t KeeperStateMachine<Storage>::getSessionWithEphemeralNodesCount() const
 {
     std::lock_guard lock(storage_and_responses_lock);
     return storage->getSessionWithEphemeralNodesCount();
 }
 
-void KeeperStateMachine::dumpWatches(WriteBufferFromOwnString & buf) const
+template<typename Storage>
+void KeeperStateMachine<Storage>::dumpWatches(WriteBufferFromOwnString & buf) const
 {
     std::lock_guard lock(storage_and_responses_lock);
     storage->dumpWatches(buf);
 }
 
-void KeeperStateMachine::dumpWatchesByPath(WriteBufferFromOwnString & buf) const
+template<typename Storage>
+void KeeperStateMachine<Storage>::dumpWatchesByPath(WriteBufferFromOwnString & buf) const
 {
     std::lock_guard lock(storage_and_responses_lock);
     storage->dumpWatchesByPath(buf);
 }
 
-void KeeperStateMachine::dumpSessionsAndEphemerals(WriteBufferFromOwnString & buf) const
+template<typename Storage>
+void KeeperStateMachine<Storage>::dumpSessionsAndEphemerals(WriteBufferFromOwnString & buf) const
 {
     std::lock_guard lock(storage_and_responses_lock);
     storage->dumpSessionsAndEphemerals(buf);
 }
 
-uint64_t KeeperStateMachine::getApproximateDataSize() const
+template<typename Storage>
+uint64_t KeeperStateMachine<Storage>::getApproximateDataSize() const
 {
     std::lock_guard lock(storage_and_responses_lock);
     return storage->getApproximateDataSize();
 }
 
-uint64_t KeeperStateMachine::getKeyArenaSize() const
+template<typename Storage>
+uint64_t KeeperStateMachine<Storage>::getKeyArenaSize() const
 {
     std::lock_guard lock(storage_and_responses_lock);
     return storage->getArenaDataSize();
 }
 
-uint64_t KeeperStateMachine::getLatestSnapshotBufSize() const
+template<typename Storage>
+uint64_t KeeperStateMachine<Storage>::getLatestSnapshotBufSize() const
 {
     std::lock_guard lock(snapshots_lock);
     if (latest_snapshot_buf)
@@ -858,7 +908,7 @@ uint64_t KeeperStateMachine::getLatestSnapshotBufSize() const
     return 0;
 }
 
-ClusterConfigPtr KeeperStateMachine::getClusterConfig() const
+ClusterConfigPtr IKeeperStateMachine::getClusterConfig() const
 {
     std::lock_guard lock(cluster_config_lock);
     if (cluster_config)
@@ -870,11 +920,15 @@ ClusterConfigPtr KeeperStateMachine::getClusterConfig() const
     return nullptr;
 }
 
-void KeeperStateMachine::recalculateStorageStats()
+template<typename Storage>
+void KeeperStateMachine<Storage>::recalculateStorageStats()
 {
     std::lock_guard lock(storage_and_responses_lock);
     LOG_INFO(log, "Recalculating storage stats");
     storage->recalculateStats();
     LOG_INFO(log, "Done recalculating storage stats");
 }
+
+template class KeeperStateMachine<KeeperMemoryStorage>;
+
 }
diff --git a/src/Coordination/KeeperStateMachine.h b/src/Coordination/KeeperStateMachine.h
index aad5d3aafd4..9ae3905cddd 100644
--- a/src/Coordination/KeeperStateMachine.h
+++ b/src/Coordination/KeeperStateMachine.h
@@ -12,27 +12,25 @@
 
 namespace DB
 {
-using ResponsesQueue = ConcurrentBoundedQueue<KeeperStorage::ResponseForSession>;
+using ResponsesQueue = ConcurrentBoundedQueue<KeeperStorageBase::ResponseForSession>;
 using SnapshotsQueue = ConcurrentBoundedQueue<CreateSnapshotTask>;
 
-/// ClickHouse Keeper state machine. Wrapper for KeeperStorage.
-/// Responsible for entries commit, snapshots creation and so on.
-class KeeperStateMachine : public nuraft::state_machine
+class IKeeperStateMachine : public nuraft::state_machine
 {
 public:
-    using CommitCallback = std::function<void(uint64_t, const KeeperStorage::RequestForSession &)>;
+    using CommitCallback = std::function<void(uint64_t, const KeeperStorageBase::RequestForSession &)>;
 
-    KeeperStateMachine(
+    IKeeperStateMachine(
         ResponsesQueue & responses_queue_,
         SnapshotsQueue & snapshots_queue_,
         const CoordinationSettingsPtr & coordination_settings_,
         const KeeperContextPtr & keeper_context_,
         KeeperSnapshotManagerS3 * snapshot_manager_s3_,
-        CommitCallback commit_callback_ = {},
-        const std::string & superdigest_ = "");
+        CommitCallback commit_callback_,
+        const std::string & superdisgest_);
 
     /// Read state from the latest snapshot
-    void init();
+    virtual void init() = 0;
 
     enum ZooKeeperLogSerializationVersion
     {
@@ -49,89 +47,66 @@ public:
     ///
     /// final - whether it's the final time we will fetch the request so we can safely remove it from cache
     /// serialization_version - information about which fields were parsed from the buffer so we can modify the buffer accordingly
-    std::shared_ptr<KeeperStorage::RequestForSession> parseRequest(nuraft::buffer & data, bool final, ZooKeeperLogSerializationVersion * serialization_version = nullptr);
+    std::shared_ptr<KeeperStorageBase::RequestForSession> parseRequest(nuraft::buffer & data, bool final, ZooKeeperLogSerializationVersion * serialization_version = nullptr);
 
-    bool preprocess(const KeeperStorage::RequestForSession & request_for_session);
+    virtual bool preprocess(const KeeperStorageBase::RequestForSession & request_for_session) = 0;
 
-    nuraft::ptr<nuraft::buffer> pre_commit(uint64_t log_idx, nuraft::buffer & data) override;
-
-    nuraft::ptr<nuraft::buffer> commit(const uint64_t log_idx, nuraft::buffer & data) override; /// NOLINT
-
-    /// Save new cluster config to our snapshot (copy of the config stored in StateManager)
     void commit_config(const uint64_t log_idx, nuraft::ptr<nuraft::cluster_config> & new_conf) override; /// NOLINT
 
     void rollback(uint64_t log_idx, nuraft::buffer & data) override;
 
     // allow_missing - whether the transaction we want to rollback can be missing from storage
     // (can happen in case of exception during preprocessing)
-    void rollbackRequest(const KeeperStorage::RequestForSession & request_for_session, bool allow_missing);
-
-    void rollbackRequestNoLock(
-        const KeeperStorage::RequestForSession & request_for_session,
-        bool allow_missing) TSA_NO_THREAD_SAFETY_ANALYSIS;
+    virtual void rollbackRequest(const KeeperStorageBase::RequestForSession & request_for_session, bool allow_missing) = 0;
 
     uint64_t last_commit_index() override { return last_committed_idx; }
 
-    /// Apply preliminarily saved (save_logical_snp_obj) snapshot to our state.
-    bool apply_snapshot(nuraft::snapshot & s) override;
-
     nuraft::ptr<nuraft::snapshot> last_snapshot() override;
 
     /// Create new snapshot from current state.
-    void create_snapshot(nuraft::snapshot & s, nuraft::async_result<bool>::handler_type & when_done) override;
+    void create_snapshot(nuraft::snapshot & s, nuraft::async_result<bool>::handler_type & when_done) override = 0;
 
     /// Save snapshot which was send by leader to us. After that we will apply it in apply_snapshot.
-    void save_logical_snp_obj(nuraft::snapshot & s, uint64_t & obj_id, nuraft::buffer & data, bool is_first_obj, bool is_last_obj) override;
+    void save_logical_snp_obj(nuraft::snapshot & s, uint64_t & obj_id, nuraft::buffer & data, bool is_first_obj, bool is_last_obj) override = 0;
 
-    /// Better name is `serialize snapshot` -- save existing snapshot (created by create_snapshot) into
-    /// in-memory buffer data_out.
     int read_logical_snp_obj(
         nuraft::snapshot & s, void *& user_snp_ctx, uint64_t obj_id, nuraft::ptr<nuraft::buffer> & data_out, bool & is_last_obj) override;
 
-    // This should be used only for tests or keeper-data-dumper because it violates
-    // TSA -- we can't acquire the lock outside of this class or return a storage under lock
-    // in a reasonable way.
-    KeeperStorage & getStorageUnsafe() TSA_NO_THREAD_SAFETY_ANALYSIS
-    {
-        return *storage;
-    }
-
-    void shutdownStorage();
+    virtual void shutdownStorage() = 0;
 
     ClusterConfigPtr getClusterConfig() const;
 
-    /// Process local read request
-    void processReadRequest(const KeeperStorage::RequestForSession & request_for_session);
+    virtual void processReadRequest(const KeeperStorageBase::RequestForSession & request_for_session) = 0;
 
-    std::vector<int64_t> getDeadSessions();
+    virtual std::vector<int64_t> getDeadSessions() = 0;
 
-    int64_t getNextZxid() const;
+    virtual int64_t getNextZxid() const = 0;
 
-    KeeperStorage::Digest getNodesDigest() const;
+    virtual KeeperStorageBase::Digest getNodesDigest() const = 0;
 
     /// Introspection functions for 4lw commands
-    uint64_t getLastProcessedZxid() const;
+    virtual uint64_t getLastProcessedZxid() const = 0;
 
-    uint64_t getNodesCount() const;
-    uint64_t getTotalWatchesCount() const;
-    uint64_t getWatchedPathsCount() const;
-    uint64_t getSessionsWithWatchesCount() const;
+    virtual uint64_t getNodesCount() const = 0;
+    virtual uint64_t getTotalWatchesCount() const = 0;
+    virtual uint64_t getWatchedPathsCount() const = 0;
+    virtual uint64_t getSessionsWithWatchesCount() const = 0;
 
-    void dumpWatches(WriteBufferFromOwnString & buf) const;
-    void dumpWatchesByPath(WriteBufferFromOwnString & buf) const;
-    void dumpSessionsAndEphemerals(WriteBufferFromOwnString & buf) const;
+    virtual void dumpWatches(WriteBufferFromOwnString & buf) const = 0;
+    virtual void dumpWatchesByPath(WriteBufferFromOwnString & buf) const = 0;
+    virtual void dumpSessionsAndEphemerals(WriteBufferFromOwnString & buf) const = 0;
 
-    uint64_t getSessionWithEphemeralNodesCount() const;
-    uint64_t getTotalEphemeralNodesCount() const;
-    uint64_t getApproximateDataSize() const;
-    uint64_t getKeyArenaSize() const;
-    uint64_t getLatestSnapshotBufSize() const;
+    virtual uint64_t getSessionWithEphemeralNodesCount() const = 0;
+    virtual uint64_t getTotalEphemeralNodesCount() const = 0;
+    virtual uint64_t getApproximateDataSize() const = 0;
+    virtual uint64_t getKeyArenaSize() const = 0;
+    virtual uint64_t getLatestSnapshotBufSize() const = 0;
 
-    void recalculateStorageStats();
+    virtual void recalculateStorageStats() = 0;
 
-    void reconfigure(const KeeperStorage::RequestForSession& request_for_session);
+    virtual void reconfigure(const KeeperStorageBase::RequestForSession& request_for_session) = 0;
 
-private:
+protected:
     CommitCallback commit_callback;
     /// In our state machine we always have a single snapshot which is stored
     /// in memory in compressed (serialized) format.
@@ -141,12 +116,7 @@ private:
 
     CoordinationSettingsPtr coordination_settings;
 
-    /// Main state machine logic
-    KeeperStoragePtr storage TSA_PT_GUARDED_BY(storage_and_responses_lock);
-
     /// Save/Load and Serialize/Deserialize logic for snapshots.
-    KeeperSnapshotManager snapshot_manager;
-
     /// Put processed responses into this queue
     ResponsesQueue & responses_queue;
 
@@ -163,7 +133,7 @@ private:
     /// for request.
     mutable std::mutex storage_and_responses_lock;
 
-    std::unordered_map<int64_t, std::unordered_map<Coordination::XID, std::shared_ptr<KeeperStorage::RequestForSession>>> parsed_request_cache;
+    std::unordered_map<int64_t, std::unordered_map<Coordination::XID, std::shared_ptr<KeeperStorageBase::RequestForSession>>> parsed_request_cache;
     uint64_t min_request_size_to_cache{0};
     /// we only need to protect the access to the map itself
     /// requests can be modified from anywhere without lock because a single request
@@ -188,8 +158,105 @@ private:
 
     KeeperSnapshotManagerS3 * snapshot_manager_s3;
 
-    KeeperStorage::ResponseForSession processReconfiguration(
-        const KeeperStorage::RequestForSession& request_for_session)
-        TSA_REQUIRES(storage_and_responses_lock);
+    virtual KeeperStorageBase::ResponseForSession processReconfiguration(
+        const KeeperStorageBase::RequestForSession& request_for_session)
+        TSA_REQUIRES(storage_and_responses_lock) = 0;
+
+};
+
+/// ClickHouse Keeper state machine. Wrapper for KeeperStorage.
+/// Responsible for entries commit, snapshots creation and so on.
+template<typename Storage>
+class KeeperStateMachine : public IKeeperStateMachine
+{
+public:
+    /// using CommitCallback = std::function<void(uint64_t, const KeeperStorage::RequestForSession &)>;
+
+    KeeperStateMachine(
+        ResponsesQueue & responses_queue_,
+        SnapshotsQueue & snapshots_queue_,
+        const CoordinationSettingsPtr & coordination_settings_,
+        const KeeperContextPtr & keeper_context_,
+        KeeperSnapshotManagerS3 * snapshot_manager_s3_,
+        CommitCallback commit_callback_ = {},
+        const std::string & superdigest_ = "");
+
+    /// Read state from the latest snapshot
+    void init() override;
+
+    bool preprocess(const KeeperStorageBase::RequestForSession & request_for_session) override;
+
+    nuraft::ptr<nuraft::buffer> pre_commit(uint64_t log_idx, nuraft::buffer & data) override;
+
+    nuraft::ptr<nuraft::buffer> commit(const uint64_t log_idx, nuraft::buffer & data) override; /// NOLINT
+
+    // allow_missing - whether the transaction we want to rollback can be missing from storage
+    // (can happen in case of exception during preprocessing)
+    void rollbackRequest(const KeeperStorageBase::RequestForSession & request_for_session, bool allow_missing) override;
+
+    void rollbackRequestNoLock(
+        const KeeperStorageBase::RequestForSession & request_for_session,
+        bool allow_missing) TSA_NO_THREAD_SAFETY_ANALYSIS;
+
+    /// Apply preliminarily saved (save_logical_snp_obj) snapshot to our state.
+    bool apply_snapshot(nuraft::snapshot & s) override;
+
+    /// Create new snapshot from current state.
+    void create_snapshot(nuraft::snapshot & s, nuraft::async_result<bool>::handler_type & when_done) override;
+
+    /// Save snapshot which was send by leader to us. After that we will apply it in apply_snapshot.
+    void save_logical_snp_obj(nuraft::snapshot & s, uint64_t & obj_id, nuraft::buffer & data, bool is_first_obj, bool is_last_obj) override;
+
+    // This should be used only for tests or keeper-data-dumper because it violates
+    // TSA -- we can't acquire the lock outside of this class or return a storage under lock
+    // in a reasonable way.
+    Storage & getStorageUnsafe() TSA_NO_THREAD_SAFETY_ANALYSIS
+    {
+        return *storage;
+    }
+
+    void shutdownStorage() override;
+
+    /// Process local read request
+    void processReadRequest(const KeeperStorageBase::RequestForSession & request_for_session) override;
+
+    std::vector<int64_t> getDeadSessions() override;
+
+    int64_t getNextZxid() const override;
+
+    KeeperStorageBase::Digest getNodesDigest() const override;
+
+    /// Introspection functions for 4lw commands
+    uint64_t getLastProcessedZxid() const override;
+
+    uint64_t getNodesCount() const override;
+    uint64_t getTotalWatchesCount() const override;
+    uint64_t getWatchedPathsCount() const override;
+    uint64_t getSessionsWithWatchesCount() const override;
+
+    void dumpWatches(WriteBufferFromOwnString & buf) const override;
+    void dumpWatchesByPath(WriteBufferFromOwnString & buf) const override;
+    void dumpSessionsAndEphemerals(WriteBufferFromOwnString & buf) const override;
+
+    uint64_t getSessionWithEphemeralNodesCount() const override;
+    uint64_t getTotalEphemeralNodesCount() const override;
+    uint64_t getApproximateDataSize() const override;
+    uint64_t getKeyArenaSize() const override;
+    uint64_t getLatestSnapshotBufSize() const override;
+
+    void recalculateStorageStats() override;
+
+    void reconfigure(const KeeperStorageBase::RequestForSession& request_for_session) override;
+
+private:
+    /// Main state machine logic
+    std::unique_ptr<Storage> storage; //TSA_PT_GUARDED_BY(storage_and_responses_lock);
+
+    /// Save/Load and Serialize/Deserialize logic for snapshots.
+    KeeperSnapshotManager<Storage> snapshot_manager;
+
+    KeeperStorageBase::ResponseForSession processReconfiguration(
+        const KeeperStorageBase::RequestForSession& request_for_session)
+        TSA_REQUIRES(storage_and_responses_lock) override;
 };
 }
diff --git a/src/Coordination/KeeperStorage.cpp b/src/Coordination/KeeperStorage.cpp
index a4bb65a2a72..e925067b6a3 100644
--- a/src/Coordination/KeeperStorage.cpp
+++ b/src/Coordination/KeeperStorage.cpp
@@ -61,10 +61,11 @@ String getSHA1(const String & userdata)
     return String{digest_id.begin(), digest_id.end()};
 }
 
+template<typename UncommittedState>
 bool fixupACL(
     const std::vector<Coordination::ACL> & request_acls,
     int64_t session_id,
-    const KeeperStorage::UncommittedState & uncommitted_state,
+    const UncommittedState & uncommitted_state,
     std::vector<Coordination::ACL> & result_acls)
 {
     if (request_acls.empty())
@@ -77,7 +78,7 @@ bool fixupACL(
         {
             uncommitted_state.forEachAuthInSession(
                 session_id,
-                [&](const KeeperStorage::AuthID & auth_id)
+                [&](const KeeperStorageBase::AuthID & auth_id)
                 {
                     valid_found = true;
                     Coordination::ACL new_acl = request_acl;
@@ -108,10 +109,10 @@ bool fixupACL(
     return valid_found;
 }
 
-KeeperStorage::ResponsesForSessions processWatchesImpl(
-    const String & path, KeeperStorage::Watches & watches, KeeperStorage::Watches & list_watches, Coordination::Event event_type)
+KeeperStorageBase::ResponsesForSessions processWatchesImpl(
+    const String & path, KeeperStorageBase::Watches & watches, KeeperStorageBase::Watches & list_watches, Coordination::Event event_type)
 {
-    KeeperStorage::ResponsesForSessions result;
+    KeeperStorageBase::ResponsesForSessions result;
     auto watch_it = watches.find(path);
     if (watch_it != watches.end())
     {
@@ -122,7 +123,7 @@ KeeperStorage::ResponsesForSessions processWatchesImpl(
         watch_response->type = event_type;
         watch_response->state = Coordination::State::CONNECTED;
         for (auto watcher_session : watch_it->second)
-            result.push_back(KeeperStorage::ResponseForSession{watcher_session, watch_response});
+            result.push_back(KeeperStorageBase::ResponseForSession{watcher_session, watch_response});
 
         watches.erase(watch_it);
     }
@@ -158,7 +159,7 @@ KeeperStorage::ResponsesForSessions processWatchesImpl(
 
             watch_list_response->state = Coordination::State::CONNECTED;
             for (auto watcher_session : watch_it->second)
-                result.push_back(KeeperStorage::ResponseForSession{watcher_session, watch_list_response});
+                result.push_back(KeeperStorageBase::ResponseForSession{watcher_session, watch_list_response});
 
             list_watches.erase(watch_it);
         }
@@ -193,31 +194,31 @@ uint64_t calculateDigest(std::string_view path, std::string_view data, const Coo
 
 }
 
-void KeeperStorage::Node::setData(String new_data)
+void KeeperMemNode::setData(String new_data)
 {
     size_bytes = size_bytes - data.size() + new_data.size();
     data = std::move(new_data);
 }
 
-void KeeperStorage::Node::addChild(StringRef child_path, bool update_size)
+void KeeperMemNode::addChild(StringRef child_path, bool update_size)
 {
     if (update_size) [[likely]]
         size_bytes += sizeof child_path;
     children.insert(child_path);
 }
 
-void KeeperStorage::Node::removeChild(StringRef child_path)
+void KeeperMemNode::removeChild(StringRef child_path)
 {
     size_bytes -= sizeof child_path;
     children.erase(child_path);
 }
 
-void KeeperStorage::Node::invalidateDigestCache() const
+void KeeperMemNode::invalidateDigestCache() const
 {
     cached_digest.reset();
 }
 
-UInt64 KeeperStorage::Node::getDigest(const std::string_view path) const
+UInt64 KeeperMemNode::getDigest(const std::string_view path) const
 {
     if (!cached_digest)
         cached_digest = calculateDigest(path, data, stat);
@@ -225,7 +226,7 @@ UInt64 KeeperStorage::Node::getDigest(const std::string_view path) const
     return *cached_digest;
 };
 
-void KeeperStorage::Node::shallowCopy(const KeeperStorage::Node & other)
+void KeeperMemNode::shallowCopy(const KeeperMemNode & other)
 {
     stat = other.stat;
     seq_num = other.seq_num;
@@ -233,14 +234,15 @@ void KeeperStorage::Node::shallowCopy(const KeeperStorage::Node & other)
     cached_digest = other.cached_digest;
 }
 
-void KeeperStorage::Node::recalculateSize()
+void KeeperMemNode::recalculateSize()
 {
-    size_bytes = sizeof(Node);
+    size_bytes = sizeof(KeeperMemNode);
     size_bytes += children.size() * sizeof(decltype(children)::value_type);
     size_bytes += data.size();
 }
 
-KeeperStorage::KeeperStorage(
+template<typename Container>
+KeeperStorage<Container>::KeeperStorage(
     int64_t tick_time_ms, const String & superdigest_, const KeeperContextPtr & keeper_context_, const bool initialize_system_nodes)
     : session_expiry_queue(tick_time_ms), keeper_context(keeper_context_), superdigest(superdigest_)
 {
@@ -252,7 +254,8 @@ KeeperStorage::KeeperStorage(
         initializeSystemNodes();
 }
 
-void KeeperStorage::initializeSystemNodes()
+template<typename Container>
+void KeeperStorage<Container>::initializeSystemNodes()
 {
     if (initialized)
         throw Exception(ErrorCodes::LOGICAL_ERROR, "KeeperStorage system nodes initialized twice");
@@ -314,12 +317,13 @@ struct Overloaded : Ts...
 template <class... Ts>
 Overloaded(Ts...) -> Overloaded<Ts...>;
 
-std::shared_ptr<KeeperStorage::Node> KeeperStorage::UncommittedState::tryGetNodeFromStorage(StringRef path) const
+template<typename Container>
+std::shared_ptr<typename Container::Node> KeeperStorage<Container>::UncommittedState::tryGetNodeFromStorage(StringRef path) const
 {
     if (auto node_it = storage.container.find(path); node_it != storage.container.end())
     {
         const auto & committed_node = node_it->value;
-        auto node = std::make_shared<KeeperStorage::Node>();
+        auto node = std::make_shared<KeeperStorage<Container>::Node>();
         node->shallowCopy(committed_node);
         return node;
     }
@@ -327,7 +331,8 @@ std::shared_ptr<KeeperStorage::Node> KeeperStorage::UncommittedState::tryGetNode
     return nullptr;
 }
 
-void KeeperStorage::UncommittedState::applyDelta(const Delta & delta)
+template<typename Container>
+void KeeperStorage<Container>::UncommittedState::applyDelta(const Delta & delta)
 {
     assert(!delta.path.empty());
     if (!nodes.contains(delta.path))
@@ -374,7 +379,8 @@ void KeeperStorage::UncommittedState::applyDelta(const Delta & delta)
         delta.operation);
 }
 
-void KeeperStorage::UncommittedState::addDelta(Delta new_delta)
+template<typename Container>
+void KeeperStorage<Container>::UncommittedState::addDelta(Delta new_delta)
 {
     const auto & added_delta = deltas.emplace_back(std::move(new_delta));
 
@@ -390,13 +396,15 @@ void KeeperStorage::UncommittedState::addDelta(Delta new_delta)
     }
 }
 
-void KeeperStorage::UncommittedState::addDeltas(std::vector<Delta> new_deltas)
+template<typename Container>
+void KeeperStorage<Container>::UncommittedState::addDeltas(std::vector<Delta> new_deltas)
 {
     for (auto & delta : new_deltas)
         addDelta(std::move(delta));
 }
 
-void KeeperStorage::UncommittedState::commit(int64_t commit_zxid)
+template<typename Container>
+void KeeperStorage<Container>::UncommittedState::commit(int64_t commit_zxid)
 {
     assert(deltas.empty() || deltas.front().zxid >= commit_zxid);
 
@@ -453,7 +461,8 @@ void KeeperStorage::UncommittedState::commit(int64_t commit_zxid)
     }
 }
 
-void KeeperStorage::UncommittedState::rollback(int64_t rollback_zxid)
+template<typename Container>
+void KeeperStorage<Container>::UncommittedState::rollback(int64_t rollback_zxid)
 {
     // we can only rollback the last zxid (if there is any)
     // if there is a delta with a larger zxid, we have invalid state
@@ -545,7 +554,8 @@ void KeeperStorage::UncommittedState::rollback(int64_t rollback_zxid)
     }
 }
 
-std::shared_ptr<KeeperStorage::Node> KeeperStorage::UncommittedState::getNode(StringRef path) const
+template<typename Container>
+std::shared_ptr<typename Container::Node> KeeperStorage<Container>::UncommittedState::getNode(StringRef path) const
 {
     if (auto node_it = nodes.find(path.toView()); node_it != nodes.end())
         return node_it->second.node;
@@ -553,7 +563,8 @@ std::shared_ptr<KeeperStorage::Node> KeeperStorage::UncommittedState::getNode(St
     return tryGetNodeFromStorage(path);
 }
 
-Coordination::ACLs KeeperStorage::UncommittedState::getACLs(StringRef path) const
+template<typename Container>
+Coordination::ACLs KeeperStorage<Container>::UncommittedState::getACLs(StringRef path) const
 {
     if (auto node_it = nodes.find(path.toView()); node_it != nodes.end())
         return node_it->second.acls;
@@ -565,7 +576,8 @@ Coordination::ACLs KeeperStorage::UncommittedState::getACLs(StringRef path) cons
     return storage.acl_map.convertNumber(node_it->value.acl_id);
 }
 
-void KeeperStorage::UncommittedState::forEachAuthInSession(int64_t session_id, std::function<void(const AuthID &)> func) const
+template<typename Container>
+void KeeperStorage<Container>::UncommittedState::forEachAuthInSession(int64_t session_id, std::function<void(const AuthID &)> func) const
 {
     const auto call_for_each_auth = [&func](const auto & auth_ids)
     {
@@ -604,7 +616,8 @@ namespace
 
 }
 
-void KeeperStorage::applyUncommittedState(KeeperStorage & other, int64_t last_log_idx)
+template<typename Container>
+void KeeperStorage<Container>::applyUncommittedState(KeeperStorage & other, int64_t last_log_idx)
 {
     std::unordered_set<int64_t> zxids_to_apply;
     for (const auto & transaction : uncommitted_transactions)
@@ -630,7 +643,8 @@ void KeeperStorage::applyUncommittedState(KeeperStorage & other, int64_t last_lo
     }
 }
 
-Coordination::Error KeeperStorage::commit(int64_t commit_zxid)
+template<typename Container>
+Coordination::Error KeeperStorage<Container>::commit(int64_t commit_zxid)
 {
     // Deltas are added with increasing ZXIDs
     // If there are no deltas for the commit_zxid (e.g. read requests), we instantly return
@@ -644,7 +658,7 @@ Coordination::Error KeeperStorage::commit(int64_t commit_zxid)
         auto result = std::visit(
             [&, &path = delta.path]<typename DeltaType>(DeltaType & operation) -> Coordination::Error
             {
-                if constexpr (std::same_as<DeltaType, KeeperStorage::CreateNodeDelta>)
+                if constexpr (std::same_as<DeltaType, CreateNodeDelta>)
                 {
                     if (!createNode(
                             path,
@@ -656,7 +670,7 @@ Coordination::Error KeeperStorage::commit(int64_t commit_zxid)
 
                     return Coordination::Error::ZOK;
                 }
-                else if constexpr (std::same_as<DeltaType, KeeperStorage::UpdateNodeDelta>)
+                else if constexpr (std::same_as<DeltaType, UpdateNodeDelta>)
                 {
                     auto node_it = container.find(path);
                     if (node_it == container.end())
@@ -671,14 +685,14 @@ Coordination::Error KeeperStorage::commit(int64_t commit_zxid)
 
                     return Coordination::Error::ZOK;
                 }
-                else if constexpr (std::same_as<DeltaType, KeeperStorage::RemoveNodeDelta>)
+                else if constexpr (std::same_as<DeltaType, RemoveNodeDelta>)
                 {
                     if (!removeNode(path, operation.version))
                         onStorageInconsistency();
 
                     return Coordination::Error::ZOK;
                 }
-                else if constexpr (std::same_as<DeltaType, KeeperStorage::SetACLDelta>)
+                else if constexpr (std::same_as<DeltaType, SetACLDelta>)
                 {
                     auto node_it = container.find(path);
                     if (node_it == container.end())
@@ -692,18 +706,18 @@ Coordination::Error KeeperStorage::commit(int64_t commit_zxid)
                     uint64_t acl_id = acl_map.convertACLs(operation.acls);
                     acl_map.addUsage(acl_id);
 
-                    container.updateValue(path, [acl_id](KeeperStorage::Node & node) { node.acl_id = acl_id; });
+                    container.updateValue(path, [acl_id](Node & node) { node.acl_id = acl_id; });
 
                     return Coordination::Error::ZOK;
                 }
-                else if constexpr (std::same_as<DeltaType, KeeperStorage::ErrorDelta>)
+                else if constexpr (std::same_as<DeltaType, ErrorDelta>)
                     return operation.error;
-                else if constexpr (std::same_as<DeltaType, KeeperStorage::SubDeltaEnd>)
+                else if constexpr (std::same_as<DeltaType, SubDeltaEnd>)
                 {
                     finish_subdelta = true;
                     return Coordination::Error::ZOK;
                 }
-                else if constexpr (std::same_as<DeltaType, KeeperStorage::AddAuthDelta>)
+                else if constexpr (std::same_as<DeltaType, AddAuthDelta>)
                 {
                     session_and_auth[operation.session_id].emplace_back(std::move(operation.auth_id));
                     return Coordination::Error::ZOK;
@@ -726,7 +740,8 @@ Coordination::Error KeeperStorage::commit(int64_t commit_zxid)
     return Coordination::Error::ZOK;
 }
 
-bool KeeperStorage::createNode(
+template<typename Container>
+bool KeeperStorage<Container>::createNode(
     const std::string & path,
     String data,
     const Coordination::Stat & stat,
@@ -745,7 +760,7 @@ bool KeeperStorage::createNode(
     if (container.contains(path))
         return false;
 
-    KeeperStorage::Node created_node;
+    Node created_node;
 
     uint64_t acl_id = acl_map.convertACLs(node_acls);
     acl_map.addUsage(acl_id);
@@ -759,7 +774,7 @@ bool KeeperStorage::createNode(
     auto child_path = getBaseNodeName(map_key->getKey());
     container.updateValue(
             parent_path,
-            [child_path](KeeperStorage::Node & parent)
+            [child_path](Node & parent)
             {
                 parent.addChild(child_path);
                 chassert(parent.stat.numChildren == static_cast<int32_t>(parent.getChildren().size()));
@@ -770,7 +785,8 @@ bool KeeperStorage::createNode(
     return true;
 };
 
-bool KeeperStorage::removeNode(const std::string & path, int32_t version)
+template<typename Container>
+bool KeeperStorage<Container>::removeNode(const std::string & path, int32_t version)
 {
     auto node_it = container.find(path);
     if (node_it == container.end())
@@ -787,7 +803,7 @@ bool KeeperStorage::removeNode(const std::string & path, int32_t version)
 
     container.updateValue(
         parentNodePath(path),
-        [child_basename = getBaseNodeName(node_it->key)](KeeperStorage::Node & parent)
+        [child_basename = getBaseNodeName(node_it->key)](Node & parent)
         {
             parent.removeChild(child_basename);
             chassert(parent.stat.numChildren == static_cast<int32_t>(parent.getChildren().size()));
@@ -800,54 +816,62 @@ bool KeeperStorage::removeNode(const std::string & path, int32_t version)
     return true;
 }
 
+template<typename Storage>
 struct KeeperStorageRequestProcessor
 {
     Coordination::ZooKeeperRequestPtr zk_request;
 
     explicit KeeperStorageRequestProcessor(const Coordination::ZooKeeperRequestPtr & zk_request_) : zk_request(zk_request_) { }
-    virtual Coordination::ZooKeeperResponsePtr process(KeeperStorage & storage, int64_t zxid) const = 0;
-    virtual std::vector<KeeperStorage::Delta>
-    preprocess(KeeperStorage & /*storage*/, int64_t /*zxid*/, int64_t /*session_id*/, int64_t /*time*/, uint64_t & /*digest*/, const KeeperContext & /*keeper_context*/) const
+
+    virtual Coordination::ZooKeeperResponsePtr process(Storage & storage, int64_t zxid) const = 0;
+
+    virtual std::vector<typename Storage::Delta>
+    preprocess(Storage & /*storage*/, int64_t /*zxid*/, int64_t /*session_id*/, int64_t /*time*/, uint64_t & /*digest*/, const KeeperContext & /*keeper_context*/) const
     {
         return {};
     }
 
     // process the request using locally committed data
     virtual Coordination::ZooKeeperResponsePtr
-    processLocal(KeeperStorage & /*storage*/, int64_t /*zxid*/) const
+    processLocal(Storage & /*storage*/, int64_t /*zxid*/) const
     {
         throw Exception{DB::ErrorCodes::LOGICAL_ERROR, "Cannot process the request locally"};
     }
 
-    virtual KeeperStorage::ResponsesForSessions
-    processWatches(KeeperStorage::Watches & /*watches*/, KeeperStorage::Watches & /*list_watches*/) const
+    virtual Storage::ResponsesForSessions
+    processWatches(Storage::Watches & /*watches*/, Storage::Watches & /*list_watches*/) const
     {
         return {};
     }
-    virtual bool checkAuth(KeeperStorage & /*storage*/, int64_t /*session_id*/, bool /*is_local*/) const { return true; }
+
+    virtual bool checkAuth(Storage & /*storage*/, int64_t /*session_id*/, bool /*is_local*/) const { return true; }
 
     virtual ~KeeperStorageRequestProcessor() = default;
 };
 
-struct KeeperStorageHeartbeatRequestProcessor final : public KeeperStorageRequestProcessor
+template<typename Storage>
+struct KeeperStorageHeartbeatRequestProcessor final : public KeeperStorageRequestProcessor<Storage>
 {
-    using KeeperStorageRequestProcessor::KeeperStorageRequestProcessor;
+    using KeeperStorageRequestProcessor<Storage>::KeeperStorageRequestProcessor;
+
     Coordination::ZooKeeperResponsePtr
-    process(KeeperStorage & /* storage */, int64_t /* zxid */) const override
+    process(Storage & /* storage */, int64_t /* zxid */) const override
     {
-        return zk_request->makeResponse();
+        return this->zk_request->makeResponse();
     }
 };
 
-struct KeeperStorageSyncRequestProcessor final : public KeeperStorageRequestProcessor
+template<typename Storage>
+struct KeeperStorageSyncRequestProcessor final : public KeeperStorageRequestProcessor<Storage>
 {
-    using KeeperStorageRequestProcessor::KeeperStorageRequestProcessor;
+    using KeeperStorageRequestProcessor<Storage>::KeeperStorageRequestProcessor;
+
     Coordination::ZooKeeperResponsePtr
-    process(KeeperStorage & /* storage */, int64_t /* zxid */) const override
+    process(Storage & /* storage */, int64_t /* zxid */) const override
     {
-        auto response = zk_request->makeResponse();
+        auto response = this->zk_request->makeResponse();
         dynamic_cast<Coordination::ZooKeeperSyncResponse &>(*response).path
-            = dynamic_cast<Coordination::ZooKeeperSyncRequest &>(*zk_request).path;
+            = dynamic_cast<Coordination::ZooKeeperSyncRequest &>(*this->zk_request).path;
         return response;
     }
 };
@@ -855,7 +879,8 @@ struct KeeperStorageSyncRequestProcessor final : public KeeperStorageRequestProc
 namespace
 {
 
-Coordination::ACLs getNodeACLs(KeeperStorage & storage, StringRef path, bool is_local)
+template<typename Storage>
+Coordination::ACLs getNodeACLs(Storage & storage, StringRef path, bool is_local)
 {
     if (is_local)
     {
@@ -883,7 +908,8 @@ void handleSystemNodeModification(const KeeperContext & keeper_context, std::str
 
 }
 
-bool KeeperStorage::checkACL(StringRef path, int32_t permission, int64_t session_id, bool is_local)
+template<typename Container>
+bool KeeperStorage<Container>::checkACL(StringRef path, int32_t permission, int64_t session_id, bool is_local)
 {
     const auto node_acls = getNodeACLs(*this, path, is_local);
     if (node_acls.empty())
@@ -910,7 +936,8 @@ bool KeeperStorage::checkACL(StringRef path, int32_t permission, int64_t session
     return false;
 }
 
-void KeeperStorage::unregisterEphemeralPath(int64_t session_id, const std::string & path)
+template<typename Container>
+void KeeperStorage<Container>::unregisterEphemeralPath(int64_t session_id, const std::string & path)
 {
     auto ephemerals_it = ephemerals.find(session_id);
     if (ephemerals_it == ephemerals.end())
@@ -921,43 +948,44 @@ void KeeperStorage::unregisterEphemeralPath(int64_t session_id, const std::strin
         ephemerals.erase(ephemerals_it);
 }
 
-struct KeeperStorageCreateRequestProcessor final : public KeeperStorageRequestProcessor
+template<typename Storage>
+struct KeeperStorageCreateRequestProcessor final : public KeeperStorageRequestProcessor<Storage>
 {
-    using KeeperStorageRequestProcessor::KeeperStorageRequestProcessor;
+    using KeeperStorageRequestProcessor<Storage>::KeeperStorageRequestProcessor;
 
-    KeeperStorage::ResponsesForSessions
-    processWatches(KeeperStorage::Watches & watches, KeeperStorage::Watches & list_watches) const override
+    KeeperStorageBase::ResponsesForSessions
+    processWatches(Storage::Watches & watches, Storage::Watches & list_watches) const override
     {
-        return processWatchesImpl(zk_request->getPath(), watches, list_watches, Coordination::Event::CREATED);
+        return processWatchesImpl(this->zk_request->getPath(), watches, list_watches, Coordination::Event::CREATED);
     }
 
-    bool checkAuth(KeeperStorage & storage, int64_t session_id, bool is_local) const override
+    bool checkAuth(Storage & storage, int64_t session_id, bool is_local) const override
     {
-        auto path = zk_request->getPath();
+        auto path = this->zk_request->getPath();
         return storage.checkACL(parentNodePath(path), Coordination::ACL::Create, session_id, is_local);
     }
 
-    std::vector<KeeperStorage::Delta>
-    preprocess(KeeperStorage & storage, int64_t zxid, int64_t session_id, int64_t time, uint64_t & digest, const KeeperContext & keeper_context) const override
+    std::vector<typename Storage::Delta>
+    preprocess(Storage & storage, int64_t zxid, int64_t session_id, int64_t time, uint64_t & digest, const KeeperContext & keeper_context) const override
     {
         ProfileEvents::increment(ProfileEvents::KeeperCreateRequest);
-        Coordination::ZooKeeperCreateRequest & request = dynamic_cast<Coordination::ZooKeeperCreateRequest &>(*zk_request);
+        Coordination::ZooKeeperCreateRequest & request = dynamic_cast<Coordination::ZooKeeperCreateRequest &>(*this->zk_request);
 
-        std::vector<KeeperStorage::Delta> new_deltas;
+        std::vector<typename Storage::Delta> new_deltas;
 
         auto parent_path = parentNodePath(request.path);
         auto parent_node = storage.uncommitted_state.getNode(parent_path);
         if (parent_node == nullptr)
-            return {KeeperStorage::Delta{zxid, Coordination::Error::ZNONODE}};
+            return {typename Storage::Delta{zxid, Coordination::Error::ZNONODE}};
 
         else if (parent_node->stat.ephemeralOwner != 0)
-            return {KeeperStorage::Delta{zxid, Coordination::Error::ZNOCHILDRENFOREPHEMERALS}};
+            return {typename Storage::Delta{zxid, Coordination::Error::ZNOCHILDRENFOREPHEMERALS}};
 
         std::string path_created = request.path;
         if (request.is_sequential)
         {
             if (request.not_exists)
-                return {KeeperStorage::Delta{zxid, Coordination::Error::ZBADARGUMENTS}};
+                return {typename Storage::Delta{zxid, Coordination::Error::ZBADARGUMENTS}};
 
             auto seq_num = parent_node->seq_num;
 
@@ -973,30 +1001,30 @@ struct KeeperStorageCreateRequestProcessor final : public KeeperStorageRequestPr
             auto error_msg = fmt::format("Trying to create a node inside the internal Keeper path ({}) which is not allowed. Path: {}", keeper_system_path, path_created);
 
             handleSystemNodeModification(keeper_context, error_msg);
-            return {KeeperStorage::Delta{zxid, Coordination::Error::ZBADARGUMENTS}};
+            return {typename Storage::Delta{zxid, Coordination::Error::ZBADARGUMENTS}};
         }
 
         if (storage.uncommitted_state.getNode(path_created))
         {
-            if (zk_request->getOpNum() == Coordination::OpNum::CreateIfNotExists)
+            if (this->zk_request->getOpNum() == Coordination::OpNum::CreateIfNotExists)
                 return new_deltas;
 
-            return {KeeperStorage::Delta{zxid, Coordination::Error::ZNODEEXISTS}};
+            return {typename Storage::Delta{zxid, Coordination::Error::ZNODEEXISTS}};
         }
 
         if (getBaseNodeName(path_created).size == 0)
-            return {KeeperStorage::Delta{zxid, Coordination::Error::ZBADARGUMENTS}};
+            return {typename Storage::Delta{zxid, Coordination::Error::ZBADARGUMENTS}};
 
         Coordination::ACLs node_acls;
         if (!fixupACL(request.acls, session_id, storage.uncommitted_state, node_acls))
-            return {KeeperStorage::Delta{zxid, Coordination::Error::ZINVALIDACL}};
+            return {typename Storage::Delta{zxid, Coordination::Error::ZINVALIDACL}};
 
         if (request.is_ephemeral)
             storage.ephemerals[session_id].emplace(path_created);
 
         int32_t parent_cversion = request.parent_cversion;
 
-        auto parent_update = [parent_cversion, zxid](KeeperStorage::Node & node)
+        auto parent_update = [parent_cversion, zxid](Storage::Node & node)
         {
             /// Increment sequential number even if node is not sequential
             ++node.seq_num;
@@ -1010,7 +1038,7 @@ struct KeeperStorageCreateRequestProcessor final : public KeeperStorageRequestPr
             ++node.stat.numChildren;
         };
 
-        new_deltas.emplace_back(std::string{parent_path}, zxid, KeeperStorage::UpdateNodeDelta{std::move(parent_update)});
+        new_deltas.emplace_back(std::string{parent_path}, zxid, typename Storage::UpdateNodeDelta{std::move(parent_update)});
 
         Coordination::Stat stat;
         stat.czxid = zxid;
@@ -1028,20 +1056,20 @@ struct KeeperStorageCreateRequestProcessor final : public KeeperStorageRequestPr
         new_deltas.emplace_back(
             std::move(path_created),
             zxid,
-            KeeperStorage::CreateNodeDelta{stat, request.is_sequential, std::move(node_acls), request.data});
+            typename Storage::CreateNodeDelta{stat, request.is_sequential, std::move(node_acls), request.data});
 
         digest = storage.calculateNodesDigest(digest, new_deltas);
         return new_deltas;
     }
 
-    Coordination::ZooKeeperResponsePtr process(KeeperStorage & storage, int64_t zxid) const override
+    Coordination::ZooKeeperResponsePtr process(Storage & storage, int64_t zxid) const override
     {
-        Coordination::ZooKeeperResponsePtr response_ptr = zk_request->makeResponse();
+        Coordination::ZooKeeperResponsePtr response_ptr = this->zk_request->makeResponse();
         Coordination::ZooKeeperCreateResponse & response = dynamic_cast<Coordination::ZooKeeperCreateResponse &>(*response_ptr);
 
         if (storage.uncommitted_state.deltas.begin()->zxid != zxid)
         {
-            response.path_created = zk_request->getPath();
+            response.path_created = this->zk_request->getPath();
             response.error = Coordination::Error::ZOK;
             return response_ptr;
         }
@@ -1057,7 +1085,7 @@ struct KeeperStorageCreateRequestProcessor final : public KeeperStorageRequestPr
             deltas.begin(),
             deltas.end(),
             [zxid](const auto & delta)
-            { return delta.zxid == zxid && std::holds_alternative<KeeperStorage::CreateNodeDelta>(delta.operation); });
+            { return delta.zxid == zxid && std::holds_alternative<typename Storage::CreateNodeDelta>(delta.operation); });
 
         response.path_created = create_delta_it->path;
         response.error = Coordination::Error::ZOK;
@@ -1065,37 +1093,38 @@ struct KeeperStorageCreateRequestProcessor final : public KeeperStorageRequestPr
     }
 };
 
-struct KeeperStorageGetRequestProcessor final : public KeeperStorageRequestProcessor
+template<typename Storage>
+struct KeeperStorageGetRequestProcessor final : public KeeperStorageRequestProcessor<Storage>
 {
-    bool checkAuth(KeeperStorage & storage, int64_t session_id, bool is_local) const override
+    bool checkAuth(Storage & storage, int64_t session_id, bool is_local) const override
     {
-        return storage.checkACL(zk_request->getPath(), Coordination::ACL::Read, session_id, is_local);
+        return storage.checkACL(this->zk_request->getPath(), Coordination::ACL::Read, session_id, is_local);
     }
 
-    using KeeperStorageRequestProcessor::KeeperStorageRequestProcessor;
+    using KeeperStorageRequestProcessor<Storage>::KeeperStorageRequestProcessor;
 
-    std::vector<KeeperStorage::Delta>
-    preprocess(KeeperStorage & storage, int64_t zxid, int64_t /*session_id*/, int64_t /*time*/, uint64_t & /*digest*/, const KeeperContext & /*keeper_context*/) const override
+    std::vector<typename Storage::Delta>
+    preprocess(Storage & storage, int64_t zxid, int64_t /*session_id*/, int64_t /*time*/, uint64_t & /*digest*/, const KeeperContext & /*keeper_context*/) const override
     {
         ProfileEvents::increment(ProfileEvents::KeeperGetRequest);
-        Coordination::ZooKeeperGetRequest & request = dynamic_cast<Coordination::ZooKeeperGetRequest &>(*zk_request);
+        Coordination::ZooKeeperGetRequest & request = dynamic_cast<Coordination::ZooKeeperGetRequest &>(*this->zk_request);
 
         if (request.path == Coordination::keeper_api_feature_flags_path
             || request.path == Coordination::keeper_config_path)
             return {};
 
         if (!storage.uncommitted_state.getNode(request.path))
-            return {KeeperStorage::Delta{zxid, Coordination::Error::ZNONODE}};
+            return {typename Storage::Delta{zxid, Coordination::Error::ZNONODE}};
 
         return {};
     }
 
     template <bool local>
-    Coordination::ZooKeeperResponsePtr processImpl(KeeperStorage & storage, int64_t zxid) const
+    Coordination::ZooKeeperResponsePtr processImpl(Storage & storage, int64_t zxid) const
     {
-        Coordination::ZooKeeperResponsePtr response_ptr = zk_request->makeResponse();
+        Coordination::ZooKeeperResponsePtr response_ptr = this->zk_request->makeResponse();
         Coordination::ZooKeeperGetResponse & response = dynamic_cast<Coordination::ZooKeeperGetResponse &>(*response_ptr);
-        Coordination::ZooKeeperGetRequest & request = dynamic_cast<Coordination::ZooKeeperGetRequest &>(*zk_request);
+        Coordination::ZooKeeperGetRequest & request = dynamic_cast<Coordination::ZooKeeperGetRequest &>(*this->zk_request);
 
         if constexpr (!local)
         {
@@ -1134,40 +1163,42 @@ struct KeeperStorageGetRequestProcessor final : public KeeperStorageRequestProce
     }
 
 
-    Coordination::ZooKeeperResponsePtr process(KeeperStorage & storage, int64_t zxid) const override
+    Coordination::ZooKeeperResponsePtr process(Storage & storage, int64_t zxid) const override
     {
         return processImpl<false>(storage, zxid);
     }
 
-    Coordination::ZooKeeperResponsePtr processLocal(KeeperStorage & storage, int64_t zxid) const override
+    Coordination::ZooKeeperResponsePtr processLocal(Storage & storage, int64_t zxid) const override
     {
         ProfileEvents::increment(ProfileEvents::KeeperGetRequest);
         return processImpl<true>(storage, zxid);
     }
 };
 
-struct KeeperStorageRemoveRequestProcessor final : public KeeperStorageRequestProcessor
+template<typename Storage>
+struct KeeperStorageRemoveRequestProcessor final : public KeeperStorageRequestProcessor<Storage>
 {
-    bool checkAuth(KeeperStorage & storage, int64_t session_id, bool is_local) const override
+    bool checkAuth(Storage & storage, int64_t session_id, bool is_local) const override
     {
-        return storage.checkACL(parentNodePath(zk_request->getPath()), Coordination::ACL::Delete, session_id, is_local);
+        return storage.checkACL(parentNodePath(this->zk_request->getPath()), Coordination::ACL::Delete, session_id, is_local);
     }
 
-    using KeeperStorageRequestProcessor::KeeperStorageRequestProcessor;
-    std::vector<KeeperStorage::Delta>
-    preprocess(KeeperStorage & storage, int64_t zxid, int64_t /*session_id*/, int64_t /*time*/, uint64_t & digest, const KeeperContext & keeper_context) const override
+    using KeeperStorageRequestProcessor<Storage>::KeeperStorageRequestProcessor;
+
+    std::vector<typename Storage::Delta>
+    preprocess(Storage & storage, int64_t zxid, int64_t /*session_id*/, int64_t /*time*/, uint64_t & digest, const KeeperContext & keeper_context) const override
     {
         ProfileEvents::increment(ProfileEvents::KeeperRemoveRequest);
-        Coordination::ZooKeeperRemoveRequest & request = dynamic_cast<Coordination::ZooKeeperRemoveRequest &>(*zk_request);
+        Coordination::ZooKeeperRemoveRequest & request = dynamic_cast<Coordination::ZooKeeperRemoveRequest &>(*this->zk_request);
 
-        std::vector<KeeperStorage::Delta> new_deltas;
+        std::vector<typename Storage::Delta> new_deltas;
 
         if (Coordination::matchPath(request.path, keeper_system_path) != Coordination::PathMatchResult::NOT_MATCH)
         {
             auto error_msg = fmt::format("Trying to delete an internal Keeper path ({}) which is not allowed", request.path);
 
             handleSystemNodeModification(keeper_context, error_msg);
-            return {KeeperStorage::Delta{zxid, Coordination::Error::ZBADARGUMENTS}};
+            return {typename Storage::Delta{zxid, Coordination::Error::ZBADARGUMENTS}};
         }
 
         const auto update_parent_pzxid = [&]()
@@ -1179,9 +1210,9 @@ struct KeeperStorageRemoveRequestProcessor final : public KeeperStorageRequestPr
             new_deltas.emplace_back(
                 std::string{parent_path},
                 zxid,
-                KeeperStorage::UpdateNodeDelta
+                typename Storage::UpdateNodeDelta
                 {
-                    [zxid](KeeperStorage::Node & parent)
+                    [zxid](Storage::Node & parent)
                     {
                         if (parent.stat.pzxid < zxid)
                             parent.stat.pzxid = zxid;
@@ -1196,12 +1227,12 @@ struct KeeperStorageRemoveRequestProcessor final : public KeeperStorageRequestPr
         {
             if (request.restored_from_zookeeper_log)
                 update_parent_pzxid();
-            return {KeeperStorage::Delta{zxid, Coordination::Error::ZNONODE}};
+            return {typename Storage::Delta{zxid, Coordination::Error::ZNONODE}};
         }
         else if (request.version != -1 && request.version != node->stat.version)
-            return {KeeperStorage::Delta{zxid, Coordination::Error::ZBADVERSION}};
+            return {typename Storage::Delta{zxid, Coordination::Error::ZBADVERSION}};
         else if (node->stat.numChildren != 0)
-            return {KeeperStorage::Delta{zxid, Coordination::Error::ZNOTEMPTY}};
+            return {typename Storage::Delta{zxid, Coordination::Error::ZNOTEMPTY}};
 
         if (request.restored_from_zookeeper_log)
             update_parent_pzxid();
@@ -1209,13 +1240,13 @@ struct KeeperStorageRemoveRequestProcessor final : public KeeperStorageRequestPr
         new_deltas.emplace_back(
             std::string{parentNodePath(request.path)},
             zxid,
-            KeeperStorage::UpdateNodeDelta{[](KeeperStorage::Node & parent)
+            typename Storage::UpdateNodeDelta{[](typename Storage::Node & parent)
                                            {
                                                ++parent.stat.cversion;
                                                --parent.stat.numChildren;
                                            }});
 
-        new_deltas.emplace_back(request.path, zxid, KeeperStorage::RemoveNodeDelta{request.version, node->stat.ephemeralOwner});
+        new_deltas.emplace_back(request.path, zxid, typename Storage::RemoveNodeDelta{request.version, node->stat.ephemeralOwner});
 
         if (node->stat.ephemeralOwner != 0)
             storage.unregisterEphemeralPath(node->stat.ephemeralOwner, request.path);
@@ -1225,44 +1256,45 @@ struct KeeperStorageRemoveRequestProcessor final : public KeeperStorageRequestPr
         return new_deltas;
     }
 
-    Coordination::ZooKeeperResponsePtr process(KeeperStorage & storage, int64_t zxid) const override
+    Coordination::ZooKeeperResponsePtr process(Storage & storage, int64_t zxid) const override
     {
-        Coordination::ZooKeeperResponsePtr response_ptr = zk_request->makeResponse();
+        Coordination::ZooKeeperResponsePtr response_ptr = this->zk_request->makeResponse();
         Coordination::ZooKeeperRemoveResponse & response = dynamic_cast<Coordination::ZooKeeperRemoveResponse &>(*response_ptr);
 
         response.error = storage.commit(zxid);
         return response_ptr;
     }
 
-    KeeperStorage::ResponsesForSessions
-    processWatches(KeeperStorage::Watches & watches, KeeperStorage::Watches & list_watches) const override
+    KeeperStorageBase::ResponsesForSessions
+    processWatches(typename Storage::Watches & watches, typename Storage::Watches & list_watches) const override
     {
-        return processWatchesImpl(zk_request->getPath(), watches, list_watches, Coordination::Event::DELETED);
+        return processWatchesImpl(this->zk_request->getPath(), watches, list_watches, Coordination::Event::DELETED);
     }
 };
 
-struct KeeperStorageExistsRequestProcessor final : public KeeperStorageRequestProcessor
+template<typename Storage>
+struct KeeperStorageExistsRequestProcessor final : public KeeperStorageRequestProcessor<Storage>
 {
-    using KeeperStorageRequestProcessor::KeeperStorageRequestProcessor;
+    using KeeperStorageRequestProcessor<Storage>::KeeperStorageRequestProcessor;
 
-    std::vector<KeeperStorage::Delta>
-    preprocess(KeeperStorage & storage, int64_t zxid, int64_t /*session_id*/, int64_t /*time*/, uint64_t & /*digest*/, const KeeperContext & /*keeper_context*/) const override
+    std::vector<typename Storage::Delta>
+    preprocess(Storage & storage, int64_t zxid, int64_t /*session_id*/, int64_t /*time*/, uint64_t & /*digest*/, const KeeperContext & /*keeper_context*/) const override
     {
         ProfileEvents::increment(ProfileEvents::KeeperExistsRequest);
-        Coordination::ZooKeeperExistsRequest & request = dynamic_cast<Coordination::ZooKeeperExistsRequest &>(*zk_request);
+        Coordination::ZooKeeperExistsRequest & request = dynamic_cast<Coordination::ZooKeeperExistsRequest &>(*this->zk_request);
 
         if (!storage.uncommitted_state.getNode(request.path))
-            return {KeeperStorage::Delta{zxid, Coordination::Error::ZNONODE}};
+            return {typename Storage::Delta{zxid, Coordination::Error::ZNONODE}};
 
         return {};
     }
 
     template <bool local>
-    Coordination::ZooKeeperResponsePtr processImpl(KeeperStorage & storage, int64_t zxid) const
+    Coordination::ZooKeeperResponsePtr processImpl(Storage & storage, int64_t zxid) const
     {
-        Coordination::ZooKeeperResponsePtr response_ptr = zk_request->makeResponse();
+        Coordination::ZooKeeperResponsePtr response_ptr = this->zk_request->makeResponse();
         Coordination::ZooKeeperExistsResponse & response = dynamic_cast<Coordination::ZooKeeperExistsResponse &>(*response_ptr);
-        Coordination::ZooKeeperExistsRequest & request = dynamic_cast<Coordination::ZooKeeperExistsRequest &>(*zk_request);
+        Coordination::ZooKeeperExistsRequest & request = dynamic_cast<Coordination::ZooKeeperExistsRequest &>(*this->zk_request);
 
         if constexpr (!local)
         {
@@ -1291,55 +1323,57 @@ struct KeeperStorageExistsRequestProcessor final : public KeeperStorageRequestPr
         return response_ptr;
     }
 
-    Coordination::ZooKeeperResponsePtr process(KeeperStorage & storage, int64_t zxid) const override
+    Coordination::ZooKeeperResponsePtr process(Storage & storage, int64_t zxid) const override
     {
         return processImpl<false>(storage, zxid);
     }
 
-    Coordination::ZooKeeperResponsePtr processLocal(KeeperStorage & storage, int64_t zxid) const override
+    Coordination::ZooKeeperResponsePtr processLocal(Storage & storage, int64_t zxid) const override
     {
         ProfileEvents::increment(ProfileEvents::KeeperExistsRequest);
         return processImpl<true>(storage, zxid);
     }
 };
 
-struct KeeperStorageSetRequestProcessor final : public KeeperStorageRequestProcessor
+template<typename Storage>
+struct KeeperStorageSetRequestProcessor final : public KeeperStorageRequestProcessor<Storage>
 {
-    bool checkAuth(KeeperStorage & storage, int64_t session_id, bool is_local) const override
+    bool checkAuth(Storage & storage, int64_t session_id, bool is_local) const override
     {
-        return storage.checkACL(zk_request->getPath(), Coordination::ACL::Write, session_id, is_local);
+        return storage.checkACL(this->zk_request->getPath(), Coordination::ACL::Write, session_id, is_local);
     }
 
-    using KeeperStorageRequestProcessor::KeeperStorageRequestProcessor;
-    std::vector<KeeperStorage::Delta>
-    preprocess(KeeperStorage & storage, int64_t zxid, int64_t /*session_id*/, int64_t time, uint64_t & digest, const KeeperContext & keeper_context) const override
+    using KeeperStorageRequestProcessor<Storage>::KeeperStorageRequestProcessor;
+
+    std::vector<typename Storage::Delta>
+    preprocess(Storage & storage, int64_t zxid, int64_t /*session_id*/, int64_t time, uint64_t & digest, const KeeperContext & keeper_context) const override
     {
         ProfileEvents::increment(ProfileEvents::KeeperSetRequest);
-        Coordination::ZooKeeperSetRequest & request = dynamic_cast<Coordination::ZooKeeperSetRequest &>(*zk_request);
+        Coordination::ZooKeeperSetRequest & request = dynamic_cast<Coordination::ZooKeeperSetRequest &>(*this->zk_request);
 
-        std::vector<KeeperStorage::Delta> new_deltas;
+        std::vector<typename Storage::Delta> new_deltas;
 
         if (Coordination::matchPath(request.path, keeper_system_path) != Coordination::PathMatchResult::NOT_MATCH)
         {
             auto error_msg = fmt::format("Trying to update an internal Keeper path ({}) which is not allowed", request.path);
 
             handleSystemNodeModification(keeper_context, error_msg);
-            return {KeeperStorage::Delta{zxid, Coordination::Error::ZBADARGUMENTS}};
+            return {typename Storage::Delta{zxid, Coordination::Error::ZBADARGUMENTS}};
         }
 
         if (!storage.uncommitted_state.getNode(request.path))
-            return {KeeperStorage::Delta{zxid, Coordination::Error::ZNONODE}};
+            return {typename Storage::Delta{zxid, Coordination::Error::ZNONODE}};
 
         auto node = storage.uncommitted_state.getNode(request.path);
 
         if (request.version != -1 && request.version != node->stat.version)
-            return {KeeperStorage::Delta{zxid, Coordination::Error::ZBADVERSION}};
+            return {typename Storage::Delta{zxid, Coordination::Error::ZBADVERSION}};
 
         new_deltas.emplace_back(
             request.path,
             zxid,
-            KeeperStorage::UpdateNodeDelta{
-                [zxid, data = request.data, time](KeeperStorage::Node & value)
+            typename Storage::UpdateNodeDelta{
+                [zxid, data = request.data, time](typename Storage::Node & value)
                 {
                     value.stat.version++;
                     value.stat.mzxid = zxid;
@@ -1352,9 +1386,9 @@ struct KeeperStorageSetRequestProcessor final : public KeeperStorageRequestProce
         new_deltas.emplace_back(
                 parentNodePath(request.path).toString(),
                 zxid,
-                KeeperStorage::UpdateNodeDelta
+                typename Storage::UpdateNodeDelta
                 {
-                    [](KeeperStorage::Node & parent)
+                    [](Storage::Node & parent)
                     {
                         parent.stat.cversion++;
                     }
@@ -1365,13 +1399,13 @@ struct KeeperStorageSetRequestProcessor final : public KeeperStorageRequestProce
         return new_deltas;
     }
 
-    Coordination::ZooKeeperResponsePtr process(KeeperStorage & storage, int64_t zxid) const override
+    Coordination::ZooKeeperResponsePtr process(Storage & storage, int64_t zxid) const override
     {
         auto & container = storage.container;
 
-        Coordination::ZooKeeperResponsePtr response_ptr = zk_request->makeResponse();
+        Coordination::ZooKeeperResponsePtr response_ptr = this->zk_request->makeResponse();
         Coordination::ZooKeeperSetResponse & response = dynamic_cast<Coordination::ZooKeeperSetResponse &>(*response_ptr);
-        Coordination::ZooKeeperSetRequest & request = dynamic_cast<Coordination::ZooKeeperSetRequest &>(*zk_request);
+        Coordination::ZooKeeperSetRequest & request = dynamic_cast<Coordination::ZooKeeperSetRequest &>(*this->zk_request);
 
         if (const auto result = storage.commit(zxid); result != Coordination::Error::ZOK)
         {
@@ -1389,40 +1423,41 @@ struct KeeperStorageSetRequestProcessor final : public KeeperStorageRequestProce
         return response_ptr;
     }
 
-    KeeperStorage::ResponsesForSessions
-    processWatches(KeeperStorage::Watches & watches, KeeperStorage::Watches & list_watches) const override
+    KeeperStorageBase::ResponsesForSessions
+    processWatches(typename Storage::Watches & watches, typename Storage::Watches & list_watches) const override
     {
-        return processWatchesImpl(zk_request->getPath(), watches, list_watches, Coordination::Event::CHANGED);
+        return processWatchesImpl(this->zk_request->getPath(), watches, list_watches, Coordination::Event::CHANGED);
     }
 };
 
-struct KeeperStorageListRequestProcessor final : public KeeperStorageRequestProcessor
+template<typename Storage>
+struct KeeperStorageListRequestProcessor final : public KeeperStorageRequestProcessor<Storage>
 {
-    bool checkAuth(KeeperStorage & storage, int64_t session_id, bool is_local) const override
+    bool checkAuth(Storage & storage, int64_t session_id, bool is_local) const override
     {
-        return storage.checkACL(zk_request->getPath(), Coordination::ACL::Read, session_id, is_local);
+        return storage.checkACL(this->zk_request->getPath(), Coordination::ACL::Read, session_id, is_local);
     }
 
-    using KeeperStorageRequestProcessor::KeeperStorageRequestProcessor;
-    std::vector<KeeperStorage::Delta>
-    preprocess(KeeperStorage & storage, int64_t zxid, int64_t /*session_id*/, int64_t /*time*/, uint64_t & /*digest*/, const KeeperContext & /*keeper_context*/) const override
+    using KeeperStorageRequestProcessor<Storage>::KeeperStorageRequestProcessor;
+
+    std::vector<typename Storage::Delta>
+    preprocess(Storage & storage, int64_t zxid, int64_t /*session_id*/, int64_t /*time*/, uint64_t & /*digest*/, const KeeperContext & /*keeper_context*/) const override
     {
         ProfileEvents::increment(ProfileEvents::KeeperListRequest);
-        Coordination::ZooKeeperListRequest & request = dynamic_cast<Coordination::ZooKeeperListRequest &>(*zk_request);
+        Coordination::ZooKeeperListRequest & request = dynamic_cast<Coordination::ZooKeeperListRequest &>(*this->zk_request);
 
         if (!storage.uncommitted_state.getNode(request.path))
-            return {KeeperStorage::Delta{zxid, Coordination::Error::ZNONODE}};
+            return {typename Storage::Delta{zxid, Coordination::Error::ZNONODE}};
 
         return {};
     }
 
-
     template <bool local>
-    Coordination::ZooKeeperResponsePtr processImpl(KeeperStorage & storage, int64_t zxid) const
+    Coordination::ZooKeeperResponsePtr processImpl(Storage & storage, int64_t zxid) const
     {
-        Coordination::ZooKeeperResponsePtr response_ptr = zk_request->makeResponse();
+        Coordination::ZooKeeperResponsePtr response_ptr = this->zk_request->makeResponse();
         Coordination::ZooKeeperListResponse & response = dynamic_cast<Coordination::ZooKeeperListResponse &>(*response_ptr);
-        Coordination::ZooKeeperListRequest & request = dynamic_cast<Coordination::ZooKeeperListRequest &>(*zk_request);
+        Coordination::ZooKeeperListRequest & request = dynamic_cast<Coordination::ZooKeeperListRequest &>(*this->zk_request);
 
         if constexpr (!local)
         {
@@ -1487,63 +1522,64 @@ struct KeeperStorageListRequestProcessor final : public KeeperStorageRequestProc
         return response_ptr;
     }
 
-    Coordination::ZooKeeperResponsePtr process(KeeperStorage & storage, int64_t zxid) const override
+    Coordination::ZooKeeperResponsePtr process(Storage & storage, int64_t zxid) const override
     {
         return processImpl<false>(storage, zxid);
     }
 
-    Coordination::ZooKeeperResponsePtr processLocal(KeeperStorage & storage, int64_t zxid) const override
+    Coordination::ZooKeeperResponsePtr processLocal(Storage & storage, int64_t zxid) const override
     {
         ProfileEvents::increment(ProfileEvents::KeeperListRequest);
         return processImpl<true>(storage, zxid);
     }
 };
 
-struct KeeperStorageCheckRequestProcessor final : public KeeperStorageRequestProcessor
+template<typename Storage>
+struct KeeperStorageCheckRequestProcessor final : public KeeperStorageRequestProcessor<Storage>
 {
     explicit KeeperStorageCheckRequestProcessor(const Coordination::ZooKeeperRequestPtr & zk_request_)
-        : KeeperStorageRequestProcessor(zk_request_)
+        : KeeperStorageRequestProcessor<Storage>(zk_request_)
     {
-        check_not_exists = zk_request->getOpNum() == Coordination::OpNum::CheckNotExists;
+        check_not_exists = this->zk_request->getOpNum() == Coordination::OpNum::CheckNotExists;
     }
 
-    bool checkAuth(KeeperStorage & storage, int64_t session_id, bool is_local) const override
+    bool checkAuth(Storage & storage, int64_t session_id, bool is_local) const override
     {
-        auto path = zk_request->getPath();
+        auto path = this->zk_request->getPath();
         return storage.checkACL(check_not_exists ? parentNodePath(path) : path, Coordination::ACL::Read, session_id, is_local);
     }
 
-    std::vector<KeeperStorage::Delta>
-    preprocess(KeeperStorage & storage, int64_t zxid, int64_t /*session_id*/, int64_t /*time*/, uint64_t & /*digest*/, const KeeperContext & /*keeper_context*/) const override
+    std::vector<typename Storage::Delta>
+    preprocess(Storage & storage, int64_t zxid, int64_t /*session_id*/, int64_t /*time*/, uint64_t & /*digest*/, const KeeperContext & /*keeper_context*/) const override
     {
         ProfileEvents::increment(ProfileEvents::KeeperCheckRequest);
 
-        Coordination::ZooKeeperCheckRequest & request = dynamic_cast<Coordination::ZooKeeperCheckRequest &>(*zk_request);
+        Coordination::ZooKeeperCheckRequest & request = dynamic_cast<Coordination::ZooKeeperCheckRequest &>(*this->zk_request);
 
         auto node = storage.uncommitted_state.getNode(request.path);
         if (check_not_exists)
         {
             if (node && (request.version == -1 || request.version == node->stat.version))
-                return {KeeperStorage::Delta{zxid, Coordination::Error::ZNODEEXISTS}};
+                return {typename Storage::Delta{zxid, Coordination::Error::ZNODEEXISTS}};
         }
         else
         {
             if (!node)
-                return {KeeperStorage::Delta{zxid, Coordination::Error::ZNONODE}};
+                return {typename Storage::Delta{zxid, Coordination::Error::ZNONODE}};
 
             if (request.version != -1 && request.version != node->stat.version)
-                return {KeeperStorage::Delta{zxid, Coordination::Error::ZBADVERSION}};
+                return {typename Storage::Delta{zxid, Coordination::Error::ZBADVERSION}};
         }
 
         return {};
     }
 
     template <bool local>
-    Coordination::ZooKeeperResponsePtr processImpl(KeeperStorage & storage, int64_t zxid) const
+    Coordination::ZooKeeperResponsePtr processImpl(Storage & storage, int64_t zxid) const
     {
-        Coordination::ZooKeeperResponsePtr response_ptr = zk_request->makeResponse();
+        Coordination::ZooKeeperResponsePtr response_ptr = this->zk_request->makeResponse();
         Coordination::ZooKeeperCheckResponse & response = dynamic_cast<Coordination::ZooKeeperCheckResponse &>(*response_ptr);
-        Coordination::ZooKeeperCheckRequest & request = dynamic_cast<Coordination::ZooKeeperCheckRequest &>(*zk_request);
+        Coordination::ZooKeeperCheckRequest & request = dynamic_cast<Coordination::ZooKeeperCheckRequest &>(*this->zk_request);
 
         if constexpr (!local)
         {
@@ -1585,12 +1621,12 @@ struct KeeperStorageCheckRequestProcessor final : public KeeperStorageRequestPro
         return response_ptr;
     }
 
-    Coordination::ZooKeeperResponsePtr process(KeeperStorage & storage, int64_t zxid) const override
+    Coordination::ZooKeeperResponsePtr process(Storage & storage, int64_t zxid) const override
     {
         return processImpl<false>(storage, zxid);
     }
 
-    Coordination::ZooKeeperResponsePtr processLocal(KeeperStorage & storage, int64_t zxid) const override
+    Coordination::ZooKeeperResponsePtr processLocal(Storage & storage, int64_t zxid) const override
     {
         ProfileEvents::increment(ProfileEvents::KeeperCheckRequest);
         return processImpl<true>(storage, zxid);
@@ -1601,55 +1637,56 @@ private:
 };
 
 
-struct KeeperStorageSetACLRequestProcessor final : public KeeperStorageRequestProcessor
+template<typename Storage>
+struct KeeperStorageSetACLRequestProcessor final : public KeeperStorageRequestProcessor<Storage>
 {
-    bool checkAuth(KeeperStorage & storage, int64_t session_id, bool is_local) const override
+    bool checkAuth(Storage & storage, int64_t session_id, bool is_local) const override
     {
-        return storage.checkACL(zk_request->getPath(), Coordination::ACL::Admin, session_id, is_local);
+        return storage.checkACL(this->zk_request->getPath(), Coordination::ACL::Admin, session_id, is_local);
     }
 
-    using KeeperStorageRequestProcessor::KeeperStorageRequestProcessor;
+    using KeeperStorageRequestProcessor<Storage>::KeeperStorageRequestProcessor;
 
-    std::vector<KeeperStorage::Delta>
-    preprocess(KeeperStorage & storage, int64_t zxid, int64_t session_id, int64_t /*time*/, uint64_t & digest, const KeeperContext & keeper_context) const override
+    std::vector<typename Storage::Delta>
+    preprocess(Storage & storage, int64_t zxid, int64_t session_id, int64_t /*time*/, uint64_t & digest, const KeeperContext & keeper_context) const override
     {
-        Coordination::ZooKeeperSetACLRequest & request = dynamic_cast<Coordination::ZooKeeperSetACLRequest &>(*zk_request);
+        Coordination::ZooKeeperSetACLRequest & request = dynamic_cast<Coordination::ZooKeeperSetACLRequest &>(*this->zk_request);
 
         if (Coordination::matchPath(request.path, keeper_system_path) != Coordination::PathMatchResult::NOT_MATCH)
         {
             auto error_msg = fmt::format("Trying to update an internal Keeper path ({}) which is not allowed", request.path);
 
             handleSystemNodeModification(keeper_context, error_msg);
-            return {KeeperStorage::Delta{zxid, Coordination::Error::ZBADARGUMENTS}};
+            return {typename Storage::Delta{zxid, Coordination::Error::ZBADARGUMENTS}};
         }
 
         auto & uncommitted_state = storage.uncommitted_state;
         if (!uncommitted_state.getNode(request.path))
-            return {KeeperStorage::Delta{zxid, Coordination::Error::ZNONODE}};
+            return {typename Storage::Delta{zxid, Coordination::Error::ZNONODE}};
 
         auto node = uncommitted_state.getNode(request.path);
 
         if (request.version != -1 && request.version != node->stat.aversion)
-            return {KeeperStorage::Delta{zxid, Coordination::Error::ZBADVERSION}};
+            return {typename Storage::Delta{zxid, Coordination::Error::ZBADVERSION}};
 
 
         Coordination::ACLs node_acls;
         if (!fixupACL(request.acls, session_id, uncommitted_state, node_acls))
-            return {KeeperStorage::Delta{zxid, Coordination::Error::ZINVALIDACL}};
+            return {typename Storage::Delta{zxid, Coordination::Error::ZINVALIDACL}};
 
-        std::vector<KeeperStorage::Delta> new_deltas
+        std::vector<typename Storage::Delta> new_deltas
         {
             {
                 request.path,
                 zxid,
-                KeeperStorage::SetACLDelta{std::move(node_acls), request.version}
+                typename Storage::SetACLDelta{std::move(node_acls), request.version}
             },
             {
                 request.path,
                 zxid,
-                KeeperStorage::UpdateNodeDelta
+                typename Storage::UpdateNodeDelta
                 {
-                    [](KeeperStorage::Node & n) { ++n.stat.aversion; }
+                    [](typename Storage::Node & n) { ++n.stat.aversion; }
                 }
             }
         };
@@ -1659,11 +1696,11 @@ struct KeeperStorageSetACLRequestProcessor final : public KeeperStorageRequestPr
         return new_deltas;
     }
 
-    Coordination::ZooKeeperResponsePtr process(KeeperStorage & storage, int64_t zxid) const override
+    Coordination::ZooKeeperResponsePtr process(Storage & storage, int64_t zxid) const override
     {
-        Coordination::ZooKeeperResponsePtr response_ptr = zk_request->makeResponse();
+        Coordination::ZooKeeperResponsePtr response_ptr = this->zk_request->makeResponse();
         Coordination::ZooKeeperSetACLResponse & response = dynamic_cast<Coordination::ZooKeeperSetACLResponse &>(*response_ptr);
-        Coordination::ZooKeeperSetACLRequest & request = dynamic_cast<Coordination::ZooKeeperSetACLRequest &>(*zk_request);
+        Coordination::ZooKeeperSetACLRequest & request = dynamic_cast<Coordination::ZooKeeperSetACLRequest &>(*this->zk_request);
 
         if (const auto result = storage.commit(zxid); result != Coordination::Error::ZOK)
         {
@@ -1681,32 +1718,33 @@ struct KeeperStorageSetACLRequestProcessor final : public KeeperStorageRequestPr
     }
 };
 
-struct KeeperStorageGetACLRequestProcessor final : public KeeperStorageRequestProcessor
+template<typename Storage>
+struct KeeperStorageGetACLRequestProcessor final : public KeeperStorageRequestProcessor<Storage>
 {
-    bool checkAuth(KeeperStorage & storage, int64_t session_id, bool is_local) const override
+    bool checkAuth(Storage & storage, int64_t session_id, bool is_local) const override
     {
-        return storage.checkACL(zk_request->getPath(), Coordination::ACL::Admin | Coordination::ACL::Read, session_id, is_local);
+        return storage.checkACL(this->zk_request->getPath(), Coordination::ACL::Admin | Coordination::ACL::Read, session_id, is_local);
     }
 
-    using KeeperStorageRequestProcessor::KeeperStorageRequestProcessor;
+    using KeeperStorageRequestProcessor<Storage>::KeeperStorageRequestProcessor;
 
-    std::vector<KeeperStorage::Delta>
-    preprocess(KeeperStorage & storage, int64_t zxid, int64_t /*session_id*/, int64_t /*time*/, uint64_t & /*digest*/, const KeeperContext & /*keeper_context*/) const override
+    std::vector<typename Storage::Delta>
+    preprocess(Storage & storage, int64_t zxid, int64_t /*session_id*/, int64_t /*time*/, uint64_t & /*digest*/, const KeeperContext & /*keeper_context*/) const override
     {
-        Coordination::ZooKeeperGetACLRequest & request = dynamic_cast<Coordination::ZooKeeperGetACLRequest &>(*zk_request);
+        Coordination::ZooKeeperGetACLRequest & request = dynamic_cast<Coordination::ZooKeeperGetACLRequest &>(*this->zk_request);
 
         if (!storage.uncommitted_state.getNode(request.path))
-            return {KeeperStorage::Delta{zxid, Coordination::Error::ZNONODE}};
+            return {typename Storage::Delta{zxid, Coordination::Error::ZNONODE}};
 
         return {};
     }
 
     template <bool local>
-    Coordination::ZooKeeperResponsePtr processImpl(KeeperStorage & storage, int64_t zxid) const
+    Coordination::ZooKeeperResponsePtr processImpl(Storage & storage, int64_t zxid) const
     {
-        Coordination::ZooKeeperResponsePtr response_ptr = zk_request->makeResponse();
+        Coordination::ZooKeeperResponsePtr response_ptr = this->zk_request->makeResponse();
         Coordination::ZooKeeperGetACLResponse & response = dynamic_cast<Coordination::ZooKeeperGetACLResponse &>(*response_ptr);
-        Coordination::ZooKeeperGetACLRequest & request = dynamic_cast<Coordination::ZooKeeperGetACLRequest &>(*zk_request);
+        Coordination::ZooKeeperGetACLRequest & request = dynamic_cast<Coordination::ZooKeeperGetACLRequest &>(*this->zk_request);
 
         if constexpr (!local)
         {
@@ -1735,23 +1773,24 @@ struct KeeperStorageGetACLRequestProcessor final : public KeeperStorageRequestPr
         return response_ptr;
     }
 
-    Coordination::ZooKeeperResponsePtr process(KeeperStorage & storage, int64_t zxid) const override
+    Coordination::ZooKeeperResponsePtr process(Storage & storage, int64_t zxid) const override
     {
         return processImpl<false>(storage, zxid);
     }
 
-    Coordination::ZooKeeperResponsePtr processLocal(KeeperStorage & storage, int64_t zxid) const override
+    Coordination::ZooKeeperResponsePtr processLocal(Storage & storage, int64_t zxid) const override
     {
         return processImpl<true>(storage, zxid);
     }
 };
 
-struct KeeperStorageMultiRequestProcessor final : public KeeperStorageRequestProcessor
+template<typename Storage>
+struct KeeperStorageMultiRequestProcessor final : public KeeperStorageRequestProcessor<Storage>
 {
     using OperationType = Coordination::ZooKeeperMultiRequest::OperationType;
     std::optional<OperationType> operation_type;
 
-    bool checkAuth(KeeperStorage & storage, int64_t session_id, bool is_local) const override
+    bool checkAuth(Storage & storage, int64_t session_id, bool is_local) const override
     {
         for (const auto & concrete_request : concrete_requests)
             if (!concrete_request->checkAuth(storage, session_id, is_local))
@@ -1759,11 +1798,11 @@ struct KeeperStorageMultiRequestProcessor final : public KeeperStorageRequestPro
         return true;
     }
 
-    std::vector<KeeperStorageRequestProcessorPtr> concrete_requests;
+    std::vector<std::shared_ptr<KeeperStorageRequestProcessor<Storage>>> concrete_requests;
     explicit KeeperStorageMultiRequestProcessor(const Coordination::ZooKeeperRequestPtr & zk_request_)
-        : KeeperStorageRequestProcessor(zk_request_)
+        : KeeperStorageRequestProcessor<Storage>(zk_request_)
     {
-        Coordination::ZooKeeperMultiRequest & request = dynamic_cast<Coordination::ZooKeeperMultiRequest &>(*zk_request);
+        Coordination::ZooKeeperMultiRequest & request = dynamic_cast<Coordination::ZooKeeperMultiRequest &>(*this->zk_request);
         concrete_requests.reserve(request.requests.size());
 
         const auto check_operation_type = [&](OperationType type)
@@ -1781,34 +1820,34 @@ struct KeeperStorageMultiRequestProcessor final : public KeeperStorageRequestPro
                 case Coordination::OpNum::Create:
                 case Coordination::OpNum::CreateIfNotExists:
                     check_operation_type(OperationType::Write);
-                    concrete_requests.push_back(std::make_shared<KeeperStorageCreateRequestProcessor>(sub_zk_request));
+                    concrete_requests.push_back(std::make_shared<KeeperStorageCreateRequestProcessor<Storage>>(sub_zk_request));
                     break;
                 case Coordination::OpNum::Remove:
                     check_operation_type(OperationType::Write);
-                    concrete_requests.push_back(std::make_shared<KeeperStorageRemoveRequestProcessor>(sub_zk_request));
+                    concrete_requests.push_back(std::make_shared<KeeperStorageRemoveRequestProcessor<Storage>>(sub_zk_request));
                     break;
                 case Coordination::OpNum::Set:
                     check_operation_type(OperationType::Write);
-                    concrete_requests.push_back(std::make_shared<KeeperStorageSetRequestProcessor>(sub_zk_request));
+                    concrete_requests.push_back(std::make_shared<KeeperStorageSetRequestProcessor<Storage>>(sub_zk_request));
                     break;
                 case Coordination::OpNum::Check:
                 case Coordination::OpNum::CheckNotExists:
                     check_operation_type(OperationType::Write);
-                    concrete_requests.push_back(std::make_shared<KeeperStorageCheckRequestProcessor>(sub_zk_request));
+                    concrete_requests.push_back(std::make_shared<KeeperStorageCheckRequestProcessor<Storage>>(sub_zk_request));
                     break;
                 case Coordination::OpNum::Get:
                     check_operation_type(OperationType::Read);
-                    concrete_requests.push_back(std::make_shared<KeeperStorageGetRequestProcessor>(sub_zk_request));
+                    concrete_requests.push_back(std::make_shared<KeeperStorageGetRequestProcessor<Storage>>(sub_zk_request));
                     break;
                 case Coordination::OpNum::Exists:
                     check_operation_type(OperationType::Read);
-                    concrete_requests.push_back(std::make_shared<KeeperStorageExistsRequestProcessor>(sub_zk_request));
+                    concrete_requests.push_back(std::make_shared<KeeperStorageExistsRequestProcessor<Storage>>(sub_zk_request));
                     break;
                 case Coordination::OpNum::List:
                 case Coordination::OpNum::FilteredList:
                 case Coordination::OpNum::SimpleList:
                     check_operation_type(OperationType::Read);
-                    concrete_requests.push_back(std::make_shared<KeeperStorageListRequestProcessor>(sub_zk_request));
+                    concrete_requests.push_back(std::make_shared<KeeperStorageListRequestProcessor<Storage>>(sub_zk_request));
                     break;
                 default:
                     throw DB::Exception(
@@ -1821,8 +1860,8 @@ struct KeeperStorageMultiRequestProcessor final : public KeeperStorageRequestPro
         assert(request.requests.empty() || operation_type.has_value());
     }
 
-    std::vector<KeeperStorage::Delta>
-    preprocess(KeeperStorage & storage, int64_t zxid, int64_t session_id, int64_t time, uint64_t & digest, const KeeperContext & keeper_context) const override
+    std::vector<typename Storage::Delta>
+    preprocess(Storage & storage, int64_t zxid, int64_t session_id, int64_t time, uint64_t & digest, const KeeperContext & keeper_context) const override
     {
         ProfileEvents::increment(ProfileEvents::KeeperMultiRequest);
         std::vector<Coordination::Error> response_errors;
@@ -1834,7 +1873,7 @@ struct KeeperStorageMultiRequestProcessor final : public KeeperStorageRequestPro
 
             if (!new_deltas.empty())
             {
-                if (auto * error = std::get_if<KeeperStorage::ErrorDelta>(&new_deltas.back().operation);
+                if (auto * error = std::get_if<typename Storage::ErrorDelta>(&new_deltas.back().operation);
                     error && *operation_type == OperationType::Write)
                 {
                     storage.uncommitted_state.rollback(zxid);
@@ -1845,10 +1884,10 @@ struct KeeperStorageMultiRequestProcessor final : public KeeperStorageRequestPro
                         response_errors.push_back(Coordination::Error::ZRUNTIMEINCONSISTENCY);
                     }
 
-                    return {KeeperStorage::Delta{zxid, KeeperStorage::FailedMultiDelta{std::move(response_errors)}}};
+                    return {typename Storage::Delta{zxid, typename Storage::FailedMultiDelta{std::move(response_errors)}}};
                 }
             }
-            new_deltas.emplace_back(zxid, KeeperStorage::SubDeltaEnd{});
+            new_deltas.emplace_back(zxid, typename Storage::SubDeltaEnd{});
             response_errors.push_back(Coordination::Error::ZOK);
 
             // manually add deltas so that the result of previous request in the transaction is used in the next request
@@ -1860,15 +1899,15 @@ struct KeeperStorageMultiRequestProcessor final : public KeeperStorageRequestPro
         return {};
     }
 
-    Coordination::ZooKeeperResponsePtr process(KeeperStorage & storage, int64_t zxid) const override
+    Coordination::ZooKeeperResponsePtr process(Storage & storage, int64_t zxid) const override
     {
-        Coordination::ZooKeeperResponsePtr response_ptr = zk_request->makeResponse();
+        Coordination::ZooKeeperResponsePtr response_ptr = this->zk_request->makeResponse();
         Coordination::ZooKeeperMultiResponse & response = dynamic_cast<Coordination::ZooKeeperMultiResponse &>(*response_ptr);
 
         auto & deltas = storage.uncommitted_state.deltas;
         // the deltas will have at least SubDeltaEnd or FailedMultiDelta
         assert(!deltas.empty());
-        if (auto * failed_multi = std::get_if<KeeperStorage::FailedMultiDelta>(&deltas.front().operation))
+        if (auto * failed_multi = std::get_if<typename Storage::FailedMultiDelta>(&deltas.front().operation))
         {
             for (size_t i = 0; i < concrete_requests.size(); ++i)
             {
@@ -1890,10 +1929,10 @@ struct KeeperStorageMultiRequestProcessor final : public KeeperStorageRequestPro
         return response_ptr;
     }
 
-    Coordination::ZooKeeperResponsePtr processLocal(KeeperStorage & storage, int64_t zxid) const override
+    Coordination::ZooKeeperResponsePtr processLocal(Storage & storage, int64_t zxid) const override
     {
         ProfileEvents::increment(ProfileEvents::KeeperMultiReadRequest);
-        Coordination::ZooKeeperResponsePtr response_ptr = zk_request->makeResponse();
+        Coordination::ZooKeeperResponsePtr response_ptr = this->zk_request->makeResponse();
         Coordination::ZooKeeperMultiResponse & response = dynamic_cast<Coordination::ZooKeeperMultiResponse &>(*response_ptr);
 
         for (size_t i = 0; i < concrete_requests.size(); ++i)
@@ -1905,10 +1944,10 @@ struct KeeperStorageMultiRequestProcessor final : public KeeperStorageRequestPro
         return response_ptr;
     }
 
-    KeeperStorage::ResponsesForSessions
-    processWatches(KeeperStorage::Watches & watches, KeeperStorage::Watches & list_watches) const override
+    KeeperStorageBase::ResponsesForSessions
+    processWatches(typename Storage::Watches & watches, typename Storage::Watches & list_watches) const override
     {
-        KeeperStorage::ResponsesForSessions result;
+        typename Storage::ResponsesForSessions result;
         for (const auto & generic_request : concrete_requests)
         {
             auto responses = generic_request->processWatches(watches, list_watches);
@@ -1918,47 +1957,50 @@ struct KeeperStorageMultiRequestProcessor final : public KeeperStorageRequestPro
     }
 };
 
-struct KeeperStorageCloseRequestProcessor final : public KeeperStorageRequestProcessor
+template<typename Storage>
+struct KeeperStorageCloseRequestProcessor final : public KeeperStorageRequestProcessor<Storage>
 {
-    using KeeperStorageRequestProcessor::KeeperStorageRequestProcessor;
-    Coordination::ZooKeeperResponsePtr process(KeeperStorage &, int64_t) const override
+    using KeeperStorageRequestProcessor<Storage>::KeeperStorageRequestProcessor;
+    Coordination::ZooKeeperResponsePtr process(Storage &, int64_t) const override
     {
         throw DB::Exception(ErrorCodes::LOGICAL_ERROR, "Called process on close request");
     }
 };
 
-struct KeeperStorageAuthRequestProcessor final : public KeeperStorageRequestProcessor
+template<typename Storage>
+struct KeeperStorageAuthRequestProcessor final : public KeeperStorageRequestProcessor<Storage>
 {
-    using KeeperStorageRequestProcessor::KeeperStorageRequestProcessor;
-    std::vector<KeeperStorage::Delta>
-    preprocess(KeeperStorage & storage, int64_t zxid, int64_t session_id, int64_t /*time*/, uint64_t & /*digest*/, const KeeperContext & /*keeper_context*/) const override
+    using KeeperStorageRequestProcessor<Storage>::KeeperStorageRequestProcessor;
+
+    std::vector<typename Storage::Delta>
+    preprocess(Storage & storage, int64_t zxid, int64_t session_id, int64_t /*time*/, uint64_t & /*digest*/, const KeeperContext & /*keeper_context*/) const override
     {
-        Coordination::ZooKeeperAuthRequest & auth_request = dynamic_cast<Coordination::ZooKeeperAuthRequest &>(*zk_request);
-        Coordination::ZooKeeperResponsePtr response_ptr = zk_request->makeResponse();
+        Coordination::ZooKeeperAuthRequest & auth_request = dynamic_cast<Coordination::ZooKeeperAuthRequest &>(*this->zk_request);
+        Coordination::ZooKeeperResponsePtr response_ptr = this->zk_request->makeResponse();
 
         if (auth_request.scheme != "digest" || std::count(auth_request.data.begin(), auth_request.data.end(), ':') != 1)
-            return {KeeperStorage::Delta{zxid, Coordination::Error::ZAUTHFAILED}};
+            return {typename Storage::Delta{zxid, Coordination::Error::ZAUTHFAILED}};
 
-        std::vector<KeeperStorage::Delta> new_deltas;
-        auto auth_digest = KeeperStorage::generateDigest(auth_request.data);
+        std::vector<typename Storage::Delta> new_deltas;
+        auto auth_digest = Storage::generateDigest(auth_request.data);
         if (auth_digest == storage.superdigest)
         {
-            KeeperStorage::AuthID auth{"super", ""};
-            new_deltas.emplace_back(zxid, KeeperStorage::AddAuthDelta{session_id, std::move(auth)});
+            typename Storage::AuthID auth{"super", ""};
+            new_deltas.emplace_back(zxid, typename Storage::AddAuthDelta{session_id, std::move(auth)});
         }
         else
         {
-            KeeperStorage::AuthID new_auth{auth_request.scheme, auth_digest};
+            typename Storage::AuthID new_auth{auth_request.scheme, auth_digest};
             if (!storage.uncommitted_state.hasACL(session_id, false, [&](const auto & auth_id) { return new_auth == auth_id; }))
-                new_deltas.emplace_back(zxid, KeeperStorage::AddAuthDelta{session_id, std::move(new_auth)});
+                new_deltas.emplace_back(zxid, typename Storage::AddAuthDelta{session_id, std::move(new_auth)});
         }
 
         return new_deltas;
     }
 
-    Coordination::ZooKeeperResponsePtr process(KeeperStorage & storage, int64_t zxid) const override
+    Coordination::ZooKeeperResponsePtr process(Storage & storage, int64_t zxid) const override
     {
-        Coordination::ZooKeeperResponsePtr response_ptr = zk_request->makeResponse();
+        Coordination::ZooKeeperResponsePtr response_ptr = this->zk_request->makeResponse();
         Coordination::ZooKeeperAuthResponse & auth_response = dynamic_cast<Coordination::ZooKeeperAuthResponse &>(*response_ptr);
 
         if (const auto result = storage.commit(zxid); result != Coordination::Error::ZOK)
@@ -1968,7 +2010,8 @@ struct KeeperStorageAuthRequestProcessor final : public KeeperStorageRequestProc
     }
 };
 
-void KeeperStorage::finalize()
+template<typename Container>
+void KeeperStorage<Container>::finalize()
 {
     if (finalized)
         throw DB::Exception(ErrorCodes::LOGICAL_ERROR, "KeeperStorage already finalized");
@@ -1983,16 +2026,17 @@ void KeeperStorage::finalize()
     session_expiry_queue.clear();
 }
 
-bool KeeperStorage::isFinalized() const
+template<typename Container>
+bool KeeperStorage<Container>::isFinalized() const
 {
     return finalized;
 }
 
-
+template<typename Storage>
 class KeeperStorageRequestProcessorsFactory final : private boost::noncopyable
 {
 public:
-    using Creator = std::function<KeeperStorageRequestProcessorPtr(const Coordination::ZooKeeperRequestPtr &)>;
+    using Creator = std::function<std::shared_ptr<KeeperStorageRequestProcessor<Storage>>(const Coordination::ZooKeeperRequestPtr &)>;
     using OpNumToRequest = std::unordered_map<Coordination::OpNum, Creator>;
 
     static KeeperStorageRequestProcessorsFactory & instance()
@@ -2001,7 +2045,7 @@ public:
         return factory;
     }
 
-    KeeperStorageRequestProcessorPtr get(const Coordination::ZooKeeperRequestPtr & zk_request) const
+    std::shared_ptr<KeeperStorageRequestProcessor<Storage>> get(const Coordination::ZooKeeperRequestPtr & zk_request) const
     {
         auto request_it = op_num_to_request.find(zk_request->getOpNum());
         if (request_it == op_num_to_request.end())
@@ -2021,39 +2065,41 @@ private:
     KeeperStorageRequestProcessorsFactory();
 };
 
-template <Coordination::OpNum num, typename RequestT>
-void registerKeeperRequestProcessor(KeeperStorageRequestProcessorsFactory & factory)
+template <Coordination::OpNum num, typename RequestT, typename Factory>
+void registerKeeperRequestProcessor(Factory & factory)
 {
     factory.registerRequest(
         num, [](const Coordination::ZooKeeperRequestPtr & zk_request) { return std::make_shared<RequestT>(zk_request); });
 }
 
 
-KeeperStorageRequestProcessorsFactory::KeeperStorageRequestProcessorsFactory()
+template<typename Storage>
+KeeperStorageRequestProcessorsFactory<Storage>::KeeperStorageRequestProcessorsFactory()
 {
-    registerKeeperRequestProcessor<Coordination::OpNum::Heartbeat, KeeperStorageHeartbeatRequestProcessor>(*this);
-    registerKeeperRequestProcessor<Coordination::OpNum::Sync, KeeperStorageSyncRequestProcessor>(*this);
-    registerKeeperRequestProcessor<Coordination::OpNum::Auth, KeeperStorageAuthRequestProcessor>(*this);
-    registerKeeperRequestProcessor<Coordination::OpNum::Close, KeeperStorageCloseRequestProcessor>(*this);
-    registerKeeperRequestProcessor<Coordination::OpNum::Create, KeeperStorageCreateRequestProcessor>(*this);
-    registerKeeperRequestProcessor<Coordination::OpNum::Remove, KeeperStorageRemoveRequestProcessor>(*this);
-    registerKeeperRequestProcessor<Coordination::OpNum::Exists, KeeperStorageExistsRequestProcessor>(*this);
-    registerKeeperRequestProcessor<Coordination::OpNum::Get, KeeperStorageGetRequestProcessor>(*this);
-    registerKeeperRequestProcessor<Coordination::OpNum::Set, KeeperStorageSetRequestProcessor>(*this);
-    registerKeeperRequestProcessor<Coordination::OpNum::List, KeeperStorageListRequestProcessor>(*this);
-    registerKeeperRequestProcessor<Coordination::OpNum::SimpleList, KeeperStorageListRequestProcessor>(*this);
-    registerKeeperRequestProcessor<Coordination::OpNum::FilteredList, KeeperStorageListRequestProcessor>(*this);
-    registerKeeperRequestProcessor<Coordination::OpNum::Check, KeeperStorageCheckRequestProcessor>(*this);
-    registerKeeperRequestProcessor<Coordination::OpNum::Multi, KeeperStorageMultiRequestProcessor>(*this);
-    registerKeeperRequestProcessor<Coordination::OpNum::MultiRead, KeeperStorageMultiRequestProcessor>(*this);
-    registerKeeperRequestProcessor<Coordination::OpNum::CreateIfNotExists, KeeperStorageCreateRequestProcessor>(*this);
-    registerKeeperRequestProcessor<Coordination::OpNum::SetACL, KeeperStorageSetACLRequestProcessor>(*this);
-    registerKeeperRequestProcessor<Coordination::OpNum::GetACL, KeeperStorageGetACLRequestProcessor>(*this);
-    registerKeeperRequestProcessor<Coordination::OpNum::CheckNotExists, KeeperStorageCheckRequestProcessor>(*this);
+    registerKeeperRequestProcessor<Coordination::OpNum::Heartbeat, KeeperStorageHeartbeatRequestProcessor<Storage>>(*this);
+    registerKeeperRequestProcessor<Coordination::OpNum::Sync, KeeperStorageSyncRequestProcessor<Storage>>(*this);
+    registerKeeperRequestProcessor<Coordination::OpNum::Auth, KeeperStorageAuthRequestProcessor<Storage>>(*this);
+    registerKeeperRequestProcessor<Coordination::OpNum::Close, KeeperStorageCloseRequestProcessor<Storage>>(*this);
+    registerKeeperRequestProcessor<Coordination::OpNum::Create, KeeperStorageCreateRequestProcessor<Storage>>(*this);
+    registerKeeperRequestProcessor<Coordination::OpNum::Remove, KeeperStorageRemoveRequestProcessor<Storage>>(*this);
+    registerKeeperRequestProcessor<Coordination::OpNum::Exists, KeeperStorageExistsRequestProcessor<Storage>>(*this);
+    registerKeeperRequestProcessor<Coordination::OpNum::Get, KeeperStorageGetRequestProcessor<Storage>>(*this);
+    registerKeeperRequestProcessor<Coordination::OpNum::Set, KeeperStorageSetRequestProcessor<Storage>>(*this);
+    registerKeeperRequestProcessor<Coordination::OpNum::List, KeeperStorageListRequestProcessor<Storage>>(*this);
+    registerKeeperRequestProcessor<Coordination::OpNum::SimpleList, KeeperStorageListRequestProcessor<Storage>>(*this);
+    registerKeeperRequestProcessor<Coordination::OpNum::FilteredList, KeeperStorageListRequestProcessor<Storage>>(*this);
+    registerKeeperRequestProcessor<Coordination::OpNum::Check, KeeperStorageCheckRequestProcessor<Storage>>(*this);
+    registerKeeperRequestProcessor<Coordination::OpNum::Multi, KeeperStorageMultiRequestProcessor<Storage>>(*this);
+    registerKeeperRequestProcessor<Coordination::OpNum::MultiRead, KeeperStorageMultiRequestProcessor<Storage>>(*this);
+    registerKeeperRequestProcessor<Coordination::OpNum::CreateIfNotExists, KeeperStorageCreateRequestProcessor<Storage>>(*this);
+    registerKeeperRequestProcessor<Coordination::OpNum::SetACL, KeeperStorageSetACLRequestProcessor<Storage>>(*this);
+    registerKeeperRequestProcessor<Coordination::OpNum::GetACL, KeeperStorageGetACLRequestProcessor<Storage>>(*this);
+    registerKeeperRequestProcessor<Coordination::OpNum::CheckNotExists, KeeperStorageCheckRequestProcessor<Storage>>(*this);
 }
 
 
-UInt64 KeeperStorage::calculateNodesDigest(UInt64 current_digest, const std::vector<Delta> & new_deltas) const
+template<typename Container>
+UInt64 KeeperStorage<Container>::calculateNodesDigest(UInt64 current_digest, const std::vector<Delta> & new_deltas) const
 {
     if (!keeper_context->digestEnabled())
         return current_digest;
@@ -2088,7 +2134,7 @@ UInt64 KeeperStorage::calculateNodesDigest(UInt64 current_digest, const std::vec
                     auto updated_node_it = updated_nodes.find(delta.path);
                     if (updated_node_it == updated_nodes.end())
                     {
-                        node = std::make_shared<KeeperStorage::Node>();
+                        node = std::make_shared<Node>();
                         node->shallowCopy(*uncommitted_state.getNode(delta.path));
                         current_digest -= node->getDigest(delta.path);
                         updated_nodes.emplace(delta.path, node);
@@ -2114,7 +2160,8 @@ UInt64 KeeperStorage::calculateNodesDigest(UInt64 current_digest, const std::vec
     return current_digest;
 }
 
-void KeeperStorage::preprocessRequest(
+template<typename Container>
+void KeeperStorage<Container>::preprocessRequest(
     const Coordination::ZooKeeperRequestPtr & zk_request,
     int64_t session_id,
     int64_t time,
@@ -2173,7 +2220,7 @@ void KeeperStorage::preprocessRequest(
         uncommitted_state.addDeltas(std::move(new_deltas));
     });
 
-    KeeperStorageRequestProcessorPtr request_processor = KeeperStorageRequestProcessorsFactory::instance().get(zk_request);
+    auto request_processor = KeeperStorageRequestProcessorsFactory<KeeperStorage<Container>>::instance().get(zk_request);
 
     if (zk_request->getOpNum() == Coordination::OpNum::Close) /// Close request is special
     {
@@ -2215,7 +2262,8 @@ void KeeperStorage::preprocessRequest(
     new_deltas = request_processor->preprocess(*this, transaction.zxid, session_id, time, new_digest, *keeper_context);
 }
 
-KeeperStorage::ResponsesForSessions KeeperStorage::processRequest(
+template<typename Container>
+KeeperStorage<Container>::ResponsesForSessions KeeperStorage<Container>::processRequest(
     const Coordination::ZooKeeperRequestPtr & zk_request,
     int64_t session_id,
     std::optional<int64_t> new_last_zxid,
@@ -2242,7 +2290,7 @@ KeeperStorage::ResponsesForSessions KeeperStorage::processRequest(
         uncommitted_transactions.pop_front();
     }
 
-    KeeperStorage::ResponsesForSessions results;
+    ResponsesForSessions results;
 
     /// ZooKeeper update sessions expirity for each request, not only for heartbeats
     session_expiry_queue.addNewSessionOrUpdate(session_id, session_and_timeout[session_id]);
@@ -2280,7 +2328,7 @@ KeeperStorage::ResponsesForSessions KeeperStorage::processRequest(
     }
     else if (zk_request->getOpNum() == Coordination::OpNum::Heartbeat) /// Heartbeat request is also special
     {
-        KeeperStorageRequestProcessorPtr storage_request = KeeperStorageRequestProcessorsFactory::instance().get(zk_request);
+        auto storage_request = KeeperStorageRequestProcessorsFactory<KeeperStorage<Container>>::instance().get(zk_request);
         auto response = storage_request->process(*this, zxid);
         response->xid = zk_request->xid;
         response->zxid = getZXID();
@@ -2289,7 +2337,7 @@ KeeperStorage::ResponsesForSessions KeeperStorage::processRequest(
     }
     else /// normal requests proccession
     {
-        KeeperStorageRequestProcessorPtr request_processor = KeeperStorageRequestProcessorsFactory::instance().get(zk_request);
+        auto request_processor = KeeperStorageRequestProcessorsFactory<KeeperStorage<Container>>::instance().get(zk_request);
         Coordination::ZooKeeperResponsePtr response;
 
         if (is_local)
@@ -2352,7 +2400,8 @@ KeeperStorage::ResponsesForSessions KeeperStorage::processRequest(
     return results;
 }
 
-void KeeperStorage::rollbackRequest(int64_t rollback_zxid, bool allow_missing)
+template<typename Container>
+void KeeperStorage<Container>::rollbackRequest(int64_t rollback_zxid, bool allow_missing)
 {
     if (allow_missing && (uncommitted_transactions.empty() || uncommitted_transactions.back().zxid < rollback_zxid))
         return;
@@ -2378,7 +2427,8 @@ void KeeperStorage::rollbackRequest(int64_t rollback_zxid, bool allow_missing)
     }
 }
 
-KeeperStorage::Digest KeeperStorage::getNodesDigest(bool committed) const
+template<typename Container>
+KeeperStorageBase::Digest KeeperStorage<Container>::getNodesDigest(bool committed) const
 {
     if (!keeper_context->digestEnabled())
         return {.version = DigestVersion::NO_DIGEST};
@@ -2389,13 +2439,15 @@ KeeperStorage::Digest KeeperStorage::getNodesDigest(bool committed) const
     return uncommitted_transactions.back().nodes_digest;
 }
 
-void KeeperStorage::removeDigest(const Node & node, const std::string_view path)
+template<typename Container>
+void KeeperStorage<Container>::removeDigest(const Node & node, const std::string_view path)
 {
     if (keeper_context->digestEnabled())
         nodes_digest -= node.getDigest(path);
 }
 
-void KeeperStorage::addDigest(const Node & node, const std::string_view path)
+template<typename Container>
+void KeeperStorage<Container>::addDigest(const Node & node, const std::string_view path)
 {
     if (keeper_context->digestEnabled())
     {
@@ -2404,7 +2456,8 @@ void KeeperStorage::addDigest(const Node & node, const std::string_view path)
     }
 }
 
-void KeeperStorage::clearDeadWatches(int64_t session_id)
+template<typename Container>
+void KeeperStorage<Container>::clearDeadWatches(int64_t session_id)
 {
     /// Clear all watches for this session
     auto watches_it = sessions_and_watchers.find(session_id);
@@ -2437,7 +2490,8 @@ void KeeperStorage::clearDeadWatches(int64_t session_id)
     }
 }
 
-void KeeperStorage::dumpWatches(WriteBufferFromOwnString & buf) const
+template<typename Container>
+void KeeperStorage<Container>::dumpWatches(WriteBufferFromOwnString & buf) const
 {
     for (const auto & [session_id, watches_paths] : sessions_and_watchers)
     {
@@ -2447,7 +2501,8 @@ void KeeperStorage::dumpWatches(WriteBufferFromOwnString & buf) const
     }
 }
 
-void KeeperStorage::dumpWatchesByPath(WriteBufferFromOwnString & buf) const
+template<typename Container>
+void KeeperStorage<Container>::dumpWatchesByPath(WriteBufferFromOwnString & buf) const
 {
     auto write_int_container = [&buf](const auto & session_ids)
     {
@@ -2470,7 +2525,8 @@ void KeeperStorage::dumpWatchesByPath(WriteBufferFromOwnString & buf) const
     }
 }
 
-void KeeperStorage::dumpSessionsAndEphemerals(WriteBufferFromOwnString & buf) const
+template<typename Container>
+void KeeperStorage<Container>::dumpSessionsAndEphemerals(WriteBufferFromOwnString & buf) const
 {
     auto write_str_set = [&buf](const std::unordered_set<String> & ephemeral_paths)
     {
@@ -2495,7 +2551,8 @@ void KeeperStorage::dumpSessionsAndEphemerals(WriteBufferFromOwnString & buf) co
     }
 }
 
-uint64_t KeeperStorage::getTotalWatchesCount() const
+template<typename Container>
+uint64_t KeeperStorage<Container>::getTotalWatchesCount() const
 {
     uint64_t ret = 0;
     for (const auto & [session, paths] : sessions_and_watchers)
@@ -2504,12 +2561,14 @@ uint64_t KeeperStorage::getTotalWatchesCount() const
     return ret;
 }
 
-uint64_t KeeperStorage::getSessionsWithWatchesCount() const
+template<typename Container>
+uint64_t KeeperStorage<Container>::getSessionsWithWatchesCount() const
 {
     return sessions_and_watchers.size();
 }
 
-uint64_t KeeperStorage::getTotalEphemeralNodesCount() const
+template<typename Container>
+uint64_t KeeperStorage<Container>::getTotalEphemeralNodesCount() const
 {
     uint64_t ret = 0;
     for (const auto & [session_id, nodes] : ephemerals)
@@ -2518,17 +2577,20 @@ uint64_t KeeperStorage::getTotalEphemeralNodesCount() const
     return ret;
 }
 
-void KeeperStorage::recalculateStats()
+template<typename Container>
+void KeeperStorage<Container>::recalculateStats()
 {
     container.recalculateDataSize();
 }
 
-String KeeperStorage::generateDigest(const String & userdata)
+template<typename Container>
+String KeeperStorage<Container>::generateDigest(const String & userdata)
 {
     std::vector<String> user_password;
     boost::split(user_password, userdata, [](char character) { return character == ':'; });
     return user_password[0] + ":" + base64Encode(getSHA1(userdata));
 }
 
+template class KeeperStorage<SnapshotableHashTable<KeeperMemNode>>;
 
 }
diff --git a/src/Coordination/KeeperStorage.h b/src/Coordination/KeeperStorage.h
index ec5df74efb6..2eff9f6651b 100644
--- a/src/Coordination/KeeperStorage.h
+++ b/src/Coordination/KeeperStorage.h
@@ -16,62 +16,55 @@
 namespace DB
 {
 
-struct KeeperStorageRequestProcessor;
-using KeeperStorageRequestProcessorPtr = std::shared_ptr<KeeperStorageRequestProcessor>;
 using ResponseCallback = std::function<void(const Coordination::ZooKeeperResponsePtr &)>;
 using ChildrenSet = absl::flat_hash_set<StringRef, StringRefHash>;
 using SessionAndTimeout = std::unordered_map<int64_t, int64_t>;
 
-struct KeeperStorageSnapshot;
+struct KeeperMemNode
+{
+    uint64_t acl_id = 0; /// 0 -- no ACL by default
+    bool is_sequental = false;
+    Coordination::Stat stat{};
+    int32_t seq_num = 0;
+    uint64_t size_bytes; // save size to avoid calculate every time
 
-/// Keeper state machine almost equal to the ZooKeeper's state machine.
-/// Implements all logic of operations, data changes, sessions allocation.
-/// In-memory and not thread safe.
-class KeeperStorage
+    KeeperMemNode() : size_bytes(sizeof(KeeperMemNode)) { }
+
+    /// Object memory size
+    uint64_t sizeInBytes() const { return size_bytes; }
+
+    void setData(String new_data);
+
+    const auto & getData() const noexcept { return data; }
+
+    void addChild(StringRef child_path, bool update_size = true);
+
+    void removeChild(StringRef child_path);
+
+    const auto & getChildren() const noexcept { return children; }
+
+    // Invalidate the calculated digest so it's recalculated again on the next
+    // getDigest call
+    void invalidateDigestCache() const;
+
+    // get the calculated digest of the node
+    UInt64 getDigest(std::string_view path) const;
+
+    // copy only necessary information for preprocessing and digest calculation
+    // (e.g. we don't need to copy list of children)
+    void shallowCopy(const KeeperMemNode & other);
+
+    void recalculateSize();
+
+private:
+    String data;
+    ChildrenSet children{};
+    mutable std::optional<UInt64> cached_digest;
+};
+
+class KeeperStorageBase
 {
 public:
-    struct Node
-    {
-        uint64_t acl_id = 0; /// 0 -- no ACL by default
-        bool is_sequental = false;
-        Coordination::Stat stat{};
-        int32_t seq_num = 0;
-        uint64_t size_bytes; // save size to avoid calculate every time
-
-        Node() : size_bytes(sizeof(Node)) { }
-
-        /// Object memory size
-        uint64_t sizeInBytes() const { return size_bytes; }
-
-        void setData(String new_data);
-
-        const auto & getData() const noexcept { return data; }
-
-        void addChild(StringRef child_path, bool update_size = true);
-
-        void removeChild(StringRef child_path);
-
-        const auto & getChildren() const noexcept { return children; }
-
-        // Invalidate the calculated digest so it's recalculated again on the next
-        // getDigest call
-        void invalidateDigestCache() const;
-
-        // get the calculated digest of the node
-        UInt64 getDigest(std::string_view path) const;
-
-        // copy only necessary information for preprocessing and digest calculation
-        // (e.g. we don't need to copy list of children)
-        void shallowCopy(const Node & other);
-
-        void recalculateSize();
-
-    private:
-        String data;
-        ChildrenSet children{};
-        mutable std::optional<UInt64> cached_digest;
-    };
-
     enum DigestVersion : uint8_t
     {
         NO_DIGEST = 0,
@@ -79,7 +72,11 @@ public:
         V2 = 2  // added system nodes that modify the digest on startup so digest from V0 is invalid
     };
 
-    static constexpr auto CURRENT_DIGEST_VERSION = DigestVersion::V2;
+    struct Digest
+    {
+        DigestVersion version{DigestVersion::NO_DIGEST};
+        uint64_t value{0};
+    };
 
     struct ResponseForSession
     {
@@ -88,11 +85,34 @@ public:
     };
     using ResponsesForSessions = std::vector<ResponseForSession>;
 
-    struct Digest
+    struct RequestForSession
     {
-        DigestVersion version{DigestVersion::NO_DIGEST};
-        uint64_t value{0};
+        int64_t session_id;
+        int64_t time{0};
+        Coordination::ZooKeeperRequestPtr request;
+        int64_t zxid{0};
+        std::optional<Digest> digest;
+        int64_t log_idx{0};
     };
+    using RequestsForSessions = std::vector<RequestForSession>;
+
+    struct AuthID
+    {
+        std::string scheme;
+        std::string id;
+
+        bool operator==(const AuthID & other) const { return scheme == other.scheme && id == other.id; }
+    };
+
+    // using Container = SnapshotableHashTable<Node>;
+    using Ephemerals = std::unordered_map<int64_t, std::unordered_set<std::string>>;
+    using SessionAndWatcher = std::unordered_map<int64_t, std::unordered_set<std::string>>;
+    using SessionIDs = std::unordered_set<int64_t>;
+
+    /// Just vector of SHA1 from user:password
+    using AuthIDs = std::vector<AuthID>;
+    using SessionAndAuth = std::unordered_map<int64_t, AuthIDs>;
+    using Watches = std::unordered_map<String /* path, relative of root_path */, SessionIDs>;
 
     static bool checkDigest(const Digest & first, const Digest & second)
     {
@@ -105,38 +125,22 @@ public:
         return first.value == second.value;
     }
 
+};
+
+/// Keeper state machine almost equal to the ZooKeeper's state machine.
+/// Implements all logic of operations, data changes, sessions allocation.
+/// In-memory and not thread safe.
+template<typename Container_>
+class KeeperStorage : public KeeperStorageBase
+{
+public:
+    using Container = Container_;
+    using Node = Container::Node;
+
+    static constexpr auto CURRENT_DIGEST_VERSION = DigestVersion::V2;
+
     static String generateDigest(const String & userdata);
 
-    struct RequestForSession
-    {
-        int64_t session_id;
-        int64_t time{0};
-        Coordination::ZooKeeperRequestPtr request;
-        int64_t zxid{0};
-        std::optional<Digest> digest;
-        int64_t log_idx{0};
-    };
-
-    struct AuthID
-    {
-        std::string scheme;
-        std::string id;
-
-        bool operator==(const AuthID & other) const { return scheme == other.scheme && id == other.id; }
-    };
-
-    using RequestsForSessions = std::vector<RequestForSession>;
-
-    using Container = SnapshotableHashTable<Node>;
-    using Ephemerals = std::unordered_map<int64_t, std::unordered_set<std::string>>;
-    using SessionAndWatcher = std::unordered_map<int64_t, std::unordered_set<std::string>>;
-    using SessionIDs = std::unordered_set<int64_t>;
-
-    /// Just vector of SHA1 from user:password
-    using AuthIDs = std::vector<AuthID>;
-    using SessionAndAuth = std::unordered_map<int64_t, AuthIDs>;
-    using Watches = std::unordered_map<String /* path, relative of root_path */, SessionIDs>;
-
     int64_t session_id_counter{1};
 
     SessionAndAuth session_and_auth;
@@ -307,7 +311,7 @@ public:
         std::unordered_map<std::string, std::list<const Delta *>, Hash, Equal> deltas_for_path;
 
         std::list<Delta> deltas;
-        KeeperStorage & storage;
+        KeeperStorage<Container> & storage;
     };
 
     UncommittedState uncommitted_state{*this};
@@ -487,6 +491,6 @@ private:
     void addDigest(const Node & node, std::string_view path);
 };
 
-using KeeperStoragePtr = std::unique_ptr<KeeperStorage>;
+using KeeperMemoryStorage = KeeperStorage<SnapshotableHashTable<KeeperMemNode>>;
 
 }
diff --git a/src/Coordination/SnapshotableHashTable.h b/src/Coordination/SnapshotableHashTable.h
index 093126237ef..2bdef189e6b 100644
--- a/src/Coordination/SnapshotableHashTable.h
+++ b/src/Coordination/SnapshotableHashTable.h
@@ -121,6 +121,7 @@ private:
 
 public:
 
+    using Node = V;
     using iterator = typename List::iterator;
     using const_iterator = typename List::const_iterator;
     using ValueUpdater = std::function<void(V & value)>;
diff --git a/src/Coordination/ZooKeeperDataReader.cpp b/src/Coordination/ZooKeeperDataReader.cpp
index 3c1550f08c8..e783582af23 100644
--- a/src/Coordination/ZooKeeperDataReader.cpp
+++ b/src/Coordination/ZooKeeperDataReader.cpp
@@ -43,7 +43,8 @@ void deserializeSnapshotMagic(ReadBuffer & in)
         throw Exception(ErrorCodes::CORRUPTED_DATA, "Incorrect magic header in file, expected {}, got {}", SNP_HEADER, magic_header);
 }
 
-int64_t deserializeSessionAndTimeout(KeeperStorage & storage, ReadBuffer & in)
+template<typename Storage>
+int64_t deserializeSessionAndTimeout(Storage & storage, ReadBuffer & in)
 {
     int32_t count;
     Coordination::read(count, in);
@@ -62,7 +63,8 @@ int64_t deserializeSessionAndTimeout(KeeperStorage & storage, ReadBuffer & in)
     return max_session_id;
 }
 
-void deserializeACLMap(KeeperStorage & storage, ReadBuffer & in)
+template<typename Storage>
+void deserializeACLMap(Storage & storage, ReadBuffer & in)
 {
     int32_t count;
     Coordination::read(count, in);
@@ -90,7 +92,8 @@ void deserializeACLMap(KeeperStorage & storage, ReadBuffer & in)
     }
 }
 
-int64_t deserializeStorageData(KeeperStorage & storage, ReadBuffer & in, Poco::Logger * log)
+template<typename Storage>
+int64_t deserializeStorageData(Storage & storage, ReadBuffer & in, Poco::Logger * log)
 {
     int64_t max_zxid = 0;
     std::string path;
@@ -98,7 +101,7 @@ int64_t deserializeStorageData(KeeperStorage & storage, ReadBuffer & in, Poco::L
     size_t count = 0;
     while (path != "/")
     {
-        KeeperStorage::Node node{};
+        typename Storage::Node node{};
         String data;
         Coordination::read(data, in);
         node.setData(std::move(data));
@@ -140,14 +143,15 @@ int64_t deserializeStorageData(KeeperStorage & storage, ReadBuffer & in, Poco::L
         if (itr.key != "/")
         {
             auto parent_path = parentNodePath(itr.key);
-            storage.container.updateValue(parent_path, [my_path = itr.key] (KeeperStorage::Node & value) { value.addChild(getBaseNodeName(my_path)); ++value.stat.numChildren; });
+            storage.container.updateValue(parent_path, [my_path = itr.key] (typename Storage::Node & value) { value.addChild(getBaseNodeName(my_path)); ++value.stat.numChildren; });
         }
     }
 
     return max_zxid;
 }
 
-void deserializeKeeperStorageFromSnapshot(KeeperStorage & storage, const std::string & snapshot_path, Poco::Logger * log)
+template<typename Storage>
+void deserializeKeeperStorageFromSnapshot(Storage & storage, const std::string & snapshot_path, Poco::Logger * log)
 {
     LOG_INFO(log, "Deserializing storage snapshot {}", snapshot_path);
     int64_t zxid = getZxidFromName(snapshot_path);
@@ -186,7 +190,8 @@ void deserializeKeeperStorageFromSnapshot(KeeperStorage & storage, const std::st
     LOG_INFO(log, "Finished, snapshot ZXID {}", storage.zxid);
 }
 
-void deserializeKeeperStorageFromSnapshotsDir(KeeperStorage & storage, const std::string & path, Poco::Logger * log)
+template<typename Storage>
+void deserializeKeeperStorageFromSnapshotsDir(Storage & storage, const std::string & path, Poco::Logger * log)
 {
     namespace fs = std::filesystem;
     std::map<int64_t, std::string> existing_snapshots;
@@ -474,7 +479,8 @@ bool hasErrorsInMultiRequest(Coordination::ZooKeeperRequestPtr request)
 
 }
 
-bool deserializeTxn(KeeperStorage & storage, ReadBuffer & in, Poco::Logger * /*log*/)
+template<typename Storage>
+bool deserializeTxn(Storage & storage, ReadBuffer & in, Poco::Logger * /*log*/)
 {
     int64_t checksum;
     Coordination::read(checksum, in);
@@ -529,7 +535,8 @@ bool deserializeTxn(KeeperStorage & storage, ReadBuffer & in, Poco::Logger * /*l
     return true;
 }
 
-void deserializeLogAndApplyToStorage(KeeperStorage & storage, const std::string & log_path, Poco::Logger * log)
+template<typename Storage>
+void deserializeLogAndApplyToStorage(Storage & storage, const std::string & log_path, Poco::Logger * log)
 {
     ReadBufferFromFile reader(log_path);
 
@@ -553,7 +560,8 @@ void deserializeLogAndApplyToStorage(KeeperStorage & storage, const std::string
     LOG_INFO(log, "Finished {} deserialization, totally read {} records", log_path, counter);
 }
 
-void deserializeLogsAndApplyToStorage(KeeperStorage & storage, const std::string & path, Poco::Logger * log)
+template<typename Storage>
+void deserializeLogsAndApplyToStorage(Storage & storage, const std::string & path, Poco::Logger * log)
 {
     namespace fs = std::filesystem;
     std::map<int64_t, std::string> existing_logs;
@@ -589,4 +597,9 @@ void deserializeLogsAndApplyToStorage(KeeperStorage & storage, const std::string
     }
 }
 
+template void deserializeKeeperStorageFromSnapshot<KeeperMemoryStorage>(KeeperMemoryStorage & storage, const std::string & snapshot_path, Poco::Logger * log);
+template void deserializeKeeperStorageFromSnapshotsDir<KeeperMemoryStorage>(KeeperMemoryStorage & storage, const std::string & path, Poco::Logger * log);
+template void deserializeLogAndApplyToStorage<KeeperMemoryStorage>(KeeperMemoryStorage & storage, const std::string & log_path, Poco::Logger * log);
+template void deserializeLogsAndApplyToStorage<KeeperMemoryStorage>(KeeperMemoryStorage & storage, const std::string & path, Poco::Logger * log);
+
 }
diff --git a/src/Coordination/ZooKeeperDataReader.h b/src/Coordination/ZooKeeperDataReader.h
index 8fd86ba99e2..928e560926b 100644
--- a/src/Coordination/ZooKeeperDataReader.h
+++ b/src/Coordination/ZooKeeperDataReader.h
@@ -5,12 +5,16 @@
 namespace DB
 {
 
-void deserializeKeeperStorageFromSnapshot(KeeperStorage & storage, const std::string & snapshot_path, Poco::Logger * log);
+template<typename Storage>
+void deserializeKeeperStorageFromSnapshot(Storage & storage, const std::string & snapshot_path, Poco::Logger * log);
 
-void deserializeKeeperStorageFromSnapshotsDir(KeeperStorage & storage, const std::string & path, Poco::Logger * log);
+template<typename Storage>
+void deserializeKeeperStorageFromSnapshotsDir(Storage & storage, const std::string & path, Poco::Logger * log);
 
-void deserializeLogAndApplyToStorage(KeeperStorage & storage, const std::string & log_path, Poco::Logger * log);
+template<typename Storage>
+void deserializeLogAndApplyToStorage(Storage & storage, const std::string & log_path, Poco::Logger * log);
 
-void deserializeLogsAndApplyToStorage(KeeperStorage & storage, const std::string & path, Poco::Logger * log);
+template<typename Storage>
+void deserializeLogsAndApplyToStorage(Storage & storage, const std::string & path, Poco::Logger * log);
 
 }
diff --git a/src/Coordination/tests/gtest_coordination.cpp b/src/Coordination/tests/gtest_coordination.cpp
index 418b5225fa4..517b55af866 100644
--- a/src/Coordination/tests/gtest_coordination.cpp
+++ b/src/Coordination/tests/gtest_coordination.cpp
@@ -1320,7 +1320,7 @@ TEST_P(CoordinationTest, SnapshotableHashMapDataSize)
     EXPECT_EQ(hello.getApproximateDataSize(), 0);
 
     /// Node
-    using Node = DB::KeeperStorage::Node;
+    using Node = DB::KeeperMemoryStorage::Node;
     DB::SnapshotableHashTable<Node> world;
     Node n1;
     n1.setData("1234");
@@ -1359,9 +1359,9 @@ TEST_P(CoordinationTest, SnapshotableHashMapDataSize)
     EXPECT_EQ(world.getApproximateDataSize(), 0);
 }
 
-void addNode(DB::KeeperStorage & storage, const std::string & path, const std::string & data, int64_t ephemeral_owner = 0)
+void addNode(DB::KeeperMemoryStorage & storage, const std::string & path, const std::string & data, int64_t ephemeral_owner = 0)
 {
-    using Node = DB::KeeperStorage::Node;
+    using Node = DB::KeeperMemoryStorage::Node;
     Node node{};
     node.setData(data);
     node.stat.ephemeralOwner = ephemeral_owner;
@@ -1383,9 +1383,9 @@ TEST_P(CoordinationTest, TestStorageSnapshotSimple)
     ChangelogDirTest test("./snapshots");
     setSnapshotDirectory("./snapshots");
 
-    DB::KeeperSnapshotManager manager(3, keeper_context, params.enable_compression);
+    DB::KeeperSnapshotManager<DB::KeeperMemoryStorage> manager(3, keeper_context, params.enable_compression);
 
-    DB::KeeperStorage storage(500, "", keeper_context);
+    DB::KeeperMemoryStorage storage(500, "", keeper_context);
     addNode(storage, "/hello", "world", 1);
     addNode(storage, "/hello/somepath", "somedata", 3);
     storage.session_id_counter = 5;
@@ -1395,7 +1395,7 @@ TEST_P(CoordinationTest, TestStorageSnapshotSimple)
     storage.getSessionID(130);
     storage.getSessionID(130);
 
-    DB::KeeperStorageSnapshot snapshot(&storage, 2);
+    DB::KeeperStorageSnapshot<DB::KeeperMemoryStorage> snapshot(&storage, 2);
 
     EXPECT_EQ(snapshot.snapshot_meta->get_last_log_idx(), 2);
     EXPECT_EQ(snapshot.session_id, 7);
@@ -1433,9 +1433,9 @@ TEST_P(CoordinationTest, TestStorageSnapshotMoreWrites)
     ChangelogDirTest test("./snapshots");
     setSnapshotDirectory("./snapshots");
 
-    DB::KeeperSnapshotManager manager(3, keeper_context, params.enable_compression);
+    DB::KeeperSnapshotManager<DB::KeeperMemoryStorage> manager(3, keeper_context, params.enable_compression);
 
-    DB::KeeperStorage storage(500, "", keeper_context);
+    DB::KeeperMemoryStorage storage(500, "", keeper_context);
     storage.getSessionID(130);
 
     for (size_t i = 0; i < 50; ++i)
@@ -1443,7 +1443,7 @@ TEST_P(CoordinationTest, TestStorageSnapshotMoreWrites)
         addNode(storage, "/hello_" + std::to_string(i), "world_" + std::to_string(i));
     }
 
-    DB::KeeperStorageSnapshot snapshot(&storage, 50);
+    DB::KeeperStorageSnapshot<DB::KeeperMemoryStorage> snapshot(&storage, 50);
     EXPECT_EQ(snapshot.snapshot_meta->get_last_log_idx(), 50);
     EXPECT_EQ(snapshot.snapshot_container_size, 54);
 
@@ -1476,9 +1476,9 @@ TEST_P(CoordinationTest, TestStorageSnapshotManySnapshots)
     ChangelogDirTest test("./snapshots");
     setSnapshotDirectory("./snapshots");
 
-    DB::KeeperSnapshotManager manager(3, keeper_context, params.enable_compression);
+    DB::KeeperSnapshotManager<DB::KeeperMemoryStorage> manager(3, keeper_context, params.enable_compression);
 
-    DB::KeeperStorage storage(500, "", keeper_context);
+    DB::KeeperMemoryStorage storage(500, "", keeper_context);
     storage.getSessionID(130);
 
     for (size_t j = 1; j <= 5; ++j)
@@ -1488,7 +1488,7 @@ TEST_P(CoordinationTest, TestStorageSnapshotManySnapshots)
             addNode(storage, "/hello_" + std::to_string(i), "world_" + std::to_string(i));
         }
 
-        DB::KeeperStorageSnapshot snapshot(&storage, j * 50);
+        DB::KeeperStorageSnapshot<DB::KeeperMemoryStorage> snapshot(&storage, j * 50);
         auto buf = manager.serializeSnapshotToBuffer(snapshot);
         manager.serializeSnapshotBufferToDisk(*buf, j * 50);
         EXPECT_TRUE(fs::exists(std::string{"./snapshots/snapshot_"} + std::to_string(j * 50) + ".bin" + params.extension));
@@ -1517,15 +1517,15 @@ TEST_P(CoordinationTest, TestStorageSnapshotMode)
     ChangelogDirTest test("./snapshots");
     setSnapshotDirectory("./snapshots");
 
-    DB::KeeperSnapshotManager manager(3, keeper_context, params.enable_compression);
-    DB::KeeperStorage storage(500, "", keeper_context);
+    DB::KeeperSnapshotManager<DB::KeeperMemoryStorage> manager(3, keeper_context, params.enable_compression);
+    DB::KeeperMemoryStorage storage(500, "", keeper_context);
     for (size_t i = 0; i < 50; ++i)
     {
         addNode(storage, "/hello_" + std::to_string(i), "world_" + std::to_string(i));
     }
 
     {
-        DB::KeeperStorageSnapshot snapshot(&storage, 50);
+        DB::KeeperStorageSnapshot<DB::KeeperMemoryStorage> snapshot(&storage, 50);
         for (size_t i = 0; i < 50; ++i)
         {
             addNode(storage, "/hello_" + std::to_string(i), "wlrd_" + std::to_string(i));
@@ -1571,14 +1571,14 @@ TEST_P(CoordinationTest, TestStorageSnapshotBroken)
     ChangelogDirTest test("./snapshots");
     setSnapshotDirectory("./snapshots");
 
-    DB::KeeperSnapshotManager manager(3, keeper_context, params.enable_compression);
-    DB::KeeperStorage storage(500, "", keeper_context);
+    DB::KeeperSnapshotManager<DB::KeeperMemoryStorage> manager(3, keeper_context, params.enable_compression);
+    DB::KeeperMemoryStorage storage(500, "", keeper_context);
     for (size_t i = 0; i < 50; ++i)
     {
         addNode(storage, "/hello_" + std::to_string(i), "world_" + std::to_string(i));
     }
     {
-        DB::KeeperStorageSnapshot snapshot(&storage, 50);
+        DB::KeeperStorageSnapshot<DB::KeeperMemoryStorage> snapshot(&storage, 50);
         auto buf = manager.serializeSnapshotToBuffer(snapshot);
         manager.serializeSnapshotBufferToDisk(*buf, 50);
     }
@@ -1602,7 +1602,7 @@ nuraft::ptr<nuraft::buffer> getBufferFromZKRequest(int64_t session_id, int64_t z
     auto time = duration_cast<milliseconds>(system_clock::now().time_since_epoch()).count();
     DB::writeIntBinary(time, buf);
     DB::writeIntBinary(zxid, buf);
-    DB::writeIntBinary(DB::KeeperStorage::DigestVersion::NO_DIGEST, buf);
+    DB::writeIntBinary(DB::KeeperMemoryStorage::DigestVersion::NO_DIGEST, buf);
     return buf.getBuffer();
 }
 
@@ -1629,7 +1629,7 @@ void testLogAndStateMachine(
 
     ResponsesQueue queue(std::numeric_limits<size_t>::max());
     SnapshotsQueue snapshots_queue{1};
-    auto state_machine = std::make_shared<KeeperStateMachine>(queue, snapshots_queue, settings, keeper_context, nullptr);
+    auto state_machine = std::make_shared<KeeperStateMachine<DB::KeeperMemoryStorage>>(queue, snapshots_queue, settings, keeper_context, nullptr);
     state_machine->init();
     DB::KeeperLogStore changelog(
         DB::LogFileSettings{
@@ -1672,7 +1672,7 @@ void testLogAndStateMachine(
     }
 
     SnapshotsQueue snapshots_queue1{1};
-    auto restore_machine = std::make_shared<KeeperStateMachine>(queue, snapshots_queue1, settings, keeper_context, nullptr);
+    auto restore_machine = std::make_shared<KeeperStateMachine<DB::KeeperMemoryStorage>>(queue, snapshots_queue1, settings, keeper_context, nullptr);
     restore_machine->init();
     EXPECT_EQ(restore_machine->last_commit_index(), total_logs - total_logs % settings->snapshot_distance);
 
@@ -1791,7 +1791,7 @@ TEST_P(CoordinationTest, TestEphemeralNodeRemove)
 
     ResponsesQueue queue(std::numeric_limits<size_t>::max());
     SnapshotsQueue snapshots_queue{1};
-    auto state_machine = std::make_shared<KeeperStateMachine>(queue, snapshots_queue, settings, keeper_context, nullptr);
+    auto state_machine = std::make_shared<KeeperStateMachine<DB::KeeperMemoryStorage>>(queue, snapshots_queue, settings, keeper_context, nullptr);
     state_machine->init();
 
     std::shared_ptr<ZooKeeperCreateRequest> request_c = std::make_shared<ZooKeeperCreateRequest>();
@@ -1825,11 +1825,11 @@ TEST_P(CoordinationTest, TestCreateNodeWithAuthSchemeForAclWhenAuthIsPrecommitte
     ResponsesQueue queue(std::numeric_limits<size_t>::max());
     SnapshotsQueue snapshots_queue{1};
 
-    auto state_machine = std::make_shared<KeeperStateMachine>(queue, snapshots_queue, settings, keeper_context, nullptr);
+    auto state_machine = std::make_shared<KeeperStateMachine<DB::KeeperMemoryStorage>>(queue, snapshots_queue, settings, keeper_context, nullptr);
     state_machine->init();
 
     String user_auth_data = "test_user:test_password";
-    String digest = KeeperStorage::generateDigest(user_auth_data);
+    String digest = KeeperMemoryStorage::generateDigest(user_auth_data);
 
     std::shared_ptr<ZooKeeperAuthRequest> auth_req = std::make_shared<ZooKeeperAuthRequest>();
     auth_req->scheme = "digest";
@@ -1877,11 +1877,11 @@ TEST_P(CoordinationTest, TestSetACLWithAuthSchemeForAclWhenAuthIsPrecommitted)
     ResponsesQueue queue(std::numeric_limits<size_t>::max());
     SnapshotsQueue snapshots_queue{1};
 
-    auto state_machine = std::make_shared<KeeperStateMachine>(queue, snapshots_queue, settings, keeper_context, nullptr);
+    auto state_machine = std::make_shared<KeeperStateMachine<DB::KeeperMemoryStorage>>(queue, snapshots_queue, settings, keeper_context, nullptr);
     state_machine->init();
 
     String user_auth_data = "test_user:test_password";
-    String digest = KeeperStorage::generateDigest(user_auth_data);
+    String digest = KeeperMemoryStorage::generateDigest(user_auth_data);
 
     std::shared_ptr<ZooKeeperAuthRequest> auth_req = std::make_shared<ZooKeeperAuthRequest>();
     auth_req->scheme = "digest";
@@ -2104,9 +2104,9 @@ TEST_P(CoordinationTest, TestStorageSnapshotDifferentCompressions)
     ChangelogDirTest test("./snapshots");
     setSnapshotDirectory("./snapshots");
 
-    DB::KeeperSnapshotManager manager(3, keeper_context, params.enable_compression);
+    DB::KeeperSnapshotManager<DB::KeeperMemoryStorage> manager(3, keeper_context, params.enable_compression);
 
-    DB::KeeperStorage storage(500, "", keeper_context);
+    DB::KeeperMemoryStorage storage(500, "", keeper_context);
     addNode(storage, "/hello", "world", 1);
     addNode(storage, "/hello/somepath", "somedata", 3);
     storage.session_id_counter = 5;
@@ -2116,13 +2116,13 @@ TEST_P(CoordinationTest, TestStorageSnapshotDifferentCompressions)
     storage.getSessionID(130);
     storage.getSessionID(130);
 
-    DB::KeeperStorageSnapshot snapshot(&storage, 2);
+    DB::KeeperStorageSnapshot<DB::KeeperMemoryStorage> snapshot(&storage, 2);
 
     auto buf = manager.serializeSnapshotToBuffer(snapshot);
     manager.serializeSnapshotBufferToDisk(*buf, 2);
     EXPECT_TRUE(fs::exists("./snapshots/snapshot_2.bin" + params.extension));
 
-    DB::KeeperSnapshotManager new_manager(3, keeper_context, !params.enable_compression);
+    DB::KeeperSnapshotManager<DB::KeeperMemoryStorage> new_manager(3, keeper_context, !params.enable_compression);
 
     auto debuf = new_manager.deserializeSnapshotBufferFromDisk(2);
 
@@ -2308,9 +2308,9 @@ TEST_P(CoordinationTest, TestStorageSnapshotEqual)
     std::optional<UInt128> snapshot_hash;
     for (size_t i = 0; i < 15; ++i)
     {
-        DB::KeeperSnapshotManager manager(3, keeper_context, params.enable_compression);
+        DB::KeeperSnapshotManager<DB::KeeperMemoryStorage> manager(3, keeper_context, params.enable_compression);
 
-        DB::KeeperStorage storage(500, "", keeper_context);
+        DB::KeeperMemoryStorage storage(500, "", keeper_context);
         addNode(storage, "/hello", "");
         for (size_t j = 0; j < 5000; ++j)
         {
@@ -2326,7 +2326,7 @@ TEST_P(CoordinationTest, TestStorageSnapshotEqual)
         for (size_t j = 0; j < 3333; ++j)
             storage.getSessionID(130 * j);
 
-        DB::KeeperStorageSnapshot snapshot(&storage, storage.zxid);
+        DB::KeeperStorageSnapshot<DB::KeeperMemoryStorage> snapshot(&storage, storage.zxid);
 
         auto buf = manager.serializeSnapshotToBuffer(snapshot);
 
@@ -2389,7 +2389,7 @@ TEST_P(CoordinationTest, TestUncommittedStateBasicCrud)
     using namespace DB;
     using namespace Coordination;
 
-    DB::KeeperStorage storage{500, "", keeper_context};
+    DB::KeeperMemoryStorage storage{500, "", keeper_context};
 
     constexpr std::string_view path = "/test";
 
@@ -2506,7 +2506,7 @@ TEST_P(CoordinationTest, TestListRequestTypes)
     using namespace DB;
     using namespace Coordination;
 
-    KeeperStorage storage{500, "", keeper_context};
+    KeeperMemoryStorage storage{500, "", keeper_context};
 
     int32_t zxid = 0;
 
@@ -2660,7 +2660,7 @@ TEST_P(CoordinationTest, TestDurableState)
 TEST_P(CoordinationTest, TestFeatureFlags)
 {
     using namespace Coordination;
-    KeeperStorage storage{500, "", keeper_context};
+    KeeperMemoryStorage storage{500, "", keeper_context};
     auto request = std::make_shared<ZooKeeperGetRequest>();
     request->path = DB::keeper_api_feature_flags_path;
     auto responses = storage.processRequest(request, 0, std::nullopt, true, true);
@@ -2679,7 +2679,7 @@ TEST_P(CoordinationTest, TestSystemNodeModify)
 
     // On INIT we abort when a system path is modified
     keeper_context->setServerState(KeeperContext::Phase::RUNNING);
-    KeeperStorage storage{500, "", keeper_context};
+    KeeperMemoryStorage storage{500, "", keeper_context};
     const auto assert_create = [&](const std::string_view path, const auto expected_code)
     {
         auto request = std::make_shared<ZooKeeperCreateRequest>();
@@ -2771,7 +2771,7 @@ TEST_P(CoordinationTest, TestCheckNotExistsRequest)
     using namespace DB;
     using namespace Coordination;
 
-    KeeperStorage storage{500, "", keeper_context};
+    KeeperMemoryStorage storage{500, "", keeper_context};
 
     int32_t zxid = 0;
 
@@ -2850,7 +2850,7 @@ TEST_P(CoordinationTest, TestReapplyingDeltas)
     create_request->path = "/test/data";
     create_request->is_sequential = true;
 
-    const auto process_create = [](KeeperStorage & storage, const auto & request, int64_t zxid)
+    const auto process_create = [](KeeperMemoryStorage & storage, const auto & request, int64_t zxid)
     {
         storage.preprocessRequest(request, 1, 0, zxid);
         auto responses = storage.processRequest(request, 1, zxid);
@@ -2871,19 +2871,19 @@ TEST_P(CoordinationTest, TestReapplyingDeltas)
             process_create(storage, create_request, zxid);
     };
 
-    KeeperStorage storage1{500, "", keeper_context};
+    KeeperMemoryStorage storage1{500, "", keeper_context};
     commit_initial_data(storage1);
 
     for (int64_t zxid = initial_zxid + 1; zxid < initial_zxid + 50; ++zxid)
         storage1.preprocessRequest(create_request, 1, 0, zxid, /*check_acl=*/true, /*digest=*/std::nullopt, /*log_idx=*/zxid);
 
     /// create identical new storage
-    KeeperStorage storage2{500, "", keeper_context};
+    KeeperMemoryStorage storage2{500, "", keeper_context};
     commit_initial_data(storage2);
 
     storage1.applyUncommittedState(storage2, initial_zxid);
 
-    const auto commit_unprocessed = [&](KeeperStorage & storage)
+    const auto commit_unprocessed = [&](KeeperMemoryStorage & storage)
     {
         for (int64_t zxid = initial_zxid + 1; zxid < initial_zxid + 50; ++zxid)
         {
@@ -2896,7 +2896,7 @@ TEST_P(CoordinationTest, TestReapplyingDeltas)
     commit_unprocessed(storage1);
     commit_unprocessed(storage2);
 
-    const auto get_children = [&](KeeperStorage & storage)
+    const auto get_children = [&](KeeperMemoryStorage & storage)
     {
         const auto list_request = std::make_shared<ZooKeeperListRequest>();
         list_request->path = "/test";

From 512705a25fac3f034a3fabcacec0fb526bd04ca0 Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Sun, 12 Nov 2023 22:21:43 +0100
Subject: [PATCH 0002/1488] fix unit test

---
 src/Coordination/KeeperSnapshotManager.h | 10 ++++++----
 src/Coordination/KeeperStateMachine.cpp  | 16 ++++++++--------
 2 files changed, 14 insertions(+), 12 deletions(-)

diff --git a/src/Coordination/KeeperSnapshotManager.h b/src/Coordination/KeeperSnapshotManager.h
index b0a9425881f..3d6c8f6ec43 100644
--- a/src/Coordination/KeeperSnapshotManager.h
+++ b/src/Coordination/KeeperSnapshotManager.h
@@ -40,8 +40,6 @@ struct SnapshotDeserializationResult
     ClusterConfigPtr cluster_config;
 };
 
-struct IKeeperStorageSnapshot {};
-
 /// In memory keeper snapshot. Keeper Storage based on a hash map which can be
 /// turned into snapshot mode. This operation is fast and KeeperStorageSnapshot
 /// class do it in constructor. It also copies iterators from storage hash table
@@ -51,7 +49,7 @@ struct IKeeperStorageSnapshot {};
 /// This representation of snapshot have to be serialized into NuRaft
 /// buffer and send over network or saved to file.
 template<typename Storage>
-struct KeeperStorageSnapshot : IKeeperStorageSnapshot
+struct KeeperStorageSnapshot
 {
 public:
     KeeperStorageSnapshot(Storage * storage_, uint64_t up_to_log_idx_, const ClusterConfigPtr & cluster_config_ = nullptr);
@@ -59,6 +57,9 @@ public:
     KeeperStorageSnapshot(
         Storage * storage_, const SnapshotMetadataPtr & snapshot_meta_, const ClusterConfigPtr & cluster_config_ = nullptr);
 
+    KeeperStorageSnapshot(const KeeperStorageSnapshot<Storage>&) = delete;
+    KeeperStorageSnapshot(KeeperStorageSnapshot<Storage>&&) = default;
+
     ~KeeperStorageSnapshot();
 
     static void serialize(const KeeperStorageSnapshot<Storage> & snapshot, WriteBuffer & out, KeeperContextPtr keeper_context);
@@ -97,7 +98,8 @@ struct SnapshotFileInfo
     DiskPtr disk;
 };
 
-using KeeperStorageSnapshotPtr = std::shared_ptr<IKeeperStorageSnapshot>;
+/// TODO(hanfei): use
+using KeeperStorageSnapshotPtr = std::unique_ptr<std::variant<KeeperStorageSnapshot<KeeperMemoryStorage>>>;
 using CreateSnapshotCallback = std::function<SnapshotFileInfo(KeeperStorageSnapshotPtr &&)>;
 
 /// Class responsible for snapshots serialization and deserialization. Each snapshot
diff --git a/src/Coordination/KeeperStateMachine.cpp b/src/Coordination/KeeperStateMachine.cpp
index a3c9378a173..097adf68e50 100644
--- a/src/Coordination/KeeperStateMachine.cpp
+++ b/src/Coordination/KeeperStateMachine.cpp
@@ -579,13 +579,13 @@ void KeeperStateMachine<Storage>::create_snapshot(nuraft::snapshot & s, nuraft::
     CreateSnapshotTask snapshot_task;
     { /// lock storage for a short period time to turn on "snapshot mode". After that we can read consistent storage state without locking.
         std::lock_guard lock(storage_and_responses_lock);
-        snapshot_task.snapshot = std::make_shared<KeeperStorageSnapshot<Storage>>(storage.get(), snapshot_meta_copy, getClusterConfig());
+        snapshot_task.snapshot = std::make_unique<std::variant<KeeperStorageSnapshot<Storage>>>(KeeperStorageSnapshot<Storage>(storage.get(), snapshot_meta_copy, getClusterConfig()));
     }
 
     /// create snapshot task for background execution (in snapshot thread)
     snapshot_task.create_snapshot = [this, when_done](KeeperStorageSnapshotPtr && snapshot_)
     {
-        auto * snapshot = typeid_cast<KeeperStorageSnapshot<Storage> *>(snapshot_.get());
+        auto && snapshot = std::get<KeeperStorageSnapshot<Storage>>(*snapshot_);
         nuraft::ptr<std::exception> exception(nullptr);
         bool ret = true;
         try
@@ -593,29 +593,29 @@ void KeeperStateMachine<Storage>::create_snapshot(nuraft::snapshot & s, nuraft::
             { /// Read storage data without locks and create snapshot
                 std::lock_guard lock(snapshots_lock);
 
-                if (latest_snapshot_meta && snapshot->snapshot_meta->get_last_log_idx() <= latest_snapshot_meta->get_last_log_idx())
+                if (latest_snapshot_meta && snapshot.snapshot_meta->get_last_log_idx() <= latest_snapshot_meta->get_last_log_idx())
                 {
                     LOG_INFO(
                         log,
                         "Will not create a snapshot with last log idx {} because a snapshot with bigger last log idx ({}) is already "
                         "created",
-                        snapshot->snapshot_meta->get_last_log_idx(),
+                        snapshot.snapshot_meta->get_last_log_idx(),
                         latest_snapshot_meta->get_last_log_idx());
                 }
                 else
                 {
-                    latest_snapshot_meta = snapshot->snapshot_meta;
+                    latest_snapshot_meta = snapshot.snapshot_meta;
                     /// we rely on the fact that the snapshot disk cannot be changed during runtime
                     if (isLocalDisk(*keeper_context->getLatestSnapshotDisk()))
                     {
-                        auto snapshot_info = snapshot_manager.serializeSnapshotToDisk(*snapshot);
+                        auto snapshot_info = snapshot_manager.serializeSnapshotToDisk(snapshot);
                         latest_snapshot_info = std::move(snapshot_info);
                         latest_snapshot_buf = nullptr;
                     }
                     else
                     {
-                        auto snapshot_buf = snapshot_manager.serializeSnapshotToBuffer(*snapshot);
-                        auto snapshot_info = snapshot_manager.serializeSnapshotBufferToDisk(*snapshot_buf, snapshot->snapshot_meta->get_last_log_idx());
+                        auto snapshot_buf = snapshot_manager.serializeSnapshotToBuffer(snapshot);
+                        auto snapshot_info = snapshot_manager.serializeSnapshotBufferToDisk(*snapshot_buf, snapshot.snapshot_meta->get_last_log_idx());
                         latest_snapshot_info = std::move(snapshot_info);
                         latest_snapshot_buf = std::move(snapshot_buf);
                     }

From bea4e28691ffc78cd549bd89794bf2eb7a416c88 Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Sun, 12 Nov 2023 23:46:53 +0100
Subject: [PATCH 0003/1488] fix special build

---
 src/Coordination/KeeperStateMachine.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Coordination/KeeperStateMachine.h b/src/Coordination/KeeperStateMachine.h
index 9ae3905cddd..e7b3ef6022d 100644
--- a/src/Coordination/KeeperStateMachine.h
+++ b/src/Coordination/KeeperStateMachine.h
@@ -27,7 +27,7 @@ public:
         const KeeperContextPtr & keeper_context_,
         KeeperSnapshotManagerS3 * snapshot_manager_s3_,
         CommitCallback commit_callback_,
-        const std::string & superdisgest_);
+        const std::string & superdigest_);
 
     /// Read state from the latest snapshot
     virtual void init() = 0;

From d67e3019af6a0a5f536237c1a9626a298d0fe740 Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Mon, 13 Nov 2023 13:53:22 +0100
Subject: [PATCH 0004/1488] try to fix data race

---
 src/Coordination/KeeperSnapshotManager.h |  3 +--
 src/Coordination/KeeperStateMachine.cpp  | 18 +++++++++---------
 src/Coordination/KeeperStorage.h         | 14 +++++++++++---
 src/Coordination/ZooKeeperDataReader.cpp |  4 ++--
 utils/keeper-data-dumper/main.cpp        |  4 ++--
 5 files changed, 25 insertions(+), 18 deletions(-)

diff --git a/src/Coordination/KeeperSnapshotManager.h b/src/Coordination/KeeperSnapshotManager.h
index 3d6c8f6ec43..fc7e65dd98a 100644
--- a/src/Coordination/KeeperSnapshotManager.h
+++ b/src/Coordination/KeeperSnapshotManager.h
@@ -98,8 +98,7 @@ struct SnapshotFileInfo
     DiskPtr disk;
 };
 
-/// TODO(hanfei): use
-using KeeperStorageSnapshotPtr = std::unique_ptr<std::variant<KeeperStorageSnapshot<KeeperMemoryStorage>>>;
+using KeeperStorageSnapshotPtr = std::variant<std::unique_ptr<KeeperStorageSnapshot<KeeperMemoryStorage>>>;
 using CreateSnapshotCallback = std::function<SnapshotFileInfo(KeeperStorageSnapshotPtr &&)>;
 
 /// Class responsible for snapshots serialization and deserialization. Each snapshot
diff --git a/src/Coordination/KeeperStateMachine.cpp b/src/Coordination/KeeperStateMachine.cpp
index 097adf68e50..5e0e3c90eef 100644
--- a/src/Coordination/KeeperStateMachine.cpp
+++ b/src/Coordination/KeeperStateMachine.cpp
@@ -579,43 +579,43 @@ void KeeperStateMachine<Storage>::create_snapshot(nuraft::snapshot & s, nuraft::
     CreateSnapshotTask snapshot_task;
     { /// lock storage for a short period time to turn on "snapshot mode". After that we can read consistent storage state without locking.
         std::lock_guard lock(storage_and_responses_lock);
-        snapshot_task.snapshot = std::make_unique<std::variant<KeeperStorageSnapshot<Storage>>>(KeeperStorageSnapshot<Storage>(storage.get(), snapshot_meta_copy, getClusterConfig()));
+        snapshot_task.snapshot = std::make_unique<KeeperStorageSnapshot<Storage>>(KeeperStorageSnapshot<Storage>(storage.get(), snapshot_meta_copy, getClusterConfig()));
     }
 
     /// create snapshot task for background execution (in snapshot thread)
     snapshot_task.create_snapshot = [this, when_done](KeeperStorageSnapshotPtr && snapshot_)
     {
-        auto && snapshot = std::get<KeeperStorageSnapshot<Storage>>(*snapshot_);
         nuraft::ptr<std::exception> exception(nullptr);
         bool ret = true;
         try
         {
+            auto && snapshot = std::get<std::unique_ptr<KeeperStorageSnapshot<Storage>>>(std::move(snapshot_));
             { /// Read storage data without locks and create snapshot
                 std::lock_guard lock(snapshots_lock);
 
-                if (latest_snapshot_meta && snapshot.snapshot_meta->get_last_log_idx() <= latest_snapshot_meta->get_last_log_idx())
+                if (latest_snapshot_meta && snapshot->snapshot_meta->get_last_log_idx() <= latest_snapshot_meta->get_last_log_idx())
                 {
                     LOG_INFO(
                         log,
                         "Will not create a snapshot with last log idx {} because a snapshot with bigger last log idx ({}) is already "
                         "created",
-                        snapshot.snapshot_meta->get_last_log_idx(),
+                        snapshot->snapshot_meta->get_last_log_idx(),
                         latest_snapshot_meta->get_last_log_idx());
                 }
                 else
                 {
-                    latest_snapshot_meta = snapshot.snapshot_meta;
+                    latest_snapshot_meta = snapshot->snapshot_meta;
                     /// we rely on the fact that the snapshot disk cannot be changed during runtime
                     if (isLocalDisk(*keeper_context->getLatestSnapshotDisk()))
                     {
-                        auto snapshot_info = snapshot_manager.serializeSnapshotToDisk(snapshot);
+                        auto snapshot_info = snapshot_manager.serializeSnapshotToDisk(*snapshot);
                         latest_snapshot_info = std::move(snapshot_info);
                         latest_snapshot_buf = nullptr;
                     }
                     else
                     {
-                        auto snapshot_buf = snapshot_manager.serializeSnapshotToBuffer(snapshot);
-                        auto snapshot_info = snapshot_manager.serializeSnapshotBufferToDisk(*snapshot_buf, snapshot.snapshot_meta->get_last_log_idx());
+                        auto snapshot_buf = snapshot_manager.serializeSnapshotToBuffer(*snapshot);
+                        auto snapshot_info = snapshot_manager.serializeSnapshotBufferToDisk(*snapshot_buf, snapshot->snapshot_meta->get_last_log_idx());
                         latest_snapshot_info = std::move(snapshot_info);
                         latest_snapshot_buf = std::move(snapshot_buf);
                     }
@@ -632,7 +632,7 @@ void KeeperStateMachine<Storage>::create_snapshot(nuraft::snapshot & s, nuraft::
                 /// Turn off "snapshot mode" and clear outdate part of storage state
                 storage->clearGarbageAfterSnapshot();
                 LOG_TRACE(log, "Cleared garbage after snapshot");
-                snapshot_.reset();
+                snapshot.reset();
             }
         }
         catch (...)
diff --git a/src/Coordination/KeeperStorage.h b/src/Coordination/KeeperStorage.h
index 2eff9f6651b..6fc36a7d04a 100644
--- a/src/Coordination/KeeperStorage.h
+++ b/src/Coordination/KeeperStorage.h
@@ -104,7 +104,6 @@ public:
         bool operator==(const AuthID & other) const { return scheme == other.scheme && id == other.id; }
     };
 
-    // using Container = SnapshotableHashTable<Node>;
     using Ephemerals = std::unordered_map<int64_t, std::unordered_set<std::string>>;
     using SessionAndWatcher = std::unordered_map<int64_t, std::unordered_set<std::string>>;
     using SessionIDs = std::unordered_set<int64_t>;
@@ -449,10 +448,19 @@ public:
     /// Set of methods for creating snapshots
 
     /// Turn on snapshot mode, so data inside Container is not deleted, but replaced with new version.
-    void enableSnapshotMode(size_t up_to_version) { container.enableSnapshotMode(up_to_version); }
+    void enableSnapshotMode(size_t up_to_version)
+    {
+        /// TODO: remove garbage log.
+        LOG_DEBUG(&Poco::Logger::get("KeeperStorage"), "enable snapshot mode {}", up_to_version);
+        container.enableSnapshotMode(up_to_version);
+    }
 
     /// Turn off snapshot mode.
-    void disableSnapshotMode() { container.disableSnapshotMode(); }
+    void disableSnapshotMode()
+    {
+        LOG_DEBUG(&Poco::Logger::get("KeeperStorage"), "disable snapshot mode");
+        container.disableSnapshotMode();
+    }
 
     Container::const_iterator getSnapshotIteratorBegin() const { return container.begin(); }
 
diff --git a/src/Coordination/ZooKeeperDataReader.cpp b/src/Coordination/ZooKeeperDataReader.cpp
index e783582af23..20080e1cdf6 100644
--- a/src/Coordination/ZooKeeperDataReader.cpp
+++ b/src/Coordination/ZooKeeperDataReader.cpp
@@ -190,10 +190,11 @@ void deserializeKeeperStorageFromSnapshot(Storage & storage, const std::string &
     LOG_INFO(log, "Finished, snapshot ZXID {}", storage.zxid);
 }
 
+namespace fs = std::filesystem;
+
 template<typename Storage>
 void deserializeKeeperStorageFromSnapshotsDir(Storage & storage, const std::string & path, Poco::Logger * log)
 {
-    namespace fs = std::filesystem;
     std::map<int64_t, std::string> existing_snapshots;
     for (const auto & p : fs::directory_iterator(path))
     {
@@ -563,7 +564,6 @@ void deserializeLogAndApplyToStorage(Storage & storage, const std::string & log_
 template<typename Storage>
 void deserializeLogsAndApplyToStorage(Storage & storage, const std::string & path, Poco::Logger * log)
 {
-    namespace fs = std::filesystem;
     std::map<int64_t, std::string> existing_logs;
     for (const auto & p : fs::directory_iterator(path))
     {
diff --git a/utils/keeper-data-dumper/main.cpp b/utils/keeper-data-dumper/main.cpp
index aa8c0efbb26..abde98e6b7d 100644
--- a/utils/keeper-data-dumper/main.cpp
+++ b/utils/keeper-data-dumper/main.cpp
@@ -13,7 +13,7 @@
 using namespace Coordination;
 using namespace DB;
 
-void dumpMachine(std::shared_ptr<KeeperStateMachine> machine)
+void dumpMachine(std::shared_ptr<KeeperStateMachine<DB::KeeperMemoryStorage>> machine)
 {
     auto & storage = machine->getStorageUnsafe();
     std::queue<std::string> keys;
@@ -67,7 +67,7 @@ int main(int argc, char *argv[])
     keeper_context->setLogDisk(std::make_shared<DB::DiskLocal>("LogDisk", argv[2]));
     keeper_context->setSnapshotDisk(std::make_shared<DB::DiskLocal>("SnapshotDisk", argv[1]));
 
-    auto state_machine = std::make_shared<KeeperStateMachine>(queue, snapshots_queue, settings, keeper_context, nullptr);
+    auto state_machine = std::make_shared<KeeperStateMachine<DB::KeeperMemoryStorage>>(queue, snapshots_queue, settings, keeper_context, nullptr);
     state_machine->init();
     size_t last_commited_index = state_machine->last_commit_index();
 

From e4c3b4c885d2d4a0295caddba0e9a8edf949bb0b Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Tue, 14 Nov 2023 14:39:10 +0100
Subject: [PATCH 0005/1488] try to fix data race again

---
 src/Coordination/KeeperSnapshotManager.h | 2 +-
 src/Coordination/KeeperStateMachine.cpp  | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/Coordination/KeeperSnapshotManager.h b/src/Coordination/KeeperSnapshotManager.h
index fc7e65dd98a..edc2607b072 100644
--- a/src/Coordination/KeeperSnapshotManager.h
+++ b/src/Coordination/KeeperSnapshotManager.h
@@ -98,7 +98,7 @@ struct SnapshotFileInfo
     DiskPtr disk;
 };
 
-using KeeperStorageSnapshotPtr = std::variant<std::unique_ptr<KeeperStorageSnapshot<KeeperMemoryStorage>>>;
+using KeeperStorageSnapshotPtr = std::variant<std::shared_ptr<KeeperStorageSnapshot<KeeperMemoryStorage>>>;
 using CreateSnapshotCallback = std::function<SnapshotFileInfo(KeeperStorageSnapshotPtr &&)>;
 
 /// Class responsible for snapshots serialization and deserialization. Each snapshot
diff --git a/src/Coordination/KeeperStateMachine.cpp b/src/Coordination/KeeperStateMachine.cpp
index 5e0e3c90eef..fbc8a4b6ec0 100644
--- a/src/Coordination/KeeperStateMachine.cpp
+++ b/src/Coordination/KeeperStateMachine.cpp
@@ -579,7 +579,7 @@ void KeeperStateMachine<Storage>::create_snapshot(nuraft::snapshot & s, nuraft::
     CreateSnapshotTask snapshot_task;
     { /// lock storage for a short period time to turn on "snapshot mode". After that we can read consistent storage state without locking.
         std::lock_guard lock(storage_and_responses_lock);
-        snapshot_task.snapshot = std::make_unique<KeeperStorageSnapshot<Storage>>(KeeperStorageSnapshot<Storage>(storage.get(), snapshot_meta_copy, getClusterConfig()));
+        snapshot_task.snapshot = std::make_shared<KeeperStorageSnapshot<Storage>>(storage.get(), snapshot_meta_copy, getClusterConfig());
     }
 
     /// create snapshot task for background execution (in snapshot thread)
@@ -589,7 +589,7 @@ void KeeperStateMachine<Storage>::create_snapshot(nuraft::snapshot & s, nuraft::
         bool ret = true;
         try
         {
-            auto && snapshot = std::get<std::unique_ptr<KeeperStorageSnapshot<Storage>>>(std::move(snapshot_));
+            auto && snapshot = std::get<std::shared_ptr<KeeperStorageSnapshot<Storage>>>(std::move(snapshot_));
             { /// Read storage data without locks and create snapshot
                 std::lock_guard lock(snapshots_lock);
 

From 13e5e6d8e7241ec9bb98c195899d969e33ca1c9b Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Wed, 27 Dec 2023 17:54:33 +0100
Subject: [PATCH 0006/1488] refine find interface

---
 src/Coordination/CoordinationSettings.h       |   3 +-
 src/Coordination/KeeperContext.cpp            | 112 ++++++++++++++++++
 src/Coordination/KeeperContext.h              |  13 ++
 src/Coordination/KeeperStorage.cpp            |  34 +++---
 src/Coordination/KeeperStorage.h              |  26 ++++
 src/Coordination/SnapshotableHashTable.h      |  14 ++-
 src/Coordination/tests/gtest_coordination.cpp |   2 +-
 7 files changed, 182 insertions(+), 22 deletions(-)

diff --git a/src/Coordination/CoordinationSettings.h b/src/Coordination/CoordinationSettings.h
index a58f2b04797..a4679ae7a87 100644
--- a/src/Coordination/CoordinationSettings.h
+++ b/src/Coordination/CoordinationSettings.h
@@ -52,7 +52,8 @@ struct Settings;
     M(UInt64, log_file_overallocate_size, 50 * 1024 * 1024, "If max_log_file_size is not set to 0, this value will be added to it for preallocating bytes on disk. If a log record is larger than this value, it could lead to uncaught out-of-space issues so a larger value is preferred", 0) \
     M(UInt64, min_request_size_for_cache, 50 * 1024, "Minimal size of the request to cache the deserialization result. Caching can have negative effect on latency for smaller requests, set to 0 to disable", 0) \
     M(UInt64, raft_limits_reconnect_limit, 50, "If connection to a peer is silent longer than this limit * (multiplied by heartbeat interval), we re-establish the connection.", 0) \
-    M(Bool, async_replication, false, "Enable async replication. All write and read guarantees are preserved while better performance is achieved. Settings is disabled by default to not break backwards compatibility.", 0)
+    M(Bool, async_replication, false, "Enable async replication. All write and read guarantees are preserved while better performance is achieved. Settings is disabled by default to not break backwards compatibility.", 0) \
+    M(Bool, use_rocksdb, false, "Use rocksdb as backend storage", 0)
 
 DECLARE_SETTINGS_TRAITS(CoordinationSettingsTraits, LIST_OF_COORDINATION_SETTINGS)
 
diff --git a/src/Coordination/KeeperContext.cpp b/src/Coordination/KeeperContext.cpp
index c3cb166abee..29569b03c49 100644
--- a/src/Coordination/KeeperContext.cpp
+++ b/src/Coordination/KeeperContext.cpp
@@ -4,12 +4,17 @@
 #include <Disks/DiskLocal.h>
 #include <Interpreters/Context.h>
 #include <IO/S3/Credentials.h>
+#include <IO/WriteHelpers.h>
 #include <Poco/Util/AbstractConfiguration.h>
 #include <Coordination/KeeperConstants.h>
 #include <Common/logger_useful.h>
 #include <Coordination/KeeperFeatureFlags.h>
 #include <boost/algorithm/string.hpp>
 
+#include <rocksdb/table.h>
+#include <rocksdb/convenience.h>
+#include <rocksdb/utilities/db_ttl.h>
+
 namespace DB
 {
 
@@ -17,6 +22,8 @@ namespace ErrorCodes
 {
 
 extern const int BAD_ARGUMENTS;
+extern const int LOCICAL_ERROR;
+extern const int ROCKSDB_ERROR;
 
 }
 
@@ -33,6 +40,93 @@ KeeperContext::KeeperContext(bool standalone_keeper_)
     system_nodes_with_data[keeper_api_version_path] = toString(static_cast<uint8_t>(KeeperApiVersion::WITH_MULTI_READ));
 }
 
+using RocksDBOptions = std::unordered_map<std::string, std::string>;
+
+static RocksDBOptions getOptionsFromConfig(const Poco::Util::AbstractConfiguration & config, const std::string & path)
+{
+    RocksDBOptions options;
+
+    Poco::Util::AbstractConfiguration::Keys keys;
+    config.keys(path, keys);
+
+    for (const auto & key : keys)
+    {
+        const String key_path = path + "." + key;
+        options[key] = config.getString(key_path);
+    }
+
+    return options;
+}
+
+static rocksdb::Options getRocksDBOptionsFromConfig(const Poco::Util::AbstractConfiguration & config)
+{
+    rocksdb::Status status;
+    rocksdb::Options base;
+
+    base.create_if_missing = true;
+    base.compression = rocksdb::CompressionType::kZSTD;
+    base.statistics = rocksdb::CreateDBStatistics();
+    /// It is too verbose by default, and in fact we don't care about rocksdb logs at all.
+    base.info_log_level = rocksdb::ERROR_LEVEL;
+
+    rocksdb::Options merged = base;
+    rocksdb::BlockBasedTableOptions table_options;
+
+    if (config.has("keeper_server.rocksdb.options"))
+    {
+        auto config_options = getOptionsFromConfig(config, "keeper_server.rocksdb.options");
+        status = rocksdb::GetDBOptionsFromMap(merged, config_options, &merged);
+        if (!status.ok())
+        {
+            throw Exception(ErrorCodes::ROCKSDB_ERROR, "Fail to merge rocksdb options from 'rocksdb.options' : {}",
+                status.ToString());
+        }
+    }
+    if (config.has("rocksdb.column_family_options"))
+    {
+        auto column_family_options = getOptionsFromConfig(config, "rocksdb.column_family_options");
+        status = rocksdb::GetColumnFamilyOptionsFromMap(merged, column_family_options, &merged);
+        if (!status.ok())
+        {
+            throw Exception(ErrorCodes::ROCKSDB_ERROR, "Fail to merge rocksdb options from 'rocksdb.column_family_options' at: {}", status.ToString());
+        }
+    }
+    if (config.has("rocksdb.block_based_table_options"))
+    {
+        auto block_based_table_options = getOptionsFromConfig(config, "rocksdb.block_based_table_options");
+        status = rocksdb::GetBlockBasedTableOptionsFromMap(table_options, block_based_table_options, &table_options);
+        if (!status.ok())
+        {
+            throw Exception(ErrorCodes::ROCKSDB_ERROR, "Fail to merge rocksdb options from 'rocksdb.block_based_table_options' at: {}", status.ToString());
+        }
+    }
+
+    merged.table_factory.reset(rocksdb::NewBlockBasedTableFactory(table_options));
+    return merged;
+}
+
+KeeperContext::Storage KeeperContext::getRocksDBPathFromConfig(const Poco::Util::AbstractConfiguration & config) const
+{
+    const auto create_local_disk = [](const auto & path)
+    {
+        if (fs::exists(path))
+            fs::remove(path);
+        fs::create_directories(path);
+
+        return std::make_shared<DiskLocal>("LocalRocksDBDisk", path);
+    };
+    if (config.has("keeper_server.rocksdb_path"))
+        return create_local_disk(config.getString("keeper_server.rocksdb_path"));
+
+    if (config.has("keeper_server.storage_path"))
+        return create_local_disk(std::filesystem::path{config.getString("keeper_server.storage_path")} / "rocksdb");
+
+    if (standalone_keeper)
+        return create_local_disk(std::filesystem::path{config.getString("path", KEEPER_DEFAULT_PATH)} / "rocksdb");
+    else
+        return create_local_disk(std::filesystem::path{config.getString("path", DBMS_DEFAULT_PATH)} / "coordination/rocksdb");
+}
+
 void KeeperContext::initialize(const Poco::Util::AbstractConfiguration & config, KeeperDispatcher * dispatcher_)
 {
     dispatcher = dispatcher_;
@@ -64,6 +158,9 @@ void KeeperContext::initialize(const Poco::Util::AbstractConfiguration & config,
 
     initializeFeatureFlags(config);
     initializeDisks(config);
+
+    if (config.getBool("keeper_server.coordination_settings.use_rocksdb", false))
+        rocksdb_options = std::make_shared<rocksdb::Options>(getRocksDBOptionsFromConfig(config));
 }
 
 namespace
@@ -99,6 +196,8 @@ void KeeperContext::initializeDisks(const Poco::Util::AbstractConfiguration & co
 {
     disk_selector->initialize(config, "storage_configuration.disks", Context::getGlobalContextInstance(), diskValidator);
 
+    rocksdb_storage = getRocksDBPathFromConfig(config);
+
     log_storage = getLogsPathFromConfig(config);
 
     if (config.has("keeper_server.latest_log_storage_disk"))
@@ -267,6 +366,19 @@ void KeeperContext::dumpConfiguration(WriteBufferFromOwnString & buf) const
     }
 }
 
+DiskPtr KeeperContext::getTemporaryRocksDBDisk() const
+{
+    DiskPtr rocksdb_disk = getDisk(rocksdb_storage);
+    if (!rocksdb_disk)
+    {
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "rocksdb storage is not initialized");
+    }
+    auto uuid_str = formatUUID(UUIDHelpers::generateV4());
+    String path_to_create = "rocks_" + std::string(uuid_str.data(), uuid_str.size());
+    rocksdb_disk->createDirectory(path_to_create);
+    return std::make_shared<DiskLocal>("LocalTmpRocksDBDisk", fullPath(rocksdb_disk, path_to_create));
+}
+
 KeeperContext::Storage KeeperContext::getLogsPathFromConfig(const Poco::Util::AbstractConfiguration & config) const
 {
     const auto create_local_disk = [](const auto & path)
diff --git a/src/Coordination/KeeperContext.h b/src/Coordination/KeeperContext.h
index 1af34b19ccf..1b3d18e8e53 100644
--- a/src/Coordination/KeeperContext.h
+++ b/src/Coordination/KeeperContext.h
@@ -6,6 +6,11 @@
 #include <cstdint>
 #include <memory>
 
+namespace rocksdb
+{
+struct Options;
+}
+
 namespace DB
 {
 
@@ -53,6 +58,10 @@ public:
 
     constexpr KeeperDispatcher * getDispatcher() const { return dispatcher; }
 
+    DiskPtr getTemporaryRocksDBDisk() const;
+
+    std::shared_ptr<rocksdb::Options> getRocksDBOptions() const { return rocksdb_options; }
+
     /// set to true when we have preprocessed or committed all the logs
     /// that were already present locally during startup
     std::atomic<bool> local_logs_preprocessed = false;
@@ -64,6 +73,7 @@ private:
     void initializeFeatureFlags(const Poco::Util::AbstractConfiguration & config);
     void initializeDisks(const Poco::Util::AbstractConfiguration & config);
 
+    Storage getRocksDBPathFromConfig(const Poco::Util::AbstractConfiguration & config) const;
     Storage getLogsPathFromConfig(const Poco::Util::AbstractConfiguration & config) const;
     Storage getSnapshotsPathFromConfig(const Poco::Util::AbstractConfiguration & config) const;
     Storage getStatePathFromConfig(const Poco::Util::AbstractConfiguration & config) const;
@@ -77,12 +87,15 @@ private:
 
     std::shared_ptr<DiskSelector> disk_selector;
 
+    Storage rocksdb_storage;
     Storage log_storage;
     Storage latest_log_storage;
     Storage snapshot_storage;
     Storage latest_snapshot_storage;
     Storage state_file_storage;
 
+    std::shared_ptr<rocksdb::Options> rocksdb_options;
+
     std::vector<std::string> old_log_disk_names;
     std::vector<std::string> old_snapshot_disk_names;
 
diff --git a/src/Coordination/KeeperStorage.cpp b/src/Coordination/KeeperStorage.cpp
index 80c6c0ad5a4..094d3664cd4 100644
--- a/src/Coordination/KeeperStorage.cpp
+++ b/src/Coordination/KeeperStorage.cpp
@@ -261,7 +261,7 @@ void KeeperStorage<Container>::initializeSystemNodes()
         throw Exception(ErrorCodes::LOGICAL_ERROR, "KeeperStorage system nodes initialized twice");
 
     // insert root system path if it isn't already inserted
-    if (container.find(keeper_system_path) == container.end())
+    if (container.find(keeper_system_path) == nullptr)
     {
         Node system_node;
         container.insert(keeper_system_path, system_node);
@@ -320,7 +320,7 @@ Overloaded(Ts...) -> Overloaded<Ts...>;
 template<typename Container>
 std::shared_ptr<typename Container::Node> KeeperStorage<Container>::UncommittedState::tryGetNodeFromStorage(StringRef path) const
 {
-    if (auto node_it = storage.container.find(path); node_it != storage.container.end())
+    if (auto node_it = storage.container.find(path); node_it)
     {
         const auto & committed_node = node_it->value;
         auto node = std::make_shared<KeeperStorage<Container>::Node>();
@@ -570,7 +570,7 @@ Coordination::ACLs KeeperStorage<Container>::UncommittedState::getACLs(StringRef
         return node_it->second.acls;
 
     auto node_it = storage.container.find(path);
-    if (node_it == storage.container.end())
+    if (node_it == nullptr)
         return {};
 
     return storage.acl_map.convertNumber(node_it->value.acl_id);
@@ -673,7 +673,7 @@ Coordination::Error KeeperStorage<Container>::commit(int64_t commit_zxid)
                 else if constexpr (std::same_as<DeltaType, UpdateNodeDelta>)
                 {
                     auto node_it = container.find(path);
-                    if (node_it == container.end())
+                    if (node_it == nullptr)
                         onStorageInconsistency();
 
                     if (operation.version != -1 && operation.version != node_it->value.stat.version)
@@ -695,7 +695,7 @@ Coordination::Error KeeperStorage<Container>::commit(int64_t commit_zxid)
                 else if constexpr (std::same_as<DeltaType, SetACLDelta>)
                 {
                     auto node_it = container.find(path);
-                    if (node_it == container.end())
+                    if (node_it == nullptr)
                         onStorageInconsistency();
 
                     if (operation.version != -1 && operation.version != node_it->value.stat.aversion)
@@ -751,7 +751,7 @@ bool KeeperStorage<Container>::createNode(
     auto parent_path = parentNodePath(path);
     auto node_it = container.find(parent_path);
 
-    if (node_it == container.end())
+    if (node_it == nullptr)
         return false;
 
     if (node_it->value.stat.ephemeralOwner != 0)
@@ -789,7 +789,7 @@ template<typename Container>
 bool KeeperStorage<Container>::removeNode(const std::string & path, int32_t version)
 {
     auto node_it = container.find(path);
-    if (node_it == container.end())
+    if (node_it == nullptr)
         return false;
 
     if (version != -1 && version != node_it->value.stat.version)
@@ -885,7 +885,7 @@ Coordination::ACLs getNodeACLs(Storage & storage, StringRef path, bool is_local)
     if (is_local)
     {
         auto node_it = storage.container.find(path);
-        if (node_it == storage.container.end())
+        if (node_it == nullptr)
             return {};
 
         return storage.acl_map.convertNumber(node_it->value.acl_id);
@@ -1146,7 +1146,7 @@ struct KeeperStorageGetRequestProcessor final : public KeeperStorageRequestProce
 
         auto & container = storage.container;
         auto node_it = container.find(request.path);
-        if (node_it == container.end())
+        if (node_it == nullptr)
         {
             if constexpr (local)
                 response.error = Coordination::Error::ZNONODE;
@@ -1308,7 +1308,7 @@ struct KeeperStorageExistsRequestProcessor final : public KeeperStorageRequestPr
 
         auto & container = storage.container;
         auto node_it = container.find(request.path);
-        if (node_it == container.end())
+        if (node_it == nullptr)
         {
             if constexpr (local)
                 response.error = Coordination::Error::ZNONODE;
@@ -1415,7 +1415,7 @@ struct KeeperStorageSetRequestProcessor final : public KeeperStorageRequestProce
         }
 
         auto node_it = container.find(request.path);
-        if (node_it == container.end())
+        if (node_it == nullptr)
             onStorageInconsistency();
 
         response.stat = node_it->value.stat;
@@ -1472,7 +1472,7 @@ struct KeeperStorageListRequestProcessor final : public KeeperStorageRequestProc
         auto & container = storage.container;
 
         auto node_it = container.find(request.path);
-        if (node_it == container.end())
+        if (node_it == nullptr)
         {
             if constexpr (local)
                 response.error = Coordination::Error::ZNONODE;
@@ -1503,7 +1503,7 @@ struct KeeperStorageListRequestProcessor final : public KeeperStorageRequestProc
 
                 auto child_path = (std::filesystem::path(request.path) / child.toView()).generic_string();
                 auto child_it = container.find(child_path);
-                if (child_it == container.end())
+                if (child_it == nullptr)
                     onStorageInconsistency();
 
                 const auto is_ephemeral = child_it->value.stat.ephemeralOwner != 0;
@@ -1604,14 +1604,14 @@ struct KeeperStorageCheckRequestProcessor final : public KeeperStorageRequestPro
 
         if (check_not_exists)
         {
-            if (node_it != container.end() && (request.version == -1 || request.version == node_it->value.stat.version))
+            if (node_it && (request.version == -1 || request.version == node_it->value.stat.version))
                 on_error(Coordination::Error::ZNODEEXISTS);
             else
                 response.error = Coordination::Error::ZOK;
         }
         else
         {
-            if (node_it == container.end())
+            if (node_it == nullptr)
                 on_error(Coordination::Error::ZNONODE);
             else if (request.version != -1 && request.version != node_it->value.stat.version)
                 on_error(Coordination::Error::ZBADVERSION);
@@ -1710,7 +1710,7 @@ struct KeeperStorageSetACLRequestProcessor final : public KeeperStorageRequestPr
         }
 
         auto node_it = storage.container.find(request.path);
-        if (node_it == storage.container.end())
+        if (node_it == nullptr)
             onStorageInconsistency();
         response.stat = node_it->value.stat;
         response.error = Coordination::Error::ZOK;
@@ -1758,7 +1758,7 @@ struct KeeperStorageGetACLRequestProcessor final : public KeeperStorageRequestPr
 
         auto & container = storage.container;
         auto node_it = container.find(request.path);
-        if (node_it == container.end())
+        if (node_it == nullptr)
         {
             if constexpr (local)
                 response.error = Coordination::Error::ZNONODE;
diff --git a/src/Coordination/KeeperStorage.h b/src/Coordination/KeeperStorage.h
index 6fc36a7d04a..288792bb488 100644
--- a/src/Coordination/KeeperStorage.h
+++ b/src/Coordination/KeeperStorage.h
@@ -20,6 +20,29 @@ using ResponseCallback = std::function<void(const Coordination::ZooKeeperRespons
 using ChildrenSet = absl::flat_hash_set<StringRef, StringRefHash>;
 using SessionAndTimeout = std::unordered_map<int64_t, int64_t>;
 
+struct KeeperRocksNode
+{
+    struct KeeperRocksNodeInfo
+    {
+        uint64_t acl_id = 0; /// 0 -- no ACL by default
+        bool is_sequental = false;
+        Coordination::Stat stat{};
+        int32_t seq_num = 0;
+        UInt64 digest = 0; /// we cached digest for this node.
+    } meta;
+
+    uint64_t sizeInBytes() const { return data.size() + sizeof(KeeperRocksNodeInfo); }
+    void setData(String new_data) {data = new_data;}
+    const auto & getData() const noexcept { return data; }
+    void shallowCopy(const KeeperRocksNode & other)
+    {
+        *this = other;
+    }
+private:
+    /// TODO(hanfei): Maybe we can store data and meta seperately.
+    String data;
+};
+
 struct KeeperMemNode
 {
     uint64_t acl_id = 0; /// 0 -- no ACL by default
@@ -136,6 +159,8 @@ public:
     using Container = Container_;
     using Node = Container::Node;
 
+    /// static constexpr bool use_rocksdb = std::is_same_v<Container_, RocksDBContainer>;
+
     static constexpr auto CURRENT_DIGEST_VERSION = DigestVersion::V2;
 
     static String generateDigest(const String & userdata);
@@ -500,5 +525,6 @@ private:
 };
 
 using KeeperMemoryStorage = KeeperStorage<SnapshotableHashTable<KeeperMemNode>>;
+//using KeeperRocksStorage = KeeperStorage<RocksDBContainer<KeeperRocksNode>>;
 
 }
diff --git a/src/Coordination/SnapshotableHashTable.h b/src/Coordination/SnapshotableHashTable.h
index 2bdef189e6b..c5ad43cc69a 100644
--- a/src/Coordination/SnapshotableHashTable.h
+++ b/src/Coordination/SnapshotableHashTable.h
@@ -119,6 +119,14 @@ private:
         }
     }
 
+    struct KVPair
+    {
+        StringRef key;
+        V value;
+    };
+
+    using KVPointer = std::shared_ptr<KVPair>;
+
 public:
 
     using Node = V;
@@ -258,12 +266,12 @@ public:
         return ret;
     }
 
-    const_iterator find(StringRef key) const
+    KVPointer find(StringRef key) const
     {
         auto map_it = map.find(key);
         if (map_it != map.end())
-            return map_it->getMapped();
-        return list.end();
+            return std::make_shared<KVPair>(KVPair{map_it->getMapped()->key, map_it->getMapped()->value});
+        return nullptr;
     }
 
 
diff --git a/src/Coordination/tests/gtest_coordination.cpp b/src/Coordination/tests/gtest_coordination.cpp
index 3d7d62b173b..6153ac12d06 100644
--- a/src/Coordination/tests/gtest_coordination.cpp
+++ b/src/Coordination/tests/gtest_coordination.cpp
@@ -2950,7 +2950,7 @@ TEST_P(CoordinationTest, TestCheckNotExistsRequest)
 
     create_path("/test_node");
     auto node_it = storage.container.find("/test_node");
-    ASSERT_NE(node_it, storage.container.end());
+    ASSERT_NE(node_it, nullptr);
     auto node_version = node_it->value.stat.version;
 
     {

From cba6629b1cc9418f15551dae7af3ba5b5ca6507e Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Fri, 29 Dec 2023 11:46:43 +0100
Subject: [PATCH 0007/1488] fix build

---
 src/Coordination/KeeperContext.cpp | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/Coordination/KeeperContext.cpp b/src/Coordination/KeeperContext.cpp
index b9fba780b66..c79bea2d7c8 100644
--- a/src/Coordination/KeeperContext.cpp
+++ b/src/Coordination/KeeperContext.cpp
@@ -11,9 +11,12 @@
 #include <Coordination/KeeperFeatureFlags.h>
 #include <boost/algorithm/string.hpp>
 
+#include "config.h"
+#if USE_ROCKSDB
 #include <rocksdb/table.h>
 #include <rocksdb/convenience.h>
 #include <rocksdb/utilities/db_ttl.h>
+#endif
 
 namespace DB
 {
@@ -40,6 +43,7 @@ KeeperContext::KeeperContext(bool standalone_keeper_)
     system_nodes_with_data[keeper_api_version_path] = toString(static_cast<uint8_t>(KeeperApiVersion::WITH_MULTI_READ));
 }
 
+#if USE_ROCKSDB
 using RocksDBOptions = std::unordered_map<std::string, std::string>;
 
 static RocksDBOptions getOptionsFromConfig(const Poco::Util::AbstractConfiguration & config, const std::string & path)
@@ -104,6 +108,7 @@ static rocksdb::Options getRocksDBOptionsFromConfig(const Poco::Util::AbstractCo
     merged.table_factory.reset(rocksdb::NewBlockBasedTableFactory(table_options));
     return merged;
 }
+#endif
 
 KeeperContext::Storage KeeperContext::getRocksDBPathFromConfig(const Poco::Util::AbstractConfiguration & config) const
 {
@@ -161,8 +166,10 @@ void KeeperContext::initialize(const Poco::Util::AbstractConfiguration & config,
     initializeFeatureFlags(config);
     initializeDisks(config);
 
+    #if USE_ROCKSDB
     if (config.getBool("keeper_server.coordination_settings.use_rocksdb", false))
         rocksdb_options = std::make_shared<rocksdb::Options>(getRocksDBOptionsFromConfig(config));
+    #endif
 }
 
 namespace

From 2e5f7c7851564cf8cf40ecb88f60ea76bb471f0a Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Tue, 9 Jan 2024 16:26:53 +0100
Subject: [PATCH 0008/1488] fix build

---
 src/Coordination/KeeperStorage.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Coordination/KeeperStorage.cpp b/src/Coordination/KeeperStorage.cpp
index c1b7d80935c..b9b2995db4e 100644
--- a/src/Coordination/KeeperStorage.cpp
+++ b/src/Coordination/KeeperStorage.cpp
@@ -271,7 +271,7 @@ void KeeperStorage<Container>::initializeSystemNodes()
 
         // update root and the digest based on it
         auto current_root_it = container.find("/");
-        assert(current_root_it != container.end());
+        assert(current_root_it);
         removeDigest(current_root_it->value, "/");
         auto updated_root_it = container.updateValue(
             "/",

From b05db179973b396d4aacd247d818742be260d712 Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Wed, 17 Jan 2024 15:32:30 +0100
Subject: [PATCH 0009/1488] support rocksdb container and see how ci is going

---
 src/Coordination/KeeperServer.cpp          |  32 +-
 src/Coordination/KeeperSnapshotManager.cpp |  51 +--
 src/Coordination/KeeperSnapshotManager.h   |  10 +
 src/Coordination/KeeperStateMachine.cpp    |   3 +
 src/Coordination/KeeperStorage.cpp         | 224 +++++++++----
 src/Coordination/KeeperStorage.h           |  51 ++-
 src/Coordination/RocksDBContainer.h        | 368 +++++++++++++++++++++
 7 files changed, 631 insertions(+), 108 deletions(-)
 create mode 100644 src/Coordination/RocksDBContainer.h

diff --git a/src/Coordination/KeeperServer.cpp b/src/Coordination/KeeperServer.cpp
index db8f34afa03..dceb01d9d96 100644
--- a/src/Coordination/KeeperServer.cpp
+++ b/src/Coordination/KeeperServer.cpp
@@ -127,14 +127,30 @@ KeeperServer::KeeperServer(
     if (coordination_settings->quorum_reads)
         LOG_WARNING(log, "Quorum reads enabled, Keeper will work slower.");
 
-    state_machine = nuraft::cs_new<KeeperStateMachine<KeeperMemoryStorage>>(
-        responses_queue_,
-        snapshots_queue_,
-        coordination_settings,
-        keeper_context,
-        config.getBool("keeper_server.upload_snapshot_on_exit", true) ? &snapshot_manager_s3 : nullptr,
-        commit_callback,
-        checkAndGetSuperdigest(configuration_and_settings_->super_digest));
+    if (coordination_settings->use_rocksdb)
+    {
+#if USE_ROCKSDB
+        state_machine = nuraft::cs_new<KeeperStateMachine<KeeperRocksStorage>>(
+            responses_queue_,
+            snapshots_queue_,
+            coordination_settings,
+            keeper_context,
+            config.getBool("keeper_server.upload_snapshot_on_exit", true) ? &snapshot_manager_s3 : nullptr,
+            commit_callback,
+            checkAndGetSuperdigest(configuration_and_settings_->super_digest));
+#else
+        throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "RocksDB support is disabled because ClickHouse was built without RocksDB support.");
+#endif
+    }
+    else
+        state_machine = nuraft::cs_new<KeeperStateMachine<KeeperMemoryStorage>>(
+            responses_queue_,
+            snapshots_queue_,
+            coordination_settings,
+            keeper_context,
+            config.getBool("keeper_server.upload_snapshot_on_exit", true) ? &snapshot_manager_s3 : nullptr,
+            commit_callback,
+            checkAndGetSuperdigest(configuration_and_settings_->super_digest));
 
     state_manager = nuraft::cs_new<KeeperStateManager>(
         server_id,
diff --git a/src/Coordination/KeeperSnapshotManager.cpp b/src/Coordination/KeeperSnapshotManager.cpp
index 7e377421e28..cccd24214cb 100644
--- a/src/Coordination/KeeperSnapshotManager.cpp
+++ b/src/Coordination/KeeperSnapshotManager.cpp
@@ -415,31 +415,34 @@ void KeeperStorageSnapshot<Storage>::deserialize(SnapshotDeserializationResult<S
             storage.nodes_digest += node.getDigest(path);
     }
 
-    for (const auto & itr : storage.container)
+    if constexpr (!use_rocksdb)
     {
-        if (itr.key != "/")
+        for (const auto & itr : storage.container)
         {
-            auto parent_path = parentNodePath(itr.key);
-            storage.container.updateValue(
-                parent_path, [version, path = itr.key](typename Storage::Node & value) { value.addChild(getBaseNodeName(path), /*update_size*/ version < SnapshotVersion::V4); });
-        }
-    }
-
-    for (const auto & itr : storage.container)
-    {
-        if (itr.key != "/")
-        {
-            if (itr.value.stat.numChildren != static_cast<int32_t>(itr.value.getChildren().size()))
+            if (itr.key != "/")
             {
-#ifdef NDEBUG
-                /// TODO (alesapin) remove this, it should be always CORRUPTED_DATA.
-                LOG_ERROR(&Poco::Logger::get("KeeperSnapshotManager"), "Children counter in stat.numChildren {}"
-                            " is different from actual children size {} for node {}", itr.value.stat.numChildren, itr.value.getChildren().size(), itr.key);
-#else
-                throw Exception(ErrorCodes::LOGICAL_ERROR, "Children counter in stat.numChildren {}"
-                                " is different from actual children size {} for node {}",
-                                itr.value.stat.numChildren, itr.value.getChildren().size(), itr.key);
-#endif
+                auto parent_path = parentNodePath(itr.key);
+                storage.container.updateValue(
+                    parent_path, [version, path = itr.key](typename Storage::Node & value) { value.addChild(getBaseNodeName(path), /*update_size*/ version < SnapshotVersion::V4); });
+            }
+        }
+
+        for (const auto & itr : storage.container)
+        {
+            if (itr.key != "/")
+            {
+                if (itr.value.stat.numChildren != static_cast<int32_t>(itr.value.getChildren().size()))
+                {
+    #ifdef NDEBUG
+                    /// TODO (alesapin) remove this, it should be always CORRUPTED_DATA.
+                    LOG_ERROR(&Poco::Logger::get("KeeperSnapshotManager"), "Children counter in stat.numChildren {}"
+                                " is different from actual children size {} for node {}", itr.value.stat.numChildren, itr.value.getChildren().size(), itr.key);
+    #else
+                    throw Exception(ErrorCodes::LOGICAL_ERROR, "Children counter in stat.numChildren {}"
+                                    " is different from actual children size {} for node {}",
+                                    itr.value.stat.numChildren, itr.value.getChildren().size(), itr.key);
+    #endif
+                }
             }
         }
     }
@@ -845,5 +848,9 @@ SnapshotFileInfo KeeperSnapshotManager<Storage>::serializeSnapshotToDisk(const K
 
 template struct KeeperStorageSnapshot<KeeperMemoryStorage>;
 template class KeeperSnapshotManager<KeeperMemoryStorage>;
+#if USE_ROCKSDB
+template struct KeeperStorageSnapshot<KeeperRocksStorage>;
+template class KeeperSnapshotManager<KeeperRocksStorage>;
+#endif
 
 }
diff --git a/src/Coordination/KeeperSnapshotManager.h b/src/Coordination/KeeperSnapshotManager.h
index edc2607b072..50ce94159b6 100644
--- a/src/Coordination/KeeperSnapshotManager.h
+++ b/src/Coordination/KeeperSnapshotManager.h
@@ -51,6 +51,12 @@ struct SnapshotDeserializationResult
 template<typename Storage>
 struct KeeperStorageSnapshot
 {
+#if USE_ROCKSDB
+    static constexpr bool use_rocksdb = std::is_same_v<Storage, KeeperRocksStorage>;
+#else
+    static constexpr bool use_rocksdb = false;
+#endif
+
 public:
     KeeperStorageSnapshot(Storage * storage_, uint64_t up_to_log_idx_, const ClusterConfigPtr & cluster_config_ = nullptr);
 
@@ -98,7 +104,11 @@ struct SnapshotFileInfo
     DiskPtr disk;
 };
 
+#if USE_ROCKSDB
+using KeeperStorageSnapshotPtr = std::variant<std::shared_ptr<KeeperStorageSnapshot<KeeperMemoryStorage>>, std::shared_ptr<KeeperStorageSnapshot<KeeperRocksStorage>>>;
+#else
 using KeeperStorageSnapshotPtr = std::variant<std::shared_ptr<KeeperStorageSnapshot<KeeperMemoryStorage>>>;
+#endif
 using CreateSnapshotCallback = std::function<SnapshotFileInfo(KeeperStorageSnapshotPtr &&)>;
 
 /// Class responsible for snapshots serialization and deserialization. Each snapshot
diff --git a/src/Coordination/KeeperStateMachine.cpp b/src/Coordination/KeeperStateMachine.cpp
index a4b27ca06fb..0c50d9a8b77 100644
--- a/src/Coordination/KeeperStateMachine.cpp
+++ b/src/Coordination/KeeperStateMachine.cpp
@@ -939,5 +939,8 @@ void KeeperStateMachine<Storage>::recalculateStorageStats()
 }
 
 template class KeeperStateMachine<KeeperMemoryStorage>;
+#if USE_ROCKSDB
+template class KeeperStateMachine<KeeperRocksStorage>;
+#endif
 
 }
diff --git a/src/Coordination/KeeperStorage.cpp b/src/Coordination/KeeperStorage.cpp
index b9b2995db4e..0d9efc87869 100644
--- a/src/Coordination/KeeperStorage.cpp
+++ b/src/Coordination/KeeperStorage.cpp
@@ -194,6 +194,40 @@ uint64_t calculateDigest(std::string_view path, std::string_view data, const Coo
 
 }
 
+void KeeperRocksNode::invalidateDigestCache() const
+{
+    if (serialized)
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "We modify node after serialized it");
+    digest = 0;
+}
+
+UInt64 KeeperRocksNode::getDigest(std::string_view path) const
+{
+    if (!digest)
+        digest = calculateDigest(path, data, stat);
+    return digest;
+}
+
+String KeeperRocksNode::getEncodedString()
+{
+    if (serialized)
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "We modify node after serialized it");
+    serialized = true;
+
+    WriteBufferFromOwnString buffer;
+    const KeeperRocksNodeInfo & node_info = *this;
+    writePODBinary(node_info, buffer);
+    return buffer.str();
+}
+
+void KeeperRocksNode::decodeFromString(const String &buffer_str)
+{
+    ReadBufferFromOwnString buffer(buffer_str);
+    KeeperRocksNodeInfo & node_info = *this;
+    readPODBinary(node_info, buffer);
+    readStringBinary(data, buffer);
+}
+
 void KeeperMemNode::setData(String new_data)
 {
     size_bytes = size_bytes - data.size() + new_data.size();
@@ -248,7 +282,8 @@ KeeperStorage<Container>::KeeperStorage(
 {
     Node root_node;
     container.insert("/", root_node);
-    addDigest(root_node, "/");
+    if constexpr (!use_rocksdb)
+        addDigest(root_node, "/");
 
     if (initialize_system_nodes)
         initializeSystemNodes();
@@ -267,21 +302,25 @@ void KeeperStorage<Container>::initializeSystemNodes()
         container.insert(keeper_system_path, system_node);
         // store digest for the empty node because we won't update
         // its stats
-        addDigest(system_node, keeper_system_path);
+        if constexpr (!use_rocksdb)
+            addDigest(system_node, keeper_system_path);
 
         // update root and the digest based on it
         auto current_root_it = container.find("/");
         assert(current_root_it);
-        removeDigest(current_root_it->value, "/");
+        if constexpr (!use_rocksdb)
+            removeDigest(current_root_it->value, "/");
         auto updated_root_it = container.updateValue(
             "/",
             [](auto & node)
             {
                 ++node.stat.numChildren;
-                node.addChild(getBaseNodeName(keeper_system_path));
+                if constexpr (!use_rocksdb)
+                    node.addChild(getBaseNodeName(keeper_system_path));
             }
         );
-        addDigest(updated_root_it->value, "/");
+        if constexpr (!use_rocksdb)
+            addDigest(updated_root_it->value, "/");
     }
 
     // insert child system nodes
@@ -290,17 +329,22 @@ void KeeperStorage<Container>::initializeSystemNodes()
         assert(path.starts_with(keeper_system_path));
         Node child_system_node;
         child_system_node.setData(data);
-        auto [map_key, _] = container.insert(std::string{path}, child_system_node);
-        /// Take child path from key owned by map.
-        auto child_path = getBaseNodeName(map_key->getKey());
-        container.updateValue(
-            parentNodePath(StringRef(path)),
-            [child_path](auto & parent)
-            {
-                // don't update stats so digest is okay
-                parent.addChild(child_path);
-            }
-        );
+        if constexpr (use_rocksdb)
+            container.insert(std::string{path}, child_system_node);
+        else
+        {
+            auto [map_key, _] = container.insert(std::string{path}, child_system_node);
+            /// Take child path from key owned by map.
+            auto child_path = getBaseNodeName(map_key->getKey());
+            container.updateValue(
+                parentNodePath(StringRef(path)),
+                [child_path](auto & parent)
+                {
+                    // don't update stats so digest is okay
+                    parent.addChild(child_path);
+                }
+            );
+        }
     }
 
     initialized = true;
@@ -679,9 +723,11 @@ Coordination::Error KeeperStorage<Container>::commit(int64_t commit_zxid)
                     if (operation.version != -1 && operation.version != node_it->value.stat.version)
                         onStorageInconsistency();
 
-                    removeDigest(node_it->value, path);
+                    if constexpr (!use_rocksdb)
+                        removeDigest(node_it->value, path);
                     auto updated_node = container.updateValue(path, operation.update_fn);
-                    addDigest(updated_node->value, path);
+                    if constexpr (!use_rocksdb)
+                        addDigest(updated_node->value, path);
 
                     return Coordination::Error::ZOK;
                 }
@@ -769,19 +815,26 @@ bool KeeperStorage<Container>::createNode(
     created_node.stat = stat;
     created_node.setData(std::move(data));
     created_node.is_sequental = is_sequental;
-    auto [map_key, _] = container.insert(path, created_node);
-    /// Take child path from key owned by map.
-    auto child_path = getBaseNodeName(map_key->getKey());
-    container.updateValue(
-            parent_path,
-            [child_path](Node & parent)
-            {
-                parent.addChild(child_path);
-                chassert(parent.stat.numChildren == static_cast<int32_t>(parent.getChildren().size()));
-            }
-    );
+    if constexpr (use_rocksdb)
+    {
+        container.insert(path, created_node);
+    }
+    else
+    {
+        auto [map_key, _] = container.insert(path, created_node);
+        /// Take child path from key owned by map.
+        auto child_path = getBaseNodeName(map_key->getKey());
+        container.updateValue(
+                parent_path,
+                [child_path](KeeperMemNode & parent)
+                {
+                    parent.addChild(child_path);
+                    chassert(parent.stat.numChildren == static_cast<int32_t>(parent.getChildren().size()));
+                }
+        );
 
-    addDigest(map_key->getMapped()->value, map_key->getKey().toView());
+        addDigest(map_key->getMapped()->value, map_key->getKey().toView());
+    }
     return true;
 };
 
@@ -801,18 +854,22 @@ bool KeeperStorage<Container>::removeNode(const std::string & path, int32_t vers
     auto prev_node = node_it->value;
     acl_map.removeUsage(prev_node.acl_id);
 
-    container.updateValue(
-        parentNodePath(path),
-        [child_basename = getBaseNodeName(node_it->key)](Node & parent)
-        {
-            parent.removeChild(child_basename);
-            chassert(parent.stat.numChildren == static_cast<int32_t>(parent.getChildren().size()));
-        }
-    );
+    if constexpr (use_rocksdb)
+        container.erase(path);
+    else {
+        container.updateValue(
+            parentNodePath(path),
+            [child_basename = getBaseNodeName(node_it->key)](KeeperMemNode & parent)
+            {
+                parent.removeChild(child_basename);
+                chassert(parent.stat.numChildren == static_cast<int32_t>(parent.getChildren().size()));
+            }
+        );
 
-    container.erase(path);
+        container.erase(path);
 
-    removeDigest(prev_node, path);
+        removeDigest(prev_node, path);
+    }
     return true;
 }
 
@@ -1485,35 +1542,65 @@ struct KeeperStorageListRequestProcessor final : public KeeperStorageRequestProc
             if (path_prefix.empty())
                 throw DB::Exception(ErrorCodes::LOGICAL_ERROR, "Logical error: path cannot be empty");
 
-            const auto & children = node_it->value.getChildren();
-            response.names.reserve(children.size());
-
-            const auto add_child = [&](const auto child)
+            if constexpr (Storage::use_rocksdb)
             {
-                using enum Coordination::ListRequestType;
-
-                auto list_request_type = ALL;
-                if (auto * filtered_list = dynamic_cast<Coordination::ZooKeeperFilteredListRequest *>(&request))
+                const auto & children = container.getChildren(request.path);
+                response.names.reserve(children.size());
+                const auto add_child = [&](const auto & child)
                 {
-                    list_request_type = filtered_list->list_request_type;
+                    using enum Coordination::ListRequestType;
+
+                    auto list_request_type = ALL;
+                    if (auto * filtered_list = dynamic_cast<Coordination::ZooKeeperFilteredListRequest *>(&request))
+                    {
+                        list_request_type = filtered_list->list_request_type;
+                    }
+
+                    if (list_request_type == ALL)
+                        return true;
+
+                    const auto is_ephemeral = child.value.stat.ephemeralOwner != 0;
+                    return (is_ephemeral && list_request_type == EPHEMERAL_ONLY) || (!is_ephemeral && list_request_type == PERSISTENT_ONLY);
+                };
+
+                for (const auto & child : children)
+                {
+                    if (add_child(child))
+                        response.names.push_back(child.key.toString());
                 }
-
-                if (list_request_type == ALL)
-                    return true;
-
-                auto child_path = (std::filesystem::path(request.path) / child.toView()).generic_string();
-                auto child_it = container.find(child_path);
-                if (child_it == nullptr)
-                    onStorageInconsistency();
-
-                const auto is_ephemeral = child_it->value.stat.ephemeralOwner != 0;
-                return (is_ephemeral && list_request_type == EPHEMERAL_ONLY) || (!is_ephemeral && list_request_type == PERSISTENT_ONLY);
-            };
-
-            for (const auto child : children)
+            }
+            else
             {
-                if (add_child(child))
-                    response.names.push_back(child.toString());
+                const auto & children = node_it->value.getChildren();
+                response.names.reserve(children.size());
+
+                const auto add_child = [&](const auto child)
+                {
+                    using enum Coordination::ListRequestType;
+
+                    auto list_request_type = ALL;
+                    if (auto * filtered_list = dynamic_cast<Coordination::ZooKeeperFilteredListRequest *>(&request))
+                    {
+                        list_request_type = filtered_list->list_request_type;
+                    }
+
+                    if (list_request_type == ALL)
+                        return true;
+
+                    auto child_path = (std::filesystem::path(request.path) / child.toView()).generic_string();
+                    auto child_it = container.find(child_path);
+                    if (child_it == nullptr)
+                        onStorageInconsistency();
+
+                    const auto is_ephemeral = child_it->value.stat.ephemeralOwner != 0;
+                    return (is_ephemeral && list_request_type == EPHEMERAL_ONLY) || (!is_ephemeral && list_request_type == PERSISTENT_ONLY);
+                };
+
+                for (const auto child : children)
+                {
+                    if (add_child(child))
+                        response.names.push_back(child.toString());
+                }
             }
 
             response.stat = node_it->value.stat;
@@ -2040,9 +2127,9 @@ public:
     using Creator = std::function<std::shared_ptr<KeeperStorageRequestProcessor<Storage>>(const Coordination::ZooKeeperRequestPtr &)>;
     using OpNumToRequest = std::unordered_map<Coordination::OpNum, Creator>;
 
-    static KeeperStorageRequestProcessorsFactory & instance()
+    static KeeperStorageRequestProcessorsFactory<Storage> & instance()
     {
-        static KeeperStorageRequestProcessorsFactory factory;
+        static KeeperStorageRequestProcessorsFactory<Storage> factory;
         return factory;
     }
 
@@ -2593,5 +2680,8 @@ String KeeperStorage<Container>::generateDigest(const String & userdata)
 }
 
 template class KeeperStorage<SnapshotableHashTable<KeeperMemNode>>;
+#if USE_ROCKSDB
+template class KeeperStorage<RocksDBContainer<KeeperRocksNode>>;
+#endif
 
 }
diff --git a/src/Coordination/KeeperStorage.h b/src/Coordination/KeeperStorage.h
index d222f8ca0a3..0eeedb8dd0c 100644
--- a/src/Coordination/KeeperStorage.h
+++ b/src/Coordination/KeeperStorage.h
@@ -6,6 +6,7 @@
 #include <Coordination/SessionExpiryQueue.h>
 #include <Coordination/SnapshotableHashTable.h>
 #include <IO/WriteBufferFromString.h>
+#include <IO/ReadBufferFromString.h>
 #include <Common/ConcurrentBoundedQueue.h>
 #include <Common/ZooKeeper/IKeeper.h>
 #include <Common/ZooKeeper/ZooKeeperCommon.h>
@@ -13,6 +14,11 @@
 
 #include <absl/container/flat_hash_set.h>
 
+#include "config.h"
+#if USE_ROCKSDB
+#include <Coordination/RocksDBContainer.h>
+#endif
+
 namespace DB
 {
 
@@ -20,16 +26,23 @@ using ResponseCallback = std::function<void(const Coordination::ZooKeeperRespons
 using ChildrenSet = absl::flat_hash_set<StringRef, StringRefHash>;
 using SessionAndTimeout = std::unordered_map<int64_t, int64_t>;
 
-struct KeeperRocksNode
+struct KeeperRocksNodeInfo
 {
-    struct KeeperRocksNodeInfo
-    {
-        uint64_t acl_id = 0; /// 0 -- no ACL by default
-        bool is_sequental = false;
-        Coordination::Stat stat{};
-        int32_t seq_num = 0;
-        UInt64 digest = 0; /// we cached digest for this node.
-    } meta;
+    uint64_t acl_id = 0; /// 0 -- no ACL by default
+    bool is_sequental = false;
+    Coordination::Stat stat{};
+    int32_t seq_num = 0;
+    mutable UInt64 digest = 0; /// we cached digest for this node.
+};
+
+struct KeeperRocksNode:KeeperRocksNodeInfo
+{
+#if USE_ROCKSDB
+    friend struct RocksDBContainer<KeeperRocksNode>;
+#endif
+    using Meta = KeeperRocksNodeInfo;
+
+    uint64_t size_bytes = 0; // only for compatible, should be deprecated
 
     uint64_t sizeInBytes() const { return data.size() + sizeof(KeeperRocksNodeInfo); }
     void setData(String new_data) {data = new_data;}
@@ -38,7 +51,17 @@ struct KeeperRocksNode
     {
         *this = other;
     }
+    void invalidateDigestCache() const;
+    UInt64 getDigest(std::string_view path) const;
+    String getEncodedString();
+    void decodeFromString(const String & buffer_str);
+    void recalculateSize() {}
+    void reset()
+    {
+        serialized = false;
+    }
 private:
+    bool serialized = false;
     String data;
 };
 
@@ -158,7 +181,11 @@ public:
     using Container = Container_;
     using Node = Container::Node;
 
-    /// static constexpr bool use_rocksdb = std::is_same_v<Container_, RocksDBContainer>;
+#if USE_ROCKSDB
+    static constexpr bool use_rocksdb = std::is_same_v<Container_, RocksDBContainer<KeeperRocksNode>>;
+#else
+    static constexpr bool use_rocksdb = false;
+#endif
 
     static constexpr auto CURRENT_DIGEST_VERSION = DigestVersion::V2;
 
@@ -521,6 +548,8 @@ private:
 };
 
 using KeeperMemoryStorage = KeeperStorage<SnapshotableHashTable<KeeperMemNode>>;
-//using KeeperRocksStorage = KeeperStorage<RocksDBContainer<KeeperRocksNode>>;
+#if USE_ROCKSDB
+using KeeperRocksStorage = KeeperStorage<RocksDBContainer<KeeperRocksNode>>;
+#endif
 
 }
diff --git a/src/Coordination/RocksDBContainer.h b/src/Coordination/RocksDBContainer.h
new file mode 100644
index 00000000000..b357bcb2dab
--- /dev/null
+++ b/src/Coordination/RocksDBContainer.h
@@ -0,0 +1,368 @@
+#pragma once
+#include <base/StringRef.h>
+#include <Coordination/CoordinationSettings.h>
+#include <Coordination/KeeperContext.h>
+#include <Coordination/pathUtils.h>
+#include <Common/SipHash.h>
+#include <IO/WriteBufferFromString.h>
+#include <IO/ReadBufferFromString.h>
+
+#include <rocksdb/convenience.h>
+#include <rocksdb/status.h>
+#include <rocksdb/table.h>
+#include <rocksdb/snapshot.h>
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int LOGICAL_ERROR;
+    extern const int ROCKSDB_ERROR;
+}
+
+/// The key-value format of rocks db will be
+/// - key: Int8 (depth of the path) + String (path)
+/// - value: SizeOf(keeperRocksNodeInfo) (meta of the node) + String (data)
+
+template <class Node_>
+struct RocksDBContainer
+{
+    using Node = Node_;
+    /// using const_iterator = std::unique_ptr<rocksdb::Iterator>;
+
+private:
+    UInt8 getKeyDepth(const std::string & key)
+    {
+        UInt8 depth = 0;
+        for (size_t i = 0; i < key.size(); i++)
+        {
+            if (key[i] == '/' && i + 1 != key.size())
+                depth ++;
+        }
+        return depth;
+    }
+
+    std::string getEncodedKey(const std::string & key, bool child_prefix = false)
+    {
+        WriteBufferFromOwnString key_buffer;
+        UInt16 depth = getKeyDepth(key) + child_prefix;
+        writeIntBinary(depth, key_buffer);
+        writeStringBinary(key, key_buffer);
+        return key_buffer.str();
+    }
+
+
+    struct KVPair
+    {
+        StringRef key;
+        Node value;
+    };
+
+    /// using KVPointer = std::shared_ptr<KVPair>;
+    using ValueUpdater = std::function<void(Node & node)>;
+
+public:
+
+    struct const_iterator
+    {
+        std::shared_ptr<rocksdb::Iterator> iter;
+
+        KVPair pair;
+
+        const_iterator()
+        {
+        }
+
+        const_iterator(KVPair && pair_) : pair(std::move(pair_)) {}
+
+        const_iterator(rocksdb::Iterator * iter_) : iter(iter_)
+        {
+            iter->SeekToFirst();
+        }
+
+        const KVPair & operator * () const
+        {
+            return pair;
+        }
+
+        const KVPair * operator->() const
+        {
+            return &pair;
+        }
+
+        bool operator != (const const_iterator & other) const
+        {
+            return pair.key.toView() != other->key.toView();
+        }
+
+        bool operator == (std::nullptr_t) const
+        {
+            return pair.key.empty();
+        }
+
+        bool operator != (std::nullptr_t) const
+        {
+            return !pair.key.empty();
+        }
+
+        operator bool() const
+        {
+            return !pair.key.empty();
+        }
+
+        const_iterator & operator ++()
+        {
+            iter->Next();
+            if (iter->Valid())
+            {
+                pair.key = iter->key().ToString();
+                pair.value.reset();
+                ReadBufferFromOwnString buffer(iter->value().ToStringView());
+                typename Node::Meta & meta = pair.value;
+                readPODBinary(meta, buffer);
+                readStringBinary(pair.value.data, buffer);
+            }
+            else
+            {
+                pair.key = StringRef{};
+            }
+            return *this;
+        }
+    };
+
+    void initialize(const KeeperContextPtr & context)
+    {
+
+        DiskPtr disk = context->getTemporaryRocksDBDisk();
+        if (disk == nullptr)
+        {
+            return;
+        }
+        auto options = context->getRocksDBOptions();
+        if (options == nullptr)
+        {
+            return;
+        }
+        rocksdb_dir = disk->getPath();
+        rocksdb::DB * db;
+        auto status = rocksdb::DB::Open(*options, rocksdb_dir, &db);
+        if (!status.ok())
+        {
+            throw Exception(ErrorCodes::ROCKSDB_ERROR, "Failed to open rocksdb path at: {}: {}",
+                rocksdb_dir, status.ToString());
+        }
+        rocksdb_ptr = std::unique_ptr<rocksdb::DB>(db);
+        /// storage_ptr = storage_;
+    }
+
+    ~RocksDBContainer()
+    {
+        rocksdb_ptr->Close();
+        rocksdb_ptr = nullptr;
+
+        fs::remove_all(rocksdb_dir);
+    }
+
+    std::vector<KVPair> getChildren(const std::string & key)
+    {
+        rocksdb::ReadOptions read_options;
+        read_options.total_order_seek = true;
+
+        auto iter = std::unique_ptr<rocksdb::Iterator>(rocksdb_ptr->NewIterator(read_options));
+        rocksdb::Slice prefix = getEncodedKey(key, true);
+        std::vector<KVPair> result;
+        for (iter->Seek(prefix); iter->Valid() && iter->key().starts_with(prefix); iter->Next())
+        {
+            Node node;
+            ReadBufferFromOwnString buffer(iter->value().ToStringView());
+            typename Node::Meta & meta = node;
+            readPODBinary(meta, buffer);
+            result.push_back(KVPair{iter->key().ToString(), node});
+        }
+        return result;
+    }
+
+    bool contains(const std::string & path)
+    {
+        const std::string & encoded_key = getEncodedKey(path);
+        std::string buffer_str;
+        rocksdb::Status status = rocksdb_ptr->Get(rocksdb::ReadOptions(), encoded_key, &buffer_str);
+        if (status.IsNotFound())
+            return false;
+        if (!status.ok())
+            throw Exception(ErrorCodes::ROCKSDB_ERROR, "Got rocksdb error during find. The error message is {}.", status.ToString());
+        return true;
+    }
+
+    const_iterator find(StringRef key_)
+    {
+        /// rocksdb::PinnableSlice slice;
+        const std::string & encoded_key = getEncodedKey(key_.toString());
+        std::string buffer_str;
+        rocksdb::Status status = rocksdb_ptr->Get(rocksdb::ReadOptions(), encoded_key, &buffer_str);
+        if (status.IsNotFound())
+            return {};
+        if (!status.ok())
+            throw Exception(ErrorCodes::ROCKSDB_ERROR, "Got rocksdb error during find. The error message is {}.", status.ToString());
+        ReadBufferFromOwnString buffer(buffer_str);
+        KVPair kv;
+        kv.key = key_;
+        typename Node::Meta & meta = kv.value;
+        readPODBinary(meta, buffer);
+        readStringBinary(kv.value.data, buffer);
+        return const_iterator(std::move(kv));
+    }
+
+    const_iterator updateValue(StringRef key_, ValueUpdater updater)
+    {
+        /// rocksdb::PinnableSlice slice;
+        const std::string & key = key_.toString();
+        const std::string & encoded_key = getEncodedKey(key);
+        std::string buffer_str;
+        rocksdb::Status status = rocksdb_ptr->Get(rocksdb::ReadOptions(), encoded_key, &buffer_str);
+        if (!status.ok())
+            throw Exception(ErrorCodes::ROCKSDB_ERROR, "Got rocksdb error during find. The error message is {}.", status.ToString());
+        Node node;
+        node.decodeFromString(buffer_str);
+        /// storage->removeDigest(node, key);
+        updater(node);
+        insertOrReplace(key, node);
+        return const_iterator(KVPair{key_, node});
+    }
+
+    bool insert(const std::string & key, Node & value)
+    {
+        std::string value_str;
+        const std::string & encoded_key = getEncodedKey(key);
+        rocksdb::Status status = rocksdb_ptr->Get(rocksdb::ReadOptions(), encoded_key, &value_str);
+        if (status.ok())
+        {
+            return false;
+        }
+        else if (status.IsNotFound())
+        {
+            // storage->addDigest(value, key);
+            status = rocksdb_ptr->Put(rocksdb::WriteOptions(), encoded_key, value.getEncodedString());
+            if (status.ok())
+            {
+                counter++;
+                return true;
+            }
+        }
+
+        throw Exception(ErrorCodes::ROCKSDB_ERROR, "Got rocksdb error during insert. The error message is {}.", status.ToString());
+    }
+
+    void insertOrReplace(const std::string & key, Node & value)
+    {
+        const std::string & encoded_key = getEncodedKey(key);
+        /// storage->addDigest(value, key);
+        std::string value_str;
+        rocksdb::Status status = rocksdb_ptr->Get(rocksdb::ReadOptions(), encoded_key, &value_str);
+        bool increase_counter = false;
+        if (status.IsNotFound())
+            increase_counter = true;
+        else if (!status.ok())
+            throw Exception(ErrorCodes::ROCKSDB_ERROR, "Got rocksdb error during get. The error message is {}.", status.ToString());
+
+        status = rocksdb_ptr->Put(rocksdb::WriteOptions(), encoded_key, value.getEncodedString());
+        if (status.ok())
+            counter += increase_counter;
+        else
+            throw Exception(ErrorCodes::ROCKSDB_ERROR, "Got rocksdb error during insert. The error message is {}.", status.ToString());
+
+    }
+
+    bool erase(const std::string & key)
+    {
+        /// storage->removeDigest(value, key);
+        auto status = rocksdb_ptr->Delete(rocksdb::WriteOptions(), key);
+        if (status.IsNotFound())
+            return false;
+        if (status.ok())
+        {
+            counter --;
+            return true;
+        }
+        throw Exception(ErrorCodes::ROCKSDB_ERROR, "Got rocksdb error during erase. The error message is {}.", status.ToString());
+    }
+
+    void recalculateDataSize() {}
+
+    uint64_t getApproximateDataSize() const
+    {
+        /// use statistics from rocksdb
+        return counter * sizeof(Node);
+    }
+
+    void enableSnapshotMode(size_t version)
+    {
+        chassert(!snapshot_mode);
+        snapshot_mode = true;
+        snapshot_up_to_version = version;
+        ++current_version;
+
+        snapshot = rocksdb_ptr->GetSnapshot();
+    }
+
+    void disableSnapshotMode()
+    {
+        chassert(snapshot_mode);
+        snapshot_mode = false;
+        rocksdb_ptr->ReleaseSnapshot(snapshot);
+    }
+
+    void clearOutdatedNodes() {}
+
+    std::pair<size_t, size_t> snapshotSizeWithVersion() const
+    {
+        return std::make_pair(counter, current_version);
+    }
+
+    const_iterator begin() const
+    {
+        rocksdb::ReadOptions read_options;
+        read_options.total_order_seek = true;
+        if (snapshot_mode)
+            read_options.snapshot = snapshot;
+        return const_iterator(rocksdb_ptr->NewIterator(read_options));
+    }
+
+    const_iterator end() const
+    {
+        return const_iterator();
+    }
+
+    size_t size() const
+    {
+        return counter;
+    }
+
+    uint64_t getArenaDataSize() const
+    {
+        return 0;
+    }
+
+    uint64_t keyArenaSize() const
+    {
+        return 0;
+    }
+
+private:
+    String rocksdb_dir;
+
+    std::unique_ptr<rocksdb::DB> rocksdb_ptr;
+
+    /// Storage* storage_ptr;
+
+    const rocksdb::Snapshot * snapshot;
+
+    bool snapshot_mode{false};
+    size_t current_version{0};
+    size_t snapshot_up_to_version{0};
+    size_t counter{0};
+
+};
+
+}

From 89ff11e967179b6005773b859136b9f909ba86b2 Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Wed, 17 Jan 2024 15:40:57 +0100
Subject: [PATCH 0010/1488] fix style

---
 src/Coordination/KeeperStorage.cpp  | 3 ++-
 src/Coordination/RocksDBContainer.h | 1 -
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Coordination/KeeperStorage.cpp b/src/Coordination/KeeperStorage.cpp
index 0d9efc87869..c115546ca71 100644
--- a/src/Coordination/KeeperStorage.cpp
+++ b/src/Coordination/KeeperStorage.cpp
@@ -856,7 +856,8 @@ bool KeeperStorage<Container>::removeNode(const std::string & path, int32_t vers
 
     if constexpr (use_rocksdb)
         container.erase(path);
-    else {
+    else
+    {
         container.updateValue(
             parentNodePath(path),
             [child_basename = getBaseNodeName(node_it->key)](KeeperMemNode & parent)
diff --git a/src/Coordination/RocksDBContainer.h b/src/Coordination/RocksDBContainer.h
index b357bcb2dab..a0eaa0b6ae7 100644
--- a/src/Coordination/RocksDBContainer.h
+++ b/src/Coordination/RocksDBContainer.h
@@ -17,7 +17,6 @@ namespace DB
 
 namespace ErrorCodes
 {
-    extern const int LOGICAL_ERROR;
     extern const int ROCKSDB_ERROR;
 }
 

From 8cddb8163480e6c77d7748d07c922d7ec7aba76d Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Wed, 17 Jan 2024 17:39:25 +0100
Subject: [PATCH 0011/1488] fix build

---
 src/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 083b959c4b6..9340e070eb2 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -131,6 +131,7 @@ endif()
 
 if (TARGET ch_contrib::rocksdb)
     add_headers_and_sources(dbms Storages/RocksDB)
+    add_headers_and_sources(dbms Coordination)
 endif()
 
 if (TARGET ch_contrib::aws_s3)

From 60674094436e8eee02f61593f23d75b17dd5de3d Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Mon, 5 Feb 2024 08:57:30 +0100
Subject: [PATCH 0012/1488] fix ut

---
 src/Coordination/KeeperSnapshotManager.cpp | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/Coordination/KeeperSnapshotManager.cpp b/src/Coordination/KeeperSnapshotManager.cpp
index 710c3be77b9..86158ee5ab6 100644
--- a/src/Coordination/KeeperSnapshotManager.cpp
+++ b/src/Coordination/KeeperSnapshotManager.cpp
@@ -427,9 +427,12 @@ void KeeperStorageSnapshot<Storage>::deserialize(SnapshotDeserializationResult<S
     {
         for (const auto & itr : storage.container)
         {
-            auto parent_path = parentNodePath(itr.key);
-            storage.container.updateValue(
-                parent_path, [path = itr.key](typename Storage::Node & value) { value.addChild(getBaseNodeName(path)); });
+            if (itr.key != "/")
+            {
+                auto parent_path = parentNodePath(itr.key);
+                storage.container.updateValue(
+                    parent_path, [path = itr.key](typename Storage::Node & value) { value.addChild(getBaseNodeName(path)); });
+            }
         }
 
         for (const auto & itr : storage.container)

From 72ac59f44faad849a21a4d857c84637e2bb068a5 Mon Sep 17 00:00:00 2001
From: Alexander Tokmakov <tavplubix@clickhouse.com>
Date: Fri, 8 Mar 2024 21:37:30 +0100
Subject: [PATCH 0013/1488] add signal handlers in client and local

---
 src/Client/ClientBase.cpp     |  17 +-
 src/Client/ClientBase.h       |   6 +
 src/Common/SignalHandlers.cpp | 592 ++++++++++++++++++++++++++++++++
 src/Common/SignalHandlers.h   | 121 +++++++
 src/Daemon/BaseDaemon.cpp     | 611 +---------------------------------
 src/Daemon/BaseDaemon.h       |   2 -
 6 files changed, 744 insertions(+), 605 deletions(-)
 create mode 100644 src/Common/SignalHandlers.cpp
 create mode 100644 src/Common/SignalHandlers.h

diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp
index 48962880b8f..6ee76576515 100644
--- a/src/Client/ClientBase.cpp
+++ b/src/Client/ClientBase.cpp
@@ -22,6 +22,7 @@
 #include <Common/StringUtils/StringUtils.h>
 #include <Common/filesystemHelpers.h>
 #include <Common/NetException.h>
+#include <Common/SignalHandlers.h>
 #include <Columns/ColumnString.h>
 #include <Columns/ColumnsNumber.h>
 #include <Formats/FormatFactory.h>
@@ -300,7 +301,13 @@ public:
 };
 
 
-ClientBase::~ClientBase() = default;
+ClientBase::~ClientBase()
+{
+    writeSignalIDtoSignalPipe(SignalListener::StopThread);
+    signal_listener_thread.join();
+    HandledSignals::instance().reset();
+}
+
 ClientBase::ClientBase() = default;
 
 
@@ -3007,6 +3014,14 @@ void ClientBase::init(int argc, char ** argv)
     }
 
     has_log_comment = config().has("log_comment");
+
+    /// Print stacktrace in case of crash
+    HandledSignals::instance().setupCommonDeadlySignalHandlers();
+
+    fatal_channel_ptr = new Poco::ConsoleChannel;
+    fatal_log = createLogger("ClientBase", fatal_channel_ptr.get(), Poco::Message::PRIO_TRACE);
+    signal_listener = std::make_unique<SignalListener>(nullptr, fatal_log);
+    signal_listener_thread.start(*signal_listener);
 }
 
 }
diff --git a/src/Client/ClientBase.h b/src/Client/ClientBase.h
index dd08e7c059b..d7e93be9435 100644
--- a/src/Client/ClientBase.h
+++ b/src/Client/ClientBase.h
@@ -10,6 +10,7 @@
 #include <Common/DNSResolver.h>
 #include <Core/ExternalTable.h>
 #include <Poco/Util/Application.h>
+#include <Poco/ConsoleChannel.h>
 #include <Interpreters/Context.h>
 #include <Client/Suggest.h>
 #include <Client/QueryFuzzer.h>
@@ -192,6 +193,11 @@ protected:
     SharedContextHolder shared_context;
     ContextMutablePtr global_context;
 
+    LoggerPtr fatal_log;
+    Poco::AutoPtr<Poco::Channel> fatal_channel_ptr;
+    Poco::Thread signal_listener_thread;
+    std::unique_ptr<Poco::Runnable> signal_listener;
+
     bool is_interactive = false; /// Use either interactive line editing interface or batch mode.
     bool is_multiquery = false;
     bool delayed_interactive = false;
diff --git a/src/Common/SignalHandlers.cpp b/src/Common/SignalHandlers.cpp
new file mode 100644
index 00000000000..9270320d536
--- /dev/null
+++ b/src/Common/SignalHandlers.cpp
@@ -0,0 +1,592 @@
+#include <Common/SignalHandlers.h>
+#include <Common/config_version.h>
+#include <Common/getHashOfLoadedBinary.h>
+#include <Daemon/BaseDaemon.h>
+#include <Daemon/SentryWriter.h>
+#include <base/sleep.h>
+#include <base/getThreadId.h>
+#include <IO/WriteBufferFromFileDescriptor.h>
+#include <IO/ReadBufferFromFileDescriptor.h>
+#include <IO/WriteBufferFromFileDescriptorDiscardOnFailure.h>
+#include <IO/WriteHelpers.h>
+#include <IO/ReadHelpers.h>
+#include <Interpreters/Context.h>
+
+namespace DB
+{
+namespace ErrorCodes
+{
+extern const int CANNOT_SET_SIGNAL_HANDLER;
+extern const int CANNOT_SEND_SIGNAL;
+}
+}
+
+using namespace DB;
+
+
+void call_default_signal_handler(int sig)
+{
+    if (SIG_ERR == signal(sig, SIG_DFL))
+        throw ErrnoException(ErrorCodes::CANNOT_SET_SIGNAL_HANDLER, "Cannot set signal handler");
+
+    if (0 != raise(sig))
+        throw ErrnoException(ErrorCodes::CANNOT_SEND_SIGNAL, "Cannot send signal");
+}
+
+
+void writeSignalIDtoSignalPipe(int sig)
+{
+    auto saved_errno = errno;   /// We must restore previous value of errno in signal handler.
+
+    char buf[signal_pipe_buf_size];
+    auto & signal_pipe = HandledSignals::instance().signal_pipe;
+    WriteBufferFromFileDescriptor out(signal_pipe.fds_rw[1], signal_pipe_buf_size, buf);
+    writeBinary(sig, out);
+    out.next();
+
+    errno = saved_errno;
+}
+
+void closeLogsSignalHandler(int sig, siginfo_t *, void *)
+{
+    DENY_ALLOCATIONS_IN_SCOPE;
+    writeSignalIDtoSignalPipe(sig);
+}
+
+void terminateRequestedSignalHandler(int sig, siginfo_t *, void *)
+{
+    DENY_ALLOCATIONS_IN_SCOPE;
+    writeSignalIDtoSignalPipe(sig);
+}
+
+
+void signalHandler(int sig, siginfo_t * info, void * context)
+{
+    DENY_ALLOCATIONS_IN_SCOPE;
+    auto saved_errno = errno;   /// We must restore previous value of errno in signal handler.
+
+    char buf[signal_pipe_buf_size];
+    auto & signal_pipe = HandledSignals::instance().signal_pipe;
+    WriteBufferFromFileDescriptorDiscardOnFailure out(signal_pipe.fds_rw[1], signal_pipe_buf_size, buf);
+
+    const ucontext_t * signal_context = reinterpret_cast<ucontext_t *>(context);
+    const StackTrace stack_trace(*signal_context);
+
+    writeBinary(sig, out);
+    writePODBinary(*info, out);
+    writePODBinary(signal_context, out);
+    writePODBinary(stack_trace, out);
+    writeVectorBinary(Exception::enable_job_stack_trace ? Exception::thread_frame_pointers : std::vector<StackTrace::FramePointers>{}, out);
+    writeBinary(static_cast<UInt32>(getThreadId()), out);
+    writePODBinary(current_thread, out);
+
+    out.next();
+
+    if (sig != SIGTSTP) /// This signal is used for debugging.
+    {
+        /// The time that is usually enough for separate thread to print info into log.
+        /// Under MSan full stack unwinding with DWARF info about inline functions takes 101 seconds in one case.
+        for (size_t i = 0; i < 300; ++i)
+        {
+            /// We will synchronize with the thread printing the messages with an atomic variable to finish earlier.
+            if (HandledSignals::instance().fatal_error_printed.test())
+                break;
+
+            /// This coarse method of synchronization is perfectly ok for fatal signals.
+            sleepForSeconds(1);
+        }
+
+        /// Wait for all logs flush operations
+        sleepForSeconds(3);
+        call_default_signal_handler(sig);
+    }
+
+    errno = saved_errno;
+}
+
+
+[[noreturn]] void terminate_handler()
+{
+    static thread_local bool terminating = false;
+    if (terminating)
+        abort();
+
+    terminating = true;
+
+    std::string log_message;
+
+    if (std::current_exception())
+        log_message = "Terminate called for uncaught exception:\n" + getCurrentExceptionMessage(true);
+    else
+        log_message = "Terminate called without an active exception";
+
+    /// POSIX.1 says that write(2)s of less than PIPE_BUF bytes must be atomic - man 7 pipe
+    /// And the buffer should not be too small because our exception messages can be large.
+    static constexpr size_t buf_size = PIPE_BUF;
+
+    if (log_message.size() > buf_size - 16)
+        log_message.resize(buf_size - 16);
+
+    char buf[buf_size];
+    auto & signal_pipe = HandledSignals::instance().signal_pipe;
+    WriteBufferFromFileDescriptor out(signal_pipe.fds_rw[1], buf_size, buf);
+
+    writeBinary(static_cast<int>(SignalListener::StdTerminate), out);
+    writeBinary(static_cast<UInt32>(getThreadId()), out);
+    writeBinary(log_message, out);
+    out.next();
+
+    abort();
+}
+
+#if defined(SANITIZER)
+static DISABLE_SANITIZER_INSTRUMENTATION void sanitizerDeathCallback()
+{
+    DENY_ALLOCATIONS_IN_SCOPE;
+    /// Also need to send data via pipe. Otherwise it may lead to deadlocks or failures in printing diagnostic info.
+
+    char buf[signal_pipe_buf_size];
+    auto & signal_pipe = HandledSignals::instance().signal_pipe;
+    WriteBufferFromFileDescriptorDiscardOnFailure out(signal_pipe.fds_rw[1], signal_pipe_buf_size, buf);
+
+    const StackTrace stack_trace;
+
+    int sig = SignalListener::SanitizerTrap;
+    writeBinary(sig, out);
+    writePODBinary(stack_trace, out);
+    writeBinary(UInt32(getThreadId()), out);
+    writePODBinary(current_thread, out);
+
+    out.next();
+
+    /// The time that is usually enough for separate thread to print info into log.
+    sleepForSeconds(20);
+}
+#endif
+
+
+void HandledSignals::addSignalHandler(const std::vector<int> & signals, signal_function handler, bool register_signal)
+{
+    struct sigaction sa;
+    memset(&sa, 0, sizeof(sa));
+    sa.sa_sigaction = handler;
+    sa.sa_flags = SA_SIGINFO;
+
+#if defined(OS_DARWIN)
+    sigemptyset(&sa.sa_mask);
+    for (auto signal : signals)
+        sigaddset(&sa.sa_mask, signal);
+#else
+    if (sigemptyset(&sa.sa_mask))
+        throw Poco::Exception("Cannot set signal handler.");
+
+    for (auto signal : signals)
+        if (sigaddset(&sa.sa_mask, signal))
+            throw Poco::Exception("Cannot set signal handler.");
+#endif
+
+    for (auto signal : signals)
+        if (sigaction(signal, &sa, nullptr))
+            throw Poco::Exception("Cannot set signal handler.");
+
+    if (register_signal)
+        std::copy(signals.begin(), signals.end(), std::back_inserter(handled_signals));
+}
+
+void blockSignals(const std::vector<int> & signals)
+{
+    sigset_t sig_set;
+
+#if defined(OS_DARWIN)
+    sigemptyset(&sig_set);
+    for (auto signal : signals)
+        sigaddset(&sig_set, signal);
+#else
+    if (sigemptyset(&sig_set))
+        throw Poco::Exception("Cannot block signal.");
+
+    for (auto signal : signals)
+        if (sigaddset(&sig_set, signal))
+            throw Poco::Exception("Cannot block signal.");
+#endif
+
+    if (pthread_sigmask(SIG_BLOCK, &sig_set, nullptr))
+        throw Poco::Exception("Cannot block signal.");
+}
+
+
+void SignalListener::run()
+{
+    static_assert(PIPE_BUF >= 512);
+    static_assert(signal_pipe_buf_size <= PIPE_BUF, "Only write of PIPE_BUF to pipe is atomic and the minimal known PIPE_BUF across supported platforms is 512");
+    char buf[signal_pipe_buf_size];
+    auto & signal_pipe = HandledSignals::instance().signal_pipe;
+    ReadBufferFromFileDescriptor in(signal_pipe.fds_rw[0], signal_pipe_buf_size, buf);
+
+    while (!in.eof())
+    {
+        int sig = 0;
+        readBinary(sig, in);
+        // We may log some specific signals afterwards, with different log
+        // levels and more info, but for completeness we log all signals
+        // here at trace level.
+        // Don't use strsignal here, because it's not thread-safe.
+        LOG_TRACE(log, "Received signal {}", sig);
+
+        if (sig == StopThread)
+        {
+            LOG_INFO(log, "Stop SignalListener thread");
+            break;
+        }
+        else if (sig == SIGHUP)
+        {
+            LOG_DEBUG(log, "Received signal to close logs.");
+            BaseDaemon::instance().closeLogs(BaseDaemon::instance().logger());
+            LOG_INFO(log, "Opened new log file after received signal.");
+        }
+        else if (sig == StdTerminate)
+        {
+            UInt32 thread_num;
+            std::string message;
+
+            readBinary(thread_num, in);
+            readBinary(message, in);
+
+            onTerminate(message, thread_num);
+        }
+        else if (sig == SIGINT ||
+                 sig == SIGQUIT ||
+                 sig == SIGTERM)
+        {
+            if (daemon)
+                daemon->handleSignal(sig);
+        }
+        else
+        {
+            siginfo_t info{};
+            ucontext_t * context{};
+            StackTrace stack_trace(NoCapture{});
+            std::vector<StackTrace::FramePointers> thread_frame_pointers;
+            UInt32 thread_num{};
+            ThreadStatus * thread_ptr{};
+
+            if (sig != SanitizerTrap)
+            {
+                readPODBinary(info, in);
+                readPODBinary(context, in);
+            }
+
+            readPODBinary(stack_trace, in);
+            readVectorBinary(thread_frame_pointers, in);
+            readBinary(thread_num, in);
+            readPODBinary(thread_ptr, in);
+
+            /// This allows to receive more signals if failure happens inside onFault function.
+            /// Example: segfault while symbolizing stack trace.
+            try
+            {
+                std::thread([=, this] { onFault(sig, info, context, stack_trace, thread_frame_pointers, thread_num, thread_ptr); }).detach();
+            }
+            catch (...)
+            {
+                /// Likely cannot allocate thread
+                onFault(sig, info, context, stack_trace, thread_frame_pointers, thread_num, thread_ptr);
+            }
+        }
+    }
+}
+
+void SignalListener::onTerminate(std::string_view message, UInt32 thread_num) const
+{
+    size_t pos = message.find('\n');
+
+    LOG_FATAL(log, "(version {}{}, build id: {}, git hash: {}) (from thread {}) {}",
+              VERSION_STRING, VERSION_OFFICIAL, daemon ? daemon->build_id : "", daemon ? daemon->git_hash : "", thread_num, message.substr(0, pos));
+
+    /// Print trace from std::terminate exception line-by-line to make it easy for grep.
+    while (pos != std::string_view::npos)
+    {
+        ++pos;
+        size_t next_pos = message.find('\n', pos);
+        size_t size = next_pos;
+        if (next_pos != std::string_view::npos)
+            size = next_pos - pos;
+
+        LOG_FATAL(log, fmt::runtime(message.substr(pos, size)));
+        pos = next_pos;
+    }
+}
+
+void SignalListener::onFault(
+    int sig,
+    const siginfo_t & info,
+    ucontext_t * context,
+    const StackTrace & stack_trace,
+    const std::vector<StackTrace::FramePointers> & thread_frame_pointers,
+    UInt32 thread_num,
+    DB::ThreadStatus * thread_ptr) const
+{
+    ThreadStatus thread_status;
+
+    /// First log those fields that are safe to access and that should not cause new fault.
+    /// That way we will have some duplicated info in the log but we don't loose important info
+    /// in case of double fault.
+
+    LOG_FATAL(log, "########## Short fault info ############");
+    LOG_FATAL(log, "(version {}{}, build id: {}, git hash: {}) (from thread {}) Received signal {}",
+              VERSION_STRING, VERSION_OFFICIAL, daemon ? daemon->build_id : "", daemon ? daemon->git_hash : "",
+              thread_num, sig);
+
+    std::string signal_description = "Unknown signal";
+
+    /// Some of these are not really signals, but our own indications on failure reason.
+    if (sig == StdTerminate)
+        signal_description = "std::terminate";
+    else if (sig == SanitizerTrap)
+        signal_description = "sanitizer trap";
+    else if (sig >= 0)
+        signal_description = strsignal(sig); // NOLINT(concurrency-mt-unsafe) // it is not thread-safe but ok in this context
+
+    LOG_FATAL(log, "Signal description: {}", signal_description);
+
+    String error_message;
+
+    if (sig != SanitizerTrap)
+        error_message = signalToErrorMessage(sig, info, *context);
+    else
+        error_message = "Sanitizer trap.";
+
+    LOG_FATAL(log, fmt::runtime(error_message));
+
+    String bare_stacktrace_str;
+    if (stack_trace.getSize())
+    {
+        /// Write bare stack trace (addresses) just in case if we will fail to print symbolized stack trace.
+        /// NOTE: This still require memory allocations and mutex lock inside logger.
+        ///       BTW we can also print it to stderr using write syscalls.
+
+        WriteBufferFromOwnString bare_stacktrace;
+        writeString("Stack trace:", bare_stacktrace);
+        for (size_t i = stack_trace.getOffset(); i < stack_trace.getSize(); ++i)
+        {
+            writeChar(' ', bare_stacktrace);
+            writePointerHex(stack_trace.getFramePointers()[i], bare_stacktrace);
+        }
+
+        LOG_FATAL(log, fmt::runtime(bare_stacktrace.str()));
+        bare_stacktrace_str = bare_stacktrace.str();
+    }
+
+    /// Now try to access potentially unsafe data in thread_ptr.
+
+    String query_id;
+    String query;
+
+    /// Send logs from this thread to client if possible.
+    /// It will allow client to see failure messages directly.
+    if (thread_ptr)
+    {
+        query_id = thread_ptr->getQueryId();
+        query = thread_ptr->getQueryForLog();
+
+        if (auto logs_queue = thread_ptr->getInternalTextLogsQueue())
+        {
+            CurrentThread::attachInternalTextLogsQueue(logs_queue, LogsLevel::trace);
+        }
+    }
+
+    LOG_FATAL(log, "########################################");
+
+    if (query_id.empty())
+    {
+        LOG_FATAL(log, "(version {}{}, build id: {}, git hash: {}) (from thread {}) (no query) Received signal {} ({})",
+                  VERSION_STRING, VERSION_OFFICIAL, daemon ? daemon->build_id : "", daemon ? daemon->git_hash : "",
+                  thread_num, signal_description, sig);
+    }
+    else
+    {
+        LOG_FATAL(log, "(version {}{}, build id: {}, git hash: {}) (from thread {}) (query_id: {}) (query: {}) Received signal {} ({})",
+                  VERSION_STRING, VERSION_OFFICIAL, daemon ? daemon->build_id : "", daemon ? daemon->git_hash : "",
+                  thread_num, query_id, query, signal_description, sig);
+    }
+
+    LOG_FATAL(log, fmt::runtime(error_message));
+
+    if (!bare_stacktrace_str.empty())
+    {
+        LOG_FATAL(log, fmt::runtime(bare_stacktrace_str));
+    }
+
+    /// Write symbolized stack trace line by line for better grep-ability.
+    stack_trace.toStringEveryLine([&](std::string_view s) { LOG_FATAL(log, fmt::runtime(s)); });
+
+    /// In case it's a scheduled job write all previous jobs origins call stacks
+    std::for_each(thread_frame_pointers.rbegin(), thread_frame_pointers.rend(),
+                  [this](const StackTrace::FramePointers & frame_pointers)
+                  {
+                      if (size_t size = std::ranges::find(frame_pointers, nullptr) - frame_pointers.begin())
+                      {
+                          LOG_FATAL(log, "========================================");
+                          WriteBufferFromOwnString bare_stacktrace;
+                          writeString("Job's origin stack trace:", bare_stacktrace);
+                          std::for_each_n(frame_pointers.begin(), size,
+                                          [&bare_stacktrace](const void * ptr)
+                                          {
+                                              writeChar(' ', bare_stacktrace);
+                                              writePointerHex(ptr, bare_stacktrace);
+                                          }
+                          );
+
+                          LOG_FATAL(log, fmt::runtime(bare_stacktrace.str()));
+
+                          StackTrace::toStringEveryLine(const_cast<void **>(frame_pointers.data()), 0, size, [this](std::string_view s) { LOG_FATAL(log, fmt::runtime(s)); });
+                      }
+                  }
+    );
+
+
+#if defined(OS_LINUX)
+    /// Write information about binary checksum. It can be difficult to calculate, so do it only after printing stack trace.
+    /// Please keep the below log messages in-sync with the ones in programs/server/Server.cpp
+
+    if (daemon && daemon->stored_binary_hash.empty())
+    {
+        LOG_FATAL(log, "Integrity check of the executable skipped because the reference checksum could not be read.");
+    }
+    else if (daemon)
+    {
+        String calculated_binary_hash = getHashOfLoadedBinaryHex();
+        if (calculated_binary_hash == daemon->stored_binary_hash)
+        {
+            LOG_FATAL(log, "Integrity check of the executable successfully passed (checksum: {})", calculated_binary_hash);
+        }
+        else
+        {
+            LOG_FATAL(
+                log,
+                "Calculated checksum of the executable ({0}) does not correspond"
+                " to the reference checksum stored in the executable ({1})."
+                " This may indicate one of the following:"
+                " - the executable was changed just after startup;"
+                " - the executable was corrupted on disk due to faulty hardware;"
+                " - the loaded executable was corrupted in memory due to faulty hardware;"
+                " - the file was intentionally modified;"
+                " - a logical error in the code.",
+                calculated_binary_hash,
+                daemon->stored_binary_hash);
+        }
+    }
+#endif
+
+    /// Write crash to system.crash_log table if available.
+    if (collectCrashLog)
+        collectCrashLog(sig, thread_num, query_id, stack_trace);
+
+#ifndef CLICKHOUSE_KEEPER_STANDALONE_BUILD
+    Context::getGlobalContextInstance()->handleCrash();
+#endif
+
+    /// Send crash report to developers (if configured)
+    if (sig != SanitizerTrap)
+    {
+        if (daemon)
+            SentryWriter::onFault(sig, error_message, stack_trace);
+
+        /// Advice the user to send it manually.
+        if (std::string_view(VERSION_OFFICIAL).contains("official build"))
+        {
+            const auto & date_lut = DateLUT::instance();
+
+            /// Approximate support period, upper bound.
+            if (time(nullptr) - date_lut.makeDate(2000 + VERSION_MAJOR, VERSION_MINOR, 1) < (365 + 30) * 86400)
+            {
+                LOG_FATAL(log, "Report this error to https://github.com/ClickHouse/ClickHouse/issues");
+            }
+            else
+            {
+                LOG_FATAL(log, "ClickHouse version {} is old and should be upgraded to the latest version.", VERSION_STRING);
+            }
+        }
+        else
+        {
+            LOG_FATAL(log, "This ClickHouse version is not official and should be upgraded to the official build.");
+        }
+    }
+
+    /// ClickHouse Keeper does not link to some part of Settings.
+#ifndef CLICKHOUSE_KEEPER_STANDALONE_BUILD
+    /// List changed settings.
+    if (!query_id.empty())
+    {
+        ContextPtr query_context = thread_ptr->getQueryContext();
+        if (query_context)
+        {
+            String changed_settings = query_context->getSettingsRef().toString();
+
+            if (changed_settings.empty())
+                LOG_FATAL(log, "No settings were changed");
+            else
+                LOG_FATAL(log, "Changed settings: {}", changed_settings);
+        }
+    }
+#endif
+
+    /// When everything is done, we will try to send these error messages to client.
+    if (thread_ptr)
+        thread_ptr->onFatalError();
+
+    HandledSignals::instance().fatal_error_printed.test_and_set();
+}
+
+
+HandledSignals::HandledSignals()
+{
+    signal_pipe.setNonBlockingWrite();
+    signal_pipe.tryIncreaseSize(1 << 20);
+}
+
+void HandledSignals::reset()
+{
+    /// Reset signals to SIG_DFL to avoid trying to write to the signal_pipe that will be closed after.
+    for (int sig : handled_signals)
+    {
+        if (SIG_ERR == signal(sig, SIG_DFL))
+        {
+            try
+            {
+                throw ErrnoException(ErrorCodes::CANNOT_SET_SIGNAL_HANDLER, "Cannot set signal handler");
+            }
+            catch (ErrnoException &)
+            {
+                tryLogCurrentException(__PRETTY_FUNCTION__);
+            }
+        }
+    }
+
+    signal_pipe.close();
+}
+
+HandledSignals::~HandledSignals()
+{
+    reset();
+};
+
+HandledSignals & HandledSignals::instance()
+{
+    static HandledSignals res;
+    return res;
+}
+
+void HandledSignals::setupCommonDeadlySignalHandlers()
+{
+    std::set_terminate(terminate_handler);
+
+    /// SIGTSTP is added for debugging purposes. To output a stack trace of any running thread at anytime.
+    addSignalHandler({SIGABRT, SIGSEGV, SIGILL, SIGBUS, SIGSYS, SIGFPE, SIGPIPE, SIGTSTP, SIGTRAP}, signalHandler, true);
+    addSignalHandler({SIGINT, SIGQUIT, SIGTERM}, terminateRequestedSignalHandler, true);
+
+#if defined(SANITIZER)
+    __sanitizer_set_death_callback(sanitizerDeathCallback);
+#endif
+}
+
diff --git a/src/Common/SignalHandlers.h b/src/Common/SignalHandlers.h
new file mode 100644
index 00000000000..b7af9dc2943
--- /dev/null
+++ b/src/Common/SignalHandlers.h
@@ -0,0 +1,121 @@
+#pragma once
+#include <csignal>
+
+#include <base/defines.h>
+#include <Common/PipeFDs.h>
+#include <Common/StackTrace.h>
+#include <Common/ThreadStatus.h>
+#include <Core/Types.h>
+#include <Poco/Runnable.h>
+
+class BaseDaemon;
+
+/** Reset signal handler to the default and send signal to itself.
+  * It's called from user signal handler to write core dump.
+  */
+void call_default_signal_handler(int sig);
+
+const size_t signal_pipe_buf_size =
+    sizeof(int)
+    + sizeof(siginfo_t)
+    + sizeof(ucontext_t*)
+    + sizeof(StackTrace)
+    + sizeof(UInt64)
+    + sizeof(UInt32)
+    + sizeof(void*);
+
+using signal_function = void(int, siginfo_t*, void*);
+
+void writeSignalIDtoSignalPipe(int sig);
+
+/** Signal handler for HUP */
+void closeLogsSignalHandler(int sig, siginfo_t *, void *);
+
+void terminateRequestedSignalHandler(int sig, siginfo_t *, void *);
+
+
+/** Handler for "fault" or diagnostic signals. Send data about fault to separate thread to write into log.
+  */
+void signalHandler(int sig, siginfo_t * info, void * context);
+
+
+/** To use with std::set_terminate.
+  * Collects slightly more info than __gnu_cxx::__verbose_terminate_handler,
+  *  and send it to pipe. Other thread will read this info from pipe and asynchronously write it to log.
+  * Look at libstdc++-v3/libsupc++/vterminate.cc for example.
+  */
+[[noreturn]] void terminate_handler();
+
+#if defined(SANITIZER)
+extern "C" void __sanitizer_set_death_callback(void (*)());
+
+/// Sanitizers may not expect some function calls from death callback.
+/// Let's try to disable instrumentation to avoid possible issues.
+/// However, this callback may call other functions that are still instrumented.
+/// We can try [[clang::always_inline]] attribute for statements in future (available in clang-15)
+/// See https://github.com/google/sanitizers/issues/1543 and https://github.com/google/sanitizers/issues/1549.
+DISABLE_SANITIZER_INSTRUMENTATION void sanitizerDeathCallback();
+#endif
+
+
+/// Avoid link time dependency on DB/Interpreters - will use this function only when linked.
+__attribute__((__weak__)) void collectCrashLog(
+    Int32 signal, UInt64 thread_id, const String & query_id, const StackTrace & stack_trace);
+
+
+void blockSignals(const std::vector<int> & signals);
+
+
+/** The thread that read info about signal or std::terminate from pipe.
+  * On HUP, close log files (for new files to be opened later).
+  * On information about std::terminate, write it to log.
+  * On other signals, write info to log.
+  */
+class SignalListener : public Poco::Runnable
+{
+public:
+    static constexpr int StdTerminate = -1;
+    static constexpr int StopThread = -2;
+    static constexpr int SanitizerTrap = -3;
+
+    explicit SignalListener(BaseDaemon * daemon_, LoggerPtr log_)
+        : daemon(daemon_), log(log_)
+    {
+    }
+
+    void run() override;
+
+private:
+    BaseDaemon * daemon;
+    LoggerPtr log;
+
+    void onTerminate(std::string_view message, UInt32 thread_num) const;
+
+    void onFault(
+        int sig,
+        const siginfo_t & info,
+        ucontext_t * context,
+        const StackTrace & stack_trace,
+        const std::vector<StackTrace::FramePointers> & thread_frame_pointers,
+        UInt32 thread_num,
+        DB::ThreadStatus * thread_ptr) const;
+};
+
+struct HandledSignals
+{
+    std::vector<int> handled_signals;
+    DB::PipeFDs signal_pipe;
+    std::atomic_flag fatal_error_printed;
+
+    HandledSignals();
+    ~HandledSignals();
+
+    void setupCommonDeadlySignalHandlers();
+
+    void addSignalHandler(const std::vector<int> & signals, signal_function handler, bool register_signal);
+
+    void reset();
+
+    static HandledSignals & instance();
+};
+
diff --git a/src/Daemon/BaseDaemon.cpp b/src/Daemon/BaseDaemon.cpp
index 289a41bb75e..15ab92464a0 100644
--- a/src/Daemon/BaseDaemon.cpp
+++ b/src/Daemon/BaseDaemon.cpp
@@ -16,7 +16,6 @@
 #endif
 #include <cerrno>
 #include <cstring>
-#include <csignal>
 #include <unistd.h>
 
 #include <algorithm>
@@ -33,6 +32,7 @@
 #include <Poco/Pipe.h>
 
 #include <Common/ErrorHandlers.h>
+#include <Common/SignalHandlers.h>
 #include <base/argsToConfig.h>
 #include <base/getThreadId.h>
 #include <base/coverage.h>
@@ -50,7 +50,6 @@
 #include <Common/Config/ConfigProcessor.h>
 #include <Common/SymbolIndex.h>
 #include <Common/getExecutablePath.h>
-#include <Common/getHashOfLoadedBinary.h>
 #include <Common/Elf.h>
 #include <Common/setThreadName.h>
 #include <Common/logger_useful.h>
@@ -75,112 +74,12 @@ namespace DB
 {
     namespace ErrorCodes
     {
-        extern const int CANNOT_SET_SIGNAL_HANDLER;
-        extern const int CANNOT_SEND_SIGNAL;
         extern const int SYSTEM_ERROR;
     }
 }
 
 using namespace DB;
 
-PipeFDs signal_pipe;
-
-
-/** Reset signal handler to the default and send signal to itself.
-  * It's called from user signal handler to write core dump.
-  */
-static void call_default_signal_handler(int sig)
-{
-    if (SIG_ERR == signal(sig, SIG_DFL))
-        throw ErrnoException(ErrorCodes::CANNOT_SET_SIGNAL_HANDLER, "Cannot set signal handler");
-
-    if (0 != raise(sig))
-        throw ErrnoException(ErrorCodes::CANNOT_SEND_SIGNAL, "Cannot send signal");
-}
-
-static const size_t signal_pipe_buf_size =
-    sizeof(int)
-    + sizeof(siginfo_t)
-    + sizeof(ucontext_t*)
-    + sizeof(StackTrace)
-    + sizeof(UInt64)
-    + sizeof(UInt32)
-    + sizeof(void*);
-
-using signal_function = void(int, siginfo_t*, void*);
-
-static void writeSignalIDtoSignalPipe(int sig)
-{
-    auto saved_errno = errno;   /// We must restore previous value of errno in signal handler.
-
-    char buf[signal_pipe_buf_size];
-    WriteBufferFromFileDescriptor out(signal_pipe.fds_rw[1], signal_pipe_buf_size, buf);
-    writeBinary(sig, out);
-    out.next();
-
-    errno = saved_errno;
-}
-
-/** Signal handler for HUP */
-static void closeLogsSignalHandler(int sig, siginfo_t *, void *)
-{
-    DENY_ALLOCATIONS_IN_SCOPE;
-    writeSignalIDtoSignalPipe(sig);
-}
-
-static void terminateRequestedSignalHandler(int sig, siginfo_t *, void *)
-{
-    DENY_ALLOCATIONS_IN_SCOPE;
-    writeSignalIDtoSignalPipe(sig);
-}
-
-
-static std::atomic_flag fatal_error_printed;
-
-/** Handler for "fault" or diagnostic signals. Send data about fault to separate thread to write into log.
-  */
-static void signalHandler(int sig, siginfo_t * info, void * context)
-{
-    DENY_ALLOCATIONS_IN_SCOPE;
-    auto saved_errno = errno;   /// We must restore previous value of errno in signal handler.
-
-    char buf[signal_pipe_buf_size];
-    WriteBufferFromFileDescriptorDiscardOnFailure out(signal_pipe.fds_rw[1], signal_pipe_buf_size, buf);
-
-    const ucontext_t * signal_context = reinterpret_cast<ucontext_t *>(context);
-    const StackTrace stack_trace(*signal_context);
-
-    writeBinary(sig, out);
-    writePODBinary(*info, out);
-    writePODBinary(signal_context, out);
-    writePODBinary(stack_trace, out);
-    writeVectorBinary(Exception::enable_job_stack_trace ? Exception::thread_frame_pointers : std::vector<StackTrace::FramePointers>{}, out);
-    writeBinary(static_cast<UInt32>(getThreadId()), out);
-    writePODBinary(current_thread, out);
-
-    out.next();
-
-    if (sig != SIGTSTP) /// This signal is used for debugging.
-    {
-        /// The time that is usually enough for separate thread to print info into log.
-        /// Under MSan full stack unwinding with DWARF info about inline functions takes 101 seconds in one case.
-        for (size_t i = 0; i < 300; ++i)
-        {
-            /// We will synchronize with the thread printing the messages with an atomic variable to finish earlier.
-            if (fatal_error_printed.test())
-                break;
-
-            /// This coarse method of synchronization is perfectly ok for fatal signals.
-            sleepForSeconds(1);
-        }
-
-        /// Wait for all logs flush operations
-        sleepForSeconds(3);
-        call_default_signal_handler(sig);
-    }
-
-    errno = saved_errno;
-}
 
 static bool getenvBool(const char * name)
 {
@@ -192,426 +91,6 @@ static bool getenvBool(const char * name)
 }
 
 
-/// Avoid link time dependency on DB/Interpreters - will use this function only when linked.
-__attribute__((__weak__)) void collectCrashLog(
-    Int32 signal, UInt64 thread_id, const String & query_id, const StackTrace & stack_trace);
-
-
-/** The thread that read info about signal or std::terminate from pipe.
-  * On HUP, close log files (for new files to be opened later).
-  * On information about std::terminate, write it to log.
-  * On other signals, write info to log.
-  */
-class SignalListener : public Poco::Runnable
-{
-public:
-    static constexpr int StdTerminate = -1;
-    static constexpr int StopThread = -2;
-    static constexpr int SanitizerTrap = -3;
-
-    explicit SignalListener(BaseDaemon & daemon_)
-        : log(getLogger("BaseDaemon"))
-        , daemon(daemon_)
-    {
-    }
-
-    void run() override
-    {
-        static_assert(PIPE_BUF >= 512);
-        static_assert(signal_pipe_buf_size <= PIPE_BUF, "Only write of PIPE_BUF to pipe is atomic and the minimal known PIPE_BUF across supported platforms is 512");
-        char buf[signal_pipe_buf_size];
-        ReadBufferFromFileDescriptor in(signal_pipe.fds_rw[0], signal_pipe_buf_size, buf);
-
-        while (!in.eof())
-        {
-            int sig = 0;
-            readBinary(sig, in);
-            // We may log some specific signals afterwards, with different log
-            // levels and more info, but for completeness we log all signals
-            // here at trace level.
-            // Don't use strsignal here, because it's not thread-safe.
-            LOG_TRACE(log, "Received signal {}", sig);
-
-            if (sig == StopThread)
-            {
-                LOG_INFO(log, "Stop SignalListener thread");
-                break;
-            }
-            else if (sig == SIGHUP)
-            {
-                LOG_DEBUG(log, "Received signal to close logs.");
-                BaseDaemon::instance().closeLogs(BaseDaemon::instance().logger());
-                LOG_INFO(log, "Opened new log file after received signal.");
-            }
-            else if (sig == StdTerminate)
-            {
-                UInt32 thread_num;
-                std::string message;
-
-                readBinary(thread_num, in);
-                readBinary(message, in);
-
-                onTerminate(message, thread_num);
-            }
-            else if (sig == SIGINT ||
-                sig == SIGQUIT ||
-                sig == SIGTERM)
-            {
-                daemon.handleSignal(sig);
-            }
-            else
-            {
-                siginfo_t info{};
-                ucontext_t * context{};
-                StackTrace stack_trace(NoCapture{});
-                std::vector<StackTrace::FramePointers> thread_frame_pointers;
-                UInt32 thread_num{};
-                ThreadStatus * thread_ptr{};
-
-                if (sig != SanitizerTrap)
-                {
-                    readPODBinary(info, in);
-                    readPODBinary(context, in);
-                }
-
-                readPODBinary(stack_trace, in);
-                readVectorBinary(thread_frame_pointers, in);
-                readBinary(thread_num, in);
-                readPODBinary(thread_ptr, in);
-
-                /// This allows to receive more signals if failure happens inside onFault function.
-                /// Example: segfault while symbolizing stack trace.
-                try
-                {
-                    std::thread([=, this] { onFault(sig, info, context, stack_trace, thread_frame_pointers, thread_num, thread_ptr); }).detach();
-                }
-                catch (...)
-                {
-                    /// Likely cannot allocate thread
-                    onFault(sig, info, context, stack_trace, thread_frame_pointers, thread_num, thread_ptr);
-                }
-            }
-        }
-    }
-
-private:
-    LoggerPtr log;
-    BaseDaemon & daemon;
-
-    void onTerminate(std::string_view message, UInt32 thread_num) const
-    {
-        size_t pos = message.find('\n');
-
-        LOG_FATAL(log, "(version {}{}, build id: {}, git hash: {}) (from thread {}) {}",
-            VERSION_STRING, VERSION_OFFICIAL, daemon.build_id, daemon.git_hash, thread_num, message.substr(0, pos));
-
-        /// Print trace from std::terminate exception line-by-line to make it easy for grep.
-        while (pos != std::string_view::npos)
-        {
-            ++pos;
-            size_t next_pos = message.find('\n', pos);
-            size_t size = next_pos;
-            if (next_pos != std::string_view::npos)
-                size = next_pos - pos;
-
-            LOG_FATAL(log, fmt::runtime(message.substr(pos, size)));
-            pos = next_pos;
-        }
-    }
-
-    void onFault(
-        int sig,
-        const siginfo_t & info,
-        ucontext_t * context,
-        const StackTrace & stack_trace,
-        const std::vector<StackTrace::FramePointers> & thread_frame_pointers,
-        UInt32 thread_num,
-        ThreadStatus * thread_ptr) const
-    {
-        ThreadStatus thread_status;
-
-        /// First log those fields that are safe to access and that should not cause new fault.
-        /// That way we will have some duplicated info in the log but we don't loose important info
-        /// in case of double fault.
-
-        LOG_FATAL(log, "########## Short fault info ############");
-        LOG_FATAL(log, "(version {}{}, build id: {}, git hash: {}) (from thread {}) Received signal {}",
-                VERSION_STRING, VERSION_OFFICIAL, daemon.build_id, daemon.git_hash,
-                thread_num, sig);
-
-        std::string signal_description = "Unknown signal";
-
-        /// Some of these are not really signals, but our own indications on failure reason.
-        if (sig == StdTerminate)
-            signal_description = "std::terminate";
-        else if (sig == SanitizerTrap)
-            signal_description = "sanitizer trap";
-        else if (sig >= 0)
-            signal_description = strsignal(sig); // NOLINT(concurrency-mt-unsafe) // it is not thread-safe but ok in this context
-
-        LOG_FATAL(log, "Signal description: {}", signal_description);
-
-        String error_message;
-
-        if (sig != SanitizerTrap)
-            error_message = signalToErrorMessage(sig, info, *context);
-        else
-            error_message = "Sanitizer trap.";
-
-        LOG_FATAL(log, fmt::runtime(error_message));
-
-        String bare_stacktrace_str;
-        if (stack_trace.getSize())
-        {
-            /// Write bare stack trace (addresses) just in case if we will fail to print symbolized stack trace.
-            /// NOTE: This still require memory allocations and mutex lock inside logger.
-            ///       BTW we can also print it to stderr using write syscalls.
-
-            WriteBufferFromOwnString bare_stacktrace;
-            writeString("Stack trace:", bare_stacktrace);
-            for (size_t i = stack_trace.getOffset(); i < stack_trace.getSize(); ++i)
-            {
-                writeChar(' ', bare_stacktrace);
-                writePointerHex(stack_trace.getFramePointers()[i], bare_stacktrace);
-            }
-
-            LOG_FATAL(log, fmt::runtime(bare_stacktrace.str()));
-            bare_stacktrace_str = bare_stacktrace.str();
-        }
-
-        /// Now try to access potentially unsafe data in thread_ptr.
-
-        String query_id;
-        String query;
-
-        /// Send logs from this thread to client if possible.
-        /// It will allow client to see failure messages directly.
-        if (thread_ptr)
-        {
-            query_id = thread_ptr->getQueryId();
-            query = thread_ptr->getQueryForLog();
-
-            if (auto logs_queue = thread_ptr->getInternalTextLogsQueue())
-            {
-                CurrentThread::attachInternalTextLogsQueue(logs_queue, LogsLevel::trace);
-            }
-        }
-
-        LOG_FATAL(log, "########################################");
-
-        if (query_id.empty())
-        {
-            LOG_FATAL(log, "(version {}{}, build id: {}, git hash: {}) (from thread {}) (no query) Received signal {} ({})",
-                VERSION_STRING, VERSION_OFFICIAL, daemon.build_id, daemon.git_hash,
-                thread_num, signal_description, sig);
-        }
-        else
-        {
-            LOG_FATAL(log, "(version {}{}, build id: {}, git hash: {}) (from thread {}) (query_id: {}) (query: {}) Received signal {} ({})",
-                VERSION_STRING, VERSION_OFFICIAL, daemon.build_id, daemon.git_hash,
-                thread_num, query_id, query, signal_description, sig);
-        }
-
-        LOG_FATAL(log, fmt::runtime(error_message));
-
-        if (!bare_stacktrace_str.empty())
-        {
-            LOG_FATAL(log, fmt::runtime(bare_stacktrace_str));
-        }
-
-        /// Write symbolized stack trace line by line for better grep-ability.
-        stack_trace.toStringEveryLine([&](std::string_view s) { LOG_FATAL(log, fmt::runtime(s)); });
-
-        /// In case it's a scheduled job write all previous jobs origins call stacks
-        std::for_each(thread_frame_pointers.rbegin(), thread_frame_pointers.rend(),
-            [this](const StackTrace::FramePointers & frame_pointers)
-            {
-                if (size_t size = std::ranges::find(frame_pointers, nullptr) - frame_pointers.begin())
-                {
-                    LOG_FATAL(log, "========================================");
-                    WriteBufferFromOwnString bare_stacktrace;
-                    writeString("Job's origin stack trace:", bare_stacktrace);
-                    std::for_each_n(frame_pointers.begin(), size,
-                        [&bare_stacktrace](const void * ptr)
-                        {
-                            writeChar(' ', bare_stacktrace);
-                            writePointerHex(ptr, bare_stacktrace);
-                        }
-                    );
-
-                    LOG_FATAL(log, fmt::runtime(bare_stacktrace.str()));
-
-                    StackTrace::toStringEveryLine(const_cast<void **>(frame_pointers.data()), 0, size, [this](std::string_view s) { LOG_FATAL(log, fmt::runtime(s)); });
-                }
-            }
-        );
-
-
-#if defined(OS_LINUX)
-        /// Write information about binary checksum. It can be difficult to calculate, so do it only after printing stack trace.
-        /// Please keep the below log messages in-sync with the ones in programs/server/Server.cpp
-
-        if (daemon.stored_binary_hash.empty())
-        {
-            LOG_FATAL(log, "Integrity check of the executable skipped because the reference checksum could not be read.");
-        }
-        else
-        {
-            String calculated_binary_hash = getHashOfLoadedBinaryHex();
-            if (calculated_binary_hash == daemon.stored_binary_hash)
-            {
-                LOG_FATAL(log, "Integrity check of the executable successfully passed (checksum: {})", calculated_binary_hash);
-            }
-            else
-            {
-                LOG_FATAL(
-                    log,
-                    "Calculated checksum of the executable ({0}) does not correspond"
-                    " to the reference checksum stored in the executable ({1})."
-                    " This may indicate one of the following:"
-                    " - the executable was changed just after startup;"
-                    " - the executable was corrupted on disk due to faulty hardware;"
-                    " - the loaded executable was corrupted in memory due to faulty hardware;"
-                    " - the file was intentionally modified;"
-                    " - a logical error in the code.",
-                    calculated_binary_hash,
-                    daemon.stored_binary_hash);
-            }
-        }
-#endif
-
-        /// Write crash to system.crash_log table if available.
-        if (collectCrashLog)
-            collectCrashLog(sig, thread_num, query_id, stack_trace);
-
-#ifndef CLICKHOUSE_KEEPER_STANDALONE_BUILD
-        Context::getGlobalContextInstance()->handleCrash();
-#endif
-
-        /// Send crash report to developers (if configured)
-        if (sig != SanitizerTrap)
-        {
-            SentryWriter::onFault(sig, error_message, stack_trace);
-
-            /// Advice the user to send it manually.
-            if (std::string_view(VERSION_OFFICIAL).contains("official build"))
-            {
-                const auto & date_lut = DateLUT::instance();
-
-                /// Approximate support period, upper bound.
-                if (time(nullptr) - date_lut.makeDate(2000 + VERSION_MAJOR, VERSION_MINOR, 1) < (365 + 30) * 86400)
-                {
-                    LOG_FATAL(log, "Report this error to https://github.com/ClickHouse/ClickHouse/issues");
-                }
-                else
-                {
-                    LOG_FATAL(log, "ClickHouse version {} is old and should be upgraded to the latest version.", VERSION_STRING);
-                }
-            }
-            else
-            {
-                LOG_FATAL(log, "This ClickHouse version is not official and should be upgraded to the official build.");
-            }
-        }
-
-        /// ClickHouse Keeper does not link to some part of Settings.
-#ifndef CLICKHOUSE_KEEPER_STANDALONE_BUILD
-        /// List changed settings.
-        if (!query_id.empty())
-        {
-            ContextPtr query_context = thread_ptr->getQueryContext();
-            if (query_context)
-            {
-                String changed_settings = query_context->getSettingsRef().toString();
-
-                if (changed_settings.empty())
-                    LOG_FATAL(log, "No settings were changed");
-                else
-                    LOG_FATAL(log, "Changed settings: {}", changed_settings);
-            }
-        }
-#endif
-
-        /// When everything is done, we will try to send these error messages to client.
-        if (thread_ptr)
-            thread_ptr->onFatalError();
-
-        fatal_error_printed.test_and_set();
-    }
-};
-
-
-#if defined(SANITIZER)
-extern "C" void __sanitizer_set_death_callback(void (*)());
-
-/// Sanitizers may not expect some function calls from death callback.
-/// Let's try to disable instrumentation to avoid possible issues.
-/// However, this callback may call other functions that are still instrumented.
-/// We can try [[clang::always_inline]] attribute for statements in future (available in clang-15)
-/// See https://github.com/google/sanitizers/issues/1543 and https://github.com/google/sanitizers/issues/1549.
-static DISABLE_SANITIZER_INSTRUMENTATION void sanitizerDeathCallback()
-{
-    DENY_ALLOCATIONS_IN_SCOPE;
-    /// Also need to send data via pipe. Otherwise it may lead to deadlocks or failures in printing diagnostic info.
-
-    char buf[signal_pipe_buf_size];
-    WriteBufferFromFileDescriptorDiscardOnFailure out(signal_pipe.fds_rw[1], signal_pipe_buf_size, buf);
-
-    const StackTrace stack_trace;
-
-    int sig = SignalListener::SanitizerTrap;
-    writeBinary(sig, out);
-    writePODBinary(stack_trace, out);
-    writeBinary(UInt32(getThreadId()), out);
-    writePODBinary(current_thread, out);
-
-    out.next();
-
-    /// The time that is usually enough for separate thread to print info into log.
-    sleepForSeconds(20);
-}
-#endif
-
-
-/** To use with std::set_terminate.
-  * Collects slightly more info than __gnu_cxx::__verbose_terminate_handler,
-  *  and send it to pipe. Other thread will read this info from pipe and asynchronously write it to log.
-  * Look at libstdc++-v3/libsupc++/vterminate.cc for example.
-  */
-[[noreturn]] static void terminate_handler()
-{
-    static thread_local bool terminating = false;
-    if (terminating)
-        abort();
-
-    terminating = true;
-
-    std::string log_message;
-
-    if (std::current_exception())
-        log_message = "Terminate called for uncaught exception:\n" + getCurrentExceptionMessage(true);
-    else
-        log_message = "Terminate called without an active exception";
-
-    /// POSIX.1 says that write(2)s of less than PIPE_BUF bytes must be atomic - man 7 pipe
-    /// And the buffer should not be too small because our exception messages can be large.
-    static constexpr size_t buf_size = PIPE_BUF;
-
-    if (log_message.size() > buf_size - 16)
-        log_message.resize(buf_size - 16);
-
-    char buf[buf_size];
-    WriteBufferFromFileDescriptor out(signal_pipe.fds_rw[1], buf_size, buf);
-
-    writeBinary(static_cast<int>(SignalListener::StdTerminate), out);
-    writeBinary(static_cast<UInt32>(getThreadId()), out);
-    writeBinary(log_message, out);
-    out.next();
-
-    abort();
-}
-
-
 static std::string createDirectory(const std::string & file)
 {
     fs::path path = fs::path(file).parent_path();
@@ -664,21 +143,7 @@ BaseDaemon::~BaseDaemon()
 {
     writeSignalIDtoSignalPipe(SignalListener::StopThread);
     signal_listener_thread.join();
-    /// Reset signals to SIG_DFL to avoid trying to write to the signal_pipe that will be closed after.
-    for (int sig : handled_signals)
-        if (SIG_ERR == signal(sig, SIG_DFL))
-        {
-            try
-            {
-                throw ErrnoException(ErrorCodes::CANNOT_SET_SIGNAL_HANDLER, "Cannot set signal handler");
-            }
-            catch (ErrnoException &)
-            {
-                tryLogCurrentException(__PRETTY_FUNCTION__);
-            }
-        }
-
-    signal_pipe.close();
+    HandledSignals::instance().reset();
 }
 
 
@@ -715,6 +180,8 @@ void BaseDaemon::closeFDs()
 #else
     fs::path proc_path{"/proc/self/fd"};
 #endif
+
+    const auto & signal_pipe = HandledSignals::instance().signal_pipe;
     if (fs::is_directory(proc_path)) /// Hooray, proc exists
     {
         /// in /proc/self/fd directory filenames are numeric file descriptors.
@@ -935,84 +402,24 @@ void BaseDaemon::initialize(Application & self)
 }
 
 
-static void addSignalHandler(const std::vector<int> & signals, signal_function handler, std::vector<int> * out_handled_signals)
-{
-    struct sigaction sa;
-    memset(&sa, 0, sizeof(sa));
-    sa.sa_sigaction = handler;
-    sa.sa_flags = SA_SIGINFO;
-
-#if defined(OS_DARWIN)
-    sigemptyset(&sa.sa_mask);
-    for (auto signal : signals)
-        sigaddset(&sa.sa_mask, signal);
-#else
-    if (sigemptyset(&sa.sa_mask))
-        throw Poco::Exception("Cannot set signal handler.");
-
-    for (auto signal : signals)
-        if (sigaddset(&sa.sa_mask, signal))
-            throw Poco::Exception("Cannot set signal handler.");
-#endif
-
-    for (auto signal : signals)
-        if (sigaction(signal, &sa, nullptr))
-            throw Poco::Exception("Cannot set signal handler.");
-
-    if (out_handled_signals)
-        std::copy(signals.begin(), signals.end(), std::back_inserter(*out_handled_signals));
-}
-
-
-static void blockSignals(const std::vector<int> & signals)
-{
-    sigset_t sig_set;
-
-#if defined(OS_DARWIN)
-    sigemptyset(&sig_set);
-    for (auto signal : signals)
-        sigaddset(&sig_set, signal);
-#else
-    if (sigemptyset(&sig_set))
-        throw Poco::Exception("Cannot block signal.");
-
-    for (auto signal : signals)
-        if (sigaddset(&sig_set, signal))
-            throw Poco::Exception("Cannot block signal.");
-#endif
-
-    if (pthread_sigmask(SIG_BLOCK, &sig_set, nullptr))
-        throw Poco::Exception("Cannot block signal.");
-}
-
 extern const char * GIT_HASH;
 
 void BaseDaemon::initializeTerminationAndSignalProcessing()
 {
     SentryWriter::initialize(config());
-    std::set_terminate(terminate_handler);
 
     /// We want to avoid SIGPIPE when working with sockets and pipes, and just handle return value/errno instead.
     blockSignals({SIGPIPE});
 
     /// Setup signal handlers.
-    /// SIGTSTP is added for debugging purposes. To output a stack trace of any running thread at anytime.
-    addSignalHandler({SIGABRT, SIGSEGV, SIGILL, SIGBUS, SIGSYS, SIGFPE, SIGPIPE, SIGTSTP, SIGTRAP}, signalHandler, &handled_signals);
-    addSignalHandler({SIGHUP}, closeLogsSignalHandler, &handled_signals);
-    addSignalHandler({SIGINT, SIGQUIT, SIGTERM}, terminateRequestedSignalHandler, &handled_signals);
-
-#if defined(SANITIZER)
-    __sanitizer_set_death_callback(sanitizerDeathCallback);
-#endif
+    HandledSignals::instance().setupCommonDeadlySignalHandlers();
+    HandledSignals::instance().addSignalHandler({SIGHUP}, closeLogsSignalHandler, true);
 
     /// Set up Poco ErrorHandler for Poco Threads.
     static KillingErrorHandler killing_error_handler;
     Poco::ErrorHandler::set(&killing_error_handler);
 
-    signal_pipe.setNonBlockingWrite();
-    signal_pipe.tryIncreaseSize(1 << 20);
-
-    signal_listener = std::make_unique<SignalListener>(*this);
+    signal_listener = std::make_unique<SignalListener>(this, getLogger("BaseDaemon"));
     signal_listener_thread.start(*signal_listener);
 
 #if defined(__ELF__) && !defined(OS_FREEBSD)
@@ -1218,7 +625,7 @@ void BaseDaemon::setupWatchdog()
         /// Forward signals to the child process.
         if (forward_signals)
         {
-            addSignalHandler(
+            HandledSignals::instance().addSignalHandler(
                 {SIGHUP, SIGINT, SIGQUIT, SIGTERM},
                 [](int sig, siginfo_t *, void *)
                 {
@@ -1234,7 +641,7 @@ void BaseDaemon::setupWatchdog()
                         (void)res;
                     }
                 },
-                nullptr);
+                false);
         }
         else
         {
diff --git a/src/Daemon/BaseDaemon.h b/src/Daemon/BaseDaemon.h
index 952cf61d8e0..0c262de328e 100644
--- a/src/Daemon/BaseDaemon.h
+++ b/src/Daemon/BaseDaemon.h
@@ -168,8 +168,6 @@ protected:
     String git_hash;
     String stored_binary_hash;
 
-    std::vector<int> handled_signals;
-
     bool should_setup_watchdog = false;
     char * argv0 = nullptr;
 };

From 159334845e0c844ca445de5f346898038ae04c03 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Sun, 10 Mar 2024 14:41:29 +0000
Subject: [PATCH 0014/1488] Revert "Revert "Fix wacky primary key sorting in
 `SHOW INDEX`""

This reverts commit 655f9cb1953c371d7b9a9ba6a58fb4a4fc3ca40e.
---
 .../InterpreterShowIndexesQuery.cpp           |  4 ++--
 .../0_stateless/02724_show_indexes.reference  | 24 +++++++++----------
 2 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/src/Interpreters/InterpreterShowIndexesQuery.cpp b/src/Interpreters/InterpreterShowIndexesQuery.cpp
index e8005ead91e..5be72dc8ce6 100644
--- a/src/Interpreters/InterpreterShowIndexesQuery.cpp
+++ b/src/Interpreters/InterpreterShowIndexesQuery.cpp
@@ -37,7 +37,7 @@ FROM (
             name AS table,
             1 AS non_unique,
             'PRIMARY' AS key_name,
-            row_number() over (order by column_name) AS seq_in_index,
+            row_number() over (order by null) AS seq_in_index,
             arrayJoin(splitByString(', ', primary_key)) AS column_name,
             'A' AS collation,
             0 AS cardinality,
@@ -75,7 +75,7 @@ FROM (
             database = '{0}'
             AND table = '{1}'))
 {2}
-ORDER BY index_type, expression, column_name, seq_in_index;)", database, table, where_expression);
+ORDER BY index_type, expression, seq_in_index;)", database, table, where_expression);
 
     /// Sorting is strictly speaking not necessary but 1. it is convenient for users, 2. SQL currently does not allow to
     /// sort the output of SHOW INDEXES otherwise (SELECT * FROM (SHOW INDEXES ...) ORDER BY ...) is rejected) and 3. some
diff --git a/tests/queries/0_stateless/02724_show_indexes.reference b/tests/queries/0_stateless/02724_show_indexes.reference
index e41f2521f5c..ac0461fc506 100644
--- a/tests/queries/0_stateless/02724_show_indexes.reference
+++ b/tests/queries/0_stateless/02724_show_indexes.reference
@@ -2,33 +2,33 @@
 tbl	1	blf_idx	1		\N	0	\N	\N	\N	BLOOM_FILTER			YES	d, b
 tbl	1	mm1_idx	1		\N	0	\N	\N	\N	MINMAX			YES	a, c, d
 tbl	1	mm2_idx	1		\N	0	\N	\N	\N	MINMAX			YES	c, d, e
-tbl	1	PRIMARY	1	a	A	0	\N	\N	\N	PRIMARY			YES	
-tbl	1	PRIMARY	2	c	A	0	\N	\N	\N	PRIMARY			YES	
+tbl	1	PRIMARY	1	c	A	0	\N	\N	\N	PRIMARY			YES	
+tbl	1	PRIMARY	2	a	A	0	\N	\N	\N	PRIMARY			YES	
 tbl	1	set_idx	1		\N	0	\N	\N	\N	SET			YES	e
 tbl	1	blf_idx	1		\N	0	\N	\N	\N	BLOOM_FILTER			YES	d, b
 tbl	1	mm1_idx	1		\N	0	\N	\N	\N	MINMAX			YES	a, c, d
 tbl	1	mm2_idx	1		\N	0	\N	\N	\N	MINMAX			YES	c, d, e
-tbl	1	PRIMARY	1	a	A	0	\N	\N	\N	PRIMARY			YES	
-tbl	1	PRIMARY	2	c	A	0	\N	\N	\N	PRIMARY			YES	
+tbl	1	PRIMARY	1	c	A	0	\N	\N	\N	PRIMARY			YES	
+tbl	1	PRIMARY	2	a	A	0	\N	\N	\N	PRIMARY			YES	
 tbl	1	set_idx	1		\N	0	\N	\N	\N	SET			YES	e
 tbl	1	blf_idx	1		\N	0	\N	\N	\N	BLOOM_FILTER			YES	d, b
 tbl	1	mm1_idx	1		\N	0	\N	\N	\N	MINMAX			YES	a, c, d
 tbl	1	mm2_idx	1		\N	0	\N	\N	\N	MINMAX			YES	c, d, e
-tbl	1	PRIMARY	1	a	A	0	\N	\N	\N	PRIMARY			YES	
-tbl	1	PRIMARY	2	c	A	0	\N	\N	\N	PRIMARY			YES	
+tbl	1	PRIMARY	1	c	A	0	\N	\N	\N	PRIMARY			YES	
+tbl	1	PRIMARY	2	a	A	0	\N	\N	\N	PRIMARY			YES	
 tbl	1	set_idx	1		\N	0	\N	\N	\N	SET			YES	e
 tbl	1	blf_idx	1		\N	0	\N	\N	\N	BLOOM_FILTER			YES	d, b
 tbl	1	mm1_idx	1		\N	0	\N	\N	\N	MINMAX			YES	a, c, d
 tbl	1	mm2_idx	1		\N	0	\N	\N	\N	MINMAX			YES	c, d, e
-tbl	1	PRIMARY	1	a	A	0	\N	\N	\N	PRIMARY			YES	
-tbl	1	PRIMARY	2	c	A	0	\N	\N	\N	PRIMARY			YES	
+tbl	1	PRIMARY	1	c	A	0	\N	\N	\N	PRIMARY			YES	
+tbl	1	PRIMARY	2	a	A	0	\N	\N	\N	PRIMARY			YES	
 tbl	1	set_idx	1		\N	0	\N	\N	\N	SET			YES	e
 --- EXTENDED
 tbl	1	blf_idx	1		\N	0	\N	\N	\N	BLOOM_FILTER			YES	d, b
 tbl	1	mm1_idx	1		\N	0	\N	\N	\N	MINMAX			YES	a, c, d
 tbl	1	mm2_idx	1		\N	0	\N	\N	\N	MINMAX			YES	c, d, e
-tbl	1	PRIMARY	1	a	A	0	\N	\N	\N	PRIMARY			YES	
-tbl	1	PRIMARY	2	c	A	0	\N	\N	\N	PRIMARY			YES	
+tbl	1	PRIMARY	1	c	A	0	\N	\N	\N	PRIMARY			YES	
+tbl	1	PRIMARY	2	a	A	0	\N	\N	\N	PRIMARY			YES	
 tbl	1	set_idx	1		\N	0	\N	\N	\N	SET			YES	e
 --- WHERE
 --- Check with weird table names
@@ -40,8 +40,8 @@ NULL	1	PRIMARY	1	c	A	0	\N	\N	\N	PRIMARY			YES
 tbl	1	blf_idx	1		\N	0	\N	\N	\N	BLOOM_FILTER			YES	d, b
 tbl	1	mm1_idx	1		\N	0	\N	\N	\N	MINMAX			YES	a, c, d
 tbl	1	mm2_idx	1		\N	0	\N	\N	\N	MINMAX			YES	c, d, e
-tbl	1	PRIMARY	1	a	A	0	\N	\N	\N	PRIMARY			YES	
-tbl	1	PRIMARY	2	c	A	0	\N	\N	\N	PRIMARY			YES	
+tbl	1	PRIMARY	1	c	A	0	\N	\N	\N	PRIMARY			YES	
+tbl	1	PRIMARY	2	a	A	0	\N	\N	\N	PRIMARY			YES	
 tbl	1	set_idx	1		\N	0	\N	\N	\N	SET			YES	e
 --- Equally named table in other database
 tbl	1	mmi_idx	1		\N	0	\N	\N	\N	MINMAX			YES	b

From 89e46807dd88f3fddbbdc095eec88745a09b60ad Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Sun, 10 Mar 2024 14:58:05 +0000
Subject: [PATCH 0015/1488] Resolve flaky DatabaseRepliated tests

---
 src/Interpreters/InterpreterShowIndexesQuery.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Interpreters/InterpreterShowIndexesQuery.cpp b/src/Interpreters/InterpreterShowIndexesQuery.cpp
index 5be72dc8ce6..c6b5d5ac727 100644
--- a/src/Interpreters/InterpreterShowIndexesQuery.cpp
+++ b/src/Interpreters/InterpreterShowIndexesQuery.cpp
@@ -37,7 +37,7 @@ FROM (
             name AS table,
             1 AS non_unique,
             'PRIMARY' AS key_name,
-            row_number() over (order by null) AS seq_in_index,
+            row_number() over (order by database) AS seq_in_index,
             arrayJoin(splitByString(', ', primary_key)) AS column_name,
             'A' AS collation,
             0 AS cardinality,

From 60a865c9c98bba7f1b18b358b502aaf9cd359ca5 Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <mikhaylovnikitka@gmail.com>
Date: Mon, 11 Mar 2024 19:21:32 +0100
Subject: [PATCH 0016/1488] Debugging

---
 src/Common/SignalHandlers.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Common/SignalHandlers.cpp b/src/Common/SignalHandlers.cpp
index 9270320d536..13679a0b611 100644
--- a/src/Common/SignalHandlers.cpp
+++ b/src/Common/SignalHandlers.cpp
@@ -231,11 +231,11 @@ void SignalListener::run()
         // levels and more info, but for completeness we log all signals
         // here at trace level.
         // Don't use strsignal here, because it's not thread-safe.
-        LOG_TRACE(log, "Received signal {}", sig);
+        // LOG_TRACE(log, "Received signal {}", sig);
 
         if (sig == StopThread)
         {
-            LOG_INFO(log, "Stop SignalListener thread");
+            // LOG_INFO(log, "Stop SignalListener thread");
             break;
         }
         else if (sig == SIGHUP)

From 6b13fc93712631b61180582520ba4f66777628c3 Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <nikitamikhaylov@clickhouse.com>
Date: Mon, 11 Mar 2024 18:24:55 +0000
Subject: [PATCH 0017/1488] Revert "Debugging"

This reverts commit 60a865c9c98bba7f1b18b358b502aaf9cd359ca5.
---
 src/Common/SignalHandlers.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Common/SignalHandlers.cpp b/src/Common/SignalHandlers.cpp
index 13679a0b611..9270320d536 100644
--- a/src/Common/SignalHandlers.cpp
+++ b/src/Common/SignalHandlers.cpp
@@ -231,11 +231,11 @@ void SignalListener::run()
         // levels and more info, but for completeness we log all signals
         // here at trace level.
         // Don't use strsignal here, because it's not thread-safe.
-        // LOG_TRACE(log, "Received signal {}", sig);
+        LOG_TRACE(log, "Received signal {}", sig);
 
         if (sig == StopThread)
         {
-            // LOG_INFO(log, "Stop SignalListener thread");
+            LOG_INFO(log, "Stop SignalListener thread");
             break;
         }
         else if (sig == SIGHUP)

From ff4ae13d9ac1c7991c3de8e55d44faa9ed223093 Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <nikitamikhaylov@clickhouse.com>
Date: Mon, 11 Mar 2024 18:26:23 +0000
Subject: [PATCH 0018/1488] Print only fatals

---
 src/Client/ClientBase.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp
index 6ee76576515..5368384b785 100644
--- a/src/Client/ClientBase.cpp
+++ b/src/Client/ClientBase.cpp
@@ -3019,7 +3019,7 @@ void ClientBase::init(int argc, char ** argv)
     HandledSignals::instance().setupCommonDeadlySignalHandlers();
 
     fatal_channel_ptr = new Poco::ConsoleChannel;
-    fatal_log = createLogger("ClientBase", fatal_channel_ptr.get(), Poco::Message::PRIO_TRACE);
+    fatal_log = createLogger("ClientBase", fatal_channel_ptr.get(), Poco::Message::PRIO_FATAL);
     signal_listener = std::make_unique<SignalListener>(nullptr, fatal_log);
     signal_listener_thread.start(*signal_listener);
 }

From 9dbd380b5040a76a2e7a8ee316689e77b6081f27 Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <nikitamikhaylov@clickhouse.com>
Date: Mon, 11 Mar 2024 22:40:19 +0000
Subject: [PATCH 0019/1488] Dont handle SIGTERM and others for client

---
 src/Client/ClientBase.cpp     |  3 +++
 src/Common/SignalHandlers.cpp | 10 ++++++++--
 src/Common/SignalHandlers.h   |  3 ++-
 src/Daemon/BaseDaemon.cpp     |  2 ++
 4 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp
index 5368384b785..9e0d5262245 100644
--- a/src/Client/ClientBase.cpp
+++ b/src/Client/ClientBase.cpp
@@ -3016,7 +3016,10 @@ void ClientBase::init(int argc, char ** argv)
     has_log_comment = config().has("log_comment");
 
     /// Print stacktrace in case of crash
+    HandledSignals::instance().setupTerminateHandler();
     HandledSignals::instance().setupCommonDeadlySignalHandlers();
+    /// We don't setup signal handlers for SIGINT, SIGQUIT, SIGTERM because we don't
+    /// have an option for client to shutdown gracefully.
 
     fatal_channel_ptr = new Poco::ConsoleChannel;
     fatal_log = createLogger("ClientBase", fatal_channel_ptr.get(), Poco::Message::PRIO_FATAL);
diff --git a/src/Common/SignalHandlers.cpp b/src/Common/SignalHandlers.cpp
index 9270320d536..8414fc54adc 100644
--- a/src/Common/SignalHandlers.cpp
+++ b/src/Common/SignalHandlers.cpp
@@ -577,16 +577,22 @@ HandledSignals & HandledSignals::instance()
     return res;
 }
 
-void HandledSignals::setupCommonDeadlySignalHandlers()
+void HandledSignals::setupTerminateHandler()
 {
     std::set_terminate(terminate_handler);
+}
 
+void HandledSignals::setupCommonDeadlySignalHandlers()
+{
     /// SIGTSTP is added for debugging purposes. To output a stack trace of any running thread at anytime.
     addSignalHandler({SIGABRT, SIGSEGV, SIGILL, SIGBUS, SIGSYS, SIGFPE, SIGPIPE, SIGTSTP, SIGTRAP}, signalHandler, true);
-    addSignalHandler({SIGINT, SIGQUIT, SIGTERM}, terminateRequestedSignalHandler, true);
 
 #if defined(SANITIZER)
     __sanitizer_set_death_callback(sanitizerDeathCallback);
 #endif
 }
 
+void HandledSignals::setupCommonTerminateRequestSignalHandlers()
+{
+    addSignalHandler({SIGINT, SIGQUIT, SIGTERM}, terminateRequestedSignalHandler, true);
+}
diff --git a/src/Common/SignalHandlers.h b/src/Common/SignalHandlers.h
index b7af9dc2943..8bfeadd612e 100644
--- a/src/Common/SignalHandlers.h
+++ b/src/Common/SignalHandlers.h
@@ -110,7 +110,9 @@ struct HandledSignals
     HandledSignals();
     ~HandledSignals();
 
+    void setupTerminateHandler();
     void setupCommonDeadlySignalHandlers();
+    void setupCommonTerminateRequestSignalHandlers();
 
     void addSignalHandler(const std::vector<int> & signals, signal_function handler, bool register_signal);
 
@@ -118,4 +120,3 @@ struct HandledSignals
 
     static HandledSignals & instance();
 };
-
diff --git a/src/Daemon/BaseDaemon.cpp b/src/Daemon/BaseDaemon.cpp
index 15ab92464a0..e807ea2706b 100644
--- a/src/Daemon/BaseDaemon.cpp
+++ b/src/Daemon/BaseDaemon.cpp
@@ -412,7 +412,9 @@ void BaseDaemon::initializeTerminationAndSignalProcessing()
     blockSignals({SIGPIPE});
 
     /// Setup signal handlers.
+    HandledSignals::instance().setupTerminateHandler();
     HandledSignals::instance().setupCommonDeadlySignalHandlers();
+    HandledSignals::instance().setupCommonTerminateRequestSignalHandlers();
     HandledSignals::instance().addSignalHandler({SIGHUP}, closeLogsSignalHandler, true);
 
     /// Set up Poco ErrorHandler for Poco Threads.

From 09860cf7fd59ed67c4ce9ea3333c2ffaf6e2f46a Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <nikitamikhaylov@clickhouse.com>
Date: Tue, 12 Mar 2024 02:55:23 +0100
Subject: [PATCH 0020/1488] Blind fix for build

---
 programs/keeper/CMakeLists.txt |  1 +
 src/Common/SignalHandlers.cpp  |  9 +++++++++
 src/Common/SignalHandlers.h    | 12 ------------
 3 files changed, 10 insertions(+), 12 deletions(-)

diff --git a/programs/keeper/CMakeLists.txt b/programs/keeper/CMakeLists.txt
index 70e0f229fd4..5752eadc2a9 100644
--- a/programs/keeper/CMakeLists.txt
+++ b/programs/keeper/CMakeLists.txt
@@ -98,6 +98,7 @@ if (BUILD_STANDALONE_KEEPER)
         ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Common/NamedCollections/NamedCollections.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Common/NamedCollections/NamedCollectionConfiguration.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Common/Jemalloc.cpp
+        ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Common/SignalHandlers.cpp
 
         ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Common/ZooKeeper/IKeeper.cpp
         ${CMAKE_CURRENT_SOURCE_DIR}/../../src/Common/ZooKeeper/TestKeeper.cpp
diff --git a/src/Common/SignalHandlers.cpp b/src/Common/SignalHandlers.cpp
index 8414fc54adc..21fd5152e6a 100644
--- a/src/Common/SignalHandlers.cpp
+++ b/src/Common/SignalHandlers.cpp
@@ -12,6 +12,8 @@
 #include <IO/ReadHelpers.h>
 #include <Interpreters/Context.h>
 
+#pragma clang diagnostic ignored "-Wreserved-identifier"
+
 namespace DB
 {
 namespace ErrorCodes
@@ -140,6 +142,13 @@ void signalHandler(int sig, siginfo_t * info, void * context)
 }
 
 #if defined(SANITIZER)
+extern "C" void __sanitizer_set_death_callback(void (*)());
+
+/// Sanitizers may not expect some function calls from death callback.
+/// Let's try to disable instrumentation to avoid possible issues.
+/// However, this callback may call other functions that are still instrumented.
+/// We can try [[clang::always_inline]] attribute for statements in future (available in clang-15)
+/// See https://github.com/google/sanitizers/issues/1543 and https://github.com/google/sanitizers/issues/1549.
 static DISABLE_SANITIZER_INSTRUMENTATION void sanitizerDeathCallback()
 {
     DENY_ALLOCATIONS_IN_SCOPE;
diff --git a/src/Common/SignalHandlers.h b/src/Common/SignalHandlers.h
index 8bfeadd612e..e7519f7aee2 100644
--- a/src/Common/SignalHandlers.h
+++ b/src/Common/SignalHandlers.h
@@ -46,18 +46,6 @@ void signalHandler(int sig, siginfo_t * info, void * context);
   */
 [[noreturn]] void terminate_handler();
 
-#if defined(SANITIZER)
-extern "C" void __sanitizer_set_death_callback(void (*)());
-
-/// Sanitizers may not expect some function calls from death callback.
-/// Let's try to disable instrumentation to avoid possible issues.
-/// However, this callback may call other functions that are still instrumented.
-/// We can try [[clang::always_inline]] attribute for statements in future (available in clang-15)
-/// See https://github.com/google/sanitizers/issues/1543 and https://github.com/google/sanitizers/issues/1549.
-DISABLE_SANITIZER_INSTRUMENTATION void sanitizerDeathCallback();
-#endif
-
-
 /// Avoid link time dependency on DB/Interpreters - will use this function only when linked.
 __attribute__((__weak__)) void collectCrashLog(
     Int32 signal, UInt64 thread_id, const String & query_id, const StackTrace & stack_trace);

From 1e4b27e923dfcda94b2d4b019fdb8e58ed5ccf33 Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <nikitamikhaylov@clickhouse.com>
Date: Tue, 12 Mar 2024 14:51:30 +0100
Subject: [PATCH 0021/1488] Bump


From b319d62620657ab9c85b9543b0f0cac41fcbccd3 Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Mon, 18 Mar 2024 01:07:03 +0100
Subject: [PATCH 0022/1488] fix

---
 src/Coordination/KeeperSnapshotManager.cpp | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/Coordination/KeeperSnapshotManager.cpp b/src/Coordination/KeeperSnapshotManager.cpp
index 5d07067ad26..c5623e746ff 100644
--- a/src/Coordination/KeeperSnapshotManager.cpp
+++ b/src/Coordination/KeeperSnapshotManager.cpp
@@ -885,14 +885,16 @@ SnapshotFileInfo KeeperSnapshotManager<Storage>::serializeSnapshotToDisk(const K
     return {snapshot_file_name, disk};
 }
 
-size_t KeeperSnapshotManager::getLatestSnapshotIndex() const
+template<typename Storage>
+size_t KeeperSnapshotManager<Storage>::getLatestSnapshotIndex() const
 {
     if (!existing_snapshots.empty())
         return existing_snapshots.rbegin()->first;
     return 0;
 }
 
-SnapshotFileInfo KeeperSnapshotManager::getLatestSnapshotInfo() const
+template<typename Storage>
+SnapshotFileInfo KeeperSnapshotManager<Storage>::getLatestSnapshotInfo() const
 {
     if (!existing_snapshots.empty())
     {

From 5f4d2f2119d5a338af0fe3ee65314eb73ed368f7 Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Mon, 18 Mar 2024 09:23:51 +0100
Subject: [PATCH 0023/1488] fix build

---
 src/Coordination/KeeperStorage.cpp       | 1 +
 src/Coordination/ZooKeeperDataReader.cpp | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/Coordination/KeeperStorage.cpp b/src/Coordination/KeeperStorage.cpp
index b1d6655a55d..992cfe8f767 100644
--- a/src/Coordination/KeeperStorage.cpp
+++ b/src/Coordination/KeeperStorage.cpp
@@ -1,6 +1,7 @@
 #include <iterator>
 #include <variant>
 #include <IO/Operators.h>
+#include <IO/ReadBufferFromString.h>
 #include <IO/WriteHelpers.h>
 #include <boost/algorithm/string.hpp>
 #include <Poco/SHA1Engine.h>
diff --git a/src/Coordination/ZooKeeperDataReader.cpp b/src/Coordination/ZooKeeperDataReader.cpp
index f3dc7ffe0b7..99d71b85e78 100644
--- a/src/Coordination/ZooKeeperDataReader.cpp
+++ b/src/Coordination/ZooKeeperDataReader.cpp
@@ -196,7 +196,7 @@ void deserializeKeeperStorageFromSnapshot(Storage & storage, const std::string &
     LOG_INFO(log, "Finished, snapshot ZXID {}", storage.zxid);
 }
 
-/// namespace fs = std::filesystem;
+namespace fs = std::filesystem;
 
 template<typename Storage>
 void deserializeKeeperStorageFromSnapshotsDir(Storage & storage, const std::string & path, LoggerPtr log)

From 587b176a2e05cf10c859e18d7c06bfa75ef7bbdb Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Tue, 19 Mar 2024 16:06:53 +0100
Subject: [PATCH 0024/1488] fix keeper build

---
 programs/keeper/CMakeLists.txt | 1 +
 src/CMakeLists.txt             | 1 -
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/programs/keeper/CMakeLists.txt b/programs/keeper/CMakeLists.txt
index 70e0f229fd4..e66df8e724e 100644
--- a/programs/keeper/CMakeLists.txt
+++ b/programs/keeper/CMakeLists.txt
@@ -192,6 +192,7 @@ if (BUILD_STANDALONE_KEEPER)
         PRIVATE
             ch_contrib::abseil_swiss_tables
             ch_contrib::nuraft
+            ch_contrib::rocksdb
             ch_contrib::lz4
             ch_contrib::zstd
             ch_contrib::cityhash
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index fd15d2df1e9..73aa409e995 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -133,7 +133,6 @@ endif()
 
 if (TARGET ch_contrib::rocksdb)
     add_headers_and_sources(dbms Storages/RocksDB)
-    add_headers_and_sources(dbms Coordination)
 endif()
 
 if (TARGET ch_contrib::aws_s3)

From 930568c8c68ad8463c38888c6843a25ba3e38429 Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Tue, 19 Mar 2024 20:41:34 +0100
Subject: [PATCH 0025/1488] fix debug build

---
 src/Coordination/KeeperSnapshotManager.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Coordination/KeeperSnapshotManager.cpp b/src/Coordination/KeeperSnapshotManager.cpp
index c5623e746ff..9f653b7dfc4 100644
--- a/src/Coordination/KeeperSnapshotManager.cpp
+++ b/src/Coordination/KeeperSnapshotManager.cpp
@@ -477,7 +477,7 @@ void KeeperStorageSnapshot<Storage>::deserialize(SnapshotDeserializationResult<S
 #else
                     throw Exception(ErrorCodes::LOGICAL_ERROR, "Children counter in stat.numChildren {}"
                                     " is different from actual children size {} for node {}",
-                                    itr.value.stat.numChildren, itr.value.getChildren().size(), itr.key);
+                                    itr.value.numChildren(), itr.value.getChildren().size(), itr.key);
 #endif
                 }
             }

From 7cd2372bfa9dfc7e4591fa0df747d1a54b393070 Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Wed, 20 Mar 2024 11:58:02 +0100
Subject: [PATCH 0026/1488] fix clang tidy

---
 src/Coordination/RocksDBContainer.h | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/src/Coordination/RocksDBContainer.h b/src/Coordination/RocksDBContainer.h
index 4bfd57e7682..a2ff6ed3f3c 100644
--- a/src/Coordination/RocksDBContainer.h
+++ b/src/Coordination/RocksDBContainer.h
@@ -71,13 +71,11 @@ public:
 
         std::shared_ptr<const KVPair> pair;
 
-        const_iterator()
-        {
-        }
+        const_iterator() = default;
 
-        const_iterator(std::shared_ptr<KVPair> pair_) : pair(std::move(pair_)) {}
+        explicit const_iterator(std::shared_ptr<KVPair> pair_) : pair(std::move(pair_)) {}
 
-        const_iterator(rocksdb::Iterator * iter_) : iter(iter_)
+        explicit const_iterator(rocksdb::Iterator * iter_) : iter(iter_)
         {
             iter->SeekToFirst();
         }
@@ -112,7 +110,7 @@ public:
             return iter != nullptr;
         }
 
-        operator bool() const
+        explicit operator bool() const
         {
             return iter != nullptr;
         }

From 0f48238a27c9167ade17bb32aa9f39710305f4f8 Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Thu, 4 Apr 2024 13:47:04 +0200
Subject: [PATCH 0027/1488] copy ut for rocks keeper

---
 src/Coordination/KeeperContext.cpp            |   19 +
 src/Coordination/KeeperContext.h              |    3 +
 src/Coordination/KeeperSnapshotManager.cpp    |    1 -
 src/Coordination/KeeperStorage.cpp            |    9 +-
 src/Coordination/RocksDBContainer.h           |  107 +-
 src/Coordination/tests/gtest_rocks_keeper.cpp | 1150 +++++++++++++++++
 6 files changed, 1263 insertions(+), 26 deletions(-)
 create mode 100644 src/Coordination/tests/gtest_rocks_keeper.cpp

diff --git a/src/Coordination/KeeperContext.cpp b/src/Coordination/KeeperContext.cpp
index 8fc4661a779..4ae3c295824 100644
--- a/src/Coordination/KeeperContext.cpp
+++ b/src/Coordination/KeeperContext.cpp
@@ -9,6 +9,7 @@
 #include <IO/S3/Credentials.h>
 #include <IO/WriteHelpers.h>
 #include <Poco/Util/AbstractConfiguration.h>
+#include <Poco/Util/JSONConfiguration.h>
 #include <Coordination/KeeperConstants.h>
 #include <Server/CloudPlacementInfo.h>
 #include <Coordination/KeeperFeatureFlags.h>
@@ -367,6 +368,12 @@ void KeeperContext::dumpConfiguration(WriteBufferFromOwnString & buf) const
     }
 }
 
+
+void KeeperContext::setRocksDBDisk(DiskPtr disk)
+{
+    rocksdb_storage = std::move(disk);
+}
+
 DiskPtr KeeperContext::getTemporaryRocksDBDisk() const
 {
     DiskPtr rocksdb_disk = getDisk(rocksdb_storage);
@@ -380,6 +387,18 @@ DiskPtr KeeperContext::getTemporaryRocksDBDisk() const
     return std::make_shared<DiskLocal>("LocalTmpRocksDBDisk", fullPath(rocksdb_disk, path_to_create));
 }
 
+void KeeperContext::setRocksDBOptions(std::shared_ptr<rocksdb::Options> rocksdb_options_)
+{
+    if (rocksdb_options_ != nullptr)
+        rocksdb_options = rocksdb_options_;
+    else
+    {
+        #if USE_ROCKSDB
+        rocksdb_options = std::make_shared<rocksdb::Options>(getRocksDBOptionsFromConfig(Poco::Util::JSONConfiguration()));
+        #endif
+    }
+}
+
 KeeperContext::Storage KeeperContext::getLogsPathFromConfig(const Poco::Util::AbstractConfiguration & config) const
 {
     const auto create_local_disk = [](const auto & path)
diff --git a/src/Coordination/KeeperContext.h b/src/Coordination/KeeperContext.h
index f84d087cb41..c9abd13b6e9 100644
--- a/src/Coordination/KeeperContext.h
+++ b/src/Coordination/KeeperContext.h
@@ -67,7 +67,10 @@ public:
 
     constexpr KeeperDispatcher * getDispatcher() const { return dispatcher; }
 
+    void setRocksDBDisk(DiskPtr disk);
     DiskPtr getTemporaryRocksDBDisk() const;
+
+    void setRocksDBOptions(std::shared_ptr<rocksdb::Options> rocksdb_options_);
     std::shared_ptr<rocksdb::Options> getRocksDBOptions() const { return rocksdb_options; }
 
     UInt64 getKeeperMemorySoftLimit() const { return memory_soft_limit; }
diff --git a/src/Coordination/KeeperSnapshotManager.cpp b/src/Coordination/KeeperSnapshotManager.cpp
index 9f653b7dfc4..593c358e80d 100644
--- a/src/Coordination/KeeperSnapshotManager.cpp
+++ b/src/Coordination/KeeperSnapshotManager.cpp
@@ -258,7 +258,6 @@ void KeeperStorageSnapshot<Storage>::serialize(const KeeperStorageSnapshot<Stora
         /// slightly bigger than required.
         if (node.mzxid > snapshot.zxid)
             break;
-
         writeBinary(path, out);
         writeNode(node, snapshot.version, out);
 
diff --git a/src/Coordination/KeeperStorage.cpp b/src/Coordination/KeeperStorage.cpp
index 992cfe8f767..905fd06359c 100644
--- a/src/Coordination/KeeperStorage.cpp
+++ b/src/Coordination/KeeperStorage.cpp
@@ -261,7 +261,10 @@ void KeeperRocksNode::decodeFromString(const String &buffer_str)
     readPODBinary(node_info, buffer);
     readVarUInt(data_size, buffer);
     if (data_size)
+    {
+        data = std::unique_ptr<char[]>(new char[data_size]);
         buffer.readStrict(data.get(), data_size);
+    }
 }
 
 KeeperMemNode & KeeperMemNode::operator=(const KeeperMemNode & other)
@@ -445,6 +448,8 @@ KeeperStorage<Container>::KeeperStorage(
     int64_t tick_time_ms, const String & superdigest_, const KeeperContextPtr & keeper_context_, const bool initialize_system_nodes)
     : session_expiry_queue(tick_time_ms), keeper_context(keeper_context_), superdigest(superdigest_)
 {
+    if constexpr (use_rocksdb)
+        container.initialize(keeper_context);
     Node root_node;
     container.insert("/", root_node);
     if constexpr (!use_rocksdb)
@@ -1757,14 +1762,14 @@ struct KeeperStorageListRequestProcessor final : public KeeperStorageRequestProc
                     if (list_request_type == ALL)
                         return true;
 
-                    const auto is_ephemeral = child.value.isEphemeral();
+                    const auto is_ephemeral = child.second.isEphemeral();
                     return (is_ephemeral && list_request_type == EPHEMERAL_ONLY) || (!is_ephemeral && list_request_type == PERSISTENT_ONLY);
                 };
 
                 for (const auto & child : children)
                 {
                     if (add_child(child))
-                        response.names.push_back(child.key.toString());
+                        response.names.push_back(child.first);
                 }
             }
             else
diff --git a/src/Coordination/RocksDBContainer.h b/src/Coordination/RocksDBContainer.h
index a2ff6ed3f3c..683678cfcd1 100644
--- a/src/Coordination/RocksDBContainer.h
+++ b/src/Coordination/RocksDBContainer.h
@@ -18,6 +18,7 @@ namespace DB
 namespace ErrorCodes
 {
     extern const int ROCKSDB_ERROR;
+    extern const int LOGICAL_ERROR;
 }
 
 namespace fs = std::filesystem;
@@ -32,9 +33,27 @@ struct RocksDBContainer
     using Node = Node_;
 
 private:
-    UInt8 getKeyDepth(const std::string & key)
+    std::string toHex(std::string origin)
     {
-        UInt8 depth = 0;
+        auto toHexChar = [](char v)
+        {
+            if (v <= 9)
+                return v + '0';
+            return 'A' + v - 10;
+        };
+        std::string result;
+        result.reserve(2 * origin.size());
+        for (size_t i = 0; i < origin.size(); ++i) {
+            unsigned char c = origin[i];
+            result.push_back(toHexChar(c >> 4));
+            result.push_back(toHexChar(c & 0xf));
+        }
+        return result;
+    }
+
+    UInt16 getKeyDepth(const std::string & key)
+    {
+        UInt16 depth = 0;
         for (size_t i = 0; i < key.size(); i++)
         {
             if (key[i] == '/' && i + 1 != key.size())
@@ -46,12 +65,17 @@ private:
     std::string getEncodedKey(const std::string & key, bool child_prefix = false)
     {
         WriteBufferFromOwnString key_buffer;
-        UInt16 depth = getKeyDepth(key) + child_prefix;
+        UInt16 depth = getKeyDepth(key) + (child_prefix ? 1 : 0);
         writeIntBinary(depth, key_buffer);
-        writeStringBinary(key, key_buffer);
+        writeString(key, key_buffer);
         return key_buffer.str();
     }
 
+    static std::string_view getDecodedKey(const std::string_view & key)
+    {
+        return std::string_view(key.begin() + 2, key.end());
+    }
+
 
     struct KVPair
     {
@@ -59,7 +83,6 @@ private:
         Node value;
     };
 
-    /// using KVPointer = std::shared_ptr<KVPair>;
     using ValueUpdater = std::function<void(Node & node)>;
 
 public:
@@ -77,7 +100,7 @@ public:
 
         explicit const_iterator(rocksdb::Iterator * iter_) : iter(iter_)
         {
-            iter->SeekToFirst();
+            updatePairFromIter();
         }
 
         const KVPair & operator * () const
@@ -92,11 +115,15 @@ public:
 
         bool operator != (const const_iterator & other) const
         {
-            return pair->key.toView() != other->key.toView();
+            return !(*this == other);
         }
 
         bool operator == (const const_iterator & other) const
         {
+            if (pair == nullptr && other == nullptr)
+                return true;
+            if (pair == nullptr || other == nullptr)
+                return false;
             return pair->key.toView() == other->key.toView() && iter == other.iter;
         }
 
@@ -118,17 +145,26 @@ public:
         const_iterator & operator ++()
         {
             iter->Next();
+            updatePairFromIter();
+            return *this;
+        }
+
+    private:
+        void updatePairFromIter()
+        {
             if (iter && iter->Valid())
             {
                 auto new_pair = std::make_shared<KVPair>();
-                new_pair->key = iter->key().ToString();
-                new_pair->value.reset();
+                new_pair->key = StringRef(getDecodedKey(iter->key().ToStringView()));
                 ReadBufferFromOwnString buffer(iter->value().ToStringView());
                 typename Node::Meta & meta = new_pair->value;
                 readPODBinary(meta, buffer);
                 readVarUInt(new_pair->value.data_size, buffer);
                 if (new_pair->value.data_size)
+                {
+                    new_pair->value.data = std::unique_ptr<char[]>(new char[new_pair->value.data_size]);
                     buffer.readStrict(new_pair->value.data.get(), new_pair->value.data_size);
+                }
                 pair = new_pair;
             }
             else
@@ -136,23 +172,25 @@ public:
                 pair = nullptr;
                 iter = nullptr;
             }
-            return *this;
         }
     };
 
+    bool initialized = false;
+
     const const_iterator end_ptr;
 
     void initialize(const KeeperContextPtr & context)
     {
-
         DiskPtr disk = context->getTemporaryRocksDBDisk();
         if (disk == nullptr)
         {
+            throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot get rocksdb disk");
             return;
         }
         auto options = context->getRocksDBOptions();
         if (options == nullptr)
         {
+            throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot get rocksdb options");
             return;
         }
         rocksdb_dir = disk->getPath();
@@ -165,24 +203,33 @@ public:
         }
         rocksdb_ptr = std::unique_ptr<rocksdb::DB>(db);
         /// storage_ptr = storage_;
+        initialized = true;
     }
 
     ~RocksDBContainer()
     {
-        rocksdb_ptr->Close();
-        rocksdb_ptr = nullptr;
+        if (initialized)
+        {
+            rocksdb_ptr->Close();
+            rocksdb_ptr = nullptr;
 
-        fs::remove_all(rocksdb_dir);
+            fs::remove_all(rocksdb_dir);
+        }
     }
 
-    std::vector<KVPair> getChildren(const std::string & key)
+    std::vector<std::pair<std::string, Node>> getChildren(const std::string & key)
     {
         rocksdb::ReadOptions read_options;
         read_options.total_order_seek = true;
 
+        size_t len = key.size() + 2;
+        if (!key.ends_with('/'))
+            len ++;
+
         auto iter = std::unique_ptr<rocksdb::Iterator>(rocksdb_ptr->NewIterator(read_options));
-        rocksdb::Slice prefix = getEncodedKey(key, true);
-        std::vector<KVPair> result;
+        std::string encoded_string = getEncodedKey(key, true);
+        rocksdb::Slice prefix(encoded_string);
+        std::vector<std::pair<std::string, Node>> result;
         for (iter->Seek(prefix); iter->Valid() && iter->key().starts_with(prefix); iter->Next())
         {
             Node node;
@@ -190,8 +237,11 @@ public:
             typename Node::Meta & meta = node;
             /// We do not read data here
             readPODBinary(meta, buffer);
-            result.emplace_back(iter->key().ToString(), std::move(node));
+            std::string real_key(iter->key().data() + len, iter->key().size() - len);
+            // std::cout << "real key: " << real_key << std::endl;
+            result.emplace_back(std::move(real_key), std::move(node));
         }
+
         return result;
     }
 
@@ -203,7 +253,7 @@ public:
         if (status.IsNotFound())
             return false;
         if (!status.ok())
-            throw Exception(ErrorCodes::ROCKSDB_ERROR, "Got rocksdb error during find. The error message is {}.", status.ToString());
+            throw Exception(ErrorCodes::ROCKSDB_ERROR, "Got rocksdb error during executing contains. The error message is {}.", status.ToString());
         return true;
     }
 
@@ -216,7 +266,7 @@ public:
         if (status.IsNotFound())
             return end();
         if (!status.ok())
-            throw Exception(ErrorCodes::ROCKSDB_ERROR, "Got rocksdb error during find. The error message is {}.", status.ToString());
+            throw Exception(ErrorCodes::ROCKSDB_ERROR, "Got rocksdb error during executing find. The error message is {}.", status.ToString());
         ReadBufferFromOwnString buffer(buffer_str);
         auto kv = std::make_shared<KVPair>();
         kv->key = key_;
@@ -225,7 +275,10 @@ public:
         /// TODO: Sometimes we don't need to load data.
         readVarUInt(kv->value.data_size, buffer);
         if (kv->value.data_size)
+        {
+            kv->value.data = std::unique_ptr<char[]>(new char[kv->value.data_size]);
             buffer.readStrict(kv->value.data.get(), kv->value.data_size);
+        }
         return const_iterator(kv);
     }
 
@@ -306,7 +359,8 @@ public:
     bool erase(const std::string & key)
     {
         /// storage->removeDigest(value, key);
-        auto status = rocksdb_ptr->Delete(rocksdb::WriteOptions(), key);
+        const std::string & encoded_key = getEncodedKey(key);
+        auto status = rocksdb_ptr->Delete(rocksdb::WriteOptions(), encoded_key);
         if (status.IsNotFound())
             return false;
         if (status.ok())
@@ -331,6 +385,7 @@ public:
         chassert(!snapshot_mode);
         snapshot_mode = true;
         snapshot_up_to_version = version;
+        snapshot_size = counter;
         ++current_version;
 
         snapshot = rocksdb_ptr->GetSnapshot();
@@ -347,7 +402,10 @@ public:
 
     std::pair<size_t, size_t> snapshotSizeWithVersion() const
     {
-        return std::make_pair(counter, current_version);
+        if (!snapshot_mode)
+            return std::make_pair(counter, current_version);
+        else
+            return std::make_pair(snapshot_size, current_version);
     }
 
     const_iterator begin() const
@@ -356,7 +414,9 @@ public:
         read_options.total_order_seek = true;
         if (snapshot_mode)
             read_options.snapshot = snapshot;
-        return const_iterator(rocksdb_ptr->NewIterator(read_options));
+        auto * iter = rocksdb_ptr->NewIterator(read_options);
+        iter->SeekToFirst();
+        return const_iterator(iter);
     }
 
     const_iterator end() const
@@ -391,6 +451,7 @@ private:
     bool snapshot_mode{false};
     size_t current_version{0};
     size_t snapshot_up_to_version{0};
+    size_t snapshot_size{0};
     size_t counter{0};
 
 };
diff --git a/src/Coordination/tests/gtest_rocks_keeper.cpp b/src/Coordination/tests/gtest_rocks_keeper.cpp
new file mode 100644
index 00000000000..6281f04a637
--- /dev/null
+++ b/src/Coordination/tests/gtest_rocks_keeper.cpp
@@ -0,0 +1,1150 @@
+#include <gtest/gtest.h>
+#include <Coordination/KeeperContext.h>
+#include <Coordination/KeeperCommon.h>
+#include <Coordination/KeeperLogStore.h>
+#include <Coordination/KeeperStorage.h>
+#include <Coordination/KeeperStateMachine.h>
+#include <Coordination/KeeperSnapshotManager.h>
+#include <Coordination/WriteBufferFromNuraftBuffer.h>
+#include <Common/Logger.h>
+#include <Disks/DiskLocal.h>
+#include <Poco/ConsoleChannel.h>
+
+#include "config.h"
+
+#if USE_NURAFT and USE_ROCKSDB
+
+namespace fs = std::filesystem;
+struct ChangelogDirTest
+{
+    std::string path;
+    bool drop;
+    explicit ChangelogDirTest(std::string path_, bool drop_ = true) : path(path_), drop(drop_)
+    {
+        EXPECT_FALSE(fs::exists(path)) << "Path " << path << " already exists, remove it to run test";
+        fs::create_directory(path);
+    }
+
+    ~ChangelogDirTest()
+    {
+        if (fs::exists(path) && drop)
+            fs::remove_all(path);
+    }
+};
+
+class RocksKeeperTest : public ::testing::Test
+{
+protected:
+    DB::KeeperContextPtr keeper_context;
+    LoggerPtr log{getLogger("RocksKeeperTest")};
+
+    void SetUp() override
+    {
+        Poco::AutoPtr<Poco::ConsoleChannel> channel(new Poco::ConsoleChannel(std::cerr));
+        Poco::Logger::root().setChannel(channel);
+        Poco::Logger::root().setLevel("trace");
+
+        auto settings = std::make_shared<DB::CoordinationSettings>();
+        settings->use_rocksdb = true;
+        keeper_context = std::make_shared<DB::KeeperContext>(true, settings);
+        keeper_context->setLocalLogsPreprocessed();
+        keeper_context->setRocksDBOptions(nullptr);
+    }
+
+    void setSnapshotDirectory(const std::string & path)
+    {
+        keeper_context->setSnapshotDisk(std::make_shared<DB::DiskLocal>("SnapshotDisk", path));
+    }
+
+    void setRocksDBDirectory(const std::string & path)
+    {
+        keeper_context->setRocksDBDisk(std::make_shared<DB::DiskLocal>("RocksDisk", path));
+    }
+};
+
+void addNode(DB::KeeperRocksStorage & storage, const std::string & path, const std::string & data, int64_t ephemeral_owner = 0)
+{
+    using Node = DB::KeeperRocksStorage::Node;
+    Node node{};
+    node.setData(data);
+    node.setEphemeralOwner(ephemeral_owner);
+    storage.container.insertOrReplace(path, node);
+    storage.container.updateValue(
+        DB::parentNodePath(StringRef{path}),
+        [&](auto & parent)
+        {
+            parent.increaseNumChildren();
+        });
+}
+
+namespace
+{
+void waitDurableLogs(nuraft::log_store & log_store)
+{
+    while (log_store.last_durable_index() != log_store.next_slot() - 1)
+        std::this_thread::sleep_for(std::chrono::milliseconds(200));
+}
+}
+
+TEST_F(RocksKeeperTest, TestStorageSnapshotSimple)
+{
+    ChangelogDirTest test("./snapshots");
+    ChangelogDirTest rocks("./rocksdb");
+    setSnapshotDirectory("./snapshots");
+    setRocksDBDirectory("./rocksdb");
+
+    DB::KeeperSnapshotManager<DB::KeeperRocksStorage> manager(3, keeper_context);
+
+    DB::KeeperRocksStorage storage(500, "", keeper_context);
+    addNode(storage, "/hello1", "world", 1);
+    addNode(storage, "/hello2", "somedata", 3);
+    storage.session_id_counter = 5;
+    storage.zxid = 2;
+    storage.ephemerals[3] = {"/hello2"};
+    storage.ephemerals[1] = {"/hello1"};
+    storage.getSessionID(130);
+    storage.getSessionID(130);
+
+    DB::KeeperStorageSnapshot<DB::KeeperRocksStorage> snapshot(&storage, 2);
+
+    EXPECT_EQ(snapshot.snapshot_meta->get_last_log_idx(), 2);
+    EXPECT_EQ(snapshot.session_id, 7);
+    EXPECT_EQ(snapshot.snapshot_container_size, 6);
+    EXPECT_EQ(snapshot.session_and_timeout.size(), 2);
+
+    auto buf = manager.serializeSnapshotToBuffer(snapshot);
+    manager.serializeSnapshotBufferToDisk(*buf, 2);
+    EXPECT_TRUE(fs::exists("./snapshots/snapshot_2.bin.zstd"));
+
+
+    auto debuf = manager.deserializeSnapshotBufferFromDisk(2);
+
+    auto [restored_storage, snapshot_meta, _] = manager.deserializeSnapshotFromBuffer(debuf);
+
+    EXPECT_EQ(restored_storage->container.size(), 6);
+    EXPECT_EQ(restored_storage->container.find("/")->value.numChildren(), 3);
+    EXPECT_EQ(restored_storage->container.find("/hello1")->value.numChildren(), 0);
+    EXPECT_EQ(restored_storage->container.find("/hello2")->value.numChildren(), 0);
+
+    EXPECT_EQ(restored_storage->container.find("/")->value.getData(), "");
+    EXPECT_EQ(restored_storage->container.find("/hello1")->value.getData(), "world");
+    EXPECT_EQ(restored_storage->container.find("/hello2")->value.getData(), "somedata");
+    EXPECT_EQ(restored_storage->session_id_counter, 7);
+    EXPECT_EQ(restored_storage->zxid, 2);
+    EXPECT_EQ(restored_storage->ephemerals.size(), 2);
+    EXPECT_EQ(restored_storage->ephemerals[3].size(), 1);
+    EXPECT_EQ(restored_storage->ephemerals[1].size(), 1);
+    EXPECT_EQ(restored_storage->session_and_timeout.size(), 2);
+}
+
+TEST_F(RocksKeeperTest, TestStorageSnapshotMoreWrites)
+{
+    ChangelogDirTest test("./snapshots");
+    ChangelogDirTest rocks("./rocksdb");
+    setSnapshotDirectory("./snapshots");
+    setRocksDBDirectory("./rocksdb");
+
+    DB::KeeperSnapshotManager<DB::KeeperRocksStorage> manager(3, keeper_context);
+
+    DB::KeeperRocksStorage storage(500, "", keeper_context);
+    storage.getSessionID(130);
+
+    for (size_t i = 0; i < 50; ++i)
+    {
+        addNode(storage, "/hello_" + std::to_string(i), "world_" + std::to_string(i));
+    }
+
+    DB::KeeperStorageSnapshot<DB::KeeperRocksStorage> snapshot(&storage, 50);
+    EXPECT_EQ(snapshot.snapshot_meta->get_last_log_idx(), 50);
+    EXPECT_EQ(snapshot.snapshot_container_size, 54);
+
+    for (size_t i = 50; i < 100; ++i)
+    {
+        addNode(storage, "/hello_" + std::to_string(i), "world_" + std::to_string(i));
+    }
+
+    EXPECT_EQ(storage.container.size(), 104);
+
+    auto buf = manager.serializeSnapshotToBuffer(snapshot);
+    manager.serializeSnapshotBufferToDisk(*buf, 50);
+    EXPECT_TRUE(fs::exists("./snapshots/snapshot_50.bin.zstd"));
+
+
+    auto debuf = manager.deserializeSnapshotBufferFromDisk(50);
+    auto [restored_storage, meta, _] = manager.deserializeSnapshotFromBuffer(debuf);
+
+    EXPECT_EQ(restored_storage->container.size(), 54);
+    for (size_t i = 0; i < 50; ++i)
+    {
+        EXPECT_EQ(restored_storage->container.find("/hello_" + std::to_string(i))->value.getData(), "world_" + std::to_string(i));
+    }
+}
+
+TEST_F(RocksKeeperTest, TestStorageSnapshotManySnapshots)
+{
+    ChangelogDirTest test("./snapshots");
+    ChangelogDirTest rocks("./rocksdb");
+    setSnapshotDirectory("./snapshots");
+    setRocksDBDirectory("./rocksdb");
+
+    DB::KeeperSnapshotManager<DB::KeeperRocksStorage> manager(3, keeper_context);
+
+    DB::KeeperRocksStorage storage(500, "", keeper_context);
+    storage.getSessionID(130);
+
+    for (size_t j = 1; j <= 5; ++j)
+    {
+        for (size_t i = (j - 1) * 50; i < j * 50; ++i)
+        {
+            addNode(storage, "/hello_" + std::to_string(i), "world_" + std::to_string(i));
+        }
+
+        DB::KeeperStorageSnapshot<DB::KeeperRocksStorage> snapshot(&storage, j * 50);
+        auto buf = manager.serializeSnapshotToBuffer(snapshot);
+        manager.serializeSnapshotBufferToDisk(*buf, j * 50);
+        EXPECT_TRUE(fs::exists(std::string{"./snapshots/snapshot_"} + std::to_string(j * 50) + ".bin.zstd"));
+    }
+
+    EXPECT_FALSE(fs::exists("./snapshots/snapshot_50.bin.zstd"));
+    EXPECT_FALSE(fs::exists("./snapshots/snapshot_100.bin.zstd"));
+    EXPECT_TRUE(fs::exists("./snapshots/snapshot_150.bin.zstd"));
+    EXPECT_TRUE(fs::exists("./snapshots/snapshot_200.bin.zstd"));
+    EXPECT_TRUE(fs::exists("./snapshots/snapshot_250.bin.zstd"));
+
+
+    auto [restored_storage, meta, _] = manager.restoreFromLatestSnapshot();
+
+    EXPECT_EQ(restored_storage->container.size(), 254);
+
+    for (size_t i = 0; i < 250; ++i)
+    {
+        EXPECT_EQ(restored_storage->container.find("/hello_" + std::to_string(i))->value.getData(), "world_" + std::to_string(i));
+    }
+}
+
+TEST_F(RocksKeeperTest, TestStorageSnapshotMode)
+{
+    ChangelogDirTest test("./snapshots");
+    setSnapshotDirectory("./snapshots");
+    ChangelogDirTest rocks("./rocksdb");
+    setRocksDBDirectory("./rocksdb");
+
+    DB::KeeperSnapshotManager<DB::KeeperRocksStorage> manager(3, keeper_context);
+    DB::KeeperRocksStorage storage(500, "", keeper_context);
+    for (size_t i = 0; i < 50; ++i)
+    {
+        addNode(storage, "/hello_" + std::to_string(i), "world_" + std::to_string(i));
+    }
+
+    {
+        DB::KeeperStorageSnapshot<DB::KeeperRocksStorage> snapshot(&storage, 50);
+        for (size_t i = 0; i < 50; ++i)
+        {
+            addNode(storage, "/hello_" + std::to_string(i), "wlrd_" + std::to_string(i));
+        }
+        for (size_t i = 0; i < 50; ++i)
+        {
+            EXPECT_EQ(storage.container.find("/hello_" + std::to_string(i))->value.getData(), "wlrd_" + std::to_string(i));
+        }
+        for (size_t i = 0; i < 50; ++i)
+        {
+            if (i % 2 == 0)
+                storage.container.erase("/hello_" + std::to_string(i));
+        }
+        EXPECT_EQ(storage.container.size(), 29);
+        EXPECT_EQ(storage.container.snapshotSizeWithVersion().first, 54);
+        EXPECT_EQ(storage.container.snapshotSizeWithVersion().second, 1);
+        auto buf = manager.serializeSnapshotToBuffer(snapshot);
+        manager.serializeSnapshotBufferToDisk(*buf, 50);
+    }
+    EXPECT_TRUE(fs::exists("./snapshots/snapshot_50.bin.zstd"));
+    EXPECT_EQ(storage.container.size(), 29);
+    storage.clearGarbageAfterSnapshot();
+    for (size_t i = 0; i < 50; ++i)
+    {
+        if (i % 2 != 0)
+            EXPECT_EQ(storage.container.find("/hello_" + std::to_string(i))->value.getData(), "wlrd_" + std::to_string(i));
+        else
+            EXPECT_FALSE(storage.container.contains("/hello_" + std::to_string(i)));
+    }
+
+    auto [restored_storage, meta, _] = manager.restoreFromLatestSnapshot();
+
+    for (size_t i = 0; i < 50; ++i)
+    {
+        EXPECT_EQ(restored_storage->container.find("/hello_" + std::to_string(i))->value.getData(), "world_" + std::to_string(i));
+    }
+}
+
+TEST_F(RocksKeeperTest, TestStorageSnapshotBroken)
+{
+    ChangelogDirTest test("./snapshots");
+    setSnapshotDirectory("./snapshots");
+    ChangelogDirTest rocks("./rocksdb");
+    setRocksDBDirectory("./rocksdb");
+
+    DB::KeeperSnapshotManager<DB::KeeperRocksStorage> manager(3, keeper_context);
+    DB::KeeperRocksStorage storage(500, "", keeper_context);
+    for (size_t i = 0; i < 50; ++i)
+    {
+        addNode(storage, "/hello_" + std::to_string(i), "world_" + std::to_string(i));
+    }
+    {
+        DB::KeeperStorageSnapshot<DB::KeeperRocksStorage> snapshot(&storage, 50);
+        auto buf = manager.serializeSnapshotToBuffer(snapshot);
+        manager.serializeSnapshotBufferToDisk(*buf, 50);
+    }
+    EXPECT_TRUE(fs::exists("./snapshots/snapshot_50.bin.zstd"));
+
+    /// Let's corrupt file
+    DB::WriteBufferFromFile plain_buf(
+        "./snapshots/snapshot_50.bin.zstd", DB::DBMS_DEFAULT_BUFFER_SIZE, O_APPEND | O_CREAT | O_WRONLY);
+    plain_buf.truncate(34);
+    plain_buf.sync();
+
+    EXPECT_THROW(manager.restoreFromLatestSnapshot(), DB::Exception);
+}
+
+nuraft::ptr<nuraft::buffer> getBufferFromZKRequest(int64_t session_id, int64_t zxid, const Coordination::ZooKeeperRequestPtr & request);
+
+nuraft::ptr<nuraft::log_entry>
+getLogEntryFromZKRequest(size_t term, int64_t session_id, int64_t zxid, const Coordination::ZooKeeperRequestPtr & request);
+
+static void testLogAndStateMachine(
+    DB::CoordinationSettingsPtr settings,
+    uint64_t total_logs,
+    bool enable_compression = true)
+{
+    using namespace Coordination;
+    using namespace DB;
+
+    ChangelogDirTest snapshots("./snapshots");
+    ChangelogDirTest logs("./logs");
+    ChangelogDirTest rocks("./rocksdb");
+
+    auto get_keeper_context = [&]
+    {
+        auto local_keeper_context = std::make_shared<DB::KeeperContext>(true, settings);
+        local_keeper_context->setSnapshotDisk(std::make_shared<DiskLocal>("SnapshotDisk", "./snapshots"));
+        local_keeper_context->setLogDisk(std::make_shared<DiskLocal>("LogDisk", "./logs"));
+        local_keeper_context->setRocksDBDisk(std::make_shared<DiskLocal>("RocksDisk", "./rocksdb"));
+        local_keeper_context->setRocksDBOptions(nullptr);
+        return local_keeper_context;
+    };
+
+    ResponsesQueue queue(std::numeric_limits<size_t>::max());
+    SnapshotsQueue snapshots_queue{1};
+
+    auto keeper_context = get_keeper_context();
+    auto state_machine = std::make_shared<KeeperStateMachine<DB::KeeperRocksStorage>>(queue, snapshots_queue, keeper_context, nullptr);
+
+    state_machine->init();
+    DB::KeeperLogStore changelog(
+        DB::LogFileSettings{
+            .force_sync = true, .compress_logs = enable_compression, .rotate_interval = settings->rotate_log_storage_interval},
+        DB::FlushSettings(),
+        keeper_context);
+    changelog.init(state_machine->last_commit_index() + 1, settings->reserved_log_items);
+
+    for (size_t i = 1; i < total_logs + 1; ++i)
+    {
+        std::shared_ptr<ZooKeeperCreateRequest> request = std::make_shared<ZooKeeperCreateRequest>();
+        request->path = "/hello_" + std::to_string(i);
+        auto entry = getLogEntryFromZKRequest(0, 1, i, request);
+        changelog.append(entry);
+        changelog.end_of_append_batch(0, 0);
+
+        waitDurableLogs(changelog);
+
+        state_machine->pre_commit(i, changelog.entry_at(i)->get_buf());
+        state_machine->commit(i, changelog.entry_at(i)->get_buf());
+        bool snapshot_created = false;
+        if (i % settings->snapshot_distance == 0)
+        {
+            nuraft::snapshot s(i, 0, std::make_shared<nuraft::cluster_config>());
+            nuraft::async_result<bool>::handler_type when_done
+                = [&snapshot_created](bool & ret, nuraft::ptr<std::exception> & /*exception*/)
+            {
+                snapshot_created = ret;
+            };
+
+            state_machine->create_snapshot(s, when_done);
+            CreateSnapshotTask snapshot_task;
+            bool pop_result = snapshots_queue.pop(snapshot_task);
+            EXPECT_TRUE(pop_result);
+
+            snapshot_task.create_snapshot(std::move(snapshot_task.snapshot));
+        }
+
+        if (snapshot_created && changelog.size() > settings->reserved_log_items)
+            changelog.compact(i - settings->reserved_log_items);
+    }
+
+    SnapshotsQueue snapshots_queue1{1};
+    keeper_context = get_keeper_context();
+    auto restore_machine = std::make_shared<KeeperStateMachine<DB::KeeperRocksStorage>>(queue, snapshots_queue1, keeper_context, nullptr);
+    restore_machine->init();
+    EXPECT_EQ(restore_machine->last_commit_index(), total_logs - total_logs % settings->snapshot_distance);
+
+    DB::KeeperLogStore restore_changelog(
+        DB::LogFileSettings{
+            .force_sync = true, .compress_logs = enable_compression, .rotate_interval = settings->rotate_log_storage_interval},
+        DB::FlushSettings(),
+        keeper_context);
+    restore_changelog.init(restore_machine->last_commit_index() + 1, settings->reserved_log_items);
+
+    EXPECT_EQ(restore_changelog.size(), std::min(settings->reserved_log_items + total_logs % settings->snapshot_distance, total_logs));
+    EXPECT_EQ(restore_changelog.next_slot(), total_logs + 1);
+    if (total_logs > settings->reserved_log_items + 1)
+        EXPECT_EQ(
+            restore_changelog.start_index(), total_logs - total_logs % settings->snapshot_distance - settings->reserved_log_items + 1);
+    else
+        EXPECT_EQ(restore_changelog.start_index(), 1);
+
+    for (size_t i = restore_machine->last_commit_index() + 1; i < restore_changelog.next_slot(); ++i)
+    {
+        restore_machine->pre_commit(i, changelog.entry_at(i)->get_buf());
+        restore_machine->commit(i, changelog.entry_at(i)->get_buf());
+    }
+
+    auto & source_storage = state_machine->getStorageUnsafe();
+    auto & restored_storage = restore_machine->getStorageUnsafe();
+
+    EXPECT_EQ(source_storage.container.size(), restored_storage.container.size());
+    for (size_t i = 1; i < total_logs + 1; ++i)
+    {
+        auto path = "/hello_" + std::to_string(i);
+        EXPECT_EQ(source_storage.container.find(path)->value.getData(), restored_storage.container.find(path)->value.getData());
+    }
+}
+
+TEST_F(RocksKeeperTest, TestStateMachineAndLogStore)
+{
+    using namespace Coordination;
+    using namespace DB;
+
+    {
+        CoordinationSettingsPtr settings = std::make_shared<CoordinationSettings>();
+        settings->snapshot_distance = 10;
+        settings->reserved_log_items = 10;
+        settings->rotate_log_storage_interval = 10;
+
+        testLogAndStateMachine(settings, 37);
+    }
+    {
+        CoordinationSettingsPtr settings = std::make_shared<CoordinationSettings>();
+        settings->snapshot_distance = 10;
+        settings->reserved_log_items = 10;
+        settings->rotate_log_storage_interval = 10;
+        testLogAndStateMachine(settings, 11);
+    }
+    {
+        CoordinationSettingsPtr settings = std::make_shared<CoordinationSettings>();
+        settings->snapshot_distance = 10;
+        settings->reserved_log_items = 10;
+        settings->rotate_log_storage_interval = 10;
+        testLogAndStateMachine(settings, 40);
+    }
+    {
+        CoordinationSettingsPtr settings = std::make_shared<CoordinationSettings>();
+        settings->snapshot_distance = 10;
+        settings->reserved_log_items = 20;
+        settings->rotate_log_storage_interval = 30;
+        testLogAndStateMachine(settings, 40);
+    }
+    {
+        CoordinationSettingsPtr settings = std::make_shared<CoordinationSettings>();
+        settings->snapshot_distance = 10;
+        settings->reserved_log_items = 0;
+        settings->rotate_log_storage_interval = 10;
+        testLogAndStateMachine(settings, 40);
+    }
+    {
+        CoordinationSettingsPtr settings = std::make_shared<CoordinationSettings>();
+        settings->snapshot_distance = 1;
+        settings->reserved_log_items = 1;
+        settings->rotate_log_storage_interval = 32;
+        testLogAndStateMachine(settings, 32);
+    }
+    {
+        CoordinationSettingsPtr settings = std::make_shared<CoordinationSettings>();
+        settings->snapshot_distance = 10;
+        settings->reserved_log_items = 7;
+        settings->rotate_log_storage_interval = 1;
+        testLogAndStateMachine(settings, 33);
+    }
+    {
+        CoordinationSettingsPtr settings = std::make_shared<CoordinationSettings>();
+        settings->snapshot_distance = 37;
+        settings->reserved_log_items = 1000;
+        settings->rotate_log_storage_interval = 5000;
+        testLogAndStateMachine(settings, 33);
+    }
+    {
+        CoordinationSettingsPtr settings = std::make_shared<CoordinationSettings>();
+        settings->snapshot_distance = 37;
+        settings->reserved_log_items = 1000;
+        settings->rotate_log_storage_interval = 5000;
+        testLogAndStateMachine(settings, 45);
+    }
+}
+
+TEST_F(RocksKeeperTest, TestEphemeralNodeRemove)
+{
+    using namespace Coordination;
+    using namespace DB;
+
+    ChangelogDirTest snapshots("./snapshots");
+    setSnapshotDirectory("./snapshots");
+    ChangelogDirTest rocks("./rocksdb");
+    setRocksDBDirectory("./rocksdb");
+
+    ResponsesQueue queue(std::numeric_limits<size_t>::max());
+    SnapshotsQueue snapshots_queue{1};
+
+    auto state_machine = std::make_shared<KeeperStateMachine<DB::KeeperRocksStorage>>(queue, snapshots_queue, keeper_context, nullptr);
+    state_machine->init();
+
+    std::shared_ptr<ZooKeeperCreateRequest> request_c = std::make_shared<ZooKeeperCreateRequest>();
+    request_c->path = "/hello";
+    request_c->is_ephemeral = true;
+    auto entry_c = getLogEntryFromZKRequest(0, 1, state_machine->getNextZxid(), request_c);
+    state_machine->pre_commit(1, entry_c->get_buf());
+    state_machine->commit(1, entry_c->get_buf());
+    const auto & storage = state_machine->getStorageUnsafe();
+
+    EXPECT_EQ(storage.ephemerals.size(), 1);
+    std::shared_ptr<ZooKeeperRemoveRequest> request_d = std::make_shared<ZooKeeperRemoveRequest>();
+    request_d->path = "/hello";
+    /// Delete from other session
+    auto entry_d = getLogEntryFromZKRequest(0, 2, state_machine->getNextZxid(), request_d);
+    state_machine->pre_commit(2, entry_d->get_buf());
+    state_machine->commit(2, entry_d->get_buf());
+
+    EXPECT_EQ(storage.ephemerals.size(), 0);
+}
+
+TEST_F(RocksKeeperTest, TestCreateNodeWithAuthSchemeForAclWhenAuthIsPrecommitted)
+{
+    using namespace Coordination;
+    using namespace DB;
+
+    ChangelogDirTest snapshots("./snapshots");
+    setSnapshotDirectory("./snapshots");
+    ChangelogDirTest rocks("./rocksdb");
+    setRocksDBDirectory("./rocksdb");
+    ResponsesQueue queue(std::numeric_limits<size_t>::max());
+    SnapshotsQueue snapshots_queue{1};
+
+    auto state_machine = std::make_shared<KeeperStateMachine<DB::KeeperRocksStorage>>(queue, snapshots_queue, keeper_context, nullptr);
+    state_machine->init();
+
+    String user_auth_data = "test_user:test_password";
+    String digest = KeeperRocksStorage::generateDigest(user_auth_data);
+
+    std::shared_ptr<ZooKeeperAuthRequest> auth_req = std::make_shared<ZooKeeperAuthRequest>();
+    auth_req->scheme = "digest";
+    auth_req->data = user_auth_data;
+
+    // Add auth data to the session
+    auto auth_entry = getLogEntryFromZKRequest(0, 1, state_machine->getNextZxid(), auth_req);
+    state_machine->pre_commit(1, auth_entry->get_buf());
+
+    // Create a node with 'auth' scheme for ACL
+    String node_path = "/hello";
+    std::shared_ptr<ZooKeeperCreateRequest> create_req = std::make_shared<ZooKeeperCreateRequest>();
+    create_req->path = node_path;
+    // When 'auth' scheme is used the creator must have been authenticated by the server (for example, using 'digest' scheme) before it can
+    // create nodes with this ACL.
+    create_req->acls = {{.permissions = 31, .scheme = "auth", .id = ""}};
+    auto create_entry = getLogEntryFromZKRequest(0, 1, state_machine->getNextZxid(), create_req);
+    state_machine->pre_commit(2, create_entry->get_buf());
+
+    const auto & uncommitted_state = state_machine->getStorageUnsafe().uncommitted_state;
+    ASSERT_TRUE(uncommitted_state.nodes.contains(node_path));
+
+    // commit log entries
+    state_machine->commit(1, auth_entry->get_buf());
+    state_machine->commit(2, create_entry->get_buf());
+
+    auto node = uncommitted_state.getNode(node_path);
+    ASSERT_NE(node, nullptr);
+    auto acls = uncommitted_state.getACLs(node_path);
+    ASSERT_EQ(acls.size(), 1);
+    EXPECT_EQ(acls[0].scheme, "digest");
+    EXPECT_EQ(acls[0].id, digest);
+    EXPECT_EQ(acls[0].permissions, 31);
+}
+
+TEST_F(RocksKeeperTest, TestSetACLWithAuthSchemeForAclWhenAuthIsPrecommitted)
+{
+    using namespace Coordination;
+    using namespace DB;
+
+    ChangelogDirTest snapshots("./snapshots");
+    setSnapshotDirectory("./snapshots");
+    ChangelogDirTest rocks("./rocksdb");
+    setRocksDBDirectory("./rocksdb");
+
+    ResponsesQueue queue(std::numeric_limits<size_t>::max());
+    SnapshotsQueue snapshots_queue{1};
+
+    auto state_machine = std::make_shared<KeeperStateMachine<DB::KeeperRocksStorage>>(queue, snapshots_queue, keeper_context, nullptr);
+    state_machine->init();
+
+    String user_auth_data = "test_user:test_password";
+    String digest = KeeperRocksStorage::generateDigest(user_auth_data);
+
+    std::shared_ptr<ZooKeeperAuthRequest> auth_req = std::make_shared<ZooKeeperAuthRequest>();
+    auth_req->scheme = "digest";
+    auth_req->data = user_auth_data;
+
+    // Add auth data to the session
+    auto auth_entry = getLogEntryFromZKRequest(0, 1, state_machine->getNextZxid(), auth_req);
+    state_machine->pre_commit(1, auth_entry->get_buf());
+
+    // Create a node
+    String node_path = "/hello";
+    std::shared_ptr<ZooKeeperCreateRequest> create_req = std::make_shared<ZooKeeperCreateRequest>();
+    create_req->path = node_path;
+    auto create_entry = getLogEntryFromZKRequest(0, 1, state_machine->getNextZxid(), create_req);
+    state_machine->pre_commit(2, create_entry->get_buf());
+
+    // Set ACL with 'auth' scheme for ACL
+    std::shared_ptr<ZooKeeperSetACLRequest> set_acl_req = std::make_shared<ZooKeeperSetACLRequest>();
+    set_acl_req->path = node_path;
+    // When 'auth' scheme is used the creator must have been authenticated by the server (for example, using 'digest' scheme) before it can
+    // set this ACL.
+    set_acl_req->acls = {{.permissions = 31, .scheme = "auth", .id = ""}};
+    auto set_acl_entry = getLogEntryFromZKRequest(0, 1, state_machine->getNextZxid(), set_acl_req);
+    state_machine->pre_commit(3, set_acl_entry->get_buf());
+
+    // commit all entries
+    state_machine->commit(1, auth_entry->get_buf());
+    state_machine->commit(2, create_entry->get_buf());
+    state_machine->commit(3, set_acl_entry->get_buf());
+
+    const auto & uncommitted_state = state_machine->getStorageUnsafe().uncommitted_state;
+    auto node = uncommitted_state.getNode(node_path);
+
+    ASSERT_NE(node, nullptr);
+    auto acls = uncommitted_state.getACLs(node_path);
+    ASSERT_EQ(acls.size(), 1);
+    EXPECT_EQ(acls[0].scheme, "digest");
+    EXPECT_EQ(acls[0].id, digest);
+    EXPECT_EQ(acls[0].permissions, 31);
+}
+
+TEST_F(RocksKeeperTest, TestStorageSnapshotEqual)
+{
+    ChangelogDirTest test("./snapshots");
+    ChangelogDirTest rocks("./rocksdb");
+    setSnapshotDirectory("./snapshots");
+    setRocksDBDirectory("./rocksdb");
+
+    std::optional<UInt128> snapshot_hash;
+    for (size_t i = 0; i < 15; ++i)
+    {
+        DB::KeeperSnapshotManager<DB::KeeperRocksStorage> manager(3, keeper_context);
+
+        DB::KeeperRocksStorage storage(500, "", keeper_context);
+        addNode(storage, "/hello", "");
+        for (size_t j = 0; j < 100; ++j)
+        {
+            addNode(storage, "/hello_" + std::to_string(j), "world", 1);
+            addNode(storage, "/hello/somepath_" + std::to_string(j), "somedata", 3);
+        }
+
+        storage.session_id_counter = 5;
+
+        storage.ephemerals[3] = {"/hello"};
+        storage.ephemerals[1] = {"/hello/somepath"};
+
+        for (size_t j = 0; j < 3333; ++j)
+            storage.getSessionID(130 * j);
+
+        DB::KeeperStorageSnapshot<DB::KeeperRocksStorage> snapshot(&storage, storage.zxid);
+
+        auto buf = manager.serializeSnapshotToBuffer(snapshot);
+
+        auto new_hash = sipHash128(reinterpret_cast<char *>(buf->data()), buf->size());
+        if (!snapshot_hash.has_value())
+        {
+            snapshot_hash = new_hash;
+        }
+        else
+        {
+            EXPECT_EQ(*snapshot_hash, new_hash);
+        }
+    }
+}
+
+TEST_F(RocksKeeperTest, TestStorageSnapshotDifferentCompressions)
+{
+    ChangelogDirTest test("./snapshots");
+    setSnapshotDirectory("./snapshots");
+    ChangelogDirTest rocks("./rocksdb");
+    setRocksDBDirectory("./rocksdb");
+
+    DB::KeeperSnapshotManager<DB::KeeperRocksStorage> manager(3, keeper_context);
+
+    DB::KeeperRocksStorage storage(500, "", keeper_context);
+    addNode(storage, "/hello1", "world", 1);
+    addNode(storage, "/hello2", "somedata", 3);
+    storage.session_id_counter = 5;
+    storage.zxid = 2;
+    storage.ephemerals[3] = {"/hello2"};
+    storage.ephemerals[1] = {"/hello1"};
+    storage.getSessionID(130);
+    storage.getSessionID(130);
+
+    DB::KeeperStorageSnapshot<DB::KeeperRocksStorage> snapshot(&storage, 2);
+
+    auto buf = manager.serializeSnapshotToBuffer(snapshot);
+    manager.serializeSnapshotBufferToDisk(*buf, 2);
+    EXPECT_TRUE(fs::exists("./snapshots/snapshot_2.bin.zstd"));
+
+    DB::KeeperSnapshotManager<DB::KeeperRocksStorage> new_manager(3, keeper_context, false);
+
+    auto debuf = new_manager.deserializeSnapshotBufferFromDisk(2);
+
+    auto [restored_storage, snapshot_meta, _] = new_manager.deserializeSnapshotFromBuffer(debuf);
+
+    EXPECT_EQ(restored_storage->container.size(), 6);
+    EXPECT_EQ(restored_storage->container.find("/")->value.numChildren(), 3);
+    EXPECT_EQ(restored_storage->container.find("/hello1")->value.numChildren(), 0);
+    EXPECT_EQ(restored_storage->container.find("/hello2")->value.numChildren(), 0);
+
+    EXPECT_EQ(restored_storage->container.find("/")->value.getData(), "");
+    EXPECT_EQ(restored_storage->container.find("/hello1")->value.getData(), "world");
+    EXPECT_EQ(restored_storage->container.find("/hello2")->value.getData(), "somedata");
+    EXPECT_EQ(restored_storage->session_id_counter, 7);
+    EXPECT_EQ(restored_storage->zxid, 2);
+    EXPECT_EQ(restored_storage->ephemerals.size(), 2);
+    EXPECT_EQ(restored_storage->ephemerals[3].size(), 1);
+    EXPECT_EQ(restored_storage->ephemerals[1].size(), 1);
+    EXPECT_EQ(restored_storage->session_and_timeout.size(), 2);
+}
+
+template <typename ResponseType>
+ResponseType getSingleResponse(const auto & responses)
+{
+    EXPECT_FALSE(responses.empty());
+    return dynamic_cast<ResponseType &>(*responses[0].response);
+}
+
+TEST_F(RocksKeeperTest, TestUncommittedStateBasicCrud)
+{
+    ChangelogDirTest test("./rocksdb");
+    setRocksDBDirectory("./rocksdb");
+
+    using namespace DB;
+    using namespace Coordination;
+
+    DB::KeeperRocksStorage storage{500, "", keeper_context};
+
+    constexpr std::string_view path = "/test";
+
+    const auto get_committed_data = [&]() -> std::optional<String>
+    {
+        auto request = std::make_shared<ZooKeeperGetRequest>();
+        request->path = path;
+        auto responses = storage.processRequest(request, 0, std::nullopt, true, true);
+        const auto & get_response = getSingleResponse<ZooKeeperGetResponse>(responses);
+
+        if (get_response.error != Error::ZOK)
+            return std::nullopt;
+
+        return get_response.data;
+    };
+
+    const auto preprocess_get = [&](int64_t zxid)
+    {
+        auto get_request = std::make_shared<ZooKeeperGetRequest>();
+        get_request->path = path;
+        storage.preprocessRequest(get_request, 0, 0, zxid);
+        return get_request;
+    };
+
+    const auto create_request = std::make_shared<ZooKeeperCreateRequest>();
+    create_request->path = path;
+    create_request->data = "initial_data";
+    storage.preprocessRequest(create_request, 0, 0, 1);
+    storage.preprocessRequest(create_request, 0, 0, 2);
+
+    ASSERT_FALSE(get_committed_data());
+
+    const auto after_create_get = preprocess_get(3);
+
+    ASSERT_FALSE(get_committed_data());
+
+    const auto set_request = std::make_shared<ZooKeeperSetRequest>();
+    set_request->path = path;
+    set_request->data = "new_data";
+    storage.preprocessRequest(set_request, 0, 0, 4);
+
+    const auto after_set_get = preprocess_get(5);
+
+    ASSERT_FALSE(get_committed_data());
+
+    const auto remove_request = std::make_shared<ZooKeeperRemoveRequest>();
+    remove_request->path = path;
+    storage.preprocessRequest(remove_request, 0, 0, 6);
+    storage.preprocessRequest(remove_request, 0, 0, 7);
+
+    const auto after_remove_get = preprocess_get(8);
+
+    ASSERT_FALSE(get_committed_data());
+
+    {
+        const auto responses = storage.processRequest(create_request, 0, 1);
+        const auto & create_response = getSingleResponse<ZooKeeperCreateResponse>(responses);
+        ASSERT_EQ(create_response.error, Error::ZOK);
+    }
+
+    {
+        const auto responses = storage.processRequest(create_request, 0, 2);
+        const auto & create_response = getSingleResponse<ZooKeeperCreateResponse>(responses);
+        ASSERT_EQ(create_response.error, Error::ZNODEEXISTS);
+    }
+
+    {
+        const auto responses = storage.processRequest(after_create_get, 0, 3);
+        const auto & get_response = getSingleResponse<ZooKeeperGetResponse>(responses);
+        ASSERT_EQ(get_response.error, Error::ZOK);
+        ASSERT_EQ(get_response.data, "initial_data");
+    }
+
+    ASSERT_EQ(get_committed_data(), "initial_data");
+
+    {
+        const auto responses = storage.processRequest(set_request, 0, 4);
+        const auto & create_response = getSingleResponse<ZooKeeperSetResponse>(responses);
+        ASSERT_EQ(create_response.error, Error::ZOK);
+    }
+
+    {
+        const auto responses = storage.processRequest(after_set_get, 0, 5);
+        const auto & get_response = getSingleResponse<ZooKeeperGetResponse>(responses);
+        ASSERT_EQ(get_response.error, Error::ZOK);
+        ASSERT_EQ(get_response.data, "new_data");
+    }
+
+    ASSERT_EQ(get_committed_data(), "new_data");
+
+    {
+        const auto responses = storage.processRequest(remove_request, 0, 6);
+        const auto & create_response = getSingleResponse<ZooKeeperRemoveResponse>(responses);
+        ASSERT_EQ(create_response.error, Error::ZOK);
+    }
+
+    {
+        const auto responses = storage.processRequest(remove_request, 0, 7);
+        const auto & remove_response = getSingleResponse<ZooKeeperRemoveResponse>(responses);
+        ASSERT_EQ(remove_response.error, Error::ZNONODE);
+    }
+
+    {
+        const auto responses = storage.processRequest(after_remove_get, 0, 8);
+        const auto & get_response = getSingleResponse<ZooKeeperGetResponse>(responses);
+        ASSERT_EQ(get_response.error, Error::ZNONODE);
+    }
+
+    ASSERT_FALSE(get_committed_data());
+}
+
+TEST_F(RocksKeeperTest, TestListRequestTypes)
+{
+    ChangelogDirTest test("./rocksdb");
+    setRocksDBDirectory("./rocksdb");
+
+    using namespace DB;
+    using namespace Coordination;
+    KeeperRocksStorage storage{500, "", keeper_context};
+
+    int32_t zxid = 0;
+
+    static constexpr std::string_view test_path = "/list_request_type/node";
+
+    const auto create_path = [&](const auto & path, bool is_ephemeral, bool is_sequential = true)
+    {
+        const auto create_request = std::make_shared<ZooKeeperCreateRequest>();
+        int new_zxid = ++zxid;
+        create_request->path = path;
+        create_request->is_sequential = is_sequential;
+        create_request->is_ephemeral = is_ephemeral;
+        storage.preprocessRequest(create_request, 1, 0, new_zxid);
+        auto responses = storage.processRequest(create_request, 1, new_zxid);
+
+        EXPECT_GE(responses.size(), 1);
+        EXPECT_EQ(responses[0].response->error, Coordination::Error::ZOK) << "Failed to create " << path;
+        const auto & create_response = dynamic_cast<ZooKeeperCreateResponse &>(*responses[0].response);
+        return create_response.path_created;
+    };
+
+    create_path(parentNodePath(StringRef{test_path}).toString(), false, false);
+
+    static constexpr size_t persistent_num = 5;
+    std::unordered_set<std::string> expected_persistent_children;
+    for (size_t i = 0; i < persistent_num; ++i)
+    {
+        expected_persistent_children.insert(getBaseNodeName(create_path(test_path, false)).toString());
+    }
+    ASSERT_EQ(expected_persistent_children.size(), persistent_num);
+
+    static constexpr size_t ephemeral_num = 5;
+    std::unordered_set<std::string> expected_ephemeral_children;
+    for (size_t i = 0; i < ephemeral_num; ++i)
+    {
+        expected_ephemeral_children.insert(getBaseNodeName(create_path(test_path, true)).toString());
+    }
+    ASSERT_EQ(expected_ephemeral_children.size(), ephemeral_num);
+
+    const auto get_children = [&](const auto list_request_type)
+    {
+        const auto list_request = std::make_shared<ZooKeeperFilteredListRequest>();
+        int new_zxid = ++zxid;
+        list_request->path = parentNodePath(StringRef{test_path}).toString();
+        list_request->list_request_type = list_request_type;
+        storage.preprocessRequest(list_request, 1, 0, new_zxid);
+        auto responses = storage.processRequest(list_request, 1, new_zxid);
+
+        EXPECT_GE(responses.size(), 1);
+        const auto & list_response = dynamic_cast<ZooKeeperListResponse &>(*responses[0].response);
+        EXPECT_EQ(list_response.error, Coordination::Error::ZOK);
+        return list_response.names;
+    };
+
+    const auto persistent_children = get_children(ListRequestType::PERSISTENT_ONLY);
+    EXPECT_EQ(persistent_children.size(), persistent_num);
+    for (const auto & child : persistent_children)
+    {
+        EXPECT_TRUE(expected_persistent_children.contains(child)) << "Missing persistent child " << child;
+    }
+
+    const auto ephemeral_children = get_children(ListRequestType::EPHEMERAL_ONLY);
+    EXPECT_EQ(ephemeral_children.size(), ephemeral_num);
+    for (const auto & child : ephemeral_children)
+    {
+        EXPECT_TRUE(expected_ephemeral_children.contains(child)) << "Missing ephemeral child " << child;
+    }
+
+    const auto all_children = get_children(ListRequestType::ALL);
+    EXPECT_EQ(all_children.size(), ephemeral_num + persistent_num);
+    for (const auto & child : all_children)
+    {
+        EXPECT_TRUE(expected_ephemeral_children.contains(child) || expected_persistent_children.contains(child))
+            << "Missing child " << child;
+    }
+}
+
+TEST_F(RocksKeeperTest, TestFeatureFlags)
+{
+    ChangelogDirTest test("./rocksdb");
+    setRocksDBDirectory("./rocksdb");
+
+    using namespace Coordination;
+    KeeperMemoryStorage storage{500, "", keeper_context};
+    auto request = std::make_shared<ZooKeeperGetRequest>();
+    request->path = DB::keeper_api_feature_flags_path;
+    auto responses = storage.processRequest(request, 0, std::nullopt, true, true);
+    const auto & get_response = getSingleResponse<ZooKeeperGetResponse>(responses);
+    DB::KeeperFeatureFlags feature_flags;
+    feature_flags.setFeatureFlags(get_response.data);
+    ASSERT_TRUE(feature_flags.isEnabled(KeeperFeatureFlag::FILTERED_LIST));
+    ASSERT_TRUE(feature_flags.isEnabled(KeeperFeatureFlag::MULTI_READ));
+    ASSERT_FALSE(feature_flags.isEnabled(KeeperFeatureFlag::CHECK_NOT_EXISTS));
+}
+
+TEST_F(RocksKeeperTest, TestSystemNodeModify)
+{
+    ChangelogDirTest test("./rocksdb");
+    setRocksDBDirectory("./rocksdb");
+
+    using namespace Coordination;
+    int64_t zxid{0};
+
+    // On INIT we abort when a system path is modified
+    keeper_context->setServerState(KeeperContext::Phase::RUNNING);
+    KeeperRocksStorage storage{500, "", keeper_context};
+    const auto assert_create = [&](const std::string_view path, const auto expected_code)
+    {
+        auto request = std::make_shared<ZooKeeperCreateRequest>();
+        request->path = path;
+        storage.preprocessRequest(request, 0, 0, zxid);
+        auto responses = storage.processRequest(request, 0, zxid);
+        ASSERT_FALSE(responses.empty());
+
+        const auto & response = responses[0];
+        ASSERT_EQ(response.response->error, expected_code) << "Unexpected error for path " << path;
+
+        ++zxid;
+    };
+
+    assert_create("/keeper", Error::ZBADARGUMENTS);
+    assert_create("/keeper/with_child", Error::ZBADARGUMENTS);
+    assert_create(DB::keeper_api_version_path, Error::ZBADARGUMENTS);
+
+    assert_create("/keeper_map", Error::ZOK);
+    assert_create("/keeper1", Error::ZOK);
+    assert_create("/keepe", Error::ZOK);
+    assert_create("/keeper1/test", Error::ZOK);
+}
+
+TEST_F(RocksKeeperTest, TestCheckNotExistsRequest)
+{
+    ChangelogDirTest test("./rocksdb");
+    setRocksDBDirectory("./rocksdb");
+
+    using namespace DB;
+    using namespace Coordination;
+
+    KeeperRocksStorage storage{500, "", keeper_context};
+
+    int32_t zxid = 0;
+
+    const auto create_path = [&](const auto & path)
+    {
+        const auto create_request = std::make_shared<ZooKeeperCreateRequest>();
+        int new_zxid = ++zxid;
+        create_request->path = path;
+        storage.preprocessRequest(create_request, 1, 0, new_zxid);
+        auto responses = storage.processRequest(create_request, 1, new_zxid);
+
+        EXPECT_GE(responses.size(), 1);
+        EXPECT_EQ(responses[0].response->error, Coordination::Error::ZOK) << "Failed to create " << path;
+    };
+
+    const auto check_request = std::make_shared<ZooKeeperCheckRequest>();
+    check_request->path = "/test_node";
+    check_request->not_exists = true;
+
+    {
+        SCOPED_TRACE("CheckNotExists returns ZOK");
+        int new_zxid = ++zxid;
+        storage.preprocessRequest(check_request, 1, 0, new_zxid);
+        auto responses = storage.processRequest(check_request, 1, new_zxid);
+        EXPECT_GE(responses.size(), 1);
+        auto error = responses[0].response->error;
+        EXPECT_EQ(error, Coordination::Error::ZOK) << "CheckNotExists returned invalid result: " << errorMessage(error);
+    }
+
+    create_path("/test_node");
+    auto node_it = storage.container.find("/test_node");
+    ASSERT_NE(node_it, storage.container.end());
+    auto node_version = node_it->value.version;
+
+    {
+        SCOPED_TRACE("CheckNotExists returns ZNODEEXISTS");
+        int new_zxid = ++zxid;
+        storage.preprocessRequest(check_request, 1, 0, new_zxid);
+        auto responses = storage.processRequest(check_request, 1, new_zxid);
+        EXPECT_GE(responses.size(), 1);
+        auto error = responses[0].response->error;
+        EXPECT_EQ(error, Coordination::Error::ZNODEEXISTS) << "CheckNotExists returned invalid result: " << errorMessage(error);
+    }
+
+    {
+        SCOPED_TRACE("CheckNotExists returns ZNODEEXISTS for same version");
+        int new_zxid = ++zxid;
+        check_request->version = node_version;
+        storage.preprocessRequest(check_request, 1, 0, new_zxid);
+        auto responses = storage.processRequest(check_request, 1, new_zxid);
+        EXPECT_GE(responses.size(), 1);
+        auto error = responses[0].response->error;
+        EXPECT_EQ(error, Coordination::Error::ZNODEEXISTS) << "CheckNotExists returned invalid result: " << errorMessage(error);
+    }
+
+    {
+        SCOPED_TRACE("CheckNotExists returns ZOK for different version");
+        int new_zxid = ++zxid;
+        check_request->version = node_version + 1;
+        storage.preprocessRequest(check_request, 1, 0, new_zxid);
+        auto responses = storage.processRequest(check_request, 1, new_zxid);
+        EXPECT_GE(responses.size(), 1);
+        auto error = responses[0].response->error;
+        EXPECT_EQ(error, Coordination::Error::ZOK) << "CheckNotExists returned invalid result: " << errorMessage(error);
+    }
+}
+
+TEST_F(RocksKeeperTest, TestReapplyingDeltas)
+{
+    ChangelogDirTest test("./rocksdb");
+    setRocksDBDirectory("./rocksdb");
+
+    using namespace DB;
+    using namespace Coordination;
+
+    static constexpr int64_t initial_zxid = 100;
+
+    const auto create_request = std::make_shared<ZooKeeperCreateRequest>();
+    create_request->path = "/test/data";
+    create_request->is_sequential = true;
+
+    const auto process_create = [](KeeperRocksStorage & storage, const auto & request, int64_t zxid)
+    {
+        storage.preprocessRequest(request, 1, 0, zxid);
+        auto responses = storage.processRequest(request, 1, zxid);
+        EXPECT_GE(responses.size(), 1);
+        EXPECT_EQ(responses[0].response->error, Error::ZOK);
+    };
+
+    const auto commit_initial_data = [&](auto & storage)
+    {
+        int64_t zxid = 1;
+
+        const auto root_create = std::make_shared<ZooKeeperCreateRequest>();
+        root_create->path = "/test";
+        process_create(storage, root_create, zxid);
+        ++zxid;
+
+        for (; zxid <= initial_zxid; ++zxid)
+            process_create(storage, create_request, zxid);
+    };
+
+    KeeperRocksStorage storage1{500, "", keeper_context};
+    commit_initial_data(storage1);
+
+    for (int64_t zxid = initial_zxid + 1; zxid < initial_zxid + 50; ++zxid)
+        storage1.preprocessRequest(create_request, 1, 0, zxid, /*check_acl=*/true, /*digest=*/std::nullopt, /*log_idx=*/zxid);
+
+    /// create identical new storage
+    KeeperRocksStorage storage2{500, "", keeper_context};
+    commit_initial_data(storage2);
+
+    storage1.applyUncommittedState(storage2, initial_zxid);
+
+    const auto commit_unprocessed = [&](KeeperRocksStorage & storage)
+    {
+        for (int64_t zxid = initial_zxid + 1; zxid < initial_zxid + 50; ++zxid)
+        {
+            auto responses = storage.processRequest(create_request, 1, zxid);
+            EXPECT_GE(responses.size(), 1);
+            EXPECT_EQ(responses[0].response->error, Error::ZOK);
+        }
+    };
+
+    commit_unprocessed(storage1);
+    commit_unprocessed(storage2);
+
+    const auto get_children = [&](KeeperRocksStorage & storage)
+    {
+        const auto list_request = std::make_shared<ZooKeeperListRequest>();
+        list_request->path = "/test";
+        auto responses = storage.processRequest(list_request, 1, std::nullopt, /*check_acl=*/true, /*is_local=*/true);
+        EXPECT_EQ(responses.size(), 1);
+        const auto * list_response = dynamic_cast<const ListResponse *>(responses[0].response.get());
+        EXPECT_TRUE(list_response);
+        return list_response->names;
+    };
+
+    auto children1 = get_children(storage1);
+    std::unordered_set<std::string> children1_set(children1.begin(), children1.end());
+
+    auto children2 = get_children(storage2);
+    std::unordered_set<std::string> children2_set(children2.begin(), children2.end());
+
+    ASSERT_TRUE(children1_set == children2_set);
+}
+
+#endif

From 96f35f9a4746654ace7bb5a54fef8f503be9ddcb Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Thu, 4 Apr 2024 18:32:53 +0200
Subject: [PATCH 0028/1488] fix style

---
 src/Coordination/RocksDBContainer.h | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/Coordination/RocksDBContainer.h b/src/Coordination/RocksDBContainer.h
index 683678cfcd1..4076e39655a 100644
--- a/src/Coordination/RocksDBContainer.h
+++ b/src/Coordination/RocksDBContainer.h
@@ -33,6 +33,7 @@ struct RocksDBContainer
     using Node = Node_;
 
 private:
+    /// this is only for debug.
     std::string toHex(std::string origin)
     {
         auto toHexChar = [](char v)
@@ -43,7 +44,8 @@ private:
         };
         std::string result;
         result.reserve(2 * origin.size());
-        for (size_t i = 0; i < origin.size(); ++i) {
+        for (size_t i = 0; i < origin.size(); ++i)
+        {
             unsigned char c = origin[i];
             result.push_back(toHexChar(c >> 4));
             result.push_back(toHexChar(c & 0xf));

From a63ee58fc4cea606cececc8edbc2bec58649861d Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Fri, 5 Apr 2024 12:16:40 +0200
Subject: [PATCH 0029/1488] fix special build

---
 src/Coordination/RocksDBContainer.h           | 3 +--
 src/Coordination/tests/gtest_rocks_keeper.cpp | 8 ++++----
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/src/Coordination/RocksDBContainer.h b/src/Coordination/RocksDBContainer.h
index 4076e39655a..9d60a49ba23 100644
--- a/src/Coordination/RocksDBContainer.h
+++ b/src/Coordination/RocksDBContainer.h
@@ -44,9 +44,8 @@ private:
         };
         std::string result;
         result.reserve(2 * origin.size());
-        for (size_t i = 0; i < origin.size(); ++i)
+        for (unsigned char c : origin)
         {
-            unsigned char c = origin[i];
             result.push_back(toHexChar(c >> 4));
             result.push_back(toHexChar(c & 0xf));
         }
diff --git a/src/Coordination/tests/gtest_rocks_keeper.cpp b/src/Coordination/tests/gtest_rocks_keeper.cpp
index 15f97676db3..d42eb66d683 100644
--- a/src/Coordination/tests/gtest_rocks_keeper.cpp
+++ b/src/Coordination/tests/gtest_rocks_keeper.cpp
@@ -1,4 +1,8 @@
 #include <gtest/gtest.h>
+#include "config.h"
+
+#if USE_NURAFT and USE_ROCKSDB
+
 #include <Coordination/KeeperContext.h>
 #include <Coordination/KeeperCommon.h>
 #include <Coordination/KeeperLogStore.h>
@@ -10,10 +14,6 @@
 #include <Disks/DiskLocal.h>
 #include <Poco/ConsoleChannel.h>
 
-#include "config.h"
-
-#if USE_NURAFT and USE_ROCKSDB
-
 namespace fs = std::filesystem;
 struct ChangelogDirTest
 {

From 55f4235d834301a6acd77964e9601a09e4b7e2ba Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Fri, 5 Apr 2024 14:59:05 +0200
Subject: [PATCH 0030/1488] enable rocksdb and check ci

---
 src/Coordination/KeeperContext.cpp | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/Coordination/KeeperContext.cpp b/src/Coordination/KeeperContext.cpp
index 4ae3c295824..949e699f436 100644
--- a/src/Coordination/KeeperContext.cpp
+++ b/src/Coordination/KeeperContext.cpp
@@ -160,8 +160,11 @@ void KeeperContext::initialize(const Poco::Util::AbstractConfiguration & config,
     initializeDisks(config);
 
     #if USE_ROCKSDB
-    if (config.getBool("keeper_server.coordination_settings.use_rocksdb", false))
+    if (config.getBool("keeper_server.coordination_settings.use_rocksdb", true))
+    {
         rocksdb_options = std::make_shared<rocksdb::Options>(getRocksDBOptionsFromConfig(config));
+        digest_enabled = false; /// TODO: support digest
+    }
     #endif
 }
 

From e93872c826be150291115b6f4fb81a2fe6000384 Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Sat, 6 Apr 2024 18:22:08 +0200
Subject: [PATCH 0031/1488] try test

---
 src/Coordination/CoordinationSettings.h      | 2 +-
 src/Coordination/KeeperServer.cpp            | 7 +++----
 src/Coordination/KeeperSnapshotManagerS3.cpp | 2 --
 3 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/src/Coordination/CoordinationSettings.h b/src/Coordination/CoordinationSettings.h
index c5bd672055f..4a898d1975c 100644
--- a/src/Coordination/CoordinationSettings.h
+++ b/src/Coordination/CoordinationSettings.h
@@ -55,7 +55,7 @@ struct Settings;
     M(UInt64, min_request_size_for_cache, 50 * 1024, "Minimal size of the request to cache the deserialization result. Caching can have negative effect on latency for smaller requests, set to 0 to disable", 0) \
     M(UInt64, raft_limits_reconnect_limit, 50, "If connection to a peer is silent longer than this limit * (multiplied by heartbeat interval), we re-establish the connection.", 0) \
     M(Bool, async_replication, false, "Enable async replication. All write and read guarantees are preserved while better performance is achieved. Settings is disabled by default to not break backwards compatibility.", 0) \
-    M(Bool, use_rocksdb, false, "Use rocksdb as backend storage", 0) \
+    M(Bool, use_rocksdb, true, "Use rocksdb as backend storage", 0) \
     M(UInt64, latest_logs_cache_size_threshold, 1 * 1024 * 1024 * 1024, "Maximum total size of in-memory cache of latest log entries.", 0) \
     M(UInt64, commit_logs_cache_size_threshold, 500 * 1024 * 1024, "Maximum total size of in-memory cache of log entries needed next for commit.", 0) \
     M(UInt64, disk_move_retries_wait_ms, 1000, "How long to wait between retries after a failure which happened while a file was being moved between disks.", 0) \
diff --git a/src/Coordination/KeeperServer.cpp b/src/Coordination/KeeperServer.cpp
index 43688ca9cda..951d3729289 100644
--- a/src/Coordination/KeeperServer.cpp
+++ b/src/Coordination/KeeperServer.cpp
@@ -131,10 +131,10 @@ KeeperServer::KeeperServer(
     if (keeper_context->getCoordinationSettings()->quorum_reads)
         LOG_WARNING(log, "Quorum reads enabled, Keeper will work slower.");
 
+#if USE_ROCKSDB
     const auto & coordination_settings = keeper_context->getCoordinationSettings();
     if (coordination_settings->use_rocksdb)
     {
-#if USE_ROCKSDB
         state_machine = nuraft::cs_new<KeeperStateMachine<KeeperRocksStorage>>(
             responses_queue_,
             snapshots_queue_,
@@ -142,11 +142,10 @@ KeeperServer::KeeperServer(
             config.getBool("keeper_server.upload_snapshot_on_exit", true) ? &snapshot_manager_s3 : nullptr,
             commit_callback,
             checkAndGetSuperdigest(configuration_and_settings_->super_digest));
-#else
-        throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "RocksDB support is disabled because ClickHouse was built without RocksDB support.");
-#endif
+        LOG_WARNING(log, "Use RocksDB as Keeper backend storage.");
     }
     else
+#endif
         state_machine = nuraft::cs_new<KeeperStateMachine<KeeperMemoryStorage>>(
             responses_queue_,
             snapshots_queue_,
diff --git a/src/Coordination/KeeperSnapshotManagerS3.cpp b/src/Coordination/KeeperSnapshotManagerS3.cpp
index b984b8ad18e..b54e2ceaa15 100644
--- a/src/Coordination/KeeperSnapshotManagerS3.cpp
+++ b/src/Coordination/KeeperSnapshotManagerS3.cpp
@@ -25,8 +25,6 @@
 
 #include <filesystem>
 
-namespace fs = std::filesystem;
-
 namespace DB
 {
 

From 801dcca2e62e68d8e6110a8ef33fc22b451913ce Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Tue, 9 Apr 2024 14:56:26 +0200
Subject: [PATCH 0032/1488] fix tests

---
 src/Coordination/KeeperContext.cpp  | 2 +-
 src/Coordination/RocksDBContainer.h | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Coordination/KeeperContext.cpp b/src/Coordination/KeeperContext.cpp
index 949e699f436..b65bb02ca5d 100644
--- a/src/Coordination/KeeperContext.cpp
+++ b/src/Coordination/KeeperContext.cpp
@@ -123,7 +123,7 @@ KeeperContext::Storage KeeperContext::getRocksDBPathFromConfig(const Poco::Util:
     const auto create_local_disk = [](const auto & path)
     {
         if (fs::exists(path))
-            fs::remove(path);
+            fs::remove_all(path);
         fs::create_directories(path);
 
         return std::make_shared<DiskLocal>("LocalRocksDBDisk", path);
diff --git a/src/Coordination/RocksDBContainer.h b/src/Coordination/RocksDBContainer.h
index 9d60a49ba23..a3ff5143c6a 100644
--- a/src/Coordination/RocksDBContainer.h
+++ b/src/Coordination/RocksDBContainer.h
@@ -353,7 +353,7 @@ public:
 
     void insertOrReplace(KeyPtr key_data, size_t key_size, Node value)
     {
-        std::string key(key_data.release(), key_size);
+        std::string key(key_data.get(), key_size);
         insertOrReplace(key, value);
     }
 

From f0fdeb6d3ef9ae51098d11a0a22bb138b6a58df0 Mon Sep 17 00:00:00 2001
From: Nikita Taranov <nikita.taranov@clickhouse.com>
Date: Sat, 24 Feb 2024 22:36:29 +0100
Subject: [PATCH 0033/1488] move adaptive task size calculation into base pool

---
 src/Core/Settings.h                           |  3 +-
 .../MergeTree/MergeTreePrefetchedReadPool.cpp | 24 +--------
 src/Storages/MergeTree/MergeTreeReadPool.cpp  | 53 +++---------------
 src/Storages/MergeTree/MergeTreeReadPool.h    |  1 -
 .../MergeTree/MergeTreeReadPoolBase.cpp       | 54 +++++++++++++++++--
 .../MergeTree/MergeTreeReadPoolBase.h         |  2 +-
 .../MergeTreeReadPoolParallelReplicas.cpp     |  1 +
 ...rgeTreeReadPoolParallelReplicasInOrder.cpp |  1 +
 src/Storages/MergeTree/MergeTreeReadTask.h    |  2 +
 9 files changed, 64 insertions(+), 77 deletions(-)

diff --git a/src/Core/Settings.h b/src/Core/Settings.h
index d5ea9534e6c..a5fcb537539 100644
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@@ -765,7 +765,7 @@ class IColumn;
     M(UInt64, merge_tree_min_rows_for_concurrent_read_for_remote_filesystem, (20 * 8192), "If at least as many lines are read from one file, the reading can be parallelized, when reading from remote filesystem.", 0) \
     M(UInt64, merge_tree_min_bytes_for_concurrent_read_for_remote_filesystem, (24 * 10 * 1024 * 1024), "If at least as many bytes are read from one file, the reading can be parallelized, when reading from remote filesystem.", 0) \
     M(UInt64, remote_read_min_bytes_for_seek, 4 * DBMS_DEFAULT_BUFFER_SIZE, "Min bytes required for remote read (url, s3) to do seek, instead of read with ignore.", 0) \
-    M(UInt64, merge_tree_min_bytes_per_task_for_remote_reading, 4 * DBMS_DEFAULT_BUFFER_SIZE, "Min bytes to read per task.", 0) \
+    M(UInt64, merge_tree_min_bytes_per_task_for_remote_reading, 2 * DBMS_DEFAULT_BUFFER_SIZE, "Min bytes to read per task.", 0) ALIAS(filesystem_prefetch_min_bytes_for_single_read_task) \
     M(Bool, merge_tree_use_const_size_tasks_for_remote_reading, true, "Whether to use constant size tasks for reading from a remote table.", 0) \
     M(Bool, merge_tree_determine_task_size_by_prewhere_columns, true, "Whether to use only prewhere columns size to determine reading task size.", 0) \
     M(UInt64, merge_tree_compact_parts_min_granules_to_multibuffer_read, 16, "Only available in ClickHouse Cloud", 0) \
@@ -806,7 +806,6 @@ class IColumn;
     M(UInt64, prefetch_buffer_size, DBMS_DEFAULT_BUFFER_SIZE, "The maximum size of the prefetch buffer to read from the filesystem.", 0) \
     M(UInt64, filesystem_prefetch_step_bytes, 0, "Prefetch step in bytes. Zero means `auto` - approximately the best prefetch step will be auto deduced, but might not be 100% the best. The actual value might be different because of setting filesystem_prefetch_min_bytes_for_single_read_task", 0) \
     M(UInt64, filesystem_prefetch_step_marks, 0, "Prefetch step in marks. Zero means `auto` - approximately the best prefetch step will be auto deduced, but might not be 100% the best. The actual value might be different because of setting filesystem_prefetch_min_bytes_for_single_read_task", 0) \
-    M(UInt64, filesystem_prefetch_min_bytes_for_single_read_task, "2Mi", "Do not parallelize within one file read less than this amount of bytes. E.g. one reader will not receive a read task of size less than this amount. This setting is recommended to avoid spikes of time for aws getObject requests to aws", 0) \
     M(UInt64, filesystem_prefetch_max_memory_usage, "1Gi", "Maximum memory usage for prefetches.", 0) \
     M(UInt64, filesystem_prefetches_limit, 200, "Maximum number of prefetches. Zero means unlimited. A setting `filesystem_prefetches_max_memory_usage` is more recommended if you want to limit the number of prefetches", 0) \
     \
diff --git a/src/Storages/MergeTree/MergeTreePrefetchedReadPool.cpp b/src/Storages/MergeTree/MergeTreePrefetchedReadPool.cpp
index c19b4ddd8a2..b752c0d0d80 100644
--- a/src/Storages/MergeTree/MergeTreePrefetchedReadPool.cpp
+++ b/src/Storages/MergeTree/MergeTreePrefetchedReadPool.cpp
@@ -411,29 +411,7 @@ void MergeTreePrefetchedReadPool::fillPerThreadTasks(size_t threads, size_t sum_
                 1, static_cast<size_t>(std::round(static_cast<double>(settings.filesystem_prefetch_step_bytes) / part_stat.approx_size_of_mark)));
         }
 
-        /// This limit is important to avoid spikes of slow aws getObject requests when parallelizing within one file.
-        /// (The default is taken from here https://docs.aws.amazon.com/whitepapers/latest/s3-optimizing-performance-best-practices/use-byte-range-fetches.html).
-        if (part_stat.approx_size_of_mark
-            && settings.filesystem_prefetch_min_bytes_for_single_read_task
-            && part_stat.approx_size_of_mark < settings.filesystem_prefetch_min_bytes_for_single_read_task)
-        {
-            const size_t min_prefetch_step_marks_by_total_cols = static_cast<size_t>(
-                std::ceil(static_cast<double>(settings.filesystem_prefetch_min_bytes_for_single_read_task) / part_stat.approx_size_of_mark));
-
-            /// At least one task to start working on it right now and another one to prefetch in the meantime.
-            const size_t new_min_prefetch_step_marks = std::min<size_t>(min_prefetch_step_marks_by_total_cols, sum_marks / threads / 2);
-            if (min_prefetch_step_marks < new_min_prefetch_step_marks)
-            {
-                LOG_DEBUG(log, "Increasing min prefetch step from {} to {}", min_prefetch_step_marks, new_min_prefetch_step_marks);
-                min_prefetch_step_marks = new_min_prefetch_step_marks;
-            }
-        }
-
-        if (part_stat.prefetch_step_marks < min_prefetch_step_marks)
-        {
-            LOG_DEBUG(log, "Increasing prefetch step from {} to {}", part_stat.prefetch_step_marks, min_prefetch_step_marks);
-            part_stat.prefetch_step_marks = min_prefetch_step_marks;
-        }
+        part_stat.prefetch_step_marks = std::max(part_stat.prefetch_step_marks, per_part_infos[i]->min_marks_per_task);
 
         LOG_DEBUG(
             log,
diff --git a/src/Storages/MergeTree/MergeTreeReadPool.cpp b/src/Storages/MergeTree/MergeTreeReadPool.cpp
index e525f7f5f65..2f9ec317386 100644
--- a/src/Storages/MergeTree/MergeTreeReadPool.cpp
+++ b/src/Storages/MergeTree/MergeTreeReadPool.cpp
@@ -25,14 +25,6 @@ extern const int CANNOT_SCHEDULE_TASK;
 extern const int LOGICAL_ERROR;
 }
 
-size_t getApproxSizeOfPart(const IMergeTreeDataPart & part, const Names & columns_to_read)
-{
-    ColumnSize columns_size{};
-    for (const auto & col_name : columns_to_read)
-        columns_size.add(part.getColumnSize(col_name));
-    return columns_size.data_compressed;
-}
-
 MergeTreeReadPool::MergeTreeReadPool(
     RangesInDataParts && parts_,
     VirtualFields shared_virtual_fields_,
@@ -53,38 +45,9 @@ MergeTreeReadPool::MergeTreeReadPool(
         column_names_,
         settings_,
         context_)
-    , min_marks_for_concurrent_read(pool_settings.min_marks_for_concurrent_read)
     , backoff_settings{context_->getSettingsRef()}
     , backoff_state{pool_settings.threads}
 {
-    if (std::ranges::count(is_part_on_remote_disk, true))
-    {
-        const auto & settings = context_->getSettingsRef();
-
-        size_t total_compressed_bytes = 0;
-        size_t total_marks = 0;
-        for (const auto & part : parts_ranges)
-        {
-            const auto & columns = settings.merge_tree_determine_task_size_by_prewhere_columns && prewhere_info
-                ? prewhere_info->prewhere_actions->getRequiredColumnsNames()
-                : column_names_;
-
-            total_compressed_bytes += getApproxSizeOfPart(*part.data_part, columns);
-            total_marks += part.getMarksCount();
-        }
-
-        if (total_marks)
-        {
-            const auto min_bytes_per_task = settings.merge_tree_min_bytes_per_task_for_remote_reading;
-            const auto avg_mark_bytes = std::max<size_t>(total_compressed_bytes / total_marks, 1);
-            /// We're taking min here because number of tasks shouldn't be too low - it will make task stealing impossible.
-            const auto heuristic_min_marks = std::min<size_t>(total_marks / pool_settings.threads, min_bytes_per_task / avg_mark_bytes);
-
-            if (heuristic_min_marks > min_marks_for_concurrent_read)
-                min_marks_for_concurrent_read = heuristic_min_marks;
-        }
-    }
-
     fillPerThreadInfo(pool_settings.threads, pool_settings.sum_marks);
 }
 
@@ -129,15 +92,16 @@ MergeTreeReadTaskPtr MergeTreeReadPool::getTask(size_t task_idx, MergeTreeReadTa
 
     const auto part_idx = thread_task.part_idx;
     auto & marks_in_part = thread_tasks.sum_marks_in_parts.back();
+    const auto min_marks_per_task = per_part_infos[part_idx]->min_marks_per_task;
 
     size_t need_marks;
     if (is_part_on_remote_disk[part_idx] && !pool_settings.use_const_size_tasks_for_remote_reading)
         need_marks = marks_in_part;
     else /// Get whole part to read if it is small enough.
-        need_marks = std::min(marks_in_part, min_marks_for_concurrent_read);
+        need_marks = std::min(marks_in_part, min_marks_per_task);
 
     /// Do not leave too little rows in part for next time.
-    if (marks_in_part > need_marks && marks_in_part - need_marks < min_marks_for_concurrent_read / 2)
+    if (marks_in_part > need_marks && marks_in_part - need_marks < min_marks_per_task / 2)
         need_marks = marks_in_part;
 
     MarkRanges ranges_to_get_from_part;
@@ -256,8 +220,6 @@ void MergeTreeReadPool::fillPerThreadInfo(size_t threads, size_t sum_marks)
             parts_queue.push(std::move(info.second));
     }
 
-    LOG_DEBUG(log, "min_marks_for_concurrent_read={}", min_marks_for_concurrent_read);
-
     const size_t min_marks_per_thread = (sum_marks - 1) / threads + 1;
 
     for (size_t i = 0; i < threads && !parts_queue.empty(); ++i)
@@ -270,15 +232,14 @@ void MergeTreeReadPool::fillPerThreadInfo(size_t threads, size_t sum_marks)
             auto & part_with_ranges = current_parts.back().part;
             size_t & marks_in_part = current_parts.back().sum_marks;
             const auto part_idx = current_parts.back().part_idx;
+            const auto min_marks_per_task = per_part_infos[part_idx]->min_marks_per_task;
 
             /// Do not get too few rows from part.
-            if (marks_in_part >= min_marks_for_concurrent_read &&
-                need_marks < min_marks_for_concurrent_read)
-                need_marks = min_marks_for_concurrent_read;
+            if (marks_in_part >= min_marks_per_task && need_marks < min_marks_per_task)
+                need_marks = min_marks_per_task;
 
             /// Do not leave too few rows in part for next time.
-            if (marks_in_part > need_marks &&
-                marks_in_part - need_marks < min_marks_for_concurrent_read)
+            if (marks_in_part > need_marks && marks_in_part - need_marks < min_marks_per_task)
                 need_marks = marks_in_part;
 
             MarkRanges ranges_to_get_from_part;
diff --git a/src/Storages/MergeTree/MergeTreeReadPool.h b/src/Storages/MergeTree/MergeTreeReadPool.h
index cb0e8a9657f..b79af82ddb2 100644
--- a/src/Storages/MergeTree/MergeTreeReadPool.h
+++ b/src/Storages/MergeTree/MergeTreeReadPool.h
@@ -78,7 +78,6 @@ private:
     void fillPerThreadInfo(size_t threads, size_t sum_marks);
 
     mutable std::mutex mutex;
-    size_t min_marks_for_concurrent_read = 0;
 
     /// State to track numbers of slow reads.
     struct BackoffState
diff --git a/src/Storages/MergeTree/MergeTreeReadPoolBase.cpp b/src/Storages/MergeTree/MergeTreeReadPoolBase.cpp
index 0cbb0a86b2f..397491a952d 100644
--- a/src/Storages/MergeTree/MergeTreeReadPoolBase.cpp
+++ b/src/Storages/MergeTree/MergeTreeReadPoolBase.cpp
@@ -1,6 +1,7 @@
-#include <Storages/MergeTree/MergeTreeReadPoolBase.h>
-#include <Storages/MergeTree/MergeTreeBlockReadUtils.h>
+#include <cmath>
 #include <Storages/MergeTree/LoadedMergeTreeDataPartInfoForReader.h>
+#include <Storages/MergeTree/MergeTreeBlockReadUtils.h>
+#include <Storages/MergeTree/MergeTreeReadPoolBase.h>
 
 
 namespace DB
@@ -29,10 +30,53 @@ MergeTreeReadPoolBase::MergeTreeReadPoolBase(
     , header(storage_snapshot->getSampleBlockForColumns(column_names))
     , profile_callback([this](ReadBufferFromFileBase::ProfileInfo info_) { profileFeedback(info_); })
 {
-    fillPerPartInfos();
+    fillPerPartInfos(context_->getSettingsRef());
 }
 
-void MergeTreeReadPoolBase::fillPerPartInfos()
+static size_t getApproxSizeOfPart(const IMergeTreeDataPart & part, const Names & columns_to_read)
+{
+    ColumnSize columns_size{};
+    for (const auto & col_name : columns_to_read)
+        columns_size.add(part.getColumnSize(col_name));
+    return columns_size.data_compressed;
+}
+
+static size_t calculateMinMarksPerTask(
+    const RangesInDataPart & part,
+    const Names & columns_to_read,
+    PrewhereInfoPtr prewhere_info,
+    const MergeTreeReadPoolBase::PoolSettings & pool_settings,
+    const Settings & settings)
+{
+    size_t min_marks_per_task = pool_settings.min_marks_for_concurrent_read;
+    const size_t part_marks_count = part.getMarksCount();
+    if (part_marks_count && part.data_part->isStoredOnRemoteDisk())
+    {
+        const auto & columns = settings.merge_tree_determine_task_size_by_prewhere_columns && prewhere_info
+            ? prewhere_info->prewhere_actions->getRequiredColumnsNames()
+            : columns_to_read;
+        const size_t part_compressed_bytes = getApproxSizeOfPart(*part.data_part, columns);
+
+        const auto avg_mark_bytes = std::max<size_t>(part_compressed_bytes / part_marks_count, 1);
+        const auto min_bytes_per_task = settings.merge_tree_min_bytes_per_task_for_remote_reading;
+        /// We're taking min here because number of tasks shouldn't be too low - it will make task stealing impossible.
+        const auto heuristic_min_marks
+            = std::min<size_t>(pool_settings.sum_marks / pool_settings.threads / 2, min_bytes_per_task / avg_mark_bytes);
+        if (heuristic_min_marks > min_marks_per_task)
+        {
+            LOG_DEBUG(
+                &Poco::Logger::get("MergeTreeReadPoolBase"),
+                "Increasing min_marks_per_task from {} to {} based on columns size heuristic",
+                min_marks_per_task,
+                heuristic_min_marks);
+            min_marks_per_task = heuristic_min_marks;
+        }
+    }
+    LOG_DEBUG(&Poco::Logger::get("MergeTreeReadPoolBase"), "Will use min_marks_per_task={}", min_marks_per_task);
+    return min_marks_per_task;
+}
+
+void MergeTreeReadPoolBase::fillPerPartInfos(const Settings & settings)
 {
     per_part_infos.reserve(parts_ranges.size());
     is_part_on_remote_disk.reserve(parts_ranges.size());
@@ -83,6 +127,8 @@ void MergeTreeReadPoolBase::fillPerPartInfos()
         }
 
         is_part_on_remote_disk.push_back(part_with_ranges.data_part->isStoredOnRemoteDisk());
+        read_task_info.min_marks_per_task
+            = calculateMinMarksPerTask(part_with_ranges, column_names, prewhere_info, pool_settings, settings);
         per_part_infos.push_back(std::make_shared<MergeTreeReadTaskInfo>(std::move(read_task_info)));
     }
 }
diff --git a/src/Storages/MergeTree/MergeTreeReadPoolBase.h b/src/Storages/MergeTree/MergeTreeReadPoolBase.h
index 1b5bfec5898..123f7538ba8 100644
--- a/src/Storages/MergeTree/MergeTreeReadPoolBase.h
+++ b/src/Storages/MergeTree/MergeTreeReadPoolBase.h
@@ -48,7 +48,7 @@ protected:
     const UncompressedCachePtr owned_uncompressed_cache;
     const Block header;
 
-    void fillPerPartInfos();
+    void fillPerPartInfos(const Settings & settings);
     std::vector<size_t> getPerPartSumMarks() const;
 
     MergeTreeReadTaskPtr createTask(
diff --git a/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicas.cpp b/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicas.cpp
index 38035d97f56..d542abb2985 100644
--- a/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicas.cpp
+++ b/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicas.cpp
@@ -33,6 +33,7 @@ MergeTreeReadPoolParallelReplicas::MergeTreeReadPoolParallelReplicas(
         context_)
     , extension(std::move(extension_))
     , coordination_mode(CoordinationMode::Default)
+    , min_marks_per_task(pool_settings.min_marks_for_concurrent_read)
 {
     extension.all_callback(
         InitialAllRangesAnnouncement(coordination_mode, parts_ranges.getDescriptions(), extension.number_of_current_replica));
diff --git a/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicasInOrder.cpp b/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicasInOrder.cpp
index 01c0a9f91be..e8228007f81 100644
--- a/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicasInOrder.cpp
+++ b/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicasInOrder.cpp
@@ -32,6 +32,7 @@ MergeTreeReadPoolParallelReplicasInOrder::MergeTreeReadPoolParallelReplicasInOrd
         context_)
     , extension(std::move(extension_))
     , mode(mode_)
+    , min_marks_per_task(pool_settings.min_marks_for_concurrent_read)
 {
     for (const auto & part : parts_ranges)
         request.push_back({part.data_part->info, MarkRanges{}});
diff --git a/src/Storages/MergeTree/MergeTreeReadTask.h b/src/Storages/MergeTree/MergeTreeReadTask.h
index c8bb501c0e8..4c94a6afe64 100644
--- a/src/Storages/MergeTree/MergeTreeReadTask.h
+++ b/src/Storages/MergeTree/MergeTreeReadTask.h
@@ -66,6 +66,8 @@ struct MergeTreeReadTaskInfo
     MergeTreeBlockSizePredictorPtr shared_size_predictor;
     /// TODO: comment
     VirtualFields const_virtual_fields;
+    /// The amount of data to read per task based on size of the queried columns.
+    size_t min_marks_per_task = 0;
 };
 
 using MergeTreeReadTaskInfoPtr = std::shared_ptr<const MergeTreeReadTaskInfo>;

From 85072356dbb4e298113d3a3a262374b1140fc8f5 Mon Sep 17 00:00:00 2001
From: Nikita Taranov <nikita.taranov@clickhouse.com>
Date: Sat, 24 Feb 2024 22:42:21 +0100
Subject: [PATCH 0034/1488] support in parallel replicas

---
 .../MergeTree/MergeTreeReadPoolParallelReplicas.cpp   |  9 ++++++---
 .../MergeTree/MergeTreeReadPoolParallelReplicas.h     |  1 +
 .../MergeTreeReadPoolParallelReplicasInOrder.cpp      | 11 +++++------
 .../MergeTreeReadPoolParallelReplicasInOrder.h        |  1 +
 4 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicas.cpp b/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicas.cpp
index d542abb2985..33eaf5a49bd 100644
--- a/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicas.cpp
+++ b/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicas.cpp
@@ -35,6 +35,9 @@ MergeTreeReadPoolParallelReplicas::MergeTreeReadPoolParallelReplicas(
     , coordination_mode(CoordinationMode::Default)
     , min_marks_per_task(pool_settings.min_marks_for_concurrent_read)
 {
+    for (const auto & info : per_part_infos)
+        min_marks_per_task = std::max(min_marks_per_task, info->min_marks_per_task);
+
     extension.all_callback(
         InitialAllRangesAnnouncement(coordination_mode, parts_ranges.getDescriptions(), extension.number_of_current_replica));
 }
@@ -51,7 +54,7 @@ MergeTreeReadTaskPtr MergeTreeReadPoolParallelReplicas::getTask(size_t /*task_id
         auto result = extension.callback(ParallelReadRequest(
             coordination_mode,
             extension.number_of_current_replica,
-            pool_settings.min_marks_for_concurrent_read * pool_settings.threads,
+            min_marks_per_task * pool_settings.threads,
             /// For Default coordination mode we don't need to pass part names.
             RangesInDataPartsDescription{}));
 
@@ -77,9 +80,9 @@ MergeTreeReadTaskPtr MergeTreeReadPoolParallelReplicas::getTask(size_t /*task_id
 
     MarkRanges ranges_to_read;
     size_t current_sum_marks = 0;
-    while (current_sum_marks < pool_settings.min_marks_for_concurrent_read && !current_task.ranges.empty())
+    while (current_sum_marks < min_marks_per_task && !current_task.ranges.empty())
     {
-        auto diff = pool_settings.min_marks_for_concurrent_read - current_sum_marks;
+        auto diff = min_marks_per_task - current_sum_marks;
         auto range = current_task.ranges.front();
         if (range.getNumberOfMarks() > diff)
         {
diff --git a/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicas.h b/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicas.h
index ca159edb91c..6ba63cc2c9a 100644
--- a/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicas.h
+++ b/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicas.h
@@ -32,6 +32,7 @@ private:
 
     const ParallelReadingExtension extension;
     const CoordinationMode coordination_mode;
+    size_t min_marks_per_task{0};
     RangesInDataPartsDescription buffered_ranges;
     bool no_more_tasks_available{false};
     LoggerPtr log = getLogger("MergeTreeReadPoolParallelReplicas");
diff --git a/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicasInOrder.cpp b/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicasInOrder.cpp
index e8228007f81..6b5cf978423 100644
--- a/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicasInOrder.cpp
+++ b/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicasInOrder.cpp
@@ -34,6 +34,9 @@ MergeTreeReadPoolParallelReplicasInOrder::MergeTreeReadPoolParallelReplicasInOrd
     , mode(mode_)
     , min_marks_per_task(pool_settings.min_marks_for_concurrent_read)
 {
+    for (const auto & info : per_part_infos)
+        min_marks_per_task = std::max(min_marks_per_task, info->min_marks_per_task);
+
     for (const auto & part : parts_ranges)
         request.push_back({part.data_part->info, MarkRanges{}});
 
@@ -77,12 +80,8 @@ MergeTreeReadTaskPtr MergeTreeReadPoolParallelReplicasInOrder::getTask(size_t ta
     if (no_more_tasks)
         return nullptr;
 
-    auto response = extension.callback(ParallelReadRequest(
-        mode,
-        extension.number_of_current_replica,
-        pool_settings.min_marks_for_concurrent_read * request.size(),
-        request
-    ));
+    auto response
+        = extension.callback(ParallelReadRequest(mode, extension.number_of_current_replica, min_marks_per_task * request.size(), request));
 
     if (!response || response->description.empty() || response->finish)
     {
diff --git a/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicasInOrder.h b/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicasInOrder.h
index 4fe3f7a699c..22841bea212 100644
--- a/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicasInOrder.h
+++ b/src/Storages/MergeTree/MergeTreeReadPoolParallelReplicasInOrder.h
@@ -30,6 +30,7 @@ private:
     const ParallelReadingExtension extension;
     const CoordinationMode mode;
 
+    size_t min_marks_per_task{0};
     bool no_more_tasks{false};
     RangesInDataPartsDescription request;
     RangesInDataPartsDescription buffered_tasks;

From 281327b4ced9aadf421646095d71ad59ba6c70ba Mon Sep 17 00:00:00 2001
From: Nikita Taranov <nikita.taranov@clickhouse.com>
Date: Mon, 26 Feb 2024 18:41:15 +0100
Subject: [PATCH 0035/1488] register settings change

---
 src/Core/SettingsChangesHistory.h | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/Core/SettingsChangesHistory.h b/src/Core/SettingsChangesHistory.h
index f43ca154d56..6e38976b4b1 100644
--- a/src/Core/SettingsChangesHistory.h
+++ b/src/Core/SettingsChangesHistory.h
@@ -85,7 +85,8 @@ namespace SettingsChangesHistory
 /// It's used to implement `compatibility` setting (see https://github.com/ClickHouse/ClickHouse/issues/35972)
 static std::map<ClickHouseVersion, SettingsChangesHistory::SettingsChanges> settings_changes_history =
 {
-    {"24.4", {{"input_format_json_throw_on_bad_escape_sequence", true, true, "Allow to save JSON strings with bad escape sequences"},
+    {"24.4", {{"merge_tree_min_bytes_per_task_for_remote_reading", 4194304, 2097152, "Value is unified with `filesystem_prefetch_min_bytes_for_single_read_task`"},
+              {"input_format_json_throw_on_bad_escape_sequence", true, true, "Allow to save JSON strings with bad escape sequences"},
               {"lightweight_deletes_sync", 2, 2, "The same as 'mutation_sync', but controls only execution of lightweight deletes"},
               }},
     {"24.3", {{"s3_connect_timeout_ms", 1000, 1000, "Introduce new dedicated setting for s3 connection timeout"},

From e8e980932fdcc87934a3d4591d9a42130a4d60a4 Mon Sep 17 00:00:00 2001
From: Nikita Taranov <nikita.taranov@clickhouse.com>
Date: Wed, 28 Feb 2024 20:06:12 +0100
Subject: [PATCH 0036/1488] fix

---
 src/Processors/QueryPlan/ReadFromMergeTree.cpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp
index bee42c3ddde..c4ba92d483e 100644
--- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp
+++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp
@@ -945,6 +945,8 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsWithOrder(
 
     PoolSettings pool_settings
     {
+        .threads = num_streams,
+        .sum_marks = parts_with_ranges.getMarksCountAllParts(),
         .min_marks_for_concurrent_read = info.min_marks_for_concurrent_read,
         .preferred_block_size_bytes = settings.preferred_block_size_bytes,
         .use_uncompressed_cache = info.use_uncompressed_cache,

From 29729c0ea8fcf2543c8d1a2f996773b54829c86d Mon Sep 17 00:00:00 2001
From: Nikita Taranov <nikita.taranov@clickhouse.com>
Date: Wed, 6 Mar 2024 23:42:37 +0100
Subject: [PATCH 0037/1488] use whole part size for Compact parts

---
 src/Storages/MergeTree/MergeTreeReadPoolBase.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/Storages/MergeTree/MergeTreeReadPoolBase.cpp b/src/Storages/MergeTree/MergeTreeReadPoolBase.cpp
index 397491a952d..2bf19e8cde5 100644
--- a/src/Storages/MergeTree/MergeTreeReadPoolBase.cpp
+++ b/src/Storages/MergeTree/MergeTreeReadPoolBase.cpp
@@ -38,7 +38,8 @@ static size_t getApproxSizeOfPart(const IMergeTreeDataPart & part, const Names &
     ColumnSize columns_size{};
     for (const auto & col_name : columns_to_read)
         columns_size.add(part.getColumnSize(col_name));
-    return columns_size.data_compressed;
+    /// For compact parts we don't know individual column sizes, let's use whole part size as approximation
+    return columns_size.data_compressed ? columns_size.data_compressed : part.getBytesOnDisk();
 }
 
 static size_t calculateMinMarksPerTask(

From af6c4a3f2d247a3b39bd48f33e16758972e31723 Mon Sep 17 00:00:00 2001
From: Nikita Taranov <nikita.taranov@clickhouse.com>
Date: Tue, 26 Mar 2024 22:09:17 +0000
Subject: [PATCH 0038/1488] fix noisy log

---
 src/Storages/MergeTree/MergeTreeReadPoolBase.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Storages/MergeTree/MergeTreeReadPoolBase.cpp b/src/Storages/MergeTree/MergeTreeReadPoolBase.cpp
index 2bf19e8cde5..9d2484ecd04 100644
--- a/src/Storages/MergeTree/MergeTreeReadPoolBase.cpp
+++ b/src/Storages/MergeTree/MergeTreeReadPoolBase.cpp
@@ -65,7 +65,7 @@ static size_t calculateMinMarksPerTask(
             = std::min<size_t>(pool_settings.sum_marks / pool_settings.threads / 2, min_bytes_per_task / avg_mark_bytes);
         if (heuristic_min_marks > min_marks_per_task)
         {
-            LOG_DEBUG(
+            LOG_TEST(
                 &Poco::Logger::get("MergeTreeReadPoolBase"),
                 "Increasing min_marks_per_task from {} to {} based on columns size heuristic",
                 min_marks_per_task,
@@ -73,7 +73,7 @@ static size_t calculateMinMarksPerTask(
             min_marks_per_task = heuristic_min_marks;
         }
     }
-    LOG_DEBUG(&Poco::Logger::get("MergeTreeReadPoolBase"), "Will use min_marks_per_task={}", min_marks_per_task);
+    LOG_TEST(&Poco::Logger::get("MergeTreeReadPoolBase"), "Will use min_marks_per_task={}", min_marks_per_task);
     return min_marks_per_task;
 }
 

From bc1042a497257526c13a08d86fc736466cd454b5 Mon Sep 17 00:00:00 2001
From: Nikita Taranov <nikita.taranov@clickhouse.com>
Date: Thu, 28 Mar 2024 19:11:18 +0000
Subject: [PATCH 0039/1488] fix test

---
 tests/queries/0_stateless/02532_send_logs_level_test.reference | 1 +
 tests/queries/0_stateless/02532_send_logs_level_test.sh        | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/02532_send_logs_level_test.reference b/tests/queries/0_stateless/02532_send_logs_level_test.reference
index 7e51b888d9c..72f4ea06184 100644
--- a/tests/queries/0_stateless/02532_send_logs_level_test.reference
+++ b/tests/queries/0_stateless/02532_send_logs_level_test.reference
@@ -1,3 +1,4 @@
+<Test> MergeTreeReadPoolBase: Will use min_marks_per_task=24
 <Test> MergeTreeMarksLoader: Loading marks from path data.cmrk3
 <Test> MergeTreeRangeReader: First reader returned: num_rows: 1, columns: 1, total_rows_per_granule: 1, no filter, column[0]:  Int32(size = 1), requested columns: key
 <Test> MergeTreeRangeReader: read() returned num_rows: 1, columns: 1, total_rows_per_granule: 1, no filter, column[0]:  Int32(size = 1), sample block key
diff --git a/tests/queries/0_stateless/02532_send_logs_level_test.sh b/tests/queries/0_stateless/02532_send_logs_level_test.sh
index 4afc6d4496b..08b3058c5a0 100755
--- a/tests/queries/0_stateless/02532_send_logs_level_test.sh
+++ b/tests/queries/0_stateless/02532_send_logs_level_test.sh
@@ -1,7 +1,8 @@
 #!/usr/bin/env bash
-# Tags: no-s3-storage, no-debug
+# Tags: no-s3-storage, no-debug, no-random-merge-tree-settings
 # - no-s3-storage - S3 has additional logging
 # - no-debug - debug builds also has additional logging
+# - no-random-merge-tree-settings - changes content of log messages
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh

From a966a94de6665095991064455470eb3a69d18c1c Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Wed, 10 Apr 2024 15:04:31 +0200
Subject: [PATCH 0040/1488] fix a bug in get children

---
 src/Coordination/RocksDBContainer.h | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/Coordination/RocksDBContainer.h b/src/Coordination/RocksDBContainer.h
index a3ff5143c6a..30950248061 100644
--- a/src/Coordination/RocksDBContainer.h
+++ b/src/Coordination/RocksDBContainer.h
@@ -218,14 +218,15 @@ public:
         }
     }
 
-    std::vector<std::pair<std::string, Node>> getChildren(const std::string & key)
+    std::vector<std::pair<std::string, Node>> getChildren(const std::string & key_)
     {
         rocksdb::ReadOptions read_options;
         read_options.total_order_seek = true;
 
-        size_t len = key.size() + 2;
+        std::string key = key_;
         if (!key.ends_with('/'))
-            len ++;
+            key += '/';
+        size_t len = key.size() + 2;
 
         auto iter = std::unique_ptr<rocksdb::Iterator>(rocksdb_ptr->NewIterator(read_options));
         std::string encoded_string = getEncodedKey(key, true);

From b5e74548b3c43116d900ee4ed573ed969c10678d Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Wed, 10 Apr 2024 15:22:57 +0200
Subject: [PATCH 0041/1488] fix clang tidy

---
 src/Coordination/KeeperSnapshotManagerS3.cpp | 2 ++
 src/Coordination/RocksDBContainer.h          | 4 +---
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/Coordination/KeeperSnapshotManagerS3.cpp b/src/Coordination/KeeperSnapshotManagerS3.cpp
index b54e2ceaa15..b984b8ad18e 100644
--- a/src/Coordination/KeeperSnapshotManagerS3.cpp
+++ b/src/Coordination/KeeperSnapshotManagerS3.cpp
@@ -25,6 +25,8 @@
 
 #include <filesystem>
 
+namespace fs = std::filesystem;
+
 namespace DB
 {
 
diff --git a/src/Coordination/RocksDBContainer.h b/src/Coordination/RocksDBContainer.h
index 30950248061..ba0f7f74b4e 100644
--- a/src/Coordination/RocksDBContainer.h
+++ b/src/Coordination/RocksDBContainer.h
@@ -21,8 +21,6 @@ namespace ErrorCodes
     extern const int LOGICAL_ERROR;
 }
 
-namespace fs = std::filesystem;
-
 /// The key-value format of rocks db will be
 /// - key: Int8 (depth of the path) + String (path)
 /// - value: SizeOf(keeperRocksNodeInfo) (meta of the node) + String (data)
@@ -214,7 +212,7 @@ public:
             rocksdb_ptr->Close();
             rocksdb_ptr = nullptr;
 
-            fs::remove_all(rocksdb_dir);
+            std::filesystem::remove_all(rocksdb_dir);
         }
     }
 

From 91ca441eed9962537ad3f3d5897810e9e876cb49 Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Fri, 3 May 2024 17:57:35 +0200
Subject: [PATCH 0042/1488] address part of comments

---
 src/Coordination/RocksDBContainer.h | 22 +---------------------
 1 file changed, 1 insertion(+), 21 deletions(-)

diff --git a/src/Coordination/RocksDBContainer.h b/src/Coordination/RocksDBContainer.h
index ba0f7f74b4e..75cc2a9e555 100644
--- a/src/Coordination/RocksDBContainer.h
+++ b/src/Coordination/RocksDBContainer.h
@@ -31,24 +31,6 @@ struct RocksDBContainer
     using Node = Node_;
 
 private:
-    /// this is only for debug.
-    std::string toHex(std::string origin)
-    {
-        auto toHexChar = [](char v)
-        {
-            if (v <= 9)
-                return v + '0';
-            return 'A' + v - 10;
-        };
-        std::string result;
-        result.reserve(2 * origin.size());
-        for (unsigned char c : origin)
-        {
-            result.push_back(toHexChar(c >> 4));
-            result.push_back(toHexChar(c & 0xf));
-        }
-        return result;
-    }
 
     UInt16 getKeyDepth(const std::string & key)
     {
@@ -184,13 +166,11 @@ public:
         if (disk == nullptr)
         {
             throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot get rocksdb disk");
-            return;
         }
         auto options = context->getRocksDBOptions();
         if (options == nullptr)
         {
             throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot get rocksdb options");
-            return;
         }
         rocksdb_dir = disk->getPath();
         rocksdb::DB * db;
@@ -365,7 +345,7 @@ public:
             return false;
         if (status.ok())
         {
-            counter --;
+            counter--;
             return true;
         }
         throw Exception(ErrorCodes::ROCKSDB_ERROR, "Got rocksdb error during erase. The error message is {}.", status.ToString());

From aa41ad12f66a56cb61b9a71bc8add9da20179808 Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Mon, 6 May 2024 15:38:29 +0200
Subject: [PATCH 0043/1488] address comments

---
 src/Coordination/KeeperContext.h              |    2 +-
 src/Coordination/KeeperStorage.cpp            |   84 +-
 src/Coordination/KeeperStorage.h              |    7 +
 src/Coordination/RocksDBContainer.h           |   21 +
 src/Coordination/tests/gtest_coordination.cpp | 1012 ++++++++-------
 src/Coordination/tests/gtest_rocks_keeper.cpp | 1150 -----------------
 6 files changed, 627 insertions(+), 1649 deletions(-)
 delete mode 100644 src/Coordination/tests/gtest_rocks_keeper.cpp

diff --git a/src/Coordination/KeeperContext.h b/src/Coordination/KeeperContext.h
index c9abd13b6e9..38013725f56 100644
--- a/src/Coordination/KeeperContext.h
+++ b/src/Coordination/KeeperContext.h
@@ -70,7 +70,7 @@ public:
     void setRocksDBDisk(DiskPtr disk);
     DiskPtr getTemporaryRocksDBDisk() const;
 
-    void setRocksDBOptions(std::shared_ptr<rocksdb::Options> rocksdb_options_);
+    void setRocksDBOptions(std::shared_ptr<rocksdb::Options> rocksdb_options_ = nullptr);
     std::shared_ptr<rocksdb::Options> getRocksDBOptions() const { return rocksdb_options; }
 
     UInt64 getKeeperMemorySoftLimit() const { return memory_soft_limit; }
diff --git a/src/Coordination/KeeperStorage.cpp b/src/Coordination/KeeperStorage.cpp
index 905fd06359c..73d735c0af8 100644
--- a/src/Coordination/KeeperStorage.cpp
+++ b/src/Coordination/KeeperStorage.cpp
@@ -1745,63 +1745,53 @@ struct KeeperStorageListRequestProcessor final : public KeeperStorageRequestProc
             if (path_prefix.empty())
                 throw DB::Exception(ErrorCodes::LOGICAL_ERROR, "Path cannot be empty");
 
-            if constexpr (Storage::use_rocksdb)
+            const auto & get_children = [&]()
             {
-                const auto & children = container.getChildren(request.path);
-                response.names.reserve(children.size());
-                const auto add_child = [&](const auto & child)
+                if constexpr (Storage::use_rocksdb)
+                    return container.getChildren(request.path);
+                else
+                    return node_it->value.getChildren();
+            };
+            const auto & children = get_children();
+            response.names.reserve(children.size());
+
+            const auto add_child = [&](const auto & child)
+            {
+                using enum Coordination::ListRequestType;
+
+                auto list_request_type = ALL;
+                if (auto * filtered_list = dynamic_cast<Coordination::ZooKeeperFilteredListRequest *>(&request))
                 {
-                    using enum Coordination::ListRequestType;
-
-                    auto list_request_type = ALL;
-                    if (auto * filtered_list = dynamic_cast<Coordination::ZooKeeperFilteredListRequest *>(&request))
-                    {
-                        list_request_type = filtered_list->list_request_type;
-                    }
-
-                    if (list_request_type == ALL)
-                        return true;
-
-                    const auto is_ephemeral = child.second.isEphemeral();
-                    return (is_ephemeral && list_request_type == EPHEMERAL_ONLY) || (!is_ephemeral && list_request_type == PERSISTENT_ONLY);
-                };
-
-                for (const auto & child : children)
-                {
-                    if (add_child(child))
-                        response.names.push_back(child.first);
+                    list_request_type = filtered_list->list_request_type;
                 }
-            }
-            else
-            {
-                const auto & children = node_it->value.getChildren();
-                response.names.reserve(children.size());
 
-                const auto add_child = [&](const auto child)
+                if (list_request_type == ALL)
+                    return true;
+
+                bool is_ephemeral;
+                if constexpr (!Storage::use_rocksdb)
                 {
-                    using enum Coordination::ListRequestType;
-
-                    auto list_request_type = ALL;
-                    if (auto * filtered_list = dynamic_cast<Coordination::ZooKeeperFilteredListRequest *>(&request))
-                    {
-                        list_request_type = filtered_list->list_request_type;
-                    }
-
-                    if (list_request_type == ALL)
-                        return true;
-
                     auto child_path = (std::filesystem::path(request.path) / child.toView()).generic_string();
                     auto child_it = container.find(child_path);
                     if (child_it == container.end())
                         onStorageInconsistency();
-
-                    const auto is_ephemeral = child_it->value.isEphemeral();
-                    return (is_ephemeral && list_request_type == EPHEMERAL_ONLY) || (!is_ephemeral && list_request_type == PERSISTENT_ONLY);
-                };
-
-                for (const auto child : children)
+                    is_ephemeral = child_it->value.isEphemeral();
+                }
+                else
                 {
-                    if (add_child(child))
+                    is_ephemeral = child.second.isEphemeral();
+                }
+
+                return (is_ephemeral && list_request_type == EPHEMERAL_ONLY) || (!is_ephemeral && list_request_type == PERSISTENT_ONLY);
+            };
+
+            for (const auto & child : children)
+            {
+                if (add_child(child))
+                {
+                    if constexpr (Storage::use_rocksdb)
+                        response.names.push_back(child.first);
+                    else
                         response.names.push_back(child.toString());
                 }
             }
diff --git a/src/Coordination/KeeperStorage.h b/src/Coordination/KeeperStorage.h
index 1d47061a17c..c2fd196b95e 100644
--- a/src/Coordination/KeeperStorage.h
+++ b/src/Coordination/KeeperStorage.h
@@ -93,6 +93,13 @@ struct KeeperRocksNodeInfo
         ephemeral_or_children_data.children_info.num_children = num_children;
     }
 
+    /// dummy interface for test
+    void addChild(StringRef) {}
+    auto getChildren() const
+    {
+        return std::vector<int>(numChildren());
+    }
+
     void increaseNumChildren()
     {
         chassert(!isEphemeral());
diff --git a/src/Coordination/RocksDBContainer.h b/src/Coordination/RocksDBContainer.h
index 75cc2a9e555..a4a236f332e 100644
--- a/src/Coordination/RocksDBContainer.h
+++ b/src/Coordination/RocksDBContainer.h
@@ -31,6 +31,20 @@ struct RocksDBContainer
     using Node = Node_;
 
 private:
+    /// MockNode is only use in test to mock `getChildren()` and `getData()`
+    struct MockNode
+    {
+        std::vector<int> children;
+        std::string data;
+        MockNode(size_t children_num, std::string_view data_)
+            : children(std::vector<int>(children_num)),
+              data(data_)
+        {
+        }
+
+        std::vector<int> getChildren() { return children; }
+        std::string getData() { return data; }
+    };
 
     UInt16 getKeyDepth(const std::string & key)
     {
@@ -262,6 +276,13 @@ public:
         return const_iterator(kv);
     }
 
+    MockNode getValue(StringRef key)
+    {
+        auto it = find(key);
+        chassert(it != end());
+        return MockNode(it->value.numChildren(), it->value.getData());
+    }
+
     const_iterator updateValue(StringRef key_, ValueUpdater updater)
     {
         /// rocksdb::PinnableSlice slice;
diff --git a/src/Coordination/tests/gtest_coordination.cpp b/src/Coordination/tests/gtest_coordination.cpp
index c303db11474..9f0937572a9 100644
--- a/src/Coordination/tests/gtest_coordination.cpp
+++ b/src/Coordination/tests/gtest_coordination.cpp
@@ -60,10 +60,22 @@ struct CompressionParam
     std::string extension;
 };
 
-class CoordinationTest : public ::testing::TestWithParam<CompressionParam>
+template <typename Storage_, bool enable_compression_>
+struct TestParam
 {
-protected:
-    DB::KeeperContextPtr keeper_context = std::make_shared<DB::KeeperContext>(true, std::make_shared<DB::CoordinationSettings>());
+    using Storage = Storage_;
+    static constexpr bool enable_compression = enable_compression_;
+};
+
+template<typename TestType>
+class CoordinationTest : public ::testing::Test
+{
+public:
+    using Storage = typename TestType::Storage;
+    static constexpr bool enable_compression = TestType::enable_compression;
+    std::string extension;
+
+    DB::KeeperContextPtr keeper_context;
     LoggerPtr log{getLogger("CoordinationTest")};
 
     void SetUp() override
@@ -72,7 +84,12 @@ protected:
         Poco::Logger::root().setChannel(channel);
         Poco::Logger::root().setLevel("trace");
 
+        auto settings = std::make_shared<DB::CoordinationSettings>();
+        settings->use_rocksdb = true;
+        keeper_context = std::make_shared<DB::KeeperContext>(true, settings);
         keeper_context->setLocalLogsPreprocessed();
+        keeper_context->setRocksDBOptions();
+        extension = enable_compression ? ".zstd" : "";
     }
 
     void setLogDirectory(const std::string & path) { keeper_context->setLogDisk(std::make_shared<DB::DiskLocal>("LogDisk", path)); }
@@ -82,13 +99,24 @@ protected:
         keeper_context->setSnapshotDisk(std::make_shared<DB::DiskLocal>("SnapshotDisk", path));
     }
 
+    void setRocksDBDirectory(const std::string & path)
+    {
+        keeper_context->setRocksDBDisk(std::make_shared<DB::DiskLocal>("RocksDisk", path));
+    }
+
     void setStateFileDirectory(const std::string & path)
     {
         keeper_context->setStateFileDisk(std::make_shared<DB::DiskLocal>("StateFile", path));
     }
 };
 
-TEST_P(CoordinationTest, RaftServerConfigParse)
+using Implementation = testing::Types<TestParam<DB::KeeperMemoryStorage, true>,
+                                      TestParam<DB::KeeperMemoryStorage, false>,
+                                      TestParam<DB::KeeperRocksStorage, true>,
+                                      TestParam<DB::KeeperRocksStorage, false>>;
+TYPED_TEST_SUITE(CoordinationTest, Implementation);
+
+TYPED_TEST(CoordinationTest, RaftServerConfigParse)
 {
     auto parse = Coordination::RaftServerConfig::parse;
     using Cfg = std::optional<DB::RaftServerConfig>;
@@ -113,7 +141,7 @@ TEST_P(CoordinationTest, RaftServerConfigParse)
         (Cfg{{1, "2001:0db8:85a3:0000:0000:8a2e:0370:7334:80"}}));
 }
 
-TEST_P(CoordinationTest, RaftServerClusterConfigParse)
+TYPED_TEST(CoordinationTest, RaftServerClusterConfigParse)
 {
     auto parse = Coordination::parseRaftServers;
     using Cfg = DB::RaftServerConfig;
@@ -129,14 +157,14 @@ TEST_P(CoordinationTest, RaftServerClusterConfigParse)
         (Servers{Cfg{1, "host:80"}, Cfg{2, "host:81"}}));
 }
 
-TEST_P(CoordinationTest, BuildTest)
+TYPED_TEST(CoordinationTest, BuildTest)
 {
     DB::InMemoryLogStore store;
     DB::SummingStateMachine machine;
     EXPECT_EQ(1, 1);
 }
 
-TEST_P(CoordinationTest, BufferSerde)
+TYPED_TEST(CoordinationTest, BufferSerde)
 {
     Coordination::ZooKeeperRequestPtr request = Coordination::ZooKeeperRequestFactory::instance().get(Coordination::OpNum::Get);
     request->xid = 3;
@@ -260,13 +288,13 @@ nuraft::ptr<nuraft::buffer> getBuffer(int64_t number)
     return ret;
 }
 
-TEST_P(CoordinationTest, TestSummingRaft1)
+TYPED_TEST(CoordinationTest, TestSummingRaft1)
 {
     ChangelogDirTest test("./logs");
-    setLogDirectory("./logs");
-    setStateFileDirectory(".");
+    this->setLogDirectory("./logs");
+    this->setStateFileDirectory(".");
 
-    SummingRaftServer s1(1, "localhost", 44444, keeper_context);
+    SummingRaftServer s1(1, "localhost", 44444, this->keeper_context);
     SCOPE_EXIT(if (std::filesystem::exists("./state")) std::filesystem::remove("./state"););
 
     /// Single node is leader
@@ -279,7 +307,7 @@ TEST_P(CoordinationTest, TestSummingRaft1)
 
     while (s1.state_machine->getValue() != 143)
     {
-        LOG_INFO(log, "Waiting s1 to apply entry");
+        LOG_INFO(this->log, "Waiting s1 to apply entry");
         std::this_thread::sleep_for(std::chrono::milliseconds(100));
     }
 
@@ -295,16 +323,16 @@ DB::LogEntryPtr getLogEntry(const std::string & s, size_t term)
     return nuraft::cs_new<nuraft::log_entry>(term, bufwriter.getBuffer());
 }
 
-TEST_P(CoordinationTest, ChangelogTestSimple)
+TYPED_TEST(CoordinationTest, ChangelogTestSimple)
 {
-    auto params = GetParam();
+    ///
     ChangelogDirTest test("./logs");
-    setLogDirectory("./logs");
+    this->setLogDirectory("./logs");
 
     DB::KeeperLogStore changelog(
-        DB::LogFileSettings{.force_sync = true, .compress_logs = params.enable_compression, .rotate_interval = 5},
+        DB::LogFileSettings{.force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 5},
         DB::FlushSettings(),
-        keeper_context);
+        this->keeper_context);
     changelog.init(1, 0);
     auto entry = getLogEntry("hello world", 77);
     changelog.append(entry);
@@ -327,16 +355,15 @@ void waitDurableLogs(nuraft::log_store & log_store)
 
 }
 
-TEST_P(CoordinationTest, ChangelogTestFile)
+TYPED_TEST(CoordinationTest, ChangelogTestFile)
 {
-    auto params = GetParam();
     ChangelogDirTest test("./logs");
-    setLogDirectory("./logs");
+    this->setLogDirectory("./logs");
 
     DB::KeeperLogStore changelog(
-        DB::LogFileSettings{.force_sync = true, .compress_logs = params.enable_compression, .rotate_interval = 5},
+        DB::LogFileSettings{.force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 5},
         DB::FlushSettings(),
-        keeper_context);
+        this->keeper_context);
     changelog.init(1, 0);
     auto entry = getLogEntry("hello world", 77);
     changelog.append(entry);
@@ -344,9 +371,9 @@ TEST_P(CoordinationTest, ChangelogTestFile)
 
     waitDurableLogs(changelog);
 
-    EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin" + params.extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin" + this->extension));
     for (const auto & p : fs::directory_iterator("./logs"))
-        EXPECT_EQ(p.path(), "./logs/changelog_1_5.bin" + params.extension);
+        EXPECT_EQ(p.path(), "./logs/changelog_1_5.bin" + this->extension);
 
     changelog.append(entry);
     changelog.append(entry);
@@ -357,20 +384,20 @@ TEST_P(CoordinationTest, ChangelogTestFile)
 
     waitDurableLogs(changelog);
 
-    EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin" + params.extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin" + this->extension));
 }
 
-TEST_P(CoordinationTest, ChangelogReadWrite)
+TYPED_TEST(CoordinationTest, ChangelogReadWrite)
 {
-    auto params = GetParam();
+
     ChangelogDirTest test("./logs");
-    setLogDirectory("./logs");
+    this->setLogDirectory("./logs");
 
     DB::KeeperLogStore changelog(
-        DB::LogFileSettings{.force_sync = true, .compress_logs = params.enable_compression, .rotate_interval = 1000},
+        DB::LogFileSettings{.force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 1000},
         DB::FlushSettings(),
-        keeper_context);
+        this->keeper_context);
     changelog.init(1, 0);
 
     for (size_t i = 0; i < 10; ++i)
@@ -385,9 +412,9 @@ TEST_P(CoordinationTest, ChangelogReadWrite)
     waitDurableLogs(changelog);
 
     DB::KeeperLogStore changelog_reader(
-        DB::LogFileSettings{.force_sync = true, .compress_logs = params.enable_compression, .rotate_interval = 1000},
+        DB::LogFileSettings{.force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 1000},
         DB::FlushSettings(),
-        keeper_context);
+        this->keeper_context);
     changelog_reader.init(1, 0);
     EXPECT_EQ(changelog_reader.size(), 10);
     EXPECT_EQ(changelog_reader.last_entry()->get_term(), changelog.last_entry()->get_term());
@@ -403,16 +430,16 @@ TEST_P(CoordinationTest, ChangelogReadWrite)
     EXPECT_EQ(10, entries_from_range->size());
 }
 
-TEST_P(CoordinationTest, ChangelogWriteAt)
+TYPED_TEST(CoordinationTest, ChangelogWriteAt)
 {
-    auto params = GetParam();
+
     ChangelogDirTest test("./logs");
-    setLogDirectory("./logs");
+    this->setLogDirectory("./logs");
 
     DB::KeeperLogStore changelog(
-        DB::LogFileSettings{.force_sync = true, .compress_logs = params.enable_compression, .rotate_interval = 1000},
+        DB::LogFileSettings{.force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 1000},
         DB::FlushSettings(),
-        keeper_context);
+        this->keeper_context);
     changelog.init(1, 0);
     for (size_t i = 0; i < 10; ++i)
     {
@@ -435,9 +462,9 @@ TEST_P(CoordinationTest, ChangelogWriteAt)
     EXPECT_EQ(changelog.next_slot(), 8);
 
     DB::KeeperLogStore changelog_reader(
-        DB::LogFileSettings{.force_sync = true, .compress_logs = params.enable_compression, .rotate_interval = 1000},
+        DB::LogFileSettings{.force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 1000},
         DB::FlushSettings(),
-        keeper_context);
+        this->keeper_context);
     changelog_reader.init(1, 0);
 
     EXPECT_EQ(changelog_reader.size(), changelog.size());
@@ -447,16 +474,16 @@ TEST_P(CoordinationTest, ChangelogWriteAt)
 }
 
 
-TEST_P(CoordinationTest, ChangelogTestAppendAfterRead)
+TYPED_TEST(CoordinationTest, ChangelogTestAppendAfterRead)
 {
-    auto params = GetParam();
+
     ChangelogDirTest test("./logs");
-    setLogDirectory("./logs");
+    this->setLogDirectory("./logs");
 
     DB::KeeperLogStore changelog(
-        DB::LogFileSettings{.force_sync = true, .compress_logs = params.enable_compression, .rotate_interval = 5},
+        DB::LogFileSettings{.force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 5},
         DB::FlushSettings(),
-        keeper_context);
+        this->keeper_context);
     changelog.init(1, 0);
     for (size_t i = 0; i < 7; ++i)
     {
@@ -469,13 +496,13 @@ TEST_P(CoordinationTest, ChangelogTestAppendAfterRead)
 
     waitDurableLogs(changelog);
 
-    EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin" + params.extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin" + this->extension));
 
     DB::KeeperLogStore changelog_reader(
-        DB::LogFileSettings{.force_sync = true, .compress_logs = params.enable_compression, .rotate_interval = 5},
+        DB::LogFileSettings{.force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 5},
         DB::FlushSettings(),
-        keeper_context);
+        this->keeper_context);
     changelog_reader.init(1, 0);
 
     EXPECT_EQ(changelog_reader.size(), 7);
@@ -488,8 +515,8 @@ TEST_P(CoordinationTest, ChangelogTestAppendAfterRead)
     EXPECT_EQ(changelog_reader.size(), 10);
 
     waitDurableLogs(changelog_reader);
-    EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin" + params.extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin" + this->extension));
 
     size_t logs_count = 0;
     for (const auto & _ [[maybe_unused]] : fs::directory_iterator("./logs"))
@@ -504,9 +531,9 @@ TEST_P(CoordinationTest, ChangelogTestAppendAfterRead)
 
     waitDurableLogs(changelog_reader);
 
-    EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_11_15.bin" + params.extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_11_15.bin" + this->extension));
 
     logs_count = 0;
     for (const auto & _ [[maybe_unused]] : fs::directory_iterator("./logs"))
@@ -533,16 +560,16 @@ void assertFileDeleted(std::string path)
 
 }
 
-TEST_P(CoordinationTest, ChangelogTestCompaction)
+TYPED_TEST(CoordinationTest, ChangelogTestCompaction)
 {
-    auto params = GetParam();
+
     ChangelogDirTest test("./logs");
-    setLogDirectory("./logs");
+    this->setLogDirectory("./logs");
 
     DB::KeeperLogStore changelog(
-        DB::LogFileSettings{.force_sync = true, .compress_logs = params.enable_compression, .rotate_interval = 5},
+        DB::LogFileSettings{.force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 5},
         DB::FlushSettings(),
-        keeper_context);
+        this->keeper_context);
     changelog.init(1, 0);
 
     for (size_t i = 0; i < 3; ++i)
@@ -556,7 +583,7 @@ TEST_P(CoordinationTest, ChangelogTestCompaction)
 
     EXPECT_EQ(changelog.size(), 3);
 
-    keeper_context->setLastCommitIndex(2);
+    this->keeper_context->setLastCommitIndex(2);
     changelog.compact(2);
 
     EXPECT_EQ(changelog.size(), 1);
@@ -564,7 +591,7 @@ TEST_P(CoordinationTest, ChangelogTestCompaction)
     EXPECT_EQ(changelog.next_slot(), 4);
     EXPECT_EQ(changelog.last_entry()->get_term(), 20);
     // nothing should be deleted
-    EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin" + params.extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin" + this->extension));
 
     auto e1 = getLogEntry("hello world", 30);
     changelog.append(e1);
@@ -578,15 +605,15 @@ TEST_P(CoordinationTest, ChangelogTestCompaction)
 
     waitDurableLogs(changelog);
 
-    EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin" + params.extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin" + this->extension));
 
-    keeper_context->setLastCommitIndex(6);
+    this->keeper_context->setLastCommitIndex(6);
     changelog.compact(6);
     std::this_thread::sleep_for(std::chrono::microseconds(1000));
 
-    assertFileDeleted("./logs/changelog_1_5.bin" + params.extension);
-    EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin" + params.extension));
+    assertFileDeleted("./logs/changelog_1_5.bin" + this->extension);
+    EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin" + this->extension));
 
     EXPECT_EQ(changelog.size(), 1);
     EXPECT_EQ(changelog.start_index(), 7);
@@ -594,9 +621,9 @@ TEST_P(CoordinationTest, ChangelogTestCompaction)
     EXPECT_EQ(changelog.last_entry()->get_term(), 60);
     /// And we able to read it
     DB::KeeperLogStore changelog_reader(
-        DB::LogFileSettings{.force_sync = true, .compress_logs = params.enable_compression, .rotate_interval = 5},
+        DB::LogFileSettings{.force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 5},
         DB::FlushSettings(),
-        keeper_context);
+        this->keeper_context);
     changelog_reader.init(7, 0);
 
     EXPECT_EQ(changelog_reader.size(), 1);
@@ -605,16 +632,16 @@ TEST_P(CoordinationTest, ChangelogTestCompaction)
     EXPECT_EQ(changelog_reader.last_entry()->get_term(), 60);
 }
 
-TEST_P(CoordinationTest, ChangelogTestBatchOperations)
+TYPED_TEST(CoordinationTest, ChangelogTestBatchOperations)
 {
-    auto params = GetParam();
+
     ChangelogDirTest test("./logs");
-    setLogDirectory("./logs");
+    this->setLogDirectory("./logs");
 
     DB::KeeperLogStore changelog(
-        DB::LogFileSettings{.force_sync = true, .compress_logs = params.enable_compression, .rotate_interval = 100},
+        DB::LogFileSettings{.force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 100},
         DB::FlushSettings(),
-        keeper_context);
+        this->keeper_context);
     changelog.init(1, 0);
     for (size_t i = 0; i < 10; ++i)
     {
@@ -630,9 +657,9 @@ TEST_P(CoordinationTest, ChangelogTestBatchOperations)
     auto entries = changelog.pack(1, 5);
 
     DB::KeeperLogStore apply_changelog(
-        DB::LogFileSettings{.force_sync = true, .compress_logs = params.enable_compression, .rotate_interval = 100},
+        DB::LogFileSettings{.force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 100},
         DB::FlushSettings(),
-        keeper_context);
+        this->keeper_context);
     apply_changelog.init(1, 0);
 
     for (size_t i = 0; i < 10; ++i)
@@ -660,18 +687,18 @@ TEST_P(CoordinationTest, ChangelogTestBatchOperations)
     EXPECT_EQ(apply_changelog.entry_at(12)->get_term(), 40);
 }
 
-TEST_P(CoordinationTest, ChangelogTestBatchOperationsEmpty)
+TYPED_TEST(CoordinationTest, ChangelogTestBatchOperationsEmpty)
 {
-    auto params = GetParam();
+
     ChangelogDirTest test("./logs");
-    setLogDirectory("./logs");
+    this->setLogDirectory("./logs");
 
     nuraft::ptr<nuraft::buffer> entries;
     {
         DB::KeeperLogStore changelog(
-            DB::LogFileSettings{.force_sync = true, .compress_logs = params.enable_compression, .rotate_interval = 100},
+            DB::LogFileSettings{.force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 100},
             DB::FlushSettings(),
-            keeper_context);
+            this->keeper_context);
         changelog.init(1, 0);
         for (size_t i = 0; i < 10; ++i)
         {
@@ -688,11 +715,11 @@ TEST_P(CoordinationTest, ChangelogTestBatchOperationsEmpty)
     }
 
     ChangelogDirTest test1("./logs1");
-    setLogDirectory("./logs1");
+    this->setLogDirectory("./logs1");
     DB::KeeperLogStore changelog_new(
-        DB::LogFileSettings{.force_sync = true, .compress_logs = params.enable_compression, .rotate_interval = 100},
+        DB::LogFileSettings{.force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 100},
         DB::FlushSettings(),
-        keeper_context);
+        this->keeper_context);
     changelog_new.init(1, 0);
     EXPECT_EQ(changelog_new.size(), 0);
 
@@ -715,23 +742,23 @@ TEST_P(CoordinationTest, ChangelogTestBatchOperationsEmpty)
     EXPECT_EQ(changelog_new.next_slot(), 11);
 
     DB::KeeperLogStore changelog_reader(
-        DB::LogFileSettings{.force_sync = true, .compress_logs = params.enable_compression, .rotate_interval = 100},
+        DB::LogFileSettings{.force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 100},
         DB::FlushSettings(),
-        keeper_context);
+        this->keeper_context);
     changelog_reader.init(5, 0);
 }
 
 
-TEST_P(CoordinationTest, ChangelogTestWriteAtPreviousFile)
+TYPED_TEST(CoordinationTest, ChangelogTestWriteAtPreviousFile)
 {
-    auto params = GetParam();
+
     ChangelogDirTest test("./logs");
-    setLogDirectory("./logs");
+    this->setLogDirectory("./logs");
 
     DB::KeeperLogStore changelog(
-        DB::LogFileSettings{.force_sync = true, .compress_logs = params.enable_compression, .rotate_interval = 5},
+        DB::LogFileSettings{.force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 5},
         DB::FlushSettings(),
-        keeper_context);
+        this->keeper_context);
     changelog.init(1, 0);
 
     for (size_t i = 0; i < 33; ++i)
@@ -743,13 +770,13 @@ TEST_P(CoordinationTest, ChangelogTestWriteAtPreviousFile)
 
     waitDurableLogs(changelog);
 
-    EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_11_15.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_16_20.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_21_25.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_26_30.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_31_35.bin" + params.extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_11_15.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_16_20.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_21_25.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_26_30.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_31_35.bin" + this->extension));
 
     EXPECT_EQ(changelog.size(), 33);
 
@@ -763,19 +790,19 @@ TEST_P(CoordinationTest, ChangelogTestWriteAtPreviousFile)
 
     waitDurableLogs(changelog);
 
-    EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin" + params.extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin" + this->extension));
 
-    EXPECT_FALSE(fs::exists("./logs/changelog_11_15.bin" + params.extension));
-    EXPECT_FALSE(fs::exists("./logs/changelog_16_20.bin" + params.extension));
-    EXPECT_FALSE(fs::exists("./logs/changelog_21_25.bin" + params.extension));
-    EXPECT_FALSE(fs::exists("./logs/changelog_26_30.bin" + params.extension));
-    EXPECT_FALSE(fs::exists("./logs/changelog_31_35.bin" + params.extension));
+    EXPECT_FALSE(fs::exists("./logs/changelog_11_15.bin" + this->extension));
+    EXPECT_FALSE(fs::exists("./logs/changelog_16_20.bin" + this->extension));
+    EXPECT_FALSE(fs::exists("./logs/changelog_21_25.bin" + this->extension));
+    EXPECT_FALSE(fs::exists("./logs/changelog_26_30.bin" + this->extension));
+    EXPECT_FALSE(fs::exists("./logs/changelog_31_35.bin" + this->extension));
 
     DB::KeeperLogStore changelog_read(
-        DB::LogFileSettings{.force_sync = true, .compress_logs = params.enable_compression, .rotate_interval = 5},
+        DB::LogFileSettings{.force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 5},
         DB::FlushSettings(),
-        keeper_context);
+        this->keeper_context);
     changelog_read.init(1, 0);
     EXPECT_EQ(changelog_read.size(), 7);
     EXPECT_EQ(changelog_read.start_index(), 1);
@@ -783,16 +810,16 @@ TEST_P(CoordinationTest, ChangelogTestWriteAtPreviousFile)
     EXPECT_EQ(changelog_read.last_entry()->get_term(), 5555);
 }
 
-TEST_P(CoordinationTest, ChangelogTestWriteAtFileBorder)
+TYPED_TEST(CoordinationTest, ChangelogTestWriteAtFileBorder)
 {
-    auto params = GetParam();
+
     ChangelogDirTest test("./logs");
-    setLogDirectory("./logs");
+    this->setLogDirectory("./logs");
 
     DB::KeeperLogStore changelog(
-        DB::LogFileSettings{.force_sync = true, .compress_logs = params.enable_compression, .rotate_interval = 5},
+        DB::LogFileSettings{.force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 5},
         DB::FlushSettings(),
-        keeper_context);
+        this->keeper_context);
     changelog.init(1, 0);
 
     for (size_t i = 0; i < 33; ++i)
@@ -804,13 +831,13 @@ TEST_P(CoordinationTest, ChangelogTestWriteAtFileBorder)
 
     waitDurableLogs(changelog);
 
-    EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_11_15.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_16_20.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_21_25.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_26_30.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_31_35.bin" + params.extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_11_15.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_16_20.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_21_25.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_26_30.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_31_35.bin" + this->extension));
 
     EXPECT_EQ(changelog.size(), 33);
 
@@ -824,19 +851,19 @@ TEST_P(CoordinationTest, ChangelogTestWriteAtFileBorder)
 
     waitDurableLogs(changelog);
 
-    EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_11_15.bin" + params.extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_11_15.bin" + this->extension));
 
-    EXPECT_FALSE(fs::exists("./logs/changelog_16_20.bin" + params.extension));
-    EXPECT_FALSE(fs::exists("./logs/changelog_21_25.bin" + params.extension));
-    EXPECT_FALSE(fs::exists("./logs/changelog_26_30.bin" + params.extension));
-    EXPECT_FALSE(fs::exists("./logs/changelog_31_35.bin" + params.extension));
+    EXPECT_FALSE(fs::exists("./logs/changelog_16_20.bin" + this->extension));
+    EXPECT_FALSE(fs::exists("./logs/changelog_21_25.bin" + this->extension));
+    EXPECT_FALSE(fs::exists("./logs/changelog_26_30.bin" + this->extension));
+    EXPECT_FALSE(fs::exists("./logs/changelog_31_35.bin" + this->extension));
 
     DB::KeeperLogStore changelog_read(
-        DB::LogFileSettings{.force_sync = true, .compress_logs = params.enable_compression, .rotate_interval = 5},
+        DB::LogFileSettings{.force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 5},
         DB::FlushSettings(),
-        keeper_context);
+        this->keeper_context);
     changelog_read.init(1, 0);
     EXPECT_EQ(changelog_read.size(), 11);
     EXPECT_EQ(changelog_read.start_index(), 1);
@@ -844,16 +871,16 @@ TEST_P(CoordinationTest, ChangelogTestWriteAtFileBorder)
     EXPECT_EQ(changelog_read.last_entry()->get_term(), 5555);
 }
 
-TEST_P(CoordinationTest, ChangelogTestWriteAtAllFiles)
+TYPED_TEST(CoordinationTest, ChangelogTestWriteAtAllFiles)
 {
-    auto params = GetParam();
+
     ChangelogDirTest test("./logs");
-    setLogDirectory("./logs");
+    this->setLogDirectory("./logs");
 
     DB::KeeperLogStore changelog(
-        DB::LogFileSettings{.force_sync = true, .compress_logs = params.enable_compression, .rotate_interval = 5},
+        DB::LogFileSettings{.force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 5},
         DB::FlushSettings(),
-        keeper_context);
+        this->keeper_context);
     changelog.init(1, 0);
     for (size_t i = 0; i < 33; ++i)
     {
@@ -864,13 +891,13 @@ TEST_P(CoordinationTest, ChangelogTestWriteAtAllFiles)
 
     waitDurableLogs(changelog);
 
-    EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_11_15.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_16_20.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_21_25.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_26_30.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_31_35.bin" + params.extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_11_15.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_16_20.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_21_25.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_26_30.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_31_35.bin" + this->extension));
 
     EXPECT_EQ(changelog.size(), 33);
 
@@ -884,26 +911,26 @@ TEST_P(CoordinationTest, ChangelogTestWriteAtAllFiles)
 
     waitDurableLogs(changelog);
 
-    EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin" + params.extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin" + this->extension));
 
-    EXPECT_FALSE(fs::exists("./logs/changelog_6_10.bin" + params.extension));
-    EXPECT_FALSE(fs::exists("./logs/changelog_11_15.bin" + params.extension));
-    EXPECT_FALSE(fs::exists("./logs/changelog_16_20.bin" + params.extension));
-    EXPECT_FALSE(fs::exists("./logs/changelog_21_25.bin" + params.extension));
-    EXPECT_FALSE(fs::exists("./logs/changelog_26_30.bin" + params.extension));
-    EXPECT_FALSE(fs::exists("./logs/changelog_31_35.bin" + params.extension));
+    EXPECT_FALSE(fs::exists("./logs/changelog_6_10.bin" + this->extension));
+    EXPECT_FALSE(fs::exists("./logs/changelog_11_15.bin" + this->extension));
+    EXPECT_FALSE(fs::exists("./logs/changelog_16_20.bin" + this->extension));
+    EXPECT_FALSE(fs::exists("./logs/changelog_21_25.bin" + this->extension));
+    EXPECT_FALSE(fs::exists("./logs/changelog_26_30.bin" + this->extension));
+    EXPECT_FALSE(fs::exists("./logs/changelog_31_35.bin" + this->extension));
 }
 
-TEST_P(CoordinationTest, ChangelogTestStartNewLogAfterRead)
+TYPED_TEST(CoordinationTest, ChangelogTestStartNewLogAfterRead)
 {
-    auto params = GetParam();
+
     ChangelogDirTest test("./logs");
-    setLogDirectory("./logs");
+    this->setLogDirectory("./logs");
 
     DB::KeeperLogStore changelog(
-        DB::LogFileSettings{.force_sync = true, .compress_logs = params.enable_compression, .rotate_interval = 5},
+        DB::LogFileSettings{.force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 5},
         DB::FlushSettings(),
-        keeper_context);
+        this->keeper_context);
     changelog.init(1, 0);
 
     for (size_t i = 0; i < 35; ++i)
@@ -915,19 +942,19 @@ TEST_P(CoordinationTest, ChangelogTestStartNewLogAfterRead)
     EXPECT_EQ(changelog.size(), 35);
 
     waitDurableLogs(changelog);
-    EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_11_15.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_16_20.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_21_25.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_26_30.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_31_35.bin" + params.extension));
-    EXPECT_FALSE(fs::exists("./logs/changelog_36_40.bin" + params.extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_11_15.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_16_20.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_21_25.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_26_30.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_31_35.bin" + this->extension));
+    EXPECT_FALSE(fs::exists("./logs/changelog_36_40.bin" + this->extension));
 
     DB::KeeperLogStore changelog_reader(
-        DB::LogFileSettings{.force_sync = true, .compress_logs = params.enable_compression, .rotate_interval = 5},
+        DB::LogFileSettings{.force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 5},
         DB::FlushSettings(),
-        keeper_context);
+        this->keeper_context);
     changelog_reader.init(1, 0);
 
     auto entry = getLogEntry("36_hello_world", 360);
@@ -937,14 +964,14 @@ TEST_P(CoordinationTest, ChangelogTestStartNewLogAfterRead)
     EXPECT_EQ(changelog_reader.size(), 36);
 
     waitDurableLogs(changelog_reader);
-    EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_11_15.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_16_20.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_21_25.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_26_30.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_31_35.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_36_40.bin" + params.extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_11_15.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_16_20.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_21_25.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_26_30.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_31_35.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_36_40.bin" + this->extension));
 }
 
 namespace
@@ -965,18 +992,18 @@ void assertBrokenFileRemoved(const fs::path & directory, const fs::path & filena
 
 }
 
-TEST_P(CoordinationTest, ChangelogTestReadAfterBrokenTruncate)
+TYPED_TEST(CoordinationTest, ChangelogTestReadAfterBrokenTruncate)
 {
     static const fs::path log_folder{"./logs"};
 
-    auto params = GetParam();
+
     ChangelogDirTest test(log_folder);
-    setLogDirectory(log_folder);
+    this->setLogDirectory(log_folder);
 
     DB::KeeperLogStore changelog(
-        DB::LogFileSettings{.force_sync = true, .compress_logs = params.enable_compression, .rotate_interval = 5},
+        DB::LogFileSettings{.force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 5},
         DB::FlushSettings(),
-        keeper_context);
+        this->keeper_context);
     changelog.init(1, 0);
 
     for (size_t i = 0; i < 35; ++i)
@@ -988,36 +1015,36 @@ TEST_P(CoordinationTest, ChangelogTestReadAfterBrokenTruncate)
     EXPECT_EQ(changelog.size(), 35);
 
     waitDurableLogs(changelog);
-    EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_11_15.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_16_20.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_21_25.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_26_30.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_31_35.bin" + params.extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_11_15.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_16_20.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_21_25.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_26_30.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_31_35.bin" + this->extension));
 
     DB::WriteBufferFromFile plain_buf(
-        "./logs/changelog_11_15.bin" + params.extension, DB::DBMS_DEFAULT_BUFFER_SIZE, O_APPEND | O_CREAT | O_WRONLY);
+        "./logs/changelog_11_15.bin" + this->extension, DB::DBMS_DEFAULT_BUFFER_SIZE, O_APPEND | O_CREAT | O_WRONLY);
     plain_buf.truncate(0);
 
     DB::KeeperLogStore changelog_reader(
-        DB::LogFileSettings{.force_sync = true, .compress_logs = params.enable_compression, .rotate_interval = 5},
+        DB::LogFileSettings{.force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 5},
         DB::FlushSettings(),
-        keeper_context);
+        this->keeper_context);
     changelog_reader.init(1, 0);
     changelog_reader.end_of_append_batch(0, 0);
 
     EXPECT_EQ(changelog_reader.size(), 10);
     EXPECT_EQ(changelog_reader.last_entry()->get_term(), 90);
 
-    EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_11_15.bin" + params.extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_11_15.bin" + this->extension));
 
-    assertBrokenFileRemoved(log_folder, "changelog_16_20.bin" + params.extension);
-    assertBrokenFileRemoved(log_folder, "changelog_21_25.bin" + params.extension);
-    assertBrokenFileRemoved(log_folder, "changelog_26_30.bin" + params.extension);
-    assertBrokenFileRemoved(log_folder, "changelog_31_35.bin" + params.extension);
+    assertBrokenFileRemoved(log_folder, "changelog_16_20.bin" + this->extension);
+    assertBrokenFileRemoved(log_folder, "changelog_21_25.bin" + this->extension);
+    assertBrokenFileRemoved(log_folder, "changelog_26_30.bin" + this->extension);
+    assertBrokenFileRemoved(log_folder, "changelog_31_35.bin" + this->extension);
 
     auto entry = getLogEntry("h", 7777);
     changelog_reader.append(entry);
@@ -1027,35 +1054,35 @@ TEST_P(CoordinationTest, ChangelogTestReadAfterBrokenTruncate)
 
     waitDurableLogs(changelog_reader);
 
-    EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_11_15.bin" + params.extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_1_5.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_11_15.bin" + this->extension));
 
-    assertBrokenFileRemoved(log_folder, "changelog_16_20.bin" + params.extension);
-    assertBrokenFileRemoved(log_folder, "changelog_21_25.bin" + params.extension);
-    assertBrokenFileRemoved(log_folder, "changelog_26_30.bin" + params.extension);
-    assertBrokenFileRemoved(log_folder, "changelog_31_35.bin" + params.extension);
+    assertBrokenFileRemoved(log_folder, "changelog_16_20.bin" + this->extension);
+    assertBrokenFileRemoved(log_folder, "changelog_21_25.bin" + this->extension);
+    assertBrokenFileRemoved(log_folder, "changelog_26_30.bin" + this->extension);
+    assertBrokenFileRemoved(log_folder, "changelog_31_35.bin" + this->extension);
 
     DB::KeeperLogStore changelog_reader2(
-        DB::LogFileSettings{.force_sync = true, .compress_logs = params.enable_compression, .rotate_interval = 5},
+        DB::LogFileSettings{.force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 5},
         DB::FlushSettings(),
-        keeper_context);
+        this->keeper_context);
     changelog_reader2.init(1, 0);
     EXPECT_EQ(changelog_reader2.size(), 11);
     EXPECT_EQ(changelog_reader2.last_entry()->get_term(), 7777);
 }
 
 /// Truncating all entries
-TEST_P(CoordinationTest, ChangelogTestReadAfterBrokenTruncate2)
+TYPED_TEST(CoordinationTest, ChangelogTestReadAfterBrokenTruncate2)
 {
-    auto params = GetParam();
+
     ChangelogDirTest test("./logs");
-    setLogDirectory("./logs");
+    this->setLogDirectory("./logs");
 
     DB::KeeperLogStore changelog(
-        DB::LogFileSettings{.force_sync = true, .compress_logs = params.enable_compression, .rotate_interval = 20},
+        DB::LogFileSettings{.force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 20},
         DB::FlushSettings(),
-        keeper_context);
+        this->keeper_context);
     changelog.init(1, 0);
 
     for (size_t i = 0; i < 35; ++i)
@@ -1066,22 +1093,22 @@ TEST_P(CoordinationTest, ChangelogTestReadAfterBrokenTruncate2)
     changelog.end_of_append_batch(0, 0);
 
     waitDurableLogs(changelog);
-    EXPECT_TRUE(fs::exists("./logs/changelog_1_20.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_21_40.bin" + params.extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_1_20.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_21_40.bin" + this->extension));
 
     DB::WriteBufferFromFile plain_buf(
-        "./logs/changelog_1_20.bin" + params.extension, DB::DBMS_DEFAULT_BUFFER_SIZE, O_APPEND | O_CREAT | O_WRONLY);
+        "./logs/changelog_1_20.bin" + this->extension, DB::DBMS_DEFAULT_BUFFER_SIZE, O_APPEND | O_CREAT | O_WRONLY);
     plain_buf.truncate(30);
 
     DB::KeeperLogStore changelog_reader(
-        DB::LogFileSettings{.force_sync = true, .compress_logs = params.enable_compression, .rotate_interval = 20},
+        DB::LogFileSettings{.force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 20},
         DB::FlushSettings(),
-        keeper_context);
+        this->keeper_context);
     changelog_reader.init(1, 0);
 
     EXPECT_EQ(changelog_reader.size(), 0);
-    EXPECT_TRUE(fs::exists("./logs/changelog_1_20.bin" + params.extension));
-    assertBrokenFileRemoved("./logs", "changelog_21_40.bin" + params.extension);
+    EXPECT_TRUE(fs::exists("./logs/changelog_1_20.bin" + this->extension));
+    assertBrokenFileRemoved("./logs", "changelog_21_40.bin" + this->extension);
     auto entry = getLogEntry("hello_world", 7777);
     changelog_reader.append(entry);
     changelog_reader.end_of_append_batch(0, 0);
@@ -1092,9 +1119,9 @@ TEST_P(CoordinationTest, ChangelogTestReadAfterBrokenTruncate2)
     EXPECT_EQ(changelog_reader.last_entry()->get_term(), 7777);
 
     DB::KeeperLogStore changelog_reader2(
-        DB::LogFileSettings{.force_sync = true, .compress_logs = params.enable_compression, .rotate_interval = 1},
+        DB::LogFileSettings{.force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 1},
         DB::FlushSettings(),
-        keeper_context);
+        this->keeper_context);
     changelog_reader2.init(1, 0);
     EXPECT_EQ(changelog_reader2.size(), 1);
     EXPECT_EQ(changelog_reader2.last_entry()->get_term(), 7777);
@@ -1103,15 +1130,15 @@ TEST_P(CoordinationTest, ChangelogTestReadAfterBrokenTruncate2)
 /// Truncating only some entries from the end
 /// For compressed logs we have no reliable way of knowing how many log entries were lost
 /// after we truncate some bytes from the end
-TEST_F(CoordinationTest, ChangelogTestReadAfterBrokenTruncate3)
+TYPED_TEST(CoordinationTest, ChangelogTestReadAfterBrokenTruncate3)
 {
     ChangelogDirTest test("./logs");
-    setLogDirectory("./logs");
+    this->setLogDirectory("./logs");
 
     DB::KeeperLogStore changelog(
         DB::LogFileSettings{.force_sync = true, .compress_logs = false, .rotate_interval = 20},
         DB::FlushSettings(),
-        keeper_context);
+        this->keeper_context);
     changelog.init(1, 0);
 
     for (size_t i = 0; i < 35; ++i)
@@ -1133,7 +1160,7 @@ TEST_F(CoordinationTest, ChangelogTestReadAfterBrokenTruncate3)
     DB::KeeperLogStore changelog_reader(
         DB::LogFileSettings{.force_sync = true, .compress_logs = false, .rotate_interval = 20},
         DB::FlushSettings(),
-        keeper_context);
+        this->keeper_context);
     changelog_reader.init(1, 0);
 
     EXPECT_EQ(changelog_reader.size(), 19);
@@ -1150,10 +1177,10 @@ TEST_F(CoordinationTest, ChangelogTestReadAfterBrokenTruncate3)
     EXPECT_EQ(changelog_reader.last_entry()->get_term(), 7777);
 }
 
-TEST_F(CoordinationTest, ChangelogTestMixedLogTypes)
+TYPED_TEST(CoordinationTest, ChangelogTestMixedLogTypes)
 {
     ChangelogDirTest test("./logs");
-    setLogDirectory("./logs");
+    this->setLogDirectory("./logs");
 
     std::vector<std::string> changelog_files;
 
@@ -1185,7 +1212,7 @@ TEST_F(CoordinationTest, ChangelogTestMixedLogTypes)
         DB::KeeperLogStore changelog(
             DB::LogFileSettings{.force_sync = true, .compress_logs = false, .rotate_interval = 20},
             DB::FlushSettings(),
-            keeper_context);
+            this->keeper_context);
         changelog.init(1, 0);
 
         for (size_t i = 0; i < 35; ++i)
@@ -1206,7 +1233,7 @@ TEST_F(CoordinationTest, ChangelogTestMixedLogTypes)
         DB::KeeperLogStore changelog_compressed(
             DB::LogFileSettings{.force_sync = true, .compress_logs = true, .rotate_interval = 20},
             DB::FlushSettings(),
-            keeper_context);
+            this->keeper_context);
         changelog_compressed.init(1, 0);
 
         verify_changelog_files();
@@ -1228,7 +1255,7 @@ TEST_F(CoordinationTest, ChangelogTestMixedLogTypes)
         DB::KeeperLogStore changelog(
             DB::LogFileSettings{.force_sync = true, .compress_logs = false, .rotate_interval = 20},
             DB::FlushSettings(),
-            keeper_context);
+            this->keeper_context);
         changelog.init(1, 0);
 
         verify_changelog_files();
@@ -1246,16 +1273,16 @@ TEST_F(CoordinationTest, ChangelogTestMixedLogTypes)
     }
 }
 
-TEST_P(CoordinationTest, ChangelogTestLostFiles)
+TYPED_TEST(CoordinationTest, ChangelogTestLostFiles)
 {
-    auto params = GetParam();
+
     ChangelogDirTest test("./logs");
-    setLogDirectory("./logs");
+    this->setLogDirectory("./logs");
 
     DB::KeeperLogStore changelog(
-        DB::LogFileSettings{.force_sync = true, .compress_logs = params.enable_compression, .rotate_interval = 20},
+        DB::LogFileSettings{.force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 20},
         DB::FlushSettings(),
-        keeper_context);
+        this->keeper_context);
     changelog.init(1, 0);
 
     for (size_t i = 0; i < 35; ++i)
@@ -1266,30 +1293,30 @@ TEST_P(CoordinationTest, ChangelogTestLostFiles)
     changelog.end_of_append_batch(0, 0);
 
     waitDurableLogs(changelog);
-    EXPECT_TRUE(fs::exists("./logs/changelog_1_20.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_21_40.bin" + params.extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_1_20.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_21_40.bin" + this->extension));
 
-    fs::remove("./logs/changelog_1_20.bin" + params.extension);
+    fs::remove("./logs/changelog_1_20.bin" + this->extension);
 
     DB::KeeperLogStore changelog_reader(
-        DB::LogFileSettings{.force_sync = true, .compress_logs = params.enable_compression, .rotate_interval = 20},
+        DB::LogFileSettings{.force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 20},
         DB::FlushSettings(),
-        keeper_context);
+        this->keeper_context);
     /// It should print error message, but still able to start
     changelog_reader.init(5, 0);
-    assertBrokenFileRemoved("./logs", "changelog_21_40.bin" + params.extension);
+    assertBrokenFileRemoved("./logs", "changelog_21_40.bin" + this->extension);
 }
 
-TEST_P(CoordinationTest, ChangelogTestLostFiles2)
+TYPED_TEST(CoordinationTest, ChangelogTestLostFiles2)
 {
-    auto params = GetParam();
+
     ChangelogDirTest test("./logs");
-    setLogDirectory("./logs");
+    this->setLogDirectory("./logs");
 
     DB::KeeperLogStore changelog(
-        DB::LogFileSettings{.force_sync = true, .compress_logs = params.enable_compression, .rotate_interval = 10},
+        DB::LogFileSettings{.force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 10},
         DB::FlushSettings(),
-        keeper_context);
+        this->keeper_context);
     changelog.init(1, 0);
 
     for (size_t i = 0; i < 35; ++i)
@@ -1301,24 +1328,24 @@ TEST_P(CoordinationTest, ChangelogTestLostFiles2)
 
     waitDurableLogs(changelog);
 
-    EXPECT_TRUE(fs::exists("./logs/changelog_1_10.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_11_20.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_21_30.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_31_40.bin" + params.extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_1_10.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_11_20.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_21_30.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_31_40.bin" + this->extension));
 
     // we have a gap in our logs, we need to remove all the logs after the gap
-    fs::remove("./logs/changelog_21_30.bin" + params.extension);
+    fs::remove("./logs/changelog_21_30.bin" + this->extension);
 
     DB::KeeperLogStore changelog_reader(
-        DB::LogFileSettings{.force_sync = true, .compress_logs = params.enable_compression, .rotate_interval = 10},
+        DB::LogFileSettings{.force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 10},
         DB::FlushSettings(),
-        keeper_context);
+        this->keeper_context);
     /// It should print error message, but still able to start
     changelog_reader.init(5, 0);
-    EXPECT_TRUE(fs::exists("./logs/changelog_1_10.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_11_20.bin" + params.extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_1_10.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_11_20.bin" + this->extension));
 
-    assertBrokenFileRemoved("./logs", "changelog_31_40.bin" + params.extension);
+    assertBrokenFileRemoved("./logs", "changelog_31_40.bin" + this->extension);
 }
 struct IntNode
 {
@@ -1334,7 +1361,7 @@ struct IntNode
     bool operator!=(const int & rhs) const { return rhs != this->value; }
 };
 
-TEST_P(CoordinationTest, SnapshotableHashMapSimple)
+TYPED_TEST(CoordinationTest, SnapshotableHashMapSimple)
 {
     DB::SnapshotableHashTable<IntNode> hello;
     EXPECT_TRUE(hello.insert("hello", 5).second);
@@ -1349,7 +1376,7 @@ TEST_P(CoordinationTest, SnapshotableHashMapSimple)
     EXPECT_EQ(hello.size(), 0);
 }
 
-TEST_P(CoordinationTest, SnapshotableHashMapTrySnapshot)
+TYPED_TEST(CoordinationTest, SnapshotableHashMapTrySnapshot)
 {
     DB::SnapshotableHashTable<IntNode> map_snp;
     EXPECT_TRUE(map_snp.insert("/hello", 7).second);
@@ -1426,7 +1453,7 @@ TEST_P(CoordinationTest, SnapshotableHashMapTrySnapshot)
     map_snp.disableSnapshotMode();
 }
 
-TEST_P(CoordinationTest, SnapshotableHashMapDataSize)
+TYPED_TEST(CoordinationTest, SnapshotableHashMapDataSize)
 {
     /// int
     DB::SnapshotableHashTable<IntNode> hello;
@@ -1503,9 +1530,10 @@ TEST_P(CoordinationTest, SnapshotableHashMapDataSize)
     EXPECT_EQ(world.getApproximateDataSize(), 0);
 }
 
-void addNode(DB::KeeperMemoryStorage & storage, const std::string & path, const std::string & data, int64_t ephemeral_owner = 0)
+template <typename Storage>
+void addNode(Storage & storage, const std::string & path, const std::string & data, int64_t ephemeral_owner = 0)
 {
-    using Node = DB::KeeperMemoryStorage::Node;
+    using Node = typename Storage::Node;
     Node node{};
     node.setData(data);
     node.setEphemeralOwner(ephemeral_owner);
@@ -1521,15 +1549,20 @@ void addNode(DB::KeeperMemoryStorage & storage, const std::string & path, const
         });
 }
 
-TEST_P(CoordinationTest, TestStorageSnapshotSimple)
+TYPED_TEST(CoordinationTest, TestStorageSnapshotSimple)
 {
-    auto params = GetParam();
+
     ChangelogDirTest test("./snapshots");
-    setSnapshotDirectory("./snapshots");
+    this->setSnapshotDirectory("./snapshots");
 
-    DB::KeeperSnapshotManager<DB::KeeperMemoryStorage> manager(3, keeper_context, params.enable_compression);
+    using Storage = typename TestFixture::Storage;
 
-    DB::KeeperMemoryStorage storage(500, "", keeper_context);
+    ChangelogDirTest rocks("./rocksdb");
+    this->setRocksDBDirectory("./rocksdb");
+
+    DB::KeeperSnapshotManager<Storage> manager(3, this->keeper_context, this->enable_compression);
+
+    Storage storage(500, "", this->keeper_context);
     addNode(storage, "/hello1", "world", 1);
     addNode(storage, "/hello2", "somedata", 3);
     storage.session_id_counter = 5;
@@ -1539,7 +1572,7 @@ TEST_P(CoordinationTest, TestStorageSnapshotSimple)
     storage.getSessionID(130);
     storage.getSessionID(130);
 
-    DB::KeeperStorageSnapshot<DB::KeeperMemoryStorage> snapshot(&storage, 2);
+    DB::KeeperStorageSnapshot<Storage> snapshot(&storage, 2);
 
     EXPECT_EQ(snapshot.snapshot_meta->get_last_log_idx(), 2);
     EXPECT_EQ(snapshot.session_id, 7);
@@ -1548,7 +1581,7 @@ TEST_P(CoordinationTest, TestStorageSnapshotSimple)
 
     auto buf = manager.serializeSnapshotToBuffer(snapshot);
     manager.serializeSnapshotBufferToDisk(*buf, 2);
-    EXPECT_TRUE(fs::exists("./snapshots/snapshot_2.bin" + params.extension));
+    EXPECT_TRUE(fs::exists("./snapshots/snapshot_2.bin" + this->extension));
 
 
     auto debuf = manager.deserializeSnapshotBufferFromDisk(2);
@@ -1571,15 +1604,20 @@ TEST_P(CoordinationTest, TestStorageSnapshotSimple)
     EXPECT_EQ(restored_storage->session_and_timeout.size(), 2);
 }
 
-TEST_P(CoordinationTest, TestStorageSnapshotMoreWrites)
+TYPED_TEST(CoordinationTest, TestStorageSnapshotMoreWrites)
 {
-    auto params = GetParam();
+
     ChangelogDirTest test("./snapshots");
-    setSnapshotDirectory("./snapshots");
+    this->setSnapshotDirectory("./snapshots");
 
-    DB::KeeperSnapshotManager<DB::KeeperMemoryStorage> manager(3, keeper_context, params.enable_compression);
+    using Storage = typename TestFixture::Storage;
 
-    DB::KeeperMemoryStorage storage(500, "", keeper_context);
+    ChangelogDirTest rocks("./rocksdb");
+    this->setRocksDBDirectory("./rocksdb");
+
+    DB::KeeperSnapshotManager<Storage> manager(3, this->keeper_context, this->enable_compression);
+
+    Storage storage(500, "", this->keeper_context);
     storage.getSessionID(130);
 
     for (size_t i = 0; i < 50; ++i)
@@ -1587,7 +1625,7 @@ TEST_P(CoordinationTest, TestStorageSnapshotMoreWrites)
         addNode(storage, "/hello_" + std::to_string(i), "world_" + std::to_string(i));
     }
 
-    DB::KeeperStorageSnapshot<DB::KeeperMemoryStorage> snapshot(&storage, 50);
+    DB::KeeperStorageSnapshot<Storage> snapshot(&storage, 50);
     EXPECT_EQ(snapshot.snapshot_meta->get_last_log_idx(), 50);
     EXPECT_EQ(snapshot.snapshot_container_size, 54);
 
@@ -1600,7 +1638,7 @@ TEST_P(CoordinationTest, TestStorageSnapshotMoreWrites)
 
     auto buf = manager.serializeSnapshotToBuffer(snapshot);
     manager.serializeSnapshotBufferToDisk(*buf, 50);
-    EXPECT_TRUE(fs::exists("./snapshots/snapshot_50.bin" + params.extension));
+    EXPECT_TRUE(fs::exists("./snapshots/snapshot_50.bin" + this->extension));
 
 
     auto debuf = manager.deserializeSnapshotBufferFromDisk(50);
@@ -1614,15 +1652,20 @@ TEST_P(CoordinationTest, TestStorageSnapshotMoreWrites)
 }
 
 
-TEST_P(CoordinationTest, TestStorageSnapshotManySnapshots)
+TYPED_TEST(CoordinationTest, TestStorageSnapshotManySnapshots)
 {
-    auto params = GetParam();
+
     ChangelogDirTest test("./snapshots");
-    setSnapshotDirectory("./snapshots");
+    this->setSnapshotDirectory("./snapshots");
 
-    DB::KeeperSnapshotManager<DB::KeeperMemoryStorage> manager(3, keeper_context, params.enable_compression);
+    using Storage = typename TestFixture::Storage;
 
-    DB::KeeperMemoryStorage storage(500, "", keeper_context);
+    ChangelogDirTest rocks("./rocksdb");
+    this->setRocksDBDirectory("./rocksdb");
+
+    DB::KeeperSnapshotManager<Storage> manager(3, this->keeper_context, this->enable_compression);
+
+    Storage storage(500, "", this->keeper_context);
     storage.getSessionID(130);
 
     for (size_t j = 1; j <= 5; ++j)
@@ -1632,17 +1675,17 @@ TEST_P(CoordinationTest, TestStorageSnapshotManySnapshots)
             addNode(storage, "/hello_" + std::to_string(i), "world_" + std::to_string(i));
         }
 
-        DB::KeeperStorageSnapshot<DB::KeeperMemoryStorage> snapshot(&storage, j * 50);
+        DB::KeeperStorageSnapshot<Storage> snapshot(&storage, j * 50);
         auto buf = manager.serializeSnapshotToBuffer(snapshot);
         manager.serializeSnapshotBufferToDisk(*buf, j * 50);
-        EXPECT_TRUE(fs::exists(std::string{"./snapshots/snapshot_"} + std::to_string(j * 50) + ".bin" + params.extension));
+        EXPECT_TRUE(fs::exists(std::string{"./snapshots/snapshot_"} + std::to_string(j * 50) + ".bin" + this->extension));
     }
 
-    EXPECT_FALSE(fs::exists("./snapshots/snapshot_50.bin" + params.extension));
-    EXPECT_FALSE(fs::exists("./snapshots/snapshot_100.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./snapshots/snapshot_150.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./snapshots/snapshot_200.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./snapshots/snapshot_250.bin" + params.extension));
+    EXPECT_FALSE(fs::exists("./snapshots/snapshot_50.bin" + this->extension));
+    EXPECT_FALSE(fs::exists("./snapshots/snapshot_100.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./snapshots/snapshot_150.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./snapshots/snapshot_200.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./snapshots/snapshot_250.bin" + this->extension));
 
 
     auto [restored_storage, meta, _] = manager.restoreFromLatestSnapshot();
@@ -1655,21 +1698,26 @@ TEST_P(CoordinationTest, TestStorageSnapshotManySnapshots)
     }
 }
 
-TEST_P(CoordinationTest, TestStorageSnapshotMode)
+TYPED_TEST(CoordinationTest, TestStorageSnapshotMode)
 {
-    auto params = GetParam();
-    ChangelogDirTest test("./snapshots");
-    setSnapshotDirectory("./snapshots");
 
-    DB::KeeperSnapshotManager<DB::KeeperMemoryStorage> manager(3, keeper_context, params.enable_compression);
-    DB::KeeperMemoryStorage storage(500, "", keeper_context);
+    ChangelogDirTest test("./snapshots");
+    this->setSnapshotDirectory("./snapshots");
+
+    using Storage = typename TestFixture::Storage;
+
+    ChangelogDirTest rocks("./rocksdb");
+    this->setRocksDBDirectory("./rocksdb");
+
+    DB::KeeperSnapshotManager<Storage> manager(3, this->keeper_context, this->enable_compression);
+    Storage storage(500, "", this->keeper_context);
     for (size_t i = 0; i < 50; ++i)
     {
         addNode(storage, "/hello_" + std::to_string(i), "world_" + std::to_string(i));
     }
 
     {
-        DB::KeeperStorageSnapshot<DB::KeeperMemoryStorage> snapshot(&storage, 50);
+        DB::KeeperStorageSnapshot<Storage> snapshot(&storage, 50);
         for (size_t i = 0; i < 50; ++i)
         {
             addNode(storage, "/hello_" + std::to_string(i), "wlrd_" + std::to_string(i));
@@ -1684,12 +1732,15 @@ TEST_P(CoordinationTest, TestStorageSnapshotMode)
                 storage.container.erase("/hello_" + std::to_string(i));
         }
         EXPECT_EQ(storage.container.size(), 29);
-        EXPECT_EQ(storage.container.snapshotSizeWithVersion().first, 105);
+        if constexpr (Storage::use_rocksdb)
+            EXPECT_EQ(storage.container.snapshotSizeWithVersion().first, 54);
+        else
+            EXPECT_EQ(storage.container.snapshotSizeWithVersion().first, 105);
         EXPECT_EQ(storage.container.snapshotSizeWithVersion().second, 1);
         auto buf = manager.serializeSnapshotToBuffer(snapshot);
         manager.serializeSnapshotBufferToDisk(*buf, 50);
     }
-    EXPECT_TRUE(fs::exists("./snapshots/snapshot_50.bin" + params.extension));
+    EXPECT_TRUE(fs::exists("./snapshots/snapshot_50.bin" + this->extension));
     EXPECT_EQ(storage.container.size(), 29);
     storage.clearGarbageAfterSnapshot();
     EXPECT_EQ(storage.container.snapshotSizeWithVersion().first, 29);
@@ -1709,28 +1760,33 @@ TEST_P(CoordinationTest, TestStorageSnapshotMode)
     }
 }
 
-TEST_P(CoordinationTest, TestStorageSnapshotBroken)
+TYPED_TEST(CoordinationTest, TestStorageSnapshotBroken)
 {
-    auto params = GetParam();
-    ChangelogDirTest test("./snapshots");
-    setSnapshotDirectory("./snapshots");
 
-    DB::KeeperSnapshotManager<DB::KeeperMemoryStorage> manager(3, keeper_context, params.enable_compression);
-    DB::KeeperMemoryStorage storage(500, "", keeper_context);
+    ChangelogDirTest test("./snapshots");
+    this->setSnapshotDirectory("./snapshots");
+
+    using Storage = typename TestFixture::Storage;
+
+    ChangelogDirTest rocks("./rocksdb");
+    this->setRocksDBDirectory("./rocksdb");
+
+    DB::KeeperSnapshotManager<Storage> manager(3, this->keeper_context, this->enable_compression);
+    Storage storage(500, "", this->keeper_context);
     for (size_t i = 0; i < 50; ++i)
     {
         addNode(storage, "/hello_" + std::to_string(i), "world_" + std::to_string(i));
     }
     {
-        DB::KeeperStorageSnapshot<DB::KeeperMemoryStorage> snapshot(&storage, 50);
+        DB::KeeperStorageSnapshot<Storage> snapshot(&storage, 50);
         auto buf = manager.serializeSnapshotToBuffer(snapshot);
         manager.serializeSnapshotBufferToDisk(*buf, 50);
     }
-    EXPECT_TRUE(fs::exists("./snapshots/snapshot_50.bin" + params.extension));
+    EXPECT_TRUE(fs::exists("./snapshots/snapshot_50.bin" + this->extension));
 
     /// Let's corrupt file
     DB::WriteBufferFromFile plain_buf(
-        "./snapshots/snapshot_50.bin" + params.extension, DB::DBMS_DEFAULT_BUFFER_SIZE, O_APPEND | O_CREAT | O_WRONLY);
+        "./snapshots/snapshot_50.bin" + this->extension, DB::DBMS_DEFAULT_BUFFER_SIZE, O_APPEND | O_CREAT | O_WRONLY);
     plain_buf.truncate(34);
     plain_buf.sync();
 
@@ -1757,6 +1813,7 @@ getLogEntryFromZKRequest(size_t term, int64_t session_id, int64_t zxid, const Co
     return nuraft::cs_new<nuraft::log_entry>(term, buffer);
 }
 
+template<typename Storage>
 void testLogAndStateMachine(
     DB::CoordinationSettingsPtr settings,
     uint64_t total_logs,
@@ -1767,12 +1824,15 @@ void testLogAndStateMachine(
 
     ChangelogDirTest snapshots("./snapshots");
     ChangelogDirTest logs("./logs");
+    ChangelogDirTest rocks("./rocksdb");
 
     auto get_keeper_context = [&]
     {
         auto local_keeper_context = std::make_shared<DB::KeeperContext>(true, settings);
         local_keeper_context->setSnapshotDisk(std::make_shared<DiskLocal>("SnapshotDisk", "./snapshots"));
         local_keeper_context->setLogDisk(std::make_shared<DiskLocal>("LogDisk", "./logs"));
+        local_keeper_context->setRocksDBDisk(std::make_shared<DiskLocal>("RocksDisk", "./rocksdb"));
+        local_keeper_context->setRocksDBOptions();
         return local_keeper_context;
     };
 
@@ -1780,7 +1840,7 @@ void testLogAndStateMachine(
     SnapshotsQueue snapshots_queue{1};
 
     auto keeper_context = get_keeper_context();
-    auto state_machine = std::make_shared<KeeperStateMachine<DB::KeeperMemoryStorage>>(queue, snapshots_queue, keeper_context, nullptr);
+    auto state_machine = std::make_shared<KeeperStateMachine<Storage>>(queue, snapshots_queue, keeper_context, nullptr);
 
     state_machine->init();
     DB::KeeperLogStore changelog(
@@ -1827,7 +1887,7 @@ void testLogAndStateMachine(
 
     SnapshotsQueue snapshots_queue1{1};
     keeper_context = get_keeper_context();
-    auto restore_machine = std::make_shared<KeeperStateMachine<DB::KeeperMemoryStorage>>(queue, snapshots_queue1, keeper_context, nullptr);
+    auto restore_machine = std::make_shared<KeeperStateMachine<Storage>>(queue, snapshots_queue1, keeper_context, nullptr);
     restore_machine->init();
     EXPECT_EQ(restore_machine->last_commit_index(), total_logs - total_logs % settings->snapshot_distance);
 
@@ -1863,11 +1923,12 @@ void testLogAndStateMachine(
     }
 }
 
-TEST_P(CoordinationTest, TestStateMachineAndLogStore)
+TYPED_TEST(CoordinationTest, TestStateMachineAndLogStore)
 {
     using namespace Coordination;
     using namespace DB;
-    auto params = GetParam();
+
+    using Storage = typename TestFixture::Storage;
 
     {
         CoordinationSettingsPtr settings = std::make_shared<CoordinationSettings>();
@@ -1875,78 +1936,83 @@ TEST_P(CoordinationTest, TestStateMachineAndLogStore)
         settings->reserved_log_items = 10;
         settings->rotate_log_storage_interval = 10;
 
-        testLogAndStateMachine(settings, 37, params.enable_compression);
+        testLogAndStateMachine<Storage>(settings, 37, this->enable_compression);
     }
     {
         CoordinationSettingsPtr settings = std::make_shared<CoordinationSettings>();
         settings->snapshot_distance = 10;
         settings->reserved_log_items = 10;
         settings->rotate_log_storage_interval = 10;
-        testLogAndStateMachine(settings, 11, params.enable_compression);
+        testLogAndStateMachine<Storage>(settings, 11, this->enable_compression);
     }
     {
         CoordinationSettingsPtr settings = std::make_shared<CoordinationSettings>();
         settings->snapshot_distance = 10;
         settings->reserved_log_items = 10;
         settings->rotate_log_storage_interval = 10;
-        testLogAndStateMachine(settings, 40, params.enable_compression);
+        testLogAndStateMachine<Storage>(settings, 40, this->enable_compression);
     }
     {
         CoordinationSettingsPtr settings = std::make_shared<CoordinationSettings>();
         settings->snapshot_distance = 10;
         settings->reserved_log_items = 20;
         settings->rotate_log_storage_interval = 30;
-        testLogAndStateMachine(settings, 40, params.enable_compression);
+        testLogAndStateMachine<Storage>(settings, 40, this->enable_compression);
     }
     {
         CoordinationSettingsPtr settings = std::make_shared<CoordinationSettings>();
         settings->snapshot_distance = 10;
         settings->reserved_log_items = 0;
         settings->rotate_log_storage_interval = 10;
-        testLogAndStateMachine(settings, 40, params.enable_compression);
+        testLogAndStateMachine<Storage>(settings, 40, this->enable_compression);
     }
     {
         CoordinationSettingsPtr settings = std::make_shared<CoordinationSettings>();
         settings->snapshot_distance = 1;
         settings->reserved_log_items = 1;
         settings->rotate_log_storage_interval = 32;
-        testLogAndStateMachine(settings, 32, params.enable_compression);
+        testLogAndStateMachine<Storage>(settings, 32, this->enable_compression);
     }
     {
         CoordinationSettingsPtr settings = std::make_shared<CoordinationSettings>();
         settings->snapshot_distance = 10;
         settings->reserved_log_items = 7;
         settings->rotate_log_storage_interval = 1;
-        testLogAndStateMachine(settings, 33, params.enable_compression);
+        testLogAndStateMachine<Storage>(settings, 33, this->enable_compression);
     }
     {
         CoordinationSettingsPtr settings = std::make_shared<CoordinationSettings>();
         settings->snapshot_distance = 37;
         settings->reserved_log_items = 1000;
         settings->rotate_log_storage_interval = 5000;
-        testLogAndStateMachine(settings, 33, params.enable_compression);
+        testLogAndStateMachine<Storage>(settings, 33, this->enable_compression);
     }
     {
         CoordinationSettingsPtr settings = std::make_shared<CoordinationSettings>();
         settings->snapshot_distance = 37;
         settings->reserved_log_items = 1000;
         settings->rotate_log_storage_interval = 5000;
-        testLogAndStateMachine(settings, 45, params.enable_compression);
+        testLogAndStateMachine<Storage>(settings, 45, this->enable_compression);
     }
 }
 
-TEST_P(CoordinationTest, TestEphemeralNodeRemove)
+TYPED_TEST(CoordinationTest, TestEphemeralNodeRemove)
 {
     using namespace Coordination;
     using namespace DB;
 
     ChangelogDirTest snapshots("./snapshots");
-    setSnapshotDirectory("./snapshots");
+    this->setSnapshotDirectory("./snapshots");
+
+    using Storage = typename TestFixture::Storage;
+
+    ChangelogDirTest rocks("./rocksdb");
+    this->setRocksDBDirectory("./rocksdb");
 
     ResponsesQueue queue(std::numeric_limits<size_t>::max());
     SnapshotsQueue snapshots_queue{1};
 
-    auto state_machine = std::make_shared<KeeperStateMachine<DB::KeeperMemoryStorage>>(queue, snapshots_queue, keeper_context, nullptr);
+    auto state_machine = std::make_shared<KeeperStateMachine<Storage>>(queue, snapshots_queue, this->keeper_context, nullptr);
     state_machine->init();
 
     std::shared_ptr<ZooKeeperCreateRequest> request_c = std::make_shared<ZooKeeperCreateRequest>();
@@ -1969,17 +2035,23 @@ TEST_P(CoordinationTest, TestEphemeralNodeRemove)
 }
 
 
-TEST_P(CoordinationTest, TestCreateNodeWithAuthSchemeForAclWhenAuthIsPrecommitted)
+TYPED_TEST(CoordinationTest, TestCreateNodeWithAuthSchemeForAclWhenAuthIsPrecommitted)
 {
     using namespace Coordination;
     using namespace DB;
 
     ChangelogDirTest snapshots("./snapshots");
-    setSnapshotDirectory("./snapshots");
+    this->setSnapshotDirectory("./snapshots");
+
+    using Storage = typename TestFixture::Storage;
+
+    ChangelogDirTest rocks("./rocksdb");
+    this->setRocksDBDirectory("./rocksdb");
+
     ResponsesQueue queue(std::numeric_limits<size_t>::max());
     SnapshotsQueue snapshots_queue{1};
 
-    auto state_machine = std::make_shared<KeeperStateMachine<DB::KeeperMemoryStorage>>(queue, snapshots_queue, keeper_context, nullptr);
+    auto state_machine = std::make_shared<KeeperStateMachine<Storage>>(queue, snapshots_queue, this->keeper_context, nullptr);
     state_machine->init();
 
     String user_auth_data = "test_user:test_password";
@@ -2019,18 +2091,18 @@ TEST_P(CoordinationTest, TestCreateNodeWithAuthSchemeForAclWhenAuthIsPrecommitte
     EXPECT_EQ(acls[0].permissions, 31);
 }
 
-TEST_P(CoordinationTest, TestSetACLWithAuthSchemeForAclWhenAuthIsPrecommitted)
+TYPED_TEST(CoordinationTest, TestSetACLWithAuthSchemeForAclWhenAuthIsPrecommitted)
 {
     using namespace Coordination;
     using namespace DB;
 
     ChangelogDirTest snapshots("./snapshots");
-    setSnapshotDirectory("./snapshots");
+    this->setSnapshotDirectory("./snapshots");
 
     ResponsesQueue queue(std::numeric_limits<size_t>::max());
     SnapshotsQueue snapshots_queue{1};
 
-    auto state_machine = std::make_shared<KeeperStateMachine<DB::KeeperMemoryStorage>>(queue, snapshots_queue, keeper_context, nullptr);
+    auto state_machine = std::make_shared<KeeperStateMachine<DB::KeeperMemoryStorage>>(queue, snapshots_queue, this->keeper_context, nullptr);
     state_machine->init();
 
     String user_auth_data = "test_user:test_password";
@@ -2077,17 +2149,17 @@ TEST_P(CoordinationTest, TestSetACLWithAuthSchemeForAclWhenAuthIsPrecommitted)
 }
 
 
-TEST_P(CoordinationTest, TestRotateIntervalChanges)
+TYPED_TEST(CoordinationTest, TestRotateIntervalChanges)
 {
     using namespace Coordination;
-    auto params = GetParam();
+
     ChangelogDirTest snapshots("./logs");
-    setLogDirectory("./logs");
+    this->setLogDirectory("./logs");
     {
         DB::KeeperLogStore changelog(
-            DB::LogFileSettings{.force_sync = true, .compress_logs = params.enable_compression, .rotate_interval = 100},
+            DB::LogFileSettings{.force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 100},
             DB::FlushSettings(),
-            keeper_context);
+            this->keeper_context);
 
         changelog.init(0, 3);
         for (size_t i = 1; i < 55; ++i)
@@ -2103,12 +2175,12 @@ TEST_P(CoordinationTest, TestRotateIntervalChanges)
     }
 
 
-    EXPECT_TRUE(fs::exists("./logs/changelog_1_100.bin" + params.extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_1_100.bin" + this->extension));
 
     DB::KeeperLogStore changelog_1(
-        DB::LogFileSettings{.force_sync = true, .compress_logs = params.enable_compression, .rotate_interval = 10},
+        DB::LogFileSettings{.force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 10},
         DB::FlushSettings(),
-        keeper_context);
+        this->keeper_context);
     changelog_1.init(0, 50);
     for (size_t i = 0; i < 55; ++i)
     {
@@ -2121,13 +2193,13 @@ TEST_P(CoordinationTest, TestRotateIntervalChanges)
 
     waitDurableLogs(changelog_1);
 
-    EXPECT_TRUE(fs::exists("./logs/changelog_1_100.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_101_110.bin" + params.extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_1_100.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_101_110.bin" + this->extension));
 
     DB::KeeperLogStore changelog_2(
-        DB::LogFileSettings{.force_sync = true, .compress_logs = params.enable_compression, .rotate_interval = 7},
+        DB::LogFileSettings{.force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 7},
         DB::FlushSettings(),
-        keeper_context);
+        this->keeper_context);
     changelog_2.init(98, 55);
 
     for (size_t i = 0; i < 17; ++i)
@@ -2141,20 +2213,20 @@ TEST_P(CoordinationTest, TestRotateIntervalChanges)
 
     waitDurableLogs(changelog_2);
 
-    keeper_context->setLastCommitIndex(105);
+    this->keeper_context->setLastCommitIndex(105);
     changelog_2.compact(105);
     std::this_thread::sleep_for(std::chrono::microseconds(1000));
 
-    assertFileDeleted("./logs/changelog_1_100.bin" + params.extension);
-    EXPECT_TRUE(fs::exists("./logs/changelog_101_110.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_111_117.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_118_124.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_125_131.bin" + params.extension));
+    assertFileDeleted("./logs/changelog_1_100.bin" + this->extension);
+    EXPECT_TRUE(fs::exists("./logs/changelog_101_110.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_111_117.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_118_124.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_125_131.bin" + this->extension));
 
     DB::KeeperLogStore changelog_3(
-        DB::LogFileSettings{.force_sync = true, .compress_logs = params.enable_compression, .rotate_interval = 5},
+        DB::LogFileSettings{.force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 5},
         DB::FlushSettings(),
-        keeper_context);
+        this->keeper_context);
     changelog_3.init(116, 3);
     for (size_t i = 0; i < 17; ++i)
     {
@@ -2167,20 +2239,20 @@ TEST_P(CoordinationTest, TestRotateIntervalChanges)
 
     waitDurableLogs(changelog_3);
 
-    keeper_context->setLastCommitIndex(125);
+    this->keeper_context->setLastCommitIndex(125);
     changelog_3.compact(125);
     std::this_thread::sleep_for(std::chrono::microseconds(1000));
-    assertFileDeleted("./logs/changelog_101_110.bin" + params.extension);
-    assertFileDeleted("./logs/changelog_111_117.bin" + params.extension);
-    assertFileDeleted("./logs/changelog_118_124.bin" + params.extension);
+    assertFileDeleted("./logs/changelog_101_110.bin" + this->extension);
+    assertFileDeleted("./logs/changelog_111_117.bin" + this->extension);
+    assertFileDeleted("./logs/changelog_118_124.bin" + this->extension);
 
-    EXPECT_TRUE(fs::exists("./logs/changelog_125_131.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_132_136.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_137_141.bin" + params.extension));
-    EXPECT_TRUE(fs::exists("./logs/changelog_142_146.bin" + params.extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_125_131.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_132_136.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_137_141.bin" + this->extension));
+    EXPECT_TRUE(fs::exists("./logs/changelog_142_146.bin" + this->extension));
 }
 
-TEST_P(CoordinationTest, TestSessionExpiryQueue)
+TYPED_TEST(CoordinationTest, TestSessionExpiryQueue)
 {
     using namespace Coordination;
     SessionExpiryQueue queue(500);
@@ -2198,16 +2270,15 @@ TEST_P(CoordinationTest, TestSessionExpiryQueue)
 }
 
 
-TEST_P(CoordinationTest, TestCompressedLogsMultipleRewrite)
+TYPED_TEST(CoordinationTest, TestCompressedLogsMultipleRewrite)
 {
     using namespace Coordination;
-    auto test_params = GetParam();
     ChangelogDirTest logs("./logs");
-    setLogDirectory("./logs");
+    this->setLogDirectory("./logs");
     DB::KeeperLogStore changelog(
-        DB::LogFileSettings{.force_sync = true, .compress_logs = test_params.enable_compression, .rotate_interval = 100},
+        DB::LogFileSettings{.force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 100},
         DB::FlushSettings(),
-        keeper_context);
+        this->keeper_context);
 
     changelog.init(0, 3);
     for (size_t i = 1; i < 55; ++i)
@@ -2222,9 +2293,9 @@ TEST_P(CoordinationTest, TestCompressedLogsMultipleRewrite)
     waitDurableLogs(changelog);
 
     DB::KeeperLogStore changelog1(
-        DB::LogFileSettings{.force_sync = true, .compress_logs = test_params.enable_compression, .rotate_interval = 100},
+        DB::LogFileSettings{.force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 100},
         DB::FlushSettings(),
-        keeper_context);
+        this->keeper_context);
     changelog1.init(0, 3);
     for (size_t i = 55; i < 70; ++i)
     {
@@ -2238,9 +2309,9 @@ TEST_P(CoordinationTest, TestCompressedLogsMultipleRewrite)
     waitDurableLogs(changelog1);
 
     DB::KeeperLogStore changelog2(
-        DB::LogFileSettings{.force_sync = true, .compress_logs = test_params.enable_compression, .rotate_interval = 100},
+        DB::LogFileSettings{.force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 100},
         DB::FlushSettings(),
-        keeper_context);
+        this->keeper_context);
     changelog2.init(0, 3);
     for (size_t i = 70; i < 80; ++i)
     {
@@ -2252,16 +2323,21 @@ TEST_P(CoordinationTest, TestCompressedLogsMultipleRewrite)
     }
 }
 
-TEST_P(CoordinationTest, TestStorageSnapshotDifferentCompressions)
+TYPED_TEST(CoordinationTest, TestStorageSnapshotDifferentCompressions)
 {
-    auto params = GetParam();
+
 
     ChangelogDirTest test("./snapshots");
-    setSnapshotDirectory("./snapshots");
+    this->setSnapshotDirectory("./snapshots");
 
-    DB::KeeperSnapshotManager<DB::KeeperMemoryStorage> manager(3, keeper_context, params.enable_compression);
+    using Storage = typename TestFixture::Storage;
 
-    DB::KeeperMemoryStorage storage(500, "", keeper_context);
+    ChangelogDirTest rocks("./rocksdb");
+    this->setRocksDBDirectory("./rocksdb");
+
+    DB::KeeperSnapshotManager<Storage> manager(3, this->keeper_context, this->enable_compression);
+
+    Storage storage(500, "", this->keeper_context);
     addNode(storage, "/hello1", "world", 1);
     addNode(storage, "/hello2", "somedata", 3);
     storage.session_id_counter = 5;
@@ -2271,13 +2347,13 @@ TEST_P(CoordinationTest, TestStorageSnapshotDifferentCompressions)
     storage.getSessionID(130);
     storage.getSessionID(130);
 
-    DB::KeeperStorageSnapshot<DB::KeeperMemoryStorage> snapshot(&storage, 2);
+    DB::KeeperStorageSnapshot<Storage> snapshot(&storage, 2);
 
     auto buf = manager.serializeSnapshotToBuffer(snapshot);
     manager.serializeSnapshotBufferToDisk(*buf, 2);
-    EXPECT_TRUE(fs::exists("./snapshots/snapshot_2.bin" + params.extension));
+    EXPECT_TRUE(fs::exists("./snapshots/snapshot_2.bin" + this->extension));
 
-    DB::KeeperSnapshotManager<DB::KeeperMemoryStorage> new_manager(3, keeper_context, !params.enable_compression);
+    DB::KeeperSnapshotManager<Storage> new_manager(3, this->keeper_context, !this->enable_compression);
 
     auto debuf = new_manager.deserializeSnapshotBufferFromDisk(2);
 
@@ -2299,17 +2375,17 @@ TEST_P(CoordinationTest, TestStorageSnapshotDifferentCompressions)
     EXPECT_EQ(restored_storage->session_and_timeout.size(), 2);
 }
 
-TEST_P(CoordinationTest, ChangelogInsertThreeTimesSmooth)
+TYPED_TEST(CoordinationTest, ChangelogInsertThreeTimesSmooth)
 {
-    auto params = GetParam();
+
     ChangelogDirTest test("./logs");
-    setLogDirectory("./logs");
+    this->setLogDirectory("./logs");
     {
-        LOG_INFO(log, "================First time=====================");
+        LOG_INFO(this->log, "================First time=====================");
         DB::KeeperLogStore changelog(
-            DB::LogFileSettings{.force_sync = true, .compress_logs = params.enable_compression, .rotate_interval = 100},
+            DB::LogFileSettings{.force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 100},
             DB::FlushSettings(),
-            keeper_context);
+            this->keeper_context);
         changelog.init(1, 0);
         auto entry = getLogEntry("hello_world", 1000);
         changelog.append(entry);
@@ -2319,11 +2395,11 @@ TEST_P(CoordinationTest, ChangelogInsertThreeTimesSmooth)
     }
 
     {
-        LOG_INFO(log, "================Second time=====================");
+        LOG_INFO(this->log, "================Second time=====================");
         DB::KeeperLogStore changelog(
-            DB::LogFileSettings{.force_sync = true, .compress_logs = params.enable_compression, .rotate_interval = 100},
+            DB::LogFileSettings{.force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 100},
             DB::FlushSettings(),
-            keeper_context);
+            this->keeper_context);
         changelog.init(1, 0);
         auto entry = getLogEntry("hello_world", 1000);
         changelog.append(entry);
@@ -2333,11 +2409,11 @@ TEST_P(CoordinationTest, ChangelogInsertThreeTimesSmooth)
     }
 
     {
-        LOG_INFO(log, "================Third time=====================");
+        LOG_INFO(this->log, "================Third time=====================");
         DB::KeeperLogStore changelog(
-            DB::LogFileSettings{.force_sync = true, .compress_logs = params.enable_compression, .rotate_interval = 100},
+            DB::LogFileSettings{.force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 100},
             DB::FlushSettings(),
-            keeper_context);
+            this->keeper_context);
         changelog.init(1, 0);
         auto entry = getLogEntry("hello_world", 1000);
         changelog.append(entry);
@@ -2347,11 +2423,11 @@ TEST_P(CoordinationTest, ChangelogInsertThreeTimesSmooth)
     }
 
     {
-        LOG_INFO(log, "================Fourth time=====================");
+        LOG_INFO(this->log, "================Fourth time=====================");
         DB::KeeperLogStore changelog(
-            DB::LogFileSettings{.force_sync = true, .compress_logs = params.enable_compression, .rotate_interval = 100},
+            DB::LogFileSettings{.force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 100},
             DB::FlushSettings(),
-            keeper_context);
+            this->keeper_context);
         changelog.init(1, 0);
         auto entry = getLogEntry("hello_world", 1000);
         changelog.append(entry);
@@ -2362,18 +2438,18 @@ TEST_P(CoordinationTest, ChangelogInsertThreeTimesSmooth)
 }
 
 
-TEST_P(CoordinationTest, ChangelogInsertMultipleTimesSmooth)
+TYPED_TEST(CoordinationTest, ChangelogInsertMultipleTimesSmooth)
 {
-    auto params = GetParam();
+
     ChangelogDirTest test("./logs");
-    setLogDirectory("./logs");
+    this->setLogDirectory("./logs");
     for (size_t i = 0; i < 36; ++i)
     {
-        LOG_INFO(log, "================First time=====================");
+        LOG_INFO(this->log, "================First time=====================");
         DB::KeeperLogStore changelog(
-            DB::LogFileSettings{.force_sync = true, .compress_logs = params.enable_compression, .rotate_interval = 100},
+            DB::LogFileSettings{.force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 100},
             DB::FlushSettings(),
-            keeper_context);
+            this->keeper_context);
         changelog.init(1, 0);
         for (size_t j = 0; j < 7; ++j)
         {
@@ -2385,24 +2461,24 @@ TEST_P(CoordinationTest, ChangelogInsertMultipleTimesSmooth)
     }
 
     DB::KeeperLogStore changelog(
-        DB::LogFileSettings{.force_sync = true, .compress_logs = params.enable_compression, .rotate_interval = 100},
+        DB::LogFileSettings{.force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 100},
         DB::FlushSettings(),
-        keeper_context);
+        this->keeper_context);
     changelog.init(1, 0);
     EXPECT_EQ(changelog.next_slot(), 36 * 7 + 1);
 }
 
-TEST_P(CoordinationTest, ChangelogInsertThreeTimesHard)
+TYPED_TEST(CoordinationTest, ChangelogInsertThreeTimesHard)
 {
-    auto params = GetParam();
+
     ChangelogDirTest test("./logs");
-    setLogDirectory("./logs");
+    this->setLogDirectory("./logs");
     {
-        LOG_INFO(log, "================First time=====================");
+        LOG_INFO(this->log, "================First time=====================");
         DB::KeeperLogStore changelog1(
-            DB::LogFileSettings{.force_sync = true, .compress_logs = params.enable_compression, .rotate_interval = 100},
+            DB::LogFileSettings{.force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 100},
             DB::FlushSettings(),
-            keeper_context);
+            this->keeper_context);
         changelog1.init(1, 0);
         auto entry = getLogEntry("hello_world", 1000);
         changelog1.append(entry);
@@ -2412,11 +2488,11 @@ TEST_P(CoordinationTest, ChangelogInsertThreeTimesHard)
     }
 
     {
-        LOG_INFO(log, "================Second time=====================");
+        LOG_INFO(this->log, "================Second time=====================");
         DB::KeeperLogStore changelog2(
-            DB::LogFileSettings{.force_sync = true, .compress_logs = params.enable_compression, .rotate_interval = 100},
+            DB::LogFileSettings{.force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 100},
             DB::FlushSettings(),
-            keeper_context);
+            this->keeper_context);
         changelog2.init(1, 0);
         auto entry = getLogEntry("hello_world", 1000);
         changelog2.append(entry);
@@ -2426,11 +2502,11 @@ TEST_P(CoordinationTest, ChangelogInsertThreeTimesHard)
     }
 
     {
-        LOG_INFO(log, "================Third time=====================");
+        LOG_INFO(this->log, "================Third time=====================");
         DB::KeeperLogStore changelog3(
-            DB::LogFileSettings{.force_sync = true, .compress_logs = params.enable_compression, .rotate_interval = 100},
+            DB::LogFileSettings{.force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 100},
             DB::FlushSettings(),
-            keeper_context);
+            this->keeper_context);
         changelog3.init(1, 0);
         auto entry = getLogEntry("hello_world", 1000);
         changelog3.append(entry);
@@ -2440,11 +2516,11 @@ TEST_P(CoordinationTest, ChangelogInsertThreeTimesHard)
     }
 
     {
-        LOG_INFO(log, "================Fourth time=====================");
+        LOG_INFO(this->log, "================Fourth time=====================");
         DB::KeeperLogStore changelog4(
-            DB::LogFileSettings{.force_sync = true, .compress_logs = params.enable_compression, .rotate_interval = 100},
+            DB::LogFileSettings{.force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 100},
             DB::FlushSettings(),
-            keeper_context);
+            this->keeper_context);
         changelog4.init(1, 0);
         auto entry = getLogEntry("hello_world", 1000);
         changelog4.append(entry);
@@ -2454,18 +2530,23 @@ TEST_P(CoordinationTest, ChangelogInsertThreeTimesHard)
     }
 }
 
-TEST_P(CoordinationTest, TestStorageSnapshotEqual)
+TYPED_TEST(CoordinationTest, TestStorageSnapshotEqual)
 {
-    auto params = GetParam();
+
     ChangelogDirTest test("./snapshots");
-    setSnapshotDirectory("./snapshots");
+    this->setSnapshotDirectory("./snapshots");
+
+    using Storage = typename TestFixture::Storage;
+
+    ChangelogDirTest rocks("./rocksdb");
+    this->setRocksDBDirectory("./rocksdb");
 
     std::optional<UInt128> snapshot_hash;
     for (size_t i = 0; i < 15; ++i)
     {
-        DB::KeeperSnapshotManager<DB::KeeperMemoryStorage> manager(3, keeper_context, params.enable_compression);
+        DB::KeeperSnapshotManager<Storage> manager(3, this->keeper_context, this->enable_compression);
 
-        DB::KeeperMemoryStorage storage(500, "", keeper_context);
+        Storage storage(500, "", this->keeper_context);
         addNode(storage, "/hello", "");
         for (size_t j = 0; j < 5000; ++j)
         {
@@ -2481,7 +2562,7 @@ TEST_P(CoordinationTest, TestStorageSnapshotEqual)
         for (size_t j = 0; j < 3333; ++j)
             storage.getSessionID(130 * j);
 
-        DB::KeeperStorageSnapshot<DB::KeeperMemoryStorage> snapshot(&storage, storage.zxid);
+        DB::KeeperStorageSnapshot<Storage> snapshot(&storage, storage.zxid);
 
         auto buf = manager.serializeSnapshotToBuffer(snapshot);
 
@@ -2498,17 +2579,16 @@ TEST_P(CoordinationTest, TestStorageSnapshotEqual)
 }
 
 
-TEST_P(CoordinationTest, TestLogGap)
+TYPED_TEST(CoordinationTest, TestLogGap)
 {
     using namespace Coordination;
-    auto test_params = GetParam();
     ChangelogDirTest logs("./logs");
-    setLogDirectory("./logs");
+    this->setLogDirectory("./logs");
 
     DB::KeeperLogStore changelog(
-        DB::LogFileSettings{.force_sync = true, .compress_logs = test_params.enable_compression, .rotate_interval = 100},
+        DB::LogFileSettings{.force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 100},
         DB::FlushSettings(),
-        keeper_context);
+        this->keeper_context);
 
     changelog.init(0, 3);
     for (size_t i = 1; i < 55; ++i)
@@ -2521,13 +2601,13 @@ TEST_P(CoordinationTest, TestLogGap)
     }
 
     DB::KeeperLogStore changelog1(
-        DB::LogFileSettings{.force_sync = true, .compress_logs = test_params.enable_compression, .rotate_interval = 100},
+        DB::LogFileSettings{.force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 100},
         DB::FlushSettings(),
-        keeper_context);
+        this->keeper_context);
     changelog1.init(61, 3);
 
     /// Logs discarded
-    EXPECT_FALSE(fs::exists("./logs/changelog_1_100.bin" + test_params.extension));
+    EXPECT_FALSE(fs::exists("./logs/changelog_1_100.bin" + this->extension));
     EXPECT_EQ(changelog1.start_index(), 61);
     EXPECT_EQ(changelog1.next_slot(), 61);
 }
@@ -2539,12 +2619,17 @@ ResponseType getSingleResponse(const auto & responses)
     return dynamic_cast<ResponseType &>(*responses[0].response);
 }
 
-TEST_P(CoordinationTest, TestUncommittedStateBasicCrud)
+TYPED_TEST(CoordinationTest, TestUncommittedStateBasicCrud)
 {
     using namespace DB;
     using namespace Coordination;
 
-    DB::KeeperMemoryStorage storage{500, "", keeper_context};
+    using Storage = typename TestFixture::Storage;
+
+    ChangelogDirTest rocks("./rocksdb");
+    this->setRocksDBDirectory("./rocksdb");
+
+    Storage storage{500, "", this->keeper_context};
 
     constexpr std::string_view path = "/test";
 
@@ -2656,12 +2741,17 @@ TEST_P(CoordinationTest, TestUncommittedStateBasicCrud)
     ASSERT_FALSE(get_committed_data());
 }
 
-TEST_P(CoordinationTest, TestListRequestTypes)
+TYPED_TEST(CoordinationTest, TestListRequestTypes)
 {
     using namespace DB;
     using namespace Coordination;
 
-    KeeperMemoryStorage storage{500, "", keeper_context};
+    using Storage = typename TestFixture::Storage;
+
+    ChangelogDirTest rocks("./rocksdb");
+    this->setRocksDBDirectory("./rocksdb");
+
+    Storage storage{500, "", this->keeper_context};
 
     int32_t zxid = 0;
 
@@ -2738,18 +2828,18 @@ TEST_P(CoordinationTest, TestListRequestTypes)
     }
 }
 
-TEST_P(CoordinationTest, TestDurableState)
+TYPED_TEST(CoordinationTest, TestDurableState)
 {
     ChangelogDirTest logs("./logs");
-    setLogDirectory("./logs");
-    setStateFileDirectory(".");
+    this->setLogDirectory("./logs");
+    this->setStateFileDirectory(".");
 
     auto state = nuraft::cs_new<nuraft::srv_state>();
     std::optional<DB::KeeperStateManager> state_manager;
 
     const auto reload_state_manager = [&]
     {
-        state_manager.emplace(1, "localhost", 9181, keeper_context);
+        state_manager.emplace(1, "localhost", 9181, this->keeper_context);
         state_manager->loadLogStore(1, 0);
     };
 
@@ -2812,10 +2902,15 @@ TEST_P(CoordinationTest, TestDurableState)
     }
 }
 
-TEST_P(CoordinationTest, TestFeatureFlags)
+TYPED_TEST(CoordinationTest, TestFeatureFlags)
 {
     using namespace Coordination;
-    KeeperMemoryStorage storage{500, "", keeper_context};
+    using Storage = typename TestFixture::Storage;
+
+    ChangelogDirTest rocks("./rocksdb");
+    this->setRocksDBDirectory("./rocksdb");
+
+    Storage storage{500, "", this->keeper_context};
     auto request = std::make_shared<ZooKeeperGetRequest>();
     request->path = DB::keeper_api_feature_flags_path;
     auto responses = storage.processRequest(request, 0, std::nullopt, true, true);
@@ -2827,14 +2922,19 @@ TEST_P(CoordinationTest, TestFeatureFlags)
     ASSERT_FALSE(feature_flags.isEnabled(KeeperFeatureFlag::CHECK_NOT_EXISTS));
 }
 
-TEST_P(CoordinationTest, TestSystemNodeModify)
+TYPED_TEST(CoordinationTest, TestSystemNodeModify)
 {
     using namespace Coordination;
     int64_t zxid{0};
 
+    using Storage = typename TestFixture::Storage;
+
+    ChangelogDirTest rocks("./rocksdb");
+    this->setRocksDBDirectory("./rocksdb");
+
     // On INIT we abort when a system path is modified
-    keeper_context->setServerState(KeeperContext::Phase::RUNNING);
-    KeeperMemoryStorage storage{500, "", keeper_context};
+    this->keeper_context->setServerState(KeeperContext::Phase::RUNNING);
+    Storage storage{500, "", this->keeper_context};
     const auto assert_create = [&](const std::string_view path, const auto expected_code)
     {
         auto request = std::make_shared<ZooKeeperCreateRequest>();
@@ -2859,11 +2959,11 @@ TEST_P(CoordinationTest, TestSystemNodeModify)
     assert_create("/keeper1/test", Error::ZOK);
 }
 
-TEST_P(CoordinationTest, ChangelogTestMaxLogSize)
+TYPED_TEST(CoordinationTest, ChangelogTestMaxLogSize)
 {
-    auto params = GetParam();
+
     ChangelogDirTest test("./logs");
-    setLogDirectory("./logs");
+    this->setLogDirectory("./logs");
 
     uint64_t last_entry_index{0};
     size_t i{0};
@@ -2871,9 +2971,9 @@ TEST_P(CoordinationTest, ChangelogTestMaxLogSize)
         SCOPED_TRACE("Small rotation interval, big size limit");
         DB::KeeperLogStore changelog(
             DB::LogFileSettings{
-                .force_sync = true, .compress_logs = params.enable_compression, .rotate_interval = 20, .max_size = 50 * 1024 * 1024},
+                .force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 20, .max_size = 50 * 1024 * 1024},
             DB::FlushSettings(),
-            keeper_context);
+            this->keeper_context);
         changelog.init(1, 0);
 
         for (; i < 100; ++i)
@@ -2891,9 +2991,9 @@ TEST_P(CoordinationTest, ChangelogTestMaxLogSize)
         SCOPED_TRACE("Large rotation interval, small size limit");
         DB::KeeperLogStore changelog(
             DB::LogFileSettings{
-                .force_sync = true, .compress_logs = params.enable_compression, .rotate_interval = 100'000, .max_size = 4000},
+                .force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 100'000, .max_size = 4000},
             DB::FlushSettings(),
-            keeper_context);
+            this->keeper_context);
         changelog.init(1, 0);
 
         ASSERT_EQ(changelog.entry_at(last_entry_index)->get_term(), (i - 1 + 44) * 10);
@@ -2913,20 +3013,25 @@ TEST_P(CoordinationTest, ChangelogTestMaxLogSize)
         SCOPED_TRACE("Final verify all logs");
         DB::KeeperLogStore changelog(
             DB::LogFileSettings{
-                .force_sync = true, .compress_logs = params.enable_compression, .rotate_interval = 100'000, .max_size = 4000},
+                .force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 100'000, .max_size = 4000},
             DB::FlushSettings(),
-            keeper_context);
+            this->keeper_context);
         changelog.init(1, 0);
         ASSERT_EQ(changelog.entry_at(last_entry_index)->get_term(), (i - 1 + 44) * 10);
     }
 }
 
-TEST_P(CoordinationTest, TestCheckNotExistsRequest)
+TYPED_TEST(CoordinationTest, TestCheckNotExistsRequest)
 {
     using namespace DB;
     using namespace Coordination;
 
-    KeeperMemoryStorage storage{500, "", keeper_context};
+    using Storage = typename TestFixture::Storage;
+
+    ChangelogDirTest rocks("./rocksdb");
+    this->setRocksDBDirectory("./rocksdb");
+
+    Storage storage{500, "", this->keeper_context};
 
     int32_t zxid = 0;
 
@@ -2994,18 +3099,23 @@ TEST_P(CoordinationTest, TestCheckNotExistsRequest)
     }
 }
 
-TEST_P(CoordinationTest, TestReapplyingDeltas)
+TYPED_TEST(CoordinationTest, TestReapplyingDeltas)
 {
     using namespace DB;
     using namespace Coordination;
 
+    using Storage = typename TestFixture::Storage;
+
+    ChangelogDirTest rocks("./rocksdb");
+    this->setRocksDBDirectory("./rocksdb");
+
     static constexpr int64_t initial_zxid = 100;
 
     const auto create_request = std::make_shared<ZooKeeperCreateRequest>();
     create_request->path = "/test/data";
     create_request->is_sequential = true;
 
-    const auto process_create = [](KeeperMemoryStorage & storage, const auto & request, int64_t zxid)
+    const auto process_create = [](Storage & storage, const auto & request, int64_t zxid)
     {
         storage.preprocessRequest(request, 1, 0, zxid);
         auto responses = storage.processRequest(request, 1, zxid);
@@ -3026,19 +3136,19 @@ TEST_P(CoordinationTest, TestReapplyingDeltas)
             process_create(storage, create_request, zxid);
     };
 
-    KeeperMemoryStorage storage1{500, "", keeper_context};
+    Storage storage1{500, "", this->keeper_context};
     commit_initial_data(storage1);
 
     for (int64_t zxid = initial_zxid + 1; zxid < initial_zxid + 50; ++zxid)
         storage1.preprocessRequest(create_request, 1, 0, zxid, /*check_acl=*/true, /*digest=*/std::nullopt, /*log_idx=*/zxid);
 
     /// create identical new storage
-    KeeperMemoryStorage storage2{500, "", keeper_context};
+    Storage storage2{500, "", this->keeper_context};
     commit_initial_data(storage2);
 
     storage1.applyUncommittedState(storage2, initial_zxid);
 
-    const auto commit_unprocessed = [&](KeeperMemoryStorage & storage)
+    const auto commit_unprocessed = [&](Storage & storage)
     {
         for (int64_t zxid = initial_zxid + 1; zxid < initial_zxid + 50; ++zxid)
         {
@@ -3051,7 +3161,7 @@ TEST_P(CoordinationTest, TestReapplyingDeltas)
     commit_unprocessed(storage1);
     commit_unprocessed(storage2);
 
-    const auto get_children = [&](KeeperMemoryStorage & storage)
+    const auto get_children = [&](Storage & storage)
     {
         const auto list_request = std::make_shared<ZooKeeperListRequest>();
         list_request->path = "/test";
@@ -3071,8 +3181,8 @@ TEST_P(CoordinationTest, TestReapplyingDeltas)
     ASSERT_TRUE(children1_set == children2_set);
 }
 
-INSTANTIATE_TEST_SUITE_P(CoordinationTestSuite,
-    CoordinationTest,
-    ::testing::ValuesIn(std::initializer_list<CompressionParam>{CompressionParam{true, ".zstd"}, CompressionParam{false, ""}}));
+/// INSTANTIATE_TEST_SUITE_P(CoordinationTestSuite,
+///     CoordinationTest,
+///     ::testing::ValuesIn(std::initializer_list<CompressionParam>{CompressionParam{true, ".zstd"}, CompressionParam{false, ""}}));
 
 #endif
diff --git a/src/Coordination/tests/gtest_rocks_keeper.cpp b/src/Coordination/tests/gtest_rocks_keeper.cpp
deleted file mode 100644
index d42eb66d683..00000000000
--- a/src/Coordination/tests/gtest_rocks_keeper.cpp
+++ /dev/null
@@ -1,1150 +0,0 @@
-#include <gtest/gtest.h>
-#include "config.h"
-
-#if USE_NURAFT and USE_ROCKSDB
-
-#include <Coordination/KeeperContext.h>
-#include <Coordination/KeeperCommon.h>
-#include <Coordination/KeeperLogStore.h>
-#include <Coordination/KeeperStorage.h>
-#include <Coordination/KeeperStateMachine.h>
-#include <Coordination/KeeperSnapshotManager.h>
-#include <Coordination/WriteBufferFromNuraftBuffer.h>
-#include <Common/Logger.h>
-#include <Disks/DiskLocal.h>
-#include <Poco/ConsoleChannel.h>
-
-namespace fs = std::filesystem;
-struct ChangelogDirTest
-{
-    std::string path;
-    bool drop;
-    explicit ChangelogDirTest(std::string path_, bool drop_ = true) : path(path_), drop(drop_)
-    {
-        EXPECT_FALSE(fs::exists(path)) << "Path " << path << " already exists, remove it to run test";
-        fs::create_directory(path);
-    }
-
-    ~ChangelogDirTest()
-    {
-        if (fs::exists(path) && drop)
-            fs::remove_all(path);
-    }
-};
-
-class RocksKeeperTest : public ::testing::Test
-{
-protected:
-    DB::KeeperContextPtr keeper_context;
-    LoggerPtr log{getLogger("RocksKeeperTest")};
-
-    void SetUp() override
-    {
-        Poco::AutoPtr<Poco::ConsoleChannel> channel(new Poco::ConsoleChannel(std::cerr));
-        Poco::Logger::root().setChannel(channel);
-        Poco::Logger::root().setLevel("trace");
-
-        auto settings = std::make_shared<DB::CoordinationSettings>();
-        settings->use_rocksdb = true;
-        keeper_context = std::make_shared<DB::KeeperContext>(true, settings);
-        keeper_context->setLocalLogsPreprocessed();
-        keeper_context->setRocksDBOptions(nullptr);
-    }
-
-    void setSnapshotDirectory(const std::string & path)
-    {
-        keeper_context->setSnapshotDisk(std::make_shared<DB::DiskLocal>("SnapshotDisk", path));
-    }
-
-    void setRocksDBDirectory(const std::string & path)
-    {
-        keeper_context->setRocksDBDisk(std::make_shared<DB::DiskLocal>("RocksDisk", path));
-    }
-};
-
-void addNode(DB::KeeperRocksStorage & storage, const std::string & path, const std::string & data, int64_t ephemeral_owner = 0)
-{
-    using Node = DB::KeeperRocksStorage::Node;
-    Node node{};
-    node.setData(data);
-    node.setEphemeralOwner(ephemeral_owner);
-    storage.container.insertOrReplace(path, node);
-    storage.container.updateValue(
-        DB::parentNodePath(StringRef{path}),
-        [&](auto & parent)
-        {
-            parent.increaseNumChildren();
-        });
-}
-
-namespace
-{
-void waitDurableLogs(nuraft::log_store & log_store)
-{
-    while (log_store.last_durable_index() != log_store.next_slot() - 1)
-        std::this_thread::sleep_for(std::chrono::milliseconds(200));
-}
-}
-
-TEST_F(RocksKeeperTest, TestStorageSnapshotSimple)
-{
-    ChangelogDirTest test("./snapshots");
-    ChangelogDirTest rocks("./rocksdb");
-    setSnapshotDirectory("./snapshots");
-    setRocksDBDirectory("./rocksdb");
-
-    DB::KeeperSnapshotManager<DB::KeeperRocksStorage> manager(3, keeper_context);
-
-    DB::KeeperRocksStorage storage(500, "", keeper_context);
-    addNode(storage, "/hello1", "world", 1);
-    addNode(storage, "/hello2", "somedata", 3);
-    storage.session_id_counter = 5;
-    storage.zxid = 2;
-    storage.ephemerals[3] = {"/hello2"};
-    storage.ephemerals[1] = {"/hello1"};
-    storage.getSessionID(130);
-    storage.getSessionID(130);
-
-    DB::KeeperStorageSnapshot<DB::KeeperRocksStorage> snapshot(&storage, 2);
-
-    EXPECT_EQ(snapshot.snapshot_meta->get_last_log_idx(), 2);
-    EXPECT_EQ(snapshot.session_id, 7);
-    EXPECT_EQ(snapshot.snapshot_container_size, 6);
-    EXPECT_EQ(snapshot.session_and_timeout.size(), 2);
-
-    auto buf = manager.serializeSnapshotToBuffer(snapshot);
-    manager.serializeSnapshotBufferToDisk(*buf, 2);
-    EXPECT_TRUE(fs::exists("./snapshots/snapshot_2.bin.zstd"));
-
-
-    auto debuf = manager.deserializeSnapshotBufferFromDisk(2);
-
-    auto [restored_storage, snapshot_meta, _] = manager.deserializeSnapshotFromBuffer(debuf);
-
-    EXPECT_EQ(restored_storage->container.size(), 6);
-    EXPECT_EQ(restored_storage->container.find("/")->value.numChildren(), 3);
-    EXPECT_EQ(restored_storage->container.find("/hello1")->value.numChildren(), 0);
-    EXPECT_EQ(restored_storage->container.find("/hello2")->value.numChildren(), 0);
-
-    EXPECT_EQ(restored_storage->container.find("/")->value.getData(), "");
-    EXPECT_EQ(restored_storage->container.find("/hello1")->value.getData(), "world");
-    EXPECT_EQ(restored_storage->container.find("/hello2")->value.getData(), "somedata");
-    EXPECT_EQ(restored_storage->session_id_counter, 7);
-    EXPECT_EQ(restored_storage->zxid, 2);
-    EXPECT_EQ(restored_storage->ephemerals.size(), 2);
-    EXPECT_EQ(restored_storage->ephemerals[3].size(), 1);
-    EXPECT_EQ(restored_storage->ephemerals[1].size(), 1);
-    EXPECT_EQ(restored_storage->session_and_timeout.size(), 2);
-}
-
-TEST_F(RocksKeeperTest, TestStorageSnapshotMoreWrites)
-{
-    ChangelogDirTest test("./snapshots");
-    ChangelogDirTest rocks("./rocksdb");
-    setSnapshotDirectory("./snapshots");
-    setRocksDBDirectory("./rocksdb");
-
-    DB::KeeperSnapshotManager<DB::KeeperRocksStorage> manager(3, keeper_context);
-
-    DB::KeeperRocksStorage storage(500, "", keeper_context);
-    storage.getSessionID(130);
-
-    for (size_t i = 0; i < 50; ++i)
-    {
-        addNode(storage, "/hello_" + std::to_string(i), "world_" + std::to_string(i));
-    }
-
-    DB::KeeperStorageSnapshot<DB::KeeperRocksStorage> snapshot(&storage, 50);
-    EXPECT_EQ(snapshot.snapshot_meta->get_last_log_idx(), 50);
-    EXPECT_EQ(snapshot.snapshot_container_size, 54);
-
-    for (size_t i = 50; i < 100; ++i)
-    {
-        addNode(storage, "/hello_" + std::to_string(i), "world_" + std::to_string(i));
-    }
-
-    EXPECT_EQ(storage.container.size(), 104);
-
-    auto buf = manager.serializeSnapshotToBuffer(snapshot);
-    manager.serializeSnapshotBufferToDisk(*buf, 50);
-    EXPECT_TRUE(fs::exists("./snapshots/snapshot_50.bin.zstd"));
-
-
-    auto debuf = manager.deserializeSnapshotBufferFromDisk(50);
-    auto [restored_storage, meta, _] = manager.deserializeSnapshotFromBuffer(debuf);
-
-    EXPECT_EQ(restored_storage->container.size(), 54);
-    for (size_t i = 0; i < 50; ++i)
-    {
-        EXPECT_EQ(restored_storage->container.find("/hello_" + std::to_string(i))->value.getData(), "world_" + std::to_string(i));
-    }
-}
-
-TEST_F(RocksKeeperTest, TestStorageSnapshotManySnapshots)
-{
-    ChangelogDirTest test("./snapshots");
-    ChangelogDirTest rocks("./rocksdb");
-    setSnapshotDirectory("./snapshots");
-    setRocksDBDirectory("./rocksdb");
-
-    DB::KeeperSnapshotManager<DB::KeeperRocksStorage> manager(3, keeper_context);
-
-    DB::KeeperRocksStorage storage(500, "", keeper_context);
-    storage.getSessionID(130);
-
-    for (size_t j = 1; j <= 5; ++j)
-    {
-        for (size_t i = (j - 1) * 50; i < j * 50; ++i)
-        {
-            addNode(storage, "/hello_" + std::to_string(i), "world_" + std::to_string(i));
-        }
-
-        DB::KeeperStorageSnapshot<DB::KeeperRocksStorage> snapshot(&storage, j * 50);
-        auto buf = manager.serializeSnapshotToBuffer(snapshot);
-        manager.serializeSnapshotBufferToDisk(*buf, j * 50);
-        EXPECT_TRUE(fs::exists(std::string{"./snapshots/snapshot_"} + std::to_string(j * 50) + ".bin.zstd"));
-    }
-
-    EXPECT_FALSE(fs::exists("./snapshots/snapshot_50.bin.zstd"));
-    EXPECT_FALSE(fs::exists("./snapshots/snapshot_100.bin.zstd"));
-    EXPECT_TRUE(fs::exists("./snapshots/snapshot_150.bin.zstd"));
-    EXPECT_TRUE(fs::exists("./snapshots/snapshot_200.bin.zstd"));
-    EXPECT_TRUE(fs::exists("./snapshots/snapshot_250.bin.zstd"));
-
-
-    auto [restored_storage, meta, _] = manager.restoreFromLatestSnapshot();
-
-    EXPECT_EQ(restored_storage->container.size(), 254);
-
-    for (size_t i = 0; i < 250; ++i)
-    {
-        EXPECT_EQ(restored_storage->container.find("/hello_" + std::to_string(i))->value.getData(), "world_" + std::to_string(i));
-    }
-}
-
-TEST_F(RocksKeeperTest, TestStorageSnapshotMode)
-{
-    ChangelogDirTest test("./snapshots");
-    setSnapshotDirectory("./snapshots");
-    ChangelogDirTest rocks("./rocksdb");
-    setRocksDBDirectory("./rocksdb");
-
-    DB::KeeperSnapshotManager<DB::KeeperRocksStorage> manager(3, keeper_context);
-    DB::KeeperRocksStorage storage(500, "", keeper_context);
-    for (size_t i = 0; i < 50; ++i)
-    {
-        addNode(storage, "/hello_" + std::to_string(i), "world_" + std::to_string(i));
-    }
-
-    {
-        DB::KeeperStorageSnapshot<DB::KeeperRocksStorage> snapshot(&storage, 50);
-        for (size_t i = 0; i < 50; ++i)
-        {
-            addNode(storage, "/hello_" + std::to_string(i), "wlrd_" + std::to_string(i));
-        }
-        for (size_t i = 0; i < 50; ++i)
-        {
-            EXPECT_EQ(storage.container.find("/hello_" + std::to_string(i))->value.getData(), "wlrd_" + std::to_string(i));
-        }
-        for (size_t i = 0; i < 50; ++i)
-        {
-            if (i % 2 == 0)
-                storage.container.erase("/hello_" + std::to_string(i));
-        }
-        EXPECT_EQ(storage.container.size(), 29);
-        EXPECT_EQ(storage.container.snapshotSizeWithVersion().first, 54);
-        EXPECT_EQ(storage.container.snapshotSizeWithVersion().second, 1);
-        auto buf = manager.serializeSnapshotToBuffer(snapshot);
-        manager.serializeSnapshotBufferToDisk(*buf, 50);
-    }
-    EXPECT_TRUE(fs::exists("./snapshots/snapshot_50.bin.zstd"));
-    EXPECT_EQ(storage.container.size(), 29);
-    storage.clearGarbageAfterSnapshot();
-    for (size_t i = 0; i < 50; ++i)
-    {
-        if (i % 2 != 0)
-            EXPECT_EQ(storage.container.find("/hello_" + std::to_string(i))->value.getData(), "wlrd_" + std::to_string(i));
-        else
-            EXPECT_FALSE(storage.container.contains("/hello_" + std::to_string(i)));
-    }
-
-    auto [restored_storage, meta, _] = manager.restoreFromLatestSnapshot();
-
-    for (size_t i = 0; i < 50; ++i)
-    {
-        EXPECT_EQ(restored_storage->container.find("/hello_" + std::to_string(i))->value.getData(), "world_" + std::to_string(i));
-    }
-}
-
-TEST_F(RocksKeeperTest, TestStorageSnapshotBroken)
-{
-    ChangelogDirTest test("./snapshots");
-    setSnapshotDirectory("./snapshots");
-    ChangelogDirTest rocks("./rocksdb");
-    setRocksDBDirectory("./rocksdb");
-
-    DB::KeeperSnapshotManager<DB::KeeperRocksStorage> manager(3, keeper_context);
-    DB::KeeperRocksStorage storage(500, "", keeper_context);
-    for (size_t i = 0; i < 50; ++i)
-    {
-        addNode(storage, "/hello_" + std::to_string(i), "world_" + std::to_string(i));
-    }
-    {
-        DB::KeeperStorageSnapshot<DB::KeeperRocksStorage> snapshot(&storage, 50);
-        auto buf = manager.serializeSnapshotToBuffer(snapshot);
-        manager.serializeSnapshotBufferToDisk(*buf, 50);
-    }
-    EXPECT_TRUE(fs::exists("./snapshots/snapshot_50.bin.zstd"));
-
-    /// Let's corrupt file
-    DB::WriteBufferFromFile plain_buf(
-        "./snapshots/snapshot_50.bin.zstd", DB::DBMS_DEFAULT_BUFFER_SIZE, O_APPEND | O_CREAT | O_WRONLY);
-    plain_buf.truncate(34);
-    plain_buf.sync();
-
-    EXPECT_THROW(manager.restoreFromLatestSnapshot(), DB::Exception);
-}
-
-nuraft::ptr<nuraft::buffer> getBufferFromZKRequest(int64_t session_id, int64_t zxid, const Coordination::ZooKeeperRequestPtr & request);
-
-nuraft::ptr<nuraft::log_entry>
-getLogEntryFromZKRequest(size_t term, int64_t session_id, int64_t zxid, const Coordination::ZooKeeperRequestPtr & request);
-
-static void testLogAndStateMachine(
-    DB::CoordinationSettingsPtr settings,
-    uint64_t total_logs,
-    bool enable_compression = true)
-{
-    using namespace Coordination;
-    using namespace DB;
-
-    ChangelogDirTest snapshots("./snapshots");
-    ChangelogDirTest logs("./logs");
-    ChangelogDirTest rocks("./rocksdb");
-
-    auto get_keeper_context = [&]
-    {
-        auto local_keeper_context = std::make_shared<DB::KeeperContext>(true, settings);
-        local_keeper_context->setSnapshotDisk(std::make_shared<DiskLocal>("SnapshotDisk", "./snapshots"));
-        local_keeper_context->setLogDisk(std::make_shared<DiskLocal>("LogDisk", "./logs"));
-        local_keeper_context->setRocksDBDisk(std::make_shared<DiskLocal>("RocksDisk", "./rocksdb"));
-        local_keeper_context->setRocksDBOptions(nullptr);
-        return local_keeper_context;
-    };
-
-    ResponsesQueue queue(std::numeric_limits<size_t>::max());
-    SnapshotsQueue snapshots_queue{1};
-
-    auto keeper_context = get_keeper_context();
-    auto state_machine = std::make_shared<KeeperStateMachine<DB::KeeperRocksStorage>>(queue, snapshots_queue, keeper_context, nullptr);
-
-    state_machine->init();
-    DB::KeeperLogStore changelog(
-        DB::LogFileSettings{
-            .force_sync = true, .compress_logs = enable_compression, .rotate_interval = settings->rotate_log_storage_interval},
-        DB::FlushSettings(),
-        keeper_context);
-    changelog.init(state_machine->last_commit_index() + 1, settings->reserved_log_items);
-
-    for (size_t i = 1; i < total_logs + 1; ++i)
-    {
-        std::shared_ptr<ZooKeeperCreateRequest> request = std::make_shared<ZooKeeperCreateRequest>();
-        request->path = "/hello_" + std::to_string(i);
-        auto entry = getLogEntryFromZKRequest(0, 1, i, request);
-        changelog.append(entry);
-        changelog.end_of_append_batch(0, 0);
-
-        waitDurableLogs(changelog);
-
-        state_machine->pre_commit(i, changelog.entry_at(i)->get_buf());
-        state_machine->commit(i, changelog.entry_at(i)->get_buf());
-        bool snapshot_created = false;
-        if (i % settings->snapshot_distance == 0)
-        {
-            nuraft::snapshot s(i, 0, std::make_shared<nuraft::cluster_config>());
-            nuraft::async_result<bool>::handler_type when_done
-                = [&snapshot_created](bool & ret, nuraft::ptr<std::exception> & /*exception*/)
-            {
-                snapshot_created = ret;
-            };
-
-            state_machine->create_snapshot(s, when_done);
-            CreateSnapshotTask snapshot_task;
-            bool pop_result = snapshots_queue.pop(snapshot_task);
-            EXPECT_TRUE(pop_result);
-
-            snapshot_task.create_snapshot(std::move(snapshot_task.snapshot), false);
-        }
-
-        if (snapshot_created && changelog.size() > settings->reserved_log_items)
-            changelog.compact(i - settings->reserved_log_items);
-    }
-
-    SnapshotsQueue snapshots_queue1{1};
-    keeper_context = get_keeper_context();
-    auto restore_machine = std::make_shared<KeeperStateMachine<DB::KeeperRocksStorage>>(queue, snapshots_queue1, keeper_context, nullptr);
-    restore_machine->init();
-    EXPECT_EQ(restore_machine->last_commit_index(), total_logs - total_logs % settings->snapshot_distance);
-
-    DB::KeeperLogStore restore_changelog(
-        DB::LogFileSettings{
-            .force_sync = true, .compress_logs = enable_compression, .rotate_interval = settings->rotate_log_storage_interval},
-        DB::FlushSettings(),
-        keeper_context);
-    restore_changelog.init(restore_machine->last_commit_index() + 1, settings->reserved_log_items);
-
-    EXPECT_EQ(restore_changelog.size(), std::min(settings->reserved_log_items + total_logs % settings->snapshot_distance, total_logs));
-    EXPECT_EQ(restore_changelog.next_slot(), total_logs + 1);
-    if (total_logs > settings->reserved_log_items + 1)
-        EXPECT_EQ(
-            restore_changelog.start_index(), total_logs - total_logs % settings->snapshot_distance - settings->reserved_log_items + 1);
-    else
-        EXPECT_EQ(restore_changelog.start_index(), 1);
-
-    for (size_t i = restore_machine->last_commit_index() + 1; i < restore_changelog.next_slot(); ++i)
-    {
-        restore_machine->pre_commit(i, changelog.entry_at(i)->get_buf());
-        restore_machine->commit(i, changelog.entry_at(i)->get_buf());
-    }
-
-    auto & source_storage = state_machine->getStorageUnsafe();
-    auto & restored_storage = restore_machine->getStorageUnsafe();
-
-    EXPECT_EQ(source_storage.container.size(), restored_storage.container.size());
-    for (size_t i = 1; i < total_logs + 1; ++i)
-    {
-        auto path = "/hello_" + std::to_string(i);
-        EXPECT_EQ(source_storage.container.find(path)->value.getData(), restored_storage.container.find(path)->value.getData());
-    }
-}
-
-TEST_F(RocksKeeperTest, TestStateMachineAndLogStore)
-{
-    using namespace Coordination;
-    using namespace DB;
-
-    {
-        CoordinationSettingsPtr settings = std::make_shared<CoordinationSettings>();
-        settings->snapshot_distance = 10;
-        settings->reserved_log_items = 10;
-        settings->rotate_log_storage_interval = 10;
-
-        testLogAndStateMachine(settings, 37);
-    }
-    {
-        CoordinationSettingsPtr settings = std::make_shared<CoordinationSettings>();
-        settings->snapshot_distance = 10;
-        settings->reserved_log_items = 10;
-        settings->rotate_log_storage_interval = 10;
-        testLogAndStateMachine(settings, 11);
-    }
-    {
-        CoordinationSettingsPtr settings = std::make_shared<CoordinationSettings>();
-        settings->snapshot_distance = 10;
-        settings->reserved_log_items = 10;
-        settings->rotate_log_storage_interval = 10;
-        testLogAndStateMachine(settings, 40);
-    }
-    {
-        CoordinationSettingsPtr settings = std::make_shared<CoordinationSettings>();
-        settings->snapshot_distance = 10;
-        settings->reserved_log_items = 20;
-        settings->rotate_log_storage_interval = 30;
-        testLogAndStateMachine(settings, 40);
-    }
-    {
-        CoordinationSettingsPtr settings = std::make_shared<CoordinationSettings>();
-        settings->snapshot_distance = 10;
-        settings->reserved_log_items = 0;
-        settings->rotate_log_storage_interval = 10;
-        testLogAndStateMachine(settings, 40);
-    }
-    {
-        CoordinationSettingsPtr settings = std::make_shared<CoordinationSettings>();
-        settings->snapshot_distance = 1;
-        settings->reserved_log_items = 1;
-        settings->rotate_log_storage_interval = 32;
-        testLogAndStateMachine(settings, 32);
-    }
-    {
-        CoordinationSettingsPtr settings = std::make_shared<CoordinationSettings>();
-        settings->snapshot_distance = 10;
-        settings->reserved_log_items = 7;
-        settings->rotate_log_storage_interval = 1;
-        testLogAndStateMachine(settings, 33);
-    }
-    {
-        CoordinationSettingsPtr settings = std::make_shared<CoordinationSettings>();
-        settings->snapshot_distance = 37;
-        settings->reserved_log_items = 1000;
-        settings->rotate_log_storage_interval = 5000;
-        testLogAndStateMachine(settings, 33);
-    }
-    {
-        CoordinationSettingsPtr settings = std::make_shared<CoordinationSettings>();
-        settings->snapshot_distance = 37;
-        settings->reserved_log_items = 1000;
-        settings->rotate_log_storage_interval = 5000;
-        testLogAndStateMachine(settings, 45);
-    }
-}
-
-TEST_F(RocksKeeperTest, TestEphemeralNodeRemove)
-{
-    using namespace Coordination;
-    using namespace DB;
-
-    ChangelogDirTest snapshots("./snapshots");
-    setSnapshotDirectory("./snapshots");
-    ChangelogDirTest rocks("./rocksdb");
-    setRocksDBDirectory("./rocksdb");
-
-    ResponsesQueue queue(std::numeric_limits<size_t>::max());
-    SnapshotsQueue snapshots_queue{1};
-
-    auto state_machine = std::make_shared<KeeperStateMachine<DB::KeeperRocksStorage>>(queue, snapshots_queue, keeper_context, nullptr);
-    state_machine->init();
-
-    std::shared_ptr<ZooKeeperCreateRequest> request_c = std::make_shared<ZooKeeperCreateRequest>();
-    request_c->path = "/hello";
-    request_c->is_ephemeral = true;
-    auto entry_c = getLogEntryFromZKRequest(0, 1, state_machine->getNextZxid(), request_c);
-    state_machine->pre_commit(1, entry_c->get_buf());
-    state_machine->commit(1, entry_c->get_buf());
-    const auto & storage = state_machine->getStorageUnsafe();
-
-    EXPECT_EQ(storage.ephemerals.size(), 1);
-    std::shared_ptr<ZooKeeperRemoveRequest> request_d = std::make_shared<ZooKeeperRemoveRequest>();
-    request_d->path = "/hello";
-    /// Delete from other session
-    auto entry_d = getLogEntryFromZKRequest(0, 2, state_machine->getNextZxid(), request_d);
-    state_machine->pre_commit(2, entry_d->get_buf());
-    state_machine->commit(2, entry_d->get_buf());
-
-    EXPECT_EQ(storage.ephemerals.size(), 0);
-}
-
-TEST_F(RocksKeeperTest, TestCreateNodeWithAuthSchemeForAclWhenAuthIsPrecommitted)
-{
-    using namespace Coordination;
-    using namespace DB;
-
-    ChangelogDirTest snapshots("./snapshots");
-    setSnapshotDirectory("./snapshots");
-    ChangelogDirTest rocks("./rocksdb");
-    setRocksDBDirectory("./rocksdb");
-    ResponsesQueue queue(std::numeric_limits<size_t>::max());
-    SnapshotsQueue snapshots_queue{1};
-
-    auto state_machine = std::make_shared<KeeperStateMachine<DB::KeeperRocksStorage>>(queue, snapshots_queue, keeper_context, nullptr);
-    state_machine->init();
-
-    String user_auth_data = "test_user:test_password";
-    String digest = KeeperRocksStorage::generateDigest(user_auth_data);
-
-    std::shared_ptr<ZooKeeperAuthRequest> auth_req = std::make_shared<ZooKeeperAuthRequest>();
-    auth_req->scheme = "digest";
-    auth_req->data = user_auth_data;
-
-    // Add auth data to the session
-    auto auth_entry = getLogEntryFromZKRequest(0, 1, state_machine->getNextZxid(), auth_req);
-    state_machine->pre_commit(1, auth_entry->get_buf());
-
-    // Create a node with 'auth' scheme for ACL
-    String node_path = "/hello";
-    std::shared_ptr<ZooKeeperCreateRequest> create_req = std::make_shared<ZooKeeperCreateRequest>();
-    create_req->path = node_path;
-    // When 'auth' scheme is used the creator must have been authenticated by the server (for example, using 'digest' scheme) before it can
-    // create nodes with this ACL.
-    create_req->acls = {{.permissions = 31, .scheme = "auth", .id = ""}};
-    auto create_entry = getLogEntryFromZKRequest(0, 1, state_machine->getNextZxid(), create_req);
-    state_machine->pre_commit(2, create_entry->get_buf());
-
-    const auto & uncommitted_state = state_machine->getStorageUnsafe().uncommitted_state;
-    ASSERT_TRUE(uncommitted_state.nodes.contains(node_path));
-
-    // commit log entries
-    state_machine->commit(1, auth_entry->get_buf());
-    state_machine->commit(2, create_entry->get_buf());
-
-    auto node = uncommitted_state.getNode(node_path);
-    ASSERT_NE(node, nullptr);
-    auto acls = uncommitted_state.getACLs(node_path);
-    ASSERT_EQ(acls.size(), 1);
-    EXPECT_EQ(acls[0].scheme, "digest");
-    EXPECT_EQ(acls[0].id, digest);
-    EXPECT_EQ(acls[0].permissions, 31);
-}
-
-TEST_F(RocksKeeperTest, TestSetACLWithAuthSchemeForAclWhenAuthIsPrecommitted)
-{
-    using namespace Coordination;
-    using namespace DB;
-
-    ChangelogDirTest snapshots("./snapshots");
-    setSnapshotDirectory("./snapshots");
-    ChangelogDirTest rocks("./rocksdb");
-    setRocksDBDirectory("./rocksdb");
-
-    ResponsesQueue queue(std::numeric_limits<size_t>::max());
-    SnapshotsQueue snapshots_queue{1};
-
-    auto state_machine = std::make_shared<KeeperStateMachine<DB::KeeperRocksStorage>>(queue, snapshots_queue, keeper_context, nullptr);
-    state_machine->init();
-
-    String user_auth_data = "test_user:test_password";
-    String digest = KeeperRocksStorage::generateDigest(user_auth_data);
-
-    std::shared_ptr<ZooKeeperAuthRequest> auth_req = std::make_shared<ZooKeeperAuthRequest>();
-    auth_req->scheme = "digest";
-    auth_req->data = user_auth_data;
-
-    // Add auth data to the session
-    auto auth_entry = getLogEntryFromZKRequest(0, 1, state_machine->getNextZxid(), auth_req);
-    state_machine->pre_commit(1, auth_entry->get_buf());
-
-    // Create a node
-    String node_path = "/hello";
-    std::shared_ptr<ZooKeeperCreateRequest> create_req = std::make_shared<ZooKeeperCreateRequest>();
-    create_req->path = node_path;
-    auto create_entry = getLogEntryFromZKRequest(0, 1, state_machine->getNextZxid(), create_req);
-    state_machine->pre_commit(2, create_entry->get_buf());
-
-    // Set ACL with 'auth' scheme for ACL
-    std::shared_ptr<ZooKeeperSetACLRequest> set_acl_req = std::make_shared<ZooKeeperSetACLRequest>();
-    set_acl_req->path = node_path;
-    // When 'auth' scheme is used the creator must have been authenticated by the server (for example, using 'digest' scheme) before it can
-    // set this ACL.
-    set_acl_req->acls = {{.permissions = 31, .scheme = "auth", .id = ""}};
-    auto set_acl_entry = getLogEntryFromZKRequest(0, 1, state_machine->getNextZxid(), set_acl_req);
-    state_machine->pre_commit(3, set_acl_entry->get_buf());
-
-    // commit all entries
-    state_machine->commit(1, auth_entry->get_buf());
-    state_machine->commit(2, create_entry->get_buf());
-    state_machine->commit(3, set_acl_entry->get_buf());
-
-    const auto & uncommitted_state = state_machine->getStorageUnsafe().uncommitted_state;
-    auto node = uncommitted_state.getNode(node_path);
-
-    ASSERT_NE(node, nullptr);
-    auto acls = uncommitted_state.getACLs(node_path);
-    ASSERT_EQ(acls.size(), 1);
-    EXPECT_EQ(acls[0].scheme, "digest");
-    EXPECT_EQ(acls[0].id, digest);
-    EXPECT_EQ(acls[0].permissions, 31);
-}
-
-TEST_F(RocksKeeperTest, TestStorageSnapshotEqual)
-{
-    ChangelogDirTest test("./snapshots");
-    ChangelogDirTest rocks("./rocksdb");
-    setSnapshotDirectory("./snapshots");
-    setRocksDBDirectory("./rocksdb");
-
-    std::optional<UInt128> snapshot_hash;
-    for (size_t i = 0; i < 15; ++i)
-    {
-        DB::KeeperSnapshotManager<DB::KeeperRocksStorage> manager(3, keeper_context);
-
-        DB::KeeperRocksStorage storage(500, "", keeper_context);
-        addNode(storage, "/hello", "");
-        for (size_t j = 0; j < 100; ++j)
-        {
-            addNode(storage, "/hello_" + std::to_string(j), "world", 1);
-            addNode(storage, "/hello/somepath_" + std::to_string(j), "somedata", 3);
-        }
-
-        storage.session_id_counter = 5;
-
-        storage.ephemerals[3] = {"/hello"};
-        storage.ephemerals[1] = {"/hello/somepath"};
-
-        for (size_t j = 0; j < 3333; ++j)
-            storage.getSessionID(130 * j);
-
-        DB::KeeperStorageSnapshot<DB::KeeperRocksStorage> snapshot(&storage, storage.zxid);
-
-        auto buf = manager.serializeSnapshotToBuffer(snapshot);
-
-        auto new_hash = sipHash128(reinterpret_cast<char *>(buf->data()), buf->size());
-        if (!snapshot_hash.has_value())
-        {
-            snapshot_hash = new_hash;
-        }
-        else
-        {
-            EXPECT_EQ(*snapshot_hash, new_hash);
-        }
-    }
-}
-
-TEST_F(RocksKeeperTest, TestStorageSnapshotDifferentCompressions)
-{
-    ChangelogDirTest test("./snapshots");
-    setSnapshotDirectory("./snapshots");
-    ChangelogDirTest rocks("./rocksdb");
-    setRocksDBDirectory("./rocksdb");
-
-    DB::KeeperSnapshotManager<DB::KeeperRocksStorage> manager(3, keeper_context);
-
-    DB::KeeperRocksStorage storage(500, "", keeper_context);
-    addNode(storage, "/hello1", "world", 1);
-    addNode(storage, "/hello2", "somedata", 3);
-    storage.session_id_counter = 5;
-    storage.zxid = 2;
-    storage.ephemerals[3] = {"/hello2"};
-    storage.ephemerals[1] = {"/hello1"};
-    storage.getSessionID(130);
-    storage.getSessionID(130);
-
-    DB::KeeperStorageSnapshot<DB::KeeperRocksStorage> snapshot(&storage, 2);
-
-    auto buf = manager.serializeSnapshotToBuffer(snapshot);
-    manager.serializeSnapshotBufferToDisk(*buf, 2);
-    EXPECT_TRUE(fs::exists("./snapshots/snapshot_2.bin.zstd"));
-
-    DB::KeeperSnapshotManager<DB::KeeperRocksStorage> new_manager(3, keeper_context, false);
-
-    auto debuf = new_manager.deserializeSnapshotBufferFromDisk(2);
-
-    auto [restored_storage, snapshot_meta, _] = new_manager.deserializeSnapshotFromBuffer(debuf);
-
-    EXPECT_EQ(restored_storage->container.size(), 6);
-    EXPECT_EQ(restored_storage->container.find("/")->value.numChildren(), 3);
-    EXPECT_EQ(restored_storage->container.find("/hello1")->value.numChildren(), 0);
-    EXPECT_EQ(restored_storage->container.find("/hello2")->value.numChildren(), 0);
-
-    EXPECT_EQ(restored_storage->container.find("/")->value.getData(), "");
-    EXPECT_EQ(restored_storage->container.find("/hello1")->value.getData(), "world");
-    EXPECT_EQ(restored_storage->container.find("/hello2")->value.getData(), "somedata");
-    EXPECT_EQ(restored_storage->session_id_counter, 7);
-    EXPECT_EQ(restored_storage->zxid, 2);
-    EXPECT_EQ(restored_storage->ephemerals.size(), 2);
-    EXPECT_EQ(restored_storage->ephemerals[3].size(), 1);
-    EXPECT_EQ(restored_storage->ephemerals[1].size(), 1);
-    EXPECT_EQ(restored_storage->session_and_timeout.size(), 2);
-}
-
-template <typename ResponseType>
-ResponseType getSingleResponse(const auto & responses)
-{
-    EXPECT_FALSE(responses.empty());
-    return dynamic_cast<ResponseType &>(*responses[0].response);
-}
-
-TEST_F(RocksKeeperTest, TestUncommittedStateBasicCrud)
-{
-    ChangelogDirTest test("./rocksdb");
-    setRocksDBDirectory("./rocksdb");
-
-    using namespace DB;
-    using namespace Coordination;
-
-    DB::KeeperRocksStorage storage{500, "", keeper_context};
-
-    constexpr std::string_view path = "/test";
-
-    const auto get_committed_data = [&]() -> std::optional<String>
-    {
-        auto request = std::make_shared<ZooKeeperGetRequest>();
-        request->path = path;
-        auto responses = storage.processRequest(request, 0, std::nullopt, true, true);
-        const auto & get_response = getSingleResponse<ZooKeeperGetResponse>(responses);
-
-        if (get_response.error != Error::ZOK)
-            return std::nullopt;
-
-        return get_response.data;
-    };
-
-    const auto preprocess_get = [&](int64_t zxid)
-    {
-        auto get_request = std::make_shared<ZooKeeperGetRequest>();
-        get_request->path = path;
-        storage.preprocessRequest(get_request, 0, 0, zxid);
-        return get_request;
-    };
-
-    const auto create_request = std::make_shared<ZooKeeperCreateRequest>();
-    create_request->path = path;
-    create_request->data = "initial_data";
-    storage.preprocessRequest(create_request, 0, 0, 1);
-    storage.preprocessRequest(create_request, 0, 0, 2);
-
-    ASSERT_FALSE(get_committed_data());
-
-    const auto after_create_get = preprocess_get(3);
-
-    ASSERT_FALSE(get_committed_data());
-
-    const auto set_request = std::make_shared<ZooKeeperSetRequest>();
-    set_request->path = path;
-    set_request->data = "new_data";
-    storage.preprocessRequest(set_request, 0, 0, 4);
-
-    const auto after_set_get = preprocess_get(5);
-
-    ASSERT_FALSE(get_committed_data());
-
-    const auto remove_request = std::make_shared<ZooKeeperRemoveRequest>();
-    remove_request->path = path;
-    storage.preprocessRequest(remove_request, 0, 0, 6);
-    storage.preprocessRequest(remove_request, 0, 0, 7);
-
-    const auto after_remove_get = preprocess_get(8);
-
-    ASSERT_FALSE(get_committed_data());
-
-    {
-        const auto responses = storage.processRequest(create_request, 0, 1);
-        const auto & create_response = getSingleResponse<ZooKeeperCreateResponse>(responses);
-        ASSERT_EQ(create_response.error, Error::ZOK);
-    }
-
-    {
-        const auto responses = storage.processRequest(create_request, 0, 2);
-        const auto & create_response = getSingleResponse<ZooKeeperCreateResponse>(responses);
-        ASSERT_EQ(create_response.error, Error::ZNODEEXISTS);
-    }
-
-    {
-        const auto responses = storage.processRequest(after_create_get, 0, 3);
-        const auto & get_response = getSingleResponse<ZooKeeperGetResponse>(responses);
-        ASSERT_EQ(get_response.error, Error::ZOK);
-        ASSERT_EQ(get_response.data, "initial_data");
-    }
-
-    ASSERT_EQ(get_committed_data(), "initial_data");
-
-    {
-        const auto responses = storage.processRequest(set_request, 0, 4);
-        const auto & create_response = getSingleResponse<ZooKeeperSetResponse>(responses);
-        ASSERT_EQ(create_response.error, Error::ZOK);
-    }
-
-    {
-        const auto responses = storage.processRequest(after_set_get, 0, 5);
-        const auto & get_response = getSingleResponse<ZooKeeperGetResponse>(responses);
-        ASSERT_EQ(get_response.error, Error::ZOK);
-        ASSERT_EQ(get_response.data, "new_data");
-    }
-
-    ASSERT_EQ(get_committed_data(), "new_data");
-
-    {
-        const auto responses = storage.processRequest(remove_request, 0, 6);
-        const auto & create_response = getSingleResponse<ZooKeeperRemoveResponse>(responses);
-        ASSERT_EQ(create_response.error, Error::ZOK);
-    }
-
-    {
-        const auto responses = storage.processRequest(remove_request, 0, 7);
-        const auto & remove_response = getSingleResponse<ZooKeeperRemoveResponse>(responses);
-        ASSERT_EQ(remove_response.error, Error::ZNONODE);
-    }
-
-    {
-        const auto responses = storage.processRequest(after_remove_get, 0, 8);
-        const auto & get_response = getSingleResponse<ZooKeeperGetResponse>(responses);
-        ASSERT_EQ(get_response.error, Error::ZNONODE);
-    }
-
-    ASSERT_FALSE(get_committed_data());
-}
-
-TEST_F(RocksKeeperTest, TestListRequestTypes)
-{
-    ChangelogDirTest test("./rocksdb");
-    setRocksDBDirectory("./rocksdb");
-
-    using namespace DB;
-    using namespace Coordination;
-    KeeperRocksStorage storage{500, "", keeper_context};
-
-    int32_t zxid = 0;
-
-    static constexpr std::string_view test_path = "/list_request_type/node";
-
-    const auto create_path = [&](const auto & path, bool is_ephemeral, bool is_sequential = true)
-    {
-        const auto create_request = std::make_shared<ZooKeeperCreateRequest>();
-        int new_zxid = ++zxid;
-        create_request->path = path;
-        create_request->is_sequential = is_sequential;
-        create_request->is_ephemeral = is_ephemeral;
-        storage.preprocessRequest(create_request, 1, 0, new_zxid);
-        auto responses = storage.processRequest(create_request, 1, new_zxid);
-
-        EXPECT_GE(responses.size(), 1);
-        EXPECT_EQ(responses[0].response->error, Coordination::Error::ZOK) << "Failed to create " << path;
-        const auto & create_response = dynamic_cast<ZooKeeperCreateResponse &>(*responses[0].response);
-        return create_response.path_created;
-    };
-
-    create_path(parentNodePath(StringRef{test_path}).toString(), false, false);
-
-    static constexpr size_t persistent_num = 5;
-    std::unordered_set<std::string> expected_persistent_children;
-    for (size_t i = 0; i < persistent_num; ++i)
-    {
-        expected_persistent_children.insert(getBaseNodeName(create_path(test_path, false)).toString());
-    }
-    ASSERT_EQ(expected_persistent_children.size(), persistent_num);
-
-    static constexpr size_t ephemeral_num = 5;
-    std::unordered_set<std::string> expected_ephemeral_children;
-    for (size_t i = 0; i < ephemeral_num; ++i)
-    {
-        expected_ephemeral_children.insert(getBaseNodeName(create_path(test_path, true)).toString());
-    }
-    ASSERT_EQ(expected_ephemeral_children.size(), ephemeral_num);
-
-    const auto get_children = [&](const auto list_request_type)
-    {
-        const auto list_request = std::make_shared<ZooKeeperFilteredListRequest>();
-        int new_zxid = ++zxid;
-        list_request->path = parentNodePath(StringRef{test_path}).toString();
-        list_request->list_request_type = list_request_type;
-        storage.preprocessRequest(list_request, 1, 0, new_zxid);
-        auto responses = storage.processRequest(list_request, 1, new_zxid);
-
-        EXPECT_GE(responses.size(), 1);
-        const auto & list_response = dynamic_cast<ZooKeeperListResponse &>(*responses[0].response);
-        EXPECT_EQ(list_response.error, Coordination::Error::ZOK);
-        return list_response.names;
-    };
-
-    const auto persistent_children = get_children(ListRequestType::PERSISTENT_ONLY);
-    EXPECT_EQ(persistent_children.size(), persistent_num);
-    for (const auto & child : persistent_children)
-    {
-        EXPECT_TRUE(expected_persistent_children.contains(child)) << "Missing persistent child " << child;
-    }
-
-    const auto ephemeral_children = get_children(ListRequestType::EPHEMERAL_ONLY);
-    EXPECT_EQ(ephemeral_children.size(), ephemeral_num);
-    for (const auto & child : ephemeral_children)
-    {
-        EXPECT_TRUE(expected_ephemeral_children.contains(child)) << "Missing ephemeral child " << child;
-    }
-
-    const auto all_children = get_children(ListRequestType::ALL);
-    EXPECT_EQ(all_children.size(), ephemeral_num + persistent_num);
-    for (const auto & child : all_children)
-    {
-        EXPECT_TRUE(expected_ephemeral_children.contains(child) || expected_persistent_children.contains(child))
-            << "Missing child " << child;
-    }
-}
-
-TEST_F(RocksKeeperTest, TestFeatureFlags)
-{
-    ChangelogDirTest test("./rocksdb");
-    setRocksDBDirectory("./rocksdb");
-
-    using namespace Coordination;
-    KeeperMemoryStorage storage{500, "", keeper_context};
-    auto request = std::make_shared<ZooKeeperGetRequest>();
-    request->path = DB::keeper_api_feature_flags_path;
-    auto responses = storage.processRequest(request, 0, std::nullopt, true, true);
-    const auto & get_response = getSingleResponse<ZooKeeperGetResponse>(responses);
-    DB::KeeperFeatureFlags feature_flags;
-    feature_flags.setFeatureFlags(get_response.data);
-    ASSERT_TRUE(feature_flags.isEnabled(KeeperFeatureFlag::FILTERED_LIST));
-    ASSERT_TRUE(feature_flags.isEnabled(KeeperFeatureFlag::MULTI_READ));
-    ASSERT_FALSE(feature_flags.isEnabled(KeeperFeatureFlag::CHECK_NOT_EXISTS));
-}
-
-TEST_F(RocksKeeperTest, TestSystemNodeModify)
-{
-    ChangelogDirTest test("./rocksdb");
-    setRocksDBDirectory("./rocksdb");
-
-    using namespace Coordination;
-    int64_t zxid{0};
-
-    // On INIT we abort when a system path is modified
-    keeper_context->setServerState(KeeperContext::Phase::RUNNING);
-    KeeperRocksStorage storage{500, "", keeper_context};
-    const auto assert_create = [&](const std::string_view path, const auto expected_code)
-    {
-        auto request = std::make_shared<ZooKeeperCreateRequest>();
-        request->path = path;
-        storage.preprocessRequest(request, 0, 0, zxid);
-        auto responses = storage.processRequest(request, 0, zxid);
-        ASSERT_FALSE(responses.empty());
-
-        const auto & response = responses[0];
-        ASSERT_EQ(response.response->error, expected_code) << "Unexpected error for path " << path;
-
-        ++zxid;
-    };
-
-    assert_create("/keeper", Error::ZBADARGUMENTS);
-    assert_create("/keeper/with_child", Error::ZBADARGUMENTS);
-    assert_create(DB::keeper_api_version_path, Error::ZBADARGUMENTS);
-
-    assert_create("/keeper_map", Error::ZOK);
-    assert_create("/keeper1", Error::ZOK);
-    assert_create("/keepe", Error::ZOK);
-    assert_create("/keeper1/test", Error::ZOK);
-}
-
-TEST_F(RocksKeeperTest, TestCheckNotExistsRequest)
-{
-    ChangelogDirTest test("./rocksdb");
-    setRocksDBDirectory("./rocksdb");
-
-    using namespace DB;
-    using namespace Coordination;
-
-    KeeperRocksStorage storage{500, "", keeper_context};
-
-    int32_t zxid = 0;
-
-    const auto create_path = [&](const auto & path)
-    {
-        const auto create_request = std::make_shared<ZooKeeperCreateRequest>();
-        int new_zxid = ++zxid;
-        create_request->path = path;
-        storage.preprocessRequest(create_request, 1, 0, new_zxid);
-        auto responses = storage.processRequest(create_request, 1, new_zxid);
-
-        EXPECT_GE(responses.size(), 1);
-        EXPECT_EQ(responses[0].response->error, Coordination::Error::ZOK) << "Failed to create " << path;
-    };
-
-    const auto check_request = std::make_shared<ZooKeeperCheckRequest>();
-    check_request->path = "/test_node";
-    check_request->not_exists = true;
-
-    {
-        SCOPED_TRACE("CheckNotExists returns ZOK");
-        int new_zxid = ++zxid;
-        storage.preprocessRequest(check_request, 1, 0, new_zxid);
-        auto responses = storage.processRequest(check_request, 1, new_zxid);
-        EXPECT_GE(responses.size(), 1);
-        auto error = responses[0].response->error;
-        EXPECT_EQ(error, Coordination::Error::ZOK) << "CheckNotExists returned invalid result: " << errorMessage(error);
-    }
-
-    create_path("/test_node");
-    auto node_it = storage.container.find("/test_node");
-    ASSERT_NE(node_it, storage.container.end());
-    auto node_version = node_it->value.version;
-
-    {
-        SCOPED_TRACE("CheckNotExists returns ZNODEEXISTS");
-        int new_zxid = ++zxid;
-        storage.preprocessRequest(check_request, 1, 0, new_zxid);
-        auto responses = storage.processRequest(check_request, 1, new_zxid);
-        EXPECT_GE(responses.size(), 1);
-        auto error = responses[0].response->error;
-        EXPECT_EQ(error, Coordination::Error::ZNODEEXISTS) << "CheckNotExists returned invalid result: " << errorMessage(error);
-    }
-
-    {
-        SCOPED_TRACE("CheckNotExists returns ZNODEEXISTS for same version");
-        int new_zxid = ++zxid;
-        check_request->version = node_version;
-        storage.preprocessRequest(check_request, 1, 0, new_zxid);
-        auto responses = storage.processRequest(check_request, 1, new_zxid);
-        EXPECT_GE(responses.size(), 1);
-        auto error = responses[0].response->error;
-        EXPECT_EQ(error, Coordination::Error::ZNODEEXISTS) << "CheckNotExists returned invalid result: " << errorMessage(error);
-    }
-
-    {
-        SCOPED_TRACE("CheckNotExists returns ZOK for different version");
-        int new_zxid = ++zxid;
-        check_request->version = node_version + 1;
-        storage.preprocessRequest(check_request, 1, 0, new_zxid);
-        auto responses = storage.processRequest(check_request, 1, new_zxid);
-        EXPECT_GE(responses.size(), 1);
-        auto error = responses[0].response->error;
-        EXPECT_EQ(error, Coordination::Error::ZOK) << "CheckNotExists returned invalid result: " << errorMessage(error);
-    }
-}
-
-TEST_F(RocksKeeperTest, TestReapplyingDeltas)
-{
-    ChangelogDirTest test("./rocksdb");
-    setRocksDBDirectory("./rocksdb");
-
-    using namespace DB;
-    using namespace Coordination;
-
-    static constexpr int64_t initial_zxid = 100;
-
-    const auto create_request = std::make_shared<ZooKeeperCreateRequest>();
-    create_request->path = "/test/data";
-    create_request->is_sequential = true;
-
-    const auto process_create = [](KeeperRocksStorage & storage, const auto & request, int64_t zxid)
-    {
-        storage.preprocessRequest(request, 1, 0, zxid);
-        auto responses = storage.processRequest(request, 1, zxid);
-        EXPECT_GE(responses.size(), 1);
-        EXPECT_EQ(responses[0].response->error, Error::ZOK);
-    };
-
-    const auto commit_initial_data = [&](auto & storage)
-    {
-        int64_t zxid = 1;
-
-        const auto root_create = std::make_shared<ZooKeeperCreateRequest>();
-        root_create->path = "/test";
-        process_create(storage, root_create, zxid);
-        ++zxid;
-
-        for (; zxid <= initial_zxid; ++zxid)
-            process_create(storage, create_request, zxid);
-    };
-
-    KeeperRocksStorage storage1{500, "", keeper_context};
-    commit_initial_data(storage1);
-
-    for (int64_t zxid = initial_zxid + 1; zxid < initial_zxid + 50; ++zxid)
-        storage1.preprocessRequest(create_request, 1, 0, zxid, /*check_acl=*/true, /*digest=*/std::nullopt, /*log_idx=*/zxid);
-
-    /// create identical new storage
-    KeeperRocksStorage storage2{500, "", keeper_context};
-    commit_initial_data(storage2);
-
-    storage1.applyUncommittedState(storage2, initial_zxid);
-
-    const auto commit_unprocessed = [&](KeeperRocksStorage & storage)
-    {
-        for (int64_t zxid = initial_zxid + 1; zxid < initial_zxid + 50; ++zxid)
-        {
-            auto responses = storage.processRequest(create_request, 1, zxid);
-            EXPECT_GE(responses.size(), 1);
-            EXPECT_EQ(responses[0].response->error, Error::ZOK);
-        }
-    };
-
-    commit_unprocessed(storage1);
-    commit_unprocessed(storage2);
-
-    const auto get_children = [&](KeeperRocksStorage & storage)
-    {
-        const auto list_request = std::make_shared<ZooKeeperListRequest>();
-        list_request->path = "/test";
-        auto responses = storage.processRequest(list_request, 1, std::nullopt, /*check_acl=*/true, /*is_local=*/true);
-        EXPECT_EQ(responses.size(), 1);
-        const auto * list_response = dynamic_cast<const ListResponse *>(responses[0].response.get());
-        EXPECT_TRUE(list_response);
-        return list_response->names;
-    };
-
-    auto children1 = get_children(storage1);
-    std::unordered_set<std::string> children1_set(children1.begin(), children1.end());
-
-    auto children2 = get_children(storage2);
-    std::unordered_set<std::string> children2_set(children2.begin(), children2.end());
-
-    ASSERT_TRUE(children1_set == children2_set);
-}
-
-#endif

From 1d4e26fa6a94799fa8ff708488d17dfeeab991e3 Mon Sep 17 00:00:00 2001
From: Justin de Guzman <justin@justindeguzman.net>
Date: Wed, 15 May 2024 13:45:18 -0700
Subject: [PATCH 0044/1488] Add back missing ProtobufList docs

---
 docs/en/interfaces/formats.md | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md
index a137eb2bdf2..a20eff0223d 100644
--- a/docs/en/interfaces/formats.md
+++ b/docs/en/interfaces/formats.md
@@ -67,6 +67,7 @@ The supported formats are:
 | [Prometheus](#prometheus)                                                                 | ✗    | ✔     |
 | [Protobuf](#protobuf)                                                                     | ✔    | ✔     |
 | [ProtobufSingle](#protobufsingle)                                                         | ✔    | ✔     |
+| [ProtobufList](#protobuflist)								    | ✔    | ✔     |
 | [Avro](#data-format-avro)                                                                 | ✔    | ✔     |
 | [AvroConfluent](#data-format-avro-confluent)                                              | ✔    | ✗     |
 | [Parquet](#data-format-parquet)                                                           | ✔    | ✔     |
@@ -1945,6 +1946,35 @@ SYSTEM DROP FORMAT SCHEMA CACHE FOR Protobuf
 
 Same as [Protobuf](#protobuf) but for storing/parsing single Protobuf message without length delimiters.
 
+## ProtobufList {#protobuflist}
+
+Similar to Protobuf but rows are represented as a sequence of sub-messages contained in a message with fixed name "Envelope".
+
+Usage example:
+
+``` sql
+SELECT * FROM test.table FORMAT ProtobufList SETTINGS format_schema = 'schemafile:MessageType'
+```
+
+``` bash
+cat protobuflist_messages.bin | clickhouse-client --query "INSERT INTO test.table FORMAT ProtobufList SETTINGS format_schema='schemafile:MessageType'"
+```
+
+where the file `schemafile.proto` looks like this:
+
+``` capnp
+syntax = "proto3";
+message Envelope {
+  message MessageType {
+    string name = 1;
+    string surname = 2;
+    uint32 birthDate = 3;
+    repeated string phoneNumbers = 4;
+  };
+  MessageType row = 1;
+};
+```
+
 ## Avro {#data-format-avro}
 
 [Apache Avro](https://avro.apache.org/) is a row-oriented data serialization framework developed within Apache’s Hadoop project.

From 48d47d26a4f18e180fa5602ad0aa89cc7af234a4 Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Thu, 14 Sep 2023 14:29:35 +0000
Subject: [PATCH 0045/1488] Add simple unit test for full sorting join

---
 .../Transforms/MergeJoinTransform.cpp         | 71 +++++++++-----
 .../Transforms/MergeJoinTransform.h           | 30 ++++--
 .../tests/gtest_merge_join_algorithm.cpp      | 95 +++++++++++++++++++
 3 files changed, 166 insertions(+), 30 deletions(-)
 create mode 100644 src/Processors/tests/gtest_merge_join_algorithm.cpp

diff --git a/src/Processors/Transforms/MergeJoinTransform.cpp b/src/Processors/Transforms/MergeJoinTransform.cpp
index 159a3244fe9..a9fd7978249 100644
--- a/src/Processors/Transforms/MergeJoinTransform.cpp
+++ b/src/Processors/Transforms/MergeJoinTransform.cpp
@@ -270,35 +270,45 @@ bool FullMergeJoinCursor::fullyCompleted() const
 }
 
 MergeJoinAlgorithm::MergeJoinAlgorithm(
-    JoinPtr table_join_,
+    JoinKind kind_,
+    JoinStrictness strictness_,
+    const TableJoin::JoinOnClause & on_clause_,
     const Blocks & input_headers,
     size_t max_block_size_)
-    : table_join(table_join_)
+    : kind(kind_)
+    , strictness(strictness_)
     , max_block_size(max_block_size_)
     , log(getLogger("MergeJoinAlgorithm"))
 {
     if (input_headers.size() != 2)
         throw Exception(ErrorCodes::LOGICAL_ERROR, "MergeJoinAlgorithm requires exactly two inputs");
 
-    auto strictness = table_join->getTableJoin().strictness();
     if (strictness != JoinStrictness::Any && strictness != JoinStrictness::All)
         throw Exception(ErrorCodes::NOT_IMPLEMENTED, "MergeJoinAlgorithm is not implemented for strictness {}", strictness);
 
-    auto kind = table_join->getTableJoin().kind();
     if (!isInner(kind) && !isLeft(kind) && !isRight(kind) && !isFull(kind))
         throw Exception(ErrorCodes::NOT_IMPLEMENTED, "MergeJoinAlgorithm is not implemented for kind {}", kind);
 
-    const auto & join_on = table_join->getTableJoin().getOnlyClause();
-
-    if (join_on.on_filter_condition_left || join_on.on_filter_condition_right)
+    if (on_clause_.on_filter_condition_left || on_clause_.on_filter_condition_right)
         throw Exception(ErrorCodes::NOT_IMPLEMENTED, "MergeJoinAlgorithm does not support ON filter conditions");
 
     cursors = {
-        createCursor(input_headers[0], join_on.key_names_left),
-        createCursor(input_headers[1], join_on.key_names_right)
+        createCursor(input_headers[0], on_clause_.key_names_left),
+        createCursor(input_headers[1], on_clause_.key_names_right)
     };
 
-    for (const auto & [left_key, right_key] : table_join->getTableJoin().leftToRightKeyRemap())
+MergeJoinAlgorithm::MergeJoinAlgorithm(
+    JoinPtr join_ptr,
+    const Blocks & input_headers,
+    size_t max_block_size_)
+    : MergeJoinAlgorithm(
+        join_ptr->getTableJoin().kind(),
+        join_ptr->getTableJoin().strictness(),
+        join_ptr->getTableJoin().getOnlyClause(),
+        input_headers,
+        max_block_size_)
+{
+    for (const auto & [left_key, right_key] : join_ptr->getTableJoin().leftToRightKeyRemap())
     {
         size_t left_idx = input_headers[0].getPositionByName(left_key);
         size_t right_idx = input_headers[1].getPositionByName(right_key);
@@ -398,7 +408,7 @@ struct AllJoinImpl
                 size_t lnum = nextDistinct(left_cursor.cursor);
                 size_t rnum = nextDistinct(right_cursor.cursor);
 
-                bool all_fit_in_block = std::max(left_map.size(), right_map.size()) + lnum * rnum <= max_block_size;
+                bool all_fit_in_block = !max_block_size || std::max(left_map.size(), right_map.size()) + lnum * rnum <= max_block_size;
                 bool have_all_ranges = left_cursor.cursor.isValid() && right_cursor.cursor.isValid();
                 if (all_fit_in_block && have_all_ranges)
                 {
@@ -498,7 +508,7 @@ std::optional<MergeJoinAlgorithm::Status> MergeJoinAlgorithm::handleAllJoinState
         }
 
         size_t total_rows = 0;
-        while (total_rows < max_block_size)
+        while (!max_block_size || total_rows < max_block_size)
         {
             const auto & left_range = all_join_state->getLeft();
             const auto & right_range = all_join_state->getRight();
@@ -523,7 +533,7 @@ std::optional<MergeJoinAlgorithm::Status> MergeJoinAlgorithm::handleAllJoinState
     return {};
 }
 
-MergeJoinAlgorithm::Status MergeJoinAlgorithm::allJoin(JoinKind kind)
+MergeJoinAlgorithm::Status MergeJoinAlgorithm::allJoin()
 {
     PaddedPODArray<UInt64> idx_map[2];
 
@@ -671,8 +681,6 @@ std::optional<MergeJoinAlgorithm::Status> MergeJoinAlgorithm::handleAnyJoinState
     if (any_join_state.empty())
         return {};
 
-    auto kind = table_join->getTableJoin().kind();
-
     Chunk result;
 
     for (size_t source_num = 0; source_num < 2; ++source_num)
@@ -717,7 +725,7 @@ std::optional<MergeJoinAlgorithm::Status> MergeJoinAlgorithm::handleAnyJoinState
     return {};
 }
 
-MergeJoinAlgorithm::Status MergeJoinAlgorithm::anyJoin(JoinKind kind)
+MergeJoinAlgorithm::Status MergeJoinAlgorithm::anyJoin()
 {
     if (auto result = handleAnyJoinState())
         return std::move(*result);
@@ -804,8 +812,6 @@ Chunk MergeJoinAlgorithm::createBlockWithDefaults(size_t source_num)
 
 IMergingAlgorithm::Status MergeJoinAlgorithm::merge()
 {
-    auto kind = table_join->getTableJoin().kind();
-
     if (!cursors[0]->cursor.isValid() && !cursors[0]->fullyCompleted())
         return Status(0);
 
@@ -849,13 +855,11 @@ IMergingAlgorithm::Status MergeJoinAlgorithm::merge()
         }
     }
 
-    auto strictness = table_join->getTableJoin().strictness();
-
     if (strictness == JoinStrictness::Any)
-        return anyJoin(kind);
+        return anyJoin();
 
     if (strictness == JoinStrictness::All)
-        return allJoin(kind);
+        return allJoin();
 
     throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Unsupported strictness '{}'", strictness);
 }
@@ -874,9 +878,30 @@ MergeJoinTransform::MergeJoinTransform(
         /* always_read_till_end_= */ false,
         /* empty_chunk_on_finish_= */ true,
         table_join, input_headers, max_block_size)
+<<<<<<< HEAD
     , log(getLogger("MergeJoinTransform"))
+=======
+>>>>>>> b4a16f38320 (Add simple unit test for full sorting join)
+{
+}
+
+MergeJoinTransform::MergeJoinTransform(
+        JoinKind kind_,
+        JoinStrictness strictness_,
+        const TableJoin::JoinOnClause & on_clause_,
+        const Blocks & input_headers,
+        const Block & output_header,
+        size_t max_block_size,
+        UInt64 limit_hint_)
+    : IMergingTransform<MergeJoinAlgorithm>(
+        input_headers,
+        output_header,
+        /* have_all_inputs_= */ true,
+        limit_hint_,
+        /* always_read_till_end_= */ false,
+        /* empty_chunk_on_finish_= */ true,
+        kind_, strictness_, on_clause_, input_headers, max_block_size)
 {
-    LOG_TRACE(log, "Use MergeJoinTransform");
 }
 
 void MergeJoinTransform::onFinish()
diff --git a/src/Processors/Transforms/MergeJoinTransform.h b/src/Processors/Transforms/MergeJoinTransform.h
index cf9331abd59..0b0efa33722 100644
--- a/src/Processors/Transforms/MergeJoinTransform.h
+++ b/src/Processors/Transforms/MergeJoinTransform.h
@@ -8,6 +8,7 @@
 #include <unordered_map>
 #include <utility>
 #include <vector>
+#include <array>
 
 #include <boost/core/noncopyable.hpp>
 
@@ -19,6 +20,7 @@
 #include <Processors/Chunk.h>
 #include <Processors/Merges/Algorithms/IMergingAlgorithm.h>
 #include <Processors/Merges/IMergingTransform.h>
+#include <Interpreters/TableJoin.h>
 
 namespace Poco { class Logger; }
 
@@ -227,7 +229,13 @@ private:
 class MergeJoinAlgorithm final : public IMergingAlgorithm
 {
 public:
-    explicit MergeJoinAlgorithm(JoinPtr table_join, const Blocks & input_headers, size_t max_block_size_);
+    MergeJoinAlgorithm(JoinKind kind_,
+                       JoinStrictness strictness_,
+                       const TableJoin::JoinOnClause & on_clause_,
+                       const Blocks & input_headers,
+                       size_t max_block_size_);
+
+    MergeJoinAlgorithm(JoinPtr join_ptr, const Blocks & input_headers, size_t max_block_size_);
 
     const char * getName() const override { return "MergeJoinAlgorithm"; }
     void initialize(Inputs inputs) override;
@@ -238,10 +246,10 @@ public:
 
 private:
     std::optional<Status> handleAnyJoinState();
-    Status anyJoin(JoinKind kind);
+    Status anyJoin();
 
     std::optional<Status> handleAllJoinState();
-    Status allJoin(JoinKind kind);
+    Status allJoin();
 
     Chunk createBlockWithDefaults(size_t source_num);
     Chunk createBlockWithDefaults(size_t source_num, size_t start, size_t num_rows) const;
@@ -251,11 +259,12 @@ private:
 
     std::array<FullMergeJoinCursorPtr, 2> cursors;
 
-    /// Keep some state to make connection between data in different blocks
+    /// Keep some state to make handle data from different blocks
     AnyJoinState any_join_state;
     std::unique_ptr<AllJoinState> all_join_state;
 
-    JoinPtr table_join;
+    JoinKind kind;
+    JoinStrictness strictness;
 
     size_t max_block_size;
     int null_direction_hint = 1;
@@ -285,12 +294,19 @@ public:
         size_t max_block_size,
         UInt64 limit_hint = 0);
 
+    MergeJoinTransform(
+        JoinKind kind_,
+        JoinStrictness strictness_,
+        const TableJoin::JoinOnClause & on_clause_,
+        const Blocks & input_headers,
+        const Block & output_header,
+        size_t max_block_size,
+        UInt64 limit_hint_ = 0);
+
     String getName() const override { return "MergeJoinTransform"; }
 
 protected:
     void onFinish() override;
-
-    LoggerPtr log;
 };
 
 }
diff --git a/src/Processors/tests/gtest_merge_join_algorithm.cpp b/src/Processors/tests/gtest_merge_join_algorithm.cpp
new file mode 100644
index 00000000000..9a8b70efc17
--- /dev/null
+++ b/src/Processors/tests/gtest_merge_join_algorithm.cpp
@@ -0,0 +1,95 @@
+#include <gtest/gtest.h>
+
+#include <Processors/Sources/SourceFromSingleChunk.h>
+#include <Processors/Sources/SourceFromChunks.h>
+#include <Processors/Sinks/NullSink.h>
+#include <Processors/Executors/PipelineExecutor.h>
+#include <Processors/Executors/PullingPipelineExecutor.h>
+#include <QueryPipeline/QueryPipeline.h>
+
+#include <Columns/ColumnsNumber.h>
+#include <DataTypes/DataTypesNumber.h>
+#include <Interpreters/TableJoin.h>
+
+#include <Processors/Transforms/MergeJoinTransform.h>
+
+
+using namespace DB;
+
+
+QueryPipeline buildJoinPipeline(std::shared_ptr<ISource> left_source, std::shared_ptr<ISource> right_source)
+{
+    Blocks inputs;
+    inputs.emplace_back(left_source->getPort().getHeader());
+    inputs.emplace_back(right_source->getPort().getHeader());
+    Block out_header = {
+        ColumnWithTypeAndName(ColumnUInt8::create(), std::make_shared<DataTypeUInt8>(), "t1.x"),
+        ColumnWithTypeAndName(ColumnUInt8::create(), std::make_shared<DataTypeUInt8>(), "t2.x"),
+    };
+
+    TableJoin::JoinOnClause on_clause;
+    on_clause.key_names_left = {"x"};
+    on_clause.key_names_right = {"x"};
+    auto joining = std::make_shared<MergeJoinTransform>(
+        JoinKind::Inner,
+        JoinStrictness::All,
+        on_clause,
+        inputs, out_header, /* max_block_size = */ 0);
+
+    chassert(joining->getInputs().size() == 2);
+
+    connect(left_source->getPort(), joining->getInputs().front());
+    connect(right_source->getPort(), joining->getInputs().back());
+
+    auto * output_port = &joining->getOutputPort();
+
+    auto processors = std::make_shared<Processors>();
+    processors->emplace_back(std::move(left_source));
+    processors->emplace_back(std::move(right_source));
+    processors->emplace_back(std::move(joining));
+
+    QueryPipeline pipeline(QueryPlanResourceHolder{}, processors, output_port);
+    return pipeline;
+}
+
+
+std::shared_ptr<ISource> createSourceWithSingleValue(size_t rows_per_chunk, size_t total_chunks)
+{
+    Block header = {
+        ColumnWithTypeAndName(ColumnUInt64::create(), std::make_shared<DataTypeUInt64>(), "x")
+    };
+
+    Chunks chunks;
+
+    for (size_t i = 0; i < total_chunks; ++i)
+    {
+        auto col = ColumnUInt64::create(rows_per_chunk, 1);
+        chunks.emplace_back(Columns{std::move(col)}, rows_per_chunk);
+    }
+
+    return std::make_shared<SourceFromChunks>(std::move(header), std::move(chunks));
+}
+
+TEST(FullSortingJoin, Simple)
+try
+{
+    auto left_source = createSourceWithSingleValue(3, 10);
+    auto right_source = createSourceWithSingleValue(2, 15);
+
+    auto pipeline = buildJoinPipeline(left_source, right_source);
+    PullingPipelineExecutor executor(pipeline);
+
+    Block block;
+
+    size_t total_result_rows = 0;
+    while (executor.pull(block))
+    {
+        total_result_rows += block.rows();
+    }
+    ASSERT_EQ(total_result_rows, 3 * 10 * 2 * 15);
+}
+catch (Exception & e)
+{
+    std::cout << e.getStackTraceString() << std::endl;
+    throw;
+}

From 2412f8521985c8d31322ed04baa502c4e7543ef6 Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Mon, 18 Sep 2023 16:14:02 +0000
Subject: [PATCH 0046/1488] wip full sorting asof join

---
 .../Transforms/MergeJoinTransform.cpp         | 138 ++++++++-
 .../Transforms/MergeJoinTransform.h           |  38 ++-
 .../tests/gtest_full_sorting_join.cpp         | 287 ++++++++++++++++++
 .../tests/gtest_merge_join_algorithm.cpp      |  95 ------
 4 files changed, 446 insertions(+), 112 deletions(-)
 create mode 100644 src/Processors/tests/gtest_full_sorting_join.cpp
 delete mode 100644 src/Processors/tests/gtest_merge_join_algorithm.cpp

diff --git a/src/Processors/Transforms/MergeJoinTransform.cpp b/src/Processors/Transforms/MergeJoinTransform.cpp
index a9fd7978249..8370e548fcb 100644
--- a/src/Processors/Transforms/MergeJoinTransform.cpp
+++ b/src/Processors/Transforms/MergeJoinTransform.cpp
@@ -34,13 +34,15 @@ namespace ErrorCodes
 namespace
 {
 
-FullMergeJoinCursorPtr createCursor(const Block & block, const Names & columns)
+constexpr UInt64 DEFAULT_VALUE_INDEX = std::numeric_limits<UInt64>::max();
+
+FullMergeJoinCursorPtr createCursor(const Block & block, const Names & columns, JoinStrictness strictness)
 {
     SortDescription desc;
     desc.reserve(columns.size());
     for (const auto & name : columns)
         desc.emplace_back(name);
-    return std::make_unique<FullMergeJoinCursor>(materializeBlock(block), desc);
+    return std::make_unique<FullMergeJoinCursor>(materializeBlock(block), desc, strictness == JoinStrictness::Asof);
 }
 
 template <bool has_left_nulls, bool has_right_nulls>
@@ -90,9 +92,10 @@ int nullableCompareAt(const IColumn & left_column, const IColumn & right_column,
 
 int ALWAYS_INLINE compareCursors(const SortCursorImpl & lhs, size_t lpos,
                                  const SortCursorImpl & rhs, size_t rpos,
+                                 size_t key_length,
                                  int null_direction_hint)
 {
-    for (size_t i = 0; i < lhs.sort_columns_size; ++i)
+    for (size_t i = 0; i < key_length; ++i)
     {
         /// TODO(@vdimir): use nullableCompareAt only if there's nullable columns
         int cmp = nullableCompareAt<true, true>(*lhs.sort_columns[i], *rhs.sort_columns[i], lpos, rpos, null_direction_hint);
@@ -104,13 +107,18 @@ int ALWAYS_INLINE compareCursors(const SortCursorImpl & lhs, size_t lpos,
 
 int ALWAYS_INLINE compareCursors(const SortCursorImpl & lhs, const SortCursorImpl & rhs, int null_direction_hint)
 {
-    return compareCursors(lhs, lhs.getRow(), rhs, rhs.getRow(), null_direction_hint);
+    return compareCursors(lhs, lhs.getRow(), rhs, rhs.getRow(), lhs.sort_columns_size, null_direction_hint);
+}
+
+int compareAsofCursors(const FullMergeJoinCursor & lhs, const FullMergeJoinCursor & rhs)
+{
+    return nullableCompareAt<false, false>(lhs.getAsofColumn(), rhs.getAsofColumn(), lhs->getRow(), rhs->getRow());
 }
 
 bool ALWAYS_INLINE totallyLess(SortCursorImpl & lhs, SortCursorImpl & rhs, int null_direction_hint)
 {
     /// The last row of left cursor is less than the current row of the right cursor.
-    int cmp = compareCursors(lhs, lhs.rows - 1, rhs, rhs.getRow(), null_direction_hint);
+    int cmp = compareCursors(lhs, lhs.rows - 1, rhs, rhs.getRow(), lhs.sort_columns_size, null_direction_hint);
     return cmp < 0;
 }
 
@@ -222,11 +230,11 @@ Chunk getRowFromChunk(const Chunk & chunk, size_t pos)
     return result;
 }
 
-void inline addRange(PaddedPODArray<UInt64> & left_map, size_t start, size_t end)
+void inline addRange(PaddedPODArray<UInt64> & values, UInt64 start, UInt64 end)
 {
     assert(end > start);
-    for (size_t i = start; i < end; ++i)
-        left_map.push_back(i);
+    for (UInt64 i = start; i < end; ++i)
+        values.push_back(i);
 }
 
 void inline addMany(PaddedPODArray<UInt64> & left_or_right_map, size_t idx, size_t num)
@@ -235,6 +243,11 @@ void inline addMany(PaddedPODArray<UInt64> & left_or_right_map, size_t idx, size
         left_or_right_map.push_back(idx);
 }
 
+void inline addMany(PaddedPODArray<UInt64> & values, UInt64 value, size_t num)
+{
+    values.resize_fill(values.size() + num, value);
+}
+
 }
 
 const Chunk & FullMergeJoinCursor::getCurrent() const
@@ -283,9 +296,15 @@ MergeJoinAlgorithm::MergeJoinAlgorithm(
     if (input_headers.size() != 2)
         throw Exception(ErrorCodes::LOGICAL_ERROR, "MergeJoinAlgorithm requires exactly two inputs");
 
-    if (strictness != JoinStrictness::Any && strictness != JoinStrictness::All)
+    if (strictness != JoinStrictness::Any && strictness != JoinStrictness::All && strictness != JoinStrictness::Asof)
         throw Exception(ErrorCodes::NOT_IMPLEMENTED, "MergeJoinAlgorithm is not implemented for strictness {}", strictness);
 
+    if (strictness == JoinStrictness::Asof)
+    {
+        if (kind != JoinKind::Left && kind != JoinKind::Inner)
+            throw Exception(ErrorCodes::NOT_IMPLEMENTED, "MergeJoinAlgorithm does not implement ASOF {} join", kind);
+    }
+
     if (!isInner(kind) && !isLeft(kind) && !isRight(kind) && !isFull(kind))
         throw Exception(ErrorCodes::NOT_IMPLEMENTED, "MergeJoinAlgorithm is not implemented for kind {}", kind);
 
@@ -293,8 +312,8 @@ MergeJoinAlgorithm::MergeJoinAlgorithm(
         throw Exception(ErrorCodes::NOT_IMPLEMENTED, "MergeJoinAlgorithm does not support ON filter conditions");
 
     cursors = {
-        createCursor(input_headers[0], on_clause_.key_names_left),
-        createCursor(input_headers[1], on_clause_.key_names_right)
+        createCursor(input_headers[0], on_clause_.key_names_left,  strictness),
+        createCursor(input_headers[1], on_clause_.key_names_right,  strictness),
     };
 
 MergeJoinAlgorithm::MergeJoinAlgorithm(
@@ -313,6 +332,8 @@ MergeJoinAlgorithm::MergeJoinAlgorithm(
         size_t left_idx = input_headers[0].getPositionByName(left_key);
         size_t right_idx = input_headers[1].getPositionByName(right_key);
         left_to_right_key_remap[left_idx] = right_idx;
+        if (strictness == JoinStrictness::Asof)
+            throw Exception(ErrorCodes::NOT_IMPLEMENTED, "MergeJoinAlgorithm does not support ASOF joins USING");
     }
 
     const auto *smjPtr = typeid_cast<const FullSortingMergeJoin *>(table_join.get());
@@ -321,6 +342,19 @@ MergeJoinAlgorithm::MergeJoinAlgorithm(
         null_direction_hint = smjPtr->getNullDirection();
     }
 
+    if (strictness == JoinStrictness::Asof)
+        setAsofInequality(join_ptr->getTableJoin().getAsofInequality());
+}
+
+void MergeJoinAlgorithm::setAsofInequality(ASOFJoinInequality asof_inequality_)
+{
+    if (strictness != JoinStrictness::Asof)
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "setAsofInequality is only supported for ASOF joins");
+
+    if (asof_inequality_ == ASOFJoinInequality::None)
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "ASOF inequality cannot be None");
+
+    asof_inequality = asof_inequality_;
 }
 
 void MergeJoinAlgorithm::logElapsed(double seconds)
@@ -770,6 +804,81 @@ MergeJoinAlgorithm::Status MergeJoinAlgorithm::anyJoin()
     return Status(std::move(result));
 }
 
+
+MergeJoinAlgorithm::Status MergeJoinAlgorithm::asofJoin()
+{
+    auto & left_cursor = *cursors[0];
+    if (!left_cursor->isValid())
+        return Status(0);
+
+    auto & right_cursor = *cursors[1];
+    if (!right_cursor->isValid())
+        return Status(1);
+
+    PaddedPODArray<UInt64> left_map;
+    PaddedPODArray<UInt64> right_map;
+
+    while (left_cursor->isValid() && right_cursor->isValid())
+    {
+        auto lpos = left_cursor->getRow();
+        auto rpos = right_cursor->getRow();
+        auto cmp = compareCursors(*left_cursor, *right_cursor);
+        if (cmp == 0)
+        {
+            auto asof_cmp = compareAsofCursors(left_cursor, right_cursor);
+            if ((asof_inequality == ASOFJoinInequality::Less && asof_cmp <= -1)
+             || (asof_inequality == ASOFJoinInequality::LessOrEquals && asof_cmp <= 0))
+            {
+                /// First row in right table that is greater (or equal) than current row in left table
+                /// matches asof join condition the best
+                left_map.push_back(lpos);
+                right_map.push_back(rpos);
+                left_cursor->next();
+                continue;
+            }
+
+            if (asof_inequality == ASOFJoinInequality::Less || asof_inequality == ASOFJoinInequality::LessOrEquals)
+            {
+                /// Asof condition is not (yet) satisfied, skip row in right table
+                right_cursor->next();
+                continue;
+            }
+            throw Exception(ErrorCodes::NOT_IMPLEMENTED, "TODO: implement ASOF equality join");
+        }
+        else if (cmp < 0)
+        {
+            /// no matches for rows in left table, just pass them through
+            size_t num = nextDistinct(*left_cursor);
+            if (isLeft(kind))
+            {
+                /// return them with default values at right side
+                addRange(left_map, lpos, lpos + num);
+                addMany(right_map, DEFAULT_VALUE_INDEX, num);
+            }
+        }
+        else
+        {
+            /// skip rows in right table until we find match for current row in left table
+            nextDistinct(*right_cursor);
+        }
+    }
+
+    chassert(left_map.size() == right_map.size());
+    Chunk result;
+    {
+        Columns lcols = indexColumns(left_cursor.getCurrent().getColumns(), left_map);
+        for (auto & col : lcols)
+            result.addColumn(std::move(col));
+
+        Columns rcols = indexColumns(right_cursor.getCurrent().getColumns(), right_map);
+        for (auto & col : rcols)
+            result.addColumn(std::move(col));
+    }
+    UNUSED(asof_inequality);
+    return Status(std::move(result));
+}
+
+
 /// if `source_num == 0` get data from left cursor and fill defaults at right
 /// otherwise - vice versa
 Chunk MergeJoinAlgorithm::createBlockWithDefaults(size_t source_num, size_t start, size_t num_rows) const
@@ -861,6 +970,9 @@ IMergingAlgorithm::Status MergeJoinAlgorithm::merge()
     if (strictness == JoinStrictness::All)
         return allJoin();
 
+    if (strictness == JoinStrictness::Asof)
+        return asofJoin();
+
     throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Unsupported strictness '{}'", strictness);
 }
 
@@ -878,10 +990,6 @@ MergeJoinTransform::MergeJoinTransform(
         /* always_read_till_end_= */ false,
         /* empty_chunk_on_finish_= */ true,
         table_join, input_headers, max_block_size)
-<<<<<<< HEAD
-    , log(getLogger("MergeJoinTransform"))
-=======
->>>>>>> b4a16f38320 (Add simple unit test for full sorting join)
 {
 }
 
diff --git a/src/Processors/Transforms/MergeJoinTransform.h b/src/Processors/Transforms/MergeJoinTransform.h
index 0b0efa33722..3ee01e57992 100644
--- a/src/Processors/Transforms/MergeJoinTransform.h
+++ b/src/Processors/Transforms/MergeJoinTransform.h
@@ -195,10 +195,27 @@ private:
 class FullMergeJoinCursor : boost::noncopyable
 {
 public:
-    explicit FullMergeJoinCursor(const Block & sample_block_, const SortDescription & description_)
+    FullMergeJoinCursor(
+        const Block & sample_block_,
+        const SortDescription & description_,
+        bool is_asof = false)
         : sample_block(sample_block_.cloneEmpty())
         , desc(description_)
     {
+        if (desc.size() == 0)
+            throw Exception(ErrorCodes::LOGICAL_ERROR, "Got empty sort description for FullMergeJoinCursor");
+
+        if (is_asof)
+        {
+            /// For ASOF join prefix of sort description is used for equality comparison
+            /// and the last column is used for inequality comparison and is handled separately
+
+            auto asof_column_description = desc.back();
+            desc.pop_back();
+
+            chassert(asof_column_description.direction == 1 && asof_column_description.nulls_direction == 1);
+            asof_column_position = sample_block.getPositionByName(asof_column_description.column_name);
+        }
     }
 
     bool fullyCompleted() const;
@@ -209,17 +226,27 @@ public:
     SortCursorImpl * operator-> () { return &cursor; }
     const SortCursorImpl * operator-> () const { return &cursor; }
 
+    SortCursorImpl & operator* () { return cursor; }
+    const SortCursorImpl & operator* () const { return cursor; }
+
     SortCursorImpl cursor;
 
     const Block & sampleBlock() const { return sample_block; }
     Columns sampleColumns() const { return sample_block.getColumns(); }
 
+    const IColumn & getAsofColumn() const
+    {
+        return *cursor.all_columns[asof_column_position];
+    }
+
 private:
     Block sample_block;
     SortDescription desc;
 
     Chunk current_chunk;
     bool recieved_all_blocks = false;
+
+    size_t asof_column_position;
 };
 
 /*
@@ -242,8 +269,9 @@ public:
     void consume(Input & input, size_t source_num) override;
     Status merge() override;
 
-    void logElapsed(double seconds);
+    void setAsofInequality(ASOFJoinInequality asof_inequality_);
 
+    void logElapsed(double seconds);
 private:
     std::optional<Status> handleAnyJoinState();
     Status anyJoin();
@@ -251,13 +279,17 @@ private:
     std::optional<Status> handleAllJoinState();
     Status allJoin();
 
+    Status asofJoin();
+
     Chunk createBlockWithDefaults(size_t source_num);
     Chunk createBlockWithDefaults(size_t source_num, size_t start, size_t num_rows) const;
 
+
     /// For `USING` join key columns should have values from right side instead of defaults
     std::unordered_map<size_t, size_t> left_to_right_key_remap;
 
     std::array<FullMergeJoinCursorPtr, 2> cursors;
+    ASOFJoinInequality asof_inequality = ASOFJoinInequality::None;
 
     /// Keep some state to make handle data from different blocks
     AnyJoinState any_join_state;
@@ -305,6 +337,8 @@ public:
 
     String getName() const override { return "MergeJoinTransform"; }
 
+    void setAsofInequality(ASOFJoinInequality asof_inequality_) { algorithm.setAsofInequality(asof_inequality_); }
+
 protected:
     void onFinish() override;
 };
diff --git a/src/Processors/tests/gtest_full_sorting_join.cpp b/src/Processors/tests/gtest_full_sorting_join.cpp
new file mode 100644
index 00000000000..888e280b55f
--- /dev/null
+++ b/src/Processors/tests/gtest_full_sorting_join.cpp
@@ -0,0 +1,287 @@
+#include <gtest/gtest.h>
+
+
+#include <Common/randomSeed.h>
+#include <pcg_random.hpp>
+#include <random>
+
+#include <Processors/Sources/SourceFromSingleChunk.h>
+#include <Processors/Sources/SourceFromChunks.h>
+#include <Processors/Sinks/NullSink.h>
+#include <Processors/Executors/PipelineExecutor.h>
+#include <Processors/Executors/PullingPipelineExecutor.h>
+#include <QueryPipeline/QueryPipeline.h>
+
+#include <Columns/ColumnsNumber.h>
+#include <DataTypes/DataTypesNumber.h>
+#include <Interpreters/TableJoin.h>
+
+#include <Processors/Transforms/MergeJoinTransform.h>
+
+
+using namespace DB;
+
+UInt64 getAndPrintRandomSeed()
+{
+    UInt64 seed = randomSeed();
+    std::cerr << "TEST_RANDOM_SEED: " << seed << std::endl;
+    return seed;
+}
+
+static UInt64 TEST_RANDOM_SEED = getAndPrintRandomSeed();
+
+static pcg64 rng(TEST_RANDOM_SEED);
+
+
+QueryPipeline buildJoinPipeline(
+    std::shared_ptr<ISource> left_source,
+    std::shared_ptr<ISource> right_source,
+    size_t key_length = 1,
+    JoinKind kind = JoinKind::Inner,
+    JoinStrictness strictness = JoinStrictness::All,
+    ASOFJoinInequality asof_inequality = ASOFJoinInequality::None)
+{
+    Blocks inputs;
+    inputs.emplace_back(left_source->getPort().getHeader());
+    inputs.emplace_back(right_source->getPort().getHeader());
+
+    Block out_header;
+    for (const auto & input : inputs)
+    {
+        for (ColumnWithTypeAndName column : input)
+        {
+            if (&input == &inputs.front())
+                column.name = "t1." + column.name;
+            else
+                column.name = "t2." + column.name;
+            out_header.insert(column);
+        }
+    }
+
+    TableJoin::JoinOnClause on_clause;
+    for (size_t i = 0; i < key_length; ++i)
+    {
+        on_clause.key_names_left.emplace_back(inputs[0].getByPosition(i).name);
+        on_clause.key_names_right.emplace_back(inputs[1].getByPosition(i).name);
+    }
+
+    auto joining = std::make_shared<MergeJoinTransform>(
+        kind,
+        strictness,
+        on_clause,
+        inputs, out_header, /* max_block_size = */ 0);
+
+    if (asof_inequality != ASOFJoinInequality::None)
+        joining->setAsofInequality(asof_inequality);
+
+    chassert(joining->getInputs().size() == 2);
+
+    connect(left_source->getPort(), joining->getInputs().front());
+    connect(right_source->getPort(), joining->getInputs().back());
+
+    auto * output_port = &joining->getOutputPort();
+
+    auto processors = std::make_shared<Processors>();
+    processors->emplace_back(std::move(left_source));
+    processors->emplace_back(std::move(right_source));
+    processors->emplace_back(std::move(joining));
+
+    QueryPipeline pipeline(QueryPlanResourceHolder{}, processors, output_port);
+    return pipeline;
+}
+
+
+std::shared_ptr<ISource> oneColumnSource(const std::vector<std::vector<UInt64>> & values)
+{
+    Block header = { ColumnWithTypeAndName(ColumnUInt64::create(), std::make_shared<DataTypeUInt64>(), "x") };
+    Chunks chunks;
+    for (const auto & chunk_values : values)
+    {
+        auto column = ColumnUInt64::create();
+        for (auto n : chunk_values)
+            column->insertValue(n);
+        chunks.emplace_back(Chunk(Columns{std::move(column)}, chunk_values.size()));
+    }
+    return std::make_shared<SourceFromChunks>(header, std::move(chunks));
+}
+
+
+TEST(FullSortingJoin, Simple)
+try
+{
+    auto left_source = oneColumnSource({ {1, 2, 3, 4, 5} });
+    auto right_source = oneColumnSource({ {1}, {2}, {3}, {4}, {5} });
+
+    auto pipeline = buildJoinPipeline(left_source, right_source);
+    PullingPipelineExecutor executor(pipeline);
+
+    Block block;
+
+    size_t total_result_rows = 0;
+    while (executor.pull(block))
+        total_result_rows += block.rows();
+
+    ASSERT_EQ(total_result_rows, 5);
+}
+catch (Exception & e)
+{
+    std::cout << e.getStackTraceString() << std::endl;
+    throw;
+}
+
+std::shared_ptr<ISource> sourceFromRows(
+    const Block & header, const std::vector<std::vector<Field>> & values, double break_prob = 0.0)
+{
+    Chunks chunks;
+    auto columns = header.cloneEmptyColumns();
+
+    std::uniform_real_distribution<> prob_dis(0.0, 1.0);
+
+
+    for (auto row : values)
+    {
+        if (!columns.empty() && (row.empty() || prob_dis(rng) < break_prob))
+        {
+            size_t rows = columns.front()->size();
+            chunks.emplace_back(std::move(columns), rows);
+            columns = header.cloneEmptyColumns();
+            continue;
+        }
+
+        for (size_t i = 0; i < columns.size(); ++i)
+            columns[i]->insert(row[i]);
+    }
+
+    if (!columns.empty())
+        chunks.emplace_back(std::move(columns), columns.front()->size());
+
+    return std::make_shared<SourceFromChunks>(header, std::move(chunks));
+}
+
+
+std::vector<std::vector<Field>> getValuesFromBlock(const Block & block, const Names & names)
+{
+    std::vector<std::vector<Field>> result;
+    for (size_t i = 0; i < block.rows(); ++i)
+    {
+        auto & row = result.emplace_back();
+        for (const auto & name : names)
+            block.getByName(name).column->get(i, row.emplace_back());
+    }
+    return result;
+}
+
+
+Block executePipeline(QueryPipeline & pipeline)
+{
+    PullingPipelineExecutor executor(pipeline);
+
+    Blocks result_blocks;
+    while (true)
+    {
+        Block block;
+        bool is_ok = executor.pull(block);
+        if (!is_ok)
+            break;
+        result_blocks.emplace_back(std::move(block));
+    }
+
+    return concatenateBlocks(result_blocks);
+}
+
+TEST(FullSortingJoin, Asof)
+try
+{
+    const std::vector<Field> chunk_break = {};
+
+    auto left_source = sourceFromRows({
+        ColumnWithTypeAndName(ColumnString::create(), std::make_shared<DataTypeString>(), "key"),
+        ColumnWithTypeAndName(ColumnUInt64::create(), std::make_shared<DataTypeUInt64>(), "t"),
+    }, {
+        {"AMZN", 3},
+        {"AMZN", 4},
+        {"AMZN", 6},
+    });
+
+    auto right_source = sourceFromRows({
+        ColumnWithTypeAndName(ColumnString::create(), std::make_shared<DataTypeString>(), "key"),
+        ColumnWithTypeAndName(ColumnUInt64::create(), std::make_shared<DataTypeUInt64>(), "t"),
+        ColumnWithTypeAndName(ColumnUInt64::create(), std::make_shared<DataTypeUInt64>(), "value"),
+    }, {
+        {"AAPL", 1, 97},
+        chunk_break,
+        {"AAPL", 2, 98},
+        {"AAPL", 3, 99},
+        {"AMZN", 1, 100},
+        {"AMZN", 2, 110},
+        chunk_break,
+        {"AMZN", 4, 130},
+        {"AMZN", 5, 140},
+    });
+
+    auto pipeline = buildJoinPipeline(
+        left_source, right_source, /* key_length = */ 2,
+        JoinKind::Inner, JoinStrictness::Asof, ASOFJoinInequality::LessOrEquals);
+
+    Block result_block = executePipeline(pipeline);
+    auto values = getValuesFromBlock(result_block, {"t1.key", "t1.t", "t2.t", "t2.value"});
+    ASSERT_EQ(values.size(), 2);
+    ASSERT_EQ(values[0], (std::vector<Field>{"AMZN", 3u, 4u, 130u}));
+    ASSERT_EQ(values[1], (std::vector<Field>{"AMZN", 4u, 4u, 130u}));
+}
+catch (Exception & e)
+{
+    std::cout << e.getStackTraceString() << std::endl;
+    throw;
+}
+
+
+TEST(FullSortingJoin, AsofOnlyColumn)
+try
+{
+    const std::vector<Field> chunk_break = {};
+
+    auto left_source = oneColumnSource({ {3}, {3, 3, 3}, {3, 5, 5, 6}, {9, 9}, {10, 20} });
+
+    UInt64 p = std::uniform_int_distribution<>(0, 2)(rng);
+    double break_prob = p == 0 ? 0.0 : (p == 1 ? 0.5 : 1.0);
+
+    auto right_source = sourceFromRows({
+        ColumnWithTypeAndName(ColumnUInt64::create(), std::make_shared<DataTypeUInt64>(), "t"),
+        ColumnWithTypeAndName(ColumnUInt64::create(), std::make_shared<DataTypeUInt64>(), "value"),
+    }, {
+        {1, 101},
+        {2, 102},
+        {4, 104},
+        {5, 105},
+        {11, 111},
+        {15, 115},
+    },
+    break_prob);
+
+    auto pipeline = buildJoinPipeline(
+        left_source, right_source, /* key_length = */ 1,
+        JoinKind::Inner, JoinStrictness::Asof, ASOFJoinInequality::LessOrEquals);
+
+    Block result_block = executePipeline(pipeline);
+
+    ASSERT_EQ(
+        assert_cast<const ColumnUInt64 *>(result_block.getByName("t1.x").column.get())->getData(),
+        (ColumnUInt64::Container{3, 3, 3, 3, 3, 5, 5, 6, 9, 9, 10})
+    );
+
+    ASSERT_EQ(
+        assert_cast<const ColumnUInt64 *>(result_block.getByName("t2.t").column.get())->getData(),
+        (ColumnUInt64::Container{4, 4, 4, 4, 4, 5, 5, 11, 11, 11, 15})
+    );
+
+    ASSERT_EQ(
+        assert_cast<const ColumnUInt64 *>(result_block.getByName("t2.value").column.get())->getData(),
+        (ColumnUInt64::Container{104, 104, 104, 104, 104, 105, 105, 111, 111, 111, 115})
+    );
+}
+catch (Exception & e)
+{
+    std::cout << e.getStackTraceString() << std::endl;
+    throw;
+}
diff --git a/src/Processors/tests/gtest_merge_join_algorithm.cpp b/src/Processors/tests/gtest_merge_join_algorithm.cpp
deleted file mode 100644
index 9a8b70efc17..00000000000
--- a/src/Processors/tests/gtest_merge_join_algorithm.cpp
+++ /dev/null
@@ -1,95 +0,0 @@
-#include <gtest/gtest.h>
-
-#include <Processors/Sources/SourceFromSingleChunk.h>
-#include <Processors/Sources/SourceFromChunks.h>
-#include <Processors/Sinks/NullSink.h>
-#include <Processors/Executors/PipelineExecutor.h>
-#include <Processors/Executors/PullingPipelineExecutor.h>
-#include <QueryPipeline/QueryPipeline.h>
-
-#include <Columns/ColumnsNumber.h>
-#include <DataTypes/DataTypesNumber.h>
-#include <Interpreters/TableJoin.h>
-
-#include <Processors/Transforms/MergeJoinTransform.h>
-
-
-using namespace DB;
-
-
-QueryPipeline buildJoinPipeline(std::shared_ptr<ISource> left_source, std::shared_ptr<ISource> right_source)
-{
-    Blocks inputs;
-    inputs.emplace_back(left_source->getPort().getHeader());
-    inputs.emplace_back(right_source->getPort().getHeader());
-    Block out_header = {
-        ColumnWithTypeAndName(ColumnUInt8::create(), std::make_shared<DataTypeUInt8>(), "t1.x"),
-        ColumnWithTypeAndName(ColumnUInt8::create(), std::make_shared<DataTypeUInt8>(), "t2.x"),
-    };
-
-    TableJoin::JoinOnClause on_clause;
-    on_clause.key_names_left = {"x"};
-    on_clause.key_names_right = {"x"};
-    auto joining = std::make_shared<MergeJoinTransform>(
-        JoinKind::Inner,
-        JoinStrictness::All,
-        on_clause,
-        inputs, out_header, /* max_block_size = */ 0);
-
-    chassert(joining->getInputs().size() == 2);
-
-    connect(left_source->getPort(), joining->getInputs().front());
-    connect(right_source->getPort(), joining->getInputs().back());
-
-    auto * output_port = &joining->getOutputPort();
-
-    auto processors = std::make_shared<Processors>();
-    processors->emplace_back(std::move(left_source));
-    processors->emplace_back(std::move(right_source));
-    processors->emplace_back(std::move(joining));
-
-    QueryPipeline pipeline(QueryPlanResourceHolder{}, processors, output_port);
-    return pipeline;
-}
-
-
-std::shared_ptr<ISource> createSourceWithSingleValue(size_t rows_per_chunk, size_t total_chunks)
-{
-    Block header = {
-        ColumnWithTypeAndName(ColumnUInt64::create(), std::make_shared<DataTypeUInt64>(), "x")
-    };
-
-    Chunks chunks;
-
-    for (size_t i = 0; i < total_chunks; ++i)
-    {
-        auto col = ColumnUInt64::create(rows_per_chunk, 1);
-        chunks.emplace_back(Columns{std::move(col)}, rows_per_chunk);
-    }
-
-    return std::make_shared<SourceFromChunks>(std::move(header), std::move(chunks));
-}
-
-TEST(FullSortingJoin, Simple)
-try
-{
-    auto left_source = createSourceWithSingleValue(3, 10);
-    auto right_source = createSourceWithSingleValue(2, 15);
-
-    auto pipeline = buildJoinPipeline(left_source, right_source);
-    PullingPipelineExecutor executor(pipeline);
-
-    Block block;
-
-    size_t total_result_rows = 0;
-    while (executor.pull(block))
-    {
-        total_result_rows += block.rows();
-    }
-    ASSERT_EQ(total_result_rows, 3 * 10 * 2 * 15);
-}
-catch (Exception & e)
-{
-    std::cout << e.getStackTraceString() << std::endl;
-    throw;
-}

From 4079f25a3865f23f23cec9c43aefe241b58e7972 Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Wed, 20 Sep 2023 12:33:29 +0000
Subject: [PATCH 0047/1488] fix test

---
 .../tests/gtest_full_sorting_join.cpp         | 35 +++++++++++++------
 1 file changed, 24 insertions(+), 11 deletions(-)

diff --git a/src/Processors/tests/gtest_full_sorting_join.cpp b/src/Processors/tests/gtest_full_sorting_join.cpp
index 888e280b55f..b3b462ee1e3 100644
--- a/src/Processors/tests/gtest_full_sorting_join.cpp
+++ b/src/Processors/tests/gtest_full_sorting_join.cpp
@@ -134,26 +134,37 @@ std::shared_ptr<ISource> sourceFromRows(
 {
     Chunks chunks;
     auto columns = header.cloneEmptyColumns();
+    chassert(!columns.empty());
 
     std::uniform_real_distribution<> prob_dis(0.0, 1.0);
 
-
-    for (auto row : values)
+    for (const auto & row : values)
     {
-        if (!columns.empty() && (row.empty() || prob_dis(rng) < break_prob))
+        if (!columns.front()->empty() && (row.empty() || prob_dis(rng) < break_prob))
         {
             size_t rows = columns.front()->size();
             chunks.emplace_back(std::move(columns), rows);
             columns = header.cloneEmptyColumns();
-            continue;
+            if (row.empty())
+                continue;
         }
 
+        chassert(row.size() == columns.size());
         for (size_t i = 0; i < columns.size(); ++i)
             columns[i]->insert(row[i]);
     }
 
-    if (!columns.empty())
-        chunks.emplace_back(std::move(columns), columns.front()->size());
+    if (!columns.front()->empty())
+    {
+        size_t rows = columns.front()->size();
+        chunks.emplace_back(std::move(columns), rows);
+    }
+
+    /// Check that code above is correct.
+    size_t total_result_rows = 0;
+    for (const auto & chunk : chunks)
+        total_result_rows += chunk.getNumRows();
+    chassert(total_result_rows == values.size());
 
     return std::make_shared<SourceFromChunks>(header, std::move(chunks));
 }
@@ -225,9 +236,11 @@ try
 
     Block result_block = executePipeline(pipeline);
     auto values = getValuesFromBlock(result_block, {"t1.key", "t1.t", "t2.t", "t2.value"});
-    ASSERT_EQ(values.size(), 2);
-    ASSERT_EQ(values[0], (std::vector<Field>{"AMZN", 3u, 4u, 130u}));
-    ASSERT_EQ(values[1], (std::vector<Field>{"AMZN", 4u, 4u, 130u}));
+
+    ASSERT_EQ(values, (std::vector<std::vector<Field>>{
+        {"AMZN", 3u, 4u, 130u},
+        {"AMZN", 4u, 4u, 130u},
+    }));
 }
 catch (Exception & e)
 {
@@ -272,12 +285,12 @@ try
 
     ASSERT_EQ(
         assert_cast<const ColumnUInt64 *>(result_block.getByName("t2.t").column.get())->getData(),
-        (ColumnUInt64::Container{4, 4, 4, 4, 4, 5, 5, 11, 11, 11, 15})
+        (ColumnUInt64::Container{4, 4, 4, 4, 4, 5, 5, 11, 11, 11, 11})
     );
 
     ASSERT_EQ(
         assert_cast<const ColumnUInt64 *>(result_block.getByName("t2.value").column.get())->getData(),
-        (ColumnUInt64::Container{104, 104, 104, 104, 104, 105, 105, 111, 111, 111, 115})
+        (ColumnUInt64::Container{104, 104, 104, 104, 104, 105, 105, 111, 111, 111, 111})
     );
 }
 catch (Exception & e)

From 984d94e5f10e7e1c3cc953e1f5394c153e8fdedc Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Wed, 20 Sep 2023 15:46:19 +0000
Subject: [PATCH 0048/1488] upd gtest_full_sorting_join

---
 .../tests/gtest_full_sorting_join.cpp         | 154 +++++++++---------
 1 file changed, 76 insertions(+), 78 deletions(-)

diff --git a/src/Processors/tests/gtest_full_sorting_join.cpp b/src/Processors/tests/gtest_full_sorting_join.cpp
index b3b462ee1e3..4d7ce25a7e8 100644
--- a/src/Processors/tests/gtest_full_sorting_join.cpp
+++ b/src/Processors/tests/gtest_full_sorting_join.cpp
@@ -93,7 +93,7 @@ QueryPipeline buildJoinPipeline(
 
 std::shared_ptr<ISource> oneColumnSource(const std::vector<std::vector<UInt64>> & values)
 {
-    Block header = { ColumnWithTypeAndName(ColumnUInt64::create(), std::make_shared<DataTypeUInt64>(), "x") };
+    Block header = { ColumnWithTypeAndName(std::make_shared<DataTypeUInt64>(), "x") };
     Chunks chunks;
     for (const auto & chunk_values : values)
     {
@@ -129,45 +129,46 @@ catch (Exception & e)
     throw;
 }
 
-std::shared_ptr<ISource> sourceFromRows(
-    const Block & header, const std::vector<std::vector<Field>> & values, double break_prob = 0.0)
+class SourceChunksBuilder
 {
+public:
+    explicit SourceChunksBuilder(const Block & header_)
+        : header(header_)
+    {
+        current_chunk = header.cloneEmptyColumns();
+        chassert(!current_chunk.empty());
+    }
+
+    SourceChunksBuilder & addRow(const std::vector<Field> & row)
+    {
+        chassert(row.size() == current_chunk.size());
+        for (size_t i = 0; i < current_chunk.size(); ++i)
+            current_chunk[i]->insert(row[i]);
+        return *this;
+    }
+
+    SourceChunksBuilder & addChunk()
+    {
+        if (current_chunk.front()->empty())
+            return *this;
+
+        size_t rows = current_chunk.front()->size();
+        chunks.emplace_back(std::move(current_chunk), rows);
+        current_chunk = header.cloneEmptyColumns();
+        return *this;
+    }
+
+    std::shared_ptr<ISource> build()
+    {
+        addChunk();
+        return std::make_shared<SourceFromChunks>(header, std::move(chunks));
+    }
+
+private:
+    Block header;
     Chunks chunks;
-    auto columns = header.cloneEmptyColumns();
-    chassert(!columns.empty());
-
-    std::uniform_real_distribution<> prob_dis(0.0, 1.0);
-
-    for (const auto & row : values)
-    {
-        if (!columns.front()->empty() && (row.empty() || prob_dis(rng) < break_prob))
-        {
-            size_t rows = columns.front()->size();
-            chunks.emplace_back(std::move(columns), rows);
-            columns = header.cloneEmptyColumns();
-            if (row.empty())
-                continue;
-        }
-
-        chassert(row.size() == columns.size());
-        for (size_t i = 0; i < columns.size(); ++i)
-            columns[i]->insert(row[i]);
-    }
-
-    if (!columns.front()->empty())
-    {
-        size_t rows = columns.front()->size();
-        chunks.emplace_back(std::move(columns), rows);
-    }
-
-    /// Check that code above is correct.
-    size_t total_result_rows = 0;
-    for (const auto & chunk : chunks)
-        total_result_rows += chunk.getNumRows();
-    chassert(total_result_rows == values.size());
-
-    return std::make_shared<SourceFromChunks>(header, std::move(chunks));
-}
+    MutableColumns current_chunk;
+};
 
 
 std::vector<std::vector<Field>> getValuesFromBlock(const Block & block, const Names & names)
@@ -203,32 +204,30 @@ Block executePipeline(QueryPipeline & pipeline)
 TEST(FullSortingJoin, Asof)
 try
 {
-    const std::vector<Field> chunk_break = {};
+    auto left_source = SourceChunksBuilder({
+        {std::make_shared<DataTypeString>(), "key"},
+        {std::make_shared<DataTypeUInt64>(), "t"},
+    })
+        .addRow({"AMZN", 3})
+        .addRow({"AMZN", 4})
+        .addRow({"AMZN", 6})
+        .build();
 
-    auto left_source = sourceFromRows({
-        ColumnWithTypeAndName(ColumnString::create(), std::make_shared<DataTypeString>(), "key"),
-        ColumnWithTypeAndName(ColumnUInt64::create(), std::make_shared<DataTypeUInt64>(), "t"),
-    }, {
-        {"AMZN", 3},
-        {"AMZN", 4},
-        {"AMZN", 6},
-    });
-
-    auto right_source = sourceFromRows({
-        ColumnWithTypeAndName(ColumnString::create(), std::make_shared<DataTypeString>(), "key"),
-        ColumnWithTypeAndName(ColumnUInt64::create(), std::make_shared<DataTypeUInt64>(), "t"),
-        ColumnWithTypeAndName(ColumnUInt64::create(), std::make_shared<DataTypeUInt64>(), "value"),
-    }, {
-        {"AAPL", 1, 97},
-        chunk_break,
-        {"AAPL", 2, 98},
-        {"AAPL", 3, 99},
-        {"AMZN", 1, 100},
-        {"AMZN", 2, 110},
-        chunk_break,
-        {"AMZN", 4, 130},
-        {"AMZN", 5, 140},
-    });
+    auto right_source = SourceChunksBuilder({
+        {std::make_shared<DataTypeString>(), "key"},
+        {std::make_shared<DataTypeUInt64>(), "t"},
+        {std::make_shared<DataTypeUInt64>(), "value"},
+    })
+        .addRow({"AAPL", 1, 97})
+        .addChunk()
+        .addRow({"AAPL", 2, 98})
+        .addRow({"AAPL", 3, 99})
+        .addRow({"AMZN", 1, 100})
+        .addRow({"AMZN", 2, 110})
+        .addChunk()
+        .addRow({"AMZN", 4, 130})
+        .addRow({"AMZN", 5, 140})
+        .build();
 
     auto pipeline = buildJoinPipeline(
         left_source, right_source, /* key_length = */ 2,
@@ -252,25 +251,24 @@ catch (Exception & e)
 TEST(FullSortingJoin, AsofOnlyColumn)
 try
 {
-    const std::vector<Field> chunk_break = {};
-
     auto left_source = oneColumnSource({ {3}, {3, 3, 3}, {3, 5, 5, 6}, {9, 9}, {10, 20} });
 
     UInt64 p = std::uniform_int_distribution<>(0, 2)(rng);
-    double break_prob = p == 0 ? 0.0 : (p == 1 ? 0.5 : 1.0);
 
-    auto right_source = sourceFromRows({
-        ColumnWithTypeAndName(ColumnUInt64::create(), std::make_shared<DataTypeUInt64>(), "t"),
-        ColumnWithTypeAndName(ColumnUInt64::create(), std::make_shared<DataTypeUInt64>(), "value"),
-    }, {
-        {1, 101},
-        {2, 102},
-        {4, 104},
-        {5, 105},
-        {11, 111},
-        {15, 115},
-    },
-    break_prob);
+    SourceChunksBuilder right_source_builder({
+        {std::make_shared<DataTypeUInt64>(), "t"},
+        {std::make_shared<DataTypeUInt64>(), "value"},
+    });
+
+    double break_prob = p == 0 ? 0.0 : (p == 1 ? 0.5 : 1.0);
+    std::uniform_real_distribution<> prob_dis(0.0, 1.0);
+    for (const auto & row : std::vector<std::vector<Field>>{ {1, 101}, {2, 102}, {4, 104}, {5, 105}, {11, 111}, {15, 115} })
+    {
+        right_source_builder.addRow(row);
+        if (prob_dis(rng) < break_prob)
+            right_source_builder.addChunk();
+    }
+    auto right_source = right_source_builder.build();
 
     auto pipeline = buildJoinPipeline(
         left_source, right_source, /* key_length = */ 1,

From 6330b466aa326e40eb7abca2699554c98241b342 Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Thu, 21 Sep 2023 11:19:20 +0000
Subject: [PATCH 0049/1488] Add randomized test
 FullSortingJoin.AsofGeneratedTestData

---
 src/Common/getRandomASCIIString.cpp           |   7 +-
 src/Common/getRandomASCIIString.h             |   3 +
 .../Transforms/MergeJoinTransform.cpp         |   2 +-
 .../tests/gtest_full_sorting_join.cpp         | 121 +++++++++++++++---
 4 files changed, 116 insertions(+), 17 deletions(-)

diff --git a/src/Common/getRandomASCIIString.cpp b/src/Common/getRandomASCIIString.cpp
index 594b4cd3228..a295277b453 100644
--- a/src/Common/getRandomASCIIString.cpp
+++ b/src/Common/getRandomASCIIString.cpp
@@ -6,12 +6,17 @@ namespace DB
 {
 
 String getRandomASCIIString(size_t length)
+{
+    return getRandomASCIIString(length, thread_local_rng);
+}
+
+String getRandomASCIIString(size_t length, pcg64 & rng)
 {
     std::uniform_int_distribution<int> distribution('a', 'z');
     String res;
     res.resize(length);
     for (auto & c : res)
-        c = distribution(thread_local_rng);
+        c = distribution(rng);
     return res;
 }
 
diff --git a/src/Common/getRandomASCIIString.h b/src/Common/getRandomASCIIString.h
index 627d2700ce3..19e1ff7120e 100644
--- a/src/Common/getRandomASCIIString.h
+++ b/src/Common/getRandomASCIIString.h
@@ -2,11 +2,14 @@
 
 #include <Core/Types.h>
 
+#include <pcg_random.hpp>
+
 
 namespace DB
 {
 
 /// Slow random string. Useful for random names and things like this. Not for generating data.
 String getRandomASCIIString(size_t length);
+String getRandomASCIIString(size_t length, pcg64 & rng);
 
 }
diff --git a/src/Processors/Transforms/MergeJoinTransform.cpp b/src/Processors/Transforms/MergeJoinTransform.cpp
index 8370e548fcb..bfde5892289 100644
--- a/src/Processors/Transforms/MergeJoinTransform.cpp
+++ b/src/Processors/Transforms/MergeJoinTransform.cpp
@@ -874,7 +874,7 @@ MergeJoinAlgorithm::Status MergeJoinAlgorithm::asofJoin()
         for (auto & col : rcols)
             result.addColumn(std::move(col));
     }
-    UNUSED(asof_inequality);
+
     return Status(std::move(result));
 }
 
diff --git a/src/Processors/tests/gtest_full_sorting_join.cpp b/src/Processors/tests/gtest_full_sorting_join.cpp
index 4d7ce25a7e8..741e945bfdb 100644
--- a/src/Processors/tests/gtest_full_sorting_join.cpp
+++ b/src/Processors/tests/gtest_full_sorting_join.cpp
@@ -1,30 +1,31 @@
 #include <gtest/gtest.h>
 
-
-#include <Common/randomSeed.h>
 #include <pcg_random.hpp>
 #include <random>
 
-#include <Processors/Sources/SourceFromSingleChunk.h>
-#include <Processors/Sources/SourceFromChunks.h>
-#include <Processors/Sinks/NullSink.h>
-#include <Processors/Executors/PipelineExecutor.h>
-#include <Processors/Executors/PullingPipelineExecutor.h>
-#include <QueryPipeline/QueryPipeline.h>
-
 #include <Columns/ColumnsNumber.h>
+#include <Common/getRandomASCIIString.h>
+#include <Common/randomSeed.h>
+
 #include <DataTypes/DataTypesNumber.h>
+
 #include <Interpreters/TableJoin.h>
 
+#include <Processors/Executors/PipelineExecutor.h>
+#include <Processors/Executors/PullingPipelineExecutor.h>
+#include <Processors/Sinks/NullSink.h>
+#include <Processors/Sources/SourceFromChunks.h>
+#include <Processors/Sources/SourceFromSingleChunk.h>
 #include <Processors/Transforms/MergeJoinTransform.h>
 
+#include <QueryPipeline/QueryPipeline.h>
 
 using namespace DB;
 
 UInt64 getAndPrintRandomSeed()
 {
     UInt64 seed = randomSeed();
-    std::cerr << "TEST_RANDOM_SEED: " << seed << std::endl;
+    std::cerr << __FILE__ << "::" << "TEST_RANDOM_SEED = " << seed << "ull" << std::endl;
     return seed;
 }
 
@@ -132,6 +133,8 @@ catch (Exception & e)
 class SourceChunksBuilder
 {
 public:
+    double break_prob = 0.0;
+
     explicit SourceChunksBuilder(const Block & header_)
         : header(header_)
     {
@@ -144,6 +147,10 @@ public:
         chassert(row.size() == current_chunk.size());
         for (size_t i = 0; i < current_chunk.size(); ++i)
             current_chunk[i]->insert(row[i]);
+
+        if (break_prob > 0.0 && std::uniform_real_distribution<>(0.0, 1.0)(rng) < break_prob)
+            addChunk();
+
         return *this;
     }
 
@@ -184,7 +191,7 @@ std::vector<std::vector<Field>> getValuesFromBlock(const Block & block, const Na
 }
 
 
-Block executePipeline(QueryPipeline & pipeline)
+Block executePipeline(QueryPipeline && pipeline)
 {
     PullingPipelineExecutor executor(pipeline);
 
@@ -233,7 +240,7 @@ try
         left_source, right_source, /* key_length = */ 2,
         JoinKind::Inner, JoinStrictness::Asof, ASOFJoinInequality::LessOrEquals);
 
-    Block result_block = executePipeline(pipeline);
+    Block result_block = executePipeline(std::move(pipeline));
     auto values = getValuesFromBlock(result_block, {"t1.key", "t1.t", "t2.t", "t2.value"});
 
     ASSERT_EQ(values, (std::vector<std::vector<Field>>{
@@ -253,13 +260,12 @@ try
 {
     auto left_source = oneColumnSource({ {3}, {3, 3, 3}, {3, 5, 5, 6}, {9, 9}, {10, 20} });
 
-    UInt64 p = std::uniform_int_distribution<>(0, 2)(rng);
-
     SourceChunksBuilder right_source_builder({
         {std::make_shared<DataTypeUInt64>(), "t"},
         {std::make_shared<DataTypeUInt64>(), "value"},
     });
 
+    UInt64 p = std::uniform_int_distribution<>(0, 2)(rng);
     double break_prob = p == 0 ? 0.0 : (p == 1 ? 0.5 : 1.0);
     std::uniform_real_distribution<> prob_dis(0.0, 1.0);
     for (const auto & row : std::vector<std::vector<Field>>{ {1, 101}, {2, 102}, {4, 104}, {5, 105}, {11, 111}, {15, 115} })
@@ -274,7 +280,7 @@ try
         left_source, right_source, /* key_length = */ 1,
         JoinKind::Inner, JoinStrictness::Asof, ASOFJoinInequality::LessOrEquals);
 
-    Block result_block = executePipeline(pipeline);
+    Block result_block = executePipeline(std::move(pipeline));
 
     ASSERT_EQ(
         assert_cast<const ColumnUInt64 *>(result_block.getByName("t1.x").column.get())->getData(),
@@ -296,3 +302,88 @@ catch (Exception & e)
     std::cout << e.getStackTraceString() << std::endl;
     throw;
 }
+
+TEST(FullSortingJoin, AsofGeneratedTestData)
+try
+{
+    auto left_source_builder = SourceChunksBuilder({
+        {std::make_shared<DataTypeUInt64>(), "k1"},
+        {std::make_shared<DataTypeString>(), "k2"},
+        {std::make_shared<DataTypeUInt64>(), "t"},
+        {std::make_shared<DataTypeUInt64>(), "attr"},
+    });
+
+    auto right_source_builder = SourceChunksBuilder({
+        {std::make_shared<DataTypeUInt64>(), "k1"},
+        {std::make_shared<DataTypeString>(), "k2"},
+        {std::make_shared<DataTypeUInt64>(), "t"},
+        {std::make_shared<DataTypeUInt64>(), "attr"},
+    });
+
+    /// uniform_int_distribution to have 0.0 and 1.0 probabilities
+    left_source_builder.break_prob = std::uniform_int_distribution<>(0, 5)(rng) / 5.0;
+    right_source_builder.break_prob = std::uniform_int_distribution<>(0, 5)(rng) / 5.0;
+
+    auto get_next_key = [](UInt64 & k1, String & k2)
+    {
+        size_t str_len = std::uniform_int_distribution<>(1, 10)(rng);
+        String new_k2 = getRandomASCIIString(str_len, rng);
+        if (new_k2.compare(k2) <= 0)
+            ++k1;
+        k2 = new_k2;
+    };
+
+    ColumnUInt64::Container expected;
+
+    UInt64 k1 = 0;
+    String k2 = "asdfg";
+    auto key_num_total = std::uniform_int_distribution<>(1, 1000)(rng);
+    for (size_t key_num = 0; key_num < key_num_total; ++key_num)
+    {
+        UInt64 left_t = 0;
+        size_t num_left_rows = std::uniform_int_distribution<>(1, 100)(rng);
+        for (size_t i = 0; i < num_left_rows; ++i)
+        {
+            left_t += std::uniform_int_distribution<>(1, 10)(rng);
+
+            left_source_builder.addRow({k1, k2, left_t, 10 * left_t});
+            expected.push_back(10 * left_t);
+
+            auto num_matches = 1 + std::poisson_distribution<>(4)(rng);
+
+            size_t right_t = left_t;
+            for (size_t j = 0; j < num_matches; ++j)
+            {
+                right_t += std::uniform_int_distribution<>(0, 3)(rng);
+                right_source_builder.addRow({k1, k2, right_t, j == 0 ? 100 * left_t : 0});
+            }
+            /// next left_t should be greater than right_t not to match with previous rows
+            left_t = right_t;
+        }
+
+        /// generate some rows with greater left_t to check that they are not matched
+        num_left_rows = std::uniform_int_distribution<>(1, 100)(rng);
+        for (size_t i = 0; i < num_left_rows; ++i)
+        {
+            left_t += std::uniform_int_distribution<>(1, 10)(rng);
+            left_source_builder.addRow({k1, k2, left_t, 10 * left_t});
+        }
+
+        get_next_key(k1, k2);
+    }
+
+    Block result_block = executePipeline(buildJoinPipeline(
+        left_source_builder.build(), right_source_builder.build(),
+        /* key_length = */ 3,
+        JoinKind::Inner, JoinStrictness::Asof, ASOFJoinInequality::LessOrEquals));
+
+    ASSERT_EQ(assert_cast<const ColumnUInt64 *>(block.getByName("t1.attr").column.get())->getData(), expected);
+
+    for (auto & e : expected)
+        e = 10 * e;
+    ASSERT_EQ(assert_cast<const ColumnUInt64 *>(block.getByName("t2.attr").column.get())->getData(), expected);
+}
+catch (Exception & e) {
+    std::cout << e.getStackTraceString() << std::endl;
+    throw;
+}

From da4f35556100847d527f194cee4b2f99b50bfa58 Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Thu, 21 Sep 2023 11:38:37 +0000
Subject: [PATCH 0050/1488] upd FullSortingJoin.AsofGeneratedTestData

---
 .../tests/gtest_full_sorting_join.cpp         | 56 +++++++++++++++----
 1 file changed, 44 insertions(+), 12 deletions(-)

diff --git a/src/Processors/tests/gtest_full_sorting_join.cpp b/src/Processors/tests/gtest_full_sorting_join.cpp
index 741e945bfdb..49c9f58b1d5 100644
--- a/src/Processors/tests/gtest_full_sorting_join.cpp
+++ b/src/Processors/tests/gtest_full_sorting_join.cpp
@@ -208,6 +208,14 @@ Block executePipeline(QueryPipeline && pipeline)
     return concatenateBlocks(result_blocks);
 }
 
+template <typename T>
+void checkColumn(const typename ColumnVector<T>::Container & expected, const Block & block, const std::string & name)
+{
+    const auto & actual = assert_cast<const ColumnVector<T> *>(block.getByName(name).column.get())->getData();
+    EXPECT_EQ(actual.size(), expected.size());
+    ASSERT_EQ(actual, expected);
+}
+
 TEST(FullSortingJoin, Asof)
 try
 {
@@ -306,18 +314,27 @@ catch (Exception & e)
 TEST(FullSortingJoin, AsofGeneratedTestData)
 try
 {
+    std::vector<JoinKind> join_kinds = {JoinKind::Inner, JoinKind::Left};
+    auto join_kind = join_kinds[std::uniform_int_distribution<size_t>(0, join_kinds.size() - 1)(rng)];
+
+    std::vector<ASOFJoinInequality> asof_inequalities = {
+        ASOFJoinInequality::Less, ASOFJoinInequality::LessOrEquals,
+        // ASOFJoinInequality::Greater, ASOFJoinInequality::GreaterOrEquals,
+    };
+    auto asof_inequality = asof_inequalities[std::uniform_int_distribution<size_t>(0, asof_inequalities.size() - 1)(rng)];
+
     auto left_source_builder = SourceChunksBuilder({
         {std::make_shared<DataTypeUInt64>(), "k1"},
         {std::make_shared<DataTypeString>(), "k2"},
         {std::make_shared<DataTypeUInt64>(), "t"},
-        {std::make_shared<DataTypeUInt64>(), "attr"},
+        {std::make_shared<DataTypeInt64>(), "attr"},
     });
 
     auto right_source_builder = SourceChunksBuilder({
         {std::make_shared<DataTypeUInt64>(), "k1"},
         {std::make_shared<DataTypeString>(), "k2"},
         {std::make_shared<DataTypeUInt64>(), "t"},
-        {std::make_shared<DataTypeUInt64>(), "attr"},
+        {std::make_shared<DataTypeInt64>(), "attr"},
     });
 
     /// uniform_int_distribution to have 0.0 and 1.0 probabilities
@@ -333,14 +350,14 @@ try
         k2 = new_k2;
     };
 
-    ColumnUInt64::Container expected;
+    ColumnInt64::Container expected;
 
     UInt64 k1 = 0;
     String k2 = "asdfg";
     auto key_num_total = std::uniform_int_distribution<>(1, 1000)(rng);
     for (size_t key_num = 0; key_num < key_num_total; ++key_num)
     {
-        UInt64 left_t = 0;
+        Int64 left_t = 0;
         size_t num_left_rows = std::uniform_int_distribution<>(1, 100)(rng);
         for (size_t i = 0; i < num_left_rows; ++i)
         {
@@ -351,11 +368,22 @@ try
 
             auto num_matches = 1 + std::poisson_distribution<>(4)(rng);
 
-            size_t right_t = left_t;
+            auto right_t = left_t;
             for (size_t j = 0; j < num_matches; ++j)
             {
-                right_t += std::uniform_int_distribution<>(0, 3)(rng);
-                right_source_builder.addRow({k1, k2, right_t, j == 0 ? 100 * left_t : 0});
+                int min_step = 1;
+                if (asof_inequality == ASOFJoinInequality::LessOrEquals || asof_inequality == ASOFJoinInequality::GreaterOrEquals)
+                    min_step = 0;
+                right_t += std::uniform_int_distribution<>(min_step, 3)(rng);
+
+                bool is_match = false;
+
+                if (asof_inequality == ASOFJoinInequality::LessOrEquals || asof_inequality == ASOFJoinInequality::Less)
+                    is_match = j == 0;
+                else if (asof_inequality == ASOFJoinInequality::GreaterOrEquals || asof_inequality == ASOFJoinInequality::Greater)
+                    is_match = j == num_matches - 1;
+
+                right_source_builder.addRow({k1, k2, right_t, is_match ? 100 * left_t : -1});
             }
             /// next left_t should be greater than right_t not to match with previous rows
             left_t = right_t;
@@ -366,7 +394,10 @@ try
         for (size_t i = 0; i < num_left_rows; ++i)
         {
             left_t += std::uniform_int_distribution<>(1, 10)(rng);
-            left_source_builder.addRow({k1, k2, left_t, 10 * left_t});
+            left_source_builder.addRow({k1, k2, left_t, -10 * left_t});
+
+            if (join_kind == JoinKind::Left)
+                expected.push_back(-10 * left_t);
         }
 
         get_next_key(k1, k2);
@@ -375,13 +406,14 @@ try
     Block result_block = executePipeline(buildJoinPipeline(
         left_source_builder.build(), right_source_builder.build(),
         /* key_length = */ 3,
-        JoinKind::Inner, JoinStrictness::Asof, ASOFJoinInequality::LessOrEquals));
+        join_kind, JoinStrictness::Asof, asof_inequality));
 
-    ASSERT_EQ(assert_cast<const ColumnUInt64 *>(block.getByName("t1.attr").column.get())->getData(), expected);
+    checkColumn<Int64>(expected, result_block, "t1.attr");
 
     for (auto & e : expected)
-        e = 10 * e;
-    ASSERT_EQ(assert_cast<const ColumnUInt64 *>(block.getByName("t2.attr").column.get())->getData(), expected);
+        e = e < 0 ? 0 : 10 * e; /// non matched rows from left table have negative attr
+
+    checkColumn<Int64>(expected, result_block, "t2.attr");
 }
 catch (Exception & e) {
     std::cout << e.getStackTraceString() << std::endl;

From 97e3ee6e661620187bf115d948115936e36cdaf4 Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Thu, 21 Sep 2023 11:40:41 +0000
Subject: [PATCH 0051/1488] upd gtest_full_sorting_join

---
 .../tests/gtest_full_sorting_join.cpp         | 92 +++++++++++--------
 1 file changed, 54 insertions(+), 38 deletions(-)

diff --git a/src/Processors/tests/gtest_full_sorting_join.cpp b/src/Processors/tests/gtest_full_sorting_join.cpp
index 49c9f58b1d5..bbd321a78d7 100644
--- a/src/Processors/tests/gtest_full_sorting_join.cpp
+++ b/src/Processors/tests/gtest_full_sorting_join.cpp
@@ -142,7 +142,7 @@ public:
         chassert(!current_chunk.empty());
     }
 
-    SourceChunksBuilder & addRow(const std::vector<Field> & row)
+    void addRow(const std::vector<Field> & row)
     {
         chassert(row.size() == current_chunk.size());
         for (size_t i = 0; i < current_chunk.size(); ++i)
@@ -150,25 +150,29 @@ public:
 
         if (break_prob > 0.0 && std::uniform_real_distribution<>(0.0, 1.0)(rng) < break_prob)
             addChunk();
-
-        return *this;
     }
 
-    SourceChunksBuilder & addChunk()
+    void addChunk()
     {
         if (current_chunk.front()->empty())
-            return *this;
+            return;
 
         size_t rows = current_chunk.front()->size();
         chunks.emplace_back(std::move(current_chunk), rows);
         current_chunk = header.cloneEmptyColumns();
-        return *this;
+        return;
     }
 
     std::shared_ptr<ISource> build()
     {
         addChunk();
-        return std::make_shared<SourceFromChunks>(header, std::move(chunks));
+
+        /// copy chunk to allow reusing same builder
+        Chunks chunks_copy;
+        chunks_copy.reserve(chunks.size());
+        for (const auto & chunk : chunks)
+            chunks_copy.emplace_back(chunk.clone());
+        return std::make_shared<SourceFromChunks>(header, std::move(chunks_copy));
     }
 
 private:
@@ -219,42 +223,54 @@ void checkColumn(const typename ColumnVector<T>::Container & expected, const Blo
 TEST(FullSortingJoin, Asof)
 try
 {
-    auto left_source = SourceChunksBuilder({
+    SourceChunksBuilder left_source({
         {std::make_shared<DataTypeString>(), "key"},
         {std::make_shared<DataTypeUInt64>(), "t"},
-    })
-        .addRow({"AMZN", 3})
-        .addRow({"AMZN", 4})
-        .addRow({"AMZN", 6})
-        .build();
+    });
 
-    auto right_source = SourceChunksBuilder({
+    left_source.addRow({"AMZN", 3});
+    left_source.addRow({"AMZN", 4});
+    left_source.addRow({"AMZN", 6});
+
+    SourceChunksBuilder right_source({
         {std::make_shared<DataTypeString>(), "key"},
         {std::make_shared<DataTypeUInt64>(), "t"},
         {std::make_shared<DataTypeUInt64>(), "value"},
-    })
-        .addRow({"AAPL", 1, 97})
-        .addChunk()
-        .addRow({"AAPL", 2, 98})
-        .addRow({"AAPL", 3, 99})
-        .addRow({"AMZN", 1, 100})
-        .addRow({"AMZN", 2, 110})
-        .addChunk()
-        .addRow({"AMZN", 4, 130})
-        .addRow({"AMZN", 5, 140})
-        .build();
+    });
+    right_source.addRow({"AAPL", 1, 97});
+    right_source.addChunk();
+    right_source.addRow({"AAPL", 2, 98});
+    right_source.addRow({"AAPL", 3, 99});
+    right_source.addRow({"AMZN", 1, 100});
+    right_source.addRow({"AMZN", 2, 110});
+    right_source.addChunk();
+    right_source.addRow({"AMZN", 4, 130});
+    right_source.addRow({"AMZN", 5, 140});
 
-    auto pipeline = buildJoinPipeline(
-        left_source, right_source, /* key_length = */ 2,
-        JoinKind::Inner, JoinStrictness::Asof, ASOFJoinInequality::LessOrEquals);
+    {
+        Block result_block = executePipeline(buildJoinPipeline(
+            left_source.build(), right_source.build(), /* key_length = */ 2,
+            JoinKind::Inner, JoinStrictness::Asof, ASOFJoinInequality::LessOrEquals));
+        auto values = getValuesFromBlock(result_block, {"t1.key", "t1.t", "t2.t", "t2.value"});
 
-    Block result_block = executePipeline(std::move(pipeline));
-    auto values = getValuesFromBlock(result_block, {"t1.key", "t1.t", "t2.t", "t2.value"});
+        ASSERT_EQ(values, (std::vector<std::vector<Field>>{
+            {"AMZN", 3u, 4u, 130u},
+            {"AMZN", 4u, 4u, 130u},
+        }));
+    }
 
-    ASSERT_EQ(values, (std::vector<std::vector<Field>>{
-        {"AMZN", 3u, 4u, 130u},
-        {"AMZN", 4u, 4u, 130u},
-    }));
+    {
+        Block result_block = executePipeline(buildJoinPipeline(
+            left_source.build(), right_source.build(), /* key_length = */ 2,
+            JoinKind::Inner, JoinStrictness::Asof, ASOFJoinInequality::GreaterOrEquals));
+        auto values = getValuesFromBlock(result_block, {"t1.key", "t1.t", "t2.t", "t2.value"});
+
+        ASSERT_EQ(values, (std::vector<std::vector<Field>>{
+            {"AMZN", 3u, 2u, 110u},
+            {"AMZN", 4u, 4u, 130u},
+            {"AMZN", 6u, 5u, 140u},
+        }));
+    }
 }
 catch (Exception & e)
 {
@@ -314,23 +330,23 @@ catch (Exception & e)
 TEST(FullSortingJoin, AsofGeneratedTestData)
 try
 {
-    std::vector<JoinKind> join_kinds = {JoinKind::Inner, JoinKind::Left};
+    std::array join_kinds{JoinKind::Inner, JoinKind::Left};
     auto join_kind = join_kinds[std::uniform_int_distribution<size_t>(0, join_kinds.size() - 1)(rng)];
 
-    std::vector<ASOFJoinInequality> asof_inequalities = {
+    std::array asof_inequalities{
         ASOFJoinInequality::Less, ASOFJoinInequality::LessOrEquals,
         // ASOFJoinInequality::Greater, ASOFJoinInequality::GreaterOrEquals,
     };
     auto asof_inequality = asof_inequalities[std::uniform_int_distribution<size_t>(0, asof_inequalities.size() - 1)(rng)];
 
-    auto left_source_builder = SourceChunksBuilder({
+    SourceChunksBuilder left_source_builder({
         {std::make_shared<DataTypeUInt64>(), "k1"},
         {std::make_shared<DataTypeString>(), "k2"},
         {std::make_shared<DataTypeUInt64>(), "t"},
         {std::make_shared<DataTypeInt64>(), "attr"},
     });
 
-    auto right_source_builder = SourceChunksBuilder({
+    SourceChunksBuilder right_source_builder({
         {std::make_shared<DataTypeUInt64>(), "k1"},
         {std::make_shared<DataTypeString>(), "k2"},
         {std::make_shared<DataTypeUInt64>(), "t"},

From 4a1a7d4c6278246e097cf0583b401aa8ac3775fa Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Mon, 25 Sep 2023 10:54:08 +0000
Subject: [PATCH 0052/1488] Add randomized test FullSortingJoin.Any

---
 .../Transforms/MergeJoinTransform.cpp         |  28 ++-
 .../Transforms/MergeJoinTransform.h           |   5 +
 .../tests/gtest_full_sorting_join.cpp         | 181 ++++++++++++++----
 3 files changed, 166 insertions(+), 48 deletions(-)

diff --git a/src/Processors/Transforms/MergeJoinTransform.cpp b/src/Processors/Transforms/MergeJoinTransform.cpp
index bfde5892289..f5e277ea8c8 100644
--- a/src/Processors/Transforms/MergeJoinTransform.cpp
+++ b/src/Processors/Transforms/MergeJoinTransform.cpp
@@ -630,7 +630,7 @@ struct AnyJoinImpl
                      FullMergeJoinCursor & right_cursor,
                      PaddedPODArray<UInt64> & left_map,
                      PaddedPODArray<UInt64> & right_map,
-                     AnyJoinState & state,
+                     AnyJoinState & any_join_state,
                      int null_direction_hint)
     {
         assert(enabled);
@@ -691,21 +691,21 @@ struct AnyJoinImpl
             }
         }
 
-        /// Remember index of last joined row to propagate it to next block
+        /// Remember last joined row to propagate it to next block
 
-        state.setValue({});
+        any_join_state.setValue({});
         if (!left_cursor->isValid())
         {
-            state.set(0, left_cursor.cursor);
+            any_join_state.set(0, left_cursor.cursor);
             if (cmp == 0 && isLeft(kind))
-                state.setValue(getRowFromChunk(right_cursor.getCurrent(), rpos));
+                any_join_state.setValue(getRowFromChunk(right_cursor.getCurrent(), rpos));
         }
 
         if (!right_cursor->isValid())
         {
-            state.set(1, right_cursor.cursor);
+            any_join_state.set(1, right_cursor.cursor);
             if (cmp == 0 && isRight(kind))
-                state.setValue(getRowFromChunk(left_cursor.getCurrent(), lpos));
+                any_join_state.setValue(getRowFromChunk(left_cursor.getCurrent(), lpos));
         }
     }
 };
@@ -720,7 +720,6 @@ std::optional<MergeJoinAlgorithm::Status> MergeJoinAlgorithm::handleAnyJoinState
     for (size_t source_num = 0; source_num < 2; ++source_num)
     {
         auto & current = *cursors[source_num];
-        auto & state = any_join_state;
         if (any_join_state.keys[source_num].equals(current.cursor))
         {
             size_t start_pos = current->getRow();
@@ -728,25 +727,22 @@ std::optional<MergeJoinAlgorithm::Status> MergeJoinAlgorithm::handleAnyJoinState
 
             if (length && isLeft(kind) && source_num == 0)
             {
-                if (state.value)
-                    result = copyChunkResized(current.getCurrent(), state.value, start_pos, length);
+                if (any_join_state.value)
+                    result = copyChunkResized(current.getCurrent(), any_join_state.value, start_pos, length);
                 else
                     result = createBlockWithDefaults(source_num, start_pos, length);
             }
 
             if (length && isRight(kind) && source_num == 1)
             {
-                if (state.value)
-                    result = copyChunkResized(state.value, current.getCurrent(), start_pos, length);
+                if (any_join_state.value)
+                    result = copyChunkResized(any_join_state.value, current.getCurrent(), start_pos, length);
                 else
                     result = createBlockWithDefaults(source_num, start_pos, length);
             }
 
-            /// We've found row with other key, no need to skip more rows with current key
             if (current->isValid())
-            {
-                state.keys[source_num].reset();
-            }
+                any_join_state.keys[source_num].reset();
         }
         else
         {
diff --git a/src/Processors/Transforms/MergeJoinTransform.h b/src/Processors/Transforms/MergeJoinTransform.h
index 3ee01e57992..375c9ebd3cc 100644
--- a/src/Processors/Transforms/MergeJoinTransform.h
+++ b/src/Processors/Transforms/MergeJoinTransform.h
@@ -85,6 +85,11 @@ public:
         keys[source_num] = JoinKeyRow(cursor, cursor.rows - 1);
     }
 
+    void reset(size_t source_num)
+    {
+        keys[source_num].reset();
+    }
+
     void setValue(Chunk value_) { value = std::move(value_); }
 
     bool empty() const { return keys[0].row.empty() && keys[1].row.empty(); }
diff --git a/src/Processors/tests/gtest_full_sorting_join.cpp b/src/Processors/tests/gtest_full_sorting_join.cpp
index bbd321a78d7..1dc410bd6b8 100644
--- a/src/Processors/tests/gtest_full_sorting_join.cpp
+++ b/src/Processors/tests/gtest_full_sorting_join.cpp
@@ -213,13 +213,142 @@ Block executePipeline(QueryPipeline && pipeline)
 }
 
 template <typename T>
-void checkColumn(const typename ColumnVector<T>::Container & expected, const Block & block, const std::string & name)
+void assertColumnVectorEq(const typename ColumnVector<T>::Container & expected, const Block & block, const std::string & name)
 {
     const auto & actual = assert_cast<const ColumnVector<T> *>(block.getByName(name).column.get())->getData();
     EXPECT_EQ(actual.size(), expected.size());
     ASSERT_EQ(actual, expected);
 }
 
+template <typename T>
+void assertColumnEq(const IColumn & expected, const Block & block, const std::string & name)
+{
+    const ColumnPtr & actual = block.getByName(name).column;
+    ASSERT_TRUE(checkColumn<T>(*actual));
+    ASSERT_TRUE(checkColumn<T>(expected));
+    EXPECT_EQ(actual->size(), expected.size());
+
+    auto dump_val = [](const IColumn & col, size_t i) -> String
+    {
+        Field value;
+        col.get(i, value);
+        return value.dump();
+    };
+
+    size_t num_rows = std::min(actual->size(), expected.size());
+    for (size_t i = 0; i < num_rows; ++i)
+        ASSERT_EQ(actual->compareAt(i, i, expected, 1), 0) << dump_val(*actual, i) << " != " << dump_val(expected, i) << " at row " << i;
+}
+
+template <typename T>
+T getRandomFrom(const std::initializer_list<T> & opts)
+{
+    std::vector<T> options(opts.begin(), opts.end());
+    size_t idx = std::uniform_int_distribution<size_t>(0, options.size() - 1)(rng);
+    return options[idx];
+}
+
+/// Used to have accurate 0.0 and 1.0 probabilities
+double getRandomDoubleQuantized(size_t quants = 5)
+{
+    return std::uniform_int_distribution<size_t>(0, quants)(rng) / static_cast<double>(quants);
+}
+
+void generateNextKey(UInt64 & k1, String & k2)
+{
+    size_t str_len = std::uniform_int_distribution<>(1, 10)(rng);
+    String new_k2 = getRandomASCIIString(str_len, rng);
+    if (new_k2.compare(k2) <= 0)
+        ++k1;
+    k2 = new_k2;
+}
+
+TEST(FullSortingJoin, Any)
+try
+{
+    JoinKind kind = getRandomFrom({JoinKind::Inner, JoinKind::Left, JoinKind::Right});
+
+    SourceChunksBuilder left_source({
+        {std::make_shared<DataTypeUInt64>(), "k1"},
+        {std::make_shared<DataTypeString>(), "k2"},
+        {std::make_shared<DataTypeString>(), "attr"},
+    });
+
+    SourceChunksBuilder right_source({
+        {std::make_shared<DataTypeUInt64>(), "k1"},
+        {std::make_shared<DataTypeString>(), "k2"},
+        {std::make_shared<DataTypeString>(), "attr"},
+    });
+
+    left_source.break_prob = getRandomDoubleQuantized();
+    right_source.break_prob = getRandomDoubleQuantized();
+
+    size_t num_keys = std::uniform_int_distribution<>(100, 1000)(rng);
+
+    auto expected_left = ColumnString::create();
+    auto expected_right = ColumnString::create();
+
+    UInt64 k1 = 0;
+    String k2 = "";
+
+    auto get_attr = [&](const String & side, size_t idx) -> String
+    {
+        return toString(k1) + "_" + k2 + "_" + side + "_" + toString(idx);
+    };
+
+    for (size_t i = 0; i < num_keys; ++i)
+    {
+        generateNextKey(k1, k2);
+
+        /// Key is present in left, right or both tables. Both tables is more probable.
+        size_t key_presence = std::uniform_int_distribution<>(0, 10)(rng);
+
+        size_t num_rows_left = key_presence == 0 ? 0 : std::uniform_int_distribution<>(1, 10)(rng);
+        for (size_t j = 0; j < num_rows_left; ++j)
+            left_source.addRow({k1, k2, get_attr("left", j)});
+
+        size_t num_rows_right = key_presence == 1 ? 0 : std::uniform_int_distribution<>(1, 10)(rng);
+        for (size_t j = 0; j < num_rows_right; ++j)
+            right_source.addRow({k1, k2, get_attr("right", j)});
+
+        String left_attr = num_rows_left ? get_attr("left", 0) : "";
+        String right_attr = num_rows_right ? get_attr("right", 0) : "";
+
+        if (kind == JoinKind::Inner && num_rows_left && num_rows_right)
+        {
+            expected_left->insert(left_attr);
+            expected_right->insert(right_attr);
+        }
+        else if (kind == JoinKind::Left)
+        {
+            for (size_t j = 0; j < num_rows_left; ++j)
+            {
+                expected_left->insert(get_attr("left", j));
+                expected_right->insert(right_attr);
+            }
+        }
+        else if (kind == JoinKind::Right)
+        {
+            for (size_t j = 0; j < num_rows_right; ++j)
+            {
+                expected_left->insert(left_attr);
+                expected_right->insert(get_attr("right", j));
+            }
+        }
+    }
+
+    Block result_block = executePipeline(buildJoinPipeline(
+        left_source.build(), right_source.build(), /* key_length = */ 2,
+        kind, JoinStrictness::Any));
+    assertColumnEq<ColumnString>(*expected_left, result_block, "t1.attr");
+    assertColumnEq<ColumnString>(*expected_right, result_block, "t2.attr");
+}
+catch (Exception & e)
+{
+    std::cout << e.getStackTraceString() << std::endl;
+    throw;
+}
+
 TEST(FullSortingJoin, Asof)
 try
 {
@@ -259,18 +388,18 @@ try
         }));
     }
 
-    {
-        Block result_block = executePipeline(buildJoinPipeline(
-            left_source.build(), right_source.build(), /* key_length = */ 2,
-            JoinKind::Inner, JoinStrictness::Asof, ASOFJoinInequality::GreaterOrEquals));
-        auto values = getValuesFromBlock(result_block, {"t1.key", "t1.t", "t2.t", "t2.value"});
+    // {
+    //     Block result_block = executePipeline(buildJoinPipeline(
+    //         left_source.build(), right_source.build(), /* key_length = */ 2,
+    //         JoinKind::Inner, JoinStrictness::Asof, ASOFJoinInequality::GreaterOrEquals));
+    //     auto values = getValuesFromBlock(result_block, {"t1.key", "t1.t", "t2.t", "t2.value"});
 
-        ASSERT_EQ(values, (std::vector<std::vector<Field>>{
-            {"AMZN", 3u, 2u, 110u},
-            {"AMZN", 4u, 4u, 130u},
-            {"AMZN", 6u, 5u, 140u},
-        }));
-    }
+    //     ASSERT_EQ(values, (std::vector<std::vector<Field>>{
+    //         {"AMZN", 3u, 2u, 110u},
+    //         {"AMZN", 4u, 4u, 130u},
+    //         {"AMZN", 6u, 5u, 140u},
+    //     }));
+    // }
 }
 catch (Exception & e)
 {
@@ -330,14 +459,12 @@ catch (Exception & e)
 TEST(FullSortingJoin, AsofGeneratedTestData)
 try
 {
-    std::array join_kinds{JoinKind::Inner, JoinKind::Left};
-    auto join_kind = join_kinds[std::uniform_int_distribution<size_t>(0, join_kinds.size() - 1)(rng)];
+    auto join_kind = getRandomFrom({JoinKind::Inner, JoinKind::Left});
 
-    std::array asof_inequalities{
+    auto asof_inequality = getRandomFrom({
         ASOFJoinInequality::Less, ASOFJoinInequality::LessOrEquals,
         // ASOFJoinInequality::Greater, ASOFJoinInequality::GreaterOrEquals,
-    };
-    auto asof_inequality = asof_inequalities[std::uniform_int_distribution<size_t>(0, asof_inequalities.size() - 1)(rng)];
+    });
 
     SourceChunksBuilder left_source_builder({
         {std::make_shared<DataTypeUInt64>(), "k1"},
@@ -353,18 +480,8 @@ try
         {std::make_shared<DataTypeInt64>(), "attr"},
     });
 
-    /// uniform_int_distribution to have 0.0 and 1.0 probabilities
-    left_source_builder.break_prob = std::uniform_int_distribution<>(0, 5)(rng) / 5.0;
-    right_source_builder.break_prob = std::uniform_int_distribution<>(0, 5)(rng) / 5.0;
-
-    auto get_next_key = [](UInt64 & k1, String & k2)
-    {
-        size_t str_len = std::uniform_int_distribution<>(1, 10)(rng);
-        String new_k2 = getRandomASCIIString(str_len, rng);
-        if (new_k2.compare(k2) <= 0)
-            ++k1;
-        k2 = new_k2;
-    };
+    left_source_builder.break_prob = getRandomDoubleQuantized();
+    right_source_builder.break_prob = getRandomDoubleQuantized();
 
     ColumnInt64::Container expected;
 
@@ -416,7 +533,7 @@ try
                 expected.push_back(-10 * left_t);
         }
 
-        get_next_key(k1, k2);
+        generateNextKey(k1, k2);
     }
 
     Block result_block = executePipeline(buildJoinPipeline(
@@ -424,12 +541,12 @@ try
         /* key_length = */ 3,
         join_kind, JoinStrictness::Asof, asof_inequality));
 
-    checkColumn<Int64>(expected, result_block, "t1.attr");
+    assertColumnVectorEq<Int64>(expected, result_block, "t1.attr");
 
     for (auto & e : expected)
         e = e < 0 ? 0 : 10 * e; /// non matched rows from left table have negative attr
 
-    checkColumn<Int64>(expected, result_block, "t2.attr");
+    assertColumnVectorEq<Int64>(expected, result_block, "t2.attr");
 }
 catch (Exception & e) {
     std::cout << e.getStackTraceString() << std::endl;

From 7e0c2d7bcb88f6d8575fc1f0e288e94f13456f87 Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Tue, 26 Sep 2023 12:25:12 +0000
Subject: [PATCH 0053/1488] wip full sorting asof join

---
 .../Transforms/MergeJoinTransform.cpp         | 300 +++++++++++++--
 .../Transforms/MergeJoinTransform.h           |  89 ++---
 .../tests/gtest_full_sorting_join.cpp         | 346 ++++++++++++++----
 .../02276_full_sort_join_unsupported.sql      |   2 +-
 4 files changed, 575 insertions(+), 162 deletions(-)

diff --git a/src/Processors/Transforms/MergeJoinTransform.cpp b/src/Processors/Transforms/MergeJoinTransform.cpp
index f5e277ea8c8..638f2f1cb10 100644
--- a/src/Processors/Transforms/MergeJoinTransform.cpp
+++ b/src/Processors/Transforms/MergeJoinTransform.cpp
@@ -34,8 +34,6 @@ namespace ErrorCodes
 namespace
 {
 
-constexpr UInt64 DEFAULT_VALUE_INDEX = std::numeric_limits<UInt64>::max();
-
 FullMergeJoinCursorPtr createCursor(const Block & block, const Names & columns, JoinStrictness strictness)
 {
     SortDescription desc;
@@ -112,7 +110,7 @@ int ALWAYS_INLINE compareCursors(const SortCursorImpl & lhs, const SortCursorImp
 
 int compareAsofCursors(const FullMergeJoinCursor & lhs, const FullMergeJoinCursor & rhs)
 {
-    return nullableCompareAt<false, false>(lhs.getAsofColumn(), rhs.getAsofColumn(), lhs->getRow(), rhs->getRow());
+    return nullableCompareAt<false, false>(*lhs.getAsofColumn(), *rhs.getAsofColumn(), lhs->getRow(), rhs->getRow());
 }
 
 bool ALWAYS_INLINE totallyLess(SortCursorImpl & lhs, SortCursorImpl & rhs, int null_direction_hint)
@@ -250,6 +248,87 @@ void inline addMany(PaddedPODArray<UInt64> & values, UInt64 value, size_t num)
 
 }
 
+JoinKeyRow::JoinKeyRow(const FullMergeJoinCursor & cursor, size_t pos)
+{
+    row.reserve(cursor->sort_columns.size());
+    for (const auto & col : cursor->sort_columns)
+    {
+        auto new_col = col->cloneEmpty();
+        new_col->insertFrom(*col, pos);
+        row.push_back(std::move(new_col));
+    }
+    if (auto asof_column = cursor.getAsofColumn())
+    {
+        auto new_col = asof_column->cloneEmpty();
+        new_col->insertFrom(*asof_column, pos);
+        row.push_back(std::move(new_col));
+    }
+}
+
+void JoinKeyRow::reset()
+{
+    row.clear();
+}
+
+bool JoinKeyRow::equals(const FullMergeJoinCursor & cursor) const
+{
+    if (row.empty())
+        return false;
+
+    assert(this->row.size() == cursor->sort_columns_size);
+    for (size_t i = 0; i < cursor->sort_columns_size; ++i)
+    {
+        int cmp = this->row[i]->compareAt(0, cursor->getRow(), *(cursor->sort_columns[i]), cursor->desc[i].nulls_direction);
+        if (cmp != 0)
+            return false;
+    }
+    return true;
+}
+
+bool JoinKeyRow::asofMatch(const FullMergeJoinCursor & cursor, ASOFJoinInequality asof_inequality) const
+{
+    if (!equals(cursor))
+        return false;
+    int cmp = cursor.getAsofColumn()->compareAt(cursor->getRow(), 0, *row.back(), 1);
+    return (asof_inequality == ASOFJoinInequality::Less && cmp < 0)
+        || (asof_inequality == ASOFJoinInequality::LessOrEquals && cmp <= 0)
+        || (asof_inequality == ASOFJoinInequality::Greater && cmp > 0)
+        || (asof_inequality == ASOFJoinInequality::GreaterOrEquals && cmp >= 0);
+}
+
+void AnyJoinState::set(size_t source_num, const FullMergeJoinCursor & cursor)
+{
+    assert(cursor->rows);
+    keys[source_num] = JoinKeyRow(cursor, cursor->rows - 1);
+}
+
+void AnyJoinState::reset(size_t source_num)
+{
+    keys[source_num].reset();
+    value.clear();
+}
+
+void AnyJoinState::setValue(Chunk value_)
+{
+    value = std::move(value_);
+}
+
+bool AnyJoinState::empty() const { return keys[0].row.empty() && keys[1].row.empty(); }
+
+
+void AsofJoinState::set(const FullMergeJoinCursor & rcursor, size_t rpos)
+{
+    key = JoinKeyRow(rcursor, rpos);
+    value = rcursor.getCurrent().clone();
+    value_row = rpos;
+}
+
+void AsofJoinState::reset()
+{
+    key.reset();
+    value.clear();
+}
+
 const Chunk & FullMergeJoinCursor::getCurrent() const
 {
     return current_chunk;
@@ -282,6 +361,31 @@ bool FullMergeJoinCursor::fullyCompleted() const
     return !cursor.isValid() && recieved_all_blocks;
 }
 
+String FullMergeJoinCursor::dump() const
+{
+    Strings row_dump;
+    if (cursor.isValid())
+    {
+        Field val;
+        for (size_t i = 0; i < cursor.sort_columns_size; ++i)
+        {
+            cursor.sort_columns[i]->get(cursor.getRow(), val);
+            row_dump.push_back(val.dump());
+        }
+
+        if (auto * asof_column = getAsofColumn())
+        {
+            asof_column->get(cursor.getRow(), val);
+            row_dump.push_back(val.dump());
+        }
+    }
+
+    return fmt::format("<{}/{}{}>[{}]",
+        cursor.getRow(), cursor.rows,
+        recieved_all_blocks ? "(finished)" : "",
+        fmt::join(row_dump, ", "));
+}
+
 MergeJoinAlgorithm::MergeJoinAlgorithm(
     JoinKind kind_,
     JoinStrictness strictness_,
@@ -456,7 +560,7 @@ struct AllJoinImpl
                 else
                 {
                     assert(state == nullptr);
-                    state = std::make_unique<AllJoinState>(left_cursor.cursor, lpos, right_cursor.cursor, rpos);
+                    state = std::make_unique<AllJoinState>(left_cursor, lpos, right_cursor, rpos);
                     state->addRange(0, left_cursor.getCurrent().clone(), lpos, lnum);
                     state->addRange(1, right_cursor.getCurrent().clone(), rpos, rnum);
                     return;
@@ -501,6 +605,17 @@ void dispatchKind(JoinKind kind, Args && ... args)
         throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Unsupported join kind: \"{}\"", kind);
 }
 
+MutableColumns MergeJoinAlgorithm::getEmptyResultColumns() const
+{
+    MutableColumns result_cols;
+    for (size_t i = 0; i < 2; ++i)
+    {
+        for (const auto & col : cursors[i]->sampleColumns())
+            result_cols.push_back(col->cloneEmpty());
+    }
+    return result_cols;
+}
+
 std::optional<MergeJoinAlgorithm::Status> MergeJoinAlgorithm::handleAllJoinState()
 {
     if (all_join_state && all_join_state->finished())
@@ -514,7 +629,7 @@ std::optional<MergeJoinAlgorithm::Status> MergeJoinAlgorithm::handleAllJoinState
         /// Accumulate blocks with same key in all_join_state
         for (size_t i = 0; i < 2; ++i)
         {
-            if (cursors[i]->cursor.isValid() && all_join_state->keys[i].equals(cursors[i]->cursor))
+            if (cursors[i]->cursor.isValid() && all_join_state->keys[i].equals(*cursors[i]))
             {
                 size_t pos = cursors[i]->cursor.getRow();
                 size_t num = nextDistinct(cursors[i]->cursor);
@@ -534,12 +649,7 @@ std::optional<MergeJoinAlgorithm::Status> MergeJoinAlgorithm::handleAllJoinState
         stat.max_blocks_loaded = std::max(stat.max_blocks_loaded, all_join_state->blocksStored());
 
         /// join all rows with current key
-        MutableColumns result_cols;
-        for (size_t i = 0; i < 2; ++i)
-        {
-            for (const auto & col : cursors[i]->sampleColumns())
-                result_cols.push_back(col->cloneEmpty());
-        }
+        MutableColumns result_cols = getEmptyResultColumns();
 
         size_t total_rows = 0;
         while (!max_block_size || total_rows < max_block_size)
@@ -567,6 +677,52 @@ std::optional<MergeJoinAlgorithm::Status> MergeJoinAlgorithm::handleAllJoinState
     return {};
 }
 
+std::optional<MergeJoinAlgorithm::Status> MergeJoinAlgorithm::handleAsofJoinState()
+{
+    if (strictness != JoinStrictness::Asof)
+        return {};
+
+    if (!cursors[1]->fullyCompleted())
+        return {};
+
+    auto & left_cursor = *cursors[0];
+    size_t lpos = left_cursor->getRow();
+    const auto & left_columns = left_cursor.getCurrent().getColumns();
+
+    MutableColumns result_cols = getEmptyResultColumns();
+
+    while (left_cursor->isValid() && asof_join_state.hasMatch(left_cursor, asof_inequality))
+    {
+        size_t i = 0;
+        for (const auto & col : left_columns)
+            result_cols[i++]->insertFrom(*col, lpos);
+        for (const auto & col : asof_join_state.value.getColumns())
+            result_cols[i++]->insertFrom(*col, asof_join_state.value_row);
+        chassert(i == result_cols.size());
+        left_cursor->next();
+    }
+
+    while (isLeft(kind) && left_cursor->isValid())
+    {
+        /// return row with default values at right side
+        size_t i = 0;
+        for (const auto & col : left_columns)
+            result_cols[i++]->insertFrom(*col, lpos);
+        for (; i < result_cols.size(); ++i)
+            result_cols[i]->insertDefault();
+        chassert(i == result_cols.size());
+
+        left_cursor->next();
+    }
+
+    size_t result_rows = result_cols.empty() ? 0 : result_cols.front()->size();
+    if (result_rows)
+        return Status(Chunk(std::move(result_cols), result_rows));
+
+    return {};
+}
+
+
 MergeJoinAlgorithm::Status MergeJoinAlgorithm::allJoin()
 {
     PaddedPODArray<UInt64> idx_map[2];
@@ -696,14 +852,14 @@ struct AnyJoinImpl
         any_join_state.setValue({});
         if (!left_cursor->isValid())
         {
-            any_join_state.set(0, left_cursor.cursor);
+            any_join_state.set(0, left_cursor);
             if (cmp == 0 && isLeft(kind))
                 any_join_state.setValue(getRowFromChunk(right_cursor.getCurrent(), rpos));
         }
 
         if (!right_cursor->isValid())
         {
-            any_join_state.set(1, right_cursor.cursor);
+            any_join_state.set(1, right_cursor);
             if (cmp == 0 && isRight(kind))
                 any_join_state.setValue(getRowFromChunk(left_cursor.getCurrent(), lpos));
         }
@@ -720,7 +876,7 @@ std::optional<MergeJoinAlgorithm::Status> MergeJoinAlgorithm::handleAnyJoinState
     for (size_t source_num = 0; source_num < 2; ++source_num)
     {
         auto & current = *cursors[source_num];
-        if (any_join_state.keys[source_num].equals(current.cursor))
+        if (any_join_state.keys[source_num].equals(current))
         {
             size_t start_pos = current->getRow();
             size_t length = nextDistinct(current.cursor);
@@ -811,24 +967,35 @@ MergeJoinAlgorithm::Status MergeJoinAlgorithm::asofJoin()
     if (!right_cursor->isValid())
         return Status(1);
 
-    PaddedPODArray<UInt64> left_map;
-    PaddedPODArray<UInt64> right_map;
+    const auto & left_columns = left_cursor.getCurrent().getColumns();
+    const auto & right_columns = right_cursor.getCurrent().getColumns();
+
+    MutableColumns result_cols = getEmptyResultColumns();
 
     while (left_cursor->isValid() && right_cursor->isValid())
     {
         auto lpos = left_cursor->getRow();
         auto rpos = right_cursor->getRow();
         auto cmp = compareCursors(*left_cursor, *right_cursor);
+        // LOG_DEBUG(&Poco::Logger::get("XXXX"), "{}:{} ({}) <=> ({}) -> {}", __FILE__, __LINE__, left_cursor.dump(), right_cursor.dump(), cmp);
+
         if (cmp == 0)
         {
             auto asof_cmp = compareAsofCursors(left_cursor, right_cursor);
+            // LOG_DEBUG(&Poco::Logger::get("XXXX"), "{}:{} ({}) <=> ({}) -> asof {}", __FILE__, __LINE__, left_cursor.dump(), right_cursor.dump(), asof_cmp);
+
             if ((asof_inequality == ASOFJoinInequality::Less && asof_cmp <= -1)
              || (asof_inequality == ASOFJoinInequality::LessOrEquals && asof_cmp <= 0))
             {
                 /// First row in right table that is greater (or equal) than current row in left table
                 /// matches asof join condition the best
-                left_map.push_back(lpos);
-                right_map.push_back(rpos);
+                size_t i = 0;
+                for (const auto & col : left_columns)
+                    result_cols[i++]->insertFrom(*col, lpos);
+                for (const auto & col : right_columns)
+                    result_cols[i++]->insertFrom(*col, rpos);
+                chassert(i == result_cols.size());
+
                 left_cursor->next();
                 continue;
             }
@@ -839,39 +1006,99 @@ MergeJoinAlgorithm::Status MergeJoinAlgorithm::asofJoin()
                 right_cursor->next();
                 continue;
             }
+
+            if ((asof_inequality == ASOFJoinInequality::Greater && asof_cmp >= 1)
+             || (asof_inequality == ASOFJoinInequality::GreaterOrEquals && asof_cmp >= 0))
+            {
+                // LOG_DEBUG(&Poco::Logger::get("XXXX"), "{}:{} ", __FILE__, __LINE__);
+                /// condition is satisfied, remember this row and move next to try to find better match
+                asof_join_state.set(right_cursor, rpos);
+                right_cursor->next();
+                continue;
+            }
+
+            if (asof_inequality == ASOFJoinInequality::Greater || asof_inequality == ASOFJoinInequality::GreaterOrEquals)
+            {
+                /// Asof condition is not satisfied anymore, use last matched row from right table
+                if (asof_join_state.hasMatch(left_cursor, asof_inequality))
+                {
+                    // LOG_DEBUG(&Poco::Logger::get("XXXX"), "{}:{} ", __FILE__, __LINE__);
+                    size_t i = 0;
+                    for (const auto & col : left_columns)
+                        result_cols[i++]->insertFrom(*col, lpos);
+                    for (const auto & col : asof_join_state.value.getColumns())
+                        result_cols[i++]->insertFrom(*col, asof_join_state.value_row);
+                    chassert(i == result_cols.size());
+                }
+                else
+                {
+                    asof_join_state.reset();
+                    if (isLeft(kind))
+                    {
+                        // LOG_DEBUG(&Poco::Logger::get("XXXX"), "{}:{} ", __FILE__, __LINE__);
+
+                        /// return row with default values at right side
+                        size_t i = 0;
+                        for (const auto & col : left_columns)
+                            result_cols[i++]->insertFrom(*col, lpos);
+                        for (; i < result_cols.size(); ++i)
+                            result_cols[i]->insertDefault();
+                        chassert(i == result_cols.size());
+                    }
+                }
+                left_cursor->next();
+                // LOG_DEBUG(&Poco::Logger::get("XXXX"), "{}:{} ", __FILE__, __LINE__);
+                continue;
+            }
+
             throw Exception(ErrorCodes::NOT_IMPLEMENTED, "TODO: implement ASOF equality join");
         }
         else if (cmp < 0)
         {
+            // LOG_DEBUG(&Poco::Logger::get("XXXX"), "{}:{} ", __FILE__, __LINE__);
+            if (asof_join_state.hasMatch(left_cursor, asof_inequality))
+            {
+                // LOG_DEBUG(&Poco::Logger::get("XXXX"), "{}:{} ", __FILE__, __LINE__);
+
+                size_t i = 0;
+                for (const auto & col : left_columns)
+                    result_cols[i++]->insertFrom(*col, lpos);
+                for (const auto & col : asof_join_state.value.getColumns())
+                    result_cols[i++]->insertFrom(*col, asof_join_state.value_row);
+                chassert(i == result_cols.size());
+                left_cursor->next();
+                continue;
+            }
+            else
+            {
+                // LOG_DEBUG(&Poco::Logger::get("XXXX"), "{}:{} ", __FILE__, __LINE__);
+                asof_join_state.reset();
+            }
+            // LOG_DEBUG(&Poco::Logger::get("XXXX"), "{}:{} ", __FILE__, __LINE__);
+
             /// no matches for rows in left table, just pass them through
             size_t num = nextDistinct(*left_cursor);
-            if (isLeft(kind))
+            if (isLeft(kind) && num)
             {
                 /// return them with default values at right side
-                addRange(left_map, lpos, lpos + num);
-                addMany(right_map, DEFAULT_VALUE_INDEX, num);
+                size_t i = 0;
+                for (const auto & col : left_columns)
+                    result_cols[i++]->insertRangeFrom(*col, lpos, num);
+                for (; i < result_cols.size(); ++i)
+                    result_cols[i]->insertManyDefaults(num);
+                chassert(i == result_cols.size());
             }
         }
         else
         {
+            // LOG_DEBUG(&Poco::Logger::get("XXXX"), "{}:{} ", __FILE__, __LINE__);
+
             /// skip rows in right table until we find match for current row in left table
             nextDistinct(*right_cursor);
         }
     }
-
-    chassert(left_map.size() == right_map.size());
-    Chunk result;
-    {
-        Columns lcols = indexColumns(left_cursor.getCurrent().getColumns(), left_map);
-        for (auto & col : lcols)
-            result.addColumn(std::move(col));
-
-        Columns rcols = indexColumns(right_cursor.getCurrent().getColumns(), right_map);
-        for (auto & col : rcols)
-            result.addColumn(std::move(col));
-    }
-
-    return Status(std::move(result));
+    size_t num_rows = result_cols.empty() ? 0 : result_cols.front()->size();
+    return Status(Chunk(std::move(result_cols), num_rows));
 }
 
 
@@ -929,6 +1156,9 @@ IMergingAlgorithm::Status MergeJoinAlgorithm::merge()
         return std::move(*result);
     }
 
+    if (auto result = handleAsofJoinState())
+        return std::move(*result);
+
     if (cursors[0]->fullyCompleted() || cursors[1]->fullyCompleted())
     {
         if (!cursors[0]->fullyCompleted() && isLeftOrFull(kind))
diff --git a/src/Processors/Transforms/MergeJoinTransform.h b/src/Processors/Transforms/MergeJoinTransform.h
index 375c9ebd3cc..dbdda0b166b 100644
--- a/src/Processors/Transforms/MergeJoinTransform.h
+++ b/src/Processors/Transforms/MergeJoinTransform.h
@@ -37,62 +37,28 @@ using FullMergeJoinCursorPtr = std::unique_ptr<FullMergeJoinCursor>;
 /// Used instead of storing previous block
 struct JoinKeyRow
 {
-    std::vector<ColumnPtr> row;
-
     JoinKeyRow() = default;
 
-    explicit JoinKeyRow(const SortCursorImpl & impl_, size_t pos)
-    {
-        row.reserve(impl_.sort_columns.size());
-        for (const auto & col : impl_.sort_columns)
-        {
-            auto new_col = col->cloneEmpty();
-            new_col->insertFrom(*col, pos);
-            row.push_back(std::move(new_col));
-        }
-    }
+    JoinKeyRow(const FullMergeJoinCursor & cursor, size_t pos);
 
-    void reset()
-    {
-        row.clear();
-    }
+    bool equals(const FullMergeJoinCursor & cursor) const;
+    bool asofMatch(const FullMergeJoinCursor & cursor, ASOFJoinInequality asof_inequality) const;
 
-    bool equals(const SortCursorImpl & impl) const
-    {
-        if (row.empty())
-            return false;
+    void reset();
 
-        assert(this->row.size() == impl.sort_columns_size);
-        for (size_t i = 0; i < impl.sort_columns_size; ++i)
-        {
-            int cmp = this->row[i]->compareAt(0, impl.getRow(), *impl.sort_columns[i], impl.desc[i].nulls_direction);
-            if (cmp != 0)
-                return false;
-        }
-        return true;
-    }
+    std::vector<ColumnPtr> row;
 };
 
 /// Remembers previous key if it was joined in previous block
 class AnyJoinState : boost::noncopyable
 {
 public:
-    AnyJoinState() = default;
+    void set(size_t source_num, const FullMergeJoinCursor & cursor);
+    void setValue(Chunk value_);
 
-    void set(size_t source_num, const SortCursorImpl & cursor)
-    {
-        assert(cursor.rows);
-        keys[source_num] = JoinKeyRow(cursor, cursor.rows - 1);
-    }
+    void reset(size_t source_num);
 
-    void reset(size_t source_num)
-    {
-        keys[source_num].reset();
-    }
-
-    void setValue(Chunk value_) { value = std::move(value_); }
-
-    bool empty() const { return keys[0].row.empty() && keys[1].row.empty(); }
+    bool empty() const;
 
     /// current keys
     JoinKeyRow keys[2];
@@ -125,8 +91,8 @@ public:
         Chunk chunk;
     };
 
-    AllJoinState(const SortCursorImpl & lcursor, size_t lpos,
-                 const SortCursorImpl & rcursor, size_t rpos)
+    AllJoinState(const FullMergeJoinCursor & lcursor, size_t lpos,
+                 const FullMergeJoinCursor & rcursor, size_t rpos)
         : keys{JoinKeyRow(lcursor, lpos), JoinKeyRow(rcursor, rpos)}
     {
     }
@@ -194,6 +160,25 @@ private:
     size_t ridx = 0;
 };
 
+
+class AsofJoinState : boost::noncopyable
+{
+public:
+    void set(const FullMergeJoinCursor & rcursor, size_t rpos);
+    void reset();
+
+    bool hasMatch(const FullMergeJoinCursor & cursor, ASOFJoinInequality asof_inequality)
+    {
+        if (value.empty())
+            return false;
+        return key.asofMatch(cursor, asof_inequality);
+    }
+
+    JoinKeyRow key;
+    Chunk value;
+    size_t value_row = 0;
+};
+
 /*
  * Wrapper for SortCursorImpl
  */
@@ -239,11 +224,15 @@ public:
     const Block & sampleBlock() const { return sample_block; }
     Columns sampleColumns() const { return sample_block.getColumns(); }
 
-    const IColumn & getAsofColumn() const
+    const IColumn * getAsofColumn() const
     {
-        return *cursor.all_columns[asof_column_position];
+        if (!asof_column_position)
+            return nullptr;
+        return cursor.all_columns[*asof_column_position];
     }
 
+    String dump() const;
+
 private:
     Block sample_block;
     SortDescription desc;
@@ -251,7 +240,7 @@ private:
     Chunk current_chunk;
     bool recieved_all_blocks = false;
 
-    size_t asof_column_position;
+    std::optional<size_t> asof_column_position;
 };
 
 /*
@@ -284,12 +273,13 @@ private:
     std::optional<Status> handleAllJoinState();
     Status allJoin();
 
+    std::optional<Status> handleAsofJoinState();
     Status asofJoin();
 
+    MutableColumns getEmptyResultColumns() const;
     Chunk createBlockWithDefaults(size_t source_num);
     Chunk createBlockWithDefaults(size_t source_num, size_t start, size_t num_rows) const;
 
-
     /// For `USING` join key columns should have values from right side instead of defaults
     std::unordered_map<size_t, size_t> left_to_right_key_remap;
 
@@ -299,6 +289,7 @@ private:
     /// Keep some state to make handle data from different blocks
     AnyJoinState any_join_state;
     std::unique_ptr<AllJoinState> all_join_state;
+    AsofJoinState asof_join_state;
 
     JoinKind kind;
     JoinStrictness strictness;
diff --git a/src/Processors/tests/gtest_full_sorting_join.cpp b/src/Processors/tests/gtest_full_sorting_join.cpp
index 1dc410bd6b8..4e0727779b7 100644
--- a/src/Processors/tests/gtest_full_sorting_join.cpp
+++ b/src/Processors/tests/gtest_full_sorting_join.cpp
@@ -18,19 +18,53 @@
 #include <Processors/Sources/SourceFromSingleChunk.h>
 #include <Processors/Transforms/MergeJoinTransform.h>
 
+#include <Processors/Formats/Impl/PrettyCompactBlockOutputFormat.h>
+#include <Processors/Executors/CompletedPipelineExecutor.h>
+
+
 #include <QueryPipeline/QueryPipeline.h>
 
 using namespace DB;
 
+namespace
+{
+
+[[ maybe_unused ]]
+String dumpBlock(std::shared_ptr<ISource> source)
+{
+    WriteBufferFromOwnString buf;
+    {
+        Block header = source->getPort().getHeader();
+        QueryPipeline pipeline(source);
+        auto format = std::make_shared<PrettyCompactBlockOutputFormat>(buf, header, FormatSettings{}, false);
+        pipeline.complete(std::move(format));
+
+        CompletedPipelineExecutor executor(pipeline);
+        executor.execute();
+    }
+    return buf.str();
+}
+
+[[ maybe_unused ]]
+String dumpBlock(const Block & block)
+{
+    Block header = block.cloneEmpty();
+    Chunk data(block.getColumns(), block.rows());
+    auto source = std::make_shared<SourceFromSingleChunk>(header, std::move(data));
+    return dumpBlock(std::move(source));
+}
+
 UInt64 getAndPrintRandomSeed()
 {
     UInt64 seed = randomSeed();
-    std::cerr << __FILE__ << "::" << "TEST_RANDOM_SEED = " << seed << "ull" << std::endl;
+    if (const char * random_seed = std::getenv("TEST_RANDOM_SEED")) // NOLINT(concurrency-mt-unsafe)
+        seed = std::stoull(random_seed);
+
+    std::cerr << __FILE__ << " :: " << "TEST_RANDOM_SEED=" << seed << std::endl;
     return seed;
 }
 
 static UInt64 TEST_RANDOM_SEED = getAndPrintRandomSeed();
-
 static pcg64 rng(TEST_RANDOM_SEED);
 
 
@@ -94,42 +128,29 @@ QueryPipeline buildJoinPipeline(
 
 std::shared_ptr<ISource> oneColumnSource(const std::vector<std::vector<UInt64>> & values)
 {
-    Block header = { ColumnWithTypeAndName(std::make_shared<DataTypeUInt64>(), "x") };
+    Block header = {
+        ColumnWithTypeAndName(std::make_shared<DataTypeUInt64>(), "key"),
+        ColumnWithTypeAndName(std::make_shared<DataTypeUInt64>(), "idx"),
+    };
+
+    UInt64 idx = 0;
     Chunks chunks;
     for (const auto & chunk_values : values)
     {
-        auto column = ColumnUInt64::create();
+        auto key_column = ColumnUInt64::create();
+        auto idx_column = ColumnUInt64::create();
+
         for (auto n : chunk_values)
-            column->insertValue(n);
-        chunks.emplace_back(Chunk(Columns{std::move(column)}, chunk_values.size()));
+        {
+            key_column->insertValue(n);
+            idx_column->insertValue(idx);
+            ++idx;
+        }
+        chunks.emplace_back(Chunk(Columns{std::move(key_column), std::move(idx_column)}, chunk_values.size()));
     }
     return std::make_shared<SourceFromChunks>(header, std::move(chunks));
 }
 
-
-TEST(FullSortingJoin, Simple)
-try
-{
-    auto left_source = oneColumnSource({ {1, 2, 3, 4, 5} });
-    auto right_source = oneColumnSource({ {1}, {2}, {3}, {4}, {5} });
-
-    auto pipeline = buildJoinPipeline(left_source, right_source);
-    PullingPipelineExecutor executor(pipeline);
-
-    Block block;
-
-    size_t total_result_rows = 0;
-    while (executor.pull(block))
-        total_result_rows += block.rows();
-
-    ASSERT_EQ(total_result_rows, 5);
-}
-catch (Exception & e)
-{
-    std::cout << e.getStackTraceString() << std::endl;
-    throw;
-}
-
 class SourceChunksBuilder
 {
 public:
@@ -163,7 +184,7 @@ public:
         return;
     }
 
-    std::shared_ptr<ISource> build()
+    std::shared_ptr<ISource> getSource()
     {
         addChunk();
 
@@ -215,9 +236,11 @@ Block executePipeline(QueryPipeline && pipeline)
 template <typename T>
 void assertColumnVectorEq(const typename ColumnVector<T>::Container & expected, const Block & block, const std::string & name)
 {
-    const auto & actual = assert_cast<const ColumnVector<T> *>(block.getByName(name).column.get())->getData();
-    EXPECT_EQ(actual.size(), expected.size());
-    ASSERT_EQ(actual, expected);
+    const auto * actual = typeid_cast<const ColumnVector<T> *>(block.getByName(name).column.get());
+    ASSERT_TRUE(actual) << "unexpected column type: " << block.getByName(name).column->dumpStructure() << "expected: " << typeid(ColumnVector<T>).name();
+
+    EXPECT_EQ(actual->getData().size(), expected.size());
+    ASSERT_EQ(actual->getData(), expected) << "column name: " << name;
 }
 
 template <typename T>
@@ -263,7 +286,90 @@ void generateNextKey(UInt64 & k1, String & k2)
     k2 = new_k2;
 }
 
-TEST(FullSortingJoin, Any)
+bool isStrict(ASOFJoinInequality inequality)
+{
+    return inequality == ASOFJoinInequality::Less || inequality == ASOFJoinInequality::Greater;
+}
+
+}
+
+TEST(FullSortingJoin, AllAnyOneKey)
+try
+{
+    {
+        SCOPED_TRACE("Inner All");
+        Block result = executePipeline(buildJoinPipeline(
+            oneColumnSource({ {1, 2, 3, 4, 5} }),
+            oneColumnSource({ {1}, {2}, {3}, {4}, {5} }),
+            1, JoinKind::Inner, JoinStrictness::All));
+
+        assertColumnVectorEq<UInt64>(ColumnUInt64::Container({0, 1, 2, 3, 4}), result, "t1.idx");
+        assertColumnVectorEq<UInt64>(ColumnUInt64::Container({0, 1, 2, 3, 4}), result, "t2.idx");
+    }
+    {
+        SCOPED_TRACE("Inner Any");
+        Block result = executePipeline(buildJoinPipeline(
+            oneColumnSource({ {1, 2, 3, 4, 5} }),
+            oneColumnSource({ {1}, {2}, {3}, {4}, {5} }),
+            1, JoinKind::Inner, JoinStrictness::Any));
+        assertColumnVectorEq<UInt64>(ColumnUInt64::Container({0, 1, 2, 3, 4}), result, "t1.idx");
+        assertColumnVectorEq<UInt64>(ColumnUInt64::Container({0, 1, 2, 3, 4}), result, "t2.idx");
+    }
+    {
+        SCOPED_TRACE("Inner All");
+        Block result = executePipeline(buildJoinPipeline(
+            oneColumnSource({ {2, 2, 2}, {2, 3}, {3, 5} }),
+            oneColumnSource({ {1, 1, 1}, {2, 2}, {3, 4} }),
+            1, JoinKind::Inner, JoinStrictness::All));
+        assertColumnVectorEq<UInt64>(ColumnUInt64::Container({0, 1, 2, 0, 1, 2, 3, 3, 4, 5}), result, "t1.idx");
+        assertColumnVectorEq<UInt64>(ColumnUInt64::Container({3, 3, 3, 4, 4, 4, 3, 4, 5, 5}), result, "t2.idx");
+    }
+    {
+        SCOPED_TRACE("Inner Any");
+        Block result = executePipeline(buildJoinPipeline(
+            oneColumnSource({ {2, 2, 2}, {2, 3}, {3, 5} }),
+            oneColumnSource({ {1, 1, 1}, {2, 2}, {3, 4} }),
+            1, JoinKind::Inner, JoinStrictness::Any));
+        assertColumnVectorEq<UInt64>(ColumnUInt64::Container({0, 4}), result, "t1.idx");
+        assertColumnVectorEq<UInt64>(ColumnUInt64::Container({3, 5}), result, "t2.idx");
+    }
+    {
+        SCOPED_TRACE("Inner Any");
+        Block result = executePipeline(buildJoinPipeline(
+            oneColumnSource({ {2, 2, 2, 2}, {3}, {3, 5} }),
+            oneColumnSource({ {1, 1, 1, 2}, {2}, {3, 4} }),
+            1, JoinKind::Inner, JoinStrictness::Any));
+        assertColumnVectorEq<UInt64>(ColumnUInt64::Container({0, 4}), result, "t1.idx");
+        assertColumnVectorEq<UInt64>(ColumnUInt64::Container({3, 5}), result, "t2.idx");
+    }
+    {
+
+        SCOPED_TRACE("Left Any");
+        Block result = executePipeline(buildJoinPipeline(
+            oneColumnSource({ {2, 2, 2}, {2, 3}, {3, 5} }),
+            oneColumnSource({ {1, 1, 1}, {2, 2}, {3, 4} }),
+            1, JoinKind::Left, JoinStrictness::Any));
+        assertColumnVectorEq<UInt64>(ColumnUInt64::Container({0, 1, 2, 3, 4, 5, 6}), result, "t1.idx");
+        assertColumnVectorEq<UInt64>(ColumnUInt64::Container({3, 3, 3, 3, 5, 5, 0}), result, "t2.idx");
+    }
+    {
+        SCOPED_TRACE("Left Any");
+        Block result = executePipeline(buildJoinPipeline(
+            oneColumnSource({ {2, 2, 2, 2}, {3}, {3, 5} }),
+            oneColumnSource({ {1, 1, 1, 2}, {2}, {3, 4} }),
+            1, JoinKind::Left, JoinStrictness::Any));
+        assertColumnVectorEq<UInt64>(ColumnUInt64::Container({0, 1, 2, 3, 4, 5, 6}), result, "t1.idx");
+        assertColumnVectorEq<UInt64>(ColumnUInt64::Container({3, 3, 3, 3, 5, 5, 0}), result, "t2.idx");
+    }
+}
+catch (Exception & e)
+{
+    std::cout << e.getStackTraceString() << std::endl;
+    throw;
+}
+
+
+TEST(FullSortingJoin, AnyRandomized)
 try
 {
     JoinKind kind = getRandomFrom({JoinKind::Inner, JoinKind::Left, JoinKind::Right});
@@ -288,7 +394,7 @@ try
     auto expected_left = ColumnString::create();
     auto expected_right = ColumnString::create();
 
-    UInt64 k1 = 0;
+    UInt64 k1 = 1;
     String k2 = "";
 
     auto get_attr = [&](const String & side, size_t idx) -> String
@@ -338,7 +444,7 @@ try
     }
 
     Block result_block = executePipeline(buildJoinPipeline(
-        left_source.build(), right_source.build(), /* key_length = */ 2,
+        left_source.getSource(), right_source.getSource(), /* key_length = */ 2,
         kind, JoinStrictness::Any));
     assertColumnEq<ColumnString>(*expected_left, result_block, "t1.attr");
     assertColumnEq<ColumnString>(*expected_right, result_block, "t2.attr");
@@ -356,10 +462,10 @@ try
         {std::make_shared<DataTypeString>(), "key"},
         {std::make_shared<DataTypeUInt64>(), "t"},
     });
-
     left_source.addRow({"AMZN", 3});
     left_source.addRow({"AMZN", 4});
     left_source.addRow({"AMZN", 6});
+    left_source.addRow({"SBUX", 10});
 
     SourceChunksBuilder right_source({
         {std::make_shared<DataTypeString>(), "key"},
@@ -371,14 +477,19 @@ try
     right_source.addRow({"AAPL", 2, 98});
     right_source.addRow({"AAPL", 3, 99});
     right_source.addRow({"AMZN", 1, 100});
+    right_source.addRow({"AMZN", 2, 0});
+    right_source.addChunk();
     right_source.addRow({"AMZN", 2, 110});
     right_source.addChunk();
     right_source.addRow({"AMZN", 4, 130});
     right_source.addRow({"AMZN", 5, 140});
+    right_source.addRow({"SBUX", 8, 180});
+    right_source.addChunk();
+    right_source.addRow({"SBUX", 9, 190});
 
     {
         Block result_block = executePipeline(buildJoinPipeline(
-            left_source.build(), right_source.build(), /* key_length = */ 2,
+            left_source.getSource(), right_source.getSource(), /* key_length = */ 2,
             JoinKind::Inner, JoinStrictness::Asof, ASOFJoinInequality::LessOrEquals));
         auto values = getValuesFromBlock(result_block, {"t1.key", "t1.t", "t2.t", "t2.value"});
 
@@ -388,18 +499,19 @@ try
         }));
     }
 
-    // {
-    //     Block result_block = executePipeline(buildJoinPipeline(
-    //         left_source.build(), right_source.build(), /* key_length = */ 2,
-    //         JoinKind::Inner, JoinStrictness::Asof, ASOFJoinInequality::GreaterOrEquals));
-    //     auto values = getValuesFromBlock(result_block, {"t1.key", "t1.t", "t2.t", "t2.value"});
+    {
+        Block result_block = executePipeline(buildJoinPipeline(
+            left_source.getSource(), right_source.getSource(), /* key_length = */ 2,
+            JoinKind::Inner, JoinStrictness::Asof, ASOFJoinInequality::GreaterOrEquals));
+        auto values = getValuesFromBlock(result_block, {"t1.key", "t1.t", "t2.t", "t2.value"});
 
-    //     ASSERT_EQ(values, (std::vector<std::vector<Field>>{
-    //         {"AMZN", 3u, 2u, 110u},
-    //         {"AMZN", 4u, 4u, 130u},
-    //         {"AMZN", 6u, 5u, 140u},
-    //     }));
-    // }
+        ASSERT_EQ(values, (std::vector<std::vector<Field>>{
+            {"AMZN", 3u, 2u, 110u},
+            {"AMZN", 4u, 4u, 130u},
+            {"AMZN", 6u, 5u, 140u},
+            {"SBUX", 10u, 9u, 190u},
+        }));
+    }
 }
 catch (Exception & e)
 {
@@ -418,8 +530,7 @@ try
         {std::make_shared<DataTypeUInt64>(), "value"},
     });
 
-    UInt64 p = std::uniform_int_distribution<>(0, 2)(rng);
-    double break_prob = p == 0 ? 0.0 : (p == 1 ? 0.5 : 1.0);
+    double break_prob = getRandomDoubleQuantized(2);
     std::uniform_real_distribution<> prob_dis(0.0, 1.0);
     for (const auto & row : std::vector<std::vector<Field>>{ {1, 101}, {2, 102}, {4, 104}, {5, 105}, {11, 111}, {15, 115} })
     {
@@ -427,7 +538,7 @@ try
         if (prob_dis(rng) < break_prob)
             right_source_builder.addChunk();
     }
-    auto right_source = right_source_builder.build();
+    auto right_source = right_source_builder.getSource();
 
     auto pipeline = buildJoinPipeline(
         left_source, right_source, /* key_length = */ 1,
@@ -436,7 +547,7 @@ try
     Block result_block = executePipeline(std::move(pipeline));
 
     ASSERT_EQ(
-        assert_cast<const ColumnUInt64 *>(result_block.getByName("t1.x").column.get())->getData(),
+        assert_cast<const ColumnUInt64 *>(result_block.getByName("t1.key").column.get())->getData(),
         (ColumnUInt64::Container{3, 3, 3, 3, 3, 5, 5, 6, 9, 9, 10})
     );
 
@@ -456,15 +567,14 @@ catch (Exception & e)
     throw;
 }
 
-TEST(FullSortingJoin, AsofGeneratedTestData)
+TEST(FullSortingJoin, AsofLessGeneratedTestData)
 try
 {
-    auto join_kind = getRandomFrom({JoinKind::Inner, JoinKind::Left});
+    auto join_kind = getRandomFrom({ JoinKind::Inner, JoinKind::Left });
 
-    auto asof_inequality = getRandomFrom({
-        ASOFJoinInequality::Less, ASOFJoinInequality::LessOrEquals,
-        // ASOFJoinInequality::Greater, ASOFJoinInequality::GreaterOrEquals,
-    });
+    auto asof_inequality = getRandomFrom({ ASOFJoinInequality::Less, ASOFJoinInequality::LessOrEquals });
+
+    SCOPED_TRACE(fmt::format("{} {}", join_kind, asof_inequality));
 
     SourceChunksBuilder left_source_builder({
         {std::make_shared<DataTypeUInt64>(), "k1"},
@@ -485,11 +595,13 @@ try
 
     ColumnInt64::Container expected;
 
-    UInt64 k1 = 0;
-    String k2 = "asdfg";
+    UInt64 k1 = 1;
+    String k2 = "";
     auto key_num_total = std::uniform_int_distribution<>(1, 1000)(rng);
     for (size_t key_num = 0; key_num < key_num_total; ++key_num)
     {
+        generateNextKey(k1, k2);
+
         Int64 left_t = 0;
         size_t num_left_rows = std::uniform_int_distribution<>(1, 100)(rng);
         for (size_t i = 0; i < num_left_rows; ++i)
@@ -504,18 +616,10 @@ try
             auto right_t = left_t;
             for (size_t j = 0; j < num_matches; ++j)
             {
-                int min_step = 1;
-                if (asof_inequality == ASOFJoinInequality::LessOrEquals || asof_inequality == ASOFJoinInequality::GreaterOrEquals)
-                    min_step = 0;
+                int min_step = isStrict(asof_inequality) ? 1 : 0;
                 right_t += std::uniform_int_distribution<>(min_step, 3)(rng);
 
-                bool is_match = false;
-
-                if (asof_inequality == ASOFJoinInequality::LessOrEquals || asof_inequality == ASOFJoinInequality::Less)
-                    is_match = j == 0;
-                else if (asof_inequality == ASOFJoinInequality::GreaterOrEquals || asof_inequality == ASOFJoinInequality::Greater)
-                    is_match = j == num_matches - 1;
-
+                bool is_match = j == 0;
                 right_source_builder.addRow({k1, k2, right_t, is_match ? 100 * left_t : -1});
             }
             /// next left_t should be greater than right_t not to match with previous rows
@@ -523,7 +627,7 @@ try
         }
 
         /// generate some rows with greater left_t to check that they are not matched
-        num_left_rows = std::uniform_int_distribution<>(1, 100)(rng);
+        num_left_rows = std::bernoulli_distribution(0.5)(rng) ? std::uniform_int_distribution<>(1, 100)(rng) : 0;
         for (size_t i = 0; i < num_left_rows; ++i)
         {
             left_t += std::uniform_int_distribution<>(1, 10)(rng);
@@ -532,12 +636,10 @@ try
             if (join_kind == JoinKind::Left)
                 expected.push_back(-10 * left_t);
         }
-
-        generateNextKey(k1, k2);
     }
 
     Block result_block = executePipeline(buildJoinPipeline(
-        left_source_builder.build(), right_source_builder.build(),
+        left_source_builder.getSource(), right_source_builder.getSource(),
         /* key_length = */ 3,
         join_kind, JoinStrictness::Asof, asof_inequality));
 
@@ -548,7 +650,97 @@ try
 
     assertColumnVectorEq<Int64>(expected, result_block, "t2.attr");
 }
-catch (Exception & e) {
+catch (Exception & e)
+{
+    std::cout << e.getStackTraceString() << std::endl;
+    throw;
+}
+
+TEST(FullSortingJoin, AsofGreaterGeneratedTestData)
+try
+{
+    auto join_kind = getRandomFrom({ JoinKind::Inner, JoinKind::Left });
+
+    auto asof_inequality = getRandomFrom({ ASOFJoinInequality::Greater, ASOFJoinInequality::GreaterOrEquals });
+
+    SCOPED_TRACE(fmt::format("{} {}", join_kind, asof_inequality));
+
+    SourceChunksBuilder left_source_builder({
+        {std::make_shared<DataTypeUInt64>(), "k1"},
+        {std::make_shared<DataTypeString>(), "k2"},
+        {std::make_shared<DataTypeUInt64>(), "t"},
+        {std::make_shared<DataTypeInt64>(), "attr"},
+    });
+
+    SourceChunksBuilder right_source_builder({
+        {std::make_shared<DataTypeUInt64>(), "k1"},
+        {std::make_shared<DataTypeString>(), "k2"},
+        {std::make_shared<DataTypeUInt64>(), "t"},
+        {std::make_shared<DataTypeInt64>(), "attr"},
+    });
+
+    left_source_builder.break_prob = getRandomDoubleQuantized();
+    right_source_builder.break_prob = getRandomDoubleQuantized();
+
+    ColumnInt64::Container expected;
+
+    UInt64 k1 = 1;
+    String k2 = "";
+    UInt64 left_t = 0;
+
+    auto key_num_total = std::uniform_int_distribution<>(1, 100)(rng);
+    for (size_t key_num = 0; key_num < key_num_total; ++key_num)
+    {
+        generateNextKey(k1, k2);
+
+        /// generate some rows with smaller left_t to check that they are not matched
+        size_t num_left_rows = std::bernoulli_distribution(0.5)(rng) ? std::uniform_int_distribution<>(1, 10)(rng) : 0;
+        for (size_t i = 0; i < num_left_rows; ++i)
+        {
+            left_t += std::uniform_int_distribution<>(1, 10)(rng);
+            left_source_builder.addRow({k1, k2, left_t, -10 * left_t});
+
+            if (join_kind == JoinKind::Left)
+                expected.push_back(-10 * left_t);
+        }
+
+        if (std::bernoulli_distribution(0.1)(rng))
+            continue;
+
+        size_t num_right_matches = std::uniform_int_distribution<>(1, 10)(rng);
+        auto right_t = left_t + std::uniform_int_distribution<>(isStrict(asof_inequality) ? 0 : 1, 10)(rng);
+        for (size_t j = 0; j < num_right_matches; ++j)
+        {
+            right_t += std::uniform_int_distribution<>(0, 3)(rng);
+            bool is_match = j == num_right_matches - 1;
+            right_source_builder.addRow({k1, k2, right_t, is_match ? 100 * right_t : -1});
+        }
+
+        /// next left_t should be greater than (or equals) right_t to match with previous rows
+        left_t = right_t + std::uniform_int_distribution<>(isStrict(asof_inequality) ? 1 : 0, 10)(rng);
+        size_t num_left_matches = std::uniform_int_distribution<>(1, 10)(rng);
+        for (size_t j = 0; j < num_left_matches; ++j)
+        {
+            left_t += std::uniform_int_distribution<>(0, 3)(rng);
+            left_source_builder.addRow({k1, k2, left_t, 10 * right_t});
+            expected.push_back(10 * right_t);
+        }
+    }
+
+    Block result_block = executePipeline(buildJoinPipeline(
+        left_source_builder.getSource(), right_source_builder.getSource(),
+        /* key_length = */ 3,
+        join_kind, JoinStrictness::Asof, asof_inequality));
+
+    assertColumnVectorEq<Int64>(expected, result_block, "t1.attr");
+
+    for (auto & e : expected)
+        e = e < 0 ? 0 : 10 * e; /// non matched rows from left table have negative attr
+
+    assertColumnVectorEq<Int64>(expected, result_block, "t2.attr");
+}
+catch (Exception & e)
+{
     std::cout << e.getStackTraceString() << std::endl;
     throw;
 }
diff --git a/tests/queries/0_stateless/02276_full_sort_join_unsupported.sql b/tests/queries/0_stateless/02276_full_sort_join_unsupported.sql
index a4e60ff54dd..03936107563 100644
--- a/tests/queries/0_stateless/02276_full_sort_join_unsupported.sql
+++ b/tests/queries/0_stateless/02276_full_sort_join_unsupported.sql
@@ -19,7 +19,7 @@ SELECT * FROM t1 ANTI JOIN t2 ON t1.key = t2.key; -- { serverError NOT_IMPLEMENT
 
 SELECT * FROM t1 SEMI JOIN t2 ON t1.key = t2.key; -- { serverError NOT_IMPLEMENTED }
 
-SELECT * FROM t1 ASOF JOIN t2 ON t1.key = t2.key AND t1.val > t2.val; -- { serverError NOT_IMPLEMENTED }
+-- SELECT * FROM t1 ASOF JOIN t2 ON t1.key = t2.key AND t1.val > t2.val; -- { serverError NOT_IMPLEMENTED }
 
 SELECT * FROM t1 ANY JOIN t2 ON t1.key = t2.key SETTINGS any_join_distinct_right_table_keys = 1; -- { serverError NOT_IMPLEMENTED }
 

From b6b55cfb187ddcbb6316d905c60d9be481bfb1a4 Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Wed, 27 Sep 2023 09:05:50 +0000
Subject: [PATCH 0054/1488] fix asof join

---
 src/Processors/Transforms/MergeJoinTransform.cpp |  2 +-
 src/Processors/tests/gtest_full_sorting_join.cpp | 10 +++++++---
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/src/Processors/Transforms/MergeJoinTransform.cpp b/src/Processors/Transforms/MergeJoinTransform.cpp
index 638f2f1cb10..ad564557c36 100644
--- a/src/Processors/Transforms/MergeJoinTransform.cpp
+++ b/src/Processors/Transforms/MergeJoinTransform.cpp
@@ -1171,7 +1171,7 @@ IMergingAlgorithm::Status MergeJoinAlgorithm::merge()
     }
 
     /// check if blocks are not intersecting at all
-    if (int cmp = totallyCompare(cursors[0]->cursor, cursors[1]->cursor, null_direction_hint); cmp != 0)
+    if (int cmp = totallyCompare(cursors[0]->cursor, cursors[1]->cursor, null_direction_hint); cmp != 0 && strictness != JoinStrictness::Asof)
     {
         if (cmp < 0)
         {
diff --git a/src/Processors/tests/gtest_full_sorting_join.cpp b/src/Processors/tests/gtest_full_sorting_join.cpp
index 4e0727779b7..2ecf7805df4 100644
--- a/src/Processors/tests/gtest_full_sorting_join.cpp
+++ b/src/Processors/tests/gtest_full_sorting_join.cpp
@@ -30,13 +30,13 @@ namespace
 {
 
 [[ maybe_unused ]]
-String dumpBlock(std::shared_ptr<ISource> source)
+String dumpBlockSource(std::shared_ptr<ISource> source, bool mono_block = false)
 {
     WriteBufferFromOwnString buf;
     {
         Block header = source->getPort().getHeader();
         QueryPipeline pipeline(source);
-        auto format = std::make_shared<PrettyCompactBlockOutputFormat>(buf, header, FormatSettings{}, false);
+        auto format = std::make_shared<PrettyCompactBlockOutputFormat>(buf, header, FormatSettings{}, mono_block);
         pipeline.complete(std::move(format));
 
         CompletedPipelineExecutor executor(pipeline);
@@ -51,7 +51,7 @@ String dumpBlock(const Block & block)
     Block header = block.cloneEmpty();
     Chunk data(block.getColumns(), block.rows());
     auto source = std::make_shared<SourceFromSingleChunk>(header, std::move(data));
-    return dumpBlock(std::move(source));
+    return dumpBlockSource(std::move(source));
 }
 
 UInt64 getAndPrintRandomSeed()
@@ -732,6 +732,10 @@ try
         /* key_length = */ 3,
         join_kind, JoinStrictness::Asof, asof_inequality));
 
+    // std::cerr << "============ left ============" << std::endl << dumpBlockSource(left_source_builder.getSource()) << std::endl;
+    // std::cerr << "============ right ============" << std::endl << dumpBlockSource(right_source_builder.getSource()) << std::endl;
+    // std::cerr << "============ result ============" << std::endl << dumpBlock(result_block) << std::endl;
+
     assertColumnVectorEq<Int64>(expected, result_block, "t1.attr");
 
     for (auto & e : expected)

From d015a023bb85c72c2747cebed12d065d5aad1159 Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Wed, 27 Sep 2023 09:56:30 +0000
Subject: [PATCH 0055/1488] unique rng seed per each test in
 gtest_full_sorting_join

---
 .../tests/gtest_full_sorting_join.cpp         | 122 ++++++++++--------
 1 file changed, 69 insertions(+), 53 deletions(-)

diff --git a/src/Processors/tests/gtest_full_sorting_join.cpp b/src/Processors/tests/gtest_full_sorting_join.cpp
index 2ecf7805df4..2bd3357eff9 100644
--- a/src/Processors/tests/gtest_full_sorting_join.cpp
+++ b/src/Processors/tests/gtest_full_sorting_join.cpp
@@ -54,20 +54,6 @@ String dumpBlock(const Block & block)
     return dumpBlockSource(std::move(source));
 }
 
-UInt64 getAndPrintRandomSeed()
-{
-    UInt64 seed = randomSeed();
-    if (const char * random_seed = std::getenv("TEST_RANDOM_SEED")) // NOLINT(concurrency-mt-unsafe)
-        seed = std::stoull(random_seed);
-
-    std::cerr << __FILE__ << " :: " << "TEST_RANDOM_SEED=" << seed << std::endl;
-    return seed;
-}
-
-static UInt64 TEST_RANDOM_SEED = getAndPrintRandomSeed();
-static pcg64 rng(TEST_RANDOM_SEED);
-
-
 QueryPipeline buildJoinPipeline(
     std::shared_ptr<ISource> left_source,
     std::shared_ptr<ISource> right_source,
@@ -154,7 +140,6 @@ std::shared_ptr<ISource> oneColumnSource(const std::vector<std::vector<UInt64>>
 class SourceChunksBuilder
 {
 public:
-    double break_prob = 0.0;
 
     explicit SourceChunksBuilder(const Block & header_)
         : header(header_)
@@ -163,13 +148,20 @@ public:
         chassert(!current_chunk.empty());
     }
 
+    void setBreakProbability(pcg64 & rng_)
+    {
+        /// random probability with possibility to have exact 0.0 and 1.0 values
+        break_prob = std::uniform_int_distribution<size_t>(0, 5)(rng_) / static_cast<double>(5);
+        rng = &rng_;
+    }
+
     void addRow(const std::vector<Field> & row)
     {
         chassert(row.size() == current_chunk.size());
         for (size_t i = 0; i < current_chunk.size(); ++i)
             current_chunk[i]->insert(row[i]);
 
-        if (break_prob > 0.0 && std::uniform_real_distribution<>(0.0, 1.0)(rng) < break_prob)
+        if (rng && std::uniform_real_distribution<>(0.0, 1.0)(*rng) < break_prob)
             addChunk();
     }
 
@@ -200,6 +192,9 @@ private:
     Block header;
     Chunks chunks;
     MutableColumns current_chunk;
+
+    pcg64 * rng = nullptr;
+    double break_prob = 0.0;
 };
 
 
@@ -239,8 +234,20 @@ void assertColumnVectorEq(const typename ColumnVector<T>::Container & expected,
     const auto * actual = typeid_cast<const ColumnVector<T> *>(block.getByName(name).column.get());
     ASSERT_TRUE(actual) << "unexpected column type: " << block.getByName(name).column->dumpStructure() << "expected: " << typeid(ColumnVector<T>).name();
 
+    auto get_first_diff = [&]() -> String
+    {
+        const auto & actual_data = actual->getData();
+        size_t num_rows = std::min(expected.size(), actual_data.size());
+        for (size_t i = 0; i < num_rows; ++i)
+        {
+            if (expected[i] != actual_data[i])
+                return fmt::format(", expected: {}, actual: {} at row {}", expected[i], actual_data[i], i);
+        }
+        return "";
+    };
+
     EXPECT_EQ(actual->getData().size(), expected.size());
-    ASSERT_EQ(actual->getData(), expected) << "column name: " << name;
+    ASSERT_EQ(actual->getData(), expected) << "column name: " << name << get_first_diff();
 }
 
 template <typename T>
@@ -264,20 +271,14 @@ void assertColumnEq(const IColumn & expected, const Block & block, const std::st
 }
 
 template <typename T>
-T getRandomFrom(const std::initializer_list<T> & opts)
+T getRandomFrom(pcg64 & rng, const std::initializer_list<T> & opts)
 {
     std::vector<T> options(opts.begin(), opts.end());
     size_t idx = std::uniform_int_distribution<size_t>(0, options.size() - 1)(rng);
     return options[idx];
 }
 
-/// Used to have accurate 0.0 and 1.0 probabilities
-double getRandomDoubleQuantized(size_t quants = 5)
-{
-    return std::uniform_int_distribution<size_t>(0, quants)(rng) / static_cast<double>(quants);
-}
-
-void generateNextKey(UInt64 & k1, String & k2)
+void generateNextKey(pcg64 & rng, UInt64 & k1, String & k2)
 {
     size_t str_len = std::uniform_int_distribution<>(1, 10)(rng);
     String new_k2 = getRandomASCIIString(str_len, rng);
@@ -293,6 +294,28 @@ bool isStrict(ASOFJoinInequality inequality)
 
 }
 
+class FullSortingJoinRandomized : public ::testing::Test
+{
+public:
+    FullSortingJoinRandomized() = default;
+
+    void SetUp() override
+    {
+        UInt64 seed = randomSeed();
+        if (const char * random_seed = std::getenv("TEST_RANDOM_SEED")) // NOLINT(concurrency-mt-unsafe)
+            seed = std::stoull(random_seed);
+
+        std::cerr << "TEST_RANDOM_SEED=" << seed << std::endl;
+        rng = pcg64(seed);
+    }
+
+    void TearDown() override
+    {
+    }
+
+    pcg64 rng;
+};
+
 TEST(FullSortingJoin, AllAnyOneKey)
 try
 {
@@ -369,10 +392,10 @@ catch (Exception & e)
 }
 
 
-TEST(FullSortingJoin, AnyRandomized)
+TEST_F(FullSortingJoinRandomized, Any)
 try
 {
-    JoinKind kind = getRandomFrom({JoinKind::Inner, JoinKind::Left, JoinKind::Right});
+    JoinKind kind = getRandomFrom(rng, {JoinKind::Inner, JoinKind::Left, JoinKind::Right});
 
     SourceChunksBuilder left_source({
         {std::make_shared<DataTypeUInt64>(), "k1"},
@@ -386,8 +409,8 @@ try
         {std::make_shared<DataTypeString>(), "attr"},
     });
 
-    left_source.break_prob = getRandomDoubleQuantized();
-    right_source.break_prob = getRandomDoubleQuantized();
+    left_source.setBreakProbability(rng);
+    right_source.setBreakProbability(rng);
 
     size_t num_keys = std::uniform_int_distribution<>(100, 1000)(rng);
 
@@ -404,7 +427,7 @@ try
 
     for (size_t i = 0; i < num_keys; ++i)
     {
-        generateNextKey(k1, k2);
+        generateNextKey(rng, k1, k2);
 
         /// Key is present in left, right or both tables. Both tables is more probable.
         size_t key_presence = std::uniform_int_distribution<>(0, 10)(rng);
@@ -520,7 +543,7 @@ catch (Exception & e)
 }
 
 
-TEST(FullSortingJoin, AsofOnlyColumn)
+TEST_F(FullSortingJoinRandomized, AsofOnlyColumn)
 try
 {
     auto left_source = oneColumnSource({ {3}, {3, 3, 3}, {3, 5, 5, 6}, {9, 9}, {10, 20} });
@@ -530,14 +553,11 @@ try
         {std::make_shared<DataTypeUInt64>(), "value"},
     });
 
-    double break_prob = getRandomDoubleQuantized(2);
-    std::uniform_real_distribution<> prob_dis(0.0, 1.0);
+    right_source_builder.setBreakProbability(rng);
+
     for (const auto & row : std::vector<std::vector<Field>>{ {1, 101}, {2, 102}, {4, 104}, {5, 105}, {11, 111}, {15, 115} })
-    {
         right_source_builder.addRow(row);
-        if (prob_dis(rng) < break_prob)
-            right_source_builder.addChunk();
-    }
+
     auto right_source = right_source_builder.getSource();
 
     auto pipeline = buildJoinPipeline(
@@ -567,12 +587,12 @@ catch (Exception & e)
     throw;
 }
 
-TEST(FullSortingJoin, AsofLessGeneratedTestData)
+TEST_F(FullSortingJoinRandomized, AsofLessGeneratedTestData)
 try
 {
-    auto join_kind = getRandomFrom({ JoinKind::Inner, JoinKind::Left });
+    auto join_kind = getRandomFrom(rng, { JoinKind::Inner, JoinKind::Left });
 
-    auto asof_inequality = getRandomFrom({ ASOFJoinInequality::Less, ASOFJoinInequality::LessOrEquals });
+    auto asof_inequality = getRandomFrom(rng, { ASOFJoinInequality::Less, ASOFJoinInequality::LessOrEquals });
 
     SCOPED_TRACE(fmt::format("{} {}", join_kind, asof_inequality));
 
@@ -590,8 +610,8 @@ try
         {std::make_shared<DataTypeInt64>(), "attr"},
     });
 
-    left_source_builder.break_prob = getRandomDoubleQuantized();
-    right_source_builder.break_prob = getRandomDoubleQuantized();
+    left_source_builder.setBreakProbability(rng);
+    right_source_builder.setBreakProbability(rng);
 
     ColumnInt64::Container expected;
 
@@ -600,7 +620,7 @@ try
     auto key_num_total = std::uniform_int_distribution<>(1, 1000)(rng);
     for (size_t key_num = 0; key_num < key_num_total; ++key_num)
     {
-        generateNextKey(k1, k2);
+        generateNextKey(rng, k1, k2);
 
         Int64 left_t = 0;
         size_t num_left_rows = std::uniform_int_distribution<>(1, 100)(rng);
@@ -656,12 +676,12 @@ catch (Exception & e)
     throw;
 }
 
-TEST(FullSortingJoin, AsofGreaterGeneratedTestData)
+TEST_F(FullSortingJoinRandomized, AsofGreaterGeneratedTestData)
 try
 {
-    auto join_kind = getRandomFrom({ JoinKind::Inner, JoinKind::Left });
+    auto join_kind = getRandomFrom(rng, { JoinKind::Inner, JoinKind::Left });
 
-    auto asof_inequality = getRandomFrom({ ASOFJoinInequality::Greater, ASOFJoinInequality::GreaterOrEquals });
+    auto asof_inequality = getRandomFrom(rng, { ASOFJoinInequality::Greater, ASOFJoinInequality::GreaterOrEquals });
 
     SCOPED_TRACE(fmt::format("{} {}", join_kind, asof_inequality));
 
@@ -679,8 +699,8 @@ try
         {std::make_shared<DataTypeInt64>(), "attr"},
     });
 
-    left_source_builder.break_prob = getRandomDoubleQuantized();
-    right_source_builder.break_prob = getRandomDoubleQuantized();
+    left_source_builder.setBreakProbability(rng);
+    right_source_builder.setBreakProbability(rng);
 
     ColumnInt64::Container expected;
 
@@ -691,7 +711,7 @@ try
     auto key_num_total = std::uniform_int_distribution<>(1, 100)(rng);
     for (size_t key_num = 0; key_num < key_num_total; ++key_num)
     {
-        generateNextKey(k1, k2);
+        generateNextKey(rng, k1, k2);
 
         /// generate some rows with smaller left_t to check that they are not matched
         size_t num_left_rows = std::bernoulli_distribution(0.5)(rng) ? std::uniform_int_distribution<>(1, 10)(rng) : 0;
@@ -732,10 +752,6 @@ try
         /* key_length = */ 3,
         join_kind, JoinStrictness::Asof, asof_inequality));
 
-    // std::cerr << "============ left ============" << std::endl << dumpBlockSource(left_source_builder.getSource()) << std::endl;
-    // std::cerr << "============ right ============" << std::endl << dumpBlockSource(right_source_builder.getSource()) << std::endl;
-    // std::cerr << "============ result ============" << std::endl << dumpBlock(result_block) << std::endl;
-
     assertColumnVectorEq<Int64>(expected, result_block, "t1.attr");
 
     for (auto & e : expected)

From 3e9090cbebb3cdb97f4dbb4b9d03c7e9d9e242b1 Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Wed, 27 Sep 2023 11:14:37 +0000
Subject: [PATCH 0056/1488] fix

---
 .../sql-reference/statements/select/join.md   |  5 ++-
 .../Transforms/MergeJoinTransform.cpp         | 41 ++++++++++++-------
 .../Transforms/MergeJoinTransform.h           | 23 +----------
 .../tests/gtest_full_sorting_join.cpp         | 36 +++++++++-------
 4 files changed, 52 insertions(+), 53 deletions(-)

diff --git a/docs/en/sql-reference/statements/select/join.md b/docs/en/sql-reference/statements/select/join.md
index 34c6016235a..96d9d26977d 100644
--- a/docs/en/sql-reference/statements/select/join.md
+++ b/docs/en/sql-reference/statements/select/join.md
@@ -297,7 +297,7 @@ Algorithm requires the special column in tables. This column:
 
 - Must contain an ordered sequence.
 - Can be one of the following types: [Int, UInt](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md), [Date](../../../sql-reference/data-types/date.md), [DateTime](../../../sql-reference/data-types/datetime.md), [Decimal](../../../sql-reference/data-types/decimal.md).
-- Can’t be the only column in the `JOIN` clause.
+- For `hash` join algorithm it can’t be the only column in the `JOIN` clause.
 
 Syntax `ASOF JOIN ... ON`:
 
@@ -337,7 +337,8 @@ For example, consider the following tables:
 `ASOF JOIN` can take the timestamp of a user event from `table_1` and find an event in `table_2` where the timestamp is closest to the timestamp of the event from `table_1` corresponding to the closest match condition. Equal timestamp values are the closest if available. Here, the `user_id` column can be used for joining on equality and the `ev_time` column can be used for joining on the closest match. In our example, `event_1_1` can be joined with `event_2_1` and `event_1_2` can be joined with `event_2_3`, but `event_2_2` can’t be joined.
 
 :::note
-`ASOF` join is **not** supported in the [Join](../../../engines/table-engines/special/join.md) table engine.
+`ASOF JOIN` is supported only by `hash` and `full_sorting_merge` join algorithms.
+It's **not** supported in the [Join](../../../engines/table-engines/special/join.md) table engine.
 :::
 
 ## PASTE JOIN Usage
diff --git a/src/Processors/Transforms/MergeJoinTransform.cpp b/src/Processors/Transforms/MergeJoinTransform.cpp
index ad564557c36..0d9eab248d8 100644
--- a/src/Processors/Transforms/MergeJoinTransform.cpp
+++ b/src/Processors/Transforms/MergeJoinTransform.cpp
@@ -329,6 +329,26 @@ void AsofJoinState::reset()
     value.clear();
 }
 
+FullMergeJoinCursor::FullMergeJoinCursor(const Block & sample_block_, const SortDescription & description_, bool is_asof)
+    : sample_block(sample_block_.cloneEmpty())
+    , desc(description_)
+{
+    if (desc.size() == 0)
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "Got empty sort description for FullMergeJoinCursor");
+
+    if (is_asof)
+    {
+        /// For ASOF join prefix of sort description is used for equality comparison
+        /// and the last column is used for inequality comparison and is handled separately
+
+        auto asof_column_description = desc.back();
+        desc.pop_back();
+
+        chassert(asof_column_description.direction == 1 && asof_column_description.nulls_direction == 1);
+        asof_column_position = sample_block.getPositionByName(asof_column_description.column_name);
+    }
+}
+
 const Chunk & FullMergeJoinCursor::getCurrent() const
 {
     return current_chunk;
@@ -686,7 +706,6 @@ std::optional<MergeJoinAlgorithm::Status> MergeJoinAlgorithm::handleAsofJoinStat
         return {};
 
     auto & left_cursor = *cursors[0];
-    size_t lpos = left_cursor->getRow();
     const auto & left_columns = left_cursor.getCurrent().getColumns();
 
     MutableColumns result_cols = getEmptyResultColumns();
@@ -695,7 +714,7 @@ std::optional<MergeJoinAlgorithm::Status> MergeJoinAlgorithm::handleAsofJoinStat
     {
         size_t i = 0;
         for (const auto & col : left_columns)
-            result_cols[i++]->insertFrom(*col, lpos);
+            result_cols[i++]->insertFrom(*col, left_cursor->getRow());
         for (const auto & col : asof_join_state.value.getColumns())
             result_cols[i++]->insertFrom(*col, asof_join_state.value_row);
         chassert(i == result_cols.size());
@@ -707,7 +726,7 @@ std::optional<MergeJoinAlgorithm::Status> MergeJoinAlgorithm::handleAsofJoinStat
         /// return row with default values at right side
         size_t i = 0;
         for (const auto & col : left_columns)
-            result_cols[i++]->insertFrom(*col, lpos);
+            result_cols[i++]->insertFrom(*col, left_cursor->getRow());
         for (; i < result_cols.size(); ++i)
             result_cols[i]->insertDefault();
         chassert(i == result_cols.size());
@@ -977,12 +996,10 @@ MergeJoinAlgorithm::Status MergeJoinAlgorithm::asofJoin()
         auto lpos = left_cursor->getRow();
         auto rpos = right_cursor->getRow();
         auto cmp = compareCursors(*left_cursor, *right_cursor);
-        // LOG_DEBUG(&Poco::Logger::get("XXXX"), "{}:{} ({}) <=> ({}) -> {}", __FILE__, __LINE__, left_cursor.dump(), right_cursor.dump(), cmp);
 
         if (cmp == 0)
         {
             auto asof_cmp = compareAsofCursors(left_cursor, right_cursor);
-            // LOG_DEBUG(&Poco::Logger::get("XXXX"), "{}:{} ({}) <=> ({}) -> asof {}", __FILE__, __LINE__, left_cursor.dump(), right_cursor.dump(), asof_cmp);
 
             if ((asof_inequality == ASOFJoinInequality::Less && asof_cmp <= -1)
              || (asof_inequality == ASOFJoinInequality::LessOrEquals && asof_cmp <= 0))
@@ -1010,7 +1027,6 @@ MergeJoinAlgorithm::Status MergeJoinAlgorithm::asofJoin()
             if ((asof_inequality == ASOFJoinInequality::Greater && asof_cmp >= 1)
              || (asof_inequality == ASOFJoinInequality::GreaterOrEquals && asof_cmp >= 0))
             {
-                // LOG_DEBUG(&Poco::Logger::get("XXXX"), "{}:{} ", __FILE__, __LINE__);
                 /// condition is satisfied, remember this row and move next to try to find better match
                 asof_join_state.set(right_cursor, rpos);
                 right_cursor->next();
@@ -1022,7 +1038,6 @@ MergeJoinAlgorithm::Status MergeJoinAlgorithm::asofJoin()
                 /// Asof condition is not satisfied anymore, use last matched row from right table
                 if (asof_join_state.hasMatch(left_cursor, asof_inequality))
                 {
-                    // LOG_DEBUG(&Poco::Logger::get("XXXX"), "{}:{} ", __FILE__, __LINE__);
                     size_t i = 0;
                     for (const auto & col : left_columns)
                         result_cols[i++]->insertFrom(*col, lpos);
@@ -1035,7 +1050,6 @@ MergeJoinAlgorithm::Status MergeJoinAlgorithm::asofJoin()
                     asof_join_state.reset();
                     if (isLeft(kind))
                     {
-                        // LOG_DEBUG(&Poco::Logger::get("XXXX"), "{}:{} ", __FILE__, __LINE__);
 
                         /// return row with default values at right side
                         size_t i = 0;
@@ -1047,7 +1061,6 @@ MergeJoinAlgorithm::Status MergeJoinAlgorithm::asofJoin()
                     }
                 }
                 left_cursor->next();
-                // LOG_DEBUG(&Poco::Logger::get("XXXX"), "{}:{} ", __FILE__, __LINE__);
                 continue;
             }
 
@@ -1055,10 +1068,8 @@ MergeJoinAlgorithm::Status MergeJoinAlgorithm::asofJoin()
         }
         else if (cmp < 0)
         {
-            // LOG_DEBUG(&Poco::Logger::get("XXXX"), "{}:{} ", __FILE__, __LINE__);
             if (asof_join_state.hasMatch(left_cursor, asof_inequality))
             {
-                // LOG_DEBUG(&Poco::Logger::get("XXXX"), "{}:{} ", __FILE__, __LINE__);
 
                 size_t i = 0;
                 for (const auto & col : left_columns)
@@ -1071,13 +1082,12 @@ MergeJoinAlgorithm::Status MergeJoinAlgorithm::asofJoin()
             }
             else
             {
-                // LOG_DEBUG(&Poco::Logger::get("XXXX"), "{}:{} ", __FILE__, __LINE__);
                 asof_join_state.reset();
             }
-            // LOG_DEBUG(&Poco::Logger::get("XXXX"), "{}:{} ", __FILE__, __LINE__);
 
             /// no matches for rows in left table, just pass them through
             size_t num = nextDistinct(*left_cursor);
+
             if (isLeft(kind) && num)
             {
                 /// return them with default values at right side
@@ -1091,7 +1101,6 @@ MergeJoinAlgorithm::Status MergeJoinAlgorithm::asofJoin()
         }
         else
         {
-            // LOG_DEBUG(&Poco::Logger::get("XXXX"), "{}:{} ", __FILE__, __LINE__);
 
             /// skip rows in right table until we find match for current row in left table
             nextDistinct(*right_cursor);
@@ -1106,6 +1115,7 @@ MergeJoinAlgorithm::Status MergeJoinAlgorithm::asofJoin()
 /// otherwise - vice versa
 Chunk MergeJoinAlgorithm::createBlockWithDefaults(size_t source_num, size_t start, size_t num_rows) const
 {
+
     ColumnRawPtrs cols;
     {
         const auto & columns_left = source_num == 0 ? cursors[0]->getCurrent().getColumns() : cursors[0]->sampleColumns();
@@ -1128,7 +1138,6 @@ Chunk MergeJoinAlgorithm::createBlockWithDefaults(size_t source_num, size_t star
             cols.push_back(col.get());
         }
     }
-
     Chunk result_chunk;
     copyColumnsResized(cols, start, num_rows, result_chunk);
     return result_chunk;
@@ -1144,6 +1153,7 @@ Chunk MergeJoinAlgorithm::createBlockWithDefaults(size_t source_num)
 
 IMergingAlgorithm::Status MergeJoinAlgorithm::merge()
 {
+
     if (!cursors[0]->cursor.isValid() && !cursors[0]->fullyCompleted())
         return Status(0);
 
@@ -1161,6 +1171,7 @@ IMergingAlgorithm::Status MergeJoinAlgorithm::merge()
 
     if (cursors[0]->fullyCompleted() || cursors[1]->fullyCompleted())
     {
+
         if (!cursors[0]->fullyCompleted() && isLeftOrFull(kind))
             return Status(createBlockWithDefaults(0));
 
diff --git a/src/Processors/Transforms/MergeJoinTransform.h b/src/Processors/Transforms/MergeJoinTransform.h
index dbdda0b166b..c8ba857781e 100644
--- a/src/Processors/Transforms/MergeJoinTransform.h
+++ b/src/Processors/Transforms/MergeJoinTransform.h
@@ -185,28 +185,7 @@ public:
 class FullMergeJoinCursor : boost::noncopyable
 {
 public:
-    FullMergeJoinCursor(
-        const Block & sample_block_,
-        const SortDescription & description_,
-        bool is_asof = false)
-        : sample_block(sample_block_.cloneEmpty())
-        , desc(description_)
-    {
-        if (desc.size() == 0)
-            throw Exception(ErrorCodes::LOGICAL_ERROR, "Got empty sort description for FullMergeJoinCursor");
-
-        if (is_asof)
-        {
-            /// For ASOF join prefix of sort description is used for equality comparison
-            /// and the last column is used for inequality comparison and is handled separately
-
-            auto asof_column_description = desc.back();
-            desc.pop_back();
-
-            chassert(asof_column_description.direction == 1 && asof_column_description.nulls_direction == 1);
-            asof_column_position = sample_block.getPositionByName(asof_column_description.column_name);
-        }
-    }
+    FullMergeJoinCursor(const Block & sample_block_, const SortDescription & description_, bool is_asof = false);
 
     bool fullyCompleted() const;
     void setChunk(Chunk && chunk);
diff --git a/src/Processors/tests/gtest_full_sorting_join.cpp b/src/Processors/tests/gtest_full_sorting_join.cpp
index 2bd3357eff9..e3423aa0386 100644
--- a/src/Processors/tests/gtest_full_sorting_join.cpp
+++ b/src/Processors/tests/gtest_full_sorting_join.cpp
@@ -2,6 +2,7 @@
 
 #include <pcg_random.hpp>
 #include <random>
+#include <Poco/ConsoleChannel.h>
 
 #include <Columns/ColumnsNumber.h>
 #include <Common/getRandomASCIIString.h>
@@ -294,18 +295,25 @@ bool isStrict(ASOFJoinInequality inequality)
 
 }
 
-class FullSortingJoinRandomized : public ::testing::Test
+class FullSortingJoinTest : public ::testing::Test
 {
 public:
-    FullSortingJoinRandomized() = default;
+    FullSortingJoinTest() = default;
 
     void SetUp() override
     {
+        Poco::AutoPtr<Poco::ConsoleChannel> channel(new Poco::ConsoleChannel(std::cerr));
+        Poco::Logger::root().setChannel(channel);
+        if (const char * test_log_level = std::getenv("TEST_LOG_LEVEL")) // NOLINT(concurrency-mt-unsafe)
+            Poco::Logger::root().setLevel(test_log_level);
+        else
+            Poco::Logger::root().setLevel("none");
+
+
         UInt64 seed = randomSeed();
         if (const char * random_seed = std::getenv("TEST_RANDOM_SEED")) // NOLINT(concurrency-mt-unsafe)
             seed = std::stoull(random_seed);
-
-        std::cerr << "TEST_RANDOM_SEED=" << seed << std::endl;
+        std::cout << "TEST_RANDOM_SEED=" << seed << std::endl;
         rng = pcg64(seed);
     }
 
@@ -316,7 +324,7 @@ public:
     pcg64 rng;
 };
 
-TEST(FullSortingJoin, AllAnyOneKey)
+TEST_F(FullSortingJoinTest, AllAnyOneKey)
 try
 {
     {
@@ -392,7 +400,7 @@ catch (Exception & e)
 }
 
 
-TEST_F(FullSortingJoinRandomized, Any)
+TEST_F(FullSortingJoinTest, AnySimple)
 try
 {
     JoinKind kind = getRandomFrom(rng, {JoinKind::Inner, JoinKind::Left, JoinKind::Right});
@@ -478,7 +486,7 @@ catch (Exception & e)
     throw;
 }
 
-TEST(FullSortingJoin, Asof)
+TEST_F(FullSortingJoinTest, AsofSimple)
 try
 {
     SourceChunksBuilder left_source({
@@ -543,7 +551,7 @@ catch (Exception & e)
 }
 
 
-TEST_F(FullSortingJoinRandomized, AsofOnlyColumn)
+TEST_F(FullSortingJoinTest, AsofOnlyColumn)
 try
 {
     auto left_source = oneColumnSource({ {3}, {3, 3, 3}, {3, 5, 5, 6}, {9, 9}, {10, 20} });
@@ -587,7 +595,7 @@ catch (Exception & e)
     throw;
 }
 
-TEST_F(FullSortingJoinRandomized, AsofLessGeneratedTestData)
+TEST_F(FullSortingJoinTest, AsofLessGeneratedTestData)
 try
 {
     auto join_kind = getRandomFrom(rng, { JoinKind::Inner, JoinKind::Left });
@@ -676,7 +684,7 @@ catch (Exception & e)
     throw;
 }
 
-TEST_F(FullSortingJoinRandomized, AsofGreaterGeneratedTestData)
+TEST_F(FullSortingJoinTest, AsofGreaterGeneratedTestData)
 try
 {
     auto join_kind = getRandomFrom(rng, { JoinKind::Inner, JoinKind::Left });
@@ -708,13 +716,13 @@ try
     String k2 = "";
     UInt64 left_t = 0;
 
-    auto key_num_total = std::uniform_int_distribution<>(1, 100)(rng);
+    auto key_num_total = std::uniform_int_distribution<>(1, 1000)(rng);
     for (size_t key_num = 0; key_num < key_num_total; ++key_num)
     {
         generateNextKey(rng, k1, k2);
 
         /// generate some rows with smaller left_t to check that they are not matched
-        size_t num_left_rows = std::bernoulli_distribution(0.5)(rng) ? std::uniform_int_distribution<>(1, 10)(rng) : 0;
+        size_t num_left_rows = std::bernoulli_distribution(0.5)(rng) ? std::uniform_int_distribution<>(1, 100)(rng) : 0;
         for (size_t i = 0; i < num_left_rows; ++i)
         {
             left_t += std::uniform_int_distribution<>(1, 10)(rng);
@@ -737,8 +745,8 @@ try
         }
 
         /// next left_t should be greater than (or equals) right_t to match with previous rows
-        left_t = right_t + std::uniform_int_distribution<>(isStrict(asof_inequality) ? 1 : 0, 10)(rng);
-        size_t num_left_matches = std::uniform_int_distribution<>(1, 10)(rng);
+        left_t = right_t + std::uniform_int_distribution<>(isStrict(asof_inequality) ? 1 : 0, 100)(rng);
+        size_t num_left_matches = std::uniform_int_distribution<>(1, 100)(rng);
         for (size_t j = 0; j < num_left_matches; ++j)
         {
             left_t += std::uniform_int_distribution<>(0, 3)(rng);

From a8690947c705339113266c1ec8db05695a36e472 Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Thu, 28 Sep 2023 12:50:24 +0000
Subject: [PATCH 0057/1488] enable stateless tessts for full sorting asof join

---
 src/Interpreters/InterpreterSelectQuery.cpp   |  7 ++-
 src/Planner/PlannerJoinTree.cpp               |  6 ++-
 .../Transforms/MergeJoinTransform.cpp         |  2 -
 .../00927_asof_join_correct_bt.reference      | 15 +++++++
 .../00927_asof_join_correct_bt.sql            | 30 ++++++++-----
 .../00927_asof_join_long.reference            |  1 +
 .../0_stateless/00927_asof_join_long.sql      | 17 ++++++-
 .../00927_asof_join_noninclusive.reference    | 29 ++++++++++++
 .../00927_asof_join_noninclusive.sql          |  5 ++-
 .../00927_asof_join_other_types.reference     | 45 +++++++++++++++++++
 .../00927_asof_join_other_types.sh            | 27 -----------
 .../00927_asof_join_other_types.sql.j2        | 27 +++++++++++
 .../0_stateless/00927_asof_joins.reference    | 15 +++++++
 .../queries/0_stateless/00927_asof_joins.sql  |  8 +++-
 .../0_stateless/00976_asof_join_on.reference  | 38 ++++++++++++++++
 ..._join_on.sql => 00976_asof_join_on.sql.j2} |  8 ++++
 16 files changed, 231 insertions(+), 49 deletions(-)
 delete mode 100755 tests/queries/0_stateless/00927_asof_join_other_types.sh
 create mode 100755 tests/queries/0_stateless/00927_asof_join_other_types.sql.j2
 rename tests/queries/0_stateless/{00976_asof_join_on.sql => 00976_asof_join_on.sql.j2} (90%)

diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp
index b72399df2c1..a0b53ad42d1 100644
--- a/src/Interpreters/InterpreterSelectQuery.cpp
+++ b/src/Interpreters/InterpreterSelectQuery.cpp
@@ -1726,7 +1726,10 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional<P
                         const auto & join_clause = table_join.getOnlyClause();
 
                         auto join_kind = table_join.kind();
-                        bool kind_allows_filtering = isInner(join_kind) || isLeft(join_kind) || isRight(join_kind);
+                        auto join_strictness = table_join.strictness();
+
+                        bool join_type_allows_filtering = (join_strictness == JoinStrictness::All || join_strictness == JoinStrictness::Any)
+                                                       && (isInner(join_kind) || isLeft(join_kind) || isRight(join_kind));
 
                         auto has_non_const = [](const Block & block, const auto & keys)
                         {
@@ -1745,7 +1748,7 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional<P
                         bool has_non_const_keys = has_non_const(query_plan.getCurrentDataStream().header, join_clause.key_names_left)
                             && has_non_const(joined_plan->getCurrentDataStream().header, join_clause.key_names_right);
 
-                        if (settings.max_rows_in_set_to_optimize_join > 0 && kind_allows_filtering && has_non_const_keys)
+                        if (settings.max_rows_in_set_to_optimize_join > 0 && join_type_allows_filtering && has_non_const_keys)
                         {
                             auto * left_set = add_create_set(query_plan, join_clause.key_names_left, JoinTableSide::Left);
                             auto * right_set = add_create_set(*joined_plan, join_clause.key_names_right, JoinTableSide::Right);
diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp
index aa46f65d2d4..63acf194139 100644
--- a/src/Planner/PlannerJoinTree.cpp
+++ b/src/Planner/PlannerJoinTree.cpp
@@ -1484,7 +1484,9 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_
         {
             const auto & join_clause = table_join->getOnlyClause();
 
-            bool kind_allows_filtering = isInner(join_kind) || isLeft(join_kind) || isRight(join_kind);
+            bool join_type_allows_filtering = (join_strictness == JoinStrictness::All || join_strictness == JoinStrictness::Any)
+                                            && (isInner(join_kind) || isLeft(join_kind) || isRight(join_kind));
+
 
             auto has_non_const = [](const Block & block, const auto & keys)
             {
@@ -1504,7 +1506,7 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_
             bool has_non_const_keys = has_non_const(left_plan.getCurrentDataStream().header, join_clause.key_names_left)
                 && has_non_const(right_plan.getCurrentDataStream().header, join_clause.key_names_right);
 
-            if (settings.max_rows_in_set_to_optimize_join > 0 && kind_allows_filtering && has_non_const_keys)
+            if (settings.max_rows_in_set_to_optimize_join > 0 && join_type_allows_filtering && has_non_const_keys)
             {
                 auto * left_set = add_create_set(left_plan, join_clause.key_names_left, JoinTableSide::Left);
                 auto * right_set = add_create_set(right_plan, join_clause.key_names_right, JoinTableSide::Right);
diff --git a/src/Processors/Transforms/MergeJoinTransform.cpp b/src/Processors/Transforms/MergeJoinTransform.cpp
index 0d9eab248d8..df56ffc2871 100644
--- a/src/Processors/Transforms/MergeJoinTransform.cpp
+++ b/src/Processors/Transforms/MergeJoinTransform.cpp
@@ -456,8 +456,6 @@ MergeJoinAlgorithm::MergeJoinAlgorithm(
         size_t left_idx = input_headers[0].getPositionByName(left_key);
         size_t right_idx = input_headers[1].getPositionByName(right_key);
         left_to_right_key_remap[left_idx] = right_idx;
-        if (strictness == JoinStrictness::Asof)
-            throw Exception(ErrorCodes::NOT_IMPLEMENTED, "MergeJoinAlgorithm does not support ASOF joins USING");
     }
 
     const auto *smjPtr = typeid_cast<const FullSortingMergeJoin *>(table_join.get());
diff --git a/tests/queries/0_stateless/00927_asof_join_correct_bt.reference b/tests/queries/0_stateless/00927_asof_join_correct_bt.reference
index bb199d0159a..a398f9604fd 100644
--- a/tests/queries/0_stateless/00927_asof_join_correct_bt.reference
+++ b/tests/queries/0_stateless/00927_asof_join_correct_bt.reference
@@ -13,3 +13,18 @@
 1	103	3	2	102	1
 1	104	4	4	104	1
 1	105	5	4	104	1
+1	101	1	0	0	0
+1	102	2	2	102	1
+1	103	3	2	102	1
+1	104	4	4	104	1
+1	105	5	4	104	1
+1	101	1	0	0	0
+1	102	2	2	102	1
+1	103	3	2	102	1
+1	104	4	4	104	1
+1	105	5	4	104	1
+1	101	1	0	0	0
+1	102	2	2	102	1
+1	103	3	2	102	1
+1	104	4	4	104	1
+1	105	5	4	104	1
diff --git a/tests/queries/0_stateless/00927_asof_join_correct_bt.sql b/tests/queries/0_stateless/00927_asof_join_correct_bt.sql
index 281a81d51c0..761d6bacde6 100644
--- a/tests/queries/0_stateless/00927_asof_join_correct_bt.sql
+++ b/tests/queries/0_stateless/00927_asof_join_correct_bt.sql
@@ -4,20 +4,26 @@ DROP TABLE IF EXISTS B;
 CREATE TABLE A(k UInt32, t UInt32, a UInt64) ENGINE = MergeTree() ORDER BY (k, t);
 INSERT INTO A(k,t,a) VALUES (1,101,1),(1,102,2),(1,103,3),(1,104,4),(1,105,5);
 
-CREATE TABLE B(k UInt32, t UInt32, b UInt64) ENGINE = MergeTree() ORDER BY (k, t);
-INSERT INTO B(k,t,b) VALUES (1,102,2), (1,104,4);
-SELECT A.k, A.t, A.a, B.b, B.t, B.k FROM A ASOF LEFT JOIN B USING(k,t) ORDER BY (A.k, A.t);
-DROP TABLE B;
+CREATE TABLE B1(k UInt32, t UInt32, b UInt64) ENGINE = MergeTree() ORDER BY (k, t);
+INSERT INTO B1(k,t,b) VALUES (1,102,2), (1,104,4);
 
+CREATE TABLE B2(t UInt32, k UInt32, b UInt64) ENGINE = MergeTree() ORDER BY (k, t);
+INSERT INTO B2(k,t,b) VALUES (1,102,2), (1,104,4);
 
-CREATE TABLE B(t UInt32, k UInt32, b UInt64) ENGINE = MergeTree() ORDER BY (k, t);
-INSERT INTO B(k,t,b) VALUES (1,102,2), (1,104,4);
-SELECT A.k, A.t, A.a, B.b, B.t, B.k FROM A ASOF LEFT JOIN B USING(k,t) ORDER BY (A.k, A.t);
-DROP TABLE B;
+CREATE TABLE B3(k UInt32, b UInt64, t UInt32) ENGINE = MergeTree() ORDER BY (k, t);
+INSERT INTO B3(k,t,b) VALUES (1,102,2), (1,104,4);
 
-CREATE TABLE B(k UInt32, b UInt64, t UInt32) ENGINE = MergeTree() ORDER BY (k, t);
-INSERT INTO B(k,t,b) VALUES (1,102,2), (1,104,4);
-SELECT A.k, A.t, A.a, B.b, B.t, B.k FROM A ASOF LEFT JOIN B USING(k,t) ORDER BY (A.k, A.t);
-DROP TABLE B;
+SELECT A.k, A.t, A.a, B.b, B.t, B.k FROM A ASOF LEFT JOIN B1 B USING(k,t) ORDER BY (A.k, A.t);
+SELECT A.k, A.t, A.a, B.b, B.t, B.k FROM A ASOF LEFT JOIN B2 B USING(k,t) ORDER BY (A.k, A.t);
+SELECT A.k, A.t, A.a, B.b, B.t, B.k FROM A ASOF LEFT JOIN B3 B USING(k,t) ORDER BY (A.k, A.t);
+
+SET join_algorithm = 'full_sorting_merge';
+SELECT A.k, A.t, A.a, B.b, B.t, B.k FROM A ASOF LEFT JOIN B1 B USING(k,t) ORDER BY (A.k, A.t);
+SELECT A.k, A.t, A.a, B.b, B.t, B.k FROM A ASOF LEFT JOIN B2 B USING(k,t) ORDER BY (A.k, A.t);
+SELECT A.k, A.t, A.a, B.b, B.t, B.k FROM A ASOF LEFT JOIN B3 B USING(k,t) ORDER BY (A.k, A.t);
+
+DROP TABLE B1;
+DROP TABLE B2;
+DROP TABLE B3;
 
 DROP TABLE A;
diff --git a/tests/queries/0_stateless/00927_asof_join_long.reference b/tests/queries/0_stateless/00927_asof_join_long.reference
index d4f015c68e4..ec40d2bc463 100644
--- a/tests/queries/0_stateless/00927_asof_join_long.reference
+++ b/tests/queries/0_stateless/00927_asof_join_long.reference
@@ -1 +1,2 @@
 3000000
+3000000
diff --git a/tests/queries/0_stateless/00927_asof_join_long.sql b/tests/queries/0_stateless/00927_asof_join_long.sql
index c03a06d48d4..7a73875e93e 100644
--- a/tests/queries/0_stateless/00927_asof_join_long.sql
+++ b/tests/queries/0_stateless/00927_asof_join_long.sql
@@ -2,15 +2,28 @@
 
 DROP TABLE IF EXISTS tvs;
 
+-- to use different algorithms for in subquery
+SET allow_experimental_analyzer = 1;
+
 CREATE TABLE tvs(k UInt32, t UInt32, tv UInt64) ENGINE = Memory;
 INSERT INTO tvs(k,t,tv) SELECT k, t, t
 FROM (SELECT toUInt32(number) AS k FROM numbers(1000)) keys
-CROSS JOIN (SELECT toUInt32(number * 3) as t FROM numbers(10000)) tv_times;
+CROSS JOIN (SELECT toUInt32(number * 3) as t FROM numbers(10000)) tv_times
+SETTINGS join_algorithm = 'hash';
 
 SELECT SUM(trades.price - tvs.tv) FROM
 (SELECT k, t, t as price
     FROM (SELECT toUInt32(number) AS k FROM numbers(1000)) keys
-    CROSS JOIN (SELECT toUInt32(number * 10) AS t FROM numbers(3000)) trade_times) trades
+    CROSS JOIN (SELECT toUInt32(number * 10) AS t FROM numbers(3000)) trade_times
+    SETTINGS join_algorithm = 'hash') trades
 ASOF LEFT JOIN tvs USING(k,t);
 
+SELECT SUM(trades.price - tvs.tv) FROM
+(SELECT k, t, t as price
+    FROM (SELECT toUInt32(number) AS k FROM numbers(1000)) keys
+    CROSS JOIN (SELECT toUInt32(number * 10) AS t FROM numbers(3000)) trade_times
+    SETTINGS join_algorithm = 'hash') trades
+ASOF LEFT JOIN tvs USING(k,t)
+SETTINGS join_algorithm = 'full_sorting_merge';
+
 DROP TABLE tvs;
diff --git a/tests/queries/0_stateless/00927_asof_join_noninclusive.reference b/tests/queries/0_stateless/00927_asof_join_noninclusive.reference
index fe2844a2a43..d856372fb4a 100644
--- a/tests/queries/0_stateless/00927_asof_join_noninclusive.reference
+++ b/tests/queries/0_stateless/00927_asof_join_noninclusive.reference
@@ -27,3 +27,32 @@
 2	1970-01-01 00:00:03	3	3	1970-01-01 00:00:03	2
 2	1970-01-01 00:00:04	4	3	1970-01-01 00:00:03	2
 2	1970-01-01 00:00:05	5	3	1970-01-01 00:00:03	2
+1	1970-01-01 00:00:01	1	0	1970-01-01 00:00:00	0
+1	1970-01-01 00:00:02	2	2	1970-01-01 00:00:02	1
+1	1970-01-01 00:00:03	3	2	1970-01-01 00:00:02	1
+1	1970-01-01 00:00:04	4	4	1970-01-01 00:00:04	1
+1	1970-01-01 00:00:05	5	4	1970-01-01 00:00:04	1
+2	1970-01-01 00:00:01	1	0	1970-01-01 00:00:00	0
+2	1970-01-01 00:00:02	2	0	1970-01-01 00:00:00	0
+2	1970-01-01 00:00:03	3	3	1970-01-01 00:00:03	2
+2	1970-01-01 00:00:04	4	3	1970-01-01 00:00:03	2
+2	1970-01-01 00:00:05	5	3	1970-01-01 00:00:03	2
+3	1970-01-01 00:00:01	1	0	1970-01-01 00:00:00	0
+3	1970-01-01 00:00:02	2	0	1970-01-01 00:00:00	0
+3	1970-01-01 00:00:03	3	0	1970-01-01 00:00:00	0
+3	1970-01-01 00:00:04	4	0	1970-01-01 00:00:00	0
+3	1970-01-01 00:00:05	5	0	1970-01-01 00:00:00	0
+1	1970-01-01 00:00:02	2	2	1970-01-01 00:00:02	1
+1	1970-01-01 00:00:03	3	2	1970-01-01 00:00:02	1
+1	1970-01-01 00:00:04	4	4	1970-01-01 00:00:04	1
+1	1970-01-01 00:00:05	5	4	1970-01-01 00:00:04	1
+2	1970-01-01 00:00:03	3	3	1970-01-01 00:00:03	2
+2	1970-01-01 00:00:04	4	3	1970-01-01 00:00:03	2
+2	1970-01-01 00:00:05	5	3	1970-01-01 00:00:03	2
+1	1970-01-01 00:00:02	2	2	1970-01-01 00:00:02	1
+1	1970-01-01 00:00:03	3	2	1970-01-01 00:00:02	1
+1	1970-01-01 00:00:04	4	4	1970-01-01 00:00:04	1
+1	1970-01-01 00:00:05	5	4	1970-01-01 00:00:04	1
+2	1970-01-01 00:00:03	3	3	1970-01-01 00:00:03	2
+2	1970-01-01 00:00:04	4	3	1970-01-01 00:00:03	2
+2	1970-01-01 00:00:05	5	3	1970-01-01 00:00:03	2
diff --git a/tests/queries/0_stateless/00927_asof_join_noninclusive.sql b/tests/queries/0_stateless/00927_asof_join_noninclusive.sql
index 5f15f3b593d..3cc99df4462 100644
--- a/tests/queries/0_stateless/00927_asof_join_noninclusive.sql
+++ b/tests/queries/0_stateless/00927_asof_join_noninclusive.sql
@@ -11,9 +11,12 @@ INSERT INTO B(k,t,b) VALUES (1,2,2),(1,4,4);
 INSERT INTO B(k,t,b) VALUES (2,3,3);
 
 SELECT A.k, toString(A.t, 'UTC'), A.a, B.b, toString(B.t, 'UTC'), B.k FROM A ASOF LEFT JOIN B USING(k,t) ORDER BY (A.k, A.t);
-
 SELECT A.k, toString(A.t, 'UTC'), A.a, B.b, toString(B.t, 'UTC'), B.k FROM A ASOF INNER JOIN B ON A.k == B.k AND A.t >= B.t ORDER BY (A.k, A.t);
+SELECT A.k, toString(A.t, 'UTC'), A.a, B.b, toString(B.t, 'UTC'), B.k FROM A ASOF JOIN B USING(k,t) ORDER BY (A.k, A.t);
 
+SET join_algorithm = 'full_sorting_merge';
+SELECT A.k, toString(A.t, 'UTC'), A.a, B.b, toString(B.t, 'UTC'), B.k FROM A ASOF LEFT JOIN B USING(k,t) ORDER BY (A.k, A.t);
+SELECT A.k, toString(A.t, 'UTC'), A.a, B.b, toString(B.t, 'UTC'), B.k FROM A ASOF INNER JOIN B ON A.k == B.k AND A.t >= B.t ORDER BY (A.k, A.t);
 SELECT A.k, toString(A.t, 'UTC'), A.a, B.b, toString(B.t, 'UTC'), B.k FROM A ASOF JOIN B USING(k,t) ORDER BY (A.k, A.t);
 
 DROP TABLE A;
diff --git a/tests/queries/0_stateless/00927_asof_join_other_types.reference b/tests/queries/0_stateless/00927_asof_join_other_types.reference
index 80c85ec1ae3..ddbc24ff925 100644
--- a/tests/queries/0_stateless/00927_asof_join_other_types.reference
+++ b/tests/queries/0_stateless/00927_asof_join_other_types.reference
@@ -1,27 +1,72 @@
+-
 2	1	1	0
 2	3	3	3
 2	5	5	3
+-
 2	1	1	0
 2	3	3	3
 2	5	5	3
+-
 2	1	1	0
 2	3	3	3
 2	5	5	3
+-
 2	1	1	0
 2	3	3	3
 2	5	5	3
+-
+2	1	1	0
+2	3	3	3
+2	5	5	3
+-
+2	1	1	0
+2	3	3	3
+2	5	5	3
+-
+2	1	1	0
+2	3	3	3
+2	5	5	3
+-
+2	1	1	0
+2	3	3	3
+2	5	5	3
+-
 2	1970-01-01 02:00:01	1	0
 2	1970-01-01 02:00:03	3	3
 2	1970-01-01 02:00:05	5	3
+-
+2	1970-01-01 02:00:01	1	0
+2	1970-01-01 02:00:03	3	3
+2	1970-01-01 02:00:05	5	3
+-
 2	1	1	0
 2	3	3	3
 2	5	5	3
+-
 2	1	1	0
 2	3	3	3
 2	5	5	3
+-
 2	1	1	0
 2	3	3	3
 2	5	5	3
+-
+2	1	1	0
+2	3	3	3
+2	5	5	3
+-
+2	1	1	0
+2	3	3	3
+2	5	5	3
+-
+2	1	1	0
+2	3	3	3
+2	5	5	3
+-
+2	1970-01-01 02:00:00.001	1	0
+2	1970-01-01 02:00:00.003	3	3
+2	1970-01-01 02:00:00.005	5	3
+-
 2	1970-01-01 02:00:00.001	1	0
 2	1970-01-01 02:00:00.003	3	3
 2	1970-01-01 02:00:00.005	5	3
diff --git a/tests/queries/0_stateless/00927_asof_join_other_types.sh b/tests/queries/0_stateless/00927_asof_join_other_types.sh
deleted file mode 100755
index 10173a3e43f..00000000000
--- a/tests/queries/0_stateless/00927_asof_join_other_types.sh
+++ /dev/null
@@ -1,27 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# shellcheck source=../shell_config.sh
-. "$CURDIR"/../shell_config.sh
-
-for typename in "UInt32" "UInt64" "Float64" "Float32" "DateTime('Asia/Istanbul')" "Decimal32(5)" "Decimal64(5)" "Decimal128(5)" "DateTime64(3, 'Asia/Istanbul')"
-do
-    $CLICKHOUSE_CLIENT -mn <<EOF
-DROP TABLE IF EXISTS A;
-DROP TABLE IF EXISTS B;
-
-CREATE TABLE A(k UInt32, t ${typename}, a Float64) ENGINE = MergeTree() ORDER BY (k, t);
-INSERT INTO A(k,t,a) VALUES (2,1,1),(2,3,3),(2,5,5);
-
-CREATE TABLE B(k UInt32, t ${typename}, b Float64) ENGINE = MergeTree() ORDER BY (k, t);
-INSERT INTO B(k,t,b) VALUES (2,3,3);
-
-SELECT k, t, a, b FROM A ASOF LEFT JOIN B USING(k,t) ORDER BY (k,t);
-
-DROP TABLE A;
-DROP TABLE B;
-EOF
-
-done
diff --git a/tests/queries/0_stateless/00927_asof_join_other_types.sql.j2 b/tests/queries/0_stateless/00927_asof_join_other_types.sql.j2
new file mode 100755
index 00000000000..7b50344d21c
--- /dev/null
+++ b/tests/queries/0_stateless/00927_asof_join_other_types.sql.j2
@@ -0,0 +1,27 @@
+
+{% for typename in ["UInt32", "UInt64", "Float64", "Float32", "DateTime('Asia/Istanbul')", "Decimal32(5)", "Decimal64(5)", "Decimal128(5)", "DateTime64(3, 'Asia/Istanbul')"] -%}
+
+
+DROP TABLE IF EXISTS A;
+DROP TABLE IF EXISTS B;
+
+CREATE TABLE A(k UInt32, t {{ typename }}, a Float64) ENGINE = MergeTree() ORDER BY (k, t);
+INSERT INTO A(k,t,a) VALUES (2,1,1),(2,3,3),(2,5,5);
+
+CREATE TABLE B(k UInt32, t {{ typename }}, b Float64) ENGINE = MergeTree() ORDER BY (k, t);
+INSERT INTO B(k,t,b) VALUES (2,3,3);
+
+SELECT '-';
+
+SELECT k, t, a, b FROM A ASOF LEFT JOIN B USING(k,t) ORDER BY (k,t)
+SETTINGS join_algorithm = 'full_sorting_merge';
+
+SELECT '-';
+
+SELECT k, t, a, b FROM A ASOF LEFT JOIN B USING(k,t) ORDER BY (k,t)
+SETTINGS join_algorithm = 'hash';
+
+DROP TABLE A;
+DROP TABLE B;
+
+{% endfor %}
diff --git a/tests/queries/0_stateless/00927_asof_joins.reference b/tests/queries/0_stateless/00927_asof_joins.reference
index 35ac9e63f42..16a1c5cff65 100644
--- a/tests/queries/0_stateless/00927_asof_joins.reference
+++ b/tests/queries/0_stateless/00927_asof_joins.reference
@@ -12,3 +12,18 @@
 2	1970-01-01 00:00:15	5	6.5	6
 2	1970-01-01 00:00:16	5	5.6	6
 2	1970-01-01 00:00:20	17	8.5	18
+-
+1	1970-01-01 00:00:05	1	1.5	2
+1	1970-01-01 00:00:06	1	1.51	2
+1	1970-01-01 00:00:10	11	11.5	12
+1	1970-01-01 00:00:11	11	11.51	12
+1	1970-01-01 00:00:15	5	5.5	6
+1	1970-01-01 00:00:16	5	5.6	6
+1	1970-01-01 00:00:20	7	7.5	8
+2	1970-01-01 00:00:05	11	2.5	12
+2	1970-01-01 00:00:06	11	2.51	12
+2	1970-01-01 00:00:10	21	12.5	22
+2	1970-01-01 00:00:11	21	12.51	22
+2	1970-01-01 00:00:15	5	6.5	6
+2	1970-01-01 00:00:16	5	5.6	6
+2	1970-01-01 00:00:20	17	8.5	18
diff --git a/tests/queries/0_stateless/00927_asof_joins.sql b/tests/queries/0_stateless/00927_asof_joins.sql
index 2dd6ca07b15..2186e88e00e 100644
--- a/tests/queries/0_stateless/00927_asof_joins.sql
+++ b/tests/queries/0_stateless/00927_asof_joins.sql
@@ -9,7 +9,13 @@ CREATE TABLE tv(key UInt32, t DateTime, tv Float64) ENGINE = MergeTree() ORDER B
 INSERT INTO tv(key,t,tv) VALUES (1,5,1.5),(1,6,1.51),(1,10,11.5),(1,11,11.51),(1,15,5.5),(1,16,5.6),(1,20,7.5);
 INSERT INTO tv(key,t,tv) VALUES (2,5,2.5),(2,6,2.51),(2,10,12.5),(2,11,12.51),(2,15,6.5),(2,16,5.6),(2,20,8.5);
 
-SELECT tv.key, toString(tv.t, 'UTC'), md.bid, tv.tv, md.ask FROM tv ASOF LEFT JOIN md USING(key,t) ORDER BY (tv.key, tv.t);
+SELECT tv.key, toString(tv.t, 'UTC'), md.bid, tv.tv, md.ask FROM tv ASOF LEFT JOIN md USING(key,t) ORDER BY (tv.key, tv.t)
+;
+
+SELECT '-';
+
+SELECT tv.key, toString(tv.t, 'UTC'), md.bid, tv.tv, md.ask FROM tv ASOF LEFT JOIN md USING(key,t) ORDER BY (tv.key, tv.t)
+SETTINGS join_algorithm = 'full_sorting_merge';
 
 DROP TABLE md;
 DROP TABLE tv;
diff --git a/tests/queries/0_stateless/00976_asof_join_on.reference b/tests/queries/0_stateless/00976_asof_join_on.reference
index 433d896426c..e13745bed9d 100644
--- a/tests/queries/0_stateless/00976_asof_join_on.reference
+++ b/tests/queries/0_stateless/00976_asof_join_on.reference
@@ -1,3 +1,41 @@
+- default -
+1	1	0	0
+1	2	1	2
+1	3	1	2
+2	1	0	0
+2	2	0	0
+2	3	2	3
+3	1	0	0
+3	2	0	0
+3	3	0	0
+9
+1	2	1	2
+1	3	1	2
+2	3	2	3
+-
+1	1	1	2
+1	2	1	2
+1	3	1	4
+2	1	2	3
+2	2	2	3
+2	3	2	3
+-
+1	1	1	2
+1	2	1	2
+1	3	1	4
+2	1	2	3
+2	2	2	3
+2	3	2	3
+-
+1	3	1	2
+-
+1	1	1	2
+1	2	1	4
+1	3	1	4
+2	1	2	3
+2	2	2	3
+1	2	1	2
+- full_sorting_merge -
 1	1	0	0
 1	2	1	2
 1	3	1	2
diff --git a/tests/queries/0_stateless/00976_asof_join_on.sql b/tests/queries/0_stateless/00976_asof_join_on.sql.j2
similarity index 90%
rename from tests/queries/0_stateless/00976_asof_join_on.sql
rename to tests/queries/0_stateless/00976_asof_join_on.sql.j2
index bd492897be9..aecf472a36c 100644
--- a/tests/queries/0_stateless/00976_asof_join_on.sql
+++ b/tests/queries/0_stateless/00976_asof_join_on.sql.j2
@@ -7,6 +7,12 @@ CREATE TABLE B(b UInt32, t UInt32) ENGINE = Memory;
 INSERT INTO A (a,t) VALUES (1,1),(1,2),(1,3), (2,1),(2,2),(2,3), (3,1),(3,2),(3,3);
 INSERT INTO B (b,t) VALUES (1,2),(1,4),(2,3);
 
+{% for join_algorithm in ['default', 'full_sorting_merge'] -%}
+
+SET join_algorithm = '{{ join_algorithm }}';
+
+SELECT '- {{ join_algorithm }} -';
+
 SELECT A.a, A.t, B.b, B.t FROM A ASOF LEFT JOIN B ON A.a == B.b AND A.t >= B.t ORDER BY (A.a, A.t);
 SELECT count() FROM A ASOF LEFT JOIN B ON A.a == B.b AND B.t <= A.t;
 SELECT A.a, A.t, B.b, B.t FROM A ASOF INNER JOIN B ON B.t <= A.t AND A.a == B.b ORDER BY (A.a, A.t);
@@ -28,5 +34,7 @@ ASOF INNER JOIN (SELECT * FROM B UNION ALL SELECT 1, 3) AS B ON B.t <= A.t AND A
 WHERE B.t != 3 ORDER BY (A.a, A.t)
 ;
 
+{% endfor %}
+
 DROP TABLE A;
 DROP TABLE B;

From 256ad60115ddd1eb8c8a5597478e057926fe30f6 Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Thu, 28 Sep 2023 12:52:20 +0000
Subject: [PATCH 0058/1488] fix style, clang tidy

---
 src/Planner/PlannerJoinTree.cpp               |  8 +--
 .../Transforms/MergeJoinTransform.cpp         | 59 +++++++++-------
 .../Transforms/MergeJoinTransform.h           |  2 +-
 .../tests/gtest_full_sorting_join.cpp         | 27 +-------
 .../00927_asof_join_other_types.sql.j2        |  0
 .../01116_asof_join_dolbyzerr.reference       |  3 +
 .../0_stateless/01116_asof_join_dolbyzerr.sql | 14 ++++
 .../02240_asof_join_biginteger.reference      |  4 ++
 .../02240_asof_join_biginteger.sql            |  8 +++
 ...n_in_left_table_clause_asof_join.reference |  1 +
 ...unction_in_left_table_clause_asof_join.sql | 12 ++++
 .../03143_asof_join_ddb_long.reference        |  2 +
 .../0_stateless/03143_asof_join_ddb_long.sql  | 48 +++++++++++++
 .../03144_asof_join_ddb_doubles.reference     | 58 ++++++++++++++++
 .../03144_asof_join_ddb_doubles.sql           | 64 +++++++++++++++++
 ...03145_asof_join_ddb_inequalities.reference | 68 +++++++++++++++++++
 .../03145_asof_join_ddb_inequalities.sql      | 63 +++++++++++++++++
 .../03146_asof_join_ddb_merge_long.reference  |  2 +
 .../03146_asof_join_ddb_merge_long.sql.j2     | 37 ++++++++++
 19 files changed, 424 insertions(+), 56 deletions(-)
 mode change 100755 => 100644 tests/queries/0_stateless/00927_asof_join_other_types.sql.j2
 create mode 100644 tests/queries/0_stateless/03143_asof_join_ddb_long.reference
 create mode 100644 tests/queries/0_stateless/03143_asof_join_ddb_long.sql
 create mode 100644 tests/queries/0_stateless/03144_asof_join_ddb_doubles.reference
 create mode 100644 tests/queries/0_stateless/03144_asof_join_ddb_doubles.sql
 create mode 100644 tests/queries/0_stateless/03145_asof_join_ddb_inequalities.reference
 create mode 100644 tests/queries/0_stateless/03145_asof_join_ddb_inequalities.sql
 create mode 100644 tests/queries/0_stateless/03146_asof_join_ddb_merge_long.reference
 create mode 100644 tests/queries/0_stateless/03146_asof_join_ddb_merge_long.sql.j2

diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp
index 63acf194139..a9d9e11f458 100644
--- a/src/Planner/PlannerJoinTree.cpp
+++ b/src/Planner/PlannerJoinTree.cpp
@@ -75,7 +75,6 @@ namespace ErrorCodes
     extern const int INVALID_JOIN_ON_EXPRESSION;
     extern const int LOGICAL_ERROR;
     extern const int NOT_IMPLEMENTED;
-    extern const int SYNTAX_ERROR;
     extern const int ACCESS_DENIED;
     extern const int PARAMETER_OUT_OF_BOUND;
     extern const int TOO_MANY_COLUMNS;
@@ -1357,12 +1356,7 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_
             {
                 if (!join_clause.hasASOF())
                     throw Exception(ErrorCodes::INVALID_JOIN_ON_EXPRESSION,
-                        "JOIN {} no inequality in ASOF JOIN ON section.",
-                        join_node.formatASTForErrorMessage());
-
-                if (table_join_clause.key_names_left.size() <= 1)
-                    throw Exception(ErrorCodes::SYNTAX_ERROR,
-                        "JOIN {} ASOF join needs at least one equi-join column",
+                        "JOIN {} no inequality in ASOF JOIN ON section",
                         join_node.formatASTForErrorMessage());
             }
 
diff --git a/src/Processors/Transforms/MergeJoinTransform.cpp b/src/Processors/Transforms/MergeJoinTransform.cpp
index df56ffc2871..26e1ebb0b60 100644
--- a/src/Processors/Transforms/MergeJoinTransform.cpp
+++ b/src/Processors/Transforms/MergeJoinTransform.cpp
@@ -43,6 +43,13 @@ FullMergeJoinCursorPtr createCursor(const Block & block, const Names & columns,
     return std::make_unique<FullMergeJoinCursor>(materializeBlock(block), desc, strictness == JoinStrictness::Asof);
 }
 
+bool isNullAt(const IColumn & column, size_t row)
+{
+    if (const auto * nullable = checkAndGetColumn<ColumnNullable>(column))
+        return nullable->isNullAt(row);
+    return false;
+}
+
 template <bool has_left_nulls, bool has_right_nulls>
 int nullableCompareAt(const IColumn & left_column, const IColumn & right_column, size_t lhs_pos, size_t rhs_pos, int null_direction_hint)
 {
@@ -54,7 +61,7 @@ int nullableCompareAt(const IColumn & left_column, const IColumn & right_column,
         if (left_nullable && right_nullable)
         {
             int res = left_nullable->compareAt(lhs_pos, rhs_pos, right_column, null_direction_hint);
-            if (res)
+            if (res != 0)
                 return res;
 
             /// NULL != NULL case
@@ -108,9 +115,9 @@ int ALWAYS_INLINE compareCursors(const SortCursorImpl & lhs, const SortCursorImp
     return compareCursors(lhs, lhs.getRow(), rhs, rhs.getRow(), lhs.sort_columns_size, null_direction_hint);
 }
 
-int compareAsofCursors(const FullMergeJoinCursor & lhs, const FullMergeJoinCursor & rhs)
+int compareAsofCursors(const FullMergeJoinCursor & lhs, const FullMergeJoinCursor & rhs, int null_direction_hint)
 {
-    return nullableCompareAt<false, false>(*lhs.getAsofColumn(), *rhs.getAsofColumn(), lhs->getRow(), rhs->getRow());
+    return nullableCompareAt<true, true>(*lhs.getAsofColumn(), *rhs.getAsofColumn(), lhs->getRow(), rhs->getRow(), null_direction_hint);
 }
 
 bool ALWAYS_INLINE totallyLess(SortCursorImpl & lhs, SortCursorImpl & rhs, int null_direction_hint)
@@ -235,12 +242,6 @@ void inline addRange(PaddedPODArray<UInt64> & values, UInt64 start, UInt64 end)
         values.push_back(i);
 }
 
-void inline addMany(PaddedPODArray<UInt64> & left_or_right_map, size_t idx, size_t num)
-{
-    for (size_t i = 0; i < num; ++i)
-        left_or_right_map.push_back(idx);
-}
-
 void inline addMany(PaddedPODArray<UInt64> & values, UInt64 value, size_t num)
 {
     values.resize_fill(values.size() + num, value);
@@ -257,7 +258,7 @@ JoinKeyRow::JoinKeyRow(const FullMergeJoinCursor & cursor, size_t pos)
         new_col->insertFrom(*col, pos);
         row.push_back(std::move(new_col));
     }
-    if (auto asof_column = cursor.getAsofColumn())
+    if (const auto * asof_column = cursor.getAsofColumn())
     {
         auto new_col = asof_column->cloneEmpty();
         new_col->insertFrom(*asof_column, pos);
@@ -275,10 +276,10 @@ bool JoinKeyRow::equals(const FullMergeJoinCursor & cursor) const
     if (row.empty())
         return false;
 
-    assert(this->row.size() == cursor->sort_columns_size);
     for (size_t i = 0; i < cursor->sort_columns_size; ++i)
     {
-        int cmp = this->row[i]->compareAt(0, cursor->getRow(), *(cursor->sort_columns[i]), cursor->desc[i].nulls_direction);
+        // int cmp = this->row[i]->compareAt(0, cursor->getRow(), *(cursor->sort_columns[i]), cursor->desc[i].nulls_direction);
+        int cmp = nullableCompareAt<true, true>(*this->row[i], *cursor->sort_columns[i], 0, cursor->getRow(), cursor->desc[i].nulls_direction);
         if (cmp != 0)
             return false;
     }
@@ -287,9 +288,16 @@ bool JoinKeyRow::equals(const FullMergeJoinCursor & cursor) const
 
 bool JoinKeyRow::asofMatch(const FullMergeJoinCursor & cursor, ASOFJoinInequality asof_inequality) const
 {
+    chassert(this->row.size() == cursor->sort_columns_size + 1);
     if (!equals(cursor))
         return false;
-    int cmp = cursor.getAsofColumn()->compareAt(cursor->getRow(), 0, *row.back(), 1);
+
+    const auto & asof_row = row.back();
+    if (isNullAt(*asof_row, 0) || isNullAt(*cursor.getAsofColumn(), cursor->getRow()))
+        return false;
+
+    int cmp = cursor.getAsofColumn()->compareAt(cursor->getRow(), 0, *asof_row, 1);
+
     return (asof_inequality == ASOFJoinInequality::Less && cmp < 0)
         || (asof_inequality == ASOFJoinInequality::LessOrEquals && cmp <= 0)
         || (asof_inequality == ASOFJoinInequality::Greater && cmp > 0)
@@ -333,7 +341,7 @@ FullMergeJoinCursor::FullMergeJoinCursor(const Block & sample_block_, const Sort
     : sample_block(sample_block_.cloneEmpty())
     , desc(description_)
 {
-    if (desc.size() == 0)
+    if (desc.empty())
         throw Exception(ErrorCodes::LOGICAL_ERROR, "Got empty sort description for FullMergeJoinCursor");
 
     if (is_asof)
@@ -393,7 +401,7 @@ String FullMergeJoinCursor::dump() const
             row_dump.push_back(val.dump());
         }
 
-        if (auto * asof_column = getAsofColumn())
+        if (const auto * asof_column = getAsofColumn())
         {
             asof_column->get(cursor.getRow(), val);
             row_dump.push_back(val.dump());
@@ -436,9 +444,10 @@ MergeJoinAlgorithm::MergeJoinAlgorithm(
         throw Exception(ErrorCodes::NOT_IMPLEMENTED, "MergeJoinAlgorithm does not support ON filter conditions");
 
     cursors = {
-        createCursor(input_headers[0], on_clause_.key_names_left,  strictness),
-        createCursor(input_headers[1], on_clause_.key_names_right,  strictness),
+        createCursor(input_headers[0], on_clause_.key_names_left, strictness),
+        createCursor(input_headers[1], on_clause_.key_names_right, strictness),
     };
+}
 
 MergeJoinAlgorithm::MergeJoinAlgorithm(
     JoinPtr join_ptr,
@@ -458,7 +467,7 @@ MergeJoinAlgorithm::MergeJoinAlgorithm(
         left_to_right_key_remap[left_idx] = right_idx;
     }
 
-    const auto *smjPtr = typeid_cast<const FullSortingMergeJoin *>(table_join.get());
+    const auto *smjPtr = typeid_cast<const FullSortingMergeJoin *>(join_ptr.get());
     if (smjPtr)
     {
         null_direction_hint = smjPtr->getNullDirection();
@@ -993,11 +1002,18 @@ MergeJoinAlgorithm::Status MergeJoinAlgorithm::asofJoin()
     {
         auto lpos = left_cursor->getRow();
         auto rpos = right_cursor->getRow();
-        auto cmp = compareCursors(*left_cursor, *right_cursor);
+        auto cmp = compareCursors(*left_cursor, *right_cursor, null_direction_hint);
+        if (cmp == 0)
+        {
+            if (isNullAt(*left_cursor.getAsofColumn(), lpos))
+                cmp = -1;
+            if (isNullAt(*right_cursor.getAsofColumn(), rpos))
+                cmp = 1;
+        }
 
         if (cmp == 0)
         {
-            auto asof_cmp = compareAsofCursors(left_cursor, right_cursor);
+            auto asof_cmp = compareAsofCursors(left_cursor, right_cursor, null_direction_hint);
 
             if ((asof_inequality == ASOFJoinInequality::Less && asof_cmp <= -1)
              || (asof_inequality == ASOFJoinInequality::LessOrEquals && asof_cmp <= 0))
@@ -1048,7 +1064,6 @@ MergeJoinAlgorithm::Status MergeJoinAlgorithm::asofJoin()
                     asof_join_state.reset();
                     if (isLeft(kind))
                     {
-
                         /// return row with default values at right side
                         size_t i = 0;
                         for (const auto & col : left_columns)
@@ -1068,7 +1083,6 @@ MergeJoinAlgorithm::Status MergeJoinAlgorithm::asofJoin()
         {
             if (asof_join_state.hasMatch(left_cursor, asof_inequality))
             {
-
                 size_t i = 0;
                 for (const auto & col : left_columns)
                     result_cols[i++]->insertFrom(*col, lpos);
@@ -1099,7 +1113,6 @@ MergeJoinAlgorithm::Status MergeJoinAlgorithm::asofJoin()
         }
         else
         {
-
             /// skip rows in right table until we find match for current row in left table
             nextDistinct(*right_cursor);
         }
diff --git a/src/Processors/Transforms/MergeJoinTransform.h b/src/Processors/Transforms/MergeJoinTransform.h
index c8ba857781e..15bf13381b8 100644
--- a/src/Processors/Transforms/MergeJoinTransform.h
+++ b/src/Processors/Transforms/MergeJoinTransform.h
@@ -167,7 +167,7 @@ public:
     void set(const FullMergeJoinCursor & rcursor, size_t rpos);
     void reset();
 
-    bool hasMatch(const FullMergeJoinCursor & cursor, ASOFJoinInequality asof_inequality)
+    bool hasMatch(const FullMergeJoinCursor & cursor, ASOFJoinInequality asof_inequality) const
     {
         if (value.empty())
             return false;
diff --git a/src/Processors/tests/gtest_full_sorting_join.cpp b/src/Processors/tests/gtest_full_sorting_join.cpp
index e3423aa0386..a3fda006eb8 100644
--- a/src/Processors/tests/gtest_full_sorting_join.cpp
+++ b/src/Processors/tests/gtest_full_sorting_join.cpp
@@ -3,6 +3,8 @@
 #include <pcg_random.hpp>
 #include <random>
 #include <Poco/ConsoleChannel.h>
+#include <Poco/Logger.h>
+#include <Poco/AutoPtr.h>
 
 #include <Columns/ColumnsNumber.h>
 #include <Common/getRandomASCIIString.h>
@@ -30,31 +32,6 @@ using namespace DB;
 namespace
 {
 
-[[ maybe_unused ]]
-String dumpBlockSource(std::shared_ptr<ISource> source, bool mono_block = false)
-{
-    WriteBufferFromOwnString buf;
-    {
-        Block header = source->getPort().getHeader();
-        QueryPipeline pipeline(source);
-        auto format = std::make_shared<PrettyCompactBlockOutputFormat>(buf, header, FormatSettings{}, mono_block);
-        pipeline.complete(std::move(format));
-
-        CompletedPipelineExecutor executor(pipeline);
-        executor.execute();
-    }
-    return buf.str();
-}
-
-[[ maybe_unused ]]
-String dumpBlock(const Block & block)
-{
-    Block header = block.cloneEmpty();
-    Chunk data(block.getColumns(), block.rows());
-    auto source = std::make_shared<SourceFromSingleChunk>(header, std::move(data));
-    return dumpBlockSource(std::move(source));
-}
-
 QueryPipeline buildJoinPipeline(
     std::shared_ptr<ISource> left_source,
     std::shared_ptr<ISource> right_source,
diff --git a/tests/queries/0_stateless/00927_asof_join_other_types.sql.j2 b/tests/queries/0_stateless/00927_asof_join_other_types.sql.j2
old mode 100755
new mode 100644
diff --git a/tests/queries/0_stateless/01116_asof_join_dolbyzerr.reference b/tests/queries/0_stateless/01116_asof_join_dolbyzerr.reference
index 1055a67ea5b..0aa1a85f19d 100644
--- a/tests/queries/0_stateless/01116_asof_join_dolbyzerr.reference
+++ b/tests/queries/0_stateless/01116_asof_join_dolbyzerr.reference
@@ -1,3 +1,6 @@
 v1	o1	['s2','s1']
 v1	o2	['s4']
 v2	o3	['s5','s3']
+v1	o1	['s2','s1']
+v1	o2	['s4']
+v2	o3	['s5','s3']
diff --git a/tests/queries/0_stateless/01116_asof_join_dolbyzerr.sql b/tests/queries/0_stateless/01116_asof_join_dolbyzerr.sql
index 8a94b6ddd24..652cb35cf2a 100644
--- a/tests/queries/0_stateless/01116_asof_join_dolbyzerr.sql
+++ b/tests/queries/0_stateless/01116_asof_join_dolbyzerr.sql
@@ -16,3 +16,17 @@ GROUP BY
 ORDER BY
     visitorId ASC,
     orderId ASC;
+
+SELECT
+    visitorId,
+    orderId,
+    groupUniqArray(sessionId)
+FROM sessions
+ASOF INNER JOIN orders ON (sessions.visitorId = orders.visitorId) AND (sessions.date <= orders.date)
+GROUP BY
+    visitorId,
+    orderId
+ORDER BY
+    visitorId ASC,
+    orderId ASC
+SETTINGS join_algorithm = 'full_sorting_merge';
diff --git a/tests/queries/0_stateless/02240_asof_join_biginteger.reference b/tests/queries/0_stateless/02240_asof_join_biginteger.reference
index cac55eec430..f7eb4d74375 100644
--- a/tests/queries/0_stateless/02240_asof_join_biginteger.reference
+++ b/tests/queries/0_stateless/02240_asof_join_biginteger.reference
@@ -2,3 +2,7 @@
 0	340282366920938463463374607431768211457
 0	18446744073709551617
 0	340282366920938463463374607431768211457
+0	18446744073709551617
+0	340282366920938463463374607431768211457
+0	18446744073709551617
+0	340282366920938463463374607431768211457
diff --git a/tests/queries/0_stateless/02240_asof_join_biginteger.sql b/tests/queries/0_stateless/02240_asof_join_biginteger.sql
index 6dc5b00f116..a5c1faae4ea 100644
--- a/tests/queries/0_stateless/02240_asof_join_biginteger.sql
+++ b/tests/queries/0_stateless/02240_asof_join_biginteger.sql
@@ -3,3 +3,11 @@ select * from (select 0 as k, toInt256('340282366920938463463374607431768211457'
 
 select * from (select 0 as k, toUInt128('18446744073709551617') as v) t1 asof join (select 0 as k, toUInt128('18446744073709551616') as v) t2 using(k, v);
 select * from (select 0 as k, toUInt256('340282366920938463463374607431768211457') as v) t1 asof join (select 0 as k, toUInt256('340282366920938463463374607431768211456') as v) t2 using(k, v);
+
+SET join_algorithm = 'full_sorting_merge';
+
+select * from (select 0 as k, toInt128('18446744073709551617') as v) t1 asof join (select 0 as k, toInt128('18446744073709551616') as v) t2 using(k, v);
+select * from (select 0 as k, toInt256('340282366920938463463374607431768211457') as v) t1 asof join (select 0 as k, toInt256('340282366920938463463374607431768211456') as v) t2 using(k, v);
+
+select * from (select 0 as k, toUInt128('18446744073709551617') as v) t1 asof join (select 0 as k, toUInt128('18446744073709551616') as v) t2 using(k, v);
+select * from (select 0 as k, toUInt256('340282366920938463463374607431768211457') as v) t1 asof join (select 0 as k, toUInt256('340282366920938463463374607431768211456') as v) t2 using(k, v);
diff --git a/tests/queries/0_stateless/02724_function_in_left_table_clause_asof_join.reference b/tests/queries/0_stateless/02724_function_in_left_table_clause_asof_join.reference
index d00491fd7e5..6ed281c757a 100644
--- a/tests/queries/0_stateless/02724_function_in_left_table_clause_asof_join.reference
+++ b/tests/queries/0_stateless/02724_function_in_left_table_clause_asof_join.reference
@@ -1 +1,2 @@
 1
+1
diff --git a/tests/queries/0_stateless/02724_function_in_left_table_clause_asof_join.sql b/tests/queries/0_stateless/02724_function_in_left_table_clause_asof_join.sql
index 13dfb5debe7..6aa70a379c1 100644
--- a/tests/queries/0_stateless/02724_function_in_left_table_clause_asof_join.sql
+++ b/tests/queries/0_stateless/02724_function_in_left_table_clause_asof_join.sql
@@ -6,3 +6,15 @@ ASOF LEFT JOIN (
   select 1 as session_id, 4 as id
 ) as visitors
 ON visitors.session_id <= sessions.id AND arrayFirst(a -> a, arrayMap((a) -> a, sessions.arr)) = visitors.id
+;
+
+select count(*)
+from (
+  select 1 as id, [1, 2, 3] as arr
+) as sessions
+ASOF LEFT JOIN (
+  select 1 as session_id, 4 as id
+) as visitors
+ON visitors.session_id <= sessions.id AND arrayFirst(a -> a, arrayMap((a) -> a, sessions.arr)) = visitors.id
+SETTINGS join_algorithm = 'full_sorting_merge'
+;
diff --git a/tests/queries/0_stateless/03143_asof_join_ddb_long.reference b/tests/queries/0_stateless/03143_asof_join_ddb_long.reference
new file mode 100644
index 00000000000..2850a8aba98
--- /dev/null
+++ b/tests/queries/0_stateless/03143_asof_join_ddb_long.reference
@@ -0,0 +1,2 @@
+49999983751397	10000032
+49999983751397	10000032
diff --git a/tests/queries/0_stateless/03143_asof_join_ddb_long.sql b/tests/queries/0_stateless/03143_asof_join_ddb_long.sql
new file mode 100644
index 00000000000..c421702bb00
--- /dev/null
+++ b/tests/queries/0_stateless/03143_asof_join_ddb_long.sql
@@ -0,0 +1,48 @@
+-- Tags: long
+
+DROP TABLE IF EXISTS build;
+DROP TABLE IF EXISTS skewed_probe;
+
+CREATE TABLE build ENGINE = MergeTree ORDER BY (key, begin)
+AS
+    SELECT
+        toDateTime('1990-03-21 13:00:00') + INTERVAL number MINUTE AS begin,
+        number % 4 AS key,
+        number AS value
+    FROM numbers(0, 10000000);
+
+CREATE TABLE skewed_probe ENGINE = MergeTree ORDER BY (key, begin)
+AS
+    SELECT
+        toDateTime('1990-04-21 13:00:01') + INTERVAL number MINUTE AS begin,
+        0 AS key
+    FROM numbers(0, 5)
+    UNION ALL
+    SELECT
+        toDateTime('1990-05-21 13:00:01') + INTERVAL number MINUTE AS begin,
+        1 AS key
+    FROM numbers(0, 10)
+    UNION ALL
+    SELECT
+        toDateTime('1990-06-21 13:00:01') + INTERVAL number MINUTE AS begin,
+        2 AS key
+    FROM numbers(0, 20)
+    UNION ALL
+    SELECT
+        toDateTime('1990-03-21 13:00:01') + INTERVAL number MINUTE AS begin,
+        3 AS key
+    FROM numbers(0, 10000000);
+
+
+SELECT SUM(value), COUNT(*)
+FROM skewed_probe
+ASOF JOIN build
+USING (key, begin)
+;
+
+SELECT SUM(value), COUNT(*)
+FROM skewed_probe
+ASOF JOIN build
+USING (key, begin)
+SETTINGS join_algorithm = 'full_sorting_merge'
+;
diff --git a/tests/queries/0_stateless/03144_asof_join_ddb_doubles.reference b/tests/queries/0_stateless/03144_asof_join_ddb_doubles.reference
new file mode 100644
index 00000000000..f130f0a3f3b
--- /dev/null
+++ b/tests/queries/0_stateless/03144_asof_join_ddb_doubles.reference
@@ -0,0 +1,58 @@
+1	0
+2	0
+3	1
+4	1
+5	1
+6	2
+7	2
+8	3
+9	3
+0	0
+1	0
+2	0
+3	1
+4	1
+5	1
+6	2
+7	2
+8	3
+9	3
+1	1	0
+1	2	0
+1	3	1
+1	4	1
+1	5	1
+1	6	2
+1	7	2
+1	8	3
+1	9	3
+2	0	10
+2	1	10
+2	2	10
+2	3	10
+2	4	10
+2	5	10
+2	6	10
+2	7	20
+2	8	20
+2	9	20
+1	0	0
+1	1	0
+1	2	0
+1	3	1
+1	4	1
+1	5	1
+1	6	2
+1	7	2
+1	8	3
+1	9	3
+2	0	10
+2	1	10
+2	2	10
+2	3	10
+2	4	10
+2	5	10
+2	6	10
+2	7	20
+2	8	20
+2	9	20
diff --git a/tests/queries/0_stateless/03144_asof_join_ddb_doubles.sql b/tests/queries/0_stateless/03144_asof_join_ddb_doubles.sql
new file mode 100644
index 00000000000..ef16ced3082
--- /dev/null
+++ b/tests/queries/0_stateless/03144_asof_join_ddb_doubles.sql
@@ -0,0 +1,64 @@
+SET join_algorithm = 'full_sorting_merge';
+
+DROP TABLE IF EXISTS events0;
+
+CREATE TABLE events0 (
+    begin Float64,
+    value Int32
+) ENGINE = MergeTree ORDER BY begin;
+
+INSERT INTO events0 VALUES (1.0, 0), (3.0, 1), (6.0, 2), (8.0, 3);
+
+SELECT p.ts, e.value
+FROM
+    (SELECT number :: Float64 AS ts FROM numbers(10)) p
+ASOF  JOIN events0 e
+ON p.ts >= e.begin
+ORDER BY p.ts ASC;
+
+SELECT p.ts, e.value
+FROM
+    (SELECT number :: Float64 AS ts FROM numbers(10)) p
+ASOF LEFT JOIN events0 e
+ON p.ts >= e.begin
+ORDER BY p.ts ASC
+-- SETTINGS join_use_nulls = 1
+;
+
+DROP TABLE IF EXISTS events0;
+
+DROP TABLE IF EXISTS events;
+DROP TABLE IF EXISTS probes;
+
+CREATE TABLE events (
+    key Int32,
+    begin Float64,
+    value Int32
+) ENGINE = MergeTree ORDER BY (key, begin);
+
+INSERT INTO events VALUES (1, 1.0, 0), (1, 3.0, 1), (1, 6.0, 2), (1, 8.0, 3), (2, 0.0, 10), (2, 7.0, 20), (2, 11.0, 30);
+
+CREATE TABLE probes (
+    key Int32,
+    ts Float64
+) ENGINE = MergeTree ORDER BY (key, ts) AS
+SELECT
+    key.number,
+    ts.number
+FROM
+    numbers(1, 2) as key,
+    numbers(10) as ts
+SETTINGS join_algorithm = 'hash';
+
+SELECT p.key, p.ts, e.value
+FROM probes p
+ASOF JOIN events e
+ON p.key = e.key AND p.ts >= e.begin
+ORDER BY p.key, p.ts ASC;
+
+SELECT p.key, p.ts, e.value
+FROM probes p
+ASOF LEFT JOIN events e
+ON p.key = e.key AND p.ts >= e.begin
+ORDER BY p.key, p.ts ASC NULLS FIRST;
+
diff --git a/tests/queries/0_stateless/03145_asof_join_ddb_inequalities.reference b/tests/queries/0_stateless/03145_asof_join_ddb_inequalities.reference
new file mode 100644
index 00000000000..73c4f7dfe25
--- /dev/null
+++ b/tests/queries/0_stateless/03145_asof_join_ddb_inequalities.reference
@@ -0,0 +1,68 @@
+2023-03-21 12:00:00	1970-01-01 00:00:00	-1
+2023-03-21 13:00:00	1970-01-01 00:00:00	-1
+2023-03-21 14:00:00	2023-03-21 13:00:00	0
+2023-03-21 15:00:00	2023-03-21 14:00:00	1
+2023-03-21 16:00:00	2023-03-21 15:00:00	2
+2023-03-21 17:00:00	2023-03-21 16:00:00	3
+2023-03-21 18:00:00	2023-03-21 16:00:00	3
+2023-03-21 19:00:00	2023-03-21 16:00:00	3
+2023-03-21 20:00:00	2023-03-21 16:00:00	3
+2023-03-21 21:00:00	2023-03-21 16:00:00	3
+2027-10-18 11:03:27	2023-03-21 16:00:00	3
+2023-03-21 12:00:00	1970-01-01 00:00:00	-1
+2023-03-21 13:00:00	1970-01-01 00:00:00	-1
+2023-03-21 14:00:00	2023-03-21 13:00:00	0
+2023-03-21 15:00:00	2023-03-21 14:00:00	1
+2023-03-21 16:00:00	2023-03-21 15:00:00	2
+2023-03-21 17:00:00	2023-03-21 16:00:00	3
+2023-03-21 18:00:00	2023-03-21 16:00:00	3
+2023-03-21 19:00:00	2023-03-21 16:00:00	3
+2023-03-21 20:00:00	2023-03-21 16:00:00	3
+2023-03-21 21:00:00	2023-03-21 16:00:00	3
+2027-10-18 11:03:27	2023-03-21 16:00:00	3
+\N	\N	\N
+2023-03-21 12:00:00	2023-03-21 13:00:00	0
+2023-03-21 13:00:00	2023-03-21 13:00:00	0
+2023-03-21 14:00:00	2023-03-21 14:00:00	1
+2023-03-21 15:00:00	2023-03-21 15:00:00	2
+2023-03-21 16:00:00	2023-03-21 16:00:00	3
+2023-03-21 17:00:00	2027-10-18 11:03:27	9
+2023-03-21 18:00:00	2027-10-18 11:03:27	9
+2023-03-21 19:00:00	2027-10-18 11:03:27	9
+2023-03-21 20:00:00	2027-10-18 11:03:27	9
+2023-03-21 21:00:00	2027-10-18 11:03:27	9
+2027-10-18 11:03:27	2027-10-18 11:03:27	9
+2023-03-21 12:00:00	2023-03-21 13:00:00	0
+2023-03-21 13:00:00	2023-03-21 13:00:00	0
+2023-03-21 14:00:00	2023-03-21 14:00:00	1
+2023-03-21 15:00:00	2023-03-21 15:00:00	2
+2023-03-21 16:00:00	2023-03-21 16:00:00	3
+2023-03-21 17:00:00	2027-10-18 11:03:27	9
+2023-03-21 18:00:00	2027-10-18 11:03:27	9
+2023-03-21 19:00:00	2027-10-18 11:03:27	9
+2023-03-21 20:00:00	2027-10-18 11:03:27	9
+2023-03-21 21:00:00	2027-10-18 11:03:27	9
+2027-10-18 11:03:27	2027-10-18 11:03:27	9
+\N	\N	\N
+2023-03-21 12:00:00	2023-03-21 13:00:00	0
+2023-03-21 13:00:00	2023-03-21 14:00:00	1
+2023-03-21 14:00:00	2023-03-21 15:00:00	2
+2023-03-21 15:00:00	2023-03-21 16:00:00	3
+2023-03-21 16:00:00	2027-10-18 11:03:27	9
+2023-03-21 17:00:00	2027-10-18 11:03:27	9
+2023-03-21 18:00:00	2027-10-18 11:03:27	9
+2023-03-21 19:00:00	2027-10-18 11:03:27	9
+2023-03-21 20:00:00	2027-10-18 11:03:27	9
+2023-03-21 21:00:00	2027-10-18 11:03:27	9
+2023-03-21 12:00:00	2023-03-21 13:00:00	0
+2023-03-21 13:00:00	2023-03-21 14:00:00	1
+2023-03-21 14:00:00	2023-03-21 15:00:00	2
+2023-03-21 15:00:00	2023-03-21 16:00:00	3
+2023-03-21 16:00:00	2027-10-18 11:03:27	9
+2023-03-21 17:00:00	2027-10-18 11:03:27	9
+2023-03-21 18:00:00	2027-10-18 11:03:27	9
+2023-03-21 19:00:00	2027-10-18 11:03:27	9
+2023-03-21 20:00:00	2027-10-18 11:03:27	9
+2023-03-21 21:00:00	2027-10-18 11:03:27	9
+2027-10-18 11:03:27	\N	\N
+\N	\N	\N
diff --git a/tests/queries/0_stateless/03145_asof_join_ddb_inequalities.sql b/tests/queries/0_stateless/03145_asof_join_ddb_inequalities.sql
new file mode 100644
index 00000000000..69de17541c1
--- /dev/null
+++ b/tests/queries/0_stateless/03145_asof_join_ddb_inequalities.sql
@@ -0,0 +1,63 @@
+DROP TABLE IF EXISTS events0;
+DROP TABLE IF EXISTS probe0;
+
+SET join_algorithm = 'full_sorting_merge';
+
+CREATE TABLE events0 (
+    begin Nullable(DateTime('UTC')),
+    value Int32
+) ENGINE = MergeTree ORDER BY tuple();
+
+INSERT INTO events0 SELECT toDateTime('2023-03-21 13:00:00', 'UTC') + INTERVAL number HOUR, number FROM numbers(4);
+INSERT INTO events0 VALUES (NULL, -10),('0000-01-01 00:00:00', -1), ('9999-12-31 23:59:59', 9);
+
+CREATE TABLE probe0 (
+    begin Nullable(DateTime('UTC'))
+) ENGINE = MergeTree ORDER BY tuple();
+
+INSERT INTO probe0 SELECT toDateTime('2023-03-21 12:00:00', 'UTC') + INTERVAl number HOUR FROM numbers(10);
+INSERT INTO probe0 VALUES (NULL),('9999-12-31 23:59:59');
+
+SET join_use_nulls = 1;
+
+SELECT p.begin, e.begin, e.value
+FROM probe0 p
+ASOF JOIN events0 e
+ON p.begin > e.begin
+ORDER BY p.begin ASC;
+
+SELECT p.begin, e.begin, e.value
+FROM probe0 p
+ASOF LEFT JOIN events0 e
+ON p.begin > e.begin
+ORDER BY p.begin ASC
+;
+
+SELECT p.begin, e.begin, e.value
+FROM probe0 p
+ASOF JOIN events0 e
+ON p.begin <= e.begin
+ORDER BY p.begin ASC;
+
+SELECT p.begin, e.begin, e.value
+FROM probe0 p
+ASOF LEFT JOIN events0 e
+ON p.begin <= e.begin
+ORDER BY p.begin ASC;
+
+SELECT p.begin, e.begin, e.value
+FROM probe0 p
+ASOF JOIN events0 e
+ON p.begin < e.begin
+ORDER BY p.begin ASC
+;
+
+SELECT p.begin, e.begin, e.value
+FROM probe0 p
+ASOF LEFT JOIN events0 e
+ON p.begin < e.begin
+ORDER BY p.begin ASC;
+
+
+DROP TABLE IF EXISTS events0;
+DROP TABLE IF EXISTS probe0;
diff --git a/tests/queries/0_stateless/03146_asof_join_ddb_merge_long.reference b/tests/queries/0_stateless/03146_asof_join_ddb_merge_long.reference
new file mode 100644
index 00000000000..ca481c7fff0
--- /dev/null
+++ b/tests/queries/0_stateless/03146_asof_join_ddb_merge_long.reference
@@ -0,0 +1,2 @@
+26790	1488
+26790	1488
diff --git a/tests/queries/0_stateless/03146_asof_join_ddb_merge_long.sql.j2 b/tests/queries/0_stateless/03146_asof_join_ddb_merge_long.sql.j2
new file mode 100644
index 00000000000..551bac0cc06
--- /dev/null
+++ b/tests/queries/0_stateless/03146_asof_join_ddb_merge_long.sql.j2
@@ -0,0 +1,37 @@
+SET allow_experimental_analyzer=1;
+
+SET session_timezone = 'UTC';
+
+{% for join_algorithm in ['default', 'full_sorting_merge'] -%}
+
+SET join_algorithm = '{{ join_algorithm }}';
+
+-- TODO: support enable for full_sorting_merge
+-- SET join_use_nulls = 1;
+
+WITH build AS (
+    SELECT
+        tk.number AS k,
+        toDateTime('2021-01-01 00:00:00') + INTERVAL i.number SECONDS AS t,
+        i.number % 37 AS v
+    FROM numbers(3000000) AS i
+    CROSS JOIN numbers(2) AS tk
+    SETTINGS join_algorithm = 'hash', join_use_nulls = 0
+),
+probe AS (
+    SELECT
+        tk.number AS k,
+        toDateTime('2021-01-01 00:00:30') + INTERVAL tt.number HOUR AS t
+    FROM numbers(2) AS tk
+    CROSS JOIN numbers(toUInt32((toDateTime('2021-02-01 00:00:30') - toDateTime('2021-01-01 00:00:30')) / 3600)) AS tt
+    SETTINGS join_algorithm = 'hash', join_use_nulls = 0
+)
+SELECT
+    SUM(v) AS v,
+    COUNT(*) AS n
+FROM probe
+ASOF LEFT JOIN build
+USING (k, t)
+;
+
+{% endfor -%}

From 29494d0bc6cc1b0f82a887a06f0643ea4880e681 Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Wed, 1 May 2024 15:08:50 +0000
Subject: [PATCH 0059/1488] add asof_join_ddb tests

---
 .../03144_asof_join_ddb_doubles.sql           |   1 +
 .../03145_asof_join_ddb_inequalities.sql      |   1 +
 .../03147_asof_join_ddb_missing.reference     |  10 +
 .../03147_asof_join_ddb_missing.sql           | 186 ++++++++++++++++++
 .../03148_asof_join_ddb_subquery.reference    |   4 +
 .../03148_asof_join_ddb_subquery.sql          |  29 +++
 .../03149_asof_join_ddb_timestamps.reference  |  56 ++++++
 .../03149_asof_join_ddb_timestamps.sql        |  95 +++++++++
 8 files changed, 382 insertions(+)
 create mode 100644 tests/queries/0_stateless/03147_asof_join_ddb_missing.reference
 create mode 100644 tests/queries/0_stateless/03147_asof_join_ddb_missing.sql
 create mode 100644 tests/queries/0_stateless/03148_asof_join_ddb_subquery.reference
 create mode 100644 tests/queries/0_stateless/03148_asof_join_ddb_subquery.sql
 create mode 100644 tests/queries/0_stateless/03149_asof_join_ddb_timestamps.reference
 create mode 100644 tests/queries/0_stateless/03149_asof_join_ddb_timestamps.sql

diff --git a/tests/queries/0_stateless/03144_asof_join_ddb_doubles.sql b/tests/queries/0_stateless/03144_asof_join_ddb_doubles.sql
index ef16ced3082..87aece14628 100644
--- a/tests/queries/0_stateless/03144_asof_join_ddb_doubles.sql
+++ b/tests/queries/0_stateless/03144_asof_join_ddb_doubles.sql
@@ -1,4 +1,5 @@
 SET join_algorithm = 'full_sorting_merge';
+SET allow_experimental_analyzer = 1;
 
 DROP TABLE IF EXISTS events0;
 
diff --git a/tests/queries/0_stateless/03145_asof_join_ddb_inequalities.sql b/tests/queries/0_stateless/03145_asof_join_ddb_inequalities.sql
index 69de17541c1..ce4badbd597 100644
--- a/tests/queries/0_stateless/03145_asof_join_ddb_inequalities.sql
+++ b/tests/queries/0_stateless/03145_asof_join_ddb_inequalities.sql
@@ -1,6 +1,7 @@
 DROP TABLE IF EXISTS events0;
 DROP TABLE IF EXISTS probe0;
 
+SET allow_experimental_analyzer = 1;
 SET join_algorithm = 'full_sorting_merge';
 
 CREATE TABLE events0 (
diff --git a/tests/queries/0_stateless/03147_asof_join_ddb_missing.reference b/tests/queries/0_stateless/03147_asof_join_ddb_missing.reference
new file mode 100644
index 00000000000..11eb84463f4
--- /dev/null
+++ b/tests/queries/0_stateless/03147_asof_join_ddb_missing.reference
@@ -0,0 +1,10 @@
+108
+108	27
+513
+1218
+3528
+14553
+121275
+1495503
+12462525
+1249625025
diff --git a/tests/queries/0_stateless/03147_asof_join_ddb_missing.sql b/tests/queries/0_stateless/03147_asof_join_ddb_missing.sql
new file mode 100644
index 00000000000..95a5f8ab3ff
--- /dev/null
+++ b/tests/queries/0_stateless/03147_asof_join_ddb_missing.sql
@@ -0,0 +1,186 @@
+SET allow_experimental_analyzer=1;
+
+SET session_timezone = 'UTC';
+SET joined_subquery_requires_alias = 0;
+SET allow_experimental_analyzer = 1;
+SET join_algorithm = 'full_sorting_merge';
+
+-- # 10 dates, 5 keys
+WITH build AS (
+    SELECT
+        k,
+        toDateTime('2001-01-01 00:00:00') + INTERVAL number MINUTE AS t,
+        number AS v
+    FROM numbers(10), (SELECT number AS k FROM numbers(5))
+    SETTINGS join_algorithm = 'default'
+),
+probe AS (
+    SELECT
+        k * 2 AS k,
+        t - INTERVAL 30 SECOND AS t
+    FROM build
+)
+SELECT SUM(v)
+FROM probe ASOF JOIN build USING (k, t);
+
+-- # Coverage: Missing right side bin
+WITH build AS (
+    SELECT
+        k * 2 AS k,
+        toDateTime('2001-01-01 00:00:00') + INTERVAL number MINUTE AS t,
+        number AS v
+    FROM numbers(10), (SELECT number AS k FROM numbers(5))
+    SETTINGS join_algorithm = 'default'
+),
+probe AS (
+    SELECT
+        intDiv(k, 2) AS k,
+        t - INTERVAL 30 SECOND AS t
+    FROM build
+)
+SELECT SUM(v), COUNT(*)
+FROM probe ASOF JOIN build USING (k, t);
+
+-- # 20 dates, 5 keys
+WITH build AS (
+    SELECT
+        k,
+        toDateTime('2001-01-01 00:00:00') + INTERVAL number MINUTE AS t,
+        number AS v
+    FROM numbers(20), (SELECT number AS k FROM numbers(5))
+    SETTINGS join_algorithm = 'default'
+),
+probe AS (
+    SELECT
+        k * 2 AS k,
+        t - INTERVAL 30 SECOND AS t
+    FROM build
+)
+SELECT SUM(v)
+FROM probe ASOF JOIN build USING (k, t);
+
+-- # 30 dates, 5 keys
+WITH build AS (
+    SELECT
+        k,
+        toDateTime('2001-01-01 00:00:00') + INTERVAL number MINUTE AS t,
+        number AS v
+    FROM numbers(30), (SELECT number AS k FROM numbers(5))
+    SETTINGS join_algorithm = 'default'
+),
+probe AS (
+    SELECT
+        k * 2 AS k,
+        t - INTERVAL 30 SECOND AS t
+    FROM build
+)
+SELECT SUM(v)
+FROM probe ASOF JOIN build USING (k, t);
+
+-- # 50 dates, 5 keys
+WITH build AS (
+    SELECT
+        k,
+        toDateTime('2001-01-01 00:00:00') + INTERVAL number MINUTE AS t,
+        number AS v
+    FROM numbers(50), (SELECT number AS k FROM numbers(5))
+    SETTINGS join_algorithm = 'default'
+),
+probe AS (
+    SELECT
+        k * 2 AS k,
+        t - INTERVAL 30 SECOND AS t
+    FROM build
+)
+SELECT SUM(v)
+FROM probe ASOF JOIN build USING (k, t);
+
+-- # 100 dates, 5 keys
+WITH build AS (
+    SELECT
+        k,
+        toDateTime('2001-01-01 00:00:00') + INTERVAL number MINUTE AS t,
+        number AS v
+    FROM numbers(100), (SELECT number AS k FROM numbers(5))
+    SETTINGS join_algorithm = 'default'
+),
+probe AS (
+    SELECT
+        k * 2 AS k,
+        t - INTERVAL 30 SECOND AS t
+    FROM build
+)
+SELECT SUM(v)
+FROM probe ASOF JOIN build USING (k, t);
+
+-- # 100 dates, 50 keys
+WITH build AS (
+    SELECT
+        k,
+        toDateTime('2001-01-01 00:00:00') + INTERVAL number MINUTE AS t,
+        number AS v
+    FROM numbers(100), (SELECT number AS k FROM numbers(50))
+    SETTINGS join_algorithm = 'default'
+),
+probe AS (
+    SELECT
+        k * 2 AS k,
+        t - INTERVAL 30 SECOND AS t
+    FROM build
+)
+SELECT SUM(v)
+FROM probe ASOF JOIN build USING (k, t);
+
+-- # 1000 dates, 5 keys
+WITH build AS (
+    SELECT
+        k,
+        toDateTime('2001-01-01 00:00:00') + INTERVAL number MINUTE AS t,
+        number AS v
+    FROM numbers(1000), (SELECT number AS k FROM numbers(5))
+    SETTINGS join_algorithm = 'default'
+),
+probe AS (
+    SELECT
+        k * 2 AS k,
+        t - INTERVAL 30 SECOND AS t
+    FROM build
+)
+SELECT SUM(v)
+FROM probe ASOF JOIN build USING (k, t);
+
+-- # 1000 dates, 50 keys
+WITH build AS (
+    SELECT
+        k,
+        toDateTime('2001-01-01 00:00:00') + INTERVAL number MINUTE AS t,
+        number AS v
+    FROM numbers(1000), (SELECT number AS k FROM numbers(50))
+    SETTINGS join_algorithm = 'default'
+),
+probe AS (
+    SELECT
+        k * 2 AS k,
+        t - INTERVAL 30 SECOND AS t
+    FROM build
+)
+SELECT SUM(v)
+FROM probe ASOF JOIN build USING (k, t);
+
+-- # 10000 dates, 50 keys
+WITH build AS (
+    SELECT
+        k,
+        toDateTime('2001-01-01 00:00:00') + INTERVAL number MINUTE AS t,
+        number AS v
+    FROM numbers(10000), (SELECT number AS k FROM numbers(50))
+    SETTINGS join_algorithm = 'default'
+),
+probe AS (
+    SELECT
+        k * 2 AS k,
+        t - INTERVAL 30 SECOND AS t
+    FROM build
+)
+SELECT SUM(v)
+FROM probe ASOF JOIN build USING (k, t);
diff --git a/tests/queries/0_stateless/03148_asof_join_ddb_subquery.reference b/tests/queries/0_stateless/03148_asof_join_ddb_subquery.reference
new file mode 100644
index 00000000000..387a4a8f249
--- /dev/null
+++ b/tests/queries/0_stateless/03148_asof_join_ddb_subquery.reference
@@ -0,0 +1,4 @@
+1	1
+3	1
+6	1
+8	1
diff --git a/tests/queries/0_stateless/03148_asof_join_ddb_subquery.sql b/tests/queries/0_stateless/03148_asof_join_ddb_subquery.sql
new file mode 100644
index 00000000000..2ddf0f09b1e
--- /dev/null
+++ b/tests/queries/0_stateless/03148_asof_join_ddb_subquery.sql
@@ -0,0 +1,29 @@
+DROP TABLE IF EXISTS events;
+CREATE TABLE events (begin Float64, value Int32) ENGINE = MergeTree() ORDER BY begin;
+
+INSERT INTO events VALUES (1, 0), (3, 1), (6, 2), (8, 3);
+
+SET allow_experimental_analyzer = 1;
+SET join_algorithm = 'full_sorting_merge';
+SET joined_subquery_requires_alias = 0;
+
+SELECT
+    begin,
+    value IN (
+        SELECT e1.value
+        FROM (
+            SELECT *
+            FROM events e1
+            WHERE e1.value = events.value
+        ) AS e1
+        ASOF JOIN (
+            SELECT number :: Float64 AS begin
+            FROM numbers(10)
+            WHERE number >= 1 AND number < 10
+        )
+        USING (begin)
+    )
+FROM events
+ORDER BY begin ASC;
+
+DROP TABLE IF EXISTS events;
diff --git a/tests/queries/0_stateless/03149_asof_join_ddb_timestamps.reference b/tests/queries/0_stateless/03149_asof_join_ddb_timestamps.reference
new file mode 100644
index 00000000000..7cfc85d23a5
--- /dev/null
+++ b/tests/queries/0_stateless/03149_asof_join_ddb_timestamps.reference
@@ -0,0 +1,56 @@
+2023-03-21 13:00:00	0
+2023-03-21 14:00:00	1
+2023-03-21 15:00:00	2
+2023-03-21 16:00:00	3
+2023-03-21 17:00:00	3
+2023-03-21 18:00:00	3
+2023-03-21 19:00:00	3
+2023-03-21 20:00:00	3
+2023-03-21 21:00:00	3
+2106-02-07 06:28:15	9
+2023-03-21 13:00:00	0
+2023-03-21 14:00:00	1
+2023-03-21 15:00:00	2
+2023-03-21 16:00:00	3
+2023-03-21 17:00:00	3
+2023-03-21 18:00:00	3
+2023-03-21 19:00:00	3
+2023-03-21 20:00:00	3
+2023-03-21 21:00:00	3
+2106-02-07 06:28:15	9
+2023-03-21 12:00:00	\N
+2023-03-21 13:00:00	0
+2023-03-21 14:00:00	1
+2023-03-21 15:00:00	2
+2023-03-21 16:00:00	3
+2023-03-21 17:00:00	3
+2023-03-21 18:00:00	3
+2023-03-21 19:00:00	3
+2023-03-21 20:00:00	3
+2023-03-21 21:00:00	3
+2106-02-07 06:28:15	9
+\N	\N
+2023-03-21 12:00:00	0
+2023-03-21 13:00:00	0
+2023-03-21 14:00:00	1
+2023-03-21 15:00:00	2
+2023-03-21 16:00:00	3
+2023-03-21 17:00:00	3
+2023-03-21 18:00:00	3
+2023-03-21 19:00:00	3
+2023-03-21 20:00:00	3
+2023-03-21 21:00:00	3
+2106-02-07 06:28:15	9
+\N	0
+2023-03-21 12:00:00	\N
+2023-03-21 13:00:00	\N
+2023-03-21 14:00:00	\N
+2023-03-21 15:00:00	\N
+2023-03-21 16:00:00	\N
+2023-03-21 17:00:00	\N
+2023-03-21 18:00:00	\N
+2023-03-21 19:00:00	\N
+2023-03-21 20:00:00	\N
+2023-03-21 21:00:00	\N
+2106-02-07 06:28:15	\N
+\N	\N
diff --git a/tests/queries/0_stateless/03149_asof_join_ddb_timestamps.sql b/tests/queries/0_stateless/03149_asof_join_ddb_timestamps.sql
new file mode 100644
index 00000000000..ff4518a3775
--- /dev/null
+++ b/tests/queries/0_stateless/03149_asof_join_ddb_timestamps.sql
@@ -0,0 +1,95 @@
+DROP TABLE IF EXISTS events0;
+DROP TABLE IF EXISTS probe0;
+
+SET session_timezone = 'UTC';
+SET allow_experimental_analyzer = 1;
+SET join_algorithm = 'full_sorting_merge';
+SET join_use_nulls = 1;
+
+CREATE TABLE events0
+ENGINE = MergeTree()
+ORDER BY COALESCE(begin, toDateTime('9999-12-31 23:59:59'))
+AS
+SELECT
+    toNullable(toDateTime('2023-03-21 13:00:00') + INTERVAL number HOUR) AS begin,
+    number AS value
+FROM numbers(4);
+
+INSERT INTO events0 VALUES (NULL, -1), (toDateTime('9999-12-31 23:59:59'), 9);
+
+CREATE TABLE probe0
+ENGINE = MergeTree()
+ORDER BY COALESCE(begin, toDateTime('9999-12-31 23:59:59'))
+AS
+SELECT
+    toNullable(toDateTime('2023-03-21 12:00:00') + INTERVAL number HOUR) AS begin
+FROM numbers(10);
+
+INSERT INTO probe0 VALUES (NULL), (toDateTime('9999-12-31 23:59:59'));
+
+SELECT
+    p.begin,
+    e.value
+FROM
+    probe0 p
+    ASOF JOIN events0 e ON p.begin >= e.begin
+ORDER BY p.begin ASC;
+
+SELECT
+    p.begin,
+    e.value
+FROM
+    probe0 p
+    ASOF JOIN events0 e USING (begin)
+ORDER BY p.begin ASC
+SETTINGS join_use_nulls = 0
+;
+
+SELECT
+    p.begin,
+    e.value
+FROM
+    probe0 p
+    ASOF LEFT JOIN events0 e ON p.begin >= e.begin
+ORDER BY p.begin ASC;
+
+SELECT
+    p.begin,
+    e.value
+FROM
+    probe0 p
+    ASOF LEFT JOIN events0 e USING (begin)
+ORDER BY p.begin ASC
+SETTINGS join_use_nulls = 0
+;
+
+SELECT
+    p.begin,
+    e.value
+FROM
+    probe0 p
+    ASOF RIGHT JOIN events0 e ON p.begin >= e.begin
+ORDER BY e.begin ASC; -- { serverError NOT_IMPLEMENTED}
+
+SELECT
+    p.begin,
+    e.value
+FROM
+    probe0 p
+    ASOF RIGHT JOIN events0 e USING (begin)
+ORDER BY e.begin ASC; -- { serverError NOT_IMPLEMENTED}
+
+
+SELECT
+    p.begin,
+    e.value
+FROM
+    probe0 p
+    ASOF LEFT JOIN (
+        SELECT * FROM events0 WHERE log(value + 5) > 10
+    ) e ON p.begin >= e.begin
+ORDER BY p.begin ASC;
+
+
+DROP TABLE IF EXISTS events0;
+DROP TABLE IF EXISTS probe0;

From d37f03201aaac06b1a175ac0b177c17d945dddbb Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Tue, 7 May 2024 10:07:43 +0000
Subject: [PATCH 0060/1488] fix

---
 src/Processors/Transforms/MergeJoinTransform.cpp                | 2 --
 tests/queries/0_stateless/03146_asof_join_ddb_merge_long.sql.j2 | 2 ++
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Processors/Transforms/MergeJoinTransform.cpp b/src/Processors/Transforms/MergeJoinTransform.cpp
index 26e1ebb0b60..9e6904f0613 100644
--- a/src/Processors/Transforms/MergeJoinTransform.cpp
+++ b/src/Processors/Transforms/MergeJoinTransform.cpp
@@ -1173,9 +1173,7 @@ IMergingAlgorithm::Status MergeJoinAlgorithm::merge()
 
 
     if (auto result = handleAllJoinState())
-    {
         return std::move(*result);
-    }
 
     if (auto result = handleAsofJoinState())
         return std::move(*result);
diff --git a/tests/queries/0_stateless/03146_asof_join_ddb_merge_long.sql.j2 b/tests/queries/0_stateless/03146_asof_join_ddb_merge_long.sql.j2
index 551bac0cc06..44c54ae2a39 100644
--- a/tests/queries/0_stateless/03146_asof_join_ddb_merge_long.sql.j2
+++ b/tests/queries/0_stateless/03146_asof_join_ddb_merge_long.sql.j2
@@ -1,3 +1,5 @@
+-- Tags: long
+
 SET allow_experimental_analyzer=1;
 
 SET session_timezone = 'UTC';

From 558b73aba4f89db4a2b3e5ed0754749caddd5dfb Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Tue, 7 May 2024 14:20:40 +0000
Subject: [PATCH 0061/1488] t

---
 tests/queries/0_stateless/03143_asof_join_ddb_long.sql | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/queries/0_stateless/03143_asof_join_ddb_long.sql b/tests/queries/0_stateless/03143_asof_join_ddb_long.sql
index c421702bb00..a41b667e6c8 100644
--- a/tests/queries/0_stateless/03143_asof_join_ddb_long.sql
+++ b/tests/queries/0_stateless/03143_asof_join_ddb_long.sql
@@ -1,5 +1,7 @@
 -- Tags: long
 
+-- https://s3.amazonaws.com/clickhouse-test-reports/55051/07f288862c56b0a98379a07101062689b0460788/stateless_tests_flaky_check__asan_.html
+
 DROP TABLE IF EXISTS build;
 DROP TABLE IF EXISTS skewed_probe;
 

From f710a67fb2e72361801f0ae9a45ccbd07e2c7f30 Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Wed, 22 May 2024 12:27:04 +0000
Subject: [PATCH 0062/1488] set timezone in 03143_asof_join_ddb_long

---
 src/Processors/Transforms/MergeJoinTransform.cpp       | 4 ++--
 tests/queries/0_stateless/03143_asof_join_ddb_long.sql | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/Processors/Transforms/MergeJoinTransform.cpp b/src/Processors/Transforms/MergeJoinTransform.cpp
index 9e6904f0613..38b63a856f6 100644
--- a/src/Processors/Transforms/MergeJoinTransform.cpp
+++ b/src/Processors/Transforms/MergeJoinTransform.cpp
@@ -45,8 +45,8 @@ FullMergeJoinCursorPtr createCursor(const Block & block, const Names & columns,
 
 bool isNullAt(const IColumn & column, size_t row)
 {
-    if (const auto * nullable = checkAndGetColumn<ColumnNullable>(column))
-        return nullable->isNullAt(row);
+    if (const auto * nullable_column = checkAndGetColumn<ColumnNullable>(&column))
+        return nullable_column->isNullAt(row);
     return false;
 }
 
diff --git a/tests/queries/0_stateless/03143_asof_join_ddb_long.sql b/tests/queries/0_stateless/03143_asof_join_ddb_long.sql
index a41b667e6c8..17a67511030 100644
--- a/tests/queries/0_stateless/03143_asof_join_ddb_long.sql
+++ b/tests/queries/0_stateless/03143_asof_join_ddb_long.sql
@@ -1,10 +1,10 @@
 -- Tags: long
 
--- https://s3.amazonaws.com/clickhouse-test-reports/55051/07f288862c56b0a98379a07101062689b0460788/stateless_tests_flaky_check__asan_.html
-
 DROP TABLE IF EXISTS build;
 DROP TABLE IF EXISTS skewed_probe;
 
+SET session_timezone = 'UTC';
+
 CREATE TABLE build ENGINE = MergeTree ORDER BY (key, begin)
 AS
     SELECT

From 11f4ed75fd18a853e70c94f10596d5b3b09a35dd Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Wed, 5 Jun 2024 12:26:05 +0200
Subject: [PATCH 0063/1488] fix build

---
 src/Processors/tests/gtest_full_sorting_join.cpp | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/Processors/tests/gtest_full_sorting_join.cpp b/src/Processors/tests/gtest_full_sorting_join.cpp
index a3fda006eb8..7294a1b381a 100644
--- a/src/Processors/tests/gtest_full_sorting_join.cpp
+++ b/src/Processors/tests/gtest_full_sorting_join.cpp
@@ -151,7 +151,6 @@ public:
         size_t rows = current_chunk.front()->size();
         chunks.emplace_back(std::move(current_chunk), rows);
         current_chunk = header.cloneEmptyColumns();
-        return;
     }
 
     std::shared_ptr<ISource> getSource()
@@ -403,7 +402,7 @@ try
     auto expected_right = ColumnString::create();
 
     UInt64 k1 = 1;
-    String k2 = "";
+    String k2;
 
     auto get_attr = [&](const String & side, size_t idx) -> String
     {
@@ -601,7 +600,7 @@ try
     ColumnInt64::Container expected;
 
     UInt64 k1 = 1;
-    String k2 = "";
+    String k2;
     auto key_num_total = std::uniform_int_distribution<>(1, 1000)(rng);
     for (size_t key_num = 0; key_num < key_num_total; ++key_num)
     {
@@ -690,7 +689,7 @@ try
     ColumnInt64::Container expected;
 
     UInt64 k1 = 1;
-    String k2 = "";
+    String k2;
     UInt64 left_t = 0;
 
     auto key_num_total = std::uniform_int_distribution<>(1, 1000)(rng);

From 4489795502e591c90f7e7978a73f6f33552cf24e Mon Sep 17 00:00:00 2001
From: Daniel Anugerah <danielanugerah2004@gmail.com>
Date: Fri, 14 Jun 2024 00:40:44 +0800
Subject: [PATCH 0064/1488] feat: support accept_invalid_certificate in
 client's config

---
 programs/client/Client.cpp                    |   7 +-
 ...9_clickhouse_accept_invalid_certificate.sh | 107 ++++++++++++++++++
 .../01889_clickhouse_client_config_format.sh  |  31 +++--
 .../02550_client_connections_credentials.sh   |  13 ++-
 4 files changed, 138 insertions(+), 20 deletions(-)
 create mode 100644 tests/queries/0_stateless/01189_clickhouse_accept_invalid_certificate.sh

diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp
index efe23d57478..f5e86b756fb 100644
--- a/programs/client/Client.cpp
+++ b/programs/client/Client.cpp
@@ -178,6 +178,9 @@ void Client::parseConnectionsCredentials(Poco::Util::AbstractConfiguration & con
                 history_file = home_path + "/" + history_file.substr(1);
             config.setString("history_file", history_file);
         }
+        if (config.has(prefix + "accept-invalid-certificate")) {
+            config.setString("accept-invalid-certificate", prefix + "accept-invalid-certificate");
+        }
     }
 
     if (!connection_name.empty() && !connection_found)
@@ -721,7 +724,7 @@ bool Client::processWithFuzzing(const String & full_query)
     }
     if (auto *q = orig_ast->as<ASTSetQuery>())
     {
-        if (auto *setDialect = q->changes.tryGet("dialect"); setDialect && setDialect->safeGet<String>() == "kusto")
+        if (auto *set_dialect = q->changes.tryGet("dialect"); set_dialect && set_dialect->safeGet<String>() == "kusto")
             return true;
     }
 
@@ -1102,7 +1105,7 @@ void Client::processOptions(const OptionsDescription & options_description,
         config().setBool("no-warnings", true);
     if (options.count("fake-drop"))
         config().setString("ignore_drop_queries_probability", "1");
-    if (options.count("accept-invalid-certificate"))
+    if (options.count("accept-invalid-certificate") || config().has("accept-invalid-certificate"))
     {
         config().setString("openSSL.client.invalidCertificateHandler.name", "AcceptCertificateHandler");
         config().setString("openSSL.client.verificationMode", "none");
diff --git a/tests/queries/0_stateless/01189_clickhouse_accept_invalid_certificate.sh b/tests/queries/0_stateless/01189_clickhouse_accept_invalid_certificate.sh
new file mode 100644
index 00000000000..00378abdb8e
--- /dev/null
+++ b/tests/queries/0_stateless/01189_clickhouse_accept_invalid_certificate.sh
@@ -0,0 +1,107 @@
+#!/usr/bin/env bash
+# Tags: no-fasttest, no-random-settings
+
+CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CUR_DIR"/../shell_config.sh
+
+# use $CLICKHOUSE_DATABASE so that clickhouse-test will replace it with default to match .reference
+config=$CLICKHOUSE_TMP/config_$CLICKHOUSE_DATABASE
+xml_config=$CLICKHOUSE_TMP/config_$CLICKHOUSE_DATABASE.xml
+XML_config=$CLICKHOUSE_TMP/config_$CLICKHOUSE_DATABASE.XML
+conf_config=$CLICKHOUSE_TMP/config_$CLICKHOUSE_DATABASE.conf
+yml_config=$CLICKHOUSE_TMP/config_$CLICKHOUSE_DATABASE.yml
+yaml_config=$CLICKHOUSE_TMP/config_$CLICKHOUSE_DATABASE.yaml
+ini_config=$CLICKHOUSE_TMP/config_$CLICKHOUSE_DATABASE.ini
+
+function cleanup() {
+	rm "${config:?}"
+	rm "${xml_config:?}"
+	rm "${XML_config:?}"
+	rm "${conf_config:?}"
+	rm "${yml_config:?}"
+	rm "${yaml_config:?}"
+	rm "${ini_config:?}"
+}
+trap cleanup EXIT
+
+cat >"$config" <<EOL
+<config>
+    <openSSL>
+        <client>
+            <invalidCertificateHandler>
+                <name>RejectCertificateHandler</name>
+            </invalidCertificateHandler>
+        </client>
+    </openSSL>
+</config>
+EOL
+cat >"$conf_config" <<EOL
+<config>
+    <openSSL>
+        <client>
+            <invalidCertificateHandler>
+                <name>RejectCertificateHandler</name>
+            </invalidCertificateHandler>
+        </client>
+    </openSSL>
+</config>
+EOL
+cat >"$xml_config" <<EOL
+<config>
+    <openSSL>
+        <client>
+            <invalidCertificateHandler>
+                <name>RejectCertificateHandler</name>
+            </invalidCertificateHandler>
+        </client>
+    </openSSL>
+</config>
+EOL
+cat >"$XML_config" <<EOL
+<config>
+    <openSSL>
+        <client>
+            <invalidCertificateHandler>
+                <name>RejectCertificateHandler</name>
+            </invalidCertificateHandler>
+        </client>
+    </openSSL>
+</config>
+EOL
+cat >"$yml_config" <<EOL
+openSSL:
+  client:
+    invalidCertificateHandler:
+      name: RejectCertificateHandler
+EOL
+cat >"$yaml_config" <<EOL
+openSSL:
+  client:
+    invalidCertificateHandler:
+      name: RejectCertificateHandler
+EOL
+cat >"$ini_config" <<EOL
+[openSSL.client.invalidCertificateHandler]
+name = RejectCertificateHandler
+EOL
+
+echo 'default'
+$CLICKHOUSE_CLIENT --config "$config" -q "select getSetting('invalidCertificateHandler')"
+echo 'xml'
+$CLICKHOUSE_CLIENT --config "$xml_config" -q "select getSetting('invalidCertificateHandler')"
+echo 'XML'
+$CLICKHOUSE_CLIENT --config "$XML_config" -q "select getSetting('invalidCertificateHandler')"
+echo 'conf'
+$CLICKHOUSE_CLIENT --config "$conf_config" -q "select getSetting('invalidCertificateHandler')"
+echo '/dev/fd/PIPE'
+# verify that /dev/fd/X parsed as XML (regardless it has .xml extension or not)
+# and that pipe does works
+$CLICKHOUSE_CLIENT --config <(echo '<config><invalidCertificateHandler><name>RejectCertificateHandler</name></invalidCertificateHandler></config>') -q "select getSetting('invalidCertificateHandler')"
+
+echo 'yml'
+$CLICKHOUSE_CLIENT --config "$yml_config" -q "select getSetting('invalidCertificateHandler')"
+echo 'yaml'
+$CLICKHOUSE_CLIENT --config "$yaml_config" -q "select getSetting('invalidCertificateHandler')"
+echo 'ini'
+$CLICKHOUSE_CLIENT --config "$ini_config" -q "select getSetting('invalidCertificateHandler')" 2>&1 |& sed -e "s#$CLICKHOUSE_TMP##" -e "s#DB::Exception: ##"
diff --git a/tests/queries/0_stateless/01889_clickhouse_client_config_format.sh b/tests/queries/0_stateless/01889_clickhouse_client_config_format.sh
index 4e970f17d3a..4af4e27653c 100755
--- a/tests/queries/0_stateless/01889_clickhouse_client_config_format.sh
+++ b/tests/queries/0_stateless/01889_clickhouse_client_config_format.sh
@@ -14,45 +14,44 @@ yml_config=$CLICKHOUSE_TMP/config_$CLICKHOUSE_DATABASE.yml
 yaml_config=$CLICKHOUSE_TMP/config_$CLICKHOUSE_DATABASE.yaml
 ini_config=$CLICKHOUSE_TMP/config_$CLICKHOUSE_DATABASE.ini
 
-function cleanup()
-{
-    rm "${config:?}"
-    rm "${xml_config:?}"
-    rm "${XML_config:?}"
-    rm "${conf_config:?}"
-    rm "${yml_config:?}"
-    rm "${yaml_config:?}"
-    rm "${ini_config:?}"
+function cleanup() {
+	rm "${config:?}"
+	rm "${xml_config:?}"
+	rm "${XML_config:?}"
+	rm "${conf_config:?}"
+	rm "${yml_config:?}"
+	rm "${yaml_config:?}"
+	rm "${ini_config:?}"
 }
 trap cleanup EXIT
 
-cat > "$config" <<EOL
+cat >"$config" <<EOL
 <config>
     <max_threads>2</max_threads>
 </config>
 EOL
-cat > "$conf_config" <<EOL
+cat >"$conf_config" <<EOL
 <config>
     <max_threads>2</max_threads>
 </config>
 EOL
-cat > "$xml_config" <<EOL
+cat >"$xml_config" <<EOL
 <config>
     <max_threads>2</max_threads>
 </config>
 EOL
-cat > "$XML_config" <<EOL
+cat >"$XML_config" <<EOL
 <config>
     <max_threads>2</max_threads>
 </config>
 EOL
-cat > "$yml_config" <<EOL
+cat >"$yml_config" <<EOL
 max_threads: 2
 EOL
-cat > "$yaml_config" <<EOL
+cat >"$yaml_config" <<EOL
 max_threads: 2
 EOL
-cat > "$ini_config" <<EOL
+cat >"$ini_config" <<EOL
 [config]
 max_threads=2
 EOL
diff --git a/tests/queries/0_stateless/02550_client_connections_credentials.sh b/tests/queries/0_stateless/02550_client_connections_credentials.sh
index a88f3fc7880..b7319d22d1b 100755
--- a/tests/queries/0_stateless/02550_client_connections_credentials.sh
+++ b/tests/queries/0_stateless/02550_client_connections_credentials.sh
@@ -14,7 +14,7 @@ CLICKHOUSE_PORT_TCP=""
 . "$CUR_DIR"/../shell_config.sh
 
 CONFIG=$CLICKHOUSE_TMP/client.xml
-cat > $CONFIG <<EOL
+cat >$CONFIG <<EOL
 <clickhouse>
     <host>$TEST_HOST</host>
     <port>$TEST_PORT</port>
@@ -66,12 +66,21 @@ cat > $CONFIG <<EOL
             <hostname>$TEST_HOST</hostname>
             <history_file>/no/such/dir/.history</history_file>
         </connection>
+
+        <connection>
+          <name> test_accept_invalid_certificate </name>
+          <hostname> $TEST_HOST </hostname>
+          <database>$CLICKHOUSE_DATABASE</database>
+          <invalidCertificateHandler>
+              <name>RejectCertificateHandler</name>
+          </invalidCertificateHandler>
+        </connection>
     </connections_credentials>
 </clickhouse>
 EOL
 
 CONFIG_ROOT_OVERRIDES=$CLICKHOUSE_TMP/client_user_pass.xml
-cat > $CONFIG_ROOT_OVERRIDES <<EOL
+cat >$CONFIG_ROOT_OVERRIDES <<EOL
 <clickhouse>
     <host>$TEST_HOST</host>
     <port>$TEST_PORT</port>

From 99456d2fa630455ebf1a2422b5e827ec056f4130 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Fri, 14 Jun 2024 17:21:15 +0200
Subject: [PATCH 0065/1488] Fix unexpeced size of low cardinality column in
 functions

---
 src/Functions/IFunction.cpp | 101 ++++++++++++++++++++++++------------
 1 file changed, 68 insertions(+), 33 deletions(-)

diff --git a/src/Functions/IFunction.cpp b/src/Functions/IFunction.cpp
index 31695fc95d5..9217071ca11 100644
--- a/src/Functions/IFunction.cpp
+++ b/src/Functions/IFunction.cpp
@@ -50,43 +50,78 @@ bool allArgumentsAreConstants(const ColumnsWithTypeAndName & args)
 ColumnPtr replaceLowCardinalityColumnsByNestedAndGetDictionaryIndexes(
     ColumnsWithTypeAndName & args, bool can_be_executed_on_default_arguments, size_t input_rows_count)
 {
-    size_t num_rows = input_rows_count;
+    /// We return the LC indexes so the LC can be reconstructed with the function result
     ColumnPtr indexes;
 
-    /// Find first LowCardinality column and replace it to nested dictionary.
-    for (auto & column : args)
+    size_t number_low_cardinality_columns = 0;
+    size_t last_low_cardinality = 0;
+    size_t number_const_columns = 0;
+    size_t number_full_columns = 0;
+
+    for (size_t i = 0; i < args.size(); i++)
     {
-        if (const auto * low_cardinality_column = checkAndGetColumn<ColumnLowCardinality>(column.column.get()))
+        auto const & arg = args[i];
+        if (checkAndGetColumn<ColumnLowCardinality>(arg.column.get()))
         {
-            /// Single LowCardinality column is supported now.
-            if (indexes)
-                throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected single dictionary argument for function.");
-
-            const auto * low_cardinality_type = checkAndGetDataType<DataTypeLowCardinality>(column.type.get());
-
-            if (!low_cardinality_type)
-                throw Exception(ErrorCodes::LOGICAL_ERROR,
-                    "Incompatible type for LowCardinality column: {}",
-                    column.type->getName());
-
-            if (can_be_executed_on_default_arguments)
-            {
-                /// Normal case, when function can be executed on values' default.
-                column.column = low_cardinality_column->getDictionary().getNestedColumn();
-                indexes = low_cardinality_column->getIndexesPtr();
-            }
-            else
-            {
-                /// Special case when default value can't be used. Example: 1 % LowCardinality(Int).
-                /// LowCardinality always contains default, so 1 % 0 will throw exception in normal case.
-                auto dict_encoded = low_cardinality_column->getMinimalDictionaryEncodedColumn(0, low_cardinality_column->size());
-                column.column = dict_encoded.dictionary;
-                indexes = dict_encoded.indexes;
-            }
-
-            num_rows = column.column->size();
-            column.type = low_cardinality_type->getDictionaryType();
+            number_low_cardinality_columns++;
+            last_low_cardinality = i;
         }
+        else if (checkAndGetColumn<ColumnConst>(arg.column.get()))
+            number_const_columns++;
+        else
+            number_full_columns++;
+    }
+
+    if (!number_low_cardinality_columns && !number_const_columns)
+        return nullptr;
+
+    if (number_full_columns > 0 || number_low_cardinality_columns > 1)
+    {
+        /// If there is a single full column, we can't replace the LC column with its dictionary, as it won't match
+        /// the size or order of the full columns. Same if there are 2 or more low cardinality columns
+        for (auto & arg : args)
+        {
+            if (const auto * column_lc = checkAndGetColumn<ColumnLowCardinality>(arg.column.get()))
+            {
+                arg.column = recursiveRemoveLowCardinality(arg.column);
+                chassert(arg.column->size() == input_rows_count);
+
+                const auto * low_cardinality_type = checkAndGetDataType<DataTypeLowCardinality>(arg.type.get());
+                if (!low_cardinality_type)
+                    throw Exception(ErrorCodes::LOGICAL_ERROR, "Incompatible type for LowCardinality column: {}", arg.type->getName());
+                arg.type = recursiveRemoveLowCardinality(arg.type);
+            }
+        }
+    }
+    else if (number_low_cardinality_columns == 1)
+    {
+        auto & lc_arg = args[last_low_cardinality];
+
+        const auto * low_cardinality_type = checkAndGetDataType<DataTypeLowCardinality>(lc_arg.type.get());
+        if (!low_cardinality_type)
+            throw Exception(ErrorCodes::LOGICAL_ERROR, "Incompatible type for LowCardinality column: {}", lc_arg.type->getName());
+
+        const auto * low_cardinality_column = checkAndGetColumn<ColumnLowCardinality>(lc_arg.column.get());
+        chassert(low_cardinality_column);
+
+        if (can_be_executed_on_default_arguments)
+        {
+            /// Normal case, when function can be executed on values' default.
+            lc_arg.column = low_cardinality_column->getDictionary().getNestedColumn();
+            indexes = low_cardinality_column->getIndexesPtr();
+        }
+        else
+        {
+            /// Special case when default value can't be used. Example: 1 % LowCardinality(Int).
+            /// LowCardinality always contains default, so 1 % 0 will throw exception in normal case.
+            auto dict_encoded = low_cardinality_column->getMinimalDictionaryEncodedColumn(0, low_cardinality_column->size());
+            lc_arg.column = dict_encoded.dictionary;
+            indexes = dict_encoded.indexes;
+        }
+
+        /// The new column will have a different number of rows, normally less but occasionally it might be more (NULL)
+        input_rows_count = lc_arg.column->size();
+        lc_arg.type = low_cardinality_type->getDictionaryType();
     }
 
     /// Change size of constants.
@@ -94,7 +129,7 @@ ColumnPtr replaceLowCardinalityColumnsByNestedAndGetDictionaryIndexes(
     {
         if (const auto * column_const = checkAndGetColumn<ColumnConst>(column.column.get()))
         {
-            column.column = ColumnConst::create(recursiveRemoveLowCardinality(column_const->getDataColumnPtr()), num_rows);
+            column.column = ColumnConst::create(recursiveRemoveLowCardinality(column_const->getDataColumnPtr()), input_rows_count);
             column.type = recursiveRemoveLowCardinality(column.type);
         }
     }

From 852318449f2ab8e0bfaeca9904b90936d751d177 Mon Sep 17 00:00:00 2001
From: Nikita Taranov <nikita.taranov@clickhouse.com>
Date: Sat, 15 Jun 2024 20:17:43 +0200
Subject: [PATCH 0066/1488] Check perf tests

---
 src/Common/PODArray_fwd.h | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/Common/PODArray_fwd.h b/src/Common/PODArray_fwd.h
index d570a90e467..bd780eb51b0 100644
--- a/src/Common/PODArray_fwd.h
+++ b/src/Common/PODArray_fwd.h
@@ -17,12 +17,12 @@ constexpr size_t integerRoundUp(size_t value, size_t dividend)
 }
 
 template <typename T, size_t initial_bytes = 4096,
-          typename TAllocator = Allocator<false>, size_t pad_right_ = 0,
+          typename TAllocator = Allocator<false, true>, size_t pad_right_ = 0,
           size_t pad_left_ = 0>
 class PODArray;
 
 /** For columns. Padding is enough to read and write xmm-register at the address of the last element. */
-template <typename T, size_t initial_bytes = 4096, typename TAllocator = Allocator<false>>
+template <typename T, size_t initial_bytes = 4096, typename TAllocator = Allocator<false, true>>
 using PaddedPODArray = PODArray<T, initial_bytes, TAllocator, PADDING_FOR_SIMD - 1, PADDING_FOR_SIMD>;
 
 /** A helper for declaring PODArray that uses inline memory.
@@ -32,6 +32,6 @@ using PaddedPODArray = PODArray<T, initial_bytes, TAllocator, PADDING_FOR_SIMD -
 template <typename T, size_t inline_bytes,
           size_t rounded_bytes = integerRoundUp(inline_bytes, sizeof(T))>
 using PODArrayWithStackMemory = PODArray<T, rounded_bytes,
-    AllocatorWithStackMemory<Allocator<false>, rounded_bytes, alignof(T)>>;
+    AllocatorWithStackMemory<Allocator<false, true>, rounded_bytes, alignof(T)>>;
 
 }

From 7568de2202a7fa99539d8d46092315bf3c7fe5e6 Mon Sep 17 00:00:00 2001
From: Nikita Taranov <nikita.taranov@clickhouse.com>
Date: Sat, 15 Jun 2024 22:03:53 +0100
Subject: [PATCH 0067/1488] Revert "Check perf tests"

This reverts commit 852318449f2ab8e0bfaeca9904b90936d751d177.
---
 src/Common/PODArray_fwd.h | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/Common/PODArray_fwd.h b/src/Common/PODArray_fwd.h
index bd780eb51b0..d570a90e467 100644
--- a/src/Common/PODArray_fwd.h
+++ b/src/Common/PODArray_fwd.h
@@ -17,12 +17,12 @@ constexpr size_t integerRoundUp(size_t value, size_t dividend)
 }
 
 template <typename T, size_t initial_bytes = 4096,
-          typename TAllocator = Allocator<false, true>, size_t pad_right_ = 0,
+          typename TAllocator = Allocator<false>, size_t pad_right_ = 0,
           size_t pad_left_ = 0>
 class PODArray;
 
 /** For columns. Padding is enough to read and write xmm-register at the address of the last element. */
-template <typename T, size_t initial_bytes = 4096, typename TAllocator = Allocator<false, true>>
+template <typename T, size_t initial_bytes = 4096, typename TAllocator = Allocator<false>>
 using PaddedPODArray = PODArray<T, initial_bytes, TAllocator, PADDING_FOR_SIMD - 1, PADDING_FOR_SIMD>;
 
 /** A helper for declaring PODArray that uses inline memory.
@@ -32,6 +32,6 @@ using PaddedPODArray = PODArray<T, initial_bytes, TAllocator, PADDING_FOR_SIMD -
 template <typename T, size_t inline_bytes,
           size_t rounded_bytes = integerRoundUp(inline_bytes, sizeof(T))>
 using PODArrayWithStackMemory = PODArray<T, rounded_bytes,
-    AllocatorWithStackMemory<Allocator<false, true>, rounded_bytes, alignof(T)>>;
+    AllocatorWithStackMemory<Allocator<false>, rounded_bytes, alignof(T)>>;
 
 }

From 57d036e5899b52fd4fdab9447630e01bf3d5382b Mon Sep 17 00:00:00 2001
From: Nikita Taranov <nikita.taranov@clickhouse.com>
Date: Sat, 15 Jun 2024 22:04:24 +0100
Subject: [PATCH 0068/1488] impl

---
 tests/performance/scripts/perf.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/performance/scripts/perf.py b/tests/performance/scripts/perf.py
index e98c158249a..94f145d82db 100755
--- a/tests/performance/scripts/perf.py
+++ b/tests/performance/scripts/perf.py
@@ -427,6 +427,8 @@ for query_index in queries_to_run:
 
         for conn_index, c in enumerate(this_query_connections):
             try:
+                c.execute("SYSTEM JEMALLOC PURGE")
+
                 res = c.execute(
                     q,
                     query_id=run_id,

From 38537a00aa9e7185b69c066cd6809c54487ecf4e Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Tue, 18 Jun 2024 18:16:12 +0000
Subject: [PATCH 0069/1488] Make ActionsDAGPtr unique_ptr.

---
 src/Core/InterpolateDescription.cpp           |  2 +-
 src/Core/InterpolateDescription.h             |  2 +-
 src/Functions/indexHint.h                     |  4 +-
 src/Interpreters/ActionsDAG.cpp               | 46 ++++++++-------
 src/Interpreters/ActionsDAG.h                 | 11 ++--
 src/Interpreters/ActionsVisitor.cpp           |  4 +-
 src/Interpreters/ActionsVisitor.h             |  2 +-
 src/Interpreters/ExpressionActions.cpp        | 20 ++++++-
 src/Interpreters/ExpressionActions.h          |  6 +-
 src/Interpreters/ExpressionAnalyzer.cpp       | 27 ++++-----
 src/Interpreters/ExpressionAnalyzer.h         |  2 +-
 src/Interpreters/InterpreterInsertQuery.cpp   |  4 +-
 src/Interpreters/InterpreterSelectQuery.cpp   | 14 ++---
 src/Interpreters/MutationsInterpreter.cpp     |  2 +-
 src/Interpreters/TableJoin.cpp                |  4 +-
 src/Interpreters/WindowDescription.h          |  6 +-
 src/Interpreters/addMissingDefaults.cpp       |  2 +-
 src/Interpreters/addMissingDefaults.h         |  2 +-
 src/Interpreters/inplaceBlockConversions.cpp  |  2 +-
 src/Interpreters/inplaceBlockConversions.h    |  2 +-
 src/Planner/CollectTableExpressionData.cpp    |  6 +-
 src/Planner/Planner.cpp                       | 49 ++++++++--------
 src/Planner/PlannerActionsVisitor.cpp         |  6 +-
 src/Planner/PlannerJoinTree.cpp               | 58 ++++++++++---------
 src/Planner/PlannerJoinTree.h                 |  2 +-
 src/Planner/PlannerJoins.cpp                  | 12 ++--
 src/Planner/PlannerWindowFunctions.cpp        | 13 ++++-
 src/Planner/PlannerWindowFunctions.h          |  2 +-
 src/Planner/Utils.cpp                         |  2 +-
 src/Processors/QueryPlan/AggregatingStep.cpp  |  4 +-
 src/Processors/QueryPlan/CubeStep.cpp         |  4 +-
 src/Processors/QueryPlan/ExpressionStep.cpp   | 12 ++--
 src/Processors/QueryPlan/ExpressionStep.h     |  2 +-
 src/Processors/QueryPlan/FilterStep.cpp       | 11 ++--
 src/Processors/QueryPlan/FilterStep.h         |  2 +-
 .../Optimizations/distinctReadInOrder.cpp     | 12 ++--
 .../Optimizations/filterPushDown.cpp          |  2 +-
 .../Optimizations/optimizePrewhere.cpp        |  2 +-
 .../optimizePrimaryKeyCondition.cpp           |  6 +-
 .../Optimizations/optimizeReadInOrder.cpp     |  4 +-
 .../optimizeUseAggregateProjection.cpp        |  4 +-
 .../optimizeUseNormalProjection.cpp           |  2 +-
 .../Optimizations/projectionsCommon.cpp       |  4 +-
 .../Optimizations/projectionsCommon.h         |  2 +-
 .../Optimizations/removeRedundantDistinct.cpp | 23 ++++----
 .../QueryPlan/ReadFromMergeTree.cpp           | 38 ++++++------
 src/Processors/QueryPlan/ReadFromMergeTree.h  |  2 +-
 .../QueryPlan/ReadFromSystemNumbersStep.cpp   |  2 +-
 .../QueryPlan/SourceStepWithFilter.cpp        |  8 +--
 .../QueryPlan/SourceStepWithFilter.h          |  3 +-
 src/Processors/QueryPlan/TotalsHavingStep.cpp |  8 +--
 src/Processors/QueryPlan/TotalsHavingStep.h   |  2 +-
 src/Processors/QueryPlan/WindowStep.h         |  2 +-
 src/Processors/SourceWithKeyCondition.h       |  4 +-
 .../Transforms/FillingTransform.cpp           |  2 +-
 src/Storages/Hive/StorageHive.cpp             |  8 +--
 src/Storages/KeyDescription.cpp               |  2 +-
 src/Storages/MergeTree/KeyCondition.cpp       |  4 +-
 src/Storages/MergeTree/KeyCondition.h         |  2 +-
 src/Storages/MergeTree/MergeTreeData.cpp      |  8 +--
 src/Storages/MergeTree/MergeTreeData.h        |  2 +-
 .../MergeTree/MergeTreeDataSelectExecutor.cpp |  8 +--
 .../MergeTree/MergeTreeDataSelectExecutor.h   |  4 +-
 .../MergeTree/MergeTreeIndexAnnoy.cpp         |  2 +-
 src/Storages/MergeTree/MergeTreeIndexAnnoy.h  |  2 +-
 .../MergeTree/MergeTreeIndexBloomFilter.cpp   |  4 +-
 .../MergeTree/MergeTreeIndexBloomFilter.h     |  4 +-
 .../MergeTreeIndexBloomFilterText.cpp         |  4 +-
 .../MergeTree/MergeTreeIndexBloomFilterText.h |  4 +-
 .../MergeTree/MergeTreeIndexFullText.cpp      |  4 +-
 .../MergeTree/MergeTreeIndexFullText.h        |  4 +-
 .../MergeTree/MergeTreeIndexHypothesis.cpp    |  2 +-
 .../MergeTree/MergeTreeIndexHypothesis.h      |  2 +-
 .../MergeTree/MergeTreeIndexMinMax.cpp        |  6 +-
 src/Storages/MergeTree/MergeTreeIndexMinMax.h |  4 +-
 src/Storages/MergeTree/MergeTreeIndexSet.cpp  |  6 +-
 src/Storages/MergeTree/MergeTreeIndexSet.h    |  4 +-
 .../MergeTree/MergeTreeIndexUSearch.cpp       |  2 +-
 .../MergeTree/MergeTreeIndexUSearch.h         |  2 +-
 src/Storages/MergeTree/MergeTreeIndices.h     |  2 +-
 .../MergeTree/MergeTreeSelectProcessor.cpp    |  4 +-
 .../MergeTree/MergeTreeSequentialSource.cpp   |  4 +-
 .../MergeTreeSplitPrewhereIntoReadSteps.cpp   | 32 +++++-----
 src/Storages/MergeTree/PartitionPruner.cpp    |  2 +-
 src/Storages/MergeTree/PartitionPruner.h      |  2 +-
 .../StorageObjectStorageSource.cpp            |  2 +-
 src/Storages/SelectQueryInfo.h                |  4 +-
 src/Storages/StorageBuffer.cpp                |  4 +-
 src/Storages/StorageDistributed.cpp           |  2 +-
 src/Storages/StorageFile.cpp                  |  2 +-
 src/Storages/StorageMerge.cpp                 | 10 ++--
 src/Storages/StorageURL.h                     |  2 +-
 src/Storages/StorageValues.cpp                |  4 +-
 src/Storages/StorageView.cpp                  |  2 +-
 src/Storages/TTLDescription.cpp               |  2 +-
 src/Storages/VirtualColumnUtils.cpp           |  4 +-
 src/Storages/VirtualColumnUtils.h             |  2 +-
 src/Storages/WindowView/StorageWindowView.cpp | 12 ++--
 98 files changed, 355 insertions(+), 318 deletions(-)

diff --git a/src/Core/InterpolateDescription.cpp b/src/Core/InterpolateDescription.cpp
index d828c2e85e9..76bbefdcfd7 100644
--- a/src/Core/InterpolateDescription.cpp
+++ b/src/Core/InterpolateDescription.cpp
@@ -14,7 +14,7 @@
 namespace DB
 {
     InterpolateDescription::InterpolateDescription(ActionsDAGPtr actions_, const Aliases & aliases)
-        : actions(actions_)
+        : actions(std::move(actions_))
     {
         for (const auto & name_type : actions->getRequiredColumns())
         {
diff --git a/src/Core/InterpolateDescription.h b/src/Core/InterpolateDescription.h
index 62d7120508b..73579aebee4 100644
--- a/src/Core/InterpolateDescription.h
+++ b/src/Core/InterpolateDescription.h
@@ -11,7 +11,7 @@ namespace DB
 {
 
 class ActionsDAG;
-using ActionsDAGPtr = std::shared_ptr<ActionsDAG>;
+using ActionsDAGPtr = std::unique_ptr<ActionsDAG>;
 using Aliases = std::unordered_map<String, ASTPtr>;
 
 /// Interpolate description
diff --git a/src/Functions/indexHint.h b/src/Functions/indexHint.h
index 3b71c7a5585..8fd7b751760 100644
--- a/src/Functions/indexHint.h
+++ b/src/Functions/indexHint.h
@@ -2,14 +2,12 @@
 #include <Functions/IFunction.h>
 #include <Functions/FunctionFactory.h>
 #include <DataTypes/DataTypesNumber.h>
+#include <Interpreters/ActionsDAG.h>
 
 
 namespace DB
 {
 
-class ActionsDAG;
-using ActionsDAGPtr = std::shared_ptr<ActionsDAG>;
-
 /** The `indexHint` function takes any number of any arguments and always returns one.
   *
   * This function has a special meaning (see ExpressionAnalyzer, KeyCondition)
diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp
index 34f3e0a98bd..23e1e5ee152 100644
--- a/src/Interpreters/ActionsDAG.cpp
+++ b/src/Interpreters/ActionsDAG.cpp
@@ -626,7 +626,7 @@ void ActionsDAG::removeAliasesForFilter(const std::string & filter_name)
 
 ActionsDAGPtr ActionsDAG::cloneSubDAG(const NodeRawConstPtrs & outputs, bool remove_aliases)
 {
-    auto actions = std::make_shared<ActionsDAG>();
+    auto actions = std::make_unique<ActionsDAG>();
     std::unordered_map<const Node *, Node *> copy_map;
 
     struct Frame
@@ -1248,25 +1248,29 @@ bool ActionsDAG::removeUnusedResult(const std::string & column_name)
 
 ActionsDAGPtr ActionsDAG::clone() const
 {
-    auto actions = std::make_shared<ActionsDAG>();
+    std::unordered_map<const Node *, Node *> old_to_new_nodes;
+    return clone(old_to_new_nodes);
+}
 
-    std::unordered_map<const Node *, Node *> copy_map;
+ActionsDAGPtr ActionsDAG::clone(std::unordered_map<const Node *, Node *> & old_to_new_nodes) const
+{
+    auto actions = std::make_unique<ActionsDAG>();
 
     for (const auto & node : nodes)
     {
         auto & copy_node = actions->nodes.emplace_back(node);
-        copy_map[&node] = &copy_node;
+        old_to_new_nodes[&node] = &copy_node;
     }
 
     for (auto & node : actions->nodes)
         for (auto & child : node.children)
-            child = copy_map[child];
+            child = old_to_new_nodes[child];
 
     for (const auto & output_node : outputs)
-        actions->outputs.push_back(copy_map[output_node]);
+        actions->outputs.push_back(old_to_new_nodes[output_node]);
 
     for (const auto & input_node : inputs)
-        actions->inputs.push_back(copy_map[input_node]);
+        actions->inputs.push_back(old_to_new_nodes[input_node]);
 
     return actions;
 }
@@ -1421,7 +1425,7 @@ ActionsDAGPtr ActionsDAG::makeConvertingActions(
     if (add_casted_columns && mode != MatchColumnsMode::Name)
         throw Exception(ErrorCodes::LOGICAL_ERROR, "Converting with add_casted_columns supported only for MatchColumnsMode::Name");
 
-    auto actions_dag = std::make_shared<ActionsDAG>(source);
+    auto actions_dag = std::make_unique<ActionsDAG>(source);
     NodeRawConstPtrs projection(num_result_columns);
 
     FunctionOverloadResolverPtr func_builder_materialize = std::make_unique<FunctionToOverloadResolverAdaptor>(std::make_shared<FunctionMaterialize>());
@@ -1549,7 +1553,7 @@ ActionsDAGPtr ActionsDAG::makeConvertingActions(
 
 ActionsDAGPtr ActionsDAG::makeAddingColumnActions(ColumnWithTypeAndName column)
 {
-    auto adding_column_action = std::make_shared<ActionsDAG>();
+    auto adding_column_action = std::make_unique<ActionsDAG>();
     FunctionOverloadResolverPtr func_builder_materialize
         = std::make_unique<FunctionToOverloadResolverAdaptor>(std::make_shared<FunctionMaterialize>());
 
@@ -1570,7 +1574,7 @@ ActionsDAGPtr ActionsDAG::merge(ActionsDAG && first, ActionsDAG && second)
     /// Some actions could become unused. Do not drop inputs to preserve the header.
     first.removeUnusedActions(false);
 
-    return std::make_shared<ActionsDAG>(std::move(first));
+    return std::make_unique<ActionsDAG>(std::move(first));
 }
 
 void ActionsDAG::mergeInplace(ActionsDAG && second)
@@ -1963,12 +1967,12 @@ ActionsDAG::SplitResult ActionsDAG::split(std::unordered_set<const Node *> split
             second_inputs.push_back(cur.to_second);
     }
 
-    auto first_actions = std::make_shared<ActionsDAG>();
+    auto first_actions = std::make_unique<ActionsDAG>();
     first_actions->nodes.swap(first_nodes);
     first_actions->outputs.swap(first_outputs);
     first_actions->inputs.swap(first_inputs);
 
-    auto second_actions = std::make_shared<ActionsDAG>();
+    auto second_actions = std::make_unique<ActionsDAG>();
     second_actions->nodes.swap(second_nodes);
     second_actions->outputs.swap(second_outputs);
     second_actions->inputs.swap(second_inputs);
@@ -2302,7 +2306,7 @@ ActionsDAGPtr ActionsDAG::createActionsForConjunction(NodeRawConstPtrs conjuncti
     if (conjunction.empty())
         return nullptr;
 
-    auto actions = std::make_shared<ActionsDAG>();
+    auto actions = std::make_unique<ActionsDAG>();
 
     FunctionOverloadResolverPtr func_builder_and = std::make_unique<FunctionToOverloadResolverAdaptor>(std::make_shared<FunctionAnd>());
 
@@ -2866,7 +2870,7 @@ ActionsDAGPtr ActionsDAG::buildFilterActionsDAG(
         bool visited_children = false;
     };
 
-    auto result_dag = std::make_shared<ActionsDAG>();
+    auto result_dag = std::make_unique<ActionsDAG>();
     std::unordered_map<std::string, const ActionsDAG::Node *> result_inputs;
     std::unordered_map<const ActionsDAG::Node *, const ActionsDAG::Node *> node_to_result_node;
 
@@ -2964,7 +2968,7 @@ ActionsDAGPtr ActionsDAG::buildFilterActionsDAG(
                             const auto & index_hint_args = index_hint->getActions()->getOutputs();
 
                             if (index_hint_args.empty())
-                                index_hint_filter_dag = std::make_shared<ActionsDAG>();
+                                index_hint_filter_dag = std::make_unique<ActionsDAG>();
                             else
                                 index_hint_filter_dag = buildFilterActionsDAG(index_hint_args,
                                     node_name_to_input_node_column,
@@ -3108,10 +3112,10 @@ ActionsDAG::NodeRawConstPtrs ActionsDAG::filterNodesByAllowedInputs(
     return nodes;
 }
 
-FindOriginalNodeForOutputName::FindOriginalNodeForOutputName(const ActionsDAGPtr & actions_)
-    :actions(actions_)
+FindOriginalNodeForOutputName::FindOriginalNodeForOutputName(const ActionsDAG & actions_)
+    //: actions(actions_)
 {
-    const auto & actions_outputs = actions->getOutputs();
+    const auto & actions_outputs = actions_.getOutputs();
     for (const auto * output_node : actions_outputs)
     {
         /// find input node which refers to the output node
@@ -3147,10 +3151,10 @@ const ActionsDAG::Node * FindOriginalNodeForOutputName::find(const String & outp
     return it->second;
 }
 
-FindAliasForInputName::FindAliasForInputName(const ActionsDAGPtr & actions_)
-    :actions(actions_)
+FindAliasForInputName::FindAliasForInputName(const ActionsDAG & actions_)
+    //: actions(actions_)
 {
-    const auto & actions_outputs = actions->getOutputs();
+    const auto & actions_outputs = actions_.getOutputs();
     for (const auto * output_node : actions_outputs)
     {
         /// find input node which corresponds to alias
diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h
index c9974fd849c..4a840885b6a 100644
--- a/src/Interpreters/ActionsDAG.h
+++ b/src/Interpreters/ActionsDAG.h
@@ -12,7 +12,7 @@ namespace DB
 {
 
 class ActionsDAG;
-using ActionsDAGPtr = std::shared_ptr<ActionsDAG>;
+using ActionsDAGPtr = std::unique_ptr<ActionsDAG>;
 
 class IExecutableFunction;
 using ExecutableFunctionPtr = std::shared_ptr<IExecutableFunction>;
@@ -262,6 +262,7 @@ public:
 #endif
 
     ActionsDAGPtr clone() const;
+    ActionsDAGPtr clone(std::unordered_map<const Node *, Node *> & old_to_new_nodes) const;
 
     static ActionsDAGPtr cloneSubDAG(const NodeRawConstPtrs & outputs, bool remove_aliases);
 
@@ -480,11 +481,11 @@ class FindOriginalNodeForOutputName
     using NameToNodeIndex = std::unordered_map<std::string_view, const ActionsDAG::Node *>;
 
 public:
-    explicit FindOriginalNodeForOutputName(const ActionsDAGPtr & actions);
+    explicit FindOriginalNodeForOutputName(const ActionsDAG & actions);
     const ActionsDAG::Node * find(const String & output_name);
 
 private:
-    ActionsDAGPtr actions;
+    //const ActionsDAG & actions;
     NameToNodeIndex index;
 };
 
@@ -493,11 +494,11 @@ class FindAliasForInputName
     using NameToNodeIndex = std::unordered_map<std::string_view, const ActionsDAG::Node *>;
 
 public:
-    explicit FindAliasForInputName(const ActionsDAGPtr & actions);
+    explicit FindAliasForInputName(const ActionsDAG & actions);
     const ActionsDAG::Node * find(const String & name);
 
 private:
-    ActionsDAGPtr actions;
+    //const ActionsDAG & actions;
     NameToNodeIndex index;
 };
 
diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp
index 9e56d740e5e..1838a7b04b9 100644
--- a/src/Interpreters/ActionsVisitor.cpp
+++ b/src/Interpreters/ActionsVisitor.cpp
@@ -1009,7 +1009,7 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data &
         dag.project(args);
 
         auto index_hint = std::make_shared<FunctionIndexHint>();
-        index_hint->setActions(std::make_shared<ActionsDAG>(std::move(dag)));
+        index_hint->setActions(std::make_unique<ActionsDAG>(std::move(dag)));
 
         // Arguments are removed. We add function instead of constant column to avoid constant folding.
         data.addFunction(std::make_unique<FunctionToOverloadResolverAdaptor>(index_hint), {}, column_name);
@@ -1272,7 +1272,7 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data &
                     lambda_dag.removeUnusedActions(Names(1, result_name));
 
                     auto lambda_actions = std::make_shared<ExpressionActions>(
-                        std::make_shared<ActionsDAG>(std::move(lambda_dag)),
+                        std::make_unique<ActionsDAG>(std::move(lambda_dag)),
                         ExpressionActionsSettings::fromContext(data.getContext(), CompileExpressions::yes));
 
                     DataTypePtr result_type = lambda_actions->getSampleBlock().getByName(result_name).type;
diff --git a/src/Interpreters/ActionsVisitor.h b/src/Interpreters/ActionsVisitor.h
index 46d2d60e461..496d9b9b587 100644
--- a/src/Interpreters/ActionsVisitor.h
+++ b/src/Interpreters/ActionsVisitor.h
@@ -22,7 +22,7 @@ class ExpressionActions;
 using ExpressionActionsPtr = std::shared_ptr<ExpressionActions>;
 
 class ActionsDAG;
-using ActionsDAGPtr = std::shared_ptr<ActionsDAG>;
+using ActionsDAGPtr = std::unique_ptr<ActionsDAG>;
 
 class IFunctionOverloadResolver;
 using FunctionOverloadResolverPtr = std::shared_ptr<IFunctionOverloadResolver>;
diff --git a/src/Interpreters/ExpressionActions.cpp b/src/Interpreters/ExpressionActions.cpp
index 7f96c927d82..7cbf5afd763 100644
--- a/src/Interpreters/ExpressionActions.cpp
+++ b/src/Interpreters/ExpressionActions.cpp
@@ -73,7 +73,25 @@ ExpressionActions::ExpressionActions(ActionsDAGPtr actions_dag_, const Expressio
 
 ExpressionActionsPtr ExpressionActions::clone() const
 {
-    return std::make_shared<ExpressionActions>(*this);
+    auto copy = std::make_shared<ExpressionActions>(ExpressionActions());
+
+    std::unordered_map<const Node *, Node *> copy_map;
+    copy->actions_dag = actions_dag->clone(copy_map);
+    copy->actions = actions;
+    for (auto & action : copy->actions)
+        action.node = copy_map[action.node];
+
+    copy->num_columns = num_columns;
+
+    copy->required_columns = required_columns;
+    copy->input_positions = input_positions;
+    copy->result_positions = result_positions;
+    copy->sample_block = sample_block;
+
+    copy->project_inputs = project_inputs;
+    copy->settings = settings;
+
+    return copy;
 }
 
 namespace
diff --git a/src/Interpreters/ExpressionActions.h b/src/Interpreters/ExpressionActions.h
index ddffe022215..63ea989bd5e 100644
--- a/src/Interpreters/ExpressionActions.h
+++ b/src/Interpreters/ExpressionActions.h
@@ -84,10 +84,9 @@ private:
     ExpressionActionsSettings settings;
 
 public:
-    ExpressionActions() = delete;
     explicit ExpressionActions(ActionsDAGPtr actions_dag_, const ExpressionActionsSettings & settings_ = {}, bool project_inputs_ = false);
-    ExpressionActions(const ExpressionActions &) = default;
-    ExpressionActions & operator=(const ExpressionActions &) = default;
+    ExpressionActions(ExpressionActions &&) = default;
+    ExpressionActions & operator=(ExpressionActions &&) = default;
 
     const Actions & getActions() const { return actions; }
     const std::list<Node> & getNodes() const { return actions_dag->getNodes(); }
@@ -131,6 +130,7 @@ public:
     ExpressionActionsPtr clone() const;
 
 private:
+    ExpressionActions() = default;
     void checkLimits(const ColumnsWithTypeAndName & columns) const;
 
     void linearizeActions(const std::unordered_set<const Node *> & lazy_executed_nodes);
diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp
index 62cddd9caf7..be00e37c751 100644
--- a/src/Interpreters/ExpressionAnalyzer.cpp
+++ b/src/Interpreters/ExpressionAnalyzer.cpp
@@ -658,7 +658,7 @@ void ExpressionAnalyzer::makeWindowDescriptionFromAST(const Context & context_,
                     with_alias->getColumnName(), 1 /* direction */,
                     1 /* nulls_direction */));
 
-            auto actions_dag = std::make_shared<ActionsDAG>(aggregated_columns);
+            auto actions_dag = std::make_unique<ActionsDAG>(aggregated_columns);
             getRootActions(column_ast, false, *actions_dag);
             desc.partition_by_actions.push_back(std::move(actions_dag));
         }
@@ -679,7 +679,7 @@ void ExpressionAnalyzer::makeWindowDescriptionFromAST(const Context & context_,
                     order_by_element.direction,
                     order_by_element.nulls_direction));
 
-            auto actions_dag = std::make_shared<ActionsDAG>(aggregated_columns);
+            auto actions_dag = std::make_unique<ActionsDAG>(aggregated_columns);
             getRootActions(column_ast, false, *actions_dag);
             desc.order_by_actions.push_back(std::move(actions_dag));
         }
@@ -823,13 +823,14 @@ void ExpressionAnalyzer::makeWindowDescriptions(ActionsDAG & actions)
             makeWindowDescriptionFromAST(*current_context, window_descriptions,
                 desc, &definition);
 
+            auto full_sort_description = desc.full_sort_description;
+
             auto [it, inserted] = window_descriptions.insert(
-                {default_window_name, desc});
+                {default_window_name, std::move(desc)});
 
             if (!inserted)
             {
-                assert(it->second.full_sort_description
-                    == desc.full_sort_description);
+                assert(it->second.full_sort_description == full_sort_description);
             }
 
             it->second.window_functions.push_back(window_function);
@@ -1353,10 +1354,10 @@ bool SelectQueryExpressionAnalyzer::appendGroupBy(ExpressionActionsChain & chain
     {
         for (auto & child : asts)
         {
-            auto actions_dag = std::make_shared<ActionsDAG>(columns_after_join);
+            auto actions_dag = std::make_unique<ActionsDAG>(columns_after_join);
             getRootActions(child, only_types, *actions_dag);
             group_by_elements_actions.emplace_back(
-                std::make_shared<ExpressionActions>(actions_dag, ExpressionActionsSettings::fromContext(getContext(), CompileExpressions::yes)));
+                std::make_shared<ExpressionActions>(std::move(actions_dag), ExpressionActionsSettings::fromContext(getContext(), CompileExpressions::yes)));
         }
     }
 
@@ -1606,10 +1607,10 @@ ActionsAndProjectInputsFlagPtr SelectQueryExpressionAnalyzer::appendOrderBy(Expr
     {
         for (const auto & child : select_query->orderBy()->children)
         {
-            auto actions_dag = std::make_shared<ActionsDAG>(columns_after_join);
+            auto actions_dag = std::make_unique<ActionsDAG>(columns_after_join);
             getRootActions(child, only_types, *actions_dag);
             order_by_elements_actions.emplace_back(
-                std::make_shared<ExpressionActions>(actions_dag, ExpressionActionsSettings::fromContext(getContext(), CompileExpressions::yes)));
+                std::make_shared<ExpressionActions>(std::move(actions_dag), ExpressionActionsSettings::fromContext(getContext(), CompileExpressions::yes)));
         }
     }
 
@@ -1799,7 +1800,7 @@ ExpressionActionsPtr ExpressionAnalyzer::getActions(bool add_aliases, bool remov
 
 ActionsDAGPtr ExpressionAnalyzer::getConstActionsDAG(const ColumnsWithTypeAndName & constant_inputs)
 {
-    auto actions = std::make_shared<ActionsDAG>(constant_inputs);
+    auto actions = std::make_unique<ActionsDAG>(constant_inputs);
     getRootActions(query, true /* no_makeset_for_subqueries */, *actions, true /* only_consts */);
     return actions;
 }
@@ -1807,7 +1808,7 @@ ActionsDAGPtr ExpressionAnalyzer::getConstActionsDAG(const ColumnsWithTypeAndNam
 ExpressionActionsPtr ExpressionAnalyzer::getConstActions(const ColumnsWithTypeAndName & constant_inputs)
 {
     auto actions = getConstActionsDAG(constant_inputs);
-    return std::make_shared<ExpressionActions>(actions, ExpressionActionsSettings::fromContext(getContext()));
+    return std::make_shared<ExpressionActions>(std::move(actions), ExpressionActionsSettings::fromContext(getContext()));
 }
 
 std::unique_ptr<QueryPlan> SelectQueryExpressionAnalyzer::getJoinedPlan()
@@ -1878,7 +1879,7 @@ ExpressionAnalysisResult::ExpressionAnalysisResult(
 
         if (prewhere_dag_and_flags)
         {
-            auto dag = std::make_shared<ActionsDAG>(std::move(prewhere_dag_and_flags->dag));
+            auto dag = std::make_unique<ActionsDAG>(std::move(prewhere_dag_and_flags->dag));
             prewhere_info = std::make_shared<PrewhereInfo>(std::move(dag), query.prewhere()->getColumnName());
             prewhere_dag_and_flags.reset();
         }
@@ -1945,7 +1946,7 @@ ExpressionAnalysisResult::ExpressionAnalysisResult(
                 {
                     auto dag = prewhere_dag_and_flags->dag.clone();
                     ExpressionActions(
-                        dag,
+                        std::move(dag),
                         ExpressionActionsSettings::fromSettings(context->getSettingsRef())).execute(before_prewhere_sample);
                     auto & column_elem = before_prewhere_sample.getByName(query.prewhere()->getColumnName());
                     /// If the filter column is a constant, record it.
diff --git a/src/Interpreters/ExpressionAnalyzer.h b/src/Interpreters/ExpressionAnalyzer.h
index 12d6dce8f72..e44a5891e77 100644
--- a/src/Interpreters/ExpressionAnalyzer.h
+++ b/src/Interpreters/ExpressionAnalyzer.h
@@ -39,7 +39,7 @@ class ArrayJoinAction;
 using ArrayJoinActionPtr = std::shared_ptr<ArrayJoinAction>;
 
 class ActionsDAG;
-using ActionsDAGPtr = std::shared_ptr<ActionsDAG>;
+using ActionsDAGPtr = std::unique_ptr<ActionsDAG>;
 
 /// Create columns in block or return false if not possible
 bool sanitizeBlock(Block & block, bool throw_if_cannot_create_column = false);
diff --git a/src/Interpreters/InterpreterInsertQuery.cpp b/src/Interpreters/InterpreterInsertQuery.cpp
index 927bafe4bfb..4dbdebd0d06 100644
--- a/src/Interpreters/InterpreterInsertQuery.cpp
+++ b/src/Interpreters/InterpreterInsertQuery.cpp
@@ -384,7 +384,7 @@ Chain InterpreterInsertQuery::buildPreSinkChain(
         context_ptr,
         null_as_default);
 
-    auto adding_missing_defaults_actions = std::make_shared<ExpressionActions>(adding_missing_defaults_dag);
+    auto adding_missing_defaults_actions = std::make_shared<ExpressionActions>(std::move(adding_missing_defaults_dag));
 
     /// Actually we don't know structure of input blocks from query/table,
     /// because some clients break insertion protocol (columns != header)
@@ -597,7 +597,7 @@ BlockIO InterpreterInsertQuery::execute()
                 pipeline.getHeader().getColumnsWithTypeAndName(),
                 header.getColumnsWithTypeAndName(),
                 ActionsDAG::MatchColumnsMode::Position);
-        auto actions = std::make_shared<ExpressionActions>(actions_dag, ExpressionActionsSettings::fromContext(getContext(), CompileExpressions::yes));
+        auto actions = std::make_shared<ExpressionActions>(std::move(actions_dag), ExpressionActionsSettings::fromContext(getContext(), CompileExpressions::yes));
 
         pipeline.addSimpleTransform([&](const Block & in_header) -> ProcessorPtr
         {
diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp
index f5b54ec64cb..64a17a7ba87 100644
--- a/src/Interpreters/InterpreterSelectQuery.cpp
+++ b/src/Interpreters/InterpreterSelectQuery.cpp
@@ -1302,7 +1302,7 @@ static InterpolateDescriptionPtr getInterpolateDescription(
             result_columns, ActionsDAG::MatchColumnsMode::Position, true);
         ActionsDAGPtr merge_dag = ActionsDAG::merge(std::move(*actions->clone()), std::move(*conv_dag));
 
-        interpolate_descr = std::make_shared<InterpolateDescription>(merge_dag, aliases);
+        interpolate_descr = std::make_shared<InterpolateDescription>(std::move(merge_dag), aliases);
     }
 
     return interpolate_descr;
@@ -2042,7 +2042,7 @@ void InterpreterSelectQuery::addEmptySourceToQueryPlan(QueryPlan & query_plan, c
             pipe.addSimpleTransform([&](const Block & header)
             {
                 return std::make_shared<FilterTransform>(header,
-                    std::make_shared<ExpressionActions>(prewhere_info.row_level_filter),
+                    std::make_shared<ExpressionActions>(prewhere_info.row_level_filter->clone()),
                     prewhere_info.row_level_column_name, true);
             });
         }
@@ -2050,7 +2050,7 @@ void InterpreterSelectQuery::addEmptySourceToQueryPlan(QueryPlan & query_plan, c
         pipe.addSimpleTransform([&](const Block & header)
         {
             return std::make_shared<FilterTransform>(
-                header, std::make_shared<ExpressionActions>(prewhere_info.prewhere_actions),
+                header, std::make_shared<ExpressionActions>(prewhere_info.prewhere_actions->clone()),
                 prewhere_info.prewhere_column_name, prewhere_info.remove_prewhere_column);
         });
     }
@@ -2094,8 +2094,8 @@ void InterpreterSelectQuery::applyFiltersToPrewhereInAnalysis(ExpressionAnalysis
         if (does_storage_support_prewhere && shouldMoveToPrewhere())
         {
             /// Execute row level filter in prewhere as a part of "move to prewhere" optimization.
-            analysis.prewhere_info = std::make_shared<PrewhereInfo>(analysis.filter_info->actions, analysis.filter_info->column_name);
-            analysis.prewhere_info->remove_prewhere_column = analysis.filter_info->do_remove_column;
+            analysis.prewhere_info = std::make_shared<PrewhereInfo>(std::move(analysis.filter_info->actions), analysis.filter_info->column_name);
+            analysis.prewhere_info->remove_prewhere_column = std::move(analysis.filter_info->do_remove_column);
             analysis.prewhere_info->need_filter = true;
             analysis.filter_info = nullptr;
         }
@@ -2103,8 +2103,8 @@ void InterpreterSelectQuery::applyFiltersToPrewhereInAnalysis(ExpressionAnalysis
     else
     {
         /// Add row level security actions to prewhere.
-        analysis.prewhere_info->row_level_filter = analysis.filter_info->actions;
-        analysis.prewhere_info->row_level_column_name = analysis.filter_info->column_name;
+        analysis.prewhere_info->row_level_filter = std::move(analysis.filter_info->actions);
+        analysis.prewhere_info->row_level_column_name = std::move(analysis.filter_info->column_name);
         analysis.filter_info = nullptr;
     }
 }
diff --git a/src/Interpreters/MutationsInterpreter.cpp b/src/Interpreters/MutationsInterpreter.cpp
index 6d3a4f30b34..1bb770bf561 100644
--- a/src/Interpreters/MutationsInterpreter.cpp
+++ b/src/Interpreters/MutationsInterpreter.cpp
@@ -1211,7 +1211,7 @@ void MutationsInterpreter::Source::read(
             MergeTreeSequentialSourceType::Mutation,
             plan, *data, storage_snapshot,
             part, required_columns,
-            apply_deleted_mask_, filter, context_,
+            apply_deleted_mask_, std::move(filter), context_,
             getLogger("MutationsInterpreter"));
     }
     else
diff --git a/src/Interpreters/TableJoin.cpp b/src/Interpreters/TableJoin.cpp
index 6191eb73fd4..baf3a743f40 100644
--- a/src/Interpreters/TableJoin.cpp
+++ b/src/Interpreters/TableJoin.cpp
@@ -470,7 +470,7 @@ static ActionsDAGPtr createWrapWithTupleActions(
     if (column_names_to_wrap.empty())
         return nullptr;
 
-    auto actions_dag = std::make_shared<ActionsDAG>(source_columns);
+    auto actions_dag = std::make_unique<ActionsDAG>(source_columns);
 
     FunctionOverloadResolverPtr func_builder = std::make_unique<FunctionToOverloadResolverAdaptor>(std::make_shared<FunctionTuple>());
 
@@ -616,7 +616,7 @@ TableJoin::createConvertingActions(
         mergeDags(right_dag, std::move(new_right_dag));
     }
 
-    return {left_dag, right_dag};
+    return {std::move(left_dag), std::move(right_dag)};
 }
 
 template <typename LeftNamesAndTypes, typename RightNamesAndTypes>
diff --git a/src/Interpreters/WindowDescription.h b/src/Interpreters/WindowDescription.h
index c26e4517c9a..17bfe619c30 100644
--- a/src/Interpreters/WindowDescription.h
+++ b/src/Interpreters/WindowDescription.h
@@ -14,7 +14,7 @@ namespace DB
 class ASTFunction;
 
 class ActionsDAG;
-using ActionsDAGPtr = std::shared_ptr<ActionsDAG>;
+using ActionsDAGPtr = std::unique_ptr<ActionsDAG>;
 
 struct WindowFunctionDescription
 {
@@ -93,8 +93,8 @@ struct WindowDescription
     // then by ORDER BY. This field holds this combined sort order.
     SortDescription full_sort_description;
 
-    std::vector<ActionsDAGPtr> partition_by_actions;
-    std::vector<ActionsDAGPtr> order_by_actions;
+    std::vector<std::shared_ptr<const ActionsDAG>> partition_by_actions;
+    std::vector<std::shared_ptr<const ActionsDAG>> order_by_actions;
 
     WindowFrame frame;
 
diff --git a/src/Interpreters/addMissingDefaults.cpp b/src/Interpreters/addMissingDefaults.cpp
index fbf17d7efb7..929999c8c37 100644
--- a/src/Interpreters/addMissingDefaults.cpp
+++ b/src/Interpreters/addMissingDefaults.cpp
@@ -21,7 +21,7 @@ ActionsDAGPtr addMissingDefaults(
     ContextPtr context,
     bool null_as_default)
 {
-    auto actions = std::make_shared<ActionsDAG>(header.getColumnsWithTypeAndName());
+    auto actions = std::make_unique<ActionsDAG>(header.getColumnsWithTypeAndName());
     auto & index = actions->getOutputs();
 
     /// For missing columns of nested structure, you need to create not a column of empty arrays, but a column of arrays of correct lengths.
diff --git a/src/Interpreters/addMissingDefaults.h b/src/Interpreters/addMissingDefaults.h
index 0a3d4de478c..94afd806dfd 100644
--- a/src/Interpreters/addMissingDefaults.h
+++ b/src/Interpreters/addMissingDefaults.h
@@ -15,7 +15,7 @@ class NamesAndTypesList;
 class ColumnsDescription;
 
 class ActionsDAG;
-using ActionsDAGPtr = std::shared_ptr<ActionsDAG>;
+using ActionsDAGPtr = std::unique_ptr<ActionsDAG>;
 
 /** Adds three types of columns into block
   * 1. Columns, that are missed inside request, but present in table without defaults (missed columns)
diff --git a/src/Interpreters/inplaceBlockConversions.cpp b/src/Interpreters/inplaceBlockConversions.cpp
index 239cce5b427..b000264ae33 100644
--- a/src/Interpreters/inplaceBlockConversions.cpp
+++ b/src/Interpreters/inplaceBlockConversions.cpp
@@ -163,7 +163,7 @@ ActionsDAGPtr createExpressions(
 
     auto syntax_result = TreeRewriter(context).analyze(expr_list, header.getNamesAndTypesList());
     auto expression_analyzer = ExpressionAnalyzer{expr_list, syntax_result, context};
-    auto dag = std::make_shared<ActionsDAG>(header.getNamesAndTypesList());
+    auto dag = std::make_unique<ActionsDAG>(header.getNamesAndTypesList());
     auto actions = expression_analyzer.getActionsDAG(true, !save_unneeded_columns);
     dag = ActionsDAG::merge(std::move(*dag), std::move(*actions));
 
diff --git a/src/Interpreters/inplaceBlockConversions.h b/src/Interpreters/inplaceBlockConversions.h
index bea44bf6db9..ffc77561e79 100644
--- a/src/Interpreters/inplaceBlockConversions.h
+++ b/src/Interpreters/inplaceBlockConversions.h
@@ -24,7 +24,7 @@ struct StorageInMemoryMetadata;
 using StorageMetadataPtr = std::shared_ptr<const StorageInMemoryMetadata>;
 
 class ActionsDAG;
-using ActionsDAGPtr = std::shared_ptr<ActionsDAG>;
+using ActionsDAGPtr = std::unique_ptr<ActionsDAG>;
 
 /// Create actions which adds missing defaults to block according to required_columns using columns description
 /// or substitute NULL into DEFAULT value in case of INSERT SELECT query (null_as_default) if according setting is 1.
diff --git a/src/Planner/CollectTableExpressionData.cpp b/src/Planner/CollectTableExpressionData.cpp
index d5e39a9f123..162d3fe8d11 100644
--- a/src/Planner/CollectTableExpressionData.cpp
+++ b/src/Planner/CollectTableExpressionData.cpp
@@ -88,7 +88,7 @@ public:
 
                 auto column_identifier = planner_context->getGlobalPlannerContext()->createColumnIdentifier(node);
 
-                ActionsDAGPtr alias_column_actions_dag = std::make_shared<ActionsDAG>();
+                ActionsDAGPtr alias_column_actions_dag = std::make_unique<ActionsDAG>();
                 PlannerActionsVisitor actions_visitor(planner_context, false);
                 auto outputs = actions_visitor.visit(*alias_column_actions_dag, column_node->getExpression());
                 if (outputs.size() != 1)
@@ -97,7 +97,7 @@ public:
                 const auto & column_name = column_node->getColumnName();
                 const auto & alias_node = alias_column_actions_dag->addAlias(*outputs[0], column_name);
                 alias_column_actions_dag->addOrReplaceInOutputs(alias_node);
-                table_expression_data.addAliasColumn(column_node->getColumn(), column_identifier, alias_column_actions_dag, select_added_columns);
+                table_expression_data.addAliasColumn(column_node->getColumn(), column_identifier, std::move(alias_column_actions_dag), select_added_columns);
             }
 
             return;
@@ -335,7 +335,7 @@ void collectTableExpressionData(QueryTreeNodePtr & query_node, PlannerContextPtr
         collect_source_columns_visitor.setKeepAliasColumns(false);
         collect_source_columns_visitor.visit(query_node_typed.getPrewhere());
 
-        auto prewhere_actions_dag = std::make_shared<ActionsDAG>();
+        auto prewhere_actions_dag = std::make_unique<ActionsDAG>();
 
         QueryTreeNodePtr query_tree_node = query_node_typed.getPrewhere();
 
diff --git a/src/Planner/Planner.cpp b/src/Planner/Planner.cpp
index 2d42ed73223..681ae7e6ac4 100644
--- a/src/Planner/Planner.cpp
+++ b/src/Planner/Planner.cpp
@@ -213,7 +213,7 @@ FiltersForTableExpressionMap collectFiltersForAnalysis(const QueryTreeNodePtr &
         if (!read_from_dummy)
             continue;
 
-        auto filter_actions = read_from_dummy->getFilterActionsDAG();
+        auto filter_actions = read_from_dummy->detachFilterActionsDAG();
         const auto & table_node = dummy_storage_to_table.at(&read_from_dummy->getStorage());
         res[table_node] = FiltersForTableExpression{std::move(filter_actions), read_from_dummy->getPrewhereInfo()};
     }
@@ -331,13 +331,13 @@ public:
 void addExpressionStep(QueryPlan & query_plan,
     const ActionsAndProjectInputsFlagPtr & expression_actions,
     const std::string & step_description,
-    std::vector<ActionsDAGPtr> & result_actions_to_execute)
+    std::vector<const ActionsDAG *> & result_actions_to_execute)
 {
     auto actions = expression_actions->dag.clone();
     if (expression_actions->project_input)
         actions->appendInputsForUnusedColumns(query_plan.getCurrentDataStream().header);
 
-    result_actions_to_execute.push_back(actions);
+    result_actions_to_execute.push_back(actions.get());
     auto expression_step = std::make_unique<ExpressionStep>(query_plan.getCurrentDataStream(), actions);
     expression_step->setStepDescription(step_description);
     query_plan.addStep(std::move(expression_step));
@@ -346,13 +346,13 @@ void addExpressionStep(QueryPlan & query_plan,
 void addFilterStep(QueryPlan & query_plan,
     const FilterAnalysisResult & filter_analysis_result,
     const std::string & step_description,
-    std::vector<ActionsDAGPtr> & result_actions_to_execute)
+    std::vector<const ActionsDAG *> & result_actions_to_execute)
 {
     auto actions = filter_analysis_result.filter_actions->dag.clone();
     if (filter_analysis_result.filter_actions->project_input)
         actions->appendInputsForUnusedColumns(query_plan.getCurrentDataStream().header);
 
-    result_actions_to_execute.push_back(actions);
+    result_actions_to_execute.push_back(actions.get());
     auto where_step = std::make_unique<FilterStep>(query_plan.getCurrentDataStream(),
         actions,
         filter_analysis_result.filter_column_name,
@@ -544,7 +544,7 @@ void addTotalsHavingStep(QueryPlan & query_plan,
     const QueryAnalysisResult & query_analysis_result,
     const PlannerContextPtr & planner_context,
     const QueryNode & query_node,
-    std::vector<ActionsDAGPtr> & result_actions_to_execute)
+    std::vector<const ActionsDAG *> & result_actions_to_execute)
 {
     const auto & query_context = planner_context->getQueryContext();
     const auto & settings = query_context->getSettingsRef();
@@ -560,7 +560,7 @@ void addTotalsHavingStep(QueryPlan & query_plan,
         if (having_analysis_result.filter_actions->project_input)
             actions->appendInputsForUnusedColumns(query_plan.getCurrentDataStream().header);
 
-        result_actions_to_execute.push_back(actions);
+        result_actions_to_execute.push_back(actions.get());
     }
 
     auto totals_having_step = std::make_unique<TotalsHavingStep>(
@@ -714,7 +714,7 @@ void addWithFillStepIfNeeded(QueryPlan & query_plan,
 
     if (query_node.hasInterpolate())
     {
-        auto interpolate_actions_dag = std::make_shared<ActionsDAG>();
+        auto interpolate_actions_dag = std::make_unique<ActionsDAG>();
         auto query_plan_columns = query_plan.getCurrentDataStream().header.getColumnsWithTypeAndName();
         for (auto & query_plan_column : query_plan_columns)
         {
@@ -885,7 +885,7 @@ void addPreliminarySortOrDistinctOrLimitStepsIfNeeded(QueryPlan & query_plan,
     const PlannerContextPtr & planner_context,
     const PlannerQueryProcessingInfo & query_processing_info,
     const QueryTreeNodePtr & query_tree,
-    std::vector<ActionsDAGPtr> & result_actions_to_execute)
+    std::vector<const ActionsDAG *> & result_actions_to_execute)
 {
     const auto & query_node = query_tree->as<QueryNode &>();
 
@@ -932,14 +932,14 @@ void addWindowSteps(QueryPlan & query_plan,
     const auto & query_context = planner_context->getQueryContext();
     const auto & settings = query_context->getSettingsRef();
 
-    auto window_descriptions = window_analysis_result.window_descriptions;
-    sortWindowDescriptions(window_descriptions);
+    const auto & window_descriptions = window_analysis_result.window_descriptions;
+    auto perm = sortWindowDescriptions(window_descriptions);
 
     size_t window_descriptions_size = window_descriptions.size();
 
     for (size_t i = 0; i < window_descriptions_size; ++i)
     {
-        const auto & window_description = window_descriptions[i];
+        const auto & window_description = window_descriptions[perm[i]];
 
         /** We don't need to sort again if the input from previous window already
           * has suitable sorting. Also don't create sort steps when there are no
@@ -952,8 +952,9 @@ void addWindowSteps(QueryPlan & query_plan,
         bool need_sort = !window_description.full_sort_description.empty();
         if (need_sort && i != 0)
         {
-            need_sort = !sortDescriptionIsPrefix(window_description.full_sort_description, window_descriptions[i - 1].full_sort_description)
-                        || (settings.max_threads != 1 && window_description.partition_by.size() != window_descriptions[i - 1].partition_by.size());
+            auto prev = perm[i - 1];
+            need_sort = !sortDescriptionIsPrefix(window_description.full_sort_description, window_descriptions[prev].full_sort_description)
+                        || (settings.max_threads != 1 && window_description.partition_by.size() != window_descriptions[prev].partition_by.size());
         }
         if (need_sort)
         {
@@ -1054,9 +1055,9 @@ void addOffsetStep(QueryPlan & query_plan, const QueryAnalysisResult & query_ana
     }
 }
 
-void collectSetsFromActionsDAG(const ActionsDAGPtr & dag, std::unordered_set<const FutureSet *> & useful_sets)
+void collectSetsFromActionsDAG(const ActionsDAG & dag, std::unordered_set<const FutureSet *> & useful_sets)
 {
-    for (const auto & node : dag->getNodes())
+    for (const auto & node : dag.getNodes())
     {
         if (node.column)
         {
@@ -1075,7 +1076,7 @@ void collectSetsFromActionsDAG(const ActionsDAGPtr & dag, std::unordered_set<con
             {
                 if (const auto * index_hint = typeid_cast<const FunctionIndexHint *>(adaptor->getFunction().get()))
                 {
-                    collectSetsFromActionsDAG(index_hint->getActions(), useful_sets);
+                    collectSetsFromActionsDAG(*index_hint->getActions(), useful_sets);
                 }
             }
         }
@@ -1086,13 +1087,13 @@ void addBuildSubqueriesForSetsStepIfNeeded(
     QueryPlan & query_plan,
     const SelectQueryOptions & select_query_options,
     const PlannerContextPtr & planner_context,
-    const std::vector<ActionsDAGPtr> & result_actions_to_execute)
+    const std::vector<const ActionsDAG *> & result_actions_to_execute)
 {
     auto subqueries = planner_context->getPreparedSets().getSubqueries();
     std::unordered_set<const FutureSet *> useful_sets;
 
-    for (const auto & actions_to_execute : result_actions_to_execute)
-        collectSetsFromActionsDAG(actions_to_execute, useful_sets);
+    for (const auto * actions_to_execute : result_actions_to_execute)
+        collectSetsFromActionsDAG(*actions_to_execute, useful_sets);
 
     auto predicate = [&useful_sets](const auto & set) { return !useful_sets.contains(set.get()); };
     auto it = std::remove_if(subqueries.begin(), subqueries.end(), std::move(predicate));
@@ -1448,7 +1449,7 @@ void Planner::buildPlanForQueryNode()
             if (it != table_filters.end())
             {
                 const auto & filters = it->second;
-                table_expression_data.setFilterActions(filters.filter_actions);
+                table_expression_data.setFilterActions(filters.filter_actions->clone());
                 table_expression_data.setPrewhereInfo(filters.prewhere_info);
             }
         }
@@ -1539,15 +1540,15 @@ void Planner::buildPlanForQueryNode()
         planner_context,
         query_processing_info);
 
-    std::vector<ActionsDAGPtr> result_actions_to_execute = std::move(join_tree_query_plan.actions_dags);
+    std::vector<const ActionsDAG *> result_actions_to_execute = std::move(join_tree_query_plan.actions_dags);
 
     for (auto & [_, table_expression_data] : planner_context->getTableExpressionNodeToData())
     {
         if (table_expression_data.getPrewhereFilterActions())
-            result_actions_to_execute.push_back(table_expression_data.getPrewhereFilterActions());
+            result_actions_to_execute.push_back(table_expression_data.getPrewhereFilterActions().get());
 
         if (table_expression_data.getRowLevelFilterActions())
-            result_actions_to_execute.push_back(table_expression_data.getRowLevelFilterActions());
+            result_actions_to_execute.push_back(table_expression_data.getRowLevelFilterActions().get());
     }
 
     if (query_processing_info.isIntermediateStage())
diff --git a/src/Planner/PlannerActionsVisitor.cpp b/src/Planner/PlannerActionsVisitor.cpp
index 7a12d5d690d..59ec7778e21 100644
--- a/src/Planner/PlannerActionsVisitor.cpp
+++ b/src/Planner/PlannerActionsVisitor.cpp
@@ -757,7 +757,7 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::vi
         lambda_arguments_names_and_types.emplace_back(lambda_argument_name, std::move(lambda_argument_type));
     }
 
-    auto lambda_actions_dag = std::make_shared<ActionsDAG>();
+    auto lambda_actions_dag = std::make_unique<ActionsDAG>();
     actions_stack.emplace_back(*lambda_actions_dag, node);
 
     auto [lambda_expression_node_name, levels] = visitImpl(lambda_node.getExpression());
@@ -765,7 +765,7 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::vi
     lambda_actions_dag->removeUnusedActions(Names(1, lambda_expression_node_name));
 
     auto expression_actions_settings = ExpressionActionsSettings::fromContext(planner_context->getQueryContext(), CompileExpressions::yes);
-    auto lambda_actions = std::make_shared<ExpressionActions>(lambda_actions_dag, expression_actions_settings);
+    auto lambda_actions = std::make_shared<ExpressionActions>(std::move(lambda_actions_dag), expression_actions_settings);
 
     Names captured_column_names;
     ActionsDAG::NodeRawConstPtrs lambda_children;
@@ -879,7 +879,7 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::vi
     const auto & function_node = node->as<FunctionNode &>();
     auto function_node_name = action_node_name_helper.calculateActionNodeName(node);
 
-    auto index_hint_actions_dag = std::make_shared<ActionsDAG>();
+    auto index_hint_actions_dag = std::make_unique<ActionsDAG>();
     auto & index_hint_actions_dag_outputs = index_hint_actions_dag->getOutputs();
     std::unordered_set<std::string_view> index_hint_actions_dag_output_node_names;
     PlannerActionsVisitor actions_visitor(planner_context);
diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp
index 43b223172e6..918cfad703e 100644
--- a/src/Planner/PlannerJoinTree.cpp
+++ b/src/Planner/PlannerJoinTree.cpp
@@ -591,7 +591,7 @@ UInt64 mainQueryNodeBlockSizeByLimit(const SelectQueryInfo & select_query_info)
 std::unique_ptr<ExpressionStep> createComputeAliasColumnsStep(
     const std::unordered_map<std::string, ActionsDAGPtr> & alias_column_expressions, const DataStream & current_data_stream)
 {
-    ActionsDAGPtr merged_alias_columns_actions_dag = std::make_shared<ActionsDAG>(current_data_stream.header.getColumnsWithTypeAndName());
+    ActionsDAGPtr merged_alias_columns_actions_dag = std::make_unique<ActionsDAG>(current_data_stream.header.getColumnsWithTypeAndName());
     ActionsDAG::NodeRawConstPtrs action_dag_outputs = merged_alias_columns_actions_dag->getInputs();
 
     for (const auto & [column_name, alias_column_actions_dag] : alias_column_expressions)
@@ -646,7 +646,7 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres
 
         auto table_expression_query_info = select_query_info;
         table_expression_query_info.table_expression = table_expression;
-        table_expression_query_info.filter_actions_dag = table_expression_data.getFilterActions();
+        table_expression_query_info.filter_actions_dag = table_expression_data.getFilterActions()->clone();
         table_expression_query_info.analyzer_can_use_parallel_replicas_on_follower = table_node == planner_context->getGlobalPlannerContext()->parallel_replicas_table;
 
         size_t max_streams = settings.max_threads;
@@ -776,7 +776,7 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres
                 if (prewhere_actions)
                 {
                     prewhere_info = std::make_shared<PrewhereInfo>();
-                    prewhere_info->prewhere_actions = prewhere_actions;
+                    prewhere_info->prewhere_actions = prewhere_actions->clone();
                     prewhere_info->prewhere_column_name = prewhere_actions->getOutputs().at(0)->result_name;
                     prewhere_info->remove_prewhere_column = true;
                     prewhere_info->need_filter = true;
@@ -787,7 +787,7 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres
                 const auto & columns_names = table_expression_data.getColumnNames();
 
                 std::vector<std::pair<FilterDAGInfo, std::string>> where_filters;
-                const auto add_filter = [&](const FilterDAGInfo & filter_info, std::string description)
+                const auto add_filter = [&](FilterDAGInfo & filter_info, std::string description)
                 {
                     if (!filter_info.actions)
                         return;
@@ -805,34 +805,34 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres
 
                         if (!prewhere_info->prewhere_actions)
                         {
-                            prewhere_info->prewhere_actions = filter_info.actions;
+                            prewhere_info->prewhere_actions = std::move(filter_info.actions);
                             prewhere_info->prewhere_column_name = filter_info.column_name;
                             prewhere_info->remove_prewhere_column = filter_info.do_remove_column;
                             prewhere_info->need_filter = true;
                         }
                         else if (!prewhere_info->row_level_filter)
                         {
-                            prewhere_info->row_level_filter = filter_info.actions;
+                            prewhere_info->row_level_filter = std::move(filter_info.actions);
                             prewhere_info->row_level_column_name = filter_info.column_name;
                             prewhere_info->need_filter = true;
                         }
                         else
                         {
-                            where_filters.emplace_back(filter_info, std::move(description));
+                            where_filters.emplace_back(std::move(filter_info), std::move(description));
                         }
 
                     }
                     else
                     {
-                        where_filters.emplace_back(filter_info, std::move(description));
+                        where_filters.emplace_back(std::move(filter_info), std::move(description));
                     }
                 };
 
                 auto row_policy_filter_info
                     = buildRowPolicyFilterIfNeeded(storage, table_expression_query_info, planner_context, used_row_policies);
-                add_filter(row_policy_filter_info, "Row-level security filter");
                 if (row_policy_filter_info.actions)
-                    table_expression_data.setRowLevelFilterActions(row_policy_filter_info.actions);
+                    table_expression_data.setRowLevelFilterActions(row_policy_filter_info.actions->clone());
+                add_filter(row_policy_filter_info, "Row-level security filter");
 
                 if (query_context->getParallelReplicasMode() == Context::ParallelReplicasMode::CUSTOM_KEY)
                 {
@@ -1063,7 +1063,7 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres
 
     if (from_stage == QueryProcessingStage::FetchColumns)
     {
-        auto rename_actions_dag = std::make_shared<ActionsDAG>(query_plan.getCurrentDataStream().header.getColumnsWithTypeAndName());
+        auto rename_actions_dag = std::make_unique<ActionsDAG>(query_plan.getCurrentDataStream().header.getColumnsWithTypeAndName());
         ActionsDAG::NodeRawConstPtrs updated_actions_dag_outputs;
 
         for (auto & output_node : rename_actions_dag->getOutputs())
@@ -1077,7 +1077,7 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres
 
         rename_actions_dag->getOutputs() = std::move(updated_actions_dag_outputs);
 
-        auto rename_step = std::make_unique<ExpressionStep>(query_plan.getCurrentDataStream(), rename_actions_dag);
+        auto rename_step = std::make_unique<ExpressionStep>(query_plan.getCurrentDataStream(), std::move(rename_actions_dag));
         rename_step->setStepDescription("Change column names to column identifiers");
         query_plan.addStep(std::move(rename_step));
     }
@@ -1117,7 +1117,7 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres
 
 void joinCastPlanColumnsToNullable(QueryPlan & plan_to_add_cast, PlannerContextPtr & planner_context, const FunctionOverloadResolverPtr & to_nullable_function)
 {
-    auto cast_actions_dag = std::make_shared<ActionsDAG>(plan_to_add_cast.getCurrentDataStream().header.getColumnsWithTypeAndName());
+    auto cast_actions_dag = std::make_unique<ActionsDAG>(plan_to_add_cast.getCurrentDataStream().header.getColumnsWithTypeAndName());
 
     for (auto & output_node : cast_actions_dag->getOutputs())
     {
@@ -1178,6 +1178,9 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_
             join_table_expression,
             planner_context);
 
+        left_join_tree_query_plan.actions_dags.emplace_back(join_clauses_and_actions.left_join_expressions_actions.get());
+        left_join_tree_query_plan.actions_dags.emplace_back(join_clauses_and_actions.right_join_expressions_actions.get());
+
         join_clauses_and_actions.left_join_expressions_actions->appendInputsForUnusedColumns(left_plan.getCurrentDataStream().header);
         auto left_join_expressions_actions_step = std::make_unique<ExpressionStep>(left_plan.getCurrentDataStream(), join_clauses_and_actions.left_join_expressions_actions);
         left_join_expressions_actions_step->setStepDescription("JOIN actions");
@@ -1223,7 +1226,7 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_
 
     auto join_cast_plan_output_nodes = [&](QueryPlan & plan_to_add_cast, std::unordered_map<std::string, DataTypePtr> & plan_column_name_to_cast_type)
     {
-        auto cast_actions_dag = std::make_shared<ActionsDAG>(plan_to_add_cast.getCurrentDataStream().header.getColumnsWithTypeAndName());
+        auto cast_actions_dag = std::make_unique<ActionsDAG>(plan_to_add_cast.getCurrentDataStream().header.getColumnsWithTypeAndName());
 
         for (auto & output_node : cast_actions_dag->getOutputs())
         {
@@ -1381,9 +1384,10 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_
 
         if (join_clauses_and_actions.mixed_join_expressions_actions)
         {
+            left_join_tree_query_plan.actions_dags.push_back(join_clauses_and_actions.mixed_join_expressions_actions.get());
             ExpressionActionsPtr & mixed_join_expression = table_join->getMixedJoinExpression();
             mixed_join_expression = std::make_shared<ExpressionActions>(
-                join_clauses_and_actions.mixed_join_expressions_actions,
+                std::move(join_clauses_and_actions.mixed_join_expressions_actions),
                 ExpressionActionsSettings::fromContext(planner_context->getQueryContext()));
         }
     }
@@ -1537,7 +1541,7 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_
         result_plan.unitePlans(std::move(join_step), {std::move(plans)});
     }
 
-    auto drop_unused_columns_after_join_actions_dag = std::make_shared<ActionsDAG>(result_plan.getCurrentDataStream().header.getColumnsWithTypeAndName());
+    auto drop_unused_columns_after_join_actions_dag = std::make_unique<ActionsDAG>(result_plan.getCurrentDataStream().header.getColumnsWithTypeAndName());
     ActionsDAG::NodeRawConstPtrs drop_unused_columns_after_join_actions_dag_updated_outputs;
     std::unordered_set<std::string_view> drop_unused_columns_after_join_actions_dag_updated_outputs_names;
     std::optional<size_t> first_skipped_column_node_index;
@@ -1582,14 +1586,14 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_
         left_join_tree_query_plan.used_row_policies.insert(right_join_tree_query_plan_row_policy);
 
     /// Collect all required actions dags in `left_join_tree_query_plan.actions_dags`
-    for (auto && action_dag : right_join_tree_query_plan.actions_dags)
+    for (const auto * action_dag : right_join_tree_query_plan.actions_dags)
         left_join_tree_query_plan.actions_dags.emplace_back(action_dag);
-    if (join_clauses_and_actions.left_join_expressions_actions)
-        left_join_tree_query_plan.actions_dags.emplace_back(std::move(join_clauses_and_actions.left_join_expressions_actions));
-    if (join_clauses_and_actions.right_join_expressions_actions)
-        left_join_tree_query_plan.actions_dags.emplace_back(std::move(join_clauses_and_actions.right_join_expressions_actions));
-    if (join_clauses_and_actions.mixed_join_expressions_actions)
-        left_join_tree_query_plan.actions_dags.push_back(join_clauses_and_actions.mixed_join_expressions_actions);
+    // if (join_clauses_and_actions.left_join_expressions_actions)
+    //     left_join_tree_query_plan.actions_dags.emplace_back(join_clauses_and_actions.left_join_expressions_actions.get());
+    // if (join_clauses_and_actions.right_join_expressions_actions)
+    //     left_join_tree_query_plan.actions_dags.emplace_back(join_clauses_and_actions.right_join_expressions_actions.get());
+    // if (join_clauses_and_actions.mixed_join_expressions_actions)
+    //     left_join_tree_query_plan.actions_dags.push_back(join_clauses_and_actions.mixed_join_expressions_actions.get());
 
     auto mapping = std::move(left_join_tree_query_plan.query_node_to_plan_step_mapping);
     auto & r_mapping = right_join_tree_query_plan.query_node_to_plan_step_mapping;
@@ -1619,7 +1623,7 @@ JoinTreeQueryPlan buildQueryPlanForArrayJoinNode(const QueryTreeNodePtr & array_
     auto plan = std::move(join_tree_query_plan.query_plan);
     auto plan_output_columns = plan.getCurrentDataStream().header.getColumnsWithTypeAndName();
 
-    ActionsDAGPtr array_join_action_dag = std::make_shared<ActionsDAG>(plan_output_columns);
+    ActionsDAGPtr array_join_action_dag = std::make_unique<ActionsDAG>(plan_output_columns);
     PlannerActionsVisitor actions_visitor(planner_context);
     std::unordered_set<std::string> array_join_expressions_output_nodes;
 
@@ -1642,13 +1646,13 @@ JoinTreeQueryPlan buildQueryPlanForArrayJoinNode(const QueryTreeNodePtr & array_
 
     array_join_action_dag->appendInputsForUnusedColumns(plan.getCurrentDataStream().header);
 
-    join_tree_query_plan.actions_dags.push_back(array_join_action_dag);
+    join_tree_query_plan.actions_dags.push_back(array_join_action_dag.get());
 
-    auto array_join_actions = std::make_unique<ExpressionStep>(plan.getCurrentDataStream(), array_join_action_dag);
+    auto array_join_actions = std::make_unique<ExpressionStep>(plan.getCurrentDataStream(), std::move(array_join_action_dag));
     array_join_actions->setStepDescription("ARRAY JOIN actions");
     plan.addStep(std::move(array_join_actions));
 
-    auto drop_unused_columns_before_array_join_actions_dag = std::make_shared<ActionsDAG>(plan.getCurrentDataStream().header.getColumnsWithTypeAndName());
+    auto drop_unused_columns_before_array_join_actions_dag = std::make_unique<ActionsDAG>(plan.getCurrentDataStream().header.getColumnsWithTypeAndName());
     ActionsDAG::NodeRawConstPtrs drop_unused_columns_before_array_join_actions_dag_updated_outputs;
     std::unordered_set<std::string_view> drop_unused_columns_before_array_join_actions_dag_updated_outputs_names;
 
diff --git a/src/Planner/PlannerJoinTree.h b/src/Planner/PlannerJoinTree.h
index 9110b2bfef9..675079427eb 100644
--- a/src/Planner/PlannerJoinTree.h
+++ b/src/Planner/PlannerJoinTree.h
@@ -16,7 +16,7 @@ struct JoinTreeQueryPlan
     QueryPlan query_plan;
     QueryProcessingStage::Enum from_stage;
     std::set<std::string> used_row_policies{};
-    std::vector<ActionsDAGPtr> actions_dags{};
+    std::vector<const ActionsDAG *> actions_dags{};
     std::unordered_map<const QueryNode *, const QueryPlan::Node *> query_node_to_plan_step_mapping{};
 };
 
diff --git a/src/Planner/PlannerJoins.cpp b/src/Planner/PlannerJoins.cpp
index 84efdd21336..45842c0d705 100644
--- a/src/Planner/PlannerJoins.cpp
+++ b/src/Planner/PlannerJoins.cpp
@@ -376,8 +376,8 @@ JoinClausesAndActions buildJoinClausesAndActions(
     const JoinNode & join_node,
     const PlannerContextPtr & planner_context)
 {
-    ActionsDAGPtr left_join_actions = std::make_shared<ActionsDAG>(left_table_expression_columns);
-    ActionsDAGPtr right_join_actions = std::make_shared<ActionsDAG>(right_table_expression_columns);
+    ActionsDAGPtr left_join_actions = std::make_unique<ActionsDAG>(left_table_expression_columns);
+    ActionsDAGPtr right_join_actions = std::make_unique<ActionsDAG>(right_table_expression_columns);
     ColumnsWithTypeAndName mixed_table_expression_columns;
     for (const auto & left_column : left_table_expression_columns)
     {
@@ -387,7 +387,7 @@ JoinClausesAndActions buildJoinClausesAndActions(
     {
         mixed_table_expression_columns.push_back(right_column);
     }
-    ActionsDAGPtr mixed_join_actions = std::make_shared<ActionsDAG>(mixed_table_expression_columns);
+    ActionsDAGPtr mixed_join_actions = std::make_unique<ActionsDAG>(mixed_table_expression_columns);
 
     /** It is possible to have constant value in JOIN ON section, that we need to ignore during DAG construction.
       * If we do not ignore it, this function will be replaced by underlying constant.
@@ -601,7 +601,7 @@ JoinClausesAndActions buildJoinClausesAndActions(
         /// So, for each column, we recalculate the value of the whole expression from JOIN ON to check if rows should be joined.
         if (result.join_clauses.size() > 1)
         {
-            auto mixed_join_expressions_actions = std::make_shared<ActionsDAG>(mixed_table_expression_columns);
+            auto mixed_join_expressions_actions = std::make_unique<ActionsDAG>(mixed_table_expression_columns);
             PlannerActionsVisitor join_expression_visitor(planner_context);
             auto join_expression_dag_node_raw_pointers = join_expression_visitor.visit(*mixed_join_expressions_actions, join_expression);
             if (join_expression_dag_node_raw_pointers.size() != 1)
@@ -611,14 +611,14 @@ JoinClausesAndActions buildJoinClausesAndActions(
             mixed_join_expressions_actions->addOrReplaceInOutputs(*join_expression_dag_node_raw_pointers[0]);
             Names required_names{join_expression_dag_node_raw_pointers[0]->result_name};
             mixed_join_expressions_actions->removeUnusedActions(required_names);
-            result.mixed_join_expressions_actions = mixed_join_expressions_actions;
+            result.mixed_join_expressions_actions = std::move(mixed_join_expressions_actions);
         }
         else
         {
             const auto & join_clause = result.join_clauses.front();
             const auto & mixed_filter_condition_nodes = join_clause.getMixedFilterConditionNodes();
             auto mixed_join_expressions_actions = ActionsDAG::buildFilterActionsDAG(mixed_filter_condition_nodes, {}, true);
-            result.mixed_join_expressions_actions = mixed_join_expressions_actions;
+            result.mixed_join_expressions_actions = std::move(mixed_join_expressions_actions);
         }
         auto outputs = result.mixed_join_expressions_actions->getOutputs();
         if (outputs.size() != 1)
diff --git a/src/Planner/PlannerWindowFunctions.cpp b/src/Planner/PlannerWindowFunctions.cpp
index ce74d82c08d..9deceeef9a3 100644
--- a/src/Planner/PlannerWindowFunctions.cpp
+++ b/src/Planner/PlannerWindowFunctions.cpp
@@ -120,7 +120,7 @@ std::vector<WindowDescription> extractWindowDescriptions(const QueryTreeNodes &
     return result;
 }
 
-void sortWindowDescriptions(std::vector<WindowDescription> & window_descriptions)
+std::vector<size_t> sortWindowDescriptions(const std::vector<WindowDescription> & window_descriptions)
 {
     auto window_description_comparator = [](const WindowDescription & lhs, const WindowDescription & rhs)
     {
@@ -151,7 +151,16 @@ void sortWindowDescriptions(std::vector<WindowDescription> & window_descriptions
         return left.size() > right.size();
     };
 
-    ::sort(window_descriptions.begin(), window_descriptions.end(), window_description_comparator);
+    auto comparator = [&](size_t lhs, size_t rhs)
+    {
+        return window_description_comparator(window_descriptions[lhs], window_descriptions[rhs]);
+    };
+
+    std::vector<size_t> perm(window_descriptions.size());
+    std::iota(perm.begin(), perm.end(), 0U);
+    ::sort(perm.begin(), perm.end(), comparator);
+
+    return perm;
 }
 
 }
diff --git a/src/Planner/PlannerWindowFunctions.h b/src/Planner/PlannerWindowFunctions.h
index 1552ef5a71f..3039ecefc4b 100644
--- a/src/Planner/PlannerWindowFunctions.h
+++ b/src/Planner/PlannerWindowFunctions.h
@@ -15,6 +15,6 @@ std::vector<WindowDescription> extractWindowDescriptions(const QueryTreeNodes &
 /** Try to sort window descriptions in such an order that the window with the longest
   * sort description goes first, and all window that use its prefixes follow.
   */
-void sortWindowDescriptions(std::vector<WindowDescription> & window_descriptions);
+std::vector<size_t> sortWindowDescriptions(const std::vector<WindowDescription> & window_descriptions);
 
 }
diff --git a/src/Planner/Utils.cpp b/src/Planner/Utils.cpp
index 18a6d297838..493ecf5ef53 100644
--- a/src/Planner/Utils.cpp
+++ b/src/Planner/Utils.cpp
@@ -440,7 +440,7 @@ FilterDAGInfo buildFilterInfo(QueryTreeNodePtr filter_query_tree,
     collectSourceColumns(filter_query_tree, planner_context, false /*keep_alias_columns*/);
     collectSets(filter_query_tree, *planner_context);
 
-    auto filter_actions_dag = std::make_shared<ActionsDAG>();
+    auto filter_actions_dag = std::make_unique<ActionsDAG>();
 
     PlannerActionsVisitor actions_visitor(planner_context, false /*use_column_identifier_as_action_node_name*/);
     auto expression_nodes = actions_visitor.visit(*filter_actions_dag, filter_query_tree);
diff --git a/src/Processors/QueryPlan/AggregatingStep.cpp b/src/Processors/QueryPlan/AggregatingStep.cpp
index 0d7e05af1de..64ba7f7cd2a 100644
--- a/src/Processors/QueryPlan/AggregatingStep.cpp
+++ b/src/Processors/QueryPlan/AggregatingStep.cpp
@@ -303,7 +303,7 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B
                 const auto & header = ports[set_counter]->getHeader();
 
                 /// Here we create a DAG which fills missing keys and adds `__grouping_set` column
-                auto dag = std::make_shared<ActionsDAG>(header.getColumnsWithTypeAndName());
+                auto dag = std::make_unique<ActionsDAG>(header.getColumnsWithTypeAndName());
                 ActionsDAG::NodeRawConstPtrs outputs;
                 outputs.reserve(output_header.columns() + 1);
 
@@ -347,7 +347,7 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B
                 }
 
                 dag->getOutputs().swap(outputs);
-                auto expression = std::make_shared<ExpressionActions>(dag, settings.getActionsSettings());
+                auto expression = std::make_shared<ExpressionActions>(std::move(dag), settings.getActionsSettings());
                 auto transform = std::make_shared<ExpressionTransform>(header, expression);
 
                 connect(*ports[set_counter], transform->getInputPort());
diff --git a/src/Processors/QueryPlan/CubeStep.cpp b/src/Processors/QueryPlan/CubeStep.cpp
index d010a3327a6..b6c70061987 100644
--- a/src/Processors/QueryPlan/CubeStep.cpp
+++ b/src/Processors/QueryPlan/CubeStep.cpp
@@ -36,7 +36,7 @@ CubeStep::CubeStep(const DataStream & input_stream_, Aggregator::Params params_,
 
 ProcessorPtr addGroupingSetForTotals(const Block & header, const Names & keys, bool use_nulls, const BuildQueryPipelineSettings & settings, UInt64 grouping_set_number)
 {
-    auto dag = std::make_shared<ActionsDAG>(header.getColumnsWithTypeAndName());
+    auto dag = std::make_unique<ActionsDAG>(header.getColumnsWithTypeAndName());
     auto & outputs = dag->getOutputs();
 
     if (use_nulls)
@@ -59,7 +59,7 @@ ProcessorPtr addGroupingSetForTotals(const Block & header, const Names & keys, b
     grouping_node = &dag->materializeNode(*grouping_node);
     outputs.insert(outputs.begin(), grouping_node);
 
-    auto expression = std::make_shared<ExpressionActions>(dag, settings.getActionsSettings());
+    auto expression = std::make_shared<ExpressionActions>(std::move(dag), settings.getActionsSettings());
     return std::make_shared<ExpressionTransform>(header, expression);
 }
 
diff --git a/src/Processors/QueryPlan/ExpressionStep.cpp b/src/Processors/QueryPlan/ExpressionStep.cpp
index 0ccb0c4492a..90ac94a1ace 100644
--- a/src/Processors/QueryPlan/ExpressionStep.cpp
+++ b/src/Processors/QueryPlan/ExpressionStep.cpp
@@ -30,13 +30,13 @@ ExpressionStep::ExpressionStep(const DataStream & input_stream_, const ActionsDA
         input_stream_,
         ExpressionTransform::transformHeader(input_stream_.header, *actions_dag_),
         getTraits(actions_dag_, input_stream_.header, input_stream_.sort_description))
-    , actions_dag(actions_dag_)
+    , actions_dag(actions_dag_->clone())
 {
 }
 
 void ExpressionStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings & settings)
 {
-    auto expression = std::make_shared<ExpressionActions>(actions_dag, settings.getActionsSettings());
+    auto expression = std::make_shared<ExpressionActions>(actions_dag->clone(), settings.getActionsSettings());
 
     pipeline.addSimpleTransform([&](const Block & header)
     {
@@ -49,7 +49,7 @@ void ExpressionStep::transformPipeline(QueryPipelineBuilder & pipeline, const Bu
                 pipeline.getHeader().getColumnsWithTypeAndName(),
                 output_stream->header.getColumnsWithTypeAndName(),
                 ActionsDAG::MatchColumnsMode::Name);
-        auto convert_actions = std::make_shared<ExpressionActions>(convert_actions_dag, settings.getActionsSettings());
+        auto convert_actions = std::make_shared<ExpressionActions>(std::move(convert_actions_dag), settings.getActionsSettings());
 
         pipeline.addSimpleTransform([&](const Block & header)
         {
@@ -61,13 +61,13 @@ void ExpressionStep::transformPipeline(QueryPipelineBuilder & pipeline, const Bu
 void ExpressionStep::describeActions(FormatSettings & settings) const
 {
     String prefix(settings.offset, settings.indent_char);
-    auto expression = std::make_shared<ExpressionActions>(actions_dag);
+    auto expression = std::make_shared<ExpressionActions>(actions_dag->clone());
     expression->describeActions(settings.out, prefix);
 }
 
 void ExpressionStep::describeActions(JSONBuilder::JSONMap & map) const
 {
-    auto expression = std::make_shared<ExpressionActions>(actions_dag);
+    auto expression = std::make_shared<ExpressionActions>(actions_dag->clone());
     map.add("Expression", expression->toTree());
 }
 
@@ -79,7 +79,7 @@ void ExpressionStep::updateOutputStream()
     if (!getDataStreamTraits().preserves_sorting)
         return;
 
-    FindAliasForInputName alias_finder(actions_dag);
+    FindAliasForInputName alias_finder(*actions_dag);
     const auto & input_sort_description = getInputStreams().front().sort_description;
     for (size_t i = 0, s = input_sort_description.size(); i < s; ++i)
     {
diff --git a/src/Processors/QueryPlan/ExpressionStep.h b/src/Processors/QueryPlan/ExpressionStep.h
index 3eef14ac129..ebbac8217cb 100644
--- a/src/Processors/QueryPlan/ExpressionStep.h
+++ b/src/Processors/QueryPlan/ExpressionStep.h
@@ -5,7 +5,7 @@ namespace DB
 {
 
 class ActionsDAG;
-using ActionsDAGPtr = std::shared_ptr<ActionsDAG>;
+using ActionsDAGPtr = std::unique_ptr<ActionsDAG>;
 
 class ExpressionTransform;
 class JoiningTransform;
diff --git a/src/Processors/QueryPlan/FilterStep.cpp b/src/Processors/QueryPlan/FilterStep.cpp
index 72934665b5c..ef9f1d17822 100644
--- a/src/Processors/QueryPlan/FilterStep.cpp
+++ b/src/Processors/QueryPlan/FilterStep.cpp
@@ -46,7 +46,6 @@ FilterStep::FilterStep(
             filter_column_name_,
             remove_filter_column_),
         getTraits(actions_dag_, input_stream_.header, input_stream_.sort_description, remove_filter_column_, filter_column_name_))
-    , actions_dag(actions_dag_)
     , filter_column_name(std::move(filter_column_name_))
     , remove_filter_column(remove_filter_column_)
 {
@@ -56,7 +55,7 @@ FilterStep::FilterStep(
 
 void FilterStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings & settings)
 {
-    auto expression = std::make_shared<ExpressionActions>(actions_dag, settings.getActionsSettings());
+    auto expression = std::make_shared<ExpressionActions>(std::move(actions_dag), settings.getActionsSettings());
 
     pipeline.addSimpleTransform([&](const Block & header, QueryPipelineBuilder::StreamType stream_type)
     {
@@ -70,7 +69,7 @@ void FilterStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQ
                 pipeline.getHeader().getColumnsWithTypeAndName(),
                 output_stream->header.getColumnsWithTypeAndName(),
                 ActionsDAG::MatchColumnsMode::Name);
-        auto convert_actions = std::make_shared<ExpressionActions>(convert_actions_dag, settings.getActionsSettings());
+        auto convert_actions = std::make_shared<ExpressionActions>(std::move(convert_actions_dag), settings.getActionsSettings());
 
         pipeline.addSimpleTransform([&](const Block & header)
         {
@@ -88,7 +87,7 @@ void FilterStep::describeActions(FormatSettings & settings) const
         settings.out << " (removed)";
     settings.out << '\n';
 
-    auto expression = std::make_shared<ExpressionActions>(actions_dag);
+    auto expression = std::make_shared<ExpressionActions>(actions_dag->clone());
     expression->describeActions(settings.out, prefix);
 }
 
@@ -97,7 +96,7 @@ void FilterStep::describeActions(JSONBuilder::JSONMap & map) const
     map.add("Filter Column", filter_column_name);
     map.add("Removes Filter", remove_filter_column);
 
-    auto expression = std::make_shared<ExpressionActions>(actions_dag);
+    auto expression = std::make_shared<ExpressionActions>(actions_dag->clone());
     map.add("Expression", expression->toTree());
 }
 
@@ -111,7 +110,7 @@ void FilterStep::updateOutputStream()
     if (!getDataStreamTraits().preserves_sorting)
         return;
 
-    FindAliasForInputName alias_finder(actions_dag);
+    FindAliasForInputName alias_finder(*actions_dag);
     const auto & input_sort_description = getInputStreams().front().sort_description;
     for (size_t i = 0, s = input_sort_description.size(); i < s; ++i)
     {
diff --git a/src/Processors/QueryPlan/FilterStep.h b/src/Processors/QueryPlan/FilterStep.h
index 939d0900c86..0f894a570b7 100644
--- a/src/Processors/QueryPlan/FilterStep.h
+++ b/src/Processors/QueryPlan/FilterStep.h
@@ -5,7 +5,7 @@ namespace DB
 {
 
 class ActionsDAG;
-using ActionsDAGPtr = std::shared_ptr<ActionsDAG>;
+using ActionsDAGPtr = std::unique_ptr<ActionsDAG>;
 
 /// Implements WHERE, HAVING operations. See FilterTransform.
 class FilterStep : public ITransformingStep
diff --git a/src/Processors/QueryPlan/Optimizations/distinctReadInOrder.cpp b/src/Processors/QueryPlan/Optimizations/distinctReadInOrder.cpp
index 0a3a4094a66..87e16b5a244 100644
--- a/src/Processors/QueryPlan/Optimizations/distinctReadInOrder.cpp
+++ b/src/Processors/QueryPlan/Optimizations/distinctReadInOrder.cpp
@@ -10,7 +10,7 @@
 namespace DB::QueryPlanOptimizations
 {
 /// build actions DAG from stack of steps
-static ActionsDAGPtr buildActionsForPlanPath(std::vector<ActionsDAGPtr> & dag_stack)
+static ActionsDAGPtr buildActionsForPlanPath(std::vector<const ActionsDAG *> & dag_stack)
 {
     if (dag_stack.empty())
         return nullptr;
@@ -27,10 +27,10 @@ static ActionsDAGPtr buildActionsForPlanPath(std::vector<ActionsDAGPtr> & dag_st
 }
 
 static std::set<std::string>
-getOriginalDistinctColumns(const ColumnsWithTypeAndName & distinct_columns, std::vector<ActionsDAGPtr> & dag_stack)
+getOriginalDistinctColumns(const ColumnsWithTypeAndName & distinct_columns, std::vector<const ActionsDAG *> & dag_stack)
 {
     auto actions = buildActionsForPlanPath(dag_stack);
-    FindOriginalNodeForOutputName original_node_finder(actions);
+    FindOriginalNodeForOutputName original_node_finder(*actions);
     std::set<std::string> original_distinct_columns;
     for (const auto & column : distinct_columns)
     {
@@ -65,7 +65,7 @@ size_t tryDistinctReadInOrder(QueryPlan::Node * parent_node)
     /// (3) gather actions DAG to find original names for columns in distinct step later
     std::vector<ITransformingStep *> steps_to_update;
     QueryPlan::Node * node = parent_node;
-    std::vector<ActionsDAGPtr> dag_stack;
+    std::vector<const ActionsDAG *> dag_stack;
     while (!node->children.empty())
     {
         auto * step = dynamic_cast<ITransformingStep *>(node->step.get());
@@ -79,9 +79,9 @@ size_t tryDistinctReadInOrder(QueryPlan::Node * parent_node)
         steps_to_update.push_back(step);
 
         if (const auto * const expr = typeid_cast<const ExpressionStep *>(step); expr)
-            dag_stack.push_back(expr->getExpression());
+            dag_stack.push_back(expr->getExpression().get());
         else if (const auto * const filter = typeid_cast<const FilterStep *>(step); filter)
-            dag_stack.push_back(filter->getExpression());
+            dag_stack.push_back(filter->getExpression().get());
 
         node = node->children.front();
     }
diff --git a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp
index 263598bdca7..ff1cefff09a 100644
--- a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp
+++ b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp
@@ -611,7 +611,7 @@ size_t tryPushDownFilter(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes
 
     if (auto * read_from_merge = typeid_cast<ReadFromMerge *>(child.get()))
     {
-        FilterDAGInfo info{filter->getExpression(), filter->getFilterColumnName(), filter->removesFilterColumn()};
+        FilterDAGInfo info{filter->getExpression()->clone(), filter->getFilterColumnName(), filter->removesFilterColumn()};
         read_from_merge->addFilter(std::move(info));
         std::swap(*parent_node, *child_node);
         return 1;
diff --git a/src/Processors/QueryPlan/Optimizations/optimizePrewhere.cpp b/src/Processors/QueryPlan/Optimizations/optimizePrewhere.cpp
index 1badd315200..13b691da888 100644
--- a/src/Processors/QueryPlan/Optimizations/optimizePrewhere.cpp
+++ b/src/Processors/QueryPlan/Optimizations/optimizePrewhere.cpp
@@ -108,7 +108,7 @@ void optimizePrewhere(Stack & stack, QueryPlan::Nodes &)
     prewhere_info->need_filter = true;
     prewhere_info->remove_prewhere_column = optimize_result.fully_moved_to_prewhere && filter_step->removesFilterColumn();
 
-    auto filter_expression = filter_step->getExpression();
+    auto filter_expression = filter_step->getExpression()->clone();
     const auto & filter_column_name = filter_step->getFilterColumnName();
 
     if (prewhere_info->remove_prewhere_column)
diff --git a/src/Processors/QueryPlan/Optimizations/optimizePrimaryKeyCondition.cpp b/src/Processors/QueryPlan/Optimizations/optimizePrimaryKeyCondition.cpp
index dbcaf5f00a7..e57d3319076 100644
--- a/src/Processors/QueryPlan/Optimizations/optimizePrimaryKeyCondition.cpp
+++ b/src/Processors/QueryPlan/Optimizations/optimizePrimaryKeyCondition.cpp
@@ -18,15 +18,15 @@ void optimizePrimaryKeyCondition(const Stack & stack)
     const auto & storage_prewhere_info = source_step_with_filter->getPrewhereInfo();
     if (storage_prewhere_info)
     {
-        source_step_with_filter->addFilter(storage_prewhere_info->prewhere_actions, storage_prewhere_info->prewhere_column_name);
+        source_step_with_filter->addFilter(storage_prewhere_info->prewhere_actions->clone(), storage_prewhere_info->prewhere_column_name);
         if (storage_prewhere_info->row_level_filter)
-            source_step_with_filter->addFilter(storage_prewhere_info->row_level_filter, storage_prewhere_info->row_level_column_name);
+            source_step_with_filter->addFilter(storage_prewhere_info->row_level_filter->clone(), storage_prewhere_info->row_level_column_name);
     }
 
     for (auto iter = stack.rbegin() + 1; iter != stack.rend(); ++iter)
     {
         if (auto * filter_step = typeid_cast<FilterStep *>(iter->node->step.get()))
-            source_step_with_filter->addFilter(filter_step->getExpression(), filter_step->getFilterColumnName());
+            source_step_with_filter->addFilter(filter_step->getExpression()->clone(), filter_step->getFilterColumnName());
 
         /// Note: actually, plan optimizations merge Filter and Expression steps.
         /// Ideally, chain should look like (Expression -> ...) -> (Filter -> ...) -> ReadFromStorage,
diff --git a/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp b/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp
index 537555afa2a..8e782e68db8 100644
--- a/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp
+++ b/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp
@@ -1066,13 +1066,13 @@ size_t tryReuseStorageOrderingForWindowFunctions(QueryPlan::Node * parent_node,
     for (const auto & actions_dag : window_desc.partition_by_actions)
     {
         order_by_elements_actions.emplace_back(
-            std::make_shared<ExpressionActions>(actions_dag, ExpressionActionsSettings::fromContext(context, CompileExpressions::yes)));
+            std::make_shared<ExpressionActions>(actions_dag->clone(), ExpressionActionsSettings::fromContext(context, CompileExpressions::yes)));
     }
 
     for (const auto & actions_dag : window_desc.order_by_actions)
     {
         order_by_elements_actions.emplace_back(
-            std::make_shared<ExpressionActions>(actions_dag, ExpressionActionsSettings::fromContext(context, CompileExpressions::yes)));
+            std::make_shared<ExpressionActions>(actions_dag->clone(), ExpressionActionsSettings::fromContext(context, CompileExpressions::yes)));
     }
 
     auto order_optimizer = std::make_shared<ReadInOrderOptimizer>(
diff --git a/src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp b/src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp
index 70327bc95b4..7c45ef48252 100644
--- a/src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp
+++ b/src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp
@@ -486,7 +486,7 @@ AggregateProjectionCandidates getAggregateProjectionCandidates(
             auto block = reading.getMergeTreeData().getMinMaxCountProjectionBlock(
                 metadata,
                 candidate.dag->getRequiredColumnsNames(),
-                (dag.filter_node ? dag.dag : nullptr),
+                (dag.filter_node ? dag.dag.get() : nullptr),
                 parts,
                 max_added_blocks.get(),
                 context);
@@ -675,7 +675,7 @@ std::optional<String> optimizeUseAggregateProjections(QueryPlan::Node & node, Qu
                 query_info,
                 context,
                 max_added_blocks,
-                candidate.dag);
+                candidate.dag.get());
 
             if (!analyzed)
                 continue;
diff --git a/src/Processors/QueryPlan/Optimizations/optimizeUseNormalProjection.cpp b/src/Processors/QueryPlan/Optimizations/optimizeUseNormalProjection.cpp
index 0af3869ccf1..c7e96d66817 100644
--- a/src/Processors/QueryPlan/Optimizations/optimizeUseNormalProjection.cpp
+++ b/src/Processors/QueryPlan/Optimizations/optimizeUseNormalProjection.cpp
@@ -172,7 +172,7 @@ std::optional<String> optimizeUseNormalProjections(Stack & stack, QueryPlan::Nod
             query_info,
             context,
             max_added_blocks,
-            query.filter_node ? query.dag : nullptr);
+            query.filter_node ? query.dag.get() : nullptr);
 
         if (!analyzed)
             continue;
diff --git a/src/Processors/QueryPlan/Optimizations/projectionsCommon.cpp b/src/Processors/QueryPlan/Optimizations/projectionsCommon.cpp
index af1578d6af8..d8b40b22904 100644
--- a/src/Processors/QueryPlan/Optimizations/projectionsCommon.cpp
+++ b/src/Processors/QueryPlan/Optimizations/projectionsCommon.cpp
@@ -213,7 +213,7 @@ bool analyzeProjectionCandidate(
     const SelectQueryInfo & query_info,
     const ContextPtr & context,
     const std::shared_ptr<PartitionIdToMaxBlock> & max_added_blocks,
-    const ActionsDAGPtr & dag)
+    const ActionsDAG * dag)
 {
     MergeTreeData::DataPartsVector projection_parts;
     MergeTreeData::DataPartsVector normal_parts;
@@ -238,7 +238,7 @@ bool analyzeProjectionCandidate(
 
     auto projection_query_info = query_info;
     projection_query_info.prewhere_info = nullptr;
-    projection_query_info.filter_actions_dag = dag;
+    projection_query_info.filter_actions_dag = dag->clone();
 
     auto projection_result_ptr = reader.estimateNumMarksToRead(
         std::move(projection_parts),
diff --git a/src/Processors/QueryPlan/Optimizations/projectionsCommon.h b/src/Processors/QueryPlan/Optimizations/projectionsCommon.h
index e1e106b988e..59ad3a43b97 100644
--- a/src/Processors/QueryPlan/Optimizations/projectionsCommon.h
+++ b/src/Processors/QueryPlan/Optimizations/projectionsCommon.h
@@ -60,6 +60,6 @@ bool analyzeProjectionCandidate(
     const SelectQueryInfo & query_info,
     const ContextPtr & context,
     const std::shared_ptr<PartitionIdToMaxBlock> & max_added_blocks,
-    const ActionsDAGPtr & dag);
+    const ActionsDAG * dag);
 
 }
diff --git a/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp b/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp
index 51df25b35f4..d3c75c988e7 100644
--- a/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp
+++ b/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp
@@ -65,7 +65,7 @@ namespace
     }
 
     /// build actions DAG from stack of steps
-    ActionsDAGPtr buildActionsForPlanPath(std::vector<ActionsDAGPtr> & dag_stack)
+    ActionsDAGPtr buildActionsForPlanPath(std::vector<const ActionsDAG *> & dag_stack)
     {
         if (dag_stack.empty())
             return nullptr;
@@ -83,7 +83,7 @@ namespace
     }
 
     bool compareAggregationKeysWithDistinctColumns(
-        const Names & aggregation_keys, const DistinctColumns & distinct_columns, std::vector<std::vector<ActionsDAGPtr>> actions_chain)
+        const Names & aggregation_keys, const DistinctColumns & distinct_columns, std::vector<std::vector<const ActionsDAG *>> actions_chain)
     {
         logDebug("aggregation_keys", aggregation_keys);
         logDebug("aggregation_keys size", aggregation_keys.size());
@@ -93,7 +93,8 @@ namespace
         std::set<String> source_columns;
         for (auto & actions : actions_chain)
         {
-            FindOriginalNodeForOutputName original_node_finder(buildActionsForPlanPath(actions));
+            auto tmp_actions = buildActionsForPlanPath(actions);
+            FindOriginalNodeForOutputName original_node_finder(*tmp_actions);
             for (const auto & column : current_columns)
             {
                 logDebug("distinct column name", column);
@@ -152,8 +153,8 @@ namespace
         const DistinctStep * distinct_step = typeid_cast<DistinctStep *>(distinct_node->step.get());
         chassert(distinct_step);
 
-        std::vector<ActionsDAGPtr> dag_stack;
-        std::vector<std::vector<ActionsDAGPtr>> actions_chain;
+        std::vector<const ActionsDAG *> dag_stack;
+        std::vector<std::vector<const ActionsDAG *>> actions_chain;
         const DistinctStep * inner_distinct_step = nullptr;
         const IQueryPlanStep * aggregation_before_distinct = nullptr;
         const QueryPlan::Node * node = distinct_node;
@@ -182,9 +183,9 @@ namespace
             }
 
             if (const auto * const expr = typeid_cast<const ExpressionStep *>(current_step); expr)
-                dag_stack.push_back(expr->getExpression());
+                dag_stack.push_back(expr->getExpression().get());
             else if (const auto * const filter = typeid_cast<const FilterStep *>(current_step); filter)
-                dag_stack.push_back(filter->getExpression());
+                dag_stack.push_back(filter->getExpression().get());
 
             node = node->children.front();
             if (inner_distinct_step = typeid_cast<DistinctStep *>(node->step.get()); inner_distinct_step)
@@ -222,7 +223,7 @@ namespace
         chassert(distinct_step);
         const auto distinct_columns = getDistinctColumns(distinct_step);
 
-        std::vector<ActionsDAGPtr> dag_stack;
+        std::vector<const ActionsDAG *> dag_stack;
         const DistinctStep * inner_distinct_step = nullptr;
         const QueryPlan::Node * node = distinct_node;
         while (!node->children.empty())
@@ -235,9 +236,9 @@ namespace
             }
 
             if (const auto * const expr = typeid_cast<const ExpressionStep *>(current_step); expr)
-                dag_stack.push_back(expr->getExpression());
+                dag_stack.push_back(expr->getExpression().get());
             else if (const auto * const filter = typeid_cast<const FilterStep *>(current_step); filter)
-                dag_stack.push_back(filter->getExpression());
+                dag_stack.push_back(filter->getExpression().get());
 
             node = node->children.front();
             inner_distinct_step = typeid_cast<DistinctStep *>(node->step.get());
@@ -267,7 +268,7 @@ namespace
             logActionsDAG("distinct pass: merged DAG", path_actions);
 
             /// compare columns of two DISTINCTs
-            FindOriginalNodeForOutputName original_node_finder(path_actions);
+            FindOriginalNodeForOutputName original_node_finder(*path_actions);
             for (const auto & column : distinct_columns)
             {
                 const auto * alias_node = original_node_finder.find(String(column));
diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp
index b2d8aa0e218..e5370c1c130 100644
--- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp
+++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp
@@ -174,7 +174,7 @@ static void updateSortDescriptionForOutputStream(
     {
         if (prewhere_info->prewhere_actions)
         {
-            FindOriginalNodeForOutputName original_column_finder(prewhere_info->prewhere_actions);
+            FindOriginalNodeForOutputName original_column_finder(*prewhere_info->prewhere_actions);
             for (auto & column : original_header)
             {
                 const auto * original_node = original_column_finder.find(column.name);
@@ -185,7 +185,7 @@ static void updateSortDescriptionForOutputStream(
 
         if (prewhere_info->row_level_filter)
         {
-            FindOriginalNodeForOutputName original_column_finder(prewhere_info->row_level_filter);
+            FindOriginalNodeForOutputName original_column_finder(*prewhere_info->row_level_filter);
             for (auto & column : original_header)
             {
                 const auto * original_node = original_column_finder.find(column.name);
@@ -830,10 +830,10 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreams(RangesInDataParts && parts_
             pipes[0].getHeader().getColumnsWithTypeAndName(),
             pipes[1].getHeader().getColumnsWithTypeAndName(),
             ActionsDAG::MatchColumnsMode::Name);
+        auto converting_expr = std::make_shared<ExpressionActions>(std::move(conversion_action));
         pipes[0].addSimpleTransform(
-            [conversion_action](const Block & header)
+            [converting_expr](const Block & header)
             {
-                auto converting_expr = std::make_shared<ExpressionActions>(conversion_action);
                 return std::make_shared<ExpressionTransform>(header, converting_expr);
             });
         return Pipe::unitePipes(std::move(pipes));
@@ -849,7 +849,7 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreams(RangesInDataParts && parts_
 
 static ActionsDAGPtr createProjection(const Block & header)
 {
-    return std::make_shared<ActionsDAG>(header.getNamesAndTypesList());
+    return std::make_unique<ActionsDAG>(header.getNamesAndTypesList());
 }
 
 Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsWithOrder(
@@ -1046,7 +1046,7 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsWithOrder(
         for (size_t j = 0; j < prefix_size; ++j)
             sort_description.emplace_back(sorting_columns[j], input_order_info->direction);
 
-        auto sorting_key_expr = std::make_shared<ExpressionActions>(sorting_key_prefix_expr);
+        auto sorting_key_expr = std::make_shared<ExpressionActions>(std::move(sorting_key_prefix_expr));
 
         auto merge_streams = [&](Pipe & pipe)
         {
@@ -1341,10 +1341,10 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal(
             pipes[0].getHeader().getColumnsWithTypeAndName(),
             pipes[1].getHeader().getColumnsWithTypeAndName(),
             ActionsDAG::MatchColumnsMode::Name);
+        auto converting_expr = std::make_shared<ExpressionActions>(std::move(conversion_action));
         pipes[0].addSimpleTransform(
-            [conversion_action](const Block & header)
+            [converting_expr](const Block & header)
             {
-                auto converting_expr = std::make_shared<ExpressionActions>(conversion_action);
                 return std::make_shared<ExpressionTransform>(header, converting_expr);
             });
         return Pipe::unitePipes(std::move(pipes));
@@ -1378,7 +1378,7 @@ ReadFromMergeTree::AnalysisResultPtr ReadFromMergeTree::selectRangesToRead(
 
 static void buildIndexes(
     std::optional<ReadFromMergeTree::Indexes> & indexes,
-    ActionsDAGPtr filter_actions_dag,
+    const ActionsDAG * filter_actions_dag,
     const MergeTreeData & data,
     const MergeTreeData::DataPartsVector & parts,
     const ContextPtr & context,
@@ -1518,11 +1518,11 @@ void ReadFromMergeTree::applyFilters(ActionDAGNodes added_filter_nodes)
         /// (1) SourceStepWithFilter::filter_nodes, (2) query_info.filter_actions_dag. Make sure there are consistent.
         /// TODO: Get rid of filter_actions_dag in query_info after we move analysis of
         /// parallel replicas and unused shards into optimization, similar to projection analysis.
-        query_info.filter_actions_dag = filter_actions_dag;
+        query_info.filter_actions_dag = std::move(filter_actions_dag);
 
         buildIndexes(
             indexes,
-            filter_actions_dag,
+            query_info.filter_actions_dag.get(),
             data,
             prepared_parts,
             context,
@@ -1564,7 +1564,7 @@ ReadFromMergeTree::AnalysisResultPtr ReadFromMergeTree::selectRangesToRead(
     const Names & primary_key_column_names = primary_key.column_names;
 
     if (!indexes)
-        buildIndexes(indexes, query_info_.filter_actions_dag, data, parts, context_, query_info_, metadata_snapshot);
+        buildIndexes(indexes, query_info_.filter_actions_dag.get(), data, parts, context_, query_info_, metadata_snapshot);
 
     if (indexes->part_values && indexes->part_values->empty())
         return std::make_shared<AnalysisResult>(std::move(result));
@@ -1993,7 +1993,7 @@ void ReadFromMergeTree::initializePipeline(QueryPipelineBuilder & pipeline, cons
 
     if (result.sampling.use_sampling)
     {
-        auto sampling_actions = std::make_shared<ExpressionActions>(result.sampling.filter_expression);
+        auto sampling_actions = std::make_shared<ExpressionActions>(result.sampling.filter_expression->clone());
         pipe.addSimpleTransform([&](const Block & header)
         {
             return std::make_shared<FilterTransform>(
@@ -2031,7 +2031,7 @@ void ReadFromMergeTree::initializePipeline(QueryPipelineBuilder & pipeline, cons
 
     if (result_projection)
     {
-        auto projection_actions = std::make_shared<ExpressionActions>(result_projection);
+        auto projection_actions = std::make_shared<ExpressionActions>(result_projection->clone());
         pipe.addSimpleTransform([&](const Block & header)
         {
             return std::make_shared<ExpressionTransform>(header, projection_actions);
@@ -2048,7 +2048,7 @@ void ReadFromMergeTree::initializePipeline(QueryPipelineBuilder & pipeline, cons
             ActionsDAG::MatchColumnsMode::Name,
             true);
 
-        auto converting_dag_expr = std::make_shared<ExpressionActions>(convert_actions_dag);
+        auto converting_dag_expr = std::make_shared<ExpressionActions>(std::move(convert_actions_dag));
 
         pipe.addSimpleTransform([&](const Block & header)
         {
@@ -2126,7 +2126,7 @@ void ReadFromMergeTree::describeActions(FormatSettings & format_settings) const
                format_settings.out << " (removed)";
             format_settings.out << '\n';
 
-            auto expression = std::make_shared<ExpressionActions>(prewhere_info->prewhere_actions);
+            auto expression = std::make_shared<ExpressionActions>(prewhere_info->prewhere_actions->clone());
             expression->describeActions(format_settings.out, prefix);
         }
 
@@ -2135,7 +2135,7 @@ void ReadFromMergeTree::describeActions(FormatSettings & format_settings) const
             format_settings.out << prefix << "Row level filter" << '\n';
             format_settings.out << prefix << "Row level filter column: " << prewhere_info->row_level_column_name << '\n';
 
-            auto expression = std::make_shared<ExpressionActions>(prewhere_info->row_level_filter);
+            auto expression = std::make_shared<ExpressionActions>(prewhere_info->row_level_filter->clone());
             expression->describeActions(format_settings.out, prefix);
         }
     }
@@ -2161,7 +2161,7 @@ void ReadFromMergeTree::describeActions(JSONBuilder::JSONMap & map) const
             std::unique_ptr<JSONBuilder::JSONMap> prewhere_filter_map = std::make_unique<JSONBuilder::JSONMap>();
             prewhere_filter_map->add("Prewhere filter column", prewhere_info->prewhere_column_name);
             prewhere_filter_map->add("Prewhere filter remove filter column", prewhere_info->remove_prewhere_column);
-            auto expression = std::make_shared<ExpressionActions>(prewhere_info->prewhere_actions);
+            auto expression = std::make_shared<ExpressionActions>(prewhere_info->prewhere_actions->clone());
             prewhere_filter_map->add("Prewhere filter expression", expression->toTree());
 
             prewhere_info_map->add("Prewhere filter", std::move(prewhere_filter_map));
@@ -2171,7 +2171,7 @@ void ReadFromMergeTree::describeActions(JSONBuilder::JSONMap & map) const
         {
             std::unique_ptr<JSONBuilder::JSONMap> row_level_filter_map = std::make_unique<JSONBuilder::JSONMap>();
             row_level_filter_map->add("Row level filter column", prewhere_info->row_level_column_name);
-            auto expression = std::make_shared<ExpressionActions>(prewhere_info->row_level_filter);
+            auto expression = std::make_shared<ExpressionActions>(prewhere_info->row_level_filter->clone());
             row_level_filter_map->add("Row level filter expression", expression->toTree());
 
             prewhere_info_map->add("Row level filter", std::move(row_level_filter_map));
diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.h b/src/Processors/QueryPlan/ReadFromMergeTree.h
index caa8aa2e1bd..e32507e1f22 100644
--- a/src/Processors/QueryPlan/ReadFromMergeTree.h
+++ b/src/Processors/QueryPlan/ReadFromMergeTree.h
@@ -23,7 +23,7 @@ struct MergeTreeDataSelectSamplingData
     bool read_nothing = false;
     Float64 used_sample_factor = 1.0;
     std::shared_ptr<ASTFunction> filter_function;
-    ActionsDAGPtr filter_expression;
+    std::shared_ptr<const ActionsDAG> filter_expression;
 };
 
 struct UsefulSkipIndexes
diff --git a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp
index 11371578c79..b9b239c721b 100644
--- a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp
+++ b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp
@@ -441,7 +441,7 @@ Pipe ReadFromSystemNumbersStep::makePipe()
     chassert(numbers_storage.step != UInt64{0});
 
     /// Build rpn of query filters
-    KeyCondition condition(filter_actions_dag, context, column_names, key_expression);
+    KeyCondition condition(filter_actions_dag.get(), context, column_names, key_expression);
 
     if (condition.extractPlainRanges(ranges))
     {
diff --git a/src/Processors/QueryPlan/SourceStepWithFilter.cpp b/src/Processors/QueryPlan/SourceStepWithFilter.cpp
index ad0940b90b9..b91debc8239 100644
--- a/src/Processors/QueryPlan/SourceStepWithFilter.cpp
+++ b/src/Processors/QueryPlan/SourceStepWithFilter.cpp
@@ -110,7 +110,7 @@ void SourceStepWithFilter::describeActions(FormatSettings & format_settings) con
                 format_settings.out << " (removed)";
             format_settings.out << '\n';
 
-            auto expression = std::make_shared<ExpressionActions>(prewhere_info->prewhere_actions);
+            auto expression = std::make_shared<ExpressionActions>(prewhere_info->prewhere_actions->clone());
             expression->describeActions(format_settings.out, prefix);
         }
 
@@ -119,7 +119,7 @@ void SourceStepWithFilter::describeActions(FormatSettings & format_settings) con
             format_settings.out << prefix << "Row level filter" << '\n';
             format_settings.out << prefix << "Row level filter column: " << prewhere_info->row_level_column_name << '\n';
 
-            auto expression = std::make_shared<ExpressionActions>(prewhere_info->row_level_filter);
+            auto expression = std::make_shared<ExpressionActions>(prewhere_info->row_level_filter->clone());
             expression->describeActions(format_settings.out, prefix);
         }
     }
@@ -137,7 +137,7 @@ void SourceStepWithFilter::describeActions(JSONBuilder::JSONMap & map) const
             std::unique_ptr<JSONBuilder::JSONMap> prewhere_filter_map = std::make_unique<JSONBuilder::JSONMap>();
             prewhere_filter_map->add("Prewhere filter column", prewhere_info->prewhere_column_name);
             prewhere_filter_map->add("Prewhere filter remove filter column", prewhere_info->remove_prewhere_column);
-            auto expression = std::make_shared<ExpressionActions>(prewhere_info->prewhere_actions);
+            auto expression = std::make_shared<ExpressionActions>(prewhere_info->prewhere_actions->clone());
             prewhere_filter_map->add("Prewhere filter expression", expression->toTree());
 
             prewhere_info_map->add("Prewhere filter", std::move(prewhere_filter_map));
@@ -147,7 +147,7 @@ void SourceStepWithFilter::describeActions(JSONBuilder::JSONMap & map) const
         {
             std::unique_ptr<JSONBuilder::JSONMap> row_level_filter_map = std::make_unique<JSONBuilder::JSONMap>();
             row_level_filter_map->add("Row level filter column", prewhere_info->row_level_column_name);
-            auto expression = std::make_shared<ExpressionActions>(prewhere_info->row_level_filter);
+            auto expression = std::make_shared<ExpressionActions>(prewhere_info->row_level_filter->clone());
             row_level_filter_map->add("Row level filter expression", expression->toTree());
 
             prewhere_info_map->add("Row level filter", std::move(row_level_filter_map));
diff --git a/src/Processors/QueryPlan/SourceStepWithFilter.h b/src/Processors/QueryPlan/SourceStepWithFilter.h
index 126d4824fff..8ac0cc24ed1 100644
--- a/src/Processors/QueryPlan/SourceStepWithFilter.h
+++ b/src/Processors/QueryPlan/SourceStepWithFilter.h
@@ -33,6 +33,7 @@ public:
     }
 
     const ActionsDAGPtr & getFilterActionsDAG() const { return filter_actions_dag; }
+    ActionsDAGPtr detachFilterActionsDAG() { return std::move(filter_actions_dag); }
 
     const SelectQueryInfo & getQueryInfo() const { return query_info; }
     const PrewhereInfoPtr & getPrewhereInfo() const { return prewhere_info; }
@@ -53,7 +54,7 @@ public:
     void applyFilters()
     {
         applyFilters(std::move(filter_nodes));
-        filter_dags = {};
+        filter_dags.clear();
     }
 
     virtual void applyFilters(ActionDAGNodes added_filter_nodes);
diff --git a/src/Processors/QueryPlan/TotalsHavingStep.cpp b/src/Processors/QueryPlan/TotalsHavingStep.cpp
index ac5e144bf4a..45de6c31d24 100644
--- a/src/Processors/QueryPlan/TotalsHavingStep.cpp
+++ b/src/Processors/QueryPlan/TotalsHavingStep.cpp
@@ -46,7 +46,7 @@ TotalsHavingStep::TotalsHavingStep(
         getTraits(!filter_column_.empty()))
     , aggregates(aggregates_)
     , overflow_row(overflow_row_)
-    , actions_dag(actions_dag_)
+    , actions_dag(actions_dag_->clone())
     , filter_column_name(filter_column_)
     , remove_filter(remove_filter_)
     , totals_mode(totals_mode_)
@@ -57,7 +57,7 @@ TotalsHavingStep::TotalsHavingStep(
 
 void TotalsHavingStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings & settings)
 {
-    auto expression_actions = actions_dag ? std::make_shared<ExpressionActions>(actions_dag, settings.getActionsSettings()) : nullptr;
+    auto expression_actions = actions_dag ? std::make_shared<ExpressionActions>(actions_dag->clone(), settings.getActionsSettings()) : nullptr;
 
     auto totals_having = std::make_shared<TotalsHavingTransform>(
         pipeline.getHeader(),
@@ -100,7 +100,7 @@ void TotalsHavingStep::describeActions(FormatSettings & settings) const
     if (actions_dag)
     {
         bool first = true;
-        auto expression = std::make_shared<ExpressionActions>(actions_dag);
+        auto expression = std::make_shared<ExpressionActions>(actions_dag->clone());
         for (const auto & action : expression->getActions())
         {
             settings.out << prefix << (first ? "Actions: "
@@ -117,7 +117,7 @@ void TotalsHavingStep::describeActions(JSONBuilder::JSONMap & map) const
     if (actions_dag)
     {
         map.add("Filter column", filter_column_name);
-        auto expression = std::make_shared<ExpressionActions>(actions_dag);
+        auto expression = std::make_shared<ExpressionActions>(actions_dag->clone());
         map.add("Expression", expression->toTree());
     }
 }
diff --git a/src/Processors/QueryPlan/TotalsHavingStep.h b/src/Processors/QueryPlan/TotalsHavingStep.h
index a81bc7bb1a9..52ef5437701 100644
--- a/src/Processors/QueryPlan/TotalsHavingStep.h
+++ b/src/Processors/QueryPlan/TotalsHavingStep.h
@@ -6,7 +6,7 @@ namespace DB
 {
 
 class ActionsDAG;
-using ActionsDAGPtr = std::shared_ptr<ActionsDAG>;
+using ActionsDAGPtr = std::unique_ptr<ActionsDAG>;
 
 enum class TotalsMode : uint8_t;
 
diff --git a/src/Processors/QueryPlan/WindowStep.h b/src/Processors/QueryPlan/WindowStep.h
index 74a0e5930c7..47883e5edf6 100644
--- a/src/Processors/QueryPlan/WindowStep.h
+++ b/src/Processors/QueryPlan/WindowStep.h
@@ -7,7 +7,7 @@ namespace DB
 {
 
 class ActionsDAG;
-using ActionsDAGPtr = std::shared_ptr<ActionsDAG>;
+using ActionsDAGPtr = std::unique_ptr<ActionsDAG>;
 
 class WindowTransform;
 
diff --git a/src/Processors/SourceWithKeyCondition.h b/src/Processors/SourceWithKeyCondition.h
index ee155d6f78c..fcf576637ff 100644
--- a/src/Processors/SourceWithKeyCondition.h
+++ b/src/Processors/SourceWithKeyCondition.h
@@ -16,13 +16,13 @@ protected:
     /// Represents pushed down filters in source
     std::shared_ptr<const KeyCondition> key_condition;
 
-    void setKeyConditionImpl(const ActionsDAGPtr & filter_actions_dag, ContextPtr context, const Block & keys)
+    void setKeyConditionImpl(const ActionsDAG * filter_actions_dag, ContextPtr context, const Block & keys)
     {
         key_condition = std::make_shared<const KeyCondition>(
             filter_actions_dag,
             context,
             keys.getNames(),
-            std::make_shared<ExpressionActions>(std::make_shared<ActionsDAG>(keys.getColumnsWithTypeAndName())));
+            std::make_shared<ExpressionActions>(std::make_unique<ActionsDAG>(keys.getColumnsWithTypeAndName())));
     }
 
 public:
diff --git a/src/Processors/Transforms/FillingTransform.cpp b/src/Processors/Transforms/FillingTransform.cpp
index bb38c3e1dc5..95267bc24e0 100644
--- a/src/Processors/Transforms/FillingTransform.cpp
+++ b/src/Processors/Transforms/FillingTransform.cpp
@@ -203,7 +203,7 @@ FillingTransform::FillingTransform(
     , use_with_fill_by_sorting_prefix(use_with_fill_by_sorting_prefix_)
 {
     if (interpolate_description)
-        interpolate_actions = std::make_shared<ExpressionActions>(interpolate_description->actions);
+        interpolate_actions = std::make_shared<ExpressionActions>(interpolate_description->actions->clone());
 
     std::vector<bool> is_fill_column(header_.columns());
     for (size_t i = 0, size = fill_description.size(); i < size; ++i)
diff --git a/src/Storages/Hive/StorageHive.cpp b/src/Storages/Hive/StorageHive.cpp
index 28d8128e052..2cd51259549 100644
--- a/src/Storages/Hive/StorageHive.cpp
+++ b/src/Storages/Hive/StorageHive.cpp
@@ -516,7 +516,7 @@ void StorageHive::initMinMaxIndexExpression()
         partition_names = partition_name_types.getNames();
         partition_types = partition_name_types.getTypes();
         partition_minmax_idx_expr = std::make_shared<ExpressionActions>(
-            std::make_shared<ActionsDAG>(partition_name_types), ExpressionActionsSettings::fromContext(getContext()));
+            std::make_unique<ActionsDAG>(partition_name_types), ExpressionActionsSettings::fromContext(getContext()));
     }
 
     NamesAndTypesList all_name_types = metadata_snapshot->getColumns().getAllPhysical();
@@ -526,7 +526,7 @@ void StorageHive::initMinMaxIndexExpression()
             hivefile_name_types.push_back(column);
     }
     hivefile_minmax_idx_expr = std::make_shared<ExpressionActions>(
-        std::make_shared<ActionsDAG>(hivefile_name_types), ExpressionActionsSettings::fromContext(getContext()));
+        std::make_unique<ActionsDAG>(hivefile_name_types), ExpressionActionsSettings::fromContext(getContext()));
 }
 
 ASTPtr StorageHive::extractKeyExpressionList(const ASTPtr & node)
@@ -647,7 +647,7 @@ HiveFiles StorageHive::collectHiveFilesFromPartition(
         for (size_t i = 0; i < partition_names.size(); ++i)
             ranges.emplace_back(fields[i]);
 
-        const KeyCondition partition_key_condition(filter_actions_dag, getContext(), partition_names, partition_minmax_idx_expr);
+        const KeyCondition partition_key_condition(filter_actions_dag.get(), getContext(), partition_names, partition_minmax_idx_expr);
         if (!partition_key_condition.checkInHyperrectangle(ranges, partition_types).can_be_true)
             return {};
     }
@@ -715,7 +715,7 @@ HiveFilePtr StorageHive::getHiveFileIfNeeded(
 
     if (prune_level >= PruneLevel::File)
     {
-        const KeyCondition hivefile_key_condition(filter_actions_dag, getContext(), hivefile_name_types.getNames(), hivefile_minmax_idx_expr);
+        const KeyCondition hivefile_key_condition(filter_actions_dag.get(), getContext(), hivefile_name_types.getNames(), hivefile_minmax_idx_expr);
         if (hive_file->useFileMinMaxIndex())
         {
             /// Load file level minmax index and apply
diff --git a/src/Storages/KeyDescription.cpp b/src/Storages/KeyDescription.cpp
index 2a697fa5654..e03ecc05064 100644
--- a/src/Storages/KeyDescription.cpp
+++ b/src/Storages/KeyDescription.cpp
@@ -160,7 +160,7 @@ KeyDescription KeyDescription::buildEmptyKey()
 {
     KeyDescription result;
     result.expression_list_ast = std::make_shared<ASTExpressionList>();
-    result.expression = std::make_shared<ExpressionActions>(std::make_shared<ActionsDAG>(), ExpressionActionsSettings{});
+    result.expression = std::make_shared<ExpressionActions>(std::make_unique<ActionsDAG>(), ExpressionActionsSettings{});
     return result;
 }
 
diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp
index f8cf19120c7..48ec5529af0 100644
--- a/src/Storages/MergeTree/KeyCondition.cpp
+++ b/src/Storages/MergeTree/KeyCondition.cpp
@@ -691,7 +691,7 @@ static const ActionsDAG::Node & cloneASTWithInversionPushDown(
 
 ActionsDAGPtr KeyCondition::cloneASTWithInversionPushDown(ActionsDAG::NodeRawConstPtrs nodes, const ContextPtr & context)
 {
-    auto res = std::make_shared<ActionsDAG>();
+    auto res = std::make_unique<ActionsDAG>();
 
     std::unordered_map<const ActionsDAG::Node *, const ActionsDAG::Node *> to_inverted;
 
@@ -777,7 +777,7 @@ void KeyCondition::getAllSpaceFillingCurves()
 }
 
 KeyCondition::KeyCondition(
-    ActionsDAGPtr filter_dag,
+    const ActionsDAG * filter_dag,
     ContextPtr context,
     const Names & key_column_names,
     const ExpressionActionsPtr & key_expr_,
diff --git a/src/Storages/MergeTree/KeyCondition.h b/src/Storages/MergeTree/KeyCondition.h
index 2bc3b108e02..14ef74ea113 100644
--- a/src/Storages/MergeTree/KeyCondition.h
+++ b/src/Storages/MergeTree/KeyCondition.h
@@ -41,7 +41,7 @@ class KeyCondition
 public:
     /// Construct key condition from ActionsDAG nodes
     KeyCondition(
-        ActionsDAGPtr filter_dag,
+        const ActionsDAG * filter_dag,
         ContextPtr context,
         const Names & key_column_names,
         const ExpressionActionsPtr & key_expr,
diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index 2e0ea4cdbcd..7b642c34f37 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -487,7 +487,7 @@ ConditionSelectivityEstimator MergeTreeData::getConditionSelectivityEstimatorByP
     ASTPtr expression_ast;
 
     ConditionSelectivityEstimator result;
-    PartitionPruner partition_pruner(storage_snapshot->metadata, filter_dag, local_context);
+    PartitionPruner partition_pruner(storage_snapshot->metadata, filter_dag.get(), local_context);
 
     if (partition_pruner.isUseless())
     {
@@ -746,7 +746,7 @@ ExpressionActionsPtr MergeTreeData::getMinMaxExpr(const KeyDescription & partiti
     if (!partition_key.column_names.empty())
         partition_key_columns = partition_key.expression->getRequiredColumnsWithTypes();
 
-    return std::make_shared<ExpressionActions>(std::make_shared<ActionsDAG>(partition_key_columns), settings);
+    return std::make_shared<ExpressionActions>(std::make_unique<ActionsDAG>(partition_key_columns), settings);
 }
 
 Names MergeTreeData::getMinMaxColumnsNames(const KeyDescription & partition_key)
@@ -1152,7 +1152,7 @@ std::optional<UInt64> MergeTreeData::totalRowsByPartitionPredicateImpl(
         if (!virtual_columns_block.has(input->result_name))
             valid = false;
 
-    PartitionPruner partition_pruner(metadata_snapshot, filter_dag, local_context, true /* strict */);
+    PartitionPruner partition_pruner(metadata_snapshot, filter_dag.get(), local_context, true /* strict */);
     if (partition_pruner.isUseless() && !valid)
         return {};
 
@@ -6819,7 +6819,7 @@ using PartitionIdToMaxBlock = std::unordered_map<String, Int64>;
 Block MergeTreeData::getMinMaxCountProjectionBlock(
     const StorageMetadataPtr & metadata_snapshot,
     const Names & required_columns,
-    const ActionsDAGPtr & filter_dag,
+    const ActionsDAG * filter_dag,
     const DataPartsVector & parts,
     const PartitionIdToMaxBlock * max_block_numbers_to_read,
     ContextPtr query_context) const
diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h
index c6f736a4afd..52916d85fef 100644
--- a/src/Storages/MergeTree/MergeTreeData.h
+++ b/src/Storages/MergeTree/MergeTreeData.h
@@ -403,7 +403,7 @@ public:
     Block getMinMaxCountProjectionBlock(
         const StorageMetadataPtr & metadata_snapshot,
         const Names & required_columns,
-        const ActionsDAGPtr & filter_dag,
+        const ActionsDAG * filter_dag,
         const DataPartsVector & parts,
         const PartitionIdToMaxBlock * max_block_numbers_to_read,
         ContextPtr query_context) const;
diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp
index 2e287ff3042..61b8b6fdaa8 100644
--- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp
+++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp
@@ -442,7 +442,7 @@ MergeTreeDataSelectSamplingData MergeTreeDataSelectExecutor::getSampling(
 }
 
 void MergeTreeDataSelectExecutor::buildKeyConditionFromPartOffset(
-    std::optional<KeyCondition> & part_offset_condition, const ActionsDAGPtr & filter_dag, ContextPtr context)
+    std::optional<KeyCondition> & part_offset_condition, const ActionsDAG * filter_dag, ContextPtr context)
 {
     if (!filter_dag)
         return;
@@ -463,10 +463,10 @@ void MergeTreeDataSelectExecutor::buildKeyConditionFromPartOffset(
         return;
 
     part_offset_condition.emplace(KeyCondition{
-        dag,
+        dag.get(),
         context,
         sample.getNames(),
-        std::make_shared<ExpressionActions>(std::make_shared<ActionsDAG>(sample.getColumnsWithTypeAndName()), ExpressionActionsSettings{}),
+        std::make_shared<ExpressionActions>(std::make_unique<ActionsDAG>(sample.getColumnsWithTypeAndName()), ExpressionActionsSettings{}),
         {}});
 }
 
@@ -474,7 +474,7 @@ std::optional<std::unordered_set<String>> MergeTreeDataSelectExecutor::filterPar
     const StorageMetadataPtr & metadata_snapshot,
     const MergeTreeData & data,
     const MergeTreeData::DataPartsVector & parts,
-    const ActionsDAGPtr & filter_dag,
+    const ActionsDAG * filter_dag,
     ContextPtr context)
 {
     if (!filter_dag)
diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h
index 788355c1e59..39bff5eacd6 100644
--- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h
+++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h
@@ -161,7 +161,7 @@ public:
 
     /// If possible, construct optional key condition from predicates containing _part_offset column.
     static void buildKeyConditionFromPartOffset(
-        std::optional<KeyCondition> & part_offset_condition, const ActionsDAGPtr & filter_dag, ContextPtr context);
+        std::optional<KeyCondition> & part_offset_condition, const ActionsDAG * filter_dag, ContextPtr context);
 
     /// If possible, filter using expression on virtual columns.
     /// Example: SELECT count() FROM table WHERE _part = 'part_name'
@@ -170,7 +170,7 @@ public:
         const StorageMetadataPtr & metadata_snapshot,
         const MergeTreeData & data,
         const MergeTreeData::DataPartsVector & parts,
-        const ActionsDAGPtr & filter_dag,
+        const ActionsDAG * filter_dag,
         ContextPtr context);
 
     /// Filter parts using minmax index and partition key.
diff --git a/src/Storages/MergeTree/MergeTreeIndexAnnoy.cpp b/src/Storages/MergeTree/MergeTreeIndexAnnoy.cpp
index e492ca0aec2..457c85eaa46 100644
--- a/src/Storages/MergeTree/MergeTreeIndexAnnoy.cpp
+++ b/src/Storages/MergeTree/MergeTreeIndexAnnoy.cpp
@@ -332,7 +332,7 @@ MergeTreeIndexConditionPtr MergeTreeIndexAnnoy::createIndexCondition(const Selec
     return std::make_shared<MergeTreeIndexConditionAnnoy>(index, query, distance_function, context);
 };
 
-MergeTreeIndexConditionPtr MergeTreeIndexAnnoy::createIndexCondition(const ActionsDAGPtr &, ContextPtr) const
+MergeTreeIndexConditionPtr MergeTreeIndexAnnoy::createIndexCondition(const ActionsDAG *, ContextPtr) const
 {
     throw Exception(ErrorCodes::NOT_IMPLEMENTED, "MergeTreeIndexAnnoy cannot be created with ActionsDAG");
 }
diff --git a/src/Storages/MergeTree/MergeTreeIndexAnnoy.h b/src/Storages/MergeTree/MergeTreeIndexAnnoy.h
index d511ab84859..282920c608e 100644
--- a/src/Storages/MergeTree/MergeTreeIndexAnnoy.h
+++ b/src/Storages/MergeTree/MergeTreeIndexAnnoy.h
@@ -99,7 +99,7 @@ public:
     MergeTreeIndexGranulePtr createIndexGranule() const override;
     MergeTreeIndexAggregatorPtr createIndexAggregator(const MergeTreeWriterSettings & settings) const override;
     MergeTreeIndexConditionPtr createIndexCondition(const SelectQueryInfo & query, ContextPtr context) const;
-    MergeTreeIndexConditionPtr createIndexCondition(const ActionsDAGPtr &, ContextPtr) const override;
+    MergeTreeIndexConditionPtr createIndexCondition(const ActionsDAG *, ContextPtr) const override;
     bool isVectorSearch() const override { return true; }
 
 private:
diff --git a/src/Storages/MergeTree/MergeTreeIndexBloomFilter.cpp b/src/Storages/MergeTree/MergeTreeIndexBloomFilter.cpp
index fc5147bb56c..c6a00751f25 100644
--- a/src/Storages/MergeTree/MergeTreeIndexBloomFilter.cpp
+++ b/src/Storages/MergeTree/MergeTreeIndexBloomFilter.cpp
@@ -201,7 +201,7 @@ bool maybeTrueOnBloomFilter(const IColumn * hash_column, const BloomFilterPtr &
 }
 
 MergeTreeIndexConditionBloomFilter::MergeTreeIndexConditionBloomFilter(
-    const ActionsDAGPtr & filter_actions_dag, ContextPtr context_, const Block & header_, size_t hash_functions_)
+    const ActionsDAG * filter_actions_dag, ContextPtr context_, const Block & header_, size_t hash_functions_)
     : WithContext(context_), header(header_), hash_functions(hash_functions_)
 {
     if (!filter_actions_dag)
@@ -897,7 +897,7 @@ MergeTreeIndexAggregatorPtr MergeTreeIndexBloomFilter::createIndexAggregator(con
     return std::make_shared<MergeTreeIndexAggregatorBloomFilter>(bits_per_row, hash_functions, index.column_names);
 }
 
-MergeTreeIndexConditionPtr MergeTreeIndexBloomFilter::createIndexCondition(const ActionsDAGPtr & filter_actions_dag, ContextPtr context) const
+MergeTreeIndexConditionPtr MergeTreeIndexBloomFilter::createIndexCondition(const ActionsDAG * filter_actions_dag, ContextPtr context) const
 {
     return std::make_shared<MergeTreeIndexConditionBloomFilter>(filter_actions_dag, context, index.sample_block, hash_functions);
 }
diff --git a/src/Storages/MergeTree/MergeTreeIndexBloomFilter.h b/src/Storages/MergeTree/MergeTreeIndexBloomFilter.h
index d66c4b8b6ca..bd1b137176a 100644
--- a/src/Storages/MergeTree/MergeTreeIndexBloomFilter.h
+++ b/src/Storages/MergeTree/MergeTreeIndexBloomFilter.h
@@ -69,7 +69,7 @@ public:
         std::vector<std::pair<size_t, ColumnPtr>> predicate;
     };
 
-    MergeTreeIndexConditionBloomFilter(const ActionsDAGPtr & filter_actions_dag, ContextPtr context_, const Block & header_, size_t hash_functions_);
+    MergeTreeIndexConditionBloomFilter(const ActionsDAG * filter_actions_dag, ContextPtr context_, const Block & header_, size_t hash_functions_);
 
     bool alwaysUnknownOrTrue() const override;
 
@@ -142,7 +142,7 @@ public:
 
     MergeTreeIndexAggregatorPtr createIndexAggregator(const MergeTreeWriterSettings & settings) const override;
 
-    MergeTreeIndexConditionPtr createIndexCondition(const ActionsDAGPtr & filter_actions_dag, ContextPtr context) const override;
+    MergeTreeIndexConditionPtr createIndexCondition(const ActionsDAG * filter_actions_dag, ContextPtr context) const override;
 
 private:
     size_t bits_per_row;
diff --git a/src/Storages/MergeTree/MergeTreeIndexBloomFilterText.cpp b/src/Storages/MergeTree/MergeTreeIndexBloomFilterText.cpp
index 8cf58687125..5b6813d12e3 100644
--- a/src/Storages/MergeTree/MergeTreeIndexBloomFilterText.cpp
+++ b/src/Storages/MergeTree/MergeTreeIndexBloomFilterText.cpp
@@ -138,7 +138,7 @@ void MergeTreeIndexAggregatorBloomFilterText::update(const Block & block, size_t
 }
 
 MergeTreeConditionBloomFilterText::MergeTreeConditionBloomFilterText(
-    const ActionsDAGPtr & filter_actions_dag,
+    const ActionsDAG * filter_actions_dag,
     ContextPtr context,
     const Block & index_sample_block,
     const BloomFilterParameters & params_,
@@ -733,7 +733,7 @@ MergeTreeIndexAggregatorPtr MergeTreeIndexBloomFilterText::createIndexAggregator
 }
 
 MergeTreeIndexConditionPtr MergeTreeIndexBloomFilterText::createIndexCondition(
-        const ActionsDAGPtr & filter_dag, ContextPtr context) const
+        const ActionsDAG * filter_dag, ContextPtr context) const
 {
     return std::make_shared<MergeTreeConditionBloomFilterText>(filter_dag, context, index.sample_block, params, token_extractor.get());
 }
diff --git a/src/Storages/MergeTree/MergeTreeIndexBloomFilterText.h b/src/Storages/MergeTree/MergeTreeIndexBloomFilterText.h
index 6fd969030df..fe042884550 100644
--- a/src/Storages/MergeTree/MergeTreeIndexBloomFilterText.h
+++ b/src/Storages/MergeTree/MergeTreeIndexBloomFilterText.h
@@ -62,7 +62,7 @@ class MergeTreeConditionBloomFilterText final : public IMergeTreeIndexCondition
 {
 public:
     MergeTreeConditionBloomFilterText(
-            const ActionsDAGPtr & filter_actions_dag,
+            const ActionsDAG * filter_actions_dag,
             ContextPtr context,
             const Block & index_sample_block,
             const BloomFilterParameters & params_,
@@ -163,7 +163,7 @@ public:
     MergeTreeIndexAggregatorPtr createIndexAggregator(const MergeTreeWriterSettings & settings) const override;
 
     MergeTreeIndexConditionPtr createIndexCondition(
-            const ActionsDAGPtr & filter_dag, ContextPtr context) const override;
+            const ActionsDAG * filter_dag, ContextPtr context) const override;
 
     BloomFilterParameters params;
     /// Function for selecting next token.
diff --git a/src/Storages/MergeTree/MergeTreeIndexFullText.cpp b/src/Storages/MergeTree/MergeTreeIndexFullText.cpp
index 47ce24b91eb..cd6af68ebcc 100644
--- a/src/Storages/MergeTree/MergeTreeIndexFullText.cpp
+++ b/src/Storages/MergeTree/MergeTreeIndexFullText.cpp
@@ -186,7 +186,7 @@ void MergeTreeIndexAggregatorFullText::update(const Block & block, size_t * pos,
 }
 
 MergeTreeConditionFullText::MergeTreeConditionFullText(
-    const ActionsDAGPtr & filter_actions_dag,
+    const ActionsDAG * filter_actions_dag,
     ContextPtr context_,
     const Block & index_sample_block,
     const GinFilterParameters & params_,
@@ -768,7 +768,7 @@ MergeTreeIndexAggregatorPtr MergeTreeIndexFullText::createIndexAggregatorForPart
 }
 
 MergeTreeIndexConditionPtr MergeTreeIndexFullText::createIndexCondition(
-        const ActionsDAGPtr & filter_actions_dag, ContextPtr context) const
+        const ActionsDAG * filter_actions_dag, ContextPtr context) const
 {
     return std::make_shared<MergeTreeConditionFullText>(filter_actions_dag, context, index.sample_block, params, token_extractor.get());
 };
diff --git a/src/Storages/MergeTree/MergeTreeIndexFullText.h b/src/Storages/MergeTree/MergeTreeIndexFullText.h
index 1a5e848e5ac..8e0b1a22acb 100644
--- a/src/Storages/MergeTree/MergeTreeIndexFullText.h
+++ b/src/Storages/MergeTree/MergeTreeIndexFullText.h
@@ -63,7 +63,7 @@ class MergeTreeConditionFullText final : public IMergeTreeIndexCondition, WithCo
 {
 public:
     MergeTreeConditionFullText(
-            const ActionsDAGPtr & filter_actions_dag,
+            const ActionsDAG * filter_actions_dag,
             ContextPtr context,
             const Block & index_sample_block,
             const GinFilterParameters & params_,
@@ -170,7 +170,7 @@ public:
     MergeTreeIndexGranulePtr createIndexGranule() const override;
     MergeTreeIndexAggregatorPtr createIndexAggregator(const MergeTreeWriterSettings & settings) const override;
     MergeTreeIndexAggregatorPtr createIndexAggregatorForPart(const GinIndexStorePtr & store, const MergeTreeWriterSettings & /*settings*/) const override;
-    MergeTreeIndexConditionPtr createIndexCondition(const ActionsDAGPtr & filter_actions_dag, ContextPtr context) const override;
+    MergeTreeIndexConditionPtr createIndexCondition(const ActionsDAG * filter_actions_dag, ContextPtr context) const override;
 
     GinFilterParameters params;
     /// Function for selecting next token.
diff --git a/src/Storages/MergeTree/MergeTreeIndexHypothesis.cpp b/src/Storages/MergeTree/MergeTreeIndexHypothesis.cpp
index 0995e2724ec..cd8065ecadf 100644
--- a/src/Storages/MergeTree/MergeTreeIndexHypothesis.cpp
+++ b/src/Storages/MergeTree/MergeTreeIndexHypothesis.cpp
@@ -79,7 +79,7 @@ MergeTreeIndexAggregatorPtr MergeTreeIndexHypothesis::createIndexAggregator(cons
 }
 
 MergeTreeIndexConditionPtr MergeTreeIndexHypothesis::createIndexCondition(
-    const ActionsDAGPtr &, ContextPtr) const
+    const ActionsDAG *, ContextPtr) const
 {
     throw Exception(ErrorCodes::LOGICAL_ERROR, "Not supported");
 }
diff --git a/src/Storages/MergeTree/MergeTreeIndexHypothesis.h b/src/Storages/MergeTree/MergeTreeIndexHypothesis.h
index 130e708d76f..e60335fe724 100644
--- a/src/Storages/MergeTree/MergeTreeIndexHypothesis.h
+++ b/src/Storages/MergeTree/MergeTreeIndexHypothesis.h
@@ -69,7 +69,7 @@ public:
     MergeTreeIndexAggregatorPtr createIndexAggregator(const MergeTreeWriterSettings & settings) const override;
 
     MergeTreeIndexConditionPtr createIndexCondition(
-        const ActionsDAGPtr & filter_actions_dag, ContextPtr context) const override;
+        const ActionsDAG * filter_actions_dag, ContextPtr context) const override;
 
     MergeTreeIndexMergedConditionPtr createIndexMergedCondition(
         const SelectQueryInfo & query_info, StorageMetadataPtr storage_metadata) const override;
diff --git a/src/Storages/MergeTree/MergeTreeIndexMinMax.cpp b/src/Storages/MergeTree/MergeTreeIndexMinMax.cpp
index 20dfed8cf8f..c60d63a59ba 100644
--- a/src/Storages/MergeTree/MergeTreeIndexMinMax.cpp
+++ b/src/Storages/MergeTree/MergeTreeIndexMinMax.cpp
@@ -157,7 +157,7 @@ void MergeTreeIndexAggregatorMinMax::update(const Block & block, size_t * pos, s
 namespace
 {
 
-KeyCondition buildCondition(const IndexDescription & index, const ActionsDAGPtr & filter_actions_dag, ContextPtr context)
+KeyCondition buildCondition(const IndexDescription & index, const ActionsDAG * filter_actions_dag, ContextPtr context)
 {
     return KeyCondition{filter_actions_dag, context, index.column_names, index.expression};
 }
@@ -165,7 +165,7 @@ KeyCondition buildCondition(const IndexDescription & index, const ActionsDAGPtr
 }
 
 MergeTreeIndexConditionMinMax::MergeTreeIndexConditionMinMax(
-    const IndexDescription & index, const ActionsDAGPtr & filter_actions_dag, ContextPtr context)
+    const IndexDescription & index, const ActionsDAG * filter_actions_dag, ContextPtr context)
     : index_data_types(index.data_types)
     , condition(buildCondition(index, filter_actions_dag, context))
 {
@@ -198,7 +198,7 @@ MergeTreeIndexAggregatorPtr MergeTreeIndexMinMax::createIndexAggregator(const Me
 }
 
 MergeTreeIndexConditionPtr MergeTreeIndexMinMax::createIndexCondition(
-    const ActionsDAGPtr & filter_actions_dag, ContextPtr context) const
+    const ActionsDAG * filter_actions_dag, ContextPtr context) const
 {
     return std::make_shared<MergeTreeIndexConditionMinMax>(index, filter_actions_dag, context);
 }
diff --git a/src/Storages/MergeTree/MergeTreeIndexMinMax.h b/src/Storages/MergeTree/MergeTreeIndexMinMax.h
index dca26fb7b28..c5031ccbb27 100644
--- a/src/Storages/MergeTree/MergeTreeIndexMinMax.h
+++ b/src/Storages/MergeTree/MergeTreeIndexMinMax.h
@@ -50,7 +50,7 @@ class MergeTreeIndexConditionMinMax final : public IMergeTreeIndexCondition
 public:
     MergeTreeIndexConditionMinMax(
         const IndexDescription & index,
-        const ActionsDAGPtr & filter_actions_dag,
+        const ActionsDAG * filter_actions_dag,
         ContextPtr context);
 
     bool alwaysUnknownOrTrue() const override;
@@ -77,7 +77,7 @@ public:
     MergeTreeIndexAggregatorPtr createIndexAggregator(const MergeTreeWriterSettings & settings) const override;
 
     MergeTreeIndexConditionPtr createIndexCondition(
-        const ActionsDAGPtr & filter_actions_dag, ContextPtr context) const override;
+        const ActionsDAG * filter_actions_dag, ContextPtr context) const override;
 
     const char* getSerializedFileExtension() const override { return ".idx2"; }
     MergeTreeIndexFormat getDeserializedFormat(const IDataPartStorage & data_part_storage, const std::string & path_prefix) const override; /// NOLINT
diff --git a/src/Storages/MergeTree/MergeTreeIndexSet.cpp b/src/Storages/MergeTree/MergeTreeIndexSet.cpp
index b11cbf1e034..7c65381b05b 100644
--- a/src/Storages/MergeTree/MergeTreeIndexSet.cpp
+++ b/src/Storages/MergeTree/MergeTreeIndexSet.cpp
@@ -245,7 +245,7 @@ MergeTreeIndexConditionSet::MergeTreeIndexConditionSet(
     const String & index_name_,
     const Block & index_sample_block,
     size_t max_rows_,
-    const ActionsDAGPtr & filter_dag,
+    const ActionsDAG * filter_dag,
     ContextPtr context)
     : index_name(index_name_)
     , max_rows(max_rows_)
@@ -272,9 +272,9 @@ MergeTreeIndexConditionSet::MergeTreeIndexConditionSet(
     filter_actions_dag->getOutputs()[0] = &traverseDAG(*filter_actions_dag_node, filter_actions_dag, context, node_to_result_node);
 
     filter_actions_dag->removeUnusedActions();
-    actions = std::make_shared<ExpressionActions>(filter_actions_dag);
 
     actions_output_column_name = filter_actions_dag->getOutputs().at(0)->result_name;
+    actions = std::make_shared<ExpressionActions>(std::move(filter_actions_dag));
 }
 
 bool MergeTreeIndexConditionSet::alwaysUnknownOrTrue() const
@@ -544,7 +544,7 @@ MergeTreeIndexAggregatorPtr MergeTreeIndexSet::createIndexAggregator(const Merge
 }
 
 MergeTreeIndexConditionPtr MergeTreeIndexSet::createIndexCondition(
-    const ActionsDAGPtr & filter_actions_dag, ContextPtr context) const
+    const ActionsDAG * filter_actions_dag, ContextPtr context) const
 {
     return std::make_shared<MergeTreeIndexConditionSet>(index.name, index.sample_block, max_rows, filter_actions_dag, context);
 }
diff --git a/src/Storages/MergeTree/MergeTreeIndexSet.h b/src/Storages/MergeTree/MergeTreeIndexSet.h
index 6efc2effafd..abd40b3cf9d 100644
--- a/src/Storages/MergeTree/MergeTreeIndexSet.h
+++ b/src/Storages/MergeTree/MergeTreeIndexSet.h
@@ -83,7 +83,7 @@ public:
         const String & index_name_,
         const Block & index_sample_block,
         size_t max_rows_,
-        const ActionsDAGPtr & filter_dag,
+        const ActionsDAG * filter_dag,
         ContextPtr context);
 
     bool alwaysUnknownOrTrue() const override;
@@ -138,7 +138,7 @@ public:
     MergeTreeIndexAggregatorPtr createIndexAggregator(const MergeTreeWriterSettings & settings) const override;
 
     MergeTreeIndexConditionPtr createIndexCondition(
-            const ActionsDAGPtr & filter_actions_dag, ContextPtr context) const override;
+            const ActionsDAG * filter_actions_dag, ContextPtr context) const override;
 
     size_t max_rows = 0;
 };
diff --git a/src/Storages/MergeTree/MergeTreeIndexUSearch.cpp b/src/Storages/MergeTree/MergeTreeIndexUSearch.cpp
index c9df7210569..59a4b0fbf9c 100644
--- a/src/Storages/MergeTree/MergeTreeIndexUSearch.cpp
+++ b/src/Storages/MergeTree/MergeTreeIndexUSearch.cpp
@@ -367,7 +367,7 @@ MergeTreeIndexConditionPtr MergeTreeIndexUSearch::createIndexCondition(const Sel
     return std::make_shared<MergeTreeIndexConditionUSearch>(index, query, distance_function, context);
 };
 
-MergeTreeIndexConditionPtr MergeTreeIndexUSearch::createIndexCondition(const ActionsDAGPtr &, ContextPtr) const
+MergeTreeIndexConditionPtr MergeTreeIndexUSearch::createIndexCondition(const ActionsDAG *, ContextPtr) const
 {
     throw Exception(ErrorCodes::NOT_IMPLEMENTED, "MergeTreeIndexAnnoy cannot be created with ActionsDAG");
 }
diff --git a/src/Storages/MergeTree/MergeTreeIndexUSearch.h b/src/Storages/MergeTree/MergeTreeIndexUSearch.h
index 5107cfee371..41de94402c9 100644
--- a/src/Storages/MergeTree/MergeTreeIndexUSearch.h
+++ b/src/Storages/MergeTree/MergeTreeIndexUSearch.h
@@ -101,7 +101,7 @@ public:
     MergeTreeIndexGranulePtr createIndexGranule() const override;
     MergeTreeIndexAggregatorPtr createIndexAggregator(const MergeTreeWriterSettings & settings) const override;
     MergeTreeIndexConditionPtr createIndexCondition(const SelectQueryInfo & query, ContextPtr context) const;
-    MergeTreeIndexConditionPtr createIndexCondition(const ActionsDAGPtr &, ContextPtr) const override;
+    MergeTreeIndexConditionPtr createIndexCondition(const ActionsDAG *, ContextPtr) const override;
     bool isVectorSearch() const override { return true; }
 
 private:
diff --git a/src/Storages/MergeTree/MergeTreeIndices.h b/src/Storages/MergeTree/MergeTreeIndices.h
index a9f1fa9378f..1be73e1c811 100644
--- a/src/Storages/MergeTree/MergeTreeIndices.h
+++ b/src/Storages/MergeTree/MergeTreeIndices.h
@@ -167,7 +167,7 @@ struct IMergeTreeIndex
     }
 
     virtual MergeTreeIndexConditionPtr createIndexCondition(
-        const ActionsDAGPtr & filter_actions_dag, ContextPtr context) const = 0;
+        const ActionsDAG * filter_actions_dag, ContextPtr context) const = 0;
 
     virtual bool isVectorSearch() const { return false; }
 
diff --git a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp
index 78b67de1a7e..8fa5b2cc955 100644
--- a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp
+++ b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp
@@ -80,7 +80,7 @@ PrewhereExprInfo MergeTreeSelectProcessor::getPrewhereActions(PrewhereInfoPtr pr
             PrewhereExprStep row_level_filter_step
             {
                 .type = PrewhereExprStep::Filter,
-                .actions = std::make_shared<ExpressionActions>(prewhere_info->row_level_filter, actions_settings),
+                .actions = std::make_shared<ExpressionActions>(prewhere_info->row_level_filter->clone(), actions_settings),
                 .filter_column_name = prewhere_info->row_level_column_name,
                 .remove_filter_column = true,
                 .need_filter = true,
@@ -96,7 +96,7 @@ PrewhereExprInfo MergeTreeSelectProcessor::getPrewhereActions(PrewhereInfoPtr pr
             PrewhereExprStep prewhere_step
             {
                 .type = PrewhereExprStep::Filter,
-                .actions = std::make_shared<ExpressionActions>(prewhere_info->prewhere_actions, actions_settings),
+                .actions = std::make_shared<ExpressionActions>(prewhere_info->prewhere_actions->clone(), actions_settings),
                 .filter_column_name = prewhere_info->prewhere_column_name,
                 .remove_filter_column = prewhere_info->remove_prewhere_column,
                 .need_filter = prewhere_info->need_filter,
diff --git a/src/Storages/MergeTree/MergeTreeSequentialSource.cpp b/src/Storages/MergeTree/MergeTreeSequentialSource.cpp
index 02f8d6f4f6a..98b35a3ca2c 100644
--- a/src/Storages/MergeTree/MergeTreeSequentialSource.cpp
+++ b/src/Storages/MergeTree/MergeTreeSequentialSource.cpp
@@ -376,7 +376,7 @@ public:
         {
             const auto & primary_key = storage_snapshot->metadata->getPrimaryKey();
             const Names & primary_key_column_names = primary_key.column_names;
-            KeyCondition key_condition(filter, context, primary_key_column_names, primary_key.expression);
+            KeyCondition key_condition(filter.get(), context, primary_key_column_names, primary_key.expression);
             LOG_DEBUG(log, "Key condition: {}", key_condition.toString());
 
             if (!key_condition.alwaysFalse())
@@ -437,7 +437,7 @@ void createReadFromPartStep(
     auto reading = std::make_unique<ReadFromPart>(type,
         storage, storage_snapshot, std::move(data_part),
         std::move(columns_to_read), apply_deleted_mask,
-        filter, std::move(context), log);
+        std::move(filter), std::move(context), log);
 
     plan.addStep(std::move(reading));
 }
diff --git a/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp b/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp
index 43e3b0c505a..25596b42951 100644
--- a/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp
+++ b/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp
@@ -50,7 +50,7 @@ void fillRequiredColumns(const ActionsDAG::Node * node, std::unordered_map<const
 /// This allows to avoid cloning the same sub-DAG into multiple step DAGs but reference previously cloned nodes from earlier steps.
 struct DAGNodeRef
 {
-    ActionsDAGPtr dag;
+    ActionsDAG * dag;
     const ActionsDAG::Node * node;
 };
 
@@ -62,7 +62,7 @@ using NodeNameToLastUsedStepMap = std::unordered_map<String, size_t>;
 const ActionsDAG::Node & addClonedDAGToDAG(
     size_t step,
     const ActionsDAG::Node * original_dag_node,
-    ActionsDAGPtr new_dag,
+    const ActionsDAGPtr & new_dag,
     OriginalToNewNodeMap & node_remap,
     NodeNameToLastUsedStepMap & node_to_step_map)
 {
@@ -72,7 +72,7 @@ const ActionsDAG::Node & addClonedDAGToDAG(
     {
         /// If the node is already in the new DAG, return it
         const auto & node_ref = node_remap.at(node_name);
-        if (node_ref.dag == new_dag)
+        if (node_ref.dag == new_dag.get())
             return *node_ref.node;
 
         /// If the node is known from the previous steps, add it as an input, except for constants
@@ -80,7 +80,7 @@ const ActionsDAG::Node & addClonedDAGToDAG(
         {
             node_ref.dag->addOrReplaceInOutputs(*node_ref.node);
             const auto & new_node = new_dag->addInput(node_ref.node->result_name, node_ref.node->result_type);
-            node_remap[node_name] = {new_dag, &new_node}; /// TODO: here we update the node reference. Is it always correct?
+            node_remap[node_name] = {new_dag.get(), &new_node}; /// TODO: here we update the node reference. Is it always correct?
 
             /// Remember the index of the last step which reuses this node.
             /// We cannot remove this node from the outputs before that step.
@@ -93,7 +93,7 @@ const ActionsDAG::Node & addClonedDAGToDAG(
     if (original_dag_node->type == ActionsDAG::ActionType::INPUT)
     {
         const auto & new_node = new_dag->addInput(original_dag_node->result_name, original_dag_node->result_type);
-        node_remap[node_name] = {new_dag, &new_node};
+        node_remap[node_name] = {new_dag.get(), &new_node};
         return new_node;
     }
 
@@ -102,7 +102,7 @@ const ActionsDAG::Node & addClonedDAGToDAG(
     {
         const auto & new_node = new_dag->addColumn(
             ColumnWithTypeAndName(original_dag_node->column, original_dag_node->result_type, original_dag_node->result_name));
-        node_remap[node_name] = {new_dag, &new_node};
+        node_remap[node_name] = {new_dag.get(), &new_node};
         return new_node;
     }
 
@@ -110,7 +110,7 @@ const ActionsDAG::Node & addClonedDAGToDAG(
     {
         const auto & alias_child = addClonedDAGToDAG(step, original_dag_node->children[0], new_dag, node_remap, node_to_step_map);
         const auto & new_node = new_dag->addAlias(alias_child, original_dag_node->result_name);
-        node_remap[node_name] = {new_dag, &new_node};
+        node_remap[node_name] = {new_dag.get(), &new_node};
         return new_node;
     }
 
@@ -125,7 +125,7 @@ const ActionsDAG::Node & addClonedDAGToDAG(
         }
 
         const auto & new_node = new_dag->addFunction(original_dag_node->function_base, new_children, original_dag_node->result_name);
-        node_remap[node_name] = {new_dag, &new_node};
+        node_remap[node_name] = {new_dag.get(), &new_node};
         return new_node;
     }
 
@@ -133,13 +133,13 @@ const ActionsDAG::Node & addClonedDAGToDAG(
 }
 
 const ActionsDAG::Node & addFunction(
-        ActionsDAGPtr new_dag,
+        const ActionsDAGPtr & new_dag,
         const FunctionOverloadResolverPtr & function,
         ActionsDAG::NodeRawConstPtrs children,
         OriginalToNewNodeMap & node_remap)
 {
     const auto & new_node = new_dag->addFunction(function, children, "");
-    node_remap[new_node.result_name] = {new_dag, &new_node};
+    node_remap[new_node.result_name] = {new_dag.get(), &new_node};
     return new_node;
 }
 
@@ -147,7 +147,7 @@ const ActionsDAG::Node & addFunction(
 /// This is different from ActionsDAG::addCast() because it set the name equal to the original name effectively hiding the value before cast,
 /// but it might be required for further steps with its original uncasted type.
 const ActionsDAG::Node & addCast(
-        ActionsDAGPtr dag,
+        const ActionsDAGPtr & dag,
         const ActionsDAG::Node & node_to_cast,
         const String & type_name,
         OriginalToNewNodeMap & node_remap)
@@ -173,7 +173,7 @@ const ActionsDAG::Node & addCast(
 /// 1. produces a result with the proper Nullable or non-Nullable UInt8 type and
 /// 2. makes sure that the result contains only 0 or 1 values even if the source column contains non-boolean values.
 const ActionsDAG::Node & addAndTrue(
-    ActionsDAGPtr dag,
+    const ActionsDAGPtr & dag,
     const ActionsDAG::Node & filter_node_to_normalize,
     OriginalToNewNodeMap & node_remap)
 {
@@ -258,7 +258,7 @@ bool tryBuildPrewhereSteps(PrewhereInfoPtr prewhere_info, const ExpressionAction
     for (size_t step_index = 0; step_index < condition_groups.size(); ++step_index)
     {
         const auto & condition_group = condition_groups[step_index];
-        ActionsDAGPtr step_dag = std::make_shared<ActionsDAG>();
+        ActionsDAGPtr step_dag = std::make_unique<ActionsDAG>();
         String result_name;
 
         std::vector<const ActionsDAG::Node *> new_condition_nodes;
@@ -299,7 +299,7 @@ bool tryBuildPrewhereSteps(PrewhereInfoPtr prewhere_info, const ExpressionAction
             }
         }
 
-        steps.push_back({step_dag, result_name});
+        steps.push_back({std::move(step_dag), result_name});
     }
 
     /// 6. Find all outputs of the original DAG
@@ -345,11 +345,11 @@ bool tryBuildPrewhereSteps(PrewhereInfoPtr prewhere_info, const ExpressionAction
     {
         for (size_t step_index = 0; step_index < steps.size(); ++step_index)
         {
-            const auto & step = steps[step_index];
+            auto & step = steps[step_index];
             PrewhereExprStep new_step
             {
                 .type = PrewhereExprStep::Filter,
-                .actions = std::make_shared<ExpressionActions>(step.actions, actions_settings),
+                .actions = std::make_shared<ExpressionActions>(std::move(step.actions), actions_settings),
                 .filter_column_name = step.column_name,
                 /// Don't remove if it's in the list of original outputs
                 .remove_filter_column =
diff --git a/src/Storages/MergeTree/PartitionPruner.cpp b/src/Storages/MergeTree/PartitionPruner.cpp
index 9de7b238f57..6df7b5aa054 100644
--- a/src/Storages/MergeTree/PartitionPruner.cpp
+++ b/src/Storages/MergeTree/PartitionPruner.cpp
@@ -4,7 +4,7 @@
 namespace DB
 {
 
-PartitionPruner::PartitionPruner(const StorageMetadataPtr & metadata, ActionsDAGPtr filter_actions_dag, ContextPtr context, bool strict)
+PartitionPruner::PartitionPruner(const StorageMetadataPtr & metadata, const ActionsDAG * filter_actions_dag, ContextPtr context, bool strict)
     : partition_key(MergeTreePartition::adjustPartitionKey(metadata, context))
     , partition_condition(filter_actions_dag, context, partition_key.column_names, partition_key.expression, true /* single_point */)
     , useless((strict && partition_condition.isRelaxed()) || partition_condition.alwaysUnknownOrTrue())
diff --git a/src/Storages/MergeTree/PartitionPruner.h b/src/Storages/MergeTree/PartitionPruner.h
index ca24559ca01..d89dfb7b245 100644
--- a/src/Storages/MergeTree/PartitionPruner.h
+++ b/src/Storages/MergeTree/PartitionPruner.h
@@ -13,7 +13,7 @@ namespace DB
 class PartitionPruner
 {
 public:
-    PartitionPruner(const StorageMetadataPtr & metadata, ActionsDAGPtr filter_actions_dag, ContextPtr context, bool strict = false);
+    PartitionPruner(const StorageMetadataPtr & metadata, const ActionsDAG * filter_actions_dag, ContextPtr context, bool strict = false);
 
     bool canBePruned(const IMergeTreeDataPart & part) const;
 
diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp
index 2fc6993369d..f640fb9ba0a 100644
--- a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp
+++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp
@@ -79,7 +79,7 @@ StorageObjectStorageSource::~StorageObjectStorageSource()
 
 void StorageObjectStorageSource::setKeyCondition(const ActionsDAGPtr & filter_actions_dag, ContextPtr context_)
 {
-    setKeyConditionImpl(filter_actions_dag, context_, read_from_format_info.format_header);
+    setKeyConditionImpl(filter_actions_dag.get(), context_, read_from_format_info.format_header);
 }
 
 std::string StorageObjectStorageSource::getUniqueStoragePathIdentifier(
diff --git a/src/Storages/SelectQueryInfo.h b/src/Storages/SelectQueryInfo.h
index 52b6674c93d..654b8b788fe 100644
--- a/src/Storages/SelectQueryInfo.h
+++ b/src/Storages/SelectQueryInfo.h
@@ -19,7 +19,7 @@ class ExpressionActions;
 using ExpressionActionsPtr = std::shared_ptr<ExpressionActions>;
 
 class ActionsDAG;
-using ActionsDAGPtr = std::shared_ptr<ActionsDAG>;
+using ActionsDAGPtr = std::unique_ptr<ActionsDAG>;
 
 struct PrewhereInfo;
 using PrewhereInfoPtr = std::shared_ptr<PrewhereInfo>;
@@ -192,7 +192,7 @@ struct SelectQueryInfo
     ASTPtr parallel_replica_custom_key_ast;
 
     /// Filter actions dag for current storage
-    ActionsDAGPtr filter_actions_dag;
+    std::shared_ptr<const ActionsDAG> filter_actions_dag;
 
     ReadInOrderOptimizerPtr order_optimizer;
     /// Can be modified while reading from storage
diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp
index a3f6b6afc5d..9bddf4f0230 100644
--- a/src/Storages/StorageBuffer.cpp
+++ b/src/Storages/StorageBuffer.cpp
@@ -432,7 +432,7 @@ void StorageBuffer::read(
                 {
                     return std::make_shared<FilterTransform>(
                             header,
-                            std::make_shared<ExpressionActions>(query_info.prewhere_info->row_level_filter, actions_settings),
+                            std::make_shared<ExpressionActions>(query_info.prewhere_info->row_level_filter->clone(), actions_settings),
                             query_info.prewhere_info->row_level_column_name,
                             false);
                 });
@@ -442,7 +442,7 @@ void StorageBuffer::read(
             {
                 return std::make_shared<FilterTransform>(
                         header,
-                        std::make_shared<ExpressionActions>(query_info.prewhere_info->prewhere_actions, actions_settings),
+                        std::make_shared<ExpressionActions>(query_info.prewhere_info->prewhere_actions->clone(), actions_settings),
                         query_info.prewhere_info->prewhere_column_name,
                         query_info.prewhere_info->remove_prewhere_column);
             });
diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp
index 5048ef4788e..be421e8e2bc 100644
--- a/src/Storages/StorageDistributed.cpp
+++ b/src/Storages/StorageDistributed.cpp
@@ -1120,7 +1120,7 @@ static ActionsDAGPtr getFilterFromQuery(const ASTPtr & ast, ContextPtr context)
     if (!source)
         return nullptr;
 
-    return source->getFilterActionsDAG();
+    return source->detachFilterActionsDAG();
 }
 
 
diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp
index aaf84f6f82c..702c257bfb6 100644
--- a/src/Storages/StorageFile.cpp
+++ b/src/Storages/StorageFile.cpp
@@ -1235,7 +1235,7 @@ StorageFileSource::~StorageFileSource()
 
 void StorageFileSource::setKeyCondition(const ActionsDAGPtr & filter_actions_dag, ContextPtr context_)
 {
-    setKeyConditionImpl(filter_actions_dag, context_, block_for_format);
+    setKeyConditionImpl(filter_actions_dag.get(), context_, block_for_format);
 }
 
 
diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp
index e2f92f08d7a..c42e3058347 100644
--- a/src/Storages/StorageMerge.cpp
+++ b/src/Storages/StorageMerge.cpp
@@ -1235,7 +1235,7 @@ ReadFromMerge::RowPolicyData::RowPolicyData(RowPolicyFilterPtr row_policy_filter
     auto expression_analyzer = ExpressionAnalyzer{expr, syntax_result, local_context};
 
     actions_dag = expression_analyzer.getActionsDAG(false /* add_aliases */, false /* project_result */);
-    filter_actions = std::make_shared<ExpressionActions>(actions_dag,
+    filter_actions = std::make_shared<ExpressionActions>(std::move(actions_dag),
         ExpressionActionsSettings::fromContext(local_context, CompileExpressions::yes));
     const auto & required_columns = filter_actions->getRequiredColumnsWithTypes();
     const auto & sample_block_columns = filter_actions->getSampleBlock().getNamesAndTypesList();
@@ -1273,12 +1273,12 @@ void ReadFromMerge::RowPolicyData::extendNames(Names & names) const
 
 void ReadFromMerge::RowPolicyData::addStorageFilter(SourceStepWithFilter * step) const
 {
-    step->addFilter(actions_dag, filter_column_name);
+    step->addFilter(actions_dag->clone(), filter_column_name);
 }
 
 void ReadFromMerge::RowPolicyData::addFilterTransform(QueryPlan & plan) const
 {
-    auto filter_step = std::make_unique<FilterStep>(plan.getCurrentDataStream(), actions_dag, filter_column_name, true /* remove filter column */);
+    auto filter_step = std::make_unique<FilterStep>(plan.getCurrentDataStream(), actions_dag->clone(), filter_column_name, true /* remove filter column */);
     plan.addStep(std::move(filter_step));
 }
 
@@ -1471,7 +1471,7 @@ void ReadFromMerge::convertAndFilterSourceStream(
         {
             pipe_columns.emplace_back(NameAndTypePair(alias.name, alias.type));
 
-            auto actions_dag = std::make_shared<ActionsDAG>(pipe_columns);
+            auto actions_dag = std::make_unique<ActionsDAG>(pipe_columns);
 
             QueryTreeNodePtr query_tree = buildQueryTree(alias.expression, local_context);
             query_tree->setAlias(alias.name);
@@ -1486,7 +1486,7 @@ void ReadFromMerge::convertAndFilterSourceStream(
                 throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected to have 1 output but got {}", nodes.size());
 
             actions_dag->addOrReplaceInOutputs(actions_dag->addAlias(*nodes.front(), alias.name));
-            auto expression_step = std::make_unique<ExpressionStep>(child.plan.getCurrentDataStream(), actions_dag);
+            auto expression_step = std::make_unique<ExpressionStep>(child.plan.getCurrentDataStream(), std::move(actions_dag));
             child.plan.addStep(std::move(expression_step));
         }
     }
diff --git a/src/Storages/StorageURL.h b/src/Storages/StorageURL.h
index f550ccb2bc4..c336f597f41 100644
--- a/src/Storages/StorageURL.h
+++ b/src/Storages/StorageURL.h
@@ -187,7 +187,7 @@ public:
 
     void setKeyCondition(const ActionsDAGPtr & filter_actions_dag, ContextPtr context_) override
     {
-        setKeyConditionImpl(filter_actions_dag, context_, block_for_format);
+        setKeyConditionImpl(filter_actions_dag.get(), context_, block_for_format);
     }
 
     Chunk generate() override;
diff --git a/src/Storages/StorageValues.cpp b/src/Storages/StorageValues.cpp
index 894b1404a21..4d73f8e5c87 100644
--- a/src/Storages/StorageValues.cpp
+++ b/src/Storages/StorageValues.cpp
@@ -48,14 +48,14 @@ Pipe StorageValues::read(
 
     if (!prepared_pipe.empty())
     {
-        auto dag = std::make_shared<ActionsDAG>(prepared_pipe.getHeader().getColumnsWithTypeAndName());
+        auto dag = std::make_unique<ActionsDAG>(prepared_pipe.getHeader().getColumnsWithTypeAndName());
         ActionsDAG::NodeRawConstPtrs outputs;
         outputs.reserve(column_names.size());
         for (const auto & name : column_names)
             outputs.push_back(dag->getOutputs()[prepared_pipe.getHeader().getPositionByName(name)]);
 
         dag->getOutputs().swap(outputs);
-        auto expression = std::make_shared<ExpressionActions>(dag);
+        auto expression = std::make_shared<ExpressionActions>(std::move(dag));
 
         prepared_pipe.addSimpleTransform([&](const Block & header)
         {
diff --git a/src/Storages/StorageView.cpp b/src/Storages/StorageView.cpp
index 016de94c17c..2c0d5c5ca85 100644
--- a/src/Storages/StorageView.cpp
+++ b/src/Storages/StorageView.cpp
@@ -177,7 +177,7 @@ void StorageView::read(
 
     /// It's expected that the columns read from storage are not constant.
     /// Because method 'getSampleBlockForColumns' is used to obtain a structure of result in InterpreterSelectQuery.
-    auto materializing_actions = std::make_shared<ActionsDAG>(query_plan.getCurrentDataStream().header.getColumnsWithTypeAndName());
+    auto materializing_actions = std::make_unique<ActionsDAG>(query_plan.getCurrentDataStream().header.getColumnsWithTypeAndName());
     materializing_actions->addMaterializingOutputActions();
 
     auto materializing = std::make_unique<ExpressionStep>(query_plan.getCurrentDataStream(), std::move(materializing_actions));
diff --git a/src/Storages/TTLDescription.cpp b/src/Storages/TTLDescription.cpp
index f831465277d..56f65b57367 100644
--- a/src/Storages/TTLDescription.cpp
+++ b/src/Storages/TTLDescription.cpp
@@ -183,7 +183,7 @@ static ExpressionAndSets buildExpressionAndSets(ASTPtr & ast, const NamesAndType
     dag->getOutputs() = {col};
     dag->removeUnusedActions();
 
-    result.expression = std::make_shared<ExpressionActions>(dag, ExpressionActionsSettings::fromContext(context_copy));
+    result.expression = std::make_shared<ExpressionActions>(std::move(dag), ExpressionActionsSettings::fromContext(context_copy));
     result.sets = analyzer.getPreparedSets();
 
     return result;
diff --git a/src/Storages/VirtualColumnUtils.cpp b/src/Storages/VirtualColumnUtils.cpp
index 778c9e13adb..6f7d1d4c39f 100644
--- a/src/Storages/VirtualColumnUtils.cpp
+++ b/src/Storages/VirtualColumnUtils.cpp
@@ -77,10 +77,10 @@ void buildSetsForDAG(const ActionsDAGPtr & dag, const ContextPtr & context)
     }
 }
 
-void filterBlockWithDAG(ActionsDAGPtr dag, Block & block, ContextPtr context)
+void filterBlockWithDAG(const ActionsDAGPtr & dag, Block & block, ContextPtr context)
 {
     buildSetsForDAG(dag, context);
-    auto actions = std::make_shared<ExpressionActions>(dag);
+    auto actions = std::make_shared<ExpressionActions>(dag->clone());
     Block block_with_filter = block;
     actions->execute(block_with_filter, /*dry_run=*/ false, /*allow_duplicates_in_input=*/ true);
 
diff --git a/src/Storages/VirtualColumnUtils.h b/src/Storages/VirtualColumnUtils.h
index fbfbdd6c6cc..0cf8470bc60 100644
--- a/src/Storages/VirtualColumnUtils.h
+++ b/src/Storages/VirtualColumnUtils.h
@@ -23,7 +23,7 @@ namespace VirtualColumnUtils
 void filterBlockWithPredicate(const ActionsDAG::Node * predicate, Block & block, ContextPtr context);
 
 /// Just filters block. Block should contain all the required columns.
-void filterBlockWithDAG(ActionsDAGPtr dag, Block & block, ContextPtr context);
+void filterBlockWithDAG(const ActionsDAGPtr & dag, Block & block, ContextPtr context);
 
 /// Builds sets used by ActionsDAG inplace.
 void buildSetsForDAG(const ActionsDAGPtr & dag, const ContextPtr & context);
diff --git a/src/Storages/WindowView/StorageWindowView.cpp b/src/Storages/WindowView/StorageWindowView.cpp
index 8bca1c97aad..da4e751a88a 100644
--- a/src/Storages/WindowView/StorageWindowView.cpp
+++ b/src/Storages/WindowView/StorageWindowView.cpp
@@ -567,7 +567,7 @@ std::pair<BlocksPtr, Block> StorageWindowView::getNewBlocks(UInt32 watermark)
     builder.addSimpleTransform([&](const Block & header)
     {
         return std::make_shared<FilterTransform>(
-            header, std::make_shared<ExpressionActions>(filter_expression), filter_function->getColumnName(), true);
+            header, std::make_shared<ExpressionActions>(std::move(filter_expression)), filter_function->getColumnName(), true);
     });
 
     /// Adding window column
@@ -592,7 +592,7 @@ std::pair<BlocksPtr, Block> StorageWindowView::getNewBlocks(UInt32 watermark)
         new_header.getColumnsWithTypeAndName(),
         ActionsDAG::MatchColumnsMode::Name);
     auto actions = std::make_shared<ExpressionActions>(
-        convert_actions_dag, ExpressionActionsSettings::fromContext(getContext(), CompileExpressions::yes));
+        std::move(convert_actions_dag), ExpressionActionsSettings::fromContext(getContext(), CompileExpressions::yes));
     builder.addSimpleTransform([&](const Block & stream_header)
     {
         return std::make_shared<ExpressionTransform>(stream_header, actions);
@@ -700,7 +700,7 @@ inline void StorageWindowView::fire(UInt32 watermark)
             getTargetTable()->getInMemoryMetadataPtr()->getColumns(),
             getContext(),
             getContext()->getSettingsRef().insert_null_as_default);
-        auto adding_missing_defaults_actions = std::make_shared<ExpressionActions>(adding_missing_defaults_dag);
+        auto adding_missing_defaults_actions = std::make_shared<ExpressionActions>(std::move(adding_missing_defaults_dag));
         pipe.addSimpleTransform([&](const Block & stream_header)
         {
             return std::make_shared<ExpressionTransform>(stream_header, adding_missing_defaults_actions);
@@ -711,7 +711,7 @@ inline void StorageWindowView::fire(UInt32 watermark)
             block_io.pipeline.getHeader().getColumnsWithTypeAndName(),
             ActionsDAG::MatchColumnsMode::Position);
         auto actions = std::make_shared<ExpressionActions>(
-            convert_actions_dag,
+            std::move(convert_actions_dag),
             ExpressionActionsSettings::fromContext(getContext(), CompileExpressions::yes));
         pipe.addSimpleTransform([&](const Block & stream_header)
         {
@@ -1475,7 +1475,7 @@ void StorageWindowView::writeIntoWindowView(
         pipe.addSimpleTransform([&](const Block & header)
         {
             return std::make_shared<FilterTransform>(
-                header, std::make_shared<ExpressionActions>(filter_expression),
+                header, std::make_shared<ExpressionActions>(std::move(filter_expression)),
                 filter_function->getColumnName(), true);
         });
     }
@@ -1583,7 +1583,7 @@ void StorageWindowView::writeIntoWindowView(
             output->getHeader().getColumnsWithTypeAndName(),
             ActionsDAG::MatchColumnsMode::Name);
         auto convert_actions = std::make_shared<ExpressionActions>(
-            convert_actions_dag, ExpressionActionsSettings::fromContext(local_context, CompileExpressions::yes));
+            std::move(convert_actions_dag), ExpressionActionsSettings::fromContext(local_context, CompileExpressions::yes));
 
         builder.addSimpleTransform([&](const Block & header) { return std::make_shared<ExpressionTransform>(header, convert_actions); });
     }

From e348186ba26072eb76e549cd2a0adcd801c92bc6 Mon Sep 17 00:00:00 2001
From: jsc0218 <jsc0218@gmail.com>
Date: Tue, 18 Jun 2024 21:30:37 +0000
Subject: [PATCH 0070/1488] clear hint in table engine and sources

---
 src/Access/Common/AccessRightsElement.cpp |  6 +-
 src/Access/Common/AccessRightsElement.h   |  1 +
 src/Access/ContextAccess.cpp              | 80 ++++++++++++++++++-----
 3 files changed, 70 insertions(+), 17 deletions(-)

diff --git a/src/Access/Common/AccessRightsElement.cpp b/src/Access/Common/AccessRightsElement.cpp
index 24ff4e7631b..2ee13d6b94f 100644
--- a/src/Access/Common/AccessRightsElement.cpp
+++ b/src/Access/Common/AccessRightsElement.cpp
@@ -224,7 +224,11 @@ void AccessRightsElement::replaceEmptyDatabase(const String & current_database)
 
 String AccessRightsElement::toString() const { return toStringImpl(*this, true); }
 String AccessRightsElement::toStringWithoutOptions() const { return toStringImpl(*this, false); }
-
+String AccessRightsElement::toStringWithoutONClause() const
+{
+    String result{access_flags.toKeywords().front()};
+    return result + " ON {db.table}";
+}
 
 bool AccessRightsElements::empty() const { return std::all_of(begin(), end(), [](const AccessRightsElement & e) { return e.empty(); }); }
 
diff --git a/src/Access/Common/AccessRightsElement.h b/src/Access/Common/AccessRightsElement.h
index ba625fc43df..49764fc727f 100644
--- a/src/Access/Common/AccessRightsElement.h
+++ b/src/Access/Common/AccessRightsElement.h
@@ -89,6 +89,7 @@ struct AccessRightsElement
     /// Returns a human-readable representation like "GRANT SELECT, UPDATE(x, y) ON db.table".
     String toString() const;
     String toStringWithoutOptions() const;
+    String toStringWithoutONClause() const;
 };
 
 
diff --git a/src/Access/ContextAccess.cpp b/src/Access/ContextAccess.cpp
index 2a658d7aaa2..3ce30a0b681 100644
--- a/src/Access/ContextAccess.cpp
+++ b/src/Access/ContextAccess.cpp
@@ -37,6 +37,24 @@ namespace ErrorCodes
 
 namespace
 {
+    static const std::vector<std::tuple<AccessFlags, std::string>> source_and_table_engines = {
+        {AccessType::FILE, "File"},
+        {AccessType::URL, "URL"},
+        {AccessType::REMOTE, "Distributed"},
+        {AccessType::MONGO, "MongoDB"},
+        {AccessType::REDIS, "Redis"},
+        {AccessType::MYSQL, "MySQL"},
+        {AccessType::POSTGRES, "PostgreSQL"},
+        {AccessType::SQLITE, "SQLite"},
+        {AccessType::ODBC, "ODBC"},
+        {AccessType::JDBC, "JDBC"},
+        {AccessType::HDFS, "HDFS"},
+        {AccessType::S3, "S3"},
+        {AccessType::HIVE, "Hive"},
+        {AccessType::AZURE, "AzureBlobStorage"}
+    };
+
+
     AccessRights mixAccessRightsFromUserAndRoles(const User & user, const EnabledRolesInfo & roles_info)
     {
         AccessRights res = user.access;
@@ -205,22 +223,6 @@ namespace
         }
 
         /// There is overlap between AccessType sources and table engines, so the following code avoids user granting twice.
-        static const std::vector<std::tuple<AccessFlags, std::string>> source_and_table_engines = {
-            {AccessType::FILE, "File"},
-            {AccessType::URL, "URL"},
-            {AccessType::REMOTE, "Distributed"},
-            {AccessType::MONGO, "MongoDB"},
-            {AccessType::REDIS, "Redis"},
-            {AccessType::MYSQL, "MySQL"},
-            {AccessType::POSTGRES, "PostgreSQL"},
-            {AccessType::SQLITE, "SQLite"},
-            {AccessType::ODBC, "ODBC"},
-            {AccessType::JDBC, "JDBC"},
-            {AccessType::HDFS, "HDFS"},
-            {AccessType::S3, "S3"},
-            {AccessType::HIVE, "Hive"},
-            {AccessType::AZURE, "AzureBlobStorage"}
-        };
 
         /// Sync SOURCE and TABLE_ENGINE, so only need to check TABLE_ENGINE later.
         if (access_control.doesTableEnginesRequireGrant())
@@ -555,6 +557,18 @@ std::shared_ptr<const AccessRights> ContextAccess::getAccessRightsWithImplicit()
     return nothing_granted;
 }
 
+/// Just Dummy to pass compile.
+template <typename... Args>
+static std::string_view getTableEngineName(const Args &... args[[maybe_unused]])
+{
+    return "";
+}
+
+template <typename... Args>
+static std::string_view getTableEngineName(std::string_view name, const Args &... args[[maybe_unused]])
+{
+    return name;
+}
 
 template <bool throw_if_denied, bool grant_option, typename... Args>
 bool ContextAccess::checkAccessImplHelper(AccessFlags flags, const Args &... args) const
@@ -611,6 +625,40 @@ bool ContextAccess::checkAccessImplHelper(AccessFlags flags, const Args &... arg
 
     if (!granted)
     {
+        /// As we check the SOURCES from the Table Engine logic, direct prompt about Table Engine would be misleading since 
+        /// SOURCES is not granted actually. In order to solve this, turn the prompt logic back to Sources.
+        if (flags & AccessType::TABLE_ENGINE && !access_control->doesTableEnginesRequireGrant())
+        {
+            AccessFlags newFlags;
+
+            String table_engine_name{getTableEngineName(args...)};
+            for (const auto & source_and_table_engine : source_and_table_engines)
+            {
+                const auto & table_engine = std::get<1>(source_and_table_engine);
+                if (table_engine != table_engine_name) continue;
+                const auto & source = std::get<0>(source_and_table_engine);
+                /// Set the flags from Table Engine to SOURCES so that prompts can be meaningful.
+                newFlags = source;
+                break;
+            }
+
+            if (newFlags == AccessType::NONE)
+                throw Exception(ErrorCodes::LOGICAL_ERROR, "Didn't find the target Source from the Table Engine");
+
+            if (grant_option && acs->isGranted(flags, args...))
+            {
+                return access_denied(ErrorCodes::ACCESS_DENIED,
+                    "{}: Not enough privileges. "
+                    "The required privileges have been granted, but without grant option. "
+                    "To execute this query, it's necessary to have the grant {} WITH GRANT OPTION",
+                    AccessRightsElement{newFlags}.toStringWithoutONClause());
+            }
+
+            return access_denied(ErrorCodes::ACCESS_DENIED,
+                "{}: Not enough privileges. To execute this query, it's necessary to have the grant {}",
+                AccessRightsElement{newFlags}.toStringWithoutONClause() + (grant_option ? " WITH GRANT OPTION" : ""));
+        }
+
         if (grant_option && acs->isGranted(flags, args...))
         {
             return access_denied(ErrorCodes::ACCESS_DENIED,

From f7e81e1ae2752020c076990395349ccd2d69cf2b Mon Sep 17 00:00:00 2001
From: jsc0218 <jsc0218@gmail.com>
Date: Tue, 18 Jun 2024 21:59:07 +0000
Subject: [PATCH 0071/1488] fix

---
 src/Access/ContextAccess.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Access/ContextAccess.cpp b/src/Access/ContextAccess.cpp
index 3ce30a0b681..de0e7e3d777 100644
--- a/src/Access/ContextAccess.cpp
+++ b/src/Access/ContextAccess.cpp
@@ -625,8 +625,8 @@ bool ContextAccess::checkAccessImplHelper(AccessFlags flags, const Args &... arg
 
     if (!granted)
     {
-        /// As we check the SOURCES from the Table Engine logic, direct prompt about Table Engine would be misleading since 
-        /// SOURCES is not granted actually. In order to solve this, turn the prompt logic back to Sources.
+        /// As we check the SOURCES from the Table Engine logic, direct prompt about Table Engine would be misleading
+        /// since SOURCES is not granted actually. In order to solve this, turn the prompt logic back to Sources.
         if (flags & AccessType::TABLE_ENGINE && !access_control->doesTableEnginesRequireGrant())
         {
             AccessFlags newFlags;

From a2ee0668f12c8cd1b88b8c4ad46c15271a5a1fd2 Mon Sep 17 00:00:00 2001
From: jsc0218 <jsc0218@gmail.com>
Date: Wed, 19 Jun 2024 02:20:22 +0000
Subject: [PATCH 0072/1488] fix

---
 src/Access/ContextAccess.cpp | 23 ++++++++---------------
 1 file changed, 8 insertions(+), 15 deletions(-)

diff --git a/src/Access/ContextAccess.cpp b/src/Access/ContextAccess.cpp
index de0e7e3d777..4620561053b 100644
--- a/src/Access/ContextAccess.cpp
+++ b/src/Access/ContextAccess.cpp
@@ -37,7 +37,7 @@ namespace ErrorCodes
 
 namespace
 {
-    static const std::vector<std::tuple<AccessFlags, std::string>> source_and_table_engines = {
+    const std::vector<std::tuple<AccessFlags, std::string>> source_and_table_engines = {
         {AccessType::FILE, "File"},
         {AccessType::URL, "URL"},
         {AccessType::REMOTE, "Distributed"},
@@ -268,6 +268,11 @@ namespace
 
     template <typename... OtherArgs>
     std::string_view getDatabase(std::string_view arg1, const OtherArgs &...) { return arg1; }
+
+    std::string_view getTableEngine() { return {}; }
+
+    template <typename... OtherArgs>
+    std::string_view getTableEngine(std::string_view arg1, const OtherArgs &...) { return arg1; }
 }
 
 
@@ -557,18 +562,6 @@ std::shared_ptr<const AccessRights> ContextAccess::getAccessRightsWithImplicit()
     return nothing_granted;
 }
 
-/// Just Dummy to pass compile.
-template <typename... Args>
-static std::string_view getTableEngineName(const Args &... args[[maybe_unused]])
-{
-    return "";
-}
-
-template <typename... Args>
-static std::string_view getTableEngineName(std::string_view name, const Args &... args[[maybe_unused]])
-{
-    return name;
-}
 
 template <bool throw_if_denied, bool grant_option, typename... Args>
 bool ContextAccess::checkAccessImplHelper(AccessFlags flags, const Args &... args) const
@@ -631,7 +624,7 @@ bool ContextAccess::checkAccessImplHelper(AccessFlags flags, const Args &... arg
         {
             AccessFlags newFlags;
 
-            String table_engine_name{getTableEngineName(args...)};
+            String table_engine_name{getTableEngine(args...)};
             for (const auto & source_and_table_engine : source_and_table_engines)
             {
                 const auto & table_engine = std::get<1>(source_and_table_engine);
@@ -642,7 +635,7 @@ bool ContextAccess::checkAccessImplHelper(AccessFlags flags, const Args &... arg
                 break;
             }
 
-            if (newFlags == AccessType::NONE)
+            if (newFlags.isEmpty())
                 throw Exception(ErrorCodes::LOGICAL_ERROR, "Didn't find the target Source from the Table Engine");
 
             if (grant_option && acs->isGranted(flags, args...))

From b125f8166f32648572c2ed0d540ded56a97ac628 Mon Sep 17 00:00:00 2001
From: Nikita Taranov <nikita.taranov@clickhouse.com>
Date: Wed, 19 Jun 2024 16:41:50 +0100
Subject: [PATCH 0073/1488] impl

---
 tests/performance/array_reduce.xml | 15 ++++++---------
 tests/performance/scripts/perf.py  | 14 ++++++++++++--
 2 files changed, 18 insertions(+), 11 deletions(-)

diff --git a/tests/performance/array_reduce.xml b/tests/performance/array_reduce.xml
index 8e271c155f3..0f50eba43bf 100644
--- a/tests/performance/array_reduce.xml
+++ b/tests/performance/array_reduce.xml
@@ -1,11 +1,8 @@
 <test>
-
-
-
-    <query>SELECT arrayReduce('count', range(100000000))</query>
-    <query>SELECT arrayReduce('sum', range(100000000))</query>
-    <query>SELECT arrayReduceInRanges('count', [(1, 100000000)], range(100000000))</query>
-    <query>SELECT arrayReduceInRanges('sum', [(1, 100000000)], range(100000000))</query>
-    <query>SELECT arrayReduceInRanges('count', arrayZip(range(1000000), range(1000000)), range(100000000))[123456]</query>
-    <query>SELECT arrayReduceInRanges('sum', arrayZip(range(1000000), range(1000000)), range(100000000))[123456]</query>
+    <query>SELECT arrayReduce('count', range(1000000)) FROM numbers_mt(500000000) format Null</query>
+    <query>SELECT arrayReduce('sum', range(1000000)) FROM numbers_mt(500000000) format Null</query>
+    <query>SELECT arrayReduceInRanges('count', [(1, 1000000)], range(1000000)) FROM numbers_mt(500000000) format Null</query>
+    <query>SELECT arrayReduceInRanges('sum', [(1, 1000000)], range(1000000)) FROM numbers_mt(500000000) format Null</query>
+    <query>SELECT arrayReduceInRanges('count', arrayZip(range(1000000), range(1000000)), range(1000000))[123456]</query>
+    <query>SELECT arrayReduceInRanges('sum', arrayZip(range(1000000), range(1000000)), range(1000000))[123456]</query>
 </test>
diff --git a/tests/performance/scripts/perf.py b/tests/performance/scripts/perf.py
index 94f145d82db..f89784a0e0b 100755
--- a/tests/performance/scripts/perf.py
+++ b/tests/performance/scripts/perf.py
@@ -345,6 +345,18 @@ for query_index in queries_to_run:
 
     print(f"display-name\t{query_index}\t{tsv_escape(query_display_name)}")
 
+    for conn_index, c in enumerate(all_connections):
+        try:
+            c.execute("SYSTEM JEMALLOC PURGE")
+
+            print(
+                f"purging jemalloc arenas\t{conn_index}\t{c.last_query.elapsed}"
+            )
+        except KeyboardInterrupt:
+            raise
+        except:
+            continue
+
     # Prewarm: run once on both servers. Helps to bring the data into memory,
     # precompile the queries, etc.
     # A query might not run on the old server if it uses a function added in the
@@ -427,8 +439,6 @@ for query_index in queries_to_run:
 
         for conn_index, c in enumerate(this_query_connections):
             try:
-                c.execute("SYSTEM JEMALLOC PURGE")
-
                 res = c.execute(
                     q,
                     query_id=run_id,

From 6f1f416700a32cf95af15b79543e27cbcffe2f14 Mon Sep 17 00:00:00 2001
From: Nikita Taranov <nikita.taranov@clickhouse.com>
Date: Thu, 20 Jun 2024 17:37:05 +0100
Subject: [PATCH 0074/1488] one more test

---
 tests/performance/final_big_column.xml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/tests/performance/final_big_column.xml b/tests/performance/final_big_column.xml
index 1fd586d2d90..4bfdfdf804f 100644
--- a/tests/performance/final_big_column.xml
+++ b/tests/performance/final_big_column.xml
@@ -1,6 +1,7 @@
 <test>
     <settings>
-        <max_insert_threads>1</max_insert_threads>
+        <!-- ingestion is quite time consuming in this test -->
+        <max_insert_threads>8</max_insert_threads>
         <max_memory_usage>20G</max_memory_usage>
     </settings>
 
@@ -10,8 +11,8 @@
         PARTITION BY toYYYYMM(d) ORDER BY key
     </create_query>
 
-    <fill_query>INSERT INTO optimized_select_final SELECT toDate('2000-01-01'), 2*number, randomPrintableASCII(1000) FROM numbers(5000000)</fill_query>
-    <fill_query>INSERT INTO optimized_select_final SELECT toDate('2020-01-01'), 2*number+1, randomPrintableASCII(1000) FROM numbers(5000000)</fill_query>
+    <fill_query>INSERT INTO optimized_select_final SELECT toDate('2000-01-01'), 2*number, randomPrintableASCII(1000) FROM numbers_mt(5000000)</fill_query>
+    <fill_query>INSERT INTO optimized_select_final SELECT toDate('2020-01-01'), 2*number+1, randomPrintableASCII(1000) FROM numbers_mt(5000000)</fill_query>
 
     <query>SELECT * FROM optimized_select_final FINAL FORMAT Null SETTINGS max_threads = 8</query>
     <query>SELECT * FROM optimized_select_final FINAL WHERE key % 10 = 0 FORMAT Null</query>

From 16baecf5a67083ecd23d621f10ec0c6250178e32 Mon Sep 17 00:00:00 2001
From: Michael Kolupaev <michael.kolupaev@clickhouse.com>
Date: Thu, 13 Jun 2024 22:01:57 +0000
Subject: [PATCH 0075/1488] attach_gdb.lib: print registers before all stacks

---
 docker/test/stateless/attach_gdb.lib | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/docker/test/stateless/attach_gdb.lib b/docker/test/stateless/attach_gdb.lib
index eb54f920b98..251fb886df6 100644
--- a/docker/test/stateless/attach_gdb.lib
+++ b/docker/test/stateless/attach_gdb.lib
@@ -25,8 +25,11 @@ handle SIG$RTMIN nostop noprint pass
 info signals
 continue
 backtrace full
-thread apply all backtrace full
 info registers
+p "top 1 KiB of the stack:"
+p/x *(uint64_t[128]*)$sp
+maintenance info sections
+thread apply all backtrace full
 disassemble /s
 up
 disassemble /s

From 557cd2c08f4791e10e1f2914bd4974c51531bf41 Mon Sep 17 00:00:00 2001
From: Michael Kolupaev <michael.kolupaev@clickhouse.com>
Date: Thu, 20 Jun 2024 18:56:06 +0000
Subject: [PATCH 0076/1488] Escape the $

---
 docker/test/stateless/attach_gdb.lib | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/test/stateless/attach_gdb.lib b/docker/test/stateless/attach_gdb.lib
index 251fb886df6..a3616ac1a04 100644
--- a/docker/test/stateless/attach_gdb.lib
+++ b/docker/test/stateless/attach_gdb.lib
@@ -27,7 +27,7 @@ continue
 backtrace full
 info registers
 p "top 1 KiB of the stack:"
-p/x *(uint64_t[128]*)$sp
+p/x *(uint64_t[128]*)"'$sp'"
 maintenance info sections
 thread apply all backtrace full
 disassemble /s

From 3ab8ba0d4ab0e4446883bf57ba3d859d1ee49a52 Mon Sep 17 00:00:00 2001
From: Michael Kolupaev <michael.kolupaev@clickhouse.com>
Date: Mon, 27 May 2024 21:33:54 +0000
Subject: [PATCH 0077/1488] Update zlib-ng from 2.0.2 to 2.1.6

---
 contrib/zlib-ng                      |  2 +-
 contrib/zlib-ng-cmake/CMakeLists.txt | 84 +++++++++++++++-------------
 2 files changed, 47 insertions(+), 39 deletions(-)

diff --git a/contrib/zlib-ng b/contrib/zlib-ng
index 50f0eae1a41..bfb184bb0fb 160000
--- a/contrib/zlib-ng
+++ b/contrib/zlib-ng
@@ -1 +1 @@
-Subproject commit 50f0eae1a411764cd6d1e85b3ce471438acd3c1c
+Subproject commit bfb184bb0fbdabe82f3a36d209e56c3e2c33866a
diff --git a/contrib/zlib-ng-cmake/CMakeLists.txt b/contrib/zlib-ng-cmake/CMakeLists.txt
index 79f343bfc75..78a5f1ae3a8 100644
--- a/contrib/zlib-ng-cmake/CMakeLists.txt
+++ b/contrib/zlib-ng-cmake/CMakeLists.txt
@@ -14,6 +14,7 @@ add_definitions(-DHAVE_VISIBILITY_HIDDEN)
 add_definitions(-DHAVE_VISIBILITY_INTERNAL)
 add_definitions(-DHAVE_BUILTIN_CTZ)
 add_definitions(-DHAVE_BUILTIN_CTZLL)
+add_definitions(-DHAVE_ATTRIBUTE_ALIGNED)
 
 set(ZLIB_ARCH_SRCS)
 set(ZLIB_ARCH_HDRS)
@@ -26,14 +27,15 @@ if(ARCH_AARCH64)
     add_definitions(-DARM_FEATURES)
     add_definitions(-DARM_AUXV_HAS_CRC32 -DARM_ASM_HWCAP)
     add_definitions(-DARM_AUXV_HAS_NEON)
-    add_definitions(-DARM_ACLE_CRC_HASH)
-    add_definitions(-DARM_NEON_ADLER32 -DARM_NEON_CHUNKSET -DARM_NEON_SLIDEHASH)
+    add_definitions(-DARM_ACLE)
+    add_definitions(-DARM_NEON)
 
-    list(APPEND ZLIB_ARCH_HDRS ${ARCHDIR}/arm.h)
-    list(APPEND ZLIB_ARCH_SRCS ${ARCHDIR}/armfeature.c)
+    list(APPEND ZLIB_ARCH_HDRS ${ARCHDIR}/arm_features.h)
+    list(APPEND ZLIB_ARCH_SRCS ${ARCHDIR}/arm_features.c)
     set(ACLE_SRCS ${ARCHDIR}/crc32_acle.c ${ARCHDIR}/insert_string_acle.c)
     list(APPEND ZLIB_ARCH_SRCS ${ACLE_SRCS})
-    set(NEON_SRCS ${ARCHDIR}/adler32_neon.c ${ARCHDIR}/chunkset_neon.c ${ARCHDIR}/slide_neon.c)
+    set(NEON_SRCS ${ARCHDIR}/adler32_neon.c ${ARCHDIR}/chunkset_neon.c
+                    ${ARCHDIR}/compare256_neon.c ${ARCHDIR}/slide_hash_neon.c)
     list(APPEND ZLIB_ARCH_SRCS ${NEON_SRCS})
 
 elseif(ARCH_PPC64LE)
@@ -41,50 +43,47 @@ elseif(ARCH_PPC64LE)
 
     add_definitions(-DPOWER8)
     add_definitions(-DPOWER_FEATURES)
-    add_definitions(-DPOWER8_VSX_ADLER32)
-    add_definitions(-DPOWER8_VSX_SLIDEHASH)
+    add_definitions(-DPOWER8_VSX)
+    add_definitions(-DPOWER8_VSX_CRC32)
 
-    list(APPEND ZLIB_ARCH_HDRS ${ARCHDIR}/power.h)
-    list(APPEND ZLIB_ARCH_SRCS ${ARCHDIR}/power.c)
-    set(POWER8_SRCS ${ARCHDIR}/adler32_power8.c ${ARCHDIR}/slide_hash_power8.c)
+    list(APPEND ZLIB_ARCH_HDRS ${ARCHDIR}/power_features.h)
+    list(APPEND ZLIB_ARCH_SRCS ${ARCHDIR}/power_features.c)
+    set(POWER8_SRCS ${ARCHDIR}/adler32_power8.c ${ARCHDIR}/chunkset_power8.c ${ARCHDIR}/slide_hash_power8.c)
+    list(APPEND POWER8_SRCS ${ARCHDIR}/crc32_power8.c)
     list(APPEND ZLIB_ARCH_SRCS ${POWER8_SRCS})
 
 elseif(ARCH_AMD64)
     set(ARCHDIR "${SOURCE_DIR}/arch/x86")
 
     add_definitions(-DX86_FEATURES)
-    list(APPEND ZLIB_ARCH_HDRS ${ARCHDIR}/x86.h)
-    list(APPEND ZLIB_ARCH_SRCS ${ARCHDIR}/x86.c)
+    list(APPEND ZLIB_ARCH_HDRS ${ARCHDIR}/x86_features.h)
+    list(APPEND ZLIB_ARCH_SRCS ${ARCHDIR}/x86_features.c)
     if(ENABLE_AVX2)
-        add_definitions(-DX86_AVX2 -DX86_AVX2_ADLER32 -DX86_AVX_CHUNKSET)
-        set(AVX2_SRCS ${ARCHDIR}/slide_avx.c)
-        list(APPEND AVX2_SRCS ${ARCHDIR}/chunkset_avx.c)
-        list(APPEND AVX2_SRCS ${ARCHDIR}/compare258_avx.c)
-        list(APPEND AVX2_SRCS ${ARCHDIR}/adler32_avx.c)
+        add_definitions(-DX86_AVX2)
+        set(AVX2_SRCS ${ARCHDIR}/slide_hash_avx2.c)
+        list(APPEND AVX2_SRCS ${ARCHDIR}/chunkset_avx2.c)
+        list(APPEND AVX2_SRCS ${ARCHDIR}/compare256_avx2.c)
+        list(APPEND AVX2_SRCS ${ARCHDIR}/adler32_avx2.c)
         list(APPEND ZLIB_ARCH_SRCS ${AVX2_SRCS})
     endif()
     if(ENABLE_SSE42)
-        add_definitions(-DX86_SSE42_CRC_HASH)
-        set(SSE42_SRCS ${ARCHDIR}/insert_string_sse.c)
-        list(APPEND ZLIB_ARCH_SRCS ${SSE42_SRCS})
-        add_definitions(-DX86_SSE42_CRC_INTRIN)
-        add_definitions(-DX86_SSE42_CMP_STR)
-        set(SSE42_SRCS ${ARCHDIR}/compare258_sse.c)
+        add_definitions(-DX86_SSE42)
+        set(SSE42_SRCS ${ARCHDIR}/adler32_sse42.c ${ARCHDIR}/insert_string_sse42.c)
         list(APPEND ZLIB_ARCH_SRCS ${SSE42_SRCS})
     endif()
     if(ENABLE_SSSE3)
-        add_definitions(-DX86_SSSE3 -DX86_SSSE3_ADLER32)
-        set(SSSE3_SRCS ${ARCHDIR}/adler32_ssse3.c)
+        add_definitions(-DX86_SSSE3)
+        set(SSSE3_SRCS ${ARCHDIR}/adler32_ssse3.c ${ARCHDIR}/chunkset_ssse3.c)
         list(APPEND ZLIB_ARCH_SRCS ${SSSE3_SRCS})
     endif()
     if(ENABLE_PCLMULQDQ)
         add_definitions(-DX86_PCLMULQDQ_CRC)
-        set(PCLMULQDQ_SRCS ${ARCHDIR}/crc_folding.c)
+        set(PCLMULQDQ_SRCS ${ARCHDIR}/crc32_pclmulqdq.c)
         list(APPEND ZLIB_ARCH_SRCS ${PCLMULQDQ_SRCS})
     endif()
 
-    add_definitions(-DX86_SSE2 -DX86_SSE2_CHUNKSET -DX86_SSE2_SLIDEHASH)
-    set(SSE2_SRCS ${ARCHDIR}/chunkset_sse.c ${ARCHDIR}/slide_sse.c)
+    add_definitions(-DX86_SSE2)
+    set(SSE2_SRCS ${ARCHDIR}/chunkset_sse2.c ${ARCHDIR}/compare256_sse2.c ${ARCHDIR}/slide_hash_sse2.c)
     list(APPEND ZLIB_ARCH_SRCS ${SSE2_SRCS})
     add_definitions(-DX86_NOCHECK_SSE2)
 endif ()
@@ -106,39 +105,45 @@ generate_cmakein(${SOURCE_DIR}/zconf.h.in ${CMAKE_CURRENT_BINARY_DIR}/zconf.h.cm
 
 set(ZLIB_SRCS
     ${SOURCE_DIR}/adler32.c
+    ${SOURCE_DIR}/adler32_fold.c
     ${SOURCE_DIR}/chunkset.c
-    ${SOURCE_DIR}/compare258.c
+    ${SOURCE_DIR}/compare256.c
     ${SOURCE_DIR}/compress.c
-    ${SOURCE_DIR}/crc32.c
-    ${SOURCE_DIR}/crc32_comb.c
+    ${SOURCE_DIR}/cpu_features.c
+    ${SOURCE_DIR}/crc32_braid.c
+    ${SOURCE_DIR}/crc32_braid_comb.c
+    ${SOURCE_DIR}/crc32_fold.c
     ${SOURCE_DIR}/deflate.c
     ${SOURCE_DIR}/deflate_fast.c
+    ${SOURCE_DIR}/deflate_huff.c
     ${SOURCE_DIR}/deflate_medium.c
     ${SOURCE_DIR}/deflate_quick.c
+    ${SOURCE_DIR}/deflate_rle.c
     ${SOURCE_DIR}/deflate_slow.c
+    ${SOURCE_DIR}/deflate_stored.c
     ${SOURCE_DIR}/functable.c
     ${SOURCE_DIR}/infback.c
-    ${SOURCE_DIR}/inffast.c
     ${SOURCE_DIR}/inflate.c
     ${SOURCE_DIR}/inftrees.c
     ${SOURCE_DIR}/insert_string.c
+    ${SOURCE_DIR}/insert_string_roll.c
+    ${SOURCE_DIR}/slide_hash.c
     ${SOURCE_DIR}/trees.c
     ${SOURCE_DIR}/uncompr.c
     ${SOURCE_DIR}/zutil.c
+)
+
+set(ZLIB_GZFILE_SRCS
     ${SOURCE_DIR}/gzlib.c
-    ${SOURCE_DIR}/gzread.c
+    ${CMAKE_CURRENT_BINARY_DIR}/gzread.c
     ${SOURCE_DIR}/gzwrite.c
 )
 
-set(ZLIB_ALL_SRCS ${ZLIB_SRCS} ${ZLIB_ARCH_SRCS})
+set(ZLIB_ALL_SRCS ${ZLIB_SRCS} ${ZLIB_ARCH_SRCS} ${ZLIB_GZFILE_SRCS})
 
 add_library(_zlib ${ZLIB_ALL_SRCS})
 add_library(ch_contrib::zlib ALIAS _zlib)
 
-# https://github.com/zlib-ng/zlib-ng/pull/733
-# This is disabed by default
-add_compile_definitions(Z_TLS=__thread)
-
 if(HAVE_UNISTD_H)
   SET(ZCONF_UNISTD_LINE "#if 1    /* was set to #if 1 by configure/cmake/etc */")
 else()
@@ -153,6 +158,9 @@ endif()
 set(ZLIB_PC ${CMAKE_CURRENT_BINARY_DIR}/zlib.pc)
 configure_file(${SOURCE_DIR}/zlib.pc.cmakein ${ZLIB_PC} @ONLY)
 configure_file(${CMAKE_CURRENT_BINARY_DIR}/zconf.h.cmakein ${CMAKE_CURRENT_BINARY_DIR}/zconf.h @ONLY)
+configure_file(${SOURCE_DIR}/zlib.h.in ${CMAKE_CURRENT_BINARY_DIR}/zlib.h @ONLY)
+configure_file(${SOURCE_DIR}/zlib_name_mangling.h.in ${CMAKE_CURRENT_BINARY_DIR}/zlib_name_mangling.h @ONLY)
+configure_file(${SOURCE_DIR}/gzread.c.in ${CMAKE_CURRENT_BINARY_DIR}/gzread.c @ONLY)
 
 # We should use same defines when including zlib.h as used when zlib compiled
 target_compile_definitions (_zlib PUBLIC ZLIB_COMPAT WITH_GZFILEOP)

From 62d6e3d3396fbcbcc6f666ac9b7bc2b60fe828b2 Mon Sep 17 00:00:00 2001
From: Michael Kolupaev <michael.kolupaev@clickhouse.com>
Date: Wed, 29 May 2024 00:19:25 +0000
Subject: [PATCH 0078/1488] Fix ARM

---
 contrib/zlib-ng-cmake/CMakeLists.txt | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/contrib/zlib-ng-cmake/CMakeLists.txt b/contrib/zlib-ng-cmake/CMakeLists.txt
index 78a5f1ae3a8..40397e14f20 100644
--- a/contrib/zlib-ng-cmake/CMakeLists.txt
+++ b/contrib/zlib-ng-cmake/CMakeLists.txt
@@ -28,7 +28,9 @@ if(ARCH_AARCH64)
     add_definitions(-DARM_AUXV_HAS_CRC32 -DARM_ASM_HWCAP)
     add_definitions(-DARM_AUXV_HAS_NEON)
     add_definitions(-DARM_ACLE)
+    add_definitions(-DHAVE_ARM_ACLE_H)
     add_definitions(-DARM_NEON)
+    add_definitions(-DARM_NEON_HASLD4)
 
     list(APPEND ZLIB_ARCH_HDRS ${ARCHDIR}/arm_features.h)
     list(APPEND ZLIB_ARCH_SRCS ${ARCHDIR}/arm_features.c)

From 8115926eb61779d669509f16863730557882b3c8 Mon Sep 17 00:00:00 2001
From: Michael Kolupaev <michael.kolupaev@clickhouse.com>
Date: Wed, 29 May 2024 04:49:19 +0000
Subject: [PATCH 0079/1488] Fix ARM some more

---
 contrib/zlib-ng-cmake/CMakeLists.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/contrib/zlib-ng-cmake/CMakeLists.txt b/contrib/zlib-ng-cmake/CMakeLists.txt
index 40397e14f20..1ed1d98ed66 100644
--- a/contrib/zlib-ng-cmake/CMakeLists.txt
+++ b/contrib/zlib-ng-cmake/CMakeLists.txt
@@ -25,6 +25,7 @@ if(ARCH_AARCH64)
     set(ARCHDIR "${SOURCE_DIR}/arch/arm")
 
     add_definitions(-DARM_FEATURES)
+    add_definitions(-DHAVE_SYS_AUXV_H)
     add_definitions(-DARM_AUXV_HAS_CRC32 -DARM_ASM_HWCAP)
     add_definitions(-DARM_AUXV_HAS_NEON)
     add_definitions(-DARM_ACLE)

From 06781efcb785a0504d9599e6a1446e235caef13d Mon Sep 17 00:00:00 2001
From: Michael Kolupaev <michael.kolupaev@clickhouse.com>
Date: Thu, 20 Jun 2024 22:59:36 +0000
Subject: [PATCH 0080/1488] Switch to a cleaner update, hopefully fix builds

---
 contrib/zlib-ng                      |  2 +-
 contrib/zlib-ng-cmake/CMakeLists.txt | 13 ++++++++++---
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/contrib/zlib-ng b/contrib/zlib-ng
index bfb184bb0fb..c19ba056b7c 160000
--- a/contrib/zlib-ng
+++ b/contrib/zlib-ng
@@ -1 +1 @@
-Subproject commit bfb184bb0fbdabe82f3a36d209e56c3e2c33866a
+Subproject commit c19ba056b7cc8029bb80f509956090c7ded58032
diff --git a/contrib/zlib-ng-cmake/CMakeLists.txt b/contrib/zlib-ng-cmake/CMakeLists.txt
index 1ed1d98ed66..91a8eb2e0a3 100644
--- a/contrib/zlib-ng-cmake/CMakeLists.txt
+++ b/contrib/zlib-ng-cmake/CMakeLists.txt
@@ -15,6 +15,7 @@ add_definitions(-DHAVE_VISIBILITY_INTERNAL)
 add_definitions(-DHAVE_BUILTIN_CTZ)
 add_definitions(-DHAVE_BUILTIN_CTZLL)
 add_definitions(-DHAVE_ATTRIBUTE_ALIGNED)
+add_definitions(-DHAVE_POSIX_MEMALIGN)
 
 set(ZLIB_ARCH_SRCS)
 set(ZLIB_ARCH_HDRS)
@@ -44,10 +45,16 @@ if(ARCH_AARCH64)
 elseif(ARCH_PPC64LE)
     set(ARCHDIR "${SOURCE_DIR}/arch/power")
 
-    add_definitions(-DPOWER8)
     add_definitions(-DPOWER_FEATURES)
-    add_definitions(-DPOWER8_VSX)
-    add_definitions(-DPOWER8_VSX_CRC32)
+    add_definitions(-DHAVE_SYS_AUXV_H)
+
+    if(POWER9)
+        add_definitions(-DPOWER9)
+    else()
+        add_definitions(-DPOWER8)
+        add_definitions(-DPOWER8_VSX)
+        add_definitions(-DPOWER8_VSX_CRC32)
+    endif()
 
     list(APPEND ZLIB_ARCH_HDRS ${ARCHDIR}/power_features.h)
     list(APPEND ZLIB_ARCH_SRCS ${ARCHDIR}/power_features.c)

From 237de4e775ba4080087496303c350cddbebc9043 Mon Sep 17 00:00:00 2001
From: Konstantin Morozov <just.morozov.k@gmail.com>
Date: Fri, 14 Jun 2024 12:31:52 +0000
Subject: [PATCH 0081/1488] add feature

---
 .../system-tables/detached_tables.md          |  36 +
 src/Databases/DatabaseAtomic.cpp              |  20 +-
 src/Databases/DatabaseAtomic.h                |   7 +-
 src/Databases/DatabaseOnDisk.cpp              |   3 +
 src/Databases/DatabaseOrdinary.cpp            |  37 +-
 src/Databases/DatabaseOrdinary.h              |   3 +
 src/Databases/DatabasesCommon.cpp             |  54 +-
 src/Databases/DatabasesCommon.h               |   6 +-
 src/Databases/IDatabase.h                     |  63 +-
 src/Storages/IStorage_fwd.h                   |   4 +-
 src/Storages/System/ReadFromSystemTables.cpp  | 138 ++++
 src/Storages/System/ReadFromSystemTables.h    |  47 ++
 .../System/StorageSystemDetachedTables.cpp    |  56 ++
 .../System/StorageSystemDetachedTables.h      |  32 +
 src/Storages/System/StorageSystemTables.cpp   | 761 ++----------------
 src/Storages/System/TablesBlockSource.cpp     | 569 +++++++++++++
 src/Storages/System/TablesBlockSource.h       |  96 +++
 src/Storages/System/attachSystemTables.cpp    |   2 +
 .../test_system_detached_tables/__init__.py   |   0
 .../test_system_detached_tables/test.py       |  38 +
 .../03172_system_detached_tables.reference    |   5 +
 .../03172_system_detached_tables.sql          |  28 +
 22 files changed, 1289 insertions(+), 716 deletions(-)
 create mode 100644 docs/en/operations/system-tables/detached_tables.md
 create mode 100644 src/Storages/System/ReadFromSystemTables.cpp
 create mode 100644 src/Storages/System/ReadFromSystemTables.h
 create mode 100644 src/Storages/System/StorageSystemDetachedTables.cpp
 create mode 100644 src/Storages/System/StorageSystemDetachedTables.h
 create mode 100644 src/Storages/System/TablesBlockSource.cpp
 create mode 100644 src/Storages/System/TablesBlockSource.h
 create mode 100644 tests/integration/test_system_detached_tables/__init__.py
 create mode 100644 tests/integration/test_system_detached_tables/test.py
 create mode 100644 tests/queries/0_stateless/03172_system_detached_tables.reference
 create mode 100644 tests/queries/0_stateless/03172_system_detached_tables.sql

diff --git a/docs/en/operations/system-tables/detached_tables.md b/docs/en/operations/system-tables/detached_tables.md
new file mode 100644
index 00000000000..d86e58d2899
--- /dev/null
+++ b/docs/en/operations/system-tables/detached_tables.md
@@ -0,0 +1,36 @@
+---
+slug: /en/operations/system-tables/detached_tables
+---
+# detached_tables
+
+Contains information of each detached table.
+
+Columns:
+
+- `database` ([String](../../sql-reference/data-types/string.md)) — The name of the database the table is in.
+
+- `name` ([String](../../sql-reference/data-types/string.md)) — Table name.
+
+- `uuid` ([UUID](../../sql-reference/data-types/uuid.md)) — Table uuid (Atomic database).
+
+- `is_permanently` ([UInt8](../../sql-reference/data-types/int-uint.md)) - Flag indicates that the table was detached PERMANENTLY.
+
+- `metadata_path` ([String](../../sql-reference/data-types/string.md)) - Path to the table metadata in the file system.
+
+
+**Example**
+
+```sql
+SELECT * FROM system.detached_tables FORMAT Vertical;
+```
+
+```text
+Row 1:
+──────
+database:                   base
+name:                       t1
+uuid:                       81b1c20a-b7c6-4116-a2ce-7583fb6b6736
+data_paths:                 ['/var/lib/clickhouse/store/81b/81b1c20a-b7c6-4116-a2ce-7583fb6b6736/']
+metadata_path:              /var/lib/clickhouse/store/461/461cf698-fd0b-406d-8c01-5d8fd5748a91/t1.sql
+is_permanently:             1
+```
diff --git a/src/Databases/DatabaseAtomic.cpp b/src/Databases/DatabaseAtomic.cpp
index ccab72cfbae..ddc30627cde 100644
--- a/src/Databases/DatabaseAtomic.cpp
+++ b/src/Databases/DatabaseAtomic.cpp
@@ -37,8 +37,10 @@ namespace ErrorCodes
 class AtomicDatabaseTablesSnapshotIterator final : public DatabaseTablesSnapshotIterator
 {
 public:
-    explicit AtomicDatabaseTablesSnapshotIterator(DatabaseTablesSnapshotIterator && base)
-        : DatabaseTablesSnapshotIterator(std::move(base)) {}
+    explicit AtomicDatabaseTablesSnapshotIterator(DatabaseTablesSnapshotIterator && base) noexcept
+        : DatabaseTablesSnapshotIterator(std::move(base))
+    {
+    }
     UUID uuid() const override { return table()->getStorageID().uuid; }
 };
 
@@ -101,6 +103,8 @@ void DatabaseAtomic::attachTable(ContextPtr /* context_ */, const String & name,
     auto table_id = table->getStorageID();
     assertDetachedTableNotInUse(table_id.uuid);
     DatabaseOrdinary::attachTableUnlocked(name, table);
+    detached_tables.erase(table_id.uuid);
+
     table_name_to_path.emplace(std::make_pair(name, relative_table_path));
 }
 
@@ -108,11 +112,11 @@ StoragePtr DatabaseAtomic::detachTable(ContextPtr /* context */, const String &
 {
     DetachedTables not_in_use;
     std::lock_guard lock(mutex);
-    auto table = DatabaseOrdinary::detachTableUnlocked(name);
+    auto detached_table = DatabaseOrdinary::detachTableUnlocked(name);
     table_name_to_path.erase(name);
-    detached_tables.emplace(table->getStorageID().uuid, table);
+    detached_tables.emplace(detached_table->getStorageID().uuid, detached_table);
     not_in_use = cleanupDetachedTables();
-    return table;
+    return detached_table;
 }
 
 void DatabaseAtomic::dropTable(ContextPtr local_context, const String & table_name, bool sync)
@@ -433,6 +437,12 @@ DatabaseAtomic::getTablesIterator(ContextPtr local_context, const IDatabase::Fil
     return std::make_unique<AtomicDatabaseTablesSnapshotIterator>(std::move(typeid_cast<DatabaseTablesSnapshotIterator &>(*base_iter)));
 }
 
+DatabaseDetachedTablesSnapshotIteratorPtr DatabaseAtomic::getDetachedTablesIterator(
+    ContextPtr local_context, const IDatabase::FilterByNameFunction & filter_by_table_name, const bool skip_not_loaded) const
+{
+    return DatabaseOrdinary::getDetachedTablesIterator(local_context, filter_by_table_name, skip_not_loaded);
+}
+
 UUID DatabaseAtomic::tryGetTableUUID(const String & table_name) const
 {
     if (auto table = tryGetTable(table_name, getContext()))
diff --git a/src/Databases/DatabaseAtomic.h b/src/Databases/DatabaseAtomic.h
index b59edd479ba..47afc4dc5ef 100644
--- a/src/Databases/DatabaseAtomic.h
+++ b/src/Databases/DatabaseAtomic.h
@@ -1,7 +1,8 @@
 #pragma once
 
-#include <Databases/DatabasesCommon.h>
 #include <Databases/DatabaseOrdinary.h>
+#include <Databases/DatabasesCommon.h>
+#include <Storages/IStorage_fwd.h>
 
 
 namespace DB
@@ -48,6 +49,9 @@ public:
 
     DatabaseTablesIteratorPtr getTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name, bool skip_not_loaded) const override;
 
+    DatabaseDetachedTablesSnapshotIteratorPtr
+    getDetachedTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name, bool skip_not_loaded) const override;
+
     void beforeLoadingMetadata(ContextMutablePtr context, LoadingStrictnessLevel mode) override;
 
     LoadTaskPtr startupDatabaseAsync(AsyncLoader & async_loader, LoadJobSet startup_after, LoadingStrictnessLevel mode) override;
@@ -81,6 +85,7 @@ protected:
 
     //TODO store path in DatabaseWithOwnTables::tables
     using NameToPathMap = std::unordered_map<String, String>;
+
     NameToPathMap table_name_to_path TSA_GUARDED_BY(mutex);
 
     DetachedTables detached_tables TSA_GUARDED_BY(mutex);
diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp
index b8154372116..25e5347e017 100644
--- a/src/Databases/DatabaseOnDisk.cpp
+++ b/src/Databases/DatabaseOnDisk.cpp
@@ -307,6 +307,9 @@ void DatabaseOnDisk::detachTablePermanently(ContextPtr query_context, const Stri
     try
     {
         FS::createFile(detached_permanently_flag);
+
+        std::lock_guard lock(mutex);
+        snapshot_detached_tables.at(table_name).is_permanently = true;
     }
     catch (Exception & e)
     {
diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp
index 10a8e06e8f0..046cbfa40be 100644
--- a/src/Databases/DatabaseOrdinary.cpp
+++ b/src/Databases/DatabaseOrdinary.cpp
@@ -187,7 +187,7 @@ void DatabaseOrdinary::loadTablesMetadata(ContextPtr local_context, ParsedTables
     size_t prev_tables_count = metadata.parsed_tables.size();
     size_t prev_total_dictionaries = metadata.total_dictionaries;
 
-    auto process_metadata = [&metadata, is_startup, this](const String & file_name)
+    auto process_metadata = [&metadata, is_startup, local_context, this](const String & file_name) mutable
     {
         fs::path path(getMetadataPath());
         fs::path file_path(file_name);
@@ -195,7 +195,7 @@ void DatabaseOrdinary::loadTablesMetadata(ContextPtr local_context, ParsedTables
 
         try
         {
-            auto ast = parseQueryFromMetadata(log, getContext(), full_path.string(), /*throw_on_error*/ true, /*remove_empty*/ false);
+            auto ast = parseQueryFromMetadata(log, local_context, full_path.string(), /*throw_on_error*/ true, /*remove_empty*/ false);
             if (ast)
             {
                 FunctionNameNormalizer::visit(ast.get());
@@ -226,6 +226,33 @@ void DatabaseOrdinary::loadTablesMetadata(ContextPtr local_context, ParsedTables
                     const std::string table_name = unescapeForFileName(file_name.substr(0, file_name.size() - 4));
                     permanently_detached_tables.push_back(table_name);
                     LOG_DEBUG(log, "Skipping permanently detached table {}.", backQuote(table_name));
+
+                    // @TODO refactoring
+                    auto parsed_table_metadata = ParsedTableMetadata{full_path.string(), ast};
+                    const auto & query = parsed_table_metadata.ast->as<const ASTCreateQuery &>();
+
+                    std::lock_guard lock(mutex);
+
+                    auto [detached_table_name, table] = createTableFromAST(
+                        query,
+                        database_name,
+                        getTableDataPath(query),
+                        std::const_pointer_cast<Context>(local_context),
+                        LoadingStrictnessLevel::CREATE);
+
+                    const auto storage_id = table->getStorageID();
+
+                    SnapshotDetachedTable snapshot_detached_table;
+                    snapshot_detached_table.detabase = storage_id.getDatabaseName();
+                    snapshot_detached_table.table = detached_table_name;
+                    snapshot_detached_table.uuid = storage_id.uuid;
+                    snapshot_detached_table.is_permanently = true;
+                    snapshot_detached_table.metadata_path = getObjectMetadataPath(snapshot_detached_table.table);
+
+
+                    snapshot_detached_tables.emplace(detached_table_name, std::move(snapshot_detached_table));
+
+                    LOG_TRACE(log, "Add detached table {} to system.detached_tables", detached_table_name);
                     return;
                 }
 
@@ -487,6 +514,12 @@ DatabaseTablesIteratorPtr DatabaseOrdinary::getTablesIterator(ContextPtr local_c
     return DatabaseWithOwnTablesBase::getTablesIterator(local_context, filter_by_table_name, skip_not_loaded);
 }
 
+DatabaseDetachedTablesSnapshotIteratorPtr DatabaseOrdinary::getDetachedTablesIterator(
+    ContextPtr local_context, const DatabaseOnDisk::FilterByNameFunction & filter_by_table_name, bool skip_not_loaded) const
+{
+    return DatabaseWithOwnTablesBase::getDetachedTablesIterator(local_context, filter_by_table_name, skip_not_loaded);
+}
+
 Strings DatabaseOrdinary::getAllTableNames(ContextPtr) const
 {
     std::set<String> unique_names;
diff --git a/src/Databases/DatabaseOrdinary.h b/src/Databases/DatabaseOrdinary.h
index ef00ac8fdfa..d2891147922 100644
--- a/src/Databases/DatabaseOrdinary.h
+++ b/src/Databases/DatabaseOrdinary.h
@@ -57,6 +57,9 @@ public:
     LoadTaskPtr startupDatabaseAsync(AsyncLoader & async_loader, LoadJobSet startup_after, LoadingStrictnessLevel mode) override;
 
     DatabaseTablesIteratorPtr getTablesIterator(ContextPtr local_context, const DatabaseOnDisk::FilterByNameFunction & filter_by_table_name, bool skip_not_loaded) const override;
+    DatabaseDetachedTablesSnapshotIteratorPtr getDetachedTablesIterator(
+        ContextPtr local_context, const DatabaseOnDisk::FilterByNameFunction & filter_by_table_name, bool skip_not_loaded) const override;
+
     Strings getAllTableNames(ContextPtr context) const override;
 
     void alterTable(
diff --git a/src/Databases/DatabasesCommon.cpp b/src/Databases/DatabasesCommon.cpp
index fd38a31da5c..312c0a4477f 100644
--- a/src/Databases/DatabasesCommon.cpp
+++ b/src/Databases/DatabasesCommon.cpp
@@ -2,12 +2,9 @@
 
 #include <Backups/BackupEntriesCollector.h>
 #include <Backups/RestorerFromBackup.h>
-#include <Common/typeid_cast.h>
-#include <Common/CurrentMetrics.h>
-#include <Common/escapeForFileName.h>
-#include <Interpreters/InterpreterCreateQuery.h>
 #include <Interpreters/Context.h>
 #include <Interpreters/DatabaseCatalog.h>
+#include <Interpreters/InterpreterCreateQuery.h>
 #include <Parsers/ASTCreateQuery.h>
 #include <Parsers/ASTSelectWithUnionQuery.h>
 #include <Parsers/ParserCreateQuery.h>
@@ -16,6 +13,10 @@
 #include <Storages/StorageFactory.h>
 #include <Storages/Utils.h>
 #include <TableFunctions/TableFunctionFactory.h>
+#include <Common/CurrentMetrics.h>
+#include <Common/escapeForFileName.h>
+#include <Common/logger_useful.h>
+#include <Common/typeid_cast.h>
 
 
 namespace DB
@@ -237,6 +238,35 @@ DatabaseTablesIteratorPtr DatabaseWithOwnTablesBase::getTablesIterator(ContextPt
     return std::make_unique<DatabaseTablesSnapshotIterator>(std::move(filtered_tables), database_name);
 }
 
+DatabaseDetachedTablesSnapshotIteratorPtr DatabaseWithOwnTablesBase::getDetachedTablesIterator(
+    ContextPtr, const FilterByNameFunction & filter_by_table_name, bool /* skip_not_loaded */) const
+{
+    std::lock_guard lock(mutex);
+    if (!filter_by_table_name)
+        return std::make_unique<DatabaseDetachedTablesSnapshotIterator>(snapshot_detached_tables);
+
+    SnapshotDetachedTables filtered_tables;
+    for (const auto & [table_name, storage] : tables)
+        if (filter_by_table_name(table_name))
+        {
+            SnapshotDetachedTable snapshot_detached_table;
+            snapshot_detached_table.detabase = storage->getStorageID().getDatabaseName();
+            snapshot_detached_table.table = table_name;
+            if (storage->getStorageID().hasUUID())
+            {
+                snapshot_detached_table.uuid = storage->getStorageID().uuid;
+            }
+
+            snapshot_detached_table.is_permanently = false;
+            snapshot_detached_table.metadata_path = getObjectMetadataPath(snapshot_detached_table.table);
+
+            filtered_tables.emplace(table_name, std::move(snapshot_detached_table));
+        }
+
+
+    return std::make_unique<DatabaseDetachedTablesSnapshotIterator>(std::move(filtered_tables));
+}
+
 bool DatabaseWithOwnTablesBase::empty() const
 {
     std::lock_guard lock(mutex);
@@ -258,6 +288,19 @@ StoragePtr DatabaseWithOwnTablesBase::detachTableUnlocked(const String & table_n
         throw Exception(ErrorCodes::UNKNOWN_TABLE, "Table {}.{} doesn't exist",
                         backQuote(database_name), backQuote(table_name));
     res = it->second;
+
+    SnapshotDetachedTable snapshot_detached_table;
+    snapshot_detached_table.detabase = it->second->getStorageID().getDatabaseName();
+    snapshot_detached_table.table = it->first;
+    if (it->second->getStorageID().hasUUID())
+    {
+        snapshot_detached_table.uuid = it->second->getStorageID().uuid;
+    }
+    snapshot_detached_table.is_permanently = false;
+    snapshot_detached_table.metadata_path = getObjectMetadataPath(snapshot_detached_table.table);
+
+    snapshot_detached_tables.emplace(it->first, std::move(snapshot_detached_table));
+
     tables.erase(it);
     res->is_detached = true;
     CurrentMetrics::sub(getAttachedCounterForStorage(res), 1);
@@ -298,6 +341,8 @@ void DatabaseWithOwnTablesBase::attachTableUnlocked(const String & table_name, c
         throw Exception(ErrorCodes::TABLE_ALREADY_EXISTS, "Table {} already exists.", table_id.getFullTableName());
     }
 
+    snapshot_detached_tables.erase(table_name);
+
     /// It is important to reset is_detached here since in case of RENAME in
     /// non-Atomic database the is_detached is set to true before RENAME.
     table->is_detached = false;
@@ -333,6 +378,7 @@ void DatabaseWithOwnTablesBase::shutdown()
 
     std::lock_guard lock(mutex);
     tables.clear();
+    snapshot_detached_tables.clear();
 }
 
 DatabaseWithOwnTablesBase::~DatabaseWithOwnTablesBase()
diff --git a/src/Databases/DatabasesCommon.h b/src/Databases/DatabasesCommon.h
index 2eecf8a564f..1ca49e90c23 100644
--- a/src/Databases/DatabasesCommon.h
+++ b/src/Databases/DatabasesCommon.h
@@ -37,6 +37,9 @@ public:
 
     DatabaseTablesIteratorPtr getTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name, bool skip_not_loaded) const override;
 
+    DatabaseDetachedTablesSnapshotIteratorPtr
+    getDetachedTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name, bool skip_not_loaded) const override;
+
     std::vector<std::pair<ASTPtr, StoragePtr>> getTablesForBackup(const FilterByNameFunction & filter, const ContextPtr & local_context) const override;
     void createTableRestoredFromBackup(const ASTPtr & create_table_query, ContextMutablePtr local_context, std::shared_ptr<IRestoreCoordination> restore_coordination, UInt64 timeout_ms) override;
 
@@ -46,12 +49,13 @@ public:
 
 protected:
     Tables tables TSA_GUARDED_BY(mutex);
+    SnapshotDetachedTables snapshot_detached_tables TSA_GUARDED_BY(mutex);
     LoggerPtr log;
 
     DatabaseWithOwnTablesBase(const String & name_, const String & logger, ContextPtr context);
 
     void attachTableUnlocked(const String & table_name, const StoragePtr & table) TSA_REQUIRES(mutex);
-    StoragePtr detachTableUnlocked(const String & table_name)  TSA_REQUIRES(mutex);
+    StoragePtr detachTableUnlocked(const String & table_name) TSA_REQUIRES(mutex);
     StoragePtr getTableUnlocked(const String & table_name) const TSA_REQUIRES(mutex);
     StoragePtr tryGetTableNoWait(const String & table_name) const;
 };
diff --git a/src/Databases/IDatabase.h b/src/Databases/IDatabase.h
index b00f2fe4baf..7c6b3b3004c 100644
--- a/src/Databases/IDatabase.h
+++ b/src/Databases/IDatabase.h
@@ -5,20 +5,22 @@
 #include <Interpreters/Context_fwd.h>
 #include <Interpreters/executeQuery.h>
 #include <Parsers/IAST_fwd.h>
+#include <QueryPipeline/BlockIO.h>
+#include <Storages/IStorage.h>
 #include <Storages/IStorage_fwd.h>
 #include <base/types.h>
-#include <Common/Exception.h>
 #include <Common/AsyncLoader.h>
+#include <Common/Exception.h>
 #include <Common/PoolId.h>
 #include <Common/ThreadPool_fwd.h>
-#include <QueryPipeline/BlockIO.h>
 
 #include <ctime>
 #include <functional>
+#include <map>
 #include <memory>
 #include <mutex>
+#include <stdexcept>
 #include <vector>
-#include <map>
 
 
 namespace DB
@@ -110,6 +112,55 @@ public:
 
 using DatabaseTablesIteratorPtr = std::unique_ptr<IDatabaseTablesIterator>;
 
+struct SnapshotDetachedTable final
+{
+    String detabase;
+    String table;
+    UUID uuid = UUIDHelpers::Nil;
+    String metadata_path;
+    bool is_permanently{false};
+};
+
+class DatabaseDetachedTablesSnapshotIterator
+{
+private:
+    SnapshotDetachedTables snapshot;
+    SnapshotDetachedTables::iterator it;
+
+protected:
+    DatabaseDetachedTablesSnapshotIterator(DatabaseDetachedTablesSnapshotIterator && other) noexcept
+    {
+        size_t idx = std::distance(other.snapshot.begin(), other.it);
+        std::swap(snapshot, other.snapshot);
+        other.it = other.snapshot.end();
+        it = snapshot.begin();
+        std::advance(it, idx);
+    }
+
+public:
+    explicit DatabaseDetachedTablesSnapshotIterator(const SnapshotDetachedTables & tables_) : snapshot(tables_), it(snapshot.begin()) { }
+
+    explicit DatabaseDetachedTablesSnapshotIterator(SnapshotDetachedTables && tables_) : snapshot(std::move(tables_)), it(snapshot.begin())
+    {
+    }
+
+    void next() { ++it; }
+
+    bool isValid() const { return it != snapshot.end(); }
+
+    String database() const { return it->second.detabase; }
+
+    String table() const { return it->second.table; }
+
+    UUID uuid() const { return it->second.uuid; }
+
+    String metadataPath() const { return it->second.metadata_path; }
+
+    bool isPermanently() const { return it->second.is_permanently; }
+};
+
+using DatabaseDetachedTablesSnapshotIteratorPtr = std::unique_ptr<DatabaseDetachedTablesSnapshotIterator>;
+
 
 /** Database engine.
   * It is responsible for:
@@ -232,6 +283,12 @@ public:
     /// Wait for all tables to be loaded and started up. If `skip_not_loaded` is true, then not yet loaded or not yet started up (at the moment of iterator creation) tables are excluded.
     virtual DatabaseTablesIteratorPtr getTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name = {}, bool skip_not_loaded = false) const = 0; /// NOLINT
 
+    virtual DatabaseDetachedTablesSnapshotIteratorPtr getDetachedTablesIterator(
+        ContextPtr /*context*/, const FilterByNameFunction & /*filter_by_table_name = {}*/, bool /*skip_not_loaded = false*/) const
+    {
+        throw Exception(ErrorCodes::NOT_IMPLEMENTED, "There is no get detached tables for Database{}", getEngineName());
+    }
+
     /// Returns list of table names.
     virtual Strings getAllTableNames(ContextPtr context) const
     {
diff --git a/src/Storages/IStorage_fwd.h b/src/Storages/IStorage_fwd.h
index b9243b029b0..4cbc586a745 100644
--- a/src/Storages/IStorage_fwd.h
+++ b/src/Storages/IStorage_fwd.h
@@ -1,5 +1,6 @@
 #pragma once
 
+#include <Core/Types_fwd.h>
 #include <base/types.h>
 
 #include <map>
@@ -9,9 +10,10 @@ namespace DB
 {
 
 class IStorage;
+struct SnapshotDetachedTable;
 
 using ConstStoragePtr = std::shared_ptr<const IStorage>;
 using StoragePtr = std::shared_ptr<IStorage>;
 using Tables = std::map<String, StoragePtr>;
-
+using SnapshotDetachedTables = std::map<String, SnapshotDetachedTable>;
 }
diff --git a/src/Storages/System/ReadFromSystemTables.cpp b/src/Storages/System/ReadFromSystemTables.cpp
new file mode 100644
index 00000000000..6e9c6a869d1
--- /dev/null
+++ b/src/Storages/System/ReadFromSystemTables.cpp
@@ -0,0 +1,138 @@
+#include "ReadFromSystemTables.h"
+
+#include <Core/Block.h>
+#include <Core/ColumnWithTypeAndName.h>
+#include <DataTypes/DataTypeString.h>
+#include <Interpreters/DatabaseCatalog.h>
+#include <QueryPipeline/QueryPipelineBuilder.h>
+#include <Storages/IStorage.h>
+#include <Storages/System/TablesBlockSource.h>
+#include <Storages/VirtualColumnUtils.h>
+
+#include <boost/range/adaptor/map.hpp>
+
+namespace DB
+{
+
+namespace
+{
+
+ColumnPtr getFilteredDatabases(const ActionsDAG::Node * predicate, ContextPtr context)
+{
+    MutableColumnPtr column = ColumnString::create();
+
+    const auto databases = DatabaseCatalog::instance().getDatabases();
+    for (const auto & database_name : databases | boost::adaptors::map_keys)
+    {
+        if (database_name == DatabaseCatalog::TEMPORARY_DATABASE)
+            continue; /// We don't want to show the internal database for temporary tables in system.tables
+
+        column->insert(database_name);
+    }
+
+    Block block{ColumnWithTypeAndName(std::move(column), std::make_shared<DataTypeString>(), "database")};
+    VirtualColumnUtils::filterBlockWithPredicate(predicate, block, context);
+    return block.getByPosition(0).column;
+}
+
+ColumnPtr getFilteredTables(
+    const ActionsDAG::Node * predicate, const ColumnPtr & filtered_databases_column, ContextPtr context, const bool need_detached_tables)
+{
+    Block sample{
+        ColumnWithTypeAndName(nullptr, std::make_shared<DataTypeString>(), "name"),
+        ColumnWithTypeAndName(nullptr, std::make_shared<DataTypeString>(), "engine")};
+
+    MutableColumnPtr database_column = ColumnString::create();
+    MutableColumnPtr engine_column;
+
+    auto dag = VirtualColumnUtils::splitFilterDagForAllowedInputs(predicate, &sample);
+    if (dag)
+    {
+        bool filter_by_engine = false;
+        for (const auto * input : dag->getInputs())
+            if (input->result_name == "engine")
+                filter_by_engine = true;
+
+        if (filter_by_engine)
+            engine_column = ColumnString::create();
+    }
+
+    for (size_t database_idx = 0; database_idx < filtered_databases_column->size(); ++database_idx)
+    {
+        const auto & database_name = filtered_databases_column->getDataAt(database_idx).toString();
+        DatabasePtr database = DatabaseCatalog::instance().tryGetDatabase(database_name);
+        if (!database)
+            continue;
+
+        if (need_detached_tables)
+        {
+            auto table_it = database->getDetachedTablesIterator(context, {}, false);
+            for (; table_it->isValid(); table_it->next())
+            {
+                database_column->insert(table_it->table());
+            }
+        }
+        else
+        {
+            auto table_it = database->getTablesIterator(context);
+            for (; table_it->isValid(); table_it->next())
+            {
+                database_column->insert(table_it->name());
+                if (engine_column)
+                    engine_column->insert(table_it->table()->getName());
+            }
+        }
+    }
+
+    Block block{ColumnWithTypeAndName(std::move(database_column), std::make_shared<DataTypeString>(), "name")};
+    if (engine_column)
+        block.insert(ColumnWithTypeAndName(std::move(engine_column), std::make_shared<DataTypeString>(), "engine"));
+
+    if (dag)
+        VirtualColumnUtils::filterBlockWithDAG(dag, block, context);
+
+    return block.getByPosition(0).column;
+}
+
+}
+
+ReadFromSystemTables::ReadFromSystemTables(
+    const Names & column_names_,
+    const SelectQueryInfo & query_info_,
+    const StorageSnapshotPtr & storage_snapshot_,
+    const ContextPtr & context_,
+    Block sample_block,
+    std::vector<UInt8> columns_mask_,
+    size_t max_block_size_,
+    const bool need_detached_tables_)
+    : SourceStepWithFilter(DataStream{.header = std::move(sample_block)}, column_names_, query_info_, storage_snapshot_, context_)
+    , columns_mask(std::move(columns_mask_))
+    , max_block_size(max_block_size_)
+    , need_detached_tables(need_detached_tables_)
+{
+}
+
+void ReadFromSystemTables::applyFilters(ActionDAGNodes added_filter_nodes)
+{
+    SourceStepWithFilter::applyFilters(std::move(added_filter_nodes));
+
+    const ActionsDAG::Node * predicate = nullptr;
+    if (filter_actions_dag)
+        predicate = filter_actions_dag->getOutputs().at(0);
+
+    filtered_databases_column = getFilteredDatabases(predicate, context);
+    filtered_tables_column = getFilteredTables(predicate, filtered_databases_column, context, need_detached_tables);
+}
+
+void ReadFromSystemTables::initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &)
+{
+    if (need_detached_tables)
+    {
+        pipeline.init(createPipe<DetachedTablesBlockSource>());
+    }
+    else
+    {
+        pipeline.init(createPipe<TablesBlockSource>());
+    }
+}
+}
diff --git a/src/Storages/System/ReadFromSystemTables.h b/src/Storages/System/ReadFromSystemTables.h
new file mode 100644
index 00000000000..f8280ffcc8d
--- /dev/null
+++ b/src/Storages/System/ReadFromSystemTables.h
@@ -0,0 +1,47 @@
+#pragma once
+
+#include <Processors/QueryPlan/SourceStepWithFilter.h>
+#include <QueryPipeline/Pipe.h>
+
+namespace DB
+{
+
+class ReadFromSystemTables : public SourceStepWithFilter
+{
+public:
+    std::string getName() const override { return "ReadFromSystemTables"; }
+    void initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) override;
+
+    ReadFromSystemTables(
+        const Names & column_names_,
+        const SelectQueryInfo & query_info_,
+        const StorageSnapshotPtr & storage_snapshot_,
+        const ContextPtr & context_,
+        Block sample_block,
+        std::vector<UInt8> columns_mask_,
+        size_t max_block_size_,
+        bool need_detached_tables);
+
+    void applyFilters(ActionDAGNodes added_filter_nodes) override;
+
+private:
+    std::vector<UInt8> columns_mask;
+    size_t max_block_size;
+    const bool need_detached_tables;
+
+    ColumnPtr filtered_databases_column;
+    ColumnPtr filtered_tables_column;
+
+    template <class T>
+    Pipe createPipe()
+    {
+        return Pipe(std::make_shared<T>(
+            std::move(columns_mask),
+            getOutputStream().header,
+            max_block_size,
+            std::move(filtered_databases_column),
+            std::move(filtered_tables_column),
+            context));
+    }
+};
+}
diff --git a/src/Storages/System/StorageSystemDetachedTables.cpp b/src/Storages/System/StorageSystemDetachedTables.cpp
new file mode 100644
index 00000000000..1596ac6a046
--- /dev/null
+++ b/src/Storages/System/StorageSystemDetachedTables.cpp
@@ -0,0 +1,56 @@
+#include "StorageSystemDetachedTables.h"
+
+#include <Core/NamesAndTypes.h>
+#include <DataTypes/DataTypeString.h>
+#include <DataTypes/DataTypeUUID.h>
+#include <DataTypes/DataTypesNumber.h>
+#include <Processors/QueryPlan/QueryPlan.h>
+#include <Storages/ColumnsDescription.h>
+#include <Storages/ProjectionsDescription.h>
+#include <Storages/StorageInMemoryMetadata.h>
+#include <Storages/System/ReadFromSystemTables.h>
+#include <Storages/System/TablesBlockSource.h>
+#include <Storages/System/getQueriedColumnsMaskAndHeader.h>
+
+
+namespace DB
+{
+
+StorageSystemDetachedTables::StorageSystemDetachedTables(const StorageID & table_id_) : IStorage(table_id_)
+{
+    StorageInMemoryMetadata storage_metadata;
+
+    auto description = ColumnsDescription{
+        ColumnDescription{"database", std::make_shared<DataTypeString>(), "The name of the database the table is in."},
+        ColumnDescription{"name", std::make_shared<DataTypeString>(), "Table name."},
+        ColumnDescription{"uuid", std::make_shared<DataTypeUUID>(), "Table uuid (Atomic database)."},
+        ColumnDescription{"metadata_path", std::make_shared<DataTypeString>(), "Path to the table metadata in the file system."},
+        ColumnDescription{"is_permanently", std::make_shared<DataTypeUInt8>(), "Table was detached permanently."},
+    };
+
+    storage_metadata.setColumns(std::move(description));
+
+    setInMemoryMetadata(storage_metadata);
+}
+
+void StorageSystemDetachedTables::read(
+    QueryPlan & query_plan,
+    const Names & column_names,
+    const StorageSnapshotPtr & storage_snapshot,
+    SelectQueryInfo & query_info,
+    ContextPtr context,
+    QueryProcessingStage::Enum /*processed_stage*/,
+    const size_t max_block_size,
+    size_t /*num_streams*/)
+{
+    storage_snapshot->check(column_names);
+    auto sample_block = storage_snapshot->metadata->getSampleBlock();
+
+    auto [columns_mask, res_block] = getQueriedColumnsMaskAndHeader(sample_block, column_names);
+
+    auto reading = std::make_unique<ReadFromSystemTables>(
+        column_names, query_info, storage_snapshot, context, std::move(res_block), std::move(columns_mask), max_block_size, true);
+
+    query_plan.addStep(std::move(reading));
+}
+}
diff --git a/src/Storages/System/StorageSystemDetachedTables.h b/src/Storages/System/StorageSystemDetachedTables.h
new file mode 100644
index 00000000000..cd042f51eaa
--- /dev/null
+++ b/src/Storages/System/StorageSystemDetachedTables.h
@@ -0,0 +1,32 @@
+#pragma once
+
+#include <Storages/IStorage.h>
+
+
+namespace DB
+{
+
+class Context;
+
+/** Implements the system table `detached_tables`, which allows you to get information about detached tables.
+  */
+class StorageSystemDetachedTables final : public IStorage
+{
+public:
+    explicit StorageSystemDetachedTables(const StorageID & table_id_);
+
+    std::string getName() const override { return "SystemDetachedTables"; }
+
+    void read(
+        QueryPlan & query_plan,
+        const Names & column_names,
+        const StorageSnapshotPtr & storage_snapshot,
+        SelectQueryInfo & /*query_info*/,
+        ContextPtr context,
+        QueryProcessingStage::Enum processed_stage,
+        size_t max_block_size,
+        size_t num_streams) override;
+
+    bool isSystemStorage() const override { return true; }
+};
+}
diff --git a/src/Storages/System/StorageSystemTables.cpp b/src/Storages/System/StorageSystemTables.cpp
index 783b899c978..5a8d6315d57 100644
--- a/src/Storages/System/StorageSystemTables.cpp
+++ b/src/Storages/System/StorageSystemTables.cpp
@@ -1,29 +1,31 @@
+#include "StorageSystemTables.h"
+
+#include <Access/ContextAccess.h>
 #include <Columns/ColumnString.h>
-#include <DataTypes/DataTypeString.h>
+#include <DataTypes/DataTypeArray.h>
 #include <DataTypes/DataTypeDateTime.h>
 #include <DataTypes/DataTypeNullable.h>
-#include <Storages/System/StorageSystemTables.h>
-#include <Storages/System/getQueriedColumnsMaskAndHeader.h>
-#include <Storages/MergeTree/MergeTreeData.h>
-#include <Storages/SelectQueryInfo.h>
-#include <Storages/VirtualColumnUtils.h>
-#include <Access/ContextAccess.h>
+#include <DataTypes/DataTypeString.h>
+#include <DataTypes/DataTypeUUID.h>
+#include <DataTypes/DataTypesNumber.h>
+#include <Disks/IStoragePolicy.h>
 #include <Interpreters/Context.h>
 #include <Interpreters/DatabaseCatalog.h>
 #include <Interpreters/formatWithPossiblyHidingSecrets.h>
 #include <Parsers/ASTCreateQuery.h>
 #include <Parsers/ASTSelectWithUnionQuery.h>
-#include <Common/typeid_cast.h>
-#include <Common/StringUtils.h>
-#include <DataTypes/DataTypesNumber.h>
-#include <DataTypes/DataTypeArray.h>
-#include <Disks/IStoragePolicy.h>
 #include <Processors/ISource.h>
 #include <Processors/QueryPlan/QueryPlan.h>
 #include <Processors/QueryPlan/SourceStepWithFilter.h>
 #include <QueryPipeline/Pipe.h>
 #include <QueryPipeline/QueryPipelineBuilder.h>
-#include <DataTypes/DataTypeUUID.h>
+#include <Storages/MergeTree/MergeTreeData.h>
+#include <Storages/SelectQueryInfo.h>
+#include <Storages/System/ReadFromSystemTables.h>
+#include <Storages/System/getQueriedColumnsMaskAndHeader.h>
+#include <Storages/VirtualColumnUtils.h>
+#include <Common/StringUtils.h>
+#include <Common/typeid_cast.h>
 
 #include <boost/range/adaptor/map.hpp>
 
@@ -32,8 +34,7 @@ namespace DB
 {
 
 
-StorageSystemTables::StorageSystemTables(const StorageID & table_id_)
-    : IStorage(table_id_)
+StorageSystemTables::StorageSystemTables(const StorageID & table_id_) : IStorage(table_id_)
 {
     StorageInMemoryMetadata storage_metadata;
 
@@ -46,9 +47,13 @@ StorageSystemTables::StorageSystemTables(const StorageID & table_id_)
         {"data_paths", std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()), "Paths to the table data in the file systems."},
         {"metadata_path", std::make_shared<DataTypeString>(), "Path to the table metadata in the file system."},
         {"metadata_modification_time", std::make_shared<DataTypeDateTime>(), "Time of latest modification of the table metadata."},
-        {"metadata_version", std::make_shared<DataTypeInt32>(), "Metadata version for ReplicatedMergeTree table, 0 for non ReplicatedMergeTree table."},
+        {"metadata_version",
+         std::make_shared<DataTypeInt32>(),
+         "Metadata version for ReplicatedMergeTree table, 0 for non ReplicatedMergeTree table."},
         {"dependencies_database", std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()), "Database dependencies."},
-        {"dependencies_table", std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()), "Table dependencies (materialized views the current table)."},
+        {"dependencies_table",
+         std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()),
+         "Table dependencies (materialized views the current table)."},
         {"create_table_query", std::make_shared<DataTypeString>(), "The query that was used to create the table."},
         {"engine_full", std::make_shared<DataTypeString>(), "Parameters of the table engine."},
         {"as_select", std::make_shared<DataTypeString>(), "SELECT query for view."},
@@ -57,676 +62,54 @@ StorageSystemTables::StorageSystemTables(const StorageID & table_id_)
         {"primary_key", std::make_shared<DataTypeString>(), "The primary key expression specified in the table."},
         {"sampling_key", std::make_shared<DataTypeString>(), "The sampling key expression specified in the table."},
         {"storage_policy", std::make_shared<DataTypeString>(), "The storage policy."},
-        {"total_rows", std::make_shared<DataTypeNullable>(std::make_shared<DataTypeUInt64>()),
-            "Total number of rows, if it is possible to quickly determine exact number of rows in the table, otherwise NULL (including underlying Buffer table)."
-        },
-        {"total_bytes", std::make_shared<DataTypeNullable>(std::make_shared<DataTypeUInt64>()),
-            "Total number of bytes, if it is possible to quickly determine exact number "
-            "of bytes for the table on storage, otherwise NULL (does not includes any underlying storage). "
-            "If the table stores data on disk, returns used space on disk (i.e. compressed). "
-            "If the table stores data in memory, returns approximated number of used bytes in memory."
-        },
-        {"total_bytes_uncompressed", std::make_shared<DataTypeNullable>(std::make_shared<DataTypeUInt64>()),
-            "Total number of uncompressed bytes, if it's possible to quickly determine the exact number "
-            "of bytes from the part checksums for the table on storage, otherwise NULL (does not take underlying storage (if any) into account)."
-        },
+        {"total_rows",
+         std::make_shared<DataTypeNullable>(std::make_shared<DataTypeUInt64>()),
+         "Total number of rows, if it is possible to quickly determine exact number of rows in the table, otherwise NULL (including "
+         "underlying Buffer table)."},
+        {"total_bytes",
+         std::make_shared<DataTypeNullable>(std::make_shared<DataTypeUInt64>()),
+         "Total number of bytes, if it is possible to quickly determine exact number "
+         "of bytes for the table on storage, otherwise NULL (does not includes any underlying storage). "
+         "If the table stores data on disk, returns used space on disk (i.e. compressed). "
+         "If the table stores data in memory, returns approximated number of used bytes in memory."},
+        {"total_bytes_uncompressed",
+         std::make_shared<DataTypeNullable>(std::make_shared<DataTypeUInt64>()),
+         "Total number of uncompressed bytes, if it's possible to quickly determine the exact number "
+         "of bytes from the part checksums for the table on storage, otherwise NULL (does not take underlying storage (if any) into "
+         "account)."},
         {"parts", std::make_shared<DataTypeNullable>(std::make_shared<DataTypeUInt64>()), "The total number of parts in this table."},
-        {"active_parts", std::make_shared<DataTypeNullable>(std::make_shared<DataTypeUInt64>()), "The number of active parts in this table."},
-        {"total_marks", std::make_shared<DataTypeNullable>(std::make_shared<DataTypeUInt64>()), "The total number of marks in all parts in this table."},
-        {"lifetime_rows", std::make_shared<DataTypeNullable>(std::make_shared<DataTypeUInt64>()),
-            "Total number of rows INSERTed since server start (only for Buffer tables)."
-        },
-        {"lifetime_bytes", std::make_shared<DataTypeNullable>(std::make_shared<DataTypeUInt64>()),
-            "Total number of bytes INSERTed since server start (only for Buffer tables)."
-        },
+        {"active_parts",
+         std::make_shared<DataTypeNullable>(std::make_shared<DataTypeUInt64>()),
+         "The number of active parts in this table."},
+        {"total_marks",
+         std::make_shared<DataTypeNullable>(std::make_shared<DataTypeUInt64>()),
+         "The total number of marks in all parts in this table."},
+        {"lifetime_rows",
+         std::make_shared<DataTypeNullable>(std::make_shared<DataTypeUInt64>()),
+         "Total number of rows INSERTed since server start (only for Buffer tables)."},
+        {"lifetime_bytes",
+         std::make_shared<DataTypeNullable>(std::make_shared<DataTypeUInt64>()),
+         "Total number of bytes INSERTed since server start (only for Buffer tables)."},
         {"comment", std::make_shared<DataTypeString>(), "The comment for the table."},
-        {"has_own_data", std::make_shared<DataTypeUInt8>(),
-            "Flag that indicates whether the table itself stores some data on disk or only accesses some other source."
-        },
-        {"loading_dependencies_database", std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()),
-            "Database loading dependencies (list of objects which should be loaded before the current object)."
-        },
-        {"loading_dependencies_table", std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()),
-            "Table loading dependencies (list of objects which should be loaded before the current object)."
-        },
-        {"loading_dependent_database", std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()),
-            "Dependent loading database."
-        },
-        {"loading_dependent_table", std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()),
-            "Dependent loading table."
-        },
+        {"has_own_data",
+         std::make_shared<DataTypeUInt8>(),
+         "Flag that indicates whether the table itself stores some data on disk or only accesses some other source."},
+        {"loading_dependencies_database",
+         std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()),
+         "Database loading dependencies (list of objects which should be loaded before the current object)."},
+        {"loading_dependencies_table",
+         std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()),
+         "Table loading dependencies (list of objects which should be loaded before the current object)."},
+        {"loading_dependent_database", std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()), "Dependent loading database."},
+        {"loading_dependent_table", std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()), "Dependent loading table."},
     };
 
-    description.setAliases({
-        {"table", std::make_shared<DataTypeString>(), "name"}
-    });
+    description.setAliases({{"table", std::make_shared<DataTypeString>(), "name"}});
 
     storage_metadata.setColumns(std::move(description));
     setInMemoryMetadata(storage_metadata);
 }
 
-
-namespace
-{
-
-ColumnPtr getFilteredDatabases(const ActionsDAG::Node * predicate, ContextPtr context)
-{
-    MutableColumnPtr column = ColumnString::create();
-
-    const auto databases = DatabaseCatalog::instance().getDatabases();
-    for (const auto & database_name : databases | boost::adaptors::map_keys)
-    {
-        if (database_name == DatabaseCatalog::TEMPORARY_DATABASE)
-            continue; /// We don't want to show the internal database for temporary tables in system.tables
-
-        column->insert(database_name);
-    }
-
-    Block block { ColumnWithTypeAndName(std::move(column), std::make_shared<DataTypeString>(), "database") };
-    VirtualColumnUtils::filterBlockWithPredicate(predicate, block, context);
-    return block.getByPosition(0).column;
-}
-
-ColumnPtr getFilteredTables(const ActionsDAG::Node * predicate, const ColumnPtr & filtered_databases_column, ContextPtr context)
-{
-    Block sample {
-        ColumnWithTypeAndName(nullptr, std::make_shared<DataTypeString>(), "name"),
-        ColumnWithTypeAndName(nullptr, std::make_shared<DataTypeString>(), "engine")
-    };
-
-    MutableColumnPtr database_column = ColumnString::create();
-    MutableColumnPtr engine_column;
-
-    auto dag = VirtualColumnUtils::splitFilterDagForAllowedInputs(predicate, &sample);
-    if (dag)
-    {
-        bool filter_by_engine = false;
-        for (const auto * input : dag->getInputs())
-            if (input->result_name == "engine")
-                filter_by_engine = true;
-
-        if (filter_by_engine)
-            engine_column = ColumnString::create();
-    }
-
-    for (size_t database_idx = 0; database_idx < filtered_databases_column->size(); ++database_idx)
-    {
-        const auto & database_name = filtered_databases_column->getDataAt(database_idx).toString();
-        DatabasePtr database = DatabaseCatalog::instance().tryGetDatabase(database_name);
-        if (!database)
-            continue;
-
-        for (auto table_it = database->getTablesIterator(context); table_it->isValid(); table_it->next())
-        {
-            database_column->insert(table_it->name());
-            if (engine_column)
-                engine_column->insert(table_it->table()->getName());
-        }
-    }
-
-    Block block {ColumnWithTypeAndName(std::move(database_column), std::make_shared<DataTypeString>(), "name")};
-    if (engine_column)
-        block.insert(ColumnWithTypeAndName(std::move(engine_column), std::make_shared<DataTypeString>(), "engine"));
-
-    if (dag)
-        VirtualColumnUtils::filterBlockWithDAG(dag, block, context);
-
-    return block.getByPosition(0).column;
-}
-
-/// Avoid heavy operation on tables if we only queried columns that we can get without table object.
-/// Otherwise it will require table initialization for Lazy database.
-bool needTable(const DatabasePtr & database, const Block & header)
-{
-    if (database->getEngineName() != "Lazy")
-        return true;
-
-    static const std::set<std::string> columns_without_table = { "database", "name", "uuid", "metadata_modification_time" };
-    for (const auto & column : header.getColumnsWithTypeAndName())
-    {
-        if (columns_without_table.find(column.name) == columns_without_table.end())
-            return true;
-    }
-    return false;
-}
-
-
-class TablesBlockSource : public ISource
-{
-public:
-    TablesBlockSource(
-        std::vector<UInt8> columns_mask_,
-        Block header,
-        UInt64 max_block_size_,
-        ColumnPtr databases_,
-        ColumnPtr tables_,
-        ContextPtr context_)
-        : ISource(std::move(header))
-        , columns_mask(std::move(columns_mask_))
-        , max_block_size(max_block_size_)
-        , databases(std::move(databases_))
-        , context(Context::createCopy(context_))
-    {
-        size_t size = tables_->size();
-        tables.reserve(size);
-        for (size_t idx = 0; idx < size; ++idx)
-            tables.insert(tables_->getDataAt(idx).toString());
-    }
-
-    String getName() const override { return "Tables"; }
-
-protected:
-    Chunk generate() override
-    {
-        if (done)
-            return {};
-
-        MutableColumns res_columns = getPort().getHeader().cloneEmptyColumns();
-
-        const auto access = context->getAccess();
-        const bool need_to_check_access_for_databases = !access->isGranted(AccessType::SHOW_TABLES);
-
-        size_t rows_count = 0;
-        while (rows_count < max_block_size)
-        {
-            if (tables_it && !tables_it->isValid())
-                ++database_idx;
-
-            while (database_idx < databases->size() && (!tables_it || !tables_it->isValid()))
-            {
-                database_name = databases->getDataAt(database_idx).toString();
-                database = DatabaseCatalog::instance().tryGetDatabase(database_name);
-
-                if (!database)
-                {
-                    /// Database was deleted just now or the user has no access.
-                    ++database_idx;
-                    continue;
-                }
-
-                break;
-            }
-
-            /// This is for temporary tables. They are output in single block regardless to max_block_size.
-            if (database_idx >= databases->size())
-            {
-                if (context->hasSessionContext())
-                {
-                    Tables external_tables = context->getSessionContext()->getExternalTables();
-
-                    for (auto & table : external_tables)
-                    {
-                        size_t src_index = 0;
-                        size_t res_index = 0;
-
-                        // database
-                        if (columns_mask[src_index++])
-                            res_columns[res_index++]->insertDefault();
-
-                        // name
-                        if (columns_mask[src_index++])
-                            res_columns[res_index++]->insert(table.first);
-
-                        // uuid
-                        if (columns_mask[src_index++])
-                            res_columns[res_index++]->insert(table.second->getStorageID().uuid);
-
-                        // engine
-                        if (columns_mask[src_index++])
-                            res_columns[res_index++]->insert(table.second->getName());
-
-                        // is_temporary
-                        if (columns_mask[src_index++])
-                            res_columns[res_index++]->insert(1u);
-
-                        // data_paths
-                        if (columns_mask[src_index++])
-                            res_columns[res_index++]->insertDefault();
-
-                        // metadata_path
-                        if (columns_mask[src_index++])
-                            res_columns[res_index++]->insertDefault();
-
-                        // metadata_modification_time
-                        if (columns_mask[src_index++])
-                            res_columns[res_index++]->insertDefault();
-
-                        // metadata_version
-                        // Temporary tables does not support replication
-                        if (columns_mask[src_index++])
-                            res_columns[res_index++]->insertDefault();
-
-                        // dependencies_database
-                        if (columns_mask[src_index++])
-                            res_columns[res_index++]->insertDefault();
-
-                        // dependencies_table
-                        if (columns_mask[src_index++])
-                            res_columns[res_index++]->insertDefault();
-
-                        // create_table_query
-                        if (columns_mask[src_index++])
-                        {
-                            auto temp_db = DatabaseCatalog::instance().getDatabaseForTemporaryTables();
-                            ASTPtr ast = temp_db ? temp_db->tryGetCreateTableQuery(table.second->getStorageID().getTableName(), context) : nullptr;
-                            res_columns[res_index++]->insert(ast ? format({context, *ast}) : "");
-                        }
-
-                        // engine_full
-                        if (columns_mask[src_index++])
-                            res_columns[res_index++]->insert(table.second->getName());
-
-                        const auto & settings = context->getSettingsRef();
-                        while (src_index < columns_mask.size())
-                        {
-                            // total_rows
-                            if (src_index == 19 && columns_mask[src_index])
-                            {
-                                if (auto total_rows = table.second->totalRows(settings))
-                                    res_columns[res_index++]->insert(*total_rows);
-                                else
-                                    res_columns[res_index++]->insertDefault();
-                            }
-                            // total_bytes
-                            else if (src_index == 20 && columns_mask[src_index])
-                            {
-                                if (auto total_bytes = table.second->totalBytes(settings))
-                                    res_columns[res_index++]->insert(*total_bytes);
-                                else
-                                    res_columns[res_index++]->insertDefault();
-                            }
-                            /// Fill the rest columns with defaults
-                            else if (columns_mask[src_index])
-                                res_columns[res_index++]->insertDefault();
-                            src_index++;
-                        }
-                    }
-                }
-
-                UInt64 num_rows = res_columns.at(0)->size();
-                done = true;
-                return Chunk(std::move(res_columns), num_rows);
-            }
-
-            const bool need_to_check_access_for_tables = need_to_check_access_for_databases && !access->isGranted(AccessType::SHOW_TABLES, database_name);
-
-            if (!tables_it || !tables_it->isValid())
-                tables_it = database->getTablesIterator(context);
-
-            const bool need_table = needTable(database, getPort().getHeader());
-
-            for (; rows_count < max_block_size && tables_it->isValid(); tables_it->next())
-            {
-                auto table_name = tables_it->name();
-                if (!tables.contains(table_name))
-                    continue;
-
-                if (need_to_check_access_for_tables && !access->isGranted(AccessType::SHOW_TABLES, database_name, table_name))
-                    continue;
-
-                StoragePtr table = nullptr;
-                TableLockHolder lock;
-                if (need_table)
-                {
-                    table = tables_it->table();
-                    if (!table)
-                        // Table might have just been removed or detached for Lazy engine (see DatabaseLazy::tryGetTable())
-                        continue;
-
-                    /// The only column that requires us to hold a shared lock is data_paths as rename might alter them (on ordinary tables)
-                    /// and it's not protected internally by other mutexes
-                    static const size_t DATA_PATHS_INDEX = 5;
-                    if (columns_mask[DATA_PATHS_INDEX])
-                    {
-                        lock = table->tryLockForShare(context->getCurrentQueryId(),
-                                                      context->getSettingsRef().lock_acquire_timeout);
-                        if (!lock)
-                            // Table was dropped while acquiring the lock, skipping table
-                            continue;
-                    }
-                }
-                ++rows_count;
-
-                size_t src_index = 0;
-                size_t res_index = 0;
-
-                if (columns_mask[src_index++])
-                    res_columns[res_index++]->insert(database_name);
-
-                if (columns_mask[src_index++])
-                    res_columns[res_index++]->insert(table_name);
-
-                if (columns_mask[src_index++])
-                    res_columns[res_index++]->insert(tables_it->uuid());
-
-                if (columns_mask[src_index++])
-                {
-                    chassert(table != nullptr);
-                    res_columns[res_index++]->insert(table->getName());
-                }
-
-                if (columns_mask[src_index++])
-                    res_columns[res_index++]->insert(0u);  // is_temporary
-
-                if (columns_mask[src_index++])
-                {
-                    chassert(lock != nullptr);
-                    Array table_paths_array;
-                    auto paths = table->getDataPaths();
-                    table_paths_array.reserve(paths.size());
-                    for (const String & path : paths)
-                        table_paths_array.push_back(path);
-                    res_columns[res_index++]->insert(table_paths_array);
-                    /// We don't need the lock anymore
-                    lock = nullptr;
-                }
-
-                if (columns_mask[src_index++])
-                    res_columns[res_index++]->insert(database->getObjectMetadataPath(table_name));
-
-                if (columns_mask[src_index++])
-                    res_columns[res_index++]->insert(static_cast<UInt64>(database->getObjectMetadataModificationTime(table_name)));
-
-                StorageMetadataPtr metadata_snapshot;
-                if (table)
-                    metadata_snapshot = table->getInMemoryMetadataPtr();
-
-                if (columns_mask[src_index++])
-                {
-                    if (metadata_snapshot && table->supportsReplication())
-                        res_columns[res_index++]->insert(metadata_snapshot->metadata_version);
-                    else
-                        res_columns[res_index++]->insertDefault();
-                }
-
-                {
-                    Array views_table_name_array;
-                    Array views_database_name_array;
-                    if (columns_mask[src_index] || columns_mask[src_index + 1])
-                    {
-                        const auto view_ids = DatabaseCatalog::instance().getDependentViews(StorageID(database_name, table_name));
-
-                        views_table_name_array.reserve(view_ids.size());
-                        views_database_name_array.reserve(view_ids.size());
-                        for (const auto & view_id : view_ids)
-                        {
-                            views_table_name_array.push_back(view_id.table_name);
-                            views_database_name_array.push_back(view_id.database_name);
-                        }
-                    }
-
-                    if (columns_mask[src_index++])
-                        res_columns[res_index++]->insert(views_database_name_array);
-
-                    if (columns_mask[src_index++])
-                        res_columns[res_index++]->insert(views_table_name_array);
-                }
-
-                if (columns_mask[src_index] || columns_mask[src_index + 1] || columns_mask[src_index + 2])
-                {
-                    ASTPtr ast = database->tryGetCreateTableQuery(table_name, context);
-                    auto * ast_create = ast ? ast->as<ASTCreateQuery>() : nullptr;
-
-                    if (ast_create && !context->getSettingsRef().show_table_uuid_in_table_create_query_if_not_nil)
-                    {
-                        ast_create->uuid = UUIDHelpers::Nil;
-                        ast_create->to_inner_uuid = UUIDHelpers::Nil;
-                    }
-
-                    if (columns_mask[src_index++])
-                        res_columns[res_index++]->insert(ast ? format({context, *ast}) : "");
-
-                    if (columns_mask[src_index++])
-                    {
-                        String engine_full;
-
-                        if (ast_create && ast_create->storage)
-                        {
-                            engine_full = format({context, *ast_create->storage});
-
-                            static const char * const extra_head = " ENGINE = ";
-                            if (startsWith(engine_full, extra_head))
-                                engine_full = engine_full.substr(strlen(extra_head));
-                        }
-
-                        res_columns[res_index++]->insert(engine_full);
-                    }
-
-                    if (columns_mask[src_index++])
-                    {
-                        String as_select;
-                        if (ast_create && ast_create->select)
-                            as_select = format({context, *ast_create->select});
-                        res_columns[res_index++]->insert(as_select);
-                    }
-                }
-                else
-                    src_index += 3;
-
-                ASTPtr expression_ptr;
-                if (columns_mask[src_index++])
-                {
-                    if (metadata_snapshot && (expression_ptr = metadata_snapshot->getPartitionKeyAST()))
-                        res_columns[res_index++]->insert(format({context, *expression_ptr}));
-                    else
-                        res_columns[res_index++]->insertDefault();
-                }
-
-                if (columns_mask[src_index++])
-                {
-                    if (metadata_snapshot && (expression_ptr = metadata_snapshot->getSortingKey().expression_list_ast))
-                        res_columns[res_index++]->insert(format({context, *expression_ptr}));
-                    else
-                        res_columns[res_index++]->insertDefault();
-                }
-
-                if (columns_mask[src_index++])
-                {
-                    if (metadata_snapshot && (expression_ptr = metadata_snapshot->getPrimaryKey().expression_list_ast))
-                        res_columns[res_index++]->insert(format({context, *expression_ptr}));
-                    else
-                        res_columns[res_index++]->insertDefault();
-                }
-
-                if (columns_mask[src_index++])
-                {
-                    if (metadata_snapshot && (expression_ptr = metadata_snapshot->getSamplingKeyAST()))
-                        res_columns[res_index++]->insert(format({context, *expression_ptr}));
-                    else
-                        res_columns[res_index++]->insertDefault();
-                }
-
-                if (columns_mask[src_index++])
-                {
-                    auto policy = table ? table->getStoragePolicy() : nullptr;
-                    if (policy)
-                        res_columns[res_index++]->insert(policy->getName());
-                    else
-                        res_columns[res_index++]->insertDefault();
-                }
-
-                auto settings = context->getSettingsRef();
-                settings.select_sequential_consistency = 0;
-                if (columns_mask[src_index++])
-                {
-                    auto total_rows = table ? table->totalRows(settings) : std::nullopt;
-                    if (total_rows)
-                        res_columns[res_index++]->insert(*total_rows);
-                    else
-                        res_columns[res_index++]->insertDefault();
-                }
-
-                if (columns_mask[src_index++])
-                {
-                    auto total_bytes = table->totalBytes(settings);
-                    if (total_bytes)
-                        res_columns[res_index++]->insert(*total_bytes);
-                    else
-                        res_columns[res_index++]->insertDefault();
-                }
-
-                if (columns_mask[src_index++])
-                {
-                    auto total_bytes_uncompressed = table->totalBytesUncompressed(settings);
-                    if (total_bytes_uncompressed)
-                        res_columns[res_index++]->insert(*total_bytes_uncompressed);
-                    else
-                        res_columns[res_index++]->insertDefault();
-                }
-
-                auto table_merge_tree = std::dynamic_pointer_cast<MergeTreeData>(table);
-                if (columns_mask[src_index++])
-                {
-                    if (table_merge_tree)
-                        res_columns[res_index++]->insert(table_merge_tree->getAllPartsCount());
-                    else
-                        res_columns[res_index++]->insertDefault();
-                }
-
-                if (columns_mask[src_index++])
-                {
-                    if (table_merge_tree)
-                        res_columns[res_index++]->insert(table_merge_tree->getActivePartsCount());
-                    else
-                        res_columns[res_index++]->insertDefault();
-                }
-
-                if (columns_mask[src_index++])
-                {
-                    if (table_merge_tree)
-                    {
-                        res_columns[res_index++]->insert(table_merge_tree->getTotalMarksCount());
-                    }
-                    else
-                        res_columns[res_index++]->insertDefault();
-                }
-
-                if (columns_mask[src_index++])
-                {
-                    auto lifetime_rows = table ? table->lifetimeRows() : std::nullopt;
-                    if (lifetime_rows)
-                        res_columns[res_index++]->insert(*lifetime_rows);
-                    else
-                        res_columns[res_index++]->insertDefault();
-                }
-
-                if (columns_mask[src_index++])
-                {
-                    auto lifetime_bytes = table ? table->lifetimeBytes() : std::nullopt;
-                    if (lifetime_bytes)
-                        res_columns[res_index++]->insert(*lifetime_bytes);
-                    else
-                        res_columns[res_index++]->insertDefault();
-                }
-
-                if (columns_mask[src_index++])
-                {
-                    if (metadata_snapshot)
-                        res_columns[res_index++]->insert(metadata_snapshot->comment);
-                    else
-                        res_columns[res_index++]->insertDefault();
-                }
-
-                if (columns_mask[src_index++])
-                {
-                    if (table)
-                        res_columns[res_index++]->insert(table->storesDataOnDisk());
-                    else
-                        res_columns[res_index++]->insertDefault();
-                }
-
-                if (columns_mask[src_index] || columns_mask[src_index + 1] || columns_mask[src_index + 2] || columns_mask[src_index + 3])
-                {
-                    auto dependencies = DatabaseCatalog::instance().getLoadingDependencies(StorageID{database_name, table_name});
-                    auto dependents = DatabaseCatalog::instance().getLoadingDependents(StorageID{database_name, table_name});
-
-                    Array dependencies_databases;
-                    Array dependencies_tables;
-                    dependencies_databases.reserve(dependencies.size());
-                    dependencies_tables.reserve(dependencies.size());
-                    for (const auto & dependency : dependencies)
-                    {
-                        dependencies_databases.push_back(dependency.database_name);
-                        dependencies_tables.push_back(dependency.table_name);
-                    }
-
-                    Array dependents_databases;
-                    Array dependents_tables;
-                    dependents_databases.reserve(dependents.size());
-                    dependents_tables.reserve(dependents.size());
-                    for (const auto & dependent : dependents)
-                    {
-                        dependents_databases.push_back(dependent.database_name);
-                        dependents_tables.push_back(dependent.table_name);
-                    }
-
-                    if (columns_mask[src_index++])
-                        res_columns[res_index++]->insert(dependencies_databases);
-                    if (columns_mask[src_index++])
-                        res_columns[res_index++]->insert(dependencies_tables);
-
-                    if (columns_mask[src_index++])
-                        res_columns[res_index++]->insert(dependents_databases);
-                    if (columns_mask[src_index++])
-                        res_columns[res_index++]->insert(dependents_tables);
-
-                }
-            }
-        }
-
-        UInt64 num_rows = res_columns.at(0)->size();
-        return Chunk(std::move(res_columns), num_rows);
-    }
-private:
-    std::vector<UInt8> columns_mask;
-    UInt64 max_block_size;
-    ColumnPtr databases;
-    NameSet tables;
-    size_t database_idx = 0;
-    DatabaseTablesIteratorPtr tables_it;
-    ContextPtr context;
-    bool done = false;
-    DatabasePtr database;
-    std::string database_name;
-};
-
-}
-
-class ReadFromSystemTables : public SourceStepWithFilter
-{
-public:
-    std::string getName() const override { return "ReadFromSystemTables"; }
-    void initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) override;
-
-    ReadFromSystemTables(
-        const Names & column_names_,
-        const SelectQueryInfo & query_info_,
-        const StorageSnapshotPtr & storage_snapshot_,
-        const ContextPtr & context_,
-        Block sample_block,
-        std::vector<UInt8> columns_mask_,
-        size_t max_block_size_)
-        : SourceStepWithFilter(
-            DataStream{.header = std::move(sample_block)},
-            column_names_,
-            query_info_,
-            storage_snapshot_,
-            context_)
-        , columns_mask(std::move(columns_mask_))
-        , max_block_size(max_block_size_)
-    {
-    }
-
-    void applyFilters(ActionDAGNodes added_filter_nodes) override;
-
-private:
-    std::vector<UInt8> columns_mask;
-    size_t max_block_size;
-
-    ColumnPtr filtered_databases_column;
-    ColumnPtr filtered_tables_column;
-};
-
 void StorageSystemTables::read(
     QueryPlan & query_plan,
     const Names & column_names,
@@ -743,28 +126,8 @@ void StorageSystemTables::read(
     auto [columns_mask, res_block] = getQueriedColumnsMaskAndHeader(sample_block, column_names);
 
     auto reading = std::make_unique<ReadFromSystemTables>(
-        column_names, query_info, storage_snapshot, context, std::move(res_block), std::move(columns_mask), max_block_size);
+        column_names, query_info, storage_snapshot, context, std::move(res_block), std::move(columns_mask), max_block_size, false);
 
     query_plan.addStep(std::move(reading));
 }
-
-void ReadFromSystemTables::applyFilters(ActionDAGNodes added_filter_nodes)
-{
-    SourceStepWithFilter::applyFilters(std::move(added_filter_nodes));
-
-    const ActionsDAG::Node * predicate = nullptr;
-    if (filter_actions_dag)
-        predicate = filter_actions_dag->getOutputs().at(0);
-
-    filtered_databases_column = getFilteredDatabases(predicate, context);
-    filtered_tables_column = getFilteredTables(predicate, filtered_databases_column, context);
-}
-
-void ReadFromSystemTables::initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &)
-{
-    Pipe pipe(std::make_shared<TablesBlockSource>(
-        std::move(columns_mask), getOutputStream().header, max_block_size, std::move(filtered_databases_column), std::move(filtered_tables_column), context));
-    pipeline.init(std::move(pipe));
-}
-
 }
diff --git a/src/Storages/System/TablesBlockSource.cpp b/src/Storages/System/TablesBlockSource.cpp
new file mode 100644
index 00000000000..4ea5bb53deb
--- /dev/null
+++ b/src/Storages/System/TablesBlockSource.cpp
@@ -0,0 +1,569 @@
+#include "TablesBlockSource.h"
+
+#include <memory>
+
+#include <Access/ContextAccess.h>
+#include <Disks/IStoragePolicy.h>
+#include <Interpreters/DatabaseCatalog.h>
+#include <Interpreters/formatWithPossiblyHidingSecrets.h>
+#include <Parsers/ASTSelectWithUnionQuery.h>
+#include <Storages/IStorage.h>
+#include <Storages/MergeTree/MergeTreeData.h>
+#include <Common/logger_useful.h>
+
+namespace DB
+{
+
+namespace
+{
+
+/// Avoid heavy operation on tables if we only queried columns that we can get without table object.
+/// Otherwise it will require table initialization for Lazy database.
+bool needTable(const DatabasePtr & database, const Block & header)
+{
+    if (database->getEngineName() != "Lazy")
+        return true;
+
+    static const std::set<std::string> columns_without_table = { "database", "name", "uuid", "metadata_modification_time" };
+    for (const auto & column : header.getColumnsWithTypeAndName())
+    {
+        if (columns_without_table.find(column.name) == columns_without_table.end())
+            return true;
+    }
+    return false;
+}
+
+}
+
+Chunk TablesBlockSource::generate()
+{
+    if (done)
+        return {};
+
+    MutableColumns res_columns = getPort().getHeader().cloneEmptyColumns();
+
+    const auto access = context->getAccess();
+    const bool need_to_check_access_for_databases = !access->isGranted(AccessType::SHOW_TABLES);
+
+    size_t rows_count = 0;
+    while (rows_count < max_block_size)
+    {
+        if (tables_it && !tables_it->isValid())
+            ++database_idx;
+
+        while (database_idx < databases->size() && (!tables_it || !tables_it->isValid()))
+        {
+            database_name = databases->getDataAt(database_idx).toString();
+            database = DatabaseCatalog::instance().tryGetDatabase(database_name);
+
+            if (!database)
+            {
+                /// Database was deleted just now or the user has no access.
+                ++database_idx;
+                continue;
+            }
+
+            break;
+        }
+
+        /// This is for temporary tables. They are output in single block regardless to max_block_size.
+        if (database_idx >= databases->size())
+        {
+            if (context->hasSessionContext())
+            {
+                Tables external_tables = context->getSessionContext()->getExternalTables();
+
+                for (auto & table : external_tables)
+                {
+                    size_t src_index = 0;
+                    size_t res_index = 0;
+
+                    // database
+                    if (columns_mask[src_index++])
+                        res_columns[res_index++]->insertDefault();
+
+                    // name
+                    if (columns_mask[src_index++])
+                        res_columns[res_index++]->insert(table.first);
+
+                    // uuid
+                    if (columns_mask[src_index++])
+                        res_columns[res_index++]->insert(table.second->getStorageID().uuid);
+
+                    // engine
+                    if (columns_mask[src_index++])
+                        res_columns[res_index++]->insert(table.second->getName());
+
+                    // is_temporary
+                    if (columns_mask[src_index++])
+                        res_columns[res_index++]->insert(1u);
+
+                    // data_paths
+                    if (columns_mask[src_index++])
+                        res_columns[res_index++]->insertDefault();
+
+                    // metadata_path
+                    if (columns_mask[src_index++])
+                        res_columns[res_index++]->insertDefault();
+
+                    // metadata_modification_time
+                    if (columns_mask[src_index++])
+                        res_columns[res_index++]->insertDefault();
+
+                    // metadata_version
+                    // Temporary tables does not support replication
+                    if (columns_mask[src_index++])
+                        res_columns[res_index++]->insertDefault();
+
+                    // dependencies_database
+                    if (columns_mask[src_index++])
+                        res_columns[res_index++]->insertDefault();
+
+                    // dependencies_table
+                    if (columns_mask[src_index++])
+                        res_columns[res_index++]->insertDefault();
+
+                    // create_table_query
+                    if (columns_mask[src_index++])
+                    {
+                        auto temp_db = DatabaseCatalog::instance().getDatabaseForTemporaryTables();
+                        ASTPtr ast
+                            = temp_db ? temp_db->tryGetCreateTableQuery(table.second->getStorageID().getTableName(), context) : nullptr;
+                        res_columns[res_index++]->insert(ast ? format({context, *ast}) : "");
+                    }
+
+                    // engine_full
+                    if (columns_mask[src_index++])
+                        res_columns[res_index++]->insert(table.second->getName());
+
+                    const auto & settings = context->getSettingsRef();
+                    while (src_index < columns_mask.size())
+                    {
+                        // total_rows
+                        if (src_index == 19 && columns_mask[src_index])
+                        {
+                            if (auto total_rows = table.second->totalRows(settings))
+                                res_columns[res_index++]->insert(*total_rows);
+                            else
+                                res_columns[res_index++]->insertDefault();
+                        }
+                        // total_bytes
+                        else if (src_index == 20 && columns_mask[src_index])
+                        {
+                            if (auto total_bytes = table.second->totalBytes(settings))
+                                res_columns[res_index++]->insert(*total_bytes);
+                            else
+                                res_columns[res_index++]->insertDefault();
+                        }
+                        /// Fill the rest columns with defaults
+                        else if (columns_mask[src_index])
+                            res_columns[res_index++]->insertDefault();
+                        src_index++;
+                    }
+                }
+            }
+
+            UInt64 num_rows = res_columns.at(0)->size();
+            done = true;
+            return Chunk(std::move(res_columns), num_rows);
+        }
+
+        const bool need_to_check_access_for_tables
+            = need_to_check_access_for_databases && !access->isGranted(AccessType::SHOW_TABLES, database_name);
+
+        if (!tables_it || !tables_it->isValid())
+            tables_it = database->getTablesIterator(context);
+
+        const bool need_table = needTable(database, getPort().getHeader());
+
+        for (; rows_count < max_block_size && tables_it->isValid(); tables_it->next())
+        {
+            auto table_name = tables_it->name();
+            if (!tables.contains(table_name))
+                continue;
+
+            if (need_to_check_access_for_tables && !access->isGranted(AccessType::SHOW_TABLES, database_name, table_name))
+                continue;
+
+            StoragePtr table = nullptr;
+            TableLockHolder lock;
+            if (need_table)
+            {
+                table = tables_it->table();
+                if (!table)
+                    // Table might have just been removed or detached for Lazy engine (see DatabaseLazy::tryGetTable())
+                    continue;
+
+                /// The only column that requires us to hold a shared lock is data_paths as rename might alter them (on ordinary tables)
+                /// and it's not protected internally by other mutexes
+                static const size_t DATA_PATHS_INDEX = 5;
+                if (columns_mask[DATA_PATHS_INDEX])
+                {
+                    lock = table->tryLockForShare(context->getCurrentQueryId(), context->getSettingsRef().lock_acquire_timeout);
+                    if (!lock)
+                        // Table was dropped while acquiring the lock, skipping table
+                        continue;
+                }
+            }
+            ++rows_count;
+
+            size_t src_index = 0;
+            size_t res_index = 0;
+
+            if (columns_mask[src_index++])
+                res_columns[res_index++]->insert(database_name);
+
+            if (columns_mask[src_index++])
+                res_columns[res_index++]->insert(table_name);
+
+            if (columns_mask[src_index++])
+                res_columns[res_index++]->insert(tables_it->uuid());
+
+            if (columns_mask[src_index++])
+            {
+                chassert(table != nullptr);
+                res_columns[res_index++]->insert(table->getName());
+            }
+
+            if (columns_mask[src_index++])
+                res_columns[res_index++]->insert(0u); // is_temporary
+
+            if (columns_mask[src_index++])
+            {
+                chassert(lock != nullptr);
+                Array table_paths_array;
+                auto paths = table->getDataPaths();
+                table_paths_array.reserve(paths.size());
+                for (const String & path : paths)
+                    table_paths_array.push_back(path);
+                res_columns[res_index++]->insert(table_paths_array);
+                /// We don't need the lock anymore
+                lock = nullptr;
+            }
+
+            if (columns_mask[src_index++])
+                res_columns[res_index++]->insert(database->getObjectMetadataPath(table_name));
+
+            if (columns_mask[src_index++])
+                res_columns[res_index++]->insert(static_cast<UInt64>(database->getObjectMetadataModificationTime(table_name)));
+
+            StorageMetadataPtr metadata_snapshot;
+            if (table)
+                metadata_snapshot = table->getInMemoryMetadataPtr();
+
+            if (columns_mask[src_index++])
+            {
+                if (metadata_snapshot && table->supportsReplication())
+                    res_columns[res_index++]->insert(metadata_snapshot->metadata_version);
+                else
+                    res_columns[res_index++]->insertDefault();
+            }
+
+            {
+                Array views_table_name_array;
+                Array views_database_name_array;
+                if (columns_mask[src_index] || columns_mask[src_index + 1])
+                {
+                    const auto view_ids = DatabaseCatalog::instance().getDependentViews(StorageID(database_name, table_name));
+
+                    views_table_name_array.reserve(view_ids.size());
+                    views_database_name_array.reserve(view_ids.size());
+                    for (const auto & view_id : view_ids)
+                    {
+                        views_table_name_array.push_back(view_id.table_name);
+                        views_database_name_array.push_back(view_id.database_name);
+                    }
+                }
+
+                if (columns_mask[src_index++])
+                    res_columns[res_index++]->insert(views_database_name_array);
+
+                if (columns_mask[src_index++])
+                    res_columns[res_index++]->insert(views_table_name_array);
+            }
+
+            if (columns_mask[src_index] || columns_mask[src_index + 1] || columns_mask[src_index + 2])
+            {
+                ASTPtr ast = database->tryGetCreateTableQuery(table_name, context);
+                auto * ast_create = ast ? ast->as<ASTCreateQuery>() : nullptr;
+
+                if (ast_create && !context->getSettingsRef().show_table_uuid_in_table_create_query_if_not_nil)
+                {
+                    ast_create->uuid = UUIDHelpers::Nil;
+                    ast_create->to_inner_uuid = UUIDHelpers::Nil;
+                }
+
+                if (columns_mask[src_index++])
+                    res_columns[res_index++]->insert(ast ? format({context, *ast}) : "");
+
+                if (columns_mask[src_index++])
+                {
+                    String engine_full;
+
+                    if (ast_create && ast_create->storage)
+                    {
+                        engine_full = format({context, *ast_create->storage});
+
+                        static const char * const extra_head = " ENGINE = ";
+                        if (startsWith(engine_full, extra_head))
+                            engine_full = engine_full.substr(strlen(extra_head));
+                    }
+
+                    res_columns[res_index++]->insert(engine_full);
+                }
+
+                if (columns_mask[src_index++])
+                {
+                    String as_select;
+                    if (ast_create && ast_create->select)
+                        as_select = format({context, *ast_create->select});
+                    res_columns[res_index++]->insert(as_select);
+                }
+            }
+            else
+                src_index += 3;
+
+            ASTPtr expression_ptr;
+            if (columns_mask[src_index++])
+            {
+                if (metadata_snapshot && (expression_ptr = metadata_snapshot->getPartitionKeyAST()))
+                    res_columns[res_index++]->insert(format({context, *expression_ptr}));
+                else
+                    res_columns[res_index++]->insertDefault();
+            }
+
+            if (columns_mask[src_index++])
+            {
+                if (metadata_snapshot && (expression_ptr = metadata_snapshot->getSortingKey().expression_list_ast))
+                    res_columns[res_index++]->insert(format({context, *expression_ptr}));
+                else
+                    res_columns[res_index++]->insertDefault();
+            }
+
+            if (columns_mask[src_index++])
+            {
+                if (metadata_snapshot && (expression_ptr = metadata_snapshot->getPrimaryKey().expression_list_ast))
+                    res_columns[res_index++]->insert(format({context, *expression_ptr}));
+                else
+                    res_columns[res_index++]->insertDefault();
+            }
+
+            if (columns_mask[src_index++])
+            {
+                if (metadata_snapshot && (expression_ptr = metadata_snapshot->getSamplingKeyAST()))
+                    res_columns[res_index++]->insert(format({context, *expression_ptr}));
+                else
+                    res_columns[res_index++]->insertDefault();
+            }
+
+            if (columns_mask[src_index++])
+            {
+                auto policy = table ? table->getStoragePolicy() : nullptr;
+                if (policy)
+                    res_columns[res_index++]->insert(policy->getName());
+                else
+                    res_columns[res_index++]->insertDefault();
+            }
+
+            auto settings = context->getSettingsRef();
+            settings.select_sequential_consistency = 0;
+            if (columns_mask[src_index++])
+            {
+                auto total_rows = table ? table->totalRows(settings) : std::nullopt;
+                if (total_rows)
+                    res_columns[res_index++]->insert(*total_rows);
+                else
+                    res_columns[res_index++]->insertDefault();
+            }
+
+            if (columns_mask[src_index++])
+            {
+                auto total_bytes = table->totalBytes(settings);
+                if (total_bytes)
+                    res_columns[res_index++]->insert(*total_bytes);
+                else
+                    res_columns[res_index++]->insertDefault();
+            }
+
+            if (columns_mask[src_index++])
+            {
+                auto total_bytes_uncompressed = table->totalBytesUncompressed(settings);
+                if (total_bytes_uncompressed)
+                    res_columns[res_index++]->insert(*total_bytes_uncompressed);
+                else
+                    res_columns[res_index++]->insertDefault();
+            }
+
+            auto table_merge_tree = std::dynamic_pointer_cast<MergeTreeData>(table);
+            if (columns_mask[src_index++])
+            {
+                if (table_merge_tree)
+                    res_columns[res_index++]->insert(table_merge_tree->getAllPartsCount());
+                else
+                    res_columns[res_index++]->insertDefault();
+            }
+
+            if (columns_mask[src_index++])
+            {
+                if (table_merge_tree)
+                    res_columns[res_index++]->insert(table_merge_tree->getActivePartsCount());
+                else
+                    res_columns[res_index++]->insertDefault();
+            }
+
+            if (columns_mask[src_index++])
+            {
+                if (table_merge_tree)
+                {
+                    res_columns[res_index++]->insert(table_merge_tree->getTotalMarksCount());
+                }
+                else
+                    res_columns[res_index++]->insertDefault();
+            }
+
+            if (columns_mask[src_index++])
+            {
+                auto lifetime_rows = table ? table->lifetimeRows() : std::nullopt;
+                if (lifetime_rows)
+                    res_columns[res_index++]->insert(*lifetime_rows);
+                else
+                    res_columns[res_index++]->insertDefault();
+            }
+
+            if (columns_mask[src_index++])
+            {
+                auto lifetime_bytes = table ? table->lifetimeBytes() : std::nullopt;
+                if (lifetime_bytes)
+                    res_columns[res_index++]->insert(*lifetime_bytes);
+                else
+                    res_columns[res_index++]->insertDefault();
+            }
+
+            if (columns_mask[src_index++])
+            {
+                if (metadata_snapshot)
+                    res_columns[res_index++]->insert(metadata_snapshot->comment);
+                else
+                    res_columns[res_index++]->insertDefault();
+            }
+
+            if (columns_mask[src_index++])
+            {
+                if (table)
+                    res_columns[res_index++]->insert(table->storesDataOnDisk());
+                else
+                    res_columns[res_index++]->insertDefault();
+            }
+
+            if (columns_mask[src_index] || columns_mask[src_index + 1] || columns_mask[src_index + 2] || columns_mask[src_index + 3])
+            {
+                auto dependencies = DatabaseCatalog::instance().getLoadingDependencies(StorageID{database_name, table_name});
+                auto dependents = DatabaseCatalog::instance().getLoadingDependents(StorageID{database_name, table_name});
+
+                Array dependencies_databases;
+                Array dependencies_tables;
+                dependencies_databases.reserve(dependencies.size());
+                dependencies_tables.reserve(dependencies.size());
+                for (const auto & dependency : dependencies)
+                {
+                    dependencies_databases.push_back(dependency.database_name);
+                    dependencies_tables.push_back(dependency.table_name);
+                }
+
+                Array dependents_databases;
+                Array dependents_tables;
+                dependents_databases.reserve(dependents.size());
+                dependents_tables.reserve(dependents.size());
+                for (const auto & dependent : dependents)
+                {
+                    dependents_databases.push_back(dependent.database_name);
+                    dependents_tables.push_back(dependent.table_name);
+                }
+
+                if (columns_mask[src_index++])
+                    res_columns[res_index++]->insert(dependencies_databases);
+                if (columns_mask[src_index++])
+                    res_columns[res_index++]->insert(dependencies_tables);
+
+                if (columns_mask[src_index++])
+                    res_columns[res_index++]->insert(dependents_databases);
+                if (columns_mask[src_index++])
+                    res_columns[res_index++]->insert(dependents_tables);
+            }
+        }
+    }
+
+    UInt64 num_rows = res_columns.at(0)->size();
+    return Chunk(std::move(res_columns), num_rows);
+}
+
+Chunk DetachedTablesBlockSource::generate()
+{
+    if (done)
+        return {};
+
+    MutableColumns result_columns = getPort().getHeader().cloneEmptyColumns();
+
+    const auto access = context->getAccess();
+    const bool need_to_check_access_for_databases = !access->isGranted(AccessType::SHOW_TABLES);
+
+    for (size_t database_idx = 0, rows_count = 0; database_idx < databases->size() && rows_count < max_block_size; ++database_idx)
+    {
+        database_name = databases->getDataAt(database_idx).toString();
+        database = DatabaseCatalog::instance().tryGetDatabase(database_name);
+
+        if (!database)
+        {
+            LOG_DEBUG(lg, "Database was deleted just now or the user has no access");
+            continue;
+        }
+
+        const bool need_to_check_access_for_tables
+            = need_to_check_access_for_databases && !access->isGranted(AccessType::SHOW_TABLES, database_name);
+
+        if (!detached_tables_it || !detached_tables_it->isValid())
+            detached_tables_it = database->getDetachedTablesIterator(context, {}, false);
+
+        for (; rows_count < max_block_size && detached_tables_it->isValid(); detached_tables_it->next())
+        {
+            const auto detached_table_name = detached_tables_it->table();
+            LOG_DEBUG(lg, "detached_table_name={}", detached_table_name);
+
+            if (!detached_tables.contains(detached_table_name))
+                continue;
+
+            if (need_to_check_access_for_tables && !access->isGranted(AccessType::SHOW_TABLES, database_name, detached_table_name))
+                continue;
+
+            fillResultColumnsByDetachedTableIterator(result_columns);
+            ++rows_count;
+        }
+    }
+
+    const UInt64 num_rows = result_columns.at(0)->size();
+    done = true;
+    return Chunk(std::move(result_columns), num_rows);
+}
+
+void DetachedTablesBlockSource::fillResultColumnsByDetachedTableIterator(MutableColumns & result_columns) const
+{
+    size_t src_index = 0;
+    size_t res_index = 0;
+
+    if (columns_mask[src_index++])
+        result_columns[res_index++]->insert(detached_tables_it->database());
+
+    if (columns_mask[src_index++])
+        result_columns[res_index++]->insert(detached_tables_it->table());
+
+
+    if (columns_mask[src_index++])
+        result_columns[res_index++]->insert(detached_tables_it->uuid());
+
+    if (columns_mask[src_index++])
+        result_columns[res_index++]->insert(detached_tables_it->metadataPath());
+
+    if (columns_mask[src_index++])
+        result_columns[res_index++]->insert(detached_tables_it->isPermanently());
+}
+}
diff --git a/src/Storages/System/TablesBlockSource.h b/src/Storages/System/TablesBlockSource.h
new file mode 100644
index 00000000000..9698fdd494c
--- /dev/null
+++ b/src/Storages/System/TablesBlockSource.h
@@ -0,0 +1,96 @@
+#pragma once
+
+#include <Databases/IDatabase.h>
+#include <Interpreters/Context.h>
+#include <Processors/ISource.h>
+#include <Common/logger_useful.h>
+
+namespace DB
+{
+
+class TablesBlockSource : public ISource
+{
+public:
+    TablesBlockSource(
+        std::vector<UInt8> columns_mask_,
+        Block header,
+        UInt64 max_block_size_,
+        ColumnPtr databases_,
+        ColumnPtr tables_,
+        ContextPtr context_)
+        : ISource(std::move(header))
+        , columns_mask(std::move(columns_mask_))
+        , max_block_size(max_block_size_)
+        , databases(std::move(databases_))
+        , context(Context::createCopy(context_))
+    {
+        size_t size = tables_->size();
+        tables.reserve(size);
+        for (size_t idx = 0; idx < size; ++idx)
+            tables.insert(tables_->getDataAt(idx).toString());
+    }
+
+    String getName() const override { return "Tables"; }
+
+protected:
+    Chunk generate() override;
+
+private:
+    std::vector<UInt8> columns_mask;
+    UInt64 max_block_size;
+    ColumnPtr databases;
+    NameSet tables;
+    size_t database_idx = 0;
+    DatabaseTablesIteratorPtr tables_it;
+    ContextPtr context;
+    bool done = false;
+    DatabasePtr database;
+    std::string database_name;
+};
+
+class DetachedTablesBlockSource : public ISource
+{
+public:
+    DetachedTablesBlockSource(
+        std::vector<UInt8> columns_mask_,
+        Block header,
+        UInt64 max_block_size_,
+        ColumnPtr databases_,
+        ColumnPtr detached_tables_,
+        ContextPtr context_)
+        : ISource(std::move(header))
+        , columns_mask(std::move(columns_mask_))
+        , max_block_size(max_block_size_)
+        , databases(std::move(databases_))
+        , context(Context::createCopy(context_))
+    {
+        size_t size = detached_tables_->size();
+        detached_tables.reserve(size);
+        for (size_t idx = 0; idx < size; ++idx)
+        {
+            detached_tables.insert(detached_tables_->getDataAt(idx).toString());
+        }
+    }
+
+    String getName() const override { return "DetachedTables"; }
+
+protected:
+    Chunk generate() override;
+
+private:
+    const std::vector<UInt8> columns_mask;
+    const UInt64 max_block_size;
+    const ColumnPtr databases;
+    NameSet detached_tables;
+    DatabaseDetachedTablesSnapshotIteratorPtr detached_tables_it;
+    ContextPtr context;
+    bool done = false;
+    DatabasePtr database;
+    std::string database_name;
+
+    // temp log for debug
+    LoggerPtr lg = getLogger("DetachedTablesBlockSource");
+
+    void fillResultColumnsByDetachedTableIterator(MutableColumns & result_columns) const;
+};
+}
diff --git a/src/Storages/System/attachSystemTables.cpp b/src/Storages/System/attachSystemTables.cpp
index 6ff86b26ca9..97eda1db3fa 100644
--- a/src/Storages/System/attachSystemTables.cpp
+++ b/src/Storages/System/attachSystemTables.cpp
@@ -18,6 +18,7 @@
 #include <Storages/System/StorageSystemDataSkippingIndices.h>
 #include <Storages/System/StorageSystemDataTypeFamilies.h>
 #include <Storages/System/StorageSystemDetachedParts.h>
+#include <Storages/System/StorageSystemDetachedTables.h>
 #include <Storages/System/StorageSystemDictionaries.h>
 #include <Storages/System/StorageSystemEvents.h>
 #include <Storages/System/StorageSystemFormats.h>
@@ -129,6 +130,7 @@ void attachSystemTablesServer(ContextPtr context, IDatabase & system_database, b
     attachNoDescription<StorageSystemZeros>(context, system_database, "zeros_mt", "Multithreaded version of system.zeros.", true);
     attach<StorageSystemDatabases>(context, system_database, "databases", "Lists all databases of the current server.");
     attachNoDescription<StorageSystemTables>(context, system_database, "tables", "Lists all tables of the current server.");
+    attachNoDescription<StorageSystemDetachedTables>(context, system_database, "detached_tables", "Lists all detached tables of the current server.");
     attachNoDescription<StorageSystemColumns>(context, system_database, "columns", "Lists all columns from all tables of the current server.");
     attach<StorageSystemFunctions>(context, system_database, "functions", "Contains a list of all available ordinary and aggregate functions with their descriptions.");
     attach<StorageSystemEvents>(context, system_database, "events", "Contains profiling events and their current value.");
diff --git a/tests/integration/test_system_detached_tables/__init__.py b/tests/integration/test_system_detached_tables/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/integration/test_system_detached_tables/test.py b/tests/integration/test_system_detached_tables/test.py
new file mode 100644
index 00000000000..2e0165deba2
--- /dev/null
+++ b/tests/integration/test_system_detached_tables/test.py
@@ -0,0 +1,38 @@
+import pytest
+from helpers.cluster import ClickHouseCluster
+
+cluster = ClickHouseCluster(__file__)
+node = cluster.add_instance("node_default", stay_alive=True)
+
+
+@pytest.fixture(scope="module", autouse=True)
+def start_cluster():
+    try:
+        cluster.start()
+        yield cluster
+    finally:
+        cluster.shutdown()
+
+
+def test_system_detached_tables():
+    node.query("CREATE TABLE test_table (n Int64) ENGINE=MergeTree ORDER BY n;")
+    node.query("CREATE TABLE test_table_perm (n Int64) ENGINE=MergeTree ORDER BY n;")
+
+    result = node.query("SELECT * FROM system.detached_tables")
+    assert result == ""
+
+    node.query("DETACH TABLE test_table")
+    node.query("DETACH TABLE test_table_perm PERMANENTLY")
+
+    result = node.query("SELECT name FROM system.detached_tables")
+    assert result == "test_table\ntest_table_perm\n"
+
+    node.restart_clickhouse()
+
+    result = node.query("SELECT name FROM system.detached_tables")
+    assert result == "test_table_perm\n"
+
+    node.restart_clickhouse()
+
+    result = node.query("SELECT name FROM system.detached_tables")
+    assert result == "test_table_perm\n"
diff --git a/tests/queries/0_stateless/03172_system_detached_tables.reference b/tests/queries/0_stateless/03172_system_detached_tables.reference
new file mode 100644
index 00000000000..1cf9e0275c9
--- /dev/null
+++ b/tests/queries/0_stateless/03172_system_detached_tables.reference
@@ -0,0 +1,5 @@
+test_system_detached_tables	test_table	0
+test_system_detached_tables	test_table_perm	1
+test_system_detached_tables	test_table	0
+test_system_detached_tables	test_table_perm	1
+test_system_detached_tables	test_table	0
diff --git a/tests/queries/0_stateless/03172_system_detached_tables.sql b/tests/queries/0_stateless/03172_system_detached_tables.sql
new file mode 100644
index 00000000000..13465d02b80
--- /dev/null
+++ b/tests/queries/0_stateless/03172_system_detached_tables.sql
@@ -0,0 +1,28 @@
+-- Tags: no-parallel
+
+DROP DATABASE IF EXISTS test_system_detached_tables;
+CREATE DATABASE IF NOT EXISTS test_system_detached_tables;
+
+CREATE TABLE test_system_detached_tables.test_table (n Int64) ENGINE=MergeTree ORDER BY n;
+SELECT * FROM system.detached_tables;
+
+DETACH TABLE test_system_detached_tables.test_table;
+SELECT database, name, is_permanently FROM system.detached_tables;
+
+ATTACH TABLE test_system_detached_tables.test_table;
+
+CREATE TABLE test_system_detached_tables.test_table_perm (n Int64) ENGINE=MergeTree ORDER BY n;
+SELECT * FROM system.detached_tables;
+
+DETACH TABLE test_system_detached_tables.test_table_perm PERMANENTLY;
+SELECT database, name, is_permanently FROM system.detached_tables;
+
+DETACH TABLE test_system_detached_tables.test_table SYNC;
+SELECT database, name, is_permanently FROM system.detached_tables;
+
+SELECT database, name, is_permanently FROM system.detached_tables WHERE name='test_table';
+
+DROP DATABASE test_system_detached_tables;
+
+
+

From 21936f32a655b1008b6233ee274549b0ce2b33c7 Mon Sep 17 00:00:00 2001
From: Konstantin Morozov <just.morozov.k@gmail.com>
Date: Fri, 21 Jun 2024 12:32:54 +0000
Subject: [PATCH 0082/1488] fix style

---
 src/Databases/DatabaseOrdinary.cpp | 2 +-
 src/Databases/DatabasesCommon.cpp  | 4 ++--
 src/Databases/IDatabase.h          | 4 ++--
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp
index 046cbfa40be..ac40bf4e90e 100644
--- a/src/Databases/DatabaseOrdinary.cpp
+++ b/src/Databases/DatabaseOrdinary.cpp
@@ -243,7 +243,7 @@ void DatabaseOrdinary::loadTablesMetadata(ContextPtr local_context, ParsedTables
                     const auto storage_id = table->getStorageID();
 
                     SnapshotDetachedTable snapshot_detached_table;
-                    snapshot_detached_table.detabase = storage_id.getDatabaseName();
+                    snapshot_detached_table.database = storage_id.getDatabaseName();
                     snapshot_detached_table.table = detached_table_name;
                     snapshot_detached_table.uuid = storage_id.uuid;
                     snapshot_detached_table.is_permanently = true;
diff --git a/src/Databases/DatabasesCommon.cpp b/src/Databases/DatabasesCommon.cpp
index 312c0a4477f..c4cad663878 100644
--- a/src/Databases/DatabasesCommon.cpp
+++ b/src/Databases/DatabasesCommon.cpp
@@ -250,7 +250,7 @@ DatabaseDetachedTablesSnapshotIteratorPtr DatabaseWithOwnTablesBase::getDetached
         if (filter_by_table_name(table_name))
         {
             SnapshotDetachedTable snapshot_detached_table;
-            snapshot_detached_table.detabase = storage->getStorageID().getDatabaseName();
+            snapshot_detached_table.database = storage->getStorageID().getDatabaseName();
             snapshot_detached_table.table = table_name;
             if (storage->getStorageID().hasUUID())
             {
@@ -290,7 +290,7 @@ StoragePtr DatabaseWithOwnTablesBase::detachTableUnlocked(const String & table_n
     res = it->second;
 
     SnapshotDetachedTable snapshot_detached_table;
-    snapshot_detached_table.detabase = it->second->getStorageID().getDatabaseName();
+    snapshot_detached_table.database = it->second->getStorageID().getDatabaseName();
     snapshot_detached_table.table = it->first;
     if (it->second->getStorageID().hasUUID())
     {
diff --git a/src/Databases/IDatabase.h b/src/Databases/IDatabase.h
index 7c6b3b3004c..2ce4650798c 100644
--- a/src/Databases/IDatabase.h
+++ b/src/Databases/IDatabase.h
@@ -114,7 +114,7 @@ using DatabaseTablesIteratorPtr = std::unique_ptr<IDatabaseTablesIterator>;
 
 struct SnapshotDetachedTable final
 {
-    String detabase;
+    String database;
     String table;
     UUID uuid = UUIDHelpers::Nil;
     String metadata_path;
@@ -148,7 +148,7 @@ public:
 
     bool isValid() const { return it != snapshot.end(); }
 
-    String database() const { return it->second.detabase; }
+    String database() const { return it->second.database; }
 
     String table() const { return it->second.table; }
 

From 7aa7b7977221d5bc9b65d0e09d272ffca96acddc Mon Sep 17 00:00:00 2001
From: Konstantin Morozov <just.morozov.k@gmail.com>
Date: Fri, 21 Jun 2024 16:33:09 +0000
Subject: [PATCH 0083/1488] refactor

---
 .../system-tables/detached_tables.md          |   9 +-
 src/Databases/DatabaseAtomic.h                |   1 -
 src/Databases/DatabaseOrdinary.cpp            |   6 +-
 src/Storages/System/ReadFromSystemTables.cpp  | 138 ----
 src/Storages/System/ReadFromSystemTables.h    |  47 --
 .../System/StorageSystemDetachedTables.cpp    | 252 +++++-
 src/Storages/System/StorageSystemTables.cpp   | 761 ++++++++++++++++--
 src/Storages/System/TablesBlockSource.cpp     | 569 -------------
 src/Storages/System/TablesBlockSource.h       |  96 ---
 .../test_system_detached_tables/test.py       |   6 +-
 .../03172_system_detached_tables.sql          |   8 +-
 11 files changed, 959 insertions(+), 934 deletions(-)
 delete mode 100644 src/Storages/System/ReadFromSystemTables.cpp
 delete mode 100644 src/Storages/System/ReadFromSystemTables.h
 delete mode 100644 src/Storages/System/TablesBlockSource.cpp
 delete mode 100644 src/Storages/System/TablesBlockSource.h

diff --git a/docs/en/operations/system-tables/detached_tables.md b/docs/en/operations/system-tables/detached_tables.md
index d86e58d2899..f669c8fd251 100644
--- a/docs/en/operations/system-tables/detached_tables.md
+++ b/docs/en/operations/system-tables/detached_tables.md
@@ -9,14 +9,14 @@ Columns:
 
 - `database` ([String](../../sql-reference/data-types/string.md)) — The name of the database the table is in.
 
-- `name` ([String](../../sql-reference/data-types/string.md)) — Table name.
+- `table` ([String](../../sql-reference/data-types/string.md)) — Table name.
 
 - `uuid` ([UUID](../../sql-reference/data-types/uuid.md)) — Table uuid (Atomic database).
 
-- `is_permanently` ([UInt8](../../sql-reference/data-types/int-uint.md)) - Flag indicates that the table was detached PERMANENTLY.
-
 - `metadata_path` ([String](../../sql-reference/data-types/string.md)) - Path to the table metadata in the file system.
 
+- `is_permanently` ([UInt8](../../sql-reference/data-types/int-uint.md)) - Flag indicates that the table was detached PERMANENTLY.
+
 
 **Example**
 
@@ -28,9 +28,8 @@ SELECT * FROM system.detached_tables FORMAT Vertical;
 Row 1:
 ──────
 database:                   base
-name:                       t1
+table:                      t1
 uuid:                       81b1c20a-b7c6-4116-a2ce-7583fb6b6736
-data_paths:                 ['/var/lib/clickhouse/store/81b/81b1c20a-b7c6-4116-a2ce-7583fb6b6736/']
 metadata_path:              /var/lib/clickhouse/store/461/461cf698-fd0b-406d-8c01-5d8fd5748a91/t1.sql
 is_permanently:             1
 ```
diff --git a/src/Databases/DatabaseAtomic.h b/src/Databases/DatabaseAtomic.h
index 47afc4dc5ef..25c5cdfbde1 100644
--- a/src/Databases/DatabaseAtomic.h
+++ b/src/Databases/DatabaseAtomic.h
@@ -85,7 +85,6 @@ protected:
 
     //TODO store path in DatabaseWithOwnTables::tables
     using NameToPathMap = std::unordered_map<String, String>;
-
     NameToPathMap table_name_to_path TSA_GUARDED_BY(mutex);
 
     DetachedTables detached_tables TSA_GUARDED_BY(mutex);
diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp
index ac40bf4e90e..7086f6e628e 100644
--- a/src/Databases/DatabaseOrdinary.cpp
+++ b/src/Databases/DatabaseOrdinary.cpp
@@ -187,7 +187,7 @@ void DatabaseOrdinary::loadTablesMetadata(ContextPtr local_context, ParsedTables
     size_t prev_tables_count = metadata.parsed_tables.size();
     size_t prev_total_dictionaries = metadata.total_dictionaries;
 
-    auto process_metadata = [&metadata, is_startup, local_context, this](const String & file_name) mutable
+    auto process_metadata = [&metadata, is_startup, local_context, this](const String & file_name)
     {
         fs::path path(getMetadataPath());
         fs::path file_path(file_name);
@@ -227,7 +227,6 @@ void DatabaseOrdinary::loadTablesMetadata(ContextPtr local_context, ParsedTables
                     permanently_detached_tables.push_back(table_name);
                     LOG_DEBUG(log, "Skipping permanently detached table {}.", backQuote(table_name));
 
-                    // @TODO refactoring
                     auto parsed_table_metadata = ParsedTableMetadata{full_path.string(), ast};
                     const auto & query = parsed_table_metadata.ast->as<const ASTCreateQuery &>();
 
@@ -249,10 +248,9 @@ void DatabaseOrdinary::loadTablesMetadata(ContextPtr local_context, ParsedTables
                     snapshot_detached_table.is_permanently = true;
                     snapshot_detached_table.metadata_path = getObjectMetadataPath(snapshot_detached_table.table);
 
-
                     snapshot_detached_tables.emplace(detached_table_name, std::move(snapshot_detached_table));
 
-                    LOG_TRACE(log, "Add detached table {} to system.detached_tables", detached_table_name);
+                    LOG_TRACE(log, "Add permanently detached table {} to system.detached_tables", detached_table_name);
                     return;
                 }
 
diff --git a/src/Storages/System/ReadFromSystemTables.cpp b/src/Storages/System/ReadFromSystemTables.cpp
deleted file mode 100644
index 6e9c6a869d1..00000000000
--- a/src/Storages/System/ReadFromSystemTables.cpp
+++ /dev/null
@@ -1,138 +0,0 @@
-#include "ReadFromSystemTables.h"
-
-#include <Core/Block.h>
-#include <Core/ColumnWithTypeAndName.h>
-#include <DataTypes/DataTypeString.h>
-#include <Interpreters/DatabaseCatalog.h>
-#include <QueryPipeline/QueryPipelineBuilder.h>
-#include <Storages/IStorage.h>
-#include <Storages/System/TablesBlockSource.h>
-#include <Storages/VirtualColumnUtils.h>
-
-#include <boost/range/adaptor/map.hpp>
-
-namespace DB
-{
-
-namespace
-{
-
-ColumnPtr getFilteredDatabases(const ActionsDAG::Node * predicate, ContextPtr context)
-{
-    MutableColumnPtr column = ColumnString::create();
-
-    const auto databases = DatabaseCatalog::instance().getDatabases();
-    for (const auto & database_name : databases | boost::adaptors::map_keys)
-    {
-        if (database_name == DatabaseCatalog::TEMPORARY_DATABASE)
-            continue; /// We don't want to show the internal database for temporary tables in system.tables
-
-        column->insert(database_name);
-    }
-
-    Block block{ColumnWithTypeAndName(std::move(column), std::make_shared<DataTypeString>(), "database")};
-    VirtualColumnUtils::filterBlockWithPredicate(predicate, block, context);
-    return block.getByPosition(0).column;
-}
-
-ColumnPtr getFilteredTables(
-    const ActionsDAG::Node * predicate, const ColumnPtr & filtered_databases_column, ContextPtr context, const bool need_detached_tables)
-{
-    Block sample{
-        ColumnWithTypeAndName(nullptr, std::make_shared<DataTypeString>(), "name"),
-        ColumnWithTypeAndName(nullptr, std::make_shared<DataTypeString>(), "engine")};
-
-    MutableColumnPtr database_column = ColumnString::create();
-    MutableColumnPtr engine_column;
-
-    auto dag = VirtualColumnUtils::splitFilterDagForAllowedInputs(predicate, &sample);
-    if (dag)
-    {
-        bool filter_by_engine = false;
-        for (const auto * input : dag->getInputs())
-            if (input->result_name == "engine")
-                filter_by_engine = true;
-
-        if (filter_by_engine)
-            engine_column = ColumnString::create();
-    }
-
-    for (size_t database_idx = 0; database_idx < filtered_databases_column->size(); ++database_idx)
-    {
-        const auto & database_name = filtered_databases_column->getDataAt(database_idx).toString();
-        DatabasePtr database = DatabaseCatalog::instance().tryGetDatabase(database_name);
-        if (!database)
-            continue;
-
-        if (need_detached_tables)
-        {
-            auto table_it = database->getDetachedTablesIterator(context, {}, false);
-            for (; table_it->isValid(); table_it->next())
-            {
-                database_column->insert(table_it->table());
-            }
-        }
-        else
-        {
-            auto table_it = database->getTablesIterator(context);
-            for (; table_it->isValid(); table_it->next())
-            {
-                database_column->insert(table_it->name());
-                if (engine_column)
-                    engine_column->insert(table_it->table()->getName());
-            }
-        }
-    }
-
-    Block block{ColumnWithTypeAndName(std::move(database_column), std::make_shared<DataTypeString>(), "name")};
-    if (engine_column)
-        block.insert(ColumnWithTypeAndName(std::move(engine_column), std::make_shared<DataTypeString>(), "engine"));
-
-    if (dag)
-        VirtualColumnUtils::filterBlockWithDAG(dag, block, context);
-
-    return block.getByPosition(0).column;
-}
-
-}
-
-ReadFromSystemTables::ReadFromSystemTables(
-    const Names & column_names_,
-    const SelectQueryInfo & query_info_,
-    const StorageSnapshotPtr & storage_snapshot_,
-    const ContextPtr & context_,
-    Block sample_block,
-    std::vector<UInt8> columns_mask_,
-    size_t max_block_size_,
-    const bool need_detached_tables_)
-    : SourceStepWithFilter(DataStream{.header = std::move(sample_block)}, column_names_, query_info_, storage_snapshot_, context_)
-    , columns_mask(std::move(columns_mask_))
-    , max_block_size(max_block_size_)
-    , need_detached_tables(need_detached_tables_)
-{
-}
-
-void ReadFromSystemTables::applyFilters(ActionDAGNodes added_filter_nodes)
-{
-    SourceStepWithFilter::applyFilters(std::move(added_filter_nodes));
-
-    const ActionsDAG::Node * predicate = nullptr;
-    if (filter_actions_dag)
-        predicate = filter_actions_dag->getOutputs().at(0);
-
-    filtered_databases_column = getFilteredDatabases(predicate, context);
-    filtered_tables_column = getFilteredTables(predicate, filtered_databases_column, context, need_detached_tables);
-}
-
-void ReadFromSystemTables::initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &)
-{
-    if (need_detached_tables)
-    {
-        pipeline.init(createPipe<DetachedTablesBlockSource>());
-    }
-    else
-    {
-        pipeline.init(createPipe<TablesBlockSource>());
-    }
-}
-}
diff --git a/src/Storages/System/ReadFromSystemTables.h b/src/Storages/System/ReadFromSystemTables.h
deleted file mode 100644
index f8280ffcc8d..00000000000
--- a/src/Storages/System/ReadFromSystemTables.h
+++ /dev/null
@@ -1,47 +0,0 @@
-#pragma once
-
-#include <Processors/QueryPlan/SourceStepWithFilter.h>
-#include <QueryPipeline/Pipe.h>
-
-namespace DB
-{
-
-class ReadFromSystemTables : public SourceStepWithFilter
-{
-public:
-    std::string getName() const override { return "ReadFromSystemTables"; }
-    void initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) override;
-
-    ReadFromSystemTables(
-        const Names & column_names_,
-        const SelectQueryInfo & query_info_,
-        const StorageSnapshotPtr & storage_snapshot_,
-        const ContextPtr & context_,
-        Block sample_block,
-        std::vector<UInt8> columns_mask_,
-        size_t max_block_size_,
-        bool need_detached_tables);
-
-    void applyFilters(ActionDAGNodes added_filter_nodes) override;
-
-private:
-    std::vector<UInt8> columns_mask;
-    size_t max_block_size;
-    const bool need_detached_tables;
-
-    ColumnPtr filtered_databases_column;
-    ColumnPtr filtered_tables_column;
-
-    template <class T>
-    Pipe createPipe()
-    {
-        return Pipe(std::make_shared<T>(
-            std::move(columns_mask),
-            getOutputStream().header,
-            max_block_size,
-            std::move(filtered_databases_column),
-            std::move(filtered_tables_column),
-            context));
-    }
-};
-}
diff --git a/src/Storages/System/StorageSystemDetachedTables.cpp b/src/Storages/System/StorageSystemDetachedTables.cpp
index 1596ac6a046..9805fa2a8cf 100644
--- a/src/Storages/System/StorageSystemDetachedTables.cpp
+++ b/src/Storages/System/StorageSystemDetachedTables.cpp
@@ -1,28 +1,232 @@
 #include "StorageSystemDetachedTables.h"
 
+#include <Access/ContextAccess.h>
 #include <Core/NamesAndTypes.h>
 #include <DataTypes/DataTypeString.h>
 #include <DataTypes/DataTypeUUID.h>
 #include <DataTypes/DataTypesNumber.h>
+#include <Interpreters/Context.h>
+#include <Interpreters/DatabaseCatalog.h>
 #include <Processors/QueryPlan/QueryPlan.h>
+#include <Processors/QueryPlan/SourceStepWithFilter.h>
+#include <QueryPipeline/QueryPipelineBuilder.h>
 #include <Storages/ColumnsDescription.h>
 #include <Storages/ProjectionsDescription.h>
 #include <Storages/StorageInMemoryMetadata.h>
-#include <Storages/System/ReadFromSystemTables.h>
-#include <Storages/System/TablesBlockSource.h>
 #include <Storages/System/getQueriedColumnsMaskAndHeader.h>
+#include <Storages/VirtualColumnUtils.h>
+
+#include <boost/range/adaptor/map.hpp>
 
 
 namespace DB
 {
 
+namespace
+{
+
+ColumnPtr getFilteredDatabases(const ActionsDAG::Node * predicate, ContextPtr context)
+{
+    MutableColumnPtr column = ColumnString::create();
+
+    const auto databases = DatabaseCatalog::instance().getDatabases();
+    for (const auto & database_name : databases | boost::adaptors::map_keys)
+    {
+        if (database_name == DatabaseCatalog::TEMPORARY_DATABASE)
+            continue; /// We don't want to show the internal database for temporary tables in system.tables
+
+        column->insert(database_name);
+    }
+
+    Block block{ColumnWithTypeAndName(std::move(column), std::make_shared<DataTypeString>(), "database")};
+    VirtualColumnUtils::filterBlockWithPredicate(predicate, block, context);
+    return block.getByPosition(0).column;
+}
+
+ColumnPtr getFilteredTables(const ActionsDAG::Node * predicate, const ColumnPtr & filtered_databases_column, ContextPtr context)
+{
+    Block sample{
+        ColumnWithTypeAndName(nullptr, std::make_shared<DataTypeString>(), "name"),
+        ColumnWithTypeAndName(nullptr, std::make_shared<DataTypeString>(), "engine")};
+
+    MutableColumnPtr database_column = ColumnString::create();
+    MutableColumnPtr engine_column;
+
+    auto dag = VirtualColumnUtils::splitFilterDagForAllowedInputs(predicate, &sample);
+    if (dag)
+    {
+        bool filter_by_engine = false;
+        for (const auto * input : dag->getInputs())
+            if (input->result_name == "engine")
+                filter_by_engine = true;
+
+        if (filter_by_engine)
+            engine_column = ColumnString::create();
+    }
+
+    for (size_t database_idx = 0; database_idx < filtered_databases_column->size(); ++database_idx)
+    {
+        const auto & database_name = filtered_databases_column->getDataAt(database_idx).toString();
+        DatabasePtr database = DatabaseCatalog::instance().tryGetDatabase(database_name);
+        if (!database)
+            continue;
+
+        auto table_it = database->getDetachedTablesIterator(context, {}, false);
+        for (; table_it->isValid(); table_it->next())
+        {
+            database_column->insert(table_it->table());
+        }
+    }
+
+    Block block{ColumnWithTypeAndName(std::move(database_column), std::make_shared<DataTypeString>(), "name")};
+    if (engine_column)
+        block.insert(ColumnWithTypeAndName(std::move(engine_column), std::make_shared<DataTypeString>(), "engine"));
+
+    if (dag)
+        VirtualColumnUtils::filterBlockWithDAG(dag, block, context);
+
+    return block.getByPosition(0).column;
+}
+
+class DetachedTablesBlockSource : public ISource
+{
+public:
+    DetachedTablesBlockSource(
+        std::vector<UInt8> columns_mask_,
+        Block header,
+        UInt64 max_block_size_,
+        ColumnPtr databases_,
+        ColumnPtr detached_tables_,
+        ContextPtr context_)
+        : ISource(std::move(header))
+        , columns_mask(std::move(columns_mask_))
+        , max_block_size(max_block_size_)
+        , databases(std::move(databases_))
+        , context(Context::createCopy(context_))
+    {
+        size_t size = detached_tables_->size();
+        detached_tables.reserve(size);
+        for (size_t idx = 0; idx < size; ++idx)
+        {
+            detached_tables.insert(detached_tables_->getDataAt(idx).toString());
+        }
+    }
+
+    String getName() const override { return "DetachedTables"; }
+
+protected:
+    Chunk generate() override
+    {
+        if (done)
+            return {};
+
+        MutableColumns result_columns = getPort().getHeader().cloneEmptyColumns();
+
+        const auto access = context->getAccess();
+        const bool need_to_check_access_for_databases = !access->isGranted(AccessType::SHOW_TABLES);
+
+        for (size_t database_idx = 0, rows_count = 0; database_idx < databases->size() && rows_count < max_block_size; ++database_idx)
+        {
+            database_name = databases->getDataAt(database_idx).toString();
+            database = DatabaseCatalog::instance().tryGetDatabase(database_name);
+
+            if (!database)
+                continue;
+
+            const bool need_to_check_access_for_tables
+                = need_to_check_access_for_databases && !access->isGranted(AccessType::SHOW_TABLES, database_name);
+
+            if (!detached_tables_it || !detached_tables_it->isValid())
+                detached_tables_it = database->getDetachedTablesIterator(context, {}, false);
+
+            for (; rows_count < max_block_size && detached_tables_it->isValid(); detached_tables_it->next())
+            {
+                const auto detached_table_name = detached_tables_it->table();
+
+                if (!detached_tables.contains(detached_table_name))
+                    continue;
+
+                if (need_to_check_access_for_tables && !access->isGranted(AccessType::SHOW_TABLES, database_name, detached_table_name))
+                    continue;
+
+                fillResultColumnsByDetachedTableIterator(result_columns);
+                ++rows_count;
+            }
+        }
+
+        const UInt64 num_rows = result_columns.at(0)->size();
+        done = true;
+        return Chunk(std::move(result_columns), num_rows);
+    }
+
+private:
+    const std::vector<UInt8> columns_mask;
+    const UInt64 max_block_size;
+    const ColumnPtr databases;
+    NameSet detached_tables;
+    DatabaseDetachedTablesSnapshotIteratorPtr detached_tables_it;
+    ContextPtr context;
+    bool done = false;
+    DatabasePtr database;
+    std::string database_name;
+
+    void fillResultColumnsByDetachedTableIterator(MutableColumns & result_columns) const
+    {
+        size_t src_index = 0;
+        size_t res_index = 0;
+
+        if (columns_mask[src_index++])
+            result_columns[res_index++]->insert(detached_tables_it->database());
+
+        if (columns_mask[src_index++])
+            result_columns[res_index++]->insert(detached_tables_it->table());
+
+
+        if (columns_mask[src_index++])
+            result_columns[res_index++]->insert(detached_tables_it->uuid());
+
+        if (columns_mask[src_index++])
+            result_columns[res_index++]->insert(detached_tables_it->metadataPath());
+
+        if (columns_mask[src_index++])
+            result_columns[res_index++]->insert(detached_tables_it->isPermanently());
+    }
+};
+
+}
+
+class ReadFromSystemDetachedTables : public SourceStepWithFilter
+{
+public:
+    std::string getName() const override { return "ReadFromSystemDetachedTables"; }
+    void initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) override;
+
+    ReadFromSystemDetachedTables(
+        const Names & column_names_,
+        const SelectQueryInfo & query_info_,
+        const StorageSnapshotPtr & storage_snapshot_,
+        const ContextPtr & context_,
+        Block sample_block,
+        std::vector<UInt8> columns_mask_,
+        size_t max_block_size_);
+
+    void applyFilters(ActionDAGNodes added_filter_nodes) override;
+
+private:
+    std::vector<UInt8> columns_mask;
+    size_t max_block_size;
+
+    ColumnPtr filtered_databases_column;
+    ColumnPtr filtered_tables_column;
+};
+
 StorageSystemDetachedTables::StorageSystemDetachedTables(const StorageID & table_id_) : IStorage(table_id_)
 {
     StorageInMemoryMetadata storage_metadata;
 
     auto description = ColumnsDescription{
         ColumnDescription{"database", std::make_shared<DataTypeString>(), "The name of the database the table is in."},
-        ColumnDescription{"name", std::make_shared<DataTypeString>(), "Table name."},
+        ColumnDescription{"table", std::make_shared<DataTypeString>(), "Table name."},
         ColumnDescription{"uuid", std::make_shared<DataTypeUUID>(), "Table uuid (Atomic database)."},
         ColumnDescription{"metadata_path", std::make_shared<DataTypeString>(), "Path to the table metadata in the file system."},
         ColumnDescription{"is_permanently", std::make_shared<DataTypeUInt8>(), "Table was detached permanently."},
@@ -48,9 +252,47 @@ void StorageSystemDetachedTables::read(
 
     auto [columns_mask, res_block] = getQueriedColumnsMaskAndHeader(sample_block, column_names);
 
-    auto reading = std::make_unique<ReadFromSystemTables>(
-        column_names, query_info, storage_snapshot, context, std::move(res_block), std::move(columns_mask), max_block_size, true);
+    auto reading = std::make_unique<ReadFromSystemDetachedTables>(
+        column_names, query_info, storage_snapshot, context, std::move(res_block), std::move(columns_mask), max_block_size);
 
     query_plan.addStep(std::move(reading));
 }
+
+ReadFromSystemDetachedTables::ReadFromSystemDetachedTables(
+    const Names & column_names_,
+    const SelectQueryInfo & query_info_,
+    const StorageSnapshotPtr & storage_snapshot_,
+    const ContextPtr & context_,
+    Block sample_block,
+    std::vector<UInt8> columns_mask_,
+    size_t max_block_size_)
+    : SourceStepWithFilter(DataStream{.header = std::move(sample_block)}, column_names_, query_info_, storage_snapshot_, context_)
+    , columns_mask(std::move(columns_mask_))
+    , max_block_size(max_block_size_)
+{
+}
+
+void ReadFromSystemDetachedTables::applyFilters(ActionDAGNodes added_filter_nodes)
+{
+    SourceStepWithFilter::applyFilters(std::move(added_filter_nodes));
+
+    const ActionsDAG::Node * predicate = nullptr;
+    if (filter_actions_dag)
+        predicate = filter_actions_dag->getOutputs().at(0);
+
+    filtered_databases_column = getFilteredDatabases(predicate, context);
+    filtered_tables_column = getFilteredTables(predicate, filtered_databases_column, context);
+}
+
+void ReadFromSystemDetachedTables::initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &)
+{
+    auto pipe = Pipe(std::make_shared<DetachedTablesBlockSource>(
+        std::move(columns_mask),
+        getOutputStream().header,
+        max_block_size,
+        std::move(filtered_databases_column),
+        std::move(filtered_tables_column),
+        context));
+    pipeline.init(std::move(pipe));
+}
 }
diff --git a/src/Storages/System/StorageSystemTables.cpp b/src/Storages/System/StorageSystemTables.cpp
index 5a8d6315d57..783b899c978 100644
--- a/src/Storages/System/StorageSystemTables.cpp
+++ b/src/Storages/System/StorageSystemTables.cpp
@@ -1,31 +1,29 @@
-#include "StorageSystemTables.h"
-
-#include <Access/ContextAccess.h>
 #include <Columns/ColumnString.h>
-#include <DataTypes/DataTypeArray.h>
+#include <DataTypes/DataTypeString.h>
 #include <DataTypes/DataTypeDateTime.h>
 #include <DataTypes/DataTypeNullable.h>
-#include <DataTypes/DataTypeString.h>
-#include <DataTypes/DataTypeUUID.h>
-#include <DataTypes/DataTypesNumber.h>
-#include <Disks/IStoragePolicy.h>
+#include <Storages/System/StorageSystemTables.h>
+#include <Storages/System/getQueriedColumnsMaskAndHeader.h>
+#include <Storages/MergeTree/MergeTreeData.h>
+#include <Storages/SelectQueryInfo.h>
+#include <Storages/VirtualColumnUtils.h>
+#include <Access/ContextAccess.h>
 #include <Interpreters/Context.h>
 #include <Interpreters/DatabaseCatalog.h>
 #include <Interpreters/formatWithPossiblyHidingSecrets.h>
 #include <Parsers/ASTCreateQuery.h>
 #include <Parsers/ASTSelectWithUnionQuery.h>
+#include <Common/typeid_cast.h>
+#include <Common/StringUtils.h>
+#include <DataTypes/DataTypesNumber.h>
+#include <DataTypes/DataTypeArray.h>
+#include <Disks/IStoragePolicy.h>
 #include <Processors/ISource.h>
 #include <Processors/QueryPlan/QueryPlan.h>
 #include <Processors/QueryPlan/SourceStepWithFilter.h>
 #include <QueryPipeline/Pipe.h>
 #include <QueryPipeline/QueryPipelineBuilder.h>
-#include <Storages/MergeTree/MergeTreeData.h>
-#include <Storages/SelectQueryInfo.h>
-#include <Storages/System/ReadFromSystemTables.h>
-#include <Storages/System/getQueriedColumnsMaskAndHeader.h>
-#include <Storages/VirtualColumnUtils.h>
-#include <Common/StringUtils.h>
-#include <Common/typeid_cast.h>
+#include <DataTypes/DataTypeUUID.h>
 
 #include <boost/range/adaptor/map.hpp>
 
@@ -34,7 +32,8 @@ namespace DB
 {
 
 
-StorageSystemTables::StorageSystemTables(const StorageID & table_id_) : IStorage(table_id_)
+StorageSystemTables::StorageSystemTables(const StorageID & table_id_)
+    : IStorage(table_id_)
 {
     StorageInMemoryMetadata storage_metadata;
 
@@ -47,13 +46,9 @@ StorageSystemTables::StorageSystemTables(const StorageID & table_id_) : IStorage
         {"data_paths", std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()), "Paths to the table data in the file systems."},
         {"metadata_path", std::make_shared<DataTypeString>(), "Path to the table metadata in the file system."},
         {"metadata_modification_time", std::make_shared<DataTypeDateTime>(), "Time of latest modification of the table metadata."},
-        {"metadata_version",
-         std::make_shared<DataTypeInt32>(),
-         "Metadata version for ReplicatedMergeTree table, 0 for non ReplicatedMergeTree table."},
+        {"metadata_version", std::make_shared<DataTypeInt32>(), "Metadata version for ReplicatedMergeTree table, 0 for non ReplicatedMergeTree table."},
         {"dependencies_database", std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()), "Database dependencies."},
-        {"dependencies_table",
-         std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()),
-         "Table dependencies (materialized views the current table)."},
+        {"dependencies_table", std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()), "Table dependencies (materialized views the current table)."},
         {"create_table_query", std::make_shared<DataTypeString>(), "The query that was used to create the table."},
         {"engine_full", std::make_shared<DataTypeString>(), "Parameters of the table engine."},
         {"as_select", std::make_shared<DataTypeString>(), "SELECT query for view."},
@@ -62,54 +57,676 @@ StorageSystemTables::StorageSystemTables(const StorageID & table_id_) : IStorage
         {"primary_key", std::make_shared<DataTypeString>(), "The primary key expression specified in the table."},
         {"sampling_key", std::make_shared<DataTypeString>(), "The sampling key expression specified in the table."},
         {"storage_policy", std::make_shared<DataTypeString>(), "The storage policy."},
-        {"total_rows",
-         std::make_shared<DataTypeNullable>(std::make_shared<DataTypeUInt64>()),
-         "Total number of rows, if it is possible to quickly determine exact number of rows in the table, otherwise NULL (including "
-         "underlying Buffer table)."},
-        {"total_bytes",
-         std::make_shared<DataTypeNullable>(std::make_shared<DataTypeUInt64>()),
-         "Total number of bytes, if it is possible to quickly determine exact number "
-         "of bytes for the table on storage, otherwise NULL (does not includes any underlying storage). "
-         "If the table stores data on disk, returns used space on disk (i.e. compressed). "
-         "If the table stores data in memory, returns approximated number of used bytes in memory."},
-        {"total_bytes_uncompressed",
-         std::make_shared<DataTypeNullable>(std::make_shared<DataTypeUInt64>()),
-         "Total number of uncompressed bytes, if it's possible to quickly determine the exact number "
-         "of bytes from the part checksums for the table on storage, otherwise NULL (does not take underlying storage (if any) into "
-         "account)."},
+        {"total_rows", std::make_shared<DataTypeNullable>(std::make_shared<DataTypeUInt64>()),
+            "Total number of rows, if it is possible to quickly determine exact number of rows in the table, otherwise NULL (including underlying Buffer table)."
+        },
+        {"total_bytes", std::make_shared<DataTypeNullable>(std::make_shared<DataTypeUInt64>()),
+            "Total number of bytes, if it is possible to quickly determine exact number "
+            "of bytes for the table on storage, otherwise NULL (does not includes any underlying storage). "
+            "If the table stores data on disk, returns used space on disk (i.e. compressed). "
+            "If the table stores data in memory, returns approximated number of used bytes in memory."
+        },
+        {"total_bytes_uncompressed", std::make_shared<DataTypeNullable>(std::make_shared<DataTypeUInt64>()),
+            "Total number of uncompressed bytes, if it's possible to quickly determine the exact number "
+            "of bytes from the part checksums for the table on storage, otherwise NULL (does not take underlying storage (if any) into account)."
+        },
         {"parts", std::make_shared<DataTypeNullable>(std::make_shared<DataTypeUInt64>()), "The total number of parts in this table."},
-        {"active_parts",
-         std::make_shared<DataTypeNullable>(std::make_shared<DataTypeUInt64>()),
-         "The number of active parts in this table."},
-        {"total_marks",
-         std::make_shared<DataTypeNullable>(std::make_shared<DataTypeUInt64>()),
-         "The total number of marks in all parts in this table."},
-        {"lifetime_rows",
-         std::make_shared<DataTypeNullable>(std::make_shared<DataTypeUInt64>()),
-         "Total number of rows INSERTed since server start (only for Buffer tables)."},
-        {"lifetime_bytes",
-         std::make_shared<DataTypeNullable>(std::make_shared<DataTypeUInt64>()),
-         "Total number of bytes INSERTed since server start (only for Buffer tables)."},
+        {"active_parts", std::make_shared<DataTypeNullable>(std::make_shared<DataTypeUInt64>()), "The number of active parts in this table."},
+        {"total_marks", std::make_shared<DataTypeNullable>(std::make_shared<DataTypeUInt64>()), "The total number of marks in all parts in this table."},
+        {"lifetime_rows", std::make_shared<DataTypeNullable>(std::make_shared<DataTypeUInt64>()),
+            "Total number of rows INSERTed since server start (only for Buffer tables)."
+        },
+        {"lifetime_bytes", std::make_shared<DataTypeNullable>(std::make_shared<DataTypeUInt64>()),
+            "Total number of bytes INSERTed since server start (only for Buffer tables)."
+        },
         {"comment", std::make_shared<DataTypeString>(), "The comment for the table."},
-        {"has_own_data",
-         std::make_shared<DataTypeUInt8>(),
-         "Flag that indicates whether the table itself stores some data on disk or only accesses some other source."},
-        {"loading_dependencies_database",
-         std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()),
-         "Database loading dependencies (list of objects which should be loaded before the current object)."},
-        {"loading_dependencies_table",
-         std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()),
-         "Table loading dependencies (list of objects which should be loaded before the current object)."},
-        {"loading_dependent_database", std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()), "Dependent loading database."},
-        {"loading_dependent_table", std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()), "Dependent loading table."},
+        {"has_own_data", std::make_shared<DataTypeUInt8>(),
+            "Flag that indicates whether the table itself stores some data on disk or only accesses some other source."
+        },
+        {"loading_dependencies_database", std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()),
+            "Database loading dependencies (list of objects which should be loaded before the current object)."
+        },
+        {"loading_dependencies_table", std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()),
+            "Table loading dependencies (list of objects which should be loaded before the current object)."
+        },
+        {"loading_dependent_database", std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()),
+            "Dependent loading database."
+        },
+        {"loading_dependent_table", std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>()),
+            "Dependent loading table."
+        },
     };
 
-    description.setAliases({{"table", std::make_shared<DataTypeString>(), "name"}});
+    description.setAliases({
+        {"table", std::make_shared<DataTypeString>(), "name"}
+    });
 
     storage_metadata.setColumns(std::move(description));
     setInMemoryMetadata(storage_metadata);
 }
 
+
+namespace
+{
+
+ColumnPtr getFilteredDatabases(const ActionsDAG::Node * predicate, ContextPtr context)
+{
+    MutableColumnPtr column = ColumnString::create();
+
+    const auto databases = DatabaseCatalog::instance().getDatabases();
+    for (const auto & database_name : databases | boost::adaptors::map_keys)
+    {
+        if (database_name == DatabaseCatalog::TEMPORARY_DATABASE)
+            continue; /// We don't want to show the internal database for temporary tables in system.tables
+
+        column->insert(database_name);
+    }
+
+    Block block { ColumnWithTypeAndName(std::move(column), std::make_shared<DataTypeString>(), "database") };
+    VirtualColumnUtils::filterBlockWithPredicate(predicate, block, context);
+    return block.getByPosition(0).column;
+}
+
+ColumnPtr getFilteredTables(const ActionsDAG::Node * predicate, const ColumnPtr & filtered_databases_column, ContextPtr context)
+{
+    Block sample {
+        ColumnWithTypeAndName(nullptr, std::make_shared<DataTypeString>(), "name"),
+        ColumnWithTypeAndName(nullptr, std::make_shared<DataTypeString>(), "engine")
+    };
+
+    MutableColumnPtr database_column = ColumnString::create();
+    MutableColumnPtr engine_column;
+
+    auto dag = VirtualColumnUtils::splitFilterDagForAllowedInputs(predicate, &sample);
+    if (dag)
+    {
+        bool filter_by_engine = false;
+        for (const auto * input : dag->getInputs())
+            if (input->result_name == "engine")
+                filter_by_engine = true;
+
+        if (filter_by_engine)
+            engine_column = ColumnString::create();
+    }
+
+    for (size_t database_idx = 0; database_idx < filtered_databases_column->size(); ++database_idx)
+    {
+        const auto & database_name = filtered_databases_column->getDataAt(database_idx).toString();
+        DatabasePtr database = DatabaseCatalog::instance().tryGetDatabase(database_name);
+        if (!database)
+            continue;
+
+        for (auto table_it = database->getTablesIterator(context); table_it->isValid(); table_it->next())
+        {
+            database_column->insert(table_it->name());
+            if (engine_column)
+                engine_column->insert(table_it->table()->getName());
+        }
+    }
+
+    Block block {ColumnWithTypeAndName(std::move(database_column), std::make_shared<DataTypeString>(), "name")};
+    if (engine_column)
+        block.insert(ColumnWithTypeAndName(std::move(engine_column), std::make_shared<DataTypeString>(), "engine"));
+
+    if (dag)
+        VirtualColumnUtils::filterBlockWithDAG(dag, block, context);
+
+    return block.getByPosition(0).column;
+}
+
+/// Avoid heavy operation on tables if we only queried columns that we can get without table object.
+/// Otherwise it will require table initialization for Lazy database.
+bool needTable(const DatabasePtr & database, const Block & header)
+{
+    if (database->getEngineName() != "Lazy")
+        return true;
+
+    static const std::set<std::string> columns_without_table = { "database", "name", "uuid", "metadata_modification_time" };
+    for (const auto & column : header.getColumnsWithTypeAndName())
+    {
+        if (columns_without_table.find(column.name) == columns_without_table.end())
+            return true;
+    }
+    return false;
+}
+
+
+class TablesBlockSource : public ISource
+{
+public:
+    TablesBlockSource(
+        std::vector<UInt8> columns_mask_,
+        Block header,
+        UInt64 max_block_size_,
+        ColumnPtr databases_,
+        ColumnPtr tables_,
+        ContextPtr context_)
+        : ISource(std::move(header))
+        , columns_mask(std::move(columns_mask_))
+        , max_block_size(max_block_size_)
+        , databases(std::move(databases_))
+        , context(Context::createCopy(context_))
+    {
+        size_t size = tables_->size();
+        tables.reserve(size);
+        for (size_t idx = 0; idx < size; ++idx)
+            tables.insert(tables_->getDataAt(idx).toString());
+    }
+
+    String getName() const override { return "Tables"; }
+
+protected:
+    Chunk generate() override
+    {
+        if (done)
+            return {};
+
+        MutableColumns res_columns = getPort().getHeader().cloneEmptyColumns();
+
+        const auto access = context->getAccess();
+        const bool need_to_check_access_for_databases = !access->isGranted(AccessType::SHOW_TABLES);
+
+        size_t rows_count = 0;
+        while (rows_count < max_block_size)
+        {
+            if (tables_it && !tables_it->isValid())
+                ++database_idx;
+
+            while (database_idx < databases->size() && (!tables_it || !tables_it->isValid()))
+            {
+                database_name = databases->getDataAt(database_idx).toString();
+                database = DatabaseCatalog::instance().tryGetDatabase(database_name);
+
+                if (!database)
+                {
+                    /// Database was deleted just now or the user has no access.
+                    ++database_idx;
+                    continue;
+                }
+
+                break;
+            }
+
+            /// This is for temporary tables. They are output in single block regardless to max_block_size.
+            if (database_idx >= databases->size())
+            {
+                if (context->hasSessionContext())
+                {
+                    Tables external_tables = context->getSessionContext()->getExternalTables();
+
+                    for (auto & table : external_tables)
+                    {
+                        size_t src_index = 0;
+                        size_t res_index = 0;
+
+                        // database
+                        if (columns_mask[src_index++])
+                            res_columns[res_index++]->insertDefault();
+
+                        // name
+                        if (columns_mask[src_index++])
+                            res_columns[res_index++]->insert(table.first);
+
+                        // uuid
+                        if (columns_mask[src_index++])
+                            res_columns[res_index++]->insert(table.second->getStorageID().uuid);
+
+                        // engine
+                        if (columns_mask[src_index++])
+                            res_columns[res_index++]->insert(table.second->getName());
+
+                        // is_temporary
+                        if (columns_mask[src_index++])
+                            res_columns[res_index++]->insert(1u);
+
+                        // data_paths
+                        if (columns_mask[src_index++])
+                            res_columns[res_index++]->insertDefault();
+
+                        // metadata_path
+                        if (columns_mask[src_index++])
+                            res_columns[res_index++]->insertDefault();
+
+                        // metadata_modification_time
+                        if (columns_mask[src_index++])
+                            res_columns[res_index++]->insertDefault();
+
+                        // metadata_version
+                        // Temporary tables does not support replication
+                        if (columns_mask[src_index++])
+                            res_columns[res_index++]->insertDefault();
+
+                        // dependencies_database
+                        if (columns_mask[src_index++])
+                            res_columns[res_index++]->insertDefault();
+
+                        // dependencies_table
+                        if (columns_mask[src_index++])
+                            res_columns[res_index++]->insertDefault();
+
+                        // create_table_query
+                        if (columns_mask[src_index++])
+                        {
+                            auto temp_db = DatabaseCatalog::instance().getDatabaseForTemporaryTables();
+                            ASTPtr ast = temp_db ? temp_db->tryGetCreateTableQuery(table.second->getStorageID().getTableName(), context) : nullptr;
+                            res_columns[res_index++]->insert(ast ? format({context, *ast}) : "");
+                        }
+
+                        // engine_full
+                        if (columns_mask[src_index++])
+                            res_columns[res_index++]->insert(table.second->getName());
+
+                        const auto & settings = context->getSettingsRef();
+                        while (src_index < columns_mask.size())
+                        {
+                            // total_rows
+                            if (src_index == 19 && columns_mask[src_index])
+                            {
+                                if (auto total_rows = table.second->totalRows(settings))
+                                    res_columns[res_index++]->insert(*total_rows);
+                                else
+                                    res_columns[res_index++]->insertDefault();
+                            }
+                            // total_bytes
+                            else if (src_index == 20 && columns_mask[src_index])
+                            {
+                                if (auto total_bytes = table.second->totalBytes(settings))
+                                    res_columns[res_index++]->insert(*total_bytes);
+                                else
+                                    res_columns[res_index++]->insertDefault();
+                            }
+                            /// Fill the rest columns with defaults
+                            else if (columns_mask[src_index])
+                                res_columns[res_index++]->insertDefault();
+                            src_index++;
+                        }
+                    }
+                }
+
+                UInt64 num_rows = res_columns.at(0)->size();
+                done = true;
+                return Chunk(std::move(res_columns), num_rows);
+            }
+
+            const bool need_to_check_access_for_tables = need_to_check_access_for_databases && !access->isGranted(AccessType::SHOW_TABLES, database_name);
+
+            if (!tables_it || !tables_it->isValid())
+                tables_it = database->getTablesIterator(context);
+
+            const bool need_table = needTable(database, getPort().getHeader());
+
+            for (; rows_count < max_block_size && tables_it->isValid(); tables_it->next())
+            {
+                auto table_name = tables_it->name();
+                if (!tables.contains(table_name))
+                    continue;
+
+                if (need_to_check_access_for_tables && !access->isGranted(AccessType::SHOW_TABLES, database_name, table_name))
+                    continue;
+
+                StoragePtr table = nullptr;
+                TableLockHolder lock;
+                if (need_table)
+                {
+                    table = tables_it->table();
+                    if (!table)
+                        // Table might have just been removed or detached for Lazy engine (see DatabaseLazy::tryGetTable())
+                        continue;
+
+                    /// The only column that requires us to hold a shared lock is data_paths as rename might alter them (on ordinary tables)
+                    /// and it's not protected internally by other mutexes
+                    static const size_t DATA_PATHS_INDEX = 5;
+                    if (columns_mask[DATA_PATHS_INDEX])
+                    {
+                        lock = table->tryLockForShare(context->getCurrentQueryId(),
+                                                      context->getSettingsRef().lock_acquire_timeout);
+                        if (!lock)
+                            // Table was dropped while acquiring the lock, skipping table
+                            continue;
+                    }
+                }
+                ++rows_count;
+
+                size_t src_index = 0;
+                size_t res_index = 0;
+
+                if (columns_mask[src_index++])
+                    res_columns[res_index++]->insert(database_name);
+
+                if (columns_mask[src_index++])
+                    res_columns[res_index++]->insert(table_name);
+
+                if (columns_mask[src_index++])
+                    res_columns[res_index++]->insert(tables_it->uuid());
+
+                if (columns_mask[src_index++])
+                {
+                    chassert(table != nullptr);
+                    res_columns[res_index++]->insert(table->getName());
+                }
+
+                if (columns_mask[src_index++])
+                    res_columns[res_index++]->insert(0u);  // is_temporary
+
+                if (columns_mask[src_index++])
+                {
+                    chassert(lock != nullptr);
+                    Array table_paths_array;
+                    auto paths = table->getDataPaths();
+                    table_paths_array.reserve(paths.size());
+                    for (const String & path : paths)
+                        table_paths_array.push_back(path);
+                    res_columns[res_index++]->insert(table_paths_array);
+                    /// We don't need the lock anymore
+                    lock = nullptr;
+                }
+
+                if (columns_mask[src_index++])
+                    res_columns[res_index++]->insert(database->getObjectMetadataPath(table_name));
+
+                if (columns_mask[src_index++])
+                    res_columns[res_index++]->insert(static_cast<UInt64>(database->getObjectMetadataModificationTime(table_name)));
+
+                StorageMetadataPtr metadata_snapshot;
+                if (table)
+                    metadata_snapshot = table->getInMemoryMetadataPtr();
+
+                if (columns_mask[src_index++])
+                {
+                    if (metadata_snapshot && table->supportsReplication())
+                        res_columns[res_index++]->insert(metadata_snapshot->metadata_version);
+                    else
+                        res_columns[res_index++]->insertDefault();
+                }
+
+                {
+                    Array views_table_name_array;
+                    Array views_database_name_array;
+                    if (columns_mask[src_index] || columns_mask[src_index + 1])
+                    {
+                        const auto view_ids = DatabaseCatalog::instance().getDependentViews(StorageID(database_name, table_name));
+
+                        views_table_name_array.reserve(view_ids.size());
+                        views_database_name_array.reserve(view_ids.size());
+                        for (const auto & view_id : view_ids)
+                        {
+                            views_table_name_array.push_back(view_id.table_name);
+                            views_database_name_array.push_back(view_id.database_name);
+                        }
+                    }
+
+                    if (columns_mask[src_index++])
+                        res_columns[res_index++]->insert(views_database_name_array);
+
+                    if (columns_mask[src_index++])
+                        res_columns[res_index++]->insert(views_table_name_array);
+                }
+
+                if (columns_mask[src_index] || columns_mask[src_index + 1] || columns_mask[src_index + 2])
+                {
+                    ASTPtr ast = database->tryGetCreateTableQuery(table_name, context);
+                    auto * ast_create = ast ? ast->as<ASTCreateQuery>() : nullptr;
+
+                    if (ast_create && !context->getSettingsRef().show_table_uuid_in_table_create_query_if_not_nil)
+                    {
+                        ast_create->uuid = UUIDHelpers::Nil;
+                        ast_create->to_inner_uuid = UUIDHelpers::Nil;
+                    }
+
+                    if (columns_mask[src_index++])
+                        res_columns[res_index++]->insert(ast ? format({context, *ast}) : "");
+
+                    if (columns_mask[src_index++])
+                    {
+                        String engine_full;
+
+                        if (ast_create && ast_create->storage)
+                        {
+                            engine_full = format({context, *ast_create->storage});
+
+                            static const char * const extra_head = " ENGINE = ";
+                            if (startsWith(engine_full, extra_head))
+                                engine_full = engine_full.substr(strlen(extra_head));
+                        }
+
+                        res_columns[res_index++]->insert(engine_full);
+                    }
+
+                    if (columns_mask[src_index++])
+                    {
+                        String as_select;
+                        if (ast_create && ast_create->select)
+                            as_select = format({context, *ast_create->select});
+                        res_columns[res_index++]->insert(as_select);
+                    }
+                }
+                else
+                    src_index += 3;
+
+                ASTPtr expression_ptr;
+                if (columns_mask[src_index++])
+                {
+                    if (metadata_snapshot && (expression_ptr = metadata_snapshot->getPartitionKeyAST()))
+                        res_columns[res_index++]->insert(format({context, *expression_ptr}));
+                    else
+                        res_columns[res_index++]->insertDefault();
+                }
+
+                if (columns_mask[src_index++])
+                {
+                    if (metadata_snapshot && (expression_ptr = metadata_snapshot->getSortingKey().expression_list_ast))
+                        res_columns[res_index++]->insert(format({context, *expression_ptr}));
+                    else
+                        res_columns[res_index++]->insertDefault();
+                }
+
+                if (columns_mask[src_index++])
+                {
+                    if (metadata_snapshot && (expression_ptr = metadata_snapshot->getPrimaryKey().expression_list_ast))
+                        res_columns[res_index++]->insert(format({context, *expression_ptr}));
+                    else
+                        res_columns[res_index++]->insertDefault();
+                }
+
+                if (columns_mask[src_index++])
+                {
+                    if (metadata_snapshot && (expression_ptr = metadata_snapshot->getSamplingKeyAST()))
+                        res_columns[res_index++]->insert(format({context, *expression_ptr}));
+                    else
+                        res_columns[res_index++]->insertDefault();
+                }
+
+                if (columns_mask[src_index++])
+                {
+                    auto policy = table ? table->getStoragePolicy() : nullptr;
+                    if (policy)
+                        res_columns[res_index++]->insert(policy->getName());
+                    else
+                        res_columns[res_index++]->insertDefault();
+                }
+
+                auto settings = context->getSettingsRef();
+                settings.select_sequential_consistency = 0;
+                if (columns_mask[src_index++])
+                {
+                    auto total_rows = table ? table->totalRows(settings) : std::nullopt;
+                    if (total_rows)
+                        res_columns[res_index++]->insert(*total_rows);
+                    else
+                        res_columns[res_index++]->insertDefault();
+                }
+
+                if (columns_mask[src_index++])
+                {
+                    auto total_bytes = table->totalBytes(settings);
+                    if (total_bytes)
+                        res_columns[res_index++]->insert(*total_bytes);
+                    else
+                        res_columns[res_index++]->insertDefault();
+                }
+
+                if (columns_mask[src_index++])
+                {
+                    auto total_bytes_uncompressed = table->totalBytesUncompressed(settings);
+                    if (total_bytes_uncompressed)
+                        res_columns[res_index++]->insert(*total_bytes_uncompressed);
+                    else
+                        res_columns[res_index++]->insertDefault();
+                }
+
+                auto table_merge_tree = std::dynamic_pointer_cast<MergeTreeData>(table);
+                if (columns_mask[src_index++])
+                {
+                    if (table_merge_tree)
+                        res_columns[res_index++]->insert(table_merge_tree->getAllPartsCount());
+                    else
+                        res_columns[res_index++]->insertDefault();
+                }
+
+                if (columns_mask[src_index++])
+                {
+                    if (table_merge_tree)
+                        res_columns[res_index++]->insert(table_merge_tree->getActivePartsCount());
+                    else
+                        res_columns[res_index++]->insertDefault();
+                }
+
+                if (columns_mask[src_index++])
+                {
+                    if (table_merge_tree)
+                    {
+                        res_columns[res_index++]->insert(table_merge_tree->getTotalMarksCount());
+                    }
+                    else
+                        res_columns[res_index++]->insertDefault();
+                }
+
+                if (columns_mask[src_index++])
+                {
+                    auto lifetime_rows = table ? table->lifetimeRows() : std::nullopt;
+                    if (lifetime_rows)
+                        res_columns[res_index++]->insert(*lifetime_rows);
+                    else
+                        res_columns[res_index++]->insertDefault();
+                }
+
+                if (columns_mask[src_index++])
+                {
+                    auto lifetime_bytes = table ? table->lifetimeBytes() : std::nullopt;
+                    if (lifetime_bytes)
+                        res_columns[res_index++]->insert(*lifetime_bytes);
+                    else
+                        res_columns[res_index++]->insertDefault();
+                }
+
+                if (columns_mask[src_index++])
+                {
+                    if (metadata_snapshot)
+                        res_columns[res_index++]->insert(metadata_snapshot->comment);
+                    else
+                        res_columns[res_index++]->insertDefault();
+                }
+
+                if (columns_mask[src_index++])
+                {
+                    if (table)
+                        res_columns[res_index++]->insert(table->storesDataOnDisk());
+                    else
+                        res_columns[res_index++]->insertDefault();
+                }
+
+                if (columns_mask[src_index] || columns_mask[src_index + 1] || columns_mask[src_index + 2] || columns_mask[src_index + 3])
+                {
+                    auto dependencies = DatabaseCatalog::instance().getLoadingDependencies(StorageID{database_name, table_name});
+                    auto dependents = DatabaseCatalog::instance().getLoadingDependents(StorageID{database_name, table_name});
+
+                    Array dependencies_databases;
+                    Array dependencies_tables;
+                    dependencies_databases.reserve(dependencies.size());
+                    dependencies_tables.reserve(dependencies.size());
+                    for (const auto & dependency : dependencies)
+                    {
+                        dependencies_databases.push_back(dependency.database_name);
+                        dependencies_tables.push_back(dependency.table_name);
+                    }
+
+                    Array dependents_databases;
+                    Array dependents_tables;
+                    dependents_databases.reserve(dependents.size());
+                    dependents_tables.reserve(dependents.size());
+                    for (const auto & dependent : dependents)
+                    {
+                        dependents_databases.push_back(dependent.database_name);
+                        dependents_tables.push_back(dependent.table_name);
+                    }
+
+                    if (columns_mask[src_index++])
+                        res_columns[res_index++]->insert(dependencies_databases);
+                    if (columns_mask[src_index++])
+                        res_columns[res_index++]->insert(dependencies_tables);
+
+                    if (columns_mask[src_index++])
+                        res_columns[res_index++]->insert(dependents_databases);
+                    if (columns_mask[src_index++])
+                        res_columns[res_index++]->insert(dependents_tables);
+
+                }
+            }
+        }
+
+        UInt64 num_rows = res_columns.at(0)->size();
+        return Chunk(std::move(res_columns), num_rows);
+    }
+private:
+    std::vector<UInt8> columns_mask;
+    UInt64 max_block_size;
+    ColumnPtr databases;
+    NameSet tables;
+    size_t database_idx = 0;
+    DatabaseTablesIteratorPtr tables_it;
+    ContextPtr context;
+    bool done = false;
+    DatabasePtr database;
+    std::string database_name;
+};
+
+}
+
+class ReadFromSystemTables : public SourceStepWithFilter
+{
+public:
+    std::string getName() const override { return "ReadFromSystemTables"; }
+    void initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) override;
+
+    ReadFromSystemTables(
+        const Names & column_names_,
+        const SelectQueryInfo & query_info_,
+        const StorageSnapshotPtr & storage_snapshot_,
+        const ContextPtr & context_,
+        Block sample_block,
+        std::vector<UInt8> columns_mask_,
+        size_t max_block_size_)
+        : SourceStepWithFilter(
+            DataStream{.header = std::move(sample_block)},
+            column_names_,
+            query_info_,
+            storage_snapshot_,
+            context_)
+        , columns_mask(std::move(columns_mask_))
+        , max_block_size(max_block_size_)
+    {
+    }
+
+    void applyFilters(ActionDAGNodes added_filter_nodes) override;
+
+private:
+    std::vector<UInt8> columns_mask;
+    size_t max_block_size;
+
+    ColumnPtr filtered_databases_column;
+    ColumnPtr filtered_tables_column;
+};
+
 void StorageSystemTables::read(
     QueryPlan & query_plan,
     const Names & column_names,
@@ -126,8 +743,28 @@ void StorageSystemTables::read(
     auto [columns_mask, res_block] = getQueriedColumnsMaskAndHeader(sample_block, column_names);
 
     auto reading = std::make_unique<ReadFromSystemTables>(
-        column_names, query_info, storage_snapshot, context, std::move(res_block), std::move(columns_mask), max_block_size, false);
+        column_names, query_info, storage_snapshot, context, std::move(res_block), std::move(columns_mask), max_block_size);
 
     query_plan.addStep(std::move(reading));
 }
+
+void ReadFromSystemTables::applyFilters(ActionDAGNodes added_filter_nodes)
+{
+    SourceStepWithFilter::applyFilters(std::move(added_filter_nodes));
+
+    const ActionsDAG::Node * predicate = nullptr;
+    if (filter_actions_dag)
+        predicate = filter_actions_dag->getOutputs().at(0);
+
+    filtered_databases_column = getFilteredDatabases(predicate, context);
+    filtered_tables_column = getFilteredTables(predicate, filtered_databases_column, context);
+}
+
+void ReadFromSystemTables::initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &)
+{
+    Pipe pipe(std::make_shared<TablesBlockSource>(
+        std::move(columns_mask), getOutputStream().header, max_block_size, std::move(filtered_databases_column), std::move(filtered_tables_column), context));
+    pipeline.init(std::move(pipe));
+}
+
 }
diff --git a/src/Storages/System/TablesBlockSource.cpp b/src/Storages/System/TablesBlockSource.cpp
deleted file mode 100644
index 4ea5bb53deb..00000000000
--- a/src/Storages/System/TablesBlockSource.cpp
+++ /dev/null
@@ -1,569 +0,0 @@
-#include "TablesBlockSource.h"
-
-#include <memory>
-
-#include <Access/ContextAccess.h>
-#include <Disks/IStoragePolicy.h>
-#include <Interpreters/DatabaseCatalog.h>
-#include <Interpreters/formatWithPossiblyHidingSecrets.h>
-#include <Parsers/ASTSelectWithUnionQuery.h>
-#include <Storages/IStorage.h>
-#include <Storages/MergeTree/MergeTreeData.h>
-#include <Common/logger_useful.h>
-
-namespace DB
-{
-
-namespace
-{
-
-/// Avoid heavy operation on tables if we only queried columns that we can get without table object.
-/// Otherwise it will require table initialization for Lazy database.
-bool needTable(const DatabasePtr & database, const Block & header)
-{
-    if (database->getEngineName() != "Lazy")
-        return true;
-
-    static const std::set<std::string> columns_without_table = { "database", "name", "uuid", "metadata_modification_time" };
-    for (const auto & column : header.getColumnsWithTypeAndName())
-    {
-        if (columns_without_table.find(column.name) == columns_without_table.end())
-            return true;
-    }
-    return false;
-}
-
-}
-
-Chunk TablesBlockSource::generate()
-{
-    if (done)
-        return {};
-
-    MutableColumns res_columns = getPort().getHeader().cloneEmptyColumns();
-
-    const auto access = context->getAccess();
-    const bool need_to_check_access_for_databases = !access->isGranted(AccessType::SHOW_TABLES);
-
-    size_t rows_count = 0;
-    while (rows_count < max_block_size)
-    {
-        if (tables_it && !tables_it->isValid())
-            ++database_idx;
-
-        while (database_idx < databases->size() && (!tables_it || !tables_it->isValid()))
-        {
-            database_name = databases->getDataAt(database_idx).toString();
-            database = DatabaseCatalog::instance().tryGetDatabase(database_name);
-
-            if (!database)
-            {
-                /// Database was deleted just now or the user has no access.
-                ++database_idx;
-                continue;
-            }
-
-            break;
-        }
-
-        /// This is for temporary tables. They are output in single block regardless to max_block_size.
-        if (database_idx >= databases->size())
-        {
-            if (context->hasSessionContext())
-            {
-                Tables external_tables = context->getSessionContext()->getExternalTables();
-
-                for (auto & table : external_tables)
-                {
-                    size_t src_index = 0;
-                    size_t res_index = 0;
-
-                    // database
-                    if (columns_mask[src_index++])
-                        res_columns[res_index++]->insertDefault();
-
-                    // name
-                    if (columns_mask[src_index++])
-                        res_columns[res_index++]->insert(table.first);
-
-                    // uuid
-                    if (columns_mask[src_index++])
-                        res_columns[res_index++]->insert(table.second->getStorageID().uuid);
-
-                    // engine
-                    if (columns_mask[src_index++])
-                        res_columns[res_index++]->insert(table.second->getName());
-
-                    // is_temporary
-                    if (columns_mask[src_index++])
-                        res_columns[res_index++]->insert(1u);
-
-                    // data_paths
-                    if (columns_mask[src_index++])
-                        res_columns[res_index++]->insertDefault();
-
-                    // metadata_path
-                    if (columns_mask[src_index++])
-                        res_columns[res_index++]->insertDefault();
-
-                    // metadata_modification_time
-                    if (columns_mask[src_index++])
-                        res_columns[res_index++]->insertDefault();
-
-                    // metadata_version
-                    // Temporary tables does not support replication
-                    if (columns_mask[src_index++])
-                        res_columns[res_index++]->insertDefault();
-
-                    // dependencies_database
-                    if (columns_mask[src_index++])
-                        res_columns[res_index++]->insertDefault();
-
-                    // dependencies_table
-                    if (columns_mask[src_index++])
-                        res_columns[res_index++]->insertDefault();
-
-                    // create_table_query
-                    if (columns_mask[src_index++])
-                    {
-                        auto temp_db = DatabaseCatalog::instance().getDatabaseForTemporaryTables();
-                        ASTPtr ast
-                            = temp_db ? temp_db->tryGetCreateTableQuery(table.second->getStorageID().getTableName(), context) : nullptr;
-                        res_columns[res_index++]->insert(ast ? format({context, *ast}) : "");
-                    }
-
-                    // engine_full
-                    if (columns_mask[src_index++])
-                        res_columns[res_index++]->insert(table.second->getName());
-
-                    const auto & settings = context->getSettingsRef();
-                    while (src_index < columns_mask.size())
-                    {
-                        // total_rows
-                        if (src_index == 19 && columns_mask[src_index])
-                        {
-                            if (auto total_rows = table.second->totalRows(settings))
-                                res_columns[res_index++]->insert(*total_rows);
-                            else
-                                res_columns[res_index++]->insertDefault();
-                        }
-                        // total_bytes
-                        else if (src_index == 20 && columns_mask[src_index])
-                        {
-                            if (auto total_bytes = table.second->totalBytes(settings))
-                                res_columns[res_index++]->insert(*total_bytes);
-                            else
-                                res_columns[res_index++]->insertDefault();
-                        }
-                        /// Fill the rest columns with defaults
-                        else if (columns_mask[src_index])
-                            res_columns[res_index++]->insertDefault();
-                        src_index++;
-                    }
-                }
-            }
-
-            UInt64 num_rows = res_columns.at(0)->size();
-            done = true;
-            return Chunk(std::move(res_columns), num_rows);
-        }
-
-        const bool need_to_check_access_for_tables
-            = need_to_check_access_for_databases && !access->isGranted(AccessType::SHOW_TABLES, database_name);
-
-        if (!tables_it || !tables_it->isValid())
-            tables_it = database->getTablesIterator(context);
-
-        const bool need_table = needTable(database, getPort().getHeader());
-
-        for (; rows_count < max_block_size && tables_it->isValid(); tables_it->next())
-        {
-            auto table_name = tables_it->name();
-            if (!tables.contains(table_name))
-                continue;
-
-            if (need_to_check_access_for_tables && !access->isGranted(AccessType::SHOW_TABLES, database_name, table_name))
-                continue;
-
-            StoragePtr table = nullptr;
-            TableLockHolder lock;
-            if (need_table)
-            {
-                table = tables_it->table();
-                if (!table)
-                    // Table might have just been removed or detached for Lazy engine (see DatabaseLazy::tryGetTable())
-                    continue;
-
-                /// The only column that requires us to hold a shared lock is data_paths as rename might alter them (on ordinary tables)
-                /// and it's not protected internally by other mutexes
-                static const size_t DATA_PATHS_INDEX = 5;
-                if (columns_mask[DATA_PATHS_INDEX])
-                {
-                    lock = table->tryLockForShare(context->getCurrentQueryId(), context->getSettingsRef().lock_acquire_timeout);
-                    if (!lock)
-                        // Table was dropped while acquiring the lock, skipping table
-                        continue;
-                }
-            }
-            ++rows_count;
-
-            size_t src_index = 0;
-            size_t res_index = 0;
-
-            if (columns_mask[src_index++])
-                res_columns[res_index++]->insert(database_name);
-
-            if (columns_mask[src_index++])
-                res_columns[res_index++]->insert(table_name);
-
-            if (columns_mask[src_index++])
-                res_columns[res_index++]->insert(tables_it->uuid());
-
-            if (columns_mask[src_index++])
-            {
-                chassert(table != nullptr);
-                res_columns[res_index++]->insert(table->getName());
-            }
-
-            if (columns_mask[src_index++])
-                res_columns[res_index++]->insert(0u); // is_temporary
-
-            if (columns_mask[src_index++])
-            {
-                chassert(lock != nullptr);
-                Array table_paths_array;
-                auto paths = table->getDataPaths();
-                table_paths_array.reserve(paths.size());
-                for (const String & path : paths)
-                    table_paths_array.push_back(path);
-                res_columns[res_index++]->insert(table_paths_array);
-                /// We don't need the lock anymore
-                lock = nullptr;
-            }
-
-            if (columns_mask[src_index++])
-                res_columns[res_index++]->insert(database->getObjectMetadataPath(table_name));
-
-            if (columns_mask[src_index++])
-                res_columns[res_index++]->insert(static_cast<UInt64>(database->getObjectMetadataModificationTime(table_name)));
-
-            StorageMetadataPtr metadata_snapshot;
-            if (table)
-                metadata_snapshot = table->getInMemoryMetadataPtr();
-
-            if (columns_mask[src_index++])
-            {
-                if (metadata_snapshot && table->supportsReplication())
-                    res_columns[res_index++]->insert(metadata_snapshot->metadata_version);
-                else
-                    res_columns[res_index++]->insertDefault();
-            }
-
-            {
-                Array views_table_name_array;
-                Array views_database_name_array;
-                if (columns_mask[src_index] || columns_mask[src_index + 1])
-                {
-                    const auto view_ids = DatabaseCatalog::instance().getDependentViews(StorageID(database_name, table_name));
-
-                    views_table_name_array.reserve(view_ids.size());
-                    views_database_name_array.reserve(view_ids.size());
-                    for (const auto & view_id : view_ids)
-                    {
-                        views_table_name_array.push_back(view_id.table_name);
-                        views_database_name_array.push_back(view_id.database_name);
-                    }
-                }
-
-                if (columns_mask[src_index++])
-                    res_columns[res_index++]->insert(views_database_name_array);
-
-                if (columns_mask[src_index++])
-                    res_columns[res_index++]->insert(views_table_name_array);
-            }
-
-            if (columns_mask[src_index] || columns_mask[src_index + 1] || columns_mask[src_index + 2])
-            {
-                ASTPtr ast = database->tryGetCreateTableQuery(table_name, context);
-                auto * ast_create = ast ? ast->as<ASTCreateQuery>() : nullptr;
-
-                if (ast_create && !context->getSettingsRef().show_table_uuid_in_table_create_query_if_not_nil)
-                {
-                    ast_create->uuid = UUIDHelpers::Nil;
-                    ast_create->to_inner_uuid = UUIDHelpers::Nil;
-                }
-
-                if (columns_mask[src_index++])
-                    res_columns[res_index++]->insert(ast ? format({context, *ast}) : "");
-
-                if (columns_mask[src_index++])
-                {
-                    String engine_full;
-
-                    if (ast_create && ast_create->storage)
-                    {
-                        engine_full = format({context, *ast_create->storage});
-
-                        static const char * const extra_head = " ENGINE = ";
-                        if (startsWith(engine_full, extra_head))
-                            engine_full = engine_full.substr(strlen(extra_head));
-                    }
-
-                    res_columns[res_index++]->insert(engine_full);
-                }
-
-                if (columns_mask[src_index++])
-                {
-                    String as_select;
-                    if (ast_create && ast_create->select)
-                        as_select = format({context, *ast_create->select});
-                    res_columns[res_index++]->insert(as_select);
-                }
-            }
-            else
-                src_index += 3;
-
-            ASTPtr expression_ptr;
-            if (columns_mask[src_index++])
-            {
-                if (metadata_snapshot && (expression_ptr = metadata_snapshot->getPartitionKeyAST()))
-                    res_columns[res_index++]->insert(format({context, *expression_ptr}));
-                else
-                    res_columns[res_index++]->insertDefault();
-            }
-
-            if (columns_mask[src_index++])
-            {
-                if (metadata_snapshot && (expression_ptr = metadata_snapshot->getSortingKey().expression_list_ast))
-                    res_columns[res_index++]->insert(format({context, *expression_ptr}));
-                else
-                    res_columns[res_index++]->insertDefault();
-            }
-
-            if (columns_mask[src_index++])
-            {
-                if (metadata_snapshot && (expression_ptr = metadata_snapshot->getPrimaryKey().expression_list_ast))
-                    res_columns[res_index++]->insert(format({context, *expression_ptr}));
-                else
-                    res_columns[res_index++]->insertDefault();
-            }
-
-            if (columns_mask[src_index++])
-            {
-                if (metadata_snapshot && (expression_ptr = metadata_snapshot->getSamplingKeyAST()))
-                    res_columns[res_index++]->insert(format({context, *expression_ptr}));
-                else
-                    res_columns[res_index++]->insertDefault();
-            }
-
-            if (columns_mask[src_index++])
-            {
-                auto policy = table ? table->getStoragePolicy() : nullptr;
-                if (policy)
-                    res_columns[res_index++]->insert(policy->getName());
-                else
-                    res_columns[res_index++]->insertDefault();
-            }
-
-            auto settings = context->getSettingsRef();
-            settings.select_sequential_consistency = 0;
-            if (columns_mask[src_index++])
-            {
-                auto total_rows = table ? table->totalRows(settings) : std::nullopt;
-                if (total_rows)
-                    res_columns[res_index++]->insert(*total_rows);
-                else
-                    res_columns[res_index++]->insertDefault();
-            }
-
-            if (columns_mask[src_index++])
-            {
-                auto total_bytes = table->totalBytes(settings);
-                if (total_bytes)
-                    res_columns[res_index++]->insert(*total_bytes);
-                else
-                    res_columns[res_index++]->insertDefault();
-            }
-
-            if (columns_mask[src_index++])
-            {
-                auto total_bytes_uncompressed = table->totalBytesUncompressed(settings);
-                if (total_bytes_uncompressed)
-                    res_columns[res_index++]->insert(*total_bytes_uncompressed);
-                else
-                    res_columns[res_index++]->insertDefault();
-            }
-
-            auto table_merge_tree = std::dynamic_pointer_cast<MergeTreeData>(table);
-            if (columns_mask[src_index++])
-            {
-                if (table_merge_tree)
-                    res_columns[res_index++]->insert(table_merge_tree->getAllPartsCount());
-                else
-                    res_columns[res_index++]->insertDefault();
-            }
-
-            if (columns_mask[src_index++])
-            {
-                if (table_merge_tree)
-                    res_columns[res_index++]->insert(table_merge_tree->getActivePartsCount());
-                else
-                    res_columns[res_index++]->insertDefault();
-            }
-
-            if (columns_mask[src_index++])
-            {
-                if (table_merge_tree)
-                {
-                    res_columns[res_index++]->insert(table_merge_tree->getTotalMarksCount());
-                }
-                else
-                    res_columns[res_index++]->insertDefault();
-            }
-
-            if (columns_mask[src_index++])
-            {
-                auto lifetime_rows = table ? table->lifetimeRows() : std::nullopt;
-                if (lifetime_rows)
-                    res_columns[res_index++]->insert(*lifetime_rows);
-                else
-                    res_columns[res_index++]->insertDefault();
-            }
-
-            if (columns_mask[src_index++])
-            {
-                auto lifetime_bytes = table ? table->lifetimeBytes() : std::nullopt;
-                if (lifetime_bytes)
-                    res_columns[res_index++]->insert(*lifetime_bytes);
-                else
-                    res_columns[res_index++]->insertDefault();
-            }
-
-            if (columns_mask[src_index++])
-            {
-                if (metadata_snapshot)
-                    res_columns[res_index++]->insert(metadata_snapshot->comment);
-                else
-                    res_columns[res_index++]->insertDefault();
-            }
-
-            if (columns_mask[src_index++])
-            {
-                if (table)
-                    res_columns[res_index++]->insert(table->storesDataOnDisk());
-                else
-                    res_columns[res_index++]->insertDefault();
-            }
-
-            if (columns_mask[src_index] || columns_mask[src_index + 1] || columns_mask[src_index + 2] || columns_mask[src_index + 3])
-            {
-                auto dependencies = DatabaseCatalog::instance().getLoadingDependencies(StorageID{database_name, table_name});
-                auto dependents = DatabaseCatalog::instance().getLoadingDependents(StorageID{database_name, table_name});
-
-                Array dependencies_databases;
-                Array dependencies_tables;
-                dependencies_databases.reserve(dependencies.size());
-                dependencies_tables.reserve(dependencies.size());
-                for (const auto & dependency : dependencies)
-                {
-                    dependencies_databases.push_back(dependency.database_name);
-                    dependencies_tables.push_back(dependency.table_name);
-                }
-
-                Array dependents_databases;
-                Array dependents_tables;
-                dependents_databases.reserve(dependents.size());
-                dependents_tables.reserve(dependents.size());
-                for (const auto & dependent : dependents)
-                {
-                    dependents_databases.push_back(dependent.database_name);
-                    dependents_tables.push_back(dependent.table_name);
-                }
-
-                if (columns_mask[src_index++])
-                    res_columns[res_index++]->insert(dependencies_databases);
-                if (columns_mask[src_index++])
-                    res_columns[res_index++]->insert(dependencies_tables);
-
-                if (columns_mask[src_index++])
-                    res_columns[res_index++]->insert(dependents_databases);
-                if (columns_mask[src_index++])
-                    res_columns[res_index++]->insert(dependents_tables);
-            }
-        }
-    }
-
-    UInt64 num_rows = res_columns.at(0)->size();
-    return Chunk(std::move(res_columns), num_rows);
-}
-
-Chunk DetachedTablesBlockSource::generate()
-{
-    if (done)
-        return {};
-
-    MutableColumns result_columns = getPort().getHeader().cloneEmptyColumns();
-
-    const auto access = context->getAccess();
-    const bool need_to_check_access_for_databases = !access->isGranted(AccessType::SHOW_TABLES);
-
-    for (size_t database_idx = 0, rows_count = 0; database_idx < databases->size() && rows_count < max_block_size; ++database_idx)
-    {
-        database_name = databases->getDataAt(database_idx).toString();
-        database = DatabaseCatalog::instance().tryGetDatabase(database_name);
-
-        if (!database)
-        {
-            LOG_DEBUG(lg, "Database was deleted just now or the user has no access");
-            continue;
-        }
-
-        const bool need_to_check_access_for_tables
-            = need_to_check_access_for_databases && !access->isGranted(AccessType::SHOW_TABLES, database_name);
-
-        if (!detached_tables_it || !detached_tables_it->isValid())
-            detached_tables_it = database->getDetachedTablesIterator(context, {}, false);
-
-        for (; rows_count < max_block_size && detached_tables_it->isValid(); detached_tables_it->next())
-        {
-            const auto detached_table_name = detached_tables_it->table();
-            LOG_DEBUG(lg, "detached_table_name={}", detached_table_name);
-
-            if (!detached_tables.contains(detached_table_name))
-                continue;
-
-            if (need_to_check_access_for_tables && !access->isGranted(AccessType::SHOW_TABLES, database_name, detached_table_name))
-                continue;
-
-            fillResultColumnsByDetachedTableIterator(result_columns);
-            ++rows_count;
-        }
-    }
-
-    const UInt64 num_rows = result_columns.at(0)->size();
-    done = true;
-    return Chunk(std::move(result_columns), num_rows);
-}
-
-void DetachedTablesBlockSource::fillResultColumnsByDetachedTableIterator(MutableColumns & result_columns) const
-{
-    size_t src_index = 0;
-    size_t res_index = 0;
-
-    if (columns_mask[src_index++])
-        result_columns[res_index++]->insert(detached_tables_it->database());
-
-    if (columns_mask[src_index++])
-        result_columns[res_index++]->insert(detached_tables_it->table());
-
-
-    if (columns_mask[src_index++])
-        result_columns[res_index++]->insert(detached_tables_it->uuid());
-
-    if (columns_mask[src_index++])
-        result_columns[res_index++]->insert(detached_tables_it->metadataPath());
-
-    if (columns_mask[src_index++])
-        result_columns[res_index++]->insert(detached_tables_it->isPermanently());
-}
-}
diff --git a/src/Storages/System/TablesBlockSource.h b/src/Storages/System/TablesBlockSource.h
deleted file mode 100644
index 9698fdd494c..00000000000
--- a/src/Storages/System/TablesBlockSource.h
+++ /dev/null
@@ -1,96 +0,0 @@
-#pragma once
-
-#include <Databases/IDatabase.h>
-#include <Interpreters/Context.h>
-#include <Processors/ISource.h>
-#include <Common/logger_useful.h>
-
-namespace DB
-{
-
-class TablesBlockSource : public ISource
-{
-public:
-    TablesBlockSource(
-        std::vector<UInt8> columns_mask_,
-        Block header,
-        UInt64 max_block_size_,
-        ColumnPtr databases_,
-        ColumnPtr tables_,
-        ContextPtr context_)
-        : ISource(std::move(header))
-        , columns_mask(std::move(columns_mask_))
-        , max_block_size(max_block_size_)
-        , databases(std::move(databases_))
-        , context(Context::createCopy(context_))
-    {
-        size_t size = tables_->size();
-        tables.reserve(size);
-        for (size_t idx = 0; idx < size; ++idx)
-            tables.insert(tables_->getDataAt(idx).toString());
-    }
-
-    String getName() const override { return "Tables"; }
-
-protected:
-    Chunk generate() override;
-
-private:
-    std::vector<UInt8> columns_mask;
-    UInt64 max_block_size;
-    ColumnPtr databases;
-    NameSet tables;
-    size_t database_idx = 0;
-    DatabaseTablesIteratorPtr tables_it;
-    ContextPtr context;
-    bool done = false;
-    DatabasePtr database;
-    std::string database_name;
-};
-
-class DetachedTablesBlockSource : public ISource
-{
-public:
-    DetachedTablesBlockSource(
-        std::vector<UInt8> columns_mask_,
-        Block header,
-        UInt64 max_block_size_,
-        ColumnPtr databases_,
-        ColumnPtr detached_tables_,
-        ContextPtr context_)
-        : ISource(std::move(header))
-        , columns_mask(std::move(columns_mask_))
-        , max_block_size(max_block_size_)
-        , databases(std::move(databases_))
-        , context(Context::createCopy(context_))
-    {
-        size_t size = detached_tables_->size();
-        detached_tables.reserve(size);
-        for (size_t idx = 0; idx < size; ++idx)
-        {
-            detached_tables.insert(detached_tables_->getDataAt(idx).toString());
-        }
-    }
-
-    String getName() const override { return "DetachedTables"; }
-
-protected:
-    Chunk generate() override;
-
-private:
-    const std::vector<UInt8> columns_mask;
-    const UInt64 max_block_size;
-    const ColumnPtr databases;
-    NameSet detached_tables;
-    DatabaseDetachedTablesSnapshotIteratorPtr detached_tables_it;
-    ContextPtr context;
-    bool done = false;
-    DatabasePtr database;
-    std::string database_name;
-
-    // temp log for debug
-    LoggerPtr lg = getLogger("DetachedTablesBlockSource");
-
-    void fillResultColumnsByDetachedTableIterator(MutableColumns & result_columns) const;
-};
-}
diff --git a/tests/integration/test_system_detached_tables/test.py b/tests/integration/test_system_detached_tables/test.py
index 2e0165deba2..02734d8bf14 100644
--- a/tests/integration/test_system_detached_tables/test.py
+++ b/tests/integration/test_system_detached_tables/test.py
@@ -24,15 +24,15 @@ def test_system_detached_tables():
     node.query("DETACH TABLE test_table")
     node.query("DETACH TABLE test_table_perm PERMANENTLY")
 
-    result = node.query("SELECT name FROM system.detached_tables")
+    result = node.query("SELECT table FROM system.detached_tables")
     assert result == "test_table\ntest_table_perm\n"
 
     node.restart_clickhouse()
 
-    result = node.query("SELECT name FROM system.detached_tables")
+    result = node.query("SELECT table FROM system.detached_tables")
     assert result == "test_table_perm\n"
 
     node.restart_clickhouse()
 
-    result = node.query("SELECT name FROM system.detached_tables")
+    result = node.query("SELECT table FROM system.detached_tables")
     assert result == "test_table_perm\n"
diff --git a/tests/queries/0_stateless/03172_system_detached_tables.sql b/tests/queries/0_stateless/03172_system_detached_tables.sql
index 13465d02b80..558dfe4cf18 100644
--- a/tests/queries/0_stateless/03172_system_detached_tables.sql
+++ b/tests/queries/0_stateless/03172_system_detached_tables.sql
@@ -7,7 +7,7 @@ CREATE TABLE test_system_detached_tables.test_table (n Int64) ENGINE=MergeTree O
 SELECT * FROM system.detached_tables;
 
 DETACH TABLE test_system_detached_tables.test_table;
-SELECT database, name, is_permanently FROM system.detached_tables;
+SELECT database, table, is_permanently FROM system.detached_tables;
 
 ATTACH TABLE test_system_detached_tables.test_table;
 
@@ -15,12 +15,12 @@ CREATE TABLE test_system_detached_tables.test_table_perm (n Int64) ENGINE=MergeT
 SELECT * FROM system.detached_tables;
 
 DETACH TABLE test_system_detached_tables.test_table_perm PERMANENTLY;
-SELECT database, name, is_permanently FROM system.detached_tables;
+SELECT database, table, is_permanently FROM system.detached_tables;
 
 DETACH TABLE test_system_detached_tables.test_table SYNC;
-SELECT database, name, is_permanently FROM system.detached_tables;
+SELECT database, table, is_permanently FROM system.detached_tables;
 
-SELECT database, name, is_permanently FROM system.detached_tables WHERE name='test_table';
+SELECT database, table, is_permanently FROM system.detached_tables WHERE table='test_table';
 
 DROP DATABASE test_system_detached_tables;
 

From 91dc9a69d844d781c7d4f94ca01d0a9bbe1a1f29 Mon Sep 17 00:00:00 2001
From: Nikita Taranov <nikita.taranov@clickhouse.com>
Date: Fri, 21 Jun 2024 22:03:38 +0100
Subject: [PATCH 0084/1488] fix final_big_column

---
 tests/performance/final_big_column.xml | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/tests/performance/final_big_column.xml b/tests/performance/final_big_column.xml
index 4bfdfdf804f..bc7c3570db6 100644
--- a/tests/performance/final_big_column.xml
+++ b/tests/performance/final_big_column.xml
@@ -1,7 +1,6 @@
 <test>
     <settings>
-        <!-- ingestion is quite time consuming in this test -->
-        <max_insert_threads>8</max_insert_threads>
+        <max_insert_threads>1</max_insert_threads>
         <max_memory_usage>20G</max_memory_usage>
     </settings>
 
@@ -11,8 +10,8 @@
         PARTITION BY toYYYYMM(d) ORDER BY key
     </create_query>
 
-    <fill_query>INSERT INTO optimized_select_final SELECT toDate('2000-01-01'), 2*number, randomPrintableASCII(1000) FROM numbers_mt(5000000)</fill_query>
-    <fill_query>INSERT INTO optimized_select_final SELECT toDate('2020-01-01'), 2*number+1, randomPrintableASCII(1000) FROM numbers_mt(5000000)</fill_query>
+    <fill_query>INSERT INTO optimized_select_final SELECT toDate('2000-01-01'), 2*number, randomPrintableASCII(1000) FROM numbers(1000000)</fill_query>
+    <fill_query>INSERT INTO optimized_select_final SELECT toDate('2020-01-01'), 2*number+1, randomPrintableASCII(1000) FROM numbers(1000000)</fill_query>
 
     <query>SELECT * FROM optimized_select_final FINAL FORMAT Null SETTINGS max_threads = 8</query>
     <query>SELECT * FROM optimized_select_final FINAL WHERE key % 10 = 0 FORMAT Null</query>

From cb0a692ba061d910712f0144cb2b5308db5d033f Mon Sep 17 00:00:00 2001
From: Nikita Taranov <nikita.taranov@clickhouse.com>
Date: Fri, 21 Jun 2024 23:42:56 +0100
Subject: [PATCH 0085/1488] fix read_from_comp_parts

---
 tests/performance/read_from_comp_parts.xml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tests/performance/read_from_comp_parts.xml b/tests/performance/read_from_comp_parts.xml
index a625075588a..055df51d12d 100644
--- a/tests/performance/read_from_comp_parts.xml
+++ b/tests/performance/read_from_comp_parts.xml
@@ -5,15 +5,16 @@
         ORDER BY (c1, c2)
         SETTINGS min_rows_for_wide_part = 1000000000 AS
         SELECT *
-        FROM generateRandom('c1 UInt32, c2 UInt64, s1 String, arr1 Array(UInt32), c3 UInt64, s2 String', 0, 30, 30)
+        FROM generateRandom('c1 UInt32, c2 UInt64, s1 String, arr1 Array(UInt32), c3 UInt64, s2 String', 0, 5, 6)
         LIMIT 50000000
+        SETTINGS max_insert_threads = 8
     </create_query>
 
     <settings>
         <max_threads>8</max_threads>
     </settings>
 
-    <query short="1">SELECT count() FROM mt_comp_parts WHERE NOT ignore(c1)</query>
+    <query short="1">SELECT count() FROM mt_comp_parts WHERE NOT ignore(s1)</query>
     <query>SELECT count() FROM mt_comp_parts WHERE NOT ignore(c2, s1, arr1, s2)</query>
     <query>SELECT count() FROM mt_comp_parts WHERE NOT ignore(c1, s1, c3)</query>
     <query>SELECT count() FROM mt_comp_parts WHERE NOT ignore(c1, c2, c3)</query>

From 15e20f56fa4a9a497e4cc247aa9215cb840203f9 Mon Sep 17 00:00:00 2001
From: JackyWoo <wuchienchao@qq.com>
Date: Fri, 21 Jun 2024 16:29:25 +0800
Subject: [PATCH 0086/1488] Add statistics cmsketch

---
 contrib/datasketches-cpp-cmake/CMakeLists.txt |  1 +
 .../statements/alter/statistics.md            |  2 +-
 src/CMakeLists.txt                            |  1 +
 .../Statistics/CMSketchStatistics.cpp         | 82 +++++++++++++++++++
 src/Storages/Statistics/CMSketchStatistics.h  | 39 +++++++++
 .../ConditionSelectivityEstimator.cpp         | 35 +++-----
 .../ConditionSelectivityEstimator.h           |  5 +-
 src/Storages/Statistics/Statistics.cpp        | 41 +++++++++-
 src/Storages/Statistics/Statistics.h          |  5 +-
 src/Storages/StatisticsDescription.cpp        |  4 +
 src/Storages/StatisticsDescription.h          |  1 +
 .../03174_statistics_cmsketch.reference       | 26 ++++++
 .../0_stateless/03174_statistics_cmsketch.sql | 78 ++++++++++++++++++
 13 files changed, 290 insertions(+), 30 deletions(-)
 create mode 100644 src/Storages/Statistics/CMSketchStatistics.cpp
 create mode 100644 src/Storages/Statistics/CMSketchStatistics.h
 create mode 100644 tests/queries/0_stateless/03174_statistics_cmsketch.reference
 create mode 100644 tests/queries/0_stateless/03174_statistics_cmsketch.sql

diff --git a/contrib/datasketches-cpp-cmake/CMakeLists.txt b/contrib/datasketches-cpp-cmake/CMakeLists.txt
index b12a88ad57b..497d6956d0e 100644
--- a/contrib/datasketches-cpp-cmake/CMakeLists.txt
+++ b/contrib/datasketches-cpp-cmake/CMakeLists.txt
@@ -9,6 +9,7 @@ set(DATASKETCHES_LIBRARY theta)
 add_library(_datasketches INTERFACE)
 target_include_directories(_datasketches SYSTEM BEFORE INTERFACE
     "${ClickHouse_SOURCE_DIR}/contrib/datasketches-cpp/common/include"
+    "${ClickHouse_SOURCE_DIR}/contrib/datasketches-cpp/count/include"
     "${ClickHouse_SOURCE_DIR}/contrib/datasketches-cpp/theta/include")
 
 add_library(ch_contrib::datasketches ALIAS _datasketches)
diff --git a/docs/en/sql-reference/statements/alter/statistics.md b/docs/en/sql-reference/statements/alter/statistics.md
index 80024781f88..22ff740d410 100644
--- a/docs/en/sql-reference/statements/alter/statistics.md
+++ b/docs/en/sql-reference/statements/alter/statistics.md
@@ -25,7 +25,7 @@ Also, they are replicated, syncing statistics metadata via ZooKeeper.
 There is an example adding two statistics types to two columns:
 
 ```
-ALTER TABLE t1 MODIFY STATISTICS c, d TYPE TDigest, Uniq;
+ALTER TABLE t1 MODIFY STATISTICS c, d TYPE TDigest, Uniq, CMSketch;
 ```
 
 :::note    
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 84aaec17a5b..4efb6004172 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -546,6 +546,7 @@ endif()
 
 if (TARGET ch_contrib::datasketches)
     target_link_libraries (clickhouse_aggregate_functions PRIVATE ch_contrib::datasketches)
+    dbms_target_link_libraries(PRIVATE ch_contrib::datasketches)
 endif ()
 
 target_link_libraries (clickhouse_common_io PRIVATE ch_contrib::lz4)
diff --git a/src/Storages/Statistics/CMSketchStatistics.cpp b/src/Storages/Statistics/CMSketchStatistics.cpp
new file mode 100644
index 00000000000..2c217d30278
--- /dev/null
+++ b/src/Storages/Statistics/CMSketchStatistics.cpp
@@ -0,0 +1,82 @@
+#include <DataTypes/DataTypeNullable.h>
+#include <DataTypes/DataTypeLowCardinality.h>
+#include <IO/ReadHelpers.h>
+#include <IO/WriteHelpers.h>
+#include <Storages/Statistics/CMSketchStatistics.h>
+
+#if USE_DATASKETCHES
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+extern const int ILLEGAL_STATISTICS;
+}
+
+
+CMSketchStatistics::CMSketchStatistics(const SingleStatisticsDescription & stat_, DataTypePtr data_type_)
+    : IStatistics(stat_), data(CMSKETCH_HASH_COUNT, CMSKETCH_BUCKET_COUNT), data_type(data_type_)
+{
+}
+
+Float64 CMSketchStatistics::estimateEqual(const Field & value) const
+{
+    if (auto float_val = getFloat64(value))
+        return data.get_estimate(&float_val.value(), 8);
+    if (auto string_val = getString(value))
+        return data.get_estimate(string_val->data(), string_val->size());
+    UNREACHABLE();
+}
+
+void CMSketchStatistics::serialize(WriteBuffer & buf)
+{
+    auto bytes = data.serialize();
+    writeIntBinary(static_cast<UInt64>(bytes.size()), buf);
+    buf.write(reinterpret_cast<const char *>(bytes.data()), bytes.size());
+}
+
+void CMSketchStatistics::deserialize(ReadBuffer & buf)
+{
+    UInt64 size;
+    readIntBinary(size, buf);
+    String s;
+    s.reserve(size);
+    buf.readStrict(s.data(), size); /// Extra copy can be avoided by implementing count_min_sketch<Float64>::deserialize with ReadBuffer
+    auto read_sketch = datasketches::count_min_sketch<Float64>::deserialize(s.data(), size, datasketches::DEFAULT_SEED);
+    data.merge(read_sketch);
+}
+
+void CMSketchStatistics::update(const ColumnPtr & column)
+{
+    size_t size = column->size();
+
+    for (size_t i = 0; i < size; ++i)
+    {
+        Field f;
+        column->get(i, f);
+        if (f.isNull())
+            continue;
+        if (auto float_val = getFloat64(f))
+            data.update(&float_val.value(), 8, 1.0);
+        else if (auto string_val = getString(f))
+            data.update(*string_val, 1.0);
+    }
+}
+
+void CMSketchValidator(const SingleStatisticsDescription &, DataTypePtr data_type)
+{
+    data_type = removeNullable(data_type);
+    data_type = removeLowCardinalityAndNullable(data_type);
+    if (!data_type->isValueRepresentedByNumber() && !isStringOrFixedString(data_type))
+        throw Exception(ErrorCodes::ILLEGAL_STATISTICS, "Statistics of type 'cmsketch' does not support type {}", data_type->getName());
+}
+
+StatisticsPtr CMSketchCreator(const SingleStatisticsDescription & stat, DataTypePtr data_type)
+{
+    return std::make_shared<CMSketchStatistics>(stat, data_type);
+}
+
+}
+
+#endif
diff --git a/src/Storages/Statistics/CMSketchStatistics.h b/src/Storages/Statistics/CMSketchStatistics.h
new file mode 100644
index 00000000000..03964614d57
--- /dev/null
+++ b/src/Storages/Statistics/CMSketchStatistics.h
@@ -0,0 +1,39 @@
+#pragma once
+
+#if USE_DATASKETCHES
+
+#include <Storages/Statistics/Statistics.h>
+#include <count_min.hpp>
+#include <Common/Allocator.h>
+
+namespace DB
+{
+
+/// CMSketchStatistics is used to estimate expression like col = 'value' or col in ('v1', 'v2').
+class CMSketchStatistics : public IStatistics
+{
+public:
+    explicit CMSketchStatistics(const SingleStatisticsDescription & stat_, DataTypePtr data_type_);
+
+    Float64 estimateEqual(const Field & value) const;
+
+    void serialize(WriteBuffer & buf) override;
+
+    void deserialize(ReadBuffer & buf) override;
+
+    void update(const ColumnPtr & column) override;
+
+private:
+    static constexpr size_t CMSKETCH_HASH_COUNT = 8;
+    static constexpr size_t CMSKETCH_BUCKET_COUNT = 2048;
+
+    datasketches::count_min_sketch<Float64> data;
+    DataTypePtr data_type;
+};
+
+StatisticsPtr CMSketchCreator(const SingleStatisticsDescription & stat, DataTypePtr);
+void CMSketchValidator(const SingleStatisticsDescription &, DataTypePtr data_type);
+
+}
+
+#endif
diff --git a/src/Storages/Statistics/ConditionSelectivityEstimator.cpp b/src/Storages/Statistics/ConditionSelectivityEstimator.cpp
index 757136fdf42..73c5c549a5d 100644
--- a/src/Storages/Statistics/ConditionSelectivityEstimator.cpp
+++ b/src/Storages/Statistics/ConditionSelectivityEstimator.cpp
@@ -35,11 +35,14 @@ Float64 ConditionSelectivityEstimator::ColumnSelectivityEstimator::estimateGreat
     return rows - estimateLess(val, rows);
 }
 
-Float64 ConditionSelectivityEstimator::ColumnSelectivityEstimator::estimateEqual(Float64 val, Float64 rows) const
+Float64 ConditionSelectivityEstimator::ColumnSelectivityEstimator::estimateEqual(Field val, Float64 rows) const
 {
+    auto float_val = getFloat64(val);
     if (part_statistics.empty())
     {
-        if (val < - threshold || val > threshold)
+        if (!float_val)
+            return default_unknown_cond_factor * rows;
+        else if (float_val.value() < - threshold || float_val.value() > threshold)
             return default_normal_cond_factor * rows;
         else
             return default_good_cond_factor * rows;
@@ -87,7 +90,7 @@ static std::pair<String, Int32> tryToExtractSingleColumn(const RPNBuilderTreeNod
     return result;
 }
 
-std::pair<String, Float64> ConditionSelectivityEstimator::extractBinaryOp(const RPNBuilderTreeNode & node, const String & column_name) const
+std::pair<String, Field> ConditionSelectivityEstimator::extractBinaryOp(const RPNBuilderTreeNode & node, const String & column_name) const
 {
     if (!node.isFunction())
         return {};
@@ -123,18 +126,7 @@ std::pair<String, Float64> ConditionSelectivityEstimator::extractBinaryOp(const
     DataTypePtr output_type;
     if (!constant_node->tryGetConstant(output_value, output_type))
         return {};
-
-    const auto type = output_value.getType();
-    Float64 value;
-    if (type == Field::Types::Int64)
-        value = output_value.get<Int64>();
-    else if (type == Field::Types::UInt64)
-        value = output_value.get<UInt64>();
-    else if (type == Field::Types::Float64)
-        value = output_value.get<Float64>();
-    else
-        return {};
-    return std::make_pair(function_name, value);
+    return std::make_pair(function_name, output_value);
 }
 
 Float64 ConditionSelectivityEstimator::estimateRowCount(const RPNBuilderTreeNode & node) const
@@ -142,7 +134,7 @@ Float64 ConditionSelectivityEstimator::estimateRowCount(const RPNBuilderTreeNode
     auto result = tryToExtractSingleColumn(node);
     if (result.second != 1)
     {
-        return default_unknown_cond_factor;
+        return default_unknown_cond_factor * total_rows;
     }
     String col = result.first;
     auto it = column_estimators.find(col);
@@ -152,19 +144,16 @@ Float64 ConditionSelectivityEstimator::estimateRowCount(const RPNBuilderTreeNode
     bool dummy = total_rows == 0;
     ColumnSelectivityEstimator estimator;
     if (it != column_estimators.end())
-    {
         estimator = it->second;
-    }
     else
-    {
         dummy = true;
-    }
     auto [op, val] = extractBinaryOp(node, col);
+    auto float_val = getFloat64(val);
     if (op == "equals")
     {
         if (dummy)
         {
-            if (val < - threshold || val > threshold)
+            if (!float_val || (float_val < - threshold || float_val > threshold))
                 return default_normal_cond_factor * total_rows;
             else
                 return default_good_cond_factor * total_rows;
@@ -175,13 +164,13 @@ Float64 ConditionSelectivityEstimator::estimateRowCount(const RPNBuilderTreeNode
     {
         if (dummy)
             return default_normal_cond_factor * total_rows;
-        return estimator.estimateLess(val, total_rows);
+        return estimator.estimateLess(float_val.value(), total_rows);
     }
     else if (op == "greater" || op == "greaterOrEquals")
     {
         if (dummy)
             return default_normal_cond_factor * total_rows;
-        return estimator.estimateGreater(val, total_rows);
+        return estimator.estimateGreater(float_val.value(), total_rows);
     }
     else
         return default_unknown_cond_factor * total_rows;
diff --git a/src/Storages/Statistics/ConditionSelectivityEstimator.h b/src/Storages/Statistics/ConditionSelectivityEstimator.h
index f0599742276..9bf4940e563 100644
--- a/src/Storages/Statistics/ConditionSelectivityEstimator.h
+++ b/src/Storages/Statistics/ConditionSelectivityEstimator.h
@@ -1,6 +1,7 @@
 #pragma once
 
 #include <Storages/Statistics/Statistics.h>
+#include <Core/Field.h>
 
 namespace DB
 {
@@ -24,7 +25,7 @@ private:
 
         Float64 estimateGreater(Float64 val, Float64 rows) const;
 
-        Float64 estimateEqual(Float64 val, Float64 rows) const;
+        Float64 estimateEqual(Field val, Float64 rows) const;
     };
 
     static constexpr auto default_good_cond_factor = 0.1;
@@ -37,7 +38,7 @@ private:
     UInt64 total_rows = 0;
     std::set<String> part_names;
     std::map<String, ColumnSelectivityEstimator> column_estimators;
-    std::pair<String, Float64> extractBinaryOp(const RPNBuilderTreeNode & node, const String & column_name) const;
+    std::pair<String, Field> extractBinaryOp(const RPNBuilderTreeNode & node, const String & column_name) const;
 
 public:
     /// TODO: Support the condition consists of CNF/DNF like (cond1 and cond2) or (cond3) ...
diff --git a/src/Storages/Statistics/Statistics.cpp b/src/Storages/Statistics/Statistics.cpp
index fed0bd61c03..b35d653b7ec 100644
--- a/src/Storages/Statistics/Statistics.cpp
+++ b/src/Storages/Statistics/Statistics.cpp
@@ -5,6 +5,7 @@
 #include <Storages/Statistics/ConditionSelectivityEstimator.h>
 #include <Storages/Statistics/TDigestStatistics.h>
 #include <Storages/Statistics/UniqStatistics.h>
+#include <Storages/Statistics/CMSketchStatistics.h>
 #include <Storages/StatisticsDescription.h>
 #include <Storages/ColumnsDescription.h>
 #include <IO/ReadHelpers.h>
@@ -26,6 +27,28 @@ enum StatisticsFileVersion : UInt16
     V0 = 0,
 };
 
+std::optional<Float64> getFloat64(const Field & f)
+{
+    const auto type = f.getType();
+    Float64 value;
+    if (type == Field::Types::Int64)
+        value = f.get<Int64>();
+    else if (type == Field::Types::UInt64)
+        value = f.get<UInt64>();
+    else if (type == Field::Types::Float64)
+        value = f.get<Float64>();
+    else
+        return {};
+    return value;
+}
+
+std::optional<String> getString(const Field & f)
+{
+    if (f.getType() == Field::Types::String)
+        return f.get<String>();
+    return {};
+}
+
 IStatistics::IStatistics(const SingleStatisticsDescription & stat_) : stat(stat_) {}
 
 ColumnStatistics::ColumnStatistics(const ColumnStatisticsDescription & stats_desc_)
@@ -54,9 +77,10 @@ Float64 ColumnStatistics::estimateGreater(Float64 val) const
     return rows - estimateLess(val);
 }
 
-Float64 ColumnStatistics::estimateEqual(Float64 val) const
+Float64 ColumnStatistics::estimateEqual(Field val) const
 {
-    if (stats.contains(StatisticsType::Uniq) && stats.contains(StatisticsType::TDigest))
+    auto float_val = getFloat64(val);
+    if (float_val && stats.contains(StatisticsType::Uniq) && stats.contains(StatisticsType::TDigest))
     {
         auto uniq_static = std::static_pointer_cast<UniqStatistics>(stats.at(StatisticsType::Uniq));
         /// 2048 is the default number of buckets in TDigest. In this case, TDigest stores exactly one value (with many rows)
@@ -64,9 +88,16 @@ Float64 ColumnStatistics::estimateEqual(Float64 val) const
         if (uniq_static->getCardinality() < 2048)
         {
             auto tdigest_static = std::static_pointer_cast<TDigestStatistics>(stats.at(StatisticsType::TDigest));
-            return tdigest_static->estimateEqual(val);
+            return tdigest_static->estimateEqual(float_val.value());
         }
     }
+#if USE_DATASKETCHES
+    if (stats.contains(StatisticsType::CMSketch))
+    {
+        auto cmsketch_static = std::static_pointer_cast<CMSketchStatistics>(stats.at(StatisticsType::CMSketch));
+        return cmsketch_static->estimateEqual(val);
+    }
+#endif
     if (val < - ConditionSelectivityEstimator::threshold || val > ConditionSelectivityEstimator::threshold)
         return rows * ConditionSelectivityEstimator::default_normal_cond_factor;
     else
@@ -145,6 +176,10 @@ MergeTreeStatisticsFactory::MergeTreeStatisticsFactory()
     registerCreator(StatisticsType::Uniq, UniqCreator);
     registerValidator(StatisticsType::TDigest, TDigestValidator);
     registerValidator(StatisticsType::Uniq, UniqValidator);
+#if USE_DATASKETCHES
+    registerCreator(StatisticsType::CMSketch, CMSketchCreator);
+    registerValidator(StatisticsType::CMSketch, CMSketchValidator);
+#endif
 }
 
 MergeTreeStatisticsFactory & MergeTreeStatisticsFactory::instance()
diff --git a/src/Storages/Statistics/Statistics.h b/src/Storages/Statistics/Statistics.h
index 2ab1337af02..f6121d72256 100644
--- a/src/Storages/Statistics/Statistics.h
+++ b/src/Storages/Statistics/Statistics.h
@@ -7,6 +7,7 @@
 #include <Common/logger_useful.h>
 #include <IO/ReadBuffer.h>
 #include <IO/WriteBuffer.h>
+#include <Core/Field.h>
 #include <Storages/StatisticsDescription.h>
 
 
@@ -58,7 +59,7 @@ public:
 
     Float64 estimateGreater(Float64 val) const;
 
-    Float64 estimateEqual(Float64 val) const;
+    Float64 estimateEqual(Field val) const;
 
 private:
 
@@ -100,4 +101,6 @@ private:
     Validators validators;
 };
 
+std::optional<Float64> getFloat64(const Field & f);
+std::optional<String> getString(const Field & f);
 }
diff --git a/src/Storages/StatisticsDescription.cpp b/src/Storages/StatisticsDescription.cpp
index f10fb78f933..08c79043ac4 100644
--- a/src/Storages/StatisticsDescription.cpp
+++ b/src/Storages/StatisticsDescription.cpp
@@ -54,6 +54,8 @@ static StatisticsType stringToStatisticsType(String type)
         return StatisticsType::TDigest;
     if (type == "uniq")
         return StatisticsType::Uniq;
+    if (type == "cmsketch")
+        return StatisticsType::CMSketch;
     throw Exception(ErrorCodes::INCORRECT_QUERY, "Unknown statistics type: {}. Supported statistics types are `tdigest` and `uniq`.", type);
 }
 
@@ -65,6 +67,8 @@ String SingleStatisticsDescription::getTypeName() const
             return "TDigest";
         case StatisticsType::Uniq:
             return "Uniq";
+        case StatisticsType::CMSketch:
+            return "CMSketch";
         default:
             throw Exception(ErrorCodes::LOGICAL_ERROR, "Unknown statistics type: {}. Supported statistics types are `tdigest` and `uniq`.", type);
     }
diff --git a/src/Storages/StatisticsDescription.h b/src/Storages/StatisticsDescription.h
index 4862fb79d45..a2005f59de1 100644
--- a/src/Storages/StatisticsDescription.h
+++ b/src/Storages/StatisticsDescription.h
@@ -13,6 +13,7 @@ enum class StatisticsType : UInt8
 {
     TDigest = 0,
     Uniq = 1,
+    CMSketch = 2,
 
     Max = 63,
 };
diff --git a/tests/queries/0_stateless/03174_statistics_cmsketch.reference b/tests/queries/0_stateless/03174_statistics_cmsketch.reference
new file mode 100644
index 00000000000..ea77f317a31
--- /dev/null
+++ b/tests/queries/0_stateless/03174_statistics_cmsketch.reference
@@ -0,0 +1,26 @@
+CREATE TABLE default.t1\n(\n    `a` String STATISTICS(cmsketch),\n    `b` Int64 STATISTICS(cmsketch),\n    `c` UInt64 STATISTICS(cmsketch),\n    `pk` String\n)\nENGINE = MergeTree\nORDER BY pk\nSETTINGS min_bytes_for_wide_part = 0, index_granularity = 8192
+        Prewhere info
+          Prewhere filter
+          Prewhere filter column: and(equals(a, \'0\'_String), equals(b, 0), equals(c, 0)) (removed)
+        Prewhere info
+          Prewhere filter
+          Prewhere filter column: and(equals(a, \'0\'_String), equals(c, 0), greater(b, 0)) (removed)
+After drop statistics for a
+        Prewhere info
+          Prewhere filter
+          Prewhere filter column: and(equals(b, 0), equals(c, 0), equals(a, \'0\'_String)) (removed)
+        Prewhere info
+          Prewhere filter
+          Prewhere filter column: and(equals(c, 0), equals(a, \'0\'_String), greater(b, 0)) (removed)
+LowCardinality
+        Prewhere info
+          Prewhere filter
+          Prewhere filter column: and(equals(a, \'0\'_String), equals(b, 0), equals(c, 0)) (removed)
+Nullable
+        Prewhere info
+          Prewhere filter
+          Prewhere filter column: and(equals(a, \'0\'_String), equals(b, 0), equals(c, 0)) (removed)
+LowCardinality(Nullable)
+        Prewhere info
+          Prewhere filter
+          Prewhere filter column: and(equals(a, \'0\'_String), equals(b, 0), equals(c, 0)) (removed)
diff --git a/tests/queries/0_stateless/03174_statistics_cmsketch.sql b/tests/queries/0_stateless/03174_statistics_cmsketch.sql
new file mode 100644
index 00000000000..c45d6186fdf
--- /dev/null
+++ b/tests/queries/0_stateless/03174_statistics_cmsketch.sql
@@ -0,0 +1,78 @@
+-- Tags: no-fasttest
+DROP TABLE IF EXISTS t1;
+
+SET allow_experimental_statistics = 1;
+SET allow_statistics_optimize = 1;
+
+CREATE TABLE t1
+(
+    a String STATISTICS(cmsketch),
+    b Int64 STATISTICS(cmsketch),
+    c UInt64 STATISTICS(cmsketch),
+    pk String,
+) Engine = MergeTree() ORDER BY pk
+SETTINGS min_bytes_for_wide_part = 0;
+
+SHOW CREATE TABLE t1;
+
+INSERT INTO t1 select toString(number % 1000), number % 100, number % 10, generateUUIDv4() FROM system.numbers LIMIT 10000;
+
+SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE c = 0 and b = 0 and a = '0') WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
+SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE c = 0 and b > 0 and a = '0') WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
+
+ALTER TABLE t1 DROP STATISTICS a;
+
+SELECT 'After drop statistics for a';
+SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE c = 0 and b = 0 and a = '0') WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
+SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE c = 0 and b > 0 and a = '0') WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
+
+DROP TABLE IF EXISTS t1;
+DROP TABLE IF EXISTS t2;
+SET allow_suspicious_low_cardinality_types=1;
+CREATE TABLE t2
+(
+    a LowCardinality(String) STATISTICS(cmsketch),
+    b Int64 STATISTICS(cmsketch),
+    c UInt64 STATISTICS(cmsketch),
+    pk String,
+) Engine = MergeTree() ORDER BY pk
+SETTINGS min_bytes_for_wide_part = 0;
+INSERT INTO t2 select toString(number % 1000), number % 100, number % 10, generateUUIDv4() FROM system.numbers LIMIT 10000;
+
+SELECT 'LowCardinality';
+SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t2 WHERE c = 0 and b = 0 and a = '0') WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
+
+
+DROP TABLE IF EXISTS t2;
+DROP TABLE IF EXISTS t3;
+
+CREATE TABLE t3
+(
+    a Nullable(String) STATISTICS(cmsketch),
+    b Int64 STATISTICS(cmsketch),
+    c UInt64 STATISTICS(cmsketch),
+    pk String,
+) Engine = MergeTree() ORDER BY pk
+SETTINGS min_bytes_for_wide_part = 0;
+INSERT INTO t3 select toString(number % 1000), number % 100, number % 10, generateUUIDv4() FROM system.numbers LIMIT 10000;
+
+SELECT 'Nullable';
+SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t3 WHERE c = 0 and b = 0 and a = '0') WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
+
+DROP TABLE IF EXISTS t3;
+DROP TABLE IF EXISTS t4;
+
+CREATE TABLE t4
+(
+    a LowCardinality(Nullable(String)) STATISTICS(cmsketch),
+    b Int64 STATISTICS(cmsketch),
+    c UInt64 STATISTICS(cmsketch),
+    pk String,
+) Engine = MergeTree() ORDER BY pk
+SETTINGS min_bytes_for_wide_part = 0;
+INSERT INTO t4 select toString(number % 1000), number % 100, number % 10, generateUUIDv4() FROM system.numbers LIMIT 10000;
+
+SELECT 'LowCardinality(Nullable)';
+SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t4 WHERE c = 0 and b = 0 and a = '0') WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
+
+DROP TABLE IF EXISTS t4;

From 00f74905a769d06a451c6472a0f461b25266b3be Mon Sep 17 00:00:00 2001
From: Konstantin Morozov <just.morozov.k@gmail.com>
Date: Mon, 24 Jun 2024 09:07:23 +0000
Subject: [PATCH 0087/1488] small up

---
 src/Databases/DatabaseOrdinary.cpp | 5 ++++-
 src/Databases/IDatabase.h          | 2 +-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp
index 7086f6e628e..3975eb46a4b 100644
--- a/src/Databases/DatabaseOrdinary.cpp
+++ b/src/Databases/DatabaseOrdinary.cpp
@@ -244,7 +244,10 @@ void DatabaseOrdinary::loadTablesMetadata(ContextPtr local_context, ParsedTables
                     SnapshotDetachedTable snapshot_detached_table;
                     snapshot_detached_table.database = storage_id.getDatabaseName();
                     snapshot_detached_table.table = detached_table_name;
-                    snapshot_detached_table.uuid = storage_id.uuid;
+                    if (storage_id.hasUUID())
+                    {
+                        snapshot_detached_table.uuid = storage_id.uuid;
+                    }
                     snapshot_detached_table.is_permanently = true;
                     snapshot_detached_table.metadata_path = getObjectMetadataPath(snapshot_detached_table.table);
 
diff --git a/src/Databases/IDatabase.h b/src/Databases/IDatabase.h
index 2ce4650798c..76d25908fda 100644
--- a/src/Databases/IDatabase.h
+++ b/src/Databases/IDatabase.h
@@ -286,7 +286,7 @@ public:
     virtual DatabaseDetachedTablesSnapshotIteratorPtr getDetachedTablesIterator(
         ContextPtr /*context*/, const FilterByNameFunction & /*filter_by_table_name = {}*/, bool /*skip_not_loaded = false*/) const
     {
-        throw Exception(ErrorCodes::NOT_IMPLEMENTED, "There is no get detached tables for Database{}", getEngineName());
+        throw Exception(ErrorCodes::NOT_IMPLEMENTED, "There is no get detached tables for Database {}", getEngineName());
     }
 
     /// Returns list of table names.

From a9331c3bcdb6911497fca06c0db5ebb1afd7b2a7 Mon Sep 17 00:00:00 2001
From: JackyWoo <wuchienchao@qq.com>
Date: Mon, 24 Jun 2024 18:26:38 +0800
Subject: [PATCH 0088/1488] Rname cmsketch to countmixsketch

---
 .../statements/alter/statistics.md            |  2 +-
 src/Storages/Statistics/CMSketchStatistics.h  | 39 -------------------
 ...stics.cpp => CountMinSketchStatistics.cpp} | 24 ++++++------
 .../Statistics/CountMinSketchStatistics.h     | 39 +++++++++++++++++++
 src/Storages/Statistics/Statistics.cpp        | 24 ++++++------
 src/Storages/StatisticsDescription.cpp        |  8 ++--
 src/Storages/StatisticsDescription.h          |  2 +-
 ...03174_statistics_countminsketch.reference} |  2 +-
 ...ql => 03174_statistics_countminsketch.sql} | 24 ++++++------
 9 files changed, 82 insertions(+), 82 deletions(-)
 delete mode 100644 src/Storages/Statistics/CMSketchStatistics.h
 rename src/Storages/Statistics/{CMSketchStatistics.cpp => CountMinSketchStatistics.cpp} (66%)
 create mode 100644 src/Storages/Statistics/CountMinSketchStatistics.h
 rename tests/queries/0_stateless/{03174_statistics_cmsketch.reference => 03174_statistics_countminsketch.reference} (80%)
 rename tests/queries/0_stateless/{03174_statistics_cmsketch.sql => 03174_statistics_countminsketch.sql} (84%)

diff --git a/docs/en/sql-reference/statements/alter/statistics.md b/docs/en/sql-reference/statements/alter/statistics.md
index 22ff740d410..0d1fa59cf86 100644
--- a/docs/en/sql-reference/statements/alter/statistics.md
+++ b/docs/en/sql-reference/statements/alter/statistics.md
@@ -25,7 +25,7 @@ Also, they are replicated, syncing statistics metadata via ZooKeeper.
 There is an example adding two statistics types to two columns:
 
 ```
-ALTER TABLE t1 MODIFY STATISTICS c, d TYPE TDigest, Uniq, CMSketch;
+ALTER TABLE t1 MODIFY STATISTICS c, d TYPE TDigest, Uniq, CountMinSketch;
 ```
 
 :::note    
diff --git a/src/Storages/Statistics/CMSketchStatistics.h b/src/Storages/Statistics/CMSketchStatistics.h
deleted file mode 100644
index 03964614d57..00000000000
--- a/src/Storages/Statistics/CMSketchStatistics.h
+++ /dev/null
@@ -1,39 +0,0 @@
-#pragma once
-
-#if USE_DATASKETCHES
-
-#include <Storages/Statistics/Statistics.h>
-#include <count_min.hpp>
-#include <Common/Allocator.h>
-
-namespace DB
-{
-
-/// CMSketchStatistics is used to estimate expression like col = 'value' or col in ('v1', 'v2').
-class CMSketchStatistics : public IStatistics
-{
-public:
-    explicit CMSketchStatistics(const SingleStatisticsDescription & stat_, DataTypePtr data_type_);
-
-    Float64 estimateEqual(const Field & value) const;
-
-    void serialize(WriteBuffer & buf) override;
-
-    void deserialize(ReadBuffer & buf) override;
-
-    void update(const ColumnPtr & column) override;
-
-private:
-    static constexpr size_t CMSKETCH_HASH_COUNT = 8;
-    static constexpr size_t CMSKETCH_BUCKET_COUNT = 2048;
-
-    datasketches::count_min_sketch<Float64> data;
-    DataTypePtr data_type;
-};
-
-StatisticsPtr CMSketchCreator(const SingleStatisticsDescription & stat, DataTypePtr);
-void CMSketchValidator(const SingleStatisticsDescription &, DataTypePtr data_type);
-
-}
-
-#endif
diff --git a/src/Storages/Statistics/CMSketchStatistics.cpp b/src/Storages/Statistics/CountMinSketchStatistics.cpp
similarity index 66%
rename from src/Storages/Statistics/CMSketchStatistics.cpp
rename to src/Storages/Statistics/CountMinSketchStatistics.cpp
index 2c217d30278..d0f52e3f6df 100644
--- a/src/Storages/Statistics/CMSketchStatistics.cpp
+++ b/src/Storages/Statistics/CountMinSketchStatistics.cpp
@@ -1,8 +1,8 @@
-#include <DataTypes/DataTypeNullable.h>
 #include <DataTypes/DataTypeLowCardinality.h>
+#include <DataTypes/DataTypeNullable.h>
 #include <IO/ReadHelpers.h>
 #include <IO/WriteHelpers.h>
-#include <Storages/Statistics/CMSketchStatistics.h>
+#include <Storages/Statistics/CountMinSketchStatistics.h>
 
 #if USE_DATASKETCHES
 
@@ -15,12 +15,12 @@ extern const int ILLEGAL_STATISTICS;
 }
 
 
-CMSketchStatistics::CMSketchStatistics(const SingleStatisticsDescription & stat_, DataTypePtr data_type_)
-    : IStatistics(stat_), data(CMSKETCH_HASH_COUNT, CMSKETCH_BUCKET_COUNT), data_type(data_type_)
+CountMinSketchStatistics::CountMinSketchStatistics(const SingleStatisticsDescription & stat_, DataTypePtr data_type_)
+    : IStatistics(stat_), data(HASH_COUNT, BUCKET_COUNT), data_type(data_type_)
 {
 }
 
-Float64 CMSketchStatistics::estimateEqual(const Field & value) const
+Float64 CountMinSketchStatistics::estimateEqual(const Field & value) const
 {
     if (auto float_val = getFloat64(value))
         return data.get_estimate(&float_val.value(), 8);
@@ -29,14 +29,14 @@ Float64 CMSketchStatistics::estimateEqual(const Field & value) const
     UNREACHABLE();
 }
 
-void CMSketchStatistics::serialize(WriteBuffer & buf)
+void CountMinSketchStatistics::serialize(WriteBuffer & buf)
 {
     auto bytes = data.serialize();
     writeIntBinary(static_cast<UInt64>(bytes.size()), buf);
     buf.write(reinterpret_cast<const char *>(bytes.data()), bytes.size());
 }
 
-void CMSketchStatistics::deserialize(ReadBuffer & buf)
+void CountMinSketchStatistics::deserialize(ReadBuffer & buf)
 {
     UInt64 size;
     readIntBinary(size, buf);
@@ -47,7 +47,7 @@ void CMSketchStatistics::deserialize(ReadBuffer & buf)
     data.merge(read_sketch);
 }
 
-void CMSketchStatistics::update(const ColumnPtr & column)
+void CountMinSketchStatistics::update(const ColumnPtr & column)
 {
     size_t size = column->size();
 
@@ -64,17 +64,17 @@ void CMSketchStatistics::update(const ColumnPtr & column)
     }
 }
 
-void CMSketchValidator(const SingleStatisticsDescription &, DataTypePtr data_type)
+void CountMinSketchValidator(const SingleStatisticsDescription &, DataTypePtr data_type)
 {
     data_type = removeNullable(data_type);
     data_type = removeLowCardinalityAndNullable(data_type);
     if (!data_type->isValueRepresentedByNumber() && !isStringOrFixedString(data_type))
-        throw Exception(ErrorCodes::ILLEGAL_STATISTICS, "Statistics of type 'cmsketch' does not support type {}", data_type->getName());
+        throw Exception(ErrorCodes::ILLEGAL_STATISTICS, "Statistics of type 'countminsketch' does not support type {}", data_type->getName());
 }
 
-StatisticsPtr CMSketchCreator(const SingleStatisticsDescription & stat, DataTypePtr data_type)
+StatisticsPtr CountMinSketchCreator(const SingleStatisticsDescription & stat, DataTypePtr data_type)
 {
-    return std::make_shared<CMSketchStatistics>(stat, data_type);
+    return std::make_shared<CountMinSketchStatistics>(stat, data_type);
 }
 
 }
diff --git a/src/Storages/Statistics/CountMinSketchStatistics.h b/src/Storages/Statistics/CountMinSketchStatistics.h
new file mode 100644
index 00000000000..23ea2cf25c9
--- /dev/null
+++ b/src/Storages/Statistics/CountMinSketchStatistics.h
@@ -0,0 +1,39 @@
+#pragma once
+
+#if USE_DATASKETCHES
+
+#include <Storages/Statistics/Statistics.h>
+#include <count_min.hpp>
+#include <Common/Allocator.h>
+
+namespace DB
+{
+
+/// CountMinSketchStatistics is used to estimate expression like col = 'value' or col in ('v1', 'v2').
+class CountMinSketchStatistics : public IStatistics
+{
+public:
+    explicit CountMinSketchStatistics(const SingleStatisticsDescription & stat_, DataTypePtr data_type_);
+
+    Float64 estimateEqual(const Field & value) const;
+
+    void serialize(WriteBuffer & buf) override;
+
+    void deserialize(ReadBuffer & buf) override;
+
+    void update(const ColumnPtr & column) override;
+
+private:
+    static constexpr size_t HASH_COUNT = 8;
+    static constexpr size_t BUCKET_COUNT = 2048;
+
+    datasketches::count_min_sketch<Float64> data;
+    DataTypePtr data_type;
+};
+
+StatisticsPtr CountMinSketchCreator(const SingleStatisticsDescription & stat, DataTypePtr);
+void CountMinSketchValidator(const SingleStatisticsDescription &, DataTypePtr data_type);
+
+}
+
+#endif
diff --git a/src/Storages/Statistics/Statistics.cpp b/src/Storages/Statistics/Statistics.cpp
index b35d653b7ec..2404d234d7a 100644
--- a/src/Storages/Statistics/Statistics.cpp
+++ b/src/Storages/Statistics/Statistics.cpp
@@ -1,15 +1,15 @@
 #include <optional>
 #include <numeric>
 
-#include <Storages/Statistics/Statistics.h>
-#include <Storages/Statistics/ConditionSelectivityEstimator.h>
-#include <Storages/Statistics/TDigestStatistics.h>
-#include <Storages/Statistics/UniqStatistics.h>
-#include <Storages/Statistics/CMSketchStatistics.h>
-#include <Storages/StatisticsDescription.h>
-#include <Storages/ColumnsDescription.h>
 #include <IO/ReadHelpers.h>
 #include <IO/WriteHelpers.h>
+#include <Storages/ColumnsDescription.h>
+#include <Storages/Statistics/ConditionSelectivityEstimator.h>
+#include <Storages/Statistics/CountMinSketchStatistics.h>
+#include <Storages/Statistics/Statistics.h>
+#include <Storages/Statistics/TDigestStatistics.h>
+#include <Storages/Statistics/UniqStatistics.h>
+#include <Storages/StatisticsDescription.h>
 #include <Common/Exception.h>
 
 namespace DB
@@ -92,10 +92,10 @@ Float64 ColumnStatistics::estimateEqual(Field val) const
         }
     }
 #if USE_DATASKETCHES
-    if (stats.contains(StatisticsType::CMSketch))
+    if (stats.contains(StatisticsType::CountMinSketch))
     {
-        auto cmsketch_static = std::static_pointer_cast<CMSketchStatistics>(stats.at(StatisticsType::CMSketch));
-        return cmsketch_static->estimateEqual(val);
+        auto count_min_sketch_static = std::static_pointer_cast<CountMinSketchStatistics>(stats.at(StatisticsType::CountMinSketch));
+        return count_min_sketch_static->estimateEqual(val);
     }
 #endif
     if (val < - ConditionSelectivityEstimator::threshold || val > ConditionSelectivityEstimator::threshold)
@@ -177,8 +177,8 @@ MergeTreeStatisticsFactory::MergeTreeStatisticsFactory()
     registerValidator(StatisticsType::TDigest, TDigestValidator);
     registerValidator(StatisticsType::Uniq, UniqValidator);
 #if USE_DATASKETCHES
-    registerCreator(StatisticsType::CMSketch, CMSketchCreator);
-    registerValidator(StatisticsType::CMSketch, CMSketchValidator);
+    registerCreator(StatisticsType::CountMinSketch, CountMinSketchCreator);
+    registerValidator(StatisticsType::CountMinSketch, CountMinSketchValidator);
 #endif
 }
 
diff --git a/src/Storages/StatisticsDescription.cpp b/src/Storages/StatisticsDescription.cpp
index 08c79043ac4..23339ca3cfe 100644
--- a/src/Storages/StatisticsDescription.cpp
+++ b/src/Storages/StatisticsDescription.cpp
@@ -54,8 +54,8 @@ static StatisticsType stringToStatisticsType(String type)
         return StatisticsType::TDigest;
     if (type == "uniq")
         return StatisticsType::Uniq;
-    if (type == "cmsketch")
-        return StatisticsType::CMSketch;
+    if (type == "countminsketch")
+        return StatisticsType::CountMinSketch;
     throw Exception(ErrorCodes::INCORRECT_QUERY, "Unknown statistics type: {}. Supported statistics types are `tdigest` and `uniq`.", type);
 }
 
@@ -67,8 +67,8 @@ String SingleStatisticsDescription::getTypeName() const
             return "TDigest";
         case StatisticsType::Uniq:
             return "Uniq";
-        case StatisticsType::CMSketch:
-            return "CMSketch";
+        case StatisticsType::CountMinSketch:
+            return "CountMinSketch";
         default:
             throw Exception(ErrorCodes::LOGICAL_ERROR, "Unknown statistics type: {}. Supported statistics types are `tdigest` and `uniq`.", type);
     }
diff --git a/src/Storages/StatisticsDescription.h b/src/Storages/StatisticsDescription.h
index a2005f59de1..03b8fb0d583 100644
--- a/src/Storages/StatisticsDescription.h
+++ b/src/Storages/StatisticsDescription.h
@@ -13,7 +13,7 @@ enum class StatisticsType : UInt8
 {
     TDigest = 0,
     Uniq = 1,
-    CMSketch = 2,
+    CountMinSketch = 2,
 
     Max = 63,
 };
diff --git a/tests/queries/0_stateless/03174_statistics_cmsketch.reference b/tests/queries/0_stateless/03174_statistics_countminsketch.reference
similarity index 80%
rename from tests/queries/0_stateless/03174_statistics_cmsketch.reference
rename to tests/queries/0_stateless/03174_statistics_countminsketch.reference
index ea77f317a31..3cec7dd7168 100644
--- a/tests/queries/0_stateless/03174_statistics_cmsketch.reference
+++ b/tests/queries/0_stateless/03174_statistics_countminsketch.reference
@@ -1,4 +1,4 @@
-CREATE TABLE default.t1\n(\n    `a` String STATISTICS(cmsketch),\n    `b` Int64 STATISTICS(cmsketch),\n    `c` UInt64 STATISTICS(cmsketch),\n    `pk` String\n)\nENGINE = MergeTree\nORDER BY pk\nSETTINGS min_bytes_for_wide_part = 0, index_granularity = 8192
+CREATE TABLE default.t1\n(\n    `a` String STATISTICS(countminsketch),\n    `b` Int64 STATISTICS(countminsketch),\n    `c` UInt64 STATISTICS(countminsketch),\n    `pk` String\n)\nENGINE = MergeTree\nORDER BY pk\nSETTINGS min_bytes_for_wide_part = 0, index_granularity = 8192
         Prewhere info
           Prewhere filter
           Prewhere filter column: and(equals(a, \'0\'_String), equals(b, 0), equals(c, 0)) (removed)
diff --git a/tests/queries/0_stateless/03174_statistics_cmsketch.sql b/tests/queries/0_stateless/03174_statistics_countminsketch.sql
similarity index 84%
rename from tests/queries/0_stateless/03174_statistics_cmsketch.sql
rename to tests/queries/0_stateless/03174_statistics_countminsketch.sql
index c45d6186fdf..0d9673309b4 100644
--- a/tests/queries/0_stateless/03174_statistics_cmsketch.sql
+++ b/tests/queries/0_stateless/03174_statistics_countminsketch.sql
@@ -6,9 +6,9 @@ SET allow_statistics_optimize = 1;
 
 CREATE TABLE t1
 (
-    a String STATISTICS(cmsketch),
-    b Int64 STATISTICS(cmsketch),
-    c UInt64 STATISTICS(cmsketch),
+    a String STATISTICS(countminsketch),
+    b Int64 STATISTICS(countminsketch),
+    c UInt64 STATISTICS(countminsketch),
     pk String,
 ) Engine = MergeTree() ORDER BY pk
 SETTINGS min_bytes_for_wide_part = 0;
@@ -31,9 +31,9 @@ DROP TABLE IF EXISTS t2;
 SET allow_suspicious_low_cardinality_types=1;
 CREATE TABLE t2
 (
-    a LowCardinality(String) STATISTICS(cmsketch),
-    b Int64 STATISTICS(cmsketch),
-    c UInt64 STATISTICS(cmsketch),
+    a LowCardinality(String) STATISTICS(countminsketch),
+    b Int64 STATISTICS(countminsketch),
+    c UInt64 STATISTICS(countminsketch),
     pk String,
 ) Engine = MergeTree() ORDER BY pk
 SETTINGS min_bytes_for_wide_part = 0;
@@ -48,9 +48,9 @@ DROP TABLE IF EXISTS t3;
 
 CREATE TABLE t3
 (
-    a Nullable(String) STATISTICS(cmsketch),
-    b Int64 STATISTICS(cmsketch),
-    c UInt64 STATISTICS(cmsketch),
+    a Nullable(String) STATISTICS(countminsketch),
+    b Int64 STATISTICS(countminsketch),
+    c UInt64 STATISTICS(countminsketch),
     pk String,
 ) Engine = MergeTree() ORDER BY pk
 SETTINGS min_bytes_for_wide_part = 0;
@@ -64,9 +64,9 @@ DROP TABLE IF EXISTS t4;
 
 CREATE TABLE t4
 (
-    a LowCardinality(Nullable(String)) STATISTICS(cmsketch),
-    b Int64 STATISTICS(cmsketch),
-    c UInt64 STATISTICS(cmsketch),
+    a LowCardinality(Nullable(String)) STATISTICS(countminsketch),
+    b Int64 STATISTICS(countminsketch),
+    c UInt64 STATISTICS(countminsketch),
     pk String,
 ) Engine = MergeTree() ORDER BY pk
 SETTINGS min_bytes_for_wide_part = 0;

From 051290e6c912108986c896916db087c71230a121 Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Mon, 24 Jun 2024 12:26:46 +0000
Subject: [PATCH 0089/1488] Add throughput introspection for scheduler queues

---
 src/Common/EventRateMeter.h                    | 18 ++++++++++++++++--
 src/Common/Scheduler/ISchedulerNode.h          | 14 ++++++++++++++
 src/Common/Scheduler/Nodes/FairPolicy.h        |  3 +--
 src/Common/Scheduler/Nodes/FifoQueue.h         |  3 +--
 src/Common/Scheduler/Nodes/PriorityPolicy.h    |  3 +--
 .../Scheduler/Nodes/SemaphoreConstraint.h      |  3 +--
 .../Scheduler/Nodes/ThrottlerConstraint.h      |  3 +--
 src/Common/Scheduler/SchedulerRoot.h           |  3 +--
 src/Storages/System/StorageSystemScheduler.cpp |  2 ++
 9 files changed, 38 insertions(+), 14 deletions(-)

diff --git a/src/Common/EventRateMeter.h b/src/Common/EventRateMeter.h
index 3a21a80ce8b..4c38d1d9371 100644
--- a/src/Common/EventRateMeter.h
+++ b/src/Common/EventRateMeter.h
@@ -14,8 +14,9 @@ namespace DB
 class EventRateMeter
 {
 public:
-    explicit EventRateMeter(double now, double period_)
+    explicit EventRateMeter(double now, double period_, double step_ = 0.0)
         : period(period_)
+        , step(step_)
         , half_decay_time(period * std::numbers::ln2) // for `ExponentiallySmoothedAverage::sumWeights()` to be equal to `1/period`
     {
         reset(now);
@@ -38,7 +39,16 @@ public:
         if (now - period <= start) // precise counting mode
             events = ExponentiallySmoothedAverage(events.value + count, now);
         else // exponential smoothing mode
-            events.add(count, now, half_decay_time);
+        {
+            // Adding events too often lead to low precision due to smoothing too often, so we buffer new events and add them in steps
+            step_count += count;
+            if (step_start + step <= now)
+            {
+                events.add(step_count, now, half_decay_time);
+                step_start = now;
+                step_count = 0;
+            }
+        }
     }
 
     /// Compute average event rate throughout `[now - period, now]` period.
@@ -58,16 +68,20 @@ public:
     void reset(double now)
     {
         start = now;
+        step_start = now;
         events = ExponentiallySmoothedAverage();
         data_points = 0;
     }
 
 private:
     const double period;
+    const double step; // duration of a step
     const double half_decay_time;
     double start; // Instant in past without events before it; when measurement started or reset
     ExponentiallySmoothedAverage events; // Estimated number of events in the last `period`
     size_t data_points = 0;
+    double step_start; // start instant of the last step
+    double step_count = 0.0; // number of events accumulated since step start
 };
 
 }
diff --git a/src/Common/Scheduler/ISchedulerNode.h b/src/Common/Scheduler/ISchedulerNode.h
index 81b491b0eda..c051829e336 100644
--- a/src/Common/Scheduler/ISchedulerNode.h
+++ b/src/Common/Scheduler/ISchedulerNode.h
@@ -3,6 +3,8 @@
 #include <Common/ErrorCodes.h>
 #include <Common/Exception.h>
 #include <Common/Priority.h>
+#include <Common/EventRateMeter.h>
+#include <Common/Stopwatch.h>
 #include <base/defines.h>
 #include <base/types.h>
 
@@ -176,6 +178,14 @@ protected:
     /// Postponed to be handled in scheduler thread, so it is intended to be called from outside.
     void scheduleActivation();
 
+    /// Helper for introspection metrics
+    void incrementDequeued(ResourceCost cost)
+    {
+        dequeued_requests++;
+        dequeued_cost += cost;
+        throughput.add(static_cast<double>(clock_gettime_ns())/1e9, cost);
+    }
+
 public:
     EventQueue * const event_queue;
     String basename;
@@ -189,6 +199,10 @@ public:
     std::atomic<ResourceCost> dequeued_cost{0};
     std::atomic<ResourceCost> canceled_cost{0};
     std::atomic<UInt64> busy_periods{0};
+
+    /// Average dequeued_cost per second
+    /// WARNING: Should only be accessed from the scheduler thread, so that locking is not required
+    EventRateMeter throughput{static_cast<double>(clock_gettime_ns())/1e9, 2, 1};
 };
 
 using SchedulerNodePtr = std::shared_ptr<ISchedulerNode>;
diff --git a/src/Common/Scheduler/Nodes/FairPolicy.h b/src/Common/Scheduler/Nodes/FairPolicy.h
index 0a4e55c253b..fba637e979e 100644
--- a/src/Common/Scheduler/Nodes/FairPolicy.h
+++ b/src/Common/Scheduler/Nodes/FairPolicy.h
@@ -188,8 +188,7 @@ public:
 
             if (request)
             {
-                dequeued_requests++;
-                dequeued_cost += request->cost;
+                incrementDequeued(request->cost);
                 return {request, heap_size > 0};
             }
         }
diff --git a/src/Common/Scheduler/Nodes/FifoQueue.h b/src/Common/Scheduler/Nodes/FifoQueue.h
index 9ec997c06d2..9fbc6d1ae65 100644
--- a/src/Common/Scheduler/Nodes/FifoQueue.h
+++ b/src/Common/Scheduler/Nodes/FifoQueue.h
@@ -59,8 +59,7 @@ public:
         if (requests.empty())
             busy_periods++;
         queue_cost -= result->cost;
-        dequeued_requests++;
-        dequeued_cost += result->cost;
+        incrementDequeued(result->cost);
         return {result, !requests.empty()};
     }
 
diff --git a/src/Common/Scheduler/Nodes/PriorityPolicy.h b/src/Common/Scheduler/Nodes/PriorityPolicy.h
index 22a5155cfeb..91dc95600d5 100644
--- a/src/Common/Scheduler/Nodes/PriorityPolicy.h
+++ b/src/Common/Scheduler/Nodes/PriorityPolicy.h
@@ -122,8 +122,7 @@ public:
 
             if (request)
             {
-                dequeued_requests++;
-                dequeued_cost += request->cost;
+                incrementDequeued(request->cost);
                 return {request, !items.empty()};
             }
         }
diff --git a/src/Common/Scheduler/Nodes/SemaphoreConstraint.h b/src/Common/Scheduler/Nodes/SemaphoreConstraint.h
index 10fce536f5d..92c6af9db18 100644
--- a/src/Common/Scheduler/Nodes/SemaphoreConstraint.h
+++ b/src/Common/Scheduler/Nodes/SemaphoreConstraint.h
@@ -81,8 +81,7 @@ public:
         child_active = child_now_active;
         if (!active())
             busy_periods++;
-        dequeued_requests++;
-        dequeued_cost += request->cost;
+        incrementDequeued(request->cost);
         return {request, active()};
     }
 
diff --git a/src/Common/Scheduler/Nodes/ThrottlerConstraint.h b/src/Common/Scheduler/Nodes/ThrottlerConstraint.h
index f4a5795bb2b..56866336f50 100644
--- a/src/Common/Scheduler/Nodes/ThrottlerConstraint.h
+++ b/src/Common/Scheduler/Nodes/ThrottlerConstraint.h
@@ -89,8 +89,7 @@ public:
         child_active = child_now_active;
         if (!active())
             busy_periods++;
-        dequeued_requests++;
-        dequeued_cost += request->cost;
+        incrementDequeued(request->cost);
         return {request, active()};
     }
 
diff --git a/src/Common/Scheduler/SchedulerRoot.h b/src/Common/Scheduler/SchedulerRoot.h
index 7af42fdbbea..5307aadc3cc 100644
--- a/src/Common/Scheduler/SchedulerRoot.h
+++ b/src/Common/Scheduler/SchedulerRoot.h
@@ -162,8 +162,7 @@ public:
             if (request == nullptr) // Possible in case of request cancel, just retry
                 continue;
 
-            dequeued_requests++;
-            dequeued_cost += request->cost;
+            incrementDequeued(request->cost);
             return {request, current != nullptr};
         }
     }
diff --git a/src/Storages/System/StorageSystemScheduler.cpp b/src/Storages/System/StorageSystemScheduler.cpp
index 339a59e88a5..b42c807d6fc 100644
--- a/src/Storages/System/StorageSystemScheduler.cpp
+++ b/src/Storages/System/StorageSystemScheduler.cpp
@@ -31,6 +31,7 @@ ColumnsDescription StorageSystemScheduler::getColumnsDescription()
         {"dequeued_requests", std::make_shared<DataTypeUInt64>(), "The total number of resource requests dequeued from this node."},
         {"canceled_requests", std::make_shared<DataTypeUInt64>(), "The total number of resource requests canceled from this node."},
         {"dequeued_cost", std::make_shared<DataTypeInt64>(), "The sum of costs (e.g. size in bytes) of all requests dequeued from this node."},
+        {"throughput", std::make_shared<DataTypeFloat64>(), "Current average throughput (dequeued cost per second)."},
         {"canceled_cost", std::make_shared<DataTypeInt64>(), "The sum of costs (e.g. size in bytes) of all requests canceled from this node."},
         {"busy_periods", std::make_shared<DataTypeUInt64>(), "The total number of deactivations of this node."},
         {"vruntime", std::make_shared<DataTypeNullable>(std::make_shared<DataTypeFloat64>()),
@@ -96,6 +97,7 @@ void StorageSystemScheduler::fillData(MutableColumns & res_columns, ContextPtr c
         res_columns[i++]->insert(node->dequeued_requests.load());
         res_columns[i++]->insert(node->canceled_requests.load());
         res_columns[i++]->insert(node->dequeued_cost.load());
+        res_columns[i++]->insert(node->throughput.rate(static_cast<double>(clock_gettime_ns())/1e9));
         res_columns[i++]->insert(node->canceled_cost.load());
         res_columns[i++]->insert(node->busy_periods.load());
 

From 7a27b10ef5a9a6a86e985d5922b85038d1cb03ee Mon Sep 17 00:00:00 2001
From: AntiTopQuark <AntiTopQuark1350@outlook.com>
Date: Mon, 24 Jun 2024 23:39:47 +0800
Subject: [PATCH 0090/1488] support MinMax hyperrectangle for MergeTreeIndexSet

---
 src/Storages/MergeTree/MergeTreeIndexSet.cpp | 66 ++++++++++++++++++--
 src/Storages/MergeTree/MergeTreeIndexSet.h   | 11 +++-
 2 files changed, 70 insertions(+), 7 deletions(-)

diff --git a/src/Storages/MergeTree/MergeTreeIndexSet.cpp b/src/Storages/MergeTree/MergeTreeIndexSet.cpp
index b11cbf1e034..e5708b67e03 100644
--- a/src/Storages/MergeTree/MergeTreeIndexSet.cpp
+++ b/src/Storages/MergeTree/MergeTreeIndexSet.cpp
@@ -44,10 +44,12 @@ MergeTreeIndexGranuleSet::MergeTreeIndexGranuleSet(
     const String & index_name_,
     const Block & index_sample_block_,
     size_t max_rows_,
-    MutableColumns && mutable_columns_)
+    MutableColumns && mutable_columns_,
+    std::vector<Range> && set_hyperrectangle_)
     : index_name(index_name_)
     , max_rows(max_rows_)
     , block(index_sample_block_.cloneWithColumns(std::move(mutable_columns_)))
+    , set_hyperrectangle(std::move(set_hyperrectangle_))
 {
 }
 
@@ -85,6 +87,15 @@ void MergeTreeIndexGranuleSet::serializeBinary(WriteBuffer & ostr) const
         serialization->serializeBinaryBulkWithMultipleStreams(column, 0, size(), settings, state);
         serialization->serializeBinaryBulkStateSuffix(settings, state);
     }
+
+    for (size_t i = 0; i < num_columns; ++i)
+    {
+        const DataTypePtr & type = block.getByPosition(i).type;
+        auto serialization = type->getDefaultSerialization();
+
+        serialization->serializeBinary(set_hyperrectangle[i].left, ostr, {});
+        serialization->serializeBinary(set_hyperrectangle[i].right, ostr, {});
+    }
 }
 
 void MergeTreeIndexGranuleSet::deserializeBinary(ReadBuffer & istr, MergeTreeIndexVersion version)
@@ -117,6 +128,25 @@ void MergeTreeIndexGranuleSet::deserializeBinary(ReadBuffer & istr, MergeTreeInd
         serialization->deserializeBinaryBulkStatePrefix(settings, state, nullptr);
         serialization->deserializeBinaryBulkWithMultipleStreams(elem.column, rows_to_read, settings, state, nullptr);
     }
+
+    set_hyperrectangle.clear();
+    Field min_val;
+    Field max_val;
+    for (size_t i = 0; i < num_columns; ++i)
+    {
+        const DataTypePtr & type = block.getByPosition(i).type;
+        auto serialization = type->getDefaultSerialization();
+
+        serialization->deserializeBinary(min_val, istr, {});
+        serialization->deserializeBinary(max_val, istr, {});
+
+        // NULL_LAST
+        if (min_val.isNull())
+            min_val = POSITIVE_INFINITY;
+        if (max_val.isNull())
+            max_val = POSITIVE_INFINITY;
+        set_hyperrectangle.emplace_back(min_val, true, max_val, true);
+    }
 }
 
 
@@ -182,10 +212,29 @@ void MergeTreeIndexAggregatorSet::update(const Block & block, size_t * pos, size
 
     if (has_new_data)
     {
+        FieldRef field_min;
+        FieldRef field_max;
         for (size_t i = 0; i < columns.size(); ++i)
         {
             auto filtered_column = block.getByName(index_columns[i]).column->filter(filter, block.rows());
             columns[i]->insertRangeFrom(*filtered_column, 0, filtered_column->size());
+
+            if (const auto * column_nullable = typeid_cast<const ColumnNullable *>(filtered_column.get()))
+                column_nullable->getExtremesNullLast(field_min, field_max);
+            else
+                filtered_column->getExtremes(field_min, field_max);
+
+            if (set_hyperrectangle.size() <= i)
+            {
+                set_hyperrectangle.emplace_back(field_min, true, field_max, true);
+            }
+            else
+            {
+                set_hyperrectangle[i].left
+                    = applyVisitor(FieldVisitorAccurateLess(), set_hyperrectangle[i].left, field_min) ? set_hyperrectangle[i].left : field_min;
+                set_hyperrectangle[i].right
+                    = applyVisitor(FieldVisitorAccurateLess(), set_hyperrectangle[i].right, field_max) ? field_max : set_hyperrectangle[i].right;
+            }
         }
     }
 
@@ -221,7 +270,7 @@ bool MergeTreeIndexAggregatorSet::buildFilter(
 
 MergeTreeIndexGranulePtr MergeTreeIndexAggregatorSet::getGranuleAndReset()
 {
-    auto granule = std::make_shared<MergeTreeIndexGranuleSet>(index_name, index_sample_block, max_rows, std::move(columns));
+    auto granule = std::make_shared<MergeTreeIndexGranuleSet>(index_name, index_sample_block, max_rows, std::move(columns), std::move(set_hyperrectangle));
 
     switch (data.type)
     {
@@ -240,15 +289,22 @@ MergeTreeIndexGranulePtr MergeTreeIndexAggregatorSet::getGranuleAndReset()
     return granule;
 }
 
+KeyCondition buildCondition(const IndexDescription & index, const ActionsDAGPtr & filter_actions_dag, ContextPtr context)
+{
+    return KeyCondition{filter_actions_dag, context, index.column_names, index.expression};
+}
 
 MergeTreeIndexConditionSet::MergeTreeIndexConditionSet(
     const String & index_name_,
     const Block & index_sample_block,
     size_t max_rows_,
     const ActionsDAGPtr & filter_dag,
-    ContextPtr context)
+    ContextPtr context,
+    const IndexDescription & index_description)
     : index_name(index_name_)
     , max_rows(max_rows_)
+    , index_data_types(index_description.data_types)
+    , condition(buildCondition(index_description, filter_dag, context))
 {
     for (const auto & name : index_sample_block.getNames())
         if (!key_columns.contains(name))
@@ -302,7 +358,7 @@ bool MergeTreeIndexConditionSet::mayBeTrueOnGranule(MergeTreeIndexGranulePtr idx
         if (!column->isNullAt(i) && (column->get64(i) & 1))
             return true;
 
-    return false;
+    return condition.checkInHyperrectangle(granule.set_hyperrectangle, index_data_types).can_be_true;
 }
 
 
@@ -546,7 +602,7 @@ MergeTreeIndexAggregatorPtr MergeTreeIndexSet::createIndexAggregator(const Merge
 MergeTreeIndexConditionPtr MergeTreeIndexSet::createIndexCondition(
     const ActionsDAGPtr & filter_actions_dag, ContextPtr context) const
 {
-    return std::make_shared<MergeTreeIndexConditionSet>(index.name, index.sample_block, max_rows, filter_actions_dag, context);
+    return std::make_shared<MergeTreeIndexConditionSet>(index.name, index.sample_block, max_rows, filter_actions_dag, context, index);
 }
 
 MergeTreeIndexPtr setIndexCreator(const IndexDescription & index)
diff --git a/src/Storages/MergeTree/MergeTreeIndexSet.h b/src/Storages/MergeTree/MergeTreeIndexSet.h
index 6efc2effafd..4fe79cb03c5 100644
--- a/src/Storages/MergeTree/MergeTreeIndexSet.h
+++ b/src/Storages/MergeTree/MergeTreeIndexSet.h
@@ -22,7 +22,8 @@ struct MergeTreeIndexGranuleSet final : public IMergeTreeIndexGranule
         const String & index_name_,
         const Block & index_sample_block_,
         size_t max_rows_,
-        MutableColumns && columns_);
+        MutableColumns && columns_,
+        std::vector<Range> && set_hyperrectangle_);
 
     void serializeBinary(WriteBuffer & ostr) const override;
     void deserializeBinary(ReadBuffer & istr, MergeTreeIndexVersion version) override;
@@ -36,6 +37,7 @@ struct MergeTreeIndexGranuleSet final : public IMergeTreeIndexGranule
     const size_t max_rows;
 
     Block block;
+    std::vector<Range> set_hyperrectangle;
 };
 
 
@@ -73,6 +75,7 @@ private:
     ClearableSetVariants data;
     Sizes key_sizes;
     MutableColumns columns;
+    std::vector<Range> set_hyperrectangle;
 };
 
 
@@ -84,7 +87,8 @@ public:
         const Block & index_sample_block,
         size_t max_rows_,
         const ActionsDAGPtr & filter_dag,
-        ContextPtr context);
+        ContextPtr context,
+        const IndexDescription & index_description);
 
     bool alwaysUnknownOrTrue() const override;
 
@@ -119,6 +123,9 @@ private:
     std::unordered_set<String> key_columns;
     ExpressionActionsPtr actions;
     String actions_output_column_name;
+    
+    DataTypes index_data_types;
+    KeyCondition condition;
 };
 
 

From b0ac0327d4d38a918e9be8fc499038cc43cafb2c Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Tue, 25 Jun 2024 11:48:29 +0000
Subject: [PATCH 0091/1488] Fix bug in EventRateMeter

It was relying on ExponentiallySmoothedCounter::get() which is designed for specific 1 second time interval between points. Now sum of weights is computed separatly in `duration` field, giving very accurate measurements independent of interval.
---
 src/Common/EventRateMeter.h                 | 52 ++++++----------
 src/Common/ProgressIndication.h             |  2 +-
 src/Common/tests/gtest_event_rate_meter.cpp | 68 +++++++++++++++++++++
 3 files changed, 86 insertions(+), 36 deletions(-)
 create mode 100644 src/Common/tests/gtest_event_rate_meter.cpp

diff --git a/src/Common/EventRateMeter.h b/src/Common/EventRateMeter.h
index 4c38d1d9371..b8a9112428f 100644
--- a/src/Common/EventRateMeter.h
+++ b/src/Common/EventRateMeter.h
@@ -4,8 +4,6 @@
 
 #include <Common/ExponentiallySmoothedCounter.h>
 
-#include <numbers>
-
 
 namespace DB
 {
@@ -14,10 +12,10 @@ namespace DB
 class EventRateMeter
 {
 public:
-    explicit EventRateMeter(double now, double period_, double step_ = 0.0)
+    explicit EventRateMeter(double now, double period_, size_t heating_ = 0)
         : period(period_)
-        , step(step_)
-        , half_decay_time(period * std::numbers::ln2) // for `ExponentiallySmoothedAverage::sumWeights()` to be equal to `1/period`
+        , max_interval(period * 10)
+        , heating(heating_)
     {
         reset(now);
     }
@@ -30,25 +28,11 @@ public:
     {
         // Remove data for initial heating stage that can present at the beginning of a query.
         // Otherwise it leads to wrong gradual increase of average value, turning algorithm into not very reactive.
-        if (count != 0.0 && ++data_points < 5)
-        {
-            start = events.time;
-            events = ExponentiallySmoothedAverage();
-        }
+        if (count != 0.0 && data_points++ <= heating)
+            reset(events.time, data_points);
 
-        if (now - period <= start) // precise counting mode
-            events = ExponentiallySmoothedAverage(events.value + count, now);
-        else // exponential smoothing mode
-        {
-            // Adding events too often lead to low precision due to smoothing too often, so we buffer new events and add them in steps
-            step_count += count;
-            if (step_start + step <= now)
-            {
-                events.add(step_count, now, half_decay_time);
-                step_start = now;
-                step_count = 0;
-            }
-        }
+        duration.add(std::min(max_interval, now - duration.time), now, period);
+        events.add(count, now, period);
     }
 
     /// Compute average event rate throughout `[now - period, now]` period.
@@ -59,29 +43,27 @@ public:
         add(now, 0);
         if (unlikely(now <= start))
             return 0;
-        if (now - period <= start) // precise counting mode
-            return events.value / (now - start);
-        else // exponential smoothing mode
-            return events.get(half_decay_time); // equals to `events.value / period`
+
+        // We do not use .get() because sum of weights will anyway be canceled out (optimization)
+        return events.value / duration.value;
     }
 
-    void reset(double now)
+    void reset(double now, size_t data_points_ = 0)
     {
         start = now;
-        step_start = now;
         events = ExponentiallySmoothedAverage();
-        data_points = 0;
+        duration = ExponentiallySmoothedAverage();
+        data_points = data_points_;
     }
 
 private:
     const double period;
-    const double step; // duration of a step
-    const double half_decay_time;
+    const double max_interval;
+    const size_t heating;
     double start; // Instant in past without events before it; when measurement started or reset
-    ExponentiallySmoothedAverage events; // Estimated number of events in the last `period`
+    ExponentiallySmoothedAverage duration; // Current duration of a period
+    ExponentiallySmoothedAverage events; // Estimated number of events in last `duration` seconds
     size_t data_points = 0;
-    double step_start; // start instant of the last step
-    double step_count = 0.0; // number of events accumulated since step start
 };
 
 }
diff --git a/src/Common/ProgressIndication.h b/src/Common/ProgressIndication.h
index a9965785889..d925077a072 100644
--- a/src/Common/ProgressIndication.h
+++ b/src/Common/ProgressIndication.h
@@ -91,7 +91,7 @@ private:
 
     bool write_progress_on_update = false;
 
-    EventRateMeter cpu_usage_meter{static_cast<double>(clock_gettime_ns()), 2'000'000'000 /*ns*/}; // average cpu utilization last 2 second
+    EventRateMeter cpu_usage_meter{static_cast<double>(clock_gettime_ns()), 2'000'000'000 /*ns*/, 4}; // average cpu utilization last 2 second, skip first 4 points
     HostToTimesMap hosts_data;
     /// In case of all of the above:
     /// - clickhouse-local
diff --git a/src/Common/tests/gtest_event_rate_meter.cpp b/src/Common/tests/gtest_event_rate_meter.cpp
new file mode 100644
index 00000000000..91ceec5eef7
--- /dev/null
+++ b/src/Common/tests/gtest_event_rate_meter.cpp
@@ -0,0 +1,68 @@
+#include <gtest/gtest.h>
+
+#include <Common/EventRateMeter.h>
+
+#include <cmath>
+
+
+TEST(EventRateMeter, ExponentiallySmoothedAverage)
+{
+    double target = 100.0;
+
+    // The test is only correct for timestep of 1 second because of
+    // how sum of weights is implemented inside `ExponentiallySmoothedAverage`
+    double time_step = 1.0;
+
+    for (double half_decay_time : { 0.1, 1.0, 10.0, 100.0})
+    {
+        DB::ExponentiallySmoothedAverage esa;
+
+        int steps = static_cast<int>(half_decay_time * 30 / time_step);
+        for (int i = 1; i <= steps; ++i)
+            esa.add(target * time_step, i * time_step, half_decay_time);
+        double measured = esa.get(half_decay_time);
+        ASSERT_LE(std::fabs(measured - target), 1e-5 * target);
+    }
+}
+
+TEST(EventRateMeter, ConstantRate)
+{
+    double target = 100.0;
+
+    for (double period : {0.1, 1.0, 10.0})
+    {
+        for (double time_step : {0.001, 0.01, 0.1, 1.0})
+        {
+            DB::EventRateMeter erm(0.0, period);
+
+            int steps = static_cast<int>(period * 30 / time_step);
+            for (int i = 1; i <= steps; ++i)
+                erm.add(i * time_step, target * time_step);
+            double measured = erm.rate(steps * time_step);
+            // std::cout << "T=" << period << " dt=" << time_step << " measured=" << measured << std::endl;
+            ASSERT_LE(std::fabs(measured - target), 1e-5 * target);
+        }
+    }
+}
+
+TEST(EventRateMeter, PreciseStart)
+{
+    double target = 100.0;
+
+    for (double period : {0.1, 1.0, 10.0})
+    {
+        for (double time_step : {0.001, 0.01, 0.1, 1.0})
+        {
+            DB::EventRateMeter erm(0.0, period);
+
+            int steps = static_cast<int>(period / time_step);
+            for (int i = 1; i <= steps; ++i)
+            {
+                erm.add(i * time_step, target * time_step);
+                double measured = erm.rate(i * time_step);
+                // std::cout << "T=" << period << " dt=" << time_step << " measured=" << measured << std::endl;
+                ASSERT_LE(std::fabs(measured - target), 1e-5 * target);
+            }
+        }
+    }
+}

From e29c8f9aeefb188efc2052d08f123e45b5b3309b Mon Sep 17 00:00:00 2001
From: AntiTopQuark <AntiTopQuark1350@outlook.com>
Date: Mon, 24 Jun 2024 23:39:47 +0800
Subject: [PATCH 0092/1488] support MinMax hyperrectangle for MergeTreeIndexSet

---
 src/Storages/MergeTree/MergeTreeIndexSet.cpp | 67 ++++++++++++++++++--
 src/Storages/MergeTree/MergeTreeIndexSet.h   | 11 +++-
 2 files changed, 72 insertions(+), 6 deletions(-)

diff --git a/src/Storages/MergeTree/MergeTreeIndexSet.cpp b/src/Storages/MergeTree/MergeTreeIndexSet.cpp
index b11cbf1e034..14ab266ded6 100644
--- a/src/Storages/MergeTree/MergeTreeIndexSet.cpp
+++ b/src/Storages/MergeTree/MergeTreeIndexSet.cpp
@@ -44,10 +44,12 @@ MergeTreeIndexGranuleSet::MergeTreeIndexGranuleSet(
     const String & index_name_,
     const Block & index_sample_block_,
     size_t max_rows_,
-    MutableColumns && mutable_columns_)
+    MutableColumns && mutable_columns_,
+    std::vector<Range> && set_hyperrectangle_)
     : index_name(index_name_)
     , max_rows(max_rows_)
     , block(index_sample_block_.cloneWithColumns(std::move(mutable_columns_)))
+    , set_hyperrectangle(std::move(set_hyperrectangle_))
 {
 }
 
@@ -85,6 +87,15 @@ void MergeTreeIndexGranuleSet::serializeBinary(WriteBuffer & ostr) const
         serialization->serializeBinaryBulkWithMultipleStreams(column, 0, size(), settings, state);
         serialization->serializeBinaryBulkStateSuffix(settings, state);
     }
+
+    for (size_t i = 0; i < num_columns; ++i)
+    {
+        const DataTypePtr & type = block.getByPosition(i).type;
+        auto serialization = type->getDefaultSerialization();
+
+        serialization->serializeBinary(set_hyperrectangle[i].left, ostr, {});
+        serialization->serializeBinary(set_hyperrectangle[i].right, ostr, {});
+    }
 }
 
 void MergeTreeIndexGranuleSet::deserializeBinary(ReadBuffer & istr, MergeTreeIndexVersion version)
@@ -117,6 +128,25 @@ void MergeTreeIndexGranuleSet::deserializeBinary(ReadBuffer & istr, MergeTreeInd
         serialization->deserializeBinaryBulkStatePrefix(settings, state, nullptr);
         serialization->deserializeBinaryBulkWithMultipleStreams(elem.column, rows_to_read, settings, state, nullptr);
     }
+
+    set_hyperrectangle.clear();
+    Field min_val;
+    Field max_val;
+    for (size_t i = 0; i < num_columns; ++i)
+    {
+        const DataTypePtr & type = block.getByPosition(i).type;
+        auto serialization = type->getDefaultSerialization();
+
+        serialization->deserializeBinary(min_val, istr, {});
+        serialization->deserializeBinary(max_val, istr, {});
+
+        // NULL_LAST
+        if (min_val.isNull())
+            min_val = POSITIVE_INFINITY;
+        if (max_val.isNull())
+            max_val = POSITIVE_INFINITY;
+        set_hyperrectangle.emplace_back(min_val, true, max_val, true);
+    }
 }
 
 
@@ -182,10 +212,29 @@ void MergeTreeIndexAggregatorSet::update(const Block & block, size_t * pos, size
 
     if (has_new_data)
     {
+        FieldRef field_min;
+        FieldRef field_max;
         for (size_t i = 0; i < columns.size(); ++i)
         {
             auto filtered_column = block.getByName(index_columns[i]).column->filter(filter, block.rows());
             columns[i]->insertRangeFrom(*filtered_column, 0, filtered_column->size());
+
+            if (const auto * column_nullable = typeid_cast<const ColumnNullable *>(filtered_column.get()))
+                column_nullable->getExtremesNullLast(field_min, field_max);
+            else
+                filtered_column->getExtremes(field_min, field_max);
+
+            if (set_hyperrectangle.size() <= i)
+            {
+                set_hyperrectangle.emplace_back(field_min, true, field_max, true);
+            }
+            else
+            {
+                set_hyperrectangle[i].left
+                    = applyVisitor(FieldVisitorAccurateLess(), set_hyperrectangle[i].left, field_min) ? set_hyperrectangle[i].left : field_min;
+                set_hyperrectangle[i].right
+                    = applyVisitor(FieldVisitorAccurateLess(), set_hyperrectangle[i].right, field_max) ? field_max : set_hyperrectangle[i].right;
+            }
         }
     }
 
@@ -221,7 +270,7 @@ bool MergeTreeIndexAggregatorSet::buildFilter(
 
 MergeTreeIndexGranulePtr MergeTreeIndexAggregatorSet::getGranuleAndReset()
 {
-    auto granule = std::make_shared<MergeTreeIndexGranuleSet>(index_name, index_sample_block, max_rows, std::move(columns));
+    auto granule = std::make_shared<MergeTreeIndexGranuleSet>(index_name, index_sample_block, max_rows, std::move(columns), std::move(set_hyperrectangle));
 
     switch (data.type)
     {
@@ -240,15 +289,22 @@ MergeTreeIndexGranulePtr MergeTreeIndexAggregatorSet::getGranuleAndReset()
     return granule;
 }
 
+KeyCondition buildCondition(const IndexDescription & index, const ActionsDAGPtr & filter_actions_dag, ContextPtr context)
+{
+    return KeyCondition{filter_actions_dag, context, index.column_names, index.expression};
+}
 
 MergeTreeIndexConditionSet::MergeTreeIndexConditionSet(
     const String & index_name_,
     const Block & index_sample_block,
     size_t max_rows_,
     const ActionsDAGPtr & filter_dag,
-    ContextPtr context)
+    ContextPtr context,
+    const IndexDescription & index_description)
     : index_name(index_name_)
     , max_rows(max_rows_)
+    , index_data_types(index_description.data_types)
+    , condition(buildCondition(index_description, filter_dag, context))
 {
     for (const auto & name : index_sample_block.getNames())
         if (!key_columns.contains(name))
@@ -292,6 +348,9 @@ bool MergeTreeIndexConditionSet::mayBeTrueOnGranule(MergeTreeIndexGranulePtr idx
     size_t size = granule.size();
     if (size == 0 || (max_rows != 0 && size > max_rows))
         return true;
+    
+    if (!condition.checkInHyperrectangle(granule.set_hyperrectangle, index_data_types).can_be_true)
+        return false;
 
     Block result = granule.block;
     actions->execute(result);
@@ -546,7 +605,7 @@ MergeTreeIndexAggregatorPtr MergeTreeIndexSet::createIndexAggregator(const Merge
 MergeTreeIndexConditionPtr MergeTreeIndexSet::createIndexCondition(
     const ActionsDAGPtr & filter_actions_dag, ContextPtr context) const
 {
-    return std::make_shared<MergeTreeIndexConditionSet>(index.name, index.sample_block, max_rows, filter_actions_dag, context);
+    return std::make_shared<MergeTreeIndexConditionSet>(index.name, index.sample_block, max_rows, filter_actions_dag, context, index);
 }
 
 MergeTreeIndexPtr setIndexCreator(const IndexDescription & index)
diff --git a/src/Storages/MergeTree/MergeTreeIndexSet.h b/src/Storages/MergeTree/MergeTreeIndexSet.h
index 6efc2effafd..4fe79cb03c5 100644
--- a/src/Storages/MergeTree/MergeTreeIndexSet.h
+++ b/src/Storages/MergeTree/MergeTreeIndexSet.h
@@ -22,7 +22,8 @@ struct MergeTreeIndexGranuleSet final : public IMergeTreeIndexGranule
         const String & index_name_,
         const Block & index_sample_block_,
         size_t max_rows_,
-        MutableColumns && columns_);
+        MutableColumns && columns_,
+        std::vector<Range> && set_hyperrectangle_);
 
     void serializeBinary(WriteBuffer & ostr) const override;
     void deserializeBinary(ReadBuffer & istr, MergeTreeIndexVersion version) override;
@@ -36,6 +37,7 @@ struct MergeTreeIndexGranuleSet final : public IMergeTreeIndexGranule
     const size_t max_rows;
 
     Block block;
+    std::vector<Range> set_hyperrectangle;
 };
 
 
@@ -73,6 +75,7 @@ private:
     ClearableSetVariants data;
     Sizes key_sizes;
     MutableColumns columns;
+    std::vector<Range> set_hyperrectangle;
 };
 
 
@@ -84,7 +87,8 @@ public:
         const Block & index_sample_block,
         size_t max_rows_,
         const ActionsDAGPtr & filter_dag,
-        ContextPtr context);
+        ContextPtr context,
+        const IndexDescription & index_description);
 
     bool alwaysUnknownOrTrue() const override;
 
@@ -119,6 +123,9 @@ private:
     std::unordered_set<String> key_columns;
     ExpressionActionsPtr actions;
     String actions_output_column_name;
+    
+    DataTypes index_data_types;
+    KeyCondition condition;
 };
 
 

From 5d9d29e37086e19dba852097e7820aff83072a8d Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Tue, 25 Jun 2024 15:00:07 +0000
Subject: [PATCH 0093/1488] Fixing some crashes

---
 src/Interpreters/ActionsDAG.cpp               | 15 ++++++++-----
 src/Interpreters/ActionsDAG.h                 |  5 +++--
 src/Interpreters/ExpressionActions.cpp        |  8 ++++---
 src/Interpreters/ExpressionAnalyzer.cpp       |  4 ++--
 src/Interpreters/InterpreterSelectQuery.cpp   | 14 ++++++------
 src/Interpreters/MutationsInterpreter.cpp     |  4 ++--
 src/Planner/Planner.cpp                       | 18 ++++++++-------
 src/Planner/PlannerJoinTree.cpp               | 22 +++++++++----------
 src/Planner/PlannerJoins.cpp                  |  4 ++--
 src/Planner/PlannerWindowFunctions.cpp        |  4 +++-
 src/Processors/QueryPlan/ExpressionStep.cpp   |  8 +++----
 src/Processors/QueryPlan/FilterStep.cpp       |  6 ++---
 .../Optimizations/distinctReadInOrder.cpp     |  4 ++--
 .../Optimizations/filterPushDown.cpp          |  4 ++--
 .../QueryPlan/Optimizations/liftUpUnion.cpp   |  2 +-
 .../Optimizations/optimizePrewhere.cpp        |  2 +-
 .../optimizePrimaryKeyCondition.cpp           |  6 ++---
 .../Optimizations/optimizeReadInOrder.cpp     |  8 +++----
 .../optimizeUseAggregateProjection.cpp        |  2 +-
 .../Optimizations/projectionsCommon.cpp       |  6 ++---
 .../Optimizations/removeRedundantDistinct.cpp |  4 ++--
 .../QueryPlan/ReadFromMergeTree.cpp           | 16 +++++++-------
 .../QueryPlan/SourceStepWithFilter.cpp        |  8 +++----
 src/Processors/QueryPlan/TotalsHavingStep.cpp |  8 +++----
 .../Transforms/FillingTransform.cpp           |  2 +-
 src/Storages/MergeTree/MergeTreeIndexSet.cpp  |  4 ++--
 .../MergeTree/MergeTreeSelectProcessor.cpp    |  4 ++--
 src/Storages/SelectQueryInfo.h                |  4 ++--
 src/Storages/StorageBuffer.cpp                |  8 +++----
 src/Storages/StorageMerge.cpp                 |  6 ++---
 src/Storages/VirtualColumnUtils.cpp           |  4 ++--
 31 files changed, 112 insertions(+), 102 deletions(-)

diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp
index 23e1e5ee152..c2626285235 100644
--- a/src/Interpreters/ActionsDAG.cpp
+++ b/src/Interpreters/ActionsDAG.cpp
@@ -1246,17 +1246,20 @@ bool ActionsDAG::removeUnusedResult(const std::string & column_name)
     return true;
 }
 
-ActionsDAGPtr ActionsDAG::clone() const
+ActionsDAGPtr ActionsDAG::clone(const ActionsDAG * from)
 {
     std::unordered_map<const Node *, Node *> old_to_new_nodes;
-    return clone(old_to_new_nodes);
+    return ActionsDAG::clone(from, old_to_new_nodes);
 }
 
-ActionsDAGPtr ActionsDAG::clone(std::unordered_map<const Node *, Node *> & old_to_new_nodes) const
+ActionsDAGPtr ActionsDAG::clone(const ActionsDAG * from, std::unordered_map<const Node *, Node *> & old_to_new_nodes)
 {
+    if (!from)
+        return nullptr;
+
     auto actions = std::make_unique<ActionsDAG>();
 
-    for (const auto & node : nodes)
+    for (const auto & node : from->nodes)
     {
         auto & copy_node = actions->nodes.emplace_back(node);
         old_to_new_nodes[&node] = &copy_node;
@@ -1266,10 +1269,10 @@ ActionsDAGPtr ActionsDAG::clone(std::unordered_map<const Node *, Node *> & old_t
         for (auto & child : node.children)
             child = old_to_new_nodes[child];
 
-    for (const auto & output_node : outputs)
+    for (const auto & output_node : from->outputs)
         actions->outputs.push_back(old_to_new_nodes[output_node]);
 
-    for (const auto & input_node : inputs)
+    for (const auto & input_node : from->inputs)
         actions->inputs.push_back(old_to_new_nodes[input_node]);
 
     return actions;
diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h
index 4a840885b6a..7ca3d1c1b0d 100644
--- a/src/Interpreters/ActionsDAG.h
+++ b/src/Interpreters/ActionsDAG.h
@@ -261,8 +261,9 @@ public:
     void compileExpressions(size_t min_count_to_compile_expression, const std::unordered_set<const Node *> & lazy_executed_nodes = {});
 #endif
 
-    ActionsDAGPtr clone() const;
-    ActionsDAGPtr clone(std::unordered_map<const Node *, Node *> & old_to_new_nodes) const;
+    static ActionsDAGPtr clone(const ActionsDAGPtr & from) { return clone(from.get()); }
+    static ActionsDAGPtr clone(const ActionsDAG * from);
+    static ActionsDAGPtr clone(const ActionsDAG * from, std::unordered_map<const Node *, Node *> & old_to_new_nodes);
 
     static ActionsDAGPtr cloneSubDAG(const NodeRawConstPtrs & outputs, bool remove_aliases);
 
diff --git a/src/Interpreters/ExpressionActions.cpp b/src/Interpreters/ExpressionActions.cpp
index 7cbf5afd763..2eca31fc75e 100644
--- a/src/Interpreters/ExpressionActions.cpp
+++ b/src/Interpreters/ExpressionActions.cpp
@@ -53,7 +53,7 @@ ExpressionActions::ExpressionActions(ActionsDAGPtr actions_dag_, const Expressio
     : project_inputs(project_inputs_)
     , settings(settings_)
 {
-    actions_dag = actions_dag_->clone();
+    actions_dag = ActionsDAG::clone(actions_dag_);
 
     /// It's important to determine lazy executed nodes before compiling expressions.
     std::unordered_set<const ActionsDAG::Node *> lazy_executed_nodes = processShortCircuitFunctions(*actions_dag, settings.short_circuit_function_evaluation);
@@ -76,15 +76,17 @@ ExpressionActionsPtr ExpressionActions::clone() const
     auto copy = std::make_shared<ExpressionActions>(ExpressionActions());
 
     std::unordered_map<const Node *, Node *> copy_map;
-    copy->actions_dag = actions_dag->clone(copy_map);
+    copy->actions_dag = ActionsDAG::clone(actions_dag.get(), copy_map);
     copy->actions = actions;
     for (auto & action : copy->actions)
         action.node = copy_map[action.node];
 
+    for (const auto * input : copy->actions_dag->getInputs())
+        copy->input_positions.emplace(input->result_name, input_positions.at(input->result_name));
+
     copy->num_columns = num_columns;
 
     copy->required_columns = required_columns;
-    copy->input_positions = input_positions;
     copy->result_positions = result_positions;
     copy->sample_block = sample_block;
 
diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp
index be00e37c751..6b49365b492 100644
--- a/src/Interpreters/ExpressionAnalyzer.cpp
+++ b/src/Interpreters/ExpressionAnalyzer.cpp
@@ -1944,7 +1944,7 @@ ExpressionAnalysisResult::ExpressionAnalysisResult(
                 Block before_prewhere_sample = source_header;
                 if (sanitizeBlock(before_prewhere_sample))
                 {
-                    auto dag = prewhere_dag_and_flags->dag.clone();
+                    auto dag = ActionsDAG::clone(&prewhere_dag_and_flags->dag);
                     ExpressionActions(
                         std::move(dag),
                         ExpressionActionsSettings::fromSettings(context->getSettingsRef())).execute(before_prewhere_sample);
@@ -1980,7 +1980,7 @@ ExpressionAnalysisResult::ExpressionAnalysisResult(
                 if (sanitizeBlock(before_where_sample))
                 {
                     ExpressionActions(
-                        before_where->dag.clone(),
+                        ActionsDAG::clone(&before_where->dag),
                         ExpressionActionsSettings::fromSettings(context->getSettingsRef())).execute(before_where_sample);
 
                     auto & column_elem
diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp
index 64a17a7ba87..71eb7dc64f8 100644
--- a/src/Interpreters/InterpreterSelectQuery.cpp
+++ b/src/Interpreters/InterpreterSelectQuery.cpp
@@ -1300,7 +1300,7 @@ static InterpolateDescriptionPtr getInterpolateDescription(
         ActionsDAGPtr actions = analyzer.getActionsDAG(true);
         ActionsDAGPtr conv_dag = ActionsDAG::makeConvertingActions(actions->getResultColumns(),
             result_columns, ActionsDAG::MatchColumnsMode::Position, true);
-        ActionsDAGPtr merge_dag = ActionsDAG::merge(std::move(*actions->clone()), std::move(*conv_dag));
+        ActionsDAGPtr merge_dag = ActionsDAG::merge(std::move(* ActionsDAG::clone(actions)), std::move(*conv_dag));
 
         interpolate_descr = std::make_shared<InterpolateDescription>(std::move(merge_dag), aliases);
     }
@@ -2042,7 +2042,7 @@ void InterpreterSelectQuery::addEmptySourceToQueryPlan(QueryPlan & query_plan, c
             pipe.addSimpleTransform([&](const Block & header)
             {
                 return std::make_shared<FilterTransform>(header,
-                    std::make_shared<ExpressionActions>(prewhere_info.row_level_filter->clone()),
+                    std::make_shared<ExpressionActions>(ActionsDAG::clone(prewhere_info.row_level_filter)),
                     prewhere_info.row_level_column_name, true);
             });
         }
@@ -2050,7 +2050,7 @@ void InterpreterSelectQuery::addEmptySourceToQueryPlan(QueryPlan & query_plan, c
         pipe.addSimpleTransform([&](const Block & header)
         {
             return std::make_shared<FilterTransform>(
-                header, std::make_shared<ExpressionActions>(prewhere_info.prewhere_actions->clone()),
+                header, std::make_shared<ExpressionActions>(ActionsDAG::clone(prewhere_info.prewhere_actions)),
                 prewhere_info.prewhere_column_name, prewhere_info.remove_prewhere_column);
         });
     }
@@ -2578,7 +2578,7 @@ void InterpreterSelectQuery::executeFetchColumns(QueryProcessingStage::Enum proc
 
 void InterpreterSelectQuery::executeWhere(QueryPlan & query_plan, const ActionsAndProjectInputsFlagPtr & expression, bool remove_filter)
 {
-    auto dag = expression->dag.clone();
+    auto dag = ActionsDAG::clone(&expression->dag);
     if (expression->project_input)
         dag->appendInputsForUnusedColumns(query_plan.getCurrentDataStream().header);
 
@@ -2752,7 +2752,7 @@ void InterpreterSelectQuery::executeMergeAggregated(QueryPlan & query_plan, bool
 
 void InterpreterSelectQuery::executeHaving(QueryPlan & query_plan, const ActionsAndProjectInputsFlagPtr & expression, bool remove_filter)
 {
-    auto dag = expression->dag.clone();
+    auto dag = ActionsDAG::clone(&expression->dag);
     if (expression->project_input)
         dag->appendInputsForUnusedColumns(query_plan.getCurrentDataStream().header);
 
@@ -2770,7 +2770,7 @@ void InterpreterSelectQuery::executeTotalsAndHaving(
     ActionsDAGPtr dag;
     if (expression)
     {
-        dag = expression->dag.clone();
+        dag = ActionsDAG::clone(&expression->dag);
         if (expression->project_input)
             dag->appendInputsForUnusedColumns(query_plan.getCurrentDataStream().header);
     }
@@ -2819,7 +2819,7 @@ void InterpreterSelectQuery::executeExpression(QueryPlan & query_plan, const Act
     if (!expression)
         return;
 
-    auto dag = expression->dag.clone();
+    auto dag = ActionsDAG::clone(&expression->dag);
     if (expression->project_input)
         dag->appendInputsForUnusedColumns(query_plan.getCurrentDataStream().header);
 
diff --git a/src/Interpreters/MutationsInterpreter.cpp b/src/Interpreters/MutationsInterpreter.cpp
index 1bb770bf561..704c5ce7d8b 100644
--- a/src/Interpreters/MutationsInterpreter.cpp
+++ b/src/Interpreters/MutationsInterpreter.cpp
@@ -1278,7 +1278,7 @@ QueryPipelineBuilder MutationsInterpreter::addStreamsForLaterStages(const std::v
 
             if (i < stage.filter_column_names.size())
             {
-                auto dag = step->actions()->dag.clone();
+                auto dag = ActionsDAG::clone(&step->actions()->dag);
                 if (step->actions()->project_input)
                     dag->appendInputsForUnusedColumns(plan.getCurrentDataStream().header);
                 /// Execute DELETEs.
@@ -1286,7 +1286,7 @@ QueryPipelineBuilder MutationsInterpreter::addStreamsForLaterStages(const std::v
             }
             else
             {
-                auto dag = step->actions()->dag.clone();
+                auto dag = ActionsDAG::clone(&step->actions()->dag);
                 if (step->actions()->project_input)
                     dag->appendInputsForUnusedColumns(plan.getCurrentDataStream().header);
                 /// Execute UPDATE or final projection.
diff --git a/src/Planner/Planner.cpp b/src/Planner/Planner.cpp
index 681ae7e6ac4..dddb7531519 100644
--- a/src/Planner/Planner.cpp
+++ b/src/Planner/Planner.cpp
@@ -333,12 +333,12 @@ void addExpressionStep(QueryPlan & query_plan,
     const std::string & step_description,
     std::vector<const ActionsDAG *> & result_actions_to_execute)
 {
-    auto actions = expression_actions->dag.clone();
+    auto actions = ActionsDAG::clone(&expression_actions->dag);
     if (expression_actions->project_input)
         actions->appendInputsForUnusedColumns(query_plan.getCurrentDataStream().header);
 
-    result_actions_to_execute.push_back(actions.get());
     auto expression_step = std::make_unique<ExpressionStep>(query_plan.getCurrentDataStream(), actions);
+    result_actions_to_execute.push_back(expression_step->getExpression().get());
     expression_step->setStepDescription(step_description);
     query_plan.addStep(std::move(expression_step));
 }
@@ -348,15 +348,15 @@ void addFilterStep(QueryPlan & query_plan,
     const std::string & step_description,
     std::vector<const ActionsDAG *> & result_actions_to_execute)
 {
-    auto actions = filter_analysis_result.filter_actions->dag.clone();
+    auto actions = ActionsDAG::clone(&filter_analysis_result.filter_actions->dag);
     if (filter_analysis_result.filter_actions->project_input)
         actions->appendInputsForUnusedColumns(query_plan.getCurrentDataStream().header);
 
-    result_actions_to_execute.push_back(actions.get());
     auto where_step = std::make_unique<FilterStep>(query_plan.getCurrentDataStream(),
         actions,
         filter_analysis_result.filter_column_name,
         filter_analysis_result.remove_filter_column);
+    result_actions_to_execute.push_back(where_step->getExpression().get());
     where_step->setStepDescription(step_description);
     query_plan.addStep(std::move(where_step));
 }
@@ -556,11 +556,9 @@ void addTotalsHavingStep(QueryPlan & query_plan,
     ActionsDAGPtr actions;
     if (having_analysis_result.filter_actions)
     {
-        actions = having_analysis_result.filter_actions->dag.clone();
+        actions = ActionsDAG::clone(&having_analysis_result.filter_actions->dag);
         if (having_analysis_result.filter_actions->project_input)
             actions->appendInputsForUnusedColumns(query_plan.getCurrentDataStream().header);
-
-        result_actions_to_execute.push_back(actions.get());
     }
 
     auto totals_having_step = std::make_unique<TotalsHavingStep>(
@@ -573,6 +571,10 @@ void addTotalsHavingStep(QueryPlan & query_plan,
         settings.totals_mode,
         settings.totals_auto_threshold,
         need_finalize);
+
+    if (having_analysis_result.filter_actions)
+        result_actions_to_execute.push_back(totals_having_step->getActions().get());
+
     query_plan.addStep(std::move(totals_having_step));
 }
 
@@ -1449,7 +1451,7 @@ void Planner::buildPlanForQueryNode()
             if (it != table_filters.end())
             {
                 const auto & filters = it->second;
-                table_expression_data.setFilterActions(filters.filter_actions->clone());
+                table_expression_data.setFilterActions(ActionsDAG::clone(filters.filter_actions));
                 table_expression_data.setPrewhereInfo(filters.prewhere_info);
             }
         }
diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp
index 918cfad703e..16b5e363bfd 100644
--- a/src/Planner/PlannerJoinTree.cpp
+++ b/src/Planner/PlannerJoinTree.cpp
@@ -646,7 +646,7 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres
 
         auto table_expression_query_info = select_query_info;
         table_expression_query_info.table_expression = table_expression;
-        table_expression_query_info.filter_actions_dag = table_expression_data.getFilterActions()->clone();
+        table_expression_query_info.filter_actions_dag = ActionsDAG::clone(table_expression_data.getFilterActions());
         table_expression_query_info.analyzer_can_use_parallel_replicas_on_follower = table_node == planner_context->getGlobalPlannerContext()->parallel_replicas_table;
 
         size_t max_streams = settings.max_threads;
@@ -776,7 +776,7 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres
                 if (prewhere_actions)
                 {
                     prewhere_info = std::make_shared<PrewhereInfo>();
-                    prewhere_info->prewhere_actions = prewhere_actions->clone();
+                    prewhere_info->prewhere_actions = ActionsDAG::clone(prewhere_actions);
                     prewhere_info->prewhere_column_name = prewhere_actions->getOutputs().at(0)->result_name;
                     prewhere_info->remove_prewhere_column = true;
                     prewhere_info->need_filter = true;
@@ -831,7 +831,7 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres
                 auto row_policy_filter_info
                     = buildRowPolicyFilterIfNeeded(storage, table_expression_query_info, planner_context, used_row_policies);
                 if (row_policy_filter_info.actions)
-                    table_expression_data.setRowLevelFilterActions(row_policy_filter_info.actions->clone());
+                    table_expression_data.setRowLevelFilterActions(ActionsDAG::clone(row_policy_filter_info.actions));
                 add_filter(row_policy_filter_info, "Row-level security filter");
 
                 if (query_context->getParallelReplicasMode() == Context::ParallelReplicasMode::CUSTOM_KEY)
@@ -1178,17 +1178,16 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_
             join_table_expression,
             planner_context);
 
-        left_join_tree_query_plan.actions_dags.emplace_back(join_clauses_and_actions.left_join_expressions_actions.get());
-        left_join_tree_query_plan.actions_dags.emplace_back(join_clauses_and_actions.right_join_expressions_actions.get());
-
         join_clauses_and_actions.left_join_expressions_actions->appendInputsForUnusedColumns(left_plan.getCurrentDataStream().header);
         auto left_join_expressions_actions_step = std::make_unique<ExpressionStep>(left_plan.getCurrentDataStream(), join_clauses_and_actions.left_join_expressions_actions);
         left_join_expressions_actions_step->setStepDescription("JOIN actions");
+        left_join_tree_query_plan.actions_dags.emplace_back(left_join_expressions_actions_step->getExpression().get());
         left_plan.addStep(std::move(left_join_expressions_actions_step));
 
         join_clauses_and_actions.right_join_expressions_actions->appendInputsForUnusedColumns(right_plan.getCurrentDataStream().header);
         auto right_join_expressions_actions_step = std::make_unique<ExpressionStep>(right_plan.getCurrentDataStream(), join_clauses_and_actions.right_join_expressions_actions);
         right_join_expressions_actions_step->setStepDescription("JOIN actions");
+        right_join_tree_query_plan.actions_dags.emplace_back(right_join_expressions_actions_step->getExpression().get());
         right_plan.addStep(std::move(right_join_expressions_actions_step));
     }
 
@@ -1434,7 +1433,8 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_
 
     auto result_plan = QueryPlan();
 
-    if (join_algorithm->isFilled())
+    bool is_filled_join = join_algorithm->isFilled();
+    if (is_filled_join)
     {
         auto filled_join_step = std::make_unique<FilledJoinStep>(
             left_plan.getCurrentDataStream(),
@@ -1586,8 +1586,9 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_
         left_join_tree_query_plan.used_row_policies.insert(right_join_tree_query_plan_row_policy);
 
     /// Collect all required actions dags in `left_join_tree_query_plan.actions_dags`
-    for (const auto * action_dag : right_join_tree_query_plan.actions_dags)
-        left_join_tree_query_plan.actions_dags.emplace_back(action_dag);
+    if (!is_filled_join)
+        for (const auto * action_dag : right_join_tree_query_plan.actions_dags)
+            left_join_tree_query_plan.actions_dags.emplace_back(action_dag);
     // if (join_clauses_and_actions.left_join_expressions_actions)
     //     left_join_tree_query_plan.actions_dags.emplace_back(join_clauses_and_actions.left_join_expressions_actions.get());
     // if (join_clauses_and_actions.right_join_expressions_actions)
@@ -1646,10 +1647,9 @@ JoinTreeQueryPlan buildQueryPlanForArrayJoinNode(const QueryTreeNodePtr & array_
 
     array_join_action_dag->appendInputsForUnusedColumns(plan.getCurrentDataStream().header);
 
-    join_tree_query_plan.actions_dags.push_back(array_join_action_dag.get());
-
     auto array_join_actions = std::make_unique<ExpressionStep>(plan.getCurrentDataStream(), std::move(array_join_action_dag));
     array_join_actions->setStepDescription("ARRAY JOIN actions");
+    join_tree_query_plan.actions_dags.push_back(array_join_actions->getExpression().get());
     plan.addStep(std::move(array_join_actions));
 
     auto drop_unused_columns_before_array_join_actions_dag = std::make_unique<ActionsDAG>(plan.getCurrentDataStream().header.getColumnsWithTypeAndName());
diff --git a/src/Planner/PlannerJoins.cpp b/src/Planner/PlannerJoins.cpp
index 45842c0d705..23b6a805ab9 100644
--- a/src/Planner/PlannerJoins.cpp
+++ b/src/Planner/PlannerJoins.cpp
@@ -588,10 +588,10 @@ JoinClausesAndActions buildJoinClausesAndActions(
         }
     }
 
-    result.left_join_expressions_actions = left_join_actions->clone();
+    result.left_join_expressions_actions = ActionsDAG::clone(left_join_actions);
     result.left_join_tmp_expression_actions = std::move(left_join_actions);
     result.left_join_expressions_actions->removeUnusedActions(join_left_actions_names);
-    result.right_join_expressions_actions = right_join_actions->clone();
+    result.right_join_expressions_actions = ActionsDAG::clone(right_join_actions);
     result.right_join_tmp_expression_actions = std::move(right_join_actions);
     result.right_join_expressions_actions->removeUnusedActions(join_right_actions_names);
 
diff --git a/src/Planner/PlannerWindowFunctions.cpp b/src/Planner/PlannerWindowFunctions.cpp
index 9deceeef9a3..b9e11578dbc 100644
--- a/src/Planner/PlannerWindowFunctions.cpp
+++ b/src/Planner/PlannerWindowFunctions.cpp
@@ -157,7 +157,9 @@ std::vector<size_t> sortWindowDescriptions(const std::vector<WindowDescription>
     };
 
     std::vector<size_t> perm(window_descriptions.size());
-    std::iota(perm.begin(), perm.end(), 0U);
+    for (size_t i = 0; i < perm.size(); ++i)
+        perm[i] = i;
+
     ::sort(perm.begin(), perm.end(), comparator);
 
     return perm;
diff --git a/src/Processors/QueryPlan/ExpressionStep.cpp b/src/Processors/QueryPlan/ExpressionStep.cpp
index 90ac94a1ace..50bc2e1533e 100644
--- a/src/Processors/QueryPlan/ExpressionStep.cpp
+++ b/src/Processors/QueryPlan/ExpressionStep.cpp
@@ -30,13 +30,13 @@ ExpressionStep::ExpressionStep(const DataStream & input_stream_, const ActionsDA
         input_stream_,
         ExpressionTransform::transformHeader(input_stream_.header, *actions_dag_),
         getTraits(actions_dag_, input_stream_.header, input_stream_.sort_description))
-    , actions_dag(actions_dag_->clone())
+    , actions_dag(ActionsDAG::clone(actions_dag_))
 {
 }
 
 void ExpressionStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings & settings)
 {
-    auto expression = std::make_shared<ExpressionActions>(actions_dag->clone(), settings.getActionsSettings());
+    auto expression = std::make_shared<ExpressionActions>(ActionsDAG::clone(actions_dag), settings.getActionsSettings());
 
     pipeline.addSimpleTransform([&](const Block & header)
     {
@@ -61,13 +61,13 @@ void ExpressionStep::transformPipeline(QueryPipelineBuilder & pipeline, const Bu
 void ExpressionStep::describeActions(FormatSettings & settings) const
 {
     String prefix(settings.offset, settings.indent_char);
-    auto expression = std::make_shared<ExpressionActions>(actions_dag->clone());
+    auto expression = std::make_shared<ExpressionActions>(ActionsDAG::clone(actions_dag));
     expression->describeActions(settings.out, prefix);
 }
 
 void ExpressionStep::describeActions(JSONBuilder::JSONMap & map) const
 {
-    auto expression = std::make_shared<ExpressionActions>(actions_dag->clone());
+    auto expression = std::make_shared<ExpressionActions>(ActionsDAG::clone(actions_dag));
     map.add("Expression", expression->toTree());
 }
 
diff --git a/src/Processors/QueryPlan/FilterStep.cpp b/src/Processors/QueryPlan/FilterStep.cpp
index ef9f1d17822..7883461f45a 100644
--- a/src/Processors/QueryPlan/FilterStep.cpp
+++ b/src/Processors/QueryPlan/FilterStep.cpp
@@ -49,7 +49,7 @@ FilterStep::FilterStep(
     , filter_column_name(std::move(filter_column_name_))
     , remove_filter_column(remove_filter_column_)
 {
-    actions_dag = actions_dag->clone();
+    actions_dag = ActionsDAG::clone(actions_dag_);
     actions_dag->removeAliasesForFilter(filter_column_name);
 }
 
@@ -87,7 +87,7 @@ void FilterStep::describeActions(FormatSettings & settings) const
         settings.out << " (removed)";
     settings.out << '\n';
 
-    auto expression = std::make_shared<ExpressionActions>(actions_dag->clone());
+    auto expression = std::make_shared<ExpressionActions>(ActionsDAG::clone(actions_dag));
     expression->describeActions(settings.out, prefix);
 }
 
@@ -96,7 +96,7 @@ void FilterStep::describeActions(JSONBuilder::JSONMap & map) const
     map.add("Filter Column", filter_column_name);
     map.add("Removes Filter", remove_filter_column);
 
-    auto expression = std::make_shared<ExpressionActions>(actions_dag->clone());
+    auto expression = std::make_shared<ExpressionActions>(ActionsDAG::clone(actions_dag));
     map.add("Expression", expression->toTree());
 }
 
diff --git a/src/Processors/QueryPlan/Optimizations/distinctReadInOrder.cpp b/src/Processors/QueryPlan/Optimizations/distinctReadInOrder.cpp
index 87e16b5a244..6cdc3cb4eb0 100644
--- a/src/Processors/QueryPlan/Optimizations/distinctReadInOrder.cpp
+++ b/src/Processors/QueryPlan/Optimizations/distinctReadInOrder.cpp
@@ -15,11 +15,11 @@ static ActionsDAGPtr buildActionsForPlanPath(std::vector<const ActionsDAG *> & d
     if (dag_stack.empty())
         return nullptr;
 
-    ActionsDAGPtr path_actions = dag_stack.back()->clone();
+    ActionsDAGPtr path_actions = ActionsDAG::clone(dag_stack.back());
     dag_stack.pop_back();
     while (!dag_stack.empty())
     {
-        ActionsDAGPtr clone = dag_stack.back()->clone();
+        ActionsDAGPtr clone = ActionsDAG::clone(dag_stack.back());
         dag_stack.pop_back();
         path_actions->mergeInplace(std::move(*clone));
     }
diff --git a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp
index ff1cefff09a..f26cd79dd97 100644
--- a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp
+++ b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp
@@ -597,7 +597,7 @@ size_t tryPushDownFilter(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes
 
             filter_node.step = std::make_unique<FilterStep>(
                 filter_node.children.front()->step->getOutputStream(),
-                filter->getExpression()->clone(),
+                ActionsDAG::clone(filter->getExpression()),
                 filter->getFilterColumnName(),
                 filter->removesFilterColumn());
         }
@@ -611,7 +611,7 @@ size_t tryPushDownFilter(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes
 
     if (auto * read_from_merge = typeid_cast<ReadFromMerge *>(child.get()))
     {
-        FilterDAGInfo info{filter->getExpression()->clone(), filter->getFilterColumnName(), filter->removesFilterColumn()};
+        FilterDAGInfo info{ActionsDAG::clone(filter->getExpression()), filter->getFilterColumnName(), filter->removesFilterColumn()};
         read_from_merge->addFilter(std::move(info));
         std::swap(*parent_node, *child_node);
         return 1;
diff --git a/src/Processors/QueryPlan/Optimizations/liftUpUnion.cpp b/src/Processors/QueryPlan/Optimizations/liftUpUnion.cpp
index 35d8b1a35e4..4629bc0af53 100644
--- a/src/Processors/QueryPlan/Optimizations/liftUpUnion.cpp
+++ b/src/Processors/QueryPlan/Optimizations/liftUpUnion.cpp
@@ -49,7 +49,7 @@ size_t tryLiftUpUnion(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes)
 
             expr_node.step = std::make_unique<ExpressionStep>(
                 expr_node.children.front()->step->getOutputStream(),
-                expression->getExpression()->clone());
+                ActionsDAG::clone(expression->getExpression()));
         }
 
         ///       - Expression - Something
diff --git a/src/Processors/QueryPlan/Optimizations/optimizePrewhere.cpp b/src/Processors/QueryPlan/Optimizations/optimizePrewhere.cpp
index 13b691da888..afe1406b65f 100644
--- a/src/Processors/QueryPlan/Optimizations/optimizePrewhere.cpp
+++ b/src/Processors/QueryPlan/Optimizations/optimizePrewhere.cpp
@@ -108,7 +108,7 @@ void optimizePrewhere(Stack & stack, QueryPlan::Nodes &)
     prewhere_info->need_filter = true;
     prewhere_info->remove_prewhere_column = optimize_result.fully_moved_to_prewhere && filter_step->removesFilterColumn();
 
-    auto filter_expression = filter_step->getExpression()->clone();
+    auto filter_expression = ActionsDAG::clone(filter_step->getExpression());
     const auto & filter_column_name = filter_step->getFilterColumnName();
 
     if (prewhere_info->remove_prewhere_column)
diff --git a/src/Processors/QueryPlan/Optimizations/optimizePrimaryKeyCondition.cpp b/src/Processors/QueryPlan/Optimizations/optimizePrimaryKeyCondition.cpp
index e57d3319076..e5ded92b105 100644
--- a/src/Processors/QueryPlan/Optimizations/optimizePrimaryKeyCondition.cpp
+++ b/src/Processors/QueryPlan/Optimizations/optimizePrimaryKeyCondition.cpp
@@ -18,15 +18,15 @@ void optimizePrimaryKeyCondition(const Stack & stack)
     const auto & storage_prewhere_info = source_step_with_filter->getPrewhereInfo();
     if (storage_prewhere_info)
     {
-        source_step_with_filter->addFilter(storage_prewhere_info->prewhere_actions->clone(), storage_prewhere_info->prewhere_column_name);
+        source_step_with_filter->addFilter(ActionsDAG::clone(storage_prewhere_info->prewhere_actions), storage_prewhere_info->prewhere_column_name);
         if (storage_prewhere_info->row_level_filter)
-            source_step_with_filter->addFilter(storage_prewhere_info->row_level_filter->clone(), storage_prewhere_info->row_level_column_name);
+            source_step_with_filter->addFilter(ActionsDAG::clone(storage_prewhere_info->row_level_filter), storage_prewhere_info->row_level_column_name);
     }
 
     for (auto iter = stack.rbegin() + 1; iter != stack.rend(); ++iter)
     {
         if (auto * filter_step = typeid_cast<FilterStep *>(iter->node->step.get()))
-            source_step_with_filter->addFilter(filter_step->getExpression()->clone(), filter_step->getFilterColumnName());
+            source_step_with_filter->addFilter(ActionsDAG::clone(filter_step->getExpression()), filter_step->getFilterColumnName());
 
         /// Note: actually, plan optimizations merge Filter and Expression steps.
         /// Ideally, chain should look like (Expression -> ...) -> (Filter -> ...) -> ReadFromStorage,
diff --git a/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp b/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp
index 8e782e68db8..a8bd98d7460 100644
--- a/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp
+++ b/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp
@@ -173,9 +173,9 @@ static void appendFixedColumnsFromFilterExpression(const ActionsDAG::Node & filt
 static void appendExpression(ActionsDAGPtr & dag, const ActionsDAGPtr & expression)
 {
     if (dag)
-        dag->mergeInplace(std::move(*expression->clone()));
+        dag->mergeInplace(std::move(*ActionsDAG::clone(expression)));
     else
-        dag = expression->clone();
+        dag = ActionsDAG::clone(expression);
 }
 
 /// This function builds a common DAG which is a merge of DAGs from Filter and Expression steps chain.
@@ -1066,13 +1066,13 @@ size_t tryReuseStorageOrderingForWindowFunctions(QueryPlan::Node * parent_node,
     for (const auto & actions_dag : window_desc.partition_by_actions)
     {
         order_by_elements_actions.emplace_back(
-            std::make_shared<ExpressionActions>(actions_dag->clone(), ExpressionActionsSettings::fromContext(context, CompileExpressions::yes)));
+            std::make_shared<ExpressionActions>(ActionsDAG::clone(actions_dag.get()), ExpressionActionsSettings::fromContext(context, CompileExpressions::yes)));
     }
 
     for (const auto & actions_dag : window_desc.order_by_actions)
     {
         order_by_elements_actions.emplace_back(
-            std::make_shared<ExpressionActions>(actions_dag->clone(), ExpressionActionsSettings::fromContext(context, CompileExpressions::yes)));
+            std::make_shared<ExpressionActions>(ActionsDAG::clone(actions_dag.get()), ExpressionActionsSettings::fromContext(context, CompileExpressions::yes)));
     }
 
     auto order_optimizer = std::make_shared<ReadInOrderOptimizer>(
diff --git a/src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp b/src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp
index 7c45ef48252..da057bd25c2 100644
--- a/src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp
+++ b/src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp
@@ -77,7 +77,7 @@ static AggregateProjectionInfo getAggregatingProjectionInfo(
 
     AggregateProjectionInfo info;
     info.context = interpreter.getContext();
-    info.before_aggregation = analysis_result.before_aggregation->dag.clone();
+    info.before_aggregation = ActionsDAG::clone(&analysis_result.before_aggregation->dag);
     info.keys = query_analyzer->aggregationKeys().getNames();
     info.aggregates = query_analyzer->aggregates();
 
diff --git a/src/Processors/QueryPlan/Optimizations/projectionsCommon.cpp b/src/Processors/QueryPlan/Optimizations/projectionsCommon.cpp
index d8b40b22904..0e2ad96a419 100644
--- a/src/Processors/QueryPlan/Optimizations/projectionsCommon.cpp
+++ b/src/Processors/QueryPlan/Optimizations/projectionsCommon.cpp
@@ -67,9 +67,9 @@ std::shared_ptr<PartitionIdToMaxBlock> getMaxAddedBlocks(ReadFromMergeTree * rea
 void QueryDAG::appendExpression(const ActionsDAGPtr & expression)
 {
     if (dag)
-        dag->mergeInplace(std::move(*expression->clone()));
+        dag->mergeInplace(std::move(*ActionsDAG::clone(expression)));
     else
-        dag = expression->clone();
+        dag = ActionsDAG::clone(expression);
 }
 
 const ActionsDAG::Node * findInOutputs(ActionsDAG & dag, const std::string & name, bool remove)
@@ -238,7 +238,7 @@ bool analyzeProjectionCandidate(
 
     auto projection_query_info = query_info;
     projection_query_info.prewhere_info = nullptr;
-    projection_query_info.filter_actions_dag = dag->clone();
+    projection_query_info.filter_actions_dag = ActionsDAG::clone(dag);
 
     auto projection_result_ptr = reader.estimateNumMarksToRead(
         std::move(projection_parts),
diff --git a/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp b/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp
index d3c75c988e7..81a8a537830 100644
--- a/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp
+++ b/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp
@@ -70,11 +70,11 @@ namespace
         if (dag_stack.empty())
             return nullptr;
 
-        ActionsDAGPtr path_actions = dag_stack.back()->clone();
+        ActionsDAGPtr path_actions = ActionsDAG::clone(dag_stack.back());
         dag_stack.pop_back();
         while (!dag_stack.empty())
         {
-            ActionsDAGPtr clone = dag_stack.back()->clone();
+            ActionsDAGPtr clone = ActionsDAG::clone(dag_stack.back());
             logActionsDAG("DAG to merge", clone);
             dag_stack.pop_back();
             path_actions->mergeInplace(std::move(*clone));
diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp
index e5370c1c130..d711b3e8472 100644
--- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp
+++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp
@@ -798,7 +798,7 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreams(RangesInDataParts && parts_
                 info.use_uncompressed_cache);
         };
 
-        auto sorting_expr = std::make_shared<ExpressionActions>(metadata_for_reading->getSortingKey().expression->getActionsDAG().clone());
+        auto sorting_expr = std::make_shared<ExpressionActions>(ActionsDAG::clone(&metadata_for_reading->getSortingKey().expression->getActionsDAG()));
 
         SplitPartsWithRangesByPrimaryKeyResult split_ranges_result = splitPartsWithRangesByPrimaryKey(
             metadata_for_reading->getPrimaryKey(),
@@ -1211,7 +1211,7 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal(
     /// we will store lonely parts with level > 0 to use parallel select on them.
     RangesInDataParts non_intersecting_parts_by_primary_key;
 
-    auto sorting_expr = std::make_shared<ExpressionActions>(metadata_for_reading->getSortingKey().expression->getActionsDAG().clone());
+    auto sorting_expr = std::make_shared<ExpressionActions>(ActionsDAG::clone(&metadata_for_reading->getSortingKey().expression->getActionsDAG()));
 
     if (prewhere_info)
     {
@@ -1993,7 +1993,7 @@ void ReadFromMergeTree::initializePipeline(QueryPipelineBuilder & pipeline, cons
 
     if (result.sampling.use_sampling)
     {
-        auto sampling_actions = std::make_shared<ExpressionActions>(result.sampling.filter_expression->clone());
+        auto sampling_actions = std::make_shared<ExpressionActions>(ActionsDAG::clone(result.sampling.filter_expression.get()));
         pipe.addSimpleTransform([&](const Block & header)
         {
             return std::make_shared<FilterTransform>(
@@ -2031,7 +2031,7 @@ void ReadFromMergeTree::initializePipeline(QueryPipelineBuilder & pipeline, cons
 
     if (result_projection)
     {
-        auto projection_actions = std::make_shared<ExpressionActions>(result_projection->clone());
+        auto projection_actions = std::make_shared<ExpressionActions>(ActionsDAG::clone(result_projection));
         pipe.addSimpleTransform([&](const Block & header)
         {
             return std::make_shared<ExpressionTransform>(header, projection_actions);
@@ -2126,7 +2126,7 @@ void ReadFromMergeTree::describeActions(FormatSettings & format_settings) const
                format_settings.out << " (removed)";
             format_settings.out << '\n';
 
-            auto expression = std::make_shared<ExpressionActions>(prewhere_info->prewhere_actions->clone());
+            auto expression = std::make_shared<ExpressionActions>(ActionsDAG::clone(prewhere_info->prewhere_actions));
             expression->describeActions(format_settings.out, prefix);
         }
 
@@ -2135,7 +2135,7 @@ void ReadFromMergeTree::describeActions(FormatSettings & format_settings) const
             format_settings.out << prefix << "Row level filter" << '\n';
             format_settings.out << prefix << "Row level filter column: " << prewhere_info->row_level_column_name << '\n';
 
-            auto expression = std::make_shared<ExpressionActions>(prewhere_info->row_level_filter->clone());
+            auto expression = std::make_shared<ExpressionActions>(ActionsDAG::clone(prewhere_info->row_level_filter));
             expression->describeActions(format_settings.out, prefix);
         }
     }
@@ -2161,7 +2161,7 @@ void ReadFromMergeTree::describeActions(JSONBuilder::JSONMap & map) const
             std::unique_ptr<JSONBuilder::JSONMap> prewhere_filter_map = std::make_unique<JSONBuilder::JSONMap>();
             prewhere_filter_map->add("Prewhere filter column", prewhere_info->prewhere_column_name);
             prewhere_filter_map->add("Prewhere filter remove filter column", prewhere_info->remove_prewhere_column);
-            auto expression = std::make_shared<ExpressionActions>(prewhere_info->prewhere_actions->clone());
+            auto expression = std::make_shared<ExpressionActions>(ActionsDAG::clone(prewhere_info->prewhere_actions));
             prewhere_filter_map->add("Prewhere filter expression", expression->toTree());
 
             prewhere_info_map->add("Prewhere filter", std::move(prewhere_filter_map));
@@ -2171,7 +2171,7 @@ void ReadFromMergeTree::describeActions(JSONBuilder::JSONMap & map) const
         {
             std::unique_ptr<JSONBuilder::JSONMap> row_level_filter_map = std::make_unique<JSONBuilder::JSONMap>();
             row_level_filter_map->add("Row level filter column", prewhere_info->row_level_column_name);
-            auto expression = std::make_shared<ExpressionActions>(prewhere_info->row_level_filter->clone());
+            auto expression = std::make_shared<ExpressionActions>(ActionsDAG::clone(prewhere_info->row_level_filter));
             row_level_filter_map->add("Row level filter expression", expression->toTree());
 
             prewhere_info_map->add("Row level filter", std::move(row_level_filter_map));
diff --git a/src/Processors/QueryPlan/SourceStepWithFilter.cpp b/src/Processors/QueryPlan/SourceStepWithFilter.cpp
index b91debc8239..79b225e7f93 100644
--- a/src/Processors/QueryPlan/SourceStepWithFilter.cpp
+++ b/src/Processors/QueryPlan/SourceStepWithFilter.cpp
@@ -110,7 +110,7 @@ void SourceStepWithFilter::describeActions(FormatSettings & format_settings) con
                 format_settings.out << " (removed)";
             format_settings.out << '\n';
 
-            auto expression = std::make_shared<ExpressionActions>(prewhere_info->prewhere_actions->clone());
+            auto expression = std::make_shared<ExpressionActions>(ActionsDAG::clone(prewhere_info->prewhere_actions));
             expression->describeActions(format_settings.out, prefix);
         }
 
@@ -119,7 +119,7 @@ void SourceStepWithFilter::describeActions(FormatSettings & format_settings) con
             format_settings.out << prefix << "Row level filter" << '\n';
             format_settings.out << prefix << "Row level filter column: " << prewhere_info->row_level_column_name << '\n';
 
-            auto expression = std::make_shared<ExpressionActions>(prewhere_info->row_level_filter->clone());
+            auto expression = std::make_shared<ExpressionActions>(ActionsDAG::clone(prewhere_info->row_level_filter));
             expression->describeActions(format_settings.out, prefix);
         }
     }
@@ -137,7 +137,7 @@ void SourceStepWithFilter::describeActions(JSONBuilder::JSONMap & map) const
             std::unique_ptr<JSONBuilder::JSONMap> prewhere_filter_map = std::make_unique<JSONBuilder::JSONMap>();
             prewhere_filter_map->add("Prewhere filter column", prewhere_info->prewhere_column_name);
             prewhere_filter_map->add("Prewhere filter remove filter column", prewhere_info->remove_prewhere_column);
-            auto expression = std::make_shared<ExpressionActions>(prewhere_info->prewhere_actions->clone());
+            auto expression = std::make_shared<ExpressionActions>(ActionsDAG::clone(prewhere_info->prewhere_actions));
             prewhere_filter_map->add("Prewhere filter expression", expression->toTree());
 
             prewhere_info_map->add("Prewhere filter", std::move(prewhere_filter_map));
@@ -147,7 +147,7 @@ void SourceStepWithFilter::describeActions(JSONBuilder::JSONMap & map) const
         {
             std::unique_ptr<JSONBuilder::JSONMap> row_level_filter_map = std::make_unique<JSONBuilder::JSONMap>();
             row_level_filter_map->add("Row level filter column", prewhere_info->row_level_column_name);
-            auto expression = std::make_shared<ExpressionActions>(prewhere_info->row_level_filter->clone());
+            auto expression = std::make_shared<ExpressionActions>(ActionsDAG::clone(prewhere_info->row_level_filter));
             row_level_filter_map->add("Row level filter expression", expression->toTree());
 
             prewhere_info_map->add("Row level filter", std::move(row_level_filter_map));
diff --git a/src/Processors/QueryPlan/TotalsHavingStep.cpp b/src/Processors/QueryPlan/TotalsHavingStep.cpp
index 45de6c31d24..19632b1862f 100644
--- a/src/Processors/QueryPlan/TotalsHavingStep.cpp
+++ b/src/Processors/QueryPlan/TotalsHavingStep.cpp
@@ -46,7 +46,7 @@ TotalsHavingStep::TotalsHavingStep(
         getTraits(!filter_column_.empty()))
     , aggregates(aggregates_)
     , overflow_row(overflow_row_)
-    , actions_dag(actions_dag_->clone())
+    , actions_dag(ActionsDAG::clone(actions_dag_))
     , filter_column_name(filter_column_)
     , remove_filter(remove_filter_)
     , totals_mode(totals_mode_)
@@ -57,7 +57,7 @@ TotalsHavingStep::TotalsHavingStep(
 
 void TotalsHavingStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings & settings)
 {
-    auto expression_actions = actions_dag ? std::make_shared<ExpressionActions>(actions_dag->clone(), settings.getActionsSettings()) : nullptr;
+    auto expression_actions = actions_dag ? std::make_shared<ExpressionActions>(ActionsDAG::clone(actions_dag), settings.getActionsSettings()) : nullptr;
 
     auto totals_having = std::make_shared<TotalsHavingTransform>(
         pipeline.getHeader(),
@@ -100,7 +100,7 @@ void TotalsHavingStep::describeActions(FormatSettings & settings) const
     if (actions_dag)
     {
         bool first = true;
-        auto expression = std::make_shared<ExpressionActions>(actions_dag->clone());
+        auto expression = std::make_shared<ExpressionActions>(ActionsDAG::clone(actions_dag));
         for (const auto & action : expression->getActions())
         {
             settings.out << prefix << (first ? "Actions: "
@@ -117,7 +117,7 @@ void TotalsHavingStep::describeActions(JSONBuilder::JSONMap & map) const
     if (actions_dag)
     {
         map.add("Filter column", filter_column_name);
-        auto expression = std::make_shared<ExpressionActions>(actions_dag->clone());
+        auto expression = std::make_shared<ExpressionActions>(ActionsDAG::clone(actions_dag));
         map.add("Expression", expression->toTree());
     }
 }
diff --git a/src/Processors/Transforms/FillingTransform.cpp b/src/Processors/Transforms/FillingTransform.cpp
index 95267bc24e0..bbe57fc6441 100644
--- a/src/Processors/Transforms/FillingTransform.cpp
+++ b/src/Processors/Transforms/FillingTransform.cpp
@@ -203,7 +203,7 @@ FillingTransform::FillingTransform(
     , use_with_fill_by_sorting_prefix(use_with_fill_by_sorting_prefix_)
 {
     if (interpolate_description)
-        interpolate_actions = std::make_shared<ExpressionActions>(interpolate_description->actions->clone());
+        interpolate_actions = std::make_shared<ExpressionActions>(ActionsDAG::clone(interpolate_description->actions));
 
     std::vector<bool> is_fill_column(header_.columns());
     for (size_t i = 0, size = fill_description.size(); i < size; ++i)
diff --git a/src/Storages/MergeTree/MergeTreeIndexSet.cpp b/src/Storages/MergeTree/MergeTreeIndexSet.cpp
index 7c65381b05b..8d4ef69b1b9 100644
--- a/src/Storages/MergeTree/MergeTreeIndexSet.cpp
+++ b/src/Storages/MergeTree/MergeTreeIndexSet.cpp
@@ -265,7 +265,7 @@ MergeTreeIndexConditionSet::MergeTreeIndexConditionSet(
         if (!set->buildOrderedSetInplace(context))
             return;
 
-    auto filter_actions_dag = filter_dag->clone();
+    auto filter_actions_dag = ActionsDAG::clone(filter_dag);
     const auto * filter_actions_dag_node = filter_actions_dag->getOutputs().at(0);
 
     std::unordered_map<const ActionsDAG::Node *, const ActionsDAG::Node *> node_to_result_node;
@@ -319,7 +319,7 @@ static const ActionsDAG::NodeRawConstPtrs & getArguments(const ActionsDAG::Node
         return index_hint.getActions()->getOutputs();
 
     /// Import the DAG and map argument pointers.
-    ActionsDAGPtr actions_clone = index_hint.getActions()->clone();
+    ActionsDAGPtr actions_clone = ActionsDAG::clone(index_hint.getActions());
     chassert(storage);
     result_dag_or_null->mergeNodes(std::move(*actions_clone), storage);
     return *storage;
diff --git a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp
index 8fa5b2cc955..e924f853524 100644
--- a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp
+++ b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp
@@ -80,7 +80,7 @@ PrewhereExprInfo MergeTreeSelectProcessor::getPrewhereActions(PrewhereInfoPtr pr
             PrewhereExprStep row_level_filter_step
             {
                 .type = PrewhereExprStep::Filter,
-                .actions = std::make_shared<ExpressionActions>(prewhere_info->row_level_filter->clone(), actions_settings),
+                .actions = std::make_shared<ExpressionActions>(ActionsDAG::clone(prewhere_info->row_level_filter), actions_settings),
                 .filter_column_name = prewhere_info->row_level_column_name,
                 .remove_filter_column = true,
                 .need_filter = true,
@@ -96,7 +96,7 @@ PrewhereExprInfo MergeTreeSelectProcessor::getPrewhereActions(PrewhereInfoPtr pr
             PrewhereExprStep prewhere_step
             {
                 .type = PrewhereExprStep::Filter,
-                .actions = std::make_shared<ExpressionActions>(prewhere_info->prewhere_actions->clone(), actions_settings),
+                .actions = std::make_shared<ExpressionActions>(ActionsDAG::clone(prewhere_info->prewhere_actions), actions_settings),
                 .filter_column_name = prewhere_info->prewhere_column_name,
                 .remove_filter_column = prewhere_info->remove_prewhere_column,
                 .need_filter = prewhere_info->need_filter,
diff --git a/src/Storages/SelectQueryInfo.h b/src/Storages/SelectQueryInfo.h
index 654b8b788fe..0b7035504ae 100644
--- a/src/Storages/SelectQueryInfo.h
+++ b/src/Storages/SelectQueryInfo.h
@@ -66,10 +66,10 @@ struct PrewhereInfo
         PrewhereInfoPtr prewhere_info = std::make_shared<PrewhereInfo>();
 
         if (row_level_filter)
-            prewhere_info->row_level_filter = row_level_filter->clone();
+            prewhere_info->row_level_filter = ActionsDAG::clone(row_level_filter);
 
         if (prewhere_actions)
-            prewhere_info->prewhere_actions = prewhere_actions->clone();
+            prewhere_info->prewhere_actions = ActionsDAG::clone(prewhere_actions);
 
         prewhere_info->row_level_column_name = row_level_column_name;
         prewhere_info->prewhere_column_name = prewhere_column_name;
diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp
index 9bddf4f0230..695b31d0c80 100644
--- a/src/Storages/StorageBuffer.cpp
+++ b/src/Storages/StorageBuffer.cpp
@@ -312,7 +312,7 @@ void StorageBuffer::read(
                     if (src_table_query_info.prewhere_info->row_level_filter)
                     {
                         src_table_query_info.prewhere_info->row_level_filter = ActionsDAG::merge(
-                            std::move(*actions_dag->clone()),
+                            std::move(*ActionsDAG::clone(actions_dag)),
                             std::move(*src_table_query_info.prewhere_info->row_level_filter));
 
                         src_table_query_info.prewhere_info->row_level_filter->removeUnusedActions();
@@ -321,7 +321,7 @@ void StorageBuffer::read(
                     if (src_table_query_info.prewhere_info->prewhere_actions)
                     {
                         src_table_query_info.prewhere_info->prewhere_actions = ActionsDAG::merge(
-                            std::move(*actions_dag->clone()),
+                            std::move(*ActionsDAG::clone(actions_dag)),
                             std::move(*src_table_query_info.prewhere_info->prewhere_actions));
 
                         src_table_query_info.prewhere_info->prewhere_actions->removeUnusedActions();
@@ -432,7 +432,7 @@ void StorageBuffer::read(
                 {
                     return std::make_shared<FilterTransform>(
                             header,
-                            std::make_shared<ExpressionActions>(query_info.prewhere_info->row_level_filter->clone(), actions_settings),
+                            std::make_shared<ExpressionActions>(ActionsDAG::clone(query_info.prewhere_info->row_level_filter), actions_settings),
                             query_info.prewhere_info->row_level_column_name,
                             false);
                 });
@@ -442,7 +442,7 @@ void StorageBuffer::read(
             {
                 return std::make_shared<FilterTransform>(
                         header,
-                        std::make_shared<ExpressionActions>(query_info.prewhere_info->prewhere_actions->clone(), actions_settings),
+                        std::make_shared<ExpressionActions>(ActionsDAG::clone(query_info.prewhere_info->prewhere_actions), actions_settings),
                         query_info.prewhere_info->prewhere_column_name,
                         query_info.prewhere_info->remove_prewhere_column);
             });
diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp
index c42e3058347..d21a6dc20dd 100644
--- a/src/Storages/StorageMerge.cpp
+++ b/src/Storages/StorageMerge.cpp
@@ -662,7 +662,7 @@ std::vector<ReadFromMerge::ChildPlan> ReadFromMerge::createChildrenPlans(SelectQ
             {
                 auto filter_step = std::make_unique<FilterStep>(
                     child.plan.getCurrentDataStream(),
-                    filter_info.actions->clone(),
+                    ActionsDAG::clone(filter_info.actions),
                     filter_info.column_name,
                     filter_info.do_remove_column);
 
@@ -1273,12 +1273,12 @@ void ReadFromMerge::RowPolicyData::extendNames(Names & names) const
 
 void ReadFromMerge::RowPolicyData::addStorageFilter(SourceStepWithFilter * step) const
 {
-    step->addFilter(actions_dag->clone(), filter_column_name);
+    step->addFilter(ActionsDAG::clone(actions_dag), filter_column_name);
 }
 
 void ReadFromMerge::RowPolicyData::addFilterTransform(QueryPlan & plan) const
 {
-    auto filter_step = std::make_unique<FilterStep>(plan.getCurrentDataStream(), actions_dag->clone(), filter_column_name, true /* remove filter column */);
+    auto filter_step = std::make_unique<FilterStep>(plan.getCurrentDataStream(), ActionsDAG::clone(actions_dag), filter_column_name, true /* remove filter column */);
     plan.addStep(std::move(filter_step));
 }
 
diff --git a/src/Storages/VirtualColumnUtils.cpp b/src/Storages/VirtualColumnUtils.cpp
index 6f7d1d4c39f..1bd5e80a4f9 100644
--- a/src/Storages/VirtualColumnUtils.cpp
+++ b/src/Storages/VirtualColumnUtils.cpp
@@ -80,7 +80,7 @@ void buildSetsForDAG(const ActionsDAGPtr & dag, const ContextPtr & context)
 void filterBlockWithDAG(const ActionsDAGPtr & dag, Block & block, ContextPtr context)
 {
     buildSetsForDAG(dag, context);
-    auto actions = std::make_shared<ExpressionActions>(dag->clone());
+    auto actions = std::make_shared<ExpressionActions>(ActionsDAG::clone(dag));
     Block block_with_filter = block;
     actions->execute(block_with_filter, /*dry_run=*/ false, /*allow_duplicates_in_input=*/ true);
 
@@ -318,7 +318,7 @@ static const ActionsDAG::Node * splitFilterNodeForAllowedInputs(
             {
                 if (const auto * index_hint = typeid_cast<const FunctionIndexHint *>(adaptor->getFunction().get()))
                 {
-                    auto index_hint_dag = index_hint->getActions()->clone();
+                    auto index_hint_dag = ActionsDAG::clone(index_hint->getActions());
                     ActionsDAG::NodeRawConstPtrs atoms;
                     for (const auto & output : index_hint_dag->getOutputs())
                         if (const auto * child_copy = splitFilterNodeForAllowedInputs(output, allowed_inputs, additional_nodes))

From 068e1c55545dfc4e004b8b5970b3f237033444cd Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Tue, 25 Jun 2024 15:59:00 +0000
Subject: [PATCH 0094/1488] Fixing build.

---
 src/Processors/QueryPlan/Optimizations/mergeExpressions.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Processors/QueryPlan/Optimizations/mergeExpressions.cpp b/src/Processors/QueryPlan/Optimizations/mergeExpressions.cpp
index 6ace1b3b5ce..97de69b1134 100644
--- a/src/Processors/QueryPlan/Optimizations/mergeExpressions.cpp
+++ b/src/Processors/QueryPlan/Optimizations/mergeExpressions.cpp
@@ -84,12 +84,12 @@ size_t tryMergeExpressions(QueryPlan::Node * parent_node, QueryPlan::Nodes &)
         if (child_actions->hasArrayJoin())
             return 0;
 
-        auto actions = child_actions->clone();
+        auto actions = ActionsDAG::clone(child_actions);
         const auto & child_filter_node = actions->findInOutputs(child_filter->getFilterColumnName());
         if (child_filter->removesFilterColumn())
             removeFromOutputs(*actions, child_filter_node);
 
-        actions->mergeInplace(std::move(*parent_actions->clone()));
+        actions->mergeInplace(std::move(*ActionsDAG::clone(parent_actions)));
 
         const auto & parent_filter_node = actions->findInOutputs(parent_filter->getFilterColumnName());
         if (parent_filter->removesFilterColumn())

From 3149e51e9254b268c41ade796d3652d0c2dec8f7 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Tue, 25 Jun 2024 17:36:07 +0000
Subject: [PATCH 0095/1488] Fix other crashes.

---
 src/Storages/WindowView/StorageWindowView.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Storages/WindowView/StorageWindowView.cpp b/src/Storages/WindowView/StorageWindowView.cpp
index 5822e46f9f8..8f39f0da5af 100644
--- a/src/Storages/WindowView/StorageWindowView.cpp
+++ b/src/Storages/WindowView/StorageWindowView.cpp
@@ -563,11 +563,11 @@ std::pair<BlocksPtr, Block> StorageWindowView::getNewBlocks(UInt32 watermark)
 
     auto syntax_result = TreeRewriter(getContext()).analyze(filter_function, builder.getHeader().getNamesAndTypesList());
     auto filter_expression = ExpressionAnalyzer(filter_function, syntax_result, getContext()).getActionsDAG(false);
+    auto filter_actions = std::make_shared<ExpressionActions>(std::move(filter_expression));
 
     builder.addSimpleTransform([&](const Block & header)
     {
-        return std::make_shared<FilterTransform>(
-            header, std::make_shared<ExpressionActions>(std::move(filter_expression)), filter_function->getColumnName(), true);
+        return std::make_shared<FilterTransform>(header, filter_actions, filter_function->getColumnName(), true);
     });
 
     /// Adding window column

From 6c0c6fd9fe452631467fcb4da4bfd764bb487bea Mon Sep 17 00:00:00 2001
From: JackyWoo <wuchienchao@qq.com>
Date: Tue, 25 Jun 2024 14:54:50 +0800
Subject: [PATCH 0096/1488] Fix error caused by type miss match

---
 src/Storages/Statistics/Statistics.cpp           | 16 ++++++++++++++++
 src/Storages/Statistics/Statistics.h             |  2 ++
 .../03174_statistics_countminsketch.reference    | 14 +++++++-------
 .../03174_statistics_countminsketch.sql          | 14 +++++++-------
 4 files changed, 32 insertions(+), 14 deletions(-)

diff --git a/src/Storages/Statistics/Statistics.cpp b/src/Storages/Statistics/Statistics.cpp
index 2404d234d7a..7afd8b8cd12 100644
--- a/src/Storages/Statistics/Statistics.cpp
+++ b/src/Storages/Statistics/Statistics.cpp
@@ -49,6 +49,20 @@ std::optional<String> getString(const Field & f)
     return {};
 }
 
+bool checkType(const Field & f)
+{
+    switch (f.getType())
+    {
+        case Field::Types::Int64:
+        case Field::Types::UInt64:
+        case Field::Types::Float64:
+        case Field::Types::String:
+            return true;
+        default:
+            return false;
+    }
+}
+
 IStatistics::IStatistics(const SingleStatisticsDescription & stat_) : stat(stat_) {}
 
 ColumnStatistics::ColumnStatistics(const ColumnStatisticsDescription & stats_desc_)
@@ -94,6 +108,8 @@ Float64 ColumnStatistics::estimateEqual(Field val) const
 #if USE_DATASKETCHES
     if (stats.contains(StatisticsType::CountMinSketch))
     {
+        if (!checkType(val))
+            return rows * ConditionSelectivityEstimator::default_normal_cond_factor;
         auto count_min_sketch_static = std::static_pointer_cast<CountMinSketchStatistics>(stats.at(StatisticsType::CountMinSketch));
         return count_min_sketch_static->estimateEqual(val);
     }
diff --git a/src/Storages/Statistics/Statistics.h b/src/Storages/Statistics/Statistics.h
index f6121d72256..b56a1959ea0 100644
--- a/src/Storages/Statistics/Statistics.h
+++ b/src/Storages/Statistics/Statistics.h
@@ -103,4 +103,6 @@ private:
 
 std::optional<Float64> getFloat64(const Field & f);
 std::optional<String> getString(const Field & f);
+bool checkType(const Field & f);
+
 }
diff --git a/tests/queries/0_stateless/03174_statistics_countminsketch.reference b/tests/queries/0_stateless/03174_statistics_countminsketch.reference
index 3cec7dd7168..b5a35c22835 100644
--- a/tests/queries/0_stateless/03174_statistics_countminsketch.reference
+++ b/tests/queries/0_stateless/03174_statistics_countminsketch.reference
@@ -1,26 +1,26 @@
 CREATE TABLE default.t1\n(\n    `a` String STATISTICS(countminsketch),\n    `b` Int64 STATISTICS(countminsketch),\n    `c` UInt64 STATISTICS(countminsketch),\n    `pk` String\n)\nENGINE = MergeTree\nORDER BY pk\nSETTINGS min_bytes_for_wide_part = 0, index_granularity = 8192
         Prewhere info
           Prewhere filter
-          Prewhere filter column: and(equals(a, \'0\'_String), equals(b, 0), equals(c, 0)) (removed)
+          Prewhere filter column: and(equals(a, \'0\'), equals(b, 0), equals(c, 0)) (removed)
         Prewhere info
           Prewhere filter
-          Prewhere filter column: and(equals(a, \'0\'_String), equals(c, 0), greater(b, 0)) (removed)
+          Prewhere filter column: and(equals(a, \'0\'), equals(c, 0), greater(b, 0)) (removed)
 After drop statistics for a
         Prewhere info
           Prewhere filter
-          Prewhere filter column: and(equals(b, 0), equals(c, 0), equals(a, \'0\'_String)) (removed)
+          Prewhere filter column: and(equals(b, 0), equals(c, 0), equals(a, \'0\')) (removed)
         Prewhere info
           Prewhere filter
-          Prewhere filter column: and(equals(c, 0), equals(a, \'0\'_String), greater(b, 0)) (removed)
+          Prewhere filter column: and(equals(c, 0), equals(a, \'0\'), greater(b, 0)) (removed)
 LowCardinality
         Prewhere info
           Prewhere filter
-          Prewhere filter column: and(equals(a, \'0\'_String), equals(b, 0), equals(c, 0)) (removed)
+          Prewhere filter column: and(equals(a, \'0\'), equals(b, 0), equals(c, 0)) (removed)
 Nullable
         Prewhere info
           Prewhere filter
-          Prewhere filter column: and(equals(a, \'0\'_String), equals(b, 0), equals(c, 0)) (removed)
+          Prewhere filter column: and(equals(a, \'0\'), equals(b, 0), equals(c, 0)) (removed)
 LowCardinality(Nullable)
         Prewhere info
           Prewhere filter
-          Prewhere filter column: and(equals(a, \'0\'_String), equals(b, 0), equals(c, 0)) (removed)
+          Prewhere filter column: and(equals(a, \'0\'), equals(b, 0), equals(c, 0)) (removed)
diff --git a/tests/queries/0_stateless/03174_statistics_countminsketch.sql b/tests/queries/0_stateless/03174_statistics_countminsketch.sql
index 0d9673309b4..200c2a4a531 100644
--- a/tests/queries/0_stateless/03174_statistics_countminsketch.sql
+++ b/tests/queries/0_stateless/03174_statistics_countminsketch.sql
@@ -17,14 +17,14 @@ SHOW CREATE TABLE t1;
 
 INSERT INTO t1 select toString(number % 1000), number % 100, number % 10, generateUUIDv4() FROM system.numbers LIMIT 10000;
 
-SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE c = 0 and b = 0 and a = '0') WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
-SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE c = 0 and b > 0 and a = '0') WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
+SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_String', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE c = 0 and b = 0 and a = '0') WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
+SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_String', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE c = 0 and b > 0 and a = '0') WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
 
 ALTER TABLE t1 DROP STATISTICS a;
 
 SELECT 'After drop statistics for a';
-SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE c = 0 and b = 0 and a = '0') WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
-SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE c = 0 and b > 0 and a = '0') WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
+SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_String', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE c = 0 and b = 0 and a = '0') WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
+SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_String', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE c = 0 and b > 0 and a = '0') WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
 
 DROP TABLE IF EXISTS t1;
 DROP TABLE IF EXISTS t2;
@@ -40,7 +40,7 @@ SETTINGS min_bytes_for_wide_part = 0;
 INSERT INTO t2 select toString(number % 1000), number % 100, number % 10, generateUUIDv4() FROM system.numbers LIMIT 10000;
 
 SELECT 'LowCardinality';
-SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t2 WHERE c = 0 and b = 0 and a = '0') WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
+SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_String', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t2 WHERE c = 0 and b = 0 and a = '0') WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
 
 
 DROP TABLE IF EXISTS t2;
@@ -57,7 +57,7 @@ SETTINGS min_bytes_for_wide_part = 0;
 INSERT INTO t3 select toString(number % 1000), number % 100, number % 10, generateUUIDv4() FROM system.numbers LIMIT 10000;
 
 SELECT 'Nullable';
-SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t3 WHERE c = 0 and b = 0 and a = '0') WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
+SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_String', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t3 WHERE c = 0 and b = 0 and a = '0') WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
 
 DROP TABLE IF EXISTS t3;
 DROP TABLE IF EXISTS t4;
@@ -73,6 +73,6 @@ SETTINGS min_bytes_for_wide_part = 0;
 INSERT INTO t4 select toString(number % 1000), number % 100, number % 10, generateUUIDv4() FROM system.numbers LIMIT 10000;
 
 SELECT 'LowCardinality(Nullable)';
-SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t4 WHERE c = 0 and b = 0 and a = '0') WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
+SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_String', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t4 WHERE c = 0 and b = 0 and a = '0') WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
 
 DROP TABLE IF EXISTS t4;

From 7b2868a1ef6ea32b4bdaac8c13186868a9aec4fe Mon Sep 17 00:00:00 2001
From: JackyWoo <wuchienchao@qq.com>
Date: Wed, 26 Jun 2024 10:09:04 +0800
Subject: [PATCH 0097/1488] A little code refactoring

---
 .../ConditionSelectivityEstimator.cpp         |  4 +--
 .../Statistics/CountMinSketchStatistics.cpp   | 31 ++++++++++++++-----
 .../Statistics/CountMinSketchStatistics.h     |  2 ++
 src/Storages/Statistics/Statistics.cpp        | 27 +++++-----------
 src/Storages/Statistics/Statistics.h          |  7 ++---
 5 files changed, 37 insertions(+), 34 deletions(-)

diff --git a/src/Storages/Statistics/ConditionSelectivityEstimator.cpp b/src/Storages/Statistics/ConditionSelectivityEstimator.cpp
index 73c5c549a5d..437b39cb537 100644
--- a/src/Storages/Statistics/ConditionSelectivityEstimator.cpp
+++ b/src/Storages/Statistics/ConditionSelectivityEstimator.cpp
@@ -37,7 +37,7 @@ Float64 ConditionSelectivityEstimator::ColumnSelectivityEstimator::estimateGreat
 
 Float64 ConditionSelectivityEstimator::ColumnSelectivityEstimator::estimateEqual(Field val, Float64 rows) const
 {
-    auto float_val = getFloat64(val);
+    auto float_val = IStatistics::getFloat64(val);
     if (part_statistics.empty())
     {
         if (!float_val)
@@ -148,7 +148,7 @@ Float64 ConditionSelectivityEstimator::estimateRowCount(const RPNBuilderTreeNode
     else
         dummy = true;
     auto [op, val] = extractBinaryOp(node, col);
-    auto float_val = getFloat64(val);
+    auto float_val = IStatistics::getFloat64(val);
     if (op == "equals")
     {
         if (dummy)
diff --git a/src/Storages/Statistics/CountMinSketchStatistics.cpp b/src/Storages/Statistics/CountMinSketchStatistics.cpp
index d0f52e3f6df..53415ff3946 100644
--- a/src/Storages/Statistics/CountMinSketchStatistics.cpp
+++ b/src/Storages/Statistics/CountMinSketchStatistics.cpp
@@ -22,13 +22,27 @@ CountMinSketchStatistics::CountMinSketchStatistics(const SingleStatisticsDescrip
 
 Float64 CountMinSketchStatistics::estimateEqual(const Field & value) const
 {
-    if (auto float_val = getFloat64(value))
+    if (auto float_val = IStatistics::getFloat64(value))
         return data.get_estimate(&float_val.value(), 8);
-    if (auto string_val = getString(value))
+    if (auto string_val = IStatistics::getString(value))
         return data.get_estimate(string_val->data(), string_val->size());
     UNREACHABLE();
 }
 
+bool CountMinSketchStatistics::checkType(const Field & f)
+{
+    switch (f.getType())
+    {
+        case Field::Types::Int64:
+        case Field::Types::UInt64:
+        case Field::Types::Float64:
+        case Field::Types::String:
+            return true;
+        default:
+            return false;
+    }
+}
+
 void CountMinSketchStatistics::serialize(WriteBuffer & buf)
 {
     auto bytes = data.serialize();
@@ -55,12 +69,13 @@ void CountMinSketchStatistics::update(const ColumnPtr & column)
     {
         Field f;
         column->get(i, f);
-        if (f.isNull())
-            continue;
-        if (auto float_val = getFloat64(f))
-            data.update(&float_val.value(), 8, 1.0);
-        else if (auto string_val = getString(f))
-            data.update(*string_val, 1.0);
+        if (checkType(f))
+        {
+            if (auto float_val = IStatistics::getFloat64(f))
+                data.update(&float_val.value(), 8, 1.0);
+            else if (auto string_val = IStatistics::getString(f))
+                data.update(*string_val, 1.0);
+        }
     }
 }
 
diff --git a/src/Storages/Statistics/CountMinSketchStatistics.h b/src/Storages/Statistics/CountMinSketchStatistics.h
index 23ea2cf25c9..37f0aa48749 100644
--- a/src/Storages/Statistics/CountMinSketchStatistics.h
+++ b/src/Storages/Statistics/CountMinSketchStatistics.h
@@ -23,6 +23,8 @@ public:
 
     void update(const ColumnPtr & column) override;
 
+    bool checkType(const Field & f);
+
 private:
     static constexpr size_t HASH_COUNT = 8;
     static constexpr size_t BUCKET_COUNT = 2048;
diff --git a/src/Storages/Statistics/Statistics.cpp b/src/Storages/Statistics/Statistics.cpp
index 7afd8b8cd12..9b184f14bfb 100644
--- a/src/Storages/Statistics/Statistics.cpp
+++ b/src/Storages/Statistics/Statistics.cpp
@@ -27,7 +27,7 @@ enum StatisticsFileVersion : UInt16
     V0 = 0,
 };
 
-std::optional<Float64> getFloat64(const Field & f)
+std::optional<Float64> IStatistics::getFloat64(const Field & f)
 {
     const auto type = f.getType();
     Float64 value;
@@ -42,27 +42,13 @@ std::optional<Float64> getFloat64(const Field & f)
     return value;
 }
 
-std::optional<String> getString(const Field & f)
+std::optional<String> IStatistics::getString(const Field & f)
 {
     if (f.getType() == Field::Types::String)
         return f.get<String>();
     return {};
 }
 
-bool checkType(const Field & f)
-{
-    switch (f.getType())
-    {
-        case Field::Types::Int64:
-        case Field::Types::UInt64:
-        case Field::Types::Float64:
-        case Field::Types::String:
-            return true;
-        default:
-            return false;
-    }
-}
-
 IStatistics::IStatistics(const SingleStatisticsDescription & stat_) : stat(stat_) {}
 
 ColumnStatistics::ColumnStatistics(const ColumnStatisticsDescription & stats_desc_)
@@ -93,7 +79,7 @@ Float64 ColumnStatistics::estimateGreater(Float64 val) const
 
 Float64 ColumnStatistics::estimateEqual(Field val) const
 {
-    auto float_val = getFloat64(val);
+    auto float_val = IStatistics::getFloat64(val);
     if (float_val && stats.contains(StatisticsType::Uniq) && stats.contains(StatisticsType::TDigest))
     {
         auto uniq_static = std::static_pointer_cast<UniqStatistics>(stats.at(StatisticsType::Uniq));
@@ -108,10 +94,11 @@ Float64 ColumnStatistics::estimateEqual(Field val) const
 #if USE_DATASKETCHES
     if (stats.contains(StatisticsType::CountMinSketch))
     {
-        if (!checkType(val))
-            return rows * ConditionSelectivityEstimator::default_normal_cond_factor;
         auto count_min_sketch_static = std::static_pointer_cast<CountMinSketchStatistics>(stats.at(StatisticsType::CountMinSketch));
-        return count_min_sketch_static->estimateEqual(val);
+        if (!count_min_sketch_static->checkType(val))
+            return rows * ConditionSelectivityEstimator::default_normal_cond_factor;
+        else
+            return count_min_sketch_static->estimateEqual(val);
     }
 #endif
     if (val < - ConditionSelectivityEstimator::threshold || val > ConditionSelectivityEstimator::threshold)
diff --git a/src/Storages/Statistics/Statistics.h b/src/Storages/Statistics/Statistics.h
index b56a1959ea0..b6df6e45b70 100644
--- a/src/Storages/Statistics/Statistics.h
+++ b/src/Storages/Statistics/Statistics.h
@@ -35,6 +35,9 @@ public:
 
     virtual void update(const ColumnPtr & column) = 0;
 
+    static std::optional<Float64> getFloat64(const Field & f);
+    static std::optional<String> getString(const Field & f);
+
 protected:
     SingleStatisticsDescription stat;
 };
@@ -101,8 +104,4 @@ private:
     Validators validators;
 };
 
-std::optional<Float64> getFloat64(const Field & f);
-std::optional<String> getString(const Field & f);
-bool checkType(const Field & f);
-
 }

From cc67efd7898483fe165eb0e72e8e61818027aab0 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Wed, 26 Jun 2024 12:39:50 +0000
Subject: [PATCH 0098/1488] Some fixups

---
 .../mergetree-family/mergetree.md             | 11 +++++++---
 .../statements/alter/statistics.md            |  4 ++--
 src/CMakeLists.txt                            |  3 +--
 .../Statistics/CountMinSketchStatistics.cpp   | 20 ++++++++++---------
 .../Statistics/CountMinSketchStatistics.h     | 16 +++++++--------
 src/Storages/Statistics/Statistics.cpp        |  2 ++
 src/Storages/Statistics/Statistics.h          |  1 -
 src/Storages/Statistics/TDigestStatistics.h   |  1 -
 src/Storages/Statistics/UniqStatistics.h      |  1 -
 src/Storages/StatisticsDescription.cpp        |  8 ++++----
 10 files changed, 36 insertions(+), 31 deletions(-)

diff --git a/docs/en/engines/table-engines/mergetree-family/mergetree.md b/docs/en/engines/table-engines/mergetree-family/mergetree.md
index f0c4e1b0e34..bb681faaf1e 100644
--- a/docs/en/engines/table-engines/mergetree-family/mergetree.md
+++ b/docs/en/engines/table-engines/mergetree-family/mergetree.md
@@ -991,13 +991,18 @@ They can be used for prewhere optimization only if we enable `set allow_statisti
 
 #### Available Types of Column Statistics {#available-types-of-column-statistics}
 
+
 - `TDigest`
 
-    Stores distribution of values from numeric columns in [TDigest](https://github.com/tdunning/t-digest) sketch.
+    [TDigest](https://github.com/tdunning/t-digest) sketches which allow to compute approximate percentiles (e.g. the 90th percentile) for numeric columns.
 
 - `Uniq`
-    
-    Estimate the number of distinct values of a column by HyperLogLog.
+
+    [HyperLogLog](https://en.wikipedia.org/wiki/HyperLogLog) sketches which provide an estimation how many distinct values a column contains.
+
+- `CountMin`
+
+    [Count-min](https://en.wikipedia.org/wiki/Count%E2%80%93min_sketch) sketches which provide an approximate count of the frequency of each value in a column.
 
 ## Column-level Settings {#column-level-settings}
 
diff --git a/docs/en/sql-reference/statements/alter/statistics.md b/docs/en/sql-reference/statements/alter/statistics.md
index 0d1fa59cf86..6880cef0e5c 100644
--- a/docs/en/sql-reference/statements/alter/statistics.md
+++ b/docs/en/sql-reference/statements/alter/statistics.md
@@ -25,9 +25,9 @@ Also, they are replicated, syncing statistics metadata via ZooKeeper.
 There is an example adding two statistics types to two columns:
 
 ```
-ALTER TABLE t1 MODIFY STATISTICS c, d TYPE TDigest, Uniq, CountMinSketch;
+ALTER TABLE t1 MODIFY STATISTICS c, d TYPE TDigest, Uniq;
 ```
 
-:::note    
+:::note
 Statistic manipulation is supported only for tables with [`*MergeTree`](../../../engines/table-engines/mergetree-family/mergetree.md) engine (including [replicated](../../../engines/table-engines/mergetree-family/replication.md) variants).
 :::
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 4efb6004172..b7138d0700e 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -545,8 +545,7 @@ if (TARGET ch_contrib::libpqxx)
 endif()
 
 if (TARGET ch_contrib::datasketches)
-    target_link_libraries (clickhouse_aggregate_functions PRIVATE ch_contrib::datasketches)
-    dbms_target_link_libraries(PRIVATE ch_contrib::datasketches)
+    dbms_target_link_libraries(PUBLIC ch_contrib::datasketches)
 endif ()
 
 target_link_libraries (clickhouse_common_io PRIVATE ch_contrib::lz4)
diff --git a/src/Storages/Statistics/CountMinSketchStatistics.cpp b/src/Storages/Statistics/CountMinSketchStatistics.cpp
index 53415ff3946..03c76c9709b 100644
--- a/src/Storages/Statistics/CountMinSketchStatistics.cpp
+++ b/src/Storages/Statistics/CountMinSketchStatistics.cpp
@@ -1,8 +1,8 @@
+#include <Storages/Statistics/CountMinSketchStatistics.h>
 #include <DataTypes/DataTypeLowCardinality.h>
 #include <DataTypes/DataTypeNullable.h>
 #include <IO/ReadHelpers.h>
 #include <IO/WriteHelpers.h>
-#include <Storages/Statistics/CountMinSketchStatistics.h>
 
 #if USE_DATASKETCHES
 
@@ -16,16 +16,18 @@ extern const int ILLEGAL_STATISTICS;
 
 
 CountMinSketchStatistics::CountMinSketchStatistics(const SingleStatisticsDescription & stat_, DataTypePtr data_type_)
-    : IStatistics(stat_), data(HASH_COUNT, BUCKET_COUNT), data_type(data_type_)
+    : IStatistics(stat_)
+    , sketch(HASH_COUNT, BUCKET_COUNT)
+    , data_type(data_type_)
 {
 }
 
 Float64 CountMinSketchStatistics::estimateEqual(const Field & value) const
 {
     if (auto float_val = IStatistics::getFloat64(value))
-        return data.get_estimate(&float_val.value(), 8);
+        return sketch.get_estimate(&float_val.value(), 8);
     if (auto string_val = IStatistics::getString(value))
-        return data.get_estimate(string_val->data(), string_val->size());
+        return sketch.get_estimate(string_val->data(), string_val->size());
     UNREACHABLE();
 }
 
@@ -45,7 +47,7 @@ bool CountMinSketchStatistics::checkType(const Field & f)
 
 void CountMinSketchStatistics::serialize(WriteBuffer & buf)
 {
-    auto bytes = data.serialize();
+    auto bytes = sketch.serialize();
     writeIntBinary(static_cast<UInt64>(bytes.size()), buf);
     buf.write(reinterpret_cast<const char *>(bytes.data()), bytes.size());
 }
@@ -58,7 +60,7 @@ void CountMinSketchStatistics::deserialize(ReadBuffer & buf)
     s.reserve(size);
     buf.readStrict(s.data(), size); /// Extra copy can be avoided by implementing count_min_sketch<Float64>::deserialize with ReadBuffer
     auto read_sketch = datasketches::count_min_sketch<Float64>::deserialize(s.data(), size, datasketches::DEFAULT_SEED);
-    data.merge(read_sketch);
+    sketch.merge(read_sketch);
 }
 
 void CountMinSketchStatistics::update(const ColumnPtr & column)
@@ -72,9 +74,9 @@ void CountMinSketchStatistics::update(const ColumnPtr & column)
         if (checkType(f))
         {
             if (auto float_val = IStatistics::getFloat64(f))
-                data.update(&float_val.value(), 8, 1.0);
+                sketch.update(&float_val.value(), 8, 1.0);
             else if (auto string_val = IStatistics::getString(f))
-                data.update(*string_val, 1.0);
+                sketch.update(*string_val, 1.0);
         }
     }
 }
@@ -84,7 +86,7 @@ void CountMinSketchValidator(const SingleStatisticsDescription &, DataTypePtr da
     data_type = removeNullable(data_type);
     data_type = removeLowCardinalityAndNullable(data_type);
     if (!data_type->isValueRepresentedByNumber() && !isStringOrFixedString(data_type))
-        throw Exception(ErrorCodes::ILLEGAL_STATISTICS, "Statistics of type 'countminsketch' does not support type {}", data_type->getName());
+        throw Exception(ErrorCodes::ILLEGAL_STATISTICS, "Statistics of type 'countmin' does not support type {}", data_type->getName());
 }
 
 StatisticsPtr CountMinSketchCreator(const SingleStatisticsDescription & stat, DataTypePtr data_type)
diff --git a/src/Storages/Statistics/CountMinSketchStatistics.h b/src/Storages/Statistics/CountMinSketchStatistics.h
index 37f0aa48749..a112de4b3aa 100644
--- a/src/Storages/Statistics/CountMinSketchStatistics.h
+++ b/src/Storages/Statistics/CountMinSketchStatistics.h
@@ -1,24 +1,24 @@
 #pragma once
 
+#include <Storages/Statistics/Statistics.h>
+
+#include "config.h"
+
 #if USE_DATASKETCHES
 
-#include <Storages/Statistics/Statistics.h>
 #include <count_min.hpp>
-#include <Common/Allocator.h>
 
 namespace DB
 {
 
-/// CountMinSketchStatistics is used to estimate expression like col = 'value' or col in ('v1', 'v2').
 class CountMinSketchStatistics : public IStatistics
 {
 public:
-    explicit CountMinSketchStatistics(const SingleStatisticsDescription & stat_, DataTypePtr data_type_);
+    CountMinSketchStatistics(const SingleStatisticsDescription & stat_, DataTypePtr data_type_);
 
     Float64 estimateEqual(const Field & value) const;
 
     void serialize(WriteBuffer & buf) override;
-
     void deserialize(ReadBuffer & buf) override;
 
     void update(const ColumnPtr & column) override;
@@ -26,10 +26,10 @@ public:
     bool checkType(const Field & f);
 
 private:
-    static constexpr size_t HASH_COUNT = 8;
-    static constexpr size_t BUCKET_COUNT = 2048;
+    static constexpr auto HASH_COUNT = 8uz;
+    static constexpr auto BUCKET_COUNT = 2048uz;
 
-    datasketches::count_min_sketch<Float64> data;
+    datasketches::count_min_sketch<Float64> sketch;
     DataTypePtr data_type;
 };
 
diff --git a/src/Storages/Statistics/Statistics.cpp b/src/Storages/Statistics/Statistics.cpp
index 9b184f14bfb..36339d7456f 100644
--- a/src/Storages/Statistics/Statistics.cpp
+++ b/src/Storages/Statistics/Statistics.cpp
@@ -12,6 +12,8 @@
 #include <Storages/StatisticsDescription.h>
 #include <Common/Exception.h>
 
+#include "config.h" /// USE_DATASKETCHES
+
 namespace DB
 {
 
diff --git a/src/Storages/Statistics/Statistics.h b/src/Storages/Statistics/Statistics.h
index b6df6e45b70..9cc50a8ad4d 100644
--- a/src/Storages/Statistics/Statistics.h
+++ b/src/Storages/Statistics/Statistics.h
@@ -30,7 +30,6 @@ public:
     virtual ~IStatistics() = default;
 
     virtual void serialize(WriteBuffer & buf) = 0;
-
     virtual void deserialize(ReadBuffer & buf) = 0;
 
     virtual void update(const ColumnPtr & column) = 0;
diff --git a/src/Storages/Statistics/TDigestStatistics.h b/src/Storages/Statistics/TDigestStatistics.h
index 7c361b8751f..396d371aae7 100644
--- a/src/Storages/Statistics/TDigestStatistics.h
+++ b/src/Storages/Statistics/TDigestStatistics.h
@@ -18,7 +18,6 @@ public:
     Float64 estimateEqual(Float64 val) const;
 
     void serialize(WriteBuffer & buf) override;
-
     void deserialize(ReadBuffer & buf) override;
 
     void update(const ColumnPtr & column) override;
diff --git a/src/Storages/Statistics/UniqStatistics.h b/src/Storages/Statistics/UniqStatistics.h
index 0d86a6e458a..16bf5858ce1 100644
--- a/src/Storages/Statistics/UniqStatistics.h
+++ b/src/Storages/Statistics/UniqStatistics.h
@@ -17,7 +17,6 @@ public:
     UInt64 getCardinality();
 
     void serialize(WriteBuffer & buf) override;
-
     void deserialize(ReadBuffer & buf) override;
 
     void update(const ColumnPtr & column) override;
diff --git a/src/Storages/StatisticsDescription.cpp b/src/Storages/StatisticsDescription.cpp
index 23339ca3cfe..8fa3400d3a6 100644
--- a/src/Storages/StatisticsDescription.cpp
+++ b/src/Storages/StatisticsDescription.cpp
@@ -54,9 +54,9 @@ static StatisticsType stringToStatisticsType(String type)
         return StatisticsType::TDigest;
     if (type == "uniq")
         return StatisticsType::Uniq;
-    if (type == "countminsketch")
+    if (type == "count_min")
         return StatisticsType::CountMinSketch;
-    throw Exception(ErrorCodes::INCORRECT_QUERY, "Unknown statistics type: {}. Supported statistics types are `tdigest` and `uniq`.", type);
+    throw Exception(ErrorCodes::INCORRECT_QUERY, "Unknown statistics type: {}. Supported statistics types are 'tdigest', 'uniq', and 'count_min'.", type);
 }
 
 String SingleStatisticsDescription::getTypeName() const
@@ -68,9 +68,9 @@ String SingleStatisticsDescription::getTypeName() const
         case StatisticsType::Uniq:
             return "Uniq";
         case StatisticsType::CountMinSketch:
-            return "CountMinSketch";
+            return "count_min";
         default:
-            throw Exception(ErrorCodes::LOGICAL_ERROR, "Unknown statistics type: {}. Supported statistics types are `tdigest` and `uniq`.", type);
+            throw Exception(ErrorCodes::LOGICAL_ERROR, "Unknown statistics type: {}. Supported statistics types are 'tdigest', 'uniq', and 'count_min'.", type);
     }
 }
 

From a2ff2a35f0ebdf0cc3b7341b1f798b4d74d780bf Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Wed, 26 Jun 2024 14:15:54 +0000
Subject: [PATCH 0099/1488] More fixups

---
 .../Statistics/CountMinSketchStatistics.cpp         | 13 +++++++------
 src/Storages/Statistics/CountMinSketchStatistics.h  |  6 ++++--
 src/Storages/Statistics/TDigestStatistics.cpp       | 10 +++++-----
 src/Storages/Statistics/TDigestStatistics.h         |  2 +-
 src/Storages/Statistics/UniqStatistics.h            |  2 +-
 5 files changed, 18 insertions(+), 15 deletions(-)

diff --git a/src/Storages/Statistics/CountMinSketchStatistics.cpp b/src/Storages/Statistics/CountMinSketchStatistics.cpp
index 03c76c9709b..c22aa966a15 100644
--- a/src/Storages/Statistics/CountMinSketchStatistics.cpp
+++ b/src/Storages/Statistics/CountMinSketchStatistics.cpp
@@ -47,7 +47,7 @@ bool CountMinSketchStatistics::checkType(const Field & f)
 
 void CountMinSketchStatistics::serialize(WriteBuffer & buf)
 {
-    auto bytes = sketch.serialize();
+    Sketch::vector_bytes bytes = sketch.serialize();
     writeIntBinary(static_cast<UInt64>(bytes.size()), buf);
     buf.write(reinterpret_cast<const char *>(bytes.data()), bytes.size());
 }
@@ -56,11 +56,12 @@ void CountMinSketchStatistics::deserialize(ReadBuffer & buf)
 {
     UInt64 size;
     readIntBinary(size, buf);
-    String s;
-    s.reserve(size);
-    buf.readStrict(s.data(), size); /// Extra copy can be avoided by implementing count_min_sketch<Float64>::deserialize with ReadBuffer
-    auto read_sketch = datasketches::count_min_sketch<Float64>::deserialize(s.data(), size, datasketches::DEFAULT_SEED);
-    sketch.merge(read_sketch);
+
+    Sketch::vector_bytes bytes;
+    bytes.reserve(size);
+    buf.readStrict(reinterpret_cast<char *>(bytes.data()), size);
+
+    sketch = datasketches::count_min_sketch<Float64>::deserialize(bytes.data(), size, datasketches::DEFAULT_SEED);
 }
 
 void CountMinSketchStatistics::update(const ColumnPtr & column)
diff --git a/src/Storages/Statistics/CountMinSketchStatistics.h b/src/Storages/Statistics/CountMinSketchStatistics.h
index a112de4b3aa..2a4f65196a2 100644
--- a/src/Storages/Statistics/CountMinSketchStatistics.h
+++ b/src/Storages/Statistics/CountMinSketchStatistics.h
@@ -29,12 +29,14 @@ private:
     static constexpr auto HASH_COUNT = 8uz;
     static constexpr auto BUCKET_COUNT = 2048uz;
 
-    datasketches::count_min_sketch<Float64> sketch;
+    using Sketch = datasketches::count_min_sketch<Float64>;
+    Sketch sketch;
+
     DataTypePtr data_type;
 };
 
-StatisticsPtr CountMinSketchCreator(const SingleStatisticsDescription & stat, DataTypePtr);
 void CountMinSketchValidator(const SingleStatisticsDescription &, DataTypePtr data_type);
+StatisticsPtr CountMinSketchCreator(const SingleStatisticsDescription & stat, DataTypePtr);
 
 }
 
diff --git a/src/Storages/Statistics/TDigestStatistics.cpp b/src/Storages/Statistics/TDigestStatistics.cpp
index aa5662c979d..d390619ea36 100644
--- a/src/Storages/Statistics/TDigestStatistics.cpp
+++ b/src/Storages/Statistics/TDigestStatistics.cpp
@@ -45,11 +45,6 @@ void TDigestStatistics::update(const ColumnPtr & column)
     }
 }
 
-StatisticsPtr TDigestCreator(const SingleStatisticsDescription & stat, DataTypePtr)
-{
-    return std::make_shared<TDigestStatistics>(stat);
-}
-
 void TDigestValidator(const SingleStatisticsDescription &, DataTypePtr data_type)
 {
     data_type = removeNullable(data_type);
@@ -57,4 +52,9 @@ void TDigestValidator(const SingleStatisticsDescription &, DataTypePtr data_type
         throw Exception(ErrorCodes::ILLEGAL_STATISTICS, "Statistics of type 'tdigest' does not support type {}", data_type->getName());
 }
 
+StatisticsPtr TDigestCreator(const SingleStatisticsDescription & stat, DataTypePtr)
+{
+    return std::make_shared<TDigestStatistics>(stat);
+}
+
 }
diff --git a/src/Storages/Statistics/TDigestStatistics.h b/src/Storages/Statistics/TDigestStatistics.h
index 396d371aae7..2113572ac38 100644
--- a/src/Storages/Statistics/TDigestStatistics.h
+++ b/src/Storages/Statistics/TDigestStatistics.h
@@ -25,7 +25,7 @@ private:
     QuantileTDigest<Float64> data;
 };
 
-StatisticsPtr TDigestCreator(const SingleStatisticsDescription & stat, DataTypePtr);
 void TDigestValidator(const SingleStatisticsDescription &, DataTypePtr data_type);
+StatisticsPtr TDigestCreator(const SingleStatisticsDescription & stat, DataTypePtr);
 
 }
diff --git a/src/Storages/Statistics/UniqStatistics.h b/src/Storages/Statistics/UniqStatistics.h
index 16bf5858ce1..bf097620a86 100644
--- a/src/Storages/Statistics/UniqStatistics.h
+++ b/src/Storages/Statistics/UniqStatistics.h
@@ -29,7 +29,7 @@ private:
 
 };
 
-StatisticsPtr UniqCreator(const SingleStatisticsDescription & stat, DataTypePtr data_type);
 void UniqValidator(const SingleStatisticsDescription &, DataTypePtr data_type);
+StatisticsPtr UniqCreator(const SingleStatisticsDescription & stat, DataTypePtr data_type);
 
 }

From c7dc99757afec4dbe4ac5b1bcfab037e141839d8 Mon Sep 17 00:00:00 2001
From: Konstantin Morozov <just.morozov.k@gmail.com>
Date: Wed, 26 Jun 2024 15:39:16 +0000
Subject: [PATCH 0100/1488] apply some comments

---
 src/Databases/DatabaseOrdinary.cpp            | 31 ++++--------
 src/Databases/DatabasesCommon.cpp             | 48 +++++++------------
 src/Databases/IDatabase.h                     |  4 +-
 .../test_system_detached_tables/test.py       | 22 +++++----
 4 files changed, 42 insertions(+), 63 deletions(-)

diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp
index 3975eb46a4b..ef0c981ddad 100644
--- a/src/Databases/DatabaseOrdinary.cpp
+++ b/src/Databases/DatabaseOrdinary.cpp
@@ -227,31 +227,18 @@ void DatabaseOrdinary::loadTablesMetadata(ContextPtr local_context, ParsedTables
                     permanently_detached_tables.push_back(table_name);
                     LOG_DEBUG(log, "Skipping permanently detached table {}.", backQuote(table_name));
 
-                    auto parsed_table_metadata = ParsedTableMetadata{full_path.string(), ast};
-                    const auto & query = parsed_table_metadata.ast->as<const ASTCreateQuery &>();
-
                     std::lock_guard lock(mutex);
 
-                    auto [detached_table_name, table] = createTableFromAST(
-                        query,
-                        database_name,
-                        getTableDataPath(query),
-                        std::const_pointer_cast<Context>(local_context),
-                        LoadingStrictnessLevel::CREATE);
+                    const auto detached_table_name = create_query->getTable();
 
-                    const auto storage_id = table->getStorageID();
-
-                    SnapshotDetachedTable snapshot_detached_table;
-                    snapshot_detached_table.database = storage_id.getDatabaseName();
-                    snapshot_detached_table.table = detached_table_name;
-                    if (storage_id.hasUUID())
-                    {
-                        snapshot_detached_table.uuid = storage_id.uuid;
-                    }
-                    snapshot_detached_table.is_permanently = true;
-                    snapshot_detached_table.metadata_path = getObjectMetadataPath(snapshot_detached_table.table);
-
-                    snapshot_detached_tables.emplace(detached_table_name, std::move(snapshot_detached_table));
+                    snapshot_detached_tables.emplace(
+                        detached_table_name,
+                        SnapshotDetachedTable{
+                            .database = create_query->getDatabase(),
+                            .table = detached_table_name,
+                            .uuid = create_query->uuid,
+                            .metadata_path = getObjectMetadataPath(detached_table_name),
+                            .is_permanently = true});
 
                     LOG_TRACE(log, "Add permanently detached table {} to system.detached_tables", detached_table_name);
                     return;
diff --git a/src/Databases/DatabasesCommon.cpp b/src/Databases/DatabasesCommon.cpp
index 3b54980597c..470625ddd8b 100644
--- a/src/Databases/DatabasesCommon.cpp
+++ b/src/Databases/DatabasesCommon.cpp
@@ -2,6 +2,10 @@
 
 #include <Backups/BackupEntriesCollector.h>
 #include <Backups/RestorerFromBackup.h>
+#include <Common/CurrentMetrics.h>
+#include <Common/escapeForFileName.h>
+#include <Common/logger_useful.h>
+#include <Common/typeid_cast.h>
 #include <Interpreters/Context.h>
 #include <Interpreters/DatabaseCatalog.h>
 #include <Interpreters/InterpreterCreateQuery.h>
@@ -13,10 +17,6 @@
 #include <Storages/StorageFactory.h>
 #include <Storages/Utils.h>
 #include <TableFunctions/TableFunctionFactory.h>
-#include <Common/CurrentMetrics.h>
-#include <Common/escapeForFileName.h>
-#include <Common/logger_useful.h>
-#include <Common/typeid_cast.h>
 
 
 namespace DB
@@ -245,26 +245,15 @@ DatabaseDetachedTablesSnapshotIteratorPtr DatabaseWithOwnTablesBase::getDetached
     if (!filter_by_table_name)
         return std::make_unique<DatabaseDetachedTablesSnapshotIterator>(snapshot_detached_tables);
 
-    SnapshotDetachedTables filtered_tables;
-    for (const auto & [table_name, storage] : tables)
-        if (filter_by_table_name(table_name))
+    SnapshotDetachedTables filtered_detached_tables;
+    for (const auto & [detached_table_name, snapshot] : snapshot_detached_tables)
+        if (filter_by_table_name(detached_table_name))
         {
-            SnapshotDetachedTable snapshot_detached_table;
-            snapshot_detached_table.database = storage->getStorageID().getDatabaseName();
-            snapshot_detached_table.table = table_name;
-            if (storage->getStorageID().hasUUID())
-            {
-                snapshot_detached_table.uuid = storage->getStorageID().uuid;
-            }
-
-            snapshot_detached_table.is_permanently = false;
-            snapshot_detached_table.metadata_path = getObjectMetadataPath(snapshot_detached_table.table);
-
-            filtered_tables.emplace(table_name, std::move(snapshot_detached_table));
+            filtered_detached_tables.emplace(detached_table_name, snapshot);
         }
 
 
-    return std::make_unique<DatabaseDetachedTablesSnapshotIterator>(std::move(filtered_tables));
+    return std::make_unique<DatabaseDetachedTablesSnapshotIterator>(std::move(filtered_detached_tables));
 }
 
 bool DatabaseWithOwnTablesBase::empty() const
@@ -289,17 +278,14 @@ StoragePtr DatabaseWithOwnTablesBase::detachTableUnlocked(const String & table_n
                         backQuote(database_name), backQuote(table_name));
     res = it->second;
 
-    SnapshotDetachedTable snapshot_detached_table;
-    snapshot_detached_table.database = it->second->getStorageID().getDatabaseName();
-    snapshot_detached_table.table = it->first;
-    if (it->second->getStorageID().hasUUID())
-    {
-        snapshot_detached_table.uuid = it->second->getStorageID().uuid;
-    }
-    snapshot_detached_table.is_permanently = false;
-    snapshot_detached_table.metadata_path = getObjectMetadataPath(snapshot_detached_table.table);
-
-    snapshot_detached_tables.emplace(it->first, std::move(snapshot_detached_table));
+    snapshot_detached_tables.emplace(
+        it->first,
+        SnapshotDetachedTable{
+            .database = it->second->getStorageID().getDatabaseName(),
+            .table = it->first,
+            .uuid = it->second->getStorageID().uuid,
+            .metadata_path = getObjectMetadataPath(it->first),
+            .is_permanently = false});
 
     tables.erase(it);
     res->is_detached = true;
diff --git a/src/Databases/IDatabase.h b/src/Databases/IDatabase.h
index 76d25908fda..e856dd3d2f0 100644
--- a/src/Databases/IDatabase.h
+++ b/src/Databases/IDatabase.h
@@ -118,7 +118,7 @@ struct SnapshotDetachedTable final
     String table;
     UUID uuid = UUIDHelpers::Nil;
     String metadata_path;
-    bool is_permanently{false};
+    bool is_permanently{};
 };
 
 class DatabaseDetachedTablesSnapshotIterator
@@ -286,7 +286,7 @@ public:
     virtual DatabaseDetachedTablesSnapshotIteratorPtr getDetachedTablesIterator(
         ContextPtr /*context*/, const FilterByNameFunction & /*filter_by_table_name = {}*/, bool /*skip_not_loaded = false*/) const
     {
-        throw Exception(ErrorCodes::NOT_IMPLEMENTED, "There is no get detached tables for Database {}", getEngineName());
+        throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot get detached tables for Database{}", getEngineName());
     }
 
     /// Returns list of table names.
diff --git a/tests/integration/test_system_detached_tables/test.py b/tests/integration/test_system_detached_tables/test.py
index 02734d8bf14..bbac5e391f8 100644
--- a/tests/integration/test_system_detached_tables/test.py
+++ b/tests/integration/test_system_detached_tables/test.py
@@ -18,21 +18,27 @@ def test_system_detached_tables():
     node.query("CREATE TABLE test_table (n Int64) ENGINE=MergeTree ORDER BY n;")
     node.query("CREATE TABLE test_table_perm (n Int64) ENGINE=MergeTree ORDER BY n;")
 
-    result = node.query("SELECT * FROM system.detached_tables")
-    assert result == ""
+    test_table_uuid = node.query("SELECT uuid FROM system.tables WHERE table='test_table'").rstrip('\n')
+    test_table_metadata_path = node.query("SELECT metadata_path FROM system.tables WHERE table='test_table'").rstrip('\n')
+
+    test_table_perm_uuid = node.query("SELECT uuid FROM system.tables WHERE table='test_table_perm'").rstrip('\n')
+    test_table_perm_metadata_path = node.query("SELECT metadata_path FROM system.tables WHERE table='test_table_perm'").rstrip('\n')
+
+    assert "" == node.query("SELECT * FROM system.detached_tables")
 
     node.query("DETACH TABLE test_table")
     node.query("DETACH TABLE test_table_perm PERMANENTLY")
 
-    result = node.query("SELECT table FROM system.detached_tables")
-    assert result == "test_table\ntest_table_perm\n"
+    querry = "SELECT database, table, is_permanently, uuid, metadata_path FROM system.detached_tables FORMAT Values"
+    result = node.query(querry)
+    assert result == f"('default','test_table',0,'{test_table_uuid}','{test_table_metadata_path}'),('default','test_table_perm',1,'{test_table_perm_uuid}','{test_table_perm_metadata_path}')"
 
     node.restart_clickhouse()
 
-    result = node.query("SELECT table FROM system.detached_tables")
-    assert result == "test_table_perm\n"
+    result = node.query(querry)
+    assert result == f"('default','test_table_perm',1,'{test_table_perm_uuid}','{test_table_perm_metadata_path}')"
 
     node.restart_clickhouse()
 
-    result = node.query("SELECT table FROM system.detached_tables")
-    assert result == "test_table_perm\n"
+    result = node.query(querry)
+    assert result == f"('default','test_table_perm',1,'{test_table_perm_uuid}','{test_table_perm_metadata_path}')"

From 91598d10e9ab6f9d7054ed7e4204665e85b7636c Mon Sep 17 00:00:00 2001
From: Tobias Florek <me@ibotty.net>
Date: Thu, 27 Jun 2024 09:29:06 +0200
Subject: [PATCH 0101/1488] document declarative ssh-keys authentication

---
 docs/en/operations/settings/settings-users.md | 33 +++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/docs/en/operations/settings/settings-users.md b/docs/en/operations/settings/settings-users.md
index 96477f777a9..ef1e58fd18e 100644
--- a/docs/en/operations/settings/settings-users.md
+++ b/docs/en/operations/settings/settings-users.md
@@ -22,6 +22,21 @@ Structure of the `users` section:
         <!-- Or -->
         <password_sha256_hex></password_sha256_hex>
 
+        <ssh_keys>
+            <ssh_key>
+                <type>ssh-ed25519</type>
+                <base64_key>AAAAC3NzaC1lZDI1NTE5AAAAIDNf0r6vRl24Ix3tv2IgPmNPO2ATa2krvt80DdcTatLj</base64_key>
+            </ssh_key>
+            <ssh_key>
+                <type>ecdsa-sha2-nistp256</type>
+                <base64_key>AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNxeV2uN5UY6CUbCzTA1rXfYimKQA5ivNIqxdax4bcMXz4D0nSk2l5E1TkR5mG8EBWtmExSPbcEPJ8V7lyWWbA8=</base64_key>
+            </ssh_key>
+            <ssh_key>
+                <type>ssh-rsa</type>
+                <base64_key>AAAAB3NzaC1yc2EAAAADAQABAAABgQCpgqL1SHhPVBOTFlOm0pu+cYBbADzC2jL41sPMawYCJHDyHuq7t+htaVVh2fRgpAPmSEnLEC2d4BEIKMtPK3bfR8plJqVXlLt6Q8t4b1oUlnjb3VPA9P6iGcW7CV1FBkZQEVx8ckOfJ3F+kI5VsrRlEDgiecm/C1VPl0/9M2llW/mPUMaD65cM9nlZgM/hUeBrfxOEqM11gDYxEZm1aRSbZoY4dfdm3vzvpSQ6lrCrkjn3X2aSmaCLcOWJhfBWMovNDB8uiPuw54g3ioZ++qEQMlfxVsqXDGYhXCrsArOVuW/5RbReO79BvXqdssiYShfwo+GhQ0+aLWMIW/jgBkkqx/n7uKLzCMX7b2F+aebRYFh+/QXEj7SnihdVfr9ud6NN3MWzZ1ltfIczlEcFLrLJ1Yq57wW6wXtviWh59WvTWFiPejGjeSjjJyqqB49tKdFVFuBnIU5u/bch2DXVgiAEdQwUrIp1ACoYPq22HFFAYUJrL32y7RxX3PGzuAv3LOc=</base64_key>
+            </ssh_key>
+        </ssh_keys>
+
         <access_management>0|1</access_management>
 
         <networks incl="networks" replace="replace">
@@ -79,6 +94,24 @@ Password can be specified in plaintext or in SHA256 (hex format).
 
     The first line of the result is the password. The second line is the corresponding double SHA1 hash.
 
+### username/ssh-key {#user-sshkey}
+
+This setting allows authenticating with SSH keys.
+
+Given a SSH key (as generated by `ssh-keygen`) like
+```
+ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDNf0r6vRl24Ix3tv2IgPmNPO2ATa2krvt80DdcTatLj john@example.com
+```
+The `ssh_key` element is expected to be
+```
+<ssh_key>
+     <type>ssh-ed25519</type>
+     <base64_key>AAAAC3NzaC1lZDI1NTE5AAAAIDNf0r6vRl24Ix3tv2IgPmNPO2ATa2krvt80DdcTatLj</base64_key>
+ </ssh_key>
+```
+
+Substitute `ssh-ed25519` with `ssh-rsa` or `ecdsa-sha2-nistp256` for the other supported algorithms.
+
 ### access_management {#access_management-user-setting}
 
 This setting enables or disables using of SQL-driven [access control and account management](../../guides/sre/user-management/index.md#access-control) for the user.

From de8f1fc6e3adb6c8f346b2fda4e34c2240e1bf8e Mon Sep 17 00:00:00 2001
From: Konstantin Morozov <just.morozov.k@gmail.com>
Date: Thu, 27 Jun 2024 07:32:56 +0000
Subject: [PATCH 0102/1488] general logic for tables

---
 src/Storages/System/ReadFromSystemTables.cpp  | 157 ++++
 src/Storages/System/ReadFromSystemTables.h    |  66 ++
 .../System/StorageSystemDetachedTables.cpp    | 237 +------
 src/Storages/System/StorageSystemTables.cpp   | 669 +-----------------
 src/Storages/System/TablesBlockSource.cpp     | 564 +++++++++++++++
 src/Storages/System/TablesBlockSource.h       |  92 +++
 6 files changed, 894 insertions(+), 891 deletions(-)
 create mode 100644 src/Storages/System/ReadFromSystemTables.cpp
 create mode 100644 src/Storages/System/ReadFromSystemTables.h
 create mode 100644 src/Storages/System/TablesBlockSource.cpp
 create mode 100644 src/Storages/System/TablesBlockSource.h

diff --git a/src/Storages/System/ReadFromSystemTables.cpp b/src/Storages/System/ReadFromSystemTables.cpp
new file mode 100644
index 00000000000..638cab1b63b
--- /dev/null
+++ b/src/Storages/System/ReadFromSystemTables.cpp
@@ -0,0 +1,157 @@
+#include "ReadFromSystemTables.h"
+
+#include <Core/Block.h>
+#include <Core/ColumnWithTypeAndName.h>
+#include <DataTypes/DataTypeString.h>
+#include <Interpreters/DatabaseCatalog.h>
+#include <QueryPipeline/QueryPipelineBuilder.h>
+#include <Storages/IStorage.h>
+#include <Storages/System/TablesBlockSource.h>
+#include <Storages/VirtualColumnUtils.h>
+
+#include <boost/range/adaptor/map.hpp>
+
+namespace DB
+{
+
+namespace
+{
+
+ColumnPtr getFilteredDatabases(const ActionsDAG::Node * predicate, ContextPtr context)
+{
+    MutableColumnPtr column = ColumnString::create();
+
+    const auto databases = DatabaseCatalog::instance().getDatabases();
+    for (const auto & database_name : databases | boost::adaptors::map_keys)
+    {
+        if (database_name == DatabaseCatalog::TEMPORARY_DATABASE)
+            continue; /// We don't want to show the internal database for temporary tables in system.tables
+
+        column->insert(database_name);
+    }
+
+    Block block{ColumnWithTypeAndName(std::move(column), std::make_shared<DataTypeString>(), "database")};
+    VirtualColumnUtils::filterBlockWithPredicate(predicate, block, context);
+    return block.getByPosition(0).column;
+}
+
+ColumnPtr getFilteredTables(
+    const ActionsDAG::Node * predicate, const ColumnPtr & filtered_databases_column, ContextPtr context, const bool need_detached_tables)
+{
+    Block sample{
+        ColumnWithTypeAndName(nullptr, std::make_shared<DataTypeString>(), "name"),
+        ColumnWithTypeAndName(nullptr, std::make_shared<DataTypeString>(), "engine")};
+
+    MutableColumnPtr database_column = ColumnString::create();
+    MutableColumnPtr engine_column;
+
+    auto dag = VirtualColumnUtils::splitFilterDagForAllowedInputs(predicate, &sample);
+    if (dag)
+    {
+        bool filter_by_engine = false;
+        for (const auto * input : dag->getInputs())
+            if (input->result_name == "engine")
+                filter_by_engine = true;
+
+        if (filter_by_engine)
+            engine_column = ColumnString::create();
+    }
+
+    for (size_t database_idx = 0; database_idx < filtered_databases_column->size(); ++database_idx)
+    {
+        const auto & database_name = filtered_databases_column->getDataAt(database_idx).toString();
+        DatabasePtr database = DatabaseCatalog::instance().tryGetDatabase(database_name);
+        if (!database)
+            continue;
+
+        if (need_detached_tables)
+        {
+            auto table_it = database->getDetachedTablesIterator(context, {}, false);
+            for (; table_it->isValid(); table_it->next())
+            {
+                database_column->insert(table_it->table());
+            }
+        }
+        else
+        {
+            auto table_it = database->getTablesIterator(context);
+            for (; table_it->isValid(); table_it->next())
+            {
+                database_column->insert(table_it->name());
+                if (engine_column)
+                    engine_column->insert(table_it->table()->getName());
+            }
+        }
+    }
+
+    Block block{ColumnWithTypeAndName(std::move(database_column), std::make_shared<DataTypeString>(), "name")};
+    if (engine_column)
+        block.insert(ColumnWithTypeAndName(std::move(engine_column), std::make_shared<DataTypeString>(), "engine"));
+
+    if (dag)
+        VirtualColumnUtils::filterBlockWithDAG(dag, block, context);
+
+    return block.getByPosition(0).column;
+}
+
+}
+
+ReadFromSystemTables::ReadFromSystemTables(
+    const Names & column_names_,
+    const SelectQueryInfo & query_info_,
+    const StorageSnapshotPtr & storage_snapshot_,
+    const ContextPtr & context_,
+    Block sample_block,
+    std::vector<UInt8> columns_mask_,
+    size_t max_block_size_)
+    : SourceStepWithFilter(DataStream{.header = std::move(sample_block)}, column_names_, query_info_, storage_snapshot_, context_)
+    , columns_mask(std::move(columns_mask_))
+    , max_block_size(max_block_size_)
+{
+}
+
+void ReadFromSystemTables::applyFilters(ActionDAGNodes added_filter_nodes)
+{
+    applyFiltersImpl(std::move(added_filter_nodes), false);
+}
+
+void ReadFromSystemTables::applyFiltersImpl(ActionDAGNodes added_filter_nodes, const bool need_detached_tables_)
+{
+    SourceStepWithFilter::applyFilters(std::move(added_filter_nodes));
+
+    const ActionsDAG::Node * predicate = nullptr;
+    if (filter_actions_dag)
+        predicate = filter_actions_dag->getOutputs().at(0);
+
+    filtered_databases_column = getFilteredDatabases(predicate, context);
+    filtered_tables_column = getFilteredTables(predicate, filtered_databases_column, context, need_detached_tables_);
+}
+
+void ReadFromSystemTables::initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &)
+{
+    pipeline.init(createPipe<TablesBlockSource>());
+}
+
+ReadFromSystemDetachedTables::ReadFromSystemDetachedTables(
+    const Names & column_names_,
+    const SelectQueryInfo & query_info_,
+    const StorageSnapshotPtr & storage_snapshot_,
+    const ContextPtr & context_,
+    Block sample_block,
+    std::vector<UInt8> columns_mask_,
+    const size_t max_block_size_)
+    : ReadFromSystemTables(
+        column_names_, query_info_, storage_snapshot_, context_, std::move(sample_block), std::move(columns_mask_), max_block_size_)
+{
+}
+
+void ReadFromSystemDetachedTables::applyFilters(ActionDAGNodes added_filter_nodes)
+{
+    applyFiltersImpl(std::move(added_filter_nodes), true);
+}
+
+void ReadFromSystemDetachedTables::initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &)
+{
+    pipeline.init(createPipe<DetachedTablesBlockSource>());
+}
+}
diff --git a/src/Storages/System/ReadFromSystemTables.h b/src/Storages/System/ReadFromSystemTables.h
new file mode 100644
index 00000000000..f8f34363159
--- /dev/null
+++ b/src/Storages/System/ReadFromSystemTables.h
@@ -0,0 +1,66 @@
+#pragma once
+
+#include <Processors/QueryPlan/SourceStepWithFilter.h>
+#include <QueryPipeline/Pipe.h>
+
+namespace DB
+{
+
+class ReadFromSystemTables : public SourceStepWithFilter
+{
+public:
+    std::string getName() const override { return "ReadFromSystemTables"; }
+    void initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) override;
+
+    ReadFromSystemTables(
+        const Names & column_names_,
+        const SelectQueryInfo & query_info_,
+        const StorageSnapshotPtr & storage_snapshot_,
+        const ContextPtr & context_,
+        Block sample_block,
+        std::vector<UInt8> columns_mask_,
+        size_t max_block_size_);
+
+    void applyFilters(ActionDAGNodes added_filter_nodes) override;
+
+protected:
+    void applyFiltersImpl(ActionDAGNodes added_filter_nodes, bool need_detached_tables_);
+
+    template <class T>
+    Pipe createPipe()
+    {
+        return Pipe(std::make_shared<T>(
+            std::move(columns_mask),
+            getOutputStream().header,
+            max_block_size,
+            std::move(filtered_databases_column),
+            std::move(filtered_tables_column),
+            context));
+    }
+
+private:
+    std::vector<UInt8> columns_mask;
+    size_t max_block_size;
+
+    ColumnPtr filtered_databases_column;
+    ColumnPtr filtered_tables_column;
+};
+
+class ReadFromSystemDetachedTables : public ReadFromSystemTables
+{
+public:
+    std::string getName() const override { return "ReadFromSystemDetachedTables"; }
+    void initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) override;
+
+    ReadFromSystemDetachedTables(
+        const Names & column_names_,
+        const SelectQueryInfo & query_info_,
+        const StorageSnapshotPtr & storage_snapshot_,
+        const ContextPtr & context_,
+        Block sample_block,
+        std::vector<UInt8> columns_mask_,
+        size_t max_block_size_);
+
+    void applyFilters(ActionDAGNodes added_filter_nodes) override;
+};
+}
diff --git a/src/Storages/System/StorageSystemDetachedTables.cpp b/src/Storages/System/StorageSystemDetachedTables.cpp
index 9805fa2a8cf..99c15fdae0e 100644
--- a/src/Storages/System/StorageSystemDetachedTables.cpp
+++ b/src/Storages/System/StorageSystemDetachedTables.cpp
@@ -13,6 +13,7 @@
 #include <Storages/ColumnsDescription.h>
 #include <Storages/ProjectionsDescription.h>
 #include <Storages/StorageInMemoryMetadata.h>
+#include <Storages/System/ReadFromSystemTables.h>
 #include <Storages/System/getQueriedColumnsMaskAndHeader.h>
 #include <Storages/VirtualColumnUtils.h>
 
@@ -22,204 +23,6 @@
 namespace DB
 {
 
-namespace
-{
-
-ColumnPtr getFilteredDatabases(const ActionsDAG::Node * predicate, ContextPtr context)
-{
-    MutableColumnPtr column = ColumnString::create();
-
-    const auto databases = DatabaseCatalog::instance().getDatabases();
-    for (const auto & database_name : databases | boost::adaptors::map_keys)
-    {
-        if (database_name == DatabaseCatalog::TEMPORARY_DATABASE)
-            continue; /// We don't want to show the internal database for temporary tables in system.tables
-
-        column->insert(database_name);
-    }
-
-    Block block{ColumnWithTypeAndName(std::move(column), std::make_shared<DataTypeString>(), "database")};
-    VirtualColumnUtils::filterBlockWithPredicate(predicate, block, context);
-    return block.getByPosition(0).column;
-}
-
-ColumnPtr getFilteredTables(const ActionsDAG::Node * predicate, const ColumnPtr & filtered_databases_column, ContextPtr context)
-{
-    Block sample{
-        ColumnWithTypeAndName(nullptr, std::make_shared<DataTypeString>(), "name"),
-        ColumnWithTypeAndName(nullptr, std::make_shared<DataTypeString>(), "engine")};
-
-    MutableColumnPtr database_column = ColumnString::create();
-    MutableColumnPtr engine_column;
-
-    auto dag = VirtualColumnUtils::splitFilterDagForAllowedInputs(predicate, &sample);
-    if (dag)
-    {
-        bool filter_by_engine = false;
-        for (const auto * input : dag->getInputs())
-            if (input->result_name == "engine")
-                filter_by_engine = true;
-
-        if (filter_by_engine)
-            engine_column = ColumnString::create();
-    }
-
-    for (size_t database_idx = 0; database_idx < filtered_databases_column->size(); ++database_idx)
-    {
-        const auto & database_name = filtered_databases_column->getDataAt(database_idx).toString();
-        DatabasePtr database = DatabaseCatalog::instance().tryGetDatabase(database_name);
-        if (!database)
-            continue;
-
-        auto table_it = database->getDetachedTablesIterator(context, {}, false);
-        for (; table_it->isValid(); table_it->next())
-        {
-            database_column->insert(table_it->table());
-        }
-    }
-
-    Block block{ColumnWithTypeAndName(std::move(database_column), std::make_shared<DataTypeString>(), "name")};
-    if (engine_column)
-        block.insert(ColumnWithTypeAndName(std::move(engine_column), std::make_shared<DataTypeString>(), "engine"));
-
-    if (dag)
-        VirtualColumnUtils::filterBlockWithDAG(dag, block, context);
-
-    return block.getByPosition(0).column;
-}
-
-class DetachedTablesBlockSource : public ISource
-{
-public:
-    DetachedTablesBlockSource(
-        std::vector<UInt8> columns_mask_,
-        Block header,
-        UInt64 max_block_size_,
-        ColumnPtr databases_,
-        ColumnPtr detached_tables_,
-        ContextPtr context_)
-        : ISource(std::move(header))
-        , columns_mask(std::move(columns_mask_))
-        , max_block_size(max_block_size_)
-        , databases(std::move(databases_))
-        , context(Context::createCopy(context_))
-    {
-        size_t size = detached_tables_->size();
-        detached_tables.reserve(size);
-        for (size_t idx = 0; idx < size; ++idx)
-        {
-            detached_tables.insert(detached_tables_->getDataAt(idx).toString());
-        }
-    }
-
-    String getName() const override { return "DetachedTables"; }
-
-protected:
-    Chunk generate() override
-    {
-        if (done)
-            return {};
-
-        MutableColumns result_columns = getPort().getHeader().cloneEmptyColumns();
-
-        const auto access = context->getAccess();
-        const bool need_to_check_access_for_databases = !access->isGranted(AccessType::SHOW_TABLES);
-
-        for (size_t database_idx = 0, rows_count = 0; database_idx < databases->size() && rows_count < max_block_size; ++database_idx)
-        {
-            database_name = databases->getDataAt(database_idx).toString();
-            database = DatabaseCatalog::instance().tryGetDatabase(database_name);
-
-            if (!database)
-                continue;
-
-            const bool need_to_check_access_for_tables
-                = need_to_check_access_for_databases && !access->isGranted(AccessType::SHOW_TABLES, database_name);
-
-            if (!detached_tables_it || !detached_tables_it->isValid())
-                detached_tables_it = database->getDetachedTablesIterator(context, {}, false);
-
-            for (; rows_count < max_block_size && detached_tables_it->isValid(); detached_tables_it->next())
-            {
-                const auto detached_table_name = detached_tables_it->table();
-
-                if (!detached_tables.contains(detached_table_name))
-                    continue;
-
-                if (need_to_check_access_for_tables && !access->isGranted(AccessType::SHOW_TABLES, database_name, detached_table_name))
-                    continue;
-
-                fillResultColumnsByDetachedTableIterator(result_columns);
-                ++rows_count;
-            }
-        }
-
-        const UInt64 num_rows = result_columns.at(0)->size();
-        done = true;
-        return Chunk(std::move(result_columns), num_rows);
-    }
-
-private:
-    const std::vector<UInt8> columns_mask;
-    const UInt64 max_block_size;
-    const ColumnPtr databases;
-    NameSet detached_tables;
-    DatabaseDetachedTablesSnapshotIteratorPtr detached_tables_it;
-    ContextPtr context;
-    bool done = false;
-    DatabasePtr database;
-    std::string database_name;
-
-    void fillResultColumnsByDetachedTableIterator(MutableColumns & result_columns) const
-    {
-        size_t src_index = 0;
-        size_t res_index = 0;
-
-        if (columns_mask[src_index++])
-            result_columns[res_index++]->insert(detached_tables_it->database());
-
-        if (columns_mask[src_index++])
-            result_columns[res_index++]->insert(detached_tables_it->table());
-
-
-        if (columns_mask[src_index++])
-            result_columns[res_index++]->insert(detached_tables_it->uuid());
-
-        if (columns_mask[src_index++])
-            result_columns[res_index++]->insert(detached_tables_it->metadataPath());
-
-        if (columns_mask[src_index++])
-            result_columns[res_index++]->insert(detached_tables_it->isPermanently());
-    }
-};
-
-}
-
-class ReadFromSystemDetachedTables : public SourceStepWithFilter
-{
-public:
-    std::string getName() const override { return "ReadFromSystemDetachedTables"; }
-    void initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) override;
-
-    ReadFromSystemDetachedTables(
-        const Names & column_names_,
-        const SelectQueryInfo & query_info_,
-        const StorageSnapshotPtr & storage_snapshot_,
-        const ContextPtr & context_,
-        Block sample_block,
-        std::vector<UInt8> columns_mask_,
-        size_t max_block_size_);
-
-    void applyFilters(ActionDAGNodes added_filter_nodes) override;
-
-private:
-    std::vector<UInt8> columns_mask;
-    size_t max_block_size;
-
-    ColumnPtr filtered_databases_column;
-    ColumnPtr filtered_tables_column;
-};
-
 StorageSystemDetachedTables::StorageSystemDetachedTables(const StorageID & table_id_) : IStorage(table_id_)
 {
     StorageInMemoryMetadata storage_metadata;
@@ -257,42 +60,4 @@ void StorageSystemDetachedTables::read(
 
     query_plan.addStep(std::move(reading));
 }
-
-ReadFromSystemDetachedTables::ReadFromSystemDetachedTables(
-    const Names & column_names_,
-    const SelectQueryInfo & query_info_,
-    const StorageSnapshotPtr & storage_snapshot_,
-    const ContextPtr & context_,
-    Block sample_block,
-    std::vector<UInt8> columns_mask_,
-    size_t max_block_size_)
-    : SourceStepWithFilter(DataStream{.header = std::move(sample_block)}, column_names_, query_info_, storage_snapshot_, context_)
-    , columns_mask(std::move(columns_mask_))
-    , max_block_size(max_block_size_)
-{
-}
-
-void ReadFromSystemDetachedTables::applyFilters(ActionDAGNodes added_filter_nodes)
-{
-    SourceStepWithFilter::applyFilters(std::move(added_filter_nodes));
-
-    const ActionsDAG::Node * predicate = nullptr;
-    if (filter_actions_dag)
-        predicate = filter_actions_dag->getOutputs().at(0);
-
-    filtered_databases_column = getFilteredDatabases(predicate, context);
-    filtered_tables_column = getFilteredTables(predicate, filtered_databases_column, context);
-}
-
-void ReadFromSystemDetachedTables::initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &)
-{
-    auto pipe = Pipe(std::make_shared<DetachedTablesBlockSource>(
-        std::move(columns_mask),
-        getOutputStream().header,
-        max_block_size,
-        std::move(filtered_databases_column),
-        std::move(filtered_tables_column),
-        context));
-    pipeline.init(std::move(pipe));
-}
 }
diff --git a/src/Storages/System/StorageSystemTables.cpp b/src/Storages/System/StorageSystemTables.cpp
index 783b899c978..40190f53fd3 100644
--- a/src/Storages/System/StorageSystemTables.cpp
+++ b/src/Storages/System/StorageSystemTables.cpp
@@ -1,29 +1,30 @@
+#include <Access/ContextAccess.h>
 #include <Columns/ColumnString.h>
-#include <DataTypes/DataTypeString.h>
+#include <Common/StringUtils.h>
+#include <Common/typeid_cast.h>
+#include <DataTypes/DataTypeArray.h>
 #include <DataTypes/DataTypeDateTime.h>
 #include <DataTypes/DataTypeNullable.h>
-#include <Storages/System/StorageSystemTables.h>
-#include <Storages/System/getQueriedColumnsMaskAndHeader.h>
-#include <Storages/MergeTree/MergeTreeData.h>
-#include <Storages/SelectQueryInfo.h>
-#include <Storages/VirtualColumnUtils.h>
-#include <Access/ContextAccess.h>
+#include <DataTypes/DataTypeString.h>
+#include <DataTypes/DataTypeUUID.h>
+#include <DataTypes/DataTypesNumber.h>
+#include <Disks/IStoragePolicy.h>
 #include <Interpreters/Context.h>
 #include <Interpreters/DatabaseCatalog.h>
 #include <Interpreters/formatWithPossiblyHidingSecrets.h>
 #include <Parsers/ASTCreateQuery.h>
 #include <Parsers/ASTSelectWithUnionQuery.h>
-#include <Common/typeid_cast.h>
-#include <Common/StringUtils.h>
-#include <DataTypes/DataTypesNumber.h>
-#include <DataTypes/DataTypeArray.h>
-#include <Disks/IStoragePolicy.h>
 #include <Processors/ISource.h>
 #include <Processors/QueryPlan/QueryPlan.h>
 #include <Processors/QueryPlan/SourceStepWithFilter.h>
 #include <QueryPipeline/Pipe.h>
 #include <QueryPipeline/QueryPipelineBuilder.h>
-#include <DataTypes/DataTypeUUID.h>
+#include <Storages/MergeTree/MergeTreeData.h>
+#include <Storages/SelectQueryInfo.h>
+#include <Storages/System/ReadFromSystemTables.h>
+#include <Storages/System/StorageSystemTables.h>
+#include <Storages/System/getQueriedColumnsMaskAndHeader.h>
+#include <Storages/VirtualColumnUtils.h>
 
 #include <boost/range/adaptor/map.hpp>
 
@@ -105,628 +106,6 @@ StorageSystemTables::StorageSystemTables(const StorageID & table_id_)
     setInMemoryMetadata(storage_metadata);
 }
 
-
-namespace
-{
-
-ColumnPtr getFilteredDatabases(const ActionsDAG::Node * predicate, ContextPtr context)
-{
-    MutableColumnPtr column = ColumnString::create();
-
-    const auto databases = DatabaseCatalog::instance().getDatabases();
-    for (const auto & database_name : databases | boost::adaptors::map_keys)
-    {
-        if (database_name == DatabaseCatalog::TEMPORARY_DATABASE)
-            continue; /// We don't want to show the internal database for temporary tables in system.tables
-
-        column->insert(database_name);
-    }
-
-    Block block { ColumnWithTypeAndName(std::move(column), std::make_shared<DataTypeString>(), "database") };
-    VirtualColumnUtils::filterBlockWithPredicate(predicate, block, context);
-    return block.getByPosition(0).column;
-}
-
-ColumnPtr getFilteredTables(const ActionsDAG::Node * predicate, const ColumnPtr & filtered_databases_column, ContextPtr context)
-{
-    Block sample {
-        ColumnWithTypeAndName(nullptr, std::make_shared<DataTypeString>(), "name"),
-        ColumnWithTypeAndName(nullptr, std::make_shared<DataTypeString>(), "engine")
-    };
-
-    MutableColumnPtr database_column = ColumnString::create();
-    MutableColumnPtr engine_column;
-
-    auto dag = VirtualColumnUtils::splitFilterDagForAllowedInputs(predicate, &sample);
-    if (dag)
-    {
-        bool filter_by_engine = false;
-        for (const auto * input : dag->getInputs())
-            if (input->result_name == "engine")
-                filter_by_engine = true;
-
-        if (filter_by_engine)
-            engine_column = ColumnString::create();
-    }
-
-    for (size_t database_idx = 0; database_idx < filtered_databases_column->size(); ++database_idx)
-    {
-        const auto & database_name = filtered_databases_column->getDataAt(database_idx).toString();
-        DatabasePtr database = DatabaseCatalog::instance().tryGetDatabase(database_name);
-        if (!database)
-            continue;
-
-        for (auto table_it = database->getTablesIterator(context); table_it->isValid(); table_it->next())
-        {
-            database_column->insert(table_it->name());
-            if (engine_column)
-                engine_column->insert(table_it->table()->getName());
-        }
-    }
-
-    Block block {ColumnWithTypeAndName(std::move(database_column), std::make_shared<DataTypeString>(), "name")};
-    if (engine_column)
-        block.insert(ColumnWithTypeAndName(std::move(engine_column), std::make_shared<DataTypeString>(), "engine"));
-
-    if (dag)
-        VirtualColumnUtils::filterBlockWithDAG(dag, block, context);
-
-    return block.getByPosition(0).column;
-}
-
-/// Avoid heavy operation on tables if we only queried columns that we can get without table object.
-/// Otherwise it will require table initialization for Lazy database.
-bool needTable(const DatabasePtr & database, const Block & header)
-{
-    if (database->getEngineName() != "Lazy")
-        return true;
-
-    static const std::set<std::string> columns_without_table = { "database", "name", "uuid", "metadata_modification_time" };
-    for (const auto & column : header.getColumnsWithTypeAndName())
-    {
-        if (columns_without_table.find(column.name) == columns_without_table.end())
-            return true;
-    }
-    return false;
-}
-
-
-class TablesBlockSource : public ISource
-{
-public:
-    TablesBlockSource(
-        std::vector<UInt8> columns_mask_,
-        Block header,
-        UInt64 max_block_size_,
-        ColumnPtr databases_,
-        ColumnPtr tables_,
-        ContextPtr context_)
-        : ISource(std::move(header))
-        , columns_mask(std::move(columns_mask_))
-        , max_block_size(max_block_size_)
-        , databases(std::move(databases_))
-        , context(Context::createCopy(context_))
-    {
-        size_t size = tables_->size();
-        tables.reserve(size);
-        for (size_t idx = 0; idx < size; ++idx)
-            tables.insert(tables_->getDataAt(idx).toString());
-    }
-
-    String getName() const override { return "Tables"; }
-
-protected:
-    Chunk generate() override
-    {
-        if (done)
-            return {};
-
-        MutableColumns res_columns = getPort().getHeader().cloneEmptyColumns();
-
-        const auto access = context->getAccess();
-        const bool need_to_check_access_for_databases = !access->isGranted(AccessType::SHOW_TABLES);
-
-        size_t rows_count = 0;
-        while (rows_count < max_block_size)
-        {
-            if (tables_it && !tables_it->isValid())
-                ++database_idx;
-
-            while (database_idx < databases->size() && (!tables_it || !tables_it->isValid()))
-            {
-                database_name = databases->getDataAt(database_idx).toString();
-                database = DatabaseCatalog::instance().tryGetDatabase(database_name);
-
-                if (!database)
-                {
-                    /// Database was deleted just now or the user has no access.
-                    ++database_idx;
-                    continue;
-                }
-
-                break;
-            }
-
-            /// This is for temporary tables. They are output in single block regardless to max_block_size.
-            if (database_idx >= databases->size())
-            {
-                if (context->hasSessionContext())
-                {
-                    Tables external_tables = context->getSessionContext()->getExternalTables();
-
-                    for (auto & table : external_tables)
-                    {
-                        size_t src_index = 0;
-                        size_t res_index = 0;
-
-                        // database
-                        if (columns_mask[src_index++])
-                            res_columns[res_index++]->insertDefault();
-
-                        // name
-                        if (columns_mask[src_index++])
-                            res_columns[res_index++]->insert(table.first);
-
-                        // uuid
-                        if (columns_mask[src_index++])
-                            res_columns[res_index++]->insert(table.second->getStorageID().uuid);
-
-                        // engine
-                        if (columns_mask[src_index++])
-                            res_columns[res_index++]->insert(table.second->getName());
-
-                        // is_temporary
-                        if (columns_mask[src_index++])
-                            res_columns[res_index++]->insert(1u);
-
-                        // data_paths
-                        if (columns_mask[src_index++])
-                            res_columns[res_index++]->insertDefault();
-
-                        // metadata_path
-                        if (columns_mask[src_index++])
-                            res_columns[res_index++]->insertDefault();
-
-                        // metadata_modification_time
-                        if (columns_mask[src_index++])
-                            res_columns[res_index++]->insertDefault();
-
-                        // metadata_version
-                        // Temporary tables does not support replication
-                        if (columns_mask[src_index++])
-                            res_columns[res_index++]->insertDefault();
-
-                        // dependencies_database
-                        if (columns_mask[src_index++])
-                            res_columns[res_index++]->insertDefault();
-
-                        // dependencies_table
-                        if (columns_mask[src_index++])
-                            res_columns[res_index++]->insertDefault();
-
-                        // create_table_query
-                        if (columns_mask[src_index++])
-                        {
-                            auto temp_db = DatabaseCatalog::instance().getDatabaseForTemporaryTables();
-                            ASTPtr ast = temp_db ? temp_db->tryGetCreateTableQuery(table.second->getStorageID().getTableName(), context) : nullptr;
-                            res_columns[res_index++]->insert(ast ? format({context, *ast}) : "");
-                        }
-
-                        // engine_full
-                        if (columns_mask[src_index++])
-                            res_columns[res_index++]->insert(table.second->getName());
-
-                        const auto & settings = context->getSettingsRef();
-                        while (src_index < columns_mask.size())
-                        {
-                            // total_rows
-                            if (src_index == 19 && columns_mask[src_index])
-                            {
-                                if (auto total_rows = table.second->totalRows(settings))
-                                    res_columns[res_index++]->insert(*total_rows);
-                                else
-                                    res_columns[res_index++]->insertDefault();
-                            }
-                            // total_bytes
-                            else if (src_index == 20 && columns_mask[src_index])
-                            {
-                                if (auto total_bytes = table.second->totalBytes(settings))
-                                    res_columns[res_index++]->insert(*total_bytes);
-                                else
-                                    res_columns[res_index++]->insertDefault();
-                            }
-                            /// Fill the rest columns with defaults
-                            else if (columns_mask[src_index])
-                                res_columns[res_index++]->insertDefault();
-                            src_index++;
-                        }
-                    }
-                }
-
-                UInt64 num_rows = res_columns.at(0)->size();
-                done = true;
-                return Chunk(std::move(res_columns), num_rows);
-            }
-
-            const bool need_to_check_access_for_tables = need_to_check_access_for_databases && !access->isGranted(AccessType::SHOW_TABLES, database_name);
-
-            if (!tables_it || !tables_it->isValid())
-                tables_it = database->getTablesIterator(context);
-
-            const bool need_table = needTable(database, getPort().getHeader());
-
-            for (; rows_count < max_block_size && tables_it->isValid(); tables_it->next())
-            {
-                auto table_name = tables_it->name();
-                if (!tables.contains(table_name))
-                    continue;
-
-                if (need_to_check_access_for_tables && !access->isGranted(AccessType::SHOW_TABLES, database_name, table_name))
-                    continue;
-
-                StoragePtr table = nullptr;
-                TableLockHolder lock;
-                if (need_table)
-                {
-                    table = tables_it->table();
-                    if (!table)
-                        // Table might have just been removed or detached for Lazy engine (see DatabaseLazy::tryGetTable())
-                        continue;
-
-                    /// The only column that requires us to hold a shared lock is data_paths as rename might alter them (on ordinary tables)
-                    /// and it's not protected internally by other mutexes
-                    static const size_t DATA_PATHS_INDEX = 5;
-                    if (columns_mask[DATA_PATHS_INDEX])
-                    {
-                        lock = table->tryLockForShare(context->getCurrentQueryId(),
-                                                      context->getSettingsRef().lock_acquire_timeout);
-                        if (!lock)
-                            // Table was dropped while acquiring the lock, skipping table
-                            continue;
-                    }
-                }
-                ++rows_count;
-
-                size_t src_index = 0;
-                size_t res_index = 0;
-
-                if (columns_mask[src_index++])
-                    res_columns[res_index++]->insert(database_name);
-
-                if (columns_mask[src_index++])
-                    res_columns[res_index++]->insert(table_name);
-
-                if (columns_mask[src_index++])
-                    res_columns[res_index++]->insert(tables_it->uuid());
-
-                if (columns_mask[src_index++])
-                {
-                    chassert(table != nullptr);
-                    res_columns[res_index++]->insert(table->getName());
-                }
-
-                if (columns_mask[src_index++])
-                    res_columns[res_index++]->insert(0u);  // is_temporary
-
-                if (columns_mask[src_index++])
-                {
-                    chassert(lock != nullptr);
-                    Array table_paths_array;
-                    auto paths = table->getDataPaths();
-                    table_paths_array.reserve(paths.size());
-                    for (const String & path : paths)
-                        table_paths_array.push_back(path);
-                    res_columns[res_index++]->insert(table_paths_array);
-                    /// We don't need the lock anymore
-                    lock = nullptr;
-                }
-
-                if (columns_mask[src_index++])
-                    res_columns[res_index++]->insert(database->getObjectMetadataPath(table_name));
-
-                if (columns_mask[src_index++])
-                    res_columns[res_index++]->insert(static_cast<UInt64>(database->getObjectMetadataModificationTime(table_name)));
-
-                StorageMetadataPtr metadata_snapshot;
-                if (table)
-                    metadata_snapshot = table->getInMemoryMetadataPtr();
-
-                if (columns_mask[src_index++])
-                {
-                    if (metadata_snapshot && table->supportsReplication())
-                        res_columns[res_index++]->insert(metadata_snapshot->metadata_version);
-                    else
-                        res_columns[res_index++]->insertDefault();
-                }
-
-                {
-                    Array views_table_name_array;
-                    Array views_database_name_array;
-                    if (columns_mask[src_index] || columns_mask[src_index + 1])
-                    {
-                        const auto view_ids = DatabaseCatalog::instance().getDependentViews(StorageID(database_name, table_name));
-
-                        views_table_name_array.reserve(view_ids.size());
-                        views_database_name_array.reserve(view_ids.size());
-                        for (const auto & view_id : view_ids)
-                        {
-                            views_table_name_array.push_back(view_id.table_name);
-                            views_database_name_array.push_back(view_id.database_name);
-                        }
-                    }
-
-                    if (columns_mask[src_index++])
-                        res_columns[res_index++]->insert(views_database_name_array);
-
-                    if (columns_mask[src_index++])
-                        res_columns[res_index++]->insert(views_table_name_array);
-                }
-
-                if (columns_mask[src_index] || columns_mask[src_index + 1] || columns_mask[src_index + 2])
-                {
-                    ASTPtr ast = database->tryGetCreateTableQuery(table_name, context);
-                    auto * ast_create = ast ? ast->as<ASTCreateQuery>() : nullptr;
-
-                    if (ast_create && !context->getSettingsRef().show_table_uuid_in_table_create_query_if_not_nil)
-                    {
-                        ast_create->uuid = UUIDHelpers::Nil;
-                        ast_create->to_inner_uuid = UUIDHelpers::Nil;
-                    }
-
-                    if (columns_mask[src_index++])
-                        res_columns[res_index++]->insert(ast ? format({context, *ast}) : "");
-
-                    if (columns_mask[src_index++])
-                    {
-                        String engine_full;
-
-                        if (ast_create && ast_create->storage)
-                        {
-                            engine_full = format({context, *ast_create->storage});
-
-                            static const char * const extra_head = " ENGINE = ";
-                            if (startsWith(engine_full, extra_head))
-                                engine_full = engine_full.substr(strlen(extra_head));
-                        }
-
-                        res_columns[res_index++]->insert(engine_full);
-                    }
-
-                    if (columns_mask[src_index++])
-                    {
-                        String as_select;
-                        if (ast_create && ast_create->select)
-                            as_select = format({context, *ast_create->select});
-                        res_columns[res_index++]->insert(as_select);
-                    }
-                }
-                else
-                    src_index += 3;
-
-                ASTPtr expression_ptr;
-                if (columns_mask[src_index++])
-                {
-                    if (metadata_snapshot && (expression_ptr = metadata_snapshot->getPartitionKeyAST()))
-                        res_columns[res_index++]->insert(format({context, *expression_ptr}));
-                    else
-                        res_columns[res_index++]->insertDefault();
-                }
-
-                if (columns_mask[src_index++])
-                {
-                    if (metadata_snapshot && (expression_ptr = metadata_snapshot->getSortingKey().expression_list_ast))
-                        res_columns[res_index++]->insert(format({context, *expression_ptr}));
-                    else
-                        res_columns[res_index++]->insertDefault();
-                }
-
-                if (columns_mask[src_index++])
-                {
-                    if (metadata_snapshot && (expression_ptr = metadata_snapshot->getPrimaryKey().expression_list_ast))
-                        res_columns[res_index++]->insert(format({context, *expression_ptr}));
-                    else
-                        res_columns[res_index++]->insertDefault();
-                }
-
-                if (columns_mask[src_index++])
-                {
-                    if (metadata_snapshot && (expression_ptr = metadata_snapshot->getSamplingKeyAST()))
-                        res_columns[res_index++]->insert(format({context, *expression_ptr}));
-                    else
-                        res_columns[res_index++]->insertDefault();
-                }
-
-                if (columns_mask[src_index++])
-                {
-                    auto policy = table ? table->getStoragePolicy() : nullptr;
-                    if (policy)
-                        res_columns[res_index++]->insert(policy->getName());
-                    else
-                        res_columns[res_index++]->insertDefault();
-                }
-
-                auto settings = context->getSettingsRef();
-                settings.select_sequential_consistency = 0;
-                if (columns_mask[src_index++])
-                {
-                    auto total_rows = table ? table->totalRows(settings) : std::nullopt;
-                    if (total_rows)
-                        res_columns[res_index++]->insert(*total_rows);
-                    else
-                        res_columns[res_index++]->insertDefault();
-                }
-
-                if (columns_mask[src_index++])
-                {
-                    auto total_bytes = table->totalBytes(settings);
-                    if (total_bytes)
-                        res_columns[res_index++]->insert(*total_bytes);
-                    else
-                        res_columns[res_index++]->insertDefault();
-                }
-
-                if (columns_mask[src_index++])
-                {
-                    auto total_bytes_uncompressed = table->totalBytesUncompressed(settings);
-                    if (total_bytes_uncompressed)
-                        res_columns[res_index++]->insert(*total_bytes_uncompressed);
-                    else
-                        res_columns[res_index++]->insertDefault();
-                }
-
-                auto table_merge_tree = std::dynamic_pointer_cast<MergeTreeData>(table);
-                if (columns_mask[src_index++])
-                {
-                    if (table_merge_tree)
-                        res_columns[res_index++]->insert(table_merge_tree->getAllPartsCount());
-                    else
-                        res_columns[res_index++]->insertDefault();
-                }
-
-                if (columns_mask[src_index++])
-                {
-                    if (table_merge_tree)
-                        res_columns[res_index++]->insert(table_merge_tree->getActivePartsCount());
-                    else
-                        res_columns[res_index++]->insertDefault();
-                }
-
-                if (columns_mask[src_index++])
-                {
-                    if (table_merge_tree)
-                    {
-                        res_columns[res_index++]->insert(table_merge_tree->getTotalMarksCount());
-                    }
-                    else
-                        res_columns[res_index++]->insertDefault();
-                }
-
-                if (columns_mask[src_index++])
-                {
-                    auto lifetime_rows = table ? table->lifetimeRows() : std::nullopt;
-                    if (lifetime_rows)
-                        res_columns[res_index++]->insert(*lifetime_rows);
-                    else
-                        res_columns[res_index++]->insertDefault();
-                }
-
-                if (columns_mask[src_index++])
-                {
-                    auto lifetime_bytes = table ? table->lifetimeBytes() : std::nullopt;
-                    if (lifetime_bytes)
-                        res_columns[res_index++]->insert(*lifetime_bytes);
-                    else
-                        res_columns[res_index++]->insertDefault();
-                }
-
-                if (columns_mask[src_index++])
-                {
-                    if (metadata_snapshot)
-                        res_columns[res_index++]->insert(metadata_snapshot->comment);
-                    else
-                        res_columns[res_index++]->insertDefault();
-                }
-
-                if (columns_mask[src_index++])
-                {
-                    if (table)
-                        res_columns[res_index++]->insert(table->storesDataOnDisk());
-                    else
-                        res_columns[res_index++]->insertDefault();
-                }
-
-                if (columns_mask[src_index] || columns_mask[src_index + 1] || columns_mask[src_index + 2] || columns_mask[src_index + 3])
-                {
-                    auto dependencies = DatabaseCatalog::instance().getLoadingDependencies(StorageID{database_name, table_name});
-                    auto dependents = DatabaseCatalog::instance().getLoadingDependents(StorageID{database_name, table_name});
-
-                    Array dependencies_databases;
-                    Array dependencies_tables;
-                    dependencies_databases.reserve(dependencies.size());
-                    dependencies_tables.reserve(dependencies.size());
-                    for (const auto & dependency : dependencies)
-                    {
-                        dependencies_databases.push_back(dependency.database_name);
-                        dependencies_tables.push_back(dependency.table_name);
-                    }
-
-                    Array dependents_databases;
-                    Array dependents_tables;
-                    dependents_databases.reserve(dependents.size());
-                    dependents_tables.reserve(dependents.size());
-                    for (const auto & dependent : dependents)
-                    {
-                        dependents_databases.push_back(dependent.database_name);
-                        dependents_tables.push_back(dependent.table_name);
-                    }
-
-                    if (columns_mask[src_index++])
-                        res_columns[res_index++]->insert(dependencies_databases);
-                    if (columns_mask[src_index++])
-                        res_columns[res_index++]->insert(dependencies_tables);
-
-                    if (columns_mask[src_index++])
-                        res_columns[res_index++]->insert(dependents_databases);
-                    if (columns_mask[src_index++])
-                        res_columns[res_index++]->insert(dependents_tables);
-
-                }
-            }
-        }
-
-        UInt64 num_rows = res_columns.at(0)->size();
-        return Chunk(std::move(res_columns), num_rows);
-    }
-private:
-    std::vector<UInt8> columns_mask;
-    UInt64 max_block_size;
-    ColumnPtr databases;
-    NameSet tables;
-    size_t database_idx = 0;
-    DatabaseTablesIteratorPtr tables_it;
-    ContextPtr context;
-    bool done = false;
-    DatabasePtr database;
-    std::string database_name;
-};
-
-}
-
-class ReadFromSystemTables : public SourceStepWithFilter
-{
-public:
-    std::string getName() const override { return "ReadFromSystemTables"; }
-    void initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) override;
-
-    ReadFromSystemTables(
-        const Names & column_names_,
-        const SelectQueryInfo & query_info_,
-        const StorageSnapshotPtr & storage_snapshot_,
-        const ContextPtr & context_,
-        Block sample_block,
-        std::vector<UInt8> columns_mask_,
-        size_t max_block_size_)
-        : SourceStepWithFilter(
-            DataStream{.header = std::move(sample_block)},
-            column_names_,
-            query_info_,
-            storage_snapshot_,
-            context_)
-        , columns_mask(std::move(columns_mask_))
-        , max_block_size(max_block_size_)
-    {
-    }
-
-    void applyFilters(ActionDAGNodes added_filter_nodes) override;
-
-private:
-    std::vector<UInt8> columns_mask;
-    size_t max_block_size;
-
-    ColumnPtr filtered_databases_column;
-    ColumnPtr filtered_tables_column;
-};
-
 void StorageSystemTables::read(
     QueryPlan & query_plan,
     const Names & column_names,
@@ -747,24 +126,4 @@ void StorageSystemTables::read(
 
     query_plan.addStep(std::move(reading));
 }
-
-void ReadFromSystemTables::applyFilters(ActionDAGNodes added_filter_nodes)
-{
-    SourceStepWithFilter::applyFilters(std::move(added_filter_nodes));
-
-    const ActionsDAG::Node * predicate = nullptr;
-    if (filter_actions_dag)
-        predicate = filter_actions_dag->getOutputs().at(0);
-
-    filtered_databases_column = getFilteredDatabases(predicate, context);
-    filtered_tables_column = getFilteredTables(predicate, filtered_databases_column, context);
-}
-
-void ReadFromSystemTables::initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &)
-{
-    Pipe pipe(std::make_shared<TablesBlockSource>(
-        std::move(columns_mask), getOutputStream().header, max_block_size, std::move(filtered_databases_column), std::move(filtered_tables_column), context));
-    pipeline.init(std::move(pipe));
-}
-
 }
diff --git a/src/Storages/System/TablesBlockSource.cpp b/src/Storages/System/TablesBlockSource.cpp
new file mode 100644
index 00000000000..d5481856f1b
--- /dev/null
+++ b/src/Storages/System/TablesBlockSource.cpp
@@ -0,0 +1,564 @@
+#include "TablesBlockSource.h"
+
+#include <memory>
+
+#include <Access/ContextAccess.h>
+#include <Disks/IStoragePolicy.h>
+#include <Interpreters/DatabaseCatalog.h>
+#include <Interpreters/formatWithPossiblyHidingSecrets.h>
+#include <Parsers/ASTSelectWithUnionQuery.h>
+#include <Storages/IStorage.h>
+#include <Storages/MergeTree/MergeTreeData.h>
+
+
+namespace DB
+{
+
+namespace
+{
+
+/// Avoid heavy operation on tables if we only queried columns that we can get without table object.
+/// Otherwise it will require table initialization for Lazy database.
+bool needTable(const DatabasePtr & database, const Block & header)
+{
+    if (database->getEngineName() != "Lazy")
+        return true;
+
+    static const std::set<std::string> columns_without_table = {"database", "name", "uuid", "metadata_modification_time"};
+    for (const auto & column : header.getColumnsWithTypeAndName())
+    {
+        if (columns_without_table.find(column.name) == columns_without_table.end())
+            return true;
+    }
+    return false;
+}
+
+}
+
+Chunk TablesBlockSource::generate()
+{
+    if (done)
+        return {};
+
+    MutableColumns res_columns = getPort().getHeader().cloneEmptyColumns();
+
+    const auto access = context->getAccess();
+    const bool need_to_check_access_for_databases = !access->isGranted(AccessType::SHOW_TABLES);
+
+    size_t rows_count = 0;
+    while (rows_count < max_block_size)
+    {
+        if (tables_it && !tables_it->isValid())
+            ++database_idx;
+
+        while (database_idx < databases->size() && (!tables_it || !tables_it->isValid()))
+        {
+            database_name = databases->getDataAt(database_idx).toString();
+            database = DatabaseCatalog::instance().tryGetDatabase(database_name);
+
+            if (!database)
+            {
+                /// Database was deleted just now or the user has no access.
+                ++database_idx;
+                continue;
+            }
+
+            break;
+        }
+
+        /// This is for temporary tables. They are output in single block regardless to max_block_size.
+        if (database_idx >= databases->size())
+        {
+            if (context->hasSessionContext())
+            {
+                Tables external_tables = context->getSessionContext()->getExternalTables();
+
+                for (auto & table : external_tables)
+                {
+                    size_t src_index = 0;
+                    size_t res_index = 0;
+
+                    // database
+                    if (columns_mask[src_index++])
+                        res_columns[res_index++]->insertDefault();
+
+                    // name
+                    if (columns_mask[src_index++])
+                        res_columns[res_index++]->insert(table.first);
+
+                    // uuid
+                    if (columns_mask[src_index++])
+                        res_columns[res_index++]->insert(table.second->getStorageID().uuid);
+
+                    // engine
+                    if (columns_mask[src_index++])
+                        res_columns[res_index++]->insert(table.second->getName());
+
+                    // is_temporary
+                    if (columns_mask[src_index++])
+                        res_columns[res_index++]->insert(1u);
+
+                    // data_paths
+                    if (columns_mask[src_index++])
+                        res_columns[res_index++]->insertDefault();
+
+                    // metadata_path
+                    if (columns_mask[src_index++])
+                        res_columns[res_index++]->insertDefault();
+
+                    // metadata_modification_time
+                    if (columns_mask[src_index++])
+                        res_columns[res_index++]->insertDefault();
+
+                    // metadata_version
+                    // Temporary tables does not support replication
+                    if (columns_mask[src_index++])
+                        res_columns[res_index++]->insertDefault();
+
+                    // dependencies_database
+                    if (columns_mask[src_index++])
+                        res_columns[res_index++]->insertDefault();
+
+                    // dependencies_table
+                    if (columns_mask[src_index++])
+                        res_columns[res_index++]->insertDefault();
+
+                    // create_table_query
+                    if (columns_mask[src_index++])
+                    {
+                        auto temp_db = DatabaseCatalog::instance().getDatabaseForTemporaryTables();
+                        ASTPtr ast
+                            = temp_db ? temp_db->tryGetCreateTableQuery(table.second->getStorageID().getTableName(), context) : nullptr;
+                        res_columns[res_index++]->insert(ast ? format({context, *ast}) : "");
+                    }
+
+                    // engine_full
+                    if (columns_mask[src_index++])
+                        res_columns[res_index++]->insert(table.second->getName());
+
+                    const auto & settings = context->getSettingsRef();
+                    while (src_index < columns_mask.size())
+                    {
+                        // total_rows
+                        if (src_index == 19 && columns_mask[src_index])
+                        {
+                            if (auto total_rows = table.second->totalRows(settings))
+                                res_columns[res_index++]->insert(*total_rows);
+                            else
+                                res_columns[res_index++]->insertDefault();
+                        }
+                        // total_bytes
+                        else if (src_index == 20 && columns_mask[src_index])
+                        {
+                            if (auto total_bytes = table.second->totalBytes(settings))
+                                res_columns[res_index++]->insert(*total_bytes);
+                            else
+                                res_columns[res_index++]->insertDefault();
+                        }
+                        /// Fill the rest columns with defaults
+                        else if (columns_mask[src_index])
+                            res_columns[res_index++]->insertDefault();
+                        src_index++;
+                    }
+                }
+            }
+
+            UInt64 num_rows = res_columns.at(0)->size();
+            done = true;
+            return Chunk(std::move(res_columns), num_rows);
+        }
+
+        const bool need_to_check_access_for_tables
+            = need_to_check_access_for_databases && !access->isGranted(AccessType::SHOW_TABLES, database_name);
+
+        if (!tables_it || !tables_it->isValid())
+            tables_it = database->getTablesIterator(context);
+
+        const bool need_table = needTable(database, getPort().getHeader());
+
+        for (; rows_count < max_block_size && tables_it->isValid(); tables_it->next())
+        {
+            auto table_name = tables_it->name();
+            if (!tables.contains(table_name))
+                continue;
+
+            if (need_to_check_access_for_tables && !access->isGranted(AccessType::SHOW_TABLES, database_name, table_name))
+                continue;
+
+            StoragePtr table = nullptr;
+            TableLockHolder lock;
+            if (need_table)
+            {
+                table = tables_it->table();
+                if (!table)
+                    // Table might have just been removed or detached for Lazy engine (see DatabaseLazy::tryGetTable())
+                    continue;
+
+                /// The only column that requires us to hold a shared lock is data_paths as rename might alter them (on ordinary tables)
+                /// and it's not protected internally by other mutexes
+                static const size_t DATA_PATHS_INDEX = 5;
+                if (columns_mask[DATA_PATHS_INDEX])
+                {
+                    lock = table->tryLockForShare(context->getCurrentQueryId(), context->getSettingsRef().lock_acquire_timeout);
+                    if (!lock)
+                        // Table was dropped while acquiring the lock, skipping table
+                        continue;
+                }
+            }
+            ++rows_count;
+
+            size_t src_index = 0;
+            size_t res_index = 0;
+
+            if (columns_mask[src_index++])
+                res_columns[res_index++]->insert(database_name);
+
+            if (columns_mask[src_index++])
+                res_columns[res_index++]->insert(table_name);
+
+            if (columns_mask[src_index++])
+                res_columns[res_index++]->insert(tables_it->uuid());
+
+            if (columns_mask[src_index++])
+            {
+                chassert(table != nullptr);
+                res_columns[res_index++]->insert(table->getName());
+            }
+
+            if (columns_mask[src_index++])
+                res_columns[res_index++]->insert(0u); // is_temporary
+
+            if (columns_mask[src_index++])
+            {
+                chassert(lock != nullptr);
+                Array table_paths_array;
+                auto paths = table->getDataPaths();
+                table_paths_array.reserve(paths.size());
+                for (const String & path : paths)
+                    table_paths_array.push_back(path);
+                res_columns[res_index++]->insert(table_paths_array);
+                /// We don't need the lock anymore
+                lock = nullptr;
+            }
+
+            if (columns_mask[src_index++])
+                res_columns[res_index++]->insert(database->getObjectMetadataPath(table_name));
+
+            if (columns_mask[src_index++])
+                res_columns[res_index++]->insert(static_cast<UInt64>(database->getObjectMetadataModificationTime(table_name)));
+
+            StorageMetadataPtr metadata_snapshot;
+            if (table)
+                metadata_snapshot = table->getInMemoryMetadataPtr();
+
+            if (columns_mask[src_index++])
+            {
+                if (metadata_snapshot && table->supportsReplication())
+                    res_columns[res_index++]->insert(metadata_snapshot->metadata_version);
+                else
+                    res_columns[res_index++]->insertDefault();
+            }
+
+            {
+                Array views_table_name_array;
+                Array views_database_name_array;
+                if (columns_mask[src_index] || columns_mask[src_index + 1])
+                {
+                    const auto view_ids = DatabaseCatalog::instance().getDependentViews(StorageID(database_name, table_name));
+
+                    views_table_name_array.reserve(view_ids.size());
+                    views_database_name_array.reserve(view_ids.size());
+                    for (const auto & view_id : view_ids)
+                    {
+                        views_table_name_array.push_back(view_id.table_name);
+                        views_database_name_array.push_back(view_id.database_name);
+                    }
+                }
+
+                if (columns_mask[src_index++])
+                    res_columns[res_index++]->insert(views_database_name_array);
+
+                if (columns_mask[src_index++])
+                    res_columns[res_index++]->insert(views_table_name_array);
+            }
+
+            if (columns_mask[src_index] || columns_mask[src_index + 1] || columns_mask[src_index + 2])
+            {
+                ASTPtr ast = database->tryGetCreateTableQuery(table_name, context);
+                auto * ast_create = ast ? ast->as<ASTCreateQuery>() : nullptr;
+
+                if (ast_create && !context->getSettingsRef().show_table_uuid_in_table_create_query_if_not_nil)
+                {
+                    ast_create->uuid = UUIDHelpers::Nil;
+                    ast_create->to_inner_uuid = UUIDHelpers::Nil;
+                }
+
+                if (columns_mask[src_index++])
+                    res_columns[res_index++]->insert(ast ? format({context, *ast}) : "");
+
+                if (columns_mask[src_index++])
+                {
+                    String engine_full;
+
+                    if (ast_create && ast_create->storage)
+                    {
+                        engine_full = format({context, *ast_create->storage});
+
+                        static const char * const extra_head = " ENGINE = ";
+                        if (startsWith(engine_full, extra_head))
+                            engine_full = engine_full.substr(strlen(extra_head));
+                    }
+
+                    res_columns[res_index++]->insert(engine_full);
+                }
+
+                if (columns_mask[src_index++])
+                {
+                    String as_select;
+                    if (ast_create && ast_create->select)
+                        as_select = format({context, *ast_create->select});
+                    res_columns[res_index++]->insert(as_select);
+                }
+            }
+            else
+                src_index += 3;
+
+            ASTPtr expression_ptr;
+            if (columns_mask[src_index++])
+            {
+                if (metadata_snapshot && (expression_ptr = metadata_snapshot->getPartitionKeyAST()))
+                    res_columns[res_index++]->insert(format({context, *expression_ptr}));
+                else
+                    res_columns[res_index++]->insertDefault();
+            }
+
+            if (columns_mask[src_index++])
+            {
+                if (metadata_snapshot && (expression_ptr = metadata_snapshot->getSortingKey().expression_list_ast))
+                    res_columns[res_index++]->insert(format({context, *expression_ptr}));
+                else
+                    res_columns[res_index++]->insertDefault();
+            }
+
+            if (columns_mask[src_index++])
+            {
+                if (metadata_snapshot && (expression_ptr = metadata_snapshot->getPrimaryKey().expression_list_ast))
+                    res_columns[res_index++]->insert(format({context, *expression_ptr}));
+                else
+                    res_columns[res_index++]->insertDefault();
+            }
+
+            if (columns_mask[src_index++])
+            {
+                if (metadata_snapshot && (expression_ptr = metadata_snapshot->getSamplingKeyAST()))
+                    res_columns[res_index++]->insert(format({context, *expression_ptr}));
+                else
+                    res_columns[res_index++]->insertDefault();
+            }
+
+            if (columns_mask[src_index++])
+            {
+                auto policy = table ? table->getStoragePolicy() : nullptr;
+                if (policy)
+                    res_columns[res_index++]->insert(policy->getName());
+                else
+                    res_columns[res_index++]->insertDefault();
+            }
+
+            auto settings = context->getSettingsRef();
+            settings.select_sequential_consistency = 0;
+            if (columns_mask[src_index++])
+            {
+                auto total_rows = table ? table->totalRows(settings) : std::nullopt;
+                if (total_rows)
+                    res_columns[res_index++]->insert(*total_rows);
+                else
+                    res_columns[res_index++]->insertDefault();
+            }
+
+            if (columns_mask[src_index++])
+            {
+                auto total_bytes = table->totalBytes(settings);
+                if (total_bytes)
+                    res_columns[res_index++]->insert(*total_bytes);
+                else
+                    res_columns[res_index++]->insertDefault();
+            }
+
+            if (columns_mask[src_index++])
+            {
+                auto total_bytes_uncompressed = table->totalBytesUncompressed(settings);
+                if (total_bytes_uncompressed)
+                    res_columns[res_index++]->insert(*total_bytes_uncompressed);
+                else
+                    res_columns[res_index++]->insertDefault();
+            }
+
+            auto table_merge_tree = std::dynamic_pointer_cast<MergeTreeData>(table);
+            if (columns_mask[src_index++])
+            {
+                if (table_merge_tree)
+                    res_columns[res_index++]->insert(table_merge_tree->getAllPartsCount());
+                else
+                    res_columns[res_index++]->insertDefault();
+            }
+
+            if (columns_mask[src_index++])
+            {
+                if (table_merge_tree)
+                    res_columns[res_index++]->insert(table_merge_tree->getActivePartsCount());
+                else
+                    res_columns[res_index++]->insertDefault();
+            }
+
+            if (columns_mask[src_index++])
+            {
+                if (table_merge_tree)
+                {
+                    res_columns[res_index++]->insert(table_merge_tree->getTotalMarksCount());
+                }
+                else
+                    res_columns[res_index++]->insertDefault();
+            }
+
+            if (columns_mask[src_index++])
+            {
+                auto lifetime_rows = table ? table->lifetimeRows() : std::nullopt;
+                if (lifetime_rows)
+                    res_columns[res_index++]->insert(*lifetime_rows);
+                else
+                    res_columns[res_index++]->insertDefault();
+            }
+
+            if (columns_mask[src_index++])
+            {
+                auto lifetime_bytes = table ? table->lifetimeBytes() : std::nullopt;
+                if (lifetime_bytes)
+                    res_columns[res_index++]->insert(*lifetime_bytes);
+                else
+                    res_columns[res_index++]->insertDefault();
+            }
+
+            if (columns_mask[src_index++])
+            {
+                if (metadata_snapshot)
+                    res_columns[res_index++]->insert(metadata_snapshot->comment);
+                else
+                    res_columns[res_index++]->insertDefault();
+            }
+
+            if (columns_mask[src_index++])
+            {
+                if (table)
+                    res_columns[res_index++]->insert(table->storesDataOnDisk());
+                else
+                    res_columns[res_index++]->insertDefault();
+            }
+
+            if (columns_mask[src_index] || columns_mask[src_index + 1] || columns_mask[src_index + 2] || columns_mask[src_index + 3])
+            {
+                auto dependencies = DatabaseCatalog::instance().getLoadingDependencies(StorageID{database_name, table_name});
+                auto dependents = DatabaseCatalog::instance().getLoadingDependents(StorageID{database_name, table_name});
+
+                Array dependencies_databases;
+                Array dependencies_tables;
+                dependencies_databases.reserve(dependencies.size());
+                dependencies_tables.reserve(dependencies.size());
+                for (const auto & dependency : dependencies)
+                {
+                    dependencies_databases.push_back(dependency.database_name);
+                    dependencies_tables.push_back(dependency.table_name);
+                }
+
+                Array dependents_databases;
+                Array dependents_tables;
+                dependents_databases.reserve(dependents.size());
+                dependents_tables.reserve(dependents.size());
+                for (const auto & dependent : dependents)
+                {
+                    dependents_databases.push_back(dependent.database_name);
+                    dependents_tables.push_back(dependent.table_name);
+                }
+
+                if (columns_mask[src_index++])
+                    res_columns[res_index++]->insert(dependencies_databases);
+                if (columns_mask[src_index++])
+                    res_columns[res_index++]->insert(dependencies_tables);
+
+                if (columns_mask[src_index++])
+                    res_columns[res_index++]->insert(dependents_databases);
+                if (columns_mask[src_index++])
+                    res_columns[res_index++]->insert(dependents_tables);
+            }
+        }
+    }
+
+    UInt64 num_rows = res_columns.at(0)->size();
+    return Chunk(std::move(res_columns), num_rows);
+}
+
+Chunk DetachedTablesBlockSource::generate()
+{
+    if (done)
+        return {};
+
+    MutableColumns result_columns = getPort().getHeader().cloneEmptyColumns();
+
+    const auto access = context->getAccess();
+    const bool need_to_check_access_for_databases = !access->isGranted(AccessType::SHOW_TABLES);
+
+    for (size_t database_idx = 0, rows_count = 0; database_idx < databases->size() && rows_count < max_block_size; ++database_idx)
+    {
+        database_name = databases->getDataAt(database_idx).toString();
+        database = DatabaseCatalog::instance().tryGetDatabase(database_name);
+
+        if (!database)
+            continue;
+
+        const bool need_to_check_access_for_tables
+            = need_to_check_access_for_databases && !access->isGranted(AccessType::SHOW_TABLES, database_name);
+
+        if (!detached_tables_it || !detached_tables_it->isValid())
+            detached_tables_it = database->getDetachedTablesIterator(context, {}, false);
+
+        for (; rows_count < max_block_size && detached_tables_it->isValid(); detached_tables_it->next())
+        {
+            const auto detached_table_name = detached_tables_it->table();
+
+            if (!detached_tables.contains(detached_table_name))
+                continue;
+
+            if (need_to_check_access_for_tables && !access->isGranted(AccessType::SHOW_TABLES, database_name, detached_table_name))
+                continue;
+
+            fillResultColumnsByDetachedTableIterator(result_columns);
+            ++rows_count;
+        }
+    }
+
+    const UInt64 num_rows = result_columns.at(0)->size();
+    done = true;
+    return Chunk(std::move(result_columns), num_rows);
+}
+
+void DetachedTablesBlockSource::fillResultColumnsByDetachedTableIterator(MutableColumns & result_columns) const
+{
+    size_t src_index = 0;
+    size_t res_index = 0;
+
+    if (columns_mask[src_index++])
+        result_columns[res_index++]->insert(detached_tables_it->database());
+
+    if (columns_mask[src_index++])
+        result_columns[res_index++]->insert(detached_tables_it->table());
+
+    if (columns_mask[src_index++])
+        result_columns[res_index++]->insert(detached_tables_it->uuid());
+
+    if (columns_mask[src_index++])
+        result_columns[res_index++]->insert(detached_tables_it->metadataPath());
+
+    if (columns_mask[src_index++])
+        result_columns[res_index++]->insert(detached_tables_it->isPermanently());
+}
+}
diff --git a/src/Storages/System/TablesBlockSource.h b/src/Storages/System/TablesBlockSource.h
new file mode 100644
index 00000000000..a9f2226dca8
--- /dev/null
+++ b/src/Storages/System/TablesBlockSource.h
@@ -0,0 +1,92 @@
+#pragma once
+
+#include <Databases/IDatabase.h>
+#include <Interpreters/Context.h>
+#include <Processors/ISource.h>
+
+namespace DB
+{
+
+class TablesBlockSource : public ISource
+{
+public:
+    TablesBlockSource(
+        std::vector<UInt8> columns_mask_,
+        Block header,
+        UInt64 max_block_size_,
+        ColumnPtr databases_,
+        ColumnPtr tables_,
+        ContextPtr context_)
+        : ISource(std::move(header))
+        , columns_mask(std::move(columns_mask_))
+        , max_block_size(max_block_size_)
+        , databases(std::move(databases_))
+        , context(Context::createCopy(context_))
+    {
+        size_t size = tables_->size();
+        tables.reserve(size);
+        for (size_t idx = 0; idx < size; ++idx)
+            tables.insert(tables_->getDataAt(idx).toString());
+    }
+
+    String getName() const override { return "Tables"; }
+
+protected:
+    Chunk generate() override;
+
+private:
+    std::vector<UInt8> columns_mask;
+    UInt64 max_block_size;
+    ColumnPtr databases;
+    NameSet tables;
+    size_t database_idx = 0;
+    DatabaseTablesIteratorPtr tables_it;
+    ContextPtr context;
+    bool done = false;
+    DatabasePtr database;
+    std::string database_name;
+};
+
+class DetachedTablesBlockSource : public ISource
+{
+public:
+    DetachedTablesBlockSource(
+        std::vector<UInt8> columns_mask_,
+        Block header,
+        UInt64 max_block_size_,
+        ColumnPtr databases_,
+        ColumnPtr detached_tables_,
+        ContextPtr context_)
+        : ISource(std::move(header))
+        , columns_mask(std::move(columns_mask_))
+        , max_block_size(max_block_size_)
+        , databases(std::move(databases_))
+        , context(Context::createCopy(context_))
+    {
+        size_t size = detached_tables_->size();
+        detached_tables.reserve(size);
+        for (size_t idx = 0; idx < size; ++idx)
+        {
+            detached_tables.insert(detached_tables_->getDataAt(idx).toString());
+        }
+    }
+
+    String getName() const override { return "DetachedTables"; }
+
+protected:
+    Chunk generate() override;
+
+private:
+    const std::vector<UInt8> columns_mask;
+    const UInt64 max_block_size;
+    const ColumnPtr databases;
+    NameSet detached_tables;
+    DatabaseDetachedTablesSnapshotIteratorPtr detached_tables_it;
+    ContextPtr context;
+    bool done = false;
+    DatabasePtr database;
+    std::string database_name;
+
+    void fillResultColumnsByDetachedTableIterator(MutableColumns & result_columns) const;
+};
+}

From e06387e6da4a89d9aab145dd057b6a99348e3ee8 Mon Sep 17 00:00:00 2001
From: Konstantin Morozov <just.morozov.k@gmail.com>
Date: Thu, 27 Jun 2024 07:37:23 +0000
Subject: [PATCH 0103/1488] fix stle

---
 .../test_system_detached_tables/test.py       | 33 +++++++++++++------
 1 file changed, 23 insertions(+), 10 deletions(-)

diff --git a/tests/integration/test_system_detached_tables/test.py b/tests/integration/test_system_detached_tables/test.py
index bbac5e391f8..45bc70147b6 100644
--- a/tests/integration/test_system_detached_tables/test.py
+++ b/tests/integration/test_system_detached_tables/test.py
@@ -18,11 +18,19 @@ def test_system_detached_tables():
     node.query("CREATE TABLE test_table (n Int64) ENGINE=MergeTree ORDER BY n;")
     node.query("CREATE TABLE test_table_perm (n Int64) ENGINE=MergeTree ORDER BY n;")
 
-    test_table_uuid = node.query("SELECT uuid FROM system.tables WHERE table='test_table'").rstrip('\n')
-    test_table_metadata_path = node.query("SELECT metadata_path FROM system.tables WHERE table='test_table'").rstrip('\n')
+    test_table_uuid = node.query(
+        "SELECT uuid FROM system.tables WHERE table='test_table'"
+    ).rstrip('\n')
+    test_table_metadata_path = node.query(
+        "SELECT metadata_path FROM system.tables WHERE table='test_table'"
+    ).rstrip('\n')
 
-    test_table_perm_uuid = node.query("SELECT uuid FROM system.tables WHERE table='test_table_perm'").rstrip('\n')
-    test_table_perm_metadata_path = node.query("SELECT metadata_path FROM system.tables WHERE table='test_table_perm'").rstrip('\n')
+    test_table_perm_uuid = node.query(
+        "SELECT uuid FROM system.tables WHERE table='test_table_perm'"
+    ).rstrip('\n')
+    test_table_perm_metadata_path = node.query(
+        "SELECT metadata_path FROM system.tables WHERE table='test_table_perm'"
+    ).rstrip('\n')
 
     assert "" == node.query("SELECT * FROM system.detached_tables")
 
@@ -31,12 +39,17 @@ def test_system_detached_tables():
 
     querry = "SELECT database, table, is_permanently, uuid, metadata_path FROM system.detached_tables FORMAT Values"
     result = node.query(querry)
-    assert result == f"('default','test_table',0,'{test_table_uuid}','{test_table_metadata_path}'),('default','test_table_perm',1,'{test_table_perm_uuid}','{test_table_perm_metadata_path}')"
-
-    node.restart_clickhouse()
-
-    result = node.query(querry)
-    assert result == f"('default','test_table_perm',1,'{test_table_perm_uuid}','{test_table_perm_metadata_path}')"
+    assert (
+        result 
+        == f"('default','test_table',0,'{test_table_uuid}','{test_table_metadata_path}'),('default','test_table_perm',1,'{test_table_perm_uuid}','{test_table_perm_metadata_path}')"
+    )
+    node.restart_clickhouse()
+
+    result = node.query(querry)
+    assert (
+        result
+        == f"('default','test_table_perm',1,'{test_table_perm_uuid}','{test_table_perm_metadata_path}')"
+    )
 
     node.restart_clickhouse()
 

From 1c9ccbae1142d585f7534441a6abc5e75acfdce3 Mon Sep 17 00:00:00 2001
From: Konstantin Morozov <just.morozov.k@gmail.com>
Date: Thu, 27 Jun 2024 07:49:07 +0000
Subject: [PATCH 0104/1488] small up

---
 src/Databases/DatabasesCommon.cpp | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/src/Databases/DatabasesCommon.cpp b/src/Databases/DatabasesCommon.cpp
index 470625ddd8b..4cf914fea56 100644
--- a/src/Databases/DatabasesCommon.cpp
+++ b/src/Databases/DatabasesCommon.cpp
@@ -270,37 +270,36 @@ StoragePtr DatabaseWithOwnTablesBase::detachTable(ContextPtr /* context_ */, con
 
 StoragePtr DatabaseWithOwnTablesBase::detachTableUnlocked(const String & table_name)
 {
-    StoragePtr res;
-
     auto it = tables.find(table_name);
     if (it == tables.end())
         throw Exception(ErrorCodes::UNKNOWN_TABLE, "Table {}.{} doesn't exist",
                         backQuote(database_name), backQuote(table_name));
-    res = it->second;
+
+    auto table_storage = it->second;
 
     snapshot_detached_tables.emplace(
-        it->first,
+        table_name,
         SnapshotDetachedTable{
             .database = it->second->getStorageID().getDatabaseName(),
-            .table = it->first,
+            .table = table_name,
             .uuid = it->second->getStorageID().uuid,
-            .metadata_path = getObjectMetadataPath(it->first),
+            .metadata_path = getObjectMetadataPath(table_name),
             .is_permanently = false});
 
     tables.erase(it);
-    res->is_detached = true;
+    table_storage->is_detached = true;
 
-    if (res->isSystemStorage() == false)
-        CurrentMetrics::sub(getAttachedCounterForStorage(res), 1);
+    if (table_storage->isSystemStorage() == false)
+        CurrentMetrics::sub(getAttachedCounterForStorage(table_storage), 1);
 
-    auto table_id = res->getStorageID();
+    auto table_id = table_storage->getStorageID();
     if (table_id.hasUUID())
     {
         assert(database_name == DatabaseCatalog::TEMPORARY_DATABASE || getUUID() != UUIDHelpers::Nil);
         DatabaseCatalog::instance().removeUUIDMapping(table_id.uuid);
     }
 
-    return res;
+    return table_storage;
 }
 
 void DatabaseWithOwnTablesBase::attachTable(ContextPtr /* context_ */, const String & table_name, const StoragePtr & table, const String &)

From 215802e74eb841449af6cb80f25b66a3610707dd Mon Sep 17 00:00:00 2001
From: Konstantin Morozov <just.morozov.k@gmail.com>
Date: Thu, 27 Jun 2024 08:28:35 +0000
Subject: [PATCH 0105/1488] fix style

---
 .../test_system_detached_tables/test.py           | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/tests/integration/test_system_detached_tables/test.py b/tests/integration/test_system_detached_tables/test.py
index 45bc70147b6..dbccac92523 100644
--- a/tests/integration/test_system_detached_tables/test.py
+++ b/tests/integration/test_system_detached_tables/test.py
@@ -20,17 +20,17 @@ def test_system_detached_tables():
 
     test_table_uuid = node.query(
         "SELECT uuid FROM system.tables WHERE table='test_table'"
-    ).rstrip('\n')
+    ).rstrip("\n")
     test_table_metadata_path = node.query(
         "SELECT metadata_path FROM system.tables WHERE table='test_table'"
-    ).rstrip('\n')
+    ).rstrip("\n")
 
     test_table_perm_uuid = node.query(
         "SELECT uuid FROM system.tables WHERE table='test_table_perm'"
-    ).rstrip('\n')
+    ).rstrip("\n")
     test_table_perm_metadata_path = node.query(
         "SELECT metadata_path FROM system.tables WHERE table='test_table_perm'"
-    ).rstrip('\n')
+    ).rstrip("\n")
 
     assert "" == node.query("SELECT * FROM system.detached_tables")
 
@@ -40,7 +40,7 @@ def test_system_detached_tables():
     querry = "SELECT database, table, is_permanently, uuid, metadata_path FROM system.detached_tables FORMAT Values"
     result = node.query(querry)
     assert (
-        result 
+        result
         == f"('default','test_table',0,'{test_table_uuid}','{test_table_metadata_path}'),('default','test_table_perm',1,'{test_table_perm_uuid}','{test_table_perm_metadata_path}')"
     )
     node.restart_clickhouse()
@@ -54,4 +54,7 @@ def test_system_detached_tables():
     node.restart_clickhouse()
 
     result = node.query(querry)
-    assert result == f"('default','test_table_perm',1,'{test_table_perm_uuid}','{test_table_perm_metadata_path}')"
+    assert (
+        result
+        == f"('default','test_table_perm',1,'{test_table_perm_uuid}','{test_table_perm_metadata_path}')"
+    )

From bda41ab03d650978cb6765746d786b25de9ed72c Mon Sep 17 00:00:00 2001
From: Konstantin Morozov <just.morozov.k@gmail.com>
Date: Thu, 27 Jun 2024 10:52:04 +0000
Subject: [PATCH 0106/1488] fix tests for CI

---
 src/Databases/DatabasesCommon.cpp                    |  8 ++++----
 src/Databases/IDatabase.h                            |  4 +++-
 src/Databases/TablesLoader.cpp                       | 10 +++++-----
 .../0_stateless/03172_system_detached_tables.sql     | 12 ++++++------
 4 files changed, 18 insertions(+), 16 deletions(-)

diff --git a/src/Databases/DatabasesCommon.cpp b/src/Databases/DatabasesCommon.cpp
index 4cf914fea56..fe0baf30e57 100644
--- a/src/Databases/DatabasesCommon.cpp
+++ b/src/Databases/DatabasesCommon.cpp
@@ -2,10 +2,6 @@
 
 #include <Backups/BackupEntriesCollector.h>
 #include <Backups/RestorerFromBackup.h>
-#include <Common/CurrentMetrics.h>
-#include <Common/escapeForFileName.h>
-#include <Common/logger_useful.h>
-#include <Common/typeid_cast.h>
 #include <Interpreters/Context.h>
 #include <Interpreters/DatabaseCatalog.h>
 #include <Interpreters/InterpreterCreateQuery.h>
@@ -17,6 +13,10 @@
 #include <Storages/StorageFactory.h>
 #include <Storages/Utils.h>
 #include <TableFunctions/TableFunctionFactory.h>
+#include <Common/CurrentMetrics.h>
+#include <Common/escapeForFileName.h>
+#include <Common/logger_useful.h>
+#include <Common/typeid_cast.h>
 
 
 namespace DB
diff --git a/src/Databases/IDatabase.h b/src/Databases/IDatabase.h
index e856dd3d2f0..ce5a52b1b0f 100644
--- a/src/Databases/IDatabase.h
+++ b/src/Databases/IDatabase.h
@@ -138,7 +138,9 @@ protected:
     }
 
 public:
-    explicit DatabaseDetachedTablesSnapshotIterator(const SnapshotDetachedTables & tables_) : snapshot(tables_), it(snapshot.begin()) { }
+    explicit DatabaseDetachedTablesSnapshotIterator(const SnapshotDetachedTables & tables_) : snapshot(tables_), it(snapshot.begin())
+    {
+    }
 
     explicit DatabaseDetachedTablesSnapshotIterator(SnapshotDetachedTables && tables_) : snapshot(std::move(tables_)), it(snapshot.begin())
     {
diff --git a/src/Databases/TablesLoader.cpp b/src/Databases/TablesLoader.cpp
index 6aa13b7b759..1f70f46da60 100644
--- a/src/Databases/TablesLoader.cpp
+++ b/src/Databases/TablesLoader.cpp
@@ -1,14 +1,14 @@
-#include <Databases/TablesLoader.h>
-#include <Databases/IDatabase.h>
+#include <numeric>
 #include <Databases/DDLDependencyVisitor.h>
 #include <Databases/DDLLoadingDependencyVisitor.h>
-#include <Interpreters/DatabaseCatalog.h>
+#include <Databases/IDatabase.h>
+#include <Databases/TablesLoader.h>
 #include <Interpreters/Context.h>
+#include <Interpreters/DatabaseCatalog.h>
 #include <Interpreters/ExternalDictionariesLoader.h>
 #include <Poco/Util/AbstractConfiguration.h>
-#include <Common/logger_useful.h>
 #include <Common/CurrentMetrics.h>
-#include <numeric>
+#include <Common/logger_useful.h>
 
 
 namespace DB
diff --git a/tests/queries/0_stateless/03172_system_detached_tables.sql b/tests/queries/0_stateless/03172_system_detached_tables.sql
index 558dfe4cf18..e361d2f34ca 100644
--- a/tests/queries/0_stateless/03172_system_detached_tables.sql
+++ b/tests/queries/0_stateless/03172_system_detached_tables.sql
@@ -4,23 +4,23 @@ DROP DATABASE IF EXISTS test_system_detached_tables;
 CREATE DATABASE IF NOT EXISTS test_system_detached_tables;
 
 CREATE TABLE test_system_detached_tables.test_table (n Int64) ENGINE=MergeTree ORDER BY n;
-SELECT * FROM system.detached_tables;
+SELECT * FROM system.detached_tables WHERE database='test_system_detached_tables';
 
 DETACH TABLE test_system_detached_tables.test_table;
-SELECT database, table, is_permanently FROM system.detached_tables;
+SELECT database, table, is_permanently FROM system.detached_tables WHERE database='test_system_detached_tables';
 
 ATTACH TABLE test_system_detached_tables.test_table;
 
 CREATE TABLE test_system_detached_tables.test_table_perm (n Int64) ENGINE=MergeTree ORDER BY n;
-SELECT * FROM system.detached_tables;
+SELECT * FROM system.detached_tables WHERE database='test_system_detached_tables';
 
 DETACH TABLE test_system_detached_tables.test_table_perm PERMANENTLY;
-SELECT database, table, is_permanently FROM system.detached_tables;
+SELECT database, table, is_permanently FROM system.detached_tables WHERE database='test_system_detached_tables';
 
 DETACH TABLE test_system_detached_tables.test_table SYNC;
-SELECT database, table, is_permanently FROM system.detached_tables;
+SELECT database, table, is_permanently FROM system.detached_tables WHERE database='test_system_detached_tables';
 
-SELECT database, table, is_permanently FROM system.detached_tables WHERE table='test_table';
+SELECT database, table, is_permanently FROM system.detached_tables WHERE database='test_system_detached_tables' AND table='test_table';
 
 DROP DATABASE test_system_detached_tables;
 

From 3f192248e86749fd23ab3a0fa8da36ab4ba0543c Mon Sep 17 00:00:00 2001
From: JackyWoo <wuchienchao@qq.com>
Date: Thu, 27 Jun 2024 22:10:57 +0800
Subject: [PATCH 0107/1488] Some fixups from code review. Fix null data_type in
 ColumnStatisticsDescription. Update total_rows event if part has no
 statistics

---
 src/Storages/MergeTree/MergeTreeData.cpp      | 14 ++-
 .../ConditionSelectivityEstimator.h           |  1 +
 .../Statistics/CountMinSketchStatistics.cpp   | 38 +++-----
 .../Statistics/CountMinSketchStatistics.h     |  8 +-
 src/Storages/Statistics/Statistics.cpp        | 49 +++++++---
 src/Storages/Statistics/Statistics.h          |  3 +
 src/Storages/Statistics/TDigestStatistics.cpp | 14 ++-
 src/Storages/Statistics/UniqStatistics.cpp    |  2 +
 src/Storages/StatisticsDescription.cpp        |  7 +-
 ...64_statistics_estimate_predicate.reference | 25 +++++
 .../02864_statistics_estimate_predicate.sql   | 94 +++++++++++++++++++
 .../02864_statistics_uniq.reference           | 35 -------
 .../0_stateless/02864_statistics_uniq.sql     | 71 --------------
 .../03174_statistics_countminsketch.reference | 26 -----
 .../03174_statistics_countminsketch.sql       | 78 ---------------
 15 files changed, 198 insertions(+), 267 deletions(-)
 create mode 100644 tests/queries/0_stateless/02864_statistics_estimate_predicate.reference
 create mode 100644 tests/queries/0_stateless/02864_statistics_estimate_predicate.sql
 delete mode 100644 tests/queries/0_stateless/02864_statistics_uniq.reference
 delete mode 100644 tests/queries/0_stateless/02864_statistics_uniq.sql
 delete mode 100644 tests/queries/0_stateless/03174_statistics_countminsketch.reference
 delete mode 100644 tests/queries/0_stateless/03174_statistics_countminsketch.sql

diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index 2e0ea4cdbcd..364076bd802 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -497,8 +497,11 @@ ConditionSelectivityEstimator MergeTreeData::getConditionSelectivityEstimatorByP
         {
             auto stats = part->loadStatistics();
             /// TODO: We only have one stats file for every part.
-            for (const auto & stat : stats)
-                result.merge(part->info.getPartNameV1(), part->rows_count, stat);
+            if (stats.empty()) /// No statistics still need add rows count.
+                result.addRows(part->rows_count);
+            else
+                for (const auto & stat : stats)
+                    result.merge(part->info.getPartNameV1(), part->rows_count, stat);
         }
         catch (...)
         {
@@ -513,8 +516,11 @@ ConditionSelectivityEstimator MergeTreeData::getConditionSelectivityEstimatorByP
             if (!partition_pruner.canBePruned(*part))
             {
                 auto stats = part->loadStatistics();
-                for (const auto & stat : stats)
-                    result.merge(part->info.getPartNameV1(), part->rows_count, stat);
+                if (stats.empty())
+                    result.addRows(part->rows_count);
+                else
+                    for (const auto & stat : stats)
+                        result.merge(part->info.getPartNameV1(), part->rows_count, stat);
             }
         }
         catch (...)
diff --git a/src/Storages/Statistics/ConditionSelectivityEstimator.h b/src/Storages/Statistics/ConditionSelectivityEstimator.h
index 9bf4940e563..ff6218e7ef1 100644
--- a/src/Storages/Statistics/ConditionSelectivityEstimator.h
+++ b/src/Storages/Statistics/ConditionSelectivityEstimator.h
@@ -46,6 +46,7 @@ public:
     Float64 estimateRowCount(const RPNBuilderTreeNode & node) const;
 
     void merge(String part_name, UInt64 part_rows, ColumnStatisticsPtr column_stat);
+    void addRows(UInt64 part_rows) { total_rows += part_rows; }
 };
 
 }
diff --git a/src/Storages/Statistics/CountMinSketchStatistics.cpp b/src/Storages/Statistics/CountMinSketchStatistics.cpp
index c22aa966a15..497570bd2d1 100644
--- a/src/Storages/Statistics/CountMinSketchStatistics.cpp
+++ b/src/Storages/Statistics/CountMinSketchStatistics.cpp
@@ -1,6 +1,6 @@
 #include <Storages/Statistics/CountMinSketchStatistics.h>
-#include <DataTypes/DataTypeLowCardinality.h>
 #include <DataTypes/DataTypeNullable.h>
+#include <DataTypes/DataTypeLowCardinality.h>
 #include <IO/ReadHelpers.h>
 #include <IO/WriteHelpers.h>
 
@@ -17,7 +17,7 @@ extern const int ILLEGAL_STATISTICS;
 
 CountMinSketchStatistics::CountMinSketchStatistics(const SingleStatisticsDescription & stat_, DataTypePtr data_type_)
     : IStatistics(stat_)
-    , sketch(HASH_COUNT, BUCKET_COUNT)
+    , sketch(num_hashes, num_buckets)
     , data_type(data_type_)
 {
 }
@@ -31,20 +31,6 @@ Float64 CountMinSketchStatistics::estimateEqual(const Field & value) const
     UNREACHABLE();
 }
 
-bool CountMinSketchStatistics::checkType(const Field & f)
-{
-    switch (f.getType())
-    {
-        case Field::Types::Int64:
-        case Field::Types::UInt64:
-        case Field::Types::Float64:
-        case Field::Types::String:
-            return true;
-        default:
-            return false;
-    }
-}
-
 void CountMinSketchStatistics::serialize(WriteBuffer & buf)
 {
     Sketch::vector_bytes bytes = sketch.serialize();
@@ -61,24 +47,24 @@ void CountMinSketchStatistics::deserialize(ReadBuffer & buf)
     bytes.reserve(size);
     buf.readStrict(reinterpret_cast<char *>(bytes.data()), size);
 
-    sketch = datasketches::count_min_sketch<Float64>::deserialize(bytes.data(), size, datasketches::DEFAULT_SEED);
+    sketch = datasketches::count_min_sketch<UInt64>::deserialize(bytes.data(), size);
 }
 
 void CountMinSketchStatistics::update(const ColumnPtr & column)
 {
     size_t size = column->size();
-
     for (size_t i = 0; i < size; ++i)
     {
         Field f;
         column->get(i, f);
-        if (checkType(f))
-        {
-            if (auto float_val = IStatistics::getFloat64(f))
-                sketch.update(&float_val.value(), 8, 1.0);
-            else if (auto string_val = IStatistics::getString(f))
-                sketch.update(*string_val, 1.0);
-        }
+
+        if (f.isNull())
+            continue;
+
+        if (auto float_val = IStatistics::getFloat64(f))
+            sketch.update(&float_val, 8, 1);
+        else
+            sketch.update(f.get<String>(), 1);
     }
 }
 
@@ -87,7 +73,7 @@ void CountMinSketchValidator(const SingleStatisticsDescription &, DataTypePtr da
     data_type = removeNullable(data_type);
     data_type = removeLowCardinalityAndNullable(data_type);
     if (!data_type->isValueRepresentedByNumber() && !isStringOrFixedString(data_type))
-        throw Exception(ErrorCodes::ILLEGAL_STATISTICS, "Statistics of type 'countmin' does not support type {}", data_type->getName());
+        throw Exception(ErrorCodes::ILLEGAL_STATISTICS, "Statistics of type 'count_min' does not support type {}", data_type->getName());
 }
 
 StatisticsPtr CountMinSketchCreator(const SingleStatisticsDescription & stat, DataTypePtr data_type)
diff --git a/src/Storages/Statistics/CountMinSketchStatistics.h b/src/Storages/Statistics/CountMinSketchStatistics.h
index 2a4f65196a2..52046a19b64 100644
--- a/src/Storages/Statistics/CountMinSketchStatistics.h
+++ b/src/Storages/Statistics/CountMinSketchStatistics.h
@@ -23,13 +23,11 @@ public:
 
     void update(const ColumnPtr & column) override;
 
-    bool checkType(const Field & f);
-
 private:
-    static constexpr auto HASH_COUNT = 8uz;
-    static constexpr auto BUCKET_COUNT = 2048uz;
+    static constexpr auto num_hashes = 8uz;
+    static constexpr auto num_buckets = 2048uz;
 
-    using Sketch = datasketches::count_min_sketch<Float64>;
+    using Sketch = datasketches::count_min_sketch<UInt64>;
     Sketch sketch;
 
     DataTypePtr data_type;
diff --git a/src/Storages/Statistics/Statistics.cpp b/src/Storages/Statistics/Statistics.cpp
index 36339d7456f..9db0ccf1023 100644
--- a/src/Storages/Statistics/Statistics.cpp
+++ b/src/Storages/Statistics/Statistics.cpp
@@ -31,17 +31,39 @@ enum StatisticsFileVersion : UInt16
 
 std::optional<Float64> IStatistics::getFloat64(const Field & f)
 {
-    const auto type = f.getType();
-    Float64 value;
-    if (type == Field::Types::Int64)
-        value = f.get<Int64>();
-    else if (type == Field::Types::UInt64)
-        value = f.get<UInt64>();
-    else if (type == Field::Types::Float64)
-        value = f.get<Float64>();
-    else
-        return {};
-    return value;
+    switch (f.getType())
+    {
+        case Field::Types::Bool:
+            return f.get<bool>();
+        case Field::Types::Int64:
+            return f.get<Int64>();
+        case Field::Types::UInt64:
+            return f.get<UInt64>();
+        case Field::Types::Float64:
+            return f.get<Float64>();
+        case Field::Types::Int128:
+            return f.get<Int128>();
+        case Field::Types::UInt128:
+            return f.get<UInt128>();
+        case Field::Types::Int256:
+            return f.get<Int256>();
+        case Field::Types::UInt256:
+            return f.get<UInt256>();
+        case Field::Types::Decimal32:
+            return f.get<Decimal32>().getValue().value;
+        case Field::Types::Decimal64:
+            return f.get<Decimal64>().getValue().value;
+        case Field::Types::Decimal128:
+            return f.get<Decimal128>().getValue().value;
+        case Field::Types::Decimal256:
+            return f.get<Decimal256>().getValue().value;
+        case Field::Types::IPv4:
+            return f.get<IPv4>().toUnderType();
+        case Field::Types::IPv6:
+            return f.get<IPv6>().toUnderType();
+        default:
+            return {};
+    }
 }
 
 std::optional<String> IStatistics::getString(const Field & f)
@@ -97,10 +119,7 @@ Float64 ColumnStatistics::estimateEqual(Field val) const
     if (stats.contains(StatisticsType::CountMinSketch))
     {
         auto count_min_sketch_static = std::static_pointer_cast<CountMinSketchStatistics>(stats.at(StatisticsType::CountMinSketch));
-        if (!count_min_sketch_static->checkType(val))
-            return rows * ConditionSelectivityEstimator::default_normal_cond_factor;
-        else
-            return count_min_sketch_static->estimateEqual(val);
+        return count_min_sketch_static->estimateEqual(val);
     }
 #endif
     if (val < - ConditionSelectivityEstimator::threshold || val > ConditionSelectivityEstimator::threshold)
diff --git a/src/Storages/Statistics/Statistics.h b/src/Storages/Statistics/Statistics.h
index 9cc50a8ad4d..acea6d3435d 100644
--- a/src/Storages/Statistics/Statistics.h
+++ b/src/Storages/Statistics/Statistics.h
@@ -34,6 +34,9 @@ public:
 
     virtual void update(const ColumnPtr & column) = 0;
 
+    /// Convert filed to Float64, used when estimating the number of rows.
+    /// Return a Float64 value if f can be represented by number, otherwise return null.
+    /// See IDataType::isValueRepresentedByNumber
     static std::optional<Float64> getFloat64(const Field & f);
     static std::optional<String> getString(const Field & f);
 
diff --git a/src/Storages/Statistics/TDigestStatistics.cpp b/src/Storages/Statistics/TDigestStatistics.cpp
index d390619ea36..7156ca8e84c 100644
--- a/src/Storages/Statistics/TDigestStatistics.cpp
+++ b/src/Storages/Statistics/TDigestStatistics.cpp
@@ -1,5 +1,6 @@
 #include <Storages/Statistics/TDigestStatistics.h>
 #include <DataTypes/DataTypeNullable.h>
+#include <DataTypes/DataTypeLowCardinality.h>
 
 namespace DB
 {
@@ -36,18 +37,23 @@ void TDigestStatistics::deserialize(ReadBuffer & buf)
 void TDigestStatistics::update(const ColumnPtr & column)
 {
     size_t size = column->size();
-
     for (size_t i = 0; i < size; ++i)
     {
-        /// TODO: support more types.
-        Float64 value = column->getFloat64(i);
-        data.add(value, 1);
+        Field f;
+        column->get(i, f);
+
+        if (f.isNull())
+            continue;
+
+        if (auto float_val = IStatistics::getFloat64(f))
+            data.add(*float_val, 1);
     }
 }
 
 void TDigestValidator(const SingleStatisticsDescription &, DataTypePtr data_type)
 {
     data_type = removeNullable(data_type);
+    data_type = removeLowCardinalityAndNullable(data_type);
     if (!data_type->isValueRepresentedByNumber())
         throw Exception(ErrorCodes::ILLEGAL_STATISTICS, "Statistics of type 'tdigest' does not support type {}", data_type->getName());
 }
diff --git a/src/Storages/Statistics/UniqStatistics.cpp b/src/Storages/Statistics/UniqStatistics.cpp
index fc748e769ca..9e65a2fee15 100644
--- a/src/Storages/Statistics/UniqStatistics.cpp
+++ b/src/Storages/Statistics/UniqStatistics.cpp
@@ -1,6 +1,7 @@
 #include <Storages/Statistics/UniqStatistics.h>
 #include <DataTypes/DataTypesNumber.h>
 #include <DataTypes/DataTypeNullable.h>
+#include <DataTypes/DataTypeLowCardinality.h>
 
 namespace DB
 {
@@ -54,6 +55,7 @@ void UniqStatistics::update(const ColumnPtr & column)
 void UniqValidator(const SingleStatisticsDescription &, DataTypePtr data_type)
 {
     data_type = removeNullable(data_type);
+    data_type = removeLowCardinalityAndNullable(data_type);
     if (!data_type->isValueRepresentedByNumber())
         throw Exception(ErrorCodes::ILLEGAL_STATISTICS, "Statistics of type 'uniq' does not support type {}", data_type->getName());
 }
diff --git a/src/Storages/StatisticsDescription.cpp b/src/Storages/StatisticsDescription.cpp
index 8fa3400d3a6..d59ffb5c8e9 100644
--- a/src/Storages/StatisticsDescription.cpp
+++ b/src/Storages/StatisticsDescription.cpp
@@ -103,10 +103,9 @@ void ColumnStatisticsDescription::merge(const ColumnStatisticsDescription & othe
     chassert(merging_column_type);
 
     if (column_name.empty())
-    {
         column_name = merging_column_name;
-        data_type = merging_column_type;
-    }
+
+    data_type = merging_column_type;
 
     for (const auto & [stats_type, stats_desc]: other.types_to_desc)
     {
@@ -125,6 +124,7 @@ void ColumnStatisticsDescription::assign(const ColumnStatisticsDescription & oth
         throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot assign statistics from column {} to {}", column_name, other.column_name);
 
     types_to_desc = other.types_to_desc;
+    data_type = other.data_type;
 }
 
 void ColumnStatisticsDescription::clear()
@@ -163,6 +163,7 @@ std::vector<ColumnStatisticsDescription> ColumnStatisticsDescription::fromAST(co
 
         const auto & column = columns.getPhysical(physical_column_name);
         stats.column_name = column.name;
+        stats.data_type = column.type;
         stats.types_to_desc = statistics_types;
         result.push_back(stats);
     }
diff --git a/tests/queries/0_stateless/02864_statistics_estimate_predicate.reference b/tests/queries/0_stateless/02864_statistics_estimate_predicate.reference
new file mode 100644
index 00000000000..7215d5fef58
--- /dev/null
+++ b/tests/queries/0_stateless/02864_statistics_estimate_predicate.reference
@@ -0,0 +1,25 @@
+CREATE TABLE default.t1\n(\n    `a` String,\n    `b` UInt64,\n    `c` Int64,\n    `pk` String\n)\nENGINE = MergeTree\nORDER BY pk\nSETTINGS min_bytes_for_wide_part = 0, index_granularity = 8192
+Test statistics TDigest:
+        Prewhere info
+          Prewhere filter
+          Prewhere filter column: and(less(c, -98), greater(b, 0)) (removed)
+        Prewhere info
+          Prewhere filter
+          Prewhere filter column: and(less(c, -98), equals(b, 0)) (removed)
+Test statistics Uniq:
+        Prewhere info
+          Prewhere filter
+          Prewhere filter column: and(equals(b, 0), equals(c, 0)) (removed)
+Test statistics count_min:
+        Prewhere info
+          Prewhere filter
+          Prewhere filter column: and(equals(a, \'0\'), equals(b, 0), equals(c, 0)) (removed)
+Test statistics multi-types:
+        Prewhere info
+          Prewhere filter
+          Prewhere filter column: and(equals(a, \'0\'), less(c, -90), greater(b, 900_UInt16)) (removed)
+        Prewhere info
+          Prewhere filter
+          Prewhere filter column: and(equals(a, \'10000\'), equals(b, 0), less(c, 0)) (removed)
+Test LowCardinality and Nullable data type:
+t2
diff --git a/tests/queries/0_stateless/02864_statistics_estimate_predicate.sql b/tests/queries/0_stateless/02864_statistics_estimate_predicate.sql
new file mode 100644
index 00000000000..a608f18a354
--- /dev/null
+++ b/tests/queries/0_stateless/02864_statistics_estimate_predicate.sql
@@ -0,0 +1,94 @@
+-- Tags: no-fasttest
+DROP TABLE IF EXISTS t1 SYNC;
+
+SET allow_experimental_statistics = 1;
+SET allow_statistics_optimize = 1;
+SET allow_suspicious_low_cardinality_types=1;
+SET mutations_sync = 2;
+
+CREATE TABLE t1
+(
+    a String,
+    b UInt64,
+    c Int64,
+    pk String,
+) Engine = MergeTree() ORDER BY pk
+SETTINGS min_bytes_for_wide_part = 0;
+
+SHOW CREATE TABLE t1;
+
+INSERT INTO t1 select toString(number % 10000), number % 1000, -(number % 100), generateUUIDv4() FROM system.numbers LIMIT 10000;
+
+SELECT 'Test statistics TDigest:';
+
+ALTER TABLE t1 ADD STATISTICS b, c TYPE tdigest;
+ALTER TABLE t1 MATERIALIZE STATISTICS b, c;
+
+SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_String', '')
+FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE b > 0/*9990*/ and c < -98/*100*/)
+WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
+SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_String', '')
+FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE b = 0/*1000*/ and c < -98/*100*/)
+WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
+
+ALTER TABLE t1 DROP STATISTICS b, c;
+
+
+SELECT 'Test statistics Uniq:';
+
+ALTER TABLE t1 ADD STATISTICS b TYPE uniq, tdigest;
+ALTER TABLE t1 MATERIALIZE STATISTICS b;
+SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_String', '')
+FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE c = 0/*1000*/ and b = 0/*10*/)
+WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
+
+ALTER TABLE t1 DROP STATISTICS b;
+
+
+SELECT 'Test statistics count_min:';
+
+ALTER TABLE t1 ADD STATISTICS a TYPE count_min;
+ALTER TABLE t1 ADD STATISTICS b TYPE count_min;
+ALTER TABLE t1 ADD STATISTICS c TYPE count_min;
+ALTER TABLE t1 MATERIALIZE STATISTICS a, b, c;
+
+SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_String', '')
+FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE c = 0/*100*/ and b = 0/*10*/ and a = '0'/*1*/) xx
+WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
+
+ALTER TABLE t1 DROP STATISTICS a, b, c;
+
+
+SELECT 'Test statistics multi-types:';
+
+ALTER TABLE t1 ADD STATISTICS a TYPE count_min;
+ALTER TABLE t1 ADD STATISTICS b TYPE count_min, uniq, tdigest;
+ALTER TABLE t1 ADD STATISTICS c TYPE count_min, uniq, tdigest;
+ALTER TABLE t1 MATERIALIZE STATISTICS a, b, c;
+
+SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_String', '')
+FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE c < -90/*900*/ and b > 900/*990*/ and a = '0'/*1*/)
+WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
+SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_String', '')
+FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE c < 0/*9900*/ and b = 0/*10*/ and a = '10000'/*0*/)
+WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
+
+ALTER TABLE t1 DROP STATISTICS a, b, c;
+
+DROP TABLE IF EXISTS t1 SYNC;
+
+
+SELECT 'Test LowCardinality and Nullable data type:';
+DROP TABLE IF EXISTS t2 SYNC;
+SET allow_suspicious_low_cardinality_types=1;
+CREATE TABLE t2
+(
+    a LowCardinality(Int64) STATISTICS(uniq, tdigest, count_min),
+    b Nullable(Int64) STATISTICS(uniq, tdigest, count_min),
+    c LowCardinality(Nullable(Int64)) STATISTICS(uniq, tdigest, count_min),
+    pk String,
+) Engine = MergeTree() ORDER BY pk;
+
+select table from system.tables where name = 't2';
+
+DROP TABLE IF EXISTS t2 SYNC;
diff --git a/tests/queries/0_stateless/02864_statistics_uniq.reference b/tests/queries/0_stateless/02864_statistics_uniq.reference
deleted file mode 100644
index 77786dbdd8c..00000000000
--- a/tests/queries/0_stateless/02864_statistics_uniq.reference
+++ /dev/null
@@ -1,35 +0,0 @@
-CREATE TABLE default.t1\n(\n    `a` Float64 STATISTICS(tdigest),\n    `b` Int64 STATISTICS(tdigest),\n    `c` Int64 STATISTICS(tdigest, uniq),\n    `pk` String\n)\nENGINE = MergeTree\nORDER BY pk\nSETTINGS min_bytes_for_wide_part = 0, index_granularity = 8192
-After insert
-        Prewhere info
-          Prewhere filter
-          Prewhere filter column: and(less(a, 10), equals(c, 0), less(b, 10)) (removed)
-        Prewhere info
-          Prewhere filter
-          Prewhere filter column: and(equals(c, 11), less(a, 10), less(b, 10)) (removed)
-After merge
-        Prewhere info
-          Prewhere filter
-          Prewhere filter column: and(less(a, 10), equals(c, 0), less(b, 10)) (removed)
-        Prewhere info
-          Prewhere filter
-          Prewhere filter column: and(equals(c, 11), less(a, 10), less(b, 10)) (removed)
-After modify TDigest
-        Prewhere info
-          Prewhere filter
-          Prewhere filter column: and(less(a, 10), equals(c, 11), less(b, 10)) (removed)
-        Prewhere info
-          Prewhere filter
-          Prewhere filter column: and(less(a, 10), equals(c, 0), less(b, 10)) (removed)
-        Prewhere info
-          Prewhere filter
-          Prewhere filter column: and(less(c, -1), less(a, 10), less(b, 10)) (removed)
-After drop
-        Prewhere info
-          Prewhere filter
-          Prewhere filter column: and(less(a, 10), equals(c, 11), less(b, 10)) (removed)
-        Prewhere info
-          Prewhere filter
-          Prewhere filter column: and(less(a, 10), equals(c, 0), less(b, 10)) (removed)
-        Prewhere info
-          Prewhere filter
-          Prewhere filter column: and(less(a, 10), less(c, -1), less(b, 10)) (removed)
diff --git a/tests/queries/0_stateless/02864_statistics_uniq.sql b/tests/queries/0_stateless/02864_statistics_uniq.sql
deleted file mode 100644
index c6b51d2a377..00000000000
--- a/tests/queries/0_stateless/02864_statistics_uniq.sql
+++ /dev/null
@@ -1,71 +0,0 @@
-DROP TABLE IF EXISTS t1;
-
-SET allow_experimental_statistics = 1;
-SET allow_statistics_optimize = 1;
-
-CREATE TABLE t1
-(
-    a Float64 STATISTICS(tdigest),
-    b Int64 STATISTICS(tdigest),
-    c Int64 STATISTICS(tdigest, uniq),
-    pk String,
-) Engine = MergeTree() ORDER BY pk
-SETTINGS min_bytes_for_wide_part = 0;
-
-SHOW CREATE TABLE t1;
-
-INSERT INTO t1 select number, -number, number/1000, generateUUIDv4() FROM system.numbers LIMIT 10000;
-INSERT INTO t1 select 0, 0, 11, generateUUIDv4();
-
-SELECT 'After insert';
-SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE b < 10 and c = 0 and a < 10) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
-SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE b < 10 and c = 11 and a < 10) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
-OPTIMIZE TABLE t1 FINAL;
-
-SELECT 'After merge';
-SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE b < 10 and c = 0 and a < 10) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
-SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE b < 10 and c = 11 and a < 10) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
-
-SELECT 'After modify TDigest';
-ALTER TABLE t1 MODIFY STATISTICS c TYPE TDigest;
-ALTER TABLE t1 MATERIALIZE STATISTICS c;
-
-SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE b < 10 and c = 11 and a < 10) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
-SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE b < 10 and c = 0 and a < 10) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
-SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE b < 10 and c < -1 and a < 10) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
-
-
-ALTER TABLE t1 DROP STATISTICS c;
-
-SELECT 'After drop';
-SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE b < 10 and c = 11 and a < 10) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
-SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE b < 10 and c = 0 and a < 10) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
-SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE b < 10 and c < -1 and a < 10) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
-
-DROP TABLE IF EXISTS t1;
-DROP TABLE IF EXISTS t2;
-SET allow_suspicious_low_cardinality_types=1;
-CREATE TABLE t2
-(
-    a Float64 STATISTICS(tdigest),
-    b Int64 STATISTICS(tdigest),
-    c LowCardinality(Int64) STATISTICS(tdigest, uniq),
-    pk String,
-) Engine = MergeTree() ORDER BY pk
-SETTINGS min_bytes_for_wide_part = 0;
-INSERT INTO t2 select number, -number, number/1000, generateUUIDv4() FROM system.numbers LIMIT 10000;
-
-DROP TABLE IF EXISTS t2;
-DROP TABLE IF EXISTS t3;
-
-CREATE TABLE t3
-(
-    a Float64 STATISTICS(tdigest),
-    b Int64 STATISTICS(tdigest),
-    c Nullable(Int64) STATISTICS(tdigest, uniq),
-    pk String,
-) Engine = MergeTree() ORDER BY pk
-SETTINGS min_bytes_for_wide_part = 0;
-INSERT INTO t3 select number, -number, number/1000, generateUUIDv4() FROM system.numbers LIMIT 10000;
-
-DROP TABLE IF EXISTS t3;
diff --git a/tests/queries/0_stateless/03174_statistics_countminsketch.reference b/tests/queries/0_stateless/03174_statistics_countminsketch.reference
deleted file mode 100644
index b5a35c22835..00000000000
--- a/tests/queries/0_stateless/03174_statistics_countminsketch.reference
+++ /dev/null
@@ -1,26 +0,0 @@
-CREATE TABLE default.t1\n(\n    `a` String STATISTICS(countminsketch),\n    `b` Int64 STATISTICS(countminsketch),\n    `c` UInt64 STATISTICS(countminsketch),\n    `pk` String\n)\nENGINE = MergeTree\nORDER BY pk\nSETTINGS min_bytes_for_wide_part = 0, index_granularity = 8192
-        Prewhere info
-          Prewhere filter
-          Prewhere filter column: and(equals(a, \'0\'), equals(b, 0), equals(c, 0)) (removed)
-        Prewhere info
-          Prewhere filter
-          Prewhere filter column: and(equals(a, \'0\'), equals(c, 0), greater(b, 0)) (removed)
-After drop statistics for a
-        Prewhere info
-          Prewhere filter
-          Prewhere filter column: and(equals(b, 0), equals(c, 0), equals(a, \'0\')) (removed)
-        Prewhere info
-          Prewhere filter
-          Prewhere filter column: and(equals(c, 0), equals(a, \'0\'), greater(b, 0)) (removed)
-LowCardinality
-        Prewhere info
-          Prewhere filter
-          Prewhere filter column: and(equals(a, \'0\'), equals(b, 0), equals(c, 0)) (removed)
-Nullable
-        Prewhere info
-          Prewhere filter
-          Prewhere filter column: and(equals(a, \'0\'), equals(b, 0), equals(c, 0)) (removed)
-LowCardinality(Nullable)
-        Prewhere info
-          Prewhere filter
-          Prewhere filter column: and(equals(a, \'0\'), equals(b, 0), equals(c, 0)) (removed)
diff --git a/tests/queries/0_stateless/03174_statistics_countminsketch.sql b/tests/queries/0_stateless/03174_statistics_countminsketch.sql
deleted file mode 100644
index 200c2a4a531..00000000000
--- a/tests/queries/0_stateless/03174_statistics_countminsketch.sql
+++ /dev/null
@@ -1,78 +0,0 @@
--- Tags: no-fasttest
-DROP TABLE IF EXISTS t1;
-
-SET allow_experimental_statistics = 1;
-SET allow_statistics_optimize = 1;
-
-CREATE TABLE t1
-(
-    a String STATISTICS(countminsketch),
-    b Int64 STATISTICS(countminsketch),
-    c UInt64 STATISTICS(countminsketch),
-    pk String,
-) Engine = MergeTree() ORDER BY pk
-SETTINGS min_bytes_for_wide_part = 0;
-
-SHOW CREATE TABLE t1;
-
-INSERT INTO t1 select toString(number % 1000), number % 100, number % 10, generateUUIDv4() FROM system.numbers LIMIT 10000;
-
-SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_String', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE c = 0 and b = 0 and a = '0') WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
-SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_String', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE c = 0 and b > 0 and a = '0') WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
-
-ALTER TABLE t1 DROP STATISTICS a;
-
-SELECT 'After drop statistics for a';
-SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_String', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE c = 0 and b = 0 and a = '0') WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
-SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_String', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE c = 0 and b > 0 and a = '0') WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
-
-DROP TABLE IF EXISTS t1;
-DROP TABLE IF EXISTS t2;
-SET allow_suspicious_low_cardinality_types=1;
-CREATE TABLE t2
-(
-    a LowCardinality(String) STATISTICS(countminsketch),
-    b Int64 STATISTICS(countminsketch),
-    c UInt64 STATISTICS(countminsketch),
-    pk String,
-) Engine = MergeTree() ORDER BY pk
-SETTINGS min_bytes_for_wide_part = 0;
-INSERT INTO t2 select toString(number % 1000), number % 100, number % 10, generateUUIDv4() FROM system.numbers LIMIT 10000;
-
-SELECT 'LowCardinality';
-SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_String', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t2 WHERE c = 0 and b = 0 and a = '0') WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
-
-
-DROP TABLE IF EXISTS t2;
-DROP TABLE IF EXISTS t3;
-
-CREATE TABLE t3
-(
-    a Nullable(String) STATISTICS(countminsketch),
-    b Int64 STATISTICS(countminsketch),
-    c UInt64 STATISTICS(countminsketch),
-    pk String,
-) Engine = MergeTree() ORDER BY pk
-SETTINGS min_bytes_for_wide_part = 0;
-INSERT INTO t3 select toString(number % 1000), number % 100, number % 10, generateUUIDv4() FROM system.numbers LIMIT 10000;
-
-SELECT 'Nullable';
-SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_String', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t3 WHERE c = 0 and b = 0 and a = '0') WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
-
-DROP TABLE IF EXISTS t3;
-DROP TABLE IF EXISTS t4;
-
-CREATE TABLE t4
-(
-    a LowCardinality(Nullable(String)) STATISTICS(countminsketch),
-    b Int64 STATISTICS(countminsketch),
-    c UInt64 STATISTICS(countminsketch),
-    pk String,
-) Engine = MergeTree() ORDER BY pk
-SETTINGS min_bytes_for_wide_part = 0;
-INSERT INTO t4 select toString(number % 1000), number % 100, number % 10, generateUUIDv4() FROM system.numbers LIMIT 10000;
-
-SELECT 'LowCardinality(Nullable)';
-SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_String', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t4 WHERE c = 0 and b = 0 and a = '0') WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
-
-DROP TABLE IF EXISTS t4;

From 6d1f37ad3db0078514dd76e857ef10474afd2ed4 Mon Sep 17 00:00:00 2001
From: Konstantin Morozov <just.morozov.k@gmail.com>
Date: Thu, 27 Jun 2024 15:37:27 +0000
Subject: [PATCH 0108/1488] support lazy db

---
 src/Databases/DatabaseLazy.cpp                | 16 ++++++
 .../03172_system_detached_tables.reference    | 16 ++++--
 .../03172_system_detached_tables.sql          | 55 ++++++++++++++-----
 3 files changed, 67 insertions(+), 20 deletions(-)

diff --git a/src/Databases/DatabaseLazy.cpp b/src/Databases/DatabaseLazy.cpp
index 233db07cd68..3c398e50371 100644
--- a/src/Databases/DatabaseLazy.cpp
+++ b/src/Databases/DatabaseLazy.cpp
@@ -187,6 +187,13 @@ void DatabaseLazy::attachTable(ContextPtr /* context_ */, const String & table_n
 
     it->second.expiration_iterator = cache_expiration_queue.emplace(cache_expiration_queue.end(), current_time, table_name);
 
+    LOG_DEBUG(log, "Add info for detached table {} to snapshot.", backQuote(table_name));
+    if (snapshot_detached_tables.contains(table_name))
+    {
+        LOG_DEBUG(log, "Clean info about detached table {} from snapshot.", backQuote(table_name));
+        snapshot_detached_tables.erase(table_name);
+    }
+
     CurrentMetrics::add(CurrentMetrics::AttachedTable, 1);
 }
 
@@ -203,6 +210,15 @@ StoragePtr DatabaseLazy::detachTable(ContextPtr /* context */, const String & ta
         if (it->second.expiration_iterator != cache_expiration_queue.end())
             cache_expiration_queue.erase(it->second.expiration_iterator);
         tables_cache.erase(it);
+        LOG_DEBUG(log, "Add info for detached table {} to snapshot.", backQuote(table_name));
+        snapshot_detached_tables.emplace(
+            table_name,
+            SnapshotDetachedTable{
+                .database = res->getStorageID().database_name,
+                .table = res->getStorageID().table_name,
+                .uuid = res->getStorageID().uuid,
+                .metadata_path = getObjectMetadataPath(table_name),
+                .is_permanently = false});
 
         CurrentMetrics::sub(CurrentMetrics::AttachedTable, 1);
     }
diff --git a/tests/queries/0_stateless/03172_system_detached_tables.reference b/tests/queries/0_stateless/03172_system_detached_tables.reference
index 1cf9e0275c9..ce19daa4cdc 100644
--- a/tests/queries/0_stateless/03172_system_detached_tables.reference
+++ b/tests/queries/0_stateless/03172_system_detached_tables.reference
@@ -1,5 +1,11 @@
-test_system_detached_tables	test_table	0
-test_system_detached_tables	test_table_perm	1
-test_system_detached_tables	test_table	0
-test_system_detached_tables	test_table_perm	1
-test_system_detached_tables	test_table	0
+database atomic tests
+test03172_system_detached_tables	test_table	0
+test03172_system_detached_tables	test_table_perm	1
+test03172_system_detached_tables	test_table	0
+test03172_system_detached_tables	test_table_perm	1
+test03172_system_detached_tables	test_table	0
+-----------------------
+database lazy tests
+test03172_system_detached_tables_lazy	test_table	0
+test03172_system_detached_tables_lazy	test_table_perm	1
+DROP TABLE
diff --git a/tests/queries/0_stateless/03172_system_detached_tables.sql b/tests/queries/0_stateless/03172_system_detached_tables.sql
index e361d2f34ca..916821d996b 100644
--- a/tests/queries/0_stateless/03172_system_detached_tables.sql
+++ b/tests/queries/0_stateless/03172_system_detached_tables.sql
@@ -1,28 +1,53 @@
 -- Tags: no-parallel
 
-DROP DATABASE IF EXISTS test_system_detached_tables;
-CREATE DATABASE IF NOT EXISTS test_system_detached_tables;
+SELECT 'database atomic tests';
+DROP DATABASE IF EXISTS test03172_system_detached_tables;
+CREATE DATABASE IF NOT EXISTS test03172_system_detached_tables;
 
-CREATE TABLE test_system_detached_tables.test_table (n Int64) ENGINE=MergeTree ORDER BY n;
-SELECT * FROM system.detached_tables WHERE database='test_system_detached_tables';
+CREATE TABLE test03172_system_detached_tables.test_table (n Int64) ENGINE=MergeTree ORDER BY n;
+SELECT * FROM system.detached_tables WHERE database='test03172_system_detached_tables';
 
-DETACH TABLE test_system_detached_tables.test_table;
-SELECT database, table, is_permanently FROM system.detached_tables WHERE database='test_system_detached_tables';
+DETACH TABLE test03172_system_detached_tables.test_table;
+SELECT database, table, is_permanently FROM system.detached_tables WHERE database='test03172_system_detached_tables';
 
-ATTACH TABLE test_system_detached_tables.test_table;
+ATTACH TABLE test03172_system_detached_tables.test_table;
 
-CREATE TABLE test_system_detached_tables.test_table_perm (n Int64) ENGINE=MergeTree ORDER BY n;
-SELECT * FROM system.detached_tables WHERE database='test_system_detached_tables';
+CREATE TABLE test03172_system_detached_tables.test_table_perm (n Int64) ENGINE=MergeTree ORDER BY n;
+SELECT * FROM system.detached_tables WHERE database='test03172_system_detached_tables';
 
-DETACH TABLE test_system_detached_tables.test_table_perm PERMANENTLY;
-SELECT database, table, is_permanently FROM system.detached_tables WHERE database='test_system_detached_tables';
+DETACH TABLE test03172_system_detached_tables.test_table_perm PERMANENTLY;
+SELECT database, table, is_permanently FROM system.detached_tables WHERE database='test03172_system_detached_tables';
 
-DETACH TABLE test_system_detached_tables.test_table SYNC;
-SELECT database, table, is_permanently FROM system.detached_tables WHERE database='test_system_detached_tables';
+DETACH TABLE test03172_system_detached_tables.test_table SYNC;
+SELECT database, table, is_permanently FROM system.detached_tables WHERE database='test03172_system_detached_tables';
 
-SELECT database, table, is_permanently FROM system.detached_tables WHERE database='test_system_detached_tables' AND table='test_table';
+SELECT database, table, is_permanently FROM system.detached_tables WHERE database='test03172_system_detached_tables' AND table='test_table';
 
-DROP DATABASE test_system_detached_tables;
+DROP DATABASE test03172_system_detached_tables SYNC;
 
+SELECT '-----------------------';
+SELECT 'database lazy tests';
 
+DROP DATABASE IF EXISTS test03172_system_detached_tables_lazy;
+CREATE DATABASE test03172_system_detached_tables_lazy Engine=Lazy(10);
 
+CREATE TABLE test03172_system_detached_tables_lazy.test_table (number UInt64) engine=Log;
+INSERT INTO test03172_system_detached_tables_lazy.test_table SELECT * FROM numbers(100);
+DETACH TABLE test03172_system_detached_tables_lazy.test_table;
+
+CREATE TABLE test03172_system_detached_tables_lazy.test_table_perm (number UInt64) engine=Log;
+INSERT INTO test03172_system_detached_tables_lazy.test_table_perm SELECT * FROM numbers(100);
+DETACH table test03172_system_detached_tables_lazy.test_table_perm PERMANENTLY;
+
+SELECT database, table, is_permanently FROM system.detached_tables WHERE database='test03172_system_detached_tables_lazy';
+
+ATTACH TABLE test03172_system_detached_tables_lazy.test_table;
+ATTACH TABLE test03172_system_detached_tables_lazy.test_table_perm;
+
+SELECT database, table, is_permanently FROM system.detached_tables WHERE database='test03172_system_detached_tables_lazy';
+
+SELECT 'DROP TABLE';
+DROP TABLE  test03172_system_detached_tables_lazy.test_table SYNC;
+DROP TABLE  test03172_system_detached_tables_lazy.test_table_perm SYNC;
+
+DROP DATABASE test03172_system_detached_tables_lazy SYNC;

From 062198b95642a52ec0c67bbc25e7d5da45814b33 Mon Sep 17 00:00:00 2001
From: Konstantin Morozov <just.morozov.k@gmail.com>
Date: Thu, 27 Jun 2024 15:44:37 +0000
Subject: [PATCH 0109/1488] add checking and exception

---
 src/Databases/DatabaseOnDisk.cpp | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp
index 25e5347e017..67c9114e793 100644
--- a/src/Databases/DatabaseOnDisk.cpp
+++ b/src/Databases/DatabaseOnDisk.cpp
@@ -22,6 +22,7 @@
 #include <Storages/IStorage.h>
 #include <Storages/StorageFactory.h>
 #include <TableFunctions/TableFunctionFactory.h>
+#include "Common/Exception.h"
 #include <Common/CurrentMetrics.h>
 #include <Common/assert_cast.h>
 #include <Common/escapeForFileName.h>
@@ -309,7 +310,11 @@ void DatabaseOnDisk::detachTablePermanently(ContextPtr query_context, const Stri
         FS::createFile(detached_permanently_flag);
 
         std::lock_guard lock(mutex);
-        snapshot_detached_tables.at(table_name).is_permanently = true;
+        if (!snapshot_detached_tables.contains(table_name))
+        {
+            throw Exception(ErrorCodes::LOGICAL_ERROR, "Snapshot doesn't contain info about detached table={}", table_name);
+        }
+        snapshot_detached_tables[table_name].is_permanently = true;
     }
     catch (Exception & e)
     {

From 1c9df94a0a0b6fc9eb7ef302b47ac909c029e345 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Thu, 27 Jun 2024 21:26:39 +0200
Subject: [PATCH 0110/1488] Replace fix with LOGICAL_ERROR

---
 src/Functions/IFunction.cpp | 24 ++++++++----------------
 1 file changed, 8 insertions(+), 16 deletions(-)

diff --git a/src/Functions/IFunction.cpp b/src/Functions/IFunction.cpp
index 9217071ca11..76ae8f33fbd 100644
--- a/src/Functions/IFunction.cpp
+++ b/src/Functions/IFunction.cpp
@@ -47,6 +47,9 @@ bool allArgumentsAreConstants(const ColumnsWithTypeAndName & args)
     return true;
 }
 
+/// Replaces single low cardinality column in a function call by its dictionary
+/// This can only happen after the arguments have been adapted in IFunctionOverloadResolver::getReturnType
+/// as it's only possible if there is one low cardinality column and, optionally, const columns
 ColumnPtr replaceLowCardinalityColumnsByNestedAndGetDictionaryIndexes(
     ColumnsWithTypeAndName & args, bool can_be_executed_on_default_arguments, size_t input_rows_count)
 {
@@ -77,21 +80,8 @@ ColumnPtr replaceLowCardinalityColumnsByNestedAndGetDictionaryIndexes(
 
     if (number_full_columns > 0 || number_low_cardinality_columns > 1)
     {
-        /// If there is a single full column, we can't replace the LC column with its dictionary, as it won't match
-        /// the size or order of the full columns. Same if there are 2 or more low cardinality columns
-        for (auto & arg : args)
-        {
-            if (const auto * column_lc = checkAndGetColumn<ColumnLowCardinality>(arg.column.get()))
-            {
-                arg.column = recursiveRemoveLowCardinality(arg.column);
-                chassert(arg.column->size() == input_rows_count);
-
-                const auto * low_cardinality_type = checkAndGetDataType<DataTypeLowCardinality>(arg.type.get());
-                if (!low_cardinality_type)
-                    throw Exception(ErrorCodes::LOGICAL_ERROR, "Incompatible type for LowCardinality column: {}", arg.type->getName());
-                arg.type = recursiveRemoveLowCardinality(arg.type);
-            }
-        }
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected low cardinality types found. Low cardinality: {}. Full {}. Const {}",
+                number_low_cardinality_columns, number_full_columns, number_const_columns);
     }
     else if (number_low_cardinality_columns == 1)
     {
@@ -124,7 +114,7 @@ ColumnPtr replaceLowCardinalityColumnsByNestedAndGetDictionaryIndexes(
         lc_arg.type = low_cardinality_type->getDictionaryType();
     }
 
-    /// Change size of constants.
+    /// Change size of constants
     for (auto & column : args)
     {
         if (const auto * column_const = checkAndGetColumn<ColumnConst>(column.column.get()))
@@ -305,6 +295,8 @@ ColumnPtr IExecutableFunction::executeWithoutSparseColumns(const ColumnsWithType
             bool can_be_executed_on_default_arguments = canBeExecutedOnDefaultArguments();
 
             const auto & dictionary_type = res_low_cardinality_type->getDictionaryType();
+            /// The arguments should have been adapted in IFunctionOverloadResolver::getReturnType
+            /// So there is only one low cardinality column (and optionally some const columns) and no full column
             ColumnPtr indexes = replaceLowCardinalityColumnsByNestedAndGetDictionaryIndexes(
                     columns_without_low_cardinality, can_be_executed_on_default_arguments, input_rows_count);
 

From c185d60375dd0fbc18d062197875ba463326f44a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Thu, 27 Jun 2024 21:59:14 +0200
Subject: [PATCH 0111/1488] Try an ugly fix

---
 src/Planner/PlannerActionsVisitor.cpp | 27 +++++++++++++++++++--------
 1 file changed, 19 insertions(+), 8 deletions(-)

diff --git a/src/Planner/PlannerActionsVisitor.cpp b/src/Planner/PlannerActionsVisitor.cpp
index 7a12d5d690d..13e96dc7016 100644
--- a/src/Planner/PlannerActionsVisitor.cpp
+++ b/src/Planner/PlannerActionsVisitor.cpp
@@ -485,16 +485,28 @@ public:
         return node;
     }
 
-    const ActionsDAG::Node * addConstantIfNecessary(const std::string & node_name, const ColumnWithTypeAndName & column)
+    [[nodiscard]] String addConstantIfNecessary(const std::string & node_name, const ColumnWithTypeAndName & column)
     {
         auto it = node_name_to_node.find(node_name);
+        if (it != node_name_to_node.end() && it->second->column)
+            return {node_name};
+
         if (it != node_name_to_node.end())
-            return it->second;
+        {
+            /// There is a node with this name, but it doesn't have a column
+            /// This likely happens because we executed the query until WithMergeableState with a const node in the
+            /// WHERE clause. As the results of headers are materialized, the column was removed
+            /// Let's add a new column and keep this
+            String dupped_name{node_name + "_dupped"};
+            const auto * node = &actions_dag.addColumn(column);
+            node_name_to_node[dupped_name] = node;
+            return dupped_name;
+        }
 
         const auto * node = &actions_dag.addColumn(column);
         node_name_to_node[node->result_name] = node;
 
-        return node;
+        return {node_name};
     }
 
     template <typename FunctionOrOverloadResolver>
@@ -723,7 +735,7 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::vi
     column.type = constant_type;
     column.column = column.type->createColumnConst(1, constant_literal);
 
-    actions_stack[0].addConstantIfNecessary(constant_node_name, column);
+    String final_name = actions_stack[0].addConstantIfNecessary(constant_node_name, column);
 
     size_t actions_stack_size = actions_stack.size();
     for (size_t i = 1; i < actions_stack_size; ++i)
@@ -732,8 +744,7 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::vi
         actions_stack_node.addInputConstantColumnIfNecessary(constant_node_name, column);
     }
 
-    return {constant_node_name, Levels(0)};
-
+    return {final_name, Levels(0)};
 }
 
 PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::visitLambda(const QueryTreeNodePtr & node)
@@ -862,7 +873,7 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::ma
     else
         column.column = std::move(column_set);
 
-    actions_stack[0].addConstantIfNecessary(column.name, column);
+    String final_name = actions_stack[0].addConstantIfNecessary(column.name, column);
 
     size_t actions_stack_size = actions_stack.size();
     for (size_t i = 1; i < actions_stack_size; ++i)
@@ -871,7 +882,7 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::ma
         actions_stack_node.addInputConstantColumnIfNecessary(column.name, column);
     }
 
-    return {column.name, Levels(0)};
+    return {final_name, Levels(0)};
 }
 
 PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::visitIndexHintFunction(const QueryTreeNodePtr & node)

From 818e8597a22cf87c560d7986818f8a094cd7db1f Mon Sep 17 00:00:00 2001
From: Konstantin Morozov <just.morozov.k@gmail.com>
Date: Fri, 28 Jun 2024 12:27:40 +0000
Subject: [PATCH 0112/1488] Revert "general logic for tables"

This reverts commit de8f1fc6e3adb6c8f346b2fda4e34c2240e1bf8e.
---
 src/Storages/System/ReadFromSystemTables.cpp  | 157 ----
 src/Storages/System/ReadFromSystemTables.h    |  66 --
 .../System/StorageSystemDetachedTables.cpp    | 237 ++++++-
 src/Storages/System/StorageSystemTables.cpp   | 669 +++++++++++++++++-
 src/Storages/System/TablesBlockSource.cpp     | 564 ---------------
 src/Storages/System/TablesBlockSource.h       |  92 ---
 6 files changed, 891 insertions(+), 894 deletions(-)
 delete mode 100644 src/Storages/System/ReadFromSystemTables.cpp
 delete mode 100644 src/Storages/System/ReadFromSystemTables.h
 delete mode 100644 src/Storages/System/TablesBlockSource.cpp
 delete mode 100644 src/Storages/System/TablesBlockSource.h

diff --git a/src/Storages/System/ReadFromSystemTables.cpp b/src/Storages/System/ReadFromSystemTables.cpp
deleted file mode 100644
index 638cab1b63b..00000000000
--- a/src/Storages/System/ReadFromSystemTables.cpp
+++ /dev/null
@@ -1,157 +0,0 @@
-#include "ReadFromSystemTables.h"
-
-#include <Core/Block.h>
-#include <Core/ColumnWithTypeAndName.h>
-#include <DataTypes/DataTypeString.h>
-#include <Interpreters/DatabaseCatalog.h>
-#include <QueryPipeline/QueryPipelineBuilder.h>
-#include <Storages/IStorage.h>
-#include <Storages/System/TablesBlockSource.h>
-#include <Storages/VirtualColumnUtils.h>
-
-#include <boost/range/adaptor/map.hpp>
-
-namespace DB
-{
-
-namespace
-{
-
-ColumnPtr getFilteredDatabases(const ActionsDAG::Node * predicate, ContextPtr context)
-{
-    MutableColumnPtr column = ColumnString::create();
-
-    const auto databases = DatabaseCatalog::instance().getDatabases();
-    for (const auto & database_name : databases | boost::adaptors::map_keys)
-    {
-        if (database_name == DatabaseCatalog::TEMPORARY_DATABASE)
-            continue; /// We don't want to show the internal database for temporary tables in system.tables
-
-        column->insert(database_name);
-    }
-
-    Block block{ColumnWithTypeAndName(std::move(column), std::make_shared<DataTypeString>(), "database")};
-    VirtualColumnUtils::filterBlockWithPredicate(predicate, block, context);
-    return block.getByPosition(0).column;
-}
-
-ColumnPtr getFilteredTables(
-    const ActionsDAG::Node * predicate, const ColumnPtr & filtered_databases_column, ContextPtr context, const bool need_detached_tables)
-{
-    Block sample{
-        ColumnWithTypeAndName(nullptr, std::make_shared<DataTypeString>(), "name"),
-        ColumnWithTypeAndName(nullptr, std::make_shared<DataTypeString>(), "engine")};
-
-    MutableColumnPtr database_column = ColumnString::create();
-    MutableColumnPtr engine_column;
-
-    auto dag = VirtualColumnUtils::splitFilterDagForAllowedInputs(predicate, &sample);
-    if (dag)
-    {
-        bool filter_by_engine = false;
-        for (const auto * input : dag->getInputs())
-            if (input->result_name == "engine")
-                filter_by_engine = true;
-
-        if (filter_by_engine)
-            engine_column = ColumnString::create();
-    }
-
-    for (size_t database_idx = 0; database_idx < filtered_databases_column->size(); ++database_idx)
-    {
-        const auto & database_name = filtered_databases_column->getDataAt(database_idx).toString();
-        DatabasePtr database = DatabaseCatalog::instance().tryGetDatabase(database_name);
-        if (!database)
-            continue;
-
-        if (need_detached_tables)
-        {
-            auto table_it = database->getDetachedTablesIterator(context, {}, false);
-            for (; table_it->isValid(); table_it->next())
-            {
-                database_column->insert(table_it->table());
-            }
-        }
-        else
-        {
-            auto table_it = database->getTablesIterator(context);
-            for (; table_it->isValid(); table_it->next())
-            {
-                database_column->insert(table_it->name());
-                if (engine_column)
-                    engine_column->insert(table_it->table()->getName());
-            }
-        }
-    }
-
-    Block block{ColumnWithTypeAndName(std::move(database_column), std::make_shared<DataTypeString>(), "name")};
-    if (engine_column)
-        block.insert(ColumnWithTypeAndName(std::move(engine_column), std::make_shared<DataTypeString>(), "engine"));
-
-    if (dag)
-        VirtualColumnUtils::filterBlockWithDAG(dag, block, context);
-
-    return block.getByPosition(0).column;
-}
-
-}
-
-ReadFromSystemTables::ReadFromSystemTables(
-    const Names & column_names_,
-    const SelectQueryInfo & query_info_,
-    const StorageSnapshotPtr & storage_snapshot_,
-    const ContextPtr & context_,
-    Block sample_block,
-    std::vector<UInt8> columns_mask_,
-    size_t max_block_size_)
-    : SourceStepWithFilter(DataStream{.header = std::move(sample_block)}, column_names_, query_info_, storage_snapshot_, context_)
-    , columns_mask(std::move(columns_mask_))
-    , max_block_size(max_block_size_)
-{
-}
-
-void ReadFromSystemTables::applyFilters(ActionDAGNodes added_filter_nodes)
-{
-    applyFiltersImpl(std::move(added_filter_nodes), false);
-}
-
-void ReadFromSystemTables::applyFiltersImpl(ActionDAGNodes added_filter_nodes, const bool need_detached_tables_)
-{
-    SourceStepWithFilter::applyFilters(std::move(added_filter_nodes));
-
-    const ActionsDAG::Node * predicate = nullptr;
-    if (filter_actions_dag)
-        predicate = filter_actions_dag->getOutputs().at(0);
-
-    filtered_databases_column = getFilteredDatabases(predicate, context);
-    filtered_tables_column = getFilteredTables(predicate, filtered_databases_column, context, need_detached_tables_);
-}
-
-void ReadFromSystemTables::initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &)
-{
-    pipeline.init(createPipe<TablesBlockSource>());
-}
-
-ReadFromSystemDetachedTables::ReadFromSystemDetachedTables(
-    const Names & column_names_,
-    const SelectQueryInfo & query_info_,
-    const StorageSnapshotPtr & storage_snapshot_,
-    const ContextPtr & context_,
-    Block sample_block,
-    std::vector<UInt8> columns_mask_,
-    const size_t max_block_size_)
-    : ReadFromSystemTables(
-        column_names_, query_info_, storage_snapshot_, context_, std::move(sample_block), std::move(columns_mask_), max_block_size_)
-{
-}
-
-void ReadFromSystemDetachedTables::applyFilters(ActionDAGNodes added_filter_nodes)
-{
-    applyFiltersImpl(std::move(added_filter_nodes), true);
-}
-
-void ReadFromSystemDetachedTables::initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &)
-{
-    pipeline.init(createPipe<DetachedTablesBlockSource>());
-}
-}
diff --git a/src/Storages/System/ReadFromSystemTables.h b/src/Storages/System/ReadFromSystemTables.h
deleted file mode 100644
index f8f34363159..00000000000
--- a/src/Storages/System/ReadFromSystemTables.h
+++ /dev/null
@@ -1,66 +0,0 @@
-#pragma once
-
-#include <Processors/QueryPlan/SourceStepWithFilter.h>
-#include <QueryPipeline/Pipe.h>
-
-namespace DB
-{
-
-class ReadFromSystemTables : public SourceStepWithFilter
-{
-public:
-    std::string getName() const override { return "ReadFromSystemTables"; }
-    void initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) override;
-
-    ReadFromSystemTables(
-        const Names & column_names_,
-        const SelectQueryInfo & query_info_,
-        const StorageSnapshotPtr & storage_snapshot_,
-        const ContextPtr & context_,
-        Block sample_block,
-        std::vector<UInt8> columns_mask_,
-        size_t max_block_size_);
-
-    void applyFilters(ActionDAGNodes added_filter_nodes) override;
-
-protected:
-    void applyFiltersImpl(ActionDAGNodes added_filter_nodes, bool need_detached_tables_);
-
-    template <class T>
-    Pipe createPipe()
-    {
-        return Pipe(std::make_shared<T>(
-            std::move(columns_mask),
-            getOutputStream().header,
-            max_block_size,
-            std::move(filtered_databases_column),
-            std::move(filtered_tables_column),
-            context));
-    }
-
-private:
-    std::vector<UInt8> columns_mask;
-    size_t max_block_size;
-
-    ColumnPtr filtered_databases_column;
-    ColumnPtr filtered_tables_column;
-};
-
-class ReadFromSystemDetachedTables : public ReadFromSystemTables
-{
-public:
-    std::string getName() const override { return "ReadFromSystemDetachedTables"; }
-    void initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) override;
-
-    ReadFromSystemDetachedTables(
-        const Names & column_names_,
-        const SelectQueryInfo & query_info_,
-        const StorageSnapshotPtr & storage_snapshot_,
-        const ContextPtr & context_,
-        Block sample_block,
-        std::vector<UInt8> columns_mask_,
-        size_t max_block_size_);
-
-    void applyFilters(ActionDAGNodes added_filter_nodes) override;
-};
-}
diff --git a/src/Storages/System/StorageSystemDetachedTables.cpp b/src/Storages/System/StorageSystemDetachedTables.cpp
index 99c15fdae0e..9805fa2a8cf 100644
--- a/src/Storages/System/StorageSystemDetachedTables.cpp
+++ b/src/Storages/System/StorageSystemDetachedTables.cpp
@@ -13,7 +13,6 @@
 #include <Storages/ColumnsDescription.h>
 #include <Storages/ProjectionsDescription.h>
 #include <Storages/StorageInMemoryMetadata.h>
-#include <Storages/System/ReadFromSystemTables.h>
 #include <Storages/System/getQueriedColumnsMaskAndHeader.h>
 #include <Storages/VirtualColumnUtils.h>
 
@@ -23,6 +22,204 @@
 namespace DB
 {
 
+namespace
+{
+
+ColumnPtr getFilteredDatabases(const ActionsDAG::Node * predicate, ContextPtr context)
+{
+    MutableColumnPtr column = ColumnString::create();
+
+    const auto databases = DatabaseCatalog::instance().getDatabases();
+    for (const auto & database_name : databases | boost::adaptors::map_keys)
+    {
+        if (database_name == DatabaseCatalog::TEMPORARY_DATABASE)
+            continue; /// We don't want to show the internal database for temporary tables in system.tables
+
+        column->insert(database_name);
+    }
+
+    Block block{ColumnWithTypeAndName(std::move(column), std::make_shared<DataTypeString>(), "database")};
+    VirtualColumnUtils::filterBlockWithPredicate(predicate, block, context);
+    return block.getByPosition(0).column;
+}
+
+ColumnPtr getFilteredTables(const ActionsDAG::Node * predicate, const ColumnPtr & filtered_databases_column, ContextPtr context)
+{
+    Block sample{
+        ColumnWithTypeAndName(nullptr, std::make_shared<DataTypeString>(), "name"),
+        ColumnWithTypeAndName(nullptr, std::make_shared<DataTypeString>(), "engine")};
+
+    MutableColumnPtr database_column = ColumnString::create();
+    MutableColumnPtr engine_column;
+
+    auto dag = VirtualColumnUtils::splitFilterDagForAllowedInputs(predicate, &sample);
+    if (dag)
+    {
+        bool filter_by_engine = false;
+        for (const auto * input : dag->getInputs())
+            if (input->result_name == "engine")
+                filter_by_engine = true;
+
+        if (filter_by_engine)
+            engine_column = ColumnString::create();
+    }
+
+    for (size_t database_idx = 0; database_idx < filtered_databases_column->size(); ++database_idx)
+    {
+        const auto & database_name = filtered_databases_column->getDataAt(database_idx).toString();
+        DatabasePtr database = DatabaseCatalog::instance().tryGetDatabase(database_name);
+        if (!database)
+            continue;
+
+        auto table_it = database->getDetachedTablesIterator(context, {}, false);
+        for (; table_it->isValid(); table_it->next())
+        {
+            database_column->insert(table_it->table());
+        }
+    }
+
+    Block block{ColumnWithTypeAndName(std::move(database_column), std::make_shared<DataTypeString>(), "name")};
+    if (engine_column)
+        block.insert(ColumnWithTypeAndName(std::move(engine_column), std::make_shared<DataTypeString>(), "engine"));
+
+    if (dag)
+        VirtualColumnUtils::filterBlockWithDAG(dag, block, context);
+
+    return block.getByPosition(0).column;
+}
+
+class DetachedTablesBlockSource : public ISource
+{
+public:
+    DetachedTablesBlockSource(
+        std::vector<UInt8> columns_mask_,
+        Block header,
+        UInt64 max_block_size_,
+        ColumnPtr databases_,
+        ColumnPtr detached_tables_,
+        ContextPtr context_)
+        : ISource(std::move(header))
+        , columns_mask(std::move(columns_mask_))
+        , max_block_size(max_block_size_)
+        , databases(std::move(databases_))
+        , context(Context::createCopy(context_))
+    {
+        size_t size = detached_tables_->size();
+        detached_tables.reserve(size);
+        for (size_t idx = 0; idx < size; ++idx)
+        {
+            detached_tables.insert(detached_tables_->getDataAt(idx).toString());
+        }
+    }
+
+    String getName() const override { return "DetachedTables"; }
+
+protected:
+    Chunk generate() override
+    {
+        if (done)
+            return {};
+
+        MutableColumns result_columns = getPort().getHeader().cloneEmptyColumns();
+
+        const auto access = context->getAccess();
+        const bool need_to_check_access_for_databases = !access->isGranted(AccessType::SHOW_TABLES);
+
+        for (size_t database_idx = 0, rows_count = 0; database_idx < databases->size() && rows_count < max_block_size; ++database_idx)
+        {
+            database_name = databases->getDataAt(database_idx).toString();
+            database = DatabaseCatalog::instance().tryGetDatabase(database_name);
+
+            if (!database)
+                continue;
+
+            const bool need_to_check_access_for_tables
+                = need_to_check_access_for_databases && !access->isGranted(AccessType::SHOW_TABLES, database_name);
+
+            if (!detached_tables_it || !detached_tables_it->isValid())
+                detached_tables_it = database->getDetachedTablesIterator(context, {}, false);
+
+            for (; rows_count < max_block_size && detached_tables_it->isValid(); detached_tables_it->next())
+            {
+                const auto detached_table_name = detached_tables_it->table();
+
+                if (!detached_tables.contains(detached_table_name))
+                    continue;
+
+                if (need_to_check_access_for_tables && !access->isGranted(AccessType::SHOW_TABLES, database_name, detached_table_name))
+                    continue;
+
+                fillResultColumnsByDetachedTableIterator(result_columns);
+                ++rows_count;
+            }
+        }
+
+        const UInt64 num_rows = result_columns.at(0)->size();
+        done = true;
+        return Chunk(std::move(result_columns), num_rows);
+    }
+
+private:
+    const std::vector<UInt8> columns_mask;
+    const UInt64 max_block_size;
+    const ColumnPtr databases;
+    NameSet detached_tables;
+    DatabaseDetachedTablesSnapshotIteratorPtr detached_tables_it;
+    ContextPtr context;
+    bool done = false;
+    DatabasePtr database;
+    std::string database_name;
+
+    void fillResultColumnsByDetachedTableIterator(MutableColumns & result_columns) const
+    {
+        size_t src_index = 0;
+        size_t res_index = 0;
+
+        if (columns_mask[src_index++])
+            result_columns[res_index++]->insert(detached_tables_it->database());
+
+        if (columns_mask[src_index++])
+            result_columns[res_index++]->insert(detached_tables_it->table());
+
+
+        if (columns_mask[src_index++])
+            result_columns[res_index++]->insert(detached_tables_it->uuid());
+
+        if (columns_mask[src_index++])
+            result_columns[res_index++]->insert(detached_tables_it->metadataPath());
+
+        if (columns_mask[src_index++])
+            result_columns[res_index++]->insert(detached_tables_it->isPermanently());
+    }
+};
+
+}
+
+class ReadFromSystemDetachedTables : public SourceStepWithFilter
+{
+public:
+    std::string getName() const override { return "ReadFromSystemDetachedTables"; }
+    void initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) override;
+
+    ReadFromSystemDetachedTables(
+        const Names & column_names_,
+        const SelectQueryInfo & query_info_,
+        const StorageSnapshotPtr & storage_snapshot_,
+        const ContextPtr & context_,
+        Block sample_block,
+        std::vector<UInt8> columns_mask_,
+        size_t max_block_size_);
+
+    void applyFilters(ActionDAGNodes added_filter_nodes) override;
+
+private:
+    std::vector<UInt8> columns_mask;
+    size_t max_block_size;
+
+    ColumnPtr filtered_databases_column;
+    ColumnPtr filtered_tables_column;
+};
+
 StorageSystemDetachedTables::StorageSystemDetachedTables(const StorageID & table_id_) : IStorage(table_id_)
 {
     StorageInMemoryMetadata storage_metadata;
@@ -60,4 +257,42 @@ void StorageSystemDetachedTables::read(
 
     query_plan.addStep(std::move(reading));
 }
+
+ReadFromSystemDetachedTables::ReadFromSystemDetachedTables(
+    const Names & column_names_,
+    const SelectQueryInfo & query_info_,
+    const StorageSnapshotPtr & storage_snapshot_,
+    const ContextPtr & context_,
+    Block sample_block,
+    std::vector<UInt8> columns_mask_,
+    size_t max_block_size_)
+    : SourceStepWithFilter(DataStream{.header = std::move(sample_block)}, column_names_, query_info_, storage_snapshot_, context_)
+    , columns_mask(std::move(columns_mask_))
+    , max_block_size(max_block_size_)
+{
+}
+
+void ReadFromSystemDetachedTables::applyFilters(ActionDAGNodes added_filter_nodes)
+{
+    SourceStepWithFilter::applyFilters(std::move(added_filter_nodes));
+
+    const ActionsDAG::Node * predicate = nullptr;
+    if (filter_actions_dag)
+        predicate = filter_actions_dag->getOutputs().at(0);
+
+    filtered_databases_column = getFilteredDatabases(predicate, context);
+    filtered_tables_column = getFilteredTables(predicate, filtered_databases_column, context);
+}
+
+void ReadFromSystemDetachedTables::initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &)
+{
+    auto pipe = Pipe(std::make_shared<DetachedTablesBlockSource>(
+        std::move(columns_mask),
+        getOutputStream().header,
+        max_block_size,
+        std::move(filtered_databases_column),
+        std::move(filtered_tables_column),
+        context));
+    pipeline.init(std::move(pipe));
+}
 }
diff --git a/src/Storages/System/StorageSystemTables.cpp b/src/Storages/System/StorageSystemTables.cpp
index 40190f53fd3..783b899c978 100644
--- a/src/Storages/System/StorageSystemTables.cpp
+++ b/src/Storages/System/StorageSystemTables.cpp
@@ -1,30 +1,29 @@
-#include <Access/ContextAccess.h>
 #include <Columns/ColumnString.h>
-#include <Common/StringUtils.h>
-#include <Common/typeid_cast.h>
-#include <DataTypes/DataTypeArray.h>
+#include <DataTypes/DataTypeString.h>
 #include <DataTypes/DataTypeDateTime.h>
 #include <DataTypes/DataTypeNullable.h>
-#include <DataTypes/DataTypeString.h>
-#include <DataTypes/DataTypeUUID.h>
-#include <DataTypes/DataTypesNumber.h>
-#include <Disks/IStoragePolicy.h>
+#include <Storages/System/StorageSystemTables.h>
+#include <Storages/System/getQueriedColumnsMaskAndHeader.h>
+#include <Storages/MergeTree/MergeTreeData.h>
+#include <Storages/SelectQueryInfo.h>
+#include <Storages/VirtualColumnUtils.h>
+#include <Access/ContextAccess.h>
 #include <Interpreters/Context.h>
 #include <Interpreters/DatabaseCatalog.h>
 #include <Interpreters/formatWithPossiblyHidingSecrets.h>
 #include <Parsers/ASTCreateQuery.h>
 #include <Parsers/ASTSelectWithUnionQuery.h>
+#include <Common/typeid_cast.h>
+#include <Common/StringUtils.h>
+#include <DataTypes/DataTypesNumber.h>
+#include <DataTypes/DataTypeArray.h>
+#include <Disks/IStoragePolicy.h>
 #include <Processors/ISource.h>
 #include <Processors/QueryPlan/QueryPlan.h>
 #include <Processors/QueryPlan/SourceStepWithFilter.h>
 #include <QueryPipeline/Pipe.h>
 #include <QueryPipeline/QueryPipelineBuilder.h>
-#include <Storages/MergeTree/MergeTreeData.h>
-#include <Storages/SelectQueryInfo.h>
-#include <Storages/System/ReadFromSystemTables.h>
-#include <Storages/System/StorageSystemTables.h>
-#include <Storages/System/getQueriedColumnsMaskAndHeader.h>
-#include <Storages/VirtualColumnUtils.h>
+#include <DataTypes/DataTypeUUID.h>
 
 #include <boost/range/adaptor/map.hpp>
 
@@ -106,6 +105,628 @@ StorageSystemTables::StorageSystemTables(const StorageID & table_id_)
     setInMemoryMetadata(storage_metadata);
 }
 
+
+namespace
+{
+
+ColumnPtr getFilteredDatabases(const ActionsDAG::Node * predicate, ContextPtr context)
+{
+    MutableColumnPtr column = ColumnString::create();
+
+    const auto databases = DatabaseCatalog::instance().getDatabases();
+    for (const auto & database_name : databases | boost::adaptors::map_keys)
+    {
+        if (database_name == DatabaseCatalog::TEMPORARY_DATABASE)
+            continue; /// We don't want to show the internal database for temporary tables in system.tables
+
+        column->insert(database_name);
+    }
+
+    Block block { ColumnWithTypeAndName(std::move(column), std::make_shared<DataTypeString>(), "database") };
+    VirtualColumnUtils::filterBlockWithPredicate(predicate, block, context);
+    return block.getByPosition(0).column;
+}
+
+ColumnPtr getFilteredTables(const ActionsDAG::Node * predicate, const ColumnPtr & filtered_databases_column, ContextPtr context)
+{
+    Block sample {
+        ColumnWithTypeAndName(nullptr, std::make_shared<DataTypeString>(), "name"),
+        ColumnWithTypeAndName(nullptr, std::make_shared<DataTypeString>(), "engine")
+    };
+
+    MutableColumnPtr database_column = ColumnString::create();
+    MutableColumnPtr engine_column;
+
+    auto dag = VirtualColumnUtils::splitFilterDagForAllowedInputs(predicate, &sample);
+    if (dag)
+    {
+        bool filter_by_engine = false;
+        for (const auto * input : dag->getInputs())
+            if (input->result_name == "engine")
+                filter_by_engine = true;
+
+        if (filter_by_engine)
+            engine_column = ColumnString::create();
+    }
+
+    for (size_t database_idx = 0; database_idx < filtered_databases_column->size(); ++database_idx)
+    {
+        const auto & database_name = filtered_databases_column->getDataAt(database_idx).toString();
+        DatabasePtr database = DatabaseCatalog::instance().tryGetDatabase(database_name);
+        if (!database)
+            continue;
+
+        for (auto table_it = database->getTablesIterator(context); table_it->isValid(); table_it->next())
+        {
+            database_column->insert(table_it->name());
+            if (engine_column)
+                engine_column->insert(table_it->table()->getName());
+        }
+    }
+
+    Block block {ColumnWithTypeAndName(std::move(database_column), std::make_shared<DataTypeString>(), "name")};
+    if (engine_column)
+        block.insert(ColumnWithTypeAndName(std::move(engine_column), std::make_shared<DataTypeString>(), "engine"));
+
+    if (dag)
+        VirtualColumnUtils::filterBlockWithDAG(dag, block, context);
+
+    return block.getByPosition(0).column;
+}
+
+/// Avoid heavy operation on tables if we only queried columns that we can get without table object.
+/// Otherwise it will require table initialization for Lazy database.
+bool needTable(const DatabasePtr & database, const Block & header)
+{
+    if (database->getEngineName() != "Lazy")
+        return true;
+
+    static const std::set<std::string> columns_without_table = { "database", "name", "uuid", "metadata_modification_time" };
+    for (const auto & column : header.getColumnsWithTypeAndName())
+    {
+        if (columns_without_table.find(column.name) == columns_without_table.end())
+            return true;
+    }
+    return false;
+}
+
+
+class TablesBlockSource : public ISource
+{
+public:
+    TablesBlockSource(
+        std::vector<UInt8> columns_mask_,
+        Block header,
+        UInt64 max_block_size_,
+        ColumnPtr databases_,
+        ColumnPtr tables_,
+        ContextPtr context_)
+        : ISource(std::move(header))
+        , columns_mask(std::move(columns_mask_))
+        , max_block_size(max_block_size_)
+        , databases(std::move(databases_))
+        , context(Context::createCopy(context_))
+    {
+        size_t size = tables_->size();
+        tables.reserve(size);
+        for (size_t idx = 0; idx < size; ++idx)
+            tables.insert(tables_->getDataAt(idx).toString());
+    }
+
+    String getName() const override { return "Tables"; }
+
+protected:
+    Chunk generate() override
+    {
+        if (done)
+            return {};
+
+        MutableColumns res_columns = getPort().getHeader().cloneEmptyColumns();
+
+        const auto access = context->getAccess();
+        const bool need_to_check_access_for_databases = !access->isGranted(AccessType::SHOW_TABLES);
+
+        size_t rows_count = 0;
+        while (rows_count < max_block_size)
+        {
+            if (tables_it && !tables_it->isValid())
+                ++database_idx;
+
+            while (database_idx < databases->size() && (!tables_it || !tables_it->isValid()))
+            {
+                database_name = databases->getDataAt(database_idx).toString();
+                database = DatabaseCatalog::instance().tryGetDatabase(database_name);
+
+                if (!database)
+                {
+                    /// Database was deleted just now or the user has no access.
+                    ++database_idx;
+                    continue;
+                }
+
+                break;
+            }
+
+            /// This is for temporary tables. They are output in single block regardless to max_block_size.
+            if (database_idx >= databases->size())
+            {
+                if (context->hasSessionContext())
+                {
+                    Tables external_tables = context->getSessionContext()->getExternalTables();
+
+                    for (auto & table : external_tables)
+                    {
+                        size_t src_index = 0;
+                        size_t res_index = 0;
+
+                        // database
+                        if (columns_mask[src_index++])
+                            res_columns[res_index++]->insertDefault();
+
+                        // name
+                        if (columns_mask[src_index++])
+                            res_columns[res_index++]->insert(table.first);
+
+                        // uuid
+                        if (columns_mask[src_index++])
+                            res_columns[res_index++]->insert(table.second->getStorageID().uuid);
+
+                        // engine
+                        if (columns_mask[src_index++])
+                            res_columns[res_index++]->insert(table.second->getName());
+
+                        // is_temporary
+                        if (columns_mask[src_index++])
+                            res_columns[res_index++]->insert(1u);
+
+                        // data_paths
+                        if (columns_mask[src_index++])
+                            res_columns[res_index++]->insertDefault();
+
+                        // metadata_path
+                        if (columns_mask[src_index++])
+                            res_columns[res_index++]->insertDefault();
+
+                        // metadata_modification_time
+                        if (columns_mask[src_index++])
+                            res_columns[res_index++]->insertDefault();
+
+                        // metadata_version
+                        // Temporary tables does not support replication
+                        if (columns_mask[src_index++])
+                            res_columns[res_index++]->insertDefault();
+
+                        // dependencies_database
+                        if (columns_mask[src_index++])
+                            res_columns[res_index++]->insertDefault();
+
+                        // dependencies_table
+                        if (columns_mask[src_index++])
+                            res_columns[res_index++]->insertDefault();
+
+                        // create_table_query
+                        if (columns_mask[src_index++])
+                        {
+                            auto temp_db = DatabaseCatalog::instance().getDatabaseForTemporaryTables();
+                            ASTPtr ast = temp_db ? temp_db->tryGetCreateTableQuery(table.second->getStorageID().getTableName(), context) : nullptr;
+                            res_columns[res_index++]->insert(ast ? format({context, *ast}) : "");
+                        }
+
+                        // engine_full
+                        if (columns_mask[src_index++])
+                            res_columns[res_index++]->insert(table.second->getName());
+
+                        const auto & settings = context->getSettingsRef();
+                        while (src_index < columns_mask.size())
+                        {
+                            // total_rows
+                            if (src_index == 19 && columns_mask[src_index])
+                            {
+                                if (auto total_rows = table.second->totalRows(settings))
+                                    res_columns[res_index++]->insert(*total_rows);
+                                else
+                                    res_columns[res_index++]->insertDefault();
+                            }
+                            // total_bytes
+                            else if (src_index == 20 && columns_mask[src_index])
+                            {
+                                if (auto total_bytes = table.second->totalBytes(settings))
+                                    res_columns[res_index++]->insert(*total_bytes);
+                                else
+                                    res_columns[res_index++]->insertDefault();
+                            }
+                            /// Fill the rest columns with defaults
+                            else if (columns_mask[src_index])
+                                res_columns[res_index++]->insertDefault();
+                            src_index++;
+                        }
+                    }
+                }
+
+                UInt64 num_rows = res_columns.at(0)->size();
+                done = true;
+                return Chunk(std::move(res_columns), num_rows);
+            }
+
+            const bool need_to_check_access_for_tables = need_to_check_access_for_databases && !access->isGranted(AccessType::SHOW_TABLES, database_name);
+
+            if (!tables_it || !tables_it->isValid())
+                tables_it = database->getTablesIterator(context);
+
+            const bool need_table = needTable(database, getPort().getHeader());
+
+            for (; rows_count < max_block_size && tables_it->isValid(); tables_it->next())
+            {
+                auto table_name = tables_it->name();
+                if (!tables.contains(table_name))
+                    continue;
+
+                if (need_to_check_access_for_tables && !access->isGranted(AccessType::SHOW_TABLES, database_name, table_name))
+                    continue;
+
+                StoragePtr table = nullptr;
+                TableLockHolder lock;
+                if (need_table)
+                {
+                    table = tables_it->table();
+                    if (!table)
+                        // Table might have just been removed or detached for Lazy engine (see DatabaseLazy::tryGetTable())
+                        continue;
+
+                    /// The only column that requires us to hold a shared lock is data_paths as rename might alter them (on ordinary tables)
+                    /// and it's not protected internally by other mutexes
+                    static const size_t DATA_PATHS_INDEX = 5;
+                    if (columns_mask[DATA_PATHS_INDEX])
+                    {
+                        lock = table->tryLockForShare(context->getCurrentQueryId(),
+                                                      context->getSettingsRef().lock_acquire_timeout);
+                        if (!lock)
+                            // Table was dropped while acquiring the lock, skipping table
+                            continue;
+                    }
+                }
+                ++rows_count;
+
+                size_t src_index = 0;
+                size_t res_index = 0;
+
+                if (columns_mask[src_index++])
+                    res_columns[res_index++]->insert(database_name);
+
+                if (columns_mask[src_index++])
+                    res_columns[res_index++]->insert(table_name);
+
+                if (columns_mask[src_index++])
+                    res_columns[res_index++]->insert(tables_it->uuid());
+
+                if (columns_mask[src_index++])
+                {
+                    chassert(table != nullptr);
+                    res_columns[res_index++]->insert(table->getName());
+                }
+
+                if (columns_mask[src_index++])
+                    res_columns[res_index++]->insert(0u);  // is_temporary
+
+                if (columns_mask[src_index++])
+                {
+                    chassert(lock != nullptr);
+                    Array table_paths_array;
+                    auto paths = table->getDataPaths();
+                    table_paths_array.reserve(paths.size());
+                    for (const String & path : paths)
+                        table_paths_array.push_back(path);
+                    res_columns[res_index++]->insert(table_paths_array);
+                    /// We don't need the lock anymore
+                    lock = nullptr;
+                }
+
+                if (columns_mask[src_index++])
+                    res_columns[res_index++]->insert(database->getObjectMetadataPath(table_name));
+
+                if (columns_mask[src_index++])
+                    res_columns[res_index++]->insert(static_cast<UInt64>(database->getObjectMetadataModificationTime(table_name)));
+
+                StorageMetadataPtr metadata_snapshot;
+                if (table)
+                    metadata_snapshot = table->getInMemoryMetadataPtr();
+
+                if (columns_mask[src_index++])
+                {
+                    if (metadata_snapshot && table->supportsReplication())
+                        res_columns[res_index++]->insert(metadata_snapshot->metadata_version);
+                    else
+                        res_columns[res_index++]->insertDefault();
+                }
+
+                {
+                    Array views_table_name_array;
+                    Array views_database_name_array;
+                    if (columns_mask[src_index] || columns_mask[src_index + 1])
+                    {
+                        const auto view_ids = DatabaseCatalog::instance().getDependentViews(StorageID(database_name, table_name));
+
+                        views_table_name_array.reserve(view_ids.size());
+                        views_database_name_array.reserve(view_ids.size());
+                        for (const auto & view_id : view_ids)
+                        {
+                            views_table_name_array.push_back(view_id.table_name);
+                            views_database_name_array.push_back(view_id.database_name);
+                        }
+                    }
+
+                    if (columns_mask[src_index++])
+                        res_columns[res_index++]->insert(views_database_name_array);
+
+                    if (columns_mask[src_index++])
+                        res_columns[res_index++]->insert(views_table_name_array);
+                }
+
+                if (columns_mask[src_index] || columns_mask[src_index + 1] || columns_mask[src_index + 2])
+                {
+                    ASTPtr ast = database->tryGetCreateTableQuery(table_name, context);
+                    auto * ast_create = ast ? ast->as<ASTCreateQuery>() : nullptr;
+
+                    if (ast_create && !context->getSettingsRef().show_table_uuid_in_table_create_query_if_not_nil)
+                    {
+                        ast_create->uuid = UUIDHelpers::Nil;
+                        ast_create->to_inner_uuid = UUIDHelpers::Nil;
+                    }
+
+                    if (columns_mask[src_index++])
+                        res_columns[res_index++]->insert(ast ? format({context, *ast}) : "");
+
+                    if (columns_mask[src_index++])
+                    {
+                        String engine_full;
+
+                        if (ast_create && ast_create->storage)
+                        {
+                            engine_full = format({context, *ast_create->storage});
+
+                            static const char * const extra_head = " ENGINE = ";
+                            if (startsWith(engine_full, extra_head))
+                                engine_full = engine_full.substr(strlen(extra_head));
+                        }
+
+                        res_columns[res_index++]->insert(engine_full);
+                    }
+
+                    if (columns_mask[src_index++])
+                    {
+                        String as_select;
+                        if (ast_create && ast_create->select)
+                            as_select = format({context, *ast_create->select});
+                        res_columns[res_index++]->insert(as_select);
+                    }
+                }
+                else
+                    src_index += 3;
+
+                ASTPtr expression_ptr;
+                if (columns_mask[src_index++])
+                {
+                    if (metadata_snapshot && (expression_ptr = metadata_snapshot->getPartitionKeyAST()))
+                        res_columns[res_index++]->insert(format({context, *expression_ptr}));
+                    else
+                        res_columns[res_index++]->insertDefault();
+                }
+
+                if (columns_mask[src_index++])
+                {
+                    if (metadata_snapshot && (expression_ptr = metadata_snapshot->getSortingKey().expression_list_ast))
+                        res_columns[res_index++]->insert(format({context, *expression_ptr}));
+                    else
+                        res_columns[res_index++]->insertDefault();
+                }
+
+                if (columns_mask[src_index++])
+                {
+                    if (metadata_snapshot && (expression_ptr = metadata_snapshot->getPrimaryKey().expression_list_ast))
+                        res_columns[res_index++]->insert(format({context, *expression_ptr}));
+                    else
+                        res_columns[res_index++]->insertDefault();
+                }
+
+                if (columns_mask[src_index++])
+                {
+                    if (metadata_snapshot && (expression_ptr = metadata_snapshot->getSamplingKeyAST()))
+                        res_columns[res_index++]->insert(format({context, *expression_ptr}));
+                    else
+                        res_columns[res_index++]->insertDefault();
+                }
+
+                if (columns_mask[src_index++])
+                {
+                    auto policy = table ? table->getStoragePolicy() : nullptr;
+                    if (policy)
+                        res_columns[res_index++]->insert(policy->getName());
+                    else
+                        res_columns[res_index++]->insertDefault();
+                }
+
+                auto settings = context->getSettingsRef();
+                settings.select_sequential_consistency = 0;
+                if (columns_mask[src_index++])
+                {
+                    auto total_rows = table ? table->totalRows(settings) : std::nullopt;
+                    if (total_rows)
+                        res_columns[res_index++]->insert(*total_rows);
+                    else
+                        res_columns[res_index++]->insertDefault();
+                }
+
+                if (columns_mask[src_index++])
+                {
+                    auto total_bytes = table->totalBytes(settings);
+                    if (total_bytes)
+                        res_columns[res_index++]->insert(*total_bytes);
+                    else
+                        res_columns[res_index++]->insertDefault();
+                }
+
+                if (columns_mask[src_index++])
+                {
+                    auto total_bytes_uncompressed = table->totalBytesUncompressed(settings);
+                    if (total_bytes_uncompressed)
+                        res_columns[res_index++]->insert(*total_bytes_uncompressed);
+                    else
+                        res_columns[res_index++]->insertDefault();
+                }
+
+                auto table_merge_tree = std::dynamic_pointer_cast<MergeTreeData>(table);
+                if (columns_mask[src_index++])
+                {
+                    if (table_merge_tree)
+                        res_columns[res_index++]->insert(table_merge_tree->getAllPartsCount());
+                    else
+                        res_columns[res_index++]->insertDefault();
+                }
+
+                if (columns_mask[src_index++])
+                {
+                    if (table_merge_tree)
+                        res_columns[res_index++]->insert(table_merge_tree->getActivePartsCount());
+                    else
+                        res_columns[res_index++]->insertDefault();
+                }
+
+                if (columns_mask[src_index++])
+                {
+                    if (table_merge_tree)
+                    {
+                        res_columns[res_index++]->insert(table_merge_tree->getTotalMarksCount());
+                    }
+                    else
+                        res_columns[res_index++]->insertDefault();
+                }
+
+                if (columns_mask[src_index++])
+                {
+                    auto lifetime_rows = table ? table->lifetimeRows() : std::nullopt;
+                    if (lifetime_rows)
+                        res_columns[res_index++]->insert(*lifetime_rows);
+                    else
+                        res_columns[res_index++]->insertDefault();
+                }
+
+                if (columns_mask[src_index++])
+                {
+                    auto lifetime_bytes = table ? table->lifetimeBytes() : std::nullopt;
+                    if (lifetime_bytes)
+                        res_columns[res_index++]->insert(*lifetime_bytes);
+                    else
+                        res_columns[res_index++]->insertDefault();
+                }
+
+                if (columns_mask[src_index++])
+                {
+                    if (metadata_snapshot)
+                        res_columns[res_index++]->insert(metadata_snapshot->comment);
+                    else
+                        res_columns[res_index++]->insertDefault();
+                }
+
+                if (columns_mask[src_index++])
+                {
+                    if (table)
+                        res_columns[res_index++]->insert(table->storesDataOnDisk());
+                    else
+                        res_columns[res_index++]->insertDefault();
+                }
+
+                if (columns_mask[src_index] || columns_mask[src_index + 1] || columns_mask[src_index + 2] || columns_mask[src_index + 3])
+                {
+                    auto dependencies = DatabaseCatalog::instance().getLoadingDependencies(StorageID{database_name, table_name});
+                    auto dependents = DatabaseCatalog::instance().getLoadingDependents(StorageID{database_name, table_name});
+
+                    Array dependencies_databases;
+                    Array dependencies_tables;
+                    dependencies_databases.reserve(dependencies.size());
+                    dependencies_tables.reserve(dependencies.size());
+                    for (const auto & dependency : dependencies)
+                    {
+                        dependencies_databases.push_back(dependency.database_name);
+                        dependencies_tables.push_back(dependency.table_name);
+                    }
+
+                    Array dependents_databases;
+                    Array dependents_tables;
+                    dependents_databases.reserve(dependents.size());
+                    dependents_tables.reserve(dependents.size());
+                    for (const auto & dependent : dependents)
+                    {
+                        dependents_databases.push_back(dependent.database_name);
+                        dependents_tables.push_back(dependent.table_name);
+                    }
+
+                    if (columns_mask[src_index++])
+                        res_columns[res_index++]->insert(dependencies_databases);
+                    if (columns_mask[src_index++])
+                        res_columns[res_index++]->insert(dependencies_tables);
+
+                    if (columns_mask[src_index++])
+                        res_columns[res_index++]->insert(dependents_databases);
+                    if (columns_mask[src_index++])
+                        res_columns[res_index++]->insert(dependents_tables);
+
+                }
+            }
+        }
+
+        UInt64 num_rows = res_columns.at(0)->size();
+        return Chunk(std::move(res_columns), num_rows);
+    }
+private:
+    std::vector<UInt8> columns_mask;
+    UInt64 max_block_size;
+    ColumnPtr databases;
+    NameSet tables;
+    size_t database_idx = 0;
+    DatabaseTablesIteratorPtr tables_it;
+    ContextPtr context;
+    bool done = false;
+    DatabasePtr database;
+    std::string database_name;
+};
+
+}
+
+class ReadFromSystemTables : public SourceStepWithFilter
+{
+public:
+    std::string getName() const override { return "ReadFromSystemTables"; }
+    void initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &) override;
+
+    ReadFromSystemTables(
+        const Names & column_names_,
+        const SelectQueryInfo & query_info_,
+        const StorageSnapshotPtr & storage_snapshot_,
+        const ContextPtr & context_,
+        Block sample_block,
+        std::vector<UInt8> columns_mask_,
+        size_t max_block_size_)
+        : SourceStepWithFilter(
+            DataStream{.header = std::move(sample_block)},
+            column_names_,
+            query_info_,
+            storage_snapshot_,
+            context_)
+        , columns_mask(std::move(columns_mask_))
+        , max_block_size(max_block_size_)
+    {
+    }
+
+    void applyFilters(ActionDAGNodes added_filter_nodes) override;
+
+private:
+    std::vector<UInt8> columns_mask;
+    size_t max_block_size;
+
+    ColumnPtr filtered_databases_column;
+    ColumnPtr filtered_tables_column;
+};
+
 void StorageSystemTables::read(
     QueryPlan & query_plan,
     const Names & column_names,
@@ -126,4 +747,24 @@ void StorageSystemTables::read(
 
     query_plan.addStep(std::move(reading));
 }
+
+void ReadFromSystemTables::applyFilters(ActionDAGNodes added_filter_nodes)
+{
+    SourceStepWithFilter::applyFilters(std::move(added_filter_nodes));
+
+    const ActionsDAG::Node * predicate = nullptr;
+    if (filter_actions_dag)
+        predicate = filter_actions_dag->getOutputs().at(0);
+
+    filtered_databases_column = getFilteredDatabases(predicate, context);
+    filtered_tables_column = getFilteredTables(predicate, filtered_databases_column, context);
+}
+
+void ReadFromSystemTables::initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &)
+{
+    Pipe pipe(std::make_shared<TablesBlockSource>(
+        std::move(columns_mask), getOutputStream().header, max_block_size, std::move(filtered_databases_column), std::move(filtered_tables_column), context));
+    pipeline.init(std::move(pipe));
+}
+
 }
diff --git a/src/Storages/System/TablesBlockSource.cpp b/src/Storages/System/TablesBlockSource.cpp
deleted file mode 100644
index d5481856f1b..00000000000
--- a/src/Storages/System/TablesBlockSource.cpp
+++ /dev/null
@@ -1,564 +0,0 @@
-#include "TablesBlockSource.h"
-
-#include <memory>
-
-#include <Access/ContextAccess.h>
-#include <Disks/IStoragePolicy.h>
-#include <Interpreters/DatabaseCatalog.h>
-#include <Interpreters/formatWithPossiblyHidingSecrets.h>
-#include <Parsers/ASTSelectWithUnionQuery.h>
-#include <Storages/IStorage.h>
-#include <Storages/MergeTree/MergeTreeData.h>
-
-
-namespace DB
-{
-
-namespace
-{
-
-/// Avoid heavy operation on tables if we only queried columns that we can get without table object.
-/// Otherwise it will require table initialization for Lazy database.
-bool needTable(const DatabasePtr & database, const Block & header)
-{
-    if (database->getEngineName() != "Lazy")
-        return true;
-
-    static const std::set<std::string> columns_without_table = {"database", "name", "uuid", "metadata_modification_time"};
-    for (const auto & column : header.getColumnsWithTypeAndName())
-    {
-        if (columns_without_table.find(column.name) == columns_without_table.end())
-            return true;
-    }
-    return false;
-}
-
-}
-
-Chunk TablesBlockSource::generate()
-{
-    if (done)
-        return {};
-
-    MutableColumns res_columns = getPort().getHeader().cloneEmptyColumns();
-
-    const auto access = context->getAccess();
-    const bool need_to_check_access_for_databases = !access->isGranted(AccessType::SHOW_TABLES);
-
-    size_t rows_count = 0;
-    while (rows_count < max_block_size)
-    {
-        if (tables_it && !tables_it->isValid())
-            ++database_idx;
-
-        while (database_idx < databases->size() && (!tables_it || !tables_it->isValid()))
-        {
-            database_name = databases->getDataAt(database_idx).toString();
-            database = DatabaseCatalog::instance().tryGetDatabase(database_name);
-
-            if (!database)
-            {
-                /// Database was deleted just now or the user has no access.
-                ++database_idx;
-                continue;
-            }
-
-            break;
-        }
-
-        /// This is for temporary tables. They are output in single block regardless to max_block_size.
-        if (database_idx >= databases->size())
-        {
-            if (context->hasSessionContext())
-            {
-                Tables external_tables = context->getSessionContext()->getExternalTables();
-
-                for (auto & table : external_tables)
-                {
-                    size_t src_index = 0;
-                    size_t res_index = 0;
-
-                    // database
-                    if (columns_mask[src_index++])
-                        res_columns[res_index++]->insertDefault();
-
-                    // name
-                    if (columns_mask[src_index++])
-                        res_columns[res_index++]->insert(table.first);
-
-                    // uuid
-                    if (columns_mask[src_index++])
-                        res_columns[res_index++]->insert(table.second->getStorageID().uuid);
-
-                    // engine
-                    if (columns_mask[src_index++])
-                        res_columns[res_index++]->insert(table.second->getName());
-
-                    // is_temporary
-                    if (columns_mask[src_index++])
-                        res_columns[res_index++]->insert(1u);
-
-                    // data_paths
-                    if (columns_mask[src_index++])
-                        res_columns[res_index++]->insertDefault();
-
-                    // metadata_path
-                    if (columns_mask[src_index++])
-                        res_columns[res_index++]->insertDefault();
-
-                    // metadata_modification_time
-                    if (columns_mask[src_index++])
-                        res_columns[res_index++]->insertDefault();
-
-                    // metadata_version
-                    // Temporary tables does not support replication
-                    if (columns_mask[src_index++])
-                        res_columns[res_index++]->insertDefault();
-
-                    // dependencies_database
-                    if (columns_mask[src_index++])
-                        res_columns[res_index++]->insertDefault();
-
-                    // dependencies_table
-                    if (columns_mask[src_index++])
-                        res_columns[res_index++]->insertDefault();
-
-                    // create_table_query
-                    if (columns_mask[src_index++])
-                    {
-                        auto temp_db = DatabaseCatalog::instance().getDatabaseForTemporaryTables();
-                        ASTPtr ast
-                            = temp_db ? temp_db->tryGetCreateTableQuery(table.second->getStorageID().getTableName(), context) : nullptr;
-                        res_columns[res_index++]->insert(ast ? format({context, *ast}) : "");
-                    }
-
-                    // engine_full
-                    if (columns_mask[src_index++])
-                        res_columns[res_index++]->insert(table.second->getName());
-
-                    const auto & settings = context->getSettingsRef();
-                    while (src_index < columns_mask.size())
-                    {
-                        // total_rows
-                        if (src_index == 19 && columns_mask[src_index])
-                        {
-                            if (auto total_rows = table.second->totalRows(settings))
-                                res_columns[res_index++]->insert(*total_rows);
-                            else
-                                res_columns[res_index++]->insertDefault();
-                        }
-                        // total_bytes
-                        else if (src_index == 20 && columns_mask[src_index])
-                        {
-                            if (auto total_bytes = table.second->totalBytes(settings))
-                                res_columns[res_index++]->insert(*total_bytes);
-                            else
-                                res_columns[res_index++]->insertDefault();
-                        }
-                        /// Fill the rest columns with defaults
-                        else if (columns_mask[src_index])
-                            res_columns[res_index++]->insertDefault();
-                        src_index++;
-                    }
-                }
-            }
-
-            UInt64 num_rows = res_columns.at(0)->size();
-            done = true;
-            return Chunk(std::move(res_columns), num_rows);
-        }
-
-        const bool need_to_check_access_for_tables
-            = need_to_check_access_for_databases && !access->isGranted(AccessType::SHOW_TABLES, database_name);
-
-        if (!tables_it || !tables_it->isValid())
-            tables_it = database->getTablesIterator(context);
-
-        const bool need_table = needTable(database, getPort().getHeader());
-
-        for (; rows_count < max_block_size && tables_it->isValid(); tables_it->next())
-        {
-            auto table_name = tables_it->name();
-            if (!tables.contains(table_name))
-                continue;
-
-            if (need_to_check_access_for_tables && !access->isGranted(AccessType::SHOW_TABLES, database_name, table_name))
-                continue;
-
-            StoragePtr table = nullptr;
-            TableLockHolder lock;
-            if (need_table)
-            {
-                table = tables_it->table();
-                if (!table)
-                    // Table might have just been removed or detached for Lazy engine (see DatabaseLazy::tryGetTable())
-                    continue;
-
-                /// The only column that requires us to hold a shared lock is data_paths as rename might alter them (on ordinary tables)
-                /// and it's not protected internally by other mutexes
-                static const size_t DATA_PATHS_INDEX = 5;
-                if (columns_mask[DATA_PATHS_INDEX])
-                {
-                    lock = table->tryLockForShare(context->getCurrentQueryId(), context->getSettingsRef().lock_acquire_timeout);
-                    if (!lock)
-                        // Table was dropped while acquiring the lock, skipping table
-                        continue;
-                }
-            }
-            ++rows_count;
-
-            size_t src_index = 0;
-            size_t res_index = 0;
-
-            if (columns_mask[src_index++])
-                res_columns[res_index++]->insert(database_name);
-
-            if (columns_mask[src_index++])
-                res_columns[res_index++]->insert(table_name);
-
-            if (columns_mask[src_index++])
-                res_columns[res_index++]->insert(tables_it->uuid());
-
-            if (columns_mask[src_index++])
-            {
-                chassert(table != nullptr);
-                res_columns[res_index++]->insert(table->getName());
-            }
-
-            if (columns_mask[src_index++])
-                res_columns[res_index++]->insert(0u); // is_temporary
-
-            if (columns_mask[src_index++])
-            {
-                chassert(lock != nullptr);
-                Array table_paths_array;
-                auto paths = table->getDataPaths();
-                table_paths_array.reserve(paths.size());
-                for (const String & path : paths)
-                    table_paths_array.push_back(path);
-                res_columns[res_index++]->insert(table_paths_array);
-                /// We don't need the lock anymore
-                lock = nullptr;
-            }
-
-            if (columns_mask[src_index++])
-                res_columns[res_index++]->insert(database->getObjectMetadataPath(table_name));
-
-            if (columns_mask[src_index++])
-                res_columns[res_index++]->insert(static_cast<UInt64>(database->getObjectMetadataModificationTime(table_name)));
-
-            StorageMetadataPtr metadata_snapshot;
-            if (table)
-                metadata_snapshot = table->getInMemoryMetadataPtr();
-
-            if (columns_mask[src_index++])
-            {
-                if (metadata_snapshot && table->supportsReplication())
-                    res_columns[res_index++]->insert(metadata_snapshot->metadata_version);
-                else
-                    res_columns[res_index++]->insertDefault();
-            }
-
-            {
-                Array views_table_name_array;
-                Array views_database_name_array;
-                if (columns_mask[src_index] || columns_mask[src_index + 1])
-                {
-                    const auto view_ids = DatabaseCatalog::instance().getDependentViews(StorageID(database_name, table_name));
-
-                    views_table_name_array.reserve(view_ids.size());
-                    views_database_name_array.reserve(view_ids.size());
-                    for (const auto & view_id : view_ids)
-                    {
-                        views_table_name_array.push_back(view_id.table_name);
-                        views_database_name_array.push_back(view_id.database_name);
-                    }
-                }
-
-                if (columns_mask[src_index++])
-                    res_columns[res_index++]->insert(views_database_name_array);
-
-                if (columns_mask[src_index++])
-                    res_columns[res_index++]->insert(views_table_name_array);
-            }
-
-            if (columns_mask[src_index] || columns_mask[src_index + 1] || columns_mask[src_index + 2])
-            {
-                ASTPtr ast = database->tryGetCreateTableQuery(table_name, context);
-                auto * ast_create = ast ? ast->as<ASTCreateQuery>() : nullptr;
-
-                if (ast_create && !context->getSettingsRef().show_table_uuid_in_table_create_query_if_not_nil)
-                {
-                    ast_create->uuid = UUIDHelpers::Nil;
-                    ast_create->to_inner_uuid = UUIDHelpers::Nil;
-                }
-
-                if (columns_mask[src_index++])
-                    res_columns[res_index++]->insert(ast ? format({context, *ast}) : "");
-
-                if (columns_mask[src_index++])
-                {
-                    String engine_full;
-
-                    if (ast_create && ast_create->storage)
-                    {
-                        engine_full = format({context, *ast_create->storage});
-
-                        static const char * const extra_head = " ENGINE = ";
-                        if (startsWith(engine_full, extra_head))
-                            engine_full = engine_full.substr(strlen(extra_head));
-                    }
-
-                    res_columns[res_index++]->insert(engine_full);
-                }
-
-                if (columns_mask[src_index++])
-                {
-                    String as_select;
-                    if (ast_create && ast_create->select)
-                        as_select = format({context, *ast_create->select});
-                    res_columns[res_index++]->insert(as_select);
-                }
-            }
-            else
-                src_index += 3;
-
-            ASTPtr expression_ptr;
-            if (columns_mask[src_index++])
-            {
-                if (metadata_snapshot && (expression_ptr = metadata_snapshot->getPartitionKeyAST()))
-                    res_columns[res_index++]->insert(format({context, *expression_ptr}));
-                else
-                    res_columns[res_index++]->insertDefault();
-            }
-
-            if (columns_mask[src_index++])
-            {
-                if (metadata_snapshot && (expression_ptr = metadata_snapshot->getSortingKey().expression_list_ast))
-                    res_columns[res_index++]->insert(format({context, *expression_ptr}));
-                else
-                    res_columns[res_index++]->insertDefault();
-            }
-
-            if (columns_mask[src_index++])
-            {
-                if (metadata_snapshot && (expression_ptr = metadata_snapshot->getPrimaryKey().expression_list_ast))
-                    res_columns[res_index++]->insert(format({context, *expression_ptr}));
-                else
-                    res_columns[res_index++]->insertDefault();
-            }
-
-            if (columns_mask[src_index++])
-            {
-                if (metadata_snapshot && (expression_ptr = metadata_snapshot->getSamplingKeyAST()))
-                    res_columns[res_index++]->insert(format({context, *expression_ptr}));
-                else
-                    res_columns[res_index++]->insertDefault();
-            }
-
-            if (columns_mask[src_index++])
-            {
-                auto policy = table ? table->getStoragePolicy() : nullptr;
-                if (policy)
-                    res_columns[res_index++]->insert(policy->getName());
-                else
-                    res_columns[res_index++]->insertDefault();
-            }
-
-            auto settings = context->getSettingsRef();
-            settings.select_sequential_consistency = 0;
-            if (columns_mask[src_index++])
-            {
-                auto total_rows = table ? table->totalRows(settings) : std::nullopt;
-                if (total_rows)
-                    res_columns[res_index++]->insert(*total_rows);
-                else
-                    res_columns[res_index++]->insertDefault();
-            }
-
-            if (columns_mask[src_index++])
-            {
-                auto total_bytes = table->totalBytes(settings);
-                if (total_bytes)
-                    res_columns[res_index++]->insert(*total_bytes);
-                else
-                    res_columns[res_index++]->insertDefault();
-            }
-
-            if (columns_mask[src_index++])
-            {
-                auto total_bytes_uncompressed = table->totalBytesUncompressed(settings);
-                if (total_bytes_uncompressed)
-                    res_columns[res_index++]->insert(*total_bytes_uncompressed);
-                else
-                    res_columns[res_index++]->insertDefault();
-            }
-
-            auto table_merge_tree = std::dynamic_pointer_cast<MergeTreeData>(table);
-            if (columns_mask[src_index++])
-            {
-                if (table_merge_tree)
-                    res_columns[res_index++]->insert(table_merge_tree->getAllPartsCount());
-                else
-                    res_columns[res_index++]->insertDefault();
-            }
-
-            if (columns_mask[src_index++])
-            {
-                if (table_merge_tree)
-                    res_columns[res_index++]->insert(table_merge_tree->getActivePartsCount());
-                else
-                    res_columns[res_index++]->insertDefault();
-            }
-
-            if (columns_mask[src_index++])
-            {
-                if (table_merge_tree)
-                {
-                    res_columns[res_index++]->insert(table_merge_tree->getTotalMarksCount());
-                }
-                else
-                    res_columns[res_index++]->insertDefault();
-            }
-
-            if (columns_mask[src_index++])
-            {
-                auto lifetime_rows = table ? table->lifetimeRows() : std::nullopt;
-                if (lifetime_rows)
-                    res_columns[res_index++]->insert(*lifetime_rows);
-                else
-                    res_columns[res_index++]->insertDefault();
-            }
-
-            if (columns_mask[src_index++])
-            {
-                auto lifetime_bytes = table ? table->lifetimeBytes() : std::nullopt;
-                if (lifetime_bytes)
-                    res_columns[res_index++]->insert(*lifetime_bytes);
-                else
-                    res_columns[res_index++]->insertDefault();
-            }
-
-            if (columns_mask[src_index++])
-            {
-                if (metadata_snapshot)
-                    res_columns[res_index++]->insert(metadata_snapshot->comment);
-                else
-                    res_columns[res_index++]->insertDefault();
-            }
-
-            if (columns_mask[src_index++])
-            {
-                if (table)
-                    res_columns[res_index++]->insert(table->storesDataOnDisk());
-                else
-                    res_columns[res_index++]->insertDefault();
-            }
-
-            if (columns_mask[src_index] || columns_mask[src_index + 1] || columns_mask[src_index + 2] || columns_mask[src_index + 3])
-            {
-                auto dependencies = DatabaseCatalog::instance().getLoadingDependencies(StorageID{database_name, table_name});
-                auto dependents = DatabaseCatalog::instance().getLoadingDependents(StorageID{database_name, table_name});
-
-                Array dependencies_databases;
-                Array dependencies_tables;
-                dependencies_databases.reserve(dependencies.size());
-                dependencies_tables.reserve(dependencies.size());
-                for (const auto & dependency : dependencies)
-                {
-                    dependencies_databases.push_back(dependency.database_name);
-                    dependencies_tables.push_back(dependency.table_name);
-                }
-
-                Array dependents_databases;
-                Array dependents_tables;
-                dependents_databases.reserve(dependents.size());
-                dependents_tables.reserve(dependents.size());
-                for (const auto & dependent : dependents)
-                {
-                    dependents_databases.push_back(dependent.database_name);
-                    dependents_tables.push_back(dependent.table_name);
-                }
-
-                if (columns_mask[src_index++])
-                    res_columns[res_index++]->insert(dependencies_databases);
-                if (columns_mask[src_index++])
-                    res_columns[res_index++]->insert(dependencies_tables);
-
-                if (columns_mask[src_index++])
-                    res_columns[res_index++]->insert(dependents_databases);
-                if (columns_mask[src_index++])
-                    res_columns[res_index++]->insert(dependents_tables);
-            }
-        }
-    }
-
-    UInt64 num_rows = res_columns.at(0)->size();
-    return Chunk(std::move(res_columns), num_rows);
-}
-
-Chunk DetachedTablesBlockSource::generate()
-{
-    if (done)
-        return {};
-
-    MutableColumns result_columns = getPort().getHeader().cloneEmptyColumns();
-
-    const auto access = context->getAccess();
-    const bool need_to_check_access_for_databases = !access->isGranted(AccessType::SHOW_TABLES);
-
-    for (size_t database_idx = 0, rows_count = 0; database_idx < databases->size() && rows_count < max_block_size; ++database_idx)
-    {
-        database_name = databases->getDataAt(database_idx).toString();
-        database = DatabaseCatalog::instance().tryGetDatabase(database_name);
-
-        if (!database)
-            continue;
-
-        const bool need_to_check_access_for_tables
-            = need_to_check_access_for_databases && !access->isGranted(AccessType::SHOW_TABLES, database_name);
-
-        if (!detached_tables_it || !detached_tables_it->isValid())
-            detached_tables_it = database->getDetachedTablesIterator(context, {}, false);
-
-        for (; rows_count < max_block_size && detached_tables_it->isValid(); detached_tables_it->next())
-        {
-            const auto detached_table_name = detached_tables_it->table();
-
-            if (!detached_tables.contains(detached_table_name))
-                continue;
-
-            if (need_to_check_access_for_tables && !access->isGranted(AccessType::SHOW_TABLES, database_name, detached_table_name))
-                continue;
-
-            fillResultColumnsByDetachedTableIterator(result_columns);
-            ++rows_count;
-        }
-    }
-
-    const UInt64 num_rows = result_columns.at(0)->size();
-    done = true;
-    return Chunk(std::move(result_columns), num_rows);
-}
-
-void DetachedTablesBlockSource::fillResultColumnsByDetachedTableIterator(MutableColumns & result_columns) const
-{
-    size_t src_index = 0;
-    size_t res_index = 0;
-
-    if (columns_mask[src_index++])
-        result_columns[res_index++]->insert(detached_tables_it->database());
-
-    if (columns_mask[src_index++])
-        result_columns[res_index++]->insert(detached_tables_it->table());
-
-    if (columns_mask[src_index++])
-        result_columns[res_index++]->insert(detached_tables_it->uuid());
-
-    if (columns_mask[src_index++])
-        result_columns[res_index++]->insert(detached_tables_it->metadataPath());
-
-    if (columns_mask[src_index++])
-        result_columns[res_index++]->insert(detached_tables_it->isPermanently());
-}
-}
diff --git a/src/Storages/System/TablesBlockSource.h b/src/Storages/System/TablesBlockSource.h
deleted file mode 100644
index a9f2226dca8..00000000000
--- a/src/Storages/System/TablesBlockSource.h
+++ /dev/null
@@ -1,92 +0,0 @@
-#pragma once
-
-#include <Databases/IDatabase.h>
-#include <Interpreters/Context.h>
-#include <Processors/ISource.h>
-
-namespace DB
-{
-
-class TablesBlockSource : public ISource
-{
-public:
-    TablesBlockSource(
-        std::vector<UInt8> columns_mask_,
-        Block header,
-        UInt64 max_block_size_,
-        ColumnPtr databases_,
-        ColumnPtr tables_,
-        ContextPtr context_)
-        : ISource(std::move(header))
-        , columns_mask(std::move(columns_mask_))
-        , max_block_size(max_block_size_)
-        , databases(std::move(databases_))
-        , context(Context::createCopy(context_))
-    {
-        size_t size = tables_->size();
-        tables.reserve(size);
-        for (size_t idx = 0; idx < size; ++idx)
-            tables.insert(tables_->getDataAt(idx).toString());
-    }
-
-    String getName() const override { return "Tables"; }
-
-protected:
-    Chunk generate() override;
-
-private:
-    std::vector<UInt8> columns_mask;
-    UInt64 max_block_size;
-    ColumnPtr databases;
-    NameSet tables;
-    size_t database_idx = 0;
-    DatabaseTablesIteratorPtr tables_it;
-    ContextPtr context;
-    bool done = false;
-    DatabasePtr database;
-    std::string database_name;
-};
-
-class DetachedTablesBlockSource : public ISource
-{
-public:
-    DetachedTablesBlockSource(
-        std::vector<UInt8> columns_mask_,
-        Block header,
-        UInt64 max_block_size_,
-        ColumnPtr databases_,
-        ColumnPtr detached_tables_,
-        ContextPtr context_)
-        : ISource(std::move(header))
-        , columns_mask(std::move(columns_mask_))
-        , max_block_size(max_block_size_)
-        , databases(std::move(databases_))
-        , context(Context::createCopy(context_))
-    {
-        size_t size = detached_tables_->size();
-        detached_tables.reserve(size);
-        for (size_t idx = 0; idx < size; ++idx)
-        {
-            detached_tables.insert(detached_tables_->getDataAt(idx).toString());
-        }
-    }
-
-    String getName() const override { return "DetachedTables"; }
-
-protected:
-    Chunk generate() override;
-
-private:
-    const std::vector<UInt8> columns_mask;
-    const UInt64 max_block_size;
-    const ColumnPtr databases;
-    NameSet detached_tables;
-    DatabaseDetachedTablesSnapshotIteratorPtr detached_tables_it;
-    ContextPtr context;
-    bool done = false;
-    DatabasePtr database;
-    std::string database_name;
-
-    void fillResultColumnsByDetachedTableIterator(MutableColumns & result_columns) const;
-};
-}

From 1e54b213850bdd34fc1c251a531da04ae9cb03aa Mon Sep 17 00:00:00 2001
From: AntiTopQuark <AntiTopQuark1350@outlook.com>
Date: Fri, 28 Jun 2024 22:43:26 +0800
Subject: [PATCH 0113/1488] fix fast test for MergeTreeIndexSet

---
 src/Storages/MergeTree/MergeTreeIndexSet.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Storages/MergeTree/MergeTreeIndexSet.cpp b/src/Storages/MergeTree/MergeTreeIndexSet.cpp
index c2a574beb90..36844648ac7 100644
--- a/src/Storages/MergeTree/MergeTreeIndexSet.cpp
+++ b/src/Storages/MergeTree/MergeTreeIndexSet.cpp
@@ -361,7 +361,7 @@ bool MergeTreeIndexConditionSet::mayBeTrueOnGranule(MergeTreeIndexGranulePtr idx
         if (!column->isNullAt(i) && (column->get64(i) & 1))
             return true;
 
-    return condition.checkInHyperrectangle(granule.set_hyperrectangle, index_data_types).can_be_true;
+    return false;
 }
 
 

From fed573ffee1bb57f0b9cff7f6f1c7d5a542af51c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Fri, 28 Jun 2024 17:12:10 +0200
Subject: [PATCH 0114/1488] Extend fix

---
 src/Planner/PlannerActionsVisitor.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/Planner/PlannerActionsVisitor.cpp b/src/Planner/PlannerActionsVisitor.cpp
index 13e96dc7016..1c9553032c2 100644
--- a/src/Planner/PlannerActionsVisitor.cpp
+++ b/src/Planner/PlannerActionsVisitor.cpp
@@ -535,7 +535,7 @@ public:
     }
 
 private:
-    std::unordered_map<std::string_view, const ActionsDAG::Node *> node_name_to_node;
+    std::unordered_map<String, const ActionsDAG::Node *> node_name_to_node;
     ActionsDAG & actions_dag;
     QueryTreeNodePtr scope_node;
 };
@@ -741,7 +741,7 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::vi
     for (size_t i = 1; i < actions_stack_size; ++i)
     {
         auto & actions_stack_node = actions_stack[i];
-        actions_stack_node.addInputConstantColumnIfNecessary(constant_node_name, column);
+        actions_stack_node.addInputConstantColumnIfNecessary(final_name, column);
     }
 
     return {final_name, Levels(0)};
@@ -879,7 +879,7 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::ma
     for (size_t i = 1; i < actions_stack_size; ++i)
     {
         auto & actions_stack_node = actions_stack[i];
-        actions_stack_node.addInputConstantColumnIfNecessary(column.name, column);
+        actions_stack_node.addInputConstantColumnIfNecessary(final_name, column);
     }
 
     return {final_name, Levels(0)};

From 51d293beb9dbbe915c6bedb92613833db0240450 Mon Sep 17 00:00:00 2001
From: Konstantin Morozov <just.morozov.k@gmail.com>
Date: Fri, 28 Jun 2024 15:29:28 +0000
Subject: [PATCH 0115/1488] general logic in detail

---
 .../System/StorageSystemDetachedTables.cpp    |  68 +-----
 src/Storages/System/StorageSystemTables.cpp   | 230 ++++++++++--------
 src/Storages/System/StorageSystemTables.h     |   9 +
 3 files changed, 139 insertions(+), 168 deletions(-)

diff --git a/src/Storages/System/StorageSystemDetachedTables.cpp b/src/Storages/System/StorageSystemDetachedTables.cpp
index 9805fa2a8cf..560056b891a 100644
--- a/src/Storages/System/StorageSystemDetachedTables.cpp
+++ b/src/Storages/System/StorageSystemDetachedTables.cpp
@@ -13,6 +13,7 @@
 #include <Storages/ColumnsDescription.h>
 #include <Storages/ProjectionsDescription.h>
 #include <Storages/StorageInMemoryMetadata.h>
+#include <Storages/System/StorageSystemTables.h>
 #include <Storages/System/getQueriedColumnsMaskAndHeader.h>
 #include <Storages/VirtualColumnUtils.h>
 
@@ -25,69 +26,6 @@ namespace DB
 namespace
 {
 
-ColumnPtr getFilteredDatabases(const ActionsDAG::Node * predicate, ContextPtr context)
-{
-    MutableColumnPtr column = ColumnString::create();
-
-    const auto databases = DatabaseCatalog::instance().getDatabases();
-    for (const auto & database_name : databases | boost::adaptors::map_keys)
-    {
-        if (database_name == DatabaseCatalog::TEMPORARY_DATABASE)
-            continue; /// We don't want to show the internal database for temporary tables in system.tables
-
-        column->insert(database_name);
-    }
-
-    Block block{ColumnWithTypeAndName(std::move(column), std::make_shared<DataTypeString>(), "database")};
-    VirtualColumnUtils::filterBlockWithPredicate(predicate, block, context);
-    return block.getByPosition(0).column;
-}
-
-ColumnPtr getFilteredTables(const ActionsDAG::Node * predicate, const ColumnPtr & filtered_databases_column, ContextPtr context)
-{
-    Block sample{
-        ColumnWithTypeAndName(nullptr, std::make_shared<DataTypeString>(), "name"),
-        ColumnWithTypeAndName(nullptr, std::make_shared<DataTypeString>(), "engine")};
-
-    MutableColumnPtr database_column = ColumnString::create();
-    MutableColumnPtr engine_column;
-
-    auto dag = VirtualColumnUtils::splitFilterDagForAllowedInputs(predicate, &sample);
-    if (dag)
-    {
-        bool filter_by_engine = false;
-        for (const auto * input : dag->getInputs())
-            if (input->result_name == "engine")
-                filter_by_engine = true;
-
-        if (filter_by_engine)
-            engine_column = ColumnString::create();
-    }
-
-    for (size_t database_idx = 0; database_idx < filtered_databases_column->size(); ++database_idx)
-    {
-        const auto & database_name = filtered_databases_column->getDataAt(database_idx).toString();
-        DatabasePtr database = DatabaseCatalog::instance().tryGetDatabase(database_name);
-        if (!database)
-            continue;
-
-        auto table_it = database->getDetachedTablesIterator(context, {}, false);
-        for (; table_it->isValid(); table_it->next())
-        {
-            database_column->insert(table_it->table());
-        }
-    }
-
-    Block block{ColumnWithTypeAndName(std::move(database_column), std::make_shared<DataTypeString>(), "name")};
-    if (engine_column)
-        block.insert(ColumnWithTypeAndName(std::move(engine_column), std::make_shared<DataTypeString>(), "engine"));
-
-    if (dag)
-        VirtualColumnUtils::filterBlockWithDAG(dag, block, context);
-
-    return block.getByPosition(0).column;
-}
-
 class DetachedTablesBlockSource : public ISource
 {
 public:
@@ -280,8 +218,8 @@ void ReadFromSystemDetachedTables::applyFilters(ActionDAGNodes added_filter_node
     if (filter_actions_dag)
         predicate = filter_actions_dag->getOutputs().at(0);
 
-    filtered_databases_column = getFilteredDatabases(predicate, context);
-    filtered_tables_column = getFilteredTables(predicate, filtered_databases_column, context);
+    filtered_databases_column = detail::getFilteredDatabases(predicate, context);
+    filtered_tables_column = detail::getFilteredTables(predicate, filtered_databases_column, context, true);
 }
 
 void ReadFromSystemDetachedTables::initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &)
diff --git a/src/Storages/System/StorageSystemTables.cpp b/src/Storages/System/StorageSystemTables.cpp
index 783b899c978..dd7f1a6fdc8 100644
--- a/src/Storages/System/StorageSystemTables.cpp
+++ b/src/Storages/System/StorageSystemTables.cpp
@@ -1,29 +1,30 @@
+#include <Storages/System/StorageSystemTables.h>
+
+#include <Access/ContextAccess.h>
 #include <Columns/ColumnString.h>
-#include <DataTypes/DataTypeString.h>
+#include <DataTypes/DataTypeArray.h>
 #include <DataTypes/DataTypeDateTime.h>
 #include <DataTypes/DataTypeNullable.h>
-#include <Storages/System/StorageSystemTables.h>
-#include <Storages/System/getQueriedColumnsMaskAndHeader.h>
-#include <Storages/MergeTree/MergeTreeData.h>
-#include <Storages/SelectQueryInfo.h>
-#include <Storages/VirtualColumnUtils.h>
-#include <Access/ContextAccess.h>
+#include <DataTypes/DataTypeString.h>
+#include <DataTypes/DataTypeUUID.h>
+#include <DataTypes/DataTypesNumber.h>
+#include <Disks/IStoragePolicy.h>
 #include <Interpreters/Context.h>
 #include <Interpreters/DatabaseCatalog.h>
 #include <Interpreters/formatWithPossiblyHidingSecrets.h>
 #include <Parsers/ASTCreateQuery.h>
 #include <Parsers/ASTSelectWithUnionQuery.h>
-#include <Common/typeid_cast.h>
-#include <Common/StringUtils.h>
-#include <DataTypes/DataTypesNumber.h>
-#include <DataTypes/DataTypeArray.h>
-#include <Disks/IStoragePolicy.h>
 #include <Processors/ISource.h>
 #include <Processors/QueryPlan/QueryPlan.h>
 #include <Processors/QueryPlan/SourceStepWithFilter.h>
 #include <QueryPipeline/Pipe.h>
 #include <QueryPipeline/QueryPipelineBuilder.h>
-#include <DataTypes/DataTypeUUID.h>
+#include <Storages/MergeTree/MergeTreeData.h>
+#include <Storages/SelectQueryInfo.h>
+#include <Storages/System/getQueriedColumnsMaskAndHeader.h>
+#include <Storages/VirtualColumnUtils.h>
+#include <Common/StringUtils.h>
+#include <Common/typeid_cast.h>
 
 #include <boost/range/adaptor/map.hpp>
 
@@ -31,6 +32,117 @@
 namespace DB
 {
 
+namespace
+{
+
+/// Avoid heavy operation on tables if we only queried columns that we can get without table object.
+/// Otherwise it will require table initialization for Lazy database.
+bool needTable(const DatabasePtr & database, const Block & header)
+{
+    if (database->getEngineName() != "Lazy")
+        return true;
+
+    static const std::set<std::string> columns_without_table = {"database", "name", "uuid", "metadata_modification_time"};
+    for (const auto & column : header.getColumnsWithTypeAndName())
+    {
+        if (columns_without_table.find(column.name) == columns_without_table.end())
+            return true;
+    }
+    return false;
+}
+
+template <typename T>
+void fillColumns(T table_it, MutableColumnPtr & database_column, MutableColumnPtr & engine_column)
+{
+    for (; table_it->isValid(); table_it->next())
+    {
+        database_column->insert(table_it->name());
+        if (engine_column)
+            engine_column->insert(table_it->table()->getName());
+    }
+}
+
+}
+
+namespace detail
+{
+ColumnPtr getFilteredDatabases(const ActionsDAG::Node * predicate, ContextPtr context)
+{
+    MutableColumnPtr column = ColumnString::create();
+
+    const auto databases = DatabaseCatalog::instance().getDatabases();
+    for (const auto & database_name : databases | boost::adaptors::map_keys)
+    {
+        if (database_name == DatabaseCatalog::TEMPORARY_DATABASE)
+            continue; /// We don't want to show the internal database for temporary tables in system.tables
+
+        column->insert(database_name);
+    }
+
+    Block block{ColumnWithTypeAndName(std::move(column), std::make_shared<DataTypeString>(), "database")};
+    VirtualColumnUtils::filterBlockWithPredicate(predicate, block, context);
+    return block.getByPosition(0).column;
+}
+
+ColumnPtr getFilteredTables(
+    const ActionsDAG::Node * predicate, const ColumnPtr & filtered_databases_column, ContextPtr context, const bool is_detached)
+{
+    Block sample{
+        ColumnWithTypeAndName(nullptr, std::make_shared<DataTypeString>(), "name"),
+        ColumnWithTypeAndName(nullptr, std::make_shared<DataTypeString>(), "engine")};
+
+    MutableColumnPtr database_column = ColumnString::create();
+    MutableColumnPtr engine_column;
+
+    auto dag = VirtualColumnUtils::splitFilterDagForAllowedInputs(predicate, &sample);
+    if (dag)
+    {
+        bool filter_by_engine = false;
+        for (const auto * input : dag->getInputs())
+            if (input->result_name == "engine")
+                filter_by_engine = true;
+
+        if (filter_by_engine)
+            engine_column = ColumnString::create();
+    }
+
+    for (size_t database_idx = 0; database_idx < filtered_databases_column->size(); ++database_idx)
+    {
+        const auto & database_name = filtered_databases_column->getDataAt(database_idx).toString();
+        DatabasePtr database = DatabaseCatalog::instance().tryGetDatabase(database_name);
+        if (!database)
+            continue;
+
+        if (is_detached)
+        {
+            auto table_it = database->getDetachedTablesIterator(context, {}, false);
+            for (; table_it->isValid(); table_it->next())
+            {
+                database_column->insert(table_it->table());
+            }
+        }
+        else
+        {
+            for (auto table_it = database->getTablesIterator(context); table_it->isValid(); table_it->next())
+            {
+                database_column->insert(table_it->name());
+                if (engine_column)
+                    engine_column->insert(table_it->table()->getName());
+            }
+        }
+    }
+
+    Block block{ColumnWithTypeAndName(std::move(database_column), std::make_shared<DataTypeString>(), "name")};
+    if (engine_column)
+        block.insert(ColumnWithTypeAndName(std::move(engine_column), std::make_shared<DataTypeString>(), "engine"));
+
+    if (dag)
+        VirtualColumnUtils::filterBlockWithDAG(dag, block, context);
+
+    return block.getByPosition(0).column;
+}
+
+}
 
 StorageSystemTables::StorageSystemTables(const StorageID & table_id_)
     : IStorage(table_id_)
@@ -105,92 +217,6 @@ StorageSystemTables::StorageSystemTables(const StorageID & table_id_)
     setInMemoryMetadata(storage_metadata);
 }
 
-
-namespace
-{
-
-ColumnPtr getFilteredDatabases(const ActionsDAG::Node * predicate, ContextPtr context)
-{
-    MutableColumnPtr column = ColumnString::create();
-
-    const auto databases = DatabaseCatalog::instance().getDatabases();
-    for (const auto & database_name : databases | boost::adaptors::map_keys)
-    {
-        if (database_name == DatabaseCatalog::TEMPORARY_DATABASE)
-            continue; /// We don't want to show the internal database for temporary tables in system.tables
-
-        column->insert(database_name);
-    }
-
-    Block block { ColumnWithTypeAndName(std::move(column), std::make_shared<DataTypeString>(), "database") };
-    VirtualColumnUtils::filterBlockWithPredicate(predicate, block, context);
-    return block.getByPosition(0).column;
-}
-
-ColumnPtr getFilteredTables(const ActionsDAG::Node * predicate, const ColumnPtr & filtered_databases_column, ContextPtr context)
-{
-    Block sample {
-        ColumnWithTypeAndName(nullptr, std::make_shared<DataTypeString>(), "name"),
-        ColumnWithTypeAndName(nullptr, std::make_shared<DataTypeString>(), "engine")
-    };
-
-    MutableColumnPtr database_column = ColumnString::create();
-    MutableColumnPtr engine_column;
-
-    auto dag = VirtualColumnUtils::splitFilterDagForAllowedInputs(predicate, &sample);
-    if (dag)
-    {
-        bool filter_by_engine = false;
-        for (const auto * input : dag->getInputs())
-            if (input->result_name == "engine")
-                filter_by_engine = true;
-
-        if (filter_by_engine)
-            engine_column = ColumnString::create();
-    }
-
-    for (size_t database_idx = 0; database_idx < filtered_databases_column->size(); ++database_idx)
-    {
-        const auto & database_name = filtered_databases_column->getDataAt(database_idx).toString();
-        DatabasePtr database = DatabaseCatalog::instance().tryGetDatabase(database_name);
-        if (!database)
-            continue;
-
-        for (auto table_it = database->getTablesIterator(context); table_it->isValid(); table_it->next())
-        {
-            database_column->insert(table_it->name());
-            if (engine_column)
-                engine_column->insert(table_it->table()->getName());
-        }
-    }
-
-    Block block {ColumnWithTypeAndName(std::move(database_column), std::make_shared<DataTypeString>(), "name")};
-    if (engine_column)
-        block.insert(ColumnWithTypeAndName(std::move(engine_column), std::make_shared<DataTypeString>(), "engine"));
-
-    if (dag)
-        VirtualColumnUtils::filterBlockWithDAG(dag, block, context);
-
-    return block.getByPosition(0).column;
-}
-
-/// Avoid heavy operation on tables if we only queried columns that we can get without table object.
-/// Otherwise it will require table initialization for Lazy database.
-bool needTable(const DatabasePtr & database, const Block & header)
-{
-    if (database->getEngineName() != "Lazy")
-        return true;
-
-    static const std::set<std::string> columns_without_table = { "database", "name", "uuid", "metadata_modification_time" };
-    for (const auto & column : header.getColumnsWithTypeAndName())
-    {
-        if (columns_without_table.find(column.name) == columns_without_table.end())
-            return true;
-    }
-    return false;
-}
-
-
 class TablesBlockSource : public ISource
 {
 public:
@@ -690,8 +716,6 @@ private:
     std::string database_name;
 };
 
-}
-
 class ReadFromSystemTables : public SourceStepWithFilter
 {
 public:
@@ -756,8 +780,8 @@ void ReadFromSystemTables::applyFilters(ActionDAGNodes added_filter_nodes)
     if (filter_actions_dag)
         predicate = filter_actions_dag->getOutputs().at(0);
 
-    filtered_databases_column = getFilteredDatabases(predicate, context);
-    filtered_tables_column = getFilteredTables(predicate, filtered_databases_column, context);
+    filtered_databases_column = detail::getFilteredDatabases(predicate, context);
+    filtered_tables_column = detail::getFilteredTables(predicate, filtered_databases_column, context, false);
 }
 
 void ReadFromSystemTables::initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &)
diff --git a/src/Storages/System/StorageSystemTables.h b/src/Storages/System/StorageSystemTables.h
index 87cdf1b6a8e..d6e3996b8e3 100644
--- a/src/Storages/System/StorageSystemTables.h
+++ b/src/Storages/System/StorageSystemTables.h
@@ -8,6 +8,15 @@ namespace DB
 
 class Context;
 
+namespace detail
+{
+
+ColumnPtr getFilteredDatabases(const ActionsDAG::Node * predicate, ContextPtr context);
+ColumnPtr
+getFilteredTables(const ActionsDAG::Node * predicate, const ColumnPtr & filtered_databases_column, ContextPtr context, bool is_detached);
+
+}
+
 
 /** Implements the system table `tables`, which allows you to get information about all tables.
   */

From c998ec1e4f1b91f8ca20c2bd5a7acb6ac8d2e1b1 Mon Sep 17 00:00:00 2001
From: jsc0218 <jsc0218@gmail.com>
Date: Sat, 29 Jun 2024 02:40:22 +0000
Subject: [PATCH 0116/1488] add test and better naming

---
 src/Access/Common/AccessRightsElement.cpp     |  4 +-
 src/Access/Common/AccessRightsElement.h       |  2 +-
 src/Access/ContextAccess.cpp                  | 10 +--
 ...xml => config_with_table_engine_grant.xml} |  0
 .../config_without_table_engine_grant.xml     |  5 ++
 ...est.py => test_with_table_engine_grant.py} |  2 +-
 .../test_without_table_engine_grant.py        | 81 +++++++++++++++++++
 7 files changed, 95 insertions(+), 9 deletions(-)
 rename tests/integration/test_grant_and_revoke/configs/{config.xml => config_with_table_engine_grant.xml} (100%)
 create mode 100644 tests/integration/test_grant_and_revoke/configs/config_without_table_engine_grant.xml
 rename tests/integration/test_grant_and_revoke/{test.py => test_with_table_engine_grant.py} (99%)
 create mode 100644 tests/integration/test_grant_and_revoke/test_without_table_engine_grant.py

diff --git a/src/Access/Common/AccessRightsElement.cpp b/src/Access/Common/AccessRightsElement.cpp
index 2ee13d6b94f..63bda09a51b 100644
--- a/src/Access/Common/AccessRightsElement.cpp
+++ b/src/Access/Common/AccessRightsElement.cpp
@@ -224,10 +224,10 @@ void AccessRightsElement::replaceEmptyDatabase(const String & current_database)
 
 String AccessRightsElement::toString() const { return toStringImpl(*this, true); }
 String AccessRightsElement::toStringWithoutOptions() const { return toStringImpl(*this, false); }
-String AccessRightsElement::toStringWithoutONClause() const
+String AccessRightsElement::toStringForAccessTypeSource() const
 {
     String result{access_flags.toKeywords().front()};
-    return result + " ON {db.table}";
+    return result + " ON *.*";
 }
 
 bool AccessRightsElements::empty() const { return std::all_of(begin(), end(), [](const AccessRightsElement & e) { return e.empty(); }); }
diff --git a/src/Access/Common/AccessRightsElement.h b/src/Access/Common/AccessRightsElement.h
index 49764fc727f..78e94e6f2e4 100644
--- a/src/Access/Common/AccessRightsElement.h
+++ b/src/Access/Common/AccessRightsElement.h
@@ -89,7 +89,7 @@ struct AccessRightsElement
     /// Returns a human-readable representation like "GRANT SELECT, UPDATE(x, y) ON db.table".
     String toString() const;
     String toStringWithoutOptions() const;
-    String toStringWithoutONClause() const;
+    String toStringForAccessTypeSource() const;
 };
 
 
diff --git a/src/Access/ContextAccess.cpp b/src/Access/ContextAccess.cpp
index 4620561053b..8ff1fc8ed21 100644
--- a/src/Access/ContextAccess.cpp
+++ b/src/Access/ContextAccess.cpp
@@ -622,7 +622,7 @@ bool ContextAccess::checkAccessImplHelper(AccessFlags flags, const Args &... arg
         /// since SOURCES is not granted actually. In order to solve this, turn the prompt logic back to Sources.
         if (flags & AccessType::TABLE_ENGINE && !access_control->doesTableEnginesRequireGrant())
         {
-            AccessFlags newFlags;
+            AccessFlags new_flags;
 
             String table_engine_name{getTableEngine(args...)};
             for (const auto & source_and_table_engine : source_and_table_engines)
@@ -631,11 +631,11 @@ bool ContextAccess::checkAccessImplHelper(AccessFlags flags, const Args &... arg
                 if (table_engine != table_engine_name) continue;
                 const auto & source = std::get<0>(source_and_table_engine);
                 /// Set the flags from Table Engine to SOURCES so that prompts can be meaningful.
-                newFlags = source;
+                new_flags = source;
                 break;
             }
 
-            if (newFlags.isEmpty())
+            if (new_flags.isEmpty())
                 throw Exception(ErrorCodes::LOGICAL_ERROR, "Didn't find the target Source from the Table Engine");
 
             if (grant_option && acs->isGranted(flags, args...))
@@ -644,12 +644,12 @@ bool ContextAccess::checkAccessImplHelper(AccessFlags flags, const Args &... arg
                     "{}: Not enough privileges. "
                     "The required privileges have been granted, but without grant option. "
                     "To execute this query, it's necessary to have the grant {} WITH GRANT OPTION",
-                    AccessRightsElement{newFlags}.toStringWithoutONClause());
+                    AccessRightsElement{new_flags}.toStringForAccessTypeSource());
             }
 
             return access_denied(ErrorCodes::ACCESS_DENIED,
                 "{}: Not enough privileges. To execute this query, it's necessary to have the grant {}",
-                AccessRightsElement{newFlags}.toStringWithoutONClause() + (grant_option ? " WITH GRANT OPTION" : ""));
+                AccessRightsElement{new_flags}.toStringForAccessTypeSource() + (grant_option ? " WITH GRANT OPTION" : ""));
         }
 
         if (grant_option && acs->isGranted(flags, args...))
diff --git a/tests/integration/test_grant_and_revoke/configs/config.xml b/tests/integration/test_grant_and_revoke/configs/config_with_table_engine_grant.xml
similarity index 100%
rename from tests/integration/test_grant_and_revoke/configs/config.xml
rename to tests/integration/test_grant_and_revoke/configs/config_with_table_engine_grant.xml
diff --git a/tests/integration/test_grant_and_revoke/configs/config_without_table_engine_grant.xml b/tests/integration/test_grant_and_revoke/configs/config_without_table_engine_grant.xml
new file mode 100644
index 00000000000..d3571f281f5
--- /dev/null
+++ b/tests/integration/test_grant_and_revoke/configs/config_without_table_engine_grant.xml
@@ -0,0 +1,5 @@
+<clickhouse>
+    <access_control_improvements>
+        <table_engines_require_grant>false</table_engines_require_grant>
+    </access_control_improvements>
+</clickhouse>
diff --git a/tests/integration/test_grant_and_revoke/test.py b/tests/integration/test_grant_and_revoke/test_with_table_engine_grant.py
similarity index 99%
rename from tests/integration/test_grant_and_revoke/test.py
rename to tests/integration/test_grant_and_revoke/test_with_table_engine_grant.py
index e533cced1e4..25ca7913e4e 100644
--- a/tests/integration/test_grant_and_revoke/test.py
+++ b/tests/integration/test_grant_and_revoke/test_with_table_engine_grant.py
@@ -5,7 +5,7 @@ from helpers.test_tools import TSV
 cluster = ClickHouseCluster(__file__)
 instance = cluster.add_instance(
     "instance",
-    main_configs=["configs/config.xml"],
+    main_configs=["configs/config_with_table_engine_grant.xml"],
     user_configs=["configs/users.d/users.xml"],
 )
 
diff --git a/tests/integration/test_grant_and_revoke/test_without_table_engine_grant.py b/tests/integration/test_grant_and_revoke/test_without_table_engine_grant.py
new file mode 100644
index 00000000000..210bb8ec465
--- /dev/null
+++ b/tests/integration/test_grant_and_revoke/test_without_table_engine_grant.py
@@ -0,0 +1,81 @@
+import pytest
+from helpers.cluster import ClickHouseCluster
+from helpers.test_tools import TSV
+
+cluster = ClickHouseCluster(__file__)
+instance = cluster.add_instance(
+    "instance",
+    main_configs=["configs/config_without_table_engine_grant.xml"],
+    user_configs=["configs/users.d/users.xml"],
+)
+
+
+@pytest.fixture(scope="module", autouse=True)
+def start_cluster():
+    try:
+        cluster.start()
+
+        instance.query("CREATE DATABASE test")
+
+        yield cluster
+
+    finally:
+        cluster.shutdown()
+
+
+@pytest.fixture(autouse=True)
+def cleanup_after_test():
+    try:
+        yield
+    finally:
+        instance.query("DROP USER IF EXISTS A")
+        instance.query("DROP TABLE IF EXISTS test.table1")
+
+
+def test_table_engine_and_source_grant():
+    instance.query("DROP USER IF EXISTS A")
+    instance.query("CREATE USER A")
+    instance.query("GRANT CREATE TABLE ON test.table1 TO A")
+
+    instance.query("GRANT POSTGRES ON *.* TO A")
+
+    instance.query(
+        """
+        CREATE TABLE test.table1(a Integer)
+        engine=PostgreSQL('localhost:5432', 'dummy', 'dummy', 'dummy', 'dummy');
+        """,
+        user="A",
+    )
+
+    instance.query("DROP TABLE test.table1")
+
+    instance.query("REVOKE POSTGRES ON *.* FROM A")
+
+    assert "Not enough privileges" in instance.query_and_get_error(
+        """
+        CREATE TABLE test.table1(a Integer)
+        engine=PostgreSQL('localhost:5432', 'dummy', 'dummy', 'dummy', 'dummy');
+        """,
+        user="A",
+    )
+
+    # expecting grant POSTGRES instead of grant PostgreSQL due to discrepancy between source access type and table engine
+    assert "grant POSTGRES ON *.*" in instance.query_and_get_error(
+        """
+        CREATE TABLE test.table1(a Integer)
+        engine=PostgreSQL('localhost:5432', 'dummy', 'dummy', 'dummy', 'dummy');
+        """,
+        user="A",
+    )
+
+    instance.query("GRANT SOURCES ON *.* TO A")
+
+    instance.query(
+        """
+        CREATE TABLE test.table1(a Integer)
+        engine=PostgreSQL('localhost:5432', 'dummy', 'dummy', 'dummy', 'dummy');
+        """,
+        user="A",
+    )
+
+    instance.query("DROP TABLE test.table1")

From 02928bb207348019a0968454e0f7d6baa6e511af Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 30 Jun 2024 05:09:53 +0200
Subject: [PATCH 0117/1488] Use Atomic database by default in clickhouse-local

---
 programs/local/LocalServer.cpp     | 12 ++++++------
 src/Databases/DatabaseAtomic.cpp   | 14 +++++++++++---
 src/Databases/DatabaseAtomic.h     |  1 +
 src/Databases/DatabaseOnDisk.cpp   | 12 ++++++++++--
 src/Databases/DatabaseOnDisk.h     |  4 +++-
 src/Databases/DatabaseOrdinary.cpp |  2 +-
 6 files changed, 32 insertions(+), 13 deletions(-)

diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp
index 670dc378b97..27fe90f68cc 100644
--- a/programs/local/LocalServer.cpp
+++ b/programs/local/LocalServer.cpp
@@ -13,6 +13,7 @@
 #include <Databases/registerDatabases.h>
 #include <Databases/DatabaseFilesystem.h>
 #include <Databases/DatabaseMemory.h>
+#include <Databases/DatabaseAtomic.h>
 #include <Databases/DatabasesOverlay.h>
 #include <Storages/System/attachSystemTables.h>
 #include <Storages/System/attachInformationSchemaTables.h>
@@ -48,7 +49,6 @@
 #include <Dictionaries/registerDictionaries.h>
 #include <Disks/registerDisks.h>
 #include <Formats/registerFormats.h>
-#include <boost/algorithm/string/replace.hpp>
 #include <boost/program_options/options_description.hpp>
 #include <base/argsToConfig.h>
 #include <filesystem>
@@ -192,11 +192,11 @@ static DatabasePtr createMemoryDatabaseIfNotExists(ContextPtr context, const Str
     return system_database;
 }
 
-static DatabasePtr createClickHouseLocalDatabaseOverlay(const String & name_, ContextPtr context_)
+static DatabasePtr createClickHouseLocalDatabaseOverlay(const String & name_, ContextPtr context)
 {
-    auto databaseCombiner = std::make_shared<DatabasesOverlay>(name_, context_);
-    databaseCombiner->registerNextDatabase(std::make_shared<DatabaseFilesystem>(name_, "", context_));
-    databaseCombiner->registerNextDatabase(std::make_shared<DatabaseMemory>(name_, context_));
+    auto databaseCombiner = std::make_shared<DatabasesOverlay>(name_, context);
+    databaseCombiner->registerNextDatabase(std::make_shared<DatabaseFilesystem>(name_, "", context));
+    databaseCombiner->registerNextDatabase(std::make_shared<DatabaseAtomic>(name_, fs::weakly_canonical(context->getPath()), UUIDHelpers::generateV4(), context));
     return databaseCombiner;
 }
 
@@ -341,7 +341,7 @@ std::string LocalServer::getInitialCreateTableQuery()
     else
         table_structure = "(" + table_structure + ")";
 
-    return fmt::format("CREATE TABLE {} {} ENGINE = File({}, {});",
+    return fmt::format("CREATE TEMPORARY TABLE {} {} ENGINE = File({}, {});",
                        table_name, table_structure, data_format, table_file);
 }
 
diff --git a/src/Databases/DatabaseAtomic.cpp b/src/Databases/DatabaseAtomic.cpp
index ccab72cfbae..985d79773aa 100644
--- a/src/Databases/DatabaseAtomic.cpp
+++ b/src/Databases/DatabaseAtomic.cpp
@@ -49,9 +49,6 @@ DatabaseAtomic::DatabaseAtomic(String name_, String metadata_path_, UUID uuid, c
     , db_uuid(uuid)
 {
     assert(db_uuid != UUIDHelpers::Nil);
-    fs::create_directories(fs::path(getContext()->getPath()) / "metadata");
-    fs::create_directories(path_to_table_symlinks);
-    tryCreateMetadataSymlink();
 }
 
 DatabaseAtomic::DatabaseAtomic(String name_, String metadata_path_, UUID uuid, ContextPtr context_)
@@ -59,6 +56,13 @@ DatabaseAtomic::DatabaseAtomic(String name_, String metadata_path_, UUID uuid, C
 {
 }
 
+void DatabaseAtomic::createDirectories()
+{
+    fs::create_directories(fs::path(getContext()->getPath()) / "metadata");
+    fs::create_directories(path_to_table_symlinks);
+    tryCreateMetadataSymlink();
+}
+
 String DatabaseAtomic::getTableDataPath(const String & table_name) const
 {
     std::lock_guard lock(mutex);
@@ -95,6 +99,7 @@ void DatabaseAtomic::drop(ContextPtr)
 void DatabaseAtomic::attachTable(ContextPtr /* context_ */, const String & name, const StoragePtr & table, const String & relative_table_path)
 {
     assert(relative_table_path != data_path && !relative_table_path.empty());
+    createDirectories();
     DetachedTables not_in_use;
     std::lock_guard lock(mutex);
     not_in_use = cleanupDetachedTables();
@@ -309,6 +314,7 @@ void DatabaseAtomic::commitCreateTable(const ASTCreateQuery & query, const Stora
                                        const String & table_metadata_tmp_path, const String & table_metadata_path,
                                        ContextPtr query_context)
 {
+    createDirectories();
     DetachedTables not_in_use;
     auto table_data_path = getTableDataPath(query);
     try
@@ -572,6 +578,7 @@ void DatabaseAtomic::renameDatabase(ContextPtr query_context, const String & new
 {
     /// CREATE, ATTACH, DROP, DETACH and RENAME DATABASE must hold DDLGuard
 
+    createDirectories();
     waitDatabaseStarted();
 
     bool check_ref_deps = query_context->getSettingsRef().check_referential_table_dependencies;
@@ -663,4 +670,5 @@ void registerDatabaseAtomic(DatabaseFactory & factory)
     };
     factory.registerDatabase("Atomic", create_fn);
 }
+
 }
diff --git a/src/Databases/DatabaseAtomic.h b/src/Databases/DatabaseAtomic.h
index b59edd479ba..26ab7657354 100644
--- a/src/Databases/DatabaseAtomic.h
+++ b/src/Databases/DatabaseAtomic.h
@@ -75,6 +75,7 @@ protected:
     using DetachedTables = std::unordered_map<UUID, StoragePtr>;
     [[nodiscard]] DetachedTables cleanupDetachedTables() TSA_REQUIRES(mutex);
 
+    void createDirectories();
     void tryCreateMetadataSymlink();
 
     virtual bool allowMoveTableToOtherDatabaseEngine(IDatabase & /*to_database*/) const { return false; }
diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp
index b8154372116..3a56c124726 100644
--- a/src/Databases/DatabaseOnDisk.cpp
+++ b/src/Databases/DatabaseOnDisk.cpp
@@ -170,7 +170,12 @@ DatabaseOnDisk::DatabaseOnDisk(
     , metadata_path(metadata_path_)
     , data_path(data_path_)
 {
-    fs::create_directories(local_context->getPath() + data_path);
+}
+
+
+void DatabaseOnDisk::createDirectories()
+{
+    fs::create_directories(std::filesystem::path(getContext()->getPath()) / data_path);
     fs::create_directories(metadata_path);
 }
 
@@ -188,6 +193,8 @@ void DatabaseOnDisk::createTable(
     const StoragePtr & table,
     const ASTPtr & query)
 {
+    createDirectories();
+
     const auto & settings = local_context->getSettingsRef();
     const auto & create = query->as<ASTCreateQuery &>();
     assert(table_name == create.getTable());
@@ -255,7 +262,6 @@ void DatabaseOnDisk::createTable(
     }
 
     commitCreateTable(create, table, table_metadata_tmp_path, table_metadata_path, local_context);
-
     removeDetachedPermanentlyFlag(local_context, table_name, table_metadata_path, false);
 }
 
@@ -283,6 +289,8 @@ void DatabaseOnDisk::commitCreateTable(const ASTCreateQuery & query, const Stora
 {
     try
     {
+        createDirectories();
+
         /// Add a table to the map of known tables.
         attachTable(query_context, query.getTable(), table, getTableDataPath(query));
 
diff --git a/src/Databases/DatabaseOnDisk.h b/src/Databases/DatabaseOnDisk.h
index 12656068643..00e7a2850b8 100644
--- a/src/Databases/DatabaseOnDisk.h
+++ b/src/Databases/DatabaseOnDisk.h
@@ -64,7 +64,7 @@ public:
     time_t getObjectMetadataModificationTime(const String & object_name) const override;
 
     String getDataPath() const override { return data_path; }
-    String getTableDataPath(const String & table_name) const override { return data_path + escapeForFileName(table_name) + "/"; }
+    String getTableDataPath(const String & table_name) const override { return std::filesystem::path(data_path) / escapeForFileName(table_name) / ""; }
     String getTableDataPath(const ASTCreateQuery & query) const override { return getTableDataPath(query.getTable()); }
     String getMetadataPath() const override { return metadata_path; }
 
@@ -99,6 +99,8 @@ protected:
     virtual void removeDetachedPermanentlyFlag(ContextPtr context, const String & table_name, const String & table_metadata_path, bool attach);
     virtual void setDetachedTableNotInUseForce(const UUID & /*uuid*/) {}
 
+    void createDirectories();
+
     const String metadata_path;
     const String data_path;
 };
diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp
index 7d4bb07e8ef..5b5c09b039d 100644
--- a/src/Databases/DatabaseOrdinary.cpp
+++ b/src/Databases/DatabaseOrdinary.cpp
@@ -52,7 +52,7 @@ static constexpr size_t METADATA_FILE_BUFFER_SIZE = 32768;
 static constexpr const char * const CONVERT_TO_REPLICATED_FLAG_NAME = "convert_to_replicated";
 
 DatabaseOrdinary::DatabaseOrdinary(const String & name_, const String & metadata_path_, ContextPtr context_)
-    : DatabaseOrdinary(name_, metadata_path_, "data/" + escapeForFileName(name_) + "/", "DatabaseOrdinary (" + name_ + ")", context_)
+    : DatabaseOrdinary(name_, metadata_path_, std::filesystem::path("data") / escapeForFileName(name_) / "", "DatabaseOrdinary (" + name_ + ")", context_)
 {
 }
 

From 488767e601e95cdf81b30b34c70e4e4ba18510e0 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 1 Jul 2024 02:38:42 +0200
Subject: [PATCH 0118/1488] Uninteresting changes

---
 programs/local/LocalServer.cpp |  1 +
 src/Client/LocalConnection.h   |  1 +
 src/Interpreters/Squashing.cpp | 96 ++++++++++++++++++----------------
 src/Interpreters/Squashing.h   | 10 +---
 4 files changed, 54 insertions(+), 54 deletions(-)

diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp
index b33e1595056..310c366b023 100644
--- a/programs/local/LocalServer.cpp
+++ b/programs/local/LocalServer.cpp
@@ -422,6 +422,7 @@ void LocalServer::connect()
 {
     connection_parameters = ConnectionParameters(getClientConfiguration(), "localhost");
 
+    /// This is needed for table function input(...).
     ReadBuffer * in;
     auto table_file = getClientConfiguration().getString("table-file", "-");
     if (table_file == "-" || table_file == "stdin")
diff --git a/src/Client/LocalConnection.h b/src/Client/LocalConnection.h
index fb6fa1b55eb..b424c5b5aa3 100644
--- a/src/Client/LocalConnection.h
+++ b/src/Client/LocalConnection.h
@@ -175,4 +175,5 @@ private:
 
     ReadBuffer * in;
 };
+
 }
diff --git a/src/Interpreters/Squashing.cpp b/src/Interpreters/Squashing.cpp
index f8b6a6542cc..858debacf0c 100644
--- a/src/Interpreters/Squashing.cpp
+++ b/src/Interpreters/Squashing.cpp
@@ -5,11 +5,60 @@
 
 namespace DB
 {
+
 namespace ErrorCodes
 {
     extern const int LOGICAL_ERROR;
 }
 
+namespace
+{
+
+Chunk squashImpl(std::vector<Chunk> & input_chunks)
+{
+    Chunk accumulated_chunk;
+    std::vector<IColumn::MutablePtr> mutable_columns = {};
+    size_t rows = 0;
+    for (const Chunk & chunk : input_chunks)
+        rows += chunk.getNumRows();
+
+    {
+        auto & first_chunk = input_chunks[0];
+        Columns columns = first_chunk.detachColumns();
+        for (auto & column : columns)
+        {
+            mutable_columns.push_back(IColumn::mutate(std::move(column)));
+            mutable_columns.back()->reserve(rows);
+        }
+    }
+
+    for (size_t i = 1; i < input_chunks.size(); ++i) // We've already processed the first chunk above
+    {
+        Columns columns = input_chunks[i].detachColumns();
+        for (size_t j = 0, size = mutable_columns.size(); j < size; ++j)
+        {
+            const auto source_column = columns[j];
+
+            mutable_columns[j]->insertRangeFrom(*source_column, 0, source_column->size());
+        }
+    }
+    accumulated_chunk.setColumns(std::move(mutable_columns), rows);
+    return accumulated_chunk;
+}
+
+const ChunksToSquash * getInfoFromChunk(const Chunk & chunk)
+{
+    const auto & info = chunk.getChunkInfo();
+    const auto * agg_info = typeid_cast<const ChunksToSquash *>(info.get());
+
+    if (!agg_info)
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "There is no ChunksToSquash in ChunkInfoPtr");
+
+    return agg_info;
+}
+
+}
+
 Squashing::Squashing(Block header_, size_t min_block_size_rows_, size_t min_block_size_bytes_)
     : header(header_)
     , min_block_size_rows(min_block_size_rows_)
@@ -27,8 +76,8 @@ Chunk Squashing::squash(Chunk && input_chunk)
     if (!input_chunk.hasChunkInfo())
         return Chunk();
 
-    const auto *info = getInfoFromChunk(input_chunk);
-    return squash(info->chunks);
+    const auto * info = getInfoFromChunk(input_chunk);
+    return squashImpl(info->chunks);
 }
 
 Chunk Squashing::add(Chunk && input_chunk)
@@ -95,49 +144,6 @@ Chunk Squashing::convertToChunk(std::vector<Chunk> && chunks) const
     return Chunk(header.cloneEmptyColumns(), 0, info);
 }
 
-Chunk Squashing::squash(std::vector<Chunk> & input_chunks)
-{
-    Chunk accumulated_chunk;
-    std::vector<IColumn::MutablePtr> mutable_columns = {};
-    size_t rows = 0;
-    for (const Chunk & chunk : input_chunks)
-        rows += chunk.getNumRows();
-
-    {
-        auto & first_chunk = input_chunks[0];
-        Columns columns = first_chunk.detachColumns();
-        for (auto & column : columns)
-        {
-            mutable_columns.push_back(IColumn::mutate(std::move(column)));
-            mutable_columns.back()->reserve(rows);
-        }
-    }
-
-    for (size_t i = 1; i < input_chunks.size(); ++i) // We've already processed the first chunk above
-    {
-        Columns columns = input_chunks[i].detachColumns();
-        for (size_t j = 0, size = mutable_columns.size(); j < size; ++j)
-        {
-            const auto source_column = columns[j];
-
-            mutable_columns[j]->insertRangeFrom(*source_column, 0, source_column->size());
-        }
-    }
-    accumulated_chunk.setColumns(std::move(mutable_columns), rows);
-    return accumulated_chunk;
-}
-
-const ChunksToSquash* Squashing::getInfoFromChunk(const Chunk & chunk)
-{
-    const auto& info = chunk.getChunkInfo();
-    const auto * agg_info = typeid_cast<const ChunksToSquash *>(info.get());
-
-    if (!agg_info)
-        throw Exception(ErrorCodes::LOGICAL_ERROR, "There is no ChunksToSquash in ChunkInfoPtr");
-
-    return agg_info;
-}
-
 void Squashing::expandCurrentSize(size_t rows, size_t bytes)
 {
     accumulated_size.rows += rows;
diff --git a/src/Interpreters/Squashing.h b/src/Interpreters/Squashing.h
index d76cca60e41..bb4cd7bed14 100644
--- a/src/Interpreters/Squashing.h
+++ b/src/Interpreters/Squashing.h
@@ -36,12 +36,8 @@ public:
     static Chunk squash(Chunk && input_chunk);
     Chunk flush();
 
-    bool isDataLeft()
-    {
-        return !chunks_to_merge_vec.empty();
-    }
-
     Block header;
+
 private:
     struct CurrentSize
     {
@@ -55,10 +51,6 @@ private:
 
     CurrentSize accumulated_size;
 
-    static const ChunksToSquash * getInfoFromChunk(const Chunk & chunk);
-
-    static Chunk squash(std::vector<Chunk> & input_chunks);
-
     void expandCurrentSize(size_t rows, size_t bytes);
     void changeCurrentSize(size_t rows, size_t bytes);
     bool isEnoughSize(size_t rows, size_t bytes) const;

From eea2d51cfe908767d6c5602a5c8f3137c757065b Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 1 Jul 2024 03:07:10 +0200
Subject: [PATCH 0119/1488] Fix fast test

---
 src/Databases/DatabaseAtomic.cpp | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/Databases/DatabaseAtomic.cpp b/src/Databases/DatabaseAtomic.cpp
index 985d79773aa..c3e93a8a324 100644
--- a/src/Databases/DatabaseAtomic.cpp
+++ b/src/Databases/DatabaseAtomic.cpp
@@ -451,6 +451,9 @@ void DatabaseAtomic::beforeLoadingMetadata(ContextMutablePtr /*context*/, Loadin
     if (mode < LoadingStrictnessLevel::FORCE_RESTORE)
         return;
 
+    if (!fs::exists(path_to_table_symlinks))
+        return;
+
     /// Recreate symlinks to table data dirs in case of force restore, because some of them may be broken
     for (const auto & table_path : fs::directory_iterator(path_to_table_symlinks))
     {

From 79fc80a3ea25309ca7f1fe7cf61d1be56526cdbc Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 1 Jul 2024 03:57:48 +0200
Subject: [PATCH 0120/1488] Fix error

---
 src/Databases/DatabaseOnDisk.cpp | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp
index 3a56c124726..832769fd043 100644
--- a/src/Databases/DatabaseOnDisk.cpp
+++ b/src/Databases/DatabaseOnDisk.cpp
@@ -611,6 +611,10 @@ time_t DatabaseOnDisk::getObjectMetadataModificationTime(const String & object_n
 
 void DatabaseOnDisk::iterateMetadataFiles(ContextPtr local_context, const IteratingFunction & process_metadata_file) const
 {
+    fs::path metadata_path = getMetadataPath();
+    if (!fs::exists(metadata_path))
+        return;
+
     auto process_tmp_drop_metadata_file = [&](const String & file_name)
     {
         assert(getUUID() == UUIDHelpers::Nil);
@@ -634,7 +638,7 @@ void DatabaseOnDisk::iterateMetadataFiles(ContextPtr local_context, const Iterat
     std::vector<std::pair<String, bool>> metadata_files;
 
     fs::directory_iterator dir_end;
-    for (fs::directory_iterator dir_it(getMetadataPath()); dir_it != dir_end; ++dir_it)
+    for (fs::directory_iterator dir_it(metadata_path); dir_it != dir_end; ++dir_it)
     {
         String file_name = dir_it->path().filename();
         /// For '.svn', '.gitignore' directory and similar.

From 4c9238a1dd9433bac7de920b6c0955ecd53a3df9 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 1 Jul 2024 03:58:27 +0200
Subject: [PATCH 0121/1488] Fix error

---
 src/Databases/DatabaseOnDisk.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp
index 832769fd043..c6b4c38a656 100644
--- a/src/Databases/DatabaseOnDisk.cpp
+++ b/src/Databases/DatabaseOnDisk.cpp
@@ -611,7 +611,6 @@ time_t DatabaseOnDisk::getObjectMetadataModificationTime(const String & object_n
 
 void DatabaseOnDisk::iterateMetadataFiles(ContextPtr local_context, const IteratingFunction & process_metadata_file) const
 {
-    fs::path metadata_path = getMetadataPath();
     if (!fs::exists(metadata_path))
         return;
 

From 2b6a47c13ee054ddf63db41445aab8a0b7187340 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 1 Jul 2024 06:10:13 +0200
Subject: [PATCH 0122/1488] Fix errors

---
 src/Databases/DatabaseAtomic.cpp | 1 +
 src/Databases/DatabaseOnDisk.cpp | 1 +
 2 files changed, 2 insertions(+)

diff --git a/src/Databases/DatabaseAtomic.cpp b/src/Databases/DatabaseAtomic.cpp
index c3e93a8a324..bebb645670b 100644
--- a/src/Databases/DatabaseAtomic.cpp
+++ b/src/Databases/DatabaseAtomic.cpp
@@ -189,6 +189,7 @@ void DatabaseAtomic::renameTable(ContextPtr local_context, const String & table_
     if (exchange && !supportsAtomicRename())
         throw Exception(ErrorCodes::NOT_IMPLEMENTED, "RENAME EXCHANGE is not supported");
 
+    createDirectories();
     waitDatabaseStarted();
 
     auto & other_db = dynamic_cast<DatabaseAtomic &>(to_database);
diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp
index c6b4c38a656..faac4b23701 100644
--- a/src/Databases/DatabaseOnDisk.cpp
+++ b/src/Databases/DatabaseOnDisk.cpp
@@ -416,6 +416,7 @@ void DatabaseOnDisk::renameTable(
             throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Moving tables between databases of different engines is not supported");
     }
 
+    createDirectories();
     waitDatabaseStarted();
 
     auto table_data_relative_path = getTableDataPath(table_name);

From ed68a29c8b3e64fa42ff8ea4f445789f72d39d46 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 1 Jul 2024 06:34:33 +0200
Subject: [PATCH 0123/1488] Fix error

---
 src/Databases/DatabasesOverlay.cpp | 12 ++++++++++++
 src/Databases/DatabasesOverlay.h   |  1 +
 2 files changed, 13 insertions(+)

diff --git a/src/Databases/DatabasesOverlay.cpp b/src/Databases/DatabasesOverlay.cpp
index 801356b3dd7..02a0aab8230 100644
--- a/src/Databases/DatabasesOverlay.cpp
+++ b/src/Databases/DatabasesOverlay.cpp
@@ -178,6 +178,18 @@ String DatabasesOverlay::getTableDataPath(const ASTCreateQuery & query) const
     return result;
 }
 
+UUID DatabasesOverlay::getUUID() const
+{
+    UUID result = UUIDHelpers::Nil;
+    for (const auto & db : databases)
+    {
+        result = db->getUUID();
+        if (result != UUIDHelpers::Nil)
+            break;
+    }
+    return result;
+}
+
 UUID DatabasesOverlay::tryGetTableUUID(const String & table_name) const
 {
     UUID result = UUIDHelpers::Nil;
diff --git a/src/Databases/DatabasesOverlay.h b/src/Databases/DatabasesOverlay.h
index b0c7e7e4032..5f6d4e601d3 100644
--- a/src/Databases/DatabasesOverlay.h
+++ b/src/Databases/DatabasesOverlay.h
@@ -41,6 +41,7 @@ public:
     String getTableDataPath(const String & table_name) const override;
     String getTableDataPath(const ASTCreateQuery & query) const override;
 
+    UUID getUUID() const override;
     UUID tryGetTableUUID(const String & table_name) const override;
 
     void drop(ContextPtr context) override;

From 229ff9af6e1c89b8d6c3e0f29945d0baccde273a Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 1 Jul 2024 06:34:54 +0200
Subject: [PATCH 0124/1488] Remove old comment

---
 src/Interpreters/StorageID.h | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/Interpreters/StorageID.h b/src/Interpreters/StorageID.h
index f9afbc7b98d..ad55d16e284 100644
--- a/src/Interpreters/StorageID.h
+++ b/src/Interpreters/StorageID.h
@@ -27,7 +27,6 @@ class ASTQueryWithTableAndOutput;
 class ASTTableIdentifier;
 class Context;
 
-// TODO(ilezhankin): refactor and merge |ASTTableIdentifier|
 struct StorageID
 {
     String database_name;

From 7f528cc1a031616bd7965a50ff98c2f662fba581 Mon Sep 17 00:00:00 2001
From: Yakov Olkhovskiy <yakov@clickhouse.com>
Date: Mon, 1 Jul 2024 06:02:41 +0000
Subject: [PATCH 0125/1488] fix client, fix tests

---
 programs/client/Client.cpp                    |  13 ++-
 .../certs/ca-cert.pem                         |  32 ++++++
 .../certs/ca-key.pem                          |  52 +++++++++
 .../certs/client-cert.pem                     |  30 +++++
 .../certs/client-key.pem                      |  52 +++++++++
 .../certs/client-req.pem                      |  27 +++++
 .../certs/generate_certs.sh                   |  13 +++
 .../certs/self-cert.pem                       |  32 ++++++
 .../certs/self-key.pem                        |  52 +++++++++
 .../configs/ssl_config.xml                    |  17 +++
 .../test_accept_invalid_certificate/test.py   |  93 +++++++++++++++
 ...9_clickhouse_accept_invalid_certificate.sh | 107 ------------------
 .../01889_clickhouse_client_config_format.sh  |  33 +++---
 .../02550_client_connections_credentials.sh   |  15 +--
 14 files changed, 429 insertions(+), 139 deletions(-)
 create mode 100644 tests/integration/test_accept_invalid_certificate/certs/ca-cert.pem
 create mode 100644 tests/integration/test_accept_invalid_certificate/certs/ca-key.pem
 create mode 100644 tests/integration/test_accept_invalid_certificate/certs/client-cert.pem
 create mode 100644 tests/integration/test_accept_invalid_certificate/certs/client-key.pem
 create mode 100644 tests/integration/test_accept_invalid_certificate/certs/client-req.pem
 create mode 100755 tests/integration/test_accept_invalid_certificate/certs/generate_certs.sh
 create mode 100644 tests/integration/test_accept_invalid_certificate/certs/self-cert.pem
 create mode 100644 tests/integration/test_accept_invalid_certificate/certs/self-key.pem
 create mode 100644 tests/integration/test_accept_invalid_certificate/configs/ssl_config.xml
 create mode 100644 tests/integration/test_accept_invalid_certificate/test.py
 delete mode 100644 tests/queries/0_stateless/01189_clickhouse_accept_invalid_certificate.sh

diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp
index f5e86b756fb..a7689bf7a79 100644
--- a/programs/client/Client.cpp
+++ b/programs/client/Client.cpp
@@ -178,9 +178,8 @@ void Client::parseConnectionsCredentials(Poco::Util::AbstractConfiguration & con
                 history_file = home_path + "/" + history_file.substr(1);
             config.setString("history_file", history_file);
         }
-        if (config.has(prefix + "accept-invalid-certificate")) {
-            config.setString("accept-invalid-certificate", prefix + "accept-invalid-certificate");
-        }
+        if (config.has(prefix + ".accept-invalid-certificate"))
+            config.setBool("accept-invalid-certificate", config.getBool(prefix + ".accept-invalid-certificate"));
     }
 
     if (!connection_name.empty() && !connection_found)
@@ -268,6 +267,12 @@ void Client::initialize(Poco::Util::Application & self)
     else if (config().has("connection"))
         throw Exception(ErrorCodes::BAD_ARGUMENTS, "--connection was specified, but config does not exist");
 
+    if (config().has("accept-invalid-certificate"))
+    {
+        config().setString("openSSL.client.invalidCertificateHandler.name", "AcceptCertificateHandler");
+        config().setString("openSSL.client.verificationMode", "none");
+    }
+
     /** getenv is thread-safe in Linux glibc and in all sane libc implementations.
       * But the standard does not guarantee that subsequent calls will not rewrite the value by returned pointer.
       *
@@ -1105,7 +1110,7 @@ void Client::processOptions(const OptionsDescription & options_description,
         config().setBool("no-warnings", true);
     if (options.count("fake-drop"))
         config().setString("ignore_drop_queries_probability", "1");
-    if (options.count("accept-invalid-certificate") || config().has("accept-invalid-certificate"))
+    if (options.count("accept-invalid-certificate"))
     {
         config().setString("openSSL.client.invalidCertificateHandler.name", "AcceptCertificateHandler");
         config().setString("openSSL.client.verificationMode", "none");
diff --git a/tests/integration/test_accept_invalid_certificate/certs/ca-cert.pem b/tests/integration/test_accept_invalid_certificate/certs/ca-cert.pem
new file mode 100644
index 00000000000..c4f88831126
--- /dev/null
+++ b/tests/integration/test_accept_invalid_certificate/certs/ca-cert.pem
@@ -0,0 +1,32 @@
+-----BEGIN CERTIFICATE-----
+MIIFhTCCA22gAwIBAgIUZHeZ4ME7rL1RYGJZGPgOTHl7btMwDQYJKoZIhvcNAQEL
+BQAwUjELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM
+GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDELMAkGA1UEAwwCY2EwHhcNMjQwNzAx
+MDUxMTI0WhcNMzQwNjI5MDUxMTI0WjBSMQswCQYDVQQGEwJSVTETMBEGA1UECAwK
+U29tZS1TdGF0ZTEhMB8GA1UECgwYSW50ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMQsw
+CQYDVQQDDAJjYTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALf623IC
+1r6Bzwa4dTufRPvFWQinT4uIBv7UvAk8tu8kmLSMOOC2COALNRCSu0s1SFcO6i5G
+CgmaWdoTXuw+NYH9n5rU06N991NBfCoLbf0eSoie7bRdTCKkqWWKaw+dCvKL5vWj
+LEZRmdh8ZN0lxa05eIsLxErvgWqE8hOOQ7jgRqYLAagp6XkMdB+YRkXFtbVcze83
+ZNxFqNjdgwYYybaqDZKGeOnRDRof7P/cyMkbGNUI7+JYA5eGmDgiOTGbaCZx5AUK
+DKmkdmy5IJBWS0rYR8Ol9Sjs6NSYkmYiLFEWFNSvI88K99+Ab3DEi9JMEqMeNTJ5
+1sx48e2OfMG7ywoGj3D3zf7SUxAfwW+7Ls+v7c/kB9hvspA9iVsqjAKcSPAPnJXk
+zR5r+zcPwjb/dvGGGB+D7PhvreEpj0n7PRFEB07jviA1E8zkwdk1L9FefYMG2oRl
+8J7miWxoyqUOtA6AYhbN36+I0iJkjxIjXLLzB/dhOzerNTzRHcJtmgSyIpcPIV42
+IsJBttLOW0Mpkfw8z88xji+1VWHm+H8sOWCzbfjncScu/z5Xhb92C2T+AG+RpakB
+LwyTsovf+xv1gGF/1gmmdKSoV/JZuHSczMmmDLJ9vXRnIpK0AgL16lfs6o0C2Pe2
+OHJv0oViokU5mW+Mge3U9kUrh6BVITk8UjUDAgMBAAGjUzBRMB0GA1UdDgQWBBRA
+x0+tgav3Q2gRt34AtKZiGfFH7zAfBgNVHSMEGDAWgBRAx0+tgav3Q2gRt34AtKZi
+GfFH7zAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4ICAQBcV7s7au6r
+bEkHaW9RbGe/7iHesOcH2uyiALGeiCF+pfN9QbfkpJJbMRDhKr+sVqJH6lS+Iys5
+MpPaRHOInNs5ajiPQcIxtxAnw2HEsy6LUVvs6RUHrS7Bxlc406YQCY/sPyItAt8t
+U/qhx5VNj6Mx1smyKZHFHZIZ0eYieriJkK6NB2E7Ji6EPGe7lkh3kEoVTfaSVCkP
+p3d4U603kzCk8PGkYlhjWUCVMhDRwgEy6FXHmU156HivGpYurKcab7cMUfr50MyK
+1Mx/NaHOYHhIIQ87t9wF2yQaMskEcyK9XOorp+HFtnTdci+6+J3u2Kmyckq81uy9
+uZreH8bUKgmntOhSqtl4tWGMWxX78c7rX2cGIbSJsZzzjmUdhvnpYeuhCCrUPzY3
+kIt9Q6q7DCKG+f+D4bHQ292emEfKYb9LgftyOX3pcAJ+d3c56Cs7c0RRVFAZyVnX
+1O9ZvHrypmii5RARNuyVtttgDpuGBrRCa2kSNlXD9PJJitI2kfoWrCI/88Pw8H/y
+bLO9/ZcCOsxBGmfrQDMUSC8rE8JLUSKV80otWwtHcxcnfpsobs64/Rz6aOLYEiS4
+/0M6xyUYdOsEGUh1uyX5qxto8iOWop4S9DLy1QVx+N8/fC6gjhMb4Vg2oTBnHlGr
+JcxdYP8jVe6tUgvRKAyNlccPIq20udfGTw==
+-----END CERTIFICATE-----
diff --git a/tests/integration/test_accept_invalid_certificate/certs/ca-key.pem b/tests/integration/test_accept_invalid_certificate/certs/ca-key.pem
new file mode 100644
index 00000000000..66d548d0cf5
--- /dev/null
+++ b/tests/integration/test_accept_invalid_certificate/certs/ca-key.pem
@@ -0,0 +1,52 @@
+-----BEGIN PRIVATE KEY-----
+MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQC3+ttyAta+gc8G
+uHU7n0T7xVkIp0+LiAb+1LwJPLbvJJi0jDjgtgjgCzUQkrtLNUhXDuouRgoJmlna
+E17sPjWB/Z+a1NOjffdTQXwqC239HkqInu20XUwipKllimsPnQryi+b1oyxGUZnY
+fGTdJcWtOXiLC8RK74FqhPITjkO44EamCwGoKel5DHQfmEZFxbW1XM3vN2TcRajY
+3YMGGMm2qg2Shnjp0Q0aH+z/3MjJGxjVCO/iWAOXhpg4Ijkxm2gmceQFCgyppHZs
+uSCQVktK2EfDpfUo7OjUmJJmIixRFhTUryPPCvffgG9wxIvSTBKjHjUyedbMePHt
+jnzBu8sKBo9w983+0lMQH8Fvuy7Pr+3P5AfYb7KQPYlbKowCnEjwD5yV5M0ea/s3
+D8I2/3bxhhgfg+z4b63hKY9J+z0RRAdO474gNRPM5MHZNS/RXn2DBtqEZfCe5ols
+aMqlDrQOgGIWzd+viNIiZI8SI1yy8wf3YTs3qzU80R3CbZoEsiKXDyFeNiLCQbbS
+zltDKZH8PM/PMY4vtVVh5vh/LDlgs23453EnLv8+V4W/dgtk/gBvkaWpAS8Mk7KL
+3/sb9YBhf9YJpnSkqFfyWbh0nMzJpgyyfb10ZyKStAIC9epX7OqNAtj3tjhyb9KF
+YqJFOZlvjIHt1PZFK4egVSE5PFI1AwIDAQABAoICACPKkBVqevddznYqvVfHgMaQ
+GmWGEwH8iQvbO+d6IcZ9Mp3xJ4GDuE9Jrt2WBgy4BWY2VxWAd6RtnCxCF3XTFlq/
+CeU3oSKNt2dBdpPBXHjhAzP34xtzCQQ2T2QbHP/RF9eeouRHha36YvkMHh0to3qz
+EwHN3LaNSHp9+ZAXXZJiQyw7OOvLl4l+hDq/R3B8aYCqWL/AYYU+of3qffpaKSe5
+hmFqeqxWCHv8GQGfpIoHFPogvnYklDJOs188PS8q4gSUpHzceyEnhwhYrX6W4xfg
+4nkr2HlJOXReaCCpZkCyEIRdvBQjuJwI4uUnv2G+feOw1T2IGWL25m3GCfwweeuZ
+AWqqbj5Xnl1+VgvaNm68aHBVlfZ14rIJ/Hu1yJV/E+5m8+3xewAor0jshWceC3OS
+lhrZp+EyrwxsKj17s2IBhk4EIOLjfosoDqdw3E2DrtqKrnI63MLUTaEqHWyXnF82
+Mn880DR/E3Byj2kKtdIVesAdO0ARtIRqedPunpcvhp1eMZ2wrLJXAHMDBJlUZF64
+ZAyO3ll79PhBn/QShGu+KQ682Ns1T3j3J14n2zqBKsU1GzSLBOZO8QXfFELsv6Xj
+NGYPg2SbkRwbYHEPr6G9ehsxKaTpMAtPesdz1devKDI4nSpd2qE9ZfREH+CW0MkK
+bpiL20d4blIG1DP84qodAoIBAQDEBXbn0IEeyL3j50/AUJY/K4seeDm1ltJmhoNU
+HC0dwt2FOHrD1J/Cx6S19zuEVt1qMwJXELxOVzixykKPy73ETW+Qfcpo2J51THrl
+fSEUpoheD37Cwjv9vzb6Qb4ZXDr4acLIB06mAUKfo0Pez2x6IybWqS/dZoD1bfh5
+wiSCITXQB4n6mI7chfDDyTqgf3C5U5b2UABLDdIKQZHY5kX55Vfr0Lz+Gt2t6uUy
+hUiN4C+Qrx1LoHY4x7g1SG9lKASG5zv5dksc+a6uYsS4N8TM3ZQz7BnZrCQPNiWz
+K70p5pMjdVTHmQqThoww9y+rRiHMU3qUSZ0tX4nrY99/MiFNAoIBAQDwRi0GzkeA
+ECWfr2R+vh5lnxXBeAvM46v+8CzwJHNxtGAhB1ojod7GxOxHiMnWDXubRiNVlhx0
+OHeCq11hSQh2+a7s8xwxYbC3paYf1ZQq81YmTGd8v+/QX9b21Kq9dpx+G2C3eikZ
+rHMuSifrSEwZMxdw/gKvQ+x66PTNeK8fIgAtZsE96VX3u7sCCNhbBElWWg5OwwMf
+0RH5Pi/O8UXxkZs2SUKFaXUFMxjYn9r9636QI8LaLsCvcbPjrhshPtHWHqgrZzia
+8gD00ty3mcydRU//xNfHruew2iRZUVVB8E0GQ80eQCsHrI0YkP876at55DKbyZLi
+pGisswDdyoePAoIBAQCfOFUSDMGksKfVY8Da1FqGKydQc5Vkk4KJdr9b5Zsb7nmi
+yXKmeKHgQ+vdRlmYsDofFhnTgKuD8sMBXuncBUE6Hy/8rMBpk3J6ksnMeGwTawWf
+aXSsW7sA7H2KnzX7QXhA6R8m/RB8NM/Z/XKWm+Nf/geBG2Hes3ESaeYHeJPRA4rD
+VCxWWe7I3x7m2R/+2CVG08fugKMMU8iFPidqh/jU994HlMC+ojHoH8FV3P9HBo98
+NVFVaZw9mLRey1x4TR4IOfD0O4rdmU2zJlTl0Z7QTTchSajhrzqHBJNwyplXXIut
+pJNdoUj+9a4P4+hbdZbkcfZCVt0NyumrQq96kMyNAoIBAQC8Jcemd6tHnd1IYPip
+wshcvSk1GYLsCzAo7vvRhEmeNSe9VVuBYYToU1XOYk/ivSYYneCXaXeYjosPGI+D
+2cO2HqiaxXhDNwNBm4YtkXp385p3XQcAifAJVLMpD7kvqYy7t0aSBiffAcoj6k++
+Dxv/86EcituhUOgCCx9/Mky3bVGEO92AVJJL7oZu/mwlQYMICnb/t/dfV8DiUo7v
+VsF1e5Z1+QINJFpNGwkOwIakpV6zEj3Hyip6d+fslu6Jpv570//OFQMfWffjvqa1
+Ysr2MEuHtiVeKRkmxHsIMgc4HirCiB872/WWCclm/kvPJ5huV4JFuZ+6uBF7gqRg
+CPMDAoIBAGs0KCL3a/eXIPWhFcOgtpcNTb3o4gK/s38+GLIVbdP7MUNEkMeWq+Pf
+PBUca2p1M8CkbpRi0OAvg1gewAR/+bCdyzoipl1+3v3goYC3J5TAs5NT/siBcjrm
+QcTKEHmTeVu82Kpi1JVtcfS2TWuSmT6vSB2A2H3R4GEPQQXnXeS/FnRA0I85MFzq
+zxMkar5VY7Eg0Z3/bN5dNqAYSzqjgMMhRMPN1W54gjL2ev65Mzt1jYf6xz7gpJOh
+6Fw2EOrM23c4OckZx2dZ7Je6ns2U82D81ffhXRwWfQxoKXvMVMccBJmHTbkVEZlE
+VAq9BWwNistc+YfRka1OOeq37MgpXq8=
+-----END PRIVATE KEY-----
diff --git a/tests/integration/test_accept_invalid_certificate/certs/client-cert.pem b/tests/integration/test_accept_invalid_certificate/certs/client-cert.pem
new file mode 100644
index 00000000000..2dd497409c3
--- /dev/null
+++ b/tests/integration/test_accept_invalid_certificate/certs/client-cert.pem
@@ -0,0 +1,30 @@
+-----BEGIN CERTIFICATE-----
+MIIFLzCCAxcCFA6o2DM1UGQ1NNCQzp0n4hb3iSd9MA0GCSqGSIb3DQEBCwUAMFIx
+CzAJBgNVBAYTAlJVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRl
+cm5ldCBXaWRnaXRzIFB0eSBMdGQxCzAJBgNVBAMMAmNhMB4XDTI0MDcwMTA1MTEy
+NloXDTM0MDYyOTA1MTEyNlowVjELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUt
+U3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEPMA0GA1UE
+AwwGY2xpZW50MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAl9i6heiD
+2inNs0lKt7XjK0QyXKyIg63uVO39ZpLY5sj1noZnbFmRo5MYC4MMIrlD3wGnqUo/
+mdev2eg8qYry0CmzhLscVEha7LN+7wmdftYSGkXOl9E08KIv9Uw2bHap4KEIRvJ6
+298bdsuv1l4ljWo3VDkPCczXIaUi94/o1uNu0sbPRluBP8kjFLhwV8Ph/7mxsofu
+V5tDYNOZ8VozA7aHAnJqXBCFZuW6igIH3XaROY9LOe2FD+qxtpqJHp7E+ZNsXXH5
+SXMQzE2/X7eKHA7zvdG4z5GLgmz3GEvq3ZBJUGUe6DCnrxqBXUeLTqyummohGE57
+LuqzrH9etFj/91NaZW9uwN42pxGJSkeZaJ5BDnzO3VYxqvvDnFs/2TFbJiZ065vI
+NpA0PnuAkGjztaNAm6qXNTXsWo8bdi8TDgIwKxoo5KIlWsxnxKB82tfBovWd+JLa
+xLlWNKbKgNb6xc9OmNCyhYkRXhqIsBrpLy6n2yppyYEiCBzPrqwJ2p3Y88yj3dsc
+xiiu0ngFSroE2pLwCgi1rQY1x9VO4XQrrq6vBvPhi530+cdXdVTnnyBhaOMX4bWX
+OOjI97xJmR/YbOvpgKw9v/na82t2fgKoqIs124iZ21QOiffOIfwwg2AKj8EaWJ9/
+Era8sZMsq/g56PWYwT1J85cCMvbNwGOXjusCAwEAATANBgkqhkiG9w0BAQsFAAOC
+AgEAAPSps9k8HEZrfBbzH5SHASNTUc7F81h2EYhaEuQrXoan39u3syCKYMSJDt0P
+AVCYbIvXmlHeaBba9YXtZVZJpCM53b+x01297x1bXzRklYxnpuD2IauuQgiaJBXV
+VlCjEy7GHBAmRQwV7llkoxx+qHsA0b+oHKvTRc9FJzk/Fx4Eb496gNMvUk47f+15
+WBmWoQolqwbH1Ivhca7BpTvTSeNGkJ14cbmuhwryLstHJCPep+vp2TYS9bHFZIPY
+g3APnWv1qShuA7Oj68GLpRVJmImgyPMK0i8LDGcLhrDiG+HYEr89MiRWDvBHun7O
+Sp5y8zDnDZSrN+EtRp8DypHeB7R0Ukmn76NP6NsigWCPMlLaHNVsvVAwWwFNae9u
+OxsoW07tJo3lZjDx+tFrUb7IRyn/9PLDIwEs4SvDP+O97o0JkCszwfRhkDnjY7Qj
+XCgXoCBalIadgnGshbEjFLH0ZSwAoCIC/35X0Zo7cd1+Fz4Xq1tgnV5ec6ci/EHi
+qsOmGfTqZz5HJlmfyxUOYicizV6SjNZSFWY07ebYRewx6ymdDFhqrbnJIEUGJYlU
+5qDY1nemoqh9QRIhm1+73hAKHtmcaa1u72gj1QBtRD5eG3ToM3GHgdbUUPvFpenx
+UPfzw/wbzITSqPoYr1/XIGq9e1muTqHbFNlGVByIW0ejeZQ=
+-----END CERTIFICATE-----
diff --git a/tests/integration/test_accept_invalid_certificate/certs/client-key.pem b/tests/integration/test_accept_invalid_certificate/certs/client-key.pem
new file mode 100644
index 00000000000..09c04c1e815
--- /dev/null
+++ b/tests/integration/test_accept_invalid_certificate/certs/client-key.pem
@@ -0,0 +1,52 @@
+-----BEGIN PRIVATE KEY-----
+MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQCX2LqF6IPaKc2z
+SUq3teMrRDJcrIiDre5U7f1mktjmyPWehmdsWZGjkxgLgwwiuUPfAaepSj+Z16/Z
+6DypivLQKbOEuxxUSFrss37vCZ1+1hIaRc6X0TTwoi/1TDZsdqngoQhG8nrb3xt2
+y6/WXiWNajdUOQ8JzNchpSL3j+jW427Sxs9GW4E/ySMUuHBXw+H/ubGyh+5Xm0Ng
+05nxWjMDtocCcmpcEIVm5bqKAgfddpE5j0s57YUP6rG2mokensT5k2xdcflJcxDM
+Tb9ft4ocDvO90bjPkYuCbPcYS+rdkElQZR7oMKevGoFdR4tOrK6aaiEYTnsu6rOs
+f160WP/3U1plb27A3janEYlKR5lonkEOfM7dVjGq+8OcWz/ZMVsmJnTrm8g2kDQ+
+e4CQaPO1o0Cbqpc1Nexajxt2LxMOAjArGijkoiVazGfEoHza18Gi9Z34ktrEuVY0
+psqA1vrFz06Y0LKFiRFeGoiwGukvLqfbKmnJgSIIHM+urAnandjzzKPd2xzGKK7S
+eAVKugTakvAKCLWtBjXH1U7hdCuurq8G8+GLnfT5x1d1VOefIGFo4xfhtZc46Mj3
+vEmZH9hs6+mArD2/+drza3Z+AqioizXbiJnbVA6J984h/DCDYAqPwRpYn38Stryx
+kyyr+Dno9ZjBPUnzlwIy9s3AY5eO6wIDAQABAoICAAl8R6ICPpCcEXrUb5qn2Ujy
+IJQdT+D0yJqGTD6QCDvsLDstGcWMcx1him5785IxJ9YpZBbnidO0euXJo8R3Ql8S
+aGHK4ZtfFyeKAYNoJ0mSIx03LdiRAEOJ7QDf2LI017yJS9IkeW5ql0fTas0za2Jm
+nCBZZrNNzvv7a/BEn8NHkfMQV5Fgy9ESvP3qcHhsNHly/aeD5ZEmbZcmWeb82orS
+nxEnR0hUSnI0JpibsycT90DimAOh3LDXfd3E0Iz9VsA8urMpqtpDYzZ1tShed7TP
+tyiaxS1+PUNIkEc9xRv2i2FQyhVmpmZdoCXfNi84nUtWhJYE6HkrmgRwnnLQp5rr
+4stSQLo9X/5ML/bo2cj1eAiH+P6bAWaZhkMYPGkhBYwwgYqZF0Jn2Ay07BPF5gqq
+rDY0mjF+Nm/NOqHfQjWHSj7KYYHm6qU1s5cAnDWLfznXGWZHpoI4UqAZtpAwGWEh
+f/nS2LyepFW8RE6mbRDe+PkOM2455QdM8MuQc8drGInJ57A5e7y1LxOvFaUP+QtG
+fqXok95XciMfttcR8nfYkUyC5QX+JmDZRf1U/DUxTJtbKXRKVG8WaFagkViiCniw
+Q4RbrS4Utu6h+8HXdI9mK2pIHlJPp202b9gPry1j3kya8/jx5cTFWCcPj4IlgY51
+ZC1+vXJUezwq60gWJrRtAoIBAQDLsR9M+ZdBC5sl5fiVGsznAvfEVv/Cf7pmeXqt
+q4BZT7kB8zfdfRRMzmZiQIX0zTbBn7Q7kWULcl8HgPbMEGZorc24CFuCq8k0n2ji
+0aJsTJlfr7kUfTZ5c3HN1fGhDrnGoCL1eElt/Es4hZTIDnbn3i+RoCqwhCqhC0Lo
+L7f6iHrVIcKyoaz+Cye9QKA89PVHMNM0f41zFJTVpSzGfWefG4vNoeU6nxqwkWVJ
+Guhkt1EsJUScHueAa7hUYlELDy99tlqOPLdG7/GG6LmE6pM7R1E6DishnL1VgOll
+KL6fouoz8nTve59eCDQECNzXN8lxPvx6TdufabGJTSZ3iNjPAoIBAQC+10DFlO3Y
+XLHfX9LWK4rdkghx9AoeCFIVZO2G29H4zyDQn7hAMISWeVCqdAKhPQMJmAXyhQcc
+PLEUdKVwifbPNO/IxsWL8SOfbI28XoZLYkQLual6JPa10jemFsinVRTPLU7RA1/T
+7t/mWvyaAHmODWUPKSs1ltzh5dNwvnSNou6h72TytwlMIk5h8Z8ZtMIY2v5Ca2Mx
+I7Wi8/5RebpszBbuLYy0GBqK5oqQov1YH0WK3ae4leaPWlZ49nyNYpx5Bvngm+d8
+4F+HvcxdpckRPfenl9nBW+0UHVXZORepQ+g0CNJxAQTomXy5VeEnrpiylrec4mE7
+YD15GKDxeHclAoIBAQC6efcr4zwDwnpRIc6HwrdfriniR7tx9rlRY1lacS3ZldDL
+5jx4uy/AeRzmSifYy56beHfqKNuWtSgmmHxYqZL3tlYlaxFL+bUERgQh0sWd3Cp1
+nUwVY+RhH71KMdk6F18gZi5bB7pBxL2gizqARC27bH+kFDvAyd60/LhOFJGb/SJm
+yLeUDctv8Ap7ihSoeaGrgRl6cGOwvWDcjpEFTn10/iTpJpPTLSGkEBfn64hh3Cml
+/qBnRSuvN+7S4gdL1qUls7IHkbJAcR5u4dQlyRLtLAdC3QHDjdkP1CrX/4s0XlOH
+JyrCrCxkS6JDJ1H2fg4fbQh7d5few0ZTapwSxuelAoIBAQCloou8MTuR6zoShsem
+TRp0npen18ikMlKT8QpbXxTJ33Q6vjWbuMCLHXYOyyuxKBQn0oNu71ixgFetSmSA
+3a2SP/7SAvtYL43cg8ossNncOciXaKgYtFZQjsjv5Zx351dtTYS30jbs6cn41AWj
+4D+gNC3/T5sKIwtJIUE3/IWKnNyDPhUeuBBgvPHM/no5RHKmQgQmP7BS6QxwB1c2
+/fD5FtXcGTqi4LzAy8F+MYizYEjqcuFsSuZWi3C8AisuR2VJge2Eravn7ZHcbbYf
+nZF63o7BLgG4WgHV0HQ/WdcYeTDjYLjNjbXl77mZ3ljSKS6s6jUo/C6p6yjPnrmp
+ewXlAoIBAHX9aLtoj06zU5dtrMVjhYip6Z5LEtHsXCvOmnAl91uUXKwtMze6Y7xc
+4ZggVWBR9lFyq9bAAuIXkRuE1jbvEwl5sLZY5Hf6GrIUR2uD7ZYFbogBTgmWCX9S
+vYRpofEwquX0AUt+a4nO09wPBX1BJMViAEZSAWGfCfzVz8rXYIgcmu471wrpQHv4
+/whNA90JX0v6tuOASgR2dGorU//0ejGcm0bcYSuivKPxBfmVNeZNgIJXLM0ZHql1
+qwLnWXS0e0kADWhp/GAWVo0xzefgD3mDmyG2Qbz8MLaDkwPU/L073ZZNT1AHFZZE
+xfVijyydtPqa/JDgqcUFBxLtiKGTbbI=
+-----END PRIVATE KEY-----
diff --git a/tests/integration/test_accept_invalid_certificate/certs/client-req.pem b/tests/integration/test_accept_invalid_certificate/certs/client-req.pem
new file mode 100644
index 00000000000..507db7bae6c
--- /dev/null
+++ b/tests/integration/test_accept_invalid_certificate/certs/client-req.pem
@@ -0,0 +1,27 @@
+-----BEGIN CERTIFICATE REQUEST-----
+MIIEmzCCAoMCAQAwVjELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUx
+ITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEPMA0GA1UEAwwGY2xp
+ZW50MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAl9i6heiD2inNs0lK
+t7XjK0QyXKyIg63uVO39ZpLY5sj1noZnbFmRo5MYC4MMIrlD3wGnqUo/mdev2eg8
+qYry0CmzhLscVEha7LN+7wmdftYSGkXOl9E08KIv9Uw2bHap4KEIRvJ6298bdsuv
+1l4ljWo3VDkPCczXIaUi94/o1uNu0sbPRluBP8kjFLhwV8Ph/7mxsofuV5tDYNOZ
+8VozA7aHAnJqXBCFZuW6igIH3XaROY9LOe2FD+qxtpqJHp7E+ZNsXXH5SXMQzE2/
+X7eKHA7zvdG4z5GLgmz3GEvq3ZBJUGUe6DCnrxqBXUeLTqyummohGE57LuqzrH9e
+tFj/91NaZW9uwN42pxGJSkeZaJ5BDnzO3VYxqvvDnFs/2TFbJiZ065vINpA0PnuA
+kGjztaNAm6qXNTXsWo8bdi8TDgIwKxoo5KIlWsxnxKB82tfBovWd+JLaxLlWNKbK
+gNb6xc9OmNCyhYkRXhqIsBrpLy6n2yppyYEiCBzPrqwJ2p3Y88yj3dscxiiu0ngF
+SroE2pLwCgi1rQY1x9VO4XQrrq6vBvPhi530+cdXdVTnnyBhaOMX4bWXOOjI97xJ
+mR/YbOvpgKw9v/na82t2fgKoqIs124iZ21QOiffOIfwwg2AKj8EaWJ9/Era8sZMs
+q/g56PWYwT1J85cCMvbNwGOXjusCAwEAAaAAMA0GCSqGSIb3DQEBCwUAA4ICAQBr
+ZWJGghyeCUESKu9v9dgQYD4Bq5YviYY7BAjdQsn5/Ti+tqkAKHEn/L+264QJMrTU
+muTKg09o7n7r+g4sHmZhT2QEtwGFwaEGvz2zRd4po29WlGQAN5a8yzXIb8b9n+C4
+Z5APeAzjBV9ooqbJCURlgeDP247XTcU8yIvvX0YpLmXYRQ5Na8TTfdBdlD3Y23mD
+aPe+2YGkK1Go4iNMoyTEoJ0cj+89WMA9o+BiGcptALYMzKBM26f/vz9hb0J49RDV
+MkccL8TKNDJQrLFCIcwpLRw08uZZdqJ/cy01hrtkMkp+SSBQO9qSmalDrTLgs4Pp
+3ml88OIsQHMkdBUjUAw5FyAJoik4iXab/sLoMmMJ/lqH0in3dT4KeEh8Xe2iAfNU
+97/6lAuZOrn2EPvqBJI/gfjwPj9wTgjlvLpmcjhBvfqkuM/dd7zTmwRoM1klRMC2
+07x2YNoG7cGXjAYk1h5HDYVJqdphgQLTS1nRqmiQ8QnPryqlZcI7T/GgpOOulHce
+lgMp6FFlGaQMysCbdYJQk4JMQXjwlj5ms/6rZSkgwmrv6WhCY7ImDj2E/7oE5XJd
+BqXHFwOQdCSgQOVH+QgmxceYolI73cMNF3Frq8/IztAOtMyBF0bpRpI1U6a3ALf4
+cK2Dkd/TLg8ZwsVBjPaRrtXlntO6tB+tn3suukCKcg==
+-----END CERTIFICATE REQUEST-----
diff --git a/tests/integration/test_accept_invalid_certificate/certs/generate_certs.sh b/tests/integration/test_accept_invalid_certificate/certs/generate_certs.sh
new file mode 100755
index 00000000000..610a1281f8b
--- /dev/null
+++ b/tests/integration/test_accept_invalid_certificate/certs/generate_certs.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+
+# 1. Generate CA's private key and self-signed certificate
+openssl req -newkey rsa:4096 -x509 -days 3650 -nodes -batch -keyout ca-key.pem -out ca-cert.pem -subj "/C=RU/ST=Some-State/O=Internet Widgits Pty Ltd/CN=ca"
+
+# 2. Generate self-signed certificate and private key for using as wrong server certificate (because it's not signed by CA)
+openssl req -newkey rsa:4096 -x509 -days 3650 -nodes -batch -keyout self-key.pem -out self-cert.pem -subj "/C=RU/ST=Some-State/O=Internet Widgits Pty Ltd/CN=server"
+
+# 3. Generate client's private key and certificate signing request (CSR)
+openssl req -newkey rsa:4096 -nodes -batch -keyout client-key.pem -out client-req.pem -subj "/C=RU/ST=Some-State/O=Internet Widgits Pty Ltd/CN=client"
+
+# 4. Use CA's private key to sign client's CSR and get back the signed certificate
+openssl x509 -req -days 3650 -in client-req.pem -CA ca-cert.pem -CAkey ca-key.pem -CAcreateserial -out client-cert.pem
diff --git a/tests/integration/test_accept_invalid_certificate/certs/self-cert.pem b/tests/integration/test_accept_invalid_certificate/certs/self-cert.pem
new file mode 100644
index 00000000000..7b3832a3b6b
--- /dev/null
+++ b/tests/integration/test_accept_invalid_certificate/certs/self-cert.pem
@@ -0,0 +1,32 @@
+-----BEGIN CERTIFICATE-----
+MIIFjTCCA3WgAwIBAgIUFpPVJytX13EWXwaLqg/rOTdu4UgwDQYJKoZIhvcNAQEL
+BQAwVjELMAkGA1UEBhMCUlUxEzARBgNVBAgMClNvbWUtU3RhdGUxITAfBgNVBAoM
+GEludGVybmV0IFdpZGdpdHMgUHR5IEx0ZDEPMA0GA1UEAwwGc2VydmVyMB4XDTI0
+MDcwMTA1MTEyNVoXDTM0MDYyOTA1MTEyNVowVjELMAkGA1UEBhMCUlUxEzARBgNV
+BAgMClNvbWUtU3RhdGUxITAfBgNVBAoMGEludGVybmV0IFdpZGdpdHMgUHR5IEx0
+ZDEPMA0GA1UEAwwGc2VydmVyMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKC
+AgEAsQO7f1Ar0ThZjh00KeuckyJDf/oPQ+67EX2Y82W4/dl8KmHksmIKp2+CJFwg
+Y7/c1ti+DjxVXr4tZynzUeDYqujTrAidUMz0zrvjXNeeyoe2hgc2X6YFtVhyEtMQ
+6q8VMaqomjuQDIyxUUdt2xHZWyxHPntDrzsmCCDH5Md27CZ4Z5jsSOdgnlrCkjd9
+NtuIzxTlrExe3whIlTL0DURTvAVV/tPoVpVSvHTWCdziaCWm1hG5T6ZCN2W0pBgV
+k+UNNtXr1ksr/Ma0Z84ShKjpN+soFOIZ3GDDGY8Xr/74jZo2c4025TpeIPcPBE0Z
+b2K24l0UNnQbgm3ryI7NRrZ7psNnhvnWZru4w5Ykr8RKBf3n1AZOt8+dXHNvJQeU
+/CUA6pKpc+sIxzAtWtS2ABqdOTJTXj2GyYNsz38dI2uzecUzT7oEtHWq9sr376EO
+xMv1wXsUBli8INhPricfojH2/ACnRPysFglAKTx0IpcwqYoV5pGkk0iK1DNkQMv4
+Kf7cD2hLWv9+Fgbrjfu/gNCL0bSJBjaTHngACePsLEQkmGomscDySYXa+PHeB8Ln
+QCWsE6MkeDa5g3gUOPV/vC6qv0ZBvQj1p3sOeyIQATDBYX+KheCO+N0ZCCdlT5Sx
+thKNDusultu6/lyjGum0c/5oGgoDKTIkibgljekKupBRAykCAwEAAaNTMFEwHQYD
+VR0OBBYEFEnPwBjLDPzta3nBrUEH+Gz1nBJRMB8GA1UdIwQYMBaAFEnPwBjLDPzt
+a3nBrUEH+Gz1nBJRMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggIB
+AIW829ODEBvBp+vUwt3hbsJdBx9DzwcBBJ+Mew95YVveiosU+3p1HaWCkc4srgOi
+hDNbpvfbqBIKVAZzesJbuTEeQ+Aj4M1lcFed9tvidVnmqDgHJApOI61GihT9MDjC
+dH6vhcsorE8E5F58FcMJ0QDynVOuCA/gHkS3ucMxBsNKGfYNe0rOGMw23CzHdnz7
+UonOKhrhIQawI3ELLQxo2LWZEmMsluznE4MB+RGdn3k/UzOY9x4bauv4hrHjYvEm
+x/zp2iWhZXvvTZ9u9xaHXEzbSU3WFCcXE/l9SDPM7ip9fabUrvaTJkmM190bD9sk
+VcEFOAFBVxfx2fvmBIYqvSkVbcWdAT6x8k+MVNfq99/1aIdh+DTB3XUB7FkE/fnU
+tXr/1sz0guwfNI/cHvhHisjmbgMhZWeeVV1YZHOJCsK7eq9rGq5Uul8nYrTdYKlN
+u0OcrBXYBB6ny7ljCTahyPPmFZh/4Mgr0xROn2HyhTCpDHjnD1M5jT783fvWqu06
+cpOjRIPYNtA/UQAWUZWrTuJTD6Kju9tDI8i1NtMWIXd7w0LDXsXeHpRoDrVvOIPT
+P0ivUpt4dFHnwR8Dos4NI/j9mkdGCblUxDsd/7or8gjedOoWjWfGCNlYa72aq0gf
+KjKGPISC3BGTNtwRTe+E7k38r+Y6YO/Tnu9RvBdDM3vF
+-----END CERTIFICATE-----
diff --git a/tests/integration/test_accept_invalid_certificate/certs/self-key.pem b/tests/integration/test_accept_invalid_certificate/certs/self-key.pem
new file mode 100644
index 00000000000..10b299a8c0c
--- /dev/null
+++ b/tests/integration/test_accept_invalid_certificate/certs/self-key.pem
@@ -0,0 +1,52 @@
+-----BEGIN PRIVATE KEY-----
+MIIJQwIBADANBgkqhkiG9w0BAQEFAASCCS0wggkpAgEAAoICAQCxA7t/UCvROFmO
+HTQp65yTIkN/+g9D7rsRfZjzZbj92XwqYeSyYgqnb4IkXCBjv9zW2L4OPFVevi1n
+KfNR4Niq6NOsCJ1QzPTOu+Nc157Kh7aGBzZfpgW1WHIS0xDqrxUxqqiaO5AMjLFR
+R23bEdlbLEc+e0OvOyYIIMfkx3bsJnhnmOxI52CeWsKSN30224jPFOWsTF7fCEiV
+MvQNRFO8BVX+0+hWlVK8dNYJ3OJoJabWEblPpkI3ZbSkGBWT5Q021evWSyv8xrRn
+zhKEqOk36ygU4hncYMMZjxev/viNmjZzjTblOl4g9w8ETRlvYrbiXRQ2dBuCbevI
+js1Gtnumw2eG+dZmu7jDliSvxEoF/efUBk63z51cc28lB5T8JQDqkqlz6wjHMC1a
+1LYAGp05MlNePYbJg2zPfx0ja7N5xTNPugS0dar2yvfvoQ7Ey/XBexQGWLwg2E+u
+Jx+iMfb8AKdE/KwWCUApPHQilzCpihXmkaSTSIrUM2RAy/gp/twPaEta/34WBuuN
++7+A0IvRtIkGNpMeeAAJ4+wsRCSYaiaxwPJJhdr48d4HwudAJawToyR4NrmDeBQ4
+9X+8Lqq/RkG9CPWnew57IhABMMFhf4qF4I743RkIJ2VPlLG2Eo0O6y6W27r+XKMa
+6bRz/mgaCgMpMiSJuCWN6Qq6kFEDKQIDAQABAoICAA4BaZhifipeTFeo395hPKY3
+k8z7EAd5SrvQ4C/XNefVNAnLwmM1dqUAzEnsMB6JqQN8FVgzAIjagI/PtedzOYbB
+nTMuC08VUwP4bGjay+mNC1gCF2nsojmoTH5IzOcvVdH0MBrj6OXVS+z5EIFrdsh2
+nGcegaB1DEagXAXcnQAvmwniA5qmaUfzjmmF58EAuwuyGj9OW5icoyw1jTHE3OCH
+hjeqCeav3I1PVpAOi1YKL0UeFIzExCsglPEBfCjoz+mCy4IiYydA4Y1FChoM80bw
+CDUpc+UH6D02y9JVA317XMuPuhmGS2RNp+vYiHv+Uide40vbq+w0nkSOvUHXC7lo
+UXf26sZWHcQP8TxewjZFcajApk03lCAb/u4skt7OQxbDOcvrcdbCsSpZPS16s3CA
+OWA9fS7WJDmFx/dhzm4xNaXpz9z5MMkUfnFqgceG3H1leamL66J+eOzlquQ8LdsJ
+/wTvCi1SsR3JmqT+TpqlXTJci7A+I/b0Yt6a0s2HunQ5LA1Xomffhn/00IxUeqyi
+4MKh7wnoCaRlwZVORgQfKnosSkbkixDabwr9BMnMryE+NdQI3JKCvchdB0idyk1z
+igTgZYpaxZg9hwTlOITGYSsprN7tiwjNQG+Kivatn1NxnxhD8lpDoUqluR7mZs1e
+CwOLrB+irCULr7tcRYtxAoIBAQDFV7E1Oz8j8NkvkLztIFJ0ej7zEyNp3HUJb4bd
+SRvF2AL6sqSw0xVLK8HbrlfPo1Nex8nZUfY3v2DmPS4+S3mrtk+8XoRkvNn0VdL4
+FqLfygYsURI8xVp4zTsAmR+7grPJbb/u3XG7CBY3T4XGHSbMmYAu7+W3A2t8Q+0U
+fptju7Gc9gEACcSfap45CzEnFbabxNq3FJJ9V92N/xVQcOzNLxPxgn6tVhPeQnXV
+0WHJQdxB/8DivgOObiTCB7on9tNbSHisTYtGwn6Kz0961ewYU7Xj4epj7A35nWhI
+E10FDAf3T7cQa9ubSFZwOiSe0voAbiWbN3/M+zgkZDp2+wlnAoIBAQDloTtzgQaA
+//VPNhB2d7iOgAl5cUSWqY0HLJobw1ETsdbkGRTgZrFXe4XlXwHB4S+zVEMmyR3p
+HY0sYoAdpmd3Ggo4efJEx9IxkD6UZ71we02szvt4MdV5mwjHOAQfEJffktVkeqEJ
+gdekSmaHTeryBf3ESpefq+OH1qaj/IjivySLI4c8fPcxNg0nMmDsdMzRaDOHDDa2
+nUpRBeMFkwiiPJpALPPp/4be0d9mrahPd4zBoBieZwaP6Zhwksi0NaM4WKEfx4Fd
+TzH0K7UfLDMJ13wuTsNPzjPlDA+R4E/O3qVZxtHzuqc7fCksE7inYZghIRSjQHmv
+mmLPaFdaIWTvAoIBAQC50BQfVyXgpYsaamI5tFFvMxvKrJ6PsvDQbZ7DBW+NHhPQ
+Eh1YEajfGthyDPrJm7OIbv30DmSToTOQFtNP/KkXsZXqe+SVJezjT1xSFJ5PsV2W
+pM9TQVAqprODoKpsWGkIpLowI3pO7RJeChafsBs0UuSLTX8yj5nJa/vzbDLL+ZMH
+g+r1Kj1lDXLlq1AgySUkDnsSH+C+/Ni0NxbOo0WU/Vr98C1u8LnMcS+aoIg+hDDs
+Q/BOjzOyDKu3tqblslbYgKh/NJSLj96uvPzNYFfpoAiIBdQ5Oa7+9eOW2gSgF+/4
+QarRX31r/PTGvzh0A5s6hHwMZnmDfJgkS+tw1HbJAoIBAQDfauQBGYROdrsieqPB
+3/H9E64kLoRhYk4IsOiwInT8CIfgHlcljkT1rQU1ih6XmJVkkZNUZVFowovcp2mn
+b4QzaW6QTHXfXduwxXtdwSGcExwcCra41yOw0ZrU2jeIUgiySpa80GNmcKJzAsCG
+3NS+7f1dwoWLoR4byEkSPF6i3v6bMdIelCeeBcz+WhHbfjjZddzMhKGnQ5pPAr9D
+aS95SiV6U0JuZWO/38rQiB2+U77rlOzfEbZcxxa+UNJeI+HOrMz3spC3TJp5yxzK
+W6K87J+yHogBWcs8C2QKtohv6khO9RbJYM3dHoygif+bO2Vctmbx/4VlyXxh+E8l
+LmCZAoIBAFR+dd7cW0pDkOsPsGZbWwDPNRwo2lxmqf2B5w9zoGNFoc4qV5k9tJfu
+kPcwol6h2P4s2olYTgYC6hjPmE17U1TyP2DFeOw5IdxzNqy7dZndmaTs4qFLnDv3
+HIbkX42p7v++EU9eKAhYKJzJpuianMZ+v5azV1wNS5TDuN2TFULyIi+1UEw0ONuJ
+rA97idb2PrO2sT/eBj02uiOcPhZH6CHSMxDp7QG+S1AVZBgutTQQdeAsb6pE/iW0
+u3IzGNxTYjTmF+u7iNdOPiaTFpdnltDviADESMJGH7PIk7r7PXMI9ID+LbBFGJ7Q
+duW6PGuZ9JMBbxbNNvYayVgPeAeUsUI=
+-----END PRIVATE KEY-----
diff --git a/tests/integration/test_accept_invalid_certificate/configs/ssl_config.xml b/tests/integration/test_accept_invalid_certificate/configs/ssl_config.xml
new file mode 100644
index 00000000000..a1dd1a24693
--- /dev/null
+++ b/tests/integration/test_accept_invalid_certificate/configs/ssl_config.xml
@@ -0,0 +1,17 @@
+<clickhouse>
+    <!-- Native interface with TLS.
+         You have to configure certificate to enable this interface.
+         See the openSSL section below.
+    -->
+    <tcp_port_secure>9440</tcp_port_secure>
+
+    <!-- Used with https_port and tcp_port_secure. Full ssl options list: https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/SSLManager.h#L71 -->
+    <openSSL replace="replace">
+        <server> <!-- Used for https server AND secure tcp port -->
+            <certificateFile>/etc/clickhouse-server/config.d/self-cert.pem</certificateFile>
+            <privateKeyFile>/etc/clickhouse-server/config.d/self-key.pem</privateKeyFile>
+            <caConfig>/etc/clickhouse-server/config.d/ca-cert.pem</caConfig>
+            <verificationMode>none</verificationMode>
+        </server>
+    </openSSL>
+</clickhouse>
diff --git a/tests/integration/test_accept_invalid_certificate/test.py b/tests/integration/test_accept_invalid_certificate/test.py
new file mode 100644
index 00000000000..13431f39854
--- /dev/null
+++ b/tests/integration/test_accept_invalid_certificate/test.py
@@ -0,0 +1,93 @@
+import pytest
+from helpers.client import Client
+from helpers.cluster import ClickHouseCluster
+import os.path
+from os import remove
+
+
+SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
+MAX_RETRY = 5
+
+cluster = ClickHouseCluster(__file__)
+instance = cluster.add_instance(
+    "node",
+    main_configs=[
+        "configs/ssl_config.xml",
+        "certs/self-key.pem",
+        "certs/self-cert.pem",
+        "certs/ca-cert.pem",
+    ],
+)
+
+@pytest.fixture(scope="module", autouse=True)
+def started_cluster():
+    try:
+        cluster.start()
+        yield cluster
+
+    finally:
+        cluster.shutdown()
+
+config_default = """<clickhouse>
+</clickhouse>"""
+
+config_accept = """<clickhouse>
+    <accept-invalid-certificate>1</accept-invalid-certificate>
+</clickhouse>"""
+
+config_connection_accept = """<clickhouse>
+    <connections_credentials>
+        <connection>
+            <name>{ip_address}</name>
+            <accept-invalid-certificate>1</accept-invalid-certificate>
+        </connection>
+    </connections_credentials>
+</clickhouse>"""
+
+
+def execute_query_native(node, query, config):
+    config_path = f"{SCRIPT_DIR}/configs/client.xml"
+
+    file = open(config_path, "w")
+    file.write(config)
+    file.close()
+
+    client = Client(
+        node.ip_address,
+        9440,
+        command=cluster.client_bin_path,
+        secure=True,
+        config=config_path,
+    )
+
+    try:
+        result = client.query(query)
+        remove(config_path)
+        return result
+    except:
+        remove(config_path)
+        raise
+
+
+def test_default():
+    with pytest.raises(Exception) as err:
+        execute_query_native(
+            instance, "SELECT 1", config_default
+        )
+    assert "certificate verify failed" in str(err.value)
+
+def test_accept():
+    assert (
+        execute_query_native(
+            instance, "SELECT 1", config_accept
+        )
+        == "1\n"
+    )
+
+def test_connection_accept():
+    assert (
+        execute_query_native(
+            instance, "SELECT 1", config_connection_accept.format(ip_address=f"{instance.ip_address}")
+        )
+        == "1\n"
+    )
diff --git a/tests/queries/0_stateless/01189_clickhouse_accept_invalid_certificate.sh b/tests/queries/0_stateless/01189_clickhouse_accept_invalid_certificate.sh
deleted file mode 100644
index 00378abdb8e..00000000000
--- a/tests/queries/0_stateless/01189_clickhouse_accept_invalid_certificate.sh
+++ /dev/null
@@ -1,107 +0,0 @@
-#!/usr/bin/env bash
-# Tags: no-fasttest, no-random-settings
-
-CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# shellcheck source=../shell_config.sh
-. "$CUR_DIR"/../shell_config.sh
-
-# use $CLICKHOUSE_DATABASE so that clickhouse-test will replace it with default to match .reference
-config=$CLICKHOUSE_TMP/config_$CLICKHOUSE_DATABASE
-xml_config=$CLICKHOUSE_TMP/config_$CLICKHOUSE_DATABASE.xml
-XML_config=$CLICKHOUSE_TMP/config_$CLICKHOUSE_DATABASE.XML
-conf_config=$CLICKHOUSE_TMP/config_$CLICKHOUSE_DATABASE.conf
-yml_config=$CLICKHOUSE_TMP/config_$CLICKHOUSE_DATABASE.yml
-yaml_config=$CLICKHOUSE_TMP/config_$CLICKHOUSE_DATABASE.yaml
-ini_config=$CLICKHOUSE_TMP/config_$CLICKHOUSE_DATABASE.ini
-
-function cleanup() {
-	rm "${config:?}"
-	rm "${xml_config:?}"
-	rm "${XML_config:?}"
-	rm "${conf_config:?}"
-	rm "${yml_config:?}"
-	rm "${yaml_config:?}"
-	rm "${ini_config:?}"
-}
-trap cleanup EXIT
-
-cat >"$config" <<EOL
-<config>
-    <openSSL>
-        <client>
-            <invalidCertificateHandler>
-                <name>RejectCertificateHandler</name>
-            </invalidCertificateHandler>
-        </client>
-    </openSSL>
-</config>
-EOL
-cat >"$conf_config" <<EOL
-<config>
-    <openSSL>
-        <client>
-            <invalidCertificateHandler>
-                <name>RejectCertificateHandler</name>
-            </invalidCertificateHandler>
-        </client>
-    </openSSL>
-</config>
-EOL
-cat >"$xml_config" <<EOL
-<config>
-    <openSSL>
-        <client>
-            <invalidCertificateHandler>
-                <name>RejectCertificateHandler</name>
-            </invalidCertificateHandler>
-        </client>
-    </openSSL>
-</config>
-EOL
-cat >"$XML_config" <<EOL
-<config>
-    <openSSL>
-        <client>
-            <invalidCertificateHandler>
-                <name>RejectCertificateHandler</name>
-            </invalidCertificateHandler>
-        </client>
-    </openSSL>
-</config>
-EOL
-cat >"$yml_config" <<EOL
-openSSL:
-  client:
-    invalidCertificateHandler:
-      name: RejectCertificateHandler
-EOL
-cat >"$yaml_config" <<EOL
-openSSL:
-  client:
-    invalidCertificateHandler:
-      name: RejectCertificateHandler
-EOL
-cat >"$ini_config" <<EOL
-[openSSL.client.invalidCertificateHandler]
-name = RejectCertificateHandler
-EOL
-
-echo 'default'
-$CLICKHOUSE_CLIENT --config "$config" -q "select getSetting('invalidCertificateHandler')"
-echo 'xml'
-$CLICKHOUSE_CLIENT --config "$xml_config" -q "select getSetting('invalidCertificateHandler')"
-echo 'XML'
-$CLICKHOUSE_CLIENT --config "$XML_config" -q "select getSetting('invalidCertificateHandler')"
-echo 'conf'
-$CLICKHOUSE_CLIENT --config "$conf_config" -q "select getSetting('invalidCertificateHandler')"
-echo '/dev/fd/PIPE'
-# verify that /dev/fd/X parsed as XML (regardless it has .xml extension or not)
-# and that pipe does works
-$CLICKHOUSE_CLIENT --config <(echo '<config><invalidCertificateHandler><name>RejectCertificateHandler</name></invalidCertificateHandler></config>') -q "select getSetting('invalidCertificateHandler')"
-
-echo 'yml'
-$CLICKHOUSE_CLIENT --config "$yml_config" -q "select getSetting('invalidCertificateHandler')"
-echo 'yaml'
-$CLICKHOUSE_CLIENT --config "$yaml_config" -q "select getSetting('invalidCertificateHandler')"
-echo 'ini'
-$CLICKHOUSE_CLIENT --config "$ini_config" -q "select getSetting('invalidCertificateHandler')" 2>&1 |& sed -e "s#$CLICKHOUSE_TMP##" -e "s#DB::Exception: ##"
diff --git a/tests/queries/0_stateless/01889_clickhouse_client_config_format.sh b/tests/queries/0_stateless/01889_clickhouse_client_config_format.sh
index 4af4e27653c..11d3c994230 100755
--- a/tests/queries/0_stateless/01889_clickhouse_client_config_format.sh
+++ b/tests/queries/0_stateless/01889_clickhouse_client_config_format.sh
@@ -14,44 +14,45 @@ yml_config=$CLICKHOUSE_TMP/config_$CLICKHOUSE_DATABASE.yml
 yaml_config=$CLICKHOUSE_TMP/config_$CLICKHOUSE_DATABASE.yaml
 ini_config=$CLICKHOUSE_TMP/config_$CLICKHOUSE_DATABASE.ini
 
-function cleanup() {
-	rm "${config:?}"
-	rm "${xml_config:?}"
-	rm "${XML_config:?}"
-	rm "${conf_config:?}"
-	rm "${yml_config:?}"
-	rm "${yaml_config:?}"
-	rm "${ini_config:?}"
+function cleanup()
+{
+    rm "${config:?}"
+    rm "${xml_config:?}"
+    rm "${XML_config:?}"
+    rm "${conf_config:?}"
+    rm "${yml_config:?}"
+    rm "${yaml_config:?}"
+    rm "${ini_config:?}"
 }
 trap cleanup EXIT
 
-cat >"$config" <<EOL
+cat > "$config" <<EOL
 <config>
     <max_threads>2</max_threads>
 </config>
 EOL
-cat >"$conf_config" <<EOL
+cat > "$conf_config" <<EOL
 <config>
     <max_threads>2</max_threads>
 </config>
 EOL
-cat >"$xml_config" <<EOL
+cat > "$xml_config" <<EOL
 <config>
     <max_threads>2</max_threads>
 </config>
 EOL
-cat >"$XML_config" <<EOL
+cat > "$XML_config" <<EOL
 <config>
     <max_threads>2</max_threads>
 </config>
 EOL
-cat >"$yml_config" <<EOL
+cat > "$yml_config" <<EOL
 max_threads: 2
 EOL
-cat >"$yaml_config" <<EOL
+cat > "$yaml_config" <<EOL
 max_threads: 2
 EOL
-cat >"$ini_config" <<EOL
+cat > "$ini_config" <<EOL
 [config]
 max_threads=2
 EOL
@@ -74,4 +75,4 @@ $CLICKHOUSE_CLIENT --config "$yml_config" -q "select getSetting('max_threads')"
 echo 'yaml'
 $CLICKHOUSE_CLIENT --config "$yaml_config" -q "select getSetting('max_threads')"
 echo 'ini'
-$CLICKHOUSE_CLIENT --config "$ini_config" -q "select getSetting('max_threads')" 2>&1 |& sed -e "s#$CLICKHOUSE_TMP##" -e "s#DB::Exception: ##"
+$CLICKHOUSE_CLIENT --config "$ini_config" -q "select getSetting('max_threads')" 2>&1 |& sed -e "s#$CLICKHOUSE_TMP##" -e "s#DB::Exception: ##"
\ No newline at end of file
diff --git a/tests/queries/0_stateless/02550_client_connections_credentials.sh b/tests/queries/0_stateless/02550_client_connections_credentials.sh
index b7319d22d1b..d89fca0fe97 100755
--- a/tests/queries/0_stateless/02550_client_connections_credentials.sh
+++ b/tests/queries/0_stateless/02550_client_connections_credentials.sh
@@ -14,7 +14,7 @@ CLICKHOUSE_PORT_TCP=""
 . "$CUR_DIR"/../shell_config.sh
 
 CONFIG=$CLICKHOUSE_TMP/client.xml
-cat >$CONFIG <<EOL
+cat > $CONFIG <<EOL
 <clickhouse>
     <host>$TEST_HOST</host>
     <port>$TEST_PORT</port>
@@ -66,21 +66,12 @@ cat >$CONFIG <<EOL
             <hostname>$TEST_HOST</hostname>
             <history_file>/no/such/dir/.history</history_file>
         </connection>
-
-        <connection>
-          <name> test_accept_invalid_certificate </name>
-          <hostname> $TEST_HOST </hostname>
-          <database>$CLICKHOUSE_DATABASE</database>
-          <invalidCertificateHandler>
-              <name>RejectCertificateHandler</name>
-          </invalidCertificateHandler>
-        </connection>
     </connections_credentials>
 </clickhouse>
 EOL
 
 CONFIG_ROOT_OVERRIDES=$CLICKHOUSE_TMP/client_user_pass.xml
-cat >$CONFIG_ROOT_OVERRIDES <<EOL
+cat > $CONFIG_ROOT_OVERRIDES <<EOL
 <clickhouse>
     <host>$TEST_HOST</host>
     <port>$TEST_PORT</port>
@@ -140,4 +131,4 @@ $CLICKHOUSE_CLIENT --config $CONFIG_ROOT_OVERRIDES --connection incorrect_auth -
 $CLICKHOUSE_CLIENT --config $CONFIG_ROOT_OVERRIDES --connection default -q 'select currentUser()'
 $CLICKHOUSE_CLIENT --config $CONFIG_ROOT_OVERRIDES --connection default --user foo -q 'select currentUser()' |& grep -F -o 'foo: Authentication failed: password is incorrect, or there is no user with such name.'
 
-rm -f "${CONFIG:?}"
+rm -f "${CONFIG:?}"
\ No newline at end of file

From 8673add9909211b3493e004ff6683d3124830f67 Mon Sep 17 00:00:00 2001
From: Yakov Olkhovskiy <yakov@clickhouse.com>
Date: Mon, 1 Jul 2024 06:06:03 +0000
Subject: [PATCH 0126/1488] restore unrelated tests

---
 .../0_stateless/01889_clickhouse_client_config_format.sh        | 2 +-
 .../queries/0_stateless/02550_client_connections_credentials.sh | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/queries/0_stateless/01889_clickhouse_client_config_format.sh b/tests/queries/0_stateless/01889_clickhouse_client_config_format.sh
index 11d3c994230..4e970f17d3a 100755
--- a/tests/queries/0_stateless/01889_clickhouse_client_config_format.sh
+++ b/tests/queries/0_stateless/01889_clickhouse_client_config_format.sh
@@ -75,4 +75,4 @@ $CLICKHOUSE_CLIENT --config "$yml_config" -q "select getSetting('max_threads')"
 echo 'yaml'
 $CLICKHOUSE_CLIENT --config "$yaml_config" -q "select getSetting('max_threads')"
 echo 'ini'
-$CLICKHOUSE_CLIENT --config "$ini_config" -q "select getSetting('max_threads')" 2>&1 |& sed -e "s#$CLICKHOUSE_TMP##" -e "s#DB::Exception: ##"
\ No newline at end of file
+$CLICKHOUSE_CLIENT --config "$ini_config" -q "select getSetting('max_threads')" 2>&1 |& sed -e "s#$CLICKHOUSE_TMP##" -e "s#DB::Exception: ##"
diff --git a/tests/queries/0_stateless/02550_client_connections_credentials.sh b/tests/queries/0_stateless/02550_client_connections_credentials.sh
index d89fca0fe97..a88f3fc7880 100755
--- a/tests/queries/0_stateless/02550_client_connections_credentials.sh
+++ b/tests/queries/0_stateless/02550_client_connections_credentials.sh
@@ -131,4 +131,4 @@ $CLICKHOUSE_CLIENT --config $CONFIG_ROOT_OVERRIDES --connection incorrect_auth -
 $CLICKHOUSE_CLIENT --config $CONFIG_ROOT_OVERRIDES --connection default -q 'select currentUser()'
 $CLICKHOUSE_CLIENT --config $CONFIG_ROOT_OVERRIDES --connection default --user foo -q 'select currentUser()' |& grep -F -o 'foo: Authentication failed: password is incorrect, or there is no user with such name.'
 
-rm -f "${CONFIG:?}"
\ No newline at end of file
+rm -f "${CONFIG:?}"

From 7c796113e83373063675a2c4189fe3ce115897f0 Mon Sep 17 00:00:00 2001
From: Yakov Olkhovskiy <yakov@clickhouse.com>
Date: Mon, 1 Jul 2024 06:34:05 +0000
Subject: [PATCH 0127/1488] fix style

---
 .../test_accept_invalid_certificate/test.py   | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/tests/integration/test_accept_invalid_certificate/test.py b/tests/integration/test_accept_invalid_certificate/test.py
index 13431f39854..87229d75f90 100644
--- a/tests/integration/test_accept_invalid_certificate/test.py
+++ b/tests/integration/test_accept_invalid_certificate/test.py
@@ -19,6 +19,7 @@ instance = cluster.add_instance(
     ],
 )
 
+
 @pytest.fixture(scope="module", autouse=True)
 def started_cluster():
     try:
@@ -28,6 +29,7 @@ def started_cluster():
     finally:
         cluster.shutdown()
 
+
 config_default = """<clickhouse>
 </clickhouse>"""
 
@@ -71,23 +73,20 @@ def execute_query_native(node, query, config):
 
 def test_default():
     with pytest.raises(Exception) as err:
-        execute_query_native(
-            instance, "SELECT 1", config_default
-        )
+        execute_query_native(instance, "SELECT 1", config_default)
     assert "certificate verify failed" in str(err.value)
 
+
 def test_accept():
-    assert (
-        execute_query_native(
-            instance, "SELECT 1", config_accept
-        )
-        == "1\n"
-    )
+    assert execute_query_native(instance, "SELECT 1", config_accept) == "1\n"
+
 
 def test_connection_accept():
     assert (
         execute_query_native(
-            instance, "SELECT 1", config_connection_accept.format(ip_address=f"{instance.ip_address}")
+            instance,
+            "SELECT 1",
+            config_connection_accept.format(ip_address=f"{instance.ip_address}"),
         )
         == "1\n"
     )

From f1105374cb30d6eb3323e25fe9a4bdcbe9e5ba1f Mon Sep 17 00:00:00 2001
From: Yakov Olkhovskiy <yakov@clickhouse.com>
Date: Mon, 1 Jul 2024 12:28:39 +0000
Subject: [PATCH 0128/1488] fix style

---
 tests/integration/test_accept_invalid_certificate/__init__.py | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 create mode 100644 tests/integration/test_accept_invalid_certificate/__init__.py

diff --git a/tests/integration/test_accept_invalid_certificate/__init__.py b/tests/integration/test_accept_invalid_certificate/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d

From 85baa91ba4774c449ef72d7f42278397598205b9 Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <nikitamikhaylov@clickhouse.com>
Date: Mon, 1 Jul 2024 15:26:19 +0200
Subject: [PATCH 0129/1488] Added spell exception

---
 .../aspell-ignore/en/aspell-dict.txt          | 21 ++++++++++---------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt
index 2bd949f102d..d100b1bc2d9 100644
--- a/utils/check-style/aspell-ignore/en/aspell-dict.txt
+++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt
@@ -48,7 +48,6 @@ AutoML
 Autocompletion
 AvroConfluent
 BIGINT
-bigrams
 BIGSERIAL
 BORO
 BSON
@@ -223,7 +222,6 @@ DatabaseOrdinaryThreadsActive
 DateTime
 DateTimes
 DbCL
-deallocated
 Decrypted
 Deduplicate
 Deduplication
@@ -295,7 +293,6 @@ FilesystemMainPathUsedBytes
 FilesystemMainPathUsedINodes
 FixedString
 FlameGraph
-flameGraph
 Flink
 ForEach
 FreeBSD
@@ -1009,7 +1006,6 @@ UncompressedCacheBytes
 UncompressedCacheCells
 UnidirectionalEdgeIsValid
 UniqThetaSketch
-unigrams
 Updatable
 Uppercased
 Uptime
@@ -1221,6 +1217,7 @@ basename
 bcrypt
 benchmarking
 bfloat
+bigrams
 binlog
 bitAnd
 bitCount
@@ -1470,6 +1467,7 @@ dbeaver
 dbgen
 dbms
 ddl
+deallocated
 deallocation
 deallocations
 debian
@@ -1509,11 +1507,11 @@ deserializing
 destructor
 destructors
 detectCharset
-detectTonality
 detectLanguage
 detectLanguageMixed
 detectLanguageUnknown
 detectProgrammingLanguage
+detectTonality
 determinator
 deterministically
 dictGet
@@ -1529,8 +1527,8 @@ dictIsIn
 disableProtocols
 disjunction
 disjunctions
-displaySecretsInShowAndSelect
 displayName
+displaySecretsInShowAndSelect
 distro
 divideDecimal
 dmesg
@@ -1580,11 +1578,11 @@ evalMLMethod
 exFAT
 expiryMsec
 exponentialMovingAverage
-exponentialmovingaverage
 exponentialTimeDecayedAvg
 exponentialTimeDecayedCount
 exponentialTimeDecayedMax
 exponentialTimeDecayedSum
+exponentialmovingaverage
 expr
 exprN
 extendedVerification
@@ -1621,6 +1619,7 @@ firstSignificantSubdomainCustom
 firstSignificantSubdomainCustomRFC
 firstSignificantSubdomainRFC
 fixedstring
+flameGraph
 flamegraph
 flatbuffers
 flattenTuple
@@ -1803,8 +1802,8 @@ incrementing
 indexHint
 indexOf
 infi
-infty
 inflight
+infty
 initcap
 initcapUTF
 initialQueryID
@@ -1952,9 +1951,9 @@ loghouse
 london
 lookups
 loongarch
-lowcardinality
 lowCardinalityIndices
 lowCardinalityKeys
+lowcardinality
 lowerUTF
 lowercased
 lttb
@@ -2262,9 +2261,9 @@ proleptic
 prometheus
 proportionsZTest
 proto
-protocol
 protobuf
 protobufsingle
+protocol
 proxied
 pseudorandom
 pseudorandomize
@@ -2516,6 +2515,7 @@ sqlite
 sqrt
 src
 srcReplicas
+sshkey
 stacktrace
 stacktraces
 startsWith
@@ -2808,6 +2808,7 @@ unescaping
 unhex
 unicode
 unidimensional
+unigrams
 unintuitive
 uniq
 uniqCombined

From 74a5d56f1a404a7271b5a177e4902d0c319195d9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Mon, 1 Jul 2024 19:32:22 +0200
Subject: [PATCH 0130/1488] Fix interpolate and add tests

---
 src/Analyzer/Resolve/QueryAnalyzer.cpp        |  4 +-
 src/Functions/IFunction.cpp                   |  6 ++
 src/Planner/Planner.cpp                       |  7 +-
 src/Planner/PlannerActionsVisitor.cpp         | 36 ++++++---
 src/Planner/PlannerActionsVisitor.h           |  9 ++-
 ..._no_aggregates_and_constant_keys.reference |  4 +-
 ...nality_group_by_distributed_plan.reference | 55 +++++++++++++
 ..._cardinality_group_by_distributed_plan.sql | 80 +++++++++++++++++++
 8 files changed, 185 insertions(+), 16 deletions(-)
 create mode 100644 tests/queries/0_stateless/03174_low_cardinality_group_by_distributed_plan.reference
 create mode 100644 tests/queries/0_stateless/03174_low_cardinality_group_by_distributed_plan.sql

diff --git a/src/Analyzer/Resolve/QueryAnalyzer.cpp b/src/Analyzer/Resolve/QueryAnalyzer.cpp
index 8860050c5b9..165256479ce 100644
--- a/src/Analyzer/Resolve/QueryAnalyzer.cpp
+++ b/src/Analyzer/Resolve/QueryAnalyzer.cpp
@@ -4100,7 +4100,9 @@ void QueryAnalyzer::resolveInterpolateColumnsNodeList(QueryTreeNodePtr & interpo
 
         auto * column_to_interpolate = interpolate_node_typed.getExpression()->as<IdentifierNode>();
         if (!column_to_interpolate)
-            throw Exception(ErrorCodes::LOGICAL_ERROR, "INTERPOLATE can work only for indentifiers, but {} is found",
+            throw Exception(
+                ErrorCodes::LOGICAL_ERROR,
+                "INTERPOLATE can work only for identifiers, but {} is found",
                 interpolate_node_typed.getExpression()->formatASTForErrorMessage());
         auto column_to_interpolate_name = column_to_interpolate->getIdentifier().getFullName();
 
diff --git a/src/Functions/IFunction.cpp b/src/Functions/IFunction.cpp
index 76ae8f33fbd..8b092ba9b6e 100644
--- a/src/Functions/IFunction.cpp
+++ b/src/Functions/IFunction.cpp
@@ -80,8 +80,14 @@ ColumnPtr replaceLowCardinalityColumnsByNestedAndGetDictionaryIndexes(
 
     if (number_full_columns > 0 || number_low_cardinality_columns > 1)
     {
+        /// This should not be possible but currently there are multiple tests in CI failing because of it
+        /// TODO: Fix those cases, then enable this exception
+#if 0
         throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected low cardinality types found. Low cardinality: {}. Full {}. Const {}",
                 number_low_cardinality_columns, number_full_columns, number_const_columns);
+#else
+        return nullptr;
+#endif
     }
     else if (number_low_cardinality_columns == 1)
     {
diff --git a/src/Planner/Planner.cpp b/src/Planner/Planner.cpp
index 2d42ed73223..93a4ea01ff0 100644
--- a/src/Planner/Planner.cpp
+++ b/src/Planner/Planner.cpp
@@ -742,7 +742,12 @@ void addWithFillStepIfNeeded(QueryPlan & query_plan,
             {
                 auto & interpolate_node_typed = interpolate_node->as<InterpolateNode &>();
 
-                PlannerActionsVisitor planner_actions_visitor(planner_context);
+                PlannerActionsVisitor planner_actions_visitor(
+                    planner_context,
+                    /* use_column_identifier_as_action_node_name_, (default value)*/ true,
+                    /// Prefer the INPUT to CONSTANT nodes (actions must be non constant)
+                    /* prefer_const_column_to_input */ false);
+
                 auto expression_to_interpolate_expression_nodes = planner_actions_visitor.visit(*interpolate_actions_dag,
                     interpolate_node_typed.getExpression());
                 if (expression_to_interpolate_expression_nodes.size() != 1)
diff --git a/src/Planner/PlannerActionsVisitor.cpp b/src/Planner/PlannerActionsVisitor.cpp
index 1c9553032c2..a199420c9bd 100644
--- a/src/Planner/PlannerActionsVisitor.cpp
+++ b/src/Planner/PlannerActionsVisitor.cpp
@@ -485,19 +485,24 @@ public:
         return node;
     }
 
-    [[nodiscard]] String addConstantIfNecessary(const std::string & node_name, const ColumnWithTypeAndName & column)
+    [[nodiscard]] String
+    addConstantIfNecessary(const std::string & node_name, const ColumnWithTypeAndName & column, bool prefer_const_column_to_input)
     {
+        chassert(column.column != nullptr);
         auto it = node_name_to_node.find(node_name);
-        if (it != node_name_to_node.end() && it->second->column)
+        if (it != node_name_to_node.end() && (!prefer_const_column_to_input || it->second->column))
             return {node_name};
 
         if (it != node_name_to_node.end())
         {
             /// There is a node with this name, but it doesn't have a column
             /// This likely happens because we executed the query until WithMergeableState with a const node in the
-            /// WHERE clause. As the results of headers are materialized, the column was removed
+            /// WHERE clause and, as the results of headers are materialized, the column was removed
             /// Let's add a new column and keep this
             String dupped_name{node_name + "_dupped"};
+            if (node_name_to_node.find(dupped_name) != node_name_to_node.end())
+                return dupped_name;
+
             const auto * node = &actions_dag.addColumn(column);
             node_name_to_node[dupped_name] = node;
             return dupped_name;
@@ -543,9 +548,11 @@ private:
 class PlannerActionsVisitorImpl
 {
 public:
-    PlannerActionsVisitorImpl(ActionsDAG & actions_dag,
+    PlannerActionsVisitorImpl(
+        ActionsDAG & actions_dag,
         const PlannerContextPtr & planner_context_,
-        bool use_column_identifier_as_action_node_name_);
+        bool use_column_identifier_as_action_node_name_,
+        bool prefer_const_column_to_input_);
 
     ActionsDAG::NodeRawConstPtrs visit(QueryTreeNodePtr expression_node);
 
@@ -605,14 +612,18 @@ private:
     const PlannerContextPtr planner_context;
     ActionNodeNameHelper action_node_name_helper;
     bool use_column_identifier_as_action_node_name;
+    bool prefer_const_column_to_input;
 };
 
-PlannerActionsVisitorImpl::PlannerActionsVisitorImpl(ActionsDAG & actions_dag,
+PlannerActionsVisitorImpl::PlannerActionsVisitorImpl(
+    ActionsDAG & actions_dag,
     const PlannerContextPtr & planner_context_,
-    bool use_column_identifier_as_action_node_name_)
+    bool use_column_identifier_as_action_node_name_,
+    bool prefer_const_column_to_input_)
     : planner_context(planner_context_)
     , action_node_name_helper(node_to_node_name, *planner_context, use_column_identifier_as_action_node_name_)
     , use_column_identifier_as_action_node_name(use_column_identifier_as_action_node_name_)
+    , prefer_const_column_to_input(prefer_const_column_to_input_)
 {
     actions_stack.emplace_back(actions_dag, nullptr);
 }
@@ -735,7 +746,7 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::vi
     column.type = constant_type;
     column.column = column.type->createColumnConst(1, constant_literal);
 
-    String final_name = actions_stack[0].addConstantIfNecessary(constant_node_name, column);
+    String final_name = actions_stack[0].addConstantIfNecessary(constant_node_name, column, prefer_const_column_to_input);
 
     size_t actions_stack_size = actions_stack.size();
     for (size_t i = 1; i < actions_stack_size; ++i)
@@ -873,7 +884,7 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::ma
     else
         column.column = std::move(column_set);
 
-    String final_name = actions_stack[0].addConstantIfNecessary(column.name, column);
+    String final_name = actions_stack[0].addConstantIfNecessary(column.name, column, prefer_const_column_to_input);
 
     size_t actions_stack_size = actions_stack.size();
     for (size_t i = 1; i < actions_stack_size; ++i)
@@ -1019,14 +1030,17 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::vi
 
 }
 
-PlannerActionsVisitor::PlannerActionsVisitor(const PlannerContextPtr & planner_context_, bool use_column_identifier_as_action_node_name_)
+PlannerActionsVisitor::PlannerActionsVisitor(
+    const PlannerContextPtr & planner_context_, bool use_column_identifier_as_action_node_name_, bool prefer_const_column_to_input_)
     : planner_context(planner_context_)
     , use_column_identifier_as_action_node_name(use_column_identifier_as_action_node_name_)
+    , prefer_const_column_to_input(prefer_const_column_to_input_)
 {}
 
 ActionsDAG::NodeRawConstPtrs PlannerActionsVisitor::visit(ActionsDAG & actions_dag, QueryTreeNodePtr expression_node)
 {
-    PlannerActionsVisitorImpl actions_visitor_impl(actions_dag, planner_context, use_column_identifier_as_action_node_name);
+    PlannerActionsVisitorImpl actions_visitor_impl(
+        actions_dag, planner_context, use_column_identifier_as_action_node_name, prefer_const_column_to_input);
     return actions_visitor_impl.visit(expression_node);
 }
 
diff --git a/src/Planner/PlannerActionsVisitor.h b/src/Planner/PlannerActionsVisitor.h
index 6bb32047327..4bec2d2bb8a 100644
--- a/src/Planner/PlannerActionsVisitor.h
+++ b/src/Planner/PlannerActionsVisitor.h
@@ -27,11 +27,17 @@ using PlannerContextPtr = std::shared_ptr<PlannerContext>;
   * During actions build, there is special handling for following functions:
   * 1. Aggregate functions are added in actions dag as INPUT nodes. Aggregate functions arguments are not added.
   * 2. For function `in` and its variants, already collected sets from planner context are used.
+  * 3. When building actions that use CONSTANT nodes, by default we ignore pre-existing INPUTs if those don't have
+  * a column (a const column always has a column). This is for compatibility with previous headers. We disable this
+  * behaviour when we explicitly want to override CONSTANT nodes with the input (resolving InterpolateNode for example)
   */
 class PlannerActionsVisitor
 {
 public:
-    explicit PlannerActionsVisitor(const PlannerContextPtr & planner_context_, bool use_column_identifier_as_action_node_name_ = true);
+    explicit PlannerActionsVisitor(
+        const PlannerContextPtr & planner_context_,
+        bool use_column_identifier_as_action_node_name_ = true,
+        bool prefer_const_column_to_input_ = true);
 
     /** Add actions necessary to calculate expression node into expression dag.
       * Necessary actions are not added in actions dag output.
@@ -42,6 +48,7 @@ public:
 private:
     const PlannerContextPtr planner_context;
     bool use_column_identifier_as_action_node_name = true;
+    bool prefer_const_column_to_input = true;
 };
 
 /** Calculate query tree expression node action dag name and add them into node to name map.
diff --git a/tests/queries/0_stateless/00257_shard_no_aggregates_and_constant_keys.reference b/tests/queries/0_stateless/00257_shard_no_aggregates_and_constant_keys.reference
index 63b8a9d14fc..fc77ed8a241 100644
--- a/tests/queries/0_stateless/00257_shard_no_aggregates_and_constant_keys.reference
+++ b/tests/queries/0_stateless/00257_shard_no_aggregates_and_constant_keys.reference
@@ -8,13 +8,13 @@
 40
 41
 
-0
+41
 2	42
 
 2	42
 43
 
-0
+43
 11
 
 11
diff --git a/tests/queries/0_stateless/03174_low_cardinality_group_by_distributed_plan.reference b/tests/queries/0_stateless/03174_low_cardinality_group_by_distributed_plan.reference
new file mode 100644
index 00000000000..1508c24f410
--- /dev/null
+++ b/tests/queries/0_stateless/03174_low_cardinality_group_by_distributed_plan.reference
@@ -0,0 +1,55 @@
+-- { echoOn }
+SELECT concatWithSeparator('.', toUInt128(6), '666' as b, materialize(toLowCardinality(8)))
+FROM system.one
+GROUP BY '666';
+6.666.8
+SELECT concatWithSeparator('.', toUInt128(6), '666' as b, materialize(toLowCardinality(8)))
+FROM remote('127.0.0.{1,1}', 'system.one')
+GROUP BY '666';
+6.666.8
+-- https://github.com/ClickHouse/ClickHouse/issues/63006
+SELECT
+    6,
+    concat(' World', toUInt128(6), 6, 6, 6, toNullable(6), materialize(toLowCardinality(toNullable(toUInt128(6))))) AS a,
+    concat(concat(' World', 6, toLowCardinality(6), ' World', toUInt256(6), materialize(6), 6, toNullable(6), 6, 6, NULL, 6, 6), ' World', 6, 6, 6, 6, toUInt256(6), NULL, 6, 6) AS b
+FROM system.one
+GROUP BY toNullable(6)
+    WITH ROLLUP
+WITH TOTALS;
+6	 World666666	\N
+6	 World666666	\N
+
+6	 World666666	\N
+SELECT
+    6,
+    concat(' World', toUInt128(6), 6, 6, 6, toNullable(6), materialize(toLowCardinality(toNullable(toUInt128(6))))) AS a,
+    concat(concat(' World', 6, toLowCardinality(6), ' World', toUInt256(6), materialize(6), 6, toNullable(6), 6, 6, NULL, 6, 6), ' World', 6, 6, 6, 6, toUInt256(6), NULL, 6, 6) AS b
+FROM remote('127.0.0.1')
+GROUP BY toNullable(6)
+    WITH ROLLUP
+    WITH TOTALS;
+6	 World666666	\N
+6	 World666666	\N
+
+6	 World666666	\N
+-- { echoOn }
+SELECT
+    '%',
+    tuple(concat('%', 1, toLowCardinality(toLowCardinality(toNullable(materialize(1)))), currentDatabase(), 101., toNullable(13), '%AS%id_02%', toNullable(toNullable(10)), toLowCardinality(toNullable(10)), 10, 10)),
+    (toDecimal128(99.67, 6), 36, 61, 14)
+FROM dist_03174
+WHERE dummy IN (0, '255')
+GROUP BY
+    toNullable(13),
+    (99.67, 61, toLowCardinality(14));
+%	('%11default10113%AS%id_02%10101010')	(99.67,36,61,14)
+-- { echoOn }
+SELECT
+    38,
+    concat(position(concat(concat(position(concat(toUInt256(3)), 'ca', 2), 3), NULLIF(1, materialize(toLowCardinality(1)))), toLowCardinality(toNullable('ca'))), concat(NULLIF(1, 1), concat(3), toNullable(3)))
+FROM set_index_not__fuzz_0
+GROUP BY
+    toNullable(3),
+    concat(concat(CAST(NULL, 'Nullable(Int8)'), toNullable(3)))
+FORMAT Null
+SETTINGS max_threads = 1, allow_experimental_analyzer = 1, cluster_for_parallel_replicas = 'parallel_replicas', max_parallel_replicas = 3, allow_experimental_parallel_reading_from_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, max_threads = 1;
diff --git a/tests/queries/0_stateless/03174_low_cardinality_group_by_distributed_plan.sql b/tests/queries/0_stateless/03174_low_cardinality_group_by_distributed_plan.sql
new file mode 100644
index 00000000000..d397d30e285
--- /dev/null
+++ b/tests/queries/0_stateless/03174_low_cardinality_group_by_distributed_plan.sql
@@ -0,0 +1,80 @@
+-- There are various tests that check that group by keys don't propagate into functions replacing const arguments
+-- by full (empty) columns
+
+DROP TABLE IF EXISTS dist_03174;
+DROP TABLE IF EXISTS set_index_not__fuzz_0;
+
+-- https://github.com/ClickHouse/ClickHouse/issues/63006
+
+SET allow_experimental_analyzer=1;
+
+-- { echoOn }
+SELECT concatWithSeparator('.', toUInt128(6), '666' as b, materialize(toLowCardinality(8)))
+FROM system.one
+GROUP BY '666';
+
+SELECT concatWithSeparator('.', toUInt128(6), '666' as b, materialize(toLowCardinality(8)))
+FROM remote('127.0.0.{1,1}', 'system.one')
+GROUP BY '666';
+
+-- https://github.com/ClickHouse/ClickHouse/issues/63006
+SELECT
+    6,
+    concat(' World', toUInt128(6), 6, 6, 6, toNullable(6), materialize(toLowCardinality(toNullable(toUInt128(6))))) AS a,
+    concat(concat(' World', 6, toLowCardinality(6), ' World', toUInt256(6), materialize(6), 6, toNullable(6), 6, 6, NULL, 6, 6), ' World', 6, 6, 6, 6, toUInt256(6), NULL, 6, 6) AS b
+FROM system.one
+GROUP BY toNullable(6)
+    WITH ROLLUP
+WITH TOTALS;
+
+SELECT
+    6,
+    concat(' World', toUInt128(6), 6, 6, 6, toNullable(6), materialize(toLowCardinality(toNullable(toUInt128(6))))) AS a,
+    concat(concat(' World', 6, toLowCardinality(6), ' World', toUInt256(6), materialize(6), 6, toNullable(6), 6, 6, NULL, 6, 6), ' World', 6, 6, 6, 6, toUInt256(6), NULL, 6, 6) AS b
+FROM remote('127.0.0.1')
+GROUP BY toNullable(6)
+    WITH ROLLUP
+    WITH TOTALS;
+
+-- https://github.com/ClickHouse/ClickHouse/issues/64945
+-- { echoOff }
+CREATE TABLE dist_03174 AS system.one ENGINE = Distributed(test_cluster_two_shards, system, one, dummy);
+
+-- { echoOn }
+SELECT
+    '%',
+    tuple(concat('%', 1, toLowCardinality(toLowCardinality(toNullable(materialize(1)))), currentDatabase(), 101., toNullable(13), '%AS%id_02%', toNullable(toNullable(10)), toLowCardinality(toNullable(10)), 10, 10)),
+    (toDecimal128(99.67, 6), 36, 61, 14)
+FROM dist_03174
+WHERE dummy IN (0, '255')
+GROUP BY
+    toNullable(13),
+    (99.67, 61, toLowCardinality(14));
+
+-- Parallel replicas
+-- { echoOff }
+CREATE TABLE set_index_not__fuzz_0
+(
+    `name` String,
+    `status` Enum8('alive' = 0, 'rip' = 1),
+    INDEX idx_status status TYPE set(2) GRANULARITY 1
+)
+ENGINE = MergeTree()
+ORDER BY name;
+
+INSERT INTO set_index_not__fuzz_0 SELECT * FROM generateRandom() LIMIT 10;
+
+-- { echoOn }
+SELECT
+    38,
+    concat(position(concat(concat(position(concat(toUInt256(3)), 'ca', 2), 3), NULLIF(1, materialize(toLowCardinality(1)))), toLowCardinality(toNullable('ca'))), concat(NULLIF(1, 1), concat(3), toNullable(3)))
+FROM set_index_not__fuzz_0
+GROUP BY
+    toNullable(3),
+    concat(concat(CAST(NULL, 'Nullable(Int8)'), toNullable(3)))
+FORMAT Null
+SETTINGS max_threads = 1, allow_experimental_analyzer = 1, cluster_for_parallel_replicas = 'parallel_replicas', max_parallel_replicas = 3, allow_experimental_parallel_reading_from_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, max_threads = 1;
+
+-- { echoOff }
+DROP TABLE IF EXISTS dist_03174;
+DROP TABLE IF EXISTS set_index_not__fuzz_0;

From 679da847da0b9593d5187997ef55116424b63b8e Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Tue, 2 Jul 2024 05:32:12 +0200
Subject: [PATCH 0131/1488] Fix tests

---
 src/Databases/DatabaseAtomic.cpp | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/Databases/DatabaseAtomic.cpp b/src/Databases/DatabaseAtomic.cpp
index bebb645670b..c06fc98d0b9 100644
--- a/src/Databases/DatabaseAtomic.cpp
+++ b/src/Databases/DatabaseAtomic.cpp
@@ -195,6 +195,9 @@ void DatabaseAtomic::renameTable(ContextPtr local_context, const String & table_
     auto & other_db = dynamic_cast<DatabaseAtomic &>(to_database);
     bool inside_database = this == &other_db;
 
+    if (!inside_database)
+        other_db.createDirectories();
+
     String old_metadata_path = getObjectMetadataPath(table_name);
     String new_metadata_path = to_database.getObjectMetadataPath(to_table_name);
 

From 0e559ff7b94c55fe35d0db7174e523180f46e998 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Tue, 2 Jul 2024 11:50:43 +0200
Subject: [PATCH 0132/1488] Better name for flag

---
 src/Planner/Planner.cpp               |  2 +-
 src/Planner/PlannerActionsVisitor.cpp | 26 ++++++++++++++------------
 src/Planner/PlannerActionsVisitor.h   |  4 ++--
 3 files changed, 17 insertions(+), 15 deletions(-)

diff --git a/src/Planner/Planner.cpp b/src/Planner/Planner.cpp
index 93a4ea01ff0..260fbabf26a 100644
--- a/src/Planner/Planner.cpp
+++ b/src/Planner/Planner.cpp
@@ -746,7 +746,7 @@ void addWithFillStepIfNeeded(QueryPlan & query_plan,
                     planner_context,
                     /* use_column_identifier_as_action_node_name_, (default value)*/ true,
                     /// Prefer the INPUT to CONSTANT nodes (actions must be non constant)
-                    /* prefer_const_column_to_input */ false);
+                    /* always_use_const_column_for_constant_nodes */ false);
 
                 auto expression_to_interpolate_expression_nodes = planner_actions_visitor.visit(*interpolate_actions_dag,
                     interpolate_node_typed.getExpression());
diff --git a/src/Planner/PlannerActionsVisitor.cpp b/src/Planner/PlannerActionsVisitor.cpp
index a199420c9bd..e9e1f4edcc2 100644
--- a/src/Planner/PlannerActionsVisitor.cpp
+++ b/src/Planner/PlannerActionsVisitor.cpp
@@ -485,12 +485,12 @@ public:
         return node;
     }
 
-    [[nodiscard]] String
-    addConstantIfNecessary(const std::string & node_name, const ColumnWithTypeAndName & column, bool prefer_const_column_to_input)
+    [[nodiscard]] String addConstantIfNecessary(
+        const std::string & node_name, const ColumnWithTypeAndName & column, bool always_use_const_column_for_constant_nodes)
     {
         chassert(column.column != nullptr);
         auto it = node_name_to_node.find(node_name);
-        if (it != node_name_to_node.end() && (!prefer_const_column_to_input || it->second->column))
+        if (it != node_name_to_node.end() && (!always_use_const_column_for_constant_nodes || it->second->column))
             return {node_name};
 
         if (it != node_name_to_node.end())
@@ -552,7 +552,7 @@ public:
         ActionsDAG & actions_dag,
         const PlannerContextPtr & planner_context_,
         bool use_column_identifier_as_action_node_name_,
-        bool prefer_const_column_to_input_);
+        bool always_use_const_column_for_constant_nodes_);
 
     ActionsDAG::NodeRawConstPtrs visit(QueryTreeNodePtr expression_node);
 
@@ -612,18 +612,18 @@ private:
     const PlannerContextPtr planner_context;
     ActionNodeNameHelper action_node_name_helper;
     bool use_column_identifier_as_action_node_name;
-    bool prefer_const_column_to_input;
+    bool always_use_const_column_for_constant_nodes;
 };
 
 PlannerActionsVisitorImpl::PlannerActionsVisitorImpl(
     ActionsDAG & actions_dag,
     const PlannerContextPtr & planner_context_,
     bool use_column_identifier_as_action_node_name_,
-    bool prefer_const_column_to_input_)
+    bool always_use_const_column_for_constant_nodes_)
     : planner_context(planner_context_)
     , action_node_name_helper(node_to_node_name, *planner_context, use_column_identifier_as_action_node_name_)
     , use_column_identifier_as_action_node_name(use_column_identifier_as_action_node_name_)
-    , prefer_const_column_to_input(prefer_const_column_to_input_)
+    , always_use_const_column_for_constant_nodes(always_use_const_column_for_constant_nodes_)
 {
     actions_stack.emplace_back(actions_dag, nullptr);
 }
@@ -746,7 +746,7 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::vi
     column.type = constant_type;
     column.column = column.type->createColumnConst(1, constant_literal);
 
-    String final_name = actions_stack[0].addConstantIfNecessary(constant_node_name, column, prefer_const_column_to_input);
+    String final_name = actions_stack[0].addConstantIfNecessary(constant_node_name, column, always_use_const_column_for_constant_nodes);
 
     size_t actions_stack_size = actions_stack.size();
     for (size_t i = 1; i < actions_stack_size; ++i)
@@ -884,7 +884,7 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::ma
     else
         column.column = std::move(column_set);
 
-    String final_name = actions_stack[0].addConstantIfNecessary(column.name, column, prefer_const_column_to_input);
+    String final_name = actions_stack[0].addConstantIfNecessary(column.name, column, always_use_const_column_for_constant_nodes);
 
     size_t actions_stack_size = actions_stack.size();
     for (size_t i = 1; i < actions_stack_size; ++i)
@@ -1031,16 +1031,18 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::vi
 }
 
 PlannerActionsVisitor::PlannerActionsVisitor(
-    const PlannerContextPtr & planner_context_, bool use_column_identifier_as_action_node_name_, bool prefer_const_column_to_input_)
+    const PlannerContextPtr & planner_context_,
+    bool use_column_identifier_as_action_node_name_,
+    bool always_use_const_column_for_constant_nodes_)
     : planner_context(planner_context_)
     , use_column_identifier_as_action_node_name(use_column_identifier_as_action_node_name_)
-    , prefer_const_column_to_input(prefer_const_column_to_input_)
+    , always_use_const_column_for_constant_nodes(always_use_const_column_for_constant_nodes_)
 {}
 
 ActionsDAG::NodeRawConstPtrs PlannerActionsVisitor::visit(ActionsDAG & actions_dag, QueryTreeNodePtr expression_node)
 {
     PlannerActionsVisitorImpl actions_visitor_impl(
-        actions_dag, planner_context, use_column_identifier_as_action_node_name, prefer_const_column_to_input);
+        actions_dag, planner_context, use_column_identifier_as_action_node_name, always_use_const_column_for_constant_nodes);
     return actions_visitor_impl.visit(expression_node);
 }
 
diff --git a/src/Planner/PlannerActionsVisitor.h b/src/Planner/PlannerActionsVisitor.h
index 4bec2d2bb8a..1dbd149bc4b 100644
--- a/src/Planner/PlannerActionsVisitor.h
+++ b/src/Planner/PlannerActionsVisitor.h
@@ -37,7 +37,7 @@ public:
     explicit PlannerActionsVisitor(
         const PlannerContextPtr & planner_context_,
         bool use_column_identifier_as_action_node_name_ = true,
-        bool prefer_const_column_to_input_ = true);
+        bool always_use_const_column_for_constant_nodes_ = true);
 
     /** Add actions necessary to calculate expression node into expression dag.
       * Necessary actions are not added in actions dag output.
@@ -48,7 +48,7 @@ public:
 private:
     const PlannerContextPtr planner_context;
     bool use_column_identifier_as_action_node_name = true;
-    bool prefer_const_column_to_input = true;
+    bool always_use_const_column_for_constant_nodes = true;
 };
 
 /** Calculate query tree expression node action dag name and add them into node to name map.

From 891f449fbdcc759053238971d979dda244c1af09 Mon Sep 17 00:00:00 2001
From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com>
Date: Tue, 2 Jul 2024 14:47:18 +0200
Subject: [PATCH 0133/1488] Update aspell-dict.txt

---
 utils/check-style/aspell-ignore/en/aspell-dict.txt | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt
index 9d5ae17b156..bf2b3794d1e 100644
--- a/utils/check-style/aspell-ignore/en/aspell-dict.txt
+++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt
@@ -759,6 +759,7 @@ PromQL
 Promql
 Promtail
 Protobuf
+ProtobufList
 ProtobufSingle
 ProxySQL
 Punycode
@@ -2201,6 +2202,7 @@ proleptic
 prometheus
 proto
 protobuf
+protobuflist
 protobufsingle
 proxied
 pseudorandom

From f76bd8ed776bc34829a7ff2de394936dc6035d01 Mon Sep 17 00:00:00 2001
From: Konstantin Morozov <just.morozov.k@gmail.com>
Date: Tue, 2 Jul 2024 14:58:04 +0000
Subject: [PATCH 0134/1488] apply some comments

---
 src/Databases/DatabaseAtomic.cpp                     |  6 ------
 src/Databases/DatabaseAtomic.h                       |  3 ---
 src/Databases/DatabaseOnDisk.cpp                     |  9 ++++++---
 src/Databases/TablesLoader.cpp                       | 10 +++++-----
 src/Storages/System/StorageSystemTables.cpp          | 12 ------------
 .../0_stateless/03172_system_detached_tables.sql     |  2 +-
 6 files changed, 12 insertions(+), 30 deletions(-)

diff --git a/src/Databases/DatabaseAtomic.cpp b/src/Databases/DatabaseAtomic.cpp
index ddc30627cde..b8d7315fc0f 100644
--- a/src/Databases/DatabaseAtomic.cpp
+++ b/src/Databases/DatabaseAtomic.cpp
@@ -437,12 +437,6 @@ DatabaseAtomic::getTablesIterator(ContextPtr local_context, const IDatabase::Fil
     return std::make_unique<AtomicDatabaseTablesSnapshotIterator>(std::move(typeid_cast<DatabaseTablesSnapshotIterator &>(*base_iter)));
 }
 
-DatabaseDetachedTablesSnapshotIteratorPtr DatabaseAtomic::getDetachedTablesIterator(
-    ContextPtr local_context, const IDatabase::FilterByNameFunction & filter_by_table_name, const bool skip_not_loaded) const
-{
-    return DatabaseOrdinary::getDetachedTablesIterator(local_context, filter_by_table_name, skip_not_loaded);
-}
-
 UUID DatabaseAtomic::tryGetTableUUID(const String & table_name) const
 {
     if (auto table = tryGetTable(table_name, getContext()))
diff --git a/src/Databases/DatabaseAtomic.h b/src/Databases/DatabaseAtomic.h
index 25c5cdfbde1..4a4ccfa2573 100644
--- a/src/Databases/DatabaseAtomic.h
+++ b/src/Databases/DatabaseAtomic.h
@@ -49,9 +49,6 @@ public:
 
     DatabaseTablesIteratorPtr getTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name, bool skip_not_loaded) const override;
 
-    DatabaseDetachedTablesSnapshotIteratorPtr
-    getDetachedTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name, bool skip_not_loaded) const override;
-
     void beforeLoadingMetadata(ContextMutablePtr context, LoadingStrictnessLevel mode) override;
 
     LoadTaskPtr startupDatabaseAsync(AsyncLoader & async_loader, LoadJobSet startup_after, LoadingStrictnessLevel mode) override;
diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp
index 67c9114e793..61de0727ba6 100644
--- a/src/Databases/DatabaseOnDisk.cpp
+++ b/src/Databases/DatabaseOnDisk.cpp
@@ -22,8 +22,8 @@
 #include <Storages/IStorage.h>
 #include <Storages/StorageFactory.h>
 #include <TableFunctions/TableFunctionFactory.h>
-#include "Common/Exception.h"
 #include <Common/CurrentMetrics.h>
+#include <Common/Exception.h>
 #include <Common/assert_cast.h>
 #include <Common/escapeForFileName.h>
 #include <Common/filesystemHelpers.h>
@@ -310,11 +310,14 @@ void DatabaseOnDisk::detachTablePermanently(ContextPtr query_context, const Stri
         FS::createFile(detached_permanently_flag);
 
         std::lock_guard lock(mutex);
-        if (!snapshot_detached_tables.contains(table_name))
+        if (const auto it = snapshot_detached_tables.find(table_name); it == snapshot_detached_tables.end())
         {
             throw Exception(ErrorCodes::LOGICAL_ERROR, "Snapshot doesn't contain info about detached table={}", table_name);
         }
-        snapshot_detached_tables[table_name].is_permanently = true;
+        else
+        {
+            it->second.is_permanently = true;
+        }
     }
     catch (Exception & e)
     {
diff --git a/src/Databases/TablesLoader.cpp b/src/Databases/TablesLoader.cpp
index 1f70f46da60..6aa13b7b759 100644
--- a/src/Databases/TablesLoader.cpp
+++ b/src/Databases/TablesLoader.cpp
@@ -1,14 +1,14 @@
-#include <numeric>
+#include <Databases/TablesLoader.h>
+#include <Databases/IDatabase.h>
 #include <Databases/DDLDependencyVisitor.h>
 #include <Databases/DDLLoadingDependencyVisitor.h>
-#include <Databases/IDatabase.h>
-#include <Databases/TablesLoader.h>
-#include <Interpreters/Context.h>
 #include <Interpreters/DatabaseCatalog.h>
+#include <Interpreters/Context.h>
 #include <Interpreters/ExternalDictionariesLoader.h>
 #include <Poco/Util/AbstractConfiguration.h>
-#include <Common/CurrentMetrics.h>
 #include <Common/logger_useful.h>
+#include <Common/CurrentMetrics.h>
+#include <numeric>
 
 
 namespace DB
diff --git a/src/Storages/System/StorageSystemTables.cpp b/src/Storages/System/StorageSystemTables.cpp
index dd7f1a6fdc8..43b761d84b1 100644
--- a/src/Storages/System/StorageSystemTables.cpp
+++ b/src/Storages/System/StorageSystemTables.cpp
@@ -50,18 +50,6 @@ bool needTable(const DatabasePtr & database, const Block & header)
     }
     return false;
 }
-
-template <typename T>
-void fillColumns(T table_it, MutableColumnPtr & database_column, MutableColumnPtr & engine_column)
-{
-    for (; table_it->isValid(); table_it->next())
-    {
-        database_column->insert(table_it->name());
-        if (engine_column)
-            engine_column->insert(table_it->table()->getName());
-    }
-}
-
 }
 
 namespace detail
diff --git a/tests/queries/0_stateless/03172_system_detached_tables.sql b/tests/queries/0_stateless/03172_system_detached_tables.sql
index 916821d996b..64f30a9d275 100644
--- a/tests/queries/0_stateless/03172_system_detached_tables.sql
+++ b/tests/queries/0_stateless/03172_system_detached_tables.sql
@@ -2,7 +2,7 @@
 
 SELECT 'database atomic tests';
 DROP DATABASE IF EXISTS test03172_system_detached_tables;
-CREATE DATABASE IF NOT EXISTS test03172_system_detached_tables;
+CREATE DATABASE IF NOT EXISTS test03172_system_detached_tables ENGINE=Atomic;
 
 CREATE TABLE test03172_system_detached_tables.test_table (n Int64) ENGINE=MergeTree ORDER BY n;
 SELECT * FROM system.detached_tables WHERE database='test03172_system_detached_tables';

From 9be404c9c7b1e14410928a4aef8396664d1e364e Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Tue, 2 Jul 2024 18:02:57 +0000
Subject: [PATCH 0135/1488] Fix another case.

---
 src/Planner/PlannerJoinTree.cpp | 2 +-
 src/Storages/StorageMerge.cpp   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp
index 16b5e363bfd..604d3366484 100644
--- a/src/Planner/PlannerJoinTree.cpp
+++ b/src/Planner/PlannerJoinTree.cpp
@@ -1383,11 +1383,11 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_
 
         if (join_clauses_and_actions.mixed_join_expressions_actions)
         {
-            left_join_tree_query_plan.actions_dags.push_back(join_clauses_and_actions.mixed_join_expressions_actions.get());
             ExpressionActionsPtr & mixed_join_expression = table_join->getMixedJoinExpression();
             mixed_join_expression = std::make_shared<ExpressionActions>(
                 std::move(join_clauses_and_actions.mixed_join_expressions_actions),
                 ExpressionActionsSettings::fromContext(planner_context->getQueryContext()));
+            left_join_tree_query_plan.actions_dags.push_back(&mixed_join_expression->getActionsDAG());
         }
     }
     else if (join_node.isUsingJoinExpression())
diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp
index 5c5ce4ecc2e..c3fdad3a8f2 100644
--- a/src/Storages/StorageMerge.cpp
+++ b/src/Storages/StorageMerge.cpp
@@ -1240,7 +1240,7 @@ ReadFromMerge::RowPolicyData::RowPolicyData(RowPolicyFilterPtr row_policy_filter
     auto expression_analyzer = ExpressionAnalyzer{expr, syntax_result, local_context};
 
     actions_dag = expression_analyzer.getActionsDAG(false /* add_aliases */, false /* project_result */);
-    filter_actions = std::make_shared<ExpressionActions>(std::move(actions_dag),
+    filter_actions = std::make_shared<ExpressionActions>(ActionsDAG::clone(actions_dag),
         ExpressionActionsSettings::fromContext(local_context, CompileExpressions::yes));
     const auto & required_columns = filter_actions->getRequiredColumnsWithTypes();
     const auto & sample_block_columns = filter_actions->getSampleBlock().getNamesAndTypesList();

From 57818990f201562d0b6938c1b8de78d16bac471f Mon Sep 17 00:00:00 2001
From: jsc0218 <jsc0218@gmail.com>
Date: Tue, 2 Jul 2024 20:41:08 +0000
Subject: [PATCH 0136/1488] fix the test

---
 src/Access/ContextAccess.cpp | 32 +++++++++++++++++++-------------
 1 file changed, 19 insertions(+), 13 deletions(-)

diff --git a/src/Access/ContextAccess.cpp b/src/Access/ContextAccess.cpp
index 8ff1fc8ed21..f534c334318 100644
--- a/src/Access/ContextAccess.cpp
+++ b/src/Access/ContextAccess.cpp
@@ -618,6 +618,22 @@ bool ContextAccess::checkAccessImplHelper(AccessFlags flags, const Args &... arg
 
     if (!granted)
     {
+        auto access_denied_no_grant = [&]<typename... FmtArgs>(AccessFlags access_flags, FmtArgs && ...fmt_args)
+        {
+            if (grant_option && acs->isGranted(access_flags, fmt_args...))
+            {
+                return access_denied(ErrorCodes::ACCESS_DENIED,
+                    "{}: Not enough privileges. "
+                    "The required privileges have been granted, but without grant option. "
+                    "To execute this query, it's necessary to have the grant {} WITH GRANT OPTION",
+                    AccessRightsElement{access_flags, fmt_args...}.toStringWithoutOptions());
+            }
+
+            return access_denied(ErrorCodes::ACCESS_DENIED,
+                "{}: Not enough privileges. To execute this query, it's necessary to have the grant {}",
+                AccessRightsElement{access_flags, fmt_args...}.toStringWithoutOptions() + (grant_option ? " WITH GRANT OPTION" : ""));
+        };
+
         /// As we check the SOURCES from the Table Engine logic, direct prompt about Table Engine would be misleading
         /// since SOURCES is not granted actually. In order to solve this, turn the prompt logic back to Sources.
         if (flags & AccessType::TABLE_ENGINE && !access_control->doesTableEnginesRequireGrant())
@@ -635,8 +651,9 @@ bool ContextAccess::checkAccessImplHelper(AccessFlags flags, const Args &... arg
                 break;
             }
 
+            /// Might happen in the case of grant Table Engine on A (but not source), then revoke A.
             if (new_flags.isEmpty())
-                throw Exception(ErrorCodes::LOGICAL_ERROR, "Didn't find the target Source from the Table Engine");
+                return access_denied_no_grant(flags, args...);
 
             if (grant_option && acs->isGranted(flags, args...))
             {
@@ -652,18 +669,7 @@ bool ContextAccess::checkAccessImplHelper(AccessFlags flags, const Args &... arg
                 AccessRightsElement{new_flags}.toStringForAccessTypeSource() + (grant_option ? " WITH GRANT OPTION" : ""));
         }
 
-        if (grant_option && acs->isGranted(flags, args...))
-        {
-            return access_denied(ErrorCodes::ACCESS_DENIED,
-                "{}: Not enough privileges. "
-                "The required privileges have been granted, but without grant option. "
-                "To execute this query, it's necessary to have the grant {} WITH GRANT OPTION",
-                AccessRightsElement{flags, args...}.toStringWithoutOptions());
-        }
-
-        return access_denied(ErrorCodes::ACCESS_DENIED,
-            "{}: Not enough privileges. To execute this query, it's necessary to have the grant {}",
-            AccessRightsElement{flags, args...}.toStringWithoutOptions() + (grant_option ? " WITH GRANT OPTION" : ""));
+        return access_denied_no_grant(flags, args...);
     }
 
     struct PrecalculatedFlags

From 9df30e4a0fe501c8e5eb1815e7a01c73d0c37ad2 Mon Sep 17 00:00:00 2001
From: Michael Kolupaev <michael.kolupaev@clickhouse.com>
Date: Wed, 3 Jul 2024 02:02:24 +0000
Subject: [PATCH 0137/1488] Fix 'Not-ready Set is passed' in system tables

---
 src/Interpreters/PreparedSets.h               | 15 ++++++++++---
 src/Planner/Planner.cpp                       |  2 +-
 src/Storages/RocksDB/StorageSystemRocksDB.cpp |  8 +++++++
 src/Storages/RocksDB/StorageSystemRocksDB.h   |  1 +
 src/Storages/StorageMergeTreeIndex.cpp        | 22 ++++++++++++++-----
 src/Storages/StorageMergeTreeIndex.h          |  2 +-
 .../System/IStorageSystemOneBlock.cpp         | 18 ++++++++++++---
 src/Storages/System/IStorageSystemOneBlock.h  |  8 +++++++
 src/Storages/System/StorageSystemColumns.cpp  | 20 +++++++++++++----
 .../StorageSystemDataSkippingIndices.cpp      | 17 +++++++++++---
 .../System/StorageSystemDatabases.cpp         |  8 +++++++
 src/Storages/System/StorageSystemDatabases.h  |  1 +
 .../System/StorageSystemDistributionQueue.cpp |  7 ++++++
 .../System/StorageSystemDistributionQueue.h   |  1 +
 .../System/StorageSystemMutations.cpp         |  7 ++++++
 src/Storages/System/StorageSystemMutations.h  |  1 +
 .../StorageSystemPartMovesBetweenShards.cpp   |  8 +++++++
 .../StorageSystemPartMovesBetweenShards.h     |  1 +
 src/Storages/System/StorageSystemReplicas.cpp | 19 +++++++++++++---
 .../System/StorageSystemReplicationQueue.cpp  |  8 +++++++
 .../System/StorageSystemReplicationQueue.h    |  1 +
 src/Storages/VirtualColumnUtils.h             |  9 ++++++++
 .../0_stateless/02841_not_ready_set_bug.sh    | 17 ++++++++++++++
 23 files changed, 177 insertions(+), 24 deletions(-)

diff --git a/src/Interpreters/PreparedSets.h b/src/Interpreters/PreparedSets.h
index bf99a8ece3c..a6aee974d0e 100644
--- a/src/Interpreters/PreparedSets.h
+++ b/src/Interpreters/PreparedSets.h
@@ -90,9 +90,18 @@ private:
 
 using FutureSetFromTuplePtr = std::shared_ptr<FutureSetFromTuple>;
 
-/// Set from subquery can be built inplace for PK or in CreatingSet step.
-/// If use_index_for_in_with_subqueries_max_values is reached, set for PK won't be created,
-/// but ordinary set would be created instead.
+/// Set from subquery can be filled (by running the subquery) in one of two ways:
+///  1. During query analysis. Specifically, inside `SourceStepWithFilter::applyFilters()`.
+///     Useful if the query plan depends on the set contents, e.g. to determine which files to read.
+///  2. During query execution. This is the preferred way.
+///     Sets are created by CreatingSetStep, which runs before other steps.
+/// Be careful: to build the set during query analysis, the `buildSetInplace()` call must happen
+/// inside `SourceStepWithFilter::applyFilters()`. Calling it later, e.g. from `initializePipeline()`
+/// will result in LOGICAL_ERROR "Not-ready Set is passed" (because a CreatingSetStep was already
+/// added to pipeline but hasn't executed yet).
+///
+/// If use_index_for_in_with_subqueries_max_values is reached, the built set won't be suitable for
+/// key analysis, but will work with function IN (the set will contain only hashes of elements).
 class FutureSetFromSubquery final : public FutureSet
 {
 public:
diff --git a/src/Planner/Planner.cpp b/src/Planner/Planner.cpp
index 2d42ed73223..8d855b8e619 100644
--- a/src/Planner/Planner.cpp
+++ b/src/Planner/Planner.cpp
@@ -1103,7 +1103,7 @@ void addBuildSubqueriesForSetsStepIfNeeded(
         auto query_tree = subquery->detachQueryTree();
         auto subquery_options = select_query_options.subquery();
         /// I don't know if this is a good decision,
-        /// But for now it is done in the same way as in old analyzer.
+        /// but for now it is done in the same way as in old analyzer.
         /// This would not ignore limits for subqueries (affects mutations only).
         /// See test_build_sets_from_multiple_threads-analyzer.
         subquery_options.ignore_limits = false;
diff --git a/src/Storages/RocksDB/StorageSystemRocksDB.cpp b/src/Storages/RocksDB/StorageSystemRocksDB.cpp
index 5105b190fd9..b6cd58f58cc 100644
--- a/src/Storages/RocksDB/StorageSystemRocksDB.cpp
+++ b/src/Storages/RocksDB/StorageSystemRocksDB.cpp
@@ -40,6 +40,14 @@ ColumnsDescription StorageSystemRocksDB::getColumnsDescription()
 }
 
 
+Block StorageSystemRocksDB::getFilterSampleBlock() const
+{
+    return {
+        { {}, std::make_shared<DataTypeString>(), "database" },
+        { {}, std::make_shared<DataTypeString>(), "table" },
+    };
+}
+
 void StorageSystemRocksDB::fillData(MutableColumns & res_columns, ContextPtr context, const ActionsDAG::Node * predicate, std::vector<UInt8>) const
 {
     const auto access = context->getAccess();
diff --git a/src/Storages/RocksDB/StorageSystemRocksDB.h b/src/Storages/RocksDB/StorageSystemRocksDB.h
index ec351c75446..be3bfaa860c 100644
--- a/src/Storages/RocksDB/StorageSystemRocksDB.h
+++ b/src/Storages/RocksDB/StorageSystemRocksDB.h
@@ -22,6 +22,7 @@ protected:
     using IStorageSystemOneBlock::IStorageSystemOneBlock;
 
     void fillData(MutableColumns & res_columns, ContextPtr context, const ActionsDAG::Node * predicate, std::vector<UInt8>) const override;
+    Block getFilterSampleBlock() const override;
 };
 
 }
diff --git a/src/Storages/StorageMergeTreeIndex.cpp b/src/Storages/StorageMergeTreeIndex.cpp
index 0b1ad02f8c9..90d01d356e9 100644
--- a/src/Storages/StorageMergeTreeIndex.cpp
+++ b/src/Storages/StorageMergeTreeIndex.cpp
@@ -275,7 +275,7 @@ public:
 private:
     std::shared_ptr<StorageMergeTreeIndex> storage;
     Poco::Logger * log;
-    const ActionsDAG::Node * predicate = nullptr;
+    ActionsDAGPtr virtual_columns_filter;
 };
 
 void ReadFromMergeTreeIndex::applyFilters(ActionDAGNodes added_filter_nodes)
@@ -283,7 +283,17 @@ void ReadFromMergeTreeIndex::applyFilters(ActionDAGNodes added_filter_nodes)
     SourceStepWithFilter::applyFilters(std::move(added_filter_nodes));
 
     if (filter_actions_dag)
-        predicate = filter_actions_dag->getOutputs().at(0);
+    {
+        Block block_to_filter
+        {
+            { {}, std::make_shared<DataTypeString>(), StorageMergeTreeIndex::part_name_column.name },
+        };
+
+        virtual_columns_filter = VirtualColumnUtils::splitFilterDagForAllowedInputs(filter_actions_dag->getOutputs().at(0), &block_to_filter);
+
+        if (virtual_columns_filter)
+            VirtualColumnUtils::buildSetsForDAG(virtual_columns_filter, context);
+    }
 }
 
 void StorageMergeTreeIndex::read(
@@ -335,7 +345,7 @@ void StorageMergeTreeIndex::read(
 
 void ReadFromMergeTreeIndex::initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &)
 {
-    auto filtered_parts = storage->getFilteredDataParts(predicate, context);
+    auto filtered_parts = storage->getFilteredDataParts(virtual_columns_filter, context);
 
     LOG_DEBUG(log, "Reading index{}from {} parts of table {}",
         storage->with_marks ? " with marks " : " ",
@@ -345,9 +355,9 @@ void ReadFromMergeTreeIndex::initializePipeline(QueryPipelineBuilder & pipeline,
     pipeline.init(Pipe(std::make_shared<MergeTreeIndexSource>(getOutputStream().header, storage->key_sample_block, std::move(filtered_parts), context, storage->with_marks)));
 }
 
-MergeTreeData::DataPartsVector StorageMergeTreeIndex::getFilteredDataParts(const ActionsDAG::Node * predicate, const ContextPtr & context) const
+MergeTreeData::DataPartsVector StorageMergeTreeIndex::getFilteredDataParts(ActionsDAGPtr virtual_columns_filter, const ContextPtr & context) const
 {
-    if (!predicate)
+    if (!virtual_columns_filter)
         return data_parts;
 
     auto all_part_names = ColumnString::create();
@@ -355,7 +365,7 @@ MergeTreeData::DataPartsVector StorageMergeTreeIndex::getFilteredDataParts(const
         all_part_names->insert(part->name);
 
     Block filtered_block{{std::move(all_part_names), std::make_shared<DataTypeString>(), part_name_column.name}};
-    VirtualColumnUtils::filterBlockWithPredicate(predicate, filtered_block, context);
+    VirtualColumnUtils::filterBlockWithDAG(virtual_columns_filter, filtered_block, context);
 
     if (!filtered_block.rows())
         return {};
diff --git a/src/Storages/StorageMergeTreeIndex.h b/src/Storages/StorageMergeTreeIndex.h
index a1fb61d5a56..652a2d6eeaf 100644
--- a/src/Storages/StorageMergeTreeIndex.h
+++ b/src/Storages/StorageMergeTreeIndex.h
@@ -36,7 +36,7 @@ public:
 private:
     friend class ReadFromMergeTreeIndex;
 
-    MergeTreeData::DataPartsVector getFilteredDataParts(const ActionsDAG::Node * predicate, const ContextPtr & context) const;
+    MergeTreeData::DataPartsVector getFilteredDataParts(ActionsDAGPtr virtual_columns_filter, const ContextPtr & context) const;
 
     StoragePtr source_table;
     bool with_marks;
diff --git a/src/Storages/System/IStorageSystemOneBlock.cpp b/src/Storages/System/IStorageSystemOneBlock.cpp
index 456b7c4f90b..7cde31905aa 100644
--- a/src/Storages/System/IStorageSystemOneBlock.cpp
+++ b/src/Storages/System/IStorageSystemOneBlock.cpp
@@ -5,6 +5,7 @@
 // #include <Storages/IStorage.h>
 #include <Storages/SelectQueryInfo.h>
 #include <Storages/System/getQueriedColumnsMaskAndHeader.h>
+#include <Storages/VirtualColumnUtils.h>
 #include <Processors/Sources/SourceFromSingleChunk.h>
 #include <Processors/QueryPlan/QueryPlan.h>
 #include <Processors/QueryPlan/SourceStepWithFilter.h>
@@ -44,7 +45,7 @@ public:
 private:
     std::shared_ptr<IStorageSystemOneBlock> storage;
     std::vector<UInt8> columns_mask;
-    const ActionsDAG::Node * predicate = nullptr;
+    ActionsDAGPtr filter;
 };
 
 void IStorageSystemOneBlock::read(
@@ -81,6 +82,7 @@ void ReadFromSystemOneBlock::initializePipeline(QueryPipelineBuilder & pipeline,
 {
     const auto & sample_block = getOutputStream().header;
     MutableColumns res_columns = sample_block.cloneEmptyColumns();
+    auto predicate = filter ? filter->getOutputs().at(0) : nullptr;
     storage->fillData(res_columns, context, predicate, std::move(columns_mask));
 
     UInt64 num_rows = res_columns.at(0)->size();
@@ -93,8 +95,18 @@ void ReadFromSystemOneBlock::applyFilters(ActionDAGNodes added_filter_nodes)
 {
     SourceStepWithFilter::applyFilters(std::move(added_filter_nodes));
 
-    if (filter_actions_dag)
-        predicate = filter_actions_dag->getOutputs().at(0);
+    if (!filter_actions_dag)
+        return;
+
+    Block sample = storage->getFilterSampleBlock();
+    if (sample.columns() == 0)
+        return;
+
+    filter = VirtualColumnUtils::splitFilterDagForAllowedInputs(filter_actions_dag->getOutputs().at(0), &sample);
+
+    /// Must prepare sets here, initializePipeline() would be too late, see comment on FutureSetFromSubquery.
+    if (filter)
+        VirtualColumnUtils::buildSetsForDAG(filter, context);
 }
 
 }
diff --git a/src/Storages/System/IStorageSystemOneBlock.h b/src/Storages/System/IStorageSystemOneBlock.h
index a20434fd97e..a47875c2537 100644
--- a/src/Storages/System/IStorageSystemOneBlock.h
+++ b/src/Storages/System/IStorageSystemOneBlock.h
@@ -22,8 +22,16 @@ class Context;
 class IStorageSystemOneBlock : public IStorage
 {
 protected:
+    /// If this method uses `predicate`, getFilterSampleBlock() must list all columns to which
+    /// it's applied. (Otherwise there'll be a LOGICAL_ERROR "Not-ready Set is passed" on subqueries.)
     virtual void fillData(MutableColumns & res_columns, ContextPtr context, const ActionsDAG::Node * predicate, std::vector<UInt8> columns_mask) const = 0;
 
+    /// Columns to which fillData() applies the `predicate`.
+    virtual Block getFilterSampleBlock() const
+    {
+        return {};
+    }
+
     virtual bool supportsColumnsMask() const { return false; }
 
     friend class ReadFromSystemOneBlock;
diff --git a/src/Storages/System/StorageSystemColumns.cpp b/src/Storages/System/StorageSystemColumns.cpp
index 8dd8d3b6154..9502a7ee2a3 100644
--- a/src/Storages/System/StorageSystemColumns.cpp
+++ b/src/Storages/System/StorageSystemColumns.cpp
@@ -337,7 +337,7 @@ private:
     std::shared_ptr<StorageSystemColumns> storage;
     std::vector<UInt8> columns_mask;
     const size_t max_block_size;
-    const ActionsDAG::Node * predicate = nullptr;
+    ActionsDAGPtr virtual_columns_filter;
 };
 
 void ReadFromSystemColumns::applyFilters(ActionDAGNodes added_filter_nodes)
@@ -345,7 +345,17 @@ void ReadFromSystemColumns::applyFilters(ActionDAGNodes added_filter_nodes)
     SourceStepWithFilter::applyFilters(std::move(added_filter_nodes));
 
     if (filter_actions_dag)
-        predicate = filter_actions_dag->getOutputs().at(0);
+    {
+        Block block_to_filter;
+        block_to_filter.insert(ColumnWithTypeAndName(ColumnString::create(), std::make_shared<DataTypeString>(), "database"));
+        block_to_filter.insert(ColumnWithTypeAndName(ColumnString::create(), std::make_shared<DataTypeString>(), "table"));
+
+        virtual_columns_filter = VirtualColumnUtils::splitFilterDagForAllowedInputs(filter_actions_dag->getOutputs().at(0), &block_to_filter);
+
+        /// Must prepare sets here, initializePipeline() would be too late, see comment on FutureSetFromSubquery.
+        if (virtual_columns_filter)
+            VirtualColumnUtils::buildSetsForDAG(virtual_columns_filter, context);
+    }
 }
 
 void StorageSystemColumns::read(
@@ -407,7 +417,8 @@ void ReadFromSystemColumns::initializePipeline(QueryPipelineBuilder & pipeline,
         block_to_filter.insert(ColumnWithTypeAndName(std::move(database_column_mut), std::make_shared<DataTypeString>(), "database"));
 
         /// Filter block with `database` column.
-        VirtualColumnUtils::filterBlockWithPredicate(predicate, block_to_filter, context);
+        if (virtual_columns_filter)
+            VirtualColumnUtils::filterBlockWithPredicate(virtual_columns_filter->getOutputs().at(0), block_to_filter, context);
 
         if (!block_to_filter.rows())
         {
@@ -455,7 +466,8 @@ void ReadFromSystemColumns::initializePipeline(QueryPipelineBuilder & pipeline,
     }
 
     /// Filter block with `database` and `table` columns.
-    VirtualColumnUtils::filterBlockWithPredicate(predicate, block_to_filter, context);
+    if (virtual_columns_filter)
+        VirtualColumnUtils::filterBlockWithDAG(virtual_columns_filter, block_to_filter, context);
 
     if (!block_to_filter.rows())
     {
diff --git a/src/Storages/System/StorageSystemDataSkippingIndices.cpp b/src/Storages/System/StorageSystemDataSkippingIndices.cpp
index 093adc59cc6..a6bba44e257 100644
--- a/src/Storages/System/StorageSystemDataSkippingIndices.cpp
+++ b/src/Storages/System/StorageSystemDataSkippingIndices.cpp
@@ -214,7 +214,7 @@ private:
     std::shared_ptr<StorageSystemDataSkippingIndices> storage;
     std::vector<UInt8> columns_mask;
     const size_t max_block_size;
-    const ActionsDAG::Node * predicate = nullptr;
+    ActionsDAGPtr virtual_columns_filter;
 };
 
 void ReadFromSystemDataSkippingIndices::applyFilters(ActionDAGNodes added_filter_nodes)
@@ -222,7 +222,17 @@ void ReadFromSystemDataSkippingIndices::applyFilters(ActionDAGNodes added_filter
     SourceStepWithFilter::applyFilters(std::move(added_filter_nodes));
 
     if (filter_actions_dag)
-        predicate = filter_actions_dag->getOutputs().at(0);
+    {
+        Block block_to_filter
+        {
+            { ColumnString::create(), std::make_shared<DataTypeString>(), "database" },
+        };
+
+        virtual_columns_filter = VirtualColumnUtils::splitFilterDagForAllowedInputs(filter_actions_dag->getOutputs().at(0), &block_to_filter);
+
+        if (virtual_columns_filter)
+            VirtualColumnUtils::buildSetsForDAG(virtual_columns_filter, context);
+    }
 }
 
 void StorageSystemDataSkippingIndices::read(
@@ -268,7 +278,8 @@ void ReadFromSystemDataSkippingIndices::initializePipeline(QueryPipelineBuilder
 
     /// Condition on "database" in a query acts like an index.
     Block block { ColumnWithTypeAndName(std::move(column), std::make_shared<DataTypeString>(), "database") };
-    VirtualColumnUtils::filterBlockWithPredicate(predicate, block, context);
+    if (virtual_columns_filter)
+        VirtualColumnUtils::filterBlockWithDAG(virtual_columns_filter, block, context);
 
     ColumnPtr & filtered_databases = block.getByPosition(0).column;
     pipeline.init(Pipe(std::make_shared<DataSkippingIndicesSource>(
diff --git a/src/Storages/System/StorageSystemDatabases.cpp b/src/Storages/System/StorageSystemDatabases.cpp
index 1dbb187c418..0585506a661 100644
--- a/src/Storages/System/StorageSystemDatabases.cpp
+++ b/src/Storages/System/StorageSystemDatabases.cpp
@@ -73,6 +73,14 @@ static String getEngineFull(const ContextPtr & ctx, const DatabasePtr & database
     return engine_full;
 }
 
+Block StorageSystemDatabases::getFilterSampleBlock() const
+{
+    return {
+        { {}, std::make_shared<DataTypeString>(), "engine" },
+        { {}, std::make_shared<DataTypeUUID>(), "uuid" },
+    };
+}
+
 static ColumnPtr getFilteredDatabases(const Databases & databases, const ActionsDAG::Node * predicate, ContextPtr context)
 {
     MutableColumnPtr name_column = ColumnString::create();
diff --git a/src/Storages/System/StorageSystemDatabases.h b/src/Storages/System/StorageSystemDatabases.h
index fa55f0aea32..d10b350435b 100644
--- a/src/Storages/System/StorageSystemDatabases.h
+++ b/src/Storages/System/StorageSystemDatabases.h
@@ -27,6 +27,7 @@ protected:
     bool supportsColumnsMask() const override { return true; }
 
     void fillData(MutableColumns & res_columns, ContextPtr context, const ActionsDAG::Node * predicate, std::vector<UInt8> columns_mask) const override;
+    Block getFilterSampleBlock() const override;
 };
 
 }
diff --git a/src/Storages/System/StorageSystemDistributionQueue.cpp b/src/Storages/System/StorageSystemDistributionQueue.cpp
index e2058448904..dab318a9c1c 100644
--- a/src/Storages/System/StorageSystemDistributionQueue.cpp
+++ b/src/Storages/System/StorageSystemDistributionQueue.cpp
@@ -107,6 +107,13 @@ ColumnsDescription StorageSystemDistributionQueue::getColumnsDescription()
     };
 }
 
+Block StorageSystemDistributionQueue::getFilterSampleBlock() const
+{
+    return {
+        { {}, std::make_shared<DataTypeString>(), "database" },
+        { {}, std::make_shared<DataTypeString>(), "table" },
+    };
+}
 
 void StorageSystemDistributionQueue::fillData(MutableColumns & res_columns, ContextPtr context, const ActionsDAG::Node * predicate, std::vector<UInt8>) const
 {
diff --git a/src/Storages/System/StorageSystemDistributionQueue.h b/src/Storages/System/StorageSystemDistributionQueue.h
index 159a86bf082..27d777a4762 100644
--- a/src/Storages/System/StorageSystemDistributionQueue.h
+++ b/src/Storages/System/StorageSystemDistributionQueue.h
@@ -22,6 +22,7 @@ protected:
     using IStorageSystemOneBlock::IStorageSystemOneBlock;
 
     void fillData(MutableColumns & res_columns, ContextPtr context, const ActionsDAG::Node * predicate, std::vector<UInt8>) const override;
+    Block getFilterSampleBlock() const override;
 };
 
 }
diff --git a/src/Storages/System/StorageSystemMutations.cpp b/src/Storages/System/StorageSystemMutations.cpp
index 94656008029..df9a71310e5 100644
--- a/src/Storages/System/StorageSystemMutations.cpp
+++ b/src/Storages/System/StorageSystemMutations.cpp
@@ -46,6 +46,13 @@ ColumnsDescription StorageSystemMutations::getColumnsDescription()
     };
 }
 
+Block StorageSystemMutations::getFilterSampleBlock() const
+{
+    return {
+        { {}, std::make_shared<DataTypeString>(), "database" },
+        { {}, std::make_shared<DataTypeString>(), "table" },
+    };
+}
 
 void StorageSystemMutations::fillData(MutableColumns & res_columns, ContextPtr context, const ActionsDAG::Node * predicate, std::vector<UInt8>) const
 {
diff --git a/src/Storages/System/StorageSystemMutations.h b/src/Storages/System/StorageSystemMutations.h
index c60157cd853..5341838a65e 100644
--- a/src/Storages/System/StorageSystemMutations.h
+++ b/src/Storages/System/StorageSystemMutations.h
@@ -22,6 +22,7 @@ protected:
     using IStorageSystemOneBlock::IStorageSystemOneBlock;
 
     void fillData(MutableColumns & res_columns, ContextPtr context, const ActionsDAG::Node * predicate, std::vector<UInt8>) const override;
+    Block getFilterSampleBlock() const override;
 };
 
 }
diff --git a/src/Storages/System/StorageSystemPartMovesBetweenShards.cpp b/src/Storages/System/StorageSystemPartMovesBetweenShards.cpp
index 9cba92bca12..ab74b205a96 100644
--- a/src/Storages/System/StorageSystemPartMovesBetweenShards.cpp
+++ b/src/Storages/System/StorageSystemPartMovesBetweenShards.cpp
@@ -43,6 +43,14 @@ ColumnsDescription StorageSystemPartMovesBetweenShards::getColumnsDescription()
 }
 
 
+Block StorageSystemPartMovesBetweenShards::getFilterSampleBlock() const
+{
+    return {
+        { {}, std::make_shared<DataTypeString>(), "database" },
+        { {}, std::make_shared<DataTypeString>(), "table" },
+    };
+}
+
 void StorageSystemPartMovesBetweenShards::fillData(MutableColumns & res_columns, ContextPtr context, const ActionsDAG::Node * predicate, std::vector<UInt8>) const
 {
     const auto access = context->getAccess();
diff --git a/src/Storages/System/StorageSystemPartMovesBetweenShards.h b/src/Storages/System/StorageSystemPartMovesBetweenShards.h
index 6a859d4de80..bc6133fcaaa 100644
--- a/src/Storages/System/StorageSystemPartMovesBetweenShards.h
+++ b/src/Storages/System/StorageSystemPartMovesBetweenShards.h
@@ -20,6 +20,7 @@ protected:
     using IStorageSystemOneBlock::IStorageSystemOneBlock;
 
     void fillData(MutableColumns & res_columns, ContextPtr context, const ActionsDAG::Node * predicate, std::vector<UInt8>) const override;
+    Block getFilterSampleBlock() const override;
 };
 
 }
diff --git a/src/Storages/System/StorageSystemReplicas.cpp b/src/Storages/System/StorageSystemReplicas.cpp
index 3bd5fd290db..9fb4dc5ed6f 100644
--- a/src/Storages/System/StorageSystemReplicas.cpp
+++ b/src/Storages/System/StorageSystemReplicas.cpp
@@ -285,7 +285,7 @@ private:
     const bool with_zk_fields;
     const size_t max_block_size;
     std::shared_ptr<StorageSystemReplicasImpl> impl;
-    const ActionsDAG::Node * predicate = nullptr;
+    ActionsDAGPtr virtual_columns_filter;
 };
 
 void ReadFromSystemReplicas::applyFilters(ActionDAGNodes added_filter_nodes)
@@ -293,7 +293,19 @@ void ReadFromSystemReplicas::applyFilters(ActionDAGNodes added_filter_nodes)
     SourceStepWithFilter::applyFilters(std::move(added_filter_nodes));
 
     if (filter_actions_dag)
-        predicate = filter_actions_dag->getOutputs().at(0);
+    {
+        Block block_to_filter
+        {
+            { ColumnString::create(), std::make_shared<DataTypeString>(), "database" },
+            { ColumnString::create(), std::make_shared<DataTypeString>(), "table" },
+            { ColumnString::create(), std::make_shared<DataTypeString>(), "engine" },
+        };
+
+        virtual_columns_filter = VirtualColumnUtils::splitFilterDagForAllowedInputs(filter_actions_dag->getOutputs().at(0), &block_to_filter);
+
+        if (virtual_columns_filter)
+            VirtualColumnUtils::buildSetsForDAG(virtual_columns_filter, context);
+    }
 }
 
 void StorageSystemReplicas::read(
@@ -430,7 +442,8 @@ void ReadFromSystemReplicas::initializePipeline(QueryPipelineBuilder & pipeline,
             { col_engine, std::make_shared<DataTypeString>(), "engine" },
         };
 
-        VirtualColumnUtils::filterBlockWithPredicate(predicate, filtered_block, context);
+        if (virtual_columns_filter)
+            VirtualColumnUtils::filterBlockWithDAG(virtual_columns_filter, filtered_block, context);
 
         if (!filtered_block.rows())
         {
diff --git a/src/Storages/System/StorageSystemReplicationQueue.cpp b/src/Storages/System/StorageSystemReplicationQueue.cpp
index 14b641f46c7..a50982de5f0 100644
--- a/src/Storages/System/StorageSystemReplicationQueue.cpp
+++ b/src/Storages/System/StorageSystemReplicationQueue.cpp
@@ -62,6 +62,14 @@ ColumnsDescription StorageSystemReplicationQueue::getColumnsDescription()
 }
 
 
+Block StorageSystemReplicationQueue::getFilterSampleBlock() const
+{
+    return {
+        { {}, std::make_shared<DataTypeString>(), "database" },
+        { {}, std::make_shared<DataTypeString>(), "table" },
+    };
+}
+
 void StorageSystemReplicationQueue::fillData(MutableColumns & res_columns, ContextPtr context, const ActionsDAG::Node * predicate, std::vector<UInt8>) const
 {
     const auto access = context->getAccess();
diff --git a/src/Storages/System/StorageSystemReplicationQueue.h b/src/Storages/System/StorageSystemReplicationQueue.h
index a9e57851be1..bcf351381ee 100644
--- a/src/Storages/System/StorageSystemReplicationQueue.h
+++ b/src/Storages/System/StorageSystemReplicationQueue.h
@@ -21,6 +21,7 @@ public:
 protected:
     using IStorageSystemOneBlock::IStorageSystemOneBlock;
     void fillData(MutableColumns & res_columns, ContextPtr context, const ActionsDAG::Node * predicate, std::vector<UInt8>) const override;
+  Block getFilterSampleBlock() const override;
 };
 
 }
diff --git a/src/Storages/VirtualColumnUtils.h b/src/Storages/VirtualColumnUtils.h
index fbfbdd6c6cc..ebec807189e 100644
--- a/src/Storages/VirtualColumnUtils.h
+++ b/src/Storages/VirtualColumnUtils.h
@@ -18,6 +18,15 @@ class NamesAndTypesList;
 namespace VirtualColumnUtils
 {
 
+/// The filtering functions are tricky to use correctly.
+/// There are 2 ways:
+///  1. Call filterBlockWithPredicate() or filterBlockWithDAG() inside SourceStepWithFilter::applyFilters().
+///  2. Call splitFilterDagForAllowedInputs() and buildSetsForDAG() inside SourceStepWithFilter::applyFilters().
+///     Then call filterBlockWithPredicate() or filterBlockWithDAG() in initializePipeline().
+///
+/// Otherwise calling filter*() outside applyFilters() will throw "Not-ready Set is passed"
+/// if there are subqueries.
+
 /// Similar to filterBlockWithQuery, but uses ActionsDAG as a predicate.
 /// Basically it is filterBlockWithDAG(splitFilterDagForAllowedInputs).
 void filterBlockWithPredicate(const ActionsDAG::Node * predicate, Block & block, ContextPtr context);
diff --git a/tests/queries/0_stateless/02841_not_ready_set_bug.sh b/tests/queries/0_stateless/02841_not_ready_set_bug.sh
index 9b2f3b0698e..556e2f52de2 100755
--- a/tests/queries/0_stateless/02841_not_ready_set_bug.sh
+++ b/tests/queries/0_stateless/02841_not_ready_set_bug.sh
@@ -11,3 +11,20 @@ $CLICKHOUSE_CLIENT --max_threads=2 --max_result_rows=1 --result_overflow_mode=br
 
 $CLICKHOUSE_CLIENT -q "SELECT * FROM system.tables WHERE 1 in (SELECT number from numbers(2)) AND database = currentDatabase() format Null"
 $CLICKHOUSE_CLIENT -q "SELECT xor(1, 0) FROM system.parts WHERE 1 IN (SELECT 1) FORMAT Null"
+
+# (Not all of these tests are effective because some of these tables are empty.)
+$CLICKHOUSE_CLIENT -nq "
+    select * from system.columns where table in (select '123');
+    select * from system.replicas where database in (select '123');
+    select * from system.data_skipping_indices where database in (select '123');
+    select * from system.databases where name in (select '123');
+    select * from system.mutations where table in (select '123');
+    select * from system.part_moves_between_shards where database in (select '123');
+    select * from system.replication_queue where database in (select '123');
+    select * from system.distribution_queue where database in (select '123');
+"
+$CLICKHOUSE_CLIENT -nq "
+    create table a (x Int8) engine MergeTree order by x;
+    insert into a values (1);
+    select * from mergeTreeIndex(currentDatabase(), 'a') where part_name in (select '123');
+"

From 8a8170d00c90545bea3e8e4881feee7b8a7fb4b7 Mon Sep 17 00:00:00 2001
From: Michael Kolupaev <michael.kolupaev@clickhouse.com>
Date: Wed, 3 Jul 2024 03:19:31 +0000
Subject: [PATCH 0138/1488] Style

---
 src/Storages/System/StorageSystemReplicationQueue.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Storages/System/StorageSystemReplicationQueue.h b/src/Storages/System/StorageSystemReplicationQueue.h
index bcf351381ee..82a4d68f300 100644
--- a/src/Storages/System/StorageSystemReplicationQueue.h
+++ b/src/Storages/System/StorageSystemReplicationQueue.h
@@ -21,7 +21,7 @@ public:
 protected:
     using IStorageSystemOneBlock::IStorageSystemOneBlock;
     void fillData(MutableColumns & res_columns, ContextPtr context, const ActionsDAG::Node * predicate, std::vector<UInt8>) const override;
-  Block getFilterSampleBlock() const override;
+    Block getFilterSampleBlock() const override;
 };
 
 }

From 192ada5ed93f93c17f26671336c6c62e53a1b669 Mon Sep 17 00:00:00 2001
From: Konstantin Morozov <just.morozov.k@gmail.com>
Date: Wed, 3 Jul 2024 08:37:50 +0000
Subject: [PATCH 0139/1488] apply comments

---
 src/Databases/DatabaseAtomic.cpp                    |  1 -
 src/Storages/System/StorageSystemDetachedTables.cpp | 13 +++++++++----
 .../03172_system_detached_tables.reference          |  4 ++--
 .../0_stateless/03172_system_detached_tables.sql    |  4 ++--
 4 files changed, 13 insertions(+), 9 deletions(-)

diff --git a/src/Databases/DatabaseAtomic.cpp b/src/Databases/DatabaseAtomic.cpp
index b8d7315fc0f..65672f0583c 100644
--- a/src/Databases/DatabaseAtomic.cpp
+++ b/src/Databases/DatabaseAtomic.cpp
@@ -103,7 +103,6 @@ void DatabaseAtomic::attachTable(ContextPtr /* context_ */, const String & name,
     auto table_id = table->getStorageID();
     assertDetachedTableNotInUse(table_id.uuid);
     DatabaseOrdinary::attachTableUnlocked(name, table);
-    detached_tables.erase(table_id.uuid);
 
     table_name_to_path.emplace(std::make_pair(name, relative_table_path));
 }
diff --git a/src/Storages/System/StorageSystemDetachedTables.cpp b/src/Storages/System/StorageSystemDetachedTables.cpp
index 560056b891a..158edb7818f 100644
--- a/src/Storages/System/StorageSystemDetachedTables.cpp
+++ b/src/Storages/System/StorageSystemDetachedTables.cpp
@@ -31,12 +31,12 @@ class DetachedTablesBlockSource : public ISource
 public:
     DetachedTablesBlockSource(
         std::vector<UInt8> columns_mask_,
-        Block header,
+        Block header_,
         UInt64 max_block_size_,
         ColumnPtr databases_,
         ColumnPtr detached_tables_,
         ContextPtr context_)
-        : ISource(std::move(header))
+        : ISource(std::move(header_))
         , columns_mask(std::move(columns_mask_))
         , max_block_size(max_block_size_)
         , databases(std::move(databases_))
@@ -63,7 +63,9 @@ protected:
         const auto access = context->getAccess();
         const bool need_to_check_access_for_databases = !access->isGranted(AccessType::SHOW_TABLES);
 
-        for (size_t database_idx = 0, rows_count = 0; database_idx < databases->size() && rows_count < max_block_size; ++database_idx)
+        size_t database_idx = 0;
+        size_t rows_count = 0;
+        for (; database_idx < databases->size() && rows_count < max_block_size; ++database_idx)
         {
             database_name = databases->getDataAt(database_idx).toString();
             database = DatabaseCatalog::instance().tryGetDatabase(database_name);
@@ -92,8 +94,11 @@ protected:
             }
         }
 
+        if (databases->size() == database_idx && max_block_size != rows_count)
+        {
+            done = true;
+        }
         const UInt64 num_rows = result_columns.at(0)->size();
-        done = true;
         return Chunk(std::move(result_columns), num_rows);
     }
 
diff --git a/tests/queries/0_stateless/03172_system_detached_tables.reference b/tests/queries/0_stateless/03172_system_detached_tables.reference
index ce19daa4cdc..83d1ff13942 100644
--- a/tests/queries/0_stateless/03172_system_detached_tables.reference
+++ b/tests/queries/0_stateless/03172_system_detached_tables.reference
@@ -6,6 +6,6 @@ test03172_system_detached_tables	test_table_perm	1
 test03172_system_detached_tables	test_table	0
 -----------------------
 database lazy tests
-test03172_system_detached_tables_lazy	test_table	0
-test03172_system_detached_tables_lazy	test_table_perm	1
+before attach	test03172_system_detached_tables_lazy	test_table	0
+before attach	test03172_system_detached_tables_lazy	test_table_perm	1
 DROP TABLE
diff --git a/tests/queries/0_stateless/03172_system_detached_tables.sql b/tests/queries/0_stateless/03172_system_detached_tables.sql
index 64f30a9d275..1a3c2d7cc0f 100644
--- a/tests/queries/0_stateless/03172_system_detached_tables.sql
+++ b/tests/queries/0_stateless/03172_system_detached_tables.sql
@@ -39,12 +39,12 @@ CREATE TABLE test03172_system_detached_tables_lazy.test_table_perm (number UInt6
 INSERT INTO test03172_system_detached_tables_lazy.test_table_perm SELECT * FROM numbers(100);
 DETACH table test03172_system_detached_tables_lazy.test_table_perm PERMANENTLY;
 
-SELECT database, table, is_permanently FROM system.detached_tables WHERE database='test03172_system_detached_tables_lazy';
+SELECT 'before attach', database, table, is_permanently FROM system.detached_tables WHERE database='test03172_system_detached_tables_lazy';
 
 ATTACH TABLE test03172_system_detached_tables_lazy.test_table;
 ATTACH TABLE test03172_system_detached_tables_lazy.test_table_perm;
 
-SELECT database, table, is_permanently FROM system.detached_tables WHERE database='test03172_system_detached_tables_lazy';
+SELECT 'after attach', database, table, is_permanently FROM system.detached_tables WHERE database='test03172_system_detached_tables_lazy';
 
 SELECT 'DROP TABLE';
 DROP TABLE  test03172_system_detached_tables_lazy.test_table SYNC;

From 6a06024983a78aaab9b7cbe6e9533255debebdb3 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Wed, 3 Jul 2024 10:25:40 +0000
Subject: [PATCH 0140/1488] Fix for prewhere optimization.

---
 src/Processors/QueryPlan/Optimizations/optimizePrewhere.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/Processors/QueryPlan/Optimizations/optimizePrewhere.cpp b/src/Processors/QueryPlan/Optimizations/optimizePrewhere.cpp
index afe1406b65f..f203d831750 100644
--- a/src/Processors/QueryPlan/Optimizations/optimizePrewhere.cpp
+++ b/src/Processors/QueryPlan/Optimizations/optimizePrewhere.cpp
@@ -59,7 +59,7 @@ void optimizePrewhere(Stack & stack, QueryPlan::Nodes &)
 
     /// TODO: We can also check for UnionStep, such as StorageBuffer and local distributed plans.
     QueryPlan::Node * filter_node = (stack.rbegin() + 1)->node;
-    const auto * filter_step = typeid_cast<FilterStep *>(filter_node->step.get());
+    auto * filter_step = typeid_cast<FilterStep *>(filter_node->step.get());
     if (!filter_step)
         return;
 
@@ -108,7 +108,7 @@ void optimizePrewhere(Stack & stack, QueryPlan::Nodes &)
     prewhere_info->need_filter = true;
     prewhere_info->remove_prewhere_column = optimize_result.fully_moved_to_prewhere && filter_step->removesFilterColumn();
 
-    auto filter_expression = ActionsDAG::clone(filter_step->getExpression());
+    auto filter_expression = std::move(filter_step->getExpression());
     const auto & filter_column_name = filter_step->getFilterColumnName();
 
     if (prewhere_info->remove_prewhere_column)
@@ -121,7 +121,7 @@ void optimizePrewhere(Stack & stack, QueryPlan::Nodes &)
         outputs.resize(size);
     }
 
-    auto split_result = filter_step->getExpression()->split(optimize_result.prewhere_nodes, true, true);
+    auto split_result = filter_expression->split(optimize_result.prewhere_nodes, true, true);
 
     /// This is the leak of abstraction.
     /// Splited actions may have inputs which are needed only for PREWHERE.

From 7616f58a0bed80525385879ac3a2a4dcdf2707f4 Mon Sep 17 00:00:00 2001
From: Konstantin Morozov <just.morozov.k@gmail.com>
Date: Wed, 3 Jul 2024 11:36:04 +0000
Subject: [PATCH 0141/1488] add tests for another db engines

---
 .../configs/remote_servers.xml                | 13 ++++
 .../test_system_detached_tables/test.py       | 69 +++++++++++++------
 2 files changed, 61 insertions(+), 21 deletions(-)
 create mode 100644 tests/integration/test_system_detached_tables/configs/remote_servers.xml

diff --git a/tests/integration/test_system_detached_tables/configs/remote_servers.xml b/tests/integration/test_system_detached_tables/configs/remote_servers.xml
new file mode 100644
index 00000000000..8b579724282
--- /dev/null
+++ b/tests/integration/test_system_detached_tables/configs/remote_servers.xml
@@ -0,0 +1,13 @@
+<clickhouse>
+    <remote_servers>
+        <test_cluster>
+            <shard>
+                <internal_replication>true</internal_replication>
+                <replica>
+                    <host>replica1</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+        </test_cluster>
+    </remote_servers>
+</clickhouse>
\ No newline at end of file
diff --git a/tests/integration/test_system_detached_tables/test.py b/tests/integration/test_system_detached_tables/test.py
index dbccac92523..f2bb5d71e85 100644
--- a/tests/integration/test_system_detached_tables/test.py
+++ b/tests/integration/test_system_detached_tables/test.py
@@ -2,8 +2,12 @@ import pytest
 from helpers.cluster import ClickHouseCluster
 
 cluster = ClickHouseCluster(__file__)
-node = cluster.add_instance("node_default", stay_alive=True)
-
+node = cluster.add_instance("replica1",
+    with_zookeeper=True,
+    main_configs=["configs/remote_servers.xml"],
+    macros={"replica": "replica1"},
+    stay_alive=True,
+)
 
 @pytest.fixture(scope="module", autouse=True)
 def start_cluster():
@@ -13,10 +17,25 @@ def start_cluster():
     finally:
         cluster.shutdown()
 
+@pytest.mark.parametrize(
+    "db_name,db_engine,table_engine,table_engine_params",
+    [
+        pytest.param(
+            "test_db_atomic", "Atomic", "MergeTree", "ORDER BY n", id="Atomic db with MergeTree table"
+        ),
+        pytest.param(
+            "test_db_lazy", "Lazy(60)", "Log", "", id="Lazy db with Log table"
+        ),
+        pytest.param(
+            "test_db_repl", "Replicated('/clickhouse/tables/test_table', 'shard1', 'replica1')", "ReplicatedMergeTree", "ORDER BY n", id="Replicated db with ReplicatedMergeTree table"
+        ),
+    ],
+)
+def test_system_detached_tables(start_cluster, db_name, db_engine, table_engine, table_engine_params):
+    node.query(f"CREATE DATABASE IF NOT EXISTS {db_name} ENGINE={db_engine};")
 
-def test_system_detached_tables():
-    node.query("CREATE TABLE test_table (n Int64) ENGINE=MergeTree ORDER BY n;")
-    node.query("CREATE TABLE test_table_perm (n Int64) ENGINE=MergeTree ORDER BY n;")
+    node.query(f"CREATE TABLE {db_name}.test_table (n Int64) ENGINE={table_engine} {table_engine_params};")
+    node.query(f"CREATE TABLE {db_name}.test_table_perm (n Int64) ENGINE={table_engine} {table_engine_params};")
 
     test_table_uuid = node.query(
         "SELECT uuid FROM system.tables WHERE table='test_table'"
@@ -32,29 +51,37 @@ def test_system_detached_tables():
         "SELECT metadata_path FROM system.tables WHERE table='test_table_perm'"
     ).rstrip("\n")
 
-    assert "" == node.query("SELECT * FROM system.detached_tables")
+    assert "" == node.query(f"SELECT * FROM system.detached_tables WHERE database='{db_name}'")
 
-    node.query("DETACH TABLE test_table")
-    node.query("DETACH TABLE test_table_perm PERMANENTLY")
+    node.query(f"SET database_replicated_always_detach_permanently=1; DETACH TABLE {db_name}.test_table")
+    node.query(f"DETACH TABLE {db_name}.test_table_perm PERMANENTLY")
 
-    querry = "SELECT database, table, is_permanently, uuid, metadata_path FROM system.detached_tables FORMAT Values"
+    querry = f"SELECT database, table, is_permanently, uuid, metadata_path FROM system.detached_tables WHERE database='{db_name}' FORMAT Values"
     result = node.query(querry)
-    assert (
-        result
-        == f"('default','test_table',0,'{test_table_uuid}','{test_table_metadata_path}'),('default','test_table_perm',1,'{test_table_perm_uuid}','{test_table_perm_metadata_path}')"
-    )
+    
+    if db_engine.startswith("Repl"):
+        expected_before_restart = f"('{db_name}','test_table',1,'{test_table_uuid}','{test_table_metadata_path}'),('{db_name}','test_table_perm',1,'{test_table_perm_uuid}','{test_table_perm_metadata_path}')"
+    else:
+        expected_before_restart = f"('{db_name}','test_table',0,'{test_table_uuid}','{test_table_metadata_path}'),('{db_name}','test_table_perm',1,'{test_table_perm_uuid}','{test_table_perm_metadata_path}')"
+
+    assert result == expected_before_restart
+
+    if db_engine.startswith("Lazy"):
+        return
+    
     node.restart_clickhouse()
 
+    if db_engine.startswith("Repl"):
+        expected_after_restart = expected_before_restart
+    else:
+        expected_after_restart = f"('{db_name}','test_table_perm',1,'{test_table_perm_uuid}','{test_table_perm_metadata_path}')"
+
     result = node.query(querry)
-    assert (
-        result
-        == f"('default','test_table_perm',1,'{test_table_perm_uuid}','{test_table_perm_metadata_path}')"
-    )
+    assert result == expected_after_restart
 
     node.restart_clickhouse()
 
     result = node.query(querry)
-    assert (
-        result
-        == f"('default','test_table_perm',1,'{test_table_perm_uuid}','{test_table_perm_metadata_path}')"
-    )
+    assert result == expected_after_restart
+
+    node.query(f"DROP DATABASE {db_name}")

From 1bb1e0189207c458550ab677983e3efeebad23b1 Mon Sep 17 00:00:00 2001
From: Konstantin Morozov <just.morozov.k@gmail.com>
Date: Wed, 3 Jul 2024 12:12:09 +0000
Subject: [PATCH 0142/1488] fix style

---
 .../test_system_detached_tables/test.py       | 42 ++++++++++++++-----
 1 file changed, 32 insertions(+), 10 deletions(-)

diff --git a/tests/integration/test_system_detached_tables/test.py b/tests/integration/test_system_detached_tables/test.py
index f2bb5d71e85..dfb55dfd99b 100644
--- a/tests/integration/test_system_detached_tables/test.py
+++ b/tests/integration/test_system_detached_tables/test.py
@@ -2,7 +2,8 @@ import pytest
 from helpers.cluster import ClickHouseCluster
 
 cluster = ClickHouseCluster(__file__)
-node = cluster.add_instance("replica1",
+node = cluster.add_instance(
+    "replica1",
     with_zookeeper=True,
     main_configs=["configs/remote_servers.xml"],
     macros={"replica": "replica1"},
@@ -17,25 +18,42 @@ def start_cluster():
     finally:
         cluster.shutdown()
 
+
 @pytest.mark.parametrize(
     "db_name,db_engine,table_engine,table_engine_params",
     [
         pytest.param(
-            "test_db_atomic", "Atomic", "MergeTree", "ORDER BY n", id="Atomic db with MergeTree table"
+            "test_db_atomic",
+            "Atomic",
+            "MergeTree",
+            "ORDER BY n",
+            id="Atomic db with MergeTree table",
         ),
         pytest.param(
-            "test_db_lazy", "Lazy(60)", "Log", "", id="Lazy db with Log table"
+            "test_db_lazy",
+            "Lazy(60)",
+            "Log",
+            "",
+            id="Lazy db with Log table",
         ),
         pytest.param(
-            "test_db_repl", "Replicated('/clickhouse/tables/test_table', 'shard1', 'replica1')", "ReplicatedMergeTree", "ORDER BY n", id="Replicated db with ReplicatedMergeTree table"
+            "test_db_repl",
+            "Replicated('/clickhouse/tables/test_table','shard1', 'replica1')", 
+            "ReplicatedMergeTree", 
+            "ORDER BY n",
+            id="Replicated db with ReplicatedMergeTree table",
         ),
     ],
 )
 def test_system_detached_tables(start_cluster, db_name, db_engine, table_engine, table_engine_params):
     node.query(f"CREATE DATABASE IF NOT EXISTS {db_name} ENGINE={db_engine};")
 
-    node.query(f"CREATE TABLE {db_name}.test_table (n Int64) ENGINE={table_engine} {table_engine_params};")
-    node.query(f"CREATE TABLE {db_name}.test_table_perm (n Int64) ENGINE={table_engine} {table_engine_params};")
+    node.query(
+        f"CREATE TABLE {db_name}.test_table (n Int64) ENGINE={table_engine} {table_engine_params};"
+    )
+    node.query(
+        f"CREATE TABLE {db_name}.test_table_perm (n Int64) ENGINE={table_engine} {table_engine_params};"
+    )
 
     test_table_uuid = node.query(
         "SELECT uuid FROM system.tables WHERE table='test_table'"
@@ -51,14 +69,18 @@ def test_system_detached_tables(start_cluster, db_name, db_engine, table_engine,
         "SELECT metadata_path FROM system.tables WHERE table='test_table_perm'"
     ).rstrip("\n")
 
-    assert "" == node.query(f"SELECT * FROM system.detached_tables WHERE database='{db_name}'")
+    assert "" == node.query(
+        f"SELECT * FROM system.detached_tables WHERE database='{db_name}'"
+    )
 
-    node.query(f"SET database_replicated_always_detach_permanently=1; DETACH TABLE {db_name}.test_table")
+    node.query(
+        f"SET database_replicated_always_detach_permanently=1; DETACH TABLE {db_name}.test_table"
+    )
     node.query(f"DETACH TABLE {db_name}.test_table_perm PERMANENTLY")
 
     querry = f"SELECT database, table, is_permanently, uuid, metadata_path FROM system.detached_tables WHERE database='{db_name}' FORMAT Values"
     result = node.query(querry)
-    
+
     if db_engine.startswith("Repl"):
         expected_before_restart = f"('{db_name}','test_table',1,'{test_table_uuid}','{test_table_metadata_path}'),('{db_name}','test_table_perm',1,'{test_table_perm_uuid}','{test_table_perm_metadata_path}')"
     else:
@@ -68,7 +90,7 @@ def test_system_detached_tables(start_cluster, db_name, db_engine, table_engine,
 
     if db_engine.startswith("Lazy"):
         return
-    
+
     node.restart_clickhouse()
 
     if db_engine.startswith("Repl"):

From f964b0b6ab48bd92a784a5c719817e2c82fa1f4a Mon Sep 17 00:00:00 2001
From: Konstantin Morozov <just.morozov.k@gmail.com>
Date: Wed, 3 Jul 2024 12:22:32 +0000
Subject: [PATCH 0143/1488] fix style again

---
 tests/integration/test_system_detached_tables/test.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/tests/integration/test_system_detached_tables/test.py b/tests/integration/test_system_detached_tables/test.py
index dfb55dfd99b..2eb870efcbc 100644
--- a/tests/integration/test_system_detached_tables/test.py
+++ b/tests/integration/test_system_detached_tables/test.py
@@ -10,6 +10,7 @@ node = cluster.add_instance(
     stay_alive=True,
 )
 
+
 @pytest.fixture(scope="module", autouse=True)
 def start_cluster():
     try:
@@ -38,14 +39,16 @@ def start_cluster():
         ),
         pytest.param(
             "test_db_repl",
-            "Replicated('/clickhouse/tables/test_table','shard1', 'replica1')", 
-            "ReplicatedMergeTree", 
+            "Replicated('/clickhouse/tables/test_table','shard1', 'replica1')",
+            "ReplicatedMergeTree",
             "ORDER BY n",
             id="Replicated db with ReplicatedMergeTree table",
         ),
     ],
 )
-def test_system_detached_tables(start_cluster, db_name, db_engine, table_engine, table_engine_params):
+def test_system_detached_tables(
+    start_cluster, db_name, db_engine, table_engine, table_engine_params
+):
     node.query(f"CREATE DATABASE IF NOT EXISTS {db_name} ENGINE={db_engine};")
 
     node.query(

From 41c62ca6636572b4d7654dc0dc329740bb5c7425 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Wed, 3 Jul 2024 14:44:47 +0000
Subject: [PATCH 0144/1488] better

---
 src/Planner/Planner.cpp         | 91 +++++++++++----------------------
 src/Planner/PlannerJoinTree.cpp | 25 ++++-----
 src/Planner/PlannerJoinTree.h   |  4 +-
 src/Planner/Utils.cpp           | 30 +++++++++++
 src/Planner/Utils.h             |  3 ++
 5 files changed, 75 insertions(+), 78 deletions(-)

diff --git a/src/Planner/Planner.cpp b/src/Planner/Planner.cpp
index dddb7531519..16ee6de73c4 100644
--- a/src/Planner/Planner.cpp
+++ b/src/Planner/Planner.cpp
@@ -10,7 +10,6 @@
 
 #include <Functions/FunctionFactory.h>
 #include <Functions/CastOverloadResolver.h>
-#include <Functions/indexHint.h>
 
 #include <QueryPipeline/Pipe.h>
 #include <Processors/Sources/SourceFromSingleChunk.h>
@@ -331,14 +330,14 @@ public:
 void addExpressionStep(QueryPlan & query_plan,
     const ActionsAndProjectInputsFlagPtr & expression_actions,
     const std::string & step_description,
-    std::vector<const ActionsDAG *> & result_actions_to_execute)
+    UsefulSets & useful_sets)
 {
     auto actions = ActionsDAG::clone(&expression_actions->dag);
     if (expression_actions->project_input)
         actions->appendInputsForUnusedColumns(query_plan.getCurrentDataStream().header);
 
     auto expression_step = std::make_unique<ExpressionStep>(query_plan.getCurrentDataStream(), actions);
-    result_actions_to_execute.push_back(expression_step->getExpression().get());
+    appendSetsFromActionsDAG(*expression_step->getExpression(), useful_sets);
     expression_step->setStepDescription(step_description);
     query_plan.addStep(std::move(expression_step));
 }
@@ -346,7 +345,7 @@ void addExpressionStep(QueryPlan & query_plan,
 void addFilterStep(QueryPlan & query_plan,
     const FilterAnalysisResult & filter_analysis_result,
     const std::string & step_description,
-    std::vector<const ActionsDAG *> & result_actions_to_execute)
+    UsefulSets & useful_sets)
 {
     auto actions = ActionsDAG::clone(&filter_analysis_result.filter_actions->dag);
     if (filter_analysis_result.filter_actions->project_input)
@@ -356,7 +355,7 @@ void addFilterStep(QueryPlan & query_plan,
         actions,
         filter_analysis_result.filter_column_name,
         filter_analysis_result.remove_filter_column);
-    result_actions_to_execute.push_back(where_step->getExpression().get());
+    appendSetsFromActionsDAG(*where_step->getExpression(), useful_sets);
     where_step->setStepDescription(step_description);
     query_plan.addStep(std::move(where_step));
 }
@@ -544,7 +543,7 @@ void addTotalsHavingStep(QueryPlan & query_plan,
     const QueryAnalysisResult & query_analysis_result,
     const PlannerContextPtr & planner_context,
     const QueryNode & query_node,
-    std::vector<const ActionsDAG *> & result_actions_to_execute)
+    UsefulSets & useful_sets)
 {
     const auto & query_context = planner_context->getQueryContext();
     const auto & settings = query_context->getSettingsRef();
@@ -573,7 +572,7 @@ void addTotalsHavingStep(QueryPlan & query_plan,
         need_finalize);
 
     if (having_analysis_result.filter_actions)
-        result_actions_to_execute.push_back(totals_having_step->getActions().get());
+        appendSetsFromActionsDAG(*totals_having_step->getActions(), useful_sets);
 
     query_plan.addStep(std::move(totals_having_step));
 }
@@ -887,7 +886,7 @@ void addPreliminarySortOrDistinctOrLimitStepsIfNeeded(QueryPlan & query_plan,
     const PlannerContextPtr & planner_context,
     const PlannerQueryProcessingInfo & query_processing_info,
     const QueryTreeNodePtr & query_tree,
-    std::vector<const ActionsDAG *> & result_actions_to_execute)
+    UsefulSets & useful_sets)
 {
     const auto & query_node = query_tree->as<QueryNode &>();
 
@@ -919,7 +918,7 @@ void addPreliminarySortOrDistinctOrLimitStepsIfNeeded(QueryPlan & query_plan,
     if (expressions_analysis_result.hasLimitBy())
     {
         const auto & limit_by_analysis_result = expressions_analysis_result.getLimitBy();
-        addExpressionStep(query_plan, limit_by_analysis_result.before_limit_by_actions, "Before LIMIT BY", result_actions_to_execute);
+        addExpressionStep(query_plan, limit_by_analysis_result.before_limit_by_actions, "Before LIMIT BY", useful_sets);
         addLimitByStep(query_plan, limit_by_analysis_result, query_node);
     }
 
@@ -1057,47 +1056,15 @@ void addOffsetStep(QueryPlan & query_plan, const QueryAnalysisResult & query_ana
     }
 }
 
-void collectSetsFromActionsDAG(const ActionsDAG & dag, std::unordered_set<const FutureSet *> & useful_sets)
-{
-    for (const auto & node : dag.getNodes())
-    {
-        if (node.column)
-        {
-            const IColumn * column = node.column.get();
-            if (const auto * column_const = typeid_cast<const ColumnConst *>(column))
-                column = &column_const->getDataColumn();
-
-            if (const auto * column_set = typeid_cast<const ColumnSet *>(column))
-                useful_sets.insert(column_set->getData().get());
-        }
-
-        if (node.type == ActionsDAG::ActionType::FUNCTION && node.function_base->getName() == "indexHint")
-        {
-            ActionsDAG::NodeRawConstPtrs children;
-            if (const auto * adaptor = typeid_cast<const FunctionToFunctionBaseAdaptor *>(node.function_base.get()))
-            {
-                if (const auto * index_hint = typeid_cast<const FunctionIndexHint *>(adaptor->getFunction().get()))
-                {
-                    collectSetsFromActionsDAG(*index_hint->getActions(), useful_sets);
-                }
-            }
-        }
-    }
-}
-
 void addBuildSubqueriesForSetsStepIfNeeded(
     QueryPlan & query_plan,
     const SelectQueryOptions & select_query_options,
     const PlannerContextPtr & planner_context,
-    const std::vector<const ActionsDAG *> & result_actions_to_execute)
+    const UsefulSets & useful_sets)
 {
     auto subqueries = planner_context->getPreparedSets().getSubqueries();
-    std::unordered_set<const FutureSet *> useful_sets;
 
-    for (const auto * actions_to_execute : result_actions_to_execute)
-        collectSetsFromActionsDAG(*actions_to_execute, useful_sets);
-
-    auto predicate = [&useful_sets](const auto & set) { return !useful_sets.contains(set.get()); };
+    auto predicate = [&useful_sets](const auto & set) { return !useful_sets.contains(set); };
     auto it = std::remove_if(subqueries.begin(), subqueries.end(), std::move(predicate));
     subqueries.erase(it, subqueries.end());
 
@@ -1542,15 +1509,15 @@ void Planner::buildPlanForQueryNode()
         planner_context,
         query_processing_info);
 
-    std::vector<const ActionsDAG *> result_actions_to_execute = std::move(join_tree_query_plan.actions_dags);
+    auto useful_sets = std::move(join_tree_query_plan.useful_sets);
 
     for (auto & [_, table_expression_data] : planner_context->getTableExpressionNodeToData())
     {
         if (table_expression_data.getPrewhereFilterActions())
-            result_actions_to_execute.push_back(table_expression_data.getPrewhereFilterActions().get());
+            appendSetsFromActionsDAG(*table_expression_data.getPrewhereFilterActions(), useful_sets);
 
         if (table_expression_data.getRowLevelFilterActions())
-            result_actions_to_execute.push_back(table_expression_data.getRowLevelFilterActions().get());
+            appendSetsFromActionsDAG(*table_expression_data.getRowLevelFilterActions(), useful_sets);
     }
 
     if (query_processing_info.isIntermediateStage())
@@ -1561,7 +1528,7 @@ void Planner::buildPlanForQueryNode()
             planner_context,
             query_processing_info,
             query_tree,
-            result_actions_to_execute);
+            useful_sets);
 
         if (expression_analysis_result.hasAggregation())
         {
@@ -1573,13 +1540,13 @@ void Planner::buildPlanForQueryNode()
     if (query_processing_info.isFirstStage())
     {
         if (expression_analysis_result.hasWhere())
-            addFilterStep(query_plan, expression_analysis_result.getWhere(), "WHERE", result_actions_to_execute);
+            addFilterStep(query_plan, expression_analysis_result.getWhere(), "WHERE", useful_sets);
 
         if (expression_analysis_result.hasAggregation())
         {
             const auto & aggregation_analysis_result = expression_analysis_result.getAggregation();
             if (aggregation_analysis_result.before_aggregation_actions)
-                addExpressionStep(query_plan, aggregation_analysis_result.before_aggregation_actions, "Before GROUP BY", result_actions_to_execute);
+                addExpressionStep(query_plan, aggregation_analysis_result.before_aggregation_actions, "Before GROUP BY", useful_sets);
 
             addAggregationStep(query_plan, aggregation_analysis_result, query_analysis_result, planner_context, select_query_info);
         }
@@ -1598,7 +1565,7 @@ void Planner::buildPlanForQueryNode()
                   */
                 const auto & window_analysis_result = expression_analysis_result.getWindow();
                 if (window_analysis_result.before_window_actions)
-                    addExpressionStep(query_plan, window_analysis_result.before_window_actions, "Before WINDOW", result_actions_to_execute);
+                    addExpressionStep(query_plan, window_analysis_result.before_window_actions, "Before WINDOW", useful_sets);
             }
             else
             {
@@ -1607,7 +1574,7 @@ void Planner::buildPlanForQueryNode()
                   * now, on shards (first_stage).
                   */
                 const auto & projection_analysis_result = expression_analysis_result.getProjection();
-                addExpressionStep(query_plan, projection_analysis_result.projection_actions, "Projection", result_actions_to_execute);
+                addExpressionStep(query_plan, projection_analysis_result.projection_actions, "Projection", useful_sets);
 
                 if (query_node.isDistinct())
                 {
@@ -1623,7 +1590,7 @@ void Planner::buildPlanForQueryNode()
                 if (expression_analysis_result.hasSort())
                 {
                     const auto & sort_analysis_result = expression_analysis_result.getSort();
-                    addExpressionStep(query_plan, sort_analysis_result.before_order_by_actions, "Before ORDER BY", result_actions_to_execute);
+                    addExpressionStep(query_plan, sort_analysis_result.before_order_by_actions, "Before ORDER BY", useful_sets);
                 }
             }
         }
@@ -1634,7 +1601,7 @@ void Planner::buildPlanForQueryNode()
             planner_context,
             query_processing_info,
             query_tree,
-            result_actions_to_execute);
+            useful_sets);
     }
 
     if (query_processing_info.isSecondStage() || query_processing_info.isFromAggregationState())
@@ -1656,14 +1623,14 @@ void Planner::buildPlanForQueryNode()
 
             if (query_node.isGroupByWithTotals())
             {
-                addTotalsHavingStep(query_plan, expression_analysis_result, query_analysis_result, planner_context, query_node, result_actions_to_execute);
+                addTotalsHavingStep(query_plan, expression_analysis_result, query_analysis_result, planner_context, query_node, useful_sets);
                 having_executed = true;
             }
 
             addCubeOrRollupStepIfNeeded(query_plan, aggregation_analysis_result, query_analysis_result, planner_context, select_query_info, query_node);
 
             if (!having_executed && expression_analysis_result.hasHaving())
-                addFilterStep(query_plan, expression_analysis_result.getHaving(), "HAVING", result_actions_to_execute);
+                addFilterStep(query_plan, expression_analysis_result.getHaving(), "HAVING", useful_sets);
         }
 
         if (query_processing_info.isFromAggregationState())
@@ -1678,16 +1645,16 @@ void Planner::buildPlanForQueryNode()
             {
                 const auto & window_analysis_result = expression_analysis_result.getWindow();
                 if (expression_analysis_result.hasAggregation())
-                    addExpressionStep(query_plan, window_analysis_result.before_window_actions, "Before window functions", result_actions_to_execute);
+                    addExpressionStep(query_plan, window_analysis_result.before_window_actions, "Before window functions", useful_sets);
 
                 addWindowSteps(query_plan, planner_context, window_analysis_result);
             }
 
             if (expression_analysis_result.hasQualify())
-                addFilterStep(query_plan, expression_analysis_result.getQualify(), "QUALIFY", result_actions_to_execute);
+                addFilterStep(query_plan, expression_analysis_result.getQualify(), "QUALIFY", useful_sets);
 
             const auto & projection_analysis_result = expression_analysis_result.getProjection();
-            addExpressionStep(query_plan, projection_analysis_result.projection_actions, "Projection", result_actions_to_execute);
+            addExpressionStep(query_plan, projection_analysis_result.projection_actions, "Projection", useful_sets);
 
             if (query_node.isDistinct())
             {
@@ -1703,7 +1670,7 @@ void Planner::buildPlanForQueryNode()
             if (expression_analysis_result.hasSort())
             {
                 const auto & sort_analysis_result = expression_analysis_result.getSort();
-                addExpressionStep(query_plan, sort_analysis_result.before_order_by_actions, "Before ORDER BY", result_actions_to_execute);
+                addExpressionStep(query_plan, sort_analysis_result.before_order_by_actions, "Before ORDER BY", useful_sets);
             }
         }
         else
@@ -1756,7 +1723,7 @@ void Planner::buildPlanForQueryNode()
         if (!query_processing_info.isFromAggregationState() && expression_analysis_result.hasLimitBy())
         {
             const auto & limit_by_analysis_result = expression_analysis_result.getLimitBy();
-            addExpressionStep(query_plan, limit_by_analysis_result.before_limit_by_actions, "Before LIMIT BY", result_actions_to_execute);
+            addExpressionStep(query_plan, limit_by_analysis_result.before_limit_by_actions, "Before LIMIT BY", useful_sets);
             addLimitByStep(query_plan, limit_by_analysis_result, query_node);
         }
 
@@ -1788,7 +1755,7 @@ void Planner::buildPlanForQueryNode()
         if (!query_processing_info.isToAggregationState())
         {
             const auto & projection_analysis_result = expression_analysis_result.getProjection();
-            addExpressionStep(query_plan, projection_analysis_result.project_names_actions, "Project names", result_actions_to_execute);
+            addExpressionStep(query_plan, projection_analysis_result.project_names_actions, "Project names", useful_sets);
         }
 
         // For additional_result_filter setting
@@ -1796,7 +1763,7 @@ void Planner::buildPlanForQueryNode()
     }
 
     if (!select_query_options.only_analyze)
-        addBuildSubqueriesForSetsStepIfNeeded(query_plan, select_query_options, planner_context, result_actions_to_execute);
+        addBuildSubqueriesForSetsStepIfNeeded(query_plan, select_query_options, planner_context, useful_sets);
 
     query_node_to_plan_step_mapping[&query_node] = query_plan.getRootNode();
 }
diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp
index 604d3366484..94054588d40 100644
--- a/src/Planner/PlannerJoinTree.cpp
+++ b/src/Planner/PlannerJoinTree.cpp
@@ -1181,13 +1181,13 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_
         join_clauses_and_actions.left_join_expressions_actions->appendInputsForUnusedColumns(left_plan.getCurrentDataStream().header);
         auto left_join_expressions_actions_step = std::make_unique<ExpressionStep>(left_plan.getCurrentDataStream(), join_clauses_and_actions.left_join_expressions_actions);
         left_join_expressions_actions_step->setStepDescription("JOIN actions");
-        left_join_tree_query_plan.actions_dags.emplace_back(left_join_expressions_actions_step->getExpression().get());
+        appendSetsFromActionsDAG(*left_join_expressions_actions_step->getExpression(), left_join_tree_query_plan.useful_sets);
         left_plan.addStep(std::move(left_join_expressions_actions_step));
 
         join_clauses_and_actions.right_join_expressions_actions->appendInputsForUnusedColumns(right_plan.getCurrentDataStream().header);
         auto right_join_expressions_actions_step = std::make_unique<ExpressionStep>(right_plan.getCurrentDataStream(), join_clauses_and_actions.right_join_expressions_actions);
         right_join_expressions_actions_step->setStepDescription("JOIN actions");
-        right_join_tree_query_plan.actions_dags.emplace_back(right_join_expressions_actions_step->getExpression().get());
+        appendSetsFromActionsDAG(*right_join_expressions_actions_step->getExpression(), right_join_tree_query_plan.useful_sets);
         right_plan.addStep(std::move(right_join_expressions_actions_step));
     }
 
@@ -1387,7 +1387,8 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_
             mixed_join_expression = std::make_shared<ExpressionActions>(
                 std::move(join_clauses_and_actions.mixed_join_expressions_actions),
                 ExpressionActionsSettings::fromContext(planner_context->getQueryContext()));
-            left_join_tree_query_plan.actions_dags.push_back(&mixed_join_expression->getActionsDAG());
+
+            appendSetsFromActionsDAG(mixed_join_expression->getActionsDAG(), left_join_tree_query_plan.useful_sets);
         }
     }
     else if (join_node.isUsingJoinExpression())
@@ -1585,16 +1586,10 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_
     for (const auto & right_join_tree_query_plan_row_policy : right_join_tree_query_plan.used_row_policies)
         left_join_tree_query_plan.used_row_policies.insert(right_join_tree_query_plan_row_policy);
 
-    /// Collect all required actions dags in `left_join_tree_query_plan.actions_dags`
+    /// Collect all required actions sets in `left_join_tree_query_plan.useful_sets`
     if (!is_filled_join)
-        for (const auto * action_dag : right_join_tree_query_plan.actions_dags)
-            left_join_tree_query_plan.actions_dags.emplace_back(action_dag);
-    // if (join_clauses_and_actions.left_join_expressions_actions)
-    //     left_join_tree_query_plan.actions_dags.emplace_back(join_clauses_and_actions.left_join_expressions_actions.get());
-    // if (join_clauses_and_actions.right_join_expressions_actions)
-    //     left_join_tree_query_plan.actions_dags.emplace_back(join_clauses_and_actions.right_join_expressions_actions.get());
-    // if (join_clauses_and_actions.mixed_join_expressions_actions)
-    //     left_join_tree_query_plan.actions_dags.push_back(join_clauses_and_actions.mixed_join_expressions_actions.get());
+        for (const auto & useful_set : right_join_tree_query_plan.useful_sets)
+            left_join_tree_query_plan.useful_sets.insert(useful_set);
 
     auto mapping = std::move(left_join_tree_query_plan.query_node_to_plan_step_mapping);
     auto & r_mapping = right_join_tree_query_plan.query_node_to_plan_step_mapping;
@@ -1604,7 +1599,7 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_
         .query_plan = std::move(result_plan),
         .from_stage = QueryProcessingStage::FetchColumns,
         .used_row_policies = std::move(left_join_tree_query_plan.used_row_policies),
-        .actions_dags = std::move(left_join_tree_query_plan.actions_dags),
+        .useful_sets = std::move(left_join_tree_query_plan.useful_sets),
         .query_node_to_plan_step_mapping = std::move(mapping),
     };
 }
@@ -1649,7 +1644,7 @@ JoinTreeQueryPlan buildQueryPlanForArrayJoinNode(const QueryTreeNodePtr & array_
 
     auto array_join_actions = std::make_unique<ExpressionStep>(plan.getCurrentDataStream(), std::move(array_join_action_dag));
     array_join_actions->setStepDescription("ARRAY JOIN actions");
-    join_tree_query_plan.actions_dags.push_back(array_join_actions->getExpression().get());
+    appendSetsFromActionsDAG(*array_join_actions->getExpression(), join_tree_query_plan.useful_sets);
     plan.addStep(std::move(array_join_actions));
 
     auto drop_unused_columns_before_array_join_actions_dag = std::make_unique<ActionsDAG>(plan.getCurrentDataStream().header.getColumnsWithTypeAndName());
@@ -1690,7 +1685,7 @@ JoinTreeQueryPlan buildQueryPlanForArrayJoinNode(const QueryTreeNodePtr & array_
         .query_plan = std::move(plan),
         .from_stage = QueryProcessingStage::FetchColumns,
         .used_row_policies = std::move(join_tree_query_plan.used_row_policies),
-        .actions_dags = std::move(join_tree_query_plan.actions_dags),
+        .useful_sets = std::move(join_tree_query_plan.useful_sets),
         .query_node_to_plan_step_mapping = std::move(join_tree_query_plan.query_node_to_plan_step_mapping),
     };
 }
diff --git a/src/Planner/PlannerJoinTree.h b/src/Planner/PlannerJoinTree.h
index 675079427eb..bc58e802a09 100644
--- a/src/Planner/PlannerJoinTree.h
+++ b/src/Planner/PlannerJoinTree.h
@@ -11,12 +11,14 @@
 namespace DB
 {
 
+using UsefulSets = std::unordered_set<FutureSetPtr>;
+
 struct JoinTreeQueryPlan
 {
     QueryPlan query_plan;
     QueryProcessingStage::Enum from_stage;
     std::set<std::string> used_row_policies{};
-    std::vector<const ActionsDAG *> actions_dags{};
+    UsefulSets useful_sets;
     std::unordered_map<const QueryNode *, const QueryPlan::Node *> query_node_to_plan_step_mapping{};
 };
 
diff --git a/src/Planner/Utils.cpp b/src/Planner/Utils.cpp
index 493ecf5ef53..7ac53e0f8c1 100644
--- a/src/Planner/Utils.cpp
+++ b/src/Planner/Utils.cpp
@@ -11,10 +11,12 @@
 #include <DataTypes/DataTypeNullable.h>
 
 #include <Columns/getLeastSuperColumn.h>
+#include <Columns/ColumnSet.h>
 
 #include <IO/WriteBufferFromString.h>
 
 #include <Functions/FunctionFactory.h>
+#include <Functions/indexHint.h>
 
 #include <Storages/StorageDummy.h>
 
@@ -475,4 +477,32 @@ ASTPtr parseAdditionalResultFilter(const Settings & settings)
     return additional_result_filter_ast;
 }
 
+void appendSetsFromActionsDAG(const ActionsDAG & dag, UsefulSets & useful_sets)
+{
+    for (const auto & node : dag.getNodes())
+    {
+        if (node.column)
+        {
+            const IColumn * column = node.column.get();
+            if (const auto * column_const = typeid_cast<const ColumnConst *>(column))
+                column = &column_const->getDataColumn();
+
+            if (const auto * column_set = typeid_cast<const ColumnSet *>(column))
+                useful_sets.insert(column_set->getData());
+        }
+
+        if (node.type == ActionsDAG::ActionType::FUNCTION && node.function_base->getName() == "indexHint")
+        {
+            ActionsDAG::NodeRawConstPtrs children;
+            if (const auto * adaptor = typeid_cast<const FunctionToFunctionBaseAdaptor *>(node.function_base.get()))
+            {
+                if (const auto * index_hint = typeid_cast<const FunctionIndexHint *>(adaptor->getFunction().get()))
+                {
+                    appendSetsFromActionsDAG(*index_hint->getActions(), useful_sets);
+                }
+            }
+        }
+    }
+}
+
 }
diff --git a/src/Planner/Utils.h b/src/Planner/Utils.h
index 3172847f053..ae60976a8d6 100644
--- a/src/Planner/Utils.h
+++ b/src/Planner/Utils.h
@@ -88,4 +88,7 @@ FilterDAGInfo buildFilterInfo(QueryTreeNodePtr filter_query_tree,
 
 ASTPtr parseAdditionalResultFilter(const Settings & settings);
 
+using UsefulSets = std::unordered_set<FutureSetPtr>;
+void appendSetsFromActionsDAG(const ActionsDAG & dag, UsefulSets & useful_sets);
+
 }

From 5d16ba57aa84ef82ccf7e34a4635ad1d14e7859d Mon Sep 17 00:00:00 2001
From: robot-clickhouse <robot-clickhouse@users.noreply.github.com>
Date: Wed, 3 Jul 2024 14:46:49 +0000
Subject: [PATCH 0145/1488] Update version_date.tsv and changelogs after
 v24.3.5.46-lts

---
 docs/changelogs/v24.3.5.46-lts.md    | 40 ++++++++++++++++++++++++++++
 utils/list-versions/version_date.tsv |  1 +
 2 files changed, 41 insertions(+)
 create mode 100644 docs/changelogs/v24.3.5.46-lts.md

diff --git a/docs/changelogs/v24.3.5.46-lts.md b/docs/changelogs/v24.3.5.46-lts.md
new file mode 100644
index 00000000000..1f2b7c8b0b7
--- /dev/null
+++ b/docs/changelogs/v24.3.5.46-lts.md
@@ -0,0 +1,40 @@
+---
+sidebar_position: 1
+sidebar_label: 2024
+---
+
+# 2024 Changelog
+
+### ClickHouse release v24.3.5.46-lts (fe54cead6b6) FIXME as compared to v24.3.4.147-lts (31a7bdc346d)
+
+#### Improvement
+* Backported in [#65463](https://github.com/ClickHouse/ClickHouse/issues/65463): Reload certificate chain during certificate reload. [#61671](https://github.com/ClickHouse/ClickHouse/pull/61671) ([Pervakov Grigorii](https://github.com/GrigoryPervakov)).
+* Backported in [#65882](https://github.com/ClickHouse/ClickHouse/issues/65882): Always start Keeper with sufficient amount of threads in global thread pool. [#64444](https://github.com/ClickHouse/ClickHouse/pull/64444) ([Duc Canh Le](https://github.com/canhld94)).
+* Backported in [#65302](https://github.com/ClickHouse/ClickHouse/issues/65302): Returned back the behaviour of how ClickHouse works and interprets Tuples in CSV format. This change effectively reverts https://github.com/ClickHouse/ClickHouse/pull/60994 and makes it available only under a few settings: `output_format_csv_serialize_tuple_into_separate_columns`, `input_format_csv_deserialize_separate_columns_into_tuple` and `input_format_csv_try_infer_strings_from_quoted_tuples`. [#65170](https://github.com/ClickHouse/ClickHouse/pull/65170) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Backported in [#65892](https://github.com/ClickHouse/ClickHouse/issues/65892): Respect cgroup CPU limit in Keeper. [#65819](https://github.com/ClickHouse/ClickHouse/pull/65819) ([Antonio Andelic](https://github.com/antonio2368)).
+
+#### Critical Bug Fix (crash, LOGICAL_ERROR, data loss, RBAC)
+* Backported in [#65283](https://github.com/ClickHouse/ClickHouse/issues/65283): Fix crash with UniqInjectiveFunctionsEliminationPass and uniqCombined. [#65188](https://github.com/ClickHouse/ClickHouse/pull/65188) ([Raúl Marín](https://github.com/Algunenano)).
+* Backported in [#65370](https://github.com/ClickHouse/ClickHouse/issues/65370): Fix a bug in ClickHouse Keeper that causes digest mismatch during closing session. [#65198](https://github.com/ClickHouse/ClickHouse/pull/65198) ([Aleksei Filatov](https://github.com/aalexfvk)).
+* Backported in [#65446](https://github.com/ClickHouse/ClickHouse/issues/65446): Use correct memory alignment for Distinct combinator. Previously, crash could happen because of invalid memory allocation when the combinator was used. [#65379](https://github.com/ClickHouse/ClickHouse/pull/65379) ([Antonio Andelic](https://github.com/antonio2368)).
+* Backported in [#65708](https://github.com/ClickHouse/ClickHouse/issues/65708): Fix crash in maxIntersections. [#65689](https://github.com/ClickHouse/ClickHouse/pull/65689) ([Raúl Marín](https://github.com/Algunenano)).
+
+#### Bug Fix (user-visible misbehavior in an official stable release)
+* Backported in [#65352](https://github.com/ClickHouse/ClickHouse/issues/65352): Fix possible abort on uncaught exception in ~WriteBufferFromFileDescriptor in StatusFile. [#64206](https://github.com/ClickHouse/ClickHouse/pull/64206) ([Kruglov Pavel](https://github.com/Avogar)).
+* Backported in [#65327](https://github.com/ClickHouse/ClickHouse/issues/65327): Fix the crash loop when restoring from backup is blocked by creating an MV with a definer that hasn't been restored yet. [#64595](https://github.com/ClickHouse/ClickHouse/pull/64595) ([pufit](https://github.com/pufit)).
+* Backported in [#65538](https://github.com/ClickHouse/ClickHouse/issues/65538): Fix crash for `ALTER TABLE ... ON CLUSTER ... MODIFY SQL SECURITY`. [#64957](https://github.com/ClickHouse/ClickHouse/pull/64957) ([pufit](https://github.com/pufit)).
+* Backported in [#65576](https://github.com/ClickHouse/ClickHouse/issues/65576): Fix crash on destroying AccessControl: add explicit shutdown. [#64993](https://github.com/ClickHouse/ClickHouse/pull/64993) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Backported in [#65159](https://github.com/ClickHouse/ClickHouse/issues/65159): Fix pushing arithmetic operations out of aggregation. In the new analyzer, optimization was applied only once. [#65104](https://github.com/ClickHouse/ClickHouse/pull/65104) ([Dmitry Novik](https://github.com/novikd)).
+* Backported in [#65615](https://github.com/ClickHouse/ClickHouse/issues/65615): Fix aggregate function name rewriting in the new analyzer. [#65110](https://github.com/ClickHouse/ClickHouse/pull/65110) ([Dmitry Novik](https://github.com/novikd)).
+* Backported in [#65728](https://github.com/ClickHouse/ClickHouse/issues/65728): Eliminate injective function in argument of functions `uniq*` recursively. This used to work correctly but was broken in the new analyzer. [#65140](https://github.com/ClickHouse/ClickHouse/pull/65140) ([Duc Canh Le](https://github.com/canhld94)).
+* Backported in [#65261](https://github.com/ClickHouse/ClickHouse/issues/65261): Fix the bug in Hashed and Hashed_Array dictionary short circuit evaluation, which may read uninitialized number, leading to various errors. [#65256](https://github.com/ClickHouse/ClickHouse/pull/65256) ([jsc0218](https://github.com/jsc0218)).
+* Backported in [#65667](https://github.com/ClickHouse/ClickHouse/issues/65667): Disable `non-intersecting-parts` optimization for queries with `FINAL` in case of `read-in-order` optimization was enabled. This could lead to an incorrect query result. As a workaround, disable `do_not_merge_across_partitions_select_final` and `split_parts_ranges_into_intersecting_and_non_intersecting_final` before this fix is merged. [#65505](https://github.com/ClickHouse/ClickHouse/pull/65505) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#65784](https://github.com/ClickHouse/ClickHouse/issues/65784): Fixed bug in MergeJoin. Column in sparse serialisation might be treated as a column of its nested type though the required conversion wasn't performed. [#65632](https://github.com/ClickHouse/ClickHouse/pull/65632) ([Nikita Taranov](https://github.com/nickitat)).
+* Backported in [#65929](https://github.com/ClickHouse/ClickHouse/issues/65929): For queries that read from `PostgreSQL`, cancel the internal `PostgreSQL` query if the ClickHouse query is finished. Otherwise, `ClickHouse` query cannot be canceled until the internal `PostgreSQL` query is finished. [#65771](https://github.com/ClickHouse/ClickHouse/pull/65771) ([Maksim Kita](https://github.com/kitaisreal)).
+* Backported in [#65824](https://github.com/ClickHouse/ClickHouse/issues/65824): Fix a bug in short circuit logic when old analyzer and dictGetOrDefault is used. [#65802](https://github.com/ClickHouse/ClickHouse/pull/65802) ([jsc0218](https://github.com/jsc0218)).
+
+#### NOT FOR CHANGELOG / INSIGNIFICANT
+
+* Backported in [#65223](https://github.com/ClickHouse/ClickHouse/issues/65223): Capture weak_ptr of ContextAccess for safety. [#65051](https://github.com/ClickHouse/ClickHouse/pull/65051) ([Alexander Gololobov](https://github.com/davenger)).
+* Backported in [#65901](https://github.com/ClickHouse/ClickHouse/issues/65901): Fix bug with session closing in Keeper. [#65735](https://github.com/ClickHouse/ClickHouse/pull/65735) ([Antonio Andelic](https://github.com/antonio2368)).
+
diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv
index 8112ed9083b..8e748a2c2ca 100644
--- a/utils/list-versions/version_date.tsv
+++ b/utils/list-versions/version_date.tsv
@@ -6,6 +6,7 @@ v24.5.1.1763-stable	2024-06-01
 v24.4.3.25-stable	2024-06-14
 v24.4.2.141-stable	2024-06-07
 v24.4.1.2088-stable	2024-05-01
+v24.3.5.46-lts	2024-07-03
 v24.3.4.147-lts	2024-06-13
 v24.3.3.102-lts	2024-05-01
 v24.3.2.23-lts	2024-04-03

From d0f36e09a964c1e8a3040d6cfd8b3edfec47474d Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Thu, 4 Jul 2024 04:58:27 +0200
Subject: [PATCH 0146/1488] Fix error

---
 src/Databases/DatabaseAtomic.cpp | 3 +++
 src/Databases/DatabaseAtomic.h   | 2 ++
 src/Databases/DatabaseOnDisk.cpp | 2 ++
 src/Databases/DatabaseOnDisk.h   | 1 +
 4 files changed, 8 insertions(+)

diff --git a/src/Databases/DatabaseAtomic.cpp b/src/Databases/DatabaseAtomic.cpp
index c06fc98d0b9..7f198042e44 100644
--- a/src/Databases/DatabaseAtomic.cpp
+++ b/src/Databases/DatabaseAtomic.cpp
@@ -58,6 +58,9 @@ DatabaseAtomic::DatabaseAtomic(String name_, String metadata_path_, UUID uuid, C
 
 void DatabaseAtomic::createDirectories()
 {
+    if (database_atomic_directories_created.test_and_set())
+        return;
+    DatabaseOnDisk::createDirectories();
     fs::create_directories(fs::path(getContext()->getPath()) / "metadata");
     fs::create_directories(path_to_table_symlinks);
     tryCreateMetadataSymlink();
diff --git a/src/Databases/DatabaseAtomic.h b/src/Databases/DatabaseAtomic.h
index 26ab7657354..9df300daa20 100644
--- a/src/Databases/DatabaseAtomic.h
+++ b/src/Databases/DatabaseAtomic.h
@@ -75,7 +75,9 @@ protected:
     using DetachedTables = std::unordered_map<UUID, StoragePtr>;
     [[nodiscard]] DetachedTables cleanupDetachedTables() TSA_REQUIRES(mutex);
 
+    std::atomic_flag database_atomic_directories_created = ATOMIC_FLAG_INIT;
     void createDirectories();
+
     void tryCreateMetadataSymlink();
 
     virtual bool allowMoveTableToOtherDatabaseEngine(IDatabase & /*to_database*/) const { return false; }
diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp
index faac4b23701..0a0ad589a9f 100644
--- a/src/Databases/DatabaseOnDisk.cpp
+++ b/src/Databases/DatabaseOnDisk.cpp
@@ -175,6 +175,8 @@ DatabaseOnDisk::DatabaseOnDisk(
 
 void DatabaseOnDisk::createDirectories()
 {
+    if (directories_created.test_and_set())
+        return;
     fs::create_directories(std::filesystem::path(getContext()->getPath()) / data_path);
     fs::create_directories(metadata_path);
 }
diff --git a/src/Databases/DatabaseOnDisk.h b/src/Databases/DatabaseOnDisk.h
index 00e7a2850b8..a8be674a4e2 100644
--- a/src/Databases/DatabaseOnDisk.h
+++ b/src/Databases/DatabaseOnDisk.h
@@ -99,6 +99,7 @@ protected:
     virtual void removeDetachedPermanentlyFlag(ContextPtr context, const String & table_name, const String & table_metadata_path, bool attach);
     virtual void setDetachedTableNotInUseForce(const UUID & /*uuid*/) {}
 
+    std::atomic_flag directories_created = ATOMIC_FLAG_INIT;
     void createDirectories();
 
     const String metadata_path;

From b52937e8580055a126f3d263cad893f212ce07b9 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Thu, 4 Jul 2024 05:00:18 +0200
Subject: [PATCH 0147/1488] Better test

---
 tests/queries/0_stateless/01191_rename_dictionary.sql | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/queries/0_stateless/01191_rename_dictionary.sql b/tests/queries/0_stateless/01191_rename_dictionary.sql
index c5012dabc81..be95e5a7d4b 100644
--- a/tests/queries/0_stateless/01191_rename_dictionary.sql
+++ b/tests/queries/0_stateless/01191_rename_dictionary.sql
@@ -27,6 +27,7 @@ RENAME DICTIONARY test_01191.t TO test_01191.dict1; -- {serverError INCORRECT_QU
 DROP DICTIONARY test_01191.t; -- {serverError INCORRECT_QUERY}
 DROP TABLE test_01191.t;
 
+DROP DATABASE IF EXISTS dummy_db;
 CREATE DATABASE dummy_db ENGINE=Atomic;
 RENAME DICTIONARY test_01191.dict TO dummy_db.dict1;
 RENAME DICTIONARY dummy_db.dict1 TO test_01191.dict;

From ee0985c5b4dbb15d28c9a034b60e438099b5c5bc Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Thu, 4 Jul 2024 05:01:35 +0200
Subject: [PATCH 0148/1488] Fix test

---
 .../02141_clickhouse_local_interactive_table.reference        | 4 ++--
 .../0_stateless/02141_clickhouse_local_interactive_table.sh   | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/queries/0_stateless/02141_clickhouse_local_interactive_table.reference b/tests/queries/0_stateless/02141_clickhouse_local_interactive_table.reference
index 0bb8966cbe4..0e74c0a083e 100644
--- a/tests/queries/0_stateless/02141_clickhouse_local_interactive_table.reference
+++ b/tests/queries/0_stateless/02141_clickhouse_local_interactive_table.reference
@@ -1,2 +1,2 @@
-CREATE TABLE default.`table`\n(\n    `key` String\n)\nENGINE = File(\'TSVWithNamesAndTypes\', \'/dev/null\')
-CREATE TABLE foo.`table`\n(\n    `key` String\n)\nENGINE = File(\'TSVWithNamesAndTypes\', \'/dev/null\')
+CREATE TEMPORARY TABLE `table`\n(\n    `key` String\n)\nENGINE = File(TSVWithNamesAndTypes, \'/dev/null\')
+CREATE TEMPORARY TABLE `table`\n(\n    `key` String\n)\nENGINE = File(TSVWithNamesAndTypes, \'/dev/null\')
diff --git a/tests/queries/0_stateless/02141_clickhouse_local_interactive_table.sh b/tests/queries/0_stateless/02141_clickhouse_local_interactive_table.sh
index 934d87616ac..3a95e59416a 100755
--- a/tests/queries/0_stateless/02141_clickhouse_local_interactive_table.sh
+++ b/tests/queries/0_stateless/02141_clickhouse_local_interactive_table.sh
@@ -4,5 +4,5 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
-$CLICKHOUSE_LOCAL --file /dev/null --structure "key String" --input-format TSVWithNamesAndTypes --interactive --send_logs_level=trace <<<'show create table table'
-$CLICKHOUSE_LOCAL --database foo --file /dev/null --structure "key String" --input-format TSVWithNamesAndTypes --interactive --send_logs_level=trace <<<'show create table table'
+$CLICKHOUSE_LOCAL --file /dev/null --structure "key String" --input-format TSVWithNamesAndTypes --interactive --send_logs_level=trace <<<'show create temporary table table'
+$CLICKHOUSE_LOCAL --database foo --file /dev/null --structure "key String" --input-format TSVWithNamesAndTypes --interactive --send_logs_level=trace <<<'show create temporary table table'

From bf312f200cb69267741778b2af04bf7a6854a2f6 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Thu, 4 Jul 2024 05:24:42 +0200
Subject: [PATCH 0149/1488] DatabaseOverlay: add support for rename

---
 programs/local/LocalServer.cpp     |  8 ++++----
 src/Databases/DatabasesOverlay.cpp | 33 ++++++++++++++++++++++++++++++
 src/Databases/DatabasesOverlay.h   |  8 ++++++++
 3 files changed, 45 insertions(+), 4 deletions(-)

diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp
index cda276c8407..41bb5604a52 100644
--- a/programs/local/LocalServer.cpp
+++ b/programs/local/LocalServer.cpp
@@ -201,10 +201,10 @@ static DatabasePtr createMemoryDatabaseIfNotExists(ContextPtr context, const Str
 
 static DatabasePtr createClickHouseLocalDatabaseOverlay(const String & name_, ContextPtr context)
 {
-    auto databaseCombiner = std::make_shared<DatabasesOverlay>(name_, context);
-    databaseCombiner->registerNextDatabase(std::make_shared<DatabaseFilesystem>(name_, "", context));
-    databaseCombiner->registerNextDatabase(std::make_shared<DatabaseAtomic>(name_, fs::weakly_canonical(context->getPath()), UUIDHelpers::generateV4(), context));
-    return databaseCombiner;
+    auto overlay = std::make_shared<DatabasesOverlay>(name_, context);
+    overlay->registerNextDatabase(std::make_shared<DatabaseAtomic>(name_, fs::weakly_canonical(context->getPath()), UUIDHelpers::generateV4(), context));
+    overlay->registerNextDatabase(std::make_shared<DatabaseFilesystem>(name_, "", context));
+    return overlay;
 }
 
 /// If path is specified and not empty, will try to setup server environment and load existing metadata
diff --git a/src/Databases/DatabasesOverlay.cpp b/src/Databases/DatabasesOverlay.cpp
index 02a0aab8230..e1a457920cf 100644
--- a/src/Databases/DatabasesOverlay.cpp
+++ b/src/Databases/DatabasesOverlay.cpp
@@ -124,6 +124,39 @@ StoragePtr DatabasesOverlay::detachTable(ContextPtr context_, const String & tab
         getEngineName());
 }
 
+void DatabasesOverlay::renameTable(
+    ContextPtr current_context,
+    const String & name,
+    IDatabase & to_database,
+    const String & to_name,
+    bool exchange,
+    bool dictionary)
+{
+    for (auto & db : databases)
+    {
+        if (db->isTableExist(name, current_context))
+        {
+            if (DatabasesOverlay * to_overlay_database = typeid_cast<DatabasesOverlay *>(&to_database))
+            {
+                /// Renaming from Overlay database inside itself or into another Overlay database.
+                /// Just use the first database in the overlay as a destination.
+                if (to_overlay_database->databases.empty())
+                    throw Exception(ErrorCodes::BAD_ARGUMENTS, "The destination Overlay database {} does not have any members", to_database.getDatabaseName());
+
+                db->renameTable(current_context, name, *to_overlay_database->databases[0], to_name, exchange, dictionary);
+            }
+            else
+            {
+                /// Renaming into a different type of database. E.g. from Overlay on top of Atomic database into just Atomic database.
+                db->renameTable(current_context, name, to_database, to_name, exchange, dictionary);
+            }
+
+            return;
+        }
+    }
+    throw Exception(ErrorCodes::UNKNOWN_TABLE, "Table {}.{} doesn't exist", backQuote(getDatabaseName()), backQuote(name));
+}
+
 ASTPtr DatabasesOverlay::getCreateTableQueryImpl(const String & name, ContextPtr context_, bool throw_on_error) const
 {
     ASTPtr result = nullptr;
diff --git a/src/Databases/DatabasesOverlay.h b/src/Databases/DatabasesOverlay.h
index 5f6d4e601d3..40c653e5cb5 100644
--- a/src/Databases/DatabasesOverlay.h
+++ b/src/Databases/DatabasesOverlay.h
@@ -35,6 +35,14 @@ public:
 
     StoragePtr detachTable(ContextPtr context, const String & table_name) override;
 
+    void renameTable(
+        ContextPtr current_context,
+        const String & name,
+        IDatabase & to_database,
+        const String & to_name,
+        bool exchange,
+        bool dictionary) override;
+
     ASTPtr getCreateTableQueryImpl(const String & name, ContextPtr context, bool throw_on_error) const override;
     ASTPtr getCreateDatabaseQuery() const override;
 

From a70710e3f19aea4434aebbd07233b8e681e4e9e4 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Thu, 4 Jul 2024 05:27:56 +0200
Subject: [PATCH 0150/1488] Add a test

---
 .../03199_atomic_clickhouse_local.reference   |  6 +++++
 .../03199_atomic_clickhouse_local.sh          | 24 +++++++++++++++++++
 2 files changed, 30 insertions(+)
 create mode 100644 tests/queries/0_stateless/03199_atomic_clickhouse_local.reference
 create mode 100755 tests/queries/0_stateless/03199_atomic_clickhouse_local.sh

diff --git a/tests/queries/0_stateless/03199_atomic_clickhouse_local.reference b/tests/queries/0_stateless/03199_atomic_clickhouse_local.reference
new file mode 100644
index 00000000000..1975397394b
--- /dev/null
+++ b/tests/queries/0_stateless/03199_atomic_clickhouse_local.reference
@@ -0,0 +1,6 @@
+123
+Hello
+['Hello','world']
+Hello
+Hello
+['Hello','world']
diff --git a/tests/queries/0_stateless/03199_atomic_clickhouse_local.sh b/tests/queries/0_stateless/03199_atomic_clickhouse_local.sh
new file mode 100755
index 00000000000..edaa83b8f95
--- /dev/null
+++ b/tests/queries/0_stateless/03199_atomic_clickhouse_local.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+${CLICKHOUSE_LOCAL} -n "
+CREATE TABLE test (x UInt8) ORDER BY x;
+INSERT INTO test VALUES (123);
+SELECT * FROM test;
+CREATE OR REPLACE TABLE test (s String) ORDER BY s;
+INSERT INTO test VALUES ('Hello');
+SELECT * FROM test;
+RENAME TABLE test TO test2;
+CREATE OR REPLACE TABLE test (s Array(String)) ORDER BY s;
+INSERT INTO test VALUES (['Hello', 'world']);
+SELECT * FROM test;
+SELECT * FROM test2;
+EXCHANGE TABLES test AND test2;
+SELECT * FROM test;
+SELECT * FROM test2;
+DROP TABLE test;
+DROP TABLE test2;
+"

From cb5d5863467a14cebdbc2dc1c6e4d72afe256515 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Thu, 4 Jul 2024 05:49:55 +0200
Subject: [PATCH 0151/1488] Fix style

---
 src/Databases/DatabasesOverlay.cpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/Databases/DatabasesOverlay.cpp b/src/Databases/DatabasesOverlay.cpp
index e1a457920cf..495733e15fd 100644
--- a/src/Databases/DatabasesOverlay.cpp
+++ b/src/Databases/DatabasesOverlay.cpp
@@ -14,6 +14,8 @@ namespace ErrorCodes
 {
     extern const int LOGICAL_ERROR;
     extern const int CANNOT_GET_CREATE_TABLE_QUERY;
+    extern const int BAD_ARGUMENTS;
+    extern const int UNKNOWN_TABLE;
 }
 
 DatabasesOverlay::DatabasesOverlay(const String & name_, ContextPtr context_)

From 9036ce9725c6d273e84d44b862cdecf31fe36d9c Mon Sep 17 00:00:00 2001
From: JackyWoo <wuchienchao@qq.com>
Date: Thu, 4 Jul 2024 11:09:47 +0800
Subject: [PATCH 0152/1488] Some fixups after merging

---
 .../mergetree-family/mergetree.md             |  2 +-
 src/Storages/MergeTree/MergeTreeData.cpp      | 16 ++++------
 .../ConditionSelectivityEstimator.cpp         | 11 ++-----
 .../ConditionSelectivityEstimator.h           | 22 +++++++-------
 src/Storages/Statistics/Statistics.cpp        | 29 +++++++++----------
 src/Storages/Statistics/Statistics.h          |  5 ++--
 ...stics.cpp => StatisticsCountMinSketch.cpp} | 20 +++++++------
 ...tatistics.h => StatisticsCountMinSketch.h} |  9 ++----
 src/Storages/Statistics/StatisticsTDigest.cpp | 18 ++++++++----
 src/Storages/Statistics/StatisticsTDigest.h   |  2 +-
 src/Storages/Statistics/TDigestStatistics.cpp |  0
 src/Storages/StatisticsDescription.cpp        |  5 ----
 .../02864_statistics_estimate_predicate.sql   |  2 ++
 13 files changed, 65 insertions(+), 76 deletions(-)
 rename src/Storages/Statistics/{CountMinSketchStatistics.cpp => StatisticsCountMinSketch.cpp} (77%)
 rename src/Storages/Statistics/{CountMinSketchStatistics.h => StatisticsCountMinSketch.h} (71%)
 delete mode 100644 src/Storages/Statistics/TDigestStatistics.cpp

diff --git a/docs/en/engines/table-engines/mergetree-family/mergetree.md b/docs/en/engines/table-engines/mergetree-family/mergetree.md
index 6466aebfaa3..7ffbd9a5bae 100644
--- a/docs/en/engines/table-engines/mergetree-family/mergetree.md
+++ b/docs/en/engines/table-engines/mergetree-family/mergetree.md
@@ -1001,7 +1001,7 @@ They can be used for prewhere optimization only if we enable `set allow_statisti
 
 - `count_min`
 
-    [count min sketch](https://en.wikipedia.org/wiki/Count%E2%80%93min_sketch) sketches which provide an approximate count of the frequency of each value in a column.
+    [Count-min](https://en.wikipedia.org/wiki/Count%E2%80%93min_sketch) sketches which provide an approximate count of the frequency of each value in a column.
 
 ## Column-level Settings {#column-level-settings}
 
diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index 6f65f65a4a0..edcd443910e 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -496,11 +496,9 @@ ConditionSelectivityEstimator MergeTreeData::getConditionSelectivityEstimatorByP
         {
             auto stats = part->loadStatistics();
             /// TODO: We only have one stats file for every part.
-            if (stats.empty()) /// No statistics still need add rows count.
-                result.addRows(part->rows_count);
-            else
-                for (const auto & stat : stats)
-                    result.merge(part->info.getPartNameV1(), part->rows_count, stat);
+            result.addRows(part->rows_count);
+            for (const auto & stat : stats)
+                result.merge(part->info.getPartNameV1(), stat);
         }
         catch (...)
         {
@@ -515,11 +513,9 @@ ConditionSelectivityEstimator MergeTreeData::getConditionSelectivityEstimatorByP
             if (!partition_pruner.canBePruned(*part))
             {
                 auto stats = part->loadStatistics();
-                if (stats.empty())
-                    result.addRows(part->rows_count);
-                else
-                    for (const auto & stat : stats)
-                        result.merge(part->info.getPartNameV1(), part->rows_count, stat);
+                result.addRows(part->rows_count);
+                for (const auto & stat : stats)
+                    result.merge(part->info.getPartNameV1(), stat);
             }
         }
         catch (...)
diff --git a/src/Storages/Statistics/ConditionSelectivityEstimator.cpp b/src/Storages/Statistics/ConditionSelectivityEstimator.cpp
index 437b39cb537..1755f0eb4df 100644
--- a/src/Storages/Statistics/ConditionSelectivityEstimator.cpp
+++ b/src/Storages/Statistics/ConditionSelectivityEstimator.cpp
@@ -35,7 +35,7 @@ Float64 ConditionSelectivityEstimator::ColumnSelectivityEstimator::estimateGreat
     return rows - estimateLess(val, rows);
 }
 
-Float64 ConditionSelectivityEstimator::ColumnSelectivityEstimator::estimateEqual(Field val, Float64 rows) const
+Float64 ConditionSelectivityEstimator::ColumnSelectivityEstimator::estimateEqual(const Field & val, Float64 rows) const
 {
     auto float_val = IStatistics::getFloat64(val);
     if (part_statistics.empty())
@@ -141,7 +141,7 @@ Float64 ConditionSelectivityEstimator::estimateRowCount(const RPNBuilderTreeNode
 
     /// If there the estimator of the column is not found or there are no data at all,
     /// we use dummy estimation.
-    bool dummy = total_rows == 0;
+    bool dummy = false;
     ColumnSelectivityEstimator estimator;
     if (it != column_estimators.end())
         estimator = it->second;
@@ -176,13 +176,8 @@ Float64 ConditionSelectivityEstimator::estimateRowCount(const RPNBuilderTreeNode
         return default_unknown_cond_factor * total_rows;
 }
 
-void ConditionSelectivityEstimator::merge(String part_name, UInt64 part_rows, ColumnStatisticsPtr column_stat)
+void ConditionSelectivityEstimator::merge(String part_name, ColumnStatisticsPtr column_stat)
 {
-    if (!part_names.contains(part_name))
-    {
-        total_rows += part_rows;
-        part_names.insert(part_name);
-    }
     if (column_stat != nullptr)
         column_estimators[column_stat->columnName()].merge(part_name, column_stat);
 }
diff --git a/src/Storages/Statistics/ConditionSelectivityEstimator.h b/src/Storages/Statistics/ConditionSelectivityEstimator.h
index ff6218e7ef1..b9127fcd5bf 100644
--- a/src/Storages/Statistics/ConditionSelectivityEstimator.h
+++ b/src/Storages/Statistics/ConditionSelectivityEstimator.h
@@ -11,6 +11,14 @@ class RPNBuilderTreeNode;
 /// It estimates the selectivity of a condition.
 class ConditionSelectivityEstimator
 {
+public:
+    /// TODO: Support the condition consists of CNF/DNF like (cond1 and cond2) or (cond3) ...
+    /// Right now we only support simple condition like col = val / col < val
+    Float64 estimateRowCount(const RPNBuilderTreeNode & node) const;
+
+    void merge(String part_name, ColumnStatisticsPtr column_stat);
+    void addRows(UInt64 part_rows) { total_rows += part_rows; }
+
 private:
     friend class ColumnStatistics;
     struct ColumnSelectivityEstimator
@@ -25,9 +33,11 @@ private:
 
         Float64 estimateGreater(Float64 val, Float64 rows) const;
 
-        Float64 estimateEqual(Field val, Float64 rows) const;
+        Float64 estimateEqual(const Field & val, Float64 rows) const;
     };
 
+    std::pair<String, Field> extractBinaryOp(const RPNBuilderTreeNode & node, const String & column_name) const;
+
     static constexpr auto default_good_cond_factor = 0.1;
     static constexpr auto default_normal_cond_factor = 0.5;
     static constexpr auto default_unknown_cond_factor = 1.0;
@@ -36,17 +46,7 @@ private:
     static constexpr auto threshold = 2;
 
     UInt64 total_rows = 0;
-    std::set<String> part_names;
     std::map<String, ColumnSelectivityEstimator> column_estimators;
-    std::pair<String, Field> extractBinaryOp(const RPNBuilderTreeNode & node, const String & column_name) const;
-
-public:
-    /// TODO: Support the condition consists of CNF/DNF like (cond1 and cond2) or (cond3) ...
-    /// Right now we only support simple condition like col = val / col < val
-    Float64 estimateRowCount(const RPNBuilderTreeNode & node) const;
-
-    void merge(String part_name, UInt64 part_rows, ColumnStatisticsPtr column_stat);
-    void addRows(UInt64 part_rows) { total_rows += part_rows; }
 };
 
 }
diff --git a/src/Storages/Statistics/Statistics.cpp b/src/Storages/Statistics/Statistics.cpp
index d3776a2d38c..cd94ed716cd 100644
--- a/src/Storages/Statistics/Statistics.cpp
+++ b/src/Storages/Statistics/Statistics.cpp
@@ -1,12 +1,12 @@
-#include <Storages/Statistics/Statistics.h>
-#include <Storages/Statistics/ConditionSelectivityEstimator.h>
-#include <Storages/Statistics/StatisticsTDigest.h>
-#include <Storages/Statistics/StatisticsUniq.h>
-#include <Storages/Statistics/CountMinSketchStatistics.h>
-#include <Storages/StatisticsDescription.h>
-#include <Storages/ColumnsDescription.h>
 #include <IO/ReadHelpers.h>
 #include <IO/WriteHelpers.h>
+#include <Storages/ColumnsDescription.h>
+#include <Storages/Statistics/ConditionSelectivityEstimator.h>
+#include <Storages/Statistics/Statistics.h>
+#include <Storages/Statistics/StatisticsCountMinSketch.h>
+#include <Storages/Statistics/StatisticsTDigest.h>
+#include <Storages/Statistics/StatisticsUniq.h>
+#include <Storages/StatisticsDescription.h>
 #include <Common/Exception.h>
 #include <Common/logger_useful.h>
 
@@ -93,7 +93,7 @@ UInt64 IStatistics::estimateCardinality() const
     throw Exception(ErrorCodes::LOGICAL_ERROR, "Cardinality estimation is not implemented for this type of statistics");
 }
 
-Float64 IStatistics::estimateEqual(Float64 /*val*/) const
+Float64 IStatistics::estimateEqual(const Field & /*val*/) const
 {
     throw Exception(ErrorCodes::LOGICAL_ERROR, "Equality estimation is not implemented for this type of statistics");
 }
@@ -125,23 +125,20 @@ Float64 ColumnStatistics::estimateGreater(Float64 val) const
     return rows - estimateLess(val);
 }
 
-Float64 ColumnStatistics::estimateEqual(Field val) const
+Float64 ColumnStatistics::estimateEqual(const Field & val) const
 {
     auto float_val = IStatistics::getFloat64(val);
-    if (stats.contains(StatisticsType::Uniq) && stats.contains(StatisticsType::TDigest))
+    if (float_val.has_value() && stats.contains(StatisticsType::Uniq) && stats.contains(StatisticsType::TDigest))
     {
         /// 2048 is the default number of buckets in TDigest. In this case, TDigest stores exactly one value (with many rows) for every bucket.
         if (stats.at(StatisticsType::Uniq)->estimateCardinality() < 2048)
-            return stats.at(StatisticsType::TDigest)->estimateEqual(float_val);
+            return stats.at(StatisticsType::TDigest)->estimateEqual(val);
     }
 #if USE_DATASKETCHES
     if (stats.contains(StatisticsType::CountMinSketch))
-    {
-        auto count_min_sketch_static = std::static_pointer_cast<CountMinSketchStatistics>(stats.at(StatisticsType::CountMinSketch));
-        return count_min_sketch_static->estimateEqual(val);
-    }
+        return stats.at(StatisticsType::CountMinSketch)->estimateEqual(val);
 #endif
-    if (float_val < - ConditionSelectivityEstimator::threshold || float_val > ConditionSelectivityEstimator::threshold)
+    if (!float_val.has_value() && (float_val < - ConditionSelectivityEstimator::threshold || float_val > ConditionSelectivityEstimator::threshold))
         return rows * ConditionSelectivityEstimator::default_normal_cond_factor;
     else
         return rows * ConditionSelectivityEstimator::default_good_cond_factor;
diff --git a/src/Storages/Statistics/Statistics.h b/src/Storages/Statistics/Statistics.h
index 4b953ddb364..c9bf3ca4847 100644
--- a/src/Storages/Statistics/Statistics.h
+++ b/src/Storages/Statistics/Statistics.h
@@ -35,7 +35,7 @@ public:
 
     /// Per-value estimations.
     /// Throws if the statistics object is not able to do a meaningful estimation.
-    virtual Float64 estimateEqual(Float64 val) const; /// cardinality of val in the column
+    virtual Float64 estimateEqual(const Field & val) const; /// cardinality of val in the column
     virtual Float64 estimateLess(Float64 val) const;  /// summarized cardinality of values < val in the column
 
     /// Convert filed to Float64, used when estimating the number of rows.
@@ -67,8 +67,7 @@ public:
 
     Float64 estimateLess(Float64 val) const;
     Float64 estimateGreater(Float64 val) const;
-    Float64 estimateEqual(Float64 val) const;
-    Float64 estimateEqual(Field val) const;
+    Float64 estimateEqual(const Field & val) const;
 
 private:
     friend class MergeTreeStatisticsFactory;
diff --git a/src/Storages/Statistics/CountMinSketchStatistics.cpp b/src/Storages/Statistics/StatisticsCountMinSketch.cpp
similarity index 77%
rename from src/Storages/Statistics/CountMinSketchStatistics.cpp
rename to src/Storages/Statistics/StatisticsCountMinSketch.cpp
index 497570bd2d1..dd8ceef4e2d 100644
--- a/src/Storages/Statistics/CountMinSketchStatistics.cpp
+++ b/src/Storages/Statistics/StatisticsCountMinSketch.cpp
@@ -1,8 +1,8 @@
-#include <Storages/Statistics/CountMinSketchStatistics.h>
-#include <DataTypes/DataTypeNullable.h>
 #include <DataTypes/DataTypeLowCardinality.h>
+#include <DataTypes/DataTypeNullable.h>
 #include <IO/ReadHelpers.h>
 #include <IO/WriteHelpers.h>
+#include <Storages/Statistics/StatisticsCountMinSketch.h>
 
 #if USE_DATASKETCHES
 
@@ -14,15 +14,17 @@ namespace ErrorCodes
 extern const int ILLEGAL_STATISTICS;
 }
 
+static constexpr auto num_hashes = 8uz;
+static constexpr auto num_buckets = 2048uz;
 
-CountMinSketchStatistics::CountMinSketchStatistics(const SingleStatisticsDescription & stat_, DataTypePtr data_type_)
+StatisticsCountMinSketch::StatisticsCountMinSketch(const SingleStatisticsDescription & stat_, DataTypePtr data_type_)
     : IStatistics(stat_)
     , sketch(num_hashes, num_buckets)
     , data_type(data_type_)
 {
 }
 
-Float64 CountMinSketchStatistics::estimateEqual(const Field & value) const
+Float64 StatisticsCountMinSketch::estimateEqual(const Field & value) const
 {
     if (auto float_val = IStatistics::getFloat64(value))
         return sketch.get_estimate(&float_val.value(), 8);
@@ -31,14 +33,14 @@ Float64 CountMinSketchStatistics::estimateEqual(const Field & value) const
     UNREACHABLE();
 }
 
-void CountMinSketchStatistics::serialize(WriteBuffer & buf)
+void StatisticsCountMinSketch::serialize(WriteBuffer & buf)
 {
     Sketch::vector_bytes bytes = sketch.serialize();
     writeIntBinary(static_cast<UInt64>(bytes.size()), buf);
     buf.write(reinterpret_cast<const char *>(bytes.data()), bytes.size());
 }
 
-void CountMinSketchStatistics::deserialize(ReadBuffer & buf)
+void StatisticsCountMinSketch::deserialize(ReadBuffer & buf)
 {
     UInt64 size;
     readIntBinary(size, buf);
@@ -47,10 +49,10 @@ void CountMinSketchStatistics::deserialize(ReadBuffer & buf)
     bytes.reserve(size);
     buf.readStrict(reinterpret_cast<char *>(bytes.data()), size);
 
-    sketch = datasketches::count_min_sketch<UInt64>::deserialize(bytes.data(), size);
+    sketch = Sketch::deserialize(bytes.data(), size);
 }
 
-void CountMinSketchStatistics::update(const ColumnPtr & column)
+void StatisticsCountMinSketch::update(const ColumnPtr & column)
 {
     size_t size = column->size();
     for (size_t i = 0; i < size; ++i)
@@ -78,7 +80,7 @@ void CountMinSketchValidator(const SingleStatisticsDescription &, DataTypePtr da
 
 StatisticsPtr CountMinSketchCreator(const SingleStatisticsDescription & stat, DataTypePtr data_type)
 {
-    return std::make_shared<CountMinSketchStatistics>(stat, data_type);
+    return std::make_shared<StatisticsCountMinSketch>(stat, data_type);
 }
 
 }
diff --git a/src/Storages/Statistics/CountMinSketchStatistics.h b/src/Storages/Statistics/StatisticsCountMinSketch.h
similarity index 71%
rename from src/Storages/Statistics/CountMinSketchStatistics.h
rename to src/Storages/Statistics/StatisticsCountMinSketch.h
index 52046a19b64..3ea3f2dbd3b 100644
--- a/src/Storages/Statistics/CountMinSketchStatistics.h
+++ b/src/Storages/Statistics/StatisticsCountMinSketch.h
@@ -11,12 +11,12 @@
 namespace DB
 {
 
-class CountMinSketchStatistics : public IStatistics
+class StatisticsCountMinSketch : public IStatistics
 {
 public:
-    CountMinSketchStatistics(const SingleStatisticsDescription & stat_, DataTypePtr data_type_);
+    StatisticsCountMinSketch(const SingleStatisticsDescription & stat_, DataTypePtr data_type_);
 
-    Float64 estimateEqual(const Field & value) const;
+    Float64 estimateEqual(const Field & value) const override;
 
     void serialize(WriteBuffer & buf) override;
     void deserialize(ReadBuffer & buf) override;
@@ -24,9 +24,6 @@ public:
     void update(const ColumnPtr & column) override;
 
 private:
-    static constexpr auto num_hashes = 8uz;
-    static constexpr auto num_buckets = 2048uz;
-
     using Sketch = datasketches::count_min_sketch<UInt64>;
     Sketch sketch;
 
diff --git a/src/Storages/Statistics/StatisticsTDigest.cpp b/src/Storages/Statistics/StatisticsTDigest.cpp
index 0747197370c..306338b4ba2 100644
--- a/src/Storages/Statistics/StatisticsTDigest.cpp
+++ b/src/Storages/Statistics/StatisticsTDigest.cpp
@@ -1,5 +1,6 @@
 #include <Storages/Statistics/StatisticsTDigest.h>
 #include <DataTypes/DataTypeNullable.h>
+#include <DataTypes/DataTypeLowCardinality.h>
 
 namespace DB
 {
@@ -16,12 +17,16 @@ StatisticsTDigest::StatisticsTDigest(const SingleStatisticsDescription & stat_)
 void StatisticsTDigest::update(const ColumnPtr & column)
 {
     size_t rows = column->size();
-
     for (size_t row = 0; row < rows; ++row)
     {
-        /// TODO: support more types.
-        Float64 value = column->getFloat64(row);
-        t_digest.add(value, 1);
+        Field f;
+        column->get(row, f);
+
+        if (f.isNull())
+            continue;
+
+        if (auto float_val = IStatistics::getFloat64(f))
+            t_digest.add(*float_val, 1);
     }
 }
 
@@ -40,14 +45,15 @@ Float64 StatisticsTDigest::estimateLess(Float64 val) const
     return t_digest.getCountLessThan(val);
 }
 
-Float64 StatisticsTDigest::estimateEqual(Float64 val) const
+Float64 StatisticsTDigest::estimateEqual(const Field & val) const
 {
-    return t_digest.getCountEqual(val);
+    return t_digest.getCountEqual(IStatistics::getFloat64(val).value());
 }
 
 void TDigestValidator(const SingleStatisticsDescription &, DataTypePtr data_type)
 {
     data_type = removeNullable(data_type);
+    data_type = removeLowCardinalityAndNullable(data_type);
     if (!data_type->isValueRepresentedByNumber())
         throw Exception(ErrorCodes::ILLEGAL_STATISTICS, "Statistics of type 'tdigest' do not support type {}", data_type->getName());
 }
diff --git a/src/Storages/Statistics/StatisticsTDigest.h b/src/Storages/Statistics/StatisticsTDigest.h
index d3a3bf115ee..8016faac7c6 100644
--- a/src/Storages/Statistics/StatisticsTDigest.h
+++ b/src/Storages/Statistics/StatisticsTDigest.h
@@ -17,7 +17,7 @@ public:
     void deserialize(ReadBuffer & buf) override;
 
     Float64 estimateLess(Float64 val) const override;
-    Float64 estimateEqual(Float64 val) const override;
+    Float64 estimateEqual(const Field & val) const override;
 
 private:
     QuantileTDigest<Float64> t_digest;
diff --git a/src/Storages/Statistics/TDigestStatistics.cpp b/src/Storages/Statistics/TDigestStatistics.cpp
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/src/Storages/StatisticsDescription.cpp b/src/Storages/StatisticsDescription.cpp
index d59ffb5c8e9..8aa954f5eb5 100644
--- a/src/Storages/StatisticsDescription.cpp
+++ b/src/Storages/StatisticsDescription.cpp
@@ -1,19 +1,14 @@
 #include <Storages/StatisticsDescription.h>
 
-#include <base/defines.h>
 #include <Parsers/ASTExpressionList.h>
 #include <Parsers/ASTFunction.h>
 #include <Parsers/ASTIdentifier.h>
 #include <Parsers/ASTStatisticsDeclaration.h>
-#include <Parsers/formatAST.h>
-#include <Parsers/parseQuery.h>
 #include <Parsers/queryToString.h>
 #include <Parsers/ParserCreateQuery.h>
 #include <Poco/Logger.h>
-#include <Storages/extractKeyExpressionList.h>
 #include <Storages/ColumnsDescription.h>
 
-#include <Common/logger_useful.h>
 
 namespace DB
 {
diff --git a/tests/queries/0_stateless/02864_statistics_estimate_predicate.sql b/tests/queries/0_stateless/02864_statistics_estimate_predicate.sql
index a608f18a354..61bae842631 100644
--- a/tests/queries/0_stateless/02864_statistics_estimate_predicate.sql
+++ b/tests/queries/0_stateless/02864_statistics_estimate_predicate.sql
@@ -27,6 +27,7 @@ ALTER TABLE t1 MATERIALIZE STATISTICS b, c;
 SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_String', '')
 FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE b > 0/*9990*/ and c < -98/*100*/)
 WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
+
 SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_String', '')
 FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE b = 0/*1000*/ and c < -98/*100*/)
 WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
@@ -69,6 +70,7 @@ ALTER TABLE t1 MATERIALIZE STATISTICS a, b, c;
 SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_String', '')
 FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE c < -90/*900*/ and b > 900/*990*/ and a = '0'/*1*/)
 WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
+
 SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_String', '')
 FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE c < 0/*9900*/ and b = 0/*10*/ and a = '10000'/*0*/)
 WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';

From 84853f6d4b6e37160888f46ba61a8a1e70d7583a Mon Sep 17 00:00:00 2001
From: JackyWoo <wuchienchao@qq.com>
Date: Thu, 4 Jul 2024 16:18:11 +0800
Subject: [PATCH 0153/1488] Fix style checking

---
 ...64_statistics_estimate_predicate.reference |  2 +-
 .../02864_statistics_estimate_predicate.sql   | 64 ++++++++++---------
 .../0_stateless/02864_statistics_uniq.sql     |  0
 3 files changed, 34 insertions(+), 32 deletions(-)
 delete mode 100644 tests/queries/0_stateless/02864_statistics_uniq.sql

diff --git a/tests/queries/0_stateless/02864_statistics_estimate_predicate.reference b/tests/queries/0_stateless/02864_statistics_estimate_predicate.reference
index 7215d5fef58..7c22f308ab9 100644
--- a/tests/queries/0_stateless/02864_statistics_estimate_predicate.reference
+++ b/tests/queries/0_stateless/02864_statistics_estimate_predicate.reference
@@ -22,4 +22,4 @@ Test statistics multi-types:
           Prewhere filter
           Prewhere filter column: and(equals(a, \'10000\'), equals(b, 0), less(c, 0)) (removed)
 Test LowCardinality and Nullable data type:
-t2
+tab2
diff --git a/tests/queries/0_stateless/02864_statistics_estimate_predicate.sql b/tests/queries/0_stateless/02864_statistics_estimate_predicate.sql
index 61bae842631..7fcb85d80f5 100644
--- a/tests/queries/0_stateless/02864_statistics_estimate_predicate.sql
+++ b/tests/queries/0_stateless/02864_statistics_estimate_predicate.sql
@@ -1,12 +1,14 @@
 -- Tags: no-fasttest
-DROP TABLE IF EXISTS t1 SYNC;
+-- Tests statistics usages in prewhere optimization.
+
+DROP TABLE IF EXISTS tab SYNC;
 
 SET allow_experimental_statistics = 1;
 SET allow_statistics_optimize = 1;
 SET allow_suspicious_low_cardinality_types=1;
 SET mutations_sync = 2;
 
-CREATE TABLE t1
+CREATE TABLE tab
 (
     a String,
     b UInt64,
@@ -15,75 +17,75 @@ CREATE TABLE t1
 ) Engine = MergeTree() ORDER BY pk
 SETTINGS min_bytes_for_wide_part = 0;
 
-SHOW CREATE TABLE t1;
+SHOW CREATE TABLE tab;
 
-INSERT INTO t1 select toString(number % 10000), number % 1000, -(number % 100), generateUUIDv4() FROM system.numbers LIMIT 10000;
+INSERT INTO tab select toString(number % 10000), number % 1000, -(number % 100), generateUUIDv4() FROM system.numbers LIMIT 10000;
 
 SELECT 'Test statistics TDigest:';
 
-ALTER TABLE t1 ADD STATISTICS b, c TYPE tdigest;
-ALTER TABLE t1 MATERIALIZE STATISTICS b, c;
+ALTER TABLE tab ADD STATISTICS b, c TYPE tdigest;
+ALTER TABLE tab MATERIALIZE STATISTICS b, c;
 
 SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_String', '')
-FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE b > 0/*9990*/ and c < -98/*100*/)
+FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE b > 0/*9990*/ and c < -98/*100*/)
 WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
 
 SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_String', '')
-FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE b = 0/*1000*/ and c < -98/*100*/)
+FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE b = 0/*1000*/ and c < -98/*100*/)
 WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
 
-ALTER TABLE t1 DROP STATISTICS b, c;
+ALTER TABLE tab DROP STATISTICS b, c;
 
 
 SELECT 'Test statistics Uniq:';
 
-ALTER TABLE t1 ADD STATISTICS b TYPE uniq, tdigest;
-ALTER TABLE t1 MATERIALIZE STATISTICS b;
+ALTER TABLE tab ADD STATISTICS b TYPE uniq, tdigest;
+ALTER TABLE tab MATERIALIZE STATISTICS b;
 SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_String', '')
-FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE c = 0/*1000*/ and b = 0/*10*/)
+FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE c = 0/*1000*/ and b = 0/*10*/)
 WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
 
-ALTER TABLE t1 DROP STATISTICS b;
+ALTER TABLE tab DROP STATISTICS b;
 
 
 SELECT 'Test statistics count_min:';
 
-ALTER TABLE t1 ADD STATISTICS a TYPE count_min;
-ALTER TABLE t1 ADD STATISTICS b TYPE count_min;
-ALTER TABLE t1 ADD STATISTICS c TYPE count_min;
-ALTER TABLE t1 MATERIALIZE STATISTICS a, b, c;
+ALTER TABLE tab ADD STATISTICS a TYPE count_min;
+ALTER TABLE tab ADD STATISTICS b TYPE count_min;
+ALTER TABLE tab ADD STATISTICS c TYPE count_min;
+ALTER TABLE tab MATERIALIZE STATISTICS a, b, c;
 
 SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_String', '')
-FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE c = 0/*100*/ and b = 0/*10*/ and a = '0'/*1*/) xx
+FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE c = 0/*100*/ and b = 0/*10*/ and a = '0'/*1*/) xx
 WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
 
-ALTER TABLE t1 DROP STATISTICS a, b, c;
+ALTER TABLE tab DROP STATISTICS a, b, c;
 
 
 SELECT 'Test statistics multi-types:';
 
-ALTER TABLE t1 ADD STATISTICS a TYPE count_min;
-ALTER TABLE t1 ADD STATISTICS b TYPE count_min, uniq, tdigest;
-ALTER TABLE t1 ADD STATISTICS c TYPE count_min, uniq, tdigest;
-ALTER TABLE t1 MATERIALIZE STATISTICS a, b, c;
+ALTER TABLE tab ADD STATISTICS a TYPE count_min;
+ALTER TABLE tab ADD STATISTICS b TYPE count_min, uniq, tdigest;
+ALTER TABLE tab ADD STATISTICS c TYPE count_min, uniq, tdigest;
+ALTER TABLE tab MATERIALIZE STATISTICS a, b, c;
 
 SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_String', '')
-FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE c < -90/*900*/ and b > 900/*990*/ and a = '0'/*1*/)
+FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE c < -90/*900*/ and b > 900/*990*/ and a = '0'/*1*/)
 WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
 
 SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_String', '')
-FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE c < 0/*9900*/ and b = 0/*10*/ and a = '10000'/*0*/)
+FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE c < 0/*9900*/ and b = 0/*10*/ and a = '10000'/*0*/)
 WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
 
-ALTER TABLE t1 DROP STATISTICS a, b, c;
+ALTER TABLE tab DROP STATISTICS a, b, c;
 
-DROP TABLE IF EXISTS t1 SYNC;
+DROP TABLE IF EXISTS tab SYNC;
 
 
 SELECT 'Test LowCardinality and Nullable data type:';
-DROP TABLE IF EXISTS t2 SYNC;
+DROP TABLE IF EXISTS tab2 SYNC;
 SET allow_suspicious_low_cardinality_types=1;
-CREATE TABLE t2
+CREATE TABLE tab2
 (
     a LowCardinality(Int64) STATISTICS(uniq, tdigest, count_min),
     b Nullable(Int64) STATISTICS(uniq, tdigest, count_min),
@@ -91,6 +93,6 @@ CREATE TABLE t2
     pk String,
 ) Engine = MergeTree() ORDER BY pk;
 
-select table from system.tables where name = 't2';
+select name from system.tables where name = 'tab2' and database = currentDatabase();
 
-DROP TABLE IF EXISTS t2 SYNC;
+DROP TABLE IF EXISTS tab2 SYNC;
diff --git a/tests/queries/0_stateless/02864_statistics_uniq.sql b/tests/queries/0_stateless/02864_statistics_uniq.sql
deleted file mode 100644
index e69de29bb2d..00000000000

From cc3e166dec6fde41c3903d486e4e18ba9b2991b2 Mon Sep 17 00:00:00 2001
From: Konstantin Morozov <just.morozov.k@gmail.com>
Date: Thu, 4 Jul 2024 12:11:15 +0000
Subject: [PATCH 0154/1488] apply comment

---
 src/Storages/System/StorageSystemDetachedTables.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Storages/System/StorageSystemDetachedTables.cpp b/src/Storages/System/StorageSystemDetachedTables.cpp
index 158edb7818f..56c5e49b467 100644
--- a/src/Storages/System/StorageSystemDetachedTables.cpp
+++ b/src/Storages/System/StorageSystemDetachedTables.cpp
@@ -94,7 +94,7 @@ protected:
             }
         }
 
-        if (databases->size() == database_idx && max_block_size != rows_count)
+        if (databases->size() == database_idx && (!detached_tables_it || !detached_tables_it->isValid()))
         {
             done = true;
         }

From accb994dbef7eb0a1e6c5eaf5f9bbeae70f1b28a Mon Sep 17 00:00:00 2001
From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com>
Date: Thu, 4 Jul 2024 18:43:45 +0200
Subject: [PATCH 0155/1488] init

---
 .../AggregateFunctionGroupArrayIntersect.cpp  | 44 ++++++++++---------
 .../03198_group_array_intersect.reference     |  4 ++
 .../03198_group_array_intersect.sql           | 10 +++++
 3 files changed, 38 insertions(+), 20 deletions(-)
 create mode 100644 tests/queries/0_stateless/03198_group_array_intersect.reference
 create mode 100644 tests/queries/0_stateless/03198_group_array_intersect.sql

diff --git a/src/AggregateFunctions/AggregateFunctionGroupArrayIntersect.cpp b/src/AggregateFunctions/AggregateFunctionGroupArrayIntersect.cpp
index 903adf5c547..0da5ade1c79 100644
--- a/src/AggregateFunctions/AggregateFunctionGroupArrayIntersect.cpp
+++ b/src/AggregateFunctions/AggregateFunctionGroupArrayIntersect.cpp
@@ -111,14 +111,12 @@ public:
             return;
 
         UInt64 version = this->data(place).version++;
-        if (version == 0)
+        if (version == 1)
         {
             for (auto & rhs_elem : rhs_set)
                 set.insert(rhs_elem.getValue());
-            return;
         }
-
-        if (!set.empty())
+        else if (!set.empty())
         {
             auto create_new_set = [](auto & lhs_val, auto & rhs_val)
             {
@@ -150,8 +148,18 @@ public:
 
     void deserialize(AggregateDataPtr __restrict place, ReadBuffer & buf, std::optional<size_t> /* version */, Arena *) const override
     {
-        readVarUInt(this->data(place).version, buf);
-        this->data(place).value.read(buf);
+        auto & set = this->data(place).value;
+        auto & version = this->data(place).version;
+        size_t size;
+        readVarUInt(version, buf);
+        readVarUInt(size, buf);
+        set.reserve(size);
+        for (size_t i = 0; i < size; ++i)
+        {
+            int key;
+            readIntBinary(key, buf);
+            set.insert(key);
+        }
     }
 
     void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena *) const override
@@ -270,30 +278,28 @@ public:
             return;
 
         UInt64 version = this->data(place).version++;
-        if (version == 0)
+        if (version == 1)
         {
             bool inserted;
             State::Set::LookupResult it;
             for (auto & rhs_elem : rhs_value)
-            {
                 set.emplace(ArenaKeyHolder{rhs_elem.getValue(), *arena}, it, inserted);
-            }
         }
         else if (!set.empty())
         {
-            auto create_new_map = [](auto & lhs_val, auto & rhs_val)
+            auto create_matched_set = [](auto & lhs_val, auto & rhs_val)
             {
-                typename State::Set new_map;
-                for (auto & lhs_elem : lhs_val)
+                typename State::Set new_set;
+                for (const auto & lhs_elem : lhs_val)
                 {
-                    auto val = rhs_val.find(lhs_elem.getValue());
-                    if (val != nullptr)
-                        new_map.insert(lhs_elem.getValue());
+                    auto is_match = rhs_val.find(lhs_elem.getValue());
+                    if (is_match != nullptr)
+                        new_set.insert(lhs_elem.getValue());
                 }
-                return new_map;
+                return new_set;
             };
-            auto new_map = rhs_value.size() < set.size() ? create_new_map(rhs_value, set) : create_new_map(set, rhs_value);
-            set = std::move(new_map);
+            auto matched_set = rhs_value.size() < set.size() ? create_matched_set(rhs_value, set) : create_matched_set(set, rhs_value);
+            set = std::move(matched_set);
         }
     }
 
@@ -316,11 +322,9 @@ public:
         readVarUInt(version, buf);
         readVarUInt(size, buf);
         set.reserve(size);
-        UInt64 elem_version;
         for (size_t i = 0; i < size; ++i)
         {
             auto key = readStringBinaryInto(*arena, buf);
-            readVarUInt(elem_version, buf);
             set.insert(key);
         }
     }
diff --git a/tests/queries/0_stateless/03198_group_array_intersect.reference b/tests/queries/0_stateless/03198_group_array_intersect.reference
new file mode 100644
index 00000000000..f6b2def2f55
--- /dev/null
+++ b/tests/queries/0_stateless/03198_group_array_intersect.reference
@@ -0,0 +1,4 @@
+['6\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0','2\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0','3\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0','5\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0','1\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0','4\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0']
+['2\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0','5\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0','1\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0','4\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0']
+['3\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0','5\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0','1\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0','0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0','4\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0']
+['5\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0','1\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0','4\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0']
diff --git a/tests/queries/0_stateless/03198_group_array_intersect.sql b/tests/queries/0_stateless/03198_group_array_intersect.sql
new file mode 100644
index 00000000000..44422fd0ba6
--- /dev/null
+++ b/tests/queries/0_stateless/03198_group_array_intersect.sql
@@ -0,0 +1,10 @@
+DROP TABLE IF EXISTS test_numbers__fuzz_29;
+CREATE TABLE test_numbers__fuzz_29 (`a` Array(Nullable(FixedString(19)))) ENGINE = MergeTree ORDER BY a SETTINGS allow_nullable_key=1;
+
+INSERT INTO test_numbers__fuzz_29 VALUES ([1,2,3,4,5,6]);
+INSERT INTO test_numbers__fuzz_29 VALUES ([1,2,4,5]);
+INSERT INTO test_numbers__fuzz_29 VALUES ([1,4,3,0,5,5,5]);
+
+SELECT groupArrayIntersect(*) FROM test_numbers__fuzz_29 GROUP BY a WITH ROLLUP;
+
+DROP TABLE test_numbers__fuzz_29;

From 07c0a72e5f06283d0d70f076eb8f33ee1339e9c8 Mon Sep 17 00:00:00 2001
From: Michael Kolupaev <michael.kolupaev@clickhouse.com>
Date: Thu, 4 Jul 2024 18:30:55 +0000
Subject: [PATCH 0156/1488] Style

---
 src/Storages/System/IStorageSystemOneBlock.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Storages/System/IStorageSystemOneBlock.cpp b/src/Storages/System/IStorageSystemOneBlock.cpp
index 7cde31905aa..308b34510ea 100644
--- a/src/Storages/System/IStorageSystemOneBlock.cpp
+++ b/src/Storages/System/IStorageSystemOneBlock.cpp
@@ -80,9 +80,9 @@ void IStorageSystemOneBlock::read(
 
 void ReadFromSystemOneBlock::initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &)
 {
-    const auto & sample_block = getOutputStream().header;
+    const Block & sample_block = getOutputStream().header;
     MutableColumns res_columns = sample_block.cloneEmptyColumns();
-    auto predicate = filter ? filter->getOutputs().at(0) : nullptr;
+    const ActionsDAG::Node * predicate = filter ? filter->getOutputs().at(0) : nullptr;
     storage->fillData(res_columns, context, predicate, std::move(columns_mask));
 
     UInt64 num_rows = res_columns.at(0)->size();

From 90df83438f9866363690239ee9ec386a303dc3ba Mon Sep 17 00:00:00 2001
From: Michael Kolupaev <michael.kolupaev@clickhouse.com>
Date: Fri, 5 Jul 2024 04:42:58 +0000
Subject: [PATCH 0157/1488] more shellcheck

---
 docker/test/stateless/attach_gdb.lib | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docker/test/stateless/attach_gdb.lib b/docker/test/stateless/attach_gdb.lib
index a3616ac1a04..a0d1b8af6d8 100644
--- a/docker/test/stateless/attach_gdb.lib
+++ b/docker/test/stateless/attach_gdb.lib
@@ -11,7 +11,8 @@ function attach_gdb_to_clickhouse()
     # explicitly ignore non-fatal signals that are used by server.
     # Number of SIGRTMIN can be determined only in runtime.
     RTMIN=$(kill -l SIGRTMIN)
-  echo "
+    # shellcheck disable=SC2016
+    echo "
 set follow-fork-mode parent
 handle SIGHUP nostop noprint pass
 handle SIGINT nostop noprint pass

From 5e4a244faf99299f0ab4b4f105cc08561c00857b Mon Sep 17 00:00:00 2001
From: Sema Checherinda <104093494+CheSema@users.noreply.github.com>
Date: Fri, 5 Jul 2024 16:52:51 +0200
Subject: [PATCH 0158/1488] Revert "Revert "insertion deduplication on retries
 for materialised views""

---
 src/Columns/ColumnObject.cpp                  |   6 +
 src/Columns/ColumnObject.h                    |   2 +-
 src/Common/CollectionOfDerived.h              | 184 ++++
 src/Core/Settings.h                           |   6 +-
 src/Interpreters/AsynchronousInsertQueue.cpp  |  19 +-
 src/Interpreters/InterpreterCheckQuery.cpp    |  18 +-
 src/Interpreters/InterpreterCreateQuery.cpp   |   9 +-
 src/Interpreters/InterpreterExplainQuery.cpp  |   8 +-
 src/Interpreters/InterpreterInsertQuery.cpp   | 678 ++++++------
 src/Interpreters/InterpreterInsertQuery.h     |  17 +-
 src/Interpreters/Squashing.cpp                | 124 +--
 src/Interpreters/Squashing.h                  |  46 +-
 src/Interpreters/SystemLog.cpp                |   8 +-
 src/Interpreters/TreeRewriter.cpp             |   2 +-
 src/Processors/Chunk.cpp                      |  20 +-
 src/Processors/Chunk.h                        |  58 +-
 .../PullingAsyncPipelineExecutor.cpp          |   9 +-
 .../Executors/PullingPipelineExecutor.cpp     |   9 +-
 .../Formats/Impl/ParquetBlockOutputFormat.cpp |   4 +-
 src/Processors/IAccumulatingTransform.cpp     |   5 +-
 .../FinishAggregatingInOrderAlgorithm.cpp     |  10 +-
 .../Algorithms/MergeTreePartLevelInfo.h       |  12 +-
 .../Algorithms/ReplacingSortedAlgorithm.cpp   |   2 +-
 .../Algorithms/ReplacingSortedAlgorithm.h     |   7 +-
 src/Processors/Merges/IMergingTransform.cpp   |   2 +-
 src/Processors/Merges/IMergingTransform.h     |   2 +-
 src/Processors/Sinks/RemoteSink.h             |   2 +-
 src/Processors/Sinks/SinkToStorage.cpp        |   5 +-
 src/Processors/Sinks/SinkToStorage.h          |   5 +-
 src/Processors/Sources/BlocksSource.h         |   5 +-
 src/Processors/Sources/RemoteSource.cpp       |   2 +-
 .../Sources/SourceFromSingleChunk.cpp         |   6 +-
 .../AggregatingInOrderTransform.cpp           |  11 +-
 .../Transforms/AggregatingInOrderTransform.h  |   5 +-
 .../Transforms/AggregatingTransform.cpp       |  16 +-
 .../Transforms/AggregatingTransform.h         |   3 +-
 .../Transforms/ApplySquashingTransform.h      |  14 +-
 .../Transforms/CountingTransform.cpp          |   5 +-
 .../DeduplicationTokenTransforms.cpp          | 236 +++++
 .../Transforms/DeduplicationTokenTransforms.h | 237 +++++
 .../Transforms/ExpressionTransform.cpp        |   2 +
 .../Transforms/JoiningTransform.cpp           |   9 +-
 src/Processors/Transforms/JoiningTransform.h  |   6 +-
 .../Transforms/MaterializingTransform.cpp     |   1 +
 .../Transforms/MemoryBoundMerging.h           |   6 +-
 ...gingAggregatedMemoryEfficientTransform.cpp |  36 +-
 ...ergingAggregatedMemoryEfficientTransform.h |   5 +-
 .../Transforms/MergingAggregatedTransform.cpp |  10 +-
 .../Transforms/PlanSquashingTransform.cpp     |  15 +-
 .../Transforms/PlanSquashingTransform.h       |   3 +-
 .../Transforms/SelectByIndicesTransform.h     |   3 +-
 .../Transforms/SquashingTransform.cpp         |  18 +-
 .../Transforms/TotalsHavingTransform.cpp      |   6 +-
 .../Transforms/buildPushingToViewsChain.cpp   | 127 ++-
 src/QueryPipeline/QueryPipelineBuilder.h      |   2 +-
 src/QueryPipeline/QueryPlanResourceHolder.cpp |   8 +-
 src/QueryPipeline/QueryPlanResourceHolder.h   |   3 +
 src/Server/TCPHandler.cpp                     |  20 +-
 src/Storages/Distributed/DistributedSink.cpp  |  20 +-
 src/Storages/Distributed/DistributedSink.h    |   2 +-
 src/Storages/FileLog/StorageFileLog.cpp       |   9 +-
 src/Storages/Kafka/StorageKafka.cpp           |   8 +-
 src/Storages/LiveView/LiveViewSink.h          |   4 +-
 src/Storages/LiveView/StorageLiveView.cpp     |  18 +-
 src/Storages/LiveView/StorageLiveView.h       |   2 +-
 src/Storages/MaterializedView/RefreshTask.cpp |   8 +-
 src/Storages/MergeTree/IMergeTreeDataPart.cpp |  17 +-
 src/Storages/MergeTree/IMergeTreeDataPart.h   |   1 +
 .../MergeTree/MergeTreeSelectProcessor.cpp    |   6 +-
 .../MergeTree/MergeTreeSequentialSource.cpp   |   5 +-
 src/Storages/MergeTree/MergeTreeSink.cpp      |  68 +-
 src/Storages/MergeTree/MergeTreeSink.h        |   3 +-
 src/Storages/MergeTree/MutateTask.cpp         |  19 +-
 .../MergeTree/ReplicatedMergeTreeSink.cpp     |  94 +-
 .../MergeTree/ReplicatedMergeTreeSink.h       |  13 +-
 src/Storages/MessageQueueSink.cpp             |   2 +-
 src/Storages/MessageQueueSink.h               |   2 +-
 src/Storages/NATS/StorageNATS.cpp             |   8 +-
 .../StorageObjectStorageSink.cpp              |   4 +-
 .../ObjectStorage/StorageObjectStorageSink.h  |   2 +-
 .../StorageObjectStorageQueue.cpp             |   8 +-
 src/Storages/PartitionedSink.cpp              |   4 +-
 src/Storages/PartitionedSink.h                |   2 +-
 .../MaterializedPostgreSQLConsumer.cpp        |   8 +-
 .../PostgreSQLReplicationHandler.cpp          |   8 +-
 src/Storages/RabbitMQ/StorageRabbitMQ.cpp     |   8 +-
 .../RocksDB/EmbeddedRocksDBBulkSink.cpp       |   7 +-
 .../RocksDB/EmbeddedRocksDBBulkSink.h         |   2 +-
 src/Storages/RocksDB/EmbeddedRocksDBSink.cpp  |   2 +-
 src/Storages/RocksDB/EmbeddedRocksDBSink.h    |   2 +-
 .../RocksDB/StorageEmbeddedRocksDB.cpp        |   3 +-
 src/Storages/StorageBuffer.cpp                |  10 +-
 src/Storages/StorageDistributed.cpp           |   8 +-
 src/Storages/StorageFile.cpp                  |   4 +-
 src/Storages/StorageKeeperMap.cpp             |   9 +-
 src/Storages/StorageLog.cpp                   |   8 +-
 src/Storages/StorageMemory.cpp                |   2 +-
 src/Storages/StorageMongoDB.cpp               |   5 +-
 src/Storages/StorageMySQL.cpp                 |   4 +-
 src/Storages/StoragePostgreSQL.cpp            |   4 +-
 src/Storages/StorageRedis.cpp                 |   9 +-
 src/Storages/StorageSQLite.cpp                |   2 +-
 src/Storages/StorageSet.cpp                   |   6 +-
 src/Storages/StorageStripeLog.cpp             |   4 +-
 src/Storages/StorageURL.cpp                   |   4 +-
 src/Storages/StorageURL.h                     |   2 +-
 .../System/StorageSystemZooKeeper.cpp         |   2 +-
 src/Storages/WindowView/StorageWindowView.cpp |  59 +-
 src/Storages/WindowView/StorageWindowView.h   |   2 +-
 .../test_force_deduplication/test.py          |  73 +-
 ...view_and_deduplication_zookeeper.reference |   2 +-
 ...lized_view_and_deduplication_zookeeper.sql |   2 +-
 ...lized_view_and_too_many_parts_zookeeper.sh |   4 +-
 .../0_stateless/01275_parallel_mv.reference   |   4 +-
 ...01927_query_views_log_current_database.sql |   1 +
 ...ication_token_materialized_views.reference |  14 +-
 ...deduplication_token_materialized_views.sql |   8 +-
 .../0_stateless/02125_query_views_log.sql     |   2 +-
 ...02912_ingestion_mv_deduplication.reference |   5 +-
 .../02912_ingestion_mv_deduplication.sql      |   5 +-
 .../0_stateless/03008_deduplication.python    | 657 ++++++++++++
 ...08_deduplication_cases_from_docs.reference |  41 +
 .../03008_deduplication_cases_from_docs.sql   | 331 ++++++
 ...on_insert_into_partitioned_table.reference |  35 +
 ...lication_insert_into_partitioned_table.sql |  83 ++
 ...ert_several_blocks_nonreplicated.reference | 962 ++++++++++++++++++
 ...ion_insert_several_blocks_nonreplicated.sh |  59 ++
 ...insert_several_blocks_replicated.reference | 962 ++++++++++++++++++
 ...cation_insert_several_blocks_replicated.sh |  59 ++
 ...tes_several_blocks_nonreplicated.reference | 962 ++++++++++++++++++
 ..._generates_several_blocks_nonreplicated.sh |  59 ++
 ...erates_several_blocks_replicated.reference | 962 ++++++++++++++++++
 ..._mv_generates_several_blocks_replicated.sh |  59 ++
 ..._mv_into_one_table_nonreplicated.reference | 706 +++++++++++++
 ...several_mv_into_one_table_nonreplicated.sh |  59 ++
 ...ral_mv_into_one_table_replicated.reference | 706 +++++++++++++
 ...on_several_mv_into_one_table_replicated.sh |  59 ++
 .../03035_max_insert_threads_support.sh       |   2 +-
 138 files changed, 8638 insertions(+), 857 deletions(-)
 create mode 100644 src/Common/CollectionOfDerived.h
 create mode 100644 src/Processors/Transforms/DeduplicationTokenTransforms.cpp
 create mode 100644 src/Processors/Transforms/DeduplicationTokenTransforms.h
 create mode 100644 tests/queries/0_stateless/03008_deduplication.python
 create mode 100644 tests/queries/0_stateless/03008_deduplication_cases_from_docs.reference
 create mode 100644 tests/queries/0_stateless/03008_deduplication_cases_from_docs.sql
 create mode 100644 tests/queries/0_stateless/03008_deduplication_insert_into_partitioned_table.reference
 create mode 100644 tests/queries/0_stateless/03008_deduplication_insert_into_partitioned_table.sql
 create mode 100644 tests/queries/0_stateless/03008_deduplication_insert_several_blocks_nonreplicated.reference
 create mode 100755 tests/queries/0_stateless/03008_deduplication_insert_several_blocks_nonreplicated.sh
 create mode 100644 tests/queries/0_stateless/03008_deduplication_insert_several_blocks_replicated.reference
 create mode 100755 tests/queries/0_stateless/03008_deduplication_insert_several_blocks_replicated.sh
 create mode 100644 tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_nonreplicated.reference
 create mode 100755 tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_nonreplicated.sh
 create mode 100644 tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_replicated.reference
 create mode 100755 tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_replicated.sh
 create mode 100644 tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_nonreplicated.reference
 create mode 100755 tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_nonreplicated.sh
 create mode 100644 tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_replicated.reference
 create mode 100755 tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_replicated.sh

diff --git a/src/Columns/ColumnObject.cpp b/src/Columns/ColumnObject.cpp
index 90ef974010c..ded56b60e64 100644
--- a/src/Columns/ColumnObject.cpp
+++ b/src/Columns/ColumnObject.cpp
@@ -1093,4 +1093,10 @@ void ColumnObject::finalize()
     checkObjectHasNoAmbiguosPaths(getKeys());
 }
 
+void ColumnObject::updateHashFast(SipHash & hash) const
+{
+    for (const auto & entry : subcolumns)
+        for (auto & part : entry->data.data)
+            part->updateHashFast(hash);
+}
 }
diff --git a/src/Columns/ColumnObject.h b/src/Columns/ColumnObject.h
index e2936b27994..b1b8827622f 100644
--- a/src/Columns/ColumnObject.h
+++ b/src/Columns/ColumnObject.h
@@ -242,7 +242,7 @@ public:
     const char * skipSerializedInArena(const char *) const override { throwMustBeConcrete(); }
     void updateHashWithValue(size_t, SipHash &) const override { throwMustBeConcrete(); }
     void updateWeakHash32(WeakHash32 &) const override { throwMustBeConcrete(); }
-    void updateHashFast(SipHash &) const override { throwMustBeConcrete(); }
+    void updateHashFast(SipHash & hash) const override;
     void expand(const Filter &, bool) override { throwMustBeConcrete(); }
     bool hasEqualValues() const override { throwMustBeConcrete(); }
     size_t byteSizeAt(size_t) const override { throwMustBeConcrete(); }
diff --git a/src/Common/CollectionOfDerived.h b/src/Common/CollectionOfDerived.h
new file mode 100644
index 00000000000..97c0c3fbc06
--- /dev/null
+++ b/src/Common/CollectionOfDerived.h
@@ -0,0 +1,184 @@
+#pragma once
+
+#include <base/defines.h>
+
+#include <Common/Exception.h>
+
+#include <algorithm>
+#include <memory>
+#include <typeindex>
+#include <vector>
+#include <string>
+
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int LOGICAL_ERROR;
+}
+
+/* This is a collections of objects derived from ItemBase.
+*  Collection contains no more than one instance for each derived type.
+*  The derived type is used to access the instance.
+*/
+
+template<class ItemBase>
+class CollectionOfDerivedItems
+{
+public:
+    using Self = CollectionOfDerivedItems<ItemBase>;
+    using ItemPtr = std::shared_ptr<ItemBase>;
+
+private:
+    struct Rec
+    {
+        std::type_index type_idx;
+        ItemPtr ptr;
+
+        bool operator<(const Rec & other) const
+        {
+            return type_idx < other.type_idx;
+        }
+
+        bool operator<(const std::type_index & value) const
+        {
+            return type_idx < value;
+        }
+
+        bool operator==(const Rec & other) const
+        {
+            return type_idx == other.type_idx;
+        }
+    };
+    using Records = std::vector<Rec>;
+
+public:
+    void swap(Self & other) noexcept
+    {
+        records.swap(other.records);
+    }
+
+    void clear()
+    {
+        records.clear();
+    }
+
+    bool empty() const
+    {
+        return records.empty();
+    }
+
+    size_t size() const
+    {
+        return records.size();
+    }
+
+    Self clone() const
+    {
+        Self result;
+        result.records.reserve(records.size());
+        for (const auto & rec : records)
+            result.records.emplace_back(rec.type_idx, rec.ptr->clone());
+        return result;
+    }
+
+    void append(Self && other)
+    {
+        auto middle_idx = records.size();
+        std::move(other.records.begin(), other.records.end(), std::back_inserter(records));
+        std::inplace_merge(records.begin(), records.begin() + middle_idx, records.end());
+        chassert(isUniqTypes());
+    }
+
+    template <class T>
+    void add(std::shared_ptr<T> info)
+    {
+        static_assert(std::is_base_of_v<ItemBase, T>, "Template parameter must inherit items base class");
+        return addImpl(std::type_index(typeid(T)), std::move(info));
+    }
+
+    template <class T>
+    std::shared_ptr<T> get() const
+    {
+        static_assert(std::is_base_of_v<ItemBase, T>, "Template parameter must inherit items base class");
+        auto it = getImpl(std::type_index(typeid(T)));
+        if (it == records.cend())
+            return nullptr;
+        auto cast = std::dynamic_pointer_cast<T>(it->ptr);
+        chassert(cast);
+        return cast;
+    }
+
+    template <class T>
+    std::shared_ptr<T> extract()
+    {
+        static_assert(std::is_base_of_v<ItemBase, T>, "Template parameter must inherit items base class");
+        auto it = getImpl(std::type_index(typeid(T)));
+        if (it == records.cend())
+            return nullptr;
+        auto cast = std::dynamic_pointer_cast<T>(it->ptr);
+        chassert(cast);
+
+        records.erase(it);
+        return cast;
+    }
+
+    std::string debug() const
+    {
+        std::string result;
+
+        for (auto & rec : records)
+        {
+            result.append(rec.type_idx.name());
+            result.append(" ");
+        }
+
+        return result;
+    }
+
+private:
+    bool isUniqTypes() const
+    {
+        auto uniq_it = std::adjacent_find(records.begin(), records.end());
+
+        return uniq_it == records.end();
+    }
+
+    void addImpl(std::type_index type_idx, ItemPtr item)
+    {
+        auto it = std::lower_bound(records.begin(), records.end(), type_idx);
+
+        if (it == records.end())
+        {
+            records.emplace_back(type_idx, item);
+            return;
+        }
+
+        if (it->type_idx == type_idx)
+            throw Exception(ErrorCodes::LOGICAL_ERROR, "inserted items must be unique by their type, type {} is inserted twice", type_idx.name());
+
+
+        records.emplace(it, type_idx, item);
+
+        chassert(isUniqTypes());
+    }
+
+    Records::const_iterator getImpl(std::type_index type_idx) const
+    {
+        auto it = std::lower_bound(records.cbegin(), records.cend(), type_idx);
+
+        if (it == records.cend())
+            return records.cend();
+
+        if (it->type_idx != type_idx)
+            return records.cend();
+
+        return it;
+    }
+
+    Records records;
+};
+
+}
diff --git a/src/Core/Settings.h b/src/Core/Settings.h
index 5903dbd32eb..81d0aa0c51d 100644
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@@ -36,7 +36,7 @@ class IColumn;
     M(Dialect, dialect, Dialect::clickhouse, "Which dialect will be used to parse query", 0)\
     M(UInt64, min_compress_block_size, 65536, "The actual size of the block to compress, if the uncompressed data less than max_compress_block_size is no less than this value and no less than the volume of data for one mark.", 0) \
     M(UInt64, max_compress_block_size, 1048576, "The maximum size of blocks of uncompressed data before compressing for writing to a table.", 0) \
-    M(UInt64, max_block_size, DEFAULT_BLOCK_SIZE, "Maximum block size for reading", 0) \
+    M(UInt64, max_block_size, DEFAULT_BLOCK_SIZE, "Maximum block size in rows for reading", 0) \
     M(UInt64, max_insert_block_size, DEFAULT_INSERT_BLOCK_SIZE, "The maximum block size for insertion, if we control the creation of blocks for insertion.", 0) \
     M(UInt64, min_insert_block_size_rows, DEFAULT_INSERT_BLOCK_SIZE, "Squash blocks passed to INSERT query to specified size in rows, if blocks are not big enough.", 0) \
     M(UInt64, min_insert_block_size_bytes, (DEFAULT_INSERT_BLOCK_SIZE * 256), "Squash blocks passed to INSERT query to specified size in bytes, if blocks are not big enough.", 0) \
@@ -634,9 +634,8 @@ class IColumn;
     M(Bool, optimize_time_filter_with_preimage, true, "Optimize Date and DateTime predicates by converting functions into equivalent comparisons without conversions (e.g. toYear(col) = 2023 -> col >= '2023-01-01' AND col <= '2023-12-31')", 0) \
     M(Bool, normalize_function_names, true, "Normalize function names to their canonical names", 0) \
     M(Bool, enable_early_constant_folding, true, "Enable query optimization where we analyze function and subqueries results and rewrite query if there are constants there", 0) \
-    M(Bool, deduplicate_blocks_in_dependent_materialized_views, false, "Should deduplicate blocks for materialized views if the block is not a duplicate for the table. Use true to always deduplicate in dependent tables.", 0) \
+    M(Bool, deduplicate_blocks_in_dependent_materialized_views, false, "Should deduplicate blocks for materialized views. Use true to always deduplicate in dependent tables.", 0) \
     M(Bool, throw_if_deduplication_in_dependent_materialized_views_enabled_with_async_insert, true, "Throw exception on INSERT query when the setting `deduplicate_blocks_in_dependent_materialized_views` is enabled along with `async_insert`. It guarantees correctness, because these features can't work together.", 0) \
-    M(Bool, update_insert_deduplication_token_in_dependent_materialized_views, false, "Should update insert deduplication token with table identifier during insert in dependent materialized views.", 0) \
     M(Bool, materialized_views_ignore_errors, false, "Allows to ignore errors for MATERIALIZED VIEW, and deliver original block to the table regardless of MVs", 0) \
     M(Bool, ignore_materialized_views_with_dropped_target_table, false, "Ignore MVs with dropped target table during pushing to views", 0) \
     M(Bool, allow_experimental_refreshable_materialized_view, false, "Allow refreshable materialized views (CREATE MATERIALIZED VIEW <name> REFRESH ...).", 0) \
@@ -953,6 +952,7 @@ class IColumn;
 
 #define OBSOLETE_SETTINGS(M, ALIAS) \
     /** Obsolete settings that do nothing but left for compatibility reasons. Remove each one after half a year of obsolescence. */ \
+    MAKE_OBSOLETE(M, Bool, update_insert_deduplication_token_in_dependent_materialized_views, 1) \
     MAKE_OBSOLETE(M, UInt64, max_memory_usage_for_all_queries, 0) \
     MAKE_OBSOLETE(M, UInt64, multiple_joins_rewriter_version, 0) \
     MAKE_OBSOLETE(M, Bool, enable_debug_queries, false) \
diff --git a/src/Interpreters/AsynchronousInsertQueue.cpp b/src/Interpreters/AsynchronousInsertQueue.cpp
index d72f3d81549..dd1166a9228 100644
--- a/src/Interpreters/AsynchronousInsertQueue.cpp
+++ b/src/Interpreters/AsynchronousInsertQueue.cpp
@@ -301,7 +301,13 @@ void AsynchronousInsertQueue::preprocessInsertQuery(const ASTPtr & query, const
     auto & insert_query = query->as<ASTInsertQuery &>();
     insert_query.async_insert_flush = true;
 
-    InterpreterInsertQuery interpreter(query, query_context, query_context->getSettingsRef().insert_allow_materialized_columns);
+    InterpreterInsertQuery interpreter(
+        query,
+        query_context,
+        query_context->getSettingsRef().insert_allow_materialized_columns,
+        /* no_squash */ false,
+        /* no_destination */ false,
+        /* async_insert */ false);
     auto table = interpreter.getTable(insert_query);
     auto sample_block = InterpreterInsertQuery::getSampleBlock(insert_query, table, table->getInMemoryMetadataPtr(), query_context);
 
@@ -781,7 +787,12 @@ try
     try
     {
         interpreter = std::make_unique<InterpreterInsertQuery>(
-            key.query, insert_context, key.settings.insert_allow_materialized_columns, false, false, true);
+            key.query,
+            insert_context,
+            key.settings.insert_allow_materialized_columns,
+            false,
+            false,
+            true);
 
         pipeline = interpreter->execute().pipeline;
         chassert(pipeline.pushing());
@@ -1000,7 +1011,7 @@ Chunk AsynchronousInsertQueue::processEntriesWithParsing(
     }
 
     Chunk chunk(executor.getResultColumns(), total_rows);
-    chunk.setChunkInfo(std::move(chunk_info));
+    chunk.getChunkInfos().add(std::move(chunk_info));
     return chunk;
 }
 
@@ -1052,7 +1063,7 @@ Chunk AsynchronousInsertQueue::processPreprocessedEntries(
     }
 
     Chunk chunk(std::move(result_columns), total_rows);
-    chunk.setChunkInfo(std::move(chunk_info));
+    chunk.getChunkInfos().add(std::move(chunk_info));
     return chunk;
 }
 
diff --git a/src/Interpreters/InterpreterCheckQuery.cpp b/src/Interpreters/InterpreterCheckQuery.cpp
index 4a84a7bf570..81bb6290acb 100644
--- a/src/Interpreters/InterpreterCheckQuery.cpp
+++ b/src/Interpreters/InterpreterCheckQuery.cpp
@@ -2,6 +2,7 @@
 #include <Interpreters/InterpreterFactory.h>
 
 #include <algorithm>
+#include <memory>
 
 #include <Access/Common/AccessFlags.h>
 
@@ -22,6 +23,7 @@
 #include <Parsers/ASTCheckQuery.h>
 #include <Parsers/ASTSetQuery.h>
 
+#include <Processors/Chunk.h>
 #include <Processors/IAccumulatingTransform.h>
 #include <Processors/IInflatingTransform.h>
 #include <Processors/ISimpleTransform.h>
@@ -91,7 +93,7 @@ Chunk getChunkFromCheckResult(const String & database, const String & table, con
     return Chunk(std::move(columns), 1);
 }
 
-class TableCheckTask : public ChunkInfo
+class TableCheckTask : public ChunkInfoCloneable<TableCheckTask>
 {
 public:
     TableCheckTask(StorageID table_id, const std::variant<std::monostate, ASTPtr, String> & partition_or_part, ContextPtr context)
@@ -110,6 +112,12 @@ public:
         context->checkAccess(AccessType::SHOW_TABLES, table_->getStorageID());
     }
 
+    TableCheckTask(const TableCheckTask & other)
+        : table(other.table)
+        , check_data_tasks(other.check_data_tasks)
+        , is_finished(other.is_finished.load())
+    {}
+
     std::optional<CheckResult> checkNext() const
     {
         if (isFinished())
@@ -121,8 +129,8 @@ public:
             std::this_thread::sleep_for(sleep_time);
         });
 
-        IStorage::DataValidationTasksPtr check_data_tasks_ = check_data_tasks;
-        auto result = table->checkDataNext(check_data_tasks_);
+        IStorage::DataValidationTasksPtr tmp = check_data_tasks;
+        auto result = table->checkDataNext(tmp);
         is_finished = !result.has_value();
         return result;
     }
@@ -180,7 +188,7 @@ protected:
         /// source should return at least one row to start pipeline
         result.addColumn(ColumnUInt8::create(1, 1));
         /// actual data stored in chunk info
-        result.setChunkInfo(std::move(current_check_task));
+        result.getChunkInfos().add(std::move(current_check_task));
         return result;
     }
 
@@ -280,7 +288,7 @@ public:
 protected:
     void transform(Chunk & chunk) override
     {
-        auto table_check_task = std::dynamic_pointer_cast<const TableCheckTask>(chunk.getChunkInfo());
+        auto table_check_task = chunk.getChunkInfos().get<TableCheckTask>();
         auto check_result = table_check_task->checkNext();
         if (!check_result)
         {
diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp
index 0ee2bb6c0e9..ee191c02ff8 100644
--- a/src/Interpreters/InterpreterCreateQuery.cpp
+++ b/src/Interpreters/InterpreterCreateQuery.cpp
@@ -1776,8 +1776,13 @@ BlockIO InterpreterCreateQuery::fillTableIfNeeded(const ASTCreateQuery & create)
         else
             insert->select = create.select->clone();
 
-        return InterpreterInsertQuery(insert, getContext(),
-            getContext()->getSettingsRef().insert_allow_materialized_columns).execute();
+        return InterpreterInsertQuery(
+            insert,
+            getContext(),
+            getContext()->getSettingsRef().insert_allow_materialized_columns,
+            /* no_squash */ false,
+            /* no_destination */ false,
+            /* async_isnert */ false).execute();
     }
 
     return {};
diff --git a/src/Interpreters/InterpreterExplainQuery.cpp b/src/Interpreters/InterpreterExplainQuery.cpp
index 7c7b4b3f95a..26b7e074fdf 100644
--- a/src/Interpreters/InterpreterExplainQuery.cpp
+++ b/src/Interpreters/InterpreterExplainQuery.cpp
@@ -534,7 +534,13 @@ QueryPipeline InterpreterExplainQuery::executeImpl()
             }
             else if (dynamic_cast<const ASTInsertQuery *>(ast.getExplainedQuery().get()))
             {
-                InterpreterInsertQuery insert(ast.getExplainedQuery(), getContext());
+                InterpreterInsertQuery insert(
+                    ast.getExplainedQuery(),
+                    getContext(),
+                    /* allow_materialized */ false,
+                    /* no_squash */ false,
+                    /* no_destination */ false,
+                    /* async_isnert */ false);
                 auto io = insert.execute();
                 printPipeline(io.pipeline.getProcessors(), buf);
             }
diff --git a/src/Interpreters/InterpreterInsertQuery.cpp b/src/Interpreters/InterpreterInsertQuery.cpp
index f396db70d21..2cbfc55d008 100644
--- a/src/Interpreters/InterpreterInsertQuery.cpp
+++ b/src/Interpreters/InterpreterInsertQuery.cpp
@@ -16,6 +16,7 @@
 #include <Interpreters/getTableExpressions.h>
 #include <Interpreters/processColumnTransformers.h>
 #include <Interpreters/InterpreterSelectQueryAnalyzer.h>
+#include <Interpreters/Context_fwd.h>
 #include <Parsers/ASTFunction.h>
 #include <Parsers/ASTInsertQuery.h>
 #include <Parsers/ASTSelectQuery.h>
@@ -26,6 +27,7 @@
 #include <Processors/Transforms/CountingTransform.h>
 #include <Processors/Transforms/ExpressionTransform.h>
 #include <Processors/Transforms/MaterializingTransform.h>
+#include <Processors/Transforms/DeduplicationTokenTransforms.h>
 #include <Processors/Transforms/SquashingTransform.h>
 #include <Processors/Transforms/PlanSquashingTransform.h>
 #include <Processors/Transforms/getSourceFromASTInsertQuery.h>
@@ -38,6 +40,7 @@
 #include <Common/ThreadStatus.h>
 #include <Common/checkStackSize.h>
 #include <Common/ProfileEvents.h>
+#include "base/defines.h"
 
 
 namespace ProfileEvents
@@ -394,28 +397,358 @@ Chain InterpreterInsertQuery::buildPreSinkChain(
     return out;
 }
 
+std::pair<std::vector<Chain>, std::vector<Chain>> InterpreterInsertQuery::buildPreAndSinkChains(size_t presink_streams, size_t sink_streams, StoragePtr table, const StorageMetadataPtr & metadata_snapshot, const Block & query_sample_block)
+{
+    chassert(presink_streams > 0);
+    chassert(sink_streams > 0);
+
+    ThreadGroupPtr running_group;
+    if (current_thread)
+        running_group = current_thread->getThreadGroup();
+    if (!running_group)
+        running_group = std::make_shared<ThreadGroup>(getContext());
+
+    std::vector<Chain> sink_chains;
+    std::vector<Chain> presink_chains;
+
+    for (size_t i = 0; i < sink_streams; ++i)
+    {
+        auto out = buildSink(table, metadata_snapshot, /* thread_status_holder= */ nullptr,
+            running_group, /* elapsed_counter_ms= */ nullptr);
+
+        sink_chains.emplace_back(std::move(out));
+    }
+
+    for (size_t i = 0; i < presink_streams; ++i)
+    {
+        auto out = buildPreSinkChain(sink_chains[0].getInputHeader(), table, metadata_snapshot, query_sample_block);
+        presink_chains.emplace_back(std::move(out));
+    }
+
+    return {std::move(presink_chains), std::move(sink_chains)};
+}
+
+
+QueryPipeline InterpreterInsertQuery::buildInsertSelectPipeline(ASTInsertQuery & query, StoragePtr table)
+{
+    const Settings & settings = getContext()->getSettingsRef();
+
+    auto metadata_snapshot = table->getInMemoryMetadataPtr();
+    auto query_sample_block = getSampleBlock(query, table, metadata_snapshot, getContext(), no_destination, allow_materialized);
+
+    bool is_trivial_insert_select = false;
+
+    if (settings.optimize_trivial_insert_select)
+    {
+        const auto & select_query = query.select->as<ASTSelectWithUnionQuery &>();
+        const auto & selects = select_query.list_of_selects->children;
+        const auto & union_modes = select_query.list_of_modes;
+
+        /// ASTSelectWithUnionQuery is not normalized now, so it may pass some queries which can be Trivial select queries
+        const auto mode_is_all = [](const auto & mode) { return mode == SelectUnionMode::UNION_ALL; };
+
+        is_trivial_insert_select =
+            std::all_of(union_modes.begin(), union_modes.end(), std::move(mode_is_all))
+            && std::all_of(selects.begin(), selects.end(), isTrivialSelect);
+    }
+
+    ContextPtr select_context = getContext();
+
+    if (is_trivial_insert_select)
+    {
+        /** When doing trivial INSERT INTO ... SELECT ... FROM table,
+            * don't need to process SELECT with more than max_insert_threads
+            * and it's reasonable to set block size for SELECT to the desired block size for INSERT
+            * to avoid unnecessary squashing.
+            */
+
+        Settings new_settings = select_context->getSettings();
+
+        new_settings.max_threads = std::max<UInt64>(1, settings.max_insert_threads);
+
+        if (table->prefersLargeBlocks())
+        {
+            if (settings.min_insert_block_size_rows)
+                new_settings.max_block_size = settings.min_insert_block_size_rows;
+            if (settings.min_insert_block_size_bytes)
+                new_settings.preferred_block_size_bytes = settings.min_insert_block_size_bytes;
+        }
+
+        auto context_for_trivial_select = Context::createCopy(context);
+        context_for_trivial_select->setSettings(new_settings);
+        context_for_trivial_select->setInsertionTable(getContext()->getInsertionTable(), getContext()->getInsertionTableColumnNames());
+
+        select_context = context_for_trivial_select;
+    }
+
+    QueryPipelineBuilder pipeline;
+
+    {
+        auto select_query_options = SelectQueryOptions(QueryProcessingStage::Complete, 1);
+
+        if (settings.allow_experimental_analyzer)
+        {
+            InterpreterSelectQueryAnalyzer interpreter_select_analyzer(query.select, select_context, select_query_options);
+            pipeline = interpreter_select_analyzer.buildQueryPipeline();
+        }
+        else
+        {
+            InterpreterSelectWithUnionQuery interpreter_select(query.select, select_context, select_query_options);
+            pipeline = interpreter_select.buildQueryPipeline();
+        }
+    }
+
+    pipeline.dropTotalsAndExtremes();
+
+    /// Allow to insert Nullable into non-Nullable columns, NULL values will be added as defaults values.
+    if (getContext()->getSettingsRef().insert_null_as_default)
+    {
+        const auto & input_columns = pipeline.getHeader().getColumnsWithTypeAndName();
+        const auto & query_columns = query_sample_block.getColumnsWithTypeAndName();
+        const auto & output_columns = metadata_snapshot->getColumns();
+
+        if (input_columns.size() == query_columns.size())
+        {
+            for (size_t col_idx = 0; col_idx < query_columns.size(); ++col_idx)
+            {
+                /// Change query sample block columns to Nullable to allow inserting nullable columns, where NULL values will be substituted with
+                /// default column values (in AddingDefaultsTransform), so all values will be cast correctly.
+                if (isNullableOrLowCardinalityNullable(input_columns[col_idx].type)
+                    && !isNullableOrLowCardinalityNullable(query_columns[col_idx].type)
+                    && !isVariant(query_columns[col_idx].type)
+                    && !isDynamic(query_columns[col_idx].type)
+                    && output_columns.has(query_columns[col_idx].name))
+                {
+                    query_sample_block.setColumn(
+                        col_idx,
+                        ColumnWithTypeAndName(
+                            makeNullableOrLowCardinalityNullable(query_columns[col_idx].column),
+                            makeNullableOrLowCardinalityNullable(query_columns[col_idx].type),
+                            query_columns[col_idx].name));
+                }
+            }
+        }
+    }
+
+    auto actions_dag = ActionsDAG::makeConvertingActions(
+            pipeline.getHeader().getColumnsWithTypeAndName(),
+            query_sample_block.getColumnsWithTypeAndName(),
+            ActionsDAG::MatchColumnsMode::Position);
+    auto actions = std::make_shared<ExpressionActions>(actions_dag, ExpressionActionsSettings::fromContext(getContext(), CompileExpressions::yes));
+
+    pipeline.addSimpleTransform([&](const Block & in_header) -> ProcessorPtr
+    {
+        return std::make_shared<ExpressionTransform>(in_header, actions);
+    });
+
+    /// We need to convert Sparse columns to full, because it's destination storage
+    /// may not support it or may have different settings for applying Sparse serialization.
+    pipeline.addSimpleTransform([&](const Block & in_header) -> ProcessorPtr
+    {
+        return std::make_shared<MaterializingTransform>(in_header);
+    });
+
+    pipeline.addSimpleTransform([&](const Block & in_header) -> ProcessorPtr
+    {
+        auto context_ptr = getContext();
+        auto counting = std::make_shared<CountingTransform>(in_header, nullptr, context_ptr->getQuota());
+        counting->setProcessListElement(context_ptr->getProcessListElement());
+        counting->setProgressCallback(context_ptr->getProgressCallback());
+
+        return counting;
+    });
+
+    size_t num_select_threads = pipeline.getNumThreads();
+
+    pipeline.resize(1);
+
+    if (shouldAddSquashingFroStorage(table))
+    {
+        pipeline.addSimpleTransform([&](const Block & in_header) -> ProcessorPtr
+        {
+            return std::make_shared<PlanSquashingTransform>(
+                in_header,
+                table->prefersLargeBlocks() ? settings.min_insert_block_size_rows : settings.max_block_size,
+                table->prefersLargeBlocks() ? settings.min_insert_block_size_bytes : 0ULL);
+        });
+    }
+
+    pipeline.addSimpleTransform([&](const Block &in_header) -> ProcessorPtr
+    {
+        return std::make_shared<DeduplicationToken::AddTokenInfoTransform>(in_header);
+    });
+
+    if (!settings.insert_deduplication_token.value.empty())
+    {
+        pipeline.addSimpleTransform([&](const Block & in_header) -> ProcessorPtr
+        {
+            return std::make_shared<DeduplicationToken::SetUserTokenTransform>(settings.insert_deduplication_token.value, in_header);
+        });
+
+        pipeline.addSimpleTransform([&](const Block & in_header) -> ProcessorPtr
+        {
+            return std::make_shared<DeduplicationToken::SetSourceBlockNumberTransform>(in_header);
+        });
+    }
+
+    /// Number of streams works like this:
+    ///  * For the SELECT, use `max_threads`, or `max_insert_threads`, or whatever
+    ///    InterpreterSelectQuery ends up with.
+    ///  * Use `max_insert_threads` streams for various insert-preparation steps, e.g.
+    ///    materializing and squashing (too slow to do in one thread). That's `presink_chains`.
+    ///  * If the table supports parallel inserts, use max_insert_threads for writing to IStorage.
+    ///    Otherwise ResizeProcessor them down to 1 stream.
+
+    size_t presink_streams_size = std::max<size_t>(settings.max_insert_threads, pipeline.getNumStreams());
+
+    size_t sink_streams_size = table->supportsParallelInsert() ? std::max<size_t>(1, settings.max_insert_threads) : 1;
+
+    if (!settings.parallel_view_processing)
+    {
+        auto table_id = table->getStorageID();
+        auto views = DatabaseCatalog::instance().getDependentViews(table_id);
+
+        if (table->isView() || !views.empty())
+            sink_streams_size = 1;
+    }
+
+    auto [presink_chains, sink_chains] = buildPreAndSinkChains(
+        presink_streams_size, sink_streams_size,
+        table, metadata_snapshot, query_sample_block);
+
+    pipeline.resize(presink_chains.size());
+
+    if (shouldAddSquashingFroStorage(table))
+    {
+        pipeline.addSimpleTransform([&](const Block & in_header) -> ProcessorPtr
+        {
+            return std::make_shared<ApplySquashingTransform>(
+                in_header,
+                table->prefersLargeBlocks() ? settings.min_insert_block_size_rows : settings.max_block_size,
+                table->prefersLargeBlocks() ? settings.min_insert_block_size_bytes : 0ULL);
+        });
+    }
+
+    for (auto & chain : presink_chains)
+        pipeline.addResources(chain.detachResources());
+    pipeline.addChains(std::move(presink_chains));
+
+    pipeline.resize(sink_streams_size);
+
+    for (auto & chain : sink_chains)
+        pipeline.addResources(chain.detachResources());
+    pipeline.addChains(std::move(sink_chains));
+
+    if (!settings.parallel_view_processing)
+    {
+        /// Don't use more threads for INSERT than for SELECT to reduce memory consumption.
+        if (pipeline.getNumThreads() > num_select_threads)
+            pipeline.setMaxThreads(num_select_threads);
+    }
+    else if (pipeline.getNumThreads() < settings.max_threads)
+    {
+        /// It is possible for query to have max_threads=1, due to optimize_trivial_insert_select,
+        /// however in case of parallel_view_processing and multiple views, views can still be processed in parallel.
+        ///
+        /// Note, number of threads will be limited by buildPushingToViewsChain() to max_threads.
+        pipeline.setMaxThreads(settings.max_threads);
+    }
+
+    pipeline.setSinks([&](const Block & cur_header, QueryPipelineBuilder::StreamType) -> ProcessorPtr
+    {
+        return std::make_shared<EmptySink>(cur_header);
+    });
+
+    return QueryPipelineBuilder::getPipeline(std::move(pipeline));
+}
+
+
+QueryPipeline InterpreterInsertQuery::buildInsertPipeline(ASTInsertQuery & query, StoragePtr table)
+{
+    const Settings & settings = getContext()->getSettingsRef();
+
+    auto metadata_snapshot = table->getInMemoryMetadataPtr();
+    auto query_sample_block = getSampleBlock(query, table, metadata_snapshot, getContext(), no_destination, allow_materialized);
+
+    Chain chain;
+
+    {
+        auto [presink_chains, sink_chains] = buildPreAndSinkChains(
+            /* presink_streams */1, /* sink_streams */1,
+            table, metadata_snapshot, query_sample_block);
+
+        chain = std::move(presink_chains.front());
+        chain.appendChain(std::move(sink_chains.front()));
+    }
+
+    if (!settings.insert_deduplication_token.value.empty())
+    {
+        chain.addSource(std::make_shared<DeduplicationToken::SetSourceBlockNumberTransform>(chain.getInputHeader()));
+        chain.addSource(std::make_shared<DeduplicationToken::SetUserTokenTransform>(settings.insert_deduplication_token.value, chain.getInputHeader()));
+    }
+
+    chain.addSource(std::make_shared<DeduplicationToken::AddTokenInfoTransform>(chain.getInputHeader()));
+
+    if (shouldAddSquashingFroStorage(table))
+    {
+        bool table_prefers_large_blocks = table->prefersLargeBlocks();
+
+        auto squashing = std::make_shared<ApplySquashingTransform>(
+                    chain.getInputHeader(),
+                    table_prefers_large_blocks ? settings.min_insert_block_size_rows : settings.max_block_size,
+                    table_prefers_large_blocks ? settings.min_insert_block_size_bytes : 0ULL);
+
+        chain.addSource(std::move(squashing));
+
+        auto balancing = std::make_shared<PlanSquashingTransform>(
+                    chain.getInputHeader(),
+                    table_prefers_large_blocks ? settings.min_insert_block_size_rows : settings.max_block_size,
+                    table_prefers_large_blocks ? settings.min_insert_block_size_bytes : 0ULL);
+
+        chain.addSource(std::move(balancing));
+    }
+
+    auto context_ptr = getContext();
+    auto counting = std::make_shared<CountingTransform>(chain.getInputHeader(), nullptr, context_ptr->getQuota());
+    counting->setProcessListElement(context_ptr->getProcessListElement());
+    counting->setProgressCallback(context_ptr->getProgressCallback());
+    chain.addSource(std::move(counting));
+
+    QueryPipeline pipeline = QueryPipeline(std::move(chain));
+
+    pipeline.setNumThreads(std::min<size_t>(pipeline.getNumThreads(), settings.max_threads));
+    pipeline.setConcurrencyControl(settings.use_concurrency_control);
+
+    if (query.hasInlinedData() && !async_insert)
+    {
+        /// can execute without additional data
+        auto format = getInputFormatFromASTInsertQuery(query_ptr, true, query_sample_block, getContext(), nullptr);
+        for (auto && buffer : owned_buffers)
+            format->addBuffer(std::move(buffer));
+
+        auto pipe = getSourceFromInputFormat(query_ptr, std::move(format), getContext(), nullptr);
+        pipeline.complete(std::move(pipe));
+    }
+
+    return pipeline;
+}
+
+
 BlockIO InterpreterInsertQuery::execute()
 {
     const Settings & settings = getContext()->getSettingsRef();
     auto & query = query_ptr->as<ASTInsertQuery &>();
 
-    QueryPipelineBuilder pipeline;
-    std::optional<QueryPipeline> distributed_pipeline;
-    QueryPlanResourceHolder resources;
 
     StoragePtr table = getTable(query);
     checkStorageSupportsTransactionsIfNeeded(table, getContext());
 
-    StoragePtr inner_table;
-    if (const auto * mv = dynamic_cast<const StorageMaterializedView *>(table.get()))
-        inner_table = mv->getTargetTable();
-
     if (query.partition_by && !table->supportsPartitionBy())
         throw Exception(ErrorCodes::NOT_IMPLEMENTED, "PARTITION BY clause is not supported by storage");
 
     auto table_lock = table->lockForShare(getContext()->getInitialQueryId(), settings.lock_acquire_timeout);
-    auto metadata_snapshot = table->getInMemoryMetadataPtr();
 
+    auto metadata_snapshot = table->getInMemoryMetadataPtr();
     auto query_sample_block = getSampleBlock(query, table, metadata_snapshot, getContext(), no_destination, allow_materialized);
 
     /// For table functions we check access while executing
@@ -423,320 +756,43 @@ BlockIO InterpreterInsertQuery::execute()
     if (!query.table_function)
         getContext()->checkAccess(AccessType::INSERT, query.table_id, query_sample_block.getNames());
 
-    if (query.select && settings.parallel_distributed_insert_select)
-        // Distributed INSERT SELECT
-        distributed_pipeline = table->distributedWrite(query, getContext());
-
-    std::vector<Chain> presink_chains;
-    std::vector<Chain> sink_chains;
-    if (!distributed_pipeline)
+    if (!allow_materialized)
     {
-        /// Number of streams works like this:
-        ///  * For the SELECT, use `max_threads`, or `max_insert_threads`, or whatever
-        ///    InterpreterSelectQuery ends up with.
-        ///  * Use `max_insert_threads` streams for various insert-preparation steps, e.g.
-        ///    materializing and squashing (too slow to do in one thread). That's `presink_chains`.
-        ///  * If the table supports parallel inserts, use the same streams for writing to IStorage.
-        ///    Otherwise ResizeProcessor them down to 1 stream.
-        ///  * If it's not an INSERT SELECT, forget all that and use one stream.
-        size_t pre_streams_size = 1;
-        size_t sink_streams_size = 1;
-
-        if (query.select)
-        {
-            bool is_trivial_insert_select = false;
-
-            if (settings.optimize_trivial_insert_select)
-            {
-                const auto & select_query = query.select->as<ASTSelectWithUnionQuery &>();
-                const auto & selects = select_query.list_of_selects->children;
-                const auto & union_modes = select_query.list_of_modes;
-
-                /// ASTSelectWithUnionQuery is not normalized now, so it may pass some queries which can be Trivial select queries
-                const auto mode_is_all = [](const auto & mode) { return mode == SelectUnionMode::UNION_ALL; };
-
-                is_trivial_insert_select =
-                    std::all_of(union_modes.begin(), union_modes.end(), std::move(mode_is_all))
-                    && std::all_of(selects.begin(), selects.end(), isTrivialSelect);
-            }
-
-            if (is_trivial_insert_select)
-            {
-                /** When doing trivial INSERT INTO ... SELECT ... FROM table,
-                  * don't need to process SELECT with more than max_insert_threads
-                  * and it's reasonable to set block size for SELECT to the desired block size for INSERT
-                  * to avoid unnecessary squashing.
-                  */
-
-                Settings new_settings = getContext()->getSettings();
-
-                new_settings.max_threads = std::max<UInt64>(1, settings.max_insert_threads);
-
-                if (table->prefersLargeBlocks())
-                {
-                    if (settings.min_insert_block_size_rows)
-                        new_settings.max_block_size = settings.min_insert_block_size_rows;
-                    if (settings.min_insert_block_size_bytes)
-                        new_settings.preferred_block_size_bytes = settings.min_insert_block_size_bytes;
-                }
-
-                auto new_context = Context::createCopy(context);
-                new_context->setSettings(new_settings);
-                new_context->setInsertionTable(getContext()->getInsertionTable(), getContext()->getInsertionTableColumnNames());
-
-                auto select_query_options = SelectQueryOptions(QueryProcessingStage::Complete, 1);
-
-                if (settings.allow_experimental_analyzer)
-                {
-                    InterpreterSelectQueryAnalyzer interpreter_select_analyzer(query.select, new_context, select_query_options);
-                    pipeline = interpreter_select_analyzer.buildQueryPipeline();
-                }
-                else
-                {
-                    InterpreterSelectWithUnionQuery interpreter_select(query.select, new_context, select_query_options);
-                    pipeline = interpreter_select.buildQueryPipeline();
-                }
-            }
-            else
-            {
-                /// Passing 1 as subquery_depth will disable limiting size of intermediate result.
-                auto select_query_options = SelectQueryOptions(QueryProcessingStage::Complete, 1);
-
-                if (settings.allow_experimental_analyzer)
-                {
-                    InterpreterSelectQueryAnalyzer interpreter_select_analyzer(query.select, getContext(), select_query_options);
-                    pipeline = interpreter_select_analyzer.buildQueryPipeline();
-                }
-                else
-                {
-                    InterpreterSelectWithUnionQuery interpreter_select(query.select, getContext(), select_query_options);
-                    pipeline = interpreter_select.buildQueryPipeline();
-                }
-            }
-
-            pipeline.dropTotalsAndExtremes();
-
-            if (settings.max_insert_threads > 1)
-            {
-                auto table_id = table->getStorageID();
-                auto views = DatabaseCatalog::instance().getDependentViews(table_id);
-
-                /// It breaks some views-related tests and we have dedicated `parallel_view_processing` for views, so let's just skip them.
-                /// Also it doesn't make sense to reshuffle data if storage doesn't support parallel inserts.
-                const bool resize_to_max_insert_threads = !table->isView() && views.empty() && table->supportsParallelInsert();
-                pre_streams_size = resize_to_max_insert_threads ? settings.max_insert_threads
-                                                                : std::min<size_t>(settings.max_insert_threads, pipeline.getNumStreams());
-
-                /// Deduplication when passing insert_deduplication_token breaks if using more than one thread
-                if (!settings.insert_deduplication_token.toString().empty())
-                {
-                    LOG_DEBUG(
-                        getLogger("InsertQuery"),
-                        "Insert-select query using insert_deduplication_token, setting streams to 1 to avoid deduplication issues");
-                    pre_streams_size = 1;
-                }
-
-                if (table->supportsParallelInsert())
-                    sink_streams_size = pre_streams_size;
-            }
-
-            pipeline.resize(pre_streams_size);
-
-            /// Allow to insert Nullable into non-Nullable columns, NULL values will be added as defaults values.
-            if (getContext()->getSettingsRef().insert_null_as_default)
-            {
-                const auto & input_columns = pipeline.getHeader().getColumnsWithTypeAndName();
-                const auto & query_columns = query_sample_block.getColumnsWithTypeAndName();
-                const auto & output_columns = metadata_snapshot->getColumns();
-
-                if (input_columns.size() == query_columns.size())
-                {
-                    for (size_t col_idx = 0; col_idx < query_columns.size(); ++col_idx)
-                    {
-                        /// Change query sample block columns to Nullable to allow inserting nullable columns, where NULL values will be substituted with
-                        /// default column values (in AddingDefaultsTransform), so all values will be cast correctly.
-                        if (isNullableOrLowCardinalityNullable(input_columns[col_idx].type)
-                            && !isNullableOrLowCardinalityNullable(query_columns[col_idx].type)
-                            && !isVariant(query_columns[col_idx].type)
-                            && !isDynamic(query_columns[col_idx].type)
-                            && output_columns.has(query_columns[col_idx].name))
-                            query_sample_block.setColumn(col_idx, ColumnWithTypeAndName(makeNullableOrLowCardinalityNullable(query_columns[col_idx].column), makeNullableOrLowCardinalityNullable(query_columns[col_idx].type), query_columns[col_idx].name));
-                    }
-                }
-            }
-        }
-
-        ThreadGroupPtr running_group;
-        if (current_thread)
-            running_group = current_thread->getThreadGroup();
-        if (!running_group)
-            running_group = std::make_shared<ThreadGroup>(getContext());
-        for (size_t i = 0; i < sink_streams_size; ++i)
-        {
-            auto out = buildSink(table, metadata_snapshot, /* thread_status_holder= */ nullptr,
-                running_group, /* elapsed_counter_ms= */ nullptr);
-            sink_chains.emplace_back(std::move(out));
-        }
-        for (size_t i = 0; i < pre_streams_size; ++i)
-        {
-            auto out = buildPreSinkChain(sink_chains[0].getInputHeader(), table, metadata_snapshot, query_sample_block);
-            presink_chains.emplace_back(std::move(out));
-        }
+        for (const auto & column : metadata_snapshot->getColumns())
+            if (column.default_desc.kind == ColumnDefaultKind::Materialized && query_sample_block.has(column.name))
+                throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Cannot insert column {}, because it is MATERIALIZED column.", column.name);
     }
 
     BlockIO res;
 
-    /// What type of query: INSERT or INSERT SELECT or INSERT WATCH?
-    if (distributed_pipeline)
+    if (query.select)
     {
-        res.pipeline = std::move(*distributed_pipeline);
-    }
-    else if (query.select)
-    {
-        const auto & header = presink_chains.at(0).getInputHeader();
-        auto actions_dag = ActionsDAG::makeConvertingActions(
-                pipeline.getHeader().getColumnsWithTypeAndName(),
-                header.getColumnsWithTypeAndName(),
-                ActionsDAG::MatchColumnsMode::Position);
-        auto actions = std::make_shared<ExpressionActions>(actions_dag, ExpressionActionsSettings::fromContext(getContext(), CompileExpressions::yes));
-
-        pipeline.addSimpleTransform([&](const Block & in_header) -> ProcessorPtr
+        if (settings.parallel_distributed_insert_select)
         {
-            return std::make_shared<ExpressionTransform>(in_header, actions);
-        });
-
-        /// We need to convert Sparse columns to full, because it's destination storage
-        /// may not support it or may have different settings for applying Sparse serialization.
-        pipeline.addSimpleTransform([&](const Block & in_header) -> ProcessorPtr
-        {
-            return std::make_shared<MaterializingTransform>(in_header);
-        });
-
-        pipeline.addSimpleTransform([&](const Block & in_header) -> ProcessorPtr
-        {
-            auto context_ptr = getContext();
-            auto counting = std::make_shared<CountingTransform>(in_header, nullptr, context_ptr->getQuota());
-            counting->setProcessListElement(context_ptr->getProcessListElement());
-            counting->setProgressCallback(context_ptr->getProgressCallback());
-
-            return counting;
-        });
-
-        if (shouldAddSquashingFroStorage(table))
-        {
-            bool table_prefers_large_blocks = table->prefersLargeBlocks();
-
-            size_t threads = presink_chains.size();
-
-            pipeline.resize(1);
-
-            pipeline.addTransform(std::make_shared<PlanSquashingTransform>(
-                    header,
-                    table_prefers_large_blocks ? settings.min_insert_block_size_rows : settings.max_block_size,
-                    table_prefers_large_blocks ? settings.min_insert_block_size_bytes : 0ULL));
-
-            pipeline.resize(threads);
-
-            pipeline.addSimpleTransform([&](const Block & in_header) -> ProcessorPtr
+            auto distributed = table->distributedWrite(query, getContext());
+            if (distributed)
             {
-                return std::make_shared<ApplySquashingTransform>(
-                    in_header,
-                    table_prefers_large_blocks ? settings.min_insert_block_size_rows : settings.max_block_size,
-                    table_prefers_large_blocks ? settings.min_insert_block_size_bytes : 0ULL);
-            });
+                res.pipeline = std::move(*distributed);
+            }
+            else
+            {
+                res.pipeline = buildInsertSelectPipeline(query, table);
+            }
         }
-
-        size_t num_select_threads = pipeline.getNumThreads();
-
-        for (auto & chain : presink_chains)
-            resources = chain.detachResources();
-        for (auto & chain : sink_chains)
-            resources = chain.detachResources();
-
-        pipeline.addChains(std::move(presink_chains));
-        pipeline.resize(sink_chains.size());
-        pipeline.addChains(std::move(sink_chains));
-
-        if (!settings.parallel_view_processing)
+        else
         {
-            /// Don't use more threads for INSERT than for SELECT to reduce memory consumption.
-            if (pipeline.getNumThreads() > num_select_threads)
-                pipeline.setMaxThreads(num_select_threads);
+            res.pipeline = buildInsertSelectPipeline(query, table);
         }
-        else if (pipeline.getNumThreads() < settings.max_threads)
-        {
-            /// It is possible for query to have max_threads=1, due to optimize_trivial_insert_select,
-            /// however in case of parallel_view_processing and multiple views, views can still be processed in parallel.
-            ///
-            /// Note, number of threads will be limited by buildPushingToViewsChain() to max_threads.
-            pipeline.setMaxThreads(settings.max_threads);
-        }
-
-        pipeline.setSinks([&](const Block & cur_header, QueryPipelineBuilder::StreamType) -> ProcessorPtr
-        {
-            return std::make_shared<EmptySink>(cur_header);
-        });
-
-        if (!allow_materialized)
-        {
-            for (const auto & column : metadata_snapshot->getColumns())
-                if (column.default_desc.kind == ColumnDefaultKind::Materialized && header.has(column.name))
-                    throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Cannot insert column {}, because it is MATERIALIZED column.", column.name);
-        }
-
-        res.pipeline = QueryPipelineBuilder::getPipeline(std::move(pipeline));
     }
     else
     {
-        auto & chain = presink_chains.at(0);
-        chain.appendChain(std::move(sink_chains.at(0)));
-
-        if (shouldAddSquashingFroStorage(table))
-        {
-            bool table_prefers_large_blocks = table->prefersLargeBlocks();
-
-            auto squashing = std::make_shared<ApplySquashingTransform>(
-                    chain.getInputHeader(),
-                    table_prefers_large_blocks ? settings.min_insert_block_size_rows : settings.max_block_size,
-                    table_prefers_large_blocks ? settings.min_insert_block_size_bytes : 0ULL);
-
-            chain.addSource(std::move(squashing));
-
-            auto balancing = std::make_shared<PlanSquashingTransform>(
-                    chain.getInputHeader(),
-                    table_prefers_large_blocks ? settings.min_insert_block_size_rows : settings.max_block_size,
-                    table_prefers_large_blocks ? settings.min_insert_block_size_bytes : 0ULL);
-
-            chain.addSource(std::move(balancing));
-        }
-
-        auto context_ptr = getContext();
-        auto counting = std::make_shared<CountingTransform>(chain.getInputHeader(), nullptr, context_ptr->getQuota());
-        counting->setProcessListElement(context_ptr->getProcessListElement());
-        counting->setProgressCallback(context_ptr->getProgressCallback());
-        chain.addSource(std::move(counting));
-
-        res.pipeline = QueryPipeline(std::move(presink_chains[0]));
-        res.pipeline.setNumThreads(std::min<size_t>(res.pipeline.getNumThreads(), settings.max_threads));
-        res.pipeline.setConcurrencyControl(settings.use_concurrency_control);
-
-        if (query.hasInlinedData() && !async_insert)
-        {
-            /// can execute without additional data
-            auto format = getInputFormatFromASTInsertQuery(query_ptr, true, query_sample_block, getContext(), nullptr);
-            for (auto && buffer : owned_buffers)
-                format->addBuffer(std::move(buffer));
-
-            auto pipe = getSourceFromInputFormat(query_ptr, std::move(format), getContext(), nullptr);
-            res.pipeline.complete(std::move(pipe));
-        }
+        res.pipeline = buildInsertPipeline(query, table);
     }
 
-    res.pipeline.addResources(std::move(resources));
-
     res.pipeline.addStorageHolder(table);
-    if (inner_table)
-        res.pipeline.addStorageHolder(inner_table);
+
+    if (const auto * mv = dynamic_cast<const StorageMaterializedView *>(table.get()))
+        res.pipeline.addStorageHolder(mv->getTargetTable());
 
     return res;
 }
@@ -757,17 +813,27 @@ void InterpreterInsertQuery::extendQueryLogElemImpl(QueryLogElement & elem, Cont
     }
 }
 
+
 void InterpreterInsertQuery::extendQueryLogElemImpl(QueryLogElement & elem, const ASTPtr &, ContextPtr context_) const
 {
     extendQueryLogElemImpl(elem, context_);
 }
 
+
 void registerInterpreterInsertQuery(InterpreterFactory & factory)
 {
     auto create_fn = [] (const InterpreterFactory::Arguments & args)
     {
-        return std::make_unique<InterpreterInsertQuery>(args.query, args.context, args.allow_materialized);
+        return std::make_unique<InterpreterInsertQuery>(
+            args.query,
+            args.context,
+            args.allow_materialized,
+            /* no_squash */false,
+            /* no_destination */false,
+            /* async_insert */false);
     };
     factory.registerInterpreter("InterpreterInsertQuery", create_fn);
 }
+
+
 }
diff --git a/src/Interpreters/InterpreterInsertQuery.h b/src/Interpreters/InterpreterInsertQuery.h
index bf73fb2a319..894c7c42144 100644
--- a/src/Interpreters/InterpreterInsertQuery.h
+++ b/src/Interpreters/InterpreterInsertQuery.h
@@ -23,10 +23,10 @@ public:
     InterpreterInsertQuery(
         const ASTPtr & query_ptr_,
         ContextPtr context_,
-        bool allow_materialized_ = false,
-        bool no_squash_ = false,
-        bool no_destination_ = false,
-        bool async_insert_ = false);
+        bool allow_materialized_,
+        bool no_squash_,
+        bool no_destination,
+        bool async_insert_);
 
     /** Prepare a request for execution. Return block streams
       * - the stream into which you can write data to execute the query, if INSERT;
@@ -73,12 +73,17 @@ private:
 
     ASTPtr query_ptr;
     const bool allow_materialized;
-    const bool no_squash;
-    const bool no_destination;
+    bool no_squash = false;
+    bool no_destination = false;
     const bool async_insert;
 
     std::vector<std::unique_ptr<ReadBuffer>> owned_buffers;
 
+    std::pair<std::vector<Chain>, std::vector<Chain>> buildPreAndSinkChains(size_t presink_streams, size_t sink_streams, StoragePtr table, const StorageMetadataPtr & metadata_snapshot, const Block & query_sample_block);
+
+    QueryPipeline buildInsertSelectPipeline(ASTInsertQuery & query, StoragePtr table);
+    QueryPipeline buildInsertPipeline(ASTInsertQuery & query, StoragePtr table);
+
     Chain buildSink(
         const StoragePtr & table,
         const StorageMetadataPtr & metadata_snapshot,
diff --git a/src/Interpreters/Squashing.cpp b/src/Interpreters/Squashing.cpp
index f8b6a6542cc..25434d1103e 100644
--- a/src/Interpreters/Squashing.cpp
+++ b/src/Interpreters/Squashing.cpp
@@ -1,6 +1,7 @@
 #include <vector>
 #include <Interpreters/Squashing.h>
 #include <Common/CurrentThread.h>
+#include <base/defines.h>
 
 
 namespace DB
@@ -11,24 +12,33 @@ namespace ErrorCodes
 }
 
 Squashing::Squashing(Block header_, size_t min_block_size_rows_, size_t min_block_size_bytes_)
-    : header(header_)
-    , min_block_size_rows(min_block_size_rows_)
+    : min_block_size_rows(min_block_size_rows_)
     , min_block_size_bytes(min_block_size_bytes_)
+    , header(header_)
 {
 }
 
 Chunk Squashing::flush()
 {
-    return convertToChunk(std::move(chunks_to_merge_vec));
+    if (!accumulated)
+        return {};
+
+    auto result = convertToChunk(accumulated.extract());
+    chassert(result);
+    return result;
 }
 
 Chunk Squashing::squash(Chunk && input_chunk)
 {
-    if (!input_chunk.hasChunkInfo())
+    if (!input_chunk)
         return Chunk();
 
-    const auto *info = getInfoFromChunk(input_chunk);
-    return squash(info->chunks);
+    auto squash_info = input_chunk.getChunkInfos().extract<ChunksToSquash>();
+
+    if (!squash_info)
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "There is no ChunksToSquash in ChunkInfoPtr");
+
+    return squash(std::move(squash_info->chunks), std::move(input_chunk.getChunkInfos()));
 }
 
 Chunk Squashing::add(Chunk && input_chunk)
@@ -37,48 +47,37 @@ Chunk Squashing::add(Chunk && input_chunk)
         return {};
 
     /// Just read block is already enough.
-    if (isEnoughSize(input_chunk.getNumRows(), input_chunk.bytes()))
+    if (isEnoughSize(input_chunk))
     {
         /// If no accumulated data, return just read block.
-        if (chunks_to_merge_vec.empty())
+        if (!accumulated)
         {
-            chunks_to_merge_vec.push_back(std::move(input_chunk));
-            Chunk res_chunk = convertToChunk(std::move(chunks_to_merge_vec));
-            chunks_to_merge_vec.clear();
-            return res_chunk;
+            accumulated.add(std::move(input_chunk));
+            return convertToChunk(accumulated.extract());
         }
 
         /// Return accumulated data (maybe it has small size) and place new block to accumulated data.
-        Chunk res_chunk = convertToChunk(std::move(chunks_to_merge_vec));
-        chunks_to_merge_vec.clear();
-        changeCurrentSize(input_chunk.getNumRows(), input_chunk.bytes());
-        chunks_to_merge_vec.push_back(std::move(input_chunk));
+        Chunk res_chunk = convertToChunk(accumulated.extract());
+        accumulated.add(std::move(input_chunk));
         return res_chunk;
     }
 
     /// Accumulated block is already enough.
-    if (isEnoughSize(accumulated_size.rows, accumulated_size.bytes))
+    if (isEnoughSize())
     {
         /// Return accumulated data and place new block to accumulated data.
-        Chunk res_chunk = convertToChunk(std::move(chunks_to_merge_vec));
-        chunks_to_merge_vec.clear();
-        changeCurrentSize(input_chunk.getNumRows(), input_chunk.bytes());
-        chunks_to_merge_vec.push_back(std::move(input_chunk));
+        Chunk res_chunk = convertToChunk(accumulated.extract());
+        accumulated.add(std::move(input_chunk));
         return res_chunk;
     }
 
     /// Pushing data into accumulating vector
-    expandCurrentSize(input_chunk.getNumRows(), input_chunk.bytes());
-    chunks_to_merge_vec.push_back(std::move(input_chunk));
+    accumulated.add(std::move(input_chunk));
 
     /// If accumulated data is big enough, we send it
-    if (isEnoughSize(accumulated_size.rows, accumulated_size.bytes))
-    {
-        Chunk res_chunk = convertToChunk(std::move(chunks_to_merge_vec));
-        changeCurrentSize(0, 0);
-        chunks_to_merge_vec.clear();
-        return res_chunk;
-    }
+    if (isEnoughSize())
+        return convertToChunk(accumulated.extract());
+
     return {};
 }
 
@@ -90,14 +89,15 @@ Chunk Squashing::convertToChunk(std::vector<Chunk> && chunks) const
     auto info = std::make_shared<ChunksToSquash>();
     info->chunks = std::move(chunks);
 
-    chunks.clear();
-
-    return Chunk(header.cloneEmptyColumns(), 0, info);
+    // It is imortant that chunk is not empty, it has to have columns even if they are empty
+    auto aggr_chunk = Chunk(header.getColumns(), 0);
+    aggr_chunk.getChunkInfos().add(std::move(info));
+    chassert(aggr_chunk);
+    return aggr_chunk;
 }
 
-Chunk Squashing::squash(std::vector<Chunk> & input_chunks)
+Chunk Squashing::squash(std::vector<Chunk> && input_chunks, Chunk::ChunkInfoCollection && infos)
 {
-    Chunk accumulated_chunk;
     std::vector<IColumn::MutablePtr> mutable_columns = {};
     size_t rows = 0;
     for (const Chunk & chunk : input_chunks)
@@ -119,35 +119,17 @@ Chunk Squashing::squash(std::vector<Chunk> & input_chunks)
         for (size_t j = 0, size = mutable_columns.size(); j < size; ++j)
         {
             const auto source_column = columns[j];
-
             mutable_columns[j]->insertRangeFrom(*source_column, 0, source_column->size());
         }
     }
-    accumulated_chunk.setColumns(std::move(mutable_columns), rows);
-    return accumulated_chunk;
-}
 
-const ChunksToSquash* Squashing::getInfoFromChunk(const Chunk & chunk)
-{
-    const auto& info = chunk.getChunkInfo();
-    const auto * agg_info = typeid_cast<const ChunksToSquash *>(info.get());
+    Chunk result;
+    result.setColumns(std::move(mutable_columns), rows);
+    result.setChunkInfos(infos);
+    result.getChunkInfos().append(std::move(input_chunks.back().getChunkInfos()));
 
-    if (!agg_info)
-        throw Exception(ErrorCodes::LOGICAL_ERROR, "There is no ChunksToSquash in ChunkInfoPtr");
-
-    return agg_info;
-}
-
-void Squashing::expandCurrentSize(size_t rows, size_t bytes)
-{
-    accumulated_size.rows += rows;
-    accumulated_size.bytes += bytes;
-}
-
-void Squashing::changeCurrentSize(size_t rows, size_t bytes)
-{
-    accumulated_size.rows = rows;
-    accumulated_size.bytes = bytes;
+    chassert(result);
+    return result;
 }
 
 bool Squashing::isEnoughSize(size_t rows, size_t bytes) const
@@ -156,4 +138,28 @@ bool Squashing::isEnoughSize(size_t rows, size_t bytes) const
         || (min_block_size_rows && rows >= min_block_size_rows)
         || (min_block_size_bytes && bytes >= min_block_size_bytes);
 }
+
+bool Squashing::isEnoughSize() const
+{
+    return isEnoughSize(accumulated.getRows(), accumulated.getBytes());
+};
+
+bool Squashing::isEnoughSize(const Chunk & chunk) const
+{
+    return isEnoughSize(chunk.getNumRows(), chunk.bytes());
+}
+
+void Squashing::CurrentSize::add(Chunk && chunk)
+{
+    rows += chunk.getNumRows();
+    bytes += chunk.bytes();
+    chunks.push_back(std::move(chunk));
+}
+
+std::vector<Chunk> Squashing::CurrentSize::extract()
+{
+    auto result = std::move(chunks);
+    *this = {};
+    return result;
+}
 }
diff --git a/src/Interpreters/Squashing.h b/src/Interpreters/Squashing.h
index d76cca60e41..64a9768a71f 100644
--- a/src/Interpreters/Squashing.h
+++ b/src/Interpreters/Squashing.h
@@ -8,9 +8,18 @@
 namespace DB
 {
 
-struct ChunksToSquash : public ChunkInfo
+class ChunksToSquash : public ChunkInfoCloneable<ChunksToSquash>
 {
-    mutable std::vector<Chunk> chunks = {};
+public:
+    ChunksToSquash() = default;
+    ChunksToSquash(const ChunksToSquash & other)
+    {
+        chunks.reserve(other.chunks.size());
+        for (const auto & chunk: other.chunks)
+           chunks.push_back(chunk.clone());
+    }
+
+    std::vector<Chunk> chunks = {};
 };
 
 /** Merging consecutive passed blocks to specified minimum size.
@@ -36,32 +45,35 @@ public:
     static Chunk squash(Chunk && input_chunk);
     Chunk flush();
 
-    bool isDataLeft()
-    {
-        return !chunks_to_merge_vec.empty();
-    }
+    void setHeader(Block header_) { header = std::move(header_); }
+    const Block & getHeader() const { return header; }
 
-    Block header;
 private:
-    struct CurrentSize
+    class CurrentSize
     {
+        std::vector<Chunk> chunks = {};
         size_t rows = 0;
         size_t bytes = 0;
+
+    public:
+        explicit operator bool () const { return !chunks.empty(); }
+        size_t getRows() const { return rows; }
+        size_t getBytes() const { return bytes; }
+        void add(Chunk && chunk);
+        std::vector<Chunk> extract();
     };
 
-    std::vector<Chunk> chunks_to_merge_vec = {};
-    size_t min_block_size_rows;
-    size_t min_block_size_bytes;
+    const size_t min_block_size_rows;
+    const size_t min_block_size_bytes;
+    Block header;
 
-    CurrentSize accumulated_size;
+    CurrentSize accumulated;
 
-    static const ChunksToSquash * getInfoFromChunk(const Chunk & chunk);
+    static Chunk squash(std::vector<Chunk> && input_chunks, Chunk::ChunkInfoCollection && infos);
 
-    static Chunk squash(std::vector<Chunk> & input_chunks);
-
-    void expandCurrentSize(size_t rows, size_t bytes);
-    void changeCurrentSize(size_t rows, size_t bytes);
+    bool isEnoughSize() const;
     bool isEnoughSize(size_t rows, size_t bytes) const;
+    bool isEnoughSize(const Chunk & chunk) const;
 
     Chunk convertToChunk(std::vector<Chunk> && chunks) const;
 };
diff --git a/src/Interpreters/SystemLog.cpp b/src/Interpreters/SystemLog.cpp
index 557065b23ff..f386e157b14 100644
--- a/src/Interpreters/SystemLog.cpp
+++ b/src/Interpreters/SystemLog.cpp
@@ -538,7 +538,13 @@ void SystemLog<LogElement>::flushImpl(const std::vector<LogElement> & to_flush,
         insert_context->makeQueryContext();
         addSettingsForQuery(insert_context, IAST::QueryKind::Insert);
 
-        InterpreterInsertQuery interpreter(query_ptr, insert_context);
+        InterpreterInsertQuery interpreter(
+            query_ptr,
+            insert_context,
+            /* allow_materialized */ false,
+            /* no_squash */ false,
+            /* no_destination */ false,
+            /* async_isnert */ false);
         BlockIO io = interpreter.execute();
 
         PushingPipelineExecutor executor(io.pipeline);
diff --git a/src/Interpreters/TreeRewriter.cpp b/src/Interpreters/TreeRewriter.cpp
index a3c5a7ed3ed..6ce6f5e454e 100644
--- a/src/Interpreters/TreeRewriter.cpp
+++ b/src/Interpreters/TreeRewriter.cpp
@@ -1188,7 +1188,7 @@ bool TreeRewriterResult::collectUsedColumns(const ASTPtr & query, bool is_select
         }
     }
 
-    /// Check for dynamic subcolums in unknown required columns.
+    /// Check for dynamic subcolumns in unknown required columns.
     if (!unknown_required_source_columns.empty())
     {
         for (const NameAndTypePair & pair : source_columns_ordinary)
diff --git a/src/Processors/Chunk.cpp b/src/Processors/Chunk.cpp
index 5f6cf2f7230..4466be5b3a7 100644
--- a/src/Processors/Chunk.cpp
+++ b/src/Processors/Chunk.cpp
@@ -19,14 +19,6 @@ Chunk::Chunk(DB::Columns columns_, UInt64 num_rows_) : columns(std::move(columns
     checkNumRowsIsConsistent();
 }
 
-Chunk::Chunk(Columns columns_, UInt64 num_rows_, ChunkInfoPtr chunk_info_)
-    : columns(std::move(columns_))
-    , num_rows(num_rows_)
-    , chunk_info(std::move(chunk_info_))
-{
-    checkNumRowsIsConsistent();
-}
-
 static Columns unmuteColumns(MutableColumns && mutable_columns)
 {
     Columns columns;
@@ -43,17 +35,11 @@ Chunk::Chunk(MutableColumns columns_, UInt64 num_rows_)
     checkNumRowsIsConsistent();
 }
 
-Chunk::Chunk(MutableColumns columns_, UInt64 num_rows_, ChunkInfoPtr chunk_info_)
-    : columns(unmuteColumns(std::move(columns_)))
-    , num_rows(num_rows_)
-    , chunk_info(std::move(chunk_info_))
-{
-    checkNumRowsIsConsistent();
-}
-
 Chunk Chunk::clone() const
 {
-    return Chunk(getColumns(), getNumRows(), chunk_info);
+    auto tmp = Chunk(getColumns(), getNumRows());
+    tmp.setChunkInfos(chunk_infos.clone());
+    return tmp;
 }
 
 void Chunk::setColumns(Columns columns_, UInt64 num_rows_)
diff --git a/src/Processors/Chunk.h b/src/Processors/Chunk.h
index 4f753798eaa..1348966c0d3 100644
--- a/src/Processors/Chunk.h
+++ b/src/Processors/Chunk.h
@@ -1,7 +1,9 @@
 #pragma once
 
+#include <Common/CollectionOfDerived.h>
 #include <Columns/IColumn.h>
-#include <unordered_map>
+
+#include <memory>
 
 namespace DB
 {
@@ -9,11 +11,29 @@ namespace DB
 class ChunkInfo
 {
 public:
-    virtual ~ChunkInfo() = default;
+    using Ptr = std::shared_ptr<ChunkInfo>;
+
     ChunkInfo() = default;
+    ChunkInfo(const ChunkInfo&) = default;
+    ChunkInfo(ChunkInfo&&) = default;
+
+    virtual Ptr clone() const = 0;
+    virtual ~ChunkInfo() = default;
 };
 
-using ChunkInfoPtr = std::shared_ptr<const ChunkInfo>;
+
+template<class Derived>
+class ChunkInfoCloneable : public ChunkInfo
+{
+public:
+    ChunkInfoCloneable() = default;
+    ChunkInfoCloneable(const ChunkInfoCloneable & other) = default;
+
+    Ptr clone() const override
+    {
+        return std::static_pointer_cast<ChunkInfo>(std::make_shared<Derived>(*static_cast<const Derived*>(this)));
+    }
+};
 
 /**
  * Chunk is a list of columns with the same length.
@@ -32,26 +52,26 @@ using ChunkInfoPtr = std::shared_ptr<const ChunkInfo>;
 class Chunk
 {
 public:
+    using ChunkInfoCollection = CollectionOfDerivedItems<ChunkInfo>;
+
     Chunk() = default;
     Chunk(const Chunk & other) = delete;
     Chunk(Chunk && other) noexcept
         : columns(std::move(other.columns))
         , num_rows(other.num_rows)
-        , chunk_info(std::move(other.chunk_info))
+        , chunk_infos(std::move(other.chunk_infos))
     {
         other.num_rows = 0;
     }
 
     Chunk(Columns columns_, UInt64 num_rows_);
-    Chunk(Columns columns_, UInt64 num_rows_, ChunkInfoPtr chunk_info_);
     Chunk(MutableColumns columns_, UInt64 num_rows_);
-    Chunk(MutableColumns columns_, UInt64 num_rows_, ChunkInfoPtr chunk_info_);
 
     Chunk & operator=(const Chunk & other) = delete;
     Chunk & operator=(Chunk && other) noexcept
     {
         columns = std::move(other.columns);
-        chunk_info = std::move(other.chunk_info);
+        chunk_infos = std::move(other.chunk_infos);
         num_rows = other.num_rows;
         other.num_rows = 0;
         return *this;
@@ -62,15 +82,15 @@ public:
     void swap(Chunk & other) noexcept
     {
         columns.swap(other.columns);
-        chunk_info.swap(other.chunk_info);
         std::swap(num_rows, other.num_rows);
+        chunk_infos.swap(other.chunk_infos);
     }
 
     void clear()
     {
         num_rows = 0;
         columns.clear();
-        chunk_info.reset();
+        chunk_infos.clear();
     }
 
     const Columns & getColumns() const { return columns; }
@@ -81,9 +101,9 @@ public:
     /** Get empty columns with the same types as in block. */
     MutableColumns cloneEmptyColumns() const;
 
-    const ChunkInfoPtr & getChunkInfo() const { return chunk_info; }
-    bool hasChunkInfo() const { return chunk_info != nullptr; }
-    void setChunkInfo(ChunkInfoPtr chunk_info_) { chunk_info = std::move(chunk_info_); }
+    ChunkInfoCollection & getChunkInfos() { return chunk_infos; }
+    const ChunkInfoCollection & getChunkInfos() const { return chunk_infos; }
+    void setChunkInfos(ChunkInfoCollection chunk_infos_) { chunk_infos = std::move(chunk_infos_); }
 
     UInt64 getNumRows() const { return num_rows; }
     UInt64 getNumColumns() const { return columns.size(); }
@@ -107,7 +127,7 @@ public:
 private:
     Columns columns;
     UInt64 num_rows = 0;
-    ChunkInfoPtr chunk_info;
+    ChunkInfoCollection chunk_infos;
 
     void checkNumRowsIsConsistent();
 };
@@ -117,11 +137,15 @@ using Chunks = std::vector<Chunk>;
 /// AsyncInsert needs two kinds of information:
 /// - offsets of different sub-chunks
 /// - tokens of different sub-chunks, which are assigned by setting `insert_deduplication_token`.
-class AsyncInsertInfo : public ChunkInfo
+class AsyncInsertInfo : public ChunkInfoCloneable<AsyncInsertInfo>
 {
 public:
     AsyncInsertInfo() = default;
-    explicit AsyncInsertInfo(const std::vector<size_t> & offsets_, const std::vector<String> & tokens_) : offsets(offsets_), tokens(tokens_) {}
+    AsyncInsertInfo(const AsyncInsertInfo & other) = default;
+    AsyncInsertInfo(const std::vector<size_t> & offsets_, const std::vector<String> & tokens_)
+        : offsets(offsets_)
+        , tokens(tokens_)
+    {}
 
     std::vector<size_t> offsets;
     std::vector<String> tokens;
@@ -130,9 +154,11 @@ public:
 using AsyncInsertInfoPtr = std::shared_ptr<AsyncInsertInfo>;
 
 /// Extension to support delayed defaults. AddingDefaultsProcessor uses it to replace missing values with column defaults.
-class ChunkMissingValues : public ChunkInfo
+class ChunkMissingValues : public ChunkInfoCloneable<ChunkMissingValues>
 {
 public:
+    ChunkMissingValues(const ChunkMissingValues & other) = default;
+
     using RowsBitMask = std::vector<bool>; /// a bit per row for a column
 
     const RowsBitMask & getDefaultsBitmask(size_t column_idx) const;
diff --git a/src/Processors/Executors/PullingAsyncPipelineExecutor.cpp b/src/Processors/Executors/PullingAsyncPipelineExecutor.cpp
index d27002197d2..d9fab88fe1f 100644
--- a/src/Processors/Executors/PullingAsyncPipelineExecutor.cpp
+++ b/src/Processors/Executors/PullingAsyncPipelineExecutor.cpp
@@ -147,13 +147,10 @@ bool PullingAsyncPipelineExecutor::pull(Block & block, uint64_t milliseconds)
 
     block = lazy_format->getPort(IOutputFormat::PortKind::Main).getHeader().cloneWithColumns(chunk.detachColumns());
 
-    if (auto chunk_info = chunk.getChunkInfo())
+    if (auto agg_info = chunk.getChunkInfos().get<AggregatedChunkInfo>())
     {
-        if (const auto * agg_info = typeid_cast<const AggregatedChunkInfo *>(chunk_info.get()))
-        {
-            block.info.bucket_num = agg_info->bucket_num;
-            block.info.is_overflows = agg_info->is_overflows;
-        }
+         block.info.bucket_num = agg_info->bucket_num;
+         block.info.is_overflows = agg_info->is_overflows;
     }
 
     return true;
diff --git a/src/Processors/Executors/PullingPipelineExecutor.cpp b/src/Processors/Executors/PullingPipelineExecutor.cpp
index cbf73c5cb07..25c15d40c9a 100644
--- a/src/Processors/Executors/PullingPipelineExecutor.cpp
+++ b/src/Processors/Executors/PullingPipelineExecutor.cpp
@@ -73,13 +73,10 @@ bool PullingPipelineExecutor::pull(Block & block)
     }
 
     block = pulling_format->getPort(IOutputFormat::PortKind::Main).getHeader().cloneWithColumns(chunk.detachColumns());
-    if (auto chunk_info = chunk.getChunkInfo())
+    if (auto agg_info = chunk.getChunkInfos().get<AggregatedChunkInfo>())
     {
-        if (const auto * agg_info = typeid_cast<const AggregatedChunkInfo *>(chunk_info.get()))
-        {
-            block.info.bucket_num = agg_info->bucket_num;
-            block.info.is_overflows = agg_info->is_overflows;
-        }
+        block.info.bucket_num = agg_info->bucket_num;
+        block.info.is_overflows = agg_info->is_overflows;
     }
 
     return true;
diff --git a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp
index a5d334f4f1d..9e499e2c400 100644
--- a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp
+++ b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp
@@ -179,7 +179,9 @@ void ParquetBlockOutputFormat::consume(Chunk chunk)
                 columns[i]->insertRangeFrom(*concatenated.getColumns()[i], offset, count);
 
             Chunks piece;
-            piece.emplace_back(std::move(columns), count, concatenated.getChunkInfo());
+            piece.emplace_back(std::move(columns), count);
+            piece.back().setChunkInfos(concatenated.getChunkInfos());
+
             writeRowGroup(std::move(piece));
         }
     }
diff --git a/src/Processors/IAccumulatingTransform.cpp b/src/Processors/IAccumulatingTransform.cpp
index 4136fc5a5f2..46be6e74693 100644
--- a/src/Processors/IAccumulatingTransform.cpp
+++ b/src/Processors/IAccumulatingTransform.cpp
@@ -8,8 +8,9 @@ namespace ErrorCodes
 }
 
 IAccumulatingTransform::IAccumulatingTransform(Block input_header, Block output_header)
-    : IProcessor({std::move(input_header)}, {std::move(output_header)}),
-    input(inputs.front()), output(outputs.front())
+    : IProcessor({std::move(input_header)}, {std::move(output_header)})
+    , input(inputs.front())
+    , output(outputs.front())
 {
 }
 
diff --git a/src/Processors/Merges/Algorithms/FinishAggregatingInOrderAlgorithm.cpp b/src/Processors/Merges/Algorithms/FinishAggregatingInOrderAlgorithm.cpp
index 466adf93538..86675bcb237 100644
--- a/src/Processors/Merges/Algorithms/FinishAggregatingInOrderAlgorithm.cpp
+++ b/src/Processors/Merges/Algorithms/FinishAggregatingInOrderAlgorithm.cpp
@@ -53,13 +53,11 @@ void FinishAggregatingInOrderAlgorithm::consume(Input & input, size_t source_num
     if (!input.chunk.hasRows())
         return;
 
-    const auto & info = input.chunk.getChunkInfo();
-    if (!info)
+    if (input.chunk.getChunkInfos().empty())
         throw Exception(ErrorCodes::LOGICAL_ERROR, "Chunk info was not set for chunk in FinishAggregatingInOrderAlgorithm");
 
     Int64 allocated_bytes = 0;
-    /// Will be set by AggregatingInOrderTransform during local aggregation; will be nullptr during merging on initiator.
-    if (const auto * arenas_info = typeid_cast<const ChunkInfoWithAllocatedBytes *>(info.get()))
+    if (auto arenas_info = input.chunk.getChunkInfos().get<ChunkInfoWithAllocatedBytes>())
         allocated_bytes = arenas_info->allocated_bytes;
 
     states[source_num] = State{input.chunk, description, allocated_bytes};
@@ -136,7 +134,7 @@ Chunk FinishAggregatingInOrderAlgorithm::prepareToMerge()
     info->chunk_num = chunk_num++;
 
     Chunk chunk;
-    chunk.setChunkInfo(std::move(info));
+    chunk.getChunkInfos().add(std::move(info));
     return chunk;
 }
 
@@ -163,7 +161,7 @@ void FinishAggregatingInOrderAlgorithm::addToAggregation()
             chunks.emplace_back(std::move(new_columns), current_rows);
         }
 
-        chunks.back().setChunkInfo(std::make_shared<AggregatedChunkInfo>());
+        chunks.back().getChunkInfos().add(std::make_shared<AggregatedChunkInfo>());
         states[i].current_row = states[i].to_row;
 
         /// We assume that sizes in bytes of rows are almost the same.
diff --git a/src/Processors/Merges/Algorithms/MergeTreePartLevelInfo.h b/src/Processors/Merges/Algorithms/MergeTreePartLevelInfo.h
index bcf4e759024..e4f22deec8d 100644
--- a/src/Processors/Merges/Algorithms/MergeTreePartLevelInfo.h
+++ b/src/Processors/Merges/Algorithms/MergeTreePartLevelInfo.h
@@ -6,18 +6,22 @@ namespace DB
 {
 
 /// To carry part level if chunk is produced by a merge tree source
-class MergeTreePartLevelInfo : public ChunkInfo
+class MergeTreePartLevelInfo : public ChunkInfoCloneable<MergeTreePartLevelInfo>
 {
 public:
     MergeTreePartLevelInfo() = delete;
-    explicit MergeTreePartLevelInfo(ssize_t part_level) : origin_merge_tree_part_level(part_level) { }
+    explicit MergeTreePartLevelInfo(ssize_t part_level)
+        : origin_merge_tree_part_level(part_level)
+    { }
+    MergeTreePartLevelInfo(const MergeTreePartLevelInfo & other) = default;
+
     size_t origin_merge_tree_part_level = 0;
 };
 
 inline size_t getPartLevelFromChunk(const Chunk & chunk)
 {
-    const auto & info = chunk.getChunkInfo();
-    if (const auto * part_level_info = typeid_cast<const MergeTreePartLevelInfo *>(info.get()))
+    const auto part_level_info = chunk.getChunkInfos().get<MergeTreePartLevelInfo>();
+    if (part_level_info)
         return part_level_info->origin_merge_tree_part_level;
     return 0;
 }
diff --git a/src/Processors/Merges/Algorithms/ReplacingSortedAlgorithm.cpp b/src/Processors/Merges/Algorithms/ReplacingSortedAlgorithm.cpp
index 7b2c7d82a01..cd347d371d9 100644
--- a/src/Processors/Merges/Algorithms/ReplacingSortedAlgorithm.cpp
+++ b/src/Processors/Merges/Algorithms/ReplacingSortedAlgorithm.cpp
@@ -17,7 +17,7 @@ namespace ErrorCodes
 
 static IMergingAlgorithm::Status emitChunk(detail::SharedChunkPtr & chunk, bool finished = false)
 {
-    chunk->setChunkInfo(std::make_shared<ChunkSelectFinalIndices>(std::move(chunk->replace_final_selection)));
+    chunk->getChunkInfos().add(std::make_shared<ChunkSelectFinalIndices>(std::move(chunk->replace_final_selection)));
     return IMergingAlgorithm::Status(std::move(*chunk), finished);
 }
 
diff --git a/src/Processors/Merges/Algorithms/ReplacingSortedAlgorithm.h b/src/Processors/Merges/Algorithms/ReplacingSortedAlgorithm.h
index a3ccccf0845..2f23f2a5c4d 100644
--- a/src/Processors/Merges/Algorithms/ReplacingSortedAlgorithm.h
+++ b/src/Processors/Merges/Algorithms/ReplacingSortedAlgorithm.h
@@ -3,6 +3,7 @@
 #include <Processors/Merges/Algorithms/MergedData.h>
 #include <Processors/Transforms/ColumnGathererTransform.h>
 #include <Processors/Merges/Algorithms/RowRef.h>
+#include <Processors/Chunk.h>
 
 namespace Poco
 {
@@ -14,11 +15,13 @@ namespace DB
 
 /** Use in skipping final to keep list of indices of selected row after merging final
   */
-struct ChunkSelectFinalIndices : public ChunkInfo
+struct ChunkSelectFinalIndices : public ChunkInfoCloneable<ChunkSelectFinalIndices>
 {
+    explicit ChunkSelectFinalIndices(MutableColumnPtr select_final_indices_);
+    ChunkSelectFinalIndices(const ChunkSelectFinalIndices & other) = default;
+
     const ColumnPtr column_holder;
     const ColumnUInt64 * select_final_indices = nullptr;
-    explicit ChunkSelectFinalIndices(MutableColumnPtr select_final_indices_);
 };
 
 /** Merges several sorted inputs into one.
diff --git a/src/Processors/Merges/IMergingTransform.cpp b/src/Processors/Merges/IMergingTransform.cpp
index fbb47969b2f..b1b0182a113 100644
--- a/src/Processors/Merges/IMergingTransform.cpp
+++ b/src/Processors/Merges/IMergingTransform.cpp
@@ -157,7 +157,7 @@ IProcessor::Status IMergingTransformBase::prepare()
     bool is_port_full = !output.canPush();
 
     /// Push if has data.
-    if ((state.output_chunk || state.output_chunk.hasChunkInfo()) && !is_port_full)
+    if ((state.output_chunk || !state.output_chunk.getChunkInfos().empty()) && !is_port_full)
         output.push(std::move(state.output_chunk));
 
     if (!is_initialized)
diff --git a/src/Processors/Merges/IMergingTransform.h b/src/Processors/Merges/IMergingTransform.h
index c218f622870..be629271736 100644
--- a/src/Processors/Merges/IMergingTransform.h
+++ b/src/Processors/Merges/IMergingTransform.h
@@ -129,7 +129,7 @@ public:
 
         IMergingAlgorithm::Status status = algorithm.merge();
 
-        if ((status.chunk && status.chunk.hasRows()) || status.chunk.hasChunkInfo())
+        if ((status.chunk && status.chunk.hasRows()) || !status.chunk.getChunkInfos().empty())
         {
             // std::cerr << "Got chunk with " << status.chunk.getNumRows() << " rows" << std::endl;
             state.output_chunk = std::move(status.chunk);
diff --git a/src/Processors/Sinks/RemoteSink.h b/src/Processors/Sinks/RemoteSink.h
index 30cf958c072..c05cc1defcb 100644
--- a/src/Processors/Sinks/RemoteSink.h
+++ b/src/Processors/Sinks/RemoteSink.h
@@ -20,7 +20,7 @@ public:
     }
 
     String getName() const override { return "RemoteSink"; }
-    void consume (Chunk chunk) override { write(RemoteInserter::getHeader().cloneWithColumns(chunk.detachColumns())); }
+    void consume (Chunk & chunk) override { write(RemoteInserter::getHeader().cloneWithColumns(chunk.getColumns())); }
     void onFinish() override { RemoteInserter::onFinish(); }
 };
 
diff --git a/src/Processors/Sinks/SinkToStorage.cpp b/src/Processors/Sinks/SinkToStorage.cpp
index 5f9f9f9b1a1..36bb70f493f 100644
--- a/src/Processors/Sinks/SinkToStorage.cpp
+++ b/src/Processors/Sinks/SinkToStorage.cpp
@@ -15,9 +15,8 @@ void SinkToStorage::onConsume(Chunk chunk)
       */
     Nested::validateArraySizes(getHeader().cloneWithColumns(chunk.getColumns()));
 
-    consume(chunk.clone());
-    if (!lastBlockIsDuplicate())
-        cur_chunk = std::move(chunk);
+    consume(chunk);
+    cur_chunk = std::move(chunk);
 }
 
 SinkToStorage::GenerateResult SinkToStorage::onGenerate()
diff --git a/src/Processors/Sinks/SinkToStorage.h b/src/Processors/Sinks/SinkToStorage.h
index 023bbd8b094..c728fa87b1e 100644
--- a/src/Processors/Sinks/SinkToStorage.h
+++ b/src/Processors/Sinks/SinkToStorage.h
@@ -18,8 +18,7 @@ public:
     void addTableLock(const TableLockHolder & lock) { table_locks.push_back(lock); }
 
 protected:
-    virtual void consume(Chunk chunk) = 0;
-    virtual bool lastBlockIsDuplicate() const { return false; }
+    virtual void consume(Chunk & chunk) = 0;
 
 private:
     std::vector<TableLockHolder> table_locks;
@@ -38,7 +37,7 @@ class NullSinkToStorage : public SinkToStorage
 public:
     using SinkToStorage::SinkToStorage;
     std::string getName() const override { return "NullSinkToStorage"; }
-    void consume(Chunk) override {}
+    void consume(Chunk &) override {}
 };
 
 using SinkPtr = std::shared_ptr<SinkToStorage>;
diff --git a/src/Processors/Sources/BlocksSource.h b/src/Processors/Sources/BlocksSource.h
index ec0dc9609f1..7ac460c14e2 100644
--- a/src/Processors/Sources/BlocksSource.h
+++ b/src/Processors/Sources/BlocksSource.h
@@ -43,7 +43,10 @@ protected:
         info->bucket_num = res.info.bucket_num;
         info->is_overflows = res.info.is_overflows;
 
-        return Chunk(res.getColumns(), res.rows(), std::move(info));
+        auto chunk = Chunk(res.getColumns(), res.rows());
+        chunk.getChunkInfos().add(std::move(info));
+
+        return chunk;
     }
 
 private:
diff --git a/src/Processors/Sources/RemoteSource.cpp b/src/Processors/Sources/RemoteSource.cpp
index 3d7dd3f76b8..1578bd389c9 100644
--- a/src/Processors/Sources/RemoteSource.cpp
+++ b/src/Processors/Sources/RemoteSource.cpp
@@ -176,7 +176,7 @@ std::optional<Chunk> RemoteSource::tryGenerate()
         auto info = std::make_shared<AggregatedChunkInfo>();
         info->bucket_num = block.info.bucket_num;
         info->is_overflows = block.info.is_overflows;
-        chunk.setChunkInfo(std::move(info));
+        chunk.getChunkInfos().add(std::move(info));
     }
 
     return chunk;
diff --git a/src/Processors/Sources/SourceFromSingleChunk.cpp b/src/Processors/Sources/SourceFromSingleChunk.cpp
index 00f40a34361..9abe0504d10 100644
--- a/src/Processors/Sources/SourceFromSingleChunk.cpp
+++ b/src/Processors/Sources/SourceFromSingleChunk.cpp
@@ -5,7 +5,9 @@
 namespace DB
 {
 
-SourceFromSingleChunk::SourceFromSingleChunk(Block header, Chunk chunk_) : ISource(std::move(header)), chunk(std::move(chunk_)) {}
+SourceFromSingleChunk::SourceFromSingleChunk(Block header, Chunk chunk_) : ISource(std::move(header)), chunk(std::move(chunk_))
+{
+}
 
 SourceFromSingleChunk::SourceFromSingleChunk(Block data) : ISource(data.cloneEmpty()), chunk(data.getColumns(), data.rows())
 {
@@ -20,7 +22,7 @@ SourceFromSingleChunk::SourceFromSingleChunk(Block data) : ISource(data.cloneEmp
         auto info = std::make_shared<AggregatedChunkInfo>();
         info->bucket_num = data.info.bucket_num;
         info->is_overflows = data.info.is_overflows;
-        chunk.setChunkInfo(std::move(info));
+        chunk.getChunkInfos().add(std::move(info));
     }
 }
 
diff --git a/src/Processors/Transforms/AggregatingInOrderTransform.cpp b/src/Processors/Transforms/AggregatingInOrderTransform.cpp
index 9ffe15d0f85..45b0960ec8f 100644
--- a/src/Processors/Transforms/AggregatingInOrderTransform.cpp
+++ b/src/Processors/Transforms/AggregatingInOrderTransform.cpp
@@ -332,7 +332,7 @@ void AggregatingInOrderTransform::generate()
     variants.aggregates_pool = variants.aggregates_pools.at(0).get();
 
     /// Pass info about used memory by aggregate functions further.
-    to_push_chunk.setChunkInfo(std::make_shared<ChunkInfoWithAllocatedBytes>(cur_block_bytes));
+    to_push_chunk.getChunkInfos().add(std::make_shared<ChunkInfoWithAllocatedBytes>(cur_block_bytes));
 
     cur_block_bytes = 0;
     cur_block_size = 0;
@@ -351,11 +351,12 @@ FinalizeAggregatedTransform::FinalizeAggregatedTransform(Block header, Aggregati
 void FinalizeAggregatedTransform::transform(Chunk & chunk)
 {
     if (params->final)
-        finalizeChunk(chunk, aggregates_mask);
-    else if (!chunk.getChunkInfo())
     {
-        auto info = std::make_shared<AggregatedChunkInfo>();
-        chunk.setChunkInfo(std::move(info));
+        finalizeChunk(chunk, aggregates_mask);
+    }
+    else if (!chunk.getChunkInfos().get<AggregatedChunkInfo>())
+    {
+        chunk.getChunkInfos().add(std::make_shared<AggregatedChunkInfo>());
     }
 }
 
diff --git a/src/Processors/Transforms/AggregatingInOrderTransform.h b/src/Processors/Transforms/AggregatingInOrderTransform.h
index 5d50e97f552..41a0d7fc7f1 100644
--- a/src/Processors/Transforms/AggregatingInOrderTransform.h
+++ b/src/Processors/Transforms/AggregatingInOrderTransform.h
@@ -5,6 +5,7 @@
 #include <Processors/ISimpleTransform.h>
 #include <Processors/Transforms/AggregatingTransform.h>
 #include <Processors/Transforms/finalizeChunk.h>
+#include <Processors/Chunk.h>
 
 namespace DB
 {
@@ -12,10 +13,12 @@ namespace DB
 struct InputOrderInfo;
 using InputOrderInfoPtr = std::shared_ptr<const InputOrderInfo>;
 
-struct ChunkInfoWithAllocatedBytes : public ChunkInfo
+struct ChunkInfoWithAllocatedBytes : public ChunkInfoCloneable<ChunkInfoWithAllocatedBytes>
 {
+    ChunkInfoWithAllocatedBytes(const ChunkInfoWithAllocatedBytes & other) = default;
     explicit ChunkInfoWithAllocatedBytes(Int64 allocated_bytes_)
         : allocated_bytes(allocated_bytes_) {}
+
     Int64 allocated_bytes;
 };
 
diff --git a/src/Processors/Transforms/AggregatingTransform.cpp b/src/Processors/Transforms/AggregatingTransform.cpp
index 65f0612d738..517f035667f 100644
--- a/src/Processors/Transforms/AggregatingTransform.cpp
+++ b/src/Processors/Transforms/AggregatingTransform.cpp
@@ -35,7 +35,7 @@ Chunk convertToChunk(const Block & block)
 
     UInt64 num_rows = block.rows();
     Chunk chunk(block.getColumns(), num_rows);
-    chunk.setChunkInfo(std::move(info));
+    chunk.getChunkInfos().add(std::move(info));
 
     return chunk;
 }
@@ -44,15 +44,11 @@ namespace
 {
     const AggregatedChunkInfo * getInfoFromChunk(const Chunk & chunk)
     {
-        const auto & info = chunk.getChunkInfo();
-        if (!info)
-            throw Exception(ErrorCodes::LOGICAL_ERROR, "Chunk info was not set for chunk.");
-
-        const auto * agg_info = typeid_cast<const AggregatedChunkInfo *>(info.get());
+        auto agg_info = chunk.getChunkInfos().get<AggregatedChunkInfo>();
         if (!agg_info)
             throw Exception(ErrorCodes::LOGICAL_ERROR, "Chunk should have AggregatedChunkInfo.");
 
-        return agg_info;
+        return agg_info.get();
     }
 
     /// Reads chunks from file in native format. Provide chunks with aggregation info.
@@ -210,11 +206,7 @@ private:
 
     void process(Chunk && chunk)
     {
-        if (!chunk.hasChunkInfo())
-            throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected chunk with chunk info in {}", getName());
-
-        const auto & info = chunk.getChunkInfo();
-        const auto * chunks_to_merge = typeid_cast<const ChunksToMerge *>(info.get());
+        auto chunks_to_merge = chunk.getChunkInfos().get<ChunksToMerge>();
         if (!chunks_to_merge)
             throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected chunk with ChunksToMerge info in {}", getName());
 
diff --git a/src/Processors/Transforms/AggregatingTransform.h b/src/Processors/Transforms/AggregatingTransform.h
index e167acde067..95983c39d1e 100644
--- a/src/Processors/Transforms/AggregatingTransform.h
+++ b/src/Processors/Transforms/AggregatingTransform.h
@@ -2,6 +2,7 @@
 #include <Compression/CompressedReadBuffer.h>
 #include <IO/ReadBufferFromFile.h>
 #include <Interpreters/Aggregator.h>
+#include <Processors/Chunk.h>
 #include <Processors/IAccumulatingTransform.h>
 #include <Common/Stopwatch.h>
 #include <Common/setThreadName.h>
@@ -19,7 +20,7 @@ namespace CurrentMetrics
 namespace DB
 {
 
-class AggregatedChunkInfo : public ChunkInfo
+class AggregatedChunkInfo : public ChunkInfoCloneable<AggregatedChunkInfo>
 {
 public:
     bool is_overflows = false;
diff --git a/src/Processors/Transforms/ApplySquashingTransform.h b/src/Processors/Transforms/ApplySquashingTransform.h
index 965a084bb13..49a6581e685 100644
--- a/src/Processors/Transforms/ApplySquashingTransform.h
+++ b/src/Processors/Transforms/ApplySquashingTransform.h
@@ -27,18 +27,12 @@ public:
         }
 
         ExceptionKeepingTransform::work();
-        if (finish_chunk)
-        {
-            data.chunk = std::move(finish_chunk);
-            ready_output = true;
-        }
     }
 
 protected:
     void onConsume(Chunk chunk) override
     {
-        if (auto res_chunk = DB::Squashing::squash(std::move(chunk)))
-            cur_chunk.setColumns(res_chunk.getColumns(), res_chunk.getNumRows());
+        cur_chunk = Squashing::squash(std::move(chunk));
     }
 
     GenerateResult onGenerate() override
@@ -48,16 +42,10 @@ protected:
         res.is_done = true;
         return res;
     }
-    void onFinish() override
-    {
-        auto chunk = DB::Squashing::squash({});
-        finish_chunk.setColumns(chunk.getColumns(), chunk.getNumRows());
-    }
 
 private:
     Squashing squashing;
     Chunk cur_chunk;
-    Chunk finish_chunk;
 };
 
 }
diff --git a/src/Processors/Transforms/CountingTransform.cpp b/src/Processors/Transforms/CountingTransform.cpp
index 3dfb9fe178f..2c6b3bd8638 100644
--- a/src/Processors/Transforms/CountingTransform.cpp
+++ b/src/Processors/Transforms/CountingTransform.cpp
@@ -1,6 +1,7 @@
-
-#include <Interpreters/ProcessList.h>
 #include <Processors/Transforms/CountingTransform.h>
+
+#include <IO/Progress.h>
+#include <Interpreters/ProcessList.h>
 #include <Common/ProfileEvents.h>
 #include <Common/ThreadStatus.h>
 
diff --git a/src/Processors/Transforms/DeduplicationTokenTransforms.cpp b/src/Processors/Transforms/DeduplicationTokenTransforms.cpp
new file mode 100644
index 00000000000..6786f76cbef
--- /dev/null
+++ b/src/Processors/Transforms/DeduplicationTokenTransforms.cpp
@@ -0,0 +1,236 @@
+#include <Processors/Transforms/DeduplicationTokenTransforms.h>
+
+#include <IO/WriteHelpers.h>
+
+#include <Common/logger_useful.h>
+#include <Common/Exception.h>
+#include <Common/SipHash.h>
+
+
+#include <fmt/core.h>
+
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int LOGICAL_ERROR;
+}
+
+void RestoreChunkInfosTransform::transform(Chunk & chunk)
+{
+    chunk.getChunkInfos().append(chunk_infos.clone());
+}
+
+namespace DeduplicationToken
+{
+
+String TokenInfo::getToken() const
+{
+    if (!isDefined())
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "token is not defined, stage {}, token {}", stage, debugToken());
+
+    return getTokenImpl();
+}
+
+String TokenInfo::getTokenImpl() const
+{
+    String result;
+    result.reserve(getTotalSize());
+
+    for (const auto & part : parts)
+    {
+        if (!result.empty())
+            result.append(":");
+        result.append(part);
+    }
+
+    return result;
+}
+
+String TokenInfo::debugToken() const
+{
+    return getTokenImpl();
+}
+
+void TokenInfo::addChunkHash(String part)
+{
+    if (stage == UNDEFINED && empty())
+        stage = DEFINE_SOURCE_WITH_HASHES;
+
+    if (stage != DEFINE_SOURCE_WITH_HASHES)
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "token is in wrong stage {}, token {}", stage, debugToken());
+
+    addTokenPart(std::move(part));
+}
+
+void TokenInfo::finishChunkHashes()
+{
+    if (stage == UNDEFINED && empty())
+        stage = DEFINE_SOURCE_WITH_HASHES;
+
+    if (stage != DEFINE_SOURCE_WITH_HASHES)
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "token is in wrong stage {}, token {}", stage, debugToken());
+
+    stage = DEFINED;
+}
+
+void TokenInfo::setUserToken(const String & token)
+{
+    if (stage == UNDEFINED && empty())
+        stage = DEFINE_SOURCE_USER_TOKEN;
+
+    if (stage != DEFINE_SOURCE_USER_TOKEN)
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "token is in wrong stage {}, token {}", stage, debugToken());
+
+    addTokenPart(fmt::format("user-token-{}", token));
+}
+
+void TokenInfo::setSourceWithUserToken(size_t block_number)
+{
+    if (stage != DEFINE_SOURCE_USER_TOKEN)
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "token is in wrong stage {}, token {}", stage, debugToken());
+
+    addTokenPart(fmt::format("source-number-{}", block_number));
+
+    stage = DEFINED;
+}
+
+void TokenInfo::setViewID(const String & id)
+{
+    if (stage == DEFINED)
+        stage = DEFINE_VIEW;
+
+    if (stage != DEFINE_VIEW)
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "token is in wrong stage {}, token {}", stage, debugToken());
+
+    addTokenPart(fmt::format("view-id-{}", id));
+}
+
+void TokenInfo::setViewBlockNumber(size_t block_number)
+{
+    if (stage != DEFINE_VIEW)
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "token is in wrong stage {}, token {}", stage, debugToken());
+
+    addTokenPart(fmt::format("view-block-{}", block_number));
+
+    stage = DEFINED;
+}
+
+void TokenInfo::reset()
+{
+    stage = UNDEFINED;
+    parts.clear();
+}
+
+void TokenInfo::addTokenPart(String part)
+{
+    parts.push_back(std::move(part));
+}
+
+size_t TokenInfo::getTotalSize() const
+{
+    if (parts.empty())
+        return 0;
+
+    size_t size = 0;
+    for (const auto & part : parts)
+        size += part.size();
+
+    // we reserve more size here to be able to add delimenter between parts.
+    return size + parts.size() - 1;
+}
+
+#ifdef ABORT_ON_LOGICAL_ERROR
+void CheckTokenTransform::transform(Chunk & chunk)
+{
+    auto token_info = chunk.getChunkInfos().get<TokenInfo>();
+
+    if (!token_info)
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "Chunk has to have DedupTokenInfo as ChunkInfo, {}", debug);
+
+    LOG_DEBUG(log, "debug: {}, token: {}", debug, token_info->debugToken());
+}
+#endif
+
+String DefineSourceWithChunkHashTransform::getChunkHash(const Chunk & chunk)
+{
+    SipHash hash;
+    for (const auto & colunm : chunk.getColumns())
+        colunm->updateHashFast(hash);
+
+    const auto hash_value = hash.get128();
+    return toString(hash_value.items[0]) + "_" + toString(hash_value.items[1]);
+}
+
+
+void DefineSourceWithChunkHashTransform::transform(Chunk & chunk)
+{
+    auto token_info = chunk.getChunkInfos().get<TokenInfo>();
+
+    if (!token_info)
+        throw Exception(
+            ErrorCodes::LOGICAL_ERROR,
+            "TokenInfo is expected for consumed chunk in DefineSourceWithChunkHashesTransform");
+
+    if (token_info->isDefined())
+        return;
+
+    token_info->addChunkHash(getChunkHash(chunk));
+    token_info->finishChunkHashes();
+}
+
+void SetUserTokenTransform::transform(Chunk & chunk)
+{
+    auto token_info = chunk.getChunkInfos().get<TokenInfo>();
+    if (!token_info)
+        throw Exception(
+            ErrorCodes::LOGICAL_ERROR,
+            "TokenInfo is expected for consumed chunk in SetUserTokenTransform");
+    token_info->setUserToken(user_token);
+}
+
+void SetSourceBlockNumberTransform::transform(Chunk & chunk)
+{
+    auto token_info = chunk.getChunkInfos().get<TokenInfo>();
+    if (!token_info)
+        throw Exception(
+            ErrorCodes::LOGICAL_ERROR,
+            "TokenInfo is expected for consumed chunk in SetSourceBlockNumberTransform");
+    token_info->setSourceWithUserToken(block_number++);
+}
+
+void SetViewIDTransform::transform(Chunk & chunk)
+{
+    auto token_info = chunk.getChunkInfos().get<TokenInfo>();
+    if (!token_info)
+        throw Exception(
+            ErrorCodes::LOGICAL_ERROR,
+            "TokenInfo is expected for consumed chunk in SetViewIDTransform");
+    token_info->setViewID(view_id);
+}
+
+void SetViewBlockNumberTransform::transform(Chunk & chunk)
+{
+    auto token_info = chunk.getChunkInfos().get<TokenInfo>();
+    if (!token_info)
+        throw Exception(
+            ErrorCodes::LOGICAL_ERROR,
+            "TokenInfo is expected for consumed chunk in SetViewBlockNumberTransform");
+    token_info->setViewBlockNumber(block_number++);
+}
+
+void ResetTokenTransform::transform(Chunk & chunk)
+{
+    auto token_info = chunk.getChunkInfos().get<TokenInfo>();
+    if (!token_info)
+        throw Exception(
+            ErrorCodes::LOGICAL_ERROR,
+            "TokenInfo is expected for consumed chunk in ResetTokenTransform");
+
+    token_info->reset();
+}
+
+}
+}
diff --git a/src/Processors/Transforms/DeduplicationTokenTransforms.h b/src/Processors/Transforms/DeduplicationTokenTransforms.h
new file mode 100644
index 00000000000..d6aff9e1370
--- /dev/null
+++ b/src/Processors/Transforms/DeduplicationTokenTransforms.h
@@ -0,0 +1,237 @@
+#pragma once
+
+#include <Processors/Chunk.h>
+#include <Processors/ISimpleTransform.h>
+
+#include <base/defines.h>
+#include "Common/Logger.h"
+
+
+namespace DB
+{
+    class RestoreChunkInfosTransform : public ISimpleTransform
+    {
+    public:
+        RestoreChunkInfosTransform(Chunk::ChunkInfoCollection chunk_infos_, const Block & header_)
+                : ISimpleTransform(header_, header_, true)
+                , chunk_infos(std::move(chunk_infos_))
+        {}
+
+        String getName() const override { return "RestoreChunkInfosTransform"; }
+
+        void transform(Chunk & chunk) override;
+
+    private:
+        Chunk::ChunkInfoCollection chunk_infos;
+    };
+
+
+namespace DeduplicationToken
+{
+    class TokenInfo : public ChunkInfoCloneable<TokenInfo>
+    {
+    public:
+        TokenInfo() = default;
+        TokenInfo(const TokenInfo & other) = default;
+
+        String getToken() const;
+        String debugToken() const;
+
+        bool empty() const { return parts.empty(); }
+
+        bool isDefined() const { return stage == DEFINED; }
+
+        void addChunkHash(String part);
+        void finishChunkHashes();
+
+        void setUserToken(const String & token);
+        void setSourceWithUserToken(size_t block_number);
+
+        void setViewID(const String & id);
+        void setViewBlockNumber(size_t block_number);
+
+        void reset();
+
+    private:
+        String getTokenImpl() const;
+
+        void addTokenPart(String part);
+        size_t getTotalSize() const;
+
+        /* Token has to be prepared in a particular order.
+        * BuildingStage ensures that token is expanded according the following order.
+        * Firstly token is expanded with information about the source.
+        * It could be done with two ways: add several hash sums from the source chunks or provide user defined deduplication token and its sequentional block number.
+        *
+        * transition // method
+        * UNDEFINED -> DEFINE_SOURCE_WITH_HASHES // addChunkHash
+        * DEFINE_SOURCE_WITH_HASHES -> DEFINE_SOURCE_WITH_HASHES // addChunkHash
+        * DEFINE_SOURCE_WITH_HASHES -> DEFINED // defineSourceWithChankHashes
+        *
+        * transition // method
+        * UNDEFINED -> DEFINE_SOURCE_USER_TOKEN // setUserToken
+        * DEFINE_SOURCE_USER_TOKEN -> DEFINED // defineSourceWithUserToken
+        *
+        * After token is defined, it could be extended with view id and view block number. Actually it has to be expanded with view details if there is one or several views.
+        *
+        * transition // method
+        * DEFINED -> DEFINE_VIEW // setViewID
+        * DEFINE_VIEW -> DEFINED // defineViewID
+        */
+
+        enum BuildingStage
+        {
+            UNDEFINED,
+            DEFINE_SOURCE_WITH_HASHES,
+            DEFINE_SOURCE_USER_TOKEN,
+            DEFINE_VIEW,
+            DEFINED,
+        };
+
+        BuildingStage stage = UNDEFINED;
+        std::vector<String> parts;
+    };
+
+
+#ifdef ABORT_ON_LOGICAL_ERROR
+    /// use that class only with debug builds in CI for introspection
+    class CheckTokenTransform : public ISimpleTransform
+    {
+    public:
+        CheckTokenTransform(String debug_, const Block & header_)
+            : ISimpleTransform(header_, header_, true)
+            , debug(std::move(debug_))
+        {
+        }
+
+        String getName() const override { return "DeduplicationToken::CheckTokenTransform"; }
+
+        void transform(Chunk & chunk) override;
+
+    private:
+        String debug;
+        LoggerPtr log = getLogger("CheckInsertDeduplicationTokenTransform");
+    };
+#endif
+
+
+    class AddTokenInfoTransform : public ISimpleTransform
+    {
+    public:
+        explicit AddTokenInfoTransform(const Block & header_)
+            : ISimpleTransform(header_, header_, true)
+        {
+        }
+
+        String getName() const override { return "DeduplicationToken::AddTokenInfoTransform"; }
+
+        void transform(Chunk & chunk) override
+        {
+            chunk.getChunkInfos().add(std::make_shared<TokenInfo>());
+        }
+    };
+
+
+    class DefineSourceWithChunkHashTransform : public ISimpleTransform
+    {
+    public:
+        explicit DefineSourceWithChunkHashTransform(const Block & header_)
+            : ISimpleTransform(header_, header_, true)
+        {
+        }
+
+        String getName() const override { return "DeduplicationToken::DefineSourceWithChunkHashesTransform"; }
+
+        // Usually MergeTreeSink/ReplicatedMergeTreeSink calls addChunkHash for the deduplication token with hashes from the parts.
+        // But if there is some table with different engine, we still need to define the source of the data in deduplication token
+        // We use that transform to define the source as a hash of entire block in deduplication token
+        void transform(Chunk & chunk) override;
+
+        static String getChunkHash(const Chunk & chunk);
+    };
+
+    class ResetTokenTransform : public ISimpleTransform
+    {
+    public:
+        explicit ResetTokenTransform(const Block & header_)
+            : ISimpleTransform(header_, header_, true)
+        {
+        }
+
+        String getName() const override { return "DeduplicationToken::ResetTokenTransform"; }
+
+        void transform(Chunk & chunk) override;
+    };
+
+
+    class SetUserTokenTransform : public ISimpleTransform
+    {
+    public:
+        SetUserTokenTransform(String user_token_, const Block & header_)
+            : ISimpleTransform(header_, header_, true)
+            , user_token(std::move(user_token_))
+        {
+        }
+
+        String getName() const override { return "DeduplicationToken::SetUserTokenTransform"; }
+
+        void transform(Chunk & chunk) override;
+
+    private:
+        String user_token;
+    };
+
+
+    class SetSourceBlockNumberTransform : public ISimpleTransform
+    {
+    public:
+        explicit SetSourceBlockNumberTransform(const Block & header_)
+            : ISimpleTransform(header_, header_, true)
+        {
+        }
+
+        String getName() const override { return "DeduplicationToken::SetSourceBlockNumberTransform"; }
+
+        void transform(Chunk & chunk) override;
+
+    private:
+        size_t block_number = 0;
+    };
+
+
+    class SetViewIDTransform : public ISimpleTransform
+    {
+    public:
+        SetViewIDTransform(String view_id_, const Block & header_)
+            : ISimpleTransform(header_, header_, true)
+            , view_id(std::move(view_id_))
+        {
+        }
+
+        String getName() const override { return "DeduplicationToken::SetViewIDTransform"; }
+
+        void transform(Chunk & chunk) override;
+
+    private:
+        String view_id;
+    };
+
+
+    class SetViewBlockNumberTransform : public ISimpleTransform
+    {
+    public:
+        explicit SetViewBlockNumberTransform(const Block & header_)
+            : ISimpleTransform(header_, header_, true)
+        {
+        }
+
+        String getName() const override { return "DeduplicationToken::SetViewBlockNumberTransform"; }
+
+        void transform(Chunk & chunk) override;
+
+    private:
+        size_t block_number = 0;
+    };
+
+}
+}
diff --git a/src/Processors/Transforms/ExpressionTransform.cpp b/src/Processors/Transforms/ExpressionTransform.cpp
index 2fbd2c21b8d..04fabc9a3c6 100644
--- a/src/Processors/Transforms/ExpressionTransform.cpp
+++ b/src/Processors/Transforms/ExpressionTransform.cpp
@@ -1,5 +1,7 @@
 #include <Processors/Transforms/ExpressionTransform.h>
 #include <Interpreters/ExpressionActions.h>
+
+
 namespace DB
 {
 
diff --git a/src/Processors/Transforms/JoiningTransform.cpp b/src/Processors/Transforms/JoiningTransform.cpp
index 3e2a9462e54..ca204bcb482 100644
--- a/src/Processors/Transforms/JoiningTransform.cpp
+++ b/src/Processors/Transforms/JoiningTransform.cpp
@@ -365,10 +365,9 @@ IProcessor::Status DelayedJoinedBlocksWorkerTransform::prepare()
             return Status::Finished;
         }
 
-        if (!data.chunk.hasChunkInfo())
+        task = data.chunk.getChunkInfos().get<DelayedBlocksTask>();
+        if (!task)
             throw Exception(ErrorCodes::LOGICAL_ERROR, "DelayedJoinedBlocksWorkerTransform must have chunk info");
-
-        task = std::dynamic_pointer_cast<const DelayedBlocksTask>(data.chunk.getChunkInfo());
     }
     else
     {
@@ -479,7 +478,7 @@ IProcessor::Status DelayedJoinedBlocksTransform::prepare()
             if (output.isFinished())
                 continue;
             Chunk chunk;
-            chunk.setChunkInfo(std::make_shared<DelayedBlocksTask>());
+            chunk.getChunkInfos().add(std::make_shared<DelayedBlocksTask>());
             output.push(std::move(chunk));
             output.finish();
         }
@@ -496,7 +495,7 @@ IProcessor::Status DelayedJoinedBlocksTransform::prepare()
         {
             Chunk chunk;
             auto task = std::make_shared<DelayedBlocksTask>(delayed_blocks, left_delayed_stream_finished_counter);
-            chunk.setChunkInfo(task);
+            chunk.getChunkInfos().add(std::move(task));
             output.push(std::move(chunk));
         }
         delayed_blocks = nullptr;
diff --git a/src/Processors/Transforms/JoiningTransform.h b/src/Processors/Transforms/JoiningTransform.h
index a308af03662..5f6d9d6fff2 100644
--- a/src/Processors/Transforms/JoiningTransform.h
+++ b/src/Processors/Transforms/JoiningTransform.h
@@ -1,6 +1,7 @@
 #pragma once
 #include <Processors/IProcessor.h>
-
+#include <Processors/Chunk.h>
+#include <memory>
 
 namespace DB
 {
@@ -111,11 +112,12 @@ private:
 };
 
 
-class DelayedBlocksTask : public ChunkInfo
+class DelayedBlocksTask : public ChunkInfoCloneable<DelayedBlocksTask>
 {
 public:
 
     DelayedBlocksTask() = default;
+    DelayedBlocksTask(const DelayedBlocksTask & other) = default;
     explicit DelayedBlocksTask(IBlocksStreamPtr delayed_blocks_, JoiningTransform::FinishCounterPtr left_delayed_stream_finish_counter_)
         : delayed_blocks(std::move(delayed_blocks_))
         , left_delayed_stream_finish_counter(left_delayed_stream_finish_counter_)
diff --git a/src/Processors/Transforms/MaterializingTransform.cpp b/src/Processors/Transforms/MaterializingTransform.cpp
index 1eaa5458d37..9ae80e21a68 100644
--- a/src/Processors/Transforms/MaterializingTransform.cpp
+++ b/src/Processors/Transforms/MaterializingTransform.cpp
@@ -1,6 +1,7 @@
 #include <Processors/Transforms/MaterializingTransform.h>
 #include <Columns/ColumnSparse.h>
 
+
 namespace DB
 {
 
diff --git a/src/Processors/Transforms/MemoryBoundMerging.h b/src/Processors/Transforms/MemoryBoundMerging.h
index 607087fb39c..d7bc320173b 100644
--- a/src/Processors/Transforms/MemoryBoundMerging.h
+++ b/src/Processors/Transforms/MemoryBoundMerging.h
@@ -150,11 +150,7 @@ private:
         if (!chunk.hasRows())
             return;
 
-        const auto & info = chunk.getChunkInfo();
-        if (!info)
-            throw Exception(ErrorCodes::LOGICAL_ERROR, "Chunk info was not set for chunk in SortingAggregatedForMemoryBoundMergingTransform.");
-
-        const auto * agg_info = typeid_cast<const AggregatedChunkInfo *>(info.get());
+        const auto & agg_info = chunk.getChunkInfos().get<AggregatedChunkInfo>();
         if (!agg_info)
             throw Exception(
                 ErrorCodes::LOGICAL_ERROR, "Chunk should have AggregatedChunkInfo in SortingAggregatedForMemoryBoundMergingTransform.");
diff --git a/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.cpp b/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.cpp
index fc40c6894bb..ea9ebb0f96e 100644
--- a/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.cpp
+++ b/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.cpp
@@ -30,10 +30,10 @@ void GroupingAggregatedTransform::pushData(Chunks chunks, Int32 bucket, bool is_
     auto info = std::make_shared<ChunksToMerge>();
     info->bucket_num = bucket;
     info->is_overflows = is_overflows;
-    info->chunks = std::make_unique<Chunks>(std::move(chunks));
+    info->chunks = std::make_shared<Chunks>(std::move(chunks));
 
     Chunk chunk;
-    chunk.setChunkInfo(std::move(info));
+    chunk.getChunkInfos().add(std::move(info));
     output.push(std::move(chunk));
 }
 
@@ -255,11 +255,10 @@ void GroupingAggregatedTransform::addChunk(Chunk chunk, size_t input)
     if (!chunk.hasRows())
         return;
 
-    const auto & info = chunk.getChunkInfo();
-    if (!info)
+    if (chunk.getChunkInfos().empty())
         throw Exception(ErrorCodes::LOGICAL_ERROR, "Chunk info was not set for chunk in GroupingAggregatedTransform.");
 
-    if (const auto * agg_info = typeid_cast<const AggregatedChunkInfo *>(info.get()))
+    if (auto agg_info = chunk.getChunkInfos().get<AggregatedChunkInfo>())
     {
         Int32 bucket = agg_info->bucket_num;
         bool is_overflows = agg_info->is_overflows;
@@ -275,7 +274,7 @@ void GroupingAggregatedTransform::addChunk(Chunk chunk, size_t input)
             last_bucket_number[input] = bucket;
         }
     }
-    else if (typeid_cast<const ChunkInfoWithAllocatedBytes *>(info.get()))
+    else if (chunk.getChunkInfos().get<ChunkInfoWithAllocatedBytes>())
     {
         single_level_chunks.emplace_back(std::move(chunk));
     }
@@ -304,7 +303,11 @@ void GroupingAggregatedTransform::work()
             Int32 bucket = cur_block.info.bucket_num;
             auto chunk_info = std::make_shared<AggregatedChunkInfo>();
             chunk_info->bucket_num = bucket;
-            chunks_map[bucket].emplace_back(Chunk(cur_block.getColumns(), cur_block.rows(), std::move(chunk_info)));
+
+            auto chunk = Chunk(cur_block.getColumns(), cur_block.rows());
+            chunk.getChunkInfos().add(std::move(chunk_info));
+
+            chunks_map[bucket].emplace_back(std::move(chunk));
         }
     }
 }
@@ -319,9 +322,7 @@ MergingAggregatedBucketTransform::MergingAggregatedBucketTransform(
 
 void MergingAggregatedBucketTransform::transform(Chunk & chunk)
 {
-    const auto & info = chunk.getChunkInfo();
-    const auto * chunks_to_merge = typeid_cast<const ChunksToMerge *>(info.get());
-
+    auto chunks_to_merge = chunk.getChunkInfos().get<ChunksToMerge>();
     if (!chunks_to_merge)
         throw Exception(ErrorCodes::LOGICAL_ERROR, "MergingAggregatedSimpleTransform chunk must have ChunkInfo with type ChunksToMerge.");
 
@@ -330,11 +331,10 @@ void MergingAggregatedBucketTransform::transform(Chunk & chunk)
     BlocksList blocks_list;
     for (auto & cur_chunk : *chunks_to_merge->chunks)
     {
-        const auto & cur_info = cur_chunk.getChunkInfo();
-        if (!cur_info)
+        if (cur_chunk.getChunkInfos().empty())
             throw Exception(ErrorCodes::LOGICAL_ERROR, "Chunk info was not set for chunk in MergingAggregatedBucketTransform.");
 
-        if (const auto * agg_info = typeid_cast<const AggregatedChunkInfo *>(cur_info.get()))
+        if (auto agg_info = cur_chunk.getChunkInfos().get<AggregatedChunkInfo>())
         {
             Block block = header.cloneWithColumns(cur_chunk.detachColumns());
             block.info.is_overflows = agg_info->is_overflows;
@@ -342,7 +342,7 @@ void MergingAggregatedBucketTransform::transform(Chunk & chunk)
 
             blocks_list.emplace_back(std::move(block));
         }
-        else if (typeid_cast<const ChunkInfoWithAllocatedBytes *>(cur_info.get()))
+        else if (cur_chunk.getChunkInfos().get<ChunkInfoWithAllocatedBytes>())
         {
             Block block = header.cloneWithColumns(cur_chunk.detachColumns());
             block.info.is_overflows = false;
@@ -361,7 +361,7 @@ void MergingAggregatedBucketTransform::transform(Chunk & chunk)
     res_info->is_overflows = chunks_to_merge->is_overflows;
     res_info->bucket_num = chunks_to_merge->bucket_num;
     res_info->chunk_num = chunks_to_merge->chunk_num;
-    chunk.setChunkInfo(std::move(res_info));
+    chunk.getChunkInfos().add(std::move(res_info));
 
     auto block = params->aggregator.mergeBlocks(blocks_list, params->final, is_cancelled);
 
@@ -405,11 +405,7 @@ bool SortingAggregatedTransform::tryPushChunk()
 
 void SortingAggregatedTransform::addChunk(Chunk chunk, size_t from_input)
 {
-    const auto & info = chunk.getChunkInfo();
-    if (!info)
-        throw Exception(ErrorCodes::LOGICAL_ERROR, "Chunk info was not set for chunk in SortingAggregatedTransform.");
-
-    const auto * agg_info = typeid_cast<const AggregatedChunkInfo *>(info.get());
+    auto agg_info = chunk.getChunkInfos().get<AggregatedChunkInfo>();
     if (!agg_info)
         throw Exception(ErrorCodes::LOGICAL_ERROR,
             "Chunk should have AggregatedChunkInfo in SortingAggregatedTransform.");
diff --git a/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.h b/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.h
index 77ee3034ffc..3a3c1bd9c1e 100644
--- a/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.h
+++ b/src/Processors/Transforms/MergingAggregatedMemoryEfficientTransform.h
@@ -3,6 +3,7 @@
 #include <Core/SortDescription.h>
 #include <Common/HashTable/HashSet.h>
 #include <Interpreters/Aggregator.h>
+#include <Processors/Chunk.h>
 #include <Processors/IProcessor.h>
 #include <Processors/ISimpleTransform.h>
 #include <Processors/ResizeProcessor.h>
@@ -142,9 +143,9 @@ private:
     void addChunk(Chunk chunk, size_t from_input);
 };
 
-struct ChunksToMerge : public ChunkInfo
+struct ChunksToMerge : public ChunkInfoCloneable<ChunksToMerge>
 {
-    std::unique_ptr<Chunks> chunks;
+    std::shared_ptr<Chunks> chunks;
     Int32 bucket_num = -1;
     bool is_overflows = false;
     UInt64 chunk_num = 0; // chunk number in order of generation, used during memory bound merging to restore chunks order
diff --git a/src/Processors/Transforms/MergingAggregatedTransform.cpp b/src/Processors/Transforms/MergingAggregatedTransform.cpp
index ad723da7527..446e60a0b81 100644
--- a/src/Processors/Transforms/MergingAggregatedTransform.cpp
+++ b/src/Processors/Transforms/MergingAggregatedTransform.cpp
@@ -32,11 +32,10 @@ void MergingAggregatedTransform::consume(Chunk chunk)
     total_input_rows += input_rows;
     ++total_input_blocks;
 
-    const auto & info = chunk.getChunkInfo();
-    if (!info)
+    if (chunk.getChunkInfos().empty())
         throw Exception(ErrorCodes::LOGICAL_ERROR, "Chunk info was not set for chunk in MergingAggregatedTransform.");
 
-    if (const auto * agg_info = typeid_cast<const AggregatedChunkInfo *>(info.get()))
+    if (auto agg_info = chunk.getChunkInfos().get<AggregatedChunkInfo>())
     {
         /** If the remote servers used a two-level aggregation method,
           * then blocks will contain information about the number of the bucket.
@@ -49,7 +48,7 @@ void MergingAggregatedTransform::consume(Chunk chunk)
 
         bucket_to_blocks[agg_info->bucket_num].emplace_back(std::move(block));
     }
-    else if (typeid_cast<const ChunkInfoWithAllocatedBytes *>(info.get()))
+    else if (chunk.getChunkInfos().get<ChunkInfoWithAllocatedBytes>())
     {
         auto block = getInputPort().getHeader().cloneWithColumns(chunk.getColumns());
         block.info.is_overflows = false;
@@ -89,7 +88,8 @@ Chunk MergingAggregatedTransform::generate()
 
     UInt64 num_rows = block.rows();
     Chunk chunk(block.getColumns(), num_rows);
-    chunk.setChunkInfo(std::move(info));
+
+    chunk.getChunkInfos().add(std::move(info));
 
     return chunk;
 }
diff --git a/src/Processors/Transforms/PlanSquashingTransform.cpp b/src/Processors/Transforms/PlanSquashingTransform.cpp
index 0f433165f14..ee4dfa6a64e 100644
--- a/src/Processors/Transforms/PlanSquashingTransform.cpp
+++ b/src/Processors/Transforms/PlanSquashingTransform.cpp
@@ -10,20 +10,20 @@ namespace ErrorCodes
 }
 
 PlanSquashingTransform::PlanSquashingTransform(
-    const Block & header, size_t min_block_size_rows, size_t min_block_size_bytes)
-    : IInflatingTransform(header, header), squashing(header, min_block_size_rows, min_block_size_bytes)
+    Block header_, size_t min_block_size_rows, size_t min_block_size_bytes)
+    : IInflatingTransform(header_, header_)
+    , squashing(header_, min_block_size_rows, min_block_size_bytes)
 {
 }
 
 void PlanSquashingTransform::consume(Chunk chunk)
 {
-    if (Chunk current_chunk = squashing.add(std::move(chunk)); current_chunk.hasChunkInfo())
-        squashed_chunk.swap(current_chunk);
+    squashed_chunk = squashing.add(std::move(chunk));
 }
 
 Chunk PlanSquashingTransform::generate()
 {
-    if (!squashed_chunk.hasChunkInfo())
+    if (!squashed_chunk)
         throw Exception(ErrorCodes::LOGICAL_ERROR, "Can't generate chunk in SimpleSquashingChunksTransform");
 
     Chunk result_chunk;
@@ -33,12 +33,11 @@ Chunk PlanSquashingTransform::generate()
 
 bool PlanSquashingTransform::canGenerate()
 {
-    return squashed_chunk.hasChunkInfo();
+    return bool(squashed_chunk);
 }
 
 Chunk PlanSquashingTransform::getRemaining()
 {
-    Chunk current_chunk = squashing.flush();
-    return current_chunk;
+    return squashing.flush();
 }
 }
diff --git a/src/Processors/Transforms/PlanSquashingTransform.h b/src/Processors/Transforms/PlanSquashingTransform.h
index 4ad2ec2d089..e6db245499e 100644
--- a/src/Processors/Transforms/PlanSquashingTransform.h
+++ b/src/Processors/Transforms/PlanSquashingTransform.h
@@ -10,7 +10,7 @@ class PlanSquashingTransform : public IInflatingTransform
 {
 public:
     PlanSquashingTransform(
-        const Block & header, size_t min_block_size_rows, size_t min_block_size_bytes);
+        Block header_, size_t min_block_size_rows, size_t min_block_size_bytes);
 
     String getName() const override { return "PlanSquashingTransform"; }
 
@@ -23,7 +23,6 @@ protected:
 private:
     Squashing squashing;
     Chunk squashed_chunk;
-    Chunk finish_chunk;
 };
 }
 
diff --git a/src/Processors/Transforms/SelectByIndicesTransform.h b/src/Processors/Transforms/SelectByIndicesTransform.h
index 480ab1a0f61..b44f5a3203e 100644
--- a/src/Processors/Transforms/SelectByIndicesTransform.h
+++ b/src/Processors/Transforms/SelectByIndicesTransform.h
@@ -26,7 +26,7 @@ public:
     void transform(Chunk & chunk) override
     {
         size_t num_rows = chunk.getNumRows();
-        const auto * select_final_indices_info = typeid_cast<const ChunkSelectFinalIndices *>(chunk.getChunkInfo().get());
+        auto select_final_indices_info = chunk.getChunkInfos().extract<ChunkSelectFinalIndices>();
 
         if (!select_final_indices_info || !select_final_indices_info->select_final_indices)
             throw Exception(ErrorCodes::LOGICAL_ERROR, "Chunk passed to SelectByIndicesTransform without indices column");
@@ -41,7 +41,6 @@ public:
 
             chunk.setColumns(std::move(columns), index_column->size());
         }
-        chunk.setChunkInfo(nullptr);
     }
 };
 
diff --git a/src/Processors/Transforms/SquashingTransform.cpp b/src/Processors/Transforms/SquashingTransform.cpp
index 34b733cde5e..1fb4433240a 100644
--- a/src/Processors/Transforms/SquashingTransform.cpp
+++ b/src/Processors/Transforms/SquashingTransform.cpp
@@ -18,9 +18,7 @@ SquashingTransform::SquashingTransform(
 
 void SquashingTransform::onConsume(Chunk chunk)
 {
-    Chunk planned_chunk = squashing.add(std::move(chunk));
-    if (planned_chunk.hasChunkInfo())
-        cur_chunk = DB::Squashing::squash(std::move(planned_chunk));
+    cur_chunk = Squashing::squash(squashing.add(std::move(chunk)));
 }
 
 SquashingTransform::GenerateResult SquashingTransform::onGenerate()
@@ -33,10 +31,7 @@ SquashingTransform::GenerateResult SquashingTransform::onGenerate()
 
 void SquashingTransform::onFinish()
 {
-    Chunk chunk = squashing.flush();
-    if (chunk.hasChunkInfo())
-        chunk = DB::Squashing::squash(std::move(chunk));
-    finish_chunk.setColumns(chunk.getColumns(), chunk.getNumRows());
+    finish_chunk = Squashing::squash(squashing.flush());
 }
 
 void SquashingTransform::work()
@@ -49,6 +44,7 @@ void SquashingTransform::work()
     }
 
     ExceptionKeepingTransform::work();
+
     if (finish_chunk)
     {
         data.chunk = std::move(finish_chunk);
@@ -67,18 +63,14 @@ void SimpleSquashingTransform::transform(Chunk & chunk)
 {
     if (!finished)
     {
-        Chunk planned_chunk = squashing.add(std::move(chunk));
-        if (planned_chunk.hasChunkInfo())
-            chunk = DB::Squashing::squash(std::move(planned_chunk));
+        chunk = Squashing::squash(squashing.add(std::move(chunk)));
     }
     else
     {
         if (chunk.hasRows())
             throw Exception(ErrorCodes::LOGICAL_ERROR, "Chunk expected to be empty, otherwise it will be lost");
 
-        chunk = squashing.flush();
-        if (chunk.hasChunkInfo())
-            chunk = DB::Squashing::squash(std::move(chunk));
+        chunk = Squashing::squash(squashing.flush());
     }
 }
 
diff --git a/src/Processors/Transforms/TotalsHavingTransform.cpp b/src/Processors/Transforms/TotalsHavingTransform.cpp
index aa86879e62c..59fceccb538 100644
--- a/src/Processors/Transforms/TotalsHavingTransform.cpp
+++ b/src/Processors/Transforms/TotalsHavingTransform.cpp
@@ -150,11 +150,7 @@ void TotalsHavingTransform::transform(Chunk & chunk)
     /// Block with values not included in `max_rows_to_group_by`. We'll postpone it.
     if (overflow_row)
     {
-        const auto & info = chunk.getChunkInfo();
-        if (!info)
-            throw Exception(ErrorCodes::LOGICAL_ERROR, "Chunk info was not set for chunk in TotalsHavingTransform.");
-
-        const auto * agg_info = typeid_cast<const AggregatedChunkInfo *>(info.get());
+        const auto & agg_info = chunk.getChunkInfos().get<AggregatedChunkInfo>();
         if (!agg_info)
             throw Exception(ErrorCodes::LOGICAL_ERROR, "Chunk should have AggregatedChunkInfo in TotalsHavingTransform.");
 
diff --git a/src/Processors/Transforms/buildPushingToViewsChain.cpp b/src/Processors/Transforms/buildPushingToViewsChain.cpp
index 25fbf13b0e7..312b333ab33 100644
--- a/src/Processors/Transforms/buildPushingToViewsChain.cpp
+++ b/src/Processors/Transforms/buildPushingToViewsChain.cpp
@@ -5,7 +5,9 @@
 #include <Interpreters/InterpreterSelectQuery.h>
 #include <Interpreters/InterpreterSelectQueryAnalyzer.h>
 #include <Parsers/ASTInsertQuery.h>
+#include <Processors/Chunk.h>
 #include <Processors/Transforms/CountingTransform.h>
+#include <Processors/Transforms/DeduplicationTokenTransforms.h>
 #include <Processors/Transforms/PlanSquashingTransform.h>
 #include <Processors/Transforms/SquashingTransform.h>
 #include <Processors/Transforms/ExpressionTransform.h>
@@ -16,6 +18,7 @@
 #include <Storages/StorageMaterializedView.h>
 #include <Storages/StorageValues.h>
 #include <QueryPipeline/QueryPipelineBuilder.h>
+#include <Common/Logger.h>
 #include <Common/Exception.h>
 #include <Common/CurrentThread.h>
 #include <Common/MemoryTracker.h>
@@ -24,9 +27,12 @@
 #include <Common/ThreadStatus.h>
 #include <Common/checkStackSize.h>
 #include <Common/logger_useful.h>
+#include "base/defines.h"
+#include <Core/Field.h>
 
 #include <atomic>
 #include <chrono>
+#include <memory>
 
 
 namespace ProfileEvents
@@ -105,7 +111,7 @@ private:
 class ExecutingInnerQueryFromViewTransform final : public ExceptionKeepingTransform
 {
 public:
-    ExecutingInnerQueryFromViewTransform(const Block & header, ViewRuntimeData & view_, ViewsDataPtr views_data_);
+    ExecutingInnerQueryFromViewTransform(const Block & header, ViewRuntimeData & view_, ViewsDataPtr views_data_, bool disable_deduplication_for_children_);
 
     String getName() const override { return "ExecutingInnerQueryFromView"; }
 
@@ -116,6 +122,7 @@ protected:
 private:
     ViewsDataPtr views_data;
     ViewRuntimeData & view;
+    bool disable_deduplication_for_children;
 
     struct State
     {
@@ -138,7 +145,7 @@ class PushingToLiveViewSink final : public SinkToStorage
 public:
     PushingToLiveViewSink(const Block & header, StorageLiveView & live_view_, StoragePtr storage_holder_, ContextPtr context_);
     String getName() const override { return "PushingToLiveViewSink"; }
-    void consume(Chunk chunk) override;
+    void consume(Chunk & chunk) override;
 
 private:
     StorageLiveView & live_view;
@@ -152,7 +159,7 @@ class PushingToWindowViewSink final : public SinkToStorage
 public:
     PushingToWindowViewSink(const Block & header, StorageWindowView & window_view_, StoragePtr storage_holder_, ContextPtr context_);
     String getName() const override { return "PushingToWindowViewSink"; }
-    void consume(Chunk chunk) override;
+    void consume(Chunk & chunk) override;
 
 private:
     StorageWindowView & window_view;
@@ -216,45 +223,10 @@ std::optional<Chain> generateViewChain(
 
     const auto & insert_settings = insert_context->getSettingsRef();
 
-    // Do not deduplicate insertions into MV if the main insertion is Ok
     if (disable_deduplication_for_children)
     {
         insert_context->setSetting("insert_deduplicate", Field{false});
     }
-    else if (insert_settings.update_insert_deduplication_token_in_dependent_materialized_views &&
-        !insert_settings.insert_deduplication_token.value.empty())
-    {
-        /** Update deduplication token passed to dependent MV with current view id. So it is possible to properly handle
-              * deduplication in complex INSERT flows.
-              *
-              * Example:
-              *
-              * landing -┬--> mv_1_1 ---> ds_1_1 ---> mv_2_1 --┬-> ds_2_1 ---> mv_3_1 ---> ds_3_1
-              *          |                                     |
-              *          └--> mv_1_2 ---> ds_1_2 ---> mv_2_2 --┘
-              *
-              * Here we want to avoid deduplication for two different blocks generated from `mv_2_1` and `mv_2_2` that will
-              * be inserted into `ds_2_1`.
-              *
-              * We are forced to use view id instead of table id because there are some possible INSERT flows where no tables
-              * are involved.
-              *
-              * Example:
-              *
-              * landing -┬--> mv_1_1 --┬-> ds_1_1
-              *          |             |
-              *          └--> mv_1_2 --┘
-              *
-              */
-        auto insert_deduplication_token = insert_settings.insert_deduplication_token.value;
-
-        if (view_id.hasUUID())
-            insert_deduplication_token += "_" + toString(view_id.uuid);
-        else
-            insert_deduplication_token += "_" + view_id.getFullNameNotQuoted();
-
-        insert_context->setSetting("insert_deduplication_token", insert_deduplication_token);
-    }
 
     // Processing of blocks for MVs is done block by block, and there will
     // be no parallel reading after (plus it is not a costless operation)
@@ -361,7 +333,13 @@ std::optional<Chain> generateViewChain(
                 insert_columns.emplace_back(column.name);
         }
 
-        InterpreterInsertQuery interpreter(nullptr, insert_context, false, false, false);
+        InterpreterInsertQuery interpreter(
+            nullptr,
+            insert_context,
+            /* allow_materialized */ false,
+            /* no_squash */ false,
+            /* no_destination */ false,
+            /* async_isnert */ false);
 
         /// TODO: remove sql_security_type check after we turn `ignore_empty_sql_security_in_create_view_query=false`
         bool check_access = !materialized_view->hasInnerTable() && materialized_view->getInMemoryMetadataPtr()->sql_security_type;
@@ -378,6 +356,10 @@ std::optional<Chain> generateViewChain(
                 table_prefers_large_blocks ? settings.min_insert_block_size_bytes : 0ULL));
         }
 
+#ifdef ABORT_ON_LOGICAL_ERROR
+        out.addSource(std::make_shared<DeduplicationToken::CheckTokenTransform>("Before squashing", out.getInputHeader()));
+#endif
+
         auto counting = std::make_shared<CountingTransform>(out.getInputHeader(), current_thread, insert_context->getQuota());
         counting->setProcessListElement(insert_context->getProcessListElement());
         counting->setProgressCallback(insert_context->getProgressCallback());
@@ -420,11 +402,19 @@ std::optional<Chain> generateViewChain(
 
     if (type == QueryViewsLogElement::ViewType::MATERIALIZED)
     {
+#ifdef ABORT_ON_LOGICAL_ERROR
+        out.addSource(std::make_shared<DeduplicationToken::CheckTokenTransform>("Right after Inner query", out.getInputHeader()));
+#endif
+
         auto executing_inner_query = std::make_shared<ExecutingInnerQueryFromViewTransform>(
-            storage_header, views_data->views.back(), views_data);
+            storage_header, views_data->views.back(), views_data, disable_deduplication_for_children);
         executing_inner_query->setRuntimeData(view_thread_status, view_counter_ms);
 
         out.addSource(std::move(executing_inner_query));
+
+#ifdef ABORT_ON_LOGICAL_ERROR
+        out.addSource(std::make_shared<DeduplicationToken::CheckTokenTransform>("Right before Inner query", out.getInputHeader()));
+#endif
     }
 
     return out;
@@ -465,11 +455,7 @@ Chain buildPushingToViewsChain(
       */
     result_chain.addTableLock(storage->lockForShare(context->getInitialQueryId(), context->getSettingsRef().lock_acquire_timeout));
 
-    /// If the "root" table deduplicates blocks, there are no need to make deduplication for children
-    /// Moreover, deduplication for AggregatingMergeTree children could produce false positives due to low size of inserting blocks
-    bool disable_deduplication_for_children = false;
-    if (!context->getSettingsRef().deduplicate_blocks_in_dependent_materialized_views)
-        disable_deduplication_for_children = !no_destination && storage->supportsDeduplication();
+    bool disable_deduplication_for_children = !context->getSettingsRef().deduplicate_blocks_in_dependent_materialized_views;
 
     auto table_id = storage->getStorageID();
     auto views = DatabaseCatalog::instance().getDependentViews(table_id);
@@ -560,12 +546,25 @@ Chain buildPushingToViewsChain(
         auto sink = std::make_shared<PushingToLiveViewSink>(live_view_header, *live_view, storage, context);
         sink->setRuntimeData(thread_status, elapsed_counter_ms);
         result_chain.addSource(std::move(sink));
+
+        result_chain.addSource(std::make_shared<DeduplicationToken::DefineSourceWithChunkHashTransform>(result_chain.getInputHeader()));
     }
     else if (auto * window_view = dynamic_cast<StorageWindowView *>(storage.get()))
     {
         auto sink = std::make_shared<PushingToWindowViewSink>(window_view->getInputHeader(), *window_view, storage, context);
         sink->setRuntimeData(thread_status, elapsed_counter_ms);
         result_chain.addSource(std::move(sink));
+
+        result_chain.addSource(std::make_shared<DeduplicationToken::DefineSourceWithChunkHashTransform>(result_chain.getInputHeader()));
+    }
+    else if (dynamic_cast<StorageMaterializedView *>(storage.get()))
+    {
+        auto sink = storage->write(query_ptr, metadata_snapshot, context, async_insert);
+        metadata_snapshot->check(sink->getHeader().getColumnsWithTypeAndName());
+        sink->setRuntimeData(thread_status, elapsed_counter_ms);
+        result_chain.addSource(std::move(sink));
+
+        result_chain.addSource(std::make_shared<DeduplicationToken::DefineSourceWithChunkHashTransform>(result_chain.getInputHeader()));
     }
     /// Do not push to destination table if the flag is set
     else if (!no_destination)
@@ -573,8 +572,15 @@ Chain buildPushingToViewsChain(
         auto sink = storage->write(query_ptr, metadata_snapshot, context, async_insert);
         metadata_snapshot->check(sink->getHeader().getColumnsWithTypeAndName());
         sink->setRuntimeData(thread_status, elapsed_counter_ms);
+
+        result_chain.addSource(std::make_shared<DeduplicationToken::DefineSourceWithChunkHashTransform>(sink->getHeader()));
+
         result_chain.addSource(std::move(sink));
     }
+    else
+    {
+        result_chain.addSource(std::make_shared<DeduplicationToken::DefineSourceWithChunkHashTransform>(storage_header));
+    }
 
     if (result_chain.empty())
         result_chain.addSink(std::make_shared<NullSinkToStorage>(storage_header));
@@ -590,7 +596,7 @@ Chain buildPushingToViewsChain(
     return result_chain;
 }
 
-static QueryPipeline process(Block block, ViewRuntimeData & view, const ViewsData & views_data)
+static QueryPipeline process(Block block, ViewRuntimeData & view, const ViewsData & views_data, Chunk::ChunkInfoCollection && chunk_infos, bool disable_deduplication_for_children)
 {
     const auto & context = view.context;
 
@@ -637,6 +643,19 @@ static QueryPipeline process(Block block, ViewRuntimeData & view, const ViewsDat
         pipeline.getHeader(),
         std::make_shared<ExpressionActions>(std::move(converting))));
 
+    pipeline.addTransform(std::make_shared<RestoreChunkInfosTransform>(std::move(chunk_infos), pipeline.getHeader()));
+
+    if (!disable_deduplication_for_children)
+    {
+        String materialize_view_id = view.table_id.hasUUID() ? toString(view.table_id.uuid) : view.table_id.getFullNameNotQuoted();
+        pipeline.addTransform(std::make_shared<DeduplicationToken::SetViewIDTransform>(std::move(materialize_view_id), pipeline.getHeader()));
+        pipeline.addTransform(std::make_shared<DeduplicationToken::SetViewBlockNumberTransform>(pipeline.getHeader()));
+    }
+    else
+    {
+        pipeline.addTransform(std::make_shared<DeduplicationToken::ResetTokenTransform>(pipeline.getHeader()));
+    }
+
     return QueryPipelineBuilder::getPipeline(std::move(pipeline));
 }
 
@@ -728,17 +747,19 @@ IProcessor::Status CopyingDataToViewsTransform::prepare()
 ExecutingInnerQueryFromViewTransform::ExecutingInnerQueryFromViewTransform(
     const Block & header,
     ViewRuntimeData & view_,
-    std::shared_ptr<ViewsData> views_data_)
+    std::shared_ptr<ViewsData> views_data_,
+    bool disable_deduplication_for_children_)
     : ExceptionKeepingTransform(header, view_.sample_block)
     , views_data(std::move(views_data_))
     , view(view_)
+    , disable_deduplication_for_children(disable_deduplication_for_children_)
 {
 }
 
 void ExecutingInnerQueryFromViewTransform::onConsume(Chunk chunk)
 {
-    auto block = getInputPort().getHeader().cloneWithColumns(chunk.getColumns());
-    state.emplace(process(block, view, *views_data));
+    auto block = getInputPort().getHeader().cloneWithColumns(chunk.detachColumns());
+    state.emplace(process(std::move(block), view, *views_data, std::move(chunk.getChunkInfos()), disable_deduplication_for_children));
 }
 
 
@@ -770,10 +791,10 @@ PushingToLiveViewSink::PushingToLiveViewSink(const Block & header, StorageLiveVi
 {
 }
 
-void PushingToLiveViewSink::consume(Chunk chunk)
+void PushingToLiveViewSink::consume(Chunk & chunk)
 {
     Progress local_progress(chunk.getNumRows(), chunk.bytes(), 0);
-    live_view.writeBlock(getHeader().cloneWithColumns(chunk.detachColumns()), context);
+    live_view.writeBlock(live_view, getHeader().cloneWithColumns(chunk.detachColumns()), std::move(chunk.getChunkInfos()), context);
 
     if (auto process = context->getProcessListElement())
         process->updateProgressIn(local_progress);
@@ -793,11 +814,11 @@ PushingToWindowViewSink::PushingToWindowViewSink(
 {
 }
 
-void PushingToWindowViewSink::consume(Chunk chunk)
+void PushingToWindowViewSink::consume(Chunk & chunk)
 {
     Progress local_progress(chunk.getNumRows(), chunk.bytes(), 0);
     StorageWindowView::writeIntoWindowView(
-        window_view, getHeader().cloneWithColumns(chunk.detachColumns()), context);
+        window_view, getHeader().cloneWithColumns(chunk.detachColumns()), std::move(chunk.getChunkInfos()), context);
 
     if (auto process = context->getProcessListElement())
         process->updateProgressIn(local_progress);
diff --git a/src/QueryPipeline/QueryPipelineBuilder.h b/src/QueryPipeline/QueryPipelineBuilder.h
index f0b2ead687e..a9e5b1535c0 100644
--- a/src/QueryPipeline/QueryPipelineBuilder.h
+++ b/src/QueryPipeline/QueryPipelineBuilder.h
@@ -193,7 +193,7 @@ public:
         return concurrency_control;
     }
 
-    void addResources(QueryPlanResourceHolder resources_) { resources = std::move(resources_); }
+    void addResources(QueryPlanResourceHolder resources_) { resources.append(std::move(resources_)); }
     void setQueryIdHolder(std::shared_ptr<QueryIdHolder> query_id_holder) { resources.query_id_holders.emplace_back(std::move(query_id_holder)); }
     void addContext(ContextPtr context) { resources.interpreter_context.emplace_back(std::move(context)); }
 
diff --git a/src/QueryPipeline/QueryPlanResourceHolder.cpp b/src/QueryPipeline/QueryPlanResourceHolder.cpp
index 2cd4dc42a83..bb2be2c8ffb 100644
--- a/src/QueryPipeline/QueryPlanResourceHolder.cpp
+++ b/src/QueryPipeline/QueryPlanResourceHolder.cpp
@@ -5,7 +5,7 @@
 namespace DB
 {
 
-QueryPlanResourceHolder & QueryPlanResourceHolder::operator=(QueryPlanResourceHolder && rhs) noexcept
+QueryPlanResourceHolder & QueryPlanResourceHolder::append(QueryPlanResourceHolder && rhs) noexcept
 {
     table_locks.insert(table_locks.end(), rhs.table_locks.begin(), rhs.table_locks.end());
     storage_holders.insert(storage_holders.end(), rhs.storage_holders.begin(), rhs.storage_holders.end());
@@ -16,6 +16,12 @@ QueryPlanResourceHolder & QueryPlanResourceHolder::operator=(QueryPlanResourceHo
     return *this;
 }
 
+QueryPlanResourceHolder & QueryPlanResourceHolder::operator=(QueryPlanResourceHolder && rhs) noexcept
+{
+    append(std::move(rhs));
+    return *this;
+}
+
 QueryPlanResourceHolder::QueryPlanResourceHolder() = default;
 QueryPlanResourceHolder::QueryPlanResourceHolder(QueryPlanResourceHolder &&) noexcept = default;
 QueryPlanResourceHolder::~QueryPlanResourceHolder() = default;
diff --git a/src/QueryPipeline/QueryPlanResourceHolder.h b/src/QueryPipeline/QueryPlanResourceHolder.h
index ed9eb68b7ba..10f7f39ab09 100644
--- a/src/QueryPipeline/QueryPlanResourceHolder.h
+++ b/src/QueryPipeline/QueryPlanResourceHolder.h
@@ -20,8 +20,11 @@ struct QueryPlanResourceHolder
     QueryPlanResourceHolder(QueryPlanResourceHolder &&) noexcept;
     ~QueryPlanResourceHolder();
 
+    QueryPlanResourceHolder & operator=(QueryPlanResourceHolder &) = delete;
+
     /// Custom move assignment does not destroy data from lhs. It appends data from rhs to lhs.
     QueryPlanResourceHolder & operator=(QueryPlanResourceHolder &&) noexcept;
+    QueryPlanResourceHolder & append(QueryPlanResourceHolder &&) noexcept;
 
     /// Some processors may implicitly use Context or temporary Storage created by Interpreter.
     /// But lifetime of Streams is not nested in lifetime of Interpreters, so we have to store it here,
diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp
index ac1423f87c1..fccea9e258e 100644
--- a/src/Server/TCPHandler.cpp
+++ b/src/Server/TCPHandler.cpp
@@ -888,12 +888,11 @@ AsynchronousInsertQueue::PushResult TCPHandler::processAsyncInsertQuery(Asynchro
 
     while (readDataNext())
     {
-        squashing.header = state.block_for_insert;
-        auto planned_chunk = squashing.add({state.block_for_insert.getColumns(), state.block_for_insert.rows()});
-        if (planned_chunk.hasChunkInfo())
+        squashing.setHeader(state.block_for_insert.cloneEmpty());
+        auto result_chunk = Squashing::squash(squashing.add({state.block_for_insert.getColumns(), state.block_for_insert.rows()}));
+        if (result_chunk)
         {
-            Chunk result_chunk = DB::Squashing::squash(std::move(planned_chunk));
-            auto result = state.block_for_insert.cloneWithColumns(result_chunk.getColumns());
+            auto result = squashing.getHeader().cloneWithColumns(result_chunk.detachColumns());
             return PushResult
             {
                 .status = PushResult::TOO_MUCH_DATA,
@@ -902,12 +901,13 @@ AsynchronousInsertQueue::PushResult TCPHandler::processAsyncInsertQuery(Asynchro
         }
     }
 
-    auto planned_chunk = squashing.flush();
-    Chunk result_chunk;
-    if (planned_chunk.hasChunkInfo())
-        result_chunk = DB::Squashing::squash(std::move(planned_chunk));
+    Chunk result_chunk = Squashing::squash(squashing.flush());
+    if (!result_chunk)
+    {
+        return insert_queue.pushQueryWithBlock(state.parsed_query, squashing.getHeader(), query_context);
+    }
 
-    auto result = squashing.header.cloneWithColumns(result_chunk.getColumns());
+    auto result = squashing.getHeader().cloneWithColumns(result_chunk.detachColumns());
     return insert_queue.pushQueryWithBlock(state.parsed_query, std::move(result), query_context);
 }
 
diff --git a/src/Storages/Distributed/DistributedSink.cpp b/src/Storages/Distributed/DistributedSink.cpp
index e556bda2561..8791668cd89 100644
--- a/src/Storages/Distributed/DistributedSink.cpp
+++ b/src/Storages/Distributed/DistributedSink.cpp
@@ -134,7 +134,7 @@ DistributedSink::DistributedSink(
 }
 
 
-void DistributedSink::consume(Chunk chunk)
+void DistributedSink::consume(Chunk & chunk)
 {
     if (is_first_chunk)
     {
@@ -142,7 +142,7 @@ void DistributedSink::consume(Chunk chunk)
         is_first_chunk = false;
     }
 
-    auto ordinary_block = getHeader().cloneWithColumns(chunk.detachColumns());
+    auto ordinary_block = getHeader().cloneWithColumns(chunk.getColumns());
 
     if (insert_sync)
         writeSync(ordinary_block);
@@ -420,7 +420,13 @@ DistributedSink::runWritingJob(JobReplica & job, const Block & current_block, si
                 /// to resolve tables (in InterpreterInsertQuery::getTable())
                 auto copy_query_ast = query_ast->clone();
 
-                InterpreterInsertQuery interp(copy_query_ast, job.local_context, allow_materialized);
+                InterpreterInsertQuery interp(
+                    copy_query_ast,
+                    job.local_context,
+                    allow_materialized,
+                    /* no_squash */ false,
+                    /* no_destination */ false,
+                    /* async_isnert */ false);
                 auto block_io = interp.execute();
 
                 job.pipeline = std::move(block_io.pipeline);
@@ -715,7 +721,13 @@ void DistributedSink::writeToLocal(const Cluster::ShardInfo & shard_info, const
 
     try
     {
-        InterpreterInsertQuery interp(query_ast, context, allow_materialized);
+        InterpreterInsertQuery interp(
+            query_ast,
+            context,
+            allow_materialized,
+            /* no_squash */ false,
+            /* no_destination */ false,
+            /* async_isnert */ false);
 
         auto block_io = interp.execute();
         PushingPipelineExecutor executor(block_io.pipeline);
diff --git a/src/Storages/Distributed/DistributedSink.h b/src/Storages/Distributed/DistributedSink.h
index a4c95633595..5b7396f2c6f 100644
--- a/src/Storages/Distributed/DistributedSink.h
+++ b/src/Storages/Distributed/DistributedSink.h
@@ -49,7 +49,7 @@ public:
         const Names & columns_to_send_);
 
     String getName() const override { return "DistributedSink"; }
-    void consume(Chunk chunk) override;
+    void consume(Chunk & chunk) override;
     void onFinish() override;
 
 private:
diff --git a/src/Storages/FileLog/StorageFileLog.cpp b/src/Storages/FileLog/StorageFileLog.cpp
index abd4b4ce23b..0f9bd8b6ff9 100644
--- a/src/Storages/FileLog/StorageFileLog.cpp
+++ b/src/Storages/FileLog/StorageFileLog.cpp
@@ -740,7 +740,14 @@ bool StorageFileLog::streamToViews()
 
     auto new_context = Context::createCopy(getContext());
 
-    InterpreterInsertQuery interpreter(insert, new_context, false, true, true);
+    InterpreterInsertQuery interpreter(
+        insert,
+        new_context,
+        /* allow_materialized */ false,
+        /* no_squash */ true,
+        /* no_destination */ true,
+        /* async_isnert */ false);
+
     auto block_io = interpreter.execute();
 
     /// Each stream responsible for closing it's files and store meta
diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp
index f5c5d093ce1..809401bb279 100644
--- a/src/Storages/Kafka/StorageKafka.cpp
+++ b/src/Storages/Kafka/StorageKafka.cpp
@@ -1099,7 +1099,13 @@ bool StorageKafka::streamToViews()
 
     // Create a stream for each consumer and join them in a union stream
     // Only insert into dependent views and expect that input blocks contain virtual columns
-    InterpreterInsertQuery interpreter(insert, kafka_context, false, true, true);
+    InterpreterInsertQuery interpreter(
+        insert,
+        kafka_context,
+        /* allow_materialized */ false,
+        /* no_squash */ true,
+        /* no_destination */ true,
+        /* async_isnert */ false);
     auto block_io = interpreter.execute();
 
     // Create a stream for each consumer and join them in a union stream
diff --git a/src/Storages/LiveView/LiveViewSink.h b/src/Storages/LiveView/LiveViewSink.h
index 792133ced64..9803fa0a160 100644
--- a/src/Storages/LiveView/LiveViewSink.h
+++ b/src/Storages/LiveView/LiveViewSink.h
@@ -71,9 +71,9 @@ public:
         new_hash.reset();
     }
 
-    void consume(Chunk chunk) override
+    void consume(Chunk & chunk) override
     {
-        auto block = getHeader().cloneWithColumns(chunk.detachColumns());
+        auto block = getHeader().cloneWithColumns(chunk.getColumns());
         block.updateHash(*new_hash);
         new_blocks->push_back(std::move(block));
     }
diff --git a/src/Storages/LiveView/StorageLiveView.cpp b/src/Storages/LiveView/StorageLiveView.cpp
index 57a1ea302f9..82759e8a851 100644
--- a/src/Storages/LiveView/StorageLiveView.cpp
+++ b/src/Storages/LiveView/StorageLiveView.cpp
@@ -21,6 +21,7 @@ limitations under the License. */
 #include <Processors/Transforms/MaterializingTransform.h>
 #include <Processors/Executors/PullingAsyncPipelineExecutor.h>
 #include <Processors/Executors/PipelineExecutor.h>
+#include <Processors/Transforms/DeduplicationTokenTransforms.h>
 #include <Processors/Transforms/SquashingTransform.h>
 #include <QueryPipeline/QueryPipelineBuilder.h>
 #include <QueryPipeline/QueryPlanResourceHolder.h>
@@ -330,7 +331,7 @@ Pipe StorageLiveView::watch(
     return reader;
 }
 
-void StorageLiveView::writeBlock(const Block & block, ContextPtr local_context)
+void StorageLiveView::writeBlock(StorageLiveView & live_view, Block && block, Chunk::ChunkInfoCollection && chunk_infos, ContextPtr local_context)
 {
     auto output = std::make_shared<LiveViewSink>(*this);
 
@@ -407,6 +408,21 @@ void StorageLiveView::writeBlock(const Block & block, ContextPtr local_context)
             builder = interpreter.buildQueryPipeline();
         }
 
+        builder.addSimpleTransform([&](const Block & cur_header)
+        {
+            return std::make_shared<RestoreChunkInfosTransform>(chunk_infos.clone(), cur_header);
+        });
+
+        String live_view_id = live_view.getStorageID().hasUUID() ? toString(live_view.getStorageID().uuid) : live_view.getStorageID().getFullNameNotQuoted();
+        builder.addSimpleTransform([&](const Block & stream_header)
+        {
+            return std::make_shared<DeduplicationToken::SetViewIDTransform>(live_view_id, stream_header);
+        });
+        builder.addSimpleTransform([&](const Block & stream_header)
+        {
+            return std::make_shared<DeduplicationToken::SetViewBlockNumberTransform>(stream_header);
+        });
+
         builder.addSimpleTransform([&](const Block & cur_header)
         {
             return std::make_shared<MaterializingTransform>(cur_header);
diff --git a/src/Storages/LiveView/StorageLiveView.h b/src/Storages/LiveView/StorageLiveView.h
index 91daac32c7b..12d8e898347 100644
--- a/src/Storages/LiveView/StorageLiveView.h
+++ b/src/Storages/LiveView/StorageLiveView.h
@@ -118,7 +118,7 @@ public:
         return 0;
     }
 
-    void writeBlock(const Block & block, ContextPtr context);
+    void writeBlock(StorageLiveView & live_view, Block && block, Chunk::ChunkInfoCollection && chunk_infos, ContextPtr context);
 
     void refresh();
 
diff --git a/src/Storages/MaterializedView/RefreshTask.cpp b/src/Storages/MaterializedView/RefreshTask.cpp
index bc8cb0ce69a..ff5214a5e51 100644
--- a/src/Storages/MaterializedView/RefreshTask.cpp
+++ b/src/Storages/MaterializedView/RefreshTask.cpp
@@ -377,7 +377,13 @@ void RefreshTask::executeRefreshUnlocked(std::shared_ptr<StorageMaterializedView
         {
             CurrentThread::QueryScope query_scope(refresh_context); // create a thread group for the query
 
-            BlockIO block_io = InterpreterInsertQuery(refresh_query, refresh_context).execute();
+            BlockIO block_io = InterpreterInsertQuery(
+                refresh_query,
+                refresh_context,
+                /* allow_materialized */ false,
+                /* no_squash */ false,
+                /* no_destination */ false,
+                /* async_isnert */ false).execute();
             QueryPipeline & pipeline = block_io.pipeline;
 
             pipeline.setProgressCallback([this](const Progress & prog)
diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp
index c2e0e778220..b28fe3d8fc6 100644
--- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp
+++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp
@@ -2339,21 +2339,26 @@ String IMergeTreeDataPart::getUniqueId() const
     return getDataPartStorage().getUniqueId();
 }
 
+UInt128 IMergeTreeDataPart::getPartBlockIDHash() const
+{
+    SipHash hash;
+    checksums.computeTotalChecksumDataOnly(hash);
+    return hash.get128();
+}
+
 String IMergeTreeDataPart::getZeroLevelPartBlockID(std::string_view token) const
 {
     if (info.level != 0)
         throw Exception(ErrorCodes::LOGICAL_ERROR, "Trying to get block id for non zero level part {}", name);
 
-    SipHash hash;
     if (token.empty())
     {
-        checksums.computeTotalChecksumDataOnly(hash);
-    }
-    else
-    {
-        hash.update(token.data(), token.size());
+        const auto hash_value = getPartBlockIDHash();
+        return info.partition_id + "_" + toString(hash_value.items[0]) + "_" + toString(hash_value.items[1]);
     }
 
+    SipHash hash;
+    hash.update(token.data(), token.size());
     const auto hash_value = hash.get128();
     return info.partition_id + "_" + toString(hash_value.items[0]) + "_" + toString(hash_value.items[1]);
 }
diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h
index 571f1389e10..9fd481b0d8e 100644
--- a/src/Storages/MergeTree/IMergeTreeDataPart.h
+++ b/src/Storages/MergeTree/IMergeTreeDataPart.h
@@ -210,6 +210,7 @@ public:
 
     /// Compute part block id for zero level part. Otherwise throws an exception.
     /// If token is not empty, block id is calculated based on it instead of block data
+    UInt128 getPartBlockIDHash() const;
     String getZeroLevelPartBlockID(std::string_view token) const;
 
     void setName(const String & new_name);
diff --git a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp
index 78b67de1a7e..a6298aab3d9 100644
--- a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp
+++ b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp
@@ -145,8 +145,12 @@ ChunkAndProgress MergeTreeSelectProcessor::read()
                 ordered_columns.push_back(res.block.getByName(name).column);
             }
 
+            auto chunk = Chunk(ordered_columns, res.row_count);
+            if (add_part_level)
+                chunk.getChunkInfos().add(std::make_shared<MergeTreePartLevelInfo>(task->getInfo().data_part->info.level));
+
             return ChunkAndProgress{
-                .chunk = Chunk(ordered_columns, res.row_count, add_part_level ? std::make_shared<MergeTreePartLevelInfo>(task->getInfo().data_part->info.level) : nullptr),
+                .chunk = std::move(chunk),
                 .num_read_rows = res.num_read_rows,
                 .num_read_bytes = res.num_read_bytes,
                 .is_finished = false};
diff --git a/src/Storages/MergeTree/MergeTreeSequentialSource.cpp b/src/Storages/MergeTree/MergeTreeSequentialSource.cpp
index 02f8d6f4f6a..4f90f7131da 100644
--- a/src/Storages/MergeTree/MergeTreeSequentialSource.cpp
+++ b/src/Storages/MergeTree/MergeTreeSequentialSource.cpp
@@ -264,7 +264,10 @@ try
                 ++it;
             }
 
-            return Chunk(std::move(res_columns), rows_read, add_part_level ? std::make_shared<MergeTreePartLevelInfo>(data_part->info.level) : nullptr);
+            auto result = Chunk(std::move(res_columns), rows_read);
+            if (add_part_level)
+                result.getChunkInfos().add(std::make_shared<MergeTreePartLevelInfo>(data_part->info.level));
+            return result;
         }
     }
     else
diff --git a/src/Storages/MergeTree/MergeTreeSink.cpp b/src/Storages/MergeTree/MergeTreeSink.cpp
index 05751e0fa6f..d8cfce1ca99 100644
--- a/src/Storages/MergeTree/MergeTreeSink.cpp
+++ b/src/Storages/MergeTree/MergeTreeSink.cpp
@@ -1,14 +1,27 @@
+#include <Common/Logger.h>
+#include <Common/logger_useful.h>
+#include <Common/Exception.h>
+#include <Common/ProfileEventsScope.h>
+#include <Core/Settings.h>
+#include <DataTypes/ObjectUtils.h>
+#include <Interpreters/StorageID.h>
+#include <Interpreters/PartLog.h>
+#include <Processors/Transforms/DeduplicationTokenTransforms.h>
 #include <Storages/MergeTree/MergeTreeSink.h>
 #include <Storages/StorageMergeTree.h>
-#include <Interpreters/PartLog.h>
-#include <DataTypes/ObjectUtils.h>
-#include <Common/ProfileEventsScope.h>
+
+#include <memory>
 
 namespace ProfileEvents
 {
     extern const Event DuplicatedInsertedBlocks;
 }
 
+namespace ErrorCodes
+{
+    extern const int LOGICAL_ERROR;
+}
+
 namespace DB
 {
 
@@ -58,12 +71,12 @@ void MergeTreeSink::onCancel()
 {
 }
 
-void MergeTreeSink::consume(Chunk chunk)
+void MergeTreeSink::consume(Chunk & chunk)
 {
     if (num_blocks_processed > 0)
         storage.delayInsertOrThrowIfNeeded(nullptr, context, false);
 
-    auto block = getHeader().cloneWithColumns(chunk.detachColumns());
+    auto block = getHeader().cloneWithColumns(chunk.getColumns());
     if (!storage_snapshot->object_columns.empty())
         convertDynamicColumnsToTuples(block, storage_snapshot);
 
@@ -76,6 +89,18 @@ void MergeTreeSink::consume(Chunk chunk)
     size_t streams = 0;
     bool support_parallel_write = false;
 
+    auto token_info = chunk.getChunkInfos().get<DeduplicationToken::TokenInfo>();
+    if (!token_info)
+        throw Exception(ErrorCodes::LOGICAL_ERROR,
+            "TokenInfo is expected for consumed chunk in MergeTreeSink for table: {}",
+            storage.getStorageID().getNameForLogs());
+
+    const bool need_to_define_dedup_token = !token_info->isDefined();
+
+    String block_dedup_token;
+    if (token_info->isDefined())
+        block_dedup_token = token_info->getToken();
+
     for (auto & current_block : part_blocks)
     {
         ProfileEvents::Counters part_counters;
@@ -100,22 +125,16 @@ void MergeTreeSink::consume(Chunk chunk)
         if (!temp_part.part)
             continue;
 
+        if (need_to_define_dedup_token)
+        {
+            chassert(temp_part.part);
+            const auto hash_value = temp_part.part->getPartBlockIDHash();
+            token_info->addChunkHash(toString(hash_value.items[0]) + "_" + toString(hash_value.items[1]));
+        }
+
         if (!support_parallel_write && temp_part.part->getDataPartStorage().supportParallelWrite())
             support_parallel_write = true;
 
-        String block_dedup_token;
-        if (storage.getDeduplicationLog())
-        {
-            const String & dedup_token = settings.insert_deduplication_token;
-            if (!dedup_token.empty())
-            {
-                /// multiple blocks can be inserted within the same insert query
-                /// an ordinal number is added to dedup token to generate a distinctive block id for each block
-                block_dedup_token = fmt::format("{}_{}", dedup_token, chunk_dedup_seqnum);
-                ++chunk_dedup_seqnum;
-            }
-        }
-
         size_t max_insert_delayed_streams_for_parallel_write;
 
         if (settings.max_insert_delayed_streams_for_parallel_write.changed)
@@ -127,6 +146,7 @@ void MergeTreeSink::consume(Chunk chunk)
 
         /// In case of too much columns/parts in block, flush explicitly.
         streams += temp_part.streams.size();
+
         if (streams > max_insert_delayed_streams_for_parallel_write)
         {
             finishDelayedChunk();
@@ -143,11 +163,16 @@ void MergeTreeSink::consume(Chunk chunk)
         {
             .temp_part = std::move(temp_part),
             .elapsed_ns = elapsed_ns,
-            .block_dedup_token = std::move(block_dedup_token),
+            .block_dedup_token = block_dedup_token,
             .part_counters = std::move(part_counters),
         });
     }
 
+    if (need_to_define_dedup_token)
+    {
+        token_info->finishChunkHashes();
+    }
+
     finishDelayedChunk();
     delayed_chunk = std::make_unique<MergeTreeSink::DelayedChunk>();
     delayed_chunk->partitions = std::move(partitions);
@@ -160,6 +185,8 @@ void MergeTreeSink::finishDelayedChunk()
     if (!delayed_chunk)
         return;
 
+    const Settings & settings = context->getSettingsRef();
+
     for (auto & partition : delayed_chunk->partitions)
     {
         ProfileEventsScope scoped_attach(&partition.part_counters);
@@ -178,7 +205,8 @@ void MergeTreeSink::finishDelayedChunk()
             storage.fillNewPartName(part, lock);
 
             auto * deduplication_log = storage.getDeduplicationLog();
-            if (deduplication_log)
+
+            if (settings.insert_deduplicate && deduplication_log)
             {
                 const String block_id = part->getZeroLevelPartBlockID(partition.block_dedup_token);
                 auto res = deduplication_log->addPart(block_id, part->info);
diff --git a/src/Storages/MergeTree/MergeTreeSink.h b/src/Storages/MergeTree/MergeTreeSink.h
index cf6715a3415..90976020d52 100644
--- a/src/Storages/MergeTree/MergeTreeSink.h
+++ b/src/Storages/MergeTree/MergeTreeSink.h
@@ -25,7 +25,7 @@ public:
     ~MergeTreeSink() override;
 
     String getName() const override { return "MergeTreeSink"; }
-    void consume(Chunk chunk) override;
+    void consume(Chunk & chunk) override;
     void onStart() override;
     void onFinish() override;
     void onCancel() override;
@@ -36,7 +36,6 @@ private:
     size_t max_parts_per_block;
     ContextPtr context;
     StorageSnapshotPtr storage_snapshot;
-    UInt64 chunk_dedup_seqnum = 0; /// input chunk ordinal number in case of dedup token
     UInt64 num_blocks_processed = 0;
 
     /// We can delay processing for previous chunk and start writing a new one.
diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp
index a552ee89aee..3dbcb5e5bda 100644
--- a/src/Storages/MergeTree/MutateTask.cpp
+++ b/src/Storages/MergeTree/MutateTask.cpp
@@ -1297,6 +1297,7 @@ void PartMergerWriter::prepare()
 bool PartMergerWriter::mutateOriginalPartAndPrepareProjections()
 {
     Block cur_block;
+    Block projection_header;
     if (MutationHelpers::checkOperationIsNotCanceled(*ctx->merges_blocker, ctx->mutate_entry) && ctx->mutating_executor->pull(cur_block))
     {
         if (ctx->minmax_idx)
@@ -1314,14 +1315,12 @@ bool PartMergerWriter::mutateOriginalPartAndPrepareProjections()
 
             ProfileEventTimeIncrement<Microseconds> watch(ProfileEvents::MutateTaskProjectionsCalculationMicroseconds);
             Block block_to_squash = projection.calculate(cur_block, ctx->context);
-            projection_squashes[i].header = block_to_squash;
-            Chunk planned_chunk = projection_squashes[i].add({block_to_squash.getColumns(), block_to_squash.rows()});
+            projection_squashes[i].setHeader(block_to_squash.cloneEmpty());
 
-            if (planned_chunk.hasChunkInfo())
+            Chunk squashed_chunk = Squashing::squash(projection_squashes[i].add({block_to_squash.getColumns(), block_to_squash.rows()}));
+            if (squashed_chunk)
             {
-                Chunk projection_chunk = DB::Squashing::squash(std::move(planned_chunk));
-
-                auto result = block_to_squash.cloneWithColumns(projection_chunk.getColumns());
+                auto result = projection_squashes[i].getHeader().cloneWithColumns(squashed_chunk.detachColumns());
                 auto tmp_part = MergeTreeDataWriter::writeTempProjectionPart(
                     *ctx->data, ctx->log, result, projection, ctx->new_data_part.get(), ++block_num);
                 tmp_part.finalize();
@@ -1342,12 +1341,10 @@ bool PartMergerWriter::mutateOriginalPartAndPrepareProjections()
     {
         const auto & projection = *ctx->projections_to_build[i];
         auto & projection_squash_plan = projection_squashes[i];
-        auto planned_chunk = projection_squash_plan.flush();
-        if (planned_chunk.hasChunkInfo())
+        auto squashed_chunk = Squashing::squash(projection_squash_plan.flush());
+        if (squashed_chunk)
         {
-            Chunk projection_chunk = DB::Squashing::squash(std::move(planned_chunk));
-
-            auto result = projection_squash_plan.header.cloneWithColumns(projection_chunk.getColumns());
+            auto result = projection_squash_plan.getHeader().cloneWithColumns(squashed_chunk.detachColumns());
             auto temp_part = MergeTreeDataWriter::writeTempProjectionPart(
                 *ctx->data, ctx->log, result, projection, ctx->new_data_part.get(), ++block_num);
             temp_part.finalize();
diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp
index 4b4f4c33e7d..bbae054fbed 100644
--- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp
+++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp
@@ -1,21 +1,25 @@
-#include <Storages/StorageReplicatedMergeTree.h>
-#include <Storages/MergeTree/ReplicatedMergeTreeQuorumEntry.h>
-#include <Storages/MergeTree/ReplicatedMergeTreeSink.h>
-#include <Storages/MergeTree/InsertBlockInfo.h>
-#include <Interpreters/PartLog.h>
 #include "Common/Exception.h"
 #include <Common/FailPoint.h>
 #include <Common/ProfileEventsScope.h>
 #include <Common/SipHash.h>
-#include <Common/ZooKeeper/KeeperException.h>
 #include <Common/ThreadFuzzer.h>
+#include <Common/ZooKeeper/KeeperException.h>
+#include <Core/Block.h>
+#include <DataTypes/ObjectUtils.h>
+#include <IO/Operators.h>
+#include <Interpreters/PartLog.h>
+#include <Processors/Transforms/DeduplicationTokenTransforms.h>
+#include <Storages/MergeTree/AsyncBlockIDsCache.h>
+#include <Storages/MergeTree/InsertBlockInfo.h>
 #include <Storages/MergeTree/MergeAlgorithm.h>
 #include <Storages/MergeTree/MergeTreeDataWriter.h>
-#include <Storages/MergeTree/AsyncBlockIDsCache.h>
-#include <DataTypes/ObjectUtils.h>
-#include <Core/Block.h>
-#include <IO/Operators.h>
+#include <Storages/MergeTree/ReplicatedMergeTreeQuorumEntry.h>
+#include <Storages/MergeTree/ReplicatedMergeTreeSink.h>
+#include <Storages/StorageReplicatedMergeTree.h>
+
 #include <fmt/core.h>
+#include <memory>
+
 
 namespace ProfileEvents
 {
@@ -253,12 +257,12 @@ size_t ReplicatedMergeTreeSinkImpl<async_insert>::checkQuorumPrecondition(const
 }
 
 template<bool async_insert>
-void ReplicatedMergeTreeSinkImpl<async_insert>::consume(Chunk chunk)
+void ReplicatedMergeTreeSinkImpl<async_insert>::consume(Chunk & chunk)
 {
     if (num_blocks_processed > 0)
         storage.delayInsertOrThrowIfNeeded(&storage.partial_shutdown_event, context, false);
 
-    auto block = getHeader().cloneWithColumns(chunk.detachColumns());
+    auto block = getHeader().cloneWithColumns(chunk.getColumns());
 
     const auto & settings = context->getSettingsRef();
 
@@ -284,13 +288,25 @@ void ReplicatedMergeTreeSinkImpl<async_insert>::consume(Chunk chunk)
 
     if constexpr (async_insert)
     {
-        const auto & chunk_info = chunk.getChunkInfo();
-        if (const auto * async_insert_info_ptr = typeid_cast<const AsyncInsertInfo *>(chunk_info.get()))
+        const auto async_insert_info_ptr = chunk.getChunkInfos().get<AsyncInsertInfo>();
+        if (async_insert_info_ptr)
             async_insert_info = std::make_shared<AsyncInsertInfo>(async_insert_info_ptr->offsets, async_insert_info_ptr->tokens);
         else
             throw Exception(ErrorCodes::LOGICAL_ERROR, "No chunk info for async inserts");
     }
 
+    String block_dedup_token;
+    auto token_info = chunk.getChunkInfos().get<DeduplicationToken::TokenInfo>();
+    if (!token_info)
+        throw Exception(ErrorCodes::LOGICAL_ERROR,
+            "TokenInfo is expected for consumed chunk in ReplicatedMergeTreeSink for table: {}",
+            storage.getStorageID().getNameForLogs());
+
+    const bool need_to_define_dedup_token = !token_info->isDefined();
+
+    if (token_info->isDefined())
+        block_dedup_token = token_info->getToken();
+
     auto part_blocks = MergeTreeDataWriter::splitBlockIntoParts(std::move(block), max_parts_per_block, metadata_snapshot, context, async_insert_info);
 
     using DelayedPartition = typename ReplicatedMergeTreeSinkImpl<async_insert>::DelayedChunk::Partition;
@@ -342,23 +358,10 @@ void ReplicatedMergeTreeSinkImpl<async_insert>::consume(Chunk chunk)
         }
         else
         {
-
             if (deduplicate)
             {
-                String block_dedup_token;
-
                 /// We add the hash from the data and partition identifier to deduplication ID.
                 /// That is, do not insert the same data to the same partition twice.
-
-                const String & dedup_token = settings.insert_deduplication_token;
-                if (!dedup_token.empty())
-                {
-                    /// multiple blocks can be inserted within the same insert query
-                    /// an ordinal number is added to dedup token to generate a distinctive block id for each block
-                    block_dedup_token = fmt::format("{}_{}", dedup_token, chunk_dedup_seqnum);
-                    ++chunk_dedup_seqnum;
-                }
-
                 block_id = temp_part.part->getZeroLevelPartBlockID(block_dedup_token);
                 LOG_DEBUG(log, "Wrote block with ID '{}', {} rows{}", block_id, current_block.block.rows(), quorumLogMessage(replicas_num));
             }
@@ -366,6 +369,13 @@ void ReplicatedMergeTreeSinkImpl<async_insert>::consume(Chunk chunk)
             {
                 LOG_DEBUG(log, "Wrote block with {} rows{}", current_block.block.rows(), quorumLogMessage(replicas_num));
             }
+
+            if (need_to_define_dedup_token)
+            {
+                chassert(temp_part.part);
+                const auto hash_value = temp_part.part->getPartBlockIDHash();
+                token_info->addChunkHash(toString(hash_value.items[0]) + "_" + toString(hash_value.items[1]));
+            }
         }
 
         profile_events_scope.reset();
@@ -411,17 +421,15 @@ void ReplicatedMergeTreeSinkImpl<async_insert>::consume(Chunk chunk)
         ));
     }
 
+    if (need_to_define_dedup_token)
+    {
+        token_info->finishChunkHashes();
+    }
+
     finishDelayedChunk(zookeeper);
     delayed_chunk = std::make_unique<ReplicatedMergeTreeSinkImpl::DelayedChunk>();
     delayed_chunk->partitions = std::move(partitions);
 
-    /// If deduplicated data should not be inserted into MV, we need to set proper
-    /// value for `last_block_is_duplicate`, which is possible only after the part is committed.
-    /// Othervide we can delay commit.
-    /// TODO: we can also delay commit if there is no MVs.
-    if (!settings.deduplicate_blocks_in_dependent_materialized_views)
-        finishDelayedChunk(zookeeper);
-
     ++num_blocks_processed;
 }
 
@@ -431,8 +439,6 @@ void ReplicatedMergeTreeSinkImpl<false>::finishDelayedChunk(const ZooKeeperWithF
     if (!delayed_chunk)
         return;
 
-    last_block_is_duplicate = false;
-
     for (auto & partition : delayed_chunk->partitions)
     {
         ProfileEventsScope scoped_attach(&partition.part_counters);
@@ -445,8 +451,6 @@ void ReplicatedMergeTreeSinkImpl<false>::finishDelayedChunk(const ZooKeeperWithF
         {
             bool deduplicated = commitPart(zookeeper, part, partition.block_id, delayed_chunk->replicas_num).second;
 
-            last_block_is_duplicate = last_block_is_duplicate || deduplicated;
-
             /// Set a special error code if the block is duplicate
             int error = (deduplicate && deduplicated) ? ErrorCodes::INSERT_WAS_DEDUPLICATED : 0;
             auto counters_snapshot = std::make_shared<ProfileEvents::Counters::Snapshot>(partition.part_counters.getPartiallyAtomicSnapshot());
@@ -535,7 +539,7 @@ bool ReplicatedMergeTreeSinkImpl<false>::writeExistingPart(MergeTreeData::Mutabl
     ProfileEventsScope profile_events_scope;
 
     String original_part_dir = part->getDataPartStorage().getPartDirectory();
-    auto try_rollback_part_rename = [this, &part, &original_part_dir]()
+    auto try_rollback_part_rename = [this, &part, &original_part_dir] ()
     {
         if (original_part_dir == part->getDataPartStorage().getPartDirectory())
             return;
@@ -1151,8 +1155,16 @@ void ReplicatedMergeTreeSinkImpl<async_insert>::onStart()
 template<bool async_insert>
 void ReplicatedMergeTreeSinkImpl<async_insert>::onFinish()
 {
-    auto zookeeper = storage.getZooKeeper();
-    finishDelayedChunk(std::make_shared<ZooKeeperWithFaultInjection>(zookeeper));
+    const auto & settings = context->getSettingsRef();
+
+    ZooKeeperWithFaultInjectionPtr zookeeper = ZooKeeperWithFaultInjection::createInstance(
+        settings.insert_keeper_fault_injection_probability,
+        settings.insert_keeper_fault_injection_seed,
+        storage.getZooKeeper(),
+        "ReplicatedMergeTreeSink::onFinish",
+        log);
+
+    finishDelayedChunk(zookeeper);
 }
 
 template<bool async_insert>
diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.h b/src/Storages/MergeTree/ReplicatedMergeTreeSink.h
index 39623c20584..7d025361717 100644
--- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.h
+++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.h
@@ -51,7 +51,7 @@ public:
     ~ReplicatedMergeTreeSinkImpl() override;
 
     void onStart() override;
-    void consume(Chunk chunk) override;
+    void consume(Chunk & chunk) override;
     void onFinish() override;
 
     String getName() const override { return "ReplicatedMergeTreeSink"; }
@@ -59,16 +59,6 @@ public:
     /// For ATTACHing existing data on filesystem.
     bool writeExistingPart(MergeTreeData::MutableDataPartPtr & part);
 
-    /// For proper deduplication in MaterializedViews
-    bool lastBlockIsDuplicate() const override
-    {
-        /// If MV is responsible for deduplication, block is not considered duplicating.
-        if (context->getSettingsRef().deduplicate_blocks_in_dependent_materialized_views)
-            return false;
-
-        return last_block_is_duplicate;
-    }
-
     struct DelayedChunk;
 private:
     std::vector<String> detectConflictsInAsyncBlockIDs(const std::vector<String> & ids);
@@ -126,7 +116,6 @@ private:
     bool allow_attach_while_readonly = false;
     bool quorum_parallel = false;
     const bool deduplicate = true;
-    bool last_block_is_duplicate = false;
     UInt64 num_blocks_processed = 0;
 
     LoggerPtr log;
diff --git a/src/Storages/MessageQueueSink.cpp b/src/Storages/MessageQueueSink.cpp
index 4fb81d69070..36899011e33 100644
--- a/src/Storages/MessageQueueSink.cpp
+++ b/src/Storages/MessageQueueSink.cpp
@@ -40,7 +40,7 @@ void MessageQueueSink::onFinish()
     producer->finish();
 }
 
-void MessageQueueSink::consume(Chunk chunk)
+void MessageQueueSink::consume(Chunk & chunk)
 {
     const auto & columns = chunk.getColumns();
     if (columns.empty())
diff --git a/src/Storages/MessageQueueSink.h b/src/Storages/MessageQueueSink.h
index b3c1e61734f..4a9248c6c4d 100644
--- a/src/Storages/MessageQueueSink.h
+++ b/src/Storages/MessageQueueSink.h
@@ -35,7 +35,7 @@ public:
 
     String getName() const override { return storage_name + "Sink"; }
 
-    void consume(Chunk chunk) override;
+    void consume(Chunk & chunk) override;
 
     void onStart() override;
     void onFinish() override;
diff --git a/src/Storages/NATS/StorageNATS.cpp b/src/Storages/NATS/StorageNATS.cpp
index 0b88a9e8929..8f0e2d76473 100644
--- a/src/Storages/NATS/StorageNATS.cpp
+++ b/src/Storages/NATS/StorageNATS.cpp
@@ -644,7 +644,13 @@ bool StorageNATS::streamToViews()
     insert->table_id = table_id;
 
     // Only insert into dependent views and expect that input blocks contain virtual columns
-    InterpreterInsertQuery interpreter(insert, nats_context, false, true, true);
+    InterpreterInsertQuery interpreter(
+        insert,
+        nats_context,
+        /* allow_materialized */ false,
+        /* no_squash */ true,
+        /* no_destination */ true,
+        /* async_isnert */ false);
     auto block_io = interpreter.execute();
 
     auto storage_snapshot = getStorageSnapshot(getInMemoryMetadataPtr(), getContext());
diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSink.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSink.cpp
index f2f6eac333c..d2bdd0af302 100644
--- a/src/Storages/ObjectStorage/StorageObjectStorageSink.cpp
+++ b/src/Storages/ObjectStorage/StorageObjectStorageSink.cpp
@@ -39,12 +39,12 @@ StorageObjectStorageSink::StorageObjectStorageSink(
         configuration->format, *write_buf, sample_block, context, format_settings_);
 }
 
-void StorageObjectStorageSink::consume(Chunk chunk)
+void StorageObjectStorageSink::consume(Chunk & chunk)
 {
     std::lock_guard lock(cancel_mutex);
     if (cancelled)
         return;
-    writer->write(getHeader().cloneWithColumns(chunk.detachColumns()));
+    writer->write(getHeader().cloneWithColumns(chunk.getColumns()));
 }
 
 void StorageObjectStorageSink::onCancel()
diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSink.h b/src/Storages/ObjectStorage/StorageObjectStorageSink.h
index e0081193686..6ab531bb21a 100644
--- a/src/Storages/ObjectStorage/StorageObjectStorageSink.h
+++ b/src/Storages/ObjectStorage/StorageObjectStorageSink.h
@@ -20,7 +20,7 @@ public:
 
     String getName() const override { return "StorageObjectStorageSink"; }
 
-    void consume(Chunk chunk) override;
+    void consume(Chunk & chunk) override;
 
     void onCancel() override;
 
diff --git a/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp b/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp
index 4388864434e..14b828e7268 100644
--- a/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp
+++ b/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp
@@ -454,7 +454,13 @@ bool StorageObjectStorageQueue::streamToViews()
 
     while (!shutdown_called && !file_iterator->isFinished())
     {
-        InterpreterInsertQuery interpreter(insert, queue_context, false, true, true);
+        InterpreterInsertQuery interpreter(
+            insert,
+            queue_context,
+            /* allow_materialized */ false,
+            /* no_squash */ true,
+            /* no_destination */ true,
+            /* async_isnert */ false);
         auto block_io = interpreter.execute();
         auto read_from_format_info = prepareReadingFromFormat(
             block_io.pipeline.getHeader().getNames(),
diff --git a/src/Storages/PartitionedSink.cpp b/src/Storages/PartitionedSink.cpp
index 09b009b26d8..ee2570756ed 100644
--- a/src/Storages/PartitionedSink.cpp
+++ b/src/Storages/PartitionedSink.cpp
@@ -51,7 +51,7 @@ SinkPtr PartitionedSink::getSinkForPartitionKey(StringRef partition_key)
     return it->second;
 }
 
-void PartitionedSink::consume(Chunk chunk)
+void PartitionedSink::consume(Chunk & chunk)
 {
     const auto & columns = chunk.getColumns();
 
@@ -104,7 +104,7 @@ void PartitionedSink::consume(Chunk chunk)
     for (const auto & [partition_key, partition_index] : partition_id_to_chunk_index)
     {
         auto sink = getSinkForPartitionKey(partition_key);
-        sink->consume(std::move(partition_index_to_chunk[partition_index]));
+        sink->consume(partition_index_to_chunk[partition_index]);
     }
 }
 
diff --git a/src/Storages/PartitionedSink.h b/src/Storages/PartitionedSink.h
index 68edeb6fd73..fcd67556dc9 100644
--- a/src/Storages/PartitionedSink.h
+++ b/src/Storages/PartitionedSink.h
@@ -20,7 +20,7 @@ public:
 
     String getName() const override { return "PartitionedSink"; }
 
-    void consume(Chunk chunk) override;
+    void consume(Chunk & chunk) override;
 
     void onException(std::exception_ptr exception) override;
 
diff --git a/src/Storages/PostgreSQL/MaterializedPostgreSQLConsumer.cpp b/src/Storages/PostgreSQL/MaterializedPostgreSQLConsumer.cpp
index ba3cc6f58d0..44479bd01e2 100644
--- a/src/Storages/PostgreSQL/MaterializedPostgreSQLConsumer.cpp
+++ b/src/Storages/PostgreSQL/MaterializedPostgreSQLConsumer.cpp
@@ -697,7 +697,13 @@ void MaterializedPostgreSQLConsumer::syncTables()
                     insert->table_id = storage->getStorageID();
                     insert->columns = std::make_shared<ASTExpressionList>(buffer->columns_ast);
 
-                    InterpreterInsertQuery interpreter(insert, insert_context, true);
+                    InterpreterInsertQuery interpreter(
+                        insert,
+                        insert_context,
+                        /* allow_materialized */ true,
+                        /* no_squash */ false,
+                        /* no_destination */ false,
+                        /* async_isnert */ false);
                     auto io = interpreter.execute();
                     auto input = std::make_shared<SourceFromSingleChunk>(
                         result_rows.cloneEmpty(), Chunk(result_rows.getColumns(), result_rows.rows()));
diff --git a/src/Storages/PostgreSQL/PostgreSQLReplicationHandler.cpp b/src/Storages/PostgreSQL/PostgreSQLReplicationHandler.cpp
index 2bb1e2dde0d..f632e553a0d 100644
--- a/src/Storages/PostgreSQL/PostgreSQLReplicationHandler.cpp
+++ b/src/Storages/PostgreSQL/PostgreSQLReplicationHandler.cpp
@@ -437,7 +437,13 @@ StorageInfo PostgreSQLReplicationHandler::loadFromSnapshot(postgres::Connection
 
     auto insert_context = materialized_storage->getNestedTableContext();
 
-    InterpreterInsertQuery interpreter(insert, insert_context);
+    InterpreterInsertQuery interpreter(
+        insert,
+        insert_context,
+        /* allow_materialized */ false,
+        /* no_squash */ false,
+        /* no_destination */ false,
+        /* async_isnert */ false);
     auto block_io = interpreter.execute();
 
     const StorageInMemoryMetadata & storage_metadata = nested_storage->getInMemoryMetadata();
diff --git a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp
index e4b19992151..f3d2aff68c8 100644
--- a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp
+++ b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp
@@ -1129,7 +1129,13 @@ bool StorageRabbitMQ::tryStreamToViews()
     }
 
     // Only insert into dependent views and expect that input blocks contain virtual columns
-    InterpreterInsertQuery interpreter(insert, rabbitmq_context, /* allow_materialized_ */ false, /* no_squash_ */ true, /* no_destination_ */ true);
+    InterpreterInsertQuery interpreter(
+        insert,
+        rabbitmq_context,
+        /* allow_materialized */ false,
+        /* no_squash */ true,
+        /* no_destination */ true,
+        /* async_isnert */ false);
     auto block_io = interpreter.execute();
 
     block_io.pipeline.complete(Pipe::unitePipes(std::move(pipes)));
diff --git a/src/Storages/RocksDB/EmbeddedRocksDBBulkSink.cpp b/src/Storages/RocksDB/EmbeddedRocksDBBulkSink.cpp
index 90792c59d38..4b5188ca9f2 100644
--- a/src/Storages/RocksDB/EmbeddedRocksDBBulkSink.cpp
+++ b/src/Storages/RocksDB/EmbeddedRocksDBBulkSink.cpp
@@ -218,7 +218,7 @@ std::pair<ColumnString::Ptr, ColumnString::Ptr> EmbeddedRocksDBBulkSink::seriali
     return {std::move(serialized_key_column), std::move(serialized_value_column)};
 }
 
-void EmbeddedRocksDBBulkSink::consume(Chunk chunk_)
+void EmbeddedRocksDBBulkSink::consume(Chunk & chunk_)
 {
     std::vector<Chunk> chunks_to_write = squash(std::move(chunk_));
 
@@ -247,7 +247,10 @@ void EmbeddedRocksDBBulkSink::onFinish()
 {
     /// If there is any data left, write it.
     if (!chunks.empty())
-        consume({});
+    {
+        Chunk empty;
+        consume(empty);
+    }
 }
 
 String EmbeddedRocksDBBulkSink::getTemporarySSTFilePath()
diff --git a/src/Storages/RocksDB/EmbeddedRocksDBBulkSink.h b/src/Storages/RocksDB/EmbeddedRocksDBBulkSink.h
index 1f548e7813d..64190c8c86f 100644
--- a/src/Storages/RocksDB/EmbeddedRocksDBBulkSink.h
+++ b/src/Storages/RocksDB/EmbeddedRocksDBBulkSink.h
@@ -32,7 +32,7 @@ public:
 
     ~EmbeddedRocksDBBulkSink() override;
 
-    void consume(Chunk chunk) override;
+    void consume(Chunk & chunk) override;
 
     void onFinish() override;
 
diff --git a/src/Storages/RocksDB/EmbeddedRocksDBSink.cpp b/src/Storages/RocksDB/EmbeddedRocksDBSink.cpp
index c451cfd1bf5..1f7f6939f40 100644
--- a/src/Storages/RocksDB/EmbeddedRocksDBSink.cpp
+++ b/src/Storages/RocksDB/EmbeddedRocksDBSink.cpp
@@ -29,7 +29,7 @@ EmbeddedRocksDBSink::EmbeddedRocksDBSink(
     serializations = getHeader().getSerializations();
 }
 
-void EmbeddedRocksDBSink::consume(Chunk chunk)
+void EmbeddedRocksDBSink::consume(Chunk & chunk)
 {
     auto rows = chunk.getNumRows();
     const auto & columns = chunk.getColumns();
diff --git a/src/Storages/RocksDB/EmbeddedRocksDBSink.h b/src/Storages/RocksDB/EmbeddedRocksDBSink.h
index 011322df829..2e1e0c7b429 100644
--- a/src/Storages/RocksDB/EmbeddedRocksDBSink.h
+++ b/src/Storages/RocksDB/EmbeddedRocksDBSink.h
@@ -17,7 +17,7 @@ public:
         StorageEmbeddedRocksDB & storage_,
         const StorageMetadataPtr & metadata_snapshot_);
 
-    void consume(Chunk chunk) override;
+    void consume(Chunk & chunk) override;
     String getName() const override { return "EmbeddedRocksDBSink"; }
 
 private:
diff --git a/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp b/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp
index b9d3e071b6c..3473166a080 100644
--- a/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp
+++ b/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp
@@ -313,7 +313,8 @@ void StorageEmbeddedRocksDB::mutate(const MutationCommands & commands, ContextPt
     Block block;
     while (executor.pull(block))
     {
-        sink->consume(Chunk{block.getColumns(), block.rows()});
+        auto chunk = Chunk(block.getColumns(), block.rows());
+        sink->consume(chunk);
     }
 }
 
diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp
index a3f6b6afc5d..b064fba223a 100644
--- a/src/Storages/StorageBuffer.cpp
+++ b/src/Storages/StorageBuffer.cpp
@@ -607,7 +607,7 @@ public:
 
     String getName() const override { return "BufferSink"; }
 
-    void consume(Chunk chunk) override
+    void consume(Chunk & chunk) override
     {
         size_t rows = chunk.getNumRows();
         if (!rows)
@@ -1020,7 +1020,13 @@ void StorageBuffer::writeBlockToDestination(const Block & block, StoragePtr tabl
     auto insert_context = Context::createCopy(getContext());
     insert_context->makeQueryContext();
 
-    InterpreterInsertQuery interpreter{insert, insert_context, allow_materialized};
+    InterpreterInsertQuery interpreter(
+        insert,
+        insert_context,
+        allow_materialized,
+        /* no_squash */ false,
+        /* no_destination */ false,
+        /* async_isnert */ false);
 
     auto block_io = interpreter.execute();
     PushingPipelineExecutor executor(block_io.pipeline);
diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp
index 849fa5dbe0b..67586985ce8 100644
--- a/src/Storages/StorageDistributed.cpp
+++ b/src/Storages/StorageDistributed.cpp
@@ -1050,7 +1050,13 @@ std::optional<QueryPipeline> StorageDistributed::distributedWriteBetweenDistribu
         const auto & shard_info = shards_info[shard_index];
         if (shard_info.isLocal())
         {
-            InterpreterInsertQuery interpreter(new_query, query_context);
+            InterpreterInsertQuery interpreter(
+                new_query,
+                query_context,
+                /* allow_materialized */ false,
+                /* no_squash */ false,
+                /* no_destination */ false,
+                /* async_isnert */ false);
             pipeline.addCompletedPipeline(interpreter.execute().pipeline);
         }
         else
diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp
index 7f39ff615f0..3fb397c7b81 100644
--- a/src/Storages/StorageFile.cpp
+++ b/src/Storages/StorageFile.cpp
@@ -1778,12 +1778,12 @@ public:
 
     String getName() const override { return "StorageFileSink"; }
 
-    void consume(Chunk chunk) override
+    void consume(Chunk & chunk) override
     {
         std::lock_guard cancel_lock(cancel_mutex);
         if (cancelled)
             return;
-        writer->write(getHeader().cloneWithColumns(chunk.detachColumns()));
+        writer->write(getHeader().cloneWithColumns(chunk.getColumns()));
     }
 
     void onCancel() override
diff --git a/src/Storages/StorageKeeperMap.cpp b/src/Storages/StorageKeeperMap.cpp
index 20f99070000..c80e799a92b 100644
--- a/src/Storages/StorageKeeperMap.cpp
+++ b/src/Storages/StorageKeeperMap.cpp
@@ -119,10 +119,10 @@ public:
 
     std::string getName() const override { return "StorageKeeperMapSink"; }
 
-    void consume(Chunk chunk) override
+    void consume(Chunk & chunk) override
     {
         auto rows = chunk.getNumRows();
-        auto block = getHeader().cloneWithColumns(chunk.detachColumns());
+        auto block = getHeader().cloneWithColumns(chunk.getColumns());
 
         WriteBufferFromOwnString wb_key;
         WriteBufferFromOwnString wb_value;
@@ -1248,7 +1248,10 @@ void StorageKeeperMap::mutate(const MutationCommands & commands, ContextPtr loca
 
     Block block;
     while (executor.pull(block))
-        sink->consume(Chunk{block.getColumns(), block.rows()});
+    {
+        auto chunk = Chunk(block.getColumns(), block.rows());
+        sink->consume(chunk);
+    }
 
     sink->finalize<true>(strict);
 }
diff --git a/src/Storages/StorageLog.cpp b/src/Storages/StorageLog.cpp
index de0324d7998..463694c63aa 100644
--- a/src/Storages/StorageLog.cpp
+++ b/src/Storages/StorageLog.cpp
@@ -1,5 +1,6 @@
 #include <Storages/StorageLog.h>
 #include <Storages/StorageFactory.h>
+#include <Storages/StorageLogSettings.h>
 
 #include <Common/Exception.h>
 #include <Common/StringUtils.h>
@@ -21,7 +22,6 @@
 #include <DataTypes/NestedUtils.h>
 
 #include <Interpreters/Context.h>
-#include "StorageLogSettings.h"
 #include <Processors/Sources/NullSource.h>
 #include <Processors/ISource.h>
 #include <QueryPipeline/Pipe.h>
@@ -341,7 +341,7 @@ public:
         }
     }
 
-    void consume(Chunk chunk) override;
+    void consume(Chunk & chunk) override;
     void onFinish() override;
 
 private:
@@ -398,9 +398,9 @@ private:
 };
 
 
-void LogSink::consume(Chunk chunk)
+void LogSink::consume(Chunk & chunk)
 {
-    auto block = getHeader().cloneWithColumns(chunk.detachColumns());
+    auto block = getHeader().cloneWithColumns(chunk.getColumns());
     metadata_snapshot->check(block, true);
 
     for (auto & stream : streams | boost::adaptors::map_values)
diff --git a/src/Storages/StorageMemory.cpp b/src/Storages/StorageMemory.cpp
index f69c4adb552..b1bd7053c2e 100644
--- a/src/Storages/StorageMemory.cpp
+++ b/src/Storages/StorageMemory.cpp
@@ -63,7 +63,7 @@ public:
 
     String getName() const override { return "MemorySink"; }
 
-    void consume(Chunk chunk) override
+    void consume(Chunk & chunk) override
     {
         auto block = getHeader().cloneWithColumns(chunk.getColumns());
         storage_snapshot->metadata->check(block, true);
diff --git a/src/Storages/StorageMongoDB.cpp b/src/Storages/StorageMongoDB.cpp
index 62a2a048642..e0818fafae9 100644
--- a/src/Storages/StorageMongoDB.cpp
+++ b/src/Storages/StorageMongoDB.cpp
@@ -17,7 +17,6 @@
 #include <QueryPipeline/Pipe.h>
 #include <Processors/Sources/MongoDBSource.h>
 #include <Processors/Sinks/SinkToStorage.h>
-#include <unordered_set>
 
 #include <DataTypes/DataTypeArray.h>
 
@@ -107,12 +106,12 @@ public:
 
     String getName() const override { return "StorageMongoDBSink"; }
 
-    void consume(Chunk chunk) override
+    void consume(Chunk & chunk) override
     {
         Poco::MongoDB::Database db(db_name);
         Poco::MongoDB::Document::Vector documents;
 
-        auto block = getHeader().cloneWithColumns(chunk.detachColumns());
+        auto block = getHeader().cloneWithColumns(chunk.getColumns());
 
         size_t num_rows = block.rows();
         size_t num_cols = block.columns();
diff --git a/src/Storages/StorageMySQL.cpp b/src/Storages/StorageMySQL.cpp
index da391909dff..2a8a7bd2ee7 100644
--- a/src/Storages/StorageMySQL.cpp
+++ b/src/Storages/StorageMySQL.cpp
@@ -151,9 +151,9 @@ public:
 
     String getName() const override { return "StorageMySQLSink"; }
 
-    void consume(Chunk chunk) override
+    void consume(Chunk & chunk) override
     {
-        auto block = getHeader().cloneWithColumns(chunk.detachColumns());
+        auto block = getHeader().cloneWithColumns(chunk.getColumns());
         auto blocks = splitBlocks(block, max_batch_rows);
         mysqlxx::Transaction trans(entry);
         try
diff --git a/src/Storages/StoragePostgreSQL.cpp b/src/Storages/StoragePostgreSQL.cpp
index a8713c61e4d..cdfeab62b58 100644
--- a/src/Storages/StoragePostgreSQL.cpp
+++ b/src/Storages/StoragePostgreSQL.cpp
@@ -227,9 +227,9 @@ public:
 
     String getName() const override { return "PostgreSQLSink"; }
 
-    void consume(Chunk chunk) override
+    void consume(Chunk & chunk) override
     {
-        auto block = getHeader().cloneWithColumns(chunk.detachColumns());
+        auto block = getHeader().cloneWithColumns(chunk.getColumns());
         if (!inserter)
         {
             if (on_conflict.empty())
diff --git a/src/Storages/StorageRedis.cpp b/src/Storages/StorageRedis.cpp
index 83bb3c606c9..1a275320f43 100644
--- a/src/Storages/StorageRedis.cpp
+++ b/src/Storages/StorageRedis.cpp
@@ -147,7 +147,7 @@ class RedisSink : public SinkToStorage
 public:
     RedisSink(StorageRedis & storage_, const StorageMetadataPtr & metadata_snapshot_);
 
-    void consume(Chunk chunk) override;
+    void consume(Chunk & chunk) override;
     String getName() const override { return "RedisSink"; }
 
 private:
@@ -169,10 +169,10 @@ RedisSink::RedisSink(StorageRedis & storage_, const StorageMetadataPtr & metadat
     }
 }
 
-void RedisSink::consume(Chunk chunk)
+void RedisSink::consume(Chunk & chunk)
 {
     auto rows = chunk.getNumRows();
-    auto block = getHeader().cloneWithColumns(chunk.detachColumns());
+    auto block = getHeader().cloneWithColumns(chunk.getColumns());
 
     WriteBufferFromOwnString wb_key;
     WriteBufferFromOwnString wb_value;
@@ -567,7 +567,8 @@ void StorageRedis::mutate(const MutationCommands & commands, ContextPtr context_
     Block block;
     while (executor.pull(block))
     {
-        sink->consume(Chunk{block.getColumns(), block.rows()});
+        Chunk chunk(block.getColumns(), block.rows());
+        sink->consume(chunk);
     }
 }
 
diff --git a/src/Storages/StorageSQLite.cpp b/src/Storages/StorageSQLite.cpp
index 179e4cee199..85417a2f2a4 100644
--- a/src/Storages/StorageSQLite.cpp
+++ b/src/Storages/StorageSQLite.cpp
@@ -141,7 +141,7 @@ public:
 
     String getName() const override { return "SQLiteSink"; }
 
-    void consume(Chunk chunk) override
+    void consume(Chunk & chunk) override
     {
         auto block = getHeader().cloneWithColumns(chunk.getColumns());
         WriteBufferFromOwnString sqlbuf;
diff --git a/src/Storages/StorageSet.cpp b/src/Storages/StorageSet.cpp
index 5b7f9fc0ac2..0d094c15880 100644
--- a/src/Storages/StorageSet.cpp
+++ b/src/Storages/StorageSet.cpp
@@ -44,7 +44,7 @@ public:
         const String & backup_file_name_, bool persistent_);
 
     String getName() const override { return "SetOrJoinSink"; }
-    void consume(Chunk chunk) override;
+    void consume(Chunk & chunk) override;
     void onFinish() override;
 
 private:
@@ -82,9 +82,9 @@ SetOrJoinSink::SetOrJoinSink(
 {
 }
 
-void SetOrJoinSink::consume(Chunk chunk)
+void SetOrJoinSink::consume(Chunk & chunk)
 {
-    Block block = getHeader().cloneWithColumns(chunk.detachColumns());
+    Block block = getHeader().cloneWithColumns(chunk.getColumns());
 
     table.insertBlock(block, getContext());
     if (persistent)
diff --git a/src/Storages/StorageStripeLog.cpp b/src/Storages/StorageStripeLog.cpp
index 8df87d6290f..9b6d9f041e1 100644
--- a/src/Storages/StorageStripeLog.cpp
+++ b/src/Storages/StorageStripeLog.cpp
@@ -226,9 +226,9 @@ public:
         }
     }
 
-    void consume(Chunk chunk) override
+    void consume(Chunk & chunk) override
     {
-        block_out->write(getHeader().cloneWithColumns(chunk.detachColumns()));
+        block_out->write(getHeader().cloneWithColumns(chunk.getColumns()));
     }
 
     void onFinish() override
diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp
index 895da028fc2..90e05c44e31 100644
--- a/src/Storages/StorageURL.cpp
+++ b/src/Storages/StorageURL.cpp
@@ -565,12 +565,12 @@ StorageURLSink::StorageURLSink(
 }
 
 
-void StorageURLSink::consume(Chunk chunk)
+void StorageURLSink::consume(Chunk & chunk)
 {
     std::lock_guard lock(cancel_mutex);
     if (cancelled)
         return;
-    writer->write(getHeader().cloneWithColumns(chunk.detachColumns()));
+    writer->write(getHeader().cloneWithColumns(chunk.getColumns()));
 }
 
 void StorageURLSink::onCancel()
diff --git a/src/Storages/StorageURL.h b/src/Storages/StorageURL.h
index fa7cc6eeeef..1804079e75f 100644
--- a/src/Storages/StorageURL.h
+++ b/src/Storages/StorageURL.h
@@ -251,7 +251,7 @@ public:
         const String & method = Poco::Net::HTTPRequest::HTTP_POST);
 
     std::string getName() const override { return "StorageURLSink"; }
-    void consume(Chunk chunk) override;
+    void consume(Chunk & chunk) override;
     void onCancel() override;
     void onException(std::exception_ptr exception) override;
     void onFinish() override;
diff --git a/src/Storages/System/StorageSystemZooKeeper.cpp b/src/Storages/System/StorageSystemZooKeeper.cpp
index cb46cd19517..c9c606de049 100644
--- a/src/Storages/System/StorageSystemZooKeeper.cpp
+++ b/src/Storages/System/StorageSystemZooKeeper.cpp
@@ -119,7 +119,7 @@ public:
     ZooKeeperSink(const Block & header, ContextPtr context) : SinkToStorage(header), zookeeper(context->getZooKeeper()) { }
     String getName() const override { return "ZooKeeperSink"; }
 
-    void consume(Chunk chunk) override
+    void consume(Chunk & chunk) override
     {
         auto block = getHeader().cloneWithColumns(chunk.getColumns());
         size_t rows = block.rows();
diff --git a/src/Storages/WindowView/StorageWindowView.cpp b/src/Storages/WindowView/StorageWindowView.cpp
index 77e6ee9cb24..e36247103c7 100644
--- a/src/Storages/WindowView/StorageWindowView.cpp
+++ b/src/Storages/WindowView/StorageWindowView.cpp
@@ -32,6 +32,7 @@
 #include <Processors/Sources/BlocksSource.h>
 #include <Processors/Sources/SourceFromSingleChunk.h>
 #include <Processors/Executors/PullingAsyncPipelineExecutor.h>
+#include <Processors/Transforms/DeduplicationTokenTransforms.h>
 #include <Processors/Transforms/ExpressionTransform.h>
 #include <Processors/Transforms/FilterTransform.h>
 #include <Processors/Transforms/WatermarkTransform.h>
@@ -304,7 +305,7 @@ namespace
     public:
         explicit AddingAggregatedChunkInfoTransform(Block header) : ISimpleTransform(header, header, false) { }
 
-        void transform(Chunk & chunk) override { chunk.setChunkInfo(std::make_shared<AggregatedChunkInfo>()); }
+        void transform(Chunk & chunk) override { chunk.getChunkInfos().add(std::make_shared<AggregatedChunkInfo>()); }
 
         String getName() const override { return "AddingAggregatedChunkInfoTransform"; }
     };
@@ -689,7 +690,13 @@ inline void StorageWindowView::fire(UInt32 watermark)
         StoragePtr target_table = getTargetTable();
         auto insert = std::make_shared<ASTInsertQuery>();
         insert->table_id = target_table->getStorageID();
-        InterpreterInsertQuery interpreter(insert, getContext());
+        InterpreterInsertQuery interpreter(
+            insert,
+            getContext(),
+            /* allow_materialized */ false,
+            /* no_squash */ false,
+            /* no_destination */ false,
+            /* async_isnert */ false);
         auto block_io = interpreter.execute();
 
         auto pipe = Pipe(std::make_shared<BlocksSource>(blocks, header));
@@ -1413,7 +1420,7 @@ void StorageWindowView::eventTimeParser(const ASTCreateQuery & query)
 }
 
 void StorageWindowView::writeIntoWindowView(
-    StorageWindowView & window_view, const Block & block, ContextPtr local_context)
+    StorageWindowView & window_view, Block && block, Chunk::ChunkInfoCollection && chunk_infos, ContextPtr local_context)
 {
     window_view.throwIfWindowViewIsDisabled(local_context);
     while (window_view.modifying_query)
@@ -1428,7 +1435,7 @@ void StorageWindowView::writeIntoWindowView(
         window_view.max_watermark = window_view.getWindowUpperBound(first_record_timestamp);
     }
 
-    Pipe pipe(std::make_shared<SourceFromSingleChunk>(block.cloneEmpty(), Chunk(block.getColumns(), block.rows())));
+    Pipe pipe(std::make_shared<SourceFromSingleChunk>(block));
 
     UInt32 lateness_bound = 0;
     UInt32 t_max_watermark = 0;
@@ -1473,10 +1480,10 @@ void StorageWindowView::writeIntoWindowView(
         auto syntax_result = TreeRewriter(local_context).analyze(query, columns);
         auto filter_expression = ExpressionAnalyzer(filter_function, syntax_result, local_context).getActionsDAG(false);
 
-        pipe.addSimpleTransform([&](const Block & header)
+        pipe.addSimpleTransform([&](const Block & header_)
         {
             return std::make_shared<FilterTransform>(
-                header, std::make_shared<ExpressionActions>(filter_expression),
+                header_, std::make_shared<ExpressionActions>(filter_expression),
                 filter_function->getColumnName(), true);
         });
     }
@@ -1531,6 +1538,30 @@ void StorageWindowView::writeIntoWindowView(
         QueryProcessingStage::WithMergeableState);
 
     builder = select_block.buildQueryPipeline();
+
+    builder.addSimpleTransform([&](const Block & stream_header)
+    {
+        // Can't move chunk_infos here, that function could be called several times
+        return std::make_shared<RestoreChunkInfosTransform>(chunk_infos.clone(), stream_header);
+    });
+
+    String window_view_id = window_view.getStorageID().hasUUID() ? toString(window_view.getStorageID().uuid) : window_view.getStorageID().getFullNameNotQuoted();
+    builder.addSimpleTransform([&](const Block & stream_header)
+    {
+        return std::make_shared<DeduplicationToken::SetViewIDTransform>(window_view_id, stream_header);
+    });
+    builder.addSimpleTransform([&](const Block & stream_header)
+    {
+        return std::make_shared<DeduplicationToken::SetViewBlockNumberTransform>(stream_header);
+    });
+
+#ifdef ABORT_ON_LOGICAL_ERROR
+    builder.addSimpleTransform([&](const Block & stream_header)
+    {
+        return std::make_shared<DeduplicationToken::CheckTokenTransform>("StorageWindowView: Afrer tmp table before squashing", stream_header);
+    });
+#endif
+
     builder.addSimpleTransform([&](const Block & current_header)
     {
         return std::make_shared<SquashingTransform>(
@@ -1570,6 +1601,13 @@ void StorageWindowView::writeIntoWindowView(
             lateness_upper_bound);
     });
 
+#ifdef ABORT_ON_LOGICAL_ERROR
+    builder.addSimpleTransform([&](const Block & stream_header)
+    {
+        return std::make_shared<DeduplicationToken::CheckTokenTransform>("StorageWindowView: Afrer WatermarkTransform", stream_header);
+    });
+#endif
+
     auto inner_table = window_view.getInnerTable();
     auto lock = inner_table->lockForShare(
         local_context->getCurrentQueryId(), local_context->getSettingsRef().lock_acquire_timeout);
@@ -1586,9 +1624,16 @@ void StorageWindowView::writeIntoWindowView(
         auto convert_actions = std::make_shared<ExpressionActions>(
             convert_actions_dag, ExpressionActionsSettings::fromContext(local_context, CompileExpressions::yes));
 
-        builder.addSimpleTransform([&](const Block & header) { return std::make_shared<ExpressionTransform>(header, convert_actions); });
+        builder.addSimpleTransform([&](const Block & header_) { return std::make_shared<ExpressionTransform>(header_, convert_actions); });
     }
 
+#ifdef ABORT_ON_LOGICAL_ERROR
+    builder.addSimpleTransform([&](const Block & stream_header)
+    {
+        return std::make_shared<DeduplicationToken::CheckTokenTransform>("StorageWindowView: Before out", stream_header);
+    });
+#endif
+
     builder.addChain(Chain(std::move(output)));
     builder.setSinks([&](const Block & cur_header, Pipe::StreamType)
     {
diff --git a/src/Storages/WindowView/StorageWindowView.h b/src/Storages/WindowView/StorageWindowView.h
index f79867df424..14ac65091d3 100644
--- a/src/Storages/WindowView/StorageWindowView.h
+++ b/src/Storages/WindowView/StorageWindowView.h
@@ -166,7 +166,7 @@ public:
 
     BlockIO populate();
 
-    static void writeIntoWindowView(StorageWindowView & window_view, const Block & block, ContextPtr context);
+    static void writeIntoWindowView(StorageWindowView & window_view, Block && block, Chunk::ChunkInfoCollection && chunk_infos, ContextPtr context);
 
     ASTPtr getMergeableQuery() const { return mergeable_query->clone(); }
 
diff --git a/tests/integration/test_force_deduplication/test.py b/tests/integration/test_force_deduplication/test.py
index 87b2c45bbc5..14c11bc8500 100644
--- a/tests/integration/test_force_deduplication/test.py
+++ b/tests/integration/test_force_deduplication/test.py
@@ -29,6 +29,8 @@ def get_counts():
 
 
 def test_basic(start_cluster):
+    old_src, old_a, old_b, old_c = 0, 0, 0, 0
+
     node.query(
         """
         CREATE TABLE test (A Int64) ENGINE = ReplicatedMergeTree ('/clickhouse/test/tables/test','1') ORDER BY tuple();
@@ -39,6 +41,15 @@ def test_basic(start_cluster):
         INSERT INTO test values(999);
         """
     )
+
+    src, a, b, c = get_counts()
+    assert src == old_src + 1
+    assert a == old_a + 2
+    assert b == old_b + 2
+    assert c == old_c + 2
+    old_src, old_a, old_b, old_c = src, a, b, c
+
+    # that issert fails on test_mv_b due to partitions by A
     with pytest.raises(QueryRuntimeException):
         node.query(
             """
@@ -46,22 +57,23 @@ def test_basic(start_cluster):
             INSERT INTO test SELECT number FROM numbers(10);
             """
         )
+    src, a, b, c = get_counts()
+    assert src == old_src + 10
+    assert a == old_a + 10
+    assert b == old_b
+    assert c == old_c + 10
+    old_src, old_a, old_b, old_c = src, a, b, c
 
-    old_src, old_a, old_b, old_c = get_counts()
-    # number of rows in test_mv_a and test_mv_c depends on order of inserts into views
-    assert old_src == 11
-    assert old_a in (1, 11)
-    assert old_b == 1
-    assert old_c in (1, 11)
-
+    # deduplication only for src table
     node.query("INSERT INTO test SELECT number FROM numbers(10)")
     src, a, b, c = get_counts()
-    # no changes because of deduplication in source table
     assert src == old_src
-    assert a == old_a
-    assert b == old_b
-    assert c == old_c
+    assert a == old_a + 10
+    assert b == old_b + 10
+    assert c == old_c + 10
+    old_src, old_a, old_b, old_c = src, a, b, c
 
+    # deduplication for MV tables does not work, because previous inserts have not written their deduplications tokens to the log due to `deduplicate_blocks_in_dependent_materialized_views = 0`.
     node.query(
         """
         SET deduplicate_blocks_in_dependent_materialized_views = 1;
@@ -69,11 +81,27 @@ def test_basic(start_cluster):
         """
     )
     src, a, b, c = get_counts()
-    assert src == 11
-    assert a == old_a + 10  # first insert could be succesfull with disabled dedup
-    assert b == 11
+    assert src == old_src
+    assert a == old_a + 10
+    assert b == old_b + 10
     assert c == old_c + 10
+    old_src, old_a, old_b, old_c = src, a, b, c
 
+    # deduplication for all the tables
+    node.query(
+        """
+        SET deduplicate_blocks_in_dependent_materialized_views = 1;
+        INSERT INTO test SELECT number FROM numbers(10);
+        """
+    )
+    src, a, b, c = get_counts()
+    assert src == old_src
+    assert a == old_a
+    assert b == old_b
+    assert c == old_c
+    old_src, old_a, old_b, old_c = src, a, b, c
+
+    # that issert fails on test_mv_b due to partitions by A, it is an uniq data which is not deduplicated
     with pytest.raises(QueryRuntimeException):
         node.query(
             """
@@ -82,16 +110,23 @@ def test_basic(start_cluster):
             INSERT INTO test SELECT number FROM numbers(100,10);
             """
         )
+    src, a, b, c = get_counts()
+    assert src == old_src + 10
+    assert a == old_a + 10
+    assert b == old_b
+    assert c == old_c + 10
+    old_src, old_a, old_b, old_c = src, a, b, c
 
+    # deduplication for all tables, except test_mv_b. For test_mv_b it is an uniq data which is not deduplicated due to exception at previous insert
     node.query(
         """
         SET deduplicate_blocks_in_dependent_materialized_views = 1;
         INSERT INTO test SELECT number FROM numbers(100,10);
         """
     )
-
     src, a, b, c = get_counts()
-    assert src == 21
-    assert a == old_a + 20
-    assert b == 21
-    assert c == old_c + 20
+    assert src == old_src
+    assert a == old_a
+    assert b == old_b + 10
+    assert c == old_c
+    old_src, old_a, old_b, old_c = src, a, b, c
diff --git a/tests/queries/0_stateless/00510_materizlized_view_and_deduplication_zookeeper.reference b/tests/queries/0_stateless/00510_materizlized_view_and_deduplication_zookeeper.reference
index adf6abb7298..9c9281dc7e4 100644
--- a/tests/queries/0_stateless/00510_materizlized_view_and_deduplication_zookeeper.reference
+++ b/tests/queries/0_stateless/00510_materizlized_view_and_deduplication_zookeeper.reference
@@ -1,7 +1,7 @@
 2
 3
 
-2
+3
 3
 
 1
diff --git a/tests/queries/0_stateless/00510_materizlized_view_and_deduplication_zookeeper.sql b/tests/queries/0_stateless/00510_materizlized_view_and_deduplication_zookeeper.sql
index d3c4da86b41..51e6a513608 100644
--- a/tests/queries/0_stateless/00510_materizlized_view_and_deduplication_zookeeper.sql
+++ b/tests/queries/0_stateless/00510_materizlized_view_and_deduplication_zookeeper.sql
@@ -29,7 +29,7 @@ INSERT INTO without_deduplication VALUES (43);
 SELECT count() FROM with_deduplication;
 SELECT count() FROM without_deduplication;
 
--- Implicit insert isn't deduplicated
+-- Implicit insert isn't deduplicated, because deduplicate_blocks_in_dependent_materialized_views = 0 by default
 SELECT '';
 SELECT countMerge(cnt) FROM with_deduplication_mv;
 SELECT countMerge(cnt) FROM without_deduplication_mv;
diff --git a/tests/queries/0_stateless/00633_materialized_view_and_too_many_parts_zookeeper.sh b/tests/queries/0_stateless/00633_materialized_view_and_too_many_parts_zookeeper.sh
index 1fb219108da..8f7d19028b0 100755
--- a/tests/queries/0_stateless/00633_materialized_view_and_too_many_parts_zookeeper.sh
+++ b/tests/queries/0_stateless/00633_materialized_view_and_too_many_parts_zookeeper.sh
@@ -36,8 +36,8 @@ ${CLICKHOUSE_CLIENT} --query "DROP TABLE c"
 echo
 ${CLICKHOUSE_CLIENT} --query "CREATE TABLE root (d UInt64) ENGINE = Null"
 ${CLICKHOUSE_CLIENT} --query "CREATE MATERIALIZED VIEW d (d UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/d', '1') ORDER BY d AS SELECT * FROM root"
-${CLICKHOUSE_CLIENT} --query "INSERT INTO root VALUES (1)";
-${CLICKHOUSE_CLIENT} --query "INSERT INTO root VALUES (1)";
+${CLICKHOUSE_CLIENT} --query "INSERT INTO root SETTINGS deduplicate_blocks_in_dependent_materialized_views=1 VALUES (1)";
+${CLICKHOUSE_CLIENT} --query "INSERT INTO root SETTINGS deduplicate_blocks_in_dependent_materialized_views=1 VALUES (1)";
 ${CLICKHOUSE_CLIENT} --query "SELECT * FROM d";
 ${CLICKHOUSE_CLIENT} --query "DROP TABLE root"
 ${CLICKHOUSE_CLIENT} --query "DROP TABLE d"
diff --git a/tests/queries/0_stateless/01275_parallel_mv.reference b/tests/queries/0_stateless/01275_parallel_mv.reference
index a9801e3b910..dadf2f35e6e 100644
--- a/tests/queries/0_stateless/01275_parallel_mv.reference
+++ b/tests/queries/0_stateless/01275_parallel_mv.reference
@@ -137,7 +137,7 @@ select arrayUniq(thread_ids) from system.query_log where
     Settings['parallel_view_processing'] = '1' and
     Settings['optimize_trivial_insert_select'] = '0' and
     Settings['max_insert_threads'] = '16';
-5
+18
 select count() from testX;
 60
 select count() from testXA;
@@ -185,7 +185,7 @@ select arrayUniq(thread_ids) from system.query_log where
     Settings['parallel_view_processing'] = '1' and
     Settings['optimize_trivial_insert_select'] = '1' and
     Settings['max_insert_threads'] = '16';
-5
+18
 select count() from testX;
 80
 select count() from testXA;
diff --git a/tests/queries/0_stateless/01927_query_views_log_current_database.sql b/tests/queries/0_stateless/01927_query_views_log_current_database.sql
index ba42795333c..6287156daaf 100644
--- a/tests/queries/0_stateless/01927_query_views_log_current_database.sql
+++ b/tests/queries/0_stateless/01927_query_views_log_current_database.sql
@@ -16,6 +16,7 @@ CREATE MATERIALIZED VIEW matview_b_to_c TO table_c AS SELECT SUM(a + sleepEachRo
 CREATE MATERIALIZED VIEW matview_join_d_e TO table_f AS SELECT table_d.a as a, table_e.count + sleepEachRow(0.000003) as count FROM table_d LEFT JOIN table_e ON table_d.a = table_e.a;
 
 -- ENABLE LOGS
+SET parallel_view_processing=0;
 SET log_query_views=1;
 SET log_queries_min_type='QUERY_FINISH';
 SET log_queries=1;
diff --git a/tests/queries/0_stateless/02124_insert_deduplication_token_materialized_views.reference b/tests/queries/0_stateless/02124_insert_deduplication_token_materialized_views.reference
index e0cc8f0ce63..2d9f236ada9 100644
--- a/tests/queries/0_stateless/02124_insert_deduplication_token_materialized_views.reference
+++ b/tests/queries/0_stateless/02124_insert_deduplication_token_materialized_views.reference
@@ -1,8 +1,8 @@
-deduplicate_blocks_in_dependent_materialized_views=0, insert_deduplication_token = no, results inconsitent
-18	18	9	18
-deduplicate_blocks_in_dependent_materialized_views=1, insert_deduplication_token = no, results inconsitent
-18	9	9	9
-deduplicate_blocks_in_dependent_materialized_views=0, insert_deduplication_token = yes, results inconsitent
-18	18	9	18
-deduplicate_blocks_in_dependent_materialized_views=1, insert_deduplication_token = yes, results consitent
+deduplicate_blocks_in_dependent_materialized_views=0, insert_deduplication_token = no, results: test_mv_a and test_mv_c have all data, test_mv_b has data obly with max_partitions_per_insert_block=0
+18	36	27	36
+deduplicate_blocks_in_dependent_materialized_views=1, insert_deduplication_token = no, results: all tables have deduplicated data
+18	18	18	18
+deduplicate_blocks_in_dependent_materialized_views=0, insert_deduplication_token = yes, results: test_mv_a and test_mv_c have all data, test_mv_b has data obly with max_partitions_per_insert_block=0
+18	36	27	36
+deduplicate_blocks_in_dependent_materialized_views=1, insert_deduplication_token = yes, results: all tables have deduplicated data
 18	18	18	18
diff --git a/tests/queries/0_stateless/02124_insert_deduplication_token_materialized_views.sql b/tests/queries/0_stateless/02124_insert_deduplication_token_materialized_views.sql
index fdd75b91b1f..465c8d6136c 100644
--- a/tests/queries/0_stateless/02124_insert_deduplication_token_materialized_views.sql
+++ b/tests/queries/0_stateless/02124_insert_deduplication_token_materialized_views.sql
@@ -1,6 +1,6 @@
 -- Tags: long
 
-select 'deduplicate_blocks_in_dependent_materialized_views=0, insert_deduplication_token = no, results inconsitent';
+select 'deduplicate_blocks_in_dependent_materialized_views=0, insert_deduplication_token = no, results: test_mv_a and test_mv_c have all data, test_mv_b has data obly with max_partitions_per_insert_block=0';
 
 drop table if exists test  sync;
 drop table if exists test_mv_a sync;
@@ -35,7 +35,7 @@ select
   (select sum(c) from test_mv_c where test='case1');
 
 
-select 'deduplicate_blocks_in_dependent_materialized_views=1, insert_deduplication_token = no, results inconsitent';
+select 'deduplicate_blocks_in_dependent_materialized_views=1, insert_deduplication_token = no, results: all tables have deduplicated data';
 
 set deduplicate_blocks_in_dependent_materialized_views=1;
 
@@ -53,7 +53,7 @@ select
   (select sum(c) from test_mv_c where test='case2');
 
 
-select 'deduplicate_blocks_in_dependent_materialized_views=0, insert_deduplication_token = yes, results inconsitent';
+select 'deduplicate_blocks_in_dependent_materialized_views=0, insert_deduplication_token = yes, results: test_mv_a and test_mv_c have all data, test_mv_b has data obly with max_partitions_per_insert_block=0';
 
 set deduplicate_blocks_in_dependent_materialized_views=0;
 
@@ -70,7 +70,7 @@ select
   (select sum(c) from test_mv_b where test='case3'),
   (select sum(c) from test_mv_c where test='case3');
 
-select 'deduplicate_blocks_in_dependent_materialized_views=1, insert_deduplication_token = yes, results consitent';
+select 'deduplicate_blocks_in_dependent_materialized_views=1, insert_deduplication_token = yes, results: all tables have deduplicated data';
 
 set deduplicate_blocks_in_dependent_materialized_views=1;
 
diff --git a/tests/queries/0_stateless/02125_query_views_log.sql b/tests/queries/0_stateless/02125_query_views_log.sql
index d2d19b76a1f..ba50902ebea 100644
--- a/tests/queries/0_stateless/02125_query_views_log.sql
+++ b/tests/queries/0_stateless/02125_query_views_log.sql
@@ -8,7 +8,7 @@ create table dst (key Int) engine=Null();
 create materialized view mv1 to dst as select * from src;
 create materialized view mv2 to dst as select * from src;
 
-insert into src select * from numbers(1e6) settings log_queries=1, max_untracked_memory=0, parallel_view_processing=1;
+insert into src select * from numbers(1e6) settings log_queries=1, max_untracked_memory=0, parallel_view_processing=0;
 system flush logs;
 
 -- { echo }
diff --git a/tests/queries/0_stateless/02912_ingestion_mv_deduplication.reference b/tests/queries/0_stateless/02912_ingestion_mv_deduplication.reference
index 335b55f05c8..07deb7c2565 100644
--- a/tests/queries/0_stateless/02912_ingestion_mv_deduplication.reference
+++ b/tests/queries/0_stateless/02912_ingestion_mv_deduplication.reference
@@ -10,13 +10,14 @@
 2022-09-01 12:23:34	42
 2023-09-01 12:23:34	42
 -- MV
-2022-09-01 12:00:00	42
+2022-09-01 12:00:00	84
+2023-09-01 12:00:00	42
 -- Original issue with deduplicate_blocks_in_dependent_materialized_views = 1 AND max_insert_delayed_streams_for_parallel_write > 1
 -- Landing
 2022-09-01 12:23:34	42
 2023-09-01 12:23:34	42
 -- MV
-2022-09-01 12:00:00	42
+2022-09-01 12:00:00	84
 2023-09-01 12:00:00	42
 -- Regression introduced in https://github.com/ClickHouse/ClickHouse/pull/54184
 -- Landing (Agg/Replacing)MergeTree
diff --git a/tests/queries/0_stateless/02912_ingestion_mv_deduplication.sql b/tests/queries/0_stateless/02912_ingestion_mv_deduplication.sql
index f206f0d7775..a2378fd8f67 100644
--- a/tests/queries/0_stateless/02912_ingestion_mv_deduplication.sql
+++ b/tests/queries/0_stateless/02912_ingestion_mv_deduplication.sql
@@ -54,8 +54,9 @@ SELECT '-- Original issue with deduplicate_blocks_in_dependent_materialized_view
 
     - 1st insert works for landing and mv tables
     - 2nd insert gets first block 20220901 deduplicated and second one inserted in landing table
-    - 2nd insert is not inserting anything in mv table due to a bug computing blocks to be discarded
+    - 2nd insert is not inserting anything in mv table due to a bug computing blocks to be discarded, now that block is inserted because deduplicate_blocks_in_dependent_materialized_views=0
 
+    Now it is fixed.
 */
 SET deduplicate_blocks_in_dependent_materialized_views = 0, max_insert_delayed_streams_for_parallel_write = 1000;
 
@@ -97,7 +98,7 @@ SELECT '-- Original issue with deduplicate_blocks_in_dependent_materialized_view
     This is what happens now:
 
     - 1st insert works for landing and mv tables
-    - 2nd insert gets first block 20220901 deduplicated and second one inserted for landing and mv tables
+    - 2nd insert gets first block 20220901 deduplicated for landing and both rows are inserted for mv tables
 
 */
 SET deduplicate_blocks_in_dependent_materialized_views = 1, max_insert_delayed_streams_for_parallel_write = 1000;
diff --git a/tests/queries/0_stateless/03008_deduplication.python b/tests/queries/0_stateless/03008_deduplication.python
new file mode 100644
index 00000000000..dd1058518c9
--- /dev/null
+++ b/tests/queries/0_stateless/03008_deduplication.python
@@ -0,0 +1,657 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+import argparse
+import string
+
+
+CURDIR = os.path.dirname(os.path.realpath(__file__))
+sys.path.insert(0, os.path.join(CURDIR, "helpers"))
+
+
+def __format(template, **params):
+    field_names = [v[1] for v in string.Formatter().parse(template) if v[1] is not None]
+    kv_args = {}
+    for field in field_names:
+        if field in params:
+            kv_args[field] = params[field]
+        else:
+            kv_args[field] = ""
+
+    return template.format(**kv_args)
+
+
+def instance_create_statement(
+    table_name,
+    table_columns,
+    table_keys,
+    table_engine,
+    with_deduplication,
+    no_merges=True,
+):
+    template = """
+        CREATE TABLE {table_name}
+        {table_columns}
+        ENGINE = {table_engine}
+        ORDER BY {table_keys}
+        {table_settings};
+        {table_no_merges}
+        """
+
+    params = dict()
+    params["table_name"] = table_name
+    params["table_columns"] = table_columns
+    params["table_keys"] = table_keys
+    params["table_no_merges"] = f"SYSTEM STOP MERGES {table_name};" if no_merges else ""
+    params["table_engine"] = (
+        "MergeTree()"
+        if table_engine == "MergeTree"
+        else f"ReplicatedMergeTree('/clickhouse/tables/{{database}}/{table_name}', '1')"
+    )
+
+    deduplication_window_setting_name = (
+        "non_replicated_deduplication_window"
+        if table_engine == "MergeTree"
+        else "replicated_deduplication_window"
+    )
+    deduplication_window_setting_value = 1000 if with_deduplication else 0
+
+    settings = list()
+    settings += [
+        f"{deduplication_window_setting_name}={deduplication_window_setting_value}"
+    ]
+    params["table_settings"] = "SETTINGS " + ",".join(settings)
+
+    return __format(template, **params)
+
+
+def instance_insert_statement(
+    table_name, count, insert_method, insert_unique_blocks, use_insert_token
+):
+    insert_settings = (
+        "" if not use_insert_token else "SETTINGS insert_deduplication_token='UDT'"
+    )
+
+    if insert_method == "InsertSelect":
+        template = """
+            INSERT INTO {table_name}
+                SELECT {insert_columns}
+                FROM numbers({count}) {insert_settings};
+            """
+        return __format(
+            template,
+            table_name=table_name,
+            count=count,
+            insert_columns="'src_4', 4"
+            if not insert_unique_blocks
+            else "'src_' || toString(number), number",
+            insert_settings=insert_settings,
+        )
+
+    else:
+        template = """
+                    INSERT INTO {table_name}
+                        {insert_settings} VALUES {insert_values};
+                    """
+
+        values = []
+        for i in range(count):
+            values += (
+                [f"('src_{i}', {i})"] if insert_unique_blocks else ["('src_4', 4)"]
+            )
+        insert_values = ", ".join(values)
+
+        return __format(
+            template,
+            table_name=table_name,
+            insert_settings=insert_settings,
+            insert_values=insert_values,
+        )
+
+
+def get_drop_tables_statements(tables):
+    return "".join(
+        [f"DROP TABLE IF EXISTS {table_name};\n" for table_name in tables[::-1]]
+    )
+
+
+def get_logs_statement(args):
+    if args.get_logs:
+        return "SET send_logs_level='test';"
+    return ""
+
+
+def str2bool(v):
+    if isinstance(v, bool):
+        return v
+    if v.lower() in ("yes", "true", "t", "y", "1"):
+        return True
+    elif v.lower() in ("no", "false", "f", "n", "0"):
+        return False
+    else:
+        raise argparse.ArgumentTypeError("Boolean value expected.")
+
+
+class ArgsFactory:
+    def __init__(self, parser):
+        self.__parser = parser
+
+    def add_opt_engine(self):
+        self.__parser.add_argument(
+            "--table-engine",
+            choices=["ReplicatedMergeTree", "MergeTree"],
+            default="MergeTree",
+        )
+
+    def add_opt_user_token(self):
+        self.__parser.add_argument(
+            "--use-insert-token", type=str2bool, nargs="?", const=True, default=False
+        )
+
+    def add_opt_single_thread(self):
+        self.__parser.add_argument(
+            "--single-thread", type=str2bool, nargs="?", const=True, default=True
+        )
+
+    def add_opt_dedup_src(self):
+        self.__parser.add_argument(
+            "--deduplicate-src-table",
+            type=str2bool,
+            nargs="?",
+            const=True,
+            default=True,
+        )
+
+    def add_opt_dedup_dst(self):
+        self.__parser.add_argument(
+            "--deduplicate-dst-table",
+            type=str2bool,
+            nargs="?",
+            const=True,
+            default=True,
+        )
+
+    def add_opt_get_logs(self):
+        self.__parser.add_argument(
+            "--get-logs", type=str2bool, nargs="?", const=True, default=False
+        )
+
+    def add_opt_uniq_blocks(self):
+        self.__parser.add_argument(
+            "--insert-unique-blocks", type=str2bool, nargs="?", const=True, default=True
+        )
+
+    def add_opt_insert_method(self):
+        self.__parser.add_argument(
+            "--insert-method",
+            choices=["InsertSelect", "InsertValues"],
+            default="InsertSelect",
+        )
+
+    def add_all(self):
+        self.add_opt_engine()
+        self.add_opt_user_token()
+        self.add_opt_single_thread()
+        self.add_opt_dedup_src()
+        self.add_opt_dedup_dst()
+        self.add_opt_get_logs()
+        self.add_opt_insert_method()
+        self.add_opt_uniq_blocks()
+
+
+def test_insert_several_blocks(parser):
+    ArgsFactory(parser).add_all()
+
+    def calle(args):
+        create_table_a_b_statement = instance_create_statement(
+            table_name="table_a_b",
+            table_columns="(a String, b UInt64)",
+            table_keys="(a, b)",
+            table_engine=args.table_engine,
+            with_deduplication=args.deduplicate_src_table,
+        )
+
+        create_table_when_b_even_statement = instance_create_statement(
+            table_name="table_when_b_even",
+            table_columns="(a String, b UInt64)",
+            table_keys="(a, b)",
+            table_engine=args.table_engine,
+            with_deduplication=args.deduplicate_dst_table,
+        )
+
+        create_mv_statement = """
+            CREATE MATERIALIZED VIEW mv_b_even
+            TO table_when_b_even
+            AS
+                SELECT a, b
+                FROM table_a_b
+                WHERE b % 2 = 0;
+            """
+
+        drop_tables_statements = get_drop_tables_statements(
+            ["table_a_b", "table_when_b_even", "mv_b_even"]
+        )
+
+        insert_statement = instance_insert_statement(
+            "table_a_b",
+            10,
+            args.insert_method,
+            args.insert_unique_blocks,
+            args.use_insert_token,
+        )
+
+        print_details_statements = f"""
+            SELECT 'table_a_b';
+            SELECT 'count', count() FROM table_a_b;
+            {"" if not args.get_logs else "SELECT _part, count() FROM table_a_b GROUP BY _part ORDER BY _part;"}
+
+            SELECT 'table_when_b_even';
+            SELECT 'count', count() FROM table_when_b_even;
+            {"" if not args.get_logs else "SELECT _part, count() FROM table_when_b_even GROUP BY _part ORDER BY _part;"}
+            """
+
+        if args.insert_unique_blocks:
+            assert_first_insert_statements = f"""
+                SELECT throwIf( count() != 10 )
+                    FROM table_a_b;
+                SELECT throwIf( count() != 5 )
+                    FROM table_when_b_even;
+                """
+            assert_second_insert_statements = f"""
+               SELECT throwIf( count() != {10 if args.deduplicate_src_table else 20} )
+                   FROM table_a_b;
+               SELECT throwIf( count() != {5 if args.deduplicate_dst_table else 10} )
+                   FROM table_when_b_even;
+               """
+        else:
+            if args.use_insert_token:
+                assert_first_insert_statements = """
+                                SELECT throwIf( count() != 10 )
+                                    FROM table_a_b;
+                                SELECT throwIf( count() != 10 )
+                                    FROM table_when_b_even;
+                                """
+                assert_second_insert_statements = f"""
+                               SELECT throwIf( count() != {10 if args.deduplicate_src_table else 20} )
+                                   FROM table_a_b;
+                               SELECT throwIf( count() != {10 if args.deduplicate_dst_table else 20} )
+                                   FROM table_when_b_even;
+                               """
+            else:
+                assert_first_insert_statements = f"""
+                    SELECT throwIf( count() != {1 if args.deduplicate_src_table else 10} )
+                        FROM table_a_b;
+                    SELECT throwIf( count() != {1 if args.deduplicate_dst_table else 10} )
+                        FROM table_when_b_even;
+                    """
+                assert_second_insert_statements = f"""
+                    SELECT throwIf( count() != {1 if args.deduplicate_src_table else 20} )
+                        FROM table_a_b;
+                    SELECT throwIf( count() != {1 if args.deduplicate_dst_table else 20} )
+                        FROM table_when_b_even;
+                    """
+
+        script = f"""
+            {get_logs_statement(args)}
+
+            SET max_insert_threads={1 if args.single_thread else 10};
+            SET update_insert_deduplication_token_in_dependent_materialized_views=1;
+            SET deduplicate_blocks_in_dependent_materialized_views=1;
+
+            SET max_block_size=1;
+            SET min_insert_block_size_rows=0;
+            SET min_insert_block_size_bytes=0;
+
+            {drop_tables_statements}
+
+            {create_table_a_b_statement}
+
+            {create_table_when_b_even_statement}
+
+            {create_mv_statement}
+
+            -- first insert
+            {insert_statement}
+
+            {print_details_statements}
+
+            {assert_first_insert_statements}
+
+            -- second insert, it is retry
+            {insert_statement}
+
+            {print_details_statements}
+
+            {assert_second_insert_statements}
+
+            {drop_tables_statements}
+            """
+
+        print(script)
+
+    parser.set_defaults(func=calle)
+
+
+def test_mv_generates_several_blocks(parser):
+    ArgsFactory(parser).add_all()
+
+    def calle(args):
+        tables = [
+            "table_for_join_with",
+            "table_a_b",
+            "table_when_b_even_and_joined",
+            "mv_b_even",
+        ]
+        drop_tables_statements = get_drop_tables_statements(tables)
+
+        details_print_for_table_for_join_with = ""
+        if args.get_logs:
+            details_print_for_table_for_join_with = """
+                SELECT 'table_for_join_with';
+                SELECT a_join, b, _part FROM table_for_join_with ORDER BY _part, a_join, b;
+                """
+
+        create_table_a_b_statement = instance_create_statement(
+            table_name="table_a_b",
+            table_columns="(a_src String, b UInt64)",
+            table_keys="(a_src, b)",
+            table_engine=args.table_engine,
+            with_deduplication=args.deduplicate_src_table,
+        )
+
+        create_table_when_b_even_and_joined_statement = instance_create_statement(
+            table_name="table_when_b_even_and_joined",
+            table_columns="(a_src String, a_join String, b UInt64)",
+            table_keys="(a_src, a_join, b)",
+            table_engine=args.table_engine,
+            with_deduplication=args.deduplicate_dst_table,
+        )
+
+        insert_statement = instance_insert_statement(
+            "table_a_b",
+            5,
+            args.insert_method,
+            args.insert_unique_blocks,
+            args.use_insert_token,
+        )
+
+        details_print_statements = f"""
+            SELECT 'table_a_b';
+            SELECT 'count', count() FROM table_a_b;
+
+            SELECT 'table_when_b_even_and_joined';
+            SELECT 'count', count() FROM table_when_b_even_and_joined;
+            {"" if not args.get_logs else "SELECT _part, a_src, a_join, b FROM table_when_b_even_and_joined ORDER BY _part;"}
+        """
+
+        if args.insert_unique_blocks:
+            assert_first_insert_statements = f"""
+                SELECT throwIf( count() != 5 )
+                    FROM table_a_b;
+
+                SELECT throwIf( count() != 9 )
+                    FROM table_when_b_even_and_joined;
+                """
+            assert_second_insert_statements = f"""
+                SELECT throwIf( count() != {5 if args.deduplicate_src_table else 10} )
+                    FROM table_a_b;
+
+                SELECT throwIf( count() != {9 if args.deduplicate_dst_table else 18} )
+                    FROM table_when_b_even_and_joined;
+                """
+        else:
+            if args.use_insert_token:
+                assert_first_insert_statements = f"""
+                    SELECT throwIf( count() != {5 if args.deduplicate_src_table else 5} )
+                        FROM table_a_b;
+
+                    SELECT throwIf( count() != {10 if args.deduplicate_dst_table else 10} )
+                        FROM table_when_b_even_and_joined;
+                    """
+                assert_second_insert_statements = f"""
+                    SELECT throwIf( count() != {5 if args.deduplicate_src_table else 10} )
+                        FROM table_a_b;
+
+                    SELECT throwIf( count() != {10 if args.deduplicate_dst_table else 20} )
+                        FROM table_when_b_even_and_joined;
+                    """
+            else:
+                assert_first_insert_statements = f"""
+                    SELECT throwIf( count() != {1 if args.deduplicate_src_table else 5} )
+                       FROM table_a_b;
+
+                    SELECT throwIf( count() != {2 if args.deduplicate_dst_table else 10} )
+                       FROM table_when_b_even_and_joined;
+                    """
+                assert_second_insert_statements = f"""
+                    SELECT throwIf( count() != {1 if args.deduplicate_src_table else 10} )
+                        FROM table_a_b;
+
+                    SELECT throwIf( count() != {2 if args.deduplicate_dst_table else 20} )
+                        FROM table_when_b_even_and_joined;
+                    """
+
+        script = f"""
+            {get_logs_statement(args)}
+
+            SET max_insert_threads={1 if args.single_thread else 10};
+            SET update_insert_deduplication_token_in_dependent_materialized_views=1;
+            SET deduplicate_blocks_in_dependent_materialized_views=1;
+
+            SET max_block_size=1;
+            SET min_insert_block_size_rows=0;
+            SET min_insert_block_size_bytes=0;
+
+            {drop_tables_statements}
+
+            CREATE TABLE table_for_join_with
+                (a_join String, b UInt64)
+                ENGINE = MergeTree()
+                ORDER BY (a_join, b);
+            INSERT INTO table_for_join_with
+                SELECT 'joined_' || toString(number), number
+                FROM numbers(1);
+            {details_print_for_table_for_join_with}
+
+            {create_table_a_b_statement}
+            SYSTEM STOP MERGES table_a_b;
+
+            {create_table_when_b_even_and_joined_statement}
+            SYSTEM STOP MERGES table_when_b_even_and_joined;
+
+            CREATE MATERIALIZED VIEW mv_b_even
+                TO table_when_b_even_and_joined
+                AS
+                    SELECT a_src, a_join, table_for_join_with.b as b
+                        FROM table_a_b
+                        FULL OUTER JOIN table_for_join_with
+                        ON table_a_b.b = table_for_join_with.b AND table_a_b.b % 2 = 0
+                        ORDER BY a_src, a_join, b;
+
+            -- first insert
+            {insert_statement}
+
+            {details_print_statements}
+
+            -- first assertion
+            {assert_first_insert_statements}
+
+            -- second insert
+            {insert_statement}
+
+            {details_print_statements}
+
+            -- second assertion
+            {assert_second_insert_statements}
+
+            {drop_tables_statements}
+        """
+
+        print(script)
+
+    parser.set_defaults(func=calle)
+
+
+def test_several_mv_into_one_table(parser):
+    ArgsFactory(parser).add_all()
+
+    def calle(args):
+        tables = ["table_src", "table_dst", "mv_b_even", "mv_b_even_even"]
+        drop_tables_statements = get_drop_tables_statements(tables)
+
+        create_table_src_statement = instance_create_statement(
+            table_name="table_src",
+            table_columns="(a String, b UInt64)",
+            table_keys="(a, b)",
+            table_engine=args.table_engine,
+            with_deduplication=args.deduplicate_src_table,
+        )
+
+        create_table_dst_statement = instance_create_statement(
+            table_name="table_dst",
+            table_columns="(a String, b UInt64)",
+            table_keys="(a, b)",
+            table_engine=args.table_engine,
+            with_deduplication=args.deduplicate_dst_table,
+        )
+
+        insert_statement = instance_insert_statement(
+            "table_src",
+            8,
+            args.insert_method,
+            args.insert_unique_blocks,
+            args.use_insert_token,
+        )
+
+        details_print_statements = f"""
+            SELECT 'table_src count', count() FROM table_src;
+
+            SELECT 'table_dst count', count() FROM table_dst;
+            {"" if not args.get_logs else "SELECT _part, count() FROM table_dst GROUP BY _part ORDER BY _part;"}
+        """
+
+        if args.insert_unique_blocks:
+            assert_first_insert_statements = f"""
+                SELECT throwIf( count() != 8 )
+                    FROM table_src;
+
+                SELECT throwIf( count() != 6 )
+                    FROM table_dst;
+                """
+            assert_second_insert_statements = f"""
+                SELECT throwIf( count() != {8 if args.deduplicate_src_table else 16} )
+                    FROM table_src;
+
+                SELECT throwIf( count() != {6 if args.deduplicate_dst_table else 12} )
+                    FROM table_dst;
+                """
+        else:
+            if args.use_insert_token:
+                assert_first_insert_statements = f"""
+                    SELECT throwIf( count() != {8 if args.deduplicate_src_table else 8} )
+                        FROM table_src;
+
+                    SELECT throwIf( count() != {16 if args.deduplicate_dst_table else 16} )
+                        FROM table_dst;
+                    """
+                assert_second_insert_statements = f"""
+                    SELECT throwIf( count() != {8 if args.deduplicate_src_table else 16} )
+                        FROM table_src;
+
+                    SELECT throwIf( count() != {16 if args.deduplicate_dst_table else 32} )
+                        FROM table_dst;
+                    """
+            else:
+                assert_first_insert_statements = f"""
+                    SELECT throwIf( count() != {1 if args.deduplicate_src_table else 8} )
+                        FROM table_src;
+
+                    SELECT throwIf( count() != {2 if args.deduplicate_dst_table else 16} )
+                        FROM table_dst;
+                    """
+                assert_second_insert_statements = f"""
+                    SELECT throwIf( count() != {1 if args.deduplicate_src_table else 16} )
+                        FROM table_src;
+
+                    SELECT throwIf( count() != {2 if args.deduplicate_dst_table else 32} )
+                        FROM table_dst;
+                    """
+
+        script = f"""
+            {get_logs_statement(args)}
+
+            SET max_insert_threads={1 if args.single_thread else 10};
+            SET update_insert_deduplication_token_in_dependent_materialized_views=1;
+            SET deduplicate_blocks_in_dependent_materialized_views=1;
+
+            SET max_block_size=1;
+            SET min_insert_block_size_rows=0;
+            SET min_insert_block_size_bytes=0;
+
+            {drop_tables_statements}
+
+            {create_table_src_statement}
+
+            {create_table_dst_statement}
+
+            CREATE MATERIALIZED VIEW mv_b_even
+                TO table_dst
+                AS
+                SELECT a, b
+                    FROM table_src
+                    WHERE b % 2 = 0;
+
+            CREATE MATERIALIZED VIEW mv_b_even_even
+                TO table_dst
+                AS
+                SELECT a, b
+                    FROM table_src
+                    WHERE b % 4 = 0;
+
+            -- first insert
+            {insert_statement}
+
+            {details_print_statements}
+
+            {assert_first_insert_statements}
+
+            -- second insert, retry
+            {insert_statement}
+
+            {details_print_statements}
+
+            {assert_second_insert_statements}
+
+            {drop_tables_statements}
+            """
+
+        print(script)
+
+    parser.set_defaults(func=calle)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    subparsers = parser.add_subparsers(dest="test")
+    test_insert_several_blocks(
+        subparsers.add_parser("insert_several_blocks_into_table")
+    )
+    test_mv_generates_several_blocks(
+        subparsers.add_parser("mv_generates_several_blocks")
+    )
+    test_several_mv_into_one_table(subparsers.add_parser("several_mv_into_one_table"))
+    args = parser.parse_args()
+    if args.test is None:
+        parser.print_help()
+    return args
+
+
+def main():
+    args = parse_args()
+    if args.test is not None:
+        args.func(args)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/tests/queries/0_stateless/03008_deduplication_cases_from_docs.reference b/tests/queries/0_stateless/03008_deduplication_cases_from_docs.reference
new file mode 100644
index 00000000000..4893274c1cd
--- /dev/null
+++ b/tests/queries/0_stateless/03008_deduplication_cases_from_docs.reference
@@ -0,0 +1,41 @@
+Different materialized view insert into one underlayed table equal data.
+first attempt
+from dst	1	A	all_1_1_0
+from mv_dst	0	A	all_1_1_0
+from mv_dst	0	A	all_2_2_0
+second attempt
+from dst	1	A	all_1_1_0
+from mv_dst	0	A	all_1_1_0
+from mv_dst	0	A	all_2_2_0
+Different insert operations generate the same data after transformation in underlied table of materialized view.
+first attempt
+from dst	1	A	all_1_1_0
+from mv_dst	0	A	all_1_1_0
+second attempt
+from dst	1	A	all_1_1_0
+from dst	2	A	all_2_2_0
+from mv_dst	0	A	all_1_1_0
+from mv_dst	0	A	all_2_2_0
+Indentical blocks in insertion with `insert_deduplication_token`
+first attempt
+from dst	0	A	all_1_1_0
+from dst	0	A	all_2_2_0
+second attempt
+from dst	0	A	all_1_1_0
+from dst	0	A	all_2_2_0
+third attempt
+from dst	0	A	all_1_1_0
+from dst	0	A	all_2_2_0
+Indentical blocks in insertion
+from dst	0	A	all_1_1_0
+Indentical blocks after materialised view`s transformation
+first attempt
+from dst	1	B	all_1_1_0
+from dst	2	B	all_2_2_0
+from mv_dst	0	B	all_1_1_0
+from mv_dst	0	B	all_2_2_0
+second attempt
+from dst	1	B	all_1_1_0
+from dst	2	B	all_2_2_0
+from mv_dst	0	B	all_1_1_0
+from mv_dst	0	B	all_2_2_0
diff --git a/tests/queries/0_stateless/03008_deduplication_cases_from_docs.sql b/tests/queries/0_stateless/03008_deduplication_cases_from_docs.sql
new file mode 100644
index 00000000000..7927a6b1edf
--- /dev/null
+++ b/tests/queries/0_stateless/03008_deduplication_cases_from_docs.sql
@@ -0,0 +1,331 @@
+-- #########
+select 'Different materialized view insert into one underlayed table equal data.';
+
+DROP TABLE IF EXISTS dst;
+DROP TABLE IF EXISTS mv_dst;
+DROP TABLE IF EXISTS mv_first;
+DROP TABLE IF EXISTS mv_second;
+
+CREATE TABLE dst
+(
+    `key` Int64,
+    `value` String
+)
+ENGINE = MergeTree
+ORDER BY tuple()
+SETTINGS non_replicated_deduplication_window=1000;
+
+CREATE TABLE mv_dst
+(
+    `key` Int64,
+    `value` String
+)
+ENGINE = MergeTree
+ORDER BY tuple()
+SETTINGS non_replicated_deduplication_window=1000;
+
+CREATE MATERIALIZED VIEW mv_first
+TO mv_dst
+AS SELECT
+    0 AS key,
+    value AS value
+FROM dst;
+
+CREATE MATERIALIZED VIEW mv_second
+TO mv_dst
+AS SELECT
+    0 AS key,
+    value AS value
+FROM dst;
+
+SET deduplicate_blocks_in_dependent_materialized_views=1;
+
+select 'first attempt';
+
+INSERT INTO dst VALUES (1, 'A');
+
+SELECT
+    'from dst',
+    *,
+    _part
+FROM dst
+ORDER by all;
+
+SELECT
+    'from mv_dst',
+    *,
+    _part
+FROM mv_dst
+ORDER by all;
+
+select 'second attempt';
+
+INSERT INTO dst VALUES (1, 'A');
+
+SELECT
+    'from dst',
+    *,
+    _part
+FROM dst
+ORDER by all;
+
+SELECT
+    'from mv_dst',
+    *,
+    _part
+FROM mv_dst
+ORDER by all;
+
+DROP TABLE mv_second;
+DROP TABLE mv_first;
+DROP TABLE mv_dst;
+DROP TABLE dst;
+
+
+-- #########
+select 'Different insert operations generate the same data after transformation in underlied table of materialized view.';
+
+DROP TABLE IF EXISTS dst;
+DROP TABLE IF EXISTS mv_dst;
+
+CREATE TABLE dst
+(
+    `key` Int64,
+    `value` String
+)
+ENGINE = MergeTree
+ORDER BY tuple()
+SETTINGS non_replicated_deduplication_window=1000;
+
+CREATE MATERIALIZED VIEW mv_dst
+(
+    `key` Int64,
+    `value` String
+)
+ENGINE = MergeTree
+ORDER BY tuple()
+SETTINGS non_replicated_deduplication_window=1000
+AS SELECT
+    0 AS key,
+    value AS value
+FROM dst;
+
+SET deduplicate_blocks_in_dependent_materialized_views=1;
+
+select 'first attempt';
+
+INSERT INTO dst VALUES (1, 'A');
+
+SELECT
+    'from dst',
+    *,
+    _part
+FROM dst
+ORDER by all;
+
+SELECT
+    'from mv_dst',
+    *,
+    _part
+FROM mv_dst
+ORDER by all;
+
+select 'second attempt';
+
+INSERT INTO dst VALUES (2, 'A');
+
+SELECT
+    'from dst',
+    *,
+    _part
+FROM dst
+ORDER by all;
+
+SELECT
+    'from mv_dst',
+    *,
+    _part
+FROM mv_dst
+ORDER by all;
+
+DROP TABLE mv_dst;
+DROP TABLE dst;
+
+
+-- #########
+select 'Indentical blocks in insertion with `insert_deduplication_token`';
+
+DROP TABLE IF EXISTS dst;
+
+CREATE TABLE dst
+(
+    `key` Int64,
+    `value` String
+)
+ENGINE = MergeTree
+ORDER BY tuple()
+SETTINGS non_replicated_deduplication_window=1000;
+
+SET max_block_size=1;
+SET min_insert_block_size_rows=0;
+SET min_insert_block_size_bytes=0;
+
+select 'first attempt';
+
+INSERT INTO dst SELECT
+    0 AS key,
+    'A' AS value
+FROM numbers(2)
+SETTINGS insert_deduplication_token='some_user_token';
+
+SELECT
+    'from dst',
+    *,
+    _part
+FROM dst
+ORDER by all;
+
+select 'second attempt';
+
+INSERT INTO dst SELECT
+    0 AS key,
+    'A' AS value
+FROM numbers(2)
+SETTINGS insert_deduplication_token='some_user_token';
+
+SELECT
+    'from dst',
+    *,
+    _part
+FROM dst
+ORDER by all;
+
+select 'third attempt';
+
+INSERT INTO dst SELECT
+    1 AS key,
+    'b' AS value
+FROM numbers(2)
+SETTINGS insert_deduplication_token='some_user_token';
+
+SELECT
+    'from dst',
+    *,
+    _part
+FROM dst
+ORDER by all;
+
+DROP TABLE dst;
+
+
+-- #########
+select 'Indentical blocks in insertion';
+
+DROP TABLE IF EXISTS dst;
+
+CREATE TABLE dst
+(
+    `key` Int64,
+    `value` String
+)
+ENGINE = MergeTree
+ORDER BY tuple()
+SETTINGS non_replicated_deduplication_window=1000;
+
+SET max_block_size=1;
+SET min_insert_block_size_rows=0;
+SET min_insert_block_size_bytes=0;
+
+INSERT INTO dst SELECT
+    0 AS key,
+    'A' AS value
+FROM numbers(2);
+
+SELECT
+    'from dst',
+    *,
+    _part
+FROM dst
+ORDER by all;
+
+DROP TABLE dst;
+
+
+-- #########
+select 'Indentical blocks after materialised view`s transformation';
+
+DROP TABLE IF EXISTS dst;
+DROP TABLE IF EXISTS mv_dst;
+
+CREATE TABLE dst
+(
+    `key` Int64,
+    `value` String
+)
+ENGINE = MergeTree
+ORDER BY tuple()
+SETTINGS non_replicated_deduplication_window=1000;
+
+CREATE MATERIALIZED VIEW mv_dst
+(
+    `key` Int64,
+    `value` String
+)
+ENGINE = MergeTree
+ORDER BY tuple()
+SETTINGS non_replicated_deduplication_window=1000
+AS SELECT
+    0 AS key,
+    value AS value
+FROM dst;
+
+SET max_block_size=1;
+SET min_insert_block_size_rows=0;
+SET min_insert_block_size_bytes=0;
+
+SET deduplicate_blocks_in_dependent_materialized_views=1;
+
+select 'first attempt';
+
+INSERT INTO dst SELECT
+    number + 1 AS key,
+    IF(key = 0, 'A', 'B') AS value
+FROM numbers(2);
+
+SELECT
+    'from dst',
+    *,
+    _part
+FROM dst
+ORDER by all;
+
+SELECT
+    'from mv_dst',
+    *,
+    _part
+FROM mv_dst
+ORDER by all;
+
+select 'second attempt';
+
+INSERT INTO dst SELECT
+    number + 1 AS key,
+    IF(key = 0, 'A', 'B') AS value
+FROM numbers(2);
+
+SELECT
+    'from dst',
+    *,
+    _part
+FROM dst
+ORDER by all;
+
+SELECT
+    'from mv_dst',
+    *,
+    _part
+FROM mv_dst
+ORDER by all;
+
+DROP TABLE mv_dst;
+DROP TABLE dst;
diff --git a/tests/queries/0_stateless/03008_deduplication_insert_into_partitioned_table.reference b/tests/queries/0_stateless/03008_deduplication_insert_into_partitioned_table.reference
new file mode 100644
index 00000000000..c82a6eaa213
--- /dev/null
+++ b/tests/queries/0_stateless/03008_deduplication_insert_into_partitioned_table.reference
@@ -0,0 +1,35 @@
+no user deduplication token
+partitioned_table is deduplicated bacause deduplication works in scope of one partiotion:
+1	A
+1	D
+2	B
+2	C
+mv_table is not deduplicated because the inserted blocks was different:
+1	A
+1	A
+1	D
+2	B
+2	B
+2	C
+with user deduplication token
+partitioned_table is not deduplicated because different tokens:
+1	A
+1	A
+1	D
+2	B
+2	B
+2	C
+mv_table is not deduplicated because different tokens:
+1	A
+1	A
+1	D
+2	B
+2	B
+2	C
+with incorrect ussage of user deduplication token
+partitioned_table is deduplicated because equal tokens:
+1	A
+2	B
+mv_table is deduplicated because equal tokens:
+1	A
+2	B
diff --git a/tests/queries/0_stateless/03008_deduplication_insert_into_partitioned_table.sql b/tests/queries/0_stateless/03008_deduplication_insert_into_partitioned_table.sql
new file mode 100644
index 00000000000..2eb931f7f73
--- /dev/null
+++ b/tests/queries/0_stateless/03008_deduplication_insert_into_partitioned_table.sql
@@ -0,0 +1,83 @@
+DROP TABLE IF EXISTS partitioned_table;
+DROP TABLE IF EXISTS mv_table;
+
+
+SET deduplicate_blocks_in_dependent_materialized_views = 1;
+
+
+SELECT 'no user deduplication token';
+
+CREATE TABLE partitioned_table
+    (key Int64, value String)
+    ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/03008_deduplication_insert_into_partitioned_table', '{replica}')
+    partition by key % 10
+    order by tuple();
+
+CREATE MATERIALIZED VIEW mv_table (key Int64, value String)
+    ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/03008_deduplication_insert_into_partitioned_table_mv', '{replica}')
+    ORDER BY tuple()
+    AS SELECT key, value FROM partitioned_table;
+
+INSERT INTO partitioned_table VALUES (1, 'A'), (2, 'B');
+INSERT INTO partitioned_table VALUES (1, 'A'), (2, 'C');
+INSERT INTO partitioned_table VALUES (1, 'D'), (2, 'B');
+
+SELECT 'partitioned_table is deduplicated bacause deduplication works in scope of one partiotion:';
+SELECT * FROM partitioned_table ORDER BY ALL;
+SELECT 'mv_table is not deduplicated because the inserted blocks was different:';
+SELECT * FROM mv_table ORDER BY ALL;
+
+DROP TABLE partitioned_table;
+DROP TABLE mv_table;
+
+
+SELECT 'with user deduplication token';
+
+CREATE TABLE partitioned_table
+    (key Int64, value String)
+    ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/03008_deduplication_insert_into_partitioned_table', '{replica}')
+    partition by key % 10
+    order by tuple();
+
+CREATE MATERIALIZED VIEW mv_table (key Int64, value String)
+    ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/03008_deduplication_insert_into_partitioned_table_mv', '{replica}')
+    ORDER BY tuple()
+    AS SELECT key, value FROM partitioned_table;
+
+INSERT INTO partitioned_table SETTINGS insert_deduplication_token='token_1' VALUES (1, 'A'), (2, 'B');
+INSERT INTO partitioned_table SETTINGS insert_deduplication_token='token_2' VALUES (1, 'A'), (2, 'C');
+INSERT INTO partitioned_table SETTINGS insert_deduplication_token='token_3' VALUES (1, 'D'), (2, 'B');
+
+SELECT 'partitioned_table is not deduplicated because different tokens:';
+SELECT * FROM partitioned_table ORDER BY ALL;
+SELECT 'mv_table is not deduplicated because different tokens:';
+SELECT * FROM mv_table ORDER BY ALL;
+
+DROP TABLE partitioned_table;
+DROP TABLE mv_table;
+
+
+SELECT 'with incorrect ussage of user deduplication token';
+
+CREATE TABLE partitioned_table
+    (key Int64, value String)
+    ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/03008_deduplication_insert_into_partitioned_table', '{replica}')
+    partition by key % 10
+    order by tuple();
+
+CREATE MATERIALIZED VIEW mv_table (key Int64, value String)
+    ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/03008_deduplication_insert_into_partitioned_table_mv', '{replica}')
+    ORDER BY tuple()
+    AS SELECT key, value FROM partitioned_table;
+
+INSERT INTO partitioned_table SETTINGS insert_deduplication_token='token_0' VALUES (1, 'A'), (2, 'B');
+INSERT INTO partitioned_table SETTINGS insert_deduplication_token='token_0' VALUES (1, 'A'), (2, 'C');
+INSERT INTO partitioned_table SETTINGS insert_deduplication_token='token_0' VALUES (1, 'D'), (2, 'B');
+
+SELECT 'partitioned_table is deduplicated because equal tokens:';
+SELECT * FROM partitioned_table ORDER BY ALL;
+SELECT 'mv_table is deduplicated because equal tokens:';
+SELECT * FROM mv_table ORDER BY ALL;
+
+DROP TABLE partitioned_table;
+DROP TABLE mv_table;
diff --git a/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_nonreplicated.reference b/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_nonreplicated.reference
new file mode 100644
index 00000000000..bf900aa84d2
--- /dev/null
+++ b/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_nonreplicated.reference
@@ -0,0 +1,962 @@
+
+Test case 0: insert_method=InsertSelect engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+OK
+
+Test case 1: insert_method=InsertSelect engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 2: insert_method=InsertSelect engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 3: insert_method=InsertSelect engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	20
+0
+0
+OK
+
+Test case 4: insert_method=InsertSelect engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	5
+0
+0
+OK
+
+Test case 5: insert_method=InsertSelect engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 6: insert_method=InsertSelect engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 7: insert_method=InsertSelect engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	20
+0
+0
+OK
+
+Test case 8: insert_method=InsertSelect engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+OK
+
+Test case 9: insert_method=InsertSelect engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 10: insert_method=InsertSelect engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 11: insert_method=InsertSelect engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	20
+0
+0
+OK
+
+Test case 12: insert_method=InsertSelect engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	5
+0
+0
+OK
+
+Test case 13: insert_method=InsertSelect engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 14: insert_method=InsertSelect engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 15: insert_method=InsertSelect engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	20
+0
+0
+OK
+
+Test case 16: insert_method=InsertSelect engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+OK
+
+Test case 17: insert_method=InsertSelect engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	1
+table_when_b_even
+count	1
+0
+0
+table_a_b
+count	1
+table_when_b_even
+count	1
+0
+0
+OK
+
+Test case 18: insert_method=InsertSelect engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 19: insert_method=InsertSelect engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	1
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	1
+table_when_b_even
+count	20
+0
+0
+OK
+
+Test case 20: insert_method=InsertSelect engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	5
+0
+0
+OK
+
+Test case 21: insert_method=InsertSelect engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	1
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	1
+0
+0
+OK
+
+Test case 22: insert_method=InsertSelect engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 23: insert_method=InsertSelect engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	20
+0
+0
+OK
+
+Test case 24: insert_method=InsertSelect engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+OK
+
+Test case 25: insert_method=InsertSelect engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	1
+table_when_b_even
+count	1
+0
+0
+table_a_b
+count	1
+table_when_b_even
+count	1
+0
+0
+OK
+
+Test case 26: insert_method=InsertSelect engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 27: insert_method=InsertSelect engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	1
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	1
+table_when_b_even
+count	20
+0
+0
+OK
+
+Test case 28: insert_method=InsertSelect engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	5
+0
+0
+OK
+
+Test case 29: insert_method=InsertSelect engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	1
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	1
+0
+0
+OK
+
+Test case 30: insert_method=InsertSelect engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 31: insert_method=InsertSelect engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	20
+0
+0
+OK
+
+Test case 32: insert_method=InsertValues engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+OK
+
+Test case 33: insert_method=InsertValues engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 34: insert_method=InsertValues engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 35: insert_method=InsertValues engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	20
+0
+0
+OK
+
+Test case 36: insert_method=InsertValues engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	5
+0
+0
+OK
+
+Test case 37: insert_method=InsertValues engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 38: insert_method=InsertValues engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 39: insert_method=InsertValues engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	20
+0
+0
+OK
+
+Test case 40: insert_method=InsertValues engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+OK
+
+Test case 41: insert_method=InsertValues engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 42: insert_method=InsertValues engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 43: insert_method=InsertValues engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	20
+0
+0
+OK
+
+Test case 44: insert_method=InsertValues engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	5
+0
+0
+OK
+
+Test case 45: insert_method=InsertValues engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 46: insert_method=InsertValues engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 47: insert_method=InsertValues engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	20
+0
+0
+OK
+
+Test case 48: insert_method=InsertValues engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+OK
+
+Test case 49: insert_method=InsertValues engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	1
+table_when_b_even
+count	1
+0
+0
+table_a_b
+count	1
+table_when_b_even
+count	1
+0
+0
+OK
+
+Test case 50: insert_method=InsertValues engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 51: insert_method=InsertValues engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	1
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	1
+table_when_b_even
+count	20
+0
+0
+OK
+
+Test case 52: insert_method=InsertValues engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	5
+0
+0
+OK
+
+Test case 53: insert_method=InsertValues engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	1
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	1
+0
+0
+OK
+
+Test case 54: insert_method=InsertValues engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 55: insert_method=InsertValues engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	20
+0
+0
+OK
+
+Test case 56: insert_method=InsertValues engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+OK
+
+Test case 57: insert_method=InsertValues engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	1
+table_when_b_even
+count	1
+0
+0
+table_a_b
+count	1
+table_when_b_even
+count	1
+0
+0
+OK
+
+Test case 58: insert_method=InsertValues engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 59: insert_method=InsertValues engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	1
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	1
+table_when_b_even
+count	20
+0
+0
+OK
+
+Test case 60: insert_method=InsertValues engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	5
+0
+0
+OK
+
+Test case 61: insert_method=InsertValues engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	1
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	1
+0
+0
+OK
+
+Test case 62: insert_method=InsertValues engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 63: insert_method=InsertValues engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	20
+0
+0
+OK
+
+All cases executed
diff --git a/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_nonreplicated.sh b/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_nonreplicated.sh
new file mode 100755
index 00000000000..49eb52b47fd
--- /dev/null
+++ b/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_nonreplicated.sh
@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+# Tags: long, no-fasttest, no-parallel
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+ENGINE="MergeTree"
+
+RUN_ONLY=""
+#RUN_ONLY="Test case 52: insert_method=InsertSelect engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True"
+
+i=0
+for insert_method in "InsertSelect" "InsertValues"; do
+    for use_insert_token in "True" "False"; do
+        for single_thread in "True" "False"; do
+            for deduplicate_src_table in "True" "False"; do
+                for deduplicate_dst_table in "True" "False"; do
+                    for insert_unique_blocks in "True" "False"; do
+
+                        THIS_RUN="Test case $i:"
+                        THIS_RUN+=" insert_method=$insert_method"
+                        THIS_RUN+=" engine=$ENGINE"
+                        THIS_RUN+=" use_insert_token=$use_insert_token"
+                        THIS_RUN+=" single_thread=$single_thread"
+                        THIS_RUN+=" deduplicate_src_table=$deduplicate_src_table"
+                        THIS_RUN+=" deduplicate_dst_table=$deduplicate_dst_table"
+                        THIS_RUN+=" insert_unique_blocks=$insert_unique_blocks"
+
+                        i=$((i+1))
+
+                        echo
+                        if [ -n "$RUN_ONLY" ] && [ "$RUN_ONLY" != "$THIS_RUN" ]; then
+                            echo "skip $THIS_RUN"
+                            continue
+                        fi
+                        echo "$THIS_RUN"
+
+                        $CLICKHOUSE_CLIENT --max_insert_block_size 1  -nmq "
+                            $(python3 $CURDIR/03008_deduplication.python insert_several_blocks_into_table \
+                                --insert-method $insert_method \
+                                --table-engine $ENGINE \
+                                --use-insert-token $use_insert_token \
+                                --single-thread $single_thread \
+                                --deduplicate-src-table $deduplicate_src_table \
+                                --deduplicate-dst-table $deduplicate_dst_table \
+                                --insert-unique-blocks $insert_unique_blocks \
+                                --get-logs false \
+                            )
+                        " && echo OK || echo FAIL
+                    done
+                done
+            done
+        done
+    done
+done
+
+echo
+echo "All cases executed"
diff --git a/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_replicated.reference b/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_replicated.reference
new file mode 100644
index 00000000000..c815324b455
--- /dev/null
+++ b/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_replicated.reference
@@ -0,0 +1,962 @@
+
+Test case 0: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+OK
+
+Test case 1: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 2: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 3: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	20
+0
+0
+OK
+
+Test case 4: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	5
+0
+0
+OK
+
+Test case 5: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 6: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 7: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	20
+0
+0
+OK
+
+Test case 8: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+OK
+
+Test case 9: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 10: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 11: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	20
+0
+0
+OK
+
+Test case 12: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	5
+0
+0
+OK
+
+Test case 13: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 14: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 15: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	20
+0
+0
+OK
+
+Test case 16: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+OK
+
+Test case 17: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	1
+table_when_b_even
+count	1
+0
+0
+table_a_b
+count	1
+table_when_b_even
+count	1
+0
+0
+OK
+
+Test case 18: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 19: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	1
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	1
+table_when_b_even
+count	20
+0
+0
+OK
+
+Test case 20: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	5
+0
+0
+OK
+
+Test case 21: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	1
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	1
+0
+0
+OK
+
+Test case 22: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 23: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	20
+0
+0
+OK
+
+Test case 24: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+OK
+
+Test case 25: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	1
+table_when_b_even
+count	1
+0
+0
+table_a_b
+count	1
+table_when_b_even
+count	1
+0
+0
+OK
+
+Test case 26: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 27: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	1
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	1
+table_when_b_even
+count	20
+0
+0
+OK
+
+Test case 28: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	5
+0
+0
+OK
+
+Test case 29: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	1
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	1
+0
+0
+OK
+
+Test case 30: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 31: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	20
+0
+0
+OK
+
+Test case 32: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+OK
+
+Test case 33: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 34: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 35: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	20
+0
+0
+OK
+
+Test case 36: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	5
+0
+0
+OK
+
+Test case 37: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 38: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 39: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	20
+0
+0
+OK
+
+Test case 40: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+OK
+
+Test case 41: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 42: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 43: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	20
+0
+0
+OK
+
+Test case 44: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	5
+0
+0
+OK
+
+Test case 45: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 46: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 47: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	20
+0
+0
+OK
+
+Test case 48: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+OK
+
+Test case 49: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	1
+table_when_b_even
+count	1
+0
+0
+table_a_b
+count	1
+table_when_b_even
+count	1
+0
+0
+OK
+
+Test case 50: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 51: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	1
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	1
+table_when_b_even
+count	20
+0
+0
+OK
+
+Test case 52: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	5
+0
+0
+OK
+
+Test case 53: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	1
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	1
+0
+0
+OK
+
+Test case 54: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 55: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	20
+0
+0
+OK
+
+Test case 56: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+OK
+
+Test case 57: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	1
+table_when_b_even
+count	1
+0
+0
+table_a_b
+count	1
+table_when_b_even
+count	1
+0
+0
+OK
+
+Test case 58: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 59: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	1
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	1
+table_when_b_even
+count	20
+0
+0
+OK
+
+Test case 60: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	5
+0
+0
+OK
+
+Test case 61: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	1
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	1
+0
+0
+OK
+
+Test case 62: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	10
+table_when_b_even
+count	5
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	10
+0
+0
+OK
+
+Test case 63: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	10
+table_when_b_even
+count	10
+0
+0
+table_a_b
+count	20
+table_when_b_even
+count	20
+0
+0
+OK
+
+All cases executed
diff --git a/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_replicated.sh b/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_replicated.sh
new file mode 100755
index 00000000000..53af06d4a6f
--- /dev/null
+++ b/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_replicated.sh
@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+# Tags: long, no-fasttest, no-parallel
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+ENGINE="ReplicatedMergeTree"
+
+RUN_ONLY=""
+#RUN_ONLY="Test case 52: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True"
+
+i=0
+for insert_method in "InsertSelect" "InsertValues"; do
+    for use_insert_token in "True" "False"; do
+        for single_thread in "True" "False"; do
+            for deduplicate_src_table in "True" "False"; do
+                for deduplicate_dst_table in "True" "False"; do
+                    for insert_unique_blocks in "True" "False"; do
+
+                        THIS_RUN="Test case $i:"
+                        THIS_RUN+=" insert_method=$insert_method"
+                        THIS_RUN+=" engine=$ENGINE"
+                        THIS_RUN+=" use_insert_token=$use_insert_token"
+                        THIS_RUN+=" single_thread=$single_thread"
+                        THIS_RUN+=" deduplicate_src_table=$deduplicate_src_table"
+                        THIS_RUN+=" deduplicate_dst_table=$deduplicate_dst_table"
+                        THIS_RUN+=" insert_unique_blocks=$insert_unique_blocks"
+
+                        i=$((i+1))
+
+                        echo
+                        if [ -n "$RUN_ONLY" ] && [ "$RUN_ONLY" != "$THIS_RUN" ]; then
+                            echo "skip $THIS_RUN"
+                            continue
+                        fi
+                        echo "$THIS_RUN"
+
+                        $CLICKHOUSE_CLIENT --max_insert_block_size 1  -nmq "
+                            $(python3 $CURDIR/03008_deduplication.python insert_several_blocks_into_table \
+                                --insert-method $insert_method \
+                                --table-engine $ENGINE \
+                                --use-insert-token $use_insert_token \
+                                --single-thread $single_thread \
+                                --deduplicate-src-table $deduplicate_src_table \
+                                --deduplicate-dst-table $deduplicate_dst_table \
+                                --insert-unique-blocks $insert_unique_blocks \
+                                --get-logs false \
+                            )
+                        " && echo OK || echo FAIL
+                    done
+                done
+            done
+        done
+    done
+done
+
+echo
+echo "All cases executed"
diff --git a/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_nonreplicated.reference b/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_nonreplicated.reference
new file mode 100644
index 00000000000..6e76ec46aa8
--- /dev/null
+++ b/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_nonreplicated.reference
@@ -0,0 +1,962 @@
+
+Test case 0: insert_method=InsertSelect engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+OK
+
+Test case 1: insert_method=InsertSelect engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	10
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	10
+0
+0
+OK
+
+Test case 2: insert_method=InsertSelect engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	18
+0
+0
+OK
+
+Test case 3: insert_method=InsertSelect engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	10
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	20
+0
+0
+OK
+
+Test case 4: insert_method=InsertSelect engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	9
+0
+0
+OK
+
+Test case 5: insert_method=InsertSelect engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	10
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	10
+0
+0
+OK
+
+Test case 6: insert_method=InsertSelect engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	18
+0
+0
+OK
+
+Test case 7: insert_method=InsertSelect engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	10
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	20
+0
+0
+OK
+
+Test case 8: insert_method=InsertSelect engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+OK
+
+Test case 9: insert_method=InsertSelect engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	10
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	10
+0
+0
+OK
+
+Test case 10: insert_method=InsertSelect engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	18
+0
+0
+OK
+
+Test case 11: insert_method=InsertSelect engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	10
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	20
+0
+0
+OK
+
+Test case 12: insert_method=InsertSelect engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	9
+0
+0
+OK
+
+Test case 13: insert_method=InsertSelect engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	10
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	10
+0
+0
+OK
+
+Test case 14: insert_method=InsertSelect engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	18
+0
+0
+OK
+
+Test case 15: insert_method=InsertSelect engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	10
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	20
+0
+0
+OK
+
+Test case 16: insert_method=InsertSelect engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+OK
+
+Test case 17: insert_method=InsertSelect engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	1
+table_when_b_even_and_joined
+count	2
+0
+0
+table_a_b
+count	1
+table_when_b_even_and_joined
+count	2
+0
+0
+OK
+
+Test case 18: insert_method=InsertSelect engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	18
+0
+0
+OK
+
+Test case 19: insert_method=InsertSelect engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	1
+table_when_b_even_and_joined
+count	10
+0
+0
+table_a_b
+count	1
+table_when_b_even_and_joined
+count	20
+0
+0
+OK
+
+Test case 20: insert_method=InsertSelect engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	9
+0
+0
+OK
+
+Test case 21: insert_method=InsertSelect engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	2
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	2
+0
+0
+OK
+
+Test case 22: insert_method=InsertSelect engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	18
+0
+0
+OK
+
+Test case 23: insert_method=InsertSelect engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	10
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	20
+0
+0
+OK
+
+Test case 24: insert_method=InsertSelect engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+OK
+
+Test case 25: insert_method=InsertSelect engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	1
+table_when_b_even_and_joined
+count	2
+0
+0
+table_a_b
+count	1
+table_when_b_even_and_joined
+count	2
+0
+0
+OK
+
+Test case 26: insert_method=InsertSelect engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	18
+0
+0
+OK
+
+Test case 27: insert_method=InsertSelect engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	1
+table_when_b_even_and_joined
+count	10
+0
+0
+table_a_b
+count	1
+table_when_b_even_and_joined
+count	20
+0
+0
+OK
+
+Test case 28: insert_method=InsertSelect engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	9
+0
+0
+OK
+
+Test case 29: insert_method=InsertSelect engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	2
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	2
+0
+0
+OK
+
+Test case 30: insert_method=InsertSelect engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	18
+0
+0
+OK
+
+Test case 31: insert_method=InsertSelect engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	10
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	20
+0
+0
+OK
+
+Test case 32: insert_method=InsertValues engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+OK
+
+Test case 33: insert_method=InsertValues engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	10
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	10
+0
+0
+OK
+
+Test case 34: insert_method=InsertValues engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	18
+0
+0
+OK
+
+Test case 35: insert_method=InsertValues engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	10
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	20
+0
+0
+OK
+
+Test case 36: insert_method=InsertValues engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	9
+0
+0
+OK
+
+Test case 37: insert_method=InsertValues engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	10
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	10
+0
+0
+OK
+
+Test case 38: insert_method=InsertValues engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	18
+0
+0
+OK
+
+Test case 39: insert_method=InsertValues engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	10
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	20
+0
+0
+OK
+
+Test case 40: insert_method=InsertValues engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+OK
+
+Test case 41: insert_method=InsertValues engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	10
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	10
+0
+0
+OK
+
+Test case 42: insert_method=InsertValues engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	18
+0
+0
+OK
+
+Test case 43: insert_method=InsertValues engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	10
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	20
+0
+0
+OK
+
+Test case 44: insert_method=InsertValues engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	9
+0
+0
+OK
+
+Test case 45: insert_method=InsertValues engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	10
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	10
+0
+0
+OK
+
+Test case 46: insert_method=InsertValues engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	18
+0
+0
+OK
+
+Test case 47: insert_method=InsertValues engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	10
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	20
+0
+0
+OK
+
+Test case 48: insert_method=InsertValues engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+OK
+
+Test case 49: insert_method=InsertValues engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	1
+table_when_b_even_and_joined
+count	2
+0
+0
+table_a_b
+count	1
+table_when_b_even_and_joined
+count	2
+0
+0
+OK
+
+Test case 50: insert_method=InsertValues engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	18
+0
+0
+OK
+
+Test case 51: insert_method=InsertValues engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	1
+table_when_b_even_and_joined
+count	10
+0
+0
+table_a_b
+count	1
+table_when_b_even_and_joined
+count	20
+0
+0
+OK
+
+Test case 52: insert_method=InsertValues engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	9
+0
+0
+OK
+
+Test case 53: insert_method=InsertValues engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	2
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	2
+0
+0
+OK
+
+Test case 54: insert_method=InsertValues engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	18
+0
+0
+OK
+
+Test case 55: insert_method=InsertValues engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	10
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	20
+0
+0
+OK
+
+Test case 56: insert_method=InsertValues engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+OK
+
+Test case 57: insert_method=InsertValues engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	1
+table_when_b_even_and_joined
+count	2
+0
+0
+table_a_b
+count	1
+table_when_b_even_and_joined
+count	2
+0
+0
+OK
+
+Test case 58: insert_method=InsertValues engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	18
+0
+0
+OK
+
+Test case 59: insert_method=InsertValues engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	1
+table_when_b_even_and_joined
+count	10
+0
+0
+table_a_b
+count	1
+table_when_b_even_and_joined
+count	20
+0
+0
+OK
+
+Test case 60: insert_method=InsertValues engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	9
+0
+0
+OK
+
+Test case 61: insert_method=InsertValues engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	2
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	2
+0
+0
+OK
+
+Test case 62: insert_method=InsertValues engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	18
+0
+0
+OK
+
+Test case 63: insert_method=InsertValues engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	10
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	20
+0
+0
+OK
+
+All cases executed
diff --git a/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_nonreplicated.sh b/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_nonreplicated.sh
new file mode 100755
index 00000000000..7d4f5240cd1
--- /dev/null
+++ b/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_nonreplicated.sh
@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+# Tags: long, no-fasttest, no-parallel
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+ENGINE="MergeTree"
+
+RUN_ONLY=""
+#RUN_ONLY="Test case 20: engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True"
+
+i=0
+for insert_method in "InsertSelect" "InsertValues"; do
+    for use_insert_token in "True" "False"; do
+        for single_thread in "True" "False"; do
+            for deduplicate_src_table in "True" "False"; do
+                for deduplicate_dst_table in "True" "False"; do
+                    for insert_unique_blocks in "True" "False"; do
+
+                        THIS_RUN="Test case $i:"
+                        THIS_RUN+=" insert_method=$insert_method"
+                        THIS_RUN+=" engine=$ENGINE"
+                        THIS_RUN+=" use_insert_token=$use_insert_token"
+                        THIS_RUN+=" single_thread=$single_thread"
+                        THIS_RUN+=" deduplicate_src_table=$deduplicate_src_table"
+                        THIS_RUN+=" deduplicate_dst_table=$deduplicate_dst_table"
+                        THIS_RUN+=" insert_unique_blocks=$insert_unique_blocks"
+
+                        i=$((i+1))
+
+                        echo
+                        if [ -n "$RUN_ONLY" ] && [ "$RUN_ONLY" != "$THIS_RUN" ]; then
+                            echo "skip $THIS_RUN"
+                            continue
+                        fi
+                        echo "$THIS_RUN"
+
+                        $CLICKHOUSE_CLIENT --max_insert_block_size 1  -nmq "
+                            $(python3 $CURDIR/03008_deduplication.python mv_generates_several_blocks \
+                                --insert-method $insert_method \
+                                --table-engine $ENGINE \
+                                --use-insert-token $use_insert_token \
+                                --single-thread $single_thread \
+                                --deduplicate-src-table $deduplicate_src_table \
+                                --deduplicate-dst-table $deduplicate_dst_table \
+                                --insert-unique-blocks $insert_unique_blocks \
+                                --get-logs false \
+                            )
+                        " && echo OK || echo FAIL
+                    done
+                done
+            done
+        done
+    done
+done
+
+echo
+echo "All cases executed"
diff --git a/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_replicated.reference b/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_replicated.reference
new file mode 100644
index 00000000000..a25e8713c61
--- /dev/null
+++ b/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_replicated.reference
@@ -0,0 +1,962 @@
+
+Test case 0: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+OK
+
+Test case 1: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	10
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	10
+0
+0
+OK
+
+Test case 2: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	18
+0
+0
+OK
+
+Test case 3: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	10
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	20
+0
+0
+OK
+
+Test case 4: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	9
+0
+0
+OK
+
+Test case 5: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	10
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	10
+0
+0
+OK
+
+Test case 6: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	18
+0
+0
+OK
+
+Test case 7: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	10
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	20
+0
+0
+OK
+
+Test case 8: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+OK
+
+Test case 9: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	10
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	10
+0
+0
+OK
+
+Test case 10: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	18
+0
+0
+OK
+
+Test case 11: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	10
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	20
+0
+0
+OK
+
+Test case 12: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	9
+0
+0
+OK
+
+Test case 13: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	10
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	10
+0
+0
+OK
+
+Test case 14: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	18
+0
+0
+OK
+
+Test case 15: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	10
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	20
+0
+0
+OK
+
+Test case 16: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+OK
+
+Test case 17: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	1
+table_when_b_even_and_joined
+count	2
+0
+0
+table_a_b
+count	1
+table_when_b_even_and_joined
+count	2
+0
+0
+OK
+
+Test case 18: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	18
+0
+0
+OK
+
+Test case 19: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	1
+table_when_b_even_and_joined
+count	10
+0
+0
+table_a_b
+count	1
+table_when_b_even_and_joined
+count	20
+0
+0
+OK
+
+Test case 20: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	9
+0
+0
+OK
+
+Test case 21: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	2
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	2
+0
+0
+OK
+
+Test case 22: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	18
+0
+0
+OK
+
+Test case 23: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	10
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	20
+0
+0
+OK
+
+Test case 24: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+OK
+
+Test case 25: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	1
+table_when_b_even_and_joined
+count	2
+0
+0
+table_a_b
+count	1
+table_when_b_even_and_joined
+count	2
+0
+0
+OK
+
+Test case 26: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	18
+0
+0
+OK
+
+Test case 27: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	1
+table_when_b_even_and_joined
+count	10
+0
+0
+table_a_b
+count	1
+table_when_b_even_and_joined
+count	20
+0
+0
+OK
+
+Test case 28: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	9
+0
+0
+OK
+
+Test case 29: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	2
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	2
+0
+0
+OK
+
+Test case 30: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	18
+0
+0
+OK
+
+Test case 31: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	10
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	20
+0
+0
+OK
+
+Test case 32: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+OK
+
+Test case 33: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	10
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	10
+0
+0
+OK
+
+Test case 34: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	18
+0
+0
+OK
+
+Test case 35: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	10
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	20
+0
+0
+OK
+
+Test case 36: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	9
+0
+0
+OK
+
+Test case 37: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	10
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	10
+0
+0
+OK
+
+Test case 38: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	18
+0
+0
+OK
+
+Test case 39: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	10
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	20
+0
+0
+OK
+
+Test case 40: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+OK
+
+Test case 41: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	10
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	10
+0
+0
+OK
+
+Test case 42: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	18
+0
+0
+OK
+
+Test case 43: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	10
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	20
+0
+0
+OK
+
+Test case 44: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	9
+0
+0
+OK
+
+Test case 45: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	10
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	10
+0
+0
+OK
+
+Test case 46: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	18
+0
+0
+OK
+
+Test case 47: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	10
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	20
+0
+0
+OK
+
+Test case 48: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+OK
+
+Test case 49: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	1
+table_when_b_even_and_joined
+count	2
+0
+0
+table_a_b
+count	1
+table_when_b_even_and_joined
+count	2
+0
+0
+OK
+
+Test case 50: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	18
+0
+0
+OK
+
+Test case 51: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	1
+table_when_b_even_and_joined
+count	10
+0
+0
+table_a_b
+count	1
+table_when_b_even_and_joined
+count	20
+0
+0
+OK
+
+Test case 52: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	9
+0
+0
+OK
+
+Test case 53: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	2
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	2
+0
+0
+OK
+
+Test case 54: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	18
+0
+0
+OK
+
+Test case 55: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	10
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	20
+0
+0
+OK
+
+Test case 56: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+OK
+
+Test case 57: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	1
+table_when_b_even_and_joined
+count	2
+0
+0
+table_a_b
+count	1
+table_when_b_even_and_joined
+count	2
+0
+0
+OK
+
+Test case 58: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	18
+0
+0
+OK
+
+Test case 59: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	1
+table_when_b_even_and_joined
+count	10
+0
+0
+table_a_b
+count	1
+table_when_b_even_and_joined
+count	20
+0
+0
+OK
+
+Test case 60: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	9
+0
+0
+OK
+
+Test case 61: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	2
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	2
+0
+0
+OK
+
+Test case 62: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	9
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	18
+0
+0
+OK
+
+Test case 63: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_a_b
+count	5
+table_when_b_even_and_joined
+count	10
+0
+0
+table_a_b
+count	10
+table_when_b_even_and_joined
+count	20
+0
+0
+OK
+
+All cases executed
diff --git a/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_replicated.sh b/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_replicated.sh
new file mode 100755
index 00000000000..109d1674f3a
--- /dev/null
+++ b/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_replicated.sh
@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+# Tags: long, no-fasttest, no-parallel
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+ENGINE="ReplicatedMergeTree"
+
+RUN_ONLY=""
+#RUN_ONLY="Test case 20: engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True"
+
+i=0
+for insert_method in "InsertSelect" "InsertValues"; do
+    for use_insert_token in "True" "False"; do
+        for single_thread in "True" "False"; do
+            for deduplicate_src_table in "True" "False"; do
+                for deduplicate_dst_table in "True" "False"; do
+                    for insert_unique_blocks in "True" "False"; do
+
+                        THIS_RUN="Test case $i:"
+                        THIS_RUN+=" insert_method=$insert_method"
+                        THIS_RUN+=" engine=$ENGINE"
+                        THIS_RUN+=" use_insert_token=$use_insert_token"
+                        THIS_RUN+=" single_thread=$single_thread"
+                        THIS_RUN+=" deduplicate_src_table=$deduplicate_src_table"
+                        THIS_RUN+=" deduplicate_dst_table=$deduplicate_dst_table"
+                        THIS_RUN+=" insert_unique_blocks=$insert_unique_blocks"
+
+                        i=$((i+1))
+
+                        echo
+                        if [ -n "$RUN_ONLY" ] && [ "$RUN_ONLY" != "$THIS_RUN" ]; then
+                            echo "skip $THIS_RUN"
+                            continue
+                        fi
+                        echo "$THIS_RUN"
+
+                        $CLICKHOUSE_CLIENT --max_insert_block_size 1  -nmq "
+                            $(python3 $CURDIR/03008_deduplication.python mv_generates_several_blocks \
+                                --insert-method $insert_method \
+                                --table-engine $ENGINE \
+                                --use-insert-token $use_insert_token \
+                                --single-thread $single_thread \
+                                --deduplicate-src-table $deduplicate_src_table \
+                                --deduplicate-dst-table $deduplicate_dst_table \
+                                --insert-unique-blocks $insert_unique_blocks \
+                                --get-logs false \
+                            )
+                        " && echo OK || echo FAIL
+                    done
+                done
+            done
+        done
+    done
+done
+
+echo
+echo "All cases executed"
diff --git a/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_nonreplicated.reference b/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_nonreplicated.reference
new file mode 100644
index 00000000000..b6a3e0175a7
--- /dev/null
+++ b/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_nonreplicated.reference
@@ -0,0 +1,706 @@
+
+Test case 0: insert_method=InsertSelect engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	8
+table_dst count	6
+0
+0
+OK
+
+Test case 1: insert_method=InsertSelect engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_src count	8
+table_dst count	16
+0
+0
+table_src count	8
+table_dst count	16
+0
+0
+OK
+
+Test case 2: insert_method=InsertSelect engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	8
+table_dst count	12
+0
+0
+OK
+
+Test case 3: insert_method=InsertSelect engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_src count	8
+table_dst count	16
+0
+0
+table_src count	8
+table_dst count	32
+0
+0
+OK
+
+Test case 4: insert_method=InsertSelect engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	16
+table_dst count	6
+0
+0
+OK
+
+Test case 5: insert_method=InsertSelect engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_src count	8
+table_dst count	16
+0
+0
+table_src count	16
+table_dst count	16
+0
+0
+OK
+
+Test case 6: insert_method=InsertSelect engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	16
+table_dst count	12
+0
+0
+OK
+
+Test case 7: insert_method=InsertSelect engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_src count	8
+table_dst count	16
+0
+0
+table_src count	16
+table_dst count	32
+0
+0
+OK
+
+Test case 8: insert_method=InsertSelect engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	8
+table_dst count	6
+0
+0
+OK
+
+Test case 9: insert_method=InsertSelect engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_src count	8
+table_dst count	16
+0
+0
+table_src count	8
+table_dst count	16
+0
+0
+OK
+
+Test case 10: insert_method=InsertSelect engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	8
+table_dst count	12
+0
+0
+OK
+
+Test case 11: insert_method=InsertSelect engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_src count	8
+table_dst count	16
+0
+0
+table_src count	8
+table_dst count	32
+0
+0
+OK
+
+Test case 12: insert_method=InsertSelect engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	16
+table_dst count	6
+0
+0
+OK
+
+Test case 13: insert_method=InsertSelect engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_src count	8
+table_dst count	16
+0
+0
+table_src count	16
+table_dst count	16
+0
+0
+OK
+
+Test case 14: insert_method=InsertSelect engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	16
+table_dst count	12
+0
+0
+OK
+
+Test case 15: insert_method=InsertSelect engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_src count	8
+table_dst count	16
+0
+0
+table_src count	16
+table_dst count	32
+0
+0
+OK
+
+Test case 16: insert_method=InsertSelect engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	8
+table_dst count	6
+0
+0
+OK
+
+Test case 17: insert_method=InsertSelect engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_src count	1
+table_dst count	2
+0
+0
+table_src count	1
+table_dst count	2
+0
+0
+OK
+
+Test case 18: insert_method=InsertSelect engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	8
+table_dst count	12
+0
+0
+OK
+
+Test case 19: insert_method=InsertSelect engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_src count	1
+table_dst count	16
+0
+0
+table_src count	1
+table_dst count	32
+0
+0
+OK
+
+Test case 20: insert_method=InsertSelect engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	16
+table_dst count	6
+0
+0
+OK
+
+Test case 21: insert_method=InsertSelect engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_src count	8
+table_dst count	2
+0
+0
+table_src count	16
+table_dst count	2
+0
+0
+OK
+
+Test case 22: insert_method=InsertSelect engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	16
+table_dst count	12
+0
+0
+OK
+
+Test case 23: insert_method=InsertSelect engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_src count	8
+table_dst count	16
+0
+0
+table_src count	16
+table_dst count	32
+0
+0
+OK
+
+Test case 24: insert_method=InsertSelect engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	8
+table_dst count	6
+0
+0
+OK
+
+Test case 25: insert_method=InsertSelect engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_src count	1
+table_dst count	2
+0
+0
+table_src count	1
+table_dst count	2
+0
+0
+OK
+
+Test case 26: insert_method=InsertSelect engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	8
+table_dst count	12
+0
+0
+OK
+
+Test case 27: insert_method=InsertSelect engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_src count	1
+table_dst count	16
+0
+0
+table_src count	1
+table_dst count	32
+0
+0
+OK
+
+Test case 28: insert_method=InsertSelect engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	16
+table_dst count	6
+0
+0
+OK
+
+Test case 29: insert_method=InsertSelect engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_src count	8
+table_dst count	2
+0
+0
+table_src count	16
+table_dst count	2
+0
+0
+OK
+
+Test case 30: insert_method=InsertSelect engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	16
+table_dst count	12
+0
+0
+OK
+
+Test case 31: insert_method=InsertSelect engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_src count	8
+table_dst count	16
+0
+0
+table_src count	16
+table_dst count	32
+0
+0
+OK
+
+Test case 32: insert_method=InsertValues engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	8
+table_dst count	6
+0
+0
+OK
+
+Test case 33: insert_method=InsertValues engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_src count	8
+table_dst count	16
+0
+0
+table_src count	8
+table_dst count	16
+0
+0
+OK
+
+Test case 34: insert_method=InsertValues engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	8
+table_dst count	12
+0
+0
+OK
+
+Test case 35: insert_method=InsertValues engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_src count	8
+table_dst count	16
+0
+0
+table_src count	8
+table_dst count	32
+0
+0
+OK
+
+Test case 36: insert_method=InsertValues engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	16
+table_dst count	6
+0
+0
+OK
+
+Test case 37: insert_method=InsertValues engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_src count	8
+table_dst count	16
+0
+0
+table_src count	16
+table_dst count	16
+0
+0
+OK
+
+Test case 38: insert_method=InsertValues engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	16
+table_dst count	12
+0
+0
+OK
+
+Test case 39: insert_method=InsertValues engine=MergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_src count	8
+table_dst count	16
+0
+0
+table_src count	16
+table_dst count	32
+0
+0
+OK
+
+Test case 40: insert_method=InsertValues engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	8
+table_dst count	6
+0
+0
+OK
+
+Test case 41: insert_method=InsertValues engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_src count	8
+table_dst count	16
+0
+0
+table_src count	8
+table_dst count	16
+0
+0
+OK
+
+Test case 42: insert_method=InsertValues engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	8
+table_dst count	12
+0
+0
+OK
+
+Test case 43: insert_method=InsertValues engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_src count	8
+table_dst count	16
+0
+0
+table_src count	8
+table_dst count	32
+0
+0
+OK
+
+Test case 44: insert_method=InsertValues engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	16
+table_dst count	6
+0
+0
+OK
+
+Test case 45: insert_method=InsertValues engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_src count	8
+table_dst count	16
+0
+0
+table_src count	16
+table_dst count	16
+0
+0
+OK
+
+Test case 46: insert_method=InsertValues engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	16
+table_dst count	12
+0
+0
+OK
+
+Test case 47: insert_method=InsertValues engine=MergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_src count	8
+table_dst count	16
+0
+0
+table_src count	16
+table_dst count	32
+0
+0
+OK
+
+Test case 48: insert_method=InsertValues engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	8
+table_dst count	6
+0
+0
+OK
+
+Test case 49: insert_method=InsertValues engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_src count	1
+table_dst count	2
+0
+0
+table_src count	1
+table_dst count	2
+0
+0
+OK
+
+Test case 50: insert_method=InsertValues engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	8
+table_dst count	12
+0
+0
+OK
+
+Test case 51: insert_method=InsertValues engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_src count	1
+table_dst count	16
+0
+0
+table_src count	1
+table_dst count	32
+0
+0
+OK
+
+Test case 52: insert_method=InsertValues engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	16
+table_dst count	6
+0
+0
+OK
+
+Test case 53: insert_method=InsertValues engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_src count	8
+table_dst count	2
+0
+0
+table_src count	16
+table_dst count	2
+0
+0
+OK
+
+Test case 54: insert_method=InsertValues engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	16
+table_dst count	12
+0
+0
+OK
+
+Test case 55: insert_method=InsertValues engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_src count	8
+table_dst count	16
+0
+0
+table_src count	16
+table_dst count	32
+0
+0
+OK
+
+Test case 56: insert_method=InsertValues engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	8
+table_dst count	6
+0
+0
+OK
+
+Test case 57: insert_method=InsertValues engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_src count	1
+table_dst count	2
+0
+0
+table_src count	1
+table_dst count	2
+0
+0
+OK
+
+Test case 58: insert_method=InsertValues engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	8
+table_dst count	12
+0
+0
+OK
+
+Test case 59: insert_method=InsertValues engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_src count	1
+table_dst count	16
+0
+0
+table_src count	1
+table_dst count	32
+0
+0
+OK
+
+Test case 60: insert_method=InsertValues engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	16
+table_dst count	6
+0
+0
+OK
+
+Test case 61: insert_method=InsertValues engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_src count	8
+table_dst count	2
+0
+0
+table_src count	16
+table_dst count	2
+0
+0
+OK
+
+Test case 62: insert_method=InsertValues engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	16
+table_dst count	12
+0
+0
+OK
+
+Test case 63: insert_method=InsertValues engine=MergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_src count	8
+table_dst count	16
+0
+0
+table_src count	16
+table_dst count	32
+0
+0
+OK
+
+All cases executed
diff --git a/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_nonreplicated.sh b/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_nonreplicated.sh
new file mode 100755
index 00000000000..fe3d610a758
--- /dev/null
+++ b/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_nonreplicated.sh
@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+# Tags: long, no-fasttest, no-parallel
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+ENGINE="MergeTree"
+
+RUN_ONLY=""
+#RUN_ONLY="Test case 17: insert_method=InsertSelect engine=MergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False"
+
+i=0
+for insert_method in "InsertSelect" "InsertValues"; do
+    for use_insert_token in "True" "False"; do
+        for single_thread in "True" "False"; do
+            for deduplicate_src_table in "True" "False"; do
+                for deduplicate_dst_table in "True" "False"; do
+                    for insert_unique_blocks in "True" "False"; do
+
+                        THIS_RUN="Test case $i:"
+                        THIS_RUN+=" insert_method=$insert_method"
+                        THIS_RUN+=" engine=$ENGINE"
+                        THIS_RUN+=" use_insert_token=$use_insert_token"
+                        THIS_RUN+=" single_thread=$single_thread"
+                        THIS_RUN+=" deduplicate_src_table=$deduplicate_src_table"
+                        THIS_RUN+=" deduplicate_dst_table=$deduplicate_dst_table"
+                        THIS_RUN+=" insert_unique_blocks=$insert_unique_blocks"
+
+                        i=$((i+1))
+
+                        echo
+                        if [ -n "$RUN_ONLY" ] && [ "$RUN_ONLY" != "$THIS_RUN" ]; then
+                            echo "skip $THIS_RUN"
+                            continue
+                        fi
+                        echo "$THIS_RUN"
+
+                        $CLICKHOUSE_CLIENT  --max_insert_block_size 1 -nmq "
+                            $(python3 $CURDIR/03008_deduplication.python several_mv_into_one_table \
+                                --insert-method $insert_method \
+                                --table-engine $ENGINE \
+                                --use-insert-token $use_insert_token \
+                                --single-thread $single_thread \
+                                --deduplicate-src-table $deduplicate_src_table \
+                                --deduplicate-dst-table $deduplicate_dst_table \
+                                --insert-unique-blocks $insert_unique_blocks \
+                                --get-logs false \
+                            )
+                        " && echo OK || echo FAIL
+                    done
+                done
+            done
+        done
+    done
+done
+
+echo
+echo "All cases executed"
diff --git a/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_replicated.reference b/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_replicated.reference
new file mode 100644
index 00000000000..1921103f49e
--- /dev/null
+++ b/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_replicated.reference
@@ -0,0 +1,706 @@
+
+Test case 0: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	8
+table_dst count	6
+0
+0
+OK
+
+Test case 1: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_src count	8
+table_dst count	16
+0
+0
+table_src count	8
+table_dst count	16
+0
+0
+OK
+
+Test case 2: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	8
+table_dst count	12
+0
+0
+OK
+
+Test case 3: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_src count	8
+table_dst count	16
+0
+0
+table_src count	8
+table_dst count	32
+0
+0
+OK
+
+Test case 4: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	16
+table_dst count	6
+0
+0
+OK
+
+Test case 5: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_src count	8
+table_dst count	16
+0
+0
+table_src count	16
+table_dst count	16
+0
+0
+OK
+
+Test case 6: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	16
+table_dst count	12
+0
+0
+OK
+
+Test case 7: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_src count	8
+table_dst count	16
+0
+0
+table_src count	16
+table_dst count	32
+0
+0
+OK
+
+Test case 8: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	8
+table_dst count	6
+0
+0
+OK
+
+Test case 9: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_src count	8
+table_dst count	16
+0
+0
+table_src count	8
+table_dst count	16
+0
+0
+OK
+
+Test case 10: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	8
+table_dst count	12
+0
+0
+OK
+
+Test case 11: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_src count	8
+table_dst count	16
+0
+0
+table_src count	8
+table_dst count	32
+0
+0
+OK
+
+Test case 12: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	16
+table_dst count	6
+0
+0
+OK
+
+Test case 13: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_src count	8
+table_dst count	16
+0
+0
+table_src count	16
+table_dst count	16
+0
+0
+OK
+
+Test case 14: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	16
+table_dst count	12
+0
+0
+OK
+
+Test case 15: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_src count	8
+table_dst count	16
+0
+0
+table_src count	16
+table_dst count	32
+0
+0
+OK
+
+Test case 16: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	8
+table_dst count	6
+0
+0
+OK
+
+Test case 17: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_src count	1
+table_dst count	2
+0
+0
+table_src count	1
+table_dst count	2
+0
+0
+OK
+
+Test case 18: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	8
+table_dst count	12
+0
+0
+OK
+
+Test case 19: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_src count	1
+table_dst count	16
+0
+0
+table_src count	1
+table_dst count	32
+0
+0
+OK
+
+Test case 20: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	16
+table_dst count	6
+0
+0
+OK
+
+Test case 21: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_src count	8
+table_dst count	2
+0
+0
+table_src count	16
+table_dst count	2
+0
+0
+OK
+
+Test case 22: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	16
+table_dst count	12
+0
+0
+OK
+
+Test case 23: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_src count	8
+table_dst count	16
+0
+0
+table_src count	16
+table_dst count	32
+0
+0
+OK
+
+Test case 24: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	8
+table_dst count	6
+0
+0
+OK
+
+Test case 25: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_src count	1
+table_dst count	2
+0
+0
+table_src count	1
+table_dst count	2
+0
+0
+OK
+
+Test case 26: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	8
+table_dst count	12
+0
+0
+OK
+
+Test case 27: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_src count	1
+table_dst count	16
+0
+0
+table_src count	1
+table_dst count	32
+0
+0
+OK
+
+Test case 28: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	16
+table_dst count	6
+0
+0
+OK
+
+Test case 29: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_src count	8
+table_dst count	2
+0
+0
+table_src count	16
+table_dst count	2
+0
+0
+OK
+
+Test case 30: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	16
+table_dst count	12
+0
+0
+OK
+
+Test case 31: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_src count	8
+table_dst count	16
+0
+0
+table_src count	16
+table_dst count	32
+0
+0
+OK
+
+Test case 32: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	8
+table_dst count	6
+0
+0
+OK
+
+Test case 33: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_src count	8
+table_dst count	16
+0
+0
+table_src count	8
+table_dst count	16
+0
+0
+OK
+
+Test case 34: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	8
+table_dst count	12
+0
+0
+OK
+
+Test case 35: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_src count	8
+table_dst count	16
+0
+0
+table_src count	8
+table_dst count	32
+0
+0
+OK
+
+Test case 36: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	16
+table_dst count	6
+0
+0
+OK
+
+Test case 37: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_src count	8
+table_dst count	16
+0
+0
+table_src count	16
+table_dst count	16
+0
+0
+OK
+
+Test case 38: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	16
+table_dst count	12
+0
+0
+OK
+
+Test case 39: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=True single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_src count	8
+table_dst count	16
+0
+0
+table_src count	16
+table_dst count	32
+0
+0
+OK
+
+Test case 40: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	8
+table_dst count	6
+0
+0
+OK
+
+Test case 41: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_src count	8
+table_dst count	16
+0
+0
+table_src count	8
+table_dst count	16
+0
+0
+OK
+
+Test case 42: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	8
+table_dst count	12
+0
+0
+OK
+
+Test case 43: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_src count	8
+table_dst count	16
+0
+0
+table_src count	8
+table_dst count	32
+0
+0
+OK
+
+Test case 44: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	16
+table_dst count	6
+0
+0
+OK
+
+Test case 45: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_src count	8
+table_dst count	16
+0
+0
+table_src count	16
+table_dst count	16
+0
+0
+OK
+
+Test case 46: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	16
+table_dst count	12
+0
+0
+OK
+
+Test case 47: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=True single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_src count	8
+table_dst count	16
+0
+0
+table_src count	16
+table_dst count	32
+0
+0
+OK
+
+Test case 48: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	8
+table_dst count	6
+0
+0
+OK
+
+Test case 49: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_src count	1
+table_dst count	2
+0
+0
+table_src count	1
+table_dst count	2
+0
+0
+OK
+
+Test case 50: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	8
+table_dst count	12
+0
+0
+OK
+
+Test case 51: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_src count	1
+table_dst count	16
+0
+0
+table_src count	1
+table_dst count	32
+0
+0
+OK
+
+Test case 52: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	16
+table_dst count	6
+0
+0
+OK
+
+Test case 53: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_src count	8
+table_dst count	2
+0
+0
+table_src count	16
+table_dst count	2
+0
+0
+OK
+
+Test case 54: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	16
+table_dst count	12
+0
+0
+OK
+
+Test case 55: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_src count	8
+table_dst count	16
+0
+0
+table_src count	16
+table_dst count	32
+0
+0
+OK
+
+Test case 56: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	8
+table_dst count	6
+0
+0
+OK
+
+Test case 57: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False
+table_src count	1
+table_dst count	2
+0
+0
+table_src count	1
+table_dst count	2
+0
+0
+OK
+
+Test case 58: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	8
+table_dst count	12
+0
+0
+OK
+
+Test case 59: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=True deduplicate_dst_table=False insert_unique_blocks=False
+table_src count	1
+table_dst count	16
+0
+0
+table_src count	1
+table_dst count	32
+0
+0
+OK
+
+Test case 60: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	16
+table_dst count	6
+0
+0
+OK
+
+Test case 61: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=True insert_unique_blocks=False
+table_src count	8
+table_dst count	2
+0
+0
+table_src count	16
+table_dst count	2
+0
+0
+OK
+
+Test case 62: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=True
+table_src count	8
+table_dst count	6
+0
+0
+table_src count	16
+table_dst count	12
+0
+0
+OK
+
+Test case 63: insert_method=InsertValues engine=ReplicatedMergeTree use_insert_token=False single_thread=False deduplicate_src_table=False deduplicate_dst_table=False insert_unique_blocks=False
+table_src count	8
+table_dst count	16
+0
+0
+table_src count	16
+table_dst count	32
+0
+0
+OK
+
+All cases executed
diff --git a/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_replicated.sh b/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_replicated.sh
new file mode 100755
index 00000000000..9adee6d53d4
--- /dev/null
+++ b/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_replicated.sh
@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+# Tags: long, no-fasttest, no-parallel
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+ENGINE="ReplicatedMergeTree"
+
+RUN_ONLY=""
+#RUN_ONLY="Test case 17: insert_method=InsertSelect engine=ReplicatedMergeTree use_insert_token=False single_thread=True deduplicate_src_table=True deduplicate_dst_table=True insert_unique_blocks=False"
+
+i=0
+for insert_method in "InsertSelect" "InsertValues"; do
+    for use_insert_token in "True" "False"; do
+        for single_thread in "True" "False"; do
+            for deduplicate_src_table in "True" "False"; do
+                for deduplicate_dst_table in "True" "False"; do
+                    for insert_unique_blocks in "True" "False"; do
+
+                        THIS_RUN="Test case $i:"
+                        THIS_RUN+=" insert_method=$insert_method"
+                        THIS_RUN+=" engine=$ENGINE"
+                        THIS_RUN+=" use_insert_token=$use_insert_token"
+                        THIS_RUN+=" single_thread=$single_thread"
+                        THIS_RUN+=" deduplicate_src_table=$deduplicate_src_table"
+                        THIS_RUN+=" deduplicate_dst_table=$deduplicate_dst_table"
+                        THIS_RUN+=" insert_unique_blocks=$insert_unique_blocks"
+
+                        i=$((i+1))
+
+                        echo
+                        if [ -n "$RUN_ONLY" ] && [ "$RUN_ONLY" != "$THIS_RUN" ]; then
+                            echo "skip $THIS_RUN"
+                            continue
+                        fi
+                        echo "$THIS_RUN"
+
+                        $CLICKHOUSE_CLIENT  --max_insert_block_size 1 -nmq "
+                            $(python3 $CURDIR/03008_deduplication.python several_mv_into_one_table \
+                                --insert-method $insert_method \
+                                --table-engine $ENGINE \
+                                --use-insert-token $use_insert_token \
+                                --single-thread $single_thread \
+                                --deduplicate-src-table $deduplicate_src_table \
+                                --deduplicate-dst-table $deduplicate_dst_table \
+                                --insert-unique-blocks $insert_unique_blocks \
+                                --get-logs false \
+                            )
+                        " && echo OK || echo FAIL
+                    done
+                done
+            done
+        done
+    done
+done
+
+echo
+echo "All cases executed"
diff --git a/tests/queries/0_stateless/03035_max_insert_threads_support.sh b/tests/queries/0_stateless/03035_max_insert_threads_support.sh
index 1e6bfb414d8..cedb651a430 100755
--- a/tests/queries/0_stateless/03035_max_insert_threads_support.sh
+++ b/tests/queries/0_stateless/03035_max_insert_threads_support.sh
@@ -8,7 +8,7 @@ DATA_FILE="data_$CLICKHOUSE_TEST_UNIQUE_NAME.csv"
 
 $CLICKHOUSE_CLIENT --max_insert_threads=4 --query="
     EXPLAIN PIPELINE INSERT INTO FUNCTION file('$DATA_FILE') SELECT * FROM numbers_mt(1000000) ORDER BY number DESC
-" | grep -o MaterializingTransform | wc -l
+" | grep -o StorageFileSink | wc -l
 
 DATA_FILE_PATH=$($CLICKHOUSE_CLIENT_BINARY --query "select _path from file('$DATA_FILE', 'One')")
 rm $DATA_FILE_PATH

From 7843313f8e09eb018a481b4ba70fcf5fc147105e Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Fri, 5 Jul 2024 18:20:50 +0200
Subject: [PATCH 0159/1488] Update PlannerJoinTree.h

---
 src/Planner/PlannerJoinTree.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Planner/PlannerJoinTree.h b/src/Planner/PlannerJoinTree.h
index bc58e802a09..259622b1d50 100644
--- a/src/Planner/PlannerJoinTree.h
+++ b/src/Planner/PlannerJoinTree.h
@@ -18,7 +18,7 @@ struct JoinTreeQueryPlan
     QueryPlan query_plan;
     QueryProcessingStage::Enum from_stage;
     std::set<std::string> used_row_policies{};
-    UsefulSets useful_sets;
+    UsefulSets useful_sets{};
     std::unordered_map<const QueryNode *, const QueryPlan::Node *> query_node_to_plan_step_mapping{};
 };
 

From 2257f9a2aee5e8a5c5e178e5f7ccaf269018756a Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Fri, 5 Jul 2024 17:49:50 +0000
Subject: [PATCH 0160/1488] Change ActionsDAGPtr to ActionsDAG where possible.

---
 src/Core/InterpolateDescription.cpp           |   6 +-
 src/Core/InterpolateDescription.h             |   7 +-
 src/Functions/indexHint.h                     |   6 +-
 src/Interpreters/ActionsDAG.cpp               | 148 +++++++++---------
 src/Interpreters/ActionsDAG.h                 |  46 +++---
 src/Interpreters/ActionsVisitor.cpp           |   4 +-
 src/Interpreters/ExpressionActions.cpp        |  19 ++-
 src/Interpreters/ExpressionActions.h          |   8 +-
 src/Interpreters/ExpressionAnalyzer.cpp       |  51 +++---
 src/Interpreters/ExpressionAnalyzer.h         |   6 +-
 src/Interpreters/GlobalSubqueriesVisitor.h    |   2 +-
 .../IInterpreterUnionOrSelectQuery.cpp        |  12 +-
 src/Interpreters/InterpreterSelectQuery.cpp   |  49 +++---
 src/Interpreters/InterpreterSelectQuery.h     |   2 +-
 src/Interpreters/MutationsInterpreter.cpp     |  14 +-
 .../MySQL/InterpretersMySQLDDLQuery.cpp       |   2 +-
 src/Interpreters/TableJoin.cpp                |  49 +++---
 src/Interpreters/TableJoin.h                  |  10 +-
 src/Interpreters/addMissingDefaults.cpp       |  24 +--
 src/Interpreters/addMissingDefaults.h         |   2 +-
 .../evaluateConstantExpression.cpp            |   2 +-
 src/Interpreters/inplaceBlockConversions.cpp  |  16 +-
 src/Interpreters/inplaceBlockConversions.h    |   6 +-
 src/Planner/Planner.cpp                       |  46 +++---
 src/Planner/PlannerActionsVisitor.cpp         |  14 +-
 src/Planner/PlannerContext.h                  |   2 +-
 src/Planner/PlannerJoinTree.cpp               |  81 +++++-----
 src/Planner/PlannerJoins.cpp                  |  56 +++----
 src/Planner/PlannerJoins.h                    |  10 +-
 src/Planner/Utils.cpp                         |  10 +-
 src/Processors/QueryPlan/AggregatingStep.cpp  |  16 +-
 src/Processors/QueryPlan/CubeStep.cpp         |  12 +-
 .../QueryPlan/DistributedCreateLocalPlan.cpp  |   2 +-
 src/Processors/QueryPlan/ExpressionStep.cpp   |  22 +--
 src/Processors/QueryPlan/ExpressionStep.h     |  11 +-
 src/Processors/QueryPlan/FilterStep.cpp       |  20 +--
 src/Processors/QueryPlan/FilterStep.h         |  12 +-
 .../convertOuterJoinToInnerJoin.cpp           |   4 +-
 .../Optimizations/distinctReadInOrder.cpp     |   4 +-
 .../Optimizations/filterPushDown.cpp          |  50 +++---
 .../Optimizations/liftUpArrayJoin.cpp         |   8 +-
 .../Optimizations/liftUpFunctions.cpp         |   6 +-
 .../QueryPlan/Optimizations/liftUpUnion.cpp   |   2 +-
 .../Optimizations/mergeExpressions.cpp        |  45 +++---
 .../Optimizations/optimizePrewhere.cpp        |  19 +--
 .../optimizePrimaryKeyConditionAndLimit.cpp   |   6 +-
 .../Optimizations/optimizeReadInOrder.cpp     |  14 +-
 .../optimizeUseAggregateProjection.cpp        |  30 ++--
 .../optimizeUseNormalProjection.cpp           |  22 +--
 .../Optimizations/projectionsCommon.cpp       |  14 +-
 .../Optimizations/projectionsCommon.h         |   4 +-
 .../Optimizations/removeRedundantDistinct.cpp |  12 +-
 .../Optimizations/removeRedundantSorting.cpp  |   4 +-
 .../QueryPlan/Optimizations/splitFilter.cpp   |  14 +-
 .../useDataParallelAggregation.cpp            |   6 +-
 src/Processors/QueryPlan/PartsSplitter.cpp    |   2 +-
 .../QueryPlan/ReadFromMergeTree.cpp           |  39 ++---
 src/Processors/QueryPlan/ReadFromMergeTree.h  |   8 +-
 .../QueryPlan/ReadFromSystemNumbersStep.cpp   |   2 +-
 .../QueryPlan/SourceStepWithFilter.cpp        |   8 +-
 .../QueryPlan/SourceStepWithFilter.h          |   6 +-
 src/Processors/QueryPlan/TotalsHavingStep.cpp |  14 +-
 src/Processors/QueryPlan/TotalsHavingStep.h   |   7 +-
 src/Processors/SourceWithKeyCondition.h       |   8 +-
 .../Transforms/AddingDefaultsTransform.cpp    |   2 +-
 .../Transforms/FillingTransform.cpp           |   2 +-
 src/Storages/Hive/StorageHive.cpp             |  24 +--
 src/Storages/Hive/StorageHive.h               |  10 +-
 src/Storages/IStorage.cpp                     |   2 +-
 src/Storages/IStorage.h                       |   4 +-
 src/Storages/KVStorageUtils.cpp               |   2 +-
 src/Storages/KVStorageUtils.h                 |   2 +-
 src/Storages/KeyDescription.cpp               |   2 +-
 src/Storages/MergeTree/IMergeTreeReader.cpp   |   4 +-
 src/Storages/MergeTree/KeyCondition.cpp       |   4 +-
 src/Storages/MergeTree/MergeTreeData.cpp      |  10 +-
 src/Storages/MergeTree/MergeTreeData.h        |   4 +-
 .../MergeTree/MergeTreeDataSelectExecutor.cpp |   4 +-
 src/Storages/MergeTree/MergeTreeIndexSet.cpp  |  38 ++---
 src/Storages/MergeTree/MergeTreeIndexSet.h    |   6 +-
 .../MergeTree/MergeTreeSelectProcessor.cpp    |   4 +-
 .../MergeTree/MergeTreeSequentialSource.cpp   |   8 +-
 .../MergeTree/MergeTreeSequentialSource.h     |   2 +-
 .../MergeTreeSplitPrewhereIntoReadSteps.cpp   |   2 +-
 .../MergeTree/MergeTreeWhereOptimizer.cpp     |   4 +-
 .../MergeTree/MergeTreeWhereOptimizer.h       |   2 +-
 src/Storages/MergeTree/RPNBuilder.cpp         |   4 +-
 .../StorageObjectStorageSource.cpp            |   4 +-
 .../StorageObjectStorageSource.h              |   2 +-
 .../ReadFinalForExternalReplicaStorage.cpp    |   2 +-
 src/Storages/SelectQueryInfo.h                |  12 +-
 src/Storages/StorageBuffer.cpp                |  12 +-
 src/Storages/StorageDistributed.cpp           |   4 +-
 src/Storages/StorageFile.cpp                  |   4 +-
 src/Storages/StorageFile.h                    |   2 +-
 src/Storages/StorageMaterializedView.cpp      |   4 +-
 src/Storages/StorageMerge.cpp                 |  30 ++--
 src/Storages/StorageMerge.h                   |   2 +-
 src/Storages/StorageMergeTree.cpp             |   2 +-
 src/Storages/StorageMergeTree.h               |   2 +-
 src/Storages/StorageReplicatedMergeTree.cpp   |   2 +-
 src/Storages/StorageReplicatedMergeTree.h     |   2 +-
 src/Storages/StorageTableFunction.h           |   2 +-
 src/Storages/StorageURL.h                     |   4 +-
 src/Storages/StorageValues.cpp                |   6 +-
 src/Storages/StorageView.cpp                  |   6 +-
 .../System/StorageSystemStackTrace.cpp        |   4 +-
 src/Storages/TTLDescription.cpp               |   8 +-
 src/Storages/VirtualColumnUtils.cpp           |  12 +-
 src/Storages/WindowView/StorageWindowView.cpp |   2 +-
 110 files changed, 718 insertions(+), 721 deletions(-)

diff --git a/src/Core/InterpolateDescription.cpp b/src/Core/InterpolateDescription.cpp
index 76bbefdcfd7..86681fdb591 100644
--- a/src/Core/InterpolateDescription.cpp
+++ b/src/Core/InterpolateDescription.cpp
@@ -13,10 +13,10 @@
 
 namespace DB
 {
-    InterpolateDescription::InterpolateDescription(ActionsDAGPtr actions_, const Aliases & aliases)
+    InterpolateDescription::InterpolateDescription(ActionsDAG actions_, const Aliases & aliases)
         : actions(std::move(actions_))
     {
-        for (const auto & name_type : actions->getRequiredColumns())
+        for (const auto & name_type : actions.getRequiredColumns())
         {
             if (const auto & p = aliases.find(name_type.name); p != aliases.end())
                 required_columns_map[p->second->getColumnName()] = name_type;
@@ -24,7 +24,7 @@ namespace DB
                 required_columns_map[name_type.name] = name_type;
         }
 
-        for (const ColumnWithTypeAndName & column : actions->getResultColumns())
+        for (const ColumnWithTypeAndName & column : actions.getResultColumns())
         {
             std::string name = column.name;
             if (const auto & p = aliases.find(name); p != aliases.end())
diff --git a/src/Core/InterpolateDescription.h b/src/Core/InterpolateDescription.h
index 73579aebee4..eeead71d780 100644
--- a/src/Core/InterpolateDescription.h
+++ b/src/Core/InterpolateDescription.h
@@ -5,21 +5,20 @@
 #include <string>
 #include <Core/NamesAndTypes.h>
 #include <Parsers/IAST_fwd.h>
+#include <Interpreters/ActionsDAG.h>
 
 
 namespace DB
 {
 
-class ActionsDAG;
-using ActionsDAGPtr = std::unique_ptr<ActionsDAG>;
 using Aliases = std::unordered_map<String, ASTPtr>;
 
 /// Interpolate description
 struct InterpolateDescription
 {
-    explicit InterpolateDescription(ActionsDAGPtr actions, const Aliases & aliases);
+    explicit InterpolateDescription(ActionsDAG actions, const Aliases & aliases);
 
-    ActionsDAGPtr actions;
+    ActionsDAG actions;
 
     std::unordered_map<std::string, NameAndTypePair> required_columns_map; /// input column name -> {alias, type}
     std::unordered_set<std::string> result_columns_set; /// result block columns
diff --git a/src/Functions/indexHint.h b/src/Functions/indexHint.h
index 8fd7b751760..3ab8a021ae1 100644
--- a/src/Functions/indexHint.h
+++ b/src/Functions/indexHint.h
@@ -58,11 +58,11 @@ public:
         return DataTypeUInt8().createColumnConst(input_rows_count, 1u);
     }
 
-    void setActions(ActionsDAGPtr actions_) { actions = std::move(actions_); }
-    const ActionsDAGPtr & getActions() const { return actions; }
+    void setActions(ActionsDAG actions_) { actions = std::move(actions_); }
+    const ActionsDAG & getActions() const { return actions; }
 
 private:
-    ActionsDAGPtr actions;
+    ActionsDAG actions;
 };
 
 }
diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp
index c2626285235..04be9d23c32 100644
--- a/src/Interpreters/ActionsDAG.cpp
+++ b/src/Interpreters/ActionsDAG.cpp
@@ -961,9 +961,9 @@ NameSet ActionsDAG::foldActionsByProjection(
 }
 
 
-ActionsDAGPtr ActionsDAG::foldActionsByProjection(const std::unordered_map<const Node *, const Node *> & new_inputs, const NodeRawConstPtrs & required_outputs)
+ActionsDAG ActionsDAG::foldActionsByProjection(const std::unordered_map<const Node *, const Node *> & new_inputs, const NodeRawConstPtrs & required_outputs)
 {
-    auto dag = std::make_unique<ActionsDAG>();
+    ActionsDAG dag;
     std::unordered_map<const Node *, const Node *> inputs_mapping;
     std::unordered_map<const Node *, const Node *> mapping;
     struct Frame
@@ -1003,9 +1003,9 @@ ActionsDAGPtr ActionsDAG::foldActionsByProjection(const std::unordered_map<const
                         {
                             bool should_rename = new_input->result_name != rename->result_name;
                             const auto & input_name = should_rename ? rename->result_name : new_input->result_name;
-                            mapped_input = &dag->addInput(input_name, new_input->result_type);
+                            mapped_input = &dag.addInput(input_name, new_input->result_type);
                             if (should_rename)
-                                mapped_input = &dag->addAlias(*mapped_input, new_input->result_name);
+                                mapped_input = &dag.addAlias(*mapped_input, new_input->result_name);
                         }
 
                         node = mapped_input;
@@ -1034,7 +1034,7 @@ ActionsDAGPtr ActionsDAG::foldActionsByProjection(const std::unordered_map<const
                     "Cannot fold actions for projection. Node {} requires input {} which does not belong to projection",
                     stack.front().node->result_name, frame.node->result_name);
 
-            auto & node = dag->nodes.emplace_back(*frame.node);
+            auto & node = dag.nodes.emplace_back(*frame.node);
             for (auto & child : node.children)
                 child = mapping[child];
 
@@ -1049,8 +1049,8 @@ ActionsDAGPtr ActionsDAG::foldActionsByProjection(const std::unordered_map<const
         /// Add an alias if the mapped node has a different result name.
         const auto * mapped_output = mapping[output];
         if (output->result_name != mapped_output->result_name)
-            mapped_output = &dag->addAlias(*mapped_output, output->result_name);
-        dag->outputs.push_back(mapped_output);
+            mapped_output = &dag.addAlias(*mapped_output, output->result_name);
+        dag.outputs.push_back(mapped_output);
     }
 
     return dag;
@@ -1411,7 +1411,7 @@ const ActionsDAG::Node & ActionsDAG::materializeNode(const Node & node)
     return addAlias(*func, name);
 }
 
-ActionsDAGPtr ActionsDAG::makeConvertingActions(
+ActionsDAG ActionsDAG::makeConvertingActions(
     const ColumnsWithTypeAndName & source,
     const ColumnsWithTypeAndName & result,
     MatchColumnsMode mode,
@@ -1428,7 +1428,7 @@ ActionsDAGPtr ActionsDAG::makeConvertingActions(
     if (add_casted_columns && mode != MatchColumnsMode::Name)
         throw Exception(ErrorCodes::LOGICAL_ERROR, "Converting with add_casted_columns supported only for MatchColumnsMode::Name");
 
-    auto actions_dag = std::make_unique<ActionsDAG>(source);
+    ActionsDAG actions_dag(source);
     NodeRawConstPtrs projection(num_result_columns);
 
     FunctionOverloadResolverPtr func_builder_materialize = std::make_unique<FunctionToOverloadResolverAdaptor>(std::make_shared<FunctionMaterialize>());
@@ -1436,9 +1436,9 @@ ActionsDAGPtr ActionsDAG::makeConvertingActions(
     std::unordered_map<std::string_view, std::list<size_t>> inputs;
     if (mode == MatchColumnsMode::Name)
     {
-        size_t input_nodes_size = actions_dag->inputs.size();
+        size_t input_nodes_size = actions_dag.inputs.size();
         for (size_t pos = 0; pos < input_nodes_size; ++pos)
-            inputs[actions_dag->inputs[pos]->result_name].push_back(pos);
+            inputs[actions_dag.inputs[pos]->result_name].push_back(pos);
     }
 
     for (size_t result_col_num = 0; result_col_num < num_result_columns; ++result_col_num)
@@ -1451,7 +1451,7 @@ ActionsDAGPtr ActionsDAG::makeConvertingActions(
         {
             case MatchColumnsMode::Position:
             {
-                src_node = dst_node = actions_dag->inputs[result_col_num];
+                src_node = dst_node = actions_dag.inputs[result_col_num];
                 break;
             }
 
@@ -1462,7 +1462,7 @@ ActionsDAGPtr ActionsDAG::makeConvertingActions(
                 {
                     const auto * res_const = typeid_cast<const ColumnConst *>(res_elem.column.get());
                     if (ignore_constant_values && res_const)
-                        src_node = dst_node = &actions_dag->addColumn(res_elem);
+                        src_node = dst_node = &actions_dag.addColumn(res_elem);
                     else
                         throw Exception(ErrorCodes::THERE_IS_NO_COLUMN,
                                         "Cannot find column `{}` in source stream, there are only columns: [{}]",
@@ -1470,7 +1470,7 @@ ActionsDAGPtr ActionsDAG::makeConvertingActions(
                 }
                 else
                 {
-                    src_node = dst_node = actions_dag->inputs[input.front()];
+                    src_node = dst_node = actions_dag.inputs[input.front()];
                     input.pop_front();
                 }
                 break;
@@ -1483,7 +1483,7 @@ ActionsDAGPtr ActionsDAG::makeConvertingActions(
             if (const auto * src_const = typeid_cast<const ColumnConst *>(dst_node->column.get()))
             {
                 if (ignore_constant_values)
-                    dst_node = &actions_dag->addColumn(res_elem);
+                    dst_node = &actions_dag.addColumn(res_elem);
                 else if (res_const->getField() != src_const->getField())
                     throw Exception(
                         ErrorCodes::ILLEGAL_COLUMN,
@@ -1505,7 +1505,7 @@ ActionsDAGPtr ActionsDAG::makeConvertingActions(
             column.column = DataTypeString().createColumnConst(0, column.name);
             column.type = std::make_shared<DataTypeString>();
 
-            const auto * right_arg = &actions_dag->addColumn(std::move(column));
+            const auto * right_arg = &actions_dag.addColumn(std::move(column));
             const auto * left_arg = dst_node;
 
             CastDiagnostic diagnostic = {dst_node->result_name, res_elem.name};
@@ -1513,13 +1513,13 @@ ActionsDAGPtr ActionsDAG::makeConvertingActions(
                 = createInternalCastOverloadResolver(CastType::nonAccurate, std::move(diagnostic));
 
             NodeRawConstPtrs children = { left_arg, right_arg };
-            dst_node = &actions_dag->addFunction(func_builder_cast, std::move(children), {});
+            dst_node = &actions_dag.addFunction(func_builder_cast, std::move(children), {});
         }
 
         if (dst_node->column && isColumnConst(*dst_node->column) && !(res_elem.column && isColumnConst(*res_elem.column)))
         {
             NodeRawConstPtrs children = {dst_node};
-            dst_node = &actions_dag->addFunction(func_builder_materialize, std::move(children), {});
+            dst_node = &actions_dag.addFunction(func_builder_materialize, std::move(children), {});
         }
 
         if (dst_node->result_name != res_elem.name)
@@ -1538,7 +1538,7 @@ ActionsDAGPtr ActionsDAG::makeConvertingActions(
             }
             else
             {
-                dst_node = &actions_dag->addAlias(*dst_node, res_elem.name);
+                dst_node = &actions_dag.addAlias(*dst_node, res_elem.name);
                 projection[result_col_num] = dst_node;
             }
         }
@@ -1548,36 +1548,36 @@ ActionsDAGPtr ActionsDAG::makeConvertingActions(
         }
     }
 
-    actions_dag->outputs.swap(projection);
-    actions_dag->removeUnusedActions(false);
+    actions_dag.outputs.swap(projection);
+    actions_dag.removeUnusedActions(false);
 
     return actions_dag;
 }
 
-ActionsDAGPtr ActionsDAG::makeAddingColumnActions(ColumnWithTypeAndName column)
+ActionsDAG ActionsDAG::makeAddingColumnActions(ColumnWithTypeAndName column)
 {
-    auto adding_column_action = std::make_unique<ActionsDAG>();
+    ActionsDAG adding_column_action;
     FunctionOverloadResolverPtr func_builder_materialize
         = std::make_unique<FunctionToOverloadResolverAdaptor>(std::make_shared<FunctionMaterialize>());
 
     auto column_name = column.name;
-    const auto * column_node = &adding_column_action->addColumn(std::move(column));
+    const auto * column_node = &adding_column_action.addColumn(std::move(column));
     NodeRawConstPtrs inputs = {column_node};
-    const auto & function_node = adding_column_action->addFunction(func_builder_materialize, std::move(inputs), {});
-    const auto & alias_node = adding_column_action->addAlias(function_node, std::move(column_name));
+    const auto & function_node = adding_column_action.addFunction(func_builder_materialize, std::move(inputs), {});
+    const auto & alias_node = adding_column_action.addAlias(function_node, std::move(column_name));
 
-    adding_column_action->outputs.push_back(&alias_node);
+    adding_column_action.outputs.push_back(&alias_node);
     return adding_column_action;
 }
 
-ActionsDAGPtr ActionsDAG::merge(ActionsDAG && first, ActionsDAG && second)
+ActionsDAG ActionsDAG::merge(ActionsDAG && first, ActionsDAG && second)
 {
     first.mergeInplace(std::move(second));
 
     /// Some actions could become unused. Do not drop inputs to preserve the header.
     first.removeUnusedActions(false);
 
-    return std::make_unique<ActionsDAG>(std::move(first));
+    return std::move(first);
 }
 
 void ActionsDAG::mergeInplace(ActionsDAG && second)
@@ -1970,15 +1970,15 @@ ActionsDAG::SplitResult ActionsDAG::split(std::unordered_set<const Node *> split
             second_inputs.push_back(cur.to_second);
     }
 
-    auto first_actions = std::make_unique<ActionsDAG>();
-    first_actions->nodes.swap(first_nodes);
-    first_actions->outputs.swap(first_outputs);
-    first_actions->inputs.swap(first_inputs);
+    ActionsDAG first_actions;
+    first_actions.nodes.swap(first_nodes);
+    first_actions.outputs.swap(first_outputs);
+    first_actions.inputs.swap(first_inputs);
 
-    auto second_actions = std::make_unique<ActionsDAG>();
-    second_actions->nodes.swap(second_nodes);
-    second_actions->outputs.swap(second_outputs);
-    second_actions->inputs.swap(second_inputs);
+    ActionsDAG second_actions;
+    second_actions.nodes.swap(second_nodes);
+    second_actions.outputs.swap(second_outputs);
+    second_actions.inputs.swap(second_inputs);
 
     std::unordered_map<const Node *, const Node *> split_nodes_mapping;
     if (create_split_nodes_mapping)
@@ -2098,7 +2098,7 @@ ActionsDAG::SplitResult ActionsDAG::splitActionsBySortingDescription(const NameS
     return res;
 }
 
-bool ActionsDAG::isFilterAlwaysFalseForDefaultValueInputs(const std::string & filter_name, const Block & input_stream_header)
+bool ActionsDAG::isFilterAlwaysFalseForDefaultValueInputs(const std::string & filter_name, const Block & input_stream_header) const
 {
     const auto * filter_node = tryFindInOutputs(filter_name);
     if (!filter_node)
@@ -2122,7 +2122,7 @@ bool ActionsDAG::isFilterAlwaysFalseForDefaultValueInputs(const std::string & fi
         input_node_name_to_default_input_column.emplace(input->result_name, std::move(constant_column_with_type_and_name));
     }
 
-    ActionsDAGPtr filter_with_default_value_inputs;
+    std::optional<ActionsDAG> filter_with_default_value_inputs;
 
     try
     {
@@ -2304,12 +2304,12 @@ ColumnsWithTypeAndName prepareFunctionArguments(const ActionsDAG::NodeRawConstPt
 ///
 /// Result actions add single column with conjunction result (it is always first in outputs).
 /// No other columns are added or removed.
-ActionsDAGPtr ActionsDAG::createActionsForConjunction(NodeRawConstPtrs conjunction, const ColumnsWithTypeAndName & all_inputs)
+std::optional<ActionsDAG> ActionsDAG::createActionsForConjunction(NodeRawConstPtrs conjunction, const ColumnsWithTypeAndName & all_inputs)
 {
     if (conjunction.empty())
-        return nullptr;
+        return {};
 
-    auto actions = std::make_unique<ActionsDAG>();
+    ActionsDAG actions;
 
     FunctionOverloadResolverPtr func_builder_and = std::make_unique<FunctionToOverloadResolverAdaptor>(std::make_shared<FunctionAnd>());
 
@@ -2350,7 +2350,7 @@ ActionsDAGPtr ActionsDAG::createActionsForConjunction(NodeRawConstPtrs conjuncti
 
             if (cur.next_child_to_visit == cur.node->children.size())
             {
-                auto & node = actions->nodes.emplace_back(*cur.node);
+                auto & node = actions.nodes.emplace_back(*cur.node);
                 nodes_mapping[cur.node] = &node;
 
                 for (auto & child : node.children)
@@ -2373,33 +2373,33 @@ ActionsDAGPtr ActionsDAG::createActionsForConjunction(NodeRawConstPtrs conjuncti
         for (const auto * predicate : conjunction)
             args.emplace_back(nodes_mapping[predicate]);
 
-        result_predicate = &actions->addFunction(func_builder_and, std::move(args), {});
+        result_predicate = &actions.addFunction(func_builder_and, std::move(args), {});
     }
 
-    actions->outputs.push_back(result_predicate);
+    actions.outputs.push_back(result_predicate);
 
     for (const auto & col : all_inputs)
     {
         const Node * input;
         auto & list = required_inputs[col.name];
         if (list.empty())
-            input = &actions->addInput(col);
+            input = &actions.addInput(col);
         else
         {
             input = list.front();
             list.pop_front();
-            actions->inputs.push_back(input);
+            actions.inputs.push_back(input);
         }
 
         /// We should not add result_predicate into the outputs for the second time.
         if (input->result_name != result_predicate->result_name)
-            actions->outputs.push_back(input);
+            actions.outputs.push_back(input);
     }
 
     return actions;
 }
 
-ActionsDAGPtr ActionsDAG::splitActionsForFilterPushDown(
+std::optional<ActionsDAG> ActionsDAG::splitActionsForFilterPushDown(
     const std::string & filter_name,
     bool removes_filter,
     const Names & available_inputs,
@@ -2415,7 +2415,7 @@ ActionsDAGPtr ActionsDAG::splitActionsForFilterPushDown(
     /// If condition is constant let's do nothing.
     /// It means there is nothing to push down or optimization was already applied.
     if (predicate->type == ActionType::COLUMN)
-        return nullptr;
+        return {};
 
     std::unordered_set<const Node *> allowed_nodes;
 
@@ -2439,7 +2439,7 @@ ActionsDAGPtr ActionsDAG::splitActionsForFilterPushDown(
     auto conjunction = getConjunctionNodes(predicate, allowed_nodes);
 
     if (conjunction.allowed.empty())
-        return nullptr;
+        return {};
 
     chassert(predicate->result_type);
 
@@ -2451,13 +2451,13 @@ ActionsDAGPtr ActionsDAG::splitActionsForFilterPushDown(
             && !conjunction.rejected.front()->result_type->equals(*predicate->result_type))
         {
             /// No further optimization can be done
-            return nullptr;
+            return {};
         }
     }
 
     auto actions = createActionsForConjunction(conjunction.allowed, all_inputs);
     if (!actions)
-        return nullptr;
+        return {};
 
     /// Now, when actions are created, update the current DAG.
     removeUnusedConjunctions(std::move(conjunction.rejected), predicate, removes_filter);
@@ -2562,11 +2562,11 @@ ActionsDAG::ActionsForJOINFilterPushDown ActionsDAG::splitActionsForJOINFilterPu
     auto left_stream_filter_to_push_down = createActionsForConjunction(left_stream_allowed_conjunctions, left_stream_header.getColumnsWithTypeAndName());
     auto right_stream_filter_to_push_down = createActionsForConjunction(right_stream_allowed_conjunctions, right_stream_header.getColumnsWithTypeAndName());
 
-    auto replace_equivalent_columns_in_filter = [](const ActionsDAGPtr & filter,
+    auto replace_equivalent_columns_in_filter = [](const ActionsDAG & filter,
         const Block & stream_header,
         const std::unordered_map<std::string, ColumnWithTypeAndName> & columns_to_replace)
     {
-        auto updated_filter = ActionsDAG::buildFilterActionsDAG({filter->getOutputs()[0]}, columns_to_replace);
+        auto updated_filter = ActionsDAG::buildFilterActionsDAG({filter.getOutputs()[0]}, columns_to_replace);
         chassert(updated_filter->getOutputs().size() == 1);
 
         /** If result filter to left or right stream has column that is one of the stream inputs, we need distinguish filter column from
@@ -2587,7 +2587,7 @@ ActionsDAG::ActionsForJOINFilterPushDown ActionsDAG::splitActionsForJOINFilterPu
         for (const auto & input : updated_filter->getInputs())
             updated_filter_inputs[input->result_name].push_back(input);
 
-        for (const auto & input : filter->getInputs())
+        for (const auto & input : filter.getInputs())
         {
             if (updated_filter_inputs.contains(input->result_name))
                 continue;
@@ -2625,12 +2625,12 @@ ActionsDAG::ActionsForJOINFilterPushDown ActionsDAG::splitActionsForJOINFilterPu
     };
 
     if (left_stream_filter_to_push_down)
-        left_stream_filter_to_push_down = replace_equivalent_columns_in_filter(left_stream_filter_to_push_down,
+        left_stream_filter_to_push_down = replace_equivalent_columns_in_filter(*left_stream_filter_to_push_down,
             left_stream_header,
             equivalent_right_stream_column_to_left_stream_column);
 
     if (right_stream_filter_to_push_down)
-        right_stream_filter_to_push_down = replace_equivalent_columns_in_filter(right_stream_filter_to_push_down,
+        right_stream_filter_to_push_down = replace_equivalent_columns_in_filter(*right_stream_filter_to_push_down,
             right_stream_header,
             equivalent_left_stream_column_to_right_stream_column);
 
@@ -2859,13 +2859,13 @@ bool ActionsDAG::isSortingPreserved(
     return true;
 }
 
-ActionsDAGPtr ActionsDAG::buildFilterActionsDAG(
+std::optional<ActionsDAG> ActionsDAG::buildFilterActionsDAG(
     const NodeRawConstPtrs & filter_nodes,
     const std::unordered_map<std::string, ColumnWithTypeAndName> & node_name_to_input_node_column,
     bool single_output_condition_node)
 {
     if (filter_nodes.empty())
-        return nullptr;
+        return {};
 
     struct Frame
     {
@@ -2873,7 +2873,7 @@ ActionsDAGPtr ActionsDAG::buildFilterActionsDAG(
         bool visited_children = false;
     };
 
-    auto result_dag = std::make_unique<ActionsDAG>();
+    ActionsDAG result_dag;
     std::unordered_map<std::string, const ActionsDAG::Node *> result_inputs;
     std::unordered_map<const ActionsDAG::Node *, const ActionsDAG::Node *> node_to_result_node;
 
@@ -2904,7 +2904,7 @@ ActionsDAGPtr ActionsDAG::buildFilterActionsDAG(
         {
             auto & result_input = result_inputs[input_node_it->second.name];
             if (!result_input)
-                result_input = &result_dag->addInput(input_node_it->second);
+                result_input = &result_dag.addInput(input_node_it->second);
 
             node_to_result_node.emplace(node, result_input);
             nodes_to_process.pop_back();
@@ -2931,25 +2931,25 @@ ActionsDAGPtr ActionsDAG::buildFilterActionsDAG(
             {
                 auto & result_input = result_inputs[node->result_name];
                 if (!result_input)
-                    result_input = &result_dag->addInput({node->column, node->result_type, node->result_name});
+                    result_input = &result_dag.addInput({node->column, node->result_type, node->result_name});
                 result_node = result_input;
                 break;
             }
             case ActionsDAG::ActionType::COLUMN:
             {
-                result_node = &result_dag->addColumn({node->column, node->result_type, node->result_name});
+                result_node = &result_dag.addColumn({node->column, node->result_type, node->result_name});
                 break;
             }
             case ActionsDAG::ActionType::ALIAS:
             {
                 const auto * child = node->children.front();
-                result_node = &result_dag->addAlias(*(node_to_result_node.find(child)->second), node->result_name);
+                result_node = &result_dag.addAlias(*(node_to_result_node.find(child)->second), node->result_name);
                 break;
             }
             case ActionsDAG::ActionType::ARRAY_JOIN:
             {
                 const auto * child = node->children.front();
-                result_node = &result_dag->addArrayJoin(*(node_to_result_node.find(child)->second), {});
+                result_node = &result_dag.addArrayJoin(*(node_to_result_node.find(child)->second), {});
                 break;
             }
             case ActionsDAG::ActionType::FUNCTION:
@@ -2967,13 +2967,11 @@ ActionsDAGPtr ActionsDAG::buildFilterActionsDAG(
                     {
                         if (const auto * index_hint = typeid_cast<const FunctionIndexHint *>(adaptor->getFunction().get()))
                         {
-                            ActionsDAGPtr index_hint_filter_dag;
-                            const auto & index_hint_args = index_hint->getActions()->getOutputs();
+                            ActionsDAG index_hint_filter_dag;
+                            const auto & index_hint_args = index_hint->getActions().getOutputs();
 
-                            if (index_hint_args.empty())
-                                index_hint_filter_dag = std::make_unique<ActionsDAG>();
-                            else
-                                index_hint_filter_dag = buildFilterActionsDAG(index_hint_args,
+                            if (!index_hint_args.empty())
+                                index_hint_filter_dag = *buildFilterActionsDAG(index_hint_args,
                                     node_name_to_input_node_column,
                                     false /*single_output_condition_node*/);
 
@@ -2995,7 +2993,7 @@ ActionsDAGPtr ActionsDAG::buildFilterActionsDAG(
                 auto [arguments, all_const] = getFunctionArguments(function_children);
                 auto function_base = function_overload_resolver ? function_overload_resolver->build(arguments) : node->function_base;
 
-                result_node = &result_dag->addFunctionImpl(
+                result_node = &result_dag.addFunctionImpl(
                     function_base,
                     std::move(function_children),
                     std::move(arguments),
@@ -3010,7 +3008,7 @@ ActionsDAGPtr ActionsDAG::buildFilterActionsDAG(
         nodes_to_process.pop_back();
     }
 
-    auto & result_dag_outputs = result_dag->getOutputs();
+    auto & result_dag_outputs = result_dag.getOutputs();
     result_dag_outputs.reserve(filter_nodes_size);
 
     for (const auto & node : filter_nodes)
@@ -3019,7 +3017,7 @@ ActionsDAGPtr ActionsDAG::buildFilterActionsDAG(
     if (result_dag_outputs.size() > 1 && single_output_condition_node)
     {
         FunctionOverloadResolverPtr func_builder_and = std::make_unique<FunctionToOverloadResolverAdaptor>(std::make_shared<FunctionAnd>());
-        result_dag_outputs = { &result_dag->addFunction(func_builder_and, result_dag_outputs, {}) };
+        result_dag_outputs = { &result_dag.addFunction(func_builder_and, result_dag_outputs, {}) };
     }
 
     return result_dag;
diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h
index 7ca3d1c1b0d..cf6a91b9fe7 100644
--- a/src/Interpreters/ActionsDAG.h
+++ b/src/Interpreters/ActionsDAG.h
@@ -247,7 +247,7 @@ public:
     ///        c * d       e
     ///            \      /
     ///            c * d - e
-    static ActionsDAGPtr foldActionsByProjection(
+    static ActionsDAG foldActionsByProjection(
         const std::unordered_map<const Node *, const Node *> & new_inputs,
         const NodeRawConstPtrs & required_outputs);
 
@@ -303,7 +303,7 @@ public:
     /// @param ignore_constant_values - Do not check that constants are same. Use value from result_header.
     /// @param add_casted_columns - Create new columns with converted values instead of replacing original.
     /// @param new_names - Output parameter for new column names when add_casted_columns is used.
-    static ActionsDAGPtr makeConvertingActions(
+    static ActionsDAG makeConvertingActions(
         const ColumnsWithTypeAndName & source,
         const ColumnsWithTypeAndName & result,
         MatchColumnsMode mode,
@@ -312,13 +312,13 @@ public:
         NameToNameMap * new_names = nullptr);
 
     /// Create expression which add const column and then materialize it.
-    static ActionsDAGPtr makeAddingColumnActions(ColumnWithTypeAndName column);
+    static ActionsDAG makeAddingColumnActions(ColumnWithTypeAndName column);
 
     /// Create ActionsDAG which represents expression equivalent to applying first and second actions consequently.
     /// Is used to replace `(first -> second)` expression chain to single `merge(first, second)` expression.
     /// If first.settings.project_input is set, then outputs of `first` must include inputs of `second`.
     /// Otherwise, any two actions may be combined.
-    static ActionsDAGPtr merge(ActionsDAG && first, ActionsDAG && second);
+    static ActionsDAG merge(ActionsDAG && first, ActionsDAG && second);
 
     /// The result is similar to merge(*this, second);
     /// Invariant : no nodes are removed from the first (this) DAG.
@@ -329,12 +329,7 @@ public:
     /// *out_outputs is filled with pointers to the nodes corresponding to second.getOutputs().
     void mergeNodes(ActionsDAG && second, NodeRawConstPtrs * out_outputs = nullptr);
 
-    struct SplitResult
-    {
-        ActionsDAGPtr first;
-        ActionsDAGPtr second;
-        std::unordered_map<const Node *, const Node *> split_nodes_mapping;
-    };
+    struct SplitResult;
 
     /// Split ActionsDAG into two DAGs, where first part contains all nodes from split_nodes and their children.
     /// Execution of first then second parts on block is equivalent to execution of initial DAG.
@@ -362,7 +357,7 @@ public:
       * @param filter_name - name of filter node in current DAG.
       * @param input_stream_header - input stream header.
       */
-    bool isFilterAlwaysFalseForDefaultValueInputs(const std::string & filter_name, const Block & input_stream_header);
+    bool isFilterAlwaysFalseForDefaultValueInputs(const std::string & filter_name, const Block & input_stream_header) const;
 
     /// Create actions which may calculate part of filter using only available_inputs.
     /// If nothing may be calculated, returns nullptr.
@@ -381,19 +376,13 @@ public:
     /// columns will be transformed like `x, y, z` -> `z > 0, z, x, y` -(remove filter)-> `z, x, y`.
     /// To avoid it, add inputs from `all_inputs` list,
     /// so actions `x, y, z -> z > 0, x, y, z` -(remove filter)-> `x, y, z` will not change columns order.
-    ActionsDAGPtr splitActionsForFilterPushDown(
+    std::optional<ActionsDAG> splitActionsForFilterPushDown(
         const std::string & filter_name,
         bool removes_filter,
         const Names & available_inputs,
         const ColumnsWithTypeAndName & all_inputs);
 
-    struct ActionsForJOINFilterPushDown
-    {
-        ActionsDAGPtr left_stream_filter_to_push_down;
-        bool left_stream_filter_removes_filter;
-        ActionsDAGPtr right_stream_filter_to_push_down;
-        bool right_stream_filter_removes_filter;
-    };
+    struct ActionsForJOINFilterPushDown;
 
     /** Split actions for JOIN filter push down.
       *
@@ -440,7 +429,7 @@ public:
       *
       * If single_output_condition_node = false, result dag has multiple output nodes.
       */
-    static ActionsDAGPtr buildFilterActionsDAG(
+    static std::optional<ActionsDAG> buildFilterActionsDAG(
         const NodeRawConstPtrs & filter_nodes,
         const std::unordered_map<std::string, ColumnWithTypeAndName> & node_name_to_input_node_column = {},
         bool single_output_condition_node = true);
@@ -472,11 +461,26 @@ private:
     void compileFunctions(size_t min_count_to_compile_expression, const std::unordered_set<const Node *> & lazy_executed_nodes = {});
 #endif
 
-    static ActionsDAGPtr createActionsForConjunction(NodeRawConstPtrs conjunction, const ColumnsWithTypeAndName & all_inputs);
+    static std::optional<ActionsDAG> createActionsForConjunction(NodeRawConstPtrs conjunction, const ColumnsWithTypeAndName & all_inputs);
 
     void removeUnusedConjunctions(NodeRawConstPtrs rejected_conjunctions, Node * predicate, bool removes_filter);
 };
 
+struct ActionsDAG::SplitResult
+{
+    ActionsDAG first;
+    ActionsDAG second;
+    std::unordered_map<const Node *, const Node *> split_nodes_mapping;
+};
+
+struct ActionsDAG::ActionsForJOINFilterPushDown
+{
+    std::optional<ActionsDAG> left_stream_filter_to_push_down;
+    bool left_stream_filter_removes_filter;
+    std::optional<ActionsDAG> right_stream_filter_to_push_down;
+    bool right_stream_filter_removes_filter;
+};
+
 class FindOriginalNodeForOutputName
 {
     using NameToNodeIndex = std::unordered_map<std::string_view, const ActionsDAG::Node *>;
diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp
index 036b5ba9be0..c2dcdcd34e7 100644
--- a/src/Interpreters/ActionsVisitor.cpp
+++ b/src/Interpreters/ActionsVisitor.cpp
@@ -1022,7 +1022,7 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data &
         dag.project(args);
 
         auto index_hint = std::make_shared<FunctionIndexHint>();
-        index_hint->setActions(std::make_unique<ActionsDAG>(std::move(dag)));
+        index_hint->setActions(std::move(dag));
 
         // Arguments are removed. We add function instead of constant column to avoid constant folding.
         data.addFunction(std::make_unique<FunctionToOverloadResolverAdaptor>(index_hint), {}, column_name);
@@ -1285,7 +1285,7 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data &
                     lambda_dag.removeUnusedActions(Names(1, result_name));
 
                     auto lambda_actions = std::make_shared<ExpressionActions>(
-                        std::make_unique<ActionsDAG>(std::move(lambda_dag)),
+                        std::move(lambda_dag),
                         ExpressionActionsSettings::fromContext(data.getContext(), CompileExpressions::yes));
 
                     DataTypePtr result_type = lambda_actions->getSampleBlock().getByName(result_name).type;
diff --git a/src/Interpreters/ExpressionActions.cpp b/src/Interpreters/ExpressionActions.cpp
index 2eca31fc75e..399f4f2ff4f 100644
--- a/src/Interpreters/ExpressionActions.cpp
+++ b/src/Interpreters/ExpressionActions.cpp
@@ -49,14 +49,13 @@ namespace ErrorCodes
 
 static std::unordered_set<const ActionsDAG::Node *> processShortCircuitFunctions(const ActionsDAG & actions_dag, ShortCircuitFunctionEvaluation short_circuit_function_evaluation);
 
-ExpressionActions::ExpressionActions(ActionsDAGPtr actions_dag_, const ExpressionActionsSettings & settings_, bool project_inputs_)
-    : project_inputs(project_inputs_)
+ExpressionActions::ExpressionActions(ActionsDAG actions_dag_, const ExpressionActionsSettings & settings_, bool project_inputs_)
+    : actions_dag(std::move(actions_dag_))
+    , project_inputs(project_inputs_)
     , settings(settings_)
 {
-    actions_dag = ActionsDAG::clone(actions_dag_);
-
     /// It's important to determine lazy executed nodes before compiling expressions.
-    std::unordered_set<const ActionsDAG::Node *> lazy_executed_nodes = processShortCircuitFunctions(*actions_dag, settings.short_circuit_function_evaluation);
+    std::unordered_set<const ActionsDAG::Node *> lazy_executed_nodes = processShortCircuitFunctions(actions_dag, settings.short_circuit_function_evaluation);
 
 #if USE_EMBEDDED_COMPILER
     if (settings.can_compile_expressions && settings.compile_expressions == CompileExpressions::yes)
@@ -68,7 +67,7 @@ ExpressionActions::ExpressionActions(ActionsDAGPtr actions_dag_, const Expressio
     if (settings.max_temporary_columns && num_columns > settings.max_temporary_columns)
         throw Exception(ErrorCodes::TOO_MANY_TEMPORARY_COLUMNS,
                         "Too many temporary columns: {}. Maximum: {}",
-                        actions_dag->dumpNames(), settings.max_temporary_columns);
+                        actions_dag.dumpNames(), settings.max_temporary_columns);
 }
 
 ExpressionActionsPtr ExpressionActions::clone() const
@@ -76,12 +75,12 @@ ExpressionActionsPtr ExpressionActions::clone() const
     auto copy = std::make_shared<ExpressionActions>(ExpressionActions());
 
     std::unordered_map<const Node *, Node *> copy_map;
-    copy->actions_dag = ActionsDAG::clone(actions_dag.get(), copy_map);
+    copy->actions_dag = std::move(*ActionsDAG::clone(&actions_dag, copy_map));
     copy->actions = actions;
     for (auto & action : copy->actions)
         action.node = copy_map[action.node];
 
-    for (const auto * input : copy->actions_dag->getInputs())
+    for (const auto * input : copy->actions_dag.getInputs())
         copy->input_positions.emplace(input->result_name, input_positions.at(input->result_name));
 
     copy->num_columns = num_columns;
@@ -357,8 +356,8 @@ void ExpressionActions::linearizeActions(const std::unordered_set<const ActionsD
     };
 
     const auto & nodes = getNodes();
-    const auto & outputs = actions_dag->getOutputs();
-    const auto & inputs = actions_dag->getInputs();
+    const auto & outputs = actions_dag.getOutputs();
+    const auto & inputs = actions_dag.getInputs();
 
     auto reverse_info = getActionsDAGReverseInfo(nodes, outputs);
     std::vector<Data> data;
diff --git a/src/Interpreters/ExpressionActions.h b/src/Interpreters/ExpressionActions.h
index 63ea989bd5e..6ff39ee07f7 100644
--- a/src/Interpreters/ExpressionActions.h
+++ b/src/Interpreters/ExpressionActions.h
@@ -70,7 +70,7 @@ public:
     using NameToInputMap = std::unordered_map<std::string_view, std::list<size_t>>;
 
 private:
-    ActionsDAGPtr actions_dag;
+    ActionsDAG actions_dag;
     Actions actions;
     size_t num_columns = 0;
 
@@ -84,13 +84,13 @@ private:
     ExpressionActionsSettings settings;
 
 public:
-    explicit ExpressionActions(ActionsDAGPtr actions_dag_, const ExpressionActionsSettings & settings_ = {}, bool project_inputs_ = false);
+    explicit ExpressionActions(ActionsDAG actions_dag_, const ExpressionActionsSettings & settings_ = {}, bool project_inputs_ = false);
     ExpressionActions(ExpressionActions &&) = default;
     ExpressionActions & operator=(ExpressionActions &&) = default;
 
     const Actions & getActions() const { return actions; }
-    const std::list<Node> & getNodes() const { return actions_dag->getNodes(); }
-    const ActionsDAG & getActionsDAG() const { return *actions_dag; }
+    const std::list<Node> & getNodes() const { return actions_dag.getNodes(); }
+    const ActionsDAG & getActionsDAG() const { return actions_dag; }
     const ColumnNumbers & getResultPositions() const { return result_positions; }
     const ExpressionActionsSettings & getSettings() const { return settings; }
 
diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp
index 6b49365b492..068b6f290fa 100644
--- a/src/Interpreters/ExpressionAnalyzer.cpp
+++ b/src/Interpreters/ExpressionAnalyzer.cpp
@@ -928,7 +928,7 @@ JoinPtr SelectQueryExpressionAnalyzer::appendJoin(
 {
     const ColumnsWithTypeAndName & left_sample_columns = chain.getLastStep().getResultColumns();
 
-    ActionsDAGPtr converting_actions;
+    std::optional<ActionsDAG> converting_actions;
     JoinPtr join = makeJoin(*syntax->ast_join, left_sample_columns, converting_actions);
 
     if (converting_actions)
@@ -1039,7 +1039,7 @@ static std::unique_ptr<QueryPlan> buildJoinedPlan(
     /// Actions which need to be calculated on joined block.
     auto joined_block_actions = analyzed_join.createJoinedBlockActions(context);
     NamesWithAliases required_columns_with_aliases = analyzed_join.getRequiredColumns(
-        Block(joined_block_actions->getResultColumns()), joined_block_actions->getRequiredColumns().getNames());
+        Block(joined_block_actions.getResultColumns()), joined_block_actions.getRequiredColumns().getNames());
 
     Names original_right_column_names;
     for (auto & pr : required_columns_with_aliases)
@@ -1060,17 +1060,17 @@ static std::unique_ptr<QueryPlan> buildJoinedPlan(
     interpreter->buildQueryPlan(*joined_plan);
     {
         Block original_right_columns = interpreter->getSampleBlock();
-        auto rename_dag = std::make_unique<ActionsDAG>(original_right_columns.getColumnsWithTypeAndName());
+        ActionsDAG rename_dag(original_right_columns.getColumnsWithTypeAndName());
         for (const auto & name_with_alias : required_columns_with_aliases)
         {
             if (name_with_alias.first != name_with_alias.second && original_right_columns.has(name_with_alias.first))
             {
                 auto pos = original_right_columns.getPositionByName(name_with_alias.first);
-                const auto & alias = rename_dag->addAlias(*rename_dag->getInputs()[pos], name_with_alias.second);
-                rename_dag->getOutputs()[pos] = &alias;
+                const auto & alias = rename_dag.addAlias(*rename_dag.getInputs()[pos], name_with_alias.second);
+                rename_dag.getOutputs()[pos] = &alias;
             }
         }
-        rename_dag->appendInputsForUnusedColumns(joined_plan->getCurrentDataStream().header);
+        rename_dag.appendInputsForUnusedColumns(joined_plan->getCurrentDataStream().header);
         auto rename_step = std::make_unique<ExpressionStep>(joined_plan->getCurrentDataStream(), std::move(rename_dag));
         rename_step->setStepDescription("Rename joined columns");
         joined_plan->addStep(std::move(rename_step));
@@ -1130,14 +1130,14 @@ std::shared_ptr<DirectKeyValueJoin> tryKeyValueJoin(std::shared_ptr<TableJoin> a
 JoinPtr SelectQueryExpressionAnalyzer::makeJoin(
     const ASTTablesInSelectQueryElement & join_element,
     const ColumnsWithTypeAndName & left_columns,
-    ActionsDAGPtr & left_convert_actions)
+    std::optional<ActionsDAG> & left_convert_actions)
 {
     /// Two JOINs are not supported with the same subquery, but different USINGs.
 
     if (joined_plan)
         throw Exception(ErrorCodes::LOGICAL_ERROR, "Table join was already created for query");
 
-    ActionsDAGPtr right_convert_actions = nullptr;
+    std::optional<ActionsDAG> right_convert_actions;
 
     const auto & analyzed_join = syntax->analyzed_join;
 
@@ -1145,7 +1145,7 @@ JoinPtr SelectQueryExpressionAnalyzer::makeJoin(
     {
         auto joined_block_actions = analyzed_join->createJoinedBlockActions(getContext());
         NamesWithAliases required_columns_with_aliases = analyzed_join->getRequiredColumns(
-            Block(joined_block_actions->getResultColumns()), joined_block_actions->getRequiredColumns().getNames());
+            Block(joined_block_actions.getResultColumns()), joined_block_actions.getRequiredColumns().getNames());
 
         Names original_right_column_names;
         for (auto & pr : required_columns_with_aliases)
@@ -1162,7 +1162,7 @@ JoinPtr SelectQueryExpressionAnalyzer::makeJoin(
     std::tie(left_convert_actions, right_convert_actions) = analyzed_join->createConvertingActions(left_columns, right_columns);
     if (right_convert_actions)
     {
-        auto converting_step = std::make_unique<ExpressionStep>(joined_plan->getCurrentDataStream(), right_convert_actions);
+        auto converting_step = std::make_unique<ExpressionStep>(joined_plan->getCurrentDataStream(), std::move(*right_convert_actions));
         converting_step->setStepDescription("Convert joined columns");
         joined_plan->addStep(std::move(converting_step));
     }
@@ -1354,8 +1354,8 @@ bool SelectQueryExpressionAnalyzer::appendGroupBy(ExpressionActionsChain & chain
     {
         for (auto & child : asts)
         {
-            auto actions_dag = std::make_unique<ActionsDAG>(columns_after_join);
-            getRootActions(child, only_types, *actions_dag);
+            ActionsDAG actions_dag(columns_after_join);
+            getRootActions(child, only_types, actions_dag);
             group_by_elements_actions.emplace_back(
                 std::make_shared<ExpressionActions>(std::move(actions_dag), ExpressionActionsSettings::fromContext(getContext(), CompileExpressions::yes)));
         }
@@ -1471,7 +1471,7 @@ void SelectQueryExpressionAnalyzer::appendGroupByModifiers(ActionsDAG & before_a
     ExpressionActionsChain::Step & step = chain.addStep(before_aggregation.getNamesAndTypesList());
     step.required_output = std::move(required_output);
 
-    step.actions()->dag = std::move(*ActionsDAG::makeConvertingActions(source_columns, result_columns, ActionsDAG::MatchColumnsMode::Position));
+    step.actions()->dag = ActionsDAG::makeConvertingActions(source_columns, result_columns, ActionsDAG::MatchColumnsMode::Position);
 }
 
 void SelectQueryExpressionAnalyzer::appendSelectSkipWindowExpressions(ExpressionActionsChain::Step & step, ASTPtr const & node)
@@ -1607,8 +1607,8 @@ ActionsAndProjectInputsFlagPtr SelectQueryExpressionAnalyzer::appendOrderBy(Expr
     {
         for (const auto & child : select_query->orderBy()->children)
         {
-            auto actions_dag = std::make_unique<ActionsDAG>(columns_after_join);
-            getRootActions(child, only_types, *actions_dag);
+            ActionsDAG actions_dag(columns_after_join);
+            getRootActions(child, only_types, actions_dag);
             order_by_elements_actions.emplace_back(
                 std::make_shared<ExpressionActions>(std::move(actions_dag), ExpressionActionsSettings::fromContext(getContext(), CompileExpressions::yes)));
         }
@@ -1737,7 +1737,7 @@ void ExpressionAnalyzer::appendExpression(ExpressionActionsChain & chain, const
     step.addRequiredOutput(expr->getColumnName());
 }
 
-ActionsDAGPtr ExpressionAnalyzer::getActionsDAG(bool add_aliases, bool remove_unused_result)
+ActionsDAG ExpressionAnalyzer::getActionsDAG(bool add_aliases, bool remove_unused_result)
 {
     ActionsDAG actions_dag(aggregated_columns);
     NamesWithAliases result_columns;
@@ -1789,7 +1789,7 @@ ActionsDAGPtr ExpressionAnalyzer::getActionsDAG(bool add_aliases, bool remove_un
         actions_dag.removeUnusedActions(name_set);
     }
 
-    return std::make_unique<ActionsDAG>(std::move(actions_dag));
+    return actions_dag;
 }
 
 ExpressionActionsPtr ExpressionAnalyzer::getActions(bool add_aliases, bool remove_unused_result, CompileExpressions compile_expressions)
@@ -1798,10 +1798,10 @@ ExpressionActionsPtr ExpressionAnalyzer::getActions(bool add_aliases, bool remov
         getActionsDAG(add_aliases, remove_unused_result), ExpressionActionsSettings::fromContext(getContext(), compile_expressions), add_aliases && remove_unused_result);
 }
 
-ActionsDAGPtr ExpressionAnalyzer::getConstActionsDAG(const ColumnsWithTypeAndName & constant_inputs)
+ActionsDAG ExpressionAnalyzer::getConstActionsDAG(const ColumnsWithTypeAndName & constant_inputs)
 {
-    auto actions = std::make_unique<ActionsDAG>(constant_inputs);
-    getRootActions(query, true /* no_makeset_for_subqueries */, *actions, true /* only_consts */);
+    ActionsDAG actions(constant_inputs);
+    getRootActions(query, true /* no_makeset_for_subqueries */, actions, true /* only_consts */);
     return actions;
 }
 
@@ -1879,8 +1879,7 @@ ExpressionAnalysisResult::ExpressionAnalysisResult(
 
         if (prewhere_dag_and_flags)
         {
-            auto dag = std::make_unique<ActionsDAG>(std::move(prewhere_dag_and_flags->dag));
-            prewhere_info = std::make_shared<PrewhereInfo>(std::move(dag), query.prewhere()->getColumnName());
+            prewhere_info = std::make_shared<PrewhereInfo>(std::move(prewhere_dag_and_flags->dag), query.prewhere()->getColumnName());
             prewhere_dag_and_flags.reset();
         }
 
@@ -1944,7 +1943,7 @@ ExpressionAnalysisResult::ExpressionAnalysisResult(
                 Block before_prewhere_sample = source_header;
                 if (sanitizeBlock(before_prewhere_sample))
                 {
-                    auto dag = ActionsDAG::clone(&prewhere_dag_and_flags->dag);
+                    ActionsDAG dag = std::move(*ActionsDAG::clone(&prewhere_dag_and_flags->dag));
                     ExpressionActions(
                         std::move(dag),
                         ExpressionActionsSettings::fromSettings(context->getSettingsRef())).execute(before_prewhere_sample);
@@ -1980,7 +1979,7 @@ ExpressionAnalysisResult::ExpressionAnalysisResult(
                 if (sanitizeBlock(before_where_sample))
                 {
                     ExpressionActions(
-                        ActionsDAG::clone(&before_where->dag),
+                        std::move(*ActionsDAG::clone(&before_where->dag)),
                         ExpressionActionsSettings::fromSettings(context->getSettingsRef())).execute(before_where_sample);
 
                     auto & column_elem
@@ -2054,7 +2053,7 @@ ExpressionAnalysisResult::ExpressionAnalysisResult(
 
                     auto & step = chain.lastStep(query_analyzer.aggregated_columns);
                     auto & actions = step.actions()->dag;
-                    actions = std::move(*ActionsDAG::merge(std::move(actions), std::move(*converting)));
+                    actions = ActionsDAG::merge(std::move(actions), std::move(converting));
                 }
             }
 
@@ -2235,7 +2234,7 @@ void ExpressionAnalysisResult::checkActions() const
     /// Check that PREWHERE doesn't contain unusual actions. Unusual actions are that can change number of rows.
     if (hasPrewhere())
     {
-        auto check_actions = [](const ActionsDAGPtr & actions)
+        auto check_actions = [](const std::optional<ActionsDAG> & actions)
         {
             if (actions)
                 for (const auto & node : actions->getNodes())
diff --git a/src/Interpreters/ExpressionAnalyzer.h b/src/Interpreters/ExpressionAnalyzer.h
index e44a5891e77..737d36eb504 100644
--- a/src/Interpreters/ExpressionAnalyzer.h
+++ b/src/Interpreters/ExpressionAnalyzer.h
@@ -117,12 +117,12 @@ public:
     /// If add_aliases, only the calculated values in the desired order and add aliases.
     ///     If also remove_unused_result, than only aliases remain in the output block.
     /// Otherwise, only temporary columns will be deleted from the block.
-    ActionsDAGPtr getActionsDAG(bool add_aliases, bool remove_unused_result = true);
+    ActionsDAG getActionsDAG(bool add_aliases, bool remove_unused_result = true);
     ExpressionActionsPtr getActions(bool add_aliases, bool remove_unused_result = true, CompileExpressions compile_expressions = CompileExpressions::no);
 
     /// Get actions to evaluate a constant expression. The function adds constants and applies functions that depend only on constants.
     /// Does not execute subqueries.
-    ActionsDAGPtr getConstActionsDAG(const ColumnsWithTypeAndName & constant_inputs = {});
+    ActionsDAG getConstActionsDAG(const ColumnsWithTypeAndName & constant_inputs = {});
     ExpressionActionsPtr getConstActions(const ColumnsWithTypeAndName & constant_inputs = {});
 
     /** Sets that require a subquery to be create.
@@ -367,7 +367,7 @@ private:
     JoinPtr makeJoin(
         const ASTTablesInSelectQueryElement & join_element,
         const ColumnsWithTypeAndName & left_columns,
-        ActionsDAGPtr & left_convert_actions);
+        std::optional<ActionsDAG> & left_convert_actions);
 
     const ASTSelectQuery * getAggregatingQuery() const;
 
diff --git a/src/Interpreters/GlobalSubqueriesVisitor.h b/src/Interpreters/GlobalSubqueriesVisitor.h
index 64b6eb5dce9..fcf0d591918 100644
--- a/src/Interpreters/GlobalSubqueriesVisitor.h
+++ b/src/Interpreters/GlobalSubqueriesVisitor.h
@@ -295,7 +295,7 @@ private:
             {
                 auto joined_block_actions = data.table_join->createJoinedBlockActions(data.getContext());
                 NamesWithAliases required_columns_with_aliases = data.table_join->getRequiredColumns(
-                    Block(joined_block_actions->getResultColumns()), joined_block_actions->getRequiredColumns().getNames());
+                    Block(joined_block_actions.getResultColumns()), joined_block_actions.getRequiredColumns().getNames());
 
                 for (auto & pr : required_columns_with_aliases)
                     required_columns.push_back(pr.first);
diff --git a/src/Interpreters/IInterpreterUnionOrSelectQuery.cpp b/src/Interpreters/IInterpreterUnionOrSelectQuery.cpp
index fed29b410db..288d06d2220 100644
--- a/src/Interpreters/IInterpreterUnionOrSelectQuery.cpp
+++ b/src/Interpreters/IInterpreterUnionOrSelectQuery.cpp
@@ -99,16 +99,16 @@ static ASTPtr parseAdditionalPostFilter(const Context & context)
                 "additional filter", settings.max_query_size, settings.max_parser_depth, settings.max_parser_backtracks);
 }
 
-static ActionsDAGPtr makeAdditionalPostFilter(ASTPtr & ast, ContextPtr context, const Block & header)
+static ActionsDAG makeAdditionalPostFilter(ASTPtr & ast, ContextPtr context, const Block & header)
 {
     auto syntax_result = TreeRewriter(context).analyze(ast, header.getNamesAndTypesList());
     String result_column_name = ast->getColumnName();
     auto dag = ExpressionAnalyzer(ast, syntax_result, context).getActionsDAG(false, false);
-    const ActionsDAG::Node * result_node = &dag->findInOutputs(result_column_name);
-    auto & outputs = dag->getOutputs();
+    const ActionsDAG::Node * result_node = &dag.findInOutputs(result_column_name);
+    auto & outputs = dag.getOutputs();
     outputs.clear();
-    outputs.reserve(dag->getInputs().size() + 1);
-    for (const auto * node : dag->getInputs())
+    outputs.reserve(dag.getInputs().size() + 1);
+    for (const auto * node : dag.getInputs())
         outputs.push_back(node);
 
     outputs.push_back(result_node);
@@ -126,7 +126,7 @@ void IInterpreterUnionOrSelectQuery::addAdditionalPostFilter(QueryPlan & plan) c
         return;
 
     auto dag = makeAdditionalPostFilter(ast, context, plan.getCurrentDataStream().header);
-    std::string filter_name = dag->getOutputs().back()->result_name;
+    std::string filter_name = dag.getOutputs().back()->result_name;
     auto filter_step = std::make_unique<FilterStep>(
         plan.getCurrentDataStream(), std::move(dag), std::move(filter_name), true);
     filter_step->setStepDescription("Additional result filter");
diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp
index 09a7e440f31..cde6e305005 100644
--- a/src/Interpreters/InterpreterSelectQuery.cpp
+++ b/src/Interpreters/InterpreterSelectQuery.cpp
@@ -175,7 +175,7 @@ FilterDAGInfoPtr generateFilterActions(
     /// Using separate expression analyzer to prevent any possible alias injection
     auto syntax_result = TreeRewriter(context).analyzeSelect(query_ast, TreeRewriterResult({}, storage, storage_snapshot));
     SelectQueryExpressionAnalyzer analyzer(query_ast, syntax_result, context, metadata_snapshot, {}, false, {}, prepared_sets);
-    filter_info->actions = std::make_unique<ActionsDAG>(std::move(analyzer.simpleSelectActions()->dag));
+    filter_info->actions = std::move(analyzer.simpleSelectActions()->dag);
 
     filter_info->column_name = expr_list->children.at(0)->getColumnName();
     filter_info->actions->removeUnusedActions(NameSet{filter_info->column_name});
@@ -938,7 +938,8 @@ bool InterpreterSelectQuery::adjustParallelReplicasAfterAnalysis()
         }
     }
 
-    query_info_copy.filter_actions_dag = ActionsDAG::buildFilterActionsDAG(added_filter_nodes.nodes);
+    if (auto filter_actions_dag = ActionsDAG::buildFilterActionsDAG(added_filter_nodes.nodes))
+        query_info_copy.filter_actions_dag = std::make_shared<const ActionsDAG>(std::move(*filter_actions_dag));
     UInt64 rows_to_read = storage_merge_tree->estimateNumberOfRowsToRead(context, storage_snapshot, query_info_copy);
     /// Note that we treat an estimation of 0 rows as a real estimation
     size_t number_of_replicas_to_use = rows_to_read / settings.parallel_replicas_min_number_of_rows_per_replica;
@@ -973,7 +974,7 @@ void InterpreterSelectQuery::buildQueryPlan(QueryPlan & query_plan)
             ActionsDAG::MatchColumnsMode::Name,
             true);
 
-        auto converting = std::make_unique<ExpressionStep>(query_plan.getCurrentDataStream(), convert_actions_dag);
+        auto converting = std::make_unique<ExpressionStep>(query_plan.getCurrentDataStream(), std::move(convert_actions_dag));
         query_plan.addStep(std::move(converting));
     }
 
@@ -1297,10 +1298,10 @@ static InterpolateDescriptionPtr getInterpolateDescription(
 
         auto syntax_result = TreeRewriter(context).analyze(exprs, source_columns);
         ExpressionAnalyzer analyzer(exprs, syntax_result, context);
-        ActionsDAGPtr actions = analyzer.getActionsDAG(true);
-        ActionsDAGPtr conv_dag = ActionsDAG::makeConvertingActions(actions->getResultColumns(),
+        ActionsDAG actions = analyzer.getActionsDAG(true);
+        ActionsDAG conv_dag = ActionsDAG::makeConvertingActions(actions.getResultColumns(),
             result_columns, ActionsDAG::MatchColumnsMode::Position, true);
-        ActionsDAGPtr merge_dag = ActionsDAG::merge(std::move(* ActionsDAG::clone(actions)), std::move(*conv_dag));
+        ActionsDAG merge_dag = ActionsDAG::merge(std::move(actions), std::move(conv_dag));
 
         interpolate_descr = std::make_shared<InterpolateDescription>(std::move(merge_dag), aliases);
     }
@@ -1485,7 +1486,7 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional<P
         {
             auto row_level_security_step = std::make_unique<FilterStep>(
                 query_plan.getCurrentDataStream(),
-                expressions.filter_info->actions,
+                std::move(*ActionsDAG::clone(&*expressions.filter_info->actions)),
                 expressions.filter_info->column_name,
                 expressions.filter_info->do_remove_column);
 
@@ -1499,7 +1500,7 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional<P
             {
                 auto row_level_filter_step = std::make_unique<FilterStep>(
                     query_plan.getCurrentDataStream(),
-                    expressions.prewhere_info->row_level_filter,
+                    std::move(*ActionsDAG::clone(&*expressions.prewhere_info->row_level_filter)),
                     expressions.prewhere_info->row_level_column_name,
                     true);
 
@@ -1509,7 +1510,7 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional<P
 
             auto prewhere_step = std::make_unique<FilterStep>(
                 query_plan.getCurrentDataStream(),
-                expressions.prewhere_info->prewhere_actions,
+                std::move(*ActionsDAG::clone(&*expressions.prewhere_info->prewhere_actions)),
                 expressions.prewhere_info->prewhere_column_name,
                 expressions.prewhere_info->remove_prewhere_column);
 
@@ -1611,7 +1612,7 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional<P
             {
                 auto row_level_security_step = std::make_unique<FilterStep>(
                     query_plan.getCurrentDataStream(),
-                    expressions.filter_info->actions,
+                    std::move(*ActionsDAG::clone(&*expressions.filter_info->actions)),
                     expressions.filter_info->column_name,
                     expressions.filter_info->do_remove_column);
 
@@ -1623,7 +1624,7 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional<P
             {
                 auto filter_step = std::make_unique<FilterStep>(
                     query_plan.getCurrentDataStream(),
-                    new_filter_info->actions,
+                    std::move(*ActionsDAG::clone(&*new_filter_info->actions)),
                     new_filter_info->column_name,
                     new_filter_info->do_remove_column);
 
@@ -2045,7 +2046,7 @@ void InterpreterSelectQuery::addEmptySourceToQueryPlan(QueryPlan & query_plan, c
             pipe.addSimpleTransform([&](const Block & header)
             {
                 return std::make_shared<FilterTransform>(header,
-                    std::make_shared<ExpressionActions>(ActionsDAG::clone(prewhere_info.row_level_filter)),
+                    std::make_shared<ExpressionActions>(std::move(*ActionsDAG::clone(&*prewhere_info.row_level_filter))),
                     prewhere_info.row_level_column_name, true);
             });
         }
@@ -2053,7 +2054,7 @@ void InterpreterSelectQuery::addEmptySourceToQueryPlan(QueryPlan & query_plan, c
         pipe.addSimpleTransform([&](const Block & header)
         {
             return std::make_shared<FilterTransform>(
-                header, std::make_shared<ExpressionActions>(ActionsDAG::clone(prewhere_info.prewhere_actions)),
+                header, std::make_shared<ExpressionActions>(std::move(*ActionsDAG::clone(&*prewhere_info.prewhere_actions))),
                 prewhere_info.prewhere_column_name, prewhere_info.remove_prewhere_column);
         });
     }
@@ -2106,7 +2107,7 @@ void InterpreterSelectQuery::applyFiltersToPrewhereInAnalysis(ExpressionAnalysis
     else
     {
         /// Add row level security actions to prewhere.
-        analysis.prewhere_info->row_level_filter = std::move(analysis.filter_info->actions);
+        analysis.prewhere_info->row_level_filter = std::move(*analysis.filter_info->actions);
         analysis.prewhere_info->row_level_column_name = std::move(analysis.filter_info->column_name);
         analysis.filter_info = nullptr;
     }
@@ -2323,7 +2324,7 @@ std::optional<UInt64> InterpreterSelectQuery::getTrivialCount(UInt64 max_paralle
         if (!filter_actions_dag)
             return {};
 
-        return storage->totalRowsByPartitionPredicate(filter_actions_dag, context);
+        return storage->totalRowsByPartitionPredicate(*filter_actions_dag, context);
     }
 }
 
@@ -2573,7 +2574,7 @@ void InterpreterSelectQuery::executeFetchColumns(QueryProcessingStage::Enum proc
     /// Aliases in table declaration.
     if (processing_stage == QueryProcessingStage::FetchColumns && alias_actions)
     {
-        auto table_aliases = std::make_unique<ExpressionStep>(query_plan.getCurrentDataStream(), alias_actions);
+        auto table_aliases = std::make_unique<ExpressionStep>(query_plan.getCurrentDataStream(), std::move(*ActionsDAG::clone(&*alias_actions)));
         table_aliases->setStepDescription("Add table aliases");
         query_plan.addStep(std::move(table_aliases));
     }
@@ -2581,9 +2582,9 @@ void InterpreterSelectQuery::executeFetchColumns(QueryProcessingStage::Enum proc
 
 void InterpreterSelectQuery::executeWhere(QueryPlan & query_plan, const ActionsAndProjectInputsFlagPtr & expression, bool remove_filter)
 {
-    auto dag = ActionsDAG::clone(&expression->dag);
+    auto dag = std::move(*ActionsDAG::clone(&expression->dag));
     if (expression->project_input)
-        dag->appendInputsForUnusedColumns(query_plan.getCurrentDataStream().header);
+        dag.appendInputsForUnusedColumns(query_plan.getCurrentDataStream().header);
 
     auto where_step = std::make_unique<FilterStep>(
         query_plan.getCurrentDataStream(), std::move(dag), getSelectQuery().where()->getColumnName(), remove_filter);
@@ -2755,9 +2756,9 @@ void InterpreterSelectQuery::executeMergeAggregated(QueryPlan & query_plan, bool
 
 void InterpreterSelectQuery::executeHaving(QueryPlan & query_plan, const ActionsAndProjectInputsFlagPtr & expression, bool remove_filter)
 {
-    auto dag = ActionsDAG::clone(&expression->dag);
+    auto dag = std::move(*ActionsDAG::clone(&expression->dag));
     if (expression->project_input)
-        dag->appendInputsForUnusedColumns(query_plan.getCurrentDataStream().header);
+        dag.appendInputsForUnusedColumns(query_plan.getCurrentDataStream().header);
 
     auto having_step
         = std::make_unique<FilterStep>(query_plan.getCurrentDataStream(), std::move(dag), getSelectQuery().having()->getColumnName(), remove_filter);
@@ -2770,10 +2771,10 @@ void InterpreterSelectQuery::executeHaving(QueryPlan & query_plan, const Actions
 void InterpreterSelectQuery::executeTotalsAndHaving(
     QueryPlan & query_plan, bool has_having, const ActionsAndProjectInputsFlagPtr & expression, bool remove_filter, bool overflow_row, bool final)
 {
-    ActionsDAGPtr dag;
+    std::optional<ActionsDAG> dag;
     if (expression)
     {
-        dag = ActionsDAG::clone(&expression->dag);
+        dag = std::move(*ActionsDAG::clone(&expression->dag));
         if (expression->project_input)
             dag->appendInputsForUnusedColumns(query_plan.getCurrentDataStream().header);
     }
@@ -2822,9 +2823,9 @@ void InterpreterSelectQuery::executeExpression(QueryPlan & query_plan, const Act
     if (!expression)
         return;
 
-    auto dag = ActionsDAG::clone(&expression->dag);
+    ActionsDAG dag = std::move(*ActionsDAG::clone(&expression->dag));
     if (expression->project_input)
-        dag->appendInputsForUnusedColumns(query_plan.getCurrentDataStream().header);
+        dag.appendInputsForUnusedColumns(query_plan.getCurrentDataStream().header);
 
     auto expression_step = std::make_unique<ExpressionStep>(query_plan.getCurrentDataStream(), std::move(dag));
 
diff --git a/src/Interpreters/InterpreterSelectQuery.h b/src/Interpreters/InterpreterSelectQuery.h
index d4ed19d45ea..ed6dd8af3b2 100644
--- a/src/Interpreters/InterpreterSelectQuery.h
+++ b/src/Interpreters/InterpreterSelectQuery.h
@@ -240,7 +240,7 @@ private:
     Block source_header;
 
     /// Actions to calculate ALIAS if required.
-    ActionsDAGPtr alias_actions;
+    std::optional<ActionsDAG> alias_actions;
 
     /// The subquery interpreter, if the subquery
     std::unique_ptr<InterpreterSelectWithUnionQuery> interpreter_subquery;
diff --git a/src/Interpreters/MutationsInterpreter.cpp b/src/Interpreters/MutationsInterpreter.cpp
index 704c5ce7d8b..2372d26e83f 100644
--- a/src/Interpreters/MutationsInterpreter.cpp
+++ b/src/Interpreters/MutationsInterpreter.cpp
@@ -1197,7 +1197,7 @@ void MutationsInterpreter::Source::read(
         const auto & names = first_stage.filter_column_names;
         size_t num_filters = names.size();
 
-        ActionsDAGPtr filter;
+        std::optional<ActionsDAG> filter;
         if (!first_stage.filter_column_names.empty())
         {
             ActionsDAG::NodeRawConstPtrs nodes(num_filters);
@@ -1278,19 +1278,19 @@ QueryPipelineBuilder MutationsInterpreter::addStreamsForLaterStages(const std::v
 
             if (i < stage.filter_column_names.size())
             {
-                auto dag = ActionsDAG::clone(&step->actions()->dag);
+                auto dag = std::move(*ActionsDAG::clone(&step->actions()->dag));
                 if (step->actions()->project_input)
-                    dag->appendInputsForUnusedColumns(plan.getCurrentDataStream().header);
+                    dag.appendInputsForUnusedColumns(plan.getCurrentDataStream().header);
                 /// Execute DELETEs.
-                plan.addStep(std::make_unique<FilterStep>(plan.getCurrentDataStream(), dag, stage.filter_column_names[i], false));
+                plan.addStep(std::make_unique<FilterStep>(plan.getCurrentDataStream(), std::move(dag), stage.filter_column_names[i], false));
             }
             else
             {
-                auto dag = ActionsDAG::clone(&step->actions()->dag);
+                auto dag = std::move(*ActionsDAG::clone(&step->actions()->dag));
                 if (step->actions()->project_input)
-                    dag->appendInputsForUnusedColumns(plan.getCurrentDataStream().header);
+                    dag.appendInputsForUnusedColumns(plan.getCurrentDataStream().header);
                 /// Execute UPDATE or final projection.
-                plan.addStep(std::make_unique<ExpressionStep>(plan.getCurrentDataStream(), dag));
+                plan.addStep(std::make_unique<ExpressionStep>(plan.getCurrentDataStream(), std::move(dag)));
             }
         }
 
diff --git a/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp b/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp
index 4821d607d0e..c1d7acf0775 100644
--- a/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp
+++ b/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp
@@ -182,7 +182,7 @@ static NamesAndTypesList getNames(const ASTFunction & expr, ContextPtr context,
 
     ASTPtr temp_ast = expr.clone();
     auto syntax = TreeRewriter(context).analyze(temp_ast, columns);
-    auto required_columns = ExpressionAnalyzer(temp_ast, syntax, context).getActionsDAG(false)->getRequiredColumns();
+    auto required_columns = ExpressionAnalyzer(temp_ast, syntax, context).getActionsDAG(false).getRequiredColumns();
     return required_columns;
 }
 
diff --git a/src/Interpreters/TableJoin.cpp b/src/Interpreters/TableJoin.cpp
index baf3a743f40..c8c926db13c 100644
--- a/src/Interpreters/TableJoin.cpp
+++ b/src/Interpreters/TableJoin.cpp
@@ -462,19 +462,19 @@ static void makeColumnNameUnique(const ColumnsWithTypeAndName & source_columns,
     }
 }
 
-static ActionsDAGPtr createWrapWithTupleActions(
+static std::optional<ActionsDAG> createWrapWithTupleActions(
     const ColumnsWithTypeAndName & source_columns,
     std::unordered_set<std::string_view> && column_names_to_wrap,
     NameToNameMap & new_names)
 {
     if (column_names_to_wrap.empty())
-        return nullptr;
+        return {};
 
-    auto actions_dag = std::make_unique<ActionsDAG>(source_columns);
+    ActionsDAG actions_dag(source_columns);
 
     FunctionOverloadResolverPtr func_builder = std::make_unique<FunctionToOverloadResolverAdaptor>(std::make_shared<FunctionTuple>());
 
-    for (const auto * input_node : actions_dag->getInputs())
+    for (const auto * input_node : actions_dag.getInputs())
     {
         const auto & column_name = input_node->result_name;
         auto it = column_names_to_wrap.find(column_name);
@@ -485,9 +485,9 @@ static ActionsDAGPtr createWrapWithTupleActions(
         String node_name = "__wrapNullsafe(" + column_name + ")";
         makeColumnNameUnique(source_columns, node_name);
 
-        const auto & dst_node = actions_dag->addFunction(func_builder, {input_node}, node_name);
+        const auto & dst_node = actions_dag.addFunction(func_builder, {input_node}, node_name);
         new_names[column_name] = dst_node.result_name;
-        actions_dag->addOrReplaceInOutputs(dst_node);
+        actions_dag.addOrReplaceInOutputs(dst_node);
     }
 
     if (!column_names_to_wrap.empty())
@@ -537,21 +537,23 @@ std::pair<NameSet, NameSet> TableJoin::getKeysForNullSafeComparion(const Columns
     return {left_keys_to_wrap, right_keys_to_wrap};
 }
 
-static void mergeDags(ActionsDAGPtr & result_dag, ActionsDAGPtr && new_dag)
+static void mergeDags(std::optional<ActionsDAG> & result_dag, std::optional<ActionsDAG> && new_dag)
 {
+    if (!new_dag)
+        return;
     if (result_dag)
         result_dag->mergeInplace(std::move(*new_dag));
     else
         result_dag = std::move(new_dag);
 }
 
-std::pair<ActionsDAGPtr, ActionsDAGPtr>
+std::pair<std::optional<ActionsDAG>, std::optional<ActionsDAG>>
 TableJoin::createConvertingActions(
     const ColumnsWithTypeAndName & left_sample_columns,
     const ColumnsWithTypeAndName & right_sample_columns)
 {
-    ActionsDAGPtr left_dag = nullptr;
-    ActionsDAGPtr right_dag = nullptr;
+    std::optional<ActionsDAG> left_dag;
+    std::optional<ActionsDAG> right_dag;
     /** If the types are not equal, we need to convert them to a common type.
       * Example:
       *   SELECT * FROM t1 JOIN t2 ON t1.a = t2.b
@@ -693,7 +695,7 @@ void TableJoin::inferJoinKeyCommonType(const LeftNamesAndTypes & left, const Rig
     }
 }
 
-static ActionsDAGPtr changeKeyTypes(const ColumnsWithTypeAndName & cols_src,
+static std::optional<ActionsDAG> changeKeyTypes(const ColumnsWithTypeAndName & cols_src,
                                     const TableJoin::NameToTypeMap & type_mapping,
                                     bool add_new_cols,
                                     NameToNameMap & key_column_rename)
@@ -710,7 +712,7 @@ static ActionsDAGPtr changeKeyTypes(const ColumnsWithTypeAndName & cols_src,
         }
     }
     if (!has_some_to_do)
-        return nullptr;
+        return {};
 
     return ActionsDAG::makeConvertingActions(
         /* source= */ cols_src,
@@ -721,7 +723,7 @@ static ActionsDAGPtr changeKeyTypes(const ColumnsWithTypeAndName & cols_src,
         /* new_names= */ &key_column_rename);
 }
 
-static ActionsDAGPtr changeTypesToNullable(
+static std::optional<ActionsDAG> changeTypesToNullable(
     const ColumnsWithTypeAndName & cols_src,
     const NameSet & exception_cols)
 {
@@ -737,7 +739,7 @@ static ActionsDAGPtr changeTypesToNullable(
     }
 
     if (!has_some_to_do)
-        return nullptr;
+        return {};
 
     return ActionsDAG::makeConvertingActions(
         /* source= */ cols_src,
@@ -748,29 +750,29 @@ static ActionsDAGPtr changeTypesToNullable(
         /* new_names= */ nullptr);
 }
 
-ActionsDAGPtr TableJoin::applyKeyConvertToTable(
+std::optional<ActionsDAG> TableJoin::applyKeyConvertToTable(
     const ColumnsWithTypeAndName & cols_src,
     const NameToTypeMap & type_mapping,
     JoinTableSide table_side,
     NameToNameMap & key_column_rename)
 {
     if (type_mapping.empty())
-        return nullptr;
+        return {};
 
     /// Create DAG to convert key columns
-    ActionsDAGPtr convert_dag = changeKeyTypes(cols_src, type_mapping, !hasUsing(), key_column_rename);
+    auto convert_dag = changeKeyTypes(cols_src, type_mapping, !hasUsing(), key_column_rename);
     applyRename(table_side, key_column_rename);
     return convert_dag;
 }
 
-ActionsDAGPtr TableJoin::applyNullsafeWrapper(
+std::optional<ActionsDAG> TableJoin::applyNullsafeWrapper(
     const ColumnsWithTypeAndName & cols_src,
     const NameSet & columns_for_nullsafe_comparison,
     JoinTableSide table_side,
     NameToNameMap & key_column_rename)
 {
     if (columns_for_nullsafe_comparison.empty())
-        return nullptr;
+        return {};
 
     std::unordered_set<std::string_view> column_names_to_wrap;
     for (const auto & name : columns_for_nullsafe_comparison)
@@ -784,7 +786,7 @@ ActionsDAGPtr TableJoin::applyNullsafeWrapper(
     }
 
     /// Create DAG to wrap keys with tuple for null-safe comparison
-    ActionsDAGPtr null_safe_wrap_dag = createWrapWithTupleActions(cols_src, std::move(column_names_to_wrap), key_column_rename);
+    auto null_safe_wrap_dag = createWrapWithTupleActions(cols_src, std::move(column_names_to_wrap), key_column_rename);
     for (auto & clause : clauses)
     {
         for (size_t i : clause.nullsafe_compare_key_indexes)
@@ -799,7 +801,7 @@ ActionsDAGPtr TableJoin::applyNullsafeWrapper(
     return null_safe_wrap_dag;
 }
 
-ActionsDAGPtr TableJoin::applyJoinUseNullsConversion(
+std::optional<ActionsDAG> TableJoin::applyJoinUseNullsConversion(
     const ColumnsWithTypeAndName & cols_src,
     const NameToNameMap & key_column_rename)
 {
@@ -809,8 +811,7 @@ ActionsDAGPtr TableJoin::applyJoinUseNullsConversion(
         exclude_columns.insert(it.second);
 
     /// Create DAG to make columns nullable if needed
-    ActionsDAGPtr add_nullable_dag = changeTypesToNullable(cols_src, exclude_columns);
-    return add_nullable_dag;
+    return changeTypesToNullable(cols_src, exclude_columns);
 }
 
 void TableJoin::setStorageJoin(std::shared_ptr<const IKeyValueEntity> storage)
@@ -957,7 +958,7 @@ bool TableJoin::allowParallelHashJoin() const
     return true;
 }
 
-ActionsDAGPtr TableJoin::createJoinedBlockActions(ContextPtr context) const
+ActionsDAG TableJoin::createJoinedBlockActions(ContextPtr context) const
 {
     ASTPtr expression_list = rightKeysList();
     auto syntax_result = TreeRewriter(context).analyze(expression_list, columnsFromJoinedTable());
diff --git a/src/Interpreters/TableJoin.h b/src/Interpreters/TableJoin.h
index 8e83233e54c..a057d46b94d 100644
--- a/src/Interpreters/TableJoin.h
+++ b/src/Interpreters/TableJoin.h
@@ -202,19 +202,19 @@ private:
     Names requiredJoinedNames() const;
 
     /// Create converting actions and change key column names if required
-    ActionsDAGPtr applyKeyConvertToTable(
+    std::optional<ActionsDAG> applyKeyConvertToTable(
         const ColumnsWithTypeAndName & cols_src,
         const NameToTypeMap & type_mapping,
         JoinTableSide table_side,
         NameToNameMap & key_column_rename);
 
-    ActionsDAGPtr applyNullsafeWrapper(
+    std::optional<ActionsDAG> applyNullsafeWrapper(
         const ColumnsWithTypeAndName & cols_src,
         const NameSet & columns_for_nullsafe_comparison,
         JoinTableSide table_side,
         NameToNameMap & key_column_rename);
 
-    ActionsDAGPtr applyJoinUseNullsConversion(
+    std::optional<ActionsDAG> applyJoinUseNullsConversion(
         const ColumnsWithTypeAndName & cols_src,
         const NameToNameMap & key_column_rename);
 
@@ -264,7 +264,7 @@ public:
 
     TemporaryDataOnDiskScopePtr getTempDataOnDisk() { return tmp_data; }
 
-    ActionsDAGPtr createJoinedBlockActions(ContextPtr context) const;
+    ActionsDAG createJoinedBlockActions(ContextPtr context) const;
 
     const std::vector<JoinAlgorithm> & getEnabledJoinAlgorithms() const { return join_algorithm; }
 
@@ -379,7 +379,7 @@ public:
     /// Calculate converting actions, rename key columns in required
     /// For `USING` join we will convert key columns inplace and affect into types in the result table
     /// For `JOIN ON` we will create new columns with converted keys to join by.
-    std::pair<ActionsDAGPtr, ActionsDAGPtr>
+    std::pair<std::optional<ActionsDAG>, std::optional<ActionsDAG>>
     createConvertingActions(
         const ColumnsWithTypeAndName & left_sample_columns,
         const ColumnsWithTypeAndName & right_sample_columns);
diff --git a/src/Interpreters/addMissingDefaults.cpp b/src/Interpreters/addMissingDefaults.cpp
index 929999c8c37..27d79e86622 100644
--- a/src/Interpreters/addMissingDefaults.cpp
+++ b/src/Interpreters/addMissingDefaults.cpp
@@ -14,15 +14,15 @@
 namespace DB
 {
 
-ActionsDAGPtr addMissingDefaults(
+ActionsDAG addMissingDefaults(
     const Block & header,
     const NamesAndTypesList & required_columns,
     const ColumnsDescription & columns,
     ContextPtr context,
     bool null_as_default)
 {
-    auto actions = std::make_unique<ActionsDAG>(header.getColumnsWithTypeAndName());
-    auto & index = actions->getOutputs();
+    ActionsDAG actions(header.getColumnsWithTypeAndName());
+    auto & index = actions.getOutputs();
 
     /// For missing columns of nested structure, you need to create not a column of empty arrays, but a column of arrays of correct lengths.
     /// First, remember the offset columns for all arrays in the block.
@@ -40,7 +40,7 @@ ActionsDAGPtr addMissingDefaults(
             if (group.empty())
                 group.push_back(nullptr);
 
-            group.push_back(actions->getInputs()[i]);
+            group.push_back(actions.getInputs()[i]);
         }
     }
 
@@ -62,11 +62,11 @@ ActionsDAGPtr addMissingDefaults(
         {
             const auto & nested_type = array_type->getNestedType();
             ColumnPtr nested_column = nested_type->createColumnConstWithDefaultValue(0);
-            const auto & constant = actions->addColumn({nested_column, nested_type, column.name});
+            const auto & constant = actions.addColumn({nested_column, nested_type, column.name});
 
             auto & group = nested_groups[offsets_name];
             group[0] = &constant;
-            index.push_back(&actions->addFunction(func_builder_replicate, group, constant.result_name));
+            index.push_back(&actions.addFunction(func_builder_replicate, group, constant.result_name));
 
             continue;
         }
@@ -75,17 +75,17 @@ ActionsDAGPtr addMissingDefaults(
         *  it can be full (or the interpreter may decide that it is constant everywhere).
         */
         auto new_column = column.type->createColumnConstWithDefaultValue(0);
-        const auto * col = &actions->addColumn({new_column, column.type, column.name});
-        index.push_back(&actions->materializeNode(*col));
+        const auto * col = &actions.addColumn({new_column, column.type, column.name});
+        index.push_back(&actions.materializeNode(*col));
     }
 
     /// Computes explicitly specified values by default and materialized columns.
-    if (auto dag = evaluateMissingDefaults(actions->getResultColumns(), required_columns, columns, context, true, null_as_default))
-        actions = ActionsDAG::merge(std::move(*actions), std::move(*dag));
+    if (auto dag = evaluateMissingDefaults(actions.getResultColumns(), required_columns, columns, context, true, null_as_default))
+        actions = ActionsDAG::merge(std::move(actions), std::move(*dag));
 
     /// Removes unused columns and reorders result.
-    actions->removeUnusedActions(required_columns.getNames(), false);
-    actions->addMaterializingOutputActions();
+    actions.removeUnusedActions(required_columns.getNames(), false);
+    actions.addMaterializingOutputActions();
 
     return actions;
 }
diff --git a/src/Interpreters/addMissingDefaults.h b/src/Interpreters/addMissingDefaults.h
index 94afd806dfd..5299bae9745 100644
--- a/src/Interpreters/addMissingDefaults.h
+++ b/src/Interpreters/addMissingDefaults.h
@@ -24,7 +24,7 @@ using ActionsDAGPtr = std::unique_ptr<ActionsDAG>;
   * Also can substitute NULL with DEFAULT value in case of INSERT SELECT query (null_as_default) if according setting is 1.
   * All three types of columns are materialized (not constants).
   */
-ActionsDAGPtr addMissingDefaults(
+ActionsDAG addMissingDefaults(
     const Block & header, const NamesAndTypesList & required_columns,
     const ColumnsDescription & columns, ContextPtr context, bool null_as_default = false);
 }
diff --git a/src/Interpreters/evaluateConstantExpression.cpp b/src/Interpreters/evaluateConstantExpression.cpp
index 4e1a2bcf5ee..d5d9fce0dbd 100644
--- a/src/Interpreters/evaluateConstantExpression.cpp
+++ b/src/Interpreters/evaluateConstantExpression.cpp
@@ -89,7 +89,7 @@ std::optional<EvaluateConstantExpressionResult> evaluateConstantExpressionImpl(c
     ColumnPtr result_column;
     DataTypePtr result_type;
     String result_name = ast->getColumnName();
-    for (const auto & action_node : actions->getOutputs())
+    for (const auto & action_node : actions.getOutputs())
     {
         if ((action_node->result_name == result_name) && action_node->column)
         {
diff --git a/src/Interpreters/inplaceBlockConversions.cpp b/src/Interpreters/inplaceBlockConversions.cpp
index b000264ae33..62f8aea86d1 100644
--- a/src/Interpreters/inplaceBlockConversions.cpp
+++ b/src/Interpreters/inplaceBlockConversions.cpp
@@ -152,22 +152,20 @@ ASTPtr convertRequiredExpressions(Block & block, const NamesAndTypesList & requi
     return conversion_expr_list;
 }
 
-ActionsDAGPtr createExpressions(
+std::optional<ActionsDAG> createExpressions(
     const Block & header,
     ASTPtr expr_list,
     bool save_unneeded_columns,
     ContextPtr context)
 {
     if (!expr_list)
-        return nullptr;
+        return {};
 
     auto syntax_result = TreeRewriter(context).analyze(expr_list, header.getNamesAndTypesList());
     auto expression_analyzer = ExpressionAnalyzer{expr_list, syntax_result, context};
-    auto dag = std::make_unique<ActionsDAG>(header.getNamesAndTypesList());
+    ActionsDAG dag(header.getNamesAndTypesList());
     auto actions = expression_analyzer.getActionsDAG(true, !save_unneeded_columns);
-    dag = ActionsDAG::merge(std::move(*dag), std::move(*actions));
-
-    return dag;
+    return ActionsDAG::merge(std::move(dag), std::move(actions));
 }
 
 }
@@ -180,7 +178,7 @@ void performRequiredConversions(Block & block, const NamesAndTypesList & require
 
     if (auto dag = createExpressions(block, conversion_expr_list, true, context))
     {
-        auto expression = std::make_shared<ExpressionActions>(std::move(dag), ExpressionActionsSettings::fromContext(context));
+        auto expression = std::make_shared<ExpressionActions>(std::move(*dag), ExpressionActionsSettings::fromContext(context));
         expression->execute(block);
     }
 }
@@ -195,7 +193,7 @@ bool needConvertAnyNullToDefault(const Block & header, const NamesAndTypesList &
     return false;
 }
 
-ActionsDAGPtr evaluateMissingDefaults(
+std::optional<ActionsDAG> evaluateMissingDefaults(
     const Block & header,
     const NamesAndTypesList & required_columns,
     const ColumnsDescription & columns,
@@ -204,7 +202,7 @@ ActionsDAGPtr evaluateMissingDefaults(
     bool null_as_default)
 {
     if (!columns.hasDefaults() && (!null_as_default || !needConvertAnyNullToDefault(header, required_columns, columns)))
-        return nullptr;
+        return {};
 
     ASTPtr expr_list = defaultRequiredExpressions(header, required_columns, columns, null_as_default);
     return createExpressions(header, expr_list, save_unneeded_columns, context);
diff --git a/src/Interpreters/inplaceBlockConversions.h b/src/Interpreters/inplaceBlockConversions.h
index ffc77561e79..570eb75dd4a 100644
--- a/src/Interpreters/inplaceBlockConversions.h
+++ b/src/Interpreters/inplaceBlockConversions.h
@@ -5,9 +5,6 @@
 #include <Common/COW.h>
 
 #include <memory>
-#include <string>
-#include <unordered_map>
-
 
 namespace DB
 {
@@ -24,12 +21,11 @@ struct StorageInMemoryMetadata;
 using StorageMetadataPtr = std::shared_ptr<const StorageInMemoryMetadata>;
 
 class ActionsDAG;
-using ActionsDAGPtr = std::unique_ptr<ActionsDAG>;
 
 /// Create actions which adds missing defaults to block according to required_columns using columns description
 /// or substitute NULL into DEFAULT value in case of INSERT SELECT query (null_as_default) if according setting is 1.
 /// Return nullptr if no actions required.
-ActionsDAGPtr evaluateMissingDefaults(
+std::optional<ActionsDAG> evaluateMissingDefaults(
     const Block & header,
     const NamesAndTypesList & required_columns,
     const ColumnsDescription & columns,
diff --git a/src/Planner/Planner.cpp b/src/Planner/Planner.cpp
index 16ee6de73c4..48e42099ce8 100644
--- a/src/Planner/Planner.cpp
+++ b/src/Planner/Planner.cpp
@@ -332,12 +332,12 @@ void addExpressionStep(QueryPlan & query_plan,
     const std::string & step_description,
     UsefulSets & useful_sets)
 {
-    auto actions = ActionsDAG::clone(&expression_actions->dag);
+    auto actions = std::move(*ActionsDAG::clone(&expression_actions->dag));
     if (expression_actions->project_input)
-        actions->appendInputsForUnusedColumns(query_plan.getCurrentDataStream().header);
+        actions.appendInputsForUnusedColumns(query_plan.getCurrentDataStream().header);
 
-    auto expression_step = std::make_unique<ExpressionStep>(query_plan.getCurrentDataStream(), actions);
-    appendSetsFromActionsDAG(*expression_step->getExpression(), useful_sets);
+    auto expression_step = std::make_unique<ExpressionStep>(query_plan.getCurrentDataStream(), std::move(actions));
+    appendSetsFromActionsDAG(expression_step->getExpression(), useful_sets);
     expression_step->setStepDescription(step_description);
     query_plan.addStep(std::move(expression_step));
 }
@@ -347,15 +347,15 @@ void addFilterStep(QueryPlan & query_plan,
     const std::string & step_description,
     UsefulSets & useful_sets)
 {
-    auto actions = ActionsDAG::clone(&filter_analysis_result.filter_actions->dag);
+    auto actions = std::move(*ActionsDAG::clone(&filter_analysis_result.filter_actions->dag));
     if (filter_analysis_result.filter_actions->project_input)
-        actions->appendInputsForUnusedColumns(query_plan.getCurrentDataStream().header);
+        actions.appendInputsForUnusedColumns(query_plan.getCurrentDataStream().header);
 
     auto where_step = std::make_unique<FilterStep>(query_plan.getCurrentDataStream(),
-        actions,
+        std::move(actions),
         filter_analysis_result.filter_column_name,
         filter_analysis_result.remove_filter_column);
-    appendSetsFromActionsDAG(*where_step->getExpression(), useful_sets);
+    appendSetsFromActionsDAG(where_step->getExpression(), useful_sets);
     where_step->setStepDescription(step_description);
     query_plan.addStep(std::move(where_step));
 }
@@ -552,10 +552,10 @@ void addTotalsHavingStep(QueryPlan & query_plan,
     const auto & having_analysis_result = expression_analysis_result.getHaving();
     bool need_finalize = !query_node.isGroupByWithRollup() && !query_node.isGroupByWithCube();
 
-    ActionsDAGPtr actions;
+    std::optional<ActionsDAG> actions;
     if (having_analysis_result.filter_actions)
     {
-        actions = ActionsDAG::clone(&having_analysis_result.filter_actions->dag);
+        actions = std::move(*ActionsDAG::clone(&having_analysis_result.filter_actions->dag));
         if (having_analysis_result.filter_actions->project_input)
             actions->appendInputsForUnusedColumns(query_plan.getCurrentDataStream().header);
     }
@@ -564,7 +564,7 @@ void addTotalsHavingStep(QueryPlan & query_plan,
         query_plan.getCurrentDataStream(),
         aggregation_analysis_result.aggregate_descriptions,
         query_analysis_result.aggregate_overflow_row,
-        actions,
+        std::move(actions),
         having_analysis_result.filter_column_name,
         having_analysis_result.remove_filter_column,
         settings.totals_mode,
@@ -715,13 +715,13 @@ void addWithFillStepIfNeeded(QueryPlan & query_plan,
 
     if (query_node.hasInterpolate())
     {
-        auto interpolate_actions_dag = std::make_unique<ActionsDAG>();
+        ActionsDAG interpolate_actions_dag;
         auto query_plan_columns = query_plan.getCurrentDataStream().header.getColumnsWithTypeAndName();
         for (auto & query_plan_column : query_plan_columns)
         {
             /// INTERPOLATE actions dag input columns must be non constant
             query_plan_column.column = nullptr;
-            interpolate_actions_dag->addInput(query_plan_column);
+            interpolate_actions_dag.addInput(query_plan_column);
         }
 
         auto & interpolate_list_node = query_node.getInterpolate()->as<ListNode &>();
@@ -729,12 +729,12 @@ void addWithFillStepIfNeeded(QueryPlan & query_plan,
 
         if (interpolate_list_nodes.empty())
         {
-            for (const auto * input_node : interpolate_actions_dag->getInputs())
+            for (const auto * input_node : interpolate_actions_dag.getInputs())
             {
                 if (column_names_with_fill.contains(input_node->result_name))
                     continue;
 
-                interpolate_actions_dag->getOutputs().push_back(input_node);
+                interpolate_actions_dag.getOutputs().push_back(input_node);
             }
         }
         else
@@ -744,12 +744,12 @@ void addWithFillStepIfNeeded(QueryPlan & query_plan,
                 auto & interpolate_node_typed = interpolate_node->as<InterpolateNode &>();
 
                 PlannerActionsVisitor planner_actions_visitor(planner_context);
-                auto expression_to_interpolate_expression_nodes = planner_actions_visitor.visit(*interpolate_actions_dag,
+                auto expression_to_interpolate_expression_nodes = planner_actions_visitor.visit(interpolate_actions_dag,
                     interpolate_node_typed.getExpression());
                 if (expression_to_interpolate_expression_nodes.size() != 1)
                     throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expression to interpolate expected to have single action node");
 
-                auto interpolate_expression_nodes = planner_actions_visitor.visit(*interpolate_actions_dag,
+                auto interpolate_expression_nodes = planner_actions_visitor.visit(interpolate_actions_dag,
                     interpolate_node_typed.getInterpolateExpression());
                 if (interpolate_expression_nodes.size() != 1)
                     throw Exception(ErrorCodes::BAD_ARGUMENTS, "Interpolate expression expected to have single action node");
@@ -760,16 +760,16 @@ void addWithFillStepIfNeeded(QueryPlan & query_plan,
                 const auto * interpolate_expression = interpolate_expression_nodes[0];
                 if (!interpolate_expression->result_type->equals(*expression_to_interpolate->result_type))
                 {
-                    interpolate_expression = &interpolate_actions_dag->addCast(*interpolate_expression,
+                    interpolate_expression = &interpolate_actions_dag.addCast(*interpolate_expression,
                         expression_to_interpolate->result_type,
                         interpolate_expression->result_name);
                 }
 
-                const auto * alias_node = &interpolate_actions_dag->addAlias(*interpolate_expression, expression_to_interpolate_name);
-                interpolate_actions_dag->getOutputs().push_back(alias_node);
+                const auto * alias_node = &interpolate_actions_dag.addAlias(*interpolate_expression, expression_to_interpolate_name);
+                interpolate_actions_dag.getOutputs().push_back(alias_node);
             }
 
-            interpolate_actions_dag->removeUnusedActions();
+            interpolate_actions_dag.removeUnusedActions();
         }
 
         Aliases empty_aliases;
@@ -1130,7 +1130,7 @@ void addAdditionalFilterStepIfNeeded(QueryPlan & query_plan,
         return;
 
     auto filter_step = std::make_unique<FilterStep>(query_plan.getCurrentDataStream(),
-        filter_info.actions,
+        std::move(*filter_info.actions),
         filter_info.column_name,
         filter_info.do_remove_column);
     filter_step->setStepDescription("additional result filter");
@@ -1418,7 +1418,7 @@ void Planner::buildPlanForQueryNode()
             if (it != table_filters.end())
             {
                 const auto & filters = it->second;
-                table_expression_data.setFilterActions(ActionsDAG::clone(filters.filter_actions));
+                table_expression_data.setFilterActions(ActionsDAG::clone(&*filters.filter_actions));
                 table_expression_data.setPrewhereInfo(filters.prewhere_info);
             }
         }
diff --git a/src/Planner/PlannerActionsVisitor.cpp b/src/Planner/PlannerActionsVisitor.cpp
index 59ec7778e21..4c0c9bc7937 100644
--- a/src/Planner/PlannerActionsVisitor.cpp
+++ b/src/Planner/PlannerActionsVisitor.cpp
@@ -757,12 +757,12 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::vi
         lambda_arguments_names_and_types.emplace_back(lambda_argument_name, std::move(lambda_argument_type));
     }
 
-    auto lambda_actions_dag = std::make_unique<ActionsDAG>();
-    actions_stack.emplace_back(*lambda_actions_dag, node);
+    ActionsDAG lambda_actions_dag;
+    actions_stack.emplace_back(lambda_actions_dag, node);
 
     auto [lambda_expression_node_name, levels] = visitImpl(lambda_node.getExpression());
-    lambda_actions_dag->getOutputs().push_back(actions_stack.back().getNodeOrThrow(lambda_expression_node_name));
-    lambda_actions_dag->removeUnusedActions(Names(1, lambda_expression_node_name));
+    lambda_actions_dag.getOutputs().push_back(actions_stack.back().getNodeOrThrow(lambda_expression_node_name));
+    lambda_actions_dag.removeUnusedActions(Names(1, lambda_expression_node_name));
 
     auto expression_actions_settings = ExpressionActionsSettings::fromContext(planner_context->getQueryContext(), CompileExpressions::yes);
     auto lambda_actions = std::make_shared<ExpressionActions>(std::move(lambda_actions_dag), expression_actions_settings);
@@ -879,14 +879,14 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::vi
     const auto & function_node = node->as<FunctionNode &>();
     auto function_node_name = action_node_name_helper.calculateActionNodeName(node);
 
-    auto index_hint_actions_dag = std::make_unique<ActionsDAG>();
-    auto & index_hint_actions_dag_outputs = index_hint_actions_dag->getOutputs();
+    ActionsDAG index_hint_actions_dag;
+    auto & index_hint_actions_dag_outputs = index_hint_actions_dag.getOutputs();
     std::unordered_set<std::string_view> index_hint_actions_dag_output_node_names;
     PlannerActionsVisitor actions_visitor(planner_context);
 
     for (const auto & argument : function_node.getArguments())
     {
-        auto index_hint_argument_expression_dag_nodes = actions_visitor.visit(*index_hint_actions_dag, argument);
+        auto index_hint_argument_expression_dag_nodes = actions_visitor.visit(index_hint_actions_dag, argument);
 
         for (auto & expression_dag_node : index_hint_argument_expression_dag_nodes)
         {
diff --git a/src/Planner/PlannerContext.h b/src/Planner/PlannerContext.h
index 418240fa34e..f35772ef7c0 100644
--- a/src/Planner/PlannerContext.h
+++ b/src/Planner/PlannerContext.h
@@ -25,7 +25,7 @@ class TableNode;
 
 struct FiltersForTableExpression
 {
-    ActionsDAGPtr filter_actions;
+    std::optional<ActionsDAG> filter_actions;
     PrewhereInfoPtr prewhere_info;
 };
 
diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp
index 94054588d40..fa3a3483a8e 100644
--- a/src/Planner/PlannerJoinTree.cpp
+++ b/src/Planner/PlannerJoinTree.cpp
@@ -591,19 +591,19 @@ UInt64 mainQueryNodeBlockSizeByLimit(const SelectQueryInfo & select_query_info)
 std::unique_ptr<ExpressionStep> createComputeAliasColumnsStep(
     const std::unordered_map<std::string, ActionsDAGPtr> & alias_column_expressions, const DataStream & current_data_stream)
 {
-    ActionsDAGPtr merged_alias_columns_actions_dag = std::make_unique<ActionsDAG>(current_data_stream.header.getColumnsWithTypeAndName());
-    ActionsDAG::NodeRawConstPtrs action_dag_outputs = merged_alias_columns_actions_dag->getInputs();
+    ActionsDAG merged_alias_columns_actions_dag(current_data_stream.header.getColumnsWithTypeAndName());
+    ActionsDAG::NodeRawConstPtrs action_dag_outputs = merged_alias_columns_actions_dag.getInputs();
 
     for (const auto & [column_name, alias_column_actions_dag] : alias_column_expressions)
     {
         const auto & current_outputs = alias_column_actions_dag->getOutputs();
         action_dag_outputs.insert(action_dag_outputs.end(), current_outputs.begin(), current_outputs.end());
-        merged_alias_columns_actions_dag->mergeNodes(std::move(*alias_column_actions_dag));
+        merged_alias_columns_actions_dag.mergeNodes(std::move(*alias_column_actions_dag));
     }
 
     for (const auto * output_node : action_dag_outputs)
-        merged_alias_columns_actions_dag->addOrReplaceInOutputs(*output_node);
-    merged_alias_columns_actions_dag->removeUnusedActions(false);
+        merged_alias_columns_actions_dag.addOrReplaceInOutputs(*output_node);
+    merged_alias_columns_actions_dag.removeUnusedActions(false);
 
     auto alias_column_step = std::make_unique<ExpressionStep>(current_data_stream, std::move(merged_alias_columns_actions_dag));
     alias_column_step->setStepDescription("Compute alias columns");
@@ -776,7 +776,7 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres
                 if (prewhere_actions)
                 {
                     prewhere_info = std::make_shared<PrewhereInfo>();
-                    prewhere_info->prewhere_actions = ActionsDAG::clone(prewhere_actions);
+                    prewhere_info->prewhere_actions = std::move(*ActionsDAG::clone(prewhere_actions));
                     prewhere_info->prewhere_column_name = prewhere_actions->getOutputs().at(0)->result_name;
                     prewhere_info->remove_prewhere_column = true;
                     prewhere_info->need_filter = true;
@@ -805,14 +805,14 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres
 
                         if (!prewhere_info->prewhere_actions)
                         {
-                            prewhere_info->prewhere_actions = std::move(filter_info.actions);
+                            prewhere_info->prewhere_actions = std::move(*filter_info.actions);
                             prewhere_info->prewhere_column_name = filter_info.column_name;
                             prewhere_info->remove_prewhere_column = filter_info.do_remove_column;
                             prewhere_info->need_filter = true;
                         }
                         else if (!prewhere_info->row_level_filter)
                         {
-                            prewhere_info->row_level_filter = std::move(filter_info.actions);
+                            prewhere_info->row_level_filter = std::move(*filter_info.actions);
                             prewhere_info->row_level_column_name = filter_info.column_name;
                             prewhere_info->need_filter = true;
                         }
@@ -831,7 +831,7 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres
                 auto row_policy_filter_info
                     = buildRowPolicyFilterIfNeeded(storage, table_expression_query_info, planner_context, used_row_policies);
                 if (row_policy_filter_info.actions)
-                    table_expression_data.setRowLevelFilterActions(ActionsDAG::clone(row_policy_filter_info.actions));
+                    table_expression_data.setRowLevelFilterActions(ActionsDAG::clone(&*row_policy_filter_info.actions));
                 add_filter(row_policy_filter_info, "Row-level security filter");
 
                 if (query_context->getParallelReplicasMode() == Context::ParallelReplicasMode::CUSTOM_KEY)
@@ -964,15 +964,14 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres
                     query_plan.addStep(std::move(alias_column_step));
                 }
 
-                for (const auto & filter_info_and_description : where_filters)
+                for (auto && [filter_info, description] : where_filters)
                 {
-                    const auto & [filter_info, description] = filter_info_and_description;
                     if (query_plan.isInitialized() &&
                         from_stage == QueryProcessingStage::FetchColumns &&
                         filter_info.actions)
                     {
                         auto filter_step = std::make_unique<FilterStep>(query_plan.getCurrentDataStream(),
-                            filter_info.actions,
+                            std::move(*filter_info.actions),
                             filter_info.column_name,
                             filter_info.do_remove_column);
                         filter_step->setStepDescription(description);
@@ -1063,19 +1062,19 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres
 
     if (from_stage == QueryProcessingStage::FetchColumns)
     {
-        auto rename_actions_dag = std::make_unique<ActionsDAG>(query_plan.getCurrentDataStream().header.getColumnsWithTypeAndName());
+        ActionsDAG rename_actions_dag(query_plan.getCurrentDataStream().header.getColumnsWithTypeAndName());
         ActionsDAG::NodeRawConstPtrs updated_actions_dag_outputs;
 
-        for (auto & output_node : rename_actions_dag->getOutputs())
+        for (auto & output_node : rename_actions_dag.getOutputs())
         {
             const auto * column_identifier = table_expression_data.getColumnIdentifierOrNull(output_node->result_name);
             if (!column_identifier)
                 continue;
 
-            updated_actions_dag_outputs.push_back(&rename_actions_dag->addAlias(*output_node, *column_identifier));
+            updated_actions_dag_outputs.push_back(&rename_actions_dag.addAlias(*output_node, *column_identifier));
         }
 
-        rename_actions_dag->getOutputs() = std::move(updated_actions_dag_outputs);
+        rename_actions_dag.getOutputs() = std::move(updated_actions_dag_outputs);
 
         auto rename_step = std::make_unique<ExpressionStep>(query_plan.getCurrentDataStream(), std::move(rename_actions_dag));
         rename_step->setStepDescription("Change column names to column identifiers");
@@ -1117,9 +1116,9 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres
 
 void joinCastPlanColumnsToNullable(QueryPlan & plan_to_add_cast, PlannerContextPtr & planner_context, const FunctionOverloadResolverPtr & to_nullable_function)
 {
-    auto cast_actions_dag = std::make_unique<ActionsDAG>(plan_to_add_cast.getCurrentDataStream().header.getColumnsWithTypeAndName());
+    ActionsDAG cast_actions_dag(plan_to_add_cast.getCurrentDataStream().header.getColumnsWithTypeAndName());
 
-    for (auto & output_node : cast_actions_dag->getOutputs())
+    for (auto & output_node : cast_actions_dag.getOutputs())
     {
         if (planner_context->getGlobalPlannerContext()->hasColumnIdentifier(output_node->result_name))
         {
@@ -1128,11 +1127,11 @@ void joinCastPlanColumnsToNullable(QueryPlan & plan_to_add_cast, PlannerContextP
                 type_to_check = type_to_check_low_cardinality->getDictionaryType();
 
             if (type_to_check->canBeInsideNullable())
-                output_node = &cast_actions_dag->addFunction(to_nullable_function, {output_node}, output_node->result_name);
+                output_node = &cast_actions_dag.addFunction(to_nullable_function, {output_node}, output_node->result_name);
         }
     }
 
-    cast_actions_dag->appendInputsForUnusedColumns(plan_to_add_cast.getCurrentDataStream().header);
+    cast_actions_dag.appendInputsForUnusedColumns(plan_to_add_cast.getCurrentDataStream().header);
     auto cast_join_columns_step = std::make_unique<ExpressionStep>(plan_to_add_cast.getCurrentDataStream(), std::move(cast_actions_dag));
     cast_join_columns_step->setStepDescription("Cast JOIN columns to Nullable");
     plan_to_add_cast.addStep(std::move(cast_join_columns_step));
@@ -1178,16 +1177,16 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_
             join_table_expression,
             planner_context);
 
-        join_clauses_and_actions.left_join_expressions_actions->appendInputsForUnusedColumns(left_plan.getCurrentDataStream().header);
-        auto left_join_expressions_actions_step = std::make_unique<ExpressionStep>(left_plan.getCurrentDataStream(), join_clauses_and_actions.left_join_expressions_actions);
+        join_clauses_and_actions.left_join_expressions_actions.appendInputsForUnusedColumns(left_plan.getCurrentDataStream().header);
+        auto left_join_expressions_actions_step = std::make_unique<ExpressionStep>(left_plan.getCurrentDataStream(), std::move(join_clauses_and_actions.left_join_expressions_actions));
         left_join_expressions_actions_step->setStepDescription("JOIN actions");
-        appendSetsFromActionsDAG(*left_join_expressions_actions_step->getExpression(), left_join_tree_query_plan.useful_sets);
+        appendSetsFromActionsDAG(left_join_expressions_actions_step->getExpression(), left_join_tree_query_plan.useful_sets);
         left_plan.addStep(std::move(left_join_expressions_actions_step));
 
-        join_clauses_and_actions.right_join_expressions_actions->appendInputsForUnusedColumns(right_plan.getCurrentDataStream().header);
-        auto right_join_expressions_actions_step = std::make_unique<ExpressionStep>(right_plan.getCurrentDataStream(), join_clauses_and_actions.right_join_expressions_actions);
+        join_clauses_and_actions.right_join_expressions_actions.appendInputsForUnusedColumns(right_plan.getCurrentDataStream().header);
+        auto right_join_expressions_actions_step = std::make_unique<ExpressionStep>(right_plan.getCurrentDataStream(), std::move(join_clauses_and_actions.right_join_expressions_actions));
         right_join_expressions_actions_step->setStepDescription("JOIN actions");
-        appendSetsFromActionsDAG(*right_join_expressions_actions_step->getExpression(), right_join_tree_query_plan.useful_sets);
+        appendSetsFromActionsDAG(right_join_expressions_actions_step->getExpression(), right_join_tree_query_plan.useful_sets);
         right_plan.addStep(std::move(right_join_expressions_actions_step));
     }
 
@@ -1225,19 +1224,19 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_
 
     auto join_cast_plan_output_nodes = [&](QueryPlan & plan_to_add_cast, std::unordered_map<std::string, DataTypePtr> & plan_column_name_to_cast_type)
     {
-        auto cast_actions_dag = std::make_unique<ActionsDAG>(plan_to_add_cast.getCurrentDataStream().header.getColumnsWithTypeAndName());
+        ActionsDAG cast_actions_dag(plan_to_add_cast.getCurrentDataStream().header.getColumnsWithTypeAndName());
 
-        for (auto & output_node : cast_actions_dag->getOutputs())
+        for (auto & output_node : cast_actions_dag.getOutputs())
         {
             auto it = plan_column_name_to_cast_type.find(output_node->result_name);
             if (it == plan_column_name_to_cast_type.end())
                 continue;
 
             const auto & cast_type = it->second;
-            output_node = &cast_actions_dag->addCast(*output_node, cast_type, output_node->result_name);
+            output_node = &cast_actions_dag.addCast(*output_node, cast_type, output_node->result_name);
         }
 
-        cast_actions_dag->appendInputsForUnusedColumns(plan_to_add_cast.getCurrentDataStream().header);
+        cast_actions_dag.appendInputsForUnusedColumns(plan_to_add_cast.getCurrentDataStream().header);
         auto cast_join_columns_step
             = std::make_unique<ExpressionStep>(plan_to_add_cast.getCurrentDataStream(), std::move(cast_actions_dag));
         cast_join_columns_step->setStepDescription("Cast JOIN USING columns");
@@ -1385,7 +1384,7 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_
         {
             ExpressionActionsPtr & mixed_join_expression = table_join->getMixedJoinExpression();
             mixed_join_expression = std::make_shared<ExpressionActions>(
-                std::move(join_clauses_and_actions.mixed_join_expressions_actions),
+                std::move(*join_clauses_and_actions.mixed_join_expressions_actions),
                 ExpressionActionsSettings::fromContext(planner_context->getQueryContext()));
 
             appendSetsFromActionsDAG(mixed_join_expression->getActionsDAG(), left_join_tree_query_plan.useful_sets);
@@ -1542,12 +1541,12 @@ JoinTreeQueryPlan buildQueryPlanForJoinNode(const QueryTreeNodePtr & join_table_
         result_plan.unitePlans(std::move(join_step), {std::move(plans)});
     }
 
-    auto drop_unused_columns_after_join_actions_dag = std::make_unique<ActionsDAG>(result_plan.getCurrentDataStream().header.getColumnsWithTypeAndName());
+    ActionsDAG drop_unused_columns_after_join_actions_dag(result_plan.getCurrentDataStream().header.getColumnsWithTypeAndName());
     ActionsDAG::NodeRawConstPtrs drop_unused_columns_after_join_actions_dag_updated_outputs;
     std::unordered_set<std::string_view> drop_unused_columns_after_join_actions_dag_updated_outputs_names;
     std::optional<size_t> first_skipped_column_node_index;
 
-    auto & drop_unused_columns_after_join_actions_dag_outputs = drop_unused_columns_after_join_actions_dag->getOutputs();
+    auto & drop_unused_columns_after_join_actions_dag_outputs = drop_unused_columns_after_join_actions_dag.getOutputs();
     size_t drop_unused_columns_after_join_actions_dag_outputs_size = drop_unused_columns_after_join_actions_dag_outputs.size();
 
     for (size_t i = 0; i < drop_unused_columns_after_join_actions_dag_outputs_size; ++i)
@@ -1619,7 +1618,7 @@ JoinTreeQueryPlan buildQueryPlanForArrayJoinNode(const QueryTreeNodePtr & array_
     auto plan = std::move(join_tree_query_plan.query_plan);
     auto plan_output_columns = plan.getCurrentDataStream().header.getColumnsWithTypeAndName();
 
-    ActionsDAGPtr array_join_action_dag = std::make_unique<ActionsDAG>(plan_output_columns);
+    ActionsDAG array_join_action_dag(plan_output_columns);
     PlannerActionsVisitor actions_visitor(planner_context);
     std::unordered_set<std::string> array_join_expressions_output_nodes;
 
@@ -1630,28 +1629,28 @@ JoinTreeQueryPlan buildQueryPlanForArrayJoinNode(const QueryTreeNodePtr & array_
         array_join_column_names.insert(array_join_column_identifier);
 
         auto & array_join_expression_column = array_join_expression->as<ColumnNode &>();
-        auto expression_dag_index_nodes = actions_visitor.visit(*array_join_action_dag, array_join_expression_column.getExpressionOrThrow());
+        auto expression_dag_index_nodes = actions_visitor.visit(array_join_action_dag, array_join_expression_column.getExpressionOrThrow());
 
         for (auto & expression_dag_index_node : expression_dag_index_nodes)
         {
-            const auto * array_join_column_node = &array_join_action_dag->addAlias(*expression_dag_index_node, array_join_column_identifier);
-            array_join_action_dag->getOutputs().push_back(array_join_column_node);
+            const auto * array_join_column_node = &array_join_action_dag.addAlias(*expression_dag_index_node, array_join_column_identifier);
+            array_join_action_dag.getOutputs().push_back(array_join_column_node);
             array_join_expressions_output_nodes.insert(array_join_column_node->result_name);
         }
     }
 
-    array_join_action_dag->appendInputsForUnusedColumns(plan.getCurrentDataStream().header);
+    array_join_action_dag.appendInputsForUnusedColumns(plan.getCurrentDataStream().header);
 
     auto array_join_actions = std::make_unique<ExpressionStep>(plan.getCurrentDataStream(), std::move(array_join_action_dag));
     array_join_actions->setStepDescription("ARRAY JOIN actions");
-    appendSetsFromActionsDAG(*array_join_actions->getExpression(), join_tree_query_plan.useful_sets);
+    appendSetsFromActionsDAG(array_join_actions->getExpression(), join_tree_query_plan.useful_sets);
     plan.addStep(std::move(array_join_actions));
 
-    auto drop_unused_columns_before_array_join_actions_dag = std::make_unique<ActionsDAG>(plan.getCurrentDataStream().header.getColumnsWithTypeAndName());
+    ActionsDAG drop_unused_columns_before_array_join_actions_dag(plan.getCurrentDataStream().header.getColumnsWithTypeAndName());
     ActionsDAG::NodeRawConstPtrs drop_unused_columns_before_array_join_actions_dag_updated_outputs;
     std::unordered_set<std::string_view> drop_unused_columns_before_array_join_actions_dag_updated_outputs_names;
 
-    auto & drop_unused_columns_before_array_join_actions_dag_outputs = drop_unused_columns_before_array_join_actions_dag->getOutputs();
+    auto & drop_unused_columns_before_array_join_actions_dag_outputs = drop_unused_columns_before_array_join_actions_dag.getOutputs();
     size_t drop_unused_columns_before_array_join_actions_dag_outputs_size = drop_unused_columns_before_array_join_actions_dag_outputs.size();
 
     for (size_t i = 0; i < drop_unused_columns_before_array_join_actions_dag_outputs_size; ++i)
diff --git a/src/Planner/PlannerJoins.cpp b/src/Planner/PlannerJoins.cpp
index 23b6a805ab9..db9678d91a6 100644
--- a/src/Planner/PlannerJoins.cpp
+++ b/src/Planner/PlannerJoins.cpp
@@ -177,13 +177,13 @@ std::set<JoinTableSide> extractJoinTableSidesFromExpression(//const ActionsDAG::
 }
 
 const ActionsDAG::Node * appendExpression(
-    ActionsDAGPtr & dag,
+    ActionsDAG & dag,
     const QueryTreeNodePtr & expression,
     const PlannerContextPtr & planner_context,
     const JoinNode & join_node)
 {
     PlannerActionsVisitor join_expression_visitor(planner_context);
-    auto join_expression_dag_node_raw_pointers = join_expression_visitor.visit(*dag, expression);
+    auto join_expression_dag_node_raw_pointers = join_expression_visitor.visit(dag, expression);
     if (join_expression_dag_node_raw_pointers.size() != 1)
         throw Exception(ErrorCodes::LOGICAL_ERROR,
             "JOIN {} ON clause contains multiple expressions",
@@ -193,9 +193,9 @@ const ActionsDAG::Node * appendExpression(
 }
 
 void buildJoinClause(
-    ActionsDAGPtr & left_dag,
-    ActionsDAGPtr & right_dag,
-    ActionsDAGPtr & mixed_dag,
+    ActionsDAG & left_dag,
+    ActionsDAG & right_dag,
+    ActionsDAG & mixed_dag,
     const PlannerContextPtr & planner_context,
     const QueryTreeNodePtr & join_expression,
     const TableExpressionSet & left_table_expressions,
@@ -376,8 +376,8 @@ JoinClausesAndActions buildJoinClausesAndActions(
     const JoinNode & join_node,
     const PlannerContextPtr & planner_context)
 {
-    ActionsDAGPtr left_join_actions = std::make_unique<ActionsDAG>(left_table_expression_columns);
-    ActionsDAGPtr right_join_actions = std::make_unique<ActionsDAG>(right_table_expression_columns);
+    ActionsDAG left_join_actions(left_table_expression_columns);
+    ActionsDAG right_join_actions(right_table_expression_columns);
     ColumnsWithTypeAndName mixed_table_expression_columns;
     for (const auto & left_column : left_table_expression_columns)
     {
@@ -387,7 +387,7 @@ JoinClausesAndActions buildJoinClausesAndActions(
     {
         mixed_table_expression_columns.push_back(right_column);
     }
-    ActionsDAGPtr mixed_join_actions = std::make_unique<ActionsDAG>(mixed_table_expression_columns);
+    ActionsDAG mixed_join_actions(mixed_table_expression_columns);
 
     /** It is possible to have constant value in JOIN ON section, that we need to ignore during DAG construction.
       * If we do not ignore it, this function will be replaced by underlying constant.
@@ -498,12 +498,12 @@ JoinClausesAndActions buildJoinClausesAndActions(
         {
             const ActionsDAG::Node * dag_filter_condition_node = nullptr;
             if (left_filter_condition_nodes.size() > 1)
-                dag_filter_condition_node = &left_join_actions->addFunction(and_function, left_filter_condition_nodes, {});
+                dag_filter_condition_node = &left_join_actions.addFunction(and_function, left_filter_condition_nodes, {});
             else
                 dag_filter_condition_node = left_filter_condition_nodes[0];
 
             join_clause.getLeftFilterConditionNodes() = {dag_filter_condition_node};
-            left_join_actions->addOrReplaceInOutputs(*dag_filter_condition_node);
+            left_join_actions.addOrReplaceInOutputs(*dag_filter_condition_node);
 
             add_necessary_name_if_needed(JoinTableSide::Left, dag_filter_condition_node->result_name);
         }
@@ -514,12 +514,12 @@ JoinClausesAndActions buildJoinClausesAndActions(
             const ActionsDAG::Node * dag_filter_condition_node = nullptr;
 
             if (right_filter_condition_nodes.size() > 1)
-                dag_filter_condition_node = &right_join_actions->addFunction(and_function, right_filter_condition_nodes, {});
+                dag_filter_condition_node = &right_join_actions.addFunction(and_function, right_filter_condition_nodes, {});
             else
                 dag_filter_condition_node = right_filter_condition_nodes[0];
 
             join_clause.getRightFilterConditionNodes() = {dag_filter_condition_node};
-            right_join_actions->addOrReplaceInOutputs(*dag_filter_condition_node);
+            right_join_actions.addOrReplaceInOutputs(*dag_filter_condition_node);
 
             add_necessary_name_if_needed(JoinTableSide::Right, dag_filter_condition_node->result_name);
         }
@@ -556,10 +556,10 @@ JoinClausesAndActions buildJoinClausesAndActions(
                 }
 
                 if (!left_key_node->result_type->equals(*common_type))
-                    left_key_node = &left_join_actions->addCast(*left_key_node, common_type, {});
+                    left_key_node = &left_join_actions.addCast(*left_key_node, common_type, {});
 
                 if (!right_key_node->result_type->equals(*common_type))
-                    right_key_node = &right_join_actions->addCast(*right_key_node, common_type, {});
+                    right_key_node = &right_join_actions.addCast(*right_key_node, common_type, {});
             }
 
             if (join_clause.isNullsafeCompareKey(i) && left_key_node->result_type->isNullable() && right_key_node->result_type->isNullable())
@@ -576,24 +576,24 @@ JoinClausesAndActions buildJoinClausesAndActions(
                   *   SELECT * FROM t1 JOIN t2 ON tuple(t1.a) == tuple(t2.b)
                   */
                 auto wrap_nullsafe_function = FunctionFactory::instance().get("tuple", planner_context->getQueryContext());
-                left_key_node = &left_join_actions->addFunction(wrap_nullsafe_function, {left_key_node}, {});
-                right_key_node = &right_join_actions->addFunction(wrap_nullsafe_function, {right_key_node}, {});
+                left_key_node = &left_join_actions.addFunction(wrap_nullsafe_function, {left_key_node}, {});
+                right_key_node = &right_join_actions.addFunction(wrap_nullsafe_function, {right_key_node}, {});
             }
 
-            left_join_actions->addOrReplaceInOutputs(*left_key_node);
-            right_join_actions->addOrReplaceInOutputs(*right_key_node);
+            left_join_actions.addOrReplaceInOutputs(*left_key_node);
+            right_join_actions.addOrReplaceInOutputs(*right_key_node);
 
             add_necessary_name_if_needed(JoinTableSide::Left, left_key_node->result_name);
             add_necessary_name_if_needed(JoinTableSide::Right, right_key_node->result_name);
         }
     }
 
-    result.left_join_expressions_actions = ActionsDAG::clone(left_join_actions);
-    result.left_join_tmp_expression_actions = std::move(left_join_actions);
-    result.left_join_expressions_actions->removeUnusedActions(join_left_actions_names);
-    result.right_join_expressions_actions = ActionsDAG::clone(right_join_actions);
-    result.right_join_tmp_expression_actions = std::move(right_join_actions);
-    result.right_join_expressions_actions->removeUnusedActions(join_right_actions_names);
+    result.left_join_expressions_actions = std::move(left_join_actions);
+    //result.left_join_tmp_expression_actions = std::move(left_join_actions);
+    result.left_join_expressions_actions.removeUnusedActions(join_left_actions_names);
+    result.right_join_expressions_actions = std::move(right_join_actions);
+    //result.right_join_tmp_expression_actions = std::move(right_join_actions);
+    result.right_join_expressions_actions.removeUnusedActions(join_right_actions_names);
 
     if (is_inequal_join)
     {
@@ -601,16 +601,16 @@ JoinClausesAndActions buildJoinClausesAndActions(
         /// So, for each column, we recalculate the value of the whole expression from JOIN ON to check if rows should be joined.
         if (result.join_clauses.size() > 1)
         {
-            auto mixed_join_expressions_actions = std::make_unique<ActionsDAG>(mixed_table_expression_columns);
+            ActionsDAG mixed_join_expressions_actions(mixed_table_expression_columns);
             PlannerActionsVisitor join_expression_visitor(planner_context);
-            auto join_expression_dag_node_raw_pointers = join_expression_visitor.visit(*mixed_join_expressions_actions, join_expression);
+            auto join_expression_dag_node_raw_pointers = join_expression_visitor.visit(mixed_join_expressions_actions, join_expression);
             if (join_expression_dag_node_raw_pointers.size() != 1)
                 throw Exception(
                     ErrorCodes::LOGICAL_ERROR, "JOIN {} ON clause contains multiple expressions", join_node.formatASTForErrorMessage());
 
-            mixed_join_expressions_actions->addOrReplaceInOutputs(*join_expression_dag_node_raw_pointers[0]);
+            mixed_join_expressions_actions.addOrReplaceInOutputs(*join_expression_dag_node_raw_pointers[0]);
             Names required_names{join_expression_dag_node_raw_pointers[0]->result_name};
-            mixed_join_expressions_actions->removeUnusedActions(required_names);
+            mixed_join_expressions_actions.removeUnusedActions(required_names);
             result.mixed_join_expressions_actions = std::move(mixed_join_expressions_actions);
         }
         else
diff --git a/src/Planner/PlannerJoins.h b/src/Planner/PlannerJoins.h
index 8adf6edd7ea..3735c373acc 100644
--- a/src/Planner/PlannerJoins.h
+++ b/src/Planner/PlannerJoins.h
@@ -182,15 +182,15 @@ struct JoinClausesAndActions
     /// Join clauses. Actions dag nodes point into join_expression_actions.
     JoinClauses join_clauses;
     /// Whole JOIN ON section expressions
-    ActionsDAGPtr left_join_tmp_expression_actions;
-    ActionsDAGPtr right_join_tmp_expression_actions;
+    // ActionsDAGPtr left_join_tmp_expression_actions;
+    // ActionsDAGPtr right_join_tmp_expression_actions;
     /// Left join expressions actions
-    ActionsDAGPtr left_join_expressions_actions;
+    ActionsDAG left_join_expressions_actions;
     /// Right join expressions actions
-    ActionsDAGPtr right_join_expressions_actions;
+    ActionsDAG right_join_expressions_actions;
     /// Originally used for inequal join. it's the total join expression.
     /// If there is no inequal join conditions, it's null.
-    ActionsDAGPtr mixed_join_expressions_actions;
+    std::optional<ActionsDAG> mixed_join_expressions_actions;
 };
 
 /** Calculate join clauses and actions for JOIN ON section.
diff --git a/src/Planner/Utils.cpp b/src/Planner/Utils.cpp
index 7ac53e0f8c1..e9f9c51d338 100644
--- a/src/Planner/Utils.cpp
+++ b/src/Planner/Utils.cpp
@@ -442,22 +442,22 @@ FilterDAGInfo buildFilterInfo(QueryTreeNodePtr filter_query_tree,
     collectSourceColumns(filter_query_tree, planner_context, false /*keep_alias_columns*/);
     collectSets(filter_query_tree, *planner_context);
 
-    auto filter_actions_dag = std::make_unique<ActionsDAG>();
+    ActionsDAG filter_actions_dag;
 
     PlannerActionsVisitor actions_visitor(planner_context, false /*use_column_identifier_as_action_node_name*/);
-    auto expression_nodes = actions_visitor.visit(*filter_actions_dag, filter_query_tree);
+    auto expression_nodes = actions_visitor.visit(filter_actions_dag, filter_query_tree);
     if (expression_nodes.size() != 1)
         throw Exception(ErrorCodes::BAD_ARGUMENTS,
             "Filter actions must return single output node. Actual {}",
             expression_nodes.size());
 
-    auto & filter_actions_outputs = filter_actions_dag->getOutputs();
+    auto & filter_actions_outputs = filter_actions_dag.getOutputs();
     filter_actions_outputs = std::move(expression_nodes);
 
     std::string filter_node_name = filter_actions_outputs[0]->result_name;
     bool remove_filter_column = true;
 
-    for (const auto & filter_input_node : filter_actions_dag->getInputs())
+    for (const auto & filter_input_node : filter_actions_dag.getInputs())
         if (table_expression_required_names_without_filter.contains(filter_input_node->result_name))
             filter_actions_outputs.push_back(filter_input_node);
 
@@ -498,7 +498,7 @@ void appendSetsFromActionsDAG(const ActionsDAG & dag, UsefulSets & useful_sets)
             {
                 if (const auto * index_hint = typeid_cast<const FunctionIndexHint *>(adaptor->getFunction().get()))
                 {
-                    appendSetsFromActionsDAG(*index_hint->getActions(), useful_sets);
+                    appendSetsFromActionsDAG(index_hint->getActions(), useful_sets);
                 }
             }
         }
diff --git a/src/Processors/QueryPlan/AggregatingStep.cpp b/src/Processors/QueryPlan/AggregatingStep.cpp
index 64ba7f7cd2a..f31de80b22d 100644
--- a/src/Processors/QueryPlan/AggregatingStep.cpp
+++ b/src/Processors/QueryPlan/AggregatingStep.cpp
@@ -303,15 +303,15 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B
                 const auto & header = ports[set_counter]->getHeader();
 
                 /// Here we create a DAG which fills missing keys and adds `__grouping_set` column
-                auto dag = std::make_unique<ActionsDAG>(header.getColumnsWithTypeAndName());
+                ActionsDAG dag(header.getColumnsWithTypeAndName());
                 ActionsDAG::NodeRawConstPtrs outputs;
                 outputs.reserve(output_header.columns() + 1);
 
                 auto grouping_col = ColumnConst::create(ColumnUInt64::create(1, set_counter), 0);
-                const auto * grouping_node = &dag->addColumn(
+                const auto * grouping_node = &dag.addColumn(
                     {ColumnPtr(std::move(grouping_col)), std::make_shared<DataTypeUInt64>(), "__grouping_set"});
 
-                grouping_node = &dag->materializeNode(*grouping_node);
+                grouping_node = &dag.materializeNode(*grouping_node);
                 outputs.push_back(grouping_node);
 
                 const auto & missing_columns = grouping_sets_params[set_counter].missing_keys;
@@ -332,21 +332,21 @@ void AggregatingStep::transformPipeline(QueryPipelineBuilder & pipeline, const B
                         column_with_default->finalize();
 
                         auto column = ColumnConst::create(std::move(column_with_default), 0);
-                        const auto * node = &dag->addColumn({ColumnPtr(std::move(column)), col.type, col.name});
-                        node = &dag->materializeNode(*node);
+                        const auto * node = &dag.addColumn({ColumnPtr(std::move(column)), col.type, col.name});
+                        node = &dag.materializeNode(*node);
                         outputs.push_back(node);
                     }
                     else
                     {
-                        const auto * column_node = dag->getOutputs()[header.getPositionByName(col.name)];
+                        const auto * column_node = dag.getOutputs()[header.getPositionByName(col.name)];
                         if (used_it != used_keys.end() && group_by_use_nulls && column_node->result_type->canBeInsideNullable())
-                            outputs.push_back(&dag->addFunction(to_nullable_function, { column_node }, col.name));
+                            outputs.push_back(&dag.addFunction(to_nullable_function, { column_node }, col.name));
                         else
                             outputs.push_back(column_node);
                     }
                 }
 
-                dag->getOutputs().swap(outputs);
+                dag.getOutputs().swap(outputs);
                 auto expression = std::make_shared<ExpressionActions>(std::move(dag), settings.getActionsSettings());
                 auto transform = std::make_shared<ExpressionTransform>(header, expression);
 
diff --git a/src/Processors/QueryPlan/CubeStep.cpp b/src/Processors/QueryPlan/CubeStep.cpp
index b6c70061987..3a98f8e4612 100644
--- a/src/Processors/QueryPlan/CubeStep.cpp
+++ b/src/Processors/QueryPlan/CubeStep.cpp
@@ -36,27 +36,27 @@ CubeStep::CubeStep(const DataStream & input_stream_, Aggregator::Params params_,
 
 ProcessorPtr addGroupingSetForTotals(const Block & header, const Names & keys, bool use_nulls, const BuildQueryPipelineSettings & settings, UInt64 grouping_set_number)
 {
-    auto dag = std::make_unique<ActionsDAG>(header.getColumnsWithTypeAndName());
-    auto & outputs = dag->getOutputs();
+    ActionsDAG dag(header.getColumnsWithTypeAndName());
+    auto & outputs = dag.getOutputs();
 
     if (use_nulls)
     {
         auto to_nullable = FunctionFactory::instance().get("toNullable", nullptr);
         for (const auto & key : keys)
         {
-            const auto * node = dag->getOutputs()[header.getPositionByName(key)];
+            const auto * node = dag.getOutputs()[header.getPositionByName(key)];
             if (node->result_type->canBeInsideNullable())
             {
-                dag->addOrReplaceInOutputs(dag->addFunction(to_nullable, { node }, node->result_name));
+                dag.addOrReplaceInOutputs(dag.addFunction(to_nullable, { node }, node->result_name));
             }
         }
     }
 
     auto grouping_col = ColumnUInt64::create(1, grouping_set_number);
-    const auto * grouping_node = &dag->addColumn(
+    const auto * grouping_node = &dag.addColumn(
         {ColumnPtr(std::move(grouping_col)), std::make_shared<DataTypeUInt64>(), "__grouping_set"});
 
-    grouping_node = &dag->materializeNode(*grouping_node);
+    grouping_node = &dag.materializeNode(*grouping_node);
     outputs.insert(outputs.begin(), grouping_node);
 
     auto expression = std::make_shared<ExpressionActions>(std::move(dag), settings.getActionsSettings());
diff --git a/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp b/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp
index 1f4f271fa6e..1c199ebedb3 100644
--- a/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp
+++ b/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp
@@ -32,7 +32,7 @@ void addConvertingActions(QueryPlan & plan, const Block & header, bool has_missi
     };
 
     auto convert_actions_dag = get_converting_dag(plan.getCurrentDataStream().header, header);
-    auto converting = std::make_unique<ExpressionStep>(plan.getCurrentDataStream(), convert_actions_dag);
+    auto converting = std::make_unique<ExpressionStep>(plan.getCurrentDataStream(), std::move(convert_actions_dag));
     plan.addStep(std::move(converting));
 }
 
diff --git a/src/Processors/QueryPlan/ExpressionStep.cpp b/src/Processors/QueryPlan/ExpressionStep.cpp
index 50bc2e1533e..94098f443d9 100644
--- a/src/Processors/QueryPlan/ExpressionStep.cpp
+++ b/src/Processors/QueryPlan/ExpressionStep.cpp
@@ -10,33 +10,33 @@
 namespace DB
 {
 
-static ITransformingStep::Traits getTraits(const ActionsDAGPtr & actions, const Block & header, const SortDescription & sort_description)
+static ITransformingStep::Traits getTraits(const ActionsDAG & actions, const Block & header, const SortDescription & sort_description)
 {
     return ITransformingStep::Traits
     {
         {
             .returns_single_stream = false,
             .preserves_number_of_streams = true,
-            .preserves_sorting = actions->isSortingPreserved(header, sort_description),
+            .preserves_sorting = actions.isSortingPreserved(header, sort_description),
         },
         {
-            .preserves_number_of_rows = !actions->hasArrayJoin(),
+            .preserves_number_of_rows = !actions.hasArrayJoin(),
         }
     };
 }
 
-ExpressionStep::ExpressionStep(const DataStream & input_stream_, const ActionsDAGPtr & actions_dag_)
+ExpressionStep::ExpressionStep(const DataStream & input_stream_, ActionsDAG actions_dag_)
     : ITransformingStep(
         input_stream_,
-        ExpressionTransform::transformHeader(input_stream_.header, *actions_dag_),
+        ExpressionTransform::transformHeader(input_stream_.header, actions_dag_),
         getTraits(actions_dag_, input_stream_.header, input_stream_.sort_description))
-    , actions_dag(ActionsDAG::clone(actions_dag_))
+    , actions_dag(std::move(actions_dag_))
 {
 }
 
 void ExpressionStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings & settings)
 {
-    auto expression = std::make_shared<ExpressionActions>(ActionsDAG::clone(actions_dag), settings.getActionsSettings());
+    auto expression = std::make_shared<ExpressionActions>(std::move(actions_dag), settings.getActionsSettings());
 
     pipeline.addSimpleTransform([&](const Block & header)
     {
@@ -61,25 +61,25 @@ void ExpressionStep::transformPipeline(QueryPipelineBuilder & pipeline, const Bu
 void ExpressionStep::describeActions(FormatSettings & settings) const
 {
     String prefix(settings.offset, settings.indent_char);
-    auto expression = std::make_shared<ExpressionActions>(ActionsDAG::clone(actions_dag));
+    auto expression = std::make_shared<ExpressionActions>(std::move(*ActionsDAG::clone(&actions_dag)));
     expression->describeActions(settings.out, prefix);
 }
 
 void ExpressionStep::describeActions(JSONBuilder::JSONMap & map) const
 {
-    auto expression = std::make_shared<ExpressionActions>(ActionsDAG::clone(actions_dag));
+    auto expression = std::make_shared<ExpressionActions>(std::move(*ActionsDAG::clone(&actions_dag)));
     map.add("Expression", expression->toTree());
 }
 
 void ExpressionStep::updateOutputStream()
 {
     output_stream = createOutputStream(
-        input_streams.front(), ExpressionTransform::transformHeader(input_streams.front().header, *actions_dag), getDataStreamTraits());
+        input_streams.front(), ExpressionTransform::transformHeader(input_streams.front().header, actions_dag), getDataStreamTraits());
 
     if (!getDataStreamTraits().preserves_sorting)
         return;
 
-    FindAliasForInputName alias_finder(*actions_dag);
+    FindAliasForInputName alias_finder(actions_dag);
     const auto & input_sort_description = getInputStreams().front().sort_description;
     for (size_t i = 0, s = input_sort_description.size(); i < s; ++i)
     {
diff --git a/src/Processors/QueryPlan/ExpressionStep.h b/src/Processors/QueryPlan/ExpressionStep.h
index ebbac8217cb..f2926318cbc 100644
--- a/src/Processors/QueryPlan/ExpressionStep.h
+++ b/src/Processors/QueryPlan/ExpressionStep.h
@@ -1,12 +1,10 @@
 #pragma once
 #include <Processors/QueryPlan/ITransformingStep.h>
+#include <Interpreters/ActionsDAG.h>
 
 namespace DB
 {
 
-class ActionsDAG;
-using ActionsDAGPtr = std::unique_ptr<ActionsDAG>;
-
 class ExpressionTransform;
 class JoiningTransform;
 
@@ -15,21 +13,22 @@ class ExpressionStep : public ITransformingStep
 {
 public:
 
-    explicit ExpressionStep(const DataStream & input_stream_, const ActionsDAGPtr & actions_dag_);
+    explicit ExpressionStep(const DataStream & input_stream_, ActionsDAG actions_dag_);
     String getName() const override { return "Expression"; }
 
     void transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings & settings) override;
 
     void describeActions(FormatSettings & settings) const override;
 
-    const ActionsDAGPtr & getExpression() const { return actions_dag; }
+    ActionsDAG & getExpression() { return actions_dag; }
+    const ActionsDAG & getExpression() const { return actions_dag; }
 
     void describeActions(JSONBuilder::JSONMap & map) const override;
 
 private:
     void updateOutputStream() override;
 
-    ActionsDAGPtr actions_dag;
+    ActionsDAG actions_dag;
 };
 
 }
diff --git a/src/Processors/QueryPlan/FilterStep.cpp b/src/Processors/QueryPlan/FilterStep.cpp
index 7883461f45a..5f15c5defac 100644
--- a/src/Processors/QueryPlan/FilterStep.cpp
+++ b/src/Processors/QueryPlan/FilterStep.cpp
@@ -9,9 +9,9 @@
 namespace DB
 {
 
-static ITransformingStep::Traits getTraits(const ActionsDAGPtr & expression, const Block & header, const SortDescription & sort_description, bool remove_filter_column, const String & filter_column_name)
+static ITransformingStep::Traits getTraits(const ActionsDAG & expression, const Block & header, const SortDescription & sort_description, bool remove_filter_column, const String & filter_column_name)
 {
-    bool preserves_sorting = expression->isSortingPreserved(header, sort_description, remove_filter_column ? filter_column_name : "");
+    bool preserves_sorting = expression.isSortingPreserved(header, sort_description, remove_filter_column ? filter_column_name : "");
     if (remove_filter_column)
     {
         preserves_sorting &= std::find_if(
@@ -35,22 +35,22 @@ static ITransformingStep::Traits getTraits(const ActionsDAGPtr & expression, con
 
 FilterStep::FilterStep(
     const DataStream & input_stream_,
-    const ActionsDAGPtr & actions_dag_,
+    ActionsDAG actions_dag_,
     String filter_column_name_,
     bool remove_filter_column_)
     : ITransformingStep(
         input_stream_,
         FilterTransform::transformHeader(
             input_stream_.header,
-            actions_dag_.get(),
+            &actions_dag_,
             filter_column_name_,
             remove_filter_column_),
         getTraits(actions_dag_, input_stream_.header, input_stream_.sort_description, remove_filter_column_, filter_column_name_))
+    , actions_dag(std::move(actions_dag_))
     , filter_column_name(std::move(filter_column_name_))
     , remove_filter_column(remove_filter_column_)
 {
-    actions_dag = ActionsDAG::clone(actions_dag_);
-    actions_dag->removeAliasesForFilter(filter_column_name);
+    actions_dag.removeAliasesForFilter(filter_column_name);
 }
 
 void FilterStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings & settings)
@@ -87,7 +87,7 @@ void FilterStep::describeActions(FormatSettings & settings) const
         settings.out << " (removed)";
     settings.out << '\n';
 
-    auto expression = std::make_shared<ExpressionActions>(ActionsDAG::clone(actions_dag));
+    auto expression = std::make_shared<ExpressionActions>(std::move(*ActionsDAG::clone(&actions_dag)));
     expression->describeActions(settings.out, prefix);
 }
 
@@ -96,7 +96,7 @@ void FilterStep::describeActions(JSONBuilder::JSONMap & map) const
     map.add("Filter Column", filter_column_name);
     map.add("Removes Filter", remove_filter_column);
 
-    auto expression = std::make_shared<ExpressionActions>(ActionsDAG::clone(actions_dag));
+    auto expression = std::make_shared<ExpressionActions>(std::move(*ActionsDAG::clone(&actions_dag)));
     map.add("Expression", expression->toTree());
 }
 
@@ -104,13 +104,13 @@ void FilterStep::updateOutputStream()
 {
     output_stream = createOutputStream(
         input_streams.front(),
-        FilterTransform::transformHeader(input_streams.front().header, actions_dag.get(), filter_column_name, remove_filter_column),
+        FilterTransform::transformHeader(input_streams.front().header, &actions_dag, filter_column_name, remove_filter_column),
         getDataStreamTraits());
 
     if (!getDataStreamTraits().preserves_sorting)
         return;
 
-    FindAliasForInputName alias_finder(*actions_dag);
+    FindAliasForInputName alias_finder(actions_dag);
     const auto & input_sort_description = getInputStreams().front().sort_description;
     for (size_t i = 0, s = input_sort_description.size(); i < s; ++i)
     {
diff --git a/src/Processors/QueryPlan/FilterStep.h b/src/Processors/QueryPlan/FilterStep.h
index 0f894a570b7..b5a31bef5fc 100644
--- a/src/Processors/QueryPlan/FilterStep.h
+++ b/src/Processors/QueryPlan/FilterStep.h
@@ -1,19 +1,17 @@
 #pragma once
 #include <Processors/QueryPlan/ITransformingStep.h>
+#include <Interpreters/ActionsDAG.h>
 
 namespace DB
 {
 
-class ActionsDAG;
-using ActionsDAGPtr = std::unique_ptr<ActionsDAG>;
-
 /// Implements WHERE, HAVING operations. See FilterTransform.
 class FilterStep : public ITransformingStep
 {
 public:
     FilterStep(
         const DataStream & input_stream_,
-        const ActionsDAGPtr & actions_dag_,
+        ActionsDAG actions_dag_,
         String filter_column_name_,
         bool remove_filter_column_);
 
@@ -23,15 +21,15 @@ public:
     void describeActions(JSONBuilder::JSONMap & map) const override;
     void describeActions(FormatSettings & settings) const override;
 
-    const ActionsDAGPtr & getExpression() const { return actions_dag; }
-    ActionsDAGPtr & getExpression() { return actions_dag; }
+    const ActionsDAG & getExpression() const { return actions_dag; }
+    ActionsDAG & getExpression() { return actions_dag; }
     const String & getFilterColumnName() const { return filter_column_name; }
     bool removesFilterColumn() const { return remove_filter_column; }
 
 private:
     void updateOutputStream() override;
 
-    ActionsDAGPtr actions_dag;
+    ActionsDAG actions_dag;
     String filter_column_name;
     bool remove_filter_column;
 };
diff --git a/src/Processors/QueryPlan/Optimizations/convertOuterJoinToInnerJoin.cpp b/src/Processors/QueryPlan/Optimizations/convertOuterJoinToInnerJoin.cpp
index d90f0e152e7..be468419cfb 100644
--- a/src/Processors/QueryPlan/Optimizations/convertOuterJoinToInnerJoin.cpp
+++ b/src/Processors/QueryPlan/Optimizations/convertOuterJoinToInnerJoin.cpp
@@ -45,10 +45,10 @@ size_t tryConvertOuterJoinToInnerJoin(QueryPlan::Node * parent_node, QueryPlan::
     bool right_stream_safe = true;
 
     if (check_left_stream)
-        left_stream_safe = filter_dag->isFilterAlwaysFalseForDefaultValueInputs(filter_column_name, left_stream_input_header);
+        left_stream_safe = filter_dag.isFilterAlwaysFalseForDefaultValueInputs(filter_column_name, left_stream_input_header);
 
     if (check_right_stream)
-        right_stream_safe = filter_dag->isFilterAlwaysFalseForDefaultValueInputs(filter_column_name, right_stream_input_header);
+        right_stream_safe = filter_dag.isFilterAlwaysFalseForDefaultValueInputs(filter_column_name, right_stream_input_header);
 
     if (!left_stream_safe || !right_stream_safe)
         return 0;
diff --git a/src/Processors/QueryPlan/Optimizations/distinctReadInOrder.cpp b/src/Processors/QueryPlan/Optimizations/distinctReadInOrder.cpp
index 6cdc3cb4eb0..8666912514e 100644
--- a/src/Processors/QueryPlan/Optimizations/distinctReadInOrder.cpp
+++ b/src/Processors/QueryPlan/Optimizations/distinctReadInOrder.cpp
@@ -79,9 +79,9 @@ size_t tryDistinctReadInOrder(QueryPlan::Node * parent_node)
         steps_to_update.push_back(step);
 
         if (const auto * const expr = typeid_cast<const ExpressionStep *>(step); expr)
-            dag_stack.push_back(expr->getExpression().get());
+            dag_stack.push_back(&expr->getExpression());
         else if (const auto * const filter = typeid_cast<const FilterStep *>(step); filter)
-            dag_stack.push_back(filter->getExpression().get());
+            dag_stack.push_back(&filter->getExpression());
 
         node = node->children.front();
     }
diff --git a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp
index f26cd79dd97..411b20b1a32 100644
--- a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp
+++ b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp
@@ -101,7 +101,7 @@ static NameSet findIdentifiersOfNode(const ActionsDAG::Node * node)
     return res;
 }
 
-static ActionsDAGPtr splitFilter(QueryPlan::Node * parent_node, const Names & available_inputs, size_t child_idx = 0)
+static std::optional<ActionsDAG> splitFilter(QueryPlan::Node * parent_node, const Names & available_inputs, size_t child_idx = 0)
 {
     QueryPlan::Node * child_node = parent_node->children.front();
     checkChildrenSize(child_node, child_idx + 1);
@@ -110,16 +110,16 @@ static ActionsDAGPtr splitFilter(QueryPlan::Node * parent_node, const Names & av
     auto & child = child_node->step;
 
     auto * filter = assert_cast<FilterStep *>(parent.get());
-    const auto & expression = filter->getExpression();
+    auto & expression = filter->getExpression();
     const auto & filter_column_name = filter->getFilterColumnName();
     bool removes_filter = filter->removesFilterColumn();
 
     const auto & all_inputs = child->getInputStreams()[child_idx].header.getColumnsWithTypeAndName();
-    return expression->splitActionsForFilterPushDown(filter_column_name, removes_filter, available_inputs, all_inputs);
+    return expression.splitActionsForFilterPushDown(filter_column_name, removes_filter, available_inputs, all_inputs);
 }
 
 static size_t
-addNewFilterStepOrThrow(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes, const ActionsDAGPtr & split_filter,
+addNewFilterStepOrThrow(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes, ActionsDAG split_filter,
                     bool can_remove_filter = true, size_t child_idx = 0, bool update_parent_filter = true)
 {
     QueryPlan::Node * child_node = parent_node->children.front();
@@ -129,14 +129,14 @@ addNewFilterStepOrThrow(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes,
     auto & child = child_node->step;
 
     auto * filter = assert_cast<FilterStep *>(parent.get());
-    const auto & expression = filter->getExpression();
+    auto & expression = filter->getExpression();
     const auto & filter_column_name = filter->getFilterColumnName();
 
-    const auto * filter_node = expression->tryFindInOutputs(filter_column_name);
+    const auto * filter_node = expression.tryFindInOutputs(filter_column_name);
     if (update_parent_filter && !filter_node && !filter->removesFilterColumn())
         throw Exception(ErrorCodes::LOGICAL_ERROR,
                         "Filter column {} was removed from ActionsDAG but it is needed in result. DAG:\n{}",
-                        filter_column_name, expression->dumpDAG());
+                        filter_column_name, expression.dumpDAG());
 
     /// Add new Filter step before Child.
     /// Expression/Filter -> Child -> Something
@@ -147,10 +147,10 @@ addNewFilterStepOrThrow(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes,
     /// Expression/Filter -> Child -> Filter -> Something
 
     /// New filter column is the first one.
-    String split_filter_column_name = split_filter->getOutputs().front()->result_name;
+    String split_filter_column_name = split_filter.getOutputs().front()->result_name;
 
     node.step = std::make_unique<FilterStep>(
-        node.children.at(0)->step->getOutputStream(), split_filter, std::move(split_filter_column_name), can_remove_filter);
+        node.children.at(0)->step->getOutputStream(), std::move(split_filter), std::move(split_filter_column_name), can_remove_filter);
 
     if (auto * transforming_step = dynamic_cast<ITransformingStep *>(child.get()))
     {
@@ -176,7 +176,7 @@ addNewFilterStepOrThrow(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes,
         {
             /// This means that all predicates of filter were pushed down.
             /// Replace current actions to expression, as we don't need to filter anything.
-            parent = std::make_unique<ExpressionStep>(child->getOutputStream(), expression);
+            parent = std::make_unique<ExpressionStep>(child->getOutputStream(), std::move(expression));
         }
         else
         {
@@ -192,7 +192,7 @@ tryAddNewFilterStep(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes, con
                     bool can_remove_filter = true, size_t child_idx = 0)
 {
     if (auto split_filter = splitFilter(parent_node, allowed_inputs, child_idx))
-        return addNewFilterStepOrThrow(parent_node, nodes, split_filter, can_remove_filter, child_idx);
+        return addNewFilterStepOrThrow(parent_node, nodes, std::move(*split_filter), can_remove_filter, child_idx);
     return 0;
 }
 
@@ -332,7 +332,7 @@ static size_t tryPushDownOverJoinStep(QueryPlan::Node * parent_node, QueryPlan::
     Names left_stream_available_columns_to_push_down = get_available_columns_for_filter(true /*push_to_left_stream*/, left_stream_filter_push_down_input_columns_available);
     Names right_stream_available_columns_to_push_down = get_available_columns_for_filter(false /*push_to_left_stream*/, right_stream_filter_push_down_input_columns_available);
 
-    auto join_filter_push_down_actions = filter->getExpression()->splitActionsForJOINFilterPushDown(filter->getFilterColumnName(),
+    auto join_filter_push_down_actions = filter->getExpression().splitActionsForJOINFilterPushDown(filter->getFilterColumnName(),
         filter->removesFilterColumn(),
         left_stream_available_columns_to_push_down,
         left_stream_input_header,
@@ -346,42 +346,44 @@ static size_t tryPushDownOverJoinStep(QueryPlan::Node * parent_node, QueryPlan::
 
     if (join_filter_push_down_actions.left_stream_filter_to_push_down)
     {
+        const auto & result_name = join_filter_push_down_actions.left_stream_filter_to_push_down->getOutputs()[0]->result_name;
         updated_steps += addNewFilterStepOrThrow(parent_node,
             nodes,
-            join_filter_push_down_actions.left_stream_filter_to_push_down,
+            std::move(*join_filter_push_down_actions.left_stream_filter_to_push_down),
             join_filter_push_down_actions.left_stream_filter_removes_filter,
             0 /*child_idx*/,
             false /*update_parent_filter*/);
         LOG_DEBUG(&Poco::Logger::get("QueryPlanOptimizations"),
             "Pushed down filter {} to the {} side of join",
-            join_filter_push_down_actions.left_stream_filter_to_push_down->getOutputs()[0]->result_name,
+            result_name,
             JoinKind::Left);
     }
 
     if (join_filter_push_down_actions.right_stream_filter_to_push_down && allow_push_down_to_right)
     {
+        const auto & result_name = join_filter_push_down_actions.right_stream_filter_to_push_down->getOutputs()[0]->result_name;
         updated_steps += addNewFilterStepOrThrow(parent_node,
             nodes,
-            join_filter_push_down_actions.right_stream_filter_to_push_down,
+            std::move(*join_filter_push_down_actions.right_stream_filter_to_push_down),
             join_filter_push_down_actions.right_stream_filter_removes_filter,
             1 /*child_idx*/,
             false /*update_parent_filter*/);
         LOG_DEBUG(&Poco::Logger::get("QueryPlanOptimizations"),
             "Pushed down filter {} to the {} side of join",
-            join_filter_push_down_actions.right_stream_filter_to_push_down->getOutputs()[0]->result_name,
+            result_name,
             JoinKind::Right);
     }
 
     if (updated_steps > 0)
     {
         const auto & filter_column_name = filter->getFilterColumnName();
-        const auto & filter_expression = filter->getExpression();
+        auto & filter_expression = filter->getExpression();
 
-        const auto * filter_node = filter_expression->tryFindInOutputs(filter_column_name);
+        const auto * filter_node = filter_expression.tryFindInOutputs(filter_column_name);
         if (!filter_node && !filter->removesFilterColumn())
             throw Exception(ErrorCodes::LOGICAL_ERROR,
                         "Filter column {} was removed from ActionsDAG but it is needed in result. DAG:\n{}",
-                        filter_column_name, filter_expression->dumpDAG());
+                        filter_column_name, filter_expression.dumpDAG());
 
 
         /// Filter column was replaced to constant.
@@ -391,7 +393,7 @@ static size_t tryPushDownOverJoinStep(QueryPlan::Node * parent_node, QueryPlan::
         {
             /// This means that all predicates of filter were pushed down.
             /// Replace current actions to expression, as we don't need to filter anything.
-            parent = std::make_unique<ExpressionStep>(child->getOutputStream(), filter_expression);
+            parent = std::make_unique<ExpressionStep>(child->getOutputStream(), std::move(filter_expression));
         }
         else
         {
@@ -416,7 +418,7 @@ size_t tryPushDownFilter(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes
     if (!filter)
         return 0;
 
-    if (filter->getExpression()->hasStatefulFunctions())
+    if (filter->getExpression().hasStatefulFunctions())
         return 0;
 
     if (auto * aggregating = typeid_cast<AggregatingStep *>(child.get()))
@@ -430,7 +432,7 @@ size_t tryPushDownFilter(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes
                 return 0;
 
             const auto & actions = filter->getExpression();
-            const auto & filter_node = actions->findInOutputs(filter->getFilterColumnName());
+            const auto & filter_node = actions.findInOutputs(filter->getFilterColumnName());
 
             auto identifiers_in_predicate = findIdentifiersOfNode(&filter_node);
 
@@ -597,7 +599,7 @@ size_t tryPushDownFilter(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes
 
             filter_node.step = std::make_unique<FilterStep>(
                 filter_node.children.front()->step->getOutputStream(),
-                ActionsDAG::clone(filter->getExpression()),
+                std::move(*ActionsDAG::clone(&filter->getExpression())),
                 filter->getFilterColumnName(),
                 filter->removesFilterColumn());
         }
@@ -611,7 +613,7 @@ size_t tryPushDownFilter(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes
 
     if (auto * read_from_merge = typeid_cast<ReadFromMerge *>(child.get()))
     {
-        FilterDAGInfo info{ActionsDAG::clone(filter->getExpression()), filter->getFilterColumnName(), filter->removesFilterColumn()};
+        FilterDAGInfo info{std::move(*ActionsDAG::clone(&filter->getExpression())), filter->getFilterColumnName(), filter->removesFilterColumn()};
         read_from_merge->addFilter(std::move(info));
         std::swap(*parent_node, *child_node);
         return 1;
diff --git a/src/Processors/QueryPlan/Optimizations/liftUpArrayJoin.cpp b/src/Processors/QueryPlan/Optimizations/liftUpArrayJoin.cpp
index 36aab41df49..0d4f2330119 100644
--- a/src/Processors/QueryPlan/Optimizations/liftUpArrayJoin.cpp
+++ b/src/Processors/QueryPlan/Optimizations/liftUpArrayJoin.cpp
@@ -28,10 +28,10 @@ size_t tryLiftUpArrayJoin(QueryPlan::Node * parent_node, QueryPlan::Nodes & node
     const auto & expression = expression_step ? expression_step->getExpression()
                                               : filter_step->getExpression();
 
-    auto split_actions = expression->splitActionsBeforeArrayJoin(array_join->columns);
+    auto split_actions = expression.splitActionsBeforeArrayJoin(array_join->columns);
 
     /// No actions can be moved before ARRAY JOIN.
-    if (split_actions.first->trivial())
+    if (split_actions.first.trivial())
         return 0;
 
     auto description = parent->getStepDescription();
@@ -49,9 +49,9 @@ size_t tryLiftUpArrayJoin(QueryPlan::Node * parent_node, QueryPlan::Nodes & node
     array_join_step->updateInputStream(node.step->getOutputStream());
 
     if (expression_step)
-        parent = std::make_unique<ExpressionStep>(array_join_step->getOutputStream(), split_actions.second);
+        parent = std::make_unique<ExpressionStep>(array_join_step->getOutputStream(), std::move(split_actions.second));
     else
-        parent = std::make_unique<FilterStep>(array_join_step->getOutputStream(), split_actions.second,
+        parent = std::make_unique<FilterStep>(array_join_step->getOutputStream(), std::move(split_actions.second),
                                               filter_step->getFilterColumnName(), filter_step->removesFilterColumn());
 
     parent->setStepDescription(description + " [split]");
diff --git a/src/Processors/QueryPlan/Optimizations/liftUpFunctions.cpp b/src/Processors/QueryPlan/Optimizations/liftUpFunctions.cpp
index b280e2d3cc6..7794ddae8fa 100644
--- a/src/Processors/QueryPlan/Optimizations/liftUpFunctions.cpp
+++ b/src/Processors/QueryPlan/Optimizations/liftUpFunctions.cpp
@@ -66,13 +66,13 @@ size_t tryExecuteFunctionsAfterSorting(QueryPlan::Node * parent_node, QueryPlan:
     NameSet sort_columns;
     for (const auto & col : sorting_step->getSortDescription())
         sort_columns.insert(col.column_name);
-    auto [needed_for_sorting, unneeded_for_sorting, _] = expression_step->getExpression()->splitActionsBySortingDescription(sort_columns);
+    auto [needed_for_sorting, unneeded_for_sorting, _] = expression_step->getExpression().splitActionsBySortingDescription(sort_columns);
 
     // No calculations can be postponed.
-    if (unneeded_for_sorting->trivial())
+    if (unneeded_for_sorting.trivial())
         return 0;
 
-    if (!areNodesConvertableToBlock(needed_for_sorting->getOutputs()) || !areNodesConvertableToBlock(unneeded_for_sorting->getInputs()))
+    if (!areNodesConvertableToBlock(needed_for_sorting.getOutputs()) || !areNodesConvertableToBlock(unneeded_for_sorting.getInputs()))
         return 0;
 
     // Sorting (parent_node) -> Expression (child_node)
diff --git a/src/Processors/QueryPlan/Optimizations/liftUpUnion.cpp b/src/Processors/QueryPlan/Optimizations/liftUpUnion.cpp
index 4629bc0af53..53f59198d0f 100644
--- a/src/Processors/QueryPlan/Optimizations/liftUpUnion.cpp
+++ b/src/Processors/QueryPlan/Optimizations/liftUpUnion.cpp
@@ -49,7 +49,7 @@ size_t tryLiftUpUnion(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes)
 
             expr_node.step = std::make_unique<ExpressionStep>(
                 expr_node.children.front()->step->getOutputStream(),
-                ActionsDAG::clone(expression->getExpression()));
+                std::move(*ActionsDAG::clone(&expression->getExpression())));
         }
 
         ///       - Expression - Something
diff --git a/src/Processors/QueryPlan/Optimizations/mergeExpressions.cpp b/src/Processors/QueryPlan/Optimizations/mergeExpressions.cpp
index 97de69b1134..d7ca96e4c64 100644
--- a/src/Processors/QueryPlan/Optimizations/mergeExpressions.cpp
+++ b/src/Processors/QueryPlan/Optimizations/mergeExpressions.cpp
@@ -38,18 +38,18 @@ size_t tryMergeExpressions(QueryPlan::Node * parent_node, QueryPlan::Nodes &)
 
     if (parent_expr && child_expr)
     {
-        const auto & child_actions = child_expr->getExpression();
-        const auto & parent_actions = parent_expr->getExpression();
+        auto & child_actions = child_expr->getExpression();
+        auto & parent_actions = parent_expr->getExpression();
 
         /// We cannot combine actions with arrayJoin and stateful function because we not always can reorder them.
         /// Example: select rowNumberInBlock() from (select arrayJoin([1, 2]))
         /// Such a query will return two zeroes if we combine actions together.
-        if (child_actions->hasArrayJoin() && parent_actions->hasStatefulFunctions())
+        if (child_actions.hasArrayJoin() && parent_actions.hasStatefulFunctions())
             return 0;
 
-        auto merged = ActionsDAG::merge(std::move(*child_actions), std::move(*parent_actions));
+        auto merged = ActionsDAG::merge(std::move(child_actions), std::move(parent_actions));
 
-        auto expr = std::make_unique<ExpressionStep>(child_expr->getInputStreams().front(), merged);
+        auto expr = std::make_unique<ExpressionStep>(child_expr->getInputStreams().front(), std::move(merged));
         expr->setStepDescription("(" + parent_expr->getStepDescription() + " + " + child_expr->getStepDescription() + ")");
 
         parent_node->step = std::move(expr);
@@ -58,16 +58,16 @@ size_t tryMergeExpressions(QueryPlan::Node * parent_node, QueryPlan::Nodes &)
     }
     else if (parent_filter && child_expr)
     {
-        const auto & child_actions = child_expr->getExpression();
-        const auto & parent_actions = parent_filter->getExpression();
+        auto & child_actions = child_expr->getExpression();
+        auto & parent_actions = parent_filter->getExpression();
 
-        if (child_actions->hasArrayJoin() && parent_actions->hasStatefulFunctions())
+        if (child_actions.hasArrayJoin() && parent_actions.hasStatefulFunctions())
             return 0;
 
-        auto merged = ActionsDAG::merge(std::move(*child_actions), std::move(*parent_actions));
+        auto merged = ActionsDAG::merge(std::move(child_actions), std::move(parent_actions));
 
         auto filter = std::make_unique<FilterStep>(child_expr->getInputStreams().front(),
-                                                   merged,
+                                                   std::move(merged),
                                                    parent_filter->getFilterColumnName(),
                                                    parent_filter->removesFilterColumn());
         filter->setStepDescription("(" + parent_filter->getStepDescription() + " + " + child_expr->getStepDescription() + ")");
@@ -78,32 +78,31 @@ size_t tryMergeExpressions(QueryPlan::Node * parent_node, QueryPlan::Nodes &)
     }
     else if (parent_filter && child_filter)
     {
-        const auto & child_actions = child_filter->getExpression();
-        const auto & parent_actions = parent_filter->getExpression();
+        auto & child_actions = child_filter->getExpression();
+        auto & parent_actions = parent_filter->getExpression();
 
-        if (child_actions->hasArrayJoin())
+        if (child_actions.hasArrayJoin())
             return 0;
 
-        auto actions = ActionsDAG::clone(child_actions);
-        const auto & child_filter_node = actions->findInOutputs(child_filter->getFilterColumnName());
+        const auto & child_filter_node = child_actions.findInOutputs(child_filter->getFilterColumnName());
         if (child_filter->removesFilterColumn())
-            removeFromOutputs(*actions, child_filter_node);
+            removeFromOutputs(child_actions, child_filter_node);
 
-        actions->mergeInplace(std::move(*ActionsDAG::clone(parent_actions)));
+        child_actions.mergeInplace(std::move(parent_actions));
 
-        const auto & parent_filter_node = actions->findInOutputs(parent_filter->getFilterColumnName());
+        const auto & parent_filter_node = child_actions.findInOutputs(parent_filter->getFilterColumnName());
         if (parent_filter->removesFilterColumn())
-            removeFromOutputs(*actions, parent_filter_node);
+            removeFromOutputs(child_actions, parent_filter_node);
 
         FunctionOverloadResolverPtr func_builder_and = std::make_unique<FunctionToOverloadResolverAdaptor>(std::make_shared<FunctionAnd>());
-        const auto & condition = actions->addFunction(func_builder_and, {&child_filter_node, &parent_filter_node}, {});
-        auto & outputs = actions->getOutputs();
+        const auto & condition = child_actions.addFunction(func_builder_and, {&child_filter_node, &parent_filter_node}, {});
+        auto & outputs = child_actions.getOutputs();
         outputs.insert(outputs.begin(), &condition);
 
-        actions->removeUnusedActions(false);
+        child_actions.removeUnusedActions(false);
 
         auto filter = std::make_unique<FilterStep>(child_filter->getInputStreams().front(),
-                                                   actions,
+                                                   std::move(child_actions),
                                                    condition.result_name,
                                                    true);
         filter->setStepDescription("(" + parent_filter->getStepDescription() + " + " + child_filter->getStepDescription() + ")");
diff --git a/src/Processors/QueryPlan/Optimizations/optimizePrewhere.cpp b/src/Processors/QueryPlan/Optimizations/optimizePrewhere.cpp
index f203d831750..0d9e050d6cb 100644
--- a/src/Processors/QueryPlan/Optimizations/optimizePrewhere.cpp
+++ b/src/Processors/QueryPlan/Optimizations/optimizePrewhere.cpp
@@ -83,10 +83,11 @@ void optimizePrewhere(Stack & stack, QueryPlan::Nodes &)
 
     Names queried_columns = source_step_with_filter->requiredSourceColumns();
 
+    const auto & source_filter_actions_dag = source_step_with_filter->getFilterActionsDAG();
     MergeTreeWhereOptimizer where_optimizer{
         std::move(column_compressed_sizes),
         storage_metadata,
-        storage.getConditionSelectivityEstimatorByPredicate(storage_snapshot, source_step_with_filter->getFilterActionsDAG(), context),
+        storage.getConditionSelectivityEstimatorByPredicate(storage_snapshot, source_filter_actions_dag ? &*source_filter_actions_dag : nullptr, context),
         queried_columns,
         storage.supportedPrewhereColumns(),
         getLogger("QueryPlanOptimizePrewhere")};
@@ -113,15 +114,15 @@ void optimizePrewhere(Stack & stack, QueryPlan::Nodes &)
 
     if (prewhere_info->remove_prewhere_column)
     {
-        removeFromOutput(*filter_expression, filter_column_name);
-        auto & outputs = filter_expression->getOutputs();
+        removeFromOutput(filter_expression, filter_column_name);
+        auto & outputs = filter_expression.getOutputs();
         size_t size = outputs.size();
         outputs.insert(outputs.end(), optimize_result.prewhere_nodes.begin(), optimize_result.prewhere_nodes.end());
-        filter_expression->removeUnusedActions(false);
+        filter_expression.removeUnusedActions(false);
         outputs.resize(size);
     }
 
-    auto split_result = filter_expression->split(optimize_result.prewhere_nodes, true, true);
+    auto split_result = filter_expression.split(optimize_result.prewhere_nodes, true, true);
 
     /// This is the leak of abstraction.
     /// Splited actions may have inputs which are needed only for PREWHERE.
@@ -137,15 +138,15 @@ void optimizePrewhere(Stack & stack, QueryPlan::Nodes &)
     /// So, here we restore removed inputs for PREWHERE actions
     {
         std::unordered_set<const ActionsDAG::Node *> first_outputs(
-            split_result.first->getOutputs().begin(), split_result.first->getOutputs().end());
-        for (const auto * input : split_result.first->getInputs())
+            split_result.first.getOutputs().begin(), split_result.first.getOutputs().end());
+        for (const auto * input : split_result.first.getInputs())
         {
             if (!first_outputs.contains(input))
             {
-                split_result.first->getOutputs().push_back(input);
+                split_result.first.getOutputs().push_back(input);
                 /// Add column to second actions as input.
                 /// Do not add it to result, so it would be removed.
-                split_result.second->addInput(input->result_name, input->result_type);
+                split_result.second.addInput(input->result_name, input->result_type);
             }
         }
     }
diff --git a/src/Processors/QueryPlan/Optimizations/optimizePrimaryKeyConditionAndLimit.cpp b/src/Processors/QueryPlan/Optimizations/optimizePrimaryKeyConditionAndLimit.cpp
index 0afddede708..71a7ca327b1 100644
--- a/src/Processors/QueryPlan/Optimizations/optimizePrimaryKeyConditionAndLimit.cpp
+++ b/src/Processors/QueryPlan/Optimizations/optimizePrimaryKeyConditionAndLimit.cpp
@@ -18,16 +18,16 @@ void optimizePrimaryKeyConditionAndLimit(const Stack & stack)
     const auto & storage_prewhere_info = source_step_with_filter->getPrewhereInfo();
     if (storage_prewhere_info)
     {
-        source_step_with_filter->addFilter(ActionsDAG::clone(storage_prewhere_info->prewhere_actions), storage_prewhere_info->prewhere_column_name);
+        source_step_with_filter->addFilter(ActionsDAG::clone(&*storage_prewhere_info->prewhere_actions), storage_prewhere_info->prewhere_column_name);
         if (storage_prewhere_info->row_level_filter)
-            source_step_with_filter->addFilter(ActionsDAG::clone(storage_prewhere_info->row_level_filter), storage_prewhere_info->row_level_column_name);
+            source_step_with_filter->addFilter(ActionsDAG::clone(&*storage_prewhere_info->row_level_filter), storage_prewhere_info->row_level_column_name);
     }
 
     for (auto iter = stack.rbegin() + 1; iter != stack.rend(); ++iter)
     {
         if (auto * filter_step = typeid_cast<FilterStep *>(iter->node->step.get()))
         {
-            source_step_with_filter->addFilter(ActionsDAG::clone(filter_step->getExpression()), filter_step->getFilterColumnName());
+            source_step_with_filter->addFilter(ActionsDAG::clone(&filter_step->getExpression()), filter_step->getFilterColumnName());
         }
         else if (auto * limit_step = typeid_cast<LimitStep *>(iter->node->step.get()))
         {
diff --git a/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp b/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp
index a8bd98d7460..b3747b81215 100644
--- a/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp
+++ b/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp
@@ -170,12 +170,12 @@ static void appendFixedColumnsFromFilterExpression(const ActionsDAG::Node & filt
     }
 }
 
-static void appendExpression(ActionsDAGPtr & dag, const ActionsDAGPtr & expression)
+static void appendExpression(ActionsDAGPtr & dag, const ActionsDAG & expression)
 {
     if (dag)
-        dag->mergeInplace(std::move(*ActionsDAG::clone(expression)));
+        dag->mergeInplace(std::move(*ActionsDAG::clone(&expression)));
     else
-        dag = ActionsDAG::clone(expression);
+        dag = ActionsDAG::clone(&expression);
 }
 
 /// This function builds a common DAG which is a merge of DAGs from Filter and Expression steps chain.
@@ -193,7 +193,7 @@ void buildSortingDAG(QueryPlan::Node & node, ActionsDAGPtr & dag, FixedColumns &
             if (prewhere_info->prewhere_actions)
             {
                 //std::cerr << "====== Adding prewhere " << std::endl;
-                appendExpression(dag, prewhere_info->prewhere_actions);
+                appendExpression(dag, *prewhere_info->prewhere_actions);
                 if (const auto * filter_expression = dag->tryFindInOutputs(prewhere_info->prewhere_column_name))
                     appendFixedColumnsFromFilterExpression(*filter_expression, fixed_columns);
             }
@@ -211,7 +211,7 @@ void buildSortingDAG(QueryPlan::Node & node, ActionsDAGPtr & dag, FixedColumns &
         const auto & actions = expression->getExpression();
 
         /// Should ignore limit because arrayJoin() can reduce the number of rows in case of empty array.
-        if (actions->hasArrayJoin())
+        if (actions.hasArrayJoin())
             limit = 0;
 
         appendExpression(dag, actions);
@@ -1066,13 +1066,13 @@ size_t tryReuseStorageOrderingForWindowFunctions(QueryPlan::Node * parent_node,
     for (const auto & actions_dag : window_desc.partition_by_actions)
     {
         order_by_elements_actions.emplace_back(
-            std::make_shared<ExpressionActions>(ActionsDAG::clone(actions_dag.get()), ExpressionActionsSettings::fromContext(context, CompileExpressions::yes)));
+            std::make_shared<ExpressionActions>(std::move(*ActionsDAG::clone(actions_dag.get())), ExpressionActionsSettings::fromContext(context, CompileExpressions::yes)));
     }
 
     for (const auto & actions_dag : window_desc.order_by_actions)
     {
         order_by_elements_actions.emplace_back(
-            std::make_shared<ExpressionActions>(ActionsDAG::clone(actions_dag.get()), ExpressionActionsSettings::fromContext(context, CompileExpressions::yes)));
+            std::make_shared<ExpressionActions>(std::move(*ActionsDAG::clone(actions_dag.get())), ExpressionActionsSettings::fromContext(context, CompileExpressions::yes)));
     }
 
     auto order_optimizer = std::make_shared<ReadInOrderOptimizer>(
diff --git a/src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp b/src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp
index da057bd25c2..34e9c8aac0e 100644
--- a/src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp
+++ b/src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp
@@ -273,7 +273,7 @@ static void appendAggregateFunctions(
     }
 }
 
-ActionsDAGPtr analyzeAggregateProjection(
+std::optional<ActionsDAG> analyzeAggregateProjection(
     const AggregateProjectionInfo & info,
     const QueryDAG & query,
     const DAGIndex & query_index,
@@ -393,7 +393,7 @@ ActionsDAGPtr analyzeAggregateProjection(
     // LOG_TRACE(getLogger("optimizeUseProjections"), "Folding actions by projection");
 
     auto proj_dag = query.dag->foldActionsByProjection(new_inputs, query_key_nodes);
-    appendAggregateFunctions(*proj_dag, aggregates, *matched_aggregates);
+    appendAggregateFunctions(proj_dag, aggregates, *matched_aggregates);
     return proj_dag;
 }
 
@@ -405,7 +405,7 @@ struct AggregateProjectionCandidate : public ProjectionCandidate
 
     /// Actions which need to be applied to columns from projection
     /// in order to get all the columns required for aggregation.
-    ActionsDAGPtr dag;
+    ActionsDAG dag;
 };
 
 struct MinMaxProjectionCandidate
@@ -480,13 +480,13 @@ AggregateProjectionCandidates getAggregateProjectionCandidates(
         if (auto proj_dag = analyzeAggregateProjection(info, dag, query_index, keys, aggregates))
         {
             // LOG_TRACE(getLogger("optimizeUseProjections"), "Projection analyzed DAG {}", proj_dag->dumpDAG());
-            AggregateProjectionCandidate candidate{.info = std::move(info), .dag = std::move(proj_dag)};
+            AggregateProjectionCandidate candidate{.info = std::move(info), .dag = std::move(*proj_dag)};
 
             // LOG_TRACE(getLogger("optimizeUseProjections"), "Projection sample block {}", sample_block.dumpStructure());
             auto block = reading.getMergeTreeData().getMinMaxCountProjectionBlock(
                 metadata,
-                candidate.dag->getRequiredColumnsNames(),
-                (dag.filter_node ? dag.dag.get() : nullptr),
+                candidate.dag.getRequiredColumnsNames(),
+                (dag.filter_node ? &*dag.dag : nullptr),
                 parts,
                 max_added_blocks.get(),
                 context);
@@ -536,7 +536,7 @@ AggregateProjectionCandidates getAggregateProjectionCandidates(
             if (auto proj_dag = analyzeAggregateProjection(info, dag, query_index, keys, aggregates))
             {
                 // LOG_TRACE(getLogger("optimizeUseProjections"), "Projection analyzed DAG {}", proj_dag->dumpDAG());
-                AggregateProjectionCandidate candidate{.info = std::move(info), .dag = std::move(proj_dag)};
+                AggregateProjectionCandidate candidate{.info = std::move(info), .dag = std::move(*proj_dag)};
                 candidate.projection = projection;
                 candidates.real.emplace_back(std::move(candidate));
             }
@@ -664,7 +664,7 @@ std::optional<String> optimizeUseAggregateProjections(QueryPlan::Node & node, Qu
         /// Selecting best candidate.
         for (auto & candidate : candidates.real)
         {
-            auto required_column_names = candidate.dag->getRequiredColumnsNames();
+            auto required_column_names = candidate.dag.getRequiredColumnsNames();
 
             bool analyzed = analyzeProjectionCandidate(
                 candidate,
@@ -675,7 +675,7 @@ std::optional<String> optimizeUseAggregateProjections(QueryPlan::Node & node, Qu
                 query_info,
                 context,
                 max_added_blocks,
-                candidate.dag.get());
+                &candidate.dag);
 
             if (!analyzed)
                 continue;
@@ -765,7 +765,7 @@ std::optional<String> optimizeUseAggregateProjections(QueryPlan::Node & node, Qu
         projection_reading = reader.readFromParts(
             /* parts = */ {},
             /* alter_conversions = */ {},
-            best_candidate->dag->getRequiredColumnsNames(),
+            best_candidate->dag.getRequiredColumnsNames(),
             proj_snapshot,
             projection_query_info,
             context,
@@ -777,7 +777,7 @@ std::optional<String> optimizeUseAggregateProjections(QueryPlan::Node & node, Qu
 
         if (!projection_reading)
         {
-            auto header = proj_snapshot->getSampleBlockForColumns(best_candidate->dag->getRequiredColumnsNames());
+            auto header = proj_snapshot->getSampleBlockForColumns(best_candidate->dag.getRequiredColumnsNames());
             Pipe pipe(std::make_shared<NullSource>(std::move(header)));
             projection_reading = std::make_unique<ReadFromPreparedSource>(std::move(pipe));
         }
@@ -808,17 +808,19 @@ std::optional<String> optimizeUseAggregateProjections(QueryPlan::Node & node, Qu
     if (best_candidate)
     {
         aggregate_projection_node = &nodes.emplace_back();
+
         if (candidates.has_filter)
         {
+            const auto & result_name = best_candidate->dag.getOutputs().front()->result_name;
             aggregate_projection_node->step = std::make_unique<FilterStep>(
                 projection_reading_node.step->getOutputStream(),
-                best_candidate->dag,
-                best_candidate->dag->getOutputs().front()->result_name,
+                std::move(best_candidate->dag),
+                result_name,
                 true);
         }
         else
             aggregate_projection_node->step
-                = std::make_unique<ExpressionStep>(projection_reading_node.step->getOutputStream(), best_candidate->dag);
+                = std::make_unique<ExpressionStep>(projection_reading_node.step->getOutputStream(), std::move(best_candidate->dag));
 
         aggregate_projection_node->children.push_back(&projection_reading_node);
     }
diff --git a/src/Processors/QueryPlan/Optimizations/optimizeUseNormalProjection.cpp b/src/Processors/QueryPlan/Optimizations/optimizeUseNormalProjection.cpp
index c7e96d66817..c0af178f08e 100644
--- a/src/Processors/QueryPlan/Optimizations/optimizeUseNormalProjection.cpp
+++ b/src/Processors/QueryPlan/Optimizations/optimizeUseNormalProjection.cpp
@@ -23,7 +23,7 @@ struct NormalProjectionCandidate : public ProjectionCandidate
 {
 };
 
-static ActionsDAGPtr makeMaterializingDAG(const Block & proj_header, const Block main_header)
+static std::optional<ActionsDAG> makeMaterializingDAG(const Block & proj_header, const Block main_header)
 {
     /// Materialize constants in case we don't have it in output header.
     /// This may happen e.g. if we have PREWHERE.
@@ -31,7 +31,7 @@ static ActionsDAGPtr makeMaterializingDAG(const Block & proj_header, const Block
     size_t num_columns = main_header.columns();
     /// This is a error; will have block structure mismatch later.
     if (proj_header.columns() != num_columns)
-        return nullptr;
+        return {};
 
     std::vector<size_t> const_positions;
     for (size_t i = 0; i < num_columns; ++i)
@@ -45,17 +45,17 @@ static ActionsDAGPtr makeMaterializingDAG(const Block & proj_header, const Block
     }
 
     if (const_positions.empty())
-        return nullptr;
+        return {};
 
-    ActionsDAGPtr dag = std::make_unique<ActionsDAG>();
-    auto & outputs = dag->getOutputs();
+    ActionsDAG dag;
+    auto & outputs = dag.getOutputs();
     for (const auto & col : proj_header.getColumnsWithTypeAndName())
-        outputs.push_back(&dag->addInput(col));
+        outputs.push_back(&dag.addInput(col));
 
     for (auto pos : const_positions)
     {
         auto & output = outputs[pos];
-        output = &dag->materializeNode(*output);
+        output = &dag.materializeNode(*output);
     }
 
     return dag;
@@ -172,7 +172,7 @@ std::optional<String> optimizeUseNormalProjections(Stack & stack, QueryPlan::Nod
             query_info,
             context,
             max_added_blocks,
-            query.filter_node ? query.dag.get() : nullptr);
+            query.filter_node ? &*query.dag : nullptr);
 
         if (!analyzed)
             continue;
@@ -242,14 +242,14 @@ std::optional<String> optimizeUseNormalProjections(Stack & stack, QueryPlan::Nod
         {
             expr_or_filter_node.step = std::make_unique<FilterStep>(
                 projection_reading_node.step->getOutputStream(),
-                query.dag,
+                std::move(*query.dag),
                 query.filter_node->result_name,
                 true);
         }
         else
             expr_or_filter_node.step = std::make_unique<ExpressionStep>(
                 projection_reading_node.step->getOutputStream(),
-                query.dag);
+                std::move(*query.dag));
 
         expr_or_filter_node.children.push_back(&projection_reading_node);
         next_node = &expr_or_filter_node;
@@ -267,7 +267,7 @@ std::optional<String> optimizeUseNormalProjections(Stack & stack, QueryPlan::Nod
 
         if (auto materializing = makeMaterializingDAG(proj_stream->header, main_stream.header))
         {
-            auto converting = std::make_unique<ExpressionStep>(*proj_stream, materializing);
+            auto converting = std::make_unique<ExpressionStep>(*proj_stream, std::move(*materializing));
             proj_stream = &converting->getOutputStream();
             auto & expr_node = nodes.emplace_back();
             expr_node.step = std::move(converting);
diff --git a/src/Processors/QueryPlan/Optimizations/projectionsCommon.cpp b/src/Processors/QueryPlan/Optimizations/projectionsCommon.cpp
index 0e2ad96a419..fb2e6c2096e 100644
--- a/src/Processors/QueryPlan/Optimizations/projectionsCommon.cpp
+++ b/src/Processors/QueryPlan/Optimizations/projectionsCommon.cpp
@@ -64,12 +64,12 @@ std::shared_ptr<PartitionIdToMaxBlock> getMaxAddedBlocks(ReadFromMergeTree * rea
     return {};
 }
 
-void QueryDAG::appendExpression(const ActionsDAGPtr & expression)
+void QueryDAG::appendExpression(const ActionsDAG & expression)
 {
     if (dag)
-        dag->mergeInplace(std::move(*ActionsDAG::clone(expression)));
+        dag->mergeInplace(std::move(*ActionsDAG::clone(&expression)));
     else
-        dag = ActionsDAG::clone(expression);
+        dag = std::move(*ActionsDAG::clone(&expression));
 }
 
 const ActionsDAG::Node * findInOutputs(ActionsDAG & dag, const std::string & name, bool remove)
@@ -120,7 +120,7 @@ bool QueryDAG::buildImpl(QueryPlan::Node & node, ActionsDAG::NodeRawConstPtrs &
         {
             if (prewhere_info->row_level_filter)
             {
-                appendExpression(prewhere_info->row_level_filter);
+                appendExpression(*prewhere_info->row_level_filter);
                 if (const auto * filter_expression = findInOutputs(*dag, prewhere_info->row_level_column_name, false))
                     filter_nodes.push_back(filter_expression);
                 else
@@ -129,7 +129,7 @@ bool QueryDAG::buildImpl(QueryPlan::Node & node, ActionsDAG::NodeRawConstPtrs &
 
             if (prewhere_info->prewhere_actions)
             {
-                appendExpression(prewhere_info->prewhere_actions);
+                appendExpression(*prewhere_info->prewhere_actions);
                 if (const auto * filter_expression
                     = findInOutputs(*dag, prewhere_info->prewhere_column_name, prewhere_info->remove_prewhere_column))
                     filter_nodes.push_back(filter_expression);
@@ -149,7 +149,7 @@ bool QueryDAG::buildImpl(QueryPlan::Node & node, ActionsDAG::NodeRawConstPtrs &
     if (auto * expression = typeid_cast<ExpressionStep *>(step))
     {
         const auto & actions = expression->getExpression();
-        if (actions->hasArrayJoin())
+        if (actions.hasArrayJoin())
             return false;
 
         appendExpression(actions);
@@ -159,7 +159,7 @@ bool QueryDAG::buildImpl(QueryPlan::Node & node, ActionsDAG::NodeRawConstPtrs &
     if (auto * filter = typeid_cast<FilterStep *>(step))
     {
         const auto & actions = filter->getExpression();
-        if (actions->hasArrayJoin())
+        if (actions.hasArrayJoin())
             return false;
 
         appendExpression(actions);
diff --git a/src/Processors/QueryPlan/Optimizations/projectionsCommon.h b/src/Processors/QueryPlan/Optimizations/projectionsCommon.h
index 59ad3a43b97..ee0dfddc326 100644
--- a/src/Processors/QueryPlan/Optimizations/projectionsCommon.h
+++ b/src/Processors/QueryPlan/Optimizations/projectionsCommon.h
@@ -25,14 +25,14 @@ std::shared_ptr<PartitionIdToMaxBlock> getMaxAddedBlocks(ReadFromMergeTree * rea
 /// Additionally, for all the Filter steps, we collect filter conditions into filter_nodes.
 struct QueryDAG
 {
-    ActionsDAGPtr dag;
+    std::optional<ActionsDAG> dag;
     const ActionsDAG::Node * filter_node = nullptr;
 
     bool build(QueryPlan::Node & node);
 
 private:
     bool buildImpl(QueryPlan::Node & node, ActionsDAG::NodeRawConstPtrs & filter_nodes);
-    void appendExpression(const ActionsDAGPtr & expression);
+    void appendExpression(const ActionsDAG & expression);
 };
 
 struct ProjectionCandidate
diff --git a/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp b/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp
index 81a8a537830..d0acd8221d4 100644
--- a/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp
+++ b/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp
@@ -132,10 +132,10 @@ namespace
             return true;
 
         if (const auto * const expr = typeid_cast<const ExpressionStep *>(step); expr)
-            return !expr->getExpression()->hasArrayJoin();
+            return !expr->getExpression().hasArrayJoin();
 
         if (const auto * const filter = typeid_cast<const FilterStep *>(step); filter)
-            return !filter->getExpression()->hasArrayJoin();
+            return !filter->getExpression().hasArrayJoin();
 
         if (typeid_cast<const LimitStep *>(step) || typeid_cast<const LimitByStep *>(step) || typeid_cast<const SortingStep *>(step)
             || typeid_cast<const WindowStep *>(step))
@@ -183,9 +183,9 @@ namespace
             }
 
             if (const auto * const expr = typeid_cast<const ExpressionStep *>(current_step); expr)
-                dag_stack.push_back(expr->getExpression().get());
+                dag_stack.push_back(&expr->getExpression());
             else if (const auto * const filter = typeid_cast<const FilterStep *>(current_step); filter)
-                dag_stack.push_back(filter->getExpression().get());
+                dag_stack.push_back(&filter->getExpression());
 
             node = node->children.front();
             if (inner_distinct_step = typeid_cast<DistinctStep *>(node->step.get()); inner_distinct_step)
@@ -236,9 +236,9 @@ namespace
             }
 
             if (const auto * const expr = typeid_cast<const ExpressionStep *>(current_step); expr)
-                dag_stack.push_back(expr->getExpression().get());
+                dag_stack.push_back(&expr->getExpression());
             else if (const auto * const filter = typeid_cast<const FilterStep *>(current_step); filter)
-                dag_stack.push_back(filter->getExpression().get());
+                dag_stack.push_back(&filter->getExpression());
 
             node = node->children.front();
             inner_distinct_step = typeid_cast<DistinctStep *>(node->step.get());
diff --git a/src/Processors/QueryPlan/Optimizations/removeRedundantSorting.cpp b/src/Processors/QueryPlan/Optimizations/removeRedundantSorting.cpp
index 632eba6ab5f..7cac7bee6ec 100644
--- a/src/Processors/QueryPlan/Optimizations/removeRedundantSorting.cpp
+++ b/src/Processors/QueryPlan/Optimizations/removeRedundantSorting.cpp
@@ -213,12 +213,12 @@ private:
             logStep("checking for stateful function", node);
             if (const auto * expr = typeid_cast<const ExpressionStep *>(step); expr)
             {
-                if (expr->getExpression()->hasStatefulFunctions())
+                if (expr->getExpression().hasStatefulFunctions())
                     return false;
             }
             else if (const auto * filter = typeid_cast<const FilterStep *>(step); filter)
             {
-                if (filter->getExpression()->hasStatefulFunctions())
+                if (filter->getExpression().hasStatefulFunctions())
                     return false;
             }
             else
diff --git a/src/Processors/QueryPlan/Optimizations/splitFilter.cpp b/src/Processors/QueryPlan/Optimizations/splitFilter.cpp
index 561ad7302c6..6aed57634b0 100644
--- a/src/Processors/QueryPlan/Optimizations/splitFilter.cpp
+++ b/src/Processors/QueryPlan/Optimizations/splitFilter.cpp
@@ -17,13 +17,13 @@ size_t trySplitFilter(QueryPlan::Node * node, QueryPlan::Nodes & nodes)
     const std::string & filter_column_name = filter_step->getFilterColumnName();
 
     /// Do not split if there are function like runningDifference.
-    if (expr->hasStatefulFunctions())
+    if (expr.hasStatefulFunctions())
         return 0;
 
     bool filter_name_clashs_with_input = false;
     if (filter_step->removesFilterColumn())
     {
-        for (const auto * input : expr->getInputs())
+        for (const auto * input : expr.getInputs())
         {
             if (input->result_name == filter_column_name)
             {
@@ -33,14 +33,14 @@ size_t trySplitFilter(QueryPlan::Node * node, QueryPlan::Nodes & nodes)
         }
     }
 
-    auto split = expr->splitActionsForFilter(filter_column_name);
+    auto split = expr.splitActionsForFilter(filter_column_name);
 
-    if (split.second->trivial())
+    if (split.second.trivial())
         return 0;
 
     bool remove_filter = false;
     if (filter_step->removesFilterColumn())
-        remove_filter = split.second->removeUnusedResult(filter_column_name);
+        remove_filter = split.second.removeUnusedResult(filter_column_name);
 
     auto description = filter_step->getStepDescription();
 
@@ -53,11 +53,11 @@ size_t trySplitFilter(QueryPlan::Node * node, QueryPlan::Nodes & nodes)
     {
         split_filter_name = "__split_filter";
 
-        for (auto & filter_output : split.first->getOutputs())
+        for (auto & filter_output : split.first.getOutputs())
         {
             if (filter_output->result_name == filter_column_name)
             {
-                filter_output = &split.first->addAlias(*filter_output, split_filter_name);
+                filter_output = &split.first.addAlias(*filter_output, split_filter_name);
                 break;
             }
         }
diff --git a/src/Processors/QueryPlan/Optimizations/useDataParallelAggregation.cpp b/src/Processors/QueryPlan/Optimizations/useDataParallelAggregation.cpp
index 124cb735d5a..7e0260c0040 100644
--- a/src/Processors/QueryPlan/Optimizations/useDataParallelAggregation.cpp
+++ b/src/Processors/QueryPlan/Optimizations/useDataParallelAggregation.cpp
@@ -146,16 +146,16 @@ bool allOutputsDependsOnlyOnAllowedNodes(
 /// 3. We match partition key actions with group by key actions to find col1', ..., coln' in partition key actions.
 /// 4. We check that partition key is indeed a deterministic function of col1', ..., coln'.
 bool isPartitionKeySuitsGroupByKey(
-    const ReadFromMergeTree & reading, const ActionsDAGPtr & group_by_actions, const AggregatingStep & aggregating)
+    const ReadFromMergeTree & reading, const ActionsDAG & group_by_actions, const AggregatingStep & aggregating)
 {
     if (aggregating.isGroupingSets())
         return false;
 
-    if (group_by_actions->hasArrayJoin() || group_by_actions->hasStatefulFunctions() || group_by_actions->hasNonDeterministic())
+    if (group_by_actions.hasArrayJoin() || group_by_actions.hasStatefulFunctions() || group_by_actions.hasNonDeterministic())
         return false;
 
     /// We are interested only in calculations required to obtain group by keys (and not aggregate function arguments for example).
-    auto key_nodes = group_by_actions->findInOutpus(aggregating.getParams().keys);
+    auto key_nodes = group_by_actions.findInOutpus(aggregating.getParams().keys);
     auto group_by_key_actions = ActionsDAG::cloneSubDAG(key_nodes, /*remove_aliases=*/ true);
 
     const auto & gb_key_required_columns = group_by_key_actions->getRequiredColumnsNames();
diff --git a/src/Processors/QueryPlan/PartsSplitter.cpp b/src/Processors/QueryPlan/PartsSplitter.cpp
index ed4b1906635..a12fce95b10 100644
--- a/src/Processors/QueryPlan/PartsSplitter.cpp
+++ b/src/Processors/QueryPlan/PartsSplitter.cpp
@@ -943,7 +943,7 @@ SplitPartsWithRangesByPrimaryKeyResult splitPartsWithRangesByPrimaryKey(
 
         auto syntax_result = TreeRewriter(context).analyze(filter_function, primary_key.expression->getRequiredColumnsWithTypes());
         auto actions = ExpressionAnalyzer(filter_function, syntax_result, context).getActionsDAG(false);
-        reorderColumns(*actions, result.merging_pipes[i].getHeader(), filter_function->getColumnName());
+        reorderColumns(actions, result.merging_pipes[i].getHeader(), filter_function->getColumnName());
         ExpressionActionsPtr expression_actions = std::make_shared<ExpressionActions>(std::move(actions));
         auto description = fmt::format(
             "filter values in ({}, {}]", i ? ::toString(borders[i - 1]) : "-inf", i < borders.size() ? ::toString(borders[i]) : "+inf");
diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp
index eca3cc54ce9..bc878e7ee49 100644
--- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp
+++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp
@@ -799,7 +799,7 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreams(RangesInDataParts && parts_
                 info.use_uncompressed_cache);
         };
 
-        auto sorting_expr = std::make_shared<ExpressionActions>(ActionsDAG::clone(&metadata_for_reading->getSortingKey().expression->getActionsDAG()));
+        auto sorting_expr = std::make_shared<ExpressionActions>(std::move(*ActionsDAG::clone(&metadata_for_reading->getSortingKey().expression->getActionsDAG())));
 
         SplitPartsWithRangesByPrimaryKeyResult split_ranges_result = splitPartsWithRangesByPrimaryKey(
             metadata_for_reading->getPrimaryKey(),
@@ -848,16 +848,16 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreams(RangesInDataParts && parts_
         info.use_uncompressed_cache);
 }
 
-static ActionsDAGPtr createProjection(const Block & header)
+static ActionsDAG createProjection(const Block & header)
 {
-    return std::make_unique<ActionsDAG>(header.getNamesAndTypesList());
+    return ActionsDAG(header.getNamesAndTypesList());
 }
 
 Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsWithOrder(
     RangesInDataParts && parts_with_ranges,
     size_t num_streams,
     const Names & column_names,
-    ActionsDAGPtr & out_projection,
+    std::optional<ActionsDAG> & out_projection,
     const InputOrderInfoPtr & input_order_info)
 {
     const auto & settings = context->getSettingsRef();
@@ -1171,7 +1171,7 @@ bool ReadFromMergeTree::doNotMergePartsAcrossPartitionsFinal() const
 }
 
 Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal(
-    RangesInDataParts && parts_with_ranges, size_t num_streams, const Names & origin_column_names, const Names & column_names, ActionsDAGPtr & out_projection)
+    RangesInDataParts && parts_with_ranges, size_t num_streams, const Names & origin_column_names, const Names & column_names, std::optional<ActionsDAG> & out_projection)
 {
     const auto & settings = context->getSettingsRef();
     const auto & data_settings = data.getSettings();
@@ -1212,7 +1212,7 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal(
     /// we will store lonely parts with level > 0 to use parallel select on them.
     RangesInDataParts non_intersecting_parts_by_primary_key;
 
-    auto sorting_expr = std::make_shared<ExpressionActions>(ActionsDAG::clone(&metadata_for_reading->getSortingKey().expression->getActionsDAG()));
+    auto sorting_expr = std::make_shared<ExpressionActions>(std::move(*ActionsDAG::clone(&metadata_for_reading->getSortingKey().expression->getActionsDAG())));
 
     if (prewhere_info)
     {
@@ -1333,7 +1333,7 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal(
 
     if (!merging_pipes.empty() && !no_merging_pipes.empty())
     {
-        out_projection = nullptr; /// We do projection here
+        out_projection = {}; /// We do projection here
         Pipes pipes;
         pipes.resize(2);
         pipes[0] = Pipe::unitePipes(std::move(merging_pipes));
@@ -1519,7 +1519,8 @@ void ReadFromMergeTree::applyFilters(ActionDAGNodes added_filter_nodes)
         /// (1) SourceStepWithFilter::filter_nodes, (2) query_info.filter_actions_dag. Make sure there are consistent.
         /// TODO: Get rid of filter_actions_dag in query_info after we move analysis of
         /// parallel replicas and unused shards into optimization, similar to projection analysis.
-        query_info.filter_actions_dag = std::move(filter_actions_dag);
+        if (filter_actions_dag)
+            query_info.filter_actions_dag = std::make_shared<const ActionsDAG>(std::move(*filter_actions_dag));
 
         buildIndexes(
             indexes,
@@ -1833,7 +1834,7 @@ bool ReadFromMergeTree::isQueryWithSampling() const
 }
 
 Pipe ReadFromMergeTree::spreadMarkRanges(
-    RangesInDataParts && parts_with_ranges, size_t num_streams, AnalysisResult & result, ActionsDAGPtr & result_projection)
+    RangesInDataParts && parts_with_ranges, size_t num_streams, AnalysisResult & result, std::optional<ActionsDAG> & result_projection)
 {
     const bool final = isQueryWithFinal();
     Names column_names_to_read = result.column_names_to_read;
@@ -1894,7 +1895,7 @@ Pipe ReadFromMergeTree::spreadMarkRanges(
     }
 }
 
-Pipe ReadFromMergeTree::groupStreamsByPartition(AnalysisResult & result, ActionsDAGPtr & result_projection)
+Pipe ReadFromMergeTree::groupStreamsByPartition(AnalysisResult & result, std::optional<ActionsDAG> & result_projection)
 {
     auto && parts_with_ranges = std::move(result.parts_with_ranges);
 
@@ -1983,7 +1984,7 @@ void ReadFromMergeTree::initializePipeline(QueryPipelineBuilder & pipeline, cons
     /// Projection, that needed to drop columns, which have appeared by execution
     /// of some extra expressions, and to allow execute the same expressions later.
     /// NOTE: It may lead to double computation of expressions.
-    ActionsDAGPtr result_projection;
+    std::optional<ActionsDAG> result_projection;
 
     Pipe pipe = output_each_partition_through_separate_port
         ? groupStreamsByPartition(result, result_projection)
@@ -2000,7 +2001,7 @@ void ReadFromMergeTree::initializePipeline(QueryPipelineBuilder & pipeline, cons
 
     if (result.sampling.use_sampling)
     {
-        auto sampling_actions = std::make_shared<ExpressionActions>(ActionsDAG::clone(result.sampling.filter_expression.get()));
+        auto sampling_actions = std::make_shared<ExpressionActions>(std::move(*ActionsDAG::clone(result.sampling.filter_expression.get())));
         pipe.addSimpleTransform([&](const Block & header)
         {
             return std::make_shared<FilterTransform>(
@@ -2013,12 +2014,12 @@ void ReadFromMergeTree::initializePipeline(QueryPipelineBuilder & pipeline, cons
 
     Block cur_header = pipe.getHeader();
 
-    auto append_actions = [&result_projection](ActionsDAGPtr actions)
+    auto append_actions = [&result_projection](ActionsDAG actions)
     {
         if (!result_projection)
             result_projection = std::move(actions);
         else
-            result_projection = ActionsDAG::merge(std::move(*result_projection), std::move(*actions));
+            result_projection = ActionsDAG::merge(std::move(*result_projection), std::move(actions));
     };
 
     if (result_projection)
@@ -2038,7 +2039,7 @@ void ReadFromMergeTree::initializePipeline(QueryPipelineBuilder & pipeline, cons
 
     if (result_projection)
     {
-        auto projection_actions = std::make_shared<ExpressionActions>(ActionsDAG::clone(result_projection));
+        auto projection_actions = std::make_shared<ExpressionActions>(std::move(*result_projection));
         pipe.addSimpleTransform([&](const Block & header)
         {
             return std::make_shared<ExpressionTransform>(header, projection_actions);
@@ -2133,7 +2134,7 @@ void ReadFromMergeTree::describeActions(FormatSettings & format_settings) const
                format_settings.out << " (removed)";
             format_settings.out << '\n';
 
-            auto expression = std::make_shared<ExpressionActions>(ActionsDAG::clone(prewhere_info->prewhere_actions));
+            auto expression = std::make_shared<ExpressionActions>(std::move(*ActionsDAG::clone(&*prewhere_info->prewhere_actions)));
             expression->describeActions(format_settings.out, prefix);
         }
 
@@ -2142,7 +2143,7 @@ void ReadFromMergeTree::describeActions(FormatSettings & format_settings) const
             format_settings.out << prefix << "Row level filter" << '\n';
             format_settings.out << prefix << "Row level filter column: " << prewhere_info->row_level_column_name << '\n';
 
-            auto expression = std::make_shared<ExpressionActions>(ActionsDAG::clone(prewhere_info->row_level_filter));
+            auto expression = std::make_shared<ExpressionActions>(std::move(*ActionsDAG::clone(&*prewhere_info->row_level_filter)));
             expression->describeActions(format_settings.out, prefix);
         }
     }
@@ -2168,7 +2169,7 @@ void ReadFromMergeTree::describeActions(JSONBuilder::JSONMap & map) const
             std::unique_ptr<JSONBuilder::JSONMap> prewhere_filter_map = std::make_unique<JSONBuilder::JSONMap>();
             prewhere_filter_map->add("Prewhere filter column", prewhere_info->prewhere_column_name);
             prewhere_filter_map->add("Prewhere filter remove filter column", prewhere_info->remove_prewhere_column);
-            auto expression = std::make_shared<ExpressionActions>(ActionsDAG::clone(prewhere_info->prewhere_actions));
+            auto expression = std::make_shared<ExpressionActions>(std::move(*ActionsDAG::clone(&*prewhere_info->prewhere_actions)));
             prewhere_filter_map->add("Prewhere filter expression", expression->toTree());
 
             prewhere_info_map->add("Prewhere filter", std::move(prewhere_filter_map));
@@ -2178,7 +2179,7 @@ void ReadFromMergeTree::describeActions(JSONBuilder::JSONMap & map) const
         {
             std::unique_ptr<JSONBuilder::JSONMap> row_level_filter_map = std::make_unique<JSONBuilder::JSONMap>();
             row_level_filter_map->add("Row level filter column", prewhere_info->row_level_column_name);
-            auto expression = std::make_shared<ExpressionActions>(ActionsDAG::clone(prewhere_info->row_level_filter));
+            auto expression = std::make_shared<ExpressionActions>(std::move(*ActionsDAG::clone(&*prewhere_info->row_level_filter)));
             row_level_filter_map->add("Row level filter expression", expression->toTree());
 
             prewhere_info_map->add("Row level filter", std::move(row_level_filter_map));
diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.h b/src/Processors/QueryPlan/ReadFromMergeTree.h
index e32507e1f22..a12f53924c3 100644
--- a/src/Processors/QueryPlan/ReadFromMergeTree.h
+++ b/src/Processors/QueryPlan/ReadFromMergeTree.h
@@ -243,9 +243,9 @@ private:
     Pipe readFromPoolParallelReplicas(RangesInDataParts parts_with_range, Names required_columns, PoolSettings pool_settings);
     Pipe readInOrder(RangesInDataParts parts_with_ranges, Names required_columns, PoolSettings pool_settings, ReadType read_type, UInt64 limit);
 
-    Pipe spreadMarkRanges(RangesInDataParts && parts_with_ranges, size_t num_streams, AnalysisResult & result, ActionsDAGPtr & result_projection);
+    Pipe spreadMarkRanges(RangesInDataParts && parts_with_ranges, size_t num_streams, AnalysisResult & result, std::optional<ActionsDAG> & result_projection);
 
-    Pipe groupStreamsByPartition(AnalysisResult & result, ActionsDAGPtr & result_projection);
+    Pipe groupStreamsByPartition(AnalysisResult & result, std::optional<ActionsDAG> & result_projection);
 
     Pipe spreadMarkRangesAmongStreams(RangesInDataParts && parts_with_ranges, size_t num_streams, const Names & column_names);
 
@@ -253,13 +253,13 @@ private:
         RangesInDataParts && parts_with_ranges,
         size_t num_streams,
         const Names & column_names,
-        ActionsDAGPtr & out_projection,
+        std::optional<ActionsDAG> & out_projection,
         const InputOrderInfoPtr & input_order_info);
 
     bool doNotMergePartsAcrossPartitionsFinal() const;
 
     Pipe spreadMarkRangesAmongStreamsFinal(
-        RangesInDataParts && parts, size_t num_streams, const Names & origin_column_names, const Names & column_names, ActionsDAGPtr & out_projection);
+        RangesInDataParts && parts, size_t num_streams, const Names & origin_column_names, const Names & column_names, std::optional<ActionsDAG> & out_projection);
 
     ReadFromMergeTree::AnalysisResult getAnalysisResult() const;
 
diff --git a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp
index 90fe609a17d..ca98f7c2110 100644
--- a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp
+++ b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp
@@ -441,7 +441,7 @@ Pipe ReadFromSystemNumbersStep::makePipe()
     chassert(numbers_storage.step != UInt64{0});
 
     /// Build rpn of query filters
-    KeyCondition condition(filter_actions_dag.get(), context, column_names, key_expression);
+    KeyCondition condition(filter_actions_dag ? &*filter_actions_dag : nullptr, context, column_names, key_expression);
 
     if (condition.extractPlainRanges(ranges))
     {
diff --git a/src/Processors/QueryPlan/SourceStepWithFilter.cpp b/src/Processors/QueryPlan/SourceStepWithFilter.cpp
index 79b225e7f93..55c9b5e442e 100644
--- a/src/Processors/QueryPlan/SourceStepWithFilter.cpp
+++ b/src/Processors/QueryPlan/SourceStepWithFilter.cpp
@@ -110,7 +110,7 @@ void SourceStepWithFilter::describeActions(FormatSettings & format_settings) con
                 format_settings.out << " (removed)";
             format_settings.out << '\n';
 
-            auto expression = std::make_shared<ExpressionActions>(ActionsDAG::clone(prewhere_info->prewhere_actions));
+            auto expression = std::make_shared<ExpressionActions>(std::move(*ActionsDAG::clone(&*prewhere_info->prewhere_actions)));
             expression->describeActions(format_settings.out, prefix);
         }
 
@@ -119,7 +119,7 @@ void SourceStepWithFilter::describeActions(FormatSettings & format_settings) con
             format_settings.out << prefix << "Row level filter" << '\n';
             format_settings.out << prefix << "Row level filter column: " << prewhere_info->row_level_column_name << '\n';
 
-            auto expression = std::make_shared<ExpressionActions>(ActionsDAG::clone(prewhere_info->row_level_filter));
+            auto expression = std::make_shared<ExpressionActions>(std::move(*ActionsDAG::clone(&*prewhere_info->row_level_filter)));
             expression->describeActions(format_settings.out, prefix);
         }
     }
@@ -137,7 +137,7 @@ void SourceStepWithFilter::describeActions(JSONBuilder::JSONMap & map) const
             std::unique_ptr<JSONBuilder::JSONMap> prewhere_filter_map = std::make_unique<JSONBuilder::JSONMap>();
             prewhere_filter_map->add("Prewhere filter column", prewhere_info->prewhere_column_name);
             prewhere_filter_map->add("Prewhere filter remove filter column", prewhere_info->remove_prewhere_column);
-            auto expression = std::make_shared<ExpressionActions>(ActionsDAG::clone(prewhere_info->prewhere_actions));
+            auto expression = std::make_shared<ExpressionActions>(std::move(*ActionsDAG::clone(&*prewhere_info->prewhere_actions)));
             prewhere_filter_map->add("Prewhere filter expression", expression->toTree());
 
             prewhere_info_map->add("Prewhere filter", std::move(prewhere_filter_map));
@@ -147,7 +147,7 @@ void SourceStepWithFilter::describeActions(JSONBuilder::JSONMap & map) const
         {
             std::unique_ptr<JSONBuilder::JSONMap> row_level_filter_map = std::make_unique<JSONBuilder::JSONMap>();
             row_level_filter_map->add("Row level filter column", prewhere_info->row_level_column_name);
-            auto expression = std::make_shared<ExpressionActions>(ActionsDAG::clone(prewhere_info->row_level_filter));
+            auto expression = std::make_shared<ExpressionActions>(std::move(*ActionsDAG::clone(&*prewhere_info->row_level_filter)));
             row_level_filter_map->add("Row level filter expression", expression->toTree());
 
             prewhere_info_map->add("Row level filter", std::move(row_level_filter_map));
diff --git a/src/Processors/QueryPlan/SourceStepWithFilter.h b/src/Processors/QueryPlan/SourceStepWithFilter.h
index 91b62efa860..f7a030c0628 100644
--- a/src/Processors/QueryPlan/SourceStepWithFilter.h
+++ b/src/Processors/QueryPlan/SourceStepWithFilter.h
@@ -33,8 +33,8 @@ public:
     {
     }
 
-    const ActionsDAGPtr & getFilterActionsDAG() const { return filter_actions_dag; }
-    ActionsDAGPtr detachFilterActionsDAG() { return std::move(filter_actions_dag); }
+    const std::optional<ActionsDAG> & getFilterActionsDAG() const { return filter_actions_dag; }
+    std::optional<ActionsDAG> detachFilterActionsDAG() { return std::move(filter_actions_dag); }
 
     const SelectQueryInfo & getQueryInfo() const { return query_info; }
     const PrewhereInfoPtr & getPrewhereInfo() const { return prewhere_info; }
@@ -81,7 +81,7 @@ protected:
     ContextPtr context;
     std::optional<size_t> limit;
 
-    ActionsDAGPtr filter_actions_dag;
+    std::optional<ActionsDAG> filter_actions_dag;
 
 private:
     /// Will be cleared after applyFilters() is called.
diff --git a/src/Processors/QueryPlan/TotalsHavingStep.cpp b/src/Processors/QueryPlan/TotalsHavingStep.cpp
index 19632b1862f..4aa4f10ac86 100644
--- a/src/Processors/QueryPlan/TotalsHavingStep.cpp
+++ b/src/Processors/QueryPlan/TotalsHavingStep.cpp
@@ -28,7 +28,7 @@ TotalsHavingStep::TotalsHavingStep(
     const DataStream & input_stream_,
     const AggregateDescriptions & aggregates_,
     bool overflow_row_,
-    const ActionsDAGPtr & actions_dag_,
+    std::optional<ActionsDAG> actions_dag_,
     const std::string & filter_column_,
     bool remove_filter_,
     TotalsMode totals_mode_,
@@ -38,7 +38,7 @@ TotalsHavingStep::TotalsHavingStep(
         input_stream_,
         TotalsHavingTransform::transformHeader(
             input_stream_.header,
-            actions_dag_.get(),
+            actions_dag_ ? &*actions_dag_ : nullptr,
             filter_column_,
             remove_filter_,
             final_,
@@ -46,7 +46,7 @@ TotalsHavingStep::TotalsHavingStep(
         getTraits(!filter_column_.empty()))
     , aggregates(aggregates_)
     , overflow_row(overflow_row_)
-    , actions_dag(ActionsDAG::clone(actions_dag_))
+    , actions_dag(std::move(actions_dag_))
     , filter_column_name(filter_column_)
     , remove_filter(remove_filter_)
     , totals_mode(totals_mode_)
@@ -57,7 +57,7 @@ TotalsHavingStep::TotalsHavingStep(
 
 void TotalsHavingStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings & settings)
 {
-    auto expression_actions = actions_dag ? std::make_shared<ExpressionActions>(ActionsDAG::clone(actions_dag), settings.getActionsSettings()) : nullptr;
+    auto expression_actions = actions_dag ? std::make_shared<ExpressionActions>(std::move(*actions_dag), settings.getActionsSettings()) : nullptr;
 
     auto totals_having = std::make_shared<TotalsHavingTransform>(
         pipeline.getHeader(),
@@ -100,7 +100,7 @@ void TotalsHavingStep::describeActions(FormatSettings & settings) const
     if (actions_dag)
     {
         bool first = true;
-        auto expression = std::make_shared<ExpressionActions>(ActionsDAG::clone(actions_dag));
+        auto expression = std::make_shared<ExpressionActions>(std::move(*ActionsDAG::clone(getActions())));
         for (const auto & action : expression->getActions())
         {
             settings.out << prefix << (first ? "Actions: "
@@ -117,7 +117,7 @@ void TotalsHavingStep::describeActions(JSONBuilder::JSONMap & map) const
     if (actions_dag)
     {
         map.add("Filter column", filter_column_name);
-        auto expression = std::make_shared<ExpressionActions>(ActionsDAG::clone(actions_dag));
+        auto expression = std::make_shared<ExpressionActions>(std::move(*ActionsDAG::clone(getActions())));
         map.add("Expression", expression->toTree());
     }
 }
@@ -128,7 +128,7 @@ void TotalsHavingStep::updateOutputStream()
         input_streams.front(),
         TotalsHavingTransform::transformHeader(
             input_streams.front().header,
-            actions_dag.get(),
+            getActions(),
             filter_column_name,
             remove_filter,
             final,
diff --git a/src/Processors/QueryPlan/TotalsHavingStep.h b/src/Processors/QueryPlan/TotalsHavingStep.h
index 52ef5437701..927b8d99de3 100644
--- a/src/Processors/QueryPlan/TotalsHavingStep.h
+++ b/src/Processors/QueryPlan/TotalsHavingStep.h
@@ -1,6 +1,7 @@
 #pragma once
 #include <Processors/QueryPlan/ITransformingStep.h>
 #include <Processors/Transforms/finalizeChunk.h>
+#include <Interpreters/ActionsDAG.h>
 
 namespace DB
 {
@@ -18,7 +19,7 @@ public:
         const DataStream & input_stream_,
         const AggregateDescriptions & aggregates_,
         bool overflow_row_,
-        const ActionsDAGPtr & actions_dag_,
+        std::optional<ActionsDAG> actions_dag_,
         const std::string & filter_column_,
         bool remove_filter_,
         TotalsMode totals_mode_,
@@ -32,7 +33,7 @@ public:
     void describeActions(JSONBuilder::JSONMap & map) const override;
     void describeActions(FormatSettings & settings) const override;
 
-    const ActionsDAGPtr & getActions() const { return actions_dag; }
+    const ActionsDAG * getActions() const { return actions_dag ? &*actions_dag : nullptr; }
 
 private:
     void updateOutputStream() override;
@@ -40,7 +41,7 @@ private:
     const AggregateDescriptions aggregates;
 
     bool overflow_row;
-    ActionsDAGPtr actions_dag;
+    std::optional<ActionsDAG> actions_dag;
     String filter_column_name;
     bool remove_filter;
     TotalsMode totals_mode;
diff --git a/src/Processors/SourceWithKeyCondition.h b/src/Processors/SourceWithKeyCondition.h
index fcf576637ff..cfd3eb236b7 100644
--- a/src/Processors/SourceWithKeyCondition.h
+++ b/src/Processors/SourceWithKeyCondition.h
@@ -16,13 +16,13 @@ protected:
     /// Represents pushed down filters in source
     std::shared_ptr<const KeyCondition> key_condition;
 
-    void setKeyConditionImpl(const ActionsDAG * filter_actions_dag, ContextPtr context, const Block & keys)
+    void setKeyConditionImpl(const std::optional<ActionsDAG> & filter_actions_dag, ContextPtr context, const Block & keys)
     {
         key_condition = std::make_shared<const KeyCondition>(
-            filter_actions_dag,
+            filter_actions_dag ? &*filter_actions_dag : nullptr,
             context,
             keys.getNames(),
-            std::make_shared<ExpressionActions>(std::make_unique<ActionsDAG>(keys.getColumnsWithTypeAndName())));
+            std::make_shared<ExpressionActions>(ActionsDAG(keys.getColumnsWithTypeAndName())));
     }
 
 public:
@@ -33,6 +33,6 @@ public:
     virtual void setKeyCondition(const std::shared_ptr<const KeyCondition> & key_condition_) { key_condition = key_condition_; }
 
     /// Set key_condition created by filter_actions_dag and context.
-    virtual void setKeyCondition(const ActionsDAGPtr & /*filter_actions_dag*/, ContextPtr /*context*/) { }
+    virtual void setKeyCondition(const std::optional<ActionsDAG> & /*filter_actions_dag*/, ContextPtr /*context*/) { }
 };
 }
diff --git a/src/Processors/Transforms/AddingDefaultsTransform.cpp b/src/Processors/Transforms/AddingDefaultsTransform.cpp
index 7945b3999c1..da4d3a0041b 100644
--- a/src/Processors/Transforms/AddingDefaultsTransform.cpp
+++ b/src/Processors/Transforms/AddingDefaultsTransform.cpp
@@ -178,7 +178,7 @@ void AddingDefaultsTransform::transform(Chunk & chunk)
     auto dag = evaluateMissingDefaults(evaluate_block, header.getNamesAndTypesList(), columns, context, false);
     if (dag)
     {
-        auto actions = std::make_shared<ExpressionActions>(std::move(dag), ExpressionActionsSettings::fromContext(context, CompileExpressions::yes), true);
+        auto actions = std::make_shared<ExpressionActions>(std::move(*dag), ExpressionActionsSettings::fromContext(context, CompileExpressions::yes), true);
         actions->execute(evaluate_block);
     }
 
diff --git a/src/Processors/Transforms/FillingTransform.cpp b/src/Processors/Transforms/FillingTransform.cpp
index bbe57fc6441..36ffc515f43 100644
--- a/src/Processors/Transforms/FillingTransform.cpp
+++ b/src/Processors/Transforms/FillingTransform.cpp
@@ -203,7 +203,7 @@ FillingTransform::FillingTransform(
     , use_with_fill_by_sorting_prefix(use_with_fill_by_sorting_prefix_)
 {
     if (interpolate_description)
-        interpolate_actions = std::make_shared<ExpressionActions>(ActionsDAG::clone(interpolate_description->actions));
+        interpolate_actions = std::make_shared<ExpressionActions>(std::move(*ActionsDAG::clone(&interpolate_description->actions)));
 
     std::vector<bool> is_fill_column(header_.columns());
     for (size_t i = 0, size = fill_description.size(); i < size; ++i)
diff --git a/src/Storages/Hive/StorageHive.cpp b/src/Storages/Hive/StorageHive.cpp
index 2cd51259549..da5a45f36d5 100644
--- a/src/Storages/Hive/StorageHive.cpp
+++ b/src/Storages/Hive/StorageHive.cpp
@@ -516,7 +516,7 @@ void StorageHive::initMinMaxIndexExpression()
         partition_names = partition_name_types.getNames();
         partition_types = partition_name_types.getTypes();
         partition_minmax_idx_expr = std::make_shared<ExpressionActions>(
-            std::make_unique<ActionsDAG>(partition_name_types), ExpressionActionsSettings::fromContext(getContext()));
+            ActionsDAG(partition_name_types), ExpressionActionsSettings::fromContext(getContext()));
     }
 
     NamesAndTypesList all_name_types = metadata_snapshot->getColumns().getAllPhysical();
@@ -526,7 +526,7 @@ void StorageHive::initMinMaxIndexExpression()
             hivefile_name_types.push_back(column);
     }
     hivefile_minmax_idx_expr = std::make_shared<ExpressionActions>(
-        std::make_unique<ActionsDAG>(hivefile_name_types), ExpressionActionsSettings::fromContext(getContext()));
+        ActionsDAG(hivefile_name_types), ExpressionActionsSettings::fromContext(getContext()));
 }
 
 ASTPtr StorageHive::extractKeyExpressionList(const ASTPtr & node)
@@ -583,7 +583,7 @@ static HiveFilePtr createHiveFile(
 
 HiveFiles StorageHive::collectHiveFilesFromPartition(
     const Apache::Hadoop::Hive::Partition & partition,
-    const ActionsDAGPtr & filter_actions_dag,
+    const ActionsDAG * filter_actions_dag,
     const HiveTableMetadataPtr & hive_table_metadata,
     const HDFSFSPtr & fs,
     const ContextPtr & context_,
@@ -647,7 +647,7 @@ HiveFiles StorageHive::collectHiveFilesFromPartition(
         for (size_t i = 0; i < partition_names.size(); ++i)
             ranges.emplace_back(fields[i]);
 
-        const KeyCondition partition_key_condition(filter_actions_dag.get(), getContext(), partition_names, partition_minmax_idx_expr);
+        const KeyCondition partition_key_condition(filter_actions_dag, getContext(), partition_names, partition_minmax_idx_expr);
         if (!partition_key_condition.checkInHyperrectangle(ranges, partition_types).can_be_true)
             return {};
     }
@@ -681,7 +681,7 @@ StorageHive::listDirectory(const String & path, const HiveTableMetadataPtr & hiv
 HiveFilePtr StorageHive::getHiveFileIfNeeded(
     const FileInfo & file_info,
     const FieldVector & fields,
-    const ActionsDAGPtr & filter_actions_dag,
+    const ActionsDAG * filter_actions_dag,
     const HiveTableMetadataPtr & hive_table_metadata,
     const ContextPtr & context_,
     PruneLevel prune_level) const
@@ -715,7 +715,7 @@ HiveFilePtr StorageHive::getHiveFileIfNeeded(
 
     if (prune_level >= PruneLevel::File)
     {
-        const KeyCondition hivefile_key_condition(filter_actions_dag.get(), getContext(), hivefile_name_types.getNames(), hivefile_minmax_idx_expr);
+        const KeyCondition hivefile_key_condition(filter_actions_dag, getContext(), hivefile_name_types.getNames(), hivefile_minmax_idx_expr);
         if (hive_file->useFileMinMaxIndex())
         {
             /// Load file level minmax index and apply
@@ -828,7 +828,7 @@ void ReadFromHive::createFiles()
     if (hive_files)
         return;
 
-    hive_files = storage->collectHiveFiles(num_streams, filter_actions_dag, hive_table_metadata, fs, context);
+    hive_files = storage->collectHiveFiles(num_streams, filter_actions_dag ? &*filter_actions_dag : nullptr, hive_table_metadata, fs, context);
     LOG_INFO(log, "Collect {} hive files to read", hive_files->size());
 }
 
@@ -950,7 +950,7 @@ void ReadFromHive::initializePipeline(QueryPipelineBuilder & pipeline, const Bui
 
 HiveFiles StorageHive::collectHiveFiles(
     size_t max_threads,
-    const ActionsDAGPtr & filter_actions_dag,
+    const ActionsDAG * filter_actions_dag,
     const HiveTableMetadataPtr & hive_table_metadata,
     const HDFSFSPtr & fs,
     const ContextPtr & context_,
@@ -1023,12 +1023,12 @@ SinkToStoragePtr StorageHive::write(const ASTPtr & /*query*/, const StorageMetad
 std::optional<UInt64> StorageHive::totalRows(const Settings & settings) const
 {
     /// query_info is not used when prune_level == PruneLevel::None
-    return totalRowsImpl(settings, nullptr, getContext(), PruneLevel::None);
+    return totalRowsImpl(settings, {}, getContext(), PruneLevel::None);
 }
 
-std::optional<UInt64> StorageHive::totalRowsByPartitionPredicate(const ActionsDAGPtr & filter_actions_dag, ContextPtr context_) const
+std::optional<UInt64> StorageHive::totalRowsByPartitionPredicate(const ActionsDAG & filter_actions_dag, ContextPtr context_) const
 {
-    return totalRowsImpl(context_->getSettingsRef(), filter_actions_dag, context_, PruneLevel::Partition);
+    return totalRowsImpl(context_->getSettingsRef(), &filter_actions_dag, context_, PruneLevel::Partition);
 }
 
 void StorageHive::checkAlterIsPossible(const AlterCommands & commands, ContextPtr /*local_context*/) const
@@ -1043,7 +1043,7 @@ void StorageHive::checkAlterIsPossible(const AlterCommands & commands, ContextPt
 }
 
 std::optional<UInt64>
-StorageHive::totalRowsImpl(const Settings & settings, const ActionsDAGPtr & filter_actions_dag, ContextPtr context_, PruneLevel prune_level) const
+StorageHive::totalRowsImpl(const Settings & settings, const ActionsDAG * filter_actions_dag, ContextPtr context_, PruneLevel prune_level) const
 {
     /// Row-based format like Text doesn't support totalRowsByPartitionPredicate
     if (!supportsSubsetOfColumns())
diff --git a/src/Storages/Hive/StorageHive.h b/src/Storages/Hive/StorageHive.h
index 8a457dd6e01..e16df22e138 100644
--- a/src/Storages/Hive/StorageHive.h
+++ b/src/Storages/Hive/StorageHive.h
@@ -57,7 +57,7 @@ public:
     bool supportsSubsetOfColumns() const;
 
     std::optional<UInt64> totalRows(const Settings & settings) const override;
-    std::optional<UInt64> totalRowsByPartitionPredicate(const ActionsDAGPtr & filter_actions_dag, ContextPtr context_) const override;
+    std::optional<UInt64> totalRowsByPartitionPredicate(const ActionsDAG & filter_actions_dag, ContextPtr context_) const override;
     void checkAlterIsPossible(const AlterCommands & commands, ContextPtr local_context) const override;
 
 protected:
@@ -90,7 +90,7 @@ private:
 
     HiveFiles collectHiveFiles(
         size_t max_threads,
-        const ActionsDAGPtr & filter_actions_dag,
+        const ActionsDAG * filter_actions_dag,
         const HiveTableMetadataPtr & hive_table_metadata,
         const HDFSFSPtr & fs,
         const ContextPtr & context_,
@@ -98,7 +98,7 @@ private:
 
     HiveFiles collectHiveFilesFromPartition(
         const Apache::Hadoop::Hive::Partition & partition,
-        const ActionsDAGPtr & filter_actions_dag,
+        const ActionsDAG * filter_actions_dag,
         const HiveTableMetadataPtr & hive_table_metadata,
         const HDFSFSPtr & fs,
         const ContextPtr & context_,
@@ -107,7 +107,7 @@ private:
     HiveFilePtr getHiveFileIfNeeded(
         const FileInfo & file_info,
         const FieldVector & fields,
-        const ActionsDAGPtr & filter_actions_dag,
+        const ActionsDAG * filter_actions_dag,
         const HiveTableMetadataPtr & hive_table_metadata,
         const ContextPtr & context_,
         PruneLevel prune_level = PruneLevel::Max) const;
@@ -115,7 +115,7 @@ private:
     void lazyInitialize();
 
     std::optional<UInt64>
-    totalRowsImpl(const Settings & settings, const ActionsDAGPtr & filter_actions_dag, ContextPtr context_, PruneLevel prune_level) const;
+    totalRowsImpl(const Settings & settings, const ActionsDAG * filter_actions_dag, ContextPtr context_, PruneLevel prune_level) const;
 
     String hive_metastore_url;
 
diff --git a/src/Storages/IStorage.cpp b/src/Storages/IStorage.cpp
index 1f7ac23eb82..57f79a2cd7f 100644
--- a/src/Storages/IStorage.cpp
+++ b/src/Storages/IStorage.cpp
@@ -236,7 +236,7 @@ StorageID IStorage::getStorageID() const
     return storage_id;
 }
 
-ConditionSelectivityEstimator IStorage::getConditionSelectivityEstimatorByPredicate(const StorageSnapshotPtr &, const ActionsDAGPtr &, ContextPtr) const
+ConditionSelectivityEstimator IStorage::getConditionSelectivityEstimatorByPredicate(const StorageSnapshotPtr &, const ActionsDAG *, ContextPtr) const
 {
     return {};
 }
diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h
index 98afd844046..c86f18d5d3b 100644
--- a/src/Storages/IStorage.h
+++ b/src/Storages/IStorage.h
@@ -135,7 +135,7 @@ public:
     /// Returns true if the storage supports queries with the PREWHERE section.
     virtual bool supportsPrewhere() const { return false; }
 
-    virtual ConditionSelectivityEstimator getConditionSelectivityEstimatorByPredicate(const StorageSnapshotPtr &, const ActionsDAGPtr &, ContextPtr) const;
+    virtual ConditionSelectivityEstimator getConditionSelectivityEstimatorByPredicate(const StorageSnapshotPtr &, const ActionsDAG *, ContextPtr) const;
 
     /// Returns which columns supports PREWHERE, or empty std::nullopt if all columns is supported.
     /// This is needed for engines whose aggregates data from multiple tables, like Merge.
@@ -682,7 +682,7 @@ public:
     virtual std::optional<UInt64> totalRows(const Settings &) const { return {}; }
 
     /// Same as above but also take partition predicate into account.
-    virtual std::optional<UInt64> totalRowsByPartitionPredicate(const ActionsDAGPtr &, ContextPtr) const { return {}; }
+    virtual std::optional<UInt64> totalRowsByPartitionPredicate(const ActionsDAG &, ContextPtr) const { return {}; }
 
     /// If it is possible to quickly determine exact number of bytes for the table on storage:
     /// - memory (approximated, resident)
diff --git a/src/Storages/KVStorageUtils.cpp b/src/Storages/KVStorageUtils.cpp
index 94319aef3b8..88783246e10 100644
--- a/src/Storages/KVStorageUtils.cpp
+++ b/src/Storages/KVStorageUtils.cpp
@@ -231,7 +231,7 @@ bool traverseDAGFilter(
 }
 
 std::pair<FieldVectorPtr, bool> getFilterKeys(
-    const String & primary_key, const DataTypePtr & primary_key_type, const ActionsDAGPtr & filter_actions_dag, const ContextPtr & context)
+    const String & primary_key, const DataTypePtr & primary_key_type, const std::optional<ActionsDAG> & filter_actions_dag, const ContextPtr & context)
 {
     if (!filter_actions_dag)
         return {{}, true};
diff --git a/src/Storages/KVStorageUtils.h b/src/Storages/KVStorageUtils.h
index e20a1ce4f37..64108290270 100644
--- a/src/Storages/KVStorageUtils.h
+++ b/src/Storages/KVStorageUtils.h
@@ -22,7 +22,7 @@ std::pair<FieldVectorPtr, bool> getFilterKeys(
     const std::string & primary_key, const DataTypePtr & primary_key_type, const SelectQueryInfo & query_info, const ContextPtr & context);
 
 std::pair<FieldVectorPtr, bool> getFilterKeys(
-    const String & primary_key, const DataTypePtr & primary_key_type, const ActionsDAGPtr & filter_actions_dag, const ContextPtr & context);
+    const String & primary_key, const DataTypePtr & primary_key_type, const std::optional<ActionsDAG> & filter_actions_dag, const ContextPtr & context);
 
 template <typename K, typename V>
 void fillColumns(const K & key, const V & value, size_t key_pos, const Block & header, MutableColumns & columns)
diff --git a/src/Storages/KeyDescription.cpp b/src/Storages/KeyDescription.cpp
index e03ecc05064..7e43966556e 100644
--- a/src/Storages/KeyDescription.cpp
+++ b/src/Storages/KeyDescription.cpp
@@ -160,7 +160,7 @@ KeyDescription KeyDescription::buildEmptyKey()
 {
     KeyDescription result;
     result.expression_list_ast = std::make_shared<ASTExpressionList>();
-    result.expression = std::make_shared<ExpressionActions>(std::make_unique<ActionsDAG>(), ExpressionActionsSettings{});
+    result.expression = std::make_shared<ExpressionActions>(ActionsDAG(), ExpressionActionsSettings{});
     return result;
 }
 
diff --git a/src/Storages/MergeTree/IMergeTreeReader.cpp b/src/Storages/MergeTree/IMergeTreeReader.cpp
index 4ad7f6ef991..264b2b397f4 100644
--- a/src/Storages/MergeTree/IMergeTreeReader.cpp
+++ b/src/Storages/MergeTree/IMergeTreeReader.cpp
@@ -163,8 +163,8 @@ void IMergeTreeReader::evaluateMissingDefaults(Block additional_columns, Columns
         if (dag)
         {
             dag->addMaterializingOutputActions();
-            auto actions = std::make_shared<
-                ExpressionActions>(std::move(dag),
+            auto actions = std::make_shared<ExpressionActions>(
+                std::move(*dag),
                 ExpressionActionsSettings::fromSettings(data_part_info_for_read->getContext()->getSettingsRef()));
             actions->execute(additional_columns);
         }
diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp
index 1efded3b064..d781345d834 100644
--- a/src/Storages/MergeTree/KeyCondition.cpp
+++ b/src/Storages/MergeTree/KeyCondition.cpp
@@ -628,7 +628,7 @@ static const ActionsDAG::Node & cloneASTWithInversionPushDown(
                     if (const auto * index_hint = typeid_cast<const FunctionIndexHint *>(adaptor->getFunction().get()))
                     {
                         const auto & index_hint_dag = index_hint->getActions();
-                        children = index_hint_dag->getOutputs();
+                        children = index_hint_dag.getOutputs();
 
                         for (auto & arg : children)
                             arg = &cloneASTWithInversionPushDown(*arg, inverted_dag, to_inverted, context, need_inversion);
@@ -729,7 +729,7 @@ Block KeyCondition::getBlockWithConstants(
     if (syntax_analyzer_result)
     {
         auto actions = ExpressionAnalyzer(query, syntax_analyzer_result, context).getConstActionsDAG();
-        for (const auto & action_node : actions->getOutputs())
+        for (const auto & action_node : actions.getOutputs())
         {
             if (action_node->column)
                 result.insert(ColumnWithTypeAndName{action_node->column, action_node->result_type, action_node->result_name});
diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index 7b642c34f37..334c8c9c5ac 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -472,7 +472,7 @@ StoragePolicyPtr MergeTreeData::getStoragePolicy() const
 }
 
 ConditionSelectivityEstimator MergeTreeData::getConditionSelectivityEstimatorByPredicate(
-    const StorageSnapshotPtr & storage_snapshot, const ActionsDAGPtr & filter_dag, ContextPtr local_context) const
+    const StorageSnapshotPtr & storage_snapshot, const ActionsDAG * filter_dag, ContextPtr local_context) const
 {
     if (!local_context->getSettings().allow_statistics_optimize)
         return {};
@@ -487,7 +487,7 @@ ConditionSelectivityEstimator MergeTreeData::getConditionSelectivityEstimatorByP
     ASTPtr expression_ast;
 
     ConditionSelectivityEstimator result;
-    PartitionPruner partition_pruner(storage_snapshot->metadata, filter_dag.get(), local_context);
+    PartitionPruner partition_pruner(storage_snapshot->metadata, filter_dag, local_context);
 
     if (partition_pruner.isUseless())
     {
@@ -746,7 +746,7 @@ ExpressionActionsPtr MergeTreeData::getMinMaxExpr(const KeyDescription & partiti
     if (!partition_key.column_names.empty())
         partition_key_columns = partition_key.expression->getRequiredColumnsWithTypes();
 
-    return std::make_shared<ExpressionActions>(std::make_unique<ActionsDAG>(partition_key_columns), settings);
+    return std::make_shared<ExpressionActions>(ActionsDAG(partition_key_columns), settings);
 }
 
 Names MergeTreeData::getMinMaxColumnsNames(const KeyDescription & partition_key)
@@ -1134,7 +1134,7 @@ Block MergeTreeData::getBlockWithVirtualsForFilter(
 
 
 std::optional<UInt64> MergeTreeData::totalRowsByPartitionPredicateImpl(
-    const ActionsDAGPtr & filter_actions_dag, ContextPtr local_context, const DataPartsVector & parts) const
+    const ActionsDAG & filter_actions_dag, ContextPtr local_context, const DataPartsVector & parts) const
 {
     if (parts.empty())
         return 0;
@@ -1142,7 +1142,7 @@ std::optional<UInt64> MergeTreeData::totalRowsByPartitionPredicateImpl(
     auto metadata_snapshot = getInMemoryMetadataPtr();
     auto virtual_columns_block = getBlockWithVirtualsForFilter(metadata_snapshot, {parts[0]});
 
-    auto filter_dag = VirtualColumnUtils::splitFilterDagForAllowedInputs(filter_actions_dag->getOutputs().at(0), nullptr);
+    auto filter_dag = VirtualColumnUtils::splitFilterDagForAllowedInputs(filter_actions_dag.getOutputs().at(0), nullptr);
     if (!filter_dag)
         return {};
 
diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h
index 52916d85fef..e490e4b0bf9 100644
--- a/src/Storages/MergeTree/MergeTreeData.h
+++ b/src/Storages/MergeTree/MergeTreeData.h
@@ -426,7 +426,7 @@ public:
 
     bool supportsPrewhere() const override { return true; }
 
-    ConditionSelectivityEstimator getConditionSelectivityEstimatorByPredicate(const StorageSnapshotPtr &, const ActionsDAGPtr &, ContextPtr) const override;
+    ConditionSelectivityEstimator getConditionSelectivityEstimatorByPredicate(const StorageSnapshotPtr &, const ActionsDAG *, ContextPtr) const override;
 
     bool supportsFinal() const override;
 
@@ -1227,7 +1227,7 @@ protected:
         boost::iterator_range<DataPartIteratorByStateAndInfo> range, const ColumnsDescription & storage_columns);
 
     std::optional<UInt64> totalRowsByPartitionPredicateImpl(
-        const ActionsDAGPtr & filter_actions_dag, ContextPtr context, const DataPartsVector & parts) const;
+        const ActionsDAG & filter_actions_dag, ContextPtr context, const DataPartsVector & parts) const;
 
     static decltype(auto) getStateModifier(DataPartState state)
     {
diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp
index 61b8b6fdaa8..5a5b6d4a6e1 100644
--- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp
+++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp
@@ -428,7 +428,7 @@ MergeTreeDataSelectSamplingData MergeTreeDataSelectExecutor::getSampling(
 
             ASTPtr query = sampling.filter_function;
             auto syntax_result = TreeRewriter(context).analyze(query, available_real_columns);
-            sampling.filter_expression = ExpressionAnalyzer(sampling.filter_function, syntax_result, context).getActionsDAG(false);
+            sampling.filter_expression = std::make_shared<const ActionsDAG>(ExpressionAnalyzer(sampling.filter_function, syntax_result, context).getActionsDAG(false));
         }
     }
 
@@ -466,7 +466,7 @@ void MergeTreeDataSelectExecutor::buildKeyConditionFromPartOffset(
         dag.get(),
         context,
         sample.getNames(),
-        std::make_shared<ExpressionActions>(std::make_unique<ActionsDAG>(sample.getColumnsWithTypeAndName()), ExpressionActionsSettings{}),
+        std::make_shared<ExpressionActions>(ActionsDAG(sample.getColumnsWithTypeAndName()), ExpressionActionsSettings{}),
         {}});
 }
 
diff --git a/src/Storages/MergeTree/MergeTreeIndexSet.cpp b/src/Storages/MergeTree/MergeTreeIndexSet.cpp
index 8d4ef69b1b9..ca31ffc9de5 100644
--- a/src/Storages/MergeTree/MergeTreeIndexSet.cpp
+++ b/src/Storages/MergeTree/MergeTreeIndexSet.cpp
@@ -265,15 +265,15 @@ MergeTreeIndexConditionSet::MergeTreeIndexConditionSet(
         if (!set->buildOrderedSetInplace(context))
             return;
 
-    auto filter_actions_dag = ActionsDAG::clone(filter_dag);
-    const auto * filter_actions_dag_node = filter_actions_dag->getOutputs().at(0);
+    auto filter_actions_dag = std::move(*ActionsDAG::clone(filter_dag));
+    const auto * filter_actions_dag_node = filter_actions_dag.getOutputs().at(0);
 
     std::unordered_map<const ActionsDAG::Node *, const ActionsDAG::Node *> node_to_result_node;
-    filter_actions_dag->getOutputs()[0] = &traverseDAG(*filter_actions_dag_node, filter_actions_dag, context, node_to_result_node);
+    filter_actions_dag.getOutputs()[0] = &traverseDAG(*filter_actions_dag_node, filter_actions_dag, context, node_to_result_node);
 
-    filter_actions_dag->removeUnusedActions();
+    filter_actions_dag.removeUnusedActions();
 
-    actions_output_column_name = filter_actions_dag->getOutputs().at(0)->result_name;
+    actions_output_column_name = filter_actions_dag.getOutputs().at(0)->result_name;
     actions = std::make_shared<ExpressionActions>(std::move(filter_actions_dag));
 }
 
@@ -306,7 +306,7 @@ bool MergeTreeIndexConditionSet::mayBeTrueOnGranule(MergeTreeIndexGranulePtr idx
 }
 
 
-static const ActionsDAG::NodeRawConstPtrs & getArguments(const ActionsDAG::Node & node, const ActionsDAGPtr & result_dag_or_null, ActionsDAG::NodeRawConstPtrs * storage)
+static const ActionsDAG::NodeRawConstPtrs & getArguments(const ActionsDAG::Node & node, ActionsDAG * result_dag_or_null, ActionsDAG::NodeRawConstPtrs * storage)
 {
     chassert(node.type == ActionsDAG::ActionType::FUNCTION);
     if (node.function_base->getName() != "indexHint")
@@ -316,17 +316,17 @@ static const ActionsDAG::NodeRawConstPtrs & getArguments(const ActionsDAG::Node
     const auto & adaptor = typeid_cast<const FunctionToFunctionBaseAdaptor &>(*node.function_base);
     const auto & index_hint = typeid_cast<const FunctionIndexHint &>(*adaptor.getFunction());
     if (!result_dag_or_null)
-        return index_hint.getActions()->getOutputs();
+        return index_hint.getActions().getOutputs();
 
     /// Import the DAG and map argument pointers.
-    ActionsDAGPtr actions_clone = ActionsDAG::clone(index_hint.getActions());
+    auto actions_clone = std::move(*ActionsDAG::clone(&index_hint.getActions()));
     chassert(storage);
-    result_dag_or_null->mergeNodes(std::move(*actions_clone), storage);
+    result_dag_or_null->mergeNodes(std::move(actions_clone), storage);
     return *storage;
 }
 
 const ActionsDAG::Node & MergeTreeIndexConditionSet::traverseDAG(const ActionsDAG::Node & node,
-    ActionsDAGPtr & result_dag,
+    ActionsDAG & result_dag,
     const ContextPtr & context,
     std::unordered_map<const ActionsDAG::Node *, const ActionsDAG::Node *> & node_to_result_node) const
 {
@@ -348,7 +348,7 @@ const ActionsDAG::Node & MergeTreeIndexConditionSet::traverseDAG(const ActionsDA
             atom_node_ptr->type == ActionsDAG::ActionType::FUNCTION)
         {
             auto bit_wrapper_function = FunctionFactory::instance().get("__bitWrapperFunc", context);
-            result_node = &result_dag->addFunction(bit_wrapper_function, {atom_node_ptr}, {});
+            result_node = &result_dag.addFunction(bit_wrapper_function, {atom_node_ptr}, {});
         }
     }
     else
@@ -359,14 +359,14 @@ const ActionsDAG::Node & MergeTreeIndexConditionSet::traverseDAG(const ActionsDA
         unknown_field_column_with_type.type = std::make_shared<DataTypeUInt8>();
         unknown_field_column_with_type.column = unknown_field_column_with_type.type->createColumnConst(1, UNKNOWN_FIELD);
 
-        result_node = &result_dag->addColumn(unknown_field_column_with_type);
+        result_node = &result_dag.addColumn(unknown_field_column_with_type);
     }
 
     node_to_result_node.emplace(&node, result_node);
     return *result_node;
 }
 
-const ActionsDAG::Node * MergeTreeIndexConditionSet::atomFromDAG(const ActionsDAG::Node & node, ActionsDAGPtr & result_dag, const ContextPtr & context) const
+const ActionsDAG::Node * MergeTreeIndexConditionSet::atomFromDAG(const ActionsDAG::Node & node, ActionsDAG & result_dag, const ContextPtr & context) const
 {
     /// Function, literal or column
 
@@ -386,7 +386,7 @@ const ActionsDAG::Node * MergeTreeIndexConditionSet::atomFromDAG(const ActionsDA
         const auto * result_node = node_to_check;
 
         if (node.type != ActionsDAG::ActionType::INPUT)
-            result_node = &result_dag->addInput(column_name, node.result_type);
+            result_node = &result_dag.addInput(column_name, node.result_type);
 
         return result_node;
     }
@@ -407,11 +407,11 @@ const ActionsDAG::Node * MergeTreeIndexConditionSet::atomFromDAG(const ActionsDA
             return nullptr;
     }
 
-    return &result_dag->addFunction(node.function_base, children, {});
+    return &result_dag.addFunction(node.function_base, children, {});
 }
 
 const ActionsDAG::Node * MergeTreeIndexConditionSet::operatorFromDAG(const ActionsDAG::Node & node,
-    ActionsDAGPtr & result_dag,
+    ActionsDAG & result_dag,
     const ContextPtr & context,
     std::unordered_map<const ActionsDAG::Node *, const ActionsDAG::Node *> & node_to_result_node) const
 {
@@ -429,7 +429,7 @@ const ActionsDAG::Node * MergeTreeIndexConditionSet::operatorFromDAG(const Actio
 
     auto function_name = node_to_check->function->getName();
     ActionsDAG::NodeRawConstPtrs temp_ptrs_to_argument;
-    const auto & arguments = getArguments(*node_to_check, result_dag, &temp_ptrs_to_argument);
+    const auto & arguments = getArguments(*node_to_check, &result_dag, &temp_ptrs_to_argument);
     size_t arguments_size = arguments.size();
 
     if (function_name == "not")
@@ -440,7 +440,7 @@ const ActionsDAG::Node * MergeTreeIndexConditionSet::operatorFromDAG(const Actio
         const ActionsDAG::Node * argument = &traverseDAG(*arguments[0], result_dag, context, node_to_result_node);
 
         auto bit_swap_last_two_function = FunctionFactory::instance().get("__bitSwapLastTwo", context);
-        return &result_dag->addFunction(bit_swap_last_two_function, {argument}, {});
+        return &result_dag.addFunction(bit_swap_last_two_function, {argument}, {});
     }
     else if (function_name == "and" || function_name == "indexHint" || function_name == "or")
     {
@@ -468,7 +468,7 @@ const ActionsDAG::Node * MergeTreeIndexConditionSet::operatorFromDAG(const Actio
             const auto * before_last_argument = children.back();
             children.pop_back();
 
-            last_argument = &result_dag->addFunction(function, {before_last_argument, last_argument}, {});
+            last_argument = &result_dag.addFunction(function, {before_last_argument, last_argument}, {});
         }
 
         return last_argument;
diff --git a/src/Storages/MergeTree/MergeTreeIndexSet.h b/src/Storages/MergeTree/MergeTreeIndexSet.h
index abd40b3cf9d..03b02515e47 100644
--- a/src/Storages/MergeTree/MergeTreeIndexSet.h
+++ b/src/Storages/MergeTree/MergeTreeIndexSet.h
@@ -93,16 +93,16 @@ public:
     ~MergeTreeIndexConditionSet() override = default;
 private:
     const ActionsDAG::Node & traverseDAG(const ActionsDAG::Node & node,
-        ActionsDAGPtr & result_dag,
+        ActionsDAG & result_dag,
         const ContextPtr & context,
         std::unordered_map<const ActionsDAG::Node *, const ActionsDAG::Node *> & node_to_result_node) const;
 
     const ActionsDAG::Node * atomFromDAG(const ActionsDAG::Node & node,
-        ActionsDAGPtr & result_dag,
+        ActionsDAG & result_dag,
         const ContextPtr & context) const;
 
     const ActionsDAG::Node * operatorFromDAG(const ActionsDAG::Node & node,
-        ActionsDAGPtr & result_dag,
+        ActionsDAG & result_dag,
         const ContextPtr & context,
         std::unordered_map<const ActionsDAG::Node *, const ActionsDAG::Node *> & node_to_result_node) const;
 
diff --git a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp
index e924f853524..aec2f988e8d 100644
--- a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp
+++ b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp
@@ -80,7 +80,7 @@ PrewhereExprInfo MergeTreeSelectProcessor::getPrewhereActions(PrewhereInfoPtr pr
             PrewhereExprStep row_level_filter_step
             {
                 .type = PrewhereExprStep::Filter,
-                .actions = std::make_shared<ExpressionActions>(ActionsDAG::clone(prewhere_info->row_level_filter), actions_settings),
+                .actions = std::make_shared<ExpressionActions>(std::move(*ActionsDAG::clone(&*prewhere_info->row_level_filter)), actions_settings),
                 .filter_column_name = prewhere_info->row_level_column_name,
                 .remove_filter_column = true,
                 .need_filter = true,
@@ -96,7 +96,7 @@ PrewhereExprInfo MergeTreeSelectProcessor::getPrewhereActions(PrewhereInfoPtr pr
             PrewhereExprStep prewhere_step
             {
                 .type = PrewhereExprStep::Filter,
-                .actions = std::make_shared<ExpressionActions>(ActionsDAG::clone(prewhere_info->prewhere_actions), actions_settings),
+                .actions = std::make_shared<ExpressionActions>(std::move(*ActionsDAG::clone(&*prewhere_info->prewhere_actions)), actions_settings),
                 .filter_column_name = prewhere_info->prewhere_column_name,
                 .remove_filter_column = prewhere_info->remove_prewhere_column,
                 .need_filter = prewhere_info->need_filter,
diff --git a/src/Storages/MergeTree/MergeTreeSequentialSource.cpp b/src/Storages/MergeTree/MergeTreeSequentialSource.cpp
index 98b35a3ca2c..15917d59c9f 100644
--- a/src/Storages/MergeTree/MergeTreeSequentialSource.cpp
+++ b/src/Storages/MergeTree/MergeTreeSequentialSource.cpp
@@ -349,7 +349,7 @@ public:
         MergeTreeData::DataPartPtr data_part_,
         Names columns_to_read_,
         bool apply_deleted_mask_,
-        ActionsDAGPtr filter_,
+        std::optional<ActionsDAG> filter_,
         ContextPtr context_,
         LoggerPtr log_)
         : ISourceStep(DataStream{.header = storage_snapshot_->getSampleBlockForColumns(columns_to_read_)})
@@ -376,7 +376,7 @@ public:
         {
             const auto & primary_key = storage_snapshot->metadata->getPrimaryKey();
             const Names & primary_key_column_names = primary_key.column_names;
-            KeyCondition key_condition(filter.get(), context, primary_key_column_names, primary_key.expression);
+            KeyCondition key_condition(&*filter, context, primary_key_column_names, primary_key.expression);
             LOG_DEBUG(log, "Key condition: {}", key_condition.toString());
 
             if (!key_condition.alwaysFalse())
@@ -417,7 +417,7 @@ private:
     MergeTreeData::DataPartPtr data_part;
     Names columns_to_read;
     bool apply_deleted_mask;
-    ActionsDAGPtr filter;
+    std::optional<ActionsDAG> filter;
     ContextPtr context;
     LoggerPtr log;
 };
@@ -430,7 +430,7 @@ void createReadFromPartStep(
     MergeTreeData::DataPartPtr data_part,
     Names columns_to_read,
     bool apply_deleted_mask,
-    ActionsDAGPtr filter,
+    std::optional<ActionsDAG> filter,
     ContextPtr context,
     LoggerPtr log)
 {
diff --git a/src/Storages/MergeTree/MergeTreeSequentialSource.h b/src/Storages/MergeTree/MergeTreeSequentialSource.h
index e6f055f776c..1b05512b9a3 100644
--- a/src/Storages/MergeTree/MergeTreeSequentialSource.h
+++ b/src/Storages/MergeTree/MergeTreeSequentialSource.h
@@ -38,7 +38,7 @@ void createReadFromPartStep(
     MergeTreeData::DataPartPtr data_part,
     Names columns_to_read,
     bool apply_deleted_mask,
-    ActionsDAGPtr filter,
+    std::optional<ActionsDAG> filter,
     ContextPtr context,
     LoggerPtr log);
 
diff --git a/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp b/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp
index 25596b42951..116edf5b9cb 100644
--- a/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp
+++ b/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp
@@ -349,7 +349,7 @@ bool tryBuildPrewhereSteps(PrewhereInfoPtr prewhere_info, const ExpressionAction
             PrewhereExprStep new_step
             {
                 .type = PrewhereExprStep::Filter,
-                .actions = std::make_shared<ExpressionActions>(std::move(step.actions), actions_settings),
+                .actions = std::make_shared<ExpressionActions>(std::move(*step.actions), actions_settings),
                 .filter_column_name = step.column_name,
                 /// Don't remove if it's in the list of original outputs
                 .remove_filter_column =
diff --git a/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp b/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp
index a9a5fddace4..8c389f00780 100644
--- a/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp
+++ b/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp
@@ -112,7 +112,7 @@ void MergeTreeWhereOptimizer::optimize(SelectQueryInfo & select_query_info, cons
     LOG_DEBUG(log, "MergeTreeWhereOptimizer: condition \"{}\" moved to PREWHERE", select.prewhere()->formatForLogging(log_queries_cut_to_length));
 }
 
-MergeTreeWhereOptimizer::FilterActionsOptimizeResult MergeTreeWhereOptimizer::optimize(const ActionsDAGPtr & filter_dag,
+MergeTreeWhereOptimizer::FilterActionsOptimizeResult MergeTreeWhereOptimizer::optimize(const ActionsDAG & filter_dag,
     const std::string & filter_column_name,
     const ContextPtr & context,
     bool is_final)
@@ -126,7 +126,7 @@ MergeTreeWhereOptimizer::FilterActionsOptimizeResult MergeTreeWhereOptimizer::op
     where_optimizer_context.use_statistics = context->getSettingsRef().allow_statistics_optimize;
 
     RPNBuilderTreeContext tree_context(context);
-    RPNBuilderTreeNode node(&filter_dag->findInOutputs(filter_column_name), tree_context);
+    RPNBuilderTreeNode node(&filter_dag.findInOutputs(filter_column_name), tree_context);
 
     auto optimize_result = optimizeImpl(node, where_optimizer_context);
     if (!optimize_result)
diff --git a/src/Storages/MergeTree/MergeTreeWhereOptimizer.h b/src/Storages/MergeTree/MergeTreeWhereOptimizer.h
index ba6b4660924..a3d035675c6 100644
--- a/src/Storages/MergeTree/MergeTreeWhereOptimizer.h
+++ b/src/Storages/MergeTree/MergeTreeWhereOptimizer.h
@@ -52,7 +52,7 @@ public:
         bool fully_moved_to_prewhere = false;
     };
 
-    FilterActionsOptimizeResult optimize(const ActionsDAGPtr & filter_dag,
+    FilterActionsOptimizeResult optimize(const ActionsDAG & filter_dag,
         const std::string & filter_column_name,
         const ContextPtr & context,
         bool is_final);
diff --git a/src/Storages/MergeTree/RPNBuilder.cpp b/src/Storages/MergeTree/RPNBuilder.cpp
index 4a18d606bb7..915a0e84902 100644
--- a/src/Storages/MergeTree/RPNBuilder.cpp
+++ b/src/Storages/MergeTree/RPNBuilder.cpp
@@ -398,7 +398,7 @@ size_t RPNBuilderFunctionTreeNode::getArgumentsSize() const
         {
             const auto * adaptor = typeid_cast<const FunctionToFunctionBaseAdaptor *>(dag_node->function_base.get());
             const auto * index_hint = typeid_cast<const FunctionIndexHint *>(adaptor->getFunction().get());
-            return index_hint->getActions()->getOutputs().size();
+            return index_hint->getActions().getOutputs().size();
         }
 
         return dag_node->children.size();
@@ -426,7 +426,7 @@ RPNBuilderTreeNode RPNBuilderFunctionTreeNode::getArgumentAt(size_t index) const
         {
             const auto & adaptor = typeid_cast<const FunctionToFunctionBaseAdaptor &>(*dag_node->function_base);
             const auto & index_hint = typeid_cast<const FunctionIndexHint &>(*adaptor.getFunction());
-            return RPNBuilderTreeNode(index_hint.getActions()->getOutputs()[index], tree_context);
+            return RPNBuilderTreeNode(index_hint.getActions().getOutputs()[index], tree_context);
         }
 
         return RPNBuilderTreeNode(dag_node->children[index], tree_context);
diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp
index 503f542f2bd..d114608d8f1 100644
--- a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp
+++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp
@@ -77,9 +77,9 @@ StorageObjectStorageSource::~StorageObjectStorageSource()
     create_reader_pool->wait();
 }
 
-void StorageObjectStorageSource::setKeyCondition(const ActionsDAGPtr & filter_actions_dag, ContextPtr context_)
+void StorageObjectStorageSource::setKeyCondition(const std::optional<ActionsDAG> & filter_actions_dag, ContextPtr context_)
 {
-    setKeyConditionImpl(filter_actions_dag.get(), context_, read_from_format_info.format_header);
+    setKeyConditionImpl(filter_actions_dag, context_, read_from_format_info.format_header);
 }
 
 std::string StorageObjectStorageSource::getUniqueStoragePathIdentifier(
diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.h b/src/Storages/ObjectStorage/StorageObjectStorageSource.h
index fd7c7aa7102..01ce980feaa 100644
--- a/src/Storages/ObjectStorage/StorageObjectStorageSource.h
+++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.h
@@ -45,7 +45,7 @@ public:
 
     String getName() const override { return name; }
 
-    void setKeyCondition(const ActionsDAGPtr & filter_actions_dag, ContextPtr context_) override;
+    void setKeyCondition(const std::optional<ActionsDAG> & filter_actions_dag, ContextPtr context_) override;
 
     Chunk generate() override;
 
diff --git a/src/Storages/ReadFinalForExternalReplicaStorage.cpp b/src/Storages/ReadFinalForExternalReplicaStorage.cpp
index e1d52eefc20..393d3f3fbb9 100644
--- a/src/Storages/ReadFinalForExternalReplicaStorage.cpp
+++ b/src/Storages/ReadFinalForExternalReplicaStorage.cpp
@@ -79,7 +79,7 @@ void readFinalFromNestedStorage(
 
         auto step = std::make_unique<FilterStep>(
             query_plan.getCurrentDataStream(),
-            actions,
+            std::move(actions),
             filter_column_name,
             false);
 
diff --git a/src/Storages/SelectQueryInfo.h b/src/Storages/SelectQueryInfo.h
index 5f48d5e795e..5276870c037 100644
--- a/src/Storages/SelectQueryInfo.h
+++ b/src/Storages/SelectQueryInfo.h
@@ -46,9 +46,9 @@ struct PrewhereInfo
 {
     /// Actions for row level security filter. Applied separately before prewhere_actions.
     /// This actions are separate because prewhere condition should not be executed over filtered rows.
-    ActionsDAGPtr row_level_filter;
+    std::optional<ActionsDAG> row_level_filter;
     /// Actions which are executed on block in order to get filter column for prewhere step.
-    ActionsDAGPtr prewhere_actions;
+    std::optional<ActionsDAG> prewhere_actions;
     String row_level_column_name;
     String prewhere_column_name;
     bool remove_prewhere_column = false;
@@ -56,7 +56,7 @@ struct PrewhereInfo
     bool generated_by_optimizer = false;
 
     PrewhereInfo() = default;
-    explicit PrewhereInfo(ActionsDAGPtr prewhere_actions_, String prewhere_column_name_)
+    explicit PrewhereInfo(std::optional<ActionsDAG> prewhere_actions_, String prewhere_column_name_)
             : prewhere_actions(std::move(prewhere_actions_)), prewhere_column_name(std::move(prewhere_column_name_)) {}
 
     std::string dump() const;
@@ -66,10 +66,10 @@ struct PrewhereInfo
         PrewhereInfoPtr prewhere_info = std::make_shared<PrewhereInfo>();
 
         if (row_level_filter)
-            prewhere_info->row_level_filter = ActionsDAG::clone(row_level_filter);
+            prewhere_info->row_level_filter = std::move(*ActionsDAG::clone(&*row_level_filter));
 
         if (prewhere_actions)
-            prewhere_info->prewhere_actions = ActionsDAG::clone(prewhere_actions);
+            prewhere_info->prewhere_actions = std::move(*ActionsDAG::clone(&*prewhere_actions));
 
         prewhere_info->row_level_column_name = row_level_column_name;
         prewhere_info->prewhere_column_name = prewhere_column_name;
@@ -93,7 +93,7 @@ struct FilterInfo
 /// Same as FilterInfo, but with ActionsDAG.
 struct FilterDAGInfo
 {
-    ActionsDAGPtr actions;
+    std::optional<ActionsDAG> actions;
     String column_name;
     bool do_remove_column = false;
 
diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp
index 695b31d0c80..fdddd84ab59 100644
--- a/src/Storages/StorageBuffer.cpp
+++ b/src/Storages/StorageBuffer.cpp
@@ -312,7 +312,7 @@ void StorageBuffer::read(
                     if (src_table_query_info.prewhere_info->row_level_filter)
                     {
                         src_table_query_info.prewhere_info->row_level_filter = ActionsDAG::merge(
-                            std::move(*ActionsDAG::clone(actions_dag)),
+                            std::move(*ActionsDAG::clone(&actions_dag)),
                             std::move(*src_table_query_info.prewhere_info->row_level_filter));
 
                         src_table_query_info.prewhere_info->row_level_filter->removeUnusedActions();
@@ -321,7 +321,7 @@ void StorageBuffer::read(
                     if (src_table_query_info.prewhere_info->prewhere_actions)
                     {
                         src_table_query_info.prewhere_info->prewhere_actions = ActionsDAG::merge(
-                            std::move(*ActionsDAG::clone(actions_dag)),
+                            std::move(*ActionsDAG::clone(&actions_dag)),
                             std::move(*src_table_query_info.prewhere_info->prewhere_actions));
 
                         src_table_query_info.prewhere_info->prewhere_actions->removeUnusedActions();
@@ -353,7 +353,7 @@ void StorageBuffer::read(
                             header.getColumnsWithTypeAndName(),
                             ActionsDAG::MatchColumnsMode::Name);
 
-                    auto converting = std::make_unique<ExpressionStep>(query_plan.getCurrentDataStream(), actions_dag);
+                    auto converting = std::make_unique<ExpressionStep>(query_plan.getCurrentDataStream(), std::move(actions_dag));
 
                     converting->setStepDescription("Convert destination table columns to Buffer table structure");
                     query_plan.addStep(std::move(converting));
@@ -432,7 +432,7 @@ void StorageBuffer::read(
                 {
                     return std::make_shared<FilterTransform>(
                             header,
-                            std::make_shared<ExpressionActions>(ActionsDAG::clone(query_info.prewhere_info->row_level_filter), actions_settings),
+                            std::make_shared<ExpressionActions>(std::move(*ActionsDAG::clone(&*query_info.prewhere_info->row_level_filter)), actions_settings),
                             query_info.prewhere_info->row_level_column_name,
                             false);
                 });
@@ -442,7 +442,7 @@ void StorageBuffer::read(
             {
                 return std::make_shared<FilterTransform>(
                         header,
-                        std::make_shared<ExpressionActions>(ActionsDAG::clone(query_info.prewhere_info->prewhere_actions), actions_settings),
+                        std::make_shared<ExpressionActions>(std::move(*ActionsDAG::clone(&*query_info.prewhere_info->prewhere_actions)), actions_settings),
                         query_info.prewhere_info->prewhere_column_name,
                         query_info.prewhere_info->remove_prewhere_column);
             });
@@ -472,7 +472,7 @@ void StorageBuffer::read(
                 result_header.getColumnsWithTypeAndName(),
                 ActionsDAG::MatchColumnsMode::Name);
 
-        auto converting = std::make_unique<ExpressionStep>(query_plan.getCurrentDataStream(), convert_actions_dag);
+        auto converting = std::make_unique<ExpressionStep>(query_plan.getCurrentDataStream(), std::move(convert_actions_dag));
         query_plan.addStep(std::move(converting));
     }
 
diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp
index 3d91da240cc..6f8a9189941 100644
--- a/src/Storages/StorageDistributed.cpp
+++ b/src/Storages/StorageDistributed.cpp
@@ -1074,7 +1074,7 @@ std::optional<QueryPipeline> StorageDistributed::distributedWriteBetweenDistribu
     return pipeline;
 }
 
-static ActionsDAGPtr getFilterFromQuery(const ASTPtr & ast, ContextPtr context)
+static std::optional<ActionsDAG> getFilterFromQuery(const ASTPtr & ast, ContextPtr context)
 {
     QueryPlan plan;
     SelectQueryOptions options;
@@ -1118,7 +1118,7 @@ static ActionsDAGPtr getFilterFromQuery(const ASTPtr & ast, ContextPtr context)
     }
 
     if (!source)
-        return nullptr;
+        return {};
 
     return source->detachFilterActionsDAG();
 }
diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp
index 2422bcd700b..4611371a471 100644
--- a/src/Storages/StorageFile.cpp
+++ b/src/Storages/StorageFile.cpp
@@ -1233,9 +1233,9 @@ StorageFileSource::~StorageFileSource()
     beforeDestroy();
 }
 
-void StorageFileSource::setKeyCondition(const ActionsDAGPtr & filter_actions_dag, ContextPtr context_)
+void StorageFileSource::setKeyCondition(const std::optional<ActionsDAG> & filter_actions_dag, ContextPtr context_)
 {
-    setKeyConditionImpl(filter_actions_dag.get(), context_, block_for_format);
+    setKeyConditionImpl(filter_actions_dag, context_, block_for_format);
 }
 
 
diff --git a/src/Storages/StorageFile.h b/src/Storages/StorageFile.h
index ac094aeb489..e9424527997 100644
--- a/src/Storages/StorageFile.h
+++ b/src/Storages/StorageFile.h
@@ -265,7 +265,7 @@ private:
         return storage->getName();
     }
 
-    void setKeyCondition(const ActionsDAGPtr & filter_actions_dag, ContextPtr context_) override;
+    void setKeyCondition(const std::optional<ActionsDAG> & filter_actions_dag, ContextPtr context_) override;
 
     bool tryGetCountFromCache(const struct stat & file_stat);
 
diff --git a/src/Storages/StorageMaterializedView.cpp b/src/Storages/StorageMaterializedView.cpp
index 316f398b476..8b6a9a4d4bb 100644
--- a/src/Storages/StorageMaterializedView.cpp
+++ b/src/Storages/StorageMaterializedView.cpp
@@ -273,8 +273,8 @@ void StorageMaterializedView::read(
              * They may be added in case of distributed query with JOIN.
              * In that case underlying table returns joined columns as well.
              */
-            converting_actions->removeUnusedActions();
-            auto converting_step = std::make_unique<ExpressionStep>(query_plan.getCurrentDataStream(), converting_actions);
+            converting_actions.removeUnusedActions();
+            auto converting_step = std::make_unique<ExpressionStep>(query_plan.getCurrentDataStream(), std::move(converting_actions));
             converting_step->setStepDescription("Convert target table structure to MaterializedView structure");
             query_plan.addStep(std::move(converting_step));
         }
diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp
index c3fdad3a8f2..374abd0b0a5 100644
--- a/src/Storages/StorageMerge.cpp
+++ b/src/Storages/StorageMerge.cpp
@@ -407,7 +407,7 @@ void ReadFromMerge::addFilter(FilterDAGInfo filter)
 {
     output_stream->header = FilterTransform::transformHeader(
             output_stream->header,
-            filter.actions.get(),
+            filter.actions ? &*filter.actions : nullptr,
             filter.column_name,
             filter.do_remove_column);
     pushed_down_filters.push_back(std::move(filter));
@@ -628,7 +628,7 @@ std::vector<ReadFromMerge::ChildPlan> ReadFromMerge::createChildrenPlans(SelectQ
 
                 auto alias_actions = ExpressionAnalyzer(required_columns_expr_list, syntax_result, context).getActionsDAG(true);
 
-                column_names_as_aliases = alias_actions->getRequiredColumns().getNames();
+                column_names_as_aliases = alias_actions.getRequiredColumns().getNames();
                 if (column_names_as_aliases.empty())
                     column_names_as_aliases.push_back(ExpressionActions::getSmallestColumn(storage_metadata_snapshot->getColumns().getAllPhysical()).name);
             }
@@ -662,7 +662,7 @@ std::vector<ReadFromMerge::ChildPlan> ReadFromMerge::createChildrenPlans(SelectQ
             {
                 auto filter_step = std::make_unique<FilterStep>(
                     child.plan.getCurrentDataStream(),
-                    ActionsDAG::clone(filter_info.actions),
+                    std::move(*ActionsDAG::clone(&*filter_info.actions)),
                     filter_info.column_name,
                     filter_info.do_remove_column);
 
@@ -1060,7 +1060,7 @@ void ReadFromMerge::addVirtualColumns(
             column.column = column.type->createColumnConst(0, Field(database_name));
 
             auto adding_column_dag = ActionsDAG::makeAddingColumnActions(std::move(column));
-            auto expression_step = std::make_unique<ExpressionStep>(child.plan.getCurrentDataStream(), adding_column_dag);
+            auto expression_step = std::make_unique<ExpressionStep>(child.plan.getCurrentDataStream(), std::move(adding_column_dag));
             child.plan.addStep(std::move(expression_step));
             plan_header = child.plan.getCurrentDataStream().header;
         }
@@ -1074,7 +1074,7 @@ void ReadFromMerge::addVirtualColumns(
             column.column = column.type->createColumnConst(0, Field(table_name));
 
             auto adding_column_dag = ActionsDAG::makeAddingColumnActions(std::move(column));
-            auto expression_step = std::make_unique<ExpressionStep>(child.plan.getCurrentDataStream(), adding_column_dag);
+            auto expression_step = std::make_unique<ExpressionStep>(child.plan.getCurrentDataStream(), std::move(adding_column_dag));
             child.plan.addStep(std::move(expression_step));
             plan_header = child.plan.getCurrentDataStream().header;
         }
@@ -1089,7 +1089,7 @@ void ReadFromMerge::addVirtualColumns(
             column.column = column.type->createColumnConst(0, Field(database_name));
 
             auto adding_column_dag = ActionsDAG::makeAddingColumnActions(std::move(column));
-            auto expression_step = std::make_unique<ExpressionStep>(child.plan.getCurrentDataStream(), adding_column_dag);
+            auto expression_step = std::make_unique<ExpressionStep>(child.plan.getCurrentDataStream(), std::move(adding_column_dag));
             child.plan.addStep(std::move(expression_step));
             plan_header = child.plan.getCurrentDataStream().header;
         }
@@ -1102,7 +1102,7 @@ void ReadFromMerge::addVirtualColumns(
             column.column = column.type->createColumnConst(0, Field(table_name));
 
             auto adding_column_dag = ActionsDAG::makeAddingColumnActions(std::move(column));
-            auto expression_step = std::make_unique<ExpressionStep>(child.plan.getCurrentDataStream(), adding_column_dag);
+            auto expression_step = std::make_unique<ExpressionStep>(child.plan.getCurrentDataStream(), std::move(adding_column_dag));
             child.plan.addStep(std::move(expression_step));
             plan_header = child.plan.getCurrentDataStream().header;
         }
@@ -1240,7 +1240,7 @@ ReadFromMerge::RowPolicyData::RowPolicyData(RowPolicyFilterPtr row_policy_filter
     auto expression_analyzer = ExpressionAnalyzer{expr, syntax_result, local_context};
 
     actions_dag = expression_analyzer.getActionsDAG(false /* add_aliases */, false /* project_result */);
-    filter_actions = std::make_shared<ExpressionActions>(ActionsDAG::clone(actions_dag),
+    filter_actions = std::make_shared<ExpressionActions>(std::move(*ActionsDAG::clone(&actions_dag)),
         ExpressionActionsSettings::fromContext(local_context, CompileExpressions::yes));
     const auto & required_columns = filter_actions->getRequiredColumnsWithTypes();
     const auto & sample_block_columns = filter_actions->getSampleBlock().getNamesAndTypesList();
@@ -1278,12 +1278,12 @@ void ReadFromMerge::RowPolicyData::extendNames(Names & names) const
 
 void ReadFromMerge::RowPolicyData::addStorageFilter(SourceStepWithFilter * step) const
 {
-    step->addFilter(ActionsDAG::clone(actions_dag), filter_column_name);
+    step->addFilter(ActionsDAG::clone(&actions_dag), filter_column_name);
 }
 
 void ReadFromMerge::RowPolicyData::addFilterTransform(QueryPlan & plan) const
 {
-    auto filter_step = std::make_unique<FilterStep>(plan.getCurrentDataStream(), ActionsDAG::clone(actions_dag), filter_column_name, true /* remove filter column */);
+    auto filter_step = std::make_unique<FilterStep>(plan.getCurrentDataStream(), std::move(*ActionsDAG::clone(&actions_dag)), filter_column_name, true /* remove filter column */);
     plan.addStep(std::move(filter_step));
 }
 
@@ -1476,7 +1476,7 @@ void ReadFromMerge::convertAndFilterSourceStream(
         {
             pipe_columns.emplace_back(NameAndTypePair(alias.name, alias.type));
 
-            auto actions_dag = std::make_unique<ActionsDAG>(pipe_columns);
+            ActionsDAG actions_dag(pipe_columns);
 
             QueryTreeNodePtr query_tree = buildQueryTree(alias.expression, local_context);
             query_tree->setAlias(alias.name);
@@ -1485,12 +1485,12 @@ void ReadFromMerge::convertAndFilterSourceStream(
             query_analysis_pass.run(query_tree, local_context);
 
             PlannerActionsVisitor actions_visitor(modified_query_info.planner_context, false /*use_column_identifier_as_action_node_name*/);
-            const auto & nodes = actions_visitor.visit(*actions_dag, query_tree);
+            const auto & nodes = actions_visitor.visit(actions_dag, query_tree);
 
             if (nodes.size() != 1)
                 throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected to have 1 output but got {}", nodes.size());
 
-            actions_dag->addOrReplaceInOutputs(actions_dag->addAlias(*nodes.front(), alias.name));
+            actions_dag.addOrReplaceInOutputs(actions_dag.addAlias(*nodes.front(), alias.name));
             auto expression_step = std::make_unique<ExpressionStep>(child.plan.getCurrentDataStream(), std::move(actions_dag));
             child.plan.addStep(std::move(expression_step));
         }
@@ -1506,7 +1506,7 @@ void ReadFromMerge::convertAndFilterSourceStream(
 
             auto dag = std::make_shared<ActionsDAG>(pipe_columns);
             auto actions_dag = expression_analyzer.getActionsDAG(true, false);
-            auto expression_step = std::make_unique<ExpressionStep>(child.plan.getCurrentDataStream(), actions_dag);
+            auto expression_step = std::make_unique<ExpressionStep>(child.plan.getCurrentDataStream(), std::move(actions_dag));
             child.plan.addStep(std::move(expression_step));
         }
     }
@@ -1524,7 +1524,7 @@ void ReadFromMerge::convertAndFilterSourceStream(
                                                                 header.getColumnsWithTypeAndName(),
                                                                 convert_actions_match_columns_mode);
 
-    auto expression_step = std::make_unique<ExpressionStep>(child.plan.getCurrentDataStream(), convert_actions_dag);
+    auto expression_step = std::make_unique<ExpressionStep>(child.plan.getCurrentDataStream(), std::move(convert_actions_dag));
     child.plan.addStep(std::move(expression_step));
 }
 
diff --git a/src/Storages/StorageMerge.h b/src/Storages/StorageMerge.h
index 94b34256d02..d6f2deca7fd 100644
--- a/src/Storages/StorageMerge.h
+++ b/src/Storages/StorageMerge.h
@@ -225,7 +225,7 @@ private:
 
     private:
         std::string filter_column_name; // complex filter, may contain logic operations
-        ActionsDAGPtr actions_dag;
+        ActionsDAG actions_dag;
         ExpressionActionsPtr filter_actions;
         StorageMetadataPtr storage_metadata_snapshot;
     };
diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp
index 9352f772ce1..b1a8a81914c 100644
--- a/src/Storages/StorageMergeTree.cpp
+++ b/src/Storages/StorageMergeTree.cpp
@@ -245,7 +245,7 @@ std::optional<UInt64> StorageMergeTree::totalRows(const Settings &) const
     return getTotalActiveSizeInRows();
 }
 
-std::optional<UInt64> StorageMergeTree::totalRowsByPartitionPredicate(const ActionsDAGPtr & filter_actions_dag, ContextPtr local_context) const
+std::optional<UInt64> StorageMergeTree::totalRowsByPartitionPredicate(const ActionsDAG & filter_actions_dag, ContextPtr local_context) const
 {
     auto parts = getVisibleDataPartsVector(local_context);
     return totalRowsByPartitionPredicateImpl(filter_actions_dag, local_context, parts);
diff --git a/src/Storages/StorageMergeTree.h b/src/Storages/StorageMergeTree.h
index 4d819508934..56324449b34 100644
--- a/src/Storages/StorageMergeTree.h
+++ b/src/Storages/StorageMergeTree.h
@@ -65,7 +65,7 @@ public:
         size_t num_streams) override;
 
     std::optional<UInt64> totalRows(const Settings &) const override;
-    std::optional<UInt64> totalRowsByPartitionPredicate(const ActionsDAGPtr & filter_actions_dag, ContextPtr) const override;
+    std::optional<UInt64> totalRowsByPartitionPredicate(const ActionsDAG & filter_actions_dag, ContextPtr) const override;
     std::optional<UInt64> totalBytes(const Settings &) const override;
     std::optional<UInt64> totalBytesUncompressed(const Settings &) const override;
 
diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp
index db58d0081c6..b472710b6d8 100644
--- a/src/Storages/StorageReplicatedMergeTree.cpp
+++ b/src/Storages/StorageReplicatedMergeTree.cpp
@@ -5556,7 +5556,7 @@ std::optional<UInt64> StorageReplicatedMergeTree::totalRows(const Settings & set
     return res;
 }
 
-std::optional<UInt64> StorageReplicatedMergeTree::totalRowsByPartitionPredicate(const ActionsDAGPtr & filter_actions_dag, ContextPtr local_context) const
+std::optional<UInt64> StorageReplicatedMergeTree::totalRowsByPartitionPredicate(const ActionsDAG & filter_actions_dag, ContextPtr local_context) const
 {
     DataPartsVector parts;
     foreachActiveParts([&](auto & part) { parts.push_back(part); }, local_context->getSettingsRef().select_sequential_consistency);
diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h
index f96206ce657..2e54f17d5d5 100644
--- a/src/Storages/StorageReplicatedMergeTree.h
+++ b/src/Storages/StorageReplicatedMergeTree.h
@@ -159,7 +159,7 @@ public:
         size_t num_streams) override;
 
     std::optional<UInt64> totalRows(const Settings & settings) const override;
-    std::optional<UInt64> totalRowsByPartitionPredicate(const ActionsDAGPtr & filter_actions_dag, ContextPtr context) const override;
+    std::optional<UInt64> totalRowsByPartitionPredicate(const ActionsDAG & filter_actions_dag, ContextPtr context) const override;
     std::optional<UInt64> totalBytes(const Settings & settings) const override;
     std::optional<UInt64> totalBytesUncompressed(const Settings & settings) const override;
 
diff --git a/src/Storages/StorageTableFunction.h b/src/Storages/StorageTableFunction.h
index 9507eb6ed8a..345dd62c687 100644
--- a/src/Storages/StorageTableFunction.h
+++ b/src/Storages/StorageTableFunction.h
@@ -112,7 +112,7 @@ public:
 
             auto step = std::make_unique<ExpressionStep>(
                 query_plan.getCurrentDataStream(),
-                convert_actions_dag);
+                std::move(convert_actions_dag));
 
             step->setStepDescription("Converting columns");
             query_plan.addStep(std::move(step));
diff --git a/src/Storages/StorageURL.h b/src/Storages/StorageURL.h
index c336f597f41..ec1f803750e 100644
--- a/src/Storages/StorageURL.h
+++ b/src/Storages/StorageURL.h
@@ -185,9 +185,9 @@ public:
 
     String getName() const override { return name; }
 
-    void setKeyCondition(const ActionsDAGPtr & filter_actions_dag, ContextPtr context_) override
+    void setKeyCondition(const std::optional<ActionsDAG> & filter_actions_dag, ContextPtr context_) override
     {
-        setKeyConditionImpl(filter_actions_dag.get(), context_, block_for_format);
+        setKeyConditionImpl(filter_actions_dag, context_, block_for_format);
     }
 
     Chunk generate() override;
diff --git a/src/Storages/StorageValues.cpp b/src/Storages/StorageValues.cpp
index 4d73f8e5c87..c1ca6244866 100644
--- a/src/Storages/StorageValues.cpp
+++ b/src/Storages/StorageValues.cpp
@@ -48,13 +48,13 @@ Pipe StorageValues::read(
 
     if (!prepared_pipe.empty())
     {
-        auto dag = std::make_unique<ActionsDAG>(prepared_pipe.getHeader().getColumnsWithTypeAndName());
+        ActionsDAG dag(prepared_pipe.getHeader().getColumnsWithTypeAndName());
         ActionsDAG::NodeRawConstPtrs outputs;
         outputs.reserve(column_names.size());
         for (const auto & name : column_names)
-            outputs.push_back(dag->getOutputs()[prepared_pipe.getHeader().getPositionByName(name)]);
+            outputs.push_back(dag.getOutputs()[prepared_pipe.getHeader().getPositionByName(name)]);
 
-        dag->getOutputs().swap(outputs);
+        dag.getOutputs().swap(outputs);
         auto expression = std::make_shared<ExpressionActions>(std::move(dag));
 
         prepared_pipe.addSimpleTransform([&](const Block & header)
diff --git a/src/Storages/StorageView.cpp b/src/Storages/StorageView.cpp
index 2c0d5c5ca85..e2c4d67c8d1 100644
--- a/src/Storages/StorageView.cpp
+++ b/src/Storages/StorageView.cpp
@@ -177,8 +177,8 @@ void StorageView::read(
 
     /// It's expected that the columns read from storage are not constant.
     /// Because method 'getSampleBlockForColumns' is used to obtain a structure of result in InterpreterSelectQuery.
-    auto materializing_actions = std::make_unique<ActionsDAG>(query_plan.getCurrentDataStream().header.getColumnsWithTypeAndName());
-    materializing_actions->addMaterializingOutputActions();
+    ActionsDAG materializing_actions(query_plan.getCurrentDataStream().header.getColumnsWithTypeAndName());
+    materializing_actions.addMaterializingOutputActions();
 
     auto materializing = std::make_unique<ExpressionStep>(query_plan.getCurrentDataStream(), std::move(materializing_actions));
     materializing->setStepDescription("Materialize constants after VIEW subquery");
@@ -203,7 +203,7 @@ void StorageView::read(
             expected_header.getColumnsWithTypeAndName(),
             ActionsDAG::MatchColumnsMode::Name);
 
-    auto converting = std::make_unique<ExpressionStep>(query_plan.getCurrentDataStream(), convert_actions_dag);
+    auto converting = std::make_unique<ExpressionStep>(query_plan.getCurrentDataStream(), std::move(convert_actions_dag));
     converting->setStepDescription("Convert VIEW subquery result to VIEW table structure");
     query_plan.addStep(std::move(converting));
 }
diff --git a/src/Storages/System/StorageSystemStackTrace.cpp b/src/Storages/System/StorageSystemStackTrace.cpp
index ba7433fb9ae..f4e6fe3df5f 100644
--- a/src/Storages/System/StorageSystemStackTrace.cpp
+++ b/src/Storages/System/StorageSystemStackTrace.cpp
@@ -276,7 +276,7 @@ public:
     StackTraceSource(
         const Names & column_names,
         Block header_,
-        ActionsDAGPtr && filter_dag_,
+        std::optional<ActionsDAG> filter_dag_,
         ContextPtr context_,
         UInt64 max_block_size_,
         LoggerPtr log_)
@@ -422,7 +422,7 @@ protected:
 private:
     ContextPtr context;
     Block header;
-    const ActionsDAGPtr filter_dag;
+    const std::optional<ActionsDAG> filter_dag;
     const ActionsDAG::Node * predicate;
 
     const size_t max_block_size;
diff --git a/src/Storages/TTLDescription.cpp b/src/Storages/TTLDescription.cpp
index 56f65b57367..a32eef20aed 100644
--- a/src/Storages/TTLDescription.cpp
+++ b/src/Storages/TTLDescription.cpp
@@ -176,12 +176,12 @@ static ExpressionAndSets buildExpressionAndSets(ASTPtr & ast, const NamesAndType
     ExpressionAnalyzer analyzer(ast, syntax_analyzer_result, context_copy);
     auto dag = analyzer.getActionsDAG(false);
 
-    const auto * col = &dag->findInOutputs(ast->getColumnName());
+    const auto * col = &dag.findInOutputs(ast->getColumnName());
     if (col->result_name != ttl_string)
-        col = &dag->addAlias(*col, ttl_string);
+        col = &dag.addAlias(*col, ttl_string);
 
-    dag->getOutputs() = {col};
-    dag->removeUnusedActions();
+    dag.getOutputs() = {col};
+    dag.removeUnusedActions();
 
     result.expression = std::make_shared<ExpressionActions>(std::move(dag), ExpressionActionsSettings::fromContext(context_copy));
     result.sets = analyzer.getPreparedSets();
diff --git a/src/Storages/VirtualColumnUtils.cpp b/src/Storages/VirtualColumnUtils.cpp
index 1bd5e80a4f9..7f54c6a6ee3 100644
--- a/src/Storages/VirtualColumnUtils.cpp
+++ b/src/Storages/VirtualColumnUtils.cpp
@@ -80,7 +80,7 @@ void buildSetsForDAG(const ActionsDAGPtr & dag, const ContextPtr & context)
 void filterBlockWithDAG(const ActionsDAGPtr & dag, Block & block, ContextPtr context)
 {
     buildSetsForDAG(dag, context);
-    auto actions = std::make_shared<ExpressionActions>(ActionsDAG::clone(dag));
+    auto actions = std::make_shared<ExpressionActions>(std::move(*ActionsDAG::clone(dag)));
     Block block_with_filter = block;
     actions->execute(block_with_filter, /*dry_run=*/ false, /*allow_duplicates_in_input=*/ true);
 
@@ -318,9 +318,9 @@ static const ActionsDAG::Node * splitFilterNodeForAllowedInputs(
             {
                 if (const auto * index_hint = typeid_cast<const FunctionIndexHint *>(adaptor->getFunction().get()))
                 {
-                    auto index_hint_dag = ActionsDAG::clone(index_hint->getActions());
+                    auto index_hint_dag = std::move(*ActionsDAG::clone(&index_hint->getActions()));
                     ActionsDAG::NodeRawConstPtrs atoms;
-                    for (const auto & output : index_hint_dag->getOutputs())
+                    for (const auto & output : index_hint_dag.getOutputs())
                         if (const auto * child_copy = splitFilterNodeForAllowedInputs(output, allowed_inputs, additional_nodes))
                             atoms.push_back(child_copy);
 
@@ -331,13 +331,13 @@ static const ActionsDAG::Node * splitFilterNodeForAllowedInputs(
                         if (atoms.size() > 1)
                         {
                             FunctionOverloadResolverPtr func_builder_and = std::make_unique<FunctionToOverloadResolverAdaptor>(std::make_shared<FunctionAnd>());
-                            res = &index_hint_dag->addFunction(func_builder_and, atoms, {});
+                            res = &index_hint_dag.addFunction(func_builder_and, atoms, {});
                         }
 
                         if (!res->result_type->equals(*node->result_type))
-                            res = &index_hint_dag->addCast(*res, node->result_type, {});
+                            res = &index_hint_dag.addCast(*res, node->result_type, {});
 
-                        additional_nodes.splice(additional_nodes.end(), ActionsDAG::detachNodes(std::move(*index_hint_dag)));
+                        additional_nodes.splice(additional_nodes.end(), ActionsDAG::detachNodes(std::move(index_hint_dag)));
                         return res;
                     }
                 }
diff --git a/src/Storages/WindowView/StorageWindowView.cpp b/src/Storages/WindowView/StorageWindowView.cpp
index 8f39f0da5af..30ae1f95593 100644
--- a/src/Storages/WindowView/StorageWindowView.cpp
+++ b/src/Storages/WindowView/StorageWindowView.cpp
@@ -1130,7 +1130,7 @@ void StorageWindowView::read(
         {
             auto converting_actions = ActionsDAG::makeConvertingActions(
                 target_header.getColumnsWithTypeAndName(), wv_header.getColumnsWithTypeAndName(), ActionsDAG::MatchColumnsMode::Name);
-            auto converting_step = std::make_unique<ExpressionStep>(query_plan.getCurrentDataStream(), converting_actions);
+            auto converting_step = std::make_unique<ExpressionStep>(query_plan.getCurrentDataStream(), std::move(converting_actions));
             converting_step->setStepDescription("Convert Target table structure to WindowView structure");
             query_plan.addStep(std::move(converting_step));
         }

From 4f510c588a34473531101c5b2e2fc9e26d18c1c9 Mon Sep 17 00:00:00 2001
From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com>
Date: Fri, 5 Jul 2024 20:49:47 +0200
Subject: [PATCH 0161/1488] quick workaround

---
 .../AggregateFunctionGroupArrayIntersect.cpp                | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/AggregateFunctions/AggregateFunctionGroupArrayIntersect.cpp b/src/AggregateFunctions/AggregateFunctionGroupArrayIntersect.cpp
index 0da5ade1c79..4f45d532179 100644
--- a/src/AggregateFunctions/AggregateFunctionGroupArrayIntersect.cpp
+++ b/src/AggregateFunctions/AggregateFunctionGroupArrayIntersect.cpp
@@ -111,7 +111,7 @@ public:
             return;
 
         UInt64 version = this->data(place).version++;
-        if (version == 1)
+        if (version == 0)
         {
             for (auto & rhs_elem : rhs_set)
                 set.insert(rhs_elem.getValue());
@@ -278,14 +278,14 @@ public:
             return;
 
         UInt64 version = this->data(place).version++;
-        if (version == 1)
+        if (version == 1 && set.empty())
         {
             bool inserted;
             State::Set::LookupResult it;
             for (auto & rhs_elem : rhs_value)
                 set.emplace(ArenaKeyHolder{rhs_elem.getValue(), *arena}, it, inserted);
         }
-        else if (!set.empty())
+        else
         {
             auto create_matched_set = [](auto & lhs_val, auto & rhs_val)
             {

From 17c6b97cbcb90e19c236708116f20ef3f88cc9c1 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 6 Jul 2024 03:16:13 +0200
Subject: [PATCH 0162/1488] Fix error

---
 programs/local/LocalServer.cpp | 7 ++++++-
 src/Databases/IDatabase.h      | 1 +
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp
index 41bb5604a52..269bffc2d56 100644
--- a/programs/local/LocalServer.cpp
+++ b/programs/local/LocalServer.cpp
@@ -737,7 +737,12 @@ void LocalServer::processConfig()
     DatabaseCatalog::instance().initializeAndLoadTemporaryDatabase();
 
     std::string default_database = server_settings.default_database;
-    DatabaseCatalog::instance().attachDatabase(default_database, createClickHouseLocalDatabaseOverlay(default_database, global_context));
+    {
+        DatabasePtr database = createClickHouseLocalDatabaseOverlay(default_database, global_context);
+        if (UUID uuid = database->getUUID(); uuid != UUIDHelpers::Nil)
+            DatabaseCatalog::instance().addUUIDMapping(uuid);
+        DatabaseCatalog::instance().attachDatabase(default_database, std::move(database));
+    }
     global_context->setCurrentDatabase(default_database);
 
     if (getClientConfiguration().has("path"))
diff --git a/src/Databases/IDatabase.h b/src/Databases/IDatabase.h
index b00f2fe4baf..3065c8ae6b5 100644
--- a/src/Databases/IDatabase.h
+++ b/src/Databases/IDatabase.h
@@ -358,6 +358,7 @@ public:
         std::lock_guard lock{mutex};
         return database_name;
     }
+
     /// Get UUID of database.
     virtual UUID getUUID() const { return UUIDHelpers::Nil; }
 

From 376472c8ceffe82a466067c9fc0039517726da62 Mon Sep 17 00:00:00 2001
From: Sema Checherinda <Sema.Checherinda@clickhouse.com>
Date: Fri, 5 Jul 2024 19:12:23 +0200
Subject: [PATCH 0163/1488] add test when chunk with 0 columns has to be
 produced in squashing

---
 src/Core/Settings.h                           |  2 +-
 src/Interpreters/Squashing.cpp                | 36 ++++++++++++-------
 src/Interpreters/Squashing.h                  | 10 +++---
 .../DeduplicationTokenTransforms.cpp          |  4 ++-
 .../0_stateless/01275_parallel_mv.sql.j2      |  6 ++--
 .../03008_deduplication_wrong_mv.reference    | 15 ++++++++
 .../03008_deduplication_wrong_mv.sql          | 21 +++++++++++
 7 files changed, 71 insertions(+), 23 deletions(-)
 create mode 100644 tests/queries/0_stateless/03008_deduplication_wrong_mv.reference
 create mode 100644 tests/queries/0_stateless/03008_deduplication_wrong_mv.sql

diff --git a/src/Core/Settings.h b/src/Core/Settings.h
index 81d0aa0c51d..adf437100b1 100644
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@@ -952,7 +952,7 @@ class IColumn;
 
 #define OBSOLETE_SETTINGS(M, ALIAS) \
     /** Obsolete settings that do nothing but left for compatibility reasons. Remove each one after half a year of obsolescence. */ \
-    MAKE_OBSOLETE(M, Bool, update_insert_deduplication_token_in_dependent_materialized_views, 1) \
+    MAKE_OBSOLETE(M, Bool, update_insert_deduplication_token_in_dependent_materialized_views, 0) \
     MAKE_OBSOLETE(M, UInt64, max_memory_usage_for_all_queries, 0) \
     MAKE_OBSOLETE(M, UInt64, multiple_joins_rewriter_version, 0) \
     MAKE_OBSOLETE(M, Bool, enable_debug_queries, false) \
diff --git a/src/Interpreters/Squashing.cpp b/src/Interpreters/Squashing.cpp
index 25434d1103e..a076494a4bc 100644
--- a/src/Interpreters/Squashing.cpp
+++ b/src/Interpreters/Squashing.cpp
@@ -1,5 +1,7 @@
 #include <vector>
 #include <Interpreters/Squashing.h>
+#include "Common/Logger.h"
+#include "Common/logger_useful.h"
 #include <Common/CurrentThread.h>
 #include <base/defines.h>
 
@@ -16,6 +18,7 @@ Squashing::Squashing(Block header_, size_t min_block_size_rows_, size_t min_bloc
     , min_block_size_bytes(min_block_size_bytes_)
     , header(header_)
 {
+    LOG_TEST(getLogger("Squashing"), "header columns {}", header.columns());
 }
 
 Chunk Squashing::flush()
@@ -23,7 +26,7 @@ Chunk Squashing::flush()
     if (!accumulated)
         return {};
 
-    auto result = convertToChunk(accumulated.extract());
+    auto result = convertToChunk(extract());
     chassert(result);
     return result;
 }
@@ -43,6 +46,8 @@ Chunk Squashing::squash(Chunk && input_chunk)
 
 Chunk Squashing::add(Chunk && input_chunk)
 {
+    LOG_TEST(getLogger("Squashing"), "add columns {} rows {}", input_chunk.getNumColumns(), input_chunk.getNumRows());
+
     if (!input_chunk)
         return {};
 
@@ -53,11 +58,11 @@ Chunk Squashing::add(Chunk && input_chunk)
         if (!accumulated)
         {
             accumulated.add(std::move(input_chunk));
-            return convertToChunk(accumulated.extract());
+            return convertToChunk(extract());
         }
 
         /// Return accumulated data (maybe it has small size) and place new block to accumulated data.
-        Chunk res_chunk = convertToChunk(accumulated.extract());
+        Chunk res_chunk = convertToChunk(extract());
         accumulated.add(std::move(input_chunk));
         return res_chunk;
     }
@@ -66,7 +71,7 @@ Chunk Squashing::add(Chunk && input_chunk)
     if (isEnoughSize())
     {
         /// Return accumulated data and place new block to accumulated data.
-        Chunk res_chunk = convertToChunk(accumulated.extract());
+        Chunk res_chunk = convertToChunk(extract());
         accumulated.add(std::move(input_chunk));
         return res_chunk;
     }
@@ -76,21 +81,25 @@ Chunk Squashing::add(Chunk && input_chunk)
 
     /// If accumulated data is big enough, we send it
     if (isEnoughSize())
-        return convertToChunk(accumulated.extract());
+        return convertToChunk(extract());
 
     return {};
 }
 
-Chunk Squashing::convertToChunk(std::vector<Chunk> && chunks) const
+Chunk Squashing::convertToChunk(CurrentData && data) const
 {
-    if (chunks.empty())
+    LOG_TEST(getLogger("Squashing"), "convertToChunk {}", data.chunks.size());
+
+    if (data.chunks.empty())
         return {};
 
     auto info = std::make_shared<ChunksToSquash>();
-    info->chunks = std::move(chunks);
+    info->chunks = std::move(data.chunks);
 
     // It is imortant that chunk is not empty, it has to have columns even if they are empty
-    auto aggr_chunk = Chunk(header.getColumns(), 0);
+    // Sometimes there are could be no columns in header but not empty rows in chunks
+    // That happens when we intend to add defaults for the missing columns after
+    auto aggr_chunk = Chunk(header.getColumns(), header.columns() ? 0 : data.getRows());
     aggr_chunk.getChunkInfos().add(std::move(info));
     chassert(aggr_chunk);
     return aggr_chunk;
@@ -149,17 +158,18 @@ bool Squashing::isEnoughSize(const Chunk & chunk) const
     return isEnoughSize(chunk.getNumRows(), chunk.bytes());
 }
 
-void Squashing::CurrentSize::add(Chunk && chunk)
+void Squashing::CurrentData::add(Chunk && chunk)
 {
     rows += chunk.getNumRows();
     bytes += chunk.bytes();
     chunks.push_back(std::move(chunk));
 }
 
-std::vector<Chunk> Squashing::CurrentSize::extract()
+Squashing::CurrentData Squashing::extract()
 {
-    auto result = std::move(chunks);
-    *this = {};
+    auto result = std::move(accumulated);
+    accumulated = {};
     return result;
 }
+
 }
diff --git a/src/Interpreters/Squashing.h b/src/Interpreters/Squashing.h
index 64a9768a71f..71ed4c4185a 100644
--- a/src/Interpreters/Squashing.h
+++ b/src/Interpreters/Squashing.h
@@ -49,25 +49,23 @@ public:
     const Block & getHeader() const { return header; }
 
 private:
-    class CurrentSize
+    struct CurrentData
     {
         std::vector<Chunk> chunks = {};
         size_t rows = 0;
         size_t bytes = 0;
 
-    public:
         explicit operator bool () const { return !chunks.empty(); }
         size_t getRows() const { return rows; }
         size_t getBytes() const { return bytes; }
         void add(Chunk && chunk);
-        std::vector<Chunk> extract();
     };
 
     const size_t min_block_size_rows;
     const size_t min_block_size_bytes;
     Block header;
 
-    CurrentSize accumulated;
+    CurrentData accumulated;
 
     static Chunk squash(std::vector<Chunk> && input_chunks, Chunk::ChunkInfoCollection && infos);
 
@@ -75,7 +73,9 @@ private:
     bool isEnoughSize(size_t rows, size_t bytes) const;
     bool isEnoughSize(const Chunk & chunk) const;
 
-    Chunk convertToChunk(std::vector<Chunk> && chunks) const;
+    CurrentData extract();
+
+    Chunk convertToChunk(CurrentData && data) const;
 };
 
 }
diff --git a/src/Processors/Transforms/DeduplicationTokenTransforms.cpp b/src/Processors/Transforms/DeduplicationTokenTransforms.cpp
index 6786f76cbef..e6f7e44e026 100644
--- a/src/Processors/Transforms/DeduplicationTokenTransforms.cpp
+++ b/src/Processors/Transforms/DeduplicationTokenTransforms.cpp
@@ -148,9 +148,11 @@ void CheckTokenTransform::transform(Chunk & chunk)
     auto token_info = chunk.getChunkInfos().get<TokenInfo>();
 
     if (!token_info)
+    {
         throw Exception(ErrorCodes::LOGICAL_ERROR, "Chunk has to have DedupTokenInfo as ChunkInfo, {}", debug);
+    }
 
-    LOG_DEBUG(log, "debug: {}, token: {}", debug, token_info->debugToken());
+    LOG_TEST(log, "debug: {}, token: {}, columns {} rows {}", debug, token_info->debugToken(), chunk.getNumColumns(), chunk.getNumRows());
 }
 #endif
 
diff --git a/tests/queries/0_stateless/01275_parallel_mv.sql.j2 b/tests/queries/0_stateless/01275_parallel_mv.sql.j2
index 9d74474c1a4..bc663bd0e63 100644
--- a/tests/queries/0_stateless/01275_parallel_mv.sql.j2
+++ b/tests/queries/0_stateless/01275_parallel_mv.sql.j2
@@ -12,9 +12,9 @@ drop table if exists testXC;
 
 create table testX (A Int64) engine=MergeTree order by tuple();
 
-create materialized view testXA engine=MergeTree order by tuple() as select sleep(0.1) from testX;
-create materialized view testXB engine=MergeTree order by tuple() as select sleep(0.2), throwIf(A=1) from testX;
-create materialized view testXC engine=MergeTree order by tuple() as select sleep(0.1) from testX;
+create materialized view testXA engine=MergeTree order by tuple() as select sleepEachRow(0.2) from testX;
+create materialized view testXB engine=MergeTree order by tuple() as select sleepEachRow(0.4), throwIf(A=1) from testX;
+create materialized view testXC engine=MergeTree order by tuple() as select sleepEachRow(0.2) from testX;
 
 -- { echoOn }
 {% for parallel_view_processing in [0, 1] %}
diff --git a/tests/queries/0_stateless/03008_deduplication_wrong_mv.reference b/tests/queries/0_stateless/03008_deduplication_wrong_mv.reference
new file mode 100644
index 00000000000..89b4f324d5e
--- /dev/null
+++ b/tests/queries/0_stateless/03008_deduplication_wrong_mv.reference
@@ -0,0 +1,15 @@
+-- { echo ON }
+CREATE TABLE src (x UInt8) ENGINE = Memory;
+CREATE TABLE dst (x UInt8) ENGINE = Memory;
+CREATE MATERIALIZED VIEW mv1 TO dst AS SELECT * FROM src;
+INSERT INTO src VALUES (0);
+SELECT * from dst;
+0
+TRUNCATE TABLE dst;
+--DROP TABLE src SYNC;
+--CREATE TABLE src (y String) ENGINE = MergeTree order by tuple();
+ALTER TABLE src ADD COLUMN y UInt8;
+ALTER TABLE src DROP COLUMN x;
+INSERT INTO src VALUES (0);
+SELECT * from dst;
+0
diff --git a/tests/queries/0_stateless/03008_deduplication_wrong_mv.sql b/tests/queries/0_stateless/03008_deduplication_wrong_mv.sql
new file mode 100644
index 00000000000..dd74c38ab17
--- /dev/null
+++ b/tests/queries/0_stateless/03008_deduplication_wrong_mv.sql
@@ -0,0 +1,21 @@
+DROP TABLE IF EXISTS mv;
+DROP TABLE IF EXISTS src;
+DROP TABLE IF EXISTS dst;
+
+-- { echo ON }
+CREATE TABLE src (x UInt8) ENGINE = Memory;
+CREATE TABLE dst (x UInt8) ENGINE = Memory;
+CREATE MATERIALIZED VIEW mv1 TO dst AS SELECT * FROM src;
+
+INSERT INTO src VALUES (0);
+SELECT * from dst;
+
+TRUNCATE TABLE dst;
+
+--DROP TABLE src SYNC;
+--CREATE TABLE src (y String) ENGINE = MergeTree order by tuple();
+ALTER TABLE src ADD COLUMN y UInt8;
+ALTER TABLE src DROP COLUMN x;
+
+INSERT INTO src VALUES (0);
+SELECT * from dst;

From 1237f93182db21f00df9ca7913619ee63d75850b Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Mon, 8 Jul 2024 15:06:52 +0000
Subject: [PATCH 0164/1488] Fixing some crashes.

---
 src/Interpreters/ExpressionActions.cpp         | 2 +-
 src/Planner/Planner.cpp                        | 8 +++++---
 src/Planner/PlannerJoins.cpp                   | 8 ++++----
 src/Planner/PlannerJoins.h                     | 4 ++--
 src/Processors/QueryPlan/ReadFromMergeTree.cpp | 2 +-
 5 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/src/Interpreters/ExpressionActions.cpp b/src/Interpreters/ExpressionActions.cpp
index 399f4f2ff4f..1c6c3f2556b 100644
--- a/src/Interpreters/ExpressionActions.cpp
+++ b/src/Interpreters/ExpressionActions.cpp
@@ -59,7 +59,7 @@ ExpressionActions::ExpressionActions(ActionsDAG actions_dag_, const ExpressionAc
 
 #if USE_EMBEDDED_COMPILER
     if (settings.can_compile_expressions && settings.compile_expressions == CompileExpressions::yes)
-        actions_dag->compileExpressions(settings.min_count_to_compile_expression, lazy_executed_nodes);
+        actions_dag.compileExpressions(settings.min_count_to_compile_expression, lazy_executed_nodes);
 #endif
 
     linearizeActions(lazy_executed_nodes);
diff --git a/src/Planner/Planner.cpp b/src/Planner/Planner.cpp
index 48e42099ce8..0b10cef82ce 100644
--- a/src/Planner/Planner.cpp
+++ b/src/Planner/Planner.cpp
@@ -212,9 +212,11 @@ FiltersForTableExpressionMap collectFiltersForAnalysis(const QueryTreeNodePtr &
         if (!read_from_dummy)
             continue;
 
-        auto filter_actions = read_from_dummy->detachFilterActionsDAG();
-        const auto & table_node = dummy_storage_to_table.at(&read_from_dummy->getStorage());
-        res[table_node] = FiltersForTableExpression{std::move(filter_actions), read_from_dummy->getPrewhereInfo()};
+        if (auto filter_actions = read_from_dummy->detachFilterActionsDAG())
+        {
+            const auto & table_node = dummy_storage_to_table.at(&read_from_dummy->getStorage());
+            res[table_node] = FiltersForTableExpression{std::move(filter_actions), read_from_dummy->getPrewhereInfo()};
+        }
     }
 
     return res;
diff --git a/src/Planner/PlannerJoins.cpp b/src/Planner/PlannerJoins.cpp
index db9678d91a6..7772336f7c0 100644
--- a/src/Planner/PlannerJoins.cpp
+++ b/src/Planner/PlannerJoins.cpp
@@ -588,11 +588,11 @@ JoinClausesAndActions buildJoinClausesAndActions(
         }
     }
 
-    result.left_join_expressions_actions = std::move(left_join_actions);
-    //result.left_join_tmp_expression_actions = std::move(left_join_actions);
+    result.left_join_expressions_actions = std::move(*ActionsDAG::clone(&left_join_actions));
+    result.left_join_tmp_expression_actions = std::move(left_join_actions);
     result.left_join_expressions_actions.removeUnusedActions(join_left_actions_names);
-    result.right_join_expressions_actions = std::move(right_join_actions);
-    //result.right_join_tmp_expression_actions = std::move(right_join_actions);
+    result.right_join_expressions_actions = std::move(*ActionsDAG::clone(&right_join_actions));
+    result.right_join_tmp_expression_actions = std::move(right_join_actions);
     result.right_join_expressions_actions.removeUnusedActions(join_right_actions_names);
 
     if (is_inequal_join)
diff --git a/src/Planner/PlannerJoins.h b/src/Planner/PlannerJoins.h
index 3735c373acc..d8665ab7739 100644
--- a/src/Planner/PlannerJoins.h
+++ b/src/Planner/PlannerJoins.h
@@ -182,8 +182,8 @@ struct JoinClausesAndActions
     /// Join clauses. Actions dag nodes point into join_expression_actions.
     JoinClauses join_clauses;
     /// Whole JOIN ON section expressions
-    // ActionsDAGPtr left_join_tmp_expression_actions;
-    // ActionsDAGPtr right_join_tmp_expression_actions;
+    ActionsDAG left_join_tmp_expression_actions;
+    ActionsDAG right_join_tmp_expression_actions;
     /// Left join expressions actions
     ActionsDAG left_join_expressions_actions;
     /// Right join expressions actions
diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp
index bc878e7ee49..9ca79fde26f 100644
--- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp
+++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp
@@ -1520,7 +1520,7 @@ void ReadFromMergeTree::applyFilters(ActionDAGNodes added_filter_nodes)
         /// TODO: Get rid of filter_actions_dag in query_info after we move analysis of
         /// parallel replicas and unused shards into optimization, similar to projection analysis.
         if (filter_actions_dag)
-            query_info.filter_actions_dag = std::make_shared<const ActionsDAG>(std::move(*filter_actions_dag));
+            query_info.filter_actions_dag = std::make_shared<const ActionsDAG>(std::move(*ActionsDAG::clone(&*filter_actions_dag)));
 
         buildIndexes(
             indexes,

From 8035582169ae0666a3e024d9329d98e4c3a9b7a9 Mon Sep 17 00:00:00 2001
From: Sema Checherinda <Sema.Checherinda@clickhouse.com>
Date: Mon, 8 Jul 2024 14:57:40 +0200
Subject: [PATCH 0165/1488] adjust test 01275_parallel_mv

---
 src/Interpreters/InterpreterInsertQuery.cpp   | 24 +++++++------------
 .../0_stateless/01275_parallel_mv.sql.j2      | 19 +++++++++------
 2 files changed, 21 insertions(+), 22 deletions(-)

diff --git a/src/Interpreters/InterpreterInsertQuery.cpp b/src/Interpreters/InterpreterInsertQuery.cpp
index 2cbfc55d008..636db546a79 100644
--- a/src/Interpreters/InterpreterInsertQuery.cpp
+++ b/src/Interpreters/InterpreterInsertQuery.cpp
@@ -37,6 +37,7 @@
 #include <Storages/StorageMaterializedView.h>
 #include <Storages/WindowView/StorageWindowView.h>
 #include <TableFunctions/TableFunctionFactory.h>
+#include <Common/logger_useful.h>
 #include <Common/ThreadStatus.h>
 #include <Common/checkStackSize.h>
 #include <Common/ProfileEvents.h>
@@ -600,16 +601,15 @@ QueryPipeline InterpreterInsertQuery::buildInsertSelectPipeline(ASTInsertQuery &
     ///    Otherwise ResizeProcessor them down to 1 stream.
 
     size_t presink_streams_size = std::max<size_t>(settings.max_insert_threads, pipeline.getNumStreams());
+    if (settings.max_insert_threads.changed)
+        presink_streams_size = std::max<size_t>(1, settings.max_insert_threads);
 
     size_t sink_streams_size = table->supportsParallelInsert() ? std::max<size_t>(1, settings.max_insert_threads) : 1;
 
-    if (!settings.parallel_view_processing)
+    size_t views_involved =  table->isView() || DatabaseCatalog::instance().getDependentViews(table->getStorageID()).size() > 0;
+    if (!settings.parallel_view_processing && views_involved)
     {
-        auto table_id = table->getStorageID();
-        auto views = DatabaseCatalog::instance().getDependentViews(table_id);
-
-        if (table->isView() || !views.empty())
-            sink_streams_size = 1;
+        sink_streams_size = 1;
     }
 
     auto [presink_chains, sink_chains] = buildPreAndSinkChains(
@@ -639,20 +639,12 @@ QueryPipeline InterpreterInsertQuery::buildInsertSelectPipeline(ASTInsertQuery &
         pipeline.addResources(chain.detachResources());
     pipeline.addChains(std::move(sink_chains));
 
-    if (!settings.parallel_view_processing)
+    if (!settings.parallel_view_processing && views_involved)
     {
         /// Don't use more threads for INSERT than for SELECT to reduce memory consumption.
         if (pipeline.getNumThreads() > num_select_threads)
             pipeline.setMaxThreads(num_select_threads);
     }
-    else if (pipeline.getNumThreads() < settings.max_threads)
-    {
-        /// It is possible for query to have max_threads=1, due to optimize_trivial_insert_select,
-        /// however in case of parallel_view_processing and multiple views, views can still be processed in parallel.
-        ///
-        /// Note, number of threads will be limited by buildPushingToViewsChain() to max_threads.
-        pipeline.setMaxThreads(settings.max_threads);
-    }
 
     pipeline.setSinks([&](const Block & cur_header, QueryPipelineBuilder::StreamType) -> ProcessorPtr
     {
@@ -794,6 +786,8 @@ BlockIO InterpreterInsertQuery::execute()
     if (const auto * mv = dynamic_cast<const StorageMaterializedView *>(table.get()))
         res.pipeline.addStorageHolder(mv->getTargetTable());
 
+    LOG_TEST(getLogger("InterpreterInsertQuery"), "Pipeline could use up to {} thread", res.pipeline.getNumThreads());
+
     return res;
 }
 
diff --git a/tests/queries/0_stateless/01275_parallel_mv.sql.j2 b/tests/queries/0_stateless/01275_parallel_mv.sql.j2
index bc663bd0e63..4e45c68b5ad 100644
--- a/tests/queries/0_stateless/01275_parallel_mv.sql.j2
+++ b/tests/queries/0_stateless/01275_parallel_mv.sql.j2
@@ -3,7 +3,12 @@
 -- no-parallel: it checks the number of threads, which can be lowered in presence of other queries
 
 -- avoid settings randomization by clickhouse-test
-set max_threads = 0;
+set max_threads = 10;
+
+
+-- more blocks to process
+set max_block_size = 10;
+set min_insert_block_size_rows = 10;
 
 drop table if exists testX;
 drop table if exists testXA;
@@ -12,23 +17,23 @@ drop table if exists testXC;
 
 create table testX (A Int64) engine=MergeTree order by tuple();
 
-create materialized view testXA engine=MergeTree order by tuple() as select sleepEachRow(0.2) from testX;
-create materialized view testXB engine=MergeTree order by tuple() as select sleepEachRow(0.4), throwIf(A=1) from testX;
-create materialized view testXC engine=MergeTree order by tuple() as select sleepEachRow(0.2) from testX;
+create materialized view testXA engine=MergeTree order by tuple() as select sleep(0.1) from testX;
+create materialized view testXB engine=MergeTree order by tuple() as select sleep(0.2), throwIf(A=1) from testX;
+create materialized view testXC engine=MergeTree order by tuple() as select sleep(0.1) from testX;
 
 -- { echoOn }
 {% for parallel_view_processing in [0, 1] %}
 {% for optimize_trivial_insert_select in [0, 1] %}
-{% for max_insert_threads in [0, 16] %}
+{% for max_insert_threads in [0, 5] %}
 select 'optimize_trivial_insert_select={{ optimize_trivial_insert_select }}', 'max_insert_threads={{ max_insert_threads }}';
 
-insert into testX select number from numbers(10) settings
+insert into testX select number from numbers(200) settings
     log_queries=1,
     parallel_view_processing={{ parallel_view_processing }},
     optimize_trivial_insert_select={{ optimize_trivial_insert_select }},
     max_insert_threads={{ max_insert_threads }}; -- { serverError FUNCTION_THROW_IF_VALUE_IS_NON_ZERO }
 system flush logs;
-select arrayUniq(thread_ids) from system.query_log where
+select peak_threads_usage from system.query_log where
     current_database = currentDatabase() and
     type != 'QueryStart' and
     query like '%insert into testX %' and

From b4e9e410e30a02799f446e635f4c9d0f0da76913 Mon Sep 17 00:00:00 2001
From: Sema Checherinda <Sema.Checherinda@clickhouse.com>
Date: Mon, 8 Jul 2024 15:00:23 +0200
Subject: [PATCH 0166/1488] fix peak_threads_usage

---
 src/Interpreters/ThreadStatusExt.cpp | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/Interpreters/ThreadStatusExt.cpp b/src/Interpreters/ThreadStatusExt.cpp
index 9ca521a4ab3..6ec6a64b13d 100644
--- a/src/Interpreters/ThreadStatusExt.cpp
+++ b/src/Interpreters/ThreadStatusExt.cpp
@@ -233,7 +233,8 @@ void ThreadStatus::attachToGroupImpl(const ThreadGroupPtr & thread_group_)
 {
     /// Attach or init current thread to thread group and copy useful information from it
     thread_group = thread_group_;
-    thread_group->linkThread(thread_id);
+    if (!internal_thread)
+        thread_group->linkThread(thread_id);
 
     performance_counters.setParent(&thread_group->performance_counters);
     memory_tracker.setParent(&thread_group->memory_tracker);
@@ -269,7 +270,8 @@ void ThreadStatus::detachFromGroup()
     /// Extract MemoryTracker out from query and user context
     memory_tracker.setParent(&total_memory_tracker);
 
-    thread_group->unlinkThread();
+    if (!internal_thread)
+        thread_group->unlinkThread();
 
     thread_group.reset();
 

From a2b17b01f9561fd1853851932a2ae77513c49e26 Mon Sep 17 00:00:00 2001
From: Anton Popov <anton@clickhouse.com>
Date: Mon, 8 Jul 2024 17:47:56 +0000
Subject: [PATCH 0167/1488] fix filling of missed subcolumns

---
 src/Interpreters/inplaceBlockConversions.cpp  | 38 ++++++++----
 src/Storages/MergeTree/IMergeTreeReader.cpp   | 48 +++++++++++----
 .../MergeTree/MergeTreeReaderCompact.cpp      | 60 ++++++++++++-------
 .../MergeTree/MergeTreeReaderCompact.h        |  1 +
 .../03203_fill_missed_subcolumns.reference    | 31 ++++++++++
 .../03203_fill_missed_subcolumns.sql          | 47 +++++++++++++++
 6 files changed, 181 insertions(+), 44 deletions(-)
 create mode 100644 tests/queries/0_stateless/03203_fill_missed_subcolumns.reference
 create mode 100644 tests/queries/0_stateless/03203_fill_missed_subcolumns.sql

diff --git a/src/Interpreters/inplaceBlockConversions.cpp b/src/Interpreters/inplaceBlockConversions.cpp
index 239cce5b427..f7d8a2a2daf 100644
--- a/src/Interpreters/inplaceBlockConversions.cpp
+++ b/src/Interpreters/inplaceBlockConversions.cpp
@@ -18,6 +18,7 @@
 #include <Storages/ColumnsDescription.h>
 #include <DataTypes/NestedUtils.h>
 #include <Columns/ColumnArray.h>
+#include <Columns/ColumnConst.h>
 #include <DataTypes/DataTypeArray.h>
 #include <Storages/StorageInMemoryMetadata.h>
 
@@ -35,8 +36,13 @@ namespace
 
 /// Add all required expressions for missing columns calculation
 void addDefaultRequiredExpressionsRecursively(
-    const Block & block, const String & required_column_name, DataTypePtr required_column_type,
-    const ColumnsDescription & columns, ASTPtr default_expr_list_accum, NameSet & added_columns, bool null_as_default)
+    const Block & block,
+    const String & required_column_name,
+    DataTypePtr required_column_type,
+    const ColumnsDescription & columns,
+    ASTPtr default_expr_list_accum,
+    NameSet & added_columns,
+    bool null_as_default)
 {
     checkStackSize();
 
@@ -273,6 +279,20 @@ static std::unordered_map<String, ColumnPtr> collectOffsetsColumns(
     return offsets_columns;
 }
 
+static ColumnPtr createColumnWithDefaultValue(const IDataType & data_type, const String & subcolumn_name, size_t num_rows)
+{
+    auto column = data_type.createColumnConstWithDefaultValue(num_rows);
+
+    if (subcolumn_name.empty())
+        return column->convertToFullColumnIfConst();
+
+    /// Firstly get subcolumn from const column and then replicate.
+    column = assert_cast<const ColumnConst &>(*column).getDataColumnPtr();
+    column = data_type.getSubcolumn(subcolumn_name, column);
+
+    return ColumnConst::create(std::move(column), num_rows)->convertToFullColumnIfConst();
+}
+
 void fillMissingColumns(
     Columns & res_columns,
     size_t num_rows,
@@ -298,21 +318,19 @@ void fillMissingColumns(
     auto requested_column = requested_columns.begin();
     for (size_t i = 0; i < num_columns; ++i, ++requested_column)
     {
-        const auto & [name, type] = *requested_column;
-
-        if (res_columns[i] && partially_read_columns.contains(name))
+        if (res_columns[i] && partially_read_columns.contains(requested_column->name))
             res_columns[i] = nullptr;
 
         if (res_columns[i])
             continue;
 
-        if (metadata_snapshot && metadata_snapshot->getColumns().hasDefault(name))
+        if (metadata_snapshot && metadata_snapshot->getColumns().hasDefault(requested_column->getNameInStorage()))
             continue;
 
         std::vector<ColumnPtr> current_offsets;
         size_t num_dimensions = 0;
 
-        const auto * array_type = typeid_cast<const DataTypeArray *>(type.get());
+        const auto * array_type = typeid_cast<const DataTypeArray *>(requested_column->type.get());
         if (array_type && !offsets_columns.empty())
         {
             num_dimensions = getNumberOfDimensions(*array_type);
@@ -348,10 +366,10 @@ void fillMissingColumns(
         if (!current_offsets.empty())
         {
             size_t num_empty_dimensions = num_dimensions - current_offsets.size();
-            auto scalar_type = createArrayOfType(getBaseTypeOfArray(type), num_empty_dimensions);
+            auto scalar_type = createArrayOfType(getBaseTypeOfArray(requested_column->getTypeInStorage()), num_empty_dimensions);
 
             size_t data_size = assert_cast<const ColumnUInt64 &>(*current_offsets.back()).getData().back();
-            res_columns[i] = scalar_type->createColumnConstWithDefaultValue(data_size)->convertToFullColumnIfConst();
+            res_columns[i] = createColumnWithDefaultValue(*scalar_type, requested_column->getSubcolumnName(), data_size);
 
             for (auto it = current_offsets.rbegin(); it != current_offsets.rend(); ++it)
                 res_columns[i] = ColumnArray::create(res_columns[i], *it);
@@ -360,7 +378,7 @@ void fillMissingColumns(
         {
             /// We must turn a constant column into a full column because the interpreter could infer
             /// that it is constant everywhere but in some blocks (from other parts) it can be a full column.
-            res_columns[i] = type->createColumnConstWithDefaultValue(num_rows)->convertToFullColumnIfConst();
+            res_columns[i] = createColumnWithDefaultValue(*requested_column->getTypeInStorage(), requested_column->getSubcolumnName(), num_rows);
         }
     }
 }
diff --git a/src/Storages/MergeTree/IMergeTreeReader.cpp b/src/Storages/MergeTree/IMergeTreeReader.cpp
index 4ad7f6ef991..aff1001163e 100644
--- a/src/Storages/MergeTree/IMergeTreeReader.cpp
+++ b/src/Storages/MergeTree/IMergeTreeReader.cpp
@@ -3,6 +3,7 @@
 #include <Storages/MergeTree/MergeTreeVirtualColumns.h>
 #include <Storages/MergeTree/LoadedMergeTreeDataPartInfoForReader.h>
 #include <DataTypes/NestedUtils.h>
+#include <DataTypes/DataTypeNested.h>
 #include <Common/escapeForFileName.h>
 #include <Compression/CachedCompressedReadBuffer.h>
 #include <Columns/ColumnArray.h>
@@ -144,19 +145,26 @@ void IMergeTreeReader::evaluateMissingDefaults(Block additional_columns, Columns
             throw Exception(ErrorCodes::LOGICAL_ERROR, "invalid number of columns passed to MergeTreeReader::fillMissingColumns. "
                             "Expected {}, got {}", num_columns, res_columns.size());
 
-        /// Convert columns list to block.
-        /// TODO: rewrite with columns interface. It will be possible after changes in ExpressionActions.
-        auto name_and_type = requested_columns.begin();
-        for (size_t pos = 0; pos < num_columns; ++pos, ++name_and_type)
-        {
-            if (res_columns[pos] == nullptr)
-                continue;
+        NameSet full_requested_columns_set;
+        NamesAndTypesList full_requested_columns;
 
-            additional_columns.insert({res_columns[pos], name_and_type->type, name_and_type->name});
+        /// Convert columns list to block. And convert subcolumns to full columns.
+        /// TODO: rewrite with columns interface. It will be possible after changes in ExpressionActions.
+
+        auto it = requested_columns.begin();
+        for (size_t pos = 0; pos < num_columns; ++pos, ++it)
+        {
+            auto name_in_storage = it->getNameInStorage();
+
+            if (full_requested_columns_set.emplace(name_in_storage).second)
+                full_requested_columns.emplace_back(name_in_storage, it->getTypeInStorage());
+
+            if (res_columns[pos])
+                additional_columns.insert({res_columns[pos], it->type, it->name});
         }
 
         auto dag = DB::evaluateMissingDefaults(
-            additional_columns, requested_columns,
+            additional_columns, full_requested_columns,
             storage_snapshot->metadata->getColumns(),
             data_part_info_for_read->getContext());
 
@@ -170,9 +178,18 @@ void IMergeTreeReader::evaluateMissingDefaults(Block additional_columns, Columns
         }
 
         /// Move columns from block.
-        name_and_type = requested_columns.begin();
-        for (size_t pos = 0; pos < num_columns; ++pos, ++name_and_type)
-            res_columns[pos] = std::move(additional_columns.getByName(name_and_type->name).column);
+        it = requested_columns.begin();
+        for (size_t pos = 0; pos < num_columns; ++pos, ++it)
+        {
+            auto name_in_storage = it->getNameInStorage();
+            res_columns[pos] = additional_columns.getByName(name_in_storage).column;
+
+            if (it->isSubcolumn())
+            {
+                const auto & type_in_storage = it->getTypeInStorage();
+                res_columns[pos] = type_in_storage->getSubcolumn(it->getSubcolumnName(), res_columns[pos]);
+            }
+        }
     }
     catch (Exception & e)
     {
@@ -192,7 +209,12 @@ bool IMergeTreeReader::isSubcolumnOffsetsOfNested(const String & name_in_storage
     if (!data_part_info_for_read->isWidePart() || subcolumn_name != "size0")
         return false;
 
-    return Nested::isSubcolumnOfNested(name_in_storage, part_columns);
+    auto split = Nested::splitName(name_in_storage);
+    if (split.second.empty())
+        return false;
+
+    auto nested_column = part_columns.tryGetColumn(GetColumnsOptions::All, split.first);
+    return nested_column && isNested(nested_column->type);
 }
 
 String IMergeTreeReader::getColumnNameInPart(const NameAndTypePair & required_column) const
diff --git a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp
index a2b8f0ad96f..fde9dafffb8 100644
--- a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp
+++ b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp
@@ -60,39 +60,25 @@ void MergeTreeReaderCompact::fillColumnPositions()
 
     for (size_t i = 0; i < columns_num; ++i)
     {
-        const auto & column_to_read = columns_to_read[i];
-
+        auto & column_to_read = columns_to_read[i];
         auto position = data_part_info_for_read->getColumnPosition(column_to_read.getNameInStorage());
-        bool is_array = isArray(column_to_read.type);
 
         if (column_to_read.isSubcolumn())
         {
-            auto storage_column_from_part = getColumnInPart(
-                {column_to_read.getNameInStorage(), column_to_read.getTypeInStorage()});
+            NameAndTypePair column_in_storage{column_to_read.getNameInStorage(), column_to_read.getTypeInStorage()};
+            auto storage_column_from_part = getColumnInPart(column_in_storage);
 
             auto subcolumn_name = column_to_read.getSubcolumnName();
             if (!storage_column_from_part.type->hasSubcolumn(subcolumn_name))
                 position.reset();
         }
 
+        column_positions[i] = std::move(position);
+
         /// If array of Nested column is missing in part,
         /// we have to read its offsets if they exist.
-        if (!position && is_array)
-        {
-            auto column_to_read_with_subcolumns = getColumnConvertedToSubcolumnOfNested(column_to_read);
-            auto name_level_for_offsets = findColumnForOffsets(column_to_read_with_subcolumns);
-
-            if (name_level_for_offsets.has_value())
-            {
-                column_positions[i] = data_part_info_for_read->getColumnPosition(name_level_for_offsets->first);
-                columns_for_offsets[i] = name_level_for_offsets;
-                partially_read_columns.insert(column_to_read.name);
-            }
-        }
-        else
-        {
-            column_positions[i] = std::move(position);
-        }
+        if (!column_positions[i])
+            findPositionForMissedNested(i);
     }
 }
 
@@ -125,6 +111,38 @@ NameAndTypePair MergeTreeReaderCompact::getColumnConvertedToSubcolumnOfNested(co
         Nested::concatenateName(name_in_storage, subcolumn_name));
 }
 
+void MergeTreeReaderCompact::findPositionForMissedNested(size_t pos)
+{
+    auto & column = columns_to_read[pos];
+
+    bool is_array = isArray(column.type);
+    bool is_offsets_subcolumn = isArray(column.getTypeInStorage()) && column.getSubcolumnName() == "size0";
+
+    if (!is_array && !is_offsets_subcolumn)
+        return;
+
+    NameAndTypePair column_in_storage{column.getNameInStorage(), column.getTypeInStorage()};
+
+    auto column_to_read_with_subcolumns = getColumnConvertedToSubcolumnOfNested(column_in_storage);
+    auto name_level_for_offsets = findColumnForOffsets(column_to_read_with_subcolumns);
+
+    if (!name_level_for_offsets)
+        return;
+
+    column_positions[pos] = data_part_info_for_read->getColumnPosition(name_level_for_offsets->first);
+
+    if (is_offsets_subcolumn)
+    {
+        /// Read offsets from antoher array from the same Nested column.
+        column = {name_level_for_offsets->first, column.getSubcolumnName(), column.getTypeInStorage(), column.type};
+    }
+    else
+    {
+        columns_for_offsets[pos] = std::move(name_level_for_offsets);
+        partially_read_columns.insert(column.name);
+    }
+}
+
 void MergeTreeReaderCompact::readData(
     const NameAndTypePair & name_and_type,
     ColumnPtr & column,
diff --git a/src/Storages/MergeTree/MergeTreeReaderCompact.h b/src/Storages/MergeTree/MergeTreeReaderCompact.h
index a783e595af5..22eabd47930 100644
--- a/src/Storages/MergeTree/MergeTreeReaderCompact.h
+++ b/src/Storages/MergeTree/MergeTreeReaderCompact.h
@@ -36,6 +36,7 @@ public:
 protected:
     void fillColumnPositions();
     NameAndTypePair getColumnConvertedToSubcolumnOfNested(const NameAndTypePair & column);
+    void findPositionForMissedNested(size_t pos);
 
     using InputStreamGetter = ISerialization::InputStreamGetter;
 
diff --git a/tests/queries/0_stateless/03203_fill_missed_subcolumns.reference b/tests/queries/0_stateless/03203_fill_missed_subcolumns.reference
new file mode 100644
index 00000000000..872eb7da3c8
--- /dev/null
+++ b/tests/queries/0_stateless/03203_fill_missed_subcolumns.reference
@@ -0,0 +1,31 @@
+0
+2
+4
+6
+8
+0
+2
+4
+6
+8
+1	['aaa',NULL]	[NULL,NULL]
+2	['ccc']	[NULL]
+3	[NULL]	[NULL]
+4	[NULL,'bbb']	['ddd',NULL]
+5	[NULL]	[NULL]
+1	2	2
+2	1	1
+3	1	1
+4	2	2
+5	1	1
+1	[0,1]	[1,1]
+2	[0]	[1]
+3	[1]	[1]
+4	[1,0]	[0,1]
+5	[1]	[1]
+1	('foo','bar')	[1,NULL,3]
+2	('aaa','bbb')	[1,NULL,3]
+3	('ccc','ddd')	[4,5,6]
+1	foo	bar	3	[0,1,0]
+2	foo	bar	3	[0,1,0]
+3	ccc	ddd	3	[0,0,0]
diff --git a/tests/queries/0_stateless/03203_fill_missed_subcolumns.sql b/tests/queries/0_stateless/03203_fill_missed_subcolumns.sql
new file mode 100644
index 00000000000..2789c9de35c
--- /dev/null
+++ b/tests/queries/0_stateless/03203_fill_missed_subcolumns.sql
@@ -0,0 +1,47 @@
+DROP TABLE IF EXISTS t_missed_subcolumns;
+
+-- .null subcolumn
+
+CREATE TABLE t_missed_subcolumns (x UInt32) ENGINE = MergeTree ORDER BY tuple();
+INSERT INTO t_missed_subcolumns SELECT * FROM numbers(10);
+
+ALTER TABLE t_missed_subcolumns ADD COLUMN `y` Nullable(UInt32);
+
+INSERT INTO t_missed_subcolumns SELECT number, if(number % 2, NULL, number) FROM numbers(10);
+
+SELECT x FROM t_missed_subcolumns WHERE y IS NOT NULL SETTINGS optimize_functions_to_subcolumns = 1;
+SELECT x FROM t_missed_subcolumns WHERE y IS NOT NULL SETTINGS optimize_functions_to_subcolumns = 0;
+
+DROP TABLE IF EXISTS t_missed_subcolumns;
+
+-- .null and .size0 subcolumn in array
+
+CREATE TABLE t_missed_subcolumns (id UInt64, `n.a` Array(Nullable(String))) ENGINE = MergeTree ORDER BY id;
+
+INSERT INTO t_missed_subcolumns VALUES (1, ['aaa', NULL]) (2, ['ccc']) (3, [NULL]);
+ALTER TABLE t_missed_subcolumns ADD COLUMN `n.b` Array(Nullable(String));
+INSERT INTO t_missed_subcolumns VALUES (4, [NULL, 'bbb'], ['ddd', NULL]), (5, [NULL], [NULL]);
+
+SELECT id, n.a, n.b FROM t_missed_subcolumns ORDER BY id;
+SELECT id, n.a.size0, n.b.size0 FROM t_missed_subcolumns ORDER BY id;
+SELECT id, n.a.null, n.b.null FROM t_missed_subcolumns ORDER BY id;
+
+DROP TABLE IF EXISTS t_missed_subcolumns;
+
+-- subcolumns and custom defaults
+
+CREATE TABLE t_missed_subcolumns (id UInt64) ENGINE = MergeTree ORDER BY id;
+SYSTEM STOP MERGES t_missed_subcolumns;
+
+INSERT INTO t_missed_subcolumns VALUES (1);
+
+ALTER TABLE t_missed_subcolumns ADD COLUMN t Tuple(a String, b String) DEFAULT ('foo', 'bar');
+INSERT INTO t_missed_subcolumns VALUES (2, ('aaa', 'bbb'));
+
+ALTER TABLE t_missed_subcolumns ADD COLUMN arr Array(Nullable(UInt64)) DEFAULT [1, NULL, 3];
+INSERT INTO t_missed_subcolumns VALUES (3, ('ccc', 'ddd'), [4, 5, 6]);
+
+SELECT id, t, arr FROM t_missed_subcolumns ORDER BY id;
+SELECT id, t.a, t.b, arr.size0, arr.null FROM t_missed_subcolumns ORDER BY id;
+
+DROP TABLE t_missed_subcolumns;

From 898260c45cfb03d715f3e5a38c4298c399a9cfd0 Mon Sep 17 00:00:00 2001
From: Sema Checherinda <Sema.Checherinda@clickhouse.com>
Date: Mon, 8 Jul 2024 21:16:05 +0200
Subject: [PATCH 0168/1488] fix style

---
 src/Processors/Transforms/SquashingTransform.cpp | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/Processors/Transforms/SquashingTransform.cpp b/src/Processors/Transforms/SquashingTransform.cpp
index 605c6afcd00..490a57d4e23 100644
--- a/src/Processors/Transforms/SquashingTransform.cpp
+++ b/src/Processors/Transforms/SquashingTransform.cpp
@@ -8,8 +8,7 @@ namespace DB
 
 namespace ErrorCodes
 {
-extern const int LOGICAL_ERROR;
-extern const int SIZES_OF_COLUMNS_DOESNT_MATCH;
+    extern const int LOGICAL_ERROR;
 }
 
 SquashingTransform::SquashingTransform(

From 943a8bce304c5610ea69a6aea7fdf354c5f71944 Mon Sep 17 00:00:00 2001
From: JackyWoo <wuchienchao@qq.com>
Date: Fri, 5 Jul 2024 15:41:21 +0800
Subject: [PATCH 0169/1488] Estimate should be based on column data type not
 only constant value

---
 .../ConditionSelectivityEstimator.cpp         | 12 +--
 .../ConditionSelectivityEstimator.h           |  4 +-
 src/Storages/Statistics/Statistics.cpp        | 38 ++--------
 src/Storages/Statistics/Statistics.h          | 20 ++---
 .../Statistics/StatisticsCountMinSketch.cpp   | 76 ++++++++++++-------
 .../Statistics/StatisticsCountMinSketch.h     |  6 +-
 src/Storages/Statistics/StatisticsTDigest.cpp | 25 +++---
 src/Storages/Statistics/StatisticsTDigest.h   |  2 +-
 src/Storages/StatisticsDescription.cpp        |  4 +-
 ...64_statistics_estimate_predicate.reference |  2 +-
 10 files changed, 100 insertions(+), 89 deletions(-)

diff --git a/src/Storages/Statistics/ConditionSelectivityEstimator.cpp b/src/Storages/Statistics/ConditionSelectivityEstimator.cpp
index 1755f0eb4df..19e3157d99a 100644
--- a/src/Storages/Statistics/ConditionSelectivityEstimator.cpp
+++ b/src/Storages/Statistics/ConditionSelectivityEstimator.cpp
@@ -16,7 +16,7 @@ void ConditionSelectivityEstimator::ColumnSelectivityEstimator::merge(String par
     part_statistics[part_name] = stats;
 }
 
-Float64 ConditionSelectivityEstimator::ColumnSelectivityEstimator::estimateLess(Float64 val, Float64 rows) const
+Float64 ConditionSelectivityEstimator::ColumnSelectivityEstimator::estimateLess(const Field & val, Float64 rows) const
 {
     if (part_statistics.empty())
         return default_normal_cond_factor * rows;
@@ -30,14 +30,14 @@ Float64 ConditionSelectivityEstimator::ColumnSelectivityEstimator::estimateLess(
     return result * rows / part_rows;
 }
 
-Float64 ConditionSelectivityEstimator::ColumnSelectivityEstimator::estimateGreater(Float64 val, Float64 rows) const
+Float64 ConditionSelectivityEstimator::ColumnSelectivityEstimator::estimateGreater(const Field & val, Float64 rows) const
 {
     return rows - estimateLess(val, rows);
 }
 
 Float64 ConditionSelectivityEstimator::ColumnSelectivityEstimator::estimateEqual(const Field & val, Float64 rows) const
 {
-    auto float_val = IStatistics::getFloat64(val);
+    auto float_val = StatisticsUtils::tryConvertToFloat64(val);
     if (part_statistics.empty())
     {
         if (!float_val)
@@ -148,7 +148,7 @@ Float64 ConditionSelectivityEstimator::estimateRowCount(const RPNBuilderTreeNode
     else
         dummy = true;
     auto [op, val] = extractBinaryOp(node, col);
-    auto float_val = IStatistics::getFloat64(val);
+    auto float_val = StatisticsUtils::tryConvertToFloat64(val);
     if (op == "equals")
     {
         if (dummy)
@@ -164,13 +164,13 @@ Float64 ConditionSelectivityEstimator::estimateRowCount(const RPNBuilderTreeNode
     {
         if (dummy)
             return default_normal_cond_factor * total_rows;
-        return estimator.estimateLess(float_val.value(), total_rows);
+        return estimator.estimateLess(val, total_rows);
     }
     else if (op == "greater" || op == "greaterOrEquals")
     {
         if (dummy)
             return default_normal_cond_factor * total_rows;
-        return estimator.estimateGreater(float_val.value(), total_rows);
+        return estimator.estimateGreater(val, total_rows);
     }
     else
         return default_unknown_cond_factor * total_rows;
diff --git a/src/Storages/Statistics/ConditionSelectivityEstimator.h b/src/Storages/Statistics/ConditionSelectivityEstimator.h
index b9127fcd5bf..ce7fdd12e92 100644
--- a/src/Storages/Statistics/ConditionSelectivityEstimator.h
+++ b/src/Storages/Statistics/ConditionSelectivityEstimator.h
@@ -29,9 +29,9 @@ private:
 
         void merge(String part_name, ColumnStatisticsPtr stats);
 
-        Float64 estimateLess(Float64 val, Float64 rows) const;
+        Float64 estimateLess(const Field & val, Float64 rows) const;
 
-        Float64 estimateGreater(Float64 val, Float64 rows) const;
+        Float64 estimateGreater(const Field & val, Float64 rows) const;
 
         Float64 estimateEqual(const Field & val, Float64 rows) const;
     };
diff --git a/src/Storages/Statistics/Statistics.cpp b/src/Storages/Statistics/Statistics.cpp
index cd94ed716cd..5663c55b263 100644
--- a/src/Storages/Statistics/Statistics.cpp
+++ b/src/Storages/Statistics/Statistics.cpp
@@ -1,8 +1,8 @@
+#include <Storages/Statistics/Statistics.h>
 #include <IO/ReadHelpers.h>
 #include <IO/WriteHelpers.h>
 #include <Storages/ColumnsDescription.h>
 #include <Storages/Statistics/ConditionSelectivityEstimator.h>
-#include <Storages/Statistics/Statistics.h>
 #include <Storages/Statistics/StatisticsCountMinSketch.h>
 #include <Storages/Statistics/StatisticsTDigest.h>
 #include <Storages/Statistics/StatisticsUniq.h>
@@ -27,44 +27,22 @@ enum StatisticsFileVersion : UInt16
     V0 = 0,
 };
 
-std::optional<Float64> IStatistics::getFloat64(const Field & f)
+std::optional<Float64> StatisticsUtils::tryConvertToFloat64(const Field & f)
 {
     switch (f.getType())
     {
-        case Field::Types::Bool:
-            return f.get<bool>();
         case Field::Types::Int64:
             return f.get<Int64>();
         case Field::Types::UInt64:
             return f.get<UInt64>();
         case Field::Types::Float64:
             return f.get<Float64>();
-        case Field::Types::Int128:
-            return f.get<Int128>();
-        case Field::Types::UInt128:
-            return f.get<UInt128>();
-        case Field::Types::Int256:
-            return f.get<Int256>();
-        case Field::Types::UInt256:
-            return f.get<UInt256>();
-        case Field::Types::Decimal32:
-            return f.get<Decimal32>().getValue().value;
-        case Field::Types::Decimal64:
-            return f.get<Decimal64>().getValue().value;
-        case Field::Types::Decimal128:
-            return f.get<Decimal128>().getValue().value;
-        case Field::Types::Decimal256:
-            return f.get<Decimal256>().getValue().value;
-        case Field::Types::IPv4:
-            return f.get<IPv4>().toUnderType();
-        case Field::Types::IPv6:
-            return f.get<IPv6>().toUnderType();
         default:
             return {};
     }
 }
 
-std::optional<String> IStatistics::getString(const Field & f)
+std::optional<String> StatisticsUtils::tryConvertToString(const DB::Field & f)
 {
     if (f.getType() == Field::Types::String)
         return f.get<String>();
@@ -98,7 +76,7 @@ Float64 IStatistics::estimateEqual(const Field & /*val*/) const
     throw Exception(ErrorCodes::LOGICAL_ERROR, "Equality estimation is not implemented for this type of statistics");
 }
 
-Float64 IStatistics::estimateLess(Float64 /*val*/) const
+Float64 IStatistics::estimateLess(const Field & /*val*/) const
 {
     throw Exception(ErrorCodes::LOGICAL_ERROR, "Less-than estimation is not implemented for this type of statistics");
 }
@@ -113,21 +91,21 @@ Float64 IStatistics::estimateLess(Float64 /*val*/) const
 /// For that reason, all estimation are performed in a central place (here), and we don't simply pass the predicate to the first statistics
 /// object that supports it natively.
 
-Float64 ColumnStatistics::estimateLess(Float64 val) const
+Float64 ColumnStatistics::estimateLess(const Field & val) const
 {
     if (stats.contains(StatisticsType::TDigest))
         return stats.at(StatisticsType::TDigest)->estimateLess(val);
     return rows * ConditionSelectivityEstimator::default_normal_cond_factor;
 }
 
-Float64 ColumnStatistics::estimateGreater(Float64 val) const
+Float64 ColumnStatistics::estimateGreater(const Field & val) const
 {
     return rows - estimateLess(val);
 }
 
 Float64 ColumnStatistics::estimateEqual(const Field & val) const
 {
-    auto float_val = IStatistics::getFloat64(val);
+    auto float_val = StatisticsUtils::tryConvertToFloat64(val);
     if (float_val.has_value() && stats.contains(StatisticsType::Uniq) && stats.contains(StatisticsType::TDigest))
     {
         /// 2048 is the default number of buckets in TDigest. In this case, TDigest stores exactly one value (with many rows) for every bucket.
@@ -254,7 +232,7 @@ ColumnStatisticsPtr MergeTreeStatisticsFactory::get(const ColumnStatisticsDescri
     {
         auto it = creators.find(type);
         if (it == creators.end())
-            throw Exception(ErrorCodes::INCORRECT_QUERY, "Unknown statistic type '{}'. Available types: 'tdigest' 'uniq' 'count_min'", type);
+            throw Exception(ErrorCodes::INCORRECT_QUERY, "Unknown statistic type '{}'. Available types: 'tdigest' 'uniq' and 'count_min'", type);
         auto stat_ptr = (it->second)(desc, stats.data_type);
         column_stat->stats[type] = stat_ptr;
     }
diff --git a/src/Storages/Statistics/Statistics.h b/src/Storages/Statistics/Statistics.h
index c9bf3ca4847..0df5359adfc 100644
--- a/src/Storages/Statistics/Statistics.h
+++ b/src/Storages/Statistics/Statistics.h
@@ -14,6 +14,14 @@ namespace DB
 constexpr auto STATS_FILE_PREFIX = "statistics_";
 constexpr auto STATS_FILE_SUFFIX = ".stats";
 
+
+///Returns std::nullopt if input Field cannot be converted to a concrete value
+struct StatisticsUtils
+{
+    static std::optional<Float64> tryConvertToFloat64(const Field & f);
+    static std::optional<String> tryConvertToString(const Field & f);
+};
+
 /// Statistics describe properties of the values in the column,
 /// e.g. how many unique values exist,
 /// what are the N most frequent values,
@@ -36,13 +44,7 @@ public:
     /// Per-value estimations.
     /// Throws if the statistics object is not able to do a meaningful estimation.
     virtual Float64 estimateEqual(const Field & val) const; /// cardinality of val in the column
-    virtual Float64 estimateLess(Float64 val) const;  /// summarized cardinality of values < val in the column
-
-    /// Convert filed to Float64, used when estimating the number of rows.
-    /// Return a Float64 value if f can be represented by number, otherwise return null.
-    /// See IDataType::isValueRepresentedByNumber
-    static std::optional<Float64> getFloat64(const Field & f);
-    static std::optional<String> getString(const Field & f);
+    virtual Float64 estimateLess(const Field & val) const;  /// summarized cardinality of values < val in the column
 
 protected:
     SingleStatisticsDescription stat;
@@ -65,8 +67,8 @@ public:
 
     void update(const ColumnPtr & column);
 
-    Float64 estimateLess(Float64 val) const;
-    Float64 estimateGreater(Float64 val) const;
+    Float64 estimateLess(const Field & val) const;
+    Float64 estimateGreater(const Field & val) const;
     Float64 estimateEqual(const Field & val) const;
 
 private:
diff --git a/src/Storages/Statistics/StatisticsCountMinSketch.cpp b/src/Storages/Statistics/StatisticsCountMinSketch.cpp
index dd8ceef4e2d..abd5cf17946 100644
--- a/src/Storages/Statistics/StatisticsCountMinSketch.cpp
+++ b/src/Storages/Statistics/StatisticsCountMinSketch.cpp
@@ -3,6 +3,8 @@
 #include <IO/ReadHelpers.h>
 #include <IO/WriteHelpers.h>
 #include <Storages/Statistics/StatisticsCountMinSketch.h>
+#include <Interpreters/convertFieldToType.h>
+#include <DataTypes/FieldToDataType.h>
 
 #if USE_DATASKETCHES
 
@@ -12,10 +14,13 @@ namespace DB
 namespace ErrorCodes
 {
 extern const int ILLEGAL_STATISTICS;
+extern const int ILLEGAL_TYPE_OF_ARGUMENT;
 }
 
-static constexpr auto num_hashes = 8uz;
-static constexpr auto num_buckets = 2048uz;
+/// Constants chosen based on rolling dices, which provides an error tolerance of 0.1% (ε = 0.001) and a confidence level of 99.9% (δ = 0.001).
+/// And sketch the size is 152kb.
+static constexpr auto num_hashes = 7uz;
+static constexpr auto num_buckets = 2718uz;
 
 StatisticsCountMinSketch::StatisticsCountMinSketch(const SingleStatisticsDescription & stat_, DataTypePtr data_type_)
     : IStatistics(stat_)
@@ -24,13 +29,49 @@ StatisticsCountMinSketch::StatisticsCountMinSketch(const SingleStatisticsDescrip
 {
 }
 
-Float64 StatisticsCountMinSketch::estimateEqual(const Field & value) const
+Float64 StatisticsCountMinSketch::estimateEqual(const Field & val) const
 {
-    if (auto float_val = IStatistics::getFloat64(value))
-        return sketch.get_estimate(&float_val.value(), 8);
-    if (auto string_val = IStatistics::getString(value))
-        return sketch.get_estimate(string_val->data(), string_val->size());
-    UNREACHABLE();
+    if (data_type->isValueRepresentedByNumber())
+    {
+        /// convertFieldToType will
+        ///     1. convert string to number, date, datetime, IPv4, Decimal etc
+        ///     2. do other conversion
+        ///     3. return null if val larger than the range of data_type
+        auto val_converted = convertFieldToType(val, *data_type);
+        if (val_converted.isNull())
+            return 0;
+
+        /// We will get the proper data type of val_converted, for example, UInt8 for 1, UInt16 for 257.
+        auto data_type_converted = applyVisitor(FieldToDataType<LeastSupertypeOnError::Null>(), val_converted);
+        DataTypes data_types = {data_type, data_type_converted};
+        auto super_type = tryGetLeastSupertype(data_types);
+
+        /// If data_type is UInt8 but val_typed is UInt16, we should return 0.
+        if (!super_type->equals(*data_type))
+            return 0;
+
+        return sketch.get_estimate(&val_converted, data_type->getSizeOfValueInMemory());
+    }
+
+    if (isStringOrFixedString(data_type))
+    {
+        return sketch.get_estimate(val.get<String>());
+    }
+
+    throw Exception(
+        ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Statistics 'count_min' does not support estimate constant value of type {}", val.getTypeName());
+}
+
+void StatisticsCountMinSketch::update(const ColumnPtr & column)
+{
+    size_t size = column->size();
+    for (size_t row = 0; row < size; ++row)
+    {
+        if (column->isNullAt(row))
+            continue;
+        auto data = column->getDataAt(row);
+        sketch.update(data.data, data.size, 1);
+    }
 }
 
 void StatisticsCountMinSketch::serialize(WriteBuffer & buf)
@@ -52,28 +93,11 @@ void StatisticsCountMinSketch::deserialize(ReadBuffer & buf)
     sketch = Sketch::deserialize(bytes.data(), size);
 }
 
-void StatisticsCountMinSketch::update(const ColumnPtr & column)
-{
-    size_t size = column->size();
-    for (size_t i = 0; i < size; ++i)
-    {
-        Field f;
-        column->get(i, f);
-
-        if (f.isNull())
-            continue;
-
-        if (auto float_val = IStatistics::getFloat64(f))
-            sketch.update(&float_val, 8, 1);
-        else
-            sketch.update(f.get<String>(), 1);
-    }
-}
-
 void CountMinSketchValidator(const SingleStatisticsDescription &, DataTypePtr data_type)
 {
     data_type = removeNullable(data_type);
     data_type = removeLowCardinalityAndNullable(data_type);
+    /// Numeric, String family, IPv4, IPv6, Date family, Enum family are supported.
     if (!data_type->isValueRepresentedByNumber() && !isStringOrFixedString(data_type))
         throw Exception(ErrorCodes::ILLEGAL_STATISTICS, "Statistics of type 'count_min' does not support type {}", data_type->getName());
 }
diff --git a/src/Storages/Statistics/StatisticsCountMinSketch.h b/src/Storages/Statistics/StatisticsCountMinSketch.h
index 3ea3f2dbd3b..2cc93d6e75a 100644
--- a/src/Storages/Statistics/StatisticsCountMinSketch.h
+++ b/src/Storages/Statistics/StatisticsCountMinSketch.h
@@ -16,13 +16,13 @@ class StatisticsCountMinSketch : public IStatistics
 public:
     StatisticsCountMinSketch(const SingleStatisticsDescription & stat_, DataTypePtr data_type_);
 
-    Float64 estimateEqual(const Field & value) const override;
+    Float64 estimateEqual(const Field & val) const override;
+
+    void update(const ColumnPtr & column) override;
 
     void serialize(WriteBuffer & buf) override;
     void deserialize(ReadBuffer & buf) override;
 
-    void update(const ColumnPtr & column) override;
-
 private:
     using Sketch = datasketches::count_min_sketch<UInt64>;
     Sketch sketch;
diff --git a/src/Storages/Statistics/StatisticsTDigest.cpp b/src/Storages/Statistics/StatisticsTDigest.cpp
index 306338b4ba2..c19d0a0328c 100644
--- a/src/Storages/Statistics/StatisticsTDigest.cpp
+++ b/src/Storages/Statistics/StatisticsTDigest.cpp
@@ -6,7 +6,8 @@ namespace DB
 {
 namespace ErrorCodes
 {
-    extern const int ILLEGAL_STATISTICS;
+extern const int ILLEGAL_STATISTICS;
+extern const int ILLEGAL_TYPE_OF_ARGUMENT;
 }
 
 StatisticsTDigest::StatisticsTDigest(const SingleStatisticsDescription & stat_)
@@ -19,14 +20,14 @@ void StatisticsTDigest::update(const ColumnPtr & column)
     size_t rows = column->size();
     for (size_t row = 0; row < rows; ++row)
     {
-        Field f;
-        column->get(row, f);
+        Field field;
+        column->get(row, field);
 
-        if (f.isNull())
+        if (field.isNull())
             continue;
 
-        if (auto float_val = IStatistics::getFloat64(f))
-            t_digest.add(*float_val, 1);
+        if (auto field_as_float = StatisticsUtils::tryConvertToFloat64(field))
+            t_digest.add(*field_as_float, 1);
     }
 }
 
@@ -40,14 +41,20 @@ void StatisticsTDigest::deserialize(ReadBuffer & buf)
     t_digest.deserialize(buf);
 }
 
-Float64 StatisticsTDigest::estimateLess(Float64 val) const
+Float64 StatisticsTDigest::estimateLess(const Field & val) const
 {
-    return t_digest.getCountLessThan(val);
+    auto val_as_float = StatisticsUtils::tryConvertToFloat64(val);
+    if (val_as_float)
+        return t_digest.getCountLessThan(*val_as_float);
+    throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Statistics 'tdigest' does not support estimate constant value of type {}", val.getTypeName());
 }
 
 Float64 StatisticsTDigest::estimateEqual(const Field & val) const
 {
-    return t_digest.getCountEqual(IStatistics::getFloat64(val).value());
+    auto val_as_float = StatisticsUtils::tryConvertToFloat64(val);
+    if (val_as_float)
+        return t_digest.getCountEqual(*val_as_float);
+    throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Statistics 'tdigest' does not support estimate constant value of type {}", val.getTypeName());
 }
 
 void TDigestValidator(const SingleStatisticsDescription &, DataTypePtr data_type)
diff --git a/src/Storages/Statistics/StatisticsTDigest.h b/src/Storages/Statistics/StatisticsTDigest.h
index 8016faac7c6..801d0787eaf 100644
--- a/src/Storages/Statistics/StatisticsTDigest.h
+++ b/src/Storages/Statistics/StatisticsTDigest.h
@@ -16,7 +16,7 @@ public:
     void serialize(WriteBuffer & buf) override;
     void deserialize(ReadBuffer & buf) override;
 
-    Float64 estimateLess(Float64 val) const override;
+    Float64 estimateLess(const Field & val) const override;
     Float64 estimateEqual(const Field & val) const override;
 
 private:
diff --git a/src/Storages/StatisticsDescription.cpp b/src/Storages/StatisticsDescription.cpp
index 8aa954f5eb5..9c5fd3604b2 100644
--- a/src/Storages/StatisticsDescription.cpp
+++ b/src/Storages/StatisticsDescription.cpp
@@ -51,7 +51,7 @@ static StatisticsType stringToStatisticsType(String type)
         return StatisticsType::Uniq;
     if (type == "count_min")
         return StatisticsType::CountMinSketch;
-    throw Exception(ErrorCodes::INCORRECT_QUERY, "Unknown statistics type: {}. Supported statistics types are 'tdigest', 'uniq', and 'count_min'.", type);
+    throw Exception(ErrorCodes::INCORRECT_QUERY, "Unknown statistics type: {}. Supported statistics types are 'tdigest', 'uniq' and 'count_min'.", type);
 }
 
 String SingleStatisticsDescription::getTypeName() const
@@ -65,7 +65,7 @@ String SingleStatisticsDescription::getTypeName() const
         case StatisticsType::CountMinSketch:
             return "count_min";
         default:
-            throw Exception(ErrorCodes::LOGICAL_ERROR, "Unknown statistics type: {}. Supported statistics types are 'tdigest', 'uniq', and 'count_min'.", type);
+            throw Exception(ErrorCodes::LOGICAL_ERROR, "Unknown statistics type: {}. Supported statistics types are 'tdigest', 'uniq' and 'count_min'.", type);
     }
 }
 
diff --git a/tests/queries/0_stateless/02864_statistics_estimate_predicate.reference b/tests/queries/0_stateless/02864_statistics_estimate_predicate.reference
index 7c22f308ab9..83b921af511 100644
--- a/tests/queries/0_stateless/02864_statistics_estimate_predicate.reference
+++ b/tests/queries/0_stateless/02864_statistics_estimate_predicate.reference
@@ -1,4 +1,4 @@
-CREATE TABLE default.t1\n(\n    `a` String,\n    `b` UInt64,\n    `c` Int64,\n    `pk` String\n)\nENGINE = MergeTree\nORDER BY pk\nSETTINGS min_bytes_for_wide_part = 0, index_granularity = 8192
+CREATE TABLE default.tab\n(\n    `a` String,\n    `b` UInt64,\n    `c` Int64,\n    `pk` String\n)\nENGINE = MergeTree\nORDER BY pk\nSETTINGS min_bytes_for_wide_part = 0, index_granularity = 8192
 Test statistics TDigest:
         Prewhere info
           Prewhere filter

From 849fb83c9770fedb937dc59df73c0cc172e115bf Mon Sep 17 00:00:00 2001
From: taiyang-li <654010905@qq.com>
Date: Tue, 9 Jul 2024 17:37:17 +0800
Subject: [PATCH 0170/1488] add function printf

---
 .../functions/string-replace-functions.md     |  25 ++
 src/Functions/printf.cpp                      | 308 ++++++++++++++++++
 .../0_stateless/032010_printf.reference       |  16 +
 tests/queries/0_stateless/032010_printf.sql   |  24 ++
 4 files changed, 373 insertions(+)
 create mode 100644 src/Functions/printf.cpp
 create mode 100644 tests/queries/0_stateless/032010_printf.reference
 create mode 100644 tests/queries/0_stateless/032010_printf.sql

diff --git a/docs/en/sql-reference/functions/string-replace-functions.md b/docs/en/sql-reference/functions/string-replace-functions.md
index 8793ebdd1a3..177790c983e 100644
--- a/docs/en/sql-reference/functions/string-replace-functions.md
+++ b/docs/en/sql-reference/functions/string-replace-functions.md
@@ -223,3 +223,28 @@ SELECT translateUTF8('Münchener Straße', 'üß', 'us') AS res;
 │ Munchener Strase │
 └──────────────────┘
 ```
+
+## printf
+
+The `printf` function formats the given string with the values (strings, integers, floating-points etc.) listed in the arguments, similar to printf function in C++. The format string can contain format specifiers starting with `%` character. Anything not contained in `%` and the following format specifier is considered literal text and copied verbatim into the output. Literal `%` character can be escaped by `%%`.
+
+**Syntax**
+
+``` sql
+printf(format, arg1, arg2, ...)
+```
+
+**Example**
+
+Query:
+
+``` sql
+select printf('%%%s %s %d', 'Hello', 'World', 2024);
+```
+
+
+``` response
+┌─printf('%%%s %s %d', 'Hello', 'World', 2024)─┐
+│ %Hello World 2024                            │
+└──────────────────────────────────────────────┘
+```
diff --git a/src/Functions/printf.cpp b/src/Functions/printf.cpp
new file mode 100644
index 00000000000..cb21d5e39ad
--- /dev/null
+++ b/src/Functions/printf.cpp
@@ -0,0 +1,308 @@
+#include <Columns/ColumnFixedString.h>
+#include <Columns/ColumnString.h>
+#include <Columns/ColumnStringHelpers.h>
+#include <DataTypes/DataTypeString.h>
+#include <Functions/FunctionFactory.h>
+#include <Functions/FunctionHelpers.h>
+#include <Functions/IFunction.h>
+#include <Functions/formatString.h>
+#include <IO/WriteHelpers.h>
+
+#include <memory>
+#include <string>
+#include <vector>
+#include <fmt/format.h>
+#include <fmt/printf.h>
+#include <sstream>
+
+namespace DB
+{
+namespace ErrorCodes
+{
+    extern const int ILLEGAL_COLUMN;
+    extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
+    extern const int ILLEGAL_TYPE_OF_ARGUMENT;
+}
+
+namespace
+{
+
+class FunctionPrintf : public IFunction
+{
+private:
+    ContextPtr context;
+    FunctionOverloadResolverPtr function_concat;
+
+    struct Instruction
+    {
+        std::string_view format;
+        size_t rows;
+        bool is_literal; /// format is literal string without any argument
+        ColumnWithTypeAndName input; /// Only used when is_literal is false
+
+        ColumnWithTypeAndName execute()
+        {
+            if (is_literal)
+                return executeLiteral(format);
+            else if (isColumnConst(*input.column))
+                return executeConstant(input);
+            else
+                return executeNonconstant(input);
+        }
+
+        String toString() const
+        {
+            std::ostringstream oss;
+            oss << "format:" << format << ", rows:" << rows << ", is_literal:" << is_literal << ", input:" << input.dumpStructure()
+                      << std::endl;
+            return oss.str();
+        }
+
+    private:
+        ColumnWithTypeAndName executeLiteral(std::string_view literal)
+        {
+            ColumnWithTypeAndName res;
+            auto str_col = ColumnString::create();
+            str_col->insert(fmt::sprintf(literal));
+            res.column = ColumnConst::create(std::move(str_col), rows);
+            res.type = std::make_shared<DataTypeString>();
+            return res;
+        }
+
+        ColumnWithTypeAndName executeConstant(const ColumnWithTypeAndName & arg)
+        {
+            ColumnWithTypeAndName tmp_arg = arg;
+            const auto & const_col = static_cast<const ColumnConst &>(*arg.column);
+            tmp_arg.column = const_col.getDataColumnPtr();
+
+            ColumnWithTypeAndName tmp_res = executeNonconstant(tmp_arg);
+            return ColumnWithTypeAndName{ColumnConst::create(tmp_res.column, arg.column->size()), tmp_res.type, tmp_res.name};
+        }
+
+        ColumnWithTypeAndName executeNonconstant(const ColumnWithTypeAndName & arg)
+        {
+            size_t size = arg.column->size();
+            auto res_col = ColumnString::create();
+            auto & res_str = static_cast<ColumnString &>(*res_col);
+            auto & res_offsets = res_str.getOffsets();
+            auto & res_chars = res_str.getChars();
+            res_offsets.reserve_exact(size);
+            res_chars.reserve(format.size() * size * 2);
+
+            String s;
+            WhichDataType which(arg.type);
+
+#define EXECUTE_BY_TYPE(IS_TYPE, GET_TYPE) \
+            else if (which.IS_TYPE()) \
+            { \
+                for (size_t i = 0; i < size; ++i) \
+                { \
+                    auto a = arg.column->GET_TYPE(i); \
+                    s = fmt::sprintf(format, a); \
+                    res_str.insertData(s.data(), s.size()); \
+                } \
+            }
+
+            if (false)
+                ;
+            EXECUTE_BY_TYPE(isNativeInt, getInt)
+            EXECUTE_BY_TYPE(isNativeUInt, getUInt)
+            EXECUTE_BY_TYPE(isFloat32, getFloat32)
+            EXECUTE_BY_TYPE(isFloat64, getFloat64)
+            else if (which.isStringOrFixedString())
+            {
+                for (size_t i = 0; i < size; ++i)
+                {
+                    auto a = arg.column->getDataAt(i).toView();
+                    s = fmt::sprintf(format, a);
+                    res_str.insertData(s.data(), s.size());
+                }
+            }
+            else throw Exception(
+                ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
+                "The argument type of function {} is {}, but native numeric or string type is expected",
+                FunctionPrintf::name,
+                arg.type->getName());
+#undef EXECUTE_BY_TYPE
+
+            ColumnWithTypeAndName res;
+            res.name = arg.name;
+            res.type = std::make_shared<DataTypeString>();
+            res.column = std::move(res_col);
+            return res;
+        }
+    };
+
+public:
+    static constexpr auto name = "printf";
+
+    static FunctionPtr create(ContextPtr context) { return std::make_shared<FunctionPrintf>(context); }
+
+    explicit FunctionPrintf(ContextPtr context_)
+        : context(context_), function_concat(FunctionFactory::instance().get("concat", context)) { }
+
+    String getName() const override { return name; }
+
+    bool isVariadic() const override { return true; }
+
+    bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; }
+
+    size_t getNumberOfArguments() const override { return 0; }
+
+    bool useDefaultImplementationForConstants() const override { return false; }
+    ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {0}; }
+
+    DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
+    {
+        if (arguments.empty())
+            throw Exception(
+                ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
+                "Number of arguments for function {} doesn't match: passed {}, should be at least 1",
+                getName(),
+                arguments.size());
+
+        /// First pattern argument must have string type
+        if (!isString(arguments[0]))
+            throw Exception(
+                ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
+                "The first argument type of function {} is {}, but String type is expected",
+                getName(),
+                arguments[0]->getName());
+
+        for (size_t i = 1; i < arguments.size(); ++i)
+        {
+            if (!isNativeNumber(arguments[i]) && !isStringOrFixedString(arguments[i]))
+                throw Exception(
+                    ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
+                    "The {}-th argument type of function {} is {}, but native numeric or string type is expected",
+                    i + 1,
+                    getName(),
+                    arguments[i]->getName());
+        }
+        return std::make_shared<DataTypeString>();
+    }
+
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
+    {
+        const ColumnPtr & c0 = arguments[0].column;
+        const ColumnConst * c0_const_string = typeid_cast<const ColumnConst *>(&*c0);
+        if (!c0_const_string)
+            throw Exception(ErrorCodes::ILLEGAL_COLUMN, "First argument of function {} must be constant string", getName());
+
+        String format = c0_const_string->getValue<String>();
+        auto instructions = buildInstructions(format, arguments, input_rows_count);
+
+        ColumnsWithTypeAndName concat_args(instructions.size());
+        for (size_t i = 0; i < instructions.size(); ++i)
+        {
+            std::cout << "instruction[" << i << "]:" << instructions[i].toString() << std::endl;
+            concat_args[i] = instructions[i].execute();
+            std::cout << "concat_args[" << i << "]:" << concat_args[i].dumpStructure() << std::endl;
+        }
+
+        auto res = function_concat->build(concat_args)->execute(concat_args, std::make_shared<DataTypeString>(), input_rows_count);
+        return res;
+    }
+
+private:
+    std::vector<Instruction> buildInstructions(const String & format , const ColumnsWithTypeAndName & arguments, size_t input_rows_count) const
+    {
+        std::vector<Instruction> instructions;
+        instructions.reserve(arguments.size());
+
+        auto append_instruction = [&](const char * begin, const char * end, const ColumnWithTypeAndName & arg)
+        {
+            Instruction instr;
+            instr.rows = input_rows_count;
+            instr.format = std::string_view(begin, end - begin);
+
+            size_t size = end - begin;
+            if (size > 1 && begin[0] == '%' and begin[1] != '%')
+            {
+                instr.is_literal = false;
+                instr.input = arg;
+            }
+            else
+            {
+                instr.is_literal = true;
+            }
+            instructions.emplace_back(std::move(instr));
+        };
+
+        auto check_index_range = [&](size_t idx)
+        {
+            if (idx >= arguments.size())
+                throw Exception(
+                    ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
+                    "Number of arguments for function {} doesn't match: passed {}, but format is {}",
+                    getName(),
+                    arguments.size(),
+                    format);
+        };
+
+        const char * begin = format.data();
+        const char * end = format.data() + format.size();
+        const char * curr = begin;
+        size_t idx = 0;
+        while (curr < end)
+        {
+            const char * tmp = curr;
+            bool is_first = curr == begin; /// If current instruction is the first one
+            bool is_literal = false;       /// If current instruction is literal string without any argument
+            if (is_first)
+            {
+                if (*curr != '%')
+                    is_literal = true;
+                else if (curr + 1 < end && *(curr + 1) == '%')
+                    is_literal = true;
+                else
+                    ++idx; /// Skip first argument if first instruction is not literal
+            }
+
+            if (!is_literal)
+                ++curr;
+
+            while (curr < end)
+            {
+                if (*curr != '%')
+                    ++curr;
+                else if (curr + 1 < end && *(curr + 1) == '%')
+                    curr += 2;
+                else
+                {
+                    check_index_range(idx);
+                    append_instruction(tmp, curr, arguments[idx]);
+                    ++idx;
+                    break;
+                }
+            }
+
+            if (curr == end)
+            {
+                check_index_range(idx);
+                append_instruction(tmp, curr, arguments[idx]);
+                ++idx;
+            }
+        }
+
+        /// Check if all arguments are used
+        if (idx != arguments.size())
+            throw Exception(
+                ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
+                "Number of arguments for function {} doesn't match: passed {}, but format is {}",
+                getName(),
+                arguments.size(),
+                format);
+
+        return instructions;
+    }
+};
+
+}
+
+REGISTER_FUNCTION(Printf)
+{
+    factory.registerFunction<FunctionPrintf>();
+}
+
+}
diff --git a/tests/queries/0_stateless/032010_printf.reference b/tests/queries/0_stateless/032010_printf.reference
new file mode 100644
index 00000000000..58501cbd0fc
--- /dev/null
+++ b/tests/queries/0_stateless/032010_printf.reference
@@ -0,0 +1,16 @@
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
+1
diff --git a/tests/queries/0_stateless/032010_printf.sql b/tests/queries/0_stateless/032010_printf.sql
new file mode 100644
index 00000000000..58fe081e499
--- /dev/null
+++ b/tests/queries/0_stateless/032010_printf.sql
@@ -0,0 +1,24 @@
+-- Testing integer formats
+select printf('%%d: %d', 123) = '%d: 123';
+select printf('%%i: %i', 123) = '%i: 123';
+select printf('%%u: %u', 123) = '%u: 123';
+select printf('%%o: %o', 123) = '%o: 173';
+select printf('%%x: %x', 123) = '%x: 7b';
+select printf('%%X: %X', 123) = '%X: 7B';
+
+-- Testing floating point formats
+select printf('%%f: %f', 123.456) = '%f: 123.456000';
+select printf('%%F: %F', 123.456) = '%F: 123.456000';
+select printf('%%e: %e', 123.456) = '%e: 1.234560e+02';
+select printf('%%E: %E', 123.456) = '%E: 1.234560E+02';
+select printf('%%g: %g', 123.456) = '%g: 123.456';
+select printf('%%G: %G', 123.456) = '%G: 123.456';
+select printf('%%a: %a', 123.456) = '%a: 0x1.edd2f1a9fbe77p+6';
+select printf('%%A: %A', 123.456) = '%A: 0X1.EDD2F1A9FBE77P+6';
+
+-- Testing character formats
+select printf('%%s: %s', 'abc') = '%s: abc';
+
+
+-- Testing the %% specifier
+select printf('%%%%: %%') = '%%: %';
\ No newline at end of file

From 64de996c03a6d0d685da6d6aa6c401a9ce710d8f Mon Sep 17 00:00:00 2001
From: JackyWoo <wuchienchao@qq.com>
Date: Tue, 9 Jul 2024 17:46:44 +0800
Subject: [PATCH 0171/1488] Fix address sanitizer container-overflow

---
 src/Storages/Statistics/StatisticsCountMinSketch.cpp |  7 +++----
 .../02864_statistics_estimate_predicate.reference    |  2 +-
 .../02864_statistics_estimate_predicate.sql          | 12 ++++++------
 3 files changed, 10 insertions(+), 11 deletions(-)

diff --git a/src/Storages/Statistics/StatisticsCountMinSketch.cpp b/src/Storages/Statistics/StatisticsCountMinSketch.cpp
index abd5cf17946..3e91b4052c4 100644
--- a/src/Storages/Statistics/StatisticsCountMinSketch.cpp
+++ b/src/Storages/Statistics/StatisticsCountMinSketch.cpp
@@ -33,10 +33,9 @@ Float64 StatisticsCountMinSketch::estimateEqual(const Field & val) const
 {
     if (data_type->isValueRepresentedByNumber())
     {
-        /// convertFieldToType will
+        /// 'val' maybe number or string, method 'convertFieldToType' will
         ///     1. convert string to number, date, datetime, IPv4, Decimal etc
-        ///     2. do other conversion
-        ///     3. return null if val larger than the range of data_type
+        ///     2. return null if val larger than the range of data_type
         auto val_converted = convertFieldToType(val, *data_type);
         if (val_converted.isNull())
             return 0;
@@ -87,7 +86,7 @@ void StatisticsCountMinSketch::deserialize(ReadBuffer & buf)
     readIntBinary(size, buf);
 
     Sketch::vector_bytes bytes;
-    bytes.reserve(size);
+    bytes.resize(size); /// To avoid 'container-overflow' in AddressSanitizer checking
     buf.readStrict(reinterpret_cast<char *>(bytes.data()), size);
 
     sketch = Sketch::deserialize(bytes.data(), size);
diff --git a/tests/queries/0_stateless/02864_statistics_estimate_predicate.reference b/tests/queries/0_stateless/02864_statistics_estimate_predicate.reference
index 83b921af511..4e41c32750f 100644
--- a/tests/queries/0_stateless/02864_statistics_estimate_predicate.reference
+++ b/tests/queries/0_stateless/02864_statistics_estimate_predicate.reference
@@ -17,7 +17,7 @@ Test statistics count_min:
 Test statistics multi-types:
         Prewhere info
           Prewhere filter
-          Prewhere filter column: and(equals(a, \'0\'), less(c, -90), greater(b, 900_UInt16)) (removed)
+          Prewhere filter column: and(equals(a, \'0\'), less(c, -90), greater(b, 900)) (removed)
         Prewhere info
           Prewhere filter
           Prewhere filter column: and(equals(a, \'10000\'), equals(b, 0), less(c, 0)) (removed)
diff --git a/tests/queries/0_stateless/02864_statistics_estimate_predicate.sql b/tests/queries/0_stateless/02864_statistics_estimate_predicate.sql
index 7fcb85d80f5..91b4f2d05cb 100644
--- a/tests/queries/0_stateless/02864_statistics_estimate_predicate.sql
+++ b/tests/queries/0_stateless/02864_statistics_estimate_predicate.sql
@@ -26,11 +26,11 @@ SELECT 'Test statistics TDigest:';
 ALTER TABLE tab ADD STATISTICS b, c TYPE tdigest;
 ALTER TABLE tab MATERIALIZE STATISTICS b, c;
 
-SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_String', '')
+SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String', '')
 FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE b > 0/*9990*/ and c < -98/*100*/)
 WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
 
-SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_String', '')
+SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String', '')
 FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE b = 0/*1000*/ and c < -98/*100*/)
 WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
 
@@ -41,7 +41,7 @@ SELECT 'Test statistics Uniq:';
 
 ALTER TABLE tab ADD STATISTICS b TYPE uniq, tdigest;
 ALTER TABLE tab MATERIALIZE STATISTICS b;
-SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_String', '')
+SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String', '')
 FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE c = 0/*1000*/ and b = 0/*10*/)
 WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
 
@@ -55,7 +55,7 @@ ALTER TABLE tab ADD STATISTICS b TYPE count_min;
 ALTER TABLE tab ADD STATISTICS c TYPE count_min;
 ALTER TABLE tab MATERIALIZE STATISTICS a, b, c;
 
-SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_String', '')
+SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String', '')
 FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE c = 0/*100*/ and b = 0/*10*/ and a = '0'/*1*/) xx
 WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
 
@@ -69,11 +69,11 @@ ALTER TABLE tab ADD STATISTICS b TYPE count_min, uniq, tdigest;
 ALTER TABLE tab ADD STATISTICS c TYPE count_min, uniq, tdigest;
 ALTER TABLE tab MATERIALIZE STATISTICS a, b, c;
 
-SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_String', '')
+SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String', '')
 FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE c < -90/*900*/ and b > 900/*990*/ and a = '0'/*1*/)
 WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
 
-SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_String', '')
+SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String', '')
 FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE c < 0/*9900*/ and b = 0/*10*/ and a = '10000'/*0*/)
 WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
 

From 751ee04e75a75629ba9b939d02070d2444afcab7 Mon Sep 17 00:00:00 2001
From: Sema Checherinda <Sema.Checherinda@clickhouse.com>
Date: Tue, 9 Jul 2024 13:10:04 +0200
Subject: [PATCH 0172/1488] fix tidy build, canonize
 01275_parallel_mv.reference

---
 src/Interpreters/InterpreterInsertQuery.cpp   |   2 +-
 .../0_stateless/01275_parallel_mv.reference   | 124 +++++++++---------
 2 files changed, 63 insertions(+), 63 deletions(-)

diff --git a/src/Interpreters/InterpreterInsertQuery.cpp b/src/Interpreters/InterpreterInsertQuery.cpp
index 636db546a79..6fd53070f41 100644
--- a/src/Interpreters/InterpreterInsertQuery.cpp
+++ b/src/Interpreters/InterpreterInsertQuery.cpp
@@ -606,7 +606,7 @@ QueryPipeline InterpreterInsertQuery::buildInsertSelectPipeline(ASTInsertQuery &
 
     size_t sink_streams_size = table->supportsParallelInsert() ? std::max<size_t>(1, settings.max_insert_threads) : 1;
 
-    size_t views_involved =  table->isView() || DatabaseCatalog::instance().getDependentViews(table->getStorageID()).size() > 0;
+    size_t views_involved =  table->isView() || !DatabaseCatalog::instance().getDependentViews(table->getStorageID()).empty();
     if (!settings.parallel_view_processing && views_involved)
     {
         sink_streams_size = 1;
diff --git a/tests/queries/0_stateless/01275_parallel_mv.reference b/tests/queries/0_stateless/01275_parallel_mv.reference
index dadf2f35e6e..221d0b5ce8d 100644
--- a/tests/queries/0_stateless/01275_parallel_mv.reference
+++ b/tests/queries/0_stateless/01275_parallel_mv.reference
@@ -4,13 +4,13 @@
 
 select 'optimize_trivial_insert_select=0', 'max_insert_threads=0';
 optimize_trivial_insert_select=0	max_insert_threads=0
-insert into testX select number from numbers(10) settings
+insert into testX select number from numbers(200) settings
     log_queries=1,
     parallel_view_processing=0,
     optimize_trivial_insert_select=0,
     max_insert_threads=0; -- { serverError FUNCTION_THROW_IF_VALUE_IS_NON_ZERO }
 system flush logs;
-select arrayUniq(thread_ids) from system.query_log where
+select peak_threads_usage from system.query_log where
     current_database = currentDatabase() and
     type != 'QueryStart' and
     query like '%insert into testX %' and
@@ -19,46 +19,46 @@ select arrayUniq(thread_ids) from system.query_log where
     Settings['max_insert_threads'] = '0';
 2
 select count() from testX;
-10
+200
 select count() from testXA;
-10
+200
 select count() from testXB;
 0
 select count() from testXC;
-10
-select 'optimize_trivial_insert_select=0', 'max_insert_threads=16';
-optimize_trivial_insert_select=0	max_insert_threads=16
-insert into testX select number from numbers(10) settings
+200
+select 'optimize_trivial_insert_select=0', 'max_insert_threads=5';
+optimize_trivial_insert_select=0	max_insert_threads=5
+insert into testX select number from numbers(200) settings
     log_queries=1,
     parallel_view_processing=0,
     optimize_trivial_insert_select=0,
-    max_insert_threads=16; -- { serverError FUNCTION_THROW_IF_VALUE_IS_NON_ZERO }
+    max_insert_threads=5; -- { serverError FUNCTION_THROW_IF_VALUE_IS_NON_ZERO }
 system flush logs;
-select arrayUniq(thread_ids) from system.query_log where
+select peak_threads_usage from system.query_log where
     current_database = currentDatabase() and
     type != 'QueryStart' and
     query like '%insert into testX %' and
     Settings['parallel_view_processing'] = '0' and
     Settings['optimize_trivial_insert_select'] = '0' and
-    Settings['max_insert_threads'] = '16';
+    Settings['max_insert_threads'] = '5';
 2
 select count() from testX;
-20
+400
 select count() from testXA;
-20
+400
 select count() from testXB;
 0
 select count() from testXC;
-20
+400
 select 'optimize_trivial_insert_select=1', 'max_insert_threads=0';
 optimize_trivial_insert_select=1	max_insert_threads=0
-insert into testX select number from numbers(10) settings
+insert into testX select number from numbers(200) settings
     log_queries=1,
     parallel_view_processing=0,
     optimize_trivial_insert_select=1,
     max_insert_threads=0; -- { serverError FUNCTION_THROW_IF_VALUE_IS_NON_ZERO }
 system flush logs;
-select arrayUniq(thread_ids) from system.query_log where
+select peak_threads_usage from system.query_log where
     current_database = currentDatabase() and
     type != 'QueryStart' and
     query like '%insert into testX %' and
@@ -67,46 +67,46 @@ select arrayUniq(thread_ids) from system.query_log where
     Settings['max_insert_threads'] = '0';
 2
 select count() from testX;
-30
+600
 select count() from testXA;
-30
+600
 select count() from testXB;
 0
 select count() from testXC;
-30
-select 'optimize_trivial_insert_select=1', 'max_insert_threads=16';
-optimize_trivial_insert_select=1	max_insert_threads=16
-insert into testX select number from numbers(10) settings
+600
+select 'optimize_trivial_insert_select=1', 'max_insert_threads=5';
+optimize_trivial_insert_select=1	max_insert_threads=5
+insert into testX select number from numbers(200) settings
     log_queries=1,
     parallel_view_processing=0,
     optimize_trivial_insert_select=1,
-    max_insert_threads=16; -- { serverError FUNCTION_THROW_IF_VALUE_IS_NON_ZERO }
+    max_insert_threads=5; -- { serverError FUNCTION_THROW_IF_VALUE_IS_NON_ZERO }
 system flush logs;
-select arrayUniq(thread_ids) from system.query_log where
+select peak_threads_usage from system.query_log where
     current_database = currentDatabase() and
     type != 'QueryStart' and
     query like '%insert into testX %' and
     Settings['parallel_view_processing'] = '0' and
     Settings['optimize_trivial_insert_select'] = '1' and
-    Settings['max_insert_threads'] = '16';
+    Settings['max_insert_threads'] = '5';
 2
 select count() from testX;
-40
+800
 select count() from testXA;
-40
+800
 select count() from testXB;
 0
 select count() from testXC;
-40
+800
 select 'optimize_trivial_insert_select=0', 'max_insert_threads=0';
 optimize_trivial_insert_select=0	max_insert_threads=0
-insert into testX select number from numbers(10) settings
+insert into testX select number from numbers(200) settings
     log_queries=1,
     parallel_view_processing=1,
     optimize_trivial_insert_select=0,
     max_insert_threads=0; -- { serverError FUNCTION_THROW_IF_VALUE_IS_NON_ZERO }
 system flush logs;
-select arrayUniq(thread_ids) from system.query_log where
+select peak_threads_usage from system.query_log where
     current_database = currentDatabase() and
     type != 'QueryStart' and
     query like '%insert into testX %' and
@@ -115,82 +115,82 @@ select arrayUniq(thread_ids) from system.query_log where
     Settings['max_insert_threads'] = '0';
 5
 select count() from testX;
-50
+1000
 select count() from testXA;
-50
+1000
 select count() from testXB;
 0
 select count() from testXC;
-50
-select 'optimize_trivial_insert_select=0', 'max_insert_threads=16';
-optimize_trivial_insert_select=0	max_insert_threads=16
-insert into testX select number from numbers(10) settings
+1000
+select 'optimize_trivial_insert_select=0', 'max_insert_threads=5';
+optimize_trivial_insert_select=0	max_insert_threads=5
+insert into testX select number from numbers(200) settings
     log_queries=1,
     parallel_view_processing=1,
     optimize_trivial_insert_select=0,
-    max_insert_threads=16; -- { serverError FUNCTION_THROW_IF_VALUE_IS_NON_ZERO }
+    max_insert_threads=5; -- { serverError FUNCTION_THROW_IF_VALUE_IS_NON_ZERO }
 system flush logs;
-select arrayUniq(thread_ids) from system.query_log where
+select peak_threads_usage from system.query_log where
     current_database = currentDatabase() and
     type != 'QueryStart' and
     query like '%insert into testX %' and
     Settings['parallel_view_processing'] = '1' and
     Settings['optimize_trivial_insert_select'] = '0' and
-    Settings['max_insert_threads'] = '16';
-18
+    Settings['max_insert_threads'] = '5';
+12
 select count() from testX;
-60
+1190
 select count() from testXA;
-60
+1130
 select count() from testXB;
-0
-select count() from testXC;
 60
+select count() from testXC;
+1130
 select 'optimize_trivial_insert_select=1', 'max_insert_threads=0';
 optimize_trivial_insert_select=1	max_insert_threads=0
-insert into testX select number from numbers(10) settings
+insert into testX select number from numbers(200) settings
     log_queries=1,
     parallel_view_processing=1,
     optimize_trivial_insert_select=1,
     max_insert_threads=0; -- { serverError FUNCTION_THROW_IF_VALUE_IS_NON_ZERO }
 system flush logs;
-select arrayUniq(thread_ids) from system.query_log where
+select peak_threads_usage from system.query_log where
     current_database = currentDatabase() and
     type != 'QueryStart' and
     query like '%insert into testX %' and
     Settings['parallel_view_processing'] = '1' and
     Settings['optimize_trivial_insert_select'] = '1' and
     Settings['max_insert_threads'] = '0';
-5
+2
 select count() from testX;
-70
+1390
 select count() from testXA;
-70
+1330
 select count() from testXB;
-0
+60
 select count() from testXC;
-70
-select 'optimize_trivial_insert_select=1', 'max_insert_threads=16';
-optimize_trivial_insert_select=1	max_insert_threads=16
-insert into testX select number from numbers(10) settings
+1330
+select 'optimize_trivial_insert_select=1', 'max_insert_threads=5';
+optimize_trivial_insert_select=1	max_insert_threads=5
+insert into testX select number from numbers(200) settings
     log_queries=1,
     parallel_view_processing=1,
     optimize_trivial_insert_select=1,
-    max_insert_threads=16; -- { serverError FUNCTION_THROW_IF_VALUE_IS_NON_ZERO }
+    max_insert_threads=5; -- { serverError FUNCTION_THROW_IF_VALUE_IS_NON_ZERO }
 system flush logs;
-select arrayUniq(thread_ids) from system.query_log where
+select peak_threads_usage from system.query_log where
     current_database = currentDatabase() and
     type != 'QueryStart' and
     query like '%insert into testX %' and
     Settings['parallel_view_processing'] = '1' and
     Settings['optimize_trivial_insert_select'] = '1' and
-    Settings['max_insert_threads'] = '16';
-18
+    Settings['max_insert_threads'] = '5';
+7
 select count() from testX;
-80
+1590
 select count() from testXA;
-80
+1480
 select count() from testXB;
-0
+160
 select count() from testXC;
-80
+1490

From 5daa28d5c5726478cbd635ce0ca6b8ce77f5fdce Mon Sep 17 00:00:00 2001
From: Sema Checherinda <Sema.Checherinda@clickhouse.com>
Date: Tue, 9 Jul 2024 13:27:58 +0200
Subject: [PATCH 0173/1488] mark heavy tests with no-asan tag, they are slow
 with asan and asuze

---
 .../03008_deduplication_insert_several_blocks_nonreplicated.sh  | 2 +-
 .../03008_deduplication_insert_several_blocks_replicated.sh     | 2 +-
 ...8_deduplication_mv_generates_several_blocks_nonreplicated.sh | 2 +-
 ...3008_deduplication_mv_generates_several_blocks_replicated.sh | 2 +-
 ...008_deduplication_several_mv_into_one_table_nonreplicated.sh | 2 +-
 .../03008_deduplication_several_mv_into_one_table_replicated.sh | 2 +-
 6 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_nonreplicated.sh b/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_nonreplicated.sh
index 49eb52b47fd..0791c7566f9 100755
--- a/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_nonreplicated.sh
+++ b/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_nonreplicated.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Tags: long, no-fasttest, no-parallel
+# Tags: long, no-fasttest, no-parallel, no-asan
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
diff --git a/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_replicated.sh b/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_replicated.sh
index 53af06d4a6f..661dfa5f930 100755
--- a/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_replicated.sh
+++ b/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_replicated.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Tags: long, no-fasttest, no-parallel
+# Tags: long, no-fasttest, no-parallel, no-asan
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
diff --git a/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_nonreplicated.sh b/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_nonreplicated.sh
index 7d4f5240cd1..10083506af4 100755
--- a/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_nonreplicated.sh
+++ b/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_nonreplicated.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Tags: long, no-fasttest, no-parallel
+# Tags: long, no-fasttest, no-parallel, no-asan
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
diff --git a/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_replicated.sh b/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_replicated.sh
index 109d1674f3a..0d5158d18cd 100755
--- a/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_replicated.sh
+++ b/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_replicated.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Tags: long, no-fasttest, no-parallel
+# Tags: long, no-fasttest, no-parallel, no-asan
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
diff --git a/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_nonreplicated.sh b/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_nonreplicated.sh
index fe3d610a758..a9a135d6839 100755
--- a/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_nonreplicated.sh
+++ b/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_nonreplicated.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Tags: long, no-fasttest, no-parallel
+# Tags: long, no-fasttest, no-parallel, no-asan
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
diff --git a/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_replicated.sh b/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_replicated.sh
index 9adee6d53d4..49d556e70f9 100755
--- a/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_replicated.sh
+++ b/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_replicated.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Tags: long, no-fasttest, no-parallel
+# Tags: long, no-fasttest, no-parallel, no-asan
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh

From d89ed1fdcb34b23858e22325f19f9e0fc38e60bb Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Tue, 9 Jul 2024 15:22:20 +0000
Subject: [PATCH 0174/1488] disable WAL

---
 src/Coordination/RocksDBContainer.h | 21 ++++++++++++++-------
 1 file changed, 14 insertions(+), 7 deletions(-)

diff --git a/src/Coordination/RocksDBContainer.h b/src/Coordination/RocksDBContainer.h
index a4a236f332e..46dc755aad8 100644
--- a/src/Coordination/RocksDBContainer.h
+++ b/src/Coordination/RocksDBContainer.h
@@ -8,6 +8,7 @@
 #include <IO/ReadBufferFromString.h>
 
 #include <rocksdb/convenience.h>
+#include <rocksdb/options.h>
 #include <rocksdb/status.h>
 #include <rocksdb/table.h>
 #include <rocksdb/snapshot.h>
@@ -195,7 +196,6 @@ public:
                 rocksdb_dir, status.ToString());
         }
         rocksdb_ptr = std::unique_ptr<rocksdb::DB>(db);
-        /// storage_ptr = storage_;
         initialized = true;
     }
 
@@ -312,8 +312,10 @@ public:
         }
         else if (status.IsNotFound())
         {
-            // storage->addDigest(value, key);
-            status = rocksdb_ptr->Put(rocksdb::WriteOptions(), encoded_key, value.getEncodedString());
+            rocksdb::WriteOptions write_options;
+            write_options.disableWAL = true;
+
+            status = rocksdb_ptr->Put(write_options, encoded_key, value.getEncodedString());
             if (status.ok())
             {
                 counter++;
@@ -336,7 +338,10 @@ public:
         else if (!status.ok())
             throw Exception(ErrorCodes::ROCKSDB_ERROR, "Got rocksdb error during get. The error message is {}.", status.ToString());
 
-        status = rocksdb_ptr->Put(rocksdb::WriteOptions(), encoded_key, value.getEncodedString());
+        rocksdb::WriteOptions write_options;
+        write_options.disableWAL = true;
+
+        status = rocksdb_ptr->Put(write_options, encoded_key, value.getEncodedString());
         if (status.ok())
             counter += increase_counter;
         else
@@ -361,7 +366,11 @@ public:
     {
         /// storage->removeDigest(value, key);
         const std::string & encoded_key = getEncodedKey(key);
-        auto status = rocksdb_ptr->Delete(rocksdb::WriteOptions(), encoded_key);
+
+        rocksdb::WriteOptions write_options;
+        write_options.disableWAL = true;
+
+        auto status = rocksdb_ptr->Delete(write_options, encoded_key);
         if (status.IsNotFound())
             return false;
         if (status.ok())
@@ -445,8 +454,6 @@ private:
 
     std::unique_ptr<rocksdb::DB> rocksdb_ptr;
 
-    /// Storage* storage_ptr;
-
     const rocksdb::Snapshot * snapshot;
 
     bool snapshot_mode{false};

From fbaf6119eaad06c7ceb4eaa54cc346fbad20f50a Mon Sep 17 00:00:00 2001
From: Anton Popov <anton@clickhouse.com>
Date: Tue, 9 Jul 2024 17:08:19 +0000
Subject: [PATCH 0175/1488] fix indexHint with fuzzer

---
 src/Functions/indexHint.h                                   | 4 ++++
 tests/queries/0_stateless/03204_index_hint_fuzzer.reference | 1 +
 tests/queries/0_stateless/03204_index_hint_fuzzer.sql       | 1 +
 3 files changed, 6 insertions(+)
 create mode 100644 tests/queries/0_stateless/03204_index_hint_fuzzer.reference
 create mode 100644 tests/queries/0_stateless/03204_index_hint_fuzzer.sql

diff --git a/src/Functions/indexHint.h b/src/Functions/indexHint.h
index 3b71c7a5585..a71e1555c78 100644
--- a/src/Functions/indexHint.h
+++ b/src/Functions/indexHint.h
@@ -42,6 +42,10 @@ public:
 
     bool useDefaultImplementationForNulls() const override { return false; }
 
+    bool useDefaultImplementationForLowCardinalityColumns() const override { return false; }
+
+    bool useDefaultImplementationForSparseColumns() const override { return false; }
+
     bool isSuitableForConstantFolding() const override { return false; }
 
     bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; }
diff --git a/tests/queries/0_stateless/03204_index_hint_fuzzer.reference b/tests/queries/0_stateless/03204_index_hint_fuzzer.reference
new file mode 100644
index 00000000000..f750ba35102
--- /dev/null
+++ b/tests/queries/0_stateless/03204_index_hint_fuzzer.reference
@@ -0,0 +1 @@
+(1,1)
diff --git a/tests/queries/0_stateless/03204_index_hint_fuzzer.sql b/tests/queries/0_stateless/03204_index_hint_fuzzer.sql
new file mode 100644
index 00000000000..5794f3eee53
--- /dev/null
+++ b/tests/queries/0_stateless/03204_index_hint_fuzzer.sql
@@ -0,0 +1 @@
+SELECT tuple(indexHint(toLowCardinality('aaa')), 1);

From 246f421f2402799fd11b22a608b4d0d497cb8438 Mon Sep 17 00:00:00 2001
From: Sema Checherinda <Sema.Checherinda@clickhouse.com>
Date: Tue, 9 Jul 2024 16:33:57 +0200
Subject: [PATCH 0176/1488] merge tree sink cancel delayed_chunk

---
 .../Transforms/ExceptionKeepingTransform.h        |  2 +-
 src/Storages/MergeTree/MergeTreeSink.cpp          |  9 +++++++++
 .../MergeTree/ReplicatedMergeTreeSink.cpp         | 15 +++++++++++++++
 src/Storages/MergeTree/ReplicatedMergeTreeSink.h  |  2 ++
 4 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/src/Processors/Transforms/ExceptionKeepingTransform.h b/src/Processors/Transforms/ExceptionKeepingTransform.h
index 000b5da798a..9aa33a8cbe5 100644
--- a/src/Processors/Transforms/ExceptionKeepingTransform.h
+++ b/src/Processors/Transforms/ExceptionKeepingTransform.h
@@ -52,7 +52,7 @@ protected:
     virtual void onConsume(Chunk chunk) = 0;
     virtual GenerateResult onGenerate() = 0;
     virtual void onFinish() {}
-    virtual void onException(std::exception_ptr /* exception */) {}
+    virtual void onException(std::exception_ptr /* exception */) { }
 
 public:
     ExceptionKeepingTransform(const Block & in_header, const Block & out_header, bool ignore_on_start_and_finish_ = true);
diff --git a/src/Storages/MergeTree/MergeTreeSink.cpp b/src/Storages/MergeTree/MergeTreeSink.cpp
index 05751e0fa6f..2e8f6db6868 100644
--- a/src/Storages/MergeTree/MergeTreeSink.cpp
+++ b/src/Storages/MergeTree/MergeTreeSink.cpp
@@ -56,6 +56,15 @@ void MergeTreeSink::onFinish()
 
 void MergeTreeSink::onCancel()
 {
+    if (!delayed_chunk)
+            return;
+
+    for (auto & partition : delayed_chunk->partitions)
+    {
+        partition.temp_part.cancel();
+    }
+
+    delayed_chunk.reset();
 }
 
 void MergeTreeSink::consume(Chunk chunk)
diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp
index 4b4f4c33e7d..93f82a5a789 100644
--- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp
+++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp
@@ -1155,6 +1155,21 @@ void ReplicatedMergeTreeSinkImpl<async_insert>::onFinish()
     finishDelayedChunk(std::make_shared<ZooKeeperWithFaultInjection>(zookeeper));
 }
 
+
+template<bool async_insert>
+void ReplicatedMergeTreeSinkImpl<async_insert>::onCancel()
+{
+    if (!delayed_chunk)
+        return;
+
+    for (auto & partition : delayed_chunk->partitions)
+    {
+        partition.temp_part.cancel();
+    }
+
+    delayed_chunk.reset();
+}
+
 template<bool async_insert>
 void ReplicatedMergeTreeSinkImpl<async_insert>::waitForQuorum(
     const ZooKeeperWithFaultInjectionPtr & zookeeper,
diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.h b/src/Storages/MergeTree/ReplicatedMergeTreeSink.h
index 39623c20584..b1796a35ed2 100644
--- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.h
+++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.h
@@ -54,6 +54,8 @@ public:
     void consume(Chunk chunk) override;
     void onFinish() override;
 
+    void onCancel() override;
+
     String getName() const override { return "ReplicatedMergeTreeSink"; }
 
     /// For ATTACHing existing data on filesystem.

From 27738747e0da63684ed1d831fd823e9966923409 Mon Sep 17 00:00:00 2001
From: Alexander Tokmakov <tavplubix@clickhouse.com>
Date: Tue, 9 Jul 2024 23:59:53 +0200
Subject: [PATCH 0177/1488] save fatal logs to file

---
 src/Client/ClientBase.cpp | 12 +++++++++++-
 src/Client/ClientBase.h   |  6 +++++-
 tests/clickhouse-test     | 23 +++++++++++++++++++++++
 3 files changed, 39 insertions(+), 2 deletions(-)

diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp
index cda80fbdc03..b72fa0edf1e 100644
--- a/src/Client/ClientBase.cpp
+++ b/src/Client/ClientBase.cpp
@@ -3068,6 +3068,8 @@ void ClientBase::init(int argc, char ** argv)
         ("max_memory_usage_in_client", po::value<std::string>(), "Set memory limit in client/local server")
 
         ("fuzzer-args", po::value<std::string>(), "Command line arguments for the LLVM's libFuzzer driver. Only relevant if the application is compiled with libFuzzer.")
+
+        ("client_log_file", po::value<std::string>(), "Path to a file for logging fatal errors in client")
     ;
 
     addOptions(options_description);
@@ -3231,7 +3233,15 @@ void ClientBase::init(int argc, char ** argv)
     /// We don't setup signal handlers for SIGINT, SIGQUIT, SIGTERM because we don't
     /// have an option for client to shutdown gracefully.
 
-    fatal_channel_ptr = new Poco::ConsoleChannel;
+    fatal_channel_ptr = new Poco::SplitterChannel;
+    fatal_console_channel_ptr = new Poco::ConsoleChannel;
+    fatal_channel_ptr->addChannel(fatal_console_channel_ptr);
+    if (options.count("client_log_file"))
+    {
+        fatal_file_channel_ptr = new Poco::SimpleFileChannel(options["client_log_file"].as<std::string>());
+        fatal_channel_ptr->addChannel(fatal_file_channel_ptr);
+    }
+
     fatal_log = createLogger("ClientBase", fatal_channel_ptr.get(), Poco::Message::PRIO_FATAL);
     signal_listener = std::make_unique<SignalListener>(nullptr, fatal_log);
     signal_listener_thread.start(*signal_listener);
diff --git a/src/Client/ClientBase.h b/src/Client/ClientBase.h
index ffc385a2e98..31039ed1e53 100644
--- a/src/Client/ClientBase.h
+++ b/src/Client/ClientBase.h
@@ -12,6 +12,8 @@
 #include <Core/ExternalTable.h>
 #include <Poco/Util/Application.h>
 #include <Poco/ConsoleChannel.h>
+#include <Poco/SimpleFileChannel.h>
+#include <Poco/SplitterChannel.h>
 #include <Interpreters/Context.h>
 #include <Client/Suggest.h>
 #include <boost/program_options.hpp>
@@ -213,7 +215,9 @@ protected:
     ContextMutablePtr global_context;
 
     LoggerPtr fatal_log;
-    Poco::AutoPtr<Poco::Channel> fatal_channel_ptr;
+    Poco::AutoPtr<Poco::SplitterChannel> fatal_channel_ptr;
+    Poco::AutoPtr<Poco::Channel> fatal_console_channel_ptr;
+    Poco::AutoPtr<Poco::Channel> fatal_file_channel_ptr;
     Poco::Thread signal_listener_thread;
     std::unique_ptr<Poco::Runnable> signal_listener;
 
diff --git a/tests/clickhouse-test b/tests/clickhouse-test
index 8e7002af889..91cecaadf7a 100755
--- a/tests/clickhouse-test
+++ b/tests/clickhouse-test
@@ -1519,6 +1519,15 @@ class TestCase:
         start_time = args.testcase_start_time
         database = args.testcase_database
 
+        if args.client_log:
+            log_opt = " --client_log_file=" + args.client_log
+            client_options += log_opt
+            os.environ["CLICKHOUSE_CLIENT_OPT"] = (
+                os.environ["CLICKHOUSE_CLIENT_OPT"]
+                if "CLICKHOUSE_CLIENT_OPT" in os.environ
+                else ""
+            ) + log_opt
+
         # This is for .sh tests
         os.environ["CLICKHOUSE_LOG_COMMENT"] = args.testcase_basename
 
@@ -2832,6 +2841,15 @@ def main(args):
         else:
             print(colored("\nNo queries hung.", args, "green", attrs=["bold"]))
 
+    if args.client_log:
+        if os.path.exists(args.client_log):
+            with open(args.client_log, "rb") as stream:
+                content = stream.read().decode()
+                if len(content):
+                    print("Has fatal logs from client:\n")
+                    print(content)
+            os.remove(args.client_log)
+
     if len(restarted_tests) > 0:
         print("\nSome tests were restarted:\n")
 
@@ -3233,6 +3251,11 @@ def parse_args():
         default=False,
         help="Replace ordinary MergeTree engine with SharedMergeTree",
     )
+    parser.add_argument(
+        "--client-log",
+        default="./client.fatal.log",
+        help="Path to file for fatal logs from client",
+    )
 
     return parser.parse_args()
 

From e0e3842772ead940f53346cc087ea54e5e6aa8fa Mon Sep 17 00:00:00 2001
From: taiyang-li <654010905@qq.com>
Date: Wed, 10 Jul 2024 10:15:33 +0800
Subject: [PATCH 0178/1488] support printf

---
 src/Functions/printf.cpp                           | 6 +++---
 utils/check-style/aspell-ignore/en/aspell-dict.txt | 1 +
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/Functions/printf.cpp b/src/Functions/printf.cpp
index cb21d5e39ad..b2a0143a3f8 100644
--- a/src/Functions/printf.cpp
+++ b/src/Functions/printf.cpp
@@ -52,7 +52,7 @@ private:
 
         String toString() const
         {
-            std::ostringstream oss;
+            std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM
             oss << "format:" << format << ", rows:" << rows << ", is_literal:" << is_literal << ", input:" << input.dumpStructure()
                       << std::endl;
             return oss.str();
@@ -195,9 +195,9 @@ public:
         ColumnsWithTypeAndName concat_args(instructions.size());
         for (size_t i = 0; i < instructions.size(); ++i)
         {
-            std::cout << "instruction[" << i << "]:" << instructions[i].toString() << std::endl;
+            // std::cout << "instruction[" << i << "]:" << instructions[i].toString() << std::endl;
             concat_args[i] = instructions[i].execute();
-            std::cout << "concat_args[" << i << "]:" << concat_args[i].dumpStructure() << std::endl;
+            // std::cout << "concat_args[" << i << "]:" << concat_args[i].dumpStructure() << std::endl;
         }
 
         auto res = function_concat->build(concat_args)->execute(concat_args, std::make_shared<DataTypeString>(), input_rows_count);
diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt
index 943caf918d6..21a9b759466 100644
--- a/utils/check-style/aspell-ignore/en/aspell-dict.txt
+++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt
@@ -2264,6 +2264,7 @@ prettyspacemonoblock
 prettyspacenoescapes
 prettyspacenoescapesmonoblock
 prewhere
+printf
 privateKeyFile
 privateKeyPassphraseHandler
 prlimit

From 8b1bc00e9a6462a6dce2946510d83a85bc69a139 Mon Sep 17 00:00:00 2001
From: taiyang-li <654010905@qq.com>
Date: Wed, 10 Jul 2024 10:52:53 +0800
Subject: [PATCH 0179/1488] fix style

---
 src/Functions/printf.cpp | 33 +++++++++++++++++----------------
 1 file changed, 17 insertions(+), 16 deletions(-)

diff --git a/src/Functions/printf.cpp b/src/Functions/printf.cpp
index b2a0143a3f8..c7c6bd228a7 100644
--- a/src/Functions/printf.cpp
+++ b/src/Functions/printf.cpp
@@ -9,19 +9,19 @@
 #include <IO/WriteHelpers.h>
 
 #include <memory>
+#include <sstream>
 #include <string>
 #include <vector>
 #include <fmt/format.h>
 #include <fmt/printf.h>
-#include <sstream>
 
 namespace DB
 {
 namespace ErrorCodes
 {
-    extern const int ILLEGAL_COLUMN;
-    extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
-    extern const int ILLEGAL_TYPE_OF_ARGUMENT;
+extern const int ILLEGAL_COLUMN;
+extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
+extern const int ILLEGAL_TYPE_OF_ARGUMENT;
 }
 
 namespace
@@ -54,7 +54,7 @@ private:
         {
             std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM
             oss << "format:" << format << ", rows:" << rows << ", is_literal:" << is_literal << ", input:" << input.dumpStructure()
-                      << std::endl;
+                << std::endl;
             return oss.str();
         }
 
@@ -93,15 +93,15 @@ private:
             WhichDataType which(arg.type);
 
 #define EXECUTE_BY_TYPE(IS_TYPE, GET_TYPE) \
-            else if (which.IS_TYPE()) \
-            { \
-                for (size_t i = 0; i < size; ++i) \
-                { \
-                    auto a = arg.column->GET_TYPE(i); \
-                    s = fmt::sprintf(format, a); \
-                    res_str.insertData(s.data(), s.size()); \
-                } \
-            }
+    else if (which.IS_TYPE()) \
+    { \
+        for (size_t i = 0; i < size; ++i) \
+        { \
+            auto a = arg.column->GET_TYPE(i); \
+            s = fmt::sprintf(format, a); \
+            res_str.insertData(s.data(), s.size()); \
+        } \
+    }
 
             if (false)
                 ;
@@ -205,7 +205,8 @@ public:
     }
 
 private:
-    std::vector<Instruction> buildInstructions(const String & format , const ColumnsWithTypeAndName & arguments, size_t input_rows_count) const
+    std::vector<Instruction>
+    buildInstructions(const String & format, const ColumnsWithTypeAndName & arguments, size_t input_rows_count) const
     {
         std::vector<Instruction> instructions;
         instructions.reserve(arguments.size());
@@ -248,7 +249,7 @@ private:
         {
             const char * tmp = curr;
             bool is_first = curr == begin; /// If current instruction is the first one
-            bool is_literal = false;       /// If current instruction is literal string without any argument
+            bool is_literal = false; /// If current instruction is literal string without any argument
             if (is_first)
             {
                 if (*curr != '%')

From 11a8de50a6283277c585fa2bad74aad1712fb1f2 Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Wed, 10 Jul 2024 08:56:35 +0000
Subject: [PATCH 0180/1488] Revert "Disable broken cases from
 02911_join_on_nullsafe_optimization"

This reverts commit 513ce9fa2f3bb0d2cc1774a07272a249b40f475f.
---
 ...2911_join_on_nullsafe_optimization.reference | 17 +++++++++++++----
 .../02911_join_on_nullsafe_optimization.sql     |  5 ++---
 2 files changed, 15 insertions(+), 7 deletions(-)

diff --git a/tests/queries/0_stateless/02911_join_on_nullsafe_optimization.reference b/tests/queries/0_stateless/02911_join_on_nullsafe_optimization.reference
index 4eb7e74446d..f0463509b80 100644
--- a/tests/queries/0_stateless/02911_join_on_nullsafe_optimization.reference
+++ b/tests/queries/0_stateless/02911_join_on_nullsafe_optimization.reference
@@ -36,10 +36,19 @@ SELECT * FROM t1 JOIN t2 ON (t1.x == t2.x AND ((t2.x IS NOT NULL) AND (t1.x IS N
 3	3	3	33
 \N	\N	\N	\N
 -- aliases defined in the join condition are valid
--- FIXME(@vdimir) broken query formatting for the following queries:
--- SELECT *, e, e2 FROM t1 FULL JOIN t2 ON ( ( ((t1.x == t2.x) AS e) AND ((t2.x IS NOT NULL) AND (t1.x IS NOT NULL)) ) OR ( (t2.x IS NULL) AND (t1.x IS NULL) ) AS e2 ) ORDER BY t1.x NULLS LAST, t2.x NULLS LAST;
--- SELECT *, e, e2 FROM t1 FULL JOIN t2 ON ( ( ((t1.x == t2.x) AS e) AND ((t2.x IS NOT NULL) AND (t1.x IS NOT NULL)) ) AS e2 ) ORDER BY t1.x NULLS LAST, t2.x NULLS LAST;
-
+SELECT *, e, e2 FROM t1 FULL JOIN t2 ON ( ( ((t1.x == t2.x) AS e) AND ((t2.x IS NOT NULL) AND (t1.x IS NOT NULL)) ) OR ( (t2.x IS NULL) AND (t1.x IS NULL) ) AS e2 ) ORDER BY t1.x NULLS LAST, t2.x NULLS LAST;
+1	42	\N	\N	\N	0
+2	2	2	2	1	1
+3	3	3	33	1	1
+\N	\N	4	42	\N	0
+\N	\N	\N	\N	\N	1
+SELECT *, e, e2 FROM t1 FULL JOIN t2 ON ( ( ((t1.x == t2.x) AS e) AND ((t2.x IS NOT NULL) AND (t1.x IS NOT NULL)) ) AS e2 ) ORDER BY t1.x NULLS LAST, t2.x NULLS LAST;
+1	42	\N	\N	\N	0
+2	2	2	2	1	1
+3	3	3	33	1	1
+\N	\N	4	42	\N	0
+\N	\N	\N	\N	\N	0
+\N	\N	\N	\N	\N	0
 -- check for non-nullable columns for which `is null` is replaced with constant
 SELECT * FROM t1n as t1 JOIN t2n as t2 ON (t1.x == t2.x AND ((t2.x IS NOT NULL) AND (t1.x IS NOT NULL)) ) OR ( (t2.x IS NULL) AND (t1.x IS NULL) ) ORDER BY t1.x NULLS LAST;
 2	2	2	2
diff --git a/tests/queries/0_stateless/02911_join_on_nullsafe_optimization.sql b/tests/queries/0_stateless/02911_join_on_nullsafe_optimization.sql
index f7813e2a1b4..67918f4302f 100644
--- a/tests/queries/0_stateless/02911_join_on_nullsafe_optimization.sql
+++ b/tests/queries/0_stateless/02911_join_on_nullsafe_optimization.sql
@@ -36,9 +36,8 @@ SELECT x = y OR (x IS NULL AND y IS NULL) FROM t1 ORDER BY x NULLS LAST;
 SELECT * FROM t1 JOIN t2 ON (t1.x == t2.x AND ((t2.x IS NOT NULL) AND (t1.x IS NOT NULL)) ) OR ( (t2.x IS NULL) AND (t1.x IS NULL) ) ORDER BY t1.x NULLS LAST;
 
 -- aliases defined in the join condition are valid
--- FIXME(@vdimir) broken query formatting for the following queries:
--- SELECT *, e, e2 FROM t1 FULL JOIN t2 ON ( ( ((t1.x == t2.x) AS e) AND ((t2.x IS NOT NULL) AND (t1.x IS NOT NULL)) ) OR ( (t2.x IS NULL) AND (t1.x IS NULL) ) AS e2 ) ORDER BY t1.x NULLS LAST, t2.x NULLS LAST;
--- SELECT *, e, e2 FROM t1 FULL JOIN t2 ON ( ( ((t1.x == t2.x) AS e) AND ((t2.x IS NOT NULL) AND (t1.x IS NOT NULL)) ) AS e2 ) ORDER BY t1.x NULLS LAST, t2.x NULLS LAST;
+SELECT *, e, e2 FROM t1 FULL JOIN t2 ON ( ( ((t1.x == t2.x) AS e) AND ((t2.x IS NOT NULL) AND (t1.x IS NOT NULL)) ) OR ( (t2.x IS NULL) AND (t1.x IS NULL) ) AS e2 ) ORDER BY t1.x NULLS LAST, t2.x NULLS LAST;
+SELECT *, e, e2 FROM t1 FULL JOIN t2 ON ( ( ((t1.x == t2.x) AS e) AND ((t2.x IS NOT NULL) AND (t1.x IS NOT NULL)) ) AS e2 ) ORDER BY t1.x NULLS LAST, t2.x NULLS LAST;
 
 -- check for non-nullable columns for which `is null` is replaced with constant
 SELECT * FROM t1n as t1 JOIN t2n as t2 ON (t1.x == t2.x AND ((t2.x IS NOT NULL) AND (t1.x IS NOT NULL)) ) OR ( (t2.x IS NULL) AND (t1.x IS NULL) ) ORDER BY t1.x NULLS LAST;

From 6e762d404456debca4ee2d5ccce94deb32c3fbad Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Wed, 10 Jul 2024 08:57:33 +0000
Subject: [PATCH 0181/1488] Fix aliased JOIN ON expression formatting

---
 src/Parsers/ASTTablesInSelectQuery.cpp | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/Parsers/ASTTablesInSelectQuery.cpp b/src/Parsers/ASTTablesInSelectQuery.cpp
index d22a4eca0fc..dbb2a008bae 100644
--- a/src/Parsers/ASTTablesInSelectQuery.cpp
+++ b/src/Parsers/ASTTablesInSelectQuery.cpp
@@ -235,7 +235,12 @@ void ASTTableJoin::formatImplAfterTable(const FormatSettings & settings, FormatS
     else if (on_expression)
     {
         settings.ostr << (settings.hilite ? hilite_keyword : "") << " ON " << (settings.hilite ? hilite_none : "");
+        bool on_has_alias = !on_expression->tryGetAlias().empty();
+        if (on_has_alias)
+            settings.ostr << "(";
         on_expression->formatImpl(settings, state, frame);
+        if (on_has_alias)
+            settings.ostr << ")";
     }
 }
 

From 678c472236f8479455cdfcd697ed2a822f68a6b4 Mon Sep 17 00:00:00 2001
From: Aleksei Filatov <alexfvk@yandex-team.ru>
Date: Wed, 10 Jul 2024 14:42:13 +0300
Subject: [PATCH 0182/1488] Close log files in watchdog

---
 src/Daemon/BaseDaemon.cpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/Daemon/BaseDaemon.cpp b/src/Daemon/BaseDaemon.cpp
index 48f76769a09..f82f02c9d9d 100644
--- a/src/Daemon/BaseDaemon.cpp
+++ b/src/Daemon/BaseDaemon.cpp
@@ -1305,6 +1305,10 @@ void BaseDaemon::setupWatchdog()
         int status = 0;
         do
         {
+            // Close log files to prevent keeping descriptors of unlinked rotated files.
+            // On next log write files will be reopened.
+            closeLogs(logger());
+
             if (-1 != waitpid(pid, &status, WUNTRACED | WCONTINUED) || errno == ECHILD)
             {
                 if (WIFSTOPPED(status))

From 0c843ae1463fd94026fb0367845e3c43bdfa2ab4 Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Wed, 10 Jul 2024 13:01:15 +0000
Subject: [PATCH 0183/1488] disable rocksdb

---
 src/Coordination/CoordinationSettings.h       | 2 +-
 src/Coordination/KeeperContext.cpp            | 2 +-
 src/Coordination/KeeperServer.cpp             | 2 +-
 src/Coordination/tests/gtest_coordination.cpp | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/Coordination/CoordinationSettings.h b/src/Coordination/CoordinationSettings.h
index 5116a814871..35a23fc9e78 100644
--- a/src/Coordination/CoordinationSettings.h
+++ b/src/Coordination/CoordinationSettings.h
@@ -55,7 +55,7 @@ struct Settings;
     M(UInt64, min_request_size_for_cache, 50 * 1024, "Minimal size of the request to cache the deserialization result. Caching can have negative effect on latency for smaller requests, set to 0 to disable", 0) \
     M(UInt64, raft_limits_reconnect_limit, 50, "If connection to a peer is silent longer than this limit * (multiplied by heartbeat interval), we re-establish the connection.", 0) \
     M(Bool, async_replication, false, "Enable async replication. All write and read guarantees are preserved while better performance is achieved. Settings is disabled by default to not break backwards compatibility.", 0) \
-    M(Bool, use_rocksdb, true, "Use rocksdb as backend storage", 0) \
+    M(Bool, experimental_use_rocksdb, false, "Use rocksdb as backend storage", 0) \
     M(UInt64, latest_logs_cache_size_threshold, 1 * 1024 * 1024 * 1024, "Maximum total size of in-memory cache of latest log entries.", 0) \
     M(UInt64, commit_logs_cache_size_threshold, 500 * 1024 * 1024, "Maximum total size of in-memory cache of log entries needed next for commit.", 0) \
     M(UInt64, disk_move_retries_wait_ms, 1000, "How long to wait between retries after a failure which happened while a file was being moved between disks.", 0) \
diff --git a/src/Coordination/KeeperContext.cpp b/src/Coordination/KeeperContext.cpp
index b65bb02ca5d..60d042403e5 100644
--- a/src/Coordination/KeeperContext.cpp
+++ b/src/Coordination/KeeperContext.cpp
@@ -160,7 +160,7 @@ void KeeperContext::initialize(const Poco::Util::AbstractConfiguration & config,
     initializeDisks(config);
 
     #if USE_ROCKSDB
-    if (config.getBool("keeper_server.coordination_settings.use_rocksdb", true))
+    if (config.getBool("keeper_server.coordination_settings.experimental_use_rocksdb", false))
     {
         rocksdb_options = std::make_shared<rocksdb::Options>(getRocksDBOptionsFromConfig(config));
         digest_enabled = false; /// TODO: support digest
diff --git a/src/Coordination/KeeperServer.cpp b/src/Coordination/KeeperServer.cpp
index 7352873107c..14b51d716de 100644
--- a/src/Coordination/KeeperServer.cpp
+++ b/src/Coordination/KeeperServer.cpp
@@ -134,7 +134,7 @@ KeeperServer::KeeperServer(
 
 #if USE_ROCKSDB
     const auto & coordination_settings = keeper_context->getCoordinationSettings();
-    if (coordination_settings->use_rocksdb)
+    if (coordination_settings->experimental_use_rocksdb)
     {
         state_machine = nuraft::cs_new<KeeperStateMachine<KeeperRocksStorage>>(
             responses_queue_,
diff --git a/src/Coordination/tests/gtest_coordination.cpp b/src/Coordination/tests/gtest_coordination.cpp
index 2f505873c65..360f848ffd1 100644
--- a/src/Coordination/tests/gtest_coordination.cpp
+++ b/src/Coordination/tests/gtest_coordination.cpp
@@ -85,7 +85,7 @@ public:
         Poco::Logger::root().setLevel("trace");
 
         auto settings = std::make_shared<DB::CoordinationSettings>();
-        settings->use_rocksdb = true;
+        settings->experimental_use_rocksdb = true;
         keeper_context = std::make_shared<DB::KeeperContext>(true, settings);
         keeper_context->setLocalLogsPreprocessed();
         keeper_context->setRocksDBOptions();

From 3ea555524dd08af3181a5d1896cbe518c0d10736 Mon Sep 17 00:00:00 2001
From: AntiTopQuark <AntiTopQuark1350@outlook.com>
Date: Thu, 11 Jul 2024 00:20:40 +0800
Subject: [PATCH 0184/1488] resolve comments

---
 src/Storages/MergeTree/MergeTreeIndexSet.cpp | 41 ++++++--------------
 src/Storages/MergeTree/MergeTreeIndexSet.h   |  2 -
 2 files changed, 11 insertions(+), 32 deletions(-)

diff --git a/src/Storages/MergeTree/MergeTreeIndexSet.cpp b/src/Storages/MergeTree/MergeTreeIndexSet.cpp
index 36844648ac7..284d47ef9e7 100644
--- a/src/Storages/MergeTree/MergeTreeIndexSet.cpp
+++ b/src/Storages/MergeTree/MergeTreeIndexSet.cpp
@@ -87,15 +87,6 @@ void MergeTreeIndexGranuleSet::serializeBinary(WriteBuffer & ostr) const
         serialization->serializeBinaryBulkWithMultipleStreams(column, 0, size(), settings, state);
         serialization->serializeBinaryBulkStateSuffix(settings, state);
     }
-
-    for (size_t i = 0; i < num_columns; ++i)
-    {
-        const DataTypePtr & type = block.getByPosition(i).type;
-        auto serialization = type->getDefaultSerialization();
-
-        serialization->serializeBinary(set_hyperrectangle[i].left, ostr, {});
-        serialization->serializeBinary(set_hyperrectangle[i].right, ostr, {});
-    }
 }
 
 void MergeTreeIndexGranuleSet::deserializeBinary(ReadBuffer & istr, MergeTreeIndexVersion version)
@@ -117,6 +108,10 @@ void MergeTreeIndexGranuleSet::deserializeBinary(ReadBuffer & istr, MergeTreeInd
     settings.getter = [&](ISerialization::SubstreamPath) -> ReadBuffer * { return &istr; };
     settings.position_independent_encoding = false;
 
+    set_hyperrectangle.clear();
+    Field min_val;
+    Field max_val;
+
     for (size_t i = 0; i < num_columns; ++i)
     {
         auto & elem = block.getByPosition(i);
@@ -127,24 +122,12 @@ void MergeTreeIndexGranuleSet::deserializeBinary(ReadBuffer & istr, MergeTreeInd
 
         serialization->deserializeBinaryBulkStatePrefix(settings, state, nullptr);
         serialization->deserializeBinaryBulkWithMultipleStreams(elem.column, rows_to_read, settings, state, nullptr);
-    }
 
-    set_hyperrectangle.clear();
-    Field min_val;
-    Field max_val;
-    for (size_t i = 0; i < num_columns; ++i)
-    {
-        const DataTypePtr & type = block.getByPosition(i).type;
-        auto serialization = type->getDefaultSerialization();
+        if (const auto * column_nullable = typeid_cast<const ColumnNullable *>(elem.column.get()))
+            column_nullable->getExtremesNullLast(min_val, max_val);
+        else
+            elem.column->getExtremes(min_val, max_val);
 
-        serialization->deserializeBinary(min_val, istr, {});
-        serialization->deserializeBinary(max_val, istr, {});
-
-        // NULL_LAST
-        if (min_val.isNull())
-            min_val = POSITIVE_INFINITY;
-        if (max_val.isNull())
-            max_val = POSITIVE_INFINITY;
         set_hyperrectangle.emplace_back(min_val, true, max_val, true);
     }
 }
@@ -295,18 +278,16 @@ KeyCondition buildCondition(const IndexDescription & index, const ActionsDAGPtr
 }
 
 MergeTreeIndexConditionSet::MergeTreeIndexConditionSet(
-    const String & index_name_,
-    const Block & index_sample_block,
     size_t max_rows_,
     const ActionsDAGPtr & filter_dag,
     ContextPtr context,
     const IndexDescription & index_description)
-    : index_name(index_name_)
+    : index_name(index_description.name)
     , max_rows(max_rows_)
     , index_data_types(index_description.data_types)
     , condition(buildCondition(index_description, filter_dag, context))
 {
-    for (const auto & name : index_sample_block.getNames())
+    for (const auto & name : index_description.sample_block.getNames())
         if (!key_columns.contains(name))
             key_columns.insert(name);
 
@@ -605,7 +586,7 @@ MergeTreeIndexAggregatorPtr MergeTreeIndexSet::createIndexAggregator(const Merge
 MergeTreeIndexConditionPtr MergeTreeIndexSet::createIndexCondition(
     const ActionsDAGPtr & filter_actions_dag, ContextPtr context) const
 {
-    return std::make_shared<MergeTreeIndexConditionSet>(index.name, index.sample_block, max_rows, filter_actions_dag, context, index);
+    return std::make_shared<MergeTreeIndexConditionSet>(max_rows, filter_actions_dag, context, index);
 }
 
 MergeTreeIndexPtr setIndexCreator(const IndexDescription & index)
diff --git a/src/Storages/MergeTree/MergeTreeIndexSet.h b/src/Storages/MergeTree/MergeTreeIndexSet.h
index 85f6c73149b..168262360fc 100644
--- a/src/Storages/MergeTree/MergeTreeIndexSet.h
+++ b/src/Storages/MergeTree/MergeTreeIndexSet.h
@@ -83,8 +83,6 @@ class MergeTreeIndexConditionSet final : public IMergeTreeIndexCondition
 {
 public:
     MergeTreeIndexConditionSet(
-        const String & index_name_,
-        const Block & index_sample_block,
         size_t max_rows_,
         const ActionsDAGPtr & filter_dag,
         ContextPtr context,

From 614e899e8d286b5b25f34f0cd94163905dceffeb Mon Sep 17 00:00:00 2001
From: Sema Checherinda <Sema.Checherinda@clickhouse.com>
Date: Wed, 10 Jul 2024 18:52:09 +0200
Subject: [PATCH 0185/1488] onCancel is concurrent call

---
 .../Executors/CompletedPipelineExecutor.cpp   |  7 +++
 .../Formats/Impl/ArrowBlockInputFormat.h      |  2 +-
 .../Formats/Impl/DWARFBlockInputFormat.h      |  2 +-
 .../Formats/Impl/NativeORCBlockInputFormat.h  |  2 +-
 .../Formats/Impl/ORCBlockInputFormat.h        |  2 +-
 .../Impl/ParallelFormattingOutputFormat.h     |  2 +-
 .../Impl/ParallelParsingInputFormat.cpp       |  2 +-
 .../Formats/Impl/ParallelParsingInputFormat.h |  2 +-
 .../Formats/Impl/ParquetBlockInputFormat.h    |  2 +-
 .../Formats/Impl/ParquetBlockOutputFormat.cpp |  2 +-
 .../Formats/Impl/ParquetBlockOutputFormat.h   |  2 +-
 .../Formats/Impl/ParquetMetadataInputFormat.h |  2 +-
 src/Processors/Formats/LazyOutputFormat.h     |  2 +-
 src/Processors/IProcessor.cpp                 |  2 +-
 src/Processors/IProcessor.h                   |  2 +-
 src/Processors/Sources/RemoteSource.cpp       |  2 +-
 src/Processors/Sources/RemoteSource.h         |  2 +-
 .../Transforms/AggregatingTransform.cpp       |  2 +-
 src/Server/TCPHandler.cpp                     |  3 ++
 src/Storages/Distributed/DistributedSink.cpp  |  2 +-
 src/Storages/Distributed/DistributedSink.h    |  2 +-
 src/Storages/LiveView/LiveViewEventsSource.h  |  2 +-
 src/Storages/LiveView/LiveViewSource.h        |  2 +-
 .../MergeTree/MergeTreeDataWriter.cpp         |  4 ++
 src/Storages/MergeTree/MergeTreeSink.cpp      | 32 +++++++------
 src/Storages/MergeTree/MergeTreeSink.h        |  1 -
 src/Storages/MergeTree/MergeTreeSource.cpp    |  2 +-
 src/Storages/MergeTree/MergeTreeSource.h      |  2 +-
 .../MergeTree/ReplicatedMergeTreeSink.cpp     | 48 ++++++++++++-------
 .../MergeTree/ReplicatedMergeTreeSink.h       |  2 -
 src/Storages/MessageQueueSink.h               |  6 ++-
 .../StorageObjectStorageSink.cpp              | 28 ++++-------
 .../ObjectStorage/StorageObjectStorageSink.h  |  8 +---
 src/Storages/StorageFile.cpp                  | 37 ++++----------
 src/Storages/StorageURL.cpp                   | 26 ++++------
 src/Storages/StorageURL.h                     |  6 +--
 36 files changed, 123 insertions(+), 131 deletions(-)

diff --git a/src/Processors/Executors/CompletedPipelineExecutor.cpp b/src/Processors/Executors/CompletedPipelineExecutor.cpp
index 598a51bf0c7..909e742ffbf 100644
--- a/src/Processors/Executors/CompletedPipelineExecutor.cpp
+++ b/src/Processors/Executors/CompletedPipelineExecutor.cpp
@@ -3,6 +3,7 @@
 #include <QueryPipeline/QueryPipeline.h>
 #include <QueryPipeline/ReadProgressCallback.h>
 #include <Poco/Event.h>
+#include "Common/Logger.h"
 #include <Common/setThreadName.h>
 #include <Common/ThreadPool.h>
 #include <Common/scope_guard_safe.h>
@@ -97,7 +98,10 @@ void CompletedPipelineExecutor::execute()
                 break;
 
             if (is_cancelled_callback())
+            {
+                LOG_INFO(getLogger("CompletedPipelineExecutor"), "execute CancelCallback FULLY_CANCELLED");
                 data->executor->cancel();
+            }
         }
 
         if (data->has_exception)
@@ -116,7 +120,10 @@ CompletedPipelineExecutor::~CompletedPipelineExecutor()
     try
     {
         if (data && data->executor)
+        {
+            LOG_INFO(getLogger("CompletedPipelineExecutor"), "~CompletedPipelineExecutor");
             data->executor->cancel();
+        }
     }
     catch (...)
     {
diff --git a/src/Processors/Formats/Impl/ArrowBlockInputFormat.h b/src/Processors/Formats/Impl/ArrowBlockInputFormat.h
index cdbc5e57e4e..4fe01d0be12 100644
--- a/src/Processors/Formats/Impl/ArrowBlockInputFormat.h
+++ b/src/Processors/Formats/Impl/ArrowBlockInputFormat.h
@@ -32,7 +32,7 @@ public:
 private:
     Chunk read() override;
 
-    void onCancel() override
+    void onCancelX() override
     {
         is_stopped = 1;
     }
diff --git a/src/Processors/Formats/Impl/DWARFBlockInputFormat.h b/src/Processors/Formats/Impl/DWARFBlockInputFormat.h
index d8f5fc3d896..6cab5d34994 100644
--- a/src/Processors/Formats/Impl/DWARFBlockInputFormat.h
+++ b/src/Processors/Formats/Impl/DWARFBlockInputFormat.h
@@ -32,7 +32,7 @@ public:
 protected:
     Chunk read() override;
 
-    void onCancel() override
+    void onCancelX() override
     {
         is_stopped = 1;
     }
diff --git a/src/Processors/Formats/Impl/NativeORCBlockInputFormat.h b/src/Processors/Formats/Impl/NativeORCBlockInputFormat.h
index a3ef9ed4b8f..de9925e3737 100644
--- a/src/Processors/Formats/Impl/NativeORCBlockInputFormat.h
+++ b/src/Processors/Formats/Impl/NativeORCBlockInputFormat.h
@@ -64,7 +64,7 @@ public:
 protected:
     Chunk read() override;
 
-    void onCancel() override { is_stopped = 1; }
+    void onCancelX() override { is_stopped = 1; }
 
 private:
     void prepareFileReader();
diff --git a/src/Processors/Formats/Impl/ORCBlockInputFormat.h b/src/Processors/Formats/Impl/ORCBlockInputFormat.h
index 34630345849..167436ad4b9 100644
--- a/src/Processors/Formats/Impl/ORCBlockInputFormat.h
+++ b/src/Processors/Formats/Impl/ORCBlockInputFormat.h
@@ -34,7 +34,7 @@ public:
 protected:
     Chunk read() override;
 
-    void onCancel() override
+    void onCancelX() override
     {
         is_stopped = 1;
     }
diff --git a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h
index 341141dd633..40774fcfbfa 100644
--- a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h
+++ b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h
@@ -122,7 +122,7 @@ public:
         started_prefix = true;
     }
 
-    void onCancel() override
+    void onCancelX() override
     {
         finishAndWait();
     }
diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp
index 447adb1ed48..d38a299cb6e 100644
--- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp
+++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp
@@ -176,7 +176,7 @@ Chunk ParallelParsingInputFormat::read()
         if (background_exception)
         {
             lock.unlock();
-            onCancel();
+            onCancelX();
             std::rethrow_exception(background_exception);
         }
 
diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.h b/src/Processors/Formats/Impl/ParallelParsingInputFormat.h
index 963ccd88def..eed40dc43e5 100644
--- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.h
+++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.h
@@ -137,7 +137,7 @@ private:
 
     Chunk read() final;
 
-    void onCancel() final
+    void onCancelX() final
     {
         /*
          * The format parsers themselves are not being cancelled here, so we'll
diff --git a/src/Processors/Formats/Impl/ParquetBlockInputFormat.h b/src/Processors/Formats/Impl/ParquetBlockInputFormat.h
index 24735ee4371..0123329f026 100644
--- a/src/Processors/Formats/Impl/ParquetBlockInputFormat.h
+++ b/src/Processors/Formats/Impl/ParquetBlockInputFormat.h
@@ -68,7 +68,7 @@ public:
 private:
     Chunk read() override;
 
-    void onCancel() override
+    void onCancelX() override
     {
         is_stopped = 1;
     }
diff --git a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp
index a5d334f4f1d..d08c91d286b 100644
--- a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp
+++ b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp
@@ -268,7 +268,7 @@ void ParquetBlockOutputFormat::resetFormatterImpl()
     staging_bytes = 0;
 }
 
-void ParquetBlockOutputFormat::onCancel()
+void ParquetBlockOutputFormat::onCancelX()
 {
     is_stopped = true;
 }
diff --git a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.h b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.h
index 422bae5c315..0704ba0ed90 100644
--- a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.h
+++ b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.h
@@ -112,7 +112,7 @@ private:
     void consume(Chunk) override;
     void finalizeImpl() override;
     void resetFormatterImpl() override;
-    void onCancel() override;
+    void onCancelX() override;
 
     void writeRowGroup(std::vector<Chunk> chunks);
     void writeUsingArrow(std::vector<Chunk> chunks);
diff --git a/src/Processors/Formats/Impl/ParquetMetadataInputFormat.h b/src/Processors/Formats/Impl/ParquetMetadataInputFormat.h
index ff63d78fa44..35180d202d8 100644
--- a/src/Processors/Formats/Impl/ParquetMetadataInputFormat.h
+++ b/src/Processors/Formats/Impl/ParquetMetadataInputFormat.h
@@ -65,7 +65,7 @@ public:
 private:
     Chunk read() override;
 
-    void onCancel() override
+    void onCancelX() override
     {
         is_stopped = 1;
     }
diff --git a/src/Processors/Formats/LazyOutputFormat.h b/src/Processors/Formats/LazyOutputFormat.h
index 9cf609ed2d7..a245df8234d 100644
--- a/src/Processors/Formats/LazyOutputFormat.h
+++ b/src/Processors/Formats/LazyOutputFormat.h
@@ -29,7 +29,7 @@ public:
 
     void setRowsBeforeLimit(size_t rows_before_limit) override;
 
-    void onCancel() override
+    void onCancelX() override
     {
         queue.clearAndFinish();
     }
diff --git a/src/Processors/IProcessor.cpp b/src/Processors/IProcessor.cpp
index f403aca2280..4d95bb5f3e0 100644
--- a/src/Processors/IProcessor.cpp
+++ b/src/Processors/IProcessor.cpp
@@ -16,7 +16,7 @@ void IProcessor::cancel()
     if (already_cancelled)
         return;
 
-    onCancel();
+    onCancelX();
 }
 
 String IProcessor::debug() const
diff --git a/src/Processors/IProcessor.h b/src/Processors/IProcessor.h
index 6f779e7a8d4..9ef7d83eefa 100644
--- a/src/Processors/IProcessor.h
+++ b/src/Processors/IProcessor.h
@@ -363,7 +363,7 @@ public:
     virtual void setRowsBeforeLimitCounter(RowsBeforeLimitCounterPtr /* counter */) {}
 
 protected:
-    virtual void onCancel() {}
+    virtual void onCancelX() {}
 
     std::atomic<bool> is_cancelled{false};
 
diff --git a/src/Processors/Sources/RemoteSource.cpp b/src/Processors/Sources/RemoteSource.cpp
index 3d7dd3f76b8..44cf26e0b01 100644
--- a/src/Processors/Sources/RemoteSource.cpp
+++ b/src/Processors/Sources/RemoteSource.cpp
@@ -182,7 +182,7 @@ std::optional<Chunk> RemoteSource::tryGenerate()
     return chunk;
 }
 
-void RemoteSource::onCancel()
+void RemoteSource::onCancelX()
 {
     query_executor->cancel();
 }
diff --git a/src/Processors/Sources/RemoteSource.h b/src/Processors/Sources/RemoteSource.h
index 052567bc261..880eb234bfb 100644
--- a/src/Processors/Sources/RemoteSource.h
+++ b/src/Processors/Sources/RemoteSource.h
@@ -36,7 +36,7 @@ public:
 
 protected:
     std::optional<Chunk> tryGenerate() override;
-    void onCancel() override;
+    void onCancelX() override;
 
 private:
     bool was_query_sent = false;
diff --git a/src/Processors/Transforms/AggregatingTransform.cpp b/src/Processors/Transforms/AggregatingTransform.cpp
index cdbe194cfac..e42c1fd3a8d 100644
--- a/src/Processors/Transforms/AggregatingTransform.cpp
+++ b/src/Processors/Transforms/AggregatingTransform.cpp
@@ -375,7 +375,7 @@ public:
         return prepareTwoLevel();
     }
 
-    void onCancel() override
+    void onCancelX() override
     {
         shared_data->is_cancelled.store(true, std::memory_order_seq_cst);
     }
diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp
index ac1423f87c1..9fffad26a72 100644
--- a/src/Server/TCPHandler.cpp
+++ b/src/Server/TCPHandler.cpp
@@ -556,7 +556,10 @@ void TCPHandler::runImpl()
                             std::scoped_lock lock(out_mutex, task_callback_mutex);
 
                             if (getQueryCancellationStatus() == CancellationStatus::FULLY_CANCELLED)
+                            {
+                                LOG_INFO(log, "CancelCallback FULLY_CANCELLED");
                                 return true;
+                            }
 
                             sendProgress();
                             sendSelectProfileEvents();
diff --git a/src/Storages/Distributed/DistributedSink.cpp b/src/Storages/Distributed/DistributedSink.cpp
index e556bda2561..6283594e0d2 100644
--- a/src/Storages/Distributed/DistributedSink.cpp
+++ b/src/Storages/Distributed/DistributedSink.cpp
@@ -596,7 +596,7 @@ void DistributedSink::onFinish()
     }
 }
 
-void DistributedSink::onCancel()
+void DistributedSink::onCancelX()
 {
     std::lock_guard lock(execution_mutex);
     if (pool && !pool->finished())
diff --git a/src/Storages/Distributed/DistributedSink.h b/src/Storages/Distributed/DistributedSink.h
index a4c95633595..1dac3eeba6d 100644
--- a/src/Storages/Distributed/DistributedSink.h
+++ b/src/Storages/Distributed/DistributedSink.h
@@ -53,7 +53,7 @@ public:
     void onFinish() override;
 
 private:
-    void onCancel() override;
+    void onCancelX() override;
 
     IColumn::Selector createSelector(const Block & source_block) const;
 
diff --git a/src/Storages/LiveView/LiveViewEventsSource.h b/src/Storages/LiveView/LiveViewEventsSource.h
index de10a98e1a2..d1ed222c185 100644
--- a/src/Storages/LiveView/LiveViewEventsSource.h
+++ b/src/Storages/LiveView/LiveViewEventsSource.h
@@ -54,7 +54,7 @@ public:
 
     String getName() const override { return "LiveViewEventsSource"; }
 
-    void onCancel() override
+    void onCancelX() override
     {
         if (storage->shutdown_called)
             return;
diff --git a/src/Storages/LiveView/LiveViewSource.h b/src/Storages/LiveView/LiveViewSource.h
index f8b428fc04d..83589067cf5 100644
--- a/src/Storages/LiveView/LiveViewSource.h
+++ b/src/Storages/LiveView/LiveViewSource.h
@@ -36,7 +36,7 @@ public:
 
     String getName() const override { return "LiveViewSource"; }
 
-    void onCancel() override
+    void onCancelX() override
     {
         if (storage->shutdown_called)
             return;
diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp
index 5c8aa32949d..0aaa7909a0f 100644
--- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp
+++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp
@@ -13,6 +13,8 @@
 #include <Storages/MergeTree/MergeTreeDataWriter.h>
 #include <Storages/MergeTree/MergedBlockOutputStream.h>
 #include <Storages/MergeTree/RowOrderOptimizer.h>
+#include "Common/Logger.h"
+#include "Common/logger_useful.h"
 #include <Common/ElapsedTimeProfileEventIncrement.h>
 #include <Common/Exception.h>
 #include <Common/HashTable/HashMap.h>
@@ -182,6 +184,8 @@ void updateTTL(
 
 void MergeTreeDataWriter::TemporaryPart::cancel()
 {
+    LOG_INFO(getLogger("MergeTreeDataWriter"), "TemporaryPart cancel");
+
     try
     {
         /// An exception context is needed to proper delete write buffers without finalization
diff --git a/src/Storages/MergeTree/MergeTreeSink.cpp b/src/Storages/MergeTree/MergeTreeSink.cpp
index 2e8f6db6868..d2e34665962 100644
--- a/src/Storages/MergeTree/MergeTreeSink.cpp
+++ b/src/Storages/MergeTree/MergeTreeSink.cpp
@@ -26,7 +26,23 @@ struct MergeTreeSink::DelayedChunk
 };
 
 
-MergeTreeSink::~MergeTreeSink() = default;
+MergeTreeSink::~MergeTreeSink()
+{
+    size_t addr = delayed_chunk ? size_t(delayed_chunk.get()) : 0;
+    LOG_INFO(storage.log, "~ReplicatedMergeTreeSinkImpl, delayed_chunk {}, called from {}", addr, StackTrace().toString());
+
+    if (!delayed_chunk)
+        return;
+
+    for (auto & partition : delayed_chunk->partitions)
+    {
+        partition.temp_part.cancel();
+    }
+
+    delayed_chunk.reset();
+
+    LOG_INFO(storage.log, "~ReplicatedMergeTreeSinkImpl end");
+}
 
 MergeTreeSink::MergeTreeSink(
     StorageMergeTree & storage_,
@@ -51,22 +67,10 @@ void MergeTreeSink::onStart()
 
 void MergeTreeSink::onFinish()
 {
+    chassert(!isCancelled());
     finishDelayedChunk();
 }
 
-void MergeTreeSink::onCancel()
-{
-    if (!delayed_chunk)
-            return;
-
-    for (auto & partition : delayed_chunk->partitions)
-    {
-        partition.temp_part.cancel();
-    }
-
-    delayed_chunk.reset();
-}
-
 void MergeTreeSink::consume(Chunk chunk)
 {
     if (num_blocks_processed > 0)
diff --git a/src/Storages/MergeTree/MergeTreeSink.h b/src/Storages/MergeTree/MergeTreeSink.h
index cf6715a3415..07ab3850df2 100644
--- a/src/Storages/MergeTree/MergeTreeSink.h
+++ b/src/Storages/MergeTree/MergeTreeSink.h
@@ -28,7 +28,6 @@ public:
     void consume(Chunk chunk) override;
     void onStart() override;
     void onFinish() override;
-    void onCancel() override;
 
 private:
     StorageMergeTree & storage;
diff --git a/src/Storages/MergeTree/MergeTreeSource.cpp b/src/Storages/MergeTree/MergeTreeSource.cpp
index e323b9f9ee7..4070ccf4433 100644
--- a/src/Storages/MergeTree/MergeTreeSource.cpp
+++ b/src/Storages/MergeTree/MergeTreeSource.cpp
@@ -149,7 +149,7 @@ std::string MergeTreeSource::getName() const
     return processor->getName();
 }
 
-void MergeTreeSource::onCancel()
+void MergeTreeSource::onCancelX()
 {
     processor->cancel();
 }
diff --git a/src/Storages/MergeTree/MergeTreeSource.h b/src/Storages/MergeTree/MergeTreeSource.h
index fc39b4f9b09..c7092aa26b1 100644
--- a/src/Storages/MergeTree/MergeTreeSource.h
+++ b/src/Storages/MergeTree/MergeTreeSource.h
@@ -26,7 +26,7 @@ public:
 protected:
     std::optional<Chunk> tryGenerate() override;
 
-    void onCancel() override;
+    void onCancelX() override;
 
 private:
     MergeTreeSelectProcessorPtr processor;
diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp
index 93f82a5a789..6c7ed9bdae0 100644
--- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp
+++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp
@@ -4,11 +4,13 @@
 #include <Storages/MergeTree/InsertBlockInfo.h>
 #include <Interpreters/PartLog.h>
 #include "Common/Exception.h"
+#include "Common/StackTrace.h"
 #include <Common/FailPoint.h>
 #include <Common/ProfileEventsScope.h>
 #include <Common/SipHash.h>
 #include <Common/ZooKeeper/KeeperException.h>
 #include <Common/ThreadFuzzer.h>
+#include "base/defines.h"
 #include <Storages/MergeTree/MergeAlgorithm.h>
 #include <Storages/MergeTree/MergeTreeDataWriter.h>
 #include <Storages/MergeTree/AsyncBlockIDsCache.h>
@@ -151,7 +153,23 @@ ReplicatedMergeTreeSinkImpl<async_insert>::ReplicatedMergeTreeSinkImpl(
 }
 
 template<bool async_insert>
-ReplicatedMergeTreeSinkImpl<async_insert>::~ReplicatedMergeTreeSinkImpl() = default;
+ReplicatedMergeTreeSinkImpl<async_insert>::~ReplicatedMergeTreeSinkImpl()
+{
+    size_t addr = delayed_chunk ? size_t(delayed_chunk.get()) : 0;
+    LOG_INFO(log, "~ReplicatedMergeTreeSinkImpl, delayed_chunk {}, called from {}", addr, StackTrace().toString());
+
+    if (!delayed_chunk)
+        return;
+
+    for (auto & partition : delayed_chunk->partitions)
+    {
+        partition.temp_part.cancel();
+    }
+
+    delayed_chunk.reset();
+
+    LOG_INFO(log, "~ReplicatedMergeTreeSinkImpl end");
+}
 
 template<bool async_insert>
 size_t ReplicatedMergeTreeSinkImpl<async_insert>::checkQuorumPrecondition(const ZooKeeperWithFaultInjectionPtr & zookeeper)
@@ -255,6 +273,8 @@ size_t ReplicatedMergeTreeSinkImpl<async_insert>::checkQuorumPrecondition(const
 template<bool async_insert>
 void ReplicatedMergeTreeSinkImpl<async_insert>::consume(Chunk chunk)
 {
+    LOG_INFO(log, "consume");
+
     if (num_blocks_processed > 0)
         storage.delayInsertOrThrowIfNeeded(&storage.partial_shutdown_event, context, false);
 
@@ -428,6 +448,9 @@ void ReplicatedMergeTreeSinkImpl<async_insert>::consume(Chunk chunk)
 template<>
 void ReplicatedMergeTreeSinkImpl<false>::finishDelayedChunk(const ZooKeeperWithFaultInjectionPtr & zookeeper)
 {
+    size_t addr = delayed_chunk ? size_t(delayed_chunk.get()) : 0;
+    LOG_INFO(log, "finishDelayedChunk {}", addr);
+
     if (!delayed_chunk)
         return;
 
@@ -457,16 +480,22 @@ void ReplicatedMergeTreeSinkImpl<false>::finishDelayedChunk(const ZooKeeperWithF
         {
             auto counters_snapshot = std::make_shared<ProfileEvents::Counters::Snapshot>(partition.part_counters.getPartiallyAtomicSnapshot());
             PartLog::addNewPart(storage.getContext(), PartLog::PartLogEntry(part, partition.elapsed_ns, counters_snapshot), ExecutionStatus::fromCurrentException("", true));
+
+            size_t addr1 = delayed_chunk ? size_t(delayed_chunk.get()) : 0;
+            LOG_INFO(log, "finishDelayedChunk exception, delayed_chunk {}", addr1);
             throw;
         }
     }
 
     delayed_chunk.reset();
+
+    LOG_INFO(log, "finishDelayedChunk end, delayed_chunk {}", bool(delayed_chunk));
 }
 
 template<>
 void ReplicatedMergeTreeSinkImpl<true>::finishDelayedChunk(const ZooKeeperWithFaultInjectionPtr & zookeeper)
 {
+
     if (!delayed_chunk)
         return;
 
@@ -1151,25 +1180,12 @@ void ReplicatedMergeTreeSinkImpl<async_insert>::onStart()
 template<bool async_insert>
 void ReplicatedMergeTreeSinkImpl<async_insert>::onFinish()
 {
+    chassert(!isCancelled());
+
     auto zookeeper = storage.getZooKeeper();
     finishDelayedChunk(std::make_shared<ZooKeeperWithFaultInjection>(zookeeper));
 }
 
-
-template<bool async_insert>
-void ReplicatedMergeTreeSinkImpl<async_insert>::onCancel()
-{
-    if (!delayed_chunk)
-        return;
-
-    for (auto & partition : delayed_chunk->partitions)
-    {
-        partition.temp_part.cancel();
-    }
-
-    delayed_chunk.reset();
-}
-
 template<bool async_insert>
 void ReplicatedMergeTreeSinkImpl<async_insert>::waitForQuorum(
     const ZooKeeperWithFaultInjectionPtr & zookeeper,
diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.h b/src/Storages/MergeTree/ReplicatedMergeTreeSink.h
index b1796a35ed2..39623c20584 100644
--- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.h
+++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.h
@@ -54,8 +54,6 @@ public:
     void consume(Chunk chunk) override;
     void onFinish() override;
 
-    void onCancel() override;
-
     String getName() const override { return "ReplicatedMergeTreeSink"; }
 
     /// For ATTACHing existing data on filesystem.
diff --git a/src/Storages/MessageQueueSink.h b/src/Storages/MessageQueueSink.h
index b3c1e61734f..38754e9475e 100644
--- a/src/Storages/MessageQueueSink.h
+++ b/src/Storages/MessageQueueSink.h
@@ -33,13 +33,17 @@ public:
         const String & storage_name_,
         const ContextPtr & context_);
 
+    ~MessageQueueSink() override
+    {
+        onFinish();
+    }
+
     String getName() const override { return storage_name + "Sink"; }
 
     void consume(Chunk chunk) override;
 
     void onStart() override;
     void onFinish() override;
-    void onCancel() override { onFinish(); }
     void onException(std::exception_ptr /* exception */) override { onFinish(); }
 
 protected:
diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSink.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSink.cpp
index f2f6eac333c..3bd0e88ecdb 100644
--- a/src/Storages/ObjectStorage/StorageObjectStorageSink.cpp
+++ b/src/Storages/ObjectStorage/StorageObjectStorageSink.cpp
@@ -2,6 +2,7 @@
 #include <Formats/FormatFactory.h>
 #include <Disks/ObjectStorages/IObjectStorage.h>
 #include <Common/isValidUTF8.h>
+#include "base/defines.h"
 #include <Storages/ObjectStorage/Utils.h>
 
 namespace DB
@@ -41,31 +42,16 @@ StorageObjectStorageSink::StorageObjectStorageSink(
 
 void StorageObjectStorageSink::consume(Chunk chunk)
 {
-    std::lock_guard lock(cancel_mutex);
-    if (cancelled)
+    if (isCancelled())
         return;
     writer->write(getHeader().cloneWithColumns(chunk.detachColumns()));
 }
 
-void StorageObjectStorageSink::onCancel()
-{
-    std::lock_guard lock(cancel_mutex);
-    cancelBuffers();
-    releaseBuffers();
-    cancelled = true;
-}
-
-void StorageObjectStorageSink::onException(std::exception_ptr)
-{
-    std::lock_guard lock(cancel_mutex);
-    cancelBuffers();
-    releaseBuffers();
-}
-
 void StorageObjectStorageSink::onFinish()
 {
-    std::lock_guard lock(cancel_mutex);
+    chassert(!isCancelled());
     finalizeBuffers();
+    releaseBuffers();
 }
 
 void StorageObjectStorageSink::finalizeBuffers()
@@ -119,6 +105,12 @@ PartitionedStorageObjectStorageSink::PartitionedStorageObjectStorageSink(
 {
 }
 
+StorageObjectStorageSink::~StorageObjectStorageSink()
+{
+    if (isCancelled())
+        cancelBuffers();
+}
+
 SinkPtr PartitionedStorageObjectStorageSink::createSinkForPartition(const String & partition_id)
 {
     auto partition_bucket = replaceWildcards(configuration->getNamespace(), partition_id);
diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSink.h b/src/Storages/ObjectStorage/StorageObjectStorageSink.h
index e0081193686..578290a92a5 100644
--- a/src/Storages/ObjectStorage/StorageObjectStorageSink.h
+++ b/src/Storages/ObjectStorage/StorageObjectStorageSink.h
@@ -18,22 +18,18 @@ public:
         ContextPtr context,
         const std::string & blob_path = "");
 
+    ~StorageObjectStorageSink() override;
+
     String getName() const override { return "StorageObjectStorageSink"; }
 
     void consume(Chunk chunk) override;
 
-    void onCancel() override;
-
-    void onException(std::exception_ptr exception) override;
-
     void onFinish() override;
 
 private:
     const Block sample_block;
     std::unique_ptr<WriteBuffer> write_buf;
     OutputFormatPtr writer;
-    bool cancelled = false;
-    std::mutex cancel_mutex;
 
     void finalizeBuffers();
     void releaseBuffers();
diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp
index 7f39ff615f0..855667b1cc6 100644
--- a/src/Storages/StorageFile.cpp
+++ b/src/Storages/StorageFile.cpp
@@ -52,6 +52,7 @@
 #include <Common/logger_useful.h>
 #include <Common/ProfileEvents.h>
 #include <Common/re2.h>
+#include "base/defines.h"
 
 #include <QueryPipeline/Pipe.h>
 #include <QueryPipeline/QueryPipelineBuilder.h>
@@ -1747,6 +1748,12 @@ public:
         initialize();
     }
 
+    ~StorageFileSink() override
+    {
+        if (isCancelled())
+            cancelBuffers();
+    }
+
     void initialize()
     {
         std::unique_ptr<WriteBufferFromFileDescriptor> naked_buffer;
@@ -1780,37 +1787,14 @@ public:
 
     void consume(Chunk chunk) override
     {
-        std::lock_guard cancel_lock(cancel_mutex);
-        if (cancelled)
+        if (isCancelled())
             return;
         writer->write(getHeader().cloneWithColumns(chunk.detachColumns()));
     }
 
-    void onCancel() override
-    {
-        std::lock_guard cancel_lock(cancel_mutex);
-        cancelBuffers();
-        releaseBuffers();
-        cancelled = true;
-    }
-
-    void onException(std::exception_ptr exception) override
-    {
-        std::lock_guard cancel_lock(cancel_mutex);
-        try
-        {
-            std::rethrow_exception(exception);
-        }
-        catch (...)
-        {
-            /// An exception context is needed to proper delete write buffers without finalization
-            releaseBuffers();
-        }
-    }
-
     void onFinish() override
     {
-        std::lock_guard cancel_lock(cancel_mutex);
+        chassert(!isCancelled());
         finalizeBuffers();
     }
 
@@ -1865,9 +1849,6 @@ private:
 
     int flags;
     std::unique_lock<std::shared_timed_mutex> lock;
-
-    std::mutex cancel_mutex;
-    bool cancelled = false;
 };
 
 class PartitionedStorageFileSink : public PartitionedSink
diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp
index 895da028fc2..6f600393263 100644
--- a/src/Storages/StorageURL.cpp
+++ b/src/Storages/StorageURL.cpp
@@ -567,31 +567,15 @@ StorageURLSink::StorageURLSink(
 
 void StorageURLSink::consume(Chunk chunk)
 {
-    std::lock_guard lock(cancel_mutex);
-    if (cancelled)
+    if (isCancelled())
         return;
     writer->write(getHeader().cloneWithColumns(chunk.detachColumns()));
 }
 
-void StorageURLSink::onCancel()
-{
-    std::lock_guard lock(cancel_mutex);
-    cancelBuffers();
-    releaseBuffers();
-    cancelled = true;
-}
-
-void StorageURLSink::onException(std::exception_ptr)
-{
-    std::lock_guard lock(cancel_mutex);
-    cancelBuffers();
-    releaseBuffers();
-}
-
 void StorageURLSink::onFinish()
 {
-    std::lock_guard lock(cancel_mutex);
     finalizeBuffers();
+    releaseBuffers();
 }
 
 void StorageURLSink::finalizeBuffers()
@@ -1396,6 +1380,11 @@ StorageURLWithFailover::StorageURLWithFailover(
     }
 }
 
+StorageURLSink::~StorageURLSink()
+{
+    if (isCancelled())
+        cancelBuffers();
+}
 
 FormatSettings StorageURL::getFormatSettingsFromArgs(const StorageFactory::Arguments & args)
 {
@@ -1586,4 +1575,5 @@ void registerStorageURL(StorageFactory & factory)
             .source_access_type = AccessType::URL,
         });
 }
+
 }
diff --git a/src/Storages/StorageURL.h b/src/Storages/StorageURL.h
index fa7cc6eeeef..12a49d3dff5 100644
--- a/src/Storages/StorageURL.h
+++ b/src/Storages/StorageURL.h
@@ -250,10 +250,10 @@ public:
         const HTTPHeaderEntries & headers = {},
         const String & method = Poco::Net::HTTPRequest::HTTP_POST);
 
+    ~StorageURLSink() override;
+
     std::string getName() const override { return "StorageURLSink"; }
     void consume(Chunk chunk) override;
-    void onCancel() override;
-    void onException(std::exception_ptr exception) override;
     void onFinish() override;
 
 private:
@@ -263,8 +263,6 @@ private:
 
     std::unique_ptr<WriteBuffer> write_buf;
     OutputFormatPtr writer;
-    std::mutex cancel_mutex;
-    bool cancelled = false;
 };
 
 class StorageURL : public IStorageURLBase

From c41424d197fe5e583100944bf4a2e47216e664c9 Mon Sep 17 00:00:00 2001
From: JackyWoo <wuchienchao@qq.com>
Date: Thu, 11 Jul 2024 10:52:55 +0800
Subject: [PATCH 0186/1488] Throw exception when datasketches is not enabled

---
 .../Statistics/StatisticsCountMinSketch.cpp   | 37 ++++++---
 .../Statistics/StatisticsCountMinSketch.h     |  4 +-
 ...64_statistics_estimate_predicate.reference | 12 ---
 .../02864_statistics_estimate_predicate.sql   | 82 +++++++++----------
 4 files changed, 69 insertions(+), 66 deletions(-)

diff --git a/src/Storages/Statistics/StatisticsCountMinSketch.cpp b/src/Storages/Statistics/StatisticsCountMinSketch.cpp
index 3e91b4052c4..c25372c69ee 100644
--- a/src/Storages/Statistics/StatisticsCountMinSketch.cpp
+++ b/src/Storages/Statistics/StatisticsCountMinSketch.cpp
@@ -13,7 +13,6 @@ namespace DB
 
 namespace ErrorCodes
 {
-extern const int ILLEGAL_STATISTICS;
 extern const int ILLEGAL_TYPE_OF_ARGUMENT;
 }
 
@@ -23,9 +22,7 @@ static constexpr auto num_hashes = 7uz;
 static constexpr auto num_buckets = 2718uz;
 
 StatisticsCountMinSketch::StatisticsCountMinSketch(const SingleStatisticsDescription & stat_, DataTypePtr data_type_)
-    : IStatistics(stat_)
-    , sketch(num_hashes, num_buckets)
-    , data_type(data_type_)
+    : IStatistics(stat_), sketch(num_hashes, num_buckets), data_type(data_type_)
 {
 }
 
@@ -40,7 +37,7 @@ Float64 StatisticsCountMinSketch::estimateEqual(const Field & val) const
         if (val_converted.isNull())
             return 0;
 
-        /// We will get the proper data type of val_converted, for example, UInt8 for 1, UInt16 for 257.
+        /// We will get the proper data type of val_converted, for example, Int8 for 1, Int16 for 257.
         auto data_type_converted = applyVisitor(FieldToDataType<LeastSupertypeOnError::Null>(), val_converted);
         DataTypes data_types = {data_type, data_type_converted};
         auto super_type = tryGetLeastSupertype(data_types);
@@ -58,7 +55,9 @@ Float64 StatisticsCountMinSketch::estimateEqual(const Field & val) const
     }
 
     throw Exception(
-        ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Statistics 'count_min' does not support estimate constant value of type {}", val.getTypeName());
+        ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
+        "Statistics 'count_min' does not support estimate constant value of type {}",
+        val.getTypeName());
 }
 
 void StatisticsCountMinSketch::update(const ColumnPtr & column)
@@ -92,20 +91,36 @@ void StatisticsCountMinSketch::deserialize(ReadBuffer & buf)
     sketch = Sketch::deserialize(bytes.data(), size);
 }
 
+}
+
+#endif
+
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+extern const int FEATURE_IS_NOT_ENABLED_AT_BUILD_TIME;
+extern const int ILLEGAL_STATISTICS;
+}
+
 void CountMinSketchValidator(const SingleStatisticsDescription &, DataTypePtr data_type)
 {
     data_type = removeNullable(data_type);
     data_type = removeLowCardinalityAndNullable(data_type);
-    /// Numeric, String family, IPv4, IPv6, Date family, Enum family are supported.
+    /// Data types of Numeric, String family, IPv4, IPv6, Date family, Enum family are supported.
     if (!data_type->isValueRepresentedByNumber() && !isStringOrFixedString(data_type))
         throw Exception(ErrorCodes::ILLEGAL_STATISTICS, "Statistics of type 'count_min' does not support type {}", data_type->getName());
 }
 
 StatisticsPtr CountMinSketchCreator(const SingleStatisticsDescription & stat, DataTypePtr data_type)
 {
+#if USE_DATASKETCHES
     return std::make_shared<StatisticsCountMinSketch>(stat, data_type);
-}
-
-}
-
+#else
+    throw Exception(ErrorCodes::FEATURE_IS_NOT_ENABLED_AT_BUILD_TIME, "Statistics of type 'count_min' is not supported in this build, to enable it turn on USE_DATASKETCHES when building.");
 #endif
+}
+
+}
diff --git a/src/Storages/Statistics/StatisticsCountMinSketch.h b/src/Storages/Statistics/StatisticsCountMinSketch.h
index 2cc93d6e75a..f4d49d37bca 100644
--- a/src/Storages/Statistics/StatisticsCountMinSketch.h
+++ b/src/Storages/Statistics/StatisticsCountMinSketch.h
@@ -30,9 +30,9 @@ private:
     DataTypePtr data_type;
 };
 
+#endif
+
 void CountMinSketchValidator(const SingleStatisticsDescription &, DataTypePtr data_type);
 StatisticsPtr CountMinSketchCreator(const SingleStatisticsDescription & stat, DataTypePtr);
 
 }
-
-#endif
diff --git a/tests/queries/0_stateless/02864_statistics_estimate_predicate.reference b/tests/queries/0_stateless/02864_statistics_estimate_predicate.reference
index 4e41c32750f..4027aeefe7b 100644
--- a/tests/queries/0_stateless/02864_statistics_estimate_predicate.reference
+++ b/tests/queries/0_stateless/02864_statistics_estimate_predicate.reference
@@ -11,15 +11,3 @@ Test statistics Uniq:
           Prewhere filter
           Prewhere filter column: and(equals(b, 0), equals(c, 0)) (removed)
 Test statistics count_min:
-        Prewhere info
-          Prewhere filter
-          Prewhere filter column: and(equals(a, \'0\'), equals(b, 0), equals(c, 0)) (removed)
-Test statistics multi-types:
-        Prewhere info
-          Prewhere filter
-          Prewhere filter column: and(equals(a, \'0\'), less(c, -90), greater(b, 900)) (removed)
-        Prewhere info
-          Prewhere filter
-          Prewhere filter column: and(equals(a, \'10000\'), equals(b, 0), less(c, 0)) (removed)
-Test LowCardinality and Nullable data type:
-tab2
diff --git a/tests/queries/0_stateless/02864_statistics_estimate_predicate.sql b/tests/queries/0_stateless/02864_statistics_estimate_predicate.sql
index 91b4f2d05cb..6647182ab05 100644
--- a/tests/queries/0_stateless/02864_statistics_estimate_predicate.sql
+++ b/tests/queries/0_stateless/02864_statistics_estimate_predicate.sql
@@ -55,44 +55,44 @@ ALTER TABLE tab ADD STATISTICS b TYPE count_min;
 ALTER TABLE tab ADD STATISTICS c TYPE count_min;
 ALTER TABLE tab MATERIALIZE STATISTICS a, b, c;
 
-SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String', '')
-FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE c = 0/*100*/ and b = 0/*10*/ and a = '0'/*1*/) xx
-WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
-
-ALTER TABLE tab DROP STATISTICS a, b, c;
-
-
-SELECT 'Test statistics multi-types:';
-
-ALTER TABLE tab ADD STATISTICS a TYPE count_min;
-ALTER TABLE tab ADD STATISTICS b TYPE count_min, uniq, tdigest;
-ALTER TABLE tab ADD STATISTICS c TYPE count_min, uniq, tdigest;
-ALTER TABLE tab MATERIALIZE STATISTICS a, b, c;
-
-SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String', '')
-FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE c < -90/*900*/ and b > 900/*990*/ and a = '0'/*1*/)
-WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
-
-SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String', '')
-FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE c < 0/*9900*/ and b = 0/*10*/ and a = '10000'/*0*/)
-WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
-
-ALTER TABLE tab DROP STATISTICS a, b, c;
-
-DROP TABLE IF EXISTS tab SYNC;
-
-
-SELECT 'Test LowCardinality and Nullable data type:';
-DROP TABLE IF EXISTS tab2 SYNC;
-SET allow_suspicious_low_cardinality_types=1;
-CREATE TABLE tab2
-(
-    a LowCardinality(Int64) STATISTICS(uniq, tdigest, count_min),
-    b Nullable(Int64) STATISTICS(uniq, tdigest, count_min),
-    c LowCardinality(Nullable(Int64)) STATISTICS(uniq, tdigest, count_min),
-    pk String,
-) Engine = MergeTree() ORDER BY pk;
-
-select name from system.tables where name = 'tab2' and database = currentDatabase();
-
-DROP TABLE IF EXISTS tab2 SYNC;
+-- SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String', '')
+-- FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE c = 0/*100*/ and b = 0/*10*/ and a = '0'/*1*/) xx
+-- WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
+--
+-- ALTER TABLE tab DROP STATISTICS a, b, c;
+--
+--
+-- SELECT 'Test statistics multi-types:';
+--
+-- ALTER TABLE tab ADD STATISTICS a TYPE count_min;
+-- ALTER TABLE tab ADD STATISTICS b TYPE count_min, uniq, tdigest;
+-- ALTER TABLE tab ADD STATISTICS c TYPE count_min, uniq, tdigest;
+-- ALTER TABLE tab MATERIALIZE STATISTICS a, b, c;
+--
+-- SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String', '')
+-- FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE c < -90/*900*/ and b > 900/*990*/ and a = '0'/*1*/)
+-- WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
+--
+-- SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String', '')
+-- FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE c < 0/*9900*/ and b = 0/*10*/ and a = '10000'/*0*/)
+-- WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
+--
+-- ALTER TABLE tab DROP STATISTICS a, b, c;
+--
+-- DROP TABLE IF EXISTS tab SYNC;
+--
+--
+-- SELECT 'Test LowCardinality and Nullable data type:';
+-- DROP TABLE IF EXISTS tab2 SYNC;
+-- SET allow_suspicious_low_cardinality_types=1;
+-- CREATE TABLE tab2
+-- (
+--     a LowCardinality(Int64) STATISTICS(uniq, tdigest, count_min),
+--     b Nullable(Int64) STATISTICS(uniq, tdigest, count_min),
+--     c LowCardinality(Nullable(Int64)) STATISTICS(uniq, tdigest, count_min),
+--     pk String,
+-- ) Engine = MergeTree() ORDER BY pk;
+--
+-- select name from system.tables where name = 'tab2' and database = currentDatabase();
+--
+-- DROP TABLE IF EXISTS tab2 SYNC;

From 96d063bcc39712c5a21a8e51244a9e216af8536a Mon Sep 17 00:00:00 2001
From: taiyang-li <654010905@qq.com>
Date: Thu, 11 Jul 2024 12:10:44 +0800
Subject: [PATCH 0187/1488] renmae ut files

---
 src/Functions/printf.cpp                      | 118 +++++++++++-------
 ...erence => 03203_function_printf.reference} |   0
 ...0_printf.sql => 03203_function_printf.sql} |   0
 3 files changed, 73 insertions(+), 45 deletions(-)
 rename tests/queries/0_stateless/{032010_printf.reference => 03203_function_printf.reference} (100%)
 rename tests/queries/0_stateless/{032010_printf.sql => 03203_function_printf.sql} (100%)

diff --git a/src/Functions/printf.cpp b/src/Functions/printf.cpp
index c7c6bd228a7..247c4a65daf 100644
--- a/src/Functions/printf.cpp
+++ b/src/Functions/printf.cpp
@@ -40,7 +40,7 @@ private:
         bool is_literal; /// format is literal string without any argument
         ColumnWithTypeAndName input; /// Only used when is_literal is false
 
-        ColumnWithTypeAndName execute()
+        ColumnWithTypeAndName execute() const
         {
             if (is_literal)
                 return executeLiteral(format);
@@ -50,7 +50,7 @@ private:
                 return executeNonconstant(input);
         }
 
-        String toString() const
+        [[maybe_unused]] String toString() const
         {
             std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM
             oss << "format:" << format << ", rows:" << rows << ", is_literal:" << is_literal << ", input:" << input.dumpStructure()
@@ -59,7 +59,7 @@ private:
         }
 
     private:
-        ColumnWithTypeAndName executeLiteral(std::string_view literal)
+        ColumnWithTypeAndName executeLiteral(std::string_view literal) const
         {
             ColumnWithTypeAndName res;
             auto str_col = ColumnString::create();
@@ -69,7 +69,7 @@ private:
             return res;
         }
 
-        ColumnWithTypeAndName executeConstant(const ColumnWithTypeAndName & arg)
+        ColumnWithTypeAndName executeConstant(const ColumnWithTypeAndName & arg) const
         {
             ColumnWithTypeAndName tmp_arg = arg;
             const auto & const_col = static_cast<const ColumnConst &>(*arg.column);
@@ -79,57 +79,85 @@ private:
             return ColumnWithTypeAndName{ColumnConst::create(tmp_res.column, arg.column->size()), tmp_res.type, tmp_res.name};
         }
 
-        ColumnWithTypeAndName executeNonconstant(const ColumnWithTypeAndName & arg)
+        template <typename T>
+        bool executeNumber(const IColumn & column, ColumnString::Chars & res_chars, ColumnString::Offsets & res_offsets) const
+        {
+            const ColumnVector<T> * concrete_column = checkAndGetColumn<ColumnVector<T>>(&column);
+            if (!concrete_column)
+                return false;
+
+            String s;
+            size_t curr_offset = 0;
+            const auto & data = concrete_column->getData();
+            for (size_t i = 0; i < data.size(); ++i)
+            {
+                T a = data[i];
+                s = fmt::sprintf(format, static_cast<NearestFieldType<T>>(a));
+                memcpy(&res_chars[curr_offset], s.data(), s.size());
+                res_chars[curr_offset + s.size()] = 0;
+
+                curr_offset += s.size() + 1;
+                res_offsets[i] = curr_offset;
+            }
+            return true;
+        }
+
+        template <typename COLUMN>
+        bool executeString(const IColumn & column, ColumnString::Chars & res_chars, ColumnString::Offsets & res_offsets) const
+        {
+            const COLUMN * concrete_column = checkAndGetColumn<COLUMN>(&column);
+            if (!concrete_column)
+                return false;
+
+            String s;
+            size_t curr_offset = 0;
+            for (size_t i = 0; i < concrete_column->size(); ++i)
+            {
+                auto a = concrete_column->getDataAt(i).toView();
+                s = fmt::sprintf(format, a);
+                memcpy(&res_chars[curr_offset], s.data(), s.size());
+                res_chars[curr_offset + s.size()] = 0;
+
+                curr_offset += s.size() + 1;
+                res_offsets[i] = curr_offset;
+            }
+            return true;
+        }
+
+        ColumnWithTypeAndName executeNonconstant(const ColumnWithTypeAndName & arg) const
         {
             size_t size = arg.column->size();
             auto res_col = ColumnString::create();
             auto & res_str = static_cast<ColumnString &>(*res_col);
             auto & res_offsets = res_str.getOffsets();
             auto & res_chars = res_str.getChars();
-            res_offsets.reserve_exact(size);
-            res_chars.reserve(format.size() * size * 2);
+            res_offsets.resize_exact(size);
+            res_chars.reserve(format.size() * size);
 
-            String s;
             WhichDataType which(arg.type);
-
-#define EXECUTE_BY_TYPE(IS_TYPE, GET_TYPE) \
-    else if (which.IS_TYPE()) \
-    { \
-        for (size_t i = 0; i < size; ++i) \
-        { \
-            auto a = arg.column->GET_TYPE(i); \
-            s = fmt::sprintf(format, a); \
-            res_str.insertData(s.data(), s.size()); \
-        } \
-    }
-
-            if (false)
-                ;
-            EXECUTE_BY_TYPE(isNativeInt, getInt)
-            EXECUTE_BY_TYPE(isNativeUInt, getUInt)
-            EXECUTE_BY_TYPE(isFloat32, getFloat32)
-            EXECUTE_BY_TYPE(isFloat64, getFloat64)
-            else if (which.isStringOrFixedString())
+            if (which.isNativeNumber()
+                && (executeNumber<UInt8>(*arg.column, res_chars, res_offsets) || executeNumber<UInt16>(*arg.column, res_chars, res_offsets)
+                    || executeNumber<UInt32>(*arg.column, res_chars, res_offsets)
+                    || executeNumber<UInt64>(*arg.column, res_chars, res_offsets)
+                    || executeNumber<Int8>(*arg.column, res_chars, res_offsets) || executeNumber<Int16>(*arg.column, res_chars, res_offsets)
+                    || executeNumber<Int32>(*arg.column, res_chars, res_offsets)
+                    || executeNumber<Int64>(*arg.column, res_chars, res_offsets)))
             {
-                for (size_t i = 0; i < size; ++i)
-                {
-                    auto a = arg.column->getDataAt(i).toView();
-                    s = fmt::sprintf(format, a);
-                    res_str.insertData(s.data(), s.size());
-                }
+                return {std::move(res_col), std::make_shared<DataTypeString>(), arg.name};
             }
-            else throw Exception(
-                ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
-                "The argument type of function {} is {}, but native numeric or string type is expected",
-                FunctionPrintf::name,
-                arg.type->getName());
-#undef EXECUTE_BY_TYPE
-
-            ColumnWithTypeAndName res;
-            res.name = arg.name;
-            res.type = std::make_shared<DataTypeString>();
-            res.column = std::move(res_col);
-            return res;
+            else if (
+                which.isStringOrFixedString()
+                && (executeString<ColumnString>(*arg.column, res_chars, res_offsets)
+                    || executeString<ColumnFixedString>(*arg.column, res_chars, res_offsets)))
+            {
+                return {std::move(res_col), std::make_shared<DataTypeString>(), arg.name};
+            }
+            else
+                throw Exception(
+                    ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
+                    "The argument type of function {} is {}, but native numeric or string type is expected",
+                    FunctionPrintf::name,
+                    arg.type->getName());
         }
     };
 
diff --git a/tests/queries/0_stateless/032010_printf.reference b/tests/queries/0_stateless/03203_function_printf.reference
similarity index 100%
rename from tests/queries/0_stateless/032010_printf.reference
rename to tests/queries/0_stateless/03203_function_printf.reference
diff --git a/tests/queries/0_stateless/032010_printf.sql b/tests/queries/0_stateless/03203_function_printf.sql
similarity index 100%
rename from tests/queries/0_stateless/032010_printf.sql
rename to tests/queries/0_stateless/03203_function_printf.sql

From 88851ddb569f9ae8c61420bde99d2ad5f3d76889 Mon Sep 17 00:00:00 2001
From: taiyang-li <654010905@qq.com>
Date: Thu, 11 Jul 2024 12:15:44 +0800
Subject: [PATCH 0188/1488]  improve uts

---
 .../0_stateless/03203_function_printf.reference    |  5 +++++
 .../queries/0_stateless/03203_function_printf.sql  | 14 ++++++++++++--
 2 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/tests/queries/0_stateless/03203_function_printf.reference b/tests/queries/0_stateless/03203_function_printf.reference
index 58501cbd0fc..338ecb0183d 100644
--- a/tests/queries/0_stateless/03203_function_printf.reference
+++ b/tests/queries/0_stateless/03203_function_printf.reference
@@ -14,3 +14,8 @@
 1
 1
 1
+1
+1
+1
+1
+1
diff --git a/tests/queries/0_stateless/03203_function_printf.sql b/tests/queries/0_stateless/03203_function_printf.sql
index 58fe081e499..c41cbf0b5e9 100644
--- a/tests/queries/0_stateless/03203_function_printf.sql
+++ b/tests/queries/0_stateless/03203_function_printf.sql
@@ -19,6 +19,16 @@ select printf('%%A: %A', 123.456) = '%A: 0X1.EDD2F1A9FBE77P+6';
 -- Testing character formats
 select printf('%%s: %s', 'abc') = '%s: abc';
 
-
 -- Testing the %% specifier
-select printf('%%%%: %%') = '%%: %';
\ No newline at end of file
+select printf('%%%%: %%') = '%%: %';
+
+-- Testing integer formats with precision
+select printf('%%.5d: %.5d', 123) = '%.5d: 00123';
+
+-- Testing floating point formats with precision
+select printf('%%.2f: %.2f', 123.456) = '%.2f: 123.46';
+select printf('%%.2e: %.2e', 123.456) = '%.2e: 1.23e+02';
+select printf('%%.2g: %.2g', 123.456) = '%.2g: 1.2e+02';
+
+-- Testing character formats with precision
+select printf('%%.2s: %.2s', 'abc') = '%.2s: ab';
\ No newline at end of file

From 465442ff7b9e7c7897c8dc754d1f6e4052303257 Mon Sep 17 00:00:00 2001
From: JackyWoo <wuchienchao@qq.com>
Date: Thu, 11 Jul 2024 11:19:44 +0800
Subject: [PATCH 0189/1488] Fix build error

---
 .../Statistics/StatisticsCountMinSketch.cpp   | 14 ++--
 .../Statistics/StatisticsCountMinSketch.h     |  5 +-
 ...64_statistics_estimate_predicate.reference | 12 +++
 .../02864_statistics_estimate_predicate.sql   | 82 +++++++++----------
 4 files changed, 66 insertions(+), 47 deletions(-)

diff --git a/src/Storages/Statistics/StatisticsCountMinSketch.cpp b/src/Storages/Statistics/StatisticsCountMinSketch.cpp
index c25372c69ee..ff985b06ee3 100644
--- a/src/Storages/Statistics/StatisticsCountMinSketch.cpp
+++ b/src/Storages/Statistics/StatisticsCountMinSketch.cpp
@@ -113,14 +113,18 @@ void CountMinSketchValidator(const SingleStatisticsDescription &, DataTypePtr da
     if (!data_type->isValueRepresentedByNumber() && !isStringOrFixedString(data_type))
         throw Exception(ErrorCodes::ILLEGAL_STATISTICS, "Statistics of type 'count_min' does not support type {}", data_type->getName());
 }
-
+#if USE_DATASKETCHES
 StatisticsPtr CountMinSketchCreator(const SingleStatisticsDescription & stat, DataTypePtr data_type)
 {
-#if USE_DATASKETCHES
     return std::make_shared<StatisticsCountMinSketch>(stat, data_type);
-#else
-    throw Exception(ErrorCodes::FEATURE_IS_NOT_ENABLED_AT_BUILD_TIME, "Statistics of type 'count_min' is not supported in this build, to enable it turn on USE_DATASKETCHES when building.");
-#endif
 }
+#else
+StatisticsPtr CountMinSketchCreator(const SingleStatisticsDescription &, DataTypePtr)
+{
+    throw Exception(
+        ErrorCodes::FEATURE_IS_NOT_ENABLED_AT_BUILD_TIME,
+        "Statistics of type 'count_min' is not supported in this build, to enable it turn on USE_DATASKETCHES when building.");
+}
+#endif
 
 }
diff --git a/src/Storages/Statistics/StatisticsCountMinSketch.h b/src/Storages/Statistics/StatisticsCountMinSketch.h
index f4d49d37bca..d6141f9f73a 100644
--- a/src/Storages/Statistics/StatisticsCountMinSketch.h
+++ b/src/Storages/Statistics/StatisticsCountMinSketch.h
@@ -30,9 +30,12 @@ private:
     DataTypePtr data_type;
 };
 
+}
+
 #endif
 
+namespace DB
+{
 void CountMinSketchValidator(const SingleStatisticsDescription &, DataTypePtr data_type);
 StatisticsPtr CountMinSketchCreator(const SingleStatisticsDescription & stat, DataTypePtr);
-
 }
diff --git a/tests/queries/0_stateless/02864_statistics_estimate_predicate.reference b/tests/queries/0_stateless/02864_statistics_estimate_predicate.reference
index 4027aeefe7b..4e41c32750f 100644
--- a/tests/queries/0_stateless/02864_statistics_estimate_predicate.reference
+++ b/tests/queries/0_stateless/02864_statistics_estimate_predicate.reference
@@ -11,3 +11,15 @@ Test statistics Uniq:
           Prewhere filter
           Prewhere filter column: and(equals(b, 0), equals(c, 0)) (removed)
 Test statistics count_min:
+        Prewhere info
+          Prewhere filter
+          Prewhere filter column: and(equals(a, \'0\'), equals(b, 0), equals(c, 0)) (removed)
+Test statistics multi-types:
+        Prewhere info
+          Prewhere filter
+          Prewhere filter column: and(equals(a, \'0\'), less(c, -90), greater(b, 900)) (removed)
+        Prewhere info
+          Prewhere filter
+          Prewhere filter column: and(equals(a, \'10000\'), equals(b, 0), less(c, 0)) (removed)
+Test LowCardinality and Nullable data type:
+tab2
diff --git a/tests/queries/0_stateless/02864_statistics_estimate_predicate.sql b/tests/queries/0_stateless/02864_statistics_estimate_predicate.sql
index 6647182ab05..91b4f2d05cb 100644
--- a/tests/queries/0_stateless/02864_statistics_estimate_predicate.sql
+++ b/tests/queries/0_stateless/02864_statistics_estimate_predicate.sql
@@ -55,44 +55,44 @@ ALTER TABLE tab ADD STATISTICS b TYPE count_min;
 ALTER TABLE tab ADD STATISTICS c TYPE count_min;
 ALTER TABLE tab MATERIALIZE STATISTICS a, b, c;
 
--- SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String', '')
--- FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE c = 0/*100*/ and b = 0/*10*/ and a = '0'/*1*/) xx
--- WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
---
--- ALTER TABLE tab DROP STATISTICS a, b, c;
---
---
--- SELECT 'Test statistics multi-types:';
---
--- ALTER TABLE tab ADD STATISTICS a TYPE count_min;
--- ALTER TABLE tab ADD STATISTICS b TYPE count_min, uniq, tdigest;
--- ALTER TABLE tab ADD STATISTICS c TYPE count_min, uniq, tdigest;
--- ALTER TABLE tab MATERIALIZE STATISTICS a, b, c;
---
--- SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String', '')
--- FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE c < -90/*900*/ and b > 900/*990*/ and a = '0'/*1*/)
--- WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
---
--- SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String', '')
--- FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE c < 0/*9900*/ and b = 0/*10*/ and a = '10000'/*0*/)
--- WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
---
--- ALTER TABLE tab DROP STATISTICS a, b, c;
---
--- DROP TABLE IF EXISTS tab SYNC;
---
---
--- SELECT 'Test LowCardinality and Nullable data type:';
--- DROP TABLE IF EXISTS tab2 SYNC;
--- SET allow_suspicious_low_cardinality_types=1;
--- CREATE TABLE tab2
--- (
---     a LowCardinality(Int64) STATISTICS(uniq, tdigest, count_min),
---     b Nullable(Int64) STATISTICS(uniq, tdigest, count_min),
---     c LowCardinality(Nullable(Int64)) STATISTICS(uniq, tdigest, count_min),
---     pk String,
--- ) Engine = MergeTree() ORDER BY pk;
---
--- select name from system.tables where name = 'tab2' and database = currentDatabase();
---
--- DROP TABLE IF EXISTS tab2 SYNC;
+SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String', '')
+FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE c = 0/*100*/ and b = 0/*10*/ and a = '0'/*1*/) xx
+WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
+
+ALTER TABLE tab DROP STATISTICS a, b, c;
+
+
+SELECT 'Test statistics multi-types:';
+
+ALTER TABLE tab ADD STATISTICS a TYPE count_min;
+ALTER TABLE tab ADD STATISTICS b TYPE count_min, uniq, tdigest;
+ALTER TABLE tab ADD STATISTICS c TYPE count_min, uniq, tdigest;
+ALTER TABLE tab MATERIALIZE STATISTICS a, b, c;
+
+SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String', '')
+FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE c < -90/*900*/ and b > 900/*990*/ and a = '0'/*1*/)
+WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
+
+SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String', '')
+FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE c < 0/*9900*/ and b = 0/*10*/ and a = '10000'/*0*/)
+WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
+
+ALTER TABLE tab DROP STATISTICS a, b, c;
+
+DROP TABLE IF EXISTS tab SYNC;
+
+
+SELECT 'Test LowCardinality and Nullable data type:';
+DROP TABLE IF EXISTS tab2 SYNC;
+SET allow_suspicious_low_cardinality_types=1;
+CREATE TABLE tab2
+(
+    a LowCardinality(Int64) STATISTICS(uniq, tdigest, count_min),
+    b Nullable(Int64) STATISTICS(uniq, tdigest, count_min),
+    c LowCardinality(Nullable(Int64)) STATISTICS(uniq, tdigest, count_min),
+    pk String,
+) Engine = MergeTree() ORDER BY pk;
+
+select name from system.tables where name = 'tab2' and database = currentDatabase();
+
+DROP TABLE IF EXISTS tab2 SYNC;

From d988399aa68ad8a6bc412bfc48d9fdefe63c1657 Mon Sep 17 00:00:00 2001
From: taiyang-li <654010905@qq.com>
Date: Thu, 11 Jul 2024 14:32:24 +0800
Subject: [PATCH 0190/1488] fix failed uts

---
 src/Functions/printf.cpp | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/Functions/printf.cpp b/src/Functions/printf.cpp
index 247c4a65daf..a890b886338 100644
--- a/src/Functions/printf.cpp
+++ b/src/Functions/printf.cpp
@@ -141,7 +141,9 @@ private:
                     || executeNumber<UInt64>(*arg.column, res_chars, res_offsets)
                     || executeNumber<Int8>(*arg.column, res_chars, res_offsets) || executeNumber<Int16>(*arg.column, res_chars, res_offsets)
                     || executeNumber<Int32>(*arg.column, res_chars, res_offsets)
-                    || executeNumber<Int64>(*arg.column, res_chars, res_offsets)))
+                    || executeNumber<Int64>(*arg.column, res_chars, res_offsets)
+                    || executeNumber<Float32>(*arg.column, res_chars, res_offsets)
+                    || executeNumber<Float64>(*arg.column, res_chars, res_offsets)))
             {
                 return {std::move(res_col), std::make_shared<DataTypeString>(), arg.name};
             }

From 21ca5f2d65c936a2c5b5fbc8f3f0c40d0ce60a6a Mon Sep 17 00:00:00 2001
From: taiyang-li <654010905@qq.com>
Date: Thu, 11 Jul 2024 15:32:05 +0800
Subject: [PATCH 0191/1488] fix failed ut

---
 .../02415_all_new_functions_must_be_documented.reference         | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference b/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference
index a152066a460..873b6bbb660 100644
--- a/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference
+++ b/tests/queries/0_stateless/02415_all_new_functions_must_be_documented.reference
@@ -562,6 +562,7 @@ positionCaseInsensitive
 positionCaseInsensitiveUTF8
 positionUTF8
 pow
+printf
 proportionsZTest
 protocol
 queryID

From 2d1f45bb985debfb20037cb5f7dba6fb4f6903d8 Mon Sep 17 00:00:00 2001
From: Igor Nikonov <igor@clickhouse.com>
Date: Thu, 11 Jul 2024 08:32:00 +0000
Subject: [PATCH 0192/1488] Remove redundant code

---
 src/Client/IConnections.h                   |  2 --
 src/Client/MultiplexedConnections.cpp       | 11 +++++------
 src/Interpreters/ClientInfo.cpp             |  4 ++--
 src/Interpreters/ClientInfo.h               |  2 +-
 src/Interpreters/Context.cpp                |  7 -------
 src/Interpreters/Context.h                  |  1 -
 src/Processors/QueryPlan/ReadFromRemote.cpp | 13 ++++++-------
 7 files changed, 14 insertions(+), 26 deletions(-)

diff --git a/src/Client/IConnections.h b/src/Client/IConnections.h
index ebc71511834..09211de53b0 100644
--- a/src/Client/IConnections.h
+++ b/src/Client/IConnections.h
@@ -54,8 +54,6 @@ public:
 
     struct ReplicaInfo
     {
-        bool collaborate_with_initiator{false};
-        size_t all_replicas_count{0};
         size_t number_of_current_replica{0};
     };
 
diff --git a/src/Client/MultiplexedConnections.cpp b/src/Client/MultiplexedConnections.cpp
index bcef286ecbc..866338bee8d 100644
--- a/src/Client/MultiplexedConnections.cpp
+++ b/src/Client/MultiplexedConnections.cpp
@@ -141,13 +141,12 @@ void MultiplexedConnections::sendQuery(
             modified_settings.group_by_two_level_threshold = 0;
             modified_settings.group_by_two_level_threshold_bytes = 0;
         }
+    }
 
-        if (replica_info)
-        {
-            client_info.collaborate_with_initiator = true;
-            client_info.count_participating_replicas = replica_info->all_replicas_count;
-            client_info.number_of_current_replica = replica_info->number_of_current_replica;
-        }
+    if (replica_info)
+    {
+        client_info.collaborate_with_initiator = true;
+        client_info.number_of_current_replica = replica_info->number_of_current_replica;
     }
 
     /// FIXME: Remove once we will make `allow_experimental_analyzer` obsolete setting.
diff --git a/src/Interpreters/ClientInfo.cpp b/src/Interpreters/ClientInfo.cpp
index ce1efb61cc0..39fdef23baa 100644
--- a/src/Interpreters/ClientInfo.cpp
+++ b/src/Interpreters/ClientInfo.cpp
@@ -95,7 +95,7 @@ void ClientInfo::write(WriteBuffer & out, UInt64 server_protocol_revision) const
     if (server_protocol_revision >= DBMS_MIN_REVISION_WITH_PARALLEL_REPLICAS)
     {
         writeVarUInt(static_cast<UInt64>(collaborate_with_initiator), out);
-        writeVarUInt(count_participating_replicas, out);
+        writeVarUInt(obsolete_count_participating_replicas, out);
         writeVarUInt(number_of_current_replica, out);
     }
 }
@@ -185,7 +185,7 @@ void ClientInfo::read(ReadBuffer & in, UInt64 client_protocol_revision)
         UInt64 value;
         readVarUInt(value, in);
         collaborate_with_initiator = static_cast<bool>(value);
-        readVarUInt(count_participating_replicas, in);
+        readVarUInt(obsolete_count_participating_replicas, in);
         readVarUInt(number_of_current_replica, in);
     }
 }
diff --git a/src/Interpreters/ClientInfo.h b/src/Interpreters/ClientInfo.h
index 3054667e264..ca32b4c5cfa 100644
--- a/src/Interpreters/ClientInfo.h
+++ b/src/Interpreters/ClientInfo.h
@@ -127,7 +127,7 @@ public:
 
     /// For parallel processing on replicas
     bool collaborate_with_initiator{false};
-    UInt64 count_participating_replicas{0};
+    UInt64 obsolete_count_participating_replicas{0};
     UInt64 number_of_current_replica{0};
 
     enum class BackgroundOperationType : uint8_t
diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp
index f2626696492..0d60f7dc5f4 100644
--- a/src/Interpreters/Context.cpp
+++ b/src/Interpreters/Context.cpp
@@ -4941,13 +4941,6 @@ void Context::setConnectionClientVersion(UInt64 client_version_major, UInt64 cli
     client_info.connection_tcp_protocol_version = client_tcp_protocol_version;
 }
 
-void Context::setReplicaInfo(bool collaborate_with_initiator, size_t all_replicas_count, size_t number_of_current_replica)
-{
-    client_info.collaborate_with_initiator = collaborate_with_initiator;
-    client_info.count_participating_replicas = all_replicas_count;
-    client_info.number_of_current_replica = number_of_current_replica;
-}
-
 void Context::increaseDistributedDepth()
 {
     ++client_info.distributed_depth;
diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h
index 8c5492bcbc8..5cd1153ab44 100644
--- a/src/Interpreters/Context.h
+++ b/src/Interpreters/Context.h
@@ -696,7 +696,6 @@ public:
     void setInitialQueryStartTime(std::chrono::time_point<std::chrono::system_clock> initial_query_start_time);
     void setQuotaClientKey(const String & quota_key);
     void setConnectionClientVersion(UInt64 client_version_major, UInt64 client_version_minor, UInt64 client_version_patch, unsigned client_tcp_protocol_version);
-    void setReplicaInfo(bool collaborate_with_initiator, size_t all_replicas_count, size_t number_of_current_replica);
     void increaseDistributedDepth();
     const OpenTelemetry::TracingContext & getClientTraceContext() const { return client_info.client_trace_context; }
     OpenTelemetry::TracingContext & getClientTraceContext() { return client_info.client_trace_context; }
diff --git a/src/Processors/QueryPlan/ReadFromRemote.cpp b/src/Processors/QueryPlan/ReadFromRemote.cpp
index 84c2515e8ca..8ebd95f27e5 100644
--- a/src/Processors/QueryPlan/ReadFromRemote.cpp
+++ b/src/Processors/QueryPlan/ReadFromRemote.cpp
@@ -411,8 +411,8 @@ void ReadFromParallelRemoteReplicasStep::initializePipeline(QueryPipelineBuilder
     auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(current_settings);
 
     const auto & shard = cluster->getShardsInfo().at(0);
-    size_t all_replicas_count = current_settings.max_parallel_replicas;
-    if (all_replicas_count > shard.getAllNodeCount())
+    size_t max_replicas_to_use = current_settings.max_parallel_replicas;
+    if (max_replicas_to_use > shard.getAllNodeCount())
     {
         LOG_INFO(
             getLogger("ReadFromParallelRemoteReplicasStep"),
@@ -420,14 +420,14 @@ void ReadFromParallelRemoteReplicasStep::initializePipeline(QueryPipelineBuilder
             "Will use the latter number to execute the query.",
             current_settings.max_parallel_replicas,
             shard.getAllNodeCount());
-        all_replicas_count = shard.getAllNodeCount();
+        max_replicas_to_use = shard.getAllNodeCount();
     }
 
     std::vector<ConnectionPoolWithFailover::Base::ShuffledPool> shuffled_pool;
-    if (all_replicas_count < shard.getAllNodeCount())
+    if (max_replicas_to_use < shard.getAllNodeCount())
     {
         shuffled_pool = shard.pool->getShuffledPools(current_settings);
-        shuffled_pool.resize(all_replicas_count);
+        shuffled_pool.resize(max_replicas_to_use);
     }
     else
     {
@@ -437,11 +437,10 @@ void ReadFromParallelRemoteReplicasStep::initializePipeline(QueryPipelineBuilder
         shuffled_pool = shard.pool->getShuffledPools(current_settings, priority_func);
     }
 
-    for (size_t i=0; i < all_replicas_count; ++i)
+    for (size_t i=0; i < max_replicas_to_use; ++i)
     {
         IConnections::ReplicaInfo replica_info
         {
-            .all_replicas_count = all_replicas_count,
             /// we should use this number specifically because efficiency of data distribution by consistent hash depends on it.
             .number_of_current_replica = i,
         };

From e8d831ce905d9ea6acbdb4e20142191c98431638 Mon Sep 17 00:00:00 2001
From: joelynch <joelynch112@gmail.com>
Date: Mon, 17 Jun 2024 19:06:23 +0200
Subject: [PATCH 0193/1488] Fix more dictGet ACL bypasses.

---
 src/Storages/StorageDictionary.cpp               |  2 ++
 .../02916_dictionary_access.reference            |  2 ++
 .../0_stateless/02916_dictionary_access.sh       | 16 +++++++++++++++-
 3 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/src/Storages/StorageDictionary.cpp b/src/Storages/StorageDictionary.cpp
index 447fd87cdc9..a64acaebff6 100644
--- a/src/Storages/StorageDictionary.cpp
+++ b/src/Storages/StorageDictionary.cpp
@@ -1,3 +1,4 @@
+#include <Access/Common/AccessFlags.h>
 #include <Storages/StorageDictionary.h>
 #include <Storages/StorageFactory.h>
 #include <DataTypes/DataTypesNumber.h>
@@ -160,6 +161,7 @@ Pipe StorageDictionary::read(
     const size_t max_block_size,
     const size_t threads)
 {
+    local_context->checkAccess(AccessType::dictGet, getStorageID());
     auto registered_dictionary_name = location == Location::SameDatabaseAndNameAsDictionary ? getStorageID().getInternalDictionaryName() : dictionary_name;
     auto dictionary = getContext()->getExternalDictionariesLoader().getDictionary(registered_dictionary_name, local_context);
     return dictionary->read(column_names, max_block_size, threads);
diff --git a/tests/queries/0_stateless/02916_dictionary_access.reference b/tests/queries/0_stateless/02916_dictionary_access.reference
index d7d02e37689..ab44e74249e 100644
--- a/tests/queries/0_stateless/02916_dictionary_access.reference
+++ b/tests/queries/0_stateless/02916_dictionary_access.reference
@@ -1,3 +1,5 @@
 0
 ACCESS_DENIED
 ACCESS_DENIED
+ACCESS_DENIED
+ACCESS_DENIED
diff --git a/tests/queries/0_stateless/02916_dictionary_access.sh b/tests/queries/0_stateless/02916_dictionary_access.sh
index 08ee517ab3b..5f80de120f4 100755
--- a/tests/queries/0_stateless/02916_dictionary_access.sh
+++ b/tests/queries/0_stateless/02916_dictionary_access.sh
@@ -6,6 +6,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
 username="user_${CLICKHOUSE_TEST_UNIQUE_NAME}"
 dictname="dict_${CLICKHOUSE_TEST_UNIQUE_NAME}"
+dicttablename="dict_table_${CLICKHOUSE_TEST_UNIQUE_NAME}"
 
 ${CLICKHOUSE_CLIENT} -nm --query "
     CREATE DICTIONARY IF NOT EXISTS ${dictname}
@@ -18,9 +19,13 @@ ${CLICKHOUSE_CLIENT} -nm --query "
     LAYOUT(FLAT())
     LIFETIME(MIN 0 MAX 1000);
     CREATE USER IF NOT EXISTS ${username} NOT IDENTIFIED;
-    GRANT CREATE TEMPORARY TABLE ON *.* to ${username};
+    GRANT SELECT, CREATE TEMPORARY TABLE ON *.* to ${username};
     SELECT * FROM dictionary(${dictname});
+    SELECT * FROM ${dictname};
     SELECT dictGet(${dictname}, 'value', 1);
+    CREATE TABLE ${dicttablename} (id UInt64, value UInt64)
+    ENGINE = Dictionary(${CLICKHOUSE_DATABASE}.${dictname});
+    SELECT * FROM ${dicttablename};
 "
 
 $CLICKHOUSE_CLIENT -nm --user="${username}" --query "
@@ -31,7 +36,16 @@ $CLICKHOUSE_CLIENT -nm --user="${username}" --query "
     SELECT dictGet(${dictname}, 'value', 1);
 " 2>&1 | grep -o ACCESS_DENIED | uniq
 
+$CLICKHOUSE_CLIENT -nm --user="${username}" --query "
+    SELECT * FROM ${dictname};
+" 2>&1 | grep -o ACCESS_DENIED | uniq
+
+$CLICKHOUSE_CLIENT -nm --user="${username}" --query "
+    SELECT * FROM ${dicttablename};
+" 2>&1 | grep -o ACCESS_DENIED | uniq
+
 ${CLICKHOUSE_CLIENT} -nm --query "
+    DROP TABLE IF EXISTS ${dicttablename} SYNC;
     DROP DICTIONARY IF EXISTS ${dictname};
     DROP USER IF EXISTS ${username};
 "

From cd9fd1880f672fe0c8b072bf63a73e4d85ecfda4 Mon Sep 17 00:00:00 2001
From: joelynch <joelynch112@gmail.com>
Date: Wed, 19 Jun 2024 16:44:47 +0200
Subject: [PATCH 0194/1488] Check for dict access on underlying dict rather
 than table

This is better for the table function because otherwise the database is
set to "_table_function" when checking access which is not the database
where the actual dict is.
---
 src/Storages/StorageDictionary.cpp             | 2 +-
 src/TableFunctions/TableFunctionDictionary.cpp | 1 -
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/Storages/StorageDictionary.cpp b/src/Storages/StorageDictionary.cpp
index a64acaebff6..72f873d1a0e 100644
--- a/src/Storages/StorageDictionary.cpp
+++ b/src/Storages/StorageDictionary.cpp
@@ -161,9 +161,9 @@ Pipe StorageDictionary::read(
     const size_t max_block_size,
     const size_t threads)
 {
-    local_context->checkAccess(AccessType::dictGet, getStorageID());
     auto registered_dictionary_name = location == Location::SameDatabaseAndNameAsDictionary ? getStorageID().getInternalDictionaryName() : dictionary_name;
     auto dictionary = getContext()->getExternalDictionariesLoader().getDictionary(registered_dictionary_name, local_context);
+    local_context->checkAccess(AccessType::dictGet, dictionary->getDatabaseOrNoDatabaseTag(), dictionary->getDictionaryID().getTableName());
     return dictionary->read(column_names, max_block_size, threads);
 }
 
diff --git a/src/TableFunctions/TableFunctionDictionary.cpp b/src/TableFunctions/TableFunctionDictionary.cpp
index 867fbf5b11e..d58c01385e0 100644
--- a/src/TableFunctions/TableFunctionDictionary.cpp
+++ b/src/TableFunctions/TableFunctionDictionary.cpp
@@ -80,7 +80,6 @@ ColumnsDescription TableFunctionDictionary::getActualTableStructure(ContextPtr c
 StoragePtr TableFunctionDictionary::executeImpl(
     const ASTPtr &, ContextPtr context, const std::string & table_name, ColumnsDescription, bool is_insert_query) const
 {
-    context->checkAccess(AccessType::dictGet, getDatabaseName(), table_name);
     StorageID dict_id(getDatabaseName(), table_name);
     auto dictionary_table_structure = getActualTableStructure(context, is_insert_query);
 

From 29dac3c97ad69736f876073ab6023d9e6c7c564a Mon Sep 17 00:00:00 2001
From: joelynch <joelynch112@gmail.com>
Date: Tue, 2 Jul 2024 20:47:02 +0200
Subject: [PATCH 0195/1488] move tests to new file

---
 .../02916_dictionary_access.reference         |  2 -
 .../0_stateless/02916_dictionary_access.sh    | 16 +-------
 .../03199_dictionary_table_access.reference   |  2 +
 .../03199_dictionary_table_access.sh          | 41 +++++++++++++++++++
 4 files changed, 44 insertions(+), 17 deletions(-)
 create mode 100644 tests/queries/0_stateless/03199_dictionary_table_access.reference
 create mode 100755 tests/queries/0_stateless/03199_dictionary_table_access.sh

diff --git a/tests/queries/0_stateless/02916_dictionary_access.reference b/tests/queries/0_stateless/02916_dictionary_access.reference
index ab44e74249e..d7d02e37689 100644
--- a/tests/queries/0_stateless/02916_dictionary_access.reference
+++ b/tests/queries/0_stateless/02916_dictionary_access.reference
@@ -1,5 +1,3 @@
 0
 ACCESS_DENIED
 ACCESS_DENIED
-ACCESS_DENIED
-ACCESS_DENIED
diff --git a/tests/queries/0_stateless/02916_dictionary_access.sh b/tests/queries/0_stateless/02916_dictionary_access.sh
index 5f80de120f4..08ee517ab3b 100755
--- a/tests/queries/0_stateless/02916_dictionary_access.sh
+++ b/tests/queries/0_stateless/02916_dictionary_access.sh
@@ -6,7 +6,6 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
 username="user_${CLICKHOUSE_TEST_UNIQUE_NAME}"
 dictname="dict_${CLICKHOUSE_TEST_UNIQUE_NAME}"
-dicttablename="dict_table_${CLICKHOUSE_TEST_UNIQUE_NAME}"
 
 ${CLICKHOUSE_CLIENT} -nm --query "
     CREATE DICTIONARY IF NOT EXISTS ${dictname}
@@ -19,13 +18,9 @@ ${CLICKHOUSE_CLIENT} -nm --query "
     LAYOUT(FLAT())
     LIFETIME(MIN 0 MAX 1000);
     CREATE USER IF NOT EXISTS ${username} NOT IDENTIFIED;
-    GRANT SELECT, CREATE TEMPORARY TABLE ON *.* to ${username};
+    GRANT CREATE TEMPORARY TABLE ON *.* to ${username};
     SELECT * FROM dictionary(${dictname});
-    SELECT * FROM ${dictname};
     SELECT dictGet(${dictname}, 'value', 1);
-    CREATE TABLE ${dicttablename} (id UInt64, value UInt64)
-    ENGINE = Dictionary(${CLICKHOUSE_DATABASE}.${dictname});
-    SELECT * FROM ${dicttablename};
 "
 
 $CLICKHOUSE_CLIENT -nm --user="${username}" --query "
@@ -36,16 +31,7 @@ $CLICKHOUSE_CLIENT -nm --user="${username}" --query "
     SELECT dictGet(${dictname}, 'value', 1);
 " 2>&1 | grep -o ACCESS_DENIED | uniq
 
-$CLICKHOUSE_CLIENT -nm --user="${username}" --query "
-    SELECT * FROM ${dictname};
-" 2>&1 | grep -o ACCESS_DENIED | uniq
-
-$CLICKHOUSE_CLIENT -nm --user="${username}" --query "
-    SELECT * FROM ${dicttablename};
-" 2>&1 | grep -o ACCESS_DENIED | uniq
-
 ${CLICKHOUSE_CLIENT} -nm --query "
-    DROP TABLE IF EXISTS ${dicttablename} SYNC;
     DROP DICTIONARY IF EXISTS ${dictname};
     DROP USER IF EXISTS ${username};
 "
diff --git a/tests/queries/0_stateless/03199_dictionary_table_access.reference b/tests/queries/0_stateless/03199_dictionary_table_access.reference
new file mode 100644
index 00000000000..4a703b3be84
--- /dev/null
+++ b/tests/queries/0_stateless/03199_dictionary_table_access.reference
@@ -0,0 +1,2 @@
+ACCESS_DENIED
+ACCESS_DENIED
diff --git a/tests/queries/0_stateless/03199_dictionary_table_access.sh b/tests/queries/0_stateless/03199_dictionary_table_access.sh
new file mode 100755
index 00000000000..952b466b5da
--- /dev/null
+++ b/tests/queries/0_stateless/03199_dictionary_table_access.sh
@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+
+CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CUR_DIR"/../shell_config.sh
+
+username="user_${CLICKHOUSE_TEST_UNIQUE_NAME}"
+dictname="dict_${CLICKHOUSE_TEST_UNIQUE_NAME}"
+dicttablename="dict_table_${CLICKHOUSE_TEST_UNIQUE_NAME}"
+
+${CLICKHOUSE_CLIENT} -nm --query "
+    CREATE DICTIONARY IF NOT EXISTS ${dictname}
+    (
+        id UInt64,
+        value UInt64
+    )
+    PRIMARY KEY id
+    SOURCE(NULL())
+    LAYOUT(FLAT())
+    LIFETIME(MIN 0 MAX 1000);
+    CREATE USER IF NOT EXISTS ${username} NOT IDENTIFIED;
+    GRANT SELECT, CREATE TEMPORARY TABLE ON *.* to ${username};
+    SELECT * FROM ${dictname};
+    CREATE TABLE ${dicttablename} (id UInt64, value UInt64)
+    ENGINE = Dictionary(${CLICKHOUSE_DATABASE}.${dictname});
+    SELECT * FROM ${dicttablename};
+"
+
+$CLICKHOUSE_CLIENT -nm --user="${username}" --query "
+    SELECT * FROM ${dictname};
+" 2>&1 | grep -o ACCESS_DENIED | uniq
+
+$CLICKHOUSE_CLIENT -nm --user="${username}" --query "
+    SELECT * FROM ${dicttablename};
+" 2>&1 | grep -o ACCESS_DENIED | uniq
+
+${CLICKHOUSE_CLIENT} -nm --query "
+    DROP TABLE IF EXISTS ${dicttablename} SYNC;
+    DROP DICTIONARY IF EXISTS ${dictname};
+    DROP USER IF EXISTS ${username};
+"

From a84dd6b7710da54e89722318755d4e4e70984e2b Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Thu, 11 Jul 2024 10:37:00 +0000
Subject: [PATCH 0196/1488] Fix some review comments

---
 .../Transforms/MergeJoinTransform.cpp         |  2 +-
 .../tests/gtest_full_sorting_join.cpp         | 48 +++++++++++++------
 .../00927_asof_join_correct_bt.reference      |  8 ++++
 .../00927_asof_join_correct_bt.sql            |  3 ++
 .../02276_full_sort_join_unsupported.sql      |  2 -
 ...03145_asof_join_ddb_inequalities.reference |  5 ++
 .../03145_asof_join_ddb_inequalities.sql      | 12 +++--
 .../03146_asof_join_ddb_merge_long.sql.j2     |  2 +-
 .../03149_asof_join_ddb_timestamps.sql        |  2 +-
 9 files changed, 59 insertions(+), 25 deletions(-)

diff --git a/src/Processors/Transforms/MergeJoinTransform.cpp b/src/Processors/Transforms/MergeJoinTransform.cpp
index 38a59cd6d9a..3b69ddaec06 100644
--- a/src/Processors/Transforms/MergeJoinTransform.cpp
+++ b/src/Processors/Transforms/MergeJoinTransform.cpp
@@ -43,7 +43,7 @@ FullMergeJoinCursorPtr createCursor(const Block & block, const Names & columns,
     return std::make_unique<FullMergeJoinCursor>(block, desc, strictness == JoinStrictness::Asof);
 }
 
-bool isNullAt(const IColumn & column, size_t row)
+bool ALWAYS_INLINE isNullAt(const IColumn & column, size_t row)
 {
     if (const auto * nullable_column = checkAndGetColumn<ColumnNullable>(&column))
         return nullable_column->isNullAt(row);
diff --git a/src/Processors/tests/gtest_full_sorting_join.cpp b/src/Processors/tests/gtest_full_sorting_join.cpp
index 7294a1b381a..f678d7984e8 100644
--- a/src/Processors/tests/gtest_full_sorting_join.cpp
+++ b/src/Processors/tests/gtest_full_sorting_join.cpp
@@ -484,7 +484,7 @@ try
     right_source.addRow({"AAPL", 2, 98});
     right_source.addRow({"AAPL", 3, 99});
     right_source.addRow({"AMZN", 1, 100});
-    right_source.addRow({"AMZN", 2, 0});
+    right_source.addRow({"AMZN", 2, 110});
     right_source.addChunk();
     right_source.addRow({"AMZN", 2, 110});
     right_source.addChunk();
@@ -574,12 +574,15 @@ catch (Exception & e)
 TEST_F(FullSortingJoinTest, AsofLessGeneratedTestData)
 try
 {
-    auto join_kind = getRandomFrom(rng, { JoinKind::Inner, JoinKind::Left });
+    /// Generate data random and build expected result at the same time.
 
+    /// Test specific combinations of join kind and inequality per each run
+    auto join_kind = getRandomFrom(rng, { JoinKind::Inner, JoinKind::Left });
     auto asof_inequality = getRandomFrom(rng, { ASOFJoinInequality::Less, ASOFJoinInequality::LessOrEquals });
 
     SCOPED_TRACE(fmt::format("{} {}", join_kind, asof_inequality));
 
+    /// Key is complex, `k1, k2` for equality and `t` for asof
     SourceChunksBuilder left_source_builder({
         {std::make_shared<DataTypeUInt64>(), "k1"},
         {std::make_shared<DataTypeString>(), "k2"},
@@ -594,9 +597,11 @@ try
         {std::make_shared<DataTypeInt64>(), "attr"},
     });
 
+    /// How small generated block should be
     left_source_builder.setBreakProbability(rng);
     right_source_builder.setBreakProbability(rng);
 
+    /// We are going to generate sorted data and remember expected result
     ColumnInt64::Container expected;
 
     UInt64 k1 = 1;
@@ -604,29 +609,34 @@ try
     auto key_num_total = std::uniform_int_distribution<>(1, 1000)(rng);
     for (size_t key_num = 0; key_num < key_num_total; ++key_num)
     {
+        /// Generate new key greater than previous
         generateNextKey(rng, k1, k2);
 
         Int64 left_t = 0;
+        /// Generate several rows for the key
         size_t num_left_rows = std::uniform_int_distribution<>(1, 100)(rng);
         for (size_t i = 0; i < num_left_rows; ++i)
         {
+            /// t is strictly greater than previous
             left_t += std::uniform_int_distribution<>(1, 10)(rng);
 
-            left_source_builder.addRow({k1, k2, left_t, 10 * left_t});
-            expected.push_back(10 * left_t);
+            auto left_arrtibute_value = 10 * left_t;
+            left_source_builder.addRow({k1, k2, left_t, left_arrtibute_value});
+            expected.push_back(left_arrtibute_value);
 
             auto num_matches = 1 + std::poisson_distribution<>(4)(rng);
-
+            /// Generate several matches in the right table
             auto right_t = left_t;
             for (size_t j = 0; j < num_matches; ++j)
             {
                 int min_step = isStrict(asof_inequality) ? 1 : 0;
                 right_t += std::uniform_int_distribution<>(min_step, 3)(rng);
 
+                /// First row should match
                 bool is_match = j == 0;
-                right_source_builder.addRow({k1, k2, right_t, is_match ? 100 * left_t : -1});
+                right_source_builder.addRow({k1, k2, right_t, is_match ? 10 * left_arrtibute_value : -1});
             }
-            /// next left_t should be greater than right_t not to match with previous rows
+            /// Next left_t should be greater than right_t not to match with previous rows
             left_t = right_t;
         }
 
@@ -650,7 +660,9 @@ try
     assertColumnVectorEq<Int64>(expected, result_block, "t1.attr");
 
     for (auto & e : expected)
-        e = e < 0 ? 0 : 10 * e; /// non matched rows from left table have negative attr
+        /// Non matched rows from left table have negative attr
+        /// Value if attribute in right table is 10 times greater than in left table
+        e = e < 0 ? 0 : 10 * e;
 
     assertColumnVectorEq<Int64>(expected, result_block, "t2.attr");
 }
@@ -663,8 +675,10 @@ catch (Exception & e)
 TEST_F(FullSortingJoinTest, AsofGreaterGeneratedTestData)
 try
 {
-    auto join_kind = getRandomFrom(rng, { JoinKind::Inner, JoinKind::Left });
+    /// Generate data random and build expected result at the same time.
 
+    /// Test specific combinations of join kind and inequality per each run
+    auto join_kind = getRandomFrom(rng, { JoinKind::Inner, JoinKind::Left });
     auto asof_inequality = getRandomFrom(rng, { ASOFJoinInequality::Greater, ASOFJoinInequality::GreaterOrEquals });
 
     SCOPED_TRACE(fmt::format("{} {}", join_kind, asof_inequality));
@@ -695,9 +709,10 @@ try
     auto key_num_total = std::uniform_int_distribution<>(1, 1000)(rng);
     for (size_t key_num = 0; key_num < key_num_total; ++key_num)
     {
+        /// Generate new key greater than previous
         generateNextKey(rng, k1, k2);
 
-        /// generate some rows with smaller left_t to check that they are not matched
+        /// Generate some rows with smaller left_t to check that they are not matched
         size_t num_left_rows = std::bernoulli_distribution(0.5)(rng) ? std::uniform_int_distribution<>(1, 100)(rng) : 0;
         for (size_t i = 0; i < num_left_rows; ++i)
         {
@@ -713,21 +728,22 @@ try
 
         size_t num_right_matches = std::uniform_int_distribution<>(1, 10)(rng);
         auto right_t = left_t + std::uniform_int_distribution<>(isStrict(asof_inequality) ? 0 : 1, 10)(rng);
+        auto attribute_value = 10 * right_t;
         for (size_t j = 0; j < num_right_matches; ++j)
         {
             right_t += std::uniform_int_distribution<>(0, 3)(rng);
             bool is_match = j == num_right_matches - 1;
-            right_source_builder.addRow({k1, k2, right_t, is_match ? 100 * right_t : -1});
+            right_source_builder.addRow({k1, k2, right_t, is_match ? 10 * attribute_value : -1});
         }
 
-        /// next left_t should be greater than (or equals) right_t to match with previous rows
+        /// Next left_t should be greater than (or equals) right_t to match with previous rows
         left_t = right_t + std::uniform_int_distribution<>(isStrict(asof_inequality) ? 1 : 0, 100)(rng);
         size_t num_left_matches = std::uniform_int_distribution<>(1, 100)(rng);
         for (size_t j = 0; j < num_left_matches; ++j)
         {
             left_t += std::uniform_int_distribution<>(0, 3)(rng);
-            left_source_builder.addRow({k1, k2, left_t, 10 * right_t});
-            expected.push_back(10 * right_t);
+            left_source_builder.addRow({k1, k2, left_t, attribute_value});
+            expected.push_back(attribute_value);
         }
     }
 
@@ -739,7 +755,9 @@ try
     assertColumnVectorEq<Int64>(expected, result_block, "t1.attr");
 
     for (auto & e : expected)
-        e = e < 0 ? 0 : 10 * e; /// non matched rows from left table have negative attr
+        /// Non matched rows from left table have negative attr
+        /// Value if attribute in right table is 10 times greater than in left table
+        e = e < 0 ? 0 : 10 * e;
 
     assertColumnVectorEq<Int64>(expected, result_block, "t2.attr");
 }
diff --git a/tests/queries/0_stateless/00927_asof_join_correct_bt.reference b/tests/queries/0_stateless/00927_asof_join_correct_bt.reference
index a398f9604fd..28c48d2e290 100644
--- a/tests/queries/0_stateless/00927_asof_join_correct_bt.reference
+++ b/tests/queries/0_stateless/00927_asof_join_correct_bt.reference
@@ -1,28 +1,36 @@
+-- { echoOn }
+SELECT A.k, A.t, A.a, B.b, B.t, B.k FROM A ASOF LEFT JOIN B1 B USING(k,t) ORDER BY (A.k, A.t);
 1	101	1	0	0	0
 1	102	2	2	102	1
 1	103	3	2	102	1
 1	104	4	4	104	1
 1	105	5	4	104	1
+SELECT A.k, A.t, A.a, B.b, B.t, B.k FROM A ASOF LEFT JOIN B2 B USING(k,t) ORDER BY (A.k, A.t);
 1	101	1	0	0	0
 1	102	2	2	102	1
 1	103	3	2	102	1
 1	104	4	4	104	1
 1	105	5	4	104	1
+SELECT A.k, A.t, A.a, B.b, B.t, B.k FROM A ASOF LEFT JOIN B3 B USING(k,t) ORDER BY (A.k, A.t);
 1	101	1	0	0	0
 1	102	2	2	102	1
 1	103	3	2	102	1
 1	104	4	4	104	1
 1	105	5	4	104	1
+SET join_algorithm = 'full_sorting_merge';
+SELECT A.k, A.t, A.a, B.b, B.t, B.k FROM A ASOF LEFT JOIN B1 B USING(k,t) ORDER BY (A.k, A.t);
 1	101	1	0	0	0
 1	102	2	2	102	1
 1	103	3	2	102	1
 1	104	4	4	104	1
 1	105	5	4	104	1
+SELECT A.k, A.t, A.a, B.b, B.t, B.k FROM A ASOF LEFT JOIN B2 B USING(k,t) ORDER BY (A.k, A.t);
 1	101	1	0	0	0
 1	102	2	2	102	1
 1	103	3	2	102	1
 1	104	4	4	104	1
 1	105	5	4	104	1
+SELECT A.k, A.t, A.a, B.b, B.t, B.k FROM A ASOF LEFT JOIN B3 B USING(k,t) ORDER BY (A.k, A.t);
 1	101	1	0	0	0
 1	102	2	2	102	1
 1	103	3	2	102	1
diff --git a/tests/queries/0_stateless/00927_asof_join_correct_bt.sql b/tests/queries/0_stateless/00927_asof_join_correct_bt.sql
index 761d6bacde6..d796b62d3b3 100644
--- a/tests/queries/0_stateless/00927_asof_join_correct_bt.sql
+++ b/tests/queries/0_stateless/00927_asof_join_correct_bt.sql
@@ -13,6 +13,7 @@ INSERT INTO B2(k,t,b) VALUES (1,102,2), (1,104,4);
 CREATE TABLE B3(k UInt32, b UInt64, t UInt32) ENGINE = MergeTree() ORDER BY (k, t);
 INSERT INTO B3(k,t,b) VALUES (1,102,2), (1,104,4);
 
+-- { echoOn }
 SELECT A.k, A.t, A.a, B.b, B.t, B.k FROM A ASOF LEFT JOIN B1 B USING(k,t) ORDER BY (A.k, A.t);
 SELECT A.k, A.t, A.a, B.b, B.t, B.k FROM A ASOF LEFT JOIN B2 B USING(k,t) ORDER BY (A.k, A.t);
 SELECT A.k, A.t, A.a, B.b, B.t, B.k FROM A ASOF LEFT JOIN B3 B USING(k,t) ORDER BY (A.k, A.t);
@@ -22,6 +23,8 @@ SELECT A.k, A.t, A.a, B.b, B.t, B.k FROM A ASOF LEFT JOIN B1 B USING(k,t) ORDER
 SELECT A.k, A.t, A.a, B.b, B.t, B.k FROM A ASOF LEFT JOIN B2 B USING(k,t) ORDER BY (A.k, A.t);
 SELECT A.k, A.t, A.a, B.b, B.t, B.k FROM A ASOF LEFT JOIN B3 B USING(k,t) ORDER BY (A.k, A.t);
 
+-- { echoOff }
+
 DROP TABLE B1;
 DROP TABLE B2;
 DROP TABLE B3;
diff --git a/tests/queries/0_stateless/02276_full_sort_join_unsupported.sql b/tests/queries/0_stateless/02276_full_sort_join_unsupported.sql
index 03936107563..0b10101d8f2 100644
--- a/tests/queries/0_stateless/02276_full_sort_join_unsupported.sql
+++ b/tests/queries/0_stateless/02276_full_sort_join_unsupported.sql
@@ -19,8 +19,6 @@ SELECT * FROM t1 ANTI JOIN t2 ON t1.key = t2.key; -- { serverError NOT_IMPLEMENT
 
 SELECT * FROM t1 SEMI JOIN t2 ON t1.key = t2.key; -- { serverError NOT_IMPLEMENTED }
 
--- SELECT * FROM t1 ASOF JOIN t2 ON t1.key = t2.key AND t1.val > t2.val; -- { serverError NOT_IMPLEMENTED }
-
 SELECT * FROM t1 ANY JOIN t2 ON t1.key = t2.key SETTINGS any_join_distinct_right_table_keys = 1; -- { serverError NOT_IMPLEMENTED }
 
 SELECT * FROM t1 JOIN t2 USING (key) SETTINGS join_use_nulls = 1; -- { serverError NOT_IMPLEMENTED }
diff --git a/tests/queries/0_stateless/03145_asof_join_ddb_inequalities.reference b/tests/queries/0_stateless/03145_asof_join_ddb_inequalities.reference
index 73c4f7dfe25..4aac918c98c 100644
--- a/tests/queries/0_stateless/03145_asof_join_ddb_inequalities.reference
+++ b/tests/queries/0_stateless/03145_asof_join_ddb_inequalities.reference
@@ -1,3 +1,4 @@
+-
 2023-03-21 12:00:00	1970-01-01 00:00:00	-1
 2023-03-21 13:00:00	1970-01-01 00:00:00	-1
 2023-03-21 14:00:00	2023-03-21 13:00:00	0
@@ -9,6 +10,7 @@
 2023-03-21 20:00:00	2023-03-21 16:00:00	3
 2023-03-21 21:00:00	2023-03-21 16:00:00	3
 2027-10-18 11:03:27	2023-03-21 16:00:00	3
+-
 2023-03-21 12:00:00	1970-01-01 00:00:00	-1
 2023-03-21 13:00:00	1970-01-01 00:00:00	-1
 2023-03-21 14:00:00	2023-03-21 13:00:00	0
@@ -32,6 +34,7 @@
 2023-03-21 20:00:00	2027-10-18 11:03:27	9
 2023-03-21 21:00:00	2027-10-18 11:03:27	9
 2027-10-18 11:03:27	2027-10-18 11:03:27	9
+-
 2023-03-21 12:00:00	2023-03-21 13:00:00	0
 2023-03-21 13:00:00	2023-03-21 13:00:00	0
 2023-03-21 14:00:00	2023-03-21 14:00:00	1
@@ -44,6 +47,7 @@
 2023-03-21 21:00:00	2027-10-18 11:03:27	9
 2027-10-18 11:03:27	2027-10-18 11:03:27	9
 \N	\N	\N
+-
 2023-03-21 12:00:00	2023-03-21 13:00:00	0
 2023-03-21 13:00:00	2023-03-21 14:00:00	1
 2023-03-21 14:00:00	2023-03-21 15:00:00	2
@@ -54,6 +58,7 @@
 2023-03-21 19:00:00	2027-10-18 11:03:27	9
 2023-03-21 20:00:00	2027-10-18 11:03:27	9
 2023-03-21 21:00:00	2027-10-18 11:03:27	9
+-
 2023-03-21 12:00:00	2023-03-21 13:00:00	0
 2023-03-21 13:00:00	2023-03-21 14:00:00	1
 2023-03-21 14:00:00	2023-03-21 15:00:00	2
diff --git a/tests/queries/0_stateless/03145_asof_join_ddb_inequalities.sql b/tests/queries/0_stateless/03145_asof_join_ddb_inequalities.sql
index ce4badbd597..d67aa254bd6 100644
--- a/tests/queries/0_stateless/03145_asof_join_ddb_inequalities.sql
+++ b/tests/queries/0_stateless/03145_asof_join_ddb_inequalities.sql
@@ -21,18 +21,19 @@ INSERT INTO probe0 VALUES (NULL),('9999-12-31 23:59:59');
 
 SET join_use_nulls = 1;
 
+SELECT '-';
 SELECT p.begin, e.begin, e.value
 FROM probe0 p
 ASOF JOIN events0 e
 ON p.begin > e.begin
 ORDER BY p.begin ASC;
 
+SELECT '-';
 SELECT p.begin, e.begin, e.value
 FROM probe0 p
 ASOF LEFT JOIN events0 e
 ON p.begin > e.begin
-ORDER BY p.begin ASC
-;
+ORDER BY p.begin ASC;
 
 SELECT p.begin, e.begin, e.value
 FROM probe0 p
@@ -40,25 +41,26 @@ ASOF JOIN events0 e
 ON p.begin <= e.begin
 ORDER BY p.begin ASC;
 
+SELECT '-';
 SELECT p.begin, e.begin, e.value
 FROM probe0 p
 ASOF LEFT JOIN events0 e
 ON p.begin <= e.begin
 ORDER BY p.begin ASC;
 
+SELECT '-';
 SELECT p.begin, e.begin, e.value
 FROM probe0 p
 ASOF JOIN events0 e
 ON p.begin < e.begin
-ORDER BY p.begin ASC
-;
+ORDER BY p.begin ASC;
 
+SELECT '-';
 SELECT p.begin, e.begin, e.value
 FROM probe0 p
 ASOF LEFT JOIN events0 e
 ON p.begin < e.begin
 ORDER BY p.begin ASC;
 
-
 DROP TABLE IF EXISTS events0;
 DROP TABLE IF EXISTS probe0;
diff --git a/tests/queries/0_stateless/03146_asof_join_ddb_merge_long.sql.j2 b/tests/queries/0_stateless/03146_asof_join_ddb_merge_long.sql.j2
index 44c54ae2a39..49ba70c471e 100644
--- a/tests/queries/0_stateless/03146_asof_join_ddb_merge_long.sql.j2
+++ b/tests/queries/0_stateless/03146_asof_join_ddb_merge_long.sql.j2
@@ -8,7 +8,7 @@ SET session_timezone = 'UTC';
 
 SET join_algorithm = '{{ join_algorithm }}';
 
--- TODO: support enable for full_sorting_merge
+-- TODO: enable once USING and `join_use_nulls` is supported by `full_sorting_merge`
 -- SET join_use_nulls = 1;
 
 WITH build AS (
diff --git a/tests/queries/0_stateless/03149_asof_join_ddb_timestamps.sql b/tests/queries/0_stateless/03149_asof_join_ddb_timestamps.sql
index ff4518a3775..cd83d62dc70 100644
--- a/tests/queries/0_stateless/03149_asof_join_ddb_timestamps.sql
+++ b/tests/queries/0_stateless/03149_asof_join_ddb_timestamps.sql
@@ -87,7 +87,7 @@ FROM
     probe0 p
     ASOF LEFT JOIN (
         SELECT * FROM events0 WHERE log(value + 5) > 10
-    ) e ON p.begin >= e.begin
+    ) e ON p.begin + INTERVAL 2 HOUR >= e.begin + INTERVAL 1 HOUR
 ORDER BY p.begin ASC;
 
 

From f85be2b452e9d62f766cee3d170dacd1906e04b7 Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Thu, 11 Jul 2024 12:17:59 +0000
Subject: [PATCH 0197/1488] Add --memory-usage option to client in non
 interactive mode

---
 src/Client/ClientBase.cpp       | 22 ++++++++++++++++++++--
 src/Common/ProgressIndication.h | 10 +++++-----
 2 files changed, 25 insertions(+), 7 deletions(-)

diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp
index 5d472ba99b9..b5de348d583 100644
--- a/src/Client/ClientBase.cpp
+++ b/src/Client/ClientBase.cpp
@@ -2069,9 +2069,18 @@ void ClientBase::processParsedSingleQuery(const String & full_query, const Strin
         progress_indication.writeFinalProgress();
         output_stream << std::endl << std::endl;
     }
-    else if (getClientConfiguration().getBool("print-time-to-stderr", false))
+    else
     {
-        error_stream << progress_indication.elapsedSeconds() << "\n";
+        const auto & config = getClientConfiguration();
+        if (config.getBool("print-time-to-stderr", false))
+            error_stream << progress_indication.elapsedSeconds() << "\n";
+
+        const auto & print_memory_mode = config.getString("print-memory-to-stderr", "");
+        auto peak_memeory_usage = std::max<Int64>(progress_indication.getMemoryUsage().peak, 0);
+        if (print_memory_mode == "default")
+            error_stream << peak_memeory_usage << "\n";
+        else if (print_memory_mode == "readable")
+            error_stream << formatReadableSizeWithBinarySuffix(peak_memeory_usage) << "\n";
     }
 
     if (!is_interactive && getClientConfiguration().getBool("print-num-processed-rows", false))
@@ -3035,6 +3044,7 @@ void ClientBase::init(int argc, char ** argv)
         ("disable_suggestion,A", "Disable loading suggestion data. Note that suggestion data is loaded asynchronously through a second connection to ClickHouse server. Also it is reasonable to disable suggestion if you want to paste a query with TAB characters. Shorthand option -A is for those who get used to mysql client.")
         ("wait_for_suggestions_to_load", "Load suggestion data synchonously.")
         ("time,t", "print query execution time to stderr in non-interactive mode (for benchmarks)")
+        ("memory-usage", po::value<std::string>()->implicit_value("default")->default_value("none"), "print memory usage to stderr in non-interactive mode (for benchmarks). Values: 'none', 'default', 'readable'")
 
         ("echo", "in batch mode, print query before execution")
 
@@ -3120,6 +3130,14 @@ void ClientBase::init(int argc, char ** argv)
     /// Output execution time to stderr in batch mode.
     if (options.count("time"))
         getClientConfiguration().setBool("print-time-to-stderr", true);
+    if (options.count("memory-usage"))
+    {
+        const auto & memory_usage_mode = options["memory-usage"].as<std::string>();
+        if (memory_usage_mode != "none" && memory_usage_mode != "default" && memory_usage_mode != "readable")
+            throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown memory-usage mode: {}", memory_usage_mode);
+        getClientConfiguration().setString("print-memory-to-stderr", memory_usage_mode);
+    }
+
     if (options.count("query"))
         queries = options["query"].as<std::vector<std::string>>();
     if (options.count("query_id"))
diff --git a/src/Common/ProgressIndication.h b/src/Common/ProgressIndication.h
index ae39fb49bcc..c7f61a7fba9 100644
--- a/src/Common/ProgressIndication.h
+++ b/src/Common/ProgressIndication.h
@@ -71,11 +71,6 @@ public:
     /// How much seconds passed since query execution start.
     double elapsedSeconds() const { return getElapsedNanoseconds() / 1e9; }
 
-    void updateThreadEventData(HostToTimesMap & new_hosts_data);
-
-private:
-    double getCPUUsage();
-
     struct MemoryUsage
     {
         UInt64 total = 0;
@@ -85,6 +80,11 @@ private:
 
     MemoryUsage getMemoryUsage() const;
 
+    void updateThreadEventData(HostToTimesMap & new_hosts_data);
+
+private:
+    double getCPUUsage();
+
     UInt64 getElapsedNanoseconds() const;
 
     /// This flag controls whether to show the progress bar. We start showing it after

From b9cd1bddd7c8612e3b43172ea8670a7e904fe237 Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Thu, 11 Jul 2024 12:20:37 +0000
Subject: [PATCH 0198/1488] upd doc

---
 docs/en/interfaces/cli.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/docs/en/interfaces/cli.md b/docs/en/interfaces/cli.md
index e18ff6f1a3f..63b7353d092 100644
--- a/docs/en/interfaces/cli.md
+++ b/docs/en/interfaces/cli.md
@@ -185,6 +185,7 @@ You can pass parameters to `clickhouse-client` (all parameters have a default va
 - `--format, -f` – Use the specified default format to output the result.
 - `--vertical, -E` – If specified, use the [Vertical format](../interfaces/formats.md#vertical) by default to output the result. This is the same as `–format=Vertical`. In this format, each value is printed on a separate line, which is helpful when displaying wide tables.
 - `--time, -t` – If specified, print the query execution time to ‘stderr’ in non-interactive mode.
+- `--memory-usage` – If specified, print the query memory usage to ‘stderr’ in non-interactive mode.
 - `--stacktrace` – If specified, also print the stack trace if an exception occurs.
 - `--config-file` – The name of the configuration file.
 - `--secure` – If specified, will connect to server over secure connection (TLS). You might need to configure your CA certificates in the [configuration file](#configuration_files). The available configuration settings are the same as for [server-side TLS configuration](../operations/server-configuration-parameters/settings.md#server_configuration_parameters-openssl).
@@ -339,7 +340,7 @@ clickhouse-client clickhouse://some_user%40some_mail.com@localhost:9000
 Connect to one of provides hosts: `192.168.1.15`, `192.168.1.25`.
 
 ``` bash
-clickhouse-client clickhouse://192.168.1.15,192.168.1.25 
+clickhouse-client clickhouse://192.168.1.15,192.168.1.25
 ```
 
 ### Configuration Files {#configuration_files}
@@ -367,7 +368,7 @@ Example of a config file:
 ```
 
 Or the same config in a YAML format:
- 
+
 ```yaml
 user: username
 password: 'password'

From 05c3692e1dfcacffc9de62bac9a53a0e28f4bea7 Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Thu, 11 Jul 2024 12:22:03 +0000
Subject: [PATCH 0199/1488] add 03203_client_benchmark_options

---
 .../03203_client_benchmark_options.reference        |  6 ++++++
 .../0_stateless/03203_client_benchmark_options.sh   | 13 +++++++++++++
 2 files changed, 19 insertions(+)
 create mode 100644 tests/queries/0_stateless/03203_client_benchmark_options.reference
 create mode 100755 tests/queries/0_stateless/03203_client_benchmark_options.sh

diff --git a/tests/queries/0_stateless/03203_client_benchmark_options.reference b/tests/queries/0_stateless/03203_client_benchmark_options.reference
new file mode 100644
index 00000000000..fd2996b1c78
--- /dev/null
+++ b/tests/queries/0_stateless/03203_client_benchmark_options.reference
@@ -0,0 +1,6 @@
+Ok
+Ok
+Ok
+Ok
+Ok
+Ok
diff --git a/tests/queries/0_stateless/03203_client_benchmark_options.sh b/tests/queries/0_stateless/03203_client_benchmark_options.sh
new file mode 100755
index 00000000000..a9b9d69822b
--- /dev/null
+++ b/tests/queries/0_stateless/03203_client_benchmark_options.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CUR_DIR"/../shell_config.sh
+
+${CLICKHOUSE_CLIENT} -t -q "SELECT sleepEachRow(2) FORMAT Null" 2>&1 | grep -q "^2\." && echo "Ok" || echo "Fail"
+${CLICKHOUSE_CLIENT} --time -q "SELECT sleepEachRow(2) FORMAT Null" 2>&1 | grep -q "^2\." && echo "Ok" || echo "Fail"
+${CLICKHOUSE_CLIENT} --memory-usage -q "SELECT sum(number) FROM numbers(10_000) FORMAT Null" 2>&1 | grep -q "^[0-9]\+$" && echo "Ok" || echo "Fail"
+${CLICKHOUSE_CLIENT} --memory-usage=none -q "SELECT sum(number) FROM numbers(10_000) FORMAT Null" # expected no output
+${CLICKHOUSE_CLIENT} --memory-usage=default -q "SELECT sum(number) FROM numbers(10_000) FORMAT Null" 2>&1 | grep -q "^[0-9]\+$" && echo "Ok" || echo "Fail"
+${CLICKHOUSE_CLIENT} --memory-usage=readable -q "SELECT sum(number) FROM numbers(10_000) FORMAT Null" 2>&1 | grep -q "^[0-9].*B$" && echo "Ok" || echo "Fail"
+${CLICKHOUSE_CLIENT} --memory-usage=unknown -q "SELECT sum(number) FROM numbers(10_000) FORMAT Null" 2>&1 | grep -q "BAD_ARGUMENTS" && echo "Ok" || echo "Fail"

From f81383b856293de861c9bb76e7eac3597ffac503 Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Thu, 11 Jul 2024 12:33:41 +0000
Subject: [PATCH 0200/1488] tiny fix

---
 src/Coordination/KeeperServer.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Coordination/KeeperServer.cpp b/src/Coordination/KeeperServer.cpp
index 14b51d716de..fd1c5475911 100644
--- a/src/Coordination/KeeperServer.cpp
+++ b/src/Coordination/KeeperServer.cpp
@@ -140,7 +140,7 @@ KeeperServer::KeeperServer(
             responses_queue_,
             snapshots_queue_,
             keeper_context,
-            config.getBool("keeper_server.upload_snapshot_on_exit", true) ? &snapshot_manager_s3 : nullptr,
+            config.getBool("keeper_server.upload_snapshot_on_exit", false) ? &snapshot_manager_s3 : nullptr,
             commit_callback,
             checkAndGetSuperdigest(configuration_and_settings_->super_digest));
         LOG_WARNING(log, "Use RocksDB as Keeper backend storage.");

From ef4f34545e46a4e0298bd4bca28597ae642f7b50 Mon Sep 17 00:00:00 2001
From: Sema Checherinda <Sema.Checherinda@clickhouse.com>
Date: Thu, 11 Jul 2024 18:00:32 +0200
Subject: [PATCH 0201/1488] make 01275_parallel_mv.sql.j2 stable

---
 .../Transforms/buildPushingToViewsChain.cpp   |   4 +-
 .../0_stateless/01275_parallel_mv.reference   | 101 +++++++++---------
 .../0_stateless/01275_parallel_mv.sql.j2      |  16 ++-
 tests/result                                  |  12 +++
 4 files changed, 78 insertions(+), 55 deletions(-)
 create mode 100644 tests/result

diff --git a/src/Processors/Transforms/buildPushingToViewsChain.cpp b/src/Processors/Transforms/buildPushingToViewsChain.cpp
index 312b333ab33..3df726aa0e8 100644
--- a/src/Processors/Transforms/buildPushingToViewsChain.cpp
+++ b/src/Processors/Transforms/buildPushingToViewsChain.cpp
@@ -794,7 +794,7 @@ PushingToLiveViewSink::PushingToLiveViewSink(const Block & header, StorageLiveVi
 void PushingToLiveViewSink::consume(Chunk & chunk)
 {
     Progress local_progress(chunk.getNumRows(), chunk.bytes(), 0);
-    live_view.writeBlock(live_view, getHeader().cloneWithColumns(chunk.detachColumns()), std::move(chunk.getChunkInfos()), context);
+    live_view.writeBlock(live_view, getHeader().cloneWithColumns(chunk.getColumns()), std::move(chunk.getChunkInfos()), context);
 
     if (auto process = context->getProcessListElement())
         process->updateProgressIn(local_progress);
@@ -818,7 +818,7 @@ void PushingToWindowViewSink::consume(Chunk & chunk)
 {
     Progress local_progress(chunk.getNumRows(), chunk.bytes(), 0);
     StorageWindowView::writeIntoWindowView(
-        window_view, getHeader().cloneWithColumns(chunk.detachColumns()), std::move(chunk.getChunkInfos()), context);
+        window_view, getHeader().cloneWithColumns(chunk.getColumns()), std::move(chunk.getChunkInfos()), context);
 
     if (auto process = context->getProcessListElement())
         process->updateProgressIn(local_progress);
diff --git a/tests/queries/0_stateless/01275_parallel_mv.reference b/tests/queries/0_stateless/01275_parallel_mv.reference
index 221d0b5ce8d..cf1f7054f6c 100644
--- a/tests/queries/0_stateless/01275_parallel_mv.reference
+++ b/tests/queries/0_stateless/01275_parallel_mv.reference
@@ -2,13 +2,18 @@
 
 
 
-select 'optimize_trivial_insert_select=0', 'max_insert_threads=0';
-optimize_trivial_insert_select=0	max_insert_threads=0
+
+
+
+select 'optimize_trivial_insert_select=0', 'max_insert_threads=0', 'iteration_num=';
+optimize_trivial_insert_select=0	max_insert_threads=0	iteration_num=
 insert into testX select number from numbers(200) settings
+    send_logs_level='fatal',
+    insert_deduplication_token='UT_1',
     log_queries=1,
     parallel_view_processing=0,
     optimize_trivial_insert_select=0,
-    max_insert_threads=0; -- { serverError FUNCTION_THROW_IF_VALUE_IS_NON_ZERO }
+    max_insert_threads=0;
 system flush logs;
 select peak_threads_usage from system.query_log where
     current_database = currentDatabase() and
@@ -22,17 +27,17 @@ select count() from testX;
 200
 select count() from testXA;
 200
-select count() from testXB;
-0
 select count() from testXC;
 200
-select 'optimize_trivial_insert_select=0', 'max_insert_threads=5';
-optimize_trivial_insert_select=0	max_insert_threads=5
+select 'optimize_trivial_insert_select=0', 'max_insert_threads=5', 'iteration_num=';
+optimize_trivial_insert_select=0	max_insert_threads=5	iteration_num=
 insert into testX select number from numbers(200) settings
+    send_logs_level='fatal',
+    insert_deduplication_token='UT_2',
     log_queries=1,
     parallel_view_processing=0,
     optimize_trivial_insert_select=0,
-    max_insert_threads=5; -- { serverError FUNCTION_THROW_IF_VALUE_IS_NON_ZERO }
+    max_insert_threads=5;
 system flush logs;
 select peak_threads_usage from system.query_log where
     current_database = currentDatabase() and
@@ -46,17 +51,17 @@ select count() from testX;
 400
 select count() from testXA;
 400
-select count() from testXB;
-0
 select count() from testXC;
 400
-select 'optimize_trivial_insert_select=1', 'max_insert_threads=0';
-optimize_trivial_insert_select=1	max_insert_threads=0
+select 'optimize_trivial_insert_select=1', 'max_insert_threads=0', 'iteration_num=';
+optimize_trivial_insert_select=1	max_insert_threads=0	iteration_num=
 insert into testX select number from numbers(200) settings
+    send_logs_level='fatal',
+    insert_deduplication_token='UT_3',
     log_queries=1,
     parallel_view_processing=0,
     optimize_trivial_insert_select=1,
-    max_insert_threads=0; -- { serverError FUNCTION_THROW_IF_VALUE_IS_NON_ZERO }
+    max_insert_threads=0;
 system flush logs;
 select peak_threads_usage from system.query_log where
     current_database = currentDatabase() and
@@ -70,17 +75,17 @@ select count() from testX;
 600
 select count() from testXA;
 600
-select count() from testXB;
-0
 select count() from testXC;
 600
-select 'optimize_trivial_insert_select=1', 'max_insert_threads=5';
-optimize_trivial_insert_select=1	max_insert_threads=5
+select 'optimize_trivial_insert_select=1', 'max_insert_threads=5', 'iteration_num=';
+optimize_trivial_insert_select=1	max_insert_threads=5	iteration_num=
 insert into testX select number from numbers(200) settings
+    send_logs_level='fatal',
+    insert_deduplication_token='UT_4',
     log_queries=1,
     parallel_view_processing=0,
     optimize_trivial_insert_select=1,
-    max_insert_threads=5; -- { serverError FUNCTION_THROW_IF_VALUE_IS_NON_ZERO }
+    max_insert_threads=5;
 system flush logs;
 select peak_threads_usage from system.query_log where
     current_database = currentDatabase() and
@@ -94,17 +99,17 @@ select count() from testX;
 800
 select count() from testXA;
 800
-select count() from testXB;
-0
 select count() from testXC;
 800
-select 'optimize_trivial_insert_select=0', 'max_insert_threads=0';
-optimize_trivial_insert_select=0	max_insert_threads=0
+select 'optimize_trivial_insert_select=0', 'max_insert_threads=0', 'iteration_num=';
+optimize_trivial_insert_select=0	max_insert_threads=0	iteration_num=
 insert into testX select number from numbers(200) settings
+    send_logs_level='fatal',
+    insert_deduplication_token='UT_5',
     log_queries=1,
     parallel_view_processing=1,
     optimize_trivial_insert_select=0,
-    max_insert_threads=0; -- { serverError FUNCTION_THROW_IF_VALUE_IS_NON_ZERO }
+    max_insert_threads=0;
 system flush logs;
 select peak_threads_usage from system.query_log where
     current_database = currentDatabase() and
@@ -118,17 +123,17 @@ select count() from testX;
 1000
 select count() from testXA;
 1000
-select count() from testXB;
-0
 select count() from testXC;
 1000
-select 'optimize_trivial_insert_select=0', 'max_insert_threads=5';
-optimize_trivial_insert_select=0	max_insert_threads=5
+select 'optimize_trivial_insert_select=0', 'max_insert_threads=5', 'iteration_num=';
+optimize_trivial_insert_select=0	max_insert_threads=5	iteration_num=
 insert into testX select number from numbers(200) settings
+    send_logs_level='fatal',
+    insert_deduplication_token='UT_6',
     log_queries=1,
     parallel_view_processing=1,
     optimize_trivial_insert_select=0,
-    max_insert_threads=5; -- { serverError FUNCTION_THROW_IF_VALUE_IS_NON_ZERO }
+    max_insert_threads=5;
 system flush logs;
 select peak_threads_usage from system.query_log where
     current_database = currentDatabase() and
@@ -139,20 +144,20 @@ select peak_threads_usage from system.query_log where
     Settings['max_insert_threads'] = '5';
 12
 select count() from testX;
-1190
+1200
 select count() from testXA;
-1130
-select count() from testXB;
-60
+1200
 select count() from testXC;
-1130
-select 'optimize_trivial_insert_select=1', 'max_insert_threads=0';
-optimize_trivial_insert_select=1	max_insert_threads=0
+1200
+select 'optimize_trivial_insert_select=1', 'max_insert_threads=0', 'iteration_num=';
+optimize_trivial_insert_select=1	max_insert_threads=0	iteration_num=
 insert into testX select number from numbers(200) settings
+    send_logs_level='fatal',
+    insert_deduplication_token='UT_7',
     log_queries=1,
     parallel_view_processing=1,
     optimize_trivial_insert_select=1,
-    max_insert_threads=0; -- { serverError FUNCTION_THROW_IF_VALUE_IS_NON_ZERO }
+    max_insert_threads=0;
 system flush logs;
 select peak_threads_usage from system.query_log where
     current_database = currentDatabase() and
@@ -163,20 +168,20 @@ select peak_threads_usage from system.query_log where
     Settings['max_insert_threads'] = '0';
 2
 select count() from testX;
-1390
+1400
 select count() from testXA;
-1330
-select count() from testXB;
-60
+1400
 select count() from testXC;
-1330
-select 'optimize_trivial_insert_select=1', 'max_insert_threads=5';
-optimize_trivial_insert_select=1	max_insert_threads=5
+1400
+select 'optimize_trivial_insert_select=1', 'max_insert_threads=5', 'iteration_num=';
+optimize_trivial_insert_select=1	max_insert_threads=5	iteration_num=
 insert into testX select number from numbers(200) settings
+    send_logs_level='fatal',
+    insert_deduplication_token='UT_8',
     log_queries=1,
     parallel_view_processing=1,
     optimize_trivial_insert_select=1,
-    max_insert_threads=5; -- { serverError FUNCTION_THROW_IF_VALUE_IS_NON_ZERO }
+    max_insert_threads=5;
 system flush logs;
 select peak_threads_usage from system.query_log where
     current_database = currentDatabase() and
@@ -187,10 +192,8 @@ select peak_threads_usage from system.query_log where
     Settings['max_insert_threads'] = '5';
 7
 select count() from testX;
-1590
+1600
 select count() from testXA;
-1480
-select count() from testXB;
-160
+1600
 select count() from testXC;
-1490
+1600
diff --git a/tests/queries/0_stateless/01275_parallel_mv.sql.j2 b/tests/queries/0_stateless/01275_parallel_mv.sql.j2
index 4e45c68b5ad..6805b8902ec 100644
--- a/tests/queries/0_stateless/01275_parallel_mv.sql.j2
+++ b/tests/queries/0_stateless/01275_parallel_mv.sql.j2
@@ -5,11 +5,12 @@
 -- avoid settings randomization by clickhouse-test
 set max_threads = 10;
 
-
 -- more blocks to process
 set max_block_size = 10;
 set min_insert_block_size_rows = 10;
 
+set materialized_views_ignore_errors = 1;
+
 drop table if exists testX;
 drop table if exists testXA;
 drop table if exists testXB;
@@ -21,17 +22,25 @@ create materialized view testXA engine=MergeTree order by tuple() as select slee
 create materialized view testXB engine=MergeTree order by tuple() as select sleep(0.2), throwIf(A=1) from testX;
 create materialized view testXC engine=MergeTree order by tuple() as select sleep(0.1) from testX;
 
+
+{% set count = namespace(value=0) %}
+
 -- { echoOn }
 {% for parallel_view_processing in [0, 1] %}
 {% for optimize_trivial_insert_select in [0, 1] %}
 {% for max_insert_threads in [0, 5] %}
-select 'optimize_trivial_insert_select={{ optimize_trivial_insert_select }}', 'max_insert_threads={{ max_insert_threads }}';
+
+{% set count.value = count.value + 1 %}
+
+select 'optimize_trivial_insert_select={{ optimize_trivial_insert_select }}', 'max_insert_threads={{ max_insert_threads }}', 'iteration_num={{ iteration_num }}';
 
 insert into testX select number from numbers(200) settings
+    send_logs_level='fatal',
+    insert_deduplication_token='UT_{{ count.value }}',
     log_queries=1,
     parallel_view_processing={{ parallel_view_processing }},
     optimize_trivial_insert_select={{ optimize_trivial_insert_select }},
-    max_insert_threads={{ max_insert_threads }}; -- { serverError FUNCTION_THROW_IF_VALUE_IS_NON_ZERO }
+    max_insert_threads={{ max_insert_threads }};
 system flush logs;
 select peak_threads_usage from system.query_log where
     current_database = currentDatabase() and
@@ -43,7 +52,6 @@ select peak_threads_usage from system.query_log where
 
 select count() from testX;
 select count() from testXA;
-select count() from testXB;
 select count() from testXC;
 {% endfor %}
 {% endfor %}
diff --git a/tests/result b/tests/result
new file mode 100644
index 00000000000..b76f44f1e6a
--- /dev/null
+++ b/tests/result
@@ -0,0 +1,12 @@
+Using queries from 'queries' directory
+Connecting to ClickHouse server...... OK
+Connected to server 24.7.1.1 @ 246f421f2402799fd11b22a608b4d0d497cb8438 chesema-processor-onCancel
+
+Running 1 stateless tests (MainProcess).
+
+00993_system_parts_race_condition_drop_zookeeper:                       [ OK ]
+
+1 tests passed. 0 tests skipped. 124.59 s elapsed (MainProcess).
+
+0 tests passed. 0 tests skipped. 0.00 s elapsed (MainProcess).
+All tests have finished.

From 92b46616005500a2674cdf4719a5476f341c9289 Mon Sep 17 00:00:00 2001
From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com>
Date: Thu, 11 Jul 2024 18:58:53 +0200
Subject: [PATCH 0202/1488] fix

---
 .../AggregateFunctionGroupArrayIntersect.cpp  | 34 +++++++++++--------
 1 file changed, 19 insertions(+), 15 deletions(-)

diff --git a/src/AggregateFunctions/AggregateFunctionGroupArrayIntersect.cpp b/src/AggregateFunctions/AggregateFunctionGroupArrayIntersect.cpp
index 4f45d532179..20acda213da 100644
--- a/src/AggregateFunctions/AggregateFunctionGroupArrayIntersect.cpp
+++ b/src/AggregateFunctions/AggregateFunctionGroupArrayIntersect.cpp
@@ -115,8 +115,10 @@ public:
         {
             for (auto & rhs_elem : rhs_set)
                 set.insert(rhs_elem.getValue());
+            return;
         }
-        else if (!set.empty())
+
+        if (!set.empty())
         {
             auto create_new_set = [](auto & lhs_val, auto & rhs_val)
             {
@@ -278,28 +280,30 @@ public:
             return;
 
         UInt64 version = this->data(place).version++;
-        if (version == 1 && set.empty())
+        if (version == 0)
         {
             bool inserted;
             State::Set::LookupResult it;
             for (auto & rhs_elem : rhs_value)
-                set.emplace(ArenaKeyHolder{rhs_elem.getValue(), *arena}, it, inserted);
-        }
-        else
-        {
-            auto create_matched_set = [](auto & lhs_val, auto & rhs_val)
             {
-                typename State::Set new_set;
-                for (const auto & lhs_elem : lhs_val)
+                set.emplace(ArenaKeyHolder{rhs_elem.getValue(), *arena}, it, inserted);
+            }
+        }
+        else if (!set.empty())
+        {
+            auto create_new_map = [](auto & lhs_val, auto & rhs_val)
+            {
+                typename State::Set new_map;
+                for (auto & lhs_elem : lhs_val)
                 {
-                    auto is_match = rhs_val.find(lhs_elem.getValue());
-                    if (is_match != nullptr)
-                        new_set.insert(lhs_elem.getValue());
+                    auto val = rhs_val.find(lhs_elem.getValue());
+                    if (val != nullptr)
+                        new_map.insert(lhs_elem.getValue());
                 }
-                return new_set;
+                return new_map;
             };
-            auto matched_set = rhs_value.size() < set.size() ? create_matched_set(rhs_value, set) : create_matched_set(set, rhs_value);
-            set = std::move(matched_set);
+            auto new_map = create_new_map(set, rhs_value);
+            set = std::move(new_map);
         }
     }
 

From 67b7cd40d42f306fe3b8ed997bded7678c9c4e78 Mon Sep 17 00:00:00 2001
From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com>
Date: Thu, 11 Jul 2024 21:28:56 +0200
Subject: [PATCH 0203/1488] Update 03198_group_array_intersect.sql

---
 tests/queries/0_stateless/03198_group_array_intersect.sql | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/queries/0_stateless/03198_group_array_intersect.sql b/tests/queries/0_stateless/03198_group_array_intersect.sql
index 44422fd0ba6..06ab0fbd819 100644
--- a/tests/queries/0_stateless/03198_group_array_intersect.sql
+++ b/tests/queries/0_stateless/03198_group_array_intersect.sql
@@ -1,4 +1,5 @@
 DROP TABLE IF EXISTS test_numbers__fuzz_29;
+SET max_threads=1, max_insert_threads=1;
 CREATE TABLE test_numbers__fuzz_29 (`a` Array(Nullable(FixedString(19)))) ENGINE = MergeTree ORDER BY a SETTINGS allow_nullable_key=1;
 
 INSERT INTO test_numbers__fuzz_29 VALUES ([1,2,3,4,5,6]);

From fa2270fd59c0de91d45ac50f2064d31f568935e4 Mon Sep 17 00:00:00 2001
From: Nikolay Degterinsky <evillique@gmail.com>
Date: Thu, 11 Jul 2024 22:35:39 +0000
Subject: [PATCH 0204/1488] Fix VALID UNTIL clause resetting after restart

---
 .../Access/InterpreterCreateUserQuery.cpp     | 17 ++++++++++++++-
 .../integration/test_user_valid_until/test.py | 21 ++++++++++++++++++-
 2 files changed, 36 insertions(+), 2 deletions(-)

diff --git a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp
index 32c51b745c7..a09de7e688a 100644
--- a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp
+++ b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp
@@ -259,7 +259,22 @@ void InterpreterCreateUserQuery::updateUserFromQuery(User & user, const ASTCreat
     if (query.auth_data)
         auth_data = AuthenticationData::fromAST(*query.auth_data, {}, !query.attach);
 
-    updateUserFromQueryImpl(user, query, auth_data, {}, {}, {}, {}, {}, allow_no_password, allow_plaintext_password, true);
+    std::optional<time_t> valid_until;
+    if (query.valid_until)
+    {
+        const String valid_until_str = checkAndGetLiteralArgument<String>(query.valid_until, "valid_until");
+        time_t time = 0;
+
+        if (valid_until_str != "infinity")
+        {
+            ReadBufferFromString in(valid_until_str);
+            readDateTimeText(time, in);
+        }
+
+        valid_until = time;
+    }
+
+    updateUserFromQueryImpl(user, query, auth_data, {}, {}, {}, {}, valid_until, allow_no_password, allow_plaintext_password, true);
 }
 
 void registerInterpreterCreateUserQuery(InterpreterFactory & factory)
diff --git a/tests/integration/test_user_valid_until/test.py b/tests/integration/test_user_valid_until/test.py
index d6d5bf8b18e..39ca5997067 100644
--- a/tests/integration/test_user_valid_until/test.py
+++ b/tests/integration/test_user_valid_until/test.py
@@ -5,7 +5,7 @@ from time import sleep
 from helpers.cluster import ClickHouseCluster
 
 cluster = ClickHouseCluster(__file__)
-node = cluster.add_instance("node")
+node = cluster.add_instance("node", stay_alive=True)
 
 
 @pytest.fixture(scope="module")
@@ -84,3 +84,22 @@ def test_details(started_cluster):
         node.query("SHOW CREATE USER user_details_time_only")
         == f"CREATE USER user_details_time_only VALID UNTIL \\'{until_year}-01-01 22:03:40\\'\n"
     )
+
+
+def test_restart(started_cluster):
+    node.query("CREATE USER user_restart VALID UNTIL '06/11/2010 08:03:20 Z+3'")
+
+    assert (
+        node.query("SHOW CREATE USER user_restart")
+        == "CREATE USER user_restart VALID UNTIL \\'2010-11-06 05:03:20\\'\n"
+    )
+
+    node.restart_clickhouse()
+
+    assert (
+        node.query("SHOW CREATE USER user_restart")
+        == "CREATE USER user_restart VALID UNTIL \\'2010-11-06 05:03:20\\'\n"
+    )
+
+    error = "Authentication failed"
+    assert error in node.query_and_get_error("SELECT 1", user="user_restart")

From f0ac0eccb16146303a4b520291c5039d86d700d2 Mon Sep 17 00:00:00 2001
From: JackyWoo <wuchienchao@qq.com>
Date: Fri, 12 Jul 2024 16:37:52 +0800
Subject: [PATCH 0205/1488] Add unit test

---
 src/Interpreters/convertFieldToType.cpp       |  2 +-
 src/Storages/Statistics/Statistics.cpp        | 16 +++---
 src/Storages/Statistics/Statistics.h          |  6 +--
 .../Statistics/StatisticsCountMinSketch.cpp   | 44 +++++++--------
 src/Storages/Statistics/tests/gtest_stats.cpp | 53 ++++++++++++++++++-
 5 files changed, 83 insertions(+), 38 deletions(-)

diff --git a/src/Interpreters/convertFieldToType.cpp b/src/Interpreters/convertFieldToType.cpp
index 184c263dbdb..1a40b780e9a 100644
--- a/src/Interpreters/convertFieldToType.cpp
+++ b/src/Interpreters/convertFieldToType.cpp
@@ -545,7 +545,7 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const ID
         catch (Exception & e)
         {
             if (e.code() == ErrorCodes::UNEXPECTED_DATA_AFTER_PARSED_VALUE)
-                throw Exception(ErrorCodes::TYPE_MISMATCH, "Cannot convert string {} to type {}", src.get<String>(), type.getName());
+                throw Exception(ErrorCodes::TYPE_MISMATCH, "Cannot convert string '{}' to type {}", src.get<String>(), type.getName());
 
             e.addMessage(fmt::format("while converting '{}' to {}", src.get<String>(), type.getName()));
             throw;
diff --git a/src/Storages/Statistics/Statistics.cpp b/src/Storages/Statistics/Statistics.cpp
index 5663c55b263..5c0e5f178e1 100644
--- a/src/Storages/Statistics/Statistics.cpp
+++ b/src/Storages/Statistics/Statistics.cpp
@@ -27,25 +27,25 @@ enum StatisticsFileVersion : UInt16
     V0 = 0,
 };
 
-std::optional<Float64> StatisticsUtils::tryConvertToFloat64(const Field & f)
+std::optional<Float64> StatisticsUtils::tryConvertToFloat64(const Field & field)
 {
-    switch (f.getType())
+    switch (field.getType())
     {
         case Field::Types::Int64:
-            return f.get<Int64>();
+            return field.get<Int64>();
         case Field::Types::UInt64:
-            return f.get<UInt64>();
+            return field.get<UInt64>();
         case Field::Types::Float64:
-            return f.get<Float64>();
+            return field.get<Float64>();
         default:
             return {};
     }
 }
 
-std::optional<String> StatisticsUtils::tryConvertToString(const DB::Field & f)
+std::optional<String> StatisticsUtils::tryConvertToString(const DB::Field & field)
 {
-    if (f.getType() == Field::Types::String)
-        return f.get<String>();
+    if (field.getType() == Field::Types::String)
+        return field.get<String>();
     return {};
 }
 
diff --git a/src/Storages/Statistics/Statistics.h b/src/Storages/Statistics/Statistics.h
index 0df5359adfc..33a5cbac4de 100644
--- a/src/Storages/Statistics/Statistics.h
+++ b/src/Storages/Statistics/Statistics.h
@@ -15,11 +15,11 @@ constexpr auto STATS_FILE_PREFIX = "statistics_";
 constexpr auto STATS_FILE_SUFFIX = ".stats";
 
 
-///Returns std::nullopt if input Field cannot be converted to a concrete value
 struct StatisticsUtils
 {
-    static std::optional<Float64> tryConvertToFloat64(const Field & f);
-    static std::optional<String> tryConvertToString(const Field & f);
+    /// Returns std::nullopt if input Field cannot be converted to a concrete value
+    static std::optional<Float64> tryConvertToFloat64(const Field & field);
+    static std::optional<String> tryConvertToString(const Field & field);
 };
 
 /// Statistics describe properties of the values in the column,
diff --git a/src/Storages/Statistics/StatisticsCountMinSketch.cpp b/src/Storages/Statistics/StatisticsCountMinSketch.cpp
index ff985b06ee3..a3c6ee8a819 100644
--- a/src/Storages/Statistics/StatisticsCountMinSketch.cpp
+++ b/src/Storages/Statistics/StatisticsCountMinSketch.cpp
@@ -4,7 +4,7 @@
 #include <IO/WriteHelpers.h>
 #include <Storages/Statistics/StatisticsCountMinSketch.h>
 #include <Interpreters/convertFieldToType.h>
-#include <DataTypes/FieldToDataType.h>
+
 
 #if USE_DATASKETCHES
 
@@ -13,7 +13,7 @@ namespace DB
 
 namespace ErrorCodes
 {
-extern const int ILLEGAL_TYPE_OF_ARGUMENT;
+extern const int LOGICAL_ERROR;
 }
 
 /// Constants chosen based on rolling dices, which provides an error tolerance of 0.1% (ε = 0.001) and a confidence level of 99.9% (δ = 0.001).
@@ -28,36 +28,32 @@ StatisticsCountMinSketch::StatisticsCountMinSketch(const SingleStatisticsDescrip
 
 Float64 StatisticsCountMinSketch::estimateEqual(const Field & val) const
 {
-    if (data_type->isValueRepresentedByNumber())
+    /// Try to convert field to data_type. Converting string to proper data types such as: number, date, datetime, IPv4, Decimal etc.
+    /// Return null if val larger than the range of data_type
+    ///
+    /// For example: if data_type is Int32:
+    ///     1. For 1.0, 1, '1', return Field(1)
+    ///     2. For 1.1, max_value_int64, return null
+    Field val_converted;
+    try
     {
-        /// 'val' maybe number or string, method 'convertFieldToType' will
-        ///     1. convert string to number, date, datetime, IPv4, Decimal etc
-        ///     2. return null if val larger than the range of data_type
-        auto val_converted = convertFieldToType(val, *data_type);
+        val_converted = convertFieldToType(val, *data_type);
         if (val_converted.isNull())
             return 0;
-
-        /// We will get the proper data type of val_converted, for example, Int8 for 1, Int16 for 257.
-        auto data_type_converted = applyVisitor(FieldToDataType<LeastSupertypeOnError::Null>(), val_converted);
-        DataTypes data_types = {data_type, data_type_converted};
-        auto super_type = tryGetLeastSupertype(data_types);
-
-        /// If data_type is UInt8 but val_typed is UInt16, we should return 0.
-        if (!super_type->equals(*data_type))
-            return 0;
-
-        return sketch.get_estimate(&val_converted, data_type->getSizeOfValueInMemory());
     }
+    catch (...)
+    {
+        /// If the conversion fails for example, when converting 'not a number' to Int32, return 0
+        return 0;
+    }
+
+    if (data_type->isValueRepresentedByNumber())
+        return sketch.get_estimate(&val_converted, data_type->getSizeOfValueInMemory());
 
     if (isStringOrFixedString(data_type))
-    {
         return sketch.get_estimate(val.get<String>());
-    }
 
-    throw Exception(
-        ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
-        "Statistics 'count_min' does not support estimate constant value of type {}",
-        val.getTypeName());
+    throw Exception(ErrorCodes::LOGICAL_ERROR, "Statistics 'count_min' does not support estimate data type of {}", data_type->getName());
 }
 
 void StatisticsCountMinSketch::update(const ColumnPtr & column)
diff --git a/src/Storages/Statistics/tests/gtest_stats.cpp b/src/Storages/Statistics/tests/gtest_stats.cpp
index c3c14632ba1..d80f64b8b6b 100644
--- a/src/Storages/Statistics/tests/gtest_stats.cpp
+++ b/src/Storages/Statistics/tests/gtest_stats.cpp
@@ -1,6 +1,10 @@
 #include <gtest/gtest.h>
 
 #include <Storages/Statistics/StatisticsTDigest.h>
+#include <Interpreters/convertFieldToType.h>
+#include <DataTypes/DataTypeFactory.h>
+
+using namespace DB;
 
 TEST(Statistics, TDigestLessThan)
 {
@@ -39,6 +43,51 @@ TEST(Statistics, TDigestLessThan)
 
     std::reverse(data.begin(), data.end());
     test_less_than(data, {-1, 1e9, 50000.0, 3000.0, 30.0}, {0, 100000, 50000, 3000, 30}, {0, 0, 0.001, 0.001, 0.001});
-
-
+}
+
+using Fields = std::vector<Field>;
+
+template <typename T>
+void testConvertFieldToDataType(const DataTypePtr & data_type, const Fields & fields, const T & expected_value, bool convert_failed = false)
+{
+    for (const auto & field : fields)
+    {
+        Field converted_value;
+        try
+        {
+            converted_value = convertFieldToType(field, *data_type);
+        }
+        catch(...)
+        {
+            /// Just ignore exceptions
+        }
+        if (convert_failed)
+            ASSERT_TRUE(converted_value.isNull());
+        else
+            ASSERT_EQ(converted_value.template get<T>(), expected_value);
+    }
+}
+
+TEST(Statistics, convertFieldToType)
+{
+    Fields fields;
+
+    auto data_type_int8 = DataTypeFactory::instance().get("Int8");
+    fields = {1, 1.0, "1"};
+    testConvertFieldToDataType(data_type_int8, fields, static_cast<Int8>(1));
+
+    fields = {256, 1.1, "not a number"};
+    testConvertFieldToDataType(data_type_int8, fields, static_cast<Int8>(1), true);
+
+    auto data_type_float64 = DataTypeFactory::instance().get("Float64");
+    fields = {1, 1.0, "1.0"};
+    testConvertFieldToDataType(data_type_float64, fields, static_cast<Float64>(1.0));
+
+    auto data_type_string = DataTypeFactory::instance().get("String");
+    fields = {1, "1"};
+    testConvertFieldToDataType(data_type_string, fields, static_cast<String>("1"));
+
+    auto data_type_date = DataTypeFactory::instance().get("Date");
+    fields = {"2024-07-12", 19916};
+    testConvertFieldToDataType(data_type_date, fields, static_cast<UInt64>(19916));
 }

From 705134413ffd321a59c1c2c82af11a5edddc962c Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Fri, 12 Jul 2024 12:00:39 +0200
Subject: [PATCH 0206/1488] Update docs/en/interfaces/cli.md

---
 docs/en/interfaces/cli.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/en/interfaces/cli.md b/docs/en/interfaces/cli.md
index 63b7353d092..849b3d4b486 100644
--- a/docs/en/interfaces/cli.md
+++ b/docs/en/interfaces/cli.md
@@ -185,7 +185,7 @@ You can pass parameters to `clickhouse-client` (all parameters have a default va
 - `--format, -f` – Use the specified default format to output the result.
 - `--vertical, -E` – If specified, use the [Vertical format](../interfaces/formats.md#vertical) by default to output the result. This is the same as `–format=Vertical`. In this format, each value is printed on a separate line, which is helpful when displaying wide tables.
 - `--time, -t` – If specified, print the query execution time to ‘stderr’ in non-interactive mode.
-- `--memory-usage` – If specified, print the query memory usage to ‘stderr’ in non-interactive mode.
+- `--memory-usage` – If specified, print memory usage to ‘stderr’ in non-interactive mode]. Possible values: 'none' - do not print memory usage, 'default' - print number of bytes, 'readable' - print memory usage in human-readable format.
 - `--stacktrace` – If specified, also print the stack trace if an exception occurs.
 - `--config-file` – The name of the configuration file.
 - `--secure` – If specified, will connect to server over secure connection (TLS). You might need to configure your CA certificates in the [configuration file](#configuration_files). The available configuration settings are the same as for [server-side TLS configuration](../operations/server-configuration-parameters/settings.md#server_configuration_parameters-openssl).

From f7fc031d9d1d63a057e5597088f1b70bc81f38a6 Mon Sep 17 00:00:00 2001
From: Konstantin Morozov <just.morozov.k@gmail.com>
Date: Fri, 12 Jul 2024 10:04:10 +0000
Subject: [PATCH 0207/1488] fix data race with permanently_detached_tables

---
 src/Databases/DatabaseOrdinary.cpp | 2 +-
 src/Databases/DatabaseOrdinary.h   | 8 ++++++--
 src/Interpreters/loadMetadata.cpp  | 2 +-
 3 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp
index 98ef4b26ae2..db812e89782 100644
--- a/src/Databases/DatabaseOrdinary.cpp
+++ b/src/Databases/DatabaseOrdinary.cpp
@@ -224,10 +224,10 @@ void DatabaseOrdinary::loadTablesMetadata(ContextPtr local_context, ParsedTables
                 if (fs::exists(full_path.string() + detached_suffix))
                 {
                     const std::string table_name = unescapeForFileName(file_name.substr(0, file_name.size() - 4));
-                    permanently_detached_tables.push_back(table_name);
                     LOG_DEBUG(log, "Skipping permanently detached table {}.", backQuote(table_name));
 
                     std::lock_guard lock(mutex);
+                    permanently_detached_tables.push_back(table_name);
 
                     const auto detached_table_name = create_query->getTable();
 
diff --git a/src/Databases/DatabaseOrdinary.h b/src/Databases/DatabaseOrdinary.h
index d2891147922..c2c5775e5ab 100644
--- a/src/Databases/DatabaseOrdinary.h
+++ b/src/Databases/DatabaseOrdinary.h
@@ -67,7 +67,11 @@ public:
         const StorageID & table_id,
         const StorageInMemoryMetadata & metadata) override;
 
-    Strings getNamesOfPermanentlyDetachedTables() const override { return permanently_detached_tables; }
+    Strings getNamesOfPermanentlyDetachedTables() const override
+    {
+        std::lock_guard lock(mutex);
+        return permanently_detached_tables;
+    }
 
 protected:
     virtual void commitAlterTable(
@@ -77,7 +81,7 @@ protected:
         const String & statement,
         ContextPtr query_context);
 
-    Strings permanently_detached_tables;
+    Strings permanently_detached_tables TSA_GUARDED_BY(mutex);
 
     std::unordered_map<String, LoadTaskPtr> load_table TSA_GUARDED_BY(mutex);
     std::unordered_map<String, LoadTaskPtr> startup_table TSA_GUARDED_BY(mutex);
diff --git a/src/Interpreters/loadMetadata.cpp b/src/Interpreters/loadMetadata.cpp
index 9c3922b8bda..03dd1714ead 100644
--- a/src/Interpreters/loadMetadata.cpp
+++ b/src/Interpreters/loadMetadata.cpp
@@ -384,7 +384,7 @@ static void maybeConvertOrdinaryDatabaseToAtomic(ContextMutablePtr context, cons
     if (database->getEngineName() != "Ordinary")
         return;
 
-    Strings permanently_detached_tables = database->getNamesOfPermanentlyDetachedTables();
+    const Strings permanently_detached_tables = database->getNamesOfPermanentlyDetachedTables();
     if (!permanently_detached_tables.empty())
     {
         throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot automatically convert database {} from Ordinary to Atomic, "

From 34e54fd089b5c4b4892e80bd426a9409b31f29c9 Mon Sep 17 00:00:00 2001
From: Sema Checherinda <Sema.Checherinda@clickhouse.com>
Date: Fri, 12 Jul 2024 13:58:41 +0200
Subject: [PATCH 0208/1488] fix test
 test_upload_s3_fail_upload_part_when_multi_part_upload

---
 tests/integration/test_checking_s3_blobs_paranoid/test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/test_checking_s3_blobs_paranoid/test.py b/tests/integration/test_checking_s3_blobs_paranoid/test.py
index 1ed70e20b79..22a5cd525f3 100644
--- a/tests/integration/test_checking_s3_blobs_paranoid/test.py
+++ b/tests/integration/test_checking_s3_blobs_paranoid/test.py
@@ -197,7 +197,7 @@ def test_upload_s3_fail_upload_part_when_multi_part_upload(
     )
     assert create_multipart == 1
     assert upload_parts >= 2
-    assert s3_errors >= 2
+    assert s3_errors == 1
 
 
 @pytest.mark.parametrize(

From c6b558c7915b070167649d4e88eafb2613570bd3 Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <nikitamikhaylov@clickhouse.com>
Date: Thu, 4 Jul 2024 22:30:18 +0200
Subject: [PATCH 0209/1488] Done

---
 .../02814_currentDatabase_for_table_functions.sql         | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/02814_currentDatabase_for_table_functions.sql b/tests/queries/0_stateless/02814_currentDatabase_for_table_functions.sql
index 74b5cf5f432..8b1e3ba1e10 100644
--- a/tests/queries/0_stateless/02814_currentDatabase_for_table_functions.sql
+++ b/tests/queries/0_stateless/02814_currentDatabase_for_table_functions.sql
@@ -13,7 +13,13 @@ CREATE MATERIALIZED VIEW null_mv Engine = Log AS SELECT * FROM null_table LEFT J
 
 CREATE TABLE null_table_buffer (number UInt64) ENGINE = Buffer(currentDatabase(), null_table, 1, 1, 1, 100, 200, 10000, 20000);
 INSERT INTO null_table_buffer VALUES (1);
-SELECT sleep(3) FORMAT Null;
+
+-- OPTIMIZE query should flush Buffer table, but still it is not guaranteed
+-- (see the comment StorageBuffer::optimize)
+-- But the combination of OPTIMIZE + sleep + OPTIMIZE should be enough.
+OPTIMIZE TABLE null_table_buffer;
+SELECT sleep(1) FORMAT Null;
+OPTIMIZE TABLE null_table_buffer;
 
 -- Insert about should've landed into `null_mv`
 SELECT count() FROM null_mv;

From ca7e003c6d7af6bf0676bba7cb61ab560c202bf3 Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <nikitamikhaylov@clickhouse.com>
Date: Fri, 12 Jul 2024 15:27:03 +0000
Subject: [PATCH 0210/1488] Fixed test

---
 .../02814_currentDatabase_for_table_functions.reference    | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/02814_currentDatabase_for_table_functions.reference b/tests/queries/0_stateless/02814_currentDatabase_for_table_functions.reference
index 7ff95106d3d..20b14d9a67b 100644
--- a/tests/queries/0_stateless/02814_currentDatabase_for_table_functions.reference
+++ b/tests/queries/0_stateless/02814_currentDatabase_for_table_functions.reference
@@ -11,7 +11,12 @@ CREATE VIEW number_view as SELECT * FROM numbers(10) as tb;
 CREATE MATERIALIZED VIEW null_mv Engine = Log AS SELECT * FROM null_table LEFT JOIN number_view as tb USING number;
 CREATE TABLE null_table_buffer (number UInt64) ENGINE = Buffer(currentDatabase(), null_table, 1, 1, 1, 100, 200, 10000, 20000);
 INSERT INTO null_table_buffer VALUES (1);
-SELECT sleep(3) FORMAT Null;
+-- OPTIMIZE query should flush Buffer table, but still it is not guaranteed
+-- (see the comment StorageBuffer::optimize)
+-- But the combination of OPTIMIZE + sleep + OPTIMIZE should be enough.
+OPTIMIZE TABLE null_table_buffer;
+SELECT sleep(1) FORMAT Null;
+OPTIMIZE TABLE null_table_buffer;
 -- Insert about should've landed into `null_mv`
 SELECT count() FROM null_mv;
 1

From ea64527ea1e88160b69f9d4d552421af07fc0d32 Mon Sep 17 00:00:00 2001
From: Alexander Tokmakov <tavplubix@clickhouse.com>
Date: Fri, 12 Jul 2024 19:18:16 +0200
Subject: [PATCH 0211/1488] suppress broken test

---
 src/Client/ClientBase.cpp                     |  6 +++---
 tests/clickhouse-test                         |  2 +-
 tests/queries/0_stateless/02895_npy_format.sh | 15 +++++++++------
 3 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp
index b72fa0edf1e..b29eebd4fde 100644
--- a/src/Client/ClientBase.cpp
+++ b/src/Client/ClientBase.cpp
@@ -3069,7 +3069,7 @@ void ClientBase::init(int argc, char ** argv)
 
         ("fuzzer-args", po::value<std::string>(), "Command line arguments for the LLVM's libFuzzer driver. Only relevant if the application is compiled with libFuzzer.")
 
-        ("client_log_file", po::value<std::string>(), "Path to a file for logging fatal errors in client")
+        ("client_logs_file", po::value<std::string>(), "Path to a file for writing client logs. Currently we only have fatal logs (when the client crashes)")
     ;
 
     addOptions(options_description);
@@ -3236,9 +3236,9 @@ void ClientBase::init(int argc, char ** argv)
     fatal_channel_ptr = new Poco::SplitterChannel;
     fatal_console_channel_ptr = new Poco::ConsoleChannel;
     fatal_channel_ptr->addChannel(fatal_console_channel_ptr);
-    if (options.count("client_log_file"))
+    if (options.count("client_logs_file"))
     {
-        fatal_file_channel_ptr = new Poco::SimpleFileChannel(options["client_log_file"].as<std::string>());
+        fatal_file_channel_ptr = new Poco::SimpleFileChannel(options["client_logs_file"].as<std::string>());
         fatal_channel_ptr->addChannel(fatal_file_channel_ptr);
     }
 
diff --git a/tests/clickhouse-test b/tests/clickhouse-test
index 1fcc9d111dd..cca0e2950b0 100755
--- a/tests/clickhouse-test
+++ b/tests/clickhouse-test
@@ -1526,7 +1526,7 @@ class TestCase:
         database = args.testcase_database
 
         if args.client_log:
-            log_opt = " --client_log_file=" + args.client_log
+            log_opt = " --client_logs_file=" + args.client_log + " "
             client_options += log_opt
             os.environ["CLICKHOUSE_CLIENT_OPT"] = (
                 os.environ["CLICKHOUSE_CLIENT_OPT"]
diff --git a/tests/queries/0_stateless/02895_npy_format.sh b/tests/queries/0_stateless/02895_npy_format.sh
index 9d05303a091..a655f275179 100755
--- a/tests/queries/0_stateless/02895_npy_format.sh
+++ b/tests/queries/0_stateless/02895_npy_format.sh
@@ -51,13 +51,16 @@ $CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/one_dim_str.npy', Npy
 $CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/two_dim.npy', Npy, 'value Array(Int8)')"
 $CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/three_dim.npy', Npy, 'value Array(Array(Int8))')"
 
-$CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/one_dim_float.npy', Npy, 'value Array(Float32)')" 2>&1 | grep -c "BAD_ARGUMENTS"
-$CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/one_dim_float.npy', Npy, 'value UUID')" 2>&1 | grep -c "BAD_ARGUMENTS"
-$CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/one_dim_float.npy', Npy, 'value Tuple(UInt8)')" 2>&1 | grep -c "BAD_ARGUMENTS"
+# The test is wrong: https://github.com/ClickHouse/ClickHouse/issues/66458
+echo -e "0\n0\n0"
+#$CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/one_dim_float.npy', Npy, 'value Array(Float32)')" 2>&1 | grep -c "BAD_ARGUMENTS"
+#$CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/one_dim_float.npy', Npy, 'value UUID')" 2>&1 | grep -c "BAD_ARGUMENTS"
+#$CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/one_dim_float.npy', Npy, 'value Tuple(UInt8)')" 2>&1 | grep -c "BAD_ARGUMENTS"
 
-$CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/one_dim_float.npy', Npy, 'value Int8')" 2>&1 | grep -c "BAD_ARGUMENTS"
-$CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/one_dim_str.npy', Npy, 'value Int8')" 2>&1 | grep -c "BAD_ARGUMENTS"
-$CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/one_dim_unicode.npy', Npy, 'value Float32')" 2>&1 | grep -c "BAD_ARGUMENTS"
+echo -e "0\n0\n0"
+#$CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/one_dim_float.npy', Npy, 'value Int8')" 2>&1 | grep -c "BAD_ARGUMENTS"
+#$CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/one_dim_str.npy', Npy, 'value Int8')" 2>&1 | grep -c "BAD_ARGUMENTS"
+#$CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/one_dim_unicode.npy', Npy, 'value Float32')" 2>&1 | grep -c "BAD_ARGUMENTS"
 
 $CLICKHOUSE_LOCAL -q "select * from file('$CURDIR/data_npy/complex.npy')" 2>&1 | grep -c "BAD_ARGUMENTS"
 

From a6e737ef2afc7fb18d661295e6f84cc3e0478ae1 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Fri, 12 Jul 2024 17:19:30 +0000
Subject: [PATCH 0212/1488] Cleaner FilterDAGInfo.

---
 src/Interpreters/ActionsDAG.cpp             | 25 +++++++-------
 src/Interpreters/ActionsDAG.h               |  2 +-
 src/Interpreters/ExpressionActions.cpp      |  2 +-
 src/Interpreters/ExpressionAnalyzer.cpp     |  2 +-
 src/Interpreters/InterpreterSelectQuery.cpp | 18 +++++------
 src/Planner/Planner.cpp                     |  4 +--
 src/Planner/PlannerJoinTree.cpp             | 36 +++++++++------------
 src/Storages/IStorage.cpp                   |  6 ++--
 src/Storages/MergeTree/MergeTreeData.cpp    |  2 +-
 src/Storages/SelectQueryInfo.h              |  2 +-
 src/Storages/StorageMerge.cpp               |  6 ++--
 11 files changed, 49 insertions(+), 56 deletions(-)

diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp
index 04be9d23c32..4401c83549f 100644
--- a/src/Interpreters/ActionsDAG.cpp
+++ b/src/Interpreters/ActionsDAG.cpp
@@ -1249,31 +1249,30 @@ bool ActionsDAG::removeUnusedResult(const std::string & column_name)
 ActionsDAGPtr ActionsDAG::clone(const ActionsDAG * from)
 {
     std::unordered_map<const Node *, Node *> old_to_new_nodes;
-    return ActionsDAG::clone(from, old_to_new_nodes);
+    if (from == nullptr)
+        return nullptr;
+    return std::make_unique<ActionsDAG>(ActionsDAG::clone(*from, old_to_new_nodes));
 }
 
-ActionsDAGPtr ActionsDAG::clone(const ActionsDAG * from, std::unordered_map<const Node *, Node *> & old_to_new_nodes)
+ActionsDAG ActionsDAG::clone(const ActionsDAG & from, std::unordered_map<const Node *, Node *> & old_to_new_nodes)
 {
-    if (!from)
-        return nullptr;
+    ActionsDAG actions;
 
-    auto actions = std::make_unique<ActionsDAG>();
-
-    for (const auto & node : from->nodes)
+    for (const auto & node : from.nodes)
     {
-        auto & copy_node = actions->nodes.emplace_back(node);
+        auto & copy_node = actions.nodes.emplace_back(node);
         old_to_new_nodes[&node] = &copy_node;
     }
 
-    for (auto & node : actions->nodes)
+    for (auto & node : actions.nodes)
         for (auto & child : node.children)
             child = old_to_new_nodes[child];
 
-    for (const auto & output_node : from->outputs)
-        actions->outputs.push_back(old_to_new_nodes[output_node]);
+    for (const auto & output_node : from.outputs)
+        actions.outputs.push_back(old_to_new_nodes[output_node]);
 
-    for (const auto & input_node : from->inputs)
-        actions->inputs.push_back(old_to_new_nodes[input_node]);
+    for (const auto & input_node : from.inputs)
+        actions.inputs.push_back(old_to_new_nodes[input_node]);
 
     return actions;
 }
diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h
index cf6a91b9fe7..f428ca2f01c 100644
--- a/src/Interpreters/ActionsDAG.h
+++ b/src/Interpreters/ActionsDAG.h
@@ -263,7 +263,7 @@ public:
 
     static ActionsDAGPtr clone(const ActionsDAGPtr & from) { return clone(from.get()); }
     static ActionsDAGPtr clone(const ActionsDAG * from);
-    static ActionsDAGPtr clone(const ActionsDAG * from, std::unordered_map<const Node *, Node *> & old_to_new_nodes);
+    static ActionsDAG clone(const ActionsDAG & from, std::unordered_map<const Node *, Node *> & old_to_new_nodes);
 
     static ActionsDAGPtr cloneSubDAG(const NodeRawConstPtrs & outputs, bool remove_aliases);
 
diff --git a/src/Interpreters/ExpressionActions.cpp b/src/Interpreters/ExpressionActions.cpp
index 1c6c3f2556b..dd1d2eb703e 100644
--- a/src/Interpreters/ExpressionActions.cpp
+++ b/src/Interpreters/ExpressionActions.cpp
@@ -75,7 +75,7 @@ ExpressionActionsPtr ExpressionActions::clone() const
     auto copy = std::make_shared<ExpressionActions>(ExpressionActions());
 
     std::unordered_map<const Node *, Node *> copy_map;
-    copy->actions_dag = std::move(*ActionsDAG::clone(&actions_dag, copy_map));
+    copy->actions_dag = ActionsDAG::clone(actions_dag, copy_map);
     copy->actions = actions;
     for (auto & action : copy->actions)
         action.node = copy_map[action.node];
diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp
index 068b6f290fa..286eda14b3f 100644
--- a/src/Interpreters/ExpressionAnalyzer.cpp
+++ b/src/Interpreters/ExpressionAnalyzer.cpp
@@ -1922,7 +1922,7 @@ ExpressionAnalysisResult::ExpressionAnalysisResult(
 
         if (storage && additional_filter)
         {
-            Names columns_for_additional_filter = additional_filter->actions->getRequiredColumnsNames();
+            Names columns_for_additional_filter = additional_filter->actions.getRequiredColumnsNames();
             additional_required_columns_after_prewhere.insert(additional_required_columns_after_prewhere.end(),
                 columns_for_additional_filter.begin(), columns_for_additional_filter.end());
         }
diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp
index cde6e305005..e723e5f7982 100644
--- a/src/Interpreters/InterpreterSelectQuery.cpp
+++ b/src/Interpreters/InterpreterSelectQuery.cpp
@@ -178,12 +178,12 @@ FilterDAGInfoPtr generateFilterActions(
     filter_info->actions = std::move(analyzer.simpleSelectActions()->dag);
 
     filter_info->column_name = expr_list->children.at(0)->getColumnName();
-    filter_info->actions->removeUnusedActions(NameSet{filter_info->column_name});
+    filter_info->actions.removeUnusedActions(NameSet{filter_info->column_name});
 
-    for (const auto * node : filter_info->actions->getInputs())
-        filter_info->actions->getOutputs().push_back(node);
+    for (const auto * node : filter_info->actions.getInputs())
+        filter_info->actions.getOutputs().push_back(node);
 
-    auto required_columns_from_filter = filter_info->actions->getRequiredColumns();
+    auto required_columns_from_filter = filter_info->actions.getRequiredColumns();
 
     for (const auto & column : required_columns_from_filter)
     {
@@ -1486,7 +1486,7 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional<P
         {
             auto row_level_security_step = std::make_unique<FilterStep>(
                 query_plan.getCurrentDataStream(),
-                std::move(*ActionsDAG::clone(&*expressions.filter_info->actions)),
+                std::move(*ActionsDAG::clone(&expressions.filter_info->actions)),
                 expressions.filter_info->column_name,
                 expressions.filter_info->do_remove_column);
 
@@ -1612,7 +1612,7 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional<P
             {
                 auto row_level_security_step = std::make_unique<FilterStep>(
                     query_plan.getCurrentDataStream(),
-                    std::move(*ActionsDAG::clone(&*expressions.filter_info->actions)),
+                    std::move(*ActionsDAG::clone(&expressions.filter_info->actions)),
                     expressions.filter_info->column_name,
                     expressions.filter_info->do_remove_column);
 
@@ -1620,11 +1620,11 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional<P
                 query_plan.addStep(std::move(row_level_security_step));
             }
 
-            const auto add_filter_step = [&](const auto & new_filter_info, const std::string & description)
+            const auto add_filter_step = [&](auto & new_filter_info, const std::string & description)
             {
                 auto filter_step = std::make_unique<FilterStep>(
                     query_plan.getCurrentDataStream(),
-                    std::move(*ActionsDAG::clone(&*new_filter_info->actions)),
+                    std::move(new_filter_info->actions),
                     new_filter_info->column_name,
                     new_filter_info->do_remove_column);
 
@@ -2107,7 +2107,7 @@ void InterpreterSelectQuery::applyFiltersToPrewhereInAnalysis(ExpressionAnalysis
     else
     {
         /// Add row level security actions to prewhere.
-        analysis.prewhere_info->row_level_filter = std::move(*analysis.filter_info->actions);
+        analysis.prewhere_info->row_level_filter = std::move(analysis.filter_info->actions);
         analysis.prewhere_info->row_level_column_name = std::move(analysis.filter_info->column_name);
         analysis.filter_info = nullptr;
     }
diff --git a/src/Planner/Planner.cpp b/src/Planner/Planner.cpp
index 0b10cef82ce..ffed19185d3 100644
--- a/src/Planner/Planner.cpp
+++ b/src/Planner/Planner.cpp
@@ -1128,11 +1128,11 @@ void addAdditionalFilterStepIfNeeded(QueryPlan & query_plan,
     auto fake_table_expression = std::make_shared<TableNode>(std::move(storage), query_context);
 
     auto filter_info = buildFilterInfo(additional_result_filter_ast, fake_table_expression, planner_context, std::move(fake_name_set));
-    if (!filter_info.actions || !query_plan.isInitialized())
+    if (!query_plan.isInitialized())
         return;
 
     auto filter_step = std::make_unique<FilterStep>(query_plan.getCurrentDataStream(),
-        std::move(*filter_info.actions),
+        std::move(filter_info.actions),
         filter_info.column_name,
         filter_info.do_remove_column);
     filter_step->setStepDescription("additional result filter");
diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp
index fa3a3483a8e..3217d3461d3 100644
--- a/src/Planner/PlannerJoinTree.cpp
+++ b/src/Planner/PlannerJoinTree.cpp
@@ -458,7 +458,7 @@ void updatePrewhereOutputsIfNeeded(SelectQueryInfo & table_expression_query_info
     prewhere_outputs.insert(prewhere_outputs.end(), required_output_nodes.begin(), required_output_nodes.end());
 }
 
-FilterDAGInfo buildRowPolicyFilterIfNeeded(const StoragePtr & storage,
+std::optional<FilterDAGInfo> buildRowPolicyFilterIfNeeded(const StoragePtr & storage,
     SelectQueryInfo & table_expression_query_info,
     PlannerContextPtr & planner_context,
     std::set<std::string> & used_row_policies)
@@ -479,7 +479,7 @@ FilterDAGInfo buildRowPolicyFilterIfNeeded(const StoragePtr & storage,
     return buildFilterInfo(row_policy_filter->expression, table_expression_query_info.table_expression, planner_context);
 }
 
-FilterDAGInfo buildCustomKeyFilterIfNeeded(const StoragePtr & storage,
+std::optional<FilterDAGInfo> buildCustomKeyFilterIfNeeded(const StoragePtr & storage,
     SelectQueryInfo & table_expression_query_info,
     PlannerContextPtr & planner_context)
 {
@@ -513,7 +513,7 @@ FilterDAGInfo buildCustomKeyFilterIfNeeded(const StoragePtr & storage,
 }
 
 /// Apply filters from additional_table_filters setting
-FilterDAGInfo buildAdditionalFiltersIfNeeded(const StoragePtr & storage,
+std::optional<FilterDAGInfo> buildAdditionalFiltersIfNeeded(const StoragePtr & storage,
     const String & table_expression_alias,
     SelectQueryInfo & table_expression_query_info,
     PlannerContextPtr & planner_context)
@@ -789,9 +789,6 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres
                 std::vector<std::pair<FilterDAGInfo, std::string>> where_filters;
                 const auto add_filter = [&](FilterDAGInfo & filter_info, std::string description)
                 {
-                    if (!filter_info.actions)
-                        return;
-
                     bool is_final = table_expression_query_info.table_expression_modifiers
                         && table_expression_query_info.table_expression_modifiers->hasFinal();
                     bool optimize_move_to_prewhere
@@ -805,14 +802,14 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres
 
                         if (!prewhere_info->prewhere_actions)
                         {
-                            prewhere_info->prewhere_actions = std::move(*filter_info.actions);
+                            prewhere_info->prewhere_actions = std::move(filter_info.actions);
                             prewhere_info->prewhere_column_name = filter_info.column_name;
                             prewhere_info->remove_prewhere_column = filter_info.do_remove_column;
                             prewhere_info->need_filter = true;
                         }
                         else if (!prewhere_info->row_level_filter)
                         {
-                            prewhere_info->row_level_filter = std::move(*filter_info.actions);
+                            prewhere_info->row_level_filter = std::move(filter_info.actions);
                             prewhere_info->row_level_column_name = filter_info.column_name;
                             prewhere_info->need_filter = true;
                         }
@@ -830,17 +827,18 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres
 
                 auto row_policy_filter_info
                     = buildRowPolicyFilterIfNeeded(storage, table_expression_query_info, planner_context, used_row_policies);
-                if (row_policy_filter_info.actions)
-                    table_expression_data.setRowLevelFilterActions(ActionsDAG::clone(&*row_policy_filter_info.actions));
-                add_filter(row_policy_filter_info, "Row-level security filter");
+                if (row_policy_filter_info)
+                {
+                    table_expression_data.setRowLevelFilterActions(ActionsDAG::clone(&row_policy_filter_info->actions));
+                    add_filter(*row_policy_filter_info, "Row-level security filter");
+                }
 
                 if (query_context->getParallelReplicasMode() == Context::ParallelReplicasMode::CUSTOM_KEY)
                 {
                     if (settings.parallel_replicas_count > 1)
                     {
-                        auto parallel_replicas_custom_key_filter_info
-                            = buildCustomKeyFilterIfNeeded(storage, table_expression_query_info, planner_context);
-                        add_filter(parallel_replicas_custom_key_filter_info, "Parallel replicas custom key filter");
+                        if (auto parallel_replicas_custom_key_filter_info= buildCustomKeyFilterIfNeeded(storage, table_expression_query_info, planner_context))
+                            add_filter(*parallel_replicas_custom_key_filter_info, "Parallel replicas custom key filter");
                     }
                     else if (auto * distributed = typeid_cast<StorageDistributed *>(storage.get());
                              distributed && query_context->canUseParallelReplicasCustomKey(*distributed->getCluster()))
@@ -850,9 +848,8 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres
                 }
 
                 const auto & table_expression_alias = table_expression->getOriginalAlias();
-                auto additional_filters_info
-                    = buildAdditionalFiltersIfNeeded(storage, table_expression_alias, table_expression_query_info, planner_context);
-                add_filter(additional_filters_info, "additional filter");
+                if (auto additional_filters_info = buildAdditionalFiltersIfNeeded(storage, table_expression_alias, table_expression_query_info, planner_context))
+                    add_filter(*additional_filters_info, "additional filter");
 
                 from_stage = storage->getQueryProcessingStage(
                     query_context, select_query_options.to_stage, storage_snapshot, table_expression_query_info);
@@ -967,11 +964,10 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres
                 for (auto && [filter_info, description] : where_filters)
                 {
                     if (query_plan.isInitialized() &&
-                        from_stage == QueryProcessingStage::FetchColumns &&
-                        filter_info.actions)
+                        from_stage == QueryProcessingStage::FetchColumns)
                     {
                         auto filter_step = std::make_unique<FilterStep>(query_plan.getCurrentDataStream(),
-                            std::move(*filter_info.actions),
+                            std::move(filter_info.actions),
                             filter_info.column_name,
                             filter_info.do_remove_column);
                         filter_step->setStepDescription(description);
diff --git a/src/Storages/IStorage.cpp b/src/Storages/IStorage.cpp
index 57f79a2cd7f..4164608b4b5 100644
--- a/src/Storages/IStorage.cpp
+++ b/src/Storages/IStorage.cpp
@@ -340,10 +340,8 @@ std::string FilterDAGInfo::dump() const
     WriteBufferFromOwnString ss;
     ss << "FilterDAGInfo for column '" << column_name <<"', do_remove_column "
        << do_remove_column << "\n";
-    if (actions)
-    {
-        ss << "actions " << actions->dumpDAG() << "\n";
-    }
+
+    ss << "actions " << actions.dumpDAG() << "\n";
 
     return ss.str();
 }
diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index 334c8c9c5ac..88fb52a94f2 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -7058,7 +7058,7 @@ ActionDAGNodes MergeTreeData::getFiltersForPrimaryKeyAnalysis(const InterpreterS
 
     ActionDAGNodes filter_nodes;
     if (auto additional_filter_info = select.getAdditionalQueryInfo())
-        filter_nodes.nodes.push_back(&additional_filter_info->actions->findInOutputs(additional_filter_info->column_name));
+        filter_nodes.nodes.push_back(&additional_filter_info->actions.findInOutputs(additional_filter_info->column_name));
 
     if (before_where)
         filter_nodes.nodes.push_back(&before_where->dag.findInOutputs(where_column_name));
diff --git a/src/Storages/SelectQueryInfo.h b/src/Storages/SelectQueryInfo.h
index 5276870c037..97b36115dfd 100644
--- a/src/Storages/SelectQueryInfo.h
+++ b/src/Storages/SelectQueryInfo.h
@@ -93,7 +93,7 @@ struct FilterInfo
 /// Same as FilterInfo, but with ActionsDAG.
 struct FilterDAGInfo
 {
-    std::optional<ActionsDAG> actions;
+    ActionsDAG actions;
     String column_name;
     bool do_remove_column = false;
 
diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp
index 374abd0b0a5..18e194491b8 100644
--- a/src/Storages/StorageMerge.cpp
+++ b/src/Storages/StorageMerge.cpp
@@ -407,7 +407,7 @@ void ReadFromMerge::addFilter(FilterDAGInfo filter)
 {
     output_stream->header = FilterTransform::transformHeader(
             output_stream->header,
-            filter.actions ? &*filter.actions : nullptr,
+            &filter.actions,
             filter.column_name,
             filter.do_remove_column);
     pushed_down_filters.push_back(std::move(filter));
@@ -662,7 +662,7 @@ std::vector<ReadFromMerge::ChildPlan> ReadFromMerge::createChildrenPlans(SelectQ
             {
                 auto filter_step = std::make_unique<FilterStep>(
                     child.plan.getCurrentDataStream(),
-                    std::move(*ActionsDAG::clone(&*filter_info.actions)),
+                    std::move(*ActionsDAG::clone(&filter_info.actions)),
                     filter_info.column_name,
                     filter_info.do_remove_column);
 
@@ -1565,7 +1565,7 @@ bool ReadFromMerge::requestReadingInOrder(InputOrderInfoPtr order_info_)
 void ReadFromMerge::applyFilters(ActionDAGNodes added_filter_nodes)
 {
     for (const auto & filter_info : pushed_down_filters)
-        added_filter_nodes.nodes.push_back(&filter_info.actions->findInOutputs(filter_info.column_name));
+        added_filter_nodes.nodes.push_back(&filter_info.actions.findInOutputs(filter_info.column_name));
 
     SourceStepWithFilter::applyFilters(added_filter_nodes);
 

From fb7cf4ab93c991b3e2cd8a3e3e1c6cecf574b936 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Fri, 12 Jul 2024 17:46:03 +0000
Subject: [PATCH 0213/1488] Better.

---
 src/Interpreters/ActionsDAG.cpp            | 16 +++++++++++-----
 src/Interpreters/ActionsDAG.h              |  4 +++-
 src/Interpreters/ExpressionActions.cpp     |  2 +-
 src/Planner/CollectTableExpressionData.cpp | 10 +++++-----
 src/Planner/Planner.cpp                    |  4 ++--
 src/Planner/PlannerJoinTree.cpp            |  7 ++++---
 src/Planner/TableExpressionData.h          | 18 +++++++++---------
 7 files changed, 35 insertions(+), 26 deletions(-)

diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp
index 4401c83549f..4f03a9e1602 100644
--- a/src/Interpreters/ActionsDAG.cpp
+++ b/src/Interpreters/ActionsDAG.cpp
@@ -1251,14 +1251,20 @@ ActionsDAGPtr ActionsDAG::clone(const ActionsDAG * from)
     std::unordered_map<const Node *, Node *> old_to_new_nodes;
     if (from == nullptr)
         return nullptr;
-    return std::make_unique<ActionsDAG>(ActionsDAG::clone(*from, old_to_new_nodes));
+    return std::make_unique<ActionsDAG>(from->clone(old_to_new_nodes));
 }
 
-ActionsDAG ActionsDAG::clone(const ActionsDAG & from, std::unordered_map<const Node *, Node *> & old_to_new_nodes)
+ActionsDAG ActionsDAG::clone() const
+{
+    std::unordered_map<const Node *, Node *> old_to_new_nodes;
+    return clone(old_to_new_nodes);
+}
+
+ActionsDAG ActionsDAG::clone(std::unordered_map<const Node *, Node *> & old_to_new_nodes) const
 {
     ActionsDAG actions;
 
-    for (const auto & node : from.nodes)
+    for (const auto & node : nodes)
     {
         auto & copy_node = actions.nodes.emplace_back(node);
         old_to_new_nodes[&node] = &copy_node;
@@ -1268,10 +1274,10 @@ ActionsDAG ActionsDAG::clone(const ActionsDAG & from, std::unordered_map<const N
         for (auto & child : node.children)
             child = old_to_new_nodes[child];
 
-    for (const auto & output_node : from.outputs)
+    for (const auto & output_node : outputs)
         actions.outputs.push_back(old_to_new_nodes[output_node]);
 
-    for (const auto & input_node : from.inputs)
+    for (const auto & input_node : inputs)
         actions.inputs.push_back(old_to_new_nodes[input_node]);
 
     return actions;
diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h
index f428ca2f01c..05948ccf928 100644
--- a/src/Interpreters/ActionsDAG.h
+++ b/src/Interpreters/ActionsDAG.h
@@ -263,7 +263,9 @@ public:
 
     static ActionsDAGPtr clone(const ActionsDAGPtr & from) { return clone(from.get()); }
     static ActionsDAGPtr clone(const ActionsDAG * from);
-    static ActionsDAG clone(const ActionsDAG & from, std::unordered_map<const Node *, Node *> & old_to_new_nodes);
+
+    ActionsDAG clone(std::unordered_map<const Node *, Node *> & old_to_new_nodes) const;
+    ActionsDAG clone() const;
 
     static ActionsDAGPtr cloneSubDAG(const NodeRawConstPtrs & outputs, bool remove_aliases);
 
diff --git a/src/Interpreters/ExpressionActions.cpp b/src/Interpreters/ExpressionActions.cpp
index dd1d2eb703e..113410b1480 100644
--- a/src/Interpreters/ExpressionActions.cpp
+++ b/src/Interpreters/ExpressionActions.cpp
@@ -75,7 +75,7 @@ ExpressionActionsPtr ExpressionActions::clone() const
     auto copy = std::make_shared<ExpressionActions>(ExpressionActions());
 
     std::unordered_map<const Node *, Node *> copy_map;
-    copy->actions_dag = ActionsDAG::clone(actions_dag, copy_map);
+    copy->actions_dag = actions_dag.clone(copy_map);
     copy->actions = actions;
     for (auto & action : copy->actions)
         action.node = copy_map[action.node];
diff --git a/src/Planner/CollectTableExpressionData.cpp b/src/Planner/CollectTableExpressionData.cpp
index 162d3fe8d11..1d85476636c 100644
--- a/src/Planner/CollectTableExpressionData.cpp
+++ b/src/Planner/CollectTableExpressionData.cpp
@@ -335,22 +335,22 @@ void collectTableExpressionData(QueryTreeNodePtr & query_node, PlannerContextPtr
         collect_source_columns_visitor.setKeepAliasColumns(false);
         collect_source_columns_visitor.visit(query_node_typed.getPrewhere());
 
-        auto prewhere_actions_dag = std::make_unique<ActionsDAG>();
+        ActionsDAG prewhere_actions_dag;
 
         QueryTreeNodePtr query_tree_node = query_node_typed.getPrewhere();
 
         PlannerActionsVisitor visitor(planner_context, false /*use_column_identifier_as_action_node_name*/);
-        auto expression_nodes = visitor.visit(*prewhere_actions_dag, query_tree_node);
+        auto expression_nodes = visitor.visit(prewhere_actions_dag, query_tree_node);
         if (expression_nodes.size() != 1)
             throw Exception(ErrorCodes::ILLEGAL_PREWHERE,
                 "Invalid PREWHERE. Expected single boolean expression. In query {}",
                 query_node->formatASTForErrorMessage());
 
-        prewhere_actions_dag->getOutputs().push_back(expression_nodes.back());
+        prewhere_actions_dag.getOutputs().push_back(expression_nodes.back());
 
-        for (const auto & prewhere_input_node : prewhere_actions_dag->getInputs())
+        for (const auto & prewhere_input_node : prewhere_actions_dag.getInputs())
             if (required_column_names_without_prewhere.contains(prewhere_input_node->result_name))
-                prewhere_actions_dag->getOutputs().push_back(prewhere_input_node);
+                prewhere_actions_dag.getOutputs().push_back(prewhere_input_node);
 
         table_expression_data.setPrewhereFilterActions(std::move(prewhere_actions_dag));
     }
diff --git a/src/Planner/Planner.cpp b/src/Planner/Planner.cpp
index ffed19185d3..9042303d0e4 100644
--- a/src/Planner/Planner.cpp
+++ b/src/Planner/Planner.cpp
@@ -1412,7 +1412,7 @@ void Planner::buildPlanForQueryNode()
     checkStoragesSupportTransactions(planner_context);
 
     const auto & table_filters = planner_context->getGlobalPlannerContext()->filters_for_table_expressions;
-    if (!select_query_options.only_analyze && !table_filters.empty()) // && top_level)
+    if (!select_query_options.only_analyze && !table_filters.empty())
     {
         for (auto & [table_node, table_expression_data] : planner_context->getTableExpressionNodeToData())
         {
@@ -1420,7 +1420,7 @@ void Planner::buildPlanForQueryNode()
             if (it != table_filters.end())
             {
                 const auto & filters = it->second;
-                table_expression_data.setFilterActions(ActionsDAG::clone(&*filters.filter_actions));
+                table_expression_data.setFilterActions(filters.filter_actions->clone());
                 table_expression_data.setPrewhereInfo(filters.prewhere_info);
             }
         }
diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp
index 3217d3461d3..d55e5e99f71 100644
--- a/src/Planner/PlannerJoinTree.cpp
+++ b/src/Planner/PlannerJoinTree.cpp
@@ -646,7 +646,8 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres
 
         auto table_expression_query_info = select_query_info;
         table_expression_query_info.table_expression = table_expression;
-        table_expression_query_info.filter_actions_dag = ActionsDAG::clone(table_expression_data.getFilterActions());
+        if (const auto & filter_actions = table_expression_data.getFilterActions())
+            table_expression_query_info.filter_actions_dag = std::make_shared<const ActionsDAG>(filter_actions->clone());
         table_expression_query_info.analyzer_can_use_parallel_replicas_on_follower = table_node == planner_context->getGlobalPlannerContext()->parallel_replicas_table;
 
         size_t max_streams = settings.max_threads;
@@ -776,7 +777,7 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres
                 if (prewhere_actions)
                 {
                     prewhere_info = std::make_shared<PrewhereInfo>();
-                    prewhere_info->prewhere_actions = std::move(*ActionsDAG::clone(prewhere_actions));
+                    prewhere_info->prewhere_actions = prewhere_actions->clone();
                     prewhere_info->prewhere_column_name = prewhere_actions->getOutputs().at(0)->result_name;
                     prewhere_info->remove_prewhere_column = true;
                     prewhere_info->need_filter = true;
@@ -829,7 +830,7 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres
                     = buildRowPolicyFilterIfNeeded(storage, table_expression_query_info, planner_context, used_row_policies);
                 if (row_policy_filter_info)
                 {
-                    table_expression_data.setRowLevelFilterActions(ActionsDAG::clone(&row_policy_filter_info->actions));
+                    table_expression_data.setRowLevelFilterActions(row_policy_filter_info->actions.clone());
                     add_filter(*row_policy_filter_info, "Row-level security filter");
                 }
 
diff --git a/src/Planner/TableExpressionData.h b/src/Planner/TableExpressionData.h
index 9723a00a356..1d04fac3dc3 100644
--- a/src/Planner/TableExpressionData.h
+++ b/src/Planner/TableExpressionData.h
@@ -211,32 +211,32 @@ public:
         is_merge_tree = is_merge_tree_value;
     }
 
-    const ActionsDAGPtr & getPrewhereFilterActions() const
+    const std::optional<ActionsDAG> & getPrewhereFilterActions() const
     {
         return prewhere_filter_actions;
     }
 
-    void setRowLevelFilterActions(ActionsDAGPtr row_level_filter_actions_value)
+    void setRowLevelFilterActions(ActionsDAG row_level_filter_actions_value)
     {
         row_level_filter_actions = std::move(row_level_filter_actions_value);
     }
 
-    const ActionsDAGPtr & getRowLevelFilterActions() const
+    const std::optional<ActionsDAG> & getRowLevelFilterActions() const
     {
         return row_level_filter_actions;
     }
 
-    void setPrewhereFilterActions(ActionsDAGPtr prewhere_filter_actions_value)
+    void setPrewhereFilterActions(ActionsDAG prewhere_filter_actions_value)
     {
         prewhere_filter_actions = std::move(prewhere_filter_actions_value);
     }
 
-    const ActionsDAGPtr & getFilterActions() const
+    const std::optional<ActionsDAG> & getFilterActions() const
     {
         return filter_actions;
     }
 
-    void setFilterActions(ActionsDAGPtr filter_actions_value)
+    void setFilterActions(ActionsDAG filter_actions_value)
     {
         filter_actions = std::move(filter_actions_value);
     }
@@ -289,16 +289,16 @@ private:
     ColumnIdentifierToColumnName column_identifier_to_column_name;
 
     /// Valid for table, table function
-    ActionsDAGPtr filter_actions;
+    std::optional<ActionsDAG> filter_actions;
 
     /// Valid for table, table function
     PrewhereInfoPtr prewhere_info;
 
     /// Valid for table, table function
-    ActionsDAGPtr prewhere_filter_actions;
+    std::optional<ActionsDAG> prewhere_filter_actions;
 
     /// Valid for table, table function
-    ActionsDAGPtr row_level_filter_actions;
+    std::optional<ActionsDAG> row_level_filter_actions;
 
     /// Is storage remote
     bool is_remote = false;

From 264d7d760fedd3fc3c900d13ee1f7976887efaa7 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Sat, 13 Jul 2024 15:52:23 +0000
Subject: [PATCH 0214/1488] Bump rocksdb to 7.0.4

---
 contrib/rocksdb                      | 2 +-
 contrib/rocksdb-cmake/CMakeLists.txt | 9 +++------
 2 files changed, 4 insertions(+), 7 deletions(-)

diff --git a/contrib/rocksdb b/contrib/rocksdb
index be366233921..4fc59e24001 160000
--- a/contrib/rocksdb
+++ b/contrib/rocksdb
@@ -1 +1 @@
-Subproject commit be366233921293bd07a84dc4ea6991858665f202
+Subproject commit 4fc59e240016a62180b09703e2938c3d7e928de0
diff --git a/contrib/rocksdb-cmake/CMakeLists.txt b/contrib/rocksdb-cmake/CMakeLists.txt
index 3a14407166c..f6479346063 100644
--- a/contrib/rocksdb-cmake/CMakeLists.txt
+++ b/contrib/rocksdb-cmake/CMakeLists.txt
@@ -59,10 +59,8 @@ if(CMAKE_SYSTEM_PROCESSOR MATCHES "arm64|aarch64|AARCH64")
   # set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=armv8-a+crc+crypto -Wno-unused-function")
 endif()
 
-set (HAVE_THREAD_LOCAL 1)
-if(HAVE_THREAD_LOCAL)
-  add_definitions(-DROCKSDB_SUPPORT_THREAD_LOCAL)
-endif()
+# thread_local is part of C++11 and later (TODO: clean up this define)
+add_definitions(-DROCKSDB_SUPPORT_THREAD_LOCAL)
 
 if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
   add_definitions(-DOS_MACOSX)
@@ -182,7 +180,6 @@ set(SOURCES
     ${ROCKSDB_SOURCE_DIR}/env/env.cc
     ${ROCKSDB_SOURCE_DIR}/env/env_chroot.cc
     ${ROCKSDB_SOURCE_DIR}/env/env_encryption.cc
-    ${ROCKSDB_SOURCE_DIR}/env/env_hdfs.cc
     ${ROCKSDB_SOURCE_DIR}/env/file_system.cc
     ${ROCKSDB_SOURCE_DIR}/env/file_system_tracer.cc
     ${ROCKSDB_SOURCE_DIR}/env/fs_remap.cc
@@ -311,7 +308,6 @@ set(SOURCES
     ${ROCKSDB_SOURCE_DIR}/util/murmurhash.cc
     ${ROCKSDB_SOURCE_DIR}/util/random.cc
     ${ROCKSDB_SOURCE_DIR}/util/rate_limiter.cc
-    ${ROCKSDB_SOURCE_DIR}/util/regex.cc
     ${ROCKSDB_SOURCE_DIR}/util/ribbon_config.cc
     ${ROCKSDB_SOURCE_DIR}/util/slice.cc
     ${ROCKSDB_SOURCE_DIR}/util/file_checksum_helper.cc
@@ -335,6 +331,7 @@ set(SOURCES
     ${ROCKSDB_SOURCE_DIR}/utilities/checkpoint/checkpoint_impl.cc
     ${ROCKSDB_SOURCE_DIR}/utilities/compaction_filters.cc
     ${ROCKSDB_SOURCE_DIR}/utilities/compaction_filters/remove_emptyvalue_compactionfilter.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/counted_fs.cc
     ${ROCKSDB_SOURCE_DIR}/utilities/debug.cc
     ${ROCKSDB_SOURCE_DIR}/utilities/env_mirror.cc
     ${ROCKSDB_SOURCE_DIR}/utilities/env_timed.cc

From 404840bf2c797d8a500d50ad1f2411b330622273 Mon Sep 17 00:00:00 2001
From: Zawa-II <zawalee33@gmail.com>
Date: Wed, 10 Jul 2024 21:29:09 -0700
Subject: [PATCH 0215/1488] [ClickHouse-66010]
 Add-machine_id-to-generateSnowflakeID

Extend existing tests and retain expr argument in generateSnowflakeID

Commit expected output file

Add argument validation in generateSnowflakeID function

Style Check

Style Check by changing error type

Style Check by removing error check

Add new test cases for  function including  parameter

Add new test cases for generateSnowflakeID function with machine_id parameter and update reference file

Add new test cases for generateSnowflakeID function with machine_id parameter and update reference file to match expected output

Fix generateSnowflakeID function to correctly handle optional machine_id parameter, update documentation and examples, and clean up tests.

Fix typo in generateSnowflakeID function and correctly handle optional machine_id parameter.
---
 src/Functions/generateSnowflakeID.cpp         | 31 ++++++++++++++-----
 .../03130_generateSnowflakeId.reference       |  4 +++
 .../0_stateless/03130_generateSnowflakeId.sql |  9 ++++--
 3 files changed, 34 insertions(+), 10 deletions(-)

diff --git a/src/Functions/generateSnowflakeID.cpp b/src/Functions/generateSnowflakeID.cpp
index a171b6bf86e..6befdf7a86f 100644
--- a/src/Functions/generateSnowflakeID.cpp
+++ b/src/Functions/generateSnowflakeID.cpp
@@ -4,6 +4,7 @@
 #include <Functions/FunctionHelpers.h>
 #include <Core/ServerUUID.h>
 #include <Common/Logger.h>
+#include <Common/ErrorCodes.h>
 #include <Common/logger_useful.h>
 #include "base/types.h"
 
@@ -165,24 +166,37 @@ public:
     {
         FunctionArgumentDescriptors mandatory_args;
         FunctionArgumentDescriptors optional_args{
-            {"expr", nullptr, nullptr, "Arbitrary expression"}
+            {"expr", nullptr, nullptr, "Arbitrary expression"},
+            {"machine_id", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNativeUInt), nullptr, "Optional machine ID in UInt*"}
         };
         validateFunctionArguments(*this, arguments, mandatory_args, optional_args);
 
         return std::make_shared<DataTypeUInt64>();
     }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & /*arguments*/, const DataTypePtr &, size_t input_rows_count) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
         auto col_res = ColumnVector<UInt64>::create();
         typename ColumnVector<UInt64>::Container & vec_to = col_res->getData();
+        vec_to.resize(input_rows_count);
+
+        uint64_t machine_id = 0;
+        if (arguments.size() == 2)
+            machine_id = arguments[1].column->getUInt(0);
+
+        if (machine_id == 0)
+            machine_id = getMachineId();
+
+        // Ensure machine_id is within the valid range
+        machine_id &= (1ull << machine_id_bits_count) - 1;
+
+        // Process expr argument here if necessary (currently a placeholder)
 
         if (input_rows_count != 0)
         {
-            vec_to.resize(input_rows_count);
-
             Data data;
             SnowflakeId snowflake_id = data.reserveRange(input_rows_count); /// returns begin of available snowflake ids range
+            snowflake_id.machine_id = machine_id; //Assign the provided machine_id
 
             for (UInt64 & to_row : vec_to)
             {
@@ -208,10 +222,13 @@ public:
 REGISTER_FUNCTION(GenerateSnowflakeID)
 {
     FunctionDocumentation::Description description = R"(Generates a Snowflake ID. The generated Snowflake ID contains the current Unix timestamp in milliseconds (41 + 1 top zero bits), followed by a machine id (10 bits), and a counter (12 bits) to distinguish IDs within a millisecond. For any given timestamp (unix_ts_ms), the counter starts at 0 and is incremented by 1 for each new Snowflake ID until the timestamp changes. In case the counter overflows, the timestamp field is incremented by 1 and the counter is reset to 0. Function generateSnowflakeID guarantees that the counter field within a timestamp increments monotonically across all function invocations in concurrently running threads and queries.)";
-    FunctionDocumentation::Syntax syntax = "generateSnowflakeID([expression])";
-    FunctionDocumentation::Arguments arguments = {{"expression", "The expression is used to bypass common subexpression elimination if the function is called multiple times in a query but otherwise ignored. Optional."}};
+    FunctionDocumentation::Syntax syntax = "generateSnowflakeID([expression, [machine_id]])";
+    FunctionDocumentation::Arguments arguments = {
+        {"expression", "The expression is used to bypass common subexpression elimination if the function is called multiple times in a query but otherwise ignored. Optional."},
+        {"machine_id", "A machine ID, the 10 least significant bits are used. Optional."}
+    };
     FunctionDocumentation::ReturnedValue returned_value = "A value of type UInt64";
-    FunctionDocumentation::Examples examples = {{"single", "SELECT generateSnowflakeID()", "7201148511606784000"}, {"multiple", "SELECT generateSnowflakeID(1), generateSnowflakeID(2)", ""}};
+    FunctionDocumentation::Examples examples = {{"single", "SELECT generateSnowflakeID()", "7201148511606784000"}, {"with_machine_id", "SELECT generateSnowflakeID(1)", ""}, {"with_machine_id_and_expression", "SELECT generateSnowflakeID('some_expression', 1)", ""}};
     FunctionDocumentation::Categories categories = {"Snowflake ID"};
 
     factory.registerFunction<FunctionGenerateSnowflakeID>({description, syntax, arguments, returned_value, examples, categories});
diff --git a/tests/queries/0_stateless/03130_generateSnowflakeId.reference b/tests/queries/0_stateless/03130_generateSnowflakeId.reference
index 39669d21bee..5cc19c73d88 100644
--- a/tests/queries/0_stateless/03130_generateSnowflakeId.reference
+++ b/tests/queries/0_stateless/03130_generateSnowflakeId.reference
@@ -1,5 +1,9 @@
 1
 0
+1
 0
 1
+1
+1
+1
 100
diff --git a/tests/queries/0_stateless/03130_generateSnowflakeId.sql b/tests/queries/0_stateless/03130_generateSnowflakeId.sql
index 0717c81aa0d..b28960cfcbf 100644
--- a/tests/queries/0_stateless/03130_generateSnowflakeId.sql
+++ b/tests/queries/0_stateless/03130_generateSnowflakeId.sql
@@ -1,12 +1,15 @@
 -- Test SQL function 'generateSnowflakeID'
 
 SELECT bitAnd(bitShiftRight(toUInt64(generateSnowflakeID()), 63), 1) = 0; -- check first bit is zero
-
-SELECT generateSnowflakeID(1) = generateSnowflakeID(2); -- disabled common subexpression elimination --> lhs != rhs
 SELECT generateSnowflakeID() = generateSnowflakeID(1); -- same as ^^
 SELECT generateSnowflakeID(1) = generateSnowflakeID(1); -- enabled common subexpression elimination
 
-SELECT generateSnowflakeID(1, 2); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH }
+SELECT generateSnowflakeID(1) = generateSnowflakeID(2); -- disabled common subexpression elimination --> lhs != rhs
+SELECT generateSnowflakeID(1) != generateSnowflakeID(); -- Check different invocations yield different results
+
+SELECT generateSnowflakeID('expr', 1) = generateSnowflakeID('expr', 1); -- enabled common subexpression elimination
+SELECT generateSnowflakeID('expr', 1) != generateSnowflakeID('expr', 2); -- different machine IDs should produce different results
+SELECT generateSnowflakeID('expr', 1) != generateSnowflakeID('different_expr', 1); -- different expressions should bypass common subexpression elimination
 
 SELECT count(*)
 FROM

From 6f08e785144b1d6794f78b8bc9d0d5b21a0f160d Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 13 Jul 2024 22:27:16 +0200
Subject: [PATCH 0216/1488] Update 03198_group_array_intersect.sql

---
 tests/queries/0_stateless/03198_group_array_intersect.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/03198_group_array_intersect.sql b/tests/queries/0_stateless/03198_group_array_intersect.sql
index 06ab0fbd819..ab7ba0b0d70 100644
--- a/tests/queries/0_stateless/03198_group_array_intersect.sql
+++ b/tests/queries/0_stateless/03198_group_array_intersect.sql
@@ -6,6 +6,6 @@ INSERT INTO test_numbers__fuzz_29 VALUES ([1,2,3,4,5,6]);
 INSERT INTO test_numbers__fuzz_29 VALUES ([1,2,4,5]);
 INSERT INTO test_numbers__fuzz_29 VALUES ([1,4,3,0,5,5,5]);
 
-SELECT groupArrayIntersect(*) FROM test_numbers__fuzz_29 GROUP BY a WITH ROLLUP;
+SELECT arraySort(groupArrayIntersect(*)) FROM test_numbers__fuzz_29 GROUP BY a WITH ROLLUP ORDER BY ALL;
 
 DROP TABLE test_numbers__fuzz_29;

From e1aac19630e1e1dcdcc9e7636b3912c01e5a38ab Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 13 Jul 2024 22:43:30 +0200
Subject: [PATCH 0217/1488] Fix test

---
 .../0_stateless/03198_group_array_intersect.reference     | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/queries/0_stateless/03198_group_array_intersect.reference b/tests/queries/0_stateless/03198_group_array_intersect.reference
index f6b2def2f55..6a3ffdba388 100644
--- a/tests/queries/0_stateless/03198_group_array_intersect.reference
+++ b/tests/queries/0_stateless/03198_group_array_intersect.reference
@@ -1,4 +1,4 @@
-['6\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0','2\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0','3\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0','5\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0','1\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0','4\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0']
-['2\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0','5\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0','1\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0','4\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0']
-['3\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0','5\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0','1\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0','0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0','4\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0']
-['5\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0','1\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0','4\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0']
+['0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0','1\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0','3\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0','4\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0','5\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0']
+['1\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0','2\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0','3\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0','4\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0','5\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0','6\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0']
+['1\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0','2\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0','4\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0','5\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0']
+['1\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0','4\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0','5\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0']

From 0925830f2edbe85a7aac6d45608551c3efb43731 Mon Sep 17 00:00:00 2001
From: Haydn <zawalee33@gmail.com>
Date: Sat, 13 Jul 2024 14:01:03 -0700
Subject: [PATCH 0218/1488] Refactored generateSnowflakeID function
 implementation and added comprehensive SQL tests

---
 src/Functions/generateSnowflakeID.cpp | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/src/Functions/generateSnowflakeID.cpp b/src/Functions/generateSnowflakeID.cpp
index 6befdf7a86f..6c5124872a0 100644
--- a/src/Functions/generateSnowflakeID.cpp
+++ b/src/Functions/generateSnowflakeID.cpp
@@ -181,22 +181,25 @@ public:
         vec_to.resize(input_rows_count);
 
         uint64_t machine_id = 0;
-        if (arguments.size() == 2)
-            machine_id = arguments[1].column->getUInt(0);
+        if (arguments.size() == 2 && input_rows_count > 0)
+        {
+            const auto & column = arguments[1].column;
+            if (column && !column->empty())
+                machine_id = column->getUInt(0);
+        }
 
         if (machine_id == 0)
             machine_id = getMachineId();
 
-        // Ensure machine_id is within the valid range
+        /// Ensure machine_id is within the valid range
         machine_id &= (1ull << machine_id_bits_count) - 1;
 
-        // Process expr argument here if necessary (currently a placeholder)
+        /// Process expr argument here if necessary (currently a placeholder)
 
-        if (input_rows_count != 0)
+        if (input_rows_count > 0)
         {
             Data data;
-            SnowflakeId snowflake_id = data.reserveRange(input_rows_count); /// returns begin of available snowflake ids range
-            snowflake_id.machine_id = machine_id; //Assign the provided machine_id
+            SnowflakeId snowflake_id = data.reserveRange(input_rows_count, machine_id);
 
             for (UInt64 & to_row : vec_to)
             {

From 08cce45258b133b2b44c8dd4c3a9c3fe70551e7e Mon Sep 17 00:00:00 2001
From: Haydn <zawalee33@gmail.com>
Date: Sat, 13 Jul 2024 14:25:42 -0700
Subject: [PATCH 0219/1488] Fix handling of machine_id in generateSnowflakeID
 to read conditionally and assign efficiently

---
 src/Functions/generateSnowflakeID.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/Functions/generateSnowflakeID.cpp b/src/Functions/generateSnowflakeID.cpp
index 6c5124872a0..e58d7cfd2d6 100644
--- a/src/Functions/generateSnowflakeID.cpp
+++ b/src/Functions/generateSnowflakeID.cpp
@@ -199,7 +199,8 @@ public:
         if (input_rows_count > 0)
         {
             Data data;
-            SnowflakeId snowflake_id = data.reserveRange(input_rows_count, machine_id);
+            SnowflakeId snowflake_id = data.reserveRange(input_rows_count);
+            snowflake_id.machine_id = machine_id;
 
             for (UInt64 & to_row : vec_to)
             {

From 5826f1f073f971031a681637491d14934612b4c4 Mon Sep 17 00:00:00 2001
From: Haydn <zawalee33@gmail.com>
Date: Sat, 13 Jul 2024 15:39:48 -0700
Subject: [PATCH 0220/1488] [ClickHouse-66010] Add machine_id parameter to
 generateSnowflakeID function and update tests

---
 src/Functions/generateSnowflakeID.cpp | 28 +++++++++++----------------
 1 file changed, 11 insertions(+), 17 deletions(-)

diff --git a/src/Functions/generateSnowflakeID.cpp b/src/Functions/generateSnowflakeID.cpp
index e58d7cfd2d6..ca639d65cc5 100644
--- a/src/Functions/generateSnowflakeID.cpp
+++ b/src/Functions/generateSnowflakeID.cpp
@@ -8,7 +8,6 @@
 #include <Common/logger_useful.h>
 #include "base/types.h"
 
-
 namespace DB
 {
 
@@ -38,7 +37,7 @@ constexpr auto machine_seq_num_bits_count = 12;
 
 /// bits masks for Snowflake ID components
 constexpr uint64_t machine_id_mask = ((1ull << machine_id_bits_count) - 1) << machine_seq_num_bits_count;
-constexpr uint64_t machine_seq_num_mask = (1ull << machine_seq_num_bits_count) - 1;
+constexpr uint64_t machine_seq_num_mask = (1ull << machine seq_num_bits_count) - 1;
 
 /// max values
 constexpr uint64_t max_machine_seq_num = machine_seq_num_mask;
@@ -97,10 +96,10 @@ struct SnowflakeIdRange
 /// 1. calculate Snowflake ID by current timestamp (`now`)
 /// 2. `begin = max(available, now)`
 /// 3. Calculate `end = begin + input_rows_count` handling `machine_seq_num` overflow
-SnowflakeIdRange getRangeOfAvailableIds(const SnowflakeId & available, size_t input_rows_count)
+SnowflakeIdRange getRangeOfAvailableIds(const SnowflakeId & available, size_t input_rows_count, uint64_t machine_id)
 {
     /// 1. `now`
-    SnowflakeId begin = {.timestamp = getTimestamp(), .machine_id = getMachineId(), .machine_seq_num = 0};
+    SnowflakeId begin = {.timestamp = getTimestamp(), .machine_id = machine_id, .machine_seq_num = 0};
 
     /// 2. `begin`
     if (begin.timestamp <= available.timestamp)
@@ -129,13 +128,13 @@ struct Data
     /// Guarantee counter monotonicity within one timestamp across all threads generating Snowflake IDs simultaneously.
     static inline std::atomic<uint64_t> lowest_available_snowflake_id = 0;
 
-    SnowflakeId reserveRange(size_t input_rows_count)
+    SnowflakeId reserveRange(size_t input_rows_count, uint64_t machine_id)
     {
         uint64_t available_snowflake_id = lowest_available_snowflake_id.load();
         SnowflakeIdRange range;
         do
         {
-            range = getRangeOfAvailableIds(toSnowflakeId(available_snowflake_id), input_rows_count);
+            range = getRangeOfAvailableIds(toSnowflakeId(available_snowflake_id), input_rows_count, machine_id);
         }
         while (!lowest_available_snowflake_id.compare_exchange_weak(available_snowflake_id, fromSnowflakeId(range.end)));
         /// CAS failed --> another thread updated `lowest_available_snowflake_id` and we re-try
@@ -167,7 +166,7 @@ public:
         FunctionArgumentDescriptors mandatory_args;
         FunctionArgumentDescriptors optional_args{
             {"expr", nullptr, nullptr, "Arbitrary expression"},
-            {"machine_id", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNativeUInt), nullptr, "Optional machine ID in UInt*"}
+            {"machine_id", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNativeUInt), nullptr, "UInt*"}
         };
         validateFunctionArguments(*this, arguments, mandatory_args, optional_args);
 
@@ -183,24 +182,19 @@ public:
         uint64_t machine_id = 0;
         if (arguments.size() == 2 && input_rows_count > 0)
         {
-            const auto & column = arguments[1].column;
-            if (column && !column->empty())
-                machine_id = column->getUInt(0);
+            const auto & col_machine_id = arguments[1].column;
+            machine_id = col_machine_id->getUInt(0);
         }
-
-        if (machine_id == 0)
+        else
             machine_id = getMachineId();
 
-        /// Ensure machine_id is within the valid range
+        /// Truncate machine id to 10 bits
         machine_id &= (1ull << machine_id_bits_count) - 1;
 
-        /// Process expr argument here if necessary (currently a placeholder)
-
         if (input_rows_count > 0)
         {
             Data data;
-            SnowflakeId snowflake_id = data.reserveRange(input_rows_count);
-            snowflake_id.machine_id = machine_id;
+            SnowflakeId snowflake_id = data.reserveRange(input_rows_count, machine_id);
 
             for (UInt64 & to_row : vec_to)
             {

From 6ed138b1a41b94da4daa456535da719af56c36f7 Mon Sep 17 00:00:00 2001
From: Haydn <zawalee33@gmail.com>
Date: Sat, 13 Jul 2024 16:06:31 -0700
Subject: [PATCH 0221/1488] Fix syntax error in generateSnowflakeID.cpp

---
 src/Functions/generateSnowflakeID.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Functions/generateSnowflakeID.cpp b/src/Functions/generateSnowflakeID.cpp
index ca639d65cc5..3aaa0b50a75 100644
--- a/src/Functions/generateSnowflakeID.cpp
+++ b/src/Functions/generateSnowflakeID.cpp
@@ -37,7 +37,7 @@ constexpr auto machine_seq_num_bits_count = 12;
 
 /// bits masks for Snowflake ID components
 constexpr uint64_t machine_id_mask = ((1ull << machine_id_bits_count) - 1) << machine_seq_num_bits_count;
-constexpr uint64_t machine_seq_num_mask = (1ull << machine seq_num_bits_count) - 1;
+constexpr uint64_t machine_seq_num_mask = (1ull << machine_seq_num_bits_count) - 1;
 
 /// max values
 constexpr uint64_t max_machine_seq_num = machine_seq_num_mask;

From fee7e22c1f60feb0a4c176355453caad18cd5bc1 Mon Sep 17 00:00:00 2001
From: xogoodnow <xo4ever9999@gmail.com>
Date: Sun, 14 Jul 2024 15:23:34 +0330
Subject: [PATCH 0222/1488] Changed the error code

---
 src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp b/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp
index 4388864434e..87a44db573d 100644
--- a/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp
+++ b/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp
@@ -36,7 +36,7 @@ namespace ErrorCodes
 {
     extern const int LOGICAL_ERROR;
     extern const int BAD_ARGUMENTS;
-    extern const int QUERY_NOT_ALLOWED;
+    extern const int BAD_QUERY_PARAMETER;
 }
 
 namespace
@@ -150,7 +150,7 @@ StorageObjectStorageQueue::StorageObjectStorageQueue(
     }
     else if (!configuration->isPathWithGlobs())
     {
-        throw Exception(ErrorCodes::QUERY_NOT_ALLOWED, "ObjectStorageQueue url must either end with '/' or contain globs");
+        throw Exception(ErrorCodes::BAD_QUERY_PARAMETER, "ObjectStorageQueue url must either end with '/' or contain globs");
     }
 
     checkAndAdjustSettings(*queue_settings, engine_args, mode > LoadingStrictnessLevel::CREATE, log);

From b5cb264b017e965037dbb0bd4623df5f5a65ec0b Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Sat, 13 Jul 2024 15:41:30 +0000
Subject: [PATCH 0223/1488] Bump ICU to 71

---
 contrib/icu                      |  2 +-
 contrib/icu-cmake/CMakeLists.txt | 10 +++++-----
 contrib/icudata                  |  2 +-
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/contrib/icu b/contrib/icu
index a56dde820dc..c205e7ee49a 160000
--- a/contrib/icu
+++ b/contrib/icu
@@ -1 +1 @@
-Subproject commit a56dde820dc35665a66f2e9ee8ba58e75049b668
+Subproject commit c205e7ee49a7086a28b9c275fcfdac9ca3dc815d
diff --git a/contrib/icu-cmake/CMakeLists.txt b/contrib/icu-cmake/CMakeLists.txt
index 0a650f2bcc0..f23b0002b8d 100644
--- a/contrib/icu-cmake/CMakeLists.txt
+++ b/contrib/icu-cmake/CMakeLists.txt
@@ -4,7 +4,9 @@ else ()
     option(ENABLE_ICU "Enable ICU" 0)
 endif ()
 
-if (NOT ENABLE_ICU)
+# Temporarily disabled s390x because the ICU build links a blob (icudt71b_dat.S) and our friends from IBM did not explain how they re-generated
+# the blob on s390x: https://github.com/ClickHouse/icudata/pull/2#issuecomment-2226957255
+if (NOT ENABLE_ICU OR ARCH_S390X)
     message(STATUS "Not using ICU")
     return()
 endif()
@@ -12,8 +14,6 @@ endif()
 set(ICU_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/icu/icu4c/source")
 set(ICUDATA_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/icudata/")
 
-set (CMAKE_CXX_STANDARD 17)
-
 # These lists of sources were generated from build log of the original ICU build system (configure + make).
 
 set(ICUUC_SOURCES
@@ -462,9 +462,9 @@ file(GENERATE OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/empty.cpp" CONTENT " ")
 enable_language(ASM)
 
 if (ARCH_S390X)
-    set(ICUDATA_SOURCE_FILE "${ICUDATA_SOURCE_DIR}/icudt70b_dat.S" )
+    set(ICUDATA_SOURCE_FILE "${ICUDATA_SOURCE_DIR}/icudt71b_dat.S" )
 else()
-    set(ICUDATA_SOURCE_FILE "${ICUDATA_SOURCE_DIR}/icudt70l_dat.S" )
+    set(ICUDATA_SOURCE_FILE "${ICUDATA_SOURCE_DIR}/icudt71l_dat.S" )
 endif()
 
 set(ICUDATA_SOURCES
diff --git a/contrib/icudata b/contrib/icudata
index c8e717892a5..e7488edd1f1 160000
--- a/contrib/icudata
+++ b/contrib/icudata
@@ -1 +1 @@
-Subproject commit c8e717892a557b4d2852317c7d628aacc0a0e5ab
+Subproject commit e7488edd1f141b0664553a985a6fcd0125279527

From 5b6956ea234962cca5414a3fb0a6191407b4305a Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Sat, 13 Jul 2024 16:48:09 +0000
Subject: [PATCH 0224/1488] Bump rocksdb to v7.10.2

---
 contrib/rocksdb                        |  2 +-
 contrib/rocksdb-cmake/CMakeLists.txt   | 53 +++++++++++++-------------
 contrib/rocksdb-cmake/build_version.cc | 31 +++++++++++----
 3 files changed, 51 insertions(+), 35 deletions(-)

diff --git a/contrib/rocksdb b/contrib/rocksdb
index 4fc59e24001..01e43568fa9 160000
--- a/contrib/rocksdb
+++ b/contrib/rocksdb
@@ -1 +1 @@
-Subproject commit 4fc59e240016a62180b09703e2938c3d7e928de0
+Subproject commit 01e43568fa9f3f7bf107b2b66c00b286b456f33e
diff --git a/contrib/rocksdb-cmake/CMakeLists.txt b/contrib/rocksdb-cmake/CMakeLists.txt
index f6479346063..98790158baa 100644
--- a/contrib/rocksdb-cmake/CMakeLists.txt
+++ b/contrib/rocksdb-cmake/CMakeLists.txt
@@ -5,6 +5,9 @@ if (NOT ENABLE_ROCKSDB)
   return()
 endif()
 
+# not in original build system, otherwise xxHash.cc fails to compile with ClickHouse C++23 default
+set (CMAKE_CXX_STANDARD 20)
+
 # Always disable jemalloc for rocksdb by default because it introduces non-standard jemalloc APIs
 option(WITH_JEMALLOC "build with JeMalloc" OFF)
 
@@ -16,14 +19,6 @@ option(WITH_LZ4 "build with lz4" ON)
 option(WITH_ZLIB "build with zlib" ON)
 option(WITH_ZSTD "build with zstd" ON)
 
-# third-party/folly is only validated to work on Linux and Windows for now.
-# So only turn it on there by default.
-if(CMAKE_SYSTEM_NAME MATCHES "Linux|Windows")
-  option(WITH_FOLLY_DISTRIBUTED_MUTEX "build with folly::DistributedMutex" ON)
-else()
-  option(WITH_FOLLY_DISTRIBUTED_MUTEX "build with folly::DistributedMutex" OFF)
-endif()
-
 if(WITH_SNAPPY)
   add_definitions(-DSNAPPY)
   list(APPEND THIRDPARTY_LIBS ch_contrib::snappy)
@@ -44,7 +39,7 @@ if(WITH_ZSTD)
   list(APPEND THIRDPARTY_LIBS ch_contrib::zstd)
 endif()
 
-option(PORTABLE "build a portable binary" ON)
+add_definitions(-DROCKSDB_PORTABLE)
 
 if(ENABLE_SSE42 AND ENABLE_PCLMULQDQ)
   add_definitions(-DHAVE_SSE42)
@@ -59,9 +54,6 @@ if(CMAKE_SYSTEM_PROCESSOR MATCHES "arm64|aarch64|AARCH64")
   # set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=armv8-a+crc+crypto -Wno-unused-function")
 endif()
 
-# thread_local is part of C++11 and later (TODO: clean up this define)
-add_definitions(-DROCKSDB_SUPPORT_THREAD_LOCAL)
-
 if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
   add_definitions(-DOS_MACOSX)
 elseif(CMAKE_SYSTEM_NAME MATCHES "Linux")
@@ -87,19 +79,21 @@ set(ROCKSDB_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/rocksdb")
 
 include_directories(${ROCKSDB_SOURCE_DIR})
 include_directories("${ROCKSDB_SOURCE_DIR}/include")
-if(WITH_FOLLY_DISTRIBUTED_MUTEX)
-  include_directories("${ROCKSDB_SOURCE_DIR}/third-party/folly")
-endif()
 
 set(SOURCES
     ${ROCKSDB_SOURCE_DIR}/cache/cache.cc
     ${ROCKSDB_SOURCE_DIR}/cache/cache_entry_roles.cc
     ${ROCKSDB_SOURCE_DIR}/cache/cache_key.cc
+    ${ROCKSDB_SOURCE_DIR}/cache/cache_helpers.cc
     ${ROCKSDB_SOURCE_DIR}/cache/cache_reservation_manager.cc
+    ${ROCKSDB_SOURCE_DIR}/cache/charged_cache.cc
     ${ROCKSDB_SOURCE_DIR}/cache/clock_cache.cc
+    ${ROCKSDB_SOURCE_DIR}/cache/compressed_secondary_cache.cc
     ${ROCKSDB_SOURCE_DIR}/cache/lru_cache.cc
+    ${ROCKSDB_SOURCE_DIR}/cache/secondary_cache.cc
     ${ROCKSDB_SOURCE_DIR}/cache/sharded_cache.cc
     ${ROCKSDB_SOURCE_DIR}/db/arena_wrapped_db_iter.cc
+    ${ROCKSDB_SOURCE_DIR}/db/blob/blob_contents.cc
     ${ROCKSDB_SOURCE_DIR}/db/blob/blob_fetcher.cc
     ${ROCKSDB_SOURCE_DIR}/db/blob/blob_file_addition.cc
     ${ROCKSDB_SOURCE_DIR}/db/blob/blob_file_builder.cc
@@ -111,6 +105,7 @@ set(SOURCES
     ${ROCKSDB_SOURCE_DIR}/db/blob/blob_log_format.cc
     ${ROCKSDB_SOURCE_DIR}/db/blob/blob_log_sequential_reader.cc
     ${ROCKSDB_SOURCE_DIR}/db/blob/blob_log_writer.cc
+    ${ROCKSDB_SOURCE_DIR}/db/blob/blob_source.cc
     ${ROCKSDB_SOURCE_DIR}/db/blob/prefetch_buffer_collection.cc
     ${ROCKSDB_SOURCE_DIR}/db/builder.cc
     ${ROCKSDB_SOURCE_DIR}/db/c.cc
@@ -122,7 +117,11 @@ set(SOURCES
     ${ROCKSDB_SOURCE_DIR}/db/compaction/compaction_picker_fifo.cc
     ${ROCKSDB_SOURCE_DIR}/db/compaction/compaction_picker_level.cc
     ${ROCKSDB_SOURCE_DIR}/db/compaction/compaction_picker_universal.cc
+    ${ROCKSDB_SOURCE_DIR}/db/compaction/compaction_service_job.cc
+    ${ROCKSDB_SOURCE_DIR}/db/compaction/compaction_state.cc
+    ${ROCKSDB_SOURCE_DIR}/db/compaction/compaction_outputs.cc
     ${ROCKSDB_SOURCE_DIR}/db/compaction/sst_partitioner.cc
+    ${ROCKSDB_SOURCE_DIR}/db/compaction/subcompaction_state.cc
     ${ROCKSDB_SOURCE_DIR}/db/convenience.cc
     ${ROCKSDB_SOURCE_DIR}/db/db_filesnapshot.cc
     ${ROCKSDB_SOURCE_DIR}/db/db_impl/compacted_db_impl.cc
@@ -157,10 +156,11 @@ set(SOURCES
     ${ROCKSDB_SOURCE_DIR}/db/merge_helper.cc
     ${ROCKSDB_SOURCE_DIR}/db/merge_operator.cc
     ${ROCKSDB_SOURCE_DIR}/db/output_validator.cc
-    ${ROCKSDB_SOURCE_DIR}/db/periodic_work_scheduler.cc
+    ${ROCKSDB_SOURCE_DIR}/db/periodic_task_scheduler.cc
     ${ROCKSDB_SOURCE_DIR}/db/range_del_aggregator.cc
     ${ROCKSDB_SOURCE_DIR}/db/range_tombstone_fragmenter.cc
     ${ROCKSDB_SOURCE_DIR}/db/repair.cc
+    ${ROCKSDB_SOURCE_DIR}/db/seqno_to_time_mapping.cc
     ${ROCKSDB_SOURCE_DIR}/db/snapshot_impl.cc
     ${ROCKSDB_SOURCE_DIR}/db/table_cache.cc
     ${ROCKSDB_SOURCE_DIR}/db/table_properties_collector.cc
@@ -172,6 +172,8 @@ set(SOURCES
     ${ROCKSDB_SOURCE_DIR}/db/version_set.cc
     ${ROCKSDB_SOURCE_DIR}/db/wal_edit.cc
     ${ROCKSDB_SOURCE_DIR}/db/wal_manager.cc
+    ${ROCKSDB_SOURCE_DIR}/db/wide/wide_column_serialization.cc
+    ${ROCKSDB_SOURCE_DIR}/db/wide/wide_columns.cc
     ${ROCKSDB_SOURCE_DIR}/db/write_batch.cc
     ${ROCKSDB_SOURCE_DIR}/db/write_batch_base.cc
     ${ROCKSDB_SOURCE_DIR}/db/write_controller.cc
@@ -230,16 +232,17 @@ set(SOURCES
     ${ROCKSDB_SOURCE_DIR}/options/options.cc
     ${ROCKSDB_SOURCE_DIR}/options/options_helper.cc
     ${ROCKSDB_SOURCE_DIR}/options/options_parser.cc
+    ${ROCKSDB_SOURCE_DIR}/port/mmap.cc
     ${ROCKSDB_SOURCE_DIR}/port/stack_trace.cc
     ${ROCKSDB_SOURCE_DIR}/table/adaptive/adaptive_table_factory.cc
     ${ROCKSDB_SOURCE_DIR}/table/block_based/binary_search_index_reader.cc
     ${ROCKSDB_SOURCE_DIR}/table/block_based/block.cc
-    ${ROCKSDB_SOURCE_DIR}/table/block_based/block_based_filter_block.cc
     ${ROCKSDB_SOURCE_DIR}/table/block_based/block_based_table_builder.cc
     ${ROCKSDB_SOURCE_DIR}/table/block_based/block_based_table_factory.cc
     ${ROCKSDB_SOURCE_DIR}/table/block_based/block_based_table_iterator.cc
     ${ROCKSDB_SOURCE_DIR}/table/block_based/block_based_table_reader.cc
     ${ROCKSDB_SOURCE_DIR}/table/block_based/block_builder.cc
+    ${ROCKSDB_SOURCE_DIR}/table/block_based/block_cache.cc
     ${ROCKSDB_SOURCE_DIR}/table/block_based/block_prefetcher.cc
     ${ROCKSDB_SOURCE_DIR}/table/block_based/block_prefix_index.cc
     ${ROCKSDB_SOURCE_DIR}/table/block_based/data_block_hash_index.cc
@@ -297,9 +300,12 @@ set(SOURCES
     ${ROCKSDB_SOURCE_DIR}/trace_replay/trace_record_result.cc
     ${ROCKSDB_SOURCE_DIR}/trace_replay/trace_record.cc
     ${ROCKSDB_SOURCE_DIR}/trace_replay/trace_replay.cc
+    ${ROCKSDB_SOURCE_DIR}/util/async_file_reader.cc
+    ${ROCKSDB_SOURCE_DIR}/util/cleanable.cc
     ${ROCKSDB_SOURCE_DIR}/util/coding.cc
     ${ROCKSDB_SOURCE_DIR}/util/compaction_job_stats_impl.cc
     ${ROCKSDB_SOURCE_DIR}/util/comparator.cc
+    ${ROCKSDB_SOURCE_DIR}/util/compression.cc
     ${ROCKSDB_SOURCE_DIR}/util/compression_context_cache.cc
     ${ROCKSDB_SOURCE_DIR}/util/concurrent_task_limiter_impl.cc
     ${ROCKSDB_SOURCE_DIR}/util/crc32c.cc
@@ -312,11 +318,13 @@ set(SOURCES
     ${ROCKSDB_SOURCE_DIR}/util/slice.cc
     ${ROCKSDB_SOURCE_DIR}/util/file_checksum_helper.cc
     ${ROCKSDB_SOURCE_DIR}/util/status.cc
+    ${ROCKSDB_SOURCE_DIR}/util/stderr_logger.cc
     ${ROCKSDB_SOURCE_DIR}/util/string_util.cc
     ${ROCKSDB_SOURCE_DIR}/util/thread_local.cc
     ${ROCKSDB_SOURCE_DIR}/util/threadpool_imp.cc
     ${ROCKSDB_SOURCE_DIR}/util/xxhash.cc
-    ${ROCKSDB_SOURCE_DIR}/utilities/backupable/backupable_db.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/agg_merge/agg_merge.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/backup/backup_engine.cc
     ${ROCKSDB_SOURCE_DIR}/utilities/blob_db/blob_compaction_filter.cc
     ${ROCKSDB_SOURCE_DIR}/utilities/blob_db/blob_db.cc
     ${ROCKSDB_SOURCE_DIR}/utilities/blob_db/blob_db_impl.cc
@@ -419,15 +427,6 @@ list(APPEND SOURCES
     "${ROCKSDB_SOURCE_DIR}/env/fs_posix.cc"
     "${ROCKSDB_SOURCE_DIR}/env/io_posix.cc")
 
-if(WITH_FOLLY_DISTRIBUTED_MUTEX)
-  list(APPEND SOURCES
-    "${ROCKSDB_SOURCE_DIR}/third-party/folly/folly/detail/Futex.cpp"
-    "${ROCKSDB_SOURCE_DIR}/third-party/folly/folly/synchronization/AtomicNotification.cpp"
-    "${ROCKSDB_SOURCE_DIR}/third-party/folly/folly/synchronization/DistributedMutex.cpp"
-    "${ROCKSDB_SOURCE_DIR}/third-party/folly/folly/synchronization/ParkingLot.cpp"
-    "${ROCKSDB_SOURCE_DIR}/third-party/folly/folly/synchronization/WaitOptions.cpp")
-endif()
-
 add_library(_rocksdb ${SOURCES})
 add_library(ch_contrib::rocksdb ALIAS _rocksdb)
 target_link_libraries(_rocksdb PRIVATE ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
diff --git a/contrib/rocksdb-cmake/build_version.cc b/contrib/rocksdb-cmake/build_version.cc
index f9639da516f..d5ea56673e0 100644
--- a/contrib/rocksdb-cmake/build_version.cc
+++ b/contrib/rocksdb-cmake/build_version.cc
@@ -1,16 +1,33 @@
 // Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
-/// This file was edited for ClickHouse.
 
 #include <memory>
 
 #include "rocksdb/version.h"
+#include "rocksdb/utilities/object_registry.h"
 #include "util/string_util.h"
 
 // The build script may replace these values with real values based
 // on whether or not GIT is available and the platform settings
-static const std::string rocksdb_build_git_sha  = "rocksdb_build_git_sha:0";
-static const std::string rocksdb_build_git_tag = "rocksdb_build_git_tag:master";
-static const std::string rocksdb_build_date = "rocksdb_build_date:2000-01-01";
+static const std::string rocksdb_build_git_sha  = "rocksdb_build_git_sha:72438a678872544809393b831c7273794c074215";
+static const std::string rocksdb_build_git_tag = "rocksdb_build_git_tag:main";
+#define HAS_GIT_CHANGES 0
+#if HAS_GIT_CHANGES == 0
+// If HAS_GIT_CHANGES is 0, the GIT date is used.
+// Use the time the branch/tag was last modified
+static const std::string rocksdb_build_date = "rocksdb_build_date:2024-07-12 16:01:57";
+#else
+// If HAS_GIT_CHANGES is > 0, the branch/tag has modifications.
+// Use the time the build was created.
+static const std::string rocksdb_build_date = "rocksdb_build_date:2024-07-13 17:15:50";
+#endif
+
+extern "C" {
+
+} // extern "C"
+
+std::unordered_map<std::string, ROCKSDB_NAMESPACE::RegistrarFunc> ROCKSDB_NAMESPACE::ObjectRegistry::builtins_ = {
+  
+};
 
 namespace ROCKSDB_NAMESPACE {
 static void AddProperty(std::unordered_map<std::string, std::string> *props, const std::string& name) {
@@ -39,12 +56,12 @@ const std::unordered_map<std::string, std::string>& GetRocksBuildProperties() {
 }
 
 std::string GetRocksVersionAsString(bool with_patch) {
-  std::string version = ToString(ROCKSDB_MAJOR) + "." + ToString(ROCKSDB_MINOR);
+  std::string version = std::to_string(ROCKSDB_MAJOR) + "." + std::to_string(ROCKSDB_MINOR);
   if (with_patch) {
-    return version + "." + ToString(ROCKSDB_PATCH);
+    return version + "." + std::to_string(ROCKSDB_PATCH);
   } else {
     return version;
-  }
+ }
 }
 
 std::string GetRocksBuildInfoAsString(const std::string& program, bool verbose) {

From ace4aa36bf0e3762711616a49d217929e2c808a0 Mon Sep 17 00:00:00 2001
From: Haydn <zawalee33@gmail.com>
Date: Sun, 14 Jul 2024 09:41:24 -0700
Subject: [PATCH 0225/1488] Fixed generateSnowflakeID function and tests
 according

Update uuid-functions.md

Adjust comments in unit tests
---
 .../sql-reference/functions/uuid-functions.md | 14 ++++++++++++-
 src/Functions/generateSnowflakeID.cpp         | 20 +++++++++----------
 .../03130_generateSnowflakeId.reference       |  5 ++---
 .../0_stateless/03130_generateSnowflakeId.sql | 11 ++++++----
 4 files changed, 32 insertions(+), 18 deletions(-)

diff --git a/docs/en/sql-reference/functions/uuid-functions.md b/docs/en/sql-reference/functions/uuid-functions.md
index e990023efbc..f1abb90b455 100644
--- a/docs/en/sql-reference/functions/uuid-functions.md
+++ b/docs/en/sql-reference/functions/uuid-functions.md
@@ -567,13 +567,15 @@ While no standard or recommendation exists for the epoch of Snowflake IDs, imple
 **Syntax**
 
 ``` sql
-generateSnowflakeID([expr])
+generateSnowflakeID([expr, [machine_id]])
 ```
 
 **Arguments**
 
 - `expr` — An arbitrary [expression](../../sql-reference/syntax.md#syntax-expressions) used to bypass [common subexpression elimination](../../sql-reference/functions/index.md#common-subexpression-elimination) if the function is called multiple times in a query. The value of the expression has no effect on the returned Snowflake ID. Optional.
 
+- `machine_id` — A machine ID, the 10 least significant bits are used. Optional.
+
 **Returned value**
 
 A value of type UInt64.
@@ -608,6 +610,16 @@ SELECT generateSnowflakeID(1), generateSnowflakeID(2);
 └────────────────────────┴────────────────────────┘
 ```
 
+Example with expression and machine ID
+```
+SELECT generateSnowflakeID('expr', 1);
+
+┌─generateSnowflakeID('expr', 1)─┐
+│            7201148511606784002 │
+└────────────────────────────────┘
+```
+
+
 ## snowflakeToDateTime
 
 :::warning
diff --git a/src/Functions/generateSnowflakeID.cpp b/src/Functions/generateSnowflakeID.cpp
index 3aaa0b50a75..b2f764d9d56 100644
--- a/src/Functions/generateSnowflakeID.cpp
+++ b/src/Functions/generateSnowflakeID.cpp
@@ -96,7 +96,8 @@ struct SnowflakeIdRange
 /// 1. calculate Snowflake ID by current timestamp (`now`)
 /// 2. `begin = max(available, now)`
 /// 3. Calculate `end = begin + input_rows_count` handling `machine_seq_num` overflow
-SnowflakeIdRange getRangeOfAvailableIds(const SnowflakeId & available, size_t input_rows_count, uint64_t machine_id)
+SnowflakeIdRange getRangeOfAvailableIds(const SnowflakeId & available, uint64_t machine_id, size_t input_rows_count)
+
 {
     /// 1. `now`
     SnowflakeId begin = {.timestamp = getTimestamp(), .machine_id = machine_id, .machine_seq_num = 0};
@@ -134,7 +135,7 @@ struct Data
         SnowflakeIdRange range;
         do
         {
-            range = getRangeOfAvailableIds(toSnowflakeId(available_snowflake_id), input_rows_count, machine_id);
+            range = getRangeOfAvailableIds(toSnowflakeId(available_snowflake_id), machine_id, input_rows_count);
         }
         while (!lowest_available_snowflake_id.compare_exchange_weak(available_snowflake_id, fromSnowflakeId(range.end)));
         /// CAS failed --> another thread updated `lowest_available_snowflake_id` and we re-try
@@ -166,7 +167,9 @@ public:
         FunctionArgumentDescriptors mandatory_args;
         FunctionArgumentDescriptors optional_args{
             {"expr", nullptr, nullptr, "Arbitrary expression"},
-            {"machine_id", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNativeUInt), nullptr, "UInt*"}
+            {"machine_id", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNativeUInt), static_cast<FunctionArgumentDescriptor::ColumnValidator>(&isColumnConst), "UInt*"}
+            {"machine_id", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNativeUInt), static_cast<FunctionArgumentDescriptor::ColumnValidator>(&isColumnConst), "const UInt*"}
+
         };
         validateFunctionArguments(*this, arguments, mandatory_args, optional_args);
 
@@ -179,17 +182,14 @@ public:
         typename ColumnVector<UInt64>::Container & vec_to = col_res->getData();
         vec_to.resize(input_rows_count);
 
-        uint64_t machine_id = 0;
+        uint64_t machine_id = getMachineId();
         if (arguments.size() == 2 && input_rows_count > 0)
         {
             const auto & col_machine_id = arguments[1].column;
             machine_id = col_machine_id->getUInt(0);
+            /// Truncate machine id to 10 bits
+            machine_id &= (1ull << machine_id_bits_count) - 1;
         }
-        else
-            machine_id = getMachineId();
-
-        /// Truncate machine id to 10 bits
-        machine_id &= (1ull << machine_id_bits_count) - 1;
 
         if (input_rows_count > 0)
         {
@@ -226,7 +226,7 @@ REGISTER_FUNCTION(GenerateSnowflakeID)
         {"machine_id", "A machine ID, the 10 least significant bits are used. Optional."}
     };
     FunctionDocumentation::ReturnedValue returned_value = "A value of type UInt64";
-    FunctionDocumentation::Examples examples = {{"single", "SELECT generateSnowflakeID()", "7201148511606784000"}, {"with_machine_id", "SELECT generateSnowflakeID(1)", ""}, {"with_machine_id_and_expression", "SELECT generateSnowflakeID('some_expression', 1)", ""}};
+    FunctionDocumentation::Examples examples = {{"no_arguments", "SELECT generateSnowflakeID()", "7201148511606784000"}, {"with_machine_id", "SELECT generateSnowflakeID(1)", "7201148511606784001"}, {"with_expression_and_machine_id", "SELECT generateSnowflakeID('some_expression', 1)", "7201148511606784002"}};
     FunctionDocumentation::Categories categories = {"Snowflake ID"};
 
     factory.registerFunction<FunctionGenerateSnowflakeID>({description, syntax, arguments, returned_value, examples, categories});
diff --git a/tests/queries/0_stateless/03130_generateSnowflakeId.reference b/tests/queries/0_stateless/03130_generateSnowflakeId.reference
index 5cc19c73d88..35592351ac6 100644
--- a/tests/queries/0_stateless/03130_generateSnowflakeId.reference
+++ b/tests/queries/0_stateless/03130_generateSnowflakeId.reference
@@ -1,9 +1,8 @@
 1
-0
 1
 0
 1
 1
+0
 1
-1
-100
+100
\ No newline at end of file
diff --git a/tests/queries/0_stateless/03130_generateSnowflakeId.sql b/tests/queries/0_stateless/03130_generateSnowflakeId.sql
index b28960cfcbf..962faec46a8 100644
--- a/tests/queries/0_stateless/03130_generateSnowflakeId.sql
+++ b/tests/queries/0_stateless/03130_generateSnowflakeId.sql
@@ -2,14 +2,17 @@
 
 SELECT bitAnd(bitShiftRight(toUInt64(generateSnowflakeID()), 63), 1) = 0; -- check first bit is zero
 SELECT generateSnowflakeID() = generateSnowflakeID(1); -- same as ^^
-SELECT generateSnowflakeID(1) = generateSnowflakeID(1); -- enabled common subexpression elimination
-
 SELECT generateSnowflakeID(1) = generateSnowflakeID(2); -- disabled common subexpression elimination --> lhs != rhs
-SELECT generateSnowflakeID(1) != generateSnowflakeID(); -- Check different invocations yield different results
+SELECT generateSnowflakeID(1) = generateSnowflakeID(1); -- enabled common subexpression elimination
 
 SELECT generateSnowflakeID('expr', 1) = generateSnowflakeID('expr', 1); -- enabled common subexpression elimination
 SELECT generateSnowflakeID('expr', 1) != generateSnowflakeID('expr', 2); -- different machine IDs should produce different results
-SELECT generateSnowflakeID('expr', 1) != generateSnowflakeID('different_expr', 1); -- different expressions should bypass common subexpression elimination
+
+SELECT bitAnd(generateSnowflakeID(1023), 1023) = 1023; -- check if the last 10 bits match the machine ID
+
+SELECT generateSnowflakeID('invalid_machine_id'); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT }
+
+SELECT generateSnowflakeID(materialize(toUInt64(1))) = generateSnowflakeID(materialize(toUInt64(1))); -- -- test with non-const machine ID
 
 SELECT count(*)
 FROM

From 27273be84f3d252562646384d8db1aab6206d4a0 Mon Sep 17 00:00:00 2001
From: Haydn <zawalee33@gmail.com>
Date: Sun, 14 Jul 2024 09:51:13 -0700
Subject: [PATCH 0226/1488] Removed Duplicte args specifications

---
 src/Functions/generateSnowflakeID.cpp | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/Functions/generateSnowflakeID.cpp b/src/Functions/generateSnowflakeID.cpp
index b2f764d9d56..ada677a9355 100644
--- a/src/Functions/generateSnowflakeID.cpp
+++ b/src/Functions/generateSnowflakeID.cpp
@@ -167,9 +167,7 @@ public:
         FunctionArgumentDescriptors mandatory_args;
         FunctionArgumentDescriptors optional_args{
             {"expr", nullptr, nullptr, "Arbitrary expression"},
-            {"machine_id", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNativeUInt), static_cast<FunctionArgumentDescriptor::ColumnValidator>(&isColumnConst), "UInt*"}
             {"machine_id", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNativeUInt), static_cast<FunctionArgumentDescriptor::ColumnValidator>(&isColumnConst), "const UInt*"}
-
         };
         validateFunctionArguments(*this, arguments, mandatory_args, optional_args);
 

From 92b2a7d33d153cafd7dc6eb17cb0d2501d886245 Mon Sep 17 00:00:00 2001
From: Haydn <zawalee33@gmail.com>
Date: Sun, 14 Jul 2024 10:34:39 -0700
Subject: [PATCH 0227/1488] Updated generateSnowflakeID tests and reference
 outputs

---
 src/Functions/generateSnowflakeID.cpp                       | 6 ++++++
 .../queries/0_stateless/03130_generateSnowflakeId.reference | 3 ++-
 tests/queries/0_stateless/03130_generateSnowflakeId.sql     | 2 +-
 3 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/src/Functions/generateSnowflakeID.cpp b/src/Functions/generateSnowflakeID.cpp
index ada677a9355..6ee61df7e3b 100644
--- a/src/Functions/generateSnowflakeID.cpp
+++ b/src/Functions/generateSnowflakeID.cpp
@@ -184,7 +184,13 @@ public:
         if (arguments.size() == 2 && input_rows_count > 0)
         {
             const auto & col_machine_id = arguments[1].column;
+            if (!isColumnConst(*col_machine_id))
+                throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "The machine_id argument must be a constant UInt64");
+
             machine_id = col_machine_id->getUInt(0);
+            if (col_machine_id->getDataType() != TypeIndex::UInt64)
+                throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "The machine_id argument must be of type UInt64");
+
             /// Truncate machine id to 10 bits
             machine_id &= (1ull << machine_id_bits_count) - 1;
         }
diff --git a/tests/queries/0_stateless/03130_generateSnowflakeId.reference b/tests/queries/0_stateless/03130_generateSnowflakeId.reference
index 35592351ac6..dff613b29d3 100644
--- a/tests/queries/0_stateless/03130_generateSnowflakeId.reference
+++ b/tests/queries/0_stateless/03130_generateSnowflakeId.reference
@@ -5,4 +5,5 @@
 1
 0
 1
-100
\ No newline at end of file
+1
+100
diff --git a/tests/queries/0_stateless/03130_generateSnowflakeId.sql b/tests/queries/0_stateless/03130_generateSnowflakeId.sql
index 962faec46a8..be49f678dca 100644
--- a/tests/queries/0_stateless/03130_generateSnowflakeId.sql
+++ b/tests/queries/0_stateless/03130_generateSnowflakeId.sql
@@ -12,7 +12,7 @@ SELECT bitAnd(generateSnowflakeID(1023), 1023) = 1023; -- check if the last 10 b
 
 SELECT generateSnowflakeID('invalid_machine_id'); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT }
 
-SELECT generateSnowflakeID(materialize(toUInt64(1))) = generateSnowflakeID(materialize(toUInt64(1))); -- -- test with non-const machine ID
+SELECT generateSnowflakeID(materialize(toUInt64(1))) = generateSnowflakeID(materialize(toUInt64(1))); -- Test with non-const machine ID
 
 SELECT count(*)
 FROM

From 5ea81366a780557f11459573e24b774494145da7 Mon Sep 17 00:00:00 2001
From: Haydn <zawalee33@gmail.com>
Date: Sun, 14 Jul 2024 10:55:57 -0700
Subject: [PATCH 0228/1488] Fix generateSnowflakeID function to handle invalid
 inputs gracefully and avoid output for invalid cases

---
 src/Functions/generateSnowflakeID.cpp             | 15 +++++++++------
 .../03130_generateSnowflakeId.reference           |  1 -
 .../0_stateless/03130_generateSnowflakeId.sql     |  4 ++--
 3 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/src/Functions/generateSnowflakeID.cpp b/src/Functions/generateSnowflakeID.cpp
index 6ee61df7e3b..2db40b10496 100644
--- a/src/Functions/generateSnowflakeID.cpp
+++ b/src/Functions/generateSnowflakeID.cpp
@@ -18,7 +18,7 @@ namespace
   https://en.wikipedia.org/wiki/Snowflake_ID
 
  0                   1                   2                   3
- 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
 ├─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┤
 |0|                         timestamp                           |
 ├─┼                 ┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┤
@@ -184,13 +184,16 @@ public:
         if (arguments.size() == 2 && input_rows_count > 0)
         {
             const auto & col_machine_id = arguments[1].column;
-            if (!isColumnConst(*col_machine_id))
-                throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "The machine_id argument must be a constant UInt64");
+
+            // Check if the provided machine_id is a constant UInt64
+            if (!isColumnConst(*col_machine_id) || !isNativeUInt(col_machine_id->getDataType()))
+            {
+                // If invalid machine_id, resize vector to 0
+                vec_to.resize(0);
+                return col_res;
+            }
 
             machine_id = col_machine_id->getUInt(0);
-            if (col_machine_id->getDataType() != TypeIndex::UInt64)
-                throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "The machine_id argument must be of type UInt64");
-
             /// Truncate machine id to 10 bits
             machine_id &= (1ull << machine_id_bits_count) - 1;
         }
diff --git a/tests/queries/0_stateless/03130_generateSnowflakeId.reference b/tests/queries/0_stateless/03130_generateSnowflakeId.reference
index dff613b29d3..b8a5829fed7 100644
--- a/tests/queries/0_stateless/03130_generateSnowflakeId.reference
+++ b/tests/queries/0_stateless/03130_generateSnowflakeId.reference
@@ -5,5 +5,4 @@
 1
 0
 1
-1
 100
diff --git a/tests/queries/0_stateless/03130_generateSnowflakeId.sql b/tests/queries/0_stateless/03130_generateSnowflakeId.sql
index be49f678dca..c641cfd4bf1 100644
--- a/tests/queries/0_stateless/03130_generateSnowflakeId.sql
+++ b/tests/queries/0_stateless/03130_generateSnowflakeId.sql
@@ -10,9 +10,9 @@ SELECT generateSnowflakeID('expr', 1) != generateSnowflakeID('expr', 2); -- diff
 
 SELECT bitAnd(generateSnowflakeID(1023), 1023) = 1023; -- check if the last 10 bits match the machine ID
 
-SELECT generateSnowflakeID('invalid_machine_id'); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT }
+SELECT generateSnowflakeID('invalid_machine_id'); -- no output for invalid type
 
-SELECT generateSnowflakeID(materialize(toUInt64(1))) = generateSnowflakeID(materialize(toUInt64(1))); -- Test with non-const machine ID
+SELECT generateSnowflakeID(materialize(toUInt64(1))); -- no output for non-const machine ID
 
 SELECT count(*)
 FROM

From 1c868356b8a1ce6e698547f7c1d2b19b2d14c095 Mon Sep 17 00:00:00 2001
From: Haydn <zawalee33@gmail.com>
Date: Sun, 14 Jul 2024 11:38:11 -0700
Subject: [PATCH 0229/1488] Fix generateSnowflakeID function to handle invalid
 inputs gracefully and update test queries

---
 src/Functions/generateSnowflakeID.cpp | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/src/Functions/generateSnowflakeID.cpp b/src/Functions/generateSnowflakeID.cpp
index 2db40b10496..812c992203d 100644
--- a/src/Functions/generateSnowflakeID.cpp
+++ b/src/Functions/generateSnowflakeID.cpp
@@ -184,15 +184,12 @@ public:
         if (arguments.size() == 2 && input_rows_count > 0)
         {
             const auto & col_machine_id = arguments[1].column;
-
-            // Check if the provided machine_id is a constant UInt64
-            if (!isColumnConst(*col_machine_id) || !isNativeUInt(col_machine_id->getDataType()))
+            if (!isColumnConst(*col_machine_id))
             {
-                // If invalid machine_id, resize vector to 0
-                vec_to.resize(0);
+                // Return an empty column if machine ID is not constant
+                vec_to.clear();
                 return col_res;
             }
-
             machine_id = col_machine_id->getUInt(0);
             /// Truncate machine id to 10 bits
             machine_id &= (1ull << machine_id_bits_count) - 1;

From a61304508f07694354c1f698ad8c8c6ba5b65edc Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 14 Jul 2024 21:47:09 +0200
Subject: [PATCH 0230/1488] Remove noise from unit tests

---
 src/Common/Exception.h                              |  9 +++++++++
 src/Common/tests/gtest_shell_command.cpp            | 13 -------------
 .../MergeTree/MergeTreeBackgroundExecutor.cpp       |  4 ++++
 src/Storages/MergeTree/tests/gtest_executor.cpp     |  4 ++--
 4 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/src/Common/Exception.h b/src/Common/Exception.h
index a4774a89f6a..4e54c411bf1 100644
--- a/src/Common/Exception.h
+++ b/src/Common/Exception.h
@@ -244,6 +244,15 @@ private:
     const char * className() const noexcept override { return "DB::ErrnoException"; }
 };
 
+/// An exception to use in unit tests to test interfaces.
+/// It is distinguished from others, so it does not have to be logged.
+class TestException : public Exception
+{
+public:
+    using Exception::Exception;
+};
+
+
 using Exceptions = std::vector<std::exception_ptr>;
 
 /** Try to write an exception to the log (and forget about it).
diff --git a/src/Common/tests/gtest_shell_command.cpp b/src/Common/tests/gtest_shell_command.cpp
index d6d0a544e9b..0ea96da9da2 100644
--- a/src/Common/tests/gtest_shell_command.cpp
+++ b/src/Common/tests/gtest_shell_command.cpp
@@ -54,16 +54,3 @@ TEST(ShellCommand, ExecuteWithInput)
 
     EXPECT_EQ(res, "Hello, world!\n");
 }
-
-TEST(ShellCommand, AutoWait)
-{
-    // <defunct> hunting:
-    for (int i = 0; i < 1000; ++i)
-    {
-        auto command = ShellCommand::execute("echo " + std::to_string(i));
-        //command->wait(); // now automatic
-    }
-
-    // std::cerr << "inspect me: ps auxwwf\n";
-    // std::this_thread::sleep_for(std::chrono::seconds(100));
-}
diff --git a/src/Storages/MergeTree/MergeTreeBackgroundExecutor.cpp b/src/Storages/MergeTree/MergeTreeBackgroundExecutor.cpp
index 8cb0badc19b..4e62c503d65 100644
--- a/src/Storages/MergeTree/MergeTreeBackgroundExecutor.cpp
+++ b/src/Storages/MergeTree/MergeTreeBackgroundExecutor.cpp
@@ -155,6 +155,10 @@ void printExceptionWithRespectToAbort(LoggerPtr log, const String & query_id)
     {
         std::rethrow_exception(ex);
     }
+    catch (const TestException &)
+    {
+        /// Exception from a unit test, ignore it.
+    }
     catch (const Exception & e)
     {
         NOEXCEPT_SCOPE({
diff --git a/src/Storages/MergeTree/tests/gtest_executor.cpp b/src/Storages/MergeTree/tests/gtest_executor.cpp
index 6f34eb4dfbd..c7057ce87c6 100644
--- a/src/Storages/MergeTree/tests/gtest_executor.cpp
+++ b/src/Storages/MergeTree/tests/gtest_executor.cpp
@@ -34,7 +34,7 @@ public:
 
         auto choice = distribution(generator);
         if (choice == 0)
-            throw std::runtime_error("Unlucky...");
+            throw TestException();
 
         return false;
     }
@@ -48,7 +48,7 @@ public:
     {
         auto choice = distribution(generator);
         if (choice == 0)
-            throw std::runtime_error("Unlucky...");
+            throw TestException();
     }
 
     Priority getPriority() const override { return {}; }

From 619152eadb702b23a439517e0ab0e848d1ade466 Mon Sep 17 00:00:00 2001
From: Haydn <zawalee33@gmail.com>
Date: Sun, 14 Jul 2024 14:13:01 -0700
Subject: [PATCH 0231/1488] Retry with returning nullptr

---
 src/Functions/generateSnowflakeID.cpp | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/src/Functions/generateSnowflakeID.cpp b/src/Functions/generateSnowflakeID.cpp
index 812c992203d..3b68e634708 100644
--- a/src/Functions/generateSnowflakeID.cpp
+++ b/src/Functions/generateSnowflakeID.cpp
@@ -185,11 +185,7 @@ public:
         {
             const auto & col_machine_id = arguments[1].column;
             if (!isColumnConst(*col_machine_id))
-            {
-                // Return an empty column if machine ID is not constant
-                vec_to.clear();
-                return col_res;
-            }
+                return nullptr;
             machine_id = col_machine_id->getUInt(0);
             /// Truncate machine id to 10 bits
             machine_id &= (1ull << machine_id_bits_count) - 1;

From 4e56b66a9f71b9e339e03f941b5016868ccdd337 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 14 Jul 2024 23:16:53 +0200
Subject: [PATCH 0232/1488] Remove unused local variables

---
 src/Parsers/ExpressionElementParsers.h   | 2 +-
 src/Parsers/ParserDescribeTableQuery.cpp | 4 ----
 2 files changed, 1 insertion(+), 5 deletions(-)

diff --git a/src/Parsers/ExpressionElementParsers.h b/src/Parsers/ExpressionElementParsers.h
index 4e3f29bfe0c..0209e785bff 100644
--- a/src/Parsers/ExpressionElementParsers.h
+++ b/src/Parsers/ExpressionElementParsers.h
@@ -9,7 +9,7 @@ namespace DB
 {
 
 
-/** The SELECT subquery is in parenthesis.
+/** The SELECT subquery, in parentheses.
   */
 class ParserSubquery : public IParserBase
 {
diff --git a/src/Parsers/ParserDescribeTableQuery.cpp b/src/Parsers/ParserDescribeTableQuery.cpp
index 92c0cfacd9b..22bbfdb03e1 100644
--- a/src/Parsers/ParserDescribeTableQuery.cpp
+++ b/src/Parsers/ParserDescribeTableQuery.cpp
@@ -11,15 +11,12 @@
 namespace DB
 {
 
-
 bool ParserDescribeTableQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
 {
     ParserKeyword s_describe(Keyword::DESCRIBE);
     ParserKeyword s_desc(Keyword::DESC);
     ParserKeyword s_table(Keyword::TABLE);
     ParserKeyword s_settings(Keyword::SETTINGS);
-    ParserToken s_dot(TokenType::Dot);
-    ParserIdentifier name_p;
     ParserSetQuery parser_settings(true);
 
     ASTPtr database;
@@ -53,5 +50,4 @@ bool ParserDescribeTableQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & ex
     return true;
 }
 
-
 }

From 15ba7534bff6cbd35ae42d42100a3910fe856882 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 15 Jul 2024 02:24:41 +0200
Subject: [PATCH 0233/1488] Fix double whitespace in EXPLAIN AST CREATE

---
 src/Parsers/ASTCreateQuery.cpp | 10 ++++++++++
 src/Parsers/ASTCreateQuery.h   |  2 +-
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/src/Parsers/ASTCreateQuery.cpp b/src/Parsers/ASTCreateQuery.cpp
index d56a2724914..6dc009da9a8 100644
--- a/src/Parsers/ASTCreateQuery.cpp
+++ b/src/Parsers/ASTCreateQuery.cpp
@@ -265,6 +265,16 @@ ASTPtr ASTCreateQuery::clone() const
     return res;
 }
 
+String ASTCreateQuery::getID(char delim) const
+{
+    String res = attach ? "AttachQuery" : "CreateQuery";
+    String database = getDatabase();
+    if (!database.empty())
+        res += (delim + getDatabase());
+    res += (delim + getTable());
+    return res;
+}
+
 void ASTCreateQuery::formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const
 {
     frame.need_parens = false;
diff --git a/src/Parsers/ASTCreateQuery.h b/src/Parsers/ASTCreateQuery.h
index 6fbf045915b..9e4364b1f25 100644
--- a/src/Parsers/ASTCreateQuery.h
+++ b/src/Parsers/ASTCreateQuery.h
@@ -136,7 +136,7 @@ public:
     bool create_or_replace{false};
 
     /** Get the text that identifies this element. */
-    String getID(char delim) const override { return (attach ? "AttachQuery" : "CreateQuery") + (delim + getDatabase()) + delim + getTable(); }
+    String getID(char delim) const override;
 
     ASTPtr clone() const override;
 

From 24545caa3a7898e754d7b1f002c95132dc6a2d8b Mon Sep 17 00:00:00 2001
From: Nikolay Degterinsky <evillique@gmail.com>
Date: Mon, 15 Jul 2024 00:52:31 +0000
Subject: [PATCH 0234/1488] Move parsing into a separate function

---
 .../Access/InterpreterCreateUserQuery.cpp     | 59 ++++++++++---------
 1 file changed, 30 insertions(+), 29 deletions(-)

diff --git a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp
index a09de7e688a..855aa36b159 100644
--- a/src/Interpreters/Access/InterpreterCreateUserQuery.cpp
+++ b/src/Interpreters/Access/InterpreterCreateUserQuery.cpp
@@ -114,6 +114,34 @@ namespace
         else if (query.grantees)
             user.grantees = *query.grantees;
     }
+
+    time_t getValidUntilFromAST(ASTPtr valid_until, ContextPtr context)
+    {
+        if (context)
+            valid_until = evaluateConstantExpressionAsLiteral(valid_until, context);
+
+        const String valid_until_str = checkAndGetLiteralArgument<String>(valid_until, "valid_until");
+
+        if (valid_until_str == "infinity")
+            return 0;
+
+        time_t time = 0;
+        ReadBufferFromString in(valid_until_str);
+
+        if (context)
+        {
+            const auto & time_zone = DateLUT::instance("");
+            const auto & utc_time_zone = DateLUT::instance("UTC");
+
+            parseDateTimeBestEffort(time, in, time_zone, utc_time_zone);
+        }
+        else
+        {
+            readDateTimeText(time, in);
+        }
+
+        return time;
+    }
 }
 
 BlockIO InterpreterCreateUserQuery::execute()
@@ -134,23 +162,7 @@ BlockIO InterpreterCreateUserQuery::execute()
 
     std::optional<time_t> valid_until;
     if (query.valid_until)
-    {
-        const ASTPtr valid_until_literal = evaluateConstantExpressionAsLiteral(query.valid_until, getContext());
-        const String valid_until_str = checkAndGetLiteralArgument<String>(valid_until_literal, "valid_until");
-
-        time_t time = 0;
-
-        if (valid_until_str != "infinity")
-        {
-            const auto & time_zone = DateLUT::instance("");
-            const auto & utc_time_zone = DateLUT::instance("UTC");
-
-            ReadBufferFromString in(valid_until_str);
-            parseDateTimeBestEffort(time, in, time_zone, utc_time_zone);
-        }
-
-        valid_until = time;
-    }
+        valid_until = getValidUntilFromAST(query.valid_until, getContext());
 
     std::optional<RolesOrUsersSet> default_roles_from_query;
     if (query.default_roles)
@@ -261,18 +273,7 @@ void InterpreterCreateUserQuery::updateUserFromQuery(User & user, const ASTCreat
 
     std::optional<time_t> valid_until;
     if (query.valid_until)
-    {
-        const String valid_until_str = checkAndGetLiteralArgument<String>(query.valid_until, "valid_until");
-        time_t time = 0;
-
-        if (valid_until_str != "infinity")
-        {
-            ReadBufferFromString in(valid_until_str);
-            readDateTimeText(time, in);
-        }
-
-        valid_until = time;
-    }
+        valid_until = getValidUntilFromAST(query.valid_until, {});
 
     updateUserFromQueryImpl(user, query, auth_data, {}, {}, {}, {}, valid_until, allow_no_password, allow_plaintext_password, true);
 }

From 09345e6b595952fd2172468c039c7f3bcb3d9494 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 15 Jul 2024 03:33:24 +0200
Subject: [PATCH 0235/1488] Fix bad test `apache_arrow_abort`

---
 tests/queries/0_stateless/02834_apache_arrow_abort.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/02834_apache_arrow_abort.sql b/tests/queries/0_stateless/02834_apache_arrow_abort.sql
index 47e1c5d3951..cb0eaebe554 100644
--- a/tests/queries/0_stateless/02834_apache_arrow_abort.sql
+++ b/tests/queries/0_stateless/02834_apache_arrow_abort.sql
@@ -1,4 +1,4 @@
--- Tags: no-fasttest
+-- Tags: no-fasttest, no-tsan, no-asan, no-msan, no-ubsan
 -- This tests depends on internet access, but it does not matter, because it only has to check that there is no abort due to a bug in Apache Arrow library.
 SET optimize_trivial_insert_select=1;
 INSERT INTO TABLE FUNCTION url('https://clickhouse-public-datasets.s3.amazonaws.com/hits_compatible/athena_partitioned/hits_9.parquet') SELECT * FROM url('https://clickhouse-public-datasets.s3.amazonaws.com/hits_compatible/athena_partitioned/hits_9.parquet'); -- { serverError CANNOT_WRITE_TO_OSTREAM, RECEIVED_ERROR_FROM_REMOTE_IO_SERVER, POCO_EXCEPTION }

From 41327260bad798494fef0914b51d95fa85043057 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 15 Jul 2024 03:47:35 +0200
Subject: [PATCH 0236/1488] Fix MSan report in GRPC

---
 contrib/grpc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contrib/grpc b/contrib/grpc
index f5b7fdc2dff..1716359d2e2 160000
--- a/contrib/grpc
+++ b/contrib/grpc
@@ -1 +1 @@
-Subproject commit f5b7fdc2dff09ada06dbf6c75df298fb40f898df
+Subproject commit 1716359d2e28d304a250f9df0e6c0ccad03de8db

From e55fcb32654dfd9cf1f22cd2411a2c48eea7aced Mon Sep 17 00:00:00 2001
From: Yakov Olkhovskiy <yakov@clickhouse.com>
Date: Mon, 15 Jul 2024 01:48:55 +0000
Subject: [PATCH 0237/1488] add docs to client xml config

---
 programs/client/clickhouse-client.xml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/programs/client/clickhouse-client.xml b/programs/client/clickhouse-client.xml
index d0deb818c1e..c32b63413e9 100644
--- a/programs/client/clickhouse-client.xml
+++ b/programs/client/clickhouse-client.xml
@@ -1,5 +1,6 @@
 <!-- Config set into /etc/clickhouse-client/. It's used if no other configs are found. -->
 <config>
+    <!-- Shorthand for self-signed combination in openSSL section below: <accept-invalid-certificate>1</accept-invalid-certificate> -->
     <openSSL>
         <client> <!-- Used for connection to server's secure tcp port -->
             <loadDefaultCAFile>true</loadDefaultCAFile>
@@ -72,6 +73,7 @@
 
                  Default: "hostname" will be used. -->
             <name>default</name>
+            <!-- For self-signed server certificate when connecting to secure tcp: <accept-invalid-certificate>1</accept-invalid-certificate> -->
             <!-- Host that will be used for connection. -->
             <hostname>127.0.0.1</hostname>
             <port>9000</port>

From d76139699178b5403255150b0774a12332728238 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 15 Jul 2024 04:08:10 +0200
Subject: [PATCH 0238/1488] Fix test

---
 .../0_stateless/02903_parameterized_view_explain_ast.reference  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/02903_parameterized_view_explain_ast.reference b/tests/queries/0_stateless/02903_parameterized_view_explain_ast.reference
index 6ee8d0c3d23..27222968b1a 100644
--- a/tests/queries/0_stateless/02903_parameterized_view_explain_ast.reference
+++ b/tests/queries/0_stateless/02903_parameterized_view_explain_ast.reference
@@ -1,4 +1,4 @@
-CreateQuery  numbers_pv (children 2)
+CreateQuery numbers_pv (children 2)
  Identifier numbers_pv
  SelectWithUnionQuery (children 1)
   ExpressionList (children 1)

From c72045710ff22405ff5a6d7441d33d64c598e9e8 Mon Sep 17 00:00:00 2001
From: Duc Canh Le <duccanh.le@ahrefs.com>
Date: Mon, 15 Jul 2024 03:07:23 +0000
Subject: [PATCH 0239/1488] fix wrong count result when there is
 non-deterministic function in predicate

Signed-off-by: Duc Canh Le <duccanh.le@ahrefs.com>
---
 src/Storages/MergeTree/MergeTreeData.cpp      |  2 +-
 src/Storages/VirtualColumnUtils.cpp           | 23 +++++++++++++------
 src/Storages/VirtualColumnUtils.h             |  2 +-
 ..._with_non_deterministic_function.reference |  2 ++
 ..._count_with_non_deterministic_function.sql |  4 ++++
 5 files changed, 24 insertions(+), 9 deletions(-)
 create mode 100644 tests/queries/0_stateless/03203_count_with_non_deterministic_function.reference
 create mode 100644 tests/queries/0_stateless/03203_count_with_non_deterministic_function.sql

diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index 467a5c82141..fe993f1a435 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -1141,7 +1141,7 @@ std::optional<UInt64> MergeTreeData::totalRowsByPartitionPredicateImpl(
     auto metadata_snapshot = getInMemoryMetadataPtr();
     auto virtual_columns_block = getBlockWithVirtualsForFilter(metadata_snapshot, {parts[0]});
 
-    auto filter_dag = VirtualColumnUtils::splitFilterDagForAllowedInputs(filter_actions_dag->getOutputs().at(0), nullptr);
+    auto filter_dag = VirtualColumnUtils::splitFilterDagForAllowedInputs(filter_actions_dag->getOutputs().at(0), nullptr, /*strict=*/ true);
     if (!filter_dag)
         return {};
 
diff --git a/src/Storages/VirtualColumnUtils.cpp b/src/Storages/VirtualColumnUtils.cpp
index 778c9e13adb..3f16f582d63 100644
--- a/src/Storages/VirtualColumnUtils.cpp
+++ b/src/Storages/VirtualColumnUtils.cpp
@@ -271,7 +271,8 @@ bool isDeterministicInScopeOfQuery(const ActionsDAG::Node * node)
 static const ActionsDAG::Node * splitFilterNodeForAllowedInputs(
     const ActionsDAG::Node * node,
     const Block * allowed_inputs,
-    ActionsDAG::Nodes & additional_nodes)
+    ActionsDAG::Nodes & additional_nodes,
+    bool strict)
 {
     if (node->type == ActionsDAG::ActionType::FUNCTION)
     {
@@ -280,8 +281,16 @@ static const ActionsDAG::Node * splitFilterNodeForAllowedInputs(
             auto & node_copy = additional_nodes.emplace_back(*node);
             node_copy.children.clear();
             for (const auto * child : node->children)
-                if (const auto * child_copy = splitFilterNodeForAllowedInputs(child, allowed_inputs, additional_nodes))
+                if (const auto * child_copy = splitFilterNodeForAllowedInputs(child, allowed_inputs, additional_nodes, strict))
+                {
+                    /// Expression like (now_allowed AND allowed) is not allowed if strict = true. This is important for
+                    /// trivial count optimization, otherwise we can get incorrect results. For example, if the query is
+                    /// SELECT count() FROM table WHERE _partition_id = '0' AND rowNumberInBlock() = 1, we cannot apply
+                    /// trivial count.
+                    if (strict)
+                        return nullptr;
                     node_copy.children.push_back(child_copy);
+                }
 
             if (node_copy.children.empty())
                 return nullptr;
@@ -307,7 +316,7 @@ static const ActionsDAG::Node * splitFilterNodeForAllowedInputs(
         {
             auto & node_copy = additional_nodes.emplace_back(*node);
             for (auto & child : node_copy.children)
-                if (child = splitFilterNodeForAllowedInputs(child, allowed_inputs, additional_nodes); !child)
+                if (child = splitFilterNodeForAllowedInputs(child, allowed_inputs, additional_nodes, strict); !child)
                     return nullptr;
 
             return &node_copy;
@@ -321,7 +330,7 @@ static const ActionsDAG::Node * splitFilterNodeForAllowedInputs(
                     auto index_hint_dag = index_hint->getActions()->clone();
                     ActionsDAG::NodeRawConstPtrs atoms;
                     for (const auto & output : index_hint_dag->getOutputs())
-                        if (const auto * child_copy = splitFilterNodeForAllowedInputs(output, allowed_inputs, additional_nodes))
+                        if (const auto * child_copy = splitFilterNodeForAllowedInputs(output, allowed_inputs, additional_nodes, strict))
                             atoms.push_back(child_copy);
 
                     if (!atoms.empty())
@@ -355,13 +364,13 @@ static const ActionsDAG::Node * splitFilterNodeForAllowedInputs(
     return node;
 }
 
-ActionsDAGPtr splitFilterDagForAllowedInputs(const ActionsDAG::Node * predicate, const Block * allowed_inputs)
+ActionsDAGPtr splitFilterDagForAllowedInputs(const ActionsDAG::Node * predicate, const Block * allowed_inputs, bool strict)
 {
     if (!predicate)
         return nullptr;
 
     ActionsDAG::Nodes additional_nodes;
-    const auto * res = splitFilterNodeForAllowedInputs(predicate, allowed_inputs, additional_nodes);
+    const auto * res = splitFilterNodeForAllowedInputs(predicate, allowed_inputs, additional_nodes, strict);
     if (!res)
         return nullptr;
 
@@ -370,7 +379,7 @@ ActionsDAGPtr splitFilterDagForAllowedInputs(const ActionsDAG::Node * predicate,
 
 void filterBlockWithPredicate(const ActionsDAG::Node * predicate, Block & block, ContextPtr context)
 {
-    auto dag = splitFilterDagForAllowedInputs(predicate, &block);
+    auto dag = splitFilterDagForAllowedInputs(predicate, &block,  /*strict=*/ false);
     if (dag)
         filterBlockWithDAG(dag, block, context);
 }
diff --git a/src/Storages/VirtualColumnUtils.h b/src/Storages/VirtualColumnUtils.h
index fbfbdd6c6cc..48a0e4fc97f 100644
--- a/src/Storages/VirtualColumnUtils.h
+++ b/src/Storages/VirtualColumnUtils.h
@@ -32,7 +32,7 @@ void buildSetsForDAG(const ActionsDAGPtr & dag, const ContextPtr & context);
 bool isDeterministicInScopeOfQuery(const ActionsDAG::Node * node);
 
 /// Extract a part of predicate that can be evaluated using only columns from input_names.
-ActionsDAGPtr splitFilterDagForAllowedInputs(const ActionsDAG::Node * predicate, const Block * allowed_inputs);
+ActionsDAGPtr splitFilterDagForAllowedInputs(const ActionsDAG::Node * predicate, const Block * allowed_inputs, bool strict = false);
 
 /// Extract from the input stream a set of `name` column values
 template <typename T>
diff --git a/tests/queries/0_stateless/03203_count_with_non_deterministic_function.reference b/tests/queries/0_stateless/03203_count_with_non_deterministic_function.reference
new file mode 100644
index 00000000000..6ed281c757a
--- /dev/null
+++ b/tests/queries/0_stateless/03203_count_with_non_deterministic_function.reference
@@ -0,0 +1,2 @@
+1
+1
diff --git a/tests/queries/0_stateless/03203_count_with_non_deterministic_function.sql b/tests/queries/0_stateless/03203_count_with_non_deterministic_function.sql
new file mode 100644
index 00000000000..d4ffa4d07ac
--- /dev/null
+++ b/tests/queries/0_stateless/03203_count_with_non_deterministic_function.sql
@@ -0,0 +1,4 @@
+CREATE TABLE t (p UInt8, x UInt64) Engine = MergeTree PARTITION BY p ORDER BY x;
+INSERT INTO t SELECT 0, number FROM numbers(10);
+SELECT count() FROM t WHERE p = 0 AND rowNumberInAllBlocks() = 1 SETTINGS allow_experimental_analyzer = 0;
+SELECT count() FROM t WHERE p = 0 AND rowNumberInAllBlocks() = 1 SETTINGS allow_experimental_analyzer = 1;

From f10cb5ffb750fb96996bcfdef0f90e396e32fb31 Mon Sep 17 00:00:00 2001
From: JackyWoo <wuchienchao@qq.com>
Date: Mon, 15 Jul 2024 12:04:44 +0800
Subject: [PATCH 0240/1488] Some fixups

---
 .../ConditionSelectivityEstimator.cpp         |  9 ++--
 src/Storages/Statistics/Statistics.h          |  2 +-
 .../Statistics/StatisticsCountMinSketch.cpp   | 43 ++++---------------
 .../Statistics/StatisticsCountMinSketch.h     | 10 ++---
 src/Storages/Statistics/StatisticsTDigest.cpp |  4 +-
 src/Storages/Statistics/tests/gtest_stats.cpp |  2 +-
 6 files changed, 21 insertions(+), 49 deletions(-)

diff --git a/src/Storages/Statistics/ConditionSelectivityEstimator.cpp b/src/Storages/Statistics/ConditionSelectivityEstimator.cpp
index 19e3157d99a..57dff958b9a 100644
--- a/src/Storages/Statistics/ConditionSelectivityEstimator.cpp
+++ b/src/Storages/Statistics/ConditionSelectivityEstimator.cpp
@@ -37,9 +37,9 @@ Float64 ConditionSelectivityEstimator::ColumnSelectivityEstimator::estimateGreat
 
 Float64 ConditionSelectivityEstimator::ColumnSelectivityEstimator::estimateEqual(const Field & val, Float64 rows) const
 {
-    auto float_val = StatisticsUtils::tryConvertToFloat64(val);
     if (part_statistics.empty())
     {
+        auto float_val = StatisticsUtils::tryConvertToFloat64(val);
         if (!float_val)
             return default_unknown_cond_factor * rows;
         else if (float_val.value() < - threshold || float_val.value() > threshold)
@@ -133,9 +133,8 @@ Float64 ConditionSelectivityEstimator::estimateRowCount(const RPNBuilderTreeNode
 {
     auto result = tryToExtractSingleColumn(node);
     if (result.second != 1)
-    {
         return default_unknown_cond_factor * total_rows;
-    }
+
     String col = result.first;
     auto it = column_estimators.find(col);
 
@@ -147,12 +146,14 @@ Float64 ConditionSelectivityEstimator::estimateRowCount(const RPNBuilderTreeNode
         estimator = it->second;
     else
         dummy = true;
+
     auto [op, val] = extractBinaryOp(node, col);
-    auto float_val = StatisticsUtils::tryConvertToFloat64(val);
+
     if (op == "equals")
     {
         if (dummy)
         {
+            auto float_val = StatisticsUtils::tryConvertToFloat64(val);
             if (!float_val || (float_val < - threshold || float_val > threshold))
                 return default_normal_cond_factor * total_rows;
             else
diff --git a/src/Storages/Statistics/Statistics.h b/src/Storages/Statistics/Statistics.h
index 33a5cbac4de..16f0c67eabd 100644
--- a/src/Storages/Statistics/Statistics.h
+++ b/src/Storages/Statistics/Statistics.h
@@ -1,9 +1,9 @@
 #pragma once
 
 #include <Core/Block.h>
+#include <Core/Field.h>
 #include <IO/ReadBuffer.h>
 #include <IO/WriteBuffer.h>
-#include <Core/Field.h>
 #include <Storages/StatisticsDescription.h>
 
 #include <boost/core/noncopyable.hpp>
diff --git a/src/Storages/Statistics/StatisticsCountMinSketch.cpp b/src/Storages/Statistics/StatisticsCountMinSketch.cpp
index a3c6ee8a819..95a8ceda8c8 100644
--- a/src/Storages/Statistics/StatisticsCountMinSketch.cpp
+++ b/src/Storages/Statistics/StatisticsCountMinSketch.cpp
@@ -1,11 +1,11 @@
+
+#include <Storages/Statistics/StatisticsCountMinSketch.h>
 #include <DataTypes/DataTypeLowCardinality.h>
 #include <DataTypes/DataTypeNullable.h>
 #include <IO/ReadHelpers.h>
 #include <IO/WriteHelpers.h>
-#include <Storages/Statistics/StatisticsCountMinSketch.h>
 #include <Interpreters/convertFieldToType.h>
 
-
 #if USE_DATASKETCHES
 
 namespace DB
@@ -14,6 +14,7 @@ namespace DB
 namespace ErrorCodes
 {
 extern const int LOGICAL_ERROR;
+extern const int ILLEGAL_STATISTICS;
 }
 
 /// Constants chosen based on rolling dices, which provides an error tolerance of 0.1% (ε = 0.001) and a confidence level of 99.9% (δ = 0.001).
@@ -34,18 +35,9 @@ Float64 StatisticsCountMinSketch::estimateEqual(const Field & val) const
     /// For example: if data_type is Int32:
     ///     1. For 1.0, 1, '1', return Field(1)
     ///     2. For 1.1, max_value_int64, return null
-    Field val_converted;
-    try
-    {
-        val_converted = convertFieldToType(val, *data_type);
-        if (val_converted.isNull())
-            return 0;
-    }
-    catch (...)
-    {
-        /// If the conversion fails for example, when converting 'not a number' to Int32, return 0
+    Field val_converted = convertFieldToType(val, *data_type);
+    if (val_converted.isNull())
         return 0;
-    }
 
     if (data_type->isValueRepresentedByNumber())
         return sketch.get_estimate(&val_converted, data_type->getSizeOfValueInMemory());
@@ -87,19 +79,6 @@ void StatisticsCountMinSketch::deserialize(ReadBuffer & buf)
     sketch = Sketch::deserialize(bytes.data(), size);
 }
 
-}
-
-#endif
-
-
-namespace DB
-{
-
-namespace ErrorCodes
-{
-extern const int FEATURE_IS_NOT_ENABLED_AT_BUILD_TIME;
-extern const int ILLEGAL_STATISTICS;
-}
 
 void CountMinSketchValidator(const SingleStatisticsDescription &, DataTypePtr data_type)
 {
@@ -109,18 +88,12 @@ void CountMinSketchValidator(const SingleStatisticsDescription &, DataTypePtr da
     if (!data_type->isValueRepresentedByNumber() && !isStringOrFixedString(data_type))
         throw Exception(ErrorCodes::ILLEGAL_STATISTICS, "Statistics of type 'count_min' does not support type {}", data_type->getName());
 }
-#if USE_DATASKETCHES
+
 StatisticsPtr CountMinSketchCreator(const SingleStatisticsDescription & stat, DataTypePtr data_type)
 {
     return std::make_shared<StatisticsCountMinSketch>(stat, data_type);
 }
-#else
-StatisticsPtr CountMinSketchCreator(const SingleStatisticsDescription &, DataTypePtr)
-{
-    throw Exception(
-        ErrorCodes::FEATURE_IS_NOT_ENABLED_AT_BUILD_TIME,
-        "Statistics of type 'count_min' is not supported in this build, to enable it turn on USE_DATASKETCHES when building.");
-}
-#endif
 
 }
+
+#endif
diff --git a/src/Storages/Statistics/StatisticsCountMinSketch.h b/src/Storages/Statistics/StatisticsCountMinSketch.h
index d6141f9f73a..aa71c643c05 100644
--- a/src/Storages/Statistics/StatisticsCountMinSketch.h
+++ b/src/Storages/Statistics/StatisticsCountMinSketch.h
@@ -30,12 +30,10 @@ private:
     DataTypePtr data_type;
 };
 
+
+void CountMinSketchValidator(const SingleStatisticsDescription &, DataTypePtr data_type);
+StatisticsPtr CountMinSketchCreator(const SingleStatisticsDescription & stat, DataTypePtr);
+
 }
 
 #endif
-
-namespace DB
-{
-void CountMinSketchValidator(const SingleStatisticsDescription &, DataTypePtr data_type);
-StatisticsPtr CountMinSketchCreator(const SingleStatisticsDescription & stat, DataTypePtr);
-}
diff --git a/src/Storages/Statistics/StatisticsTDigest.cpp b/src/Storages/Statistics/StatisticsTDigest.cpp
index c19d0a0328c..f3c6ce1566b 100644
--- a/src/Storages/Statistics/StatisticsTDigest.cpp
+++ b/src/Storages/Statistics/StatisticsTDigest.cpp
@@ -7,7 +7,7 @@ namespace DB
 namespace ErrorCodes
 {
 extern const int ILLEGAL_STATISTICS;
-extern const int ILLEGAL_TYPE_OF_ARGUMENT;
+extern const int LOGICAL_ERROR;
 }
 
 StatisticsTDigest::StatisticsTDigest(const SingleStatisticsDescription & stat_)
@@ -54,7 +54,7 @@ Float64 StatisticsTDigest::estimateEqual(const Field & val) const
     auto val_as_float = StatisticsUtils::tryConvertToFloat64(val);
     if (val_as_float)
         return t_digest.getCountEqual(*val_as_float);
-    throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Statistics 'tdigest' does not support estimate constant value of type {}", val.getTypeName());
+    throw Exception(ErrorCodes::LOGICAL_ERROR, "Statistics 'tdigest' does not support estimating value of type {}", val.getTypeName());
 }
 
 void TDigestValidator(const SingleStatisticsDescription &, DataTypePtr data_type)
diff --git a/src/Storages/Statistics/tests/gtest_stats.cpp b/src/Storages/Statistics/tests/gtest_stats.cpp
index d80f64b8b6b..f82a535bebc 100644
--- a/src/Storages/Statistics/tests/gtest_stats.cpp
+++ b/src/Storages/Statistics/tests/gtest_stats.cpp
@@ -59,7 +59,7 @@ void testConvertFieldToDataType(const DataTypePtr & data_type, const Fields & fi
         }
         catch(...)
         {
-            /// Just ignore exceptions
+            ASSERT_FALSE(convert_failed);
         }
         if (convert_failed)
             ASSERT_TRUE(converted_value.isNull());

From e8aa157c56e035449b90e3c428f27db0c40263b5 Mon Sep 17 00:00:00 2001
From: JackyWoo <wuchienchao@qq.com>
Date: Mon, 15 Jul 2024 14:25:39 +0800
Subject: [PATCH 0241/1488] Fix code style

---
 src/Storages/Statistics/StatisticsTDigest.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Storages/Statistics/StatisticsTDigest.cpp b/src/Storages/Statistics/StatisticsTDigest.cpp
index f3c6ce1566b..e3a59f3251a 100644
--- a/src/Storages/Statistics/StatisticsTDigest.cpp
+++ b/src/Storages/Statistics/StatisticsTDigest.cpp
@@ -46,7 +46,7 @@ Float64 StatisticsTDigest::estimateLess(const Field & val) const
     auto val_as_float = StatisticsUtils::tryConvertToFloat64(val);
     if (val_as_float)
         return t_digest.getCountLessThan(*val_as_float);
-    throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Statistics 'tdigest' does not support estimate constant value of type {}", val.getTypeName());
+    throw Exception(ErrorCodes::LOGICAL_ERROR, "Statistics 'tdigest' does not support estimate value of type {}", val.getTypeName());
 }
 
 Float64 StatisticsTDigest::estimateEqual(const Field & val) const

From 5b0bcf79f024514fbc2e2d7850d9eb1fdfd25e64 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Mon, 15 Jul 2024 06:39:33 +0000
Subject: [PATCH 0242/1488] Some fixups

---
 .../sql-reference/functions/uuid-functions.md |  7 ++---
 src/Functions/generateSnowflakeID.cpp         | 29 +++++++++----------
 .../03130_generateSnowflakeId.reference       |  9 ++++--
 .../0_stateless/03130_generateSnowflakeId.sql | 25 +++++++++-------
 4 files changed, 36 insertions(+), 34 deletions(-)

diff --git a/docs/en/sql-reference/functions/uuid-functions.md b/docs/en/sql-reference/functions/uuid-functions.md
index f1abb90b455..0bde207dcc9 100644
--- a/docs/en/sql-reference/functions/uuid-functions.md
+++ b/docs/en/sql-reference/functions/uuid-functions.md
@@ -573,8 +573,7 @@ generateSnowflakeID([expr, [machine_id]])
 **Arguments**
 
 - `expr` — An arbitrary [expression](../../sql-reference/syntax.md#syntax-expressions) used to bypass [common subexpression elimination](../../sql-reference/functions/index.md#common-subexpression-elimination) if the function is called multiple times in a query. The value of the expression has no effect on the returned Snowflake ID. Optional.
-
-- `machine_id` — A machine ID, the 10 least significant bits are used. Optional.
+- `machine_id` — A machine ID, the lowest 10 bits are used. [Int64](../data-types/int-uint.md). Optional.
 
 **Returned value**
 
@@ -610,7 +609,8 @@ SELECT generateSnowflakeID(1), generateSnowflakeID(2);
 └────────────────────────┴────────────────────────┘
 ```
 
-Example with expression and machine ID
+**Example with expression and a machine ID**
+
 ```
 SELECT generateSnowflakeID('expr', 1);
 
@@ -619,7 +619,6 @@ SELECT generateSnowflakeID('expr', 1);
 └────────────────────────────────┘
 ```
 
-
 ## snowflakeToDateTime
 
 :::warning
diff --git a/src/Functions/generateSnowflakeID.cpp b/src/Functions/generateSnowflakeID.cpp
index 3b68e634708..c95e3edf4ca 100644
--- a/src/Functions/generateSnowflakeID.cpp
+++ b/src/Functions/generateSnowflakeID.cpp
@@ -18,7 +18,7 @@ namespace
   https://en.wikipedia.org/wiki/Snowflake_ID
 
  0                   1                   2                   3
- 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
 ├─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┤
 |0|                         timestamp                           |
 ├─┼                 ┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┼─┤
@@ -129,7 +129,7 @@ struct Data
     /// Guarantee counter monotonicity within one timestamp across all threads generating Snowflake IDs simultaneously.
     static inline std::atomic<uint64_t> lowest_available_snowflake_id = 0;
 
-    SnowflakeId reserveRange(size_t input_rows_count, uint64_t machine_id)
+    SnowflakeId reserveRange(uint64_t machine_id, size_t input_rows_count)
     {
         uint64_t available_snowflake_id = lowest_available_snowflake_id.load();
         SnowflakeIdRange range;
@@ -178,23 +178,20 @@ public:
     {
         auto col_res = ColumnVector<UInt64>::create();
         typename ColumnVector<UInt64>::Container & vec_to = col_res->getData();
-        vec_to.resize(input_rows_count);
-
-        uint64_t machine_id = getMachineId();
-        if (arguments.size() == 2 && input_rows_count > 0)
-        {
-            const auto & col_machine_id = arguments[1].column;
-            if (!isColumnConst(*col_machine_id))
-                return nullptr;
-            machine_id = col_machine_id->getUInt(0);
-            /// Truncate machine id to 10 bits
-            machine_id &= (1ull << machine_id_bits_count) - 1;
-        }
 
         if (input_rows_count > 0)
         {
+            vec_to.resize(input_rows_count);
+
+            uint64_t machine_id = getMachineId();
+            if (arguments.size() == 2)
+            {
+                machine_id = arguments[1].column->getUInt(0);
+                machine_id &= (1ull << machine_id_bits_count) - 1;
+            }
+
             Data data;
-            SnowflakeId snowflake_id = data.reserveRange(input_rows_count, machine_id);
+            SnowflakeId snowflake_id = data.reserveRange(machine_id, input_rows_count);
 
             for (UInt64 & to_row : vec_to)
             {
@@ -223,7 +220,7 @@ REGISTER_FUNCTION(GenerateSnowflakeID)
     FunctionDocumentation::Syntax syntax = "generateSnowflakeID([expression, [machine_id]])";
     FunctionDocumentation::Arguments arguments = {
         {"expression", "The expression is used to bypass common subexpression elimination if the function is called multiple times in a query but otherwise ignored. Optional."},
-        {"machine_id", "A machine ID, the 10 least significant bits are used. Optional."}
+        {"machine_id", "A machine ID, the lowest 10 bits are used. Optional."}
     };
     FunctionDocumentation::ReturnedValue returned_value = "A value of type UInt64";
     FunctionDocumentation::Examples examples = {{"no_arguments", "SELECT generateSnowflakeID()", "7201148511606784000"}, {"with_machine_id", "SELECT generateSnowflakeID(1)", "7201148511606784001"}, {"with_expression_and_machine_id", "SELECT generateSnowflakeID('some_expression', 1)", "7201148511606784002"}};
diff --git a/tests/queries/0_stateless/03130_generateSnowflakeId.reference b/tests/queries/0_stateless/03130_generateSnowflakeId.reference
index b8a5829fed7..fd264f00d36 100644
--- a/tests/queries/0_stateless/03130_generateSnowflakeId.reference
+++ b/tests/queries/0_stateless/03130_generateSnowflakeId.reference
@@ -1,8 +1,11 @@
+Negative tests
+The first bit must be zero
 1
-1
+Test disabling of common subexpression elimination via first parameter
+0
 0
 1
+Test user-provided machine ID
 1
-0
-1
+Generated Snowflake IDs are unique
 100
diff --git a/tests/queries/0_stateless/03130_generateSnowflakeId.sql b/tests/queries/0_stateless/03130_generateSnowflakeId.sql
index c641cfd4bf1..6c0a6cc0f2c 100644
--- a/tests/queries/0_stateless/03130_generateSnowflakeId.sql
+++ b/tests/queries/0_stateless/03130_generateSnowflakeId.sql
@@ -1,19 +1,22 @@
 -- Test SQL function 'generateSnowflakeID'
 
-SELECT bitAnd(bitShiftRight(toUInt64(generateSnowflakeID()), 63), 1) = 0; -- check first bit is zero
-SELECT generateSnowflakeID() = generateSnowflakeID(1); -- same as ^^
+SELECT 'Negative tests';
+SELECT generateSnowflakeID(1, 2, 3); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH }
+SELECT generateSnowflakeID(1, 'not_an_int'); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT }
+SELECT generateSnowflakeID(1, materialize(2)); -- { serverError ILLEGAL_COLUMN }
+
+ SELECT 'The first bit must be zero';
+SELECT bitAnd(bitShiftRight(generateSnowflakeID(), 63), 1) = 0;
+
+SELECT 'Test disabling of common subexpression elimination via first parameter';
 SELECT generateSnowflakeID(1) = generateSnowflakeID(2); -- disabled common subexpression elimination --> lhs != rhs
-SELECT generateSnowflakeID(1) = generateSnowflakeID(1); -- enabled common subexpression elimination
+SELECT generateSnowflakeID() = generateSnowflakeID(1); -- same as ^^
+SELECT generateSnowflakeID(1) = generateSnowflakeID(1); -- with common subexpression elimination
 
-SELECT generateSnowflakeID('expr', 1) = generateSnowflakeID('expr', 1); -- enabled common subexpression elimination
-SELECT generateSnowflakeID('expr', 1) != generateSnowflakeID('expr', 2); -- different machine IDs should produce different results
-
-SELECT bitAnd(generateSnowflakeID(1023), 1023) = 1023; -- check if the last 10 bits match the machine ID
-
-SELECT generateSnowflakeID('invalid_machine_id'); -- no output for invalid type
-
-SELECT generateSnowflakeID(materialize(toUInt64(1))); -- no output for non-const machine ID
+SELECT 'Test user-provided machine ID';
+SELECT bitAnd(bitShiftRight(generateSnowflakeID(1, 123), 12), 1024 - 1) = 123; -- the machine id is actually set in the generated snowflake ID (1024 = 2^10)
 
+SELECT 'Generated Snowflake IDs are unique';
 SELECT count(*)
 FROM
 (

From 68aebce89f30eb7766c420d25e852e21e27dfe7d Mon Sep 17 00:00:00 2001
From: taiyang-li <654010905@qq.com>
Date: Mon, 15 Jul 2024 14:41:37 +0800
Subject: [PATCH 0243/1488] fix failed uts

---
 src/Functions/printf.cpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/Functions/printf.cpp b/src/Functions/printf.cpp
index a890b886338..3efe854a53b 100644
--- a/src/Functions/printf.cpp
+++ b/src/Functions/printf.cpp
@@ -93,6 +93,8 @@ private:
             {
                 T a = data[i];
                 s = fmt::sprintf(format, static_cast<NearestFieldType<T>>(a));
+
+                res_chars.resize(curr_offset + s.size() + 1);
                 memcpy(&res_chars[curr_offset], s.data(), s.size());
                 res_chars[curr_offset + s.size()] = 0;
 
@@ -115,6 +117,8 @@ private:
             {
                 auto a = concrete_column->getDataAt(i).toView();
                 s = fmt::sprintf(format, a);
+
+                res_chars.resize(curr_offset + s.size() + 1);
                 memcpy(&res_chars[curr_offset], s.data(), s.size());
                 res_chars[curr_offset + s.size()] = 0;
 

From 2132ce52e0f72afe90e72e756d5ef494ad081ea9 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Mon, 15 Jul 2024 06:48:39 +0000
Subject: [PATCH 0244/1488] Bump ICU to 75

---
 contrib/icu                      | 2 +-
 contrib/icu-cmake/CMakeLists.txt | 6 +++---
 contrib/icudata                  | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/contrib/icu b/contrib/icu
index c205e7ee49a..7750081bda4 160000
--- a/contrib/icu
+++ b/contrib/icu
@@ -1 +1 @@
-Subproject commit c205e7ee49a7086a28b9c275fcfdac9ca3dc815d
+Subproject commit 7750081bda4b3bc1768ae03849ec70f67ea10625
diff --git a/contrib/icu-cmake/CMakeLists.txt b/contrib/icu-cmake/CMakeLists.txt
index f23b0002b8d..f9d05f7fe97 100644
--- a/contrib/icu-cmake/CMakeLists.txt
+++ b/contrib/icu-cmake/CMakeLists.txt
@@ -4,7 +4,7 @@ else ()
     option(ENABLE_ICU "Enable ICU" 0)
 endif ()
 
-# Temporarily disabled s390x because the ICU build links a blob (icudt71b_dat.S) and our friends from IBM did not explain how they re-generated
+# Temporarily disabled s390x because the ICU build links a blob (icudt71b_dat.S) and our friends from IBM did not explain how they generated
 # the blob on s390x: https://github.com/ClickHouse/icudata/pull/2#issuecomment-2226957255
 if (NOT ENABLE_ICU OR ARCH_S390X)
     message(STATUS "Not using ICU")
@@ -462,9 +462,9 @@ file(GENERATE OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/empty.cpp" CONTENT " ")
 enable_language(ASM)
 
 if (ARCH_S390X)
-    set(ICUDATA_SOURCE_FILE "${ICUDATA_SOURCE_DIR}/icudt71b_dat.S" )
+    set(ICUDATA_SOURCE_FILE "${ICUDATA_SOURCE_DIR}/icudt75b_dat.S" )
 else()
-    set(ICUDATA_SOURCE_FILE "${ICUDATA_SOURCE_DIR}/icudt71l_dat.S" )
+    set(ICUDATA_SOURCE_FILE "${ICUDATA_SOURCE_DIR}/icudt75l_dat.S" )
 endif()
 
 set(ICUDATA_SOURCES
diff --git a/contrib/icudata b/contrib/icudata
index e7488edd1f1..d345d6ac22f 160000
--- a/contrib/icudata
+++ b/contrib/icudata
@@ -1 +1 @@
-Subproject commit e7488edd1f141b0664553a985a6fcd0125279527
+Subproject commit d345d6ac22f381c882420de9053d30ae1ff38d75

From 77272c925dc15acc5fdd0260a1c0aab35b1df3c3 Mon Sep 17 00:00:00 2001
From: Xu Jia <jia.xu@clickhouse.com>
Date: Mon, 15 Jul 2024 15:10:39 +0800
Subject: [PATCH 0245/1488] disable insertion and mutation

---
 docs/en/operations/settings/settings.md       |  6 ++
 src/Core/ServerSettings.h                     |  1 +
 src/Interpreters/InterpreterAlterQuery.cpp    |  7 ++
 src/Interpreters/InterpreterDeleteQuery.cpp   |  4 +
 src/Interpreters/InterpreterInsertQuery.cpp   |  5 ++
 .../__init__.py                               |  0
 .../config/cluster.xml                        | 16 ++++
 .../config/reading_node.xml                   |  3 +
 .../config/storage_policy.xml                 | 21 +++++
 .../config/writing_node.xml                   |  3 +
 .../test.py                                   | 84 +++++++++++++++++++
 11 files changed, 150 insertions(+)
 create mode 100644 tests/integration/test_disable_insertion_and_mutation/__init__.py
 create mode 100644 tests/integration/test_disable_insertion_and_mutation/config/cluster.xml
 create mode 100644 tests/integration/test_disable_insertion_and_mutation/config/reading_node.xml
 create mode 100644 tests/integration/test_disable_insertion_and_mutation/config/storage_policy.xml
 create mode 100644 tests/integration/test_disable_insertion_and_mutation/config/writing_node.xml
 create mode 100644 tests/integration/test_disable_insertion_and_mutation/test.py

diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md
index c3f697c3bdc..143ce836beb 100644
--- a/docs/en/operations/settings/settings.md
+++ b/docs/en/operations/settings/settings.md
@@ -5608,3 +5608,9 @@ Default value: `10000000`.
 Minimal size of block to compress in CROSS JOIN. Zero value means - disable this threshold. This block is compressed when any of the two thresholds (by rows or by bytes) are reached.
 
 Default value: `1GiB`.
+
+## disable_insertion_and_mutation
+
+Disable all insert and mutations (alter table update / alter table delete / alter table drop partition). Set to true, can make this node focus on reading queries.
+
+Default value: `false`.
diff --git a/src/Core/ServerSettings.h b/src/Core/ServerSettings.h
index 28b32a6e6a5..cf09874125d 100644
--- a/src/Core/ServerSettings.h
+++ b/src/Core/ServerSettings.h
@@ -157,6 +157,7 @@ namespace DB
     M(Bool, prepare_system_log_tables_on_startup, false, "If true, ClickHouse creates all configured `system.*_log` tables before the startup. It can be helpful if some startup scripts depend on these tables.", 0) \
     M(Double, gwp_asan_force_sample_probability, 0.0003, "Probability that an allocation from specific places will be sampled by GWP Asan (i.e. PODArray allocations)", 0) \
     M(UInt64, config_reload_interval_ms, 2000, "How often clickhouse will reload config and check for new changes", 0) \
+    M(Bool, disable_insertion_and_mutation, false, "Disable all insert/alter/delete queries. This setting will be enabled if someone needs read-only nodes to prevent insertion and mutation affect reading performance.", 0)
 
 /// If you add a setting which can be updated at runtime, please update 'changeable_settings' map in StorageSystemServerSettings.cpp
 
diff --git a/src/Interpreters/InterpreterAlterQuery.cpp b/src/Interpreters/InterpreterAlterQuery.cpp
index c70a3397f4e..b9dd59909e6 100644
--- a/src/Interpreters/InterpreterAlterQuery.cpp
+++ b/src/Interpreters/InterpreterAlterQuery.cpp
@@ -46,6 +46,7 @@ namespace ErrorCodes
     extern const int BAD_ARGUMENTS;
     extern const int UNKNOWN_TABLE;
     extern const int UNKNOWN_DATABASE;
+    extern const int QUERY_IS_PROHIBITED;
 }
 
 
@@ -191,6 +192,12 @@ BlockIO InterpreterAlterQuery::executeToTable(const ASTAlterQuery & alter)
                                                          "to execute ALTERs of different types (replicated and non replicated) in single query");
     }
 
+    if (mutation_commands.hasNonEmptyMutationCommands() || !partition_commands.empty())
+    {
+        if (getContext()->getServerSettings().disable_insertion_and_mutation)
+            throw Exception(ErrorCodes::QUERY_IS_PROHIBITED, "Mutations are prohibited");
+    }
+
     if (!alter_commands.empty())
     {
         auto alter_lock = table->lockForAlter(getContext()->getSettingsRef().lock_acquire_timeout);
diff --git a/src/Interpreters/InterpreterDeleteQuery.cpp b/src/Interpreters/InterpreterDeleteQuery.cpp
index 39d5d9e9cef..5f3e3385148 100644
--- a/src/Interpreters/InterpreterDeleteQuery.cpp
+++ b/src/Interpreters/InterpreterDeleteQuery.cpp
@@ -26,6 +26,7 @@ namespace ErrorCodes
     extern const int SUPPORT_IS_DISABLED;
     extern const int BAD_ARGUMENTS;
     extern const int NOT_IMPLEMENTED;
+    extern const int QUERY_IS_PROHIBITED;
 }
 
 
@@ -50,6 +51,9 @@ BlockIO InterpreterDeleteQuery::execute()
     if (table->isStaticStorage())
         throw Exception(ErrorCodes::TABLE_IS_READ_ONLY, "Table is read-only");
 
+    if (getContext()->getGlobalContext()->getServerSettings().disable_insertion_and_mutation)
+        throw Exception(ErrorCodes::QUERY_IS_PROHIBITED, "Delete queries are prohibited");
+
     DatabasePtr database = DatabaseCatalog::instance().getDatabase(table_id.database_name);
     if (database->shouldReplicateQuery(getContext(), query_ptr))
     {
diff --git a/src/Interpreters/InterpreterInsertQuery.cpp b/src/Interpreters/InterpreterInsertQuery.cpp
index f396db70d21..b62a71de884 100644
--- a/src/Interpreters/InterpreterInsertQuery.cpp
+++ b/src/Interpreters/InterpreterInsertQuery.cpp
@@ -44,6 +44,7 @@ namespace ProfileEvents
 {
     extern const Event InsertQueriesWithSubqueries;
     extern const Event QueriesWithSubqueries;
+    extern const int QUERY_IS_PROHIBITED;
 }
 
 namespace DB
@@ -406,6 +407,10 @@ BlockIO InterpreterInsertQuery::execute()
     StoragePtr table = getTable(query);
     checkStorageSupportsTransactionsIfNeeded(table, getContext());
 
+    if (getContext()->getServerSettings().disable_insertion_and_mutation
+        && query.table_id.database_name != DatabaseCatalog::SYSTEM_DATABASE)
+        throw Exception(ErrorCodes::QUERY_IS_PROHIBITED, "Insert queries are prohibited");
+
     StoragePtr inner_table;
     if (const auto * mv = dynamic_cast<const StorageMaterializedView *>(table.get()))
         inner_table = mv->getTargetTable();
diff --git a/tests/integration/test_disable_insertion_and_mutation/__init__.py b/tests/integration/test_disable_insertion_and_mutation/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/integration/test_disable_insertion_and_mutation/config/cluster.xml b/tests/integration/test_disable_insertion_and_mutation/config/cluster.xml
new file mode 100644
index 00000000000..17782a77679
--- /dev/null
+++ b/tests/integration/test_disable_insertion_and_mutation/config/cluster.xml
@@ -0,0 +1,16 @@
+<clickhouse>
+	<remote_servers>
+		<default>
+			<shard>
+				<replica>
+					<host>writing_node</host>
+					<port>9000</port>
+				</replica>
+				<replica>
+					<host>reading_node</host>
+					<port>9000</port>
+				</replica>
+			</shard>
+        </default>
+	</remote_servers>
+</clickhouse>
\ No newline at end of file
diff --git a/tests/integration/test_disable_insertion_and_mutation/config/reading_node.xml b/tests/integration/test_disable_insertion_and_mutation/config/reading_node.xml
new file mode 100644
index 00000000000..becabce8a44
--- /dev/null
+++ b/tests/integration/test_disable_insertion_and_mutation/config/reading_node.xml
@@ -0,0 +1,3 @@
+<clickhouse>
+	<disable_insertion_and_mutation>true</disable_insertion_and_mutation>
+</clickhouse>
\ No newline at end of file
diff --git a/tests/integration/test_disable_insertion_and_mutation/config/storage_policy.xml b/tests/integration/test_disable_insertion_and_mutation/config/storage_policy.xml
new file mode 100644
index 00000000000..cec96cfcc1a
--- /dev/null
+++ b/tests/integration/test_disable_insertion_and_mutation/config/storage_policy.xml
@@ -0,0 +1,21 @@
+<clickhouse>
+    <storage_configuration>
+        <disks>
+            <s3_with_keeper>
+                <type>s3_with_keeper</type>
+                <endpoint>http://minio1:9001/root/data/</endpoint>
+                <access_key_id>minio</access_key_id>
+                <secret_access_key>minio123</secret_access_key>
+            </s3_with_keeper>
+        </disks>
+        <policies>
+            <s3_with_keeper>
+                <volumes>
+                    <main>
+                        <disk>s3_with_keeper</disk>
+                    </main>
+                </volumes>
+            </s3_with_keeper>
+        </policies>
+    </storage_configuration>
+</clickhouse>
\ No newline at end of file
diff --git a/tests/integration/test_disable_insertion_and_mutation/config/writing_node.xml b/tests/integration/test_disable_insertion_and_mutation/config/writing_node.xml
new file mode 100644
index 00000000000..0737af7afc7
--- /dev/null
+++ b/tests/integration/test_disable_insertion_and_mutation/config/writing_node.xml
@@ -0,0 +1,3 @@
+<clickhouse>
+	<disable_insertion_and_mutation>false</disable_insertion_and_mutation>
+</clickhouse>
\ No newline at end of file
diff --git a/tests/integration/test_disable_insertion_and_mutation/test.py b/tests/integration/test_disable_insertion_and_mutation/test.py
new file mode 100644
index 00000000000..5234ae9c57c
--- /dev/null
+++ b/tests/integration/test_disable_insertion_and_mutation/test.py
@@ -0,0 +1,84 @@
+import pytest
+from helpers.client import QueryRuntimeException
+from helpers.cluster import ClickHouseCluster
+import time
+
+cluster = ClickHouseCluster(__file__)
+
+writing_node = cluster.add_instance(
+    "writing_node",
+    main_configs=["config/writing_node.xml", "config/storage_policy.xml", "config/cluster.xml"],
+    with_zookeeper=True,
+    with_minio=True,
+    stay_alive=True,
+    macros={"shard": 1, "replica": 1},
+)
+reading_node = cluster.add_instance(
+    "reading_node",
+    main_configs=["config/reading_node.xml", "config/storage_policy.xml", "config/cluster.xml"],
+    with_zookeeper=True,
+    with_minio=True,
+    stay_alive=True,
+    macros={"shard": 1, "replica": 2},
+)
+
+
+@pytest.fixture(scope="module")
+def started_cluster():
+    try:
+        cluster.start()
+
+        yield cluster
+
+    finally:
+        cluster.shutdown()
+
+
+def test_disable_insertion_and_mutation(started_cluster):
+    writing_node.query("""CREATE TABLE my_table on cluster default (key UInt64, value String) ENGINE=ReplicatedMergeTree('/clickhouse/tables/{shard}/default.my_table', '{replica}') ORDER BY key partition by (key % 5) SETTINGS storage_policy='s3_with_keeper' """)
+
+    assert (
+        "QUERY_IS_PROHIBITED"
+        in reading_node.query_and_get_error("INSERT INTO my_table VALUES (1, 'hello')")
+    )
+
+    assert (
+        "QUERY_IS_PROHIBITED"
+        in reading_node.query_and_get_error("INSERT INTO my_table SETTINGS async_insert = 1 VALUES (1, 'hello')")
+    )
+
+    assert (
+        "QUERY_IS_PROHIBITED"
+        in reading_node.query_and_get_error("ALTER TABLE my_table delete where 1")
+    )
+
+    assert (
+        "QUERY_IS_PROHIBITED"
+        in reading_node.query_and_get_error("ALTER table my_table update key = 1 where 1")
+    )
+
+    assert (
+        "QUERY_IS_PROHIBITED"
+        in reading_node.query_and_get_error("ALTER TABLE my_table drop partition 0")
+    )
+
+    reading_node.query("SELECT * from my_table");
+    writing_node.query("INSERT INTO my_table VALUES (1, 'hello')")
+    writing_node.query("ALTER TABLE my_table delete where 1")
+    writing_node.query("ALTER table my_table update value = 'no hello' where 1")
+
+    reading_node.query("ALTER TABLE my_table ADD COLUMN new_column UInt64")
+    writing_node.query("SELECT new_column from my_table")
+    reading_node.query("SELECT new_column from my_table")
+
+    reading_node.query("ALter Table my_table MODIFY COLUMN new_column String")
+
+    assert(
+        "new_column\tString"
+        in reading_node.query("DESC my_table")
+    )
+
+    assert(
+        "new_column\tString"
+        in writing_node.query("DESC my_table")
+    )

From fc29ac7891eddd3a714f5af574c71040f91f451d Mon Sep 17 00:00:00 2001
From: Xu Jia <jia.xu@clickhouse.com>
Date: Mon, 15 Jul 2024 17:06:37 +0800
Subject: [PATCH 0246/1488] add error extern to fix compile error

---
 src/Interpreters/InterpreterInsertQuery.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/Interpreters/InterpreterInsertQuery.cpp b/src/Interpreters/InterpreterInsertQuery.cpp
index b62a71de884..c01b2196ac9 100644
--- a/src/Interpreters/InterpreterInsertQuery.cpp
+++ b/src/Interpreters/InterpreterInsertQuery.cpp
@@ -56,6 +56,7 @@ namespace ErrorCodes
     extern const int NO_SUCH_COLUMN_IN_TABLE;
     extern const int ILLEGAL_COLUMN;
     extern const int DUPLICATE_COLUMN;
+    extern const int QUERY_IS_PROHIBITED;
 }
 
 InterpreterInsertQuery::InterpreterInsertQuery(

From ccba078da10bed8d42e821f8bcdd47f448d198a0 Mon Sep 17 00:00:00 2001
From: Xu Jia <jia.xu@clickhouse.com>
Date: Mon, 15 Jul 2024 09:46:31 +0000
Subject: [PATCH 0247/1488] change storage policy to default

---
 .../config/storage_policy.xml                 | 21 -------------------
 .../test.py                                   |  6 +++---
 2 files changed, 3 insertions(+), 24 deletions(-)
 delete mode 100644 tests/integration/test_disable_insertion_and_mutation/config/storage_policy.xml

diff --git a/tests/integration/test_disable_insertion_and_mutation/config/storage_policy.xml b/tests/integration/test_disable_insertion_and_mutation/config/storage_policy.xml
deleted file mode 100644
index cec96cfcc1a..00000000000
--- a/tests/integration/test_disable_insertion_and_mutation/config/storage_policy.xml
+++ /dev/null
@@ -1,21 +0,0 @@
-<clickhouse>
-    <storage_configuration>
-        <disks>
-            <s3_with_keeper>
-                <type>s3_with_keeper</type>
-                <endpoint>http://minio1:9001/root/data/</endpoint>
-                <access_key_id>minio</access_key_id>
-                <secret_access_key>minio123</secret_access_key>
-            </s3_with_keeper>
-        </disks>
-        <policies>
-            <s3_with_keeper>
-                <volumes>
-                    <main>
-                        <disk>s3_with_keeper</disk>
-                    </main>
-                </volumes>
-            </s3_with_keeper>
-        </policies>
-    </storage_configuration>
-</clickhouse>
\ No newline at end of file
diff --git a/tests/integration/test_disable_insertion_and_mutation/test.py b/tests/integration/test_disable_insertion_and_mutation/test.py
index 5234ae9c57c..f098f130d2b 100644
--- a/tests/integration/test_disable_insertion_and_mutation/test.py
+++ b/tests/integration/test_disable_insertion_and_mutation/test.py
@@ -7,7 +7,7 @@ cluster = ClickHouseCluster(__file__)
 
 writing_node = cluster.add_instance(
     "writing_node",
-    main_configs=["config/writing_node.xml", "config/storage_policy.xml", "config/cluster.xml"],
+    main_configs=["config/writing_node.xml", "config/cluster.xml"],
     with_zookeeper=True,
     with_minio=True,
     stay_alive=True,
@@ -15,7 +15,7 @@ writing_node = cluster.add_instance(
 )
 reading_node = cluster.add_instance(
     "reading_node",
-    main_configs=["config/reading_node.xml", "config/storage_policy.xml", "config/cluster.xml"],
+    main_configs=["config/reading_node.xml", "config/cluster.xml"],
     with_zookeeper=True,
     with_minio=True,
     stay_alive=True,
@@ -35,7 +35,7 @@ def started_cluster():
 
 
 def test_disable_insertion_and_mutation(started_cluster):
-    writing_node.query("""CREATE TABLE my_table on cluster default (key UInt64, value String) ENGINE=ReplicatedMergeTree('/clickhouse/tables/{shard}/default.my_table', '{replica}') ORDER BY key partition by (key % 5) SETTINGS storage_policy='s3_with_keeper' """)
+    writing_node.query("""CREATE TABLE my_table on cluster default (key UInt64, value String) ENGINE=ReplicatedMergeTree('/clickhouse/tables/{shard}/default.my_table', '{replica}') ORDER BY key partition by (key % 5) """)
 
     assert (
         "QUERY_IS_PROHIBITED"

From 76e1dea7ccdabc27129fdf911845f2622bbbc371 Mon Sep 17 00:00:00 2001
From: JackyWoo <wuchienchao@qq.com>
Date: Mon, 15 Jul 2024 18:17:09 +0800
Subject: [PATCH 0248/1488] Fix unit test

---
 src/Storages/Statistics/tests/gtest_stats.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Storages/Statistics/tests/gtest_stats.cpp b/src/Storages/Statistics/tests/gtest_stats.cpp
index f82a535bebc..9b9fae83109 100644
--- a/src/Storages/Statistics/tests/gtest_stats.cpp
+++ b/src/Storages/Statistics/tests/gtest_stats.cpp
@@ -59,7 +59,7 @@ void testConvertFieldToDataType(const DataTypePtr & data_type, const Fields & fi
         }
         catch(...)
         {
-            ASSERT_FALSE(convert_failed);
+            ASSERT_TRUE(convert_failed);
         }
         if (convert_failed)
             ASSERT_TRUE(converted_value.isNull());

From 0dfcaaa28fd4717fcf217d771333077e86401529 Mon Sep 17 00:00:00 2001
From: Sema Checherinda <Sema.Checherinda@clickhouse.com>
Date: Mon, 15 Jul 2024 13:05:53 +0200
Subject: [PATCH 0249/1488] work with review notes

---
 src/Interpreters/Squashing.cpp | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/src/Interpreters/Squashing.cpp b/src/Interpreters/Squashing.cpp
index a076494a4bc..5fe0ba9a737 100644
--- a/src/Interpreters/Squashing.cpp
+++ b/src/Interpreters/Squashing.cpp
@@ -18,7 +18,6 @@ Squashing::Squashing(Block header_, size_t min_block_size_rows_, size_t min_bloc
     , min_block_size_bytes(min_block_size_bytes_)
     , header(header_)
 {
-    LOG_TEST(getLogger("Squashing"), "header columns {}", header.columns());
 }
 
 Chunk Squashing::flush()
@@ -46,8 +45,6 @@ Chunk Squashing::squash(Chunk && input_chunk)
 
 Chunk Squashing::add(Chunk && input_chunk)
 {
-    LOG_TEST(getLogger("Squashing"), "add columns {} rows {}", input_chunk.getNumColumns(), input_chunk.getNumRows());
-
     if (!input_chunk)
         return {};
 
@@ -88,8 +85,6 @@ Chunk Squashing::add(Chunk && input_chunk)
 
 Chunk Squashing::convertToChunk(CurrentData && data) const
 {
-    LOG_TEST(getLogger("Squashing"), "convertToChunk {}", data.chunks.size());
-
     if (data.chunks.empty())
         return {};
 
@@ -99,7 +94,10 @@ Chunk Squashing::convertToChunk(CurrentData && data) const
     // It is imortant that chunk is not empty, it has to have columns even if they are empty
     // Sometimes there are could be no columns in header but not empty rows in chunks
     // That happens when we intend to add defaults for the missing columns after
-    auto aggr_chunk = Chunk(header.getColumns(), header.columns() ? 0 : data.getRows());
+    auto aggr_chunk = Chunk(header.getColumns(), 0);
+    if (header.columns() == 0)
+        aggr_chunk = Chunk(header.getColumns(), data.getRows());
+
     aggr_chunk.getChunkInfos().add(std::move(info));
     chassert(aggr_chunk);
     return aggr_chunk;

From b2466466d46ab1828b67693edf520d52ff45bc65 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Mon, 15 Jul 2024 14:28:12 +0200
Subject: [PATCH 0250/1488] Rename Context::getSettings()

---
 programs/local/LocalServer.cpp                              | 2 +-
 src/Analyzer/QueryTreeBuilder.cpp                           | 2 +-
 src/Analyzer/Resolve/QueryAnalyzer.cpp                      | 2 +-
 src/Analyzer/Utils.cpp                                      | 2 +-
 src/Bridge/IBridge.cpp                                      | 2 +-
 src/Client/ClientBase.cpp                                   | 6 +++---
 src/Databases/DatabaseDictionary.cpp                        | 4 ++--
 src/Databases/DatabaseOnDisk.cpp                            | 4 ++--
 src/Databases/MySQL/MaterializedMySQLSyncThread.cpp         | 2 +-
 src/Functions/formatQuery.cpp                               | 2 +-
 src/Functions/hasColumnInTable.cpp                          | 2 +-
 src/Interpreters/Context.cpp                                | 2 +-
 src/Interpreters/Context.h                                  | 5 ++---
 src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp         | 2 +-
 src/Interpreters/ExpressionAnalyzer.cpp                     | 4 ++--
 src/Interpreters/InterpreterInsertQuery.cpp                 | 2 +-
 src/Interpreters/InterpreterSelectQuery.cpp                 | 2 +-
 src/Interpreters/JoinedTables.cpp                           | 2 +-
 src/Interpreters/ProcessList.cpp                            | 2 +-
 src/Interpreters/interpretSubquery.cpp                      | 2 +-
 src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp      | 2 +-
 .../QueryPlan/Optimizations/optimizeReadInOrder.cpp         | 2 +-
 src/Server/MySQLHandler.cpp                                 | 2 +-
 .../Distributed/DistributedAsyncInsertDirectoryQueue.cpp    | 2 +-
 src/Storages/MergeTree/MergedBlockOutputStream.cpp          | 2 +-
 src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp     | 2 +-
 src/Storages/StorageURL.cpp                                 | 4 ++--
 src/Storages/StorageView.cpp                                | 2 +-
 src/Storages/getStructureOfRemoteTable.cpp                  | 2 +-
 src/TableFunctions/Hive/TableFunctionHive.cpp               | 2 +-
 30 files changed, 37 insertions(+), 38 deletions(-)

diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp
index 46b543e49e9..a7265ef0de4 100644
--- a/programs/local/LocalServer.cpp
+++ b/programs/local/LocalServer.cpp
@@ -80,7 +80,7 @@ namespace ErrorCodes
 
 void applySettingsOverridesForLocal(ContextMutablePtr context)
 {
-    Settings settings = context->getSettings();
+    Settings settings = context->getSettingsCopy();
 
     settings.allow_introspection_functions = true;
     settings.storage_file_read_method = LocalFSReadMethod::mmap;
diff --git a/src/Analyzer/QueryTreeBuilder.cpp b/src/Analyzer/QueryTreeBuilder.cpp
index a62b6e56ac5..0a732a3b3b3 100644
--- a/src/Analyzer/QueryTreeBuilder.cpp
+++ b/src/Analyzer/QueryTreeBuilder.cpp
@@ -237,7 +237,7 @@ QueryTreeNodePtr QueryTreeBuilder::buildSelectExpression(const ASTPtr & select_q
     /// Remove global settings limit and offset
     if (const auto & settings_ref = updated_context->getSettingsRef(); settings_ref.limit || settings_ref.offset)
     {
-        Settings settings = updated_context->getSettings();
+        Settings settings = updated_context->getSettingsCopy();
         limit = settings.limit;
         offset = settings.offset;
         settings.limit = 0;
diff --git a/src/Analyzer/Resolve/QueryAnalyzer.cpp b/src/Analyzer/Resolve/QueryAnalyzer.cpp
index 5f7b06231d9..92618dfe346 100644
--- a/src/Analyzer/Resolve/QueryAnalyzer.cpp
+++ b/src/Analyzer/Resolve/QueryAnalyzer.cpp
@@ -503,7 +503,7 @@ void QueryAnalyzer::evaluateScalarSubqueryIfNeeded(QueryTreeNodePtr & node, Iden
         ProfileEvents::increment(ProfileEvents::ScalarSubqueriesCacheMiss);
         auto subquery_context = Context::createCopy(context);
 
-        Settings subquery_settings = context->getSettings();
+        Settings subquery_settings = context->getSettingsCopy();
         subquery_settings.max_result_rows = 1;
         subquery_settings.extremes = false;
         subquery_context->setSettings(subquery_settings);
diff --git a/src/Analyzer/Utils.cpp b/src/Analyzer/Utils.cpp
index d10bbd9bd23..e5f372b7368 100644
--- a/src/Analyzer/Utils.cpp
+++ b/src/Analyzer/Utils.cpp
@@ -867,7 +867,7 @@ void updateContextForSubqueryExecution(ContextMutablePtr & mutable_context)
       *  max_rows_in_join, max_bytes_in_join, join_overflow_mode,
       *  which are checked separately (in the Set, Join objects).
       */
-    Settings subquery_settings = mutable_context->getSettings();
+    Settings subquery_settings = mutable_context->getSettingsCopy();
     subquery_settings.max_result_rows = 0;
     subquery_settings.max_result_bytes = 0;
     /// The calculation of extremes does not make sense and is not necessary (if you do it, then the extremes of the subquery can be taken for whole query).
diff --git a/src/Bridge/IBridge.cpp b/src/Bridge/IBridge.cpp
index de48a4f2b84..5682a28f899 100644
--- a/src/Bridge/IBridge.cpp
+++ b/src/Bridge/IBridge.cpp
@@ -232,7 +232,7 @@ int IBridge::main(const std::vector<std::string> & /*args*/)
     auto context = Context::createGlobal(shared_context.get());
     context->makeGlobalContext();
 
-    auto settings = context->getSettings();
+    auto settings = context->getSettingsCopy();
     settings.set("http_max_field_value_size", http_max_field_value_size);
     context->setSettings(settings);
 
diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp
index 5d472ba99b9..2dc603a307f 100644
--- a/src/Client/ClientBase.cpp
+++ b/src/Client/ClientBase.cpp
@@ -723,7 +723,7 @@ void ClientBase::initLogsOutputStream()
 
 void ClientBase::adjustSettings()
 {
-    Settings settings = global_context->getSettings();
+    Settings settings = global_context->getSettingsCopy();
 
     /// NOTE: Do not forget to set changed=false to avoid sending it to the server (to avoid breakage read only profiles)
 
@@ -931,7 +931,7 @@ bool ClientBase::isSyncInsertWithData(const ASTInsertQuery & insert_query, const
     if (!insert_query.data)
         return false;
 
-    auto settings = context->getSettings();
+    auto settings = context->getSettingsCopy();
     if (insert_query.settings_ast)
         settings.applyChanges(insert_query.settings_ast->as<ASTSetQuery>()->changes);
 
@@ -2696,7 +2696,7 @@ bool ClientBase::processMultiQueryFromFile(const String & file_name)
 
     if (!getClientConfiguration().has("log_comment"))
     {
-        Settings settings = global_context->getSettings();
+        Settings settings = global_context->getSettingsCopy();
         /// NOTE: cannot use even weakly_canonical() since it fails for /dev/stdin due to resolving of "pipe:[X]"
         settings.log_comment = fs::absolute(fs::path(file_name));
         global_context->setSettings(settings);
diff --git a/src/Databases/DatabaseDictionary.cpp b/src/Databases/DatabaseDictionary.cpp
index 52196e75c4a..a9569408814 100644
--- a/src/Databases/DatabaseDictionary.cpp
+++ b/src/Databases/DatabaseDictionary.cpp
@@ -111,7 +111,7 @@ ASTPtr DatabaseDictionary::getCreateTableQueryImpl(const String & table_name, Co
         buffer << ") Engine = Dictionary(" << backQuoteIfNeed(table_name) << ")";
     }
 
-    auto settings = getContext()->getSettingsRef();
+    const auto & settings = getContext()->getSettingsRef();
     ParserCreateQuery parser;
     const char * pos = query.data();
     std::string error_message;
@@ -133,7 +133,7 @@ ASTPtr DatabaseDictionary::getCreateDatabaseQuery() const
         if (const auto comment_value = getDatabaseComment(); !comment_value.empty())
             buffer << " COMMENT " << backQuote(comment_value);
     }
-    auto settings = getContext()->getSettingsRef();
+    const auto & settings = getContext()->getSettingsRef();
     ParserCreateQuery parser;
     return parseQuery(parser, query.data(), query.data() + query.size(), "", 0, settings.max_parser_depth, settings.max_parser_backtracks);
 }
diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp
index 07a250e72c7..261a917c595 100644
--- a/src/Databases/DatabaseOnDisk.cpp
+++ b/src/Databases/DatabaseOnDisk.cpp
@@ -523,7 +523,7 @@ ASTPtr DatabaseOnDisk::getCreateDatabaseQuery() const
 {
     ASTPtr ast;
 
-    auto settings = getContext()->getSettingsRef();
+    const auto & settings = getContext()->getSettingsRef();
     {
         std::lock_guard lock(mutex);
         auto database_metadata_path = getContext()->getPath() + "metadata/" + escapeForFileName(database_name) + ".sql";
@@ -722,7 +722,7 @@ ASTPtr DatabaseOnDisk::parseQueryFromMetadata(
         return nullptr;
     }
 
-    auto settings = local_context->getSettingsRef();
+    const auto & settings = local_context->getSettingsRef();
     ParserCreateQuery parser;
     const char * pos = query.data();
     std::string error_message;
diff --git a/src/Databases/MySQL/MaterializedMySQLSyncThread.cpp b/src/Databases/MySQL/MaterializedMySQLSyncThread.cpp
index 2c342755337..04b4070d5af 100644
--- a/src/Databases/MySQL/MaterializedMySQLSyncThread.cpp
+++ b/src/Databases/MySQL/MaterializedMySQLSyncThread.cpp
@@ -89,7 +89,7 @@ static constexpr auto MYSQL_BACKGROUND_THREAD_NAME = "MySQLDBSync";
 
 static ContextMutablePtr createQueryContext(ContextPtr context)
 {
-    Settings new_query_settings = context->getSettings();
+    Settings new_query_settings = context->getSettingsCopy();
     new_query_settings.insert_allow_materialized_columns = true;
 
     /// To avoid call AST::format
diff --git a/src/Functions/formatQuery.cpp b/src/Functions/formatQuery.cpp
index d10b3f9a5b7..f7f7e4b5bcb 100644
--- a/src/Functions/formatQuery.cpp
+++ b/src/Functions/formatQuery.cpp
@@ -39,7 +39,7 @@ public:
     FunctionFormatQuery(ContextPtr context, String name_, OutputFormatting output_formatting_, ErrorHandling error_handling_)
         : name(name_), output_formatting(output_formatting_), error_handling(error_handling_)
     {
-        const Settings & settings = context->getSettings();
+        const Settings & settings = context->getSettingsRef();
         max_query_size = settings.max_query_size;
         max_parser_depth = settings.max_parser_depth;
         max_parser_backtracks = settings.max_parser_backtracks;
diff --git a/src/Functions/hasColumnInTable.cpp b/src/Functions/hasColumnInTable.cpp
index 00714997b4a..cc496270b01 100644
--- a/src/Functions/hasColumnInTable.cpp
+++ b/src/Functions/hasColumnInTable.cpp
@@ -143,7 +143,7 @@ ColumnPtr FunctionHasColumnInTable::executeImpl(const ColumnsWithTypeAndName & a
             /* cluster_name= */ "",
             /* password= */ ""
         };
-        auto cluster = std::make_shared<Cluster>(getContext()->getSettings(), host_names, params);
+        auto cluster = std::make_shared<Cluster>(getContext()->getSettingsRef(), host_names, params);
 
         // FIXME this (probably) needs a non-constant access to query context,
         // because it might initialized a storage. Ideally, the tables required
diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp
index fc1e87e7b7e..3a88e0ccfe1 100644
--- a/src/Interpreters/Context.cpp
+++ b/src/Interpreters/Context.cpp
@@ -2267,7 +2267,7 @@ bool Context::displaySecretsInShowAndSelect() const
     return shared->server_settings.display_secrets_in_show_and_select;
 }
 
-Settings Context::getSettings() const
+Settings Context::getSettingsCopy() const
 {
     SharedLockGuard lock(mutex);
     return *settings;
diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h
index 284cac50769..61095e53a17 100644
--- a/src/Interpreters/Context.h
+++ b/src/Interpreters/Context.h
@@ -830,7 +830,8 @@ public:
     void setMacros(std::unique_ptr<Macros> && macros);
 
     bool displaySecretsInShowAndSelect() const;
-    Settings getSettings() const;
+    Settings getSettingsCopy() const;
+    const Settings & getSettingsRef() const { return *settings; }
     void setSettings(const Settings & settings_);
 
     /// Set settings by name.
@@ -955,8 +956,6 @@ public:
     void makeSessionContext();
     void makeGlobalContext();
 
-    const Settings & getSettingsRef() const { return *settings; }
-
     void setProgressCallback(ProgressCallback callback);
     /// Used in executeQuery() to pass it to the QueryPipeline.
     ProgressCallback getProgressCallback() const;
diff --git a/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp b/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp
index 4bd1c47d5a0..1ca8c40460c 100644
--- a/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp
+++ b/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp
@@ -74,7 +74,7 @@ void ExecuteScalarSubqueriesMatcher::visit(ASTPtr & ast, Data & data)
 static auto getQueryInterpreter(const ASTSubquery & subquery, ExecuteScalarSubqueriesMatcher::Data & data)
 {
     auto subquery_context = Context::createCopy(data.getContext());
-    Settings subquery_settings = data.getContext()->getSettings();
+    Settings subquery_settings = data.getContext()->getSettingsCopy();
     subquery_settings.max_result_rows = 1;
     subquery_settings.extremes = false;
     subquery_context->setSettings(subquery_settings);
diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp
index 16d0eb71278..1a4c02bdebb 100644
--- a/src/Interpreters/ExpressionAnalyzer.cpp
+++ b/src/Interpreters/ExpressionAnalyzer.cpp
@@ -171,7 +171,7 @@ ExpressionAnalyzer::ExpressionAnalyzer(
     PreparedSetsPtr prepared_sets_,
     bool is_create_parameterized_view_)
     : WithContext(context_)
-    , query(query_), settings(getContext()->getSettings())
+    , query(query_), settings(getContext()->getSettingsRef())
     , subquery_depth(subquery_depth_)
     , syntax(syntax_analyzer_result_)
     , is_create_parameterized_view(is_create_parameterized_view_)
@@ -983,7 +983,7 @@ static std::shared_ptr<IJoin> tryCreateJoin(
         algorithm == JoinAlgorithm::PARALLEL_HASH ||
         algorithm == JoinAlgorithm::DEFAULT)
     {
-        const auto & settings = context->getSettings();
+        const auto & settings = context->getSettingsRef();
 
         if (analyzed_join->allowParallelHashJoin())
             return std::make_shared<ConcurrentHashJoin>(
diff --git a/src/Interpreters/InterpreterInsertQuery.cpp b/src/Interpreters/InterpreterInsertQuery.cpp
index ef298d4d45a..dffa0cbaa5b 100644
--- a/src/Interpreters/InterpreterInsertQuery.cpp
+++ b/src/Interpreters/InterpreterInsertQuery.cpp
@@ -469,7 +469,7 @@ BlockIO InterpreterInsertQuery::execute()
                   * to avoid unnecessary squashing.
                   */
 
-                Settings new_settings = getContext()->getSettings();
+                Settings new_settings = getContext()->getSettingsCopy();
 
                 new_settings.max_threads = std::max<UInt64>(1, settings.max_insert_threads);
 
diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp
index cd91f9532b9..cb3c478dbb1 100644
--- a/src/Interpreters/InterpreterSelectQuery.cpp
+++ b/src/Interpreters/InterpreterSelectQuery.cpp
@@ -253,7 +253,7 @@ namespace
 ContextPtr getSubqueryContext(const ContextPtr & context)
 {
     auto subquery_context = Context::createCopy(context);
-    Settings subquery_settings = context->getSettings();
+    Settings subquery_settings = context->getSettingsCopy();
     subquery_settings.max_result_rows = 0;
     subquery_settings.max_result_bytes = 0;
     /// The calculation of extremes does not make sense and is not necessary (if you do it, then the extremes of the subquery can be taken for whole query).
diff --git a/src/Interpreters/JoinedTables.cpp b/src/Interpreters/JoinedTables.cpp
index 0de2bf9cb1f..c5226107f8d 100644
--- a/src/Interpreters/JoinedTables.cpp
+++ b/src/Interpreters/JoinedTables.cpp
@@ -308,7 +308,7 @@ std::shared_ptr<TableJoin> JoinedTables::makeTableJoin(const ASTSelectQuery & se
     if (tables_with_columns.size() < 2)
         return {};
 
-    auto settings = context->getSettingsRef();
+    const auto & settings = context->getSettingsRef();
     MultiEnum<JoinAlgorithm> join_algorithm = settings.join_algorithm;
     bool try_use_direct_join = join_algorithm.isSet(JoinAlgorithm::DIRECT) || join_algorithm.isSet(JoinAlgorithm::DEFAULT);
     auto table_join = std::make_shared<TableJoin>(settings, context->getGlobalTemporaryVolume(), context->getTempDataOnDisk());
diff --git a/src/Interpreters/ProcessList.cpp b/src/Interpreters/ProcessList.cpp
index 5b07852d9e3..271e23a7288 100644
--- a/src/Interpreters/ProcessList.cpp
+++ b/src/Interpreters/ProcessList.cpp
@@ -657,7 +657,7 @@ QueryStatusInfo QueryStatus::getInfo(bool get_thread_list, bool get_profile_even
     {
         if (auto ctx = context.lock())
         {
-            res.query_settings = std::make_shared<Settings>(ctx->getSettings());
+            res.query_settings = std::make_shared<Settings>(ctx->getSettingsRef());
             res.current_database = ctx->getCurrentDatabase();
         }
     }
diff --git a/src/Interpreters/interpretSubquery.cpp b/src/Interpreters/interpretSubquery.cpp
index 340f6d1d805..909875b99a0 100644
--- a/src/Interpreters/interpretSubquery.cpp
+++ b/src/Interpreters/interpretSubquery.cpp
@@ -62,7 +62,7 @@ std::shared_ptr<InterpreterSelectWithUnionQuery> interpretSubquery(
       *  which are checked separately (in the Set, Join objects).
       */
     auto subquery_context = Context::createCopy(context);
-    Settings subquery_settings = context->getSettings();
+    Settings subquery_settings = context->getSettingsCopy();
     subquery_settings.max_result_rows = 0;
     subquery_settings.max_result_bytes = 0;
     /// The calculation of `extremes` does not make sense and is not necessary (if you do it, then the `extremes` of the subquery can be taken instead of the whole query).
diff --git a/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp b/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp
index c23d717d52f..e467c358d1d 100644
--- a/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp
+++ b/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp
@@ -406,7 +406,7 @@ bool ValuesBlockInputFormat::parseExpression(IColumn & column, size_t column_idx
 {
     const Block & header = getPort().getHeader();
     const IDataType & type = *header.getByPosition(column_idx).type;
-    auto settings = context->getSettingsRef();
+    const auto & settings = context->getSettingsRef();
 
     /// Advance the token iterator until the start of the column expression
     readUntilTheEndOfRowAndReTokenize(column_idx);
diff --git a/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp b/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp
index 28eb4da2e17..415a6a11999 100644
--- a/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp
+++ b/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp
@@ -1057,7 +1057,7 @@ size_t tryReuseStorageOrderingForWindowFunctions(QueryPlan::Node * parent_node,
     }
 
     auto context = read_from_merge_tree->getContext();
-    const auto & settings = context->getSettings();
+    const auto & settings = context->getSettingsRef();
     if (!settings.optimize_read_in_window_order || (settings.optimize_read_in_order && settings.query_plan_read_in_order) || context->getSettingsRef().allow_experimental_analyzer)
     {
         return 0;
diff --git a/src/Server/MySQLHandler.cpp b/src/Server/MySQLHandler.cpp
index b6d795b1e69..3deb09bae88 100644
--- a/src/Server/MySQLHandler.cpp
+++ b/src/Server/MySQLHandler.cpp
@@ -473,7 +473,7 @@ void MySQLHandler::comQuery(ReadBuffer & payload, bool binary_protocol)
         query_context->setCurrentQueryId(fmt::format("mysql:{}:{}", connection_id, toString(UUIDHelpers::generateV4())));
 
         /// --- Workaround for Bug 56173. Can be removed when the analyzer is on by default.
-        auto settings = query_context->getSettings();
+        auto settings = query_context->getSettingsCopy();
         settings.prefer_column_name_to_alias = true;
         query_context->setSettings(settings);
 
diff --git a/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp b/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp
index d471c67553d..c287fc817eb 100644
--- a/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp
+++ b/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp
@@ -283,7 +283,7 @@ ConnectionPoolWithFailoverPtr DistributedAsyncInsertDirectoryQueue::createPool(c
 
     auto pools = createPoolsForAddresses(addresses, pool_factory, storage.log);
 
-    const auto settings = storage.getContext()->getSettings();
+    const auto & settings = storage.getContext()->getSettingsRef();
     return std::make_shared<ConnectionPoolWithFailover>(std::move(pools),
         settings.load_balancing,
         settings.distributed_replica_error_half_life.totalSeconds(),
diff --git a/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/src/Storages/MergeTree/MergedBlockOutputStream.cpp
index 38869aebaa5..4ee68580d3f 100644
--- a/src/Storages/MergeTree/MergedBlockOutputStream.cpp
+++ b/src/Storages/MergeTree/MergedBlockOutputStream.cpp
@@ -34,7 +34,7 @@ MergedBlockOutputStream::MergedBlockOutputStream(
     , write_settings(write_settings_)
 {
     MergeTreeWriterSettings writer_settings(
-        data_part->storage.getContext()->getSettings(),
+        data_part->storage.getContext()->getSettingsRef(),
         write_settings,
         storage_settings,
         data_part->index_granularity_info.mark_type.adaptive,
diff --git a/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp b/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp
index c167ac87317..05cd77dcd40 100644
--- a/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp
+++ b/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp
@@ -23,7 +23,7 @@ MergedColumnOnlyOutputStream::MergedColumnOnlyOutputStream(
     const MergeTreeIndexGranularityInfo * index_granularity_info)
     : IMergedBlockOutputStream(data_part->storage.getSettings(), data_part->getDataPartStoragePtr(), metadata_snapshot_, columns_list_, /*reset_columns=*/ true)
 {
-    const auto & global_settings = data_part->storage.getContext()->getSettings();
+    const auto & global_settings = data_part->storage.getContext()->getSettingsRef();
 
     MergeTreeWriterSettings writer_settings(
         global_settings,
diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp
index 9cec8c75ebe..731bd7ec3d3 100644
--- a/src/Storages/StorageURL.cpp
+++ b/src/Storages/StorageURL.cpp
@@ -462,7 +462,7 @@ std::pair<Poco::URI, std::unique_ptr<ReadWriteBufferFromHTTP>> StorageURLSource:
 
         setCredentials(credentials, request_uri);
 
-        const auto settings = context_->getSettings();
+        const auto & settings = context_->getSettingsRef();
 
         auto proxy_config = getProxyConfiguration(request_uri.getScheme());
 
@@ -1324,7 +1324,7 @@ std::optional<time_t> IStorageURLBase::tryGetLastModificationTime(
     const Poco::Net::HTTPBasicCredentials & credentials,
     const ContextPtr & context)
 {
-    auto settings = context->getSettingsRef();
+    const auto & settings = context->getSettingsRef();
 
     auto uri = Poco::URI(url);
 
diff --git a/src/Storages/StorageView.cpp b/src/Storages/StorageView.cpp
index 5f768bce978..929896e3246 100644
--- a/src/Storages/StorageView.cpp
+++ b/src/Storages/StorageView.cpp
@@ -97,7 +97,7 @@ bool hasJoin(const ASTSelectWithUnionQuery & ast)
 ContextPtr getViewContext(ContextPtr context, const StorageSnapshotPtr & storage_snapshot)
 {
     auto view_context = storage_snapshot->metadata->getSQLSecurityOverriddenContext(context);
-    Settings view_settings = view_context->getSettings();
+    Settings view_settings = view_context->getSettingsCopy();
     view_settings.max_result_rows = 0;
     view_settings.max_result_bytes = 0;
     view_settings.extremes = false;
diff --git a/src/Storages/getStructureOfRemoteTable.cpp b/src/Storages/getStructureOfRemoteTable.cpp
index 56071abaa95..9d23f132759 100644
--- a/src/Storages/getStructureOfRemoteTable.cpp
+++ b/src/Storages/getStructureOfRemoteTable.cpp
@@ -65,7 +65,7 @@ ColumnsDescription getStructureOfRemoteTableInShard(
     /// Ignore limit for result number of rows (that could be set during handling CSE/CTE),
     /// since this is a service query and should not lead to query failure.
     {
-        Settings new_settings = new_context->getSettings();
+        Settings new_settings = new_context->getSettingsCopy();
         new_settings.max_result_rows = 0;
         new_settings.max_result_bytes = 0;
         new_context->setSettings(new_settings);
diff --git a/src/TableFunctions/Hive/TableFunctionHive.cpp b/src/TableFunctions/Hive/TableFunctionHive.cpp
index 80494dbe5a8..759807d7a4f 100644
--- a/src/TableFunctions/Hive/TableFunctionHive.cpp
+++ b/src/TableFunctions/Hive/TableFunctionHive.cpp
@@ -93,7 +93,7 @@ StoragePtr TableFunctionHive::executeImpl(
     ColumnsDescription /*cached_columns_*/,
     bool /*is_insert_query*/) const
 {
-    const Settings & settings = context_->getSettings();
+    const Settings & settings = context_->getSettingsRef();
     ParserExpression partition_by_parser;
     ASTPtr partition_by_ast = parseQuery(
         partition_by_parser,

From bcd08b89531ba0734a689be64f41b46064f0f8b9 Mon Sep 17 00:00:00 2001
From: robot-clickhouse <robot-clickhouse@users.noreply.github.com>
Date: Mon, 15 Jul 2024 12:50:53 +0000
Subject: [PATCH 0251/1488] Automatic style fix

---
 tests/queries/0_stateless/03008_deduplication.python | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/tests/queries/0_stateless/03008_deduplication.python b/tests/queries/0_stateless/03008_deduplication.python
index dd1058518c9..bfaa496805f 100644
--- a/tests/queries/0_stateless/03008_deduplication.python
+++ b/tests/queries/0_stateless/03008_deduplication.python
@@ -83,9 +83,11 @@ def instance_insert_statement(
             template,
             table_name=table_name,
             count=count,
-            insert_columns="'src_4', 4"
-            if not insert_unique_blocks
-            else "'src_' || toString(number), number",
+            insert_columns=(
+                "'src_4', 4"
+                if not insert_unique_blocks
+                else "'src_' || toString(number), number"
+            ),
             insert_settings=insert_settings,
         )
 

From 981a2c4658cab15cbb6fe17ffabf74bc4562103a Mon Sep 17 00:00:00 2001
From: Yakov Olkhovskiy <99031427+yakov-olkhovskiy@users.noreply.github.com>
Date: Mon, 15 Jul 2024 09:27:22 -0400
Subject: [PATCH 0252/1488] Update clickhouse-client.xml


From e346535641bff8998c126f5fb74f02d63014090a Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Mon, 15 Jul 2024 16:27:53 +0200
Subject: [PATCH 0253/1488] fix log in keeper tcp handler

---
 src/Server/KeeperTCPHandler.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Server/KeeperTCPHandler.cpp b/src/Server/KeeperTCPHandler.cpp
index 4849f5827c1..aa79d8814a7 100644
--- a/src/Server/KeeperTCPHandler.cpp
+++ b/src/Server/KeeperTCPHandler.cpp
@@ -623,7 +623,7 @@ void KeeperTCPHandler::updateStats(Coordination::ZooKeeperResponsePtr & response
             LOG_INFO(
                 log,
                 "Total time to process a request took too long ({}ms).\nRequest info: {}",
-                elapsed,
+                elapsed_ms,
                 request->toString(/*short_format=*/true));
         }
 

From dec5931262ab4dfe541abfca3fb1e7b67f3d9a5e Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Mon, 15 Jul 2024 16:43:41 +0200
Subject: [PATCH 0254/1488] log session id

---
 src/Server/KeeperTCPHandler.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/Server/KeeperTCPHandler.cpp b/src/Server/KeeperTCPHandler.cpp
index aa79d8814a7..b61df45133a 100644
--- a/src/Server/KeeperTCPHandler.cpp
+++ b/src/Server/KeeperTCPHandler.cpp
@@ -622,7 +622,8 @@ void KeeperTCPHandler::updateStats(Coordination::ZooKeeperResponsePtr & response
         {
             LOG_INFO(
                 log,
-                "Total time to process a request took too long ({}ms).\nRequest info: {}",
+                "Total time to process a request in session {} took too long ({}ms).\nRequest info: {}",
+                session_id,
                 elapsed_ms,
                 request->toString(/*short_format=*/true));
         }

From 22b37d526bacfa281372211a81a3daf1518ca5d6 Mon Sep 17 00:00:00 2001
From: Blargian <shaunstruwig.ss@gmail.com>
Date: Mon, 15 Jul 2024 17:00:47 +0200
Subject: [PATCH 0255/1488] update joingGet and add joinGetOrNull

---
 .../functions/other-functions.md              | 138 +++++++++++++++---
 1 file changed, 121 insertions(+), 17 deletions(-)

diff --git a/docs/en/sql-reference/functions/other-functions.md b/docs/en/sql-reference/functions/other-functions.md
index 260457b3be1..12d082fe0f3 100644
--- a/docs/en/sql-reference/functions/other-functions.md
+++ b/docs/en/sql-reference/functions/other-functions.md
@@ -2449,11 +2449,11 @@ As you can see, `runningAccumulate` merges states for each group of rows separat
 
 ## joinGet
 
-The function lets you extract data from the table the same way as from a [dictionary](../../sql-reference/dictionaries/index.md).
-
-Gets the data from [Join](../../engines/table-engines/special/join.md#creating-a-table) tables using the specified join key.
+The function lets you extract data from the table the same way as from a [dictionary](../../sql-reference/dictionaries/index.md). Gets the data from [Join](../../engines/table-engines/special/join.md#creating-a-table) tables using the specified join key.
 
+:::note
 Only supports tables created with the `ENGINE = Join(ANY, LEFT, <join_keys>)` statement.
+:::
 
 **Syntax**
 
@@ -2463,26 +2463,32 @@ joinGet(join_storage_table_name, `value_column`, join_keys)
 
 **Arguments**
 
-- `join_storage_table_name` — an [identifier](../../sql-reference/syntax.md#syntax-identifiers) indicating where the search is performed. The identifier is searched in the default database (see setting `default_database` in the config file). To override the default database, use `USE db_name` or specify the database and the table through the separator `db_name.db_table` as in the example.
+- `join_storage_table_name` — an [identifier](../../sql-reference/syntax.md#syntax-identifiers) indicating where the search is performed.
 - `value_column` — name of the column of the table that contains required data.
 - `join_keys` — list of keys.
 
+:::note
+The identifier is searched for in the default database (see setting `default_database` in the config file). To override the default database, use `USE db_name` or specify the database and the table through the separator `db_name.db_table` as in the example.
+:::
+
 **Returned value**
 
-Returns a list of values corresponded to list of keys.
-
-If certain does not exist in source table then `0` or `null` will be returned based on [join_use_nulls](../../operations/settings/settings.md#join_use_nulls) setting.
+- Returns a list of values corresponded to the list of keys.
 
+:::note
+If a certain key does not exist in source table then `0` or `null` will be returned based on [join_use_nulls](../../operations/settings/settings.md#join_use_nulls) setting during table creation.
 More info about `join_use_nulls` in [Join operation](../../engines/table-engines/special/join.md).
+:::
 
 **Example**
 
 Input table:
 
 ```sql
-CREATE DATABASE db_test
-CREATE TABLE db_test.id_val(`id` UInt32, `val` UInt32) ENGINE = Join(ANY, LEFT, id) SETTINGS join_use_nulls = 1
-INSERT INTO db_test.id_val VALUES (1,11)(2,12)(4,13)
+CREATE DATABASE db_test;
+CREATE TABLE db_test.id_val(`id` UInt32, `val` UInt32) ENGINE = Join(ANY, LEFT, id);
+INSERT INTO db_test.id_val VALUES (1, 11)(2, 12)(4, 13);
+SELECT * FROM db_test.id_val;
 ```
 
 ```text
@@ -2496,18 +2502,116 @@ INSERT INTO db_test.id_val VALUES (1,11)(2,12)(4,13)
 Query:
 
 ```sql
-SELECT joinGet(db_test.id_val, 'val', toUInt32(number)) from numbers(4) SETTINGS join_use_nulls = 1
+SELECT number, joinGet(db_test.id_val, 'val', toUInt32(number)) from numbers(4);
 ```
 
 Result:
 
 ```text
-┌─joinGet(db_test.id_val, 'val', toUInt32(number))─┐
-│                                                0 │
-│                                               11 │
-│                                               12 │
-│                                                0 │
-└──────────────────────────────────────────────────┘
+   ┌─number─┬─joinGet('db_test.id_val', 'val', toUInt32(number))─┐
+1. │      0 │                                                  0 │
+2. │      1 │                                                 11 │
+3. │      2 │                                                 12 │
+4. │      3 │                                                  0 │
+   └────────┴────────────────────────────────────────────────────┘
+```
+
+Setting `join_use_nulls` can be used during table creation to change the behaviour of what gets returned if no key exists in the source table.
+
+```sql
+CREATE DATABASE db_test;
+CREATE TABLE db_test.id_val_nulls(`id` UInt32, `val` UInt32) ENGINE = Join(ANY, LEFT, id) SETTINGS join_use_nulls=1;
+INSERT INTO db_test.id_val_nulls VALUES (1, 11)(2, 12)(4, 13);
+SELECT * FROM db_test.id_val_nulls;
+```
+
+```text
+┌─id─┬─val─┐
+│  4 │  13 │
+│  2 │  12 │
+│  1 │  11 │
+└────┴─────┘
+```
+
+Query:
+
+```sql
+SELECT number, joinGet(db_test.id_val_nulls, 'val', toUInt32(number)) from numbers(4);
+```
+
+Result:
+
+```text
+   ┌─number─┬─joinGet('db_test.id_val_nulls', 'val', toUInt32(number))─┐
+1. │      0 │                                                     ᴺᵁᴸᴸ │
+2. │      1 │                                                       11 │
+3. │      2 │                                                       12 │
+4. │      3 │                                                     ᴺᵁᴸᴸ │
+   └────────┴──────────────────────────────────────────────────────────┘
+```
+
+## joinGetOrNull
+
+Like [joinGet](#joinget) but returns `NULL` when the key is missing instead of returning the default value.
+
+**Syntax**
+
+```sql
+joinGetOrNull(join_storage_table_name, `value_column`, join_keys)
+```
+
+**Arguments**
+
+- `join_storage_table_name` — an [identifier](../../sql-reference/syntax.md#syntax-identifiers) indicating where the search is performed. 
+- `value_column` — name of the column of the table that contains required data.
+- `join_keys` — list of keys.
+
+:::note
+The identifier is searched for in the default database (see setting `default_database` in the config file). To override the default database, use `USE db_name` or specify the database and the table through the separator `db_name.db_table` as in the example.
+:::
+
+**Returned value**
+
+- Returns a list of values corresponded to the list of keys.
+
+:::note
+If a certain key does not exist in source table then `NULL` is returned for that key.
+:::
+
+**Example**
+
+Input table:
+
+```sql
+CREATE DATABASE db_test;
+CREATE TABLE db_test.id_val(`id` UInt32, `val` UInt32) ENGINE = Join(ANY, LEFT, id);
+INSERT INTO db_test.id_val VALUES (1, 11)(2, 12)(4, 13);
+SELECT * FROM db_test.id_val;
+```
+
+```text
+┌─id─┬─val─┐
+│  4 │  13 │
+│  2 │  12 │
+│  1 │  11 │
+└────┴─────┘
+```
+
+Query:
+
+```sql
+SELECT number, joinGetOrNull(db_test.id_val, 'val', toUInt32(number)) from numbers(4);
+```
+
+Result:
+
+```text
+   ┌─number─┬─joinGetOrNull('db_test.id_val', 'val', toUInt32(number))─┐
+1. │      0 │                                                     ᴺᵁᴸᴸ │
+2. │      1 │                                                       11 │
+3. │      2 │                                                       12 │
+4. │      3 │                                                     ᴺᵁᴸᴸ │
+   └────────┴──────────────────────────────────────────────────────────┘
 ```
 
 ## catboostEvaluate

From 38027abbecfc7b60594ed1073ce9104133152aa9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?=
 <benjamin.antal@clickhouse.com>
Date: Mon, 8 Jul 2024 13:06:50 +0000
Subject: [PATCH 0256/1488] Fix bug in `numbers` when both limit and offset is
 used but the index cannot be used

---
 src/Storages/System/StorageSystemNumbers.cpp                 | 5 +++++
 .../03203_system_numbers_limit_and_offset.reference          | 3 +++
 .../0_stateless/03203_system_numbers_limit_and_offset.sql    | 5 +++++
 3 files changed, 13 insertions(+)
 create mode 100644 tests/queries/0_stateless/03203_system_numbers_limit_and_offset.reference
 create mode 100644 tests/queries/0_stateless/03203_system_numbers_limit_and_offset.sql

diff --git a/src/Storages/System/StorageSystemNumbers.cpp b/src/Storages/System/StorageSystemNumbers.cpp
index da700a7a4e9..6ddfb5e3213 100644
--- a/src/Storages/System/StorageSystemNumbers.cpp
+++ b/src/Storages/System/StorageSystemNumbers.cpp
@@ -38,6 +38,11 @@ void StorageSystemNumbers::read(
     size_t max_block_size,
     size_t num_streams)
 {
+    if (limit && *limit < max_block_size)
+    {
+        max_block_size = static_cast<size_t>(*limit);
+        multithreaded = false;
+    }
     query_plan.addStep(std::make_unique<ReadFromSystemNumbersStep>(
         column_names, query_info, storage_snapshot, context, shared_from_this(), max_block_size, num_streams));
 }
diff --git a/tests/queries/0_stateless/03203_system_numbers_limit_and_offset.reference b/tests/queries/0_stateless/03203_system_numbers_limit_and_offset.reference
new file mode 100644
index 00000000000..d55fa7c71fc
--- /dev/null
+++ b/tests/queries/0_stateless/03203_system_numbers_limit_and_offset.reference
@@ -0,0 +1,3 @@
+9900
+9910
+9920
diff --git a/tests/queries/0_stateless/03203_system_numbers_limit_and_offset.sql b/tests/queries/0_stateless/03203_system_numbers_limit_and_offset.sql
new file mode 100644
index 00000000000..6b7f998c4d4
--- /dev/null
+++ b/tests/queries/0_stateless/03203_system_numbers_limit_and_offset.sql
@@ -0,0 +1,5 @@
+SELECT number FROM numbers_mt(10000)
+WHERE (number % 10) = 0
+ORDER BY number ASC
+LIMIT 990, 3;
+

From a177bb63077353e15f6780ae775c9519f2795695 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?=
 <benjamin.antal@clickhouse.com>
Date: Mon, 8 Jul 2024 16:14:20 +0000
Subject: [PATCH 0257/1488] Really fix the bug

---
 .../QueryPlan/ReadFromSystemNumbersStep.cpp   | 35 +++++++------------
 src/Storages/System/StorageSystemNumbers.cpp  |  5 ---
 ..._system_numbers_limit_and_offset.reference |  4 +++
 .../03203_system_numbers_limit_and_offset.sql | 34 +++++++++++++++++-
 4 files changed, 49 insertions(+), 29 deletions(-)

diff --git a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp
index a3ae035afdd..049fe762640 100644
--- a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp
+++ b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp
@@ -39,15 +39,10 @@ inline void iotaWithStepOptimized(T * begin, size_t count, T first_value, T step
 class NumbersSource : public ISource
 {
 public:
-    NumbersSource(UInt64 block_size_, UInt64 offset_, std::optional<UInt64> limit_, UInt64 chunk_step_, const std::string & column_name, UInt64 step_)
-        : ISource(createHeader(column_name))
-        , block_size(block_size_)
-        , next(offset_)
-        , chunk_step(chunk_step_)
-        , step(step_)
+    NumbersSource(
+        UInt64 block_size_, UInt64 offset_, std::optional<UInt64> end_, UInt64 chunk_step_, const std::string & column_name, UInt64 step_)
+        : ISource(createHeader(column_name)), block_size(block_size_), next(offset_), chunk_step(chunk_step_), end(end_), step(step_)
     {
-        if (limit_.has_value())
-            end = limit_.value() + offset_;
     }
     String getName() const override { return "Numbers"; }
 
@@ -79,7 +74,6 @@ protected:
         next += chunk_step;
 
         progress(column->size(), column->byteSize());
-
         return {Columns{std::move(column)}, real_block_size};
     }
 
@@ -549,13 +543,21 @@ Pipe ReadFromSystemNumbersStep::makePipe()
         return pipe;
     }
 
+    const auto end = std::invoke(
+        [&]() -> std::optional<UInt64>
+        {
+            if (numbers_storage.limit.has_value())
+                return *(numbers_storage.limit) + numbers_storage.offset;
+            return {};
+        });
+
     /// Fall back to NumbersSource
     for (size_t i = 0; i < num_streams; ++i)
     {
         auto source = std::make_shared<NumbersSource>(
             max_block_size,
             numbers_storage.offset + i * max_block_size * numbers_storage.step,
-            numbers_storage.limit,
+            end,
             num_streams * max_block_size * numbers_storage.step,
             numbers_storage.column_name,
             numbers_storage.step);
@@ -571,19 +573,6 @@ Pipe ReadFromSystemNumbersStep::makePipe()
         pipe.addSource(std::move(source));
     }
 
-    if (numbers_storage.limit)
-    {
-        size_t i = 0;
-        auto storage_limit = (*numbers_storage.limit - 1) / numbers_storage.step + 1;
-        /// This formula is how to split 'limit' elements to 'num_streams' chunks almost uniformly.
-        pipe.addSimpleTransform(
-            [&](const Block & header)
-            {
-                ++i;
-                return std::make_shared<LimitTransform>(header, storage_limit * i / num_streams - storage_limit * (i - 1) / num_streams, 0);
-            });
-    }
-
     return pipe;
 }
 
diff --git a/src/Storages/System/StorageSystemNumbers.cpp b/src/Storages/System/StorageSystemNumbers.cpp
index 6ddfb5e3213..da700a7a4e9 100644
--- a/src/Storages/System/StorageSystemNumbers.cpp
+++ b/src/Storages/System/StorageSystemNumbers.cpp
@@ -38,11 +38,6 @@ void StorageSystemNumbers::read(
     size_t max_block_size,
     size_t num_streams)
 {
-    if (limit && *limit < max_block_size)
-    {
-        max_block_size = static_cast<size_t>(*limit);
-        multithreaded = false;
-    }
     query_plan.addStep(std::make_unique<ReadFromSystemNumbersStep>(
         column_names, query_info, storage_snapshot, context, shared_from_this(), max_block_size, num_streams));
 }
diff --git a/tests/queries/0_stateless/03203_system_numbers_limit_and_offset.reference b/tests/queries/0_stateless/03203_system_numbers_limit_and_offset.reference
index d55fa7c71fc..05deae711c6 100644
--- a/tests/queries/0_stateless/03203_system_numbers_limit_and_offset.reference
+++ b/tests/queries/0_stateless/03203_system_numbers_limit_and_offset.reference
@@ -1,3 +1,7 @@
+case 1
 9900
 9910
 9920
+case 2
+9990
+18679	31
diff --git a/tests/queries/0_stateless/03203_system_numbers_limit_and_offset.sql b/tests/queries/0_stateless/03203_system_numbers_limit_and_offset.sql
index 6b7f998c4d4..1a1133c8a7f 100644
--- a/tests/queries/0_stateless/03203_system_numbers_limit_and_offset.sql
+++ b/tests/queries/0_stateless/03203_system_numbers_limit_and_offset.sql
@@ -1,5 +1,37 @@
+SELECT 'case 1';
 SELECT number FROM numbers_mt(10000)
 WHERE (number % 10) = 0
 ORDER BY number ASC
 LIMIT 990, 3;
-
+SELECT 'case 2';
+SELECT number FROM numbers_mt(10000)
+WHERE (number % 10) = 0
+ORDER BY number ASC
+LIMIT 999, 20 SETTINGS max_block_size = 31;
+--- The following query was buggy before, so let's use it as a test case
+WITH
+    toUInt64(-1) AS umax,
+    toUInt8(ceil(log10(umax))) AS max_digits,
+    9 * max_digits AS max_digits_sum,
+    (x -> ((x > 1) AND (arraySum(arrayMap(y -> ((y > 1) AND (y < x) AND ((x % y) = 0)), range(toUInt64(sqrt(x)) + 1))) = 0))) AS is_prime_slow
+SELECT
+    num,
+    ds
+FROM
+(
+    WITH x -> arraySum(arrayMap(y -> toUInt8(y), splitByString('', toString(x)))) AS digits_sum
+    SELECT
+        1 + (number * 2) AS num,
+        digits_sum(num) AS ds
+    FROM numbers_mt(10000)
+    WHERE ds IN (
+        WITH x -> ((x > 1) AND (arraySum(arrayMap(y -> ((y > 1) AND (y < x) AND ((x % y) = 0)), range(toUInt64(sqrt(x)) + 1))) = 0)) AS is_prime_slow
+        SELECT number
+        FROM numbers(max_digits_sum + 1)
+        WHERE is_prime_slow(number)
+    )
+)
+WHERE is_prime_slow(num)
+ORDER BY num ASC
+LIMIT 998, 1
+SETTINGS max_block_size = 64, max_threads=16;

From 8b78cf1c6139931132ff84bdf1ae495eef9714be Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?=
 <benjamin.antal@clickhouse.com>
Date: Mon, 8 Jul 2024 16:22:12 +0000
Subject: [PATCH 0258/1488] Add one more test case

---
 .../03203_system_numbers_limit_and_offset.reference  | 12 ++++++++++++
 .../03203_system_numbers_limit_and_offset.sql        |  8 ++++++++
 2 files changed, 20 insertions(+)

diff --git a/tests/queries/0_stateless/03203_system_numbers_limit_and_offset.reference b/tests/queries/0_stateless/03203_system_numbers_limit_and_offset.reference
index 05deae711c6..e95864a6fc9 100644
--- a/tests/queries/0_stateless/03203_system_numbers_limit_and_offset.reference
+++ b/tests/queries/0_stateless/03203_system_numbers_limit_and_offset.reference
@@ -5,3 +5,15 @@ case 1
 case 2
 9990
 18679	31
+0
+10
+20
+30
+40
+50
+60
+70
+80
+90
+100
+110
diff --git a/tests/queries/0_stateless/03203_system_numbers_limit_and_offset.sql b/tests/queries/0_stateless/03203_system_numbers_limit_and_offset.sql
index 1a1133c8a7f..dc9e06b806f 100644
--- a/tests/queries/0_stateless/03203_system_numbers_limit_and_offset.sql
+++ b/tests/queries/0_stateless/03203_system_numbers_limit_and_offset.sql
@@ -3,11 +3,13 @@ SELECT number FROM numbers_mt(10000)
 WHERE (number % 10) = 0
 ORDER BY number ASC
 LIMIT 990, 3;
+
 SELECT 'case 2';
 SELECT number FROM numbers_mt(10000)
 WHERE (number % 10) = 0
 ORDER BY number ASC
 LIMIT 999, 20 SETTINGS max_block_size = 31;
+
 --- The following query was buggy before, so let's use it as a test case
 WITH
     toUInt64(-1) AS umax,
@@ -35,3 +37,9 @@ WHERE is_prime_slow(num)
 ORDER BY num ASC
 LIMIT 998, 1
 SETTINGS max_block_size = 64, max_threads=16;
+
+SELECT number
+FROM numbers_mt(120)
+WHERE (number % 10) = 0
+ORDER BY number ASC
+SETTINGS max_block_size = 31, max_threads = 11

From 2442473b253c4d8cf8f4474065b196bd4be4782c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?=
 <benjamin.antal@clickhouse.com>
Date: Wed, 10 Jul 2024 11:12:28 +0000
Subject: [PATCH 0259/1488] Fix `generate_series`

---
 .../QueryPlan/ReadFromSystemNumbersStep.cpp   | 50 +++++++++++++++----
 .../02970_generate_series.reference           |  1 +
 .../0_stateless/02970_generate_series.sql     |  1 +
 3 files changed, 41 insertions(+), 11 deletions(-)

diff --git a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp
index 049fe762640..67c228d43f4 100644
--- a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp
+++ b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp
@@ -36,12 +36,32 @@ inline void iotaWithStepOptimized(T * begin, size_t count, T first_value, T step
         iotaWithStep(begin, count, first_value, step);
 }
 
+/// The range is defined as [start, end)
+UInt64 itemCountInRange(UInt64 start, UInt64 end, UInt64 step)
+{
+    const auto range_count = end - start;
+    if (step == 1)
+        return range_count;
+
+    return (range_count - 1) / step + 1;
+}
+
 class NumbersSource : public ISource
 {
 public:
     NumbersSource(
-        UInt64 block_size_, UInt64 offset_, std::optional<UInt64> end_, UInt64 chunk_step_, const std::string & column_name, UInt64 step_)
-        : ISource(createHeader(column_name)), block_size(block_size_), next(offset_), chunk_step(chunk_step_), end(end_), step(step_)
+        UInt64 block_size_,
+        UInt64 offset_,
+        std::optional<UInt64> end_,
+        const std::string & column_name,
+        UInt64 step_in_chunk_,
+        UInt64 step_between_chunks_)
+        : ISource(createHeader(column_name))
+        , block_size(block_size_)
+        , next(offset_)
+        , end(end_)
+        , step_in_chunk(step_in_chunk_)
+        , step_between_chunks(step_between_chunks_)
     {
     }
     String getName() const override { return "Numbers"; }
@@ -59,7 +79,10 @@ protected:
         {
             if (end.value() <= next)
                 return {};
-            real_block_size = std::min(block_size, end.value() - next);
+
+            auto max_items_to_generate = itemCountInRange(next, *end, step_in_chunk);
+
+            real_block_size = std::min(block_size, max_items_to_generate);
         }
         auto column = ColumnUInt64::create(real_block_size);
         ColumnUInt64::Container & vec = column->getData();
@@ -69,9 +92,9 @@ protected:
 
         UInt64 * current_end = &vec[real_block_size];
 
-        iotaWithStepOptimized(pos, static_cast<size_t>(current_end - pos), curr, step);
+        iotaWithStepOptimized(pos, static_cast<size_t>(current_end - pos), curr, step_in_chunk);
 
-        next += chunk_step;
+        next += step_between_chunks;
 
         progress(column->size(), column->byteSize());
         return {Columns{std::move(column)}, real_block_size};
@@ -80,9 +103,9 @@ protected:
 private:
     UInt64 block_size;
     UInt64 next;
-    UInt64 chunk_step;
     std::optional<UInt64> end; /// not included
-    UInt64 step;
+    UInt64 step_in_chunk;
+    UInt64 step_between_chunks;
 };
 
 struct RangeWithStep
@@ -552,19 +575,24 @@ Pipe ReadFromSystemNumbersStep::makePipe()
         });
 
     /// Fall back to NumbersSource
+    /// Range in a single block
+    const auto block_range = max_block_size * numbers_storage.step;
+    /// Step between chunks in a single source.
+    /// It is bigger than block_range in case of multiple threads, because we have to account for other sources as well.
+    const auto step_between_chunks = num_streams * block_range;
     for (size_t i = 0; i < num_streams; ++i)
     {
         auto source = std::make_shared<NumbersSource>(
             max_block_size,
-            numbers_storage.offset + i * max_block_size * numbers_storage.step,
+            numbers_storage.offset + i * block_range,
             end,
-            num_streams * max_block_size * numbers_storage.step,
             numbers_storage.column_name,
-            numbers_storage.step);
+            numbers_storage.step,
+            step_between_chunks);
 
         if (numbers_storage.limit && i == 0)
         {
-            auto rows_appr = (*numbers_storage.limit - 1) / numbers_storage.step + 1;
+            auto rows_appr = itemCountInRange(numbers_storage.offset, *numbers_storage.limit, numbers_storage.step);
             if (limit > 0 && limit < rows_appr)
                 rows_appr = query_info_limit;
             source->addTotalRowsApprox(rows_appr);
diff --git a/tests/queries/0_stateless/02970_generate_series.reference b/tests/queries/0_stateless/02970_generate_series.reference
index 6e6f3c81587..f8a8e1891b7 100644
--- a/tests/queries/0_stateless/02970_generate_series.reference
+++ b/tests/queries/0_stateless/02970_generate_series.reference
@@ -5,6 +5,7 @@
 501
 50
 17928
+17928
 0
 10
 13
diff --git a/tests/queries/0_stateless/02970_generate_series.sql b/tests/queries/0_stateless/02970_generate_series.sql
index 0844714b3a6..edae884a561 100644
--- a/tests/queries/0_stateless/02970_generate_series.sql
+++ b/tests/queries/0_stateless/02970_generate_series.sql
@@ -5,6 +5,7 @@ SELECT count() FROM generate_series(7, 77, 10);
 SELECT count() FROM generate_series(0, 1000, 2);
 SELECT count() FROM generate_series(0, 999, 20);
 SELECT sum(generate_series) FROM generate_series(4, 1008, 4) WHERE generate_series % 7 = 1;
+SELECT sum(generate_series) FROM generate_series(4, 1008, 4) WHERE generate_series % 7 = 1 SETTINGS max_block_size = 71;
 
 SELECT * FROM generate_series(5, 4);
 SELECT * FROM generate_series(0, 0);

From a92eb1a4c38be7868fd0cb80261ae882c8fbb70e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?=
 <benjamin.antal@clickhouse.com>
Date: Wed, 10 Jul 2024 11:20:05 +0000
Subject: [PATCH 0260/1488] Avoid creating unnecessary sources

---
 src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp
index 67c228d43f4..da8a6933d6f 100644
--- a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp
+++ b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp
@@ -582,6 +582,10 @@ Pipe ReadFromSystemNumbersStep::makePipe()
     const auto step_between_chunks = num_streams * block_range;
     for (size_t i = 0; i < num_streams; ++i)
     {
+        const auto source_start = numbers_storage.offset + i * block_range;
+        if (numbers_storage.limit.has_value() && *numbers_storage.limit < source_start)
+            break;
+
         auto source = std::make_shared<NumbersSource>(
             max_block_size,
             numbers_storage.offset + i * block_range,

From 17352b6ed1cdcad359d2f17a96040f7fdaba9422 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?=
 <benjamin.antal@clickhouse.com>
Date: Wed, 10 Jul 2024 12:06:11 +0000
Subject: [PATCH 0261/1488] Fix the case when offset is bigger than limit

---
 src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp    | 8 +++++---
 .../03203_system_numbers_limit_and_offset.reference       | 1 +
 .../0_stateless/03203_system_numbers_limit_and_offset.sql | 8 +++++++-
 3 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp
index da8a6933d6f..b070bbe739b 100644
--- a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp
+++ b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp
@@ -582,13 +582,15 @@ Pipe ReadFromSystemNumbersStep::makePipe()
     const auto step_between_chunks = num_streams * block_range;
     for (size_t i = 0; i < num_streams; ++i)
     {
-        const auto source_start = numbers_storage.offset + i * block_range;
-        if (numbers_storage.limit.has_value() && *numbers_storage.limit < source_start)
+        const auto source_offset = i * block_range;
+        if (numbers_storage.limit.has_value() && *numbers_storage.limit < source_offset)
             break;
 
+        const auto source_start = numbers_storage.offset + source_offset;
+
         auto source = std::make_shared<NumbersSource>(
             max_block_size,
-            numbers_storage.offset + i * block_range,
+            source_start,
             end,
             numbers_storage.column_name,
             numbers_storage.step,
diff --git a/tests/queries/0_stateless/03203_system_numbers_limit_and_offset.reference b/tests/queries/0_stateless/03203_system_numbers_limit_and_offset.reference
index e95864a6fc9..960459f42ac 100644
--- a/tests/queries/0_stateless/03203_system_numbers_limit_and_offset.reference
+++ b/tests/queries/0_stateless/03203_system_numbers_limit_and_offset.reference
@@ -17,3 +17,4 @@ case 2
 90
 100
 110
+4250
diff --git a/tests/queries/0_stateless/03203_system_numbers_limit_and_offset.sql b/tests/queries/0_stateless/03203_system_numbers_limit_and_offset.sql
index dc9e06b806f..e258b0bd2d2 100644
--- a/tests/queries/0_stateless/03203_system_numbers_limit_and_offset.sql
+++ b/tests/queries/0_stateless/03203_system_numbers_limit_and_offset.sql
@@ -42,4 +42,10 @@ SELECT number
 FROM numbers_mt(120)
 WHERE (number % 10) = 0
 ORDER BY number ASC
-SETTINGS max_block_size = 31, max_threads = 11
+SETTINGS max_block_size = 31, max_threads = 11;
+
+SELECT number
+FROM numbers_mt(4242, 9)
+WHERE (number % 10) = 0
+ORDER BY number ASC
+SETTINGS max_block_size = 31, max_threads = 11;

From 6be9d7b2544ccb5642999b53409c0d9326163a62 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?=
 <antaljanosbenjamin@users.noreply.github.com>
Date: Thu, 11 Jul 2024 13:40:44 +0200
Subject: [PATCH 0262/1488] Make the tests works with old analyzer

---
 .../0_stateless/03203_system_numbers_limit_and_offset.sql     | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/queries/0_stateless/03203_system_numbers_limit_and_offset.sql b/tests/queries/0_stateless/03203_system_numbers_limit_and_offset.sql
index e258b0bd2d2..b72ab4a98fd 100644
--- a/tests/queries/0_stateless/03203_system_numbers_limit_and_offset.sql
+++ b/tests/queries/0_stateless/03203_system_numbers_limit_and_offset.sql
@@ -21,10 +21,10 @@ SELECT
     ds
 FROM
 (
-    WITH x -> arraySum(arrayMap(y -> toUInt8(y), splitByString('', toString(x)))) AS digits_sum
+    WITH arraySum(arrayMap(y -> toUInt8(y), splitByString('', toString(num)))) AS digits_sum
     SELECT
         1 + (number * 2) AS num,
-        digits_sum(num) AS ds
+        digits_sum AS ds
     FROM numbers_mt(10000)
     WHERE ds IN (
         WITH x -> ((x > 1) AND (arraySum(arrayMap(y -> ((y > 1) AND (y < x) AND ((x % y) = 0)), range(toUInt64(sqrt(x)) + 1))) = 0)) AS is_prime_slow

From 35eabcc9d1cf3a5339c20ea34d006edfc858ae68 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?=
 <benjamin.antal@clickhouse.com>
Date: Sat, 13 Jul 2024 16:01:15 +0000
Subject: [PATCH 0263/1488] Split test cases

---
 ...ystem_numbers_limit_and_offset_complex.reference} |  6 ------
 ...3203_system_numbers_limit_and_offset_complex.sql} | 12 ------------
 ..._system_numbers_limit_and_offset_simple.reference |  6 ++++++
 .../03203_system_numbers_limit_and_offset_simple.sql | 11 +++++++++++
 4 files changed, 17 insertions(+), 18 deletions(-)
 rename tests/queries/0_stateless/{03203_system_numbers_limit_and_offset.reference => 03203_system_numbers_limit_and_offset_complex.reference} (60%)
 rename tests/queries/0_stateless/{03203_system_numbers_limit_and_offset.sql => 03203_system_numbers_limit_and_offset_complex.sql} (82%)
 create mode 100644 tests/queries/0_stateless/03203_system_numbers_limit_and_offset_simple.reference
 create mode 100644 tests/queries/0_stateless/03203_system_numbers_limit_and_offset_simple.sql

diff --git a/tests/queries/0_stateless/03203_system_numbers_limit_and_offset.reference b/tests/queries/0_stateless/03203_system_numbers_limit_and_offset_complex.reference
similarity index 60%
rename from tests/queries/0_stateless/03203_system_numbers_limit_and_offset.reference
rename to tests/queries/0_stateless/03203_system_numbers_limit_and_offset_complex.reference
index 960459f42ac..d38b21d2d05 100644
--- a/tests/queries/0_stateless/03203_system_numbers_limit_and_offset.reference
+++ b/tests/queries/0_stateless/03203_system_numbers_limit_and_offset_complex.reference
@@ -1,9 +1,3 @@
-case 1
-9900
-9910
-9920
-case 2
-9990
 18679	31
 0
 10
diff --git a/tests/queries/0_stateless/03203_system_numbers_limit_and_offset.sql b/tests/queries/0_stateless/03203_system_numbers_limit_and_offset_complex.sql
similarity index 82%
rename from tests/queries/0_stateless/03203_system_numbers_limit_and_offset.sql
rename to tests/queries/0_stateless/03203_system_numbers_limit_and_offset_complex.sql
index b72ab4a98fd..8e8620361fa 100644
--- a/tests/queries/0_stateless/03203_system_numbers_limit_and_offset.sql
+++ b/tests/queries/0_stateless/03203_system_numbers_limit_and_offset_complex.sql
@@ -1,15 +1,3 @@
-SELECT 'case 1';
-SELECT number FROM numbers_mt(10000)
-WHERE (number % 10) = 0
-ORDER BY number ASC
-LIMIT 990, 3;
-
-SELECT 'case 2';
-SELECT number FROM numbers_mt(10000)
-WHERE (number % 10) = 0
-ORDER BY number ASC
-LIMIT 999, 20 SETTINGS max_block_size = 31;
-
 --- The following query was buggy before, so let's use it as a test case
 WITH
     toUInt64(-1) AS umax,
diff --git a/tests/queries/0_stateless/03203_system_numbers_limit_and_offset_simple.reference b/tests/queries/0_stateless/03203_system_numbers_limit_and_offset_simple.reference
new file mode 100644
index 00000000000..6aad3ee0b4b
--- /dev/null
+++ b/tests/queries/0_stateless/03203_system_numbers_limit_and_offset_simple.reference
@@ -0,0 +1,6 @@
+case 1
+9900
+9910
+9920
+case 2
+9990
diff --git a/tests/queries/0_stateless/03203_system_numbers_limit_and_offset_simple.sql b/tests/queries/0_stateless/03203_system_numbers_limit_and_offset_simple.sql
new file mode 100644
index 00000000000..df01ddf4312
--- /dev/null
+++ b/tests/queries/0_stateless/03203_system_numbers_limit_and_offset_simple.sql
@@ -0,0 +1,11 @@
+SELECT 'case 1';
+SELECT number FROM numbers_mt(10000)
+WHERE (number % 10) = 0
+ORDER BY number ASC
+LIMIT 990, 3;
+
+SELECT 'case 2';
+SELECT number FROM numbers_mt(10000)
+WHERE (number % 10) = 0
+ORDER BY number ASC
+LIMIT 999, 20 SETTINGS max_block_size = 31;

From 8e469af2a41b444bbeb58b63fa0ece7d883553c0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?=
 <benjamin.antal@clickhouse.com>
Date: Sat, 13 Jul 2024 20:36:01 +0000
Subject: [PATCH 0264/1488] Make query really work with old analyzer

---
 ...ystem_numbers_limit_and_offset_complex.sql | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/tests/queries/0_stateless/03203_system_numbers_limit_and_offset_complex.sql b/tests/queries/0_stateless/03203_system_numbers_limit_and_offset_complex.sql
index 8e8620361fa..756e08da27d 100644
--- a/tests/queries/0_stateless/03203_system_numbers_limit_and_offset_complex.sql
+++ b/tests/queries/0_stateless/03203_system_numbers_limit_and_offset_complex.sql
@@ -1,27 +1,26 @@
 --- The following query was buggy before, so let's use it as a test case
 WITH
-    toUInt64(-1) AS umax,
-    toUInt8(ceil(log10(umax))) AS max_digits,
-    9 * max_digits AS max_digits_sum,
-    (x -> ((x > 1) AND (arraySum(arrayMap(y -> ((y > 1) AND (y < x) AND ((x % y) = 0)), range(toUInt64(sqrt(x)) + 1))) = 0))) AS is_prime_slow
+    (num > 1) AND (arraySum(arrayMap(y -> ((y > 1) AND (y < num) AND ((num % y) = 0)), range(toUInt64(sqrt(num)) + 1))) = 0) AS is_prime_slow
 SELECT
     num,
-    ds
+    ds,
 FROM
 (
-    WITH arraySum(arrayMap(y -> toUInt8(y), splitByString('', toString(num)))) AS digits_sum
+    WITH
+        arraySum(arrayMap(y -> toUInt8(y), splitByString('', toString(num)))) AS digits_sum
     SELECT
         1 + (number * 2) AS num,
         digits_sum AS ds
     FROM numbers_mt(10000)
     WHERE ds IN (
-        WITH x -> ((x > 1) AND (arraySum(arrayMap(y -> ((y > 1) AND (y < x) AND ((x % y) = 0)), range(toUInt64(sqrt(x)) + 1))) = 0)) AS is_prime_slow
+        WITH
+            (number > 1) AND (arraySum(arrayMap(y -> ((y > 1) AND (y < number) AND ((number % y) = 0)), range(toUInt64(sqrt(number)) + 1))) = 0) AS is_prime_slow
         SELECT number
-        FROM numbers(max_digits_sum + 1)
-        WHERE is_prime_slow(number)
+        FROM numbers(180 + 1)
+        WHERE is_prime_slow
     )
 )
-WHERE is_prime_slow(num)
+WHERE is_prime_slow
 ORDER BY num ASC
 LIMIT 998, 1
 SETTINGS max_block_size = 64, max_threads=16;

From 908f5899ddfdc701df5e9e6189760431e88b6695 Mon Sep 17 00:00:00 2001
From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com>
Date: Mon, 15 Jul 2024 17:28:33 +0200
Subject: [PATCH 0265/1488] Add settings to replace external engines to Null

---
 src/Core/Settings.h                         |  1 +
 src/Core/SettingsChangesHistory.cpp         |  3 +-
 src/Interpreters/InterpreterCreateQuery.cpp | 40 +++++++++++++++++++++
 3 files changed, 43 insertions(+), 1 deletion(-)

diff --git a/src/Core/Settings.h b/src/Core/Settings.h
index 52fa28a4481..7bf97896357 100644
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@@ -891,6 +891,7 @@ class IColumn;
     M(Bool, optimize_distinct_in_order, true, "Enable DISTINCT optimization if some columns in DISTINCT form a prefix of sorting. For example, prefix of sorting key in merge tree or ORDER BY statement", 0) \
     M(Bool, keeper_map_strict_mode, false, "Enforce additional checks during operations on KeeperMap. E.g. throw an exception on an insert for already existing key", 0) \
     M(UInt64, extract_key_value_pairs_max_pairs_per_row, 1000, "Max number of pairs that can be produced by the `extractKeyValuePairs` function. Used as a safeguard against consuming too much memory.", 0) ALIAS(extract_kvp_max_pairs_per_row) \
+    M(Bool, restore_replace_external_engine_to_null, false, "Replace all the External table engines to Null on restore. Useful for testing purposes", 0) \
     \
     \
     /* ###################################### */ \
diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp
index b9b72209103..a23d9d17da2 100644
--- a/src/Core/SettingsChangesHistory.cpp
+++ b/src/Core/SettingsChangesHistory.cpp
@@ -78,7 +78,8 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
               {"azure_sdk_retry_max_backoff_ms", 1000, 1000, "Maximal backoff between retries in azure sdk"},
               {"ignore_on_cluster_for_replicated_named_collections_queries", false, false, "Ignore ON CLUSTER clause for replicated named collections management queries."},
               {"postgresql_connection_attempt_timeout", 2, 2, "Allow to control 'connect_timeout' parameter of PostgreSQL connection."},
-              {"postgresql_connection_pool_retries", 2, 2, "Allow to control the number of retries in PostgreSQL connection pool."}
+              {"postgresql_connection_pool_retries", 2, 2, "Allow to control the number of retries in PostgreSQL connection pool."},
+              {"restore_replace_external_engine_to_null", false, false, "New setting."}
               }},
     {"24.6", {{"materialize_skip_indexes_on_insert", true, true, "Added new setting to allow to disable materialization of skip indexes on insert"},
               {"materialize_statistics_on_insert", true, true, "Added new setting to allow to disable materialization of statistics on insert"},
diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp
index 84d7f0a587c..f8696caebe7 100644
--- a/src/Interpreters/InterpreterCreateQuery.cpp
+++ b/src/Interpreters/InterpreterCreateQuery.cpp
@@ -960,6 +960,14 @@ namespace
         engine_ast->no_empty_args = true;
         storage.set(storage.engine, engine_ast);
     }
+
+    void setNullTableEngine(ASTStorage &storage)
+    {
+        auto engine_ast = std::make_shared<ASTFunction>();
+        engine_ast->name = "Null";
+        engine_ast->no_empty_args = true;
+        storage.set(storage.engine, engine_ast);
+    }
 }
 
 void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const
@@ -1000,6 +1008,38 @@ void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const
         /// Some part of storage definition (such as PARTITION BY) is specified, but ENGINE is not: just set default one.
         if (!create.storage->engine)
             setDefaultTableEngine(*create.storage, getContext()->getSettingsRef().default_table_engine.value);
+        /// For exrternal tables with restore_replace_external_engine_to_null setting we replace external engines to
+        /// Null table engine.
+        else (create.storage->engine == "AzureBlobStorage" ||
+              create.storage->engine == "AzureQueue" ||
+              create.storage->engine == "COSN" ||
+              create.storage->engine == "DeltaLake" ||
+              create.storage->engine == "Dictionary" ||
+              create.storage->engine == "Executable" ||
+              create.storage->engine == "ExecutablePool" ||
+              create.storage->engine == "ExternalDistributed" ||
+              create.storage->engine == "File" ||
+              create.storage->engine == "Hudi" ||
+              create.storage->engine == "Iceberg" ||
+              create.storage->engine == "JDBC" ||
+              create.storage->engine == "Kafka" ||
+              create.storage->engine == "MaterializedPostgreSQL" ||
+              create.storage->engine == "MongoDB" ||
+              create.storage->engine == "MySQL" ||
+              create.storage->engine == "NATS" ||
+              create.storage->engine == "ODBC" ||
+              create.storage->engine == "OSS" ||
+              create.storage->engine == "PostgreSQL" ||
+              create.storage->engine == "RabbitMQ" ||
+              create.storage->engine == "Redis" ||
+              create.storage->engine == "S3" ||
+              create.storage->engine == "S3Queue" ||
+              create.storage->engine == "TinyLog" ||
+              create.storage->engine == "URL")
+        {
+            if (getContext()->getSettingsRef().restore_replace_external_engine_to_null)
+                setNullTableEngine(*create.storage)
+        }
         return;
     }
 

From 344e5b716d49eda59783b6fff85757e6a5b6e98f Mon Sep 17 00:00:00 2001
From: Sema Checherinda <Sema.Checherinda@clickhouse.com>
Date: Mon, 15 Jul 2024 17:41:32 +0200
Subject: [PATCH 0266/1488] cancel method is noexcept

---
 src/Common/ConcurrentBoundedQueue.h           |  2 +-
 .../Executors/CompletedPipelineExecutor.cpp   |  2 --
 .../Formats/Impl/ArrowBlockInputFormat.h      |  2 +-
 .../Formats/Impl/DWARFBlockInputFormat.h      |  2 +-
 .../Formats/Impl/NativeORCBlockInputFormat.h  |  2 +-
 .../Formats/Impl/ORCBlockInputFormat.h        |  2 +-
 .../Impl/ParallelFormattingOutputFormat.cpp   |  2 +-
 .../Impl/ParallelFormattingOutputFormat.h     |  4 ++--
 .../Impl/ParallelParsingInputFormat.cpp       |  2 +-
 .../Formats/Impl/ParallelParsingInputFormat.h |  4 ++--
 .../Formats/Impl/ParquetBlockInputFormat.h    |  2 +-
 .../Formats/Impl/ParquetBlockOutputFormat.cpp |  2 +-
 .../Formats/Impl/ParquetBlockOutputFormat.h   |  2 +-
 .../Formats/Impl/ParquetMetadataInputFormat.h |  2 +-
 src/Processors/Formats/LazyOutputFormat.h     |  2 +-
 src/Processors/IProcessor.cpp                 |  4 ++--
 src/Processors/IProcessor.h                   |  4 ++--
 src/Processors/Sources/RemoteSource.cpp       | 13 ++++++++++--
 src/Processors/Sources/RemoteSource.h         |  2 +-
 .../Transforms/AggregatingTransform.cpp       |  2 +-
 src/Server/TCPHandler.cpp                     |  1 -
 src/Storages/Distributed/DistributedSink.cpp  | 20 +++++++++++++++----
 src/Storages/Distributed/DistributedSink.h    |  2 +-
 src/Storages/LiveView/LiveViewEventsSource.h  |  2 +-
 src/Storages/LiveView/LiveViewSource.h        |  2 +-
 .../MergeTree/MergeTreeSelectProcessor.h      |  2 +-
 src/Storages/MergeTree/MergeTreeSink.cpp      |  5 -----
 src/Storages/MergeTree/MergeTreeSource.cpp    |  2 +-
 src/Storages/MergeTree/MergeTreeSource.h      |  2 +-
 .../MergeTree/ReplicatedMergeTreeSink.cpp     | 16 ---------------
 src/Storages/MessageQueueSink.cpp             | 13 ++++++++++++
 src/Storages/MessageQueueSink.h               |  6 +-----
 .../StorageObjectStorageSink.cpp              |  2 +-
 33 files changed, 70 insertions(+), 64 deletions(-)

diff --git a/src/Common/ConcurrentBoundedQueue.h b/src/Common/ConcurrentBoundedQueue.h
index 16b9488c98d..a830ae157a5 100644
--- a/src/Common/ConcurrentBoundedQueue.h
+++ b/src/Common/ConcurrentBoundedQueue.h
@@ -243,7 +243,7 @@ public:
     }
 
     /// Clear and finish queue
-    void clearAndFinish()
+    void clearAndFinish() noexcept
     {
         {
             std::lock_guard lock(queue_mutex);
diff --git a/src/Processors/Executors/CompletedPipelineExecutor.cpp b/src/Processors/Executors/CompletedPipelineExecutor.cpp
index 909e742ffbf..1eeee896ede 100644
--- a/src/Processors/Executors/CompletedPipelineExecutor.cpp
+++ b/src/Processors/Executors/CompletedPipelineExecutor.cpp
@@ -99,7 +99,6 @@ void CompletedPipelineExecutor::execute()
 
             if (is_cancelled_callback())
             {
-                LOG_INFO(getLogger("CompletedPipelineExecutor"), "execute CancelCallback FULLY_CANCELLED");
                 data->executor->cancel();
             }
         }
@@ -121,7 +120,6 @@ CompletedPipelineExecutor::~CompletedPipelineExecutor()
     {
         if (data && data->executor)
         {
-            LOG_INFO(getLogger("CompletedPipelineExecutor"), "~CompletedPipelineExecutor");
             data->executor->cancel();
         }
     }
diff --git a/src/Processors/Formats/Impl/ArrowBlockInputFormat.h b/src/Processors/Formats/Impl/ArrowBlockInputFormat.h
index 4fe01d0be12..cb74a9dd93e 100644
--- a/src/Processors/Formats/Impl/ArrowBlockInputFormat.h
+++ b/src/Processors/Formats/Impl/ArrowBlockInputFormat.h
@@ -32,7 +32,7 @@ public:
 private:
     Chunk read() override;
 
-    void onCancelX() override
+    void onCancel() noexcept override
     {
         is_stopped = 1;
     }
diff --git a/src/Processors/Formats/Impl/DWARFBlockInputFormat.h b/src/Processors/Formats/Impl/DWARFBlockInputFormat.h
index 6cab5d34994..2d94d166708 100644
--- a/src/Processors/Formats/Impl/DWARFBlockInputFormat.h
+++ b/src/Processors/Formats/Impl/DWARFBlockInputFormat.h
@@ -32,7 +32,7 @@ public:
 protected:
     Chunk read() override;
 
-    void onCancelX() override
+    void onCancel() noexcept override
     {
         is_stopped = 1;
     }
diff --git a/src/Processors/Formats/Impl/NativeORCBlockInputFormat.h b/src/Processors/Formats/Impl/NativeORCBlockInputFormat.h
index de9925e3737..e4f2ef9ebe3 100644
--- a/src/Processors/Formats/Impl/NativeORCBlockInputFormat.h
+++ b/src/Processors/Formats/Impl/NativeORCBlockInputFormat.h
@@ -64,7 +64,7 @@ public:
 protected:
     Chunk read() override;
 
-    void onCancelX() override { is_stopped = 1; }
+    void onCancel() noexcept override { is_stopped = 1; }
 
 private:
     void prepareFileReader();
diff --git a/src/Processors/Formats/Impl/ORCBlockInputFormat.h b/src/Processors/Formats/Impl/ORCBlockInputFormat.h
index 167436ad4b9..85f1636d3dc 100644
--- a/src/Processors/Formats/Impl/ORCBlockInputFormat.h
+++ b/src/Processors/Formats/Impl/ORCBlockInputFormat.h
@@ -34,7 +34,7 @@ public:
 protected:
     Chunk read() override;
 
-    void onCancelX() override
+    void onCancel() noexcept override
     {
         is_stopped = 1;
     }
diff --git a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.cpp b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.cpp
index b2871310be5..5d404d493a6 100644
--- a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.cpp
+++ b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.cpp
@@ -96,7 +96,7 @@ namespace DB
     }
 
 
-    void ParallelFormattingOutputFormat::finishAndWait()
+    void ParallelFormattingOutputFormat::finishAndWait() noexcept
     {
         emergency_stop = true;
 
diff --git a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h
index 40774fcfbfa..54617c77f86 100644
--- a/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h
+++ b/src/Processors/Formats/Impl/ParallelFormattingOutputFormat.h
@@ -122,7 +122,7 @@ public:
         started_prefix = true;
     }
 
-    void onCancelX() override
+    void onCancel() noexcept override
     {
         finishAndWait();
     }
@@ -268,7 +268,7 @@ private:
     bool collected_suffix = false;
     bool collected_finalize = false;
 
-    void finishAndWait();
+    void finishAndWait() noexcept;
 
     void onBackgroundException()
     {
diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp
index d38a299cb6e..447adb1ed48 100644
--- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp
+++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp
@@ -176,7 +176,7 @@ Chunk ParallelParsingInputFormat::read()
         if (background_exception)
         {
             lock.unlock();
-            onCancelX();
+            onCancel();
             std::rethrow_exception(background_exception);
         }
 
diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.h b/src/Processors/Formats/Impl/ParallelParsingInputFormat.h
index eed40dc43e5..b97bf5213e6 100644
--- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.h
+++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.h
@@ -137,7 +137,7 @@ private:
 
     Chunk read() final;
 
-    void onCancelX() final
+    void onCancel() noexcept final
     {
         /*
          * The format parsers themselves are not being cancelled here, so we'll
@@ -292,7 +292,7 @@ private:
             first_parser_finished.wait();
     }
 
-    void finishAndWait()
+    void finishAndWait() noexcept
     {
         /// Defending concurrent segmentator thread join
         std::lock_guard finish_and_wait_lock(finish_and_wait_mutex);
diff --git a/src/Processors/Formats/Impl/ParquetBlockInputFormat.h b/src/Processors/Formats/Impl/ParquetBlockInputFormat.h
index 0123329f026..ed528cc077c 100644
--- a/src/Processors/Formats/Impl/ParquetBlockInputFormat.h
+++ b/src/Processors/Formats/Impl/ParquetBlockInputFormat.h
@@ -68,7 +68,7 @@ public:
 private:
     Chunk read() override;
 
-    void onCancelX() override
+    void onCancel() noexcept override
     {
         is_stopped = 1;
     }
diff --git a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp
index d08c91d286b..01fb97223f1 100644
--- a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp
+++ b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp
@@ -268,7 +268,7 @@ void ParquetBlockOutputFormat::resetFormatterImpl()
     staging_bytes = 0;
 }
 
-void ParquetBlockOutputFormat::onCancelX()
+void ParquetBlockOutputFormat::onCancel() noexcept
 {
     is_stopped = true;
 }
diff --git a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.h b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.h
index 0704ba0ed90..f8f5d2556a5 100644
--- a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.h
+++ b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.h
@@ -112,7 +112,7 @@ private:
     void consume(Chunk) override;
     void finalizeImpl() override;
     void resetFormatterImpl() override;
-    void onCancelX() override;
+    void onCancel() noexcept override;
 
     void writeRowGroup(std::vector<Chunk> chunks);
     void writeUsingArrow(std::vector<Chunk> chunks);
diff --git a/src/Processors/Formats/Impl/ParquetMetadataInputFormat.h b/src/Processors/Formats/Impl/ParquetMetadataInputFormat.h
index 35180d202d8..5d2d8989859 100644
--- a/src/Processors/Formats/Impl/ParquetMetadataInputFormat.h
+++ b/src/Processors/Formats/Impl/ParquetMetadataInputFormat.h
@@ -65,7 +65,7 @@ public:
 private:
     Chunk read() override;
 
-    void onCancelX() override
+    void onCancel() noexcept override
     {
         is_stopped = 1;
     }
diff --git a/src/Processors/Formats/LazyOutputFormat.h b/src/Processors/Formats/LazyOutputFormat.h
index a245df8234d..c803ed5dc61 100644
--- a/src/Processors/Formats/LazyOutputFormat.h
+++ b/src/Processors/Formats/LazyOutputFormat.h
@@ -29,7 +29,7 @@ public:
 
     void setRowsBeforeLimit(size_t rows_before_limit) override;
 
-    void onCancelX() override
+    void onCancel() noexcept override
     {
         queue.clearAndFinish();
     }
diff --git a/src/Processors/IProcessor.cpp b/src/Processors/IProcessor.cpp
index 4d95bb5f3e0..edb4d662d8b 100644
--- a/src/Processors/IProcessor.cpp
+++ b/src/Processors/IProcessor.cpp
@@ -9,14 +9,14 @@
 namespace DB
 {
 
-void IProcessor::cancel()
+void IProcessor::cancel() noexcept
 {
 
     bool already_cancelled = is_cancelled.exchange(true, std::memory_order_acq_rel);
     if (already_cancelled)
         return;
 
-    onCancelX();
+    onCancel();
 }
 
 String IProcessor::debug() const
diff --git a/src/Processors/IProcessor.h b/src/Processors/IProcessor.h
index 5f4d71fbf54..680a446173e 100644
--- a/src/Processors/IProcessor.h
+++ b/src/Processors/IProcessor.h
@@ -238,7 +238,7 @@ public:
     /// In case if query was cancelled executor will wait till all processors finish their jobs.
     /// Generally, there is no reason to check this flag. However, it may be reasonable for long operations (e.g. i/o).
     bool isCancelled() const { return is_cancelled.load(std::memory_order_acquire); }
-    void cancel();
+    void cancel() noexcept;
 
     /// Additional method which is called in case if ports were updated while work() method.
     /// May be used to stop execution in rare cases.
@@ -363,7 +363,7 @@ public:
     virtual void setRowsBeforeLimitCounter(RowsBeforeLimitCounterPtr /* counter */) {}
 
 protected:
-    virtual void onCancelX() {}
+    virtual void onCancel() noexcept {}
 
     std::atomic<bool> is_cancelled{false};
 
diff --git a/src/Processors/Sources/RemoteSource.cpp b/src/Processors/Sources/RemoteSource.cpp
index 44cf26e0b01..42696f9c3ce 100644
--- a/src/Processors/Sources/RemoteSource.cpp
+++ b/src/Processors/Sources/RemoteSource.cpp
@@ -4,6 +4,8 @@
 #include <QueryPipeline/StreamLocalLimits.h>
 #include <Processors/Transforms/AggregatingTransform.h>
 #include <DataTypes/DataTypeAggregateFunction.h>
+#include <Common/Exception.h>
+#include <Common/Logger.h>
 
 namespace DB
 {
@@ -182,9 +184,16 @@ std::optional<Chunk> RemoteSource::tryGenerate()
     return chunk;
 }
 
-void RemoteSource::onCancelX()
+void RemoteSource::onCancel() noexcept
 {
-    query_executor->cancel();
+    try
+    {
+        query_executor->cancel();
+    }
+    catch (...)
+    {
+        tryLogCurrentException(getLogger("RemoteSource"), "Error occurs on cancelation.");
+    }
 }
 
 void RemoteSource::onUpdatePorts()
diff --git a/src/Processors/Sources/RemoteSource.h b/src/Processors/Sources/RemoteSource.h
index 880eb234bfb..adf1b8e9fac 100644
--- a/src/Processors/Sources/RemoteSource.h
+++ b/src/Processors/Sources/RemoteSource.h
@@ -36,7 +36,7 @@ public:
 
 protected:
     std::optional<Chunk> tryGenerate() override;
-    void onCancelX() override;
+    void onCancel() noexcept override;
 
 private:
     bool was_query_sent = false;
diff --git a/src/Processors/Transforms/AggregatingTransform.cpp b/src/Processors/Transforms/AggregatingTransform.cpp
index e42c1fd3a8d..870ba84722d 100644
--- a/src/Processors/Transforms/AggregatingTransform.cpp
+++ b/src/Processors/Transforms/AggregatingTransform.cpp
@@ -375,7 +375,7 @@ public:
         return prepareTwoLevel();
     }
 
-    void onCancelX() override
+    void onCancel() noexcept override
     {
         shared_data->is_cancelled.store(true, std::memory_order_seq_cst);
     }
diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp
index ba12cad7771..e7342ff9b6f 100644
--- a/src/Server/TCPHandler.cpp
+++ b/src/Server/TCPHandler.cpp
@@ -557,7 +557,6 @@ void TCPHandler::runImpl()
 
                             if (getQueryCancellationStatus() == CancellationStatus::FULLY_CANCELLED)
                             {
-                                LOG_INFO(log, "CancelCallback FULLY_CANCELLED");
                                 return true;
                             }
 
diff --git a/src/Storages/Distributed/DistributedSink.cpp b/src/Storages/Distributed/DistributedSink.cpp
index 6283594e0d2..d67910a141f 100644
--- a/src/Storages/Distributed/DistributedSink.cpp
+++ b/src/Storages/Distributed/DistributedSink.cpp
@@ -596,7 +596,7 @@ void DistributedSink::onFinish()
     }
 }
 
-void DistributedSink::onCancelX()
+void DistributedSink::onCancel() noexcept
 {
     std::lock_guard lock(execution_mutex);
     if (pool && !pool->finished())
@@ -607,14 +607,26 @@ void DistributedSink::onCancelX()
         }
         catch (...)
         {
-            tryLogCurrentException(storage.log);
+            tryLogCurrentException(storage.log, "Error occurs on cancelation.");
         }
     }
 
     for (auto & shard_jobs : per_shard_jobs)
+    {
         for (JobReplica & job : shard_jobs.replicas_jobs)
-            if (job.executor)
-                job.executor->cancel();
+        {
+            try
+            {
+                if (job.executor)
+                    job.executor->cancel();
+            }
+            catch (...)
+            {
+                tryLogCurrentException(storage.log, "Error occurs on cancelation.");
+            }
+        }
+    }
+
 }
 
 
diff --git a/src/Storages/Distributed/DistributedSink.h b/src/Storages/Distributed/DistributedSink.h
index 1dac3eeba6d..65a5eb52787 100644
--- a/src/Storages/Distributed/DistributedSink.h
+++ b/src/Storages/Distributed/DistributedSink.h
@@ -53,7 +53,7 @@ public:
     void onFinish() override;
 
 private:
-    void onCancelX() override;
+    void onCancel() noexcept override;
 
     IColumn::Selector createSelector(const Block & source_block) const;
 
diff --git a/src/Storages/LiveView/LiveViewEventsSource.h b/src/Storages/LiveView/LiveViewEventsSource.h
index d1ed222c185..4210acbc5bc 100644
--- a/src/Storages/LiveView/LiveViewEventsSource.h
+++ b/src/Storages/LiveView/LiveViewEventsSource.h
@@ -54,7 +54,7 @@ public:
 
     String getName() const override { return "LiveViewEventsSource"; }
 
-    void onCancelX() override
+    void onCancel() noexcept override
     {
         if (storage->shutdown_called)
             return;
diff --git a/src/Storages/LiveView/LiveViewSource.h b/src/Storages/LiveView/LiveViewSource.h
index 83589067cf5..81dd5620e57 100644
--- a/src/Storages/LiveView/LiveViewSource.h
+++ b/src/Storages/LiveView/LiveViewSource.h
@@ -36,7 +36,7 @@ public:
 
     String getName() const override { return "LiveViewSource"; }
 
-    void onCancelX() override
+    void onCancel() noexcept override
     {
         if (storage->shutdown_called)
             return;
diff --git a/src/Storages/MergeTree/MergeTreeSelectProcessor.h b/src/Storages/MergeTree/MergeTreeSelectProcessor.h
index 03ca30dd5b3..7a9cebbcb2e 100644
--- a/src/Storages/MergeTree/MergeTreeSelectProcessor.h
+++ b/src/Storages/MergeTree/MergeTreeSelectProcessor.h
@@ -48,7 +48,7 @@ public:
 
     ChunkAndProgress read();
 
-    void cancel() { is_cancelled = true; }
+    void cancel() noexcept { is_cancelled = true; }
 
     const MergeTreeReaderSettings & getSettings() const { return reader_settings; }
 
diff --git a/src/Storages/MergeTree/MergeTreeSink.cpp b/src/Storages/MergeTree/MergeTreeSink.cpp
index d2e34665962..210a7057f94 100644
--- a/src/Storages/MergeTree/MergeTreeSink.cpp
+++ b/src/Storages/MergeTree/MergeTreeSink.cpp
@@ -28,9 +28,6 @@ struct MergeTreeSink::DelayedChunk
 
 MergeTreeSink::~MergeTreeSink()
 {
-    size_t addr = delayed_chunk ? size_t(delayed_chunk.get()) : 0;
-    LOG_INFO(storage.log, "~ReplicatedMergeTreeSinkImpl, delayed_chunk {}, called from {}", addr, StackTrace().toString());
-
     if (!delayed_chunk)
         return;
 
@@ -40,8 +37,6 @@ MergeTreeSink::~MergeTreeSink()
     }
 
     delayed_chunk.reset();
-
-    LOG_INFO(storage.log, "~ReplicatedMergeTreeSinkImpl end");
 }
 
 MergeTreeSink::MergeTreeSink(
diff --git a/src/Storages/MergeTree/MergeTreeSource.cpp b/src/Storages/MergeTree/MergeTreeSource.cpp
index 4070ccf4433..380c47723bc 100644
--- a/src/Storages/MergeTree/MergeTreeSource.cpp
+++ b/src/Storages/MergeTree/MergeTreeSource.cpp
@@ -149,7 +149,7 @@ std::string MergeTreeSource::getName() const
     return processor->getName();
 }
 
-void MergeTreeSource::onCancelX()
+void MergeTreeSource::onCancel() noexcept
 {
     processor->cancel();
 }
diff --git a/src/Storages/MergeTree/MergeTreeSource.h b/src/Storages/MergeTree/MergeTreeSource.h
index c7092aa26b1..7506af4f9b8 100644
--- a/src/Storages/MergeTree/MergeTreeSource.h
+++ b/src/Storages/MergeTree/MergeTreeSource.h
@@ -26,7 +26,7 @@ public:
 protected:
     std::optional<Chunk> tryGenerate() override;
 
-    void onCancelX() override;
+    void onCancel() noexcept override;
 
 private:
     MergeTreeSelectProcessorPtr processor;
diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp
index 6c7ed9bdae0..7bfe647fa7f 100644
--- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp
+++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp
@@ -155,9 +155,6 @@ ReplicatedMergeTreeSinkImpl<async_insert>::ReplicatedMergeTreeSinkImpl(
 template<bool async_insert>
 ReplicatedMergeTreeSinkImpl<async_insert>::~ReplicatedMergeTreeSinkImpl()
 {
-    size_t addr = delayed_chunk ? size_t(delayed_chunk.get()) : 0;
-    LOG_INFO(log, "~ReplicatedMergeTreeSinkImpl, delayed_chunk {}, called from {}", addr, StackTrace().toString());
-
     if (!delayed_chunk)
         return;
 
@@ -167,8 +164,6 @@ ReplicatedMergeTreeSinkImpl<async_insert>::~ReplicatedMergeTreeSinkImpl()
     }
 
     delayed_chunk.reset();
-
-    LOG_INFO(log, "~ReplicatedMergeTreeSinkImpl end");
 }
 
 template<bool async_insert>
@@ -273,8 +268,6 @@ size_t ReplicatedMergeTreeSinkImpl<async_insert>::checkQuorumPrecondition(const
 template<bool async_insert>
 void ReplicatedMergeTreeSinkImpl<async_insert>::consume(Chunk chunk)
 {
-    LOG_INFO(log, "consume");
-
     if (num_blocks_processed > 0)
         storage.delayInsertOrThrowIfNeeded(&storage.partial_shutdown_event, context, false);
 
@@ -448,9 +441,6 @@ void ReplicatedMergeTreeSinkImpl<async_insert>::consume(Chunk chunk)
 template<>
 void ReplicatedMergeTreeSinkImpl<false>::finishDelayedChunk(const ZooKeeperWithFaultInjectionPtr & zookeeper)
 {
-    size_t addr = delayed_chunk ? size_t(delayed_chunk.get()) : 0;
-    LOG_INFO(log, "finishDelayedChunk {}", addr);
-
     if (!delayed_chunk)
         return;
 
@@ -480,22 +470,16 @@ void ReplicatedMergeTreeSinkImpl<false>::finishDelayedChunk(const ZooKeeperWithF
         {
             auto counters_snapshot = std::make_shared<ProfileEvents::Counters::Snapshot>(partition.part_counters.getPartiallyAtomicSnapshot());
             PartLog::addNewPart(storage.getContext(), PartLog::PartLogEntry(part, partition.elapsed_ns, counters_snapshot), ExecutionStatus::fromCurrentException("", true));
-
-            size_t addr1 = delayed_chunk ? size_t(delayed_chunk.get()) : 0;
-            LOG_INFO(log, "finishDelayedChunk exception, delayed_chunk {}", addr1);
             throw;
         }
     }
 
     delayed_chunk.reset();
-
-    LOG_INFO(log, "finishDelayedChunk end, delayed_chunk {}", bool(delayed_chunk));
 }
 
 template<>
 void ReplicatedMergeTreeSinkImpl<true>::finishDelayedChunk(const ZooKeeperWithFaultInjectionPtr & zookeeper)
 {
-
     if (!delayed_chunk)
         return;
 
diff --git a/src/Storages/MessageQueueSink.cpp b/src/Storages/MessageQueueSink.cpp
index 4fb81d69070..10617422f40 100644
--- a/src/Storages/MessageQueueSink.cpp
+++ b/src/Storages/MessageQueueSink.cpp
@@ -1,6 +1,7 @@
 #include <Storages/MessageQueueSink.h>
 #include <Formats/FormatFactory.h>
 #include <Processors/Formats/IRowOutputFormat.h>
+#include <Common/Exception.h>
 #include <Common/logger_useful.h>
 
 namespace DB
@@ -79,4 +80,16 @@ void MessageQueueSink::consume(Chunk chunk)
 }
 
 
+void MessageQueueSink::onCancel() noexcept
+{
+    try
+    {
+        onFinish();
+    }
+    catch (...)
+    {
+        tryLogCurrentException(getLogger("MessageQueueSink"), "Error occurs on cancelation.");
+    }
+}
+
 }
diff --git a/src/Storages/MessageQueueSink.h b/src/Storages/MessageQueueSink.h
index 38754e9475e..6964af8cb4d 100644
--- a/src/Storages/MessageQueueSink.h
+++ b/src/Storages/MessageQueueSink.h
@@ -33,17 +33,13 @@ public:
         const String & storage_name_,
         const ContextPtr & context_);
 
-    ~MessageQueueSink() override
-    {
-        onFinish();
-    }
-
     String getName() const override { return storage_name + "Sink"; }
 
     void consume(Chunk chunk) override;
 
     void onStart() override;
     void onFinish() override;
+    void onCancel() noexcept override;
     void onException(std::exception_ptr /* exception */) override { onFinish(); }
 
 protected:
diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSink.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSink.cpp
index 3bd0e88ecdb..d5813015e13 100644
--- a/src/Storages/ObjectStorage/StorageObjectStorageSink.cpp
+++ b/src/Storages/ObjectStorage/StorageObjectStorageSink.cpp
@@ -2,8 +2,8 @@
 #include <Formats/FormatFactory.h>
 #include <Disks/ObjectStorages/IObjectStorage.h>
 #include <Common/isValidUTF8.h>
-#include "base/defines.h"
 #include <Storages/ObjectStorage/Utils.h>
+#include <base/defines.h>
 
 namespace DB
 {

From d4e7188750755a33148eb12536cd3687efb8354d Mon Sep 17 00:00:00 2001
From: Max K <maxkaynov@gmail.com>
Date: Mon, 15 Jul 2024 16:32:49 +0200
Subject: [PATCH 0267/1488] CI: Report job start and finish to CI DB

---
 tests/ci/ci.py     | 236 ++++++++++++++++++++++++++-------------------
 tests/ci/report.py |  14 ++-
 2 files changed, 146 insertions(+), 104 deletions(-)

diff --git a/tests/ci/ci.py b/tests/ci/ci.py
index cf285f4b97d..110a7b2a49c 100644
--- a/tests/ci/ci.py
+++ b/tests/ci/ci.py
@@ -48,7 +48,19 @@ from git_helper import GIT_PREFIX, Git
 from git_helper import Runner as GitRunner
 from github_helper import GitHub
 from pr_info import PRInfo
-from report import ERROR, FAILURE, PENDING, SUCCESS, BuildResult, JobReport, TestResult
+from report import (
+    ERROR,
+    FAILURE,
+    PENDING,
+    SUCCESS,
+    BuildResult,
+    JobReport,
+    TestResult,
+    OK,
+    JOB_STARTED_TEST_NAME,
+    JOB_FINISHED_TEST_NAME,
+    FAIL,
+)
 from s3_helper import S3Helper
 from stopwatch import Stopwatch
 from tee_popen import TeePopen
@@ -263,7 +275,8 @@ def check_missing_images_on_dockerhub(
     return result
 
 
-def _pre_action(s3, indata, pr_info):
+def _pre_action(s3, job_name, batch, indata, pr_info):
+    no_cache = CiSettings.create_from_run_config(indata).no_ci_cache
     print("Clear dmesg")
     Utils.clear_dmesg()
     CommitStatusData.cleanup()
@@ -282,6 +295,90 @@ def _pre_action(s3, indata, pr_info):
 
     ci_cache.dump_run_config(indata)
 
+    to_be_skipped = False
+    skip_status = SUCCESS
+    # check if job was run already
+    if CI.is_build_job(job_name):
+        # this is a build job - check if a build report is present
+        build_result = (
+            BuildResult.load_any(job_name, pr_info.number, pr_info.head_ref)
+            if not no_cache
+            else None
+        )
+        if build_result:
+            if build_result.status == SUCCESS:
+                to_be_skipped = True
+            else:
+                print(
+                    "Build report found but status is unsuccessful - will try to rerun"
+                )
+            print("::group::Build Report")
+            print(build_result.as_json())
+            print("::endgroup::")
+    else:
+        # this is a test job - check if GH commit status or cache record is present
+        commit = get_commit(GitHub(get_best_robot_token(), per_page=100), pr_info.sha)
+        # rerun helper check
+        # FIXME: Find a way to identify if job restarted manually (by developer) or by automatic workflow restart (died spot-instance)
+        #  disable rerun check for the former
+        if job_name not in (
+            CI.JobNames.BUILD_CHECK,
+        ):  # we might want to rerun build report job
+            rerun_helper = RerunHelper(commit, _get_ext_check_name(job_name))
+            if rerun_helper.is_already_finished_by_status():
+                print("WARNING: Rerunning job with GH status ")
+                status = rerun_helper.get_finished_status()
+                assert status
+                print("::group::Commit Status")
+                print(status)
+                print("::endgroup::")
+                to_be_skipped = True
+                skip_status = status.state
+
+        # ci cache check
+        if not to_be_skipped and not no_cache:
+            ci_cache = CiCache(s3, indata["jobs_data"]["digests"]).update()
+            job_config = CI.get_job_config(job_name)
+            if ci_cache.is_successful(
+                job_name,
+                batch,
+                job_config.num_batches,
+                job_config.required_on_release_branch,
+            ):
+                print("CICache record has be found - job will be skipped")
+                job_status = ci_cache.get_successful(
+                    job_name, batch, job_config.num_batches
+                )
+                assert job_status, "BUG"
+                _create_gh_status(
+                    commit,
+                    job_name,
+                    batch,
+                    job_config.num_batches,
+                    job_status,
+                )
+                to_be_skipped = True
+                # skip_status = SUCCESS already there
+                GHActions.print_in_group("Commit Status Data", job_status)
+
+    # create pre report
+    jr = JobReport.create_pre_report(status=skip_status, job_skipped=to_be_skipped)
+    jr.dump()
+
+    if not to_be_skipped:
+        print("push start record to ci db")
+        prepared_events = prepare_tests_results_for_clickhouse(
+            pr_info,
+            [TestResult(JOB_STARTED_TEST_NAME, OK)],
+            SUCCESS,
+            0.0,
+            JobReport.get_start_time_from_current(),
+            "",
+            _get_ext_check_name(job_name),
+        )
+        ClickHouseHelper().insert_events_into(
+            db="default", table="checks", events=prepared_events
+        )
     print(f"Pre action done. Report files [{reports_files}] have been downloaded")
 
 
@@ -1045,108 +1142,23 @@ def main() -> int:
     ### PRE action: start
     elif args.pre:
         assert indata, "Run config must be provided via --infile"
-        _pre_action(s3, indata, pr_info)
-        JobReport.create_pre_report().dump()
+        _pre_action(s3, args.job_name, args.batch, indata, pr_info)
 
     ### RUN action: start
     elif args.run:
         assert indata
-        ci_settings = CiSettings.create_from_run_config(indata)
+        job_report = JobReport.load()
         check_name = args.job_name
         check_name_with_group = _get_ext_check_name(check_name)
         print(
             f"Check if rerun for name: [{check_name}], extended name [{check_name_with_group}]"
         )
-        previous_status = None
-        if CI.is_build_job(check_name):
-            # this is a build job - check if a build report is present
-            build_result = (
-                BuildResult.load_any(check_name, pr_info.number, pr_info.head_ref)
-                if not ci_settings.no_ci_cache
-                else None
-            )
-            if build_result:
-                if build_result.status == SUCCESS:
-                    previous_status = build_result.status
-                    JobReport(
-                        status=SUCCESS,
-                        description="",
-                        test_results=[],
-                        start_time="",
-                        duration=0.0,
-                        additional_files=[],
-                        job_skipped=True,
-                    ).dump()
-                else:
-                    # FIXME: Consider reusing failures for build jobs.
-                    #   Just remove this if/else - that makes build job starting and failing immediately
-                    print(
-                        "Build report found but status is unsuccessful - will try to rerun"
-                    )
-                print("::group::Build Report")
-                print(build_result.as_json())
-                print("::endgroup::")
-        else:
-            # this is a test job - check if GH commit status or cache record is present
-            commit = get_commit(
-                GitHub(get_best_robot_token(), per_page=100), pr_info.sha
-            )
 
-            # rerun helper check
-            # FIXME: Find a way to identify if job restarted manually (by developer) or by automatic workflow restart (died spot-instance)
-            #  disable rerun check for the former
-            if check_name not in (
-                CI.JobNames.BUILD_CHECK,
-            ):  # we might want to rerun build report job
-                rerun_helper = RerunHelper(commit, check_name_with_group)
-                if rerun_helper.is_already_finished_by_status():
-                    print("WARNING: Rerunning job with GH status ")
-                    status = rerun_helper.get_finished_status()
-                    assert status
-                    print("::group::Commit Status")
-                    print(status)
-                    print("::endgroup::")
-                    previous_status = status.state
-                    print("Create dummy job report with job_skipped flag")
-                    JobReport(
-                        status=status.state,
-                        description="",
-                        test_results=[],
-                        start_time="",
-                        duration=0.0,
-                        additional_files=[],
-                        job_skipped=True,
-                    ).dump()
-
-            # ci cache check
-            if not previous_status and not ci_settings.no_ci_cache:
-                ci_cache = CiCache(s3, indata["jobs_data"]["digests"]).update()
-                job_config = CI.get_job_config(check_name)
-                if ci_cache.is_successful(
-                    check_name,
-                    args.batch,
-                    job_config.num_batches,
-                    job_config.required_on_release_branch,
-                ):
-                    job_status = ci_cache.get_successful(
-                        check_name, args.batch, job_config.num_batches
-                    )
-                    assert job_status, "BUG"
-                    _create_gh_status(
-                        commit,
-                        check_name,
-                        args.batch,
-                        job_config.num_batches,
-                        job_status,
-                    )
-                    previous_status = job_status.status
-                    GHActions.print_in_group("Commit Status Data", job_status)
-
-        if previous_status and not args.force:
+        if job_report.job_skipped and not args.force:
             print(
-                f"Commit status or Build Report is already present - job will be skipped with status: [{previous_status}]"
+                f"Commit status or Build Report is already present - job will be skipped with status: [{job_report.status}]"
             )
-            if previous_status == SUCCESS:
+            if job_report.status == SUCCESS:
                 exit_code = 0
             else:
                 exit_code = 1
@@ -1166,7 +1178,8 @@ def main() -> int:
         assert (
             job_report
         ), "BUG. There must be job report either real report, or pre-report if job was killed"
-        if not job_report.job_skipped and not job_report.pre_report:
+        error_description = ""
+        if not job_report.pre_report:
             # it's a real job report
             ch_helper = ClickHouseHelper()
             check_url = ""
@@ -1244,7 +1257,6 @@ def main() -> int:
                     pr_info,
                     dump_to_file=True,
                 )
-
             print(f"Job report url: [{check_url}]")
             prepared_events = prepare_tests_results_for_clickhouse(
                 pr_info,
@@ -1269,9 +1281,7 @@ def main() -> int:
                 )
         elif job_report.job_skipped:
             print(f"Skipped after rerun check {[args.job_name]} - do nothing")
-        elif job_report.job_skipped:
-            print(f"Job was skipped {[args.job_name]} - do nothing")
-        elif job_report.pre_report:
+        else:
             print(f"ERROR: Job was killed - generate evidence")
             job_report.update_duration()
             ret_code = os.getenv("JOB_EXIT_CODE", "")
@@ -1282,10 +1292,13 @@ def main() -> int:
                     pass
             if Utils.is_killed_with_oom():
                 print("WARNING: OOM while job execution")
-                error = f"Out Of Memory, exit_code {job_report.exit_code}, after {int(job_report.duration)}s"
+                error_description = f"Out Of Memory, exit_code {job_report.exit_code}"
             else:
-                error = f"Unknown, exit_code {job_report.exit_code}, after {int(job_report.duration)}s"
-            CIBuddy().post_error(error, job_name=_get_ext_check_name(args.job_name))
+                error_description = f"Unknown, exit_code {job_report.exit_code}"
+            CIBuddy().post_error(
+                error_description + f" after {int(job_report.duration)}s",
+                job_name=_get_ext_check_name(args.job_name),
+            )
             if CI.is_test_job(args.job_name):
                 gh = GitHub(get_best_robot_token(), per_page=100)
                 commit = get_commit(gh, pr_info.sha)
@@ -1293,11 +1306,32 @@ def main() -> int:
                     commit,
                     ERROR,
                     "",
-                    "Error: " + error,
+                    "Error: " + error_description,
                     _get_ext_check_name(args.job_name),
                     pr_info,
                     dump_to_file=True,
                 )
+
+        if not job_report.job_skipped:
+            print("push finish record to ci db")
+            prepared_events = prepare_tests_results_for_clickhouse(
+                pr_info,
+                [
+                    TestResult(
+                        JOB_FINISHED_TEST_NAME,
+                        FAIL if error_description else OK,
+                        raw_logs=error_description or None,
+                    )
+                ],
+                SUCCESS if not error_description else ERROR,
+                0.0,
+                JobReport.get_start_time_from_current(),
+                "",
+                _get_ext_check_name(args.job_name),
+            )
+            ClickHouseHelper().insert_events_into(
+                db="default", table="checks", events=prepared_events
+            )
     ### POST action: end
 
     ### MARK SUCCESS action: start
diff --git a/tests/ci/report.py b/tests/ci/report.py
index 4be7b438f4f..77043dfc8be 100644
--- a/tests/ci/report.py
+++ b/tests/ci/report.py
@@ -247,6 +247,9 @@ BASE_HEADERS = ["Test name", "Test status"]
 # should not be in TEMP directory or any directory that may be cleaned during the job execution
 JOB_REPORT_FILE = Path(GITHUB_WORKSPACE) / "job_report.json"
 
+JOB_STARTED_TEST_NAME = "STARTED"
+JOB_FINISHED_TEST_NAME = "COMPLETED"
+
 
 @dataclass
 class TestResult:
@@ -304,14 +307,19 @@ class JobReport:
     exit_code: int = -1
 
     @staticmethod
-    def create_pre_report() -> "JobReport":
+    def get_start_time_from_current():
+        return datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
+
+    @classmethod
+    def create_pre_report(cls, status: str, job_skipped: bool) -> "JobReport":
         return JobReport(
-            status=ERROR,
+            status=status,
             description="",
             test_results=[],
-            start_time=datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S"),
+            start_time=cls.get_start_time_from_current(),
             duration=0.0,
             additional_files=[],
+            job_skipped=job_skipped,
             pre_report=True,
         )
 

From 12101f455b06a3df1bd00b8b070b4c7862d087bf Mon Sep 17 00:00:00 2001
From: Sema Checherinda <Sema.Checherinda@clickhouse.com>
Date: Mon, 15 Jul 2024 17:55:02 +0200
Subject: [PATCH 0268/1488] fix typo

---
 src/Processors/Sources/RemoteSource.cpp      | 2 +-
 src/Storages/Distributed/DistributedSink.cpp | 4 ++--
 src/Storages/MessageQueueSink.cpp            | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/Processors/Sources/RemoteSource.cpp b/src/Processors/Sources/RemoteSource.cpp
index 42696f9c3ce..48a6804de9a 100644
--- a/src/Processors/Sources/RemoteSource.cpp
+++ b/src/Processors/Sources/RemoteSource.cpp
@@ -192,7 +192,7 @@ void RemoteSource::onCancel() noexcept
     }
     catch (...)
     {
-        tryLogCurrentException(getLogger("RemoteSource"), "Error occurs on cancelation.");
+        tryLogCurrentException(getLogger("RemoteSource"), "Error occurs on cancellation.");
     }
 }
 
diff --git a/src/Storages/Distributed/DistributedSink.cpp b/src/Storages/Distributed/DistributedSink.cpp
index fa4ba01a37c..197905c0849 100644
--- a/src/Storages/Distributed/DistributedSink.cpp
+++ b/src/Storages/Distributed/DistributedSink.cpp
@@ -608,7 +608,7 @@ void DistributedSink::onCancel() noexcept
         }
         catch (...)
         {
-            tryLogCurrentException(storage.log, "Error occurs on cancelation.");
+            tryLogCurrentException(storage.log, "Error occurs on cancellation.");
         }
     }
 
@@ -623,7 +623,7 @@ void DistributedSink::onCancel() noexcept
             }
             catch (...)
             {
-                tryLogCurrentException(storage.log, "Error occurs on cancelation.");
+                tryLogCurrentException(storage.log, "Error occurs on cancellation.");
             }
         }
     }
diff --git a/src/Storages/MessageQueueSink.cpp b/src/Storages/MessageQueueSink.cpp
index 10617422f40..9cddb2e7ce8 100644
--- a/src/Storages/MessageQueueSink.cpp
+++ b/src/Storages/MessageQueueSink.cpp
@@ -88,7 +88,7 @@ void MessageQueueSink::onCancel() noexcept
     }
     catch (...)
     {
-        tryLogCurrentException(getLogger("MessageQueueSink"), "Error occurs on cancelation.");
+        tryLogCurrentException(getLogger("MessageQueueSink"), "Error occurs on cancellation.");
     }
 }
 

From 51c1936b7f15327a1685472b41178f607f1c8670 Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Mon, 15 Jul 2024 18:28:06 +0200
Subject: [PATCH 0269/1488] Correctly track memory for Allocator::realloc

---
 src/Common/Allocator.cpp                      | 12 +++--
 .../configs/async_metrics_no.xml              | 21 ++++++++
 tests/integration/test_memory_limit/test.py   | 52 +++++++++++++++++++
 3 files changed, 82 insertions(+), 3 deletions(-)
 create mode 100644 tests/integration/test_memory_limit/configs/async_metrics_no.xml
 create mode 100644 tests/integration/test_memory_limit/test.py

diff --git a/src/Common/Allocator.cpp b/src/Common/Allocator.cpp
index bfc85559fe8..c8373e74501 100644
--- a/src/Common/Allocator.cpp
+++ b/src/Common/Allocator.cpp
@@ -187,10 +187,10 @@ void * Allocator<clear_memory_, populate>::realloc(void * buf, size_t old_size,
 #if USE_GWP_ASAN
     if (unlikely(GWPAsan::GuardedAlloc.shouldSample()))
     {
+        auto trace_alloc = CurrentMemoryTracker::alloc(new_size);
         if (void * ptr = GWPAsan::GuardedAlloc.allocate(new_size, alignment))
         {
             auto trace_free = CurrentMemoryTracker::free(old_size);
-            auto trace_alloc = CurrentMemoryTracker::alloc(new_size);
             trace_free.onFree(buf, old_size);
 
             memcpy(ptr, buf, std::min(old_size, new_size));
@@ -231,13 +231,17 @@ void * Allocator<clear_memory_, populate>::realloc(void * buf, size_t old_size,
     if (alignment <= MALLOC_MIN_ALIGNMENT)
     {
         /// Resize malloc'd memory region with no special alignment requirement.
-        auto trace_free = CurrentMemoryTracker::free(old_size);
+        /// Realloc can do 2 possible things:
+        /// - expand existing memory region
+        /// - allocate new memory block and free the old one
+        /// Because we don't know which option will be picked we need to make sure there is enough
+        /// memory for all options
         auto trace_alloc = CurrentMemoryTracker::alloc(new_size);
-        trace_free.onFree(buf, old_size);
 
         void * new_buf = ::realloc(buf, new_size);
         if (nullptr == new_buf)
         {
+            [[maybe_unused]] auto trace_free = CurrentMemoryTracker::free(old_size);
             throw DB::ErrnoException(
                 DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY,
                 "Allocator: Cannot realloc from {} to {}",
@@ -246,6 +250,8 @@ void * Allocator<clear_memory_, populate>::realloc(void * buf, size_t old_size,
         }
 
         buf = new_buf;
+        auto trace_free = CurrentMemoryTracker::free(old_size);
+        trace_free.onFree(buf, old_size);
         trace_alloc.onAlloc(buf, new_size);
 
         if constexpr (clear_memory)
diff --git a/tests/integration/test_memory_limit/configs/async_metrics_no.xml b/tests/integration/test_memory_limit/configs/async_metrics_no.xml
new file mode 100644
index 00000000000..96cae3bf387
--- /dev/null
+++ b/tests/integration/test_memory_limit/configs/async_metrics_no.xml
@@ -0,0 +1,21 @@
+<clickhouse>
+    <!-- this update period also syncs MemoryTracking with RSS, disable this, by using period = 1 day -->
+    <asynchronous_metrics_update_period_s>86400</asynchronous_metrics_update_period_s>
+    <query_masking_rules remove="remove"/>
+
+    <query_thread_log remove="remove"/>
+    <query_log remove="remove" />
+    <query_views_log remove="remove" />
+    <metric_log remove="remove"/>
+    <error_log remove="remove"/>
+    <text_log remove="remove"/>
+    <trace_log remove="remove"/>
+    <asynchronous_metric_log remove="remove" />
+    <session_log remove="remove" />
+    <part_log remove="remove" />
+    <crash_log remove="remove" />
+    <opentelemetry_span_log remove="remove" />
+    <!-- just in case it will be enabled by default -->
+    <zookeeper_log remove="remove" />
+    <transactions_info_log remove="remove" />
+</clickhouse>
diff --git a/tests/integration/test_memory_limit/test.py b/tests/integration/test_memory_limit/test.py
new file mode 100644
index 00000000000..292d3ad5719
--- /dev/null
+++ b/tests/integration/test_memory_limit/test.py
@@ -0,0 +1,52 @@
+#!/usr/bin/env python3
+import logging
+import time
+import pytest
+from helpers.cluster import ClickHouseCluster
+from multiprocessing.dummy import Pool
+
+cluster = ClickHouseCluster(__file__)
+
+node = cluster.add_instance(
+    "node",
+    main_configs=[
+        "configs/async_metrics_no.xml",
+    ],
+    env_variables={
+        "MALLOC_CONF": "dirty_decay_ms:0"
+    }
+)
+
+@pytest.fixture(scope="module", autouse=True)
+def start_cluster():
+    try:
+        cluster.start()
+        yield cluster
+    finally:
+        cluster.shutdown()
+
+def test_multiple_queries():
+    p = Pool(15)
+
+    def run_query(node):
+        try:
+            node.query("SELECT * FROM system.numbers GROUP BY number")
+        except Exception as ex:
+            print("Exception", ex)
+            raise ex
+
+    tasks = []
+    for i in range(30):
+        tasks.append(p.apply_async(run_query, (node, )))
+        time.sleep(i * 0.1)
+
+
+    for task in tasks:
+        try:
+            task.get()
+        except Exception as ex:
+            print("Exception", ex)
+
+
+    # test that we didn't kill the server
+    node.query("SELECT 1")

From 10c553b06450dfb34d63b8b37b8fbdd19e98ad5f Mon Sep 17 00:00:00 2001
From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com>
Date: Mon, 15 Jul 2024 18:36:19 +0200
Subject: [PATCH 0270/1488] empty commit


From 7d70968db3527d894bc6c02d51dc70f932f7eacd Mon Sep 17 00:00:00 2001
From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com>
Date: Mon, 15 Jul 2024 18:47:04 +0200
Subject: [PATCH 0271/1488] try fix

---
 src/Interpreters/InterpreterCreateQuery.cpp | 52 ++++++++++-----------
 1 file changed, 26 insertions(+), 26 deletions(-)

diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp
index f8696caebe7..9eb13a29af7 100644
--- a/src/Interpreters/InterpreterCreateQuery.cpp
+++ b/src/Interpreters/InterpreterCreateQuery.cpp
@@ -1010,32 +1010,32 @@ void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const
             setDefaultTableEngine(*create.storage, getContext()->getSettingsRef().default_table_engine.value);
         /// For exrternal tables with restore_replace_external_engine_to_null setting we replace external engines to
         /// Null table engine.
-        else (create.storage->engine == "AzureBlobStorage" ||
-              create.storage->engine == "AzureQueue" ||
-              create.storage->engine == "COSN" ||
-              create.storage->engine == "DeltaLake" ||
-              create.storage->engine == "Dictionary" ||
-              create.storage->engine == "Executable" ||
-              create.storage->engine == "ExecutablePool" ||
-              create.storage->engine == "ExternalDistributed" ||
-              create.storage->engine == "File" ||
-              create.storage->engine == "Hudi" ||
-              create.storage->engine == "Iceberg" ||
-              create.storage->engine == "JDBC" ||
-              create.storage->engine == "Kafka" ||
-              create.storage->engine == "MaterializedPostgreSQL" ||
-              create.storage->engine == "MongoDB" ||
-              create.storage->engine == "MySQL" ||
-              create.storage->engine == "NATS" ||
-              create.storage->engine == "ODBC" ||
-              create.storage->engine == "OSS" ||
-              create.storage->engine == "PostgreSQL" ||
-              create.storage->engine == "RabbitMQ" ||
-              create.storage->engine == "Redis" ||
-              create.storage->engine == "S3" ||
-              create.storage->engine == "S3Queue" ||
-              create.storage->engine == "TinyLog" ||
-              create.storage->engine == "URL")
+        else (create.storage->engine->name == "AzureBlobStorage" ||
+              create.storage->engine->name == "AzureQueue" ||
+              create.storage->engine->name == "COSN" ||
+              create.storage->engine->name == "DeltaLake" ||
+              create.storage->engine->name == "Dictionary" ||
+              create.storage->engine->name == "Executable" ||
+              create.storage->engine->name == "ExecutablePool" ||
+              create.storage->engine->name == "ExternalDistributed" ||
+              create.storage->engine->name == "File" ||
+              create.storage->engine->name == "Hudi" ||
+              create.storage->engine->name == "Iceberg" ||
+              create.storage->engine->name == "JDBC" ||
+              create.storage->engine->name == "Kafka" ||
+              create.storage->engine->name == "MaterializedPostgreSQL" ||
+              create.storage->engine->name == "MongoDB" ||
+              create.storage->engine->name == "MySQL" ||
+              create.storage->engine->name == "NATS" ||
+              create.storage->engine->name == "ODBC" ||
+              create.storage->engine->name == "OSS" ||
+              create.storage->engine->name == "PostgreSQL" ||
+              create.storage->engine->name == "RabbitMQ" ||
+              create.storage->engine->name == "Redis" ||
+              create.storage->engine->name == "S3" ||
+              create.storage->engine->name == "S3Queue" ||
+              create.storage->engine->name == "TinyLog" ||
+              create.storage->engine->name == "URL")
         {
             if (getContext()->getSettingsRef().restore_replace_external_engine_to_null)
                 setNullTableEngine(*create.storage)

From fc49b1b75f9b075f28cdc4b7eeb768339bb1ebd5 Mon Sep 17 00:00:00 2001
From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com>
Date: Mon, 15 Jul 2024 19:02:21 +0200
Subject: [PATCH 0272/1488] semicolon

---
 src/Interpreters/InterpreterCreateQuery.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp
index 9eb13a29af7..94230f0e7d1 100644
--- a/src/Interpreters/InterpreterCreateQuery.cpp
+++ b/src/Interpreters/InterpreterCreateQuery.cpp
@@ -1038,7 +1038,7 @@ void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const
               create.storage->engine->name == "URL")
         {
             if (getContext()->getSettingsRef().restore_replace_external_engine_to_null)
-                setNullTableEngine(*create.storage)
+                setNullTableEngine(*create.storage);
         }
         return;
     }

From 7b4c6a201bbc000b1a3482c772a9c6762141c6ee Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Mon, 15 Jul 2024 19:05:23 +0200
Subject: [PATCH 0273/1488] Limit the memory please

---
 tests/integration/test_memory_limit/__init__.py |  0
 tests/integration/test_memory_limit/test.py     | 11 +++++------
 2 files changed, 5 insertions(+), 6 deletions(-)
 create mode 100644 tests/integration/test_memory_limit/__init__.py

diff --git a/tests/integration/test_memory_limit/__init__.py b/tests/integration/test_memory_limit/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/integration/test_memory_limit/test.py b/tests/integration/test_memory_limit/test.py
index 292d3ad5719..942cf04f896 100644
--- a/tests/integration/test_memory_limit/test.py
+++ b/tests/integration/test_memory_limit/test.py
@@ -12,11 +12,11 @@ node = cluster.add_instance(
     main_configs=[
         "configs/async_metrics_no.xml",
     ],
-    env_variables={
-        "MALLOC_CONF": "dirty_decay_ms:0"
-    }
+    mem_limit="4g",
+    env_variables={"MALLOC_CONF": "dirty_decay_ms:0"},
 )
 
+
 @pytest.fixture(scope="module", autouse=True)
 def start_cluster():
     try:
@@ -25,6 +25,7 @@ def start_cluster():
     finally:
         cluster.shutdown()
 
+
 def test_multiple_queries():
     p = Pool(15)
 
@@ -37,16 +38,14 @@ def test_multiple_queries():
 
     tasks = []
     for i in range(30):
-        tasks.append(p.apply_async(run_query, (node, )))
+        tasks.append(p.apply_async(run_query, (node,)))
         time.sleep(i * 0.1)
 
-
     for task in tasks:
         try:
             task.get()
         except Exception as ex:
             print("Exception", ex)
 
-
     # test that we didn't kill the server
     node.query("SELECT 1")

From 3aca36c54585ea3fcb2f8a303b96c178fbdf7ecc Mon Sep 17 00:00:00 2001
From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com>
Date: Mon, 15 Jul 2024 19:23:29 +0200
Subject: [PATCH 0274/1488] Update AggregateFunctionGroupArrayIntersect.cpp


From 9adc5b9ba4650dc87d361bc88f7323fdac776125 Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Mon, 15 Jul 2024 19:23:33 +0200
Subject: [PATCH 0275/1488] more fixes

---
 src/Common/Allocator.cpp | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/Common/Allocator.cpp b/src/Common/Allocator.cpp
index c8373e74501..bcc5d187047 100644
--- a/src/Common/Allocator.cpp
+++ b/src/Common/Allocator.cpp
@@ -190,9 +190,6 @@ void * Allocator<clear_memory_, populate>::realloc(void * buf, size_t old_size,
         auto trace_alloc = CurrentMemoryTracker::alloc(new_size);
         if (void * ptr = GWPAsan::GuardedAlloc.allocate(new_size, alignment))
         {
-            auto trace_free = CurrentMemoryTracker::free(old_size);
-            trace_free.onFree(buf, old_size);
-
             memcpy(ptr, buf, std::min(old_size, new_size));
             free(buf, old_size);
             trace_alloc.onAlloc(buf, new_size);
@@ -209,6 +206,7 @@ void * Allocator<clear_memory_, populate>::realloc(void * buf, size_t old_size,
         }
         else
         {
+            [[maybe_unused]] auto trace_free = CurrentMemoryTracker::free(old_size);
             ProfileEvents::increment(ProfileEvents::GWPAsanAllocateFailed);
         }
     }

From bc5e2afc86317d832e6e25fb7d7719e44bdc1876 Mon Sep 17 00:00:00 2001
From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com>
Date: Tue, 16 Jul 2024 00:52:17 +0200
Subject: [PATCH 0276/1488] Update StorageDictionary.cpp


From 04735f42297c6526b837e537c74ccbf0d38c0581 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Tue, 16 Jul 2024 02:27:58 +0200
Subject: [PATCH 0277/1488] Fix clang tidy

---
 src/Interpreters/Context.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp
index fc1e87e7b7e..2602afd8b78 100644
--- a/src/Interpreters/Context.cpp
+++ b/src/Interpreters/Context.cpp
@@ -51,7 +51,6 @@
 #include <Interpreters/SessionTracker.h>
 #include <Core/ServerSettings.h>
 #include <Interpreters/PreparedSets.h>
-#include <Core/Settings.h>
 #include <Core/SettingsQuirks.h>
 #include <Access/AccessControl.h>
 #include <Access/ContextAccess.h>

From 083e4b17db62121d6905c35480c3a462dc26477b Mon Sep 17 00:00:00 2001
From: Xu Jia <jia.xu@clickhouse.com>
Date: Tue, 16 Jul 2024 09:34:52 +0800
Subject: [PATCH 0278/1488] trigger CI


From 68aedc219187b0ffe89a3382c7f8136085e72ecf Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Tue, 16 Jul 2024 03:58:55 +0200
Subject: [PATCH 0279/1488] Fix something in Fast Test

---
 docker/test/fasttest/run.sh | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh
index 0d975d64010..58c436b469b 100755
--- a/docker/test/fasttest/run.sh
+++ b/docker/test/fasttest/run.sh
@@ -9,7 +9,11 @@ trap 'kill $(jobs -pr) ||:' EXIT
 stage=${stage:-}
 
 # Compiler version, normally set by Dockerfile
-export LLVM_VERSION=${LLVM_VERSION:-17}
+export LLVM_VERSION=${LLVM_VERSION:-18}
+
+# For some reason it is needed for tests with 'expect' but I don't know, why
+groupadd --system --gid 1000 clickhouse
+useradd --system --gid 1000 --uid 1000 clickhouse
 
 # A variable to pass additional flags to CMake.
 # Here we explicitly default it to nothing so that bash doesn't complain about

From 26f8b81081dd75941bb72b00642391529b9d5bda Mon Sep 17 00:00:00 2001
From: Xu Jia <jia.xu@clickhouse.com>
Date: Tue, 16 Jul 2024 10:54:00 +0800
Subject: [PATCH 0280/1488] update trusted contributors

---
 tests/ci/lambda_shared_package/lambda_shared/pr.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/ci/lambda_shared_package/lambda_shared/pr.py b/tests/ci/lambda_shared_package/lambda_shared/pr.py
index e981e28a454..e3aceaab0de 100644
--- a/tests/ci/lambda_shared_package/lambda_shared/pr.py
+++ b/tests/ci/lambda_shared_package/lambda_shared/pr.py
@@ -45,6 +45,7 @@ TRUSTED_CONTRIBUTORS = {
         "k-morozov",  # Konstantin Morozov, Yandex Cloud
         "justindeguzman",  # ClickHouse, Inc
         "jrdi",  # ClickHouse contributor, TinyBird
+        "XuJia0210", # ClickHouse, Inc
     ]
 }
 

From 919bbf57a7bca538f4a75c0e63af10555b75aa44 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Tue, 16 Jul 2024 05:07:39 +0200
Subject: [PATCH 0281/1488] Fix hashing of empty tuples

---
 src/Functions/FunctionsHashing.h              | 20 ++++++++++++----
 .../03205_hashing_empty_tuples.reference      | 22 ++++++++++++++++++
 .../03205_hashing_empty_tuples.sql            | 23 +++++++++++++++++++
 3 files changed, 60 insertions(+), 5 deletions(-)
 create mode 100644 tests/queries/0_stateless/03205_hashing_empty_tuples.reference
 create mode 100644 tests/queries/0_stateless/03205_hashing_empty_tuples.sql

diff --git a/src/Functions/FunctionsHashing.h b/src/Functions/FunctionsHashing.h
index 27717ea3611..95c54ac9528 100644
--- a/src/Functions/FunctionsHashing.h
+++ b/src/Functions/FunctionsHashing.h
@@ -1184,7 +1184,7 @@ private:
 
         if (icolumn->size() != vec_to.size())
             throw Exception(ErrorCodes::LOGICAL_ERROR, "Argument column '{}' size {} doesn't match result column size {} of function {}",
-                    icolumn->getName(), icolumn->size(), vec_to.size(), getName());
+                icolumn->getName(), icolumn->size(), vec_to.size(), getName());
 
         if constexpr (Keyed)
             if (key_cols.size() != vec_to.size() && key_cols.size() != 1)
@@ -1223,6 +1223,9 @@ private:
         else executeGeneric<first>(key_cols, icolumn, vec_to);
     }
 
+    /// Return a fixed random-looking magic number when input is empty.
+    static constexpr auto filler = 0xe28dbde7fe22e41c;
+
     void executeForArgument(const KeyColumnsType & key_cols, const IDataType * type, const IColumn * column, typename ColumnVector<ToType>::Container & vec_to, bool & is_first) const
     {
         /// Flattening of tuples.
@@ -1231,6 +1234,11 @@ private:
             const auto & tuple_columns = tuple->getColumns();
             const DataTypes & tuple_types = typeid_cast<const DataTypeTuple &>(*type).getElements();
             size_t tuple_size = tuple_columns.size();
+
+            if (0 == tuple_size && is_first)
+                for (auto & hash : vec_to)
+                    hash = static_cast<ToType>(filler);
+
             for (size_t i = 0; i < tuple_size; ++i)
                 executeForArgument(key_cols, tuple_types[i].get(), tuple_columns[i].get(), vec_to, is_first);
         }
@@ -1239,6 +1247,11 @@ private:
             const auto & tuple_columns = tuple_const->getColumns();
             const DataTypes & tuple_types = typeid_cast<const DataTypeTuple &>(*type).getElements();
             size_t tuple_size = tuple_columns.size();
+
+            if (0 == tuple_size && is_first)
+                for (auto & hash : vec_to)
+                    hash = static_cast<ToType>(filler);
+
             for (size_t i = 0; i < tuple_size; ++i)
             {
                 auto tmp = ColumnConst::create(tuple_columns[i], column->size());
@@ -1300,10 +1313,7 @@ public:
             constexpr size_t first_data_argument = Keyed;
 
             if (arguments.size() <= first_data_argument)
-            {
-                /// Return a fixed random-looking magic number when input is empty
-                vec_to.assign(input_rows_count, static_cast<ToType>(0xe28dbde7fe22e41c));
-            }
+                vec_to.assign(input_rows_count, static_cast<ToType>(filler));
 
             KeyColumnsType key_cols{};
             if constexpr (Keyed)
diff --git a/tests/queries/0_stateless/03205_hashing_empty_tuples.reference b/tests/queries/0_stateless/03205_hashing_empty_tuples.reference
new file mode 100644
index 00000000000..e24b5809aee
--- /dev/null
+++ b/tests/queries/0_stateless/03205_hashing_empty_tuples.reference
@@ -0,0 +1,22 @@
+16324913028386710556
+16324913028386710556
+5049034479224883533
+7385293435322750976
+12248912094175844631
+5049034479224883533
+5887129541803688833
+5887129541803688833
+13747979201178469747
+5887129541803688833
+15520217392480966957
+16324913028386710556
+16324913028386710556
+5049034479224883533
+7385293435322750976
+12248912094175844631
+5049034479224883533
+5887129541803688833
+5887129541803688833
+13747979201178469747
+5887129541803688833
+15520217392480966957
diff --git a/tests/queries/0_stateless/03205_hashing_empty_tuples.sql b/tests/queries/0_stateless/03205_hashing_empty_tuples.sql
new file mode 100644
index 00000000000..4a97f30ced3
--- /dev/null
+++ b/tests/queries/0_stateless/03205_hashing_empty_tuples.sql
@@ -0,0 +1,23 @@
+SELECT sipHash64(());
+SELECT sipHash64((), ());
+SELECT sipHash64((), 1);
+SELECT sipHash64(1, ());
+SELECT sipHash64(1, (), 1);
+SELECT sipHash64((), 1, ());
+SELECT sipHash64((), (1, 2));
+SELECT sipHash64((), (1, 2));
+SELECT sipHash64((1, 2), ());
+SELECT sipHash64((), (1, 2), ());
+SELECT sipHash64((1, 2), (), (3, 4));
+
+SELECT sipHash64(materialize(()));
+SELECT sipHash64(materialize(()), materialize(()));
+SELECT sipHash64(materialize(()), 1);
+SELECT sipHash64(1, materialize(()));
+SELECT sipHash64(1, materialize(()), 1);
+SELECT sipHash64((), 1, materialize(()));
+SELECT sipHash64(materialize(()), (1, 2));
+SELECT sipHash64(materialize(()), (1, 2));
+SELECT sipHash64((1, 2), materialize(()));
+SELECT sipHash64(materialize(()), (1, 2), ());
+SELECT sipHash64((1, 2), materialize(()), (3, 4));

From 2c62a6b5283c68aac604a97591571e05e6877fab Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Tue, 16 Jul 2024 05:10:26 +0200
Subject: [PATCH 0282/1488] Update pr.py

---
 .../lambda_shared_package/lambda_shared/pr.py | 21 +------------------
 1 file changed, 1 insertion(+), 20 deletions(-)

diff --git a/tests/ci/lambda_shared_package/lambda_shared/pr.py b/tests/ci/lambda_shared_package/lambda_shared/pr.py
index e3aceaab0de..d970662d38b 100644
--- a/tests/ci/lambda_shared_package/lambda_shared/pr.py
+++ b/tests/ci/lambda_shared_package/lambda_shared/pr.py
@@ -3,48 +3,29 @@
 import re
 from typing import Tuple
 
-# Individual trusted contirbutors who are not in any trusted organization.
+# Individual trusted contributors who are not in any trusted organization.
 # Can be changed in runtime: we will append users that we learned to be in
 # a trusted org, to save GitHub API calls.
 TRUSTED_CONTRIBUTORS = {
     e.lower()
     for e in [
-        "achimbab",  # Kakao corp
         "Algunenano",  # Raúl Marín, ClickHouse, Inc
         "amosbird",
         "azat",  # SEMRush
         "bharatnc",  # Many contributions.
-        "bobrik",  # Seasoned contributor, CloudFlare
         "cwurm",  # ClickHouse, Inc
         "den-crane",  # Documentation contributor
-        "hagen1778",  # Roman Khavronenko, seasoned contributor
-        "hczhcz",
-        "hexiaoting",  # Seasoned contributor
         "ildus",  # adjust, ex-pgpro
-        "javisantana",  # a Spanish ClickHouse enthusiast, ex-Carto
-        "kreuzerkrieg",
-        "nikvas0",
         "nvartolomei",  # Seasoned contributor, CloudFlare
-        "spongedu",  # Seasoned contributor
         "taiyang-li",
         "ucasFL",  # Amos Bird's friend
         "vdimir",  # ClickHouse, Inc
-        "YiuRULE",
-        "zlobober",  # Developer of YT
-        "ilejn",  # Arenadata, responsible for Kerberized Kafka
         "thomoco",  # ClickHouse, Inc
-        "BoloniniD",  # Seasoned contributor, HSE
         "tonickkozlov",  # Cloudflare
         "tylerhannan",  # ClickHouse, Inc
-        "myrrc",  # Mike Kot, DoubleCloud
         "thevar1able",  # ClickHouse, Inc
-        "aalexfvk",
-        "MikhailBurdukov",
         "tsolodov",  # ClickHouse, Inc
-        "kitaisreal",
-        "k-morozov",  # Konstantin Morozov, Yandex Cloud
         "justindeguzman",  # ClickHouse, Inc
-        "jrdi",  # ClickHouse contributor, TinyBird
         "XuJia0210", # ClickHouse, Inc
     ]
 }

From 77d6e781f593fdcfe50506aba855eda747c05c01 Mon Sep 17 00:00:00 2001
From: Duc Canh Le <duccanh.le@ahrefs.com>
Date: Tue, 16 Jul 2024 11:25:18 +0800
Subject: [PATCH 0283/1488] Update src/Storages/VirtualColumnUtils.cpp

Co-authored-by: Kruglov Pavel <48961922+Avogar@users.noreply.github.com>
---
 src/Storages/VirtualColumnUtils.cpp | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/src/Storages/VirtualColumnUtils.cpp b/src/Storages/VirtualColumnUtils.cpp
index f7ffd47f9e8..df833fa6a66 100644
--- a/src/Storages/VirtualColumnUtils.cpp
+++ b/src/Storages/VirtualColumnUtils.cpp
@@ -282,15 +282,13 @@ static const ActionsDAG::Node * splitFilterNodeForAllowedInputs(
             node_copy.children.clear();
             for (const auto * child : node->children)
                 if (const auto * child_copy = splitFilterNodeForAllowedInputs(child, allowed_inputs, additional_nodes, strict))
-                {
-                    /// Expression like (now_allowed AND allowed) is not allowed if strict = true. This is important for
-                    /// trivial count optimization, otherwise we can get incorrect results. For example, if the query is
-                    /// SELECT count() FROM table WHERE _partition_id = '0' AND rowNumberInBlock() = 1, we cannot apply
-                    /// trivial count.
-                    if (strict)
-                        return nullptr;
                     node_copy.children.push_back(child_copy);
-                }
+                /// Expression like (now_allowed AND allowed) is not allowed if strict = true. This is important for
+                /// trivial count optimization, otherwise we can get incorrect results. For example, if the query is
+                /// SELECT count() FROM table WHERE _partition_id = '0' AND rowNumberInBlock() = 1, we cannot apply
+                /// trivial count.
+                else if (strict)
+                    return nullptr;
 
             if (node_copy.children.empty())
                 return nullptr;

From 5d96b684a640ef7cc0f911d25ab8fbde4db9f590 Mon Sep 17 00:00:00 2001
From: Xu Jia <jia.xu@clickhouse.com>
Date: Tue, 16 Jul 2024 11:30:13 +0800
Subject: [PATCH 0284/1488] update format

---
 tests/ci/lambda_shared_package/lambda_shared/pr.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/ci/lambda_shared_package/lambda_shared/pr.py b/tests/ci/lambda_shared_package/lambda_shared/pr.py
index d970662d38b..9320cc4ce8c 100644
--- a/tests/ci/lambda_shared_package/lambda_shared/pr.py
+++ b/tests/ci/lambda_shared_package/lambda_shared/pr.py
@@ -26,7 +26,7 @@ TRUSTED_CONTRIBUTORS = {
         "thevar1able",  # ClickHouse, Inc
         "tsolodov",  # ClickHouse, Inc
         "justindeguzman",  # ClickHouse, Inc
-        "XuJia0210", # ClickHouse, Inc
+        "XuJia0210",  # ClickHouse, Inc
     ]
 }
 

From 1d2f3a1ed5274764a2a051994ebd69464b6b1212 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Tue, 16 Jul 2024 05:56:26 +0200
Subject: [PATCH 0285/1488] Update pr.py

---
 tests/ci/lambda_shared_package/lambda_shared/pr.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/tests/ci/lambda_shared_package/lambda_shared/pr.py b/tests/ci/lambda_shared_package/lambda_shared/pr.py
index 9320cc4ce8c..95130fc2a0f 100644
--- a/tests/ci/lambda_shared_package/lambda_shared/pr.py
+++ b/tests/ci/lambda_shared_package/lambda_shared/pr.py
@@ -9,7 +9,6 @@ from typing import Tuple
 TRUSTED_CONTRIBUTORS = {
     e.lower()
     for e in [
-        "Algunenano",  # Raúl Marín, ClickHouse, Inc
         "amosbird",
         "azat",  # SEMRush
         "bharatnc",  # Many contributions.
@@ -19,11 +18,9 @@ TRUSTED_CONTRIBUTORS = {
         "nvartolomei",  # Seasoned contributor, CloudFlare
         "taiyang-li",
         "ucasFL",  # Amos Bird's friend
-        "vdimir",  # ClickHouse, Inc
         "thomoco",  # ClickHouse, Inc
         "tonickkozlov",  # Cloudflare
         "tylerhannan",  # ClickHouse, Inc
-        "thevar1able",  # ClickHouse, Inc
         "tsolodov",  # ClickHouse, Inc
         "justindeguzman",  # ClickHouse, Inc
         "XuJia0210",  # ClickHouse, Inc

From e292358ade930e2779792365407fa229cd37d079 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Tue, 16 Jul 2024 06:12:42 +0200
Subject: [PATCH 0286/1488] Update run.sh

---
 docker/test/fasttest/run.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh
index 58c436b469b..57e7b6b2f56 100755
--- a/docker/test/fasttest/run.sh
+++ b/docker/test/fasttest/run.sh
@@ -11,9 +11,9 @@ stage=${stage:-}
 # Compiler version, normally set by Dockerfile
 export LLVM_VERSION=${LLVM_VERSION:-18}
 
-# For some reason it is needed for tests with 'expect' but I don't know, why
-groupadd --system --gid 1000 clickhouse
-useradd --system --gid 1000 --uid 1000 clickhouse
+# For some reason, it is needed for tests with 'expect', but I don't know, why.
+sudo groupadd --system --gid 1000 clickhouse
+sudo useradd --system --gid 1000 --uid 1000 clickhouse
 
 # A variable to pass additional flags to CMake.
 # Here we explicitly default it to nothing so that bash doesn't complain about

From 3a8507b6eb906c71177e810a25878bf645147165 Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Tue, 16 Jul 2024 09:04:38 +0200
Subject: [PATCH 0287/1488] Fix test

---
 .../queries/0_stateless/02231_buffer_aggregate_states_leak.sql  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/02231_buffer_aggregate_states_leak.sql b/tests/queries/0_stateless/02231_buffer_aggregate_states_leak.sql
index f1f7c876ba6..3d6b69fe161 100644
--- a/tests/queries/0_stateless/02231_buffer_aggregate_states_leak.sql
+++ b/tests/queries/0_stateless/02231_buffer_aggregate_states_leak.sql
@@ -29,7 +29,7 @@ from in_02231
 group by key;
 
 set optimize_trivial_insert_select = 1;
-insert into in_02231 select * from numbers(10e6) settings max_memory_usage='310Mi', max_threads=1;
+insert into in_02231 select * from numbers(10e6) settings max_memory_usage='400Mi', max_threads=1;
 
 drop table buffer_02231;
 drop table out_02231;

From a9b8c2a29a69d85be827048f02ca85a84ec7afca Mon Sep 17 00:00:00 2001
From: Max K <maxkaynov@gmail.com>
Date: Tue, 16 Jul 2024 10:48:43 +0200
Subject: [PATCH 0288/1488] CI: Do not block Tests_3 unless
 MAX_TOTAL_FAILURES_PER_JOB_BEFORE_BLOCKING_CI

---
 tests/ci/functional_test_check.py  | 28 +++++++++++-
 tests/ci/integration_test_check.py | 19 ++++++++
 tests/ci/merge_pr.py               | 70 ++++--------------------------
 3 files changed, 54 insertions(+), 63 deletions(-)

diff --git a/tests/ci/functional_test_check.py b/tests/ci/functional_test_check.py
index 41c7ed963c9..74dd4d8fbd7 100644
--- a/tests/ci/functional_test_check.py
+++ b/tests/ci/functional_test_check.py
@@ -17,9 +17,19 @@ from download_release_packages import download_last_release
 from env_helper import REPO_COPY, REPORT_PATH, TEMP_PATH
 from get_robot_token import get_parameter_from_ssm
 from pr_info import PRInfo
-from report import ERROR, SUCCESS, JobReport, StatusType, TestResults, read_test_results
+from report import (
+    ERROR,
+    SUCCESS,
+    JobReport,
+    StatusType,
+    TestResults,
+    read_test_results,
+    FAILURE,
+)
 from stopwatch import Stopwatch
 from tee_popen import TeePopen
+from ci_config import CI
+from ci_utils import Utils
 
 NO_CHANGES_MSG = "Nothing to run"
 
@@ -351,7 +361,23 @@ def main():
         additional_files=additional_logs,
     ).dump(to_file=args.report_to_file if args.report_to_file else None)
 
+    should_block_ci = False
     if state != SUCCESS:
+        should_block_ci = True
+
+    if state == FAILURE and CI.is_required(check_name):
+        failed_cnt = Utils.get_failed_tests_number(description)
+        print(
+            f"Job status is [{state}] with [{failed_cnt}] failed test cases. status description [{description}]"
+        )
+        if (
+            failed_cnt
+            and failed_cnt <= CI.MAX_TOTAL_FAILURES_PER_JOB_BEFORE_BLOCKING_CI
+        ):
+            print(f"Won't block the CI workflow")
+            should_block_ci = False
+
+    if should_block_ci:
         sys.exit(1)
 
 
diff --git a/tests/ci/integration_test_check.py b/tests/ci/integration_test_check.py
index 80ac1935d95..6245f0490fc 100644
--- a/tests/ci/integration_test_check.py
+++ b/tests/ci/integration_test_check.py
@@ -23,10 +23,13 @@ from report import (
     TestResult,
     TestResults,
     read_test_results,
+    FAILURE,
 )
 from stopwatch import Stopwatch
 
 import integration_tests_runner as runner
+from ci_config import CI
+from ci_utils import Utils
 
 
 def get_json_params_dict(
@@ -233,7 +236,23 @@ def main():
         additional_files=additional_logs,
     ).dump(to_file=args.report_to_file if args.report_to_file else None)
 
+    should_block_ci = False
     if state != SUCCESS:
+        should_block_ci = True
+
+    if state == FAILURE and CI.is_required(check_name):
+        failed_cnt = Utils.get_failed_tests_number(description)
+        print(
+            f"Job status is [{state}] with [{failed_cnt}] failed test cases. status description [{description}]"
+        )
+        if (
+            failed_cnt
+            and failed_cnt <= CI.MAX_TOTAL_FAILURES_PER_JOB_BEFORE_BLOCKING_CI
+        ):
+            print(f"Won't block the CI workflow")
+            should_block_ci = False
+
+    if should_block_ci:
         sys.exit(1)
 
 
diff --git a/tests/ci/merge_pr.py b/tests/ci/merge_pr.py
index 94456506879..4b57467ebdc 100644
--- a/tests/ci/merge_pr.py
+++ b/tests/ci/merge_pr.py
@@ -27,7 +27,6 @@ from report import SUCCESS, FAILURE
 from env_helper import GITHUB_UPSTREAM_REPOSITORY, GITHUB_REPOSITORY
 from synchronizer_utils import SYNC_BRANCH_PREFIX
 from ci_config import CI
-from ci_utils import Utils
 
 # The team name for accepted approvals
 TEAM_NAME = getenv("GITHUB_TEAM_NAME", "core")
@@ -249,74 +248,21 @@ def main():
     repo = gh.get_repo(args.repo)
 
     if args.set_ci_status:
+        # set Mergeable check status and exit
         assert args.wf_status in (FAILURE, SUCCESS)
-        # set mergeable check status and exit
         commit = get_commit(gh, args.pr_info.sha)
         statuses = get_commit_filtered_statuses(commit)
 
-        max_failed_tests_per_job = 0
-        job_name_with_max_failures = None
-        total_failed_tests = 0
-        failed_to_get_info = False
         has_failed_statuses = False
         for status in statuses:
-            if not CI.is_required(status.context) or status.context in (
-                CI.StatusNames.SYNC,
-                CI.StatusNames.PR_CHECK,
-            ):
-                # CI.StatusNames.SYNC or CI.StatusNames.PR_CHECK should not be checked
-                continue
             print(f"Check status [{status.context}], [{status.state}]")
-            if status.state == FAILURE:
+            if not CI.is_required(status.context) and status.state != SUCCESS:
                 has_failed_statuses = True
-                failed_cnt = Utils.get_failed_tests_number(status.description)
-                if failed_cnt is None:
-                    failed_to_get_info = True
-                    print(
-                        f"WARNING: failed to get number of failed tests from [{status.description}]"
-                    )
-                else:
-                    if failed_cnt > max_failed_tests_per_job:
-                        job_name_with_max_failures = status.context
-                        max_failed_tests_per_job = failed_cnt
-                    total_failed_tests += failed_cnt
-                    print(
-                        f"Failed test cases in [{status.context}] is [{failed_cnt}], total failures [{total_failed_tests}]"
-                    )
-            elif status.state != SUCCESS and status.context not in (
-                CI.StatusNames.SYNC,
-                CI.StatusNames.PR_CHECK,
-            ):
-                # do not block CI on failures in (CI.StatusNames.SYNC, CI.StatusNames.PR_CHECK)
-                has_failed_statuses = True
-                print(
-                    f"Unexpected status for [{status.context}]: [{status.state}] - block further testing"
-                )
-                failed_to_get_info = True
-
-        can_continue = True
-        if total_failed_tests > CI.MAX_TOTAL_FAILURES_BEFORE_BLOCKING_CI:
-            print(
-                f"Required check has [{total_failed_tests}] failed - block further testing"
-            )
-            can_continue = False
-        if max_failed_tests_per_job > CI.MAX_TOTAL_FAILURES_PER_JOB_BEFORE_BLOCKING_CI:
-            print(
-                f"Job [{job_name_with_max_failures}] has [{max_failed_tests_per_job}] failures - block further testing"
-            )
-            can_continue = False
-        if failed_to_get_info:
-            print("Unexpected commit status state - block further testing")
-            can_continue = False
-        if args.wf_status != SUCCESS and not has_failed_statuses:
-            # workflow failed but reason is unknown as no failed statuses present
-            can_continue = False
-            print(
-                "WARNING: Either the runner is faulty or the operating status is unknown. The first is self-healing, the second requires investigation."
-            )
 
         if args.wf_status == SUCCESS or has_failed_statuses:
-            # do not set mergeable check status if args.wf_status == failure, apparently it has died runners and is to be restarted
+            # set Mergeable check if workflow is successful (green)
+            # or if we have GH statuses with failures (red)
+            #    to avoid false-green on a died runner
             state = trigger_mergeable_check(
                 commit,
                 statuses,
@@ -333,10 +279,10 @@ def main():
             print(
                 "Workflow failed but no failed statuses found (died runner?) - cannot set Mergeable Check status"
             )
-
-        if not can_continue:
+        if args.wf_status == SUCCESS and not has_failed_statuses:
+            sys.exit(0)
+        else:
             sys.exit(1)
-        sys.exit(0)
 
     # An ugly and not nice fix to patch the wrong organization URL,
     # see https://github.com/PyGithub/PyGithub/issues/2395#issuecomment-1378629710

From 2bc65fe2080260d16e27df965610197a38052705 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Tue, 16 Jul 2024 09:44:05 +0000
Subject: [PATCH 0289/1488] Make ColumnSparse::updateWeakHash32 consistent with
 internal column.

---
 src/Columns/ColumnSparse.cpp | 38 +++++++++++++++++++++++++++++++++---
 src/Columns/ColumnSparse.h   |  3 +++
 2 files changed, 38 insertions(+), 3 deletions(-)

diff --git a/src/Columns/ColumnSparse.cpp b/src/Columns/ColumnSparse.cpp
index 809586d8810..ea4d23c1678 100644
--- a/src/Columns/ColumnSparse.cpp
+++ b/src/Columns/ColumnSparse.cpp
@@ -4,6 +4,7 @@
 #include <Columns/ColumnConst.h>
 #include <Columns/ColumnsCommon.h>
 #include <Columns/ColumnTuple.h>
+#include <Columns/ColumnVector.h>
 #include <Common/HashTable/Hash.h>
 #include <Common/SipHash.h>
 #include <Common/WeakHash.h>
@@ -684,13 +685,26 @@ void ColumnSparse::updateWeakHash32(WeakHash32 & hash) const
         throw Exception(ErrorCodes::LOGICAL_ERROR, "Size of WeakHash32 does not match size of column: "
         "column size is {}, hash size is {}", _size, hash.getData().size());
 
-    auto offset_it = begin();
+    size_t values_size = values->size();
+    WeakHash32 values_hash(values_size);
+
     auto & hash_data = hash.getData();
+    auto & values_hash_data = values_hash.getData();
+    const auto & offsets_data = getOffsetsData();
+
+    if (getNumberOfDefaultRows() > 0)
+        values_hash_data[0] = hash_data[getFirstDefaultValueIndex()];
+
+    for (size_t i = 0; i < values_size; ++i)
+        values_hash_data[i + 1] = hash_data[offsets_data[i]];
+
+    values->updateWeakHash32(values_hash);
+
+    auto offset_it = begin();
     for (size_t i = 0; i < _size; ++i, ++offset_it)
     {
         size_t value_index = offset_it.getValueIndex();
-        auto data_ref = values->getDataAt(value_index);
-        hash_data[i] = ::updateWeakHash32(reinterpret_cast<const UInt8 *>(data_ref.data), data_ref.size, hash_data[i]);
+        hash_data[i] = values_hash_data[value_index];
     }
 }
 
@@ -807,6 +821,24 @@ size_t ColumnSparse::getValueIndex(size_t n) const
     return it - offsets_data.begin() + 1;
 }
 
+size_t ColumnSparse::getFirstDefaultValueIndex() const
+{
+    if (getNumberOfDefaultRows() == 0)
+        return size();
+
+    const auto & offsets_data = getOffsetsData();
+    size_t off_size = offsets_data.size();
+
+    if (off_size == 0 || offsets_data[0] > 0)
+        return 0;
+
+    size_t idx = 0;
+    while (idx + 1 < off_size && offsets_data[idx] + 1 == offsets_data[idx + 1])
+        ++idx;
+
+    return offsets_data[idx] + 1;
+}
+
 ColumnSparse::Iterator ColumnSparse::getIterator(size_t n) const
 {
     const auto & offsets_data = getOffsetsData();
diff --git a/src/Columns/ColumnSparse.h b/src/Columns/ColumnSparse.h
index 3e34d1de94a..4860f5171f7 100644
--- a/src/Columns/ColumnSparse.h
+++ b/src/Columns/ColumnSparse.h
@@ -173,6 +173,9 @@ public:
     /// O(log(offsets.size())) complexity,
     size_t getValueIndex(size_t n) const;
 
+    /// Returns an index of the first default value, or size() if there is no defaults.
+    size_t getFirstDefaultValueIndex() const;
+
     const IColumn & getValuesColumn() const { return *values; }
     IColumn & getValuesColumn() { return *values; }
 

From 24b9e1885216a62294e68a977437b4d4c62a12f9 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Tue, 16 Jul 2024 09:50:27 +0000
Subject: [PATCH 0290/1488] Add a test.

---
 ...l_window_finctions_and_column_sparce_bug.reference |  8 ++++++++
 ...arallel_window_finctions_and_column_sparce_bug.sql | 11 +++++++++++
 2 files changed, 19 insertions(+)
 create mode 100644 tests/queries/0_stateless/03205_parallel_window_finctions_and_column_sparce_bug.reference
 create mode 100644 tests/queries/0_stateless/03205_parallel_window_finctions_and_column_sparce_bug.sql

diff --git a/tests/queries/0_stateless/03205_parallel_window_finctions_and_column_sparce_bug.reference b/tests/queries/0_stateless/03205_parallel_window_finctions_and_column_sparce_bug.reference
new file mode 100644
index 00000000000..f11ec57a425
--- /dev/null
+++ b/tests/queries/0_stateless/03205_parallel_window_finctions_and_column_sparce_bug.reference
@@ -0,0 +1,8 @@
+false	1	1
+true	1	1
+---
+false	1	1
+false	1	2
+false	1	3
+true	1	1
+true	1	2
diff --git a/tests/queries/0_stateless/03205_parallel_window_finctions_and_column_sparce_bug.sql b/tests/queries/0_stateless/03205_parallel_window_finctions_and_column_sparce_bug.sql
new file mode 100644
index 00000000000..a4c0200813c
--- /dev/null
+++ b/tests/queries/0_stateless/03205_parallel_window_finctions_and_column_sparce_bug.sql
@@ -0,0 +1,11 @@
+create table t(c Int32, d Bool) Engine=MergeTree order by c;
+system stop merges t;
+
+insert into t values (1, 0);
+insert into t values (1, 0);
+insert into t values (1, 1);
+insert into t values (1, 0)(1, 1);
+
+SELECT d, c, row_number() over (partition by d order by c) as c8 FROM t qualify c8=1 order by d settings max_threads=2;
+SELECT '---';
+SELECT d, c, row_number() over (partition by d order by c) as c8 FROM t order by d, c8 settings max_threads=2;

From 902e548a2daf087bdb4363694a91bf665a7f7f9b Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Tue, 16 Jul 2024 09:52:00 +0000
Subject: [PATCH 0291/1488] Fix typo.

---
 ...205_parallel_window_finctions_and_column_sparse_bug.reference} | 0
 ... => 03205_parallel_window_finctions_and_column_sparse_bug.sql} | 0
 2 files changed, 0 insertions(+), 0 deletions(-)
 rename tests/queries/0_stateless/{03205_parallel_window_finctions_and_column_sparce_bug.reference => 03205_parallel_window_finctions_and_column_sparse_bug.reference} (100%)
 rename tests/queries/0_stateless/{03205_parallel_window_finctions_and_column_sparce_bug.sql => 03205_parallel_window_finctions_and_column_sparse_bug.sql} (100%)

diff --git a/tests/queries/0_stateless/03205_parallel_window_finctions_and_column_sparce_bug.reference b/tests/queries/0_stateless/03205_parallel_window_finctions_and_column_sparse_bug.reference
similarity index 100%
rename from tests/queries/0_stateless/03205_parallel_window_finctions_and_column_sparce_bug.reference
rename to tests/queries/0_stateless/03205_parallel_window_finctions_and_column_sparse_bug.reference
diff --git a/tests/queries/0_stateless/03205_parallel_window_finctions_and_column_sparce_bug.sql b/tests/queries/0_stateless/03205_parallel_window_finctions_and_column_sparse_bug.sql
similarity index 100%
rename from tests/queries/0_stateless/03205_parallel_window_finctions_and_column_sparce_bug.sql
rename to tests/queries/0_stateless/03205_parallel_window_finctions_and_column_sparse_bug.sql

From 478616de3d03495cf8c324da9464a9807b51ba41 Mon Sep 17 00:00:00 2001
From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com>
Date: Tue, 16 Jul 2024 10:54:39 +0000
Subject: [PATCH 0292/1488] forgot

---
 src/Interpreters/InterpreterCreateQuery.cpp | 52 ++++++++++-----------
 1 file changed, 26 insertions(+), 26 deletions(-)

diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp
index 94230f0e7d1..3b23c6899e9 100644
--- a/src/Interpreters/InterpreterCreateQuery.cpp
+++ b/src/Interpreters/InterpreterCreateQuery.cpp
@@ -1010,32 +1010,32 @@ void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const
             setDefaultTableEngine(*create.storage, getContext()->getSettingsRef().default_table_engine.value);
         /// For exrternal tables with restore_replace_external_engine_to_null setting we replace external engines to
         /// Null table engine.
-        else (create.storage->engine->name == "AzureBlobStorage" ||
-              create.storage->engine->name == "AzureQueue" ||
-              create.storage->engine->name == "COSN" ||
-              create.storage->engine->name == "DeltaLake" ||
-              create.storage->engine->name == "Dictionary" ||
-              create.storage->engine->name == "Executable" ||
-              create.storage->engine->name == "ExecutablePool" ||
-              create.storage->engine->name == "ExternalDistributed" ||
-              create.storage->engine->name == "File" ||
-              create.storage->engine->name == "Hudi" ||
-              create.storage->engine->name == "Iceberg" ||
-              create.storage->engine->name == "JDBC" ||
-              create.storage->engine->name == "Kafka" ||
-              create.storage->engine->name == "MaterializedPostgreSQL" ||
-              create.storage->engine->name == "MongoDB" ||
-              create.storage->engine->name == "MySQL" ||
-              create.storage->engine->name == "NATS" ||
-              create.storage->engine->name == "ODBC" ||
-              create.storage->engine->name == "OSS" ||
-              create.storage->engine->name == "PostgreSQL" ||
-              create.storage->engine->name == "RabbitMQ" ||
-              create.storage->engine->name == "Redis" ||
-              create.storage->engine->name == "S3" ||
-              create.storage->engine->name == "S3Queue" ||
-              create.storage->engine->name == "TinyLog" ||
-              create.storage->engine->name == "URL")
+        else if (create.storage->engine->name == "AzureBlobStorage" ||
+            create.storage->engine->name == "AzureQueue" ||
+            create.storage->engine->name == "COSN" ||
+            create.storage->engine->name == "DeltaLake" ||
+            create.storage->engine->name == "Dictionary" ||
+            create.storage->engine->name == "Executable" ||
+            create.storage->engine->name == "ExecutablePool" ||
+            create.storage->engine->name == "ExternalDistributed" ||
+            create.storage->engine->name == "File" ||
+            create.storage->engine->name == "Hudi" ||
+            create.storage->engine->name == "Iceberg" ||
+            create.storage->engine->name == "JDBC" ||
+            create.storage->engine->name == "Kafka" ||
+            create.storage->engine->name == "MaterializedPostgreSQL" ||
+            create.storage->engine->name == "MongoDB" ||
+            create.storage->engine->name == "MySQL" ||
+            create.storage->engine->name == "NATS" ||
+            create.storage->engine->name == "ODBC" ||
+            create.storage->engine->name == "OSS" ||
+            create.storage->engine->name == "PostgreSQL" ||
+            create.storage->engine->name == "RabbitMQ" ||
+            create.storage->engine->name == "Redis" ||
+            create.storage->engine->name == "S3" ||
+            create.storage->engine->name == "S3Queue" ||
+            create.storage->engine->name == "TinyLog" ||
+            create.storage->engine->name == "URL")
         {
             if (getContext()->getSettingsRef().restore_replace_external_engine_to_null)
                 setNullTableEngine(*create.storage);

From 85c42348bcb47325be61505a7d908b46be9fe3b3 Mon Sep 17 00:00:00 2001
From: Duc Canh Le <duccanh.le@ahrefs.com>
Date: Tue, 16 Jul 2024 11:28:59 +0000
Subject: [PATCH 0293/1488] address reviews

Signed-off-by: Duc Canh Le <duccanh.le@ahrefs.com>
---
 src/Storages/MergeTree/MergeTreeData.cpp       |  2 +-
 src/Storages/VirtualColumnUtils.cpp            | 18 +++++++++---------
 src/Storages/VirtualColumnUtils.h              | 10 +++++++++-
 ...3_count_with_non_deterministic_function.sql |  2 +-
 4 files changed, 20 insertions(+), 12 deletions(-)

diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index eb07df8f012..faf2741a456 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -1144,7 +1144,7 @@ std::optional<UInt64> MergeTreeData::totalRowsByPartitionPredicateImpl(
     auto metadata_snapshot = getInMemoryMetadataPtr();
     auto virtual_columns_block = getBlockWithVirtualsForFilter(metadata_snapshot, {parts[0]});
 
-    auto filter_dag = VirtualColumnUtils::splitFilterDagForAllowedInputs(filter_actions_dag->getOutputs().at(0), nullptr, /*strict=*/ true);
+    auto filter_dag = VirtualColumnUtils::splitFilterDagForAllowedInputs(filter_actions_dag->getOutputs().at(0), nullptr, /*allow_non_deterministic_functions=*/ false);
     if (!filter_dag)
         return {};
 
diff --git a/src/Storages/VirtualColumnUtils.cpp b/src/Storages/VirtualColumnUtils.cpp
index df833fa6a66..151079154b1 100644
--- a/src/Storages/VirtualColumnUtils.cpp
+++ b/src/Storages/VirtualColumnUtils.cpp
@@ -272,7 +272,7 @@ static const ActionsDAG::Node * splitFilterNodeForAllowedInputs(
     const ActionsDAG::Node * node,
     const Block * allowed_inputs,
     ActionsDAG::Nodes & additional_nodes,
-    bool strict)
+    bool allow_non_deterministic_functions)
 {
     if (node->type == ActionsDAG::ActionType::FUNCTION)
     {
@@ -281,13 +281,13 @@ static const ActionsDAG::Node * splitFilterNodeForAllowedInputs(
             auto & node_copy = additional_nodes.emplace_back(*node);
             node_copy.children.clear();
             for (const auto * child : node->children)
-                if (const auto * child_copy = splitFilterNodeForAllowedInputs(child, allowed_inputs, additional_nodes, strict))
+                if (const auto * child_copy = splitFilterNodeForAllowedInputs(child, allowed_inputs, additional_nodes, allow_non_deterministic_functions))
                     node_copy.children.push_back(child_copy);
-                /// Expression like (now_allowed AND allowed) is not allowed if strict = true. This is important for
+                /// Expression like (now_allowed AND allowed) is not allowed if allow_non_deterministic_functions = true. This is important for
                 /// trivial count optimization, otherwise we can get incorrect results. For example, if the query is
                 /// SELECT count() FROM table WHERE _partition_id = '0' AND rowNumberInBlock() = 1, we cannot apply
                 /// trivial count.
-                else if (strict)
+                else if (!allow_non_deterministic_functions)
                     return nullptr;
 
             if (node_copy.children.empty())
@@ -314,7 +314,7 @@ static const ActionsDAG::Node * splitFilterNodeForAllowedInputs(
         {
             auto & node_copy = additional_nodes.emplace_back(*node);
             for (auto & child : node_copy.children)
-                if (child = splitFilterNodeForAllowedInputs(child, allowed_inputs, additional_nodes, strict); !child)
+                if (child = splitFilterNodeForAllowedInputs(child, allowed_inputs, additional_nodes, allow_non_deterministic_functions); !child)
                     return nullptr;
 
             return &node_copy;
@@ -328,7 +328,7 @@ static const ActionsDAG::Node * splitFilterNodeForAllowedInputs(
                     auto index_hint_dag = index_hint->getActions()->clone();
                     ActionsDAG::NodeRawConstPtrs atoms;
                     for (const auto & output : index_hint_dag->getOutputs())
-                        if (const auto * child_copy = splitFilterNodeForAllowedInputs(output, allowed_inputs, additional_nodes, strict))
+                        if (const auto * child_copy = splitFilterNodeForAllowedInputs(output, allowed_inputs, additional_nodes, allow_non_deterministic_functions))
                             atoms.push_back(child_copy);
 
                     if (!atoms.empty())
@@ -362,13 +362,13 @@ static const ActionsDAG::Node * splitFilterNodeForAllowedInputs(
     return node;
 }
 
-ActionsDAGPtr splitFilterDagForAllowedInputs(const ActionsDAG::Node * predicate, const Block * allowed_inputs, bool strict)
+ActionsDAGPtr splitFilterDagForAllowedInputs(const ActionsDAG::Node * predicate, const Block * allowed_inputs, bool allow_non_deterministic_functions)
 {
     if (!predicate)
         return nullptr;
 
     ActionsDAG::Nodes additional_nodes;
-    const auto * res = splitFilterNodeForAllowedInputs(predicate, allowed_inputs, additional_nodes, strict);
+    const auto * res = splitFilterNodeForAllowedInputs(predicate, allowed_inputs, additional_nodes, allow_non_deterministic_functions);
     if (!res)
         return nullptr;
 
@@ -377,7 +377,7 @@ ActionsDAGPtr splitFilterDagForAllowedInputs(const ActionsDAG::Node * predicate,
 
 void filterBlockWithPredicate(const ActionsDAG::Node * predicate, Block & block, ContextPtr context)
 {
-    auto dag = splitFilterDagForAllowedInputs(predicate, &block,  /*strict=*/ false);
+    auto dag = splitFilterDagForAllowedInputs(predicate, &block,  /*allow_non_deterministic_functions=*/ false);
     if (dag)
         filterBlockWithDAG(dag, block, context);
 }
diff --git a/src/Storages/VirtualColumnUtils.h b/src/Storages/VirtualColumnUtils.h
index 5b41e56c0ef..e5cfa47c8f6 100644
--- a/src/Storages/VirtualColumnUtils.h
+++ b/src/Storages/VirtualColumnUtils.h
@@ -32,7 +32,15 @@ void buildSetsForDAG(const ActionsDAG & dag, const ContextPtr & context);
 bool isDeterministicInScopeOfQuery(const ActionsDAG::Node * node);
 
 /// Extract a part of predicate that can be evaluated using only columns from input_names.
-ActionsDAGPtr splitFilterDagForAllowedInputs(const ActionsDAG::Node * predicate, const Block * allowed_inputs, bool strict = false);
+/// When allow_non_deterministic_functions is true then even if the predicate contains non-deterministic
+/// functions, we still allow to extract a part of the predicate, otherwise we return nullptr.
+/// allow_non_deterministic_functions must be false when we are going to use the result to filter parts in
+/// MergeTreeData::totalRowsByPartitionPredicateImp. For example, if the query is
+/// `SELECT count() FROM table  WHERE _partition_id = '0' AND rowNumberInBlock() = 1`
+/// The predicate will be `_partition_id = '0' AND rowNumberInBlock() = 1`, and `rowNumberInBlock()` is
+/// non-deterministic. If we still extract the part `_partition_id = '0'` for filtering parts, then trivial
+/// count optimization will be mistakenly applied to the query.
+ActionsDAGPtr splitFilterDagForAllowedInputs(const ActionsDAG::Node * predicate, const Block * allowed_inputs, bool allow_non_deterministic_functions = true);
 
 /// Extract from the input stream a set of `name` column values
 template <typename T>
diff --git a/tests/queries/0_stateless/03203_count_with_non_deterministic_function.sql b/tests/queries/0_stateless/03203_count_with_non_deterministic_function.sql
index d4ffa4d07ac..bb3269da597 100644
--- a/tests/queries/0_stateless/03203_count_with_non_deterministic_function.sql
+++ b/tests/queries/0_stateless/03203_count_with_non_deterministic_function.sql
@@ -1,4 +1,4 @@
 CREATE TABLE t (p UInt8, x UInt64) Engine = MergeTree PARTITION BY p ORDER BY x;
-INSERT INTO t SELECT 0, number FROM numbers(10);
+INSERT INTO t SELECT 0, number FROM numbers(10) SETTINGS max_block_size = 100;
 SELECT count() FROM t WHERE p = 0 AND rowNumberInAllBlocks() = 1 SETTINGS allow_experimental_analyzer = 0;
 SELECT count() FROM t WHERE p = 0 AND rowNumberInAllBlocks() = 1 SETTINGS allow_experimental_analyzer = 1;

From ff7f5fe80873aad2f0f7b6f4e1e73c0178a69503 Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitlibar@clickhouse.com>
Date: Wed, 1 May 2024 03:09:13 +0200
Subject: [PATCH 0294/1488] Move view targets to separate AST class
 "ASTViewTargets" in order to allow extending it to support more kinds of view
 targets.

---
 src/Backups/BackupUtils.cpp                   |   2 +-
 src/Backups/RestoreCoordinationLocal.cpp      |   6 +-
 src/Backups/RestoreCoordinationLocal.h        |   9 +-
 src/Backups/RestoreCoordinationRemote.cpp     |   9 +-
 src/Databases/DDLDependencyVisitor.cpp        |  19 +-
 src/Databases/DDLRenamingVisitor.cpp          |  17 +-
 src/Databases/DatabaseReplicated.cpp          | 160 +++++-----
 src/Databases/DatabaseReplicated.h            |   1 +
 src/Interpreters/InterpreterCreateQuery.cpp   | 126 ++++++--
 .../InterpreterShowCreateQuery.cpp            |   3 +-
 src/Parsers/ASTCreateQuery.cpp                |  95 +++---
 src/Parsers/ASTCreateQuery.h                  |  40 ++-
 src/Parsers/ASTViewTargets.cpp                | 300 ++++++++++++++++++
 src/Parsers/ASTViewTargets.h                  | 102 ++++++
 src/Parsers/CreateQueryUUIDs.cpp              | 168 ++++++++++
 src/Parsers/CreateQueryUUIDs.h                |  40 +++
 src/Parsers/ParserCreateQuery.cpp             |  92 ++++--
 src/Parsers/ParserViewTargets.cpp             |  88 +++++
 src/Parsers/ParserViewTargets.h               |  24 ++
 src/Storages/StorageMaterializedView.cpp      |  40 ++-
 src/Storages/System/StorageSystemTables.cpp   |   3 +-
 src/Storages/WindowView/StorageWindowView.cpp |  15 +-
 22 files changed, 1116 insertions(+), 243 deletions(-)
 create mode 100644 src/Parsers/ASTViewTargets.cpp
 create mode 100644 src/Parsers/ASTViewTargets.h
 create mode 100644 src/Parsers/CreateQueryUUIDs.cpp
 create mode 100644 src/Parsers/CreateQueryUUIDs.h
 create mode 100644 src/Parsers/ParserViewTargets.cpp
 create mode 100644 src/Parsers/ParserViewTargets.h

diff --git a/src/Backups/BackupUtils.cpp b/src/Backups/BackupUtils.cpp
index fa8ed5855dd..cd3f963b15d 100644
--- a/src/Backups/BackupUtils.cpp
+++ b/src/Backups/BackupUtils.cpp
@@ -105,7 +105,7 @@ bool compareRestoredTableDef(const IAST & restored_table_create_query, const IAS
         auto new_query = query.clone();
         adjustCreateQueryForBackup(new_query, global_context);
         ASTCreateQuery & create = typeid_cast<ASTCreateQuery &>(*new_query);
-        create.setUUID({});
+        create.resetUUIDs();
         create.if_not_exists = false;
         return new_query;
     };
diff --git a/src/Backups/RestoreCoordinationLocal.cpp b/src/Backups/RestoreCoordinationLocal.cpp
index f51d6c0c1d8..9fe22f874b4 100644
--- a/src/Backups/RestoreCoordinationLocal.cpp
+++ b/src/Backups/RestoreCoordinationLocal.cpp
@@ -1,4 +1,5 @@
 #include <Backups/RestoreCoordinationLocal.h>
+#include <Parsers/ASTCreateQuery.h>
 #include <Parsers/formatAST.h>
 #include <Common/logger_useful.h>
 
@@ -67,7 +68,7 @@ void RestoreCoordinationLocal::generateUUIDForTable(ASTCreateQuery & create_quer
         auto it = create_query_uuids.find(query_str);
         if (it != create_query_uuids.end())
         {
-            create_query.setUUID(it->second);
+            it->second.copyToQuery(create_query);
             return true;
         }
         return false;
@@ -79,7 +80,8 @@ void RestoreCoordinationLocal::generateUUIDForTable(ASTCreateQuery & create_quer
             return;
     }
 
-    auto new_uuids = create_query.generateRandomUUID(/* always_generate_new_uuid= */ true);
+    CreateQueryUUIDs new_uuids{create_query, /* generate_random= */ true, /* force_random= */ true};
+    new_uuids.copyToQuery(create_query);
 
     {
         std::lock_guard lock{mutex};
diff --git a/src/Backups/RestoreCoordinationLocal.h b/src/Backups/RestoreCoordinationLocal.h
index 5e51b719d63..35f93574b68 100644
--- a/src/Backups/RestoreCoordinationLocal.h
+++ b/src/Backups/RestoreCoordinationLocal.h
@@ -1,16 +1,17 @@
 #pragma once
 
 #include <Backups/IRestoreCoordination.h>
-#include <Parsers/ASTCreateQuery.h>
+#include <Parsers/CreateQueryUUIDs.h>
+#include <Common/Logger.h>
 #include <mutex>
 #include <set>
 #include <unordered_set>
 
-namespace Poco { class Logger; }
-
 
 namespace DB
 {
+class ASTCreateQuery;
+
 
 /// Implementation of the IRestoreCoordination interface performing coordination in memory.
 class RestoreCoordinationLocal : public IRestoreCoordination
@@ -55,7 +56,7 @@ private:
 
     std::set<std::pair<String /* database_zk_path */, String /* table_name */>> acquired_tables_in_replicated_databases;
     std::unordered_set<String /* table_zk_path */> acquired_data_in_replicated_tables;
-    std::unordered_map<String, ASTCreateQuery::UUIDs> create_query_uuids;
+    std::unordered_map<String, CreateQueryUUIDs> create_query_uuids;
     std::unordered_set<String /* root_zk_path */> acquired_data_in_keeper_map_tables;
 
     mutable std::mutex mutex;
diff --git a/src/Backups/RestoreCoordinationRemote.cpp b/src/Backups/RestoreCoordinationRemote.cpp
index 84106737fc9..44214d00be5 100644
--- a/src/Backups/RestoreCoordinationRemote.cpp
+++ b/src/Backups/RestoreCoordinationRemote.cpp
@@ -3,6 +3,7 @@
 #include <Backups/RestoreCoordinationRemote.h>
 #include <Backups/BackupCoordinationStageSync.h>
 #include <Parsers/ASTCreateQuery.h>
+#include <Parsers/CreateQueryUUIDs.h>
 #include <Parsers/formatAST.h>
 #include <Functions/UserDefined/UserDefinedSQLObjectType.h>
 #include <Common/ZooKeeper/KeeperException.h>
@@ -269,7 +270,8 @@ bool RestoreCoordinationRemote::acquireInsertingDataForKeeperMap(const String &
 void RestoreCoordinationRemote::generateUUIDForTable(ASTCreateQuery & create_query)
 {
     String query_str = serializeAST(create_query);
-    String new_uuids_str = create_query.generateRandomUUID(/* always_generate_new_uuid= */ true).toString();
+    CreateQueryUUIDs new_uuids{create_query, /* generate_random= */ true, /* force_random= */ true};
+    String new_uuids_str = new_uuids.toString();
 
     auto holder = with_retries.createRetriesControlHolder("generateUUIDForTable");
     holder.retries_ctl.retryLoop(
@@ -281,11 +283,14 @@ void RestoreCoordinationRemote::generateUUIDForTable(ASTCreateQuery & create_que
             Coordination::Error res = zk->tryCreate(path, new_uuids_str, zkutil::CreateMode::Persistent);
 
             if (res == Coordination::Error::ZOK)
+            {
+                new_uuids.copyToQuery(create_query);
                 return;
+            }
 
             if (res == Coordination::Error::ZNODEEXISTS)
             {
-                create_query.setUUID(ASTCreateQuery::UUIDs::fromString(zk->get(path)));
+                CreateQueryUUIDs::fromString(zk->get(path)).copyToQuery(create_query);
                 return;
             }
 
diff --git a/src/Databases/DDLDependencyVisitor.cpp b/src/Databases/DDLDependencyVisitor.cpp
index d81dc7a76d8..d149b49d465 100644
--- a/src/Databases/DDLDependencyVisitor.cpp
+++ b/src/Databases/DDLDependencyVisitor.cpp
@@ -80,13 +80,20 @@ namespace
         /// CREATE TABLE or CREATE DICTIONARY or CREATE VIEW or CREATE TEMPORARY TABLE or CREATE DATABASE query.
         void visitCreateQuery(const ASTCreateQuery & create)
         {
-            QualifiedTableName to_table{create.to_table_id.database_name, create.to_table_id.table_name};
-            if (!to_table.table.empty())
+            if (create.targets)
             {
-                /// TO target_table (for materialized views)
-                if (to_table.database.empty())
-                    to_table.database = current_database;
-                dependencies.emplace(to_table);
+                for (const auto & target : create.targets->targets)
+                {
+                    const auto & table_id = target.table_id;
+                    if (!table_id.table_name.empty())
+                    {
+                        /// TO target_table (for materialized views)
+                        QualifiedTableName target_name{table_id.database_name, table_id.table_name};
+                        if (target_name.database.empty())
+                            target_name.database = current_database;
+                        dependencies.emplace(target_name);
+                    }
+                }
             }
 
             QualifiedTableName as_table{create.as_database, create.as_table};
diff --git a/src/Databases/DDLRenamingVisitor.cpp b/src/Databases/DDLRenamingVisitor.cpp
index 6cd414635a0..38e100e2470 100644
--- a/src/Databases/DDLRenamingVisitor.cpp
+++ b/src/Databases/DDLRenamingVisitor.cpp
@@ -86,12 +86,19 @@ namespace
             create.as_table = as_table_new.table;
         }
 
-        QualifiedTableName to_table{create.to_table_id.database_name, create.to_table_id.table_name};
-        if (!to_table.table.empty() && !to_table.database.empty())
+        if (create.targets)
         {
-            auto to_table_new = data.renaming_map.getNewTableName(to_table);
-            if (to_table_new != to_table)
-                create.to_table_id = StorageID{to_table_new.database, to_table_new.table};
+            for (auto & target : create.targets->targets)
+            {
+                auto & table_id = target.table_id;
+                if (!table_id.database_name.empty() && !table_id.table_name.empty())
+                {
+                    QualifiedTableName target_name{table_id.database_name, table_id.table_name};
+                    auto new_target_name = data.renaming_map.getNewTableName(target_name);
+                    if (new_target_name != target_name)
+                        table_id = StorageID{new_target_name.database, new_target_name.table};
+                }
+            }
         }
     }
 
diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp
index 7ce2859e962..25d1ad90a3c 100644
--- a/src/Databases/DatabaseReplicated.cpp
+++ b/src/Databases/DatabaseReplicated.cpp
@@ -729,81 +729,14 @@ void DatabaseReplicated::checkQueryValid(const ASTPtr & query, ContextPtr query_
 
         if (auto * create = query->as<ASTCreateQuery>())
         {
-            bool replicated_table = create->storage && create->storage->engine &&
-                (startsWith(create->storage->engine->name, "Replicated") || startsWith(create->storage->engine->name, "Shared"));
-            if (!replicated_table || !create->storage->engine->arguments)
-                return;
+            if (create->storage)
+                checkTableEngine(*create, *create->storage, query_context);
 
-            ASTs & args_ref = create->storage->engine->arguments->children;
-            ASTs args = args_ref;
-            if (args.size() < 2)
-                return;
-
-            /// It can be a constant expression. Try to evaluate it, ignore exception if we cannot.
-            bool has_expression_argument = args_ref[0]->as<ASTFunction>() || args_ref[1]->as<ASTFunction>();
-            if (has_expression_argument)
+            if (create->targets)
             {
-                try
-                {
-                    args[0] = evaluateConstantExpressionAsLiteral(args_ref[0]->clone(), query_context);
-                    args[1] = evaluateConstantExpressionAsLiteral(args_ref[1]->clone(), query_context);
-                }
-                catch (...) // NOLINT(bugprone-empty-catch)
-                {
-                }
+                for (auto inner_table_engine : create->targets->getInnerEngines())
+                    checkTableEngine(*create, *inner_table_engine, query_context);
             }
-
-            ASTLiteral * arg1 = args[0]->as<ASTLiteral>();
-            ASTLiteral * arg2 = args[1]->as<ASTLiteral>();
-            if (!arg1 || !arg2 || arg1->value.getType() != Field::Types::String || arg2->value.getType() != Field::Types::String)
-                return;
-
-            String maybe_path = arg1->value.get<String>();
-            String maybe_replica = arg2->value.get<String>();
-
-            /// Looks like it's ReplicatedMergeTree with explicit zookeeper_path and replica_name arguments.
-            /// Let's ensure that some macros are used.
-            /// NOTE: we cannot check here that substituted values will be actually different on shards and replicas.
-
-            Macros::MacroExpansionInfo info;
-            info.table_id = {getDatabaseName(), create->getTable(), create->uuid};
-            info.shard = getShardName();
-            info.replica = getReplicaName();
-            query_context->getMacros()->expand(maybe_path, info);
-            bool maybe_shard_macros = info.expanded_other;
-            info.expanded_other = false;
-            query_context->getMacros()->expand(maybe_replica, info);
-            bool maybe_replica_macros = info.expanded_other;
-            bool enable_functional_tests_helper = getContext()->getConfigRef().has("_functional_tests_helper_database_replicated_replace_args_macros");
-
-            if (!enable_functional_tests_helper)
-            {
-                if (query_context->getSettingsRef().database_replicated_allow_replicated_engine_arguments)
-                    LOG_WARNING(log, "It's not recommended to explicitly specify zookeeper_path and replica_name in ReplicatedMergeTree arguments");
-                else
-                    throw Exception(ErrorCodes::INCORRECT_QUERY,
-                                    "It's not allowed to specify explicit zookeeper_path and replica_name "
-                                    "for ReplicatedMergeTree arguments in Replicated database. If you really want to "
-                                    "specify them explicitly, enable setting "
-                                    "database_replicated_allow_replicated_engine_arguments.");
-            }
-
-            if (maybe_shard_macros && maybe_replica_macros)
-                return;
-
-            if (enable_functional_tests_helper && !has_expression_argument)
-            {
-                if (maybe_path.empty() || maybe_path.back() != '/')
-                    maybe_path += '/';
-                args_ref[0]->as<ASTLiteral>()->value = maybe_path + "auto_{shard}";
-                args_ref[1]->as<ASTLiteral>()->value = maybe_replica + "auto_{replica}";
-                return;
-            }
-
-            throw Exception(ErrorCodes::INCORRECT_QUERY,
-                            "Explicit zookeeper_path and replica_name are specified in ReplicatedMergeTree arguments. "
-                            "If you really want to specify it explicitly, then you should use some macros "
-                            "to distinguish different shards and replicas");
         }
     }
 
@@ -827,6 +760,85 @@ void DatabaseReplicated::checkQueryValid(const ASTPtr & query, ContextPtr query_
     }
 }
 
+void DatabaseReplicated::checkTableEngine(const ASTCreateQuery & query, ASTStorage & storage, ContextPtr query_context) const
+{
+    bool replicated_table = storage.engine &&
+        (startsWith(storage.engine->name, "Replicated") || startsWith(storage.engine->name, "Shared"));
+    if (!replicated_table || !storage.engine->arguments)
+        return;
+
+    ASTs & args_ref = storage.engine->arguments->children;
+    ASTs args = args_ref;
+    if (args.size() < 2)
+        return;
+
+    /// It can be a constant expression. Try to evaluate it, ignore exception if we cannot.
+    bool has_expression_argument = args_ref[0]->as<ASTFunction>() || args_ref[1]->as<ASTFunction>();
+    if (has_expression_argument)
+    {
+        try
+        {
+            args[0] = evaluateConstantExpressionAsLiteral(args_ref[0]->clone(), query_context);
+            args[1] = evaluateConstantExpressionAsLiteral(args_ref[1]->clone(), query_context);
+        }
+        catch (...) // NOLINT(bugprone-empty-catch)
+        {
+        }
+    }
+
+    ASTLiteral * arg1 = args[0]->as<ASTLiteral>();
+    ASTLiteral * arg2 = args[1]->as<ASTLiteral>();
+    if (!arg1 || !arg2 || arg1->value.getType() != Field::Types::String || arg2->value.getType() != Field::Types::String)
+        return;
+
+    String maybe_path = arg1->value.get<String>();
+    String maybe_replica = arg2->value.get<String>();
+
+    /// Looks like it's ReplicatedMergeTree with explicit zookeeper_path and replica_name arguments.
+    /// Let's ensure that some macros are used.
+    /// NOTE: we cannot check here that substituted values will be actually different on shards and replicas.
+
+    Macros::MacroExpansionInfo info;
+    info.table_id = {getDatabaseName(), query.getTable(), query.uuid};
+    info.shard = getShardName();
+    info.replica = getReplicaName();
+    query_context->getMacros()->expand(maybe_path, info);
+    bool maybe_shard_macros = info.expanded_other;
+    info.expanded_other = false;
+    query_context->getMacros()->expand(maybe_replica, info);
+    bool maybe_replica_macros = info.expanded_other;
+    bool enable_functional_tests_helper = getContext()->getConfigRef().has("_functional_tests_helper_database_replicated_replace_args_macros");
+
+    if (!enable_functional_tests_helper)
+    {
+        if (query_context->getSettingsRef().database_replicated_allow_replicated_engine_arguments)
+            LOG_WARNING(log, "It's not recommended to explicitly specify zookeeper_path and replica_name in ReplicatedMergeTree arguments");
+        else
+            throw Exception(ErrorCodes::INCORRECT_QUERY,
+                            "It's not allowed to specify explicit zookeeper_path and replica_name "
+                            "for ReplicatedMergeTree arguments in Replicated database. If you really want to "
+                            "specify them explicitly, enable setting "
+                            "database_replicated_allow_replicated_engine_arguments.");
+    }
+
+    if (maybe_shard_macros && maybe_replica_macros)
+        return;
+
+    if (enable_functional_tests_helper && !has_expression_argument)
+    {
+        if (maybe_path.empty() || maybe_path.back() != '/')
+            maybe_path += '/';
+        args_ref[0]->as<ASTLiteral>()->value = maybe_path + "auto_{shard}";
+        args_ref[1]->as<ASTLiteral>()->value = maybe_replica + "auto_{replica}";
+        return;
+    }
+
+    throw Exception(ErrorCodes::INCORRECT_QUERY,
+                    "Explicit zookeeper_path and replica_name are specified in ReplicatedMergeTree arguments. "
+                    "If you really want to specify it explicitly, then you should use some macros "
+                    "to distinguish different shards and replicas");
+}
+
 BlockIO DatabaseReplicated::tryEnqueueReplicatedDDL(const ASTPtr & query, ContextPtr query_context, QueryFlags flags)
 {
     waitDatabaseStarted();
@@ -1312,11 +1324,9 @@ ASTPtr DatabaseReplicated::parseQueryFromMetadataInZooKeeper(const String & node
     if (create.uuid == UUIDHelpers::Nil || create.getTable() != TABLE_WITH_UUID_NAME_PLACEHOLDER || create.database)
         throw Exception(ErrorCodes::LOGICAL_ERROR, "Got unexpected query from {}: {}", node_name, query);
 
-    bool is_materialized_view_with_inner_table = create.is_materialized_view && create.to_table_id.empty();
-
     create.setDatabase(getDatabaseName());
     create.setTable(unescapeForFileName(node_name));
-    create.attach = is_materialized_view_with_inner_table;
+    create.attach = create.is_materialized_view_with_inner_table();
 
     return ast;
 }
diff --git a/src/Databases/DatabaseReplicated.h b/src/Databases/DatabaseReplicated.h
index eab5b2ff931..8c3fa7c87f6 100644
--- a/src/Databases/DatabaseReplicated.h
+++ b/src/Databases/DatabaseReplicated.h
@@ -107,6 +107,7 @@ private:
     void fillClusterAuthInfo(String collection_name, const Poco::Util::AbstractConfiguration & config);
 
     void checkQueryValid(const ASTPtr & query, ContextPtr query_context) const;
+    void checkTableEngine(const ASTCreateQuery & query, ASTStorage & storage, ContextPtr query_context) const;
 
     void recoverLostReplica(const ZooKeeperPtr & current_zookeeper, UInt32 our_log_ptr, UInt32 & max_log_ptr);
 
diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp
index 84d7f0a587c..45e2881ae5c 100644
--- a/src/Interpreters/InterpreterCreateQuery.cpp
+++ b/src/Interpreters/InterpreterCreateQuery.cpp
@@ -950,7 +950,7 @@ namespace
             throw Exception(ErrorCodes::INCORRECT_QUERY, "Temporary tables cannot be created with Replicated, Shared or KeeperMap table engines");
     }
 
-    void setDefaultTableEngine(ASTStorage &storage, DefaultTableEngine engine)
+    void setDefaultTableEngine(ASTStorage & storage, DefaultTableEngine engine)
     {
         if (engine == DefaultTableEngine::None)
             throw Exception(ErrorCodes::ENGINE_REQUIRED, "Table engine is not specified in CREATE query");
@@ -970,9 +970,6 @@ void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const
     if (create.is_dictionary || create.is_ordinary_view || create.is_live_view || create.is_window_view)
         return;
 
-    if (create.is_materialized_view && create.to_table_id)
-        return;
-
     if (create.temporary)
     {
         /// Some part of storage definition is specified, but ENGINE is not: just set the one from default_temporary_table_engine setting.
@@ -987,22 +984,44 @@ void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const
         }
 
         if (!create.storage->engine)
-        {
             setDefaultTableEngine(*create.storage, getContext()->getSettingsRef().default_temporary_table_engine.value);
-        }
 
         checkTemporaryTableEngineName(create.storage->engine->name);
         return;
     }
 
+    if (create.is_materialized_view)
+    {
+        /// A materialized view with an external target doesn't need a table engine.
+        if (create.is_materialized_view_with_external_target())
+            return;
+
+        if (auto to_engine = create.getTargetInnerEngine(ViewTarget::To))
+        {
+            /// This materialized view already has a storage definition.
+            if (!to_engine->engine)
+            {
+                /// Some part of storage definition (such as PARTITION BY) is specified, but ENGINE is not: just set default one.
+                setDefaultTableEngine(*to_engine, getContext()->getSettingsRef().default_table_engine.value);
+            }
+            return;
+        }
+    }
+
     if (create.storage)
     {
-        /// Some part of storage definition (such as PARTITION BY) is specified, but ENGINE is not: just set default one.
+        /// This table already has a storage definition.
         if (!create.storage->engine)
+        {
+            /// Some part of storage definition (such as PARTITION BY) is specified, but ENGINE is not: just set default one.
             setDefaultTableEngine(*create.storage, getContext()->getSettingsRef().default_table_engine.value);
+        }
         return;
     }
 
+    /// We'll try to extract a storage definition from clause `AS`:
+    ///     CREATE TABLE table_name AS other_table_name
+    std::shared_ptr<ASTStorage> storage_def;
     if (!create.as_table.empty())
     {
         /// NOTE Getting the structure from the table specified in the AS is done not atomically with the creation of the table.
@@ -1018,12 +1037,14 @@ void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const
         if (as_create.is_ordinary_view)
             throw Exception(ErrorCodes::INCORRECT_QUERY, "Cannot CREATE a table AS {}, it is a View", qualified_name);
 
-        if (as_create.is_materialized_view && as_create.to_table_id)
+        if (as_create.is_materialized_view_with_external_target())
+        {
             throw Exception(
                 ErrorCodes::INCORRECT_QUERY,
-                "Cannot CREATE a table AS {}, it is a Materialized View without storage. Use \"AS `{}`\" instead",
+                "Cannot CREATE a table AS {}, it is a Materialized View without storage. Use \"AS {}\" instead",
                 qualified_name,
-                as_create.to_table_id.getQualifiedName());
+                as_create.getTargetTableID(ViewTarget::To).getFullTableName());
+        }
 
         if (as_create.is_live_view)
             throw Exception(ErrorCodes::INCORRECT_QUERY, "Cannot CREATE a table AS {}, it is a Live View", qualified_name);
@@ -1034,18 +1055,37 @@ void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const
         if (as_create.is_dictionary)
             throw Exception(ErrorCodes::INCORRECT_QUERY, "Cannot CREATE a table AS {}, it is a Dictionary", qualified_name);
 
-        if (as_create.storage)
-            create.set(create.storage, as_create.storage->ptr());
+        if (as_create.is_materialized_view)
+        {
+            storage_def = as_create.getTargetInnerEngine(ViewTarget::To);
+        }
         else if (as_create.as_table_function)
+        {
             create.set(create.as_table_function, as_create.as_table_function->ptr());
+            return;
+        }
+        else if (as_create.storage)
+        {
+            storage_def = typeid_cast<std::shared_ptr<ASTStorage>>(as_create.storage->ptr());
+        }
         else
+        {
             throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot set engine, it's a bug.");
-
-        return;
+        }
     }
 
-    create.set(create.storage, std::make_shared<ASTStorage>());
-    setDefaultTableEngine(*create.storage, getContext()->getSettingsRef().default_table_engine.value);
+    if (!storage_def)
+    {
+        /// Set ENGINE by default.
+        storage_def = std::make_shared<ASTStorage>();
+        setDefaultTableEngine(*storage_def, getContext()->getSettingsRef().default_table_engine.value);
+    }
+
+    /// Use the found table engine to modify the create query.
+    if (create.is_materialized_view)
+        create.setTargetInnerEngine(ViewTarget::To, storage_def);
+    else
+        create.set(create.storage, storage_def);
 }
 
 void InterpreterCreateQuery::assertOrSetUUID(ASTCreateQuery & create, const DatabasePtr & database) const
@@ -1087,11 +1127,11 @@ void InterpreterCreateQuery::assertOrSetUUID(ASTCreateQuery & create, const Data
                             kind_upper, create.table);
         }
 
-        create.generateRandomUUID();
+        create.generateRandomUUIDs();
     }
     else
     {
-        bool has_uuid = create.uuid != UUIDHelpers::Nil || create.to_inner_uuid != UUIDHelpers::Nil;
+        bool has_uuid = (create.uuid != UUIDHelpers::Nil) || (create.targets && create.targets->hasInnerUUIDs());
         if (has_uuid && !is_on_cluster && !internal)
         {
             /// We don't show the following error message either
@@ -1106,8 +1146,7 @@ void InterpreterCreateQuery::assertOrSetUUID(ASTCreateQuery & create, const Data
         /// The database doesn't support UUID so we'll ignore it. The UUID could be set here because of either
         /// a) the initiator of `ON CLUSTER` query generated it to ensure the same UUIDs are used on different hosts; or
         /// b) `RESTORE from backup` query generated it to ensure the same UUIDs are used on different hosts.
-        create.uuid = UUIDHelpers::Nil;
-        create.to_inner_uuid = UUIDHelpers::Nil;
+        create.resetUUIDs();
     }
 }
 
@@ -1131,6 +1170,14 @@ void checkTableCanBeAddedWithNoCyclicDependencies(const ASTCreateQuery & create,
     DatabaseCatalog::instance().checkTableCanBeAddedWithNoCyclicDependencies(qualified_name, ref_dependencies, loading_dependencies);
 }
 
+bool isReplicated(const ASTStorage & storage)
+{
+    if (!storage.engine)
+        return false;
+    const auto & storage_name = storage.engine->name;
+    return storage_name.starts_with("Replicated") || storage_name.starts_with("Shared");
+}
+
 }
 
 BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create)
@@ -1247,8 +1294,9 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create)
 
     if (!create.temporary && !create.database)
         create.setDatabase(current_database);
-    if (create.to_table_id && create.to_table_id.database_name.empty())
-        create.to_table_id.database_name = current_database;
+
+    if (create.targets)
+        create.targets->setCurrentDatabase(current_database);
 
     if (create.select && create.isView())
     {
@@ -1282,12 +1330,9 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create)
     TableProperties properties = getTablePropertiesAndNormalizeCreateQuery(create, mode);
 
     /// Check type compatible for materialized dest table and select columns
-    if (create.select && create.is_materialized_view && create.to_table_id && mode <= LoadingStrictnessLevel::CREATE)
+    if (create.is_materialized_view_with_external_target() && create.select && mode <= LoadingStrictnessLevel::CREATE)
     {
-        if (StoragePtr to_table = DatabaseCatalog::instance().tryGetTable(
-            {create.to_table_id.database_name, create.to_table_id.table_name, create.to_table_id.uuid},
-            getContext()
-        ))
+        if (StoragePtr to_table = DatabaseCatalog::instance().tryGetTable(create.getTargetTableID(ViewTarget::To), getContext()))
         {
             Block input_block;
 
@@ -1333,11 +1378,17 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create)
     if (!allow_heavy_create && database && database->getEngineName() == "Replicated" && (create.select || create.is_populate))
     {
         bool is_storage_replicated = false;
-        if (create.storage && create.storage->engine)
+
+        if (create.storage && isReplicated(*create.storage))
+            is_storage_replicated = true;
+
+        if (create.targets)
         {
-            const auto & storage_name = create.storage->engine->name;
-            if (storage_name.starts_with("Replicated") || storage_name.starts_with("Shared"))
-                is_storage_replicated = true;
+            for (auto inner_table_engine : create.targets->getInnerEngines())
+            {
+                if (isReplicated(*inner_table_engine))
+                    is_storage_replicated = true;
+            }
         }
 
         const bool allow_create_select_for_replicated = (create.isView() && !create.is_populate) || create.is_create_empty || !is_storage_replicated;
@@ -1791,7 +1842,7 @@ void InterpreterCreateQuery::prepareOnClusterQuery(ASTCreateQuery & create, Cont
 
     /// For CREATE query generate UUID on initiator, so it will be the same on all hosts.
     /// It will be ignored if database does not support UUIDs.
-    create.generateRandomUUID();
+    create.generateRandomUUIDs();
 
     /// For cross-replication cluster we cannot use UUID in replica path.
     String cluster_name_expanded = local_context->getMacros()->expand(cluster_name);
@@ -1913,8 +1964,15 @@ AccessRightsElements InterpreterCreateQuery::getRequiredAccess() const
         }
     }
 
-    if (create.to_table_id)
-        required_access.emplace_back(AccessType::SELECT | AccessType::INSERT, create.to_table_id.database_name, create.to_table_id.table_name);
+    if (create.targets)
+    {
+        for (const auto & target : create.targets->targets)
+        {
+            const auto & target_id = target.table_id;
+            if (target_id)
+                required_access.emplace_back(AccessType::SELECT | AccessType::INSERT, target_id.database_name, target_id.table_name);
+        }
+    }
 
     if (create.storage && create.storage->engine)
         required_access.emplace_back(AccessType::TABLE_ENGINE, create.storage->engine->name);
diff --git a/src/Interpreters/InterpreterShowCreateQuery.cpp b/src/Interpreters/InterpreterShowCreateQuery.cpp
index 0fca7b64d5a..16add79d226 100644
--- a/src/Interpreters/InterpreterShowCreateQuery.cpp
+++ b/src/Interpreters/InterpreterShowCreateQuery.cpp
@@ -94,7 +94,8 @@ QueryPipeline InterpreterShowCreateQuery::executeImpl()
     {
         auto & create = create_query->as<ASTCreateQuery &>();
         create.uuid = UUIDHelpers::Nil;
-        create.to_inner_uuid = UUIDHelpers::Nil;
+        if (create.targets)
+            create.targets->resetInnerUUIDs();
     }
 
     MutableColumnPtr column = ColumnString::create();
diff --git a/src/Parsers/ASTCreateQuery.cpp b/src/Parsers/ASTCreateQuery.cpp
index d56a2724914..770a63c6e75 100644
--- a/src/Parsers/ASTCreateQuery.cpp
+++ b/src/Parsers/ASTCreateQuery.cpp
@@ -2,6 +2,8 @@
 #include <Parsers/ASTExpressionList.h>
 #include <Parsers/ASTFunction.h>
 #include <Parsers/ASTSelectWithUnionQuery.h>
+#include <Parsers/CommonParsers.h>
+#include <Parsers/CreateQueryUUIDs.h>
 #include <Common/quoteString.h>
 #include <Interpreters/StorageID.h>
 #include <IO/Operators.h>
@@ -240,12 +242,12 @@ ASTPtr ASTCreateQuery::clone() const
         res->set(res->columns_list, columns_list->clone());
     if (storage)
         res->set(res->storage, storage->clone());
-    if (inner_storage)
-        res->set(res->inner_storage, inner_storage->clone());
     if (select)
         res->set(res->select, select->clone());
     if (table_overrides)
         res->set(res->table_overrides, table_overrides->clone());
+    if (targets)
+        res->set(res->targets, targets->clone());
 
     if (dictionary)
     {
@@ -388,20 +390,18 @@ void ASTCreateQuery::formatQueryImpl(const FormatSettings & settings, FormatStat
         refresh_strategy->formatImpl(settings, state, frame);
     }
 
-    if (to_table_id)
+    if (auto to_table_id = getTargetTableID(ViewTarget::To))
     {
-        assert((is_materialized_view || is_window_view) && to_inner_uuid == UUIDHelpers::Nil);
-        settings.ostr
-            << (settings.hilite ? hilite_keyword : "") << " TO " << (settings.hilite ? hilite_none : "")
-            << (!to_table_id.database_name.empty() ? backQuoteIfNeed(to_table_id.database_name) + "." : "")
-            << backQuoteIfNeed(to_table_id.table_name);
+        settings.ostr <<  " " << (settings.hilite ? hilite_keyword : "") << toStringView(Keyword::TO)
+                      << (settings.hilite ? hilite_none : "") << " "
+                      << (!to_table_id.database_name.empty() ? backQuoteIfNeed(to_table_id.database_name) + "." : "")
+                      << backQuoteIfNeed(to_table_id.table_name);
     }
 
-    if (to_inner_uuid != UUIDHelpers::Nil)
+    if (auto to_inner_uuid = getTargetInnerUUID(ViewTarget::To); to_inner_uuid != UUIDHelpers::Nil)
     {
-        assert(is_materialized_view && !to_table_id);
-        settings.ostr << (settings.hilite ? hilite_keyword : "") << " TO INNER UUID " << (settings.hilite ? hilite_none : "")
-                      << quoteString(toString(to_inner_uuid));
+        settings.ostr << " " << (settings.hilite ? hilite_keyword : "") << toStringView(Keyword::TO_INNER_UUID)
+                      << (settings.hilite ? hilite_none : "") << " " << quoteString(toString(to_inner_uuid));
     }
 
     bool should_add_empty = is_create_empty;
@@ -461,14 +461,17 @@ void ASTCreateQuery::formatQueryImpl(const FormatSettings & settings, FormatStat
 
     frame.expression_list_always_start_on_new_line = false;
 
-    if (inner_storage)
+    if (storage)
+        storage->formatImpl(settings, state, frame);
+
+    if (auto inner_storage = getTargetInnerEngine(ViewTarget::Inner))
     {
-        settings.ostr << (settings.hilite ? hilite_keyword : "") << " INNER" << (settings.hilite ? hilite_none : "");
+        settings.ostr << " " << (settings.hilite ? hilite_keyword : "") << toStringView(Keyword::INNER) << (settings.hilite ? hilite_none : "");
         inner_storage->formatImpl(settings, state, frame);
     }
 
-    if (storage)
-        storage->formatImpl(settings, state, frame);
+    if (auto to_storage = getTargetInnerEngine(ViewTarget::To))
+        to_storage->formatImpl(settings, state, frame);
 
     if (dictionary)
         dictionary->formatImpl(settings, state, frame);
@@ -528,48 +531,50 @@ bool ASTCreateQuery::isParameterizedView() const
 }
 
 
-ASTCreateQuery::UUIDs::UUIDs(const ASTCreateQuery & query)
-    : uuid(query.uuid)
-    , to_inner_uuid(query.to_inner_uuid)
+void ASTCreateQuery::generateRandomUUIDs()
 {
+    CreateQueryUUIDs{*this, /* generate_random= */ true}.copyToQuery(*this);
 }
 
-String ASTCreateQuery::UUIDs::toString() const
+void ASTCreateQuery::resetUUIDs()
 {
-    WriteBufferFromOwnString out;
-    out << "{" << uuid << "," << to_inner_uuid << "}";
-    return out.str();
+    CreateQueryUUIDs{}.copyToQuery(*this);
 }
 
-ASTCreateQuery::UUIDs ASTCreateQuery::UUIDs::fromString(const String & str)
+
+StorageID ASTCreateQuery::getTargetTableID(ViewTarget::Kind target_kind) const
 {
-    ReadBufferFromString in{str};
-    ASTCreateQuery::UUIDs res;
-    in >> "{" >> res.uuid >> "," >> res.to_inner_uuid >> "}";
-    return res;
+    if (targets)
+        return targets->getTableID(target_kind);
+    return StorageID::createEmpty();
 }
 
-ASTCreateQuery::UUIDs ASTCreateQuery::generateRandomUUID(bool always_generate_new_uuid)
+bool ASTCreateQuery::hasTargetTableID(ViewTarget::Kind target_kind) const
 {
-    if (always_generate_new_uuid)
-        setUUID({});
-
-    if (uuid == UUIDHelpers::Nil)
-        uuid = UUIDHelpers::generateV4();
-
-    /// If destination table (to_table_id) is not specified for materialized view,
-    /// then MV will create inner table. We should generate UUID of inner table here.
-    bool need_uuid_for_inner_table = !attach && is_materialized_view && !to_table_id;
-    if (need_uuid_for_inner_table && (to_inner_uuid == UUIDHelpers::Nil))
-        to_inner_uuid = UUIDHelpers::generateV4();
-
-    return UUIDs{*this};
+    if (targets)
+        return targets->hasTableID(target_kind);
+    return false;
 }
 
-void ASTCreateQuery::setUUID(const UUIDs & uuids)
+UUID ASTCreateQuery::getTargetInnerUUID(ViewTarget::Kind target_kind) const
 {
-    uuid = uuids.uuid;
-    to_inner_uuid = uuids.to_inner_uuid;
+    if (targets)
+        return targets->getInnerUUID(target_kind);
+    return UUIDHelpers::Nil;
+}
+
+std::shared_ptr<ASTStorage> ASTCreateQuery::getTargetInnerEngine(ViewTarget::Kind target_kind) const
+{
+    if (targets)
+        return targets->getInnerEngine(target_kind);
+    return nullptr;
+}
+
+void ASTCreateQuery::setTargetInnerEngine(ViewTarget::Kind target_kind, ASTPtr storage_def)
+{
+    if (!targets)
+        set(targets, std::make_shared<ASTViewTargets>());
+    targets->setInnerEngine(target_kind, storage_def);
 }
 
 }
diff --git a/src/Parsers/ASTCreateQuery.h b/src/Parsers/ASTCreateQuery.h
index 6fbf045915b..f751a09169c 100644
--- a/src/Parsers/ASTCreateQuery.h
+++ b/src/Parsers/ASTCreateQuery.h
@@ -5,6 +5,7 @@
 #include <Parsers/ASTDictionary.h>
 #include <Parsers/ASTDictionaryAttributeDeclaration.h>
 #include <Parsers/ASTTableOverrides.h>
+#include <Parsers/ASTViewTargets.h>
 #include <Parsers/ASTSQLSecurity.h>
 #include <Parsers/ASTRefreshStrategy.h>
 #include <Interpreters/StorageID.h>
@@ -15,6 +16,7 @@ namespace DB
 class ASTFunction;
 class ASTSetQuery;
 class ASTSelectWithUnionQuery;
+struct CreateQueryUUIDs;
 
 
 class ASTStorage : public IAST
@@ -101,17 +103,15 @@ public:
     bool has_uuid{false}; // CREATE TABLE x UUID '...'
 
     ASTColumns * columns_list = nullptr;
-
-    StorageID to_table_id = StorageID::createEmpty();   /// For CREATE MATERIALIZED VIEW mv TO table.
-    UUID to_inner_uuid = UUIDHelpers::Nil;      /// For materialized view with inner table
-    ASTStorage * inner_storage = nullptr;      /// For window view with inner table
     ASTStorage * storage = nullptr;
+
     ASTPtr watermark_function;
     ASTPtr lateness_function;
     String as_database;
     String as_table;
     IAST * as_table_function = nullptr;
     ASTSelectWithUnionQuery * select = nullptr;
+    ASTViewTargets * targets = nullptr;
     IAST * comment = nullptr;
     ASTPtr sql_security = nullptr;
 
@@ -153,17 +153,25 @@ public:
 
     QueryKind getQueryKind() const override { return QueryKind::Create; }
 
-    struct UUIDs
-    {
-        UUID uuid = UUIDHelpers::Nil;
-        UUID to_inner_uuid = UUIDHelpers::Nil;
-        UUIDs() = default;
-        explicit UUIDs(const ASTCreateQuery & query);
-        String toString() const;
-        static UUIDs fromString(const String & str);
-    };
-    UUIDs generateRandomUUID(bool always_generate_new_uuid = false);
-    void setUUID(const UUIDs & uuids);
+    /// Generates a random UUID for this create query if it's not specified already.
+    /// The function also generates random UUIDs for inner target tables if this create query implies that
+    /// (for example, if it's a `CREATE MATERIALIZED VIEW` query with an inner storage).
+    void generateRandomUUIDs();
+
+    /// Removes UUID from this create query.
+    /// The function also removes UUIDs for inner target tables from this create query (see also generateRandomUUID()).
+    void resetUUIDs();
+
+    /// Returns information about a target table.
+    /// If that information isn't specified in this create query (or even not allowed) then the function returns an empty value.
+    StorageID getTargetTableID(ViewTarget::Kind target_kind) const;
+    bool hasTargetTableID(ViewTarget::Kind target_kind) const;
+    UUID getTargetInnerUUID(ViewTarget::Kind target_kind) const;
+    std::shared_ptr<ASTStorage> getTargetInnerEngine(ViewTarget::Kind target_kind) const;
+    void setTargetInnerEngine(ViewTarget::Kind target_kind, ASTPtr storage_def);
+
+    bool is_materialized_view_with_external_target() const { return is_materialized_view && hasTargetTableID(ViewTarget::To); }
+    bool is_materialized_view_with_inner_table() const { return is_materialized_view && !hasTargetTableID(ViewTarget::To); }
 
 protected:
     void formatQueryImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override;
@@ -171,8 +179,8 @@ protected:
     void forEachPointerToChild(std::function<void(void**)> f) override
     {
         f(reinterpret_cast<void **>(&columns_list));
-        f(reinterpret_cast<void **>(&inner_storage));
         f(reinterpret_cast<void **>(&storage));
+        f(reinterpret_cast<void **>(&targets));
         f(reinterpret_cast<void **>(&as_table_function));
         f(reinterpret_cast<void **>(&select));
         f(reinterpret_cast<void **>(&comment));
diff --git a/src/Parsers/ASTViewTargets.cpp b/src/Parsers/ASTViewTargets.cpp
new file mode 100644
index 00000000000..38f103b6e55
--- /dev/null
+++ b/src/Parsers/ASTViewTargets.cpp
@@ -0,0 +1,300 @@
+#include <Parsers/ASTViewTargets.h>
+
+#include <Parsers/ASTCreateQuery.h>
+#include <Parsers/CommonParsers.h>
+#include <IO/WriteHelpers.h>
+
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int BAD_ARGUMENTS;
+    extern const int LOGICAL_ERROR;
+}
+
+
+std::string_view toString(ViewTarget::Kind kind)
+{
+    switch (kind)
+    {
+        case ViewTarget::To:      return "to";
+        case ViewTarget::Inner:   return "inner";
+    }
+    throw Exception(ErrorCodes::LOGICAL_ERROR, "{} doesn't support kind {}", __FUNCTION__, kind);
+}
+
+void parseFromString(ViewTarget::Kind & out, std::string_view str)
+{
+    for (auto kind : magic_enum::enum_values<ViewTarget::Kind>())
+    {
+        if (toString(kind) == str)
+        {
+            out = kind;
+            return;
+        }
+    }
+    throw Exception(ErrorCodes::BAD_ARGUMENTS, "{}: Unexpected string {}", __FUNCTION__, str);
+}
+
+
+std::vector<ViewTarget::Kind> ASTViewTargets::getKinds() const
+{
+    std::vector<ViewTarget::Kind> kinds;
+    kinds.reserve(targets.size());
+    for (auto & target : targets)
+        kinds.push_back(target.kind);
+    return kinds;
+}
+
+
+void ASTViewTargets::setTableID(ViewTarget::Kind kind, const StorageID & table_id_)
+{
+    for (auto & target : targets)
+    {
+        if (target.kind == kind)
+        {
+            target.table_id = table_id_;
+            return;
+        }
+    }
+    if (table_id_)
+        targets.emplace_back(kind).table_id = table_id_;
+}
+
+StorageID ASTViewTargets::getTableID(ViewTarget::Kind kind) const
+{
+    if (const auto * target = tryGetTarget(kind))
+        return target->table_id;
+    return StorageID::createEmpty();
+}
+
+bool ASTViewTargets::hasTableID(ViewTarget::Kind kind) const
+{
+    if (const auto * target = tryGetTarget(kind))
+        return !target->table_id.empty();
+    return false;
+}
+
+void ASTViewTargets::setCurrentDatabase(const String & current_database)
+{
+    for (auto & target : targets)
+    {
+        auto & table_id = target.table_id;
+        if (!table_id.table_name.empty() && table_id.database_name.empty())
+            table_id.database_name = current_database;
+    }
+}
+
+void ASTViewTargets::setInnerUUID(ViewTarget::Kind kind, const UUID & inner_uuid_)
+{
+    for (auto & target : targets)
+    {
+        if (target.kind == kind)
+        {
+            target.inner_uuid = inner_uuid_;
+            return;
+        }
+    }
+    if (inner_uuid_ != UUIDHelpers::Nil)
+        targets.emplace_back(kind).inner_uuid = inner_uuid_;
+}
+
+UUID ASTViewTargets::getInnerUUID(ViewTarget::Kind kind) const
+{
+    if (const auto * target = tryGetTarget(kind))
+        return target->inner_uuid;
+    return UUIDHelpers::Nil;
+}
+
+bool ASTViewTargets::hasInnerUUID(ViewTarget::Kind kind) const
+{
+    return getInnerUUID(kind) != UUIDHelpers::Nil;
+}
+
+void ASTViewTargets::resetInnerUUIDs()
+{
+    for (auto & target : targets)
+        target.inner_uuid = UUIDHelpers::Nil;
+}
+
+bool ASTViewTargets::hasInnerUUIDs() const
+{
+    for (auto & target : targets)
+    {
+        if (target.inner_uuid != UUIDHelpers::Nil)
+            return true;
+    }
+    return false;
+}
+
+void ASTViewTargets::setInnerEngine(ViewTarget::Kind kind, ASTPtr storage_def)
+{
+    auto new_inner_engine = typeid_cast<std::shared_ptr<ASTStorage>>(storage_def);
+    if (!new_inner_engine && storage_def)
+        throw Exception(DB::ErrorCodes::LOGICAL_ERROR, "Bad cast from type {} to ASTStorage", storage_def->getID());
+
+    for (auto & target : targets)
+    {
+        if (target.kind == kind)
+        {
+            if (target.inner_engine == new_inner_engine)
+                return;
+            if (new_inner_engine)
+                children.push_back(new_inner_engine);
+            if (target.inner_engine)
+                std::erase(children, target.inner_engine);
+            target.inner_engine = new_inner_engine;
+            return;
+        }
+    }
+
+    if (new_inner_engine)
+    {
+        targets.emplace_back(kind).inner_engine = new_inner_engine;
+        children.push_back(new_inner_engine);
+    }
+}
+
+std::shared_ptr<ASTStorage> ASTViewTargets::getInnerEngine(ViewTarget::Kind kind) const
+{
+    if (const auto * target = tryGetTarget(kind))
+        return target->inner_engine;
+    return nullptr;
+}
+
+std::vector<std::shared_ptr<ASTStorage>> ASTViewTargets::getInnerEngines() const
+{
+    std::vector<std::shared_ptr<ASTStorage>> res;
+    res.reserve(targets.size());
+    for (const auto & target : targets)
+    {
+        if (target.inner_engine)
+            res.push_back(target.inner_engine);
+    }
+    return res;
+}
+
+const ViewTarget * ASTViewTargets::tryGetTarget(ViewTarget::Kind kind) const
+{
+    for (const auto & target : targets)
+    {
+        if (target.kind == kind)
+            return &target;
+    }
+    return nullptr;
+}
+
+ASTPtr ASTViewTargets::clone() const
+{
+    auto res = std::make_shared<ASTViewTargets>(*this);
+    res->children.clear();
+    for (auto & target : res->targets)
+    {
+        if (target.inner_engine)
+        {
+            target.inner_engine = typeid_cast<std::shared_ptr<ASTStorage>>(target.inner_engine->clone());
+            res->children.push_back(target.inner_engine);
+        }
+    }
+    return res;
+}
+
+void ASTViewTargets::formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const
+{
+    for (const auto & target : targets)
+        formatTarget(target, s, state, frame);
+}
+
+void ASTViewTargets::formatTarget(ViewTarget::Kind kind, const FormatSettings & s, FormatState & state, FormatStateStacked frame) const
+{
+    for (const auto & target : targets)
+    {
+        if (target.kind == kind)
+            formatTarget(target, s, state, frame);
+    }
+}
+
+void ASTViewTargets::formatTarget(const ViewTarget & target, const FormatSettings & s, FormatState & state, FormatStateStacked frame)
+{
+    if (target.table_id)
+    {
+        auto keyword = getKeywordForTableID(target.kind);
+        if (!keyword)
+            throw Exception(ErrorCodes::LOGICAL_ERROR, "No keyword for table name of kind {}", toString(target.kind));
+        s.ostr <<  " " << (s.hilite ? hilite_keyword : "") << toStringView(*keyword)
+               << (s.hilite ? hilite_none : "") << " "
+               << (!target.table_id.database_name.empty() ? backQuoteIfNeed(target.table_id.database_name) + "." : "")
+               << backQuoteIfNeed(target.table_id.table_name);
+    }
+
+    if (target.inner_uuid != UUIDHelpers::Nil)
+    {
+        auto keyword = getKeywordForInnerUUID(target.kind);
+        if (!keyword)
+            throw Exception(ErrorCodes::LOGICAL_ERROR, "No prefix keyword for inner UUID of kind {}", toString(target.kind));
+        s.ostr << " " << (s.hilite ? hilite_keyword : "") << toStringView(*keyword)
+               << (s.hilite ? hilite_none : "") << " " << quoteString(toString(target.inner_uuid));
+    }
+
+    if (target.inner_engine)
+    {
+        auto keyword = getKeywordForInnerStorage(target.kind);
+        if (!keyword)
+            throw Exception(ErrorCodes::LOGICAL_ERROR, "No prefix keyword for table engine of kind {}", toString(target.kind));
+        s.ostr << " " << (s.hilite ? hilite_keyword : "") << toStringView(*keyword) << (s.hilite ? hilite_none : "");
+        target.inner_engine->formatImpl(s, state, frame);
+    }
+}
+
+std::optional<Keyword> ASTViewTargets::getKeywordForTableID(ViewTarget::Kind kind)
+{
+    switch (kind)
+    {
+        case ViewTarget::To:      return Keyword::TO;      /// TO mydb.mydata
+        case ViewTarget::Inner:   return std::nullopt;
+    }
+    UNREACHABLE();
+}
+
+std::optional<Keyword> ASTViewTargets::getKeywordForInnerStorage(ViewTarget::Kind kind)
+{
+    switch (kind)
+    {
+        case ViewTarget::To:      return std::nullopt;      /// ENGINE = MergeTree()
+        case ViewTarget::Inner:   return Keyword::INNER;    /// INNER ENGINE = MergeTree()
+    }
+    UNREACHABLE();
+}
+
+std::optional<Keyword> ASTViewTargets::getKeywordForInnerUUID(ViewTarget::Kind kind)
+{
+    switch (kind)
+    {
+        case ViewTarget::To:      return Keyword::TO_INNER_UUID;       /// TO INNER UUID 'XXX'
+        case ViewTarget::Inner:   return std::nullopt;
+    }
+    UNREACHABLE();
+}
+
+void ASTViewTargets::forEachPointerToChild(std::function<void(void**)> f)
+{
+    for (auto & target : targets)
+    {
+        if (target.inner_engine)
+        {
+            ASTStorage * new_inner_engine = target.inner_engine.get();
+            f(reinterpret_cast<void **>(&new_inner_engine));
+            if (new_inner_engine != target.inner_engine.get())
+            {
+                if (new_inner_engine)
+                    target.inner_engine = typeid_cast<std::shared_ptr<ASTStorage>>(new_inner_engine->ptr());
+                else
+                    target.inner_engine.reset();
+            }
+        }
+    }
+}
+
+}
diff --git a/src/Parsers/ASTViewTargets.h b/src/Parsers/ASTViewTargets.h
new file mode 100644
index 00000000000..33a7bc5fcb1
--- /dev/null
+++ b/src/Parsers/ASTViewTargets.h
@@ -0,0 +1,102 @@
+#pragma once
+
+#include <Parsers/IAST.h>
+#include <Interpreters/StorageID.h>
+
+
+namespace DB
+{
+class ASTStorage;
+enum class Keyword : size_t;
+
+/// Information about the target table for a materialized view or a window view.
+struct ViewTarget
+{
+    enum Kind
+    {
+        /// Target table for a materialized view or a window view.
+        To,
+
+        /// Table with intermediate results for a window view.
+        Inner,
+    };
+
+    Kind kind = To;
+
+    /// StorageID of the target table, if it's not inner.
+    /// That storage ID can be seen for example after "TO" in a statement like CREATE MATERIALIZED VIEW ... TO ...
+    StorageID table_id = StorageID::createEmpty();
+
+    /// UUID of the target table, if it's inner.
+    /// The UUID is calculated automatically and can be seen for example after "TO INNER UUID" in a statement like
+    /// CREATE MATERIALIZED VIEW ... TO INNER UUID ...
+    UUID inner_uuid = UUIDHelpers::Nil;
+
+    /// Table engine of the target table, if it's inner.
+    /// That engine can be seen for example after "ENGINE" in a statement like CREATE MATERIALIZED VIEW ... ENGINE ...
+    std::shared_ptr<ASTStorage> inner_engine;
+};
+
+/// Converts ViewTarget::Kind to a string.
+std::string_view toString(ViewTarget::Kind kind);
+void parseFromString(ViewTarget::Kind & out, std::string_view str);
+
+
+/// Information about all the target tables for a view.
+class ASTViewTargets : public IAST
+{
+public:
+    std::vector<ViewTarget> targets;
+
+    /// Sets the StorageID of the target table, if it's not inner.
+    /// That storage ID can be seen for example after "TO" in a statement like CREATE MATERIALIZED VIEW ... TO ...
+    void setTableID(ViewTarget::Kind kind, const StorageID & table_id_);
+    StorageID getTableID(ViewTarget::Kind kind) const;
+    bool hasTableID(ViewTarget::Kind kind) const;
+
+    /// Replaces an empty database in the StorageID of the target table with a specified database.
+    void setCurrentDatabase(const String & current_database);
+
+    /// Sets the UUID of the target table, if it's inner.
+    /// The UUID is calculated automatically and can be seen for example after "TO INNER UUID" in a statement like
+    /// CREATE MATERIALIZED VIEW ... TO INNER UUID ...
+    void setInnerUUID(ViewTarget::Kind kind, const UUID & inner_uuid_);
+    UUID getInnerUUID(ViewTarget::Kind kind) const;
+    bool hasInnerUUID(ViewTarget::Kind kind) const;
+
+    void resetInnerUUIDs();
+    bool hasInnerUUIDs() const;
+
+    /// Sets the table engine of the target table, if it's inner.
+    /// That engine can be seen for example after "ENGINE" in a statement like CREATE MATERIALIZED VIEW ... ENGINE ...
+    void setInnerEngine(ViewTarget::Kind kind, ASTPtr storage_def);
+    std::shared_ptr<ASTStorage> getInnerEngine(ViewTarget::Kind kind) const;
+    std::vector<std::shared_ptr<ASTStorage>> getInnerEngines() const;
+
+    /// Returns a list of all kinds of views in this ASTViewTargets.
+    std::vector<ViewTarget::Kind> getKinds() const;
+
+    /// Returns information about a target table.
+    /// The function returns null if such target doesn't exist.
+    const ViewTarget * tryGetTarget(ViewTarget::Kind kind) const;
+
+    String getID(char) const override { return "ViewTargets"; }
+
+    ASTPtr clone() const override;
+
+    void formatImpl(const FormatSettings & s, FormatState & state, FormatStateStacked frame) const override;
+
+    /// Formats information only about a specific target table.
+    void formatTarget(ViewTarget::Kind kind, const FormatSettings & s, FormatState & state, FormatStateStacked frame) const;
+    static void formatTarget(const ViewTarget & target, const FormatSettings & s, FormatState & state, FormatStateStacked frame);
+
+    /// Helper functions for class ParserViewTargets. Returns a prefix keyword matching a specified target kind.
+    static std::optional<Keyword> getKeywordForTableID(ViewTarget::Kind kind);
+    static std::optional<Keyword> getKeywordForInnerUUID(ViewTarget::Kind kind);
+    static std::optional<Keyword> getKeywordForInnerStorage(ViewTarget::Kind kind);
+
+protected:
+    void forEachPointerToChild(std::function<void(void**)> f) override;
+};
+
+}
diff --git a/src/Parsers/CreateQueryUUIDs.cpp b/src/Parsers/CreateQueryUUIDs.cpp
new file mode 100644
index 00000000000..4dfee67b537
--- /dev/null
+++ b/src/Parsers/CreateQueryUUIDs.cpp
@@ -0,0 +1,168 @@
+#include <Parsers/CreateQueryUUIDs.h>
+
+#include <Parsers/ASTCreateQuery.h>
+#include <Parsers/ASTFunction.h>
+#include <IO/ReadBufferFromString.h>
+#include <IO/ReadHelpers.h>
+
+
+namespace DB
+{
+
+CreateQueryUUIDs::CreateQueryUUIDs(const ASTCreateQuery & query, bool generate_random, bool force_random)
+{
+    if (!generate_random || !force_random)
+    {
+        uuid = query.uuid;
+        if (query.targets)
+        {
+            for (const auto & target : query.targets->targets)
+                setTargetInnerUUID(target.kind, target.inner_uuid);
+        }
+    }
+
+    if (generate_random)
+    {
+        if (uuid == UUIDHelpers::Nil)
+            uuid = UUIDHelpers::generateV4();
+
+        /// For an ATTACH query we should never generate UUIDs for its inner target tables
+        /// because for an ATTACH query those inner target tables probably already exist and can be accessible by names.
+        /// If we generate random UUIDs for already existing tables then those UUIDs will not be correct making those inner target table inaccessible.
+        /// Thus it's not safe for example to replace
+        /// "ATTACH MATERIALIZED VIEW mv AS SELECT a FROM b" with
+        /// "ATTACH MATERIALIZED VIEW mv TO INNER UUID "XXXX" AS SELECT a FROM b"
+        /// This replacement is safe only for CREATE queries when inner target tables don't exist yet.
+        if (!query.attach)
+        {
+            auto generate_target_uuid = [&](ViewTarget::Kind target_kind)
+            {
+                if ((query.getTargetInnerUUID(target_kind) == UUIDHelpers::Nil) && query.getTargetTableID(target_kind).empty())
+                    setTargetInnerUUID(target_kind, UUIDHelpers::generateV4());
+            };
+
+            /// If destination table (to_table_id) is not specified for materialized view,
+            /// then MV will create inner table. We should generate UUID of inner table here.
+            if (query.is_materialized_view)
+                generate_target_uuid(ViewTarget::To);
+        }
+    }
+}
+
+bool CreateQueryUUIDs::empty() const
+{
+    if (uuid != UUIDHelpers::Nil)
+        return false;
+    for (const auto & [_, inner_uuid] : targets_inner_uuids)
+    {
+        if (inner_uuid != UUIDHelpers::Nil)
+            return false;
+    }
+    return true;
+}
+
+String CreateQueryUUIDs::toString() const
+{
+    WriteBufferFromOwnString out;
+    out << "{";
+    bool need_comma = false;
+    auto add_name_and_uuid_to_string = [&](std::string_view name_, const UUID & uuid_)
+    {
+        if (std::exchange(need_comma, true))
+            out << ", ";
+        out << "\"" << name_ << "\": \"" << uuid_ << "\"";
+    };
+    if (uuid != UUIDHelpers::Nil)
+        add_name_and_uuid_to_string("uuid", uuid);
+    for (const auto & [kind, inner_uuid] : targets_inner_uuids)
+    {
+        if (inner_uuid != UUIDHelpers::Nil)
+            add_name_and_uuid_to_string(::DB::toString(kind), inner_uuid);
+    }
+    out << "}";
+    return out.str();
+}
+
+CreateQueryUUIDs CreateQueryUUIDs::fromString(const String & str)
+{
+    ReadBufferFromString in{str};
+    CreateQueryUUIDs res;
+    skipWhitespaceIfAny(in);
+    in >> "{";
+    skipWhitespaceIfAny(in);
+    char c;
+    while (in.peek(c) && c != '}')
+    {
+        String name;
+        String value;
+        readDoubleQuotedString(name, in);
+        skipWhitespaceIfAny(in);
+        in >> ":";
+        skipWhitespaceIfAny(in);
+        readDoubleQuotedString(value, in);
+        skipWhitespaceIfAny(in);
+        if (name == "uuid")
+        {
+            res.uuid = parse<UUID>(value);
+        }
+        else
+        {
+            ViewTarget::Kind kind;
+            parseFromString(kind, name);
+            res.setTargetInnerUUID(kind, parse<UUID>(value));
+        }
+        if (in.peek(c) && c == ',')
+        {
+            in.ignore(1);
+            skipWhitespaceIfAny(in);
+        }
+    }
+    in >> "}";
+    return res;
+}
+
+void CreateQueryUUIDs::setTargetInnerUUID(ViewTarget::Kind kind, const UUID & new_inner_uuid)
+{
+    for (auto & pair : targets_inner_uuids)
+    {
+        if (pair.first == kind)
+        {
+            pair.second = new_inner_uuid;
+            return;
+        }
+    }
+    if (new_inner_uuid != UUIDHelpers::Nil)
+        targets_inner_uuids.emplace_back(kind, new_inner_uuid);
+}
+
+UUID CreateQueryUUIDs::getTargetInnerUUID(ViewTarget::Kind kind) const
+{
+    for (const auto & pair : targets_inner_uuids)
+    {
+        if (pair.first == kind)
+            return pair.second;
+    }
+    return UUIDHelpers::Nil;
+}
+
+void CreateQueryUUIDs::copyToQuery(ASTCreateQuery & query) const
+{
+    query.uuid = uuid;
+
+    if (query.targets)
+        query.targets->resetInnerUUIDs();
+
+    if (!targets_inner_uuids.empty())
+    {
+        if (!query.targets)
+            query.set(query.targets, std::make_shared<ASTViewTargets>());
+
+        for (const auto & [kind, inner_uuid] : targets_inner_uuids)
+        {
+            if (inner_uuid != UUIDHelpers::Nil)
+                query.targets->setInnerUUID(kind, inner_uuid);
+        }
+    }
+}
+
+}
diff --git a/src/Parsers/CreateQueryUUIDs.h b/src/Parsers/CreateQueryUUIDs.h
new file mode 100644
index 00000000000..419dad24b35
--- /dev/null
+++ b/src/Parsers/CreateQueryUUIDs.h
@@ -0,0 +1,40 @@
+#pragma once
+
+#include <Parsers/ASTViewTargets.h>
+
+
+namespace DB
+{
+class ASTCreateQuery;
+
+/// The UUID of a table or a database defined with a CREATE QUERY along with the UUIDs of its inner targets.
+struct CreateQueryUUIDs
+{
+    CreateQueryUUIDs() = default;
+
+    /// Collect UUIDs from ASTCreateQuery.
+    /// Parameters:
+    /// `generate_random` - if it's true then unspecified in the query UUIDs will be generated randomly;
+    /// `force_random` - if it's true then all UUIDs (even specified in the query) will be (re)generated randomly.
+    explicit CreateQueryUUIDs(const ASTCreateQuery & query, bool generate_random = false, bool force_random = false);
+
+    bool empty() const;
+    explicit operator bool() const { return !empty(); }
+
+    String toString() const;
+    static CreateQueryUUIDs fromString(const String & str);
+
+    void setTargetInnerUUID(ViewTarget::Kind kind, const UUID & new_inner_uuid);
+    UUID getTargetInnerUUID(ViewTarget::Kind kind) const;
+
+    /// Copies UUIDs to ASTCreateQuery.
+    void copyToQuery(ASTCreateQuery & query) const;
+
+    /// UUID of the table.
+    UUID uuid = UUIDHelpers::Nil;
+
+    /// UUIDs of its target table (or tables).
+    std::vector<std::pair<ViewTarget::Kind, UUID>> targets_inner_uuids;
+};
+
+}
diff --git a/src/Parsers/ParserCreateQuery.cpp b/src/Parsers/ParserCreateQuery.cpp
index 014dc7bd3bf..41379a845e7 100644
--- a/src/Parsers/ParserCreateQuery.cpp
+++ b/src/Parsers/ParserCreateQuery.cpp
@@ -22,6 +22,7 @@
 #include <Parsers/ParserSelectWithUnionQuery.h>
 #include <Parsers/ParserSetQuery.h>
 #include <Parsers/ParserRefreshStrategy.h>
+#include <Parsers/ParserViewTargets.h>
 #include <Common/typeid_cast.h>
 #include <Parsers/ASTColumnDeclaration.h>
 
@@ -693,7 +694,8 @@ bool ParserCreateTableQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe
 
     ASTPtr table;
     ASTPtr columns_list;
-    ASTPtr storage;
+    std::shared_ptr<ASTStorage> storage;
+    ASTPtr targets;
     ASTPtr as_database;
     ASTPtr as_table;
     ASTPtr as_table_function;
@@ -773,6 +775,17 @@ bool ParserCreateTableQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe
         return true;
     }
 
+    auto parse_storage = [&]
+    {
+        chassert(!storage);
+        ASTPtr ast;
+        if (!storage_p.parse(pos, ast, expected))
+            return false;
+
+        storage = typeid_cast<std::shared_ptr<ASTStorage>>(ast);
+        return true;
+    };
+
     auto need_parse_as_select = [&is_create_empty, &pos, &expected]()
     {
         if (ParserKeyword{Keyword::EMPTY_AS}.ignore(pos, expected))
@@ -798,7 +811,7 @@ bool ParserCreateTableQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe
         if (!s_rparen.ignore(pos, expected))
             return false;
 
-        auto storage_parse_result = storage_p.parse(pos, storage, expected);
+        auto storage_parse_result = parse_storage();
 
         if ((storage_parse_result || is_temporary) && need_parse_as_select())
         {
@@ -820,7 +833,7 @@ bool ParserCreateTableQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe
       */
     else
     {
-        storage_p.parse(pos, storage, expected);
+        parse_storage();
 
         /// CREATE|ATTACH TABLE ... AS ...
         if (need_parse_as_select())
@@ -843,7 +856,7 @@ bool ParserCreateTableQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe
 
                     /// Optional - ENGINE can be specified.
                     if (!storage)
-                        storage_p.parse(pos, storage, expected);
+                        parse_storage();
                 }
             }
         }
@@ -904,6 +917,7 @@ bool ParserCreateTableQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe
     tryGetIdentifierNameInto(as_database, query->as_database);
     tryGetIdentifierNameInto(as_table, query->as_table);
     query->set(query->select, select);
+    query->set(query->targets, targets);
     query->is_create_empty = is_create_empty;
 
     if (from_path)
@@ -977,6 +991,13 @@ bool ParserCreateLiveViewQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & e
             return false;
     }
 
+    std::shared_ptr<ASTViewTargets> targets;
+    if (to_table)
+    {
+        targets = std::make_shared<ASTViewTargets>();
+        targets->setTableID(ViewTarget::To, to_table->as<ASTTableIdentifier>()->getTableId());
+    }
+
     /// Optional - a list of columns can be specified. It must fully comply with SELECT.
     if (s_lparen.ignore(pos, expected))
     {
@@ -1017,14 +1038,12 @@ bool ParserCreateLiveViewQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & e
     if (query->table)
         query->children.push_back(query->table);
 
-    if (to_table)
-        query->to_table_id = to_table->as<ASTTableIdentifier>()->getTableId();
-
     query->set(query->columns_list, columns_list);
 
     tryGetIdentifierNameInto(as_database, query->as_database);
     tryGetIdentifierNameInto(as_table, query->as_table);
     query->set(query->select, select);
+    query->set(query->targets, targets);
 
     if (comment)
         query->set(query->comment, comment);
@@ -1139,6 +1158,18 @@ bool ParserCreateWindowViewQuery::parseImpl(Pos & pos, ASTPtr & node, Expected &
         storage_p.parse(pos, storage, expected);
     }
 
+    std::shared_ptr<ASTViewTargets> targets;
+    if (to_table || storage || inner_storage)
+    {
+        targets = std::make_shared<ASTViewTargets>();
+        if (to_table)
+            targets->setTableID(ViewTarget::To, to_table->as<ASTTableIdentifier>()->getTableId());
+        if (storage)
+            targets->setInnerEngine(ViewTarget::To, storage);
+        if (inner_storage)
+            targets->setInnerEngine(ViewTarget::Inner, inner_storage);
+    }
+
     // WATERMARK
     if (s_watermark.ignore(pos, expected))
     {
@@ -1195,12 +1226,8 @@ bool ParserCreateWindowViewQuery::parseImpl(Pos & pos, ASTPtr & node, Expected &
     if (query->table)
         query->children.push_back(query->table);
 
-    if (to_table)
-        query->to_table_id = to_table->as<ASTTableIdentifier>()->getTableId();
-
     query->set(query->columns_list, columns_list);
-    query->set(query->storage, storage);
-    query->set(query->inner_storage, inner_storage);
+
     query->is_watermark_strictly_ascending = is_watermark_strictly_ascending;
     query->is_watermark_ascending = is_watermark_ascending;
     query->is_watermark_bounded = is_watermark_bounded;
@@ -1213,6 +1240,7 @@ bool ParserCreateWindowViewQuery::parseImpl(Pos & pos, ASTPtr & node, Expected &
     tryGetIdentifierNameInto(as_database, query->as_database);
     tryGetIdentifierNameInto(as_table, query->as_table);
     query->set(query->select, select);
+    query->set(query->targets, targets);
 
     return true;
 }
@@ -1436,6 +1464,7 @@ bool ParserCreateDatabaseQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & e
     return true;
 }
 
+
 bool ParserCreateViewQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
 {
     ParserKeyword s_create(Keyword::CREATE);
@@ -1622,13 +1651,8 @@ bool ParserCreateViewQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec
     if (query->table)
         query->children.push_back(query->table);
 
-    if (to_table)
-        query->to_table_id = to_table->as<ASTTableIdentifier>()->getTableId();
-    if (to_inner_uuid)
-        query->to_inner_uuid = parseFromString<UUID>(to_inner_uuid->as<ASTLiteral>()->value.get<String>());
-
     query->set(query->columns_list, columns_list);
-    query->set(query->storage, storage);
+
     if (refresh_strategy)
         query->set(query->refresh_strategy, refresh_strategy);
     if (comment)
@@ -1639,29 +1663,41 @@ bool ParserCreateViewQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec
     if (query->columns_list && query->columns_list->primary_key)
     {
         /// If engine is not set will use default one
-        if (!query->storage)
-            query->set(query->storage, std::make_shared<ASTStorage>());
-        else if (query->storage->primary_key)
+        if (!storage)
+            storage = std::make_shared<ASTStorage>();
+        auto & storage_ref = typeid_cast<ASTStorage &>(*storage);
+        if (storage_ref.primary_key)
             throw Exception(ErrorCodes::BAD_ARGUMENTS, "Multiple primary keys are not allowed.");
-
-        query->storage->primary_key = query->columns_list->primary_key;
-
+        storage_ref.primary_key = query->columns_list->primary_key;
     }
 
     if (query->columns_list && (query->columns_list->primary_key_from_columns))
     {
         /// If engine is not set will use default one
-        if (!query->storage)
-            query->set(query->storage, std::make_shared<ASTStorage>());
-        else if (query->storage->primary_key)
+        if (!storage)
+            storage = std::make_shared<ASTStorage>();
+        auto & storage_ref = typeid_cast<ASTStorage &>(*storage);
+        if (storage_ref.primary_key)
             throw Exception(ErrorCodes::BAD_ARGUMENTS, "Multiple primary keys are not allowed.");
+        storage_ref.primary_key = query->columns_list->primary_key_from_columns;
+    }
 
-        query->storage->primary_key = query->columns_list->primary_key_from_columns;
+    std::shared_ptr<ASTViewTargets> targets;
+    if (to_table || to_inner_uuid || storage)
+    {
+        targets = std::make_shared<ASTViewTargets>();
+        if (to_table)
+            targets->setTableID(ViewTarget::To, to_table->as<ASTTableIdentifier>()->getTableId());
+        if (to_inner_uuid)
+            targets->setInnerUUID(ViewTarget::To, parseFromString<UUID>(to_inner_uuid->as<ASTLiteral>()->value.safeGet<String>()));
+        if (storage)
+            targets->setInnerEngine(ViewTarget::To, storage);
     }
 
     tryGetIdentifierNameInto(as_database, query->as_database);
     tryGetIdentifierNameInto(as_table, query->as_table);
     query->set(query->select, select);
+    query->set(query->targets, targets);
 
     return true;
 }
diff --git a/src/Parsers/ParserViewTargets.cpp b/src/Parsers/ParserViewTargets.cpp
new file mode 100644
index 00000000000..8f010882cdd
--- /dev/null
+++ b/src/Parsers/ParserViewTargets.cpp
@@ -0,0 +1,88 @@
+#include <Parsers/ParserViewTargets.h>
+
+#include <Parsers/ASTIdentifier.h>
+#include <Parsers/ASTViewTargets.h>
+#include <Parsers/ExpressionElementParsers.h>
+#include <Parsers/ParserCreateQuery.h>
+#include <IO/ReadHelpers.h>
+
+
+namespace DB
+{
+
+ParserViewTargets::ParserViewTargets()
+{
+    for (auto kind : magic_enum::enum_values<ViewTarget::Kind>())
+        accept_kinds.push_back(kind);
+}
+
+bool ParserViewTargets::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
+{
+    ParserStringLiteral literal_p;
+    ParserStorage storage_p{ParserStorage::TABLE_ENGINE};
+    ParserCompoundIdentifier table_name_p(/*table_name_with_optional_uuid*/ true, /*allow_query_parameter*/ true);
+
+    std::shared_ptr<ASTViewTargets> res;
+
+    auto result = [&] -> ASTViewTargets &
+    {
+        if (!res)
+            res = std::make_shared<ASTViewTargets>();
+        return *res;
+    };
+
+    for (;;)
+    {
+        auto start = pos;
+        for (auto kind : accept_kinds)
+        {
+            auto current = pos;
+
+            auto keyword = ASTViewTargets::getKeywordForInnerUUID(kind);
+            if (keyword && ParserKeyword{*keyword}.ignore(pos, expected))
+            {
+                ASTPtr ast;
+                if (literal_p.parse(pos, ast, expected))
+                {
+                    result().setInnerUUID(kind, parseFromString<UUID>(ast->as<ASTLiteral>()->value.safeGet<String>()));
+                    break;
+                }
+            }
+            pos = current;
+
+            keyword = ASTViewTargets::getKeywordForInnerStorage(kind);
+            if (keyword && ParserKeyword{*keyword}.ignore(pos, expected))
+            {
+                ASTPtr ast;
+                if (storage_p.parse(pos, ast, expected))
+                {
+                    result().setInnerEngine(kind, ast);
+                    break;
+                }
+            }
+            pos = current;
+
+            keyword = ASTViewTargets::getKeywordForTableID(kind);
+            if (keyword && ParserKeyword{*keyword}.ignore(pos, expected))
+            {
+                ASTPtr ast;
+                if (table_name_p.parse(pos, ast, expected))
+                {
+                    result().setTableID(kind, ast->as<ASTTableIdentifier>()->getTableId());
+                    break;
+                }
+            }
+            pos = current;
+        }
+        if (pos == start)
+            break;
+    }
+
+    if (!res || res->targets.empty())
+        return false;
+
+    node = res;
+    return true;
+}
+
+}
diff --git a/src/Parsers/ParserViewTargets.h b/src/Parsers/ParserViewTargets.h
new file mode 100644
index 00000000000..f5d1850e974
--- /dev/null
+++ b/src/Parsers/ParserViewTargets.h
@@ -0,0 +1,24 @@
+#pragma once
+
+#include <Parsers/IParserBase.h>
+#include <Parsers/ASTViewTargets.h>
+
+
+namespace DB
+{
+
+/// Parses information about target views of a table.
+class ParserViewTargets : public IParserBase
+{
+public:
+    ParserViewTargets();
+    explicit ParserViewTargets(const std::vector<ViewTarget::Kind> & accept_kinds_) : accept_kinds(accept_kinds_) { }
+
+protected:
+    const char * getName() const override { return "ViewTargets"; }
+    bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
+
+    std::vector<ViewTarget::Kind> accept_kinds;
+};
+
+}
diff --git a/src/Storages/StorageMaterializedView.cpp b/src/Storages/StorageMaterializedView.cpp
index 57d95a98f11..b603d0ecf87 100644
--- a/src/Storages/StorageMaterializedView.cpp
+++ b/src/Storages/StorageMaterializedView.cpp
@@ -93,11 +93,6 @@ StorageMaterializedView::StorageMaterializedView(
 {
     StorageInMemoryMetadata storage_metadata;
     storage_metadata.setColumns(columns_);
-    auto * storage_def = query.storage;
-    if (storage_def && storage_def->primary_key)
-        storage_metadata.primary_key = KeyDescription::getKeyFromAST(storage_def->primary_key->ptr(),
-                                                                     storage_metadata.columns,
-                                                                     local_context->getGlobalContext());
 
     if (query.sql_security)
         storage_metadata.setSQLSecurity(query.sql_security->as<ASTSQLSecurity &>());
@@ -110,12 +105,21 @@ StorageMaterializedView::StorageMaterializedView(
         throw Exception(ErrorCodes::INCORRECT_QUERY, "SELECT query is not specified for {}", getName());
 
     /// If the destination table is not set, use inner table
-    has_inner_table = query.to_table_id.empty();
-    if (has_inner_table && !query.storage)
+    auto to_table_id = query.getTargetTableID(ViewTarget::To);
+    has_inner_table = to_table_id.empty();
+    auto to_inner_uuid = query.getTargetInnerUUID(ViewTarget::To);
+    auto to_table_engine = query.getTargetInnerEngine(ViewTarget::To);
+
+    if (has_inner_table && !to_table_engine)
         throw Exception(ErrorCodes::INCORRECT_QUERY,
                         "You must specify where to save results of a MaterializedView query: "
                         "either ENGINE or an existing table in a TO clause");
 
+    if (to_table_engine && to_table_engine->primary_key)
+        storage_metadata.primary_key = KeyDescription::getKeyFromAST(to_table_engine->primary_key->ptr(),
+                                                                     storage_metadata.columns,
+                                                                     local_context->getGlobalContext());
+
     auto select = SelectQueryDescription::getSelectQueryFromASTForMatView(query.select->clone(), query.refresh_strategy != nullptr, local_context);
     if (select.select_table_id)
     {
@@ -135,25 +139,25 @@ StorageMaterializedView::StorageMaterializedView(
 
     setInMemoryMetadata(storage_metadata);
 
-    bool point_to_itself_by_uuid = has_inner_table && query.to_inner_uuid != UUIDHelpers::Nil
-                                                   && query.to_inner_uuid == table_id_.uuid;
-    bool point_to_itself_by_name = !has_inner_table && query.to_table_id.database_name == table_id_.database_name
-                                                    && query.to_table_id.table_name == table_id_.table_name;
+    bool point_to_itself_by_uuid = has_inner_table && to_inner_uuid != UUIDHelpers::Nil
+                                                   && to_inner_uuid == table_id_.uuid;
+    bool point_to_itself_by_name = !has_inner_table && to_table_id.database_name == table_id_.database_name
+                                                    && to_table_id.table_name == table_id_.table_name;
     if (point_to_itself_by_uuid || point_to_itself_by_name)
         throw Exception(ErrorCodes::BAD_ARGUMENTS, "Materialized view {} cannot point to itself", table_id_.getFullTableName());
 
     if (!has_inner_table)
     {
-        target_table_id = query.to_table_id;
+        target_table_id = to_table_id;
     }
     else if (LoadingStrictnessLevel::ATTACH <= mode)
     {
         /// If there is an ATTACH request, then the internal table must already be created.
-        target_table_id = StorageID(getStorageID().database_name, generateInnerTableName(getStorageID()), query.to_inner_uuid);
+        target_table_id = StorageID(getStorageID().database_name, generateInnerTableName(getStorageID()), to_inner_uuid);
     }
     else
     {
-        const String & engine = query.storage->engine->name;
+        const String & engine = to_table_engine->engine->name;
         const auto & storage_features = StorageFactory::instance().getStorageFeatures(engine);
 
         /// We will create a query to create an internal table.
@@ -161,8 +165,8 @@ StorageMaterializedView::StorageMaterializedView(
         auto manual_create_query = std::make_shared<ASTCreateQuery>();
         manual_create_query->setDatabase(getStorageID().database_name);
         manual_create_query->setTable(generateInnerTableName(getStorageID()));
-        manual_create_query->uuid = query.to_inner_uuid;
-        manual_create_query->has_uuid = query.to_inner_uuid != UUIDHelpers::Nil;
+        manual_create_query->uuid = to_inner_uuid;
+        manual_create_query->has_uuid = to_inner_uuid != UUIDHelpers::Nil;
 
         auto new_columns_list = std::make_shared<ASTColumns>();
         new_columns_list->set(new_columns_list->columns, query.columns_list->columns->ptr());
@@ -184,7 +188,9 @@ StorageMaterializedView::StorageMaterializedView(
         }
 
         manual_create_query->set(manual_create_query->columns_list, new_columns_list);
-        manual_create_query->set(manual_create_query->storage, query.storage->ptr());
+
+        if (to_table_engine)
+            manual_create_query->set(manual_create_query->storage, to_table_engine);
 
         InterpreterCreateQuery create_interpreter(manual_create_query, create_context);
         create_interpreter.setInternal(true);
diff --git a/src/Storages/System/StorageSystemTables.cpp b/src/Storages/System/StorageSystemTables.cpp
index 783b899c978..14af3bad700 100644
--- a/src/Storages/System/StorageSystemTables.cpp
+++ b/src/Storages/System/StorageSystemTables.cpp
@@ -470,7 +470,8 @@ protected:
                     if (ast_create && !context->getSettingsRef().show_table_uuid_in_table_create_query_if_not_nil)
                     {
                         ast_create->uuid = UUIDHelpers::Nil;
-                        ast_create->to_inner_uuid = UUIDHelpers::Nil;
+                        if (ast_create->targets)
+                            ast_create->targets->resetInnerUUIDs();
                     }
 
                     if (columns_mask[src_index++])
diff --git a/src/Storages/WindowView/StorageWindowView.cpp b/src/Storages/WindowView/StorageWindowView.cpp
index b842cdda022..2b1d39fd3b6 100644
--- a/src/Storages/WindowView/StorageWindowView.cpp
+++ b/src/Storages/WindowView/StorageWindowView.cpp
@@ -1202,8 +1202,11 @@ StorageWindowView::StorageWindowView(
     setInMemoryMetadata(storage_metadata);
 
     /// If the target table is not set, use inner target table
-    has_inner_target_table = query.to_table_id.empty();
-    if (has_inner_target_table && !query.storage)
+    auto to_table_id = query.getTargetTableID(ViewTarget::To);
+    has_inner_target_table = to_table_id.empty();
+    auto to_table_engine = query.getTargetInnerEngine(ViewTarget::To);
+
+    if (has_inner_target_table && !to_table_engine)
         throw Exception(ErrorCodes::INCORRECT_QUERY,
                         "You must specify where to save results of a WindowView query: "
                         "either ENGINE or an existing table in a TO clause");
@@ -1218,12 +1221,12 @@ StorageWindowView::StorageWindowView(
 
     auto inner_query = initInnerQuery(query.select->list_of_selects->children.at(0)->as<ASTSelectQuery &>(), context_);
 
-    if (query.inner_storage)
-        inner_table_engine = query.inner_storage->clone();
+    if (auto inner_storage = query.getTargetInnerEngine(ViewTarget::Inner))
+        inner_table_engine = inner_storage->clone();
     inner_table_id = StorageID(getStorageID().database_name, generateInnerTableName(getStorageID()));
     inner_fetch_query = generateInnerFetchQuery(inner_table_id);
 
-    target_table_id = has_inner_target_table ? StorageID(table_id_.database_name, generateTargetTableName(table_id_)) : query.to_table_id;
+    target_table_id = has_inner_target_table ? StorageID(table_id_.database_name, generateTargetTableName(table_id_)) : to_table_id;
 
     if (is_proctime)
         next_fire_signal = getWindowUpperBound(now());
@@ -1248,7 +1251,7 @@ StorageWindowView::StorageWindowView(
             new_columns_list->set(new_columns_list->columns, query.columns_list->columns->ptr());
 
             target_create_query->set(target_create_query->columns_list, new_columns_list);
-            target_create_query->set(target_create_query->storage, query.storage->ptr());
+            target_create_query->set(target_create_query->storage, to_table_engine);
 
             InterpreterCreateQuery create_interpreter_(target_create_query, create_context_);
             create_interpreter_.setInternal(true);

From 9b6bdee5f3f80661577e0204f622dd6e41571806 Mon Sep 17 00:00:00 2001
From: Nikita Taranov <nikita.taranov@clickhouse.com>
Date: Tue, 16 Jul 2024 13:44:50 +0100
Subject: [PATCH 0295/1488] one more change after merging with master

---
 src/Core/SettingsChangesHistory.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp
index 8c096c13634..b78fed7e1c1 100644
--- a/src/Core/SettingsChangesHistory.cpp
+++ b/src/Core/SettingsChangesHistory.cpp
@@ -66,6 +66,7 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
               {"azure_sdk_max_retries", 10, 10, "Maximum number of retries in azure sdk"},
               {"azure_sdk_retry_initial_backoff_ms", 10, 10, "Minimal backoff between retries in azure sdk"},
               {"azure_sdk_retry_max_backoff_ms", 1000, 1000, "Maximal backoff between retries in azure sdk"},
+              {"merge_tree_min_bytes_per_task_for_remote_reading", 4194304, 2097152, "Value is unified with `filesystem_prefetch_min_bytes_for_single_read_task`"},
               }},
     {"24.6", {{"materialize_skip_indexes_on_insert", true, true, "Added new setting to allow to disable materialization of skip indexes on insert"},
               {"materialize_statistics_on_insert", true, true, "Added new setting to allow to disable materialization of statistics on insert"},

From 2018c5e760c148a1b5bc3a6b967fbb29125c5f84 Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Tue, 16 Jul 2024 14:55:14 +0200
Subject: [PATCH 0296/1488] Disable sanitizer

---
 tests/integration/test_memory_limit/test.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/integration/test_memory_limit/test.py b/tests/integration/test_memory_limit/test.py
index 942cf04f896..6d6745711da 100644
--- a/tests/integration/test_memory_limit/test.py
+++ b/tests/integration/test_memory_limit/test.py
@@ -27,6 +27,9 @@ def start_cluster():
 
 
 def test_multiple_queries():
+    if node.is_built_with_sanitizer():
+        return
+
     p = Pool(15)
 
     def run_query(node):

From 35725b666f6afdf115aa977fadd15d2076cba020 Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Tue, 16 Jul 2024 13:23:46 +0000
Subject: [PATCH 0297/1488] address comments

---
 src/Coordination/KeeperStateMachine.h |  2 --
 src/Coordination/KeeperStorage.h      |  7 ++++---
 src/Coordination/RocksDBContainer.h   | 11 ++---------
 3 files changed, 6 insertions(+), 14 deletions(-)

diff --git a/src/Coordination/KeeperStateMachine.h b/src/Coordination/KeeperStateMachine.h
index 4305e3b173d..dd2f239f551 100644
--- a/src/Coordination/KeeperStateMachine.h
+++ b/src/Coordination/KeeperStateMachine.h
@@ -114,8 +114,6 @@ protected:
     nuraft::ptr<nuraft::buffer> latest_snapshot_buf = nullptr;
 
     CoordinationSettingsPtr coordination_settings;
-    /// Main state machine logic
-    /// KeeperStoragePtr storage TSA_PT_GUARDED_BY(storage_and_responses_lock);
 
     /// Save/Load and Serialize/Deserialize logic for snapshots.
     /// Put processed responses into this queue
diff --git a/src/Coordination/KeeperStorage.h b/src/Coordination/KeeperStorage.h
index ef8a8d1410e..4a9286d4835 100644
--- a/src/Coordination/KeeperStorage.h
+++ b/src/Coordination/KeeperStorage.h
@@ -19,12 +19,12 @@ namespace DB
 class KeeperContext;
 using KeeperContextPtr = std::shared_ptr<KeeperContext>;
 
-/// struct KeeperStorageRequestProcessor;
-/// using KeeperStorageRequestProcessorPtr = std::shared_ptr<KeeperStorageRequestProcessor>;
 using ResponseCallback = std::function<void(const Coordination::ZooKeeperResponsePtr &)>;
 using ChildrenSet = absl::flat_hash_set<StringRef, StringRefHash>;
 using SessionAndTimeout = std::unordered_map<int64_t, int64_t>;
 
+/// KeeperRocksNodeInfo is used in RocksDB keeper.
+/// It is serialized directly as POD to RocksDB.
 struct KeeperRocksNodeInfo
 {
     int64_t czxid{0};
@@ -144,7 +144,8 @@ struct KeeperRocksNodeInfo
     void copyStats(const Coordination::Stat & stat);
 };
 
-struct KeeperRocksNode:KeeperRocksNodeInfo
+/// KeeperRocksNode is the memory structure used by RocksDB
+struct KeeperRocksNode : public KeeperRocksNodeInfo
 {
 #if USE_ROCKSDB
     friend struct RocksDBContainer<KeeperRocksNode>;
diff --git a/src/Coordination/RocksDBContainer.h b/src/Coordination/RocksDBContainer.h
index 46dc755aad8..12b40bbb87e 100644
--- a/src/Coordination/RocksDBContainer.h
+++ b/src/Coordination/RocksDBContainer.h
@@ -196,6 +196,7 @@ public:
                 rocksdb_dir, status.ToString());
         }
         rocksdb_ptr = std::unique_ptr<rocksdb::DB>(db);
+        write_options.disableWAL = true;
         initialized = true;
     }
 
@@ -312,9 +313,6 @@ public:
         }
         else if (status.IsNotFound())
         {
-            rocksdb::WriteOptions write_options;
-            write_options.disableWAL = true;
-
             status = rocksdb_ptr->Put(write_options, encoded_key, value.getEncodedString());
             if (status.ok())
             {
@@ -338,9 +336,6 @@ public:
         else if (!status.ok())
             throw Exception(ErrorCodes::ROCKSDB_ERROR, "Got rocksdb error during get. The error message is {}.", status.ToString());
 
-        rocksdb::WriteOptions write_options;
-        write_options.disableWAL = true;
-
         status = rocksdb_ptr->Put(write_options, encoded_key, value.getEncodedString());
         if (status.ok())
             counter += increase_counter;
@@ -367,9 +362,6 @@ public:
         /// storage->removeDigest(value, key);
         const std::string & encoded_key = getEncodedKey(key);
 
-        rocksdb::WriteOptions write_options;
-        write_options.disableWAL = true;
-
         auto status = rocksdb_ptr->Delete(write_options, encoded_key);
         if (status.IsNotFound())
             return false;
@@ -453,6 +445,7 @@ private:
     String rocksdb_dir;
 
     std::unique_ptr<rocksdb::DB> rocksdb_ptr;
+    rocksdb::WriteOptions write_options;
 
     const rocksdb::Snapshot * snapshot;
 

From 04f3c29b60658938a93aa6de6f15f8c50121e53e Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Tue, 16 Jul 2024 13:47:38 +0000
Subject: [PATCH 0298/1488] Fix crash.

---
 src/Columns/ColumnSparse.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Columns/ColumnSparse.cpp b/src/Columns/ColumnSparse.cpp
index ea4d23c1678..0922eb5ea2d 100644
--- a/src/Columns/ColumnSparse.cpp
+++ b/src/Columns/ColumnSparse.cpp
@@ -695,7 +695,7 @@ void ColumnSparse::updateWeakHash32(WeakHash32 & hash) const
     if (getNumberOfDefaultRows() > 0)
         values_hash_data[0] = hash_data[getFirstDefaultValueIndex()];
 
-    for (size_t i = 0; i < values_size; ++i)
+    for (size_t i = 0; i + 1 < values_size; ++i)
         values_hash_data[i + 1] = hash_data[offsets_data[i]];
 
     values->updateWeakHash32(values_hash);

From 8acc5d90ca295c2ff6b574da15dec70268a19015 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Tue, 16 Jul 2024 13:47:58 +0000
Subject: [PATCH 0299/1488] Add more test cases.

---
 ..._finctions_and_column_sparse_bug.reference | 75 ++++++++++++++++++
 ...window_finctions_and_column_sparse_bug.sql | 76 +++++++++++++++++++
 2 files changed, 151 insertions(+)

diff --git a/tests/queries/0_stateless/03205_parallel_window_finctions_and_column_sparse_bug.reference b/tests/queries/0_stateless/03205_parallel_window_finctions_and_column_sparse_bug.reference
index f11ec57a425..356329a392d 100644
--- a/tests/queries/0_stateless/03205_parallel_window_finctions_and_column_sparse_bug.reference
+++ b/tests/queries/0_stateless/03205_parallel_window_finctions_and_column_sparse_bug.reference
@@ -6,3 +6,78 @@ false	1	2
 false	1	3
 true	1	1
 true	1	2
+---
+-755809149	0
+---
+1	-2081147898
+1	-1981899149
+2	-2051538534
+2	-1650266905
+3	-1975508531
+3	-1646738223
+4	-1700730666
+4	-1618912877
+5	-1465484835
+5	-1317193174
+6	-1458338029
+6	-1219769753
+7	-1450619195
+7	-1154269118
+8	-1365934326
+8	-1150980622
+9	-1203382363
+9	-1098155311
+10	-1197430632
+10	-841067875
+11	-1176267855
+11	-816935497
+12	-1020892864
+12	-599948807
+13	-991301833
+13	-526570556
+14	-685902265
+14	-504713125
+15	-653505826
+15	-411038390
+16	-451392958
+16	-331834394
+17	-262516786
+17	-176934810
+18	-222873194
+18	-2
+19	-153185515
+19	6
+20	-74234560
+20	255
+21	-41
+21	406615258
+22	-6
+22	541758331
+23	-5
+23	720985423
+24	-3
+24	745669725
+25	15
+25	897064234
+26	65535
+26	1116921321
+27	77089559
+27	1207796283
+28	100663045
+28	1603772265
+29	561061873
+29	1664059402
+30	643897141
+30	1688303275
+31	914629990
+31	1913361922
+32	1159852204
+32	1929066636
+33	1258218855
+33	1968095908
+34	1459407556
+34	2054878592
+35	1936334332
+35	2125075305
+36	1962246186
+37	2030467062
diff --git a/tests/queries/0_stateless/03205_parallel_window_finctions_and_column_sparse_bug.sql b/tests/queries/0_stateless/03205_parallel_window_finctions_and_column_sparse_bug.sql
index a4c0200813c..6e326d0a67f 100644
--- a/tests/queries/0_stateless/03205_parallel_window_finctions_and_column_sparse_bug.sql
+++ b/tests/queries/0_stateless/03205_parallel_window_finctions_and_column_sparse_bug.sql
@@ -9,3 +9,79 @@ insert into t values (1, 0)(1, 1);
 SELECT d, c, row_number() over (partition by d order by c) as c8 FROM t qualify c8=1 order by d settings max_threads=2;
 SELECT '---';
 SELECT d, c, row_number() over (partition by d order by c) as c8 FROM t order by d, c8 settings max_threads=2;
+SELECT '---';
+
+drop table t;
+
+create table t (
+  c Int32 primary key ,
+  s Bool ,
+  w Float64
+  );
+
+system stop merges t;
+
+insert into t values(439499072,true,0),(1393290072,true,0);
+insert into t values(-1317193174,false,0),(1929066636,false,0);
+insert into t values(-2,false,0),(1962246186,true,0),(2054878592,false,0);
+insert into t values(-1893563136,true,41.55);
+insert into t values(-1338380855,true,-0.7),(-991301833,true,0),(-755809149,false,43.18),(-41,true,0),(3,false,0),(255,false,0),(255,false,0),(189195893,false,0),(195550885,false,9223372036854776000);
+
+SELECT * FROM (
+SELECT c, min(w) OVER (PARTITION BY s ORDER BY c ASC, s ASC, w ASC)
+FROM t limit toUInt64(-1))
+WHERE c = -755809149;
+
+SELECT '---';
+
+create table t_vkx4cc (
+  c_ylzjpt Int32,
+  c_hqfr9 Bool ,
+  ) engine = MergeTree order by c_ylzjpt;
+
+system stop merges t_vkx4cc;
+
+insert into t_vkx4cc (c_ylzjpt, c_hqfr9) values (-2081147898, coalesce((NOT NOT(cast( (53 < 539704722) as Nullable(Bool)))), true)), (-1219769753, coalesce((true) and (false), false)), (-1981899149, coalesce(false, false)), (-1646738223, coalesce((NOT NOT(cast( (23.5 <= -26) as Nullable(Bool)))), false));
+
+insert into t_vkx4cc (c_ylzjpt, c_hqfr9) values (255, coalesce(false, false)), (-1317193174, coalesce(false, false)), (-41, coalesce(true, false)), (1929066636, coalesce(false, true));
+
+insert into t_vkx4cc (c_ylzjpt, c_hqfr9) values (-1700730666, coalesce((NOT NOT(cast( (-2022515471055597472 AND -29) as Nullable(Bool)))), false)), (1664059402, coalesce((NOT NOT(cast( (-19643 >= -122) as Nullable(Bool)))), false)), (1688303275, coalesce((NOT NOT(cast( (737275892 < 105) as Nullable(Bool)))), true)), (406615258, coalesce((NOT NOT(cast( (-657730213 = 82.86) as Nullable(Bool)))), false));
+
+insert into t_vkx4cc (c_ylzjpt, c_hqfr9) values (-2, coalesce(false, false)), (1962246186, coalesce(true, false)), (-991301833, coalesce(true, true)), (2054878592, coalesce(false, false));
+
+insert into t_vkx4cc (c_ylzjpt, c_hqfr9) values (643897141, coalesce((NOT NOT(cast( (-60 AND cast(null as Nullable(Int64))) as Nullable(Bool)))), true)), (-2051538534, coalesce(((-1616816511 between 332225780 and -1883087387)) or ((-573375170 between -1427445977 and 615586748)), false)), (77089559, coalesce((NOT NOT(cast( ((true) and (true) != 925456787) as Nullable(Bool)))), false)), (1116921321, coalesce((0 is NULL), true));
+
+insert into t_vkx4cc (c_ylzjpt, c_hqfr9) values (-816935497, coalesce(false, false)), (1207796283, coalesce((-129 between -5 and -5), false)), (-1365934326, coalesce(true, false)), (-1618912877, coalesce((NOT NOT(cast( (false >= 31833) as Nullable(Bool)))), false));
+
+insert into t_vkx4cc (c_ylzjpt, c_hqfr9) values (-331834394, coalesce((NOT NOT(cast( (-63 <= -1822810052) as Nullable(Bool)))), true)), (-1020892864, coalesce((NOT NOT(cast( (40.31 <= 8146037365746019777) as Nullable(Bool)))), true)), (-1150980622, coalesce(((94019304 between -730556489 and 32)) and ((-956354236 is not NULL)), true)), (-1203382363, coalesce(true, true));
+
+insert into t_vkx4cc (c_ylzjpt, c_hqfr9) values (-653505826, coalesce((true) or (true), false)), (-1975508531, coalesce(((-796885845 between 65536 and cast(null as Nullable(Int32)))) or ((NOT NOT(cast( (-7467729336434250795 < 100.20) as Nullable(Bool))))), false)), (-1465484835, coalesce(((NOT NOT(cast( (19209 <= 75.96) as Nullable(Bool))))) or (true), false)), (1968095908, coalesce((NOT NOT(cast( (-1309960412156062327 > 13102) as Nullable(Bool)))), true));
+
+alter table t_vkx4cc add column c_zosphq2t1 Float64;
+
+insert into t_vkx4cc (c_ylzjpt, c_hqfr9, c_zosphq2t1) values (-153185515, coalesce((NOT NOT(cast( (1291639145 >= 30.22) as Nullable(Bool)))), false), -1.8), (-411038390, coalesce(((-762326135 between 16 and 177530758)) or (false), true), 26.34), (914629990, coalesce((-1125832977 is not NULL), true), 59.2), (541758331, coalesce(false, true), -255.1);
+
+insert into t_vkx4cc (c_ylzjpt, c_hqfr9, c_zosphq2t1) values (2125075305, coalesce(false, false), 55.36), (-1176267855, coalesce(true, true), 55.45), (1459407556, coalesce((true) and ((NOT NOT(cast( (95.96 != 65) as Nullable(Bool))))), true), 85.80), (-1098155311, coalesce(false, false), 2147483649.9);
+
+insert into t_vkx4cc (c_ylzjpt, c_hqfr9, c_zosphq2t1) values (6, coalesce((NOT NOT(cast( (1546334968 < -4) as Nullable(Bool)))), true), 57.42), (-5, coalesce((NOT NOT(cast( (59 AND 13) as Nullable(Bool)))), false), 65536.3), (100663045, coalesce((-1190355242 is not NULL), true), 73.80), (-451392958, coalesce((NOT NOT(cast( (false != -443845933) as Nullable(Bool)))), false), -4294967294.0);
+
+insert into t_vkx4cc (c_ylzjpt, c_hqfr9, c_zosphq2t1) values (561061873, coalesce(true, false), 12.17), (-526570556, coalesce(false, false), 64.73), (-1450619195, coalesce(true, true), 54.33), (-3, coalesce(true, true), 52.9);
+
+insert into t_vkx4cc (c_ylzjpt, c_hqfr9, c_zosphq2t1) values (-504713125, coalesce(false, true), 27.58), (897064234, coalesce((836516994 between cast(null as Nullable(Int32)) and -1832647080), true), 9223372036854775809.2), (65535, coalesce(true, true), 4294967297.5), (-599948807, coalesce((false) or ((NOT NOT(cast( (6.52 = 65.49) as Nullable(Bool))))), false), 256.5);
+
+insert into t_vkx4cc (c_ylzjpt, c_hqfr9, c_zosphq2t1) values (-1650266905, coalesce((NOT NOT(cast( (-83 = -218055084) as Nullable(Bool)))), true), 1.9), (-841067875, coalesce(false, true), -126.5), (15, coalesce(((NOT NOT(cast( (cast(null as Nullable(Decimal)) = cast(null as Nullable(Int32))) as Nullable(Bool))))) or (true), true), 33.65), (1913361922, coalesce((NOT NOT(cast( (false AND 0) as Nullable(Bool)))), false), 6.4);
+
+insert into t_vkx4cc (c_ylzjpt, c_hqfr9, c_zosphq2t1) values (1159852204, coalesce((-2057115045 is not NULL), false), 20.61), (-6, coalesce(true, true), 66.33), (-1154269118, coalesce(false, true), 8.89), (1258218855, coalesce(true, false), 19.80);
+
+insert into t_vkx4cc (c_ylzjpt, c_hqfr9, c_zosphq2t1) values (1603772265, coalesce(false, true), 57.87), (-176934810, coalesce(false, true), 128.8), (-1458338029, coalesce((NOT NOT(cast( (20908 != (NOT NOT(cast( (cast(null as Nullable(Decimal)) <= (true)         or ((NOT NOT(cast( (973511022 <= -112) as Nullable(Bool)))))) as Nullable(Bool))))) as Nullable(Bool)))), true), 76.54), (-262516786, coalesce((cast(null as Nullable(Int32)) is NULL), false), 21.49);
+
+insert into t_vkx4cc (c_ylzjpt, c_hqfr9, c_zosphq2t1) values (-1197430632, coalesce(true, false), 45.40), (-685902265, coalesce((NOT NOT(cast( (cast(null as Nullable(Decimal)) < cast(null as Nullable(Decimal))) as Nullable(Bool)))), true), 5.55), (1936334332, coalesce((-1565552735 is not NULL), false), 26.28), (2030467062, coalesce((NOT NOT(cast( (127.3 != cast(null as Nullable(Int32))) as Nullable(Bool)))), true), 89.50);
+
+insert into t_vkx4cc (c_ylzjpt, c_hqfr9, c_zosphq2t1) values (720985423, coalesce((NOT NOT(cast( (-451448940 = cast(null as Nullable(Decimal))) as Nullable(Bool)))), false), 52.65), (-222873194, coalesce(((-20 between -1419620477 and 1616455043)) or ((25624502 between 1312431316 and 1757361651)), false), 127.2), (745669725, coalesce((NOT NOT(cast( ((NOT NOT(cast( (cast(null as Nullable(UInt64)) <= 42) as Nullable(Bool)))) >= 3233811255032796928) as Nullable(Bool)))), false), 7.74), (-74234560, coalesce((NOT NOT(cast( (cast(null as Nullable(Decimal)) >= cast(null as Nullable(Decimal))) as Nullable(Bool)))), true), 19.25);
+
+SELECT DISTINCT
+  count(ref_0.c_zosphq2t1) over (partition by ref_0.c_hqfr9 order by ref_0.c_ylzjpt, ref_0.c_hqfr9, ref_0.c_zosphq2t1) as c0,
+  ref_0.c_ylzjpt as c1
+FROM
+  t_vkx4cc as ref_0
+  order by c0, c1;

From 4dea89df763fb9504cb681379d46830f4ec98db3 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Tue, 16 Jul 2024 13:50:58 +0000
Subject: [PATCH 0300/1488] Cleanup.

---
 src/Columns/ColumnSparse.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/Columns/ColumnSparse.cpp b/src/Columns/ColumnSparse.cpp
index 0922eb5ea2d..98a66e87387 100644
--- a/src/Columns/ColumnSparse.cpp
+++ b/src/Columns/ColumnSparse.cpp
@@ -4,7 +4,6 @@
 #include <Columns/ColumnConst.h>
 #include <Columns/ColumnsCommon.h>
 #include <Columns/ColumnTuple.h>
-#include <Columns/ColumnVector.h>
 #include <Common/HashTable/Hash.h>
 #include <Common/SipHash.h>
 #include <Common/WeakHash.h>

From f88e825b33273bc369be246b0dbbe7cfaa80855d Mon Sep 17 00:00:00 2001
From: Nikita Taranov <nikita.taranov@clickhouse.com>
Date: Tue, 16 Jul 2024 15:14:56 +0100
Subject: [PATCH 0301/1488] small fix

---
 src/Storages/MergeTree/MergeTreePrefetchedReadPool.cpp | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/Storages/MergeTree/MergeTreePrefetchedReadPool.cpp b/src/Storages/MergeTree/MergeTreePrefetchedReadPool.cpp
index 94213e2ba61..26595fbb36d 100644
--- a/src/Storages/MergeTree/MergeTreePrefetchedReadPool.cpp
+++ b/src/Storages/MergeTree/MergeTreePrefetchedReadPool.cpp
@@ -379,7 +379,6 @@ void MergeTreePrefetchedReadPool::fillPerThreadTasks(size_t threads, size_t sum_
     for (const auto & part : per_part_statistics)
         total_size_approx += part.sum_marks * part.approx_size_of_mark;
 
-    size_t min_prefetch_step_marks = pool_settings.min_marks_for_concurrent_read;
     for (size_t i = 0; i < per_part_infos.size(); ++i)
     {
         auto & part_stat = per_part_statistics[i];
@@ -411,11 +410,10 @@ void MergeTreePrefetchedReadPool::fillPerThreadTasks(size_t threads, size_t sum_
 
     LOG_DEBUG(
         log,
-        "Sum marks: {}, threads: {}, min_marks_per_thread: {}, min prefetch step marks: {}, prefetches limit: {}, total_size_approx: {}",
+        "Sum marks: {}, threads: {}, min_marks_per_thread: {}, prefetches limit: {}, total_size_approx: {}",
         sum_marks,
         threads,
         min_marks_per_thread,
-        min_prefetch_step_marks,
         settings.filesystem_prefetches_limit,
         total_size_approx);
 

From 288b0aaeb097c714e57d4cfd71a606b6942bd57c Mon Sep 17 00:00:00 2001
From: Nikita Taranov <nikita.taranov@clickhouse.com>
Date: Tue, 16 Jul 2024 15:16:48 +0100
Subject: [PATCH 0302/1488] fix build

---
 src/Storages/MergeTree/MergeTreeReadPoolBase.cpp | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/Storages/MergeTree/MergeTreeReadPoolBase.cpp b/src/Storages/MergeTree/MergeTreeReadPoolBase.cpp
index 79eaae14f59..0e713150625 100644
--- a/src/Storages/MergeTree/MergeTreeReadPoolBase.cpp
+++ b/src/Storages/MergeTree/MergeTreeReadPoolBase.cpp
@@ -1,7 +1,10 @@
-#include <cmath>
+#include <Storages/MergeTree/MergeTreeReadPoolBase.h>
+
+#include <Core/Settings.h>
 #include <Storages/MergeTree/LoadedMergeTreeDataPartInfoForReader.h>
 #include <Storages/MergeTree/MergeTreeBlockReadUtils.h>
-#include <Storages/MergeTree/MergeTreeReadPoolBase.h>
+
+#include <cmath>
 
 
 namespace DB

From 2af361eb14617d067732795424c22a6f511c9f18 Mon Sep 17 00:00:00 2001
From: Max K <maxkaynov@gmail.com>
Date: Tue, 16 Jul 2024 16:18:41 +0200
Subject: [PATCH 0303/1488] minor fix

---
 tests/ci/merge_pr.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/tests/ci/merge_pr.py b/tests/ci/merge_pr.py
index 4b57467ebdc..b005be472c7 100644
--- a/tests/ci/merge_pr.py
+++ b/tests/ci/merge_pr.py
@@ -256,7 +256,12 @@ def main():
         has_failed_statuses = False
         for status in statuses:
             print(f"Check status [{status.context}], [{status.state}]")
-            if not CI.is_required(status.context) and status.state != SUCCESS:
+            if (
+                not CI.is_required(status.context)
+                and status.context not in (CI.StatusNames.SYNC, CI.StatusNames.PR_CHECK)
+                and status.state != SUCCESS
+            ):
+                print(f"WARNING: Failed status [{status.context}], [{status.state}]")
                 has_failed_statuses = True
 
         if args.wf_status == SUCCESS or has_failed_statuses:

From e29d78e4cfbdd66bf4b5ed754f607267a57a9134 Mon Sep 17 00:00:00 2001
From: MikhailBurdukov <burdukvmikhail@gmail.com>
Date: Tue, 16 Jul 2024 14:43:08 +0000
Subject: [PATCH 0304/1488] Add option for validating the Primary key type in
 Dictionaries

---
 src/Core/Settings.h                           |  1 +
 src/Core/SettingsChangesHistory.cpp           |  1 +
 src/Databases/DatabaseDictionary.cpp          |  2 +-
 src/Dictionaries/DictionaryStructure.cpp      | 12 +-----
 src/Dictionaries/DictionaryStructure.h        | 10 +----
 .../getDictionaryConfigurationFromAST.cpp     |  9 ++++
 src/Storages/StorageDictionary.cpp            | 13 ++++--
 src/Storages/StorageDictionary.h              |  2 +-
 .../TableFunctionDictionary.cpp               |  2 +-
 ...ionary_validate_primary_key_type.reference |  4 ++
 ...8_dictionary_validate_primary_key_type.sql | 41 +++++++++++++++++++
 11 files changed, 71 insertions(+), 26 deletions(-)
 create mode 100644 tests/queries/0_stateless/03198_dictionary_validate_primary_key_type.reference
 create mode 100644 tests/queries/0_stateless/03198_dictionary_validate_primary_key_type.sql

diff --git a/src/Core/Settings.h b/src/Core/Settings.h
index d84e5b149f6..c0bc1489b31 100644
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@@ -148,6 +148,7 @@ class IColumn;
     M(UInt64, max_local_write_bandwidth, 0, "The maximum speed of local writes in bytes per second.", 0) \
     M(Bool, stream_like_engine_allow_direct_select, false, "Allow direct SELECT query for Kafka, RabbitMQ, FileLog, Redis Streams, and NATS engines. In case there are attached materialized views, SELECT query is not allowed even if this setting is enabled.", 0) \
     M(String, stream_like_engine_insert_queue, "", "When stream like engine reads from multiple queues, user will need to select one queue to insert into when writing. Used by Redis Streams and NATS.", 0) \
+    M(Bool, dictionary_validate_id_type, false, "Validate primary key type for dictionaries. By default id type for simple layouts will be implicitly converted to UInt64.", 0) \
     \
     M(Bool, distributed_insert_skip_read_only_replicas, false, "If true, INSERT into Distributed will skip read-only replicas.", 0) \
     M(Bool, distributed_foreground_insert, false, "If setting is enabled, insert query into distributed waits until data are sent to all nodes in a cluster. \n\nEnables or disables synchronous data insertion into a `Distributed` table.\n\nBy default, when inserting data into a Distributed table, the ClickHouse server sends data to cluster nodes in the background. When `distributed_foreground_insert` = 1, the data is processed synchronously, and the `INSERT` operation succeeds only after all the data is saved on all shards (at least one replica for each shard if `internal_replication` is true).", 0) ALIAS(insert_distributed_sync) \
diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp
index b0725340f46..0b3082529aa 100644
--- a/src/Core/SettingsChangesHistory.cpp
+++ b/src/Core/SettingsChangesHistory.cpp
@@ -59,6 +59,7 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
 {
     {"24.7", {{"output_format_parquet_write_page_index", false, true, "Add a possibility to write page index into parquet files."},
               {"optimize_trivial_insert_select", true, false, "The optimization does not make sense in many cases."},
+              {"dictionary_validate_id_type", false, false, "Validate primary key type for dictionaries. By default id type for simple layouts will be implicitly converted to UInt64."},
               }},
     {"24.6", {{"materialize_skip_indexes_on_insert", true, true, "Added new setting to allow to disable materialization of skip indexes on insert"},
               {"materialize_statistics_on_insert", true, true, "Added new setting to allow to disable materialization of statistics on insert"},
diff --git a/src/Databases/DatabaseDictionary.cpp b/src/Databases/DatabaseDictionary.cpp
index adb9a659fcd..6aece753e43 100644
--- a/src/Databases/DatabaseDictionary.cpp
+++ b/src/Databases/DatabaseDictionary.cpp
@@ -104,7 +104,7 @@ ASTPtr DatabaseDictionary::getCreateTableQueryImpl(const String & table_name, Co
             return {};
         }
 
-        auto names_and_types = StorageDictionary::getNamesAndTypes(ExternalDictionariesLoader::getDictionaryStructure(*load_result.config));
+        auto names_and_types = StorageDictionary::getNamesAndTypes(ExternalDictionariesLoader::getDictionaryStructure(*load_result.config), false);
         buffer << "CREATE TABLE " << backQuoteIfNeed(getDatabaseName()) << '.' << backQuoteIfNeed(table_name) << " (";
         buffer << names_and_types.toNamesAndTypesDescription();
         buffer << ") Engine = Dictionary(" << backQuoteIfNeed(table_name) << ")";
diff --git a/src/Dictionaries/DictionaryStructure.cpp b/src/Dictionaries/DictionaryStructure.cpp
index c2f2f4a8532..bd9ec57af1f 100644
--- a/src/Dictionaries/DictionaryStructure.cpp
+++ b/src/Dictionaries/DictionaryStructure.cpp
@@ -58,15 +58,6 @@ std::optional<AttributeUnderlyingType> tryGetAttributeUnderlyingType(TypeIndex i
 
 }
 
-
-DictionarySpecialAttribute::DictionarySpecialAttribute(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix)
-    : name{config.getString(config_prefix + ".name", "")}, expression{config.getString(config_prefix + ".expression", "")}
-{
-    if (name.empty() && !expression.empty())
-        throw Exception(ErrorCodes::BAD_ARGUMENTS, "Element {}.name is empty", config_prefix);
-}
-
-
 DictionaryStructure::DictionaryStructure(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix)
 {
     std::string structure_prefix = config_prefix + ".structure";
@@ -79,7 +70,8 @@ DictionaryStructure::DictionaryStructure(const Poco::Util::AbstractConfiguration
 
     if (has_id)
     {
-        id.emplace(config, structure_prefix + ".id");
+        static constexpr auto id_default_type = "UInt64";
+        id.emplace(makeDictionaryTypedSpecialAttribute(config, structure_prefix + ".id", id_default_type));
     }
     else if (has_key)
     {
diff --git a/src/Dictionaries/DictionaryStructure.h b/src/Dictionaries/DictionaryStructure.h
index 55060b1592f..0d44b696d74 100644
--- a/src/Dictionaries/DictionaryStructure.h
+++ b/src/Dictionaries/DictionaryStructure.h
@@ -89,14 +89,6 @@ constexpr void callOnDictionaryAttributeType(AttributeUnderlyingType type, F &&
     });
 }
 
-struct DictionarySpecialAttribute final
-{
-    const std::string name;
-    const std::string expression;
-
-    DictionarySpecialAttribute(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix);
-};
-
 struct DictionaryTypedSpecialAttribute final
 {
     const std::string name;
@@ -108,7 +100,7 @@ struct DictionaryTypedSpecialAttribute final
 /// Name of identifier plus list of attributes
 struct DictionaryStructure final
 {
-    std::optional<DictionarySpecialAttribute> id;
+    std::optional<DictionaryTypedSpecialAttribute> id;
     std::optional<std::vector<DictionaryAttribute>> key;
     std::vector<DictionaryAttribute> attributes;
     std::unordered_map<std::string, size_t> attribute_name_to_index;
diff --git a/src/Dictionaries/getDictionaryConfigurationFromAST.cpp b/src/Dictionaries/getDictionaryConfigurationFromAST.cpp
index 9ee2027afc7..4ec2e1f5260 100644
--- a/src/Dictionaries/getDictionaryConfigurationFromAST.cpp
+++ b/src/Dictionaries/getDictionaryConfigurationFromAST.cpp
@@ -382,6 +382,15 @@ void buildPrimaryKeyConfiguration(
         name_element->appendChild(name);
 
         buildAttributeExpressionIfNeeded(doc, id_element, dict_attr);
+
+        if (!dict_attr->type)
+            return;
+
+        AutoPtr<Element> type_element(doc->createElement("type"));
+        id_element->appendChild(type_element);
+
+        AutoPtr<Text> type(doc->createTextNode(queryToString(dict_attr->type)));
+        type_element->appendChild(type);
     }
     else
     {
diff --git a/src/Storages/StorageDictionary.cpp b/src/Storages/StorageDictionary.cpp
index 447fd87cdc9..69af73122ba 100644
--- a/src/Storages/StorageDictionary.cpp
+++ b/src/Storages/StorageDictionary.cpp
@@ -25,13 +25,14 @@ namespace ErrorCodes
     extern const int CANNOT_DETACH_DICTIONARY_AS_TABLE;
     extern const int DICTIONARY_ALREADY_EXISTS;
     extern const int NOT_IMPLEMENTED;
+    extern const int BAD_ARGUMENTS;
 }
 
 namespace
 {
     void checkNamesAndTypesCompatibleWithDictionary(const String & dictionary_name, const ColumnsDescription & columns, const DictionaryStructure & dictionary_structure)
     {
-        auto dictionary_names_and_types = StorageDictionary::getNamesAndTypes(dictionary_structure);
+        auto dictionary_names_and_types = StorageDictionary::getNamesAndTypes(dictionary_structure, false);
         std::set<NameAndTypePair> names_and_types_set(dictionary_names_and_types.begin(), dictionary_names_and_types.end());
 
         for (const auto & column : columns.getOrdinary())
@@ -47,13 +48,17 @@ namespace
 }
 
 
-NamesAndTypesList StorageDictionary::getNamesAndTypes(const DictionaryStructure & dictionary_structure)
+NamesAndTypesList StorageDictionary::getNamesAndTypes(const DictionaryStructure & dictionary_structure, bool validate_id_type)
 {
     NamesAndTypesList dictionary_names_and_types;
 
     if (dictionary_structure.id)
-        dictionary_names_and_types.emplace_back(dictionary_structure.id->name, std::make_shared<DataTypeUInt64>());
+    {
+        if (validate_id_type && dictionary_structure.id->type->getTypeId() != TypeIndex::UInt64)
+            throw Exception(ErrorCodes::BAD_ARGUMENTS, "Incorrect type of ID column: must be UInt64, but it is {}", dictionary_structure.id->type->getFamilyName());
 
+        dictionary_names_and_types.emplace_back(dictionary_structure.id->name, std::make_shared<DataTypeUInt64>());
+    }
     /// In old-style (XML) configuration we don't have this attributes in the
     /// main attribute list, so we have to add them to columns list explicitly.
     /// In the new configuration (DDL) we have them both in range_* nodes and
@@ -105,7 +110,7 @@ StorageDictionary::StorageDictionary(
     Location location_,
     ContextPtr context_)
     : StorageDictionary(
-        table_id_, dictionary_name_, ColumnsDescription{getNamesAndTypes(dictionary_structure_)}, comment, location_, context_)
+        table_id_, dictionary_name_, ColumnsDescription{getNamesAndTypes(dictionary_structure_, context_->getSettingsRef().dictionary_validate_id_type)}, comment, location_, context_)
 {
 }
 
diff --git a/src/Storages/StorageDictionary.h b/src/Storages/StorageDictionary.h
index 17e4efda2cd..44a274cf97c 100644
--- a/src/Storages/StorageDictionary.h
+++ b/src/Storages/StorageDictionary.h
@@ -80,7 +80,7 @@ public:
 
     std::shared_ptr<const IDictionary> getDictionary() const;
 
-    static NamesAndTypesList getNamesAndTypes(const DictionaryStructure & dictionary_structure);
+    static NamesAndTypesList getNamesAndTypes(const DictionaryStructure & dictionary_structure, bool validate_id_type);
 
     bool isDictionary() const override { return true; }
     void shutdown(bool is_drop) override;
diff --git a/src/TableFunctions/TableFunctionDictionary.cpp b/src/TableFunctions/TableFunctionDictionary.cpp
index 867fbf5b11e..9e0491975ae 100644
--- a/src/TableFunctions/TableFunctionDictionary.cpp
+++ b/src/TableFunctions/TableFunctionDictionary.cpp
@@ -74,7 +74,7 @@ ColumnsDescription TableFunctionDictionary::getActualTableStructure(ContextPtr c
 
     /// otherwise, we get table structure by dictionary structure.
     auto dictionary_structure = external_loader.getDictionaryStructure(dictionary_name, context);
-    return ColumnsDescription(StorageDictionary::getNamesAndTypes(dictionary_structure));
+    return ColumnsDescription(StorageDictionary::getNamesAndTypes(dictionary_structure, false));
 }
 
 StoragePtr TableFunctionDictionary::executeImpl(
diff --git a/tests/queries/0_stateless/03198_dictionary_validate_primary_key_type.reference b/tests/queries/0_stateless/03198_dictionary_validate_primary_key_type.reference
new file mode 100644
index 00000000000..c7f2781aed9
--- /dev/null
+++ b/tests/queries/0_stateless/03198_dictionary_validate_primary_key_type.reference
@@ -0,0 +1,4 @@
+n1	UInt64					
+n2	UInt32					
+n1	UInt64					
+n2	UInt32					
diff --git a/tests/queries/0_stateless/03198_dictionary_validate_primary_key_type.sql b/tests/queries/0_stateless/03198_dictionary_validate_primary_key_type.sql
new file mode 100644
index 00000000000..eaf51ef7ead
--- /dev/null
+++ b/tests/queries/0_stateless/03198_dictionary_validate_primary_key_type.sql
@@ -0,0 +1,41 @@
+CREATE DICTIONARY `test_dictionary0` (
+    `n1` String,
+    `n2` UInt32
+)
+PRIMARY KEY n1
+SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 DB 'test_db' TABLE 'table_01' USER 'default'))
+LIFETIME(MIN 1 MAX 10)
+LAYOUT(FLAT());
+
+SET dictionary_validate_id_type=1;
+
+CREATE DICTIONARY `test_dictionary1` (
+    `n1` String,
+    `n2` UInt32
+)
+PRIMARY KEY n1
+SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 DB 'test_db' TABLE 'table_01' USER 'default'))
+LIFETIME(MIN 1 MAX 10)
+LAYOUT(FLAT());  -- { serverError 36 }
+
+CREATE DICTIONARY `test_dictionary2` (
+    `n1` UInt32,
+    `n2` UInt32
+)
+PRIMARY KEY n1
+SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 DB 'test_db' TABLE 'table_01' USER 'default'))
+LIFETIME(MIN 1 MAX 10)
+LAYOUT(FLAT()); -- { serverError 36 }
+
+CREATE DICTIONARY `test_dictionary3` (
+    `n1` UInt64,
+    `n2` UInt32
+)
+PRIMARY KEY n1
+SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 DB 'test_db' TABLE 'table_01' USER 'default'))
+LIFETIME(MIN 1 MAX 10)
+LAYOUT(FLAT());
+
+DESCRIBE `test_dictionary0`;
+DESCRIBE `test_dictionary3`;
+

From 7ea3324776bd4cb8cc886822a9b30d3dfcaff5a2 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Tue, 16 Jul 2024 16:31:34 +0000
Subject: [PATCH 0305/1488] Refactor in VirtualColumnUtils

---
 src/Interpreters/ActionsDAG.cpp               | 12 ++++-----
 src/Interpreters/ActionsDAG.h                 |  3 +--
 .../useDataParallelAggregation.cpp            |  8 +++---
 src/Processors/QueryPlan/SortingStep.cpp      | 10 +++++++
 src/Storages/MergeTree/MergeTreeData.cpp      |  4 +--
 .../MergeTree/MergeTreeDataSelectExecutor.cpp |  4 +--
 .../StorageObjectStorageSource.cpp            | 15 ++++++++---
 .../StorageObjectStorageSource.h              |  2 +-
 src/Storages/StorageFile.cpp                  |  8 ++++--
 src/Storages/StorageURL.cpp                   |  6 +++--
 .../System/StorageSystemDetachedParts.cpp     |  4 +--
 .../StorageSystemDroppedTablesParts.cpp       |  4 +--
 .../System/StorageSystemDroppedTablesParts.h  |  6 ++---
 .../System/StorageSystemPartsBase.cpp         | 12 ++++-----
 src/Storages/System/StorageSystemPartsBase.h  |  6 ++---
 src/Storages/System/StorageSystemTables.cpp   |  2 +-
 src/Storages/VirtualColumnUtils.cpp           | 27 +++++++++++--------
 src/Storages/VirtualColumnUtils.h             | 13 ++++-----
 18 files changed, 87 insertions(+), 59 deletions(-)

diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp
index 4f03a9e1602..e001406408f 100644
--- a/src/Interpreters/ActionsDAG.cpp
+++ b/src/Interpreters/ActionsDAG.cpp
@@ -624,9 +624,9 @@ void ActionsDAG::removeAliasesForFilter(const std::string & filter_name)
     }
 }
 
-ActionsDAGPtr ActionsDAG::cloneSubDAG(const NodeRawConstPtrs & outputs, bool remove_aliases)
+ActionsDAG ActionsDAG::cloneSubDAG(const NodeRawConstPtrs & outputs, bool remove_aliases)
 {
-    auto actions = std::make_unique<ActionsDAG>();
+    ActionsDAG actions;
     std::unordered_map<const Node *, Node *> copy_map;
 
     struct Frame
@@ -661,21 +661,21 @@ ActionsDAGPtr ActionsDAG::cloneSubDAG(const NodeRawConstPtrs & outputs, bool rem
             if (remove_aliases && frame.node->type == ActionType::ALIAS)
                 copy_node = copy_map[frame.node->children.front()];
             else
-                copy_node = &actions->nodes.emplace_back(*frame.node);
+                copy_node = &actions.nodes.emplace_back(*frame.node);
 
             if (frame.node->type == ActionType::INPUT)
-                actions->inputs.push_back(copy_node);
+                actions.inputs.push_back(copy_node);
 
             stack.pop();
         }
     }
 
-    for (auto & node : actions->nodes)
+    for (auto & node : actions.nodes)
         for (auto & child : node.children)
             child = copy_map[child];
 
     for (const auto * output : outputs)
-        actions->outputs.push_back(copy_map[output]);
+        actions.outputs.push_back(copy_map[output]);
 
     return actions;
 }
diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h
index 05948ccf928..6f5c3d3b0df 100644
--- a/src/Interpreters/ActionsDAG.h
+++ b/src/Interpreters/ActionsDAG.h
@@ -261,13 +261,12 @@ public:
     void compileExpressions(size_t min_count_to_compile_expression, const std::unordered_set<const Node *> & lazy_executed_nodes = {});
 #endif
 
-    static ActionsDAGPtr clone(const ActionsDAGPtr & from) { return clone(from.get()); }
     static ActionsDAGPtr clone(const ActionsDAG * from);
 
     ActionsDAG clone(std::unordered_map<const Node *, Node *> & old_to_new_nodes) const;
     ActionsDAG clone() const;
 
-    static ActionsDAGPtr cloneSubDAG(const NodeRawConstPtrs & outputs, bool remove_aliases);
+    static ActionsDAG cloneSubDAG(const NodeRawConstPtrs & outputs, bool remove_aliases);
 
     /// Execute actions for header. Input block must have empty columns.
     /// Result should be equal to the execution of ExpressionActions built from this DAG.
diff --git a/src/Processors/QueryPlan/Optimizations/useDataParallelAggregation.cpp b/src/Processors/QueryPlan/Optimizations/useDataParallelAggregation.cpp
index 7e0260c0040..0eeaec9bde7 100644
--- a/src/Processors/QueryPlan/Optimizations/useDataParallelAggregation.cpp
+++ b/src/Processors/QueryPlan/Optimizations/useDataParallelAggregation.cpp
@@ -74,11 +74,11 @@ void removeInjectiveFunctionsFromResultsRecursively(const ActionsDAG::Node * nod
 /// Our objective is to replace injective function nodes in `actions` results with its children
 /// until only the irreducible subset of nodes remains. Against these set of nodes we will match partition key expression
 /// to determine if it maps all rows with the same value of group by key to the same partition.
-NodeSet removeInjectiveFunctionsFromResultsRecursively(const ActionsDAGPtr & actions)
+NodeSet removeInjectiveFunctionsFromResultsRecursively(const ActionsDAG & actions)
 {
     NodeSet irreducible;
     NodeSet visited;
-    for (const auto & node : actions->getOutputs())
+    for (const auto & node : actions.getOutputs())
         removeInjectiveFunctionsFromResultsRecursively(node, irreducible, visited);
     return irreducible;
 }
@@ -158,7 +158,7 @@ bool isPartitionKeySuitsGroupByKey(
     auto key_nodes = group_by_actions.findInOutpus(aggregating.getParams().keys);
     auto group_by_key_actions = ActionsDAG::cloneSubDAG(key_nodes, /*remove_aliases=*/ true);
 
-    const auto & gb_key_required_columns = group_by_key_actions->getRequiredColumnsNames();
+    const auto & gb_key_required_columns = group_by_key_actions.getRequiredColumnsNames();
 
     const auto & partition_actions = reading.getStorageMetadata()->getPartitionKey().expression->getActionsDAG();
 
@@ -169,7 +169,7 @@ bool isPartitionKeySuitsGroupByKey(
 
     const auto irreducibe_nodes = removeInjectiveFunctionsFromResultsRecursively(group_by_key_actions);
 
-    const auto matches = matchTrees(group_by_key_actions->getOutputs(), partition_actions);
+    const auto matches = matchTrees(group_by_key_actions.getOutputs(), partition_actions);
 
     return allOutputsDependsOnlyOnAllowedNodes(partition_actions, irreducibe_nodes, matches);
 }
diff --git a/src/Processors/QueryPlan/SortingStep.cpp b/src/Processors/QueryPlan/SortingStep.cpp
index 48fad9f5fdb..e8e761e7ab0 100644
--- a/src/Processors/QueryPlan/SortingStep.cpp
+++ b/src/Processors/QueryPlan/SortingStep.cpp
@@ -447,6 +447,13 @@ void SortingStep::describeActions(FormatSettings & settings) const
         settings.out << '\n';
     }
 
+    if (!partition_by_description.empty())
+    {
+        settings.out << prefix << "Partition by description: ";
+        dumpSortDescription(partition_by_description, settings.out);
+        settings.out << '\n';
+    }
+
     if (limit)
         settings.out << prefix << "Limit " << limit << '\n';
 }
@@ -461,6 +468,9 @@ void SortingStep::describeActions(JSONBuilder::JSONMap & map) const
     else
         map.add("Sort Description", explainSortDescription(result_description));
 
+    if (!partition_by_description.empty())
+         map.add("Partition By Description", explainSortDescription(partition_by_description));
+
     if (limit)
         map.add("Limit", limit);
 }
diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index 878e0420665..9aa9490198a 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -1154,7 +1154,7 @@ std::optional<UInt64> MergeTreeData::totalRowsByPartitionPredicateImpl(
         if (!virtual_columns_block.has(input->result_name))
             valid = false;
 
-    PartitionPruner partition_pruner(metadata_snapshot, filter_dag.get(), local_context, true /* strict */);
+    PartitionPruner partition_pruner(metadata_snapshot, &*filter_dag, local_context, true /* strict */);
     if (partition_pruner.isUseless() && !valid)
         return {};
 
@@ -1162,7 +1162,7 @@ std::optional<UInt64> MergeTreeData::totalRowsByPartitionPredicateImpl(
     if (valid)
     {
         virtual_columns_block = getBlockWithVirtualsForFilter(metadata_snapshot, parts);
-        VirtualColumnUtils::filterBlockWithDAG(filter_dag, virtual_columns_block, local_context);
+        VirtualColumnUtils::filterBlockWithDAG(std::move(*filter_dag), virtual_columns_block, local_context);
         part_values = VirtualColumnUtils::extractSingleValueFromBlock<String>(virtual_columns_block, "_part");
         if (part_values.empty())
             return 0;
diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp
index a6a40a808e5..a37dbfa554c 100644
--- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp
+++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp
@@ -465,7 +465,7 @@ void MergeTreeDataSelectExecutor::buildKeyConditionFromPartOffset(
         return;
 
     part_offset_condition.emplace(KeyCondition{
-        dag.get(),
+        &*dag,
         context,
         sample.getNames(),
         std::make_shared<ExpressionActions>(ActionsDAG(sample.getColumnsWithTypeAndName()), ExpressionActionsSettings{}),
@@ -488,7 +488,7 @@ std::optional<std::unordered_set<String>> MergeTreeDataSelectExecutor::filterPar
         return {};
 
     auto virtual_columns_block = data.getBlockWithVirtualsForFilter(metadata_snapshot, parts);
-    VirtualColumnUtils::filterBlockWithDAG(dag, virtual_columns_block, context);
+    VirtualColumnUtils::filterBlockWithDAG(std::move(*dag), virtual_columns_block, context);
     return VirtualColumnUtils::extractSingleValueFromBlock<String>(virtual_columns_block, "_part");
 }
 
diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp
index c86b56d3f1b..e760098f10f 100644
--- a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp
+++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp
@@ -139,7 +139,10 @@ std::shared_ptr<StorageObjectStorageSource::IIterator> StorageObjectStorageSourc
             paths.reserve(keys.size());
             for (const auto & key : keys)
                 paths.push_back(fs::path(configuration->getNamespace()) / key);
-            VirtualColumnUtils::filterByPathOrFile(keys, paths, filter_dag, virtual_columns, local_context);
+
+            VirtualColumnUtils::buildSetsForDAG(*filter_dag, local_context);
+            auto actions = std::make_shared<ExpressionActions>(std::move(*filter_dag));
+            VirtualColumnUtils::filterByPathOrFile(keys, paths, actions, virtual_columns);
             copy_configuration->setPaths(keys);
         }
 
@@ -506,7 +509,11 @@ StorageObjectStorageSource::GlobIterator::GlobIterator(
         }
 
         recursive = key_with_globs == "/**";
-        filter_dag = VirtualColumnUtils::createPathAndFileFilterDAG(predicate, virtual_columns);
+        if (auto filter_dag = VirtualColumnUtils::createPathAndFileFilterDAG(predicate, virtual_columns))
+        {
+            VirtualColumnUtils::buildSetsForDAG(*filter_dag, getContext());
+            filter_expr = std::make_shared<ExpressionActions>(std::move(*filter_dag));
+        }
     }
     else
     {
@@ -570,14 +577,14 @@ StorageObjectStorage::ObjectInfoPtr StorageObjectStorageSource::GlobIterator::ne
                     ++it;
             }
 
-            if (filter_dag)
+            if (filter_expr)
             {
                 std::vector<String> paths;
                 paths.reserve(new_batch.size());
                 for (const auto & object_info : new_batch)
                     paths.push_back(getUniqueStoragePathIdentifier(*configuration, *object_info, false));
 
-                VirtualColumnUtils::filterByPathOrFile(new_batch, paths, filter_dag, virtual_columns, getContext());
+                VirtualColumnUtils::filterByPathOrFile(new_batch, paths, filter_expr, virtual_columns);
 
                 LOG_TEST(logger, "Filtered files: {} -> {}", paths.size(), new_batch.size());
             }
diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.h b/src/Storages/ObjectStorage/StorageObjectStorageSource.h
index b8418ddd07c..e466621e1e1 100644
--- a/src/Storages/ObjectStorage/StorageObjectStorageSource.h
+++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.h
@@ -208,7 +208,7 @@ private:
 
     ObjectInfos object_infos;
     ObjectInfos * read_keys;
-    ActionsDAGPtr filter_dag;
+    ExpressionActionsPtr filter_expr;
     ObjectStorageIteratorPtr object_storage_iterator;
     bool recursive{false};
     std::vector<String> expanded_keys;
diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp
index c6acb358d89..fe6f494db00 100644
--- a/src/Storages/StorageFile.cpp
+++ b/src/Storages/StorageFile.cpp
@@ -1130,12 +1130,16 @@ StorageFileSource::FilesIterator::FilesIterator(
     bool distributed_processing_)
     : WithContext(context_), files(files_), archive_info(std::move(archive_info_)), distributed_processing(distributed_processing_)
 {
-    ActionsDAGPtr filter_dag;
+    std::optional<ActionsDAG> filter_dag;
     if (!distributed_processing && !archive_info && !files.empty())
         filter_dag = VirtualColumnUtils::createPathAndFileFilterDAG(predicate, virtual_columns);
 
     if (filter_dag)
-        VirtualColumnUtils::filterByPathOrFile(files, files, filter_dag, virtual_columns, context_);
+    {
+        VirtualColumnUtils::buildSetsForDAG(*filter_dag, context_);
+        auto actions = std::make_shared<ExpressionActions>(std::move(*filter_dag));
+        VirtualColumnUtils::filterByPathOrFile(files, files, actions, virtual_columns);
+    }
 }
 
 String StorageFileSource::FilesIterator::next()
diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp
index 9cec8c75ebe..c61bb8ac980 100644
--- a/src/Storages/StorageURL.cpp
+++ b/src/Storages/StorageURL.cpp
@@ -198,7 +198,7 @@ public:
     {
         uris = parseRemoteDescription(uri_, 0, uri_.size(), ',', max_addresses);
 
-        ActionsDAGPtr filter_dag;
+        std::optional<ActionsDAG> filter_dag;
         if (!uris.empty())
             filter_dag = VirtualColumnUtils::createPathAndFileFilterDAG(predicate, virtual_columns);
 
@@ -209,7 +209,9 @@ public:
             for (const auto & uri : uris)
                 paths.push_back(Poco::URI(uri).getPath());
 
-            VirtualColumnUtils::filterByPathOrFile(uris, paths, filter_dag, virtual_columns, context);
+            VirtualColumnUtils::buildSetsForDAG(*filter_dag, context);
+            auto actions = std::make_shared<ExpressionActions>(std::move(*filter_dag));
+            VirtualColumnUtils::filterByPathOrFile(uris, paths, actions, virtual_columns);
         }
     }
 
diff --git a/src/Storages/System/StorageSystemDetachedParts.cpp b/src/Storages/System/StorageSystemDetachedParts.cpp
index 7e4c1de1c65..0d0ae666c10 100644
--- a/src/Storages/System/StorageSystemDetachedParts.cpp
+++ b/src/Storages/System/StorageSystemDetachedParts.cpp
@@ -307,7 +307,7 @@ protected:
     std::shared_ptr<StorageSystemDetachedParts> storage;
     std::vector<UInt8> columns_mask;
 
-    ActionsDAGPtr filter;
+    std::optional<ActionsDAG> filter;
     const size_t max_block_size;
     const size_t num_streams;
 };
@@ -359,7 +359,7 @@ void StorageSystemDetachedParts::read(
 
 void ReadFromSystemDetachedParts::initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &)
 {
-    auto state = std::make_shared<SourceState>(StoragesInfoStream(nullptr, filter, context));
+    auto state = std::make_shared<SourceState>(StoragesInfoStream({}, std::move(filter), context));
 
     Pipe pipe;
 
diff --git a/src/Storages/System/StorageSystemDroppedTablesParts.cpp b/src/Storages/System/StorageSystemDroppedTablesParts.cpp
index c17d6402d88..defc4ec2d2a 100644
--- a/src/Storages/System/StorageSystemDroppedTablesParts.cpp
+++ b/src/Storages/System/StorageSystemDroppedTablesParts.cpp
@@ -11,7 +11,7 @@ namespace DB
 {
 
 
-StoragesDroppedInfoStream::StoragesDroppedInfoStream(const ActionsDAGPtr & filter, ContextPtr context)
+StoragesDroppedInfoStream::StoragesDroppedInfoStream(std::optional<ActionsDAG> filter, ContextPtr context)
         : StoragesInfoStreamBase(context)
 {
     /// Will apply WHERE to subset of columns and then add more columns.
@@ -75,7 +75,7 @@ StoragesDroppedInfoStream::StoragesDroppedInfoStream(const ActionsDAGPtr & filte
     {
         /// Filter block_to_filter with columns 'database', 'table', 'engine', 'active'.
         if (filter)
-            VirtualColumnUtils::filterBlockWithDAG(filter, block_to_filter, context);
+            VirtualColumnUtils::filterBlockWithDAG(std::move(*filter), block_to_filter, context);
         rows = block_to_filter.rows();
     }
 
diff --git a/src/Storages/System/StorageSystemDroppedTablesParts.h b/src/Storages/System/StorageSystemDroppedTablesParts.h
index dff9e41cce3..32468fc31b2 100644
--- a/src/Storages/System/StorageSystemDroppedTablesParts.h
+++ b/src/Storages/System/StorageSystemDroppedTablesParts.h
@@ -9,7 +9,7 @@ namespace DB
 class StoragesDroppedInfoStream : public StoragesInfoStreamBase
 {
 public:
-    StoragesDroppedInfoStream(const ActionsDAGPtr & filter, ContextPtr context);
+    StoragesDroppedInfoStream(std::optional<ActionsDAG> filter, ContextPtr context);
 protected:
     bool tryLockTable(StoragesInfo &) override
     {
@@ -30,9 +30,9 @@ public:
 
     std::string getName() const override { return "SystemDroppedTablesParts"; }
 protected:
-    std::unique_ptr<StoragesInfoStreamBase> getStoragesInfoStream(const ActionsDAGPtr &, const ActionsDAGPtr & filter, ContextPtr context) override
+    std::unique_ptr<StoragesInfoStreamBase> getStoragesInfoStream(std::optional<ActionsDAG>, std::optional<ActionsDAG> filter, ContextPtr context) override
     {
-        return std::make_unique<StoragesDroppedInfoStream>(filter, context);
+        return std::make_unique<StoragesDroppedInfoStream>(std::move(filter), context);
     }
 };
 
diff --git a/src/Storages/System/StorageSystemPartsBase.cpp b/src/Storages/System/StorageSystemPartsBase.cpp
index f7d1c1b3eb8..a0c9a5c61bd 100644
--- a/src/Storages/System/StorageSystemPartsBase.cpp
+++ b/src/Storages/System/StorageSystemPartsBase.cpp
@@ -91,7 +91,7 @@ StoragesInfo::getProjectionParts(MergeTreeData::DataPartStateVector & state, boo
     return data->getProjectionPartsVectorForInternalUsage({State::Active}, &state);
 }
 
-StoragesInfoStream::StoragesInfoStream(const ActionsDAGPtr & filter_by_database, const ActionsDAGPtr & filter_by_other_columns, ContextPtr context)
+StoragesInfoStream::StoragesInfoStream(std::optional<ActionsDAG> filter_by_database, std::optional<ActionsDAG> filter_by_other_columns, ContextPtr context)
     : StoragesInfoStreamBase(context)
 {
     /// Will apply WHERE to subset of columns and then add more columns.
@@ -124,7 +124,7 @@ StoragesInfoStream::StoragesInfoStream(const ActionsDAGPtr & filter_by_database,
 
         /// Filter block_to_filter with column 'database'.
         if (filter_by_database)
-            VirtualColumnUtils::filterBlockWithDAG(filter_by_database, block_to_filter, context);
+            VirtualColumnUtils::filterBlockWithDAG(std::move(*filter_by_database), block_to_filter, context);
         rows = block_to_filter.rows();
 
         /// Block contains new columns, update database_column.
@@ -204,7 +204,7 @@ StoragesInfoStream::StoragesInfoStream(const ActionsDAGPtr & filter_by_database,
     {
         /// Filter block_to_filter with columns 'database', 'table', 'engine', 'active'.
         if (filter_by_other_columns)
-            VirtualColumnUtils::filterBlockWithDAG(filter_by_other_columns, block_to_filter, context);
+            VirtualColumnUtils::filterBlockWithDAG(std::move(*filter_by_other_columns), block_to_filter, context);
         rows = block_to_filter.rows();
     }
 
@@ -236,8 +236,8 @@ protected:
     std::shared_ptr<StorageSystemPartsBase> storage;
     std::vector<UInt8> columns_mask;
     const bool has_state_column;
-    ActionsDAGPtr filter_by_database;
-    ActionsDAGPtr filter_by_other_columns;
+    std::optional<ActionsDAG> filter_by_database;
+    std::optional<ActionsDAG> filter_by_other_columns;
 };
 
 ReadFromSystemPartsBase::ReadFromSystemPartsBase(
@@ -318,7 +318,7 @@ void StorageSystemPartsBase::read(
 
 void ReadFromSystemPartsBase::initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &)
 {
-    auto stream = storage->getStoragesInfoStream(filter_by_database, filter_by_other_columns, context);
+    auto stream = storage->getStoragesInfoStream(std::move(filter_by_database), std::move(filter_by_other_columns), context);
     auto header = getOutputStream().header;
 
     MutableColumns res_columns = header.cloneEmptyColumns();
diff --git a/src/Storages/System/StorageSystemPartsBase.h b/src/Storages/System/StorageSystemPartsBase.h
index 8671fd850f8..806af4a7bf8 100644
--- a/src/Storages/System/StorageSystemPartsBase.h
+++ b/src/Storages/System/StorageSystemPartsBase.h
@@ -116,7 +116,7 @@ protected:
 class StoragesInfoStream : public StoragesInfoStreamBase
 {
 public:
-    StoragesInfoStream(const ActionsDAGPtr & filter_by_database, const ActionsDAGPtr & filter_by_other_columns, ContextPtr context);
+    StoragesInfoStream(std::optional<ActionsDAG> filter_by_database, std::optional<ActionsDAG> filter_by_other_columns, ContextPtr context);
 };
 
 /** Implements system table 'parts' which allows to get information about data parts for tables of MergeTree family.
@@ -146,9 +146,9 @@ protected:
 
     StorageSystemPartsBase(const StorageID & table_id_, ColumnsDescription && columns);
 
-    virtual std::unique_ptr<StoragesInfoStreamBase> getStoragesInfoStream(const ActionsDAGPtr & filter_by_database, const ActionsDAGPtr & filter_by_other_columns, ContextPtr context)
+    virtual std::unique_ptr<StoragesInfoStreamBase> getStoragesInfoStream(std::optional<ActionsDAG> filter_by_database, std::optional<ActionsDAG> filter_by_other_columns, ContextPtr context)
     {
-        return std::make_unique<StoragesInfoStream>(filter_by_database, filter_by_other_columns, context);
+        return std::make_unique<StoragesInfoStream>(std::move(filter_by_database), std::move(filter_by_other_columns), context);
     }
 
     virtual void
diff --git a/src/Storages/System/StorageSystemTables.cpp b/src/Storages/System/StorageSystemTables.cpp
index 783b899c978..85aaf4ad186 100644
--- a/src/Storages/System/StorageSystemTables.cpp
+++ b/src/Storages/System/StorageSystemTables.cpp
@@ -169,7 +169,7 @@ ColumnPtr getFilteredTables(const ActionsDAG::Node * predicate, const ColumnPtr
         block.insert(ColumnWithTypeAndName(std::move(engine_column), std::make_shared<DataTypeString>(), "engine"));
 
     if (dag)
-        VirtualColumnUtils::filterBlockWithDAG(dag, block, context);
+        VirtualColumnUtils::filterBlockWithDAG(std::move(*dag), block, context);
 
     return block.getByPosition(0).column;
 }
diff --git a/src/Storages/VirtualColumnUtils.cpp b/src/Storages/VirtualColumnUtils.cpp
index 1630d9fd9c4..32c6a558340 100644
--- a/src/Storages/VirtualColumnUtils.cpp
+++ b/src/Storages/VirtualColumnUtils.cpp
@@ -77,15 +77,20 @@ void buildSetsForDAG(const ActionsDAG & dag, const ContextPtr & context)
     }
 }
 
-void filterBlockWithDAG(const ActionsDAGPtr & dag, Block & block, ContextPtr context)
+void filterBlockWithDAG(ActionsDAG dag, Block & block, ContextPtr context)
+{
+    buildSetsForDAG(dag, context);
+    auto actions = std::make_shared<ExpressionActions>(std::move(dag));
+    filterBlockWithExpression(actions, block);
+}
+
+void filterBlockWithExpression(const ExpressionActionsPtr & actions, Block & block)
 {
-    buildSetsForDAG(*dag, context);
-    auto actions = std::make_shared<ExpressionActions>(std::move(*ActionsDAG::clone(dag)));
     Block block_with_filter = block;
     actions->execute(block_with_filter, /*dry_run=*/ false, /*allow_duplicates_in_input=*/ true);
 
     /// Filter the block.
-    String filter_column_name = dag->getOutputs().at(0)->result_name;
+    String filter_column_name = actions->getActionsDAG().getOutputs().at(0)->result_name;
     ColumnPtr filter_column = block_with_filter.getByName(filter_column_name).column->convertToFullColumnIfConst();
 
     ConstantFilterDescription constant_filter(*filter_column);
@@ -155,7 +160,7 @@ static void addPathAndFileToVirtualColumns(Block & block, const String & path, s
     block.getByName("_idx").column->assumeMutableRef().insert(idx);
 }
 
-ActionsDAGPtr createPathAndFileFilterDAG(const ActionsDAG::Node * predicate, const NamesAndTypesList & virtual_columns)
+std::optional<ActionsDAG> createPathAndFileFilterDAG(const ActionsDAG::Node * predicate, const NamesAndTypesList & virtual_columns)
 {
     if (!predicate || virtual_columns.empty())
         return {};
@@ -171,7 +176,7 @@ ActionsDAGPtr createPathAndFileFilterDAG(const ActionsDAG::Node * predicate, con
     return splitFilterDagForAllowedInputs(predicate, &block);
 }
 
-ColumnPtr getFilterByPathAndFileIndexes(const std::vector<String> & paths, const ActionsDAGPtr & dag, const NamesAndTypesList & virtual_columns, const ContextPtr & context)
+ColumnPtr getFilterByPathAndFileIndexes(const std::vector<String> & paths, const ExpressionActionsPtr & actions, const NamesAndTypesList & virtual_columns)
 {
     Block block;
     for (const auto & column : virtual_columns)
@@ -184,7 +189,7 @@ ColumnPtr getFilterByPathAndFileIndexes(const std::vector<String> & paths, const
     for (size_t i = 0; i != paths.size(); ++i)
         addPathAndFileToVirtualColumns(block, paths[i], i);
 
-    filterBlockWithDAG(dag, block, context);
+    filterBlockWithExpression(actions, block);
 
     return block.getByName("_idx").column;
 }
@@ -355,15 +360,15 @@ static const ActionsDAG::Node * splitFilterNodeForAllowedInputs(
     return node;
 }
 
-ActionsDAGPtr splitFilterDagForAllowedInputs(const ActionsDAG::Node * predicate, const Block * allowed_inputs)
+std::optional<ActionsDAG> splitFilterDagForAllowedInputs(const ActionsDAG::Node * predicate, const Block * allowed_inputs)
 {
     if (!predicate)
-        return nullptr;
+        return {};
 
     ActionsDAG::Nodes additional_nodes;
     const auto * res = splitFilterNodeForAllowedInputs(predicate, allowed_inputs, additional_nodes);
     if (!res)
-        return nullptr;
+        return {};
 
     return ActionsDAG::cloneSubDAG({res}, true);
 }
@@ -372,7 +377,7 @@ void filterBlockWithPredicate(const ActionsDAG::Node * predicate, Block & block,
 {
     auto dag = splitFilterDagForAllowedInputs(predicate, &block);
     if (dag)
-        filterBlockWithDAG(dag, block, context);
+        filterBlockWithDAG(std::move(*dag), block, context);
 }
 
 }
diff --git a/src/Storages/VirtualColumnUtils.h b/src/Storages/VirtualColumnUtils.h
index 208aa7a8100..72c45964ff4 100644
--- a/src/Storages/VirtualColumnUtils.h
+++ b/src/Storages/VirtualColumnUtils.h
@@ -23,7 +23,8 @@ namespace VirtualColumnUtils
 void filterBlockWithPredicate(const ActionsDAG::Node * predicate, Block & block, ContextPtr context);
 
 /// Just filters block. Block should contain all the required columns.
-void filterBlockWithDAG(const ActionsDAGPtr & dag, Block & block, ContextPtr context);
+void filterBlockWithDAG(ActionsDAG dag, Block & block, ContextPtr context);
+void filterBlockWithExpression(const ExpressionActionsPtr & actions, Block & block);
 
 /// Builds sets used by ActionsDAG inplace.
 void buildSetsForDAG(const ActionsDAG & dag, const ContextPtr & context);
@@ -32,7 +33,7 @@ void buildSetsForDAG(const ActionsDAG & dag, const ContextPtr & context);
 bool isDeterministicInScopeOfQuery(const ActionsDAG::Node * node);
 
 /// Extract a part of predicate that can be evaluated using only columns from input_names.
-ActionsDAGPtr splitFilterDagForAllowedInputs(const ActionsDAG::Node * predicate, const Block * allowed_inputs);
+std::optional<ActionsDAG> splitFilterDagForAllowedInputs(const ActionsDAG::Node * predicate, const Block * allowed_inputs);
 
 /// Extract from the input stream a set of `name` column values
 template <typename T>
@@ -49,14 +50,14 @@ auto extractSingleValueFromBlock(const Block & block, const String & name)
 NameSet getVirtualNamesForFileLikeStorage();
 VirtualColumnsDescription getVirtualsForFileLikeStorage(const ColumnsDescription & storage_columns);
 
-ActionsDAGPtr createPathAndFileFilterDAG(const ActionsDAG::Node * predicate, const NamesAndTypesList & virtual_columns);
+std::optional<ActionsDAG> createPathAndFileFilterDAG(const ActionsDAG::Node * predicate, const NamesAndTypesList & virtual_columns);
 
-ColumnPtr getFilterByPathAndFileIndexes(const std::vector<String> & paths, const ActionsDAGPtr & dag, const NamesAndTypesList & virtual_columns, const ContextPtr & context);
+ColumnPtr getFilterByPathAndFileIndexes(const std::vector<String> & paths, const ExpressionActionsPtr & actions, const NamesAndTypesList & virtual_columns);
 
 template <typename T>
-void filterByPathOrFile(std::vector<T> & sources, const std::vector<String> & paths, const ActionsDAGPtr & dag, const NamesAndTypesList & virtual_columns, const ContextPtr & context)
+void filterByPathOrFile(std::vector<T> & sources, const std::vector<String> & paths, const ExpressionActionsPtr & actions, const NamesAndTypesList & virtual_columns)
 {
-    auto indexes_column = getFilterByPathAndFileIndexes(paths, dag, virtual_columns, context);
+    auto indexes_column = getFilterByPathAndFileIndexes(paths, actions, virtual_columns);
     const auto & indexes = typeid_cast<const ColumnUInt64 &>(*indexes_column).getData();
     if (indexes.size() == sources.size())
         return;

From 03381bdf22c93b7762f8ef1b63a6148b31599323 Mon Sep 17 00:00:00 2001
From: MikhailBurdukov <burdukvmikhail@gmail.com>
Date: Tue, 16 Jul 2024 16:44:04 +0000
Subject: [PATCH 0306/1488] Build fix

---
 src/Storages/StorageDictionary.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/Storages/StorageDictionary.cpp b/src/Storages/StorageDictionary.cpp
index 69af73122ba..09ca53d7582 100644
--- a/src/Storages/StorageDictionary.cpp
+++ b/src/Storages/StorageDictionary.cpp
@@ -9,6 +9,7 @@
 #include <Parsers/ASTLiteral.h>
 #include <Common/Config/ConfigHelper.h>
 #include <Common/quoteString.h>
+#include <Core/Settings.h>
 #include <QueryPipeline/Pipe.h>
 #include <IO/Operators.h>
 #include <Dictionaries/getDictionaryConfigurationFromAST.h>

From 0954eefb076d36ec5804b46e594005cd7f4030bf Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Tue, 16 Jul 2024 17:01:35 +0000
Subject: [PATCH 0307/1488] Revert SortingStep changes.

---
 src/Processors/QueryPlan/SortingStep.cpp | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/src/Processors/QueryPlan/SortingStep.cpp b/src/Processors/QueryPlan/SortingStep.cpp
index e8e761e7ab0..48fad9f5fdb 100644
--- a/src/Processors/QueryPlan/SortingStep.cpp
+++ b/src/Processors/QueryPlan/SortingStep.cpp
@@ -447,13 +447,6 @@ void SortingStep::describeActions(FormatSettings & settings) const
         settings.out << '\n';
     }
 
-    if (!partition_by_description.empty())
-    {
-        settings.out << prefix << "Partition by description: ";
-        dumpSortDescription(partition_by_description, settings.out);
-        settings.out << '\n';
-    }
-
     if (limit)
         settings.out << prefix << "Limit " << limit << '\n';
 }
@@ -468,9 +461,6 @@ void SortingStep::describeActions(JSONBuilder::JSONMap & map) const
     else
         map.add("Sort Description", explainSortDescription(result_description));
 
-    if (!partition_by_description.empty())
-         map.add("Partition By Description", explainSortDescription(partition_by_description));
-
     if (limit)
         map.add("Limit", limit);
 }

From 40b4a3de6dfddcb3ef206730334821183df06963 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Tue, 16 Jul 2024 19:19:51 +0200
Subject: [PATCH 0308/1488] Update run.sh

---
 docker/test/fasttest/run.sh | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh
index 58c436b469b..970bf12a81a 100755
--- a/docker/test/fasttest/run.sh
+++ b/docker/test/fasttest/run.sh
@@ -11,10 +11,6 @@ stage=${stage:-}
 # Compiler version, normally set by Dockerfile
 export LLVM_VERSION=${LLVM_VERSION:-18}
 
-# For some reason it is needed for tests with 'expect' but I don't know, why
-groupadd --system --gid 1000 clickhouse
-useradd --system --gid 1000 --uid 1000 clickhouse
-
 # A variable to pass additional flags to CMake.
 # Here we explicitly default it to nothing so that bash doesn't complain about
 # it being undefined. Also read it as array so that we can pass an empty list

From 68526487ed51119a638af4554557fe813aa45bf7 Mon Sep 17 00:00:00 2001
From: Alexander Tokmakov <tavplubix@clickhouse.com>
Date: Tue, 16 Jul 2024 19:35:24 +0200
Subject: [PATCH 0309/1488] fix

---
 src/Common/SignalHandlers.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/Common/SignalHandlers.cpp b/src/Common/SignalHandlers.cpp
index 97b890469b9..a4b7784df5c 100644
--- a/src/Common/SignalHandlers.cpp
+++ b/src/Common/SignalHandlers.cpp
@@ -12,6 +12,7 @@
 #include <IO/WriteHelpers.h>
 #include <IO/ReadHelpers.h>
 #include <Interpreters/Context.h>
+#include <Core/Settings.h>
 
 #pragma clang diagnostic ignored "-Wreserved-identifier"
 

From db7fb78a04280179b7ca09f588c957d2e0eab6be Mon Sep 17 00:00:00 2001
From: Max K <maxkaynov@gmail.com>
Date: Tue, 16 Jul 2024 19:49:21 +0200
Subject: [PATCH 0310/1488] major fix

---
 tests/ci/ci_config.py | 6 ++++--
 tests/ci/merge_pr.py  | 6 +-----
 2 files changed, 5 insertions(+), 7 deletions(-)

diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py
index 9b9ddee5326..f5e39f343b2 100644
--- a/tests/ci/ci_config.py
+++ b/tests/ci/ci_config.py
@@ -415,7 +415,8 @@ class CI:
         JobNames.INTEGRATION_TEST_FLAKY: CommonJobConfigs.INTEGRATION_TEST.with_properties(
             required_builds=[BuildNames.PACKAGE_ASAN],
             pr_only=True,
-            reference_job_name=JobNames.INTEGRATION_TEST_TSAN,
+            # TODO: approach with reference job names does not work because digest may not be calculated if job skipped in wf
+            # reference_job_name=JobNames.INTEGRATION_TEST_TSAN,
         ),
         JobNames.COMPATIBILITY_TEST: CommonJobConfigs.COMPATIBILITY_TEST.with_properties(
             required_builds=[BuildNames.PACKAGE_RELEASE],
@@ -460,7 +461,8 @@ class CI:
             required_builds=[BuildNames.PACKAGE_ASAN],
             pr_only=True,
             timeout=3600,
-            reference_job_name=JobNames.STATELESS_TEST_RELEASE,
+            # TODO: approach with reference job names does not work because digest may not be calculated if job skipped in wf
+            # reference_job_name=JobNames.STATELESS_TEST_RELEASE,
         ),
         JobNames.JEPSEN_KEEPER: JobConfig(
             required_builds=[BuildNames.BINARY_RELEASE],
diff --git a/tests/ci/merge_pr.py b/tests/ci/merge_pr.py
index b005be472c7..4d8facafb84 100644
--- a/tests/ci/merge_pr.py
+++ b/tests/ci/merge_pr.py
@@ -256,11 +256,7 @@ def main():
         has_failed_statuses = False
         for status in statuses:
             print(f"Check status [{status.context}], [{status.state}]")
-            if (
-                not CI.is_required(status.context)
-                and status.context not in (CI.StatusNames.SYNC, CI.StatusNames.PR_CHECK)
-                and status.state != SUCCESS
-            ):
+            if CI.is_required(status.context) and status.state != SUCCESS:
                 print(f"WARNING: Failed status [{status.context}], [{status.state}]")
                 has_failed_statuses = True
 

From b6a790124cd670749b4c504f58a4854307bf7d83 Mon Sep 17 00:00:00 2001
From: Igor Nikonov <igor@clickhouse.com>
Date: Tue, 16 Jul 2024 20:16:47 +0000
Subject: [PATCH 0311/1488] Handling parallel replicas protocol with priority
 for async communication

---
 src/Processors/IProcessor.h                   |  2 +
 src/Processors/Sources/RemoteSource.cpp       | 23 +++++++++++
 src/Processors/Sources/RemoteSource.h         |  3 ++
 src/QueryPipeline/RemoteQueryExecutor.cpp     | 38 +++++++++++++++++--
 src/QueryPipeline/RemoteQueryExecutor.h       |  4 +-
 .../RemoteQueryExecutorReadContext.h          |  2 +
 6 files changed, 67 insertions(+), 5 deletions(-)

diff --git a/src/Processors/IProcessor.h b/src/Processors/IProcessor.h
index 02f7b6b3d12..358983a2179 100644
--- a/src/Processors/IProcessor.h
+++ b/src/Processors/IProcessor.h
@@ -221,6 +221,8 @@ public:
         throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method 'schedule' is not implemented for {} processor", getName());
     }
 
+    virtual void asyncJobReady() {}
+
     /** You must call this method if 'prepare' returned ExpandPipeline.
       * This method cannot access any port, but it can create new ports for current processor.
       *
diff --git a/src/Processors/Sources/RemoteSource.cpp b/src/Processors/Sources/RemoteSource.cpp
index 3d7dd3f76b8..f1d47f69782 100644
--- a/src/Processors/Sources/RemoteSource.cpp
+++ b/src/Processors/Sources/RemoteSource.cpp
@@ -89,6 +89,12 @@ ISource::Status RemoteSource::prepare()
 
 void RemoteSource::work()
 {
+    if (async_immediate_work.exchange(false))
+    {
+        LOG_DEBUG(getLogger(__PRETTY_FUNCTION__), "async_immediate_work was true");
+        return;
+    }
+
     /// Connection drain is a heavy operation that may take a long time.
     /// Therefore we move connection drain from prepare() to work(), and drain multiple connections in parallel.
     /// See issue: https://github.com/ClickHouse/ClickHouse/issues/60844
@@ -101,6 +107,23 @@ void RemoteSource::work()
     ISource::work();
 }
 
+void RemoteSource::asyncJobReady()
+{
+    chassert(async_read);
+
+    if (!was_query_sent)
+        return;
+
+    auto res = query_executor->readAsync(/*probe=*/true);
+    if (res.type == RemoteQueryExecutor::ReadResult::Type::ParallelReplicasToken)
+    {
+        LOG_DEBUG(getLogger(__PRETTY_FUNCTION__), "async_immediate_work is {}", async_immediate_work);
+        work();
+        async_immediate_work = true;
+        LOG_DEBUG(getLogger(__PRETTY_FUNCTION__), "async_immediate_work is true");
+    }
+}
+
 std::optional<Chunk> RemoteSource::tryGenerate()
 {
     /// onCancel() will do the cancel if the query was sent.
diff --git a/src/Processors/Sources/RemoteSource.h b/src/Processors/Sources/RemoteSource.h
index 052567bc261..fa04985f101 100644
--- a/src/Processors/Sources/RemoteSource.h
+++ b/src/Processors/Sources/RemoteSource.h
@@ -32,6 +32,8 @@ public:
 
     int schedule() override { return fd; }
 
+    void asyncJobReady() override;
+
     void setStorageLimits(const std::shared_ptr<const StorageLimitsList> & storage_limits_) override;
 
 protected:
@@ -52,6 +54,7 @@ private:
     int fd = -1;
     size_t rows = 0;
     bool manually_add_rows_before_limit_counter = false;
+    std::atomic_bool async_immediate_work{false};
 };
 
 /// Totals source from RemoteQueryExecutor.
diff --git a/src/QueryPipeline/RemoteQueryExecutor.cpp b/src/QueryPipeline/RemoteQueryExecutor.cpp
index b08f2002f64..3ca05b53417 100644
--- a/src/QueryPipeline/RemoteQueryExecutor.cpp
+++ b/src/QueryPipeline/RemoteQueryExecutor.cpp
@@ -469,7 +469,7 @@ RemoteQueryExecutor::ReadResult RemoteQueryExecutor::read()
     return restartQueryWithoutDuplicatedUUIDs();
 }
 
-RemoteQueryExecutor::ReadResult RemoteQueryExecutor::readAsync()
+RemoteQueryExecutor::ReadResult RemoteQueryExecutor::readAsync(bool check_packet_type_only)
 {
 #if defined(OS_LINUX)
     if (!read_context || (resent_query && recreate_read_context))
@@ -486,7 +486,21 @@ RemoteQueryExecutor::ReadResult RemoteQueryExecutor::readAsync()
     {
         std::lock_guard lock(was_cancelled_mutex);
         if (was_cancelled)
+        {
+            LOG_DEBUG(getLogger(__PRETTY_FUNCTION__), "was_cancelled");
             return ReadResult(Block());
+        }
+
+        if (has_postponed_packet)
+        {
+            has_postponed_packet = false;
+            auto read_result = processPacket(read_context->getPacket());
+            if (read_result.getType() == ReadResult::Type::Data || read_result.getType() == ReadResult::Type::ParallelReplicasToken)
+                return read_result;
+
+            if (got_duplicated_part_uuids)
+                break;
+        }
 
         read_context->resume();
 
@@ -506,12 +520,28 @@ RemoteQueryExecutor::ReadResult RemoteQueryExecutor::readAsync()
 
         /// Check if packet is not ready yet.
         if (read_context->isInProgress())
+        {
+            LOG_DEBUG(getLogger(__PRETTY_FUNCTION__), "read_context still in progress");
             return ReadResult(read_context->getFileDescriptor());
+        }
 
-        auto anything = processPacket(read_context->getPacket());
+        const auto packet_type = read_context->getPacketType();
+        LOG_DEBUG(getLogger(__PRETTY_FUNCTION__), "Packet type: {}", packet_type);
 
-        if (anything.getType() == ReadResult::Type::Data || anything.getType() == ReadResult::Type::ParallelReplicasToken)
-            return anything;
+        if (check_packet_type_only)
+        {
+            has_postponed_packet = true;
+            if (packet_type == Protocol::Server::MergeTreeReadTaskRequest
+                || packet_type == Protocol::Server::MergeTreeAllRangesAnnouncement)
+            {
+                return ReadResult(ReadResult::Type::ParallelReplicasToken);
+            }
+            return ReadResult(ReadResult::Type::Nothing);
+        }
+
+        auto read_result = processPacket(read_context->getPacket());
+        if (read_result.getType() == ReadResult::Type::Data || read_result.getType() == ReadResult::Type::ParallelReplicasToken)
+            return read_result;
 
         if (got_duplicated_part_uuids)
             break;
diff --git a/src/QueryPipeline/RemoteQueryExecutor.h b/src/QueryPipeline/RemoteQueryExecutor.h
index 04a59cc3b7e..6849c3e0a07 100644
--- a/src/QueryPipeline/RemoteQueryExecutor.h
+++ b/src/QueryPipeline/RemoteQueryExecutor.h
@@ -183,7 +183,7 @@ public:
     ReadResult read();
 
     /// Async variant of read. Returns ready block or file descriptor which may be used for polling.
-    ReadResult readAsync();
+    ReadResult readAsync(bool check_packet_type_only = false);
 
     /// Receive all remain packets and finish query.
     /// It should be cancelled after read returned empty block.
@@ -303,6 +303,8 @@ private:
       */
     bool got_duplicated_part_uuids = false;
 
+    bool has_postponed_packet = false;
+
     /// Parts uuids, collected from remote replicas
     std::vector<UUID> duplicated_part_uuids;
 
diff --git a/src/QueryPipeline/RemoteQueryExecutorReadContext.h b/src/QueryPipeline/RemoteQueryExecutorReadContext.h
index b8aa8bb9111..c054e75f6f1 100644
--- a/src/QueryPipeline/RemoteQueryExecutorReadContext.h
+++ b/src/QueryPipeline/RemoteQueryExecutorReadContext.h
@@ -39,6 +39,8 @@ public:
 
     Packet getPacket() { return std::move(packet); }
 
+    UInt64 getPacketType() const { return packet.type; }
+
 private:
     bool checkTimeout(bool blocking = false);
 

From 67f5ffc5920892b34b1c8b7176337c39cafe7141 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Tue, 16 Jul 2024 20:02:53 +0000
Subject: [PATCH 0312/1488] Fixups

---
 tests/performance/final_big_column.xml    | 4 ++--
 tests/performance/function_tokens.xml     | 4 ++--
 tests/performance/polymorphic_parts_s.xml | 8 ++++----
 tests/performance/scripts/report.py       | 2 +-
 4 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/tests/performance/final_big_column.xml b/tests/performance/final_big_column.xml
index 1fd586d2d90..5225b3d7ad4 100644
--- a/tests/performance/final_big_column.xml
+++ b/tests/performance/final_big_column.xml
@@ -10,8 +10,8 @@
         PARTITION BY toYYYYMM(d) ORDER BY key
     </create_query>
 
-    <fill_query>INSERT INTO optimized_select_final SELECT toDate('2000-01-01'), 2*number, randomPrintableASCII(1000) FROM numbers(5000000)</fill_query>
-    <fill_query>INSERT INTO optimized_select_final SELECT toDate('2020-01-01'), 2*number+1, randomPrintableASCII(1000) FROM numbers(5000000)</fill_query>
+    <fill_query>INSERT INTO optimized_select_final SELECT toDate('2000-01-01'), 2*number, randomPrintableASCII(1000) FROM numbers(2500000)</fill_query>
+    <fill_query>INSERT INTO optimized_select_final SELECT toDate('2020-01-01'), 2*number+1, randomPrintableASCII(1000) FROM numbers(2500000)</fill_query>
 
     <query>SELECT * FROM optimized_select_final FINAL FORMAT Null SETTINGS max_threads = 8</query>
     <query>SELECT * FROM optimized_select_final FINAL WHERE key % 10 = 0 FORMAT Null</query>
diff --git a/tests/performance/function_tokens.xml b/tests/performance/function_tokens.xml
index 1ff56323d62..bc2bc71a933 100644
--- a/tests/performance/function_tokens.xml
+++ b/tests/performance/function_tokens.xml
@@ -1,5 +1,5 @@
 <test>
     <query>with 'Many years later as he faced the firing squad, Colonel Aureliano Buendia was to remember that distant afternoon when his father took him to discover ice.' as s select splitByChar(' ', materialize(s)) as w from numbers(1000000)</query>
-    <query>with 'Many years later as he faced the firing squad, Colonel Aureliano Buendia was to remember that distant afternoon when his father took him to discover ice.' as s select splitByRegexp(' ', materialize(s)) as w from numbers(1000000)</query>
-    <query>with 'Many years later as he faced the firing squad, Colonel Aureliano Buendia was to remember that distant afternoon when his father took him to discover ice.' as s select splitByRegexp('\s+', materialize(s)) as w from numbers(100000)</query>
+    <query>with 'Many years later as he faced the firing squad, Colonel Aureliano Buendia was to remember that distant afternoon when his father took him to discover ice.' as s select splitByRegexp(' ', materialize(s)) as w from numbers(200000)</query>
+    <query>with 'Many years later as he faced the firing squad, Colonel Aureliano Buendia was to remember that distant afternoon when his father took him to discover ice.' as s select splitByRegexp('\s+', materialize(s)) as w from numbers(20000)</query>
 </test>
diff --git a/tests/performance/polymorphic_parts_s.xml b/tests/performance/polymorphic_parts_s.xml
index b4dd87a7ae3..5fe1ffffe1d 100644
--- a/tests/performance/polymorphic_parts_s.xml
+++ b/tests/performance/polymorphic_parts_s.xml
@@ -24,10 +24,10 @@
         <min_insert_block_size_rows>1</min_insert_block_size_rows>
     </settings>
 
-    <!-- 100 parts -->
-    <query>INSERT INTO hits_wide(UserID)    SELECT rand() FROM  numbers(100)</query>
-    <query>INSERT INTO hits_compact(UserID) SELECT rand() FROM numbers(1000)</query>
-    <query>INSERT INTO hits_buffer(UserID)  SELECT rand() FROM numbers(100)</query>
+    <!-- 50 parts -->
+    <query>INSERT INTO hits_wide(UserID)    SELECT rand() FROM  numbers(50)</query>
+    <query>INSERT INTO hits_compact(UserID) SELECT rand() FROM numbers(500)</query>
+    <query>INSERT INTO hits_buffer(UserID)  SELECT rand() FROM numbers(50)</query>
 
     <drop_query>DROP TABLE IF EXISTS hits_wide</drop_query>
     <drop_query>DROP TABLE IF EXISTS hits_compact</drop_query>
diff --git a/tests/performance/scripts/report.py b/tests/performance/scripts/report.py
index c2bc773bd54..e45d709ca2c 100755
--- a/tests/performance/scripts/report.py
+++ b/tests/performance/scripts/report.py
@@ -555,7 +555,7 @@ if args.report == "main":
             "Total client time for measured query runs,&nbsp;s",  # 2
             "Queries",  # 3
             "Longest query, total for measured runs,&nbsp;s",  # 4
-            "Wall clock time per query,&nbsp;s",  # 5
+            "Average query wall clock time,&nbsp;s",  # 5
             "Shortest query, total for measured runs,&nbsp;s",  # 6
             "",  # Runs                                               #7
         ]

From ea3a333a0711a33ad9193e67939b5f8a970834ea Mon Sep 17 00:00:00 2001
From: Shaun Struwig <41984034+Blargian@users.noreply.github.com>
Date: Tue, 16 Jul 2024 22:25:45 +0200
Subject: [PATCH 0313/1488] Fix formatting error

---
 docs/en/sql-reference/functions/other-functions.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/en/sql-reference/functions/other-functions.md b/docs/en/sql-reference/functions/other-functions.md
index 260457b3be1..5e2802eb08b 100644
--- a/docs/en/sql-reference/functions/other-functions.md
+++ b/docs/en/sql-reference/functions/other-functions.md
@@ -86,7 +86,7 @@ Returns the fully qualified domain name of the ClickHouse server.
 fqdn();
 ```
 
-Aliases: `fullHostName`, 'FQDN'. 
+Aliases: `fullHostName`, `FQDN`. 
 
 **Returned value**
 

From 8c0ade413cca3ea63b47909c54c80de76c1e2a26 Mon Sep 17 00:00:00 2001
From: Shaun Struwig <41984034+Blargian@users.noreply.github.com>
Date: Tue, 16 Jul 2024 22:28:12 +0200
Subject: [PATCH 0314/1488] Update other-functions.md

---
 docs/en/sql-reference/functions/other-functions.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/en/sql-reference/functions/other-functions.md b/docs/en/sql-reference/functions/other-functions.md
index 5e2802eb08b..b7e4094f30e 100644
--- a/docs/en/sql-reference/functions/other-functions.md
+++ b/docs/en/sql-reference/functions/other-functions.md
@@ -76,7 +76,7 @@ WHERE macro = 'test';
 └───────┴──────────────┘
 ```
 
-## FQDN
+## fqdn
 
 Returns the fully qualified domain name of the ClickHouse server.
 

From a7310e51939ad6053d6ab94b07f0171457e5d779 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Mon, 8 Jul 2024 19:32:27 +0200
Subject: [PATCH 0315/1488] Ignore async_load_databases for ATTACH query

It is quite odd that when ATTACH finishes the tables may not be exists,
due to async_load_databases.

For server startup it makes total sense, but not for queries.

Plus, you can execute queries in parallel if you want to make it faster.

Note, that server startup does not uses this code, see loadMetadata.cpp.

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 src/Interpreters/InterpreterCreateQuery.cpp | 16 ++++------------
 1 file changed, 4 insertions(+), 12 deletions(-)

diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp
index 84d7f0a587c..1d8d885b216 100644
--- a/src/Interpreters/InterpreterCreateQuery.cpp
+++ b/src/Interpreters/InterpreterCreateQuery.cpp
@@ -362,18 +362,10 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create)
             TablesLoader loader{getContext()->getGlobalContext(), {{database_name, database}}, mode};
             auto load_tasks = loader.loadTablesAsync();
             auto startup_tasks = loader.startupTablesAsync();
-            if (getContext()->getGlobalContext()->getServerSettings().async_load_databases)
-            {
-                scheduleLoad(load_tasks);
-                scheduleLoad(startup_tasks);
-            }
-            else
-            {
-                /// First prioritize, schedule and wait all the load table tasks
-                waitLoad(currentPoolOr(TablesLoaderForegroundPoolId), load_tasks);
-                /// Only then prioritize, schedule and wait all the startup tasks
-                waitLoad(currentPoolOr(TablesLoaderForegroundPoolId), startup_tasks);
-            }
+            /// First prioritize, schedule and wait all the load table tasks
+            waitLoad(currentPoolOr(TablesLoaderForegroundPoolId), load_tasks);
+            /// Only then prioritize, schedule and wait all the startup tasks
+            waitLoad(currentPoolOr(TablesLoaderForegroundPoolId), startup_tasks);
         }
     }
     catch (...)

From f30d35ae2926948f1e6a268917113e757df4e2df Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Mon, 8 Jul 2024 19:34:18 +0200
Subject: [PATCH 0316/1488] Revert "Merge pull request #65571 from
 ClickHouse/fix-flaky-test-4"

Reverts: https://github.com/ClickHouse/ClickHouse/pull/65571

This reverts commit da9a34ea46b504881ffe5aa605c933106862ba25, reversing
changes made to cbdb9833f207d4b0e35ad09cf4757f5d5b506b77.

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 .../0_stateless/01254_dict_load_after_detach_attach.reference   | 2 +-
 .../queries/0_stateless/01254_dict_load_after_detach_attach.sql | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/queries/0_stateless/01254_dict_load_after_detach_attach.reference b/tests/queries/0_stateless/01254_dict_load_after_detach_attach.reference
index 9c2c59f6379..2f2d638a294 100644
--- a/tests/queries/0_stateless/01254_dict_load_after_detach_attach.reference
+++ b/tests/queries/0_stateless/01254_dict_load_after_detach_attach.reference
@@ -1,4 +1,4 @@
-NOT_LOADED
+0	NOT_LOADED
 0	LOADED
 10
 1	LOADED
diff --git a/tests/queries/0_stateless/01254_dict_load_after_detach_attach.sql b/tests/queries/0_stateless/01254_dict_load_after_detach_attach.sql
index 11473c6ce32..ef9e940df8b 100644
--- a/tests/queries/0_stateless/01254_dict_load_after_detach_attach.sql
+++ b/tests/queries/0_stateless/01254_dict_load_after_detach_attach.sql
@@ -12,7 +12,7 @@ LAYOUT(FLAT());
 DETACH DATABASE {CLICKHOUSE_DATABASE:Identifier};
 ATTACH DATABASE {CLICKHOUSE_DATABASE:Identifier};
 
-SELECT COALESCE((SELECT status FROM system.dictionaries WHERE database = currentDatabase() AND name = 'dict')::Nullable(String), 'NOT_LOADED');
+SELECT query_count, status FROM system.dictionaries WHERE database = currentDatabase() AND name = 'dict';
 SYSTEM RELOAD DICTIONARY dict;
 SELECT query_count, status FROM system.dictionaries WHERE database = currentDatabase() AND name = 'dict';
 SELECT dictGetUInt64('dict', 'val', toUInt64(0));

From d579e9f64f1142934eeb8628f9b2764b5a022ec9 Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitlibar@clickhouse.com>
Date: Tue, 16 Jul 2024 23:12:02 +0200
Subject: [PATCH 0317/1488] Fix logical error in PrometheusRequestHandler.

---
 src/Server/PrometheusRequestHandler.cpp | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)

diff --git a/src/Server/PrometheusRequestHandler.cpp b/src/Server/PrometheusRequestHandler.cpp
index 1f3e038a1f5..87c106c3fc0 100644
--- a/src/Server/PrometheusRequestHandler.cpp
+++ b/src/Server/PrometheusRequestHandler.cpp
@@ -30,15 +30,8 @@ void PrometheusRequestHandler::handleRequest(HTTPServerRequest & request, HTTPSe
         response.setContentType("text/plain; version=0.0.4; charset=UTF-8");
 
         WriteBufferFromHTTPServerResponse wb(response, request.getMethod() == Poco::Net::HTTPRequest::HTTP_HEAD, keep_alive_timeout, write_event);
-        try
-        {
-            metrics_writer->write(wb);
-            wb.finalize();
-        }
-        catch (...)
-        {
-            wb.finalize();
-        }
+        metrics_writer->write(wb);
+        wb.finalize();
     }
     catch (...)
     {

From 9ff6de4eba0e82ff6dc2c1d007d4866637e1d0b2 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Tue, 16 Jul 2024 23:17:45 +0200
Subject: [PATCH 0318/1488] Remove unused header

---
 src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp b/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp
index b9edff39b82..a904b29e12f 100644
--- a/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp
+++ b/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp
@@ -21,7 +21,6 @@
 
 #include <Processors/QueryPlan/QueryPlan.h>
 #include <Processors/QueryPlan/ReadFromPreparedSource.h>
-#include <Processors/Transforms/FilterTransform.h>
 #include <Parsers/ASTFunction.h>
 #include <Parsers/ASTIdentifier.h>
 #include <Parsers/ASTTablesInSelectQuery.h>

From 5ca7241865f863d68b3a3b96011b9238e49ac3f8 Mon Sep 17 00:00:00 2001
From: Max K <maxkaynov@gmail.com>
Date: Wed, 17 Jul 2024 00:23:14 +0200
Subject: [PATCH 0319/1488] CI: AWS ASG decrease capacity from runners

---
 tests/ci/worker/init_runner.sh | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/tests/ci/worker/init_runner.sh b/tests/ci/worker/init_runner.sh
index ec7793db2aa..91bffa876b2 100644
--- a/tests/ci/worker/init_runner.sh
+++ b/tests/ci/worker/init_runner.sh
@@ -50,7 +50,7 @@ set -uo pipefail
 # set accordingly to a runner role #
 ####################################
 
-echo "Running init script"
+echo "Running init script v1"
 export DEBIAN_FRONTEND=noninteractive
 export RUNNER_HOME=/home/ubuntu/actions-runner
 
@@ -90,7 +90,6 @@ terminate_delayed() {
     # IF `sleep` IS CHANGED, CHANGE ANOTHER VALUE IN `pgrep`
     sleep=13.14159265358979323846
     echo "Going to terminate the runner's instance in $sleep seconds"
-    INSTANCE_ID=$(ec2metadata --instance-id)
     # We execute it with `at` to not have it as an orphan process, but launched independently
     # GH Runners kill all remain processes
     echo "sleep '$sleep'; aws ec2 terminate-instances --instance-ids $INSTANCE_ID" | at now || \
@@ -111,11 +110,17 @@ declare -f terminate_delayed >> /tmp/actions-hooks/common.sh
 terminate_and_exit() {
     # Terminate instance and exit from the script instantly
     echo "Going to terminate the runner's instance"
-    INSTANCE_ID=$(ec2metadata --instance-id)
     aws ec2 terminate-instances --instance-ids "$INSTANCE_ID"
     exit 0
 }
 
+terminate_decrease_and_exit() {
+    # Terminate instance and exit from the script instantly
+    echo "Going to terminate the runner's instance and decrease asg capacity"
+    aws autoscaling terminate-instance-in-auto-scaling-group --instance-id "$INSTANCE_ID" --should-decrement-desired-capacity
+    exit 0
+}
+
 declare -f terminate_and_exit >> /tmp/actions-hooks/common.sh
 
 check_spot_instance_is_old() {
@@ -324,7 +329,7 @@ while true; do
                 sudo -u ubuntu ./config.sh remove --token "$(get_runner_token)" \
                     || continue
                 echo "Runner didn't launch or have assigned jobs after ${RUNNER_AGE} seconds, shutting down"
-                terminate_and_exit
+                terminate_decrease_and_exit
             fi
         fi
     else

From a1a802cee9f1e75cfdaed7c86aac63fd4e3010db Mon Sep 17 00:00:00 2001
From: Max K <maxkaynov@gmail.com>
Date: Wed, 17 Jul 2024 00:32:13 +0200
Subject: [PATCH 0320/1488] fix deployment script

---
 tests/ci/worker/deploy-runner-init.sh | 4 +++-
 tests/ci/worker/init_runner.sh        | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/tests/ci/worker/deploy-runner-init.sh b/tests/ci/worker/deploy-runner-init.sh
index 06edede48fa..96fbd82a99c 100755
--- a/tests/ci/worker/deploy-runner-init.sh
+++ b/tests/ci/worker/deploy-runner-init.sh
@@ -1,5 +1,7 @@
 #!/usr/bin/env bash
 
+set -e
+
 usage() {
   echo "Usage: $0 ENVIRONMENT" >&2
   echo "Valid values for ENVIRONMENT: staging, production" >&2
@@ -55,7 +57,7 @@ EOF
 
 body() {
   local first_line
-  first_line=$(sed -n '/^# THE SCRIPT START$/{=;q}' "$SOURCE_SCRIPT")
+  first_line=$(sed -n '/^# THE SCRIPT START$/{=;q;}' "$SOURCE_SCRIPT")
   if [ -z "$first_line" ]; then
     echo "The pattern '# THE SCRIPT START' is not found in $SOURCE_SCRIPT" >&2
     exit 1
diff --git a/tests/ci/worker/init_runner.sh b/tests/ci/worker/init_runner.sh
index 91bffa876b2..d6cdb6d9c57 100644
--- a/tests/ci/worker/init_runner.sh
+++ b/tests/ci/worker/init_runner.sh
@@ -50,7 +50,7 @@ set -uo pipefail
 # set accordingly to a runner role #
 ####################################
 
-echo "Running init script v1"
+echo "Running init v1"
 export DEBIAN_FRONTEND=noninteractive
 export RUNNER_HOME=/home/ubuntu/actions-runner
 

From a6ee9a98f54396394460a6e8492c371b53e47fbd Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 17 Jul 2024 01:59:22 +0200
Subject: [PATCH 0321/1488] Fix clang-tidy

---
 src/Storages/transformQueryForExternalDatabaseAnalyzer.cpp | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/Storages/transformQueryForExternalDatabaseAnalyzer.cpp b/src/Storages/transformQueryForExternalDatabaseAnalyzer.cpp
index dc1749b3196..a7f56999d73 100644
--- a/src/Storages/transformQueryForExternalDatabaseAnalyzer.cpp
+++ b/src/Storages/transformQueryForExternalDatabaseAnalyzer.cpp
@@ -69,9 +69,6 @@ ASTPtr getASTForExternalDatabaseFromQueryTree(const QueryTreeNodePtr & query_tre
     bool allow_where = true;
     if (const auto * join_node = join_tree->as<JoinNode>())
     {
-        if (join_node->getStrictness() != JoinStrictness::All)
-            allow_where = false;
-
         if (join_node->getKind() == JoinKind::Left)
             allow_where = join_node->getLeftTableExpression()->isEqual(*table_expression);
         else if (join_node->getKind() == JoinKind::Right)

From 694c0074ab3ccafae265485062b2075b0b98d0b9 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 17 Jul 2024 02:08:37 +0200
Subject: [PATCH 0322/1488] Allow to run clang-tidy with clang-19

---
 .clang-tidy | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.clang-tidy b/.clang-tidy
index de19059d09e..bb63bf2eea6 100644
--- a/.clang-tidy
+++ b/.clang-tidy
@@ -27,6 +27,8 @@ Checks: [
   '-bugprone-not-null-terminated-result',
   '-bugprone-reserved-identifier', # useful but too slow, TODO retry when https://reviews.llvm.org/rG1c282052624f9d0bd273bde0b47b30c96699c6c7 is merged
   '-bugprone-unchecked-optional-access',
+  '-bugprone-crtp-constructor-accessibility',
+  '-bugprone-suspicious-stringview-data-usage',
 
   '-cert-dcl16-c',
   '-cert-dcl37-c',
@@ -36,6 +38,7 @@ Checks: [
   '-cert-msc51-cpp',
   '-cert-oop54-cpp',
   '-cert-oop57-cpp',
+  '-cert-err33-c', # Misreports on clang-19: it warns about all functions containing 'remove' in the name, not only about the standard library.
 
   '-clang-analyzer-optin.performance.Padding',
 
@@ -99,6 +102,7 @@ Checks: [
   '-modernize-use-emplace',
   '-modernize-use-nodiscard',
   '-modernize-use-trailing-return-type',
+  '-modernize-use-designated-initializers',
 
   '-performance-enum-size',
   '-performance-inefficient-string-concatenation',

From 0d1789292bc480c33c0f2b2ee54058976dea74fe Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 17 Jul 2024 02:11:09 +0200
Subject: [PATCH 0323/1488] Add sudo for convenience

---
 docker/test/util/Dockerfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docker/test/util/Dockerfile b/docker/test/util/Dockerfile
index f13bb576f79..babddbd228c 100644
--- a/docker/test/util/Dockerfile
+++ b/docker/test/util/Dockerfile
@@ -17,6 +17,7 @@ ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=18
 
 RUN apt-get update \
     && apt-get install \
+        sudo \
         apt-transport-https \
         apt-utils \
         ca-certificates \

From c913a36ed6580e87ebb641cb3024c7182725d8ea Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 17 Jul 2024 02:13:57 +0200
Subject: [PATCH 0324/1488] Minor modifications

---
 docker/test/fasttest/Dockerfile | 5 ++++-
 docker/test/fasttest/run.sh     | 4 ----
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/docker/test/fasttest/Dockerfile b/docker/test/fasttest/Dockerfile
index 2512268be0f..72e33226f38 100644
--- a/docker/test/fasttest/Dockerfile
+++ b/docker/test/fasttest/Dockerfile
@@ -26,7 +26,10 @@ RUN apt-get update \
         zstd \
         --yes --no-install-recommends \
     && apt-get clean \
-    && rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/*
+    && rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/* \
+    && groupadd --system --gid 1000 clickhouse \
+    && useradd --system --gid 1000 --uid 1000 clickhouse
+# ^ For some reason, groupadd and useradd are needed for tests with 'expect', but I don't know, why.
 
 COPY requirements.txt /
 RUN pip3 install --no-cache-dir -r /requirements.txt
diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh
index 57e7b6b2f56..970bf12a81a 100755
--- a/docker/test/fasttest/run.sh
+++ b/docker/test/fasttest/run.sh
@@ -11,10 +11,6 @@ stage=${stage:-}
 # Compiler version, normally set by Dockerfile
 export LLVM_VERSION=${LLVM_VERSION:-18}
 
-# For some reason, it is needed for tests with 'expect', but I don't know, why.
-sudo groupadd --system --gid 1000 clickhouse
-sudo useradd --system --gid 1000 --uid 1000 clickhouse
-
 # A variable to pass additional flags to CMake.
 # Here we explicitly default it to nothing so that bash doesn't complain about
 # it being undefined. Also read it as array so that we can pass an empty list

From a40edea5c99ac290a3b04a015065c0cad5266ab0 Mon Sep 17 00:00:00 2001
From: Xu Jia <jia.xu@clickhouse.com>
Date: Wed, 17 Jul 2024 09:37:28 +0800
Subject: [PATCH 0325/1488] trigger CI


From 60f45cb993ccf15812605d51b6e7ce8489c98d05 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 17 Jul 2024 03:52:35 +0200
Subject: [PATCH 0326/1488] A fix

---
 docker/test/fasttest/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/test/fasttest/Dockerfile b/docker/test/fasttest/Dockerfile
index 72e33226f38..4cac2ee6135 100644
--- a/docker/test/fasttest/Dockerfile
+++ b/docker/test/fasttest/Dockerfile
@@ -28,7 +28,7 @@ RUN apt-get update \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/* \
     && groupadd --system --gid 1000 clickhouse \
-    && useradd --system --gid 1000 --uid 1000 clickhouse
+    && useradd --system --gid 1000 --uid 1000 -m clickhouse
 # ^ For some reason, groupadd and useradd are needed for tests with 'expect', but I don't know, why.
 
 COPY requirements.txt /

From b05e02625844930892d7ceaa77f2655765c51b88 Mon Sep 17 00:00:00 2001
From: JackyWoo <wuchienchao@qq.com>
Date: Wed, 17 Jul 2024 09:58:57 +0800
Subject: [PATCH 0327/1488] Fix logical error

---
 src/Storages/Statistics/Statistics.cpp | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/Storages/Statistics/Statistics.cpp b/src/Storages/Statistics/Statistics.cpp
index 5c0e5f178e1..588e20e801f 100644
--- a/src/Storages/Statistics/Statistics.cpp
+++ b/src/Storages/Statistics/Statistics.cpp
@@ -37,6 +37,14 @@ std::optional<Float64> StatisticsUtils::tryConvertToFloat64(const Field & field)
             return field.get<UInt64>();
         case Field::Types::Float64:
             return field.get<Float64>();
+        case Field::Types::Int128:
+            return field.get<Int128>();
+        case Field::Types::UInt128:
+            return field.get<UInt128>();
+        case Field::Types::Int256:
+            return field.get<Int256>();
+        case Field::Types::UInt256:
+            return field.get<UInt256>();
         default:
             return {};
     }

From 10c19d2be123845cb2d452a4e3e735e0e28e41cf Mon Sep 17 00:00:00 2001
From: jiaosenvip <62206110+jiaosenvip@users.noreply.github.com>
Date: Wed, 17 Jul 2024 11:11:34 +0800
Subject: [PATCH 0328/1488] Conditional statement error

The conditional statement is used incorrectly here, and the two cases are correctly covered after modification.
---
 docs/zh/sql-reference/functions/conditional-functions.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/zh/sql-reference/functions/conditional-functions.md b/docs/zh/sql-reference/functions/conditional-functions.md
index e0f8f7a4777..378e4d897c0 100644
--- a/docs/zh/sql-reference/functions/conditional-functions.md
+++ b/docs/zh/sql-reference/functions/conditional-functions.md
@@ -82,14 +82,14 @@ FROM LEFT_RIGHT
 SELECT
     left,
     right,
-    if(left < right, 'left is smaller than right', 'right is greater or equal than left') AS is_smaller
+    if(left < right, 'left is smaller than right', 'right is smaller or equal than left') AS is_smaller
 FROM LEFT_RIGHT
 WHERE isNotNull(left) AND isNotNull(right)
 
 ┌─left─┬─right─┬─is_smaller──────────────────────────┐
 │    1 │     3 │ left is smaller than right          │
-│    2 │     2 │ right is greater or equal than left │
-│    3 │     1 │ right is greater or equal than left │
+│    2 │     2 │ right is smaller or equal than left │
+│    3 │     1 │ right is smaller or equal than left │
 └──────┴───────┴─────────────────────────────────────┘
 ```
 

From 187a9a623c2f23978d0b1161d6950a9cf7d9d8ad Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 17 Jul 2024 06:56:32 +0200
Subject: [PATCH 0329/1488] Remove `is_deterministic` from system.functions

---
 .../System/StorageSystemFunctions.cpp         | 67 ++++++-------------
 .../02117_show_create_table_system.reference  |  1 -
 ...ckhouse_local_interactive_exception.expect |  2 +-
 .../02815_no_throw_in_simple_queries.sh       |  7 +-
 4 files changed, 21 insertions(+), 56 deletions(-)

diff --git a/src/Storages/System/StorageSystemFunctions.cpp b/src/Storages/System/StorageSystemFunctions.cpp
index 6e4ac8b2747..0a233dc052b 100644
--- a/src/Storages/System/StorageSystemFunctions.cpp
+++ b/src/Storages/System/StorageSystemFunctions.cpp
@@ -40,7 +40,6 @@ namespace
         MutableColumns & res_columns,
         const String & name,
         UInt64 is_aggregate,
-        std::optional<UInt64> is_deterministic,
         const String & create_query,
         FunctionOrigin function_origin,
         const Factory & factory)
@@ -48,58 +47,53 @@ namespace
         res_columns[0]->insert(name);
         res_columns[1]->insert(is_aggregate);
 
-        if (!is_deterministic.has_value())
-            res_columns[2]->insertDefault();
-        else
-            res_columns[2]->insert(*is_deterministic);
-
         if constexpr (std::is_same_v<Factory, UserDefinedSQLFunctionFactory> || std::is_same_v<Factory, UserDefinedExecutableFunctionFactory>)
         {
-            res_columns[3]->insert(false);
-            res_columns[4]->insertDefault();
+            res_columns[2]->insert(false);
+            res_columns[3]->insertDefault();
         }
         else
         {
-            res_columns[3]->insert(factory.isCaseInsensitive(name));
+            res_columns[2]->insert(factory.isCaseInsensitive(name));
             if (factory.isAlias(name))
-                res_columns[4]->insert(factory.aliasTo(name));
+                res_columns[3]->insert(factory.aliasTo(name));
             else
-                res_columns[4]->insertDefault();
+                res_columns[3]->insertDefault();
         }
 
-        res_columns[5]->insert(create_query);
-        res_columns[6]->insert(static_cast<Int8>(function_origin));
+        res_columns[4]->insert(create_query);
+        res_columns[5]->insert(static_cast<Int8>(function_origin));
 
         if constexpr (std::is_same_v<Factory, FunctionFactory>)
         {
             if (factory.isAlias(name))
             {
+                res_columns[6]->insertDefault();
                 res_columns[7]->insertDefault();
                 res_columns[8]->insertDefault();
                 res_columns[9]->insertDefault();
                 res_columns[10]->insertDefault();
                 res_columns[11]->insertDefault();
-                res_columns[12]->insertDefault();
             }
             else
             {
                 auto documentation = factory.getDocumentation(name);
-                res_columns[7]->insert(documentation.description);
-                res_columns[8]->insert(documentation.syntax);
-                res_columns[9]->insert(documentation.argumentsAsString());
-                res_columns[10]->insert(documentation.returned_value);
-                res_columns[11]->insert(documentation.examplesAsString());
-                res_columns[12]->insert(documentation.categoriesAsString());
+                res_columns[6]->insert(documentation.description);
+                res_columns[7]->insert(documentation.syntax);
+                res_columns[8]->insert(documentation.argumentsAsString());
+                res_columns[9]->insert(documentation.returned_value);
+                res_columns[10]->insert(documentation.examplesAsString());
+                res_columns[11]->insert(documentation.categoriesAsString());
             }
         }
         else
         {
+            res_columns[6]->insertDefault();
             res_columns[7]->insertDefault();
             res_columns[8]->insertDefault();
             res_columns[9]->insertDefault();
             res_columns[10]->insertDefault();
             res_columns[11]->insertDefault();
-            res_columns[12]->insertDefault();
         }
     }
 }
@@ -120,7 +114,6 @@ ColumnsDescription StorageSystemFunctions::getColumnsDescription()
     {
         {"name", std::make_shared<DataTypeString>(), "The name of the function."},
         {"is_aggregate", std::make_shared<DataTypeUInt8>(), "Whether the function is an aggregate function."},
-        {"is_deterministic", std::make_shared<DataTypeNullable>(std::make_shared<DataTypeUInt8>()), "Whether the function is deterministic."},
         {"case_insensitive", std::make_shared<DataTypeUInt8>(), "Whether the function name can be used case-insensitively."},
         {"alias_to", std::make_shared<DataTypeString>(), "The original function name, if the function name is an alias."},
         {"create_query", std::make_shared<DataTypeString>(), "Obsolete."},
@@ -140,36 +133,14 @@ void StorageSystemFunctions::fillData(MutableColumns & res_columns, ContextPtr c
     const auto & function_names = functions_factory.getAllRegisteredNames();
     for (const auto & function_name : function_names)
     {
-        std::optional<UInt64> is_deterministic;
-        try
-        {
-            DO_NOT_UPDATE_ERROR_STATISTICS();
-            is_deterministic = functions_factory.tryGet(function_name, context)->isDeterministic();
-        }
-        catch (const Exception & e)
-        {
-            /// Some functions throw because they need special configuration or setup before use.
-            if (e.code() == ErrorCodes::DICTIONARIES_WAS_NOT_LOADED
-                || e.code() == ErrorCodes::FUNCTION_NOT_ALLOWED
-                || e.code() == ErrorCodes::NOT_IMPLEMENTED
-                || e.code() == ErrorCodes::SUPPORT_IS_DISABLED
-                || e.code() == ErrorCodes::ACCESS_DENIED
-                || e.code() == ErrorCodes::DEPRECATED_FUNCTION)
-            {
-                /// Ignore exception, show is_deterministic = NULL.
-            }
-            else
-                throw;
-        }
-
-        fillRow(res_columns, function_name, 0, is_deterministic, "", FunctionOrigin::SYSTEM, functions_factory);
+        fillRow(res_columns, function_name, 0, "", FunctionOrigin::SYSTEM, functions_factory);
     }
 
     const auto & aggregate_functions_factory = AggregateFunctionFactory::instance();
     const auto & aggregate_function_names = aggregate_functions_factory.getAllRegisteredNames();
     for (const auto & function_name : aggregate_function_names)
     {
-        fillRow(res_columns, function_name, 1, {1}, "", FunctionOrigin::SYSTEM, aggregate_functions_factory);
+        fillRow(res_columns, function_name, 1, "", FunctionOrigin::SYSTEM, aggregate_functions_factory);
     }
 
     const auto & user_defined_sql_functions_factory = UserDefinedSQLFunctionFactory::instance();
@@ -177,14 +148,14 @@ void StorageSystemFunctions::fillData(MutableColumns & res_columns, ContextPtr c
     for (const auto & function_name : user_defined_sql_functions_names)
     {
         auto create_query = queryToString(user_defined_sql_functions_factory.get(function_name));
-        fillRow(res_columns, function_name, 0, {0}, create_query, FunctionOrigin::SQL_USER_DEFINED, user_defined_sql_functions_factory);
+        fillRow(res_columns, function_name, 0, create_query, FunctionOrigin::SQL_USER_DEFINED, user_defined_sql_functions_factory);
     }
 
     const auto & user_defined_executable_functions_factory = UserDefinedExecutableFunctionFactory::instance();
     const auto & user_defined_executable_functions_names = user_defined_executable_functions_factory.getRegisteredNames(context); /// NOLINT(readability-static-accessed-through-instance)
     for (const auto & function_name : user_defined_executable_functions_names)
     {
-        fillRow(res_columns, function_name, 0, {0}, "", FunctionOrigin::EXECUTABLE_USER_DEFINED, user_defined_executable_functions_factory);
+        fillRow(res_columns, function_name, 0, "", FunctionOrigin::EXECUTABLE_USER_DEFINED, user_defined_executable_functions_factory);
     }
 }
 
diff --git a/tests/queries/0_stateless/02117_show_create_table_system.reference b/tests/queries/0_stateless/02117_show_create_table_system.reference
index 8f62eda9233..cfae4fee6c2 100644
--- a/tests/queries/0_stateless/02117_show_create_table_system.reference
+++ b/tests/queries/0_stateless/02117_show_create_table_system.reference
@@ -286,7 +286,6 @@ CREATE TABLE system.functions
 (
     `name` String,
     `is_aggregate` UInt8,
-    `is_deterministic` Nullable(UInt8),
     `case_insensitive` UInt8,
     `alias_to` String,
     `create_query` String,
diff --git a/tests/queries/0_stateless/02164_clickhouse_local_interactive_exception.expect b/tests/queries/0_stateless/02164_clickhouse_local_interactive_exception.expect
index add977c4fce..76902bdc69d 100755
--- a/tests/queries/0_stateless/02164_clickhouse_local_interactive_exception.expect
+++ b/tests/queries/0_stateless/02164_clickhouse_local_interactive_exception.expect
@@ -9,7 +9,7 @@ if {[info exists env(CLICKHOUSE_TMP)]} {
 }
 exp_internal -f $CLICKHOUSE_TMP/$basename.debuglog 0
 
-log_user 0
+log_user 1
 set timeout 20
 match_max 100000
 
diff --git a/tests/queries/0_stateless/02815_no_throw_in_simple_queries.sh b/tests/queries/0_stateless/02815_no_throw_in_simple_queries.sh
index 50f5e565088..68c55f9b66a 100755
--- a/tests/queries/0_stateless/02815_no_throw_in_simple_queries.sh
+++ b/tests/queries/0_stateless/02815_no_throw_in_simple_queries.sh
@@ -45,9 +45,4 @@ expect eof
 EOF
 }
 
-run "$CLICKHOUSE_LOCAL --disable_suggestion"
-# Suggestions are off because the suggestion feature initializes itself by reading all available function
-# names from "system.functions". Getting the value for field "is_obsolete" occasionally throws (e.g. for
-# certain dictionary functions when dictionaries are not set up yet). Exceptions are properly handled, but
-# they exist for a short time. This, in combination with CLICKHOUSE_TERMINATE_ON_ANY_EXCEPTION, terminates
-# clickhouse-local and clickhouse-client when run in interactive mode *with* suggestions.
+run "$CLICKHOUSE_LOCAL"

From 855f03d1d3c5b34d3de1b0919b9f7975d64b3cb8 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 17 Jul 2024 06:57:16 +0200
Subject: [PATCH 0330/1488] Remove `is_deterministic` from system.functions

---
 docs/en/operations/system-tables/functions.md | 1 -
 1 file changed, 1 deletion(-)

diff --git a/docs/en/operations/system-tables/functions.md b/docs/en/operations/system-tables/functions.md
index d52bf24f289..da0ef04bd27 100644
--- a/docs/en/operations/system-tables/functions.md
+++ b/docs/en/operations/system-tables/functions.md
@@ -9,7 +9,6 @@ Columns:
 
 - `name` ([String](../../sql-reference/data-types/string.md)) – The name of the function.
 - `is_aggregate` ([UInt8](../../sql-reference/data-types/int-uint.md)) — Whether the function is an aggregate function.
-- `is_deterministic` ([Nullable](../../sql-reference/data-types/nullable.md)([UInt8](../../sql-reference/data-types/int-uint.md))) - Whether the function is deterministic.
 - `case_insensitive`, ([UInt8](../../sql-reference/data-types/int-uint.md)) - Whether the function name can be used case-insensitively.
 - `alias_to`, ([String](../../sql-reference/data-types/string.md)) - The original function name, if the function name is an alias.
 - `create_query`, ([String](../../sql-reference/data-types/enum.md)) - Unused.

From 968d28f1f4bae02eae2f7b96c774ddfd9656c308 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 17 Jul 2024 07:24:46 +0200
Subject: [PATCH 0331/1488] Remove outdated articles from the docs

---
 docs/en/sql-reference/data-types/lowcardinality.md | 3 +--
 docs/zh/changelog/index.md                         | 2 +-
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/docs/en/sql-reference/data-types/lowcardinality.md b/docs/en/sql-reference/data-types/lowcardinality.md
index 133ac2bd72e..a92f6f4dce6 100644
--- a/docs/en/sql-reference/data-types/lowcardinality.md
+++ b/docs/en/sql-reference/data-types/lowcardinality.md
@@ -56,7 +56,6 @@ Functions:
 
 ## Related content
 
-- [Reducing ClickHouse Storage Cost with the Low Cardinality Type – Lessons from an Instana Engineer](https://altinity.com/blog/2020-5-20-reducing-clickhouse-storage-cost-with-the-low-cardinality-type-lessons-from-an-instana-engineer)
-- [String Optimization (video presentation in Russian)](https://youtu.be/rqf-ILRgBdY?list=PL0Z2YDlm0b3iwXCpEFiOOYmwXzVmjJfEt). [Slides in English](https://github.com/ClickHouse/clickhouse-presentations/raw/master/meetup19/string_optimization.pdf)
 - Blog: [Optimizing ClickHouse with Schemas and Codecs](https://clickhouse.com/blog/optimize-clickhouse-codecs-compression-schema)
 - Blog: [Working with time series data in ClickHouse](https://clickhouse.com/blog/working-with-time-series-data-and-functions-ClickHouse)
+- [String Optimization (video presentation in Russian)](https://youtu.be/rqf-ILRgBdY?list=PL0Z2YDlm0b3iwXCpEFiOOYmwXzVmjJfEt). [Slides in English](https://github.com/ClickHouse/clickhouse-presentations/raw/master/meetup19/string_optimization.pdf)
diff --git a/docs/zh/changelog/index.md b/docs/zh/changelog/index.md
index c91d8bcf4d1..cd77a8c03cf 100644
--- a/docs/zh/changelog/index.md
+++ b/docs/zh/changelog/index.md
@@ -356,7 +356,7 @@ sidebar_label: "\u53D8\u66F4\u65E5\u5FD7"
 
 #### 新功能 {#new-feature-1}
 
--   添加 `deduplicate_blocks_in_dependent_materialized_views` 用于控制具有实例化视图的表中幂等插入的行为的选项。 这个新功能是由Altinity的特殊要求添加到错误修正版本中的。
+-   添加 `deduplicate_blocks_in_dependent_materialized_views` 用于控制具有实例化视图的表中幂等插入的行为的选项。
     [#9070](https://github.com/ClickHouse/ClickHouse/pull/9070) [(urykhy)](https://github.com/urykhy)
 
 ### ClickHouse版本v20.1.2.4,2020-01-22 {#clickhouse-release-v20-1-2-4-2020-01-22}

From 4a69bd78819c1bb97988fce96d17be99bbffe00e Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 17 Jul 2024 07:37:45 +0200
Subject: [PATCH 0332/1488] Fix terrible test @arthurpassos

---
 ...tiple_batches_array_inconsistent_offsets.sh | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/tests/queries/0_stateless/02874_parquet_multiple_batches_array_inconsistent_offsets.sh b/tests/queries/0_stateless/02874_parquet_multiple_batches_array_inconsistent_offsets.sh
index 83196458a84..c96531ffea9 100755
--- a/tests/queries/0_stateless/02874_parquet_multiple_batches_array_inconsistent_offsets.sh
+++ b/tests/queries/0_stateless/02874_parquet_multiple_batches_array_inconsistent_offsets.sh
@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
-# Tags: no-ubsan, no-fasttest
+# Tags: long, no-ubsan, no-fasttest, no-parallel, no-asan, no-msan, no-tsan
+# This test requires around 10 GB of memory and it is just too much.
 
 CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
@@ -121,9 +122,12 @@ echo "Parquet"
 #}
 
 DATA_FILE=$CUR_DIR/data_parquet/string_int_list_inconsistent_offset_multiple_batches.parquet
-${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS parquet_load"
-${CLICKHOUSE_CLIENT} --query="CREATE TABLE parquet_load (ints Array(Int64), strings Nullable(String)) ENGINE = Memory"
-cat "$DATA_FILE" | ${CLICKHOUSE_CLIENT} -q "INSERT INTO parquet_load FORMAT Parquet"
-${CLICKHOUSE_CLIENT} --query="SELECT * FROM parquet_load" | md5sum
-${CLICKHOUSE_CLIENT} --query="SELECT count() FROM parquet_load"
-${CLICKHOUSE_CLIENT} --query="drop table parquet_load"
\ No newline at end of file
+
+${CLICKHOUSE_LOCAL} --multiquery "
+DROP TABLE IF EXISTS parquet_load;
+CREATE TABLE parquet_load (ints Array(Int64), strings Nullable(String)) ENGINE = Memory;
+INSERT INTO parquet_load FROM INFILE '$DATA_FILE';
+SELECT sum(cityHash64(*)) FROM parquet_load;
+SELECT count() FROM parquet_load;
+DROP TABLE parquet_load;
+"

From 73ecbefd4123bae9ca9902111c28ace0cc05b994 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 17 Jul 2024 07:52:33 +0200
Subject: [PATCH 0333/1488] Fix bad log message

---
 src/Common/ZooKeeper/ZooKeeper.cpp | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/src/Common/ZooKeeper/ZooKeeper.cpp b/src/Common/ZooKeeper/ZooKeeper.cpp
index 56db9adb787..01bb508da95 100644
--- a/src/Common/ZooKeeper/ZooKeeper.cpp
+++ b/src/Common/ZooKeeper/ZooKeeper.cpp
@@ -13,14 +13,14 @@
 #include <Common/ZooKeeper/Types.h>
 #include <Common/ZooKeeper/ZooKeeperCommon.h>
 #include <Common/randomSeed.h>
-#include <base/find_symbols.h>
 #include <base/sort.h>
+#include <base/map.h>
 #include <base/getFQDNOrHostName.h>
 #include <Core/ServerUUID.h>
 #include <Core/BackgroundSchedulePool.h>
-#include "Common/ZooKeeper/IKeeper.h"
-#include <Common/DNSResolver.h>
+#include <Common/ZooKeeper/IKeeper.h>
 #include <Common/StringUtils.h>
+#include <Common/quoteString.h>
 #include <Common/Exception.h>
 #include <Interpreters/Context.h>
 
@@ -114,7 +114,11 @@ void ZooKeeper::init(ZooKeeperArgs args_, std::unique_ptr<Coordination::IKeeper>
             /// availability_zones is empty on server startup or after config reloading
             /// We will keep the az info when starting new sessions
             availability_zones = args.availability_zones;
-            LOG_TEST(log, "Availability zones from config: [{}], client: {}", fmt::join(availability_zones, ", "), args.client_availability_zone);
+
+            LOG_TEST(log, "Availability zones from config: [{}], client: {}",
+                fmt::join(collections::map(availability_zones, [](auto s){ return DB::quoteString(s); }), ", "),
+                DB::quoteString(args.client_availability_zone));
+
             if (args.availability_zone_autodetect)
                 updateAvailabilityZones();
         }

From 88f9a87ce6960c97990b801d8c6f99e614f329a8 Mon Sep 17 00:00:00 2001
From: Alexander Gololobov <davenger@clickhouse.com>
Date: Wed, 17 Jul 2024 08:43:38 +0200
Subject: [PATCH 0334/1488] unused error codes

---
 src/Storages/System/StorageSystemFunctions.cpp | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/src/Storages/System/StorageSystemFunctions.cpp b/src/Storages/System/StorageSystemFunctions.cpp
index 0a233dc052b..f10ce9e3987 100644
--- a/src/Storages/System/StorageSystemFunctions.cpp
+++ b/src/Storages/System/StorageSystemFunctions.cpp
@@ -16,16 +16,6 @@
 namespace DB
 {
 
-namespace ErrorCodes
-{
-    extern const int DICTIONARIES_WAS_NOT_LOADED;
-    extern const int FUNCTION_NOT_ALLOWED;
-    extern const int NOT_IMPLEMENTED;
-    extern const int SUPPORT_IS_DISABLED;
-    extern const int ACCESS_DENIED;
-    extern const int DEPRECATED_FUNCTION;
-};
-
 enum class FunctionOrigin : int8_t
 {
     SYSTEM = 0,

From b4755b36de41e493e121f5846a71ede11d5155d3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?=
 <benjamin.antal@clickhouse.com>
Date: Tue, 16 Jul 2024 15:22:41 +0000
Subject: [PATCH 0335/1488] Fix dropping named collection in local storage

---
 .../NamedCollectionsMetadataStorage.cpp       | 54 +++++++++----------
 1 file changed, 27 insertions(+), 27 deletions(-)

diff --git a/src/Common/NamedCollections/NamedCollectionsMetadataStorage.cpp b/src/Common/NamedCollections/NamedCollectionsMetadataStorage.cpp
index 3c6561fa51e..79b674965de 100644
--- a/src/Common/NamedCollections/NamedCollectionsMetadataStorage.cpp
+++ b/src/Common/NamedCollections/NamedCollectionsMetadataStorage.cpp
@@ -116,32 +116,32 @@ public:
         return elements;
     }
 
-    bool exists(const std::string & path) const override
+    bool exists(const std::string & file_name) const override
     {
-        return fs::exists(getPath(path));
+        return fs::exists(getPath(file_name));
     }
 
-    std::string read(const std::string & path) const override
+    std::string read(const std::string & file_name) const override
     {
-        ReadBufferFromFile in(getPath(path));
+        ReadBufferFromFile in(getPath(file_name));
         std::string data;
         readStringUntilEOF(data, in);
         return data;
     }
 
-    void write(const std::string & path, const std::string & data, bool replace) override
+    void write(const std::string & file_name, const std::string & data, bool replace) override
     {
-        if (!replace && fs::exists(path))
+        if (!replace && fs::exists(file_name))
         {
             throw Exception(
                 ErrorCodes::NAMED_COLLECTION_ALREADY_EXISTS,
                 "Metadata file {} for named collection already exists",
-                path);
+                file_name);
         }
 
         fs::create_directories(root_path);
 
-        auto tmp_path = getPath(path + ".tmp");
+        auto tmp_path = getPath(file_name + ".tmp");
         WriteBufferFromFile out(tmp_path, data.size(), O_WRONLY | O_CREAT | O_EXCL);
         writeString(data, out);
 
@@ -150,22 +150,22 @@ public:
             out.sync();
         out.close();
 
-        fs::rename(tmp_path, getPath(path));
+        fs::rename(tmp_path, getPath(file_name));
     }
 
-    void remove(const std::string & path) override
+    void remove(const std::string & file_name) override
     {
-        if (!removeIfExists(getPath(path)))
+        if (!removeIfExists(file_name))
         {
             throw Exception(
                 ErrorCodes::NAMED_COLLECTION_DOESNT_EXIST,
-                "Cannot remove `{}`, because it doesn't exist", path);
+                "Cannot remove `{}`, because it doesn't exist", file_name);
         }
     }
 
-    bool removeIfExists(const std::string & path) override
+    bool removeIfExists(const std::string & file_name) override
     {
-        return fs::remove(getPath(path));
+        return fs::remove(getPath(file_name));
     }
 
 private:
@@ -264,49 +264,49 @@ public:
         return children;
     }
 
-    bool exists(const std::string & path) const override
+    bool exists(const std::string & file_name) const override
     {
-        return getClient()->exists(getPath(path));
+        return getClient()->exists(getPath(file_name));
     }
 
-    std::string read(const std::string & path) const override
+    std::string read(const std::string & file_name) const override
     {
-        return getClient()->get(getPath(path));
+        return getClient()->get(getPath(file_name));
     }
 
-    void write(const std::string & path, const std::string & data, bool replace) override
+    void write(const std::string & file_name, const std::string & data, bool replace) override
     {
         if (replace)
         {
-            getClient()->createOrUpdate(getPath(path), data, zkutil::CreateMode::Persistent);
+            getClient()->createOrUpdate(getPath(file_name), data, zkutil::CreateMode::Persistent);
         }
         else
         {
-            auto code = getClient()->tryCreate(getPath(path), data, zkutil::CreateMode::Persistent);
+            auto code = getClient()->tryCreate(getPath(file_name), data, zkutil::CreateMode::Persistent);
 
             if (code == Coordination::Error::ZNODEEXISTS)
             {
                 throw Exception(
                     ErrorCodes::NAMED_COLLECTION_ALREADY_EXISTS,
                     "Metadata file {} for named collection already exists",
-                    path);
+                    file_name);
             }
         }
     }
 
-    void remove(const std::string & path) override
+    void remove(const std::string & file_name) override
     {
-        getClient()->remove(getPath(path));
+        getClient()->remove(getPath(file_name));
     }
 
-    bool removeIfExists(const std::string & path) override
+    bool removeIfExists(const std::string & file_name) override
     {
-        auto code = getClient()->tryRemove(getPath(path));
+        auto code = getClient()->tryRemove(getPath(file_name));
         if (code == Coordination::Error::ZOK)
             return true;
         if (code == Coordination::Error::ZNONODE)
             return false;
-        throw Coordination::Exception::fromPath(code, getPath(path));
+        throw Coordination::Exception::fromPath(code, getPath(file_name));
     }
 
 private:

From df636100d3643031ee8d2102b737e992a6e44a40 Mon Sep 17 00:00:00 2001
From: Samuele Guerrini <sguerrini97@protonmail.com>
Date: Mon, 8 Jul 2024 12:00:53 +0200
Subject: [PATCH 0336/1488] add entry in documentation for
 use_same_password_for_base_backup

---
 docs/en/operations/backup.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docs/en/operations/backup.md b/docs/en/operations/backup.md
index fc861e25e9f..248fdbc156f 100644
--- a/docs/en/operations/backup.md
+++ b/docs/en/operations/backup.md
@@ -85,6 +85,7 @@ The BACKUP and RESTORE statements take a list of DATABASE and TABLE names, a des
     - `password` for the file on disk
     - `base_backup`: the destination of the previous backup of this source.  For example, `Disk('backups', '1.zip')`
     - `use_same_s3_credentials_for_base_backup`: whether base backup to S3 should inherit credentials from the query. Only works with `S3`.
+    - `use_same_password_for_base_backup`: whether base backup archive should inherit the password from the query.
     - `structure_only`: if enabled, allows to only backup or restore the CREATE statements without the data of tables
     - `storage_policy`: storage policy for the tables being restored. See [Using Multiple Block Devices for Data Storage](../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-multiple-volumes). This setting is only applicable to the `RESTORE` command. The specified storage policy applies only to tables with an engine from the `MergeTree` family.
     - `s3_storage_class`: the storage class used for S3 backup. For example, `STANDARD`

From d8c68a27744dca3f68e4de08036148c2ecfd2ed6 Mon Sep 17 00:00:00 2001
From: Samuele Guerrini <sguerrini97@protonmail.com>
Date: Mon, 8 Jul 2024 12:00:53 +0200
Subject: [PATCH 0337/1488] working incremental password protected backups

---
 src/Backups/BackupFactory.h                         |  1 +
 src/Backups/BackupImpl.cpp                          | 13 +++++++++++--
 src/Backups/BackupImpl.h                            |  7 +++++--
 src/Backups/BackupSettings.cpp                      |  1 +
 src/Backups/BackupSettings.h                        |  3 +++
 src/Backups/BackupsWorker.cpp                       |  2 ++
 src/Backups/RestoreSettings.cpp                     |  1 +
 src/Backups/RestoreSettings.h                       |  3 +++
 .../registerBackupEngineAzureBlobStorage.cpp        |  6 ++++--
 src/Backups/registerBackupEngineS3.cpp              |  6 ++++--
 src/Backups/registerBackupEnginesFileAndDisk.cpp    |  6 ++++--
 11 files changed, 39 insertions(+), 10 deletions(-)

diff --git a/src/Backups/BackupFactory.h b/src/Backups/BackupFactory.h
index e13a9a12ca2..807b8516d49 100644
--- a/src/Backups/BackupFactory.h
+++ b/src/Backups/BackupFactory.h
@@ -41,6 +41,7 @@ public:
         bool allow_s3_native_copy = true;
         bool allow_azure_native_copy = true;
         bool use_same_s3_credentials_for_base_backup = false;
+        bool use_same_password_for_base_backup = false;
         bool azure_attempt_to_create_container = true;
         ReadSettings read_settings;
         WriteSettings write_settings;
diff --git a/src/Backups/BackupImpl.cpp b/src/Backups/BackupImpl.cpp
index 3f972c36e47..23f067a62f5 100644
--- a/src/Backups/BackupImpl.cpp
+++ b/src/Backups/BackupImpl.cpp
@@ -92,7 +92,8 @@ BackupImpl::BackupImpl(
     std::shared_ptr<IBackupReader> reader_,
     const ContextPtr & context_,
     bool is_internal_backup_,
-    bool use_same_s3_credentials_for_base_backup_)
+    bool use_same_s3_credentials_for_base_backup_,
+    bool use_same_password_for_base_backup_)
     : backup_info(backup_info_)
     , backup_name_for_logging(backup_info.toStringForLogging())
     , use_archive(!archive_params_.archive_name.empty())
@@ -104,6 +105,7 @@ BackupImpl::BackupImpl(
     , version(INITIAL_BACKUP_VERSION)
     , base_backup_info(base_backup_info_)
     , use_same_s3_credentials_for_base_backup(use_same_s3_credentials_for_base_backup_)
+    , use_same_password_for_base_backup(use_same_password_for_base_backup_)
     , log(getLogger("BackupImpl"))
 {
     open();
@@ -120,7 +122,8 @@ BackupImpl::BackupImpl(
     const std::shared_ptr<IBackupCoordination> & coordination_,
     const std::optional<UUID> & backup_uuid_,
     bool deduplicate_files_,
-    bool use_same_s3_credentials_for_base_backup_)
+    bool use_same_s3_credentials_for_base_backup_,
+    bool use_same_password_for_base_backup_)
     : backup_info(backup_info_)
     , backup_name_for_logging(backup_info.toStringForLogging())
     , use_archive(!archive_params_.archive_name.empty())
@@ -135,6 +138,7 @@ BackupImpl::BackupImpl(
     , base_backup_info(base_backup_info_)
     , deduplicate_files(deduplicate_files_)
     , use_same_s3_credentials_for_base_backup(use_same_s3_credentials_for_base_backup_)
+    , use_same_password_for_base_backup(use_same_password_for_base_backup_)
     , log(getLogger("BackupImpl"))
 {
     open();
@@ -258,6 +262,11 @@ std::shared_ptr<const IBackup> BackupImpl::getBaseBackupUnlocked() const
         params.is_internal_backup = is_internal_backup;
         /// use_same_s3_credentials_for_base_backup should be inherited for base backups
         params.use_same_s3_credentials_for_base_backup = use_same_s3_credentials_for_base_backup;
+        /// use_same_password_for_base_backup should be inherited for base backups
+        params.use_same_password_for_base_backup = use_same_password_for_base_backup;
+
+        if (params.use_same_password_for_base_backup)
+            params.password = archive_params.password;
 
         base_backup = BackupFactory::instance().createBackup(params);
 
diff --git a/src/Backups/BackupImpl.h b/src/Backups/BackupImpl.h
index 2b27e2ab090..d7846104c4c 100644
--- a/src/Backups/BackupImpl.h
+++ b/src/Backups/BackupImpl.h
@@ -41,7 +41,8 @@ public:
         std::shared_ptr<IBackupReader> reader_,
         const ContextPtr & context_,
         bool is_internal_backup_,
-        bool use_same_s3_credentials_for_base_backup_);
+        bool use_same_s3_credentials_for_base_backup_,
+        bool use_same_password_for_base_backup_);
 
     BackupImpl(
         const BackupInfo & backup_info_,
@@ -53,7 +54,8 @@ public:
         const std::shared_ptr<IBackupCoordination> & coordination_,
         const std::optional<UUID> & backup_uuid_,
         bool deduplicate_files_,
-        bool use_same_s3_credentials_for_base_backup_);
+        bool use_same_s3_credentials_for_base_backup_,
+        bool use_same_password_for_base_backup_);
 
     ~BackupImpl() override;
 
@@ -153,6 +155,7 @@ private:
     bool writing_finalized = false;
     bool deduplicate_files = true;
     bool use_same_s3_credentials_for_base_backup = false;
+    bool use_same_password_for_base_backup = false;
     const LoggerPtr log;
 };
 
diff --git a/src/Backups/BackupSettings.cpp b/src/Backups/BackupSettings.cpp
index e33880f88e3..37ddd344001 100644
--- a/src/Backups/BackupSettings.cpp
+++ b/src/Backups/BackupSettings.cpp
@@ -29,6 +29,7 @@ namespace ErrorCodes
     M(Bool, allow_s3_native_copy) \
     M(Bool, allow_azure_native_copy) \
     M(Bool, use_same_s3_credentials_for_base_backup) \
+    M(Bool, use_same_password_for_base_backup) \
     M(Bool, azure_attempt_to_create_container) \
     M(Bool, read_from_filesystem_cache) \
     M(UInt64, shard_num) \
diff --git a/src/Backups/BackupSettings.h b/src/Backups/BackupSettings.h
index a6c4d5d7181..d8f48f6e1ac 100644
--- a/src/Backups/BackupSettings.h
+++ b/src/Backups/BackupSettings.h
@@ -50,6 +50,9 @@ struct BackupSettings
     /// Whether base backup to S3 should inherit credentials from the BACKUP query.
     bool use_same_s3_credentials_for_base_backup = false;
 
+    /// Wheter base backup archive should be unlocked using the same password as the incremental archive
+    bool use_same_password_for_base_backup = false;
+
     /// Whether a new Azure container should be created if it does not exist (requires permissions at storage account level)
     bool azure_attempt_to_create_container = true;
 
diff --git a/src/Backups/BackupsWorker.cpp b/src/Backups/BackupsWorker.cpp
index 15a7d7c1eca..0614fb2da01 100644
--- a/src/Backups/BackupsWorker.cpp
+++ b/src/Backups/BackupsWorker.cpp
@@ -601,6 +601,7 @@ void BackupsWorker::doBackup(
     backup_create_params.allow_s3_native_copy = backup_settings.allow_s3_native_copy;
     backup_create_params.allow_azure_native_copy = backup_settings.allow_azure_native_copy;
     backup_create_params.use_same_s3_credentials_for_base_backup = backup_settings.use_same_s3_credentials_for_base_backup;
+    backup_create_params.use_same_password_for_base_backup = backup_settings.use_same_password_for_base_backup;
     backup_create_params.azure_attempt_to_create_container = backup_settings.azure_attempt_to_create_container;
     backup_create_params.read_settings = getReadSettingsForBackup(context, backup_settings);
     backup_create_params.write_settings = getWriteSettingsForBackup(context);
@@ -912,6 +913,7 @@ void BackupsWorker::doRestore(
     backup_open_params.password = restore_settings.password;
     backup_open_params.allow_s3_native_copy = restore_settings.allow_s3_native_copy;
     backup_open_params.use_same_s3_credentials_for_base_backup = restore_settings.use_same_s3_credentials_for_base_backup;
+    backup_open_params.use_same_password_for_base_backup = restore_settings.use_same_password_for_base_backup;
     backup_open_params.read_settings = getReadSettingsForRestore(context);
     backup_open_params.write_settings = getWriteSettingsForRestore(context);
     backup_open_params.is_internal_backup = restore_settings.internal;
diff --git a/src/Backups/RestoreSettings.cpp b/src/Backups/RestoreSettings.cpp
index 7bbfd9ed751..a974fc11d00 100644
--- a/src/Backups/RestoreSettings.cpp
+++ b/src/Backups/RestoreSettings.cpp
@@ -164,6 +164,7 @@ namespace
     M(RestoreUDFCreationMode, create_function) \
     M(Bool, allow_s3_native_copy) \
     M(Bool, use_same_s3_credentials_for_base_backup) \
+    M(Bool, use_same_password_for_base_backup) \
     M(Bool, restore_broken_parts_as_detached) \
     M(Bool, internal) \
     M(String, host_id) \
diff --git a/src/Backups/RestoreSettings.h b/src/Backups/RestoreSettings.h
index 06ecbc80aef..0fe5ee1a4bf 100644
--- a/src/Backups/RestoreSettings.h
+++ b/src/Backups/RestoreSettings.h
@@ -113,6 +113,9 @@ struct RestoreSettings
     /// Whether base backup from S3 should inherit credentials from the RESTORE query.
     bool use_same_s3_credentials_for_base_backup = false;
 
+    /// Wheter base backup archive should be unlocked using the same password as the incremental archive
+    bool use_same_password_for_base_backup = false;
+
     /// If it's true RESTORE won't stop on broken parts while restoring, instead they will be restored as detached parts
     /// to the `detached` folder with names starting with `broken-from-backup'.
     bool restore_broken_parts_as_detached = false;
diff --git a/src/Backups/registerBackupEngineAzureBlobStorage.cpp b/src/Backups/registerBackupEngineAzureBlobStorage.cpp
index 626df99b00c..45f0386375a 100644
--- a/src/Backups/registerBackupEngineAzureBlobStorage.cpp
+++ b/src/Backups/registerBackupEngineAzureBlobStorage.cpp
@@ -141,7 +141,8 @@ void registerBackupEngineAzureBlobStorage(BackupFactory & factory)
                 reader,
                 params.context,
                 params.is_internal_backup,
-                /* use_same_s3_credentials_for_base_backup*/ false);
+                /* use_same_s3_credentials_for_base_backup*/ false,
+                params.use_same_password_for_base_backup);
         }
         else
         {
@@ -164,7 +165,8 @@ void registerBackupEngineAzureBlobStorage(BackupFactory & factory)
                 params.backup_coordination,
                 params.backup_uuid,
                 params.deduplicate_files,
-                /* use_same_s3_credentials_for_base_backup */ false);
+                /* use_same_s3_credentials_for_base_backup */ false,
+                params.use_same_password_for_base_backup);
         }
 #else
         throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "AzureBlobStorage support is disabled");
diff --git a/src/Backups/registerBackupEngineS3.cpp b/src/Backups/registerBackupEngineS3.cpp
index 59ed9506af0..79e3e945557 100644
--- a/src/Backups/registerBackupEngineS3.cpp
+++ b/src/Backups/registerBackupEngineS3.cpp
@@ -120,7 +120,8 @@ void registerBackupEngineS3(BackupFactory & factory)
                 reader,
                 params.context,
                 params.is_internal_backup,
-                params.use_same_s3_credentials_for_base_backup);
+                params.use_same_s3_credentials_for_base_backup,
+                params.use_same_password_for_base_backup);
         }
         else
         {
@@ -144,7 +145,8 @@ void registerBackupEngineS3(BackupFactory & factory)
                 params.backup_coordination,
                 params.backup_uuid,
                 params.deduplicate_files,
-                params.use_same_s3_credentials_for_base_backup);
+                params.use_same_s3_credentials_for_base_backup,
+                params.use_same_password_for_base_backup);
         }
 #else
         throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "S3 support is disabled");
diff --git a/src/Backups/registerBackupEnginesFileAndDisk.cpp b/src/Backups/registerBackupEnginesFileAndDisk.cpp
index 35263d39cba..c486f79a77a 100644
--- a/src/Backups/registerBackupEnginesFileAndDisk.cpp
+++ b/src/Backups/registerBackupEnginesFileAndDisk.cpp
@@ -178,7 +178,8 @@ void registerBackupEnginesFileAndDisk(BackupFactory & factory)
                 reader,
                 params.context,
                 params.is_internal_backup,
-                params.use_same_s3_credentials_for_base_backup);
+                params.use_same_s3_credentials_for_base_backup,
+                params.use_same_password_for_base_backup);
         }
         else
         {
@@ -197,7 +198,8 @@ void registerBackupEnginesFileAndDisk(BackupFactory & factory)
                 params.backup_coordination,
                 params.backup_uuid,
                 params.deduplicate_files,
-                params.use_same_s3_credentials_for_base_backup);
+                params.use_same_s3_credentials_for_base_backup,
+                params.use_same_password_for_base_backup);
         }
     };
 

From 74de7833b8d429edcac9c4735143fc92349abdaf Mon Sep 17 00:00:00 2001
From: Samuele Guerrini <sguerrini97@protonmail.com>
Date: Mon, 8 Jul 2024 12:34:53 +0200
Subject: [PATCH 0338/1488] fix typo

---
 src/Backups/BackupSettings.h  | 2 +-
 src/Backups/RestoreSettings.h | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Backups/BackupSettings.h b/src/Backups/BackupSettings.h
index d8f48f6e1ac..0abeb897db4 100644
--- a/src/Backups/BackupSettings.h
+++ b/src/Backups/BackupSettings.h
@@ -50,7 +50,7 @@ struct BackupSettings
     /// Whether base backup to S3 should inherit credentials from the BACKUP query.
     bool use_same_s3_credentials_for_base_backup = false;
 
-    /// Wheter base backup archive should be unlocked using the same password as the incremental archive
+    /// Whether base backup archive should be unlocked using the same password as the incremental archive
     bool use_same_password_for_base_backup = false;
 
     /// Whether a new Azure container should be created if it does not exist (requires permissions at storage account level)
diff --git a/src/Backups/RestoreSettings.h b/src/Backups/RestoreSettings.h
index 0fe5ee1a4bf..fe07a0a7208 100644
--- a/src/Backups/RestoreSettings.h
+++ b/src/Backups/RestoreSettings.h
@@ -113,7 +113,7 @@ struct RestoreSettings
     /// Whether base backup from S3 should inherit credentials from the RESTORE query.
     bool use_same_s3_credentials_for_base_backup = false;
 
-    /// Wheter base backup archive should be unlocked using the same password as the incremental archive
+    /// Whether base backup archive should be unlocked using the same password as the incremental archive
     bool use_same_password_for_base_backup = false;
 
     /// If it's true RESTORE won't stop on broken parts while restoring, instead they will be restored as detached parts

From 12fb08648670dcf39659596ccb552f3462004fd8 Mon Sep 17 00:00:00 2001
From: Samuele Guerrini <sguerrini97@protonmail.com>
Date: Mon, 8 Jul 2024 16:25:07 +0200
Subject: [PATCH 0339/1488] added test for use_same_password_for_base_backup

---
 ...se_same_password_for_base_backup.reference | 21 ++++++++
 ...ackup_use_same_password_for_base_backup.sh | 50 +++++++++++++++++++
 2 files changed, 71 insertions(+)
 create mode 100644 tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.reference
 create mode 100755 tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.sh

diff --git a/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.reference b/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.reference
new file mode 100644
index 00000000000..7354d50a7c0
--- /dev/null
+++ b/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.reference
@@ -0,0 +1,21 @@
+use_same_password_for_base_backup
+base
+BACKUP_CREATED
+add_more_data_1
+inc_1
+BACKUP_CREATED
+add_more_data_2
+inc_2
+BACKUP_CREATED
+inc_2_bad
+Couldn't unpack zip archive '02843_backup_use_same_password_for_base_backup_test_inc_1.zip': Password is required. (CANNOT_UNPACK_ARCHIVE)
+restore_inc_1
+RESTORED
+restore_inc_2
+RESTORED
+restore_inc_2_bad
+Couldn't unpack zip archive '02843_backup_use_same_password_for_base_backup_test_inc_1.zip': Password is required. (CANNOT_UNPACK_ARCHIVE)
+count_inc_1
+20
+count_inc_2
+30
diff --git a/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.sh b/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.sh
new file mode 100755
index 00000000000..a2b1a953e24
--- /dev/null
+++ b/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.sh
@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+# Tags: no-fasttest
+
+CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CUR_DIR"/../shell_config.sh
+
+$CLICKHOUSE_CLIENT -nm -q "
+  DROP TABLE IF EXISTS data;
+  DROP TABLE IF EXISTS data_1;
+  DROP TABLE IF EXISTS data_2;
+  CREATE TABLE data (key Int) ENGINE=MergeTree() ORDER BY tuple();
+  INSERT INTO data SELECT * from numbers(10);
+"
+
+echo 'use_same_password_for_base_backup'
+echo "base"
+$CLICKHOUSE_CLIENT -q "BACKUP TABLE data TO Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_base.zip') SETTINGS password='password';" | cut -f2
+
+echo 'add_more_data_1'
+$CLICKHOUSE_CLIENT -q "INSERT INTO data SELECT * FROM numbers(10,10);"
+
+echo "inc_1"
+$CLICKHOUSE_CLIENT -q "BACKUP TABLE data TO Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_1.zip') SETTINGS base_backup=Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_base.zip'),password='password',use_same_password_for_base_backup=1" | cut -f2
+
+echo 'add_more_data_2'
+$CLICKHOUSE_CLIENT -q "INSERT INTO data SELECT * FROM numbers(20,10);"
+
+echo "inc_2"
+$CLICKHOUSE_CLIENT -q "BACKUP TABLE data TO Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_2.zip') SETTINGS base_backup=Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_1.zip'),password='password',use_same_password_for_base_backup=1" | cut -f2
+
+echo "inc_2_bad"
+$CLICKHOUSE_CLIENT -q "BACKUP TABLE data TO Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_2_bad.zip') SETTINGS base_backup=Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_1.zip'),password='password'" |& grep -m1 -o "Couldn't unpack zip archive '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_1.zip': Password is required. (CANNOT_UNPACK_ARCHIVE)"
+
+echo "restore_inc_1"
+$CLICKHOUSE_CLIENT -q "RESTORE TABLE data AS data_1 FROM Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_1.zip') SETTINGS password='password',use_same_password_for_base_backup=1" | cut -f2
+
+echo "restore_inc_2"
+$CLICKHOUSE_CLIENT -q "RESTORE TABLE data AS data_2 FROM Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_2.zip') SETTINGS password='password',use_same_password_for_base_backup=1" | cut -f2
+
+echo "restore_inc_2_bad"
+$CLICKHOUSE_CLIENT -q "RESTORE TABLE data AS data_2 FROM Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_2.zip') SETTINGS password='password'" |& grep -m1 -o "Couldn't unpack zip archive '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_1.zip': Password is required. (CANNOT_UNPACK_ARCHIVE)"
+
+echo "count_inc_1"
+$CLICKHOUSE_CLIENT -q "SELECT COUNT(*) FROM data_1" | cut -f2
+
+echo "count_inc_2"
+$CLICKHOUSE_CLIENT -q "SELECT COUNT(*) FROM data_2" | cut -f2
+
+exit 0

From 6f25aacd71948b682630889e75774eb4f2469aaa Mon Sep 17 00:00:00 2001
From: Samuele Guerrini <sguerrini97@protonmail.com>
Date: Tue, 9 Jul 2024 16:36:59 +0200
Subject: [PATCH 0340/1488] use CLICKHOUSE_TEST_NAME instead of
 CLICKHOUSE_TEST_UNIQUE_NAME for backup filename

---
 ...kup_use_same_password_for_base_backup.reference |  4 ++--
 ...843_backup_use_same_password_for_base_backup.sh | 14 +++++++-------
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.reference b/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.reference
index 7354d50a7c0..1a331cca46b 100644
--- a/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.reference
+++ b/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.reference
@@ -8,13 +8,13 @@ add_more_data_2
 inc_2
 BACKUP_CREATED
 inc_2_bad
-Couldn't unpack zip archive '02843_backup_use_same_password_for_base_backup_test_inc_1.zip': Password is required. (CANNOT_UNPACK_ARCHIVE)
+Couldn't unpack zip archive '02843_backup_use_same_password_for_base_backup_inc_1.zip': Password is required. (CANNOT_UNPACK_ARCHIVE)
 restore_inc_1
 RESTORED
 restore_inc_2
 RESTORED
 restore_inc_2_bad
-Couldn't unpack zip archive '02843_backup_use_same_password_for_base_backup_test_inc_1.zip': Password is required. (CANNOT_UNPACK_ARCHIVE)
+Couldn't unpack zip archive '02843_backup_use_same_password_for_base_backup_inc_1.zip': Password is required. (CANNOT_UNPACK_ARCHIVE)
 count_inc_1
 20
 count_inc_2
diff --git a/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.sh b/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.sh
index a2b1a953e24..4c5bec3775c 100755
--- a/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.sh
+++ b/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.sh
@@ -15,31 +15,31 @@ $CLICKHOUSE_CLIENT -nm -q "
 
 echo 'use_same_password_for_base_backup'
 echo "base"
-$CLICKHOUSE_CLIENT -q "BACKUP TABLE data TO Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_base.zip') SETTINGS password='password';" | cut -f2
+$CLICKHOUSE_CLIENT -q "BACKUP TABLE data TO Disk('backups', '${CLICKHOUSE_TEST_NAME}_base.zip') SETTINGS password='password';" | cut -f2
 
 echo 'add_more_data_1'
 $CLICKHOUSE_CLIENT -q "INSERT INTO data SELECT * FROM numbers(10,10);"
 
 echo "inc_1"
-$CLICKHOUSE_CLIENT -q "BACKUP TABLE data TO Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_1.zip') SETTINGS base_backup=Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_base.zip'),password='password',use_same_password_for_base_backup=1" | cut -f2
+$CLICKHOUSE_CLIENT -q "BACKUP TABLE data TO Disk('backups', '${CLICKHOUSE_TEST_NAME}_inc_1.zip') SETTINGS base_backup=Disk('backups', '${CLICKHOUSE_TEST_NAME}_base.zip'),password='password',use_same_password_for_base_backup=1" | cut -f2
 
 echo 'add_more_data_2'
 $CLICKHOUSE_CLIENT -q "INSERT INTO data SELECT * FROM numbers(20,10);"
 
 echo "inc_2"
-$CLICKHOUSE_CLIENT -q "BACKUP TABLE data TO Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_2.zip') SETTINGS base_backup=Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_1.zip'),password='password',use_same_password_for_base_backup=1" | cut -f2
+$CLICKHOUSE_CLIENT -q "BACKUP TABLE data TO Disk('backups', '${CLICKHOUSE_TEST_NAME}_inc_2.zip') SETTINGS base_backup=Disk('backups', '${CLICKHOUSE_TEST_NAME}_inc_1.zip'),password='password',use_same_password_for_base_backup=1" | cut -f2
 
 echo "inc_2_bad"
-$CLICKHOUSE_CLIENT -q "BACKUP TABLE data TO Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_2_bad.zip') SETTINGS base_backup=Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_1.zip'),password='password'" |& grep -m1 -o "Couldn't unpack zip archive '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_1.zip': Password is required. (CANNOT_UNPACK_ARCHIVE)"
+$CLICKHOUSE_CLIENT -q "BACKUP TABLE data TO Disk('backups', '${CLICKHOUSE_TEST_NAME}_inc_2_bad.zip') SETTINGS base_backup=Disk('backups', '${CLICKHOUSE_TEST_NAME}_inc_1.zip'),password='password'" |& grep -m1 -o "Couldn't unpack zip archive '${CLICKHOUSE_TEST_NAME}_inc_1.zip': Password is required. (CANNOT_UNPACK_ARCHIVE)"
 
 echo "restore_inc_1"
-$CLICKHOUSE_CLIENT -q "RESTORE TABLE data AS data_1 FROM Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_1.zip') SETTINGS password='password',use_same_password_for_base_backup=1" | cut -f2
+$CLICKHOUSE_CLIENT -q "RESTORE TABLE data AS data_1 FROM Disk('backups', '${CLICKHOUSE_TEST_NAME}_inc_1.zip') SETTINGS password='password',use_same_password_for_base_backup=1" | cut -f2
 
 echo "restore_inc_2"
-$CLICKHOUSE_CLIENT -q "RESTORE TABLE data AS data_2 FROM Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_2.zip') SETTINGS password='password',use_same_password_for_base_backup=1" | cut -f2
+$CLICKHOUSE_CLIENT -q "RESTORE TABLE data AS data_2 FROM Disk('backups', '${CLICKHOUSE_TEST_NAME}_inc_2.zip') SETTINGS password='password',use_same_password_for_base_backup=1" | cut -f2
 
 echo "restore_inc_2_bad"
-$CLICKHOUSE_CLIENT -q "RESTORE TABLE data AS data_2 FROM Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_2.zip') SETTINGS password='password'" |& grep -m1 -o "Couldn't unpack zip archive '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_1.zip': Password is required. (CANNOT_UNPACK_ARCHIVE)"
+$CLICKHOUSE_CLIENT -q "RESTORE TABLE data AS data_2 FROM Disk('backups', '${CLICKHOUSE_TEST_NAME}_inc_2.zip') SETTINGS password='password'" |& grep -m1 -o "Couldn't unpack zip archive '${CLICKHOUSE_TEST_NAME}_inc_1.zip': Password is required. (CANNOT_UNPACK_ARCHIVE)"
 
 echo "count_inc_1"
 $CLICKHOUSE_CLIENT -q "SELECT COUNT(*) FROM data_1" | cut -f2

From cdd955f421e442cdb776e95e8e7ae8608bc8636e Mon Sep 17 00:00:00 2001
From: Samuele Guerrini <sguerrini97@protonmail.com>
Date: Wed, 10 Jul 2024 09:20:39 +0200
Subject: [PATCH 0341/1488] Revert "use CLICKHOUSE_TEST_NAME instead of
 CLICKHOUSE_TEST_UNIQUE_NAME for backup filename"

This reverts commit 72f6368a2ede6a01390db770f1b9ddfa00d3f1fe.
---
 ...kup_use_same_password_for_base_backup.reference |  4 ++--
 ...843_backup_use_same_password_for_base_backup.sh | 14 +++++++-------
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.reference b/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.reference
index 1a331cca46b..7354d50a7c0 100644
--- a/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.reference
+++ b/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.reference
@@ -8,13 +8,13 @@ add_more_data_2
 inc_2
 BACKUP_CREATED
 inc_2_bad
-Couldn't unpack zip archive '02843_backup_use_same_password_for_base_backup_inc_1.zip': Password is required. (CANNOT_UNPACK_ARCHIVE)
+Couldn't unpack zip archive '02843_backup_use_same_password_for_base_backup_test_inc_1.zip': Password is required. (CANNOT_UNPACK_ARCHIVE)
 restore_inc_1
 RESTORED
 restore_inc_2
 RESTORED
 restore_inc_2_bad
-Couldn't unpack zip archive '02843_backup_use_same_password_for_base_backup_inc_1.zip': Password is required. (CANNOT_UNPACK_ARCHIVE)
+Couldn't unpack zip archive '02843_backup_use_same_password_for_base_backup_test_inc_1.zip': Password is required. (CANNOT_UNPACK_ARCHIVE)
 count_inc_1
 20
 count_inc_2
diff --git a/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.sh b/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.sh
index 4c5bec3775c..a2b1a953e24 100755
--- a/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.sh
+++ b/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.sh
@@ -15,31 +15,31 @@ $CLICKHOUSE_CLIENT -nm -q "
 
 echo 'use_same_password_for_base_backup'
 echo "base"
-$CLICKHOUSE_CLIENT -q "BACKUP TABLE data TO Disk('backups', '${CLICKHOUSE_TEST_NAME}_base.zip') SETTINGS password='password';" | cut -f2
+$CLICKHOUSE_CLIENT -q "BACKUP TABLE data TO Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_base.zip') SETTINGS password='password';" | cut -f2
 
 echo 'add_more_data_1'
 $CLICKHOUSE_CLIENT -q "INSERT INTO data SELECT * FROM numbers(10,10);"
 
 echo "inc_1"
-$CLICKHOUSE_CLIENT -q "BACKUP TABLE data TO Disk('backups', '${CLICKHOUSE_TEST_NAME}_inc_1.zip') SETTINGS base_backup=Disk('backups', '${CLICKHOUSE_TEST_NAME}_base.zip'),password='password',use_same_password_for_base_backup=1" | cut -f2
+$CLICKHOUSE_CLIENT -q "BACKUP TABLE data TO Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_1.zip') SETTINGS base_backup=Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_base.zip'),password='password',use_same_password_for_base_backup=1" | cut -f2
 
 echo 'add_more_data_2'
 $CLICKHOUSE_CLIENT -q "INSERT INTO data SELECT * FROM numbers(20,10);"
 
 echo "inc_2"
-$CLICKHOUSE_CLIENT -q "BACKUP TABLE data TO Disk('backups', '${CLICKHOUSE_TEST_NAME}_inc_2.zip') SETTINGS base_backup=Disk('backups', '${CLICKHOUSE_TEST_NAME}_inc_1.zip'),password='password',use_same_password_for_base_backup=1" | cut -f2
+$CLICKHOUSE_CLIENT -q "BACKUP TABLE data TO Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_2.zip') SETTINGS base_backup=Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_1.zip'),password='password',use_same_password_for_base_backup=1" | cut -f2
 
 echo "inc_2_bad"
-$CLICKHOUSE_CLIENT -q "BACKUP TABLE data TO Disk('backups', '${CLICKHOUSE_TEST_NAME}_inc_2_bad.zip') SETTINGS base_backup=Disk('backups', '${CLICKHOUSE_TEST_NAME}_inc_1.zip'),password='password'" |& grep -m1 -o "Couldn't unpack zip archive '${CLICKHOUSE_TEST_NAME}_inc_1.zip': Password is required. (CANNOT_UNPACK_ARCHIVE)"
+$CLICKHOUSE_CLIENT -q "BACKUP TABLE data TO Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_2_bad.zip') SETTINGS base_backup=Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_1.zip'),password='password'" |& grep -m1 -o "Couldn't unpack zip archive '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_1.zip': Password is required. (CANNOT_UNPACK_ARCHIVE)"
 
 echo "restore_inc_1"
-$CLICKHOUSE_CLIENT -q "RESTORE TABLE data AS data_1 FROM Disk('backups', '${CLICKHOUSE_TEST_NAME}_inc_1.zip') SETTINGS password='password',use_same_password_for_base_backup=1" | cut -f2
+$CLICKHOUSE_CLIENT -q "RESTORE TABLE data AS data_1 FROM Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_1.zip') SETTINGS password='password',use_same_password_for_base_backup=1" | cut -f2
 
 echo "restore_inc_2"
-$CLICKHOUSE_CLIENT -q "RESTORE TABLE data AS data_2 FROM Disk('backups', '${CLICKHOUSE_TEST_NAME}_inc_2.zip') SETTINGS password='password',use_same_password_for_base_backup=1" | cut -f2
+$CLICKHOUSE_CLIENT -q "RESTORE TABLE data AS data_2 FROM Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_2.zip') SETTINGS password='password',use_same_password_for_base_backup=1" | cut -f2
 
 echo "restore_inc_2_bad"
-$CLICKHOUSE_CLIENT -q "RESTORE TABLE data AS data_2 FROM Disk('backups', '${CLICKHOUSE_TEST_NAME}_inc_2.zip') SETTINGS password='password'" |& grep -m1 -o "Couldn't unpack zip archive '${CLICKHOUSE_TEST_NAME}_inc_1.zip': Password is required. (CANNOT_UNPACK_ARCHIVE)"
+$CLICKHOUSE_CLIENT -q "RESTORE TABLE data AS data_2 FROM Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_2.zip') SETTINGS password='password'" |& grep -m1 -o "Couldn't unpack zip archive '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_1.zip': Password is required. (CANNOT_UNPACK_ARCHIVE)"
 
 echo "count_inc_1"
 $CLICKHOUSE_CLIENT -q "SELECT COUNT(*) FROM data_1" | cut -f2

From ba1c6fe3ef3a7f263939b2ae9e6c249d9c6b38c0 Mon Sep 17 00:00:00 2001
From: Samuele Guerrini <sguerrini97@protonmail.com>
Date: Wed, 10 Jul 2024 09:23:19 +0200
Subject: [PATCH 0342/1488] shorter grep output

---
 .../02843_backup_use_same_password_for_base_backup.reference  | 4 ++--
 .../02843_backup_use_same_password_for_base_backup.sh         | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.reference b/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.reference
index 7354d50a7c0..cbcb6b4cb7c 100644
--- a/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.reference
+++ b/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.reference
@@ -8,13 +8,13 @@ add_more_data_2
 inc_2
 BACKUP_CREATED
 inc_2_bad
-Couldn't unpack zip archive '02843_backup_use_same_password_for_base_backup_test_inc_1.zip': Password is required. (CANNOT_UNPACK_ARCHIVE)
+_inc_1.zip': Password is required. (CANNOT_UNPACK_ARCHIVE)
 restore_inc_1
 RESTORED
 restore_inc_2
 RESTORED
 restore_inc_2_bad
-Couldn't unpack zip archive '02843_backup_use_same_password_for_base_backup_test_inc_1.zip': Password is required. (CANNOT_UNPACK_ARCHIVE)
+_inc_1.zip': Password is required. (CANNOT_UNPACK_ARCHIVE)
 count_inc_1
 20
 count_inc_2
diff --git a/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.sh b/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.sh
index a2b1a953e24..f2f1265c1a0 100755
--- a/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.sh
+++ b/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.sh
@@ -30,7 +30,7 @@ echo "inc_2"
 $CLICKHOUSE_CLIENT -q "BACKUP TABLE data TO Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_2.zip') SETTINGS base_backup=Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_1.zip'),password='password',use_same_password_for_base_backup=1" | cut -f2
 
 echo "inc_2_bad"
-$CLICKHOUSE_CLIENT -q "BACKUP TABLE data TO Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_2_bad.zip') SETTINGS base_backup=Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_1.zip'),password='password'" |& grep -m1 -o "Couldn't unpack zip archive '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_1.zip': Password is required. (CANNOT_UNPACK_ARCHIVE)"
+$CLICKHOUSE_CLIENT -q "BACKUP TABLE data TO Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_2_bad.zip') SETTINGS base_backup=Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_1.zip'),password='password'" |& grep -m1 -o "_inc_1.zip': Password is required. (CANNOT_UNPACK_ARCHIVE)"
 
 echo "restore_inc_1"
 $CLICKHOUSE_CLIENT -q "RESTORE TABLE data AS data_1 FROM Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_1.zip') SETTINGS password='password',use_same_password_for_base_backup=1" | cut -f2
@@ -39,7 +39,7 @@ echo "restore_inc_2"
 $CLICKHOUSE_CLIENT -q "RESTORE TABLE data AS data_2 FROM Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_2.zip') SETTINGS password='password',use_same_password_for_base_backup=1" | cut -f2
 
 echo "restore_inc_2_bad"
-$CLICKHOUSE_CLIENT -q "RESTORE TABLE data AS data_2 FROM Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_2.zip') SETTINGS password='password'" |& grep -m1 -o "Couldn't unpack zip archive '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_1.zip': Password is required. (CANNOT_UNPACK_ARCHIVE)"
+$CLICKHOUSE_CLIENT -q "RESTORE TABLE data AS data_2 FROM Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_2.zip') SETTINGS password='password'" |& grep -m1 -o "_inc_1.zip': Password is required. (CANNOT_UNPACK_ARCHIVE)"
 
 echo "count_inc_1"
 $CLICKHOUSE_CLIENT -q "SELECT COUNT(*) FROM data_1" | cut -f2

From 01c8faf190462d508d9cb1b7d342d6ccfbeeff88 Mon Sep 17 00:00:00 2001
From: Samuele Guerrini <sguerrini97@protonmail.com>
Date: Wed, 10 Jul 2024 11:15:33 +0200
Subject: [PATCH 0343/1488] Revert "shorter grep output"

This reverts commit b0cbf1495dd0bcaba828706e895347e4ec550e29.
---
 .../02843_backup_use_same_password_for_base_backup.reference  | 4 ++--
 .../02843_backup_use_same_password_for_base_backup.sh         | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.reference b/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.reference
index cbcb6b4cb7c..7354d50a7c0 100644
--- a/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.reference
+++ b/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.reference
@@ -8,13 +8,13 @@ add_more_data_2
 inc_2
 BACKUP_CREATED
 inc_2_bad
-_inc_1.zip': Password is required. (CANNOT_UNPACK_ARCHIVE)
+Couldn't unpack zip archive '02843_backup_use_same_password_for_base_backup_test_inc_1.zip': Password is required. (CANNOT_UNPACK_ARCHIVE)
 restore_inc_1
 RESTORED
 restore_inc_2
 RESTORED
 restore_inc_2_bad
-_inc_1.zip': Password is required. (CANNOT_UNPACK_ARCHIVE)
+Couldn't unpack zip archive '02843_backup_use_same_password_for_base_backup_test_inc_1.zip': Password is required. (CANNOT_UNPACK_ARCHIVE)
 count_inc_1
 20
 count_inc_2
diff --git a/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.sh b/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.sh
index f2f1265c1a0..a2b1a953e24 100755
--- a/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.sh
+++ b/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.sh
@@ -30,7 +30,7 @@ echo "inc_2"
 $CLICKHOUSE_CLIENT -q "BACKUP TABLE data TO Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_2.zip') SETTINGS base_backup=Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_1.zip'),password='password',use_same_password_for_base_backup=1" | cut -f2
 
 echo "inc_2_bad"
-$CLICKHOUSE_CLIENT -q "BACKUP TABLE data TO Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_2_bad.zip') SETTINGS base_backup=Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_1.zip'),password='password'" |& grep -m1 -o "_inc_1.zip': Password is required. (CANNOT_UNPACK_ARCHIVE)"
+$CLICKHOUSE_CLIENT -q "BACKUP TABLE data TO Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_2_bad.zip') SETTINGS base_backup=Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_1.zip'),password='password'" |& grep -m1 -o "Couldn't unpack zip archive '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_1.zip': Password is required. (CANNOT_UNPACK_ARCHIVE)"
 
 echo "restore_inc_1"
 $CLICKHOUSE_CLIENT -q "RESTORE TABLE data AS data_1 FROM Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_1.zip') SETTINGS password='password',use_same_password_for_base_backup=1" | cut -f2
@@ -39,7 +39,7 @@ echo "restore_inc_2"
 $CLICKHOUSE_CLIENT -q "RESTORE TABLE data AS data_2 FROM Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_2.zip') SETTINGS password='password',use_same_password_for_base_backup=1" | cut -f2
 
 echo "restore_inc_2_bad"
-$CLICKHOUSE_CLIENT -q "RESTORE TABLE data AS data_2 FROM Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_2.zip') SETTINGS password='password'" |& grep -m1 -o "_inc_1.zip': Password is required. (CANNOT_UNPACK_ARCHIVE)"
+$CLICKHOUSE_CLIENT -q "RESTORE TABLE data AS data_2 FROM Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_2.zip') SETTINGS password='password'" |& grep -m1 -o "Couldn't unpack zip archive '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_1.zip': Password is required. (CANNOT_UNPACK_ARCHIVE)"
 
 echo "count_inc_1"
 $CLICKHOUSE_CLIENT -q "SELECT COUNT(*) FROM data_1" | cut -f2

From 9dae370569054b4933b374c12dee6461c938fea8 Mon Sep 17 00:00:00 2001
From: Samuele Guerrini <sguerrini97@protonmail.com>
Date: Wed, 10 Jul 2024 11:23:36 +0200
Subject: [PATCH 0344/1488] fix db name in
 backup_use_same_password_for_base_backup test reference file

---
 .../02843_backup_use_same_password_for_base_backup.reference  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.reference b/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.reference
index 7354d50a7c0..cdcf0532cd9 100644
--- a/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.reference
+++ b/tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.reference
@@ -8,13 +8,13 @@ add_more_data_2
 inc_2
 BACKUP_CREATED
 inc_2_bad
-Couldn't unpack zip archive '02843_backup_use_same_password_for_base_backup_test_inc_1.zip': Password is required. (CANNOT_UNPACK_ARCHIVE)
+Couldn't unpack zip archive '02843_backup_use_same_password_for_base_backup_default_inc_1.zip': Password is required. (CANNOT_UNPACK_ARCHIVE)
 restore_inc_1
 RESTORED
 restore_inc_2
 RESTORED
 restore_inc_2_bad
-Couldn't unpack zip archive '02843_backup_use_same_password_for_base_backup_test_inc_1.zip': Password is required. (CANNOT_UNPACK_ARCHIVE)
+Couldn't unpack zip archive '02843_backup_use_same_password_for_base_backup_default_inc_1.zip': Password is required. (CANNOT_UNPACK_ARCHIVE)
 count_inc_1
 20
 count_inc_2

From b8d6c68f5fdc691f18000878203971689d2f812c Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Wed, 17 Jul 2024 10:30:03 +0200
Subject: [PATCH 0345/1488] Update
 02874_parquet_multiple_batches_array_inconsistent_offsets.reference

---
 ...arquet_multiple_batches_array_inconsistent_offsets.reference | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/02874_parquet_multiple_batches_array_inconsistent_offsets.reference b/tests/queries/0_stateless/02874_parquet_multiple_batches_array_inconsistent_offsets.reference
index ba63f2f7e9c..a9eae234dba 100644
--- a/tests/queries/0_stateless/02874_parquet_multiple_batches_array_inconsistent_offsets.reference
+++ b/tests/queries/0_stateless/02874_parquet_multiple_batches_array_inconsistent_offsets.reference
@@ -1,3 +1,3 @@
 Parquet
-e76a749f346078a6a43e0cbd25f0d18a  -
+3249508141921544766
 400

From f1350b442c79854de25a06efca66fc888225f93b Mon Sep 17 00:00:00 2001
From: cangyin <excangyin@gmail.com>
Date: Wed, 17 Jul 2024 17:30:27 +0800
Subject: [PATCH 0346/1488] Negative sign in prewhere optimization

---
 src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp b/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp
index e7b3585ecda..44ab12bde2a 100644
--- a/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp
+++ b/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp
@@ -233,7 +233,7 @@ static bool isConditionGood(const RPNBuilderTreeNode & condition, const NameSet
     else if (type == Field::Types::Float64)
     {
         const auto value = output_value.get<Float64>();
-        return value < threshold || threshold < value;
+        return value < -threshold || threshold < value;
     }
 
     return false;

From 005256429fb4d6a9aec88b3bb5662382166f6f9b Mon Sep 17 00:00:00 2001
From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com>
Date: Wed, 17 Jul 2024 11:49:15 +0200
Subject: [PATCH 0347/1488] Don't run test 03198_dynamic_read_subcolumns in
 azure

---
 tests/queries/0_stateless/03198_dynamic_read_subcolumns.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/03198_dynamic_read_subcolumns.sql b/tests/queries/0_stateless/03198_dynamic_read_subcolumns.sql
index 27fca179580..155009cc2fe 100644
--- a/tests/queries/0_stateless/03198_dynamic_read_subcolumns.sql
+++ b/tests/queries/0_stateless/03198_dynamic_read_subcolumns.sql
@@ -1,4 +1,4 @@
--- Tags: no-random-settings, no-s3-storage
+-- Tags: no-random-settings, no-object-storage
 
 SET allow_experimental_dynamic_type = 1;
 DROP TABLE IF EXISTS test_dynamic;

From 67852402aa2eb0b57e1f89ff78ab4a831cbf33c6 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Wed, 17 Jul 2024 10:30:56 +0000
Subject: [PATCH 0348/1488] Better (perhaps, if the stars align)

---
 .../InterpreterShowIndexesQuery.cpp           | 37 +++++++++++++++----
 1 file changed, 29 insertions(+), 8 deletions(-)

diff --git a/src/Interpreters/InterpreterShowIndexesQuery.cpp b/src/Interpreters/InterpreterShowIndexesQuery.cpp
index c6b5d5ac727..31f0404e123 100644
--- a/src/Interpreters/InterpreterShowIndexesQuery.cpp
+++ b/src/Interpreters/InterpreterShowIndexesQuery.cpp
@@ -33,12 +33,33 @@ String InterpreterShowIndexesQuery::getRewrittenQuery()
     String rewritten_query = fmt::format(R"(
 SELECT *
 FROM (
-        (SELECT
+        (WITH
+            t1 AS (
+                SELECT
+                    name,
+                    arrayJoin(splitByString(', ', primary_key)) AS pk_col
+                FROM
+                    system.tables
+                WHERE
+                    database = '{0}'
+                    AND name = '{1}'
+            ),
+            t2 AS (
+                SELECT
+                    name,
+                    pk_col,
+                    row_number() OVER (ORDER BY 1) AS row_num
+                FROM
+                    t1
+            )
+        SELECT
             name AS table,
             1 AS non_unique,
             'PRIMARY' AS key_name,
-            row_number() over (order by database) AS seq_in_index,
-            arrayJoin(splitByString(', ', primary_key)) AS column_name,
+            -- row_number() over (order by database) AS seq_in_index,
+            row_num AS seq_in_index,
+            -- arrayJoin(splitByString(', ', primary_key)) AS column_name,
+            pk_col,
             'A' AS collation,
             0 AS cardinality,
             NULL AS sub_part,
@@ -49,10 +70,9 @@ FROM (
             '' AS index_comment,
             'YES' AS visible,
             '' AS expression
-        FROM system.tables
-        WHERE
-            database = '{0}'
-            AND name = '{1}')
+        FROM
+            t2
+        )
     UNION ALL (
         SELECT
             table AS table,
@@ -70,7 +90,8 @@ FROM (
             '' AS index_comment,
             'YES' AS visible,
             expr AS expression
-        FROM system.data_skipping_indices
+        FROM
+            system.data_skipping_indices
         WHERE
             database = '{0}'
             AND table = '{1}'))

From 2958803255f0f646147eca4f9432f72496318bbf Mon Sep 17 00:00:00 2001
From: Max K <maxkaynov@gmail.com>
Date: Wed, 17 Jul 2024 11:26:38 +0200
Subject: [PATCH 0349/1488] do not decrease from lambda

---
 tests/ci/autoscale_runners_lambda/app.py      | 77 ++++++++++---------
 .../test_autoscale.py                         | 25 ++++--
 2 files changed, 60 insertions(+), 42 deletions(-)

diff --git a/tests/ci/autoscale_runners_lambda/app.py b/tests/ci/autoscale_runners_lambda/app.py
index 520abcb41bb..aed5cf3a8dd 100644
--- a/tests/ci/autoscale_runners_lambda/app.py
+++ b/tests/ci/autoscale_runners_lambda/app.py
@@ -99,7 +99,8 @@ def set_capacity(
             continue
         raise ValueError("Queue status is not in ['in_progress', 'queued']")
 
-    scale_down, scale_up = get_scales(runner_type)
+    #scale_down, scale_up = get_scales(runner_type)
+    _, scale_up = get_scales(runner_type)
     # With lyfecycle hooks some instances are actually free because some of
     # them are in 'Terminating:Wait' state
     effective_capacity = max(
@@ -110,7 +111,7 @@ def set_capacity(
     # How much nodes are free (positive) or need to be added (negative)
     capacity_reserve = effective_capacity - running - queued
     stop = False
-    if capacity_reserve < 0:
+    if capacity_reserve <= 0:
         # This part is about scaling up
         capacity_deficit = -capacity_reserve
         # It looks that we are still OK, since no queued jobs exist
@@ -158,41 +159,43 @@ def set_capacity(
             )
         return
 
-    # Now we will calculate if we need to scale down
-    stop = stop or asg["DesiredCapacity"] == asg["MinSize"]
-    new_capacity = asg["DesiredCapacity"] - (capacity_reserve // scale_down)
-    new_capacity = max(new_capacity, asg["MinSize"])
-    new_capacity = min(new_capacity, asg["MaxSize"])
-    stop = stop or asg["DesiredCapacity"] == new_capacity
-    if stop:
-        logging.info(
-            "Do not decrease ASG %s capacity, current capacity=%s, effective "
-            "capacity=%s, minimum capacity=%s, running jobs=%s, queue size=%s",
-            asg["AutoScalingGroupName"],
-            asg["DesiredCapacity"],
-            effective_capacity,
-            asg["MinSize"],
-            running,
-            queued,
-        )
-        return
-
-    logging.info(
-        "The ASG %s capacity will be decreased to %s, current capacity=%s, effective "
-        "capacity=%s, minimum capacity=%s, running jobs=%s, queue size=%s",
-        asg["AutoScalingGroupName"],
-        new_capacity,
-        asg["DesiredCapacity"],
-        effective_capacity,
-        asg["MinSize"],
-        running,
-        queued,
-    )
-    if not dry_run:
-        client.set_desired_capacity(
-            AutoScalingGroupName=asg["AutoScalingGroupName"],
-            DesiredCapacity=new_capacity,
-        )
+    # FIXME: try decreasing capacity from runners that finished their jobs and have no job assigned
+    #   IMPORTANT: Runner init script must be of version that supports ASG decrease
+    # # Now we will calculate if we need to scale down
+    # stop = stop or asg["DesiredCapacity"] == asg["MinSize"]
+    # new_capacity = asg["DesiredCapacity"] - (capacity_reserve // scale_down)
+    # new_capacity = max(new_capacity, asg["MinSize"])
+    # new_capacity = min(new_capacity, asg["MaxSize"])
+    # stop = stop or asg["DesiredCapacity"] == new_capacity
+    # if stop:
+    #     logging.info(
+    #         "Do not decrease ASG %s capacity, current capacity=%s, effective "
+    #         "capacity=%s, minimum capacity=%s, running jobs=%s, queue size=%s",
+    #         asg["AutoScalingGroupName"],
+    #         asg["DesiredCapacity"],
+    #         effective_capacity,
+    #         asg["MinSize"],
+    #         running,
+    #         queued,
+    #     )
+    #     return
+    #
+    # logging.info(
+    #     "The ASG %s capacity will be decreased to %s, current capacity=%s, effective "
+    #     "capacity=%s, minimum capacity=%s, running jobs=%s, queue size=%s",
+    #     asg["AutoScalingGroupName"],
+    #     new_capacity,
+    #     asg["DesiredCapacity"],
+    #     effective_capacity,
+    #     asg["MinSize"],
+    #     running,
+    #     queued,
+    # )
+    # if not dry_run:
+    #     client.set_desired_capacity(
+    #         AutoScalingGroupName=asg["AutoScalingGroupName"],
+    #         DesiredCapacity=new_capacity,
+    #     )
 
 
 def main(dry_run: bool = True) -> None:
diff --git a/tests/ci/autoscale_runners_lambda/test_autoscale.py b/tests/ci/autoscale_runners_lambda/test_autoscale.py
index 3aeab49ffc7..d1a1f9b358f 100644
--- a/tests/ci/autoscale_runners_lambda/test_autoscale.py
+++ b/tests/ci/autoscale_runners_lambda/test_autoscale.py
@@ -97,19 +97,34 @@ class TestSetCapacity(unittest.TestCase):
             ),
             TestCase("lower-min", 10, 5, 20, [Queue("queued", 5, "lower-min")], 10),
             # Decrease capacity
-            TestCase("w/reserve", 1, 13, 20, [Queue("queued", 5, "w/reserve")], 5),
+            # FIXME: Tests changed for lambda that can only scale up
+            # TestCase("w/reserve", 1, 13, 20, [Queue("queued", 5, "w/reserve")], 5),
+            TestCase("w/reserve", 1, 13, 20, [Queue("queued", 5, "w/reserve")], -1),
+            # TestCase(
+            #     "style-checker", 1, 13, 20, [Queue("queued", 5, "style-checker")], 5
+            # ),
             TestCase(
-                "style-checker", 1, 13, 20, [Queue("queued", 5, "style-checker")], 5
+                "style-checker", 1, 13, 20, [Queue("queued", 5, "style-checker")], -1
             ),
-            TestCase("w/reserve", 1, 23, 20, [Queue("queued", 17, "w/reserve")], 17),
-            TestCase("decrease", 1, 13, 20, [Queue("in_progress", 3, "decrease")], 3),
+            # TestCase("w/reserve", 1, 23, 20, [Queue("queued", 17, "w/reserve")], 17),
+            TestCase("w/reserve", 1, 23, 20, [Queue("queued", 17, "w/reserve")], -1),
+            # TestCase("decrease", 1, 13, 20, [Queue("in_progress", 3, "decrease")], 3),
+            TestCase("decrease", 1, 13, 20, [Queue("in_progress", 3, "decrease")], -1),
+            # TestCase(
+            #     "style-checker",
+            #     1,
+            #     13,
+            #     20,
+            #     [Queue("in_progress", 5, "style-checker")],
+            #     5,
+            # ),
             TestCase(
                 "style-checker",
                 1,
                 13,
                 20,
                 [Queue("in_progress", 5, "style-checker")],
-                5,
+                -1,
             ),
         )
         for t in test_cases:

From c40473ff80c0cdf09b85b7dda695a41ee2df0112 Mon Sep 17 00:00:00 2001
From: robot-clickhouse <robot-clickhouse@users.noreply.github.com>
Date: Wed, 17 Jul 2024 10:50:52 +0000
Subject: [PATCH 0350/1488] Automatic style fix

---
 tests/ci/autoscale_runners_lambda/app.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/ci/autoscale_runners_lambda/app.py b/tests/ci/autoscale_runners_lambda/app.py
index aed5cf3a8dd..c790a5081dd 100644
--- a/tests/ci/autoscale_runners_lambda/app.py
+++ b/tests/ci/autoscale_runners_lambda/app.py
@@ -99,7 +99,7 @@ def set_capacity(
             continue
         raise ValueError("Queue status is not in ['in_progress', 'queued']")
 
-    #scale_down, scale_up = get_scales(runner_type)
+    # scale_down, scale_up = get_scales(runner_type)
     _, scale_up = get_scales(runner_type)
     # With lyfecycle hooks some instances are actually free because some of
     # them are in 'Terminating:Wait' state

From 8a81946033eac461c155f388fa1595f8e42db794 Mon Sep 17 00:00:00 2001
From: MikhailBurdukov <burdukvmikhail@gmail.com>
Date: Wed, 17 Jul 2024 10:52:01 +0000
Subject: [PATCH 0351/1488] Rename opt

---
 src/Core/Settings.h                                            | 3 +--
 src/Core/SettingsChangesHistory.cpp                            | 2 +-
 src/Storages/StorageDictionary.cpp                             | 2 +-
 .../0_stateless/03198_dictionary_validate_primary_key_type.sql | 2 +-
 4 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/src/Core/Settings.h b/src/Core/Settings.h
index c116c888f28..9d2e3606e67 100644
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@@ -151,8 +151,7 @@ class IColumn;
     M(UInt64, max_local_write_bandwidth, 0, "The maximum speed of local writes in bytes per second.", 0) \
     M(Bool, stream_like_engine_allow_direct_select, false, "Allow direct SELECT query for Kafka, RabbitMQ, FileLog, Redis Streams, and NATS engines. In case there are attached materialized views, SELECT query is not allowed even if this setting is enabled.", 0) \
     M(String, stream_like_engine_insert_queue, "", "When stream like engine reads from multiple queues, user will need to select one queue to insert into when writing. Used by Redis Streams and NATS.", 0) \
-    M(Bool, dictionary_validate_id_type, false, "Validate primary key type for dictionaries. By default id type for simple layouts will be implicitly converted to UInt64.", 0) \
-    \
+    M(Bool, dictionary_validate_primary_key_type, false, "Validate primary key type for dictionaries. By default id type for simple layouts will be implicitly converted to UInt64.", 0) \
     M(Bool, distributed_insert_skip_read_only_replicas, false, "If true, INSERT into Distributed will skip read-only replicas.", 0) \
     M(Bool, distributed_foreground_insert, false, "If setting is enabled, insert query into distributed waits until data are sent to all nodes in a cluster. \n\nEnables or disables synchronous data insertion into a `Distributed` table.\n\nBy default, when inserting data into a Distributed table, the ClickHouse server sends data to cluster nodes in the background. When `distributed_foreground_insert` = 1, the data is processed synchronously, and the `INSERT` operation succeeds only after all the data is saved on all shards (at least one replica for each shard if `internal_replication` is true).", 0) ALIAS(insert_distributed_sync) \
     M(UInt64, distributed_background_insert_timeout, 0, "Timeout for insert query into distributed. Setting is used only with insert_distributed_sync enabled. Zero value means no timeout.", 0) ALIAS(insert_distributed_timeout) \
diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp
index 4e9e402f1d4..b2669a0e1c5 100644
--- a/src/Core/SettingsChangesHistory.cpp
+++ b/src/Core/SettingsChangesHistory.cpp
@@ -67,7 +67,7 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
               {"enable_named_columns_in_function_tuple", false, true, "Generate named tuples in function tuple() when all names are unique and can be treated as unquoted identifiers."},
               {"input_format_json_ignore_key_case", false, false, "Ignore json key case while read json field from string."},
               {"optimize_trivial_insert_select", true, false, "The optimization does not make sense in many cases."},
-              {"dictionary_validate_id_type", false, false, "Validate primary key type for dictionaries. By default id type for simple layouts will be implicitly converted to UInt64."},
+              {"dictionary_validate_primary_key_type", false, false, "Validate primary key type for dictionaries. By default id type for simple layouts will be implicitly converted to UInt64."},
               {"collect_hash_table_stats_during_joins", false, true, "New setting."},
               {"max_size_to_preallocate_for_joins", 0, 100'000'000, "New setting."},
               {"input_format_orc_read_use_writer_time_zone", false, false, "Whether use the writer's time zone in ORC stripe for ORC row reader, the default ORC row reader's time zone is GMT."},
diff --git a/src/Storages/StorageDictionary.cpp b/src/Storages/StorageDictionary.cpp
index 09ca53d7582..e7d8e9a6848 100644
--- a/src/Storages/StorageDictionary.cpp
+++ b/src/Storages/StorageDictionary.cpp
@@ -111,7 +111,7 @@ StorageDictionary::StorageDictionary(
     Location location_,
     ContextPtr context_)
     : StorageDictionary(
-        table_id_, dictionary_name_, ColumnsDescription{getNamesAndTypes(dictionary_structure_, context_->getSettingsRef().dictionary_validate_id_type)}, comment, location_, context_)
+        table_id_, dictionary_name_, ColumnsDescription{getNamesAndTypes(dictionary_structure_, context_->getSettingsRef().dictionary_validate_primary_key_type)}, comment, location_, context_)
 {
 }
 
diff --git a/tests/queries/0_stateless/03198_dictionary_validate_primary_key_type.sql b/tests/queries/0_stateless/03198_dictionary_validate_primary_key_type.sql
index eaf51ef7ead..e7b8730abbd 100644
--- a/tests/queries/0_stateless/03198_dictionary_validate_primary_key_type.sql
+++ b/tests/queries/0_stateless/03198_dictionary_validate_primary_key_type.sql
@@ -7,7 +7,7 @@ SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 DB 'test_db' TABLE 'table_01' USER
 LIFETIME(MIN 1 MAX 10)
 LAYOUT(FLAT());
 
-SET dictionary_validate_id_type=1;
+SET dictionary_validate_primary_key_type=1;
 
 CREATE DICTIONARY `test_dictionary1` (
     `n1` String,

From 3ade6de95df356adfaf7f6a6e3fb22d2545c6c50 Mon Sep 17 00:00:00 2001
From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com>
Date: Wed, 17 Jul 2024 13:13:51 +0200
Subject: [PATCH 0352/1488] Add description about no-object-storage tag

---
 tests/queries/0_stateless/03198_dynamic_read_subcolumns.sql | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/queries/0_stateless/03198_dynamic_read_subcolumns.sql b/tests/queries/0_stateless/03198_dynamic_read_subcolumns.sql
index 155009cc2fe..5d473064c68 100644
--- a/tests/queries/0_stateless/03198_dynamic_read_subcolumns.sql
+++ b/tests/queries/0_stateless/03198_dynamic_read_subcolumns.sql
@@ -1,4 +1,5 @@
 -- Tags: no-random-settings, no-object-storage
+-- Tag no-object-storage: this test relies on the number of opened files in MergeTree that can differ in object storages
 
 SET allow_experimental_dynamic_type = 1;
 DROP TABLE IF EXISTS test_dynamic;

From 7356e809aba191cdcff1785032be7c0e44f9eea3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?=
 <benjamin.antal@clickhouse.com>
Date: Wed, 17 Jul 2024 11:32:02 +0000
Subject: [PATCH 0353/1488] Ensure that `getPath` is not called with an
 absolute path

---
 .../NamedCollectionsMetadataStorage.cpp          | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/src/Common/NamedCollections/NamedCollectionsMetadataStorage.cpp b/src/Common/NamedCollections/NamedCollectionsMetadataStorage.cpp
index 79b674965de..189eb58196d 100644
--- a/src/Common/NamedCollections/NamedCollectionsMetadataStorage.cpp
+++ b/src/Common/NamedCollections/NamedCollectionsMetadataStorage.cpp
@@ -169,9 +169,13 @@ public:
     }
 
 private:
-    std::string getPath(const std::string & path) const
+    std::string getPath(const std::string & file_name) const
     {
-        return fs::path(root_path) / path;
+        const auto file_name_as_path = fs::path(file_name);
+        if (file_name_as_path.is_absolute())
+            throw Exception(ErrorCodes::LOGICAL_ERROR, "Filename {} cannot be an absolute path!", file_name);
+
+        return fs::path(root_path) / file_name_as_path;
     }
 
     /// Delete .tmp files. They could be left undeleted in case of
@@ -320,9 +324,13 @@ private:
         return zookeeper_client;
     }
 
-    std::string getPath(const std::string & path) const
+    std::string getPath(const std::string & file_name) const
     {
-        return fs::path(root_path) / path;
+        const auto file_name_as_path = fs::path(file_name);
+        if (file_name_as_path.is_absolute())
+            throw Exception(ErrorCodes::LOGICAL_ERROR, "Filename {} cannot be an absolute path!", file_name);
+
+        return fs::path(root_path) / file_name_as_path;
     }
 };
 

From e4816e5c2283dd6254e9d36991b60ad58c2c2dc0 Mon Sep 17 00:00:00 2001
From: Sema Checherinda <Sema.Checherinda@clickhouse.com>
Date: Wed, 17 Jul 2024 13:42:32 +0200
Subject: [PATCH 0354/1488] replace no-asan with no-azure-blob-storage tag in
 heavy tests

---
 .../03008_deduplication_insert_several_blocks_nonreplicated.sh  | 2 +-
 .../03008_deduplication_insert_several_blocks_replicated.sh     | 2 +-
 ...8_deduplication_mv_generates_several_blocks_nonreplicated.sh | 2 +-
 ...3008_deduplication_mv_generates_several_blocks_replicated.sh | 2 +-
 ...008_deduplication_several_mv_into_one_table_nonreplicated.sh | 2 +-
 .../03008_deduplication_several_mv_into_one_table_replicated.sh | 2 +-
 6 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_nonreplicated.sh b/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_nonreplicated.sh
index 0791c7566f9..f0a723501de 100755
--- a/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_nonreplicated.sh
+++ b/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_nonreplicated.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Tags: long, no-fasttest, no-parallel, no-asan
+# Tags: long, no-fasttest, no-parallel, no-azure-blob-storage
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
diff --git a/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_replicated.sh b/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_replicated.sh
index 661dfa5f930..1b7f9be9510 100755
--- a/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_replicated.sh
+++ b/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_replicated.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Tags: long, no-fasttest, no-parallel, no-asan
+# Tags: long, no-fasttest, no-parallel, no-azure-blob-storage
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
diff --git a/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_nonreplicated.sh b/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_nonreplicated.sh
index 10083506af4..e76b6eb77ce 100755
--- a/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_nonreplicated.sh
+++ b/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_nonreplicated.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Tags: long, no-fasttest, no-parallel, no-asan
+# Tags: long, no-fasttest, no-parallel, no-azure-blob-storage
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
diff --git a/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_replicated.sh b/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_replicated.sh
index 0d5158d18cd..7e6cf5e95c9 100755
--- a/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_replicated.sh
+++ b/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_replicated.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Tags: long, no-fasttest, no-parallel, no-asan
+# Tags: long, no-fasttest, no-parallel, no-azure-blob-storage
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
diff --git a/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_nonreplicated.sh b/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_nonreplicated.sh
index a9a135d6839..193e15d311c 100755
--- a/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_nonreplicated.sh
+++ b/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_nonreplicated.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Tags: long, no-fasttest, no-parallel, no-asan
+# Tags: long, no-fasttest, no-parallel, no-azure-blob-storage
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
diff --git a/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_replicated.sh b/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_replicated.sh
index 49d556e70f9..3f4f0b17258 100755
--- a/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_replicated.sh
+++ b/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_replicated.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Tags: long, no-fasttest, no-parallel, no-asan
+# Tags: long, no-fasttest, no-parallel, no-azure-blob-storage
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh

From 8211d8eabfed2fee115ec9d60b0692918547cd32 Mon Sep 17 00:00:00 2001
From: Konstantin Morozov <just.morozov.k@gmail.com>
Date: Wed, 17 Jul 2024 12:04:37 +0000
Subject: [PATCH 0355/1488] fix data race

---
 src/IO/S3/Client.cpp | 11 +++++++++++
 src/IO/S3/Client.h   | 11 ++++-------
 2 files changed, 15 insertions(+), 7 deletions(-)

diff --git a/src/IO/S3/Client.cpp b/src/IO/S3/Client.cpp
index 3b958dea046..b86e181c369 100644
--- a/src/IO/S3/Client.cpp
+++ b/src/IO/S3/Client.cpp
@@ -825,6 +825,17 @@ void Client::updateURIForBucket(const std::string & bucket, S3::URI new_uri) con
     cache->uri_for_bucket_cache.emplace(bucket, std::move(new_uri));
 }
 
+ClientCache::ClientCache(const ClientCache & other)
+{
+    {
+        std::lock_guard lock(region_cache_mutex);
+        region_for_bucket_cache = other.region_for_bucket_cache;
+    }
+    {
+        std::lock_guard lock(uri_cache_mutex);
+        uri_for_bucket_cache = other.uri_for_bucket_cache;
+    }
+}
 
 void ClientCache::clearCache()
 {
diff --git a/src/IO/S3/Client.h b/src/IO/S3/Client.h
index 2fd0a9cd2d1..6d97ec29ae6 100644
--- a/src/IO/S3/Client.h
+++ b/src/IO/S3/Client.h
@@ -54,10 +54,7 @@ struct ClientCache
 {
     ClientCache() = default;
 
-    ClientCache(const ClientCache & other)
-        : region_for_bucket_cache(other.region_for_bucket_cache)
-        , uri_for_bucket_cache(other.uri_for_bucket_cache)
-    {}
+    ClientCache(const ClientCache & other);
 
     ClientCache(ClientCache && other) = delete;
 
@@ -67,10 +64,10 @@ struct ClientCache
     void clearCache();
 
     std::mutex region_cache_mutex;
-    std::unordered_map<std::string, std::string> region_for_bucket_cache;
+    std::unordered_map<std::string, std::string> region_for_bucket_cache TSA_GUARDED_BY(region_cache_mutex);
 
     std::mutex uri_cache_mutex;
-    std::unordered_map<std::string, URI> uri_for_bucket_cache;
+    std::unordered_map<std::string, URI> uri_for_bucket_cache TSA_GUARDED_BY(uri_cache_mutex);
 };
 
 class ClientCacheRegistry
@@ -89,7 +86,7 @@ private:
     ClientCacheRegistry() = default;
 
     std::mutex clients_mutex;
-    std::unordered_map<ClientCache *, std::weak_ptr<ClientCache>> client_caches;
+    std::unordered_map<ClientCache *, std::weak_ptr<ClientCache>> client_caches TSA_GUARDED_BY(clients_mutex);
 };
 
 bool isS3ExpressEndpoint(const std::string & endpoint);

From 0de54b9d296927b7a8bc84bf1f8bfe5a5a01f080 Mon Sep 17 00:00:00 2001
From: Konstantin Morozov <just.morozov.k@gmail.com>
Date: Wed, 17 Jul 2024 13:01:04 +0000
Subject: [PATCH 0356/1488] fix comma

---
 src/IO/S3/Client.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/IO/S3/Client.cpp b/src/IO/S3/Client.cpp
index b86e181c369..6e659c82def 100644
--- a/src/IO/S3/Client.cpp
+++ b/src/IO/S3/Client.cpp
@@ -828,11 +828,11 @@ void Client::updateURIForBucket(const std::string & bucket, S3::URI new_uri) con
 ClientCache::ClientCache(const ClientCache & other)
 {
     {
-        std::lock_guard lock(region_cache_mutex);
+        std::lock_guard lock(other.region_cache_mutex);
         region_for_bucket_cache = other.region_for_bucket_cache;
     }
     {
-        std::lock_guard lock(uri_cache_mutex);
+        std::lock_guard lock(other.uri_cache_mutex);
         uri_for_bucket_cache = other.uri_for_bucket_cache;
     }
 }

From 523e0abb4ec329c0535602c43c17991f4ef043a3 Mon Sep 17 00:00:00 2001
From: Igor Nikonov <igor@clickhouse.com>
Date: Wed, 17 Jul 2024 13:15:14 +0000
Subject: [PATCH 0357/1488] Remove debug logs

---
 src/Processors/Sources/RemoteSource.cpp   |  7 +------
 src/QueryPipeline/RemoteQueryExecutor.cpp | 10 +---------
 2 files changed, 2 insertions(+), 15 deletions(-)

diff --git a/src/Processors/Sources/RemoteSource.cpp b/src/Processors/Sources/RemoteSource.cpp
index f1d47f69782..e33613564a2 100644
--- a/src/Processors/Sources/RemoteSource.cpp
+++ b/src/Processors/Sources/RemoteSource.cpp
@@ -90,10 +90,7 @@ ISource::Status RemoteSource::prepare()
 void RemoteSource::work()
 {
     if (async_immediate_work.exchange(false))
-    {
-        LOG_DEBUG(getLogger(__PRETTY_FUNCTION__), "async_immediate_work was true");
         return;
-    }
 
     /// Connection drain is a heavy operation that may take a long time.
     /// Therefore we move connection drain from prepare() to work(), and drain multiple connections in parallel.
@@ -114,13 +111,11 @@ void RemoteSource::asyncJobReady()
     if (!was_query_sent)
         return;
 
-    auto res = query_executor->readAsync(/*probe=*/true);
+    auto res = query_executor->readAsync(/*check_packet_type_only=*/true);
     if (res.type == RemoteQueryExecutor::ReadResult::Type::ParallelReplicasToken)
     {
-        LOG_DEBUG(getLogger(__PRETTY_FUNCTION__), "async_immediate_work is {}", async_immediate_work);
         work();
         async_immediate_work = true;
-        LOG_DEBUG(getLogger(__PRETTY_FUNCTION__), "async_immediate_work is true");
     }
 }
 
diff --git a/src/QueryPipeline/RemoteQueryExecutor.cpp b/src/QueryPipeline/RemoteQueryExecutor.cpp
index 3ca05b53417..87f634b8334 100644
--- a/src/QueryPipeline/RemoteQueryExecutor.cpp
+++ b/src/QueryPipeline/RemoteQueryExecutor.cpp
@@ -486,10 +486,7 @@ RemoteQueryExecutor::ReadResult RemoteQueryExecutor::readAsync(bool check_packet
     {
         std::lock_guard lock(was_cancelled_mutex);
         if (was_cancelled)
-        {
-            LOG_DEBUG(getLogger(__PRETTY_FUNCTION__), "was_cancelled");
             return ReadResult(Block());
-        }
 
         if (has_postponed_packet)
         {
@@ -520,17 +517,12 @@ RemoteQueryExecutor::ReadResult RemoteQueryExecutor::readAsync(bool check_packet
 
         /// Check if packet is not ready yet.
         if (read_context->isInProgress())
-        {
-            LOG_DEBUG(getLogger(__PRETTY_FUNCTION__), "read_context still in progress");
             return ReadResult(read_context->getFileDescriptor());
-        }
-
-        const auto packet_type = read_context->getPacketType();
-        LOG_DEBUG(getLogger(__PRETTY_FUNCTION__), "Packet type: {}", packet_type);
 
         if (check_packet_type_only)
         {
             has_postponed_packet = true;
+            const auto packet_type = read_context->getPacketType();
             if (packet_type == Protocol::Server::MergeTreeReadTaskRequest
                 || packet_type == Protocol::Server::MergeTreeAllRangesAnnouncement)
             {

From baade8baf45c7919f106160b8b8633d2c59e3ae8 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Wed, 17 Jul 2024 13:22:50 +0000
Subject: [PATCH 0358/1488] Replace updateWeakHash to getWeakHash

---
 src/Columns/ColumnAggregateFunction.cpp       |  9 +--
 src/Columns/ColumnAggregateFunction.h         |  2 +-
 src/Columns/ColumnArray.cpp                   | 11 ++-
 src/Columns/ColumnArray.h                     |  2 +-
 src/Columns/ColumnCompressed.h                |  3 +-
 src/Columns/ColumnConst.cpp                   | 14 +---
 src/Columns/ColumnConst.h                     |  2 +-
 src/Columns/ColumnDecimal.cpp                 |  9 +--
 src/Columns/ColumnDecimal.h                   |  2 +-
 src/Columns/ColumnDynamic.h                   |  5 +-
 src/Columns/ColumnFixedString.cpp             | 10 +--
 src/Columns/ColumnFixedString.h               |  2 +-
 src/Columns/ColumnFunction.h                  |  5 +-
 src/Columns/ColumnLowCardinality.cpp          | 26 +++---
 src/Columns/ColumnLowCardinality.h            |  4 +-
 src/Columns/ColumnMap.cpp                     |  4 +-
 src/Columns/ColumnMap.h                       |  2 +-
 src/Columns/ColumnNullable.cpp                | 16 ++--
 src/Columns/ColumnNullable.h                  |  2 +-
 src/Columns/ColumnObject.h                    |  3 +-
 src/Columns/ColumnSparse.cpp                  | 21 ++---
 src/Columns/ColumnSparse.h                    |  2 +-
 src/Columns/ColumnString.cpp                  |  9 +--
 src/Columns/ColumnString.h                    |  2 +-
 src/Columns/ColumnTuple.cpp                   | 11 ++-
 src/Columns/ColumnTuple.h                     |  2 +-
 src/Columns/ColumnVariant.cpp                 | 27 ++-----
 src/Columns/ColumnVariant.h                   |  2 +-
 src/Columns/ColumnVector.cpp                  |  9 +--
 src/Columns/ColumnVector.h                    |  2 +-
 src/Columns/IColumn.h                         |  4 +-
 src/Columns/IColumnDummy.h                    |  4 +-
 src/Columns/IColumnUnique.h                   |  5 +-
 src/Columns/tests/gtest_weak_hash_32.cpp      | 81 +++++++------------
 src/Common/WeakHash.cpp                       | 22 +++++
 src/Common/WeakHash.h                         |  5 +-
 src/Interpreters/ConcurrentHashJoin.cpp       |  2 +-
 src/Interpreters/JoinUtils.cpp                |  2 +-
 .../ScatterByPartitionTransform.cpp           |  2 +-
 39 files changed, 150 insertions(+), 197 deletions(-)

diff --git a/src/Columns/ColumnAggregateFunction.cpp b/src/Columns/ColumnAggregateFunction.cpp
index cfd07c27765..33bd1266c90 100644
--- a/src/Columns/ColumnAggregateFunction.cpp
+++ b/src/Columns/ColumnAggregateFunction.cpp
@@ -366,13 +366,10 @@ void ColumnAggregateFunction::updateHashWithValue(size_t n, SipHash & hash) cons
     hash.update(wbuf.str().c_str(), wbuf.str().size());
 }
 
-void ColumnAggregateFunction::updateWeakHash32(WeakHash32 & hash) const
+WeakHash32 ColumnAggregateFunction::getWeakHash32() const
 {
     auto s = data.size();
-    if (hash.getData().size() != data.size())
-        throw Exception(ErrorCodes::LOGICAL_ERROR, "Size of WeakHash32 does not match size of column: "
-                        "column size is {}, hash size is {}", std::to_string(s), hash.getData().size());
-
+    WeakHash32 hash(s);
     auto & hash_data = hash.getData();
 
     std::vector<UInt8> v;
@@ -383,6 +380,8 @@ void ColumnAggregateFunction::updateWeakHash32(WeakHash32 & hash) const
         wbuf.finalize();
         hash_data[i] = ::updateWeakHash32(v.data(), v.size(), hash_data[i]);
     }
+
+    return hash;
 }
 
 void ColumnAggregateFunction::updateHashFast(SipHash & hash) const
diff --git a/src/Columns/ColumnAggregateFunction.h b/src/Columns/ColumnAggregateFunction.h
index 1be7a862438..330a707b75c 100644
--- a/src/Columns/ColumnAggregateFunction.h
+++ b/src/Columns/ColumnAggregateFunction.h
@@ -177,7 +177,7 @@ public:
 
     void updateHashWithValue(size_t n, SipHash & hash) const override;
 
-    void updateWeakHash32(WeakHash32 & hash) const override;
+    WeakHash32 getWeakHash32() const override;
 
     void updateHashFast(SipHash & hash) const override;
 
diff --git a/src/Columns/ColumnArray.cpp b/src/Columns/ColumnArray.cpp
index 5d7350f3a79..9203fb8042f 100644
--- a/src/Columns/ColumnArray.cpp
+++ b/src/Columns/ColumnArray.cpp
@@ -271,15 +271,12 @@ void ColumnArray::updateHashWithValue(size_t n, SipHash & hash) const
         getData().updateHashWithValue(offset + i, hash);
 }
 
-void ColumnArray::updateWeakHash32(WeakHash32 & hash) const
+WeakHash32 ColumnArray::getWeakHash32() const
 {
     auto s = offsets->size();
-    if (hash.getData().size() != s)
-        throw Exception(ErrorCodes::LOGICAL_ERROR, "Size of WeakHash32 does not match size of column: "
-                        "column size is {}, hash size is {}", s, hash.getData().size());
+    WeakHash32 hash(s);
 
-    WeakHash32 internal_hash(data->size());
-    data->updateWeakHash32(internal_hash);
+    WeakHash32 internal_hash = data->getWeakHash32();
 
     Offset prev_offset = 0;
     const auto & offsets_data = getOffsets();
@@ -300,6 +297,8 @@ void ColumnArray::updateWeakHash32(WeakHash32 & hash) const
 
         prev_offset = offsets_data[i];
     }
+
+    return hash;
 }
 
 void ColumnArray::updateHashFast(SipHash & hash) const
diff --git a/src/Columns/ColumnArray.h b/src/Columns/ColumnArray.h
index 6cd3e2f6c3b..5e01b9144d7 100644
--- a/src/Columns/ColumnArray.h
+++ b/src/Columns/ColumnArray.h
@@ -82,7 +82,7 @@ public:
     const char * deserializeAndInsertFromArena(const char * pos) override;
     const char * skipSerializedInArena(const char * pos) const override;
     void updateHashWithValue(size_t n, SipHash & hash) const override;
-    void updateWeakHash32(WeakHash32 & hash) const override;
+    WeakHash32 getWeakHash32() const override;
     void updateHashFast(SipHash & hash) const override;
 #if !defined(ABORT_ON_LOGICAL_ERROR)
     void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
diff --git a/src/Columns/ColumnCompressed.h b/src/Columns/ColumnCompressed.h
index 5e455709fec..19470113394 100644
--- a/src/Columns/ColumnCompressed.h
+++ b/src/Columns/ColumnCompressed.h
@@ -3,6 +3,7 @@
 #include <optional>
 #include <Core/Field.h>
 #include <Columns/IColumn.h>
+#include <Common/WeakHash.h>
 #include <IO/BufferWithOwnMemory.h>
 
 
@@ -98,7 +99,7 @@ public:
     const char * deserializeAndInsertFromArena(const char *) override { throwMustBeDecompressed(); }
     const char * skipSerializedInArena(const char *) const override { throwMustBeDecompressed(); }
     void updateHashWithValue(size_t, SipHash &) const override { throwMustBeDecompressed(); }
-    void updateWeakHash32(WeakHash32 &) const override { throwMustBeDecompressed(); }
+    WeakHash32 getWeakHash32() const override { throwMustBeDecompressed(); }
     void updateHashFast(SipHash &) const override { throwMustBeDecompressed(); }
     ColumnPtr filter(const Filter &, ssize_t) const override { throwMustBeDecompressed(); }
     void expand(const Filter &, bool) override { throwMustBeDecompressed(); }
diff --git a/src/Columns/ColumnConst.cpp b/src/Columns/ColumnConst.cpp
index f2cea83db0e..84427e7be2b 100644
--- a/src/Columns/ColumnConst.cpp
+++ b/src/Columns/ColumnConst.cpp
@@ -137,18 +137,10 @@ void ColumnConst::updatePermutation(PermutationSortDirection /*direction*/, Perm
 {
 }
 
-void ColumnConst::updateWeakHash32(WeakHash32 & hash) const
+WeakHash32 ColumnConst::getWeakHash32() const
 {
-    if (hash.getData().size() != s)
-        throw Exception(ErrorCodes::LOGICAL_ERROR, "Size of WeakHash32 does not match size of column: "
-                        "column size is {}, hash size is {}", std::to_string(s), std::to_string(hash.getData().size()));
-
-    WeakHash32 element_hash(1);
-    data->updateWeakHash32(element_hash);
-    size_t data_hash = element_hash.getData()[0];
-
-    for (auto & value : hash.getData())
-        value = static_cast<UInt32>(intHashCRC32(data_hash, value));
+    WeakHash32 element_hash = data->getWeakHash32();
+    return WeakHash32(s, element_hash.getData()[0]);
 }
 
 void ColumnConst::compareColumn(
diff --git a/src/Columns/ColumnConst.h b/src/Columns/ColumnConst.h
index b55a1f42037..65ce53687b9 100644
--- a/src/Columns/ColumnConst.h
+++ b/src/Columns/ColumnConst.h
@@ -204,7 +204,7 @@ public:
         data->updateHashWithValue(0, hash);
     }
 
-    void updateWeakHash32(WeakHash32 & hash) const override;
+    WeakHash32 getWeakHash32() const override;
 
     void updateHashFast(SipHash & hash) const override
     {
diff --git a/src/Columns/ColumnDecimal.cpp b/src/Columns/ColumnDecimal.cpp
index cf413f790a7..ed9c699a841 100644
--- a/src/Columns/ColumnDecimal.cpp
+++ b/src/Columns/ColumnDecimal.cpp
@@ -76,13 +76,10 @@ void ColumnDecimal<T>::updateHashWithValue(size_t n, SipHash & hash) const
 }
 
 template <is_decimal T>
-void ColumnDecimal<T>::updateWeakHash32(WeakHash32 & hash) const
+WeakHash32 ColumnDecimal<T>::getWeakHash32() const
 {
     auto s = data.size();
-
-    if (hash.getData().size() != s)
-        throw Exception(ErrorCodes::LOGICAL_ERROR, "Size of WeakHash32 does not match size of column: "
-                        "column size is {}, hash size is {}", std::to_string(s), std::to_string(hash.getData().size()));
+    WeakHash32 hash(s);
 
     const T * begin = data.data();
     const T * end = begin + s;
@@ -94,6 +91,8 @@ void ColumnDecimal<T>::updateWeakHash32(WeakHash32 & hash) const
         ++begin;
         ++hash_data;
     }
+
+    return hash;
 }
 
 template <is_decimal T>
diff --git a/src/Columns/ColumnDecimal.h b/src/Columns/ColumnDecimal.h
index 32efeb643a6..eb8a305a822 100644
--- a/src/Columns/ColumnDecimal.h
+++ b/src/Columns/ColumnDecimal.h
@@ -102,7 +102,7 @@ public:
     const char * deserializeAndInsertFromArena(const char * pos) override;
     const char * skipSerializedInArena(const char * pos) const override;
     void updateHashWithValue(size_t n, SipHash & hash) const override;
-    void updateWeakHash32(WeakHash32 & hash) const override;
+    WeakHash32 getWeakHash32() const override;
     void updateHashFast(SipHash & hash) const override;
 #if !defined(ABORT_ON_LOGICAL_ERROR)
     int compareAt(size_t n, size_t m, const IColumn & rhs_, int nan_direction_hint) const override;
diff --git a/src/Columns/ColumnDynamic.h b/src/Columns/ColumnDynamic.h
index 9abddc7a26d..6f09abb945a 100644
--- a/src/Columns/ColumnDynamic.h
+++ b/src/Columns/ColumnDynamic.h
@@ -4,6 +4,7 @@
 #include <Columns/ColumnVector.h>
 #include <Columns/ColumnVariant.h>
 #include <DataTypes/IDataType.h>
+#include <Common/WeakHash.h>
 
 
 namespace DB
@@ -174,9 +175,9 @@ public:
 
     void updateHashWithValue(size_t n, SipHash & hash) const override;
 
-    void updateWeakHash32(WeakHash32 & hash) const override
+    WeakHash32 getWeakHash32() const override
     {
-        variant_column->updateWeakHash32(hash);
+        return variant_column->getWeakHash32();
     }
 
     void updateHashFast(SipHash & hash) const override
diff --git a/src/Columns/ColumnFixedString.cpp b/src/Columns/ColumnFixedString.cpp
index 1c2de203a94..4d17eb0bebd 100644
--- a/src/Columns/ColumnFixedString.cpp
+++ b/src/Columns/ColumnFixedString.cpp
@@ -137,14 +137,10 @@ void ColumnFixedString::updateHashWithValue(size_t index, SipHash & hash) const
     hash.update(reinterpret_cast<const char *>(&chars[n * index]), n);
 }
 
-void ColumnFixedString::updateWeakHash32(WeakHash32 & hash) const
+WeakHash32 ColumnFixedString::getWeakHash32() const
 {
     auto s = size();
-
-    if (hash.getData().size() != s)
-        throw Exception(ErrorCodes::LOGICAL_ERROR, "Size of WeakHash32 does not match size of column: "
-                        "column size is {}, "
-                        "hash size is {}", std::to_string(s), std::to_string(hash.getData().size()));
+    WeakHash32 hash(s);
 
     const UInt8 * pos = chars.data();
     UInt32 * hash_data = hash.getData().data();
@@ -156,6 +152,8 @@ void ColumnFixedString::updateWeakHash32(WeakHash32 & hash) const
         pos += n;
         ++hash_data;
     }
+
+    return hash;
 }
 
 void ColumnFixedString::updateHashFast(SipHash & hash) const
diff --git a/src/Columns/ColumnFixedString.h b/src/Columns/ColumnFixedString.h
index 6e88136fc50..630c6c1c0a6 100644
--- a/src/Columns/ColumnFixedString.h
+++ b/src/Columns/ColumnFixedString.h
@@ -133,7 +133,7 @@ public:
 
     void updateHashWithValue(size_t index, SipHash & hash) const override;
 
-    void updateWeakHash32(WeakHash32 & hash) const override;
+    WeakHash32 getWeakHash32() const override;
 
     void updateHashFast(SipHash & hash) const override;
 
diff --git a/src/Columns/ColumnFunction.h b/src/Columns/ColumnFunction.h
index ba924c49a82..dcd67aecad7 100644
--- a/src/Columns/ColumnFunction.h
+++ b/src/Columns/ColumnFunction.h
@@ -4,6 +4,7 @@
 #include <Core/NamesAndTypes.h>
 #include <Core/ColumnsWithTypeAndName.h>
 #include <Columns/IColumn.h>
+#include <Common/WeakHash.h>
 
 
 namespace DB
@@ -130,9 +131,9 @@ public:
         throw Exception(ErrorCodes::NOT_IMPLEMENTED, "updateHashWithValue is not implemented for {}", getName());
     }
 
-    void updateWeakHash32(WeakHash32 &) const override
+    WeakHash32 getWeakHash32() const override
     {
-        throw Exception(ErrorCodes::NOT_IMPLEMENTED, "updateWeakHash32 is not implemented for {}", getName());
+        throw Exception(ErrorCodes::NOT_IMPLEMENTED, "getWeakHash32 is not implemented for {}", getName());
     }
 
     void updateHashFast(SipHash &) const override
diff --git a/src/Columns/ColumnLowCardinality.cpp b/src/Columns/ColumnLowCardinality.cpp
index eb694a10b0f..49ce948bf78 100644
--- a/src/Columns/ColumnLowCardinality.cpp
+++ b/src/Columns/ColumnLowCardinality.cpp
@@ -7,8 +7,7 @@
 #include <Common/HashTable/HashMap.h>
 #include <Common/WeakHash.h>
 #include <Common/assert_cast.h>
-#include "Storages/IndicesDescription.h"
-#include "base/types.h"
+#include <base/types.h>
 #include <base/sort.h>
 #include <base/scope_guard.h>
 
@@ -320,19 +319,10 @@ const char * ColumnLowCardinality::skipSerializedInArena(const char * pos) const
     return getDictionary().skipSerializedInArena(pos);
 }
 
-void ColumnLowCardinality::updateWeakHash32(WeakHash32 & hash) const
+WeakHash32 ColumnLowCardinality::getWeakHash32() const
 {
-    auto s = size();
-
-    if (hash.getData().size() != s)
-        throw Exception(ErrorCodes::LOGICAL_ERROR, "Size of WeakHash32 does not match size of column: "
-                        "column size is {}, hash size is {}", std::to_string(s), std::to_string(hash.getData().size()));
-
-    const auto & dict = getDictionary().getNestedColumn();
-    WeakHash32 dict_hash(dict->size());
-    dict->updateWeakHash32(dict_hash);
-
-    idx.updateWeakHash(hash, dict_hash);
+    WeakHash32 dict_hash = getDictionary().getNestedColumn()->getWeakHash32();
+    return idx.getWeakHash(dict_hash);
 }
 
 void ColumnLowCardinality::updateHashFast(SipHash & hash) const
@@ -832,10 +822,11 @@ bool ColumnLowCardinality::Index::containsDefault() const
     return contains;
 }
 
-void ColumnLowCardinality::Index::updateWeakHash(WeakHash32 & hash, WeakHash32 & dict_hash) const
+WeakHash32 ColumnLowCardinality::Index::getWeakHash(const WeakHash32 & dict_hash) const
 {
+    WeakHash32 hash(positions->size());
     auto & hash_data = hash.getData();
-    auto & dict_hash_data = dict_hash.getData();
+    const auto & dict_hash_data = dict_hash.getData();
 
     auto update_weak_hash = [&](auto x)
     {
@@ -844,10 +835,11 @@ void ColumnLowCardinality::Index::updateWeakHash(WeakHash32 & hash, WeakHash32 &
         auto size = data.size();
 
         for (size_t i = 0; i < size; ++i)
-            hash_data[i] = static_cast<UInt32>(intHashCRC32(dict_hash_data[data[i]], hash_data[i]));
+            hash_data[i] = dict_hash_data[data[i]];
     };
 
     callForType(std::move(update_weak_hash), size_of_type);
+    return hash;
 }
 
 void ColumnLowCardinality::Index::collectSerializedValueSizes(
diff --git a/src/Columns/ColumnLowCardinality.h b/src/Columns/ColumnLowCardinality.h
index e99be07cd8d..fb0c1237fcf 100644
--- a/src/Columns/ColumnLowCardinality.h
+++ b/src/Columns/ColumnLowCardinality.h
@@ -111,7 +111,7 @@ public:
         getDictionary().updateHashWithValue(getIndexes().getUInt(n), hash);
     }
 
-    void updateWeakHash32(WeakHash32 & hash) const override;
+    WeakHash32 getWeakHash32() const override;
 
     void updateHashFast(SipHash &) const override;
 
@@ -325,7 +325,7 @@ public:
 
         bool containsDefault() const;
 
-        void updateWeakHash(WeakHash32 & hash, WeakHash32 & dict_hash) const;
+        WeakHash32 getWeakHash(const WeakHash32 & dict_hash) const;
 
         void collectSerializedValueSizes(PaddedPODArray<UInt64> & sizes, const PaddedPODArray<UInt64> & dict_sizes) const;
 
diff --git a/src/Columns/ColumnMap.cpp b/src/Columns/ColumnMap.cpp
index 2dffddb2dc9..08d7734ac6b 100644
--- a/src/Columns/ColumnMap.cpp
+++ b/src/Columns/ColumnMap.cpp
@@ -143,9 +143,9 @@ void ColumnMap::updateHashWithValue(size_t n, SipHash & hash) const
     nested->updateHashWithValue(n, hash);
 }
 
-void ColumnMap::updateWeakHash32(WeakHash32 & hash) const
+WeakHash32 ColumnMap::getWeakHash32() const
 {
-    nested->updateWeakHash32(hash);
+    return nested->getWeakHash32();
 }
 
 void ColumnMap::updateHashFast(SipHash & hash) const
diff --git a/src/Columns/ColumnMap.h b/src/Columns/ColumnMap.h
index a54071a2974..29275e1b5f7 100644
--- a/src/Columns/ColumnMap.h
+++ b/src/Columns/ColumnMap.h
@@ -64,7 +64,7 @@ public:
     const char * deserializeAndInsertFromArena(const char * pos) override;
     const char * skipSerializedInArena(const char * pos) const override;
     void updateHashWithValue(size_t n, SipHash & hash) const override;
-    void updateWeakHash32(WeakHash32 & hash) const override;
+    WeakHash32 getWeakHash32() const override;
     void updateHashFast(SipHash & hash) const override;
 
 #if !defined(ABORT_ON_LOGICAL_ERROR)
diff --git a/src/Columns/ColumnNullable.cpp b/src/Columns/ColumnNullable.cpp
index f060e74b315..64e99a3bbe8 100644
--- a/src/Columns/ColumnNullable.cpp
+++ b/src/Columns/ColumnNullable.cpp
@@ -56,25 +56,21 @@ void ColumnNullable::updateHashWithValue(size_t n, SipHash & hash) const
         getNestedColumn().updateHashWithValue(n, hash);
 }
 
-void ColumnNullable::updateWeakHash32(WeakHash32 & hash) const
+WeakHash32 ColumnNullable::getWeakHash32() const
 {
     auto s = size();
 
-    if (hash.getData().size() != s)
-        throw Exception(ErrorCodes::LOGICAL_ERROR, "Size of WeakHash32 does not match size of column: "
-                        "column size is {}, hash size is {}", std::to_string(s), std::to_string(hash.getData().size()));
-
-    WeakHash32 old_hash = hash;
-    nested_column->updateWeakHash32(hash);
+    WeakHash32 hash = nested_column->getWeakHash32();
 
     const auto & null_map_data = getNullMapData();
     auto & hash_data = hash.getData();
-    auto & old_hash_data = old_hash.getData();
 
-    /// Use old data for nulls.
+    /// Use defualt for nulls.
     for (size_t row = 0; row < s; ++row)
         if (null_map_data[row])
-            hash_data[row] = old_hash_data[row];
+            hash_data[row] = WeakHash32::kDefaultInitialValue;
+
+    return hash;
 }
 
 void ColumnNullable::updateHashFast(SipHash & hash) const
diff --git a/src/Columns/ColumnNullable.h b/src/Columns/ColumnNullable.h
index a6d0483e527..15bbd8c3b57 100644
--- a/src/Columns/ColumnNullable.h
+++ b/src/Columns/ColumnNullable.h
@@ -133,7 +133,7 @@ public:
     void protect() override;
     ColumnPtr replicate(const Offsets & replicate_offsets) const override;
     void updateHashWithValue(size_t n, SipHash & hash) const override;
-    void updateWeakHash32(WeakHash32 & hash) const override;
+    WeakHash32 getWeakHash32() const override;
     void updateHashFast(SipHash & hash) const override;
     void getExtremes(Field & min, Field & max) const override;
     // Special function for nullable minmax index
diff --git a/src/Columns/ColumnObject.h b/src/Columns/ColumnObject.h
index fadf2e18779..21607e003f2 100644
--- a/src/Columns/ColumnObject.h
+++ b/src/Columns/ColumnObject.h
@@ -5,6 +5,7 @@
 #include <Core/Names.h>
 #include <DataTypes/Serializations/SubcolumnsTree.h>
 #include <Common/PODArray.h>
+#include <Common/WeakHash.h>
 
 #include <DataTypes/IDataType.h>
 
@@ -252,7 +253,7 @@ public:
     const char * deserializeAndInsertFromArena(const char *) override { throwMustBeConcrete(); }
     const char * skipSerializedInArena(const char *) const override { throwMustBeConcrete(); }
     void updateHashWithValue(size_t, SipHash &) const override { throwMustBeConcrete(); }
-    void updateWeakHash32(WeakHash32 &) const override { throwMustBeConcrete(); }
+    WeakHash32 getWeakHash32() const override { throwMustBeConcrete(); }
     void updateHashFast(SipHash &) const override { throwMustBeConcrete(); }
     void expand(const Filter &, bool) override { throwMustBeConcrete(); }
     bool hasEqualValues() const override { throwMustBeConcrete(); }
diff --git a/src/Columns/ColumnSparse.cpp b/src/Columns/ColumnSparse.cpp
index 98a66e87387..0d103a263dd 100644
--- a/src/Columns/ColumnSparse.cpp
+++ b/src/Columns/ColumnSparse.cpp
@@ -678,26 +678,13 @@ void ColumnSparse::updateHashWithValue(size_t n, SipHash & hash) const
     values->updateHashWithValue(getValueIndex(n), hash);
 }
 
-void ColumnSparse::updateWeakHash32(WeakHash32 & hash) const
+WeakHash32 ColumnSparse::getWeakHash32() const
 {
-    if (hash.getData().size() != _size)
-        throw Exception(ErrorCodes::LOGICAL_ERROR, "Size of WeakHash32 does not match size of column: "
-        "column size is {}, hash size is {}", _size, hash.getData().size());
-
-    size_t values_size = values->size();
-    WeakHash32 values_hash(values_size);
+    WeakHash32 values_hash = values->getWeakHash32();
+    WeakHash32 hash(size());
 
     auto & hash_data = hash.getData();
     auto & values_hash_data = values_hash.getData();
-    const auto & offsets_data = getOffsetsData();
-
-    if (getNumberOfDefaultRows() > 0)
-        values_hash_data[0] = hash_data[getFirstDefaultValueIndex()];
-
-    for (size_t i = 0; i + 1 < values_size; ++i)
-        values_hash_data[i + 1] = hash_data[offsets_data[i]];
-
-    values->updateWeakHash32(values_hash);
 
     auto offset_it = begin();
     for (size_t i = 0; i < _size; ++i, ++offset_it)
@@ -705,6 +692,8 @@ void ColumnSparse::updateWeakHash32(WeakHash32 & hash) const
         size_t value_index = offset_it.getValueIndex();
         hash_data[i] = values_hash_data[value_index];
     }
+
+    return hash;
 }
 
 void ColumnSparse::updateHashFast(SipHash & hash) const
diff --git a/src/Columns/ColumnSparse.h b/src/Columns/ColumnSparse.h
index 4860f5171f7..a5d4d788b17 100644
--- a/src/Columns/ColumnSparse.h
+++ b/src/Columns/ColumnSparse.h
@@ -139,7 +139,7 @@ public:
     void protect() override;
     ColumnPtr replicate(const Offsets & replicate_offsets) const override;
     void updateHashWithValue(size_t n, SipHash & hash) const override;
-    void updateWeakHash32(WeakHash32 & hash) const override;
+    WeakHash32 getWeakHash32() const override;
     void updateHashFast(SipHash & hash) const override;
     void getExtremes(Field & min, Field & max) const override;
 
diff --git a/src/Columns/ColumnString.cpp b/src/Columns/ColumnString.cpp
index 1eda9714d62..4accfbe8f41 100644
--- a/src/Columns/ColumnString.cpp
+++ b/src/Columns/ColumnString.cpp
@@ -108,13 +108,10 @@ MutableColumnPtr ColumnString::cloneResized(size_t to_size) const
     return res;
 }
 
-void ColumnString::updateWeakHash32(WeakHash32 & hash) const
+WeakHash32 ColumnString::getWeakHash32() const
 {
     auto s = offsets.size();
-
-    if (hash.getData().size() != s)
-        throw Exception(ErrorCodes::LOGICAL_ERROR, "Size of WeakHash32 does not match size of column: "
-                        "column size is {}, hash size is {}", std::to_string(s), std::to_string(hash.getData().size()));
+    WeakHash32 hash(s);
 
     const UInt8 * pos = chars.data();
     UInt32 * hash_data = hash.getData().data();
@@ -130,6 +127,8 @@ void ColumnString::updateWeakHash32(WeakHash32 & hash) const
         prev_offset = offset;
         ++hash_data;
     }
+
+    return hash;
 }
 
 
diff --git a/src/Columns/ColumnString.h b/src/Columns/ColumnString.h
index 602ffac65e8..faaaa8848ca 100644
--- a/src/Columns/ColumnString.h
+++ b/src/Columns/ColumnString.h
@@ -212,7 +212,7 @@ public:
         hash.update(reinterpret_cast<const char *>(&chars[offset]), string_size);
     }
 
-    void updateWeakHash32(WeakHash32 & hash) const override;
+    WeakHash32 getWeakHash32() const override;
 
     void updateHashFast(SipHash & hash) const override
     {
diff --git a/src/Columns/ColumnTuple.cpp b/src/Columns/ColumnTuple.cpp
index 9b822d7f570..cb0b05d2154 100644
--- a/src/Columns/ColumnTuple.cpp
+++ b/src/Columns/ColumnTuple.cpp
@@ -308,16 +308,15 @@ void ColumnTuple::updateHashWithValue(size_t n, SipHash & hash) const
         column->updateHashWithValue(n, hash);
 }
 
-void ColumnTuple::updateWeakHash32(WeakHash32 & hash) const
+WeakHash32 ColumnTuple::getWeakHash32() const
 {
     auto s = size();
-
-    if (hash.getData().size() != s)
-        throw Exception(ErrorCodes::LOGICAL_ERROR, "Size of WeakHash32 does not match size of column: "
-                        "column size is {}, hash size is {}", std::to_string(s), std::to_string(hash.getData().size()));
+    WeakHash32 hash(s);
 
     for (const auto & column : columns)
-        column->updateWeakHash32(hash);
+        hash.update(column->getWeakHash32());
+
+    return hash;
 }
 
 void ColumnTuple::updateHashFast(SipHash & hash) const
diff --git a/src/Columns/ColumnTuple.h b/src/Columns/ColumnTuple.h
index 38e479791d4..2fafd93f776 100644
--- a/src/Columns/ColumnTuple.h
+++ b/src/Columns/ColumnTuple.h
@@ -81,7 +81,7 @@ public:
     const char * deserializeAndInsertFromArena(const char * pos) override;
     const char * skipSerializedInArena(const char * pos) const override;
     void updateHashWithValue(size_t n, SipHash & hash) const override;
-    void updateWeakHash32(WeakHash32 & hash) const override;
+    WeakHash32 getWeakHash32() const override;
     void updateHashFast(SipHash & hash) const override;
 #if !defined(ABORT_ON_LOGICAL_ERROR)
     void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
diff --git a/src/Columns/ColumnVariant.cpp b/src/Columns/ColumnVariant.cpp
index ee5de4c2dde..8fd6e1bbac1 100644
--- a/src/Columns/ColumnVariant.cpp
+++ b/src/Columns/ColumnVariant.cpp
@@ -789,36 +789,26 @@ void ColumnVariant::updateHashWithValue(size_t n, SipHash & hash) const
         variants[localDiscriminatorByGlobal(global_discr)]->updateHashWithValue(offsetAt(n), hash);
 }
 
-void ColumnVariant::updateWeakHash32(WeakHash32 & hash) const
+WeakHash32 ColumnVariant::getWeakHash32() const
 {
     auto s = size();
 
-    if (hash.getData().size() != s)
-        throw Exception(ErrorCodes::LOGICAL_ERROR, "Size of WeakHash32 does not match size of column: "
-                                                   "column size is {}, hash size is {}", std::to_string(s), std::to_string(hash.getData().size()));
-
     /// If we have only NULLs, keep hash unchanged.
     if (hasOnlyNulls())
-        return;
+        return WeakHash32(s);
 
     /// Optimization for case when there is only 1 non-empty variant and no NULLs.
     /// In this case we can just calculate weak hash for this variant.
     if (auto non_empty_local_discr = getLocalDiscriminatorOfOneNoneEmptyVariantNoNulls())
-    {
-        variants[*non_empty_local_discr]->updateWeakHash32(hash);
-        return;
-    }
+        return variants[*non_empty_local_discr]->getWeakHash32();
 
     /// Calculate weak hash for all variants.
     std::vector<WeakHash32> nested_hashes;
     for (const auto & variant : variants)
-    {
-        WeakHash32 nested_hash(variant->size());
-        variant->updateWeakHash32(nested_hash);
-        nested_hashes.emplace_back(std::move(nested_hash));
-    }
+        nested_hashes.emplace_back(variant->getWeakHash32());
 
     /// For each row hash is a hash of corresponding row from corresponding variant.
+    WeakHash32 hash(s);
     auto & hash_data = hash.getData();
     const auto & local_discriminators_data = getLocalDiscriminators();
     const auto & offsets_data = getOffsets();
@@ -827,11 +817,10 @@ void ColumnVariant::updateWeakHash32(WeakHash32 & hash) const
         Discriminator discr = local_discriminators_data[i];
         /// Update hash only for non-NULL values
         if (discr != NULL_DISCRIMINATOR)
-        {
-            auto nested_hash = nested_hashes[local_discriminators_data[i]].getData()[offsets_data[i]];
-            hash_data[i] = static_cast<UInt32>(hashCRC32(nested_hash, hash_data[i]));
-        }
+            hash_data[i] = nested_hashes[discr].getData()[offsets_data[i]];
     }
+
+    return hash;
 }
 
 void ColumnVariant::updateHashFast(SipHash & hash) const
diff --git a/src/Columns/ColumnVariant.h b/src/Columns/ColumnVariant.h
index d91b8e93a7d..94f3066e676 100644
--- a/src/Columns/ColumnVariant.h
+++ b/src/Columns/ColumnVariant.h
@@ -213,7 +213,7 @@ public:
     const char * deserializeVariantAndInsertFromArena(Discriminator global_discr, const char * pos);
     const char * skipSerializedInArena(const char * pos) const override;
     void updateHashWithValue(size_t n, SipHash & hash) const override;
-    void updateWeakHash32(WeakHash32 & hash) const override;
+    WeakHash32 getWeakHash32() const override;
     void updateHashFast(SipHash & hash) const override;
     ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override;
     void expand(const Filter & mask, bool inverted) override;
diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp
index 19d1b800961..185a1e0f615 100644
--- a/src/Columns/ColumnVector.cpp
+++ b/src/Columns/ColumnVector.cpp
@@ -73,13 +73,10 @@ void ColumnVector<T>::updateHashWithValue(size_t n, SipHash & hash) const
 }
 
 template <typename T>
-void ColumnVector<T>::updateWeakHash32(WeakHash32 & hash) const
+WeakHash32 ColumnVector<T>::getWeakHash32() const
 {
     auto s = data.size();
-
-    if (hash.getData().size() != s)
-        throw Exception(ErrorCodes::LOGICAL_ERROR, "Size of WeakHash32 does not match size of column: "
-                        "column size is {}, hash size is {}", std::to_string(s), std::to_string(hash.getData().size()));
+    WeakHash32 hash(s);
 
     const T * begin = data.data();
     const T * end = begin + s;
@@ -91,6 +88,8 @@ void ColumnVector<T>::updateWeakHash32(WeakHash32 & hash) const
         ++begin;
         ++hash_data;
     }
+
+    return hash;
 }
 
 template <typename T>
diff --git a/src/Columns/ColumnVector.h b/src/Columns/ColumnVector.h
index 3a0acf5e312..c01778ecf32 100644
--- a/src/Columns/ColumnVector.h
+++ b/src/Columns/ColumnVector.h
@@ -114,7 +114,7 @@ public:
 
     void updateHashWithValue(size_t n, SipHash & hash) const override;
 
-    void updateWeakHash32(WeakHash32 & hash) const override;
+    WeakHash32 getWeakHash32() const override;
 
     void updateHashFast(SipHash & hash) const override;
 
diff --git a/src/Columns/IColumn.h b/src/Columns/IColumn.h
index 4b6f34e5aa2..3798d3b7466 100644
--- a/src/Columns/IColumn.h
+++ b/src/Columns/IColumn.h
@@ -300,10 +300,10 @@ public:
     ///  passed bytes to hash must identify sequence of values unambiguously.
     virtual void updateHashWithValue(size_t n, SipHash & hash) const = 0;
 
-    /// Update hash function value. Hash is calculated for each element.
+    /// Get hash function value. Hash is calculated for each element.
     /// It's a fast weak hash function. Mainly need to scatter data between threads.
     /// WeakHash32 must have the same size as column.
-    virtual void updateWeakHash32(WeakHash32 & hash) const = 0;
+    virtual WeakHash32 getWeakHash32() const = 0;
 
     /// Update state of hash with all column.
     virtual void updateHashFast(SipHash & hash) const = 0;
diff --git a/src/Columns/IColumnDummy.h b/src/Columns/IColumnDummy.h
index c19fb704d9b..b18f4fdb302 100644
--- a/src/Columns/IColumnDummy.h
+++ b/src/Columns/IColumnDummy.h
@@ -1,6 +1,7 @@
 #pragma once
 
 #include <Columns/IColumn.h>
+#include <Common/WeakHash.h>
 
 
 namespace DB
@@ -63,8 +64,9 @@ public:
     {
     }
 
-    void updateWeakHash32(WeakHash32 & /*hash*/) const override
+    WeakHash32 getWeakHash32() const override
     {
+        return WeakHash32(s);
     }
 
     void updateHashFast(SipHash & /*hash*/) const override
diff --git a/src/Columns/IColumnUnique.h b/src/Columns/IColumnUnique.h
index 3398452b7ee..1b86204f5b1 100644
--- a/src/Columns/IColumnUnique.h
+++ b/src/Columns/IColumnUnique.h
@@ -1,6 +1,7 @@
 #pragma once
 #include <optional>
 #include <Columns/IColumn.h>
+#include <Common/WeakHash.h>
 
 namespace DB
 {
@@ -166,9 +167,9 @@ public:
         throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method scatter is not supported for ColumnUnique.");
     }
 
-    void updateWeakHash32(WeakHash32 &) const override
+    WeakHash32 getWeakHash32() const override
     {
-        throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method updateWeakHash32 is not supported for ColumnUnique.");
+        throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method getWeakHash32 is not supported for ColumnUnique.");
     }
 
     void updateHashFast(SipHash &) const override
diff --git a/src/Columns/tests/gtest_weak_hash_32.cpp b/src/Columns/tests/gtest_weak_hash_32.cpp
index 2c95998761b..3143d0ff83c 100644
--- a/src/Columns/tests/gtest_weak_hash_32.cpp
+++ b/src/Columns/tests/gtest_weak_hash_32.cpp
@@ -60,8 +60,7 @@ TEST(WeakHash32, ColumnVectorU8)
             data.push_back(i);
     }
 
-    WeakHash32 hash(col->size());
-    col->updateWeakHash32(hash);
+    WeakHash32 hash = col->getWeakHash32();
 
     checkColumn(hash.getData(), col->getData());
 }
@@ -77,8 +76,7 @@ TEST(WeakHash32, ColumnVectorI8)
             data.push_back(i);
     }
 
-    WeakHash32 hash(col->size());
-    col->updateWeakHash32(hash);
+    WeakHash32 hash = col->getWeakHash32();
 
     checkColumn(hash.getData(), col->getData());
 }
@@ -94,8 +92,7 @@ TEST(WeakHash32, ColumnVectorU16)
             data.push_back(i);
     }
 
-    WeakHash32 hash(col->size());
-    col->updateWeakHash32(hash);
+    WeakHash32 hash = col->getWeakHash32();
 
     checkColumn(hash.getData(), col->getData());
 }
@@ -111,8 +108,7 @@ TEST(WeakHash32, ColumnVectorI16)
             data.push_back(i);
     }
 
-    WeakHash32 hash(col->size());
-    col->updateWeakHash32(hash);
+    WeakHash32 hash = col->getWeakHash32();
 
     checkColumn(hash.getData(), col->getData());
 }
@@ -128,8 +124,7 @@ TEST(WeakHash32, ColumnVectorU32)
             data.push_back(i << 16u);
     }
 
-    WeakHash32 hash(col->size());
-    col->updateWeakHash32(hash);
+    WeakHash32 hash = col->getWeakHash32();
 
     checkColumn(hash.getData(), col->getData());
 }
@@ -145,8 +140,7 @@ TEST(WeakHash32, ColumnVectorI32)
             data.push_back(i << 16);
     }
 
-    WeakHash32 hash(col->size());
-    col->updateWeakHash32(hash);
+    WeakHash32 hash = col->getWeakHash32();
 
     checkColumn(hash.getData(), col->getData());
 }
@@ -162,8 +156,7 @@ TEST(WeakHash32, ColumnVectorU64)
             data.push_back(i << 32u);
     }
 
-    WeakHash32 hash(col->size());
-    col->updateWeakHash32(hash);
+    WeakHash32 hash = col->getWeakHash32();
 
     checkColumn(hash.getData(), col->getData());
 }
@@ -179,8 +172,7 @@ TEST(WeakHash32, ColumnVectorI64)
             data.push_back(i << 32);
     }
 
-    WeakHash32 hash(col->size());
-    col->updateWeakHash32(hash);
+    WeakHash32 hash = col->getWeakHash32();
 
     checkColumn(hash.getData(), col->getData());
 }
@@ -204,8 +196,7 @@ TEST(WeakHash32, ColumnVectorU128)
         }
     }
 
-    WeakHash32 hash(col->size());
-    col->updateWeakHash32(hash);
+    WeakHash32 hash = col->getWeakHash32();
 
     checkColumn(hash.getData(), eq_data);
 }
@@ -221,8 +212,7 @@ TEST(WeakHash32, ColumnVectorI128)
             data.push_back(i << 32);
     }
 
-    WeakHash32 hash(col->size());
-    col->updateWeakHash32(hash);
+    WeakHash32 hash = col->getWeakHash32();
 
     checkColumn(hash.getData(), col->getData());
 }
@@ -238,8 +228,7 @@ TEST(WeakHash32, ColumnDecimal32)
             data.push_back(i << 16);
     }
 
-    WeakHash32 hash(col->size());
-    col->updateWeakHash32(hash);
+    WeakHash32 hash = col->getWeakHash32();
 
     checkColumn(hash.getData(), col->getData());
 }
@@ -255,8 +244,7 @@ TEST(WeakHash32, ColumnDecimal64)
             data.push_back(i << 32);
     }
 
-    WeakHash32 hash(col->size());
-    col->updateWeakHash32(hash);
+    WeakHash32 hash = col->getWeakHash32();
 
     checkColumn(hash.getData(), col->getData());
 }
@@ -272,8 +260,7 @@ TEST(WeakHash32, ColumnDecimal128)
             data.push_back(i << 32);
     }
 
-    WeakHash32 hash(col->size());
-    col->updateWeakHash32(hash);
+    WeakHash32 hash = col->getWeakHash32();
 
     checkColumn(hash.getData(), col->getData());
 }
@@ -294,8 +281,7 @@ TEST(WeakHash32, ColumnString1)
         }
     }
 
-    WeakHash32 hash(col->size());
-    col->updateWeakHash32(hash);
+    WeakHash32 hash = col->getWeakHash32();
 
     checkColumn(hash.getData(), data);
 }
@@ -331,8 +317,7 @@ TEST(WeakHash32, ColumnString2)
         }
     }
 
-    WeakHash32 hash(col->size());
-    col->updateWeakHash32(hash);
+    WeakHash32 hash = col->getWeakHash32();
 
     checkColumn(hash.getData(), data);
 }
@@ -369,8 +354,7 @@ TEST(WeakHash32, ColumnString3)
         }
     }
 
-    WeakHash32 hash(col->size());
-    col->updateWeakHash32(hash);
+    WeakHash32 hash = col->getWeakHash32();
 
     checkColumn(hash.getData(), data);
 }
@@ -397,8 +381,7 @@ TEST(WeakHash32, ColumnFixedString)
         }
     }
 
-    WeakHash32 hash(col->size());
-    col->updateWeakHash32(hash);
+    WeakHash32 hash = col->getWeakHash32();
 
     checkColumn(hash.getData(), data);
 }
@@ -444,8 +427,7 @@ TEST(WeakHash32, ColumnArray)
 
     auto col_arr = ColumnArray::create(std::move(val), std::move(off));
 
-    WeakHash32 hash(col_arr->size());
-    col_arr->updateWeakHash32(hash);
+    WeakHash32 hash = col_arr->getWeakHash32();
 
     checkColumn(hash.getData(), eq_data);
 }
@@ -479,8 +461,7 @@ TEST(WeakHash32, ColumnArray2)
 
     auto col_arr = ColumnArray::create(std::move(val), std::move(off));
 
-    WeakHash32 hash(col_arr->size());
-    col_arr->updateWeakHash32(hash);
+    WeakHash32 hash = col_arr->getWeakHash32();
 
     checkColumn(hash.getData(), eq_data);
 }
@@ -536,8 +517,7 @@ TEST(WeakHash32, ColumnArrayArray)
     auto col_arr = ColumnArray::create(std::move(val), std::move(off));
     auto col_arr_arr = ColumnArray::create(std::move(col_arr), std::move(off2));
 
-    WeakHash32 hash(col_arr_arr->size());
-    col_arr_arr->updateWeakHash32(hash);
+    WeakHash32 hash = col_arr_arr->getWeakHash32();
 
     checkColumn(hash.getData(), eq_data);
 }
@@ -555,8 +535,7 @@ TEST(WeakHash32, ColumnConst)
 
     auto col_const = ColumnConst::create(std::move(inner_col), 256);
 
-    WeakHash32 hash(col_const->size());
-    col_const->updateWeakHash32(hash);
+    WeakHash32 hash = col_const->getWeakHash32();
 
     checkColumn(hash.getData(), data);
 }
@@ -576,8 +555,7 @@ TEST(WeakHash32, ColumnLowcardinality)
         }
     }
 
-    WeakHash32 hash(col->size());
-    col->updateWeakHash32(hash);
+    WeakHash32 hash = col->getWeakHash32();
 
     checkColumn(hash.getData(), data);
 }
@@ -602,8 +580,7 @@ TEST(WeakHash32, ColumnNullable)
 
     auto col_null = ColumnNullable::create(std::move(col), std::move(mask));
 
-    WeakHash32 hash(col_null->size());
-    col_null->updateWeakHash32(hash);
+    WeakHash32 hash = col_null->getWeakHash32();
 
     checkColumn(hash.getData(), eq);
 }
@@ -633,8 +610,7 @@ TEST(WeakHash32, ColumnTupleUInt64UInt64)
     columns.emplace_back(std::move(col2));
     auto col_tuple = ColumnTuple::create(std::move(columns));
 
-    WeakHash32 hash(col_tuple->size());
-    col_tuple->updateWeakHash32(hash);
+    WeakHash32 hash = col_tuple->getWeakHash32();
 
     checkColumn(hash.getData(), eq);
 }
@@ -671,8 +647,7 @@ TEST(WeakHash32, ColumnTupleUInt64String)
     columns.emplace_back(std::move(col2));
     auto col_tuple = ColumnTuple::create(std::move(columns));
 
-    WeakHash32 hash(col_tuple->size());
-    col_tuple->updateWeakHash32(hash);
+    WeakHash32 hash = col_tuple->getWeakHash32();
 
     checkColumn(hash.getData(), eq);
 }
@@ -709,8 +684,7 @@ TEST(WeakHash32, ColumnTupleUInt64FixedString)
     columns.emplace_back(std::move(col2));
     auto col_tuple = ColumnTuple::create(std::move(columns));
 
-    WeakHash32 hash(col_tuple->size());
-    col_tuple->updateWeakHash32(hash);
+    WeakHash32 hash = col_tuple->getWeakHash32();
 
     checkColumn(hash.getData(), eq);
 }
@@ -756,8 +730,7 @@ TEST(WeakHash32, ColumnTupleUInt64Array)
     columns.emplace_back(ColumnArray::create(std::move(val), std::move(off)));
     auto col_tuple = ColumnTuple::create(std::move(columns));
 
-    WeakHash32 hash(col_tuple->size());
-    col_tuple->updateWeakHash32(hash);
+    WeakHash32 hash = col_tuple->getWeakHash32();
 
     checkColumn(hash.getData(), eq_data);
 }
diff --git a/src/Common/WeakHash.cpp b/src/Common/WeakHash.cpp
index 54d973b6296..cb12df84db1 100644
--- a/src/Common/WeakHash.cpp
+++ b/src/Common/WeakHash.cpp
@@ -1,2 +1,24 @@
 #include <Common/WeakHash.h>
+#include <Common/Exception.h>
+#include <Common/HashTable/Hash.h>
 
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int LOGICAL_ERROR;
+}
+
+void WeakHash32::update(const WeakHash32 & other)
+{
+    size_t size = data.size();
+    if (size != other.data.size())
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "Size of WeakHash32 does not match:"
+                        "left size is {}, right size is {}", size, other.data.size());
+
+    for (size_t i = 0; i < size; ++i)
+        data[i] = static_cast<UInt32>(intHashCRC32(other.data[i], data[i]));
+}
+
+}
diff --git a/src/Common/WeakHash.h b/src/Common/WeakHash.h
index b59624e64f2..d4a8d63868c 100644
--- a/src/Common/WeakHash.h
+++ b/src/Common/WeakHash.h
@@ -11,9 +11,8 @@ namespace DB
 /// The main purpose why this class needed is to support data initialization. Initially, every bit is 1.
 class WeakHash32
 {
-    static constexpr UInt32 kDefaultInitialValue = ~UInt32(0);
-
 public:
+    static constexpr UInt32 kDefaultInitialValue = ~UInt32(0);
 
     using Container = PaddedPODArray<UInt32>;
 
@@ -22,6 +21,8 @@ public:
 
     void reset(size_t size, UInt32 initial_value = kDefaultInitialValue) { data.assign(size, initial_value); }
 
+    void update(const WeakHash32 & other);
+
     const Container & getData() const { return data; }
     Container & getData() { return data; }
 
diff --git a/src/Interpreters/ConcurrentHashJoin.cpp b/src/Interpreters/ConcurrentHashJoin.cpp
index 4493a9f4dbd..ac940c62a1a 100644
--- a/src/Interpreters/ConcurrentHashJoin.cpp
+++ b/src/Interpreters/ConcurrentHashJoin.cpp
@@ -310,7 +310,7 @@ IColumn::Selector ConcurrentHashJoin::selectDispatchBlock(const Strings & key_co
     {
         const auto & key_col = from_block.getByName(key_name).column->convertToFullColumnIfConst();
         const auto & key_col_no_lc = recursiveRemoveLowCardinality(recursiveRemoveSparse(key_col));
-        key_col_no_lc->updateWeakHash32(hash);
+        hash.update(key_col_no_lc->getWeakHash32());
     }
     return hashToSelector(hash, num_shards);
 }
diff --git a/src/Interpreters/JoinUtils.cpp b/src/Interpreters/JoinUtils.cpp
index 1788c9aca48..180a45d4295 100644
--- a/src/Interpreters/JoinUtils.cpp
+++ b/src/Interpreters/JoinUtils.cpp
@@ -554,7 +554,7 @@ static Blocks scatterBlockByHashImpl(const Strings & key_columns_names, const Bl
     for (const auto & key_name : key_columns_names)
     {
         ColumnPtr key_col = materializeColumn(block, key_name);
-        key_col->updateWeakHash32(hash);
+        hash.update(key_col->getWeakHash32());
     }
     auto selector = hashToSelector(hash, sharder);
 
diff --git a/src/Processors/Transforms/ScatterByPartitionTransform.cpp b/src/Processors/Transforms/ScatterByPartitionTransform.cpp
index 6e3cdc0fda1..16d265c9bcb 100644
--- a/src/Processors/Transforms/ScatterByPartitionTransform.cpp
+++ b/src/Processors/Transforms/ScatterByPartitionTransform.cpp
@@ -109,7 +109,7 @@ void ScatterByPartitionTransform::generateOutputChunks()
     hash.reset(num_rows);
 
     for (const auto & column_number : key_columns)
-        columns[column_number]->updateWeakHash32(hash);
+        hash.update(columns[column_number]->getWeakHash32());
 
     const auto & hash_data = hash.getData();
     IColumn::Selector selector(num_rows);

From fd443a1721439aa455d431763e38eec2e51822a9 Mon Sep 17 00:00:00 2001
From: Konstantin Morozov <just.morozov.k@gmail.com>
Date: Wed, 17 Jul 2024 13:43:36 +0000
Subject: [PATCH 0359/1488] add mutable

---
 src/IO/S3/Client.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/IO/S3/Client.h b/src/IO/S3/Client.h
index 6d97ec29ae6..94f1fd05c05 100644
--- a/src/IO/S3/Client.h
+++ b/src/IO/S3/Client.h
@@ -63,10 +63,10 @@ struct ClientCache
 
     void clearCache();
 
-    std::mutex region_cache_mutex;
+    mutable std::mutex region_cache_mutex;
     std::unordered_map<std::string, std::string> region_for_bucket_cache TSA_GUARDED_BY(region_cache_mutex);
 
-    std::mutex uri_cache_mutex;
+    mutable std::mutex uri_cache_mutex;
     std::unordered_map<std::string, URI> uri_for_bucket_cache TSA_GUARDED_BY(uri_cache_mutex);
 };
 

From 2dbd04c8a77bf262f0965ddef9a2c166c22fcf55 Mon Sep 17 00:00:00 2001
From: Blargian <shaunstruwig.ss@gmail.com>
Date: Wed, 17 Jul 2024 17:01:22 +0200
Subject: [PATCH 0360/1488] add toIntXYZ documentation

---
 .../functions/type-conversion-functions.md    | 1275 ++++++++++++++++-
 1 file changed, 1239 insertions(+), 36 deletions(-)

diff --git a/docs/en/sql-reference/functions/type-conversion-functions.md b/docs/en/sql-reference/functions/type-conversion-functions.md
index 61e84ca72d1..057083d317f 100644
--- a/docs/en/sql-reference/functions/type-conversion-functions.md
+++ b/docs/en/sql-reference/functions/type-conversion-functions.md
@@ -49,105 +49,1308 @@ SETTINGS cast_keep_nullable = 1
 └──────────────────┴─────────────────────┴──────────────────┘
 ```
 
-## toInt(8\|16\|32\|64\|128\|256)
+## toInt8
 
-Converts an input value to a value the [Int](../data-types/int-uint.md) data type. This function family includes:
+Converts an input value to a value of type `Int8`.
 
-- `toInt8(expr)` — Converts to a value of data type `Int8`.
-- `toInt16(expr)` — Converts to a value of data type `Int16`.
-- `toInt32(expr)` — Converts to a value of data type `Int32`.
-- `toInt64(expr)` — Converts to a value of data type `Int64`.
-- `toInt128(expr)` — Converts to a value of data type `Int128`.
-- `toInt256(expr)` — Converts to a value of data type `Int256`.
+**Syntax**
+
+```sql
+toInt8(expr)
+```
 
 **Arguments**
 
-- `expr` — [Expression](../syntax.md/#syntax-expressions) returning a number or a string with the decimal representation of a number. Binary, octal, and hexadecimal representations of numbers are not supported. Leading zeroes are stripped.
+- `expr` — Expression returning a number or a string with the decimal representation of a number. [Expression](../syntax.md/#syntax-expressions).
+
+:::note
+Binary, octal, and hexadecimal representations of numbers are not supported. Leading zeroes are stripped.
+:::
 
 **Returned value**
 
-Integer value in the `Int8`, `Int16`, `Int32`, `Int64`, `Int128` or `Int256` data type.
+- 8-bit integer value. [Int8](../data-types/int-uint.md).
 
-Functions use [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning they truncate fractional digits of numbers.
+:::note
+Function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
+:::
 
-The behavior of functions for the [NaN and Inf](../data-types/float.md/#data_type-float-nan-inf) arguments is undefined. Remember about [numeric conversions issues](#common-issues-with-data-conversion), when using the functions.
+:::danger
+An exception is thrown for [NaN and Inf](../data-types/float.md/#data_type-float-nan-inf) arguments. Keep in mind [numeric conversions issues](#common-issues-with-data-conversion), when using this function.
+:::
+
+**Example**
+
+Query:
+
+```sql
+SELECT
+    toInt8(-8),
+    toInt8(-8.8),
+    toInt8('-8');
+```
+
+Result:
+
+```response
+   ┌─toInt8(-8)─┬─toInt8(-8.8)─┬─toInt8('-8')─┐
+1. │         -8 │           -8 │           -8 │
+   └────────────┴──────────────┴──────────────┘
+```
+
+**See also**
+
+- [`toInt8OrZero`](#toint8orzero).
+- [`toInt8OrNull`](#toint8ornull).
+- [`toInt8OrDefault`](#toint8ordefault).
+
+## toInt8OrZero
+
+Like [`toInt8`](#toint8), it takes an argument of type [String](../data-types/string.md) and tries to parse it to type `Int8`. If unsuccessful, returns `0`.
+
+**Syntax**
+
+```sql
+toInt8OrZero(expr)
+```
+
+**Arguments**
+
+- `expr` — Expression returning a number or a string with the decimal representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
+
+:::note
+Binary, octal, and hexadecimal representations of numbers are not supported. Leading zeroes are stripped.
+:::
+
+**Returned value**
+
+- 8-bit integer value if successful, otherwise `0`. [Int8](../data-types/int-uint.md).
+
+:::note
+Function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers. 
+:::
+
+:::danger
+An exception is thrown for [NaN and Inf](../data-types/float.md/#data_type-float-nan-inf) arguments. Keep in mind [numeric conversions issues](#common-issues-with-data-conversion), when using this function.
+:::
 
 **Example**
 
 Query:
 
 ``` sql
-SELECT toInt64(nan), toInt32(32), toInt16('16'), toInt8(8.8);
+SELECT
+    toInt8OrZero('-8'),
+    toInt8OrZero('abc');
 ```
 
 Result:
 
 ```response
-┌─────────toInt64(nan)─┬─toInt32(32)─┬─toInt16('16')─┬─toInt8(8.8)─┐
-│ -9223372036854775808 │          32 │            16 │           8 │
-└──────────────────────┴─────────────┴───────────────┴─────────────┘
+   ┌─toInt8OrZero('-8')─┬─toInt8OrZero('abc')─┐
+1. │                 -8 │                   0 │
+   └────────────────────┴─────────────────────┘
 ```
 
-## toInt(8\|16\|32\|64\|128\|256)OrZero
+**See also**
 
-Takes an argument of type [String](../data-types/string.md) and tries to parse it into an Int (8 \| 16 \| 32 \| 64 \| 128 \| 256). If unsuccessful, returns `0`.
+- [`toInt8`](#toint8).
+- [`toInt8OrNull`](#toint8ornull).
+- [`toInt8OrDefault`](#toint8ordefault).
+
+## toInt8OrNull
+
+Like [`toInt8`](#toint8), takes an argument of type [String](../data-types/string.md) and tries to parse it to type `Int8`. If unsuccessful, returns `NULL`.
+
+**Syntax**
+
+```sql
+toInt8OrNull(expr)
+```
+
+**Arguments**
+
+- `expr` — Expression returning a number or a string with the decimal representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
+
+:::note
+Binary, octal, and hexadecimal representations of numbers are not supported. Leading zeroes are stripped.
+:::
+
+**Returned value**
+
+- 8-bit integer value if successful, otherwise `NULL`. [Int8](../data-types/int-uint.md) / [NULL](../data-types/nullable.md).
+
+:::note
+Function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
+:::
+
+:::danger
+An exception is thrown for [NaN and Inf](../data-types/float.md/#data_type-float-nan-inf) arguments. Keep in mind [numeric conversions issues](#common-issues-with-data-conversion), when using this function.
+:::
 
 **Example**
 
 Query:
 
 ``` sql
-SELECT toInt64OrZero('123123'), toInt8OrZero('123qwe123');
+SELECT toInt8OrNull('-8'), toInt8OrNull('abc');
 ```
 
 Result:
 
 ```response
-┌─toInt64OrZero('123123')─┬─toInt8OrZero('123qwe123')─┐
-│                  123123 │                         0 │
-└─────────────────────────┴───────────────────────────┘
+   ┌─toInt8OrNull('-8')─┬─toInt8OrNull('abc')─┐
+1. │                 -8 │                ᴺᵁᴸᴸ │
+   └────────────────────┴─────────────────────┘
 ```
 
-## toInt(8\|16\|32\|64\|128\|256)OrNull
+**See also**
 
-It takes an argument of type String and tries to parse it into Int (8 \| 16 \| 32 \| 64 \| 128 \| 256). If unsuccessful, returns `NULL`.
+- [`toInt8`](#toint8).
+- [`toInt8OrZero`](#toint8orzero).
+- [`toInt8OrDefault`](#toint8ordefault).
+
+## toInt8OrDefault
+
+Like [`toInt8`](#toint8), takes an argument of type [String](../data-types/string.md) and tries to parse it to type `Int8`. If unsuccessful, returns the default type value.
+
+**Syntax**
+
+```sql
+toInt8OrDefault(expr, def)
+```
+
+**Arguments**
+
+- `expr` — Expression returning a number or a string with the decimal representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
+- `def` — The default value to return if parsing to type `Int8` is unsuccessful. [Int8](../data-types/int-uint.md).
+
+:::note
+Binary, octal, and hexadecimal representations of numbers are not supported. Leading zeroes are stripped.
+:::
+
+**Returned value**
+
+- 8-bit integer value if successful, otherwise returns the default value. [Int8](../data-types/int-uint.md).
+
+:::note
+- Function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
+- The default value type should be the same as the cast type.
+:::
+
+:::danger
+An exception is thrown for [NaN and Inf](../data-types/float.md/#data_type-float-nan-inf) arguments. Keep in mind [numeric conversions issues](#common-issues-with-data-conversion), when using this function.
+:::
 
 **Example**
 
 Query:
 
 ``` sql
-SELECT toInt64OrNull('123123'), toInt8OrNull('123qwe123');
+SELECT
+    toInt8OrDefault('-8', CAST('-1', 'Int8')),
+    toInt8OrDefault('abc', CAST('-1', 'Int8'));
 ```
 
 Result:
 
 ```response
-┌─toInt64OrNull('123123')─┬─toInt8OrNull('123qwe123')─┐
-│                  123123 │                      ᴺᵁᴸᴸ │
-└─────────────────────────┴───────────────────────────┘
+   ┌─toInt8OrDefault('-8', CAST('-1', 'Int8'))─┬─toInt8OrDefault('abc', CAST('-1', 'Int8'))─┐
+1. │                                        -8 │                                         -1 │
+   └───────────────────────────────────────────┴────────────────────────────────────────────┘
 ```
 
-## toInt(8\|16\|32\|64\|128\|256)OrDefault
+**See also**
 
-It takes an argument of type String and tries to parse it into Int (8 \| 16 \| 32 \| 64 \| 128 \| 256). If unsuccessful, returns the default type value.
+- [`toInt8`](#toint8).
+- [`toInt8OrZero`](#toint8orzero).
+- [`toInt8OrNull`](#toint8orNull).
+
+## toInt16
+
+Converts an input value to a value of type `Int16`.
+
+**Syntax**
+
+```sql
+toInt16(expr)
+```
+
+**Arguments**
+
+- `expr` — Expression returning a number or a string with the decimal representation of a number. [Expression](../syntax.md/#syntax-expressions).
+
+:::note
+Binary, octal, and hexadecimal representations of numbers are not supported. Leading zeroes are stripped.
+:::
+
+**Returned value**
+
+- 16-bit integer value. [Int16](../data-types/int-uint.md).
+
+:::note
+Function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
+:::
+
+:::danger
+An exception is thrown for [NaN and Inf](../data-types/float.md/#data_type-float-nan-inf) arguments. Keep in mind [numeric conversions issues](#common-issues-with-data-conversion), when using this function.
+:::
+
+**Example**
+
+Query:
+
+```sql
+SELECT
+    toInt16(-16),
+    toInt16(-16.16),
+    toInt16('-16');
+```
+
+Result:
+
+```response
+   ┌─toInt16(-16)─┬─toInt16(-16.16)─┬─toInt16('-16')─┐
+1. │          -16 │             -16 │            -16 │
+   └──────────────┴─────────────────┴────────────────┘
+```
+
+**See also**
+
+- [`toInt16OrZero`](#toint16orzero).
+- [`toInt16OrNull`](#toint16ornull).
+- [`toInt16OrDefault`](#toint16ordefault).
+
+## toInt16OrZero
+
+Like [`toInt16`](#toint16), takes an argument of type [String](../data-types/string.md) and tries to parse it to type `Int16`. If unsuccessful, returns `0`.
+
+**Syntax**
+
+```sql
+toInt16OrZero(expr)
+```
+
+**Arguments**
+
+- `expr` — Expression returning a number or a string with the decimal representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
+
+:::note
+Binary, octal, and hexadecimal representations of numbers are not supported. Leading zeroes are stripped.
+:::
+
+**Returned value**
+
+- 16-bit integer value if successful, otherwise `0`. [Int16](../data-types/int-uint.md).
+
+:::note
+Function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
+:::
+
+:::danger
+An exception is thrown for [NaN and Inf](../data-types/float.md/#data_type-float-nan-inf) arguments. Keep in mind [numeric conversions issues](#common-issues-with-data-conversion), when using this function.
+:::
 
 **Example**
 
 Query:
 
 ``` sql
-SELECT toInt64OrDefault('123123', cast('-1' as Int64)), toInt8OrDefault('123qwe123', cast('-1' as Int8));
+SELECT
+    toInt16OrZero('-16'),
+    toInt16OrZero('abc');
 ```
 
 Result:
 
 ```response
-┌─toInt64OrDefault('123123', CAST('-1', 'Int64'))─┬─toInt8OrDefault('123qwe123', CAST('-1', 'Int8'))─┐
-│                                          123123 │                                               -1 │
-└─────────────────────────────────────────────────┴──────────────────────────────────────────────────┘
+   ┌─toInt16OrZero('-16')─┬─toInt16OrZero('abc')─┐
+1. │                  -16 │                    0 │
+   └──────────────────────┴──────────────────────┘
 ```
 
+**See also**
+
+- [`toInt16`](#toint16).
+- [`toInt16OrNull`](#toint16ornull).
+- [`toInt16OrDefault`](#toint16ordefault).
+
+## toInt16OrNull
+
+Like [`toInt16`](#toint16), takes an argument of type [String](../data-types/string.md) and tries to parse it to type `Int16`. If unsuccessful, returns `NULL`.
+
+**Syntax**
+
+```sql
+toInt16OrNull(expr)
+```
+
+**Arguments**
+
+- `expr` — Expression returning a number or a string with the decimal representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
+
+:::note
+Binary, octal, and hexadecimal representations of numbers are not supported. Leading zeroes are stripped.
+:::
+
+**Returned value**
+
+- 16-bit integer value if successful, otherwise `NULL`. [Int16](../data-types/int-uint.md) / [NULL](../data-types/nullable.md).
+
+:::note
+Function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
+:::
+
+:::danger
+An exception is thrown for [NaN and Inf](../data-types/float.md/#data_type-float-nan-inf) arguments. Keep in mind [numeric conversions issues](#common-issues-with-data-conversion), when using this function.
+:::
+
+**Example**
+
+Query:
+
+``` sql
+SELECT
+    toInt16OrNull('-16'),
+    toInt16OrNull('abc');
+```
+
+Result:
+
+```response
+   ┌─toInt16OrNull('-16')─┬─toInt16OrNull('abc')─┐
+1. │                  -16 │                 ᴺᵁᴸᴸ │
+   └──────────────────────┴──────────────────────┘
+```
+
+**See also**
+
+- [`toInt16`](#toint16).
+- [`toInt16OrZero`](#toint16orzero).
+- [`toInt16OrDefault`](#toint16ordefault).
+
+## toInt16OrDefault
+
+Like [`toInt16`](#toint16), takes an argument of type [String](../data-types/string.md) and tries to parse it to type `Int16`. If unsuccessful, returns the default type value.
+
+**Syntax**
+
+```sql
+toInt16OrDefault(expr, def)
+```
+
+**Arguments**
+
+- `expr` — Expression returning a number or a string with the decimal representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
+- `def` — The default value to return if parsing to type `Int16` is unsuccessful. [Int8](../data-types/int-uint.md).
+
+:::note
+Binary, octal, and hexadecimal representations of numbers are not supported. Leading zeroes are stripped.
+:::
+
+**Returned value**
+
+- 16-bit integer value if successful, otherwise returns the default value. [Int16](../data-types/int-uint.md).
+
+:::note
+- Function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
+- The default value type should be the same as the cast type.
+:::
+
+:::danger
+An exception is thrown for [NaN and Inf](../data-types/float.md/#data_type-float-nan-inf) arguments. Keep in mind [numeric conversions issues](#common-issues-with-data-conversion), when using this function.
+:::
+
+**Example**
+
+Query:
+
+``` sql
+SELECT toInt16OrDefault('-16', cast('-1' as Int16)), toInt16OrDefault('abc', cast('-1' as Int16));
+```
+
+Result:
+
+```response
+   ┌─toInt16OrDefault('-16', CAST('-1', 'Int16'))─┬─toInt16OrDefault('abc', CAST('-1', 'Int16'))─┐
+1. │                                          -16 │                                           -1 │
+   └──────────────────────────────────────────────┴──────────────────────────────────────────────┘
+```
+
+**See also**
+
+- [`toInt16`](#toint16).
+- [`toInt16OrZero`](#toint16orzero).
+- [`toInt16OrNull`](#toint16ornull).
+
+## toInt32
+
+Converts an input value to a value of type `Int32`.
+
+**Syntax**
+
+```sql
+toInt32(expr)
+```
+
+**Arguments**
+
+- `expr` — Expression returning a number or a string with the decimal representation of a number. [Expression](../syntax.md/#syntax-expressions).
+
+:::note
+Binary, octal, and hexadecimal representations of numbers are not supported. Leading zeroes are stripped.
+:::
+
+**Returned value**
+
+- 32-bit integer value. [Int32](../data-types/int-uint.md).
+
+:::note
+Function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
+:::
+
+:::danger
+An exception is thrown for [NaN and Inf](../data-types/float.md/#data_type-float-nan-inf) arguments. Keep in mind [numeric conversions issues](#common-issues-with-data-conversion), when using this function.
+:::
+
+**Example**
+
+Query:
+
+```sql
+SELECT
+    toInt32(-32),
+    toInt32(-32.32),
+    toInt32('-32')
+```
+
+Result:
+
+```response
+   ┌─toInt32(-32)─┬─toInt32(-32.32)─┬─toInt32('-32')─┐
+1. │          -32 │             -32 │            -32 │
+   └──────────────┴─────────────────┴────────────────┘
+```
+
+**See also**
+
+- [`toInt32OrZero`](#toint32orzero).
+- [`toInt32OrNull`](#toint32ornull).
+- [`toInt32OrDefault`](#toint32ordefault).
+
+## toInt32OrZero
+
+Like [`toInt32`](#toint32), takes an argument of type [String](../data-types/string.md) and tries to parse it to type `Int32`. If unsuccessful, returns `0`.
+
+**Syntax**
+
+```sql
+toInt32OrZero(expr)
+```
+
+**Arguments**
+
+- `expr` — Expression returning a number or a string with the decimal representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
+
+:::note
+Binary, octal, and hexadecimal representations of numbers are not supported. Leading zeroes are stripped.
+:::
+
+**Returned value**
+
+- 32-bit integer value if successful, otherwise `0`. [Int32](../data-types/int-uint.md)
+
+:::note
+Function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncate fractional digits of numbers.
+:::
+
+:::danger
+An exception is thrown for [NaN and Inf](../data-types/float.md/#data_type-float-nan-inf) arguments. Keep in mind [numeric conversions issues](#common-issues-with-data-conversion), when using this function.
+:::
+
+**Example**
+
+Query:
+
+``` sql
+SELECT toInt32OrZero('-32'), toInt32OrZero('abc');
+```
+
+Result:
+
+```response
+   ┌─toInt32OrZero('-32')─┬─toInt32OrZero('abc')─┐
+1. │                  -32 │                    0 │
+   └──────────────────────┴──────────────────────┘
+```
+**See also**
+
+- [`toInt32`](#toint32).
+- [`toInt32OrNull`](#toint32ornull).
+- [`toInt32OrDefault`](#toint32ordefault).
+- 
+## toInt32OrNull
+
+Like [`toInt32`](#toint32), takes an argument of type [String](../data-types/string.md) and tries to parse it to type `Int32`. If unsuccessful, returns `NULL`.
+
+**Syntax**
+
+```sql
+toInt32OrNull(expr)
+```
+
+**Arguments**
+
+- `expr` — Expression returning a number or a string with the decimal representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
+
+:::note
+Binary, octal, and hexadecimal representations of numbers are not supported. Leading zeroes are stripped.
+:::
+
+**Returned value**
+
+- 32-bit integer value if successful, otherwise `NULL`. [Int32](../data-types/int-uint.md) / [NULL](../data-types/nullable.md).
+
+:::note
+Function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
+:::
+
+:::danger
+An exception is thrown for [NaN and Inf](../data-types/float.md/#data_type-float-nan-inf) arguments. Keep in mind [numeric conversions issues](#common-issues-with-data-conversion), when using this function.
+:::
+
+**Example**
+
+Query:
+
+``` sql
+SELECT toInt32OrNull('-32'), toInt32OrNull('abc');
+```
+
+Result:
+
+```response
+   ┌─toInt32OrNull('-32')─┬─toInt32OrNull('abc')─┐
+1. │                  -32 │                 ᴺᵁᴸᴸ │
+   └──────────────────────┴──────────────────────┘
+```
+
+**See also**
+
+- [`toInt32`](#toint32).
+- [`toInt32OrZero`](#toint32orzero).
+- [`toInt32OrDefault`](#toint32ordefault).
+
+## toInt32OrDefault
+
+Like [`toInt32`](#toint32), takes an argument of type [String](../data-types/string.md) and tries to parse it to type `Int32`. If unsuccessful, returns the default type value.
+
+**Syntax**
+
+```sql
+toInt32OrDefault(expr, def)
+```
+
+**Arguments**
+
+- `expr` — Expression returning a number or a string with the decimal representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
+- `def` — The default value to return if parsing to type `Int32` is unsuccessful. [Int32](../data-types/int-uint.md).
+
+:::note
+Binary, octal, and hexadecimal representations of numbers are not supported. Leading zeroes are stripped.
+:::
+
+**Returned value**
+
+- 32-bit integer value if successful, otherwise returns the default value. [Int32](../data-types/int-uint.md).
+
+:::note
+- Function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
+- The default value type should be the same as the cast type.
+  :::
+
+:::danger
+An exception is thrown for [NaN and Inf](../data-types/float.md/#data_type-float-nan-inf) arguments. Keep in mind [numeric conversions issues](#common-issues-with-data-conversion), when using this function.
+:::
+
+**Example**
+
+Query:
+
+``` sql
+SELECT toInt32OrDefault('-32', cast('-1' as Int32)), toInt32OrDefault('abc', cast('-1' as Int32));
+```
+
+Result:
+
+```response
+   ┌─toInt32OrDefault('-32', CAST('-1', 'Int32'))─┬─toInt32OrDefault('abc', CAST('-1', 'Int32'))─┐
+1. │                                          -32 │                                           -1 │
+   └──────────────────────────────────────────────┴──────────────────────────────────────────────┘
+```
+
+**See also**
+
+- [`toInt32`](#toint32).
+- [`toInt32OrZero`](#toint32orzero).
+- [`toInt32OrNull`](#toint32ornull).
+
+## toInt64
+
+Converts an input value to a value of type `Int64`.
+
+**Syntax**
+
+```sql
+toInt64(expr)
+```
+
+**Arguments**
+
+- `expr` — Expression returning a number or a string with the decimal representation of a number. [Expression](../syntax.md/#syntax-expressions).
+
+:::note
+Binary, octal, and hexadecimal representations of numbers are not supported. Leading zeroes are stripped.
+:::
+
+**Returned value**
+
+- 64-bit integer value. [Int64](../data-types/int-uint.md). [Int64](../data-types/int-uint.md).
+
+:::note
+Function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
+:::
+
+:::danger
+An exception is thrown for [NaN and Inf](../data-types/float.md/#data_type-float-nan-inf) arguments. Keep in mind [numeric conversions issues](#common-issues-with-data-conversion), when using this function.
+:::
+
+**Example**
+
+Query:
+
+```sql
+SELECT
+    toInt64(-64),
+    toInt64(-64.64),
+    toInt64('-64');
+```
+
+Result:
+
+```response
+   ┌─toInt64(-64)─┬─toInt64(-64.64)─┬─toInt64('-64')─┐
+1. │          -64 │             -64 │            -64 │
+   └──────────────┴─────────────────┴────────────────┘
+```
+
+**See also**
+
+- [`toInt64OrZero`](#toint64orzero).
+- [`toInt64OrNull`](#toint64ornull).
+- [`toInt64OrDefault`](#toint64ordefault).
+
+## toInt64OrZero
+
+Like [`toInt64`](#toint64), takes an argument of type [String](../data-types/string.md) and tries to parse it to type `Int64`. If unsuccessful, returns `0`.
+
+**Syntax**
+
+```sql
+toInt64OrZero(expr)
+```
+
+**Arguments**
+
+- `expr` — Expression returning a number or a string with the decimal representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
+
+:::note
+Binary, octal, and hexadecimal representations of numbers are not supported. Leading zeroes are stripped.
+:::
+
+**Returned value**
+
+- 64-bit integer value if successful, otherwise `0`. [Int64](../data-types/int-uint.md).
+
+:::note
+Function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
+:::
+
+:::danger
+An exception is thrown for [NaN and Inf](../data-types/float.md/#data_type-float-nan-inf) arguments. Keep in mind [numeric conversions issues](#common-issues-with-data-conversion), when using this function.
+:::
+
+**Example**
+
+Query:
+
+``` sql
+SELECT
+    toInt64OrZero('-64'),
+    toInt64OrZero('abc');
+```
+
+Result:
+
+```response
+   ┌─toInt64OrZero('-64')─┬─toInt64OrZero('abc')─┐
+1. │                  -64 │                    0 │
+   └──────────────────────┴──────────────────────┘
+```
+
+**See also**
+
+- [`toInt64`](#toint64).
+- [`toInt64OrNull`](#toint64ornull).
+- [`toInt64OrDefault`](#toint64ordefault).
+
+## toInt64OrNull
+
+Like [`toInt64`], takes an argument of type [String](../data-types/string.md) and tries to parse it to type `Int64`. If unsuccessful, returns `NULL`.
+
+**Syntax**
+
+```sql
+toInt64OrNull(expr)
+```
+
+**Arguments**
+
+- `expr` — Expression returning a number or a string with the decimal representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
+
+:::note
+Binary, octal, and hexadecimal representations of numbers are not supported. Leading zeroes are stripped.
+:::
+
+**Returned value**
+
+- Integer value of type `Int64` if successful, otherwise `NULL`. [Int64](../data-types/int-uint.md) / [NULL](../data-types/nullable.md).
+
+:::note
+Function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
+:::
+
+:::danger
+An exception is thrown for [NaN and Inf](../data-types/float.md/#data_type-float-nan-inf) arguments. Keep in mind [numeric conversions issues](#common-issues-with-data-conversion), when using this function.
+:::
+
+**Example**
+
+Query:
+
+``` sql
+SELECT
+    toInt64OrNull('-64'),
+    toInt64OrNull('abc');
+```
+
+Result:
+
+```response
+   ┌─toInt64OrNull('-64')─┬─toInt64OrNull('abc')─┐
+1. │                  -64 │                 ᴺᵁᴸᴸ │
+   └──────────────────────┴──────────────────────┘
+```
+
+**See also**
+
+- [`toInt64`](#toint64).
+- [`toInt64OrZero`](#toint64orzero).
+- [`toInt64OrDefault`](#toint64ordefault).
+
+## toInt64OrDefault
+
+Like [`toInt64`](#toint64), takes an argument of type [String](../data-types/string.md) and tries to parse it to type `Int64`. If unsuccessful, returns the default type value.
+
+**Syntax**
+
+```sql
+toInt64OrDefault(expr, def)
+```
+
+**Arguments**
+
+- `expr` — Expression returning a number or a string with the decimal representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
+- `def` — The default value to return if parsing to type `Int64` is unsuccessful. [Int64](../data-types/int-uint.md).
+
+:::note
+Binary, octal, and hexadecimal representations of numbers are not supported. Leading zeroes are stripped.
+:::
+
+**Returned value**
+
+- Integer value of type `Int64` if successful, otherwise returns the default value. [Int64](../data-types/int-uint.md).
+
+:::note
+- Function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
+- The default value type should be the same as the cast type.
+  :::
+
+:::danger
+An exception is thrown for [NaN and Inf](../data-types/float.md/#data_type-float-nan-inf) arguments. Keep in mind [numeric conversions issues](#common-issues-with-data-conversion), when using this function.
+:::
+
+**Example**
+
+Query:
+
+``` sql
+SELECT
+    toInt64OrDefault('-64', CAST('-1', 'Int64')),
+    toInt64OrDefault('abc', CAST('-1', 'Int64'));
+```
+
+Result:
+
+```response
+   ┌─toInt64OrDefault('-64', CAST('-1', 'Int64'))─┬─toInt64OrDefault('abc', CAST('-1', 'Int64'))─┐
+1. │                                          -64 │                                           -1 │
+   └──────────────────────────────────────────────┴──────────────────────────────────────────────┘
+```
+
+**See also**
+
+- [`toInt64`](#toint64).
+- [`toInt64OrZero`](#toint64orzero).
+- [`toInt64OrNull`](#toint64ornull).
+
+## toInt128
+
+Converts an input value to a value of type `Int128`.
+
+**Syntax**
+
+```sql
+toInt128(expr)
+```
+
+**Arguments**
+
+- `expr` — Expression returning a number or a string with the decimal representation of a number. [Expression](../syntax.md/#syntax-expressions).
+
+:::note
+Binary, octal, and hexadecimal representations of numbers are not supported. Leading zeroes are stripped.
+:::
+
+**Returned value**
+
+- 128-bit integer value. [Int128](../data-types/int-uint.md).
+
+:::note
+Function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
+:::
+
+:::danger
+An exception is thrown for [NaN and Inf](../data-types/float.md/#data_type-float-nan-inf) arguments. Keep in mind [numeric conversions issues](#common-issues-with-data-conversion), when using this function.
+:::
+
+**Example**
+
+Query:
+
+```sql
+SELECT
+    toInt128(-128),
+    toInt128(-128.8),
+    toInt128('-128'),
+```
+
+Result:
+
+```response
+   ┌─toInt128(-128)─┬─toInt128(-128.8)─┬─toInt128('-128')─┐
+1. │           -128 │             -128 │             -128 │
+   └────────────────┴──────────────────┴──────────────────┘
+```
+
+**See also**
+
+- [`toInt128OrZero`](#toint128orzero).
+- [`toInt128OrNull`](#toint128ornull).
+- [`toInt128OrDefault`](#toint128ordefault).
+
+## toInt128OrZero
+
+Like [`toInt128`](#toint128), takes an argument of type [String](../data-types/string.md) and tries to parse it to type `Int128`. If unsuccessful, returns `0`.
+
+**Syntax**
+
+```sql
+toInt128OrZero(expr)
+```
+
+**Arguments**
+
+- `expr` — Expression returning a number or a string with the decimal representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
+
+:::note
+Binary, octal, and hexadecimal representations of numbers are not supported. Leading zeroes are stripped.
+:::
+
+**Returned value**
+
+- 128-bit integer value if successful, otherwise `0`. [Int128](../data-types/int-uint.md).
+
+:::note
+Function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
+:::
+
+:::danger
+An exception is thrown for [NaN and Inf](../data-types/float.md/#data_type-float-nan-inf) arguments. Keep in mind [numeric conversions issues](#common-issues-with-data-conversion), when using this function.
+:::
+
+**Example**
+
+Query:
+
+``` sql
+SELECT
+    toInt128OrZero('-128'),
+    toInt128OrZero('abc');
+```
+
+Result:
+
+```response
+   ┌─toInt128OrZero('-128')─┬─toInt128OrZero('abc')─┐
+1. │                   -128 │                     0 │
+   └────────────────────────┴───────────────────────┘
+```
+
+**See also**
+
+- [`toInt128`](#toint128).
+- [`toInt128OrNull`](#toint128ornull).
+- [`toInt128OrDefault`](#toint128ordefault).
+
+## toInt128OrNull
+
+Like [`toInt128`](#toint128), takes an argument of type [String](../data-types/string.md) and tries to parse it to type `Int128`. If unsuccessful, returns `NULL`.
+
+**Syntax**
+
+```sql
+toInt128OrNull(expr)
+```
+
+**Arguments**
+
+- `expr` — Expression returning a number or a string with the decimal representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
+
+:::note
+Binary, octal, and hexadecimal representations of numbers are not supported. Leading zeroes are stripped.
+:::
+
+**Returned value**
+
+- 128-bit integer value if successful, otherwise `NULL`. [Int128](../data-types/int-uint.md) / [NULL](../data-types/nullable.md).
+
+:::note
+Function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
+:::
+
+:::danger
+An exception is thrown for [NaN and Inf](../data-types/float.md/#data_type-float-nan-inf) arguments. Keep in mind [numeric conversions issues](#common-issues-with-data-conversion), when using this function.
+:::
+
+**Example**
+
+Query:
+
+``` sql
+SELECT
+    toInt128OrNull('-128'),
+    toInt128OrNull('abc');
+```
+
+Result:
+
+```response
+   ┌─toInt128OrNull('-128')─┬─toInt128OrNull('abc')─┐
+1. │                   -128 │                  ᴺᵁᴸᴸ │
+   └────────────────────────┴───────────────────────┘
+```
+
+**See also**
+
+- [`toInt128`](#toint128).
+- [`toInt128OrZero`](#toint128orzero).
+- [`toInt128OrDefault`](#toint128ordefault).
+
+## toInt128OrDefault
+
+Like [`toInt128`](#toint128), takes an argument of type [String](../data-types/string.md) and tries to parse it to type `Int128`. If unsuccessful, returns the default type value.
+
+**Syntax**
+
+```sql
+toInt128OrDefault(expr, def)
+```
+
+**Arguments**
+
+- `expr` — Expression returning a number or a string with the decimal representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
+- `def` — The default value to return if parsing to type `Int128` is unsuccessful. [Int128](../data-types/int-uint.md).
+
+:::note
+Binary, octal, and hexadecimal representations of numbers are not supported. Leading zeroes are stripped.
+:::
+
+**Returned value**
+
+- 128-bit integer value if successful, otherwise returns the default value. [Int128](../data-types/int-uint.md).
+
+:::note
+- Function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
+- The default value type should be the same as the cast type.
+:::
+
+:::danger
+An exception is thrown for [NaN and Inf](../data-types/float.md/#data_type-float-nan-inf) arguments. Keep in mind [numeric conversions issues](#common-issues-with-data-conversion), when using this function.
+:::
+
+**Example**
+
+Query:
+
+``` sql
+SELECT
+    toInt128OrDefault('-128', CAST('-1', 'Int128')),
+    toInt128OrDefault('abc', CAST('-1', 'Int128'));
+```
+
+Result:
+
+```response
+   ┌─toInt128OrDefault('-128', CAST('-1', 'Int128'))─┬─toInt128OrDefault('abc', CAST('-1', 'Int128'))─┐
+1. │                                            -128 │                                             -1 │
+   └─────────────────────────────────────────────────┴────────────────────────────────────────────────┘
+```
+
+**See also**
+
+- [`toInt128`](#toint128).
+- [`toInt128OrZero`](#toint128orzero).
+- [`toInt128OrNull`](#toint128ornull).
+
+## toInt256
+
+Converts an input value to a value of type `Int256`.
+
+**Syntax**
+
+```sql
+toInt256(expr)
+```
+
+**Arguments**
+
+- `expr` — Expression returning a number or a string with the decimal representation of a number. [Expression](../syntax.md/#syntax-expressions).
+
+:::note
+Binary, octal, and hexadecimal representations of numbers are not supported. Leading zeroes are stripped.
+:::
+
+**Returned value**
+
+- 256-bit integer value. [Int256](../data-types/int-uint.md).
+
+:::note
+Function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
+:::
+
+:::danger
+An exception is thrown for [NaN and Inf](../data-types/float.md/#data_type-float-nan-inf) arguments. Keep in mind [numeric conversions issues](#common-issues-with-data-conversion), when using this function.
+:::
+
+**Example**
+
+Query:
+
+```sql
+SELECT
+    toInt256(-256),
+    toInt256(-256.256),
+    toInt256('-256');
+```
+
+Result:
+
+```response
+   ┌─toInt256(-256)─┬─toInt256(-256.256)─┬─toInt256('-256')─┐
+1. │           -256 │               -256 │             -256 │
+   └────────────────┴────────────────────┴──────────────────┘
+```
+
+**See also**
+
+- [`toInt256OrZero`](#toint256orzero).
+- [`toInt256OrNull`](#toint256ornull).
+- [`toInt256OrDefault`](#toint256ordefault).
+
+## toInt256OrZero
+
+Like [`toInt256`](#toint256), takes an argument of type [String](../data-types/string.md) and tries to parse it to type `Int256`. If unsuccessful, returns `0`.
+
+**Syntax**
+
+```sql
+toInt256OrZero(expr)
+```
+
+**Arguments**
+
+- `expr` — Expression returning a number or a string with the decimal representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
+
+:::note
+Binary, octal, and hexadecimal representations of numbers are not supported. Leading zeroes are stripped.
+:::
+
+**Returned value**
+
+- 256-bit integer value if successful, otherwise `0`. [Int256](../data-types/int-uint.md).
+
+:::note
+Functions uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
+:::
+
+:::danger
+An exception is thrown for [NaN and Inf](../data-types/float.md/#data_type-float-nan-inf) arguments. Keep in mind [numeric conversions issues](#common-issues-with-data-conversion), when using this function.
+:::
+
+**Example**
+
+Query:
+
+``` sql
+SELECT
+    toInt256OrZero('-256'),
+    toInt256OrZero('abc');
+```
+
+Result:
+
+```response
+   ┌─toInt256OrZero('-256')─┬─toInt256OrZero('abc')─┐
+1. │                   -256 │                     0 │
+   └────────────────────────┴───────────────────────┘
+```
+
+**See also**
+
+- [`toInt256`](#toint256).
+- [`toInt256OrNull`](#toint256ornull).
+- [`toInt256OrDefault`](#toint256ordefault).
+
+## toInt256OrNull
+
+Like [`toInt256`](#toint256), takes an argument of type [String](../data-types/string.md) and tries to parse it to type `Int256`. If unsuccessful, returns `NULL`.
+
+**Syntax**
+
+```sql
+toInt256OrNull(expr)
+```
+
+**Arguments**
+
+- `expr` — Expression returning a number or a string with the decimal representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
+
+:::note
+Binary, octal, and hexadecimal representations of numbers are not supported. Leading zeroes are stripped.
+:::
+
+**Returned value**
+
+- 256-bit integer value if successful, otherwise `NULL`. [Int256](../data-types/int-uint.md) / [NULL](../data-types/nullable.md).
+
+:::note
+Function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
+:::
+
+:::danger
+An exception is thrown for [NaN and Inf](../data-types/float.md/#data_type-float-nan-inf) arguments. Keep in mind [numeric conversions issues](#common-issues-with-data-conversion), when using this function.
+:::
+
+**Example**
+
+Query:
+
+``` sql
+SELECT
+    toInt256OrNull('-256'),
+    toInt256OrNull('abc');
+```
+
+Result:
+
+```response
+   ┌─toInt256OrNull('-256')─┬─toInt256OrNull('abc')─┐
+1. │                   -256 │                  ᴺᵁᴸᴸ  │
+   └────────────────────────┴───────────────────────┘
+```
+
+**See also**
+
+- [`toInt256`](#toint256).
+- [`toInt256OrZero`](#toint256orzero).
+- [`toInt256OrDefault`](#toint256ordefault).
+
+## toInt256OrDefault
+
+Like [`toInt256`](#toint256), takes an argument of type [String](../data-types/string.md) and tries to parse it to type `Int256`. If unsuccessful, returns the default type value.
+
+**Syntax**
+
+```sql
+toInt256OrDefault(expr, def)
+```
+
+**Arguments**
+
+- `expr` — Expression returning a number or a string with the decimal representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
+- `def` — The default value to return if parsing to type `Int256` is unsuccessful. [Int256](../data-types/int-uint.md).
+
+:::note
+Binary, octal, and hexadecimal representations of numbers are not supported. Leading zeroes are stripped.
+:::
+
+**Returned value**
+
+- 256-bit integer value if successful, otherwise returns the default value. [Int256](../data-types/int-uint.md).
+
+:::note
+- Function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
+- The default value type should be the same as the cast type.
+:::
+
+:::danger
+An exception is thrown for [NaN and Inf](../data-types/float.md/#data_type-float-nan-inf) arguments. Keep in mind [numeric conversions issues](#common-issues-with-data-conversion), when using this function.
+:::
+
+**Example**
+
+Query:
+
+``` sql
+SELECT
+    toInt256OrDefault('-256', CAST('-1', 'Int256')),
+    toInt256OrDefault('abc', CAST('-1', 'Int256'));
+```
+
+Result:
+
+```response
+   ┌─toInt256OrDefault('-256', CAST('-1', 'Int256'))─┬─toInt256OrDefault('abc', CAST('-1', 'Int256'))─┐
+1. │                                            -256 │                                             -1 │
+   └─────────────────────────────────────────────────┴────────────────────────────────────────────────┘
+```
+
+**See also**
+
+- [`toInt256`](#toint256).
+- [`toInt256OrZero`](#toint256orzero).
+- [`toInt256OrNull`](#toint256ornull).
 
 ## toUInt(8\|16\|32\|64\|256)
 
@@ -167,7 +1370,7 @@ Converts an input value to the [UInt](../data-types/int-uint.md) data type. This
 
 - Integer value in the `UInt8`, `UInt16`, `UInt32`, `UInt64` or `UInt256` data type.
 
-Functions use [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning they truncate fractional digits of numbers.
+Functions use [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
 
 The behavior of functions for negative arguments and for the [NaN and Inf](../data-types/float.md/#data_type-float-nan-inf) arguments is undefined. If you pass a string with a negative number, for example `'-32'`, ClickHouse raises an exception. Remember about [numeric conversions issues](#common-issues-with-data-conversion), when using the functions.
 
@@ -2289,7 +3492,7 @@ Result:
 └─────────────────────┴─────────────────┴─────────────────────────────────────┘
 ```
 
-**See Also**
+**See also**
 
 - [RFC 1123](https://datatracker.ietf.org/doc/html/rfc1123)
 - [toDate](#todate)

From e25fbb6d478cc36af44d6ea60d35c28309bcb603 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?=
 <benjamin.antal@clickhouse.com>
Date: Wed, 17 Jul 2024 15:28:08 +0000
Subject: [PATCH 0361/1488] Fix style

---
 .../NamedCollections/NamedCollectionsMetadataStorage.cpp      | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Common/NamedCollections/NamedCollectionsMetadataStorage.cpp b/src/Common/NamedCollections/NamedCollectionsMetadataStorage.cpp
index 189eb58196d..36191b89e86 100644
--- a/src/Common/NamedCollections/NamedCollectionsMetadataStorage.cpp
+++ b/src/Common/NamedCollections/NamedCollectionsMetadataStorage.cpp
@@ -173,7 +173,7 @@ private:
     {
         const auto file_name_as_path = fs::path(file_name);
         if (file_name_as_path.is_absolute())
-            throw Exception(ErrorCodes::LOGICAL_ERROR, "Filename {} cannot be an absolute path!", file_name);
+            throw Exception(ErrorCodes::LOGICAL_ERROR, "Filename {} cannot be an absolute path", file_name);
 
         return fs::path(root_path) / file_name_as_path;
     }
@@ -328,7 +328,7 @@ private:
     {
         const auto file_name_as_path = fs::path(file_name);
         if (file_name_as_path.is_absolute())
-            throw Exception(ErrorCodes::LOGICAL_ERROR, "Filename {} cannot be an absolute path!", file_name);
+            throw Exception(ErrorCodes::LOGICAL_ERROR, "Filename {} cannot be an absolute path", file_name);
 
         return fs::path(root_path) / file_name_as_path;
     }

From 05874d0b85d78c1067c1db5332c7cc74b94d88cd Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Wed, 17 Jul 2024 15:37:55 +0000
Subject: [PATCH 0362/1488] Fixing style.

---
 src/Columns/ColumnDecimal.cpp  |  1 -
 src/Columns/ColumnNullable.cpp |  2 +-
 src/Columns/ColumnSparse.cpp   | 18 ------------------
 src/Columns/ColumnSparse.h     |  3 ---
 4 files changed, 1 insertion(+), 23 deletions(-)

diff --git a/src/Columns/ColumnDecimal.cpp b/src/Columns/ColumnDecimal.cpp
index ed9c699a841..e27807950ae 100644
--- a/src/Columns/ColumnDecimal.cpp
+++ b/src/Columns/ColumnDecimal.cpp
@@ -28,7 +28,6 @@ namespace ErrorCodes
     extern const int PARAMETER_OUT_OF_BOUND;
     extern const int SIZES_OF_COLUMNS_DOESNT_MATCH;
     extern const int NOT_IMPLEMENTED;
-    extern const int LOGICAL_ERROR;
 }
 
 template <is_decimal T>
diff --git a/src/Columns/ColumnNullable.cpp b/src/Columns/ColumnNullable.cpp
index 64e99a3bbe8..867c9149242 100644
--- a/src/Columns/ColumnNullable.cpp
+++ b/src/Columns/ColumnNullable.cpp
@@ -65,7 +65,7 @@ WeakHash32 ColumnNullable::getWeakHash32() const
     const auto & null_map_data = getNullMapData();
     auto & hash_data = hash.getData();
 
-    /// Use defualt for nulls.
+    /// Use default for nulls.
     for (size_t row = 0; row < s; ++row)
         if (null_map_data[row])
             hash_data[row] = WeakHash32::kDefaultInitialValue;
diff --git a/src/Columns/ColumnSparse.cpp b/src/Columns/ColumnSparse.cpp
index 0d103a263dd..8f98a4433d3 100644
--- a/src/Columns/ColumnSparse.cpp
+++ b/src/Columns/ColumnSparse.cpp
@@ -809,24 +809,6 @@ size_t ColumnSparse::getValueIndex(size_t n) const
     return it - offsets_data.begin() + 1;
 }
 
-size_t ColumnSparse::getFirstDefaultValueIndex() const
-{
-    if (getNumberOfDefaultRows() == 0)
-        return size();
-
-    const auto & offsets_data = getOffsetsData();
-    size_t off_size = offsets_data.size();
-
-    if (off_size == 0 || offsets_data[0] > 0)
-        return 0;
-
-    size_t idx = 0;
-    while (idx + 1 < off_size && offsets_data[idx] + 1 == offsets_data[idx + 1])
-        ++idx;
-
-    return offsets_data[idx] + 1;
-}
-
 ColumnSparse::Iterator ColumnSparse::getIterator(size_t n) const
 {
     const auto & offsets_data = getOffsetsData();
diff --git a/src/Columns/ColumnSparse.h b/src/Columns/ColumnSparse.h
index a5d4d788b17..392a6910956 100644
--- a/src/Columns/ColumnSparse.h
+++ b/src/Columns/ColumnSparse.h
@@ -173,9 +173,6 @@ public:
     /// O(log(offsets.size())) complexity,
     size_t getValueIndex(size_t n) const;
 
-    /// Returns an index of the first default value, or size() if there is no defaults.
-    size_t getFirstDefaultValueIndex() const;
-
     const IColumn & getValuesColumn() const { return *values; }
     IColumn & getValuesColumn() { return *values; }
 

From 245bc587ae5d9cb6751153b7bf24dcf88487d5dd Mon Sep 17 00:00:00 2001
From: Shaun Struwig <41984034+Blargian@users.noreply.github.com>
Date: Wed, 17 Jul 2024 17:57:28 +0200
Subject: [PATCH 0363/1488] add 'fqdn' to list

---
 utils/check-style/aspell-ignore/en/aspell-dict.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt
index 4e7bf7f686e..b4b18f9bbf6 100644
--- a/utils/check-style/aspell-ignore/en/aspell-dict.txt
+++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt
@@ -1653,6 +1653,7 @@ formated
 formatschema
 formatter
 formatters
+fqdn
 frac
 freezed
 fromDaysSinceYearZero

From 5608914bca8b36920f8012fa48b6617512629cfe Mon Sep 17 00:00:00 2001
From: Nikita Taranov <nikita.taranov@clickhouse.com>
Date: Wed, 17 Jul 2024 16:59:31 +0100
Subject: [PATCH 0364/1488] impl

---
 base/base/defines.h                               | 15 +++++++++++----
 src/Analyzer/QueryTreePassManager.cpp             |  6 +++---
 src/Columns/ColumnAggregateFunction.cpp           |  4 ++--
 src/Columns/ColumnAggregateFunction.h             |  6 +++---
 src/Columns/ColumnArray.cpp                       |  6 +++---
 src/Columns/ColumnArray.h                         |  6 +++---
 src/Columns/ColumnCompressed.h                    |  4 ++--
 src/Columns/ColumnConst.h                         |  8 ++++----
 src/Columns/ColumnDecimal.cpp                     |  4 ++--
 src/Columns/ColumnDecimal.h                       |  8 ++++----
 src/Columns/ColumnDynamic.cpp                     |  8 ++++----
 src/Columns/ColumnDynamic.h                       |  4 ++--
 src/Columns/ColumnFixedString.cpp                 |  6 +++---
 src/Columns/ColumnFixedString.h                   |  8 ++++----
 src/Columns/ColumnFunction.cpp                    |  4 ++--
 src/Columns/ColumnFunction.h                      |  6 +++---
 src/Columns/ColumnLowCardinality.cpp              |  6 +++---
 src/Columns/ColumnLowCardinality.h                |  6 +++---
 src/Columns/ColumnMap.cpp                         |  8 ++++----
 src/Columns/ColumnMap.h                           |  4 ++--
 src/Columns/ColumnNullable.cpp                    |  8 ++++----
 src/Columns/ColumnNullable.h                      |  6 +++---
 src/Columns/ColumnObject.cpp                      |  4 ++--
 src/Columns/ColumnObject.h                        |  4 ++--
 src/Columns/ColumnSparse.cpp                      |  6 +++---
 src/Columns/ColumnSparse.h                        |  6 +++---
 src/Columns/ColumnString.cpp                      |  4 ++--
 src/Columns/ColumnString.h                        |  8 ++++----
 src/Columns/ColumnTuple.cpp                       |  8 ++++----
 src/Columns/ColumnTuple.h                         |  6 +++---
 src/Columns/ColumnUnique.h                        |  4 ++--
 src/Columns/ColumnVariant.cpp                     |  8 ++++----
 src/Columns/ColumnVariant.h                       |  4 ++--
 src/Columns/ColumnVector.cpp                      |  2 +-
 src/Columns/ColumnVector.h                        |  8 ++++----
 src/Columns/IColumn.cpp                           |  2 +-
 src/Columns/IColumn.h                             | 10 +++++-----
 src/Columns/IColumnDummy.h                        |  6 +++---
 src/Columns/IColumnUnique.h                       |  2 +-
 .../benchmark_column_insert_many_from.cpp         |  2 +-
 .../Config/AbstractConfigurationComparison.cpp    |  2 +-
 src/Common/MemoryTracker.cpp                      |  2 +-
 src/Common/PageCache.cpp                          |  2 +-
 src/Common/assert_cast.h                          |  2 +-
 src/Common/tests/gtest_rw_lock.cpp                |  2 +-
 src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp   |  6 +++---
 src/IO/tests/gtest_memory_resize.cpp              |  6 +++---
 src/IO/tests/gtest_writebuffer_s3.cpp             |  4 ++--
 src/Interpreters/Cache/FileCache.cpp              |  8 ++++----
 src/Interpreters/Cache/FileSegment.cpp            |  6 +++---
 src/Interpreters/Cache/Metadata.cpp               |  2 +-
 src/Interpreters/executeDDLQueryOnCluster.cpp     |  2 +-
 .../gtest_exception_on_incorrect_pipeline.cpp     |  2 +-
 .../tests/gtest_check_sorted_stream.cpp           |  8 ++++----
 src/Server/TCPHandler.cpp                         |  2 +-
 src/Storages/MaterializedView/RefreshTask.cpp     |  2 +-
 src/Storages/StorageReplicatedMergeTree.cpp       |  4 ++--
 57 files changed, 152 insertions(+), 145 deletions(-)

diff --git a/base/base/defines.h b/base/base/defines.h
index 2fc54c37bde..cf3d357da18 100644
--- a/base/base/defines.h
+++ b/base/base/defines.h
@@ -87,10 +87,17 @@
 #   define ASAN_POISON_MEMORY_REGION(a, b)
 #endif
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
-    #if !defined(NDEBUG) || defined(ADDRESS_SANITIZER) || defined(THREAD_SANITIZER) || defined(MEMORY_SANITIZER) || defined(UNDEFINED_BEHAVIOR_SANITIZER)
-        #define ABORT_ON_LOGICAL_ERROR
-    #endif
+/// We used to have only ABORT_ON_LOGICAL_ERROR macro, but most of its uses were actually in places where we didn't care about logical errors
+/// but wanted to check exactly if the current build type is debug or with sanitizer. This new macro is introduced to fix those places.
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
+#    if !defined(NDEBUG) || defined(ADDRESS_SANITIZER) || defined(THREAD_SANITIZER) || defined(MEMORY_SANITIZER) \
+        || defined(UNDEFINED_BEHAVIOR_SANITIZER)
+#        define DEBUG_OR_SANITIZER_BUILD
+#    endif
+#endif
+
+#if !defined(ABORT_ON_LOGICAL_ERROR) && defined(DEBUG_OR_SANITIZER_BUILD)
+#    define ABORT_ON_LOGICAL_ERROR
 #endif
 
 /// chassert(x) is similar to assert(x), but:
diff --git a/src/Analyzer/QueryTreePassManager.cpp b/src/Analyzer/QueryTreePassManager.cpp
index f7919b6422c..4443f83596f 100644
--- a/src/Analyzer/QueryTreePassManager.cpp
+++ b/src/Analyzer/QueryTreePassManager.cpp
@@ -62,7 +62,7 @@ namespace ErrorCodes
 namespace
 {
 
-#if defined(ABORT_ON_LOGICAL_ERROR)
+#if defined(DEBUG_OR_SANITIZER_BUILD)
 
 /** This visitor checks if Query Tree structure is valid after each pass
   * in debug build.
@@ -183,7 +183,7 @@ void QueryTreePassManager::run(QueryTreeNodePtr query_tree_node)
     for (size_t i = 0; i < passes_size; ++i)
     {
         passes[i]->run(query_tree_node, current_context);
-#if defined(ABORT_ON_LOGICAL_ERROR)
+#if defined(DEBUG_OR_SANITIZER_BUILD)
         ValidationChecker(passes[i]->getName()).visit(query_tree_node);
 #endif
     }
@@ -208,7 +208,7 @@ void QueryTreePassManager::run(QueryTreeNodePtr query_tree_node, size_t up_to_pa
     for (size_t i = 0; i < up_to_pass_index; ++i)
     {
         passes[i]->run(query_tree_node, current_context);
-#if defined(ABORT_ON_LOGICAL_ERROR)
+#if defined(DEBUG_OR_SANITIZER_BUILD)
         ValidationChecker(passes[i]->getName()).visit(query_tree_node);
 #endif
     }
diff --git a/src/Columns/ColumnAggregateFunction.cpp b/src/Columns/ColumnAggregateFunction.cpp
index cfd07c27765..955981a972d 100644
--- a/src/Columns/ColumnAggregateFunction.cpp
+++ b/src/Columns/ColumnAggregateFunction.cpp
@@ -267,7 +267,7 @@ bool ColumnAggregateFunction::structureEquals(const IColumn & to) const
 }
 
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
 void ColumnAggregateFunction::insertRangeFrom(const IColumn & from, size_t start, size_t length)
 #else
 void ColumnAggregateFunction::doInsertRangeFrom(const IColumn & from, size_t start, size_t length)
@@ -466,7 +466,7 @@ void ColumnAggregateFunction::insertFromWithOwnership(const IColumn & from, size
     insertMergeFrom(from, n);
 }
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
 void ColumnAggregateFunction::insertFrom(const IColumn & from, size_t n)
 #else
 void ColumnAggregateFunction::doInsertFrom(const IColumn & from, size_t n)
diff --git a/src/Columns/ColumnAggregateFunction.h b/src/Columns/ColumnAggregateFunction.h
index 1be7a862438..fe678fc1eaa 100644
--- a/src/Columns/ColumnAggregateFunction.h
+++ b/src/Columns/ColumnAggregateFunction.h
@@ -145,7 +145,7 @@ public:
 
     void insertData(const char * pos, size_t length) override;
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     void insertFrom(const IColumn & from, size_t n) override;
 #else
     using IColumn::insertFrom;
@@ -189,7 +189,7 @@ public:
 
     void protect() override;
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     void insertRangeFrom(const IColumn & from, size_t start, size_t length) override;
 #else
     void doInsertRangeFrom(const IColumn & from, size_t start, size_t length) override;
@@ -212,7 +212,7 @@ public:
 
     MutableColumns scatter(ColumnIndex num_columns, const Selector & selector) const override;
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     int compareAt(size_t, size_t, const IColumn &, int) const override
 #else
     int doCompareAt(size_t, size_t, const IColumn &, int) const override
diff --git a/src/Columns/ColumnArray.cpp b/src/Columns/ColumnArray.cpp
index 5d7350f3a79..598d501a2b8 100644
--- a/src/Columns/ColumnArray.cpp
+++ b/src/Columns/ColumnArray.cpp
@@ -337,7 +337,7 @@ bool ColumnArray::tryInsert(const Field & x)
     return true;
 }
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
 void ColumnArray::insertFrom(const IColumn & src_, size_t n)
 #else
 void ColumnArray::doInsertFrom(const IColumn & src_, size_t n)
@@ -396,7 +396,7 @@ int ColumnArray::compareAtImpl(size_t n, size_t m, const IColumn & rhs_, int nan
             : 1);
 }
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
 int ColumnArray::compareAt(size_t n, size_t m, const IColumn & rhs_, int nan_direction_hint) const
 #else
 int ColumnArray::doCompareAt(size_t n, size_t m, const IColumn & rhs_, int nan_direction_hint) const
@@ -543,7 +543,7 @@ void ColumnArray::getExtremes(Field & min, Field & max) const
 }
 
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
 void ColumnArray::insertRangeFrom(const IColumn & src, size_t start, size_t length)
 #else
 void ColumnArray::doInsertRangeFrom(const IColumn & src, size_t start, size_t length)
diff --git a/src/Columns/ColumnArray.h b/src/Columns/ColumnArray.h
index 6cd3e2f6c3b..6f735fe9dc3 100644
--- a/src/Columns/ColumnArray.h
+++ b/src/Columns/ColumnArray.h
@@ -84,14 +84,14 @@ public:
     void updateHashWithValue(size_t n, SipHash & hash) const override;
     void updateWeakHash32(WeakHash32 & hash) const override;
     void updateHashFast(SipHash & hash) const override;
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
 #else
     void doInsertRangeFrom(const IColumn & src, size_t start, size_t length) override;
 #endif
     void insert(const Field & x) override;
     bool tryInsert(const Field & x) override;
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     void insertFrom(const IColumn & src_, size_t n) override;
 #else
     void doInsertFrom(const IColumn & src_, size_t n) override;
@@ -103,7 +103,7 @@ public:
     ColumnPtr permute(const Permutation & perm, size_t limit) const override;
     ColumnPtr index(const IColumn & indexes, size_t limit) const override;
     template <typename Type> ColumnPtr indexImpl(const PaddedPODArray<Type> & indexes, size_t limit) const;
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     int compareAt(size_t n, size_t m, const IColumn & rhs_, int nan_direction_hint) const override;
 #else
     int doCompareAt(size_t n, size_t m, const IColumn & rhs_, int nan_direction_hint) const override;
diff --git a/src/Columns/ColumnCompressed.h b/src/Columns/ColumnCompressed.h
index 5e455709fec..10649602920 100644
--- a/src/Columns/ColumnCompressed.h
+++ b/src/Columns/ColumnCompressed.h
@@ -85,7 +85,7 @@ public:
     bool isDefaultAt(size_t) const override { throwMustBeDecompressed(); }
     void insert(const Field &) override { throwMustBeDecompressed(); }
     bool tryInsert(const Field &) override { throwMustBeDecompressed(); }
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     void insertRangeFrom(const IColumn &, size_t, size_t) override { throwMustBeDecompressed(); }
 #else
     void doInsertRangeFrom(const IColumn &, size_t, size_t) override { throwMustBeDecompressed(); }
@@ -104,7 +104,7 @@ public:
     void expand(const Filter &, bool) override { throwMustBeDecompressed(); }
     ColumnPtr permute(const Permutation &, size_t) const override { throwMustBeDecompressed(); }
     ColumnPtr index(const IColumn &, size_t) const override { throwMustBeDecompressed(); }
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     int compareAt(size_t, size_t, const IColumn &, int) const override { throwMustBeDecompressed(); }
 #else
     int doCompareAt(size_t, size_t, const IColumn &, int) const override { throwMustBeDecompressed(); }
diff --git a/src/Columns/ColumnConst.h b/src/Columns/ColumnConst.h
index b55a1f42037..e419dbd2c4c 100644
--- a/src/Columns/ColumnConst.h
+++ b/src/Columns/ColumnConst.h
@@ -123,7 +123,7 @@ public:
         return data->isNullAt(0);
     }
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     void insertRangeFrom(const IColumn &, size_t /*start*/, size_t length) override
 #else
     void doInsertRangeFrom(const IColumn &, size_t /*start*/, size_t length) override
@@ -151,7 +151,7 @@ public:
         ++s;
     }
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     void insertFrom(const IColumn &, size_t) override
 #else
     void doInsertFrom(const IColumn &, size_t) override
@@ -160,7 +160,7 @@ public:
         ++s;
     }
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     void insertManyFrom(const IColumn & /*src*/, size_t /* position */, size_t length) override { s += length; }
 #else
     void doInsertManyFrom(const IColumn & /*src*/, size_t /* position */, size_t length) override { s += length; }
@@ -237,7 +237,7 @@ public:
         return data->allocatedBytes() + sizeof(s);
     }
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     int compareAt(size_t, size_t, const IColumn & rhs, int nan_direction_hint) const override
 #else
     int doCompareAt(size_t, size_t, const IColumn & rhs, int nan_direction_hint) const override
diff --git a/src/Columns/ColumnDecimal.cpp b/src/Columns/ColumnDecimal.cpp
index cf413f790a7..8e1a96b6ed2 100644
--- a/src/Columns/ColumnDecimal.cpp
+++ b/src/Columns/ColumnDecimal.cpp
@@ -32,7 +32,7 @@ namespace ErrorCodes
 }
 
 template <is_decimal T>
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
 int ColumnDecimal<T>::compareAt(size_t n, size_t m, const IColumn & rhs_, int) const
 #else
 int ColumnDecimal<T>::doCompareAt(size_t n, size_t m, const IColumn & rhs_, int) const
@@ -335,7 +335,7 @@ void ColumnDecimal<T>::insertData(const char * src, size_t /*length*/)
 }
 
 template <is_decimal T>
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
 void ColumnDecimal<T>::insertRangeFrom(const IColumn & src, size_t start, size_t length)
 #else
 void ColumnDecimal<T>::doInsertRangeFrom(const IColumn & src, size_t start, size_t length)
diff --git a/src/Columns/ColumnDecimal.h b/src/Columns/ColumnDecimal.h
index 32efeb643a6..3985a667135 100644
--- a/src/Columns/ColumnDecimal.h
+++ b/src/Columns/ColumnDecimal.h
@@ -55,13 +55,13 @@ public:
     void reserve(size_t n) override { data.reserve_exact(n); }
     void shrinkToFit() override { data.shrink_to_fit(); }
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     void insertFrom(const IColumn & src, size_t n) override { data.push_back(static_cast<const Self &>(src).getData()[n]); }
 #else
     void doInsertFrom(const IColumn & src, size_t n) override { data.push_back(static_cast<const Self &>(src).getData()[n]); }
 #endif
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     void insertManyFrom(const IColumn & src, size_t position, size_t length) override
 #else
     void doInsertManyFrom(const IColumn & src, size_t position, size_t length) override
@@ -76,7 +76,7 @@ public:
     void insertManyDefaults(size_t length) override { data.resize_fill(data.size() + length); }
     void insert(const Field & x) override { data.push_back(x.get<T>()); }
     bool tryInsert(const Field & x) override;
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
 #else
     void doInsertRangeFrom(const IColumn & src, size_t start, size_t length) override;
@@ -104,7 +104,7 @@ public:
     void updateHashWithValue(size_t n, SipHash & hash) const override;
     void updateWeakHash32(WeakHash32 & hash) const override;
     void updateHashFast(SipHash & hash) const override;
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     int compareAt(size_t n, size_t m, const IColumn & rhs_, int nan_direction_hint) const override;
 #else
     int doCompareAt(size_t n, size_t m, const IColumn & rhs_, int nan_direction_hint) const override;
diff --git a/src/Columns/ColumnDynamic.cpp b/src/Columns/ColumnDynamic.cpp
index c735238f515..a92d54dd675 100644
--- a/src/Columns/ColumnDynamic.cpp
+++ b/src/Columns/ColumnDynamic.cpp
@@ -215,7 +215,7 @@ bool ColumnDynamic::tryInsert(const DB::Field & x)
 }
 
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
 void ColumnDynamic::insertFrom(const DB::IColumn & src_, size_t n)
 #else
 void ColumnDynamic::doInsertFrom(const DB::IColumn & src_, size_t n)
@@ -269,7 +269,7 @@ void ColumnDynamic::doInsertFrom(const DB::IColumn & src_, size_t n)
     variant_col.insertIntoVariantFrom(string_variant_discr, *tmp_string_column, 0);
 }
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
 void ColumnDynamic::insertRangeFrom(const DB::IColumn & src_, size_t start, size_t length)
 #else
 void ColumnDynamic::doInsertRangeFrom(const DB::IColumn & src_, size_t start, size_t length)
@@ -439,7 +439,7 @@ void ColumnDynamic::doInsertRangeFrom(const DB::IColumn & src_, size_t start, si
     }
 }
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
 void ColumnDynamic::insertManyFrom(const DB::IColumn & src_, size_t position, size_t length)
 #else
 void ColumnDynamic::doInsertManyFrom(const DB::IColumn & src_, size_t position, size_t length)
@@ -603,7 +603,7 @@ void ColumnDynamic::updateHashWithValue(size_t n, SipHash & hash) const
     variant_col.getVariantByGlobalDiscriminator(discr).updateHashWithValue(variant_col.offsetAt(n), hash);
 }
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
 int ColumnDynamic::compareAt(size_t n, size_t m, const DB::IColumn & rhs, int nan_direction_hint) const
 #else
 int ColumnDynamic::doCompareAt(size_t n, size_t m, const DB::IColumn & rhs, int nan_direction_hint) const
diff --git a/src/Columns/ColumnDynamic.h b/src/Columns/ColumnDynamic.h
index 9abddc7a26d..fa8ec55c60a 100644
--- a/src/Columns/ColumnDynamic.h
+++ b/src/Columns/ColumnDynamic.h
@@ -143,7 +143,7 @@ public:
     void insert(const Field & x) override;
     bool tryInsert(const Field & x) override;
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     void insertFrom(const IColumn & src_, size_t n) override;
     void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
     void insertManyFrom(const IColumn & src, size_t position, size_t length) override;
@@ -220,7 +220,7 @@ public:
         return scattered_columns;
     }
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     int compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const override;
 #else
     int doCompareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const override;
diff --git a/src/Columns/ColumnFixedString.cpp b/src/Columns/ColumnFixedString.cpp
index 1c2de203a94..db697f6372b 100644
--- a/src/Columns/ColumnFixedString.cpp
+++ b/src/Columns/ColumnFixedString.cpp
@@ -74,7 +74,7 @@ bool ColumnFixedString::tryInsert(const Field & x)
     return true;
 }
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
 void ColumnFixedString::insertFrom(const IColumn & src_, size_t index)
 #else
 void ColumnFixedString::doInsertFrom(const IColumn & src_, size_t index)
@@ -90,7 +90,7 @@ void ColumnFixedString::doInsertFrom(const IColumn & src_, size_t index)
     memcpySmallAllowReadWriteOverflow15(chars.data() + old_size, &src.chars[n * index], n);
 }
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
 void ColumnFixedString::insertManyFrom(const IColumn & src, size_t position, size_t length)
 #else
 void ColumnFixedString::doInsertManyFrom(const IColumn & src, size_t position, size_t length)
@@ -227,7 +227,7 @@ size_t ColumnFixedString::estimateCardinalityInPermutedRange(const Permutation &
     return elements.size();
 }
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
 void ColumnFixedString::insertRangeFrom(const IColumn & src, size_t start, size_t length)
 #else
 void ColumnFixedString::doInsertRangeFrom(const IColumn & src, size_t start, size_t length)
diff --git a/src/Columns/ColumnFixedString.h b/src/Columns/ColumnFixedString.h
index 6e88136fc50..6e7ff488f9a 100644
--- a/src/Columns/ColumnFixedString.h
+++ b/src/Columns/ColumnFixedString.h
@@ -98,13 +98,13 @@ public:
 
     bool tryInsert(const Field & x) override;
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     void insertFrom(const IColumn & src_, size_t index) override;
 #else
     void doInsertFrom(const IColumn & src_, size_t index) override;
 #endif
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     void insertManyFrom(const IColumn & src, size_t position, size_t length) override;
 #else
     void doInsertManyFrom(const IColumn & src, size_t position, size_t length) override;
@@ -137,7 +137,7 @@ public:
 
     void updateHashFast(SipHash & hash) const override;
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     int compareAt(size_t p1, size_t p2, const IColumn & rhs_, int /*nan_direction_hint*/) const override
 #else
     int doCompareAt(size_t p1, size_t p2, const IColumn & rhs_, int /*nan_direction_hint*/) const override
@@ -156,7 +156,7 @@ public:
 
     size_t estimateCardinalityInPermutedRange(const Permutation & permutation, const EqualRange & equal_range) const override;
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
 #else
     void doInsertRangeFrom(const IColumn & src, size_t start, size_t length) override;
diff --git a/src/Columns/ColumnFunction.cpp b/src/Columns/ColumnFunction.cpp
index fa57f35a823..fc81efaac0c 100644
--- a/src/Columns/ColumnFunction.cpp
+++ b/src/Columns/ColumnFunction.cpp
@@ -72,7 +72,7 @@ ColumnPtr ColumnFunction::cut(size_t start, size_t length) const
     return ColumnFunction::create(length, function, capture, is_short_circuit_argument, is_function_compiled);
 }
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
 void ColumnFunction::insertFrom(const IColumn & src, size_t n)
 #else
 void ColumnFunction::doInsertFrom(const IColumn & src, size_t n)
@@ -93,7 +93,7 @@ void ColumnFunction::doInsertFrom(const IColumn & src, size_t n)
     ++elements_size;
 }
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
 void ColumnFunction::insertRangeFrom(const IColumn & src, size_t start, size_t length)
 #else
 void ColumnFunction::doInsertRangeFrom(const IColumn & src, size_t start, size_t length)
diff --git a/src/Columns/ColumnFunction.h b/src/Columns/ColumnFunction.h
index ba924c49a82..dfc592ab281 100644
--- a/src/Columns/ColumnFunction.h
+++ b/src/Columns/ColumnFunction.h
@@ -94,12 +94,12 @@ public:
         throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot insert into {}", getName());
     }
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     void insertFrom(const IColumn & src, size_t n) override;
 #else
     void doInsertFrom(const IColumn & src, size_t n) override;
 #endif
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     void insertRangeFrom(const IColumn &, size_t start, size_t length) override;
 #else
     void doInsertRangeFrom(const IColumn &, size_t start, size_t length) override;
@@ -145,7 +145,7 @@ public:
         throw Exception(ErrorCodes::NOT_IMPLEMENTED, "popBack is not implemented for {}", getName());
     }
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     int compareAt(size_t, size_t, const IColumn &, int) const override
 #else
     int doCompareAt(size_t, size_t, const IColumn &, int) const override
diff --git a/src/Columns/ColumnLowCardinality.cpp b/src/Columns/ColumnLowCardinality.cpp
index eb694a10b0f..237e157f420 100644
--- a/src/Columns/ColumnLowCardinality.cpp
+++ b/src/Columns/ColumnLowCardinality.cpp
@@ -159,7 +159,7 @@ void ColumnLowCardinality::insertDefault()
     idx.insertPosition(getDictionary().getDefaultValueIndex());
 }
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
 void ColumnLowCardinality::insertFrom(const IColumn & src, size_t n)
 #else
 void ColumnLowCardinality::doInsertFrom(const IColumn & src, size_t n)
@@ -191,7 +191,7 @@ void ColumnLowCardinality::insertFromFullColumn(const IColumn & src, size_t n)
     idx.insertPosition(getDictionary().uniqueInsertFrom(src, n));
 }
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
 void ColumnLowCardinality::insertRangeFrom(const IColumn & src, size_t start, size_t length)
 #else
 void ColumnLowCardinality::doInsertRangeFrom(const IColumn & src, size_t start, size_t length)
@@ -372,7 +372,7 @@ int ColumnLowCardinality::compareAtImpl(size_t n, size_t m, const IColumn & rhs,
     return getDictionary().compareAt(n_index, m_index, low_cardinality_column.getDictionary(), nan_direction_hint);
 }
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
 int ColumnLowCardinality::compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const
 #else
 int ColumnLowCardinality::doCompareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const
diff --git a/src/Columns/ColumnLowCardinality.h b/src/Columns/ColumnLowCardinality.h
index e99be07cd8d..1b74518037f 100644
--- a/src/Columns/ColumnLowCardinality.h
+++ b/src/Columns/ColumnLowCardinality.h
@@ -78,14 +78,14 @@ public:
     bool tryInsert(const Field & x) override;
     void insertDefault() override;
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     void insertFrom(const IColumn & src, size_t n) override;
 #else
     void doInsertFrom(const IColumn & src, size_t n) override;
 #endif
     void insertFromFullColumn(const IColumn & src, size_t n);
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
 #else
     void doInsertRangeFrom(const IColumn & src, size_t start, size_t length) override;
@@ -135,7 +135,7 @@ public:
         return ColumnLowCardinality::create(dictionary.getColumnUniquePtr(), getIndexes().index(indexes_, limit));
     }
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     int compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const override;
 #else
     int doCompareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const override;
diff --git a/src/Columns/ColumnMap.cpp b/src/Columns/ColumnMap.cpp
index 2dffddb2dc9..77bf8802ab4 100644
--- a/src/Columns/ColumnMap.cpp
+++ b/src/Columns/ColumnMap.cpp
@@ -153,7 +153,7 @@ void ColumnMap::updateHashFast(SipHash & hash) const
     nested->updateHashFast(hash);
 }
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
 void ColumnMap::insertFrom(const IColumn & src, size_t n)
 #else
 void ColumnMap::doInsertFrom(const IColumn & src, size_t n)
@@ -162,7 +162,7 @@ void ColumnMap::doInsertFrom(const IColumn & src, size_t n)
     nested->insertFrom(assert_cast<const ColumnMap &>(src).getNestedColumn(), n);
 }
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
 void ColumnMap::insertManyFrom(const IColumn & src, size_t position, size_t length)
 #else
 void ColumnMap::doInsertManyFrom(const IColumn & src, size_t position, size_t length)
@@ -171,7 +171,7 @@ void ColumnMap::doInsertManyFrom(const IColumn & src, size_t position, size_t le
     assert_cast<ColumnArray &>(*nested).insertManyFrom(assert_cast<const ColumnMap &>(src).getNestedColumn(), position, length);
 }
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
 void ColumnMap::insertRangeFrom(const IColumn & src, size_t start, size_t length)
 #else
 void ColumnMap::doInsertRangeFrom(const IColumn & src, size_t start, size_t length)
@@ -222,7 +222,7 @@ MutableColumns ColumnMap::scatter(ColumnIndex num_columns, const Selector & sele
     return res;
 }
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
 int ColumnMap::compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const
 #else
 int ColumnMap::doCompareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const
diff --git a/src/Columns/ColumnMap.h b/src/Columns/ColumnMap.h
index a54071a2974..592f83732df 100644
--- a/src/Columns/ColumnMap.h
+++ b/src/Columns/ColumnMap.h
@@ -67,7 +67,7 @@ public:
     void updateWeakHash32(WeakHash32 & hash) const override;
     void updateHashFast(SipHash & hash) const override;
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     void insertFrom(const IColumn & src_, size_t n) override;
     void insertManyFrom(const IColumn & src, size_t position, size_t length) override;
     void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
@@ -83,7 +83,7 @@ public:
     ColumnPtr index(const IColumn & indexes, size_t limit) const override;
     ColumnPtr replicate(const Offsets & offsets) const override;
     MutableColumns scatter(ColumnIndex num_columns, const Selector & selector) const override;
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     int compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const override;
 #else
     int doCompareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const override;
diff --git a/src/Columns/ColumnNullable.cpp b/src/Columns/ColumnNullable.cpp
index f060e74b315..11447a7966e 100644
--- a/src/Columns/ColumnNullable.cpp
+++ b/src/Columns/ColumnNullable.cpp
@@ -221,7 +221,7 @@ const char * ColumnNullable::skipSerializedInArena(const char * pos) const
     return pos;
 }
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
 void ColumnNullable::insertRangeFrom(const IColumn & src, size_t start, size_t length)
 #else
 void ColumnNullable::doInsertRangeFrom(const IColumn & src, size_t start, size_t length)
@@ -262,7 +262,7 @@ bool ColumnNullable::tryInsert(const Field & x)
     return true;
 }
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
 void ColumnNullable::insertFrom(const IColumn & src, size_t n)
 #else
 void ColumnNullable::doInsertFrom(const IColumn & src, size_t n)
@@ -274,7 +274,7 @@ void ColumnNullable::doInsertFrom(const IColumn & src, size_t n)
 }
 
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
 void ColumnNullable::insertManyFrom(const IColumn & src, size_t position, size_t length)
 #else
 void ColumnNullable::doInsertManyFrom(const IColumn & src, size_t position, size_t length)
@@ -414,7 +414,7 @@ int ColumnNullable::compareAtImpl(size_t n, size_t m, const IColumn & rhs_, int
     return getNestedColumn().compareAt(n, m, nested_rhs, null_direction_hint);
 }
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
 int ColumnNullable::compareAt(size_t n, size_t m, const IColumn & rhs_, int null_direction_hint) const
 #else
 int ColumnNullable::doCompareAt(size_t n, size_t m, const IColumn & rhs_, int null_direction_hint) const
diff --git a/src/Columns/ColumnNullable.h b/src/Columns/ColumnNullable.h
index a6d0483e527..cf7efba6235 100644
--- a/src/Columns/ColumnNullable.h
+++ b/src/Columns/ColumnNullable.h
@@ -69,7 +69,7 @@ public:
     char * serializeValueIntoMemory(size_t n, char * memory) const override;
     const char * deserializeAndInsertFromArena(const char * pos) override;
     const char * skipSerializedInArena(const char * pos) const override;
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
 #else
     void doInsertRangeFrom(const IColumn & src, size_t start, size_t length) override;
@@ -77,7 +77,7 @@ public:
     void insert(const Field & x) override;
     bool tryInsert(const Field & x) override;
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     void insertFrom(const IColumn & src, size_t n) override;
     void insertManyFrom(const IColumn & src, size_t position, size_t length) override;
 #else
@@ -100,7 +100,7 @@ public:
     void expand(const Filter & mask, bool inverted) override;
     ColumnPtr permute(const Permutation & perm, size_t limit) const override;
     ColumnPtr index(const IColumn & indexes, size_t limit) const override;
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     int compareAt(size_t n, size_t m, const IColumn & rhs_, int null_direction_hint) const override;
 #else
     int doCompareAt(size_t n, size_t m, const IColumn & rhs_, int null_direction_hint) const override;
diff --git a/src/Columns/ColumnObject.cpp b/src/Columns/ColumnObject.cpp
index adcd42b16e9..39e587368fe 100644
--- a/src/Columns/ColumnObject.cpp
+++ b/src/Columns/ColumnObject.cpp
@@ -763,7 +763,7 @@ void ColumnObject::get(size_t n, Field & res) const
     }
 }
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
 void ColumnObject::insertFrom(const IColumn & src, size_t n)
 #else
 void ColumnObject::doInsertFrom(const IColumn & src, size_t n)
@@ -772,7 +772,7 @@ void ColumnObject::doInsertFrom(const IColumn & src, size_t n)
     insert(src[n]);
 }
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
 void ColumnObject::insertRangeFrom(const IColumn & src, size_t start, size_t length)
 #else
 void ColumnObject::doInsertRangeFrom(const IColumn & src, size_t start, size_t length)
diff --git a/src/Columns/ColumnObject.h b/src/Columns/ColumnObject.h
index fadf2e18779..09c28d460e4 100644
--- a/src/Columns/ColumnObject.h
+++ b/src/Columns/ColumnObject.h
@@ -210,7 +210,7 @@ public:
     bool tryInsert(const Field & field) override;
     void insertDefault() override;
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     void insertFrom(const IColumn & src, size_t n) override;
     void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
 #else
@@ -235,7 +235,7 @@ public:
     /// Order of rows in ColumnObject is undefined.
     void getPermutation(PermutationSortDirection, PermutationSortStability, size_t, int, Permutation & res) const override;
     void updatePermutation(PermutationSortDirection, PermutationSortStability, size_t, int, Permutation &, EqualRanges &) const override {}
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     int compareAt(size_t, size_t, const IColumn &, int) const override { return 0; }
 #else
     int doCompareAt(size_t, size_t, const IColumn &, int) const override { return 0; }
diff --git a/src/Columns/ColumnSparse.cpp b/src/Columns/ColumnSparse.cpp
index 809586d8810..49c953c5aab 100644
--- a/src/Columns/ColumnSparse.cpp
+++ b/src/Columns/ColumnSparse.cpp
@@ -174,7 +174,7 @@ const char * ColumnSparse::skipSerializedInArena(const char * pos) const
     return values->skipSerializedInArena(pos);
 }
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
 void ColumnSparse::insertRangeFrom(const IColumn & src, size_t start, size_t length)
 #else
 void ColumnSparse::doInsertRangeFrom(const IColumn & src, size_t start, size_t length)
@@ -252,7 +252,7 @@ bool ColumnSparse::tryInsert(const Field & x)
     return true;
 }
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
 void ColumnSparse::insertFrom(const IColumn & src, size_t n)
 #else
 void ColumnSparse::doInsertFrom(const IColumn & src, size_t n)
@@ -454,7 +454,7 @@ ColumnPtr ColumnSparse::indexImpl(const PaddedPODArray<Type> & indexes, size_t l
     return ColumnSparse::create(std::move(res_values), std::move(res_offsets), limit);
 }
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
 int ColumnSparse::compareAt(size_t n, size_t m, const IColumn & rhs_, int null_direction_hint) const
 #else
 int ColumnSparse::doCompareAt(size_t n, size_t m, const IColumn & rhs_, int null_direction_hint) const
diff --git a/src/Columns/ColumnSparse.h b/src/Columns/ColumnSparse.h
index 3e34d1de94a..732fad292af 100644
--- a/src/Columns/ColumnSparse.h
+++ b/src/Columns/ColumnSparse.h
@@ -81,14 +81,14 @@ public:
     char * serializeValueIntoMemory(size_t n, char * memory) const override;
     const char * deserializeAndInsertFromArena(const char * pos) override;
     const char * skipSerializedInArena(const char *) const override;
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
 #else
     void doInsertRangeFrom(const IColumn & src, size_t start, size_t length) override;
 #endif
     void insert(const Field & x) override;
     bool tryInsert(const Field & x) override;
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     void insertFrom(const IColumn & src, size_t n) override;
 #else
     void doInsertFrom(const IColumn & src, size_t n) override;
@@ -106,7 +106,7 @@ public:
     template <typename Type>
     ColumnPtr indexImpl(const PaddedPODArray<Type> & indexes, size_t limit) const;
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     int compareAt(size_t n, size_t m, const IColumn & rhs_, int null_direction_hint) const override;
 #else
     int doCompareAt(size_t n, size_t m, const IColumn & rhs_, int null_direction_hint) const override;
diff --git a/src/Columns/ColumnString.cpp b/src/Columns/ColumnString.cpp
index 1eda9714d62..37a0d6b31a2 100644
--- a/src/Columns/ColumnString.cpp
+++ b/src/Columns/ColumnString.cpp
@@ -39,7 +39,7 @@ ColumnString::ColumnString(const ColumnString & src)
             last_offset, chars.size());
 }
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
 void ColumnString::insertManyFrom(const IColumn & src, size_t position, size_t length)
 #else
 void ColumnString::doInsertManyFrom(const IColumn & src, size_t position, size_t length)
@@ -133,7 +133,7 @@ void ColumnString::updateWeakHash32(WeakHash32 & hash) const
 }
 
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
 void ColumnString::insertRangeFrom(const IColumn & src, size_t start, size_t length)
 #else
 void ColumnString::doInsertRangeFrom(const IColumn & src, size_t start, size_t length)
diff --git a/src/Columns/ColumnString.h b/src/Columns/ColumnString.h
index 602ffac65e8..157ca9fc9cd 100644
--- a/src/Columns/ColumnString.h
+++ b/src/Columns/ColumnString.h
@@ -142,7 +142,7 @@ public:
         return true;
     }
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     void insertFrom(const IColumn & src_, size_t n) override
 #else
     void doInsertFrom(const IColumn & src_, size_t n) override
@@ -169,7 +169,7 @@ public:
         }
     }
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     void insertManyFrom(const IColumn & src, size_t position, size_t length) override;
 #else
     void doInsertManyFrom(const IColumn & src, size_t position, size_t length) override;
@@ -220,7 +220,7 @@ public:
         hash.update(reinterpret_cast<const char *>(chars.data()), chars.size() * sizeof(chars[0]));
     }
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
 #else
     void doInsertRangeFrom(const IColumn & src, size_t start, size_t length) override;
@@ -250,7 +250,7 @@ public:
             offsets.push_back(offsets.back() + 1);
     }
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     int compareAt(size_t n, size_t m, const IColumn & rhs_, int /*nan_direction_hint*/) const override
 #else
     int doCompareAt(size_t n, size_t m, const IColumn & rhs_, int /*nan_direction_hint*/) const override
diff --git a/src/Columns/ColumnTuple.cpp b/src/Columns/ColumnTuple.cpp
index 9b822d7f570..50e698f57c2 100644
--- a/src/Columns/ColumnTuple.cpp
+++ b/src/Columns/ColumnTuple.cpp
@@ -205,7 +205,7 @@ bool ColumnTuple::tryInsert(const Field & x)
     return true;
 }
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
 void ColumnTuple::insertFrom(const IColumn & src_, size_t n)
 #else
 void ColumnTuple::doInsertFrom(const IColumn & src_, size_t n)
@@ -222,7 +222,7 @@ void ColumnTuple::doInsertFrom(const IColumn & src_, size_t n)
         columns[i]->insertFrom(*src.columns[i], n);
 }
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
 void ColumnTuple::insertManyFrom(const IColumn & src, size_t position, size_t length)
 #else
 void ColumnTuple::doInsertManyFrom(const IColumn & src, size_t position, size_t length)
@@ -326,7 +326,7 @@ void ColumnTuple::updateHashFast(SipHash & hash) const
         column->updateHashFast(hash);
 }
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
 void ColumnTuple::insertRangeFrom(const IColumn & src, size_t start, size_t length)
 #else
 void ColumnTuple::doInsertRangeFrom(const IColumn & src, size_t start, size_t length)
@@ -482,7 +482,7 @@ int ColumnTuple::compareAtImpl(size_t n, size_t m, const IColumn & rhs, int nan_
     return 0;
 }
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
 int ColumnTuple::compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const
 #else
 int ColumnTuple::doCompareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const
diff --git a/src/Columns/ColumnTuple.h b/src/Columns/ColumnTuple.h
index 38e479791d4..71a47e58401 100644
--- a/src/Columns/ColumnTuple.h
+++ b/src/Columns/ColumnTuple.h
@@ -66,7 +66,7 @@ public:
     void insert(const Field & x) override;
     bool tryInsert(const Field & x) override;
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     void insertFrom(const IColumn & src_, size_t n) override;
     void insertManyFrom(const IColumn & src, size_t position, size_t length) override;
 #else
@@ -83,7 +83,7 @@ public:
     void updateHashWithValue(size_t n, SipHash & hash) const override;
     void updateWeakHash32(WeakHash32 & hash) const override;
     void updateHashFast(SipHash & hash) const override;
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
 #else
     void doInsertRangeFrom(const IColumn & src, size_t start, size_t length) override;
@@ -94,7 +94,7 @@ public:
     ColumnPtr index(const IColumn & indexes, size_t limit) const override;
     ColumnPtr replicate(const Offsets & offsets) const override;
     MutableColumns scatter(ColumnIndex num_columns, const Selector & selector) const override;
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     int compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const override;
 #else
     int doCompareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const override;
diff --git a/src/Columns/ColumnUnique.h b/src/Columns/ColumnUnique.h
index ec1f8e0a4d5..d6cb75679be 100644
--- a/src/Columns/ColumnUnique.h
+++ b/src/Columns/ColumnUnique.h
@@ -90,7 +90,7 @@ public:
         return getNestedColumn()->updateHashWithValue(n, hash_func);
     }
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     int compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const override;
 #else
     int doCompareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const override;
@@ -492,7 +492,7 @@ const char * ColumnUnique<ColumnType>::skipSerializedInArena(const char *) const
 }
 
 template <typename ColumnType>
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
 int ColumnUnique<ColumnType>::compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const
 #else
 int ColumnUnique<ColumnType>::doCompareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const
diff --git a/src/Columns/ColumnVariant.cpp b/src/Columns/ColumnVariant.cpp
index ee5de4c2dde..18e9cffd61a 100644
--- a/src/Columns/ColumnVariant.cpp
+++ b/src/Columns/ColumnVariant.cpp
@@ -595,7 +595,7 @@ void ColumnVariant::insertManyFromImpl(const DB::IColumn & src_, size_t position
     }
 }
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
 void ColumnVariant::insertFrom(const IColumn & src_, size_t n)
 #else
 void ColumnVariant::doInsertFrom(const IColumn & src_, size_t n)
@@ -604,7 +604,7 @@ void ColumnVariant::doInsertFrom(const IColumn & src_, size_t n)
     insertFromImpl(src_, n, nullptr);
 }
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
 void ColumnVariant::insertRangeFrom(const IColumn & src_, size_t start, size_t length)
 #else
 void ColumnVariant::doInsertRangeFrom(const IColumn & src_, size_t start, size_t length)
@@ -613,7 +613,7 @@ void ColumnVariant::doInsertRangeFrom(const IColumn & src_, size_t start, size_t
     insertRangeFromImpl(src_, start, length, nullptr);
 }
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
 void ColumnVariant::insertManyFrom(const DB::IColumn & src_, size_t position, size_t length)
 #else
 void ColumnVariant::doInsertManyFrom(const DB::IColumn & src_, size_t position, size_t length)
@@ -1186,7 +1186,7 @@ bool ColumnVariant::hasEqualValues() const
     return local_discriminators->hasEqualValues() && variants[localDiscriminatorAt(0)]->hasEqualValues();
 }
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
 int ColumnVariant::compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const
 #else
 int ColumnVariant::doCompareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const
diff --git a/src/Columns/ColumnVariant.h b/src/Columns/ColumnVariant.h
index d91b8e93a7d..2dbe1494823 100644
--- a/src/Columns/ColumnVariant.h
+++ b/src/Columns/ColumnVariant.h
@@ -180,7 +180,7 @@ public:
     void insert(const Field & x) override;
     bool tryInsert(const Field & x) override;
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     void insertFrom(const IColumn & src_, size_t n) override;
     void insertRangeFrom(const IColumn & src_, size_t start, size_t length) override;
     void insertManyFrom(const IColumn & src_, size_t position, size_t length) override;
@@ -223,7 +223,7 @@ public:
     ColumnPtr indexImpl(const PaddedPODArray<Type> & indexes, size_t limit) const;
     ColumnPtr replicate(const Offsets & replicate_offsets) const override;
     MutableColumns scatter(ColumnIndex num_columns, const Selector & selector) const override;
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     int compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const override;
 #else
     int doCompareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const override;
diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp
index 19d1b800961..d5b7786a702 100644
--- a/src/Columns/ColumnVector.cpp
+++ b/src/Columns/ColumnVector.cpp
@@ -503,7 +503,7 @@ bool ColumnVector<T>::tryInsert(const DB::Field & x)
 }
 
 template <typename T>
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
 void ColumnVector<T>::insertRangeFrom(const IColumn & src, size_t start, size_t length)
 #else
 void ColumnVector<T>::doInsertRangeFrom(const IColumn & src, size_t start, size_t length)
diff --git a/src/Columns/ColumnVector.h b/src/Columns/ColumnVector.h
index 3a0acf5e312..57c108fff23 100644
--- a/src/Columns/ColumnVector.h
+++ b/src/Columns/ColumnVector.h
@@ -64,7 +64,7 @@ public:
         return data.size();
     }
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     void insertFrom(const IColumn & src, size_t n) override
 #else
     void doInsertFrom(const IColumn & src, size_t n) override
@@ -73,7 +73,7 @@ public:
         data.push_back(assert_cast<const Self &>(src).getData()[n]);
     }
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     void insertManyFrom(const IColumn & src, size_t position, size_t length) override
 #else
     void doInsertManyFrom(const IColumn & src, size_t position, size_t length) override
@@ -150,7 +150,7 @@ public:
     }
 
     /// This method implemented in header because it could be possibly devirtualized.
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     int compareAt(size_t n, size_t m, const IColumn & rhs_, int nan_direction_hint) const override
 #else
     int doCompareAt(size_t n, size_t m, const IColumn & rhs_, int nan_direction_hint) const override
@@ -240,7 +240,7 @@ public:
 
     bool tryInsert(const DB::Field & x) override;
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     void insertRangeFrom(const IColumn & src, size_t start, size_t length) override;
 #else
     void doInsertRangeFrom(const IColumn & src, size_t start, size_t length) override;
diff --git a/src/Columns/IColumn.cpp b/src/Columns/IColumn.cpp
index 552e52cf51c..a189903b617 100644
--- a/src/Columns/IColumn.cpp
+++ b/src/Columns/IColumn.cpp
@@ -46,7 +46,7 @@ String IColumn::dumpStructure() const
     return res.str();
 }
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
 void IColumn::insertFrom(const IColumn & src, size_t n)
 #else
 void IColumn::doInsertFrom(const IColumn & src, size_t n)
diff --git a/src/Columns/IColumn.h b/src/Columns/IColumn.h
index 4b6f34e5aa2..005cb314201 100644
--- a/src/Columns/IColumn.h
+++ b/src/Columns/IColumn.h
@@ -179,7 +179,7 @@ public:
 
     /// Appends n-th element from other column with the same type.
     /// Is used in merge-sort and merges. It could be implemented in inherited classes more optimally than default implementation.
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     virtual void insertFrom(const IColumn & src, size_t n);
 #else
     void insertFrom(const IColumn & src, size_t n)
@@ -191,7 +191,7 @@ public:
 
     /// Appends range of elements from other column with the same type.
     /// Could be used to concatenate columns.
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     virtual void insertRangeFrom(const IColumn & src, size_t start, size_t length) = 0;
 #else
     void insertRangeFrom(const IColumn & src, size_t start, size_t length)
@@ -202,7 +202,7 @@ public:
 #endif
 
     /// Appends one element from other column with the same type multiple times.
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     virtual void insertManyFrom(const IColumn & src, size_t position, size_t length)
     {
         for (size_t i = 0; i < length; ++i)
@@ -345,7 +345,7 @@ public:
       *
       * For non Nullable and non floating point types, nan_direction_hint is ignored.
       */
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     [[nodiscard]] virtual int compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const = 0;
 #else
     [[nodiscard]] int compareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const
@@ -667,7 +667,7 @@ protected:
         Sort full_sort,
         PartialSort partial_sort) const;
 
-#if defined(ABORT_ON_LOGICAL_ERROR)
+#if defined(DEBUG_OR_SANITIZER_BUILD)
     virtual void doInsertFrom(const IColumn & src, size_t n);
 
     virtual void doInsertRangeFrom(const IColumn & src, size_t start, size_t length) = 0;
diff --git a/src/Columns/IColumnDummy.h b/src/Columns/IColumnDummy.h
index c19fb704d9b..db21ec05aae 100644
--- a/src/Columns/IColumnDummy.h
+++ b/src/Columns/IColumnDummy.h
@@ -26,7 +26,7 @@ public:
     size_t byteSize() const override { return 0; }
     size_t byteSizeAt(size_t) const override { return 0; }
     size_t allocatedBytes() const override { return 0; }
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     int compareAt(size_t, size_t, const IColumn &, int) const override { return 0; }
 #else
     int doCompareAt(size_t, size_t, const IColumn &, int) const override { return 0; }
@@ -71,7 +71,7 @@ public:
     {
     }
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     void insertFrom(const IColumn &, size_t) override
 #else
     void doInsertFrom(const IColumn &, size_t) override
@@ -80,7 +80,7 @@ public:
         ++s;
     }
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     void insertRangeFrom(const IColumn & /*src*/, size_t /*start*/, size_t length) override
 #else
     void doInsertRangeFrom(const IColumn & /*src*/, size_t /*start*/, size_t length) override
diff --git a/src/Columns/IColumnUnique.h b/src/Columns/IColumnUnique.h
index 3398452b7ee..1faabe55772 100644
--- a/src/Columns/IColumnUnique.h
+++ b/src/Columns/IColumnUnique.h
@@ -85,7 +85,7 @@ public:
         throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method tryInsert is not supported for ColumnUnique.");
     }
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
     void insertRangeFrom(const IColumn &, size_t, size_t) override
 #else
     void doInsertRangeFrom(const IColumn &, size_t, size_t) override
diff --git a/src/Columns/benchmarks/benchmark_column_insert_many_from.cpp b/src/Columns/benchmarks/benchmark_column_insert_many_from.cpp
index 645f6ed79f3..240099f0ae5 100644
--- a/src/Columns/benchmarks/benchmark_column_insert_many_from.cpp
+++ b/src/Columns/benchmarks/benchmark_column_insert_many_from.cpp
@@ -52,7 +52,7 @@ static ColumnPtr mockColumn(const DataTypePtr & type, size_t rows)
 }
 
 
-#if !defined(ABORT_ON_LOGICAL_ERROR)
+#if !defined(DEBUG_OR_SANITIZER_BUILD)
 static NO_INLINE void insertManyFrom(IColumn & dst, const IColumn & src)
 #else
 static NO_INLINE void doInsertManyFrom(IColumn & dst, const IColumn & src)
diff --git a/src/Common/Config/AbstractConfigurationComparison.cpp b/src/Common/Config/AbstractConfigurationComparison.cpp
index 607b583cf31..80c837ed43b 100644
--- a/src/Common/Config/AbstractConfigurationComparison.cpp
+++ b/src/Common/Config/AbstractConfigurationComparison.cpp
@@ -38,7 +38,7 @@ namespace
             std::erase_if(left_subkeys, [&](const String & key) { return ignore_keys->contains(key); });
             std::erase_if(right_subkeys, [&](const String & key) { return ignore_keys->contains(key); });
 
-#if defined(ABORT_ON_LOGICAL_ERROR)
+#if defined(DEBUG_OR_SANITIZER_BUILD)
             /// Compound `ignore_keys` are not yet implemented.
             for (const auto & ignore_key : *ignore_keys)
                 chassert(ignore_key.find('.') == std::string_view::npos);
diff --git a/src/Common/MemoryTracker.cpp b/src/Common/MemoryTracker.cpp
index 28cfa98666a..ac412684cf7 100644
--- a/src/Common/MemoryTracker.cpp
+++ b/src/Common/MemoryTracker.cpp
@@ -192,7 +192,7 @@ void MemoryTracker::debugLogBigAllocationWithoutCheck(Int64 size [[maybe_unused]
 {
     /// Big allocations through allocNoThrow (without checking memory limits) may easily lead to OOM (and it's hard to debug).
     /// Let's find them.
-#ifdef ABORT_ON_LOGICAL_ERROR
+#ifdef DEBUG_OR_SANITIZER_BUILD
     if (size < 0)
         return;
 
diff --git a/src/Common/PageCache.cpp b/src/Common/PageCache.cpp
index 56bd8c1a339..d719a387e14 100644
--- a/src/Common/PageCache.cpp
+++ b/src/Common/PageCache.cpp
@@ -424,7 +424,7 @@ static void logUnexpectedSyscallError(std::string name)
 {
     std::string message = fmt::format("{} failed: {}", name, errnoToString());
     LOG_WARNING(&Poco::Logger::get("PageCache"), "{}", message);
-#if defined(ABORT_ON_LOGICAL_ERROR)
+#if defined(DEBUG_OR_SANITIZER_BUILD)
     volatile bool true_ = true;
     if (true_) // suppress warning about missing [[noreturn]]
         abortOnFailedAssertion(message);
diff --git a/src/Common/assert_cast.h b/src/Common/assert_cast.h
index f9d0bf0e595..7a04372ffad 100644
--- a/src/Common/assert_cast.h
+++ b/src/Common/assert_cast.h
@@ -25,7 +25,7 @@ namespace DB
 template <typename To, typename From>
 inline To assert_cast(From && from)
 {
-#ifdef ABORT_ON_LOGICAL_ERROR
+#ifdef DEBUG_OR_SANITIZER_BUILD
     try
     {
         if constexpr (std::is_pointer_v<To>)
diff --git a/src/Common/tests/gtest_rw_lock.cpp b/src/Common/tests/gtest_rw_lock.cpp
index d8c6e9cb99d..9b0c9aeafbe 100644
--- a/src/Common/tests/gtest_rw_lock.cpp
+++ b/src/Common/tests/gtest_rw_lock.cpp
@@ -166,7 +166,7 @@ TEST(Common, RWLockRecursive)
 
             auto lock2 = fifo_lock->getLock(RWLockImpl::Read, "q2");
 
-#ifndef ABORT_ON_LOGICAL_ERROR
+#ifndef DEBUG_OR_SANITIZER_BUILD
             /// It throws LOGICAL_ERROR
             EXPECT_ANY_THROW({fifo_lock->getLock(RWLockImpl::Write, "q2");});
 #endif
diff --git a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp
index e9c642666d3..198f6c0ea04 100644
--- a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp
+++ b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp
@@ -59,7 +59,7 @@ CachedOnDiskReadBufferFromFile::CachedOnDiskReadBufferFromFile(
     std::optional<size_t> read_until_position_,
     std::shared_ptr<FilesystemCacheLog> cache_log_)
     : ReadBufferFromFileBase(use_external_buffer_ ? 0 : settings_.remote_fs_buffer_size, nullptr, 0, file_size_)
-#ifdef ABORT_ON_LOGICAL_ERROR
+#ifdef DEBUG_OR_SANITIZER_BUILD
     , log(getLogger(fmt::format("CachedOnDiskReadBufferFromFile({})", cache_key_)))
 #else
     , log(getLogger("CachedOnDiskReadBufferFromFile"))
@@ -452,7 +452,7 @@ CachedOnDiskReadBufferFromFile::getImplementationBuffer(FileSegment & file_segme
     {
         case ReadType::CACHED:
         {
-#ifdef ABORT_ON_LOGICAL_ERROR
+#ifdef DEBUG_OR_SANITIZER_BUILD
             size_t file_size = getFileSizeFromReadBuffer(*read_buffer_for_file_segment);
             if (file_size == 0 || range.left + file_size <= file_offset_of_buffer_end)
                 throw Exception(
@@ -937,7 +937,7 @@ bool CachedOnDiskReadBufferFromFile::nextImplStep()
 
     if (!result)
     {
-#ifdef ABORT_ON_LOGICAL_ERROR
+#ifdef DEBUG_OR_SANITIZER_BUILD
         if (read_type == ReadType::CACHED)
         {
             size_t cache_file_size = getFileSizeFromReadBuffer(*implementation_buffer);
diff --git a/src/IO/tests/gtest_memory_resize.cpp b/src/IO/tests/gtest_memory_resize.cpp
index d760a948075..c3b34c352b2 100644
--- a/src/IO/tests/gtest_memory_resize.cpp
+++ b/src/IO/tests/gtest_memory_resize.cpp
@@ -134,7 +134,7 @@ TEST(MemoryResizeTest, SmallInitAndBigResizeOverflowWhenPadding)
         ASSERT_EQ(memory.m_capacity, 0x8000000000000000ULL - 1);
         ASSERT_EQ(memory.m_size, 0x8000000000000000ULL - PADDING_FOR_SIMD);
 
-#ifndef ABORT_ON_LOGICAL_ERROR
+#ifndef DEBUG_OR_SANITIZER_BUILD
         EXPECT_THROW_ERROR_CODE(memory.resize(0x8000000000000000ULL - (PADDING_FOR_SIMD - 1)), Exception, ErrorCodes::LOGICAL_ERROR);
         ASSERT_TRUE(memory.m_data);  // state is intact after exception
         ASSERT_EQ(memory.m_capacity, 0x8000000000000000ULL - 1);
@@ -158,7 +158,7 @@ TEST(MemoryResizeTest, SmallInitAndBigResizeOverflowWhenPadding)
         ASSERT_EQ(memory.m_capacity, PADDING_FOR_SIMD);
         ASSERT_EQ(memory.m_size, 1);
 
-#ifndef ABORT_ON_LOGICAL_ERROR
+#ifndef DEBUG_OR_SANITIZER_BUILD
         EXPECT_THROW_ERROR_CODE(memory.resize(0x8000000000000000ULL - (PADDING_FOR_SIMD - 1)), Exception, ErrorCodes::LOGICAL_ERROR);
         ASSERT_TRUE(memory.m_data); // state is intact after exception
         ASSERT_EQ(memory.m_capacity, PADDING_FOR_SIMD);
@@ -197,7 +197,7 @@ TEST(MemoryResizeTest, BigInitAndSmallResizeOverflowWhenPadding)
         , ErrorCodes::ARGUMENT_OUT_OF_BOUND);
     }
 
-#ifndef ABORT_ON_LOGICAL_ERROR
+#ifndef DEBUG_OR_SANITIZER_BUILD
     {
         EXPECT_THROW_ERROR_CODE(
         {
diff --git a/src/IO/tests/gtest_writebuffer_s3.cpp b/src/IO/tests/gtest_writebuffer_s3.cpp
index 3c1af6538ad..b53a8b58023 100644
--- a/src/IO/tests/gtest_writebuffer_s3.cpp
+++ b/src/IO/tests/gtest_writebuffer_s3.cpp
@@ -917,8 +917,8 @@ TEST_P(SyncAsync, ExceptionOnUploadPart) {
 
 
 TEST_F(WBS3Test, PrefinalizeCalledMultipleTimes) {
-#ifdef ABORT_ON_LOGICAL_ERROR
-    GTEST_SKIP() << "this test trigger LOGICAL_ERROR, runs only if ABORT_ON_LOGICAL_ERROR is not defined";
+#ifdef DEBUG_OR_SANITIZER_BUILD
+    GTEST_SKIP() << "this test trigger LOGICAL_ERROR, runs only if DEBUG_OR_SANITIZER_BUILD is not defined";
 #else
     EXPECT_THROW({
         try {
diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp
index a3848fa3a75..a88c0de2cfe 100644
--- a/src/Interpreters/Cache/FileCache.cpp
+++ b/src/Interpreters/Cache/FileCache.cpp
@@ -1007,7 +1007,7 @@ void FileCache::freeSpaceRatioKeepingThreadFunc()
         limits_satisfied = main_priority->collectCandidatesForEviction(
             desired_size, desired_elements_num, keep_up_free_space_remove_batch, stat, eviction_candidates, lock);
 
-#ifdef ABORT_ON_LOGICAL_ERROR
+#ifdef DEBUG_OR_SANITIZER_BUILD
         /// Let's make sure that we correctly processed the limits.
         if (limits_satisfied && eviction_candidates.size() < keep_up_free_space_remove_batch)
         {
@@ -1110,7 +1110,7 @@ void FileCache::removeAllReleasable(const UserID & user_id)
 {
     assertInitialized();
 
-#ifdef ABORT_ON_LOGICAL_ERROR
+#ifdef DEBUG_OR_SANITIZER_BUILD
     assertCacheCorrectness();
 #endif
 
@@ -1226,7 +1226,7 @@ void FileCache::loadMetadataImpl()
     if (first_exception)
         std::rethrow_exception(first_exception);
 
-#ifdef ABORT_ON_LOGICAL_ERROR
+#ifdef DEBUG_OR_SANITIZER_BUILD
     assertCacheCorrectness();
 #endif
 }
@@ -1393,7 +1393,7 @@ void FileCache::loadMetadataForKeys(const fs::path & keys_dir)
 FileCache::~FileCache()
 {
     deactivateBackgroundOperations();
-#ifdef ABORT_ON_LOGICAL_ERROR
+#ifdef DEBUG_OR_SANITIZER_BUILD
     assertCacheCorrectness();
 #endif
 }
diff --git a/src/Interpreters/Cache/FileSegment.cpp b/src/Interpreters/Cache/FileSegment.cpp
index 838ca0b491e..1664a91b694 100644
--- a/src/Interpreters/Cache/FileSegment.cpp
+++ b/src/Interpreters/Cache/FileSegment.cpp
@@ -67,7 +67,7 @@ FileSegment::FileSegment(
     , key_metadata(key_metadata_)
     , queue_iterator(queue_iterator_)
     , cache(cache_)
-#ifdef ABORT_ON_LOGICAL_ERROR
+#ifdef DEBUG_OR_SANITIZER_BUILD
     , log(getLogger(fmt::format("FileSegment({}) : {}", key_.toString(), range().toString())))
 #else
     , log(getLogger("FileSegment"))
@@ -385,9 +385,9 @@ void FileSegment::write(char * from, size_t size, size_t offset_in_file)
 
     try
     {
-#ifdef ABORT_ON_LOGICAL_ERROR
+#ifdef DEBUG_OR_SANITIZER_BUILD
         /// This mutex is only needed to have a valid assertion in assertCacheCorrectness(),
-        /// which is only executed in debug/sanitizer builds (under ABORT_ON_LOGICAL_ERROR).
+        /// which is only executed in debug/sanitizer builds (under DEBUG_OR_SANITIZER_BUILD).
         std::lock_guard lock(write_mutex);
 #endif
 
diff --git a/src/Interpreters/Cache/Metadata.cpp b/src/Interpreters/Cache/Metadata.cpp
index 1d23278a255..7e4b76d3cc6 100644
--- a/src/Interpreters/Cache/Metadata.cpp
+++ b/src/Interpreters/Cache/Metadata.cpp
@@ -963,7 +963,7 @@ KeyMetadata::iterator LockedKey::removeFileSegmentImpl(
         }
         else if (!can_be_broken)
         {
-#ifdef ABORT_ON_LOGICAL_ERROR
+#ifdef DEBUG_OR_SANITIZER_BUILD
             throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected path {} to exist", path);
 #else
             LOG_WARNING(key_metadata->logger(), "Expected path {} to exist, while removing {}:{}",
diff --git a/src/Interpreters/executeDDLQueryOnCluster.cpp b/src/Interpreters/executeDDLQueryOnCluster.cpp
index 5d237d28089..1b57ad2b622 100644
--- a/src/Interpreters/executeDDLQueryOnCluster.cpp
+++ b/src/Interpreters/executeDDLQueryOnCluster.cpp
@@ -538,7 +538,7 @@ Chunk DDLQueryStatusSource::generate()
             ExecutionStatus status(-1, "Cannot obtain error message");
 
             /// Replicated database retries in case of error, it should not write error status.
-#ifdef ABORT_ON_LOGICAL_ERROR
+#ifdef DEBUG_OR_SANITIZER_BUILD
             bool need_check_status = true;
 #else
             bool need_check_status = !is_replicated_database;
diff --git a/src/Processors/tests/gtest_exception_on_incorrect_pipeline.cpp b/src/Processors/tests/gtest_exception_on_incorrect_pipeline.cpp
index ce5992c2548..364d7c69071 100644
--- a/src/Processors/tests/gtest_exception_on_incorrect_pipeline.cpp
+++ b/src/Processors/tests/gtest_exception_on_incorrect_pipeline.cpp
@@ -50,7 +50,7 @@ TEST(Processors, PortsNotConnected)
     processors->emplace_back(std::move(source));
     processors->emplace_back(std::move(sink));
 
-#ifndef ABORT_ON_LOGICAL_ERROR
+#ifndef DEBUG_OR_SANITIZER_BUILD
     try
     {
         QueryStatusPtr element;
diff --git a/src/QueryPipeline/tests/gtest_check_sorted_stream.cpp b/src/QueryPipeline/tests/gtest_check_sorted_stream.cpp
index c8ab2e3a973..34bc2eb2b5e 100644
--- a/src/QueryPipeline/tests/gtest_check_sorted_stream.cpp
+++ b/src/QueryPipeline/tests/gtest_check_sorted_stream.cpp
@@ -133,7 +133,7 @@ TEST(CheckSortedTransform, CheckBadLastRow)
     EXPECT_NO_THROW(executor.pull(chunk));
     EXPECT_NO_THROW(executor.pull(chunk));
 
-#ifndef ABORT_ON_LOGICAL_ERROR
+#ifndef DEBUG_OR_SANITIZER_BUILD
     EXPECT_THROW(executor.pull(chunk), DB::Exception);
 #endif
 }
@@ -158,7 +158,7 @@ TEST(CheckSortedTransform, CheckUnsortedBlock1)
 
     Chunk chunk;
 
-#ifndef ABORT_ON_LOGICAL_ERROR
+#ifndef DEBUG_OR_SANITIZER_BUILD
     EXPECT_THROW(executor.pull(chunk), DB::Exception);
 #endif
 }
@@ -181,7 +181,7 @@ TEST(CheckSortedTransform, CheckUnsortedBlock2)
     PullingPipelineExecutor executor(pipeline);
 
     Chunk chunk;
-#ifndef ABORT_ON_LOGICAL_ERROR
+#ifndef DEBUG_OR_SANITIZER_BUILD
     EXPECT_THROW(executor.pull(chunk), DB::Exception);
 #endif
 }
@@ -204,7 +204,7 @@ TEST(CheckSortedTransform, CheckUnsortedBlock3)
     PullingPipelineExecutor executor(pipeline);
 
     Chunk chunk;
-#ifndef ABORT_ON_LOGICAL_ERROR
+#ifndef DEBUG_OR_SANITIZER_BUILD
     EXPECT_THROW(executor.pull(chunk), DB::Exception);
 #endif
 }
diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp
index 8d69df8de76..1306a3869c7 100644
--- a/src/Server/TCPHandler.cpp
+++ b/src/Server/TCPHandler.cpp
@@ -666,7 +666,7 @@ void TCPHandler::runImpl()
 // Server should die on std logic errors in debug, like with assert()
 // or ErrorCodes::LOGICAL_ERROR. This helps catch these errors in
 // tests.
-#ifdef ABORT_ON_LOGICAL_ERROR
+#ifdef DEBUG_OR_SANITIZER_BUILD
         catch (const std::logic_error & e)
         {
             state.io.onException();
diff --git a/src/Storages/MaterializedView/RefreshTask.cpp b/src/Storages/MaterializedView/RefreshTask.cpp
index 29631b95542..41e90aafd42 100644
--- a/src/Storages/MaterializedView/RefreshTask.cpp
+++ b/src/Storages/MaterializedView/RefreshTask.cpp
@@ -357,7 +357,7 @@ void RefreshTask::refreshTask()
         stop_requested = true;
         tryLogCurrentException(log,
             "Unexpected exception in refresh scheduling, please investigate. The view will be stopped.");
-#ifdef ABORT_ON_LOGICAL_ERROR
+#ifdef DEBUG_OR_SANITIZER_BUILD
         abortOnFailedAssertion("Unexpected exception in refresh scheduling");
 #endif
     }
diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp
index 72f725965e0..3f02486ed15 100644
--- a/src/Storages/StorageReplicatedMergeTree.cpp
+++ b/src/Storages/StorageReplicatedMergeTree.cpp
@@ -1516,7 +1516,7 @@ static time_t tryGetPartCreateTime(zkutil::ZooKeeperPtr & zookeeper, const Strin
 
 void StorageReplicatedMergeTree::paranoidCheckForCoveredPartsInZooKeeperOnStart(const Strings & parts_in_zk, const Strings & parts_to_fetch) const
 {
-#ifdef ABORT_ON_LOGICAL_ERROR
+#ifdef DEBUG_OR_SANITIZER_BUILD
     constexpr bool paranoid_check_for_covered_parts_default = true;
 #else
     constexpr bool paranoid_check_for_covered_parts_default = false;
@@ -2383,7 +2383,7 @@ static void paranoidCheckForCoveredPartsInZooKeeper(
     const String & covering_part_name,
     const StorageReplicatedMergeTree & storage)
 {
-#ifdef ABORT_ON_LOGICAL_ERROR
+#ifdef DEBUG_OR_SANITIZER_BUILD
     constexpr bool paranoid_check_for_covered_parts_default = true;
 #else
     constexpr bool paranoid_check_for_covered_parts_default = false;

From 8e013a1c5938e7cecba5804ac8e9f653a95e4ddc Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Wed, 17 Jul 2024 16:23:36 +0000
Subject: [PATCH 0365/1488] Disable getConstantResultForNonConstArguments for
 IS NULL with old analyzer.

---
 src/Functions/isNotNull.cpp                   | 10 +++++++-
 src/Functions/isNull.cpp                      | 15 ++++++++++--
 src/Functions/isNullable.cpp                  | 15 ++++++++++--
 ...constant_result_old_analyzer_bug.reference |  0
 ..._null_constant_result_old_analyzer_bug.sql | 23 +++++++++++++++++++
 5 files changed, 58 insertions(+), 5 deletions(-)
 create mode 100644 tests/queries/0_stateless/03206_is_null_constant_result_old_analyzer_bug.reference
 create mode 100644 tests/queries/0_stateless/03206_is_null_constant_result_old_analyzer_bug.sql

diff --git a/src/Functions/isNotNull.cpp b/src/Functions/isNotNull.cpp
index a10e7ebd40c..a48ace4243f 100644
--- a/src/Functions/isNotNull.cpp
+++ b/src/Functions/isNotNull.cpp
@@ -22,7 +22,9 @@ class FunctionIsNotNull : public IFunction
 public:
     static constexpr auto name = "isNotNull";
 
-    static FunctionPtr create(ContextPtr) { return std::make_shared<FunctionIsNotNull>(); }
+    static FunctionPtr create(ContextPtr context) { return std::make_shared<FunctionIsNotNull>(context->getSettingsRef().allow_experimental_analyzer); }
+
+    explicit FunctionIsNotNull(bool use_analyzer_) : use_analyzer(use_analyzer_) {}
 
     std::string getName() const override
     {
@@ -31,6 +33,10 @@ public:
 
     ColumnPtr getConstantResultForNonConstArguments(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type) const override
     {
+        /// (column IS NULL) triggers a bug in old analyzer when it is replaced to constant.
+        if (!use_analyzer)
+            return nullptr;
+
         const ColumnWithTypeAndName & elem = arguments[0];
         if (elem.type->onlyNull())
             return result_type->createColumnConst(1, UInt8(0));
@@ -123,6 +129,8 @@ private:
 #endif
         vectorImpl(null_map, res);
     }
+
+    bool use_analyzer;
 };
 }
 
diff --git a/src/Functions/isNull.cpp b/src/Functions/isNull.cpp
index 95d659b103b..3317d73ba27 100644
--- a/src/Functions/isNull.cpp
+++ b/src/Functions/isNull.cpp
@@ -7,6 +7,8 @@
 #include <Columns/ColumnLowCardinality.h>
 #include <Columns/ColumnVariant.h>
 #include <Columns/ColumnDynamic.h>
+#include <Core/Settings.h>
+#include <Interpreters/Context.h>
 
 
 namespace DB
@@ -21,11 +23,13 @@ class FunctionIsNull : public IFunction
 public:
     static constexpr auto name = "isNull";
 
-    static FunctionPtr create(ContextPtr)
+    static FunctionPtr create(ContextPtr context)
     {
-        return std::make_shared<FunctionIsNull>();
+        return std::make_shared<FunctionIsNull>(context->getSettingsRef().allow_experimental_analyzer);
     }
 
+    explicit FunctionIsNull(bool use_analyzer_) : use_analyzer(use_analyzer_) {}
+
     std::string getName() const override
     {
         return name;
@@ -33,6 +37,10 @@ public:
 
     ColumnPtr getConstantResultForNonConstArguments(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type) const override
     {
+        /// (column IS NULL) triggers a bug in old analyzer when it is replaced to constant.
+        if (!use_analyzer)
+            return nullptr;
+
         const ColumnWithTypeAndName & elem = arguments[0];
         if (elem.type->onlyNull())
             return result_type->createColumnConst(1, UInt8(1));
@@ -95,6 +103,9 @@ public:
             return DataTypeUInt8().createColumnConst(elem.column->size(), 0u);
         }
     }
+
+private:
+    bool use_analyzer;
 };
 
 }
diff --git a/src/Functions/isNullable.cpp b/src/Functions/isNullable.cpp
index b24ee4f5e73..d21ac9cf07c 100644
--- a/src/Functions/isNullable.cpp
+++ b/src/Functions/isNullable.cpp
@@ -3,6 +3,8 @@
 #include <DataTypes/DataTypesNumber.h>
 #include <Columns/ColumnsNumber.h>
 #include <DataTypes/DataTypeNullable.h>
+#include <Core/Settings.h>
+#include <Interpreters/Context.h>
 
 namespace DB
 {
@@ -14,11 +16,13 @@ class FunctionIsNullable : public IFunction
 {
 public:
     static constexpr auto name = "isNullable";
-    static FunctionPtr create(ContextPtr)
+    static FunctionPtr create(ContextPtr context)
     {
-        return std::make_shared<FunctionIsNullable>();
+        return std::make_shared<FunctionIsNullable>(context->getSettingsRef().allow_experimental_analyzer);
     }
 
+    explicit FunctionIsNullable(bool use_analyzer_) : use_analyzer(use_analyzer_) {}
+
     String getName() const override
     {
         return name;
@@ -26,6 +30,10 @@ public:
 
     ColumnPtr getConstantResultForNonConstArguments(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type) const override
     {
+        /// isNullable(column) triggers a bug in old analyzer when it is replaced to constant.
+        if (!use_analyzer)
+            return nullptr;
+
         const ColumnWithTypeAndName & elem = arguments[0];
         if (elem.type->onlyNull() || canContainNull(*elem.type))
             return result_type->createColumnConst(1, UInt8(1));
@@ -60,6 +68,9 @@ public:
         const auto & elem = arguments[0];
         return ColumnUInt8::create(input_rows_count, isColumnNullable(*elem.column) || elem.type->isLowCardinalityNullable());
     }
+
+private:
+    bool use_analyzer;
 };
 
 }
diff --git a/tests/queries/0_stateless/03206_is_null_constant_result_old_analyzer_bug.reference b/tests/queries/0_stateless/03206_is_null_constant_result_old_analyzer_bug.reference
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/queries/0_stateless/03206_is_null_constant_result_old_analyzer_bug.sql b/tests/queries/0_stateless/03206_is_null_constant_result_old_analyzer_bug.sql
new file mode 100644
index 00000000000..39e9ebfcd77
--- /dev/null
+++ b/tests/queries/0_stateless/03206_is_null_constant_result_old_analyzer_bug.sql
@@ -0,0 +1,23 @@
+CREATE TABLE left (x UUID) ORDER BY tuple();
+
+CREATE TABLE right (x UUID) ORDER BY tuple();
+
+set allow_experimental_analyzer=0;
+
+SELECT left.x, (right.x IS NULL)::Boolean FROM left LEFT OUTER JOIN right ON left.x = right.x GROUP BY ALL;
+
+SELECT isNullable(number)::Boolean, now() FROM numbers(2) GROUP BY isNullable(number)::Boolean, now() FORMAT Null;
+
+SELECT isNull(number)::Boolean, now() FROM numbers(2) GROUP BY isNull(number)::Boolean, now() FORMAT Null;
+
+SELECT (number IS NULL)::Boolean, now() FROM numbers(2) GROUP BY (number IS NULL)::Boolean, now() FORMAT Null;
+
+set allow_experimental_analyzer=1;
+
+SELECT left.x, (right.x IS NULL)::Boolean FROM left LEFT OUTER JOIN right ON left.x = right.x GROUP BY ALL;
+
+SELECT isNullable(number)::Boolean, now() FROM numbers(2) GROUP BY isNullable(number)::Boolean, now() FORMAT Null;
+
+SELECT isNull(number)::Boolean, now() FROM numbers(2) GROUP BY isNull(number)::Boolean, now() FORMAT Null;
+
+SELECT (number IS NULL)::Boolean, now() FROM numbers(2) GROUP BY (number IS NULL)::Boolean, now() FORMAT Null;

From a3d02291b42ad0de78534bf9ae64a2f0dc3a3826 Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Wed, 17 Jul 2024 18:24:13 +0200
Subject: [PATCH 0366/1488] Increase backoff a little for inserts

---
 tests/config/users.d/insert_keeper_retries.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/config/users.d/insert_keeper_retries.xml b/tests/config/users.d/insert_keeper_retries.xml
index f01157d40ac..0cf41473616 100644
--- a/tests/config/users.d/insert_keeper_retries.xml
+++ b/tests/config/users.d/insert_keeper_retries.xml
@@ -4,7 +4,7 @@
             <insert_keeper_max_retries>100</insert_keeper_max_retries>
             <insert_keeper_fault_injection_probability>0.01</insert_keeper_fault_injection_probability>
             <insert_keeper_retry_initial_backoff_ms>1</insert_keeper_retry_initial_backoff_ms>
-            <insert_keeper_retry_max_backoff_ms>1</insert_keeper_retry_max_backoff_ms>
+            <insert_keeper_retry_max_backoff_ms>10</insert_keeper_retry_max_backoff_ms>
         </default>
     </profiles>
 </clickhouse>

From d5345c5aae8c8598a043311e58318d22c5396392 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 17 Jul 2024 18:31:10 +0200
Subject: [PATCH 0367/1488] Update Squashing.cpp

---
 src/Interpreters/Squashing.cpp | 32 --------------------------------
 1 file changed, 32 deletions(-)

diff --git a/src/Interpreters/Squashing.cpp b/src/Interpreters/Squashing.cpp
index 4af7d57f5a0..49e95f202dd 100644
--- a/src/Interpreters/Squashing.cpp
+++ b/src/Interpreters/Squashing.cpp
@@ -17,38 +17,6 @@ namespace ErrorCodes
 namespace
 {
 
-Chunk squashImpl(std::vector<Chunk> & input_chunks)
-{
-    Chunk accumulated_chunk;
-    std::vector<IColumn::MutablePtr> mutable_columns = {};
-    size_t rows = 0;
-    for (const Chunk & chunk : input_chunks)
-        rows += chunk.getNumRows();
-
-    {
-        auto & first_chunk = input_chunks[0];
-        Columns columns = first_chunk.detachColumns();
-        for (auto & column : columns)
-        {
-            mutable_columns.push_back(IColumn::mutate(std::move(column)));
-            mutable_columns.back()->reserve(rows);
-        }
-    }
-
-    for (size_t i = 1; i < input_chunks.size(); ++i) // We've already processed the first chunk above
-    {
-        Columns columns = input_chunks[i].detachColumns();
-        for (size_t j = 0, size = mutable_columns.size(); j < size; ++j)
-        {
-            const auto source_column = columns[j];
-
-            mutable_columns[j]->insertRangeFrom(*source_column, 0, source_column->size());
-        }
-    }
-    accumulated_chunk.setColumns(std::move(mutable_columns), rows);
-    return accumulated_chunk;
-}
-
 const ChunksToSquash * getInfoFromChunk(const Chunk & chunk)
 {
     const auto & info = chunk.getChunkInfo();

From 37dd7756434c4a2fa780fc187505c67e92be6f45 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 17 Jul 2024 18:31:27 +0200
Subject: [PATCH 0368/1488] Update Squashing.cpp

---
 src/Interpreters/Squashing.cpp | 16 ----------------
 1 file changed, 16 deletions(-)

diff --git a/src/Interpreters/Squashing.cpp b/src/Interpreters/Squashing.cpp
index 49e95f202dd..3a7f28ed837 100644
--- a/src/Interpreters/Squashing.cpp
+++ b/src/Interpreters/Squashing.cpp
@@ -14,22 +14,6 @@ namespace ErrorCodes
     extern const int LOGICAL_ERROR;
 }
 
-namespace
-{
-
-const ChunksToSquash * getInfoFromChunk(const Chunk & chunk)
-{
-    const auto & info = chunk.getChunkInfo();
-    const auto * agg_info = typeid_cast<const ChunksToSquash *>(info.get());
-
-    if (!agg_info)
-        throw Exception(ErrorCodes::LOGICAL_ERROR, "There is no ChunksToSquash in ChunkInfoPtr");
-
-    return agg_info;
-}
-
-}
-
 Squashing::Squashing(Block header_, size_t min_block_size_rows_, size_t min_block_size_bytes_)
     : min_block_size_rows(min_block_size_rows_)
     , min_block_size_bytes(min_block_size_bytes_)

From 8bc537b490c5fc8ac499227e0b1cbb4ccb6a5af2 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Wed, 17 Jul 2024 17:26:51 +0000
Subject: [PATCH 0369/1488] Allo scalar subquery in fisrt argiment of IN with
 new analyzer.

---
 src/Analyzer/Resolve/QueryAnalyzer.cpp                | 11 +++++++++++
 .../0_stateless/02370_analyzer_in_function.reference  |  3 +++
 .../0_stateless/02370_analyzer_in_function.sql        |  4 ++++
 3 files changed, 18 insertions(+)

diff --git a/src/Analyzer/Resolve/QueryAnalyzer.cpp b/src/Analyzer/Resolve/QueryAnalyzer.cpp
index 63788560959..5b31d727e37 100644
--- a/src/Analyzer/Resolve/QueryAnalyzer.cpp
+++ b/src/Analyzer/Resolve/QueryAnalyzer.cpp
@@ -2919,6 +2919,17 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi
 
             resolveExpressionNode(in_second_argument, scope, false /*allow_lambda_expression*/, true /*allow_table_expression*/);
         }
+
+        /// Edge case when the first argument of IN is scalar subquery.
+        auto & in_first_argument = function_in_arguments_nodes[0];
+        auto first_argument_type = in_first_argument->getNodeType();
+        if (first_argument_type == QueryTreeNodeType::QUERY || first_argument_type == QueryTreeNodeType::UNION)
+        {
+            IdentifierResolveScope subquery_scope(in_first_argument, &scope /*parent_scope*/);
+            subquery_scope.subquery_depth = scope.subquery_depth + 1;
+
+            evaluateScalarSubqueryIfNeeded(in_first_argument, subquery_scope);
+        }
     }
 
     /// Initialize function argument columns
diff --git a/tests/queries/0_stateless/02370_analyzer_in_function.reference b/tests/queries/0_stateless/02370_analyzer_in_function.reference
index fda174c0b7c..49a080c2616 100644
--- a/tests/queries/0_stateless/02370_analyzer_in_function.reference
+++ b/tests/queries/0_stateless/02370_analyzer_in_function.reference
@@ -12,3 +12,6 @@
 1
 1
 0
+1
+1
+1
diff --git a/tests/queries/0_stateless/02370_analyzer_in_function.sql b/tests/queries/0_stateless/02370_analyzer_in_function.sql
index 7287c94deda..a6e4400e101 100644
--- a/tests/queries/0_stateless/02370_analyzer_in_function.sql
+++ b/tests/queries/0_stateless/02370_analyzer_in_function.sql
@@ -21,3 +21,7 @@ SELECT (1, 2) IN 1; -- { serverError ILLEGAL_TYPE_OF_ARGUMENT }
 SELECT (1, 2) IN [1]; -- { serverError INCORRECT_ELEMENT_OF_SET }
 SELECT (1, 2) IN (((1, 2), (1, 2)), ((1, 2), (1, 2))); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT }
 SELECT (1, 2) IN [((1, 2), (1, 2)), ((1, 2), (1, 2))]; -- { serverError ILLEGAL_TYPE_OF_ARGUMENT }
+
+select (select 1) in (1);
+select in(untuple(((1), (1))));
+select in(untuple(((select 1), (1))));

From 80e1377e5d2223176274c319938187f0da799280 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Wed, 17 Jul 2024 17:30:28 +0000
Subject: [PATCH 0370/1488] Fixing build.

---
 src/Storages/System/StorageSystemTables.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Storages/System/StorageSystemTables.cpp b/src/Storages/System/StorageSystemTables.cpp
index 43b761d84b1..d6b577bf6c8 100644
--- a/src/Storages/System/StorageSystemTables.cpp
+++ b/src/Storages/System/StorageSystemTables.cpp
@@ -125,7 +125,7 @@ ColumnPtr getFilteredTables(
         block.insert(ColumnWithTypeAndName(std::move(engine_column), std::make_shared<DataTypeString>(), "engine"));
 
     if (dag)
-        VirtualColumnUtils::filterBlockWithDAG(dag, block, context);
+        VirtualColumnUtils::filterBlockWithDAG(std::move(*dag), block, context);
 
     return block.getByPosition(0).column;
 }

From c7fa082eac37bd109085f2c65f85f752781864a2 Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Wed, 17 Jul 2024 19:32:54 +0200
Subject: [PATCH 0371/1488] add log for splitBlockIntoParts

---
 src/Storages/MergeTree/MergeTreeDataWriter.cpp | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp
index 73244b714bf..ee3ac4207cc 100644
--- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp
+++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp
@@ -315,6 +315,12 @@ BlocksWithPartition MergeTreeDataWriter::splitBlockIntoParts(
 
     for (size_t i = 0; i < async_insert_info_with_partition.size(); ++i)
     {
+        if (async_insert_info_with_partition[i] == nullptr)
+        {
+            LOG_ERROR(getLogger("MergeTreeDataWriter"), "The {}th element in async_insert_info_with_partition is nullptr. There are totally {} partitions in the insert. Selector content is {}",
+                      i, partitions_count, fmt::join(selector.begin(), selector.end(), ","));
+            throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected error for async deduplicated insert, please check error logs");
+        }
         result[i].offsets = std::move(async_insert_info_with_partition[i]->offsets);
         result[i].tokens = std::move(async_insert_info_with_partition[i]->tokens);
     }

From c57ce063a346698b550134762277c81e12072d88 Mon Sep 17 00:00:00 2001
From: Nikita Taranov <nikita.taranov@clickhouse.com>
Date: Wed, 17 Jul 2024 18:37:38 +0100
Subject: [PATCH 0372/1488] impl

---
 .../02044_url_glob_parallel_connection_refused.sh            | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/02044_url_glob_parallel_connection_refused.sh b/tests/queries/0_stateless/02044_url_glob_parallel_connection_refused.sh
index 7e8579f7cbe..b4b0ee8a023 100755
--- a/tests/queries/0_stateless/02044_url_glob_parallel_connection_refused.sh
+++ b/tests/queries/0_stateless/02044_url_glob_parallel_connection_refused.sh
@@ -8,7 +8,10 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
 i=0 retries=5
 # Connecting to wrong address and checking for race condition
+# http_max_tries is limited to 2 because with the default 10 retries the execution time might go as high as around 3 minutes (because of exponential back-off).
+# because of that we might see wrong 'tests hung' reports depending on how close to the end of tests run this particular test was executed.
+# proper fix should be implemented in https://github.com/ClickHouse/ClickHouse/issues/66656
 while [[ $i -lt $retries ]]; do
-    timeout 5s ${CLICKHOUSE_CLIENT} --max_threads 10 --query "SELECT * FROM url('http://128.0.0.{1..10}:${CLICKHOUSE_PORT_HTTP}/?query=SELECT+sleep(1)', TSV, 'x UInt8')" --format Null 2>/dev/null
+    timeout 5s ${CLICKHOUSE_CLIENT} --max_threads 10 --http_max_tries 2 --query "SELECT * FROM url('http://128.0.0.{1..10}:${CLICKHOUSE_PORT_HTTP}/?query=SELECT+sleep(1)', TSV, 'x UInt8')" --format Null 2>/dev/null
     ((++i))
 done

From eb129232ff27c8103aa5e71d4ce8a9cdbc905dde Mon Sep 17 00:00:00 2001
From: Amos Bird <amosbird@gmail.com>
Date: Wed, 17 Jul 2024 18:47:33 +0800
Subject: [PATCH 0373/1488] Avoid generating named tuple for special keywords

---
 src/Parsers/isUnquotedIdentifier.cpp                | 13 +++++++++++++
 src/Parsers/isUnquotedIdentifier.h                  |  8 ++++++++
 .../02890_named_tuple_functions.reference           |  1 +
 .../0_stateless/02890_named_tuple_functions.sql     |  5 ++++-
 4 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/src/Parsers/isUnquotedIdentifier.cpp b/src/Parsers/isUnquotedIdentifier.cpp
index 6f2442635ec..26cb3992a50 100644
--- a/src/Parsers/isUnquotedIdentifier.cpp
+++ b/src/Parsers/isUnquotedIdentifier.cpp
@@ -1,5 +1,6 @@
 #include <Parsers/isUnquotedIdentifier.h>
 
+#include <Parsers/CommonParsers.h>
 #include <Parsers/Lexer.h>
 
 namespace DB
@@ -7,6 +8,18 @@ namespace DB
 
 bool isUnquotedIdentifier(const String & name)
 {
+    auto is_keyword = [&name](Keyword keyword)
+    {
+        auto s = toStringView(keyword);
+        if (name.size() != s.size())
+            return false;
+        return strncasecmp(s.data(), name.data(), s.size()) == 0;
+    };
+
+    /// Special keywords are parsed as literals instead of identifiers.
+    if (is_keyword(Keyword::NULL_KEYWORD) || is_keyword(Keyword::TRUE_KEYWORD) || is_keyword(Keyword::FALSE_KEYWORD))
+        return false;
+
     Lexer lexer(name.data(), name.data() + name.size());
 
     auto maybe_ident = lexer.nextToken();
diff --git a/src/Parsers/isUnquotedIdentifier.h b/src/Parsers/isUnquotedIdentifier.h
index 839e5860ad3..9c9f9239eb3 100644
--- a/src/Parsers/isUnquotedIdentifier.h
+++ b/src/Parsers/isUnquotedIdentifier.h
@@ -5,6 +5,14 @@
 namespace DB
 {
 
+/// Checks if the input string @name is a valid unquoted identifier.
+///
+/// Example Usage:
+///   abc     -> true   (valid unquoted identifier)
+///   123     -> false  (identifiers cannot start with digits)
+///   `123`   -> false  (quoted identifiers are not considered)
+///   `abc`   -> false  (quoted identifiers are not considered)
+///   null    -> false  (reserved literal keyword)
 bool isUnquotedIdentifier(const String & name);
 
 }
diff --git a/tests/queries/0_stateless/02890_named_tuple_functions.reference b/tests/queries/0_stateless/02890_named_tuple_functions.reference
index f7a0c440b5a..6b36ff2c54c 100644
--- a/tests/queries/0_stateless/02890_named_tuple_functions.reference
+++ b/tests/queries/0_stateless/02890_named_tuple_functions.reference
@@ -7,3 +7,4 @@ Tuple(\n    k UInt8,\n    j Int32)
 Tuple(Int32, Int32, Int32, Int32)
 ['1','2','3','4']
 (1,2,3)
+Tuple(Nullable(Nothing))	Tuple(Bool)	Tuple(Bool)
diff --git a/tests/queries/0_stateless/02890_named_tuple_functions.sql b/tests/queries/0_stateless/02890_named_tuple_functions.sql
index 8e0c9c2b10e..0033102bd53 100644
--- a/tests/queries/0_stateless/02890_named_tuple_functions.sql
+++ b/tests/queries/0_stateless/02890_named_tuple_functions.sql
@@ -28,4 +28,7 @@ create table tbl (x Tuple(a Int32, b Int32, c Int32)) engine MergeTree order by
 insert into tbl values (tuple(1, 2, 3)); -- without tuple it's interpreted differently inside values block.
 select * from tbl;
 
-drop table tbl
+drop table tbl;
+
+-- Avoid generating named tuple for special keywords
+select toTypeName(tuple(null)), toTypeName(tuple(true)), toTypeName(tuple(false));

From b4a3cba519b397f91ec38e806dcc5597b846a1c5 Mon Sep 17 00:00:00 2001
From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com>
Date: Wed, 17 Jul 2024 20:09:14 +0200
Subject: [PATCH 0374/1488] Fix possible PARAMETER_OUT_OF_BOUND error during
 reading variant subcolumn

---
 .../SerializationVariantElement.cpp           | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/src/DataTypes/Serializations/SerializationVariantElement.cpp b/src/DataTypes/Serializations/SerializationVariantElement.cpp
index 8ceab17cba4..03b5d9584e0 100644
--- a/src/DataTypes/Serializations/SerializationVariantElement.cpp
+++ b/src/DataTypes/Serializations/SerializationVariantElement.cpp
@@ -11,6 +11,7 @@ namespace DB
 namespace ErrorCodes
 {
     extern const int NOT_IMPLEMENTED;
+    extern const int LOGICAL_ERROR;
 }
 
 struct SerializationVariantElement::DeserializeBinaryBulkStateVariantElement : public ISerialization::DeserializeBinaryBulkState
@@ -188,13 +189,6 @@ void SerializationVariantElement::deserializeBinaryBulkWithMultipleStreams(
             assert_cast<ColumnLowCardinality &>(*variant_element_state->variant->assumeMutable()).nestedRemoveNullable();
     }
 
-    /// If nothing to deserialize, just insert defaults.
-    if (variant_limit == 0)
-    {
-        mutable_column->insertManyDefaults(num_new_discriminators);
-        return;
-    }
-
     addVariantToPath(settings.path);
     nested_serialization->deserializeBinaryBulkWithMultipleStreams(variant_element_state->variant, *variant_limit, settings, variant_element_state->variant_element_state, cache);
     removeVariantFromPath(settings.path);
@@ -209,6 +203,17 @@ void SerializationVariantElement::deserializeBinaryBulkWithMultipleStreams(
         return;
     }
 
+    /// If there was nothing to deserialize or nothing was actually deserialized when variant_limit > 0, just insert defaults.
+    /// The second case means that we don't have a stream for such sub-column. It may happen during ALTER MODIFY column with Variant extension.
+    if (variant_limit == 0 || variant_element_state->variant->empty())
+    {
+        mutable_column->insertManyDefaults(num_new_discriminators);
+        return;
+    }
+
+    if (variant_element_state->variant->size() < *variant_limit)
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "Size of deserialized variant column less than the limit: {} < {}", variant_element_state->variant->size(), *variant_limit);
+
     size_t variant_offset = variant_element_state->variant->size() - *variant_limit;
 
     /// If we have only our discriminator in range, insert the whole range to result column.

From 29d3f9598d58aa6010509fbf4e2344cefd4cef24 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 17 Jul 2024 20:53:05 +0200
Subject: [PATCH 0375/1488] Fix test

---
 .../02164_clickhouse_local_interactive_exception.expect         | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/02164_clickhouse_local_interactive_exception.expect b/tests/queries/0_stateless/02164_clickhouse_local_interactive_exception.expect
index 76902bdc69d..add977c4fce 100755
--- a/tests/queries/0_stateless/02164_clickhouse_local_interactive_exception.expect
+++ b/tests/queries/0_stateless/02164_clickhouse_local_interactive_exception.expect
@@ -9,7 +9,7 @@ if {[info exists env(CLICKHOUSE_TMP)]} {
 }
 exp_internal -f $CLICKHOUSE_TMP/$basename.debuglog 0
 
-log_user 1
+log_user 0
 set timeout 20
 match_max 100000
 

From 47573f1ae158faabd6ac0a62598c907c85fbd4c1 Mon Sep 17 00:00:00 2001
From: jsc0218 <jsc0218@gmail.com>
Date: Wed, 17 Jul 2024 20:44:44 +0000
Subject: [PATCH 0376/1488] add options

---
 src/Core/SettingsEnums.h                   | 9 +++++++++
 src/Storages/MergeTree/MergeTreeSettings.h | 1 +
 2 files changed, 10 insertions(+)

diff --git a/src/Core/SettingsEnums.h b/src/Core/SettingsEnums.h
index f6d9593ca56..ac3264fe041 100644
--- a/src/Core/SettingsEnums.h
+++ b/src/Core/SettingsEnums.h
@@ -315,6 +315,15 @@ enum class LightweightMutationProjectionMode : uint8_t
 
 DECLARE_SETTING_ENUM(LightweightMutationProjectionMode)
 
+enum class DeduplicateMergeProjectionMode : uint8_t
+{
+    THROW,
+    DROP,
+    REBUILD,
+};
+
+DECLARE_SETTING_ENUM(DeduplicateMergeProjectionMode)
+
 DECLARE_SETTING_ENUM(LocalFSReadMethod)
 
 enum class ObjectStorageQueueMode : uint8_t
diff --git a/src/Storages/MergeTree/MergeTreeSettings.h b/src/Storages/MergeTree/MergeTreeSettings.h
index f5ada81cf55..166a18a7bab 100644
--- a/src/Storages/MergeTree/MergeTreeSettings.h
+++ b/src/Storages/MergeTree/MergeTreeSettings.h
@@ -215,6 +215,7 @@ struct Settings;
     M(Float, primary_key_ratio_of_unique_prefix_values_to_skip_suffix_columns, 0.9f, "If the value of a column of the primary key in data part changes at least in this ratio of times, skip loading next columns in memory. This allows to save memory usage by not loading useless columns of the primary key.", 0) \
     /** Projection settings. */ \
     M(UInt64, max_projections, 25, "The maximum number of merge tree projections.", 0) \
+    M(DeduplicateMergeProjectionMode, deduplicate_merge_projection_mode, DeduplicateMergeProjectionMode::THROW, "Whether to allow create projection for the table with non-classic MergeTree, if allowed, what is the next action.", 0) \
 
 #define MAKE_OBSOLETE_MERGE_TREE_SETTING(M, TYPE, NAME, DEFAULT) \
     M(TYPE, NAME, DEFAULT, "Obsolete setting, does nothing.", BaseSettingsHelpers::Flags::OBSOLETE)

From 79402aa71b62d2a3f9cbd462c40e8d710714e37e Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Wed, 17 Jul 2024 20:39:37 +0000
Subject: [PATCH 0377/1488] Make CaseSensitiveness an enum class

---
 .../AggregateFunctionAnalysisOfVariance.cpp   |  4 +-
 .../AggregateFunctionAny.cpp                  |  6 +-
 .../AggregateFunctionAnyRespectNulls.cpp      |  6 +-
 .../AggregateFunctionAvg.cpp                  |  2 +-
 .../AggregateFunctionBitwise.cpp              |  6 +-
 .../AggregateFunctionCorr.cpp                 |  2 +-
 .../AggregateFunctionCount.cpp                |  2 +-
 .../AggregateFunctionCovar.cpp                |  4 +-
 .../AggregateFunctionFactory.cpp              |  4 +-
 .../AggregateFunctionFactory.h                |  2 +-
 .../AggregateFunctionGroupArray.cpp           |  4 +-
 .../AggregateFunctionGroupConcat.cpp          |  2 +-
 ...AggregateFunctionKolmogorovSmirnovTest.cpp |  2 +-
 .../AggregateFunctionSecondMoment.cpp         | 10 +--
 .../AggregateFunctionSum.cpp                  |  2 +-
 .../AggregateFunctionTopK.cpp                 |  6 +-
 .../AggregateFunctionsMinMax.cpp              |  4 +-
 src/Common/IFactoryWithAliases.h              | 12 ++--
 src/DataTypes/DataTypeDate.cpp                |  2 +-
 src/DataTypes/DataTypeDate32.cpp              |  2 +-
 src/DataTypes/DataTypeDomainBool.cpp          |  4 +-
 src/DataTypes/DataTypeEnum.cpp                |  2 +-
 src/DataTypes/DataTypeFactory.cpp             | 10 +--
 src/DataTypes/DataTypeFactory.h               |  8 +--
 src/DataTypes/DataTypeFixedString.cpp         |  2 +-
 src/DataTypes/DataTypeIPv4andIPv6.cpp         |  4 +-
 src/DataTypes/DataTypeObject.cpp              |  2 +-
 src/DataTypes/DataTypeString.cpp              | 62 ++++++++---------
 src/DataTypes/DataTypesDecimal.cpp            | 16 ++---
 src/DataTypes/DataTypesNumber.cpp             | 66 +++++++++----------
 src/DataTypes/registerDataTypeDateTime.cpp    |  8 +--
 src/Functions/CRC.cpp                         |  6 +-
 src/Functions/CastOverloadResolver.cpp        |  4 +-
 src/Functions/FunctionChar.cpp                |  2 +-
 src/Functions/FunctionFQDN.cpp                |  2 +-
 src/Functions/FunctionFactory.cpp             |  6 +-
 src/Functions/FunctionFactory.h               |  8 +--
 .../FunctionGenerateRandomStructure.cpp       |  3 +-
 .../FunctionsBinaryRepresentation.cpp         |  8 +--
 src/Functions/FunctionsCodingIP.cpp           |  8 +--
 src/Functions/FunctionsCodingULID.cpp         |  3 +-
 src/Functions/FunctionsCodingUUID.cpp         |  7 +-
 src/Functions/FunctionsConversion.cpp         |  2 +-
 src/Functions/FunctionsHashingMisc.cpp        |  3 +-
 src/Functions/FunctionsLogical.cpp            |  2 +-
 src/Functions/FunctionsOpDate.cpp             |  4 +-
 src/Functions/FunctionsRound.cpp              | 14 ++--
 .../FunctionsStringHashFixedString.cpp        |  3 +-
 src/Functions/JSONArrayLength.cpp             |  2 +-
 src/Functions/UTCTimestamp.cpp                |  4 +-
 src/Functions/UTCTimestampTransform.cpp       |  4 +-
 src/Functions/abs.cpp                         |  2 +-
 src/Functions/acos.cpp                        |  2 +-
 src/Functions/array/arrayFlatten.cpp          |  2 +-
 src/Functions/array/arrayShuffle.cpp          |  4 +-
 src/Functions/array/length.cpp                |  4 +-
 src/Functions/ascii.cpp                       |  2 +-
 src/Functions/asin.cpp                        |  2 +-
 src/Functions/atan.cpp                        |  2 +-
 src/Functions/atan2.cpp                       |  2 +-
 src/Functions/base64Decode.cpp                |  2 +-
 src/Functions/base64Encode.cpp                |  2 +-
 src/Functions/byteSwap.cpp                    |  2 +-
 src/Functions/coalesce.cpp                    |  2 +-
 src/Functions/concat.cpp                      |  2 +-
 src/Functions/concatWithSeparator.cpp         |  2 +-
 src/Functions/connectionId.cpp                |  4 +-
 src/Functions/cos.cpp                         |  2 +-
 src/Functions/countMatches.cpp                |  4 +-
 src/Functions/countSubstrings.cpp             |  2 +-
 src/Functions/currentDatabase.cpp             |  6 +-
 src/Functions/currentSchemas.cpp              |  4 +-
 src/Functions/currentUser.cpp                 |  4 +-
 src/Functions/dateDiff.cpp                    |  6 +-
 src/Functions/dateName.cpp                    |  2 +-
 src/Functions/date_trunc.cpp                  |  2 +-
 src/Functions/degrees.cpp                     |  2 +-
 src/Functions/exp.cpp                         |  2 +-
 src/Functions/extractAllGroupsVertical.cpp    |  2 +-
 src/Functions/factorial.cpp                   |  2 +-
 src/Functions/formatDateTime.cpp              |  4 +-
 src/Functions/formatReadableDecimalSize.cpp   |  3 +-
 src/Functions/formatReadableSize.cpp          |  2 +-
 src/Functions/fromDaysSinceYearZero.cpp       |  2 +-
 src/Functions/generateULID.cpp                |  3 +-
 src/Functions/greatest.cpp                    |  2 +-
 src/Functions/hasSubsequence.cpp              |  2 +-
 .../hasSubsequenceCaseInsensitive.cpp         |  2 +-
 .../hasSubsequenceCaseInsensitiveUTF8.cpp     |  2 +-
 src/Functions/hasSubsequenceUTF8.cpp          |  2 +-
 src/Functions/hasToken.cpp                    |  4 +-
 src/Functions/hasTokenCaseInsensitive.cpp     |  4 +-
 src/Functions/hypot.cpp                       |  2 +-
 src/Functions/if.cpp                          |  2 +-
 src/Functions/ifNull.cpp                      |  2 +-
 src/Functions/initcap.cpp                     |  2 +-
 src/Functions/initialQueryID.cpp              |  2 +-
 src/Functions/isNull.cpp                      |  2 +-
 .../keyvaluepair/extractKeyValuePairs.cpp     |  2 +-
 src/Functions/least.cpp                       |  2 +-
 src/Functions/left.cpp                        |  4 +-
 src/Functions/lengthUTF8.cpp                  |  4 +-
 src/Functions/locate.cpp                      |  2 +-
 src/Functions/log.cpp                         |  4 +-
 src/Functions/log10.cpp                       |  2 +-
 src/Functions/log2.cpp                        |  2 +-
 src/Functions/lower.cpp                       |  4 +-
 src/Functions/makeDate.cpp                    |  2 +-
 src/Functions/match.cpp                       |  2 +-
 src/Functions/mathConstants.cpp               |  2 +-
 src/Functions/max2.cpp                        |  2 +-
 src/Functions/min2.cpp                        |  2 +-
 src/Functions/modulo.cpp                      |  8 +--
 src/Functions/monthName.cpp                   |  2 +-
 src/Functions/now.cpp                         |  4 +-
 src/Functions/now64.cpp                       |  2 +-
 src/Functions/nullIf.cpp                      |  2 +-
 src/Functions/padString.cpp                   |  4 +-
 src/Functions/parseDateTime.cpp               |  4 +-
 src/Functions/position.cpp                    |  2 +-
 src/Functions/positionCaseInsensitive.cpp     |  2 +-
 src/Functions/pow.cpp                         |  4 +-
 src/Functions/queryID.cpp                     |  2 +-
 src/Functions/radians.cpp                     |  2 +-
 src/Functions/rand.cpp                        |  2 +-
 src/Functions/regexpExtract.cpp               |  2 +-
 src/Functions/repeat.cpp                      |  2 +-
 src/Functions/replaceAll.cpp                  |  2 +-
 src/Functions/replaceRegexpAll.cpp            |  2 +-
 src/Functions/reverse.cpp                     |  2 +-
 src/Functions/right.cpp                       |  4 +-
 src/Functions/serverConstants.cpp             |  7 +-
 src/Functions/sign.cpp                        |  2 +-
 src/Functions/sin.cpp                         |  2 +-
 src/Functions/soundex.cpp                     |  2 +-
 src/Functions/space.cpp                       |  2 +-
 src/Functions/sqrt.cpp                        |  2 +-
 src/Functions/structureToFormatSchema.cpp     |  6 +-
 src/Functions/substring.cpp                   | 10 +--
 src/Functions/substringIndex.cpp              |  2 +-
 src/Functions/synonyms.cpp                    |  2 +-
 src/Functions/tan.cpp                         |  2 +-
 src/Functions/tanh.cpp                        |  2 +-
 src/Functions/timestamp.cpp                   |  2 +-
 src/Functions/toCustomWeek.cpp                |  4 +-
 src/Functions/toDayOfMonth.cpp                |  4 +-
 src/Functions/toDayOfWeek.cpp                 |  2 +-
 src/Functions/toDayOfYear.cpp                 |  2 +-
 src/Functions/toDaysSinceYearZero.cpp         |  2 +-
 src/Functions/toDecimalString.cpp             |  2 +-
 src/Functions/toHour.cpp                      |  2 +-
 src/Functions/toLastDayOfMonth.cpp            |  2 +-
 src/Functions/toMillisecond.cpp               |  2 +-
 src/Functions/toMinute.cpp                    |  2 +-
 src/Functions/toMonth.cpp                     |  2 +-
 src/Functions/toQuarter.cpp                   |  2 +-
 src/Functions/toSecond.cpp                    |  2 +-
 src/Functions/toYear.cpp                      |  2 +-
 src/Functions/today.cpp                       |  4 +-
 src/Functions/upper.cpp                       |  4 +-
 src/Functions/vectorFunctions.cpp             | 34 +++++-----
 src/Functions/widthBucket.cpp                 |  2 +-
 src/Processors/Transforms/WindowTransform.cpp | 12 ++--
 src/TableFunctions/TableFunctionFactory.cpp   |  6 +-
 src/TableFunctions/TableFunctionFactory.h     |  4 +-
 src/TableFunctions/TableFunctionFormat.cpp    |  2 +-
 src/TableFunctions/TableFunctionValues.cpp    |  2 +-
 167 files changed, 355 insertions(+), 365 deletions(-)

diff --git a/src/AggregateFunctions/AggregateFunctionAnalysisOfVariance.cpp b/src/AggregateFunctions/AggregateFunctionAnalysisOfVariance.cpp
index 934a8dffd90..5d833796510 100644
--- a/src/AggregateFunctions/AggregateFunctionAnalysisOfVariance.cpp
+++ b/src/AggregateFunctions/AggregateFunctionAnalysisOfVariance.cpp
@@ -118,10 +118,10 @@ AggregateFunctionPtr createAggregateFunctionAnalysisOfVariance(const std::string
 void registerAggregateFunctionAnalysisOfVariance(AggregateFunctionFactory & factory)
 {
     AggregateFunctionProperties properties = { .is_order_dependent = false };
-    factory.registerFunction("analysisOfVariance", {createAggregateFunctionAnalysisOfVariance, properties}, AggregateFunctionFactory::CaseInsensitive);
+    factory.registerFunction("analysisOfVariance", {createAggregateFunctionAnalysisOfVariance, properties}, AggregateFunctionFactory::Case::Insensitive);
 
     /// This is widely used term
-    factory.registerAlias("anova", "analysisOfVariance", AggregateFunctionFactory::CaseInsensitive);
+    factory.registerAlias("anova", "analysisOfVariance", AggregateFunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/AggregateFunctions/AggregateFunctionAny.cpp b/src/AggregateFunctions/AggregateFunctionAny.cpp
index f727ab04aa9..2bcee0fdd5f 100644
--- a/src/AggregateFunctions/AggregateFunctionAny.cpp
+++ b/src/AggregateFunctions/AggregateFunctionAny.cpp
@@ -361,9 +361,9 @@ void registerAggregateFunctionsAny(AggregateFunctionFactory & factory)
     AggregateFunctionProperties default_properties = {.returns_default_when_only_null = false, .is_order_dependent = true};
 
     factory.registerFunction("any", {createAggregateFunctionAny, default_properties});
-    factory.registerAlias("any_value", "any", AggregateFunctionFactory::CaseInsensitive);
-    factory.registerAlias("first_value", "any", AggregateFunctionFactory::CaseInsensitive);
+    factory.registerAlias("any_value", "any", AggregateFunctionFactory::Case::Insensitive);
+    factory.registerAlias("first_value", "any", AggregateFunctionFactory::Case::Insensitive);
     factory.registerFunction("anyLast", {createAggregateFunctionAnyLast, default_properties});
-    factory.registerAlias("last_value", "anyLast", AggregateFunctionFactory::CaseInsensitive);
+    factory.registerAlias("last_value", "anyLast", AggregateFunctionFactory::Case::Insensitive);
 }
 }
diff --git a/src/AggregateFunctions/AggregateFunctionAnyRespectNulls.cpp b/src/AggregateFunctions/AggregateFunctionAnyRespectNulls.cpp
index 7275409c151..0b6642bffac 100644
--- a/src/AggregateFunctions/AggregateFunctionAnyRespectNulls.cpp
+++ b/src/AggregateFunctions/AggregateFunctionAnyRespectNulls.cpp
@@ -221,11 +221,11 @@ void registerAggregateFunctionsAnyRespectNulls(AggregateFunctionFactory & factor
         = {.returns_default_when_only_null = false, .is_order_dependent = true, .is_window_function = true};
 
     factory.registerFunction("any_respect_nulls", {createAggregateFunctionAnyRespectNulls, default_properties_for_respect_nulls});
-    factory.registerAlias("any_value_respect_nulls", "any_respect_nulls", AggregateFunctionFactory::CaseInsensitive);
-    factory.registerAlias("first_value_respect_nulls", "any_respect_nulls", AggregateFunctionFactory::CaseInsensitive);
+    factory.registerAlias("any_value_respect_nulls", "any_respect_nulls", AggregateFunctionFactory::Case::Insensitive);
+    factory.registerAlias("first_value_respect_nulls", "any_respect_nulls", AggregateFunctionFactory::Case::Insensitive);
 
     factory.registerFunction("anyLast_respect_nulls", {createAggregateFunctionAnyLastRespectNulls, default_properties_for_respect_nulls});
-    factory.registerAlias("last_value_respect_nulls", "anyLast_respect_nulls", AggregateFunctionFactory::CaseInsensitive);
+    factory.registerAlias("last_value_respect_nulls", "anyLast_respect_nulls", AggregateFunctionFactory::Case::Insensitive);
 
     /// Must happen after registering any and anyLast
     factory.registerNullsActionTransformation("any", "any_respect_nulls");
diff --git a/src/AggregateFunctions/AggregateFunctionAvg.cpp b/src/AggregateFunctions/AggregateFunctionAvg.cpp
index ac6d2cf7fb4..57b14921c99 100644
--- a/src/AggregateFunctions/AggregateFunctionAvg.cpp
+++ b/src/AggregateFunctions/AggregateFunctionAvg.cpp
@@ -46,6 +46,6 @@ AggregateFunctionPtr createAggregateFunctionAvg(const std::string & name, const
 
 void registerAggregateFunctionAvg(AggregateFunctionFactory & factory)
 {
-    factory.registerFunction("avg", createAggregateFunctionAvg, AggregateFunctionFactory::CaseInsensitive);
+    factory.registerFunction("avg", createAggregateFunctionAvg, AggregateFunctionFactory::Case::Insensitive);
 }
 }
diff --git a/src/AggregateFunctions/AggregateFunctionBitwise.cpp b/src/AggregateFunctions/AggregateFunctionBitwise.cpp
index 619251552e4..ecced5f3e32 100644
--- a/src/AggregateFunctions/AggregateFunctionBitwise.cpp
+++ b/src/AggregateFunctions/AggregateFunctionBitwise.cpp
@@ -234,9 +234,9 @@ void registerAggregateFunctionsBitwise(AggregateFunctionFactory & factory)
     factory.registerFunction("groupBitXor", createAggregateFunctionBitwise<AggregateFunctionGroupBitXorData>);
 
     /// Aliases for compatibility with MySQL.
-    factory.registerAlias("BIT_OR", "groupBitOr", AggregateFunctionFactory::CaseInsensitive);
-    factory.registerAlias("BIT_AND", "groupBitAnd", AggregateFunctionFactory::CaseInsensitive);
-    factory.registerAlias("BIT_XOR", "groupBitXor", AggregateFunctionFactory::CaseInsensitive);
+    factory.registerAlias("BIT_OR", "groupBitOr", AggregateFunctionFactory::Case::Insensitive);
+    factory.registerAlias("BIT_AND", "groupBitAnd", AggregateFunctionFactory::Case::Insensitive);
+    factory.registerAlias("BIT_XOR", "groupBitXor", AggregateFunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/AggregateFunctions/AggregateFunctionCorr.cpp b/src/AggregateFunctions/AggregateFunctionCorr.cpp
index 2e8ff3af933..02d3a4aa912 100644
--- a/src/AggregateFunctions/AggregateFunctionCorr.cpp
+++ b/src/AggregateFunctions/AggregateFunctionCorr.cpp
@@ -9,7 +9,7 @@ template <typename T1, typename T2> using AggregateFunctionCorr = AggregateFunct
 
 void registerAggregateFunctionsStatisticsCorr(AggregateFunctionFactory & factory)
 {
-    factory.registerFunction("corr", createAggregateFunctionStatisticsBinary<AggregateFunctionCorr, StatisticsFunctionKind::corr>, AggregateFunctionFactory::CaseInsensitive);
+    factory.registerFunction("corr", createAggregateFunctionStatisticsBinary<AggregateFunctionCorr, StatisticsFunctionKind::corr>, AggregateFunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/AggregateFunctions/AggregateFunctionCount.cpp b/src/AggregateFunctions/AggregateFunctionCount.cpp
index 25f991ab693..ad3aee90c37 100644
--- a/src/AggregateFunctions/AggregateFunctionCount.cpp
+++ b/src/AggregateFunctions/AggregateFunctionCount.cpp
@@ -37,7 +37,7 @@ AggregateFunctionPtr createAggregateFunctionCount(const std::string & name, cons
 void registerAggregateFunctionCount(AggregateFunctionFactory & factory)
 {
     AggregateFunctionProperties properties = { .returns_default_when_only_null = true, .is_order_dependent = false };
-    factory.registerFunction("count", {createAggregateFunctionCount, properties}, AggregateFunctionFactory::CaseInsensitive);
+    factory.registerFunction("count", {createAggregateFunctionCount, properties}, AggregateFunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/AggregateFunctions/AggregateFunctionCovar.cpp b/src/AggregateFunctions/AggregateFunctionCovar.cpp
index 9645685483f..e4877a0aed3 100644
--- a/src/AggregateFunctions/AggregateFunctionCovar.cpp
+++ b/src/AggregateFunctions/AggregateFunctionCovar.cpp
@@ -13,8 +13,8 @@ void registerAggregateFunctionsStatisticsCovar(AggregateFunctionFactory & factor
     factory.registerFunction("covarPop", createAggregateFunctionStatisticsBinary<AggregateFunctionCovar, StatisticsFunctionKind::covarPop>);
 
     /// Synonyms for compatibility.
-    factory.registerAlias("COVAR_SAMP", "covarSamp", AggregateFunctionFactory::CaseInsensitive);
-    factory.registerAlias("COVAR_POP", "covarPop", AggregateFunctionFactory::CaseInsensitive);
+    factory.registerAlias("COVAR_SAMP", "covarSamp", AggregateFunctionFactory::Case::Insensitive);
+    factory.registerAlias("COVAR_POP", "covarPop", AggregateFunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/AggregateFunctions/AggregateFunctionFactory.cpp b/src/AggregateFunctions/AggregateFunctionFactory.cpp
index b5c6440a69c..082fa11ca8a 100644
--- a/src/AggregateFunctions/AggregateFunctionFactory.cpp
+++ b/src/AggregateFunctions/AggregateFunctionFactory.cpp
@@ -29,7 +29,7 @@ const String & getAggregateFunctionCanonicalNameIfAny(const String & name)
     return AggregateFunctionFactory::instance().getCanonicalNameIfAny(name);
 }
 
-void AggregateFunctionFactory::registerFunction(const String & name, Value creator_with_properties, CaseSensitiveness case_sensitiveness)
+void AggregateFunctionFactory::registerFunction(const String & name, Value creator_with_properties, Case case_sensitiveness)
 {
     if (creator_with_properties.creator == nullptr)
         throw Exception(ErrorCodes::LOGICAL_ERROR, "AggregateFunctionFactory: "
@@ -39,7 +39,7 @@ void AggregateFunctionFactory::registerFunction(const String & name, Value creat
         throw Exception(ErrorCodes::LOGICAL_ERROR, "AggregateFunctionFactory: the aggregate function name '{}' is not unique",
             name);
 
-    if (case_sensitiveness == CaseInsensitive)
+    if (case_sensitiveness == Case::Insensitive)
     {
         auto key = Poco::toLower(name);
         if (!case_insensitive_aggregate_functions.emplace(key, creator_with_properties).second)
diff --git a/src/AggregateFunctions/AggregateFunctionFactory.h b/src/AggregateFunctions/AggregateFunctionFactory.h
index b1dc422fcb0..a5fa3424543 100644
--- a/src/AggregateFunctions/AggregateFunctionFactory.h
+++ b/src/AggregateFunctions/AggregateFunctionFactory.h
@@ -60,7 +60,7 @@ public:
     void registerFunction(
         const String & name,
         Value creator,
-        CaseSensitiveness case_sensitiveness = CaseSensitive);
+        Case case_sensitiveness = Case::Sensitive);
 
     /// Register how to transform from one aggregate function to other based on NullsAction
     /// Registers them both ways:
diff --git a/src/AggregateFunctions/AggregateFunctionGroupArray.cpp b/src/AggregateFunctions/AggregateFunctionGroupArray.cpp
index 16907e0f24f..7034e6373b1 100644
--- a/src/AggregateFunctions/AggregateFunctionGroupArray.cpp
+++ b/src/AggregateFunctions/AggregateFunctionGroupArray.cpp
@@ -840,8 +840,8 @@ void registerAggregateFunctionGroupArray(AggregateFunctionFactory & factory)
     AggregateFunctionProperties properties = { .returns_default_when_only_null = false, .is_order_dependent = true };
 
     factory.registerFunction("groupArray", { createAggregateFunctionGroupArray<false>, properties });
-    factory.registerAlias("array_agg", "groupArray", AggregateFunctionFactory::CaseInsensitive);
-    factory.registerAliasUnchecked("array_concat_agg", "groupArrayArray", AggregateFunctionFactory::CaseInsensitive);
+    factory.registerAlias("array_agg", "groupArray", AggregateFunctionFactory::Case::Insensitive);
+    factory.registerAliasUnchecked("array_concat_agg", "groupArrayArray", AggregateFunctionFactory::Case::Insensitive);
     factory.registerFunction("groupArraySample", { createAggregateFunctionGroupArraySample, properties });
     factory.registerFunction("groupArrayLast", { createAggregateFunctionGroupArray<true>, properties });
 }
diff --git a/src/AggregateFunctions/AggregateFunctionGroupConcat.cpp b/src/AggregateFunctions/AggregateFunctionGroupConcat.cpp
index 1c059dc52aa..5494ef74705 100644
--- a/src/AggregateFunctions/AggregateFunctionGroupConcat.cpp
+++ b/src/AggregateFunctions/AggregateFunctionGroupConcat.cpp
@@ -277,7 +277,7 @@ void registerAggregateFunctionGroupConcat(AggregateFunctionFactory & factory)
     AggregateFunctionProperties properties = { .returns_default_when_only_null = false, .is_order_dependent = true };
 
     factory.registerFunction("groupConcat", { createAggregateFunctionGroupConcat, properties });
-    factory.registerAlias("group_concat", "groupConcat", AggregateFunctionFactory::CaseInsensitive);
+    factory.registerAlias("group_concat", "groupConcat", AggregateFunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/AggregateFunctions/AggregateFunctionKolmogorovSmirnovTest.cpp b/src/AggregateFunctions/AggregateFunctionKolmogorovSmirnovTest.cpp
index 736cca11f1e..04eebe9f485 100644
--- a/src/AggregateFunctions/AggregateFunctionKolmogorovSmirnovTest.cpp
+++ b/src/AggregateFunctions/AggregateFunctionKolmogorovSmirnovTest.cpp
@@ -350,7 +350,7 @@ AggregateFunctionPtr createAggregateFunctionKolmogorovSmirnovTest(
 
 void registerAggregateFunctionKolmogorovSmirnovTest(AggregateFunctionFactory & factory)
 {
-    factory.registerFunction("kolmogorovSmirnovTest", createAggregateFunctionKolmogorovSmirnovTest, AggregateFunctionFactory::CaseInsensitive);
+    factory.registerFunction("kolmogorovSmirnovTest", createAggregateFunctionKolmogorovSmirnovTest, AggregateFunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/AggregateFunctions/AggregateFunctionSecondMoment.cpp b/src/AggregateFunctions/AggregateFunctionSecondMoment.cpp
index 80fbe2511d9..4aa6a0a4429 100644
--- a/src/AggregateFunctions/AggregateFunctionSecondMoment.cpp
+++ b/src/AggregateFunctions/AggregateFunctionSecondMoment.cpp
@@ -15,11 +15,11 @@ void registerAggregateFunctionsStatisticsSecondMoment(AggregateFunctionFactory &
     factory.registerFunction("stddevPop", createAggregateFunctionStatisticsUnary<AggregateFunctionSecondMoment, StatisticsFunctionKind::stddevPop>);
 
     /// Synonyms for compatibility.
-    factory.registerAlias("VAR_SAMP", "varSamp", AggregateFunctionFactory::CaseInsensitive);
-    factory.registerAlias("VAR_POP", "varPop", AggregateFunctionFactory::CaseInsensitive);
-    factory.registerAlias("STDDEV_SAMP", "stddevSamp", AggregateFunctionFactory::CaseInsensitive);
-    factory.registerAlias("STDDEV_POP", "stddevPop", AggregateFunctionFactory::CaseInsensitive);
-    factory.registerAlias("STD", "stddevPop", AggregateFunctionFactory::CaseInsensitive);
+    factory.registerAlias("VAR_SAMP", "varSamp", AggregateFunctionFactory::Case::Insensitive);
+    factory.registerAlias("VAR_POP", "varPop", AggregateFunctionFactory::Case::Insensitive);
+    factory.registerAlias("STDDEV_SAMP", "stddevSamp", AggregateFunctionFactory::Case::Insensitive);
+    factory.registerAlias("STDDEV_POP", "stddevPop", AggregateFunctionFactory::Case::Insensitive);
+    factory.registerAlias("STD", "stddevPop", AggregateFunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/AggregateFunctions/AggregateFunctionSum.cpp b/src/AggregateFunctions/AggregateFunctionSum.cpp
index e393cb6dd38..910e49f388d 100644
--- a/src/AggregateFunctions/AggregateFunctionSum.cpp
+++ b/src/AggregateFunctions/AggregateFunctionSum.cpp
@@ -72,7 +72,7 @@ AggregateFunctionPtr createAggregateFunctionSum(const std::string & name, const
 
 void registerAggregateFunctionSum(AggregateFunctionFactory & factory)
 {
-    factory.registerFunction("sum", createAggregateFunctionSum<AggregateFunctionSumSimple>, AggregateFunctionFactory::CaseInsensitive);
+    factory.registerFunction("sum", createAggregateFunctionSum<AggregateFunctionSumSimple>, AggregateFunctionFactory::Case::Insensitive);
     factory.registerFunction("sumWithOverflow", createAggregateFunctionSum<AggregateFunctionSumWithOverflow>);
     factory.registerFunction("sumKahan", createAggregateFunctionSum<AggregateFunctionSumKahan>);
 }
diff --git a/src/AggregateFunctions/AggregateFunctionTopK.cpp b/src/AggregateFunctions/AggregateFunctionTopK.cpp
index 26f756abe18..f949f6b7e4a 100644
--- a/src/AggregateFunctions/AggregateFunctionTopK.cpp
+++ b/src/AggregateFunctions/AggregateFunctionTopK.cpp
@@ -535,9 +535,9 @@ void registerAggregateFunctionTopK(AggregateFunctionFactory & factory)
 
     factory.registerFunction("topK", { createAggregateFunctionTopK<false, false>, properties });
     factory.registerFunction("topKWeighted", { createAggregateFunctionTopK<true, false>, properties });
-    factory.registerFunction("approx_top_k", { createAggregateFunctionTopK<false, true>, properties }, AggregateFunctionFactory::CaseInsensitive);
-    factory.registerFunction("approx_top_sum", { createAggregateFunctionTopK<true, true>, properties }, AggregateFunctionFactory::CaseInsensitive);
-    factory.registerAlias("approx_top_count", "approx_top_k", AggregateFunctionFactory::CaseInsensitive);
+    factory.registerFunction("approx_top_k", { createAggregateFunctionTopK<false, true>, properties }, AggregateFunctionFactory::Case::Insensitive);
+    factory.registerFunction("approx_top_sum", { createAggregateFunctionTopK<true, true>, properties }, AggregateFunctionFactory::Case::Insensitive);
+    factory.registerAlias("approx_top_count", "approx_top_k", AggregateFunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/AggregateFunctions/AggregateFunctionsMinMax.cpp b/src/AggregateFunctions/AggregateFunctionsMinMax.cpp
index 03e21c15a75..5fa9a4ff5d1 100644
--- a/src/AggregateFunctions/AggregateFunctionsMinMax.cpp
+++ b/src/AggregateFunctions/AggregateFunctionsMinMax.cpp
@@ -195,8 +195,8 @@ AggregateFunctionPtr createAggregateFunctionMinMax(
 
 void registerAggregateFunctionsMinMax(AggregateFunctionFactory & factory)
 {
-    factory.registerFunction("min", createAggregateFunctionMinMax<true>, AggregateFunctionFactory::CaseInsensitive);
-    factory.registerFunction("max", createAggregateFunctionMinMax<false>, AggregateFunctionFactory::CaseInsensitive);
+    factory.registerFunction("min", createAggregateFunctionMinMax<true>, AggregateFunctionFactory::Case::Insensitive);
+    factory.registerFunction("max", createAggregateFunctionMinMax<false>, AggregateFunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Common/IFactoryWithAliases.h b/src/Common/IFactoryWithAliases.h
index 74d4b6e3bcb..431e5c7b733 100644
--- a/src/Common/IFactoryWithAliases.h
+++ b/src/Common/IFactoryWithAliases.h
@@ -39,16 +39,16 @@ protected:
 
 public:
     /// For compatibility with SQL, it's possible to specify that certain function name is case insensitive.
-    enum CaseSensitiveness
+    enum Case
     {
-        CaseSensitive,
-        CaseInsensitive
+        Sensitive,
+        Insensitive
     };
 
     /** Register additional name for value
       * real_name have to be already registered.
       */
-    void registerAlias(const String & alias_name, const String & real_name, CaseSensitiveness case_sensitiveness = CaseSensitive)
+    void registerAlias(const String & alias_name, const String & real_name, Case case_sensitiveness = Sensitive)
     {
         const auto & creator_map = getMap();
         const auto & case_insensitive_creator_map = getCaseInsensitiveMap();
@@ -66,12 +66,12 @@ public:
     }
 
     /// We need sure the real_name exactly exists when call the function directly.
-    void registerAliasUnchecked(const String & alias_name, const String & real_name, CaseSensitiveness case_sensitiveness = CaseSensitive)
+    void registerAliasUnchecked(const String & alias_name, const String & real_name, Case case_sensitiveness = Sensitive)
     {
         String alias_name_lowercase = Poco::toLower(alias_name);
         const String factory_name = getFactoryName();
 
-        if (case_sensitiveness == CaseInsensitive)
+        if (case_sensitiveness == Insensitive)
         {
             if (!case_insensitive_aliases.emplace(alias_name_lowercase, real_name).second)
                 throw Exception(ErrorCodes::LOGICAL_ERROR, "{}: case insensitive alias name '{}' is not unique", factory_name, alias_name);
diff --git a/src/DataTypes/DataTypeDate.cpp b/src/DataTypes/DataTypeDate.cpp
index ee4b0065e59..0a7aa7deac6 100644
--- a/src/DataTypes/DataTypeDate.cpp
+++ b/src/DataTypes/DataTypeDate.cpp
@@ -17,7 +17,7 @@ SerializationPtr DataTypeDate::doGetDefaultSerialization() const
 
 void registerDataTypeDate(DataTypeFactory & factory)
 {
-    factory.registerSimpleDataType("Date", [] { return DataTypePtr(std::make_shared<DataTypeDate>()); }, DataTypeFactory::CaseInsensitive);
+    factory.registerSimpleDataType("Date", [] { return DataTypePtr(std::make_shared<DataTypeDate>()); }, DataTypeFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/DataTypes/DataTypeDate32.cpp b/src/DataTypes/DataTypeDate32.cpp
index 343e498d303..b2b8e7c0c1c 100644
--- a/src/DataTypes/DataTypeDate32.cpp
+++ b/src/DataTypes/DataTypeDate32.cpp
@@ -24,7 +24,7 @@ Field DataTypeDate32::getDefault() const
 void registerDataTypeDate32(DataTypeFactory & factory)
 {
     factory.registerSimpleDataType(
-        "Date32", [] { return DataTypePtr(std::make_shared<DataTypeDate32>()); }, DataTypeFactory::CaseInsensitive);
+        "Date32", [] { return DataTypePtr(std::make_shared<DataTypeDate32>()); }, DataTypeFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/DataTypes/DataTypeDomainBool.cpp b/src/DataTypes/DataTypeDomainBool.cpp
index 3d19b6262d8..30dbba2d8c0 100644
--- a/src/DataTypes/DataTypeDomainBool.cpp
+++ b/src/DataTypes/DataTypeDomainBool.cpp
@@ -15,8 +15,8 @@ void registerDataTypeDomainBool(DataTypeFactory & factory)
                 std::make_unique<DataTypeCustomFixedName>("Bool"), std::make_unique<SerializationBool>(type->getDefaultSerialization())));
     });
 
-    factory.registerAlias("bool", "Bool", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("boolean", "Bool", DataTypeFactory::CaseInsensitive);
+    factory.registerAlias("bool", "Bool", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("boolean", "Bool", DataTypeFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/DataTypes/DataTypeEnum.cpp b/src/DataTypes/DataTypeEnum.cpp
index a1d5e4b39b7..08e0c0d2045 100644
--- a/src/DataTypes/DataTypeEnum.cpp
+++ b/src/DataTypes/DataTypeEnum.cpp
@@ -318,7 +318,7 @@ void registerDataTypeEnum(DataTypeFactory & factory)
     factory.registerDataType("Enum", create);
 
     /// MySQL
-    factory.registerAlias("ENUM", "Enum", DataTypeFactory::CaseInsensitive);
+    factory.registerAlias("ENUM", "Enum", DataTypeFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/DataTypes/DataTypeFactory.cpp b/src/DataTypes/DataTypeFactory.cpp
index 07dc4395bfe..af37cde2846 100644
--- a/src/DataTypes/DataTypeFactory.cpp
+++ b/src/DataTypes/DataTypeFactory.cpp
@@ -175,7 +175,7 @@ DataTypePtr DataTypeFactory::getCustom(DataTypeCustomDescPtr customization) cons
 }
 
 
-void DataTypeFactory::registerDataType(const String & family_name, Value creator, CaseSensitiveness case_sensitiveness)
+void DataTypeFactory::registerDataType(const String & family_name, Value creator, Case case_sensitiveness)
 {
     if (creator == nullptr)
         throw Exception(ErrorCodes::LOGICAL_ERROR, "DataTypeFactory: the data type family {} has been provided  a null constructor", family_name);
@@ -189,12 +189,12 @@ void DataTypeFactory::registerDataType(const String & family_name, Value creator
         throw Exception(ErrorCodes::LOGICAL_ERROR, "DataTypeFactory: the data type family name '{}' is not unique",
             family_name);
 
-    if (case_sensitiveness == CaseInsensitive
+    if (case_sensitiveness == Case::Insensitive
         && !case_insensitive_data_types.emplace(family_name_lowercase, creator).second)
         throw Exception(ErrorCodes::LOGICAL_ERROR, "DataTypeFactory: the case insensitive data type family name '{}' is not unique", family_name);
 }
 
-void DataTypeFactory::registerSimpleDataType(const String & name, SimpleCreator creator, CaseSensitiveness case_sensitiveness)
+void DataTypeFactory::registerSimpleDataType(const String & name, SimpleCreator creator, Case case_sensitiveness)
 {
     if (creator == nullptr)
         throw Exception(ErrorCodes::LOGICAL_ERROR, "DataTypeFactory: the data type {} has been provided  a null constructor",
@@ -208,7 +208,7 @@ void DataTypeFactory::registerSimpleDataType(const String & name, SimpleCreator
     }, case_sensitiveness);
 }
 
-void DataTypeFactory::registerDataTypeCustom(const String & family_name, CreatorWithCustom creator, CaseSensitiveness case_sensitiveness)
+void DataTypeFactory::registerDataTypeCustom(const String & family_name, CreatorWithCustom creator, Case case_sensitiveness)
 {
     registerDataType(family_name, [creator](const ASTPtr & ast)
     {
@@ -219,7 +219,7 @@ void DataTypeFactory::registerDataTypeCustom(const String & family_name, Creator
     }, case_sensitiveness);
 }
 
-void DataTypeFactory::registerSimpleDataTypeCustom(const String & name, SimpleCreatorWithCustom creator, CaseSensitiveness case_sensitiveness)
+void DataTypeFactory::registerSimpleDataTypeCustom(const String & name, SimpleCreatorWithCustom creator, Case case_sensitiveness)
 {
     registerDataTypeCustom(name, [name, creator](const ASTPtr & ast)
     {
diff --git a/src/DataTypes/DataTypeFactory.h b/src/DataTypes/DataTypeFactory.h
index 86e0203358d..edba9886d1c 100644
--- a/src/DataTypes/DataTypeFactory.h
+++ b/src/DataTypes/DataTypeFactory.h
@@ -41,16 +41,16 @@ public:
     DataTypePtr tryGet(const ASTPtr & ast) const;
 
     /// Register a type family by its name.
-    void registerDataType(const String & family_name, Value creator, CaseSensitiveness case_sensitiveness = CaseSensitive);
+    void registerDataType(const String & family_name, Value creator, Case case_sensitiveness = Case::Sensitive);
 
     /// Register a simple data type, that have no parameters.
-    void registerSimpleDataType(const String & name, SimpleCreator creator, CaseSensitiveness case_sensitiveness = CaseSensitive);
+    void registerSimpleDataType(const String & name, SimpleCreator creator, Case case_sensitiveness = Case::Sensitive);
 
     /// Register a customized type family
-    void registerDataTypeCustom(const String & family_name, CreatorWithCustom creator, CaseSensitiveness case_sensitiveness = CaseSensitive);
+    void registerDataTypeCustom(const String & family_name, CreatorWithCustom creator, Case case_sensitiveness = Case::Sensitive);
 
     /// Register a simple customized data type
-    void registerSimpleDataTypeCustom(const String & name, SimpleCreatorWithCustom creator, CaseSensitiveness case_sensitiveness = CaseSensitive);
+    void registerSimpleDataTypeCustom(const String & name, SimpleCreatorWithCustom creator, Case case_sensitiveness = Case::Sensitive);
 
 private:
     template <bool nullptr_on_error>
diff --git a/src/DataTypes/DataTypeFixedString.cpp b/src/DataTypes/DataTypeFixedString.cpp
index 85af59e852d..080ff8826a5 100644
--- a/src/DataTypes/DataTypeFixedString.cpp
+++ b/src/DataTypes/DataTypeFixedString.cpp
@@ -64,7 +64,7 @@ void registerDataTypeFixedString(DataTypeFactory & factory)
     factory.registerDataType("FixedString", create);
 
     /// Compatibility alias.
-    factory.registerAlias("BINARY", "FixedString", DataTypeFactory::CaseInsensitive);
+    factory.registerAlias("BINARY", "FixedString", DataTypeFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/DataTypes/DataTypeIPv4andIPv6.cpp b/src/DataTypes/DataTypeIPv4andIPv6.cpp
index 4c0b45f472a..de11cc50107 100644
--- a/src/DataTypes/DataTypeIPv4andIPv6.cpp
+++ b/src/DataTypes/DataTypeIPv4andIPv6.cpp
@@ -9,9 +9,9 @@ namespace DB
 void registerDataTypeIPv4andIPv6(DataTypeFactory & factory)
 {
     factory.registerSimpleDataType("IPv4", [] { return DataTypePtr(std::make_shared<DataTypeIPv4>()); });
-    factory.registerAlias("INET4", "IPv4", DataTypeFactory::CaseInsensitive);
+    factory.registerAlias("INET4", "IPv4", DataTypeFactory::Case::Insensitive);
     factory.registerSimpleDataType("IPv6", [] { return DataTypePtr(std::make_shared<DataTypeIPv6>()); });
-    factory.registerAlias("INET6", "IPv6", DataTypeFactory::CaseInsensitive);
+    factory.registerAlias("INET6", "IPv6", DataTypeFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/DataTypes/DataTypeObject.cpp b/src/DataTypes/DataTypeObject.cpp
index 720436d0e0d..5636a46373f 100644
--- a/src/DataTypes/DataTypeObject.cpp
+++ b/src/DataTypes/DataTypeObject.cpp
@@ -76,7 +76,7 @@ void registerDataTypeObject(DataTypeFactory & factory)
     factory.registerDataType("Object", create);
     factory.registerSimpleDataType("JSON",
         [] { return std::make_shared<DataTypeObject>("JSON", false); },
-        DataTypeFactory::CaseInsensitive);
+        DataTypeFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/DataTypes/DataTypeString.cpp b/src/DataTypes/DataTypeString.cpp
index 95e49420009..ca65fb42cc8 100644
--- a/src/DataTypes/DataTypeString.cpp
+++ b/src/DataTypes/DataTypeString.cpp
@@ -62,38 +62,38 @@ void registerDataTypeString(DataTypeFactory & factory)
 
     /// These synonims are added for compatibility.
 
-    factory.registerAlias("CHAR", "String", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("NCHAR", "String", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("CHARACTER", "String", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("VARCHAR", "String", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("NVARCHAR", "String", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("VARCHAR2", "String", DataTypeFactory::CaseInsensitive); /// Oracle
-    factory.registerAlias("TEXT", "String", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("TINYTEXT", "String", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("MEDIUMTEXT", "String", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("LONGTEXT", "String", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("BLOB", "String", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("CLOB", "String", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("TINYBLOB", "String", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("MEDIUMBLOB", "String", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("LONGBLOB", "String", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("BYTEA", "String", DataTypeFactory::CaseInsensitive); /// PostgreSQL
+    factory.registerAlias("CHAR", "String", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("NCHAR", "String", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("CHARACTER", "String", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("VARCHAR", "String", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("NVARCHAR", "String", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("VARCHAR2", "String", DataTypeFactory::Case::Insensitive); /// Oracle
+    factory.registerAlias("TEXT", "String", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("TINYTEXT", "String", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("MEDIUMTEXT", "String", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("LONGTEXT", "String", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("BLOB", "String", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("CLOB", "String", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("TINYBLOB", "String", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("MEDIUMBLOB", "String", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("LONGBLOB", "String", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("BYTEA", "String", DataTypeFactory::Case::Insensitive); /// PostgreSQL
 
-    factory.registerAlias("CHARACTER LARGE OBJECT", "String", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("CHARACTER VARYING", "String", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("CHAR LARGE OBJECT", "String", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("CHAR VARYING", "String", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("NATIONAL CHAR", "String", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("NATIONAL CHARACTER", "String", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("NATIONAL CHARACTER LARGE OBJECT", "String", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("NATIONAL CHARACTER VARYING", "String", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("NATIONAL CHAR VARYING", "String", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("NCHAR VARYING", "String", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("NCHAR LARGE OBJECT", "String", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("BINARY LARGE OBJECT", "String", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("BINARY VARYING", "String", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("VARBINARY", "String", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("GEOMETRY", "String", DataTypeFactory::CaseInsensitive); //mysql
+    factory.registerAlias("CHARACTER LARGE OBJECT", "String", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("CHARACTER VARYING", "String", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("CHAR LARGE OBJECT", "String", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("CHAR VARYING", "String", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("NATIONAL CHAR", "String", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("NATIONAL CHARACTER", "String", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("NATIONAL CHARACTER LARGE OBJECT", "String", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("NATIONAL CHARACTER VARYING", "String", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("NATIONAL CHAR VARYING", "String", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("NCHAR VARYING", "String", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("NCHAR LARGE OBJECT", "String", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("BINARY LARGE OBJECT", "String", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("BINARY VARYING", "String", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("VARBINARY", "String", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("GEOMETRY", "String", DataTypeFactory::Case::Insensitive); //mysql
 
 }
 }
diff --git a/src/DataTypes/DataTypesDecimal.cpp b/src/DataTypes/DataTypesDecimal.cpp
index 77a7a3e7237..a427fd0717a 100644
--- a/src/DataTypes/DataTypesDecimal.cpp
+++ b/src/DataTypes/DataTypesDecimal.cpp
@@ -364,15 +364,15 @@ template class DataTypeDecimal<Decimal256>;
 
 void registerDataTypeDecimal(DataTypeFactory & factory)
 {
-    factory.registerDataType("Decimal32", createExact<Decimal32>, DataTypeFactory::CaseInsensitive);
-    factory.registerDataType("Decimal64", createExact<Decimal64>, DataTypeFactory::CaseInsensitive);
-    factory.registerDataType("Decimal128", createExact<Decimal128>, DataTypeFactory::CaseInsensitive);
-    factory.registerDataType("Decimal256", createExact<Decimal256>, DataTypeFactory::CaseInsensitive);
+    factory.registerDataType("Decimal32", createExact<Decimal32>, DataTypeFactory::Case::Insensitive);
+    factory.registerDataType("Decimal64", createExact<Decimal64>, DataTypeFactory::Case::Insensitive);
+    factory.registerDataType("Decimal128", createExact<Decimal128>, DataTypeFactory::Case::Insensitive);
+    factory.registerDataType("Decimal256", createExact<Decimal256>, DataTypeFactory::Case::Insensitive);
 
-    factory.registerDataType("Decimal", create, DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("DEC", "Decimal", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("NUMERIC", "Decimal", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("FIXED", "Decimal", DataTypeFactory::CaseInsensitive);
+    factory.registerDataType("Decimal", create, DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("DEC", "Decimal", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("NUMERIC", "Decimal", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("FIXED", "Decimal", DataTypeFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/DataTypes/DataTypesNumber.cpp b/src/DataTypes/DataTypesNumber.cpp
index 99446d24eed..72020b0a5aa 100644
--- a/src/DataTypes/DataTypesNumber.cpp
+++ b/src/DataTypes/DataTypesNumber.cpp
@@ -65,41 +65,41 @@ void registerDataTypeNumbers(DataTypeFactory & factory)
 
     /// These synonyms are added for compatibility.
 
-    factory.registerAlias("TINYINT", "Int8", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("INT1", "Int8", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("BYTE", "Int8", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("TINYINT SIGNED", "Int8", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("INT1 SIGNED", "Int8", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("SMALLINT", "Int16", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("SMALLINT SIGNED", "Int16", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("INT", "Int32", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("INTEGER", "Int32", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("MEDIUMINT", "Int32", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("MEDIUMINT SIGNED", "Int32", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("INT SIGNED", "Int32", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("INTEGER SIGNED", "Int32", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("BIGINT", "Int64", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("SIGNED", "Int64", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("BIGINT SIGNED", "Int64", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("TIME", "Int64", DataTypeFactory::CaseInsensitive);
+    factory.registerAlias("TINYINT", "Int8", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("INT1", "Int8", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("BYTE", "Int8", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("TINYINT SIGNED", "Int8", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("INT1 SIGNED", "Int8", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("SMALLINT", "Int16", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("SMALLINT SIGNED", "Int16", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("INT", "Int32", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("INTEGER", "Int32", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("MEDIUMINT", "Int32", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("MEDIUMINT SIGNED", "Int32", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("INT SIGNED", "Int32", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("INTEGER SIGNED", "Int32", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("BIGINT", "Int64", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("SIGNED", "Int64", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("BIGINT SIGNED", "Int64", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("TIME", "Int64", DataTypeFactory::Case::Insensitive);
 
-    factory.registerAlias("TINYINT UNSIGNED", "UInt8", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("INT1 UNSIGNED", "UInt8", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("SMALLINT UNSIGNED", "UInt16", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("YEAR", "UInt16", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("MEDIUMINT UNSIGNED", "UInt32", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("INT UNSIGNED", "UInt32", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("INTEGER UNSIGNED", "UInt32", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("UNSIGNED", "UInt64", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("BIGINT UNSIGNED", "UInt64", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("BIT", "UInt64", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("SET", "UInt64", DataTypeFactory::CaseInsensitive);
+    factory.registerAlias("TINYINT UNSIGNED", "UInt8", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("INT1 UNSIGNED", "UInt8", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("SMALLINT UNSIGNED", "UInt16", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("YEAR", "UInt16", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("MEDIUMINT UNSIGNED", "UInt32", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("INT UNSIGNED", "UInt32", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("INTEGER UNSIGNED", "UInt32", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("UNSIGNED", "UInt64", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("BIGINT UNSIGNED", "UInt64", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("BIT", "UInt64", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("SET", "UInt64", DataTypeFactory::Case::Insensitive);
 
-    factory.registerAlias("FLOAT", "Float32", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("REAL", "Float32", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("SINGLE", "Float32", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("DOUBLE", "Float64", DataTypeFactory::CaseInsensitive);
-    factory.registerAlias("DOUBLE PRECISION", "Float64", DataTypeFactory::CaseInsensitive);
+    factory.registerAlias("FLOAT", "Float32", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("REAL", "Float32", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("SINGLE", "Float32", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("DOUBLE", "Float64", DataTypeFactory::Case::Insensitive);
+    factory.registerAlias("DOUBLE PRECISION", "Float64", DataTypeFactory::Case::Insensitive);
 }
 
 /// Explicit template instantiations.
diff --git a/src/DataTypes/registerDataTypeDateTime.cpp b/src/DataTypes/registerDataTypeDateTime.cpp
index 802356cc108..84a52d4affb 100644
--- a/src/DataTypes/registerDataTypeDateTime.cpp
+++ b/src/DataTypes/registerDataTypeDateTime.cpp
@@ -108,11 +108,11 @@ static DataTypePtr create64(const ASTPtr & arguments)
 
 void registerDataTypeDateTime(DataTypeFactory & factory)
 {
-    factory.registerDataType("DateTime", create, DataTypeFactory::CaseInsensitive);
-    factory.registerDataType("DateTime32", create32, DataTypeFactory::CaseInsensitive);
-    factory.registerDataType("DateTime64", create64, DataTypeFactory::CaseInsensitive);
+    factory.registerDataType("DateTime", create, DataTypeFactory::Case::Insensitive);
+    factory.registerDataType("DateTime32", create32, DataTypeFactory::Case::Insensitive);
+    factory.registerDataType("DateTime64", create64, DataTypeFactory::Case::Insensitive);
 
-    factory.registerAlias("TIMESTAMP", "DateTime", DataTypeFactory::CaseInsensitive);
+    factory.registerAlias("TIMESTAMP", "DateTime", DataTypeFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/CRC.cpp b/src/Functions/CRC.cpp
index 49d6dd6fa52..0ba976669a3 100644
--- a/src/Functions/CRC.cpp
+++ b/src/Functions/CRC.cpp
@@ -150,9 +150,9 @@ using FunctionCRC64ECMA = FunctionCRC<CRC64ECMAImpl>;
 
 REGISTER_FUNCTION(CRC)
 {
-    factory.registerFunction<FunctionCRC32ZLib>({}, FunctionFactory::CaseInsensitive);
-    factory.registerFunction<FunctionCRC32IEEE>({}, FunctionFactory::CaseInsensitive);
-    factory.registerFunction<FunctionCRC64ECMA>({}, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<FunctionCRC32ZLib>({}, FunctionFactory::Case::Insensitive);
+    factory.registerFunction<FunctionCRC32IEEE>({}, FunctionFactory::Case::Insensitive);
+    factory.registerFunction<FunctionCRC64ECMA>({}, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/CastOverloadResolver.cpp b/src/Functions/CastOverloadResolver.cpp
index 10a08c6e35f..49f63073aaf 100644
--- a/src/Functions/CastOverloadResolver.cpp
+++ b/src/Functions/CastOverloadResolver.cpp
@@ -137,10 +137,10 @@ FunctionOverloadResolverPtr createInternalCastOverloadResolver(CastType type, st
 
 REGISTER_FUNCTION(CastOverloadResolvers)
 {
-    factory.registerFunction("_CAST", [](ContextPtr context){ return CastOverloadResolverImpl::create(context, CastType::nonAccurate, true, {}); }, {}, FunctionFactory::CaseInsensitive);
+    factory.registerFunction("_CAST", [](ContextPtr context){ return CastOverloadResolverImpl::create(context, CastType::nonAccurate, true, {}); }, {}, FunctionFactory::Case::Insensitive);
     /// Note: "internal" (not affected by null preserving setting) versions of accurate cast functions are unneeded.
 
-    factory.registerFunction("CAST", [](ContextPtr context){ return CastOverloadResolverImpl::create(context, CastType::nonAccurate, false, {}); }, {}, FunctionFactory::CaseInsensitive);
+    factory.registerFunction("CAST", [](ContextPtr context){ return CastOverloadResolverImpl::create(context, CastType::nonAccurate, false, {}); }, {}, FunctionFactory::Case::Insensitive);
     factory.registerFunction("accurateCast", [](ContextPtr context){ return CastOverloadResolverImpl::create(context, CastType::accurate, false, {}); }, {});
     factory.registerFunction("accurateCastOrNull", [](ContextPtr context){ return CastOverloadResolverImpl::create(context, CastType::accurateOrNull, false, {}); }, {});
 }
diff --git a/src/Functions/FunctionChar.cpp b/src/Functions/FunctionChar.cpp
index 055eb08f0c7..0ebe1442f08 100644
--- a/src/Functions/FunctionChar.cpp
+++ b/src/Functions/FunctionChar.cpp
@@ -116,7 +116,7 @@ private:
 
 REGISTER_FUNCTION(Char)
 {
-    factory.registerFunction<FunctionChar>({}, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<FunctionChar>({}, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/FunctionFQDN.cpp b/src/Functions/FunctionFQDN.cpp
index 108a96216fd..8948c948265 100644
--- a/src/Functions/FunctionFQDN.cpp
+++ b/src/Functions/FunctionFQDN.cpp
@@ -46,7 +46,7 @@ public:
 
 REGISTER_FUNCTION(FQDN)
 {
-    factory.registerFunction<FunctionFQDN>({}, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<FunctionFQDN>({}, FunctionFactory::Case::Insensitive);
     factory.registerAlias("fullHostName", "FQDN");
 }
 
diff --git a/src/Functions/FunctionFactory.cpp b/src/Functions/FunctionFactory.cpp
index de6d5a9e1c1..501cf6e725c 100644
--- a/src/Functions/FunctionFactory.cpp
+++ b/src/Functions/FunctionFactory.cpp
@@ -31,7 +31,7 @@ void FunctionFactory::registerFunction(
     const std::string & name,
     FunctionCreator creator,
     FunctionDocumentation doc,
-    CaseSensitiveness case_sensitiveness)
+    Case case_sensitiveness)
 {
     if (!functions.emplace(name, FunctionFactoryData{creator, doc}).second)
         throw Exception(ErrorCodes::LOGICAL_ERROR, "FunctionFactory: the function name '{}' is not unique", name);
@@ -41,7 +41,7 @@ void FunctionFactory::registerFunction(
         throw Exception(ErrorCodes::LOGICAL_ERROR, "FunctionFactory: the function name '{}' is already registered as alias",
                         name);
 
-    if (case_sensitiveness == CaseInsensitive)
+    if (case_sensitiveness == Case::Insensitive)
     {
         if (!case_insensitive_functions.emplace(function_name_lowercase, FunctionFactoryData{creator, doc}).second)
             throw Exception(ErrorCodes::LOGICAL_ERROR, "FunctionFactory: the case insensitive function name '{}' is not unique",
@@ -54,7 +54,7 @@ void FunctionFactory::registerFunction(
     const std::string & name,
     FunctionSimpleCreator creator,
     FunctionDocumentation doc,
-    CaseSensitiveness case_sensitiveness)
+    Case case_sensitiveness)
 {
     registerFunction(name, [my_creator = std::move(creator)](ContextPtr context)
     {
diff --git a/src/Functions/FunctionFactory.h b/src/Functions/FunctionFactory.h
index bb43d4719b8..d05e84439be 100644
--- a/src/Functions/FunctionFactory.h
+++ b/src/Functions/FunctionFactory.h
@@ -30,7 +30,7 @@ public:
     static FunctionFactory & instance();
 
     template <typename Function>
-    void registerFunction(FunctionDocumentation doc = {}, CaseSensitiveness case_sensitiveness = CaseSensitive)
+    void registerFunction(FunctionDocumentation doc = {}, Case case_sensitiveness = Case::Sensitive)
     {
         registerFunction<Function>(Function::name, std::move(doc), case_sensitiveness);
     }
@@ -56,13 +56,13 @@ public:
         const std::string & name,
         FunctionCreator creator,
         FunctionDocumentation doc = {},
-        CaseSensitiveness case_sensitiveness = CaseSensitive);
+        Case case_sensitiveness = Case::Sensitive);
 
     void registerFunction(
         const std::string & name,
         FunctionSimpleCreator creator,
         FunctionDocumentation doc = {},
-        CaseSensitiveness case_sensitiveness = CaseSensitive);
+        Case case_sensitiveness = Case::Sensitive);
 
     FunctionDocumentation getDocumentation(const std::string & name) const;
 
@@ -79,7 +79,7 @@ private:
     String getFactoryName() const override { return "FunctionFactory"; }
 
     template <typename Function>
-    void registerFunction(const std::string & name, FunctionDocumentation doc = {}, CaseSensitiveness case_sensitiveness = CaseSensitive)
+    void registerFunction(const std::string & name, FunctionDocumentation doc = {}, Case case_sensitiveness = Case::Sensitive)
     {
         registerFunction(name, &Function::create, std::move(doc), case_sensitiveness);
     }
diff --git a/src/Functions/FunctionGenerateRandomStructure.cpp b/src/Functions/FunctionGenerateRandomStructure.cpp
index 0c6f4287ecb..2bead8737fd 100644
--- a/src/Functions/FunctionGenerateRandomStructure.cpp
+++ b/src/Functions/FunctionGenerateRandomStructure.cpp
@@ -445,8 +445,7 @@ The function returns a value of type String.
                 {"with specified seed", "SELECT generateRandomStructure(1, 42)", "c1 UInt128"},
             },
             .categories{"Random"}
-        },
-        FunctionFactory::CaseSensitive);
+        });
 }
 
 }
diff --git a/src/Functions/FunctionsBinaryRepresentation.cpp b/src/Functions/FunctionsBinaryRepresentation.cpp
index ab10d402df4..f77d2f1f350 100644
--- a/src/Functions/FunctionsBinaryRepresentation.cpp
+++ b/src/Functions/FunctionsBinaryRepresentation.cpp
@@ -728,10 +728,10 @@ public:
 
 REGISTER_FUNCTION(BinaryRepr)
 {
-    factory.registerFunction<EncodeToBinaryRepresentation<HexImpl>>({}, FunctionFactory::CaseInsensitive);
-    factory.registerFunction<DecodeFromBinaryRepresentation<UnhexImpl>>({}, FunctionFactory::CaseInsensitive);
-    factory.registerFunction<EncodeToBinaryRepresentation<BinImpl>>({}, FunctionFactory::CaseInsensitive);
-    factory.registerFunction<DecodeFromBinaryRepresentation<UnbinImpl>>({}, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<EncodeToBinaryRepresentation<HexImpl>>({}, FunctionFactory::Case::Insensitive);
+    factory.registerFunction<DecodeFromBinaryRepresentation<UnhexImpl>>({}, FunctionFactory::Case::Insensitive);
+    factory.registerFunction<EncodeToBinaryRepresentation<BinImpl>>({}, FunctionFactory::Case::Insensitive);
+    factory.registerFunction<DecodeFromBinaryRepresentation<UnbinImpl>>({}, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/FunctionsCodingIP.cpp b/src/Functions/FunctionsCodingIP.cpp
index a134e39fbcd..0a97d029f84 100644
--- a/src/Functions/FunctionsCodingIP.cpp
+++ b/src/Functions/FunctionsCodingIP.cpp
@@ -1169,10 +1169,10 @@ REGISTER_FUNCTION(Coding)
     factory.registerFunction<FunctionIPv6StringToNum<IPStringToNumExceptionMode::Null>>();
 
     /// MySQL compatibility aliases:
-    factory.registerAlias("INET_ATON", FunctionIPv4StringToNum<IPStringToNumExceptionMode::Throw>::name, FunctionFactory::CaseInsensitive);
-    factory.registerAlias("INET6_NTOA", FunctionIPv6NumToString::name, FunctionFactory::CaseInsensitive);
-    factory.registerAlias("INET6_ATON", FunctionIPv6StringToNum<IPStringToNumExceptionMode::Throw>::name, FunctionFactory::CaseInsensitive);
-    factory.registerAlias("INET_NTOA", NameFunctionIPv4NumToString::name, FunctionFactory::CaseInsensitive);
+    factory.registerAlias("INET_ATON", FunctionIPv4StringToNum<IPStringToNumExceptionMode::Throw>::name, FunctionFactory::Case::Insensitive);
+    factory.registerAlias("INET6_NTOA", FunctionIPv6NumToString::name, FunctionFactory::Case::Insensitive);
+    factory.registerAlias("INET6_ATON", FunctionIPv6StringToNum<IPStringToNumExceptionMode::Throw>::name, FunctionFactory::Case::Insensitive);
+    factory.registerAlias("INET_NTOA", NameFunctionIPv4NumToString::name, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/FunctionsCodingULID.cpp b/src/Functions/FunctionsCodingULID.cpp
index ff040945a15..b67224a5625 100644
--- a/src/Functions/FunctionsCodingULID.cpp
+++ b/src/Functions/FunctionsCodingULID.cpp
@@ -180,8 +180,7 @@ An optional second argument can be passed to specify a timezone for the timestam
                 {"ulid", "SELECT ULIDStringToDateTime(generateULID())", ""},
                 {"timezone", "SELECT ULIDStringToDateTime(generateULID(), 'Asia/Istanbul')", ""}},
             .categories{"ULID"}
-        },
-        FunctionFactory::CaseSensitive);
+        });
 }
 
 }
diff --git a/src/Functions/FunctionsCodingUUID.cpp b/src/Functions/FunctionsCodingUUID.cpp
index 6a44f4263a8..83fdcbc4af9 100644
--- a/src/Functions/FunctionsCodingUUID.cpp
+++ b/src/Functions/FunctionsCodingUUID.cpp
@@ -496,8 +496,8 @@ This function accepts a UUID and returns a FixedString(16) as its binary represe
 │ 612f3c40-5d3b-217e-707b-6a546a3d7b29 │ a/<@];!~p{jTj={) │ @</a];!~p{jTj={) │
 └──────────────────────────────────────┴──────────────────┴──────────────────┘
 )"}},
-            .categories{"UUID"}},
-        FunctionFactory::CaseSensitive);
+            .categories{"UUID"}});
+
 
     factory.registerFunction<FunctionUUIDv7ToDateTime>(
         FunctionDocumentation{
@@ -509,8 +509,7 @@ An optional second argument can be passed to specify a timezone for the timestam
             .examples{
                 {"uuid","select UUIDv7ToDateTime(generateUUIDv7())", ""},
                 {"uuid","select generateUUIDv7() as uuid, UUIDv7ToDateTime(uuid), UUIDv7ToDateTime(uuid, 'America/New_York')", ""}},
-            .categories{"UUID"}},
-        FunctionFactory::CaseSensitive);
+            .categories{"UUID"}});
 }
 
 }
diff --git a/src/Functions/FunctionsConversion.cpp b/src/Functions/FunctionsConversion.cpp
index 1342e3f2c5d..675283d011e 100644
--- a/src/Functions/FunctionsConversion.cpp
+++ b/src/Functions/FunctionsConversion.cpp
@@ -5224,7 +5224,7 @@ REGISTER_FUNCTION(Conversion)
     /// MySQL compatibility alias. Cannot be registered as alias,
     /// because we don't want it to be normalized to toDate in queries,
     /// otherwise CREATE DICTIONARY query breaks.
-    factory.registerFunction("DATE", &FunctionToDate::create, {}, FunctionFactory::CaseInsensitive);
+    factory.registerFunction("DATE", &FunctionToDate::create, {}, FunctionFactory::Case::Insensitive);
 
     factory.registerFunction<FunctionToDate32>();
     factory.registerFunction<FunctionToDateTime>();
diff --git a/src/Functions/FunctionsHashingMisc.cpp b/src/Functions/FunctionsHashingMisc.cpp
index 38f16af0e6d..5cc29215fe3 100644
--- a/src/Functions/FunctionsHashingMisc.cpp
+++ b/src/Functions/FunctionsHashingMisc.cpp
@@ -41,8 +41,7 @@ REGISTER_FUNCTION(Hashing)
             .description="Calculates value of XXH3 64-bit hash function. Refer to https://github.com/Cyan4973/xxHash for detailed documentation.",
             .examples{{"hash", "SELECT xxh3('ClickHouse')", ""}},
             .categories{"Hash"}
-        },
-        FunctionFactory::CaseSensitive);
+        });
 
     factory.registerFunction<FunctionWyHash64>();
 
diff --git a/src/Functions/FunctionsLogical.cpp b/src/Functions/FunctionsLogical.cpp
index 2f5ce6deebf..e1814150da6 100644
--- a/src/Functions/FunctionsLogical.cpp
+++ b/src/Functions/FunctionsLogical.cpp
@@ -29,7 +29,7 @@ REGISTER_FUNCTION(Logical)
     factory.registerFunction<FunctionAnd>();
     factory.registerFunction<FunctionOr>();
     factory.registerFunction<FunctionXor>();
-    factory.registerFunction<FunctionNot>({}, FunctionFactory::CaseInsensitive); /// Operator NOT(x) can be parsed as a function.
+    factory.registerFunction<FunctionNot>({}, FunctionFactory::Case::Insensitive); /// Operator NOT(x) can be parsed as a function.
 }
 
 namespace ErrorCodes
diff --git a/src/Functions/FunctionsOpDate.cpp b/src/Functions/FunctionsOpDate.cpp
index 7355848f73f..c4b154736e0 100644
--- a/src/Functions/FunctionsOpDate.cpp
+++ b/src/Functions/FunctionsOpDate.cpp
@@ -99,8 +99,8 @@ using FunctionSubDate = FunctionOpDate<SubDate>;
 
 REGISTER_FUNCTION(AddInterval)
 {
-    factory.registerFunction<FunctionAddDate>({}, FunctionFactory::CaseInsensitive);
-    factory.registerFunction<FunctionSubDate>({}, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<FunctionAddDate>({}, FunctionFactory::Case::Insensitive);
+    factory.registerFunction<FunctionSubDate>({}, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/FunctionsRound.cpp b/src/Functions/FunctionsRound.cpp
index 059476acb40..d87a9e7ca43 100644
--- a/src/Functions/FunctionsRound.cpp
+++ b/src/Functions/FunctionsRound.cpp
@@ -7,16 +7,16 @@ namespace DB
 
 REGISTER_FUNCTION(Round)
 {
-    factory.registerFunction<FunctionRound>({}, FunctionFactory::CaseInsensitive);
-    factory.registerFunction<FunctionRoundBankers>({}, FunctionFactory::CaseSensitive);
-    factory.registerFunction<FunctionFloor>({}, FunctionFactory::CaseInsensitive);
-    factory.registerFunction<FunctionCeil>({}, FunctionFactory::CaseInsensitive);
-    factory.registerFunction<FunctionTrunc>({}, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<FunctionRound>({}, FunctionFactory::Case::Insensitive);
+    factory.registerFunction<FunctionRoundBankers>({}, FunctionFactory::Case::Sensitive);
+    factory.registerFunction<FunctionFloor>({}, FunctionFactory::Case::Insensitive);
+    factory.registerFunction<FunctionCeil>({}, FunctionFactory::Case::Insensitive);
+    factory.registerFunction<FunctionTrunc>({}, FunctionFactory::Case::Insensitive);
     factory.registerFunction<FunctionRoundDown>();
 
     /// Compatibility aliases.
-    factory.registerAlias("ceiling", "ceil", FunctionFactory::CaseInsensitive);
-    factory.registerAlias("truncate", "trunc", FunctionFactory::CaseInsensitive);
+    factory.registerAlias("ceiling", "ceil", FunctionFactory::Case::Insensitive);
+    factory.registerAlias("truncate", "trunc", FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/FunctionsStringHashFixedString.cpp b/src/Functions/FunctionsStringHashFixedString.cpp
index e3b1b82c92f..01e989a7f2c 100644
--- a/src/Functions/FunctionsStringHashFixedString.cpp
+++ b/src/Functions/FunctionsStringHashFixedString.cpp
@@ -428,8 +428,7 @@ REGISTER_FUNCTION(HashFixedStrings)
     It returns a BLAKE3 hash as a byte array with type FixedString(32).
     )",
             .examples{{"hash", "SELECT hex(BLAKE3('ABC'))", ""}},
-            .categories{"Hash"}},
-        FunctionFactory::CaseSensitive);
+            .categories{"Hash"}});
 #    endif
 }
 #endif
diff --git a/src/Functions/JSONArrayLength.cpp b/src/Functions/JSONArrayLength.cpp
index 73dd55f1266..24e93440454 100644
--- a/src/Functions/JSONArrayLength.cpp
+++ b/src/Functions/JSONArrayLength.cpp
@@ -104,7 +104,7 @@ REGISTER_FUNCTION(JSONArrayLength)
         .description="Returns the number of elements in the outermost JSON array. The function returns NULL if input JSON string is invalid."});
 
     /// For Spark compatibility.
-    factory.registerAlias("JSON_ARRAY_LENGTH", "JSONArrayLength", FunctionFactory::CaseInsensitive);
+    factory.registerAlias("JSON_ARRAY_LENGTH", "JSONArrayLength", FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/UTCTimestamp.cpp b/src/Functions/UTCTimestamp.cpp
index acc34b0a974..bc8e1b28431 100644
--- a/src/Functions/UTCTimestamp.cpp
+++ b/src/Functions/UTCTimestamp.cpp
@@ -117,8 +117,8 @@ Example:
 )",
     .examples{
         {"typical", "SELECT UTCTimestamp();", ""}},
-    .categories{"Dates and Times"}}, FunctionFactory::CaseInsensitive);
-    factory.registerAlias("UTC_timestamp", UTCTimestampOverloadResolver::name, FunctionFactory::CaseInsensitive);
+    .categories{"Dates and Times"}}, FunctionFactory::Case::Insensitive);
+    factory.registerAlias("UTC_timestamp", UTCTimestampOverloadResolver::name, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/UTCTimestampTransform.cpp b/src/Functions/UTCTimestampTransform.cpp
index 6d301270d8e..36ec520068f 100644
--- a/src/Functions/UTCTimestampTransform.cpp
+++ b/src/Functions/UTCTimestampTransform.cpp
@@ -144,8 +144,8 @@ REGISTER_FUNCTION(UTCTimestampTransform)
 {
     factory.registerFunction<ToUTCTimestampFunction>();
     factory.registerFunction<FromUTCTimestampFunction>();
-    factory.registerAlias("to_utc_timestamp", NameToUTCTimestamp::name, FunctionFactory::CaseInsensitive);
-    factory.registerAlias("from_utc_timestamp", NameFromUTCTimestamp::name, FunctionFactory::CaseInsensitive);
+    factory.registerAlias("to_utc_timestamp", NameToUTCTimestamp::name, FunctionFactory::Case::Insensitive);
+    factory.registerAlias("from_utc_timestamp", NameFromUTCTimestamp::name, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/abs.cpp b/src/Functions/abs.cpp
index 9ac2363f765..742d3b85619 100644
--- a/src/Functions/abs.cpp
+++ b/src/Functions/abs.cpp
@@ -51,7 +51,7 @@ template <> struct FunctionUnaryArithmeticMonotonicity<NameAbs>
 
 REGISTER_FUNCTION(Abs)
 {
-    factory.registerFunction<FunctionAbs>({}, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<FunctionAbs>({}, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/acos.cpp b/src/Functions/acos.cpp
index bc300ee77fb..39895fed64a 100644
--- a/src/Functions/acos.cpp
+++ b/src/Functions/acos.cpp
@@ -14,7 +14,7 @@ using FunctionAcos = FunctionMathUnary<UnaryFunctionVectorized<AcosName, acos>>;
 
 REGISTER_FUNCTION(Acos)
 {
-    factory.registerFunction<FunctionAcos>({}, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<FunctionAcos>({}, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/array/arrayFlatten.cpp b/src/Functions/array/arrayFlatten.cpp
index d4eb8eebeee..553ad82bd53 100644
--- a/src/Functions/array/arrayFlatten.cpp
+++ b/src/Functions/array/arrayFlatten.cpp
@@ -123,7 +123,7 @@ private:
 REGISTER_FUNCTION(ArrayFlatten)
 {
     factory.registerFunction<ArrayFlatten>();
-    factory.registerAlias("flatten", "arrayFlatten", FunctionFactory::CaseInsensitive);
+    factory.registerAlias("flatten", "arrayFlatten", FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/array/arrayShuffle.cpp b/src/Functions/array/arrayShuffle.cpp
index 10cb51d27d2..fa17aa46464 100644
--- a/src/Functions/array/arrayShuffle.cpp
+++ b/src/Functions/array/arrayShuffle.cpp
@@ -196,7 +196,7 @@ It is possible to override the seed to produce stable results:
                 {"explicit_seed", "SELECT arrayShuffle([1, 2, 3, 4], 41)", ""},
                 {"materialize", "SELECT arrayShuffle(materialize([1, 2, 3]), 42), arrayShuffle([1, 2, 3], 42) FROM numbers(10)", ""}},
             .categories{"Array"}},
-        FunctionFactory::CaseInsensitive);
+        FunctionFactory::Case::Insensitive);
 
     factory.registerFunction<FunctionArrayShuffleImpl<FunctionArrayPartialShuffleTraits>>(
         FunctionDocumentation{
@@ -224,7 +224,7 @@ It is possible to override the seed to produce stable results:
                 {"materialize",
                  "SELECT arrayPartialShuffle(materialize([1, 2, 3, 4]), 2, 42), arrayPartialShuffle([1, 2, 3], 2, 42) FROM numbers(10)", ""}},
             .categories{"Array"}},
-        FunctionFactory::CaseInsensitive);
+        FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/array/length.cpp b/src/Functions/array/length.cpp
index 91a5e5fdec2..d81c071b55e 100644
--- a/src/Functions/array/length.cpp
+++ b/src/Functions/array/length.cpp
@@ -100,8 +100,8 @@ It is ok to have ASCII NUL bytes in strings, and they will be counted as well.
                 },
             .categories{"String", "Array"}
         },
-        FunctionFactory::CaseInsensitive);
-    factory.registerAlias("OCTET_LENGTH", "length", FunctionFactory::CaseInsensitive);
+        FunctionFactory::Case::Insensitive);
+    factory.registerAlias("OCTET_LENGTH", "length", FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/ascii.cpp b/src/Functions/ascii.cpp
index b43c3221391..7c8158b53d4 100644
--- a/src/Functions/ascii.cpp
+++ b/src/Functions/ascii.cpp
@@ -90,7 +90,7 @@ If s is empty, the result is 0. If the first character is not an ASCII character
         )",
         .examples{{"ascii", "SELECT ascii('234')", ""}},
         .categories{"String"}
-        }, FunctionFactory::CaseInsensitive);
+        }, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/asin.cpp b/src/Functions/asin.cpp
index 3049b025d5e..85faf8c275d 100644
--- a/src/Functions/asin.cpp
+++ b/src/Functions/asin.cpp
@@ -41,7 +41,7 @@ For more details, see [https://en.wikipedia.org/wiki/Inverse_trigonometric_funct
                 {"nan", "SELECT asin(1.1), asin(-2), asin(inf), asin(nan)", ""}},
             .categories{"Mathematical", "Trigonometric"}
         },
-        FunctionFactory::CaseInsensitive);
+        FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/atan.cpp b/src/Functions/atan.cpp
index 32a0f06db8a..3f74c510487 100644
--- a/src/Functions/atan.cpp
+++ b/src/Functions/atan.cpp
@@ -14,7 +14,7 @@ using FunctionAtan = FunctionMathUnary<UnaryFunctionVectorized<AtanName, atan>>;
 
 REGISTER_FUNCTION(Atan)
 {
-    factory.registerFunction<FunctionAtan>({}, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<FunctionAtan>({}, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/atan2.cpp b/src/Functions/atan2.cpp
index 7be177f6dfb..42294e11458 100644
--- a/src/Functions/atan2.cpp
+++ b/src/Functions/atan2.cpp
@@ -15,7 +15,7 @@ namespace
 
 REGISTER_FUNCTION(Atan2)
 {
-    factory.registerFunction<FunctionAtan2>({}, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<FunctionAtan2>({}, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/base64Decode.cpp b/src/Functions/base64Decode.cpp
index 50278c4b0b2..4d06ac99d6f 100644
--- a/src/Functions/base64Decode.cpp
+++ b/src/Functions/base64Decode.cpp
@@ -17,7 +17,7 @@ REGISTER_FUNCTION(Base64Decode)
     factory.registerFunction<FunctionBase64Conversion<Base64Decode<Base64Variant::Normal>>>({description, syntax, arguments, returned_value, examples, categories});
 
     /// MySQL compatibility alias.
-    factory.registerAlias("FROM_BASE64", "base64Decode", FunctionFactory::CaseInsensitive);
+    factory.registerAlias("FROM_BASE64", "base64Decode", FunctionFactory::Case::Insensitive);
 }
 }
 
diff --git a/src/Functions/base64Encode.cpp b/src/Functions/base64Encode.cpp
index d6e63c38a4c..64142995552 100644
--- a/src/Functions/base64Encode.cpp
+++ b/src/Functions/base64Encode.cpp
@@ -17,7 +17,7 @@ REGISTER_FUNCTION(Base64Encode)
     factory.registerFunction<FunctionBase64Conversion<Base64Encode<Base64Variant::Normal>>>({description, syntax, arguments, returned_value, examples, categories});
 
     /// MySQL compatibility alias.
-    factory.registerAlias("TO_BASE64", "base64Encode", FunctionFactory::CaseInsensitive);
+    factory.registerAlias("TO_BASE64", "base64Encode", FunctionFactory::Case::Insensitive);
 }
 }
 
diff --git a/src/Functions/byteSwap.cpp b/src/Functions/byteSwap.cpp
index 2a343a07720..6c824b851b0 100644
--- a/src/Functions/byteSwap.cpp
+++ b/src/Functions/byteSwap.cpp
@@ -100,7 +100,7 @@ One use-case of this function is reversing IPv4s:
                 {"64-bit", "SELECT byteSwap(123294967295)", "18439412204227788800"},
             },
             .categories{"Mathematical", "Arithmetic"}},
-        FunctionFactory::CaseInsensitive);
+        FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/coalesce.cpp b/src/Functions/coalesce.cpp
index 722f32af523..19da6a85b38 100644
--- a/src/Functions/coalesce.cpp
+++ b/src/Functions/coalesce.cpp
@@ -180,7 +180,7 @@ private:
 
 REGISTER_FUNCTION(Coalesce)
 {
-    factory.registerFunction<FunctionCoalesce>({}, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<FunctionCoalesce>({}, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/concat.cpp b/src/Functions/concat.cpp
index b011c33e02a..5c5e089e740 100644
--- a/src/Functions/concat.cpp
+++ b/src/Functions/concat.cpp
@@ -240,7 +240,7 @@ private:
 
 REGISTER_FUNCTION(Concat)
 {
-    factory.registerFunction<ConcatOverloadResolver>({}, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<ConcatOverloadResolver>({}, FunctionFactory::Case::Insensitive);
     factory.registerFunction<FunctionConcatAssumeInjective>();
 }
 
diff --git a/src/Functions/concatWithSeparator.cpp b/src/Functions/concatWithSeparator.cpp
index ed02f331192..1d38ef87558 100644
--- a/src/Functions/concatWithSeparator.cpp
+++ b/src/Functions/concatWithSeparator.cpp
@@ -193,7 +193,7 @@ The function is named “injective” if it always returns different result for
         .categories{"String"}});
 
     /// Compatibility with Spark and MySQL:
-    factory.registerAlias("concat_ws", "concatWithSeparator", FunctionFactory::CaseInsensitive);
+    factory.registerAlias("concat_ws", "concatWithSeparator", FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/connectionId.cpp b/src/Functions/connectionId.cpp
index 9c53482482b..c1036b2ddbe 100644
--- a/src/Functions/connectionId.cpp
+++ b/src/Functions/connectionId.cpp
@@ -33,8 +33,8 @@ public:
 
 REGISTER_FUNCTION(ConnectionId)
 {
-    factory.registerFunction<FunctionConnectionId>({}, FunctionFactory::CaseInsensitive);
-    factory.registerAlias("connection_id", "connectionID", FunctionFactory::CaseInsensitive);
+    factory.registerFunction<FunctionConnectionId>({}, FunctionFactory::Case::Insensitive);
+    factory.registerAlias("connection_id", "connectionID", FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/cos.cpp b/src/Functions/cos.cpp
index 3496373a9d5..40fdede0e1c 100644
--- a/src/Functions/cos.cpp
+++ b/src/Functions/cos.cpp
@@ -13,7 +13,7 @@ using FunctionCos = FunctionMathUnary<UnaryFunctionVectorized<CosName, cos>>;
 
 REGISTER_FUNCTION(Cos)
 {
-    factory.registerFunction<FunctionCos>({}, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<FunctionCos>({}, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/countMatches.cpp b/src/Functions/countMatches.cpp
index a8620080012..4db48b1305f 100644
--- a/src/Functions/countMatches.cpp
+++ b/src/Functions/countMatches.cpp
@@ -22,8 +22,8 @@ namespace DB
 
 REGISTER_FUNCTION(CountMatches)
 {
-    factory.registerFunction<FunctionCountMatches<FunctionCountMatchesCaseSensitive>>({}, FunctionFactory::CaseSensitive);
-    factory.registerFunction<FunctionCountMatches<FunctionCountMatchesCaseInsensitive>>({}, FunctionFactory::CaseSensitive);
+    factory.registerFunction<FunctionCountMatches<FunctionCountMatchesCaseSensitive>>();
+    factory.registerFunction<FunctionCountMatches<FunctionCountMatchesCaseInsensitive>>();
 }
 
 }
diff --git a/src/Functions/countSubstrings.cpp b/src/Functions/countSubstrings.cpp
index 843b81437f5..137edb179b2 100644
--- a/src/Functions/countSubstrings.cpp
+++ b/src/Functions/countSubstrings.cpp
@@ -19,6 +19,6 @@ using FunctionCountSubstrings = FunctionsStringSearch<CountSubstringsImpl<NameCo
 
 REGISTER_FUNCTION(CountSubstrings)
 {
-    factory.registerFunction<FunctionCountSubstrings>({}, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<FunctionCountSubstrings>({}, FunctionFactory::Case::Insensitive);
 }
 }
diff --git a/src/Functions/currentDatabase.cpp b/src/Functions/currentDatabase.cpp
index 954899c3c2b..16cb43ebb04 100644
--- a/src/Functions/currentDatabase.cpp
+++ b/src/Functions/currentDatabase.cpp
@@ -54,9 +54,9 @@ public:
 REGISTER_FUNCTION(CurrentDatabase)
 {
     factory.registerFunction<FunctionCurrentDatabase>();
-    factory.registerAlias("DATABASE", FunctionCurrentDatabase::name, FunctionFactory::CaseInsensitive);
-    factory.registerAlias("SCHEMA", FunctionCurrentDatabase::name, FunctionFactory::CaseInsensitive);
-    factory.registerAlias("current_database", FunctionCurrentDatabase::name, FunctionFactory::CaseInsensitive);
+    factory.registerAlias("DATABASE", FunctionCurrentDatabase::name, FunctionFactory::Case::Insensitive);
+    factory.registerAlias("SCHEMA", FunctionCurrentDatabase::name, FunctionFactory::Case::Insensitive);
+    factory.registerAlias("current_database", FunctionCurrentDatabase::name, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/currentSchemas.cpp b/src/Functions/currentSchemas.cpp
index 322e719eb17..0a128d0e908 100644
--- a/src/Functions/currentSchemas.cpp
+++ b/src/Functions/currentSchemas.cpp
@@ -80,8 +80,8 @@ Requires a boolean parameter, but it is ignored actually. It is required just fo
              {"common", "SELECT current_schemas(true);", "['default']"}
         }
         },
-        FunctionFactory::CaseInsensitive);
-    factory.registerAlias("current_schemas", FunctionCurrentSchemas::name, FunctionFactory::CaseInsensitive);
+        FunctionFactory::Case::Insensitive);
+    factory.registerAlias("current_schemas", FunctionCurrentSchemas::name, FunctionFactory::Case::Insensitive);
 
 }
 
diff --git a/src/Functions/currentUser.cpp b/src/Functions/currentUser.cpp
index 1679c56a929..9f48f15ffb3 100644
--- a/src/Functions/currentUser.cpp
+++ b/src/Functions/currentUser.cpp
@@ -54,8 +54,8 @@ public:
 REGISTER_FUNCTION(CurrentUser)
 {
     factory.registerFunction<FunctionCurrentUser>();
-    factory.registerAlias("user", FunctionCurrentUser::name, FunctionFactory::CaseInsensitive);
-    factory.registerAlias("current_user", FunctionCurrentUser::name, FunctionFactory::CaseInsensitive);
+    factory.registerAlias("user", FunctionCurrentUser::name, FunctionFactory::Case::Insensitive);
+    factory.registerAlias("current_user", FunctionCurrentUser::name, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/dateDiff.cpp b/src/Functions/dateDiff.cpp
index 8e8865db7ed..f49e8dee6b7 100644
--- a/src/Functions/dateDiff.cpp
+++ b/src/Functions/dateDiff.cpp
@@ -490,7 +490,7 @@ private:
 
 REGISTER_FUNCTION(DateDiff)
 {
-    factory.registerFunction<FunctionDateDiff<true>>({}, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<FunctionDateDiff<true>>({}, FunctionFactory::Case::Insensitive);
     factory.registerAlias("date_diff", FunctionDateDiff<true>::name);
     factory.registerAlias("DATE_DIFF", FunctionDateDiff<true>::name);
     factory.registerAlias("timestampDiff", FunctionDateDiff<true>::name);
@@ -509,12 +509,12 @@ Example:
 )",
     .examples{
         {"typical", "SELECT timeDiff(UTCTimestamp(), now());", ""}},
-    .categories{"Dates and Times"}}, FunctionFactory::CaseInsensitive);
+    .categories{"Dates and Times"}}, FunctionFactory::Case::Insensitive);
 }
 
 REGISTER_FUNCTION(Age)
 {
-    factory.registerFunction<FunctionDateDiff<false>>({}, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<FunctionDateDiff<false>>({}, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/dateName.cpp b/src/Functions/dateName.cpp
index c06dfe15dc4..8165ea1b8d3 100644
--- a/src/Functions/dateName.cpp
+++ b/src/Functions/dateName.cpp
@@ -354,7 +354,7 @@ private:
 
 REGISTER_FUNCTION(DateName)
 {
-    factory.registerFunction<FunctionDateNameImpl>({}, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<FunctionDateNameImpl>({}, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/date_trunc.cpp b/src/Functions/date_trunc.cpp
index b8c60dd164e..dd3ea0b877b 100644
--- a/src/Functions/date_trunc.cpp
+++ b/src/Functions/date_trunc.cpp
@@ -178,7 +178,7 @@ REGISTER_FUNCTION(DateTrunc)
     factory.registerFunction<FunctionDateTrunc>();
 
     /// Compatibility alias.
-    factory.registerAlias("DATE_TRUNC", "dateTrunc", FunctionFactory::CaseInsensitive);
+    factory.registerAlias("DATE_TRUNC", "dateTrunc", FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/degrees.cpp b/src/Functions/degrees.cpp
index 3aa20a77a0d..8646eb54d9a 100644
--- a/src/Functions/degrees.cpp
+++ b/src/Functions/degrees.cpp
@@ -23,7 +23,7 @@ namespace
 
 REGISTER_FUNCTION(Degrees)
 {
-    factory.registerFunction<FunctionDegrees>({}, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<FunctionDegrees>({}, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/exp.cpp b/src/Functions/exp.cpp
index d352cda7460..e67cbd6d819 100644
--- a/src/Functions/exp.cpp
+++ b/src/Functions/exp.cpp
@@ -36,7 +36,7 @@ using FunctionExp = FunctionMathUnary<UnaryFunctionVectorized<ExpName, exp>>;
 
 REGISTER_FUNCTION(Exp)
 {
-    factory.registerFunction<FunctionExp>({}, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<FunctionExp>({}, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/extractAllGroupsVertical.cpp b/src/Functions/extractAllGroupsVertical.cpp
index 87a0b4cf7bc..6a968d89354 100644
--- a/src/Functions/extractAllGroupsVertical.cpp
+++ b/src/Functions/extractAllGroupsVertical.cpp
@@ -18,7 +18,7 @@ namespace DB
 REGISTER_FUNCTION(ExtractAllGroupsVertical)
 {
     factory.registerFunction<FunctionExtractAllGroups<VerticalImpl>>();
-    factory.registerAlias("extractAllGroups", VerticalImpl::Name, FunctionFactory::CaseSensitive);
+    factory.registerAlias("extractAllGroups", VerticalImpl::Name);
 }
 
 }
diff --git a/src/Functions/factorial.cpp b/src/Functions/factorial.cpp
index 7ff9126c004..9b319caad63 100644
--- a/src/Functions/factorial.cpp
+++ b/src/Functions/factorial.cpp
@@ -106,7 +106,7 @@ The factorial of 0 is 1. Likewise, the factorial() function returns 1 for any ne
 )",
             .examples{{"factorial", "SELECT factorial(10)", ""}},
             .categories{"Mathematical"}},
-        FunctionFactory::CaseInsensitive);
+        FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/formatDateTime.cpp b/src/Functions/formatDateTime.cpp
index c3a5fe036c3..f89afd67e78 100644
--- a/src/Functions/formatDateTime.cpp
+++ b/src/Functions/formatDateTime.cpp
@@ -1834,10 +1834,10 @@ using FunctionFromUnixTimestampInJodaSyntax = FunctionFormatDateTimeImpl<NameFro
 REGISTER_FUNCTION(FormatDateTime)
 {
     factory.registerFunction<FunctionFormatDateTime>();
-    factory.registerAlias("DATE_FORMAT", FunctionFormatDateTime::name, FunctionFactory::CaseInsensitive);
+    factory.registerAlias("DATE_FORMAT", FunctionFormatDateTime::name, FunctionFactory::Case::Insensitive);
 
     factory.registerFunction<FunctionFromUnixTimestamp>();
-    factory.registerAlias("FROM_UNIXTIME", FunctionFromUnixTimestamp::name, FunctionFactory::CaseInsensitive);
+    factory.registerAlias("FROM_UNIXTIME", FunctionFromUnixTimestamp::name, FunctionFactory::Case::Insensitive);
 
     factory.registerFunction<FunctionFormatDateTimeInJodaSyntax>();
     factory.registerFunction<FunctionFromUnixTimestampInJodaSyntax>();
diff --git a/src/Functions/formatReadableDecimalSize.cpp b/src/Functions/formatReadableDecimalSize.cpp
index 1aa5abc526e..9298360aebc 100644
--- a/src/Functions/formatReadableDecimalSize.cpp
+++ b/src/Functions/formatReadableDecimalSize.cpp
@@ -29,8 +29,7 @@ Accepts the size (number of bytes). Returns a rounded size with a suffix (KB, MB
         .examples{
             {"formatReadableDecimalSize", "SELECT formatReadableDecimalSize(1000)", ""}},
         .categories{"OtherFunctions"}
-    },
-    FunctionFactory::CaseSensitive);
+    });
 }
 
 }
diff --git a/src/Functions/formatReadableSize.cpp b/src/Functions/formatReadableSize.cpp
index 5c11603e9d7..ee66a0396df 100644
--- a/src/Functions/formatReadableSize.cpp
+++ b/src/Functions/formatReadableSize.cpp
@@ -22,7 +22,7 @@ namespace
 REGISTER_FUNCTION(FormatReadableSize)
 {
     factory.registerFunction<FunctionFormatReadable<Impl>>();
-    factory.registerAlias("FORMAT_BYTES", Impl::name, FunctionFactory::CaseInsensitive);
+    factory.registerAlias("FORMAT_BYTES", Impl::name, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/fromDaysSinceYearZero.cpp b/src/Functions/fromDaysSinceYearZero.cpp
index 0543e6bf229..e1ba9ea533e 100644
--- a/src/Functions/fromDaysSinceYearZero.cpp
+++ b/src/Functions/fromDaysSinceYearZero.cpp
@@ -125,7 +125,7 @@ The calculation is the same as in MySQL's FROM_DAYS() function.
         .examples{{"typical", "SELECT fromDaysSinceYearZero32(713569)", "2023-09-08"}},
         .categories{"Dates and Times"}});
 
-    factory.registerAlias("FROM_DAYS", FunctionFromDaysSinceYearZero<DateTraits>::name, FunctionFactory::CaseInsensitive);
+    factory.registerAlias("FROM_DAYS", FunctionFromDaysSinceYearZero<DateTraits>::name, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/generateULID.cpp b/src/Functions/generateULID.cpp
index f2f2d8ae3b9..933618ccec3 100644
--- a/src/Functions/generateULID.cpp
+++ b/src/Functions/generateULID.cpp
@@ -85,8 +85,7 @@ The function returns a value of type FixedString(26).
             {"ulid", "SELECT generateULID()", ""},
             {"multiple", "SELECT generateULID(1), generateULID(2)", ""}},
         .categories{"ULID"}
-    },
-    FunctionFactory::CaseSensitive);
+    });
 }
 
 }
diff --git a/src/Functions/greatest.cpp b/src/Functions/greatest.cpp
index 87a48c887b4..88539bda4a5 100644
--- a/src/Functions/greatest.cpp
+++ b/src/Functions/greatest.cpp
@@ -65,7 +65,7 @@ using FunctionGreatest = FunctionBinaryArithmetic<GreatestImpl, NameGreatest>;
 
 REGISTER_FUNCTION(Greatest)
 {
-    factory.registerFunction<LeastGreatestOverloadResolver<LeastGreatest::Greatest, FunctionGreatest>>({}, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<LeastGreatestOverloadResolver<LeastGreatest::Greatest, FunctionGreatest>>({}, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/hasSubsequence.cpp b/src/Functions/hasSubsequence.cpp
index 4bcce53b4db..1426e8cb7a9 100644
--- a/src/Functions/hasSubsequence.cpp
+++ b/src/Functions/hasSubsequence.cpp
@@ -24,7 +24,7 @@ using FunctionHasSubsequence = HasSubsequenceImpl<NameHasSubsequence, HasSubsequ
 
 REGISTER_FUNCTION(hasSubsequence)
 {
-    factory.registerFunction<FunctionHasSubsequence>({}, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<FunctionHasSubsequence>({}, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/hasSubsequenceCaseInsensitive.cpp b/src/Functions/hasSubsequenceCaseInsensitive.cpp
index c93bbead58c..8e5751066a9 100644
--- a/src/Functions/hasSubsequenceCaseInsensitive.cpp
+++ b/src/Functions/hasSubsequenceCaseInsensitive.cpp
@@ -23,7 +23,7 @@ using FunctionHasSubsequenceCaseInsensitive = HasSubsequenceImpl<NameHasSubseque
 
 REGISTER_FUNCTION(hasSubsequenceCaseInsensitive)
 {
-    factory.registerFunction<FunctionHasSubsequenceCaseInsensitive>({}, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<FunctionHasSubsequenceCaseInsensitive>({}, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/hasSubsequenceCaseInsensitiveUTF8.cpp b/src/Functions/hasSubsequenceCaseInsensitiveUTF8.cpp
index 18438bc8b16..039af061bf5 100644
--- a/src/Functions/hasSubsequenceCaseInsensitiveUTF8.cpp
+++ b/src/Functions/hasSubsequenceCaseInsensitiveUTF8.cpp
@@ -25,7 +25,7 @@ using FunctionHasSubsequenceCaseInsensitiveUTF8 = HasSubsequenceImpl<NameHasSubs
 
 REGISTER_FUNCTION(hasSubsequenceCaseInsensitiveUTF8)
 {
-    factory.registerFunction<FunctionHasSubsequenceCaseInsensitiveUTF8>({}, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<FunctionHasSubsequenceCaseInsensitiveUTF8>({}, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/hasSubsequenceUTF8.cpp b/src/Functions/hasSubsequenceUTF8.cpp
index 7a22211eb8c..636fbfab85f 100644
--- a/src/Functions/hasSubsequenceUTF8.cpp
+++ b/src/Functions/hasSubsequenceUTF8.cpp
@@ -24,7 +24,7 @@ using FunctionHasSubsequenceUTF8 = HasSubsequenceImpl<NameHasSubsequenceUTF8, Ha
 
 REGISTER_FUNCTION(hasSubsequenceUTF8)
 {
-    factory.registerFunction<FunctionHasSubsequenceUTF8>({}, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<FunctionHasSubsequenceUTF8>({}, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/hasToken.cpp b/src/Functions/hasToken.cpp
index fa41abf2641..299a8a16b35 100644
--- a/src/Functions/hasToken.cpp
+++ b/src/Functions/hasToken.cpp
@@ -25,10 +25,10 @@ using FunctionHasTokenOrNull
 REGISTER_FUNCTION(HasToken)
 {
     factory.registerFunction<FunctionHasToken>(FunctionDocumentation
-        {.description="Performs lookup of needle in haystack using tokenbf_v1 index."}, FunctionFactory::CaseSensitive);
+        {.description="Performs lookup of needle in haystack using tokenbf_v1 index."});
 
     factory.registerFunction<FunctionHasTokenOrNull>(FunctionDocumentation
-        {.description="Performs lookup of needle in haystack using tokenbf_v1 index. Returns null if needle is ill-formed."}, FunctionFactory::CaseSensitive);
+        {.description="Performs lookup of needle in haystack using tokenbf_v1 index. Returns null if needle is ill-formed."});
 }
 
 }
diff --git a/src/Functions/hasTokenCaseInsensitive.cpp b/src/Functions/hasTokenCaseInsensitive.cpp
index 32675b9384d..6ff134194e3 100644
--- a/src/Functions/hasTokenCaseInsensitive.cpp
+++ b/src/Functions/hasTokenCaseInsensitive.cpp
@@ -26,11 +26,11 @@ REGISTER_FUNCTION(HasTokenCaseInsensitive)
 {
     factory.registerFunction<FunctionHasTokenCaseInsensitive>(
         FunctionDocumentation{.description="Performs case insensitive lookup of needle in haystack using tokenbf_v1 index."},
-        DB::FunctionFactory::CaseInsensitive);
+        DB::FunctionFactory::Case::Insensitive);
 
     factory.registerFunction<FunctionHasTokenCaseInsensitiveOrNull>(
         FunctionDocumentation{.description="Performs case insensitive lookup of needle in haystack using tokenbf_v1 index. Returns null if needle is ill-formed."},
-        DB::FunctionFactory::CaseInsensitive);
+        DB::FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/hypot.cpp b/src/Functions/hypot.cpp
index 465471cb09b..8845d1fa8ae 100644
--- a/src/Functions/hypot.cpp
+++ b/src/Functions/hypot.cpp
@@ -15,7 +15,7 @@ namespace
 
 REGISTER_FUNCTION(Hypot)
 {
-    factory.registerFunction<FunctionHypot>({}, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<FunctionHypot>({}, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/if.cpp b/src/Functions/if.cpp
index 30eaa26fa20..07dbee27a9d 100644
--- a/src/Functions/if.cpp
+++ b/src/Functions/if.cpp
@@ -1309,7 +1309,7 @@ public:
 
 REGISTER_FUNCTION(If)
 {
-    factory.registerFunction<FunctionIf>({}, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<FunctionIf>({}, FunctionFactory::Case::Insensitive);
 }
 
 FunctionOverloadResolverPtr createInternalFunctionIfOverloadResolver(bool allow_experimental_variant_type, bool use_variant_as_common_type)
diff --git a/src/Functions/ifNull.cpp b/src/Functions/ifNull.cpp
index 1093f3f817f..358a52c8394 100644
--- a/src/Functions/ifNull.cpp
+++ b/src/Functions/ifNull.cpp
@@ -91,7 +91,7 @@ private:
 
 REGISTER_FUNCTION(IfNull)
 {
-    factory.registerFunction<FunctionIfNull>({}, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<FunctionIfNull>({}, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/initcap.cpp b/src/Functions/initcap.cpp
index 6b2958227bc..4661ce117c0 100644
--- a/src/Functions/initcap.cpp
+++ b/src/Functions/initcap.cpp
@@ -60,7 +60,7 @@ using FunctionInitcap = FunctionStringToString<InitcapImpl, NameInitcap>;
 
 REGISTER_FUNCTION(Initcap)
 {
-    factory.registerFunction<FunctionInitcap>({}, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<FunctionInitcap>({}, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/initialQueryID.cpp b/src/Functions/initialQueryID.cpp
index 9c9390d4e50..f32f92a2f46 100644
--- a/src/Functions/initialQueryID.cpp
+++ b/src/Functions/initialQueryID.cpp
@@ -41,6 +41,6 @@ public:
 REGISTER_FUNCTION(InitialQueryID)
 {
     factory.registerFunction<FunctionInitialQueryID>();
-    factory.registerAlias("initial_query_id", FunctionInitialQueryID::name, FunctionFactory::CaseInsensitive);
+    factory.registerAlias("initial_query_id", FunctionInitialQueryID::name, FunctionFactory::Case::Insensitive);
 }
 }
diff --git a/src/Functions/isNull.cpp b/src/Functions/isNull.cpp
index 95d659b103b..9347d81c45f 100644
--- a/src/Functions/isNull.cpp
+++ b/src/Functions/isNull.cpp
@@ -101,7 +101,7 @@ public:
 
 REGISTER_FUNCTION(IsNull)
 {
-    factory.registerFunction<FunctionIsNull>({}, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<FunctionIsNull>({}, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/keyvaluepair/extractKeyValuePairs.cpp b/src/Functions/keyvaluepair/extractKeyValuePairs.cpp
index 02e8412bbf3..1c5164e132d 100644
--- a/src/Functions/keyvaluepair/extractKeyValuePairs.cpp
+++ b/src/Functions/keyvaluepair/extractKeyValuePairs.cpp
@@ -241,7 +241,7 @@ REGISTER_FUNCTION(ExtractKeyValuePairs)
             └──────────────────┘
             ```)"}
     );
-    factory.registerAlias("str_to_map", NameExtractKeyValuePairs::name, FunctionFactory::CaseInsensitive);
+    factory.registerAlias("str_to_map", NameExtractKeyValuePairs::name, FunctionFactory::Case::Insensitive);
     factory.registerAlias("mapFromString", NameExtractKeyValuePairs::name);
 }
 
diff --git a/src/Functions/least.cpp b/src/Functions/least.cpp
index babb8378d80..091a868e8e2 100644
--- a/src/Functions/least.cpp
+++ b/src/Functions/least.cpp
@@ -65,7 +65,7 @@ using FunctionLeast = FunctionBinaryArithmetic<LeastImpl, NameLeast>;
 
 REGISTER_FUNCTION(Least)
 {
-    factory.registerFunction<LeastGreatestOverloadResolver<LeastGreatest::Least, FunctionLeast>>({}, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<LeastGreatestOverloadResolver<LeastGreatest::Least, FunctionLeast>>({}, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/left.cpp b/src/Functions/left.cpp
index 006706c8f21..c9f62a0f8f1 100644
--- a/src/Functions/left.cpp
+++ b/src/Functions/left.cpp
@@ -6,8 +6,8 @@ namespace DB
 
 REGISTER_FUNCTION(Left)
 {
-    factory.registerFunction<FunctionLeftRight<false, SubstringDirection::Left>>({}, FunctionFactory::CaseInsensitive);
-    factory.registerFunction<FunctionLeftRight<true, SubstringDirection::Left>>({}, FunctionFactory::CaseSensitive);
+    factory.registerFunction<FunctionLeftRight<false, SubstringDirection::Left>>({}, FunctionFactory::Case::Insensitive);
+    factory.registerFunction<FunctionLeftRight<true, SubstringDirection::Left>>({}, FunctionFactory::Case::Sensitive);
 }
 
 }
diff --git a/src/Functions/lengthUTF8.cpp b/src/Functions/lengthUTF8.cpp
index 5a4af4934df..59a0d532602 100644
--- a/src/Functions/lengthUTF8.cpp
+++ b/src/Functions/lengthUTF8.cpp
@@ -83,8 +83,8 @@ REGISTER_FUNCTION(LengthUTF8)
     factory.registerFunction<FunctionLengthUTF8>();
 
     /// Compatibility aliases.
-    factory.registerAlias("CHAR_LENGTH", "lengthUTF8", FunctionFactory::CaseInsensitive);
-    factory.registerAlias("CHARACTER_LENGTH", "lengthUTF8", FunctionFactory::CaseInsensitive);
+    factory.registerAlias("CHAR_LENGTH", "lengthUTF8", FunctionFactory::Case::Insensitive);
+    factory.registerAlias("CHARACTER_LENGTH", "lengthUTF8", FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/locate.cpp b/src/Functions/locate.cpp
index d9a727ab3ef..076aa1bdc6d 100644
--- a/src/Functions/locate.cpp
+++ b/src/Functions/locate.cpp
@@ -29,6 +29,6 @@ REGISTER_FUNCTION(Locate)
     FunctionDocumentation::Categories doc_categories = {"String search"};
 
 
-    factory.registerFunction<FunctionLocate>({doc_description, doc_syntax, doc_arguments, doc_returned_value, doc_examples, doc_categories}, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<FunctionLocate>({doc_description, doc_syntax, doc_arguments, doc_returned_value, doc_examples, doc_categories}, FunctionFactory::Case::Insensitive);
 }
 }
diff --git a/src/Functions/log.cpp b/src/Functions/log.cpp
index 9096b8c6f22..8bebdb8d7bd 100644
--- a/src/Functions/log.cpp
+++ b/src/Functions/log.cpp
@@ -34,8 +34,8 @@ using FunctionLog = FunctionMathUnary<UnaryFunctionVectorized<LogName, log>>;
 
 REGISTER_FUNCTION(Log)
 {
-    factory.registerFunction<FunctionLog>({}, FunctionFactory::CaseInsensitive);
-    factory.registerAlias("ln", "log", FunctionFactory::CaseInsensitive);
+    factory.registerFunction<FunctionLog>({}, FunctionFactory::Case::Insensitive);
+    factory.registerAlias("ln", "log", FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/log10.cpp b/src/Functions/log10.cpp
index 5dfe4ac9357..6241df3e092 100644
--- a/src/Functions/log10.cpp
+++ b/src/Functions/log10.cpp
@@ -13,7 +13,7 @@ using FunctionLog10 = FunctionMathUnary<UnaryFunctionVectorized<Log10Name, log10
 
 REGISTER_FUNCTION(Log10)
 {
-    factory.registerFunction<FunctionLog10>({}, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<FunctionLog10>({}, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/log2.cpp b/src/Functions/log2.cpp
index 9457ac64bc6..52b3ab52ea7 100644
--- a/src/Functions/log2.cpp
+++ b/src/Functions/log2.cpp
@@ -13,7 +13,7 @@ using FunctionLog2 = FunctionMathUnary<UnaryFunctionVectorized<Log2Name, log2>>;
 
 REGISTER_FUNCTION(Log2)
 {
-    factory.registerFunction<FunctionLog2>({}, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<FunctionLog2>({}, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/lower.cpp b/src/Functions/lower.cpp
index 38ae5a8a7f0..5210a20b026 100644
--- a/src/Functions/lower.cpp
+++ b/src/Functions/lower.cpp
@@ -19,8 +19,8 @@ using FunctionLower = FunctionStringToString<LowerUpperImpl<'A', 'Z'>, NameLower
 
 REGISTER_FUNCTION(Lower)
 {
-    factory.registerFunction<FunctionLower>({}, FunctionFactory::CaseInsensitive);
-    factory.registerAlias("lcase", NameLower::name, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<FunctionLower>({}, FunctionFactory::Case::Insensitive);
+    factory.registerAlias("lcase", NameLower::name, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/makeDate.cpp b/src/Functions/makeDate.cpp
index 41a09793994..21d466d7708 100644
--- a/src/Functions/makeDate.cpp
+++ b/src/Functions/makeDate.cpp
@@ -724,7 +724,7 @@ public:
 
 REGISTER_FUNCTION(MakeDate)
 {
-    factory.registerFunction<FunctionMakeDate<DateTraits>>({}, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<FunctionMakeDate<DateTraits>>({}, FunctionFactory::Case::Insensitive);
     factory.registerFunction<FunctionMakeDate<Date32Traits>>();
     factory.registerFunction<FunctionMakeDateTime>();
     factory.registerFunction<FunctionMakeDateTime64>();
diff --git a/src/Functions/match.cpp b/src/Functions/match.cpp
index c719cc6dd82..6cd65597032 100644
--- a/src/Functions/match.cpp
+++ b/src/Functions/match.cpp
@@ -20,7 +20,7 @@ using FunctionMatch = FunctionsStringSearch<MatchImpl<NameMatch, MatchTraits::Sy
 REGISTER_FUNCTION(Match)
 {
     factory.registerFunction<FunctionMatch>();
-    factory.registerAlias("REGEXP_MATCHES", NameMatch::name, FunctionFactory::CaseInsensitive);
+    factory.registerAlias("REGEXP_MATCHES", NameMatch::name, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/mathConstants.cpp b/src/Functions/mathConstants.cpp
index 2b199a30616..37ababbc0e5 100644
--- a/src/Functions/mathConstants.cpp
+++ b/src/Functions/mathConstants.cpp
@@ -44,7 +44,7 @@ REGISTER_FUNCTION(E)
 
 REGISTER_FUNCTION(Pi)
 {
-    factory.registerFunction<FunctionPi>({}, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<FunctionPi>({}, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/max2.cpp b/src/Functions/max2.cpp
index 928e6f22918..88b5c7c08c0 100644
--- a/src/Functions/max2.cpp
+++ b/src/Functions/max2.cpp
@@ -21,6 +21,6 @@ namespace
 
 REGISTER_FUNCTION(Max2)
 {
-    factory.registerFunction<FunctionMax2>({}, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<FunctionMax2>({}, FunctionFactory::Case::Insensitive);
 }
 }
diff --git a/src/Functions/min2.cpp b/src/Functions/min2.cpp
index f031530edf5..8ab56dbe90d 100644
--- a/src/Functions/min2.cpp
+++ b/src/Functions/min2.cpp
@@ -22,6 +22,6 @@ namespace
 
 REGISTER_FUNCTION(Min2)
 {
-    factory.registerFunction<FunctionMin2>({}, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<FunctionMin2>({}, FunctionFactory::Case::Insensitive);
 }
 }
diff --git a/src/Functions/modulo.cpp b/src/Functions/modulo.cpp
index ebc1c4f5275..76a07aeda2e 100644
--- a/src/Functions/modulo.cpp
+++ b/src/Functions/modulo.cpp
@@ -155,7 +155,7 @@ using FunctionModulo = BinaryArithmeticOverloadResolver<ModuloImpl, NameModulo,
 REGISTER_FUNCTION(Modulo)
 {
     factory.registerFunction<FunctionModulo>();
-    factory.registerAlias("mod", "modulo", FunctionFactory::CaseInsensitive);
+    factory.registerAlias("mod", "modulo", FunctionFactory::Case::Insensitive);
 }
 
 struct NameModuloLegacy { static constexpr auto name = "moduloLegacy"; };
@@ -183,11 +183,11 @@ In other words, the function returning the modulus (modulo) in the terms of Modu
         )",
             .examples{{"positiveModulo", "SELECT positiveModulo(-1, 10);", ""}},
             .categories{"Arithmetic"}},
-        FunctionFactory::CaseInsensitive);
+        FunctionFactory::Case::Insensitive);
 
-    factory.registerAlias("positive_modulo", "positiveModulo", FunctionFactory::CaseInsensitive);
+    factory.registerAlias("positive_modulo", "positiveModulo", FunctionFactory::Case::Insensitive);
     /// Compatibility with Spark:
-    factory.registerAlias("pmod", "positiveModulo", FunctionFactory::CaseInsensitive);
+    factory.registerAlias("pmod", "positiveModulo", FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/monthName.cpp b/src/Functions/monthName.cpp
index f49f77bd6e7..ae444460170 100644
--- a/src/Functions/monthName.cpp
+++ b/src/Functions/monthName.cpp
@@ -74,7 +74,7 @@ private:
 
 REGISTER_FUNCTION(MonthName)
 {
-    factory.registerFunction<FunctionMonthName>({}, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<FunctionMonthName>({}, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/now.cpp b/src/Functions/now.cpp
index 352c05f8f32..7b2150e3534 100644
--- a/src/Functions/now.cpp
+++ b/src/Functions/now.cpp
@@ -138,8 +138,8 @@ private:
 
 REGISTER_FUNCTION(Now)
 {
-    factory.registerFunction<NowOverloadResolver>({}, FunctionFactory::CaseInsensitive);
-    factory.registerAlias("current_timestamp", NowOverloadResolver::name, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<NowOverloadResolver>({}, FunctionFactory::Case::Insensitive);
+    factory.registerAlias("current_timestamp", NowOverloadResolver::name, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/now64.cpp b/src/Functions/now64.cpp
index 23ab8cad003..9786a0c9f39 100644
--- a/src/Functions/now64.cpp
+++ b/src/Functions/now64.cpp
@@ -170,7 +170,7 @@ private:
 
 REGISTER_FUNCTION(Now64)
 {
-    factory.registerFunction<Now64OverloadResolver>({}, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<Now64OverloadResolver>({}, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/nullIf.cpp b/src/Functions/nullIf.cpp
index 392cc20cfcf..550287885a1 100644
--- a/src/Functions/nullIf.cpp
+++ b/src/Functions/nullIf.cpp
@@ -69,7 +69,7 @@ public:
 
 REGISTER_FUNCTION(NullIf)
 {
-    factory.registerFunction<FunctionNullIf>({}, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<FunctionNullIf>({}, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/padString.cpp b/src/Functions/padString.cpp
index 8670c837e21..23554c3fbbc 100644
--- a/src/Functions/padString.cpp
+++ b/src/Functions/padString.cpp
@@ -335,8 +335,8 @@ REGISTER_FUNCTION(PadString)
     factory.registerFunction<FunctionPadString<true, false>>();  /// rightPad
     factory.registerFunction<FunctionPadString<true, true>>();   /// rightPadUTF8
 
-    factory.registerAlias("lpad", "leftPad", FunctionFactory::CaseInsensitive);
-    factory.registerAlias("rpad", "rightPad", FunctionFactory::CaseInsensitive);
+    factory.registerAlias("lpad", "leftPad", FunctionFactory::Case::Insensitive);
+    factory.registerAlias("rpad", "rightPad", FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/parseDateTime.cpp b/src/Functions/parseDateTime.cpp
index d2353c19a61..bdca0151bba 100644
--- a/src/Functions/parseDateTime.cpp
+++ b/src/Functions/parseDateTime.cpp
@@ -2098,10 +2098,10 @@ namespace
 REGISTER_FUNCTION(ParseDateTime)
 {
     factory.registerFunction<FunctionParseDateTime>();
-    factory.registerAlias("TO_UNIXTIME", FunctionParseDateTime::name, FunctionFactory::CaseInsensitive);
+    factory.registerAlias("TO_UNIXTIME", FunctionParseDateTime::name, FunctionFactory::Case::Insensitive);
     factory.registerFunction<FunctionParseDateTimeOrZero>();
     factory.registerFunction<FunctionParseDateTimeOrNull>();
-    factory.registerAlias("str_to_date", FunctionParseDateTimeOrNull::name, FunctionFactory::CaseInsensitive);
+    factory.registerAlias("str_to_date", FunctionParseDateTimeOrNull::name, FunctionFactory::Case::Insensitive);
 
     factory.registerFunction<FunctionParseDateTimeInJodaSyntax>();
     factory.registerFunction<FunctionParseDateTimeInJodaSyntaxOrZero>();
diff --git a/src/Functions/position.cpp b/src/Functions/position.cpp
index 29a5db2eb24..aad47cc5b3f 100644
--- a/src/Functions/position.cpp
+++ b/src/Functions/position.cpp
@@ -19,6 +19,6 @@ using FunctionPosition = FunctionsStringSearch<PositionImpl<NamePosition, Positi
 
 REGISTER_FUNCTION(Position)
 {
-    factory.registerFunction<FunctionPosition>({}, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<FunctionPosition>({}, FunctionFactory::Case::Insensitive);
 }
 }
diff --git a/src/Functions/positionCaseInsensitive.cpp b/src/Functions/positionCaseInsensitive.cpp
index f71ce0078cc..7c59ffa83cd 100644
--- a/src/Functions/positionCaseInsensitive.cpp
+++ b/src/Functions/positionCaseInsensitive.cpp
@@ -20,6 +20,6 @@ using FunctionPositionCaseInsensitive = FunctionsStringSearch<PositionImpl<NameP
 REGISTER_FUNCTION(PositionCaseInsensitive)
 {
     factory.registerFunction<FunctionPositionCaseInsensitive>();
-    factory.registerAlias("instr", NamePositionCaseInsensitive::name, FunctionFactory::CaseInsensitive);
+    factory.registerAlias("instr", NamePositionCaseInsensitive::name, FunctionFactory::Case::Insensitive);
 }
 }
diff --git a/src/Functions/pow.cpp b/src/Functions/pow.cpp
index 9b383da97e7..f2976b4812e 100644
--- a/src/Functions/pow.cpp
+++ b/src/Functions/pow.cpp
@@ -13,8 +13,8 @@ using FunctionPow = FunctionMathBinaryFloat64<BinaryFunctionVectorized<PowName,
 
 REGISTER_FUNCTION(Pow)
 {
-    factory.registerFunction<FunctionPow>({}, FunctionFactory::CaseInsensitive);
-    factory.registerAlias("power", "pow", FunctionFactory::CaseInsensitive);
+    factory.registerFunction<FunctionPow>({}, FunctionFactory::Case::Insensitive);
+    factory.registerAlias("power", "pow", FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/queryID.cpp b/src/Functions/queryID.cpp
index 5d0ac719797..7299714e464 100644
--- a/src/Functions/queryID.cpp
+++ b/src/Functions/queryID.cpp
@@ -41,6 +41,6 @@ public:
 REGISTER_FUNCTION(QueryID)
 {
     factory.registerFunction<FunctionQueryID>();
-    factory.registerAlias("query_id", FunctionQueryID::name, FunctionFactory::CaseInsensitive);
+    factory.registerAlias("query_id", FunctionQueryID::name, FunctionFactory::Case::Insensitive);
 }
 }
diff --git a/src/Functions/radians.cpp b/src/Functions/radians.cpp
index 2c2c2743532..9185340be15 100644
--- a/src/Functions/radians.cpp
+++ b/src/Functions/radians.cpp
@@ -23,7 +23,7 @@ namespace
 
 REGISTER_FUNCTION(Radians)
 {
-    factory.registerFunction<FunctionRadians>({}, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<FunctionRadians>({}, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/rand.cpp b/src/Functions/rand.cpp
index ea30922d731..35b325e59fd 100644
--- a/src/Functions/rand.cpp
+++ b/src/Functions/rand.cpp
@@ -13,7 +13,7 @@ using FunctionRand = FunctionRandom<UInt32, NameRand>;
 
 REGISTER_FUNCTION(Rand)
 {
-    factory.registerFunction<FunctionRand>({}, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<FunctionRand>({}, FunctionFactory::Case::Insensitive);
     factory.registerAlias("rand32", NameRand::name);
 }
 
diff --git a/src/Functions/regexpExtract.cpp b/src/Functions/regexpExtract.cpp
index 3cc5393296c..6bedac54e39 100644
--- a/src/Functions/regexpExtract.cpp
+++ b/src/Functions/regexpExtract.cpp
@@ -253,7 +253,7 @@ REGISTER_FUNCTION(RegexpExtract)
         FunctionDocumentation{.description="Extracts the first string in haystack that matches the regexp pattern and corresponds to the regex group index."});
 
     /// For Spark compatibility.
-    factory.registerAlias("REGEXP_EXTRACT", "regexpExtract", FunctionFactory::CaseInsensitive);
+    factory.registerAlias("REGEXP_EXTRACT", "regexpExtract", FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/repeat.cpp b/src/Functions/repeat.cpp
index aa90bf2490d..c001959b465 100644
--- a/src/Functions/repeat.cpp
+++ b/src/Functions/repeat.cpp
@@ -278,7 +278,7 @@ public:
 
 REGISTER_FUNCTION(Repeat)
 {
-    factory.registerFunction<FunctionRepeat>({}, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<FunctionRepeat>({}, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/replaceAll.cpp b/src/Functions/replaceAll.cpp
index 6c06f5984b3..9ce525390bf 100644
--- a/src/Functions/replaceAll.cpp
+++ b/src/Functions/replaceAll.cpp
@@ -20,7 +20,7 @@ using FunctionReplaceAll = FunctionStringReplace<ReplaceStringImpl<NameReplaceAl
 REGISTER_FUNCTION(ReplaceAll)
 {
     factory.registerFunction<FunctionReplaceAll>();
-    factory.registerAlias("replace", NameReplaceAll::name, FunctionFactory::CaseInsensitive);
+    factory.registerAlias("replace", NameReplaceAll::name, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/replaceRegexpAll.cpp b/src/Functions/replaceRegexpAll.cpp
index f5f56fb0f35..77f21b6efee 100644
--- a/src/Functions/replaceRegexpAll.cpp
+++ b/src/Functions/replaceRegexpAll.cpp
@@ -20,7 +20,7 @@ using FunctionReplaceRegexpAll = FunctionStringReplace<ReplaceRegexpImpl<NameRep
 REGISTER_FUNCTION(ReplaceRegexpAll)
 {
     factory.registerFunction<FunctionReplaceRegexpAll>();
-    factory.registerAlias("REGEXP_REPLACE", NameReplaceRegexpAll::name, FunctionFactory::CaseInsensitive);
+    factory.registerAlias("REGEXP_REPLACE", NameReplaceRegexpAll::name, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/reverse.cpp b/src/Functions/reverse.cpp
index 39608b77997..d23e48b8d42 100644
--- a/src/Functions/reverse.cpp
+++ b/src/Functions/reverse.cpp
@@ -113,7 +113,7 @@ private:
 
 REGISTER_FUNCTION(Reverse)
 {
-    factory.registerFunction<ReverseOverloadResolver>({}, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<ReverseOverloadResolver>({}, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/right.cpp b/src/Functions/right.cpp
index a8ab4bf9685..ef3303ab968 100644
--- a/src/Functions/right.cpp
+++ b/src/Functions/right.cpp
@@ -6,8 +6,8 @@ namespace DB
 
 REGISTER_FUNCTION(Right)
 {
-    factory.registerFunction<FunctionLeftRight<false, SubstringDirection::Right>>({}, FunctionFactory::CaseInsensitive);
-    factory.registerFunction<FunctionLeftRight<true, SubstringDirection::Right>>({}, FunctionFactory::CaseSensitive);
+    factory.registerFunction<FunctionLeftRight<false, SubstringDirection::Right>>({}, FunctionFactory::Case::Insensitive);
+    factory.registerFunction<FunctionLeftRight<true, SubstringDirection::Right>>({}, FunctionFactory::Case::Sensitive);
 }
 
 }
diff --git a/src/Functions/serverConstants.cpp b/src/Functions/serverConstants.cpp
index 761c8964f12..fe999d66701 100644
--- a/src/Functions/serverConstants.cpp
+++ b/src/Functions/serverConstants.cpp
@@ -206,12 +206,12 @@ REGISTER_FUNCTION(Uptime)
 
 REGISTER_FUNCTION(Version)
 {
-    factory.registerFunction<FunctionVersion>({}, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<FunctionVersion>({}, FunctionFactory::Case::Insensitive);
 }
 
 REGISTER_FUNCTION(Revision)
 {
-    factory.registerFunction<FunctionRevision>({}, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<FunctionRevision>({}, FunctionFactory::Case::Insensitive);
 }
 
 REGISTER_FUNCTION(ZooKeeperSessionUptime)
@@ -237,8 +237,7 @@ Returns the value of `display_name` from config or server FQDN if not set.
 )",
             .examples{{"displayName", "SELECT displayName();", ""}},
             .categories{"Constant", "Miscellaneous"}
-        },
-        FunctionFactory::CaseSensitive);
+        });
 }
 
 
diff --git a/src/Functions/sign.cpp b/src/Functions/sign.cpp
index 3dd2ac8e3aa..914e1ad9e1f 100644
--- a/src/Functions/sign.cpp
+++ b/src/Functions/sign.cpp
@@ -44,7 +44,7 @@ struct FunctionUnaryArithmeticMonotonicity<NameSign>
 
 REGISTER_FUNCTION(Sign)
 {
-    factory.registerFunction<FunctionSign>({}, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<FunctionSign>({}, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/sin.cpp b/src/Functions/sin.cpp
index 914f431adb4..945486b26a8 100644
--- a/src/Functions/sin.cpp
+++ b/src/Functions/sin.cpp
@@ -21,7 +21,7 @@ REGISTER_FUNCTION(Sin)
             .returned_value = "The sine of x.",
             .examples = {{.name = "simple", .query = "SELECT sin(1.23)", .result = "0.9424888019316975"}},
             .categories{"Mathematical", "Trigonometric"}},
-        FunctionFactory::CaseInsensitive);
+        FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/soundex.cpp b/src/Functions/soundex.cpp
index 77ddb14a6ec..fcf1523d1a3 100644
--- a/src/Functions/soundex.cpp
+++ b/src/Functions/soundex.cpp
@@ -112,7 +112,7 @@ struct NameSoundex
 REGISTER_FUNCTION(Soundex)
 {
     factory.registerFunction<FunctionStringToString<SoundexImpl, NameSoundex>>(
-        FunctionDocumentation{.description="Returns Soundex code of a string."}, FunctionFactory::CaseInsensitive);
+        FunctionDocumentation{.description="Returns Soundex code of a string."}, FunctionFactory::Case::Insensitive);
 }
 
 
diff --git a/src/Functions/space.cpp b/src/Functions/space.cpp
index ce12f2f541c..cd6ca73c088 100644
--- a/src/Functions/space.cpp
+++ b/src/Functions/space.cpp
@@ -173,7 +173,7 @@ public:
 
 REGISTER_FUNCTION(Space)
 {
-    factory.registerFunction<FunctionSpace>({}, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<FunctionSpace>({}, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/sqrt.cpp b/src/Functions/sqrt.cpp
index 3c50f994391..a6e2dee71d9 100644
--- a/src/Functions/sqrt.cpp
+++ b/src/Functions/sqrt.cpp
@@ -13,7 +13,7 @@ using FunctionSqrt = FunctionMathUnary<UnaryFunctionVectorized<SqrtName, sqrt>>;
 
 REGISTER_FUNCTION(Sqrt)
 {
-    factory.registerFunction<FunctionSqrt>({}, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<FunctionSqrt>({}, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/structureToFormatSchema.cpp b/src/Functions/structureToFormatSchema.cpp
index 406da372c04..4fc2bf707a4 100644
--- a/src/Functions/structureToFormatSchema.cpp
+++ b/src/Functions/structureToFormatSchema.cpp
@@ -116,8 +116,7 @@ Function that converts ClickHouse table structure to CapnProto format schema
 "}"},
             },
             .categories{"Other"}
-        },
-        FunctionFactory::CaseSensitive);
+        });
 }
 
 
@@ -138,8 +137,7 @@ Function that converts ClickHouse table structure to Protobuf format schema
 "}"},
             },
             .categories{"Other"}
-        },
-        FunctionFactory::CaseSensitive);
+        });
 }
 
 }
diff --git a/src/Functions/substring.cpp b/src/Functions/substring.cpp
index f1dea7db018..51980eb6b9c 100644
--- a/src/Functions/substring.cpp
+++ b/src/Functions/substring.cpp
@@ -201,12 +201,12 @@ public:
 
 REGISTER_FUNCTION(Substring)
 {
-    factory.registerFunction<FunctionSubstring<false>>({}, FunctionFactory::CaseInsensitive);
-    factory.registerAlias("substr", "substring", FunctionFactory::CaseInsensitive); // MySQL alias
-    factory.registerAlias("mid", "substring", FunctionFactory::CaseInsensitive); /// MySQL alias
-    factory.registerAlias("byteSlice", "substring", FunctionFactory::CaseInsensitive); /// resembles PostgreSQL's get_byte function, similar to ClickHouse's bitSlice
+    factory.registerFunction<FunctionSubstring<false>>({}, FunctionFactory::Case::Insensitive);
+    factory.registerAlias("substr", "substring", FunctionFactory::Case::Insensitive); // MySQL alias
+    factory.registerAlias("mid", "substring", FunctionFactory::Case::Insensitive); /// MySQL alias
+    factory.registerAlias("byteSlice", "substring", FunctionFactory::Case::Insensitive); /// resembles PostgreSQL's get_byte function, similar to ClickHouse's bitSlice
 
-    factory.registerFunction<FunctionSubstring<true>>({}, FunctionFactory::CaseSensitive);
+    factory.registerFunction<FunctionSubstring<true>>();
 }
 
 }
diff --git a/src/Functions/substringIndex.cpp b/src/Functions/substringIndex.cpp
index 15a321bd5b0..eccd849059b 100644
--- a/src/Functions/substringIndex.cpp
+++ b/src/Functions/substringIndex.cpp
@@ -314,7 +314,7 @@ REGISTER_FUNCTION(SubstringIndex)
     factory.registerFunction<FunctionSubstringIndex<false>>(); /// substringIndex
     factory.registerFunction<FunctionSubstringIndex<true>>(); /// substringIndexUTF8
 
-    factory.registerAlias("SUBSTRING_INDEX", "substringIndex", FunctionFactory::CaseInsensitive);
+    factory.registerAlias("SUBSTRING_INDEX", "substringIndex", FunctionFactory::Case::Insensitive);
 }
 
 
diff --git a/src/Functions/synonyms.cpp b/src/Functions/synonyms.cpp
index 28dd83627d9..18c1557115f 100644
--- a/src/Functions/synonyms.cpp
+++ b/src/Functions/synonyms.cpp
@@ -121,7 +121,7 @@ public:
 
 REGISTER_FUNCTION(Synonyms)
 {
-    factory.registerFunction<FunctionSynonyms>({}, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<FunctionSynonyms>({}, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/tan.cpp b/src/Functions/tan.cpp
index e39f8598419..51cf0bbcceb 100644
--- a/src/Functions/tan.cpp
+++ b/src/Functions/tan.cpp
@@ -13,7 +13,7 @@ using FunctionTan = FunctionMathUnary<UnaryFunctionVectorized<TanName, tan>>;
 
 REGISTER_FUNCTION(Tan)
 {
-    factory.registerFunction<FunctionTan>({}, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<FunctionTan>({}, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/tanh.cpp b/src/Functions/tanh.cpp
index bdefa5263d7..62755737f70 100644
--- a/src/Functions/tanh.cpp
+++ b/src/Functions/tanh.cpp
@@ -39,7 +39,7 @@ using FunctionTanh = FunctionMathUnary<UnaryFunctionVectorized<TanhName, tanh>>;
 
 REGISTER_FUNCTION(Tanh)
 {
-    factory.registerFunction<FunctionTanh>({}, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<FunctionTanh>({}, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/timestamp.cpp b/src/Functions/timestamp.cpp
index 6f2bd2030d5..c2e10a2d220 100644
--- a/src/Functions/timestamp.cpp
+++ b/src/Functions/timestamp.cpp
@@ -187,7 +187,7 @@ If the second argument 'expr_time' is provided, it adds the specified time to th
             {"timestamp", "SELECT timestamp('2013-12-31 12:00:00')", "2013-12-31 12:00:00.000000"},
             {"timestamp", "SELECT timestamp('2013-12-31 12:00:00', '12:00:00.11')", "2014-01-01 00:00:00.110000"},
         },
-        .categories{"DateTime"}}, FunctionFactory::CaseInsensitive);
+        .categories{"DateTime"}}, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/toCustomWeek.cpp b/src/Functions/toCustomWeek.cpp
index 98e7aaf1d6b..61c0767654e 100644
--- a/src/Functions/toCustomWeek.cpp
+++ b/src/Functions/toCustomWeek.cpp
@@ -21,8 +21,8 @@ REGISTER_FUNCTION(ToCustomWeek)
     factory.registerFunction<FunctionToLastDayOfWeek>();
 
     /// Compatibility aliases for mysql.
-    factory.registerAlias("week", "toWeek", FunctionFactory::CaseInsensitive);
-    factory.registerAlias("yearweek", "toYearWeek", FunctionFactory::CaseInsensitive);
+    factory.registerAlias("week", "toWeek", FunctionFactory::Case::Insensitive);
+    factory.registerAlias("yearweek", "toYearWeek", FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/toDayOfMonth.cpp b/src/Functions/toDayOfMonth.cpp
index c20b0b75797..93013c3528b 100644
--- a/src/Functions/toDayOfMonth.cpp
+++ b/src/Functions/toDayOfMonth.cpp
@@ -14,8 +14,8 @@ REGISTER_FUNCTION(ToDayOfMonth)
     factory.registerFunction<FunctionToDayOfMonth>();
 
     /// MySQL compatibility alias.
-    factory.registerAlias("DAY", "toDayOfMonth", FunctionFactory::CaseInsensitive);
-    factory.registerAlias("DAYOFMONTH", "toDayOfMonth", FunctionFactory::CaseInsensitive);
+    factory.registerAlias("DAY", "toDayOfMonth", FunctionFactory::Case::Insensitive);
+    factory.registerAlias("DAYOFMONTH", "toDayOfMonth", FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/toDayOfWeek.cpp b/src/Functions/toDayOfWeek.cpp
index dc508d70814..d1f55bbddab 100644
--- a/src/Functions/toDayOfWeek.cpp
+++ b/src/Functions/toDayOfWeek.cpp
@@ -13,7 +13,7 @@ REGISTER_FUNCTION(ToDayOfWeek)
     factory.registerFunction<FunctionToDayOfWeek>();
 
     /// MySQL compatibility alias.
-    factory.registerAlias("DAYOFWEEK", "toDayOfWeek", FunctionFactory::CaseInsensitive);
+    factory.registerAlias("DAYOFWEEK", "toDayOfWeek", FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/toDayOfYear.cpp b/src/Functions/toDayOfYear.cpp
index 0cbafd6275a..9a27c41b0ed 100644
--- a/src/Functions/toDayOfYear.cpp
+++ b/src/Functions/toDayOfYear.cpp
@@ -14,7 +14,7 @@ REGISTER_FUNCTION(ToDayOfYear)
     factory.registerFunction<FunctionToDayOfYear>();
 
     /// MySQL compatibility alias.
-    factory.registerAlias("DAYOFYEAR", "toDayOfYear", FunctionFactory::CaseInsensitive);
+    factory.registerAlias("DAYOFYEAR", "toDayOfYear", FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/toDaysSinceYearZero.cpp b/src/Functions/toDaysSinceYearZero.cpp
index f6239b2900b..b5c053a11b3 100644
--- a/src/Functions/toDaysSinceYearZero.cpp
+++ b/src/Functions/toDaysSinceYearZero.cpp
@@ -20,7 +20,7 @@ The calculation is the same as in MySQL's TO_DAYS() function.
         .categories{"Dates and Times"}});
 
     /// MySQL compatibility alias.
-    factory.registerAlias("TO_DAYS", FunctionToDaysSinceYearZero::name, FunctionFactory::CaseInsensitive);
+    factory.registerAlias("TO_DAYS", FunctionToDaysSinceYearZero::name, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/toDecimalString.cpp b/src/Functions/toDecimalString.cpp
index 4ee664ad237..523948a5396 100644
--- a/src/Functions/toDecimalString.cpp
+++ b/src/Functions/toDecimalString.cpp
@@ -273,7 +273,7 @@ second argument is the desired number of digits in fractional part. Returns Stri
         )",
             .examples{{"toDecimalString", "SELECT toDecimalString(2.1456,2)", ""}},
             .categories{"String"}
-        }, FunctionFactory::CaseInsensitive);
+        }, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/toHour.cpp b/src/Functions/toHour.cpp
index fc9ec657adf..bc122538661 100644
--- a/src/Functions/toHour.cpp
+++ b/src/Functions/toHour.cpp
@@ -14,7 +14,7 @@ REGISTER_FUNCTION(ToHour)
     factory.registerFunction<FunctionToHour>();
 
     /// MySQL compatibility alias.
-    factory.registerAlias("HOUR", "toHour", FunctionFactory::CaseInsensitive);
+    factory.registerAlias("HOUR", "toHour", FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/toLastDayOfMonth.cpp b/src/Functions/toLastDayOfMonth.cpp
index 9365880bfb8..004ae2718e7 100644
--- a/src/Functions/toLastDayOfMonth.cpp
+++ b/src/Functions/toLastDayOfMonth.cpp
@@ -13,7 +13,7 @@ REGISTER_FUNCTION(ToLastDayOfMonth)
     factory.registerFunction<FunctionToLastDayOfMonth>();
 
     /// MySQL compatibility alias.
-    factory.registerAlias("LAST_DAY", "toLastDayOfMonth", FunctionFactory::CaseInsensitive);
+    factory.registerAlias("LAST_DAY", "toLastDayOfMonth", FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/toMillisecond.cpp b/src/Functions/toMillisecond.cpp
index aaef517c996..efa08c322a2 100644
--- a/src/Functions/toMillisecond.cpp
+++ b/src/Functions/toMillisecond.cpp
@@ -27,7 +27,7 @@ Returns the millisecond component (0-999) of a date with time.
             );
 
     /// MySQL compatibility alias.
-    factory.registerAlias("MILLISECOND", "toMillisecond", FunctionFactory::CaseInsensitive);
+    factory.registerAlias("MILLISECOND", "toMillisecond", FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/toMinute.cpp b/src/Functions/toMinute.cpp
index 162ecb282df..291da33d2e8 100644
--- a/src/Functions/toMinute.cpp
+++ b/src/Functions/toMinute.cpp
@@ -14,7 +14,7 @@ REGISTER_FUNCTION(ToMinute)
     factory.registerFunction<FunctionToMinute>();
 
     /// MySQL compatibility alias.
-    factory.registerAlias("MINUTE", "toMinute", FunctionFactory::CaseInsensitive);
+    factory.registerAlias("MINUTE", "toMinute", FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/toMonth.cpp b/src/Functions/toMonth.cpp
index 422f21e7df8..3ef73bf1be3 100644
--- a/src/Functions/toMonth.cpp
+++ b/src/Functions/toMonth.cpp
@@ -13,7 +13,7 @@ REGISTER_FUNCTION(ToMonth)
 {
     factory.registerFunction<FunctionToMonth>();
     /// MySQL compatibility alias.
-    factory.registerAlias("MONTH", "toMonth", FunctionFactory::CaseInsensitive);
+    factory.registerAlias("MONTH", "toMonth", FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/toQuarter.cpp b/src/Functions/toQuarter.cpp
index 3c301095ff2..2e6d4fa93de 100644
--- a/src/Functions/toQuarter.cpp
+++ b/src/Functions/toQuarter.cpp
@@ -13,7 +13,7 @@ REGISTER_FUNCTION(ToQuarter)
 {
     factory.registerFunction<FunctionToQuarter>();
     /// MySQL compatibility alias.
-    factory.registerAlias("QUARTER", "toQuarter", FunctionFactory::CaseInsensitive);
+    factory.registerAlias("QUARTER", "toQuarter", FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/toSecond.cpp b/src/Functions/toSecond.cpp
index 372097fd488..1ad3b46fbd7 100644
--- a/src/Functions/toSecond.cpp
+++ b/src/Functions/toSecond.cpp
@@ -14,7 +14,7 @@ REGISTER_FUNCTION(ToSecond)
     factory.registerFunction<FunctionToSecond>();
 
     /// MySQL compatibility alias.
-    factory.registerAlias("SECOND", "toSecond", FunctionFactory::CaseInsensitive);
+    factory.registerAlias("SECOND", "toSecond", FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/toYear.cpp b/src/Functions/toYear.cpp
index 75479adb82c..0d2c8136337 100644
--- a/src/Functions/toYear.cpp
+++ b/src/Functions/toYear.cpp
@@ -14,7 +14,7 @@ REGISTER_FUNCTION(ToYear)
     factory.registerFunction<FunctionToYear>();
 
     /// MySQL compatibility alias.
-    factory.registerAlias("YEAR", "toYear", FunctionFactory::CaseInsensitive);
+    factory.registerAlias("YEAR", "toYear", FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/today.cpp b/src/Functions/today.cpp
index 356660fa7b5..88eddc9b60e 100644
--- a/src/Functions/today.cpp
+++ b/src/Functions/today.cpp
@@ -84,8 +84,8 @@ public:
 REGISTER_FUNCTION(Today)
 {
     factory.registerFunction<TodayOverloadResolver>();
-    factory.registerAlias("current_date", TodayOverloadResolver::name, FunctionFactory::CaseInsensitive);
-    factory.registerAlias("curdate", TodayOverloadResolver::name, FunctionFactory::CaseInsensitive);
+    factory.registerAlias("current_date", TodayOverloadResolver::name, FunctionFactory::Case::Insensitive);
+    factory.registerAlias("curdate", TodayOverloadResolver::name, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/upper.cpp b/src/Functions/upper.cpp
index 3e1c7b1d800..5af0f059e3f 100644
--- a/src/Functions/upper.cpp
+++ b/src/Functions/upper.cpp
@@ -18,8 +18,8 @@ using FunctionUpper = FunctionStringToString<LowerUpperImpl<'a', 'z'>, NameUpper
 
 REGISTER_FUNCTION(Upper)
 {
-    factory.registerFunction<FunctionUpper>({}, FunctionFactory::CaseInsensitive);
-    factory.registerAlias("ucase", FunctionUpper::name, FunctionFactory::CaseInsensitive);
+    factory.registerFunction<FunctionUpper>({}, FunctionFactory::Case::Insensitive);
+    factory.registerAlias("ucase", FunctionUpper::name, FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Functions/vectorFunctions.cpp b/src/Functions/vectorFunctions.cpp
index 7a2598f5f4b..5e23493c86d 100644
--- a/src/Functions/vectorFunctions.cpp
+++ b/src/Functions/vectorFunctions.cpp
@@ -1576,9 +1576,9 @@ using TupleOrArrayFunctionCosineDistance = TupleOrArrayFunction<CosineDistanceTr
 REGISTER_FUNCTION(VectorFunctions)
 {
     factory.registerFunction<FunctionTuplePlus>();
-    factory.registerAlias("vectorSum", FunctionTuplePlus::name, FunctionFactory::CaseInsensitive);
+    factory.registerAlias("vectorSum", FunctionTuplePlus::name, FunctionFactory::Case::Insensitive);
     factory.registerFunction<FunctionTupleMinus>();
-    factory.registerAlias("vectorDifference", FunctionTupleMinus::name, FunctionFactory::CaseInsensitive);
+    factory.registerAlias("vectorDifference", FunctionTupleMinus::name, FunctionFactory::Case::Insensitive);
     factory.registerFunction<FunctionTupleMultiply>();
     factory.registerFunction<FunctionTupleDivide>();
     factory.registerFunction<FunctionTupleModulo>();
@@ -1652,7 +1652,7 @@ If the types of the first interval (or the interval in the tuple) and the second
     factory.registerFunction<FunctionTupleIntDivOrZeroByNumber>();
 
     factory.registerFunction<TupleOrArrayFunctionDotProduct>();
-    factory.registerAlias("scalarProduct", TupleOrArrayFunctionDotProduct::name, FunctionFactory::CaseInsensitive);
+    factory.registerAlias("scalarProduct", TupleOrArrayFunctionDotProduct::name, FunctionFactory::Case::Insensitive);
 
     factory.registerFunction<TupleOrArrayFunctionL1Norm>();
     factory.registerFunction<TupleOrArrayFunctionL2Norm>();
@@ -1660,11 +1660,11 @@ If the types of the first interval (or the interval in the tuple) and the second
     factory.registerFunction<TupleOrArrayFunctionLinfNorm>();
     factory.registerFunction<TupleOrArrayFunctionLpNorm>();
 
-    factory.registerAlias("normL1", TupleOrArrayFunctionL1Norm::name, FunctionFactory::CaseInsensitive);
-    factory.registerAlias("normL2", TupleOrArrayFunctionL2Norm::name, FunctionFactory::CaseInsensitive);
-    factory.registerAlias("normL2Squared", TupleOrArrayFunctionL2SquaredNorm::name, FunctionFactory::CaseInsensitive);
-    factory.registerAlias("normLinf", TupleOrArrayFunctionLinfNorm::name, FunctionFactory::CaseInsensitive);
-    factory.registerAlias("normLp", FunctionLpNorm::name, FunctionFactory::CaseInsensitive);
+    factory.registerAlias("normL1", TupleOrArrayFunctionL1Norm::name, FunctionFactory::Case::Insensitive);
+    factory.registerAlias("normL2", TupleOrArrayFunctionL2Norm::name, FunctionFactory::Case::Insensitive);
+    factory.registerAlias("normL2Squared", TupleOrArrayFunctionL2SquaredNorm::name, FunctionFactory::Case::Insensitive);
+    factory.registerAlias("normLinf", TupleOrArrayFunctionLinfNorm::name, FunctionFactory::Case::Insensitive);
+    factory.registerAlias("normLp", FunctionLpNorm::name, FunctionFactory::Case::Insensitive);
 
     factory.registerFunction<TupleOrArrayFunctionL1Distance>();
     factory.registerFunction<TupleOrArrayFunctionL2Distance>();
@@ -1672,21 +1672,21 @@ If the types of the first interval (or the interval in the tuple) and the second
     factory.registerFunction<TupleOrArrayFunctionLinfDistance>();
     factory.registerFunction<TupleOrArrayFunctionLpDistance>();
 
-    factory.registerAlias("distanceL1", FunctionL1Distance::name, FunctionFactory::CaseInsensitive);
-    factory.registerAlias("distanceL2", FunctionL2Distance::name, FunctionFactory::CaseInsensitive);
-    factory.registerAlias("distanceL2Squared", FunctionL2SquaredDistance::name, FunctionFactory::CaseInsensitive);
-    factory.registerAlias("distanceLinf", FunctionLinfDistance::name, FunctionFactory::CaseInsensitive);
-    factory.registerAlias("distanceLp", FunctionLpDistance::name, FunctionFactory::CaseInsensitive);
+    factory.registerAlias("distanceL1", FunctionL1Distance::name, FunctionFactory::Case::Insensitive);
+    factory.registerAlias("distanceL2", FunctionL2Distance::name, FunctionFactory::Case::Insensitive);
+    factory.registerAlias("distanceL2Squared", FunctionL2SquaredDistance::name, FunctionFactory::Case::Insensitive);
+    factory.registerAlias("distanceLinf", FunctionLinfDistance::name, FunctionFactory::Case::Insensitive);
+    factory.registerAlias("distanceLp", FunctionLpDistance::name, FunctionFactory::Case::Insensitive);
 
     factory.registerFunction<FunctionL1Normalize>();
     factory.registerFunction<FunctionL2Normalize>();
     factory.registerFunction<FunctionLinfNormalize>();
     factory.registerFunction<FunctionLpNormalize>();
 
-    factory.registerAlias("normalizeL1", FunctionL1Normalize::name, FunctionFactory::CaseInsensitive);
-    factory.registerAlias("normalizeL2", FunctionL2Normalize::name, FunctionFactory::CaseInsensitive);
-    factory.registerAlias("normalizeLinf", FunctionLinfNormalize::name, FunctionFactory::CaseInsensitive);
-    factory.registerAlias("normalizeLp", FunctionLpNormalize::name, FunctionFactory::CaseInsensitive);
+    factory.registerAlias("normalizeL1", FunctionL1Normalize::name, FunctionFactory::Case::Insensitive);
+    factory.registerAlias("normalizeL2", FunctionL2Normalize::name, FunctionFactory::Case::Insensitive);
+    factory.registerAlias("normalizeLinf", FunctionLinfNormalize::name, FunctionFactory::Case::Insensitive);
+    factory.registerAlias("normalizeLp", FunctionLpNormalize::name, FunctionFactory::Case::Insensitive);
 
     factory.registerFunction<TupleOrArrayFunctionCosineDistance>();
 }
diff --git a/src/Functions/widthBucket.cpp b/src/Functions/widthBucket.cpp
index e804808b699..d007cc968f0 100644
--- a/src/Functions/widthBucket.cpp
+++ b/src/Functions/widthBucket.cpp
@@ -287,7 +287,7 @@ Result:
         .categories{"Mathematical"},
     });
 
-    factory.registerAlias("width_bucket", "widthBucket", FunctionFactory::CaseInsensitive);
+    factory.registerAlias("width_bucket", "widthBucket", FunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/Processors/Transforms/WindowTransform.cpp b/src/Processors/Transforms/WindowTransform.cpp
index a694fa43e46..86421adf4fb 100644
--- a/src/Processors/Transforms/WindowTransform.cpp
+++ b/src/Processors/Transforms/WindowTransform.cpp
@@ -2716,42 +2716,42 @@ void registerWindowFunctions(AggregateFunctionFactory & factory)
         {
             return std::make_shared<WindowFunctionRank>(name, argument_types,
                 parameters);
-        }, properties}, AggregateFunctionFactory::CaseInsensitive);
+        }, properties}, AggregateFunctionFactory::Case::Insensitive);
 
     factory.registerFunction("dense_rank", {[](const std::string & name,
             const DataTypes & argument_types, const Array & parameters, const Settings *)
         {
             return std::make_shared<WindowFunctionDenseRank>(name, argument_types,
                 parameters);
-        }, properties}, AggregateFunctionFactory::CaseInsensitive);
+        }, properties}, AggregateFunctionFactory::Case::Insensitive);
 
     factory.registerFunction("percent_rank", {[](const std::string & name,
             const DataTypes & argument_types, const Array & parameters, const Settings *)
         {
             return std::make_shared<WindowFunctionPercentRank>(name, argument_types,
                 parameters);
-        }, properties}, AggregateFunctionFactory::CaseInsensitive);
+        }, properties}, AggregateFunctionFactory::Case::Insensitive);
 
     factory.registerFunction("row_number", {[](const std::string & name,
             const DataTypes & argument_types, const Array & parameters, const Settings *)
         {
             return std::make_shared<WindowFunctionRowNumber>(name, argument_types,
                 parameters);
-        }, properties}, AggregateFunctionFactory::CaseInsensitive);
+        }, properties}, AggregateFunctionFactory::Case::Insensitive);
 
     factory.registerFunction("ntile", {[](const std::string & name,
             const DataTypes & argument_types, const Array & parameters, const Settings *)
         {
             return std::make_shared<WindowFunctionNtile>(name, argument_types,
                 parameters);
-        }, properties}, AggregateFunctionFactory::CaseInsensitive);
+        }, properties}, AggregateFunctionFactory::Case::Insensitive);
 
     factory.registerFunction("nth_value", {[](const std::string & name,
             const DataTypes & argument_types, const Array & parameters, const Settings *)
         {
             return std::make_shared<WindowFunctionNthValue>(
                 name, argument_types, parameters);
-        }, properties}, AggregateFunctionFactory::CaseInsensitive);
+        }, properties}, AggregateFunctionFactory::Case::Insensitive);
 
     factory.registerFunction("lagInFrame", {[](const std::string & name,
             const DataTypes & argument_types, const Array & parameters, const Settings *)
diff --git a/src/TableFunctions/TableFunctionFactory.cpp b/src/TableFunctions/TableFunctionFactory.cpp
index 6ecdeb47779..e505535ae76 100644
--- a/src/TableFunctions/TableFunctionFactory.cpp
+++ b/src/TableFunctions/TableFunctionFactory.cpp
@@ -19,17 +19,17 @@ namespace ErrorCodes
 }
 
 void TableFunctionFactory::registerFunction(
-    const std::string & name, Value value, CaseSensitiveness case_sensitiveness)
+    const std::string & name, Value value, Case case_sensitiveness)
 {
     if (!table_functions.emplace(name, value).second)
         throw Exception(ErrorCodes::LOGICAL_ERROR, "TableFunctionFactory: the table function name '{}' is not unique", name);
 
-    if (case_sensitiveness == CaseInsensitive
+    if (case_sensitiveness == Case::Insensitive
         && !case_insensitive_table_functions.emplace(Poco::toLower(name), value).second)
         throw Exception(ErrorCodes::LOGICAL_ERROR, "TableFunctionFactory: "
                         "the case insensitive table function name '{}' is not unique", name);
 
-    KnownTableFunctionNames::instance().add(name, (case_sensitiveness == CaseInsensitive));
+    KnownTableFunctionNames::instance().add(name, (case_sensitiveness == Case::Insensitive));
 }
 
 TableFunctionPtr TableFunctionFactory::get(
diff --git a/src/TableFunctions/TableFunctionFactory.h b/src/TableFunctions/TableFunctionFactory.h
index 2cc648ba181..adc74c2e735 100644
--- a/src/TableFunctions/TableFunctionFactory.h
+++ b/src/TableFunctions/TableFunctionFactory.h
@@ -48,10 +48,10 @@ public:
     void registerFunction(
         const std::string & name,
         Value value,
-        CaseSensitiveness case_sensitiveness = CaseSensitive);
+        Case case_sensitiveness = Case::Sensitive);
 
     template <typename Function>
-    void registerFunction(TableFunctionProperties properties = {}, CaseSensitiveness case_sensitiveness = CaseSensitive)
+    void registerFunction(TableFunctionProperties properties = {}, Case case_sensitiveness = Case::Sensitive)
     {
         auto creator = []() -> TableFunctionPtr { return std::make_shared<Function>(); };
         registerFunction(Function::name,
diff --git a/src/TableFunctions/TableFunctionFormat.cpp b/src/TableFunctions/TableFunctionFormat.cpp
index 5a0f47653d5..7e4fdea1ff3 100644
--- a/src/TableFunctions/TableFunctionFormat.cpp
+++ b/src/TableFunctions/TableFunctionFormat.cpp
@@ -219,7 +219,7 @@ Result:
 
 void registerTableFunctionFormat(TableFunctionFactory & factory)
 {
-    factory.registerFunction<TableFunctionFormat>({format_table_function_documentation, false}, TableFunctionFactory::CaseInsensitive);
+    factory.registerFunction<TableFunctionFormat>({format_table_function_documentation, false}, TableFunctionFactory::Case::Insensitive);
 }
 
 }
diff --git a/src/TableFunctions/TableFunctionValues.cpp b/src/TableFunctions/TableFunctionValues.cpp
index 4b56fa57091..95c531f8a3b 100644
--- a/src/TableFunctions/TableFunctionValues.cpp
+++ b/src/TableFunctions/TableFunctionValues.cpp
@@ -174,7 +174,7 @@ StoragePtr TableFunctionValues::executeImpl(const ASTPtr & ast_function, Context
 
 void registerTableFunctionValues(TableFunctionFactory & factory)
 {
-    factory.registerFunction<TableFunctionValues>({.documentation = {}, .allow_readonly = true}, TableFunctionFactory::CaseInsensitive);
+    factory.registerFunction<TableFunctionValues>({.documentation = {}, .allow_readonly = true}, TableFunctionFactory::Case::Insensitive);
 }
 
 }

From 3985a4012d70762480d50083eb0d30a3b70e0981 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Thu, 18 Jul 2024 00:06:25 +0200
Subject: [PATCH 0378/1488] Fix tidy

---
 src/Storages/MergeTree/MergeTreeBackgroundExecutor.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Storages/MergeTree/MergeTreeBackgroundExecutor.cpp b/src/Storages/MergeTree/MergeTreeBackgroundExecutor.cpp
index 4e62c503d65..4a4deb07eee 100644
--- a/src/Storages/MergeTree/MergeTreeBackgroundExecutor.cpp
+++ b/src/Storages/MergeTree/MergeTreeBackgroundExecutor.cpp
@@ -155,7 +155,7 @@ void printExceptionWithRespectToAbort(LoggerPtr log, const String & query_id)
     {
         std::rethrow_exception(ex);
     }
-    catch (const TestException &)
+    catch (const TestException &) // NOLINT
     {
         /// Exception from a unit test, ignore it.
     }

From b8cf8829927dedc701b97893b3a1e7651193b586 Mon Sep 17 00:00:00 2001
From: Nikita Taranov <nikita.taranov@clickhouse.com>
Date: Wed, 17 Jul 2024 23:22:24 +0100
Subject: [PATCH 0379/1488] impl

---
 src/Common/CgroupsMemoryUsageObserver.cpp | 49 ++++++++++++++++++-----
 src/Common/CgroupsMemoryUsageObserver.h   |  2 +
 2 files changed, 40 insertions(+), 11 deletions(-)

diff --git a/src/Common/CgroupsMemoryUsageObserver.cpp b/src/Common/CgroupsMemoryUsageObserver.cpp
index 02bde0d80b7..cf661174789 100644
--- a/src/Common/CgroupsMemoryUsageObserver.cpp
+++ b/src/Common/CgroupsMemoryUsageObserver.cpp
@@ -11,6 +11,7 @@
 #include <base/cgroupsv2.h>
 #include <base/getMemoryAmount.h>
 #include <base/sleep.h>
+#include <fmt/ranges.h>
 
 #include <cstdint>
 #include <filesystem>
@@ -45,26 +46,33 @@ namespace
 ///   kernel 5
 ///   rss 15
 ///   [...]
-uint64_t readMetricFromStatFile(ReadBufferFromFile & buf, const std::string & key)
+using Metrics = std::map<std::string, uint64_t>;
+
+Metrics readAllMetricsFromStatFile(ReadBufferFromFile & buf)
 {
+    Metrics metrics;
     while (!buf.eof())
     {
         std::string current_key;
         readStringUntilWhitespace(current_key, buf);
-        if (current_key != key)
-        {
-            std::string dummy;
-            readStringUntilNewlineInto(dummy, buf);
-            buf.ignore();
-            continue;
-        }
 
         assertChar(' ', buf);
+
         uint64_t value = 0;
         readIntText(value, buf);
-        return value;
-    }
+        assertChar('\n', buf);
 
+        auto [_, inserted] = metrics.emplace(std::move(current_key), value);
+        chassert(inserted, "Duplicate keys in stat file");
+    }
+    return metrics;
+}
+
+uint64_t readMetricFromStatFile(ReadBufferFromFile & buf, const std::string & key)
+{
+    const auto all_metrics = readAllMetricsFromStatFile(buf);
+    if (const auto it = all_metrics.find(key); it != all_metrics.end())
+        return it->second;
     throw Exception(ErrorCodes::INCORRECT_DATA, "Cannot find '{}' in '{}'", key, buf.getFileName());
 }
 
@@ -79,6 +87,13 @@ struct CgroupsV1Reader : ICgroupsReader
         return readMetricFromStatFile(buf, "rss");
     }
 
+    std::string dumpAllStats() override
+    {
+        std::lock_guard lock(mutex);
+        buf.rewind();
+        return fmt::format("{}", readAllMetricsFromStatFile(buf));
+    }
+
 private:
     std::mutex mutex;
     ReadBufferFromFile buf TSA_GUARDED_BY(mutex);
@@ -106,6 +121,13 @@ struct CgroupsV2Reader : ICgroupsReader
         return mem_usage;
     }
 
+    std::string dumpAllStats() override
+    {
+        std::lock_guard lock(mutex);
+        stat_buf.rewind();
+        return fmt::format("{}", readAllMetricsFromStatFile(stat_buf));
+    }
+
 private:
     std::mutex mutex;
     ReadBufferFromFile current_buf TSA_GUARDED_BY(mutex);
@@ -234,7 +256,12 @@ void CgroupsMemoryUsageObserver::setMemoryUsageLimits(uint64_t hard_limit_, uint
 #    endif
             /// Reset current usage in memory tracker. Expect zero for free_memory_in_allocator_arenas as we just purged them.
             uint64_t memory_usage = cgroup_reader->readMemoryUsage();
-            LOG_TRACE(log, "Read current memory usage {} bytes ({}) from cgroups", memory_usage, ReadableSize(memory_usage));
+            LOG_TRACE(
+                log,
+                "Read current memory usage {} bytes ({}) from cgroups, full available stats: {}",
+                memory_usage,
+                ReadableSize(memory_usage),
+                cgroup_reader->dumpAllStats());
             MemoryTracker::setRSS(memory_usage, 0);
 
             LOG_INFO(log, "Purged jemalloc arenas. Current memory usage is {}", ReadableSize(memory_usage));
diff --git a/src/Common/CgroupsMemoryUsageObserver.h b/src/Common/CgroupsMemoryUsageObserver.h
index b848a2bff3c..0d5d07597c8 100644
--- a/src/Common/CgroupsMemoryUsageObserver.h
+++ b/src/Common/CgroupsMemoryUsageObserver.h
@@ -14,6 +14,8 @@ struct ICgroupsReader
     virtual ~ICgroupsReader() = default;
 
     virtual uint64_t readMemoryUsage() = 0;
+
+    virtual std::string dumpAllStats() = 0;
 };
 
 /// Does two things:

From 787a9137df53d6c46f0b6f47ba21b23f49f8e749 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Thu, 18 Jul 2024 03:08:34 +0200
Subject: [PATCH 0380/1488] Fix bad test `02210_processors_profile_log`

---
 tests/queries/0_stateless/02210_processors_profile_log.sql | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tests/queries/0_stateless/02210_processors_profile_log.sql b/tests/queries/0_stateless/02210_processors_profile_log.sql
index 59edbb71457..75e5bcbb585 100644
--- a/tests/queries/0_stateless/02210_processors_profile_log.sql
+++ b/tests/queries/0_stateless/02210_processors_profile_log.sql
@@ -14,10 +14,12 @@ SELECT
     multiIf(
         -- ExpressionTransform executes sleep(),
         -- so IProcessor::work() will spend 1 sec.
-        name = 'ExpressionTransform', elapsed_us>=1e6 ? 1 : elapsed_us,
+        -- We use two different timers to measure time: CLOCK_MONOTONIC for sleep and CLOCK_MONOTONIC_COARSE for profiling
+        -- that's why we cannot compare directly with 1,000,000 microseconds - let's compare with 900,000 microseconds.
+        name = 'ExpressionTransform', elapsed_us >= 0.9e6 ? 1 : elapsed_us,
         -- SourceFromSingleChunk, that feed data to ExpressionTransform,
         -- will feed first block and then wait in PortFull.
-        name = 'SourceFromSingleChunk', output_wait_elapsed_us>=1e6 ? 1 : output_wait_elapsed_us,
+        name = 'SourceFromSingleChunk', output_wait_elapsed_us >= 0.9e6 ? 1 : output_wait_elapsed_us,
         -- NullSource/LazyOutputFormatLazyOutputFormat are the outputs
         -- so they cannot starts to execute before sleep(1) will be executed.
         input_wait_elapsed_us>=1e6 ? 1 : input_wait_elapsed_us)

From 4217d029c0f6b69b0b67c414f2b6df750392561b Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Thu, 18 Jul 2024 03:33:36 +0200
Subject: [PATCH 0381/1488] Fix something

---
 programs/server/Server.cpp                    |  3 +-
 programs/server/Server.h                      |  3 +-
 src/Databases/IDatabase.h                     |  1 -
 src/Storages/MergeTree/AlterConversions.h     |  1 -
 .../MergeTree/MergeTreeDataMergerMutator.cpp  |  4 +-
 src/Storages/StorageMergeTree.cpp             | 50 +++++++------------
 src/Storages/StorageMergeTree.h               |  2 +-
 7 files changed, 23 insertions(+), 41 deletions(-)

diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp
index 053ddaf8d8b..0b695c3dde6 100644
--- a/programs/server/Server.cpp
+++ b/programs/server/Server.cpp
@@ -2731,8 +2731,7 @@ void Server::createInterserverServers(
 
 void Server::stopServers(
     std::vector<ProtocolServerAdapter> & servers,
-    const ServerType & server_type
-) const
+    const ServerType & server_type) const
 {
     LoggerRawPtr log = &logger();
 
diff --git a/programs/server/Server.h b/programs/server/Server.h
index 3f03dd137ef..feaf61f1ffd 100644
--- a/programs/server/Server.h
+++ b/programs/server/Server.h
@@ -129,8 +129,7 @@ private:
 
     void stopServers(
         std::vector<ProtocolServerAdapter> & servers,
-        const ServerType & server_type
-    ) const;
+        const ServerType & server_type) const;
 };
 
 }
diff --git a/src/Databases/IDatabase.h b/src/Databases/IDatabase.h
index ce5a52b1b0f..f94326d220e 100644
--- a/src/Databases/IDatabase.h
+++ b/src/Databases/IDatabase.h
@@ -6,7 +6,6 @@
 #include <Interpreters/executeQuery.h>
 #include <Parsers/IAST_fwd.h>
 #include <QueryPipeline/BlockIO.h>
-#include <Storages/IStorage.h>
 #include <Storages/IStorage_fwd.h>
 #include <base/types.h>
 #include <Common/AsyncLoader.h>
diff --git a/src/Storages/MergeTree/AlterConversions.h b/src/Storages/MergeTree/AlterConversions.h
index 0f857d351dd..7bcd4af8b36 100644
--- a/src/Storages/MergeTree/AlterConversions.h
+++ b/src/Storages/MergeTree/AlterConversions.h
@@ -8,7 +8,6 @@
 namespace DB
 {
 
-
 /// Alter conversions which should be applied on-fly for part.
 /// Built from of the most recent mutation commands for part.
 /// Now only ALTER RENAME COLUMN is applied.
diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp
index 3c223b8d748..140a226f2d1 100644
--- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp
+++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp
@@ -118,11 +118,11 @@ UInt64 MergeTreeDataMergerMutator::getMaxSourcePartSizeForMutation() const
         occupied >= data_settings->max_number_of_mutations_for_replica)
         return 0;
 
-    /// DataPart can be store only at one disk. Get maximum reservable free space at all disks.
+    /// A DataPart can be stored only at a single disk. Get the maximum reservable free space at all disks.
     UInt64 disk_space = data.getStoragePolicy()->getMaxUnreservedFreeSpace();
     auto max_tasks_count = data.getContext()->getMergeMutateExecutor()->getMaxTasksCount();
 
-    /// Allow mutations only if there are enough threads, leave free threads for merges else
+    /// Allow mutations only if there are enough threads, otherwise, leave free threads for merges.
     if (occupied <= 1
         || max_tasks_count - occupied >= data_settings->number_of_free_entries_in_pool_to_execute_mutation)
         return static_cast<UInt64>(disk_space / DISK_USAGE_COEFFICIENT_TO_RESERVE);
diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp
index 40b3a12297b..444b3fbae4c 100644
--- a/src/Storages/StorageMergeTree.cpp
+++ b/src/Storages/StorageMergeTree.cpp
@@ -11,22 +11,17 @@
 #include <Interpreters/ClusterProxy/SelectStreamFactory.h>
 #include <Interpreters/ClusterProxy/executeQuery.h>
 #include <Interpreters/Context.h>
-#include <Interpreters/InterpreterSelectQueryAnalyzer.h>
 #include <Interpreters/MutationsInterpreter.h>
 #include <Interpreters/PartLog.h>
 #include <Interpreters/TransactionLog.h>
 #include <Parsers/ASTCheckQuery.h>
 #include <Parsers/ASTFunction.h>
-#include <Parsers/ASTLiteral.h>
 #include <Parsers/ASTPartition.h>
-#include <Parsers/ASTSetQuery.h>
 #include <Parsers/formatAST.h>
-#include <Parsers/queryToString.h>
 #include <Planner/Utils.h>
 #include <Processors/QueryPlan/BuildQueryPipelineSettings.h>
 #include <Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h>
 #include <Processors/QueryPlan/QueryPlan.h>
-#include <QueryPipeline/Pipe.h>
 #include <Storages/AlterCommands.h>
 #include <Storages/MergeTree/ActiveDataPartSet.h>
 #include <Storages/MergeTree/IMergeTreeDataPart.h>
@@ -35,19 +30,14 @@
 #include <Storages/MergeTree/MergeTreeData.h>
 #include <Storages/MergeTree/MergeTreeSettings.h>
 #include <Storages/MergeTree/MergeTreeSink.h>
-#include <Storages/MergeTree/PartitionPruner.h>
 #include <Storages/MergeTree/checkDataPart.h>
 #include <Storages/PartitionCommands.h>
 #include <Storages/buildQueryTreeForShard.h>
-#include <base/sort.h>
 #include <fmt/core.h>
-#include <Poco/Timestamp.h>
 #include <Common/Exception.h>
 #include <Common/MemoryTracker.h>
 #include <Common/ProfileEventsScope.h>
-#include <Common/ThreadPool.h>
 #include <Common/escapeForFileName.h>
-#include <Common/typeid_cast.h>
 
 
 namespace DB
@@ -1197,7 +1187,6 @@ bool StorageMergeTree::merge(
     task->setCurrentTransaction(MergeTreeTransactionHolder{}, MergeTreeTransactionPtr{txn});
 
     executeHere(task);
-
     return true;
 }
 
@@ -1493,7 +1482,6 @@ bool StorageMergeTree::scheduleDataProcessingJob(BackgroundJobsAssignee & assign
         scheduled = true;
     }
 
-
     return scheduled;
 }
 
@@ -1600,9 +1588,7 @@ bool StorageMergeTree::optimize(
     if (!partition && final)
     {
         if (cleanup && this->merging_params.mode != MergingParams::Mode::Replacing)
-        {
             throw Exception(ErrorCodes::CANNOT_ASSIGN_OPTIMIZE, "Cannot OPTIMIZE with CLEANUP table: only ReplacingMergeTree can be CLEANUP");
-        }
 
         if (cleanup && !getSettings()->allow_experimental_replacing_merge_with_cleanup)
             throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Experimental merges with CLEANUP are not allowed");
@@ -1616,15 +1602,15 @@ bool StorageMergeTree::optimize(
         for (const String & partition_id : partition_ids)
         {
             if (!merge(
-                    true,
-                    partition_id,
-                    true,
-                    deduplicate,
-                    deduplicate_by_columns,
-                    cleanup,
-                    txn,
-                    disable_reason,
-                    local_context->getSettingsRef().optimize_skip_merged_partitions))
+                true,
+                partition_id,
+                true,
+                deduplicate,
+                deduplicate_by_columns,
+                cleanup,
+                txn,
+                disable_reason,
+                local_context->getSettingsRef().optimize_skip_merged_partitions))
             {
                 constexpr auto message = "Cannot OPTIMIZE table: {}";
                 if (disable_reason.text.empty())
@@ -1644,15 +1630,15 @@ bool StorageMergeTree::optimize(
             partition_id = getPartitionIDFromQuery(partition, local_context);
 
         if (!merge(
-                true,
-                partition_id,
-                final,
-                deduplicate,
-                deduplicate_by_columns,
-                cleanup,
-                txn,
-                disable_reason,
-                local_context->getSettingsRef().optimize_skip_merged_partitions))
+            true,
+            partition_id,
+            final,
+            deduplicate,
+            deduplicate_by_columns,
+            cleanup,
+            txn,
+            disable_reason,
+            local_context->getSettingsRef().optimize_skip_merged_partitions))
         {
             constexpr auto message = "Cannot OPTIMIZE table: {}";
             if (disable_reason.text.empty())
diff --git a/src/Storages/StorageMergeTree.h b/src/Storages/StorageMergeTree.h
index 4d819508934..064b51739bd 100644
--- a/src/Storages/StorageMergeTree.h
+++ b/src/Storages/StorageMergeTree.h
@@ -211,7 +211,6 @@ private:
         bool optimize_skip_merged_partitions = false,
         SelectPartsDecision * select_decision_out = nullptr);
 
-
     MergeMutateSelectedEntryPtr selectPartsToMutate(
         const StorageMetadataPtr & metadata_snapshot, PreformattedMessage & disable_reason,
         TableLockHolder & table_lock_holder, std::unique_lock<std::mutex> & currently_processing_in_background_mutex_lock);
@@ -310,6 +309,7 @@ private:
     };
 
 protected:
+    /// Collect mutations that have to be applied on the fly: currently they are only RENAME COLUMN.
     MutationCommands getAlterMutationCommandsForPart(const DataPartPtr & part) const override;
 };
 

From 55355f43ad420456467121ce43072a10791c5cc8 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 17 Jul 2024 05:19:58 +0200
Subject: [PATCH 0382/1488] Fix bad code: it was catching exceptions

---
 src/IO/WithFileSize.cpp                       | 48 +++++++++----------
 ...ry_and_native_with_binary_encoded_types.sh |  4 +-
 2 files changed, 25 insertions(+), 27 deletions(-)

diff --git a/src/IO/WithFileSize.cpp b/src/IO/WithFileSize.cpp
index 3660d962c08..8cea12fa200 100644
--- a/src/IO/WithFileSize.cpp
+++ b/src/IO/WithFileSize.cpp
@@ -14,40 +14,38 @@ namespace ErrorCodes
 }
 
 template <typename T>
-static size_t getFileSize(T & in)
+static std::optional<size_t> tryGetFileSize(T & in)
 {
     if (auto * with_file_size = dynamic_cast<WithFileSize *>(&in))
-    {
         return with_file_size->getFileSize();
-    }
+
+    return std::nullopt;
+}
+
+template <typename T>
+static size_t getFileSize(T & in)
+{
+    if (auto maybe_size = tryGetFileSize(in))
+        return *maybe_size;
 
     throw Exception(ErrorCodes::UNKNOWN_FILE_SIZE, "Cannot find out file size");
 }
 
-size_t getFileSizeFromReadBuffer(ReadBuffer & in)
-{
-    if (auto * delegate = dynamic_cast<ReadBufferFromFileDecorator *>(&in))
-    {
-        return getFileSize(delegate->getWrappedReadBuffer());
-    }
-    else if (auto * compressed = dynamic_cast<CompressedReadBufferWrapper *>(&in))
-    {
-        return getFileSize(compressed->getWrappedReadBuffer());
-    }
-
-    return getFileSize(in);
-}
-
 std::optional<size_t> tryGetFileSizeFromReadBuffer(ReadBuffer & in)
 {
-    try
-    {
-        return getFileSizeFromReadBuffer(in);
-    }
-    catch (...)
-    {
-        return std::nullopt;
-    }
+    if (auto * delegate = dynamic_cast<ReadBufferFromFileDecorator *>(&in))
+        return tryGetFileSize(delegate->getWrappedReadBuffer());
+    else if (auto * compressed = dynamic_cast<CompressedReadBufferWrapper *>(&in))
+        return tryGetFileSize(compressed->getWrappedReadBuffer());
+    return tryGetFileSize(in);
+}
+
+size_t getFileSizeFromReadBuffer(ReadBuffer & in)
+{
+    if (auto maybe_size = tryGetFileSizeFromReadBuffer(in))
+        return *maybe_size;
+
+    throw Exception(ErrorCodes::UNKNOWN_FILE_SIZE, "Cannot find out file size");
 }
 
 bool isBufferWithFileSize(const ReadBuffer & in)
diff --git a/tests/queries/0_stateless/03173_row_binary_and_native_with_binary_encoded_types.sh b/tests/queries/0_stateless/03173_row_binary_and_native_with_binary_encoded_types.sh
index 723b11ad620..0c585d36348 100755
--- a/tests/queries/0_stateless/03173_row_binary_and_native_with_binary_encoded_types.sh
+++ b/tests/queries/0_stateless/03173_row_binary_and_native_with_binary_encoded_types.sh
@@ -6,8 +6,8 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
 function test
 {
-  $CLICKHOUSE_LOCAL --allow_experimental_dynamic_type=1 --allow_experimental_variant_type=1 --output_format_binary_encode_types_in_binary_format=1 -q "select $1 as value format RowBinaryWithNamesAndTypes" | $CLICKHOUSE_LOCAL --input-format RowBinaryWithNamesAndTypes --input_format_binary_decode_types_in_binary_format=1 -q "select value, toTypeName(value) from table"
-  $CLICKHOUSE_LOCAL --allow_experimental_dynamic_type=1 --allow_experimental_variant_type=1 --output_format_native_encode_types_in_binary_format=1 -q "select $1 as value format Native" | $CLICKHOUSE_LOCAL --input-format Native --input_format_native_decode_types_in_binary_format=1 -q "select value, toTypeName(value) from table"
+    $CLICKHOUSE_LOCAL --allow_experimental_dynamic_type=1 --allow_experimental_variant_type=1 --output_format_binary_encode_types_in_binary_format=1 -q "select $1 as value format RowBinaryWithNamesAndTypes" | $CLICKHOUSE_LOCAL --input-format RowBinaryWithNamesAndTypes --input_format_binary_decode_types_in_binary_format=1 -q "select value, toTypeName(value) from table"
+    $CLICKHOUSE_LOCAL --allow_experimental_dynamic_type=1 --allow_experimental_variant_type=1 --output_format_native_encode_types_in_binary_format=1 -q "select $1 as value format Native" | $CLICKHOUSE_LOCAL --input-format Native --input_format_native_decode_types_in_binary_format=1 -q "select value, toTypeName(value) from table"
 }
 
 test "materialize(42)::UInt8"

From e0aedb992f647a8dcd226bc8775795ecad91a551 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 17 Jul 2024 05:34:04 +0200
Subject: [PATCH 0383/1488] Add a test

---
 .../03206_no_exceptions_clickhouse_local.reference       | 1 +
 .../0_stateless/03206_no_exceptions_clickhouse_local.sh  | 9 +++++++++
 2 files changed, 10 insertions(+)
 create mode 100644 tests/queries/0_stateless/03206_no_exceptions_clickhouse_local.reference
 create mode 100755 tests/queries/0_stateless/03206_no_exceptions_clickhouse_local.sh

diff --git a/tests/queries/0_stateless/03206_no_exceptions_clickhouse_local.reference b/tests/queries/0_stateless/03206_no_exceptions_clickhouse_local.reference
new file mode 100644
index 00000000000..11277a62b06
--- /dev/null
+++ b/tests/queries/0_stateless/03206_no_exceptions_clickhouse_local.reference
@@ -0,0 +1 @@
+Hello	world
diff --git a/tests/queries/0_stateless/03206_no_exceptions_clickhouse_local.sh b/tests/queries/0_stateless/03206_no_exceptions_clickhouse_local.sh
new file mode 100755
index 00000000000..86839a228dc
--- /dev/null
+++ b/tests/queries/0_stateless/03206_no_exceptions_clickhouse_local.sh
@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+# Tags: no-fasttest
+# Tag no-fasttest: In fasttest, ENABLE_LIBRARIES=0, so the grpc library is not built
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+CLICKHOUSE_TERMINATE_ON_ANY_EXCEPTION=1 ${CLICKHOUSE_LOCAL} --query "SELECT * FROM table" --input-format CSV <<<"Hello, world"

From 89f6f74418fa361c65261447ebbd0d79d88cc748 Mon Sep 17 00:00:00 2001
From: JackyWoo <wuchienchao@qq.com>
Date: Thu, 18 Jul 2024 10:31:11 +0800
Subject: [PATCH 0384/1488] Revert 02864_statistics_uniq

---
 .../02864_statistics_uniq.reference           | 113 ++++++++++++++++++
 .../0_stateless/02864_statistics_uniq.sql     |  73 +++++++++++
 2 files changed, 186 insertions(+)
 create mode 100644 tests/queries/0_stateless/02864_statistics_uniq.reference
 create mode 100644 tests/queries/0_stateless/02864_statistics_uniq.sql

diff --git a/tests/queries/0_stateless/02864_statistics_uniq.reference b/tests/queries/0_stateless/02864_statistics_uniq.reference
new file mode 100644
index 00000000000..861e35ddd48
--- /dev/null
+++ b/tests/queries/0_stateless/02864_statistics_uniq.reference
@@ -0,0 +1,113 @@
+CREATE TABLE default.t1\n(\n    `a` Float64 STATISTICS(tdigest),\n    `b` Int64 STATISTICS(tdigest),\n    `c` Int64 STATISTICS(tdigest, uniq),\n    `pk` String\n)\nENGINE = MergeTree\nORDER BY pk\nSETTINGS min_bytes_for_wide_part = 0, index_granularity = 8192
+After insert
+        Prewhere info
+          Prewhere filter
+          Prewhere filter column: and(less(a, 10), equals(c, 0), less(b, 10)) (removed)
+        Prewhere info
+          Prewhere filter
+          Prewhere filter column: and(equals(c, 11), less(a, 10), less(b, 10)) (removed)
+After merge
+        Prewhere info
+          Prewhere filter
+          Prewhere filter column: and(less(a, 10), equals(c, 0), less(b, 10)) (removed)
+        Prewhere info
+          Prewhere filter
+          Prewhere filter column: and(equals(c, 11), less(a, 10), less(b, 10)) (removed)
+After modify TDigest
+        Prewhere info
+          Prewhere filter
+          Prewhere filter column: and(less(a, 10), equals(c, 11), less(b, 10)) (removed)
+        Prewhere info
+          Prewhere filter
+          Prewhere filter column: and(less(a, 10), equals(c, 0), less(b, 10)) (removed)
+        Prewhere info
+          Prewhere filter
+          Prewhere filter column: and(less(c, -1), less(a, 10), less(b, 10)) (removed)
+After drop
+        Prewhere info
+          Prewhere filter
+          Prewhere filter column: and(less(a, 10), equals(c, 11), less(b, 10)) (removed)
+        Prewhere info
+          Prewhere filter
+          Prewhere filter column: and(less(a, 10), equals(c, 0), less(b, 10)) (removed)
+        Prewhere info
+          Prewhere filter
+          Prewhere filter column: and(less(a, 10), less(c, -1), less(b, 10)) (removed)
+ 72 changes: 0 additions & 72 deletions72  
+tests/queries/0_stateless/02864_statistics_uniq.sql
+Viewed
+Original file line number	Diff line number	Diff line change
+@@ -1,72 +0,0 @@
+DROP TABLE IF EXISTS t1;
+
+SET allow_experimental_statistics = 1;
+SET allow_statistics_optimize = 1;
+SET mutations_sync = 1;
+
+CREATE TABLE t1
+(
+    a Float64 STATISTICS(tdigest),
+    b Int64 STATISTICS(tdigest),
+    c Int64 STATISTICS(tdigest, uniq),
+    pk String,
+) Engine = MergeTree() ORDER BY pk
+SETTINGS min_bytes_for_wide_part = 0;
+
+SHOW CREATE TABLE t1;
+
+INSERT INTO t1 select number, -number, number/1000, generateUUIDv4() FROM system.numbers LIMIT 10000;
+INSERT INTO t1 select 0, 0, 11, generateUUIDv4();
+
+SELECT 'After insert';
+SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE b < 10 and c = 0 and a < 10) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
+SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE b < 10 and c = 11 and a < 10) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
+OPTIMIZE TABLE t1 FINAL;
+
+SELECT 'After merge';
+SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE b < 10 and c = 0 and a < 10) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
+SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE b < 10 and c = 11 and a < 10) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
+
+SELECT 'After modify TDigest';
+ALTER TABLE t1 MODIFY STATISTICS c TYPE TDigest;
+ALTER TABLE t1 MATERIALIZE STATISTICS c;
+
+SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE b < 10 and c = 11 and a < 10) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
+SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE b < 10 and c = 0 and a < 10) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
+SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE b < 10 and c < -1 and a < 10) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
+
+
+ALTER TABLE t1 DROP STATISTICS c;
+
+SELECT 'After drop';
+SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE b < 10 and c = 11 and a < 10) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
+SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE b < 10 and c = 0 and a < 10) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
+SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE b < 10 and c < -1 and a < 10) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
+
+DROP TABLE IF EXISTS t1;
+DROP TABLE IF EXISTS t2;
+SET allow_suspicious_low_cardinality_types=1;
+CREATE TABLE t2
+(
+    a Float64 STATISTICS(tdigest),
+    b Int64 STATISTICS(tdigest),
+    c LowCardinality(Int64) STATISTICS(tdigest, uniq),
+    pk String,
+) Engine = MergeTree() ORDER BY pk
+SETTINGS min_bytes_for_wide_part = 0;
+INSERT INTO t2 select number, -number, number/1000, generateUUIDv4() FROM system.numbers LIMIT 10000;
+
+DROP TABLE IF EXISTS t2;
+DROP TABLE IF EXISTS t3;
+
+CREATE TABLE t3
+(
+    a Float64 STATISTICS(tdigest),
+    b Int64 STATISTICS(tdigest),
+    c Nullable(Int64) STATISTICS(tdigest, uniq),
+    pk String,
+) Engine = MergeTree() ORDER BY pk
+SETTINGS min_bytes_for_wide_part = 0;
+INSERT INTO t3 select number, -number, number/1000, generateUUIDv4() FROM system.numbers LIMIT 10000;
+
+DROP TABLE IF EXISTS t3;
+
diff --git a/tests/queries/0_stateless/02864_statistics_uniq.sql b/tests/queries/0_stateless/02864_statistics_uniq.sql
new file mode 100644
index 00000000000..0f5f353c045
--- /dev/null
+++ b/tests/queries/0_stateless/02864_statistics_uniq.sql
@@ -0,0 +1,73 @@
+DROP TABLE IF EXISTS t1;
+
+SET allow_experimental_statistics = 1;
+SET allow_statistics_optimize = 1;
+SET mutations_sync = 1;
+
+CREATE TABLE t1
+(
+    a Float64 STATISTICS(tdigest),
+    b Int64 STATISTICS(tdigest),
+    c Int64 STATISTICS(tdigest, uniq),
+    pk String,
+) Engine = MergeTree() ORDER BY pk
+SETTINGS min_bytes_for_wide_part = 0;
+
+SHOW CREATE TABLE t1;
+
+INSERT INTO t1 select number, -number, number/1000, generateUUIDv4() FROM system.numbers LIMIT 10000;
+INSERT INTO t1 select 0, 0, 11, generateUUIDv4();
+
+SELECT 'After insert';
+SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE b < 10 and c = 0 and a < 10) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
+SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE b < 10 and c = 11 and a < 10) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
+OPTIMIZE TABLE t1 FINAL;
+
+SELECT 'After merge';
+SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE b < 10 and c = 0 and a < 10) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
+SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE b < 10 and c = 11 and a < 10) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
+
+SELECT 'After modify TDigest';
+ALTER TABLE t1 MODIFY STATISTICS c TYPE TDigest;
+ALTER TABLE t1 MATERIALIZE STATISTICS c;
+
+SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE b < 10 and c = 11 and a < 10) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
+SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE b < 10 and c = 0 and a < 10) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
+SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE b < 10 and c < -1 and a < 10) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
+
+
+ALTER TABLE t1 DROP STATISTICS c;
+
+SELECT 'After drop';
+SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE b < 10 and c = 11 and a < 10) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
+SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE b < 10 and c = 0 and a < 10) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
+SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE b < 10 and c < -1 and a < 10) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
+
+DROP TABLE IF EXISTS t1;
+DROP TABLE IF EXISTS t2;
+SET allow_suspicious_low_cardinality_types=1;
+CREATE TABLE t2
+(
+    a Float64 STATISTICS(tdigest),
+    b Int64 STATISTICS(tdigest),
+    c LowCardinality(Int64) STATISTICS(tdigest, uniq),
+    pk String,
+) Engine = MergeTree() ORDER BY pk
+SETTINGS min_bytes_for_wide_part = 0;
+INSERT INTO t2 select number, -number, number/1000, generateUUIDv4() FROM system.numbers LIMIT 10000;
+
+DROP TABLE IF EXISTS t2;
+DROP TABLE IF EXISTS t3;
+
+CREATE TABLE t3
+(
+    a Float64 STATISTICS(tdigest),
+    b Int64 STATISTICS(tdigest),
+    c Nullable(Int64) STATISTICS(tdigest, uniq),
+    pk String,
+) Engine = MergeTree() ORDER BY pk
+SETTINGS min_bytes_for_wide_part = 0;
+INSERT INTO t3 select number, -number, number/1000, generateUUIDv4() FROM system.numbers LIMIT 10000;
+
+DROP TABLE IF EXISTS t3;
+

From 1c883b802db839220a8e0a81892f653152a59cc1 Mon Sep 17 00:00:00 2001
From: JackyWoo <wuchienchao@qq.com>
Date: Thu, 18 Jul 2024 10:33:05 +0800
Subject: [PATCH 0385/1488] Rename test 02864_statistics_count_min_sketch

---
 ...864_statistics_count_min_sketch.reference} | 11 ---
 ... => 02864_statistics_count_min_sketch.sql} | 34 +-------
 .../02864_statistics_uniq.reference           | 78 -------------------
 3 files changed, 3 insertions(+), 120 deletions(-)
 rename tests/queries/0_stateless/{02864_statistics_estimate_predicate.reference => 02864_statistics_count_min_sketch.reference} (63%)
 rename tests/queries/0_stateless/{02864_statistics_estimate_predicate.sql => 02864_statistics_count_min_sketch.sql} (62%)

diff --git a/tests/queries/0_stateless/02864_statistics_estimate_predicate.reference b/tests/queries/0_stateless/02864_statistics_count_min_sketch.reference
similarity index 63%
rename from tests/queries/0_stateless/02864_statistics_estimate_predicate.reference
rename to tests/queries/0_stateless/02864_statistics_count_min_sketch.reference
index 4e41c32750f..02c41656a36 100644
--- a/tests/queries/0_stateless/02864_statistics_estimate_predicate.reference
+++ b/tests/queries/0_stateless/02864_statistics_count_min_sketch.reference
@@ -1,15 +1,4 @@
 CREATE TABLE default.tab\n(\n    `a` String,\n    `b` UInt64,\n    `c` Int64,\n    `pk` String\n)\nENGINE = MergeTree\nORDER BY pk\nSETTINGS min_bytes_for_wide_part = 0, index_granularity = 8192
-Test statistics TDigest:
-        Prewhere info
-          Prewhere filter
-          Prewhere filter column: and(less(c, -98), greater(b, 0)) (removed)
-        Prewhere info
-          Prewhere filter
-          Prewhere filter column: and(less(c, -98), equals(b, 0)) (removed)
-Test statistics Uniq:
-        Prewhere info
-          Prewhere filter
-          Prewhere filter column: and(equals(b, 0), equals(c, 0)) (removed)
 Test statistics count_min:
         Prewhere info
           Prewhere filter
diff --git a/tests/queries/0_stateless/02864_statistics_estimate_predicate.sql b/tests/queries/0_stateless/02864_statistics_count_min_sketch.sql
similarity index 62%
rename from tests/queries/0_stateless/02864_statistics_estimate_predicate.sql
rename to tests/queries/0_stateless/02864_statistics_count_min_sketch.sql
index 91b4f2d05cb..c730aa7b4a7 100644
--- a/tests/queries/0_stateless/02864_statistics_estimate_predicate.sql
+++ b/tests/queries/0_stateless/02864_statistics_count_min_sketch.sql
@@ -1,5 +1,4 @@
 -- Tags: no-fasttest
--- Tests statistics usages in prewhere optimization.
 
 DROP TABLE IF EXISTS tab SYNC;
 
@@ -21,33 +20,6 @@ SHOW CREATE TABLE tab;
 
 INSERT INTO tab select toString(number % 10000), number % 1000, -(number % 100), generateUUIDv4() FROM system.numbers LIMIT 10000;
 
-SELECT 'Test statistics TDigest:';
-
-ALTER TABLE tab ADD STATISTICS b, c TYPE tdigest;
-ALTER TABLE tab MATERIALIZE STATISTICS b, c;
-
-SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String', '')
-FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE b > 0/*9990*/ and c < -98/*100*/)
-WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
-
-SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String', '')
-FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE b = 0/*1000*/ and c < -98/*100*/)
-WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
-
-ALTER TABLE tab DROP STATISTICS b, c;
-
-
-SELECT 'Test statistics Uniq:';
-
-ALTER TABLE tab ADD STATISTICS b TYPE uniq, tdigest;
-ALTER TABLE tab MATERIALIZE STATISTICS b;
-SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8|_UInt16|_String', '')
-FROM (EXPLAIN actions=1 SELECT count(*) FROM tab WHERE c = 0/*1000*/ and b = 0/*10*/)
-WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
-
-ALTER TABLE tab DROP STATISTICS b;
-
-
 SELECT 'Test statistics count_min:';
 
 ALTER TABLE tab ADD STATISTICS a TYPE count_min;
@@ -87,9 +59,9 @@ DROP TABLE IF EXISTS tab2 SYNC;
 SET allow_suspicious_low_cardinality_types=1;
 CREATE TABLE tab2
 (
-    a LowCardinality(Int64) STATISTICS(uniq, tdigest, count_min),
-    b Nullable(Int64) STATISTICS(uniq, tdigest, count_min),
-    c LowCardinality(Nullable(Int64)) STATISTICS(uniq, tdigest, count_min),
+    a LowCardinality(Int64) STATISTICS(count_min),
+    b Nullable(Int64) STATISTICS(count_min),
+    c LowCardinality(Nullable(Int64)) STATISTICS(count_min),
     pk String,
 ) Engine = MergeTree() ORDER BY pk;
 
diff --git a/tests/queries/0_stateless/02864_statistics_uniq.reference b/tests/queries/0_stateless/02864_statistics_uniq.reference
index 861e35ddd48..77786dbdd8c 100644
--- a/tests/queries/0_stateless/02864_statistics_uniq.reference
+++ b/tests/queries/0_stateless/02864_statistics_uniq.reference
@@ -33,81 +33,3 @@ After drop
         Prewhere info
           Prewhere filter
           Prewhere filter column: and(less(a, 10), less(c, -1), less(b, 10)) (removed)
- 72 changes: 0 additions & 72 deletions72  
-tests/queries/0_stateless/02864_statistics_uniq.sql
-Viewed
-Original file line number	Diff line number	Diff line change
-@@ -1,72 +0,0 @@
-DROP TABLE IF EXISTS t1;
-
-SET allow_experimental_statistics = 1;
-SET allow_statistics_optimize = 1;
-SET mutations_sync = 1;
-
-CREATE TABLE t1
-(
-    a Float64 STATISTICS(tdigest),
-    b Int64 STATISTICS(tdigest),
-    c Int64 STATISTICS(tdigest, uniq),
-    pk String,
-) Engine = MergeTree() ORDER BY pk
-SETTINGS min_bytes_for_wide_part = 0;
-
-SHOW CREATE TABLE t1;
-
-INSERT INTO t1 select number, -number, number/1000, generateUUIDv4() FROM system.numbers LIMIT 10000;
-INSERT INTO t1 select 0, 0, 11, generateUUIDv4();
-
-SELECT 'After insert';
-SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE b < 10 and c = 0 and a < 10) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
-SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE b < 10 and c = 11 and a < 10) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
-OPTIMIZE TABLE t1 FINAL;
-
-SELECT 'After merge';
-SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE b < 10 and c = 0 and a < 10) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
-SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE b < 10 and c = 11 and a < 10) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
-
-SELECT 'After modify TDigest';
-ALTER TABLE t1 MODIFY STATISTICS c TYPE TDigest;
-ALTER TABLE t1 MATERIALIZE STATISTICS c;
-
-SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE b < 10 and c = 11 and a < 10) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
-SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE b < 10 and c = 0 and a < 10) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
-SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE b < 10 and c < -1 and a < 10) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
-
-
-ALTER TABLE t1 DROP STATISTICS c;
-
-SELECT 'After drop';
-SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE b < 10 and c = 11 and a < 10) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
-SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE b < 10 and c = 0 and a < 10) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
-SELECT replaceRegexpAll(explain, '__table1.|_UInt8|_Int8', '') FROM (EXPLAIN actions=1 SELECT count(*) FROM t1 WHERE b < 10 and c < -1 and a < 10) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
-
-DROP TABLE IF EXISTS t1;
-DROP TABLE IF EXISTS t2;
-SET allow_suspicious_low_cardinality_types=1;
-CREATE TABLE t2
-(
-    a Float64 STATISTICS(tdigest),
-    b Int64 STATISTICS(tdigest),
-    c LowCardinality(Int64) STATISTICS(tdigest, uniq),
-    pk String,
-) Engine = MergeTree() ORDER BY pk
-SETTINGS min_bytes_for_wide_part = 0;
-INSERT INTO t2 select number, -number, number/1000, generateUUIDv4() FROM system.numbers LIMIT 10000;
-
-DROP TABLE IF EXISTS t2;
-DROP TABLE IF EXISTS t3;
-
-CREATE TABLE t3
-(
-    a Float64 STATISTICS(tdigest),
-    b Int64 STATISTICS(tdigest),
-    c Nullable(Int64) STATISTICS(tdigest, uniq),
-    pk String,
-) Engine = MergeTree() ORDER BY pk
-SETTINGS min_bytes_for_wide_part = 0;
-INSERT INTO t3 select number, -number, number/1000, generateUUIDv4() FROM system.numbers LIMIT 10000;
-
-DROP TABLE IF EXISTS t3;
-

From c7be25f0a167c2c5ab6944b47779be2f90af443d Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Thu, 18 Jul 2024 04:54:36 +0200
Subject: [PATCH 0386/1488] Fix everything

---
 src/Disks/IO/AsynchronousBoundedReadBuffer.h           |  2 +-
 src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp        |  2 +-
 src/Disks/IO/ReadBufferFromAzureBlobStorage.h          |  2 +-
 src/Disks/IO/ReadBufferFromRemoteFSGather.h            |  2 +-
 src/IO/Archives/LibArchiveReader.cpp                   |  2 +-
 src/IO/Archives/ZipArchiveReader.cpp                   |  2 +-
 src/IO/AsynchronousReadBufferFromFileDescriptor.cpp    |  2 +-
 src/IO/AsynchronousReadBufferFromFileDescriptor.h      |  2 +-
 src/IO/ConcatSeekableReadBuffer.h                      |  2 +-
 src/IO/MMapReadBufferFromFileDescriptor.cpp            |  2 +-
 src/IO/MMapReadBufferFromFileDescriptor.h              |  2 +-
 src/IO/ParallelReadBuffer.cpp                          |  2 +-
 src/IO/ParallelReadBuffer.h                            |  2 +-
 src/IO/ReadBufferFromEmptyFile.h                       |  2 +-
 src/IO/ReadBufferFromEncryptedFile.h                   |  2 +-
 src/IO/ReadBufferFromFileBase.cpp                      |  6 ++----
 src/IO/ReadBufferFromFileBase.h                        |  2 +-
 src/IO/ReadBufferFromFileDecorator.cpp                 |  4 ++--
 src/IO/ReadBufferFromFileDecorator.h                   |  2 +-
 src/IO/ReadBufferFromFileDescriptor.cpp                |  2 +-
 src/IO/ReadBufferFromFileDescriptor.h                  |  2 +-
 src/IO/ReadBufferFromS3.cpp                            |  6 +++---
 src/IO/ReadBufferFromS3.h                              |  2 +-
 src/IO/ReadWriteBufferFromHTTP.cpp                     |  7 ++-----
 src/IO/ReadWriteBufferFromHTTP.h                       |  2 +-
 src/IO/WithFileSize.cpp                                | 10 +++++++++-
 src/IO/WithFileSize.h                                  |  7 ++++---
 src/Storages/Cache/ExternalDataSourceCache.h           |  2 +-
 .../HDFS/AsynchronousReadBufferFromHDFS.cpp            |  4 ++--
 .../HDFS/AsynchronousReadBufferFromHDFS.h              |  2 +-
 src/Storages/ObjectStorage/HDFS/ReadBufferFromHDFS.cpp |  8 ++++----
 src/Storages/ObjectStorage/HDFS/ReadBufferFromHDFS.h   |  2 +-
 32 files changed, 52 insertions(+), 48 deletions(-)

diff --git a/src/Disks/IO/AsynchronousBoundedReadBuffer.h b/src/Disks/IO/AsynchronousBoundedReadBuffer.h
index 9a802348998..3dc8fcc39cb 100644
--- a/src/Disks/IO/AsynchronousBoundedReadBuffer.h
+++ b/src/Disks/IO/AsynchronousBoundedReadBuffer.h
@@ -34,7 +34,7 @@ public:
 
     String getFileName() const override { return impl->getFileName(); }
 
-    size_t getFileSize() override { return impl->getFileSize(); }
+    std::optional<size_t> tryGetFileSize() override { return impl->tryGetFileSize(); }
 
     String getInfoForLog() override { return impl->getInfoForLog(); }
 
diff --git a/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp b/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp
index da1ea65f2ea..a36a8b031b4 100644
--- a/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp
+++ b/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp
@@ -253,7 +253,7 @@ void ReadBufferFromAzureBlobStorage::initialize()
     initialized = true;
 }
 
-size_t ReadBufferFromAzureBlobStorage::getFileSize()
+std::optional<size_t> ReadBufferFromAzureBlobStorage::tryGetFileSize()
 {
     if (!blob_client)
         blob_client = std::make_unique<Azure::Storage::Blobs::BlobClient>(blob_container_client->GetBlobClient(path));
diff --git a/src/Disks/IO/ReadBufferFromAzureBlobStorage.h b/src/Disks/IO/ReadBufferFromAzureBlobStorage.h
index d328195cc26..f407f27e099 100644
--- a/src/Disks/IO/ReadBufferFromAzureBlobStorage.h
+++ b/src/Disks/IO/ReadBufferFromAzureBlobStorage.h
@@ -42,7 +42,7 @@ public:
 
     bool supportsRightBoundedReads() const override { return true; }
 
-    size_t getFileSize() override;
+    std::optional<size_t> tryGetFileSize() override;
 
     size_t readBigAt(char * to, size_t n, size_t range_begin, const std::function<bool(size_t)> & progress_callback) const override;
 
diff --git a/src/Disks/IO/ReadBufferFromRemoteFSGather.h b/src/Disks/IO/ReadBufferFromRemoteFSGather.h
index e36365a8174..9f1cb681f1a 100644
--- a/src/Disks/IO/ReadBufferFromRemoteFSGather.h
+++ b/src/Disks/IO/ReadBufferFromRemoteFSGather.h
@@ -41,7 +41,7 @@ public:
 
     void setReadUntilEnd() override { setReadUntilPosition(getFileSize()); }
 
-    size_t getFileSize() override { return getTotalSize(blobs_to_read); }
+    std::optional<size_t> tryGetFileSize() override { return getTotalSize(blobs_to_read); }
 
     size_t getFileOffsetOfBufferEnd() const override { return file_offset_of_buffer_end; }
 
diff --git a/src/IO/Archives/LibArchiveReader.cpp b/src/IO/Archives/LibArchiveReader.cpp
index e3fe63fa40d..31bad4d6638 100644
--- a/src/IO/Archives/LibArchiveReader.cpp
+++ b/src/IO/Archives/LibArchiveReader.cpp
@@ -321,7 +321,7 @@ public:
     off_t getPosition() override { throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "getPosition not supported when reading from archive"); }
     String getFileName() const override { return handle.getFileName(); }
 
-    size_t getFileSize() override { return handle.getFileInfo().uncompressed_size; }
+    std::optional<size_t> tryGetFileSize() override { return handle.getFileInfo().uncompressed_size; }
 
     Handle releaseHandle() && { return std::move(handle); }
 
diff --git a/src/IO/Archives/ZipArchiveReader.cpp b/src/IO/Archives/ZipArchiveReader.cpp
index 2a9b7a43519..12b07d550c2 100644
--- a/src/IO/Archives/ZipArchiveReader.cpp
+++ b/src/IO/Archives/ZipArchiveReader.cpp
@@ -317,7 +317,7 @@ public:
 
     String getFileName() const override { return handle.getFileName(); }
 
-    size_t getFileSize() override { return handle.getFileInfo().uncompressed_size; }
+    std::optional<size_t> tryGetFileSize() override { return handle.getFileInfo().uncompressed_size; }
 
     /// Releases owned handle to pass it to an enumerator.
     HandleHolder releaseHandle() &&
diff --git a/src/IO/AsynchronousReadBufferFromFileDescriptor.cpp b/src/IO/AsynchronousReadBufferFromFileDescriptor.cpp
index f8c00d62732..6c4bd09b76f 100644
--- a/src/IO/AsynchronousReadBufferFromFileDescriptor.cpp
+++ b/src/IO/AsynchronousReadBufferFromFileDescriptor.cpp
@@ -244,7 +244,7 @@ void AsynchronousReadBufferFromFileDescriptor::rewind()
     file_offset_of_buffer_end = 0;
 }
 
-size_t AsynchronousReadBufferFromFileDescriptor::getFileSize()
+std::optional<size_t> AsynchronousReadBufferFromFileDescriptor::tryGetFileSize()
 {
     return getSizeFromFileDescriptor(fd, getFileName());
 }
diff --git a/src/IO/AsynchronousReadBufferFromFileDescriptor.h b/src/IO/AsynchronousReadBufferFromFileDescriptor.h
index 82659b1aca7..097979fbe00 100644
--- a/src/IO/AsynchronousReadBufferFromFileDescriptor.h
+++ b/src/IO/AsynchronousReadBufferFromFileDescriptor.h
@@ -68,7 +68,7 @@ public:
     /// Seek to the beginning, discarding already read data if any. Useful to reread file that changes on every read.
     void rewind();
 
-    size_t getFileSize() override;
+    std::optional<size_t> tryGetFileSize() override;
 
     size_t getFileOffsetOfBufferEnd() const override { return file_offset_of_buffer_end; }
 
diff --git a/src/IO/ConcatSeekableReadBuffer.h b/src/IO/ConcatSeekableReadBuffer.h
index c8c16c5d887..609f0dc25b8 100644
--- a/src/IO/ConcatSeekableReadBuffer.h
+++ b/src/IO/ConcatSeekableReadBuffer.h
@@ -21,7 +21,7 @@ public:
     off_t seek(off_t off, int whence) override;
     off_t getPosition() override;
 
-    size_t getFileSize() override { return total_size; }
+    std::optional<size_t> tryGetFileSize() override { return total_size; }
 
 private:
     bool nextImpl() override;
diff --git a/src/IO/MMapReadBufferFromFileDescriptor.cpp b/src/IO/MMapReadBufferFromFileDescriptor.cpp
index f27828f71b2..83dd192de54 100644
--- a/src/IO/MMapReadBufferFromFileDescriptor.cpp
+++ b/src/IO/MMapReadBufferFromFileDescriptor.cpp
@@ -87,7 +87,7 @@ off_t MMapReadBufferFromFileDescriptor::seek(off_t offset, int whence)
     return new_pos;
 }
 
-size_t MMapReadBufferFromFileDescriptor::getFileSize()
+std::optional<size_t> MMapReadBufferFromFileDescriptor::tryGetFileSize()
 {
     return getSizeFromFileDescriptor(getFD(), getFileName());
 }
diff --git a/src/IO/MMapReadBufferFromFileDescriptor.h b/src/IO/MMapReadBufferFromFileDescriptor.h
index f774538374a..de44ec3f9d8 100644
--- a/src/IO/MMapReadBufferFromFileDescriptor.h
+++ b/src/IO/MMapReadBufferFromFileDescriptor.h
@@ -38,7 +38,7 @@ public:
 
     int getFD() const;
 
-    size_t getFileSize() override;
+    std::optional<size_t> tryGetFileSize() override;
 
     size_t readBigAt(char * to, size_t n, size_t offset, const std::function<bool(size_t)> &) const override;
     bool supportsReadAt() override { return true; }
diff --git a/src/IO/ParallelReadBuffer.cpp b/src/IO/ParallelReadBuffer.cpp
index e6771235a8e..89cff670e37 100644
--- a/src/IO/ParallelReadBuffer.cpp
+++ b/src/IO/ParallelReadBuffer.cpp
@@ -152,7 +152,7 @@ off_t ParallelReadBuffer::seek(off_t offset, int whence)
     return offset;
 }
 
-size_t ParallelReadBuffer::getFileSize()
+std::optional<size_t> ParallelReadBuffer::tryGetFileSize()
 {
     return file_size;
 }
diff --git a/src/IO/ParallelReadBuffer.h b/src/IO/ParallelReadBuffer.h
index cfeec2b3677..8852472a8bc 100644
--- a/src/IO/ParallelReadBuffer.h
+++ b/src/IO/ParallelReadBuffer.h
@@ -33,7 +33,7 @@ public:
     ~ParallelReadBuffer() override { finishAndWait(); }
 
     off_t seek(off_t off, int whence) override;
-    size_t getFileSize() override;
+    std::optional<size_t> tryGetFileSize() override;
     off_t getPosition() override;
 
     const SeekableReadBuffer & getReadBuffer() const { return input; }
diff --git a/src/IO/ReadBufferFromEmptyFile.h b/src/IO/ReadBufferFromEmptyFile.h
index f21f2f507dc..b15299dafee 100644
--- a/src/IO/ReadBufferFromEmptyFile.h
+++ b/src/IO/ReadBufferFromEmptyFile.h
@@ -19,7 +19,7 @@ private:
     std::string getFileName() const override { return "<empty>"; }
     off_t seek(off_t /*off*/, int /*whence*/) override { return 0; }
     off_t getPosition() override { return 0; }
-    size_t getFileSize() override { return 0; }
+    std::optional<size_t> tryGetFileSize() override { return 0; }
 };
 
 }
diff --git a/src/IO/ReadBufferFromEncryptedFile.h b/src/IO/ReadBufferFromEncryptedFile.h
index 3626daccb3e..213d242bb91 100644
--- a/src/IO/ReadBufferFromEncryptedFile.h
+++ b/src/IO/ReadBufferFromEncryptedFile.h
@@ -30,7 +30,7 @@ public:
 
     void setReadUntilEnd() override { in->setReadUntilEnd(); }
 
-    size_t getFileSize() override { return in->getFileSize(); }
+    std::optional<size_t> tryGetFileSize() override { return in->tryGetFileSize(); }
 
 private:
     bool nextImpl() override;
diff --git a/src/IO/ReadBufferFromFileBase.cpp b/src/IO/ReadBufferFromFileBase.cpp
index 4ac3f984f78..d42b12ba49b 100644
--- a/src/IO/ReadBufferFromFileBase.cpp
+++ b/src/IO/ReadBufferFromFileBase.cpp
@@ -26,11 +26,9 @@ ReadBufferFromFileBase::ReadBufferFromFileBase(
 
 ReadBufferFromFileBase::~ReadBufferFromFileBase() = default;
 
-size_t ReadBufferFromFileBase::getFileSize()
+std::optional<size_t> ReadBufferFromFileBase::tryGetFileSize()
 {
-    if (file_size)
-        return *file_size;
-    throw Exception(ErrorCodes::UNKNOWN_FILE_SIZE, "Cannot find out file size for read buffer");
+    return file_size;
 }
 
 void ReadBufferFromFileBase::setProgressCallback(ContextPtr context)
diff --git a/src/IO/ReadBufferFromFileBase.h b/src/IO/ReadBufferFromFileBase.h
index 9870d8bbe43..c98dcd5a93e 100644
--- a/src/IO/ReadBufferFromFileBase.h
+++ b/src/IO/ReadBufferFromFileBase.h
@@ -50,7 +50,7 @@ public:
         clock_type = clock_type_;
     }
 
-    size_t getFileSize() override;
+    std::optional<size_t> tryGetFileSize() override;
 
     void setProgressCallback(ContextPtr context);
 
diff --git a/src/IO/ReadBufferFromFileDecorator.cpp b/src/IO/ReadBufferFromFileDecorator.cpp
index 9ac0fb4e475..8a6468b9bd0 100644
--- a/src/IO/ReadBufferFromFileDecorator.cpp
+++ b/src/IO/ReadBufferFromFileDecorator.cpp
@@ -52,9 +52,9 @@ bool ReadBufferFromFileDecorator::nextImpl()
     return result;
 }
 
-size_t ReadBufferFromFileDecorator::getFileSize()
+std::optional<size_t> ReadBufferFromFileDecorator::tryGetFileSize()
 {
-    return getFileSizeFromReadBuffer(*impl);
+    return tryGetFileSizeFromReadBuffer(*impl);
 }
 
 }
diff --git a/src/IO/ReadBufferFromFileDecorator.h b/src/IO/ReadBufferFromFileDecorator.h
index 6e62c7f741b..69f029c5cf7 100644
--- a/src/IO/ReadBufferFromFileDecorator.h
+++ b/src/IO/ReadBufferFromFileDecorator.h
@@ -27,7 +27,7 @@ public:
 
     ReadBuffer & getWrappedReadBuffer() { return *impl; }
 
-    size_t getFileSize() override;
+    std::optional<size_t> tryGetFileSize() override;
 
 protected:
     std::unique_ptr<SeekableReadBuffer> impl;
diff --git a/src/IO/ReadBufferFromFileDescriptor.cpp b/src/IO/ReadBufferFromFileDescriptor.cpp
index 76a80f145e7..51a1a5d8d93 100644
--- a/src/IO/ReadBufferFromFileDescriptor.cpp
+++ b/src/IO/ReadBufferFromFileDescriptor.cpp
@@ -253,7 +253,7 @@ void ReadBufferFromFileDescriptor::rewind()
     file_offset_of_buffer_end = 0;
 }
 
-size_t ReadBufferFromFileDescriptor::getFileSize()
+std::optional<size_t> ReadBufferFromFileDescriptor::tryGetFileSize()
 {
     return getSizeFromFileDescriptor(fd, getFileName());
 }
diff --git a/src/IO/ReadBufferFromFileDescriptor.h b/src/IO/ReadBufferFromFileDescriptor.h
index db256ef91c7..6083e744c95 100644
--- a/src/IO/ReadBufferFromFileDescriptor.h
+++ b/src/IO/ReadBufferFromFileDescriptor.h
@@ -69,7 +69,7 @@ public:
     /// Seek to the beginning, discarding already read data if any. Useful to reread file that changes on every read.
     void rewind();
 
-    size_t getFileSize() override;
+    std::optional<size_t> tryGetFileSize() override;
 
     bool checkIfActuallySeekable() override;
 
diff --git a/src/IO/ReadBufferFromS3.cpp b/src/IO/ReadBufferFromS3.cpp
index 9e001232e65..94f317802e3 100644
--- a/src/IO/ReadBufferFromS3.cpp
+++ b/src/IO/ReadBufferFromS3.cpp
@@ -313,15 +313,15 @@ off_t ReadBufferFromS3::seek(off_t offset_, int whence)
     return offset;
 }
 
-size_t ReadBufferFromS3::getFileSize()
+std::optional<size_t> ReadBufferFromS3::tryGetFileSize()
 {
     if (file_size)
-        return *file_size;
+        return file_size;
 
     auto object_size = S3::getObjectSize(*client_ptr, bucket, key, version_id);
 
     file_size = object_size;
-    return *file_size;
+    return file_size;
 }
 
 off_t ReadBufferFromS3::getPosition()
diff --git a/src/IO/ReadBufferFromS3.h b/src/IO/ReadBufferFromS3.h
index c6625c2d632..ff04f78ce7b 100644
--- a/src/IO/ReadBufferFromS3.h
+++ b/src/IO/ReadBufferFromS3.h
@@ -63,7 +63,7 @@ public:
 
     off_t getPosition() override;
 
-    size_t getFileSize() override;
+    std::optional<size_t> tryGetFileSize() override;
 
     void setReadUntilPosition(size_t position) override;
     void setReadUntilEnd() override;
diff --git a/src/IO/ReadWriteBufferFromHTTP.cpp b/src/IO/ReadWriteBufferFromHTTP.cpp
index b753e66da48..2a62b11aa44 100644
--- a/src/IO/ReadWriteBufferFromHTTP.cpp
+++ b/src/IO/ReadWriteBufferFromHTTP.cpp
@@ -121,15 +121,12 @@ void ReadWriteBufferFromHTTP::prepareRequest(Poco::Net::HTTPRequest & request, s
         credentials.authenticate(request);
 }
 
-size_t ReadWriteBufferFromHTTP::getFileSize()
+std::optional<size_t> ReadWriteBufferFromHTTP::tryGetFileSize()
 {
     if (!file_info)
         file_info = getFileInfo();
 
-    if (file_info->file_size)
-        return *file_info->file_size;
-
-    throw Exception(ErrorCodes::UNKNOWN_FILE_SIZE, "Cannot find out file size for: {}", initial_uri.toString());
+    return file_info->file_size;
 }
 
 bool ReadWriteBufferFromHTTP::supportsReadAt()
diff --git a/src/IO/ReadWriteBufferFromHTTP.h b/src/IO/ReadWriteBufferFromHTTP.h
index f496fe3ddcd..1c9bda53008 100644
--- a/src/IO/ReadWriteBufferFromHTTP.h
+++ b/src/IO/ReadWriteBufferFromHTTP.h
@@ -118,7 +118,7 @@ private:
 
     std::unique_ptr<ReadBuffer> initialize();
 
-    size_t getFileSize() override;
+    std::optional<size_t> tryGetFileSize() override;
 
     bool supportsReadAt() override;
 
diff --git a/src/IO/WithFileSize.cpp b/src/IO/WithFileSize.cpp
index 8cea12fa200..cbbcab83de2 100644
--- a/src/IO/WithFileSize.cpp
+++ b/src/IO/WithFileSize.cpp
@@ -13,11 +13,19 @@ namespace ErrorCodes
     extern const int UNKNOWN_FILE_SIZE;
 }
 
+size_t WithFileSize::getFileSize()
+{
+    if (auto maybe_size = tryGetFileSize())
+        return *maybe_size;
+
+    throw Exception(ErrorCodes::UNKNOWN_FILE_SIZE, "Cannot find out file size");
+}
+
 template <typename T>
 static std::optional<size_t> tryGetFileSize(T & in)
 {
     if (auto * with_file_size = dynamic_cast<WithFileSize *>(&in))
-        return with_file_size->getFileSize();
+        return with_file_size->tryGetFileSize();
 
     return std::nullopt;
 }
diff --git a/src/IO/WithFileSize.h b/src/IO/WithFileSize.h
index 0ae3af98ea0..e5dc383fab0 100644
--- a/src/IO/WithFileSize.h
+++ b/src/IO/WithFileSize.h
@@ -10,15 +10,16 @@ class ReadBuffer;
 class WithFileSize
 {
 public:
-    virtual size_t getFileSize() = 0;
+    /// Returns nullopt if couldn't find out file size;
+    virtual std::optional<size_t> tryGetFileSize() = 0;
     virtual ~WithFileSize() = default;
+
+    size_t getFileSize();
 };
 
 bool isBufferWithFileSize(const ReadBuffer & in);
 
 size_t getFileSizeFromReadBuffer(ReadBuffer & in);
-
-/// Return nullopt if couldn't find out file size;
 std::optional<size_t> tryGetFileSizeFromReadBuffer(ReadBuffer & in);
 
 size_t getDataOffsetMaybeCompressed(const ReadBuffer & in);
diff --git a/src/Storages/Cache/ExternalDataSourceCache.h b/src/Storages/Cache/ExternalDataSourceCache.h
index 4c8c7974005..3b4eff28307 100644
--- a/src/Storages/Cache/ExternalDataSourceCache.h
+++ b/src/Storages/Cache/ExternalDataSourceCache.h
@@ -53,7 +53,7 @@ public:
     bool nextImpl() override;
     off_t seek(off_t off, int whence) override;
     off_t getPosition() override;
-    size_t getFileSize() override { return remote_file_size; }
+    std::optional<size_t> tryGetFileSize() override { return remote_file_size; }
 
 private:
     std::unique_ptr<LocalFileHolder> local_file_holder;
diff --git a/src/Storages/ObjectStorage/HDFS/AsynchronousReadBufferFromHDFS.cpp b/src/Storages/ObjectStorage/HDFS/AsynchronousReadBufferFromHDFS.cpp
index 21df7e35284..3bbc4e8a2ea 100644
--- a/src/Storages/ObjectStorage/HDFS/AsynchronousReadBufferFromHDFS.cpp
+++ b/src/Storages/ObjectStorage/HDFS/AsynchronousReadBufferFromHDFS.cpp
@@ -91,9 +91,9 @@ void AsynchronousReadBufferFromHDFS::prefetch(Priority priority)
 }
 
 
-size_t AsynchronousReadBufferFromHDFS::getFileSize()
+std::optional<size_t> AsynchronousReadBufferFromHDFS::tryGetFileSize()
 {
-    return impl->getFileSize();
+    return impl->tryGetFileSize();
 }
 
 String AsynchronousReadBufferFromHDFS::getFileName() const
diff --git a/src/Storages/ObjectStorage/HDFS/AsynchronousReadBufferFromHDFS.h b/src/Storages/ObjectStorage/HDFS/AsynchronousReadBufferFromHDFS.h
index 5aef92315a4..9846d74453b 100644
--- a/src/Storages/ObjectStorage/HDFS/AsynchronousReadBufferFromHDFS.h
+++ b/src/Storages/ObjectStorage/HDFS/AsynchronousReadBufferFromHDFS.h
@@ -35,7 +35,7 @@ public:
 
     void prefetch(Priority priority) override;
 
-    size_t getFileSize() override;
+    std::optional<size_t> tryGetFileSize() override;
 
     String getFileName() const override;
 
diff --git a/src/Storages/ObjectStorage/HDFS/ReadBufferFromHDFS.cpp b/src/Storages/ObjectStorage/HDFS/ReadBufferFromHDFS.cpp
index be339d021dc..bf6f9db722c 100644
--- a/src/Storages/ObjectStorage/HDFS/ReadBufferFromHDFS.cpp
+++ b/src/Storages/ObjectStorage/HDFS/ReadBufferFromHDFS.cpp
@@ -31,7 +31,7 @@ namespace ErrorCodes
 }
 
 
-struct ReadBufferFromHDFS::ReadBufferFromHDFSImpl : public BufferWithOwnMemory<SeekableReadBuffer>
+struct ReadBufferFromHDFS::ReadBufferFromHDFSImpl : public BufferWithOwnMemory<SeekableReadBuffer>, public WithFileSize
 {
     String hdfs_uri;
     String hdfs_file_path;
@@ -90,7 +90,7 @@ struct ReadBufferFromHDFS::ReadBufferFromHDFSImpl : public BufferWithOwnMemory<S
         hdfsCloseFile(fs.get(), fin);
     }
 
-    size_t getFileSize() const
+    std::optional<size_t> tryGetFileSize() override
     {
         return file_size;
     }
@@ -191,9 +191,9 @@ ReadBufferFromHDFS::ReadBufferFromHDFS(
 
 ReadBufferFromHDFS::~ReadBufferFromHDFS() = default;
 
-size_t ReadBufferFromHDFS::getFileSize()
+std::optional<size_t> ReadBufferFromHDFS::tryGetFileSize()
 {
-    return impl->getFileSize();
+    return impl->tryGetFileSize();
 }
 
 bool ReadBufferFromHDFS::nextImpl()
diff --git a/src/Storages/ObjectStorage/HDFS/ReadBufferFromHDFS.h b/src/Storages/ObjectStorage/HDFS/ReadBufferFromHDFS.h
index d9671e7e445..5363f07967b 100644
--- a/src/Storages/ObjectStorage/HDFS/ReadBufferFromHDFS.h
+++ b/src/Storages/ObjectStorage/HDFS/ReadBufferFromHDFS.h
@@ -40,7 +40,7 @@ public:
 
     off_t getPosition() override;
 
-    size_t getFileSize() override;
+    std::optional<size_t> tryGetFileSize() override;
 
     size_t getFileOffsetOfBufferEnd() const override;
 

From b8fbfd227fb60e0f244bda716ef5a9bb89376986 Mon Sep 17 00:00:00 2001
From: Xu Jia <jia.xu@clickhouse.com>
Date: Thu, 18 Jul 2024 15:41:08 +0800
Subject: [PATCH 0387/1488] format

---
 src/Interpreters/InterpreterInsertQuery.cpp   |  1 -
 .../test.py                                   | 40 ++++++++-----------
 2 files changed, 17 insertions(+), 24 deletions(-)

diff --git a/src/Interpreters/InterpreterInsertQuery.cpp b/src/Interpreters/InterpreterInsertQuery.cpp
index 4064cd82b67..181fb064b54 100644
--- a/src/Interpreters/InterpreterInsertQuery.cpp
+++ b/src/Interpreters/InterpreterInsertQuery.cpp
@@ -49,7 +49,6 @@ namespace ProfileEvents
 {
     extern const Event InsertQueriesWithSubqueries;
     extern const Event QueriesWithSubqueries;
-    extern const int QUERY_IS_PROHIBITED;
 }
 
 namespace DB
diff --git a/tests/integration/test_disable_insertion_and_mutation/test.py b/tests/integration/test_disable_insertion_and_mutation/test.py
index f098f130d2b..f25964d27b8 100644
--- a/tests/integration/test_disable_insertion_and_mutation/test.py
+++ b/tests/integration/test_disable_insertion_and_mutation/test.py
@@ -37,29 +37,25 @@ def started_cluster():
 def test_disable_insertion_and_mutation(started_cluster):
     writing_node.query("""CREATE TABLE my_table on cluster default (key UInt64, value String) ENGINE=ReplicatedMergeTree('/clickhouse/tables/{shard}/default.my_table', '{replica}') ORDER BY key partition by (key % 5) """)
 
-    assert (
-        "QUERY_IS_PROHIBITED"
-        in reading_node.query_and_get_error("INSERT INTO my_table VALUES (1, 'hello')")
+    assert "QUERY_IS_PROHIBITED" in reading_node.query_and_get_error(
+        "INSERT INTO my_table VALUES (1, 'hello')"
     )
 
-    assert (
-        "QUERY_IS_PROHIBITED"
-        in reading_node.query_and_get_error("INSERT INTO my_table SETTINGS async_insert = 1 VALUES (1, 'hello')")
+    assert "QUERY_IS_PROHIBITED" in reading_node.query_and_get_error(
+        "INSERT INTO my_table SETTINGS async_insert = 1 VALUES (1, 'hello')"
     )
 
-    assert (
-        "QUERY_IS_PROHIBITED"
-        in reading_node.query_and_get_error("ALTER TABLE my_table delete where 1")
+    assert "QUERY_IS_PROHIBITED" in reading_node.query_and_get_error(
+        "ALTER TABLE my_table delete where 1"
+    )
+    
+
+    assert "QUERY_IS_PROHIBITED" in reading_node.query_and_get_error(
+        "ALTER table my_table update key = 1 where 1"
     )
 
-    assert (
-        "QUERY_IS_PROHIBITED"
-        in reading_node.query_and_get_error("ALTER table my_table update key = 1 where 1")
-    )
-
-    assert (
-        "QUERY_IS_PROHIBITED"
-        in reading_node.query_and_get_error("ALTER TABLE my_table drop partition 0")
+    assert "QUERY_IS_PROHIBITED" in reading_node.query_and_get_error(
+        "ALTER TABLE my_table drop partition 0"
     )
 
     reading_node.query("SELECT * from my_table");
@@ -73,12 +69,10 @@ def test_disable_insertion_and_mutation(started_cluster):
 
     reading_node.query("ALter Table my_table MODIFY COLUMN new_column String")
 
-    assert(
-        "new_column\tString"
-        in reading_node.query("DESC my_table")
+    assert "new_column\tString" in reading_node.query(
+        "DESC my_table"
     )
 
-    assert(
-        "new_column\tString"
-        in writing_node.query("DESC my_table")
+    assert "new_column\tString" in writing_node.query(
+        "DESC my_table"
     )

From 2e1f679ceb05afe4d5d813eb4048555c6311c3e1 Mon Sep 17 00:00:00 2001
From: Michael Stetsyuk <mike.stetsyuk@gmail.com>
Date: Tue, 16 Jul 2024 18:35:33 +0200
Subject: [PATCH 0388/1488] add S3DiskNoKeyErrors metric

---
 src/Common/CurrentMetrics.cpp |  2 ++
 src/IO/S3/Client.cpp          | 21 +++++++++++++++------
 2 files changed, 17 insertions(+), 6 deletions(-)

diff --git a/src/Common/CurrentMetrics.cpp b/src/Common/CurrentMetrics.cpp
index 7c97e73f278..2fedba0175b 100644
--- a/src/Common/CurrentMetrics.cpp
+++ b/src/Common/CurrentMetrics.cpp
@@ -306,6 +306,8 @@
     \
     M(FilteringMarksWithPrimaryKey, "Number of threads currently doing filtering of mark ranges by the primary key") \
     M(FilteringMarksWithSecondaryKeys, "Number of threads currently doing filtering of mark ranges by secondary keys") \
+    \
+    M(S3DiskNoKeyErrors, "Number of no-key S3 disk errors") \
 
 #ifdef APPLY_FOR_EXTERNAL_METRICS
     #define APPLY_FOR_METRICS(M) APPLY_FOR_BUILTIN_METRICS(M) APPLY_FOR_EXTERNAL_METRICS(M)
diff --git a/src/IO/S3/Client.cpp b/src/IO/S3/Client.cpp
index 3b958dea046..db20420db9f 100644
--- a/src/IO/S3/Client.cpp
+++ b/src/IO/S3/Client.cpp
@@ -24,6 +24,7 @@
 
 #include <Common/assert_cast.h>
 #include <Common/logger_useful.h>
+#include <Common/CurrentMetrics.h>
 #include <Common/ProxyConfigurationResolverProvider.h>
 
 #include <Core/Settings.h>
@@ -43,6 +44,11 @@ namespace ProfileEvents
     extern const Event TinyS3Clients;
 }
 
+namespace CurrentMetrics
+{
+    extern const Metric S3DiskNoKeyErrors;
+}
+
 namespace DB
 {
 
@@ -379,10 +385,10 @@ Model::HeadObjectOutcome Client::HeadObject(HeadObjectRequest & request) const
 
     request.overrideURI(std::move(*bucket_uri));
 
-    /// The next call is NOT a recurcive call
-    /// This is a virtuall call Aws::S3::S3Client::HeadObject(const Model::HeadObjectRequest&)
-    return enrichErrorMessage(
-        HeadObject(static_cast<const Model::HeadObjectRequest&>(request)));
+    if (isClientForDisk())
+        CurrentMetrics::add(CurrentMetrics::S3DiskNoKeyErrors);
+
+    return enrichErrorMessage(std::move(result));
 }
 
 /// For each request, we wrap the request functions from Aws::S3::Client with doRequest
@@ -402,8 +408,11 @@ Model::ListObjectsOutcome Client::ListObjects(ListObjectsRequest & request) cons
 
 Model::GetObjectOutcome Client::GetObject(GetObjectRequest & request) const
 {
-    return enrichErrorMessage(
-        doRequest(request, [this](const Model::GetObjectRequest & req) { return GetObject(req); }));
+    auto resp = doRequest(request, [this](const Model::GetObjectRequest & req) { return GetObject(req); });
+    if (!resp.IsSuccess() && isClientForDisk())
+        CurrentMetrics::add(CurrentMetrics::S3DiskNoKeyErrors);
+
+    return enrichErrorMessage(std::move(resp));
 }
 
 Model::AbortMultipartUploadOutcome Client::AbortMultipartUpload(AbortMultipartUploadRequest & request) const

From f16a05cf2d31c1306e89e2fa3e990233a5d34288 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Thu, 18 Jul 2024 10:51:53 +0000
Subject: [PATCH 0389/1488] Fix a test.

---
 tests/queries/0_stateless/02892_orc_filter_pushdown.reference | 2 +-
 tests/queries/0_stateless/02892_orc_filter_pushdown.sql       | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/queries/0_stateless/02892_orc_filter_pushdown.reference b/tests/queries/0_stateless/02892_orc_filter_pushdown.reference
index 903d42bf492..0be6d13adb3 100644
--- a/tests/queries/0_stateless/02892_orc_filter_pushdown.reference
+++ b/tests/queries/0_stateless/02892_orc_filter_pushdown.reference
@@ -205,7 +205,7 @@ select count(), sum(number) from file('02892.orc', ORC, 'number UInt64, negative
 600	419700
 select count(), min(negative_or_null), max(negative_or_null) from file('02892.orc', ORC, 'number UInt64, negative_or_null Int64') where (negative_or_null < -500);
 596	-1099	-501
-select count(), sum(number) from file('02892.orc', ORC, 'number UInt64, negative_or_null Int64') where indexHint(negative_or_null is null);
+select count(), sum(number) from file('02892.orc', ORC, 'number UInt64, negative_or_null Int64') where indexHint(negative_or_null is null) settings allow_experimental_analyzer=1;
 0	0
 select count(), min(negative_or_null), max(negative_or_null) from file('02892.orc', ORC, 'number UInt64, negative_or_null Int64') where (negative_or_null is null);
 0	0	0
diff --git a/tests/queries/0_stateless/02892_orc_filter_pushdown.sql b/tests/queries/0_stateless/02892_orc_filter_pushdown.sql
index e3736de6a17..29055eb911a 100644
--- a/tests/queries/0_stateless/02892_orc_filter_pushdown.sql
+++ b/tests/queries/0_stateless/02892_orc_filter_pushdown.sql
@@ -206,7 +206,7 @@ select count(), min(nEgAtIvE_oR_nUlL), max(nEgAtIvE_oR_nUlL) from file('02892.or
 select count(), sum(number) from file('02892.orc', ORC, 'number UInt64, negative_or_null Int64') where indexHint(negative_or_null < -500);
 select count(), min(negative_or_null), max(negative_or_null) from file('02892.orc', ORC, 'number UInt64, negative_or_null Int64') where (negative_or_null < -500);
 
-select count(), sum(number) from file('02892.orc', ORC, 'number UInt64, negative_or_null Int64') where indexHint(negative_or_null is null);
+select count(), sum(number) from file('02892.orc', ORC, 'number UInt64, negative_or_null Int64') where indexHint(negative_or_null is null) settings allow_experimental_analyzer=1;
 select count(), min(negative_or_null), max(negative_or_null) from file('02892.orc', ORC, 'number UInt64, negative_or_null Int64') where (negative_or_null is null);
 
 select count(), sum(number) from file('02892.orc', ORC, 'number UInt64, negative_or_null Int64') where indexHint(negative_or_null in (0, -1, -10, -100, -1000));

From 87fafaa9f5f8406228440631acf597c782a3ecdd Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Thu, 18 Jul 2024 11:00:50 +0000
Subject: [PATCH 0390/1488] Remove flaky case from the test.

---
 ..._finctions_and_column_sparse_bug.reference | 73 -------------------
 ...window_finctions_and_column_sparse_bug.sql | 54 --------------
 2 files changed, 127 deletions(-)

diff --git a/tests/queries/0_stateless/03205_parallel_window_finctions_and_column_sparse_bug.reference b/tests/queries/0_stateless/03205_parallel_window_finctions_and_column_sparse_bug.reference
index 356329a392d..13e229432ae 100644
--- a/tests/queries/0_stateless/03205_parallel_window_finctions_and_column_sparse_bug.reference
+++ b/tests/queries/0_stateless/03205_parallel_window_finctions_and_column_sparse_bug.reference
@@ -8,76 +8,3 @@ true	1	1
 true	1	2
 ---
 -755809149	0
----
-1	-2081147898
-1	-1981899149
-2	-2051538534
-2	-1650266905
-3	-1975508531
-3	-1646738223
-4	-1700730666
-4	-1618912877
-5	-1465484835
-5	-1317193174
-6	-1458338029
-6	-1219769753
-7	-1450619195
-7	-1154269118
-8	-1365934326
-8	-1150980622
-9	-1203382363
-9	-1098155311
-10	-1197430632
-10	-841067875
-11	-1176267855
-11	-816935497
-12	-1020892864
-12	-599948807
-13	-991301833
-13	-526570556
-14	-685902265
-14	-504713125
-15	-653505826
-15	-411038390
-16	-451392958
-16	-331834394
-17	-262516786
-17	-176934810
-18	-222873194
-18	-2
-19	-153185515
-19	6
-20	-74234560
-20	255
-21	-41
-21	406615258
-22	-6
-22	541758331
-23	-5
-23	720985423
-24	-3
-24	745669725
-25	15
-25	897064234
-26	65535
-26	1116921321
-27	77089559
-27	1207796283
-28	100663045
-28	1603772265
-29	561061873
-29	1664059402
-30	643897141
-30	1688303275
-31	914629990
-31	1913361922
-32	1159852204
-32	1929066636
-33	1258218855
-33	1968095908
-34	1459407556
-34	2054878592
-35	1936334332
-35	2125075305
-36	1962246186
-37	2030467062
diff --git a/tests/queries/0_stateless/03205_parallel_window_finctions_and_column_sparse_bug.sql b/tests/queries/0_stateless/03205_parallel_window_finctions_and_column_sparse_bug.sql
index 6e326d0a67f..f2391e0d165 100644
--- a/tests/queries/0_stateless/03205_parallel_window_finctions_and_column_sparse_bug.sql
+++ b/tests/queries/0_stateless/03205_parallel_window_finctions_and_column_sparse_bug.sql
@@ -31,57 +31,3 @@ SELECT * FROM (
 SELECT c, min(w) OVER (PARTITION BY s ORDER BY c ASC, s ASC, w ASC)
 FROM t limit toUInt64(-1))
 WHERE c = -755809149;
-
-SELECT '---';
-
-create table t_vkx4cc (
-  c_ylzjpt Int32,
-  c_hqfr9 Bool ,
-  ) engine = MergeTree order by c_ylzjpt;
-
-system stop merges t_vkx4cc;
-
-insert into t_vkx4cc (c_ylzjpt, c_hqfr9) values (-2081147898, coalesce((NOT NOT(cast( (53 < 539704722) as Nullable(Bool)))), true)), (-1219769753, coalesce((true) and (false), false)), (-1981899149, coalesce(false, false)), (-1646738223, coalesce((NOT NOT(cast( (23.5 <= -26) as Nullable(Bool)))), false));
-
-insert into t_vkx4cc (c_ylzjpt, c_hqfr9) values (255, coalesce(false, false)), (-1317193174, coalesce(false, false)), (-41, coalesce(true, false)), (1929066636, coalesce(false, true));
-
-insert into t_vkx4cc (c_ylzjpt, c_hqfr9) values (-1700730666, coalesce((NOT NOT(cast( (-2022515471055597472 AND -29) as Nullable(Bool)))), false)), (1664059402, coalesce((NOT NOT(cast( (-19643 >= -122) as Nullable(Bool)))), false)), (1688303275, coalesce((NOT NOT(cast( (737275892 < 105) as Nullable(Bool)))), true)), (406615258, coalesce((NOT NOT(cast( (-657730213 = 82.86) as Nullable(Bool)))), false));
-
-insert into t_vkx4cc (c_ylzjpt, c_hqfr9) values (-2, coalesce(false, false)), (1962246186, coalesce(true, false)), (-991301833, coalesce(true, true)), (2054878592, coalesce(false, false));
-
-insert into t_vkx4cc (c_ylzjpt, c_hqfr9) values (643897141, coalesce((NOT NOT(cast( (-60 AND cast(null as Nullable(Int64))) as Nullable(Bool)))), true)), (-2051538534, coalesce(((-1616816511 between 332225780 and -1883087387)) or ((-573375170 between -1427445977 and 615586748)), false)), (77089559, coalesce((NOT NOT(cast( ((true) and (true) != 925456787) as Nullable(Bool)))), false)), (1116921321, coalesce((0 is NULL), true));
-
-insert into t_vkx4cc (c_ylzjpt, c_hqfr9) values (-816935497, coalesce(false, false)), (1207796283, coalesce((-129 between -5 and -5), false)), (-1365934326, coalesce(true, false)), (-1618912877, coalesce((NOT NOT(cast( (false >= 31833) as Nullable(Bool)))), false));
-
-insert into t_vkx4cc (c_ylzjpt, c_hqfr9) values (-331834394, coalesce((NOT NOT(cast( (-63 <= -1822810052) as Nullable(Bool)))), true)), (-1020892864, coalesce((NOT NOT(cast( (40.31 <= 8146037365746019777) as Nullable(Bool)))), true)), (-1150980622, coalesce(((94019304 between -730556489 and 32)) and ((-956354236 is not NULL)), true)), (-1203382363, coalesce(true, true));
-
-insert into t_vkx4cc (c_ylzjpt, c_hqfr9) values (-653505826, coalesce((true) or (true), false)), (-1975508531, coalesce(((-796885845 between 65536 and cast(null as Nullable(Int32)))) or ((NOT NOT(cast( (-7467729336434250795 < 100.20) as Nullable(Bool))))), false)), (-1465484835, coalesce(((NOT NOT(cast( (19209 <= 75.96) as Nullable(Bool))))) or (true), false)), (1968095908, coalesce((NOT NOT(cast( (-1309960412156062327 > 13102) as Nullable(Bool)))), true));
-
-alter table t_vkx4cc add column c_zosphq2t1 Float64;
-
-insert into t_vkx4cc (c_ylzjpt, c_hqfr9, c_zosphq2t1) values (-153185515, coalesce((NOT NOT(cast( (1291639145 >= 30.22) as Nullable(Bool)))), false), -1.8), (-411038390, coalesce(((-762326135 between 16 and 177530758)) or (false), true), 26.34), (914629990, coalesce((-1125832977 is not NULL), true), 59.2), (541758331, coalesce(false, true), -255.1);
-
-insert into t_vkx4cc (c_ylzjpt, c_hqfr9, c_zosphq2t1) values (2125075305, coalesce(false, false), 55.36), (-1176267855, coalesce(true, true), 55.45), (1459407556, coalesce((true) and ((NOT NOT(cast( (95.96 != 65) as Nullable(Bool))))), true), 85.80), (-1098155311, coalesce(false, false), 2147483649.9);
-
-insert into t_vkx4cc (c_ylzjpt, c_hqfr9, c_zosphq2t1) values (6, coalesce((NOT NOT(cast( (1546334968 < -4) as Nullable(Bool)))), true), 57.42), (-5, coalesce((NOT NOT(cast( (59 AND 13) as Nullable(Bool)))), false), 65536.3), (100663045, coalesce((-1190355242 is not NULL), true), 73.80), (-451392958, coalesce((NOT NOT(cast( (false != -443845933) as Nullable(Bool)))), false), -4294967294.0);
-
-insert into t_vkx4cc (c_ylzjpt, c_hqfr9, c_zosphq2t1) values (561061873, coalesce(true, false), 12.17), (-526570556, coalesce(false, false), 64.73), (-1450619195, coalesce(true, true), 54.33), (-3, coalesce(true, true), 52.9);
-
-insert into t_vkx4cc (c_ylzjpt, c_hqfr9, c_zosphq2t1) values (-504713125, coalesce(false, true), 27.58), (897064234, coalesce((836516994 between cast(null as Nullable(Int32)) and -1832647080), true), 9223372036854775809.2), (65535, coalesce(true, true), 4294967297.5), (-599948807, coalesce((false) or ((NOT NOT(cast( (6.52 = 65.49) as Nullable(Bool))))), false), 256.5);
-
-insert into t_vkx4cc (c_ylzjpt, c_hqfr9, c_zosphq2t1) values (-1650266905, coalesce((NOT NOT(cast( (-83 = -218055084) as Nullable(Bool)))), true), 1.9), (-841067875, coalesce(false, true), -126.5), (15, coalesce(((NOT NOT(cast( (cast(null as Nullable(Decimal)) = cast(null as Nullable(Int32))) as Nullable(Bool))))) or (true), true), 33.65), (1913361922, coalesce((NOT NOT(cast( (false AND 0) as Nullable(Bool)))), false), 6.4);
-
-insert into t_vkx4cc (c_ylzjpt, c_hqfr9, c_zosphq2t1) values (1159852204, coalesce((-2057115045 is not NULL), false), 20.61), (-6, coalesce(true, true), 66.33), (-1154269118, coalesce(false, true), 8.89), (1258218855, coalesce(true, false), 19.80);
-
-insert into t_vkx4cc (c_ylzjpt, c_hqfr9, c_zosphq2t1) values (1603772265, coalesce(false, true), 57.87), (-176934810, coalesce(false, true), 128.8), (-1458338029, coalesce((NOT NOT(cast( (20908 != (NOT NOT(cast( (cast(null as Nullable(Decimal)) <= (true)         or ((NOT NOT(cast( (973511022 <= -112) as Nullable(Bool)))))) as Nullable(Bool))))) as Nullable(Bool)))), true), 76.54), (-262516786, coalesce((cast(null as Nullable(Int32)) is NULL), false), 21.49);
-
-insert into t_vkx4cc (c_ylzjpt, c_hqfr9, c_zosphq2t1) values (-1197430632, coalesce(true, false), 45.40), (-685902265, coalesce((NOT NOT(cast( (cast(null as Nullable(Decimal)) < cast(null as Nullable(Decimal))) as Nullable(Bool)))), true), 5.55), (1936334332, coalesce((-1565552735 is not NULL), false), 26.28), (2030467062, coalesce((NOT NOT(cast( (127.3 != cast(null as Nullable(Int32))) as Nullable(Bool)))), true), 89.50);
-
-insert into t_vkx4cc (c_ylzjpt, c_hqfr9, c_zosphq2t1) values (720985423, coalesce((NOT NOT(cast( (-451448940 = cast(null as Nullable(Decimal))) as Nullable(Bool)))), false), 52.65), (-222873194, coalesce(((-20 between -1419620477 and 1616455043)) or ((25624502 between 1312431316 and 1757361651)), false), 127.2), (745669725, coalesce((NOT NOT(cast( ((NOT NOT(cast( (cast(null as Nullable(UInt64)) <= 42) as Nullable(Bool)))) >= 3233811255032796928) as Nullable(Bool)))), false), 7.74), (-74234560, coalesce((NOT NOT(cast( (cast(null as Nullable(Decimal)) >= cast(null as Nullable(Decimal))) as Nullable(Bool)))), true), 19.25);
-
-SELECT DISTINCT
-  count(ref_0.c_zosphq2t1) over (partition by ref_0.c_hqfr9 order by ref_0.c_ylzjpt, ref_0.c_hqfr9, ref_0.c_zosphq2t1) as c0,
-  ref_0.c_ylzjpt as c1
-FROM
-  t_vkx4cc as ref_0
-  order by c0, c1;

From 959ac9a768530cb9e2c7b013df37bfee000a1644 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Thu, 18 Jul 2024 13:16:35 +0200
Subject: [PATCH 0391/1488] ci: dump dmesg in case of OOM

Without additional info it is unclear how to tune paralelism or maybe
split some modules.

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 tests/ci/ci.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/ci/ci.py b/tests/ci/ci.py
index cf285f4b97d..c5ad97088aa 100644
--- a/tests/ci/ci.py
+++ b/tests/ci/ci.py
@@ -1281,7 +1281,8 @@ def main() -> int:
                 except ValueError:
                     pass
             if Utils.is_killed_with_oom():
-                print("WARNING: OOM while job execution")
+                print("WARNING: OOM while job execution:")
+                print(subprocess.run("sudo dmesg -T", check=False))
                 error = f"Out Of Memory, exit_code {job_report.exit_code}, after {int(job_report.duration)}s"
             else:
                 error = f"Unknown, exit_code {job_report.exit_code}, after {int(job_report.duration)}s"

From e3be4a95bd6118ca789642c56edf88b6a880c02e Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Thu, 18 Jul 2024 13:29:02 +0200
Subject: [PATCH 0392/1488] fix clang tidy

---
 src/Databases/IDatabase.h | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/Databases/IDatabase.h b/src/Databases/IDatabase.h
index ce5a52b1b0f..f94326d220e 100644
--- a/src/Databases/IDatabase.h
+++ b/src/Databases/IDatabase.h
@@ -6,7 +6,6 @@
 #include <Interpreters/executeQuery.h>
 #include <Parsers/IAST_fwd.h>
 #include <QueryPipeline/BlockIO.h>
-#include <Storages/IStorage.h>
 #include <Storages/IStorage_fwd.h>
 #include <base/types.h>
 #include <Common/AsyncLoader.h>

From 5c1db5fc66170fd2b194962b2914f48b21e15453 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Thu, 18 Jul 2024 13:30:22 +0200
Subject: [PATCH 0393/1488] Fix column injection in merges after drop column

---
 src/Storages/MergeTree/IMergeTreeDataPart.cpp              | 7 ++-----
 src/Storages/MergeTree/IMergeTreeDataPart.h                | 4 +++-
 src/Storages/MergeTree/IMergeTreeDataPartInfoForReader.h   | 2 +-
 .../MergeTree/LoadedMergeTreeDataPartInfoForReader.h       | 5 ++++-
 src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp         | 3 ++-
 src/Storages/MergeTree/MergeTreeDataPartWide.cpp           | 5 ++++-
 ...parallel_alter_add_drop_column_zookeeper_on_steroids.sh | 4 ++--
 7 files changed, 18 insertions(+), 12 deletions(-)

diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp
index 15863e74455..c2f87018872 100644
--- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp
+++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp
@@ -652,15 +652,12 @@ size_t IMergeTreeDataPart::getFileSizeOrZero(const String & file_name) const
     return checksum->second.file_size;
 }
 
-String IMergeTreeDataPart::getColumnNameWithMinimumCompressedSize(bool with_subcolumns) const
+String IMergeTreeDataPart::getColumnNameWithMinimumCompressedSize(const NamesAndTypesList & available_columns) const
 {
-    auto options = GetColumnsOptions(GetColumnsOptions::AllPhysical).withSubcolumns(with_subcolumns);
-    auto columns_list = columns_description.get(options);
-
     std::optional<std::string> minimum_size_column;
     UInt64 minimum_size = std::numeric_limits<UInt64>::max();
 
-    for (const auto & column : columns_list)
+    for (const auto & column : available_columns)
     {
         if (!hasColumnFiles(column))
             continue;
diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h
index 9fd481b0d8e..85ef0472ce7 100644
--- a/src/Storages/MergeTree/IMergeTreeDataPart.h
+++ b/src/Storages/MergeTree/IMergeTreeDataPart.h
@@ -196,7 +196,9 @@ public:
 
     /// Returns the name of a column with minimum compressed size (as returned by getColumnSize()).
     /// If no checksums are present returns the name of the first physically existing column.
-    String getColumnNameWithMinimumCompressedSize(bool with_subcolumns) const;
+    /// We pass a list of available columns since the ones available in the current storage snapshot might be smaller
+    /// than the one the table has (e.g a DROP COLUMN happened) and we don't want to get a column not in the snapshot
+    String getColumnNameWithMinimumCompressedSize(const NamesAndTypesList & available_columns) const;
 
     bool contains(const IMergeTreeDataPart & other) const { return info.contains(other.info); }
 
diff --git a/src/Storages/MergeTree/IMergeTreeDataPartInfoForReader.h b/src/Storages/MergeTree/IMergeTreeDataPartInfoForReader.h
index ccc88079daa..7d4fb1df1c2 100644
--- a/src/Storages/MergeTree/IMergeTreeDataPartInfoForReader.h
+++ b/src/Storages/MergeTree/IMergeTreeDataPartInfoForReader.h
@@ -47,7 +47,7 @@ public:
 
     virtual std::optional<size_t> getColumnPosition(const String & column_name) const = 0;
 
-    virtual String getColumnNameWithMinimumCompressedSize(bool with_subcolumns) const = 0;
+    virtual String getColumnNameWithMinimumCompressedSize(const NamesAndTypesList & available_columns) const = 0;
 
     virtual const MergeTreeDataPartChecksums & getChecksums() const = 0;
 
diff --git a/src/Storages/MergeTree/LoadedMergeTreeDataPartInfoForReader.h b/src/Storages/MergeTree/LoadedMergeTreeDataPartInfoForReader.h
index f5111ccaacc..aff1cf0edb0 100644
--- a/src/Storages/MergeTree/LoadedMergeTreeDataPartInfoForReader.h
+++ b/src/Storages/MergeTree/LoadedMergeTreeDataPartInfoForReader.h
@@ -36,7 +36,10 @@ public:
 
     AlterConversionsPtr getAlterConversions() const override { return alter_conversions; }
 
-    String getColumnNameWithMinimumCompressedSize(bool with_subcolumns) const override { return data_part->getColumnNameWithMinimumCompressedSize(with_subcolumns); }
+    String getColumnNameWithMinimumCompressedSize(const NamesAndTypesList & available_columns) const override
+    {
+        return data_part->getColumnNameWithMinimumCompressedSize(available_columns);
+    }
 
     const MergeTreeDataPartChecksums & getChecksums() const override { return data_part->checksums; }
 
diff --git a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp
index 570387a7046..aaa4ecd8eee 100644
--- a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp
+++ b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp
@@ -127,7 +127,8 @@ NameSet injectRequiredColumns(
         */
     if (!have_at_least_one_physical_column)
     {
-        const auto minimum_size_column_name = data_part_info_for_reader.getColumnNameWithMinimumCompressedSize(with_subcolumns);
+        auto available_columns = storage_snapshot->metadata->getColumns().get(options);
+        const auto minimum_size_column_name = data_part_info_for_reader.getColumnNameWithMinimumCompressedSize(available_columns);
         columns.push_back(minimum_size_column_name);
         /// correctly report added column
         injected_columns.insert(columns.back());
diff --git a/src/Storages/MergeTree/MergeTreeDataPartWide.cpp b/src/Storages/MergeTree/MergeTreeDataPartWide.cpp
index ee1a9b7f8ed..5bab523a9f1 100644
--- a/src/Storages/MergeTree/MergeTreeDataPartWide.cpp
+++ b/src/Storages/MergeTree/MergeTreeDataPartWide.cpp
@@ -266,10 +266,13 @@ void MergeTreeDataPartWide::doCheckConsistency(bool require_part_metadata) const
 
 bool MergeTreeDataPartWide::hasColumnFiles(const NameAndTypePair & column) const
 {
+    auto serialization = tryGetSerialization(column.name);
+    if (!serialization)
+        return false;
     auto marks_file_extension = index_granularity_info.mark_type.getFileExtension();
 
     bool res = true;
-    getSerialization(column.name)->enumerateStreams([&](const auto & substream_path)
+    serialization->enumerateStreams([&](const auto & substream_path)
     {
         auto stream_name = getStreamNameForColumn(column, substream_path, checksums);
         if (!stream_name || !checksums.files.contains(*stream_name + marks_file_extension))
diff --git a/tests/queries/0_stateless/03144_parallel_alter_add_drop_column_zookeeper_on_steroids.sh b/tests/queries/0_stateless/03144_parallel_alter_add_drop_column_zookeeper_on_steroids.sh
index c27dfffcfc2..b3d0b08948b 100755
--- a/tests/queries/0_stateless/03144_parallel_alter_add_drop_column_zookeeper_on_steroids.sh
+++ b/tests/queries/0_stateless/03144_parallel_alter_add_drop_column_zookeeper_on_steroids.sh
@@ -30,9 +30,9 @@ function alter_thread()
     while true; do
         REPLICA=$(($RANDOM % 3 + 1))
         ADD=$(($RANDOM % 5 + 1))
-        $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_add_drop_steroids_$REPLICA ADD COLUMN value$ADD UInt32 DEFAULT 42 SETTINGS replication_alter_partitions_sync=0"; # additionaly we don't wait anything for more heavy concurrency
+        $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_add_drop_steroids_$REPLICA ADD COLUMN value$ADD UInt32 DEFAULT 42 SETTINGS replication_alter_partitions_sync=0"; # additionally we don't wait anything for more heavy concurrency
         DROP=$(($RANDOM % 5 + 1))
-        $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_add_drop_steroids_$REPLICA DROP COLUMN value$DROP SETTINGS replication_alter_partitions_sync=0"; # additionaly we don't wait anything for more heavy concurrency
+        $CLICKHOUSE_CLIENT --query "ALTER TABLE concurrent_alter_add_drop_steroids_$REPLICA DROP COLUMN value$DROP SETTINGS replication_alter_partitions_sync=0"; # additionally we don't wait anything for more heavy concurrency
         sleep 0.$RANDOM
     done
 }

From 85714e7d7e18a9d91dfe385b658e1ae1ddedb2ea Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Thu, 18 Jul 2024 11:31:23 +0000
Subject: [PATCH 0394/1488] fix asof join on nulls

---
 .../Transforms/MergeJoinTransform.cpp         | 10 ++-
 .../0_stateless/00976_asof_join_on.reference  | 78 ++++++++++++++++++-
 .../0_stateless/00976_asof_join_on.sql.j2     |  7 +-
 3 files changed, 88 insertions(+), 7 deletions(-)

diff --git a/src/Processors/Transforms/MergeJoinTransform.cpp b/src/Processors/Transforms/MergeJoinTransform.cpp
index 3b69ddaec06..7bdb3d4f30f 100644
--- a/src/Processors/Transforms/MergeJoinTransform.cpp
+++ b/src/Processors/Transforms/MergeJoinTransform.cpp
@@ -257,8 +257,14 @@ JoinKeyRow::JoinKeyRow(const FullMergeJoinCursor & cursor, size_t pos)
         new_col->insertFrom(*col, pos);
         row.push_back(std::move(new_col));
     }
-    if (const auto * asof_column = cursor.getAsofColumn())
+    if (const IColumn * asof_column = cursor.getAsofColumn())
     {
+        if (const auto * nullable_asof_column = checkAndGetColumn<ColumnNullable>(asof_column))
+        {
+            /// We save matched column, and since NULL do not match anything, we can't use it as a key
+            chassert(!nullable_asof_column->isNullAt(pos));
+            asof_column = nullable_asof_column->getNestedColumnPtr().get();
+        }
         auto new_col = asof_column->cloneEmpty();
         new_col->insertFrom(*asof_column, pos);
         row.push_back(std::move(new_col));
@@ -1174,7 +1180,6 @@ IMergingAlgorithm::Status MergeJoinAlgorithm::merge()
     if (!cursors[1]->cursor.isValid() && !cursors[1]->fullyCompleted())
         return Status(1);
 
-
     if (auto result = handleAllJoinState())
         return std::move(*result);
 
@@ -1183,7 +1188,6 @@ IMergingAlgorithm::Status MergeJoinAlgorithm::merge()
 
     if (cursors[0]->fullyCompleted() || cursors[1]->fullyCompleted())
     {
-
         if (!cursors[0]->fullyCompleted() && isLeftOrFull(kind))
             return Status(createBlockWithDefaults(0));
 
diff --git a/tests/queries/0_stateless/00976_asof_join_on.reference b/tests/queries/0_stateless/00976_asof_join_on.reference
index e13745bed9d..80af4287d3c 100644
--- a/tests/queries/0_stateless/00976_asof_join_on.reference
+++ b/tests/queries/0_stateless/00976_asof_join_on.reference
@@ -1,4 +1,4 @@
-- default -
+- default / join_use_nulls = 0 -
 1	1	0	0
 1	2	1	2
 1	3	1	2
@@ -35,7 +35,7 @@
 2	1	2	3
 2	2	2	3
 1	2	1	2
-- full_sorting_merge -
+- full_sorting_merge / join_use_nulls = 0 -
 1	1	0	0
 1	2	1	2
 1	3	1	2
@@ -72,3 +72,77 @@
 2	1	2	3
 2	2	2	3
 1	2	1	2
+- default / join_use_nulls = 1 -
+1	1	\N	\N
+1	2	1	2
+1	3	1	2
+2	1	\N	\N
+2	2	\N	\N
+2	3	2	3
+3	1	\N	\N
+3	2	\N	\N
+3	3	\N	\N
+9
+1	2	1	2
+1	3	1	2
+2	3	2	3
+-
+1	1	1	2
+1	2	1	2
+1	3	1	4
+2	1	2	3
+2	2	2	3
+2	3	2	3
+-
+1	1	1	2
+1	2	1	2
+1	3	1	4
+2	1	2	3
+2	2	2	3
+2	3	2	3
+-
+1	3	1	2
+-
+1	1	1	2
+1	2	1	4
+1	3	1	4
+2	1	2	3
+2	2	2	3
+1	2	1	2
+- full_sorting_merge / join_use_nulls = 1 -
+1	1	\N	\N
+1	2	1	2
+1	3	1	2
+2	1	\N	\N
+2	2	\N	\N
+2	3	2	3
+3	1	\N	\N
+3	2	\N	\N
+3	3	\N	\N
+9
+1	2	1	2
+1	3	1	2
+2	3	2	3
+-
+1	1	1	2
+1	2	1	2
+1	3	1	4
+2	1	2	3
+2	2	2	3
+2	3	2	3
+-
+1	1	1	2
+1	2	1	2
+1	3	1	4
+2	1	2	3
+2	2	2	3
+2	3	2	3
+-
+1	3	1	2
+-
+1	1	1	2
+1	2	1	4
+1	3	1	4
+2	1	2	3
+2	2	2	3
+1	2	1	2
diff --git a/tests/queries/0_stateless/00976_asof_join_on.sql.j2 b/tests/queries/0_stateless/00976_asof_join_on.sql.j2
index aecf472a36c..ea642366a05 100644
--- a/tests/queries/0_stateless/00976_asof_join_on.sql.j2
+++ b/tests/queries/0_stateless/00976_asof_join_on.sql.j2
@@ -7,11 +7,13 @@ CREATE TABLE B(b UInt32, t UInt32) ENGINE = Memory;
 INSERT INTO A (a,t) VALUES (1,1),(1,2),(1,3), (2,1),(2,2),(2,3), (3,1),(3,2),(3,3);
 INSERT INTO B (b,t) VALUES (1,2),(1,4),(2,3);
 
+{% for join_use_nulls in [0, 1] -%}
 {% for join_algorithm in ['default', 'full_sorting_merge'] -%}
 
 SET join_algorithm = '{{ join_algorithm }}';
 
-SELECT '- {{ join_algorithm }} -';
+SELECT '- {{ join_algorithm }} / join_use_nulls = {{ join_use_nulls }} -';
+set join_use_nulls = {{ join_use_nulls }};
 
 SELECT A.a, A.t, B.b, B.t FROM A ASOF LEFT JOIN B ON A.a == B.b AND A.t >= B.t ORDER BY (A.a, A.t);
 SELECT count() FROM A ASOF LEFT JOIN B ON A.a == B.b AND B.t <= A.t;
@@ -34,7 +36,8 @@ ASOF INNER JOIN (SELECT * FROM B UNION ALL SELECT 1, 3) AS B ON B.t <= A.t AND A
 WHERE B.t != 3 ORDER BY (A.a, A.t)
 ;
 
-{% endfor %}
+{% endfor -%}
+{% endfor -%}
 
 DROP TABLE A;
 DROP TABLE B;

From 884dc496a0a978074d3e0bd70f4df8d0225e69c1 Mon Sep 17 00:00:00 2001
From: Xu Jia <jia.xu@clickhouse.com>
Date: Thu, 18 Jul 2024 20:58:37 +0800
Subject: [PATCH 0395/1488] format

---
 .../test_disable_insertion_and_mutation/test.py    | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/tests/integration/test_disable_insertion_and_mutation/test.py b/tests/integration/test_disable_insertion_and_mutation/test.py
index f25964d27b8..b6431690245 100644
--- a/tests/integration/test_disable_insertion_and_mutation/test.py
+++ b/tests/integration/test_disable_insertion_and_mutation/test.py
@@ -35,7 +35,9 @@ def started_cluster():
 
 
 def test_disable_insertion_and_mutation(started_cluster):
-    writing_node.query("""CREATE TABLE my_table on cluster default (key UInt64, value String) ENGINE=ReplicatedMergeTree('/clickhouse/tables/{shard}/default.my_table', '{replica}') ORDER BY key partition by (key % 5) """)
+    writing_node.query(
+        """CREATE TABLE my_table on cluster default (key UInt64, value String) ENGINE=ReplicatedMergeTree('/clickhouse/tables/{shard}/default.my_table', '{replica}') ORDER BY key partition by (key % 5) """
+    )
 
     assert "QUERY_IS_PROHIBITED" in reading_node.query_and_get_error(
         "INSERT INTO my_table VALUES (1, 'hello')"
@@ -58,7 +60,7 @@ def test_disable_insertion_and_mutation(started_cluster):
         "ALTER TABLE my_table drop partition 0"
     )
 
-    reading_node.query("SELECT * from my_table");
+    reading_node.query("SELECT * from my_table")
     writing_node.query("INSERT INTO my_table VALUES (1, 'hello')")
     writing_node.query("ALTER TABLE my_table delete where 1")
     writing_node.query("ALTER table my_table update value = 'no hello' where 1")
@@ -69,10 +71,6 @@ def test_disable_insertion_and_mutation(started_cluster):
 
     reading_node.query("ALter Table my_table MODIFY COLUMN new_column String")
 
-    assert "new_column\tString" in reading_node.query(
-        "DESC my_table"
-    )
+    assert "new_column\tString" in reading_node.query("DESC my_table")
 
-    assert "new_column\tString" in writing_node.query(
-        "DESC my_table"
-    )
+    assert "new_column\tString" in writing_node.query("DESC my_table")

From ab2522b17020f1fb31b760f0594784cd58468797 Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Thu, 18 Jul 2024 13:17:34 +0000
Subject: [PATCH 0396/1488] fix asof join on nulls

---
 src/Processors/Transforms/MergeJoinTransform.cpp | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/Processors/Transforms/MergeJoinTransform.cpp b/src/Processors/Transforms/MergeJoinTransform.cpp
index 7bdb3d4f30f..e96a75d277b 100644
--- a/src/Processors/Transforms/MergeJoinTransform.cpp
+++ b/src/Processors/Transforms/MergeJoinTransform.cpp
@@ -301,7 +301,11 @@ bool JoinKeyRow::asofMatch(const FullMergeJoinCursor & cursor, ASOFJoinInequalit
     if (isNullAt(*asof_row, 0) || isNullAt(*cursor.getAsofColumn(), cursor->getRow()))
         return false;
 
-    int cmp = cursor.getAsofColumn()->compareAt(cursor->getRow(), 0, *asof_row, 1);
+    int cmp = 0;
+    if (const auto * nullable_column = checkAndGetColumn<ColumnNullable>(cursor.getAsofColumn()))
+        cmp = nullable_column->getNestedColumn().compareAt(cursor->getRow(), 0, *asof_row, 1);
+    else
+        cmp = cursor.getAsofColumn()->compareAt(cursor->getRow(), 0, *asof_row, 1);
 
     return (asof_inequality == ASOFJoinInequality::Less && cmp < 0)
         || (asof_inequality == ASOFJoinInequality::LessOrEquals && cmp <= 0)

From 617840c7270bf90068e04d348db138fd8c1f3456 Mon Sep 17 00:00:00 2001
From: Sema Checherinda <Sema.Checherinda@clickhouse.com>
Date: Thu, 18 Jul 2024 15:19:41 +0200
Subject: [PATCH 0397/1488] add test

---
 ...duplication_remote_insert_select.reference |  0
 ...008_deduplication_remote_insert_select.sql | 45 +++++++++++++++++++
 2 files changed, 45 insertions(+)
 create mode 100644 tests/queries/0_stateless/03008_deduplication_remote_insert_select.reference
 create mode 100644 tests/queries/0_stateless/03008_deduplication_remote_insert_select.sql

diff --git a/tests/queries/0_stateless/03008_deduplication_remote_insert_select.reference b/tests/queries/0_stateless/03008_deduplication_remote_insert_select.reference
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/queries/0_stateless/03008_deduplication_remote_insert_select.sql b/tests/queries/0_stateless/03008_deduplication_remote_insert_select.sql
new file mode 100644
index 00000000000..1c229cdbc26
--- /dev/null
+++ b/tests/queries/0_stateless/03008_deduplication_remote_insert_select.sql
@@ -0,0 +1,45 @@
+DROP TABLE IF EXISTS tt;
+CREATE TABLE src (a UInt64, b UInt64)
+    ENGINE=ReplicatedMergeTree('/clickhouse/tables/{database}/03008_deduplication_remote_insert_select/src', '{replica}')
+    ORDER BY tuple();
+
+INSERT INTO src SELECT number % 10 as a, number as b FROM numbers(50);
+
+SET allow_experimental_parallel_reading_from_replicas=1;
+SET max_parallel_replicas=3;
+SET parallel_replicas_for_non_replicated_merge_tree=1;
+SET cluster_for_parallel_replicas='parallel_replicas';
+
+SELECT count() FROM remote('127.0.0.{1..6}', currentDatabase(), src);
+
+CREATE TABLE dst_null(a UInt64, b UInt64) ENGINE = Null;
+
+set allow_deprecated_syntax_for_merge_tree=1;
+CREATE MATERIALIZED VIEW mv_dst
+ENGINE = AggregatingMergeTree()
+ORDER BY a
+AS SELECT
+    a,
+    sumState(b)  AS sum_b,
+    uniqState(b) AS uniq_b
+FROM dst_null
+GROUP BY a;
+
+INSERT INTO dst_null
+SELECT
+    a,
+    b
+FROM src;
+
+SELECT
+    a,
+    sumMerge(sum_b)                AS sum_b,
+    uniqMerge(uniq_b)              AS uniq_b
+FROM mv_dst
+GROUP BY a
+ORDER BY a;
+
+DROP TABLE src;
+DROP TABLE mv_dst;
+DROP TABLE dst_null;
+

From e128d88ff11f78be47e0b6ae52164f1313e2ed6c Mon Sep 17 00:00:00 2001
From: Sema Checherinda <Sema.Checherinda@clickhouse.com>
Date: Thu, 18 Jul 2024 16:42:47 +0200
Subject: [PATCH 0398/1488] fix CollectionOfDerivedItems::append to
 appendIfUniq

---
 src/Common/CollectionOfDerived.h              | 13 +++---
 src/Interpreters/Squashing.cpp                |  2 +-
 .../DeduplicationTokenTransforms.cpp          |  2 +-
 ...duplication_remote_insert_select.reference | 36 +++++++++++++++
 ...008_deduplication_remote_insert_select.sql | 44 +++++++++++--------
 5 files changed, 71 insertions(+), 26 deletions(-)

diff --git a/src/Common/CollectionOfDerived.h b/src/Common/CollectionOfDerived.h
index 97c0c3fbc06..9f80ff727b4 100644
--- a/src/Common/CollectionOfDerived.h
+++ b/src/Common/CollectionOfDerived.h
@@ -84,12 +84,18 @@ public:
         return result;
     }
 
-    void append(Self && other)
+    // append items for other inscnace only if there is no such item in current instance
+    void appendIfUniq(Self && other)
     {
         auto middle_idx = records.size();
         std::move(other.records.begin(), other.records.end(), std::back_inserter(records));
+        // merge is stable
         std::inplace_merge(records.begin(), records.begin() + middle_idx, records.end());
-        chassert(isUniqTypes());
+        // remove duplicates
+        records.erase(std::unique(records.begin(), records.end()), records.end());
+
+        assert(std::is_sorted(records.begin(), records.end()));
+        assert(isUniqTypes());
     }
 
     template <class T>
@@ -142,7 +148,6 @@ private:
     bool isUniqTypes() const
     {
         auto uniq_it = std::adjacent_find(records.begin(), records.end());
-
         return uniq_it == records.end();
     }
 
@@ -161,8 +166,6 @@ private:
 
 
         records.emplace(it, type_idx, item);
-
-        chassert(isUniqTypes());
     }
 
     Records::const_iterator getImpl(std::type_index type_idx) const
diff --git a/src/Interpreters/Squashing.cpp b/src/Interpreters/Squashing.cpp
index 3a7f28ed837..488177c3b4f 100644
--- a/src/Interpreters/Squashing.cpp
+++ b/src/Interpreters/Squashing.cpp
@@ -134,7 +134,7 @@ Chunk Squashing::squash(std::vector<Chunk> && input_chunks, Chunk::ChunkInfoColl
     Chunk result;
     result.setColumns(std::move(mutable_columns), rows);
     result.setChunkInfos(infos);
-    result.getChunkInfos().append(std::move(input_chunks.back().getChunkInfos()));
+    result.getChunkInfos().appendIfUniq(std::move(input_chunks.back().getChunkInfos()));
 
     chassert(result);
     return result;
diff --git a/src/Processors/Transforms/DeduplicationTokenTransforms.cpp b/src/Processors/Transforms/DeduplicationTokenTransforms.cpp
index e6f7e44e026..841090f029e 100644
--- a/src/Processors/Transforms/DeduplicationTokenTransforms.cpp
+++ b/src/Processors/Transforms/DeduplicationTokenTransforms.cpp
@@ -20,7 +20,7 @@ namespace ErrorCodes
 
 void RestoreChunkInfosTransform::transform(Chunk & chunk)
 {
-    chunk.getChunkInfos().append(chunk_infos.clone());
+    chunk.getChunkInfos().appendIfUniq(chunk_infos.clone());
 }
 
 namespace DeduplicationToken
diff --git a/tests/queries/0_stateless/03008_deduplication_remote_insert_select.reference b/tests/queries/0_stateless/03008_deduplication_remote_insert_select.reference
index e69de29bb2d..9dd45974e40 100644
--- a/tests/queries/0_stateless/03008_deduplication_remote_insert_select.reference
+++ b/tests/queries/0_stateless/03008_deduplication_remote_insert_select.reference
@@ -0,0 +1,36 @@
+-- { echoOn }
+SELECT count() FROM src;
+100
+SELECT a, sum(b), uniq(b), FROM src GROUP BY a ORDER BY a;
+0	450	10
+1	460	10
+2	470	10
+3	480	10
+4	490	10
+5	500	10
+6	510	10
+7	520	10
+8	530	10
+9	540	10
+SELECT count() FROM remote('127.0.0.{1..2}', currentDatabase(), src);
+200
+-- { echoOn }
+INSERT INTO dst_null
+    SELECT a, b FROM src;
+SELECT
+    a,
+    sumMerge(sum_b) AS sum_b,
+    uniqMerge(uniq_b) AS uniq_b
+FROM mv_dst
+GROUP BY a
+ORDER BY a;
+0	450	10
+1	460	10
+2	470	10
+3	480	10
+4	490	10
+5	500	10
+6	510	10
+7	520	10
+8	530	10
+9	540	10
diff --git a/tests/queries/0_stateless/03008_deduplication_remote_insert_select.sql b/tests/queries/0_stateless/03008_deduplication_remote_insert_select.sql
index 1c229cdbc26..c8e092822da 100644
--- a/tests/queries/0_stateless/03008_deduplication_remote_insert_select.sql
+++ b/tests/queries/0_stateless/03008_deduplication_remote_insert_select.sql
@@ -1,43 +1,49 @@
-DROP TABLE IF EXISTS tt;
+DROP TABLE IF EXISTS src;
+
 CREATE TABLE src (a UInt64, b UInt64)
     ENGINE=ReplicatedMergeTree('/clickhouse/tables/{database}/03008_deduplication_remote_insert_select/src', '{replica}')
     ORDER BY tuple();
 
-INSERT INTO src SELECT number % 10 as a, number as b FROM numbers(50);
+INSERT INTO src SELECT number % 10 as a, number as b FROM numbers(100);
 
 SET allow_experimental_parallel_reading_from_replicas=1;
 SET max_parallel_replicas=3;
 SET parallel_replicas_for_non_replicated_merge_tree=1;
 SET cluster_for_parallel_replicas='parallel_replicas';
 
-SELECT count() FROM remote('127.0.0.{1..6}', currentDatabase(), src);
+-- { echoOn }
+SELECT count() FROM src;
+SELECT a, sum(b), uniq(b), FROM src GROUP BY a ORDER BY a;
+SELECT count() FROM remote('127.0.0.{1..2}', currentDatabase(), src);
+-- { echoOff }
 
-CREATE TABLE dst_null(a UInt64, b UInt64) ENGINE = Null;
+DROP TABLE IF EXISTS dst_null;
+CREATE TABLE dst_null(a UInt64, b UInt64)
+    ENGINE = Null;
 
-set allow_deprecated_syntax_for_merge_tree=1;
+DROP TABLE IF EXISTS mv_dst;
 CREATE MATERIALIZED VIEW mv_dst
-ENGINE = AggregatingMergeTree()
-ORDER BY a
-AS SELECT
-    a,
-    sumState(b)  AS sum_b,
-    uniqState(b) AS uniq_b
-FROM dst_null
-GROUP BY a;
+    ENGINE = AggregatingMergeTree()
+    ORDER BY a
+    AS SELECT
+        a,
+        sumState(b)  AS sum_b,
+        uniqState(b) AS uniq_b
+    FROM dst_null
+    GROUP BY a;
 
+-- { echoOn }
 INSERT INTO dst_null
-SELECT
-    a,
-    b
-FROM src;
+    SELECT a, b FROM src;
 
 SELECT
     a,
-    sumMerge(sum_b)                AS sum_b,
-    uniqMerge(uniq_b)              AS uniq_b
+    sumMerge(sum_b) AS sum_b,
+    uniqMerge(uniq_b) AS uniq_b
 FROM mv_dst
 GROUP BY a
 ORDER BY a;
+-- { echoOff }
 
 DROP TABLE src;
 DROP TABLE mv_dst;

From c01e2cbeea02ebecfc4dea4692baffff3087b043 Mon Sep 17 00:00:00 2001
From: Xu Jia <jia.xu@clickhouse.com>
Date: Thu, 18 Jul 2024 22:58:30 +0800
Subject: [PATCH 0399/1488] format

---
 tests/integration/test_disable_insertion_and_mutation/test.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/integration/test_disable_insertion_and_mutation/test.py b/tests/integration/test_disable_insertion_and_mutation/test.py
index b6431690245..b6da7ed548f 100644
--- a/tests/integration/test_disable_insertion_and_mutation/test.py
+++ b/tests/integration/test_disable_insertion_and_mutation/test.py
@@ -50,7 +50,6 @@ def test_disable_insertion_and_mutation(started_cluster):
     assert "QUERY_IS_PROHIBITED" in reading_node.query_and_get_error(
         "ALTER TABLE my_table delete where 1"
     )
-    
 
     assert "QUERY_IS_PROHIBITED" in reading_node.query_and_get_error(
         "ALTER table my_table update key = 1 where 1"

From cda846339be22c66cd0d35d49273a314fa3bdf69 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Thu, 18 Jul 2024 15:12:32 +0000
Subject: [PATCH 0400/1488] Remove ActionsDAG::clone

---
 src/Interpreters/ActionsDAG.cpp               |  8 ----
 src/Interpreters/ActionsDAG.h                 |  3 --
 src/Interpreters/ExpressionAnalyzer.cpp       |  9 +----
 src/Interpreters/InterpreterSelectQuery.cpp   | 24 ++++++------
 src/Interpreters/MutationsInterpreter.cpp     |  4 +-
 src/Planner/Planner.cpp                       | 38 +++++++++----------
 src/Planner/PlannerExpressionAnalysis.h       | 16 ++++----
 src/Planner/PlannerJoins.cpp                  |  4 +-
 src/Processors/QueryPlan/ExpressionStep.cpp   |  4 +-
 src/Processors/QueryPlan/FilterStep.cpp       |  4 +-
 .../Optimizations/distinctReadInOrder.cpp     | 10 ++---
 .../Optimizations/filterPushDown.cpp          |  4 +-
 .../QueryPlan/Optimizations/liftUpUnion.cpp   |  2 +-
 .../optimizePrimaryKeyConditionAndLimit.cpp   |  6 +--
 .../Optimizations/optimizeReadInOrder.cpp     | 28 +++++++-------
 .../optimizeUseAggregateProjection.cpp        |  4 +-
 .../Optimizations/projectionsCommon.cpp       |  7 ++--
 .../Optimizations/removeRedundantDistinct.cpp | 20 +++++-----
 .../QueryPlan/ReadFromMergeTree.cpp           | 16 ++++----
 .../QueryPlan/SourceStepWithFilter.cpp        |  8 ++--
 src/Processors/QueryPlan/TotalsHavingStep.cpp | 22 +++++++----
 .../Transforms/FillingTransform.cpp           |  2 +-
 src/Storages/MergeTree/MergeTreeIndexSet.cpp  |  4 +-
 .../MergeTree/MergeTreeSelectProcessor.cpp    |  4 +-
 src/Storages/SelectQueryInfo.h                |  4 +-
 src/Storages/StorageBuffer.cpp                | 10 +++--
 src/Storages/StorageMerge.cpp                 |  8 ++--
 src/Storages/VirtualColumnUtils.cpp           |  2 +-
 28 files changed, 135 insertions(+), 140 deletions(-)

diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp
index e001406408f..53e04f24829 100644
--- a/src/Interpreters/ActionsDAG.cpp
+++ b/src/Interpreters/ActionsDAG.cpp
@@ -1246,14 +1246,6 @@ bool ActionsDAG::removeUnusedResult(const std::string & column_name)
     return true;
 }
 
-ActionsDAGPtr ActionsDAG::clone(const ActionsDAG * from)
-{
-    std::unordered_map<const Node *, Node *> old_to_new_nodes;
-    if (from == nullptr)
-        return nullptr;
-    return std::make_unique<ActionsDAG>(from->clone(old_to_new_nodes));
-}
-
 ActionsDAG ActionsDAG::clone() const
 {
     std::unordered_map<const Node *, Node *> old_to_new_nodes;
diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h
index 6f5c3d3b0df..6f6c3f9bccb 100644
--- a/src/Interpreters/ActionsDAG.h
+++ b/src/Interpreters/ActionsDAG.h
@@ -261,8 +261,6 @@ public:
     void compileExpressions(size_t min_count_to_compile_expression, const std::unordered_set<const Node *> & lazy_executed_nodes = {});
 #endif
 
-    static ActionsDAGPtr clone(const ActionsDAG * from);
-
     ActionsDAG clone(std::unordered_map<const Node *, Node *> & old_to_new_nodes) const;
     ActionsDAG clone() const;
 
@@ -491,7 +489,6 @@ public:
     const ActionsDAG::Node * find(const String & output_name);
 
 private:
-    //const ActionsDAG & actions;
     NameToNodeIndex index;
 };
 
diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp
index 0f350602777..6b5b129085d 100644
--- a/src/Interpreters/ExpressionAnalyzer.cpp
+++ b/src/Interpreters/ExpressionAnalyzer.cpp
@@ -1944,10 +1944,7 @@ ExpressionAnalysisResult::ExpressionAnalysisResult(
                 Block before_prewhere_sample = source_header;
                 if (sanitizeBlock(before_prewhere_sample))
                 {
-                    ActionsDAG dag = std::move(*ActionsDAG::clone(&prewhere_dag_and_flags->dag));
-                    ExpressionActions(
-                        std::move(dag),
-                        ExpressionActionsSettings::fromSettings(context->getSettingsRef())).execute(before_prewhere_sample);
+                    prewhere_dag_and_flags->dag.updateHeader(before_prewhere_sample);
                     auto & column_elem = before_prewhere_sample.getByName(query.prewhere()->getColumnName());
                     /// If the filter column is a constant, record it.
                     if (column_elem.column)
@@ -1979,9 +1976,7 @@ ExpressionAnalysisResult::ExpressionAnalysisResult(
                     before_where_sample = source_header;
                 if (sanitizeBlock(before_where_sample))
                 {
-                    ExpressionActions(
-                        std::move(*ActionsDAG::clone(&before_where->dag)),
-                        ExpressionActionsSettings::fromSettings(context->getSettingsRef())).execute(before_where_sample);
+                    before_where->dag.updateHeader(before_where_sample);
 
                     auto & column_elem
                         = before_where_sample.getByName(query.where()->getColumnName());
diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp
index c85eb8310dc..e0073a6af5d 100644
--- a/src/Interpreters/InterpreterSelectQuery.cpp
+++ b/src/Interpreters/InterpreterSelectQuery.cpp
@@ -1501,7 +1501,7 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional<P
         {
             auto row_level_security_step = std::make_unique<FilterStep>(
                 query_plan.getCurrentDataStream(),
-                std::move(*ActionsDAG::clone(&expressions.filter_info->actions)),
+                expressions.filter_info->actions.clone(),
                 expressions.filter_info->column_name,
                 expressions.filter_info->do_remove_column);
 
@@ -1515,7 +1515,7 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional<P
             {
                 auto row_level_filter_step = std::make_unique<FilterStep>(
                     query_plan.getCurrentDataStream(),
-                    std::move(*ActionsDAG::clone(&*expressions.prewhere_info->row_level_filter)),
+                    expressions.prewhere_info->row_level_filter->clone(),
                     expressions.prewhere_info->row_level_column_name,
                     true);
 
@@ -1525,7 +1525,7 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional<P
 
             auto prewhere_step = std::make_unique<FilterStep>(
                 query_plan.getCurrentDataStream(),
-                std::move(*ActionsDAG::clone(&*expressions.prewhere_info->prewhere_actions)),
+                expressions.prewhere_info->prewhere_actions->clone(),
                 expressions.prewhere_info->prewhere_column_name,
                 expressions.prewhere_info->remove_prewhere_column);
 
@@ -1627,7 +1627,7 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional<P
             {
                 auto row_level_security_step = std::make_unique<FilterStep>(
                     query_plan.getCurrentDataStream(),
-                    std::move(*ActionsDAG::clone(&expressions.filter_info->actions)),
+                    expressions.filter_info->actions.clone(),
                     expressions.filter_info->column_name,
                     expressions.filter_info->do_remove_column);
 
@@ -2056,20 +2056,22 @@ void InterpreterSelectQuery::addEmptySourceToQueryPlan(QueryPlan & query_plan, c
     {
         auto & prewhere_info = *query_info.prewhere_info;
 
+        auto row_level_actions = std::make_shared<ExpressionActions>(prewhere_info.row_level_filter->clone());
         if (prewhere_info.row_level_filter)
         {
             pipe.addSimpleTransform([&](const Block & header)
             {
                 return std::make_shared<FilterTransform>(header,
-                    std::make_shared<ExpressionActions>(std::move(*ActionsDAG::clone(&*prewhere_info.row_level_filter))),
+                    row_level_actions,
                     prewhere_info.row_level_column_name, true);
             });
         }
 
+        auto filter_actions = std::make_shared<ExpressionActions>(prewhere_info.prewhere_actions->clone());
         pipe.addSimpleTransform([&](const Block & header)
         {
             return std::make_shared<FilterTransform>(
-                header, std::make_shared<ExpressionActions>(std::move(*ActionsDAG::clone(&*prewhere_info.prewhere_actions))),
+                header, filter_actions,
                 prewhere_info.prewhere_column_name, prewhere_info.remove_prewhere_column);
         });
     }
@@ -2589,7 +2591,7 @@ void InterpreterSelectQuery::executeFetchColumns(QueryProcessingStage::Enum proc
     /// Aliases in table declaration.
     if (processing_stage == QueryProcessingStage::FetchColumns && alias_actions)
     {
-        auto table_aliases = std::make_unique<ExpressionStep>(query_plan.getCurrentDataStream(), std::move(*ActionsDAG::clone(&*alias_actions)));
+        auto table_aliases = std::make_unique<ExpressionStep>(query_plan.getCurrentDataStream(), alias_actions->clone());
         table_aliases->setStepDescription("Add table aliases");
         query_plan.addStep(std::move(table_aliases));
     }
@@ -2597,7 +2599,7 @@ void InterpreterSelectQuery::executeFetchColumns(QueryProcessingStage::Enum proc
 
 void InterpreterSelectQuery::executeWhere(QueryPlan & query_plan, const ActionsAndProjectInputsFlagPtr & expression, bool remove_filter)
 {
-    auto dag = std::move(*ActionsDAG::clone(&expression->dag));
+    auto dag = expression->dag.clone();
     if (expression->project_input)
         dag.appendInputsForUnusedColumns(query_plan.getCurrentDataStream().header);
 
@@ -2771,7 +2773,7 @@ void InterpreterSelectQuery::executeMergeAggregated(QueryPlan & query_plan, bool
 
 void InterpreterSelectQuery::executeHaving(QueryPlan & query_plan, const ActionsAndProjectInputsFlagPtr & expression, bool remove_filter)
 {
-    auto dag = std::move(*ActionsDAG::clone(&expression->dag));
+    auto dag = expression->dag.clone();
     if (expression->project_input)
         dag.appendInputsForUnusedColumns(query_plan.getCurrentDataStream().header);
 
@@ -2789,7 +2791,7 @@ void InterpreterSelectQuery::executeTotalsAndHaving(
     std::optional<ActionsDAG> dag;
     if (expression)
     {
-        dag = std::move(*ActionsDAG::clone(&expression->dag));
+        dag = expression->dag.clone();
         if (expression->project_input)
             dag->appendInputsForUnusedColumns(query_plan.getCurrentDataStream().header);
     }
@@ -2838,7 +2840,7 @@ void InterpreterSelectQuery::executeExpression(QueryPlan & query_plan, const Act
     if (!expression)
         return;
 
-    ActionsDAG dag = std::move(*ActionsDAG::clone(&expression->dag));
+    auto dag = expression->dag.clone();
     if (expression->project_input)
         dag.appendInputsForUnusedColumns(query_plan.getCurrentDataStream().header);
 
diff --git a/src/Interpreters/MutationsInterpreter.cpp b/src/Interpreters/MutationsInterpreter.cpp
index 5b710149d85..57ad5caa4c7 100644
--- a/src/Interpreters/MutationsInterpreter.cpp
+++ b/src/Interpreters/MutationsInterpreter.cpp
@@ -1281,7 +1281,7 @@ QueryPipelineBuilder MutationsInterpreter::addStreamsForLaterStages(const std::v
 
             if (i < stage.filter_column_names.size())
             {
-                auto dag = std::move(*ActionsDAG::clone(&step->actions()->dag));
+                auto dag = step->actions()->dag.clone();
                 if (step->actions()->project_input)
                     dag.appendInputsForUnusedColumns(plan.getCurrentDataStream().header);
                 /// Execute DELETEs.
@@ -1289,7 +1289,7 @@ QueryPipelineBuilder MutationsInterpreter::addStreamsForLaterStages(const std::v
             }
             else
             {
-                auto dag = std::move(*ActionsDAG::clone(&step->actions()->dag));
+                auto dag = step->actions()->dag.clone();
                 if (step->actions()->project_input)
                     dag.appendInputsForUnusedColumns(plan.getCurrentDataStream().header);
                 /// Execute UPDATE or final projection.
diff --git a/src/Planner/Planner.cpp b/src/Planner/Planner.cpp
index e087c3691b4..fb721069e6e 100644
--- a/src/Planner/Planner.cpp
+++ b/src/Planner/Planner.cpp
@@ -333,11 +333,11 @@ public:
 };
 
 void addExpressionStep(QueryPlan & query_plan,
-    const ActionsAndProjectInputsFlagPtr & expression_actions,
+    ActionsAndProjectInputsFlagPtr & expression_actions,
     const std::string & step_description,
     UsefulSets & useful_sets)
 {
-    auto actions = std::move(*ActionsDAG::clone(&expression_actions->dag));
+    auto actions = std::move(expression_actions->dag);
     if (expression_actions->project_input)
         actions.appendInputsForUnusedColumns(query_plan.getCurrentDataStream().header);
 
@@ -348,11 +348,11 @@ void addExpressionStep(QueryPlan & query_plan,
 }
 
 void addFilterStep(QueryPlan & query_plan,
-    const FilterAnalysisResult & filter_analysis_result,
+    FilterAnalysisResult & filter_analysis_result,
     const std::string & step_description,
     UsefulSets & useful_sets)
 {
-    auto actions = std::move(*ActionsDAG::clone(&filter_analysis_result.filter_actions->dag));
+    auto actions = std::move(filter_analysis_result.filter_actions->dag);
     if (filter_analysis_result.filter_actions->project_input)
         actions.appendInputsForUnusedColumns(query_plan.getCurrentDataStream().header);
 
@@ -544,7 +544,7 @@ void addMergingAggregatedStep(QueryPlan & query_plan,
 }
 
 void addTotalsHavingStep(QueryPlan & query_plan,
-    const PlannerExpressionsAnalysisResult & expression_analysis_result,
+    PlannerExpressionsAnalysisResult & expression_analysis_result,
     const QueryAnalysisResult & query_analysis_result,
     const PlannerContextPtr & planner_context,
     const QueryNode & query_node,
@@ -553,14 +553,14 @@ void addTotalsHavingStep(QueryPlan & query_plan,
     const auto & query_context = planner_context->getQueryContext();
     const auto & settings = query_context->getSettingsRef();
 
-    const auto & aggregation_analysis_result = expression_analysis_result.getAggregation();
-    const auto & having_analysis_result = expression_analysis_result.getHaving();
+    auto & aggregation_analysis_result = expression_analysis_result.getAggregation();
+    auto & having_analysis_result = expression_analysis_result.getHaving();
     bool need_finalize = !query_node.isGroupByWithRollup() && !query_node.isGroupByWithCube();
 
     std::optional<ActionsDAG> actions;
     if (having_analysis_result.filter_actions)
     {
-        actions = std::move(*ActionsDAG::clone(&having_analysis_result.filter_actions->dag));
+        actions = std::move(having_analysis_result.filter_actions->dag);
         if (having_analysis_result.filter_actions->project_input)
             actions->appendInputsForUnusedColumns(query_plan.getCurrentDataStream().header);
     }
@@ -886,7 +886,7 @@ bool addPreliminaryLimitOptimizationStepIfNeeded(QueryPlan & query_plan,
   * WINDOW functions.
   */
 void addPreliminarySortOrDistinctOrLimitStepsIfNeeded(QueryPlan & query_plan,
-    const PlannerExpressionsAnalysisResult & expressions_analysis_result,
+    PlannerExpressionsAnalysisResult & expressions_analysis_result,
     const QueryAnalysisResult & query_analysis_result,
     const PlannerContextPtr & planner_context,
     const PlannerQueryProcessingInfo & query_processing_info,
@@ -922,7 +922,7 @@ void addPreliminarySortOrDistinctOrLimitStepsIfNeeded(QueryPlan & query_plan,
 
     if (expressions_analysis_result.hasLimitBy())
     {
-        const auto & limit_by_analysis_result = expressions_analysis_result.getLimitBy();
+        auto & limit_by_analysis_result = expressions_analysis_result.getLimitBy();
         addExpressionStep(query_plan, limit_by_analysis_result.before_limit_by_actions, "Before LIMIT BY", useful_sets);
         addLimitByStep(query_plan, limit_by_analysis_result, query_node);
     }
@@ -1549,7 +1549,7 @@ void Planner::buildPlanForQueryNode()
 
         if (expression_analysis_result.hasAggregation())
         {
-            const auto & aggregation_analysis_result = expression_analysis_result.getAggregation();
+            auto & aggregation_analysis_result = expression_analysis_result.getAggregation();
             if (aggregation_analysis_result.before_aggregation_actions)
                 addExpressionStep(query_plan, aggregation_analysis_result.before_aggregation_actions, "Before GROUP BY", useful_sets);
 
@@ -1568,7 +1568,7 @@ void Planner::buildPlanForQueryNode()
                   * window functions, we can't execute ORDER BY and DISTINCT
                   * now, on shard (first_stage).
                   */
-                const auto & window_analysis_result = expression_analysis_result.getWindow();
+                auto & window_analysis_result = expression_analysis_result.getWindow();
                 if (window_analysis_result.before_window_actions)
                     addExpressionStep(query_plan, window_analysis_result.before_window_actions, "Before WINDOW", useful_sets);
             }
@@ -1578,7 +1578,7 @@ void Planner::buildPlanForQueryNode()
                   * Projection expressions, preliminary DISTINCT and before ORDER BY expressions
                   * now, on shards (first_stage).
                   */
-                const auto & projection_analysis_result = expression_analysis_result.getProjection();
+                auto & projection_analysis_result = expression_analysis_result.getProjection();
                 addExpressionStep(query_plan, projection_analysis_result.projection_actions, "Projection", useful_sets);
 
                 if (query_node.isDistinct())
@@ -1594,7 +1594,7 @@ void Planner::buildPlanForQueryNode()
 
                 if (expression_analysis_result.hasSort())
                 {
-                    const auto & sort_analysis_result = expression_analysis_result.getSort();
+                    auto & sort_analysis_result = expression_analysis_result.getSort();
                     addExpressionStep(query_plan, sort_analysis_result.before_order_by_actions, "Before ORDER BY", useful_sets);
                 }
             }
@@ -1648,7 +1648,7 @@ void Planner::buildPlanForQueryNode()
         {
             if (expression_analysis_result.hasWindow())
             {
-                const auto & window_analysis_result = expression_analysis_result.getWindow();
+                auto & window_analysis_result = expression_analysis_result.getWindow();
                 if (expression_analysis_result.hasAggregation())
                     addExpressionStep(query_plan, window_analysis_result.before_window_actions, "Before window functions", useful_sets);
 
@@ -1658,7 +1658,7 @@ void Planner::buildPlanForQueryNode()
             if (expression_analysis_result.hasQualify())
                 addFilterStep(query_plan, expression_analysis_result.getQualify(), "QUALIFY", useful_sets);
 
-            const auto & projection_analysis_result = expression_analysis_result.getProjection();
+            auto & projection_analysis_result = expression_analysis_result.getProjection();
             addExpressionStep(query_plan, projection_analysis_result.projection_actions, "Projection", useful_sets);
 
             if (query_node.isDistinct())
@@ -1674,7 +1674,7 @@ void Planner::buildPlanForQueryNode()
 
             if (expression_analysis_result.hasSort())
             {
-                const auto & sort_analysis_result = expression_analysis_result.getSort();
+                auto & sort_analysis_result = expression_analysis_result.getSort();
                 addExpressionStep(query_plan, sort_analysis_result.before_order_by_actions, "Before ORDER BY", useful_sets);
             }
         }
@@ -1727,7 +1727,7 @@ void Planner::buildPlanForQueryNode()
 
         if (!query_processing_info.isFromAggregationState() && expression_analysis_result.hasLimitBy())
         {
-            const auto & limit_by_analysis_result = expression_analysis_result.getLimitBy();
+            auto & limit_by_analysis_result = expression_analysis_result.getLimitBy();
             addExpressionStep(query_plan, limit_by_analysis_result.before_limit_by_actions, "Before LIMIT BY", useful_sets);
             addLimitByStep(query_plan, limit_by_analysis_result, query_node);
         }
@@ -1759,7 +1759,7 @@ void Planner::buildPlanForQueryNode()
         /// Project names is not done on shards, because initiator will not find columns in blocks
         if (!query_processing_info.isToAggregationState())
         {
-            const auto & projection_analysis_result = expression_analysis_result.getProjection();
+            auto & projection_analysis_result = expression_analysis_result.getProjection();
             addExpressionStep(query_plan, projection_analysis_result.project_names_actions, "Project names", useful_sets);
         }
 
diff --git a/src/Planner/PlannerExpressionAnalysis.h b/src/Planner/PlannerExpressionAnalysis.h
index 820df7131a7..283fcac7aba 100644
--- a/src/Planner/PlannerExpressionAnalysis.h
+++ b/src/Planner/PlannerExpressionAnalysis.h
@@ -64,7 +64,7 @@ public:
         : projection_analysis_result(std::move(projection_analysis_result_))
     {}
 
-    const ProjectionAnalysisResult & getProjection() const
+    ProjectionAnalysisResult & getProjection()
     {
         return projection_analysis_result;
     }
@@ -74,7 +74,7 @@ public:
         return where_analysis_result.filter_actions != nullptr;
     }
 
-    const FilterAnalysisResult & getWhere() const
+    FilterAnalysisResult & getWhere()
     {
         return where_analysis_result;
     }
@@ -89,7 +89,7 @@ public:
         return !aggregation_analysis_result.aggregation_keys.empty() || !aggregation_analysis_result.aggregate_descriptions.empty();
     }
 
-    const AggregationAnalysisResult & getAggregation() const
+    AggregationAnalysisResult & getAggregation()
     {
         return aggregation_analysis_result;
     }
@@ -104,7 +104,7 @@ public:
         return having_analysis_result.filter_actions != nullptr;
     }
 
-    const FilterAnalysisResult & getHaving() const
+    FilterAnalysisResult & getHaving()
     {
         return having_analysis_result;
     }
@@ -119,7 +119,7 @@ public:
         return !window_analysis_result.window_descriptions.empty();
     }
 
-    const WindowAnalysisResult & getWindow() const
+    WindowAnalysisResult & getWindow()
     {
         return window_analysis_result;
     }
@@ -134,7 +134,7 @@ public:
         return qualify_analysis_result.filter_actions != nullptr;
     }
 
-    const FilterAnalysisResult & getQualify() const
+    FilterAnalysisResult & getQualify()
     {
         return qualify_analysis_result;
     }
@@ -149,7 +149,7 @@ public:
         return sort_analysis_result.before_order_by_actions != nullptr;
     }
 
-    const SortAnalysisResult & getSort() const
+    SortAnalysisResult & getSort()
     {
         return sort_analysis_result;
     }
@@ -164,7 +164,7 @@ public:
         return limit_by_analysis_result.before_limit_by_actions != nullptr;
     }
 
-    const LimitByAnalysisResult & getLimitBy() const
+    LimitByAnalysisResult & getLimitBy()
     {
         return limit_by_analysis_result;
     }
diff --git a/src/Planner/PlannerJoins.cpp b/src/Planner/PlannerJoins.cpp
index 4cf1b138bed..5acff9dac82 100644
--- a/src/Planner/PlannerJoins.cpp
+++ b/src/Planner/PlannerJoins.cpp
@@ -591,10 +591,10 @@ JoinClausesAndActions buildJoinClausesAndActions(
         }
     }
 
-    result.left_join_expressions_actions = std::move(*ActionsDAG::clone(&left_join_actions));
+    result.left_join_expressions_actions = left_join_actions.clone();
     result.left_join_tmp_expression_actions = std::move(left_join_actions);
     result.left_join_expressions_actions.removeUnusedActions(join_left_actions_names);
-    result.right_join_expressions_actions = std::move(*ActionsDAG::clone(&right_join_actions));
+    result.right_join_expressions_actions = right_join_actions.clone();
     result.right_join_tmp_expression_actions = std::move(right_join_actions);
     result.right_join_expressions_actions.removeUnusedActions(join_right_actions_names);
 
diff --git a/src/Processors/QueryPlan/ExpressionStep.cpp b/src/Processors/QueryPlan/ExpressionStep.cpp
index 94098f443d9..6f88c4527a4 100644
--- a/src/Processors/QueryPlan/ExpressionStep.cpp
+++ b/src/Processors/QueryPlan/ExpressionStep.cpp
@@ -61,13 +61,13 @@ void ExpressionStep::transformPipeline(QueryPipelineBuilder & pipeline, const Bu
 void ExpressionStep::describeActions(FormatSettings & settings) const
 {
     String prefix(settings.offset, settings.indent_char);
-    auto expression = std::make_shared<ExpressionActions>(std::move(*ActionsDAG::clone(&actions_dag)));
+    auto expression = std::make_shared<ExpressionActions>(actions_dag.clone());
     expression->describeActions(settings.out, prefix);
 }
 
 void ExpressionStep::describeActions(JSONBuilder::JSONMap & map) const
 {
-    auto expression = std::make_shared<ExpressionActions>(std::move(*ActionsDAG::clone(&actions_dag)));
+    auto expression = std::make_shared<ExpressionActions>(actions_dag.clone());
     map.add("Expression", expression->toTree());
 }
 
diff --git a/src/Processors/QueryPlan/FilterStep.cpp b/src/Processors/QueryPlan/FilterStep.cpp
index 5f15c5defac..0c6b71387b7 100644
--- a/src/Processors/QueryPlan/FilterStep.cpp
+++ b/src/Processors/QueryPlan/FilterStep.cpp
@@ -87,7 +87,7 @@ void FilterStep::describeActions(FormatSettings & settings) const
         settings.out << " (removed)";
     settings.out << '\n';
 
-    auto expression = std::make_shared<ExpressionActions>(std::move(*ActionsDAG::clone(&actions_dag)));
+    auto expression = std::make_shared<ExpressionActions>(actions_dag.clone());
     expression->describeActions(settings.out, prefix);
 }
 
@@ -96,7 +96,7 @@ void FilterStep::describeActions(JSONBuilder::JSONMap & map) const
     map.add("Filter Column", filter_column_name);
     map.add("Removes Filter", remove_filter_column);
 
-    auto expression = std::make_shared<ExpressionActions>(std::move(*ActionsDAG::clone(&actions_dag)));
+    auto expression = std::make_shared<ExpressionActions>(actions_dag.clone());
     map.add("Expression", expression->toTree());
 }
 
diff --git a/src/Processors/QueryPlan/Optimizations/distinctReadInOrder.cpp b/src/Processors/QueryPlan/Optimizations/distinctReadInOrder.cpp
index 8666912514e..37e61a6c388 100644
--- a/src/Processors/QueryPlan/Optimizations/distinctReadInOrder.cpp
+++ b/src/Processors/QueryPlan/Optimizations/distinctReadInOrder.cpp
@@ -10,18 +10,18 @@
 namespace DB::QueryPlanOptimizations
 {
 /// build actions DAG from stack of steps
-static ActionsDAGPtr buildActionsForPlanPath(std::vector<const ActionsDAG *> & dag_stack)
+static std::optional<ActionsDAG> buildActionsForPlanPath(std::vector<const ActionsDAG *> & dag_stack)
 {
     if (dag_stack.empty())
-        return nullptr;
+        return {};
 
-    ActionsDAGPtr path_actions = ActionsDAG::clone(dag_stack.back());
+    ActionsDAG path_actions = dag_stack.back()->clone();
     dag_stack.pop_back();
     while (!dag_stack.empty())
     {
-        ActionsDAGPtr clone = ActionsDAG::clone(dag_stack.back());
+        ActionsDAG clone = dag_stack.back()->clone();
         dag_stack.pop_back();
-        path_actions->mergeInplace(std::move(*clone));
+        path_actions.mergeInplace(std::move(clone));
     }
     return path_actions;
 }
diff --git a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp
index 411b20b1a32..73314f005b6 100644
--- a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp
+++ b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp
@@ -599,7 +599,7 @@ size_t tryPushDownFilter(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes
 
             filter_node.step = std::make_unique<FilterStep>(
                 filter_node.children.front()->step->getOutputStream(),
-                std::move(*ActionsDAG::clone(&filter->getExpression())),
+                filter->getExpression().clone(),
                 filter->getFilterColumnName(),
                 filter->removesFilterColumn());
         }
@@ -613,7 +613,7 @@ size_t tryPushDownFilter(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes
 
     if (auto * read_from_merge = typeid_cast<ReadFromMerge *>(child.get()))
     {
-        FilterDAGInfo info{std::move(*ActionsDAG::clone(&filter->getExpression())), filter->getFilterColumnName(), filter->removesFilterColumn()};
+        FilterDAGInfo info{filter->getExpression().clone(), filter->getFilterColumnName(), filter->removesFilterColumn()};
         read_from_merge->addFilter(std::move(info));
         std::swap(*parent_node, *child_node);
         return 1;
diff --git a/src/Processors/QueryPlan/Optimizations/liftUpUnion.cpp b/src/Processors/QueryPlan/Optimizations/liftUpUnion.cpp
index 53f59198d0f..c48551732c9 100644
--- a/src/Processors/QueryPlan/Optimizations/liftUpUnion.cpp
+++ b/src/Processors/QueryPlan/Optimizations/liftUpUnion.cpp
@@ -49,7 +49,7 @@ size_t tryLiftUpUnion(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes)
 
             expr_node.step = std::make_unique<ExpressionStep>(
                 expr_node.children.front()->step->getOutputStream(),
-                std::move(*ActionsDAG::clone(&expression->getExpression())));
+                expression->getExpression().clone());
         }
 
         ///       - Expression - Something
diff --git a/src/Processors/QueryPlan/Optimizations/optimizePrimaryKeyConditionAndLimit.cpp b/src/Processors/QueryPlan/Optimizations/optimizePrimaryKeyConditionAndLimit.cpp
index 71a7ca327b1..63b4e019066 100644
--- a/src/Processors/QueryPlan/Optimizations/optimizePrimaryKeyConditionAndLimit.cpp
+++ b/src/Processors/QueryPlan/Optimizations/optimizePrimaryKeyConditionAndLimit.cpp
@@ -18,16 +18,16 @@ void optimizePrimaryKeyConditionAndLimit(const Stack & stack)
     const auto & storage_prewhere_info = source_step_with_filter->getPrewhereInfo();
     if (storage_prewhere_info)
     {
-        source_step_with_filter->addFilter(ActionsDAG::clone(&*storage_prewhere_info->prewhere_actions), storage_prewhere_info->prewhere_column_name);
+        source_step_with_filter->addFilter(std::make_unique<ActionsDAG>(storage_prewhere_info->prewhere_actions->clone()), storage_prewhere_info->prewhere_column_name);
         if (storage_prewhere_info->row_level_filter)
-            source_step_with_filter->addFilter(ActionsDAG::clone(&*storage_prewhere_info->row_level_filter), storage_prewhere_info->row_level_column_name);
+            source_step_with_filter->addFilter(std::make_unique<ActionsDAG>(storage_prewhere_info->row_level_filter->clone()), storage_prewhere_info->row_level_column_name);
     }
 
     for (auto iter = stack.rbegin() + 1; iter != stack.rend(); ++iter)
     {
         if (auto * filter_step = typeid_cast<FilterStep *>(iter->node->step.get()))
         {
-            source_step_with_filter->addFilter(ActionsDAG::clone(&filter_step->getExpression()), filter_step->getFilterColumnName());
+            source_step_with_filter->addFilter(std::make_unique<ActionsDAG>(filter_step->getExpression().clone()), filter_step->getFilterColumnName());
         }
         else if (auto * limit_step = typeid_cast<LimitStep *>(iter->node->step.get()))
         {
diff --git a/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp b/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp
index 99aaef6d054..252420e19fe 100644
--- a/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp
+++ b/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp
@@ -171,17 +171,17 @@ static void appendFixedColumnsFromFilterExpression(const ActionsDAG::Node & filt
     }
 }
 
-static void appendExpression(ActionsDAGPtr & dag, const ActionsDAG & expression)
+static void appendExpression(std::optional<ActionsDAG> & dag, const ActionsDAG & expression)
 {
     if (dag)
-        dag->mergeInplace(std::move(*ActionsDAG::clone(&expression)));
+        dag->mergeInplace(expression.clone());
     else
-        dag = ActionsDAG::clone(&expression);
+        dag = expression.clone();
 }
 
 /// This function builds a common DAG which is a merge of DAGs from Filter and Expression steps chain.
 /// Additionally, build a set of fixed columns.
-void buildSortingDAG(QueryPlan::Node & node, ActionsDAGPtr & dag, FixedColumns & fixed_columns, size_t & limit)
+void buildSortingDAG(QueryPlan::Node & node, std::optional<ActionsDAG> & dag, FixedColumns & fixed_columns, size_t & limit)
 {
     IQueryPlanStep * step = node.step.get();
     if (auto * reading = typeid_cast<ReadFromMergeTree *>(step))
@@ -330,7 +330,7 @@ void enreachFixedColumns(const ActionsDAG & dag, FixedColumns & fixed_columns)
 
 InputOrderInfoPtr buildInputOrderInfo(
     const FixedColumns & fixed_columns,
-    const ActionsDAGPtr & dag,
+    const std::optional<ActionsDAG> & dag,
     const SortDescription & description,
     const KeyDescription & sorting_key,
     size_t limit)
@@ -507,7 +507,7 @@ struct AggregationInputOrder
 
 AggregationInputOrder buildInputOrderInfo(
     const FixedColumns & fixed_columns,
-    const ActionsDAGPtr & dag,
+    const std::optional<ActionsDAG> & dag,
     const Names & group_by_keys,
     const ActionsDAG & sorting_key_dag,
     const Names & sorting_key_columns)
@@ -693,7 +693,7 @@ AggregationInputOrder buildInputOrderInfo(
 InputOrderInfoPtr buildInputOrderInfo(
     const ReadFromMergeTree * reading,
     const FixedColumns & fixed_columns,
-    const ActionsDAGPtr & dag,
+    const std::optional<ActionsDAG> & dag,
     const SortDescription & description,
     size_t limit)
 {
@@ -709,7 +709,7 @@ InputOrderInfoPtr buildInputOrderInfo(
 InputOrderInfoPtr buildInputOrderInfo(
     ReadFromMerge * merge,
     const FixedColumns & fixed_columns,
-    const ActionsDAGPtr & dag,
+    const std::optional<ActionsDAG> & dag,
     const SortDescription & description,
     size_t limit)
 {
@@ -745,7 +745,7 @@ InputOrderInfoPtr buildInputOrderInfo(
 AggregationInputOrder buildInputOrderInfo(
     ReadFromMergeTree * reading,
     const FixedColumns & fixed_columns,
-    const ActionsDAGPtr & dag,
+    const std::optional<ActionsDAG> & dag,
     const Names & group_by_keys)
 {
     const auto & sorting_key = reading->getStorageMetadata()->getSortingKey();
@@ -760,7 +760,7 @@ AggregationInputOrder buildInputOrderInfo(
 AggregationInputOrder buildInputOrderInfo(
     ReadFromMerge * merge,
     const FixedColumns & fixed_columns,
-    const ActionsDAGPtr & dag,
+    const std::optional<ActionsDAG> & dag,
     const Names & group_by_keys)
 {
     const auto & tables = merge->getSelectedTables();
@@ -801,7 +801,7 @@ InputOrderInfoPtr buildInputOrderInfo(SortingStep & sorting, QueryPlan::Node & n
     const auto & description = sorting.getSortDescription();
     size_t limit = sorting.getLimit();
 
-    ActionsDAGPtr dag;
+    std::optional<ActionsDAG> dag;
     FixedColumns fixed_columns;
     buildSortingDAG(node, dag, fixed_columns, limit);
 
@@ -855,7 +855,7 @@ AggregationInputOrder buildInputOrderInfo(AggregatingStep & aggregating, QueryPl
     const auto & keys = aggregating.getParams().keys;
     size_t limit = 0;
 
-    ActionsDAGPtr dag;
+    std::optional<ActionsDAG> dag;
     FixedColumns fixed_columns;
     buildSortingDAG(node, dag, fixed_columns, limit);
 
@@ -1076,13 +1076,13 @@ size_t tryReuseStorageOrderingForWindowFunctions(QueryPlan::Node * parent_node,
     for (const auto & actions_dag : window_desc.partition_by_actions)
     {
         order_by_elements_actions.emplace_back(
-            std::make_shared<ExpressionActions>(std::move(*ActionsDAG::clone(actions_dag.get())), ExpressionActionsSettings::fromContext(context, CompileExpressions::yes)));
+            std::make_shared<ExpressionActions>(actions_dag->clone(), ExpressionActionsSettings::fromContext(context, CompileExpressions::yes)));
     }
 
     for (const auto & actions_dag : window_desc.order_by_actions)
     {
         order_by_elements_actions.emplace_back(
-            std::make_shared<ExpressionActions>(std::move(*ActionsDAG::clone(actions_dag.get())), ExpressionActionsSettings::fromContext(context, CompileExpressions::yes)));
+            std::make_shared<ExpressionActions>(actions_dag->clone(), ExpressionActionsSettings::fromContext(context, CompileExpressions::yes)));
     }
 
     auto order_optimizer = std::make_shared<ReadInOrderOptimizer>(
diff --git a/src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp b/src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp
index 4448d4b7869..ad89cec5f79 100644
--- a/src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp
+++ b/src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp
@@ -43,7 +43,7 @@ static DAGIndex buildDAGIndex(const ActionsDAG & dag)
 /// Required analysis info from aggregate projection.
 struct AggregateProjectionInfo
 {
-    ActionsDAGPtr before_aggregation;
+    std::optional<ActionsDAG> before_aggregation;
     Names keys;
     AggregateDescriptions aggregates;
 
@@ -78,7 +78,7 @@ static AggregateProjectionInfo getAggregatingProjectionInfo(
 
     AggregateProjectionInfo info;
     info.context = interpreter.getContext();
-    info.before_aggregation = ActionsDAG::clone(&analysis_result.before_aggregation->dag);
+    info.before_aggregation = analysis_result.before_aggregation->dag.clone();
     info.keys = query_analyzer->aggregationKeys().getNames();
     info.aggregates = query_analyzer->aggregates();
 
diff --git a/src/Processors/QueryPlan/Optimizations/projectionsCommon.cpp b/src/Processors/QueryPlan/Optimizations/projectionsCommon.cpp
index ad76976becc..571d1dd0cc1 100644
--- a/src/Processors/QueryPlan/Optimizations/projectionsCommon.cpp
+++ b/src/Processors/QueryPlan/Optimizations/projectionsCommon.cpp
@@ -68,9 +68,9 @@ std::shared_ptr<PartitionIdToMaxBlock> getMaxAddedBlocks(ReadFromMergeTree * rea
 void QueryDAG::appendExpression(const ActionsDAG & expression)
 {
     if (dag)
-        dag->mergeInplace(std::move(*ActionsDAG::clone(&expression)));
+        dag->mergeInplace(expression.clone());
     else
-        dag = std::move(*ActionsDAG::clone(&expression));
+        dag = expression.clone();
 }
 
 const ActionsDAG::Node * findInOutputs(ActionsDAG & dag, const std::string & name, bool remove)
@@ -239,7 +239,8 @@ bool analyzeProjectionCandidate(
 
     auto projection_query_info = query_info;
     projection_query_info.prewhere_info = nullptr;
-    projection_query_info.filter_actions_dag = ActionsDAG::clone(dag);
+    if (dag)
+        projection_query_info.filter_actions_dag = std::make_unique<ActionsDAG>(dag->clone());
 
     auto projection_result_ptr = reader.estimateNumMarksToRead(
         std::move(projection_parts),
diff --git a/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp b/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp
index d0acd8221d4..7664822cc7e 100644
--- a/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp
+++ b/src/Processors/QueryPlan/Optimizations/removeRedundantDistinct.cpp
@@ -43,10 +43,10 @@ namespace
         }
     }
 
-    void logActionsDAG(const String & prefix, const ActionsDAGPtr & actions)
+    void logActionsDAG(const String & prefix, const ActionsDAG & actions)
     {
         if constexpr (debug_logging_enabled)
-            LOG_DEBUG(getLogger("redundantDistinct"), "{} :\n{}", prefix, actions->dumpDAG());
+            LOG_DEBUG(getLogger("redundantDistinct"), "{} :\n{}", prefix, actions.dumpDAG());
     }
 
     using DistinctColumns = std::set<std::string_view>;
@@ -65,19 +65,19 @@ namespace
     }
 
     /// build actions DAG from stack of steps
-    ActionsDAGPtr buildActionsForPlanPath(std::vector<const ActionsDAG *> & dag_stack)
+    std::optional<ActionsDAG> buildActionsForPlanPath(std::vector<const ActionsDAG *> & dag_stack)
     {
         if (dag_stack.empty())
-            return nullptr;
+            return {};
 
-        ActionsDAGPtr path_actions = ActionsDAG::clone(dag_stack.back());
+        ActionsDAG path_actions = dag_stack.back()->clone();
         dag_stack.pop_back();
         while (!dag_stack.empty())
         {
-            ActionsDAGPtr clone = ActionsDAG::clone(dag_stack.back());
+            ActionsDAG clone = dag_stack.back()->clone();
             logActionsDAG("DAG to merge", clone);
             dag_stack.pop_back();
-            path_actions->mergeInplace(std::move(*clone));
+            path_actions.mergeInplace(std::move(clone));
         }
         return path_actions;
     }
@@ -260,15 +260,15 @@ namespace
         if (distinct_columns.size() != inner_distinct_columns.size())
             return false;
 
-        ActionsDAGPtr path_actions;
+        ActionsDAG path_actions;
         if (!dag_stack.empty())
         {
             /// build actions DAG to find original column names
-            path_actions = buildActionsForPlanPath(dag_stack);
+            path_actions = std::move(*buildActionsForPlanPath(dag_stack));
             logActionsDAG("distinct pass: merged DAG", path_actions);
 
             /// compare columns of two DISTINCTs
-            FindOriginalNodeForOutputName original_node_finder(*path_actions);
+            FindOriginalNodeForOutputName original_node_finder(path_actions);
             for (const auto & column : distinct_columns)
             {
                 const auto * alias_node = original_node_finder.find(String(column));
diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp
index add53f9d6b3..5dda4ddc18b 100644
--- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp
+++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp
@@ -802,7 +802,7 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreams(RangesInDataParts && parts_
                 info.use_uncompressed_cache);
         };
 
-        auto sorting_expr = std::make_shared<ExpressionActions>(std::move(*ActionsDAG::clone(&metadata_for_reading->getSortingKey().expression->getActionsDAG())));
+        auto sorting_expr = metadata_for_reading->getSortingKey().expression;
 
         SplitPartsWithRangesByPrimaryKeyResult split_ranges_result = splitPartsWithRangesByPrimaryKey(
             metadata_for_reading->getPrimaryKey(),
@@ -1215,7 +1215,7 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal(
     /// we will store lonely parts with level > 0 to use parallel select on them.
     RangesInDataParts non_intersecting_parts_by_primary_key;
 
-    auto sorting_expr = std::make_shared<ExpressionActions>(std::move(*ActionsDAG::clone(&metadata_for_reading->getSortingKey().expression->getActionsDAG())));
+    auto sorting_expr = metadata_for_reading->getSortingKey().expression;
 
     if (prewhere_info)
     {
@@ -1523,7 +1523,7 @@ void ReadFromMergeTree::applyFilters(ActionDAGNodes added_filter_nodes)
         /// TODO: Get rid of filter_actions_dag in query_info after we move analysis of
         /// parallel replicas and unused shards into optimization, similar to projection analysis.
         if (filter_actions_dag)
-            query_info.filter_actions_dag = std::make_shared<const ActionsDAG>(std::move(*ActionsDAG::clone(&*filter_actions_dag)));
+            query_info.filter_actions_dag = std::make_shared<const ActionsDAG>(std::move(*filter_actions_dag));
 
         buildIndexes(
             indexes,
@@ -2004,7 +2004,7 @@ void ReadFromMergeTree::initializePipeline(QueryPipelineBuilder & pipeline, cons
 
     if (result.sampling.use_sampling)
     {
-        auto sampling_actions = std::make_shared<ExpressionActions>(std::move(*ActionsDAG::clone(result.sampling.filter_expression.get())));
+        auto sampling_actions = std::make_shared<ExpressionActions>(result.sampling.filter_expression->clone());
         pipe.addSimpleTransform([&](const Block & header)
         {
             return std::make_shared<FilterTransform>(
@@ -2137,7 +2137,7 @@ void ReadFromMergeTree::describeActions(FormatSettings & format_settings) const
                format_settings.out << " (removed)";
             format_settings.out << '\n';
 
-            auto expression = std::make_shared<ExpressionActions>(std::move(*ActionsDAG::clone(&*prewhere_info->prewhere_actions)));
+            auto expression = std::make_shared<ExpressionActions>(prewhere_info->prewhere_actions->clone());
             expression->describeActions(format_settings.out, prefix);
         }
 
@@ -2146,7 +2146,7 @@ void ReadFromMergeTree::describeActions(FormatSettings & format_settings) const
             format_settings.out << prefix << "Row level filter" << '\n';
             format_settings.out << prefix << "Row level filter column: " << prewhere_info->row_level_column_name << '\n';
 
-            auto expression = std::make_shared<ExpressionActions>(std::move(*ActionsDAG::clone(&*prewhere_info->row_level_filter)));
+            auto expression = std::make_shared<ExpressionActions>(prewhere_info->row_level_filter->clone());
             expression->describeActions(format_settings.out, prefix);
         }
     }
@@ -2172,7 +2172,7 @@ void ReadFromMergeTree::describeActions(JSONBuilder::JSONMap & map) const
             std::unique_ptr<JSONBuilder::JSONMap> prewhere_filter_map = std::make_unique<JSONBuilder::JSONMap>();
             prewhere_filter_map->add("Prewhere filter column", prewhere_info->prewhere_column_name);
             prewhere_filter_map->add("Prewhere filter remove filter column", prewhere_info->remove_prewhere_column);
-            auto expression = std::make_shared<ExpressionActions>(std::move(*ActionsDAG::clone(&*prewhere_info->prewhere_actions)));
+            auto expression = std::make_shared<ExpressionActions>(prewhere_info->prewhere_actions->clone());
             prewhere_filter_map->add("Prewhere filter expression", expression->toTree());
 
             prewhere_info_map->add("Prewhere filter", std::move(prewhere_filter_map));
@@ -2182,7 +2182,7 @@ void ReadFromMergeTree::describeActions(JSONBuilder::JSONMap & map) const
         {
             std::unique_ptr<JSONBuilder::JSONMap> row_level_filter_map = std::make_unique<JSONBuilder::JSONMap>();
             row_level_filter_map->add("Row level filter column", prewhere_info->row_level_column_name);
-            auto expression = std::make_shared<ExpressionActions>(std::move(*ActionsDAG::clone(&*prewhere_info->row_level_filter)));
+            auto expression = std::make_shared<ExpressionActions>(prewhere_info->row_level_filter->clone());
             row_level_filter_map->add("Row level filter expression", expression->toTree());
 
             prewhere_info_map->add("Row level filter", std::move(row_level_filter_map));
diff --git a/src/Processors/QueryPlan/SourceStepWithFilter.cpp b/src/Processors/QueryPlan/SourceStepWithFilter.cpp
index 55c9b5e442e..b91debc8239 100644
--- a/src/Processors/QueryPlan/SourceStepWithFilter.cpp
+++ b/src/Processors/QueryPlan/SourceStepWithFilter.cpp
@@ -110,7 +110,7 @@ void SourceStepWithFilter::describeActions(FormatSettings & format_settings) con
                 format_settings.out << " (removed)";
             format_settings.out << '\n';
 
-            auto expression = std::make_shared<ExpressionActions>(std::move(*ActionsDAG::clone(&*prewhere_info->prewhere_actions)));
+            auto expression = std::make_shared<ExpressionActions>(prewhere_info->prewhere_actions->clone());
             expression->describeActions(format_settings.out, prefix);
         }
 
@@ -119,7 +119,7 @@ void SourceStepWithFilter::describeActions(FormatSettings & format_settings) con
             format_settings.out << prefix << "Row level filter" << '\n';
             format_settings.out << prefix << "Row level filter column: " << prewhere_info->row_level_column_name << '\n';
 
-            auto expression = std::make_shared<ExpressionActions>(std::move(*ActionsDAG::clone(&*prewhere_info->row_level_filter)));
+            auto expression = std::make_shared<ExpressionActions>(prewhere_info->row_level_filter->clone());
             expression->describeActions(format_settings.out, prefix);
         }
     }
@@ -137,7 +137,7 @@ void SourceStepWithFilter::describeActions(JSONBuilder::JSONMap & map) const
             std::unique_ptr<JSONBuilder::JSONMap> prewhere_filter_map = std::make_unique<JSONBuilder::JSONMap>();
             prewhere_filter_map->add("Prewhere filter column", prewhere_info->prewhere_column_name);
             prewhere_filter_map->add("Prewhere filter remove filter column", prewhere_info->remove_prewhere_column);
-            auto expression = std::make_shared<ExpressionActions>(std::move(*ActionsDAG::clone(&*prewhere_info->prewhere_actions)));
+            auto expression = std::make_shared<ExpressionActions>(prewhere_info->prewhere_actions->clone());
             prewhere_filter_map->add("Prewhere filter expression", expression->toTree());
 
             prewhere_info_map->add("Prewhere filter", std::move(prewhere_filter_map));
@@ -147,7 +147,7 @@ void SourceStepWithFilter::describeActions(JSONBuilder::JSONMap & map) const
         {
             std::unique_ptr<JSONBuilder::JSONMap> row_level_filter_map = std::make_unique<JSONBuilder::JSONMap>();
             row_level_filter_map->add("Row level filter column", prewhere_info->row_level_column_name);
-            auto expression = std::make_shared<ExpressionActions>(std::move(*ActionsDAG::clone(&*prewhere_info->row_level_filter)));
+            auto expression = std::make_shared<ExpressionActions>(prewhere_info->row_level_filter->clone());
             row_level_filter_map->add("Row level filter expression", expression->toTree());
 
             prewhere_info_map->add("Row level filter", std::move(row_level_filter_map));
diff --git a/src/Processors/QueryPlan/TotalsHavingStep.cpp b/src/Processors/QueryPlan/TotalsHavingStep.cpp
index 70457918de1..2554053064f 100644
--- a/src/Processors/QueryPlan/TotalsHavingStep.cpp
+++ b/src/Processors/QueryPlan/TotalsHavingStep.cpp
@@ -101,13 +101,16 @@ void TotalsHavingStep::describeActions(FormatSettings & settings) const
     if (actions_dag)
     {
         bool first = true;
-        auto expression = std::make_shared<ExpressionActions>(std::move(*ActionsDAG::clone(getActions())));
-        for (const auto & action : expression->getActions())
+        if (actions_dag)
         {
-            settings.out << prefix << (first ? "Actions: "
-                                             : "         ");
-            first = false;
-            settings.out << action.toString() << '\n';
+            auto expression = std::make_shared<ExpressionActions>(actions_dag->clone());
+            for (const auto & action : expression->getActions())
+            {
+                settings.out << prefix << (first ? "Actions: "
+                                                : "         ");
+                first = false;
+                settings.out << action.toString() << '\n';
+            }
         }
     }
 }
@@ -118,8 +121,11 @@ void TotalsHavingStep::describeActions(JSONBuilder::JSONMap & map) const
     if (actions_dag)
     {
         map.add("Filter column", filter_column_name);
-        auto expression = std::make_shared<ExpressionActions>(std::move(*ActionsDAG::clone(getActions())));
-        map.add("Expression", expression->toTree());
+        if (actions_dag)
+        {
+            auto expression = std::make_shared<ExpressionActions>(actions_dag->clone());
+            map.add("Expression", expression->toTree());
+        }
     }
 }
 
diff --git a/src/Processors/Transforms/FillingTransform.cpp b/src/Processors/Transforms/FillingTransform.cpp
index 36ffc515f43..9601f821cc8 100644
--- a/src/Processors/Transforms/FillingTransform.cpp
+++ b/src/Processors/Transforms/FillingTransform.cpp
@@ -203,7 +203,7 @@ FillingTransform::FillingTransform(
     , use_with_fill_by_sorting_prefix(use_with_fill_by_sorting_prefix_)
 {
     if (interpolate_description)
-        interpolate_actions = std::make_shared<ExpressionActions>(std::move(*ActionsDAG::clone(&interpolate_description->actions)));
+        interpolate_actions = std::make_shared<ExpressionActions>(interpolate_description->actions.clone());
 
     std::vector<bool> is_fill_column(header_.columns());
     for (size_t i = 0, size = fill_description.size(); i < size; ++i)
diff --git a/src/Storages/MergeTree/MergeTreeIndexSet.cpp b/src/Storages/MergeTree/MergeTreeIndexSet.cpp
index ca31ffc9de5..c0875ed184d 100644
--- a/src/Storages/MergeTree/MergeTreeIndexSet.cpp
+++ b/src/Storages/MergeTree/MergeTreeIndexSet.cpp
@@ -265,7 +265,7 @@ MergeTreeIndexConditionSet::MergeTreeIndexConditionSet(
         if (!set->buildOrderedSetInplace(context))
             return;
 
-    auto filter_actions_dag = std::move(*ActionsDAG::clone(filter_dag));
+    auto filter_actions_dag = filter_dag->clone();
     const auto * filter_actions_dag_node = filter_actions_dag.getOutputs().at(0);
 
     std::unordered_map<const ActionsDAG::Node *, const ActionsDAG::Node *> node_to_result_node;
@@ -319,7 +319,7 @@ static const ActionsDAG::NodeRawConstPtrs & getArguments(const ActionsDAG::Node
         return index_hint.getActions().getOutputs();
 
     /// Import the DAG and map argument pointers.
-    auto actions_clone = std::move(*ActionsDAG::clone(&index_hint.getActions()));
+    auto actions_clone = index_hint.getActions().clone();
     chassert(storage);
     result_dag_or_null->mergeNodes(std::move(actions_clone), storage);
     return *storage;
diff --git a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp
index 22289187cfa..f1df9e231c4 100644
--- a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp
+++ b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp
@@ -80,7 +80,7 @@ PrewhereExprInfo MergeTreeSelectProcessor::getPrewhereActions(PrewhereInfoPtr pr
             PrewhereExprStep row_level_filter_step
             {
                 .type = PrewhereExprStep::Filter,
-                .actions = std::make_shared<ExpressionActions>(std::move(*ActionsDAG::clone(&*prewhere_info->row_level_filter)), actions_settings),
+                .actions = std::make_shared<ExpressionActions>(prewhere_info->row_level_filter->clone(), actions_settings),
                 .filter_column_name = prewhere_info->row_level_column_name,
                 .remove_filter_column = true,
                 .need_filter = true,
@@ -96,7 +96,7 @@ PrewhereExprInfo MergeTreeSelectProcessor::getPrewhereActions(PrewhereInfoPtr pr
             PrewhereExprStep prewhere_step
             {
                 .type = PrewhereExprStep::Filter,
-                .actions = std::make_shared<ExpressionActions>(std::move(*ActionsDAG::clone(&*prewhere_info->prewhere_actions)), actions_settings),
+                .actions = std::make_shared<ExpressionActions>(prewhere_info->prewhere_actions->clone(), actions_settings),
                 .filter_column_name = prewhere_info->prewhere_column_name,
                 .remove_filter_column = prewhere_info->remove_prewhere_column,
                 .need_filter = prewhere_info->need_filter,
diff --git a/src/Storages/SelectQueryInfo.h b/src/Storages/SelectQueryInfo.h
index 589698fcc30..60f103fdb70 100644
--- a/src/Storages/SelectQueryInfo.h
+++ b/src/Storages/SelectQueryInfo.h
@@ -66,10 +66,10 @@ struct PrewhereInfo
         PrewhereInfoPtr prewhere_info = std::make_shared<PrewhereInfo>();
 
         if (row_level_filter)
-            prewhere_info->row_level_filter = std::move(*ActionsDAG::clone(&*row_level_filter));
+            prewhere_info->row_level_filter = row_level_filter->clone();
 
         if (prewhere_actions)
-            prewhere_info->prewhere_actions = std::move(*ActionsDAG::clone(&*prewhere_actions));
+            prewhere_info->prewhere_actions = prewhere_actions->clone();
 
         prewhere_info->row_level_column_name = row_level_column_name;
         prewhere_info->prewhere_column_name = prewhere_column_name;
diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp
index c096504170e..aee4e4683ad 100644
--- a/src/Storages/StorageBuffer.cpp
+++ b/src/Storages/StorageBuffer.cpp
@@ -313,7 +313,7 @@ void StorageBuffer::read(
                     if (src_table_query_info.prewhere_info->row_level_filter)
                     {
                         src_table_query_info.prewhere_info->row_level_filter = ActionsDAG::merge(
-                            std::move(*ActionsDAG::clone(&actions_dag)),
+                            actions_dag.clone(),
                             std::move(*src_table_query_info.prewhere_info->row_level_filter));
 
                         src_table_query_info.prewhere_info->row_level_filter->removeUnusedActions();
@@ -322,7 +322,7 @@ void StorageBuffer::read(
                     if (src_table_query_info.prewhere_info->prewhere_actions)
                     {
                         src_table_query_info.prewhere_info->prewhere_actions = ActionsDAG::merge(
-                            std::move(*ActionsDAG::clone(&actions_dag)),
+                            actions_dag.clone(),
                             std::move(*src_table_query_info.prewhere_info->prewhere_actions));
 
                         src_table_query_info.prewhere_info->prewhere_actions->removeUnusedActions();
@@ -429,21 +429,23 @@ void StorageBuffer::read(
 
             if (query_info.prewhere_info->row_level_filter)
             {
+                auto actions = std::make_shared<ExpressionActions>(query_info.prewhere_info->row_level_filter->clone(), actions_settings);
                 pipe_from_buffers.addSimpleTransform([&](const Block & header)
                 {
                     return std::make_shared<FilterTransform>(
                             header,
-                            std::make_shared<ExpressionActions>(std::move(*ActionsDAG::clone(&*query_info.prewhere_info->row_level_filter)), actions_settings),
+                            actions,
                             query_info.prewhere_info->row_level_column_name,
                             false);
                 });
             }
 
+            auto actions = std::make_shared<ExpressionActions>(query_info.prewhere_info->prewhere_actions->clone(), actions_settings);
             pipe_from_buffers.addSimpleTransform([&](const Block & header)
             {
                 return std::make_shared<FilterTransform>(
                         header,
-                        std::make_shared<ExpressionActions>(std::move(*ActionsDAG::clone(&*query_info.prewhere_info->prewhere_actions)), actions_settings),
+                        actions,
                         query_info.prewhere_info->prewhere_column_name,
                         query_info.prewhere_info->remove_prewhere_column);
             });
diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp
index e16e2a07685..0e1568c8e79 100644
--- a/src/Storages/StorageMerge.cpp
+++ b/src/Storages/StorageMerge.cpp
@@ -663,7 +663,7 @@ std::vector<ReadFromMerge::ChildPlan> ReadFromMerge::createChildrenPlans(SelectQ
             {
                 auto filter_step = std::make_unique<FilterStep>(
                     child.plan.getCurrentDataStream(),
-                    std::move(*ActionsDAG::clone(&filter_info.actions)),
+                    filter_info.actions.clone(),
                     filter_info.column_name,
                     filter_info.do_remove_column);
 
@@ -1241,7 +1241,7 @@ ReadFromMerge::RowPolicyData::RowPolicyData(RowPolicyFilterPtr row_policy_filter
     auto expression_analyzer = ExpressionAnalyzer{expr, syntax_result, local_context};
 
     actions_dag = expression_analyzer.getActionsDAG(false /* add_aliases */, false /* project_result */);
-    filter_actions = std::make_shared<ExpressionActions>(std::move(*ActionsDAG::clone(&actions_dag)),
+    filter_actions = std::make_shared<ExpressionActions>(actions_dag.clone(),
         ExpressionActionsSettings::fromContext(local_context, CompileExpressions::yes));
     const auto & required_columns = filter_actions->getRequiredColumnsWithTypes();
     const auto & sample_block_columns = filter_actions->getSampleBlock().getNamesAndTypesList();
@@ -1279,12 +1279,12 @@ void ReadFromMerge::RowPolicyData::extendNames(Names & names) const
 
 void ReadFromMerge::RowPolicyData::addStorageFilter(SourceStepWithFilter * step) const
 {
-    step->addFilter(ActionsDAG::clone(&actions_dag), filter_column_name);
+    step->addFilter(std::make_unique<ActionsDAG>(actions_dag.clone()), filter_column_name);
 }
 
 void ReadFromMerge::RowPolicyData::addFilterTransform(QueryPlan & plan) const
 {
-    auto filter_step = std::make_unique<FilterStep>(plan.getCurrentDataStream(), std::move(*ActionsDAG::clone(&actions_dag)), filter_column_name, true /* remove filter column */);
+    auto filter_step = std::make_unique<FilterStep>(plan.getCurrentDataStream(), actions_dag.clone(), filter_column_name, true /* remove filter column */);
     plan.addStep(std::move(filter_step));
 }
 
diff --git a/src/Storages/VirtualColumnUtils.cpp b/src/Storages/VirtualColumnUtils.cpp
index 32c6a558340..146a54eda78 100644
--- a/src/Storages/VirtualColumnUtils.cpp
+++ b/src/Storages/VirtualColumnUtils.cpp
@@ -323,7 +323,7 @@ static const ActionsDAG::Node * splitFilterNodeForAllowedInputs(
             {
                 if (const auto * index_hint = typeid_cast<const FunctionIndexHint *>(adaptor->getFunction().get()))
                 {
-                    auto index_hint_dag = std::move(*ActionsDAG::clone(&index_hint->getActions()));
+                    auto index_hint_dag = index_hint->getActions().clone();
                     ActionsDAG::NodeRawConstPtrs atoms;
                     for (const auto & output : index_hint_dag.getOutputs())
                         if (const auto * child_copy = splitFilterNodeForAllowedInputs(output, allowed_inputs, additional_nodes))

From 81fb9b18f623bc85b1044242ab908d50731ae624 Mon Sep 17 00:00:00 2001
From: Alexander Tokmakov <tavplubix@clickhouse.com>
Date: Thu, 18 Jul 2024 17:41:36 +0200
Subject: [PATCH 0401/1488] Update 02125_many_mutations.sh

---
 tests/queries/0_stateless/02125_many_mutations.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/queries/0_stateless/02125_many_mutations.sh b/tests/queries/0_stateless/02125_many_mutations.sh
index 474fc8ef199..239f30811d4 100755
--- a/tests/queries/0_stateless/02125_many_mutations.sh
+++ b/tests/queries/0_stateless/02125_many_mutations.sh
@@ -48,6 +48,7 @@ $CLICKHOUSE_CLIENT --multiquery -q "
 select count() from system.mutations where database = currentDatabase() and table = 'many_mutations' and not is_done;
 system start merges many_mutations;
 optimize table many_mutations final SETTINGS optimize_throw_if_noop = 1;
+alter table many_mutations update y = y + 1 where 1 settings mutations_sync=2;
 select count() from system.mutations where database = currentDatabase() and table = 'many_mutations' and not is_done;
 select x, y from many_mutations order by x;
 truncate table many_mutations;

From f71e91b19f579580346dc430e9603c6dfa8102e5 Mon Sep 17 00:00:00 2001
From: Alexander Tokmakov <tavplubix@clickhouse.com>
Date: Thu, 18 Jul 2024 17:42:25 +0200
Subject: [PATCH 0402/1488] Update 02125_many_mutations.reference

---
 tests/queries/0_stateless/02125_many_mutations.reference | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/queries/0_stateless/02125_many_mutations.reference b/tests/queries/0_stateless/02125_many_mutations.reference
index c98d8221c7f..a94b8b78480 100644
--- a/tests/queries/0_stateless/02125_many_mutations.reference
+++ b/tests/queries/0_stateless/02125_many_mutations.reference
@@ -2,5 +2,5 @@
 1	1
 20000
 0
-0	20000
-1	20001
+0	20001
+1	20002

From 36275fdacc34206931f69087fe77539e25bbbedd Mon Sep 17 00:00:00 2001
From: Alexander Tokmakov <tavplubix@clickhouse.com>
Date: Thu, 18 Jul 2024 17:45:52 +0200
Subject: [PATCH 0403/1488] Update 02125_many_mutations_2.sh

---
 tests/queries/0_stateless/02125_many_mutations_2.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/queries/0_stateless/02125_many_mutations_2.sh b/tests/queries/0_stateless/02125_many_mutations_2.sh
index b7585991738..958f50287b2 100755
--- a/tests/queries/0_stateless/02125_many_mutations_2.sh
+++ b/tests/queries/0_stateless/02125_many_mutations_2.sh
@@ -52,6 +52,7 @@ $CLICKHOUSE_CLIENT --multiquery -q "
 select count() from system.mutations where database = currentDatabase() and table = 'many_mutations' and not is_done;
 system start merges many_mutations;
 optimize table many_mutations final SETTINGS optimize_throw_if_noop = 1;
+alter table many_mutations update y = y + 1 where 1 settings mutations_sync=2;
 system flush logs;
 select count() from system.mutations where database = currentDatabase() and table = 'many_mutations' and not is_done;
 select count() from many_mutations;

From 093b13329c4206b4ecd604fb373d8d60cf345a1f Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Thu, 18 Jul 2024 15:46:05 +0000
Subject: [PATCH 0404/1488] One more check in JOIN ON ... IS NULL

---
 .../Passes/LogicalExpressionOptimizerPass.cpp | 46 +++++++++++++++++--
 ...11_join_on_nullsafe_optimization.reference |  4 ++
 .../02911_join_on_nullsafe_optimization.sql   |  1 +
 3 files changed, 47 insertions(+), 4 deletions(-)

diff --git a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp
index 854697bca9f..7e54b5a4b42 100644
--- a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp
+++ b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp
@@ -67,6 +67,41 @@ QueryTreeNodePtr findEqualsFunction(const QueryTreeNodes & nodes)
     return nullptr;
 }
 
+/// Checks if the node is combination of isNull and notEquals functions of two the same arguments
+bool matchIsNullOfTwoArgs(const QueryTreeNodes & nodes, QueryTreeNodePtr & lhs, QueryTreeNodePtr & rhs)
+{
+    QueryTreeNodePtrWithHashSet all_arguments;
+    for (const auto & node : nodes)
+    {
+        const auto * func_node = node->as<FunctionNode>();
+        if (!func_node)
+            return false;
+
+        const auto & arguments = func_node->getArguments().getNodes();
+        if (func_node->getFunctionName() == "isNull" && arguments.size() == 1)
+            all_arguments.insert(QueryTreeNodePtrWithHash(arguments[0]));
+        else if (func_node->getFunctionName() == "notEquals" && arguments.size() == 2)
+        {
+            if (arguments[0]->isEqual(*arguments[1]))
+                return false;
+            all_arguments.insert(QueryTreeNodePtrWithHash(arguments[0]));
+            all_arguments.insert(QueryTreeNodePtrWithHash(arguments[1]));
+        }
+        else
+            return false;
+
+        if (all_arguments.size() > 2)
+            return false;
+    }
+
+    if (all_arguments.size() != 2)
+        return false;
+
+    lhs = all_arguments.begin()->node;
+    rhs = std::next(all_arguments.begin())->node;
+    return true;
+}
+
 bool isBooleanConstant(const QueryTreeNodePtr & node, bool expected_value)
 {
     const auto * constant_node = node->as<ConstantNode>();
@@ -212,11 +247,14 @@ private:
             else if (func_name == "and")
             {
                 const auto & and_arguments = argument_function->getArguments().getNodes();
-                bool all_are_is_null = and_arguments.size() == 2 && isNodeFunction(and_arguments[0], "isNull") && isNodeFunction(and_arguments[1], "isNull");
-                if (all_are_is_null)
+
+                QueryTreeNodePtr is_null_lhs_arg;
+                QueryTreeNodePtr is_null_rhs_arg;
+                if (matchIsNullOfTwoArgs(and_arguments, is_null_lhs_arg, is_null_rhs_arg))
                 {
-                    is_null_argument_to_indices[getFunctionArgument(and_arguments.front(), 0)].push_back(or_operands.size() - 1);
-                    is_null_argument_to_indices[getFunctionArgument(and_arguments.back(), 0)].push_back(or_operands.size() - 1);
+                    is_null_argument_to_indices[is_null_lhs_arg].push_back(or_operands.size() - 1);
+                    is_null_argument_to_indices[is_null_rhs_arg].push_back(or_operands.size() - 1);
+                    continue;
                 }
 
                 /// Expression `a = b AND (a IS NOT NULL) AND true AND (b IS NOT NULL)` we can be replaced with `a = b`
diff --git a/tests/queries/0_stateless/02911_join_on_nullsafe_optimization.reference b/tests/queries/0_stateless/02911_join_on_nullsafe_optimization.reference
index 4eb7e74446d..31a1cda18e7 100644
--- a/tests/queries/0_stateless/02911_join_on_nullsafe_optimization.reference
+++ b/tests/queries/0_stateless/02911_join_on_nullsafe_optimization.reference
@@ -35,6 +35,10 @@ SELECT * FROM t1 JOIN t2 ON (t1.x == t2.x AND ((t2.x IS NOT NULL) AND (t1.x IS N
 2	2	2	2
 3	3	3	33
 \N	\N	\N	\N
+SELECT * FROM t1 JOIN t2 ON (t1.x == t2.x AND ((t2.x IS NOT NULL) AND (t1.x IS NOT NULL)) ) OR ( t2.x <> t1.x AND (t2.x IS NULL) AND (t1.x IS NULL) ) ORDER BY t1.x NULLS LAST;
+2	2	2	2
+3	3	3	33
+\N	\N	\N	\N
 -- aliases defined in the join condition are valid
 -- FIXME(@vdimir) broken query formatting for the following queries:
 -- SELECT *, e, e2 FROM t1 FULL JOIN t2 ON ( ( ((t1.x == t2.x) AS e) AND ((t2.x IS NOT NULL) AND (t1.x IS NOT NULL)) ) OR ( (t2.x IS NULL) AND (t1.x IS NULL) ) AS e2 ) ORDER BY t1.x NULLS LAST, t2.x NULLS LAST;
diff --git a/tests/queries/0_stateless/02911_join_on_nullsafe_optimization.sql b/tests/queries/0_stateless/02911_join_on_nullsafe_optimization.sql
index f7813e2a1b4..f739259caf9 100644
--- a/tests/queries/0_stateless/02911_join_on_nullsafe_optimization.sql
+++ b/tests/queries/0_stateless/02911_join_on_nullsafe_optimization.sql
@@ -34,6 +34,7 @@ SELECT * FROM t1 JOIN t2 ON (t1.x <=> t2.x OR (t1.x IS NULL AND t2.x IS NULL)) A
 SELECT x = y OR (x IS NULL AND y IS NULL) FROM t1 ORDER BY x NULLS LAST;
 
 SELECT * FROM t1 JOIN t2 ON (t1.x == t2.x AND ((t2.x IS NOT NULL) AND (t1.x IS NOT NULL)) ) OR ( (t2.x IS NULL) AND (t1.x IS NULL) ) ORDER BY t1.x NULLS LAST;
+SELECT * FROM t1 JOIN t2 ON (t1.x == t2.x AND ((t2.x IS NOT NULL) AND (t1.x IS NOT NULL)) ) OR ( t2.x <> t1.x AND (t2.x IS NULL) AND (t1.x IS NULL) ) ORDER BY t1.x NULLS LAST;
 
 -- aliases defined in the join condition are valid
 -- FIXME(@vdimir) broken query formatting for the following queries:

From 7fc8ee726e3ef2dfb7d778fbb1a70fb147a33067 Mon Sep 17 00:00:00 2001
From: Michael Stetsyuk <mike.stetsyuk@gmail.com>
Date: Thu, 18 Jul 2024 12:32:16 +0200
Subject: [PATCH 0405/1488] add replication lag and recovery time metrics

---
 src/Databases/DatabaseReplicated.cpp          | 44 +++++++++----------
 src/Databases/DatabaseReplicated.h            | 10 ++++-
 src/Databases/DatabaseReplicatedWorker.cpp    |  4 ++
 src/Databases/DatabaseReplicatedWorker.h      |  4 ++
 src/Storages/System/StorageSystemClusters.cpp | 33 +++++++++-----
 src/Storages/System/StorageSystemClusters.h   |  2 +-
 .../test_recovery_time_metric/__init__.py     |  0
 .../configs/config.xml                        | 41 +++++++++++++++++
 .../test_recovery_time_metric/test.py         | 26 +++++++++++
 9 files changed, 129 insertions(+), 35 deletions(-)
 create mode 100644 tests/integration/test_recovery_time_metric/__init__.py
 create mode 100644 tests/integration/test_recovery_time_metric/configs/config.xml
 create mode 100644 tests/integration/test_recovery_time_metric/test.py

diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp
index 7ce2859e962..b11b9382732 100644
--- a/src/Databases/DatabaseReplicated.cpp
+++ b/src/Databases/DatabaseReplicated.cpp
@@ -338,42 +338,40 @@ ClusterPtr DatabaseReplicated::getClusterImpl(bool all_groups) const
     return std::make_shared<Cluster>(getContext()->getSettingsRef(), shards, params);
 }
 
-std::vector<UInt8> DatabaseReplicated::tryGetAreReplicasActive(const ClusterPtr & cluster_) const
+ReplicasInfo DatabaseReplicated::tryGetReplicasInfo(const ClusterPtr & cluster_) const
 {
-    Strings paths;
+    ReplicasInfo res;
+
+    auto zookeeper = getZooKeeper();
     const auto & addresses_with_failover = cluster_->getShardsAddresses();
     const auto & shards_info = cluster_->getShardsInfo();
-    for (size_t shard_index = 0; shard_index < shards_info.size(); ++shard_index)
-    {
-        for (const auto & replica : addresses_with_failover[shard_index])
-        {
-            String full_name = getFullReplicaName(replica.database_shard_name, replica.database_replica_name);
-            paths.emplace_back(fs::path(zookeeper_path) / "replicas" / full_name / "active");
-        }
-    }
 
     try
     {
-        auto current_zookeeper = getZooKeeper();
-        auto res = current_zookeeper->exists(paths);
+        UInt32 max_log_ptr = parse<UInt32>(zookeeper->get(zookeeper_path + "/max_log_ptr"));
 
-        std::vector<UInt8> statuses;
-        statuses.resize(paths.size());
-
-        for (size_t i = 0; i < res.size(); ++i)
-            if (res[i].error == Coordination::Error::ZOK)
-                statuses[i] = 1;
-
-        return statuses;
-    }
-    catch (...)
+        for (size_t shard_index = 0; shard_index < shards_info.size(); ++shard_index)
+        {
+            for (const auto & replica : addresses_with_failover[shard_index])
+            {
+                String full_name = getFullReplicaName(replica.database_shard_name, replica.database_replica_name);
+                UInt32 log_ptr = parse<UInt32>(zookeeper->get(fs::path(zookeeper_path) / "replicas" / full_name / "log_ptr"));
+                bool is_active = zookeeper->exists(fs::path(zookeeper_path) / "replicas" / full_name / "active");
+                res.push_back(ReplicaInfo{
+                    .is_active = is_active,
+                    .replication_lag = max_log_ptr - log_ptr,
+                    .recovery_time = replica.is_local ? ddl_worker->getCurrentInitializationDurationMs() : 0,
+                });
+            }
+        }
+        return res;
+    } catch (...)
     {
         tryLogCurrentException(log);
         return {};
     }
 }
 
-
 void DatabaseReplicated::fillClusterAuthInfo(String collection_name, const Poco::Util::AbstractConfiguration & config_ref)
 {
     const auto & config_prefix = fmt::format("named_collections.{}", collection_name);
diff --git a/src/Databases/DatabaseReplicated.h b/src/Databases/DatabaseReplicated.h
index eab5b2ff931..db02b5ef30f 100644
--- a/src/Databases/DatabaseReplicated.h
+++ b/src/Databases/DatabaseReplicated.h
@@ -17,6 +17,14 @@ using ZooKeeperPtr = std::shared_ptr<zkutil::ZooKeeper>;
 class Cluster;
 using ClusterPtr = std::shared_ptr<Cluster>;
 
+struct ReplicaInfo
+{
+    bool is_active;
+    UInt32 replication_lag;
+    UInt64 recovery_time;
+};
+using ReplicasInfo = std::vector<ReplicaInfo>;
+
 class DatabaseReplicated : public DatabaseAtomic
 {
 public:
@@ -84,7 +92,7 @@ public:
 
     static void dropReplica(DatabaseReplicated * database, const String & database_zookeeper_path, const String & shard, const String & replica, bool throw_if_noop);
 
-    std::vector<UInt8> tryGetAreReplicasActive(const ClusterPtr & cluster_) const;
+    ReplicasInfo tryGetReplicasInfo(const ClusterPtr & cluster_) const;
 
     void renameDatabase(ContextPtr query_context, const String & new_name) override;
 
diff --git a/src/Databases/DatabaseReplicatedWorker.cpp b/src/Databases/DatabaseReplicatedWorker.cpp
index 1ef88dc03bc..cea2d123f87 100644
--- a/src/Databases/DatabaseReplicatedWorker.cpp
+++ b/src/Databases/DatabaseReplicatedWorker.cpp
@@ -32,6 +32,8 @@ DatabaseReplicatedDDLWorker::DatabaseReplicatedDDLWorker(DatabaseReplicated * db
 
 bool DatabaseReplicatedDDLWorker::initializeMainThread()
 {
+    initialization_duration_timer.emplace();
+
     while (!stop_flag)
     {
         try
@@ -69,6 +71,7 @@ bool DatabaseReplicatedDDLWorker::initializeMainThread()
 
             initializeReplication();
             initialized = true;
+            initialization_duration_timer.reset();
             return true;
         }
         catch (...)
@@ -78,6 +81,7 @@ bool DatabaseReplicatedDDLWorker::initializeMainThread()
         }
     }
 
+    initialization_duration_timer.reset();
     return false;
 }
 
diff --git a/src/Databases/DatabaseReplicatedWorker.h b/src/Databases/DatabaseReplicatedWorker.h
index 41edf2221b8..aea3b71173d 100644
--- a/src/Databases/DatabaseReplicatedWorker.h
+++ b/src/Databases/DatabaseReplicatedWorker.h
@@ -36,6 +36,8 @@ public:
                                    DatabaseReplicated * const database, bool committed = false); /// NOLINT
 
     UInt32 getLogPointer() const;
+
+    UInt64 getCurrentInitializationDurationMs() const { return initialization_duration_timer ? initialization_duration_timer->elapsedMilliseconds() : 0; }
 private:
     bool initializeMainThread() override;
     void initializeReplication();
@@ -56,6 +58,8 @@ private:
     ZooKeeperPtr active_node_holder_zookeeper;
     /// It will remove "active" node when database is detached
     zkutil::EphemeralNodeHolderPtr active_node_holder;
+
+    std::optional<Stopwatch> initialization_duration_timer;
 };
 
 }
diff --git a/src/Storages/System/StorageSystemClusters.cpp b/src/Storages/System/StorageSystemClusters.cpp
index 160c8d6270e..0da4bd70cbd 100644
--- a/src/Storages/System/StorageSystemClusters.cpp
+++ b/src/Storages/System/StorageSystemClusters.cpp
@@ -31,6 +31,8 @@ ColumnsDescription StorageSystemClusters::getColumnsDescription()
         {"database_shard_name", std::make_shared<DataTypeString>(), "The name of the `Replicated` database shard (for clusters that belong to a `Replicated` database)."},
         {"database_replica_name", std::make_shared<DataTypeString>(), "The name of the `Replicated` database replica (for clusters that belong to a `Replicated` database)."},
         {"is_active", std::make_shared<DataTypeNullable>(std::make_shared<DataTypeUInt8>()), "The status of the Replicated database replica (for clusters that belong to a Replicated database): 1 means 'replica is online', 0 means 'replica is offline', NULL means 'unknown'."},
+        {"replication_lag", std::make_shared<DataTypeNullable>(std::make_shared<DataTypeUInt32>()), "The replication lag of the `Replicated` database replica (for clusters that belong to a Replicated database)."},
+        {"recovery_time", std::make_shared<DataTypeNullable>(std::make_shared<DataTypeUInt64>()), "The recovery time of the `Replicated` database replica (for clusters that belong to a Replicated database), in milliseconds."},
     };
 
     description.setAliases({
@@ -46,24 +48,23 @@ void StorageSystemClusters::fillData(MutableColumns & res_columns, ContextPtr co
         writeCluster(res_columns, name_and_cluster, {});
 
     const auto databases = DatabaseCatalog::instance().getDatabases();
-    for (const auto & name_and_database : databases)
+    for (const auto & [database_name, database] : databases)
     {
-        if (const auto * replicated = typeid_cast<const DatabaseReplicated *>(name_and_database.second.get()))
+        if (const auto * replicated = typeid_cast<const DatabaseReplicated *>(database.get()))
         {
-
             if (auto database_cluster = replicated->tryGetCluster())
-                writeCluster(res_columns, {name_and_database.first, database_cluster},
-                             replicated->tryGetAreReplicasActive(database_cluster));
+                writeCluster(res_columns, {database_name, database_cluster},
+                             replicated->tryGetReplicasInfo(database_cluster));
 
             if (auto database_cluster = replicated->tryGetAllGroupsCluster())
-                writeCluster(res_columns, {DatabaseReplicated::ALL_GROUPS_CLUSTER_PREFIX + name_and_database.first, database_cluster},
-                             replicated->tryGetAreReplicasActive(database_cluster));
+                writeCluster(res_columns, {DatabaseReplicated::ALL_GROUPS_CLUSTER_PREFIX + database_name, database_cluster},
+                             replicated->tryGetReplicasInfo(database_cluster));
         }
     }
 }
 
 void StorageSystemClusters::writeCluster(MutableColumns & res_columns, const NameAndCluster & name_and_cluster,
-                                         const std::vector<UInt8> & is_active)
+                                         const ReplicasInfo & replicas_info)
 {
     const String & cluster_name = name_and_cluster.first;
     const ClusterPtr & cluster = name_and_cluster.second;
@@ -99,10 +100,22 @@ void StorageSystemClusters::writeCluster(MutableColumns & res_columns, const Nam
             res_columns[i++]->insert(pool_status[replica_index].estimated_recovery_time.count());
             res_columns[i++]->insert(address.database_shard_name);
             res_columns[i++]->insert(address.database_replica_name);
-            if (is_active.empty())
+            if (replicas_info.empty())
+            {
                 res_columns[i++]->insertDefault();
+                res_columns[i++]->insertDefault();
+                res_columns[i++]->insertDefault();
+            }
             else
-                res_columns[i++]->insert(is_active[replica_idx++]);
+            {
+                const auto & replica_info = replicas_info[replica_idx++];
+                res_columns[i++]->insert(replica_info.is_active);
+                res_columns[i++]->insert(replica_info.replication_lag);
+                if (replica_info.recovery_time != 0)
+                    res_columns[i++]->insert(replica_info.recovery_time);
+                else
+                    res_columns[i++]->insertDefault();
+            }
         }
     }
 }
diff --git a/src/Storages/System/StorageSystemClusters.h b/src/Storages/System/StorageSystemClusters.h
index 0f7c792261d..ead123aa79e 100644
--- a/src/Storages/System/StorageSystemClusters.h
+++ b/src/Storages/System/StorageSystemClusters.h
@@ -27,7 +27,7 @@ protected:
     using NameAndCluster = std::pair<String, std::shared_ptr<Cluster>>;
 
     void fillData(MutableColumns & res_columns, ContextPtr context, const ActionsDAG::Node *, std::vector<UInt8>) const override;
-    static void writeCluster(MutableColumns & res_columns, const NameAndCluster & name_and_cluster, const std::vector<UInt8> & is_active);
+    static void writeCluster(MutableColumns & res_columns, const NameAndCluster & name_and_cluster, const ReplicasInfo & replicas_info);
 };
 
 }
diff --git a/tests/integration/test_recovery_time_metric/__init__.py b/tests/integration/test_recovery_time_metric/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/integration/test_recovery_time_metric/configs/config.xml b/tests/integration/test_recovery_time_metric/configs/config.xml
new file mode 100644
index 00000000000..bad9b1fa9ea
--- /dev/null
+++ b/tests/integration/test_recovery_time_metric/configs/config.xml
@@ -0,0 +1,41 @@
+<clickhouse>
+    <tcp_port>9000</tcp_port>
+
+    <profiles>
+        <default>
+        </default>
+    </profiles>
+
+    <users>
+        <default>
+            <profile>default</profile>
+            <no_password></no_password>
+        </default>
+    </users>
+
+    <keeper_server>
+        <tcp_port>2181</tcp_port>
+        <server_id>1</server_id>
+        <log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
+        <snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
+        <coordination_settings>
+            <session_timeout_ms>20000</session_timeout_ms>
+        </coordination_settings>
+        <raft_configuration>
+            <server>
+                <id>1</id>
+                <hostname>localhost</hostname>
+                <port>9444</port>
+            </server>
+        </raft_configuration>
+    </keeper_server>
+
+    <zookeeper>
+        <node index="1">
+            <host>localhost</host>
+            <port>2181</port>
+        </node>
+        <session_timeout_ms>20000</session_timeout_ms>
+    </zookeeper>
+
+</clickhouse>
diff --git a/tests/integration/test_recovery_time_metric/test.py b/tests/integration/test_recovery_time_metric/test.py
new file mode 100644
index 00000000000..9ceb0cce288
--- /dev/null
+++ b/tests/integration/test_recovery_time_metric/test.py
@@ -0,0 +1,26 @@
+import pytest
+from helpers.cluster import ClickHouseCluster
+
+cluster = ClickHouseCluster(__file__)
+node = cluster.add_instance("node", main_configs=["configs/config.xml"], with_zookeeper=True)
+
+
+@pytest.fixture(scope="module")
+def start_cluster():
+    try:
+        cluster.start()
+        yield cluster
+    finally:
+        cluster.shutdown()
+
+
+def test_recovery_time_metric(start_cluster):
+    node.query("CREATE DATABASE rdb ENGINE = Replicated('/test/test_recovery_time_metric', 'shard1', 'replica1');")
+    node.query("CREATE TABLE rdb.t (x UInt32) ENGINE = MergeTree ORDER BY x;")
+    node.exec_in_container(["bash", "-c", "rm /var/lib/clickhouse/metadata/rdb/t.sql"])
+    node.restart_clickhouse()
+    assert (
+        node.query("SELECT any(recovery_time) FROM system.clusters;")
+        != "0\n"
+    )
+

From 392183832834bf7041a0cae4fd27fc1112f51bc2 Mon Sep 17 00:00:00 2001
From: avogar <avogar@clickhouse.com>
Date: Thu, 18 Jul 2024 16:45:26 +0000
Subject: [PATCH 0406/1488] Fix runtime error while converting [Null] field to
 Array(Variant)

---
 src/Interpreters/convertFieldToType.cpp                | 10 +++++-----
 .../03203_variant_convert_field_to_type_bug.reference  |  0
 .../03203_variant_convert_field_to_type_bug.sql        |  5 +++++
 3 files changed, 10 insertions(+), 5 deletions(-)
 create mode 100644 tests/queries/0_stateless/03203_variant_convert_field_to_type_bug.reference
 create mode 100644 tests/queries/0_stateless/03203_variant_convert_field_to_type_bug.sql

diff --git a/src/Interpreters/convertFieldToType.cpp b/src/Interpreters/convertFieldToType.cpp
index 184c263dbdb..9ee214f4415 100644
--- a/src/Interpreters/convertFieldToType.cpp
+++ b/src/Interpreters/convertFieldToType.cpp
@@ -356,7 +356,7 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const ID
             for (size_t i = 0; i < src_arr_size; ++i)
             {
                 res[i] = convertFieldToType(src_arr[i], element_type);
-                if (res[i].isNull() && !element_type.isNullable())
+                if (res[i].isNull() && !canContainNull(element_type))
                 {
                     // See the comment for Tuples below.
                     have_unconvertible_element = true;
@@ -384,7 +384,7 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const ID
             {
                 const auto & element_type = *(type_tuple->getElements()[i]);
                 res[i] = convertFieldToType(src_tuple[i], element_type);
-                if (!res[i].isNull() || element_type.isNullable())
+                if (!res[i].isNull() || canContainNull(element_type))
                     continue;
 
                 /*
@@ -433,11 +433,11 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const ID
 
                 updated_entry[0] = convertFieldToType(key, key_type);
 
-                if (updated_entry[0].isNull() && !key_type.isNullable())
+                if (updated_entry[0].isNull() && !canContainNull(key_type))
                     have_unconvertible_element = true;
 
                 updated_entry[1] = convertFieldToType(value, value_type);
-                if (updated_entry[1].isNull() && !value_type.isNullable())
+                if (updated_entry[1].isNull() && !canContainNull(value_type))
                     have_unconvertible_element = true;
 
                 res[i] = updated_entry;
@@ -592,7 +592,7 @@ Field convertFieldToType(const Field & from_value, const IDataType & to_type, co
 Field convertFieldToTypeOrThrow(const Field & from_value, const IDataType & to_type, const IDataType * from_type_hint)
 {
     bool is_null = from_value.isNull();
-    if (is_null && !to_type.isNullable() && !to_type.isLowCardinalityNullable())
+    if (is_null && !canContainNull(to_type))
         throw Exception(ErrorCodes::TYPE_MISMATCH, "Cannot convert NULL to {}", to_type.getName());
 
     Field converted = convertFieldToType(from_value, to_type, from_type_hint);
diff --git a/tests/queries/0_stateless/03203_variant_convert_field_to_type_bug.reference b/tests/queries/0_stateless/03203_variant_convert_field_to_type_bug.reference
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/queries/0_stateless/03203_variant_convert_field_to_type_bug.sql b/tests/queries/0_stateless/03203_variant_convert_field_to_type_bug.sql
new file mode 100644
index 00000000000..b73bb8ffa6d
--- /dev/null
+++ b/tests/queries/0_stateless/03203_variant_convert_field_to_type_bug.sql
@@ -0,0 +1,5 @@
+set allow_experimental_variant_type=1;
+set use_variant_as_common_type=1;
+
+SELECT * FROM numbers([tuple(1, 2), NULL], 2); -- {serverError ILLEGAL_TYPE_OF_ARGUMENT}
+

From a6d4db342b2fc83e385d549ba5ce9ebf9e63064e Mon Sep 17 00:00:00 2001
From: robot-clickhouse <robot-clickhouse@users.noreply.github.com>
Date: Thu, 18 Jul 2024 16:45:40 +0000
Subject: [PATCH 0407/1488] Automatic style fix

---
 .../integration/test_recovery_time_metric/test.py  | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/tests/integration/test_recovery_time_metric/test.py b/tests/integration/test_recovery_time_metric/test.py
index 9ceb0cce288..90155f81ba2 100644
--- a/tests/integration/test_recovery_time_metric/test.py
+++ b/tests/integration/test_recovery_time_metric/test.py
@@ -2,7 +2,9 @@ import pytest
 from helpers.cluster import ClickHouseCluster
 
 cluster = ClickHouseCluster(__file__)
-node = cluster.add_instance("node", main_configs=["configs/config.xml"], with_zookeeper=True)
+node = cluster.add_instance(
+    "node", main_configs=["configs/config.xml"], with_zookeeper=True
+)
 
 
 @pytest.fixture(scope="module")
@@ -15,12 +17,10 @@ def start_cluster():
 
 
 def test_recovery_time_metric(start_cluster):
-    node.query("CREATE DATABASE rdb ENGINE = Replicated('/test/test_recovery_time_metric', 'shard1', 'replica1');")
+    node.query(
+        "CREATE DATABASE rdb ENGINE = Replicated('/test/test_recovery_time_metric', 'shard1', 'replica1');"
+    )
     node.query("CREATE TABLE rdb.t (x UInt32) ENGINE = MergeTree ORDER BY x;")
     node.exec_in_container(["bash", "-c", "rm /var/lib/clickhouse/metadata/rdb/t.sql"])
     node.restart_clickhouse()
-    assert (
-        node.query("SELECT any(recovery_time) FROM system.clusters;")
-        != "0\n"
-    )
-
+    assert node.query("SELECT any(recovery_time) FROM system.clusters;") != "0\n"

From 225af356c7ca3fd3c401f0fa8273d3dd751297fa Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Thu, 18 Jul 2024 17:06:03 +0000
Subject: [PATCH 0408/1488] Better.

---
 src/Interpreters/ActionsDAG.cpp       | 1 -
 src/Interpreters/ActionsDAG.h         | 1 -
 src/Interpreters/ActionsVisitor.h     | 8 +-------
 src/Interpreters/ExpressionAnalyzer.h | 3 ---
 src/Interpreters/WindowDescription.h  | 1 -
 src/Interpreters/addMissingDefaults.h | 6 ------
 6 files changed, 1 insertion(+), 19 deletions(-)

diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp
index 53e04f24829..85b2b38da17 100644
--- a/src/Interpreters/ActionsDAG.cpp
+++ b/src/Interpreters/ActionsDAG.cpp
@@ -3150,7 +3150,6 @@ const ActionsDAG::Node * FindOriginalNodeForOutputName::find(const String & outp
 }
 
 FindAliasForInputName::FindAliasForInputName(const ActionsDAG & actions_)
-    //: actions(actions_)
 {
     const auto & actions_outputs = actions_.getOutputs();
     for (const auto * output_node : actions_outputs)
diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h
index 6f6c3f9bccb..76cc9327530 100644
--- a/src/Interpreters/ActionsDAG.h
+++ b/src/Interpreters/ActionsDAG.h
@@ -501,7 +501,6 @@ public:
     const ActionsDAG::Node * find(const String & name);
 
 private:
-    //const ActionsDAG & actions;
     NameToNodeIndex index;
 };
 
diff --git a/src/Interpreters/ActionsVisitor.h b/src/Interpreters/ActionsVisitor.h
index 496d9b9b587..5b638fc14c8 100644
--- a/src/Interpreters/ActionsVisitor.h
+++ b/src/Interpreters/ActionsVisitor.h
@@ -18,12 +18,6 @@ namespace DB
 class ASTExpressionList;
 class ASTFunction;
 
-class ExpressionActions;
-using ExpressionActionsPtr = std::shared_ptr<ExpressionActions>;
-
-class ActionsDAG;
-using ActionsDAGPtr = std::unique_ptr<ActionsDAG>;
-
 class IFunctionOverloadResolver;
 using FunctionOverloadResolverPtr = std::shared_ptr<IFunctionOverloadResolver>;
 
@@ -32,7 +26,7 @@ FutureSetPtr makeExplicitSet(
     const ASTFunction * node, const ActionsDAG & actions, ContextPtr context, PreparedSets & prepared_sets);
 
 /** For ActionsVisitor
-  * A stack of ExpressionActions corresponding to nested lambda expressions.
+  * A stack of ActionsDAG corresponding to nested lambda expressions.
   * The new action should be added to the highest possible level.
   * For example, in the expression "select arrayMap(x -> x + column1 * column2, array1)"
   *  calculation of the product must be done outside the lambda expression (it does not depend on x),
diff --git a/src/Interpreters/ExpressionAnalyzer.h b/src/Interpreters/ExpressionAnalyzer.h
index 737d36eb504..0c00247df85 100644
--- a/src/Interpreters/ExpressionAnalyzer.h
+++ b/src/Interpreters/ExpressionAnalyzer.h
@@ -38,9 +38,6 @@ using StorageMetadataPtr = std::shared_ptr<const StorageInMemoryMetadata>;
 class ArrayJoinAction;
 using ArrayJoinActionPtr = std::shared_ptr<ArrayJoinAction>;
 
-class ActionsDAG;
-using ActionsDAGPtr = std::unique_ptr<ActionsDAG>;
-
 /// Create columns in block or return false if not possible
 bool sanitizeBlock(Block & block, bool throw_if_cannot_create_column = false);
 
diff --git a/src/Interpreters/WindowDescription.h b/src/Interpreters/WindowDescription.h
index 17bfe619c30..d51d9ca94d8 100644
--- a/src/Interpreters/WindowDescription.h
+++ b/src/Interpreters/WindowDescription.h
@@ -14,7 +14,6 @@ namespace DB
 class ASTFunction;
 
 class ActionsDAG;
-using ActionsDAGPtr = std::unique_ptr<ActionsDAG>;
 
 struct WindowFunctionDescription
 {
diff --git a/src/Interpreters/addMissingDefaults.h b/src/Interpreters/addMissingDefaults.h
index 5299bae9745..551583a0006 100644
--- a/src/Interpreters/addMissingDefaults.h
+++ b/src/Interpreters/addMissingDefaults.h
@@ -2,11 +2,6 @@
 
 #include <Interpreters/Context_fwd.h>
 
-#include <memory>
-#include <string>
-#include <unordered_map>
-
-
 namespace DB
 {
 
@@ -15,7 +10,6 @@ class NamesAndTypesList;
 class ColumnsDescription;
 
 class ActionsDAG;
-using ActionsDAGPtr = std::unique_ptr<ActionsDAG>;
 
 /** Adds three types of columns into block
   * 1. Columns, that are missed inside request, but present in table without defaults (missed columns)

From 9eba5975d8d5738998418bb0be613aede5f77c6b Mon Sep 17 00:00:00 2001
From: Max K <maxkaynov@gmail.com>
Date: Thu, 18 Jul 2024 18:43:34 +0200
Subject: [PATCH 0409/1488] CI: Fix issue with a skipped Build report

---
 tests/ci/ci_cache.py        |  5 -----
 tests/ci/ci_settings.py     |  4 ++++
 tests/ci/test_ci_options.py | 16 +++++++++++++---
 3 files changed, 17 insertions(+), 8 deletions(-)

diff --git a/tests/ci/ci_cache.py b/tests/ci/ci_cache.py
index 9486a286a8d..cfefb954fcd 100644
--- a/tests/ci/ci_cache.py
+++ b/tests/ci/ci_cache.py
@@ -737,17 +737,12 @@ class CiCache:
                     if job_name not in required_builds:
                         remove_from_to_do.append(job_name)
 
-        if not required_builds:
-            remove_from_to_do.append(CI.JobNames.BUILD_CHECK)
-
         for job in remove_from_to_do:
             print(f"Filter job [{job}] - not affected by the change")
             if job in self.jobs_to_do:
                 del self.jobs_to_do[job]
             if job in self.jobs_to_wait:
                 del self.jobs_to_wait[job]
-            if job in self.jobs_to_skip:
-                self.jobs_to_skip.remove(job)
 
     def await_pending_jobs(self, is_release: bool, dry_run: bool = False) -> None:
         """
diff --git a/tests/ci/ci_settings.py b/tests/ci/ci_settings.py
index 7b2dd12c310..54323ef868f 100644
--- a/tests/ci/ci_settings.py
+++ b/tests/ci/ci_settings.py
@@ -160,6 +160,10 @@ class CiSettings:
             else:
                 return False
 
+        if CI.is_build_job(job):
+            print(f"Build job [{job}] - always run")
+            return True
+
         if self.exclude_keywords:
             for keyword in self.exclude_keywords:
                 if keyword in normalize_string(job):
diff --git a/tests/ci/test_ci_options.py b/tests/ci/test_ci_options.py
index f4d14a17512..f71320abf2c 100644
--- a/tests/ci/test_ci_options.py
+++ b/tests/ci/test_ci_options.py
@@ -197,6 +197,10 @@ class TestCIOptions(unittest.TestCase):
                 "package_debug",
                 "package_msan",
                 "package_ubsan",
+                "package_aarch64",
+                "package_release_coverage",
+                "package_tsan",
+                "binary_release",
                 "Stateless tests (asan)",
                 "Stateless tests (azure, asan)",
                 "Stateless tests flaky check (asan)",
@@ -276,6 +280,7 @@ class TestCIOptions(unittest.TestCase):
             filtered_jobs,
             [
                 "Style check",
+                "fuzzers",
             ],
         )
 
@@ -291,9 +296,7 @@ class TestCIOptions(unittest.TestCase):
         )
         self.assertCountEqual(
             filtered_jobs,
-            [
-                "Style check",
-            ],
+            ["Style check", "fuzzers"],
         )
 
     def test_options_applied_4(self):
@@ -329,5 +332,12 @@ class TestCIOptions(unittest.TestCase):
                 "Stateless tests (release, old analyzer, s3, DatabaseReplicated)",
                 "package_asan",
                 "fuzzers",
+                "package_aarch64",
+                "package_release_coverage",
+                "package_debug",
+                "package_tsan",
+                "package_msan",
+                "package_ubsan",
+                "binary_release",
             ],
         )

From f50538c596050974c7afef5a87e10c97c2b1656b Mon Sep 17 00:00:00 2001
From: Sema Checherinda <Sema.Checherinda@clickhouse.com>
Date: Thu, 18 Jul 2024 19:31:49 +0200
Subject: [PATCH 0410/1488] add 03008_deduplication_rundom_setttings.sh

---
 ...ion_insert_several_blocks_nonreplicated.sh |  2 +-
 ...cation_insert_several_blocks_replicated.sh |  2 +-
 ..._generates_several_blocks_nonreplicated.sh |  2 +-
 ..._mv_generates_several_blocks_replicated.sh |  2 +-
 ...8_deduplication_rundom_setttings.reference |  3 +
 .../03008_deduplication_rundom_setttings.sh   | 78 +++++++++++++++++++
 ...several_mv_into_one_table_nonreplicated.sh |  2 +-
 ...on_several_mv_into_one_table_replicated.sh |  2 +-
 8 files changed, 87 insertions(+), 6 deletions(-)
 create mode 100644 tests/queries/0_stateless/03008_deduplication_rundom_setttings.reference
 create mode 100755 tests/queries/0_stateless/03008_deduplication_rundom_setttings.sh

diff --git a/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_nonreplicated.sh b/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_nonreplicated.sh
index f0a723501de..ef4866cb69e 100755
--- a/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_nonreplicated.sh
+++ b/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_nonreplicated.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Tags: long, no-fasttest, no-parallel, no-azure-blob-storage
+# Tags: long, no-fasttest, no-parallel, no-azure-blob-storage, no-s3-storage
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
diff --git a/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_replicated.sh b/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_replicated.sh
index 1b7f9be9510..dd4b1f7cec0 100755
--- a/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_replicated.sh
+++ b/tests/queries/0_stateless/03008_deduplication_insert_several_blocks_replicated.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Tags: long, no-fasttest, no-parallel, no-azure-blob-storage
+# Tags: long, no-fasttest, no-parallel, no-azure-blob-storage, no-s3-storage
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
diff --git a/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_nonreplicated.sh b/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_nonreplicated.sh
index e76b6eb77ce..33386c76edb 100755
--- a/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_nonreplicated.sh
+++ b/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_nonreplicated.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Tags: long, no-fasttest, no-parallel, no-azure-blob-storage
+# Tags: long, no-fasttest, no-parallel, no-azure-blob-storage, no-s3-storage
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
diff --git a/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_replicated.sh b/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_replicated.sh
index 7e6cf5e95c9..b66ef83abf2 100755
--- a/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_replicated.sh
+++ b/tests/queries/0_stateless/03008_deduplication_mv_generates_several_blocks_replicated.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Tags: long, no-fasttest, no-parallel, no-azure-blob-storage
+# Tags: long, no-fasttest, no-parallel, no-azure-blob-storage, no-s3-storage
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
diff --git a/tests/queries/0_stateless/03008_deduplication_rundom_setttings.reference b/tests/queries/0_stateless/03008_deduplication_rundom_setttings.reference
new file mode 100644
index 00000000000..8e7e24a7dd2
--- /dev/null
+++ b/tests/queries/0_stateless/03008_deduplication_rundom_setttings.reference
@@ -0,0 +1,3 @@
+insert_several_blocks_into_table OK
+mv_generates_several_blocks OK
+several_mv_into_one_table OK
diff --git a/tests/queries/0_stateless/03008_deduplication_rundom_setttings.sh b/tests/queries/0_stateless/03008_deduplication_rundom_setttings.sh
new file mode 100755
index 00000000000..20513c35c71
--- /dev/null
+++ b/tests/queries/0_stateless/03008_deduplication_rundom_setttings.sh
@@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+ENGINE="ReplicatedMergeTree"
+
+engine_options=("ReplicatedMergeTree" "MergeTree")
+engine=${engine_options[ $RANDOM % ${#engine_options[@]} ]}
+
+insert_method_option=("InsertSelect" "InsertValues")
+insert_method=${insert_method_option[ $RANDOM % ${#insert_method_option[@]} ]}
+
+use_insert_token_option=("True" "False")
+use_insert_token=${use_insert_token_option[ $RANDOM % ${#use_insert_token_option[@]} ]}
+
+single_thread_options=("True" "False")
+single_thread=${single_thread_options[ $RANDOM % ${#single_thread_options[@]} ]}
+
+deduplicate_src_table_options=("True" "False")
+deduplicate_src_table=${deduplicate_src_table_options[ $RANDOM % ${#deduplicate_src_table_options[@]} ]}
+
+deduplicate_dst_table_options=("True" "False")
+deduplicate_dst_table=${deduplicate_dst_table_options[ $RANDOM % ${#deduplicate_dst_table_options[@]} ]}
+
+insert_unique_blocks_options=("True" "False")
+insert_unique_blocks=${insert_unique_blocks_options[ $RANDOM % ${#insert_unique_blocks_options[@]} ]}
+
+
+THIS_RUN="Test case:"
+THIS_RUN+=" insert_method=$insert_method"
+THIS_RUN+=" engine=$ENGINE"
+THIS_RUN+=" use_insert_token=$use_insert_token"
+THIS_RUN+=" single_thread=$single_thread"
+THIS_RUN+=" deduplicate_src_table=$deduplicate_src_table"
+THIS_RUN+=" deduplicate_dst_table=$deduplicate_dst_table"
+THIS_RUN+=" insert_unique_blocks=$insert_unique_blocks"
+
+$CLICKHOUSE_CLIENT --max_insert_block_size 1  -nmq "
+    $(python3 $CURDIR/03008_deduplication.python insert_several_blocks_into_table \
+        --insert-method $insert_method \
+        --table-engine $ENGINE \
+        --use-insert-token $use_insert_token \
+        --single-thread $single_thread \
+        --deduplicate-src-table $deduplicate_src_table \
+        --deduplicate-dst-table $deduplicate_dst_table \
+        --insert-unique-blocks $insert_unique_blocks \
+        --get-logs false \
+    )
+" 1>/dev/null 2>&1 && echo 'insert_several_blocks_into_table OK'  || echo "FAIL: insert_several_blocks_into_table ${THIS_RUN}"
+
+$CLICKHOUSE_CLIENT --max_insert_block_size 1  -nmq "
+    $(python3 $CURDIR/03008_deduplication.python mv_generates_several_blocks \
+        --insert-method $insert_method \
+        --table-engine $ENGINE \
+        --use-insert-token $use_insert_token \
+        --single-thread $single_thread \
+        --deduplicate-src-table $deduplicate_src_table \
+        --deduplicate-dst-table $deduplicate_dst_table \
+        --insert-unique-blocks $insert_unique_blocks \
+        --get-logs false \
+    )
+" 1>/dev/null 2>&1 && echo 'mv_generates_several_blocks OK'  || echo "FAIL: mv_generates_several_blocks ${THIS_RUN}"
+
+$CLICKHOUSE_CLIENT  --max_insert_block_size 1 -nmq "
+    $(python3 $CURDIR/03008_deduplication.python several_mv_into_one_table \
+        --insert-method $insert_method \
+        --table-engine $ENGINE \
+        --use-insert-token $use_insert_token \
+        --single-thread $single_thread \
+        --deduplicate-src-table $deduplicate_src_table \
+        --deduplicate-dst-table $deduplicate_dst_table \
+        --insert-unique-blocks $insert_unique_blocks \
+        --get-logs false \
+    )
+" 1>/dev/null 2>&1 && echo 'several_mv_into_one_table OK'  || echo "FAIL: several_mv_into_one_table ${THIS_RUN}"
+
diff --git a/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_nonreplicated.sh b/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_nonreplicated.sh
index 193e15d311c..f9e1838f491 100755
--- a/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_nonreplicated.sh
+++ b/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_nonreplicated.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Tags: long, no-fasttest, no-parallel, no-azure-blob-storage
+# Tags: long, no-fasttest, no-parallel, no-azure-blob-storage, no-s3-storage
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
diff --git a/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_replicated.sh b/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_replicated.sh
index 3f4f0b17258..698e70d4064 100755
--- a/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_replicated.sh
+++ b/tests/queries/0_stateless/03008_deduplication_several_mv_into_one_table_replicated.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Tags: long, no-fasttest, no-parallel, no-azure-blob-storage
+# Tags: long, no-fasttest, no-parallel, no-azure-blob-storage, no-s3-storage
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh

From a7220ffff4a96683e0a9fb648efbe43dd66c19c0 Mon Sep 17 00:00:00 2001
From: Sema Checherinda <Sema.Checherinda@clickhouse.com>
Date: Thu, 18 Jul 2024 19:35:47 +0200
Subject: [PATCH 0411/1488] fix test name

---
 ...s.reference => 03008_deduplication_random_setttings.reference} | 0
 ...undom_setttings.sh => 03008_deduplication_random_setttings.sh} | 0
 2 files changed, 0 insertions(+), 0 deletions(-)
 rename tests/queries/0_stateless/{03008_deduplication_rundom_setttings.reference => 03008_deduplication_random_setttings.reference} (100%)
 rename tests/queries/0_stateless/{03008_deduplication_rundom_setttings.sh => 03008_deduplication_random_setttings.sh} (100%)

diff --git a/tests/queries/0_stateless/03008_deduplication_rundom_setttings.reference b/tests/queries/0_stateless/03008_deduplication_random_setttings.reference
similarity index 100%
rename from tests/queries/0_stateless/03008_deduplication_rundom_setttings.reference
rename to tests/queries/0_stateless/03008_deduplication_random_setttings.reference
diff --git a/tests/queries/0_stateless/03008_deduplication_rundom_setttings.sh b/tests/queries/0_stateless/03008_deduplication_random_setttings.sh
similarity index 100%
rename from tests/queries/0_stateless/03008_deduplication_rundom_setttings.sh
rename to tests/queries/0_stateless/03008_deduplication_random_setttings.sh

From 4249d869a952595a124528e1af6b6a91d21e3d3e Mon Sep 17 00:00:00 2001
From: Max K <maxkaynov@gmail.com>
Date: Thu, 18 Jul 2024 19:48:13 +0200
Subject: [PATCH 0412/1488] fix tests

---
 tests/ci/test_ci_config.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/tests/ci/test_ci_config.py b/tests/ci/test_ci_config.py
index 4336783e0d5..12e863c4d8d 100644
--- a/tests/ci/test_ci_config.py
+++ b/tests/ci/test_ci_config.py
@@ -295,6 +295,13 @@ class TestCIConfig(unittest.TestCase):
                     continue
                 expected_jobs_to_do.append(job)
         for job, config in CI.JOB_CONFIGS.items():
+            if (
+                CI.is_build_job(job)
+                and not config.run_by_label
+                and job not in expected_jobs_to_do
+            ):
+                # expected to run all builds jobs
+                expected_jobs_to_do.append(job)
             if not any(
                 keyword in normalize_string(job)
                 for keyword in settings.include_keywords

From 36468c528debe37bb6023a37d7ed9b93e7b56836 Mon Sep 17 00:00:00 2001
From: Sema Checherinda <Sema.Checherinda@clickhouse.com>
Date: Thu, 18 Jul 2024 19:55:09 +0200
Subject: [PATCH 0413/1488] relax condition in test, remove unused counters

---
 src/Common/ProfileEvents.cpp                                 | 2 --
 src/Coordination/KeeperConstants.cpp                         | 2 --
 src/IO/ReadBufferFromS3.cpp                                  | 2 --
 .../02789_reading_from_s3_with_connection_pool.sh            | 5 +----
 4 files changed, 1 insertion(+), 10 deletions(-)

diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp
index 871ba7cab8b..2b090136e2e 100644
--- a/src/Common/ProfileEvents.cpp
+++ b/src/Common/ProfileEvents.cpp
@@ -442,8 +442,6 @@ The server successfully detected this situation and will download merged part fr
     M(ReadBufferFromS3InitMicroseconds, "Time spent initializing connection to S3.") \
     M(ReadBufferFromS3Bytes, "Bytes read from S3.") \
     M(ReadBufferFromS3RequestsErrors, "Number of exceptions while reading from S3.") \
-    M(ReadBufferFromS3ResetSessions, "Number of HTTP sessions that were reset in ReadBufferFromS3.") \
-    M(ReadBufferFromS3PreservedSessions, "Number of HTTP sessions that were preserved in ReadBufferFromS3.") \
     \
     M(WriteBufferFromS3Microseconds, "Time spent on writing to S3.") \
     M(WriteBufferFromS3Bytes, "Bytes written to S3.") \
diff --git a/src/Coordination/KeeperConstants.cpp b/src/Coordination/KeeperConstants.cpp
index 7589e3393be..daa6a4bcb3f 100644
--- a/src/Coordination/KeeperConstants.cpp
+++ b/src/Coordination/KeeperConstants.cpp
@@ -183,8 +183,6 @@
     M(ReadBufferFromS3InitMicroseconds) \
     M(ReadBufferFromS3Bytes) \
     M(ReadBufferFromS3RequestsErrors) \
-    M(ReadBufferFromS3ResetSessions) \
-    M(ReadBufferFromS3PreservedSessions) \
 \
     M(WriteBufferFromS3Microseconds) \
     M(WriteBufferFromS3Bytes) \
diff --git a/src/IO/ReadBufferFromS3.cpp b/src/IO/ReadBufferFromS3.cpp
index 9e001232e65..bf97e959ee0 100644
--- a/src/IO/ReadBufferFromS3.cpp
+++ b/src/IO/ReadBufferFromS3.cpp
@@ -25,8 +25,6 @@ namespace ProfileEvents
     extern const Event ReadBufferFromS3InitMicroseconds;
     extern const Event ReadBufferFromS3Bytes;
     extern const Event ReadBufferFromS3RequestsErrors;
-    extern const Event ReadBufferFromS3ResetSessions;
-    extern const Event ReadBufferFromS3PreservedSessions;
     extern const Event ReadBufferSeekCancelConnection;
     extern const Event S3GetObject;
     extern const Event DiskS3GetObject;
diff --git a/tests/queries/0_stateless/02789_reading_from_s3_with_connection_pool.sh b/tests/queries/0_stateless/02789_reading_from_s3_with_connection_pool.sh
index e4a1de9a2ec..751b2798243 100755
--- a/tests/queries/0_stateless/02789_reading_from_s3_with_connection_pool.sh
+++ b/tests/queries/0_stateless/02789_reading_from_s3_with_connection_pool.sh
@@ -18,10 +18,7 @@ query="SELECT a, b FROM test_s3"
 query_id=$(${CLICKHOUSE_CLIENT} --query "select queryID() from ($query) limit 1" 2>&1)
 ${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS"
 ${CLICKHOUSE_CLIENT} -nm --query "
-WITH
-    ProfileEvents['DiskConnectionsReset'] AS reset,
-    ProfileEvents['DiskConnectionsPreserved'] AS preserved
-SELECT preserved > reset
+SELECT ProfileEvents['DiskConnectionsPreserved'] > 0
 FROM system.query_log
 WHERE type = 'QueryFinish'
     AND current_database = currentDatabase()

From c73dd41c58e8b59ef97373f64429a07045a3923a Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Thu, 18 Jul 2024 19:55:36 +0200
Subject: [PATCH 0414/1488] Remove bad test `host_resolver_fail_count`

---
 .../test_host_resolver_fail_count/__init__.py |   0
 .../configs/config.d/cluster.xml              |  12 --
 .../configs/config.d/s3.xml                   |  21 ---
 .../test_case.py                              | 126 ------------------
 4 files changed, 159 deletions(-)
 delete mode 100644 tests/integration/test_host_resolver_fail_count/__init__.py
 delete mode 100644 tests/integration/test_host_resolver_fail_count/configs/config.d/cluster.xml
 delete mode 100644 tests/integration/test_host_resolver_fail_count/configs/config.d/s3.xml
 delete mode 100644 tests/integration/test_host_resolver_fail_count/test_case.py

diff --git a/tests/integration/test_host_resolver_fail_count/__init__.py b/tests/integration/test_host_resolver_fail_count/__init__.py
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/tests/integration/test_host_resolver_fail_count/configs/config.d/cluster.xml b/tests/integration/test_host_resolver_fail_count/configs/config.d/cluster.xml
deleted file mode 100644
index bde62b82719..00000000000
--- a/tests/integration/test_host_resolver_fail_count/configs/config.d/cluster.xml
+++ /dev/null
@@ -1,12 +0,0 @@
-<clickhouse>
-    <profiles>
-        <default>
-            <connect_timeout>5</connect_timeout>
-            <receive_timeout>5</receive_timeout>
-            <send_timeout>5</send_timeout>
-            <http_connection_timeout>5</http_connection_timeout>
-            <http_send_timeout>5</http_send_timeout>
-            <http_receive_timeout>5</http_receive_timeout>
-        </default>
-    </profiles>
-</clickhouse>
\ No newline at end of file
diff --git a/tests/integration/test_host_resolver_fail_count/configs/config.d/s3.xml b/tests/integration/test_host_resolver_fail_count/configs/config.d/s3.xml
deleted file mode 100644
index 94ac83b32ac..00000000000
--- a/tests/integration/test_host_resolver_fail_count/configs/config.d/s3.xml
+++ /dev/null
@@ -1,21 +0,0 @@
-<clickhouse>
-    <storage_configuration>
-        <disks>
-            <s3>
-                <type>s3</type>
-                <endpoint>http://minio1:9001/root/data/</endpoint>
-                <access_key_id>minio</access_key_id>
-                <secret_access_key>minio123</secret_access_key>
-            </s3>
-        </disks>
-        <policies>
-            <s3>
-                <volumes>
-                    <main>
-                        <disk>s3</disk>
-                    </main>
-                </volumes>
-            </s3>
-        </policies>
-    </storage_configuration>
-</clickhouse>
diff --git a/tests/integration/test_host_resolver_fail_count/test_case.py b/tests/integration/test_host_resolver_fail_count/test_case.py
deleted file mode 100644
index d25681f0781..00000000000
--- a/tests/integration/test_host_resolver_fail_count/test_case.py
+++ /dev/null
@@ -1,126 +0,0 @@
-"""Test Interserver responses on configured IP."""
-
-import pytest
-import time
-from helpers.cluster import ClickHouseCluster
-
-cluster = ClickHouseCluster(__file__)
-
-node = cluster.add_instance(
-    "node",
-    main_configs=["configs/config.d/cluster.xml", "configs/config.d/s3.xml"],
-    with_minio=True,
-)
-
-
-@pytest.fixture(scope="module")
-def start_cluster():
-    try:
-        cluster.start()
-        yield cluster
-
-    finally:
-        cluster.shutdown()
-
-
-# The same value as in ClickHouse, this can't be confugured via config now
-DEFAULT_RESOLVE_TIME_HISTORY_SECONDS = 2 * 60
-
-
-def test_host_resolver(start_cluster):
-    minio_ip = cluster.get_instance_ip("minio1")
-
-    # drop DNS cache
-    node.set_hosts(
-        [
-            (minio_ip, "minio1"),
-            (node.ip_address, "minio1"),  # no answer on 9001 port on this IP
-        ]
-    )
-
-    node.query("SYSTEM DROP DNS CACHE")
-    node.query("SYSTEM DROP CONNECTIONS CACHE")
-
-    node.query(
-        """
-                CREATE TABLE test (key UInt32, value UInt32)
-                Engine=MergeTree()
-                ORDER BY key PARTITION BY key
-                SETTINGS storage_policy='s3'
-                """
-    )
-
-    initial_fails = "0\n"
-    k = 0
-    limit = 100
-    while initial_fails == "0\n":
-        node.query(
-            f"""
-                    INSERT INTO test VALUES (0,{k})
-                    """
-        )
-        # HostResolver chooses IP randomly, so on single call can choose worked ID
-        initial_fails = node.query(
-            "SELECT value FROM system.events WHERE event LIKE 'AddressesMarkedAsFailed'"
-        )
-        k += 1
-        if k >= limit:
-            # Dead IP was not choosen for 100 iteration.
-            # This is not expected, but not an error actually.
-            # And test should be stopped.
-            return
-
-    # initial_fails can be more than 1 if clickhouse does something in several parallel threads
-
-    for j in range(10):
-        for i in range(10):
-            node.query(
-                f"""
-                        INSERT INTO test VALUES ({i+1},{j+1})
-                        """
-            )
-            fails = node.query(
-                "SELECT value FROM system.events WHERE event LIKE 'AddressesMarkedAsFailed'"
-            )
-            assert fails == initial_fails
-
-    # Check that clickhouse tries to recheck IP after 2 minutes
-    time.sleep(DEFAULT_RESOLVE_TIME_HISTORY_SECONDS)
-
-    intermediate_fails = initial_fails
-    limit = k + 100
-    while intermediate_fails == initial_fails:
-        node.query(
-            f"""
-                    INSERT INTO test VALUES (101,{k})
-                    """
-        )
-        intermediate_fails = node.query(
-            "SELECT value FROM system.events WHERE event LIKE 'AddressesMarkedAsFailed'"
-        )
-        k += 1
-        if k >= limit:
-            # Dead IP was not choosen for 100 iteration.
-            # This is not expected, but not an error actually.
-            # And test should be stopped.
-            return
-
-    # After another 2 minutes shoudl not be new fails, next retry after 4 minutes
-    time.sleep(DEFAULT_RESOLVE_TIME_HISTORY_SECONDS)
-
-    initial_fails = intermediate_fails
-    limit = k + 100
-    while intermediate_fails == initial_fails:
-        node.query(
-            f"""
-                    INSERT INTO test VALUES (102,{k})
-                    """
-        )
-        intermediate_fails = node.query(
-            "SELECT value FROM system.events WHERE event LIKE 'AddressesMarkedAsFailed'"
-        )
-        k += 1
-        if k >= limit:
-            break
-
-    assert k == limit

From 4ebb189691c2d553887e9d49b52f9e0a45eaf004 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Thu, 18 Jul 2024 17:59:59 +0000
Subject: [PATCH 0415/1488] Better.

---
 src/Planner/CollectTableExpressionData.cpp |  8 ++++----
 src/Planner/PlannerJoinTree.cpp            | 12 ++++++------
 src/Planner/TableExpressionData.h          |  6 +++---
 src/Storages/SelectQueryInfo.cpp           |  2 +-
 4 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/src/Planner/CollectTableExpressionData.cpp b/src/Planner/CollectTableExpressionData.cpp
index 1d85476636c..2fe62aa9be0 100644
--- a/src/Planner/CollectTableExpressionData.cpp
+++ b/src/Planner/CollectTableExpressionData.cpp
@@ -88,15 +88,15 @@ public:
 
                 auto column_identifier = planner_context->getGlobalPlannerContext()->createColumnIdentifier(node);
 
-                ActionsDAGPtr alias_column_actions_dag = std::make_unique<ActionsDAG>();
+                ActionsDAG alias_column_actions_dag;
                 PlannerActionsVisitor actions_visitor(planner_context, false);
-                auto outputs = actions_visitor.visit(*alias_column_actions_dag, column_node->getExpression());
+                auto outputs = actions_visitor.visit(alias_column_actions_dag, column_node->getExpression());
                 if (outputs.size() != 1)
                     throw Exception(ErrorCodes::LOGICAL_ERROR,
                         "Expected single output in actions dag for alias column {}. Actual {}", column_node->dumpTree(), outputs.size());
                 const auto & column_name = column_node->getColumnName();
-                const auto & alias_node = alias_column_actions_dag->addAlias(*outputs[0], column_name);
-                alias_column_actions_dag->addOrReplaceInOutputs(alias_node);
+                const auto & alias_node = alias_column_actions_dag.addAlias(*outputs[0], column_name);
+                alias_column_actions_dag.addOrReplaceInOutputs(alias_node);
                 table_expression_data.addAliasColumn(column_node->getColumn(), column_identifier, std::move(alias_column_actions_dag), select_added_columns);
             }
 
diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp
index 10b5a761d58..048bfa4b577 100644
--- a/src/Planner/PlannerJoinTree.cpp
+++ b/src/Planner/PlannerJoinTree.cpp
@@ -591,16 +591,16 @@ UInt64 mainQueryNodeBlockSizeByLimit(const SelectQueryInfo & select_query_info)
 }
 
 std::unique_ptr<ExpressionStep> createComputeAliasColumnsStep(
-    const std::unordered_map<std::string, ActionsDAGPtr> & alias_column_expressions, const DataStream & current_data_stream)
+    std::unordered_map<std::string, ActionsDAG> & alias_column_expressions, const DataStream & current_data_stream)
 {
     ActionsDAG merged_alias_columns_actions_dag(current_data_stream.header.getColumnsWithTypeAndName());
     ActionsDAG::NodeRawConstPtrs action_dag_outputs = merged_alias_columns_actions_dag.getInputs();
 
-    for (const auto & [column_name, alias_column_actions_dag] : alias_column_expressions)
+    for (auto & [column_name, alias_column_actions_dag] : alias_column_expressions)
     {
-        const auto & current_outputs = alias_column_actions_dag->getOutputs();
+        const auto & current_outputs = alias_column_actions_dag.getOutputs();
         action_dag_outputs.insert(action_dag_outputs.end(), current_outputs.begin(), current_outputs.end());
-        merged_alias_columns_actions_dag.mergeNodes(std::move(*alias_column_actions_dag));
+        merged_alias_columns_actions_dag.mergeNodes(std::move(alias_column_actions_dag));
     }
 
     for (const auto * output_node : action_dag_outputs)
@@ -996,7 +996,7 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres
                     }
                 }
 
-                const auto & alias_column_expressions = table_expression_data.getAliasColumnExpressions();
+                auto & alias_column_expressions = table_expression_data.getAliasColumnExpressions();
                 if (!alias_column_expressions.empty() && query_plan.isInitialized() && from_stage == QueryProcessingStage::FetchColumns)
                 {
                     auto alias_column_step = createComputeAliasColumnsStep(alias_column_expressions, query_plan.getCurrentDataStream());
@@ -1085,7 +1085,7 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres
             query_plan = std::move(subquery_planner).extractQueryPlan();
         }
 
-        const auto & alias_column_expressions = table_expression_data.getAliasColumnExpressions();
+        auto & alias_column_expressions = table_expression_data.getAliasColumnExpressions();
         if (!alias_column_expressions.empty() && query_plan.isInitialized() && from_stage == QueryProcessingStage::FetchColumns)
         {
             auto alias_column_step = createComputeAliasColumnsStep(alias_column_expressions, query_plan.getCurrentDataStream());
diff --git a/src/Planner/TableExpressionData.h b/src/Planner/TableExpressionData.h
index 1d04fac3dc3..72412a869e4 100644
--- a/src/Planner/TableExpressionData.h
+++ b/src/Planner/TableExpressionData.h
@@ -73,7 +73,7 @@ public:
     }
 
     /// Add alias column
-    void addAliasColumn(const NameAndTypePair & column, const ColumnIdentifier & column_identifier, ActionsDAGPtr actions_dag, bool is_selected_column = true)
+    void addAliasColumn(const NameAndTypePair & column, const ColumnIdentifier & column_identifier, ActionsDAG actions_dag, bool is_selected_column = true)
     {
         alias_column_expressions.emplace(column.name, std::move(actions_dag));
         addColumnImpl(column, column_identifier, is_selected_column);
@@ -94,7 +94,7 @@ public:
     }
 
     /// Get ALIAS columns names mapped to expressions
-    const std::unordered_map<std::string, ActionsDAGPtr> & getAliasColumnExpressions() const
+    std::unordered_map<std::string, ActionsDAG> & getAliasColumnExpressions()
     {
         return alias_column_expressions;
     }
@@ -277,7 +277,7 @@ private:
     NameSet selected_column_names_set;
 
     /// Expression to calculate ALIAS columns
-    std::unordered_map<std::string, ActionsDAGPtr> alias_column_expressions;
+    std::unordered_map<std::string, ActionsDAG> alias_column_expressions;
 
     /// Valid for table, table function, array join, query, union nodes
     ColumnNameToColumn column_name_to_column;
diff --git a/src/Storages/SelectQueryInfo.cpp b/src/Storages/SelectQueryInfo.cpp
index d59ccf0dfaf..c9c96ed5837 100644
--- a/src/Storages/SelectQueryInfo.cpp
+++ b/src/Storages/SelectQueryInfo.cpp
@@ -18,7 +18,7 @@ std::unordered_map<std::string, ColumnWithTypeAndName> SelectQueryInfo::buildNod
     std::unordered_map<std::string, ColumnWithTypeAndName> node_name_to_input_node_column;
     if (planner_context)
     {
-        const auto & table_expression_data = planner_context->getTableExpressionDataOrThrow(table_expression);
+        auto & table_expression_data = planner_context->getTableExpressionDataOrThrow(table_expression);
         const auto & alias_column_expressions = table_expression_data.getAliasColumnExpressions();
         for (const auto & [column_identifier, column_name] : table_expression_data.getColumnIdentifierToColumnName())
         {

From 4eb5a404430f69785fd756d1706621fe5410f43c Mon Sep 17 00:00:00 2001
From: Nikita Taranov <nikita.taranov@clickhouse.com>
Date: Thu, 18 Jul 2024 19:04:42 +0100
Subject: [PATCH 0416/1488] add unit tests

---
 src/Common/CgroupsMemoryUsageObserver.cpp |  12 +-
 src/Common/CgroupsMemoryUsageObserver.h   |   3 +
 src/Common/tests/gtest_cgroups_reader.cpp | 175 ++++++++++++++++++++++
 3 files changed, 186 insertions(+), 4 deletions(-)
 create mode 100644 src/Common/tests/gtest_cgroups_reader.cpp

diff --git a/src/Common/CgroupsMemoryUsageObserver.cpp b/src/Common/CgroupsMemoryUsageObserver.cpp
index cf661174789..ef8bdfc1823 100644
--- a/src/Common/CgroupsMemoryUsageObserver.cpp
+++ b/src/Common/CgroupsMemoryUsageObserver.cpp
@@ -200,10 +200,7 @@ CgroupsMemoryUsageObserver::CgroupsMemoryUsageObserver(std::chrono::seconds wait
 {
     const auto [cgroup_path, version] = getCgroupsPath();
 
-    if (version == CgroupsVersion::V2)
-        cgroup_reader = std::make_unique<CgroupsV2Reader>(cgroup_path);
-    else
-        cgroup_reader = std::make_unique<CgroupsV1Reader>(cgroup_path);
+    cgroup_reader = createCgroupsReader(version, cgroup_path);
 
     LOG_INFO(
         log,
@@ -365,6 +362,13 @@ void CgroupsMemoryUsageObserver::runThread()
     }
 }
 
+std::unique_ptr<ICgroupsReader> createCgroupsReader(CgroupsMemoryUsageObserver::CgroupsVersion version, const fs::path & cgroup_path)
+{
+    if (version == CgroupsMemoryUsageObserver::CgroupsVersion::V2)
+        return std::make_unique<CgroupsV2Reader>(cgroup_path);
+    else
+        return std::make_unique<CgroupsV1Reader>(cgroup_path);
+}
 }
 
 #endif
diff --git a/src/Common/CgroupsMemoryUsageObserver.h b/src/Common/CgroupsMemoryUsageObserver.h
index 0d5d07597c8..7f888fe631b 100644
--- a/src/Common/CgroupsMemoryUsageObserver.h
+++ b/src/Common/CgroupsMemoryUsageObserver.h
@@ -83,6 +83,9 @@ private:
     bool quit = false;
 };
 
+std::unique_ptr<ICgroupsReader>
+createCgroupsReader(CgroupsMemoryUsageObserver::CgroupsVersion version, const std::filesystem::path & cgroup_path);
+
 #else
 class CgroupsMemoryUsageObserver
 {
diff --git a/src/Common/tests/gtest_cgroups_reader.cpp b/src/Common/tests/gtest_cgroups_reader.cpp
new file mode 100644
index 00000000000..38e56401401
--- /dev/null
+++ b/src/Common/tests/gtest_cgroups_reader.cpp
@@ -0,0 +1,175 @@
+#include <cstdint>
+#include <filesystem>
+#include <stdint.h>
+#include <gtest/gtest.h>
+
+#include <IO/WriteBufferFromFile.h>
+#include <Common/CgroupsMemoryUsageObserver.h>
+#include <Common/filesystemHelpers.h>
+#include "IO/WriteBufferFromFileBase.h"
+
+using namespace DB;
+
+
+const std::string SAMPLE_FILE[2] = {
+    R"(cache 4673703936
+rss 2232029184
+rss_huge 0
+shmem 0
+mapped_file 344678400
+dirty 4730880
+writeback 135168
+swap 0
+pgpgin 2038569918
+pgpgout 2036883790
+pgfault 2055373287
+pgmajfault 0
+inactive_anon 2156335104
+active_anon 0
+inactive_file 2841305088
+active_file 1653915648
+unevictable 256008192
+hierarchical_memory_limit 8589934592
+hierarchical_memsw_limit 8589934592
+total_cache 4673703936
+total_rss 2232029184
+total_rss_huge 0
+total_shmem 0
+total_mapped_file 344678400
+total_dirty 4730880
+total_writeback 135168
+total_swap 0
+total_pgpgin 2038569918
+total_pgpgout 2036883790
+total_pgfault 2055373287
+total_pgmajfault 0
+total_inactive_anon 2156335104
+total_active_anon 0
+total_inactive_file 2841305088
+total_active_file 1653915648
+total_unevictable 256008192
+)",
+    R"(anon 10429399040
+file 17410793472
+kernel 1537789952
+kernel_stack 3833856
+pagetables 65441792
+sec_pagetables 0
+percpu 15232
+sock 0
+vmalloc 0
+shmem 0
+zswap 0
+zswapped 0
+file_mapped 344010752
+file_dirty 2060857344
+file_writeback 0
+swapcached 0
+anon_thp 0
+file_thp 0
+shmem_thp 0
+inactive_anon 0
+active_anon 10429370368
+inactive_file 8693084160
+active_file 8717561856
+unevictable 0
+slab_reclaimable 1460982504
+slab_unreclaimable 5152864
+slab 1466135368
+workingset_refault_anon 0
+workingset_refault_file 0
+workingset_activate_anon 0
+workingset_activate_file 0
+workingset_restore_anon 0
+workingset_restore_file 0
+workingset_nodereclaim 0
+pgscan 0
+pgsteal 0
+pgscan_kswapd 0
+pgscan_direct 0
+pgscan_khugepaged 0
+pgsteal_kswapd 0
+pgsteal_direct 0
+pgsteal_khugepaged 0
+pgfault 43026352
+pgmajfault 36762
+pgrefill 0
+pgactivate 0
+pgdeactivate 0
+pglazyfree 259
+pglazyfreed 0
+zswpin 0
+zswpout 0
+thp_fault_alloc 0
+thp_collapse_alloc 0
+)"};
+
+const std::string EXPECTED[2]
+    = {"{\"active_anon\": 0, \"active_file\": 1653915648, \"cache\": 4673703936, \"dirty\": 4730880, \"hierarchical_memory_limit\": "
+       "8589934592, \"hierarchical_memsw_limit\": 8589934592, \"inactive_anon\": 2156335104, \"inactive_file\": 2841305088, "
+       "\"mapped_file\": 344678400, \"pgfault\": 2055373287, \"pgmajfault\": 0, \"pgpgin\": 2038569918, \"pgpgout\": 2036883790, \"rss\": "
+       "2232029184, \"rss_huge\": 0, \"shmem\": 0, \"swap\": 0, \"total_active_anon\": 0, \"total_active_file\": 1653915648, "
+       "\"total_cache\": 4673703936, \"total_dirty\": 4730880, \"total_inactive_anon\": 2156335104, \"total_inactive_file\": 2841305088, "
+       "\"total_mapped_file\": 344678400, \"total_pgfault\": 2055373287, \"total_pgmajfault\": 0, \"total_pgpgin\": 2038569918, "
+       "\"total_pgpgout\": 2036883790, \"total_rss\": 2232029184, \"total_rss_huge\": 0, \"total_shmem\": 0, \"total_swap\": 0, "
+       "\"total_unevictable\": 256008192, \"total_writeback\": 135168, \"unevictable\": 256008192, \"writeback\": 135168}",
+       "{\"active_anon\": 10429370368, \"active_file\": 8717561856, \"anon\": 10429399040, \"anon_thp\": 0, \"file\": 17410793472, "
+       "\"file_dirty\": 2060857344, \"file_mapped\": 344010752, \"file_thp\": 0, \"file_writeback\": 0, \"inactive_anon\": 0, "
+       "\"inactive_file\": 8693084160, \"kernel\": 1537789952, \"kernel_stack\": 3833856, \"pagetables\": 65441792, \"percpu\": 15232, "
+       "\"pgactivate\": 0, \"pgdeactivate\": 0, \"pgfault\": 43026352, \"pglazyfree\": 259, \"pglazyfreed\": 0, \"pgmajfault\": 36762, "
+       "\"pgrefill\": 0, \"pgscan\": 0, \"pgscan_direct\": 0, \"pgscan_khugepaged\": 0, \"pgscan_kswapd\": 0, \"pgsteal\": 0, "
+       "\"pgsteal_direct\": 0, \"pgsteal_khugepaged\": 0, \"pgsteal_kswapd\": 0, \"sec_pagetables\": 0, \"shmem\": 0, \"shmem_thp\": 0, "
+       "\"slab\": 1466135368, \"slab_reclaimable\": 1460982504, \"slab_unreclaimable\": 5152864, \"sock\": 0, \"swapcached\": 0, "
+       "\"thp_collapse_alloc\": 0, \"thp_fault_alloc\": 0, \"unevictable\": 0, \"vmalloc\": 0, \"workingset_activate_anon\": 0, "
+       "\"workingset_activate_file\": 0, \"workingset_nodereclaim\": 0, \"workingset_refault_anon\": 0, \"workingset_refault_file\": 0, "
+       "\"workingset_restore_anon\": 0, \"workingset_restore_file\": 0, \"zswap\": 0, \"zswapped\": 0, \"zswpin\": 0, \"zswpout\": 0}"};
+
+
+class CgroupsMemoryUsageObserverFixture : public ::testing::TestWithParam<CgroupsMemoryUsageObserver::CgroupsVersion>
+{
+    void SetUp() override
+    {
+        const uint8_t version = static_cast<uint8_t>(GetParam());
+        tmp_dir = fmt::format("./test_cgroups_{}", magic_enum::enum_name(GetParam()));
+        fs::create_directories(tmp_dir);
+
+        auto stat_file = WriteBufferFromFile(tmp_dir + "/memory.stat");
+        stat_file.write(SAMPLE_FILE[version].data(), SAMPLE_FILE[version].size());
+        stat_file.sync();
+
+        if (GetParam() == CgroupsMemoryUsageObserver::CgroupsVersion::V2)
+        {
+            auto current_file = WriteBufferFromFile(tmp_dir + "/memory.current");
+            current_file.write("29645422592", 11);
+            current_file.sync();
+        }
+    }
+
+protected:
+    std::string tmp_dir;
+};
+
+
+TEST_P(CgroupsMemoryUsageObserverFixture, ReadMemoryUsageTest)
+{
+    const auto version = GetParam();
+    auto reader = createCgroupsReader(version, tmp_dir);
+    ASSERT_EQ(
+        reader->readMemoryUsage(),
+        version == CgroupsMemoryUsageObserver::CgroupsVersion::V1 ? /* rss from memory.stat */ 2232029184
+                                                                  : /* value from memory.current - inactive_file */ 20952338432);
+}
+
+
+TEST_P(CgroupsMemoryUsageObserverFixture, DumpAllStatsTest)
+{
+    const auto version = GetParam();
+    auto reader = createCgroupsReader(version, tmp_dir);
+    ASSERT_EQ(reader->dumpAllStats(), EXPECTED[static_cast<uint8_t>(version)]);
+}
+
+
+INSTANTIATE_TEST_SUITE_P(
+    CgroupsMemoryUsageObserverTests,
+    CgroupsMemoryUsageObserverFixture,
+    ::testing::Values(CgroupsMemoryUsageObserver::CgroupsVersion::V1, CgroupsMemoryUsageObserver::CgroupsVersion::V2));

From 3ef2cbdec5b960a873ee4fd70d9788248d4324a1 Mon Sep 17 00:00:00 2001
From: Yong Wang <yong.wang@ibm.com>
Date: Thu, 18 Jul 2024 11:21:52 -0700
Subject: [PATCH 0417/1488] openssl-dynamic-build-ppc64le : fix openssl dynamic
 build for ppc64le

---
 contrib/openssl-cmake/CMakeLists.txt | 1 -
 1 file changed, 1 deletion(-)

diff --git a/contrib/openssl-cmake/CMakeLists.txt b/contrib/openssl-cmake/CMakeLists.txt
index 85de0340996..a9e4a3df698 100644
--- a/contrib/openssl-cmake/CMakeLists.txt
+++ b/contrib/openssl-cmake/CMakeLists.txt
@@ -1298,7 +1298,6 @@ elseif(ARCH_PPC64LE)
         ${OPENSSL_SOURCE_DIR}/crypto/camellia/camellia.c
         ${OPENSSL_SOURCE_DIR}/crypto/camellia/cmll_cbc.c
         ${OPENSSL_SOURCE_DIR}/crypto/chacha/chacha_enc.c
-        ${OPENSSL_SOURCE_DIR}/crypto/mem_clr.c
         ${OPENSSL_SOURCE_DIR}/crypto/rc4/rc4_enc.c
         ${OPENSSL_SOURCE_DIR}/crypto/rc4/rc4_skey.c
         ${OPENSSL_SOURCE_DIR}/crypto/sha/keccak1600.c

From 465e4ad73f60a3843426c88301b9d3c1376fc851 Mon Sep 17 00:00:00 2001
From: Max K <maxkaynov@gmail.com>
Date: Wed, 17 Jul 2024 17:19:56 +0200
Subject: [PATCH 0418/1488] CI: Remove aws lambda packages from oss

---
 tests/ci/autoscale_runners_lambda/app.py      | 235 ----------
 .../build_and_deploy_archive.sh               |   1 -
 .../ci/autoscale_runners_lambda/lambda_shared |   1 -
 .../autoscale_runners_lambda/requirements.txt |   1 -
 .../test_autoscale.py                         | 196 ---------
 tests/ci/build_check.py                       |   5 +-
 .../cancel_and_rerun_workflow_lambda/app.py   | 376 ----------------
 .../build_and_deploy_archive.sh               |   1 -
 .../lambda_shared                             |   1 -
 .../requirements.txt                          |   1 -
 tests/ci/cherry_pick.py                       |   2 +-
 tests/ci/ci_config.py                         |   3 +
 tests/ci/ci_definitions.py                    |  47 ++
 tests/ci/ci_runners_metrics_lambda/app.py     | 164 -------
 .../build_and_deploy_archive.sh               |   1 -
 .../ci_runners_metrics_lambda/lambda_shared   |   1 -
 .../requirements.txt                          |   2 -
 tests/ci/ci_utils.py                          | 115 ++++-
 tests/ci/clean_lost_instances_lambda/app.py   | 336 ---------------
 .../build_and_deploy_archive.sh               |   1 -
 .../clean_lost_instances_lambda/lambda_shared |   1 -
 .../requirements.txt                          |   2 -
 tests/ci/lambda_shared_package/.gitignore     |   2 -
 tests/ci/lambda_shared_package/__init__.py    |   0
 .../lambda_shared/__init__.py                 | 237 ----------
 .../lambda_shared_package/lambda_shared/pr.py | 168 --------
 .../lambda_shared/token.py                    |  95 ----
 tests/ci/lambda_shared_package/pyproject.toml |  24 --
 tests/ci/lambda_shared_package/setup.cfg      |   8 -
 tests/ci/pr_info.py                           |   2 +-
 tests/ci/release.py                           |   2 +-
 tests/ci/run_check.py                         |  50 ++-
 tests/ci/runner_token_rotation_lambda/app.py  |  93 ----
 .../build_and_deploy_archive.sh               |   1 -
 .../lambda_shared                             |   1 -
 .../requirements.txt                          |   1 -
 tests/ci/slack_bot_ci_lambda/app.py           | 323 --------------
 .../build_and_deploy_archive.sh               |   1 -
 tests/ci/slack_bot_ci_lambda/requirements.txt |   1 -
 tests/ci/team_keys_lambda/app.py              | 136 ------
 .../build_and_deploy_archive.sh               |  76 ----
 tests/ci/team_keys_lambda/lambda_shared       |   1 -
 tests/ci/team_keys_lambda/requirements.txt    |   1 -
 tests/ci/terminate_runner_lambda/app.py       | 278 ------------
 .../build_and_deploy_archive.sh               |   1 -
 .../ci/terminate_runner_lambda/lambda_shared  |   1 -
 .../terminate_runner_lambda/requirements.txt  |   1 -
 tests/ci/workflow_approve_rerun_lambda/app.py | 404 ------------------
 .../build_and_deploy_archive.sh               |   1 -
 .../lambda_shared                             |   1 -
 .../requirements.txt                          |   1 -
 tests/ci/workflow_jobs_lambda/app.py          | 202 ---------
 .../build_and_deploy_archive.sh               |   1 -
 tests/ci/workflow_jobs_lambda/lambda_shared   |   1 -
 .../ci/workflow_jobs_lambda/requirements.txt  |   1 -
 55 files changed, 194 insertions(+), 3415 deletions(-)
 delete mode 100644 tests/ci/autoscale_runners_lambda/app.py
 delete mode 120000 tests/ci/autoscale_runners_lambda/build_and_deploy_archive.sh
 delete mode 120000 tests/ci/autoscale_runners_lambda/lambda_shared
 delete mode 100644 tests/ci/autoscale_runners_lambda/requirements.txt
 delete mode 100644 tests/ci/autoscale_runners_lambda/test_autoscale.py
 delete mode 100644 tests/ci/cancel_and_rerun_workflow_lambda/app.py
 delete mode 120000 tests/ci/cancel_and_rerun_workflow_lambda/build_and_deploy_archive.sh
 delete mode 120000 tests/ci/cancel_and_rerun_workflow_lambda/lambda_shared
 delete mode 100644 tests/ci/cancel_and_rerun_workflow_lambda/requirements.txt
 delete mode 100644 tests/ci/ci_runners_metrics_lambda/app.py
 delete mode 120000 tests/ci/ci_runners_metrics_lambda/build_and_deploy_archive.sh
 delete mode 120000 tests/ci/ci_runners_metrics_lambda/lambda_shared
 delete mode 100644 tests/ci/ci_runners_metrics_lambda/requirements.txt
 delete mode 100644 tests/ci/clean_lost_instances_lambda/app.py
 delete mode 120000 tests/ci/clean_lost_instances_lambda/build_and_deploy_archive.sh
 delete mode 120000 tests/ci/clean_lost_instances_lambda/lambda_shared
 delete mode 100644 tests/ci/clean_lost_instances_lambda/requirements.txt
 delete mode 100644 tests/ci/lambda_shared_package/.gitignore
 delete mode 100644 tests/ci/lambda_shared_package/__init__.py
 delete mode 100644 tests/ci/lambda_shared_package/lambda_shared/__init__.py
 delete mode 100644 tests/ci/lambda_shared_package/lambda_shared/pr.py
 delete mode 100644 tests/ci/lambda_shared_package/lambda_shared/token.py
 delete mode 100644 tests/ci/lambda_shared_package/pyproject.toml
 delete mode 100644 tests/ci/lambda_shared_package/setup.cfg
 delete mode 100644 tests/ci/runner_token_rotation_lambda/app.py
 delete mode 120000 tests/ci/runner_token_rotation_lambda/build_and_deploy_archive.sh
 delete mode 120000 tests/ci/runner_token_rotation_lambda/lambda_shared
 delete mode 100644 tests/ci/runner_token_rotation_lambda/requirements.txt
 delete mode 100755 tests/ci/slack_bot_ci_lambda/app.py
 delete mode 120000 tests/ci/slack_bot_ci_lambda/build_and_deploy_archive.sh
 delete mode 100644 tests/ci/slack_bot_ci_lambda/requirements.txt
 delete mode 100644 tests/ci/team_keys_lambda/app.py
 delete mode 100644 tests/ci/team_keys_lambda/build_and_deploy_archive.sh
 delete mode 120000 tests/ci/team_keys_lambda/lambda_shared
 delete mode 100644 tests/ci/team_keys_lambda/requirements.txt
 delete mode 100644 tests/ci/terminate_runner_lambda/app.py
 delete mode 120000 tests/ci/terminate_runner_lambda/build_and_deploy_archive.sh
 delete mode 120000 tests/ci/terminate_runner_lambda/lambda_shared
 delete mode 100644 tests/ci/terminate_runner_lambda/requirements.txt
 delete mode 100644 tests/ci/workflow_approve_rerun_lambda/app.py
 delete mode 120000 tests/ci/workflow_approve_rerun_lambda/build_and_deploy_archive.sh
 delete mode 120000 tests/ci/workflow_approve_rerun_lambda/lambda_shared
 delete mode 100644 tests/ci/workflow_approve_rerun_lambda/requirements.txt
 delete mode 100644 tests/ci/workflow_jobs_lambda/app.py
 delete mode 120000 tests/ci/workflow_jobs_lambda/build_and_deploy_archive.sh
 delete mode 120000 tests/ci/workflow_jobs_lambda/lambda_shared
 delete mode 100644 tests/ci/workflow_jobs_lambda/requirements.txt

diff --git a/tests/ci/autoscale_runners_lambda/app.py b/tests/ci/autoscale_runners_lambda/app.py
deleted file mode 100644
index c790a5081dd..00000000000
--- a/tests/ci/autoscale_runners_lambda/app.py
+++ /dev/null
@@ -1,235 +0,0 @@
-#!/usr/bin/env python3
-
-"""The lambda to decrease/increase ASG desired capacity based on current queue"""
-
-import logging
-from dataclasses import dataclass
-from pprint import pformat
-from typing import Any, List, Literal, Optional, Tuple
-
-import boto3  # type: ignore
-from lambda_shared import (
-    RUNNER_TYPE_LABELS,
-    CHException,
-    ClickHouseHelper,
-    get_parameter_from_ssm,
-)
-
-### Update comment on the change ###
-# 4 HOUR - is a balance to get the most precise values
-#   - Our longest possible running check is around 5h on the worst scenario
-#   - The long queue won't be wiped out and replaced, so the measurmenet is fine
-#   - If the data is spoiled by something, we are from the bills perspective
-# Changed it to 3 HOUR: in average we have 1h tasks, but p90 is around 2h.
-# With 4h we have too much wasted computing time in case of issues with DB
-QUEUE_QUERY = f"""SELECT
-    last_status AS status,
-    toUInt32(count()) AS length,
-    labels
-FROM
-(
-    SELECT
-        arraySort(groupArray(status))[-1] AS last_status,
-        labels,
-        id,
-        html_url
-    FROM default.workflow_jobs
-    WHERE has(labels, 'self-hosted')
-        AND hasAny({RUNNER_TYPE_LABELS}, labels)
-        AND started_at > now() - INTERVAL 3 HOUR
-    GROUP BY ALL
-    HAVING last_status IN ('in_progress', 'queued')
-)
-GROUP BY ALL
-ORDER BY labels, last_status"""
-
-
-@dataclass
-class Queue:
-    status: Literal["in_progress", "queued"]
-    lentgh: int
-    label: str
-
-
-def get_scales(runner_type: str) -> Tuple[int, int]:
-    "returns the multipliers for scaling down and up ASG by types"
-    # Scaling down is quicker on the lack of running jobs than scaling up on
-    # queue
-
-    # The ASG should deflate almost instantly
-    scale_down = 1
-    # the style checkers have so many noise, so it scales up too quickly
-    # The 5 was too quick, there are complainings regarding too slow with
-    # 10. I am trying 7 now.
-    # 7 still looks a bit slow, so I try 6
-    # Let's have it the same as the other ASG
-    #
-    # All type of style-checkers should be added very quickly to not block the workflows
-    # UPDATE THE COMMENT ON CHANGES
-    scale_up = 3
-    if "style" in runner_type:
-        scale_up = 1
-    return scale_down, scale_up
-
-
-CH_CLIENT = None  # type: Optional[ClickHouseHelper]
-
-
-def set_capacity(
-    runner_type: str, queues: List[Queue], client: Any, dry_run: bool = True
-) -> None:
-    assert len(queues) in (1, 2)
-    assert all(q.label == runner_type for q in queues)
-    as_groups = client.describe_auto_scaling_groups(
-        Filters=[
-            {"Name": "tag-key", "Values": ["github:runner-type"]},
-            {"Name": "tag-value", "Values": [runner_type]},
-        ]
-    )["AutoScalingGroups"]
-    assert len(as_groups) == 1
-    asg = as_groups[0]
-    running = 0
-    queued = 0
-    for q in queues:
-        if q.status == "in_progress":
-            running = q.lentgh
-            continue
-        if q.status == "queued":
-            queued = q.lentgh
-            continue
-        raise ValueError("Queue status is not in ['in_progress', 'queued']")
-
-    # scale_down, scale_up = get_scales(runner_type)
-    _, scale_up = get_scales(runner_type)
-    # With lyfecycle hooks some instances are actually free because some of
-    # them are in 'Terminating:Wait' state
-    effective_capacity = max(
-        asg["DesiredCapacity"],
-        len([ins for ins in asg["Instances"] if ins["HealthStatus"] == "Healthy"]),
-    )
-
-    # How much nodes are free (positive) or need to be added (negative)
-    capacity_reserve = effective_capacity - running - queued
-    stop = False
-    if capacity_reserve <= 0:
-        # This part is about scaling up
-        capacity_deficit = -capacity_reserve
-        # It looks that we are still OK, since no queued jobs exist
-        stop = stop or queued == 0
-        # Are we already at the capacity limits
-        stop = stop or asg["MaxSize"] <= asg["DesiredCapacity"]
-        # Let's calculate a new desired capacity
-        # (capacity_deficit + scale_up - 1) // scale_up : will increase min by 1
-        # if there is any capacity_deficit
-        new_capacity = (
-            asg["DesiredCapacity"] + (capacity_deficit + scale_up - 1) // scale_up
-        )
-        new_capacity = max(new_capacity, asg["MinSize"])
-        new_capacity = min(new_capacity, asg["MaxSize"])
-        # Finally, should the capacity be even changed
-        stop = stop or asg["DesiredCapacity"] == new_capacity
-        if stop:
-            logging.info(
-                "Do not increase ASG %s capacity, current capacity=%s, effective "
-                "capacity=%s, maximum capacity=%s, running jobs=%s, queue size=%s",
-                asg["AutoScalingGroupName"],
-                asg["DesiredCapacity"],
-                effective_capacity,
-                asg["MaxSize"],
-                running,
-                queued,
-            )
-            return
-
-        logging.info(
-            "The ASG %s capacity will be increased to %s, current capacity=%s, "
-            "effective capacity=%s, maximum capacity=%s, running jobs=%s, queue size=%s",
-            asg["AutoScalingGroupName"],
-            new_capacity,
-            asg["DesiredCapacity"],
-            effective_capacity,
-            asg["MaxSize"],
-            running,
-            queued,
-        )
-        if not dry_run:
-            client.set_desired_capacity(
-                AutoScalingGroupName=asg["AutoScalingGroupName"],
-                DesiredCapacity=new_capacity,
-            )
-        return
-
-    # FIXME: try decreasing capacity from runners that finished their jobs and have no job assigned
-    #   IMPORTANT: Runner init script must be of version that supports ASG decrease
-    # # Now we will calculate if we need to scale down
-    # stop = stop or asg["DesiredCapacity"] == asg["MinSize"]
-    # new_capacity = asg["DesiredCapacity"] - (capacity_reserve // scale_down)
-    # new_capacity = max(new_capacity, asg["MinSize"])
-    # new_capacity = min(new_capacity, asg["MaxSize"])
-    # stop = stop or asg["DesiredCapacity"] == new_capacity
-    # if stop:
-    #     logging.info(
-    #         "Do not decrease ASG %s capacity, current capacity=%s, effective "
-    #         "capacity=%s, minimum capacity=%s, running jobs=%s, queue size=%s",
-    #         asg["AutoScalingGroupName"],
-    #         asg["DesiredCapacity"],
-    #         effective_capacity,
-    #         asg["MinSize"],
-    #         running,
-    #         queued,
-    #     )
-    #     return
-    #
-    # logging.info(
-    #     "The ASG %s capacity will be decreased to %s, current capacity=%s, effective "
-    #     "capacity=%s, minimum capacity=%s, running jobs=%s, queue size=%s",
-    #     asg["AutoScalingGroupName"],
-    #     new_capacity,
-    #     asg["DesiredCapacity"],
-    #     effective_capacity,
-    #     asg["MinSize"],
-    #     running,
-    #     queued,
-    # )
-    # if not dry_run:
-    #     client.set_desired_capacity(
-    #         AutoScalingGroupName=asg["AutoScalingGroupName"],
-    #         DesiredCapacity=new_capacity,
-    #     )
-
-
-def main(dry_run: bool = True) -> None:
-    logging.getLogger().setLevel(logging.INFO)
-    asg_client = boto3.client("autoscaling")
-    try:
-        global CH_CLIENT
-        CH_CLIENT = CH_CLIENT or ClickHouseHelper(
-            get_parameter_from_ssm("clickhouse-test-stat-url"), "play"
-        )
-        queues = CH_CLIENT.select_json_each_row("default", QUEUE_QUERY)
-    except CHException as ex:
-        logging.exception(
-            "Got an exception on insert, tryuing to update the client "
-            "credentials and repeat",
-            exc_info=ex,
-        )
-        CH_CLIENT = ClickHouseHelper(
-            get_parameter_from_ssm("clickhouse-test-stat-url"), "play"
-        )
-        queues = CH_CLIENT.select_json_each_row("default", QUEUE_QUERY)
-
-    logging.info("Received queue data:\n%s", pformat(queues, width=120))
-    for runner_type in RUNNER_TYPE_LABELS:
-        runner_queues = [
-            Queue(queue["status"], queue["length"], runner_type)
-            for queue in queues
-            if runner_type in queue["labels"]
-        ]
-        runner_queues = runner_queues or [Queue("in_progress", 0, runner_type)]
-        set_capacity(runner_type, runner_queues, asg_client, dry_run)
-
-
-def handler(event: dict, context: Any) -> None:
-    _ = event
-    _ = context
-    return main(False)
diff --git a/tests/ci/autoscale_runners_lambda/build_and_deploy_archive.sh b/tests/ci/autoscale_runners_lambda/build_and_deploy_archive.sh
deleted file mode 120000
index 96ba3fa024e..00000000000
--- a/tests/ci/autoscale_runners_lambda/build_and_deploy_archive.sh
+++ /dev/null
@@ -1 +0,0 @@
-../team_keys_lambda/build_and_deploy_archive.sh
\ No newline at end of file
diff --git a/tests/ci/autoscale_runners_lambda/lambda_shared b/tests/ci/autoscale_runners_lambda/lambda_shared
deleted file mode 120000
index ba86e090f6c..00000000000
--- a/tests/ci/autoscale_runners_lambda/lambda_shared
+++ /dev/null
@@ -1 +0,0 @@
-../lambda_shared_package/lambda_shared
\ No newline at end of file
diff --git a/tests/ci/autoscale_runners_lambda/requirements.txt b/tests/ci/autoscale_runners_lambda/requirements.txt
deleted file mode 100644
index 098e04a9798..00000000000
--- a/tests/ci/autoscale_runners_lambda/requirements.txt
+++ /dev/null
@@ -1 +0,0 @@
-../lambda_shared_package
diff --git a/tests/ci/autoscale_runners_lambda/test_autoscale.py b/tests/ci/autoscale_runners_lambda/test_autoscale.py
deleted file mode 100644
index d1a1f9b358f..00000000000
--- a/tests/ci/autoscale_runners_lambda/test_autoscale.py
+++ /dev/null
@@ -1,196 +0,0 @@
-#!/usr/bin/env python
-
-import unittest
-from dataclasses import dataclass
-from typing import Any, List
-
-from app import Queue, set_capacity
-
-
-@dataclass
-class TestCase:
-    name: str
-    min_size: int
-    desired_capacity: int
-    max_size: int
-    queues: List[Queue]
-    expected_capacity: int
-
-
-class TestSetCapacity(unittest.TestCase):
-    class FakeClient:
-        def __init__(self):
-            self._expected_data = {}  # type: dict
-            self._expected_capacity = -1
-
-        @property
-        def expected_data(self) -> dict:
-            """a one-time property"""
-            data, self._expected_data = self._expected_data, {}
-            return data
-
-        @expected_data.setter
-        def expected_data(self, value: dict) -> None:
-            self._expected_data = value
-
-        @property
-        def expected_capacity(self) -> int:
-            """a one-time property"""
-            capacity, self._expected_capacity = self._expected_capacity, -1
-            return capacity
-
-        def describe_auto_scaling_groups(self, **kwargs: Any) -> dict:
-            _ = kwargs
-            return self.expected_data
-
-        def set_desired_capacity(self, **kwargs: Any) -> None:
-            self._expected_capacity = kwargs["DesiredCapacity"]
-
-        def data_helper(
-            self, name: str, min_size: int, desired_capacity: int, max_size: int
-        ) -> None:
-            self.expected_data = {
-                "AutoScalingGroups": [
-                    {
-                        "AutoScalingGroupName": name,
-                        "DesiredCapacity": desired_capacity,
-                        "MinSize": min_size,
-                        "MaxSize": max_size,
-                        "Instances": [],  # necessary for ins["HealthStatus"] check
-                    }
-                ]
-            }
-
-    def setUp(self):
-        self.client = self.FakeClient()
-
-    def test_normal_cases(self):
-        test_cases = (
-            # Do not change capacity
-            TestCase("noqueue", 1, 13, 20, [Queue("in_progress", 155, "noqueue")], -1),
-            TestCase("reserve", 1, 13, 20, [Queue("queued", 13, "reserve")], -1),
-            # Increase capacity
-            TestCase(
-                "increase-always",
-                1,
-                13,
-                20,
-                [Queue("queued", 14, "increase-always")],
-                14,
-            ),
-            TestCase("increase-1", 1, 13, 20, [Queue("queued", 23, "increase-1")], 17),
-            TestCase(
-                "style-checker", 1, 13, 20, [Queue("queued", 19, "style-checker")], 19
-            ),
-            TestCase("increase-2", 1, 13, 20, [Queue("queued", 18, "increase-2")], 15),
-            TestCase("increase-3", 1, 13, 20, [Queue("queued", 183, "increase-3")], 20),
-            TestCase(
-                "increase-w/o reserve",
-                1,
-                13,
-                20,
-                [
-                    Queue("in_progress", 11, "increase-w/o reserve"),
-                    Queue("queued", 12, "increase-w/o reserve"),
-                ],
-                17,
-            ),
-            TestCase("lower-min", 10, 5, 20, [Queue("queued", 5, "lower-min")], 10),
-            # Decrease capacity
-            # FIXME: Tests changed for lambda that can only scale up
-            # TestCase("w/reserve", 1, 13, 20, [Queue("queued", 5, "w/reserve")], 5),
-            TestCase("w/reserve", 1, 13, 20, [Queue("queued", 5, "w/reserve")], -1),
-            # TestCase(
-            #     "style-checker", 1, 13, 20, [Queue("queued", 5, "style-checker")], 5
-            # ),
-            TestCase(
-                "style-checker", 1, 13, 20, [Queue("queued", 5, "style-checker")], -1
-            ),
-            # TestCase("w/reserve", 1, 23, 20, [Queue("queued", 17, "w/reserve")], 17),
-            TestCase("w/reserve", 1, 23, 20, [Queue("queued", 17, "w/reserve")], -1),
-            # TestCase("decrease", 1, 13, 20, [Queue("in_progress", 3, "decrease")], 3),
-            TestCase("decrease", 1, 13, 20, [Queue("in_progress", 3, "decrease")], -1),
-            # TestCase(
-            #     "style-checker",
-            #     1,
-            #     13,
-            #     20,
-            #     [Queue("in_progress", 5, "style-checker")],
-            #     5,
-            # ),
-            TestCase(
-                "style-checker",
-                1,
-                13,
-                20,
-                [Queue("in_progress", 5, "style-checker")],
-                -1,
-            ),
-        )
-        for t in test_cases:
-            self.client.data_helper(t.name, t.min_size, t.desired_capacity, t.max_size)
-            set_capacity(t.name, t.queues, self.client, False)
-            self.assertEqual(t.expected_capacity, self.client.expected_capacity, t.name)
-
-    def test_effective_capacity(self):
-        """Normal cases test increasing w/o considering
-        effective_capacity much lower than DesiredCapacity"""
-        test_cases = (
-            TestCase(
-                "desired-overwritten",
-                1,
-                20,  # DesiredCapacity, overwritten by effective_capacity
-                50,
-                [
-                    Queue("in_progress", 30, "desired-overwritten"),
-                    Queue("queued", 60, "desired-overwritten"),
-                ],
-                40,
-            ),
-        )
-        for t in test_cases:
-            self.client.data_helper(t.name, t.min_size, t.desired_capacity, t.max_size)
-            # we test that effective_capacity is 30 (a half of 60)
-            data_with_instances = self.client.expected_data
-            data_with_instances["AutoScalingGroups"][0]["Instances"] = [
-                {"HealthStatus": "Healthy" if i % 2 else "Unhealthy"} for i in range(60)
-            ]
-            self.client.expected_data = data_with_instances
-            set_capacity(t.name, t.queues, self.client, False)
-            self.assertEqual(t.expected_capacity, self.client.expected_capacity, t.name)
-
-    def test_exceptions(self):
-        test_cases = (
-            (
-                TestCase(
-                    "different names",
-                    1,
-                    1,
-                    1,
-                    [Queue("queued", 5, "another name")],
-                    -1,
-                ),
-                AssertionError,
-            ),
-            (TestCase("wrong queue len", 1, 1, 1, [], -1), AssertionError),
-            (
-                TestCase(
-                    "wrong queue", 1, 1, 1, [Queue("wrong", 1, "wrong queue")], -1  # type: ignore
-                ),
-                ValueError,
-            ),
-        )
-        for t, error in test_cases:
-            with self.assertRaises(error):
-                self.client.data_helper(
-                    t.name, t.min_size, t.desired_capacity, t.max_size
-                )
-                set_capacity(t.name, t.queues, self.client, False)
-
-        with self.assertRaises(AssertionError):
-            self.client.expected_data = {"AutoScalingGroups": [1, 2]}
-            set_capacity(
-                "wrong number of ASGs",
-                [Queue("queued", 1, "wrong number of ASGs")],
-                self.client,
-            )
diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py
index 39f34ed9ccf..77d91c8400b 100644
--- a/tests/ci/build_check.py
+++ b/tests/ci/build_check.py
@@ -12,7 +12,6 @@ import docker_images_helper
 from ci_config import CI
 from env_helper import REPO_COPY, S3_BUILDS_BUCKET, TEMP_PATH
 from git_helper import Git
-from lambda_shared_package.lambda_shared.pr import Labels
 from pr_info import PRInfo
 from report import FAILURE, SUCCESS, JobReport, StatusType
 from stopwatch import Stopwatch
@@ -108,7 +107,9 @@ def build_clickhouse(
 
 
 def is_release_pr(pr_info: PRInfo) -> bool:
-    return Labels.RELEASE in pr_info.labels or Labels.RELEASE_LTS in pr_info.labels
+    return (
+        CI.Labels.RELEASE in pr_info.labels or CI.Labels.RELEASE_LTS in pr_info.labels
+    )
 
 
 def get_release_or_pr(pr_info: PRInfo, version: ClickHouseVersion) -> Tuple[str, str]:
diff --git a/tests/ci/cancel_and_rerun_workflow_lambda/app.py b/tests/ci/cancel_and_rerun_workflow_lambda/app.py
deleted file mode 100644
index 578ade5c8a0..00000000000
--- a/tests/ci/cancel_and_rerun_workflow_lambda/app.py
+++ /dev/null
@@ -1,376 +0,0 @@
-#!/usr/bin/env python3
-
-import json
-import time
-from base64 import b64decode
-from collections import namedtuple
-from queue import Queue
-from threading import Thread
-from typing import Any, Dict, List, Optional
-
-import requests
-from lambda_shared.pr import Labels
-from lambda_shared.token import get_cached_access_token
-
-NEED_RERUN_OR_CANCELL_WORKFLOWS = {
-    "BackportPR",
-    "DocsCheck",
-    "MasterCI",
-    "PullRequestCI",
-}
-
-MAX_RETRY = 5
-
-DEBUG_INFO = {}  # type: Dict[str, Any]
-
-
-class Worker(Thread):
-    def __init__(
-        self, request_queue: Queue, token: str, ignore_exception: bool = False
-    ):
-        Thread.__init__(self)
-        self.queue = request_queue
-        self.token = token
-        self.ignore_exception = ignore_exception
-        self.response = {}  # type: Dict
-
-    def run(self):
-        m = self.queue.get()
-        try:
-            self.response = _exec_get_with_retry(m, self.token)
-        except Exception as e:
-            if not self.ignore_exception:
-                raise
-            print(f"Exception occured, still continue: {e}")
-        self.queue.task_done()
-
-
-def _exec_get_with_retry(url: str, token: str) -> dict:
-    headers = {"Authorization": f"token {token}"}
-    e = Exception()
-    for i in range(MAX_RETRY):
-        try:
-            response = requests.get(url, headers=headers, timeout=30)
-            response.raise_for_status()
-            return response.json()  # type: ignore
-        except Exception as ex:
-            print("Got exception executing request", ex)
-            e = ex
-            time.sleep(i + 1)
-
-    raise requests.HTTPError("Cannot execute GET request with retries") from e
-
-
-WorkflowDescription = namedtuple(
-    "WorkflowDescription",
-    [
-        "url",
-        "run_id",
-        "name",
-        "head_sha",
-        "status",
-        "rerun_url",
-        "cancel_url",
-        "conclusion",
-    ],
-)
-
-
-def get_workflows_description_for_pull_request(
-    pull_request_event: dict, token: str
-) -> List[WorkflowDescription]:
-    head_repo = pull_request_event["head"]["repo"]["full_name"]
-    head_branch = pull_request_event["head"]["ref"]
-    print("PR", pull_request_event["number"], "has head ref", head_branch)
-
-    workflows_data = []
-    repo_url = pull_request_event["base"]["repo"]["url"]
-    request_url = f"{repo_url}/actions/runs?per_page=100"
-    # Get all workflows for the current branch
-    for i in range(1, 11):
-        workflows = _exec_get_with_retry(
-            f"{request_url}&event=pull_request&branch={head_branch}&page={i}", token
-        )
-        if not workflows["workflow_runs"]:
-            break
-        workflows_data += workflows["workflow_runs"]
-        if i == 10:
-            print("Too many workflows found")
-
-    if not workflows_data:
-        print("No workflows found by filter")
-        return []
-
-    print(f"Total workflows for the branch {head_branch} found: {len(workflows_data)}")
-
-    DEBUG_INFO["workflows"] = []
-    workflow_descriptions = []
-    for workflow in workflows_data:
-        # Some time workflow["head_repository"]["full_name"] is None
-        if workflow["head_repository"] is None:
-            continue
-        DEBUG_INFO["workflows"].append(
-            {
-                "full_name": workflow["head_repository"]["full_name"],
-                "name": workflow["name"],
-                "branch": workflow["head_branch"],
-            }
-        )
-        # unfortunately we cannot filter workflows from forks in request to API
-        # so doing it manually
-        if (
-            workflow["head_repository"]["full_name"] == head_repo
-            and workflow["name"] in NEED_RERUN_OR_CANCELL_WORKFLOWS
-        ):
-            workflow_descriptions.append(
-                WorkflowDescription(
-                    url=workflow["url"],
-                    run_id=workflow["id"],
-                    name=workflow["name"],
-                    head_sha=workflow["head_sha"],
-                    status=workflow["status"],
-                    rerun_url=workflow["rerun_url"],
-                    cancel_url=workflow["cancel_url"],
-                    conclusion=workflow["conclusion"],
-                )
-            )
-
-    return workflow_descriptions
-
-
-def get_workflow_description_fallback(
-    pull_request_event: dict, token: str
-) -> List[WorkflowDescription]:
-    head_repo = pull_request_event["head"]["repo"]["full_name"]
-    head_branch = pull_request_event["head"]["ref"]
-    print("Get last 500 workflows from API to search related there")
-    # Fallback for a case of an already deleted branch and no workflows received
-    repo_url = pull_request_event["base"]["repo"]["url"]
-    request_url = f"{repo_url}/actions/runs?per_page=100"
-    q = Queue()  # type: Queue
-    workers = []
-    workflows_data = []
-    i = 1
-    for i in range(1, 6):
-        q.put(f"{request_url}&page={i}")
-        worker = Worker(q, token, True)
-        worker.start()
-        workers.append(worker)
-
-    for worker in workers:
-        worker.join()
-        if not worker.response:
-            # We ignore get errors, so response can be empty
-            continue
-        # Prefilter workflows
-        workflows_data += [
-            wf
-            for wf in worker.response["workflow_runs"]
-            if wf["head_repository"] is not None
-            and wf["head_repository"]["full_name"] == head_repo
-            and wf["head_branch"] == head_branch
-            and wf["name"] in NEED_RERUN_OR_CANCELL_WORKFLOWS
-        ]
-
-    print(f"Total workflows in last 500 actions matches: {len(workflows_data)}")
-
-    DEBUG_INFO["workflows"] = [
-        {
-            "full_name": wf["head_repository"]["full_name"],
-            "name": wf["name"],
-            "branch": wf["head_branch"],
-        }
-        for wf in workflows_data
-    ]
-
-    workflow_descriptions = [
-        WorkflowDescription(
-            url=wf["url"],
-            run_id=wf["id"],
-            name=wf["name"],
-            head_sha=wf["head_sha"],
-            status=wf["status"],
-            rerun_url=wf["rerun_url"],
-            cancel_url=wf["cancel_url"],
-            conclusion=wf["conclusion"],
-        )
-        for wf in workflows_data
-    ]
-
-    return workflow_descriptions
-
-
-def get_workflow_description(workflow_url: str, token: str) -> WorkflowDescription:
-    workflow = _exec_get_with_retry(workflow_url, token)
-    return WorkflowDescription(
-        url=workflow["url"],
-        run_id=workflow["id"],
-        name=workflow["name"],
-        head_sha=workflow["head_sha"],
-        status=workflow["status"],
-        rerun_url=workflow["rerun_url"],
-        cancel_url=workflow["cancel_url"],
-        conclusion=workflow["conclusion"],
-    )
-
-
-def _exec_post_with_retry(url: str, token: str, json: Optional[Any] = None) -> Any:
-    headers = {"Authorization": f"token {token}"}
-    e = Exception()
-    for i in range(MAX_RETRY):
-        try:
-            response = requests.post(url, headers=headers, json=json, timeout=30)
-            response.raise_for_status()
-            return response.json()
-        except Exception as ex:
-            print("Got exception executing request", ex)
-            e = ex
-            time.sleep(i + 1)
-
-    raise requests.HTTPError("Cannot execute POST request with retry") from e
-
-
-def exec_workflow_url(urls_to_post, token):
-    for url in urls_to_post:
-        print("Post for workflow workflow using url", url)
-        _exec_post_with_retry(url, token)
-        print("Workflow post finished")
-
-
-def main(event):
-    token = get_cached_access_token()
-    DEBUG_INFO["event"] = event
-    if event["isBase64Encoded"]:
-        event_data = json.loads(b64decode(event["body"]))
-    else:
-        event_data = json.loads(event["body"])
-
-    print("Got event for PR", event_data["number"])
-    action = event_data["action"]
-    print("Got action", event_data["action"])
-    pull_request = event_data["pull_request"]
-    label = ""
-    if action == "labeled":
-        label = event_data["label"]["name"]
-        print("Added label:", label)
-
-    print("PR has labels", {label["name"] for label in pull_request["labels"]})
-    if action == "opened" or (
-        action == "labeled" and pull_request["created_at"] == pull_request["updated_at"]
-    ):
-        print("Freshly opened PR, nothing to do")
-        return
-
-    if action == "closed" or label == Labels.DO_NOT_TEST:
-        print("PR merged/closed or manually labeled 'do not test', will kill workflows")
-        workflow_descriptions = get_workflows_description_for_pull_request(
-            pull_request, token
-        )
-        workflow_descriptions = (
-            workflow_descriptions
-            or get_workflow_description_fallback(pull_request, token)
-        )
-        urls_to_cancel = []
-        for workflow_description in workflow_descriptions:
-            if (
-                workflow_description.status != "completed"
-                and workflow_description.conclusion != "cancelled"
-            ):
-                urls_to_cancel.append(workflow_description.cancel_url)
-        print(f"Found {len(urls_to_cancel)} workflows to cancel")
-        exec_workflow_url(urls_to_cancel, token)
-        return
-
-    if label == Labels.CAN_BE_TESTED:
-        print("PR marked with can be tested label, rerun workflow")
-        workflow_descriptions = get_workflows_description_for_pull_request(
-            pull_request, token
-        )
-        workflow_descriptions = (
-            workflow_descriptions
-            or get_workflow_description_fallback(pull_request, token)
-        )
-        if not workflow_descriptions:
-            print("Not found any workflows")
-            return
-
-        workflow_descriptions.sort(key=lambda x: x.run_id)  # type: ignore
-        most_recent_workflow = workflow_descriptions[-1]
-        print("Latest workflow", most_recent_workflow)
-        if (
-            most_recent_workflow.status != "completed"
-            and most_recent_workflow.conclusion != "cancelled"
-        ):
-            print("Latest workflow is not completed, cancelling")
-            exec_workflow_url([most_recent_workflow.cancel_url], token)
-            print("Cancelled")
-
-        for _ in range(45):
-            # If the number of retries is changed: tune the lambda limits accordingly
-            latest_workflow_desc = get_workflow_description(
-                most_recent_workflow.url, token
-            )
-            print("Checking latest workflow", latest_workflow_desc)
-            if latest_workflow_desc.status in ("completed", "cancelled"):
-                print("Finally latest workflow done, going to rerun")
-                exec_workflow_url([most_recent_workflow.rerun_url], token)
-                print("Rerun finished, exiting")
-                break
-            print("Still have strange status")
-            time.sleep(3)
-        return
-
-    if action == "edited":
-        print("PR is edited - do nothing")
-        # error, _ = check_pr_description(
-        #     pull_request["body"], pull_request["base"]["repo"]["full_name"]
-        # )
-        # if error:
-        #     print(
-        #         f"The PR's body is wrong, is going to comment it. The error is: {error}"
-        #     )
-        #     post_json = {
-        #         "body": "This is an automatic comment. The PR descriptions does not "
-        #         f"match the [template]({pull_request['base']['repo']['html_url']}/"
-        #         "blob/master/.github/PULL_REQUEST_TEMPLATE.md?plain=1).\n\n"
-        #         f"Please, edit it accordingly.\n\nThe error is: {error}"
-        #     }
-        #     _exec_post_with_retry(pull_request["comments_url"], token, json=post_json)
-        return
-
-    if action == "synchronize":
-        print("PR is synchronized, going to stop old actions")
-        workflow_descriptions = get_workflows_description_for_pull_request(
-            pull_request, token
-        )
-        workflow_descriptions = (
-            workflow_descriptions
-            or get_workflow_description_fallback(pull_request, token)
-        )
-        urls_to_cancel = []
-        for workflow_description in workflow_descriptions:
-            if (
-                workflow_description.status != "completed"
-                and workflow_description.conclusion != "cancelled"
-                and workflow_description.head_sha != pull_request["head"]["sha"]
-            ):
-                urls_to_cancel.append(workflow_description.cancel_url)
-        print(f"Found {len(urls_to_cancel)} workflows to cancel")
-        exec_workflow_url(urls_to_cancel, token)
-        return
-
-    print("Nothing to do")
-
-
-def handler(event, _):
-    try:
-        main(event)
-
-        return {
-            "statusCode": 200,
-            "headers": {"Content-Type": "application/json"},
-            "body": '{"status": "OK"}',
-        }
-    finally:
-        for name, value in DEBUG_INFO.items():
-            print(f"Value of {name}: ", value)
diff --git a/tests/ci/cancel_and_rerun_workflow_lambda/build_and_deploy_archive.sh b/tests/ci/cancel_and_rerun_workflow_lambda/build_and_deploy_archive.sh
deleted file mode 120000
index 96ba3fa024e..00000000000
--- a/tests/ci/cancel_and_rerun_workflow_lambda/build_and_deploy_archive.sh
+++ /dev/null
@@ -1 +0,0 @@
-../team_keys_lambda/build_and_deploy_archive.sh
\ No newline at end of file
diff --git a/tests/ci/cancel_and_rerun_workflow_lambda/lambda_shared b/tests/ci/cancel_and_rerun_workflow_lambda/lambda_shared
deleted file mode 120000
index ba86e090f6c..00000000000
--- a/tests/ci/cancel_and_rerun_workflow_lambda/lambda_shared
+++ /dev/null
@@ -1 +0,0 @@
-../lambda_shared_package/lambda_shared
\ No newline at end of file
diff --git a/tests/ci/cancel_and_rerun_workflow_lambda/requirements.txt b/tests/ci/cancel_and_rerun_workflow_lambda/requirements.txt
deleted file mode 100644
index 4cb3fba0f7b..00000000000
--- a/tests/ci/cancel_and_rerun_workflow_lambda/requirements.txt
+++ /dev/null
@@ -1 +0,0 @@
-../lambda_shared_package[token]
diff --git a/tests/ci/cherry_pick.py b/tests/ci/cherry_pick.py
index 459be12ada0..a7fc6d02853 100644
--- a/tests/ci/cherry_pick.py
+++ b/tests/ci/cherry_pick.py
@@ -38,7 +38,7 @@ from env_helper import TEMP_PATH
 from get_robot_token import get_best_robot_token
 from git_helper import GIT_PREFIX, git_runner, is_shallow
 from github_helper import GitHub, PullRequest, PullRequests, Repository
-from lambda_shared_package.lambda_shared.pr import Labels
+from ci_config import Labels
 from ssh import SSHKey
 
 
diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py
index f5e39f343b2..98c7e99a495 100644
--- a/tests/ci/ci_config.py
+++ b/tests/ci/ci_config.py
@@ -32,6 +32,9 @@ class CI:
     from ci_definitions import MQ_JOBS as MQ_JOBS
     from ci_definitions import WorkflowStages as WorkflowStages
     from ci_definitions import Runners as Runners
+    from ci_definitions import Labels as Labels
+    from ci_definitions import TRUSTED_CONTRIBUTORS as TRUSTED_CONTRIBUTORS
+    from ci_utils import CATEGORY_TO_LABEL as CATEGORY_TO_LABEL
 
     # Jobs that run for doc related updates
     _DOCS_CHECK_JOBS = [JobNames.DOCS_CHECK, JobNames.STYLE_CHECK]
diff --git a/tests/ci/ci_definitions.py b/tests/ci/ci_definitions.py
index acd9b7fa904..d41a621bc2e 100644
--- a/tests/ci/ci_definitions.py
+++ b/tests/ci/ci_definitions.py
@@ -7,6 +7,53 @@ from ci_utils import WithIter
 from integration_test_images import IMAGES
 
 
+class Labels:
+    PR_BUGFIX = "pr-bugfix"
+    PR_CRITICAL_BUGFIX = "pr-critical-bugfix"
+    CAN_BE_TESTED = "can be tested"
+    DO_NOT_TEST = "do not test"
+    MUST_BACKPORT = "pr-must-backport"
+    MUST_BACKPORT_CLOUD = "pr-must-backport-cloud"
+    JEPSEN_TEST = "jepsen-test"
+    SKIP_MERGEABLE_CHECK = "skip mergeable check"
+    PR_BACKPORT = "pr-backport"
+    PR_BACKPORTS_CREATED = "pr-backports-created"
+    PR_BACKPORTS_CREATED_CLOUD = "pr-backports-created-cloud"
+    PR_CHERRYPICK = "pr-cherrypick"
+    PR_CI = "pr-ci"
+    PR_FEATURE = "pr-feature"
+    PR_SYNCED_TO_CLOUD = "pr-synced-to-cloud"
+    PR_SYNC_UPSTREAM = "pr-sync-upstream"
+    RELEASE = "release"
+    RELEASE_LTS = "release-lts"
+    SUBMODULE_CHANGED = "submodule changed"
+
+    # automatic backport for critical bug fixes
+    AUTO_BACKPORT = {"pr-critical-bugfix"}
+
+
+TRUSTED_CONTRIBUTORS = {
+    e.lower()
+    for e in [
+        "amosbird",
+        "azat",  # SEMRush
+        "bharatnc",  # Many contributions.
+        "cwurm",  # ClickHouse, Inc
+        "den-crane",  # Documentation contributor
+        "ildus",  # adjust, ex-pgpro
+        "nvartolomei",  # Seasoned contributor, CloudFlare
+        "taiyang-li",
+        "ucasFL",  # Amos Bird's friend
+        "thomoco",  # ClickHouse, Inc
+        "tonickkozlov",  # Cloudflare
+        "tylerhannan",  # ClickHouse, Inc
+        "tsolodov",  # ClickHouse, Inc
+        "justindeguzman",  # ClickHouse, Inc
+        "XuJia0210",  # ClickHouse, Inc
+    ]
+}
+
+
 class WorkflowStages(metaclass=WithIter):
     """
     Stages of GitHUb actions workflow
diff --git a/tests/ci/ci_runners_metrics_lambda/app.py b/tests/ci/ci_runners_metrics_lambda/app.py
deleted file mode 100644
index 47161215a97..00000000000
--- a/tests/ci/ci_runners_metrics_lambda/app.py
+++ /dev/null
@@ -1,164 +0,0 @@
-#!/usr/bin/env python3
-"""
-Lambda function to:
-    - calculate number of running runners
-    - cleaning dead runners from GitHub
-    - terminating stale lost runners in EC2
-"""
-
-import argparse
-import sys
-from typing import Dict
-
-import boto3  # type: ignore
-from lambda_shared import RUNNER_TYPE_LABELS, RunnerDescriptions, list_runners
-from lambda_shared.token import (
-    get_access_token_by_key_app,
-    get_cached_access_token,
-    get_key_and_app_from_aws,
-)
-
-UNIVERSAL_LABEL = "universal"
-
-
-def handler(event, context):
-    _ = event
-    _ = context
-    main(get_cached_access_token(), True)
-
-
-def group_runners_by_tag(
-    listed_runners: RunnerDescriptions,
-) -> Dict[str, RunnerDescriptions]:
-    result = {}  # type: Dict[str, RunnerDescriptions]
-
-    def add_to_result(tag, runner):
-        if tag not in result:
-            result[tag] = []
-        result[tag].append(runner)
-
-    for runner in listed_runners:
-        if UNIVERSAL_LABEL in runner.tags:
-            # Do not proceed other labels if UNIVERSAL_LABEL is included
-            add_to_result(UNIVERSAL_LABEL, runner)
-            continue
-
-        for tag in runner.tags:
-            if tag in RUNNER_TYPE_LABELS:
-                add_to_result(tag, runner)
-                break
-        else:
-            add_to_result("unlabeled", runner)
-    return result
-
-
-def push_metrics_to_cloudwatch(
-    listed_runners: RunnerDescriptions, group_name: str
-) -> None:
-    client = boto3.client("cloudwatch")
-    namespace = "RunnersMetrics"
-    metrics_data = []
-    busy_runners = sum(
-        1 for runner in listed_runners if runner.busy and not runner.offline
-    )
-    dimensions = [{"Name": "group", "Value": group_name}]
-    metrics_data.append(
-        {
-            "MetricName": "BusyRunners",
-            "Value": busy_runners,
-            "Unit": "Count",
-            "Dimensions": dimensions,
-        }
-    )
-    total_active_runners = sum(1 for runner in listed_runners if not runner.offline)
-    metrics_data.append(
-        {
-            "MetricName": "ActiveRunners",
-            "Value": total_active_runners,
-            "Unit": "Count",
-            "Dimensions": dimensions,
-        }
-    )
-    total_runners = len(listed_runners)
-    metrics_data.append(
-        {
-            "MetricName": "TotalRunners",
-            "Value": total_runners,
-            "Unit": "Count",
-            "Dimensions": dimensions,
-        }
-    )
-    if total_active_runners == 0:
-        busy_ratio = 100.0
-    else:
-        busy_ratio = busy_runners / total_active_runners * 100
-
-    metrics_data.append(
-        {
-            "MetricName": "BusyRunnersRatio",
-            "Value": busy_ratio,
-            "Unit": "Percent",
-            "Dimensions": dimensions,
-        }
-    )
-
-    client.put_metric_data(Namespace=namespace, MetricData=metrics_data)
-
-
-def main(
-    access_token: str,
-    push_to_cloudwatch: bool,
-) -> None:
-    gh_runners = list_runners(access_token)
-    grouped_runners = group_runners_by_tag(gh_runners)
-    for group, group_runners in grouped_runners.items():
-        if push_to_cloudwatch:
-            print(f"Pushing metrics for group '{group}'")
-            push_metrics_to_cloudwatch(group_runners, group)
-        else:
-            print(group, f"({len(group_runners)})")
-            for runner in group_runners:
-                print("\t", runner)
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Get list of runners and their states")
-    parser.add_argument(
-        "-p", "--private-key-path", help="Path to file with private key"
-    )
-    parser.add_argument("-k", "--private-key", help="Private key")
-    parser.add_argument(
-        "-a", "--app-id", type=int, help="GitHub application ID", required=True
-    )
-    parser.add_argument(
-        "--push-to-cloudwatch",
-        action="store_true",
-        help="Push metrics for active and busy runners to cloudwatch",
-    )
-
-    args = parser.parse_args()
-
-    if not args.private_key_path and not args.private_key:
-        print(
-            "Either --private-key-path or --private-key must be specified",
-            file=sys.stderr,
-        )
-
-    if args.private_key_path and args.private_key:
-        print(
-            "Either --private-key-path or --private-key must be specified",
-            file=sys.stderr,
-        )
-
-    if args.private_key:
-        private_key = args.private_key
-    elif args.private_key_path:
-        with open(args.private_key_path, "r", encoding="utf-8") as key_file:
-            private_key = key_file.read()
-    else:
-        print("Attempt to get key and id from AWS secret manager")
-        private_key, args.app_id = get_key_and_app_from_aws()
-
-    token = get_access_token_by_key_app(private_key, args.app_id)
-
-    main(token, args.push_to_cloudwatch)
diff --git a/tests/ci/ci_runners_metrics_lambda/build_and_deploy_archive.sh b/tests/ci/ci_runners_metrics_lambda/build_and_deploy_archive.sh
deleted file mode 120000
index 96ba3fa024e..00000000000
--- a/tests/ci/ci_runners_metrics_lambda/build_and_deploy_archive.sh
+++ /dev/null
@@ -1 +0,0 @@
-../team_keys_lambda/build_and_deploy_archive.sh
\ No newline at end of file
diff --git a/tests/ci/ci_runners_metrics_lambda/lambda_shared b/tests/ci/ci_runners_metrics_lambda/lambda_shared
deleted file mode 120000
index ba86e090f6c..00000000000
--- a/tests/ci/ci_runners_metrics_lambda/lambda_shared
+++ /dev/null
@@ -1 +0,0 @@
-../lambda_shared_package/lambda_shared
\ No newline at end of file
diff --git a/tests/ci/ci_runners_metrics_lambda/requirements.txt b/tests/ci/ci_runners_metrics_lambda/requirements.txt
deleted file mode 100644
index e2b16067a93..00000000000
--- a/tests/ci/ci_runners_metrics_lambda/requirements.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-../lambda_shared_package
-../lambda_shared_package[token]
diff --git a/tests/ci/ci_utils.py b/tests/ci/ci_utils.py
index 25875e55df6..49f0447b5ca 100644
--- a/tests/ci/ci_utils.py
+++ b/tests/ci/ci_utils.py
@@ -3,7 +3,42 @@ import re
 import subprocess
 from contextlib import contextmanager
 from pathlib import Path
-from typing import Any, Iterator, List, Union, Optional
+from typing import Any, Iterator, List, Union, Optional, Tuple
+
+
+LABEL_CATEGORIES = {
+    "pr-backward-incompatible": ["Backward Incompatible Change"],
+    "pr-bugfix": [
+        "Bug Fix",
+        "Bug Fix (user-visible misbehavior in an official stable release)",
+        "Bug Fix (user-visible misbehaviour in official stable or prestable release)",
+        "Bug Fix (user-visible misbehavior in official stable or prestable release)",
+    ],
+    "pr-critical-bugfix": ["Critical Bug Fix (crash, LOGICAL_ERROR, data loss, RBAC)"],
+    "pr-build": [
+        "Build/Testing/Packaging Improvement",
+        "Build Improvement",
+        "Build/Testing Improvement",
+        "Build",
+        "Packaging Improvement",
+    ],
+    "pr-documentation": [
+        "Documentation (changelog entry is not required)",
+        "Documentation",
+    ],
+    "pr-feature": ["New Feature"],
+    "pr-improvement": ["Improvement"],
+    "pr-not-for-changelog": [
+        "Not for changelog (changelog entry is not required)",
+        "Not for changelog",
+    ],
+    "pr-performance": ["Performance Improvement"],
+    "pr-ci": ["CI Fix or Improvement (changelog entry is not required)"],
+}
+
+CATEGORY_TO_LABEL = {
+    c: lb for lb, categories in LABEL_CATEGORIES.items() for c in categories
+}
 
 
 class WithIter(type):
@@ -109,3 +144,81 @@ class Utils:
     @staticmethod
     def clear_dmesg():
         Shell.run("sudo dmesg --clear ||:")
+
+    @staticmethod
+    def check_pr_description(pr_body: str, repo_name: str) -> Tuple[str, str]:
+        """The function checks the body to being properly formatted according to
+        .github/PULL_REQUEST_TEMPLATE.md, if the first returned string is not empty,
+        then there is an error."""
+        lines = list(map(lambda x: x.strip(), pr_body.split("\n") if pr_body else []))
+        lines = [re.sub(r"\s+", " ", line) for line in lines]
+
+        # Check if body contains "Reverts ClickHouse/ClickHouse#36337"
+        if [
+            True for line in lines if re.match(rf"\AReverts {repo_name}#[\d]+\Z", line)
+        ]:
+            return "", LABEL_CATEGORIES["pr-not-for-changelog"][0]
+
+        category = ""
+        entry = ""
+        description_error = ""
+
+        i = 0
+        while i < len(lines):
+            if re.match(r"(?i)^[#>*_ ]*change\s*log\s*category", lines[i]):
+                i += 1
+                if i >= len(lines):
+                    break
+                # Can have one empty line between header and the category
+                # itself. Filter it out.
+                if not lines[i]:
+                    i += 1
+                    if i >= len(lines):
+                        break
+                category = re.sub(r"^[-*\s]*", "", lines[i])
+                i += 1
+
+                # Should not have more than one category. Require empty line
+                # after the first found category.
+                if i >= len(lines):
+                    break
+                if lines[i]:
+                    second_category = re.sub(r"^[-*\s]*", "", lines[i])
+                    description_error = (
+                        "More than one changelog category specified: "
+                        f"'{category}', '{second_category}'"
+                    )
+                    return description_error, category
+
+            elif re.match(
+                r"(?i)^[#>*_ ]*(short\s*description|change\s*log\s*entry)", lines[i]
+            ):
+                i += 1
+                # Can have one empty line between header and the entry itself.
+                # Filter it out.
+                if i < len(lines) and not lines[i]:
+                    i += 1
+                # All following lines until empty one are the changelog entry.
+                entry_lines = []
+                while i < len(lines) and lines[i]:
+                    entry_lines.append(lines[i])
+                    i += 1
+                entry = " ".join(entry_lines)
+                # Don't accept changelog entries like '...'.
+                entry = re.sub(r"[#>*_.\- ]", "", entry)
+                # Don't accept changelog entries like 'Close #12345'.
+                entry = re.sub(r"^[\w\-\s]{0,10}#?\d{5,6}\.?$", "", entry)
+            else:
+                i += 1
+
+        if not category:
+            description_error = "Changelog category is empty"
+        # Filter out the PR categories that are not for changelog.
+        elif "(changelog entry is not required)" in category:
+            pass  # to not check the rest of the conditions
+        elif category not in CATEGORY_TO_LABEL:
+            description_error, category = f"Category '{category}' is not valid", ""
+        elif not entry:
+            description_error = f"Changelog entry required for category '{category}'"
+
+        return description_error, category
diff --git a/tests/ci/clean_lost_instances_lambda/app.py b/tests/ci/clean_lost_instances_lambda/app.py
deleted file mode 100644
index 4accc14f7ae..00000000000
--- a/tests/ci/clean_lost_instances_lambda/app.py
+++ /dev/null
@@ -1,336 +0,0 @@
-#!/usr/bin/env python3
-"""
-Lambda function to:
-    - calculate number of running runners
-    - cleaning dead runners from GitHub
-    - terminating stale lost runners in EC2
-"""
-
-import argparse
-import sys
-from dataclasses import dataclass
-from datetime import datetime
-from typing import Dict, List
-
-import boto3  # type: ignore
-import requests
-from botocore.exceptions import ClientError  # type: ignore
-from lambda_shared import (
-    RUNNER_TYPE_LABELS,
-    RunnerDescription,
-    RunnerDescriptions,
-    list_runners,
-)
-from lambda_shared.token import (
-    get_access_token_by_key_app,
-    get_cached_access_token,
-    get_key_and_app_from_aws,
-)
-
-UNIVERSAL_LABEL = "universal"
-
-
-@dataclass
-class LostInstance:
-    counter: int
-    seen: datetime
-
-    def set_offline(self) -> None:
-        now = datetime.now()
-        if now.timestamp() <= self.seen.timestamp() + 120:
-            # the instance is offline for more than 2 minutes, so we increase
-            # the counter
-            self.counter += 1
-        else:
-            self.counter = 1
-        self.seen = now
-
-    @property
-    def recently_offline(self) -> bool:
-        """Returns True if the instance has been seen less than 5 minutes ago"""
-        return datetime.now().timestamp() <= self.seen.timestamp() + 300
-
-    @property
-    def stable_offline(self) -> bool:
-        return self.counter >= 3
-
-
-LOST_INSTANCES = {}  # type: Dict["str", LostInstance]
-
-
-def get_dead_runners_in_ec2(runners: RunnerDescriptions) -> RunnerDescriptions:
-    """Returns instances that are offline/dead in EC2, or not found in EC2"""
-    ids = {
-        runner.name: runner
-        for runner in runners
-        # Only `i-deadbead123` are valid names for an instance ID
-        if runner.name.startswith("i-") and runner.offline and not runner.busy
-    }
-    if not ids:
-        return []
-
-    # Delete all offline runners with wrong name
-    result_to_delete = [
-        runner
-        for runner in runners
-        if not ids.get(runner.name) and runner.offline and not runner.busy
-    ]
-
-    client = boto3.client("ec2")
-
-    i = 0
-    inc = 100
-
-    print("Checking ids: ", " ".join(ids.keys()))
-    instances_statuses = []
-    while i < len(ids.keys()):
-        try:
-            instances_statuses.append(
-                client.describe_instance_status(
-                    InstanceIds=list(ids.keys())[i : i + inc]
-                )
-            )
-            # It applied only if all ids exist in EC2
-            i += inc
-        except ClientError as e:
-            # The list of non-existent instances is in the message:
-            #   The instance IDs 'i-069b1c256c06cf4e3, i-0f26430432b044035,
-            #   i-0faa2ff44edbc147e, i-0eccf2514585045ec, i-0ee4ee53e0daa7d4a,
-            #   i-07928f15acd473bad, i-0eaddda81298f9a85' do not exist
-            message = e.response["Error"]["Message"]
-            if message.startswith("The instance IDs '") and message.endswith(
-                "' do not exist"
-            ):
-                non_existent = message[18:-14].split(", ")
-                for n in non_existent:
-                    result_to_delete.append(ids.pop(n))
-            else:
-                raise
-
-    found_instances = set([])
-    print("Response", instances_statuses)
-    for instances_status in instances_statuses:
-        for instance_status in instances_status["InstanceStatuses"]:
-            if instance_status["InstanceState"]["Name"] in ("pending", "running"):
-                found_instances.add(instance_status["InstanceId"])
-
-    print("Found instances", found_instances)
-    for runner in result_to_delete:
-        print("Instance", runner.name, "is not alive, going to remove it")
-    for instance_id, runner in ids.items():
-        if instance_id not in found_instances:
-            print("Instance", instance_id, "is not found in EC2, going to remove it")
-            result_to_delete.append(runner)
-    return result_to_delete
-
-
-def handler(event, context):
-    _ = event
-    _ = context
-    main(get_cached_access_token(), True)
-
-
-def delete_runner(access_token: str, runner: RunnerDescription) -> bool:
-    headers = {
-        "Authorization": f"token {access_token}",
-        "Accept": "application/vnd.github.v3+json",
-    }
-
-    response = requests.delete(
-        f"https://api.github.com/orgs/ClickHouse/actions/runners/{runner.id}",
-        headers=headers,
-        timeout=30,
-    )
-    response.raise_for_status()
-    print(f"Response code deleting {runner.name} is {response.status_code}")
-    return bool(response.status_code == 204)
-
-
-def get_lost_ec2_instances(runners: RunnerDescriptions) -> List[str]:
-    global LOST_INSTANCES
-    now = datetime.now()
-    client = boto3.client("ec2")
-    reservations = client.describe_instances(
-        Filters=[
-            {"Name": "tag-key", "Values": ["github:runner-type"]},
-            {"Name": "instance-state-name", "Values": ["pending", "running"]},
-        ],
-    )["Reservations"]
-    # flatten the reservation into instances
-    instances = [
-        instance
-        for reservation in reservations
-        for instance in reservation["Instances"]
-    ]
-    offline_runner_names = {
-        runner.name for runner in runners if runner.offline and not runner.busy
-    }
-    runner_names = {runner.name for runner in runners}
-
-    def offline_instance(iid: str) -> None:
-        if iid in LOST_INSTANCES:
-            LOST_INSTANCES[iid].set_offline()
-            return
-        LOST_INSTANCES[iid] = LostInstance(1, now)
-
-    for instance in instances:
-        # Do not consider instances started 20 minutes ago as problematic
-        if now.timestamp() - instance["LaunchTime"].timestamp() < 1200:
-            continue
-
-        runner_type = [
-            tag["Value"]
-            for tag in instance["Tags"]
-            if tag["Key"] == "github:runner-type"
-        ][0]
-        # If there's no necessary labels in runner type it's fine
-        if not (UNIVERSAL_LABEL in runner_type or runner_type in RUNNER_TYPE_LABELS):
-            continue
-
-        if instance["InstanceId"] in offline_runner_names:
-            offline_instance(instance["InstanceId"])
-            continue
-
-        if (
-            instance["State"]["Name"] == "running"
-            and not instance["InstanceId"] in runner_names
-        ):
-            offline_instance(instance["InstanceId"])
-
-    instance_ids = [instance["InstanceId"] for instance in instances]
-    # clean out long unseen instances
-    LOST_INSTANCES = {
-        instance_id: stats
-        for instance_id, stats in LOST_INSTANCES.items()
-        if stats.recently_offline and instance_id in instance_ids
-    }
-    print("The remained LOST_INSTANCES: ", LOST_INSTANCES)
-
-    return [
-        instance_id
-        for instance_id, stats in LOST_INSTANCES.items()
-        if stats.stable_offline
-    ]
-
-
-def continue_lifecycle_hooks(delete_offline_runners: bool) -> None:
-    """The function to trigger CONTINUE for instances' lifectycle hooks"""
-    client = boto3.client("ec2")
-    reservations = client.describe_instances(
-        Filters=[
-            {"Name": "tag-key", "Values": ["github:runner-type"]},
-            {"Name": "instance-state-name", "Values": ["shutting-down", "terminated"]},
-        ],
-    )["Reservations"]
-    # flatten the reservation into instances
-    terminated_instances = [
-        instance["InstanceId"]
-        for reservation in reservations
-        for instance in reservation["Instances"]
-    ]
-
-    asg_client = boto3.client("autoscaling")
-    as_groups = asg_client.describe_auto_scaling_groups(
-        Filters=[{"Name": "tag-key", "Values": ["github:runner-type"]}]
-    )["AutoScalingGroups"]
-    for asg in as_groups:
-        lifecycle_hooks = [
-            lch
-            for lch in asg_client.describe_lifecycle_hooks(
-                AutoScalingGroupName=asg["AutoScalingGroupName"]
-            )["LifecycleHooks"]
-            if lch["LifecycleTransition"] == "autoscaling:EC2_INSTANCE_TERMINATING"
-        ]
-        if not lifecycle_hooks:
-            continue
-        for instance in asg["Instances"]:
-            continue_instance = False
-            if instance["LifecycleState"] == "Terminating:Wait":
-                if instance["HealthStatus"] == "Unhealthy":
-                    print(f"The instance {instance['InstanceId']} is Unhealthy")
-                    continue_instance = True
-                elif (
-                    instance["HealthStatus"] == "Healthy"
-                    and instance["InstanceId"] in terminated_instances
-                ):
-                    print(
-                        f"The instance {instance['InstanceId']} is already terminated"
-                    )
-                    continue_instance = True
-            if continue_instance:
-                if delete_offline_runners:
-                    for lch in lifecycle_hooks:
-                        print(f"Continue lifecycle hook {lch['LifecycleHookName']}")
-                        asg_client.complete_lifecycle_action(
-                            LifecycleHookName=lch["LifecycleHookName"],
-                            AutoScalingGroupName=asg["AutoScalingGroupName"],
-                            LifecycleActionResult="CONTINUE",
-                            InstanceId=instance["InstanceId"],
-                        )
-
-
-def main(
-    access_token: str,
-    delete_offline_runners: bool,
-) -> None:
-    gh_runners = list_runners(access_token)
-
-    dead_runners = get_dead_runners_in_ec2(gh_runners)
-    print("Runners in GH API to terminate: ", [runner.name for runner in dead_runners])
-    if delete_offline_runners and dead_runners:
-        print("Going to delete offline runners")
-        for runner in dead_runners:
-            print("Deleting runner", runner)
-            delete_runner(access_token, runner)
-    elif dead_runners:
-        print("Would delete dead runners: ", dead_runners)
-
-    lost_instances = get_lost_ec2_instances(gh_runners)
-    print("Instances to terminate: ", lost_instances)
-    if delete_offline_runners:
-        if lost_instances:
-            print("Going to terminate lost instances")
-            boto3.client("ec2").terminate_instances(InstanceIds=lost_instances)
-
-    continue_lifecycle_hooks(delete_offline_runners)
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Get list of runners and their states")
-    parser.add_argument(
-        "-p", "--private-key-path", help="Path to file with private key"
-    )
-    parser.add_argument("-k", "--private-key", help="Private key")
-    parser.add_argument(
-        "-a", "--app-id", type=int, help="GitHub application ID", required=True
-    )
-    parser.add_argument(
-        "--delete-offline", action="store_true", help="Remove offline runners"
-    )
-
-    args = parser.parse_args()
-
-    if not args.private_key_path and not args.private_key:
-        print(
-            "Either --private-key-path or --private-key must be specified",
-            file=sys.stderr,
-        )
-
-    if args.private_key_path and args.private_key:
-        print(
-            "Either --private-key-path or --private-key must be specified",
-            file=sys.stderr,
-        )
-
-    if args.private_key:
-        private_key = args.private_key
-    elif args.private_key_path:
-        with open(args.private_key_path, "r", encoding="utf-8") as key_file:
-            private_key = key_file.read()
-    else:
-        print("Attempt to get key and id from AWS secret manager")
-        private_key, args.app_id = get_key_and_app_from_aws()
-
-    token = get_access_token_by_key_app(private_key, args.app_id)
-
-    main(token, args.delete_offline)
diff --git a/tests/ci/clean_lost_instances_lambda/build_and_deploy_archive.sh b/tests/ci/clean_lost_instances_lambda/build_and_deploy_archive.sh
deleted file mode 120000
index 96ba3fa024e..00000000000
--- a/tests/ci/clean_lost_instances_lambda/build_and_deploy_archive.sh
+++ /dev/null
@@ -1 +0,0 @@
-../team_keys_lambda/build_and_deploy_archive.sh
\ No newline at end of file
diff --git a/tests/ci/clean_lost_instances_lambda/lambda_shared b/tests/ci/clean_lost_instances_lambda/lambda_shared
deleted file mode 120000
index ba86e090f6c..00000000000
--- a/tests/ci/clean_lost_instances_lambda/lambda_shared
+++ /dev/null
@@ -1 +0,0 @@
-../lambda_shared_package/lambda_shared
\ No newline at end of file
diff --git a/tests/ci/clean_lost_instances_lambda/requirements.txt b/tests/ci/clean_lost_instances_lambda/requirements.txt
deleted file mode 100644
index e2b16067a93..00000000000
--- a/tests/ci/clean_lost_instances_lambda/requirements.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-../lambda_shared_package
-../lambda_shared_package[token]
diff --git a/tests/ci/lambda_shared_package/.gitignore b/tests/ci/lambda_shared_package/.gitignore
deleted file mode 100644
index 59d52651e06..00000000000
--- a/tests/ci/lambda_shared_package/.gitignore
+++ /dev/null
@@ -1,2 +0,0 @@
-build
-*.egg-info
diff --git a/tests/ci/lambda_shared_package/__init__.py b/tests/ci/lambda_shared_package/__init__.py
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/tests/ci/lambda_shared_package/lambda_shared/__init__.py b/tests/ci/lambda_shared_package/lambda_shared/__init__.py
deleted file mode 100644
index 8b53f9dcb23..00000000000
--- a/tests/ci/lambda_shared_package/lambda_shared/__init__.py
+++ /dev/null
@@ -1,237 +0,0 @@
-"""The shared code and types for all our CI lambdas
-It exists as __init__.py and lambda_shared/__init__.py to work both in local and venv"""
-
-import json
-import logging
-import time
-from collections import namedtuple
-from typing import Any, Dict, Iterable, List, Optional
-
-import boto3  # type: ignore
-import requests
-
-RUNNER_TYPE_LABELS = [
-    "builder",
-    "func-tester",
-    "func-tester-aarch64",
-    "fuzzer-unit-tester",
-    "limited-tester",
-    "stress-tester",
-    "style-checker",
-    "style-checker-aarch64",
-    # private runners
-    "private-builder",
-    "private-clickpipes",
-    "private-func-tester",
-    "private-fuzzer-unit-tester",
-    "private-stress-tester",
-    "private-style-checker",
-]
-
-
-### VENDORING
-def get_parameter_from_ssm(
-    name: str, decrypt: bool = True, client: Optional[Any] = None
-) -> str:
-    if not client:
-        client = boto3.client("ssm", region_name="us-east-1")
-    return client.get_parameter(Name=name, WithDecryption=decrypt)[  # type: ignore
-        "Parameter"
-    ]["Value"]
-
-
-class CHException(Exception):
-    pass
-
-
-class InsertException(CHException):
-    pass
-
-
-class ClickHouseHelper:
-    def __init__(
-        self,
-        url: str,
-        user: Optional[str] = None,
-        password: Optional[str] = None,
-    ):
-        self.url = url
-        self.auth = {}
-        if user:
-            self.auth["X-ClickHouse-User"] = user
-        if password:
-            self.auth["X-ClickHouse-Key"] = password
-
-    @staticmethod
-    def _insert_json_str_info_impl(
-        url: str, auth: Dict[str, str], db: str, table: str, json_str: str
-    ) -> None:
-        params = {
-            "database": db,
-            "query": f"INSERT INTO {table} FORMAT JSONEachRow",
-            "date_time_input_format": "best_effort",
-            "send_logs_level": "warning",
-        }
-
-        for i in range(5):
-            try:
-                response = requests.post(
-                    url, params=params, data=json_str, headers=auth
-                )
-            except Exception as e:
-                error = f"Received exception while sending data to {url} on {i} attempt: {e}"
-                logging.warning(error)
-                continue
-
-            logging.info("Response content '%s'", response.content)
-
-            if response.ok:
-                break
-
-            error = (
-                "Cannot insert data into clickhouse at try "
-                + str(i)
-                + ": HTTP code "
-                + str(response.status_code)
-                + ": '"
-                + str(response.text)
-                + "'"
-            )
-
-            if response.status_code >= 500:
-                # A retriable error
-                time.sleep(1)
-                continue
-
-            logging.info(
-                "Request headers '%s', body '%s'",
-                response.request.headers,
-                response.request.body,
-            )
-
-            raise InsertException(error)
-        else:
-            raise InsertException(error)
-
-    def _insert_json_str_info(self, db: str, table: str, json_str: str) -> None:
-        self._insert_json_str_info_impl(self.url, self.auth, db, table, json_str)
-
-    def insert_event_into(
-        self, db: str, table: str, event: object, safe: bool = True
-    ) -> None:
-        event_str = json.dumps(event)
-        try:
-            self._insert_json_str_info(db, table, event_str)
-        except InsertException as e:
-            logging.error(
-                "Exception happened during inserting data into clickhouse: %s", e
-            )
-            if not safe:
-                raise
-
-    def insert_events_into(
-        self, db: str, table: str, events: Iterable[object], safe: bool = True
-    ) -> None:
-        jsons = []
-        for event in events:
-            jsons.append(json.dumps(event))
-
-        try:
-            self._insert_json_str_info(db, table, ",".join(jsons))
-        except InsertException as e:
-            logging.error(
-                "Exception happened during inserting data into clickhouse: %s", e
-            )
-            if not safe:
-                raise
-
-    def _select_and_get_json_each_row(self, db: str, query: str) -> str:
-        params = {
-            "database": db,
-            "query": query,
-            "default_format": "JSONEachRow",
-        }
-        for i in range(5):
-            response = None
-            try:
-                response = requests.get(self.url, params=params, headers=self.auth)
-                response.raise_for_status()
-                return response.text  # type: ignore
-            except Exception as ex:
-                logging.warning("Cannot fetch data with exception %s", str(ex))
-                if response:
-                    logging.warning("Reponse text %s", response.text)
-                time.sleep(0.1 * i)
-
-        raise CHException("Cannot fetch data from clickhouse")
-
-    def select_json_each_row(self, db: str, query: str) -> List[dict]:
-        text = self._select_and_get_json_each_row(db, query)
-        result = []
-        for line in text.split("\n"):
-            if line:
-                result.append(json.loads(line))
-        return result
-
-
-### Runners
-
-RunnerDescription = namedtuple(
-    "RunnerDescription", ["id", "name", "tags", "offline", "busy"]
-)
-RunnerDescriptions = List[RunnerDescription]
-
-
-def list_runners(access_token: str) -> RunnerDescriptions:
-    headers = {
-        "Authorization": f"token {access_token}",
-        "Accept": "application/vnd.github.v3+json",
-    }
-    per_page = 100
-    response = requests.get(
-        f"https://api.github.com/orgs/ClickHouse/actions/runners?per_page={per_page}",
-        headers=headers,
-    )
-    response.raise_for_status()
-    data = response.json()
-    total_runners = data["total_count"]
-    print("Expected total runners", total_runners)
-    runners = data["runners"]
-
-    # round to 0 for 0, 1 for 1..100, but to 2 for 101..200
-    total_pages = (total_runners - 1) // per_page + 1
-
-    print("Total pages", total_pages)
-    for i in range(2, total_pages + 1):
-        response = requests.get(
-            "https://api.github.com/orgs/ClickHouse/actions/runners"
-            f"?page={i}&per_page={per_page}",
-            headers=headers,
-        )
-        response.raise_for_status()
-        data = response.json()
-        runners += data["runners"]
-
-    print("Total runners", len(runners))
-    result = []
-    for runner in runners:
-        tags = [tag["name"] for tag in runner["labels"]]
-        desc = RunnerDescription(
-            id=runner["id"],
-            name=runner["name"],
-            tags=tags,
-            offline=runner["status"] == "offline",
-            busy=runner["busy"],
-        )
-        result.append(desc)
-
-    return result
-
-
-def cached_value_is_valid(updated_at: float, ttl: float) -> bool:
-    "a common function to identify if cachable value is still valid"
-    if updated_at == 0:
-        return False
-    if time.time() - ttl < updated_at:
-        return True
-    return False
diff --git a/tests/ci/lambda_shared_package/lambda_shared/pr.py b/tests/ci/lambda_shared_package/lambda_shared/pr.py
deleted file mode 100644
index 95130fc2a0f..00000000000
--- a/tests/ci/lambda_shared_package/lambda_shared/pr.py
+++ /dev/null
@@ -1,168 +0,0 @@
-#!/usr/bin/env python
-
-import re
-from typing import Tuple
-
-# Individual trusted contributors who are not in any trusted organization.
-# Can be changed in runtime: we will append users that we learned to be in
-# a trusted org, to save GitHub API calls.
-TRUSTED_CONTRIBUTORS = {
-    e.lower()
-    for e in [
-        "amosbird",
-        "azat",  # SEMRush
-        "bharatnc",  # Many contributions.
-        "cwurm",  # ClickHouse, Inc
-        "den-crane",  # Documentation contributor
-        "ildus",  # adjust, ex-pgpro
-        "nvartolomei",  # Seasoned contributor, CloudFlare
-        "taiyang-li",
-        "ucasFL",  # Amos Bird's friend
-        "thomoco",  # ClickHouse, Inc
-        "tonickkozlov",  # Cloudflare
-        "tylerhannan",  # ClickHouse, Inc
-        "tsolodov",  # ClickHouse, Inc
-        "justindeguzman",  # ClickHouse, Inc
-        "XuJia0210",  # ClickHouse, Inc
-    ]
-}
-
-
-class Labels:
-    PR_BUGFIX = "pr-bugfix"
-    PR_CRITICAL_BUGFIX = "pr-critical-bugfix"
-    CAN_BE_TESTED = "can be tested"
-    DO_NOT_TEST = "do not test"
-    MUST_BACKPORT = "pr-must-backport"
-    MUST_BACKPORT_CLOUD = "pr-must-backport-cloud"
-    JEPSEN_TEST = "jepsen-test"
-    SKIP_MERGEABLE_CHECK = "skip mergeable check"
-    PR_BACKPORT = "pr-backport"
-    PR_BACKPORTS_CREATED = "pr-backports-created"
-    PR_BACKPORTS_CREATED_CLOUD = "pr-backports-created-cloud"
-    PR_CHERRYPICK = "pr-cherrypick"
-    PR_CI = "pr-ci"
-    PR_FEATURE = "pr-feature"
-    PR_SYNCED_TO_CLOUD = "pr-synced-to-cloud"
-    PR_SYNC_UPSTREAM = "pr-sync-upstream"
-    RELEASE = "release"
-    RELEASE_LTS = "release-lts"
-    SUBMODULE_CHANGED = "submodule changed"
-
-    # automatic backport for critical bug fixes
-    AUTO_BACKPORT = {"pr-critical-bugfix"}
-
-
-# Descriptions are used in .github/PULL_REQUEST_TEMPLATE.md, keep comments there
-# updated accordingly
-# The following lists are append only, try to avoid editing them
-# They still could be cleaned out after the decent time though.
-LABEL_CATEGORIES = {
-    "pr-backward-incompatible": ["Backward Incompatible Change"],
-    "pr-bugfix": [
-        "Bug Fix",
-        "Bug Fix (user-visible misbehavior in an official stable release)",
-        "Bug Fix (user-visible misbehaviour in official stable or prestable release)",
-        "Bug Fix (user-visible misbehavior in official stable or prestable release)",
-    ],
-    "pr-critical-bugfix": ["Critical Bug Fix (crash, LOGICAL_ERROR, data loss, RBAC)"],
-    "pr-build": [
-        "Build/Testing/Packaging Improvement",
-        "Build Improvement",
-        "Build/Testing Improvement",
-        "Build",
-        "Packaging Improvement",
-    ],
-    "pr-documentation": [
-        "Documentation (changelog entry is not required)",
-        "Documentation",
-    ],
-    "pr-feature": ["New Feature"],
-    "pr-improvement": ["Improvement"],
-    "pr-not-for-changelog": [
-        "Not for changelog (changelog entry is not required)",
-        "Not for changelog",
-    ],
-    "pr-performance": ["Performance Improvement"],
-    "pr-ci": ["CI Fix or Improvement (changelog entry is not required)"],
-}
-
-CATEGORY_TO_LABEL = {
-    c: lb for lb, categories in LABEL_CATEGORIES.items() for c in categories
-}
-
-
-def check_pr_description(pr_body: str, repo_name: str) -> Tuple[str, str]:
-    """The function checks the body to being properly formatted according to
-    .github/PULL_REQUEST_TEMPLATE.md, if the first returned string is not empty,
-    then there is an error."""
-    lines = list(map(lambda x: x.strip(), pr_body.split("\n") if pr_body else []))
-    lines = [re.sub(r"\s+", " ", line) for line in lines]
-
-    # Check if body contains "Reverts ClickHouse/ClickHouse#36337"
-    if [True for line in lines if re.match(rf"\AReverts {repo_name}#[\d]+\Z", line)]:
-        return "", LABEL_CATEGORIES["pr-not-for-changelog"][0]
-
-    category = ""
-    entry = ""
-    description_error = ""
-
-    i = 0
-    while i < len(lines):
-        if re.match(r"(?i)^[#>*_ ]*change\s*log\s*category", lines[i]):
-            i += 1
-            if i >= len(lines):
-                break
-            # Can have one empty line between header and the category
-            # itself. Filter it out.
-            if not lines[i]:
-                i += 1
-                if i >= len(lines):
-                    break
-            category = re.sub(r"^[-*\s]*", "", lines[i])
-            i += 1
-
-            # Should not have more than one category. Require empty line
-            # after the first found category.
-            if i >= len(lines):
-                break
-            if lines[i]:
-                second_category = re.sub(r"^[-*\s]*", "", lines[i])
-                description_error = (
-                    "More than one changelog category specified: "
-                    f"'{category}', '{second_category}'"
-                )
-                return description_error, category
-
-        elif re.match(
-            r"(?i)^[#>*_ ]*(short\s*description|change\s*log\s*entry)", lines[i]
-        ):
-            i += 1
-            # Can have one empty line between header and the entry itself.
-            # Filter it out.
-            if i < len(lines) and not lines[i]:
-                i += 1
-            # All following lines until empty one are the changelog entry.
-            entry_lines = []
-            while i < len(lines) and lines[i]:
-                entry_lines.append(lines[i])
-                i += 1
-            entry = " ".join(entry_lines)
-            # Don't accept changelog entries like '...'.
-            entry = re.sub(r"[#>*_.\- ]", "", entry)
-            # Don't accept changelog entries like 'Close #12345'.
-            entry = re.sub(r"^[\w\-\s]{0,10}#?\d{5,6}\.?$", "", entry)
-        else:
-            i += 1
-
-    if not category:
-        description_error = "Changelog category is empty"
-    # Filter out the PR categories that are not for changelog.
-    elif "(changelog entry is not required)" in category:
-        pass  # to not check the rest of the conditions
-    elif category not in CATEGORY_TO_LABEL:
-        description_error, category = f"Category '{category}' is not valid", ""
-    elif not entry:
-        description_error = f"Changelog entry required for category '{category}'"
-
-    return description_error, category
diff --git a/tests/ci/lambda_shared_package/lambda_shared/token.py b/tests/ci/lambda_shared_package/lambda_shared/token.py
deleted file mode 100644
index 3fb8f10c0e2..00000000000
--- a/tests/ci/lambda_shared_package/lambda_shared/token.py
+++ /dev/null
@@ -1,95 +0,0 @@
-"""Module to get the token for GitHub"""
-
-from dataclasses import dataclass
-import json
-import time
-from typing import Tuple
-
-import boto3  # type: ignore
-import jwt
-import requests
-
-from . import cached_value_is_valid
-
-
-def get_key_and_app_from_aws() -> Tuple[str, int]:
-    secret_name = "clickhouse_github_secret_key"
-    session = boto3.session.Session()
-    client = session.client(
-        service_name="secretsmanager",
-    )
-    get_secret_value_response = client.get_secret_value(SecretId=secret_name)
-    data = json.loads(get_secret_value_response["SecretString"])
-    return data["clickhouse-app-key"], int(data["clickhouse-app-id"])
-
-
-def get_installation_id(jwt_token: str) -> int:
-    headers = {
-        "Authorization": f"Bearer {jwt_token}",
-        "Accept": "application/vnd.github.v3+json",
-    }
-    response = requests.get("https://api.github.com/app/installations", headers=headers)
-    response.raise_for_status()
-    data = response.json()
-    for installation in data:
-        if installation["account"]["login"] == "ClickHouse":
-            installation_id = installation["id"]
-
-    return installation_id  # type: ignore
-
-
-def get_access_token_by_jwt(jwt_token: str, installation_id: int) -> str:
-    headers = {
-        "Authorization": f"Bearer {jwt_token}",
-        "Accept": "application/vnd.github.v3+json",
-    }
-    response = requests.post(
-        f"https://api.github.com/app/installations/{installation_id}/access_tokens",
-        headers=headers,
-    )
-    response.raise_for_status()
-    data = response.json()
-    return data["token"]  # type: ignore
-
-
-def get_token_from_aws() -> str:
-    private_key, app_id = get_key_and_app_from_aws()
-    return get_access_token_by_key_app(private_key, app_id)
-
-
-def get_access_token_by_key_app(private_key: str, app_id: int) -> str:
-    payload = {
-        "iat": int(time.time()) - 60,
-        "exp": int(time.time()) + (10 * 60),
-        "iss": app_id,
-    }
-
-    # FIXME: apparently should be switched to this so that mypy is happy
-    # jwt_instance = JWT()
-    # encoded_jwt = jwt_instance.encode(payload, private_key, algorithm="RS256")
-    encoded_jwt = jwt.encode(payload, private_key, algorithm="RS256")  # type: ignore
-    installation_id = get_installation_id(encoded_jwt)
-    return get_access_token_by_jwt(encoded_jwt, installation_id)
-
-
-@dataclass
-class CachedToken:
-    time: float
-    value: str
-    updating: bool = False
-
-
-_cached_token = CachedToken(0, "")
-
-
-def get_cached_access_token() -> str:
-    if time.time() - 550 < _cached_token.time or _cached_token.updating:
-        return _cached_token.value
-    # Indicate that the value is updating now, so the cached value can be
-    # used. The first setting and close-to-ttl are not counted as update
-    _cached_token.updating = cached_value_is_valid(_cached_token.time, 590)
-    private_key, app_id = get_key_and_app_from_aws()
-    _cached_token.time = time.time()
-    _cached_token.value = get_access_token_by_key_app(private_key, app_id)
-    _cached_token.updating = False
-    return _cached_token.value
diff --git a/tests/ci/lambda_shared_package/pyproject.toml b/tests/ci/lambda_shared_package/pyproject.toml
deleted file mode 100644
index dff36b89fbb..00000000000
--- a/tests/ci/lambda_shared_package/pyproject.toml
+++ /dev/null
@@ -1,24 +0,0 @@
-[build-system]
-requires = ["setuptools"]
-build-backend = "setuptools.build_meta"
-
-[project]
-name = "lambda_shared"
-version = "0.0.1"
-dependencies = [
-  "requests",
-  "urllib3 < 2"
-]
-
-[project.optional-dependencies]
-token = [
-  "PyJWT",
-  "cryptography",
-]
-dev = [
-  "boto3",
-  "lambda_shared[token]",
-]
-
-[tool.distutils.bdist_wheel]
-universal = true
diff --git a/tests/ci/lambda_shared_package/setup.cfg b/tests/ci/lambda_shared_package/setup.cfg
deleted file mode 100644
index 744280ae41b..00000000000
--- a/tests/ci/lambda_shared_package/setup.cfg
+++ /dev/null
@@ -1,8 +0,0 @@
-### This file exists for clear builds in docker ###
-#   without it the `build` directory wouldn't be  #
-# updated on the fly and will require manual clean #
-[build]
-build_base = /tmp/lambda_shared
-
-[egg_info]
-egg_base = /tmp/
diff --git a/tests/ci/pr_info.py b/tests/ci/pr_info.py
index a411fc4e8f6..9f3b5a586cc 100644
--- a/tests/ci/pr_info.py
+++ b/tests/ci/pr_info.py
@@ -15,7 +15,7 @@ from env_helper import (
     GITHUB_SERVER_URL,
     GITHUB_UPSTREAM_REPOSITORY,
 )
-from lambda_shared_package.lambda_shared.pr import Labels
+from ci_config import Labels
 from get_robot_token import get_best_robot_token
 from github_helper import GitHub
 
diff --git a/tests/ci/release.py b/tests/ci/release.py
index 7ab1149310f..2de20d00a00 100755
--- a/tests/ci/release.py
+++ b/tests/ci/release.py
@@ -25,7 +25,7 @@ from contextlib import contextmanager
 from typing import Any, Final, Iterator, List, Optional, Tuple
 
 from git_helper import Git, commit, release_branch
-from lambda_shared_package.lambda_shared.pr import Labels
+from ci_config import Labels
 from report import SUCCESS
 from version_helper import (
     FILE_WITH_VERSION_PATH,
diff --git a/tests/ci/run_check.py b/tests/ci/run_check.py
index 00942352dde..0ad01e3accd 100644
--- a/tests/ci/run_check.py
+++ b/tests/ci/run_check.py
@@ -15,26 +15,22 @@ from commit_status_helper import (
 )
 from env_helper import GITHUB_REPOSITORY, GITHUB_SERVER_URL
 from get_robot_token import get_best_robot_token
-from lambda_shared_package.lambda_shared.pr import (
-    CATEGORY_TO_LABEL,
-    TRUSTED_CONTRIBUTORS,
-    Labels,
-    check_pr_description,
-)
+from ci_config import CI
+from ci_utils import Utils
 from pr_info import PRInfo
 from report import FAILURE, PENDING, SUCCESS, StatusType
-from ci_config import CI
+
 
 TRUSTED_ORG_IDS = {
     54801242,  # clickhouse
 }
 
-OK_SKIP_LABELS = {Labels.RELEASE, Labels.PR_BACKPORT, Labels.PR_CHERRYPICK}
+OK_SKIP_LABELS = {CI.Labels.RELEASE, CI.Labels.PR_BACKPORT, CI.Labels.PR_CHERRYPICK}
 PR_CHECK = "PR Check"
 
 
 def pr_is_by_trusted_user(pr_user_login, pr_user_orgs):
-    if pr_user_login.lower() in TRUSTED_CONTRIBUTORS:
+    if pr_user_login.lower() in CI.TRUSTED_CONTRIBUTORS:
         logging.info("User '%s' is trusted", pr_user_login)
         return True
 
@@ -63,13 +59,13 @@ def should_run_ci_for_pr(pr_info: PRInfo) -> Tuple[bool, str]:
     if OK_SKIP_LABELS.intersection(pr_info.labels):
         return True, "Don't try new checks for release/backports/cherry-picks"
 
-    if Labels.CAN_BE_TESTED not in pr_info.labels and not pr_is_by_trusted_user(
+    if CI.Labels.CAN_BE_TESTED not in pr_info.labels and not pr_is_by_trusted_user(
         pr_info.user_login, pr_info.user_orgs
     ):
         logging.info(
             "PRs by untrusted users need the '%s' label - "
             "please contact a member of the core team",
-            Labels.CAN_BE_TESTED,
+            CI.Labels.CAN_BE_TESTED,
         )
         return False, "Needs 'can be tested' label"
 
@@ -96,30 +92,32 @@ def main():
     commit = get_commit(gh, pr_info.sha)
     status = SUCCESS  # type: StatusType
 
-    description_error, category = check_pr_description(pr_info.body, GITHUB_REPOSITORY)
+    description_error, category = Utils.check_pr_description(
+        pr_info.body, GITHUB_REPOSITORY
+    )
     pr_labels_to_add = []
     pr_labels_to_remove = []
     if (
-        category in CATEGORY_TO_LABEL
-        and CATEGORY_TO_LABEL[category] not in pr_info.labels
+        category in CI.CATEGORY_TO_LABEL
+        and CI.CATEGORY_TO_LABEL[category] not in pr_info.labels
     ):
-        pr_labels_to_add.append(CATEGORY_TO_LABEL[category])
+        pr_labels_to_add.append(CI.CATEGORY_TO_LABEL[category])
 
     for label in pr_info.labels:
         if (
-            label in CATEGORY_TO_LABEL.values()
-            and category in CATEGORY_TO_LABEL
-            and label != CATEGORY_TO_LABEL[category]
+            label in CI.CATEGORY_TO_LABEL.values()
+            and category in CI.CATEGORY_TO_LABEL
+            and label != CI.CATEGORY_TO_LABEL[category]
         ):
             pr_labels_to_remove.append(label)
 
     if pr_info.has_changes_in_submodules():
-        pr_labels_to_add.append(Labels.SUBMODULE_CHANGED)
-    elif Labels.SUBMODULE_CHANGED in pr_info.labels:
-        pr_labels_to_remove.append(Labels.SUBMODULE_CHANGED)
+        pr_labels_to_add.append(CI.Labels.SUBMODULE_CHANGED)
+    elif CI.Labels.SUBMODULE_CHANGED in pr_info.labels:
+        pr_labels_to_remove.append(CI.Labels.SUBMODULE_CHANGED)
 
-    if any(label in Labels.AUTO_BACKPORT for label in pr_labels_to_add):
-        backport_labels = [Labels.MUST_BACKPORT, Labels.MUST_BACKPORT_CLOUD]
+    if any(label in CI.Labels.AUTO_BACKPORT for label in pr_labels_to_add):
+        backport_labels = [CI.Labels.MUST_BACKPORT, CI.Labels.MUST_BACKPORT_CLOUD]
         pr_labels_to_add += [
             label for label in backport_labels if label not in pr_info.labels
         ]
@@ -164,15 +162,15 @@ def main():
 
     # 2. Then we check if the documentation is not created to fail the Mergeable check
     if (
-        Labels.PR_FEATURE in pr_info.labels
+        CI.Labels.PR_FEATURE in pr_info.labels
         and not pr_info.has_changes_in_documentation()
     ):
         print(
-            f"::error ::The '{Labels.PR_FEATURE}' in the labels, "
+            f"::error ::The '{CI.Labels.PR_FEATURE}' in the labels, "
             "but there's no changed documentation"
         )
         status = FAILURE
-        description = f"expect adding docs for {Labels.PR_FEATURE}"
+        description = f"expect adding docs for {CI.Labels.PR_FEATURE}"
     # 3. But we allow the workflow to continue
 
     # 4. And post only a single commit status on a failure
diff --git a/tests/ci/runner_token_rotation_lambda/app.py b/tests/ci/runner_token_rotation_lambda/app.py
deleted file mode 100644
index 445704648a9..00000000000
--- a/tests/ci/runner_token_rotation_lambda/app.py
+++ /dev/null
@@ -1,93 +0,0 @@
-#!/usr/bin/env python3
-
-import argparse
-import sys
-
-import boto3  # type: ignore
-import requests
-from lambda_shared.token import get_access_token_by_key_app, get_cached_access_token
-
-
-def get_runner_registration_token(access_token):
-    headers = {
-        "Authorization": f"token {access_token}",
-        "Accept": "application/vnd.github.v3+json",
-    }
-    response = requests.post(
-        "https://api.github.com/orgs/ClickHouse/actions/runners/registration-token",
-        headers=headers,
-        timeout=30,
-    )
-    response.raise_for_status()
-    data = response.json()
-    return data["token"]
-
-
-def main(access_token, push_to_ssm, ssm_parameter_name):
-    runner_registration_token = get_runner_registration_token(access_token)
-
-    if push_to_ssm:
-        print("Trying to put params into ssm manager")
-        client = boto3.client("ssm")
-        client.put_parameter(
-            Name=ssm_parameter_name,
-            Value=runner_registration_token,
-            Type="SecureString",
-            Overwrite=True,
-        )
-    else:
-        print(
-            "Not push token to AWS Parameter Store, just print:",
-            runner_registration_token,
-        )
-
-
-def handler(event, context):
-    _, _ = event, context
-    main(get_cached_access_token(), True, "github_runner_registration_token")
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(
-        description="Get new token from github to add runners"
-    )
-    parser.add_argument(
-        "-p", "--private-key-path", help="Path to file with private key"
-    )
-    parser.add_argument("-k", "--private-key", help="Private key")
-    parser.add_argument(
-        "-a", "--app-id", type=int, help="GitHub application ID", required=True
-    )
-    parser.add_argument(
-        "--push-to-ssm",
-        action="store_true",
-        help="Store received token in parameter store",
-    )
-    parser.add_argument(
-        "--ssm-parameter-name",
-        default="github_runner_registration_token",
-        help="AWS paramater store parameter name",
-    )
-
-    args = parser.parse_args()
-
-    if not args.private_key_path and not args.private_key:
-        print(
-            "Either --private-key-path or --private-key must be specified",
-            file=sys.stderr,
-        )
-
-    if args.private_key_path and args.private_key:
-        print(
-            "Either --private-key-path or --private-key must be specified",
-            file=sys.stderr,
-        )
-
-    if args.private_key:
-        private_key = args.private_key
-    else:
-        with open(args.private_key_path, "r", encoding="utf-8") as key_file:
-            private_key = key_file.read()
-
-    token = get_access_token_by_key_app(private_key, args.app_id)
-    main(token, args.push_to_ssm, args.ssm_parameter_name)
diff --git a/tests/ci/runner_token_rotation_lambda/build_and_deploy_archive.sh b/tests/ci/runner_token_rotation_lambda/build_and_deploy_archive.sh
deleted file mode 120000
index 96ba3fa024e..00000000000
--- a/tests/ci/runner_token_rotation_lambda/build_and_deploy_archive.sh
+++ /dev/null
@@ -1 +0,0 @@
-../team_keys_lambda/build_and_deploy_archive.sh
\ No newline at end of file
diff --git a/tests/ci/runner_token_rotation_lambda/lambda_shared b/tests/ci/runner_token_rotation_lambda/lambda_shared
deleted file mode 120000
index ba86e090f6c..00000000000
--- a/tests/ci/runner_token_rotation_lambda/lambda_shared
+++ /dev/null
@@ -1 +0,0 @@
-../lambda_shared_package/lambda_shared
\ No newline at end of file
diff --git a/tests/ci/runner_token_rotation_lambda/requirements.txt b/tests/ci/runner_token_rotation_lambda/requirements.txt
deleted file mode 100644
index 4cb3fba0f7b..00000000000
--- a/tests/ci/runner_token_rotation_lambda/requirements.txt
+++ /dev/null
@@ -1 +0,0 @@
-../lambda_shared_package[token]
diff --git a/tests/ci/slack_bot_ci_lambda/app.py b/tests/ci/slack_bot_ci_lambda/app.py
deleted file mode 100755
index 94b71724b1c..00000000000
--- a/tests/ci/slack_bot_ci_lambda/app.py
+++ /dev/null
@@ -1,323 +0,0 @@
-#!/usr/bin/env python3
-
-"""
-A trivial stateless slack bot that notifies about new broken tests in ClickHouse CI.
-It checks what happened to our CI during the last check_period hours (1 hour) and
-  notifies us in slack if necessary.
-This script should be executed once each check_period hours (1 hour).
-It will post duplicate messages if you run it more often; it will lose some messages
-  if you run it less often.
-
-You can run it locally with no arguments, it will work in a dry-run mode.
-  Or you can set your own SLACK_URL_DEFAULT.
-Feel free to add more checks, more details to messages, or better heuristics.
-
-It's deployed to slack-bot-ci-lambda in CI/CD account
-
-See also: https://aretestsgreenyet.com/
-"""
-
-import base64
-import json
-import os
-import random
-
-import requests
-
-DRY_RUN_MARK = "<no url, dry run>"
-
-MAX_FAILURES_DEFAULT = 30
-SLACK_URL_DEFAULT = DRY_RUN_MARK
-
-FLAKY_ALERT_PROBABILITY = 0.50
-REPORT_NO_FAILURES_PROBABILITY = 0.99
-
-MAX_TESTS_TO_REPORT = 4
-
-# Slack has a stupid limitation on message size, it splits long messages into multiple,
-# ones breaking formatting
-MESSAGE_LENGTH_LIMIT = 4000
-
-# Find tests that failed in master during the last check_period * 24 hours,
-# but did not fail during the last 2 weeks. Assuming these tests were broken recently.
-# Counts number of failures in check_period and check_period * 24 time windows
-# to distinguish rare flaky tests from completely broken tests
-NEW_BROKEN_TESTS_QUERY = """
-WITH
-    1 AS check_period,
-    check_period * 24 AS extended_check_period,
-    now() as now
-SELECT
-    test_name,
-    any(report_url),
-    countIf((check_start_time + check_duration_ms / 1000) < now - INTERVAL check_period HOUR) AS count_prev_periods,
-    countIf((check_start_time + check_duration_ms / 1000) >= now - INTERVAL check_period HOUR) AS count
-FROM checks
-WHERE 1
-    AND check_start_time BETWEEN now - INTERVAL 1 WEEK AND now
-    AND (check_start_time + check_duration_ms / 1000) >= now - INTERVAL extended_check_period HOUR
-    AND pull_request_number = 0
-    AND test_status LIKE 'F%'
-    AND check_status != 'success'
-    AND test_name NOT IN (
-        SELECT test_name FROM checks WHERE 1
-        AND check_start_time >= now - INTERVAL 1 MONTH
-        AND (check_start_time + check_duration_ms / 1000) BETWEEN now - INTERVAL 2 WEEK AND now - INTERVAL extended_check_period HOUR
-        AND pull_request_number = 0
-        AND check_status != 'success'
-        AND test_status LIKE 'F%')
-    AND test_context_raw NOT LIKE '%CannotSendRequest%' and test_context_raw NOT LIKE '%Server does not respond to health check%'
-GROUP BY test_name
-ORDER BY (count_prev_periods + count) DESC
-"""
-
-# Returns total number of failed checks during the last 24 hours
-# and previous value of that metric (check_period hours ago)
-COUNT_FAILURES_QUERY = """
-WITH
-    1 AS check_period,
-    '%' AS check_name_pattern,
-    now() as now
-SELECT
-    countIf((check_start_time + check_duration_ms / 1000) >= now - INTERVAL 24 HOUR) AS new_val,
-    countIf((check_start_time + check_duration_ms / 1000) <= now - INTERVAL check_period HOUR) AS prev_val
-FROM checks
-WHERE 1
-    AND check_start_time >= now - INTERVAL 1 WEEK
-    AND (check_start_time + check_duration_ms / 1000) >= now - INTERVAL 24 + check_period HOUR
-    AND pull_request_number = 0
-    AND test_status LIKE 'F%'
-    AND check_status != 'success'
-    AND check_name ILIKE check_name_pattern
-"""
-
-# Returns percentage of failed checks (once per day, at noon)
-FAILED_CHECKS_PERCENTAGE_QUERY = """
-SELECT if(toHour(now('Europe/Amsterdam')) = 12, v, 0)
-FROM
-(
-    SELECT
-        countDistinctIf((commit_sha, check_name), (test_status LIKE 'F%') AND (check_status != 'success'))
-            / countDistinct((commit_sha, check_name)) AS v
-    FROM checks
-    WHERE 1
-        AND (pull_request_number = 0)
-        AND (test_status != 'SKIPPED')
-        AND (check_start_time > (now() - toIntervalDay(1)))
-)
-"""
-
-# It shows all recent failures of the specified test (helps to find when it started)
-ALL_RECENT_FAILURES_QUERY = """
-WITH
-    '{}' AS name_substr,
-    90 AS interval_days,
-    ('Stateless tests (asan)', 'Stateless tests (address)', 'Stateless tests (address, actions)', 'Integration tests (asan) [1/3]', 'Stateless tests (tsan) [1/3]') AS backport_and_release_specific_checks
-SELECT
-    toStartOfDay(check_start_time) AS d,
-    count(),
-    groupUniqArray(pull_request_number) AS prs,
-    any(report_url)
-FROM checks
-WHERE ((now() - toIntervalDay(interval_days)) <= check_start_time) AND (pull_request_number NOT IN (
-    SELECT pull_request_number AS prn
-    FROM checks
-    WHERE (prn != 0) AND ((now() - toIntervalDay(interval_days)) <= check_start_time) AND (check_name IN (backport_and_release_specific_checks))
-)) AND (position(test_name, name_substr) > 0) AND (test_status IN ('FAIL', 'ERROR', 'FLAKY'))
-GROUP BY d
-ORDER BY d DESC
-"""
-
-SLACK_MESSAGE_JSON = {"type": "mrkdwn", "text": None}
-
-
-def get_play_url(query):
-    return (
-        "https://play.clickhouse.com/play?user=play#"
-        + base64.b64encode(query.encode()).decode()
-    )
-
-
-def run_clickhouse_query(query):
-    url = "https://play.clickhouse.com/?user=play&query=" + requests.compat.quote(query)
-    res = requests.get(url, timeout=30)
-    if res.status_code != 200:
-        print("Failed to execute query: ", res.status_code, res.content)
-        res.raise_for_status()
-
-    lines = res.text.strip().splitlines()
-    return [x.split("\t") for x in lines]
-
-
-def split_broken_and_flaky_tests(failed_tests):
-    if not failed_tests:
-        return None
-
-    broken_tests = []
-    flaky_tests = []
-    for name, report, count_prev_str, count_str in failed_tests:
-        count_prev, count = int(count_prev_str), int(count_str)
-        if (count_prev < 2 <= count) or (count_prev == count == 1):
-            # It failed 2 times or more within extended time window, it's definitely broken.
-            # 2 <= count means that it was not reported as broken on previous runs
-            broken_tests.append([name, report])
-        elif 0 < count and count_prev == 0:
-            # It failed only once, can be a rare flaky test
-            flaky_tests.append([name, report])
-
-    return broken_tests, flaky_tests
-
-
-def format_failed_tests_list(failed_tests, failure_type):
-    if len(failed_tests) == 1:
-        res = f"There is a new {failure_type} test:\n"
-    else:
-        res = f"There are {len(failed_tests)} new {failure_type} tests:\n"
-
-    for name, report in failed_tests[:MAX_TESTS_TO_REPORT]:
-        cidb_url = get_play_url(ALL_RECENT_FAILURES_QUERY.format(name))
-        res += f"-   *{name}*  -  <{report}|Report>  -  <{cidb_url}|CI DB> \n"
-
-    if MAX_TESTS_TO_REPORT < len(failed_tests):
-        res += (
-            f"-   and {len(failed_tests) - MAX_TESTS_TO_REPORT} other "
-            "tests... :this-is-fine-fire:"
-        )
-
-    return res
-
-
-def get_new_broken_tests_message(failed_tests):
-    if not failed_tests:
-        return None
-
-    broken_tests, flaky_tests = split_broken_and_flaky_tests(failed_tests)
-    if len(broken_tests) == 0 and len(flaky_tests) == 0:
-        return None
-
-    msg = ""
-    if len(broken_tests) > 0:
-        msg += format_failed_tests_list(broken_tests, "*BROKEN*")
-    elif random.random() > FLAKY_ALERT_PROBABILITY:
-        looks_like_fuzzer = [x[0].count(" ") > 2 for x in flaky_tests]
-        if not any(looks_like_fuzzer):
-            print("Will not report flaky tests to avoid noise: ", flaky_tests)
-            return None
-
-    if len(flaky_tests) > 0:
-        if len(msg) > 0:
-            msg += "\n"
-        msg += format_failed_tests_list(flaky_tests, "flaky")
-
-    return msg
-
-
-def get_too_many_failures_message_impl(failures_count):
-    MAX_FAILURES = int(os.environ.get("MAX_FAILURES", MAX_FAILURES_DEFAULT))
-    curr_failures = int(failures_count[0][0])
-    prev_failures = int(failures_count[0][1])
-    if curr_failures == 0 and prev_failures != 0:
-        if random.random() < REPORT_NO_FAILURES_PROBABILITY:
-            return None
-        return "Wow, there are *no failures* at all... 0_o"
-    return_none = (
-        curr_failures < MAX_FAILURES
-        or curr_failures < prev_failures
-        or (curr_failures - prev_failures) / prev_failures < 0.2
-    )
-    if return_none:
-        return None
-    if prev_failures < MAX_FAILURES:
-        return f":alert: *CI is broken: there are {curr_failures} failures during the last 24 hours*"
-    return "CI is broken and it's getting worse: there are {curr_failures} failures during the last 24 hours"
-
-
-def get_too_many_failures_message(failures_count):
-    msg = get_too_many_failures_message_impl(failures_count)
-    if msg:
-        msg += "\nSee https://aretestsgreenyet.com/"
-    return msg
-
-
-def get_failed_checks_percentage_message(percentage):
-    p = float(percentage[0][0]) * 100
-
-    # Always report more than 1% of failed checks
-    # For <= 1%: higher percentage of failures == higher probability
-    if p <= random.random():
-        return None
-
-    msg = ":alert: " if p > 1 else "Only " if p < 0.5 else ""
-    msg += f"*{p:.2f}%* of all checks in master have failed yesterday"
-    return msg
-
-
-def split_slack_message(long_message):
-    lines = long_message.split("\n")
-    messages = []
-    curr_msg = ""
-    for line in lines:
-        if len(curr_msg) + len(line) < MESSAGE_LENGTH_LIMIT:
-            curr_msg += "\n"
-            curr_msg += line
-        else:
-            messages.append(curr_msg)
-            curr_msg = line
-    messages.append(curr_msg)
-    return messages
-
-
-def send_to_slack_impl(message):
-    SLACK_URL = os.environ.get("SLACK_URL", SLACK_URL_DEFAULT)
-    if SLACK_URL == DRY_RUN_MARK:
-        return
-
-    payload = SLACK_MESSAGE_JSON.copy()
-    payload["text"] = message
-    res = requests.post(SLACK_URL, json.dumps(payload), timeout=30)
-    if res.status_code != 200:
-        print("Failed to send a message to Slack: ", res.status_code, res.content)
-        res.raise_for_status()
-
-
-def send_to_slack(message):
-    messages = split_slack_message(message)
-    for msg in messages:
-        send_to_slack_impl(msg)
-
-
-def query_and_alert_if_needed(query, get_message_func):
-    query_res = run_clickhouse_query(query)
-    print("Got result {} for query {}", query_res, query)
-    msg = get_message_func(query_res)
-    if msg is None:
-        return
-
-    msg += f"\nCI DB query: <{get_play_url(query)}|link>"
-    print("Sending message to slack:", msg)
-    send_to_slack(msg)
-
-
-def check_and_alert():
-    query_and_alert_if_needed(NEW_BROKEN_TESTS_QUERY, get_new_broken_tests_message)
-    query_and_alert_if_needed(COUNT_FAILURES_QUERY, get_too_many_failures_message)
-    query_and_alert_if_needed(
-        FAILED_CHECKS_PERCENTAGE_QUERY, get_failed_checks_percentage_message
-    )
-
-
-def handler(event, context):
-    _, _ = event, context
-    try:
-        check_and_alert()
-        return {"statusCode": 200, "body": "OK"}
-    except Exception as e:
-        send_to_slack(
-            "I failed, please help me "
-            f"(see ClickHouse/ClickHouse/tests/ci/slack_bot_ci_lambda/app.py): {e}"
-        )
-        return {"statusCode": 200, "body": "FAIL"}
-
-
-if __name__ == "__main__":
-    check_and_alert()
diff --git a/tests/ci/slack_bot_ci_lambda/build_and_deploy_archive.sh b/tests/ci/slack_bot_ci_lambda/build_and_deploy_archive.sh
deleted file mode 120000
index 96ba3fa024e..00000000000
--- a/tests/ci/slack_bot_ci_lambda/build_and_deploy_archive.sh
+++ /dev/null
@@ -1 +0,0 @@
-../team_keys_lambda/build_and_deploy_archive.sh
\ No newline at end of file
diff --git a/tests/ci/slack_bot_ci_lambda/requirements.txt b/tests/ci/slack_bot_ci_lambda/requirements.txt
deleted file mode 100644
index 098e04a9798..00000000000
--- a/tests/ci/slack_bot_ci_lambda/requirements.txt
+++ /dev/null
@@ -1 +0,0 @@
-../lambda_shared_package
diff --git a/tests/ci/team_keys_lambda/app.py b/tests/ci/team_keys_lambda/app.py
deleted file mode 100644
index c056808a9b4..00000000000
--- a/tests/ci/team_keys_lambda/app.py
+++ /dev/null
@@ -1,136 +0,0 @@
-#!/usr/bin/env python3
-
-import argparse
-import json
-from datetime import datetime
-from queue import Queue
-from threading import Thread
-
-import boto3  # type: ignore
-import requests
-
-
-class Keys(set):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.updated_at = 0.0
-
-    def update_now(self):
-        self.updated_at = datetime.now().timestamp()
-
-
-keys = Keys()
-
-
-class Worker(Thread):
-    def __init__(self, request_queue):
-        Thread.__init__(self)
-        self.queue = request_queue
-        self.results = set()
-
-    def run(self):
-        while True:
-            m = self.queue.get()
-            if m == "":
-                break
-            response = requests.get(f"https://github.com/{m}.keys", timeout=30)
-            self.results.add(f"# {m}\n{response.text}\n")
-            self.queue.task_done()
-
-
-def get_org_team_members(token: str, org: str, team_slug: str) -> set:
-    headers = {
-        "Authorization": f"token {token}",
-        "Accept": "application/vnd.github.v3+json",
-    }
-    response = requests.get(
-        f"https://api.github.com/orgs/{org}/teams/{team_slug}/members",
-        headers=headers,
-        timeout=30,
-    )
-    response.raise_for_status()
-    data = response.json()
-    return set(m["login"] for m in data)
-
-
-def get_cached_members_keys(members: set) -> Keys:
-    if (datetime.now().timestamp() - 3600) <= keys.updated_at:
-        return keys
-
-    q = Queue()  # type: Queue
-    workers = []
-    for m in members:
-        q.put(m)
-        # Create workers and add to the queue
-        worker = Worker(q)
-        worker.start()
-        workers.append(worker)
-
-    # Workers keep working till they receive an empty string
-    for _ in workers:
-        q.put("")
-
-    # Join workers to wait till they finished
-    for worker in workers:
-        worker.join()
-
-    keys.clear()
-    for worker in workers:
-        keys.update(worker.results)
-    keys.update_now()
-    return keys
-
-
-def get_token_from_aws() -> str:
-    # We need a separate token, since the clickhouse-ci app does not have
-    # access to the organization members' endpoint
-    secret_name = "clickhouse_robot_token"
-    session = boto3.session.Session()
-    client = session.client(
-        service_name="secretsmanager",
-    )
-    get_secret_value_response = client.get_secret_value(SecretId=secret_name)
-    data = json.loads(get_secret_value_response["SecretString"])
-    return data["clickhouse_robot_token"]  # type: ignore
-
-
-def main(token: str, org: str, team_slug: str) -> str:
-    members = get_org_team_members(token, org, team_slug)
-    keys = get_cached_members_keys(members)
-
-    return "".join(sorted(keys))
-
-
-def handler(event, context):
-    _ = context
-    _ = event
-    if keys.updated_at < (datetime.now().timestamp() - 3600):
-        token = get_token_from_aws()
-        body = main(token, "ClickHouse", "core")
-    else:
-        body = "".join(sorted(keys))
-
-    result = {
-        "statusCode": 200,
-        "headers": {
-            "Content-Type": "text/html",
-        },
-        "body": body,
-    }
-    return result
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(
-        description="Get the public SSH keys for members of given org and team"
-    )
-    parser.add_argument("--token", required=True, help="Github PAT")
-    parser.add_argument(
-        "--organization", help="GitHub organization name", default="ClickHouse"
-    )
-    parser.add_argument("--team", help="GitHub team name", default="core")
-
-    args = parser.parse_args()
-    output = main(args.token, args.organization, args.team)
-
-    print(f"# Just showing off the keys:\n{output}")
diff --git a/tests/ci/team_keys_lambda/build_and_deploy_archive.sh b/tests/ci/team_keys_lambda/build_and_deploy_archive.sh
deleted file mode 100644
index b72bce4a677..00000000000
--- a/tests/ci/team_keys_lambda/build_and_deploy_archive.sh
+++ /dev/null
@@ -1,76 +0,0 @@
-#!/usr/bin/env bash
-set -xeo pipefail
-
-WORKDIR=$(dirname "$0")
-WORKDIR=$(readlink -f "${WORKDIR}")
-DIR_NAME=$(basename "$WORKDIR")
-cd "$WORKDIR"
-
-# Do not deploy the lambda to AWS
-DRY_RUN=${DRY_RUN:-}
-# Python runtime to install dependencies
-PY_VERSION=${PY_VERSION:-3.10}
-PY_EXEC="python${PY_VERSION}"
-# Image to build the lambda zip package
-DOCKER_IMAGE="public.ecr.aws/lambda/python:${PY_VERSION}"
-# Rename the_lambda_name directory to the-lambda-name lambda in AWS
-LAMBDA_NAME=${DIR_NAME//_/-}
-# The name of directory with lambda code
-PACKAGE=lambda-package
-
-# Do not rebuild and deploy the archive if it's newer than sources
-if [ -e "$PACKAGE.zip" ] && [ -z "$FORCE" ]; then
-  REBUILD=""
-  for src in app.py build_and_deploy_archive.sh requirements.txt lambda_shared/*; do
-    if [ "$src" -nt "$PACKAGE.zip" ]; then
-      REBUILD=1
-    fi
-  done
-  [ -n "$REBUILD" ] || exit 0
-fi
-
-docker_cmd=(
-  docker run -i --net=host --rm --user="${UID}" -e HOME=/tmp --entrypoint=/bin/bash
-  --volume="${WORKDIR}/..:/ci" --workdir="/ci/${DIR_NAME}" "${DOCKER_IMAGE}"
-)
-rm -rf "$PACKAGE" "$PACKAGE".zip
-mkdir "$PACKAGE"
-cp app.py "$PACKAGE"
-if [ -f requirements.txt ]; then
-  VENV=lambda-venv
-  rm -rf "$VENV"
-  "${docker_cmd[@]}" -ex <<EOF
-    '$PY_EXEC' -m venv '$VENV' &&
-    source '$VENV/bin/activate' &&
-    pip install -r requirements.txt &&
-    # To have consistent pyc files
-    find '$VENV/lib' -name '*.pyc' -delete
-    cp -rT '$VENV/lib/$PY_EXEC/site-packages/' '$PACKAGE'
-    rm -r '$PACKAGE'/{pip,pip-*,setuptools,setuptools-*}
-    chmod 0777 -R '$PACKAGE'
-EOF
-fi
-# Create zip archive via python zipfile to have it cross-platform
-"${docker_cmd[@]}" -ex <<EOF
-cd '$PACKAGE'
-find ! -type d -exec touch -t 201212121212 {} +
-
-python <<'EOP'
-import zipfile
-import os
-files_path = []
-for root, _, files in os.walk('.'):
-    files_path.extend(os.path.join(root, file) for file in files)
-# persistent file order
-files_path.sort()
-with zipfile.ZipFile('../$PACKAGE.zip', 'w') as zf:
-    for file in files_path:
-        zf.write(file)
-EOP
-EOF
-
-ECHO=()
-if [ -n "$DRY_RUN" ]; then
-  ECHO=(echo Run the following command to push the changes:)
-fi
-"${ECHO[@]}" aws lambda update-function-code --function-name "$LAMBDA_NAME" --zip-file fileb://"$WORKDIR/$PACKAGE".zip
diff --git a/tests/ci/team_keys_lambda/lambda_shared b/tests/ci/team_keys_lambda/lambda_shared
deleted file mode 120000
index ba86e090f6c..00000000000
--- a/tests/ci/team_keys_lambda/lambda_shared
+++ /dev/null
@@ -1 +0,0 @@
-../lambda_shared_package/lambda_shared
\ No newline at end of file
diff --git a/tests/ci/team_keys_lambda/requirements.txt b/tests/ci/team_keys_lambda/requirements.txt
deleted file mode 100644
index 098e04a9798..00000000000
--- a/tests/ci/team_keys_lambda/requirements.txt
+++ /dev/null
@@ -1 +0,0 @@
-../lambda_shared_package
diff --git a/tests/ci/terminate_runner_lambda/app.py b/tests/ci/terminate_runner_lambda/app.py
deleted file mode 100644
index ab3e33d9e11..00000000000
--- a/tests/ci/terminate_runner_lambda/app.py
+++ /dev/null
@@ -1,278 +0,0 @@
-#!/usr/bin/env python3
-
-import argparse
-import json
-import sys
-import time
-from dataclasses import dataclass
-from typing import Any, Dict, List
-
-import boto3  # type: ignore
-from lambda_shared import RunnerDescriptions, cached_value_is_valid, list_runners
-from lambda_shared.token import get_access_token_by_key_app, get_cached_access_token
-
-
-@dataclass
-class CachedInstances:
-    time: float
-    value: dict
-    updating: bool = False
-
-
-cached_instances = CachedInstances(0, {})
-
-
-def get_cached_instances() -> dict:
-    """return cached instances description with updating it once per five minutes"""
-    if time.time() - 250 < cached_instances.time or cached_instances.updating:
-        return cached_instances.value
-    cached_instances.updating = cached_value_is_valid(cached_instances.time, 300)
-    ec2_client = boto3.client("ec2")
-    instances_response = ec2_client.describe_instances(
-        Filters=[{"Name": "instance-state-name", "Values": ["running"]}]
-    )
-    cached_instances.time = time.time()
-    cached_instances.value = {
-        instance["InstanceId"]: instance
-        for reservation in instances_response["Reservations"]
-        for instance in reservation["Instances"]
-    }
-    cached_instances.updating = False
-    return cached_instances.value
-
-
-@dataclass
-class CachedRunners:
-    time: float
-    value: RunnerDescriptions
-    updating: bool = False
-
-
-cached_runners = CachedRunners(0, [])
-
-
-def get_cached_runners(access_token: str) -> RunnerDescriptions:
-    """From time to time request to GH api costs up to 3 seconds, and
-    it's a disaster from the termination lambda perspective"""
-    if time.time() - 5 < cached_runners.time or cached_instances.updating:
-        return cached_runners.value
-    cached_runners.updating = cached_value_is_valid(cached_runners.time, 15)
-    cached_runners.value = list_runners(access_token)
-    cached_runners.time = time.time()
-    cached_runners.updating = False
-    return cached_runners.value
-
-
-def how_many_instances_to_kill(event_data: dict) -> Dict[str, int]:
-    data_array = event_data["CapacityToTerminate"]
-    to_kill_by_zone = {}  # type: Dict[str, int]
-    for av_zone in data_array:
-        zone_name = av_zone["AvailabilityZone"]
-        to_kill = av_zone["Capacity"]
-        if zone_name not in to_kill_by_zone:
-            to_kill_by_zone[zone_name] = 0
-
-        to_kill_by_zone[zone_name] += to_kill
-
-    return to_kill_by_zone
-
-
-def get_candidates_to_be_killed(event_data: dict) -> Dict[str, List[str]]:
-    data_array = event_data["Instances"]
-    instances_by_zone = {}  # type: Dict[str, List[str]]
-    for instance in data_array:
-        zone_name = instance["AvailabilityZone"]
-        instance_id = instance["InstanceId"]  # type: str
-        if zone_name not in instances_by_zone:
-            instances_by_zone[zone_name] = []
-        instances_by_zone[zone_name].append(instance_id)
-
-    return instances_by_zone
-
-
-def main(access_token: str, event: dict) -> Dict[str, List[str]]:
-    start = time.time()
-    print("Got event", json.dumps(event, sort_keys=True).replace("\n", ""))
-    to_kill_by_zone = how_many_instances_to_kill(event)
-    instances_by_zone = get_candidates_to_be_killed(event)
-    # Getting ASG and instances' descriptions from the API
-    # We don't kill instances that alive for less than 10 minutes, since they
-    # could be not in the GH active runners yet
-    print(f"Check other hosts from the same ASG {event['AutoScalingGroupName']}")
-    asg_client = boto3.client("autoscaling")
-    as_groups_response = asg_client.describe_auto_scaling_groups(
-        AutoScalingGroupNames=[event["AutoScalingGroupName"]]
-    )
-    assert len(as_groups_response["AutoScalingGroups"]) == 1
-    asg = as_groups_response["AutoScalingGroups"][0]
-    asg_instance_ids = [instance["InstanceId"] for instance in asg["Instances"]]
-    instance_descriptions = get_cached_instances()
-    # The instances launched less than 10 minutes ago
-    immune_ids = [
-        instance["InstanceId"]
-        for instance in instance_descriptions.values()
-        if start - instance["LaunchTime"].timestamp() < 600
-    ]
-    # if the ASG's instance ID not in instance_descriptions, it's most probably
-    # is not cached yet, so we must mark it as immuned
-    immune_ids.extend(
-        iid for iid in asg_instance_ids if iid not in instance_descriptions
-    )
-    print("Time spent on the requests to AWS: ", time.time() - start)
-
-    runners = get_cached_runners(access_token)
-    runner_ids = set(runner.name for runner in runners)
-    # We used to delete potential hosts to terminate from GitHub runners pool,
-    # but the documentation states:
-    # --- Returning an instance first in the response data does not guarantee its termination
-    # so they will be cleaned out by ci_runners_metrics_lambda eventually
-
-    instances_to_kill = []
-    total_to_kill = 0
-    for zone, num_to_kill in to_kill_by_zone.items():
-        candidates = instances_by_zone[zone]
-        total_to_kill += num_to_kill
-        if num_to_kill > len(candidates):
-            raise RuntimeError(
-                f"Required to kill {num_to_kill}, but have only {len(candidates)}"
-                f" candidates in AV {zone}"
-            )
-
-        delete_for_av = []  # type: RunnerDescriptions
-        for candidate in candidates:
-            if candidate in immune_ids:
-                print(
-                    f"Candidate {candidate} started less than 10 minutes ago, won't touch a child"
-                )
-                break
-            if candidate not in runner_ids:
-                print(
-                    f"Candidate {candidate} was not in runners list, simply delete it"
-                )
-                instances_to_kill.append(candidate)
-                break
-            if len(delete_for_av) + len(instances_to_kill) == num_to_kill:
-                break
-            if candidate in instances_to_kill:
-                continue
-
-            for runner in runners:
-                if runner.name == candidate:
-                    if not runner.busy:
-                        print(
-                            f"Runner {runner.name} is not busy and can be deleted from AV {zone}"
-                        )
-                        delete_for_av.append(runner)
-                    else:
-                        print(f"Runner {runner.name} is busy, not going to delete it")
-                    break
-
-        if len(delete_for_av) < num_to_kill:
-            print(
-                f"Checked all candidates for av {zone}, get to delete "
-                f"{len(delete_for_av)}, but still cannot get required {num_to_kill}"
-            )
-
-        instances_to_kill += [runner.name for runner in delete_for_av]
-
-    if len(instances_to_kill) < total_to_kill:
-        for instance in asg_instance_ids:
-            if instance in immune_ids:
-                continue
-            for runner in runners:
-                if runner.name == instance and not runner.busy:
-                    print(f"Runner {runner.name} is not busy and can be deleted")
-                    instances_to_kill.append(runner.name)
-
-            if total_to_kill <= len(instances_to_kill):
-                print("Got enough instances to kill")
-                break
-
-    response = {"InstanceIDs": instances_to_kill}
-    print("Got instances to kill: ", response)
-    print("Time spent on the request: ", time.time() - start)
-    return response
-
-
-def handler(event: dict, context: Any) -> Dict[str, List[str]]:
-    _ = context
-    return main(get_cached_access_token(), event)
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Get list of runners and their states")
-    parser.add_argument(
-        "-p", "--private-key-path", help="Path to file with private key"
-    )
-    parser.add_argument("-k", "--private-key", help="Private key")
-    parser.add_argument(
-        "-a", "--app-id", type=int, help="GitHub application ID", required=True
-    )
-
-    args = parser.parse_args()
-
-    if not args.private_key_path and not args.private_key:
-        print(
-            "Either --private-key-path or --private-key must be specified",
-            file=sys.stderr,
-        )
-
-    if args.private_key_path and args.private_key:
-        print(
-            "Either --private-key-path or --private-key must be specified",
-            file=sys.stderr,
-        )
-
-    if args.private_key:
-        private_key = args.private_key
-    else:
-        with open(args.private_key_path, "r", encoding="utf-8") as key_file:
-            private_key = key_file.read()
-
-    token = get_access_token_by_key_app(private_key, args.app_id)
-
-    sample_event = {
-        "AutoScalingGroupARN": "arn:aws:autoscaling:us-east-1:<account-id>:autoScalingGroup:d4738357-2d40-4038-ae7e-b00ae0227003:autoScalingGroupName/my-asg",
-        "AutoScalingGroupName": "my-asg",
-        "CapacityToTerminate": [
-            {
-                "AvailabilityZone": "us-east-1b",
-                "Capacity": 1,
-                "InstanceMarketOption": "OnDemand",
-            },
-            {
-                "AvailabilityZone": "us-east-1c",
-                "Capacity": 2,
-                "InstanceMarketOption": "OnDemand",
-            },
-        ],
-        "Instances": [
-            {
-                "AvailabilityZone": "us-east-1b",
-                "InstanceId": "i-08d0b3c1a137e02a5",
-                "InstanceType": "t2.nano",
-                "InstanceMarketOption": "OnDemand",
-            },
-            {
-                "AvailabilityZone": "us-east-1c",
-                "InstanceId": "ip-172-31-45-253.eu-west-1.compute.internal",
-                "InstanceType": "t2.nano",
-                "InstanceMarketOption": "OnDemand",
-            },
-            {
-                "AvailabilityZone": "us-east-1c",
-                "InstanceId": "ip-172-31-27-227.eu-west-1.compute.internal",
-                "InstanceType": "t2.nano",
-                "InstanceMarketOption": "OnDemand",
-            },
-            {
-                "AvailabilityZone": "us-east-1c",
-                "InstanceId": "ip-172-31-45-253.eu-west-1.compute.internal",
-                "InstanceType": "t2.nano",
-                "InstanceMarketOption": "OnDemand",
-            },
-        ],
-        "Cause": "SCALE_IN",
-    }
-
-    main(token, sample_event)
diff --git a/tests/ci/terminate_runner_lambda/build_and_deploy_archive.sh b/tests/ci/terminate_runner_lambda/build_and_deploy_archive.sh
deleted file mode 120000
index 96ba3fa024e..00000000000
--- a/tests/ci/terminate_runner_lambda/build_and_deploy_archive.sh
+++ /dev/null
@@ -1 +0,0 @@
-../team_keys_lambda/build_and_deploy_archive.sh
\ No newline at end of file
diff --git a/tests/ci/terminate_runner_lambda/lambda_shared b/tests/ci/terminate_runner_lambda/lambda_shared
deleted file mode 120000
index ba86e090f6c..00000000000
--- a/tests/ci/terminate_runner_lambda/lambda_shared
+++ /dev/null
@@ -1 +0,0 @@
-../lambda_shared_package/lambda_shared
\ No newline at end of file
diff --git a/tests/ci/terminate_runner_lambda/requirements.txt b/tests/ci/terminate_runner_lambda/requirements.txt
deleted file mode 100644
index 4cb3fba0f7b..00000000000
--- a/tests/ci/terminate_runner_lambda/requirements.txt
+++ /dev/null
@@ -1 +0,0 @@
-../lambda_shared_package[token]
diff --git a/tests/ci/workflow_approve_rerun_lambda/app.py b/tests/ci/workflow_approve_rerun_lambda/app.py
deleted file mode 100644
index 8ed47a54ab1..00000000000
--- a/tests/ci/workflow_approve_rerun_lambda/app.py
+++ /dev/null
@@ -1,404 +0,0 @@
-#!/usr/bin/env python3
-
-import fnmatch
-import json
-import time
-from collections import namedtuple
-from urllib.parse import quote
-
-import requests
-from lambda_shared.pr import TRUSTED_CONTRIBUTORS
-from lambda_shared.token import get_cached_access_token
-
-SUSPICIOUS_CHANGED_FILES_NUMBER = 200
-
-SUSPICIOUS_PATTERNS = [
-    ".github/*",
-    "docker/*",
-    "docs/tools/*",
-    "packages/*",
-    "tests/ci/*",
-]
-
-# Number of retries for API calls.
-MAX_RETRY = 5
-
-# Number of times a check can re-run as a whole.
-# It is needed, because we are using AWS "spot" instances, that are terminated often
-MAX_WORKFLOW_RERUN = 30
-
-WorkflowDescription = namedtuple(
-    "WorkflowDescription",
-    [
-        "name",
-        "action",
-        "run_id",
-        "event",
-        "workflow_id",
-        "conclusion",
-        "status",
-        "api_url",
-        "fork_owner_login",
-        "fork_branch",
-        "rerun_url",
-        "jobs_url",
-        "attempt",
-        "repo_url",
-        "url",
-    ],
-)
-
-# See https://api.github.com/orgs/{name}
-TRUSTED_ORG_IDS = {
-    54801242,  # clickhouse
-}
-
-# See https://api.github.com/repos/ClickHouse/ClickHouse/actions/workflows
-# Use ID to not inject a malicious workflow
-TRUSTED_WORKFLOW_IDS = {
-    14586616,  # Cancel workflows, always trusted
-}
-
-NEED_RERUN_WORKFLOWS = {
-    "BackportPR",
-    "DocsCheck",
-    "MasterCI",
-    "NightlyBuilds",
-    "PublishedReleaseCI",
-    "PullRequestCI",
-    "ReleaseBranchCI",
-}
-
-
-def is_trusted_contributor(pr_user_login, pr_user_orgs):
-    if pr_user_login.lower() in TRUSTED_CONTRIBUTORS:
-        print(f"User '{pr_user_login}' is trusted")
-        return True
-
-    print(f"User '{pr_user_login}' is not trusted")
-
-    for org_id in pr_user_orgs:
-        if org_id in TRUSTED_ORG_IDS:
-            print(
-                f"Org '{org_id}' is trusted; will mark user {pr_user_login} as trusted"
-            )
-            return True
-        print(f"Org '{org_id}' is not trusted")
-
-    return False
-
-
-def _exec_get_with_retry(url, token):
-    headers = {"Authorization": f"token {token}"}
-    e = Exception()
-    for i in range(MAX_RETRY):
-        try:
-            response = requests.get(url, headers=headers, timeout=30)
-            response.raise_for_status()
-            return response.json()
-        except Exception as ex:
-            print("Got exception executing request", ex)
-            e = ex
-            time.sleep(i + 1)
-
-    raise requests.HTTPError("Cannot execute GET request with retries") from e
-
-
-def _exec_post_with_retry(url, token, data=None):
-    headers = {"Authorization": f"token {token}"}
-    e = Exception()
-    for i in range(MAX_RETRY):
-        try:
-            if data:
-                response = requests.post(url, headers=headers, json=data, timeout=30)
-            else:
-                response = requests.post(url, headers=headers, timeout=30)
-            if response.status_code == 403:
-                data = response.json()
-                if (
-                    "message" in data
-                    and data["message"]
-                    == "This workflow run is not waiting for approval"
-                ):
-                    print("Workflow doesn't need approval")
-                    return data
-            response.raise_for_status()
-            return response.json()
-        except Exception as ex:
-            print("Got exception executing request", ex)
-            e = ex
-            time.sleep(i + 1)
-
-    raise requests.HTTPError("Cannot execute POST request with retry") from e
-
-
-def _get_pull_requests_from(repo_url, owner, branch, token):
-    url = f"{repo_url}/pulls?head={quote(owner)}:{quote(branch)}"
-    return _exec_get_with_retry(url, token)
-
-
-def get_workflow_description_from_event(event):
-    action = event["action"]
-    run_id = event["workflow_run"]["id"]
-    event_type = event["workflow_run"]["event"]
-    fork_owner = event["workflow_run"]["head_repository"]["owner"]["login"]
-    fork_branch = event["workflow_run"]["head_branch"]
-    name = event["workflow_run"]["name"]
-    workflow_id = event["workflow_run"]["workflow_id"]
-    conclusion = event["workflow_run"]["conclusion"]
-    attempt = event["workflow_run"]["run_attempt"]
-    status = event["workflow_run"]["status"]
-    jobs_url = event["workflow_run"]["jobs_url"]
-    rerun_url = event["workflow_run"]["rerun_url"]
-    url = event["workflow_run"]["html_url"]
-    api_url = event["workflow_run"]["url"]
-    repo_url = event["repository"]["url"]
-    return WorkflowDescription(
-        name=name,
-        action=action,
-        run_id=run_id,
-        event=event_type,
-        fork_owner_login=fork_owner,
-        fork_branch=fork_branch,
-        workflow_id=workflow_id,
-        conclusion=conclusion,
-        attempt=attempt,
-        status=status,
-        jobs_url=jobs_url,
-        rerun_url=rerun_url,
-        url=url,
-        repo_url=repo_url,
-        api_url=api_url,
-    )
-
-
-def get_pr_author_and_orgs(pull_request, token):
-    author = pull_request["user"]["login"]
-    orgs = _exec_get_with_retry(pull_request["user"]["organizations_url"], token)
-    return author, [org["id"] for org in orgs]
-
-
-def get_changed_files_for_pull_request(pull_request, token):
-    url = pull_request["url"]
-
-    changed_files = set([])
-    for i in range(1, 31):
-        print("Requesting changed files page", i)
-        data = _exec_get_with_retry(f"{url}/files?page={i}&per_page=100", token)
-        print(f"Got {len(data)} changed files")
-        if len(data) == 0:
-            print("No more changed files")
-            break
-
-        for change in data:
-            # print("Adding changed file", change['filename'])
-            changed_files.add(change["filename"])
-
-        if len(changed_files) >= SUSPICIOUS_CHANGED_FILES_NUMBER:
-            print(
-                f"More than {len(changed_files)} changed files. "
-                "Will stop fetching new files."
-            )
-            break
-
-    return changed_files
-
-
-def check_suspicious_changed_files(changed_files):
-    if len(changed_files) >= SUSPICIOUS_CHANGED_FILES_NUMBER:
-        print(f"Too many files changed {len(changed_files)}, need manual approve")
-        return True
-
-    for path in changed_files:
-        for pattern in SUSPICIOUS_PATTERNS:
-            if fnmatch.fnmatch(path, pattern):
-                print(
-                    f"File {path} match suspicious pattern {pattern}, "
-                    "will not approve automatically"
-                )
-                return True
-
-    print("No changed files match suspicious patterns, run could be approved")
-    return False
-
-
-def approve_run(workflow_description: WorkflowDescription, token: str) -> None:
-    print("Approving run")
-    url = f"{workflow_description.api_url}/approve"
-    _exec_post_with_retry(url, token)
-
-
-def label_manual_approve(pull_request, token):
-    url = f"{pull_request['issue_url']}/labels"
-    data = {"labels": ["manual approve"]}
-
-    _exec_post_with_retry(url, token, data)
-
-
-def get_workflow_jobs(workflow_description, token):
-    jobs_url = (
-        workflow_description.api_url + f"/attempts/{workflow_description.attempt}/jobs"
-    )
-    jobs = []
-    i = 1
-    while True:
-        got_jobs = _exec_get_with_retry(jobs_url + f"?page={i}", token)
-        if len(got_jobs["jobs"]) == 0:
-            break
-
-        jobs += got_jobs["jobs"]
-        i += 1
-
-    return jobs
-
-
-def check_need_to_rerun(workflow_description, token):
-    if workflow_description.attempt >= MAX_WORKFLOW_RERUN:
-        print(
-            "Not going to rerun workflow because it's already tried more than two times"
-        )
-        return False
-    print("Going to check jobs")
-
-    jobs = get_workflow_jobs(workflow_description, token)
-    print("Got jobs", len(jobs))
-    for job in jobs:
-        print(f"Job {job['name']} has a conclusion '{job['conclusion']}'")
-        if job["conclusion"] not in ("success", "skipped"):
-            print("Job", job["name"], "failed, checking steps")
-            for step in job["steps"]:
-                # always the last job
-                if step["name"] == "Complete job":
-                    print("Found Complete job step for job", job["name"])
-                    break
-            else:
-                print(
-                    "Checked all steps and doesn't found Complete job, going to rerun"
-                )
-                return True
-
-    return False
-
-
-def rerun_workflow(workflow_description, token):
-    print("Going to rerun workflow")
-    try:
-        _exec_post_with_retry(f"{workflow_description.rerun_url}-failed-jobs", token)
-    except Exception:
-        _exec_post_with_retry(workflow_description.rerun_url, token)
-
-
-def check_workflow_completed(
-    event_data: dict, workflow_description: WorkflowDescription, token: str
-) -> bool:
-    if workflow_description.action == "completed":
-        attempt = 0
-        # Nice and reliable GH API sends from time to time such events, e.g:
-        # action='completed', conclusion=None, status='in_progress',
-        # So let's try receiving a real workflow data
-        while workflow_description.conclusion is None and attempt < MAX_RETRY:
-            progressive_sleep = 3 * sum(i + 1 for i in range(attempt))
-            time.sleep(progressive_sleep)
-            event_data["workflow_run"] = _exec_get_with_retry(
-                workflow_description.api_url, token
-            )
-            workflow_description = get_workflow_description_from_event(event_data)
-            attempt += 1
-
-        if workflow_description.conclusion != "failure":
-            print(
-                "Workflow finished with status "
-                f"{workflow_description.conclusion}, exiting"
-            )
-            return True
-
-        print(
-            "Workflow",
-            workflow_description.url,
-            "completed and failed, let's check for rerun",
-        )
-
-        if workflow_description.name not in NEED_RERUN_WORKFLOWS:
-            print(
-                "Workflow",
-                workflow_description.name,
-                "not in list of rerunable workflows",
-            )
-            return True
-
-        if check_need_to_rerun(workflow_description, token):
-            rerun_workflow(workflow_description, token)
-            return True
-
-    return False
-
-
-def main(event):
-    token = get_cached_access_token()
-    event_data = json.loads(event["body"])
-    print("The body received:", event["body"])
-    workflow_description = get_workflow_description_from_event(event_data)
-
-    print("Got workflow description", workflow_description)
-    if check_workflow_completed(event_data, workflow_description, token):
-        return
-
-    if workflow_description.action != "requested":
-        print("Exiting, event action is", workflow_description.action)
-        return
-
-    if workflow_description.workflow_id in TRUSTED_WORKFLOW_IDS:
-        print("Workflow in trusted list, approving run")
-        approve_run(workflow_description, token)
-        return
-
-    pull_requests = _get_pull_requests_from(
-        workflow_description.repo_url,
-        workflow_description.fork_owner_login,
-        workflow_description.fork_branch,
-        token,
-    )
-
-    print("Got pull requests for workflow", len(pull_requests))
-    if len(pull_requests) != 1:
-        print(f"Can't continue with non-uniq PRs: {pull_requests}")
-        return
-
-    pull_request = pull_requests[0]
-    print("Pull request for workflow number", pull_request["number"])
-
-    author, author_orgs = get_pr_author_and_orgs(pull_request, token)
-    if is_trusted_contributor(author, author_orgs):
-        print("Contributor is trusted, approving run")
-        approve_run(workflow_description, token)
-        return
-
-    labels = {label["name"] for label in pull_request["labels"]}
-    if "can be tested" not in labels:
-        print("Label 'can be tested' is required for untrusted users")
-        return
-
-    changed_files = get_changed_files_for_pull_request(pull_request, token)
-    print(f"Totally have {len(changed_files)} changed files in PR:", changed_files)
-    if check_suspicious_changed_files(changed_files):
-        print(f"Pull Request {pull_request['number']} has suspicious changes")
-        if "manual approve" not in labels:
-            print("Label the PR as needed for manuall approve")
-            label_manual_approve(pull_request, token)
-    else:
-        print(f"Pull Request {pull_request['number']} has no suspicious changes")
-        approve_run(workflow_description, token)
-
-
-def handler(event, _):
-    try:
-        main(event)
-
-        return {
-            "statusCode": 200,
-            "headers": {"Content-Type": "application/json"},
-            "body": '{"status": "OK"}',
-        }
-    except Exception:
-        print("Received event: ", event)
-        raise
diff --git a/tests/ci/workflow_approve_rerun_lambda/build_and_deploy_archive.sh b/tests/ci/workflow_approve_rerun_lambda/build_and_deploy_archive.sh
deleted file mode 120000
index 96ba3fa024e..00000000000
--- a/tests/ci/workflow_approve_rerun_lambda/build_and_deploy_archive.sh
+++ /dev/null
@@ -1 +0,0 @@
-../team_keys_lambda/build_and_deploy_archive.sh
\ No newline at end of file
diff --git a/tests/ci/workflow_approve_rerun_lambda/lambda_shared b/tests/ci/workflow_approve_rerun_lambda/lambda_shared
deleted file mode 120000
index ba86e090f6c..00000000000
--- a/tests/ci/workflow_approve_rerun_lambda/lambda_shared
+++ /dev/null
@@ -1 +0,0 @@
-../lambda_shared_package/lambda_shared
\ No newline at end of file
diff --git a/tests/ci/workflow_approve_rerun_lambda/requirements.txt b/tests/ci/workflow_approve_rerun_lambda/requirements.txt
deleted file mode 100644
index 4cb3fba0f7b..00000000000
--- a/tests/ci/workflow_approve_rerun_lambda/requirements.txt
+++ /dev/null
@@ -1 +0,0 @@
-../lambda_shared_package[token]
diff --git a/tests/ci/workflow_jobs_lambda/app.py b/tests/ci/workflow_jobs_lambda/app.py
deleted file mode 100644
index 4210ca22da9..00000000000
--- a/tests/ci/workflow_jobs_lambda/app.py
+++ /dev/null
@@ -1,202 +0,0 @@
-#!/usr/bin/env python
-
-"""
-Lambda gets the workflow_job events, see
-https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#workflow_job
-
-Then it either posts it as is to the play.clickhouse.com, or anonymizes the sensitive
-fields for private repositories
-"""
-
-import json
-import logging
-from base64 import b64decode
-from dataclasses import dataclass
-from typing import Any, List, Optional
-
-from lambda_shared import ClickHouseHelper, InsertException, get_parameter_from_ssm
-
-logging.getLogger().setLevel(logging.INFO)
-
-
-@dataclass
-class WorkflowJob:
-    id: int
-    run_id: int
-    workflow_name: str
-    head_branch: str
-    run_url: str
-    run_attempt: int
-    node_id: str
-    head_sha: str
-    url: str
-    html_url: str
-    status: str
-    conclusion: str
-    started_at: str
-    completed_at: str
-    name: str
-    steps: int  # just number of steps, we don't keep steps
-    check_run_url: str
-    labels: List[str]
-    runner_id: int
-    runner_name: str
-    runner_group_id: int
-    runner_group_name: str
-    repository: str
-
-    def anonimyze_url(self, url: str) -> str:
-        return url.replace(self.repository, "ANONYMIZED_REPO")
-
-    def anonimyze(self):
-        anm = "ANONYMIZED"
-        self.workflow_name = anm
-        self.head_branch = anm
-        self.run_url = self.anonimyze_url(self.run_url)
-        self.node_id = anm
-        self.url = self.anonimyze_url(self.url)
-        self.html_url = self.anonimyze_url(self.html_url)
-        self.name = anm
-        self.check_run_url = self.anonimyze_url(self.check_run_url)
-        self.repository = anm
-
-    def as_dict(self) -> dict:
-        return self.__dict__
-
-
-CH_CLIENT = None  # type: Optional[ClickHouseHelper]
-
-
-def send_event_workflow_job(workflow_job: WorkflowJob) -> None:
-    # # SHOW CREATE TABLE default.workflow_jobs
-    # CREATE TABLE default.workflow_jobs UUID 'c0351924-8ccd-47a6-9db0-e28a9eee2fdf'
-    # (
-    #     `id` UInt64,
-    #     `run_id` UInt64,
-    #     `workflow_name` LowCardinality(String),
-    #     `head_branch` LowCardinality(String),
-    #     `run_url` String,
-    #     `run_attempt` UInt16,
-    #     `node_id` String,
-    #     `head_sha` String,
-    #     `url` String,
-    #     `html_url` String,
-    #     `status` Enum8('waiting' = 1, 'queued' = 2, 'in_progress' = 3, 'completed' = 4),
-    #     `conclusion` LowCardinality(String),
-    #     `started_at` DateTime,
-    #     `completed_at` DateTime,
-    #     `name` LowCardinality(String),
-    #     `steps` UInt16,
-    #     `check_run_url` String,
-    #     `labels` Array(LowCardinality(String)),
-    #     `runner_id` UInt64,
-    #     `runner_name` String,
-    #     `runner_group_id` UInt64,
-    #     `runner_group_name` LowCardinality(String),
-    #     `repository` LowCardinality(String),
-    #     `updated_at` DateTime DEFAULT now()
-    # )
-    # ENGINE = ReplicatedMergeTree('/clickhouse/tables/c0351924-8ccd-47a6-9db0-e28a9eee2fdf/{shard}', '{replica}')
-    # PARTITION BY toStartOfMonth(started_at)
-    # ORDER BY (id, updated_at)
-    # SETTINGS index_granularity = 8192
-    global CH_CLIENT
-    CH_CLIENT = CH_CLIENT or ClickHouseHelper(
-        get_parameter_from_ssm("clickhouse-test-stat-url"),
-        get_parameter_from_ssm("clickhouse-test-stat-login"),
-        get_parameter_from_ssm("clickhouse-test-stat-password"),
-    )
-    try:
-        CH_CLIENT.insert_event_into(
-            "default", "workflow_jobs", workflow_job.as_dict(), False
-        )
-    except InsertException as ex:
-        logging.exception(
-            "Got an exception on insert, tryuing to update the client "
-            "credentials and repeat",
-            exc_info=ex,
-        )
-        CH_CLIENT = ClickHouseHelper(
-            get_parameter_from_ssm("clickhouse-test-stat-url"),
-            get_parameter_from_ssm("clickhouse-test-stat-login"),
-            get_parameter_from_ssm("clickhouse-test-stat-password"),
-        )
-        CH_CLIENT.insert_event_into(
-            "default", "workflow_jobs", workflow_job.as_dict(), False
-        )
-
-
-def killed_job(wf_job: dict) -> bool:
-    """a hack to identify the killed runner if "Complete job" is omit"""
-    if (
-        wf_job.get("status", "") != "completed"
-        or wf_job.get("conclusion", "") != "failure"
-    ):
-        # The task either success or in progress
-        return False
-    return not any(
-        step["name"] == "Complete job" and step["conclusion"] is not None
-        for step in wf_job["steps"]
-    )
-
-
-def handler(event: dict, context: Any) -> dict:
-    if event["isBase64Encoded"]:
-        event_data = json.loads(b64decode(event["body"]))
-    else:
-        event_data = json.loads(event["body"])
-
-    logging.info("Got the next raw event from the github hook: %s", event_data)
-    repo = event_data["repository"]
-    try:
-        wf_job = event_data["workflow_job"]
-    except KeyError:
-        logging.error("The event does not contain valid workflow_jobs data")
-        logging.error("The event data: %s", event)
-        logging.error("The context data: %s", context)
-
-    if killed_job(wf_job):
-        # for killed job we record 0
-        steps = 0
-    else:
-        # We record only finished steps
-        steps = sum(1 for st in wf_job["steps"] if st["conclusion"] is not None)
-
-    workflow_job = WorkflowJob(
-        wf_job["id"],
-        wf_job["run_id"],
-        wf_job["workflow_name"] or "",  # nullable
-        wf_job["head_branch"],
-        wf_job["run_url"],
-        wf_job["run_attempt"],
-        wf_job["node_id"],
-        wf_job["head_sha"],
-        wf_job["url"],
-        wf_job["html_url"],
-        wf_job["status"],
-        wf_job["conclusion"] or "",  # nullable
-        wf_job["started_at"],
-        wf_job["completed_at"] or "1970-01-01T00:00:00",  # nullable date
-        wf_job["name"],
-        steps,
-        wf_job["check_run_url"],
-        wf_job["labels"],
-        wf_job["runner_id"] or 0,  # nullable
-        wf_job["runner_name"] or "",  # nullable
-        wf_job["runner_group_id"] or 0,  # nullable
-        wf_job["runner_group_name"] or "",  # nullable
-        repo["full_name"],
-    )
-    logging.info(
-        "Got the next event (private_repo=%s): %s", repo["private"], workflow_job
-    )
-    if repo["private"]:
-        workflow_job.anonimyze()
-
-    send_event_workflow_job(workflow_job)
-
-    return {
-        "statusCode": 200,
-        "headers": {"Content-Type": "application/json"},
-        "body": '{"status": "OK"}',
-    }
diff --git a/tests/ci/workflow_jobs_lambda/build_and_deploy_archive.sh b/tests/ci/workflow_jobs_lambda/build_and_deploy_archive.sh
deleted file mode 120000
index 96ba3fa024e..00000000000
--- a/tests/ci/workflow_jobs_lambda/build_and_deploy_archive.sh
+++ /dev/null
@@ -1 +0,0 @@
-../team_keys_lambda/build_and_deploy_archive.sh
\ No newline at end of file
diff --git a/tests/ci/workflow_jobs_lambda/lambda_shared b/tests/ci/workflow_jobs_lambda/lambda_shared
deleted file mode 120000
index ba86e090f6c..00000000000
--- a/tests/ci/workflow_jobs_lambda/lambda_shared
+++ /dev/null
@@ -1 +0,0 @@
-../lambda_shared_package/lambda_shared
\ No newline at end of file
diff --git a/tests/ci/workflow_jobs_lambda/requirements.txt b/tests/ci/workflow_jobs_lambda/requirements.txt
deleted file mode 100644
index 098e04a9798..00000000000
--- a/tests/ci/workflow_jobs_lambda/requirements.txt
+++ /dev/null
@@ -1 +0,0 @@
-../lambda_shared_package

From cdadef78471b47d05d6d1c437a823b17f8867991 Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitlibar@clickhouse.com>
Date: Thu, 18 Jul 2024 21:26:33 +0200
Subject: [PATCH 0419/1488] Add more comments.

---
 src/Parsers/ASTViewTargets.h    | 21 +++++++++++++++++----
 src/Parsers/ParserViewTargets.h |  7 ++++++-
 2 files changed, 23 insertions(+), 5 deletions(-)

diff --git a/src/Parsers/ASTViewTargets.h b/src/Parsers/ASTViewTargets.h
index 33a7bc5fcb1..12182919f0e 100644
--- a/src/Parsers/ASTViewTargets.h
+++ b/src/Parsers/ASTViewTargets.h
@@ -9,15 +9,20 @@ namespace DB
 class ASTStorage;
 enum class Keyword : size_t;
 
-/// Information about the target table for a materialized view or a window view.
+/// Information about target tables (external or inner) of a materialized view or a window view.
+/// See ASTViewTargets for more details.
 struct ViewTarget
 {
     enum Kind
     {
-        /// Target table for a materialized view or a window view.
+        /// If `kind == ViewTarget::To` then `ViewTarget` contains information about the "TO" table of a materialized view or a window view:
+        ///     CREATE MATERIALIZED VIEW db.mv_name {TO [db.]to_target | ENGINE to_engine} AS SELECT ...
+        /// or
+        ///     CREATE WINDOW VIEW db.wv_name {TO [db.]to_target | ENGINE to_engine} AS SELECT ...
         To,
 
-        /// Table with intermediate results for a window view.
+        /// If `kind == ViewTarget::Inner` then `ViewTarget` contains information about the "INNER" table of a window view:
+        ///     CREATE WINDOW VIEW db.wv_name {INNER ENGINE inner_engine} AS SELECT ...
         Inner,
     };
 
@@ -42,7 +47,15 @@ std::string_view toString(ViewTarget::Kind kind);
 void parseFromString(ViewTarget::Kind & out, std::string_view str);
 
 
-/// Information about all the target tables for a view.
+/// Information about all target tables (external or inner) of a view.
+///
+/// For example, for a materialized view:
+///     CREATE MATERIALIZED VIEW db.mv_name [TO [db.]to_target | ENGINE to_engine] AS SELECT ...
+/// this class contains information about the "TO" table: its name and database (if it's external), its UUID and engine (if it's inner).
+///
+/// For a window view:
+///     CREATE WINDOW VIEW db.wv_name [TO [db.]to_target | ENGINE to_engine] [INNER ENGINE inner_engine] AS SELECT ...
+/// this class contains information about both the "TO" table and the "INNER" table.
 class ASTViewTargets : public IAST
 {
 public:
diff --git a/src/Parsers/ParserViewTargets.h b/src/Parsers/ParserViewTargets.h
index f5d1850e974..3af3c0b8df3 100644
--- a/src/Parsers/ParserViewTargets.h
+++ b/src/Parsers/ParserViewTargets.h
@@ -7,7 +7,12 @@
 namespace DB
 {
 
-/// Parses information about target views of a table.
+/// Parses information about target tables (external or inner) of a materialized view or a window view.
+/// The function parses one or multiple parts of a CREATE query looking like this:
+///     TO db.table_name
+///     TO INNER UUID 'XXX'
+///     {ENGINE / INNER ENGINE} TableEngine(arguments) [ORDER BY ...] [SETTINGS ...]
+/// Returns ASTViewTargets if succeeded.
 class ParserViewTargets : public IParserBase
 {
 public:

From d7250c1d63c561c14f41f5f1a18f79fe0efc5972 Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitlibar@clickhouse.com>
Date: Thu, 18 Jul 2024 21:27:04 +0200
Subject: [PATCH 0420/1488] Add function ASTCreateQuery::hasInnerUUIDs().

---
 src/Interpreters/InterpreterCreateQuery.cpp | 2 +-
 src/Parsers/ASTCreateQuery.cpp              | 7 +++++++
 src/Parsers/ASTCreateQuery.h                | 1 +
 3 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp
index 45e2881ae5c..faa91341a7c 100644
--- a/src/Interpreters/InterpreterCreateQuery.cpp
+++ b/src/Interpreters/InterpreterCreateQuery.cpp
@@ -1131,7 +1131,7 @@ void InterpreterCreateQuery::assertOrSetUUID(ASTCreateQuery & create, const Data
     }
     else
     {
-        bool has_uuid = (create.uuid != UUIDHelpers::Nil) || (create.targets && create.targets->hasInnerUUIDs());
+        bool has_uuid = (create.uuid != UUIDHelpers::Nil) || create.hasInnerUUIDs();
         if (has_uuid && !is_on_cluster && !internal)
         {
             /// We don't show the following error message either
diff --git a/src/Parsers/ASTCreateQuery.cpp b/src/Parsers/ASTCreateQuery.cpp
index 770a63c6e75..348b54203fc 100644
--- a/src/Parsers/ASTCreateQuery.cpp
+++ b/src/Parsers/ASTCreateQuery.cpp
@@ -563,6 +563,13 @@ UUID ASTCreateQuery::getTargetInnerUUID(ViewTarget::Kind target_kind) const
     return UUIDHelpers::Nil;
 }
 
+bool ASTCreateQuery::hasInnerUUIDs() const
+{
+    if (targets)
+        return targets->hasInnerUUIDs();
+    return false;
+}
+
 std::shared_ptr<ASTStorage> ASTCreateQuery::getTargetInnerEngine(ViewTarget::Kind target_kind) const
 {
     if (targets)
diff --git a/src/Parsers/ASTCreateQuery.h b/src/Parsers/ASTCreateQuery.h
index f751a09169c..08d26f28efa 100644
--- a/src/Parsers/ASTCreateQuery.h
+++ b/src/Parsers/ASTCreateQuery.h
@@ -167,6 +167,7 @@ public:
     StorageID getTargetTableID(ViewTarget::Kind target_kind) const;
     bool hasTargetTableID(ViewTarget::Kind target_kind) const;
     UUID getTargetInnerUUID(ViewTarget::Kind target_kind) const;
+    bool hasInnerUUIDs() const;
     std::shared_ptr<ASTStorage> getTargetInnerEngine(ViewTarget::Kind target_kind) const;
     void setTargetInnerEngine(ViewTarget::Kind target_kind, ASTPtr storage_def);
 

From 626e1f70cb6f46407a11771cce67c879a9f1a0e5 Mon Sep 17 00:00:00 2001
From: Nikita Fomichev <nikita.fomichev@clickhouse.com>
Date: Sun, 14 Jul 2024 19:33:19 +0200
Subject: [PATCH 0421/1488] Stateful tests: decrease tests timeout

---
 docker/test/stateful/run.sh | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/docker/test/stateful/run.sh b/docker/test/stateful/run.sh
index 35ffeee5438..857385f4715 100755
--- a/docker/test/stateful/run.sh
+++ b/docker/test/stateful/run.sh
@@ -4,6 +4,9 @@
 source /setup_export_logs.sh
 set -e -x
 
+MAX_RUN_TIME=${MAX_RUN_TIME:-3600}
+MAX_RUN_TIME=$((MAX_RUN_TIME == 0 ? 3600 : MAX_RUN_TIME))
+
 # Choose random timezone for this test run
 TZ="$(rg -v '#' /usr/share/zoneinfo/zone.tab | awk '{print $3}' | shuf | head -n1)"
 echo "Choosen random timezone $TZ"
@@ -242,7 +245,22 @@ function run_tests()
 }
 
 export -f run_tests
-timeout "$MAX_RUN_TIME" bash -c run_tests ||:
+
+function timeout_with_logging() {
+    local exit_code=0
+
+    timeout -s TERM --preserve-status "${@}" || exit_code="${?}"
+
+    if [[ "${exit_code}" -eq "124" ]]
+    then
+      echo "The command 'timeout ${*}' has been killed by timeout"
+    fi
+
+    return $exit_code
+}
+
+TIMEOUT=$((MAX_RUN_TIME - 700))
+timeout_with_logging "$TIMEOUT" bash -c run_tests ||:
 
 echo "Files in current directory"
 ls -la ./

From 17fc423062fea64c392d48028eb32957e185269a Mon Sep 17 00:00:00 2001
From: Nikita Fomichev <nikita.fomichev@clickhouse.com>
Date: Mon, 15 Jul 2024 11:58:56 +0200
Subject: [PATCH 0422/1488] Stateless tests: add `no-parallel` tag to high-load
 tests

---
 .../00840_long_concurrent_select_and_drop_deadlock.sh           | 2 +-
 .../queries/0_stateless/00965_shard_unresolvable_addresses.sql  | 2 +-
 tests/queries/0_stateless/01004_rename_deadlock.sh              | 2 +-
 tests/queries/0_stateless/01005_rwr_shard_deadlock.sh           | 2 +-
 tests/queries/0_stateless/01007_r1r2_w_r2r1_deadlock.sh         | 2 +-
 ...azy_database_concurrent_recreate_reattach_and_show_tables.sh | 2 +-
 .../01035_concurrent_move_partition_from_table_zookeeper.sh     | 2 +-
 .../0_stateless/01079_parallel_alter_modify_zookeeper_long.sh   | 2 +-
 .../0_stateless/01111_create_drop_replicated_db_stress.sh       | 2 +-
 tests/queries/0_stateless/01412_cache_dictionary_race.sh        | 2 +-
 .../0_stateless/01454_storagememory_data_race_challenge.sh      | 2 +-
 tests/queries/0_stateless/01475_read_subcolumns.sql             | 2 +-
 .../queries/0_stateless/01502_long_log_tinylog_deadlock_race.sh | 2 +-
 tests/queries/0_stateless/01533_multiple_nested.sql             | 2 +-
 tests/queries/0_stateless/01778_mmap_cache_infra.sql            | 1 +
 .../01921_concurrent_ttl_and_normal_merges_zookeeper_long.sh    | 2 +-
 tests/queries/0_stateless/02417_load_marks_async.sh             | 1 +
 .../0_stateless/03148_async_queries_in_query_log_errors.sh      | 1 +
 .../0_stateless/03164_s3_settings_for_queries_and_merges.sql    | 2 +-
 tests/queries/0_stateless/03198_dynamic_read_subcolumns.sql     | 2 +-
 20 files changed, 20 insertions(+), 17 deletions(-)

diff --git a/tests/queries/0_stateless/00840_long_concurrent_select_and_drop_deadlock.sh b/tests/queries/0_stateless/00840_long_concurrent_select_and_drop_deadlock.sh
index ae728c8d10d..f0bc52ee356 100755
--- a/tests/queries/0_stateless/00840_long_concurrent_select_and_drop_deadlock.sh
+++ b/tests/queries/0_stateless/00840_long_concurrent_select_and_drop_deadlock.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Tags: deadlock, no-debug
+# Tags: deadlock, no-debug, no-parallel
 
 # NOTE: database = $CLICKHOUSE_DATABASE is unwanted
 
diff --git a/tests/queries/0_stateless/00965_shard_unresolvable_addresses.sql b/tests/queries/0_stateless/00965_shard_unresolvable_addresses.sql
index 79600c6f67e..16b62c37d80 100644
--- a/tests/queries/0_stateless/00965_shard_unresolvable_addresses.sql
+++ b/tests/queries/0_stateless/00965_shard_unresolvable_addresses.sql
@@ -1,4 +1,4 @@
--- Tags: shard
+-- Tags: shard, no-parallel
 
 SET prefer_localhost_replica = 1;
 
diff --git a/tests/queries/0_stateless/01004_rename_deadlock.sh b/tests/queries/0_stateless/01004_rename_deadlock.sh
index f0adf136e94..60d67e1744a 100755
--- a/tests/queries/0_stateless/01004_rename_deadlock.sh
+++ b/tests/queries/0_stateless/01004_rename_deadlock.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Tags: deadlock
+# Tags: deadlock, no-parallel
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
diff --git a/tests/queries/0_stateless/01005_rwr_shard_deadlock.sh b/tests/queries/0_stateless/01005_rwr_shard_deadlock.sh
index ef352606b69..51e8cdbc8e8 100755
--- a/tests/queries/0_stateless/01005_rwr_shard_deadlock.sh
+++ b/tests/queries/0_stateless/01005_rwr_shard_deadlock.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Tags: deadlock, shard
+# Tags: deadlock, shard, no-parallel
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
diff --git a/tests/queries/0_stateless/01007_r1r2_w_r2r1_deadlock.sh b/tests/queries/0_stateless/01007_r1r2_w_r2r1_deadlock.sh
index 9f4b2241732..d602ff6df3a 100755
--- a/tests/queries/0_stateless/01007_r1r2_w_r2r1_deadlock.sh
+++ b/tests/queries/0_stateless/01007_r1r2_w_r2r1_deadlock.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Tags: deadlock
+# Tags: deadlock, no-parallel
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
diff --git a/tests/queries/0_stateless/01014_lazy_database_concurrent_recreate_reattach_and_show_tables.sh b/tests/queries/0_stateless/01014_lazy_database_concurrent_recreate_reattach_and_show_tables.sh
index 3046fcbcd73..e4b3a31b13f 100755
--- a/tests/queries/0_stateless/01014_lazy_database_concurrent_recreate_reattach_and_show_tables.sh
+++ b/tests/queries/0_stateless/01014_lazy_database_concurrent_recreate_reattach_and_show_tables.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Tags: no-fasttest
+# Tags: no-fasttest, no-parallel
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
diff --git a/tests/queries/0_stateless/01035_concurrent_move_partition_from_table_zookeeper.sh b/tests/queries/0_stateless/01035_concurrent_move_partition_from_table_zookeeper.sh
index 0d57bb25543..59a1e3cb980 100755
--- a/tests/queries/0_stateless/01035_concurrent_move_partition_from_table_zookeeper.sh
+++ b/tests/queries/0_stateless/01035_concurrent_move_partition_from_table_zookeeper.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Tags: zookeeper, no-fasttest
+# Tags: zookeeper, no-fasttest, long, no-parallel
 
 set -e
 
diff --git a/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper_long.sh b/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper_long.sh
index 399c9e488a4..382e3279711 100755
--- a/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper_long.sh
+++ b/tests/queries/0_stateless/01079_parallel_alter_modify_zookeeper_long.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Tags: long, zookeeper, no-fasttest
+# Tags: long, zookeeper, no-fasttest, no-parallel
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
diff --git a/tests/queries/0_stateless/01111_create_drop_replicated_db_stress.sh b/tests/queries/0_stateless/01111_create_drop_replicated_db_stress.sh
index 90128d7a8ad..619b6e91d11 100755
--- a/tests/queries/0_stateless/01111_create_drop_replicated_db_stress.sh
+++ b/tests/queries/0_stateless/01111_create_drop_replicated_db_stress.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Tags: race, zookeeper
+# Tags: race, zookeeper, no-parallel
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
diff --git a/tests/queries/0_stateless/01412_cache_dictionary_race.sh b/tests/queries/0_stateless/01412_cache_dictionary_race.sh
index 9aa39652021..36295ca01ea 100755
--- a/tests/queries/0_stateless/01412_cache_dictionary_race.sh
+++ b/tests/queries/0_stateless/01412_cache_dictionary_race.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Tags: race
+# Tags: race, no-parallel
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
diff --git a/tests/queries/0_stateless/01454_storagememory_data_race_challenge.sh b/tests/queries/0_stateless/01454_storagememory_data_race_challenge.sh
index 2bfd350ec51..fa9238041b1 100755
--- a/tests/queries/0_stateless/01454_storagememory_data_race_challenge.sh
+++ b/tests/queries/0_stateless/01454_storagememory_data_race_challenge.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Tags: race
+# Tags: race, no-parallel
 
 set -e
 
diff --git a/tests/queries/0_stateless/01475_read_subcolumns.sql b/tests/queries/0_stateless/01475_read_subcolumns.sql
index d6eec2f84a1..d387230d9e5 100644
--- a/tests/queries/0_stateless/01475_read_subcolumns.sql
+++ b/tests/queries/0_stateless/01475_read_subcolumns.sql
@@ -1,4 +1,4 @@
--- Tags: no-object-storage, no-random-settings
+-- Tags: no-object-storage, no-random-settings, no-parallel
 
 SET use_uncompressed_cache = 0;
 
diff --git a/tests/queries/0_stateless/01502_long_log_tinylog_deadlock_race.sh b/tests/queries/0_stateless/01502_long_log_tinylog_deadlock_race.sh
index b8efee89b4a..5a146968a3b 100755
--- a/tests/queries/0_stateless/01502_long_log_tinylog_deadlock_race.sh
+++ b/tests/queries/0_stateless/01502_long_log_tinylog_deadlock_race.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Tags: deadlock
+# Tags: deadlock, no-parallel
 
 set -e
 
diff --git a/tests/queries/0_stateless/01533_multiple_nested.sql b/tests/queries/0_stateless/01533_multiple_nested.sql
index 80e9fc7e2fb..4756bd4e919 100644
--- a/tests/queries/0_stateless/01533_multiple_nested.sql
+++ b/tests/queries/0_stateless/01533_multiple_nested.sql
@@ -1,4 +1,4 @@
--- Tags: no-object-storage, no-random-merge-tree-settings
+-- Tags: no-object-storage, no-random-merge-tree-settings, no-parallel
 -- no-s3 because read FileOpen metric
 DROP TABLE IF EXISTS nested;
 
diff --git a/tests/queries/0_stateless/01778_mmap_cache_infra.sql b/tests/queries/0_stateless/01778_mmap_cache_infra.sql
index 29a84c5507b..50fdb6ffbba 100644
--- a/tests/queries/0_stateless/01778_mmap_cache_infra.sql
+++ b/tests/queries/0_stateless/01778_mmap_cache_infra.sql
@@ -1,3 +1,4 @@
+-- Tags: no-parallel
 -- We check the existence of queries and metrics and don't check the results (a smoke test).
 
 SYSTEM DROP MMAP CACHE;
diff --git a/tests/queries/0_stateless/01921_concurrent_ttl_and_normal_merges_zookeeper_long.sh b/tests/queries/0_stateless/01921_concurrent_ttl_and_normal_merges_zookeeper_long.sh
index edffc0a3807..0dfde4997a7 100755
--- a/tests/queries/0_stateless/01921_concurrent_ttl_and_normal_merges_zookeeper_long.sh
+++ b/tests/queries/0_stateless/01921_concurrent_ttl_and_normal_merges_zookeeper_long.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Tags: long, zookeeper
+# Tags: long, zookeeper, no-parallel
 
 CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=error
 
diff --git a/tests/queries/0_stateless/02417_load_marks_async.sh b/tests/queries/0_stateless/02417_load_marks_async.sh
index 72b35a565df..950656e7ab6 100755
--- a/tests/queries/0_stateless/02417_load_marks_async.sh
+++ b/tests/queries/0_stateless/02417_load_marks_async.sh
@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
+# Tags: no-parallel
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
diff --git a/tests/queries/0_stateless/03148_async_queries_in_query_log_errors.sh b/tests/queries/0_stateless/03148_async_queries_in_query_log_errors.sh
index 9c290133bf9..2b4b96a9cbf 100755
--- a/tests/queries/0_stateless/03148_async_queries_in_query_log_errors.sh
+++ b/tests/queries/0_stateless/03148_async_queries_in_query_log_errors.sh
@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
+# Tags: no-parallel
 
 CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
diff --git a/tests/queries/0_stateless/03164_s3_settings_for_queries_and_merges.sql b/tests/queries/0_stateless/03164_s3_settings_for_queries_and_merges.sql
index 652b27b8a67..ac2070fbd76 100644
--- a/tests/queries/0_stateless/03164_s3_settings_for_queries_and_merges.sql
+++ b/tests/queries/0_stateless/03164_s3_settings_for_queries_and_merges.sql
@@ -1,4 +1,4 @@
--- Tags: no-random-settings, no-fasttest
+-- Tags: no-random-settings, no-fasttest, no-parallel
 
 SET allow_prefetched_read_pool_for_remote_filesystem=0;
 SET allow_prefetched_read_pool_for_local_filesystem=0;
diff --git a/tests/queries/0_stateless/03198_dynamic_read_subcolumns.sql b/tests/queries/0_stateless/03198_dynamic_read_subcolumns.sql
index 5d473064c68..bbe701f022b 100644
--- a/tests/queries/0_stateless/03198_dynamic_read_subcolumns.sql
+++ b/tests/queries/0_stateless/03198_dynamic_read_subcolumns.sql
@@ -1,4 +1,4 @@
--- Tags: no-random-settings, no-object-storage
+-- Tags: no-random-settings, no-object-storage, no-parallel
 -- Tag no-object-storage: this test relies on the number of opened files in MergeTree that can differ in object storages
 
 SET allow_experimental_dynamic_type = 1;

From bd10951db8d6f9abc0d1e74aa122adf7ed7f0843 Mon Sep 17 00:00:00 2001
From: Nikita Fomichev <nikita.fomichev@clickhouse.com>
Date: Tue, 16 Jul 2024 10:32:10 +0200
Subject: [PATCH 0423/1488] Stateless tests: tune MAX_RUN_TIME

---
 docker/test/stateless/run.sh      | 6 ------
 tests/ci/functional_test_check.py | 4 ++--
 2 files changed, 2 insertions(+), 8 deletions(-)

diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh
index 2b535f8dd23..54bc2f184b1 100755
--- a/docker/test/stateless/run.sh
+++ b/docker/test/stateless/run.sh
@@ -183,9 +183,6 @@ if [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
     --keeper_server.tcp_port 29181 --keeper_server.server_id 3 \
     --prometheus.port 29988 \
     --macros.shard s2   # It doesn't work :(
-
-    MAX_RUN_TIME=$((MAX_RUN_TIME < 9000 ? MAX_RUN_TIME : 9000))  # min(MAX_RUN_TIME, 2.5 hours)
-    MAX_RUN_TIME=$((MAX_RUN_TIME != 0 ? MAX_RUN_TIME : 9000))    # set to 2.5 hours if 0 (unlimited)
 fi
 
 if [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then
@@ -210,9 +207,6 @@ if [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then
     --keeper_server.tcp_port 19181 --keeper_server.server_id 2 \
     --prometheus.port 19988 \
     --macros.replica r2   # It doesn't work :(
-
-    MAX_RUN_TIME=$((MAX_RUN_TIME < 9000 ? MAX_RUN_TIME : 9000))  # min(MAX_RUN_TIME, 2.5 hours)
-    MAX_RUN_TIME=$((MAX_RUN_TIME != 0 ? MAX_RUN_TIME : 9000))    # set to 2.5 hours if 0 (unlimited)
 fi
 
 # Wait for the server to start, but not for too long.
diff --git a/tests/ci/functional_test_check.py b/tests/ci/functional_test_check.py
index 74dd4d8fbd7..66db082677f 100644
--- a/tests/ci/functional_test_check.py
+++ b/tests/ci/functional_test_check.py
@@ -112,8 +112,8 @@ def get_run_command(
     ]
 
     if flaky_check:
-        envs.append("-e NUM_TRIES=100")
-        envs.append("-e MAX_RUN_TIME=1800")
+        envs.append("-e NUM_TRIES=50")
+        envs.append("-e MAX_RUN_TIME=2800")
 
     envs += [f"-e {e}" for e in additional_envs]
 

From 466c9f55c143b22f9fd9697420be1079b0adee8c Mon Sep 17 00:00:00 2001
From: Nikita Fomichev <nikita.fomichev@clickhouse.com>
Date: Tue, 16 Jul 2024 11:37:16 +0200
Subject: [PATCH 0424/1488] Stateless tests: fix test `01053_ssd_dictionary.sh`

---
 tests/queries/0_stateless/01053_ssd_dictionary.sh             | 4 ++--
 tests/queries/0_stateless/01280_ssd_complex_key_dictionary.sh | 3 +--
 2 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/tests/queries/0_stateless/01053_ssd_dictionary.sh b/tests/queries/0_stateless/01053_ssd_dictionary.sh
index 00e5719a9a9..fdd40940ab5 100755
--- a/tests/queries/0_stateless/01053_ssd_dictionary.sh
+++ b/tests/queries/0_stateless/01053_ssd_dictionary.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 # Tags: no-parallel, no-fasttest
-# Tag no-fasttest: this test mistakenly requires acces to /var/lib/clickhouse -- can't run this locally, disabled
+# Tag no-fasttest: this test mistakenly requires access to /var/lib/clickhouse -- can't run this locally, disabled
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
@@ -9,7 +9,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 $CLICKHOUSE_CLIENT --allow_deprecated_database_ordinary=1 -n --query="
   DROP DATABASE IF EXISTS 01053_db;
 
-  CREATE DATABASE 01053_db Engine = Ordinary;
+  CREATE DATABASE 01053_db;
 
   DROP TABLE IF EXISTS 01053_db.table_for_dict;
 
diff --git a/tests/queries/0_stateless/01280_ssd_complex_key_dictionary.sh b/tests/queries/0_stateless/01280_ssd_complex_key_dictionary.sh
index 9a80820dd58..60a65b9a253 100755
--- a/tests/queries/0_stateless/01280_ssd_complex_key_dictionary.sh
+++ b/tests/queries/0_stateless/01280_ssd_complex_key_dictionary.sh
@@ -6,9 +6,8 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CURDIR"/../shell_config.sh
 
 $CLICKHOUSE_CLIENT -n --query="
-    set allow_deprecated_database_ordinary=1;
     DROP DATABASE IF EXISTS 01280_db;
-    CREATE DATABASE 01280_db Engine = Ordinary;
+    CREATE DATABASE 01280_db;
     DROP TABLE IF EXISTS 01280_db.table_for_dict;
     CREATE TABLE 01280_db.table_for_dict
     (

From 7c1ed1b445822fa91d6554497fd4b6d0f76ad8cb Mon Sep 17 00:00:00 2001
From: Nikita Fomichev <nikita.fomichev@clickhouse.com>
Date: Thu, 18 Jul 2024 19:48:12 +0200
Subject: [PATCH 0425/1488] Stateless tests: check parallelization

---
 docker/test/stateless/run.sh | 79 +-----------------------------------
 tests/clickhouse-test        | 74 ++++-----------------------------
 2 files changed, 10 insertions(+), 143 deletions(-)

diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh
index 54bc2f184b1..0647ed02839 100755
--- a/docker/test/stateless/run.sh
+++ b/docker/test/stateless/run.sh
@@ -12,12 +12,6 @@ MAX_RUN_TIME=$((MAX_RUN_TIME == 0 ? 7200 : MAX_RUN_TIME))
 USE_DATABASE_REPLICATED=${USE_DATABASE_REPLICATED:=0}
 USE_SHARED_CATALOG=${USE_SHARED_CATALOG:=0}
 
-RUN_SEQUENTIAL_TESTS_IN_PARALLEL=0
-
-if [[ "$USE_DATABASE_REPLICATED" -eq 1 ]] || [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then
-  RUN_SEQUENTIAL_TESTS_IN_PARALLEL=0
-fi
-
 # Choose random timezone for this test run.
 #
 # NOTE: that clickhouse-test will randomize session_timezone by itself as well
@@ -101,53 +95,6 @@ if [ "$NUM_TRIES" -gt "1" ]; then
     mkdir -p /var/run/clickhouse-server
 fi
 
-# Run a CH instance to execute sequential tests on it in parallel with all other tests.
-if [[ "$RUN_SEQUENTIAL_TESTS_IN_PARALLEL" -eq 1 ]]; then
-  mkdir -p /var/run/clickhouse-server3 /etc/clickhouse-server3 /var/lib/clickhouse3
-  cp -r -L /etc/clickhouse-server/* /etc/clickhouse-server3/
-
-  sudo chown clickhouse:clickhouse /var/run/clickhouse-server3 /var/lib/clickhouse3 /etc/clickhouse-server3/
-  sudo chown -R clickhouse:clickhouse /etc/clickhouse-server3/*
-
-  function replace(){
-      sudo find /etc/clickhouse-server3/ -type f -name '*.xml' -exec sed -i "$1" {} \;
-  }
-
-  replace "s|<port>9000</port>|<port>19000</port>|g"
-  replace "s|<port>9440</port>|<port>19440</port>|g"
-  replace "s|<port>9988</port>|<port>19988</port>|g"
-  replace "s|<port>9234</port>|<port>19234</port>|g"
-  replace "s|<port>9181</port>|<port>19181</port>|g"
-  replace "s|<https_port>8443</https_port>|<https_port>18443</https_port>|g"
-  replace "s|<tcp_port>9000</tcp_port>|<tcp_port>19000</tcp_port>|g"
-  replace "s|<tcp_port>9181</tcp_port>|<tcp_port>19181</tcp_port>|g"
-  replace "s|<tcp_port_secure>9440</tcp_port_secure>|<tcp_port_secure>19440</tcp_port_secure>|g"
-  replace "s|<tcp_with_proxy_port>9010</tcp_with_proxy_port>|<tcp_with_proxy_port>19010</tcp_with_proxy_port>|g"
-  replace "s|<mysql_port>9004</mysql_port>|<mysql_port>19004</mysql_port>|g"
-  replace "s|<postgresql_port>9005</postgresql_port>|<postgresql_port>19005</postgresql_port>|g"
-  replace "s|<interserver_http_port>9009</interserver_http_port>|<interserver_http_port>19009</interserver_http_port>|g"
-  replace "s|8123|18123|g"
-  replace "s|/var/lib/clickhouse/|/var/lib/clickhouse3/|g"
-  replace "s|/etc/clickhouse-server/|/etc/clickhouse-server3/|g"
-  # distributed cache
-  replace "s|<tcp_port>10001</tcp_port>|<tcp_port>10003</tcp_port>|g"
-  replace "s|<tcp_port>10002</tcp_port>|<tcp_port>10004</tcp_port>|g"
-
-  sudo -E -u clickhouse /usr/bin/clickhouse server --daemon --config /etc/clickhouse-server3/config.xml \
-  --pid-file /var/run/clickhouse-server3/clickhouse-server.pid \
-  -- --path /var/lib/clickhouse3/ --logger.stderr /var/log/clickhouse-server/stderr3.log \
-  --logger.log /var/log/clickhouse-server/clickhouse-server3.log --logger.errorlog /var/log/clickhouse-server/clickhouse-server3.err.log \
-  --tcp_port 19000 --tcp_port_secure 19440 --http_port 18123 --https_port 18443 --interserver_http_port 19009 --tcp_with_proxy_port 19010 \
-  --prometheus.port 19988 --keeper_server.raft_configuration.server.port 19234 --keeper_server.tcp_port 19181 \
-  --mysql_port 19004 --postgresql_port 19005
-
-  for _ in {1..100}
-  do
-      clickhouse-client --port 19000 --query "SELECT 1" && break
-      sleep 1
-  done
-fi
-
 # simplest way to forward env variables to server
 sudo -E -u clickhouse /usr/bin/clickhouse-server --config /etc/clickhouse-server/config.xml --daemon --pid-file /var/run/clickhouse-server/clickhouse-server.pid
 
@@ -217,7 +164,6 @@ do
 done
 
 setup_logs_replication
-
 attach_gdb_to_clickhouse || true  # FIXME: to not break old builds, clean on 2023-09-01
 
 function fn_exists() {
@@ -278,11 +224,7 @@ function run_tests()
     else
         # All other configurations are OK.
         ADDITIONAL_OPTIONS+=('--jobs')
-        ADDITIONAL_OPTIONS+=('5')
-    fi
-
-    if [[ "$RUN_SEQUENTIAL_TESTS_IN_PARALLEL" -eq 1 ]]; then
-        ADDITIONAL_OPTIONS+=('--run-sequential-tests-in-parallel')
+        ADDITIONAL_OPTIONS+=('7')
     fi
 
     if [[ -n "$RUN_BY_HASH_NUM" ]] && [[ -n "$RUN_BY_HASH_TOTAL" ]]; then
@@ -367,9 +309,6 @@ done
 # Because it's the simplest way to read it when server has crashed.
 sudo clickhouse stop ||:
 
-if [[ "$RUN_SEQUENTIAL_TESTS_IN_PARALLEL" -eq 1 ]]; then
-    sudo clickhouse stop --pid-path /var/run/clickhouse-server3 ||:
-fi
 
 if [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
     sudo clickhouse stop --pid-path /var/run/clickhouse-server1 ||:
@@ -387,12 +326,6 @@ rg -Fa "<Fatal>" /var/log/clickhouse-server/clickhouse-server.log ||:
 rg -A50 -Fa "============" /var/log/clickhouse-server/stderr.log ||:
 zstd --threads=0 < /var/log/clickhouse-server/clickhouse-server.log > /test_output/clickhouse-server.log.zst &
 
-if [[ "$RUN_SEQUENTIAL_TESTS_IN_PARALLEL" -eq 1 ]]; then
-  rg -Fa "<Fatal>" /var/log/clickhouse-server3/clickhouse-server.log ||:
-  rg -A50 -Fa "============" /var/log/clickhouse-server3/stderr.log ||:
-  zstd --threads=0 < /var/log/clickhouse-server3/clickhouse-server.log > /test_output/clickhouse-server3.log.zst &
-fi
-
 data_path_config="--path=/var/lib/clickhouse/"
 if [[ -n "$USE_S3_STORAGE_FOR_MERGE_TREE" ]] && [[ "$USE_S3_STORAGE_FOR_MERGE_TREE" -eq 1 ]]; then
     # We need s3 storage configuration (but it's more likely that clickhouse-local will fail for some reason)
@@ -413,10 +346,6 @@ if [ $failed_to_save_logs -ne 0 ]; then
     do
         clickhouse-local "$data_path_config" --only-system-tables --stacktrace -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.tsv.zst ||:
 
-        if [[ "$RUN_SEQUENTIAL_TESTS_IN_PARALLEL" -eq 1 ]]; then
-            clickhouse-local --path /var/lib/clickhouse3/ --only-system-tables --stacktrace -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.3.tsv.zst ||:
-        fi
-
         if [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
             clickhouse-local --path /var/lib/clickhouse1/ --only-system-tables --stacktrace -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.1.tsv.zst ||:
             clickhouse-local --path /var/lib/clickhouse2/ --only-system-tables --stacktrace -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.2.tsv.zst ||:
@@ -458,12 +387,6 @@ rm -rf /var/lib/clickhouse/data/system/*/
 tar -chf /test_output/store.tar /var/lib/clickhouse/store ||:
 tar -chf /test_output/metadata.tar /var/lib/clickhouse/metadata/*.sql ||:
 
-if [[ "$RUN_SEQUENTIAL_TESTS_IN_PARALLEL" -eq 1 ]]; then
-    rm -rf /var/lib/clickhouse3/data/system/*/
-    tar -chf /test_output/store.tar /var/lib/clickhouse3/store ||:
-    tar -chf /test_output/metadata.tar /var/lib/clickhouse3/metadata/*.sql ||:
-fi
-
 
 if [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
     rg -Fa "<Fatal>" /var/log/clickhouse-server/clickhouse-server1.log ||:
diff --git a/tests/clickhouse-test b/tests/clickhouse-test
index 60160e71e81..a8a6ce68b8f 100755
--- a/tests/clickhouse-test
+++ b/tests/clickhouse-test
@@ -2187,7 +2187,7 @@ def run_tests_array(all_tests_with_params: Tuple[List[str], int, TestSuite, bool
             description = ""
             test_case_name = removesuffix(test_case.name, ".gen", ".sql") + ": "
 
-            if is_concurrent or args.run_sequential_tests_in_parallel:
+            if is_concurrent:
                 description = f"{test_case_name:72}"
             else:
                 sys.stdout.flush()
@@ -2447,35 +2447,6 @@ def extract_key(key: str) -> str:
     )[1]
 
 
-def override_envs(*args_, **kwargs):
-    global args
-    args.client += " --port 19000"
-    args.http_port = 18123
-    args.https_port = 18443
-
-    updated_env = {
-        "CLICKHOUSE_CONFIG": "/etc/clickhouse-server3/config.xml",
-        "CLICKHOUSE_CONFIG_DIR": "/etc/clickhouse-server3",
-        "CLICKHOUSE_CONFIG_GREP": "/etc/clickhouse-server3/preprocessed/config.xml",
-        "CLICKHOUSE_USER_FILES": "/var/lib/clickhouse3/user_files",
-        "CLICKHOUSE_SCHEMA_FILES": "/var/lib/clickhouse3/format_schemas",
-        "CLICKHOUSE_PATH": "/var/lib/clickhouse3",
-        "CLICKHOUSE_PORT_TCP": "19000",
-        "CLICKHOUSE_PORT_TCP_SECURE": "19440",
-        "CLICKHOUSE_PORT_TCP_WITH_PROXY": "19010",
-        "CLICKHOUSE_PORT_HTTP": "18123",
-        "CLICKHOUSE_PORT_HTTPS": "18443",
-        "CLICKHOUSE_PORT_INTERSERVER": "19009",
-        "CLICKHOUSE_PORT_KEEPER": "19181",
-        "CLICKHOUSE_PORT_PROMTHEUS_PORT": "19988",
-        "CLICKHOUSE_PORT_MYSQL": "19004",
-        "CLICKHOUSE_PORT_POSTGRESQL": "19005",
-    }
-    os.environ.update(updated_env)
-
-    run_tests_array(*args_, **kwargs)
-
-
 def run_tests_process(*args, **kwargs):
     return run_tests_array(*args, **kwargs)
 
@@ -2519,24 +2490,6 @@ def do_run_tests(jobs, test_suite: TestSuite):
             processes.append(process)
             process.start()
 
-        if args.run_sequential_tests_in_parallel:
-            # Run parallel tests and sequential tests at the same time
-            # Sequential tests will use different ClickHouse instance
-            # In this process we can safely override values in `args` and `os.environ`
-            process = multiprocessing.Process(
-                target=override_envs,
-                args=(
-                    (
-                        test_suite.sequential_tests,
-                        len(test_suite.sequential_tests),
-                        test_suite,
-                        False,
-                    ),
-                ),
-            )
-            processes.append(process)
-            process.start()
-
         while processes:
             sys.stdout.flush()
             # Periodically check the server for hangs
@@ -2568,15 +2521,15 @@ def do_run_tests(jobs, test_suite: TestSuite):
 
             sleep(5)
 
-        if not args.run_sequential_tests_in_parallel:
-            run_tests_array(
-                (
-                    test_suite.sequential_tests,
-                    len(test_suite.sequential_tests),
-                    test_suite,
-                    False,
-                )
+        run_tests_array(
+            (
+                test_suite.sequential_tests,
+                len(test_suite.sequential_tests),
+                test_suite,
+                False,
             )
+        )
+
         return len(test_suite.sequential_tests) + len(test_suite.parallel_tests)
     else:
         num_tests = len(test_suite.all_tests)
@@ -3419,15 +3372,6 @@ def parse_args():
         help="Path to file for fatal logs from client",
     )
 
-    parser.add_argument(
-        "--run-sequential-tests-in-parallel",
-        action="store_true",
-        default=False,
-        help="If `true`, tests with the tag `no-parallel` will run on a "
-        "separate ClickHouse instance in parallel with other tests. "
-        "This is used in CI to make test jobs run faster.",
-    )
-
     return parser.parse_args()
 
 

From e806123856f5ded0f2e92f4f4b42c38132276c15 Mon Sep 17 00:00:00 2001
From: Igor Nikonov <igor@clickhouse.com>
Date: Thu, 18 Jul 2024 20:30:56 +0000
Subject: [PATCH 0426/1488] Fix non x86 build

---
 src/QueryPipeline/RemoteQueryExecutor.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/QueryPipeline/RemoteQueryExecutor.cpp b/src/QueryPipeline/RemoteQueryExecutor.cpp
index 87f634b8334..d7edbc9ed35 100644
--- a/src/QueryPipeline/RemoteQueryExecutor.cpp
+++ b/src/QueryPipeline/RemoteQueryExecutor.cpp
@@ -469,7 +469,7 @@ RemoteQueryExecutor::ReadResult RemoteQueryExecutor::read()
     return restartQueryWithoutDuplicatedUUIDs();
 }
 
-RemoteQueryExecutor::ReadResult RemoteQueryExecutor::readAsync(bool check_packet_type_only)
+RemoteQueryExecutor::ReadResult RemoteQueryExecutor::readAsync([[maybe_unused]] bool check_packet_type_only)
 {
 #if defined(OS_LINUX)
     if (!read_context || (resent_query && recreate_read_context))

From c021f9dcd363d01d46108a9084b90c9463c9116a Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Thu, 18 Jul 2024 22:56:15 +0200
Subject: [PATCH 0427/1488] Fix SHOW MERGES

---
 src/Interpreters/InterpreterShowTablesQuery.cpp | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/src/Interpreters/InterpreterShowTablesQuery.cpp b/src/Interpreters/InterpreterShowTablesQuery.cpp
index 51038aaca46..78580f2ec02 100644
--- a/src/Interpreters/InterpreterShowTablesQuery.cpp
+++ b/src/Interpreters/InterpreterShowTablesQuery.cpp
@@ -121,9 +121,18 @@ String InterpreterShowTablesQuery::getRewrittenQuery()
     if (query.merges)
     {
         WriteBufferFromOwnString rewritten_query;
-        rewritten_query << "SELECT table, database, round((elapsed * (1 / merges.progress)) - merges.elapsed, 2) AS estimate_complete, round(elapsed,2) elapsed, "
-                           "round(progress*100, 2) AS progress, is_mutation, formatReadableSize(total_size_bytes_compressed) AS size_compressed, "
-                           "formatReadableSize(memory_usage) AS memory_usage FROM system.merges";
+        rewritten_query << R"(
+            SELECT
+                table,
+                database,
+                merges.progress > 0 ? round(merges.elapsed * (1 - merges.progress) / merges.progress, 2) : NULL AS estimate_complete,
+                round(elapsed, 2) AS elapsed,
+                round(progress * 100, 2) AS progress,
+                is_mutation,
+                formatReadableSize(total_size_bytes_compressed) AS size_compressed,
+                formatReadableSize(memory_usage) AS memory_usage
+            FROM system.merges
+            )";
 
         if (!query.like.empty())
         {

From 76904d4ae645aaf9a4cfce938be23a0c60a03fb6 Mon Sep 17 00:00:00 2001
From: jsc0218 <jsc0218@gmail.com>
Date: Thu, 18 Jul 2024 21:08:35 +0000
Subject: [PATCH 0428/1488] support throw option

---
 src/Core/SettingsEnums.cpp                    |  5 +++
 src/Interpreters/InterpreterCreateQuery.cpp   | 21 ++++++++++++
 ...ojection_merge_special_mergetree.reference |  0
 ...206_projection_merge_special_mergetree.sql | 32 +++++++++++++++++++
 4 files changed, 58 insertions(+)
 create mode 100644 tests/queries/0_stateless/03206_projection_merge_special_mergetree.reference
 create mode 100644 tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql

diff --git a/src/Core/SettingsEnums.cpp b/src/Core/SettingsEnums.cpp
index 82e7d6db410..74b6c793849 100644
--- a/src/Core/SettingsEnums.cpp
+++ b/src/Core/SettingsEnums.cpp
@@ -177,6 +177,11 @@ IMPLEMENT_SETTING_ENUM(LightweightMutationProjectionMode, ErrorCodes::BAD_ARGUME
     {{"throw", LightweightMutationProjectionMode::THROW},
      {"drop", LightweightMutationProjectionMode::DROP}})
 
+IMPLEMENT_SETTING_ENUM(DeduplicateMergeProjectionMode, ErrorCodes::BAD_ARGUMENTS,
+    {{"throw", DeduplicateMergeProjectionMode::THROW},
+     {"drop", DeduplicateMergeProjectionMode::DROP},
+     {"rebuild", DeduplicateMergeProjectionMode::THROW}})
+
 IMPLEMENT_SETTING_AUTO_ENUM(LocalFSReadMethod, ErrorCodes::BAD_ARGUMENTS)
 
 IMPLEMENT_SETTING_ENUM(ParquetVersion, ErrorCodes::BAD_ARGUMENTS,
diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp
index ea10ad59db4..df5ec4525eb 100644
--- a/src/Interpreters/InterpreterCreateQuery.cpp
+++ b/src/Interpreters/InterpreterCreateQuery.cpp
@@ -1281,6 +1281,27 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create)
     /// Set and retrieve list of columns, indices and constraints. Set table engine if needed. Rewrite query in canonical way.
     TableProperties properties = getTablePropertiesAndNormalizeCreateQuery(create, mode);
 
+    /// Projection is only supported in (Replictaed)MergeTree.
+    if (std::string_view engine_name(create.storage->engine->name);
+        !properties.projections.empty() && engine_name != "MergeTree" && engine_name != "ReplicatedMergeTree")
+    {
+        bool projection_support = false;
+        if (auto * setting = create.storage->settings; setting != nullptr)
+        {
+            for (const auto & change : setting->changes)
+            {
+                if (change.name == "deduplicate_merge_projection_mode" && change.value != Field("throw"))
+                {
+                    projection_support = true;
+                    break;
+                }
+            }
+        }
+        if (!projection_support)
+            throw Exception(ErrorCodes::SUPPORT_IS_DISABLED,
+                "Projection is only supported in (Replictaed)MergeTree. Consider drop or rebuild option of deduplicate_merge_projection_mode.");
+    }
+
     /// Check type compatible for materialized dest table and select columns
     if (create.select && create.is_materialized_view && create.to_table_id && mode <= LoadingStrictnessLevel::CREATE)
     {
diff --git a/tests/queries/0_stateless/03206_projection_merge_special_mergetree.reference b/tests/queries/0_stateless/03206_projection_merge_special_mergetree.reference
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql b/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql
new file mode 100644
index 00000000000..6b5e516ad21
--- /dev/null
+++ b/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql
@@ -0,0 +1,32 @@
+DROP TABLE IF EXISTS tp;
+
+CREATE TABLE tp (
+    type Int32,
+    eventcnt UInt64,
+    PROJECTION p (select sum(eventcnt), type group by type)
+) engine = ReplacingMergeTree order by type;  -- { serverError SUPPORT_IS_DISABLED }
+
+CREATE TABLE tp (
+    type Int32,
+    eventcnt UInt64,
+    PROJECTION p (select sum(eventcnt), type group by type)
+) engine = ReplacingMergeTree order by type
+SETTINGS deduplicate_merge_projection_mode = 'throw';  -- { serverError SUPPORT_IS_DISABLED }
+
+CREATE TABLE tp (
+    type Int32,
+    eventcnt UInt64,
+    PROJECTION p (select sum(eventcnt), type group by type)
+) engine = ReplacingMergeTree order by type
+SETTINGS deduplicate_merge_projection_mode = 'drop';
+
+DROP TABLE tp;
+
+CREATE TABLE tp (
+    type Int32,
+    eventcnt UInt64,
+    PROJECTION p (select sum(eventcnt), type group by type)
+) engine = ReplacingMergeTree order by type
+SETTINGS deduplicate_merge_projection_mode = 'rebuild';
+
+DROP TABLE tp;

From 4ef9cb6d7aa32aeb56c26bfa6ecad94beacba540 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Thu, 18 Jul 2024 23:13:32 +0200
Subject: [PATCH 0429/1488] Fix style

---
 src/IO/ReadBufferFromFileBase.cpp  | 5 -----
 src/IO/ReadWriteBufferFromHTTP.cpp | 1 -
 2 files changed, 6 deletions(-)

diff --git a/src/IO/ReadBufferFromFileBase.cpp b/src/IO/ReadBufferFromFileBase.cpp
index d42b12ba49b..b7a1438cff8 100644
--- a/src/IO/ReadBufferFromFileBase.cpp
+++ b/src/IO/ReadBufferFromFileBase.cpp
@@ -5,11 +5,6 @@
 namespace DB
 {
 
-namespace ErrorCodes
-{
-    extern const int UNKNOWN_FILE_SIZE;
-}
-
 ReadBufferFromFileBase::ReadBufferFromFileBase() : BufferWithOwnMemory<SeekableReadBuffer>(0)
 {
 }
diff --git a/src/IO/ReadWriteBufferFromHTTP.cpp b/src/IO/ReadWriteBufferFromHTTP.cpp
index 2a62b11aa44..4d27a78c8dc 100644
--- a/src/IO/ReadWriteBufferFromHTTP.cpp
+++ b/src/IO/ReadWriteBufferFromHTTP.cpp
@@ -72,7 +72,6 @@ namespace ErrorCodes
     extern const int BAD_ARGUMENTS;
     extern const int CANNOT_SEEK_THROUGH_FILE;
     extern const int SEEK_POSITION_OUT_OF_BOUND;
-    extern const int UNKNOWN_FILE_SIZE;
 }
 
 std::unique_ptr<ReadBuffer> ReadWriteBufferFromHTTP::CallResult::transformToReadBuffer(size_t buf_size) &&

From 09141163b4444fca79f0ab05f3c012bd69ecf1c3 Mon Sep 17 00:00:00 2001
From: Max K <maxkaynov@gmail.com>
Date: Wed, 17 Jul 2024 18:19:54 +0200
Subject: [PATCH 0430/1488] remove lambda unit tests

---
 .github/workflows/backport_branches.yml | 4 ----
 .github/workflows/master.yml            | 4 ----
 .github/workflows/merge_queue.yml       | 4 ----
 .github/workflows/pull_request.yml      | 4 ----
 .github/workflows/release_branches.yml  | 4 ----
 tests/ci/.gitignore                     | 3 ---
 tests/ci/create_release.py              | 5 ++---
 7 files changed, 2 insertions(+), 26 deletions(-)

diff --git a/.github/workflows/backport_branches.yml b/.github/workflows/backport_branches.yml
index 60bd79560eb..9645d0e46de 100644
--- a/.github/workflows/backport_branches.yml
+++ b/.github/workflows/backport_branches.yml
@@ -36,10 +36,6 @@ jobs:
           cd "$GITHUB_WORKSPACE/tests/ci"
           echo "Testing the main ci directory"
           python3 -m unittest discover -s . -p 'test_*.py'
-          for dir in *_lambda/; do
-            echo "Testing $dir"
-            python3 -m unittest discover -s "$dir" -p 'test_*.py'
-          done
       - name: PrepareRunConfig
         id: runconfig
         run: |
diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index d27b1987532..09acef5eb8b 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -33,10 +33,6 @@ jobs:
 #          cd "$GITHUB_WORKSPACE/tests/ci"
 #          echo "Testing the main ci directory"
 #          python3 -m unittest discover -s . -p 'test_*.py'
-#          for dir in *_lambda/; do
-#            echo "Testing $dir"
-#            python3 -m unittest discover -s "$dir" -p 'test_*.py'
-#          done
       - name: PrepareRunConfig
         id: runconfig
         run: |
diff --git a/.github/workflows/merge_queue.yml b/.github/workflows/merge_queue.yml
index c08c3fb30ac..31a65ac3d15 100644
--- a/.github/workflows/merge_queue.yml
+++ b/.github/workflows/merge_queue.yml
@@ -30,10 +30,6 @@ jobs:
           cd "$GITHUB_WORKSPACE/tests/ci"
           echo "Testing the main ci directory"
           python3 -m unittest discover -s . -p 'test_*.py'
-          for dir in *_lambda/; do
-            echo "Testing $dir"
-            python3 -m unittest discover -s "$dir" -p 'test_*.py'
-          done
       - name: PrepareRunConfig
         id: runconfig
         run: |
diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
index 04ce4d29ce9..5124e4dba2c 100644
--- a/.github/workflows/pull_request.yml
+++ b/.github/workflows/pull_request.yml
@@ -48,10 +48,6 @@ jobs:
           cd "$GITHUB_WORKSPACE/tests/ci"
           echo "Testing the main ci directory"
           python3 -m unittest discover -s . -p 'test_*.py'
-          for dir in *_lambda/; do
-            echo "Testing $dir"
-            python3 -m unittest discover -s "$dir" -p 'test_*.py'
-          done
       - name: PrepareRunConfig
         id: runconfig
         run: |
diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml
index e1a97b91016..6a18999d74e 100644
--- a/.github/workflows/release_branches.yml
+++ b/.github/workflows/release_branches.yml
@@ -33,10 +33,6 @@ jobs:
           cd "$GITHUB_WORKSPACE/tests/ci"
           echo "Testing the main ci directory"
           python3 -m unittest discover -s . -p 'test_*.py'
-          for dir in *_lambda/; do
-            echo "Testing $dir"
-            python3 -m unittest discover -s "$dir" -p 'test_*.py'
-          done
       - name: PrepareRunConfig
         id: runconfig
         run: |
diff --git a/tests/ci/.gitignore b/tests/ci/.gitignore
index bc407a1f961..b5e58611040 100644
--- a/tests/ci/.gitignore
+++ b/tests/ci/.gitignore
@@ -1,4 +1 @@
-*_lambda/lambda-venv
-*_lambda/lambda-package
-*_lambda/lambda-package.zip
 gh_cache
\ No newline at end of file
diff --git a/tests/ci/create_release.py b/tests/ci/create_release.py
index e5723e00e2a..277134c3991 100755
--- a/tests/ci/create_release.py
+++ b/tests/ci/create_release.py
@@ -13,7 +13,6 @@ from git_helper import Git, GIT_PREFIX
 from ssh import SSHAgent
 from env_helper import GITHUB_REPOSITORY, S3_BUILDS_BUCKET
 from s3_helper import S3Helper
-from autoscale_runners_lambda.lambda_shared.pr import Labels
 from ci_utils import Shell
 from version_helper import (
     FILE_WITH_VERSION_PATH,
@@ -220,9 +219,9 @@ class ReleaseInfo:
         )
         with checkout(self.release_branch):
             with checkout_new(new_release_branch):
-                pr_labels = f"--label {Labels.RELEASE}"
+                pr_labels = f"--label {CI.Labels.RELEASE}"
                 if stable_release_type == VersionType.LTS:
-                    pr_labels += f" --label {Labels.RELEASE_LTS}"
+                    pr_labels += f" --label {CI.Labels.RELEASE_LTS}"
                 cmd_push_branch = (
                     f"{GIT_PREFIX} push --set-upstream origin {new_release_branch}"
                 )

From 0bf9346b07dc6fb07180a4221477512ba4eae024 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 19 Jul 2024 00:08:36 +0200
Subject: [PATCH 0431/1488] Update 03206_no_exceptions_clickhouse_local.sh

---
 .../queries/0_stateless/03206_no_exceptions_clickhouse_local.sh | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tests/queries/0_stateless/03206_no_exceptions_clickhouse_local.sh b/tests/queries/0_stateless/03206_no_exceptions_clickhouse_local.sh
index 86839a228dc..00efd1f4591 100755
--- a/tests/queries/0_stateless/03206_no_exceptions_clickhouse_local.sh
+++ b/tests/queries/0_stateless/03206_no_exceptions_clickhouse_local.sh
@@ -1,6 +1,4 @@
 #!/usr/bin/env bash
-# Tags: no-fasttest
-# Tag no-fasttest: In fasttest, ENABLE_LIBRARIES=0, so the grpc library is not built
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh

From b29518537061bda2d73a0a98e7b2d1fcdda3cedf Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 19 Jul 2024 00:11:29 +0200
Subject: [PATCH 0432/1488] Fix bad test
 03036_join_filter_push_down_equivalent_sets

---
 .../03036_join_filter_push_down_equivalent_sets.sql           | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/queries/0_stateless/03036_join_filter_push_down_equivalent_sets.sql b/tests/queries/0_stateless/03036_join_filter_push_down_equivalent_sets.sql
index 3893ec41c0e..d94a68aa4d8 100644
--- a/tests/queries/0_stateless/03036_join_filter_push_down_equivalent_sets.sql
+++ b/tests/queries/0_stateless/03036_join_filter_push_down_equivalent_sets.sql
@@ -7,13 +7,13 @@ CREATE TABLE test_table_1
 (
     id UInt64,
     value String
-) ENGINE=MergeTree ORDER BY id;
+) ENGINE=MergeTree ORDER BY id SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';
 
 CREATE TABLE test_table_2
 (
     id UInt64,
     value String
-) ENGINE=MergeTree ORDER BY id;
+) ENGINE=MergeTree ORDER BY id SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';
 
 INSERT INTO test_table_1 SELECT number, number FROM numbers(10);
 INSERT INTO test_table_2 SELECT number, number FROM numbers(10);

From 7d9b7cc79611751adc6d22aa47c5e179228a2840 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 15 Jul 2024 02:54:29 +0200
Subject: [PATCH 0433/1488] Introduce ASTDataType

---
 src/DataTypes/DataTypeFactory.cpp             | 14 ++---
 src/Databases/DatabasesCommon.cpp             |  2 +-
 .../PostgreSQL/DatabasePostgreSQL.cpp         | 15 +++--
 .../InterpreterShowCreateQuery.cpp            |  2 -
 .../MySQL/InterpretersMySQLDDLQuery.cpp       | 28 ++++-----
 .../formatWithPossiblyHidingSecrets.h         |  8 ++-
 src/Parsers/ASTColumnDeclaration.cpp          |  4 --
 src/Parsers/ASTDataType.cpp                   | 57 +++++++++++++++++++
 src/Parsers/ASTDataType.h                     | 36 ++++++++++++
 src/Parsers/ASTFunction.cpp                   |  4 --
 src/Parsers/ASTFunction.h                     |  2 +-
 src/Parsers/IAST.h                            | 15 +++--
 src/Parsers/ParserCreateQuery.cpp             |  7 ++-
 src/Parsers/ParserCreateQuery.h               |  8 +--
 src/Parsers/ParserDataType.cpp                | 14 ++---
 .../StorageMaterializedPostgreSQL.cpp         | 13 ++---
 src/Storages/WindowView/StorageWindowView.cpp |  3 +-
 17 files changed, 156 insertions(+), 76 deletions(-)
 create mode 100644 src/Parsers/ASTDataType.cpp
 create mode 100644 src/Parsers/ASTDataType.h

diff --git a/src/DataTypes/DataTypeFactory.cpp b/src/DataTypes/DataTypeFactory.cpp
index af37cde2846..45552e506cd 100644
--- a/src/DataTypes/DataTypeFactory.cpp
+++ b/src/DataTypes/DataTypeFactory.cpp
@@ -2,7 +2,7 @@
 #include <DataTypes/DataTypeCustom.h>
 #include <Parsers/parseQuery.h>
 #include <Parsers/ParserCreateQuery.h>
-#include <Parsers/ASTFunction.h>
+#include <Parsers/ASTDataType.h>
 #include <Parsers/ASTIdentifier.h>
 #include <Parsers/ASTLiteral.h>
 #include <Common/typeid_cast.h>
@@ -83,15 +83,9 @@ DataTypePtr DataTypeFactory::tryGet(const ASTPtr & ast) const
 template <bool nullptr_on_error>
 DataTypePtr DataTypeFactory::getImpl(const ASTPtr & ast) const
 {
-    if (const auto * func = ast->as<ASTFunction>())
+    if (const auto * type = ast->as<ASTDataType>())
     {
-        if (func->parameters)
-        {
-            if constexpr (nullptr_on_error)
-                return nullptr;
-            throw Exception(ErrorCodes::ILLEGAL_SYNTAX_FOR_DATA_TYPE, "Data type cannot have multiple parenthesized parameters.");
-        }
-        return getImpl<nullptr_on_error>(func->name, func->arguments);
+        return getImpl<nullptr_on_error>(type->name, type->arguments);
     }
 
     if (const auto * ident = ast->as<ASTIdentifier>())
@@ -107,7 +101,7 @@ DataTypePtr DataTypeFactory::getImpl(const ASTPtr & ast) const
 
     if constexpr (nullptr_on_error)
         return nullptr;
-    throw Exception(ErrorCodes::UNEXPECTED_AST_STRUCTURE, "Unexpected AST element for data type.");
+    throw Exception(ErrorCodes::UNEXPECTED_AST_STRUCTURE, "Unexpected AST element for data type: {}.", ast->getID());
 }
 
 DataTypePtr DataTypeFactory::get(const String & family_name_param, const ASTPtr & parameters) const
diff --git a/src/Databases/DatabasesCommon.cpp b/src/Databases/DatabasesCommon.cpp
index fe0baf30e57..cacba581745 100644
--- a/src/Databases/DatabasesCommon.cpp
+++ b/src/Databases/DatabasesCommon.cpp
@@ -149,7 +149,7 @@ ASTPtr getCreateQueryFromStorage(const StoragePtr & storage, const ASTPtr & ast_
             columns = metadata_ptr->columns.getAll();
         for (const auto & column_name_and_type: columns)
         {
-            const auto & ast_column_declaration = std::make_shared<ASTColumnDeclaration>();
+            const auto ast_column_declaration = std::make_shared<ASTColumnDeclaration>();
             ast_column_declaration->name = column_name_and_type.name;
             /// parser typename
             {
diff --git a/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp b/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp
index a846e23cd4f..032fc33ea16 100644
--- a/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp
+++ b/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp
@@ -12,9 +12,9 @@
 #include <Interpreters/evaluateConstantExpression.h>
 #include <Parsers/ASTCreateQuery.h>
 #include <Parsers/ASTFunction.h>
+#include <Parsers/ASTDataType.h>
 #include <Parsers/ParserCreateQuery.h>
 #include <Parsers/parseQuery.h>
-#include <Parsers/queryToString.h>
 #include <Common/escapeForFileName.h>
 #include <Common/parseRemoteDescription.h>
 #include <Databases/DatabaseFactory.h>
@@ -25,6 +25,7 @@
 #include <Core/Settings.h>
 #include <filesystem>
 
+
 namespace fs = std::filesystem;
 
 namespace DB
@@ -432,7 +433,7 @@ ASTPtr DatabasePostgreSQL::getCreateTableQueryImpl(const String & table_name, Co
     auto metadata_snapshot = storage->getInMemoryMetadataPtr();
     for (const auto & column_type_and_name : metadata_snapshot->getColumns().getOrdinary())
     {
-        const auto & column_declaration = std::make_shared<ASTColumnDeclaration>();
+        const auto column_declaration = std::make_shared<ASTColumnDeclaration>();
         column_declaration->name = column_type_and_name.name;
         column_declaration->type = getColumnDeclaration(column_type_and_name.type);
         columns_expression_list->children.emplace_back(column_declaration);
@@ -470,17 +471,15 @@ ASTPtr DatabasePostgreSQL::getColumnDeclaration(const DataTypePtr & data_type) c
     WhichDataType which(data_type);
 
     if (which.isNullable())
-        return makeASTFunction("Nullable", getColumnDeclaration(typeid_cast<const DataTypeNullable *>(data_type.get())->getNestedType()));
+        return makeASTDataType("Nullable", getColumnDeclaration(typeid_cast<const DataTypeNullable *>(data_type.get())->getNestedType()));
 
     if (which.isArray())
-        return makeASTFunction("Array", getColumnDeclaration(typeid_cast<const DataTypeArray *>(data_type.get())->getNestedType()));
+        return makeASTDataType("Array", getColumnDeclaration(typeid_cast<const DataTypeArray *>(data_type.get())->getNestedType()));
 
     if (which.isDateTime64())
-    {
-        return makeASTFunction("DateTime64", std::make_shared<ASTLiteral>(static_cast<UInt32>(6)));
-    }
+        return makeASTDataType("DateTime64", std::make_shared<ASTLiteral>(static_cast<UInt32>(6)));
 
-    return std::make_shared<ASTIdentifier>(data_type->getName());
+    return makeASTDataType(data_type->getName());
 }
 
 void registerDatabasePostgreSQL(DatabaseFactory & factory)
diff --git a/src/Interpreters/InterpreterShowCreateQuery.cpp b/src/Interpreters/InterpreterShowCreateQuery.cpp
index 0fca7b64d5a..ca5b7a3b5c1 100644
--- a/src/Interpreters/InterpreterShowCreateQuery.cpp
+++ b/src/Interpreters/InterpreterShowCreateQuery.cpp
@@ -1,9 +1,7 @@
 #include <Storages/IStorage.h>
 #include <Parsers/TablePropertiesQueriesASTs.h>
-#include <Parsers/formatAST.h>
 #include <Processors/Sources/SourceFromSingleChunk.h>
 #include <QueryPipeline/BlockIO.h>
-#include <DataTypes/DataTypesNumber.h>
 #include <DataTypes/DataTypeString.h>
 #include <Columns/ColumnString.h>
 #include <Common/typeid_cast.h>
diff --git a/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp b/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp
index 4821d607d0e..f73965cfcc8 100644
--- a/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp
+++ b/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp
@@ -3,6 +3,7 @@
 #include <Parsers/IAST.h>
 #include <Parsers/ASTLiteral.h>
 #include <Parsers/ASTFunction.h>
+#include <Parsers/ASTDataType.h>
 #include <Parsers/ASTIdentifier.h>
 #include <Parsers/ASTAlterQuery.h>
 #include <Parsers/ASTCreateQuery.h>
@@ -29,6 +30,7 @@
 #include <Interpreters/applyTableOverride.h>
 #include <Storages/IStorage.h>
 
+
 namespace DB
 {
 
@@ -95,22 +97,22 @@ NamesAndTypesList getColumnsList(const ASTExpressionList * columns_definition)
         }
 
         ASTPtr data_type = declare_column->data_type;
-        auto * data_type_function = data_type->as<ASTFunction>();
+        auto * data_type_node = data_type->as<ASTDataType>();
 
-        if (data_type_function)
+        if (data_type_node)
         {
-            String type_name_upper = Poco::toUpper(data_type_function->name);
+            String type_name_upper = Poco::toUpper(data_type_node->name);
 
             if (is_unsigned)
             {
                 /// For example(in MySQL): CREATE TABLE test(column_name INT NOT NULL ... UNSIGNED)
                 if (type_name_upper.find("INT") != String::npos && !endsWith(type_name_upper, "SIGNED")
                     && !endsWith(type_name_upper, "UNSIGNED"))
-                    data_type_function->name = type_name_upper + " UNSIGNED";
+                    data_type_node->name = type_name_upper + " UNSIGNED";
             }
 
             if (type_name_upper == "SET")
-                data_type_function->arguments.reset();
+                data_type_node->arguments.reset();
 
             /// Transforms MySQL ENUM's list of strings to ClickHouse string-integer pairs
             /// For example ENUM('a', 'b', 'c') -> ENUM('a'=1, 'b'=2, 'c'=3)
@@ -119,7 +121,7 @@ NamesAndTypesList getColumnsList(const ASTExpressionList * columns_definition)
             if (type_name_upper.find("ENUM") != String::npos)
             {
                 UInt16 i = 0;
-                for (ASTPtr & child : data_type_function->arguments->children)
+                for (ASTPtr & child : data_type_node->arguments->children)
                 {
                     auto new_child = std::make_shared<ASTFunction>();
                     new_child->name = "equals";
@@ -133,10 +135,10 @@ NamesAndTypesList getColumnsList(const ASTExpressionList * columns_definition)
             }
 
             if (type_name_upper == "DATE")
-                data_type_function->name = "Date32";
+                data_type_node->name = "Date32";
         }
         if (is_nullable)
-            data_type = makeASTFunction("Nullable", data_type);
+            data_type = makeASTDataType("Nullable", data_type);
 
         columns_name_and_type.emplace_back(declare_column->name, DataTypeFactory::instance().get(data_type));
     }
@@ -175,7 +177,7 @@ static ColumnsDescription createColumnsDescription(const NamesAndTypesList & col
     return columns_description;
 }
 
-static NamesAndTypesList getNames(const ASTFunction & expr, ContextPtr context, const NamesAndTypesList & columns)
+static NamesAndTypesList getNames(const ASTDataType & expr, ContextPtr context, const NamesAndTypesList & columns)
 {
     if (expr.arguments->children.empty())
         return NamesAndTypesList{};
@@ -219,9 +221,9 @@ static std::tuple<NamesAndTypesList, NamesAndTypesList, NamesAndTypesList, NameS
     ASTExpressionList * columns_definition, ASTExpressionList * indices_define, ContextPtr context, NamesAndTypesList & columns)
 {
     NameSet increment_columns;
-    auto keys = makeASTFunction("tuple");
-    auto unique_keys = makeASTFunction("tuple");
-    auto primary_keys = makeASTFunction("tuple");
+    auto keys = makeASTDataType("tuple");
+    auto unique_keys = makeASTDataType("tuple");
+    auto primary_keys = makeASTDataType("tuple");
 
     if (indices_define && !indices_define->children.empty())
     {
@@ -482,7 +484,7 @@ ASTs InterpreterCreateImpl::getRewrittenQueries(
     {
         auto column_declaration = std::make_shared<ASTColumnDeclaration>();
         column_declaration->name = name;
-        column_declaration->type = makeASTFunction(type);
+        column_declaration->type = makeASTDataType(type);
         column_declaration->default_specifier = "MATERIALIZED";
         column_declaration->default_expression = std::make_shared<ASTLiteral>(default_value);
         column_declaration->children.emplace_back(column_declaration->type);
diff --git a/src/Interpreters/formatWithPossiblyHidingSecrets.h b/src/Interpreters/formatWithPossiblyHidingSecrets.h
index 039bcbc2bca..ea8c295b169 100644
--- a/src/Interpreters/formatWithPossiblyHidingSecrets.h
+++ b/src/Interpreters/formatWithPossiblyHidingSecrets.h
@@ -1,11 +1,14 @@
 #pragma once
-#include "Access/ContextAccess.h"
-#include "Interpreters/Context.h"
+
+#include <Access/ContextAccess.h>
+#include <Interpreters/Context.h>
+
 
 #include <Core/Settings.h>
 
 namespace DB
 {
+
 struct SecretHidingFormatSettings
 {
     // We can't store const Context& as there's a dangerous usage {.ctx = *getContext()}
@@ -24,4 +27,5 @@ inline String format(const SecretHidingFormatSettings & settings)
 
     return settings.query.formatWithPossiblyHidingSensitiveData(settings.max_length, settings.one_line, show_secrets);
 }
+
 }
diff --git a/src/Parsers/ASTColumnDeclaration.cpp b/src/Parsers/ASTColumnDeclaration.cpp
index 4a8a3d2967d..c96499095d5 100644
--- a/src/Parsers/ASTColumnDeclaration.cpp
+++ b/src/Parsers/ASTColumnDeclaration.cpp
@@ -1,8 +1,6 @@
 #include <Parsers/ASTColumnDeclaration.h>
 #include <Common/quoteString.h>
 #include <IO/Operators.h>
-#include <Parsers/ASTLiteral.h>
-#include <DataTypes/DataTypeFactory.h>
 
 
 namespace DB
@@ -15,8 +13,6 @@ ASTPtr ASTColumnDeclaration::clone() const
 
     if (type)
     {
-        // Type may be an ASTFunction (e.g. `create table t (a Decimal(9,0))`),
-        // so we have to clone it properly as well.
         res->type = type->clone();
         res->children.push_back(res->type);
     }
diff --git a/src/Parsers/ASTDataType.cpp b/src/Parsers/ASTDataType.cpp
new file mode 100644
index 00000000000..3c17ae8c380
--- /dev/null
+++ b/src/Parsers/ASTDataType.cpp
@@ -0,0 +1,57 @@
+#include <Parsers/ASTDataType.h>
+#include <Common/SipHash.h>
+#include <IO/Operators.h>
+
+
+namespace DB
+{
+
+String ASTDataType::getID(char delim) const
+{
+    return "DataType" + (delim + name);
+}
+
+ASTPtr ASTDataType::clone() const
+{
+    auto res = std::make_shared<ASTDataType>(*this);
+    res->children.clear();
+
+    if (arguments)
+    {
+        res->arguments = arguments->clone();
+        res->children.push_back(res->arguments);
+    }
+
+    return res;
+}
+
+void ASTDataType::updateTreeHashImpl(SipHash & hash_state, bool) const
+{
+    hash_state.update(name.size());
+    hash_state.update(name);
+    /// Children are hashed automatically.
+}
+
+void ASTDataType::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const
+{
+    settings.ostr << (settings.hilite ? hilite_function : "") << name;
+
+    if (arguments && !arguments->children.empty())
+    {
+        settings.ostr << '(' << (settings.hilite ? hilite_none : "");
+
+        for (size_t i = 0, size = arguments->children.size(); i < size; ++i)
+        {
+            if (i != 0)
+                settings.ostr << ", ";
+
+            arguments->children[i]->formatImpl(settings, state, frame);
+        }
+
+        settings.ostr << (settings.hilite ? hilite_function : "") << ')';
+    }
+
+    settings.ostr << (settings.hilite ? hilite_none : "");
+}
+
+}
diff --git a/src/Parsers/ASTDataType.h b/src/Parsers/ASTDataType.h
new file mode 100644
index 00000000000..c8f3c6e2e9d
--- /dev/null
+++ b/src/Parsers/ASTDataType.h
@@ -0,0 +1,36 @@
+#pragma once
+
+#include <Parsers/ASTExpressionList.h>
+
+
+namespace DB
+{
+
+/// AST for data types, e.g. UInt8 or Tuple(x UInt8, y Enum(a = 1))
+class ASTDataType : public IAST
+{
+public:
+    String name;
+    ASTPtr arguments;
+
+    String getID(char delim) const override;
+    ASTPtr clone() const override;
+    void updateTreeHashImpl(SipHash & hash_state, bool ignore_aliases) const override;
+    void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override;
+};
+
+template <typename... Args>
+std::shared_ptr<ASTDataType> makeASTDataType(const String & name, Args &&... args)
+{
+    auto function = std::make_shared<ASTDataType>();
+
+    function->name = name;
+    function->arguments = std::make_shared<ASTExpressionList>();
+    function->children.push_back(function->arguments);
+
+    function->arguments->children = { std::forward<Args>(args)... };
+
+    return function;
+}
+
+}
diff --git a/src/Parsers/ASTFunction.cpp b/src/Parsers/ASTFunction.cpp
index f39229d7566..7d42b6d1e9c 100644
--- a/src/Parsers/ASTFunction.cpp
+++ b/src/Parsers/ASTFunction.cpp
@@ -7,7 +7,6 @@
 #include <Common/FieldVisitorToString.h>
 #include <Common/KnownObjectNames.h>
 #include <Common/SipHash.h>
-#include <Common/typeid_cast.h>
 #include <IO/Operators.h>
 #include <IO/WriteBufferFromString.h>
 #include <IO/WriteHelpers.h>
@@ -19,9 +18,6 @@
 #include <Parsers/queryToString.h>
 #include <Parsers/ASTSetQuery.h>
 #include <Parsers/FunctionSecretArgumentsFinderAST.h>
-#include <Core/QualifiedTableName.h>
-
-#include <boost/algorithm/string.hpp>
 
 
 using namespace std::literals;
diff --git a/src/Parsers/ASTFunction.h b/src/Parsers/ASTFunction.h
index 3a94691f25d..be2b6beae54 100644
--- a/src/Parsers/ASTFunction.h
+++ b/src/Parsers/ASTFunction.h
@@ -46,7 +46,7 @@ public:
 
     NullsAction nulls_action = NullsAction::EMPTY;
 
-    /// do not print empty parentheses if there are no args - compatibility with new AST for data types and engine names.
+    /// do not print empty parentheses if there are no args - compatibility with engine names.
     bool no_empty_args = false;
 
     /// Specifies where this function-like expression is used.
diff --git a/src/Parsers/IAST.h b/src/Parsers/IAST.h
index ee70fed0f07..e7e2ff2ec4a 100644
--- a/src/Parsers/IAST.h
+++ b/src/Parsers/IAST.h
@@ -271,16 +271,15 @@ public:
         throw Exception(ErrorCodes::LOGICAL_ERROR, "Unknown element in AST: {}", getID());
     }
 
-    // Secrets are displayed regarding show_secrets, then SensitiveDataMasker is applied.
-    // You can use Interpreters/formatWithPossiblyHidingSecrets.h for convenience.
+    /// Secrets are displayed regarding show_secrets, then SensitiveDataMasker is applied.
+    /// You can use Interpreters/formatWithPossiblyHidingSecrets.h for convenience.
     String formatWithPossiblyHidingSensitiveData(size_t max_length, bool one_line, bool show_secrets) const;
 
-    /*
-     * formatForLogging and formatForErrorMessage always hide secrets. This inconsistent
-     * behaviour is due to the fact such functions are called from Client which knows nothing about
-     * access rights and settings. Moreover, the only use case for displaying secrets are backups,
-     * and backup tools use only direct input and ignore logs and error messages.
-     */
+    /** formatForLogging and formatForErrorMessage always hide secrets. This inconsistent
+      * behaviour is due to the fact such functions are called from Client which knows nothing about
+      * access rights and settings. Moreover, the only use case for displaying secrets are backups,
+      * and backup tools use only direct input and ignore logs and error messages.
+      */
     String formatForLogging(size_t max_length = 0) const
     {
         return formatWithPossiblyHidingSensitiveData(max_length, true, false);
diff --git a/src/Parsers/ParserCreateQuery.cpp b/src/Parsers/ParserCreateQuery.cpp
index 014dc7bd3bf..fa232954cd6 100644
--- a/src/Parsers/ParserCreateQuery.cpp
+++ b/src/Parsers/ParserCreateQuery.cpp
@@ -5,6 +5,7 @@
 #include <Parsers/ASTExpressionList.h>
 #include <Parsers/ASTForeignKeyDeclaration.h>
 #include <Parsers/ASTFunction.h>
+#include <Parsers/ASTDataType.h>
 #include <Parsers/ASTIdentifier.h>
 #include <Parsers/ASTIndexDeclaration.h>
 #include <Parsers/ASTStatisticsDeclaration.h>
@@ -75,9 +76,9 @@ bool ParserNestedTable::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
     if (!close.ignore(pos, expected))
         return false;
 
-    auto func = std::make_shared<ASTFunction>();
+    auto func = std::make_shared<ASTDataType>();
     tryGetIdentifierNameInto(name, func->name);
-    // FIXME(ilezhankin): func->no_empty_args = true; ?
+
     func->arguments = columns;
     func->children.push_back(columns);
     node = func;
@@ -749,7 +750,7 @@ bool ParserCreateTableQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe
 
     auto * table_id = table->as<ASTTableIdentifier>();
 
-    // Shortcut for ATTACH a previously detached table
+    /// A shortcut for ATTACH a previously detached table.
     bool short_attach = attach && !from_path;
     if (short_attach && (!pos.isValid() || pos.get().type == TokenType::Semicolon))
     {
diff --git a/src/Parsers/ParserCreateQuery.h b/src/Parsers/ParserCreateQuery.h
index bb37491a366..7bd1d1bf588 100644
--- a/src/Parsers/ParserCreateQuery.h
+++ b/src/Parsers/ParserCreateQuery.h
@@ -101,17 +101,15 @@ class IParserColumnDeclaration : public IParserBase
 {
 public:
     explicit IParserColumnDeclaration(bool require_type_ = true, bool allow_null_modifiers_ = false, bool check_keywords_after_name_ = false)
-    : require_type(require_type_)
-    , allow_null_modifiers(allow_null_modifiers_)
-    , check_keywords_after_name(check_keywords_after_name_)
+        : require_type(require_type_)
+        , allow_null_modifiers(allow_null_modifiers_)
+        , check_keywords_after_name(check_keywords_after_name_)
     {
     }
 
     void enableCheckTypeKeyword() { check_type_keyword = true; }
 
 protected:
-    using ASTDeclarePtr = std::shared_ptr<ASTColumnDeclaration>;
-
     const char * getName() const  override{ return "column declaration"; }
 
     bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
diff --git a/src/Parsers/ParserDataType.cpp b/src/Parsers/ParserDataType.cpp
index ad33c7e4558..63800819899 100644
--- a/src/Parsers/ParserDataType.cpp
+++ b/src/Parsers/ParserDataType.cpp
@@ -1,6 +1,7 @@
 #include <Parsers/ParserDataType.h>
 
 #include <boost/algorithm/string/case_conv.hpp>
+#include <Parsers/ASTDataType.h>
 #include <Parsers/ASTFunction.h>
 #include <Parsers/ASTIdentifier.h>
 #include <Parsers/ASTIdentifier_fwd.h>
@@ -198,13 +199,12 @@ bool ParserDataType::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
         }
     }
 
-    auto function_node = std::make_shared<ASTFunction>();
-    function_node->name = type_name;
-    function_node->no_empty_args = true;
+    auto data_type_node = std::make_shared<ASTDataType>();
+    data_type_node->name = type_name;
 
     if (pos->type != TokenType::OpeningRoundBracket)
     {
-        node = function_node;
+        node = data_type_node;
         return true;
     }
     ++pos;
@@ -222,10 +222,10 @@ bool ParserDataType::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
         return false;
     ++pos;
 
-    function_node->arguments = expr_list_args;
-    function_node->children.push_back(function_node->arguments);
+    data_type_node->arguments = expr_list_args;
+    data_type_node->children.push_back(data_type_node->arguments);
 
-    node = function_node;
+    node = data_type_node;
     return true;
 }
 
diff --git a/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp b/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp
index a904b29e12f..f7dde509d4e 100644
--- a/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp
+++ b/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp
@@ -7,10 +7,8 @@
 #include <Common/parseAddress.h>
 #include <Common/assert_cast.h>
 
-#include <Core/Settings.h>
 #include <Core/PostgreSQL/Connection.h>
 
-#include <DataTypes/DataTypeString.h>
 #include <DataTypes/DataTypeNullable.h>
 #include <DataTypes/DataTypeArray.h>
 #include <DataTypes/DataTypesDecimal.h>
@@ -22,6 +20,7 @@
 #include <Processors/QueryPlan/QueryPlan.h>
 #include <Processors/QueryPlan/ReadFromPreparedSource.h>
 #include <Parsers/ASTFunction.h>
+#include <Parsers/ASTDataType.h>
 #include <Parsers/ASTIdentifier.h>
 #include <Parsers/ASTTablesInSelectQuery.h>
 #include <Parsers/ExpressionListParsers.h>
@@ -295,7 +294,7 @@ std::shared_ptr<ASTColumnDeclaration> StorageMaterializedPostgreSQL::getMaterial
     auto column_declaration = std::make_shared<ASTColumnDeclaration>();
 
     column_declaration->name = std::move(name);
-    column_declaration->type = makeASTFunction(type);
+    column_declaration->type = makeASTDataType(type);
 
     column_declaration->default_specifier = "MATERIALIZED";
     column_declaration->default_expression = std::make_shared<ASTLiteral>(default_value);
@@ -312,17 +311,17 @@ ASTPtr StorageMaterializedPostgreSQL::getColumnDeclaration(const DataTypePtr & d
     WhichDataType which(data_type);
 
     if (which.isNullable())
-        return makeASTFunction("Nullable", getColumnDeclaration(typeid_cast<const DataTypeNullable *>(data_type.get())->getNestedType()));
+        return makeASTDataType("Nullable", getColumnDeclaration(typeid_cast<const DataTypeNullable *>(data_type.get())->getNestedType()));
 
     if (which.isArray())
-        return makeASTFunction("Array", getColumnDeclaration(typeid_cast<const DataTypeArray *>(data_type.get())->getNestedType()));
+        return makeASTDataType("Array", getColumnDeclaration(typeid_cast<const DataTypeArray *>(data_type.get())->getNestedType()));
 
     /// getName() for decimal returns 'Decimal(precision, scale)', will get an error with it
     if (which.isDecimal())
     {
         auto make_decimal_expression = [&](std::string type_name)
         {
-            auto ast_expression = std::make_shared<ASTFunction>();
+            auto ast_expression = std::make_shared<ASTDataType>();
 
             ast_expression->name = type_name;
             ast_expression->arguments = std::make_shared<ASTExpressionList>();
@@ -354,7 +353,7 @@ ASTPtr StorageMaterializedPostgreSQL::getColumnDeclaration(const DataTypePtr & d
         return ast_expression;
     }
 
-    return std::make_shared<ASTIdentifier>(data_type->getName());
+    return makeASTDataType(data_type->getName());
 }
 
 
diff --git a/src/Storages/WindowView/StorageWindowView.cpp b/src/Storages/WindowView/StorageWindowView.cpp
index e15da0074d5..65a30b18e96 100644
--- a/src/Storages/WindowView/StorageWindowView.cpp
+++ b/src/Storages/WindowView/StorageWindowView.cpp
@@ -20,6 +20,7 @@
 #include <Parsers/ASTAsterisk.h>
 #include <Parsers/ASTCreateQuery.h>
 #include <Parsers/ASTDropQuery.h>
+#include <Parsers/ASTDataType.h>
 #include <Parsers/ASTIdentifier.h>
 #include <Parsers/ASTLiteral.h>
 #include <Parsers/ASTOrderByElement.h>
@@ -805,7 +806,7 @@ ASTPtr StorageWindowView::getInnerTableCreateQuery(const ASTPtr & inner_query, c
     {
         auto column_window = std::make_shared<ASTColumnDeclaration>();
         column_window->name = window_id_name;
-        column_window->type = std::make_shared<ASTIdentifier>("UInt32");
+        column_window->type = makeASTDataType("UInt32");
         columns_list->children.push_back(column_window);
     }
 

From 2860aa514d0d8837bd72a87390686b20e05ecae7 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 15 Jul 2024 03:50:38 +0200
Subject: [PATCH 0434/1488] Fix style

---
 src/DataTypes/DataTypeFactory.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/DataTypes/DataTypeFactory.cpp b/src/DataTypes/DataTypeFactory.cpp
index 45552e506cd..db6e1738d22 100644
--- a/src/DataTypes/DataTypeFactory.cpp
+++ b/src/DataTypes/DataTypeFactory.cpp
@@ -22,7 +22,6 @@ namespace ErrorCodes
 {
     extern const int LOGICAL_ERROR;
     extern const int UNKNOWN_TYPE;
-    extern const int ILLEGAL_SYNTAX_FOR_DATA_TYPE;
     extern const int UNEXPECTED_AST_STRUCTURE;
     extern const int DATA_TYPE_CANNOT_HAVE_ARGUMENTS;
 }

From 985f398925266c5867cd25fcdf655a5a306928fd Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 15 Jul 2024 05:33:44 +0200
Subject: [PATCH 0435/1488] Fix error

---
 src/Client/ClientBase.cpp                   |  1 -
 src/DataTypes/DataTypeAggregateFunction.cpp |  4 +--
 src/Parsers/ParserDataType.cpp              | 36 ++++++++++++++-------
 src/TableFunctions/ITableFunction.cpp       |  1 -
 4 files changed, 26 insertions(+), 16 deletions(-)

diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp
index 13dce05cabc..95d2dff54e6 100644
--- a/src/Client/ClientBase.cpp
+++ b/src/Client/ClientBase.cpp
@@ -70,7 +70,6 @@
 
 #include <boost/algorithm/string/case_conv.hpp>
 #include <boost/algorithm/string/replace.hpp>
-#include <boost/algorithm/string/split.hpp>
 #include <iostream>
 #include <filesystem>
 #include <limits>
diff --git a/src/DataTypes/DataTypeAggregateFunction.cpp b/src/DataTypes/DataTypeAggregateFunction.cpp
index ef7d86d2a81..09175617bf1 100644
--- a/src/DataTypes/DataTypeAggregateFunction.cpp
+++ b/src/DataTypes/DataTypeAggregateFunction.cpp
@@ -257,8 +257,8 @@ static DataTypePtr create(const ASTPtr & arguments)
     }
     else
         throw Exception(ErrorCodes::BAD_ARGUMENTS,
-                        "Unexpected AST element passed as aggregate function name for data type AggregateFunction. "
-                        "Must be identifier or function.");
+                        "Unexpected AST element {} passed as aggregate function name for data type AggregateFunction. "
+                        "Must be identifier or function", data_type_ast->getID());
 
     for (size_t i = argument_types_start_idx; i < arguments->children.size(); ++i)
         argument_types.push_back(DataTypeFactory::instance().get(arguments->children[i]));
diff --git a/src/Parsers/ParserDataType.cpp b/src/Parsers/ParserDataType.cpp
index 63800819899..78b5aaa93a6 100644
--- a/src/Parsers/ParserDataType.cpp
+++ b/src/Parsers/ParserDataType.cpp
@@ -70,20 +70,32 @@ private:
             DynamicArgumentsParser parser;
             return parser.parse(pos, node, expected);
         }
-
-        ParserNestedTable nested_parser;
-        ParserDataType data_type_parser;
-        ParserAllCollectionsOfLiterals literal_parser(false);
-
-        const char * operators[] = {"=", "equals", nullptr};
-        ParserLeftAssociativeBinaryOperatorList enum_parser(operators, std::make_unique<ParserLiteral>());
-
-        if (pos->type == TokenType::BareWord && std::string_view(pos->begin, pos->size()) == "Nested")
+        else if (type_name == "Nested")
+        {
+            ParserNestedTable nested_parser;
             return nested_parser.parse(pos, node, expected);
+        }
+        else if (type_name == "AggregateFunction")
+        {
+            ParserFunction function_parser;
+            ParserIdentifier identifier_parser;
+            ParserAllCollectionsOfLiterals literal_parser(false);
+            return literal_parser.parse(pos, node, expected)
+                || identifier_parser.parse(pos, node, expected)
+                || function_parser.parse(pos, node, expected);
+        }
+        else
+        {
+            ParserDataType data_type_parser;
+            ParserAllCollectionsOfLiterals literal_parser(false);
 
-        return enum_parser.parse(pos, node, expected)
-            || literal_parser.parse(pos, node, expected)
-            || data_type_parser.parse(pos, node, expected);
+            const char * operators[] = {"=", "equals", nullptr};
+            ParserLeftAssociativeBinaryOperatorList enum_parser(operators, std::make_unique<ParserLiteral>());
+
+            return enum_parser.parse(pos, node, expected)
+                || literal_parser.parse(pos, node, expected)
+                || data_type_parser.parse(pos, node, expected);
+        }
     }
 
     std::string_view type_name;
diff --git a/src/TableFunctions/ITableFunction.cpp b/src/TableFunctions/ITableFunction.cpp
index e5676c5c25d..916ff7ec022 100644
--- a/src/TableFunctions/ITableFunction.cpp
+++ b/src/TableFunctions/ITableFunction.cpp
@@ -1,5 +1,4 @@
 #include <TableFunctions/ITableFunction.h>
-#include <Interpreters/Context.h>
 #include <Storages/StorageFactory.h>
 #include <Storages/StorageTableFunction.h>
 #include <Access/Common/AccessFlags.h>

From 15be94ee14a7affe6643dd4c3ac1b104e3c69eeb Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Tue, 16 Jul 2024 05:47:27 +0200
Subject: [PATCH 0436/1488] Update src/Parsers/ASTDataType.h
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
---
 src/Parsers/ASTDataType.h | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/Parsers/ASTDataType.h b/src/Parsers/ASTDataType.h
index c8f3c6e2e9d..abe5cbb8626 100644
--- a/src/Parsers/ASTDataType.h
+++ b/src/Parsers/ASTDataType.h
@@ -22,13 +22,13 @@ public:
 template <typename... Args>
 std::shared_ptr<ASTDataType> makeASTDataType(const String & name, Args &&... args)
 {
-    auto function = std::make_shared<ASTDataType>();
+    auto data_type = std::make_shared<ASTDataType>();
 
-    function->name = name;
-    function->arguments = std::make_shared<ASTExpressionList>();
-    function->children.push_back(function->arguments);
+    data_type->name = name;
+    data_type->arguments = std::make_shared<ASTExpressionList>();
+    data_type->children.push_back(function->arguments);
 
-    function->arguments->children = { std::forward<Args>(args)... };
+    data_type->arguments->children = { std::forward<Args>(args)... };
 
     return function;
 }

From 87fa2c64e96c6bac67275207bd708ac231fa9fb6 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Tue, 16 Jul 2024 05:48:00 +0200
Subject: [PATCH 0437/1488] Apply review comments

---
 src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp b/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp
index f7dde509d4e..e795cd9c6c6 100644
--- a/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp
+++ b/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp
@@ -345,7 +345,7 @@ ASTPtr StorageMaterializedPostgreSQL::getColumnDeclaration(const DataTypePtr & d
 
     if (which.isDateTime64())
     {
-        auto ast_expression = std::make_shared<ASTFunction>();
+        auto ast_expression = std::make_shared<ASTDataType>();
 
         ast_expression->name = "DateTime64";
         ast_expression->arguments = std::make_shared<ASTExpressionList>();

From 88dce34be907863b6f3cf6890be87b0d7278a101 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Tue, 16 Jul 2024 05:48:31 +0200
Subject: [PATCH 0438/1488] Fix error

---
 src/Parsers/ASTDataType.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Parsers/ASTDataType.h b/src/Parsers/ASTDataType.h
index abe5cbb8626..d9427c2fd9e 100644
--- a/src/Parsers/ASTDataType.h
+++ b/src/Parsers/ASTDataType.h
@@ -26,11 +26,11 @@ std::shared_ptr<ASTDataType> makeASTDataType(const String & name, Args &&... arg
 
     data_type->name = name;
     data_type->arguments = std::make_shared<ASTExpressionList>();
-    data_type->children.push_back(function->arguments);
+    data_type->children.push_back(data_type->arguments);
 
     data_type->arguments->children = { std::forward<Args>(args)... };
 
-    return function;
+    return data_type;
 }
 
 }

From 4d4e0901881221b39cce0e0527f530fe90eb7ad3 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Tue, 16 Jul 2024 05:58:06 +0200
Subject: [PATCH 0439/1488] Fix build

---
 src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp b/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp
index e795cd9c6c6..e7b58841c4e 100644
--- a/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp
+++ b/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp
@@ -7,6 +7,7 @@
 #include <Common/parseAddress.h>
 #include <Common/assert_cast.h>
 
+#include <Core/Settings.h>
 #include <Core/PostgreSQL/Connection.h>
 
 #include <DataTypes/DataTypeNullable.h>

From 2832f8c57e37a1fc7d0c91b9ad621785c6d2a5a5 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 17 Jul 2024 04:59:04 +0200
Subject: [PATCH 0440/1488] Fix a typo

---
 src/Databases/DatabasesCommon.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Databases/DatabasesCommon.cpp b/src/Databases/DatabasesCommon.cpp
index cacba581745..fdbdb610275 100644
--- a/src/Databases/DatabasesCommon.cpp
+++ b/src/Databases/DatabasesCommon.cpp
@@ -164,7 +164,7 @@ ASTPtr getCreateQueryFromStorage(const StoragePtr & storage, const ASTPtr & ast_
                 if (!parser.parse(pos, ast_type, expected))
                 {
                     if (throw_on_error)
-                        throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot parser metadata of {}.{}",
+                        throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot parse metadata of {}.{}",
                                         backQuote(table_id.database_name), backQuote(table_id.table_name));
                     else
                         return nullptr;

From 6a7a4df1eca0cda4fd2efdcc8aaf2e8741f4cbcc Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 17 Jul 2024 05:02:26 +0200
Subject: [PATCH 0441/1488] Fix error

---
 src/Parsers/ParserDataType.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Parsers/ParserDataType.cpp b/src/Parsers/ParserDataType.cpp
index 78b5aaa93a6..af1a299a887 100644
--- a/src/Parsers/ParserDataType.cpp
+++ b/src/Parsers/ParserDataType.cpp
@@ -75,7 +75,7 @@ private:
             ParserNestedTable nested_parser;
             return nested_parser.parse(pos, node, expected);
         }
-        else if (type_name == "AggregateFunction")
+        else if (type_name == "AggregateFunction" || type_name == "SimpleAggregateFunction")
         {
             ParserFunction function_parser;
             ParserIdentifier identifier_parser;

From cc201745620490c7d885a9e45d9f4b92f7492c10 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 17 Jul 2024 05:19:58 +0200
Subject: [PATCH 0442/1488] Fix bad code: it was catching exceptions

---
 src/IO/WithFileSize.cpp                       | 48 +++++++++----------
 ...ry_and_native_with_binary_encoded_types.sh |  4 +-
 2 files changed, 25 insertions(+), 27 deletions(-)

diff --git a/src/IO/WithFileSize.cpp b/src/IO/WithFileSize.cpp
index 3660d962c08..8cea12fa200 100644
--- a/src/IO/WithFileSize.cpp
+++ b/src/IO/WithFileSize.cpp
@@ -14,40 +14,38 @@ namespace ErrorCodes
 }
 
 template <typename T>
-static size_t getFileSize(T & in)
+static std::optional<size_t> tryGetFileSize(T & in)
 {
     if (auto * with_file_size = dynamic_cast<WithFileSize *>(&in))
-    {
         return with_file_size->getFileSize();
-    }
+
+    return std::nullopt;
+}
+
+template <typename T>
+static size_t getFileSize(T & in)
+{
+    if (auto maybe_size = tryGetFileSize(in))
+        return *maybe_size;
 
     throw Exception(ErrorCodes::UNKNOWN_FILE_SIZE, "Cannot find out file size");
 }
 
-size_t getFileSizeFromReadBuffer(ReadBuffer & in)
-{
-    if (auto * delegate = dynamic_cast<ReadBufferFromFileDecorator *>(&in))
-    {
-        return getFileSize(delegate->getWrappedReadBuffer());
-    }
-    else if (auto * compressed = dynamic_cast<CompressedReadBufferWrapper *>(&in))
-    {
-        return getFileSize(compressed->getWrappedReadBuffer());
-    }
-
-    return getFileSize(in);
-}
-
 std::optional<size_t> tryGetFileSizeFromReadBuffer(ReadBuffer & in)
 {
-    try
-    {
-        return getFileSizeFromReadBuffer(in);
-    }
-    catch (...)
-    {
-        return std::nullopt;
-    }
+    if (auto * delegate = dynamic_cast<ReadBufferFromFileDecorator *>(&in))
+        return tryGetFileSize(delegate->getWrappedReadBuffer());
+    else if (auto * compressed = dynamic_cast<CompressedReadBufferWrapper *>(&in))
+        return tryGetFileSize(compressed->getWrappedReadBuffer());
+    return tryGetFileSize(in);
+}
+
+size_t getFileSizeFromReadBuffer(ReadBuffer & in)
+{
+    if (auto maybe_size = tryGetFileSizeFromReadBuffer(in))
+        return *maybe_size;
+
+    throw Exception(ErrorCodes::UNKNOWN_FILE_SIZE, "Cannot find out file size");
 }
 
 bool isBufferWithFileSize(const ReadBuffer & in)
diff --git a/tests/queries/0_stateless/03173_row_binary_and_native_with_binary_encoded_types.sh b/tests/queries/0_stateless/03173_row_binary_and_native_with_binary_encoded_types.sh
index 723b11ad620..0c585d36348 100755
--- a/tests/queries/0_stateless/03173_row_binary_and_native_with_binary_encoded_types.sh
+++ b/tests/queries/0_stateless/03173_row_binary_and_native_with_binary_encoded_types.sh
@@ -6,8 +6,8 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
 function test
 {
-  $CLICKHOUSE_LOCAL --allow_experimental_dynamic_type=1 --allow_experimental_variant_type=1 --output_format_binary_encode_types_in_binary_format=1 -q "select $1 as value format RowBinaryWithNamesAndTypes" | $CLICKHOUSE_LOCAL --input-format RowBinaryWithNamesAndTypes --input_format_binary_decode_types_in_binary_format=1 -q "select value, toTypeName(value) from table"
-  $CLICKHOUSE_LOCAL --allow_experimental_dynamic_type=1 --allow_experimental_variant_type=1 --output_format_native_encode_types_in_binary_format=1 -q "select $1 as value format Native" | $CLICKHOUSE_LOCAL --input-format Native --input_format_native_decode_types_in_binary_format=1 -q "select value, toTypeName(value) from table"
+    $CLICKHOUSE_LOCAL --allow_experimental_dynamic_type=1 --allow_experimental_variant_type=1 --output_format_binary_encode_types_in_binary_format=1 -q "select $1 as value format RowBinaryWithNamesAndTypes" | $CLICKHOUSE_LOCAL --input-format RowBinaryWithNamesAndTypes --input_format_binary_decode_types_in_binary_format=1 -q "select value, toTypeName(value) from table"
+    $CLICKHOUSE_LOCAL --allow_experimental_dynamic_type=1 --allow_experimental_variant_type=1 --output_format_native_encode_types_in_binary_format=1 -q "select $1 as value format Native" | $CLICKHOUSE_LOCAL --input-format Native --input_format_native_decode_types_in_binary_format=1 -q "select value, toTypeName(value) from table"
 }
 
 test "materialize(42)::UInt8"

From e1a24c9dd6f6320ce02714265e91e83f8dbf43f6 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 17 Jul 2024 05:45:57 +0200
Subject: [PATCH 0443/1488] Fix error

---
 src/Parsers/ParserDataType.cpp | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/Parsers/ParserDataType.cpp b/src/Parsers/ParserDataType.cpp
index af1a299a887..c5da4a32e92 100644
--- a/src/Parsers/ParserDataType.cpp
+++ b/src/Parsers/ParserDataType.cpp
@@ -3,7 +3,6 @@
 #include <boost/algorithm/string/case_conv.hpp>
 #include <Parsers/ASTDataType.h>
 #include <Parsers/ASTFunction.h>
-#include <Parsers/ASTIdentifier.h>
 #include <Parsers/ASTIdentifier_fwd.h>
 #include <Parsers/CommonParsers.h>
 #include <Parsers/ExpressionElementParsers.h>
@@ -80,9 +79,9 @@ private:
             ParserFunction function_parser;
             ParserIdentifier identifier_parser;
             ParserAllCollectionsOfLiterals literal_parser(false);
-            return literal_parser.parse(pos, node, expected)
-                || identifier_parser.parse(pos, node, expected)
-                || function_parser.parse(pos, node, expected);
+            return function_parser.parse(pos, node, expected)
+                || literal_parser.parse(pos, node, expected)
+                || identifier_parser.parse(pos, node, expected);
         }
         else
         {

From bc1146389617f9e9198d0dd25eea89a9206421ba Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 19 Jul 2024 03:16:23 +0200
Subject: [PATCH 0444/1488] Fix error

---
 src/Parsers/ParserDataType.cpp | 142 +++++++++++++++++++--------------
 1 file changed, 84 insertions(+), 58 deletions(-)

diff --git a/src/Parsers/ParserDataType.cpp b/src/Parsers/ParserDataType.cpp
index c5da4a32e92..2edb0141e12 100644
--- a/src/Parsers/ParserDataType.cpp
+++ b/src/Parsers/ParserDataType.cpp
@@ -46,60 +46,6 @@ private:
     }
 };
 
-/// Wrapper to allow mixed lists of nested and normal types.
-/// Parameters are either:
-/// - Nested table elements;
-/// - Enum element in form of 'a' = 1;
-/// - literal;
-/// - Dynamic type arguments;
-/// - another data type (or identifier);
-class ParserDataTypeArgument : public IParserBase
-{
-public:
-    explicit ParserDataTypeArgument(std::string_view type_name_) : type_name(type_name_)
-    {
-    }
-
-private:
-    const char * getName() const override { return "data type argument"; }
-    bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override
-    {
-        if (type_name == "Dynamic")
-        {
-            DynamicArgumentsParser parser;
-            return parser.parse(pos, node, expected);
-        }
-        else if (type_name == "Nested")
-        {
-            ParserNestedTable nested_parser;
-            return nested_parser.parse(pos, node, expected);
-        }
-        else if (type_name == "AggregateFunction" || type_name == "SimpleAggregateFunction")
-        {
-            ParserFunction function_parser;
-            ParserIdentifier identifier_parser;
-            ParserAllCollectionsOfLiterals literal_parser(false);
-            return function_parser.parse(pos, node, expected)
-                || literal_parser.parse(pos, node, expected)
-                || identifier_parser.parse(pos, node, expected);
-        }
-        else
-        {
-            ParserDataType data_type_parser;
-            ParserAllCollectionsOfLiterals literal_parser(false);
-
-            const char * operators[] = {"=", "equals", nullptr};
-            ParserLeftAssociativeBinaryOperatorList enum_parser(operators, std::make_unique<ParserLiteral>());
-
-            return enum_parser.parse(pos, node, expected)
-                || literal_parser.parse(pos, node, expected)
-                || data_type_parser.parse(pos, node, expected);
-        }
-    }
-
-    std::string_view type_name;
-};
-
 }
 
 bool ParserDataType::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
@@ -221,11 +167,91 @@ bool ParserDataType::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
     ++pos;
 
     /// Parse optional parameters
-    ParserList args_parser(std::make_unique<ParserDataTypeArgument>(type_name), std::make_unique<ParserToken>(TokenType::Comma));
-    ASTPtr expr_list_args;
+    ASTPtr expr_list_args = std::make_shared<ASTExpressionList>();
+
+    /// Allow mixed lists of nested and normal types.
+    /// Parameters are either:
+    /// - Nested table elements;
+    /// - Enum element in form of 'a' = 1;
+    /// - literal;
+    /// - Dynamic type arguments;
+    /// - another data type (or identifier);
+
+    size_t arg_num = 0;
+    bool have_version_of_aggregate_function = false;
+    while (true)
+    {
+        if (arg_num > 0)
+        {
+            if (pos->type == TokenType::Comma)
+                ++pos;
+            else
+                break;
+        }
+
+        ASTPtr arg;
+        if (type_name == "Dynamic")
+        {
+            DynamicArgumentsParser parser;
+            parser.parse(pos, arg, expected);
+        }
+        else if (type_name == "Nested")
+        {
+            ParserNestedTable nested_parser;
+            nested_parser.parse(pos, arg, expected);
+        }
+        else if (type_name == "AggregateFunction" || type_name == "SimpleAggregateFunction")
+        {
+            /// This is less trivial.
+            /// The first optional argument for AggregateFunction is a numeric literal, defining the version.
+            /// The next argument is the function name, optionally with parameters.
+            /// Subsequent arguments are data types.
+
+            if (arg_num == 0 && type_name == "AggregateFunction")
+            {
+                ParserUnsignedInteger version_parser;
+                if (version_parser.parse(pos, arg, expected))
+                {
+                    have_version_of_aggregate_function = true;
+                    expr_list_args->children.emplace_back(std::move(arg));
+                    ++arg_num;
+                    continue;
+                }
+            }
+
+            if (arg_num == (have_version_of_aggregate_function ? 1 : 0))
+            {
+                ParserFunction function_parser;
+                ParserIdentifier identifier_parser;
+                function_parser.parse(pos, arg, expected)
+                    || identifier_parser.parse(pos, arg, expected);
+            }
+            else
+            {
+                ParserDataType data_type_parser;
+                data_type_parser.parse(pos, arg, expected);
+            }
+        }
+        else
+        {
+            ParserDataType data_type_parser;
+            ParserAllCollectionsOfLiterals literal_parser(false);
+
+            const char * operators[] = {"=", "equals", nullptr};
+            ParserLeftAssociativeBinaryOperatorList enum_parser(operators, std::make_unique<ParserLiteral>());
+
+            enum_parser.parse(pos, arg, expected)
+               || literal_parser.parse(pos, arg, expected)
+               || data_type_parser.parse(pos, arg, expected);
+        }
+
+        if (!arg)
+            break;
+
+        expr_list_args->children.emplace_back(std::move(arg));
+        ++arg_num;
+    }
 
-    if (!args_parser.parse(pos, expr_list_args, expected))
-        return false;
     if (pos->type == TokenType::Comma)
         // ignore trailing comma inside Nested structures like Tuple(Int, Tuple(Int, String),)
         ++pos;

From 06594935f0e2dd9fc61882c7e643677de474fd7c Mon Sep 17 00:00:00 2001
From: jsc0218 <jsc0218@gmail.com>
Date: Fri, 19 Jul 2024 01:34:03 +0000
Subject: [PATCH 0445/1488] forbid optimize deduplicate

---
 src/Interpreters/InterpreterOptimizeQuery.cpp            | 5 +++++
 .../03206_projection_merge_special_mergetree.sql         | 9 +++++++++
 2 files changed, 14 insertions(+)

diff --git a/src/Interpreters/InterpreterOptimizeQuery.cpp b/src/Interpreters/InterpreterOptimizeQuery.cpp
index 907a01b0432..8d1ac3455b7 100644
--- a/src/Interpreters/InterpreterOptimizeQuery.cpp
+++ b/src/Interpreters/InterpreterOptimizeQuery.cpp
@@ -20,6 +20,7 @@ namespace DB
 namespace ErrorCodes
 {
     extern const int THERE_IS_NO_COLUMN;
+    extern const int NOT_IMPLEMENTED;
 }
 
 
@@ -42,6 +43,10 @@ BlockIO InterpreterOptimizeQuery::execute()
     auto metadata_snapshot = table->getInMemoryMetadataPtr();
     auto storage_snapshot = table->getStorageSnapshot(metadata_snapshot, getContext());
 
+    /// Don't allow OPTIMIZE DEDUPLICATE for all engines with projections.
+    if (ast.deduplicate && !metadata_snapshot->projections.empty())
+        throw Exception(ErrorCodes::NOT_IMPLEMENTED, "DEDUPLICATE with projections are not supported yet");
+
     // Empty list of names means we deduplicate by all columns, but user can explicitly state which columns to use.
     Names column_names;
     if (ast.deduplicate_by_columns)
diff --git a/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql b/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql
index 6b5e516ad21..06fb9a30aca 100644
--- a/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql
+++ b/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql
@@ -30,3 +30,12 @@ CREATE TABLE tp (
 SETTINGS deduplicate_merge_projection_mode = 'rebuild';
 
 DROP TABLE tp;
+
+
+-- don't allow OPTIMIZE DEDUPLICATE for all engines with projections
+CREATE TABLE test (
+    a INT PRIMARY KEY,
+    PROJECTION p (SELECT * ORDER BY a)
+) engine = MergeTree;
+
+OPTIMIZE TABLE test DEDUPLICATE;  -- { serverError NOT_IMPLEMENTED }

From f19de4effdf1e9d15acab69cecd882d7fd9b156b Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 19 Jul 2024 03:38:05 +0200
Subject: [PATCH 0446/1488] Fix bad tests long_select_and_alter, CC @alesapin

---
 .../0_stateless/01338_long_select_and_alter.reference       | 2 +-
 tests/queries/0_stateless/01338_long_select_and_alter.sh    | 6 +++---
 .../01338_long_select_and_alter_zookeeper.reference         | 2 +-
 .../0_stateless/01338_long_select_and_alter_zookeeper.sh    | 6 +++---
 4 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/tests/queries/0_stateless/01338_long_select_and_alter.reference b/tests/queries/0_stateless/01338_long_select_and_alter.reference
index c2678e7052e..921730b17ce 100644
--- a/tests/queries/0_stateless/01338_long_select_and_alter.reference
+++ b/tests/queries/0_stateless/01338_long_select_and_alter.reference
@@ -1,3 +1,3 @@
-10
 5
+10
 CREATE TABLE default.alter_mt\n(\n    `key` UInt64,\n    `value` UInt64\n)\nENGINE = MergeTree\nORDER BY key\nSETTINGS index_granularity = 8192
diff --git a/tests/queries/0_stateless/01338_long_select_and_alter.sh b/tests/queries/0_stateless/01338_long_select_and_alter.sh
index 2b0709162a3..2659e5c16cf 100755
--- a/tests/queries/0_stateless/01338_long_select_and_alter.sh
+++ b/tests/queries/0_stateless/01338_long_select_and_alter.sh
@@ -13,15 +13,15 @@ $CLICKHOUSE_CLIENT --query "INSERT INTO alter_mt SELECT number, toString(number)
 
 $CLICKHOUSE_CLIENT --function_sleep_max_microseconds_per_block 10000000 --query "SELECT count(distinct concat(value, '_')) FROM alter_mt WHERE not sleepEachRow(2)" &
 
-# to be sure that select took all required locks
+# To be sure that select took all required locks for better test sensitivity, although it isn't guaranteed (then the test will also succeed).
 sleep 2
 
 $CLICKHOUSE_CLIENT --query "ALTER TABLE alter_mt MODIFY COLUMN value UInt64"
 
-$CLICKHOUSE_CLIENT --query "SELECT sum(value) FROM alter_mt"
-
 wait
 
+$CLICKHOUSE_CLIENT --query "SELECT sum(value) FROM alter_mt"
+
 $CLICKHOUSE_CLIENT --query "SHOW CREATE TABLE alter_mt"
 
 $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS alter_mt"
diff --git a/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.reference b/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.reference
index b4ed8efab63..9c5ad0fa468 100644
--- a/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.reference
+++ b/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.reference
@@ -1,3 +1,3 @@
-10
 5
+10
 CREATE TABLE default.alter_mt\n(\n    `key` UInt64,\n    `value` UInt64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/01338_long_select_and_alter_zookeeper_default/alter_mt\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192
diff --git a/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.sh b/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.sh
index 41e0a12f369..6eb795408f4 100755
--- a/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.sh
+++ b/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.sh
@@ -13,15 +13,15 @@ $CLICKHOUSE_CLIENT --query "INSERT INTO alter_mt SELECT number, toString(number)
 
 $CLICKHOUSE_CLIENT --function_sleep_max_microseconds_per_block 10000000 --query "SELECT count(distinct concat(value, '_')) FROM alter_mt WHERE not sleepEachRow(2)" &
 
-# to be sure that select took all required locks
+# To be sure that select took all required locks for better test sensitivity, although it isn't guaranteed (then the test will also succeed).
 sleep 2
 
 $CLICKHOUSE_CLIENT --query "ALTER TABLE alter_mt MODIFY COLUMN value UInt64"
 
-$CLICKHOUSE_CLIENT --query "SELECT sum(value) FROM alter_mt"
-
 wait
 
+$CLICKHOUSE_CLIENT --query "SELECT sum(value) FROM alter_mt"
+
 $CLICKHOUSE_CLIENT --query "SHOW CREATE TABLE alter_mt"
 
 $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS alter_mt"

From 69ad57a2a52d50510b4ec5beea08c0c599846859 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 19 Jul 2024 03:58:07 +0200
Subject: [PATCH 0447/1488] Update
 03205_parallel_window_finctions_and_column_sparse_bug.sql

---
 .../03205_parallel_window_finctions_and_column_sparse_bug.sql   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/03205_parallel_window_finctions_and_column_sparse_bug.sql b/tests/queries/0_stateless/03205_parallel_window_finctions_and_column_sparse_bug.sql
index f2391e0d165..4cc54bb5ac2 100644
--- a/tests/queries/0_stateless/03205_parallel_window_finctions_and_column_sparse_bug.sql
+++ b/tests/queries/0_stateless/03205_parallel_window_finctions_and_column_sparse_bug.sql
@@ -6,7 +6,7 @@ insert into t values (1, 0);
 insert into t values (1, 1);
 insert into t values (1, 0)(1, 1);
 
-SELECT d, c, row_number() over (partition by d order by c) as c8 FROM t qualify c8=1 order by d settings max_threads=2;
+SELECT d, c, row_number() over (partition by d order by c) as c8 FROM t qualify c8=1 order by d settings max_threads=2, allow_experimental_analyzer = 1;
 SELECT '---';
 SELECT d, c, row_number() over (partition by d order by c) as c8 FROM t order by d, c8 settings max_threads=2;
 SELECT '---';

From f6380dac0cbf08f913069606beeea433fb7f8857 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 19 Jul 2024 04:03:28 +0200
Subject: [PATCH 0448/1488] Fix tidy.

---
 src/Databases/IDatabase.h | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/Databases/IDatabase.h b/src/Databases/IDatabase.h
index ce5a52b1b0f..f94326d220e 100644
--- a/src/Databases/IDatabase.h
+++ b/src/Databases/IDatabase.h
@@ -6,7 +6,6 @@
 #include <Interpreters/executeQuery.h>
 #include <Parsers/IAST_fwd.h>
 #include <QueryPipeline/BlockIO.h>
-#include <Storages/IStorage.h>
 #include <Storages/IStorage_fwd.h>
 #include <base/types.h>
 #include <Common/AsyncLoader.h>

From 48e7708d7bcf575123ea20cee9455e0a4cf26791 Mon Sep 17 00:00:00 2001
From: Xu Jia <jia.xu@clickhouse.com>
Date: Fri, 19 Jul 2024 10:29:13 +0800
Subject: [PATCH 0449/1488] fix compile error

---
 src/Interpreters/InterpreterAlterQuery.cpp  | 1 +
 src/Interpreters/InterpreterDeleteQuery.cpp | 1 +
 src/Interpreters/InterpreterInsertQuery.cpp | 8 ++++----
 3 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/Interpreters/InterpreterAlterQuery.cpp b/src/Interpreters/InterpreterAlterQuery.cpp
index 398fe31f1a9..9b5b5dfc20a 100644
--- a/src/Interpreters/InterpreterAlterQuery.cpp
+++ b/src/Interpreters/InterpreterAlterQuery.cpp
@@ -5,6 +5,7 @@
 #include <Access/Common/AccessRightsElement.h>
 #include <Common/typeid_cast.h>
 #include <Core/Settings.h>
+#include <Core/ServerSettings.h>
 #include <Databases/DatabaseFactory.h>
 #include <Databases/DatabaseReplicated.h>
 #include <Databases/IDatabase.h>
diff --git a/src/Interpreters/InterpreterDeleteQuery.cpp b/src/Interpreters/InterpreterDeleteQuery.cpp
index b37ec4de4ab..291c8e19db0 100644
--- a/src/Interpreters/InterpreterDeleteQuery.cpp
+++ b/src/Interpreters/InterpreterDeleteQuery.cpp
@@ -3,6 +3,7 @@
 
 #include <Access/ContextAccess.h>
 #include <Core/Settings.h>
+#include <Core/ServerSettings.h>
 #include <Databases/DatabaseReplicated.h>
 #include <Databases/IDatabase.h>
 #include <Interpreters/Context.h>
diff --git a/src/Interpreters/InterpreterInsertQuery.cpp b/src/Interpreters/InterpreterInsertQuery.cpp
index 181fb064b54..aef6c1249d5 100644
--- a/src/Interpreters/InterpreterInsertQuery.cpp
+++ b/src/Interpreters/InterpreterInsertQuery.cpp
@@ -6,6 +6,7 @@
 #include <AggregateFunctions/AggregateFunctionFactory.h>
 #include <Columns/ColumnNullable.h>
 #include <Core/Settings.h>
+#include <Core/ServerSettings.h>
 #include <Processors/Transforms/buildPushingToViewsChain.h>
 #include <DataTypes/DataTypeNullable.h>
 #include <Interpreters/DatabaseCatalog.h>
@@ -411,10 +412,6 @@ std::pair<std::vector<Chain>, std::vector<Chain>> InterpreterInsertQuery::buildP
     if (!running_group)
         running_group = std::make_shared<ThreadGroup>(getContext());
 
-    if (getContext()->getServerSettings().disable_insertion_and_mutation
-        && query.table_id.database_name != DatabaseCatalog::SYSTEM_DATABASE)
-        throw Exception(ErrorCodes::QUERY_IS_PROHIBITED, "Insert queries are prohibited");
-
     std::vector<Chain> sink_chains;
     std::vector<Chain> presink_chains;
 
@@ -737,6 +734,9 @@ BlockIO InterpreterInsertQuery::execute()
     const Settings & settings = getContext()->getSettingsRef();
     auto & query = query_ptr->as<ASTInsertQuery &>();
 
+    if (getContext()->getServerSettings().disable_insertion_and_mutation
+        && query.table_id.database_name != DatabaseCatalog::SYSTEM_DATABASE)
+        throw Exception(ErrorCodes::QUERY_IS_PROHIBITED, "Insert queries are prohibited");
 
     StoragePtr table = getTable(query);
     checkStorageSupportsTransactionsIfNeeded(table, getContext());

From b2adabd59eedad6805b7f8a250cb4a787d0e0998 Mon Sep 17 00:00:00 2001
From: Max K <maxkaynov@gmail.com>
Date: Fri, 19 Jul 2024 10:27:15 +0200
Subject: [PATCH 0450/1488] fix pylint

---
 utils/check-style/check-pylint | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/utils/check-style/check-pylint b/utils/check-style/check-pylint
index 8cfbc68ac96..919a1661e45 100755
--- a/utils/check-style/check-pylint
+++ b/utils/check-style/check-pylint
@@ -12,5 +12,4 @@ function xargs-pylint {
 
 # exclude ci unittest scripts from check: test_*
 find "$ROOT_PATH/tests" -maxdepth 2 -type f -exec file -F' ' --mime-type {} + | grep -v "/test_" | xargs-pylint 50
-# Beware, there lambdas are checked. All of them contain `app`, and it causes brain-cucumber-zalgo
-find "$ROOT_PATH/tests/ci" -mindepth 2 -type f -exec file -F' ' --mime-type {} + | xargs-pylint 1
+

From 5cca0f1ab712e9814ce9e68844e98c241ce2fba1 Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Fri, 19 Jul 2024 09:30:26 +0000
Subject: [PATCH 0451/1488] Add test test_storage_mysql/test.py::test_joins,
 close #65989

---
 tests/integration/test_storage_mysql/test.py | 57 ++++++++++++++++++++
 1 file changed, 57 insertions(+)

diff --git a/tests/integration/test_storage_mysql/test.py b/tests/integration/test_storage_mysql/test.py
index b131d8fe51c..5948954ff5f 100644
--- a/tests/integration/test_storage_mysql/test.py
+++ b/tests/integration/test_storage_mysql/test.py
@@ -908,6 +908,63 @@ def test_mysql_point(started_cluster):
     conn.close()
 
 
+def test_joins(started_cluster):
+    conn = get_mysql_conn(started_cluster, cluster.mysql8_ip)
+    drop_mysql_table(conn, "test_joins_mysql_users")
+    with conn.cursor() as cursor:
+        cursor.execute(
+            "CREATE TABLE clickhouse.test_joins_mysql_users (id INT NOT NULL, name varchar(50) NOT NULL, created TIMESTAMP, PRIMARY KEY (`id`)) ENGINE=InnoDB;"
+        )
+        cursor.execute(
+            f"INSERT INTO clickhouse.test_joins_mysql_users VALUES (469722, 'user@example.com', '2019-08-30 07:55:01')"
+        )
+
+    drop_mysql_table(conn, "test_joins_mysql_tickets")
+    with conn.cursor() as cursor:
+        cursor.execute(
+            "CREATE TABLE clickhouse.test_joins_mysql_tickets (id INT NOT NULL, subject varchar(50), created TIMESTAMP, creator INT NOT NULL, PRIMARY KEY (`id`)) ENGINE=InnoDB;"
+        )
+        cursor.execute(
+            f"INSERT INTO clickhouse.test_joins_mysql_tickets VALUES (281607, 'Feedback', '2024-06-25 12:09:41', 469722)"
+        )
+
+    conn.commit()
+
+    node1.query(
+        """
+        CREATE TABLE test_joins_table_users
+        (
+            `id` Int32,
+            `Name` String,
+            `Created` Nullable(DateTime)
+        )
+        ENGINE = MySQL('mysql80:3306', 'clickhouse', 'test_joins_mysql_users', 'root', 'clickhouse');
+        """
+    )
+
+    node1.query(
+        """
+        CREATE TABLE test_joins_table_tickets
+        (
+            `id` Int32,
+            `Subject` Nullable(String),
+            `Created` Nullable(DateTime),
+            `Creator` Int32
+        )
+        ENGINE = MySQL('mysql80:3306', 'clickhouse', 'test_joins_mysql_tickets', 'root', 'clickhouse');
+        """
+    )
+
+    node1.query(
+        """
+        SELECT test_joins_table_tickets.id, Subject, test_joins_table_tickets.Created, Name
+        FROM test_joins_table_tickets
+        LEFT JOIN test_joins_table_users ON test_joins_table_tickets.Creator = test_joins_table_users.id
+        WHERE test_joins_table_tickets.Created = '2024-06-25 12:09:41'
+        """
+    ) == "281607\tFeedback\t2024-06-25 12:09:41\tuser@example.com\n"
+
+
 if __name__ == "__main__":
     with contextmanager(started_cluster)() as cluster:
         for name, instance in list(cluster.instances.items()):

From 4cb862432c50848e3406899f5c7079b4cf1d62a8 Mon Sep 17 00:00:00 2001
From: Igor Nikonov <igor@clickhouse.com>
Date: Fri, 19 Jul 2024 09:34:20 +0000
Subject: [PATCH 0452/1488] Rename method

---
 src/Processors/IProcessor.h             | 2 +-
 src/Processors/Sources/RemoteSource.cpp | 2 +-
 src/Processors/Sources/RemoteSource.h   | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/Processors/IProcessor.h b/src/Processors/IProcessor.h
index 358983a2179..0776921a814 100644
--- a/src/Processors/IProcessor.h
+++ b/src/Processors/IProcessor.h
@@ -221,7 +221,7 @@ public:
         throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method 'schedule' is not implemented for {} processor", getName());
     }
 
-    virtual void asyncJobReady() {}
+    virtual void onAsyncJobReady() {}
 
     /** You must call this method if 'prepare' returned ExpandPipeline.
       * This method cannot access any port, but it can create new ports for current processor.
diff --git a/src/Processors/Sources/RemoteSource.cpp b/src/Processors/Sources/RemoteSource.cpp
index 3ec2356a121..587f6e2001b 100644
--- a/src/Processors/Sources/RemoteSource.cpp
+++ b/src/Processors/Sources/RemoteSource.cpp
@@ -104,7 +104,7 @@ void RemoteSource::work()
     ISource::work();
 }
 
-void RemoteSource::asyncJobReady()
+void RemoteSource::onAsyncJobReady()
 {
     chassert(async_read);
 
diff --git a/src/Processors/Sources/RemoteSource.h b/src/Processors/Sources/RemoteSource.h
index fa04985f101..2247c781584 100644
--- a/src/Processors/Sources/RemoteSource.h
+++ b/src/Processors/Sources/RemoteSource.h
@@ -32,7 +32,7 @@ public:
 
     int schedule() override { return fd; }
 
-    void asyncJobReady() override;
+    void onAsyncJobReady() override;
 
     void setStorageLimits(const std::shared_ptr<const StorageLimitsList> & storage_limits_) override;
 

From 983d2b474b3d3adba2d7c7f8824f0e45eb8ea68e Mon Sep 17 00:00:00 2001
From: Max K <maxkaynov@gmail.com>
Date: Fri, 19 Jul 2024 12:44:31 +0200
Subject: [PATCH 0453/1488] CI: Next fix for builds in CI workflow

---
 tests/ci/ci_config.py       | 18 ++++--------------
 tests/ci/ci_definitions.py  |  2 --
 tests/ci/ci_settings.py     | 10 ++++------
 tests/ci/test_ci_options.py |  6 +++---
 4 files changed, 11 insertions(+), 25 deletions(-)

diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py
index 98c7e99a495..90797b36d8e 100644
--- a/tests/ci/ci_config.py
+++ b/tests/ci/ci_config.py
@@ -48,24 +48,14 @@ class CI:
                 JobNames.INTEGRATION_TEST_ARM,
             ]
         ),
-        Tags.CI_SET_REQUIRED: LabelConfig(run_jobs=REQUIRED_CHECKS),
+        Tags.CI_SET_REQUIRED: LabelConfig(
+            run_jobs=REQUIRED_CHECKS
+            + [build for build in BuildNames if build != BuildNames.FUZZERS]
+        ),
         Tags.CI_SET_BUILDS: LabelConfig(
             run_jobs=[JobNames.STYLE_CHECK, JobNames.BUILD_CHECK]
             + [build for build in BuildNames if build != BuildNames.FUZZERS]
         ),
-        Tags.CI_SET_NON_REQUIRED: LabelConfig(
-            run_jobs=[job for job in JobNames if job not in REQUIRED_CHECKS]
-        ),
-        Tags.CI_SET_OLD_ANALYZER: LabelConfig(
-            run_jobs=[
-                JobNames.STYLE_CHECK,
-                JobNames.FAST_TEST,
-                BuildNames.PACKAGE_RELEASE,
-                BuildNames.PACKAGE_ASAN,
-                JobNames.STATELESS_TEST_OLD_ANALYZER_S3_REPLICATED_RELEASE,
-                JobNames.INTEGRATION_TEST_ASAN_OLD_ANALYZER,
-            ]
-        ),
         Tags.CI_SET_SYNC: LabelConfig(
             run_jobs=[
                 BuildNames.PACKAGE_ASAN,
diff --git a/tests/ci/ci_definitions.py b/tests/ci/ci_definitions.py
index d41a621bc2e..20d29d2337e 100644
--- a/tests/ci/ci_definitions.py
+++ b/tests/ci/ci_definitions.py
@@ -102,8 +102,6 @@ class Tags(metaclass=WithIter):
     CI_SET_ARM = "ci_set_arm"
     CI_SET_REQUIRED = "ci_set_required"
     CI_SET_BUILDS = "ci_set_builds"
-    CI_SET_NON_REQUIRED = "ci_set_non_required"
-    CI_SET_OLD_ANALYZER = "ci_set_old_analyzer"
 
     libFuzzer = "libFuzzer"
 
diff --git a/tests/ci/ci_settings.py b/tests/ci/ci_settings.py
index 54323ef868f..c29c5777dba 100644
--- a/tests/ci/ci_settings.py
+++ b/tests/ci/ci_settings.py
@@ -160,11 +160,8 @@ class CiSettings:
             else:
                 return False
 
-        if CI.is_build_job(job):
-            print(f"Build job [{job}] - always run")
-            return True
-
-        if self.exclude_keywords:
+        # do not exclude builds
+        if self.exclude_keywords and not CI.is_build_job(job):
             for keyword in self.exclude_keywords:
                 if keyword in normalize_string(job):
                     print(f"Job [{job}] matches Exclude keyword [{keyword}] - deny")
@@ -172,7 +169,8 @@ class CiSettings:
 
         to_deny = False
         if self.include_keywords:
-            if job == CI.JobNames.STYLE_CHECK:
+            # do not exclude builds
+            if job == CI.JobNames.STYLE_CHECK or CI.is_build_job(job):
                 # never exclude Style Check by include keywords
                 return True
             for keyword in self.include_keywords:
diff --git a/tests/ci/test_ci_options.py b/tests/ci/test_ci_options.py
index f71320abf2c..534885dda2b 100644
--- a/tests/ci/test_ci_options.py
+++ b/tests/ci/test_ci_options.py
@@ -8,8 +8,8 @@ from ci_config import CI
 
 _TEST_BODY_1 = """
 #### Run only:
-- [x] <!---ci_set_non_required--> Non required
-- [ ] <!---ci_set_arm--> Integration tests (arm64)
+- [ ] <!---ci_set_required--> Some Set
+- [x] <!---ci_set_arm--> Integration tests (arm64)
 - [x] <!---ci_include_foo--> Integration tests
 - [x] <!---ci_include_foo_Bar--> Integration tests
 - [ ] <!---ci_include_bar--> Integration tests
@@ -149,7 +149,7 @@ class TestCIOptions(unittest.TestCase):
         self.assertFalse(ci_options.no_ci_cache)
         self.assertTrue(ci_options.no_merge_commit)
         self.assertTrue(ci_options.woolen_wolfdog)
-        self.assertEqual(ci_options.ci_sets, ["ci_set_non_required"])
+        self.assertEqual(ci_options.ci_sets, ["ci_set_arm"])
         self.assertCountEqual(ci_options.include_keywords, ["foo", "foo_bar"])
         self.assertCountEqual(ci_options.exclude_keywords, ["foo", "foo_bar"])
 

From 74033e6fbcd63c741447cc5501dd27292e2d79f7 Mon Sep 17 00:00:00 2001
From: Sema Checherinda <Sema.Checherinda@clickhouse.com>
Date: Fri, 19 Jul 2024 13:14:19 +0200
Subject: [PATCH 0454/1488] fix var names

---
 .../03008_deduplication_random_setttings.sh            | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/tests/queries/0_stateless/03008_deduplication_random_setttings.sh b/tests/queries/0_stateless/03008_deduplication_random_setttings.sh
index 20513c35c71..e9f59138177 100755
--- a/tests/queries/0_stateless/03008_deduplication_random_setttings.sh
+++ b/tests/queries/0_stateless/03008_deduplication_random_setttings.sh
@@ -4,8 +4,6 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
-ENGINE="ReplicatedMergeTree"
-
 engine_options=("ReplicatedMergeTree" "MergeTree")
 engine=${engine_options[ $RANDOM % ${#engine_options[@]} ]}
 
@@ -30,7 +28,7 @@ insert_unique_blocks=${insert_unique_blocks_options[ $RANDOM % ${#insert_unique_
 
 THIS_RUN="Test case:"
 THIS_RUN+=" insert_method=$insert_method"
-THIS_RUN+=" engine=$ENGINE"
+THIS_RUN+=" engine=$engine"
 THIS_RUN+=" use_insert_token=$use_insert_token"
 THIS_RUN+=" single_thread=$single_thread"
 THIS_RUN+=" deduplicate_src_table=$deduplicate_src_table"
@@ -40,7 +38,7 @@ THIS_RUN+=" insert_unique_blocks=$insert_unique_blocks"
 $CLICKHOUSE_CLIENT --max_insert_block_size 1  -nmq "
     $(python3 $CURDIR/03008_deduplication.python insert_several_blocks_into_table \
         --insert-method $insert_method \
-        --table-engine $ENGINE \
+        --table-engine $engine \
         --use-insert-token $use_insert_token \
         --single-thread $single_thread \
         --deduplicate-src-table $deduplicate_src_table \
@@ -53,7 +51,7 @@ $CLICKHOUSE_CLIENT --max_insert_block_size 1  -nmq "
 $CLICKHOUSE_CLIENT --max_insert_block_size 1  -nmq "
     $(python3 $CURDIR/03008_deduplication.python mv_generates_several_blocks \
         --insert-method $insert_method \
-        --table-engine $ENGINE \
+        --table-engine $engine \
         --use-insert-token $use_insert_token \
         --single-thread $single_thread \
         --deduplicate-src-table $deduplicate_src_table \
@@ -66,7 +64,7 @@ $CLICKHOUSE_CLIENT --max_insert_block_size 1  -nmq "
 $CLICKHOUSE_CLIENT  --max_insert_block_size 1 -nmq "
     $(python3 $CURDIR/03008_deduplication.python several_mv_into_one_table \
         --insert-method $insert_method \
-        --table-engine $ENGINE \
+        --table-engine $engine \
         --use-insert-token $use_insert_token \
         --single-thread $single_thread \
         --deduplicate-src-table $deduplicate_src_table \

From 8349d260952a6daeb84c653c37ac000cf5302cfd Mon Sep 17 00:00:00 2001
From: Igor Nikonov <igor@clickhouse.com>
Date: Fri, 19 Jul 2024 11:25:34 +0000
Subject: [PATCH 0455/1488] Simplified implementation

---
 src/Processors/Sources/RemoteSource.cpp   | 17 +++++------
 src/Processors/Sources/RemoteSource.h     |  2 +-
 src/QueryPipeline/RemoteQueryExecutor.cpp | 37 +++++++++++++++++++++++
 src/QueryPipeline/RemoteQueryExecutor.h   |  3 ++
 4 files changed, 49 insertions(+), 10 deletions(-)

diff --git a/src/Processors/Sources/RemoteSource.cpp b/src/Processors/Sources/RemoteSource.cpp
index 587f6e2001b..46c27676e12 100644
--- a/src/Processors/Sources/RemoteSource.cpp
+++ b/src/Processors/Sources/RemoteSource.cpp
@@ -89,9 +89,6 @@ ISource::Status RemoteSource::prepare()
 
 void RemoteSource::work()
 {
-    if (async_immediate_work.exchange(false))
-        return;
-
     /// Connection drain is a heavy operation that may take a long time.
     /// Therefore we move connection drain from prepare() to work(), and drain multiple connections in parallel.
     /// See issue: https://github.com/ClickHouse/ClickHouse/issues/60844
@@ -101,6 +98,13 @@ void RemoteSource::work()
         executor_finished = true;
         return;
     }
+
+    if (preprocessed_packet)
+    {
+        preprocessed_packet = false;
+        return;
+    }
+
     ISource::work();
 }
 
@@ -111,12 +115,7 @@ void RemoteSource::onAsyncJobReady()
     if (!was_query_sent)
         return;
 
-    auto res = query_executor->readAsync(/*check_packet_type_only=*/true);
-    if (res.type == RemoteQueryExecutor::ReadResult::Type::ParallelReplicasToken)
-    {
-        work();
-        async_immediate_work = true;
-    }
+    preprocessed_packet = query_executor->processParallelReplicaPacketIfAny();
 }
 
 std::optional<Chunk> RemoteSource::tryGenerate()
diff --git a/src/Processors/Sources/RemoteSource.h b/src/Processors/Sources/RemoteSource.h
index 2247c781584..22d3921708b 100644
--- a/src/Processors/Sources/RemoteSource.h
+++ b/src/Processors/Sources/RemoteSource.h
@@ -54,7 +54,7 @@ private:
     int fd = -1;
     size_t rows = 0;
     bool manually_add_rows_before_limit_counter = false;
-    std::atomic_bool async_immediate_work{false};
+    bool preprocessed_packet = false;
 };
 
 /// Totals source from RemoteQueryExecutor.
diff --git a/src/QueryPipeline/RemoteQueryExecutor.cpp b/src/QueryPipeline/RemoteQueryExecutor.cpp
index d7edbc9ed35..b15e31a120f 100644
--- a/src/QueryPipeline/RemoteQueryExecutor.cpp
+++ b/src/QueryPipeline/RemoteQueryExecutor.cpp
@@ -936,4 +936,41 @@ bool RemoteQueryExecutor::needToSkipUnavailableShard() const
     return context->getSettingsRef().skip_unavailable_shards && (0 == connections->size());
 }
 
+bool  RemoteQueryExecutor::processParallelReplicaPacketIfAny()
+{
+#if defined(OS_LINUX)
+    if (!read_context || (resent_query && recreate_read_context))
+    {
+        std::lock_guard lock(was_cancelled_mutex);
+        if (was_cancelled)
+            return false;
+
+        read_context = std::make_unique<ReadContext>(*this);
+        recreate_read_context = false;
+    }
+
+    {
+        std::lock_guard lock(was_cancelled_mutex);
+        if (was_cancelled)
+            return false;
+
+        chassert(!has_postponed_packet);
+
+        read_context->resume();
+        if (read_context->isInProgress()) // <- nothing to process
+            return false;
+
+        const auto packet_type = read_context->getPacketType();
+        if (packet_type == Protocol::Server::MergeTreeReadTaskRequest || packet_type == Protocol::Server::MergeTreeAllRangesAnnouncement)
+        {
+            processPacket(read_context->getPacket());
+            return true;
+        }
+
+        has_postponed_packet = true;
+        return false;
+    }
+#endif
+}
+
 }
diff --git a/src/QueryPipeline/RemoteQueryExecutor.h b/src/QueryPipeline/RemoteQueryExecutor.h
index 6849c3e0a07..6f56df71f1d 100644
--- a/src/QueryPipeline/RemoteQueryExecutor.h
+++ b/src/QueryPipeline/RemoteQueryExecutor.h
@@ -222,6 +222,9 @@ public:
 
     bool isReplicaUnavailable() const { return extension && extension->parallel_reading_coordinator && connections->size() == 0; }
 
+    /// return true if parallel replica packet was processed
+    bool processParallelReplicaPacketIfAny();
+
 private:
     RemoteQueryExecutor(
         const String & query_,

From 4e3fdfc2d6482d42b8e152911e24ee38b1bafc89 Mon Sep 17 00:00:00 2001
From: Alexander Gololobov <davenger@clickhouse.com>
Date: Fri, 19 Jul 2024 13:26:35 +0200
Subject: [PATCH 0456/1488] Save writer thread id for debugging

---
 src/Common/SharedMutex.cpp | 10 +++++++++-
 src/Common/SharedMutex.h   |  2 ++
 2 files changed, 11 insertions(+), 1 deletion(-)

diff --git a/src/Common/SharedMutex.cpp b/src/Common/SharedMutex.cpp
index 1df09ca998a..7b00ef0b28b 100644
--- a/src/Common/SharedMutex.cpp
+++ b/src/Common/SharedMutex.cpp
@@ -1,4 +1,5 @@
 #include <Common/SharedMutex.h>
+#include <base/getThreadId.h>
 
 #ifdef OS_LINUX /// Because of futex
 
@@ -12,6 +13,7 @@ namespace DB
 SharedMutex::SharedMutex()
     : state(0)
     , waiters(0)
+    , writer_thread_id(0)
 {}
 
 void SharedMutex::lock()
@@ -32,16 +34,22 @@ void SharedMutex::lock()
     value |= writers;
     while (value & readers)
         futexWaitLowerFetch(state, value);
+
+    writer_thread_id.store(getThreadId());
 }
 
 bool SharedMutex::try_lock()
 {
     UInt64 value = 0;
-    return state.compare_exchange_strong(value, writers);
+    bool success = state.compare_exchange_strong(value, writers);
+    if (success)
+        writer_thread_id.store(getThreadId());
+    return success;
 }
 
 void SharedMutex::unlock()
 {
+    writer_thread_id.store(0);
     state.store(0);
     if (waiters)
         futexWakeUpperAll(state);
diff --git a/src/Common/SharedMutex.h b/src/Common/SharedMutex.h
index 9215ff62af3..a53e2984239 100644
--- a/src/Common/SharedMutex.h
+++ b/src/Common/SharedMutex.h
@@ -36,6 +36,8 @@ private:
 
     alignas(64) std::atomic<UInt64> state;
     std::atomic<UInt32> waiters;
+    /// Is set while the lock is held in exclusive mode only to facilitate debugging
+    std::atomic<UInt64> writer_thread_id;
 };
 
 }

From 55d1656f4d0da2f23b2df719dabeed7999645349 Mon Sep 17 00:00:00 2001
From: Alexander Gololobov <davenger@clickhouse.com>
Date: Fri, 19 Jul 2024 13:27:41 +0200
Subject: [PATCH 0457/1488] Moving is not safe, prohibit it

---
 src/Common/SharedMutex.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/Common/SharedMutex.h b/src/Common/SharedMutex.h
index a53e2984239..c77c8765885 100644
--- a/src/Common/SharedMutex.h
+++ b/src/Common/SharedMutex.h
@@ -19,6 +19,8 @@ public:
     ~SharedMutex() = default;
     SharedMutex(const SharedMutex &) = delete;
     SharedMutex & operator=(const SharedMutex &) = delete;
+    SharedMutex(SharedMutex &&) = delete;
+    SharedMutex & operator=(SharedMutex &&) = delete;
 
     // Exclusive ownership
     void lock() TSA_ACQUIRE();

From 53ea5510143ded0862fd51922077a7cdc1344fe2 Mon Sep 17 00:00:00 2001
From: Igor Nikonov <igor@clickhouse.com>
Date: Fri, 19 Jul 2024 11:30:55 +0000
Subject: [PATCH 0458/1488] Remove unused code

---
 src/QueryPipeline/RemoteQueryExecutor.cpp | 14 +-------------
 src/QueryPipeline/RemoteQueryExecutor.h   |  2 +-
 2 files changed, 2 insertions(+), 14 deletions(-)

diff --git a/src/QueryPipeline/RemoteQueryExecutor.cpp b/src/QueryPipeline/RemoteQueryExecutor.cpp
index b15e31a120f..b78c38a4134 100644
--- a/src/QueryPipeline/RemoteQueryExecutor.cpp
+++ b/src/QueryPipeline/RemoteQueryExecutor.cpp
@@ -469,7 +469,7 @@ RemoteQueryExecutor::ReadResult RemoteQueryExecutor::read()
     return restartQueryWithoutDuplicatedUUIDs();
 }
 
-RemoteQueryExecutor::ReadResult RemoteQueryExecutor::readAsync([[maybe_unused]] bool check_packet_type_only)
+RemoteQueryExecutor::ReadResult RemoteQueryExecutor::readAsync()
 {
 #if defined(OS_LINUX)
     if (!read_context || (resent_query && recreate_read_context))
@@ -519,18 +519,6 @@ RemoteQueryExecutor::ReadResult RemoteQueryExecutor::readAsync([[maybe_unused]]
         if (read_context->isInProgress())
             return ReadResult(read_context->getFileDescriptor());
 
-        if (check_packet_type_only)
-        {
-            has_postponed_packet = true;
-            const auto packet_type = read_context->getPacketType();
-            if (packet_type == Protocol::Server::MergeTreeReadTaskRequest
-                || packet_type == Protocol::Server::MergeTreeAllRangesAnnouncement)
-            {
-                return ReadResult(ReadResult::Type::ParallelReplicasToken);
-            }
-            return ReadResult(ReadResult::Type::Nothing);
-        }
-
         auto read_result = processPacket(read_context->getPacket());
         if (read_result.getType() == ReadResult::Type::Data || read_result.getType() == ReadResult::Type::ParallelReplicasToken)
             return read_result;
diff --git a/src/QueryPipeline/RemoteQueryExecutor.h b/src/QueryPipeline/RemoteQueryExecutor.h
index 6f56df71f1d..7289e2a2243 100644
--- a/src/QueryPipeline/RemoteQueryExecutor.h
+++ b/src/QueryPipeline/RemoteQueryExecutor.h
@@ -183,7 +183,7 @@ public:
     ReadResult read();
 
     /// Async variant of read. Returns ready block or file descriptor which may be used for polling.
-    ReadResult readAsync(bool check_packet_type_only = false);
+    ReadResult readAsync();
 
     /// Receive all remain packets and finish query.
     /// It should be cancelled after read returned empty block.

From 3fb01ed2c9154d79cd9d23e3ae2e8708e86d0a34 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?=
 <benjamin.antal@clickhouse.com>
Date: Fri, 19 Jul 2024 12:10:28 +0000
Subject: [PATCH 0459/1488] Use nonexistent address to check connection error
 at table creation

---
 tests/integration/test_storage_rabbitmq/test.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/tests/integration/test_storage_rabbitmq/test.py b/tests/integration/test_storage_rabbitmq/test.py
index 3240039ee81..c163f3f7aed 100644
--- a/tests/integration/test_storage_rabbitmq/test.py
+++ b/tests/integration/test_storage_rabbitmq/test.py
@@ -2220,13 +2220,11 @@ def test_rabbitmq_commit_on_block_write(rabbitmq_cluster):
 
 
 def test_rabbitmq_no_connection_at_startup_1(rabbitmq_cluster):
-    # no connection when table is initialized
-    rabbitmq_cluster.pause_container("rabbitmq1")
-    instance.query_and_get_error(
+    error = instance.query_and_get_error(
         """
         CREATE TABLE test.cs (key UInt64, value UInt64)
             ENGINE = RabbitMQ
-            SETTINGS rabbitmq_host_port = 'rabbitmq1:5672',
+            SETTINGS rabbitmq_host_port = 'no_connection_at_startup:5672',
                      rabbitmq_exchange_name = 'cs',
                      rabbitmq_format = 'JSONEachRow',
                      rabbitmq_flush_interval_ms=1000,
@@ -2234,7 +2232,7 @@ def test_rabbitmq_no_connection_at_startup_1(rabbitmq_cluster):
                      rabbitmq_row_delimiter = '\\n';
     """
     )
-    rabbitmq_cluster.unpause_container("rabbitmq1")
+    assert "CANNOT_CONNECT_RABBITMQ" in error
 
 
 def test_rabbitmq_no_connection_at_startup_2(rabbitmq_cluster):

From 064bd643dad9153e7c35b6a235585a962d51df2d Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Fri, 19 Jul 2024 12:23:56 +0000
Subject: [PATCH 0460/1488] Cosmetics

---
 src/Functions/dateDiff.cpp | 196 +++++++++++++++----------------------
 1 file changed, 79 insertions(+), 117 deletions(-)

diff --git a/src/Functions/dateDiff.cpp b/src/Functions/dateDiff.cpp
index 8e8865db7ed..5c46ad40daa 100644
--- a/src/Functions/dateDiff.cpp
+++ b/src/Functions/dateDiff.cpp
@@ -45,26 +45,26 @@ public:
 
     template <typename Transform>
     void dispatchForColumns(
-        const IColumn & x, const IColumn & y,
+        const IColumn & col_x, const IColumn & col_y,
         const DateLUTImpl & timezone_x, const DateLUTImpl & timezone_y,
         ColumnInt64::Container & result) const
     {
-        if (const auto * x_vec_16 = checkAndGetColumn<ColumnDate>(&x))
-            dispatchForSecondColumn<Transform>(*x_vec_16, y, timezone_x, timezone_y, result);
-        else if (const auto * x_vec_32 = checkAndGetColumn<ColumnDateTime>(&x))
-            dispatchForSecondColumn<Transform>(*x_vec_32, y, timezone_x, timezone_y, result);
-        else if (const auto * x_vec_32_s = checkAndGetColumn<ColumnDate32>(&x))
-            dispatchForSecondColumn<Transform>(*x_vec_32_s, y, timezone_x, timezone_y, result);
-        else if (const auto * x_vec_64 = checkAndGetColumn<ColumnDateTime64>(&x))
-            dispatchForSecondColumn<Transform>(*x_vec_64, y, timezone_x, timezone_y, result);
-        else if (const auto * x_const_16 = checkAndGetColumnConst<ColumnDate>(&x))
-            dispatchConstForSecondColumn<Transform>(x_const_16->getValue<UInt16>(), y, timezone_x, timezone_y, result);
-        else if (const auto * x_const_32 = checkAndGetColumnConst<ColumnDateTime>(&x))
-            dispatchConstForSecondColumn<Transform>(x_const_32->getValue<UInt32>(), y, timezone_x, timezone_y, result);
-        else if (const auto * x_const_32_s = checkAndGetColumnConst<ColumnDate32>(&x))
-            dispatchConstForSecondColumn<Transform>(x_const_32_s->getValue<Int32>(), y, timezone_x, timezone_y, result);
-        else if (const auto * x_const_64 = checkAndGetColumnConst<ColumnDateTime64>(&x))
-            dispatchConstForSecondColumn<Transform>(x_const_64->getValue<DecimalField<DateTime64>>(), y, timezone_x, timezone_y, result);
+        if (const auto * x_vec_16 = checkAndGetColumn<ColumnDate>(&col_x))
+            dispatchForSecondColumn<Transform>(*x_vec_16, col_y, timezone_x, timezone_y, result);
+        else if (const auto * x_vec_32 = checkAndGetColumn<ColumnDateTime>(&col_x))
+            dispatchForSecondColumn<Transform>(*x_vec_32, col_y, timezone_x, timezone_y, result);
+        else if (const auto * x_vec_32_s = checkAndGetColumn<ColumnDate32>(&col_x))
+            dispatchForSecondColumn<Transform>(*x_vec_32_s, col_y, timezone_x, timezone_y, result);
+        else if (const auto * x_vec_64 = checkAndGetColumn<ColumnDateTime64>(&col_x))
+            dispatchForSecondColumn<Transform>(*x_vec_64, col_y, timezone_x, timezone_y, result);
+        else if (const auto * x_const_16 = checkAndGetColumnConst<ColumnDate>(&col_x))
+            dispatchConstForSecondColumn<Transform>(x_const_16->getValue<UInt16>(), col_y, timezone_x, timezone_y, result);
+        else if (const auto * x_const_32 = checkAndGetColumnConst<ColumnDateTime>(&col_x))
+            dispatchConstForSecondColumn<Transform>(x_const_32->getValue<UInt32>(), col_y, timezone_x, timezone_y, result);
+        else if (const auto * x_const_32_s = checkAndGetColumnConst<ColumnDate32>(&col_x))
+            dispatchConstForSecondColumn<Transform>(x_const_32_s->getValue<Int32>(), col_y, timezone_x, timezone_y, result);
+        else if (const auto * x_const_64 = checkAndGetColumnConst<ColumnDateTime64>(&col_x))
+            dispatchConstForSecondColumn<Transform>(x_const_64->getValue<DecimalField<DateTime64>>(), col_y, timezone_x, timezone_y, result);
         else
             throw Exception(ErrorCodes::ILLEGAL_COLUMN,
                 "Illegal column for first argument of function {}, must be Date, Date32, DateTime or DateTime64",
@@ -73,25 +73,25 @@ public:
 
     template <typename Transform, typename LeftColumnType>
     void dispatchForSecondColumn(
-        const LeftColumnType & x, const IColumn & y,
+        const LeftColumnType & x, const IColumn & col_y,
         const DateLUTImpl & timezone_x, const DateLUTImpl & timezone_y,
         ColumnInt64::Container & result) const
     {
-        if (const auto * y_vec_16 = checkAndGetColumn<ColumnDate>(&y))
+        if (const auto * y_vec_16 = checkAndGetColumn<ColumnDate>(&col_y))
             vectorVector<Transform>(x, *y_vec_16, timezone_x, timezone_y, result);
-        else if (const auto * y_vec_32 = checkAndGetColumn<ColumnDateTime>(&y))
+        else if (const auto * y_vec_32 = checkAndGetColumn<ColumnDateTime>(&col_y))
             vectorVector<Transform>(x, *y_vec_32, timezone_x, timezone_y, result);
-        else if (const auto * y_vec_32_s = checkAndGetColumn<ColumnDate32>(&y))
+        else if (const auto * y_vec_32_s = checkAndGetColumn<ColumnDate32>(&col_y))
             vectorVector<Transform>(x, *y_vec_32_s, timezone_x, timezone_y, result);
-        else if (const auto * y_vec_64 = checkAndGetColumn<ColumnDateTime64>(&y))
+        else if (const auto * y_vec_64 = checkAndGetColumn<ColumnDateTime64>(&col_y))
             vectorVector<Transform>(x, *y_vec_64, timezone_x, timezone_y, result);
-        else if (const auto * y_const_16 = checkAndGetColumnConst<ColumnDate>(&y))
+        else if (const auto * y_const_16 = checkAndGetColumnConst<ColumnDate>(&col_y))
             vectorConstant<Transform>(x, y_const_16->getValue<UInt16>(), timezone_x, timezone_y, result);
-        else if (const auto * y_const_32 = checkAndGetColumnConst<ColumnDateTime>(&y))
+        else if (const auto * y_const_32 = checkAndGetColumnConst<ColumnDateTime>(&col_y))
             vectorConstant<Transform>(x, y_const_32->getValue<UInt32>(), timezone_x, timezone_y, result);
-        else if (const auto * y_const_32_s = checkAndGetColumnConst<ColumnDate32>(&y))
+        else if (const auto * y_const_32_s = checkAndGetColumnConst<ColumnDate32>(&col_y))
             vectorConstant<Transform>(x, y_const_32_s->getValue<Int32>(), timezone_x, timezone_y, result);
-        else if (const auto * y_const_64 = checkAndGetColumnConst<ColumnDateTime64>(&y))
+        else if (const auto * y_const_64 = checkAndGetColumnConst<ColumnDateTime64>(&col_y))
             vectorConstant<Transform>(x, y_const_64->getValue<DecimalField<DateTime64>>(), timezone_x, timezone_y, result);
         else
             throw Exception(ErrorCodes::ILLEGAL_COLUMN,
@@ -101,17 +101,17 @@ public:
 
     template <typename Transform, typename T1>
     void dispatchConstForSecondColumn(
-        T1 x, const IColumn & y,
+        T1 x, const IColumn & col_y,
         const DateLUTImpl & timezone_x, const DateLUTImpl & timezone_y,
         ColumnInt64::Container & result) const
     {
-        if (const auto * y_vec_16 = checkAndGetColumn<ColumnDate>(&y))
+        if (const auto * y_vec_16 = checkAndGetColumn<ColumnDate>(&col_y))
             constantVector<Transform>(x, *y_vec_16, timezone_x, timezone_y, result);
-        else if (const auto * y_vec_32 = checkAndGetColumn<ColumnDateTime>(&y))
+        else if (const auto * y_vec_32 = checkAndGetColumn<ColumnDateTime>(&col_y))
             constantVector<Transform>(x, *y_vec_32, timezone_x, timezone_y, result);
-        else if (const auto * y_vec_32_s = checkAndGetColumn<ColumnDate32>(&y))
+        else if (const auto * y_vec_32_s = checkAndGetColumn<ColumnDate32>(&col_y))
             constantVector<Transform>(x, *y_vec_32_s, timezone_x, timezone_y, result);
-        else if (const auto * y_vec_64 = checkAndGetColumn<ColumnDateTime64>(&y))
+        else if (const auto * y_vec_64 = checkAndGetColumn<ColumnDateTime64>(&col_y))
             constantVector<Transform>(x, *y_vec_64, timezone_x, timezone_y, result);
         else
             throw Exception(ErrorCodes::ILLEGAL_COLUMN,
@@ -168,8 +168,7 @@ public:
     Int64 calculate(const TransformX & transform_x, const TransformY & transform_y, T1 x, T2 y, const DateLUTImpl & timezone_x, const DateLUTImpl & timezone_y) const
     {
         if constexpr (is_diff)
-            return static_cast<Int64>(transform_y.execute(y, timezone_y))
-                - static_cast<Int64>(transform_x.execute(x, timezone_x));
+            return static_cast<Int64>(transform_y.execute(y, timezone_y)) - static_cast<Int64>(transform_x.execute(x, timezone_x));
         else
         {
             auto res = static_cast<Int64>(transform_y.execute(y, timezone_y))
@@ -332,95 +331,73 @@ public:
     static constexpr auto name = is_relative ? "dateDiff" : "age";
     static FunctionPtr create(ContextPtr) { return std::make_shared<FunctionDateDiff>(); }
 
-    String getName() const override
-    {
-        return name;
-    }
+    String getName() const override { return name; }
 
     bool isVariadic() const override { return true; }
     bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; }
     size_t getNumberOfArguments() const override { return 0; }
+    bool useDefaultImplementationForConstants() const override { return true; }
+    ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {0, 3}; }
 
-    DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
+    DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
     {
-        if (arguments.size() != 3 && arguments.size() != 4)
-            throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
-                "Number of arguments for function {} doesn't match: passed {}, should be 3 or 4",
-                getName(), arguments.size());
+        FunctionArgumentDescriptors mandatory_args{
+            {"unit", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"},
+            {"startdate", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isDateOrDate32OrDateTimeOrDateTime64), nullptr, "Date[32] or DateTime[64]"},
+            {"enddate", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isDateOrDate32OrDateTimeOrDateTime64), nullptr, "Date[32] or DateTime[64]"},
+        };
 
-        if (!isString(arguments[0]))
-            throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
-                "First argument for function {} (unit) must be String",
-                getName());
+        FunctionArgumentDescriptors optional_args{
+            {"timezone", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"},
+        };
 
-        if (!isDate(arguments[1]) && !isDate32(arguments[1]) && !isDateTime(arguments[1]) && !isDateTime64(arguments[1]))
-            throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
-                "Second argument for function {} must be Date, Date32, DateTime or DateTime64",
-                getName());
-
-        if (!isDate(arguments[2]) && !isDate32(arguments[2]) && !isDateTime(arguments[2]) && !isDateTime64(arguments[2]))
-            throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
-                "Third argument for function {} must be Date, Date32, DateTime or DateTime64",
-                getName()
-                );
-
-        if (arguments.size() == 4 && !isString(arguments[3]))
-            throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
-                "Fourth argument for function {} (timezone) must be String",
-                getName());
+        validateFunctionArguments(*this, arguments, mandatory_args, optional_args);
 
         return std::make_shared<DataTypeInt64>();
     }
 
-    bool useDefaultImplementationForConstants() const override { return true; }
-    ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {0, 3}; }
-
     ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
-        const auto * unit_column = checkAndGetColumnConst<ColumnString>(arguments[0].column.get());
-        if (!unit_column)
-            throw Exception(ErrorCodes::ILLEGAL_COLUMN,
-                "First argument for function {} must be constant String",
-                getName());
+        const auto * col_unit = checkAndGetColumnConst<ColumnString>(arguments[0].column.get());
+        if (!col_unit)
+            throw Exception(ErrorCodes::ILLEGAL_COLUMN, "First argument for function {} must be constant String", getName());
 
-        String unit = Poco::toLower(unit_column->getValue<String>());
+        String unit = Poco::toLower(col_unit->getValue<String>());
 
-        const IColumn & x = *arguments[1].column;
-        const IColumn & y = *arguments[2].column;
+        const IColumn & col_x = *arguments[1].column;
+        const IColumn & col_y = *arguments[2].column;
 
-        size_t rows = input_rows_count;
-        auto res = ColumnInt64::create(rows);
+        auto col_res = ColumnInt64::create(input_rows_count);
 
         const auto & timezone_x = extractTimeZoneFromFunctionArguments(arguments, 3, 1);
         const auto & timezone_y = extractTimeZoneFromFunctionArguments(arguments, 3, 2);
 
         if (unit == "year" || unit == "years" || unit == "yy" || unit == "yyyy")
-            impl.template dispatchForColumns<ToRelativeYearNumImpl<ResultPrecision::Extended>>(x, y, timezone_x, timezone_y, res->getData());
+            impl.template dispatchForColumns<ToRelativeYearNumImpl<ResultPrecision::Extended>>(col_x, col_y, timezone_x, timezone_y, col_res->getData());
         else if (unit == "quarter" || unit == "quarters" || unit == "qq" || unit == "q")
-            impl.template dispatchForColumns<ToRelativeQuarterNumImpl<ResultPrecision::Extended>>(x, y, timezone_x, timezone_y, res->getData());
+            impl.template dispatchForColumns<ToRelativeQuarterNumImpl<ResultPrecision::Extended>>(col_x, col_y, timezone_x, timezone_y, col_res->getData());
         else if (unit == "month" || unit == "months" || unit == "mm" || unit == "m")
-            impl.template dispatchForColumns<ToRelativeMonthNumImpl<ResultPrecision::Extended>>(x, y, timezone_x, timezone_y, res->getData());
+            impl.template dispatchForColumns<ToRelativeMonthNumImpl<ResultPrecision::Extended>>(col_x, col_y, timezone_x, timezone_y, col_res->getData());
         else if (unit == "week" || unit == "weeks" || unit == "wk" || unit == "ww")
-            impl.template dispatchForColumns<ToRelativeWeekNumImpl<ResultPrecision::Extended>>(x, y, timezone_x, timezone_y, res->getData());
+            impl.template dispatchForColumns<ToRelativeWeekNumImpl<ResultPrecision::Extended>>(col_x, col_y, timezone_x, timezone_y, col_res->getData());
         else if (unit == "day" || unit == "days" || unit == "dd" || unit == "d")
-            impl.template dispatchForColumns<ToRelativeDayNumImpl<ResultPrecision::Extended>>(x, y, timezone_x, timezone_y, res->getData());
+            impl.template dispatchForColumns<ToRelativeDayNumImpl<ResultPrecision::Extended>>(col_x, col_y, timezone_x, timezone_y, col_res->getData());
         else if (unit == "hour" || unit == "hours" || unit == "hh" || unit == "h")
-            impl.template dispatchForColumns<ToRelativeHourNumImpl<ResultPrecision::Extended>>(x, y, timezone_x, timezone_y, res->getData());
+            impl.template dispatchForColumns<ToRelativeHourNumImpl<ResultPrecision::Extended>>(col_x, col_y, timezone_x, timezone_y, col_res->getData());
         else if (unit == "minute" || unit == "minutes" || unit == "mi" || unit == "n")
-            impl.template dispatchForColumns<ToRelativeMinuteNumImpl<ResultPrecision::Extended>>(x, y, timezone_x, timezone_y, res->getData());
+            impl.template dispatchForColumns<ToRelativeMinuteNumImpl<ResultPrecision::Extended>>(col_x, col_y, timezone_x, timezone_y, col_res->getData());
         else if (unit == "second" || unit == "seconds" || unit == "ss" || unit == "s")
-            impl.template dispatchForColumns<ToRelativeSecondNumImpl<ResultPrecision::Extended>>(x, y, timezone_x, timezone_y, res->getData());
+            impl.template dispatchForColumns<ToRelativeSecondNumImpl<ResultPrecision::Extended>>(col_x, col_y, timezone_x, timezone_y, col_res->getData());
         else if (unit == "millisecond" || unit == "milliseconds" || unit == "ms")
-            impl.template dispatchForColumns<ToRelativeSubsecondNumImpl<millisecond_multiplier>>(x, y, timezone_x, timezone_y, res->getData());
+            impl.template dispatchForColumns<ToRelativeSubsecondNumImpl<millisecond_multiplier>>(col_x, col_y, timezone_x, timezone_y, col_res->getData());
         else if (unit == "microsecond" || unit == "microseconds" || unit == "us" || unit == "u")
-            impl.template dispatchForColumns<ToRelativeSubsecondNumImpl<microsecond_multiplier>>(x, y, timezone_x, timezone_y, res->getData());
+            impl.template dispatchForColumns<ToRelativeSubsecondNumImpl<microsecond_multiplier>>(col_x, col_y, timezone_x, timezone_y, col_res->getData());
         else if (unit == "nanosecond" || unit == "nanoseconds" || unit == "ns")
-            impl.template dispatchForColumns<ToRelativeSubsecondNumImpl<nanosecond_multiplier>>(x, y, timezone_x, timezone_y, res->getData());
+            impl.template dispatchForColumns<ToRelativeSubsecondNumImpl<nanosecond_multiplier>>(col_x, col_y, timezone_x, timezone_y, col_res->getData());
         else
-            throw Exception(ErrorCodes::BAD_ARGUMENTS,
-                "Function {} does not support '{}' unit", getName(), unit);
+            throw Exception(ErrorCodes::BAD_ARGUMENTS, "Function {} does not support '{}' unit", getName(), unit);
 
-        return res;
+        return col_res;
     }
 private:
     DateDiffImpl<is_relative> impl{name};
@@ -437,50 +414,35 @@ public:
     static constexpr auto name = "timeDiff";
     static FunctionPtr create(ContextPtr) { return std::make_shared<FunctionTimeDiff>(); }
 
-    String getName() const override
-    {
-        return name;
-    }
-
+    String getName() const override { return name; }
+    bool useDefaultImplementationForConstants() const override { return true; }
+    ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {}; }
     bool isVariadic() const override { return false; }
     bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; }
     size_t getNumberOfArguments() const override { return 2; }
 
-    DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
+    DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
     {
-        if (arguments.size() != 2)
-            throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
-                "Number of arguments for function {} doesn't match: passed {}, should be 2",
-                getName(), arguments.size());
+        FunctionArgumentDescriptors args{
+            {"first_datetime", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isDateOrDate32OrDateTimeOrDateTime64), nullptr, "Date[32] or DateTime[64]"},
+            {"second_datetime", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isDateOrDate32OrDateTimeOrDateTime64), nullptr, "Date[32] or DateTime[64]"},
+        };
 
-        if (!isDate(arguments[0]) && !isDate32(arguments[0]) && !isDateTime(arguments[0]) && !isDateTime64(arguments[0]))
-            throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
-                "First argument for function {} must be Date, Date32, DateTime or DateTime64",
-                getName());
-
-        if (!isDate(arguments[1]) && !isDate32(arguments[1]) && !isDateTime(arguments[1]) && !isDateTime64(arguments[1]))
-            throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
-                "Second argument for function {} must be Date, Date32, DateTime or DateTime64",
-                getName()
-                );
+        validateFunctionArguments(*this, arguments, args);
 
         return std::make_shared<DataTypeInt64>();
     }
 
-    bool useDefaultImplementationForConstants() const override { return true; }
-    ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {}; }
-
     ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
-        const IColumn & x = *arguments[0].column;
-        const IColumn & y = *arguments[1].column;
+        const IColumn & col_x = *arguments[0].column;
+        const IColumn & col_y = *arguments[1].column;
 
-        size_t rows = input_rows_count;
-        auto res = ColumnInt64::create(rows);
+        auto col_res = ColumnInt64::create(input_rows_count);
 
-        impl.dispatchForColumns<ToRelativeSecondNumImpl<ResultPrecision::Extended>>(x, y, DateLUT::instance(), DateLUT::instance(), res->getData());
+        impl.dispatchForColumns<ToRelativeSecondNumImpl<ResultPrecision::Extended>>(col_x, col_y, DateLUT::instance(), DateLUT::instance(), col_res->getData());
 
-        return res;
+        return col_res;
     }
 private:
     DateDiffImpl<true> impl{name};

From 09cf5b1188d8b7a7485647ad62c936a7b28d41c7 Mon Sep 17 00:00:00 2001
From: lgbo-ustc <lgbo.ustc@gmail.com>
Date: Wed, 17 Jul 2024 09:31:31 +0800
Subject: [PATCH 0461/1488] column_length is not updated in
 ColumnTuple::insertManyFrom

---
 src/Columns/ColumnTuple.cpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/Columns/ColumnTuple.cpp b/src/Columns/ColumnTuple.cpp
index 9b822d7f570..9310d92e2ea 100644
--- a/src/Columns/ColumnTuple.cpp
+++ b/src/Columns/ColumnTuple.cpp
@@ -201,6 +201,7 @@ bool ColumnTuple::tryInsert(const Field & x)
             return false;
         }
     }
+    ++column_length;
 
     return true;
 }
@@ -236,6 +237,7 @@ void ColumnTuple::doInsertManyFrom(const IColumn & src, size_t position, size_t
 
     for (size_t i = 0; i < tuple_size; ++i)
         columns[i]->insertManyFrom(*src_tuple.columns[i], position, length);
+    column_length += length;
 }
 
 void ColumnTuple::insertDefault()

From 9164c989368adcdc4b6fe9f138706beef57706f1 Mon Sep 17 00:00:00 2001
From: lgbo-ustc <lgbo.ustc@gmail.com>
Date: Wed, 17 Jul 2024 09:51:24 +0800
Subject: [PATCH 0462/1488] add test and merge #66623

---
 src/Storages/transformQueryForExternalDatabaseAnalyzer.cpp  | 4 +---
 tests/queries/0_stateless/03205_column_type_check.reference | 3 +++
 tests/queries/0_stateless/03205_column_type_check.sql       | 2 ++
 3 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/Storages/transformQueryForExternalDatabaseAnalyzer.cpp b/src/Storages/transformQueryForExternalDatabaseAnalyzer.cpp
index ff40b1fcc2d..a7f56999d73 100644
--- a/src/Storages/transformQueryForExternalDatabaseAnalyzer.cpp
+++ b/src/Storages/transformQueryForExternalDatabaseAnalyzer.cpp
@@ -69,9 +69,7 @@ ASTPtr getASTForExternalDatabaseFromQueryTree(const QueryTreeNodePtr & query_tre
     bool allow_where = true;
     if (const auto * join_node = join_tree->as<JoinNode>())
     {
-        if (join_node->getStrictness() != JoinStrictness::All)
-            allow_where = false;
-        else if (join_node->getKind() == JoinKind::Left)
+        if (join_node->getKind() == JoinKind::Left)
             allow_where = join_node->getLeftTableExpression()->isEqual(*table_expression);
         else if (join_node->getKind() == JoinKind::Right)
             allow_where = join_node->getRightTableExpression()->isEqual(*table_expression);
diff --git a/tests/queries/0_stateless/03205_column_type_check.reference b/tests/queries/0_stateless/03205_column_type_check.reference
index 3b6c93a0610..466cabedd2e 100644
--- a/tests/queries/0_stateless/03205_column_type_check.reference
+++ b/tests/queries/0_stateless/03205_column_type_check.reference
@@ -1,2 +1,5 @@
 1	nan	1048575	2
 1	1	1	1	1
+()	1	nan	1048575	2
+
+()	1	nan	1048575	2
diff --git a/tests/queries/0_stateless/03205_column_type_check.sql b/tests/queries/0_stateless/03205_column_type_check.sql
index ab122821eb0..332061d6be8 100644
--- a/tests/queries/0_stateless/03205_column_type_check.sql
+++ b/tests/queries/0_stateless/03205_column_type_check.sql
@@ -5,3 +5,5 @@ SET join_algorithm='hash';
 SET allow_experimental_join_condition=1;
 SELECT * FROM ( SELECT 1 AS a, toLowCardinality(1), 1) AS t1 CROSS  JOIN (SELECT toLowCardinality(1 AS a), 1 AS b) AS t2;
 
+
+SELECT * FROM (SELECT tuple(),  1 GROUP BY greatCircleAngle(toNullable(1048575), 257, toInt128(-9223372036854775808), materialize(1048576)) WITH TOTALS) AS t, (SELECT greatCircleAngle(toUInt256(1048575), 257, toNullable(-9223372036854775808), 1048576), 1048575,  2) AS u

From 3f66b39a18a7bf271a9a9f97dfc075866e2409eb Mon Sep 17 00:00:00 2001
From: Michael Stetsyuk <mike.stetsyuk@gmail.com>
Date: Fri, 19 Jul 2024 14:30:27 +0200
Subject: [PATCH 0463/1488] test replication lag metric

---
 .../0_stateless/03206_replication_lag_metric.reference   | 4 ++++
 .../queries/0_stateless/03206_replication_lag_metric.sql | 9 +++++++++
 2 files changed, 13 insertions(+)
 create mode 100644 tests/queries/0_stateless/03206_replication_lag_metric.reference
 create mode 100644 tests/queries/0_stateless/03206_replication_lag_metric.sql

diff --git a/tests/queries/0_stateless/03206_replication_lag_metric.reference b/tests/queries/0_stateless/03206_replication_lag_metric.reference
new file mode 100644
index 00000000000..02f4a7264b1
--- /dev/null
+++ b/tests/queries/0_stateless/03206_replication_lag_metric.reference
@@ -0,0 +1,4 @@
+0
+2
+0
+2
diff --git a/tests/queries/0_stateless/03206_replication_lag_metric.sql b/tests/queries/0_stateless/03206_replication_lag_metric.sql
new file mode 100644
index 00000000000..6b86553fcaf
--- /dev/null
+++ b/tests/queries/0_stateless/03206_replication_lag_metric.sql
@@ -0,0 +1,9 @@
+CREATE DATABASE rdb1 ENGINE = Replicated('/test/test_replication_lag_metric', 'shard1', 'replica1');
+CREATE DATABASE rdb2 ENGINE = Replicated('/test/test_replication_lag_metric', 'shard1', 'replica2');
+
+SET distributed_ddl_task_timeout = 0;
+CREATE TABLE rdb1.t (id UInt32) ENGINE = ReplicatedMergeTree ORDER BY id;
+SELECT replication_lag FROM system.clusters;
+
+DROP DATABASE rdb1;
+DROP DATABASE rdb2;

From 245626e5789064fda39ccc7288b83162284a3617 Mon Sep 17 00:00:00 2001
From: Michael Stetsyuk <mike.stetsyuk@gmail.com>
Date: Fri, 19 Jul 2024 14:30:44 +0200
Subject: [PATCH 0464/1488] small fix

---
 src/Storages/System/StorageSystemClusters.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Storages/System/StorageSystemClusters.h b/src/Storages/System/StorageSystemClusters.h
index ead123aa79e..f6e08734896 100644
--- a/src/Storages/System/StorageSystemClusters.h
+++ b/src/Storages/System/StorageSystemClusters.h
@@ -1,10 +1,10 @@
 #pragma once
 
+#include <Databases/DatabaseReplicated.h>
 #include <DataTypes/DataTypeString.h>
 #include <DataTypes/DataTypesNumber.h>
 #include <Storages/System/IStorageSystemOneBlock.h>
 
-
 namespace DB
 {
 

From 212b81da533d18a6b9f02c66b34a2161ff1e5d71 Mon Sep 17 00:00:00 2001
From: Sema Checherinda <Sema.Checherinda@clickhouse.com>
Date: Fri, 19 Jul 2024 14:33:20 +0200
Subject: [PATCH 0465/1488] rm debug prints, fix headers

---
 src/Processors/Executors/CompletedPipelineExecutor.cpp | 1 -
 src/Storages/MergeTree/MergeTreeDataWriter.cpp         | 4 ----
 src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp     | 1 -
 3 files changed, 6 deletions(-)

diff --git a/src/Processors/Executors/CompletedPipelineExecutor.cpp b/src/Processors/Executors/CompletedPipelineExecutor.cpp
index 1eeee896ede..888835c9beb 100644
--- a/src/Processors/Executors/CompletedPipelineExecutor.cpp
+++ b/src/Processors/Executors/CompletedPipelineExecutor.cpp
@@ -3,7 +3,6 @@
 #include <QueryPipeline/QueryPipeline.h>
 #include <QueryPipeline/ReadProgressCallback.h>
 #include <Poco/Event.h>
-#include "Common/Logger.h"
 #include <Common/setThreadName.h>
 #include <Common/ThreadPool.h>
 #include <Common/scope_guard_safe.h>
diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp
index 97335b601f9..73244b714bf 100644
--- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp
+++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp
@@ -14,8 +14,6 @@
 #include <Storages/MergeTree/MergedBlockOutputStream.h>
 #include <Storages/MergeTree/MergeTreeSettings.h>
 #include <Storages/MergeTree/RowOrderOptimizer.h>
-#include "Common/Logger.h"
-#include "Common/logger_useful.h"
 #include <Common/ElapsedTimeProfileEventIncrement.h>
 #include <Common/Exception.h>
 #include <Common/HashTable/HashMap.h>
@@ -186,8 +184,6 @@ void updateTTL(
 
 void MergeTreeDataWriter::TemporaryPart::cancel()
 {
-    LOG_INFO(getLogger("MergeTreeDataWriter"), "TemporaryPart cancel");
-
     try
     {
         /// An exception context is needed to proper delete write buffers without finalization
diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp
index f6d6b8cb7a3..17662f92035 100644
--- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp
+++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp
@@ -4,7 +4,6 @@
 #include <Storages/MergeTree/InsertBlockInfo.h>
 #include <Interpreters/PartLog.h>
 #include <Common/Exception.h>
-#include <Common/StackTrace.h>
 #include <Common/FailPoint.h>
 #include <Common/ProfileEventsScope.h>
 #include <Common/SipHash.h>

From 57c1d7a1011f96cea21ca66a3064b7481f8ce40b Mon Sep 17 00:00:00 2001
From: Anton Popov <anton@clickhouse.com>
Date: Fri, 19 Jul 2024 12:36:57 +0000
Subject: [PATCH 0466/1488] fix filling of empty Nested

---
 src/DataTypes/IDataType.cpp                   |  4 +-
 src/DataTypes/ObjectUtils.cpp                 | 31 ++++++++++
 src/DataTypes/ObjectUtils.h                   |  3 +
 src/DataTypes/Serializations/ISerialization.h |  3 +-
 src/Interpreters/inplaceBlockConversions.cpp  | 57 ++++++++++++++++---
 src/Storages/MergeTree/IMergeTreeReader.cpp   |  7 ++-
 src/Storages/MergeTree/IMergeTreeReader.h     |  3 +
 7 files changed, 95 insertions(+), 13 deletions(-)

diff --git a/src/DataTypes/IDataType.cpp b/src/DataTypes/IDataType.cpp
index 1cb64b65d3a..824bc6e33b0 100644
--- a/src/DataTypes/IDataType.cpp
+++ b/src/DataTypes/IDataType.cpp
@@ -90,7 +90,9 @@ void IDataType::forEachSubcolumn(
             {
                 auto name = ISerialization::getSubcolumnNameForStream(subpath, prefix_len);
                 auto subdata = ISerialization::createFromPath(subpath, prefix_len);
-                callback(subpath, name, subdata);
+                auto path_copy = subpath;
+                path_copy.resize(prefix_len);
+                callback(path_copy, name, subdata);
             }
             subpath[i].visited = true;
         }
diff --git a/src/DataTypes/ObjectUtils.cpp b/src/DataTypes/ObjectUtils.cpp
index 1d525e5987f..356e609e77a 100644
--- a/src/DataTypes/ObjectUtils.cpp
+++ b/src/DataTypes/ObjectUtils.cpp
@@ -8,6 +8,7 @@
 #include <DataTypes/DataTypeNothing.h>
 #include <DataTypes/DataTypeArray.h>
 #include <DataTypes/DataTypeMap.h>
+#include <DataTypes/DataTypeTuple.h>
 #include <DataTypes/DataTypeNullable.h>
 #include <DataTypes/DataTypesNumber.h>
 #include <DataTypes/DataTypeNested.h>
@@ -66,6 +67,36 @@ DataTypePtr getBaseTypeOfArray(const DataTypePtr & type)
     return last_array ? last_array->getNestedType() : type;
 }
 
+DataTypePtr getBaseTypeOfArray(DataTypePtr type, const Names & tuple_elements)
+{
+    auto it = tuple_elements.begin();
+    while (true)
+    {
+        if (const auto * type_array = typeid_cast<const DataTypeArray *>(type.get()))
+        {
+            type = type_array->getNestedType();
+        }
+        else if (const auto * type_tuple = typeid_cast<const DataTypeTuple *>(type.get()))
+        {
+            if (it == tuple_elements.end())
+                break;
+
+            auto pos = type_tuple->tryGetPositionByName(*it);
+            if (!pos)
+                break;
+
+            ++it;
+            type = type_tuple->getElement(*pos);
+        }
+        else
+        {
+            break;
+        }
+    }
+
+    return type;
+}
+
 ColumnPtr getBaseColumnOfArray(const ColumnPtr & column)
 {
     /// Get raw pointers to avoid extra copying of column pointers.
diff --git a/src/DataTypes/ObjectUtils.h b/src/DataTypes/ObjectUtils.h
index 6599d8adef1..21e5c3b2f59 100644
--- a/src/DataTypes/ObjectUtils.h
+++ b/src/DataTypes/ObjectUtils.h
@@ -27,6 +27,9 @@ size_t getNumberOfDimensions(const IColumn & column);
 /// Returns type of scalars of Array of arbitrary dimensions.
 DataTypePtr getBaseTypeOfArray(const DataTypePtr & type);
 
+/// The same as above but takes into account Tuples of Nested.
+DataTypePtr getBaseTypeOfArray(DataTypePtr type, const Names & tuple_elements);
+
 /// Returns Array type with requested scalar type and number of dimensions.
 DataTypePtr createArrayOfType(DataTypePtr type, size_t num_dimensions);
 
diff --git a/src/DataTypes/Serializations/ISerialization.h b/src/DataTypes/Serializations/ISerialization.h
index 255dbbfadd2..5d0bf60c59f 100644
--- a/src/DataTypes/Serializations/ISerialization.h
+++ b/src/DataTypes/Serializations/ISerialization.h
@@ -195,7 +195,7 @@ public:
         /// Types of substreams that can have arbitrary name.
         static const std::set<Type> named_types;
 
-        Type type;
+        Type type = Type::Regular;
 
         /// The name of a variant element type.
         String variant_element_name;
@@ -212,6 +212,7 @@ public:
         /// Flag, that may help to traverse substream paths.
         mutable bool visited = false;
 
+        Substream() = default;
         Substream(Type type_) : type(type_) {} /// NOLINT
         String toString() const;
     };
diff --git a/src/Interpreters/inplaceBlockConversions.cpp b/src/Interpreters/inplaceBlockConversions.cpp
index f7d8a2a2daf..ce3f25d16f8 100644
--- a/src/Interpreters/inplaceBlockConversions.cpp
+++ b/src/Interpreters/inplaceBlockConversions.cpp
@@ -283,6 +283,9 @@ static ColumnPtr createColumnWithDefaultValue(const IDataType & data_type, const
 {
     auto column = data_type.createColumnConstWithDefaultValue(num_rows);
 
+    /// We must turn a constant column into a full column because the interpreter could infer
+    /// that it is constant everywhere but in some blocks (from other parts) it can be a full column.
+
     if (subcolumn_name.empty())
         return column->convertToFullColumnIfConst();
 
@@ -293,6 +296,35 @@ static ColumnPtr createColumnWithDefaultValue(const IDataType & data_type, const
     return ColumnConst::create(std::move(column), num_rows)->convertToFullColumnIfConst();
 }
 
+static bool hasDefault(const StorageMetadataPtr & metadata_snapshot, const NameAndTypePair & column)
+{
+    if (!metadata_snapshot)
+        return false;
+
+    const auto & columns = metadata_snapshot->getColumns();
+    if (columns.has(column.name))
+        return columns.hasDefault(column.name);
+
+    auto name_in_storage = column.getNameInStorage();
+    return columns.hasDefault(name_in_storage);
+}
+
+static String removeTupleElementsFromSubcolumn(String subcolumn_name, const Names & tuple_elements)
+{
+    subcolumn_name += ".";
+    for (const auto & elem : tuple_elements)
+    {
+        auto pos = subcolumn_name.find(elem + ".");
+        if (pos != std::string::npos)
+            subcolumn_name.erase(pos, elem.size());
+    }
+
+    if (subcolumn_name.ends_with("."))
+        subcolumn_name.pop_back();
+
+    return subcolumn_name;
+}
+
 void fillMissingColumns(
     Columns & res_columns,
     size_t num_rows,
@@ -321,10 +353,8 @@ void fillMissingColumns(
         if (res_columns[i] && partially_read_columns.contains(requested_column->name))
             res_columns[i] = nullptr;
 
-        if (res_columns[i])
-            continue;
-
-        if (metadata_snapshot && metadata_snapshot->getColumns().hasDefault(requested_column->getNameInStorage()))
+        /// Nothing to fill or default should be filled in evaluateMissingDefaults
+        if (res_columns[i] || hasDefault(metadata_snapshot, *requested_column))
             continue;
 
         std::vector<ColumnPtr> current_offsets;
@@ -365,19 +395,30 @@ void fillMissingColumns(
 
         if (!current_offsets.empty())
         {
+
+            Names tuple_elements;
+            auto serialization = IDataType::getSerialization(*requested_column);
+
+            IDataType::forEachSubcolumn([&](const auto & path, const auto &, const auto &)
+            {
+                if (path.back().type == ISerialization::Substream::TupleElement)
+                    tuple_elements.push_back(path.back().name_of_substream);
+            }, ISerialization::SubstreamData(serialization));
+
             size_t num_empty_dimensions = num_dimensions - current_offsets.size();
-            auto scalar_type = createArrayOfType(getBaseTypeOfArray(requested_column->getTypeInStorage()), num_empty_dimensions);
+            auto base_type = getBaseTypeOfArray(requested_column->getTypeInStorage(), tuple_elements);
+            auto scalar_type = createArrayOfType(base_type, num_empty_dimensions);
 
             size_t data_size = assert_cast<const ColumnUInt64 &>(*current_offsets.back()).getData().back();
-            res_columns[i] = createColumnWithDefaultValue(*scalar_type, requested_column->getSubcolumnName(), data_size);
+            auto subcolumn_name = removeTupleElementsFromSubcolumn(requested_column->getSubcolumnName(), tuple_elements);
+
+            res_columns[i] = createColumnWithDefaultValue(*scalar_type, subcolumn_name, data_size);
 
             for (auto it = current_offsets.rbegin(); it != current_offsets.rend(); ++it)
                 res_columns[i] = ColumnArray::create(res_columns[i], *it);
         }
         else
         {
-            /// We must turn a constant column into a full column because the interpreter could infer
-            /// that it is constant everywhere but in some blocks (from other parts) it can be a full column.
             res_columns[i] = createColumnWithDefaultValue(*requested_column->getTypeInStorage(), requested_column->getSubcolumnName(), num_rows);
         }
     }
diff --git a/src/Storages/MergeTree/IMergeTreeReader.cpp b/src/Storages/MergeTree/IMergeTreeReader.cpp
index aff1001163e..5f36e4c7c13 100644
--- a/src/Storages/MergeTree/IMergeTreeReader.cpp
+++ b/src/Storages/MergeTree/IMergeTreeReader.cpp
@@ -44,6 +44,7 @@ IMergeTreeReader::IMergeTreeReader(
     , alter_conversions(data_part_info_for_read->getAlterConversions())
     /// For wide parts convert plain arrays of Nested to subcolumns
     /// to allow to use shared offset column from cache.
+    , original_requested_columns(columns_)
     , requested_columns(data_part_info_for_read->isWidePart()
         ? Nested::convertToSubcolumns(columns_)
         : columns_)
@@ -139,7 +140,7 @@ void IMergeTreeReader::evaluateMissingDefaults(Block additional_columns, Columns
 {
     try
     {
-        size_t num_columns = requested_columns.size();
+        size_t num_columns = original_requested_columns.size();
 
         if (res_columns.size() != num_columns)
             throw Exception(ErrorCodes::LOGICAL_ERROR, "invalid number of columns passed to MergeTreeReader::fillMissingColumns. "
@@ -151,7 +152,7 @@ void IMergeTreeReader::evaluateMissingDefaults(Block additional_columns, Columns
         /// Convert columns list to block. And convert subcolumns to full columns.
         /// TODO: rewrite with columns interface. It will be possible after changes in ExpressionActions.
 
-        auto it = requested_columns.begin();
+        auto it = original_requested_columns.begin();
         for (size_t pos = 0; pos < num_columns; ++pos, ++it)
         {
             auto name_in_storage = it->getNameInStorage();
@@ -178,7 +179,7 @@ void IMergeTreeReader::evaluateMissingDefaults(Block additional_columns, Columns
         }
 
         /// Move columns from block.
-        it = requested_columns.begin();
+        it = original_requested_columns.begin();
         for (size_t pos = 0; pos < num_columns; ++pos, ++it)
         {
             auto name_in_storage = it->getNameInStorage();
diff --git a/src/Storages/MergeTree/IMergeTreeReader.h b/src/Storages/MergeTree/IMergeTreeReader.h
index a1ec0339fd6..d799ce57b40 100644
--- a/src/Storages/MergeTree/IMergeTreeReader.h
+++ b/src/Storages/MergeTree/IMergeTreeReader.h
@@ -112,6 +112,9 @@ protected:
 
 private:
     /// Columns that are requested to read.
+    NamesAndTypesList original_requested_columns;
+
+    /// The same as above but with converted Arrays to subcolumns of Nested.
     NamesAndTypesList requested_columns;
 
     /// Actual columns description in part.

From a0171256fd5cb6932ceb9b2cc98c9b553dfa82fd Mon Sep 17 00:00:00 2001
From: Alexander Sapin <alesapin@gmail.com>
Date: Fri, 19 Jul 2024 14:41:17 +0200
Subject: [PATCH 0467/1488] Small improvement for background pool in Keeper

---
 src/Coordination/KeeperServer.cpp | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/Coordination/KeeperServer.cpp b/src/Coordination/KeeperServer.cpp
index dc9658e895f..68515debe3b 100644
--- a/src/Coordination/KeeperServer.cpp
+++ b/src/Coordination/KeeperServer.cpp
@@ -368,7 +368,10 @@ void KeeperServer::launchRaftServer(const Poco::Util::AbstractConfiguration & co
         LockMemoryExceptionInThread::removeUniqueLock();
     };
 
-    asio_opts.thread_pool_size_ = getNumberOfPhysicalCPUCores();
+    /// At least 16 threads for network communication in asio.
+    /// asio is async framework, so even with 1 thread it should be ok, but
+    /// still as safeguard it's better to have some redundant capacity here
+    asio_opts.thread_pool_size_ = std::max(16U, getNumberOfPhysicalCPUCores());
 
     if (state_manager->isSecure())
     {

From 189b3d306fc0e488010564384b193412acd0358b Mon Sep 17 00:00:00 2001
From: Anton Popov <anton@clickhouse.com>
Date: Fri, 19 Jul 2024 13:17:17 +0000
Subject: [PATCH 0468/1488] fix tests

---
 src/Storages/MergeTree/MergeTreeReaderCompact.cpp               | 2 +-
 .../0_stateless/02026_describe_include_subcolumns.reference     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp
index fde9dafffb8..ff0311dc1ca 100644
--- a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp
+++ b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp
@@ -101,7 +101,7 @@ NameAndTypePair MergeTreeReaderCompact::getColumnConvertedToSubcolumnOfNested(co
 
     if (!storage_columns_with_collected_nested)
     {
-        auto options = GetColumnsOptions(GetColumnsOptions::AllPhysical).withExtendedObjects();
+        auto options = GetColumnsOptions(GetColumnsOptions::All).withExtendedObjects();
         auto storage_columns_list = Nested::collect(storage_snapshot->getColumns(options));
         storage_columns_with_collected_nested = ColumnsDescription(std::move(storage_columns_list));
     }
diff --git a/tests/queries/0_stateless/02026_describe_include_subcolumns.reference b/tests/queries/0_stateless/02026_describe_include_subcolumns.reference
index dec65f62748..62efafceaad 100644
--- a/tests/queries/0_stateless/02026_describe_include_subcolumns.reference
+++ b/tests/queries/0_stateless/02026_describe_include_subcolumns.reference
@@ -26,7 +26,7 @@
 10. │ t.s        │ String                                                                    │              │                    │                         │ ZSTD(1)          │                      │            1 │
 11. │ t.a        │ Array(Tuple(
     a UInt32,
-    b UInt32))                                   │              │                    │                         │                  │                      │            1 │
+    b UInt32))                                   │              │                    │                         │ ZSTD(1)          │                      │            1 │
 12. │ t.a.size0  │ UInt64                                                                    │              │                    │                         │                  │                      │            1 │
 13. │ t.a.a      │ Array(UInt32)                                                             │              │                    │                         │ ZSTD(1)          │                      │            1 │
 14. │ t.a.b      │ Array(UInt32)                                                             │              │                    │                         │ ZSTD(1)          │                      │            1 │

From 79ef630d85cb445a743ee2d5950197709d75325f Mon Sep 17 00:00:00 2001
From: Michael Stetsyuk <mike.stetsyuk@gmail.com>
Date: Fri, 19 Jul 2024 15:25:08 +0200
Subject: [PATCH 0469/1488] fix tests

---
 .../0_stateless/02117_show_create_table_system.reference      | 2 ++
 tests/queries/0_stateless/03206_replication_lag_metric.sql    | 4 +++-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/02117_show_create_table_system.reference b/tests/queries/0_stateless/02117_show_create_table_system.reference
index 8f62eda9233..28356632a66 100644
--- a/tests/queries/0_stateless/02117_show_create_table_system.reference
+++ b/tests/queries/0_stateless/02117_show_create_table_system.reference
@@ -52,6 +52,8 @@ CREATE TABLE system.clusters
     `database_shard_name` String,
     `database_replica_name` String,
     `is_active` Nullable(UInt8),
+    `replication_lag` Nullable(UInt32),
+    `recovery_time` Nullable(UInt64),
     `name` String ALIAS cluster
 )
 ENGINE = SystemClusters
diff --git a/tests/queries/0_stateless/03206_replication_lag_metric.sql b/tests/queries/0_stateless/03206_replication_lag_metric.sql
index 6b86553fcaf..998c332a11c 100644
--- a/tests/queries/0_stateless/03206_replication_lag_metric.sql
+++ b/tests/queries/0_stateless/03206_replication_lag_metric.sql
@@ -1,9 +1,11 @@
+-- Tags: no-parallel
+
 CREATE DATABASE rdb1 ENGINE = Replicated('/test/test_replication_lag_metric', 'shard1', 'replica1');
 CREATE DATABASE rdb2 ENGINE = Replicated('/test/test_replication_lag_metric', 'shard1', 'replica2');
 
 SET distributed_ddl_task_timeout = 0;
 CREATE TABLE rdb1.t (id UInt32) ENGINE = ReplicatedMergeTree ORDER BY id;
-SELECT replication_lag FROM system.clusters;
+SELECT replication_lag FROM system.clusters WHERE cluster IN ('rdb1', 'rdb2') ORDER BY cluster ASC, replica_num ASC;
 
 DROP DATABASE rdb1;
 DROP DATABASE rdb2;

From eb51dc8980a4fa29e7836132158023fbba39db7f Mon Sep 17 00:00:00 2001
From: Alex Katsman <alex.katsman@clickhouse.com>
Date: Mon, 15 Jul 2024 12:08:18 +0000
Subject: [PATCH 0470/1488] Don't start GWP allocations until init is finished

---
 programs/keeper/Keeper.cpp |  8 ++++++++
 programs/server/Server.cpp |  1 +
 src/Client/ClientBase.cpp  |  9 +++++++++
 src/Common/Allocator.cpp   |  4 ++--
 src/Common/GWPAsan.cpp     |  7 +++++++
 src/Common/GWPAsan.h       | 18 ++++++++++++++++++
 src/Common/memory.h        |  6 +++---
 7 files changed, 48 insertions(+), 5 deletions(-)

diff --git a/programs/keeper/Keeper.cpp b/programs/keeper/Keeper.cpp
index 44c2daa33ad..8cf1a4d1999 100644
--- a/programs/keeper/Keeper.cpp
+++ b/programs/keeper/Keeper.cpp
@@ -52,6 +52,10 @@
 #    include <Server/CertificateReloader.h>
 #endif
 
+#if USE_GWP_ASAN
+#    include <Common/GWPAsan.h>
+#endif
+
 #include <Server/ProtocolServerAdapter.h>
 #include <Server/KeeperTCPHandlerFactory.h>
 
@@ -639,6 +643,10 @@ try
         tryLogCurrentException(log, "Disabling cgroup memory observer because of an error during initialization");
     }
 
+#if USE_GWP_ASAN
+    GWPAsan::initFinished();
+#endif
+
 
     LOG_INFO(log, "Ready for connections.");
 
diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp
index 0b695c3dde6..16888015f8b 100644
--- a/programs/server/Server.cpp
+++ b/programs/server/Server.cpp
@@ -2213,6 +2213,7 @@ try
         CannotAllocateThreadFaultInjector::setFaultProbability(server_settings.cannot_allocate_thread_fault_injection_probability);
 
 #if USE_GWP_ASAN
+        GWPAsan::initFinished();
         GWPAsan::setForceSampleProbability(server_settings.gwp_asan_force_sample_probability);
 #endif
 
diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp
index 13dce05cabc..7af199131b6 100644
--- a/src/Client/ClientBase.cpp
+++ b/src/Client/ClientBase.cpp
@@ -81,6 +81,10 @@
 #include <Common/config_version.h>
 #include "config.h"
 
+#if USE_GWP_ASAN
+#    include <Common/GWPAsan.h>
+#endif
+
 
 namespace fs = std::filesystem;
 using namespace std::literals;
@@ -3264,6 +3268,11 @@ void ClientBase::init(int argc, char ** argv)
     fatal_log = createLogger("ClientBase", fatal_channel_ptr.get(), Poco::Message::PRIO_FATAL);
     signal_listener = std::make_unique<SignalListener>(nullptr, fatal_log);
     signal_listener_thread.start(*signal_listener);
+
+#if USE_GWP_ASAN
+    GWPAsan::initFinished();
+#endif
+
 }
 
 }
diff --git a/src/Common/Allocator.cpp b/src/Common/Allocator.cpp
index bcc5d187047..7f2241ab4c0 100644
--- a/src/Common/Allocator.cpp
+++ b/src/Common/Allocator.cpp
@@ -68,7 +68,7 @@ void * allocNoTrack(size_t size, size_t alignment)
 {
     void * buf;
 #if USE_GWP_ASAN
-    if (unlikely(GWPAsan::GuardedAlloc.shouldSample()))
+    if (unlikely(GWPAsan::shouldSample()))
     {
         if (void * ptr = GWPAsan::GuardedAlloc.allocate(size, alignment))
         {
@@ -185,7 +185,7 @@ void * Allocator<clear_memory_, populate>::realloc(void * buf, size_t old_size,
     }
 
 #if USE_GWP_ASAN
-    if (unlikely(GWPAsan::GuardedAlloc.shouldSample()))
+    if (unlikely(GWPAsan::shouldSample()))
     {
         auto trace_alloc = CurrentMemoryTracker::alloc(new_size);
         if (void * ptr = GWPAsan::GuardedAlloc.allocate(new_size, alignment))
diff --git a/src/Common/GWPAsan.cpp b/src/Common/GWPAsan.cpp
index 48fbd07ec34..de6991191ea 100644
--- a/src/Common/GWPAsan.cpp
+++ b/src/Common/GWPAsan.cpp
@@ -217,6 +217,13 @@ void printReport([[maybe_unused]] uintptr_t fault_address)
         reinterpret_cast<void **>(trace.data()), 0, trace_length, [&](const auto line) { LOG_FATAL(logger, fmt::runtime(line)); });
 }
 
+std::atomic<bool> init_finished = false;
+
+void initFinished()
+{
+    init_finished.store(true, std::memory_order_relaxed);
+}
+
 std::atomic<double> force_sample_probability = 0.0;
 
 void setForceSampleProbability(double value)
diff --git a/src/Common/GWPAsan.h b/src/Common/GWPAsan.h
index b3215c6157e..846c3417db4 100644
--- a/src/Common/GWPAsan.h
+++ b/src/Common/GWPAsan.h
@@ -19,12 +19,30 @@ bool isGWPAsanError(uintptr_t fault_address);
 
 void printReport(uintptr_t fault_address);
 
+extern std::atomic<bool> init_finished;
+
+void initFinished();
+
 extern std::atomic<double> force_sample_probability;
 
 void setForceSampleProbability(double value);
 
+/**
+ * We'd like to postpone sampling allocations under the startup is finished. There are mainly
+ * two reasons for that:
+ *
+ * - To avoid complex issues with initialization order
+ * - Don't waste MaxSimultaneousAllocations on global objects as it's not useful
+*/
+inline bool shouldSample()
+{
+    return init_finished.load(std::memory_order_relaxed) && GuardedAlloc.shouldSample();
+}
+
 inline bool shouldForceSample()
 {
+    if (!init_finished.load(std::memory_order_relaxed))
+        return false;
     std::bernoulli_distribution dist(force_sample_probability.load(std::memory_order_relaxed));
     return dist(thread_local_rng);
 }
diff --git a/src/Common/memory.h b/src/Common/memory.h
index dbef069b408..d673f954fb2 100644
--- a/src/Common/memory.h
+++ b/src/Common/memory.h
@@ -37,7 +37,7 @@ requires DB::OptionalArgument<TAlign...>
 inline ALWAYS_INLINE void * newImpl(std::size_t size, TAlign... align)
 {
 #if USE_GWP_ASAN
-    if (unlikely(GWPAsan::GuardedAlloc.shouldSample()))
+    if (unlikely(GWPAsan::shouldSample()))
     {
         if constexpr (sizeof...(TAlign) == 1)
         {
@@ -83,7 +83,7 @@ inline ALWAYS_INLINE void * newImpl(std::size_t size, TAlign... align)
 inline ALWAYS_INLINE void * newNoExcept(std::size_t size) noexcept
 {
 #if USE_GWP_ASAN
-    if (unlikely(GWPAsan::GuardedAlloc.shouldSample()))
+    if (unlikely(GWPAsan::shouldSample()))
     {
         if (void * ptr = GWPAsan::GuardedAlloc.allocate(size))
         {
@@ -102,7 +102,7 @@ inline ALWAYS_INLINE void * newNoExcept(std::size_t size) noexcept
 inline ALWAYS_INLINE void * newNoExcept(std::size_t size, std::align_val_t align) noexcept
 {
 #if USE_GWP_ASAN
-    if (unlikely(GWPAsan::GuardedAlloc.shouldSample()))
+    if (unlikely(GWPAsan::shouldSample()))
     {
         if (void * ptr = GWPAsan::GuardedAlloc.allocate(size, alignToSizeT(align)))
         {

From 0ef2c7d486669d8c604e332594a02dfc1c38278a Mon Sep 17 00:00:00 2001
From: Alex Katsman <alex.katsman@clickhouse.com>
Date: Thu, 18 Jul 2024 11:53:55 +0000
Subject: [PATCH 0471/1488] Disable parallel run for
 02440_mutations_finalization test

---
 tests/queries/0_stateless/02440_mutations_finalization.sql | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/queries/0_stateless/02440_mutations_finalization.sql b/tests/queries/0_stateless/02440_mutations_finalization.sql
index c522d8ab9df..92ed6a327ad 100644
--- a/tests/queries/0_stateless/02440_mutations_finalization.sql
+++ b/tests/queries/0_stateless/02440_mutations_finalization.sql
@@ -1,3 +1,4 @@
+-- Tags: no-parallel
 
 create table mut (n int) engine=ReplicatedMergeTree('/test/02440/{database}/mut', '1') order by tuple();
 set insert_keeper_fault_injection_probability=0;

From e91c4b2e057323fc2eba16317dbcb210eedd0cab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Fri, 19 Jul 2024 16:13:36 +0200
Subject: [PATCH 0472/1488] Fix flakiness of async insert tests due to
 adaptative timeout

---
 .../queries/0_stateless/02726_async_insert_flush_queue.sql | 3 ++-
 .../0_stateless/02884_async_insert_skip_settings.reference | 4 ++--
 .../0_stateless/02884_async_insert_skip_settings.sql       | 7 ++++---
 3 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/tests/queries/0_stateless/02726_async_insert_flush_queue.sql b/tests/queries/0_stateless/02726_async_insert_flush_queue.sql
index 1ae24e4f3da..e393d96fc40 100644
--- a/tests/queries/0_stateless/02726_async_insert_flush_queue.sql
+++ b/tests/queries/0_stateless/02726_async_insert_flush_queue.sql
@@ -6,7 +6,8 @@ CREATE TABLE t_async_inserts_flush (a UInt64) ENGINE = Memory;
 
 SET async_insert = 1;
 SET wait_for_async_insert = 0;
-SET async_insert_busy_timeout_min_ms = 1000000;
+-- Disable adaptive timeout to prevent immediate push of the first message (if the queue last push was old)
+SET async_insert_use_adaptive_busy_timeout=0;
 SET async_insert_busy_timeout_max_ms = 10000000;
 
 INSERT INTO t_async_inserts_flush VALUES (1) (2);
diff --git a/tests/queries/0_stateless/02884_async_insert_skip_settings.reference b/tests/queries/0_stateless/02884_async_insert_skip_settings.reference
index 318966ce93b..957be52c59b 100644
--- a/tests/queries/0_stateless/02884_async_insert_skip_settings.reference
+++ b/tests/queries/0_stateless/02884_async_insert_skip_settings.reference
@@ -1,5 +1,5 @@
-4
+pending to flush	4
 1
 1
 2
-1
+flush queries	1
diff --git a/tests/queries/0_stateless/02884_async_insert_skip_settings.sql b/tests/queries/0_stateless/02884_async_insert_skip_settings.sql
index 9bc689fb4ec..df715af873a 100644
--- a/tests/queries/0_stateless/02884_async_insert_skip_settings.sql
+++ b/tests/queries/0_stateless/02884_async_insert_skip_settings.sql
@@ -9,7 +9,8 @@ ORDER BY id;
 SET async_insert = 1;
 SET async_insert_deduplicate = 1;
 SET wait_for_async_insert = 0;
-SET async_insert_busy_timeout_min_ms = 100000;
+-- Disable adaptive timeout to prevent immediate push of the first message (if the queue last push was old)
+SET async_insert_use_adaptive_busy_timeout=0;
 SET async_insert_busy_timeout_max_ms = 1000000;
 
 SET insert_deduplication_token = '1';
@@ -30,7 +31,7 @@ INSERT INTO t_async_insert_skip_settings VALUES (2);
 
 SYSTEM FLUSH LOGS;
 
-SELECT length(entries.bytes) FROM system.asynchronous_inserts
+SELECT 'pending to flush', length(entries.bytes) FROM system.asynchronous_inserts
 WHERE database = currentDatabase() AND table = 't_async_insert_skip_settings'
 ORDER BY first_update;
 
@@ -40,7 +41,7 @@ SELECT * FROM t_async_insert_skip_settings ORDER BY id;
 
 SYSTEM FLUSH LOGS;
 
-SELECT uniqExact(flush_query_id) FROM system.asynchronous_insert_log
+SELECT 'flush queries', uniqExact(flush_query_id) FROM system.asynchronous_insert_log
 WHERE database = currentDatabase() AND table = 't_async_insert_skip_settings';
 
 DROP TABLE t_async_insert_skip_settings SYNC;

From a54a0614a464d686aff48aab583daa527f74e932 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Fri, 19 Jul 2024 12:27:18 +0000
Subject: [PATCH 0473/1488] Iterate over input_rows_count instead of a size of
 a random input column

---
 src/Functions/dateDiff.cpp | 99 ++++++++++++++++++++------------------
 1 file changed, 51 insertions(+), 48 deletions(-)

diff --git a/src/Functions/dateDiff.cpp b/src/Functions/dateDiff.cpp
index 5c46ad40daa..a39cbae4e30 100644
--- a/src/Functions/dateDiff.cpp
+++ b/src/Functions/dateDiff.cpp
@@ -47,82 +47,80 @@ public:
     void dispatchForColumns(
         const IColumn & col_x, const IColumn & col_y,
         const DateLUTImpl & timezone_x, const DateLUTImpl & timezone_y,
+        size_t input_rows_count,
         ColumnInt64::Container & result) const
     {
         if (const auto * x_vec_16 = checkAndGetColumn<ColumnDate>(&col_x))
-            dispatchForSecondColumn<Transform>(*x_vec_16, col_y, timezone_x, timezone_y, result);
+            dispatchForSecondColumn<Transform>(*x_vec_16, col_y, timezone_x, timezone_y, input_rows_count, result);
         else if (const auto * x_vec_32 = checkAndGetColumn<ColumnDateTime>(&col_x))
-            dispatchForSecondColumn<Transform>(*x_vec_32, col_y, timezone_x, timezone_y, result);
+            dispatchForSecondColumn<Transform>(*x_vec_32, col_y, timezone_x, timezone_y, input_rows_count, result);
         else if (const auto * x_vec_32_s = checkAndGetColumn<ColumnDate32>(&col_x))
-            dispatchForSecondColumn<Transform>(*x_vec_32_s, col_y, timezone_x, timezone_y, result);
+            dispatchForSecondColumn<Transform>(*x_vec_32_s, col_y, timezone_x, timezone_y, input_rows_count, result);
         else if (const auto * x_vec_64 = checkAndGetColumn<ColumnDateTime64>(&col_x))
-            dispatchForSecondColumn<Transform>(*x_vec_64, col_y, timezone_x, timezone_y, result);
+            dispatchForSecondColumn<Transform>(*x_vec_64, col_y, timezone_x, timezone_y, input_rows_count, result);
         else if (const auto * x_const_16 = checkAndGetColumnConst<ColumnDate>(&col_x))
-            dispatchConstForSecondColumn<Transform>(x_const_16->getValue<UInt16>(), col_y, timezone_x, timezone_y, result);
+            dispatchConstForSecondColumn<Transform>(x_const_16->getValue<UInt16>(), col_y, timezone_x, timezone_y, input_rows_count, result);
         else if (const auto * x_const_32 = checkAndGetColumnConst<ColumnDateTime>(&col_x))
-            dispatchConstForSecondColumn<Transform>(x_const_32->getValue<UInt32>(), col_y, timezone_x, timezone_y, result);
+            dispatchConstForSecondColumn<Transform>(x_const_32->getValue<UInt32>(), col_y, timezone_x, timezone_y, input_rows_count, result);
         else if (const auto * x_const_32_s = checkAndGetColumnConst<ColumnDate32>(&col_x))
-            dispatchConstForSecondColumn<Transform>(x_const_32_s->getValue<Int32>(), col_y, timezone_x, timezone_y, result);
+            dispatchConstForSecondColumn<Transform>(x_const_32_s->getValue<Int32>(), col_y, timezone_x, timezone_y, input_rows_count, result);
         else if (const auto * x_const_64 = checkAndGetColumnConst<ColumnDateTime64>(&col_x))
-            dispatchConstForSecondColumn<Transform>(x_const_64->getValue<DecimalField<DateTime64>>(), col_y, timezone_x, timezone_y, result);
+            dispatchConstForSecondColumn<Transform>(x_const_64->getValue<DecimalField<DateTime64>>(), col_y, timezone_x, timezone_y, input_rows_count, result);
         else
-            throw Exception(ErrorCodes::ILLEGAL_COLUMN,
-                "Illegal column for first argument of function {}, must be Date, Date32, DateTime or DateTime64",
-                name);
+            throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column for first argument of function {}, must be Date, Date32, DateTime or DateTime64", name);
     }
 
     template <typename Transform, typename LeftColumnType>
     void dispatchForSecondColumn(
         const LeftColumnType & x, const IColumn & col_y,
         const DateLUTImpl & timezone_x, const DateLUTImpl & timezone_y,
+        size_t input_rows_count,
         ColumnInt64::Container & result) const
     {
         if (const auto * y_vec_16 = checkAndGetColumn<ColumnDate>(&col_y))
-            vectorVector<Transform>(x, *y_vec_16, timezone_x, timezone_y, result);
+            vectorVector<Transform>(x, *y_vec_16, timezone_x, timezone_y, input_rows_count, result);
         else if (const auto * y_vec_32 = checkAndGetColumn<ColumnDateTime>(&col_y))
-            vectorVector<Transform>(x, *y_vec_32, timezone_x, timezone_y, result);
+            vectorVector<Transform>(x, *y_vec_32, timezone_x, timezone_y, input_rows_count, result);
         else if (const auto * y_vec_32_s = checkAndGetColumn<ColumnDate32>(&col_y))
-            vectorVector<Transform>(x, *y_vec_32_s, timezone_x, timezone_y, result);
+            vectorVector<Transform>(x, *y_vec_32_s, timezone_x, timezone_y, input_rows_count, result);
         else if (const auto * y_vec_64 = checkAndGetColumn<ColumnDateTime64>(&col_y))
-            vectorVector<Transform>(x, *y_vec_64, timezone_x, timezone_y, result);
+            vectorVector<Transform>(x, *y_vec_64, timezone_x, timezone_y, input_rows_count, result);
         else if (const auto * y_const_16 = checkAndGetColumnConst<ColumnDate>(&col_y))
-            vectorConstant<Transform>(x, y_const_16->getValue<UInt16>(), timezone_x, timezone_y, result);
+            vectorConstant<Transform>(x, y_const_16->getValue<UInt16>(), timezone_x, timezone_y, input_rows_count, result);
         else if (const auto * y_const_32 = checkAndGetColumnConst<ColumnDateTime>(&col_y))
-            vectorConstant<Transform>(x, y_const_32->getValue<UInt32>(), timezone_x, timezone_y, result);
+            vectorConstant<Transform>(x, y_const_32->getValue<UInt32>(), timezone_x, timezone_y, input_rows_count, result);
         else if (const auto * y_const_32_s = checkAndGetColumnConst<ColumnDate32>(&col_y))
-            vectorConstant<Transform>(x, y_const_32_s->getValue<Int32>(), timezone_x, timezone_y, result);
+            vectorConstant<Transform>(x, y_const_32_s->getValue<Int32>(), timezone_x, timezone_y, input_rows_count, result);
         else if (const auto * y_const_64 = checkAndGetColumnConst<ColumnDateTime64>(&col_y))
-            vectorConstant<Transform>(x, y_const_64->getValue<DecimalField<DateTime64>>(), timezone_x, timezone_y, result);
+            vectorConstant<Transform>(x, y_const_64->getValue<DecimalField<DateTime64>>(), timezone_x, timezone_y, input_rows_count, result);
         else
-            throw Exception(ErrorCodes::ILLEGAL_COLUMN,
-                "Illegal column for second argument of function {}, must be Date, Date32, DateTime or DateTime64",
-                name);
+            throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column for second argument of function {}, must be Date, Date32, DateTime or DateTime64", name);
     }
 
     template <typename Transform, typename T1>
     void dispatchConstForSecondColumn(
         T1 x, const IColumn & col_y,
         const DateLUTImpl & timezone_x, const DateLUTImpl & timezone_y,
+        size_t input_rows_count,
         ColumnInt64::Container & result) const
     {
         if (const auto * y_vec_16 = checkAndGetColumn<ColumnDate>(&col_y))
-            constantVector<Transform>(x, *y_vec_16, timezone_x, timezone_y, result);
+            constantVector<Transform>(x, *y_vec_16, timezone_x, timezone_y, input_rows_count, result);
         else if (const auto * y_vec_32 = checkAndGetColumn<ColumnDateTime>(&col_y))
-            constantVector<Transform>(x, *y_vec_32, timezone_x, timezone_y, result);
+            constantVector<Transform>(x, *y_vec_32, timezone_x, timezone_y, input_rows_count, result);
         else if (const auto * y_vec_32_s = checkAndGetColumn<ColumnDate32>(&col_y))
-            constantVector<Transform>(x, *y_vec_32_s, timezone_x, timezone_y, result);
+            constantVector<Transform>(x, *y_vec_32_s, timezone_x, timezone_y, input_rows_count, result);
         else if (const auto * y_vec_64 = checkAndGetColumn<ColumnDateTime64>(&col_y))
-            constantVector<Transform>(x, *y_vec_64, timezone_x, timezone_y, result);
+            constantVector<Transform>(x, *y_vec_64, timezone_x, timezone_y, input_rows_count, result);
         else
-            throw Exception(ErrorCodes::ILLEGAL_COLUMN,
-                "Illegal column for second argument of function {}, must be Date, Date32, DateTime or DateTime64",
-                name);
+            throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column for second argument of function {}, must be Date, Date32, DateTime or DateTime64", name);
     }
 
     template <typename Transform, typename LeftColumnType, typename RightColumnType>
     void vectorVector(
         const LeftColumnType & x, const RightColumnType & y,
         const DateLUTImpl & timezone_x, const DateLUTImpl & timezone_y,
+        size_t input_rows_count,
         ColumnInt64::Container & result) const
     {
         const auto & x_data = x.getData();
@@ -130,14 +128,15 @@ public:
 
         const auto transform_x = TransformDateTime64<Transform>(getScale(x));
         const auto transform_y = TransformDateTime64<Transform>(getScale(y));
-        for (size_t i = 0, size = x.size(); i < size; ++i)
-                result[i] = calculate(transform_x, transform_y, x_data[i], y_data[i], timezone_x, timezone_y);
+        for (size_t i = 0; i < input_rows_count; ++i)
+            result[i] = calculate(transform_x, transform_y, x_data[i], y_data[i], timezone_x, timezone_y);
     }
 
     template <typename Transform, typename LeftColumnType, typename T2>
     void vectorConstant(
         const LeftColumnType & x, T2 y,
         const DateLUTImpl & timezone_x, const DateLUTImpl & timezone_y,
+        size_t input_rows_count,
         ColumnInt64::Container & result) const
     {
         const auto & x_data = x.getData();
@@ -145,7 +144,7 @@ public:
         const auto transform_y = TransformDateTime64<Transform>(getScale(y));
         const auto y_value = stripDecimalFieldValue(y);
 
-        for (size_t i = 0, size = x.size(); i < size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
             result[i] = calculate(transform_x, transform_y, x_data[i], y_value, timezone_x, timezone_y);
     }
 
@@ -153,6 +152,7 @@ public:
     void constantVector(
         T1 x, const RightColumnType & y,
         const DateLUTImpl & timezone_x, const DateLUTImpl & timezone_y,
+        size_t input_rows_count,
         ColumnInt64::Container & result) const
     {
         const auto & y_data = y.getData();
@@ -160,19 +160,22 @@ public:
         const auto transform_y = TransformDateTime64<Transform>(getScale(y));
         const auto x_value = stripDecimalFieldValue(x);
 
-        for (size_t i = 0, size = y.size(); i < size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
             result[i] = calculate(transform_x, transform_y, x_value, y_data[i], timezone_x, timezone_y);
     }
 
     template <typename TransformX, typename TransformY, typename T1, typename T2>
     Int64 calculate(const TransformX & transform_x, const TransformY & transform_y, T1 x, T2 y, const DateLUTImpl & timezone_x, const DateLUTImpl & timezone_y) const
     {
+        auto res =  static_cast<Int64>(transform_y.execute(y, timezone_y)) - static_cast<Int64>(transform_x.execute(x, timezone_x));
+
         if constexpr (is_diff)
-            return static_cast<Int64>(transform_y.execute(y, timezone_y)) - static_cast<Int64>(transform_x.execute(x, timezone_x));
+        {
+            return res;
+        }
         else
         {
-            auto res = static_cast<Int64>(transform_y.execute(y, timezone_y))
-                - static_cast<Int64>(transform_x.execute(x, timezone_x));
+            /// Adjust res:
             DateTimeComponentsWithFractionalPart a_comp;
             DateTimeComponentsWithFractionalPart b_comp;
             Int64 adjust_value;
@@ -373,27 +376,27 @@ public:
         const auto & timezone_y = extractTimeZoneFromFunctionArguments(arguments, 3, 2);
 
         if (unit == "year" || unit == "years" || unit == "yy" || unit == "yyyy")
-            impl.template dispatchForColumns<ToRelativeYearNumImpl<ResultPrecision::Extended>>(col_x, col_y, timezone_x, timezone_y, col_res->getData());
+            impl.template dispatchForColumns<ToRelativeYearNumImpl<ResultPrecision::Extended>>(col_x, col_y, timezone_x, timezone_y, input_rows_count, col_res->getData());
         else if (unit == "quarter" || unit == "quarters" || unit == "qq" || unit == "q")
-            impl.template dispatchForColumns<ToRelativeQuarterNumImpl<ResultPrecision::Extended>>(col_x, col_y, timezone_x, timezone_y, col_res->getData());
+            impl.template dispatchForColumns<ToRelativeQuarterNumImpl<ResultPrecision::Extended>>(col_x, col_y, timezone_x, timezone_y, input_rows_count, col_res->getData());
         else if (unit == "month" || unit == "months" || unit == "mm" || unit == "m")
-            impl.template dispatchForColumns<ToRelativeMonthNumImpl<ResultPrecision::Extended>>(col_x, col_y, timezone_x, timezone_y, col_res->getData());
+            impl.template dispatchForColumns<ToRelativeMonthNumImpl<ResultPrecision::Extended>>(col_x, col_y, timezone_x, timezone_y, input_rows_count, col_res->getData());
         else if (unit == "week" || unit == "weeks" || unit == "wk" || unit == "ww")
-            impl.template dispatchForColumns<ToRelativeWeekNumImpl<ResultPrecision::Extended>>(col_x, col_y, timezone_x, timezone_y, col_res->getData());
+            impl.template dispatchForColumns<ToRelativeWeekNumImpl<ResultPrecision::Extended>>(col_x, col_y, timezone_x, timezone_y, input_rows_count, col_res->getData());
         else if (unit == "day" || unit == "days" || unit == "dd" || unit == "d")
-            impl.template dispatchForColumns<ToRelativeDayNumImpl<ResultPrecision::Extended>>(col_x, col_y, timezone_x, timezone_y, col_res->getData());
+            impl.template dispatchForColumns<ToRelativeDayNumImpl<ResultPrecision::Extended>>(col_x, col_y, timezone_x, timezone_y, input_rows_count, col_res->getData());
         else if (unit == "hour" || unit == "hours" || unit == "hh" || unit == "h")
-            impl.template dispatchForColumns<ToRelativeHourNumImpl<ResultPrecision::Extended>>(col_x, col_y, timezone_x, timezone_y, col_res->getData());
+            impl.template dispatchForColumns<ToRelativeHourNumImpl<ResultPrecision::Extended>>(col_x, col_y, timezone_x, timezone_y, input_rows_count, col_res->getData());
         else if (unit == "minute" || unit == "minutes" || unit == "mi" || unit == "n")
-            impl.template dispatchForColumns<ToRelativeMinuteNumImpl<ResultPrecision::Extended>>(col_x, col_y, timezone_x, timezone_y, col_res->getData());
+            impl.template dispatchForColumns<ToRelativeMinuteNumImpl<ResultPrecision::Extended>>(col_x, col_y, timezone_x, timezone_y, input_rows_count, col_res->getData());
         else if (unit == "second" || unit == "seconds" || unit == "ss" || unit == "s")
-            impl.template dispatchForColumns<ToRelativeSecondNumImpl<ResultPrecision::Extended>>(col_x, col_y, timezone_x, timezone_y, col_res->getData());
+            impl.template dispatchForColumns<ToRelativeSecondNumImpl<ResultPrecision::Extended>>(col_x, col_y, timezone_x, timezone_y, input_rows_count, col_res->getData());
         else if (unit == "millisecond" || unit == "milliseconds" || unit == "ms")
-            impl.template dispatchForColumns<ToRelativeSubsecondNumImpl<millisecond_multiplier>>(col_x, col_y, timezone_x, timezone_y, col_res->getData());
+            impl.template dispatchForColumns<ToRelativeSubsecondNumImpl<millisecond_multiplier>>(col_x, col_y, timezone_x, timezone_y, input_rows_count, col_res->getData());
         else if (unit == "microsecond" || unit == "microseconds" || unit == "us" || unit == "u")
-            impl.template dispatchForColumns<ToRelativeSubsecondNumImpl<microsecond_multiplier>>(col_x, col_y, timezone_x, timezone_y, col_res->getData());
+            impl.template dispatchForColumns<ToRelativeSubsecondNumImpl<microsecond_multiplier>>(col_x, col_y, timezone_x, timezone_y, input_rows_count, col_res->getData());
         else if (unit == "nanosecond" || unit == "nanoseconds" || unit == "ns")
-            impl.template dispatchForColumns<ToRelativeSubsecondNumImpl<nanosecond_multiplier>>(col_x, col_y, timezone_x, timezone_y, col_res->getData());
+            impl.template dispatchForColumns<ToRelativeSubsecondNumImpl<nanosecond_multiplier>>(col_x, col_y, timezone_x, timezone_y, input_rows_count, col_res->getData());
         else
             throw Exception(ErrorCodes::BAD_ARGUMENTS, "Function {} does not support '{}' unit", getName(), unit);
 
@@ -440,7 +443,7 @@ public:
 
         auto col_res = ColumnInt64::create(input_rows_count);
 
-        impl.dispatchForColumns<ToRelativeSecondNumImpl<ResultPrecision::Extended>>(col_x, col_y, DateLUT::instance(), DateLUT::instance(), col_res->getData());
+        impl.dispatchForColumns<ToRelativeSecondNumImpl<ResultPrecision::Extended>>(col_x, col_y, DateLUT::instance(), DateLUT::instance(), input_rows_count, col_res->getData());
 
         return col_res;
     }

From 0cab22fd16caf260306e1b1feb77d2ddcced5205 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Fri, 19 Jul 2024 16:44:56 +0200
Subject: [PATCH 0474/1488] Attempt to fix flakiness of 01194_http_query_id

---
 tests/queries/0_stateless/01194_http_query_id.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/queries/0_stateless/01194_http_query_id.sh b/tests/queries/0_stateless/01194_http_query_id.sh
index 5aebdc10dfc..fac17cca3c6 100755
--- a/tests/queries/0_stateless/01194_http_query_id.sh
+++ b/tests/queries/0_stateless/01194_http_query_id.sh
@@ -4,14 +4,14 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
-url="http://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_HTTP}/?session_id=test_01194"
 rnd=$RANDOM
+url="${CLICKHOUSE_URL}&session_id=test_01194_$RANDOM"
 
 ${CLICKHOUSE_CURL} -sS "$url&query=SELECT+'test_01194',$rnd,1" > /dev/null
 ${CLICKHOUSE_CURL} -sS "$url&query=SELECT+'test_01194',$rnd,2" > /dev/null
 ${CLICKHOUSE_CURL} -sS "$url" --data "SELECT 'test_01194',$rnd,3" > /dev/null
 ${CLICKHOUSE_CURL} -sS "$url" --data "SELECT 'test_01194',$rnd,4" > /dev/null
 
-${CLICKHOUSE_CURL} -sS "$url" --data "SYSTEM FLUSH LOGS"
+${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" --data "SYSTEM FLUSH LOGS"
 
-${CLICKHOUSE_CURL} -sS "$url&query=SELECT+count(DISTINCT+query_id)+FROM+system.query_log+WHERE+current_database+LIKE+currentDatabase()+AND+query+LIKE+'SELECT+''test_01194'',$rnd%25'"
+${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" --data "SELECT count(DISTINCT query_id) FROM system.query_log WHERE current_database = currentDatabase() AND event_date >= yesterday() AND query LIKE 'SELECT ''test_01194'',$rnd%'"

From 7d379388d24fba77de9eab9d99e69e7dc23763c2 Mon Sep 17 00:00:00 2001
From: Alexander Sapin <alesapin@gmail.com>
Date: Fri, 19 Jul 2024 17:12:34 +0200
Subject: [PATCH 0475/1488] Turn off randomization of harmful setting

---
 tests/clickhouse-test                     | 5 ++++-
 tests/queries/0_stateless/replication.lib | 8 ++++++--
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/tests/clickhouse-test b/tests/clickhouse-test
index 60160e71e81..f166f8287d5 100755
--- a/tests/clickhouse-test
+++ b/tests/clickhouse-test
@@ -821,7 +821,10 @@ class SettingsRandomizer:
                 get_localzone(),
             ]
         ),
-        "prefer_warmed_unmerged_parts_seconds": lambda: random.randint(0, 10),
+        # This setting affect part names and their content which can be read from tables in tests.
+        # We have a lot of tests which relies on part names, so it's very unsafe to enable randomization
+        # of this setting
+        # "prefer_warmed_unmerged_parts_seconds": lambda: random.randint(0, 10),
         "use_page_cache_for_disks_without_file_cache": lambda: random.random() < 0.7,
         "page_cache_inject_eviction": lambda: random.random() < 0.5,
         "merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability": lambda: round(
diff --git a/tests/queries/0_stateless/replication.lib b/tests/queries/0_stateless/replication.lib
index 2e21f351d2a..1a86cd9f8db 100755
--- a/tests/queries/0_stateless/replication.lib
+++ b/tests/queries/0_stateless/replication.lib
@@ -79,7 +79,9 @@ function check_replication_consistency()
     fi
 
     # Touch all data to check that it's readable (and trigger PartCheckThread if needed)
-    while ! $CLICKHOUSE_CLIENT -q "SELECT * FROM merge(currentDatabase(), '$table_name_prefix') FORMAT Null" 2>/dev/null; do
+    # it's important to disable prefer warmed unmerged parts because
+    # otherwise it can read non-syncrhonized state of replicas
+    while ! $CLICKHOUSE_CLIENT --prefer_warmed_unmerged_parts_seconds=0 -q "SELECT * FROM merge(currentDatabase(), '$table_name_prefix') FORMAT Null" 2>/dev/null; do
         sleep 1;
         num_tries=$((num_tries+1))
         if [ $num_tries -eq 250 ]; then
@@ -102,7 +104,9 @@ function check_replication_consistency()
 
     try_sync_replicas "$table_name_prefix" "$time_left" || exit 1
 
-    res=$($CLICKHOUSE_CLIENT -q \
+    # it's important to disable prefer warmed unmerged parts because
+    # otherwise it can read non-syncrhonized state of replicas
+    res=$($CLICKHOUSE_CLIENT --prefer_warmed_unmerged_parts_seconds=0 -q \
     "SELECT
         if((countDistinct(data) as c) == 0, 1, c)
     FROM

From 6aa754decf9ce156c8e912e9f4e3a0b5e0ece9a8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Fri, 19 Jul 2024 17:38:04 +0200
Subject: [PATCH 0476/1488] Fix docs for default value on from_env

---
 docs/en/operations/configuration-files.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/en/operations/configuration-files.md b/docs/en/operations/configuration-files.md
index 57fea3cca3a..4a818410ebe 100644
--- a/docs/en/operations/configuration-files.md
+++ b/docs/en/operations/configuration-files.md
@@ -124,7 +124,7 @@ which is equal to
 
 #### Default values for from_env and from_zk attributes
 
-It's possible to set the default value and substitute it only if the environment variable or zookeeper node is set using `replace="1"`.
+It's possible to set the default value and substitute it only if the environment variable or zookeeper node is set using `replace="1"` (must be declared before from_env).
 
 With previous example, but `MAX_QUERY_SIZE` is unset:
 
@@ -132,7 +132,7 @@ With previous example, but `MAX_QUERY_SIZE` is unset:
 <clickhouse>
     <profiles>
         <default>
-            <max_query_size from_env="MAX_QUERY_SIZE" replace="1">150000</max_query_size>
+            <max_query_size replace="1" from_env="MAX_QUERY_SIZE">150000</max_query_size>
         </default>
     </profiles>
 </clickhouse>

From e57a10189c5e2444beae08f6e55bab9959cde063 Mon Sep 17 00:00:00 2001
From: Alexander Tokmakov <tavplubix@clickhouse.com>
Date: Fri, 19 Jul 2024 19:09:51 +0200
Subject: [PATCH 0477/1488] Update ci_config.py

---
 tests/ci/ci_config.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py
index 98c7e99a495..d18faaf8f1b 100644
--- a/tests/ci/ci_config.py
+++ b/tests/ci/ci_config.py
@@ -317,42 +317,42 @@ class CI:
             random_bucket="parrepl_with_sanitizer",
         ),
         JobNames.STATELESS_TEST_ASAN: CommonJobConfigs.STATELESS_TEST.with_properties(
-            required_builds=[BuildNames.PACKAGE_ASAN], num_batches=2
+            required_builds=[BuildNames.PACKAGE_ASAN], num_batches=4
         ),
         JobNames.STATELESS_TEST_TSAN: CommonJobConfigs.STATELESS_TEST.with_properties(
-            required_builds=[BuildNames.PACKAGE_TSAN], num_batches=2
+            required_builds=[BuildNames.PACKAGE_TSAN], num_batches=5
         ),
         JobNames.STATELESS_TEST_MSAN: CommonJobConfigs.STATELESS_TEST.with_properties(
-            required_builds=[BuildNames.PACKAGE_MSAN], num_batches=3
+            required_builds=[BuildNames.PACKAGE_MSAN], num_batches=6
         ),
         JobNames.STATELESS_TEST_UBSAN: CommonJobConfigs.STATELESS_TEST.with_properties(
-            required_builds=[BuildNames.PACKAGE_UBSAN], num_batches=1
+            required_builds=[BuildNames.PACKAGE_UBSAN], num_batches=2
         ),
         JobNames.STATELESS_TEST_DEBUG: CommonJobConfigs.STATELESS_TEST.with_properties(
-            required_builds=[BuildNames.PACKAGE_DEBUG], num_batches=2
+            required_builds=[BuildNames.PACKAGE_DEBUG], num_batches=5
         ),
         JobNames.STATELESS_TEST_RELEASE: CommonJobConfigs.STATELESS_TEST.with_properties(
             required_builds=[BuildNames.PACKAGE_RELEASE],
         ),
         JobNames.STATELESS_TEST_RELEASE_COVERAGE: CommonJobConfigs.STATELESS_TEST.with_properties(
-            required_builds=[BuildNames.PACKAGE_RELEASE_COVERAGE], num_batches=5
+            required_builds=[BuildNames.PACKAGE_RELEASE_COVERAGE], num_batches=6
         ),
         JobNames.STATELESS_TEST_AARCH64: CommonJobConfigs.STATELESS_TEST.with_properties(
             required_builds=[BuildNames.PACKAGE_AARCH64],
             runner_type=Runners.FUNC_TESTER_ARM,
         ),
         JobNames.STATELESS_TEST_OLD_ANALYZER_S3_REPLICATED_RELEASE: CommonJobConfigs.STATELESS_TEST.with_properties(
-            required_builds=[BuildNames.PACKAGE_RELEASE], num_batches=3
+            required_builds=[BuildNames.PACKAGE_RELEASE], num_batches=4
         ),
         JobNames.STATELESS_TEST_S3_DEBUG: CommonJobConfigs.STATELESS_TEST.with_properties(
-            required_builds=[BuildNames.PACKAGE_DEBUG], num_batches=2
+            required_builds=[BuildNames.PACKAGE_DEBUG], num_batches=6
         ),
         JobNames.STATELESS_TEST_AZURE_ASAN: CommonJobConfigs.STATELESS_TEST.with_properties(
-            required_builds=[BuildNames.PACKAGE_ASAN], num_batches=2, release_only=True
+            required_builds=[BuildNames.PACKAGE_ASAN], num_batches=4, release_only=True
         ),
         JobNames.STATELESS_TEST_S3_TSAN: CommonJobConfigs.STATELESS_TEST.with_properties(
             required_builds=[BuildNames.PACKAGE_TSAN],
-            num_batches=3,
+            num_batches=5,
         ),
         JobNames.STRESS_TEST_DEBUG: CommonJobConfigs.STRESS_TEST.with_properties(
             required_builds=[BuildNames.PACKAGE_DEBUG],

From 38126bb436c3f46cfdc321abca421f7fae969f5b Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitlibar@clickhouse.com>
Date: Fri, 19 Jul 2024 19:09:55 +0200
Subject: [PATCH 0478/1488] Add test.

---
 .../configs/disk_s3.xml                       | 22 +++++++++++++++
 .../test_backup_restore_s3/test.py            | 28 ++++++-------------
 2 files changed, 31 insertions(+), 19 deletions(-)

diff --git a/tests/integration/test_backup_restore_s3/configs/disk_s3.xml b/tests/integration/test_backup_restore_s3/configs/disk_s3.xml
index 45a1e17b039..7ac1a052c30 100644
--- a/tests/integration/test_backup_restore_s3/configs/disk_s3.xml
+++ b/tests/integration/test_backup_restore_s3/configs/disk_s3.xml
@@ -21,6 +21,13 @@
                 <secret_access_key>minio123</secret_access_key>
                 <s3_max_single_part_upload_size>33554432</s3_max_single_part_upload_size>
             </disk_s3_plain>
+            <disk_s3_plain_rewritable>
+                <type>s3_plain_rewritable</type>
+                <endpoint>http://minio1:9001/root/data/disks/disk_s3_plain_rewritable/</endpoint>
+                <access_key_id>minio</access_key_id>
+                <secret_access_key>minio123</secret_access_key>
+                <s3_max_single_part_upload_size>33554432</s3_max_single_part_upload_size>
+            </disk_s3_plain_rewritable>
             <disk_s3_cache>
                 <type>cache</type>
                 <disk>disk_s3</disk>
@@ -37,6 +44,20 @@
                     </main>
                 </volumes>
             </policy_s3>
+            <policy_s3_plain>
+                <volumes>
+                    <main>
+                        <disk>disk_s3_plain</disk>
+                    </main>
+                </volumes>
+            </policy_s3_plain>
+            <policy_s3_plain_rewritable>
+                <volumes>
+                    <main>
+                        <disk>disk_s3_plain_rewritable</disk>
+                    </main>
+                </volumes>
+            </policy_s3_plain_rewritable>
             <policy_s3_other_bucket>
                 <volumes>
                     <main>
@@ -57,6 +78,7 @@
         <allowed_disk>default</allowed_disk>
         <allowed_disk>disk_s3</allowed_disk>
         <allowed_disk>disk_s3_plain</allowed_disk>
+        <allowed_disk>disk_s3_plain_rewritable</allowed_disk>
         <allowed_disk>disk_s3_cache</allowed_disk>
         <allowed_disk>disk_s3_other_bucket</allowed_disk>
     </backups>
diff --git a/tests/integration/test_backup_restore_s3/test.py b/tests/integration/test_backup_restore_s3/test.py
index d53335000a6..4840f5afc66 100644
--- a/tests/integration/test_backup_restore_s3/test.py
+++ b/tests/integration/test_backup_restore_s3/test.py
@@ -419,25 +419,15 @@ def test_backup_to_s3_multipart():
     assert "ReadBufferFromS3RequestsErrors" not in restore_events
 
 
-def test_backup_to_s3_native_copy():
-    storage_policy = "policy_s3"
-    backup_name = new_backup_name()
-    backup_destination = (
-        f"S3('http://minio1:9001/root/data/backups/{backup_name}', 'minio', 'minio123')"
-    )
-    (backup_events, restore_events) = check_backup_and_restore(
-        storage_policy, backup_destination
-    )
-    # single part upload
-    assert backup_events["S3CopyObject"] > 0
-    assert restore_events["S3CopyObject"] > 0
-    assert node.contains_in_log(
-        f"copyS3File: Single operation copy has completed. Bucket: root, Key: data/backups/{backup_name}"
-    )
-
-
-def test_backup_to_s3_native_copy_other_bucket():
-    storage_policy = "policy_s3_other_bucket"
+@pytest.mark.parametrize(
+    "storage_policy",
+    [
+        "policy_s3",
+        "policy_s3_other_bucket",
+        "policy_s3_plain_rewritable",
+    ],
+)
+def test_backup_to_s3_native_copy(storage_policy):
     backup_name = new_backup_name()
     backup_destination = (
         f"S3('http://minio1:9001/root/data/backups/{backup_name}', 'minio', 'minio123')"

From 5bea6751e07ef9753db0e532d476056679a4393a Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitlibar@clickhouse.com>
Date: Fri, 19 Jul 2024 19:11:07 +0200
Subject: [PATCH 0479/1488] Make the error message about broken parts more
 useful.

---
 src/Storages/MergeTree/IMergeTreeDataPart.cpp | 17 ++++++++++++++++-
 src/Storages/MergeTree/MergeTreeData.cpp      |  4 +++-
 2 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp
index c2f87018872..3a44359b537 100644
--- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp
+++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp
@@ -739,10 +739,25 @@ void IMergeTreeDataPart::loadColumnsChecksumsIndexes(bool require_columns_checks
     }
     catch (...)
     {
-        /// Don't scare people with broken part error
+        /// Don't scare people with broken part error if it's retryable.
         if (!isRetryableException(std::current_exception()))
+        {
             LOG_ERROR(storage.log, "Part {} is broken and needs manual correction", getDataPartStorage().getFullPath());
 
+            if (Exception * e = exception_cast<Exception *>(std::current_exception()))
+            {
+                /// Probably there is something wrong with files of this part.
+                /// So it can be helpful to add to the error message some information about those files.
+                String files_in_part;
+                for (auto it = getDataPartStorage().iterate(); it->isValid(); it->next())
+                    files_in_part += fmt::format("{}{} ({} bytes)", (files_in_part.empty() ? "" : ", "), it->name(), getDataPartStorage().getFileSize(it->name()));
+                if (!files_in_part.empty())
+                    e->addMessage("Part contains files: {}", files_in_part);
+                if (isEmpty())
+                    e->addMessage("Part is empty");
+            }
+        }
+
         // There could be conditions that data part to be loaded is broken, but some of meta infos are already written
         // into metadata before exception, need to clean them all.
         metadata_manager->deleteAll(/*include_projection*/ true);
diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index 78a551591a6..12a4effe33c 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -5634,9 +5634,11 @@ void MergeTreeData::restorePartFromBackup(std::shared_ptr<RestoredPartsHolder> r
     String part_name = part_info.getPartNameAndCheckFormat(format_version);
     auto backup = restored_parts_holder->getBackup();
 
+    /// Find all files of this part in the backup.
+    Strings filenames = backup->listFiles(part_path_in_backup, /* recursive= */ true);
+
     /// Calculate the total size of the part.
     UInt64 total_size_of_part = 0;
-    Strings filenames = backup->listFiles(part_path_in_backup, /* recursive= */ true);
     fs::path part_path_in_backup_fs = part_path_in_backup;
     for (const String & filename : filenames)
         total_size_of_part += backup->getFileSize(part_path_in_backup_fs / filename);

From 3abf636853e755d0a5264eb568ce895e55749920 Mon Sep 17 00:00:00 2001
From: Alexander Tokmakov <tavplubix@clickhouse.com>
Date: Fri, 19 Jul 2024 19:16:53 +0200
Subject: [PATCH 0480/1488] fix for 992 and friends

---
 src/Storages/StorageReplicatedMergeTree.cpp                  | 2 +-
 .../00992_system_parts_race_condition_zookeeper_long.sh      | 2 +-
 tests/queries/0_stateless/replication.lib                    | 5 ++++-
 3 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp
index 72f725965e0..416100def4c 100644
--- a/src/Storages/StorageReplicatedMergeTree.cpp
+++ b/src/Storages/StorageReplicatedMergeTree.cpp
@@ -3940,7 +3940,7 @@ void StorageReplicatedMergeTree::mergeSelectingTask()
         merge_selecting_task->schedule();
     else
     {
-        LOG_TRACE(log, "Scheduling next merge selecting task after {}ms", merge_selecting_sleep_ms);
+        LOG_TRACE(log, "Scheduling next merge selecting task after {}ms, current attempt status: {}", merge_selecting_sleep_ms, result);
         merge_selecting_task->scheduleAfter(merge_selecting_sleep_ms);
     }
 }
diff --git a/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper_long.sh b/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper_long.sh
index 4887c409844..02a739ece4a 100755
--- a/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper_long.sh
+++ b/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper_long.sh
@@ -41,7 +41,7 @@ function thread3()
 
 function thread4()
 {
-    while true; do $CLICKHOUSE_CLIENT --receive_timeout=3 -q "OPTIMIZE TABLE alter_table0 FINAL" | grep -Fv "Timeout exceeded while receiving data from server"; done
+    while true; do $CLICKHOUSE_CLIENT --receive_timeout=1 -q "OPTIMIZE TABLE alter_table0 FINAL" | grep -Fv "Timeout exceeded while receiving data from server"; done
 }
 
 function thread5()
diff --git a/tests/queries/0_stateless/replication.lib b/tests/queries/0_stateless/replication.lib
index 2e21f351d2a..6331a5eb406 100755
--- a/tests/queries/0_stateless/replication.lib
+++ b/tests/queries/0_stateless/replication.lib
@@ -51,6 +51,9 @@ function check_replication_consistency()
     table_name_prefix=$1
     check_query_part=$2
 
+    # Try to kill some mutations because sometimes tests run too much (it's not guarenteed to kill all mutations, see below)
+    ${CLICKHOUSE_CLIENT} -q "KILL MUTATION WHERE database=currentDatabase() AND table like '$table_name_prefix%'" > /dev/null
+
     # Wait for all queries to finish (query may still be running if thread is killed by timeout)
     num_tries=0
     while [[ $($CLICKHOUSE_CLIENT -q "SELECT count() FROM system.processes WHERE current_database=currentDatabase() AND query LIKE '%$table_name_prefix%'") -ne 1 ]]; do
@@ -94,7 +97,7 @@ function check_replication_consistency()
     some_table=$($CLICKHOUSE_CLIENT -q "SELECT name FROM system.tables WHERE database=currentDatabase() AND name like '$table_name_prefix%' ORDER BY rand() LIMIT 1")
     $CLICKHOUSE_CLIENT -q "SYSTEM SYNC REPLICA $some_table PULL" 1>/dev/null 2>/dev/null ||:
 
-    # Forcefully cancel mutations to avoid waiting for them to finish
+    # Forcefully cancel mutations to avoid waiting for them to finish. Kills the remaining mutations
     ${CLICKHOUSE_CLIENT} -q "KILL MUTATION WHERE database=currentDatabase() AND table like '$table_name_prefix%'" > /dev/null
 
     # SYNC REPLICA is not enough if some MUTATE_PARTs are not assigned yet

From e1a659f83b96e33b9482157f0a18dd3c3efb3926 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 19 Jul 2024 19:24:22 +0200
Subject: [PATCH 0481/1488] Remove obsolete code from CMakeLists

---
 src/CMakeLists.txt | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index d985595154c..dac20aaca48 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -419,9 +419,6 @@ dbms_target_link_libraries (
         boost::circular_buffer
         boost::heap)
 
-target_include_directories(clickhouse_common_io PUBLIC "${CMAKE_CURRENT_BINARY_DIR}/Core/include") # uses some includes from core
-dbms_target_include_directories(PUBLIC "${CMAKE_CURRENT_BINARY_DIR}/Core/include")
-
 target_link_libraries(clickhouse_common_io PUBLIC
     ch_contrib::miniselect
     ch_contrib::pdqsort)

From a11e89f4f70b101553a138d367d7f6dcd8318554 Mon Sep 17 00:00:00 2001
From: Nikita Taranov <nikita.taranov@clickhouse.com>
Date: Fri, 19 Jul 2024 19:00:45 +0100
Subject: [PATCH 0482/1488] impl

---
 base/base/defines.h      | 6 +-----
 src/Common/Exception.cpp | 4 ++--
 2 files changed, 3 insertions(+), 7 deletions(-)

diff --git a/base/base/defines.h b/base/base/defines.h
index cf3d357da18..5685a6d9833 100644
--- a/base/base/defines.h
+++ b/base/base/defines.h
@@ -96,10 +96,6 @@
 #    endif
 #endif
 
-#if !defined(ABORT_ON_LOGICAL_ERROR) && defined(DEBUG_OR_SANITIZER_BUILD)
-#    define ABORT_ON_LOGICAL_ERROR
-#endif
-
 /// chassert(x) is similar to assert(x), but:
 ///     - works in builds with sanitizers, not only in debug builds
 ///     - tries to print failed assertion into server log
@@ -108,7 +104,7 @@
 /// Also it makes sense to call abort() instead of __builtin_unreachable() in debug builds,
 /// because SIGABRT is easier to debug than SIGTRAP (the second one makes gdb crazy)
 #if !defined(chassert)
-    #if defined(ABORT_ON_LOGICAL_ERROR)
+#    if defined(DEBUG_OR_SANITIZER_BUILD)
         // clang-format off
         #include <base/types.h>
         namespace DB
diff --git a/src/Common/Exception.cpp b/src/Common/Exception.cpp
index 111280074dd..33befa64946 100644
--- a/src/Common/Exception.cpp
+++ b/src/Common/Exception.cpp
@@ -64,7 +64,7 @@ void handle_error_code(const std::string & msg, int code, bool remote, const Exc
 {
     // In debug builds and builds with sanitizers, treat LOGICAL_ERROR as an assertion failure.
     // Log the message before we fail.
-#ifdef ABORT_ON_LOGICAL_ERROR
+#ifdef DEBUG_OR_SANITIZER_BUILD
     if (code == ErrorCodes::LOGICAL_ERROR)
     {
         abortOnFailedAssertion(msg, trace.data(), 0, trace.size());
@@ -443,7 +443,7 @@ PreformattedMessage getCurrentExceptionMessageAndPattern(bool with_stacktrace, b
         }
         catch (...) {} // NOLINT(bugprone-empty-catch)
 
-#ifdef ABORT_ON_LOGICAL_ERROR
+#ifdef DEBUG_OR_SANITIZER_BUILD
         try
         {
             throw;

From 00e14bde80c343919bc237bdcdeb15f22f153eab Mon Sep 17 00:00:00 2001
From: Nikita Fomichev <nikita.fomichev@clickhouse.com>
Date: Fri, 19 Jul 2024 19:52:30 +0200
Subject: [PATCH 0483/1488] Update ci_config.py

---
 tests/ci/ci_config.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py
index d18faaf8f1b..00b545f9f33 100644
--- a/tests/ci/ci_config.py
+++ b/tests/ci/ci_config.py
@@ -317,19 +317,19 @@ class CI:
             random_bucket="parrepl_with_sanitizer",
         ),
         JobNames.STATELESS_TEST_ASAN: CommonJobConfigs.STATELESS_TEST.with_properties(
-            required_builds=[BuildNames.PACKAGE_ASAN], num_batches=4
+            required_builds=[BuildNames.PACKAGE_ASAN], num_batches=2
         ),
         JobNames.STATELESS_TEST_TSAN: CommonJobConfigs.STATELESS_TEST.with_properties(
-            required_builds=[BuildNames.PACKAGE_TSAN], num_batches=5
+            required_builds=[BuildNames.PACKAGE_TSAN], num_batches=4
         ),
         JobNames.STATELESS_TEST_MSAN: CommonJobConfigs.STATELESS_TEST.with_properties(
-            required_builds=[BuildNames.PACKAGE_MSAN], num_batches=6
+            required_builds=[BuildNames.PACKAGE_MSAN], num_batches=4
         ),
         JobNames.STATELESS_TEST_UBSAN: CommonJobConfigs.STATELESS_TEST.with_properties(
             required_builds=[BuildNames.PACKAGE_UBSAN], num_batches=2
         ),
         JobNames.STATELESS_TEST_DEBUG: CommonJobConfigs.STATELESS_TEST.with_properties(
-            required_builds=[BuildNames.PACKAGE_DEBUG], num_batches=5
+            required_builds=[BuildNames.PACKAGE_DEBUG], num_batches=2
         ),
         JobNames.STATELESS_TEST_RELEASE: CommonJobConfigs.STATELESS_TEST.with_properties(
             required_builds=[BuildNames.PACKAGE_RELEASE],
@@ -345,14 +345,14 @@ class CI:
             required_builds=[BuildNames.PACKAGE_RELEASE], num_batches=4
         ),
         JobNames.STATELESS_TEST_S3_DEBUG: CommonJobConfigs.STATELESS_TEST.with_properties(
-            required_builds=[BuildNames.PACKAGE_DEBUG], num_batches=6
+            required_builds=[BuildNames.PACKAGE_DEBUG], num_batches=2
         ),
         JobNames.STATELESS_TEST_AZURE_ASAN: CommonJobConfigs.STATELESS_TEST.with_properties(
-            required_builds=[BuildNames.PACKAGE_ASAN], num_batches=4, release_only=True
+            required_builds=[BuildNames.PACKAGE_ASAN], num_batches=3, release_only=True
         ),
         JobNames.STATELESS_TEST_S3_TSAN: CommonJobConfigs.STATELESS_TEST.with_properties(
             required_builds=[BuildNames.PACKAGE_TSAN],
-            num_batches=5,
+            num_batches=4,
         ),
         JobNames.STRESS_TEST_DEBUG: CommonJobConfigs.STRESS_TEST.with_properties(
             required_builds=[BuildNames.PACKAGE_DEBUG],

From fa2659a87cde713011b2a2452269397216c54d6f Mon Sep 17 00:00:00 2001
From: Nikita Fomichev <nikita.fomichev@clickhouse.com>
Date: Fri, 19 Jul 2024 20:10:08 +0200
Subject: [PATCH 0484/1488] Increase CI timeout for stateless tests

---
 docker/test/stateless/run.sh | 4 ++--
 tests/ci/ci_definitions.py   | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh
index 0647ed02839..264dcd09da3 100755
--- a/docker/test/stateless/run.sh
+++ b/docker/test/stateless/run.sh
@@ -6,8 +6,8 @@ source /setup_export_logs.sh
 # fail on errors, verbose and export all env variables
 set -e -x -a
 
-MAX_RUN_TIME=${MAX_RUN_TIME:-7200}
-MAX_RUN_TIME=$((MAX_RUN_TIME == 0 ? 7200 : MAX_RUN_TIME))
+MAX_RUN_TIME=${MAX_RUN_TIME:-9000}
+MAX_RUN_TIME=$((MAX_RUN_TIME == 0 ? 9000 : MAX_RUN_TIME))
 
 USE_DATABASE_REPLICATED=${USE_DATABASE_REPLICATED:=0}
 USE_SHARED_CATALOG=${USE_SHARED_CATALOG:=0}
diff --git a/tests/ci/ci_definitions.py b/tests/ci/ci_definitions.py
index d41a621bc2e..8addc0de077 100644
--- a/tests/ci/ci_definitions.py
+++ b/tests/ci/ci_definitions.py
@@ -432,7 +432,7 @@ class CommonJobConfigs:
         ),
         run_command='functional_test_check.py "$CHECK_NAME"',
         runner_type=Runners.FUNC_TESTER,
-        timeout=7200,
+        timeout=9000,
     )
     STATEFUL_TEST = JobConfig(
         job_name_keyword="stateful",

From 1e160189f0652f476b3efc0ea1ebeb0a1eb5e162 Mon Sep 17 00:00:00 2001
From: Nikita Fomichev <nikita.fomichev@clickhouse.com>
Date: Fri, 19 Jul 2024 20:10:41 +0200
Subject: [PATCH 0485/1488] Increase parallel jobs for stateless tests

---
 docker/test/stateless/run.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh
index 264dcd09da3..670a2420ddf 100755
--- a/docker/test/stateless/run.sh
+++ b/docker/test/stateless/run.sh
@@ -224,7 +224,7 @@ function run_tests()
     else
         # All other configurations are OK.
         ADDITIONAL_OPTIONS+=('--jobs')
-        ADDITIONAL_OPTIONS+=('7')
+        ADDITIONAL_OPTIONS+=('8')
     fi
 
     if [[ -n "$RUN_BY_HASH_NUM" ]] && [[ -n "$RUN_BY_HASH_TOTAL" ]]; then

From 7fd779cc6f6b271366a27b2cbcfe6bad8cb15913 Mon Sep 17 00:00:00 2001
From: Nikita Fomichev <nikita.fomichev@clickhouse.com>
Date: Fri, 19 Jul 2024 20:41:52 +0200
Subject: [PATCH 0486/1488] Stateless tests: split parallel tests more evenly

---
 tests/clickhouse-test | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/tests/clickhouse-test b/tests/clickhouse-test
index a8a6ce68b8f..c543e181c4f 100755
--- a/tests/clickhouse-test
+++ b/tests/clickhouse-test
@@ -2168,7 +2168,10 @@ def run_tests_array(all_tests_with_params: Tuple[List[str], int, TestSuite, bool
 
     while True:
         if all_tests:
-            case = all_tests.pop(0)
+            try:
+                case = all_tests.pop(0)
+            except IndexError:
+                break
         else:
             break
 
@@ -2474,18 +2477,16 @@ def do_run_tests(jobs, test_suite: TestSuite):
         # of failures will be nearly the same for all tests from the group.
         random.shuffle(test_suite.parallel_tests)
 
-        batch_size = max(1, (len(test_suite.parallel_tests) // jobs) + 1)
-        parallel_tests_array = []
-        for job in range(jobs):
-            range_ = job * batch_size, job * batch_size + batch_size
-            batch = test_suite.parallel_tests[range_[0] : range_[1]]
-            parallel_tests_array.append((batch, batch_size, test_suite, True))
+        batch_size = len(test_suite.parallel_tests) // jobs
+        manager = multiprocessing.Manager()
+        parallel_tests = manager.list()
+        parallel_tests.extend(test_suite.parallel_tests)
 
         processes = []
-
-        for test_batch in parallel_tests_array:
+        for job in range(jobs):
             process = multiprocessing.Process(
-                target=run_tests_process, args=(test_batch,)
+                target=run_tests_process,
+                args=((parallel_tests, batch_size, test_suite, True),),
             )
             processes.append(process)
             process.start()

From 565ee9ad66ba1198fdbec9ad6c1b19b76e9cf185 Mon Sep 17 00:00:00 2001
From: Anton Popov <anton@clickhouse.com>
Date: Fri, 19 Jul 2024 21:05:21 +0200
Subject: [PATCH 0487/1488] Update 02724_limit_num_mutations.sh

---
 tests/queries/0_stateless/02724_limit_num_mutations.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/queries/0_stateless/02724_limit_num_mutations.sh b/tests/queries/0_stateless/02724_limit_num_mutations.sh
index a9d69b2ed48..60888db0e2e 100755
--- a/tests/queries/0_stateless/02724_limit_num_mutations.sh
+++ b/tests/queries/0_stateless/02724_limit_num_mutations.sh
@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
+# Tags: no-parallel
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh

From 14a0c7e300e1f451ab9d46fc6cfd8c1e5f811946 Mon Sep 17 00:00:00 2001
From: Nikita Fomichev <nikita.fomichev@clickhouse.com>
Date: Fri, 19 Jul 2024 21:13:51 +0200
Subject: [PATCH 0488/1488] Stateless tests: fix stylecheck

---
 tests/clickhouse-test | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/clickhouse-test b/tests/clickhouse-test
index c543e181c4f..6edd3159c7f 100755
--- a/tests/clickhouse-test
+++ b/tests/clickhouse-test
@@ -2483,7 +2483,7 @@ def do_run_tests(jobs, test_suite: TestSuite):
         parallel_tests.extend(test_suite.parallel_tests)
 
         processes = []
-        for job in range(jobs):
+        for _ in range(jobs):
             process = multiprocessing.Process(
                 target=run_tests_process,
                 args=((parallel_tests, batch_size, test_suite, True),),

From f3fb729f53860d55db1d72ccfc88f9c5d018aea1 Mon Sep 17 00:00:00 2001
From: Igor Nikonov <igor@clickhouse.com>
Date: Fri, 19 Jul 2024 20:12:14 +0000
Subject: [PATCH 0489/1488] Call onAsyncJobReady()

---
 src/Processors/Executors/ExecutorTasks.cpp  | 2 ++
 src/Processors/Executors/ExecutorTasks.h    | 2 +-
 src/Processors/Executors/PipelineExecutor.h | 1 -
 3 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/Processors/Executors/ExecutorTasks.cpp b/src/Processors/Executors/ExecutorTasks.cpp
index 7e3bee239ef..d045f59a2e2 100644
--- a/src/Processors/Executors/ExecutorTasks.cpp
+++ b/src/Processors/Executors/ExecutorTasks.cpp
@@ -204,6 +204,8 @@ void ExecutorTasks::processAsyncTasks()
         while (auto task = async_task_queue.wait(lock))
         {
             auto * node = static_cast<ExecutingGraph::Node *>(task.data);
+            node->processor->onAsyncJobReady();
+
             executor_contexts[task.thread_num]->pushAsyncTask(node);
             ++num_waiting_async_tasks;
 
diff --git a/src/Processors/Executors/ExecutorTasks.h b/src/Processors/Executors/ExecutorTasks.h
index 202ca253c6c..b2201873edf 100644
--- a/src/Processors/Executors/ExecutorTasks.h
+++ b/src/Processors/Executors/ExecutorTasks.h
@@ -28,7 +28,7 @@ class ExecutorTasks
     TaskQueue<ExecutingGraph::Node> task_queue;
 
     /// Queue which stores tasks where processors returned Async status after prepare.
-    /// If multiple threads are using, main thread will wait for async tasks.
+    /// If multiple threads are used, main thread will wait for async tasks.
     /// For single thread, will wait for async tasks only when task_queue is empty.
     PollingQueue async_task_queue;
 
diff --git a/src/Processors/Executors/PipelineExecutor.h b/src/Processors/Executors/PipelineExecutor.h
index 03f0f7f1a0a..ae119355cb5 100644
--- a/src/Processors/Executors/PipelineExecutor.h
+++ b/src/Processors/Executors/PipelineExecutor.h
@@ -9,7 +9,6 @@
 
 #include <deque>
 #include <queue>
-#include <mutex>
 #include <memory>
 
 

From c948103dee50bf5bddeff9af485d4df2acc8b0f7 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 19 Jul 2024 22:23:50 +0200
Subject: [PATCH 0490/1488] Better tests

---
 .../0_stateless/01338_long_select_and_alter.reference       | 2 +-
 tests/queries/0_stateless/01338_long_select_and_alter.sh    | 6 +++---
 .../01338_long_select_and_alter_zookeeper.reference         | 2 +-
 .../0_stateless/01338_long_select_and_alter_zookeeper.sh    | 6 +++---
 4 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/tests/queries/0_stateless/01338_long_select_and_alter.reference b/tests/queries/0_stateless/01338_long_select_and_alter.reference
index 921730b17ce..276d6bcc29d 100644
--- a/tests/queries/0_stateless/01338_long_select_and_alter.reference
+++ b/tests/queries/0_stateless/01338_long_select_and_alter.reference
@@ -1,3 +1,3 @@
 5
-10
+5
 CREATE TABLE default.alter_mt\n(\n    `key` UInt64,\n    `value` UInt64\n)\nENGINE = MergeTree\nORDER BY key\nSETTINGS index_granularity = 8192
diff --git a/tests/queries/0_stateless/01338_long_select_and_alter.sh b/tests/queries/0_stateless/01338_long_select_and_alter.sh
index 2659e5c16cf..08609546ff5 100755
--- a/tests/queries/0_stateless/01338_long_select_and_alter.sh
+++ b/tests/queries/0_stateless/01338_long_select_and_alter.sh
@@ -9,7 +9,7 @@ $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS alter_mt"
 
 $CLICKHOUSE_CLIENT --query "CREATE TABLE alter_mt (key UInt64, value String) ENGINE=MergeTree() ORDER BY key"
 
-$CLICKHOUSE_CLIENT --query "INSERT INTO alter_mt SELECT number, toString(number) FROM numbers(5)"
+$CLICKHOUSE_CLIENT --query "INSERT INTO alter_mt SELECT number - 1 AS x, toString(x) FROM numbers(5)"
 
 $CLICKHOUSE_CLIENT --function_sleep_max_microseconds_per_block 10000000 --query "SELECT count(distinct concat(value, '_')) FROM alter_mt WHERE not sleepEachRow(2)" &
 
@@ -18,10 +18,10 @@ sleep 2
 
 $CLICKHOUSE_CLIENT --query "ALTER TABLE alter_mt MODIFY COLUMN value UInt64"
 
-wait
-
 $CLICKHOUSE_CLIENT --query "SELECT sum(value) FROM alter_mt"
 
+wait
+
 $CLICKHOUSE_CLIENT --query "SHOW CREATE TABLE alter_mt"
 
 $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS alter_mt"
diff --git a/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.reference b/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.reference
index 9c5ad0fa468..aab1b93f6bd 100644
--- a/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.reference
+++ b/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.reference
@@ -1,3 +1,3 @@
 5
-10
+5
 CREATE TABLE default.alter_mt\n(\n    `key` UInt64,\n    `value` UInt64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/01338_long_select_and_alter_zookeeper_default/alter_mt\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192
diff --git a/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.sh b/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.sh
index 6eb795408f4..def6d2ab127 100755
--- a/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.sh
+++ b/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.sh
@@ -9,7 +9,7 @@ $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS alter_mt"
 
 $CLICKHOUSE_CLIENT --query "CREATE TABLE alter_mt (key UInt64, value String) ENGINE=ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/alter_mt', '1') ORDER BY key"
 
-$CLICKHOUSE_CLIENT --query "INSERT INTO alter_mt SELECT number, toString(number) FROM numbers(5)"
+$CLICKHOUSE_CLIENT --query "INSERT INTO alter_mt SELECT number - 1 AS x, toString(x) FROM numbers(5)"
 
 $CLICKHOUSE_CLIENT --function_sleep_max_microseconds_per_block 10000000 --query "SELECT count(distinct concat(value, '_')) FROM alter_mt WHERE not sleepEachRow(2)" &
 
@@ -18,10 +18,10 @@ sleep 2
 
 $CLICKHOUSE_CLIENT --query "ALTER TABLE alter_mt MODIFY COLUMN value UInt64"
 
-wait
-
 $CLICKHOUSE_CLIENT --query "SELECT sum(value) FROM alter_mt"
 
+wait
+
 $CLICKHOUSE_CLIENT --query "SHOW CREATE TABLE alter_mt"
 
 $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS alter_mt"

From 465a34d3dfe3e313471e10d59cab8219b3e5837e Mon Sep 17 00:00:00 2001
From: Igor Nikonov <igor@clickhouse.com>
Date: Fri, 19 Jul 2024 20:27:57 +0000
Subject: [PATCH 0491/1488] Simplify, fix build

---
 src/QueryPipeline/RemoteQueryExecutor.cpp | 48 +++++++++++------------
 1 file changed, 22 insertions(+), 26 deletions(-)

diff --git a/src/QueryPipeline/RemoteQueryExecutor.cpp b/src/QueryPipeline/RemoteQueryExecutor.cpp
index b78c38a4134..61a512bcfc5 100644
--- a/src/QueryPipeline/RemoteQueryExecutor.cpp
+++ b/src/QueryPipeline/RemoteQueryExecutor.cpp
@@ -924,41 +924,37 @@ bool RemoteQueryExecutor::needToSkipUnavailableShard() const
     return context->getSettingsRef().skip_unavailable_shards && (0 == connections->size());
 }
 
-bool  RemoteQueryExecutor::processParallelReplicaPacketIfAny()
+bool RemoteQueryExecutor::processParallelReplicaPacketIfAny()
 {
 #if defined(OS_LINUX)
+
+    std::lock_guard lock(was_cancelled_mutex);
+    if (was_cancelled)
+        return false;
+
     if (!read_context || (resent_query && recreate_read_context))
     {
-        std::lock_guard lock(was_cancelled_mutex);
-        if (was_cancelled)
-            return false;
-
         read_context = std::make_unique<ReadContext>(*this);
         recreate_read_context = false;
     }
 
-    {
-        std::lock_guard lock(was_cancelled_mutex);
-        if (was_cancelled)
-            return false;
+    chassert(!has_postponed_packet);
 
-        chassert(!has_postponed_packet);
-
-        read_context->resume();
-        if (read_context->isInProgress()) // <- nothing to process
-            return false;
-
-        const auto packet_type = read_context->getPacketType();
-        if (packet_type == Protocol::Server::MergeTreeReadTaskRequest || packet_type == Protocol::Server::MergeTreeAllRangesAnnouncement)
-        {
-            processPacket(read_context->getPacket());
-            return true;
-        }
-
-        has_postponed_packet = true;
+    read_context->resume();
+    if (read_context->isInProgress()) // <- nothing to process
         return false;
-    }
-#endif
-}
 
+    const auto packet_type = read_context->getPacketType();
+    if (packet_type == Protocol::Server::MergeTreeReadTaskRequest || packet_type == Protocol::Server::MergeTreeAllRangesAnnouncement)
+    {
+        processPacket(read_context->getPacket());
+        return true;
+    }
+
+    has_postponed_packet = true;
+
+#endif
+
+    return false;
+}
 }

From 277dbfa0574b567241d169494a459e6f2b04d5e6 Mon Sep 17 00:00:00 2001
From: "Zhukova, Maria" <maria.zhukova@intel.com>
Date: Fri, 19 Jul 2024 13:13:42 -0700
Subject: [PATCH 0492/1488] update QPL to 1.6.0 + missing header fix

---
 contrib/qpl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contrib/qpl b/contrib/qpl
index d4715e0e798..c2ced94c53c 160000
--- a/contrib/qpl
+++ b/contrib/qpl
@@ -1 +1 @@
-Subproject commit d4715e0e79896b85612158e135ee1a85f3b3e04d
+Subproject commit c2ced94c53c1ee22191201a59878e9280bc9b9b8

From 1347bc32187b694148657459c0b745c45cb92a8d Mon Sep 17 00:00:00 2001
From: "Zhukova, Maria" <maria.zhukova@intel.com>
Date: Fri, 19 Jul 2024 13:32:18 -0700
Subject: [PATCH 0493/1488] update qpl-cmake to reflect changes in QPL
 1.5.0-1.6.0

---
 contrib/qpl-cmake/CMakeLists.txt | 92 ++++++++++++++++++++------------
 1 file changed, 57 insertions(+), 35 deletions(-)

diff --git a/contrib/qpl-cmake/CMakeLists.txt b/contrib/qpl-cmake/CMakeLists.txt
index 7a84048e16b..b2f263252c2 100644
--- a/contrib/qpl-cmake/CMakeLists.txt
+++ b/contrib/qpl-cmake/CMakeLists.txt
@@ -4,7 +4,6 @@ set (QPL_PROJECT_DIR "${ClickHouse_SOURCE_DIR}/contrib/qpl")
 set (QPL_SRC_DIR "${ClickHouse_SOURCE_DIR}/contrib/qpl/sources")
 set (QPL_BINARY_DIR "${ClickHouse_BINARY_DIR}/build/contrib/qpl")
 set (EFFICIENT_WAIT OFF)
-set (BLOCK_ON_FAULT ON)
 set (LOG_HW_INIT OFF)
 set (SANITIZE_MEMORY OFF)
 set (SANITIZE_THREADS OFF)
@@ -16,16 +15,18 @@ function(GetLibraryVersion _content _outputVar)
     SET(${_outputVar} ${CMAKE_MATCH_1} PARENT_SCOPE)
 endfunction()
 
-set (QPL_VERSION 1.2.0)
+set (QPL_VERSION 1.6.0)
 
 message(STATUS "Intel QPL version: ${QPL_VERSION}")
 
-# There are 5 source subdirectories under $QPL_SRC_DIR: isal, c_api, core-sw, middle-layer, c_api.
-# Generate 8 library targets: middle_layer_lib, isal, isal_asm, qplcore_px, qplcore_avx512, qplcore_sw_dispatcher, core_iaa, middle_layer_lib.
+# There are 5 source subdirectories under $QPL_SRC_DIR: c_api, core-iaa, core-sw, middle-layer and isal.
+# Generate 8 library targets: qpl_c_api, core_iaa, qplcore_px, qplcore_avx512, qplcore_sw_dispatcher, middle_layer_lib, isal and isal_asm,
+# which are then combined into static or shared qpl.
 # Output ch_contrib::qpl by linking with 8 library targets.
 
-# The qpl submodule comes with its own version of isal. It contains code which does not exist in upstream isal. It would be nice to link
-# only upstream isal (ch_contrib::isal) but at this point we can't.
+# Note, qpl submodule comes with its own version of isal that is not compatible with upstream isal (e.g., ch_contrib::isal).
+
+## cmake/CompileOptions.cmake and automatic wrappers generation
 
 # ==========================================================================
 # Copyright (C) 2022 Intel Corporation
@@ -442,6 +443,7 @@ function(generate_unpack_kernel_arrays current_directory PLATFORMS_LIST)
     endforeach()
 endfunction()
 
+# [SUBDIR]isal
 
 enable_language(ASM_NASM)
 
@@ -479,7 +481,6 @@ set(ISAL_ASM_SRC ${QPL_SRC_DIR}/isal/igzip/igzip_body.asm
                  ${QPL_SRC_DIR}/isal/igzip/igzip_set_long_icf_fg_04.asm
                  ${QPL_SRC_DIR}/isal/igzip/igzip_set_long_icf_fg_06.asm
                  ${QPL_SRC_DIR}/isal/igzip/igzip_multibinary.asm
-                 ${QPL_SRC_DIR}/isal/igzip/stdmac.asm
                  ${QPL_SRC_DIR}/isal/crc/crc_multibinary.asm
                  ${QPL_SRC_DIR}/isal/crc/crc32_gzip_refl_by8.asm
                  ${QPL_SRC_DIR}/isal/crc/crc32_gzip_refl_by8_02.asm
@@ -505,7 +506,6 @@ set_property(GLOBAL APPEND PROPERTY QPL_LIB_DEPS
 # Setting external and internal interfaces for ISA-L library
 target_include_directories(isal
                         PUBLIC $<BUILD_INTERFACE:${QPL_SRC_DIR}/isal/include>
-                        PRIVATE ${QPL_SRC_DIR}/isal/include
                         PUBLIC ${QPL_SRC_DIR}/isal/igzip)
 
 set_target_properties(isal PROPERTIES
@@ -617,12 +617,9 @@ target_compile_options(qplcore_sw_dispatcher
 
 # [SUBDIR]core-iaa
 file(GLOB HW_PATH_SRC ${QPL_SRC_DIR}/core-iaa/sources/aecs/*.c
-                      ${QPL_SRC_DIR}/core-iaa/sources/aecs/*.cpp
                       ${QPL_SRC_DIR}/core-iaa/sources/driver_loader/*.c
-                      ${QPL_SRC_DIR}/core-iaa/sources/driver_loader/*.cpp
                       ${QPL_SRC_DIR}/core-iaa/sources/descriptors/*.c
-                      ${QPL_SRC_DIR}/core-iaa/sources/descriptors/*.cpp
-                      ${QPL_SRC_DIR}/core-iaa/sources/bit_rev.c)
+                      ${QPL_SRC_DIR}/core-iaa/sources/*.c)
 
 # Create library
 add_library(core_iaa OBJECT ${HW_PATH_SRC})
@@ -634,31 +631,27 @@ target_include_directories(core_iaa
         PRIVATE ${UUID_DIR}
         PUBLIC $<BUILD_INTERFACE:${QPL_SRC_DIR}/core-iaa/include>
         PUBLIC $<BUILD_INTERFACE:${QPL_SRC_DIR}/core-iaa/sources/include>
-        PRIVATE $<BUILD_INTERFACE:${QPL_PROJECT_DIR}/include>  # status.h in own_checkers.h
-        PRIVATE $<BUILD_INTERFACE:${QPL_PROJECT_DIR}/sources/c_api> # own_checkers.h
+        PRIVATE $<BUILD_INTERFACE:${QPL_PROJECT_DIR}/include> # status.h in own_checkers.h
+        PRIVATE $<TARGET_PROPERTY:qpl_c_api,INTERFACE_INCLUDE_DIRECTORIES> # for own_checkers.h
         PRIVATE $<TARGET_PROPERTY:qplcore_sw_dispatcher,INTERFACE_INCLUDE_DIRECTORIES>)
 
 target_compile_features(core_iaa PRIVATE c_std_11)
 
 target_compile_definitions(core_iaa PRIVATE QPL_BADARG_CHECK
-        PRIVATE $<$<BOOL:${BLOCK_ON_FAULT}>: BLOCK_ON_FAULT_ENABLED>
         PRIVATE $<$<BOOL:${LOG_HW_INIT}>:LOG_HW_INIT>
         PRIVATE $<$<BOOL:${DYNAMIC_LOADING_LIBACCEL_CONFIG}>:DYNAMIC_LOADING_LIBACCEL_CONFIG>)
 
 # [SUBDIR]middle-layer
 file(GLOB MIDDLE_LAYER_SRC
-        ${QPL_SRC_DIR}/middle-layer/analytics/*.cpp
-        ${QPL_SRC_DIR}/middle-layer/c_wrapper/*.cpp
-        ${QPL_SRC_DIR}/middle-layer/checksum/*.cpp
+        ${QPL_SRC_DIR}/middle-layer/accelerator/*.cpp
+	    ${QPL_SRC_DIR}/middle-layer/analytics/*.cpp
         ${QPL_SRC_DIR}/middle-layer/common/*.cpp
         ${QPL_SRC_DIR}/middle-layer/compression/*.cpp
         ${QPL_SRC_DIR}/middle-layer/compression/*/*.cpp
         ${QPL_SRC_DIR}/middle-layer/compression/*/*/*.cpp
         ${QPL_SRC_DIR}/middle-layer/dispatcher/*.cpp
         ${QPL_SRC_DIR}/middle-layer/other/*.cpp
-        ${QPL_SRC_DIR}/middle-layer/util/*.cpp
-        ${QPL_SRC_DIR}/middle-layer/inflate/*.cpp
-        ${QPL_SRC_DIR}/core-iaa/sources/accelerator/*.cpp) # todo
+        ${QPL_SRC_DIR}/middle-layer/util/*.cpp)
 
 add_library(middle_layer_lib OBJECT
         ${MIDDLE_LAYER_SRC})
@@ -667,6 +660,7 @@ set_property(GLOBAL APPEND PROPERTY QPL_LIB_DEPS
         $<TARGET_OBJECTS:middle_layer_lib>)
 
 target_compile_options(middle_layer_lib
+        PRIVATE $<$<C_COMPILER_ID:GNU,Clang>:$<$<CONFIG:Release>:-O3;-U_FORTIFY_SOURCE;-D_FORTIFY_SOURCE=2>>
         PRIVATE ${QPL_LINUX_TOOLCHAIN_CPP_EMBEDDED_FLAGS})
 
 target_compile_definitions(middle_layer_lib
@@ -682,6 +676,7 @@ target_include_directories(middle_layer_lib
         PRIVATE ${UUID_DIR}
         PUBLIC $<BUILD_INTERFACE:${QPL_SRC_DIR}/middle-layer>
         PUBLIC $<TARGET_PROPERTY:_qpl,INTERFACE_INCLUDE_DIRECTORIES>
+        PRIVATE $<TARGET_PROPERTY:qpl_c_api,INTERFACE_INCLUDE_DIRECTORIES>
         PUBLIC $<TARGET_PROPERTY:qplcore_sw_dispatcher,INTERFACE_INCLUDE_DIRECTORIES>
         PUBLIC $<TARGET_PROPERTY:isal,INTERFACE_INCLUDE_DIRECTORIES>
         PUBLIC $<TARGET_PROPERTY:core_iaa,INTERFACE_INCLUDE_DIRECTORIES>)
@@ -689,31 +684,58 @@ target_include_directories(middle_layer_lib
 target_compile_definitions(middle_layer_lib PUBLIC -DQPL_LIB)
 
 # [SUBDIR]c_api
-file(GLOB_RECURSE QPL_C_API_SRC
-        ${QPL_SRC_DIR}/c_api/*.c
-        ${QPL_SRC_DIR}/c_api/*.cpp)
+file(GLOB QPL_C_API_SRC
+        ${QPL_SRC_DIR}/c_api/compression_operations/*.c
+        ${QPL_SRC_DIR}/c_api/compression_operations/*.cpp
+	    ${QPL_SRC_DIR}/c_api/filter_operations/*.cpp
+	    ${QPL_SRC_DIR}/c_api/legacy_hw_path/*.c
+	    ${QPL_SRC_DIR}/c_api/legacy_hw_path/*.cpp
+	    ${QPL_SRC_DIR}/c_api/other_operations/*.cpp
+	    ${QPL_SRC_DIR}/c_api/serialization/*.cpp
+	    ${QPL_SRC_DIR}/c_api/*.cpp)
+
+add_library(qpl_c_api OBJECT ${QPL_C_API_SRC})
+
+target_include_directories(qpl_c_api
+	    PUBLIC $<BUILD_INTERFACE:${QPL_SRC_DIR}/c_api/>
+        PUBLIC $<BUILD_INTERFACE:${QPL_SRC_DIR}/include/> $<INSTALL_INTERFACE:include>
+        PRIVATE $<TARGET_PROPERTY:middle_layer_lib,INTERFACE_INCLUDE_DIRECTORIES>)
+
+set_target_properties(qpl_c_api PROPERTIES
+	$<$<C_COMPILER_ID:GNU,Clang>:C_STANDARD 17
+	CXX_STANDARD 17)
+
+target_compile_options(qpl_c_api
+        PRIVATE $<$<C_COMPILER_ID:GNU,Clang>:$<$<CONFIG:Release>:-O3;-U_FORTIFY_SOURCE;-D_FORTIFY_SOURCE=2>>
+	    PRIVATE $<$<COMPILE_LANG_AND_ID:CXX,GNU,Clang>:${QPL_LINUX_TOOLCHAIN_CPP_EMBEDDED_FLAGS}>)
+
+target_compile_definitions(qpl_c_api
+        PUBLIC -DQPL_BADARG_CHECK # own_checkers.h
+        PUBLIC -DQPL_LIB          # needed for middle_layer_lib
+        PUBLIC $<$<BOOL:${LOG_HW_INIT}>:LOG_HW_INIT>) # needed for middle_layer_lib
+
+set_property(GLOBAL APPEND PROPERTY QPL_LIB_DEPS
+        $<TARGET_OBJECTS:qpl_c_api>)
+
+# Final _qpl target
 
 get_property(LIB_DEPS GLOBAL PROPERTY QPL_LIB_DEPS)
 
-add_library(_qpl STATIC ${QPL_C_API_SRC} ${LIB_DEPS})
+add_library(_qpl STATIC ${LIB_DEPS})
 
 target_include_directories(_qpl
-        PUBLIC $<BUILD_INTERFACE:${QPL_PROJECT_DIR}/include/> $<INSTALL_INTERFACE:include>
-        PRIVATE $<TARGET_PROPERTY:middle_layer_lib,INTERFACE_INCLUDE_DIRECTORIES>
-        PRIVATE $<BUILD_INTERFACE:${QPL_SRC_DIR}/c_api>)
+        PUBLIC $<BUILD_INTERFACE:${QPL_PROJECT_DIR}/include/> $<INSTALL_INTERFACE:include>)
 
-target_compile_options(_qpl
-        PRIVATE ${QPL_LINUX_TOOLCHAIN_CPP_EMBEDDED_FLAGS})
 
 target_compile_definitions(_qpl
-        PRIVATE -DQPL_LIB
-        PRIVATE -DQPL_BADARG_CHECK
-        PRIVATE $<$<BOOL:${DYNAMIC_LOADING_LIBACCEL_CONFIG}>:DYNAMIC_LOADING_LIBACCEL_CONFIG>
         PUBLIC -DENABLE_QPL_COMPRESSION)
 
 target_link_libraries(_qpl
-        PRIVATE ch_contrib::accel-config
-        PRIVATE ch_contrib::isal)
+        PRIVATE ch_contrib::accel-config)
+
+# C++ filesystem library requires additional linking for older GNU/Clang
+target_link_libraries(_qpl PRIVATE $<$<AND:$<CXX_COMPILER_ID:GNU>,$<VERSION_LESS:$<CXX_COMPILER_VERSION>,9.1>>:stdc++fs>)
+target_link_libraries(_qpl PRIVATE $<$<AND:$<CXX_COMPILER_ID:Clang>,$<VERSION_LESS:$<CXX_COMPILER_VERSION>,9.0>>:c++fs>)
 
 target_include_directories(_qpl SYSTEM BEFORE
         PUBLIC "${QPL_PROJECT_DIR}/include"

From a373b62bbf8083ffa96210fc1c959f13939526fc Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 20 Jul 2024 00:04:52 +0200
Subject: [PATCH 0494/1488] Better diagnostics in functional tests

---
 docker/test/stateless/run.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh
index 0647ed02839..f43bb5da33d 100755
--- a/docker/test/stateless/run.sh
+++ b/docker/test/stateless/run.sh
@@ -264,7 +264,7 @@ if [ "$NUM_TRIES" -gt "1" ]; then
     # We don't run tests with Ordinary database in PRs, only in master.
     # So run new/changed tests with Ordinary at least once in flaky check.
     timeout_with_logging "$TIMEOUT" bash -c 'NUM_TRIES=1; USE_DATABASE_ORDINARY=1; run_tests' \
-      | sed 's/All tests have finished//' | sed 's/No tests were run//' ||:
+      | sed 's/All tests have finished/Redacted: a message about tests finish is deleted/' | sed 's/No tests were run/Redacted: a message about no tests run is deleted/' ||:
 fi
 
 timeout_with_logging "$TIMEOUT" bash -c run_tests ||:

From 134c0065407bd3f9394a720fbdfef7edf241ef84 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 20 Jul 2024 00:08:25 +0200
Subject: [PATCH 0495/1488] Whitespace

---
 docker/test/stateless/utils.lib | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/test/stateless/utils.lib b/docker/test/stateless/utils.lib
index c3bb8ae9ea4..6b7b659296b 100644
--- a/docker/test/stateless/utils.lib
+++ b/docker/test/stateless/utils.lib
@@ -47,7 +47,7 @@ function timeout_with_logging() {
 
     if [[ "${exit_code}" -eq "124" ]]
     then
-      echo "The command 'timeout ${*}' has been killed by timeout"
+        echo "The command 'timeout ${*}' has been killed by timeout"
     fi
 
     return $exit_code

From ba6b7b86ba3e868cd001efbce2c6cf8a5236a024 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 20 Jul 2024 00:09:53 +0200
Subject: [PATCH 0496/1488] Log messages

---
 docker/test/stateless/utils.lib | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/docker/test/stateless/utils.lib b/docker/test/stateless/utils.lib
index 6b7b659296b..cb257536c36 100644
--- a/docker/test/stateless/utils.lib
+++ b/docker/test/stateless/utils.lib
@@ -45,9 +45,12 @@ function timeout_with_logging() {
 
     timeout -s TERM --preserve-status "${@}" || exit_code="${?}"
 
+    echo "Checking if it is a timeout. The code 124 will indicate a timeout."
     if [[ "${exit_code}" -eq "124" ]]
     then
-        echo "The command 'timeout ${*}' has been killed by timeout"
+        echo "The command 'timeout ${*}' has been killed by timeout."
+    else
+        echo "No, it isn't a timeout."
     fi
 
     return $exit_code

From 5ae3a421e0bf90c3d1755371fe1d6ff5662207ca Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 20 Jul 2024 00:12:02 +0200
Subject: [PATCH 0497/1488] Copy-paste

---
 docker/test/fasttest/run.sh | 5 ++++-
 docker/test/stateful/run.sh | 5 ++++-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh
index 970bf12a81a..26283afc86a 100755
--- a/docker/test/fasttest/run.sh
+++ b/docker/test/fasttest/run.sh
@@ -261,9 +261,12 @@ function timeout_with_logging() {
 
     timeout -s TERM --preserve-status "${@}" || exit_code="${?}"
 
+    echo "Checking if it is a timeout. The code 124 will indicate a timeout."
     if [[ "${exit_code}" -eq "124" ]]
     then
-      echo "The command 'timeout ${*}' has been killed by timeout"
+        echo "The command 'timeout ${*}' has been killed by timeout."
+    else
+        echo "No, it isn't a timeout."
     fi
 
     return $exit_code
diff --git a/docker/test/stateful/run.sh b/docker/test/stateful/run.sh
index 857385f4715..3a0e3a8be48 100755
--- a/docker/test/stateful/run.sh
+++ b/docker/test/stateful/run.sh
@@ -251,9 +251,12 @@ function timeout_with_logging() {
 
     timeout -s TERM --preserve-status "${@}" || exit_code="${?}"
 
+    echo "Checking if it is a timeout. The code 124 will indicate a timeout."
     if [[ "${exit_code}" -eq "124" ]]
     then
-      echo "The command 'timeout ${*}' has been killed by timeout"
+        echo "The command 'timeout ${*}' has been killed by timeout."
+    else
+        echo "No, it isn't a timeout."
     fi
 
     return $exit_code

From 444303cb7117c92e578ad4ea20f7c0001edb3c8b Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 20 Jul 2024 00:25:57 +0200
Subject: [PATCH 0498/1488] Better diagnostics

---
 docker/test/stateless/run.sh | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh
index f43bb5da33d..b24af431ff1 100755
--- a/docker/test/stateless/run.sh
+++ b/docker/test/stateless/run.sh
@@ -247,12 +247,22 @@ function run_tests()
 
     try_run_with_retry 10 clickhouse-client -q "insert into system.zookeeper (name, path, value) values ('auxiliary_zookeeper2', '/test/chroot/', '')"
 
+    TIMEOUT=$((MAX_RUN_TIME - 800 > 8400 ? 8400 : MAX_RUN_TIME - 800))
+    START_TIME=${SECONDS}
     set +e
-    timeout -k 60m -s TERM --preserve-status 140m  clickhouse-test --testname --shard --zookeeper --check-zookeeper-session --hung-check --print-time \
-         --no-drop-if-fail --test-runs "$NUM_TRIES" "${ADDITIONAL_OPTIONS[@]}" 2>&1 \
+    timeout --preserve-status --signal TERM --kill-after 60m ${TIMEOUT}s \
+        clickhouse-test --testname --shard --zookeeper --check-zookeeper-session --hung-check --print-time \
+            --no-drop-if-fail --test-runs "$NUM_TRIES" "${ADDITIONAL_OPTIONS[@]}" 2>&1 \
     | ts '%Y-%m-%d %H:%M:%S' \
     | tee -a test_output/test_result.txt
     set -e
+    DURATION=$((START_TIME - SECONDS))
+
+    echo "Elapsed ${DURATION} seconds."
+    if [[ $DURATION -ge $TIMEOUT ]]
+    then
+        echo "It looks like the command is terminated by the timeout, which is ${TIMEOUT} seconds."
+    fi
 }
 
 export -f run_tests

From 42ba3229b5484ebe29f4191cd5672433ba1f6588 Mon Sep 17 00:00:00 2001
From: Rich Raposa <richraposa@gmail.com>
Date: Fri, 19 Jul 2024 16:33:15 -0600
Subject: [PATCH 0499/1488] Update azureBlobStorageCluster.md
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This has been driving me crazy for a while 😄 The table functions are listed alphabetically except for this one - so it's a trivial fix.
---
 .../en/sql-reference/table-functions/azureBlobStorageCluster.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/en/sql-reference/table-functions/azureBlobStorageCluster.md b/docs/en/sql-reference/table-functions/azureBlobStorageCluster.md
index 20dfd35d5db..ca3959d6ed1 100644
--- a/docs/en/sql-reference/table-functions/azureBlobStorageCluster.md
+++ b/docs/en/sql-reference/table-functions/azureBlobStorageCluster.md
@@ -1,6 +1,6 @@
 ---
 slug: /en/sql-reference/table-functions/azureBlobStorageCluster
-sidebar_position: 55
+sidebar_position: 15
 sidebar_label: azureBlobStorageCluster
 title: "azureBlobStorageCluster Table Function"
 ---

From 8a67713e63bbaec8bf820bd6813affefe305d2cc Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 20 Jul 2024 01:09:45 +0200
Subject: [PATCH 0500/1488] Fix error

---
 tests/queries/0_stateless/01338_long_select_and_alter.sh      | 4 ++--
 .../0_stateless/01338_long_select_and_alter_zookeeper.sh      | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/queries/0_stateless/01338_long_select_and_alter.sh b/tests/queries/0_stateless/01338_long_select_and_alter.sh
index 08609546ff5..5d2759ac884 100755
--- a/tests/queries/0_stateless/01338_long_select_and_alter.sh
+++ b/tests/queries/0_stateless/01338_long_select_and_alter.sh
@@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
 $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS alter_mt"
 
-$CLICKHOUSE_CLIENT --query "CREATE TABLE alter_mt (key UInt64, value String) ENGINE=MergeTree() ORDER BY key"
+$CLICKHOUSE_CLIENT --query "CREATE TABLE alter_mt (key Int64, value String) ENGINE=MergeTree() ORDER BY key"
 
 $CLICKHOUSE_CLIENT --query "INSERT INTO alter_mt SELECT number - 1 AS x, toString(x) FROM numbers(5)"
 
@@ -16,7 +16,7 @@ $CLICKHOUSE_CLIENT --function_sleep_max_microseconds_per_block 10000000 --query
 # To be sure that select took all required locks for better test sensitivity, although it isn't guaranteed (then the test will also succeed).
 sleep 2
 
-$CLICKHOUSE_CLIENT --query "ALTER TABLE alter_mt MODIFY COLUMN value UInt64"
+$CLICKHOUSE_CLIENT --query "ALTER TABLE alter_mt MODIFY COLUMN value Int64"
 
 $CLICKHOUSE_CLIENT --query "SELECT sum(value) FROM alter_mt"
 
diff --git a/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.sh b/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.sh
index def6d2ab127..593a96a7cc8 100755
--- a/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.sh
+++ b/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.sh
@@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
 $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS alter_mt"
 
-$CLICKHOUSE_CLIENT --query "CREATE TABLE alter_mt (key UInt64, value String) ENGINE=ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/alter_mt', '1') ORDER BY key"
+$CLICKHOUSE_CLIENT --query "CREATE TABLE alter_mt (key Int64, value String) ENGINE=ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/alter_mt', '1') ORDER BY key"
 
 $CLICKHOUSE_CLIENT --query "INSERT INTO alter_mt SELECT number - 1 AS x, toString(x) FROM numbers(5)"
 
@@ -16,7 +16,7 @@ $CLICKHOUSE_CLIENT --function_sleep_max_microseconds_per_block 10000000 --query
 # To be sure that select took all required locks for better test sensitivity, although it isn't guaranteed (then the test will also succeed).
 sleep 2
 
-$CLICKHOUSE_CLIENT --query "ALTER TABLE alter_mt MODIFY COLUMN value UInt64"
+$CLICKHOUSE_CLIENT --query "ALTER TABLE alter_mt MODIFY COLUMN value Int64"
 
 $CLICKHOUSE_CLIENT --query "SELECT sum(value) FROM alter_mt"
 

From 7caa7e20601b2500f513476dfe10819f328be3d3 Mon Sep 17 00:00:00 2001
From: jsc0218 <jsc0218@gmail.com>
Date: Sat, 20 Jul 2024 02:28:13 +0000
Subject: [PATCH 0501/1488] block deduplicate only in throw mode

---
 src/Interpreters/InterpreterCreateQuery.cpp   |  2 +-
 src/Interpreters/InterpreterOptimizeQuery.cpp |  4 ----
 src/Storages/StorageMergeTree.cpp             |  6 ++++--
 src/Storages/StorageReplicatedMergeTree.cpp   |  6 ++++--
 ...206_projection_merge_special_mergetree.sql | 19 +++++++++----------
 5 files changed, 18 insertions(+), 19 deletions(-)

diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp
index df5ec4525eb..05df26b0d31 100644
--- a/src/Interpreters/InterpreterCreateQuery.cpp
+++ b/src/Interpreters/InterpreterCreateQuery.cpp
@@ -1298,7 +1298,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create)
             }
         }
         if (!projection_support)
-            throw Exception(ErrorCodes::SUPPORT_IS_DISABLED,
+            throw Exception(ErrorCodes::NOT_IMPLEMENTED,
                 "Projection is only supported in (Replictaed)MergeTree. Consider drop or rebuild option of deduplicate_merge_projection_mode.");
     }
 
diff --git a/src/Interpreters/InterpreterOptimizeQuery.cpp b/src/Interpreters/InterpreterOptimizeQuery.cpp
index 8d1ac3455b7..3bee235185d 100644
--- a/src/Interpreters/InterpreterOptimizeQuery.cpp
+++ b/src/Interpreters/InterpreterOptimizeQuery.cpp
@@ -43,10 +43,6 @@ BlockIO InterpreterOptimizeQuery::execute()
     auto metadata_snapshot = table->getInMemoryMetadataPtr();
     auto storage_snapshot = table->getStorageSnapshot(metadata_snapshot, getContext());
 
-    /// Don't allow OPTIMIZE DEDUPLICATE for all engines with projections.
-    if (ast.deduplicate && !metadata_snapshot->projections.empty())
-        throw Exception(ErrorCodes::NOT_IMPLEMENTED, "DEDUPLICATE with projections are not supported yet");
-
     // Empty list of names means we deduplicate by all columns, but user can explicitly state which columns to use.
     Names column_names;
     if (ast.deduplicate_by_columns)
diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp
index 40b3a12297b..a5d434796ba 100644
--- a/src/Storages/StorageMergeTree.cpp
+++ b/src/Storages/StorageMergeTree.cpp
@@ -1580,10 +1580,12 @@ bool StorageMergeTree::optimize(
 {
     assertNotReadonly();
 
-    if (deduplicate && getInMemoryMetadataPtr()->hasProjections())
+    if (deduplicate && getInMemoryMetadataPtr()->hasProjections()
+        && getSettings()->deduplicate_merge_projection_mode == DeduplicateMergeProjectionMode::THROW)
         throw Exception(ErrorCodes::NOT_IMPLEMENTED,
                     "OPTIMIZE DEDUPLICATE query is not supported for table {} as it has projections. "
-                    "User should drop all the projections manually before running the query",
+                    "User should drop all the projections manually before running the query, "
+                    "or consider drop or rebuild option of deduplicate_merge_projection_mode",
                     getStorageID().getTableName());
 
     if (deduplicate)
diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp
index 72f725965e0..3751883df24 100644
--- a/src/Storages/StorageReplicatedMergeTree.cpp
+++ b/src/Storages/StorageReplicatedMergeTree.cpp
@@ -5794,10 +5794,12 @@ bool StorageReplicatedMergeTree::optimize(
     if (!is_leader)
         throw Exception(ErrorCodes::NOT_A_LEADER, "OPTIMIZE cannot be done on this replica because it is not a leader");
 
-    if (deduplicate && getInMemoryMetadataPtr()->hasProjections())
+    if (deduplicate && getInMemoryMetadataPtr()->hasProjections()
+        && getSettings()->deduplicate_merge_projection_mode == DeduplicateMergeProjectionMode::THROW)
         throw Exception(ErrorCodes::NOT_IMPLEMENTED,
                     "OPTIMIZE DEDUPLICATE query is not supported for table {} as it has projections. "
-                    "User should drop all the projections manually before running the query",
+                    "User should drop all the projections manually before running the query, "
+                    "or consider drop or rebuild option of deduplicate_merge_projection_mode",
                     getStorageID().getTableName());
 
     if (cleanup)
diff --git a/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql b/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql
index 06fb9a30aca..c8945fd784c 100644
--- a/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql
+++ b/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql
@@ -4,14 +4,14 @@ CREATE TABLE tp (
     type Int32,
     eventcnt UInt64,
     PROJECTION p (select sum(eventcnt), type group by type)
-) engine = ReplacingMergeTree order by type;  -- { serverError SUPPORT_IS_DISABLED }
+) engine = ReplacingMergeTree order by type;  -- { serverError NOT_IMPLEMENTED }
 
 CREATE TABLE tp (
     type Int32,
     eventcnt UInt64,
     PROJECTION p (select sum(eventcnt), type group by type)
 ) engine = ReplacingMergeTree order by type
-SETTINGS deduplicate_merge_projection_mode = 'throw';  -- { serverError SUPPORT_IS_DISABLED }
+SETTINGS deduplicate_merge_projection_mode = 'throw';  -- { serverError NOT_IMPLEMENTED }
 
 CREATE TABLE tp (
     type Int32,
@@ -20,6 +20,10 @@ CREATE TABLE tp (
 ) engine = ReplacingMergeTree order by type
 SETTINGS deduplicate_merge_projection_mode = 'drop';
 
+ALTER TABLE tp MODIFY SETTING deduplicate_merge_projection_mode = 'throw';
+
+OPTIMIZE TABLE tp DEDUPLICATE;  -- { serverError NOT_IMPLEMENTED }
+
 DROP TABLE tp;
 
 CREATE TABLE tp (
@@ -29,13 +33,8 @@ CREATE TABLE tp (
 ) engine = ReplacingMergeTree order by type
 SETTINGS deduplicate_merge_projection_mode = 'rebuild';
 
-DROP TABLE tp;
+ALTER TABLE tp MODIFY SETTING deduplicate_merge_projection_mode = 'throw';
 
+OPTIMIZE TABLE tp DEDUPLICATE;  -- { serverError NOT_IMPLEMENTED }
 
--- don't allow OPTIMIZE DEDUPLICATE for all engines with projections
-CREATE TABLE test (
-    a INT PRIMARY KEY,
-    PROJECTION p (SELECT * ORDER BY a)
-) engine = MergeTree;
-
-OPTIMIZE TABLE test DEDUPLICATE;  -- { serverError NOT_IMPLEMENTED }
+DROP TABLE tp;
\ No newline at end of file

From 7c89ca59af17576e3a4e6b42167c84b5045fb969 Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Sat, 20 Jul 2024 11:46:24 +0200
Subject: [PATCH 0502/1488] Bump


From e3b2fbf7ec1a322fa4fcbd808a3a19dd6ec3b8ee Mon Sep 17 00:00:00 2001
From: Max K <maxkaynov@gmail.com>
Date: Tue, 16 Jul 2024 15:37:50 +0200
Subject: [PATCH 0503/1488] CI: POC for Auto Releases

---
 .github/workflows/auto_release.yml   |  42 ++++---
 .github/workflows/create_release.yml |   9 +-
 tests/ci/auto_release.py             | 169 +++++++++++++--------------
 tests/ci/ci_utils.py                 |   6 +-
 4 files changed, 121 insertions(+), 105 deletions(-)

diff --git a/.github/workflows/auto_release.yml b/.github/workflows/auto_release.yml
index f1a6b307b40..e90b183b8d6 100644
--- a/.github/workflows/auto_release.yml
+++ b/.github/workflows/auto_release.yml
@@ -1,44 +1,58 @@
 name: AutoRelease
 
 env:
-  # Force the stdout and stderr streams to be unbuffered
   PYTHONUNBUFFERED: 1
 
 concurrency:
-  group: auto-release
+  group: release
 on: # yamllint disable-line rule:truthy
   # schedule:
   #   - cron: '0 10-16 * * 1-5'
   workflow_dispatch:
 
 jobs:
-  CherryPick:
-    runs-on: [self-hosted, style-checker-aarch64]
+  AutoRelease:
+    runs-on: [self-hosted, release-maker]
     steps:
+      - name: DebugInfo
+        uses: hmarr/debug-action@f7318c783045ac39ed9bb497e22ce835fdafbfe6
       - name: Set envs
-        # https://docs.github.com/en/actions/learn-github-actions/workflow-commands-for-github-actions#multiline-strings
         run: |
           cat >> "$GITHUB_ENV" << 'EOF'
-          TEMP_PATH=${{runner.temp}}/cherry_pick
           ROBOT_CLICKHOUSE_SSH_KEY<<RCSK
           ${{secrets.ROBOT_CLICKHOUSE_SSH_KEY}}
           RCSK
-          REPO_OWNER=ClickHouse
-          REPO_NAME=ClickHouse
-          REPO_TEAM=core
           EOF
       - name: Check out repository code
         uses: ClickHouse/checkout@v1
         with:
-          clear-repository: true
           token: ${{secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN}}
           fetch-depth: 0
-      - name: Auto-release
+      - name: Auto Release Prepare
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
-          python3 auto_release.py --release-after-days=3
-      - name: Cleanup
-        if: always()
+          python3 auto_release.py
+          echo "::group::Auto Release Info"
+          python3 -m json.tool /tmp/autorelease_info.json
+          echo "::endgroup::"
+          {
+              echo 'AUTO_RELEASE_PARAMS<<EOF'
+              cat  /tmp/autorelease_info.json
+              echo 'EOF'
+          } >> "$GITHUB_ENV"
+      - name: Release ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[0].release_branch }}
+        if: ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[0] }}
+        uses: ./.github/workflows/create_release.yml
+        with:
+          type: patch
+          ref: ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[0].commit_sha }}
+          dry-run: true
+          autorelease: true
+      - name: Post Slack Message
+        if: ${{ !cancelled() }}
+        run: |
+          echo Slack Message
+      - name: Clean up
         run: |
           docker ps --quiet | xargs --no-run-if-empty docker kill ||:
           docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml
index d8d27531f28..96cd46f583a 100644
--- a/.github/workflows/create_release.yml
+++ b/.github/workflows/create_release.yml
@@ -22,6 +22,10 @@ concurrency:
         required: false
         default: true
         type: boolean
+      autorelease:
+        required: false
+        default: false
+        type: boolean
 
 jobs:
   CreateRelease:
@@ -30,8 +34,10 @@ jobs:
     runs-on: [self-hosted, release-maker]
     steps:
       - name: DebugInfo
+        if: ${{ ! inputs.autorelease }}
         uses: hmarr/debug-action@f7318c783045ac39ed9bb497e22ce835fdafbfe6
       - name: Set envs
+        if: ${{ ! inputs.autorelease }}
         # https://docs.github.com/en/actions/learn-github-actions/workflow-commands-for-github-actions#multiline-strings
         run: |
           cat >> "$GITHUB_ENV" << 'EOF'
@@ -41,6 +47,7 @@ jobs:
           RELEASE_INFO_FILE=${{ runner.temp }}/release_info.json
           EOF
       - name: Check out repository code
+        if: ${{ ! inputs.autorelease }}
         uses: ClickHouse/checkout@v1
         with:
           token: ${{secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN}}
@@ -161,6 +168,6 @@ jobs:
           export CHECK_NAME="Docker keeper image"
           python3 docker_server.py --release-type auto --version ${{ env.RELEASE_TAG }} --check-name "$CHECK_NAME" --sha ${{ env.COMMIT_SHA }} ${{ ! inputs.dry-run && '--push'  || '' }}
       - name: Post Slack Message
-        if: always()
+        if: ${{ !cancelled() }}
         run: |
           echo Slack Message
diff --git a/tests/ci/auto_release.py b/tests/ci/auto_release.py
index bfd19b11e6d..88e91eb8aed 100644
--- a/tests/ci/auto_release.py
+++ b/tests/ci/auto_release.py
@@ -1,13 +1,16 @@
 import argparse
-from datetime import timedelta, datetime
+import dataclasses
+import json
 import logging
 import os
-from commit_status_helper import get_commit_filtered_statuses
+from typing import List
+
 from get_robot_token import get_best_robot_token
 from github_helper import GitHub
-from release import Release, Repo as ReleaseRepo, RELEASE_READY_STATUS
-from report import SUCCESS
 from ssh import SSHKey
+from ci_utils import Shell
+from env_helper import GITHUB_REPOSITORY
+from report import SUCCESS
 
 LOGGER_NAME = __name__
 HELPER_LOGGERS = ["github_helper", LOGGER_NAME]
@@ -20,116 +23,104 @@ def parse_args():
         "branches and do a release in case for green builds."
     )
     parser.add_argument("--token", help="GitHub token, if not set, used from smm")
-    parser.add_argument(
-        "--repo", default="ClickHouse/ClickHouse", help="Repo owner/name"
-    )
-    parser.add_argument("--dry-run", action="store_true", help="Do not create anything")
-    parser.add_argument(
-        "--release-after-days",
-        type=int,
-        default=3,
-        help="Do automatic release on the latest green commit after the latest "
-        "release if the newest release is older than the specified days",
-    )
-    parser.add_argument(
-        "--debug-helpers",
-        action="store_true",
-        help="Add debug logging for this script and github_helper",
-    )
-    parser.add_argument(
-        "--remote-protocol",
-        "-p",
-        default="ssh",
-        choices=ReleaseRepo.VALID,
-        help="repo protocol for git commands remote, 'origin' is a special case and "
-        "uses 'origin' as a remote",
-    )
 
     return parser.parse_args()
 
 
+MAX_NUMBER_OF_COMMITS_TO_CONSIDER_FOR_RELEASE = 5
+AUTORELEASE_INFO_FILE = "/tmp/autorelease_info.json"
+
+
+@dataclasses.dataclass
+class ReleaseParams:
+    release_branch: str
+    commit_sha: str
+
+
+@dataclasses.dataclass
+class AutoReleaseInfo:
+    releases: List[ReleaseParams]
+
+    def add_release(self, release_params: ReleaseParams):
+        self.releases.append(release_params)
+
+    def dump(self):
+        print(f"Dump release info into [{AUTORELEASE_INFO_FILE}]")
+        with open(AUTORELEASE_INFO_FILE, "w", encoding="utf-8") as f:
+            print(json.dumps(dataclasses.asdict(self), indent=2), file=f)
+
+
 def main():
     args = parse_args()
-    logging.basicConfig(level=logging.INFO)
-    if args.debug_helpers:
-        for logger_name in HELPER_LOGGERS:
-            logging.getLogger(logger_name).setLevel(logging.DEBUG)
 
     token = args.token or get_best_robot_token()
-    days_as_timedelta = timedelta(days=args.release_after_days)
-    now = datetime.now()
-
+    assert len(token) > 10
+    os.environ["GH_TOKEN"] = token
+    (Shell.run("gh auth status", check=True))
     gh = GitHub(token)
-    prs = gh.get_release_pulls(args.repo)
+    prs = gh.get_release_pulls(GITHUB_REPOSITORY)
     branch_names = [pr.head.ref for pr in prs]
 
-    logger.info("Found release branches: %s\n ", " \n".join(branch_names))
-    repo = gh.get_repo(args.repo)
+    print(f"Found release branches [{branch_names}]")
+    repo = gh.get_repo(GITHUB_REPOSITORY)
 
-    # In general there is no guarantee on which order the refs/commits are
-    # returned from the API, so we have to order them.
+    autoRelease_info = AutoReleaseInfo(releases=[])
     for pr in prs:
-        logger.info("Checking PR %s", pr.head.ref)
+        print(f"Checking PR [{pr.head.ref}]")
 
         refs = list(repo.get_git_matching_refs(f"tags/v{pr.head.ref}"))
         refs.sort(key=lambda ref: ref.ref)
 
         latest_release_tag_ref = refs[-1]
         latest_release_tag = repo.get_git_tag(latest_release_tag_ref.object.sha)
-        logger.info("That last release was done at %s", latest_release_tag.tagger.date)
-
-        if latest_release_tag.tagger.date + days_as_timedelta > now:
-            logger.info(
-                "Not enough days since the last release %s,"
-                " no automatic release can be done",
-                latest_release_tag.tag,
+        commit_num = int(
+            Shell.run(
+                f"git rev-list --count {latest_release_tag.tag}..origin/{pr.head.ref}",
+                check=True,
             )
-            continue
-
-        unreleased_commits = list(
-            repo.get_commits(sha=pr.head.ref, since=latest_release_tag.tagger.date)
         )
-        unreleased_commits.sort(
-            key=lambda commit: commit.commit.committer.date, reverse=True
+        print(
+            f"Previous release is [{latest_release_tag}] was [{commit_num}] commits before, date [{latest_release_tag.tagger.date}]"
         )
-
-        for commit in unreleased_commits:
-            logger.info("Checking statuses of commit %s", commit.sha)
-            statuses = get_commit_filtered_statuses(commit)
-            all_success = all(st.state == SUCCESS for st in statuses)
-            passed_ready_for_release_check = any(
-                st.context == RELEASE_READY_STATUS and st.state == SUCCESS
-                for st in statuses
+        commit_reverse_index = 0
+        commit_found = False
+        commit_checked = False
+        commit_sha = ""
+        while (
+            commit_reverse_index < commit_num - 1
+            and commit_reverse_index < MAX_NUMBER_OF_COMMITS_TO_CONSIDER_FOR_RELEASE
+        ):
+            commit_checked = True
+            commit_sha = Shell.run(
+                f"git rev-list --max-count=1 --skip={commit_reverse_index} origin/{pr.head.ref}",
+                check=True,
             )
-            if not (all_success and passed_ready_for_release_check):
-                logger.info("Commit is not green, thus not suitable for release")
-                continue
-
-            logger.info("Commit is ready for release, let's release!")
-
-            release = Release(
-                ReleaseRepo(args.repo, args.remote_protocol),
-                commit.sha,
-                "patch",
-                args.dry_run,
-                True,
+            print(
+                f"Check if commit [{commit_sha}] [{pr.head.ref}~{commit_reverse_index}] is ready for release"
             )
-            try:
-                release.do(True, True, True)
-            except:
-                if release.has_rollback:
-                    logging.error(
-                        "!!The release process finished with error, read the output carefully!!"
-                    )
-                    logging.error(
-                        "Probably, rollback finished with error. "
-                        "If you don't see any of the following commands in the output, "
-                        "execute them manually:"
-                    )
-                    release.log_rollback()
-                raise
-            logging.info("New release is done!")
+            commit_reverse_index += 1
+
+            cmd = f"gh api -H 'Accept: application/vnd.github.v3+json' /repos/{GITHUB_REPOSITORY}/commits/{commit_sha}/status"
+            ci_status_json = Shell.run(cmd, check=True)
+            ci_status = json.loads(ci_status_json)["state"]
+            if ci_status == SUCCESS:
+                commit_found = True
             break
+        if commit_found:
+            print(
+                f"Add release ready info for commit [{commit_sha}] and release branch [{pr.head.ref}]"
+            )
+            autoRelease_info.add_release(
+                ReleaseParams(release_branch=pr.head.ref, commit_sha=commit_sha)
+            )
+        else:
+            print(f"WARNING: No good commits found for release branch [{pr.head.ref}]")
+            if commit_checked:
+                print(
+                    f"ERROR: CI is failed. check CI status for branch [{pr.head.ref}]"
+                )
+
+    autoRelease_info.dump()
 
 
 if __name__ == "__main__":
diff --git a/tests/ci/ci_utils.py b/tests/ci/ci_utils.py
index 49f0447b5ca..0653374356f 100644
--- a/tests/ci/ci_utils.py
+++ b/tests/ci/ci_utils.py
@@ -95,7 +95,8 @@ class Shell:
         return res.stdout.strip()
 
     @classmethod
-    def run(cls, command):
+    def run(cls, command, check=False):
+        print(f"Run command [{command}]")
         res = ""
         result = subprocess.run(
             command,
@@ -107,6 +108,9 @@ class Shell:
         )
         if result.returncode == 0:
             res = result.stdout
+        elif check:
+            print(f"ERROR: stdout {result.stdout}, stderr {result.stderr}")
+            assert result.returncode == 0
         return res.strip()
 
     @classmethod

From 3de472cedc22c1b109256507d81a491f9cf57d58 Mon Sep 17 00:00:00 2001
From: Max K <maxkaynov@gmail.com>
Date: Tue, 16 Jul 2024 17:07:49 +0200
Subject: [PATCH 0504/1488] add buddy, move release into action.yml

try callable wf

fix

add ci buddy
---
 .github/actions/release/action.yml   | 165 ++++++++++++++++++++++++
 .github/workflows/auto_release.yml   |  55 +++++++-
 .github/workflows/create_release.yml | 140 +-------------------
 tests/ci/artifactory.py              |  32 ++---
 tests/ci/auto_release.py             | 183 ++++++++++++++++++---------
 tests/ci/ci.py                       |   2 +-
 tests/ci/ci_buddy.py                 |  56 +++++++-
 tests/ci/ci_config.py                |   3 +
 tests/ci/ci_utils.py                 |  76 ++++++++++-
 tests/ci/create_release.py           | 106 ++++++++++------
 tests/ci/pr_info.py                  |  13 +-
 11 files changed, 560 insertions(+), 271 deletions(-)
 create mode 100644 .github/actions/release/action.yml

diff --git a/.github/actions/release/action.yml b/.github/actions/release/action.yml
new file mode 100644
index 00000000000..fd4436f1f16
--- /dev/null
+++ b/.github/actions/release/action.yml
@@ -0,0 +1,165 @@
+name: Release
+
+description: Makes patch releases and creates new release branch
+
+inputs:
+  ref:
+    description: 'Git reference (branch or commit sha) from which to create the release'
+    required: true
+    type: string
+  type:
+    description: 'The type of release: "new" for a new release or "patch" for a patch release'
+    required: true
+    type: choice
+    options:
+      - patch
+      - new
+  dry-run:
+    description: 'Dry run'
+    required: false
+    default: true
+    type: boolean
+  token:
+    required: true
+    type: string
+
+runs:
+  using: "composite"
+  steps:
+    - name: Prepare Release Info
+      shell: bash
+      run: |
+        python3 ./tests/ci/create_release.py --prepare-release-info \
+          --ref ${{ inputs.ref }} --release-type ${{ inputs.type }} \
+          ${{ inputs.dry-run && '--dry-run' || '' }}
+        echo "::group::Release Info"
+        python3 -m json.tool /tmp/release_info.json
+        echo "::endgroup::"
+        release_tag=$(jq -r '.release_tag' /tmp/release_info.json)
+        commit_sha=$(jq -r '.commit_sha' /tmp/release_info.json)
+        echo "Release Tag: $release_tag"
+        echo "RELEASE_TAG=$release_tag" >> "$GITHUB_ENV"
+        echo "COMMIT_SHA=$commit_sha" >> "$GITHUB_ENV"
+    - name: Download All Release Artifacts
+      if: ${{ inputs.type == 'patch' }}
+      shell: bash
+      run: |
+        python3 ./tests/ci/create_release.py --download-packages ${{ inputs.dry-run && '--dry-run' || '' }}
+    - name: Push Git Tag for the Release
+      shell: bash
+      run: |
+        python3 ./tests/ci/create_release.py --push-release-tag ${{ inputs.dry-run && '--dry-run' || '' }}
+    - name: Push New Release Branch
+      if: ${{ inputs.type == 'new' }}
+      shell: bash
+      run: |
+        python3 ./tests/ci/create_release.py --push-new-release-branch ${{ inputs.dry-run && '--dry-run' || '' }}
+    - name: Bump CH Version and Update Contributors' List
+      shell: bash
+      run: |
+        python3 ./tests/ci/create_release.py --create-bump-version-pr ${{ inputs.dry-run && '--dry-run' || '' }}
+    - name: Checkout master
+      shell: bash
+      run: |
+        git checkout master
+    - name: Bump Docker versions, Changelog, Security
+      if: ${{ inputs.type == 'patch' }}
+      shell: bash
+      run: |
+        [ "$(git branch --show-current)" != "master" ] && echo "not on the master" && exit 1
+        echo "List versions"
+        ./utils/list-versions/list-versions.sh > ./utils/list-versions/version_date.tsv
+        echo "Update docker version"
+        ./utils/list-versions/update-docker-version.sh
+        echo "Generate ChangeLog"
+        export CI=1
+        docker run -u "${UID}:${GID}" -e PYTHONUNBUFFERED=1 -e CI=1 --network=host \
+          --volume=".:/ClickHouse" clickhouse/style-test \
+          /ClickHouse/tests/ci/changelog.py -v --debug-helpers \
+          --gh-user-or-token=${{ inputs.token }} --jobs=5 \
+          --output="/ClickHouse/docs/changelogs/${{ env.RELEASE_TAG }}.md" ${{ env.RELEASE_TAG }}
+        git add ./docs/changelogs/${{ env.RELEASE_TAG }}.md
+        echo "Generate Security"
+        python3 ./utils/security-generator/generate_security.py > SECURITY.md
+        git diff HEAD
+    - name: Create ChangeLog PR
+      if: ${{ inputs.type == 'patch' && ! inputs.dry-run }}
+      uses: peter-evans/create-pull-request@v6
+      with:
+        author: "robot-clickhouse <robot-clickhouse@users.noreply.github.com>"
+        token: ${{ inputs.token }}
+        committer: "robot-clickhouse <robot-clickhouse@users.noreply.github.com>"
+        commit-message: Update version_date.tsv and changelogs after ${{ env.RELEASE_TAG }}
+        branch: auto/${{ env.RELEASE_TAG }}
+        assignees: ${{ github.event.sender.login }}  # assign the PR to the tag pusher
+        delete-branch: true
+        title: Update version_date.tsv and changelog after ${{ env.RELEASE_TAG }}
+        labels: do not test
+        body: |
+          Update version_date.tsv and changelogs after ${{ env.RELEASE_TAG }}
+          ### Changelog category (leave one):
+          - Not for changelog (changelog entry is not required)
+    - name: Reset changes if Dry-run
+      if: ${{ inputs.dry-run }}
+      shell: bash
+      run: |
+        git reset --hard HEAD
+    - name: Checkout back to GITHUB_REF
+      shell: bash
+      run: |
+        git checkout "$GITHUB_REF_NAME"
+    - name: Create GH Release
+      shell: bash
+      if: ${{ inputs.type == 'patch' }}
+      run: |
+        python3 ./tests/ci/create_release.py --create-gh-release \
+          ${{ inputs.dry-run && '--dry-run' || '' }}
+    - name: Export TGZ Packages
+      if: ${{ inputs.type == 'patch' }}
+      shell: bash
+      run: |
+        python3 ./tests/ci/artifactory.py --export-tgz ${{ inputs.dry-run && '--dry-run' || '' }}
+    - name: Test TGZ Packages
+      if: ${{ inputs.type == 'patch' }}
+      shell: bash
+      run: |
+        python3 ./tests/ci/artifactory.py --test-tgz ${{ inputs.dry-run && '--dry-run' || '' }}
+    - name: Export RPM Packages
+      if: ${{ inputs.type == 'patch' }}
+      shell: bash
+      run: |
+        python3 ./tests/ci/artifactory.py --export-rpm ${{ inputs.dry-run && '--dry-run' || '' }}
+    - name: Test RPM Packages
+      if: ${{ inputs.type == 'patch' }}
+      shell: bash
+      run: |
+        python3 ./tests/ci/artifactory.py --test-rpm ${{ inputs.dry-run && '--dry-run' || '' }}
+    - name: Export Debian Packages
+      if: ${{ inputs.type == 'patch' }}
+      shell: bash
+      run: |
+        python3 ./tests/ci/artifactory.py --export-debian ${{ inputs.dry-run && '--dry-run' || '' }}
+    - name: Test Debian Packages
+      if: ${{ inputs.type == 'patch' }}
+      shell: bash
+      run: |
+        python3 ./tests/ci/artifactory.py --test-debian ${{ inputs.dry-run && '--dry-run' || '' }}
+    - name: Docker clickhouse/clickhouse-server building
+      if: ${{ inputs.type == 'patch' }}
+      shell: bash
+      run: |
+        cd "./tests/ci"
+        export CHECK_NAME="Docker server image"
+        python3 docker_server.py --release-type auto --version ${{ env.RELEASE_TAG }} --check-name "$CHECK_NAME" --sha ${{ env.COMMIT_SHA }} ${{ ! inputs.dry-run && '--push' || '' }}
+    - name: Docker clickhouse/clickhouse-keeper building
+      if: ${{ inputs.type == 'patch' }}
+      shell: bash
+      run: |
+        cd "./tests/ci"
+        export CHECK_NAME="Docker keeper image"
+        python3 docker_server.py --release-type auto --version ${{ env.RELEASE_TAG }} --check-name "$CHECK_NAME" --sha ${{ env.COMMIT_SHA }} ${{ ! inputs.dry-run && '--push'  || '' }}
+    - name: Post Slack Message
+      if: ${{ !cancelled() }}
+      shell: bash
+      run: |
+        python3 ./tests/ci/create_release.py --post-status ${{ inputs.dry-run && '--dry-run' || '' }}
diff --git a/.github/workflows/auto_release.yml b/.github/workflows/auto_release.yml
index e90b183b8d6..e0a94d3bbb1 100644
--- a/.github/workflows/auto_release.yml
+++ b/.github/workflows/auto_release.yml
@@ -2,6 +2,7 @@ name: AutoRelease
 
 env:
   PYTHONUNBUFFERED: 1
+  DRY_RUN: true
 
 concurrency:
   group: release
@@ -9,6 +10,12 @@ on: # yamllint disable-line rule:truthy
   # schedule:
   #   - cron: '0 10-16 * * 1-5'
   workflow_dispatch:
+    inputs:
+      dry-run:
+        description: 'Dry run'
+        required: false
+        default: true
+        type: boolean
 
 jobs:
   AutoRelease:
@@ -31,7 +38,7 @@ jobs:
       - name: Auto Release Prepare
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
-          python3 auto_release.py
+          python3 auto_release.py --prepare
           echo "::group::Auto Release Info"
           python3 -m json.tool /tmp/autorelease_info.json
           echo "::endgroup::"
@@ -40,14 +47,50 @@ jobs:
               cat  /tmp/autorelease_info.json
               echo 'EOF'
           } >> "$GITHUB_ENV"
+      - name: Post Release Branch statuses
+        run: |
+          cd "$GITHUB_WORKSPACE/tests/ci"
+          python3 auto_release.py --post-status
       - name: Release ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[0].release_branch }}
-        if: ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[0] }}
-        uses: ./.github/workflows/create_release.yml
+        if: ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[0] && fromJson(env.AUTO_RELEASE_PARAMS).releases[0].ready }}
+        uses: ./.github/actions/release
         with:
-          type: patch
           ref: ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[0].commit_sha }}
-          dry-run: true
-          autorelease: true
+          type: patch
+          dry-run: ${{ inputs.dry-run }}
+          token: ${{secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN}}
+      - name: Release ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[1].release_branch }}
+        if: ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[0] && fromJson(env.AUTO_RELEASE_PARAMS).releases[1].ready }}
+        uses: ./.github/actions/release
+        with:
+          ref: ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[1].commit_sha }}
+          type: patch
+          dry-run: ${{ inputs.dry-run }}
+          token: ${{secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN}}
+      - name: Release ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[2].release_branch }}
+        if: ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[2] && fromJson(env.AUTO_RELEASE_PARAMS).releases[2].ready }}
+        uses: ./.github/actions/release
+        with:
+          ref: ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[2].commit_sha }}
+          type: patch
+          dry-run: ${{ inputs.dry-run }}
+          token: ${{secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN}}
+      - name: Release ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[3].release_branch }}
+        if: ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[3] && fromJson(env.AUTO_RELEASE_PARAMS).releases[3].ready }}
+        uses: ./.github/actions/release
+        with:
+          ref: ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[3].commit_sha }}
+          type: patch
+          dry-run: ${{ inputs.dry-run }}
+          token: ${{secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN}}
+      - name: Release ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[4].release_branch }}
+        if: ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[4] && fromJson(env.AUTO_RELEASE_PARAMS).releases[4].ready }}
+        uses: ./.github/actions/release
+        with:
+          ref: ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[4].commit_sha }}
+          type: patch
+          dry-run: ${{ inputs.dry-run }}
+          token: ${{secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN}}
       - name: Post Slack Message
         if: ${{ !cancelled() }}
         run: |
diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml
index 96cd46f583a..6246306e536 100644
--- a/.github/workflows/create_release.yml
+++ b/.github/workflows/create_release.yml
@@ -3,7 +3,7 @@ name: CreateRelease
 concurrency:
   group: release
 
-'on':
+on:
   workflow_dispatch:
     inputs:
       ref:
@@ -22,10 +22,6 @@ concurrency:
         required: false
         default: true
         type: boolean
-      autorelease:
-        required: false
-        default: false
-        type: boolean
 
 jobs:
   CreateRelease:
@@ -36,138 +32,16 @@ jobs:
       - name: DebugInfo
         if: ${{ ! inputs.autorelease }}
         uses: hmarr/debug-action@f7318c783045ac39ed9bb497e22ce835fdafbfe6
-      - name: Set envs
-        if: ${{ ! inputs.autorelease }}
-        # https://docs.github.com/en/actions/learn-github-actions/workflow-commands-for-github-actions#multiline-strings
-        run: |
-          cat >> "$GITHUB_ENV" << 'EOF'
-          ROBOT_CLICKHOUSE_SSH_KEY<<RCSK
-          ${{secrets.ROBOT_CLICKHOUSE_SSH_KEY}}
-          RCSK
-          RELEASE_INFO_FILE=${{ runner.temp }}/release_info.json
-          EOF
       - name: Check out repository code
         if: ${{ ! inputs.autorelease }}
         uses: ClickHouse/checkout@v1
         with:
           token: ${{secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN}}
           fetch-depth: 0
-      - name: Prepare Release Info
-        run: |
-          python3 ./tests/ci/create_release.py --prepare-release-info \
-            --ref ${{ inputs.ref }} --release-type ${{ inputs.type }} \
-            --outfile ${{ env.RELEASE_INFO_FILE }} ${{ inputs.dry-run && '--dry-run' || '' }}
-          echo "::group::Release Info"
-          python3 -m json.tool "$RELEASE_INFO_FILE"
-          echo "::endgroup::"
-          release_tag=$(jq -r '.release_tag' "$RELEASE_INFO_FILE")
-          commit_sha=$(jq -r '.commit_sha' "$RELEASE_INFO_FILE")
-          echo "Release Tag: $release_tag"
-          echo "RELEASE_TAG=$release_tag" >> "$GITHUB_ENV"
-          echo "COMMIT_SHA=$commit_sha" >> "$GITHUB_ENV"
-      - name: Download All Release Artifacts
-        if: ${{ inputs.type == 'patch' }}
-        run: |
-          python3 ./tests/ci/create_release.py --infile "$RELEASE_INFO_FILE" --download-packages ${{ inputs.dry-run && '--dry-run' || '' }}
-      - name: Push Git Tag for the Release
-        run: |
-          python3 ./tests/ci/create_release.py --push-release-tag --infile "$RELEASE_INFO_FILE" ${{ inputs.dry-run && '--dry-run' || '' }}
-      - name: Push New Release Branch
-        if: ${{ inputs.type == 'new' }}
-        run: |
-          python3 ./tests/ci/create_release.py --push-new-release-branch --infile "$RELEASE_INFO_FILE" ${{ inputs.dry-run && '--dry-run' || '' }}
-      - name: Bump CH Version and Update Contributors' List
-        run: |
-          python3 ./tests/ci/create_release.py --create-bump-version-pr --infile "$RELEASE_INFO_FILE"  ${{ inputs.dry-run && '--dry-run' || '' }}
-      - name: Checkout master
-        run: |
-          git checkout master
-      - name: Bump Docker versions, Changelog, Security
-        if: ${{ inputs.type == 'patch' }}
-        run: |
-          [ "$(git branch --show-current)" != "master" ] && echo "not on the master" && exit 1
-          echo "List versions"
-          ./utils/list-versions/list-versions.sh > ./utils/list-versions/version_date.tsv
-          echo "Update docker version"
-          ./utils/list-versions/update-docker-version.sh
-          echo "Generate ChangeLog"
-          export CI=1
-          docker run -u "${UID}:${GID}" -e PYTHONUNBUFFERED=1 -e CI=1 --network=host \
-            --volume=".:/ClickHouse" clickhouse/style-test \
-            /ClickHouse/tests/ci/changelog.py -v --debug-helpers \
-            --gh-user-or-token="$GH_TOKEN" --jobs=5 \
-            --output="/ClickHouse/docs/changelogs/${{ env.RELEASE_TAG }}.md" ${{ env.RELEASE_TAG }}
-          git add ./docs/changelogs/${{ env.RELEASE_TAG }}.md
-          echo "Generate Security"
-          python3 ./utils/security-generator/generate_security.py > SECURITY.md
-          git diff HEAD
-      - name: Create ChangeLog PR
-        if: ${{ inputs.type == 'patch' && ! inputs.dry-run }}
-        uses: peter-evans/create-pull-request@v6
+      - name: Call Release Action
+        uses: ./.github/actions/release
         with:
-          author: "robot-clickhouse <robot-clickhouse@users.noreply.github.com>"
-          token: ${{ secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN }}
-          committer: "robot-clickhouse <robot-clickhouse@users.noreply.github.com>"
-          commit-message: Update version_date.tsv and changelogs after ${{ env.RELEASE_TAG }}
-          branch: auto/${{ env.RELEASE_TAG }}
-          assignees: ${{ github.event.sender.login }}  # assign the PR to the tag pusher
-          delete-branch: true
-          title: Update version_date.tsv and changelog after ${{ env.RELEASE_TAG }}
-          labels: do not test
-          body: |
-            Update version_date.tsv and changelogs after ${{ env.RELEASE_TAG }}
-            ### Changelog category (leave one):
-            - Not for changelog (changelog entry is not required)
-      - name: Reset changes if Dry-run
-        if: ${{ inputs.dry-run }}
-        run: |
-          git reset --hard HEAD
-      - name: Checkout back to GITHUB_REF
-        run: |
-          git checkout "$GITHUB_REF_NAME"
-      - name: Create GH Release
-        if: ${{ inputs.type == 'patch' }}
-        run: |
-          python3 ./tests/ci/create_release.py --create-gh-release \
-            --infile ${{ env.RELEASE_INFO_FILE }} ${{ inputs.dry-run && '--dry-run' || '' }}
-
-      - name: Export TGZ Packages
-        if: ${{ inputs.type == 'patch' }}
-        run: |
-          python3 ./tests/ci/artifactory.py --export-tgz --infile ${{ env.RELEASE_INFO_FILE }} ${{ inputs.dry-run && '--dry-run' || '' }}
-      - name: Test TGZ Packages
-        if: ${{ inputs.type == 'patch' }}
-        run: |
-          python3 ./tests/ci/artifactory.py --test-tgz --infile ${{ env.RELEASE_INFO_FILE }} ${{ inputs.dry-run && '--dry-run' || '' }}
-      - name: Export RPM Packages
-        if: ${{ inputs.type == 'patch' }}
-        run: |
-          python3 ./tests/ci/artifactory.py --export-rpm --infile ${{ env.RELEASE_INFO_FILE }} ${{ inputs.dry-run && '--dry-run' || '' }}
-      - name: Test RPM Packages
-        if: ${{ inputs.type == 'patch' }}
-        run: |
-          python3 ./tests/ci/artifactory.py --test-rpm --infile ${{ env.RELEASE_INFO_FILE }} ${{ inputs.dry-run && '--dry-run' || '' }}
-      - name: Export Debian Packages
-        if: ${{ inputs.type == 'patch' }}
-        run: |
-            python3 ./tests/ci/artifactory.py --export-debian --infile ${{ env.RELEASE_INFO_FILE }} ${{ inputs.dry-run && '--dry-run' || '' }}
-      - name: Test Debian Packages
-        if: ${{ inputs.type == 'patch' }}
-        run: |
-          python3 ./tests/ci/artifactory.py --test-debian --infile ${{ env.RELEASE_INFO_FILE }} ${{ inputs.dry-run && '--dry-run' || '' }}
-      - name: Docker clickhouse/clickhouse-server building
-        if: ${{ inputs.type == 'patch' }}
-        run: |
-          cd "./tests/ci"
-          export CHECK_NAME="Docker server image"
-          python3 docker_server.py --release-type auto --version ${{ env.RELEASE_TAG }} --check-name "$CHECK_NAME" --sha ${{ env.COMMIT_SHA }} ${{ ! inputs.dry-run && '--push' || '' }}
-      - name: Docker clickhouse/clickhouse-keeper building
-        if: ${{ inputs.type == 'patch' }}
-        run: |
-          cd "./tests/ci"
-          export CHECK_NAME="Docker keeper image"
-          python3 docker_server.py --release-type auto --version ${{ env.RELEASE_TAG }} --check-name "$CHECK_NAME" --sha ${{ env.COMMIT_SHA }} ${{ ! inputs.dry-run && '--push'  || '' }}
-      - name: Post Slack Message
-        if: ${{ !cancelled() }}
-        run: |
-          echo Slack Message
+          ref: ${{ inputs.ref }}
+          type: inputs.type
+          dry-run: ${{ inputs.dry-run }}
+          token: ${{secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN}}
diff --git a/tests/ci/artifactory.py b/tests/ci/artifactory.py
index 1a062d05a23..2009b122a18 100644
--- a/tests/ci/artifactory.py
+++ b/tests/ci/artifactory.py
@@ -4,7 +4,7 @@ from pathlib import Path
 from typing import Optional
 from shutil import copy2
 from create_release import PackageDownloader, ReleaseInfo, ShellRunner
-from ci_utils import WithIter
+from ci_utils import WithIter, Shell
 
 
 class MountPointApp(metaclass=WithIter):
@@ -141,12 +141,16 @@ class DebianArtifactory:
             ShellRunner.run("sync")
 
     def test_packages(self):
-        ShellRunner.run("docker pull ubuntu:latest")
+        Shell.run("docker pull ubuntu:latest")
         print(f"Test packages installation, version [{self.version}]")
-        cmd = f"docker run --rm ubuntu:latest bash -c \"apt update -y; apt install -y sudo gnupg ca-certificates; apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 8919F6BD2B48D754; echo 'deb {self.repo_url} stable main' | tee /etc/apt/sources.list.d/clickhouse.list; apt update -y; apt-get install -y clickhouse-client={self.version}\""
+        debian_command = f"echo 'deb {self.repo_url} stable main' | tee /etc/apt/sources.list.d/clickhouse.list; apt update -y; apt-get install -y clickhouse-common-static={self.version} clickhouse-client={self.version}"
+        cmd = f'docker run --rm ubuntu:latest bash -c "apt update -y; apt install -y sudo gnupg ca-certificates; apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 8919F6BD2B48D754; {debian_command}"'
         print("Running test command:")
         print(f"  {cmd}")
-        ShellRunner.run(cmd)
+        Shell.run(cmd, check=True)
+        release_info = ReleaseInfo.from_file()
+        release_info.debian_command = debian_command
+        release_info.dump()
 
 
 def _copy_if_not_exists(src: Path, dst: Path) -> Path:
@@ -210,15 +214,19 @@ class RpmArtifactory:
         pub_key_path.write_text(ShellRunner.run(update_public_key)[1])
         if codename == RepoCodenames.LTS:
             self.export_packages(RepoCodenames.STABLE)
-        ShellRunner.run("sync")
+        Shell.run("sync")
 
     def test_packages(self):
-        ShellRunner.run("docker pull fedora:latest")
+        Shell.run("docker pull fedora:latest")
         print(f"Test package installation, version [{self.version}]")
-        cmd = f'docker run --rm fedora:latest /bin/bash -c "dnf -y install dnf-plugins-core && dnf config-manager --add-repo={self.repo_url} && dnf makecache && dnf -y install clickhouse-client-{self.version}-1"'
+        rpm_command = f"dnf config-manager --add-repo={self.repo_url} && dnf makecache && dnf -y install clickhouse-client-{self.version}-1"
+        cmd = f'docker run --rm fedora:latest /bin/bash -c "dnf -y install dnf-plugins-core && dnf config-manager --add-repo={self.repo_url} && {rpm_command}"'
         print("Running test command:")
         print(f"  {cmd}")
-        ShellRunner.run(cmd)
+        Shell.run(cmd, check=True)
+        release_info = ReleaseInfo.from_file()
+        release_info.rpm_command = rpm_command
+        release_info.dump()
 
 
 class TgzArtifactory:
@@ -280,12 +288,6 @@ def parse_args() -> argparse.Namespace:
         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
         description="Adds release packages to the repository",
     )
-    parser.add_argument(
-        "--infile",
-        type=str,
-        required=True,
-        help="input file with release info",
-    )
     parser.add_argument(
         "--export-debian",
         action="store_true",
@@ -328,7 +330,7 @@ if __name__ == "__main__":
     args = parse_args()
     assert args.dry_run
 
-    release_info = ReleaseInfo.from_file(args.infile)
+    release_info = ReleaseInfo.from_file()
     """
     Use S3FS. RCLONE has some errors with r2 remote which I didn't figure out how to resolve:
            ERROR : IO error: NotImplemented: versionId not implemented
diff --git a/tests/ci/auto_release.py b/tests/ci/auto_release.py
index 88e91eb8aed..b8f781c4d73 100644
--- a/tests/ci/auto_release.py
+++ b/tests/ci/auto_release.py
@@ -1,20 +1,17 @@
 import argparse
 import dataclasses
 import json
-import logging
 import os
+import sys
 from typing import List
 
 from get_robot_token import get_best_robot_token
 from github_helper import GitHub
-from ssh import SSHKey
 from ci_utils import Shell
 from env_helper import GITHUB_REPOSITORY
 from report import SUCCESS
-
-LOGGER_NAME = __name__
-HELPER_LOGGERS = ["github_helper", LOGGER_NAME]
-logger = logging.getLogger(LOGGER_NAME)
+from ci_buddy import CIBuddy
+from ci_config import CI
 
 
 def parse_args():
@@ -23,8 +20,17 @@ def parse_args():
         "branches and do a release in case for green builds."
     )
     parser.add_argument("--token", help="GitHub token, if not set, used from smm")
-
-    return parser.parse_args()
+    parser.add_argument(
+        "--post-status",
+        action="store_true",
+        help="Post release branch statuses",
+    )
+    parser.add_argument(
+        "--prepare",
+        action="store_true",
+        help="Prepare autorelease info",
+    )
+    return parser.parse_args(), parser
 
 
 MAX_NUMBER_OF_COMMITS_TO_CONSIDER_FOR_RELEASE = 5
@@ -33,8 +39,16 @@ AUTORELEASE_INFO_FILE = "/tmp/autorelease_info.json"
 
 @dataclasses.dataclass
 class ReleaseParams:
+    ready: bool
+    ci_status: str
+    num_patches: int
     release_branch: str
     commit_sha: str
+    commits_to_branch_head: int
+    latest: bool
+
+    def to_dict(self):
+        return dataclasses.asdict(self)
 
 
 @dataclasses.dataclass
@@ -49,83 +63,128 @@ class AutoReleaseInfo:
         with open(AUTORELEASE_INFO_FILE, "w", encoding="utf-8") as f:
             print(json.dumps(dataclasses.asdict(self), indent=2), file=f)
 
+    @staticmethod
+    def from_file() -> "AutoReleaseInfo":
+        with open(AUTORELEASE_INFO_FILE, "r", encoding="utf-8") as json_file:
+            res = json.load(json_file)
+        releases = [ReleaseParams(**release) for release in res["releases"]]
+        return AutoReleaseInfo(releases=releases)
 
-def main():
-    args = parse_args()
 
-    token = args.token or get_best_robot_token()
+def _prepare(token):
     assert len(token) > 10
     os.environ["GH_TOKEN"] = token
-    (Shell.run("gh auth status", check=True))
+    Shell.run("gh auth status", check=True)
+
     gh = GitHub(token)
     prs = gh.get_release_pulls(GITHUB_REPOSITORY)
+    prs.sort(key=lambda x: x.head.ref)
     branch_names = [pr.head.ref for pr in prs]
-
     print(f"Found release branches [{branch_names}]")
-    repo = gh.get_repo(GITHUB_REPOSITORY)
 
+    repo = gh.get_repo(GITHUB_REPOSITORY)
     autoRelease_info = AutoReleaseInfo(releases=[])
+
     for pr in prs:
-        print(f"Checking PR [{pr.head.ref}]")
+        print(f"\nChecking PR [{pr.head.ref}]")
 
         refs = list(repo.get_git_matching_refs(f"tags/v{pr.head.ref}"))
-        refs.sort(key=lambda ref: ref.ref)
+        assert refs
 
+        refs.sort(key=lambda ref: ref.ref)
         latest_release_tag_ref = refs[-1]
         latest_release_tag = repo.get_git_tag(latest_release_tag_ref.object.sha)
-        commit_num = int(
-            Shell.run(
-                f"git rev-list --count {latest_release_tag.tag}..origin/{pr.head.ref}",
-                check=True,
-            )
-        )
-        print(
-            f"Previous release is [{latest_release_tag}] was [{commit_num}] commits before, date [{latest_release_tag.tagger.date}]"
-        )
-        commit_reverse_index = 0
-        commit_found = False
-        commit_checked = False
-        commit_sha = ""
-        while (
-            commit_reverse_index < commit_num - 1
-            and commit_reverse_index < MAX_NUMBER_OF_COMMITS_TO_CONSIDER_FOR_RELEASE
-        ):
-            commit_checked = True
-            commit_sha = Shell.run(
-                f"git rev-list --max-count=1 --skip={commit_reverse_index} origin/{pr.head.ref}",
-                check=True,
-            )
-            print(
-                f"Check if commit [{commit_sha}] [{pr.head.ref}~{commit_reverse_index}] is ready for release"
-            )
-            commit_reverse_index += 1
 
-            cmd = f"gh api -H 'Accept: application/vnd.github.v3+json' /repos/{GITHUB_REPOSITORY}/commits/{commit_sha}/status"
-            ci_status_json = Shell.run(cmd, check=True)
-            ci_status = json.loads(ci_status_json)["state"]
-            if ci_status == SUCCESS:
-                commit_found = True
-            break
-        if commit_found:
+        commits = Shell.run(
+            f"git rev-list --first-parent {latest_release_tag.tag}..origin/{pr.head.ref}",
+            check=True,
+        ).split("\n")
+        commit_num = len(commits)
+        print(
+            f"Previous release [{latest_release_tag.tag}] was [{commit_num}] commits ago, date [{latest_release_tag.tagger.date}]"
+        )
+
+        commits_to_check = commits[:-1]  # Exclude the version bump commit
+        commit_sha = ""
+        commit_ci_status = ""
+        commits_to_branch_head = 0
+
+        for idx, commit in enumerate(
+            commits_to_check[:MAX_NUMBER_OF_COMMITS_TO_CONSIDER_FOR_RELEASE]
+        ):
+            print(
+                f"Check commit [{commit}] [{pr.head.ref}~{idx+1}] as release candidate"
+            )
+            commit_num -= 1
+
+            is_completed = CI.GHActions.check_wf_completed(
+                token=token, commit_sha=commit
+            )
+            if not is_completed:
+                print(f"CI is in progress for [{commit}] - check previous commit")
+                commits_to_branch_head += 1
+                continue
+
+            commit_ci_status = CI.GHActions.get_commit_status_by_name(
+                token=token,
+                commit_sha=commit,
+                status_name=(CI.JobNames.BUILD_CHECK, "ClickHouse build check"),
+            )
+            commit_sha = commit
+            if commit_ci_status == SUCCESS:
+                break
+            else:
+                print(f"CI status [{commit_ci_status}] - skip")
+            commits_to_branch_head += 1
+
+        ready = commit_ci_status == SUCCESS and commit_sha
+        if ready:
             print(
                 f"Add release ready info for commit [{commit_sha}] and release branch [{pr.head.ref}]"
             )
-            autoRelease_info.add_release(
-                ReleaseParams(release_branch=pr.head.ref, commit_sha=commit_sha)
-            )
         else:
-            print(f"WARNING: No good commits found for release branch [{pr.head.ref}]")
-            if commit_checked:
-                print(
-                    f"ERROR: CI is failed. check CI status for branch [{pr.head.ref}]"
-                )
+            print(f"WARNING: No ready commits found for release branch [{pr.head.ref}]")
+
+        autoRelease_info.add_release(
+            ReleaseParams(
+                release_branch=pr.head.ref,
+                commit_sha=commit_sha,
+                ready=ready,
+                ci_status=commit_ci_status,
+                num_patches=commit_num,
+                commits_to_branch_head=commits_to_branch_head,
+                latest=False,
+            )
+        )
+
+    if autoRelease_info.releases:
+        autoRelease_info.releases[-1].latest = True
 
     autoRelease_info.dump()
 
 
-if __name__ == "__main__":
-    if os.getenv("ROBOT_CLICKHOUSE_SSH_KEY", ""):
-        with SSHKey("ROBOT_CLICKHOUSE_SSH_KEY"):
-            main()
+def main():
+    args, parser = parse_args()
+
+    if args.post_status:
+        info = AutoReleaseInfo.from_file()
+        for release_info in info.releases:
+            if release_info.ready:
+                CIBuddy(dry_run=False).post_info(
+                    title=f"Auto Release Status for {release_info.release_branch}",
+                    body=release_info.to_dict(),
+                )
+            else:
+                CIBuddy(dry_run=False).post_warning(
+                    title=f"Auto Release Status for {release_info.release_branch}",
+                    body=release_info.to_dict(),
+                )
+    elif args.prepare:
+        _prepare(token=args.token or get_best_robot_token())
     else:
-        main()
+        parser.print_help()
+        sys.exit(2)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/tests/ci/ci.py b/tests/ci/ci.py
index 110a7b2a49c..8ad358cb874 100644
--- a/tests/ci/ci.py
+++ b/tests/ci/ci.py
@@ -1295,7 +1295,7 @@ def main() -> int:
                 error_description = f"Out Of Memory, exit_code {job_report.exit_code}"
             else:
                 error_description = f"Unknown, exit_code {job_report.exit_code}"
-            CIBuddy().post_error(
+            CIBuddy().post_job_error(
                 error_description + f" after {int(job_report.duration)}s",
                 job_name=_get_ext_check_name(args.job_name),
             )
diff --git a/tests/ci/ci_buddy.py b/tests/ci/ci_buddy.py
index c650b876610..727a3d88359 100644
--- a/tests/ci/ci_buddy.py
+++ b/tests/ci/ci_buddy.py
@@ -1,5 +1,6 @@
 import json
 import os
+from typing import Union, Dict
 
 import boto3
 import requests
@@ -60,7 +61,55 @@ class CIBuddy:
         except Exception as e:
             print(f"ERROR: Failed to post message, ex {e}")
 
-    def post_error(self, error_description, job_name="", with_instance_info=True):
+    def _post_formatted(
+        self, title, body: Union[Dict, str], with_wf_link: bool
+    ) -> None:
+        message = title
+        if isinstance(body, dict):
+            for name, value in body.items():
+                if "commit_sha" in name:
+                    value = (
+                        f"<https://github.com/{self.repo}/commit/{value}|{value[:8]}>"
+                    )
+                message += f"      *{name}*:    {value}\n"
+        else:
+            message += body + "\n"
+        run_id = os.getenv("GITHUB_RUN_ID", "")
+        if with_wf_link and run_id:
+            message += f"      *workflow*: <https://github.com/{self.repo}/actions/runs/{run_id}|{run_id}>\n"
+        self.post(message)
+
+    def post_info(
+        self, title, body: Union[Dict, str], with_wf_link: bool = True
+    ) -> None:
+        title_extended = f":white_circle:    *{title}*\n\n"
+        self._post_formatted(title_extended, body, with_wf_link)
+
+    def post_done(
+        self, title, body: Union[Dict, str], with_wf_link: bool = True
+    ) -> None:
+        title_extended = f":white_check_mark:    *{title}*\n\n"
+        self._post_formatted(title_extended, body, with_wf_link)
+
+    def post_warning(
+        self, title, body: Union[Dict, str], with_wf_link: bool = True
+    ) -> None:
+        title_extended = f":warning:    *{title}*\n\n"
+        self._post_formatted(title_extended, body, with_wf_link)
+
+    def post_critical(
+        self, title, body: Union[Dict, str], with_wf_link: bool = True
+    ) -> None:
+        title_extended = f":black_circle:    *{title}*\n\n"
+        self._post_formatted(title_extended, body, with_wf_link)
+
+    def post_job_error(
+        self,
+        error_description,
+        job_name="",
+        with_instance_info=True,
+        with_wf_link: bool = True,
+    ):
         instance_id, instance_type = "unknown", "unknown"
         if with_instance_info:
             instance_id = Shell.run("ec2metadata --instance-id") or instance_id
@@ -82,10 +131,13 @@ class CIBuddy:
             message += line_pr_
         else:
             message += line_br_
+        run_id = os.getenv("GITHUB_RUN_ID", "")
+        if with_wf_link and run_id:
+            message += f"      *workflow*: <https://github.com/{self.repo}/actions/runs/{run_id}|{run_id}>\n"
         self.post(message)
 
 
 if __name__ == "__main__":
     # test
     buddy = CIBuddy(dry_run=True)
-    buddy.post_error("TEst")
+    buddy.post_job_error("TEst")
diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py
index 0a4ab3a823b..a44b15f34c1 100644
--- a/tests/ci/ci_config.py
+++ b/tests/ci/ci_config.py
@@ -32,6 +32,9 @@ class CI:
     from ci_definitions import MQ_JOBS as MQ_JOBS
     from ci_definitions import WorkflowStages as WorkflowStages
     from ci_definitions import Runners as Runners
+    from ci_utils import Envs as Envs
+    from ci_utils import Utils as Utils
+    from ci_utils import GHActions as GHActions
     from ci_definitions import Labels as Labels
     from ci_definitions import TRUSTED_CONTRIBUTORS as TRUSTED_CONTRIBUTORS
     from ci_utils import CATEGORY_TO_LABEL as CATEGORY_TO_LABEL
diff --git a/tests/ci/ci_utils.py b/tests/ci/ci_utils.py
index 0653374356f..4536d1f2b54 100644
--- a/tests/ci/ci_utils.py
+++ b/tests/ci/ci_utils.py
@@ -1,9 +1,16 @@
 import os
 import re
 import subprocess
+import time
 from contextlib import contextmanager
 from pathlib import Path
-from typing import Any, Iterator, List, Union, Optional, Tuple
+from typing import Any, Iterator, List, Union, Optional, Sequence
+
+import requests
+
+
+class Envs:
+    GITHUB_REPOSITORY = os.getenv("GITHUB_REPOSITORY", "ClickHouse/ClickHouse")
 
 
 LABEL_CATEGORIES = {
@@ -80,6 +87,62 @@ class GHActions:
             print(line)
         print("::endgroup::")
 
+    @staticmethod
+    def get_commit_status_by_name(
+        token: str, commit_sha: str, status_name: Union[str, Sequence]
+    ) -> Optional[str]:
+        assert len(token) == 40
+        assert len(commit_sha) == 40
+        assert is_hex(commit_sha)
+        assert not is_hex(token)
+        url = f"https://api.github.com/repos/{Envs.GITHUB_REPOSITORY}/commits/{commit_sha}/statuses?per_page={200}"
+        headers = {
+            "Authorization": f"token {token}",
+            "Accept": "application/vnd.github.v3+json",
+        }
+        response = requests.get(url, headers=headers, timeout=5)
+
+        if isinstance(status_name, str):
+            status_name = (status_name,)
+        if response.status_code == 200:
+            assert "next" not in response.links, "Response truncated"
+            statuses = response.json()
+            for status in statuses:
+                if status["context"] in status_name:
+                    return status["state"]
+        return None
+
+    @staticmethod
+    def check_wf_completed(token: str, commit_sha: str) -> bool:
+        headers = {
+            "Authorization": f"token {token}",
+            "Accept": "application/vnd.github.v3+json",
+        }
+        url = f"https://api.github.com/repos/{Envs.GITHUB_REPOSITORY}/commits/{commit_sha}/check-runs?per_page={100}"
+
+        for i in range(3):
+            try:
+                response = requests.get(url, headers=headers, timeout=5)
+                response.raise_for_status()
+                # assert "next" not in response.links, "Response truncated"
+
+                data = response.json()
+                assert data["check_runs"], "?"
+
+                for check in data["check_runs"]:
+                    if check["status"] != "completed":
+                        print(
+                            f"   Check workflow status: Check not completed [{check['name']}]"
+                        )
+                        return False
+                else:
+                    return True
+            except Exception as e:
+                print(f"ERROR: exception {e}")
+                time.sleep(1)
+
+        return False
+
 
 class Shell:
     @classmethod
@@ -108,15 +171,18 @@ class Shell:
         )
         if result.returncode == 0:
             res = result.stdout
-        elif check:
-            print(f"ERROR: stdout {result.stdout}, stderr {result.stderr}")
-            assert result.returncode == 0
+        else:
+            print(
+                f"ERROR: stdout {result.stdout.strip()}, stderr {result.stderr.strip()}"
+            )
+            if check:
+                assert result.returncode == 0
         return res.strip()
 
     @classmethod
     def check(cls, command):
         result = subprocess.run(
-            command + " 2>&1",
+            command,
             shell=True,
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
diff --git a/tests/ci/create_release.py b/tests/ci/create_release.py
index 277134c3991..414ec8afd3e 100755
--- a/tests/ci/create_release.py
+++ b/tests/ci/create_release.py
@@ -14,6 +14,7 @@ from ssh import SSHAgent
 from env_helper import GITHUB_REPOSITORY, S3_BUILDS_BUCKET
 from s3_helper import S3Helper
 from ci_utils import Shell
+from ci_buddy import CIBuddy
 from version_helper import (
     FILE_WITH_VERSION_PATH,
     GENERATED_CONTRIBUTORS,
@@ -27,6 +28,7 @@ from ci_config import CI
 
 CMAKE_PATH = get_abs_path(FILE_WITH_VERSION_PATH)
 CONTRIBUTORS_PATH = get_abs_path(GENERATED_CONTRIBUTORS)
+RELEASE_INFO_FILE = "/tmp/release_info.json"
 
 
 class ShellRunner:
@@ -67,17 +69,25 @@ class ReleaseInfo:
     codename: str
     previous_release_tag: str
     previous_release_sha: str
+    changelog_pr: str = ""
+    version_bump_pr: str = ""
+    release_url: str = ""
+    debian_command: str = ""
+    rpm_command: str = ""
 
     @staticmethod
-    def from_file(file_path: str) -> "ReleaseInfo":
-        with open(file_path, "r", encoding="utf-8") as json_file:
+    def from_file() -> "ReleaseInfo":
+        with open(RELEASE_INFO_FILE, "r", encoding="utf-8") as json_file:
             res = json.load(json_file)
         return ReleaseInfo(**res)
 
+    def dump(self):
+        print(f"Dump release info into [{RELEASE_INFO_FILE}]")
+        with open(RELEASE_INFO_FILE, "w", encoding="utf-8") as f:
+            print(json.dumps(dataclasses.asdict(self), indent=2), file=f)
+
     @staticmethod
-    def prepare(commit_ref: str, release_type: str, outfile: str) -> None:
-        Path(outfile).parent.mkdir(parents=True, exist_ok=True)
-        Path(outfile).unlink(missing_ok=True)
+    def prepare(commit_ref: str, release_type: str) -> None:
         version = None
         release_branch = None
         release_tag = None
@@ -91,7 +101,7 @@ class ReleaseInfo:
                 f"git merge-base --is-ancestor origin/{commit_ref} origin/master"
             )
             with checkout(commit_ref):
-                _, commit_sha = ShellRunner.run(f"git rev-parse {commit_ref}")
+                commit_sha = Shell.run(f"git rev-parse {commit_ref}", check=True)
                 # Git() must be inside "with checkout" contextmanager
                 git = Git()
                 version = get_version_from_repo(git=git)
@@ -112,7 +122,7 @@ class ReleaseInfo:
                 assert previous_release_sha
         if release_type == "patch":
             with checkout(commit_ref):
-                _, commit_sha = ShellRunner.run(f"git rev-parse {commit_ref}")
+                commit_sha = Shell.run(f"git rev-parse {commit_ref}", check=True)
                 # Git() must be inside "with checkout" contextmanager
                 git = Git()
                 version = get_version_from_repo(git=git)
@@ -171,8 +181,7 @@ class ReleaseInfo:
             previous_release_tag=previous_release_tag,
             previous_release_sha=previous_release_sha,
         )
-        with open(outfile, "w", encoding="utf-8") as f:
-            print(json.dumps(dataclasses.asdict(res), indent=2), file=f)
+        res.dump()
 
     def push_release_tag(self, dry_run: bool) -> None:
         if dry_run:
@@ -276,21 +285,38 @@ class ReleaseInfo:
                         f"{GIT_PREFIX} checkout '{CMAKE_PATH}' '{CONTRIBUTORS_PATH}'"
                     )
 
+    def update_release_info(self, dry_run: bool) -> None:
+        branch = f"auto/{release_info.release_tag}"
+        if not dry_run:
+            get_url_cmd = f"gh pr list --repo {GITHUB_REPOSITORY} --head {branch} --json url --jq '.[0].url'"
+            url = Shell.run(get_url_cmd)
+            if url:
+                print(f"Update release info with Changelog PR link [{url}]")
+            else:
+                print(f"WARNING: Changelog PR not found, branch [{branch}]")
+        else:
+            url = "dry-run"
+
+        self.changelog_pr = url
+        self.dump()
+
     def create_gh_release(self, packages_files: List[str], dry_run: bool) -> None:
         repo = os.getenv("GITHUB_REPOSITORY")
         assert repo
-        cmds = []
-        cmds.append(
+        cmds = [
             f"gh release create --repo {repo} --title 'Release {self.release_tag}' {self.release_tag}"
-        )
+        ]
         for file in packages_files:
             cmds.append(f"gh release upload {self.release_tag} {file}")
         if not dry_run:
             for cmd in cmds:
-                ShellRunner.run(cmd)
+                Shell.run(cmd, check=True)
+            self.release_url = f"https://github.com/{GITHUB_REPOSITORY}/releases/tag/{self.release_tag}"
         else:
             print("Dry-run, would run commands:")
             print("\n  * ".join(cmds))
+            self.release_url = f"dry-run"
+        self.dump()
 
 
 class RepoTypes:
@@ -508,6 +534,11 @@ def parse_args() -> argparse.Namespace:
         action="store_true",
         help="Create GH Release object and attach all packages",
     )
+    parser.add_argument(
+        "--post-status",
+        action="store_true",
+        help="Post release status into Slack",
+    )
     parser.add_argument(
         "--ref",
         type=str,
@@ -525,18 +556,6 @@ def parse_args() -> argparse.Namespace:
         action="store_true",
         help="do not make any actual changes in the repo, just show what will be done",
     )
-    parser.add_argument(
-        "--outfile",
-        default="",
-        type=str,
-        help="output file to write json result to, if not set - stdout",
-    )
-    parser.add_argument(
-        "--infile",
-        default="",
-        type=str,
-        help="input file with release info",
-    )
 
     return parser.parse_args()
 
@@ -547,7 +566,7 @@ def checkout(ref: str) -> Iterator[None]:
     rollback_cmd = f"{GIT_PREFIX} checkout {orig_ref}"
     assert orig_ref
     if ref not in (orig_ref,):
-        ShellRunner.run(f"{GIT_PREFIX} checkout {ref}")
+        Shell.run(f"{GIT_PREFIX} checkout {ref}")
     try:
         yield
     except (Exception, KeyboardInterrupt) as e:
@@ -587,27 +606,21 @@ if __name__ == "__main__":
 
     if args.prepare_release_info:
         assert (
-            args.ref and args.release_type and args.outfile
-        ), "--ref, --release-type and --outfile must be provided with --prepare-release-info"
-        ReleaseInfo.prepare(
-            commit_ref=args.ref, release_type=args.release_type, outfile=args.outfile
-        )
+            args.ref and args.release_type
+        ), "--ref and --release-type must be provided with --prepare-release-info"
+        ReleaseInfo.prepare(commit_ref=args.ref, release_type=args.release_type)
     if args.push_release_tag:
-        assert args.infile, "--infile <release info file path> must be provided"
-        release_info = ReleaseInfo.from_file(args.infile)
+        release_info = ReleaseInfo.from_file()
         release_info.push_release_tag(dry_run=args.dry_run)
     if args.push_new_release_branch:
-        assert args.infile, "--infile <release info file path> must be provided"
-        release_info = ReleaseInfo.from_file(args.infile)
+        release_info = ReleaseInfo.from_file()
         release_info.push_new_release_branch(dry_run=args.dry_run)
     if args.create_bump_version_pr:
         # TODO: store link to PR in release info
-        assert args.infile, "--infile <release info file path> must be provided"
-        release_info = ReleaseInfo.from_file(args.infile)
+        release_info = ReleaseInfo.from_file()
         release_info.update_version_and_contributors_list(dry_run=args.dry_run)
     if args.download_packages:
-        assert args.infile, "--infile <release info file path> must be provided"
-        release_info = ReleaseInfo.from_file(args.infile)
+        release_info = ReleaseInfo.from_file()
         p = PackageDownloader(
             release=release_info.release_branch,
             commit_sha=release_info.commit_sha,
@@ -615,14 +628,23 @@ if __name__ == "__main__":
         )
         p.run()
     if args.create_gh_release:
-        assert args.infile, "--infile <release info file path> must be provided"
-        release_info = ReleaseInfo.from_file(args.infile)
+        release_info = ReleaseInfo.from_file()
         p = PackageDownloader(
             release=release_info.release_branch,
             commit_sha=release_info.commit_sha,
             version=release_info.version,
         )
-        release_info.create_gh_release(p.get_all_packages_files(), args.dry_run)
+    if args.post_status:
+        release_info = ReleaseInfo.from_file()
+        release_info.update_release_info(dry_run=args.dry_run)
+        if release_info.debian_command:
+            CIBuddy(dry_run=args.dry_run).post_done(
+                f"New release issued", dataclasses.asdict(release_info)
+            )
+        else:
+            CIBuddy(dry_run=args.dry_run).post_critical(
+                f"Failed to issue new release", dataclasses.asdict(release_info)
+            )
 
     # tear down ssh
     if _ssh_agent and _key_pub:
diff --git a/tests/ci/pr_info.py b/tests/ci/pr_info.py
index 9f3b5a586cc..59806a2a8fa 100644
--- a/tests/ci/pr_info.py
+++ b/tests/ci/pr_info.py
@@ -296,13 +296,16 @@ class PRInfo:
         else:
             if "schedule" in github_event:
                 self.event_type = EventType.SCHEDULE
-            else:
+            elif "inputs" in github_event:
                 # assume this is a dispatch
                 self.event_type = EventType.DISPATCH
-            logging.warning(
-                "event.json does not match pull_request or push:\n%s",
-                json.dumps(github_event, sort_keys=True, indent=4),
-            )
+                print("PR Info:")
+                print(self)
+            else:
+                logging.warning(
+                    "event.json does not match pull_request or push:\n%s",
+                    json.dumps(github_event, sort_keys=True, indent=4),
+                )
             self.sha = os.getenv(
                 "GITHUB_SHA", "0000000000000000000000000000000000000000"
             )

From 952ab302ce1fa5d9b739bbdf6acdf2fcdd208a04 Mon Sep 17 00:00:00 2001
From: Max K <maxkaynov@gmail.com>
Date: Fri, 19 Jul 2024 09:00:11 +0200
Subject: [PATCH 0505/1488] run test auto release by schedule

---
 .github/workflows/auto_release.yml | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/auto_release.yml b/.github/workflows/auto_release.yml
index e0a94d3bbb1..7588e7998bf 100644
--- a/.github/workflows/auto_release.yml
+++ b/.github/workflows/auto_release.yml
@@ -7,8 +7,10 @@ env:
 concurrency:
   group: release
 on: # yamllint disable-line rule:truthy
-  # schedule:
-  #   - cron: '0 10-16 * * 1-5'
+  # Workflow uses a test bucket for packages and dry run mode (no real releases)
+  schedule:
+    - cron: '0 9 * * *'
+    - cron: '0 15 * * *'
   workflow_dispatch:
     inputs:
       dry-run:
@@ -19,6 +21,8 @@ on: # yamllint disable-line rule:truthy
 
 jobs:
   AutoRelease:
+    env:
+      DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run != '' && github.event.inputs.dry_run || true }}
     runs-on: [self-hosted, release-maker]
     steps:
       - name: DebugInfo
@@ -57,7 +61,7 @@ jobs:
         with:
           ref: ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[0].commit_sha }}
           type: patch
-          dry-run: ${{ inputs.dry-run }}
+          dry-run: ${{ env.DRY_RUN }}
           token: ${{secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN}}
       - name: Release ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[1].release_branch }}
         if: ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[0] && fromJson(env.AUTO_RELEASE_PARAMS).releases[1].ready }}
@@ -65,7 +69,7 @@ jobs:
         with:
           ref: ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[1].commit_sha }}
           type: patch
-          dry-run: ${{ inputs.dry-run }}
+          dry-run: ${{ env.DRY_RUN }}
           token: ${{secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN}}
       - name: Release ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[2].release_branch }}
         if: ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[2] && fromJson(env.AUTO_RELEASE_PARAMS).releases[2].ready }}
@@ -73,7 +77,7 @@ jobs:
         with:
           ref: ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[2].commit_sha }}
           type: patch
-          dry-run: ${{ inputs.dry-run }}
+          dry-run: ${{ env.DRY_RUN }}
           token: ${{secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN}}
       - name: Release ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[3].release_branch }}
         if: ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[3] && fromJson(env.AUTO_RELEASE_PARAMS).releases[3].ready }}
@@ -81,7 +85,7 @@ jobs:
         with:
           ref: ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[3].commit_sha }}
           type: patch
-          dry-run: ${{ inputs.dry-run }}
+          dry-run: ${{ env.DRY_RUN }}
           token: ${{secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN}}
       - name: Release ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[4].release_branch }}
         if: ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[4] && fromJson(env.AUTO_RELEASE_PARAMS).releases[4].ready }}
@@ -89,7 +93,7 @@ jobs:
         with:
           ref: ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[4].commit_sha }}
           type: patch
-          dry-run: ${{ inputs.dry-run }}
+          dry-run: ${{ env.DRY_RUN }}
           token: ${{secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN}}
       - name: Post Slack Message
         if: ${{ !cancelled() }}

From 5d09f205e5bb5ab8aa860b14334eb5656b2c2a1b Mon Sep 17 00:00:00 2001
From: Max Kainov <max.kainov@clickhouse.com>
Date: Fri, 19 Jul 2024 09:35:43 +0000
Subject: [PATCH 0506/1488] style fixes

---
 .github/workflows/auto_release.yml   |  8 ++++++--
 .github/workflows/create_release.yml |  5 +----
 tests/ci/auto_release.py             | 11 ++++++-----
 tests/ci/ci_buddy.py                 | 18 +++++++++---------
 tests/ci/ci_utils.py                 | 11 +++++------
 5 files changed, 27 insertions(+), 26 deletions(-)

diff --git a/.github/workflows/auto_release.yml b/.github/workflows/auto_release.yml
index 7588e7998bf..f2cbf771190 100644
--- a/.github/workflows/auto_release.yml
+++ b/.github/workflows/auto_release.yml
@@ -21,8 +21,6 @@ on: # yamllint disable-line rule:truthy
 
 jobs:
   AutoRelease:
-    env:
-      DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.dry_run != '' && github.event.inputs.dry_run || true }}
     runs-on: [self-hosted, release-maker]
     steps:
       - name: DebugInfo
@@ -34,6 +32,12 @@ jobs:
           ${{secrets.ROBOT_CLICKHOUSE_SSH_KEY}}
           RCSK
           EOF
+      - name: Set DRY_RUN for schedule
+        if: ${{ github.event_name == 'schedule' }}
+        run: echo "DRY_RUN=true" >> "$GITHUB_ENV"
+      - name: Set DRY_RUN for dispatch
+        if: ${{ github.event_name == 'workflow_dispatch' }}
+        run: echo "DRY_RUN=${{ github.event.inputs.dry-run }}" >> "$GITHUB_ENV"
       - name: Check out repository code
         uses: ClickHouse/checkout@v1
         with:
diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml
index 6246306e536..6d914d1567e 100644
--- a/.github/workflows/create_release.yml
+++ b/.github/workflows/create_release.yml
@@ -2,8 +2,7 @@ name: CreateRelease
 
 concurrency:
   group: release
-
-on:
+'on':
   workflow_dispatch:
     inputs:
       ref:
@@ -30,10 +29,8 @@ jobs:
     runs-on: [self-hosted, release-maker]
     steps:
       - name: DebugInfo
-        if: ${{ ! inputs.autorelease }}
         uses: hmarr/debug-action@f7318c783045ac39ed9bb497e22ce835fdafbfe6
       - name: Check out repository code
-        if: ${{ ! inputs.autorelease }}
         uses: ClickHouse/checkout@v1
         with:
           token: ${{secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN}}
diff --git a/tests/ci/auto_release.py b/tests/ci/auto_release.py
index b8f781c4d73..5d98d8810a4 100644
--- a/tests/ci/auto_release.py
+++ b/tests/ci/auto_release.py
@@ -55,7 +55,7 @@ class ReleaseParams:
 class AutoReleaseInfo:
     releases: List[ReleaseParams]
 
-    def add_release(self, release_params: ReleaseParams):
+    def add_release(self, release_params: ReleaseParams) -> None:
         self.releases.append(release_params)
 
     def dump(self):
@@ -133,15 +133,16 @@ def _prepare(token):
             commit_sha = commit
             if commit_ci_status == SUCCESS:
                 break
-            else:
-                print(f"CI status [{commit_ci_status}] - skip")
+
+            print(f"CI status [{commit_ci_status}] - skip")
             commits_to_branch_head += 1
 
-        ready = commit_ci_status == SUCCESS and commit_sha
-        if ready:
+        ready = False
+        if commit_ci_status == SUCCESS and commit_sha:
             print(
                 f"Add release ready info for commit [{commit_sha}] and release branch [{pr.head.ref}]"
             )
+            ready = True
         else:
             print(f"WARNING: No ready commits found for release branch [{pr.head.ref}]")
 
diff --git a/tests/ci/ci_buddy.py b/tests/ci/ci_buddy.py
index 727a3d88359..ff8fdba2b6c 100644
--- a/tests/ci/ci_buddy.py
+++ b/tests/ci/ci_buddy.py
@@ -62,7 +62,7 @@ class CIBuddy:
             print(f"ERROR: Failed to post message, ex {e}")
 
     def _post_formatted(
-        self, title, body: Union[Dict, str], with_wf_link: bool
+        self, title: str, body: Union[Dict, str], with_wf_link: bool
     ) -> None:
         message = title
         if isinstance(body, dict):
@@ -80,36 +80,36 @@ class CIBuddy:
         self.post(message)
 
     def post_info(
-        self, title, body: Union[Dict, str], with_wf_link: bool = True
+        self, title: str, body: Union[Dict, str], with_wf_link: bool = True
     ) -> None:
         title_extended = f":white_circle:    *{title}*\n\n"
         self._post_formatted(title_extended, body, with_wf_link)
 
     def post_done(
-        self, title, body: Union[Dict, str], with_wf_link: bool = True
+        self, title: str, body: Union[Dict, str], with_wf_link: bool = True
     ) -> None:
         title_extended = f":white_check_mark:    *{title}*\n\n"
         self._post_formatted(title_extended, body, with_wf_link)
 
     def post_warning(
-        self, title, body: Union[Dict, str], with_wf_link: bool = True
+        self, title: str, body: Union[Dict, str], with_wf_link: bool = True
     ) -> None:
         title_extended = f":warning:    *{title}*\n\n"
         self._post_formatted(title_extended, body, with_wf_link)
 
     def post_critical(
-        self, title, body: Union[Dict, str], with_wf_link: bool = True
+        self, title: str, body: Union[Dict, str], with_wf_link: bool = True
     ) -> None:
         title_extended = f":black_circle:    *{title}*\n\n"
         self._post_formatted(title_extended, body, with_wf_link)
 
     def post_job_error(
         self,
-        error_description,
-        job_name="",
-        with_instance_info=True,
+        error_description: str,
+        job_name: str = "",
+        with_instance_info: bool = True,
         with_wf_link: bool = True,
-    ):
+    ) -> None:
         instance_id, instance_type = "unknown", "unknown"
         if with_instance_info:
             instance_id = Shell.run("ec2metadata --instance-id") or instance_id
diff --git a/tests/ci/ci_utils.py b/tests/ci/ci_utils.py
index 4536d1f2b54..eb25a53d492 100644
--- a/tests/ci/ci_utils.py
+++ b/tests/ci/ci_utils.py
@@ -90,7 +90,7 @@ class GHActions:
     @staticmethod
     def get_commit_status_by_name(
         token: str, commit_sha: str, status_name: Union[str, Sequence]
-    ) -> Optional[str]:
+    ) -> str:
         assert len(token) == 40
         assert len(commit_sha) == 40
         assert is_hex(commit_sha)
@@ -109,8 +109,8 @@ class GHActions:
             statuses = response.json()
             for status in statuses:
                 if status["context"] in status_name:
-                    return status["state"]
-        return None
+                    return status["state"]  # type: ignore
+        return ""
 
     @staticmethod
     def check_wf_completed(token: str, commit_sha: str) -> bool:
@@ -135,10 +135,9 @@ class GHActions:
                             f"   Check workflow status: Check not completed [{check['name']}]"
                         )
                         return False
-                else:
-                    return True
+                return True
             except Exception as e:
-                print(f"ERROR: exception {e}")
+                print(f"ERROR: exception after attempt [{i}]: {e}")
                 time.sleep(1)
 
         return False

From 3b842885779e123af1fd8aeaca4e7c131c5a33a9 Mon Sep 17 00:00:00 2001
From: Max K <maxkaynov@gmail.com>
Date: Fri, 19 Jul 2024 12:05:19 +0200
Subject: [PATCH 0507/1488] fix create release

---
 .github/workflows/create_release.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml
index 6d914d1567e..55644bdd503 100644
--- a/.github/workflows/create_release.yml
+++ b/.github/workflows/create_release.yml
@@ -39,6 +39,6 @@ jobs:
         uses: ./.github/actions/release
         with:
           ref: ${{ inputs.ref }}
-          type: inputs.type
+          type: ${{ inputs.type }}
           dry-run: ${{ inputs.dry-run }}
           token: ${{secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN}}

From a3a4548d96dac4b480e3a54519cfa34fab17ce4c Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitlibar@clickhouse.com>
Date: Sat, 20 Jul 2024 15:01:29 +0200
Subject: [PATCH 0508/1488] Fix removing files after restoring to
 s3_plain_rewritable.

---
 src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp
index e7c85bea1c6..b5805f6d23a 100644
--- a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp
+++ b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp
@@ -874,7 +874,9 @@ void DiskObjectStorageTransaction::writeFileUsingBlobWritingFunction(
     /// Create metadata (see create_metadata_callback in DiskObjectStorageTransaction::writeFile()).
     if (mode == WriteMode::Rewrite)
     {
-        if (!object_storage.isWriteOnce() && metadata_storage.exists(path))
+        /// Otherwise we will produce lost blobs which nobody points to
+        /// WriteOnce storages are not affected by the issue
+        if (!object_storage.isPlain() && metadata_storage.exists(path))
             object_storage.removeObjectsIfExist(metadata_storage.getStorageObjects(path));
 
         metadata_transaction->createMetadataFile(path, std::move(object_key), object_size);

From 3767f723489507dad7fe275c245253e7885aab8e Mon Sep 17 00:00:00 2001
From: Max K <maxkaynov@gmail.com>
Date: Fri, 19 Jul 2024 20:43:14 +0200
Subject: [PATCH 0509/1488] more info for buddy

---
 .github/actions/release/action.yml |  20 +-
 .github/workflows/auto_release.yml |   3 +-
 tests/ci/artifactory.py            |  86 ++++---
 tests/ci/auto_release.py           |  27 +++
 tests/ci/ci_buddy.py               |   4 +-
 tests/ci/ci_utils.py               |  23 +-
 tests/ci/create_release.py         | 348 ++++++++++++++++++-----------
 7 files changed, 338 insertions(+), 173 deletions(-)

diff --git a/.github/actions/release/action.yml b/.github/actions/release/action.yml
index fd4436f1f16..99ec02662f6 100644
--- a/.github/actions/release/action.yml
+++ b/.github/actions/release/action.yml
@@ -58,14 +58,11 @@ runs:
       shell: bash
       run: |
         python3 ./tests/ci/create_release.py --create-bump-version-pr ${{ inputs.dry-run && '--dry-run' || '' }}
-    - name: Checkout master
-      shell: bash
-      run: |
-        git checkout master
     - name: Bump Docker versions, Changelog, Security
       if: ${{ inputs.type == 'patch' }}
       shell: bash
       run: |
+        python3 ./tests/ci/create_release.py --set-progress-started --progress "update ChangeLog"
         [ "$(git branch --show-current)" != "master" ] && echo "not on the master" && exit 1
         echo "List versions"
         ./utils/list-versions/list-versions.sh > ./utils/list-versions/version_date.tsv
@@ -108,12 +105,13 @@ runs:
       shell: bash
       run: |
         git checkout "$GITHUB_REF_NAME"
+        # set current progress to OK
+        python3 ./tests/ci/create_release.py --set-progress-completed
     - name: Create GH Release
       shell: bash
       if: ${{ inputs.type == 'patch' }}
       run: |
-        python3 ./tests/ci/create_release.py --create-gh-release \
-          ${{ inputs.dry-run && '--dry-run' || '' }}
+        python3 ./tests/ci/create_release.py --create-gh-release ${{ inputs.dry-run && '--dry-run' || '' }}
     - name: Export TGZ Packages
       if: ${{ inputs.type == 'patch' }}
       shell: bash
@@ -148,16 +146,26 @@ runs:
       if: ${{ inputs.type == 'patch' }}
       shell: bash
       run: |
+        python3 ./tests/ci/create_release.py --set-progress-started --progress "docker server release"
         cd "./tests/ci"
         export CHECK_NAME="Docker server image"
         python3 docker_server.py --release-type auto --version ${{ env.RELEASE_TAG }} --check-name "$CHECK_NAME" --sha ${{ env.COMMIT_SHA }} ${{ ! inputs.dry-run && '--push' || '' }}
+        python3 ./tests/ci/create_release.py --set-progress-completed
     - name: Docker clickhouse/clickhouse-keeper building
       if: ${{ inputs.type == 'patch' }}
       shell: bash
       run: |
+        python3 ./tests/ci/create_release.py --set-progress-started --progress "docker keeper release"
         cd "./tests/ci"
         export CHECK_NAME="Docker keeper image"
         python3 docker_server.py --release-type auto --version ${{ env.RELEASE_TAG }} --check-name "$CHECK_NAME" --sha ${{ env.COMMIT_SHA }} ${{ ! inputs.dry-run && '--push'  || '' }}
+        python3 ./tests/ci/create_release.py --set-progress-completed
+    - name: Set Release progress completed
+      shell: bash
+      run: |
+        # If we here - set completed status, to post proper Slack OK or FAIL message in the next step
+        python3 ./tests/ci/create_release.py --set-progress-started --progress "completed"
+        python3 ./tests/ci/create_release.py --set-progress-completed
     - name: Post Slack Message
       if: ${{ !cancelled() }}
       shell: bash
diff --git a/.github/workflows/auto_release.yml b/.github/workflows/auto_release.yml
index f2cbf771190..457ffacc7a8 100644
--- a/.github/workflows/auto_release.yml
+++ b/.github/workflows/auto_release.yml
@@ -102,7 +102,8 @@ jobs:
       - name: Post Slack Message
         if: ${{ !cancelled() }}
         run: |
-          echo Slack Message
+          cd "$GITHUB_WORKSPACE/tests/ci"
+          python3 auto_release.py --post-auto-release-complete --wf-status ${{ job.status }}
       - name: Clean up
         run: |
           docker ps --quiet | xargs --no-run-if-empty docker kill ||:
diff --git a/tests/ci/artifactory.py b/tests/ci/artifactory.py
index 2009b122a18..98a0345c6bd 100644
--- a/tests/ci/artifactory.py
+++ b/tests/ci/artifactory.py
@@ -3,7 +3,12 @@ import time
 from pathlib import Path
 from typing import Optional
 from shutil import copy2
-from create_release import PackageDownloader, ReleaseInfo, ShellRunner
+from create_release import (
+    PackageDownloader,
+    ReleaseInfo,
+    ReleaseContextManager,
+    ReleaseProgress,
+)
 from ci_utils import WithIter, Shell
 
 
@@ -76,19 +81,20 @@ class R2MountPoint:
         )
 
         _TEST_MOUNT_CMD = f"mount | grep -q {self.MOUNT_POINT}"
-        ShellRunner.run(_CLEAN_LOG_FILE_CMD)
-        ShellRunner.run(_UNMOUNT_CMD)
-        ShellRunner.run(_MKDIR_CMD)
-        ShellRunner.run(_MKDIR_FOR_CACHE)
-        ShellRunner.run(self.mount_cmd, async_=self.async_mount)
+        Shell.run(_CLEAN_LOG_FILE_CMD)
+        Shell.run(_UNMOUNT_CMD)
+        Shell.run(_MKDIR_CMD)
+        Shell.run(_MKDIR_FOR_CACHE)
+        # didn't manage to use simple run() and not block or fail
+        Shell.run_as_daemon(self.mount_cmd)
         if self.async_mount:
             time.sleep(3)
-        ShellRunner.run(_TEST_MOUNT_CMD)
+        Shell.run(_TEST_MOUNT_CMD, check=True)
 
     @classmethod
     def teardown(cls):
         print(f"Unmount [{cls.MOUNT_POINT}]")
-        ShellRunner.run(f"umount {cls.MOUNT_POINT}")
+        Shell.run(f"umount {cls.MOUNT_POINT}")
 
 
 class RepoCodenames(metaclass=WithIter):
@@ -124,8 +130,8 @@ class DebianArtifactory:
         cmd = f"{REPREPRO_CMD_PREFIX} includedeb {self.codename} {' '.join(paths)}"
         print("Running export command:")
         print(f"  {cmd}")
-        ShellRunner.run(cmd)
-        ShellRunner.run("sync")
+        Shell.run(cmd, check=True)
+        Shell.run("sync")
 
         if self.codename == RepoCodenames.LTS:
             packages_with_version = [
@@ -137,8 +143,8 @@ class DebianArtifactory:
             cmd = f"{REPREPRO_CMD_PREFIX} copy {RepoCodenames.STABLE} {RepoCodenames.LTS} {' '.join(packages_with_version)}"
             print("Running copy command:")
             print(f"  {cmd}")
-            ShellRunner.run(cmd)
-            ShellRunner.run("sync")
+            Shell.run(cmd, check=True)
+            Shell.run("sync")
 
     def test_packages(self):
         Shell.run("docker pull ubuntu:latest")
@@ -206,12 +212,12 @@ class RpmArtifactory:
         for command in commands:
             print("Running command:")
             print(f"    {command}")
-            ShellRunner.run(command)
+            Shell.run(command, check=True)
 
         update_public_key = f"gpg --armor --export {self._SIGN_KEY}"
         pub_key_path = dest_dir / "repodata" / "repomd.xml.key"
         print("Updating repomd.xml.key")
-        pub_key_path.write_text(ShellRunner.run(update_public_key)[1])
+        pub_key_path.write_text(Shell.run(update_public_key, check=True))
         if codename == RepoCodenames.LTS:
             self.export_packages(RepoCodenames.STABLE)
         Shell.run("sync")
@@ -264,23 +270,29 @@ class TgzArtifactory:
 
         if codename == RepoCodenames.LTS:
             self.export_packages(RepoCodenames.STABLE)
-        ShellRunner.run("sync")
+        Shell.run("sync")
 
     def test_packages(self):
         tgz_file = "/tmp/tmp.tgz"
         tgz_sha_file = "/tmp/tmp.tgz.sha512"
-        ShellRunner.run(
-            f"curl -o {tgz_file} -f0 {self.repo_url}/stable/clickhouse-client-{self.version}-arm64.tgz"
+        cmd = f"curl -o {tgz_file} -f0 {self.repo_url}/stable/clickhouse-client-{self.version}-arm64.tgz"
+        Shell.run(
+            cmd,
+            check=True,
         )
-        ShellRunner.run(
-            f"curl -o {tgz_sha_file} -f0 {self.repo_url}/stable/clickhouse-client-{self.version}-arm64.tgz.sha512"
+        Shell.run(
+            f"curl -o {tgz_sha_file} -f0 {self.repo_url}/stable/clickhouse-client-{self.version}-arm64.tgz.sha512",
+            check=True,
         )
-        expected_checksum = ShellRunner.run(f"cut -d ' ' -f 1 {tgz_sha_file}")
-        actual_checksum = ShellRunner.run(f"sha512sum {tgz_file} | cut -d ' ' -f 1")
+        expected_checksum = Shell.run(f"cut -d ' ' -f 1 {tgz_sha_file}", check=True)
+        actual_checksum = Shell.run(f"sha512sum {tgz_file} | cut -d ' ' -f 1")
         assert (
             expected_checksum == actual_checksum
         ), f"[{actual_checksum} != {expected_checksum}]"
-        ShellRunner.run("rm /tmp/tmp.tgz*")
+        Shell.run("rm /tmp/tmp.tgz*")
+        release_info = ReleaseInfo.from_file()
+        release_info.tgz_command = cmd
+        release_info.dump()
 
 
 def parse_args() -> argparse.Namespace:
@@ -338,20 +350,26 @@ if __name__ == "__main__":
     """
     mp = R2MountPoint(MountPointApp.S3FS, dry_run=args.dry_run)
     if args.export_debian:
-        mp.init()
-        DebianArtifactory(release_info, dry_run=args.dry_run).export_packages()
-        mp.teardown()
+        with ReleaseContextManager(release_progress=ReleaseProgress.EXPORT_DEB) as _:
+            mp.init()
+            DebianArtifactory(release_info, dry_run=args.dry_run).export_packages()
+            mp.teardown()
     if args.export_rpm:
-        mp.init()
-        RpmArtifactory(release_info, dry_run=args.dry_run).export_packages()
-        mp.teardown()
+        with ReleaseContextManager(release_progress=ReleaseProgress.EXPORT_RPM) as _:
+            mp.init()
+            RpmArtifactory(release_info, dry_run=args.dry_run).export_packages()
+            mp.teardown()
     if args.export_tgz:
-        mp.init()
-        TgzArtifactory(release_info, dry_run=args.dry_run).export_packages()
-        mp.teardown()
+        with ReleaseContextManager(release_progress=ReleaseProgress.EXPORT_TGZ) as _:
+            mp.init()
+            TgzArtifactory(release_info, dry_run=args.dry_run).export_packages()
+            mp.teardown()
     if args.test_debian:
-        DebianArtifactory(release_info, dry_run=args.dry_run).test_packages()
+        with ReleaseContextManager(release_progress=ReleaseProgress.TEST_DEB) as _:
+            DebianArtifactory(release_info, dry_run=args.dry_run).test_packages()
     if args.test_tgz:
-        TgzArtifactory(release_info, dry_run=args.dry_run).test_packages()
+        with ReleaseContextManager(release_progress=ReleaseProgress.TEST_TGZ) as _:
+            TgzArtifactory(release_info, dry_run=args.dry_run).test_packages()
     if args.test_rpm:
-        RpmArtifactory(release_info, dry_run=args.dry_run).test_packages()
+        with ReleaseContextManager(release_progress=ReleaseProgress.TEST_RPM) as _:
+            RpmArtifactory(release_info, dry_run=args.dry_run).test_packages()
diff --git a/tests/ci/auto_release.py b/tests/ci/auto_release.py
index 5d98d8810a4..39ab3156c80 100644
--- a/tests/ci/auto_release.py
+++ b/tests/ci/auto_release.py
@@ -25,11 +25,22 @@ def parse_args():
         action="store_true",
         help="Post release branch statuses",
     )
+    parser.add_argument(
+        "--post-auto-release-complete",
+        action="store_true",
+        help="Post autorelease completion status",
+    )
     parser.add_argument(
         "--prepare",
         action="store_true",
         help="Prepare autorelease info",
     )
+    parser.add_argument(
+        "--wf-status",
+        type=str,
+        default="",
+        help="overall workflow status [success|failure]",
+    )
     return parser.parse_args(), parser
 
 
@@ -180,6 +191,22 @@ def main():
                     title=f"Auto Release Status for {release_info.release_branch}",
                     body=release_info.to_dict(),
                 )
+    if args.post_auto_release_complete:
+        assert args.wf_status, "--wf-status Required with --post-auto-release-complete"
+        if args.wf_status != SUCCESS:
+            CIBuddy(dry_run=False).post_job_error(
+                error_description="Autorelease workflow failed",
+                job_name="Autorelease",
+                with_instance_info=False,
+                with_wf_link=True,
+                critical=True,
+            )
+        else:
+            CIBuddy(dry_run=False).post_info(
+                title=f"Autorelease completed",
+                body="",
+                with_wf_link=True,
+            )
     elif args.prepare:
         _prepare(token=args.token or get_best_robot_token())
     else:
diff --git a/tests/ci/ci_buddy.py b/tests/ci/ci_buddy.py
index ff8fdba2b6c..3eba5532e66 100644
--- a/tests/ci/ci_buddy.py
+++ b/tests/ci/ci_buddy.py
@@ -109,6 +109,7 @@ class CIBuddy:
         job_name: str = "",
         with_instance_info: bool = True,
         with_wf_link: bool = True,
+        critical: bool = False,
     ) -> None:
         instance_id, instance_type = "unknown", "unknown"
         if with_instance_info:
@@ -116,7 +117,8 @@ class CIBuddy:
             instance_type = Shell.run("ec2metadata --instance-type") or instance_type
         if not job_name:
             job_name = os.getenv("CHECK_NAME", "unknown")
-        line_err = f":red_circle:    *Error: {error_description}*\n\n"
+        sign = ":red_circle:" if not critical else ":black_circle:"
+        line_err = f"{sign}    *Error: {error_description}*\n\n"
         line_ghr = f"   *Runner:*    `{instance_type}`, `{instance_id}`\n"
         line_job = f"   *Job:*          `{job_name}`\n"
         line_pr_ = f"   *PR:*           <https://github.com/{self.repo}/pull/{self.pr_number}|#{self.pr_number}>, <{self.commit_url}|{self.sha}>\n"
diff --git a/tests/ci/ci_utils.py b/tests/ci/ci_utils.py
index eb25a53d492..efbf014cd52 100644
--- a/tests/ci/ci_utils.py
+++ b/tests/ci/ci_utils.py
@@ -4,7 +4,7 @@ import subprocess
 import time
 from contextlib import contextmanager
 from pathlib import Path
-from typing import Any, Iterator, List, Union, Optional, Sequence
+from typing import Any, Iterator, List, Union, Optional, Sequence, Tuple
 
 import requests
 
@@ -142,6 +142,16 @@ class GHActions:
 
         return False
 
+    @staticmethod
+    def get_pr_url_by_branch(repo, branch):
+        get_url_cmd = (
+            f"gh pr list --repo {repo} --head {branch} --json url --jq '.[0].url'"
+        )
+        url = Shell.run(get_url_cmd)
+        if not url:
+            print(f"ERROR: PR nor found, branch [{branch}]")
+        return url
+
 
 class Shell:
     @classmethod
@@ -157,7 +167,10 @@ class Shell:
         return res.stdout.strip()
 
     @classmethod
-    def run(cls, command, check=False):
+    def run(cls, command, check=False, dry_run=False):
+        if dry_run:
+            print(f"Dry-ryn. Would run command [{command}]")
+            return ""
         print(f"Run command [{command}]")
         res = ""
         result = subprocess.run(
@@ -178,6 +191,12 @@ class Shell:
                 assert result.returncode == 0
         return res.strip()
 
+    @classmethod
+    def run_as_daemon(cls, command):
+        print(f"Run daemon command [{command}]")
+        subprocess.Popen(command.split(" "))  # pylint:disable=consider-using-with
+        return 0, ""
+
     @classmethod
     def check(cls, command):
         result = subprocess.run(
diff --git a/tests/ci/create_release.py b/tests/ci/create_release.py
index 414ec8afd3e..4347cfebb54 100755
--- a/tests/ci/create_release.py
+++ b/tests/ci/create_release.py
@@ -2,7 +2,6 @@ import argparse
 import dataclasses
 import json
 import os
-import subprocess
 
 from contextlib import contextmanager
 from copy import copy
@@ -13,7 +12,7 @@ from git_helper import Git, GIT_PREFIX
 from ssh import SSHAgent
 from env_helper import GITHUB_REPOSITORY, S3_BUILDS_BUCKET
 from s3_helper import S3Helper
-from ci_utils import Shell
+from ci_utils import Shell, GHActions
 from ci_buddy import CIBuddy
 from version_helper import (
     FILE_WITH_VERSION_PATH,
@@ -31,32 +30,62 @@ CONTRIBUTORS_PATH = get_abs_path(GENERATED_CONTRIBUTORS)
 RELEASE_INFO_FILE = "/tmp/release_info.json"
 
 
-class ShellRunner:
+class ReleaseProgress:
+    STARTED = "started"
+    DOWNLOAD_PACKAGES = "download packages"
+    PUSH_RELEASE_TAG = "push release tag"
+    PUSH_NEW_RELEASE_BRANCH = "push new release branch"
+    BUMP_VERSION = "bump version"
+    CREATE_GH_RELEASE = "create GH release"
+    EXPORT_TGZ = "export TGZ packages"
+    EXPORT_RPM = "export RPM packages"
+    EXPORT_DEB = "export DEB packages"
+    TEST_TGZ = "test TGZ packages"
+    TEST_RPM = "test RPM packages"
+    TEST_DEB = "test DEB packages"
 
-    @classmethod
-    def run(
-        cls, command, check_retcode=True, print_output=True, async_=False, dry_run=False
-    ):
-        if dry_run:
-            print(f"Dry-run: Would run shell command: [{command}]")
-            return 0, ""
-        print(f"Running shell command: [{command}]")
-        if async_:
-            subprocess.Popen(command.split(" "))  # pylint:disable=consider-using-with
-            return 0, ""
-        result = subprocess.run(
-            command + " 2>&1",
-            shell=True,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
-            text=True,
-            check=True,
-        )
-        if print_output:
-            print(result.stdout)
-        if check_retcode:
-            assert result.returncode == 0, f"Return code [{result.returncode}]"
-        return result.returncode, result.stdout
+
+class ReleaseProgressDescription:
+    OK = "OK"
+    FAILED = "FAILED"
+
+
+class ReleaseContextManager:
+    def __init__(self, release_progress):
+        self.release_progress = release_progress
+        self.release_info = None
+
+    def __enter__(self):
+        if self.release_progress == ReleaseProgress.STARTED:
+            # create initial release info
+            self.release_info = ReleaseInfo(
+                release_branch="NA",
+                commit_sha=args.ref,
+                release_tag="NA",
+                version="NA",
+                codename="NA",
+                previous_release_tag="NA",
+                previous_release_sha="NA",
+                release_progress=ReleaseProgress.STARTED,
+            ).dump()
+        else:
+            # fetch release info from fs and update
+            self.release_info = ReleaseInfo.from_file()
+            assert self.release_info
+            assert (
+                self.release_info.progress_description == ReleaseProgressDescription.OK
+            ), "Must be OK on the start of new context"
+            self.release_info.release_progress = self.release_progress
+            self.release_info.dump()
+        return self.release_info
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        assert self.release_info
+        if exc_type is not None:
+            self.release_info.progress_description = ReleaseProgressDescription.FAILED
+        else:
+            self.release_info.progress_description = ReleaseProgressDescription.OK
+        self.release_info.dump()
 
 
 @dataclasses.dataclass
@@ -74,6 +103,10 @@ class ReleaseInfo:
     release_url: str = ""
     debian_command: str = ""
     rpm_command: str = ""
+    tgz_command: str = ""
+    docker_command: str = ""
+    release_progress: str = ""
+    progress_description: str = ""
 
     @staticmethod
     def from_file() -> "ReleaseInfo":
@@ -85,9 +118,9 @@ class ReleaseInfo:
         print(f"Dump release info into [{RELEASE_INFO_FILE}]")
         with open(RELEASE_INFO_FILE, "w", encoding="utf-8") as f:
             print(json.dumps(dataclasses.asdict(self), indent=2), file=f)
+        return self
 
-    @staticmethod
-    def prepare(commit_ref: str, release_type: str) -> None:
+    def prepare(self, commit_ref: str, release_type: str) -> "ReleaseInfo":
         version = None
         release_branch = None
         release_tag = None
@@ -97,8 +130,9 @@ class ReleaseInfo:
         assert release_type in ("patch", "new")
         if release_type == "new":
             # check commit_ref is right and on a right branch
-            ShellRunner.run(
-                f"git merge-base --is-ancestor origin/{commit_ref} origin/master"
+            Shell.run(
+                f"git merge-base --is-ancestor origin/{commit_ref} origin/master",
+                check=True,
             )
             with checkout(commit_ref):
                 commit_sha = Shell.run(f"git rev-parse {commit_ref}", check=True)
@@ -130,10 +164,11 @@ class ReleaseInfo:
                 version.with_description(codename)
                 release_branch = f"{version.major}.{version.minor}"
                 release_tag = version.describe
-            ShellRunner.run(f"{GIT_PREFIX} fetch origin {release_branch} --tags")
+            Shell.run(f"{GIT_PREFIX} fetch origin {release_branch} --tags", check=True)
             # check commit is right and on a right branch
-            ShellRunner.run(
-                f"git merge-base --is-ancestor {commit_ref} origin/{release_branch}"
+            Shell.run(
+                f"git merge-base --is-ancestor {commit_ref} origin/{release_branch}",
+                check=True,
             )
             if version.patch == 1:
                 expected_version = copy(version)
@@ -172,21 +207,22 @@ class ReleaseInfo:
             and version
             and codename in ("lts", "stable")
         )
-        res = ReleaseInfo(
-            release_branch=release_branch,
-            commit_sha=commit_sha,
-            release_tag=release_tag,
-            version=version.string,
-            codename=codename,
-            previous_release_tag=previous_release_tag,
-            previous_release_sha=previous_release_sha,
-        )
-        res.dump()
+
+        self.release_branch = release_branch
+        self.commit_sha = commit_sha
+        self.release_tag = release_tag
+        self.version = version.string
+        self.codename = codename
+        self.previous_release_tag = previous_release_tag
+        self.previous_release_sha = previous_release_sha
+        self.release_progress = ReleaseProgress.STARTED
+        self.progress_description = ReleaseProgressDescription.OK
+        return self
 
     def push_release_tag(self, dry_run: bool) -> None:
         if dry_run:
             # remove locally created tag from prev run
-            ShellRunner.run(
+            Shell.run(
                 f"{GIT_PREFIX} tag -l | grep -q {self.release_tag} && git tag -d {self.release_tag} ||:"
             )
         # Create release tag
@@ -194,16 +230,17 @@ class ReleaseInfo:
             f"Create and push release tag [{self.release_tag}], commit [{self.commit_sha}]"
         )
         tag_message = f"Release {self.release_tag}"
-        ShellRunner.run(
-            f"{GIT_PREFIX} tag -a -m '{tag_message}' {self.release_tag} {self.commit_sha}"
+        Shell.run(
+            f"{GIT_PREFIX} tag -a -m '{tag_message}' {self.release_tag} {self.commit_sha}",
+            check=True,
         )
         cmd_push_tag = f"{GIT_PREFIX} push origin {self.release_tag}:{self.release_tag}"
-        ShellRunner.run(cmd_push_tag, dry_run=dry_run)
+        Shell.run(cmd_push_tag, dry_run=dry_run, check=True)
 
     @staticmethod
     def _create_gh_label(label: str, color_hex: str, dry_run: bool) -> None:
         cmd = f"gh api repos/{GITHUB_REPOSITORY}/labels -f name={label} -f color={color_hex}"
-        ShellRunner.run(cmd, dry_run=dry_run)
+        Shell.run(cmd, dry_run=dry_run, check=True)
 
     def push_new_release_branch(self, dry_run: bool) -> None:
         assert (
@@ -220,8 +257,8 @@ class ReleaseInfo:
         ), f"Unexpected current version in git, must precede [{self.version}] by one step, actual [{version.string}]"
         if dry_run:
             # remove locally created branch from prev run
-            ShellRunner.run(
-                f"{GIT_PREFIX} branch -l | grep -q {new_release_branch} && git branch -d {new_release_branch} ||:"
+            Shell.run(
+                f"{GIT_PREFIX} branch -l | grep -q {new_release_branch} && git branch -d {new_release_branch}"
             )
         print(
             f"Create and push new release branch [{new_release_branch}], commit [{self.commit_sha}]"
@@ -234,7 +271,7 @@ class ReleaseInfo:
                 cmd_push_branch = (
                     f"{GIT_PREFIX} push --set-upstream origin {new_release_branch}"
                 )
-                ShellRunner.run(cmd_push_branch, dry_run=dry_run)
+                Shell.run(cmd_push_branch, dry_run=dry_run, check=True)
 
         print("Create and push backport tags for new release branch")
         ReleaseInfo._create_gh_label(
@@ -243,12 +280,13 @@ class ReleaseInfo:
         ReleaseInfo._create_gh_label(
             f"v{new_release_branch}-affected", "c2bfff", dry_run=dry_run
         )
-        ShellRunner.run(
+        Shell.run(
             f"""gh pr create --repo {GITHUB_REPOSITORY} --title 'Release pull request for branch {new_release_branch}'
             --head {new_release_branch} {pr_labels}
             --body 'This PullRequest is a part of ClickHouse release cycle. It is used by CI system only. Do not perform any changes with it.'
             """,
             dry_run=dry_run,
+            check=True,
         )
 
     def update_version_and_contributors_list(self, dry_run: bool) -> None:
@@ -274,31 +312,34 @@ class ReleaseInfo:
                 body_file = get_abs_path(".github/PULL_REQUEST_TEMPLATE.md")
                 actor = os.getenv("GITHUB_ACTOR", "") or "me"
                 cmd_create_pr = f"gh pr create --repo {GITHUB_REPOSITORY} --title 'Update version after release' --head {branch_upd_version_contributors} --base {self.release_branch} --body-file '{body_file} --label 'do not test' --assignee @{actor}"
-                ShellRunner.run(cmd_commit_version_upd, dry_run=dry_run)
-                ShellRunner.run(cmd_push_branch, dry_run=dry_run)
-                ShellRunner.run(cmd_create_pr, dry_run=dry_run)
+                Shell.run(cmd_commit_version_upd, check=True, dry_run=dry_run)
+                Shell.run(cmd_push_branch, check=True, dry_run=dry_run)
+                Shell.run(cmd_create_pr, check=True, dry_run=dry_run)
                 if dry_run:
-                    ShellRunner.run(
-                        f"{GIT_PREFIX} diff '{CMAKE_PATH}' '{CONTRIBUTORS_PATH}'"
-                    )
-                    ShellRunner.run(
+                    Shell.run(f"{GIT_PREFIX} diff '{CMAKE_PATH}' '{CONTRIBUTORS_PATH}'")
+                    Shell.run(
                         f"{GIT_PREFIX} checkout '{CMAKE_PATH}' '{CONTRIBUTORS_PATH}'"
                     )
+                self.version_bump_pr = GHActions.get_pr_url_by_branch(
+                    repo=GITHUB_REPOSITORY, branch=branch_upd_version_contributors
+                )
 
-    def update_release_info(self, dry_run: bool) -> None:
+    def update_release_info(self, dry_run: bool) -> "ReleaseInfo":
         branch = f"auto/{release_info.release_tag}"
         if not dry_run:
-            get_url_cmd = f"gh pr list --repo {GITHUB_REPOSITORY} --head {branch} --json url --jq '.[0].url'"
-            url = Shell.run(get_url_cmd)
-            if url:
-                print(f"Update release info with Changelog PR link [{url}]")
-            else:
-                print(f"WARNING: Changelog PR not found, branch [{branch}]")
+            url = GHActions.get_pr_url_by_branch(repo=GITHUB_REPOSITORY, branch=branch)
         else:
             url = "dry-run"
 
+        print(f"ChangeLog PR url [{url}]")
         self.changelog_pr = url
+        print(f"Release url [{url}]")
+        self.release_url = (
+            f"https://github.com/{GITHUB_REPOSITORY}/releases/tag/{self.release_tag}"
+        )
+        self.docker_command = f"docker run --rm clickhouse/clickhouse:{self.release_branch} clickhouse --version"
         self.dump()
+        return self
 
     def create_gh_release(self, packages_files: List[str], dry_run: bool) -> None:
         repo = os.getenv("GITHUB_REPOSITORY")
@@ -376,7 +417,7 @@ class PackageDownloader:
         self.macos_package_files = ["clickhouse-macos", "clickhouse-macos-aarch64"]
         self.file_to_type = {}
 
-        ShellRunner.run(f"mkdir -p {self.LOCAL_DIR}")
+        Shell.run(f"mkdir -p {self.LOCAL_DIR}")
 
         for package_type in self.PACKAGE_TYPES:
             for package in self.package_names:
@@ -426,7 +467,7 @@ class PackageDownloader:
         return res
 
     def run(self):
-        ShellRunner.run(f"rm -rf {self.LOCAL_DIR}/*")
+        Shell.run(f"rm -rf {self.LOCAL_DIR}/*")
         for package_file in (
             self.deb_package_files + self.rpm_package_files + self.tgz_package_files
         ):
@@ -499,6 +540,37 @@ class PackageDownloader:
         return True
 
 
+@contextmanager
+def checkout(ref: str) -> Iterator[None]:
+    orig_ref = Shell.run(f"{GIT_PREFIX} symbolic-ref --short HEAD", check=True)
+    rollback_cmd = f"{GIT_PREFIX} checkout {orig_ref}"
+    assert orig_ref
+    if ref not in (orig_ref,):
+        Shell.run(f"{GIT_PREFIX} checkout {ref}")
+    try:
+        yield
+    except (Exception, KeyboardInterrupt) as e:
+        print(f"ERROR: Exception [{e}]")
+        Shell.run(rollback_cmd)
+        raise
+    Shell.run(rollback_cmd)
+
+
+@contextmanager
+def checkout_new(ref: str) -> Iterator[None]:
+    orig_ref = Shell.run(f"{GIT_PREFIX} symbolic-ref --short HEAD", check=True)
+    rollback_cmd = f"{GIT_PREFIX} checkout {orig_ref}"
+    assert orig_ref
+    Shell.run(f"{GIT_PREFIX} checkout -b {ref}", check=True)
+    try:
+        yield
+    except (Exception, KeyboardInterrupt) as e:
+        print(f"ERROR: Exception [{e}]")
+        Shell.run(rollback_cmd)
+        raise
+    Shell.run(rollback_cmd)
+
+
 def parse_args() -> argparse.Namespace:
     parser = argparse.ArgumentParser(
         formatter_class=argparse.ArgumentDefaultsHelpFormatter,
@@ -556,44 +628,26 @@ def parse_args() -> argparse.Namespace:
         action="store_true",
         help="do not make any actual changes in the repo, just show what will be done",
     )
-
+    parser.add_argument(
+        "--set-progress-started",
+        action="store_true",
+        help="Set new progress step, --progress <PROGRESS STEP> must be set",
+    )
+    parser.add_argument(
+        "--progress",
+        type=str,
+        help="Progress step name, see @ReleaseProgress",
+    )
+    parser.add_argument(
+        "--set-progress-completed",
+        action="store_true",
+        help="Set current progress step to OK (completed)",
+    )
     return parser.parse_args()
 
 
-@contextmanager
-def checkout(ref: str) -> Iterator[None]:
-    _, orig_ref = ShellRunner.run(f"{GIT_PREFIX} symbolic-ref --short HEAD")
-    rollback_cmd = f"{GIT_PREFIX} checkout {orig_ref}"
-    assert orig_ref
-    if ref not in (orig_ref,):
-        Shell.run(f"{GIT_PREFIX} checkout {ref}")
-    try:
-        yield
-    except (Exception, KeyboardInterrupt) as e:
-        print(f"ERROR: Exception [{e}]")
-        ShellRunner.run(rollback_cmd)
-        raise
-    ShellRunner.run(rollback_cmd)
-
-
-@contextmanager
-def checkout_new(ref: str) -> Iterator[None]:
-    _, orig_ref = ShellRunner.run(f"{GIT_PREFIX} symbolic-ref --short HEAD")
-    rollback_cmd = f"{GIT_PREFIX} checkout {orig_ref}"
-    assert orig_ref
-    ShellRunner.run(f"{GIT_PREFIX} checkout -b {ref}")
-    try:
-        yield
-    except (Exception, KeyboardInterrupt) as e:
-        print(f"ERROR: Exception [{e}]")
-        ShellRunner.run(rollback_cmd)
-        raise
-    ShellRunner.run(rollback_cmd)
-
-
 if __name__ == "__main__":
     args = parse_args()
-    assert args.dry_run
 
     # prepare ssh for git if needed
     _ssh_agent = None
@@ -605,35 +659,56 @@ if __name__ == "__main__":
         _ssh_agent.print_keys()
 
     if args.prepare_release_info:
-        assert (
-            args.ref and args.release_type
-        ), "--ref and --release-type must be provided with --prepare-release-info"
-        ReleaseInfo.prepare(commit_ref=args.ref, release_type=args.release_type)
-    if args.push_release_tag:
-        release_info = ReleaseInfo.from_file()
-        release_info.push_release_tag(dry_run=args.dry_run)
-    if args.push_new_release_branch:
-        release_info = ReleaseInfo.from_file()
-        release_info.push_new_release_branch(dry_run=args.dry_run)
-    if args.create_bump_version_pr:
-        # TODO: store link to PR in release info
-        release_info = ReleaseInfo.from_file()
-        release_info.update_version_and_contributors_list(dry_run=args.dry_run)
+        with ReleaseContextManager(
+            release_progress=ReleaseProgress.STARTED
+        ) as release_info:
+            assert (
+                args.ref and args.release_type
+            ), "--ref and --release-type must be provided with --prepare-release-info"
+            release_info.prepare(commit_ref=args.ref, release_type=args.release_type)
+
     if args.download_packages:
-        release_info = ReleaseInfo.from_file()
-        p = PackageDownloader(
-            release=release_info.release_branch,
-            commit_sha=release_info.commit_sha,
-            version=release_info.version,
-        )
-        p.run()
+        with ReleaseContextManager(
+            release_progress=ReleaseProgress.DOWNLOAD_PACKAGES
+        ) as release_info:
+            p = PackageDownloader(
+                release=release_info.release_branch,
+                commit_sha=release_info.commit_sha,
+                version=release_info.version,
+            )
+            p.run()
+
+    if args.push_release_tag:
+        with ReleaseContextManager(
+            release_progress=ReleaseProgress.PUSH_RELEASE_TAG
+        ) as release_info:
+            release_info.push_release_tag(dry_run=args.dry_run)
+
+    if args.push_new_release_branch:
+        with ReleaseContextManager(
+            release_progress=ReleaseProgress.PUSH_NEW_RELEASE_BRANCH
+        ) as release_info:
+            release_info.push_new_release_branch(dry_run=args.dry_run)
+
+    if args.create_bump_version_pr:
+        with ReleaseContextManager(
+            release_progress=ReleaseProgress.BUMP_VERSION
+        ) as release_info:
+            release_info.update_version_and_contributors_list(dry_run=args.dry_run)
+
     if args.create_gh_release:
-        release_info = ReleaseInfo.from_file()
-        p = PackageDownloader(
-            release=release_info.release_branch,
-            commit_sha=release_info.commit_sha,
-            version=release_info.version,
-        )
+        with ReleaseContextManager(
+            release_progress=ReleaseProgress.CREATE_GH_RELEASE
+        ) as release_info:
+            p = PackageDownloader(
+                release=release_info.release_branch,
+                commit_sha=release_info.commit_sha,
+                version=release_info.version,
+            )
+            release_info.create_gh_release(
+                packages_files=p.get_all_packages_files(), dry_run=args.dry_run
+            )
+
     if args.post_status:
         release_info = ReleaseInfo.from_file()
         release_info.update_release_info(dry_run=args.dry_run)
@@ -646,6 +721,21 @@ if __name__ == "__main__":
                 f"Failed to issue new release", dataclasses.asdict(release_info)
             )
 
+    if args.set_progress_started:
+        ri = ReleaseInfo.from_file()
+        ri.release_progress = args.progress
+        ri.progress_description = ReleaseProgressDescription.FAILED
+        ri.dump()
+        assert args.progress, "Progress step name must be provided"
+
+    if args.set_progress_completed:
+        ri = ReleaseInfo.from_file()
+        assert (
+            ri.progress_description == ReleaseProgressDescription.FAILED
+        ), "Must be FAILED before set to OK"
+        ri.progress_description = ReleaseProgressDescription.OK
+        ri.dump()
+
     # tear down ssh
     if _ssh_agent and _key_pub:
         _ssh_agent.remove(_key_pub)

From c786e6eb49152d7992ee3ee5c76300b7e4712309 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 20 Jul 2024 21:27:35 +0200
Subject: [PATCH 0510/1488] Fix tests

---
 tests/queries/0_stateless/01338_long_select_and_alter.reference | 2 +-
 .../0_stateless/01338_long_select_and_alter_zookeeper.reference | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/queries/0_stateless/01338_long_select_and_alter.reference b/tests/queries/0_stateless/01338_long_select_and_alter.reference
index 276d6bcc29d..027109252e1 100644
--- a/tests/queries/0_stateless/01338_long_select_and_alter.reference
+++ b/tests/queries/0_stateless/01338_long_select_and_alter.reference
@@ -1,3 +1,3 @@
 5
 5
-CREATE TABLE default.alter_mt\n(\n    `key` UInt64,\n    `value` UInt64\n)\nENGINE = MergeTree\nORDER BY key\nSETTINGS index_granularity = 8192
+CREATE TABLE default.alter_mt\n(\n    `key` Int64,\n    `value` Int64\n)\nENGINE = MergeTree\nORDER BY key\nSETTINGS index_granularity = 8192
diff --git a/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.reference b/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.reference
index aab1b93f6bd..65e638bc3a4 100644
--- a/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.reference
+++ b/tests/queries/0_stateless/01338_long_select_and_alter_zookeeper.reference
@@ -1,3 +1,3 @@
 5
 5
-CREATE TABLE default.alter_mt\n(\n    `key` UInt64,\n    `value` UInt64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/01338_long_select_and_alter_zookeeper_default/alter_mt\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192
+CREATE TABLE default.alter_mt\n(\n    `key` Int64,\n    `value` Int64\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/01338_long_select_and_alter_zookeeper_default/alter_mt\', \'1\')\nORDER BY key\nSETTINGS index_granularity = 8192

From ee3c0e7e1f37a4e9388866145e9cbb8f0220e42c Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 20 Jul 2024 21:50:53 +0200
Subject: [PATCH 0511/1488] Better diagnostics in `test_disk_configuration`

---
 .../test_disk_configuration/test.py            | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/tests/integration/test_disk_configuration/test.py b/tests/integration/test_disk_configuration/test.py
index c003ff85755..afc5303298c 100644
--- a/tests/integration/test_disk_configuration/test.py
+++ b/tests/integration/test_disk_configuration/test.py
@@ -208,13 +208,21 @@ def test_merge_tree_custom_disk_setting(start_cluster):
                 secret_access_key='minio123');
     """
     )
-    count = len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True)))
+
+    list1 = list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))
+    count1 = len(list1)
+
     node1.query(f"INSERT INTO {TABLE_NAME}_3 SELECT number FROM numbers(100)")
     assert int(node1.query(f"SELECT count() FROM {TABLE_NAME}_3")) == 100
-    assert (
-        len(list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True)))
-        == count
-    )
+
+    list2 = list(minio.list_objects(cluster.minio_bucket, "data/", recursive=True))
+    count2 = len(list2)
+
+    if count1 != count2:
+        print("list1: ", list1)
+        print("list2: ", list2)
+
+    assert count1 == count2
     assert (
         len(list(minio.list_objects(cluster.minio_bucket, "data2/", recursive=True)))
         > 0

From f9b97aac84d6ab5f377261e5b57c2d6e8a432cef Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 20 Jul 2024 23:19:33 +0200
Subject: [PATCH 0512/1488] Fix bad test `02950_part_log_bytes_uncompressed`

---
 .../0_stateless/02950_part_log_bytes_uncompressed.sql       | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/02950_part_log_bytes_uncompressed.sql b/tests/queries/0_stateless/02950_part_log_bytes_uncompressed.sql
index 248475ab84b..cfed02eaeeb 100644
--- a/tests/queries/0_stateless/02950_part_log_bytes_uncompressed.sql
+++ b/tests/queries/0_stateless/02950_part_log_bytes_uncompressed.sql
@@ -1,3 +1,6 @@
+-- Tags: no-random-merge-tree-settings, no-random-settings
+-- Because we compare part sizes, and they could be affected by index granularity and index compression settings.
+
 CREATE TABLE part_log_bytes_uncompressed (
     key UInt8,
     value UInt8
@@ -17,7 +20,8 @@ ALTER TABLE part_log_bytes_uncompressed DROP PART 'all_4_4_0' SETTINGS mutations
 
 SYSTEM FLUSH LOGS;
 
-SELECT event_type, table, part_name, bytes_uncompressed > 0, size_in_bytes < bytes_uncompressed FROM system.part_log
+SELECT event_type, table, part_name, bytes_uncompressed > 0, size_in_bytes < bytes_uncompressed ? '1' : toString((size_in_bytes, bytes_uncompressed))
+FROM system.part_log
 WHERE event_date >= yesterday() AND database = currentDatabase() AND table = 'part_log_bytes_uncompressed'
     AND (event_type != 'RemovePart' OR part_name = 'all_4_4_0') -- ignore removal of other parts
 ORDER BY part_name, event_type;

From 67f51153661f37370f4e365ed3f19c6a4b84c2f4 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 21 Jul 2024 04:16:31 +0200
Subject: [PATCH 0513/1488] Update 02950_part_log_bytes_uncompressed.sql

---
 tests/queries/0_stateless/02950_part_log_bytes_uncompressed.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/02950_part_log_bytes_uncompressed.sql b/tests/queries/0_stateless/02950_part_log_bytes_uncompressed.sql
index cfed02eaeeb..24425062116 100644
--- a/tests/queries/0_stateless/02950_part_log_bytes_uncompressed.sql
+++ b/tests/queries/0_stateless/02950_part_log_bytes_uncompressed.sql
@@ -20,7 +20,7 @@ ALTER TABLE part_log_bytes_uncompressed DROP PART 'all_4_4_0' SETTINGS mutations
 
 SYSTEM FLUSH LOGS;
 
-SELECT event_type, table, part_name, bytes_uncompressed > 0, size_in_bytes < bytes_uncompressed ? '1' : toString((size_in_bytes, bytes_uncompressed))
+SELECT event_type, table, part_name, bytes_uncompressed > 0, (bytes_uncompressed > 0 ? (size_in_bytes < bytes_uncompressed ? '1' : toString((size_in_bytes, bytes_uncompressed))) : '0')
 FROM system.part_log
 WHERE event_date >= yesterday() AND database = currentDatabase() AND table = 'part_log_bytes_uncompressed'
     AND (event_type != 'RemovePart' OR part_name = 'all_4_4_0') -- ignore removal of other parts

From f1021b70f7f79fc6f921989573435ab5df406bdc Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 21 Jul 2024 07:05:09 +0200
Subject: [PATCH 0514/1488] Better diagnostics for test trace_events_stress

---
 src/Common/TraceSender.cpp | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/src/Common/TraceSender.cpp b/src/Common/TraceSender.cpp
index 91d07367a82..064da1b3d76 100644
--- a/src/Common/TraceSender.cpp
+++ b/src/Common/TraceSender.cpp
@@ -23,8 +23,15 @@ namespace DB
 
 LazyPipeFDs TraceSender::pipe;
 
+static thread_local bool inside_send = false;
 void TraceSender::send(TraceType trace_type, const StackTrace & stack_trace, Extras extras)
 {
+    DENY_ALLOCATIONS_IN_SCOPE;
+
+    if (unlikely(inside_send))
+        abort(); /// The method shouldn't be called recursively or throw exceptions.
+    inside_send = true;
+
     constexpr size_t buf_size = sizeof(char) /// TraceCollector stop flag
         + sizeof(UInt8)                      /// String size
         + QUERY_ID_MAX_LEN                   /// Maximum query_id length
@@ -80,6 +87,8 @@ void TraceSender::send(TraceType trace_type, const StackTrace & stack_trace, Ext
     writePODBinary(extras.increment, out);
 
     out.next();
+
+    inside_send = false;
 }
 
 }

From 6437088d81200909245172c441de3bf301f553b0 Mon Sep 17 00:00:00 2001
From: Mikhail <salazar290720035017@gmail.com>
Date: Sun, 21 Jul 2024 12:52:26 +0700
Subject: [PATCH 0515/1488] translate playground page to russian in ru locale

---
 docs/ru/getting-started/playground.md | 40 +++++++++++++--------------
 1 file changed, 20 insertions(+), 20 deletions(-)

diff --git a/docs/ru/getting-started/playground.md b/docs/ru/getting-started/playground.md
index c8c987eec9e..eb990c6071e 100644
--- a/docs/ru/getting-started/playground.md
+++ b/docs/ru/getting-started/playground.md
@@ -1,43 +1,43 @@
 ---
 slug: /ru/getting-started/playground
 sidebar_position: 14
-sidebar_label: Playground
+sidebar_label: Песочница
 ---
 
-# ClickHouse Playground {#clickhouse-playground}
+# Песочница ClickHouse {#clickhouse-playground}
 
-[ClickHouse Playground](https://play.clickhouse.com/play?user=play) allows people to experiment with ClickHouse by running queries instantly, without setting up their server or cluster.
-Several example datasets are available in Playground.
+[Песочница ClickHouse](https://play.clickhouse.com/play?user=play) позволяет пользователям экспериментировать с ClickHouse, выполняя запросы мгновенно, без необходимости настройки сервера или кластера.
+В Песочнице доступны несколько примеров наборов данных.
 
-You can make queries to Playground using any HTTP client, for example [curl](https://curl.haxx.se) or [wget](https://www.gnu.org/software/wget/), or set up a connection using [JDBC](../interfaces/jdbc.md) or [ODBC](../interfaces/odbc.md) drivers. More information about software products that support ClickHouse is available [here](../interfaces/index.md).
+Вы можете выполнять запросы к Песочнице, используя любой HTTP-клиент, например [curl](https://curl.haxx.se) или [wget](https://www.gnu.org/software/wget/), или настроить соединение, используя драйверы [JDBC](../interfaces/jdbc.md) или [ODBC](../interfaces/odbc.md). Дополнительную информацию о программных продуктах, поддерживающих ClickHouse, можно найти [здесь](../interfaces/index.md).
 
-## Credentials {#credentials}
+## Учетные данные {#credentials}
 
-| Parameter           | Value                              |
+| Параметр            | Значение                           |
 |:--------------------|:-----------------------------------|
-| HTTPS endpoint      | `https://play.clickhouse.com:443/` |
-| Native TCP endpoint | `play.clickhouse.com:9440`         |
-| User                | `explorer` or `play`               |
-| Password            | (empty)                            |
+| HTTPS-адрес         | `https://play.clickhouse.com:443/` |
+| TCP-адрес           | `play.clickhouse.com:9440`         |
+| Пользователь        | `explorer` или `play`              |
+| Пароль              | (пусто)                            |
 
-## Limitations {#limitations}
+## Ограничения {#limitations}
 
-The queries are executed as a read-only user. It implies some limitations:
+Запросы выполняются от имени пользователя с правами только на чтение. Это предполагает некоторые ограничения:
 
--   DDL queries are not allowed
--   INSERT queries are not allowed
+-   DDL-запросы не разрешены
+-   INSERT-запросы не разрешены
 
-The service also have quotas on its usage.
+Сервис также имеет квоты на использование.
 
-## Examples {#examples}
+## Примеры {#examples}
 
-HTTPS endpoint example with `curl`:
+Пример использования HTTPS-адреса с `curl`:
 
-``` bash
+```bash
 curl "https://play.clickhouse.com/?user=explorer" --data-binary "SELECT 'Play ClickHouse'"
 ```
 
-TCP endpoint example with [CLI](../interfaces/cli.md):
+Пример использования TCP-адреса с [CLI](../interfaces/cli.md):
 
 ``` bash
 clickhouse client --secure --host play.clickhouse.com --user explorer

From 4e1afb9c4d30ffb217fe3ee1ced586eefeffaa76 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 21 Jul 2024 08:16:27 +0200
Subject: [PATCH 0516/1488] Fix bad test
 `01042_system_reload_dictionary_reloads_completely`

---
 ...ad_dictionary_reloads_completely.reference |  6 ++---
 ...em_reload_dictionary_reloads_completely.sh | 22 ++++++++++++-------
 2 files changed, 17 insertions(+), 11 deletions(-)

diff --git a/tests/queries/0_stateless/01042_system_reload_dictionary_reloads_completely.reference b/tests/queries/0_stateless/01042_system_reload_dictionary_reloads_completely.reference
index f12dcd8258a..10bc7981d3e 100644
--- a/tests/queries/0_stateless/01042_system_reload_dictionary_reloads_completely.reference
+++ b/tests/queries/0_stateless/01042_system_reload_dictionary_reloads_completely.reference
@@ -1,6 +1,6 @@
 12 -> 	102
 13 -> 	103
 14 -> 	-1
-12(r) -> 	102
-13(r) -> 	103
-14(r) -> 	104
+12 (after reloading) -> 	102
+13 (after reloading) -> 	103
+14 (after reloading) -> 	104
diff --git a/tests/queries/0_stateless/01042_system_reload_dictionary_reloads_completely.sh b/tests/queries/0_stateless/01042_system_reload_dictionary_reloads_completely.sh
index 2b075566ac3..42488ca946c 100755
--- a/tests/queries/0_stateless/01042_system_reload_dictionary_reloads_completely.sh
+++ b/tests/queries/0_stateless/01042_system_reload_dictionary_reloads_completely.sh
@@ -1,4 +1,6 @@
 #!/usr/bin/env bash
+# Tags: no-random-settings
+# Dictionaries are updated using the server time.
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
@@ -6,8 +8,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
 set -e -o pipefail
 
-# NOTE: dictionaries TTLs works with server timezone, so session_timeout cannot be used
-$CLICKHOUSE_CLIENT --session_timezone '' --multiquery <<EOF
+$CLICKHOUSE_CLIENT --multiquery <<EOF
 CREATE TABLE ${CLICKHOUSE_DATABASE}.table(x Int64, y Int64, insert_time DateTime) ENGINE = MergeTree ORDER BY tuple();
 INSERT INTO ${CLICKHOUSE_DATABASE}.table VALUES (12, 102, now());
 
@@ -28,16 +29,21 @@ $CLICKHOUSE_CLIENT --query "SELECT '12 -> ', dictGetInt64('${CLICKHOUSE_DATABASE
 $CLICKHOUSE_CLIENT --query "INSERT INTO ${CLICKHOUSE_DATABASE}.table VALUES (13, 103, now())"
 $CLICKHOUSE_CLIENT --query "INSERT INTO ${CLICKHOUSE_DATABASE}.table VALUES (14, 104, now() - INTERVAL 1 DAY)"
 
+# Wait when the dictionary will update the value for 13 on its own:
 while [ "$(${CLICKHOUSE_CLIENT} --query "SELECT dictGetInt64('${CLICKHOUSE_DATABASE}.dict', 'y', toUInt64(13))")" = -1 ]
-    do
-        sleep 0.5
-    done
+do
+    sleep 0.5
+done
 
 $CLICKHOUSE_CLIENT --query "SELECT '13 -> ', dictGetInt64('${CLICKHOUSE_DATABASE}.dict', 'y', toUInt64(13))"
+
+# By the way, the value for 14 is expected to not be updated at this moment,
+# because the values were selected by the update field insert_time, and for 14 it was set as one day ago.
 $CLICKHOUSE_CLIENT --query "SELECT '14 -> ', dictGetInt64('${CLICKHOUSE_DATABASE}.dict', 'y', toUInt64(14))"
 
+# SYSTEM RELOAD DICTIONARY reloads it completely, regardless of the update field, so we will see new values, even for key 14.
 $CLICKHOUSE_CLIENT --query "SYSTEM RELOAD DICTIONARY '${CLICKHOUSE_DATABASE}.dict'"
 
-$CLICKHOUSE_CLIENT --query "SELECT '12(r) -> ', dictGetInt64('${CLICKHOUSE_DATABASE}.dict', 'y', toUInt64(12))"
-$CLICKHOUSE_CLIENT --query "SELECT '13(r) -> ', dictGetInt64('${CLICKHOUSE_DATABASE}.dict', 'y', toUInt64(13))"
-$CLICKHOUSE_CLIENT --query "SELECT '14(r) -> ', dictGetInt64('${CLICKHOUSE_DATABASE}.dict', 'y', toUInt64(14))"
+$CLICKHOUSE_CLIENT --query "SELECT '12 (after reloading) -> ', dictGetInt64('${CLICKHOUSE_DATABASE}.dict', 'y', toUInt64(12))"
+$CLICKHOUSE_CLIENT --query "SELECT '13 (after reloading) -> ', dictGetInt64('${CLICKHOUSE_DATABASE}.dict', 'y', toUInt64(13))"
+$CLICKHOUSE_CLIENT --query "SELECT '14 (after reloading) -> ', dictGetInt64('${CLICKHOUSE_DATABASE}.dict', 'y', toUInt64(14))"

From e74892bfaf4aa98654a6253a34bbeb3f1740de41 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 21 Jul 2024 08:21:09 +0200
Subject: [PATCH 0517/1488] Update playground.md

---
 docs/ru/getting-started/playground.md | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/docs/ru/getting-started/playground.md b/docs/ru/getting-started/playground.md
index eb990c6071e..a2d5498fb9a 100644
--- a/docs/ru/getting-started/playground.md
+++ b/docs/ru/getting-started/playground.md
@@ -1,15 +1,15 @@
 ---
 slug: /ru/getting-started/playground
 sidebar_position: 14
-sidebar_label: Песочница
+sidebar_label: Playground
 ---
 
-# Песочница ClickHouse {#clickhouse-playground}
+# ClickHouse Playground {#clickhouse-playground}
 
-[Песочница ClickHouse](https://play.clickhouse.com/play?user=play) позволяет пользователям экспериментировать с ClickHouse, выполняя запросы мгновенно, без необходимости настройки сервера или кластера.
-В Песочнице доступны несколько примеров наборов данных.
+[ClickHouse Playground](https://play.clickhouse.com/play?user=play) позволяет пользователям экспериментировать с ClickHouse, выполняя запросы мгновенно, без необходимости настройки сервера или кластера.
+В Playground доступны несколько примеров наборов данных.
 
-Вы можете выполнять запросы к Песочнице, используя любой HTTP-клиент, например [curl](https://curl.haxx.se) или [wget](https://www.gnu.org/software/wget/), или настроить соединение, используя драйверы [JDBC](../interfaces/jdbc.md) или [ODBC](../interfaces/odbc.md). Дополнительную информацию о программных продуктах, поддерживающих ClickHouse, можно найти [здесь](../interfaces/index.md).
+Вы можете выполнять запросы к Playground, используя любой HTTP-клиент, например [curl](https://curl.haxx.se) или [wget](https://www.gnu.org/software/wget/), или настроить соединение, используя драйверы [JDBC](../interfaces/jdbc.md) или [ODBC](../interfaces/odbc.md). Дополнительную информацию о программных продуктах, поддерживающих ClickHouse, можно найти [здесь](../interfaces/index.md).
 
 ## Учетные данные {#credentials}
 

From f71244c85c76425bdeeb89ce017caed512f9576c Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 21 Jul 2024 08:53:54 +0200
Subject: [PATCH 0518/1488] Something is strange with the test about
 refreshable materialized views

---
 src/Storages/MaterializedView/RefreshTask.cpp               | 2 +-
 .../0_stateless/02932_refreshable_materialized_views.sh     | 6 ++----
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/src/Storages/MaterializedView/RefreshTask.cpp b/src/Storages/MaterializedView/RefreshTask.cpp
index 857cfd78910..cff0e2cf40b 100644
--- a/src/Storages/MaterializedView/RefreshTask.cpp
+++ b/src/Storages/MaterializedView/RefreshTask.cpp
@@ -304,7 +304,7 @@ void RefreshTask::refreshTask()
                 {
                     PreformattedMessage message = getCurrentExceptionMessageAndPattern(true);
                     auto text = message.text;
-                    message.text = fmt::format("Refresh failed: {}", message.text);
+                    message.text = fmt::format("Refresh view {} failed: {}", view->getStorageID().getFullTableName(), message.text);
                     LOG_ERROR(log, message);
                     exception = text;
                 }
diff --git a/tests/queries/0_stateless/02932_refreshable_materialized_views.sh b/tests/queries/0_stateless/02932_refreshable_materialized_views.sh
index 89942e25b67..36cdc8d88e3 100755
--- a/tests/queries/0_stateless/02932_refreshable_materialized_views.sh
+++ b/tests/queries/0_stateless/02932_refreshable_materialized_views.sh
@@ -2,8 +2,6 @@
 # Tags: atomic-database
 
 CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# reset --log_comment
-CLICKHOUSE_LOG_COMMENT=
 # shellcheck source=../shell_config.sh
 . "$CUR_DIR"/../shell_config.sh
 
@@ -172,13 +170,13 @@ $CLICKHOUSE_CLIENT -nq "
     drop table b;
     create materialized view c refresh every 1 second (x Int64) engine Memory empty as select * from src;
     drop table src;"
-while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result from refreshes -- $LINENO" | xargs`" != 'Exception' ]
+while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result from refreshes where view = 'c' -- $LINENO" | xargs`" != 'Exception' ]
 do
     sleep 0.1
 done
 # Check exception, create src, expect successful refresh.
 $CLICKHOUSE_CLIENT -nq "
-    select '<19: exception>', exception ilike '%UNKNOWN_TABLE%' from refreshes;
+    select '<19: exception>', exception ilike '%UNKNOWN_TABLE%' ? '1' : exception from refreshes where view = 'c';
     create table src (x Int64) engine Memory as select 1;
     system refresh view c;"
 while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result from refreshes -- $LINENO" | xargs`" != 'Finished' ]

From fab0bb6aa41c9deccb69666ae0d22559c0784121 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 21 Jul 2024 09:01:44 +0200
Subject: [PATCH 0519/1488] Remove all sleeps

---
 .../02932_refreshable_materialized_views.sh   | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/tests/queries/0_stateless/02932_refreshable_materialized_views.sh b/tests/queries/0_stateless/02932_refreshable_materialized_views.sh
index 36cdc8d88e3..9081035579d 100755
--- a/tests/queries/0_stateless/02932_refreshable_materialized_views.sh
+++ b/tests/queries/0_stateless/02932_refreshable_materialized_views.sh
@@ -132,7 +132,7 @@ while [ "`$CLICKHOUSE_CLIENT -nq "select status, next_refresh_time from refreshe
 do
     sleep 0.1
 done
-sleep 1
+
 $CLICKHOUSE_CLIENT -nq "
     select '<14: waiting for next cycle>', view, status, remaining_dependencies, next_refresh_time from refreshes;
     truncate src;
@@ -222,22 +222,27 @@ done
 $CLICKHOUSE_CLIENT -nq "
     rename table e to f;
     select '<24: rename during refresh>', * from f;
-    select '<25: rename during refresh>', view, status from refreshes;
+    select '<25: rename during refresh>', view, status from refreshes where view = 'f';
     alter table f modify refresh after 10 year;"
-sleep 2 # make it likely that at least one row was processed
+
 # Cancel.
 $CLICKHOUSE_CLIENT -nq "
     system cancel view f;"
-while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result from refreshes -- $LINENO" | xargs`" != 'Cancelled' ]
+while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result from refreshes where view = 'f' -- $LINENO" | xargs`" != 'Cancelled' ]
 do
     sleep 0.1
 done
+
+while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes where view = 'f' -- $LINENO" | xargs`" = 'Running' ]
+do
+    sleep 0.1
+done
+
 # Check that another refresh doesn't immediately start after the cancelled one.
-sleep 1
 $CLICKHOUSE_CLIENT -nq "
-    select '<27: cancelled>', view, status from refreshes;
+    select '<27: cancelled>', view, status from refreshes where view = 'f';
     system refresh view f;"
-while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes -- $LINENO" | xargs`" != 'Running' ]
+while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes where view = 'f' -- $LINENO" | xargs`" != 'Running' ]
 do
     sleep 0.1
 done

From a371557f07ab97b88da40e6b790a5d361f437a47 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 21 Jul 2024 09:06:45 +0200
Subject: [PATCH 0520/1488] Make test `00997_set_index_array` lighter

---
 tests/queries/0_stateless/00997_set_index_array.sql | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/queries/0_stateless/00997_set_index_array.sql b/tests/queries/0_stateless/00997_set_index_array.sql
index d6d27f5a6a0..ed972d1a545 100644
--- a/tests/queries/0_stateless/00997_set_index_array.sql
+++ b/tests/queries/0_stateless/00997_set_index_array.sql
@@ -12,10 +12,10 @@ ORDER BY (primary_key);
 
 INSERT INTO set_array
 select
-  toString(intDiv(number, 1000000)) as primary_key,
+  toString(intDiv(number, 100000)) as primary_key,
   array(number) as index_array
 from system.numbers
-limit 10000000;
+limit 1000000;
 
 OPTIMIZE TABLE set_array FINAL;
 

From 5ff125e37ef35ce5b451e54482e92d7663e25bdb Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 21 Jul 2024 10:32:54 +0200
Subject: [PATCH 0521/1488] Miscellaneous

---
 src/Common/CurrentMetrics.cpp         | 8 ++++----
 src/Databases/DatabaseLazy.cpp        | 4 ++--
 src/Databases/DatabasesCommon.cpp     | 8 ++++----
 src/Parsers/ExpressionListParsers.cpp | 2 +-
 4 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/src/Common/CurrentMetrics.cpp b/src/Common/CurrentMetrics.cpp
index 7c97e73f278..1011ab12d15 100644
--- a/src/Common/CurrentMetrics.cpp
+++ b/src/Common/CurrentMetrics.cpp
@@ -234,10 +234,10 @@
     M(PartsCommitted, "Deprecated. See PartsActive.") \
     M(PartsPreActive, "The part is in data_parts, but not used for SELECTs.") \
     M(PartsActive, "Active data part, used by current and upcoming SELECTs.") \
-    M(AttachedDatabase, "Active database, used by current and upcoming SELECTs.") \
-    M(AttachedTable, "Active table, used by current and upcoming SELECTs.") \
-    M(AttachedView, "Active view, used by current and upcoming SELECTs.") \
-    M(AttachedDictionary, "Active dictionary, used by current and upcoming SELECTs.") \
+    M(AttachedDatabase, "Active databases.") \
+    M(AttachedTable, "Active tables.") \
+    M(AttachedView, "Active views.") \
+    M(AttachedDictionary, "Active dictionaries.") \
     M(PartsOutdated, "Not active data part, but could be used by only current SELECTs, could be deleted after SELECTs finishes.") \
     M(PartsDeleting, "Not active data part with identity refcounter, it is deleting right now by a cleaner.") \
     M(PartsDeleteOnDestroy, "Part was moved to another disk and should be deleted in own destructor.") \
diff --git a/src/Databases/DatabaseLazy.cpp b/src/Databases/DatabaseLazy.cpp
index 5017c9b25cb..ca30ee6db15 100644
--- a/src/Databases/DatabaseLazy.cpp
+++ b/src/Databases/DatabaseLazy.cpp
@@ -195,7 +195,7 @@ void DatabaseLazy::attachTable(ContextPtr /* context_ */, const String & table_n
         snapshot_detached_tables.erase(table_name);
     }
 
-    CurrentMetrics::add(CurrentMetrics::AttachedTable, 1);
+    CurrentMetrics::add(CurrentMetrics::AttachedTable);
 }
 
 StoragePtr DatabaseLazy::detachTable(ContextPtr /* context */, const String & table_name)
@@ -221,7 +221,7 @@ StoragePtr DatabaseLazy::detachTable(ContextPtr /* context */, const String & ta
                 .metadata_path = getObjectMetadataPath(table_name),
                 .is_permanently = false});
 
-        CurrentMetrics::sub(CurrentMetrics::AttachedTable, 1);
+        CurrentMetrics::sub(CurrentMetrics::AttachedTable);
     }
     return res;
 }
diff --git a/src/Databases/DatabasesCommon.cpp b/src/Databases/DatabasesCommon.cpp
index fe0baf30e57..6ccaf811764 100644
--- a/src/Databases/DatabasesCommon.cpp
+++ b/src/Databases/DatabasesCommon.cpp
@@ -289,8 +289,8 @@ StoragePtr DatabaseWithOwnTablesBase::detachTableUnlocked(const String & table_n
     tables.erase(it);
     table_storage->is_detached = true;
 
-    if (table_storage->isSystemStorage() == false)
-        CurrentMetrics::sub(getAttachedCounterForStorage(table_storage), 1);
+    if (!table_storage->isSystemStorage() && database_name != DatabaseCatalog::SYSTEM_DATABASE)
+        CurrentMetrics::sub(getAttachedCounterForStorage(table_storage));
 
     auto table_id = table_storage->getStorageID();
     if (table_id.hasUUID())
@@ -334,8 +334,8 @@ void DatabaseWithOwnTablesBase::attachTableUnlocked(const String & table_name, c
     /// non-Atomic database the is_detached is set to true before RENAME.
     table->is_detached = false;
 
-    if (table->isSystemStorage() == false && table_id.database_name != DatabaseCatalog::SYSTEM_DATABASE)
-        CurrentMetrics::add(getAttachedCounterForStorage(table), 1);
+    if (!table->isSystemStorage() && table_id.database_name != DatabaseCatalog::SYSTEM_DATABASE)
+        CurrentMetrics::add(getAttachedCounterForStorage(table));
 }
 
 void DatabaseWithOwnTablesBase::shutdown()
diff --git a/src/Parsers/ExpressionListParsers.cpp b/src/Parsers/ExpressionListParsers.cpp
index fff8383e7b3..f97c042e91e 100644
--- a/src/Parsers/ExpressionListParsers.cpp
+++ b/src/Parsers/ExpressionListParsers.cpp
@@ -2743,7 +2743,7 @@ Action ParserExpressionImpl::tryParseOperator(Layers & layers, IParser::Pos & po
     /// 'AND' can be both boolean function and part of the '... BETWEEN ... AND ...' operator
     if (op.function_name == "and" && layers.back()->between_counter)
     {
-        layers.back()->between_counter--;
+        --layers.back()->between_counter;
         op = finish_between_operator;
     }
 

From ea83f89374f723d1d53dca64ec58e8fd993f013d Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 21 Jul 2024 10:39:03 +0200
Subject: [PATCH 0522/1488] Fix inconsistent formatting of lambda functions
 inside composite types

---
 src/Parsers/ASTFunction.cpp                               | 8 ++++----
 ...ite_expressions_lambda_consistent_formatting.reference | 0
 ...composite_expressions_lambda_consistent_formatting.sql | 6 ++++++
 3 files changed, 10 insertions(+), 4 deletions(-)
 create mode 100644 tests/queries/0_stateless/03207_composite_expressions_lambda_consistent_formatting.reference
 create mode 100644 tests/queries/0_stateless/03207_composite_expressions_lambda_consistent_formatting.sql

diff --git a/src/Parsers/ASTFunction.cpp b/src/Parsers/ASTFunction.cpp
index f39229d7566..b04ec1c22b2 100644
--- a/src/Parsers/ASTFunction.cpp
+++ b/src/Parsers/ASTFunction.cpp
@@ -7,7 +7,6 @@
 #include <Common/FieldVisitorToString.h>
 #include <Common/KnownObjectNames.h>
 #include <Common/SipHash.h>
-#include <Common/typeid_cast.h>
 #include <IO/Operators.h>
 #include <IO/WriteBufferFromString.h>
 #include <IO/WriteHelpers.h>
@@ -19,9 +18,6 @@
 #include <Parsers/queryToString.h>
 #include <Parsers/ASTSetQuery.h>
 #include <Parsers/FunctionSecretArgumentsFinderAST.h>
-#include <Core/QualifiedTableName.h>
-
-#include <boost/algorithm/string.hpp>
 
 
 using namespace std::literals;
@@ -632,6 +628,7 @@ void ASTFunction::formatImplWithoutAlias(const FormatSettings & settings, Format
                     settings.ostr << ", ";
                 if (arguments->children[i]->as<ASTSetQuery>())
                     settings.ostr << "SETTINGS ";
+                nested_dont_need_parens.list_element_index = i;
                 arguments->children[i]->formatImpl(settings, state, nested_dont_need_parens);
             }
             settings.ostr << (settings.hilite ? hilite_operator : "") << ']' << (settings.hilite ? hilite_none : "");
@@ -642,12 +639,14 @@ void ASTFunction::formatImplWithoutAlias(const FormatSettings & settings, Format
         {
             settings.ostr << (settings.hilite ? hilite_operator : "") << ((frame.need_parens && !alias.empty()) ? "tuple" : "") << '('
                           << (settings.hilite ? hilite_none : "");
+
             for (size_t i = 0; i < arguments->children.size(); ++i)
             {
                 if (i != 0)
                     settings.ostr << ", ";
                 if (arguments->children[i]->as<ASTSetQuery>())
                     settings.ostr << "SETTINGS ";
+                nested_dont_need_parens.list_element_index = i;
                 arguments->children[i]->formatImpl(settings, state, nested_dont_need_parens);
             }
             settings.ostr << (settings.hilite ? hilite_operator : "") << ')' << (settings.hilite ? hilite_none : "");
@@ -663,6 +662,7 @@ void ASTFunction::formatImplWithoutAlias(const FormatSettings & settings, Format
                     settings.ostr << ", ";
                 if (arguments->children[i]->as<ASTSetQuery>())
                     settings.ostr << "SETTINGS ";
+                nested_dont_need_parens.list_element_index = i;
                 arguments->children[i]->formatImpl(settings, state, nested_dont_need_parens);
             }
             settings.ostr << (settings.hilite ? hilite_operator : "") << ')' << (settings.hilite ? hilite_none : "");
diff --git a/tests/queries/0_stateless/03207_composite_expressions_lambda_consistent_formatting.reference b/tests/queries/0_stateless/03207_composite_expressions_lambda_consistent_formatting.reference
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/queries/0_stateless/03207_composite_expressions_lambda_consistent_formatting.sql b/tests/queries/0_stateless/03207_composite_expressions_lambda_consistent_formatting.sql
new file mode 100644
index 00000000000..42c823cf476
--- /dev/null
+++ b/tests/queries/0_stateless/03207_composite_expressions_lambda_consistent_formatting.sql
@@ -0,0 +1,6 @@
+SELECT [1, (x -> 1)]; -- { serverError ILLEGAL_TYPE_OF_ARGUMENT }
+SELECT (1, (x -> 1)); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT }
+SELECT map(1, (x -> 1)); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT }
+SELECT [1, lambda(x, 1)]; -- { serverError UNKNOWN_IDENTIFIER }
+SELECT (1, lambda(x, 1)); -- { serverError UNKNOWN_IDENTIFIER }
+SELECT map(1, lambda(x, 1)); -- { serverError UNKNOWN_IDENTIFIER }

From 7f03b189667161de61e33523586a594694b02071 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 21 Jul 2024 10:49:53 +0200
Subject: [PATCH 0523/1488] Fix two terrible bugs

---
 src/Interpreters/InterpreterCreateQuery.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp
index ea10ad59db4..8c1a3cb4dff 100644
--- a/src/Interpreters/InterpreterCreateQuery.cpp
+++ b/src/Interpreters/InterpreterCreateQuery.cpp
@@ -1606,7 +1606,7 @@ bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create,
         UInt64 table_count = CurrentMetrics::get(CurrentMetrics::AttachedTable);
         if (table_count >= table_num_limit)
             throw Exception(ErrorCodes::TOO_MANY_TABLES,
-                            "Too many tables in the Clickhouse. "
+                            "Too many tables. "
                             "The limit (setting 'max_table_num_to_throw') is set to {}, current number of tables is {}",
                             table_num_limit, table_count);
     }

From a6f1c46448a17f294cc6047dfc3fef3802b40abf Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 21 Jul 2024 11:08:38 +0200
Subject: [PATCH 0524/1488] Miscellaneous

---
 src/Interpreters/InterpreterCreateQuery.cpp   |  9 ++++----
 .../test_table_db_num_limit/test.py           | 22 +++++++++++--------
 2 files changed, 17 insertions(+), 14 deletions(-)

diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp
index 8c1a3cb4dff..2a4a02597fe 100644
--- a/src/Interpreters/InterpreterCreateQuery.cpp
+++ b/src/Interpreters/InterpreterCreateQuery.cpp
@@ -16,7 +16,6 @@
 #include <Common/logger_useful.h>
 #include <Core/Settings.h>
 #include <Parsers/ASTSetQuery.h>
-#include <base/hex.h>
 
 #include <Core/Defines.h>
 #include <Core/SettingsEnums.h>
@@ -82,13 +81,13 @@
 #include <Interpreters/ApplyWithSubqueryVisitor.h>
 
 #include <TableFunctions/TableFunctionFactory.h>
-#include <DataTypes/DataTypeFixedString.h>
 
 #include <Functions/UserDefined/UserDefinedSQLFunctionFactory.h>
 #include <Functions/UserDefined/UserDefinedSQLFunctionVisitor.h>
 #include <Interpreters/ReplaceQueryParameterVisitor.h>
 #include <Parsers/QueryParameterVisitor.h>
 
+
 namespace CurrentMetrics
 {
     extern const Metric AttachedTable;
@@ -166,8 +165,8 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create)
 
         if (db_count >= db_num_limit)
             throw Exception(ErrorCodes::TOO_MANY_DATABASES,
-                            "Too many databases in the Clickhouse. "
-                            "The limit (setting 'max_database_num_to_throw') is set to {}, current number of databases is {}",
+                            "Too many databases. "
+                            "The limit (server configuration parameter `max_database_num_to_throw`) is set to {}, the current number of databases is {}",
                             db_num_limit, db_count);
     }
 
@@ -1607,7 +1606,7 @@ bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create,
         if (table_count >= table_num_limit)
             throw Exception(ErrorCodes::TOO_MANY_TABLES,
                             "Too many tables. "
-                            "The limit (setting 'max_table_num_to_throw') is set to {}, current number of tables is {}",
+                            "The limit (server configuration parameter `max_table_num_to_throw`) is set to {}, the current number of tables is {}",
                             table_num_limit, table_count);
     }
 
diff --git a/tests/integration/test_table_db_num_limit/test.py b/tests/integration/test_table_db_num_limit/test.py
index aa8030b077c..56403d165b2 100644
--- a/tests/integration/test_table_db_num_limit/test.py
+++ b/tests/integration/test_table_db_num_limit/test.py
@@ -4,8 +4,8 @@ from helpers.cluster import ClickHouseCluster
 
 cluster = ClickHouseCluster(__file__)
 
-node1 = cluster.add_instance(
-    "node1", main_configs=["config/config.xml"], with_zookeeper=True
+node = cluster.add_instance(
+    "node", main_configs=["config/config.xml"]
 )
 
 
@@ -22,22 +22,26 @@ def started_cluster():
 
 def test_table_db_limit(started_cluster):
     for i in range(10):
-        node1.query("create database db{}".format(i))
+        node.query("create database db{}".format(i))
 
     with pytest.raises(QueryRuntimeException) as exp_info:
-        node1.query("create database db_exp".format(i))
+        node.query("create database db_exp".format(i))
 
     assert "TOO_MANY_DATABASES" in str(exp_info)
 
     for i in range(10):
-        node1.query("create table t{} (a Int32) Engine = Log".format(i))
+        node.query("create table t{} (a Int32) Engine = Log".format(i))
+
+    # This checks that system tables are not accounted in the number of tables.
+    node.query("system flush logs")
 
-    node1.query("system flush logs")
     for i in range(10):
-        node1.query("drop table t{}".format(i))
+        node.query("drop table t{}".format(i))
+
     for i in range(10):
-        node1.query("create table t{} (a Int32) Engine = Log".format(i))
+        node.query("create table t{} (a Int32) Engine = Log".format(i))
 
     with pytest.raises(QueryRuntimeException) as exp_info:
-        node1.query("create table default.tx (a Int32) Engine = Log")
+        node.query("create table default.tx (a Int32) Engine = Log")
+
     assert "TOO_MANY_TABLES" in str(exp_info)

From efd5ff9f4c63c82d77cfda43af786ad3b7a4b8e0 Mon Sep 17 00:00:00 2001
From: robot-clickhouse <robot-clickhouse@users.noreply.github.com>
Date: Sun, 21 Jul 2024 09:15:13 +0000
Subject: [PATCH 0525/1488] Automatic style fix

---
 tests/integration/test_table_db_num_limit/test.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/tests/integration/test_table_db_num_limit/test.py b/tests/integration/test_table_db_num_limit/test.py
index 56403d165b2..a7bb04905f2 100644
--- a/tests/integration/test_table_db_num_limit/test.py
+++ b/tests/integration/test_table_db_num_limit/test.py
@@ -4,9 +4,7 @@ from helpers.cluster import ClickHouseCluster
 
 cluster = ClickHouseCluster(__file__)
 
-node = cluster.add_instance(
-    "node", main_configs=["config/config.xml"]
-)
+node = cluster.add_instance("node", main_configs=["config/config.xml"])
 
 
 @pytest.fixture(scope="module")

From 7ed9ab2338f7de5a2495aad267cc688799c5ff18 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 21 Jul 2024 11:29:12 +0200
Subject: [PATCH 0526/1488] Remove wrong logic from InterpreterCreateQuery

---
 src/Interpreters/InterpreterCreateQuery.cpp       | 12 ++++++------
 tests/integration/test_table_db_num_limit/test.py |  3 ++-
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp
index 2a4a02597fe..7e0b6eb4193 100644
--- a/src/Interpreters/InterpreterCreateQuery.cpp
+++ b/src/Interpreters/InterpreterCreateQuery.cpp
@@ -146,21 +146,21 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create)
     }
 
     auto db_num_limit = getContext()->getGlobalContext()->getServerSettings().max_database_num_to_throw;
-    if (db_num_limit > 0)
+    if (db_num_limit > 0 && !internal)
     {
         size_t db_count = DatabaseCatalog::instance().getDatabases().size();
-        std::vector<String> system_databases = {
+        std::initializer_list<std::string_view> system_databases =
+        {
             DatabaseCatalog::TEMPORARY_DATABASE,
             DatabaseCatalog::SYSTEM_DATABASE,
             DatabaseCatalog::INFORMATION_SCHEMA,
             DatabaseCatalog::INFORMATION_SCHEMA_UPPERCASE,
-            DatabaseCatalog::DEFAULT_DATABASE
         };
 
         for (const auto & system_database : system_databases)
         {
-            if (db_count > 0 && DatabaseCatalog::instance().isDatabaseExist(system_database))
-                db_count--;
+            if (db_count > 0 && DatabaseCatalog::instance().isDatabaseExist(std::string(system_database)))
+                --db_count;
         }
 
         if (db_count >= db_num_limit)
@@ -1600,7 +1600,7 @@ bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create,
     }
 
     UInt64 table_num_limit = getContext()->getGlobalContext()->getServerSettings().max_table_num_to_throw;
-    if (table_num_limit > 0 && create.getDatabase() != DatabaseCatalog::SYSTEM_DATABASE)
+    if (table_num_limit > 0 && !internal)
     {
         UInt64 table_count = CurrentMetrics::get(CurrentMetrics::AttachedTable);
         if (table_count >= table_num_limit)
diff --git a/tests/integration/test_table_db_num_limit/test.py b/tests/integration/test_table_db_num_limit/test.py
index 56403d165b2..aa6404c385c 100644
--- a/tests/integration/test_table_db_num_limit/test.py
+++ b/tests/integration/test_table_db_num_limit/test.py
@@ -21,7 +21,8 @@ def started_cluster():
 
 
 def test_table_db_limit(started_cluster):
-    for i in range(10):
+    # By the way, default database already exists.
+    for i in range(9):
         node.query("create database db{}".format(i))
 
     with pytest.raises(QueryRuntimeException) as exp_info:

From 433ac55d13276e7a0b26ded07f27f17726703dab Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Sun, 21 Jul 2024 11:41:42 +0200
Subject: [PATCH 0527/1488] Correctly handle failure

---
 src/Common/Allocator.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Common/Allocator.cpp b/src/Common/Allocator.cpp
index 7f2241ab4c0..1b43e746c69 100644
--- a/src/Common/Allocator.cpp
+++ b/src/Common/Allocator.cpp
@@ -206,7 +206,7 @@ void * Allocator<clear_memory_, populate>::realloc(void * buf, size_t old_size,
         }
         else
         {
-            [[maybe_unused]] auto trace_free = CurrentMemoryTracker::free(old_size);
+            [[maybe_unused]] auto trace_free = CurrentMemoryTracker::free(new_size);
             ProfileEvents::increment(ProfileEvents::GWPAsanAllocateFailed);
         }
     }
@@ -239,7 +239,7 @@ void * Allocator<clear_memory_, populate>::realloc(void * buf, size_t old_size,
         void * new_buf = ::realloc(buf, new_size);
         if (nullptr == new_buf)
         {
-            [[maybe_unused]] auto trace_free = CurrentMemoryTracker::free(old_size);
+            [[maybe_unused]] auto trace_free = CurrentMemoryTracker::free(new_size);
             throw DB::ErrnoException(
                 DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY,
                 "Allocator: Cannot realloc from {} to {}",

From 08353419d0798f209627f16f840f4d101227431d Mon Sep 17 00:00:00 2001
From: Max K <maxkaynov@gmail.com>
Date: Sun, 21 Jul 2024 11:50:15 +0200
Subject: [PATCH 0528/1488] CI: Never await on BuildReport job, Skip
 BuildReport if no builds in workflow

---
 tests/ci/ci_cache.py       | 27 ++++++++++----
 tests/ci/ci_definitions.py |  3 ++
 tests/ci/merge_pr.py       |  5 ++-
 tests/ci/test_ci_config.py | 74 ++++++++++++++++++++++++++++++++++----
 4 files changed, 96 insertions(+), 13 deletions(-)

diff --git a/tests/ci/ci_cache.py b/tests/ci/ci_cache.py
index cfefb954fcd..16b6eac1ecb 100644
--- a/tests/ci/ci_cache.py
+++ b/tests/ci/ci_cache.py
@@ -638,7 +638,7 @@ class CiCache:
         pushes pending records for all jobs that supposed to be run
         """
         for job, job_config in self.jobs_to_do.items():
-            if not job_config.has_digest():
+            if not job_config.has_digest() or job_config.disable_await:
                 continue
             pending_state = PendingState(time.time(), run_url=GITHUB_RUN_URL)
             assert job_config.batches
@@ -708,7 +708,7 @@ class CiCache:
         Filter is to be applied in PRs to remove jobs that are not affected by the change
         :return:
         """
-        remove_from_to_do = []
+        remove_from_workflow = []
         required_builds = []
         has_test_jobs_to_skip = False
         for job_name, job_config in self.jobs_to_do.items():
@@ -723,26 +723,41 @@ class CiCache:
                     job=reference_name,
                     job_config=reference_config,
                 ):
-                    remove_from_to_do.append(job_name)
+                    remove_from_workflow.append(job_name)
                     has_test_jobs_to_skip = True
                 else:
                     required_builds += (
                         job_config.required_builds if job_config.required_builds else []
                     )
         if has_test_jobs_to_skip:
-            # If there are tests to skip, it means build digest has not been changed.
+            # If there are tests to skip, it means builds are not affected as well.
             # No need to test builds. Let's keep all builds required for test jobs and skip the others
             for job_name, job_config in self.jobs_to_do.items():
                 if CI.is_build_job(job_name):
                     if job_name not in required_builds:
-                        remove_from_to_do.append(job_name)
+                        remove_from_workflow.append(job_name)
 
-        for job in remove_from_to_do:
+        for job in remove_from_workflow:
             print(f"Filter job [{job}] - not affected by the change")
             if job in self.jobs_to_do:
                 del self.jobs_to_do[job]
             if job in self.jobs_to_wait:
                 del self.jobs_to_wait[job]
+            if job in self.jobs_to_skip:
+                self.jobs_to_skip.remove(job)
+
+        # special handling for the special job: BUILD_CHECK
+        has_builds = False
+        for job in list(self.jobs_to_do) + self.jobs_to_skip:
+            if CI.is_build_job(job):
+                has_builds = True
+                break
+        if not has_builds:
+            if CI.JobNames.BUILD_CHECK in self.jobs_to_do:
+                print(
+                    f"Filter job [{CI.JobNames.BUILD_CHECK}] - no builds are required in the workflow"
+                )
+                del self.jobs_to_do[CI.JobNames.BUILD_CHECK]
 
     def await_pending_jobs(self, is_release: bool, dry_run: bool = False) -> None:
         """
diff --git a/tests/ci/ci_definitions.py b/tests/ci/ci_definitions.py
index 4c78efd39a2..a8d9793f1d3 100644
--- a/tests/ci/ci_definitions.py
+++ b/tests/ci/ci_definitions.py
@@ -351,6 +351,8 @@ class JobConfig:
     run_by_label: str = ""
     # to run always regardless of the job digest or/and label
     run_always: bool = False
+    # disables CI await for a given job
+    disable_await: bool = False
     # if the job needs to be run on the release branch, including master (building packages, docker server).
     # NOTE: Subsequent runs on the same branch with the similar digest are still considered skip-able.
     required_on_release_branch: bool = False
@@ -395,6 +397,7 @@ class CommonJobConfigs:
             ],
         ),
         runner_type=Runners.STYLE_CHECKER_ARM,
+        disable_await=True,
     )
     COMPATIBILITY_TEST = JobConfig(
         job_name_keyword="compatibility",
diff --git a/tests/ci/merge_pr.py b/tests/ci/merge_pr.py
index 4d8facafb84..13c7537a84b 100644
--- a/tests/ci/merge_pr.py
+++ b/tests/ci/merge_pr.py
@@ -254,11 +254,14 @@ def main():
         statuses = get_commit_filtered_statuses(commit)
 
         has_failed_statuses = False
+        has_native_failed_status = False
         for status in statuses:
             print(f"Check status [{status.context}], [{status.state}]")
             if CI.is_required(status.context) and status.state != SUCCESS:
                 print(f"WARNING: Failed status [{status.context}], [{status.state}]")
                 has_failed_statuses = True
+                if status.context != CI.StatusNames.SYNC:
+                    has_native_failed_status = True
 
         if args.wf_status == SUCCESS or has_failed_statuses:
             # set Mergeable check if workflow is successful (green)
@@ -280,7 +283,7 @@ def main():
             print(
                 "Workflow failed but no failed statuses found (died runner?) - cannot set Mergeable Check status"
             )
-        if args.wf_status == SUCCESS and not has_failed_statuses:
+        if args.wf_status == SUCCESS and not has_native_failed_status:
             sys.exit(0)
         else:
             sys.exit(1)
diff --git a/tests/ci/test_ci_config.py b/tests/ci/test_ci_config.py
index 12e863c4d8d..10867ea1444 100644
--- a/tests/ci/test_ci_config.py
+++ b/tests/ci/test_ci_config.py
@@ -587,11 +587,11 @@ class TestCIConfig(unittest.TestCase):
         for job, job_config in ci_cache.jobs_to_do.items():
             if job in MOCK_AFFECTED_JOBS:
                 MOCK_REQUIRED_BUILDS += job_config.required_builds
-            elif job not in MOCK_AFFECTED_JOBS:
+            elif job not in MOCK_AFFECTED_JOBS and not job_config.disable_await:
                 ci_cache.jobs_to_wait[job] = job_config
 
         for job, job_config in ci_cache.jobs_to_do.items():
-            if job_config.reference_job_name:
+            if job_config.reference_job_name or job_config.disable_await:
                 # jobs with reference_job_name in config are not supposed to have records in the cache - continue
                 continue
             if job in MOCK_AFFECTED_JOBS:
@@ -624,11 +624,73 @@ class TestCIConfig(unittest.TestCase):
             + MOCK_AFFECTED_JOBS
             + MOCK_REQUIRED_BUILDS
         )
+        self.assertTrue(CI.JobNames.BUILD_CHECK not in ci_cache.jobs_to_wait, "We must never await on Builds Report")
         self.assertCountEqual(
             list(ci_cache.jobs_to_wait),
-            [
-                CI.JobNames.BUILD_CHECK,
-            ]
-            + MOCK_REQUIRED_BUILDS,
+            MOCK_REQUIRED_BUILDS,
+        )
+        self.assertCountEqual(list(ci_cache.jobs_to_do), expected_to_do)
+
+    def test_ci_py_filters_not_affected_jobs_in_prs_no_builds(self):
+        """
+        checks ci.py filters not affected jobs in PRs, no builds required
+        """
+        settings = CiSettings()
+        settings.no_ci_cache = True
+        pr_info = PRInfo(github_event=_TEST_EVENT_JSON)
+        pr_info.event_type = EventType.PULL_REQUEST
+        pr_info.number = 123
+        assert pr_info.is_pr
+        ci_cache = CIPY._configure_jobs(
+            S3Helper(), pr_info, settings, skip_jobs=False, dry_run=True
+        )
+        self.assertTrue(not ci_cache.jobs_to_skip, "Must be no jobs in skip list")
+        assert not ci_cache.jobs_to_wait
+        assert not ci_cache.jobs_to_skip
+
+        MOCK_AFFECTED_JOBS = [
+            CI.JobNames.DOCS_CHECK,
+        ]
+        MOCK_REQUIRED_BUILDS = []
+
+        # pretend there are pending jobs that we need to wait
+        for job, job_config in ci_cache.jobs_to_do.items():
+            if job in MOCK_AFFECTED_JOBS:
+                if job_config.required_builds:
+                    MOCK_REQUIRED_BUILDS += job_config.required_builds
+            elif job not in MOCK_AFFECTED_JOBS and not job_config.disable_await:
+                ci_cache.jobs_to_wait[job] = job_config
+
+        for job, job_config in ci_cache.jobs_to_do.items():
+            if job_config.reference_job_name or job_config.disable_await:
+                # jobs with reference_job_name in config are not supposed to have records in the cache - continue
+                continue
+            if job in MOCK_AFFECTED_JOBS:
+                continue
+            for batch in range(job_config.num_batches):
+                # add any record into cache
+                record = CiCache.Record(
+                    record_type=random.choice(
+                        [
+                            CiCache.RecordType.FAILED,
+                            CiCache.RecordType.PENDING,
+                            CiCache.RecordType.SUCCESSFUL,
+                        ]
+                    ),
+                    job_name=job,
+                    job_digest=ci_cache.job_digests[job],
+                    batch=batch,
+                    num_batches=job_config.num_batches,
+                    release_branch=True,
+                )
+                for record_t_, records_ in ci_cache.records.items():
+                    if record_t_.value == CiCache.RecordType.FAILED.value:
+                        records_[record.to_str_key()] = record
+
+        ci_cache.filter_out_not_affected_jobs()
+        expected_to_do = MOCK_AFFECTED_JOBS + MOCK_REQUIRED_BUILDS
+        self.assertCountEqual(
+            list(ci_cache.jobs_to_wait),
+            MOCK_REQUIRED_BUILDS,
         )
         self.assertCountEqual(list(ci_cache.jobs_to_do), expected_to_do)

From 19f8e1768f5095618c4e0cded2c8299968aaf0fb Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitlibar@clickhouse.com>
Date: Sat, 20 Jul 2024 16:36:19 +0200
Subject: [PATCH 0529/1488] Add const to cycles.

---
 src/Databases/DatabaseReplicated.cpp        | 2 +-
 src/Interpreters/InterpreterCreateQuery.cpp | 2 +-
 src/Parsers/ASTViewTargets.cpp              | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp
index 25d1ad90a3c..4c079ae5300 100644
--- a/src/Databases/DatabaseReplicated.cpp
+++ b/src/Databases/DatabaseReplicated.cpp
@@ -734,7 +734,7 @@ void DatabaseReplicated::checkQueryValid(const ASTPtr & query, ContextPtr query_
 
             if (create->targets)
             {
-                for (auto inner_table_engine : create->targets->getInnerEngines())
+                for (const auto & inner_table_engine : create->targets->getInnerEngines())
                     checkTableEngine(*create, *inner_table_engine, query_context);
             }
         }
diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp
index faa91341a7c..342374aa580 100644
--- a/src/Interpreters/InterpreterCreateQuery.cpp
+++ b/src/Interpreters/InterpreterCreateQuery.cpp
@@ -1384,7 +1384,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create)
 
         if (create.targets)
         {
-            for (auto inner_table_engine : create.targets->getInnerEngines())
+            for (const auto & inner_table_engine : create.targets->getInnerEngines())
             {
                 if (isReplicated(*inner_table_engine))
                     is_storage_replicated = true;
diff --git a/src/Parsers/ASTViewTargets.cpp b/src/Parsers/ASTViewTargets.cpp
index 38f103b6e55..8ee98e704df 100644
--- a/src/Parsers/ASTViewTargets.cpp
+++ b/src/Parsers/ASTViewTargets.cpp
@@ -43,7 +43,7 @@ std::vector<ViewTarget::Kind> ASTViewTargets::getKinds() const
 {
     std::vector<ViewTarget::Kind> kinds;
     kinds.reserve(targets.size());
-    for (auto & target : targets)
+    for (const auto & target : targets)
         kinds.push_back(target.kind);
     return kinds;
 }
@@ -121,7 +121,7 @@ void ASTViewTargets::resetInnerUUIDs()
 
 bool ASTViewTargets::hasInnerUUIDs() const
 {
-    for (auto & target : targets)
+    for (const auto & target : targets)
     {
         if (target.inner_uuid != UUIDHelpers::Nil)
             return true;

From c61581ca69757ce9c1d15b6ccc139d48d27f5b07 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 21 Jul 2024 12:06:53 +0200
Subject: [PATCH 0530/1488] Fix error

---
 src/Common/TraceSender.cpp | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/Common/TraceSender.cpp b/src/Common/TraceSender.cpp
index 064da1b3d76..f1adf7c516a 100644
--- a/src/Common/TraceSender.cpp
+++ b/src/Common/TraceSender.cpp
@@ -26,11 +26,16 @@ LazyPipeFDs TraceSender::pipe;
 static thread_local bool inside_send = false;
 void TraceSender::send(TraceType trace_type, const StackTrace & stack_trace, Extras extras)
 {
-    DENY_ALLOCATIONS_IN_SCOPE;
-
+    /** The method shouldn't be called recursively or throw exceptions.
+      * There are several reasons:
+      * - avoid infinite recursion when some of subsequent functions invoke tracing;
+      * - avoid inconsistent writes if the method was interrupted by a signal handler in the middle of writing,
+      *   and then another tracing is invoked (e.g., from query profiler).
+      */
     if (unlikely(inside_send))
-        abort(); /// The method shouldn't be called recursively or throw exceptions.
+        return;
     inside_send = true;
+    DENY_ALLOCATIONS_IN_SCOPE;
 
     constexpr size_t buf_size = sizeof(char) /// TraceCollector stop flag
         + sizeof(UInt8)                      /// String size

From 0c2c027af63fcbababffbe3a39ed2631884e1938 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 21 Jul 2024 12:30:20 +0200
Subject: [PATCH 0531/1488] Remove bad tests @azat

---
 ...2_part_log_rmt_fetch_merge_error.reference | 10 -----
 .../03002_part_log_rmt_fetch_merge_error.sql  | 35 ----------------
 ..._part_log_rmt_fetch_mutate_error.reference | 10 -----
 .../03002_part_log_rmt_fetch_mutate_error.sql | 41 -------------------
 4 files changed, 96 deletions(-)
 delete mode 100644 tests/queries/0_stateless/03002_part_log_rmt_fetch_merge_error.reference
 delete mode 100644 tests/queries/0_stateless/03002_part_log_rmt_fetch_merge_error.sql
 delete mode 100644 tests/queries/0_stateless/03002_part_log_rmt_fetch_mutate_error.reference
 delete mode 100644 tests/queries/0_stateless/03002_part_log_rmt_fetch_mutate_error.sql

diff --git a/tests/queries/0_stateless/03002_part_log_rmt_fetch_merge_error.reference b/tests/queries/0_stateless/03002_part_log_rmt_fetch_merge_error.reference
deleted file mode 100644
index b19d389d8d0..00000000000
--- a/tests/queries/0_stateless/03002_part_log_rmt_fetch_merge_error.reference
+++ /dev/null
@@ -1,10 +0,0 @@
-before
-rmt_master	NewPart	0	1
-rmt_master	MergeParts	0	1
-rmt_slave	MergeParts	1	0
-rmt_slave	DownloadPart	0	1
-after
-rmt_master	NewPart	0	1
-rmt_master	MergeParts	0	1
-rmt_slave	MergeParts	1	0
-rmt_slave	DownloadPart	0	2
diff --git a/tests/queries/0_stateless/03002_part_log_rmt_fetch_merge_error.sql b/tests/queries/0_stateless/03002_part_log_rmt_fetch_merge_error.sql
deleted file mode 100644
index 548a8e5570a..00000000000
--- a/tests/queries/0_stateless/03002_part_log_rmt_fetch_merge_error.sql
+++ /dev/null
@@ -1,35 +0,0 @@
--- Tags: no-replicated-database, no-parallel, no-shared-merge-tree
--- SMT: The merge process is completely different from RMT
-
-drop table if exists rmt_master;
-drop table if exists rmt_slave;
-
-create table rmt_master (key Int) engine=ReplicatedMergeTree('/clickhouse/{database}', 'master') order by key settings always_fetch_merged_part=0;
--- always_fetch_merged_part=1, consider this table as a "slave"
-create table rmt_slave (key Int) engine=ReplicatedMergeTree('/clickhouse/{database}', 'slave') order by key settings always_fetch_merged_part=1;
-
-insert into rmt_master values (1);
-
-system sync replica rmt_master;
-system sync replica rmt_slave;
-system stop replicated sends rmt_master;
-optimize table rmt_master final settings alter_sync=1, optimize_throw_if_noop=1;
-
-select sleep(3) format Null;
-
-system flush logs;
-select 'before';
-select table, event_type, error>0, countIf(error=0) from system.part_log where database = currentDatabase() group by 1, 2, 3 order by 1, 2, 3;
-
-system start replicated sends rmt_master;
--- sleep few seconds to try rmt_slave to fetch the part and reflect this error
--- in system.part_log
-select sleep(3) format Null;
-system sync replica rmt_slave;
-
-system flush logs;
-select 'after';
-select table, event_type, error>0, countIf(error=0) from system.part_log where database = currentDatabase() group by 1, 2, 3 order by 1, 2, 3;
-
-drop table rmt_master;
-drop table rmt_slave;
diff --git a/tests/queries/0_stateless/03002_part_log_rmt_fetch_mutate_error.reference b/tests/queries/0_stateless/03002_part_log_rmt_fetch_mutate_error.reference
deleted file mode 100644
index aac9e7527d1..00000000000
--- a/tests/queries/0_stateless/03002_part_log_rmt_fetch_mutate_error.reference
+++ /dev/null
@@ -1,10 +0,0 @@
-before
-rmt_master	NewPart	0	1
-rmt_master	MutatePart	0	1
-rmt_slave	DownloadPart	0	1
-rmt_slave	MutatePart	1	0
-after
-rmt_master	NewPart	0	1
-rmt_master	MutatePart	0	1
-rmt_slave	DownloadPart	0	2
-rmt_slave	MutatePart	1	0
diff --git a/tests/queries/0_stateless/03002_part_log_rmt_fetch_mutate_error.sql b/tests/queries/0_stateless/03002_part_log_rmt_fetch_mutate_error.sql
deleted file mode 100644
index d8b5ebb3148..00000000000
--- a/tests/queries/0_stateless/03002_part_log_rmt_fetch_mutate_error.sql
+++ /dev/null
@@ -1,41 +0,0 @@
--- Tags: no-replicated-database, no-parallel, no-shared-merge-tree
--- SMT: The merge process is completely different from RMT
-
-drop table if exists rmt_master;
-drop table if exists rmt_slave;
-
-create table rmt_master (key Int) engine=ReplicatedMergeTree('/clickhouse/{database}', 'master') order by tuple() settings always_fetch_merged_part=0, old_parts_lifetime=600;
--- prefer_fetch_merged_part_*_threshold=0, consider this table as a "slave"
-create table rmt_slave (key Int) engine=ReplicatedMergeTree('/clickhouse/{database}', 'slave') order by tuple() settings prefer_fetch_merged_part_time_threshold=0, prefer_fetch_merged_part_size_threshold=0, old_parts_lifetime=600;
-
-insert into rmt_master values (1);
-
-system sync replica rmt_master;
-system sync replica rmt_slave;
-system stop replicated sends rmt_master;
-system stop pulling replication log rmt_slave;
-alter table rmt_master update key=key+100 where 1 settings alter_sync=1;
-
--- first we need to make the rmt_master execute mutation so that it will have
--- the part, and rmt_slave will consider it instead of performing mutation on
--- it's own, otherwise prefer_fetch_merged_part_*_threshold will be simply ignored
-select sleep(3) format Null;
-system start pulling replication log rmt_slave;
--- and sleep few more seconds to try rmt_slave to fetch the part and reflect
--- this error in system.part_log
-select sleep(3) format Null;
-
-system flush logs;
-select 'before';
-select table, event_type, error>0, countIf(error=0) from system.part_log where database = currentDatabase() group by 1, 2, 3 order by 1, 2, 3;
-
-system start replicated sends rmt_master;
-select sleep(3) format Null;
-system sync replica rmt_slave;
-
-system flush logs;
-select 'after';
-select table, event_type, error>0, countIf(error=0) from system.part_log where database = currentDatabase() group by 1, 2, 3 order by 1, 2, 3;
-
-drop table rmt_master;
-drop table rmt_slave;

From 02bf9e4243c329611fc0be43432d3f9c290512d5 Mon Sep 17 00:00:00 2001
From: Max K <maxkaynov@gmail.com>
Date: Sun, 21 Jul 2024 12:46:58 +0200
Subject: [PATCH 0532/1488] push pending records before await call

---
 tests/ci/ci.py             |  5 +-
 tests/ci/ci_cache.py       | 93 +++++++++++++++++++++++++++++++++++++-
 tests/ci/test_ci_config.py |  5 +-
 3 files changed, 99 insertions(+), 4 deletions(-)

diff --git a/tests/ci/ci.py b/tests/ci/ci.py
index ff31d1ce489..f6bec5304a0 100644
--- a/tests/ci/ci.py
+++ b/tests/ci/ci.py
@@ -1110,13 +1110,14 @@ def main() -> int:
             ci_cache.print_status()
 
         if IS_CI and not pr_info.is_merge_queue:
-            # wait for pending jobs to be finished, await_jobs is a long blocking call
-            ci_cache.await_pending_jobs(pr_info.is_release)
 
             if pr_info.is_release:
                 print("Release/master: CI Cache add pending records for all todo jobs")
                 ci_cache.push_pending_all(pr_info.is_release)
 
+            # wait for pending jobs to be finished, await_jobs is a long blocking call
+            ci_cache.await_pending_jobs(pr_info.is_release)
+
         # conclude results
         result["git_ref"] = git_ref
         result["version"] = version
diff --git a/tests/ci/ci_cache.py b/tests/ci/ci_cache.py
index 16b6eac1ecb..85eabb84f9f 100644
--- a/tests/ci/ci_cache.py
+++ b/tests/ci/ci_cache.py
@@ -638,7 +638,14 @@ class CiCache:
         pushes pending records for all jobs that supposed to be run
         """
         for job, job_config in self.jobs_to_do.items():
-            if not job_config.has_digest() or job_config.disable_await:
+            if (
+                job in self.jobs_to_wait
+                or not job_config.has_digest()
+                or job_config.disable_await
+            ):
+                # 1. "job in self.jobs_to_wait" - this job already has a pending record in cache
+                # 2. "not job_config.has_digest()" - cache is not used for these jobs
+                # 3. "job_config.disable_await" - await is explicitly disabled
                 continue
             pending_state = PendingState(time.time(), run_url=GITHUB_RUN_URL)
             assert job_config.batches
@@ -899,3 +906,87 @@ class CiCache:
                 self.jobs_to_wait[job] = job_config
 
         return self
+
+
+if __name__ == "__main__":
+    # for testing
+    job_digest = {
+        "package_release": "bbbd3519d1",
+        "package_aarch64": "bbbd3519d1",
+        "package_asan": "bbbd3519d1",
+        "package_ubsan": "bbbd3519d1",
+        "package_tsan": "bbbd3519d1",
+        "package_msan": "bbbd3519d1",
+        "package_debug": "bbbd3519d1",
+        "package_release_coverage": "bbbd3519d1",
+        "binary_release": "bbbd3519d1",
+        "binary_tidy": "bbbd3519d1",
+        "binary_darwin": "bbbd3519d1",
+        "binary_aarch64": "bbbd3519d1",
+        "binary_aarch64_v80compat": "bbbd3519d1",
+        "binary_freebsd": "bbbd3519d1",
+        "binary_darwin_aarch64": "bbbd3519d1",
+        "binary_ppc64le": "bbbd3519d1",
+        "binary_amd64_compat": "bbbd3519d1",
+        "binary_amd64_musl": "bbbd3519d1",
+        "binary_riscv64": "bbbd3519d1",
+        "binary_s390x": "bbbd3519d1",
+        "binary_loongarch64": "bbbd3519d1",
+        "Builds": "f5dffeecb8",
+        "Install packages (release)": "ba0c89660e",
+        "Install packages (aarch64)": "ba0c89660e",
+        "Stateful tests (asan)": "32a9a1aba9",
+        "Stateful tests (tsan)": "32a9a1aba9",
+        "Stateful tests (msan)": "32a9a1aba9",
+        "Stateful tests (ubsan)": "32a9a1aba9",
+        "Stateful tests (debug)": "32a9a1aba9",
+        "Stateful tests (release)": "32a9a1aba9",
+        "Stateful tests (coverage)": "32a9a1aba9",
+        "Stateful tests (aarch64)": "32a9a1aba9",
+        "Stateful tests (release, ParallelReplicas)": "32a9a1aba9",
+        "Stateful tests (debug, ParallelReplicas)": "32a9a1aba9",
+        "Stateless tests (asan)": "deb6778b88",
+        "Stateless tests (tsan)": "deb6778b88",
+        "Stateless tests (msan)": "deb6778b88",
+        "Stateless tests (ubsan)": "deb6778b88",
+        "Stateless tests (debug)": "deb6778b88",
+        "Stateless tests (release)": "deb6778b88",
+        "Stateless tests (coverage)": "deb6778b88",
+        "Stateless tests (aarch64)": "deb6778b88",
+        "Stateless tests (release, old analyzer, s3, DatabaseReplicated)": "deb6778b88",
+        "Stateless tests (debug, s3 storage)": "deb6778b88",
+        "Stateless tests (tsan, s3 storage)": "deb6778b88",
+        "Stress test (debug)": "aa298abf10",
+        "Stress test (tsan)": "aa298abf10",
+        "Upgrade check (debug)": "5ce4d3ee02",
+        "Integration tests (asan, old analyzer)": "42e58be3aa",
+        "Integration tests (tsan)": "42e58be3aa",
+        "Integration tests (aarch64)": "42e58be3aa",
+        "Integration tests flaky check (asan)": "42e58be3aa",
+        "Compatibility check (release)": "ecb69d8c4b",
+        "Compatibility check (aarch64)": "ecb69d8c4b",
+        "Unit tests (release)": "09d00b702e",
+        "Unit tests (asan)": "09d00b702e",
+        "Unit tests (msan)": "09d00b702e",
+        "Unit tests (tsan)": "09d00b702e",
+        "Unit tests (ubsan)": "09d00b702e",
+        "AST fuzzer (debug)": "c38ebf947f",
+        "AST fuzzer (asan)": "c38ebf947f",
+        "AST fuzzer (msan)": "c38ebf947f",
+        "AST fuzzer (tsan)": "c38ebf947f",
+        "AST fuzzer (ubsan)": "c38ebf947f",
+        "Stateless tests flaky check (asan)": "deb6778b88",
+        "Performance Comparison (release)": "a8a7179258",
+        "ClickBench (release)": "45c07c4aa6",
+        "ClickBench (aarch64)": "45c07c4aa6",
+        "Docker server image": "6a24d5b187",
+        "Docker keeper image": "6a24d5b187",
+        "Docs check": "4764154c62",
+        "Fast test": "cb269133f2",
+        "Style check": "ffffffffff",
+        "Stateful tests (ubsan, ParallelReplicas)": "32a9a1aba9",
+        "Stress test (msan)": "aa298abf10",
+        "Upgrade check (asan)": "5ce4d3ee02",
+    }
+    ci_cache = CiCache(job_digests=job_digest, cache_enabled=True, s3=S3Helper())
+    ci_cache.update()
diff --git a/tests/ci/test_ci_config.py b/tests/ci/test_ci_config.py
index 10867ea1444..04fd44a87e9 100644
--- a/tests/ci/test_ci_config.py
+++ b/tests/ci/test_ci_config.py
@@ -624,7 +624,10 @@ class TestCIConfig(unittest.TestCase):
             + MOCK_AFFECTED_JOBS
             + MOCK_REQUIRED_BUILDS
         )
-        self.assertTrue(CI.JobNames.BUILD_CHECK not in ci_cache.jobs_to_wait, "We must never await on Builds Report")
+        self.assertTrue(
+            CI.JobNames.BUILD_CHECK not in ci_cache.jobs_to_wait,
+            "We must never await on Builds Report",
+        )
         self.assertCountEqual(
             list(ci_cache.jobs_to_wait),
             MOCK_REQUIRED_BUILDS,

From d83428daafc665542023fac0a9add048603ad224 Mon Sep 17 00:00:00 2001
From: Max K <maxkaynov@gmail.com>
Date: Sun, 21 Jul 2024 14:52:33 +0200
Subject: [PATCH 0533/1488] fix in ci unittests

---
 tests/ci/test_ci_config.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/ci/test_ci_config.py b/tests/ci/test_ci_config.py
index 04fd44a87e9..44142050821 100644
--- a/tests/ci/test_ci_config.py
+++ b/tests/ci/test_ci_config.py
@@ -652,7 +652,7 @@ class TestCIConfig(unittest.TestCase):
         assert not ci_cache.jobs_to_skip
 
         MOCK_AFFECTED_JOBS = [
-            CI.JobNames.DOCS_CHECK,
+            CI.JobNames.FAST_TEST,
         ]
         MOCK_REQUIRED_BUILDS = []
 

From 09e4faf2dbadfdf1eaedc0eec127098c6b9540f1 Mon Sep 17 00:00:00 2001
From: jsc0218 <jsc0218@gmail.com>
Date: Sun, 21 Jul 2024 15:05:26 +0000
Subject: [PATCH 0534/1488] fix

---
 src/Interpreters/InterpreterOptimizeQuery.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/Interpreters/InterpreterOptimizeQuery.cpp b/src/Interpreters/InterpreterOptimizeQuery.cpp
index 3bee235185d..907a01b0432 100644
--- a/src/Interpreters/InterpreterOptimizeQuery.cpp
+++ b/src/Interpreters/InterpreterOptimizeQuery.cpp
@@ -20,7 +20,6 @@ namespace DB
 namespace ErrorCodes
 {
     extern const int THERE_IS_NO_COLUMN;
-    extern const int NOT_IMPLEMENTED;
 }
 
 

From 4b8b9b1503800398ccbf68e6a6134838fd67dac5 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Sun, 21 Jul 2024 16:30:59 +0000
Subject: [PATCH 0535/1488] Fix UB in function "age"

---
 src/Functions/DateTimeTransforms.h                       | 5 ++++-
 tests/queries/0_stateless/02477_age_datetime64.reference | 5 +++++
 tests/queries/0_stateless/02477_age_datetime64.sql       | 4 ++++
 3 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/src/Functions/DateTimeTransforms.h b/src/Functions/DateTimeTransforms.h
index 34c59ecab08..5f745f3ccad 100644
--- a/src/Functions/DateTimeTransforms.h
+++ b/src/Functions/DateTimeTransforms.h
@@ -1954,7 +1954,10 @@ struct ToRelativeSubsecondNumImpl
             return t.value;
         if (scale > scale_multiplier)
             return t.value / (scale / scale_multiplier);
-        return t.value * (scale_multiplier / scale);
+        return static_cast<UInt128>(t.value) * static_cast<UInt128>((scale_multiplier / scale));
+        /// Casting ^^: All integers are Int64, yet if t.value is big enough the multiplication can still
+        /// overflow which is UB. This place is too low-level and generic to check if t.value is sane.
+        /// Therefore just let it overflow safely and don't bother further.
     }
     static Int64 execute(UInt32 t, const DateLUTImpl &)
     {
diff --git a/tests/queries/0_stateless/02477_age_datetime64.reference b/tests/queries/0_stateless/02477_age_datetime64.reference
index 3b4459dd26d..fb085f461c9 100644
--- a/tests/queries/0_stateless/02477_age_datetime64.reference
+++ b/tests/queries/0_stateless/02477_age_datetime64.reference
@@ -111,3 +111,8 @@ SELECT age('day', materialize(toDateTime64('2015-08-18 00:00:00', 0, 'UTC')), ma
 1
 SELECT age('day', materialize(toDate('2015-08-18', 'UTC')), materialize(toDateTime64('2015-08-19 00:00:00', 3, 'UTC')));
 1
+-- UBsan bug #66638
+set session_timezone = 'UTC';
+SELECT age('second', toDateTime(1157339245694594829, 6, 'UTC'), toDate('2015-08-18'))
+
+-8973935999
diff --git a/tests/queries/0_stateless/02477_age_datetime64.sql b/tests/queries/0_stateless/02477_age_datetime64.sql
index 1bed93991ca..b5fa4da8837 100644
--- a/tests/queries/0_stateless/02477_age_datetime64.sql
+++ b/tests/queries/0_stateless/02477_age_datetime64.sql
@@ -75,3 +75,7 @@ SELECT age('second', materialize(toDateTime64('2015-08-18 00:00:00', 0, 'UTC')),
 SELECT age('second', materialize(toDateTime('2015-08-18 00:00:00', 'UTC')), materialize(toDateTime64('2015-08-18 00:00:10', 3, 'UTC')));
 SELECT age('day', materialize(toDateTime64('2015-08-18 00:00:00', 0, 'UTC')), materialize(toDate('2015-08-19', 'UTC')));
 SELECT age('day', materialize(toDate('2015-08-18', 'UTC')), materialize(toDateTime64('2015-08-19 00:00:00', 3, 'UTC')));
+
+-- UBsan bug #66638
+set session_timezone = 'UTC';
+SELECT age('second', toDateTime(1157339245694594829, 6, 'UTC'), toDate('2015-08-18'))

From 8786d9b5dd5f93d5c1f22c4d618093d69f8d57a3 Mon Sep 17 00:00:00 2001
From: joelynch <joelynch112@gmail.com>
Date: Sun, 21 Jul 2024 20:00:38 +0200
Subject: [PATCH 0536/1488] Ensure COMMENT clause works for all table engines

---
 src/Databases/SQLite/DatabaseSQLite.cpp                |  1 +
 src/Storages/Kafka/StorageKafka.cpp                    | 10 +++++++---
 src/Storages/Kafka/StorageKafka.h                      |  1 +
 src/Storages/NATS/StorageNATS.cpp                      |  4 +++-
 src/Storages/NATS/StorageNATS.h                        |  1 +
 .../PostgreSQL/StorageMaterializedPostgreSQL.cpp       |  1 +
 src/Storages/RabbitMQ/StorageRabbitMQ.cpp              |  4 +++-
 src/Storages/RabbitMQ/StorageRabbitMQ.h                |  1 +
 src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp        |  1 +
 src/Storages/StorageExecutable.cpp                     |  7 ++++---
 src/Storages/StorageExecutable.h                       |  3 ++-
 src/Storages/StorageKeeperMap.cpp                      |  1 +
 src/Storages/StorageSQLite.cpp                         |  4 +++-
 src/Storages/StorageSQLite.h                           |  1 +
 src/Storages/WindowView/StorageWindowView.cpp          |  4 +++-
 src/Storages/WindowView/StorageWindowView.h            |  1 +
 src/TableFunctions/TableFunctionExecutable.cpp         |  9 ++++++++-
 src/TableFunctions/TableFunctionSQLite.cpp             |  2 +-
 18 files changed, 43 insertions(+), 13 deletions(-)

diff --git a/src/Databases/SQLite/DatabaseSQLite.cpp b/src/Databases/SQLite/DatabaseSQLite.cpp
index 132a978140c..471730fce29 100644
--- a/src/Databases/SQLite/DatabaseSQLite.cpp
+++ b/src/Databases/SQLite/DatabaseSQLite.cpp
@@ -154,6 +154,7 @@ StoragePtr DatabaseSQLite::fetchTable(const String & table_name, ContextPtr loca
         table_name,
         ColumnsDescription{*columns},
         ConstraintsDescription{},
+        /* comment = */ "",
         local_context);
 
     return storage;
diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp
index 809401bb279..3aad64a0cfb 100644
--- a/src/Storages/Kafka/StorageKafka.cpp
+++ b/src/Storages/Kafka/StorageKafka.cpp
@@ -418,8 +418,11 @@ namespace
 }
 
 StorageKafka::StorageKafka(
-    const StorageID & table_id_, ContextPtr context_,
-    const ColumnsDescription & columns_, std::unique_ptr<KafkaSettings> kafka_settings_,
+    const StorageID & table_id_,
+    ContextPtr context_,
+    const ColumnsDescription & columns_,
+    const String & comment,
+    std::unique_ptr<KafkaSettings> kafka_settings_,
     const String & collection_name_)
     : IStorage(table_id_)
     , WithContext(context_->getGlobalContext())
@@ -451,6 +454,7 @@ StorageKafka::StorageKafka(
 
     StorageInMemoryMetadata storage_metadata;
     storage_metadata.setColumns(columns_);
+    storage_metadata.setComment(comment);
     setInMemoryMetadata(storage_metadata);
     setVirtuals(createVirtuals(kafka_settings->kafka_handle_error_mode));
 
@@ -1317,7 +1321,7 @@ void registerStorageKafka(StorageFactory & factory)
                                                        "See https://clickhouse.com/docs/en/engines/table-engines/integrations/kafka/#configuration");
         }
 
-        return std::make_shared<StorageKafka>(args.table_id, args.getContext(), args.columns, std::move(kafka_settings), collection_name);
+        return std::make_shared<StorageKafka>(args.table_id, args.getContext(), args.columns, args.comment, std::move(kafka_settings), collection_name);
     };
 
     factory.registerStorage("Kafka", creator_fn, StorageFactory::StorageFeatures{ .supports_settings = true, });
diff --git a/src/Storages/Kafka/StorageKafka.h b/src/Storages/Kafka/StorageKafka.h
index fa4affbda36..31e1a6076b6 100644
--- a/src/Storages/Kafka/StorageKafka.h
+++ b/src/Storages/Kafka/StorageKafka.h
@@ -40,6 +40,7 @@ public:
         const StorageID & table_id_,
         ContextPtr context_,
         const ColumnsDescription & columns_,
+        const String & comment,
         std::unique_ptr<KafkaSettings> kafka_settings_,
         const String & collection_name_);
 
diff --git a/src/Storages/NATS/StorageNATS.cpp b/src/Storages/NATS/StorageNATS.cpp
index 8f0e2d76473..9d728c3395f 100644
--- a/src/Storages/NATS/StorageNATS.cpp
+++ b/src/Storages/NATS/StorageNATS.cpp
@@ -49,6 +49,7 @@ StorageNATS::StorageNATS(
     const StorageID & table_id_,
     ContextPtr context_,
     const ColumnsDescription & columns_,
+    const String & comment,
     std::unique_ptr<NATSSettings> nats_settings_,
     LoadingStrictnessLevel mode)
     : IStorage(table_id_)
@@ -87,6 +88,7 @@ StorageNATS::StorageNATS(
 
     StorageInMemoryMetadata storage_metadata;
     storage_metadata.setColumns(columns_);
+    storage_metadata.setComment(comment);
     setInMemoryMetadata(storage_metadata);
     setVirtuals(createVirtuals(nats_settings->nats_handle_error_mode));
 
@@ -760,7 +762,7 @@ void registerStorageNATS(StorageFactory & factory)
         if (!nats_settings->nats_subjects.changed)
             throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "You must specify `nats_subjects` setting");
 
-        return std::make_shared<StorageNATS>(args.table_id, args.getContext(), args.columns, std::move(nats_settings), args.mode);
+        return std::make_shared<StorageNATS>(args.table_id, args.getContext(), args.columns, args.comment, std::move(nats_settings), args.mode);
     };
 
     factory.registerStorage("NATS", creator_fn, StorageFactory::StorageFeatures{ .supports_settings = true, });
diff --git a/src/Storages/NATS/StorageNATS.h b/src/Storages/NATS/StorageNATS.h
index 41d77acfde6..5fca8cb0163 100644
--- a/src/Storages/NATS/StorageNATS.h
+++ b/src/Storages/NATS/StorageNATS.h
@@ -23,6 +23,7 @@ public:
         const StorageID & table_id_,
         ContextPtr context_,
         const ColumnsDescription & columns_,
+        const String & comment,
         std::unique_ptr<NATSSettings> nats_settings_,
         LoadingStrictnessLevel mode);
 
diff --git a/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp b/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp
index a904b29e12f..f4c38a52a3f 100644
--- a/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp
+++ b/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp
@@ -571,6 +571,7 @@ void registerStorageMaterializedPostgreSQL(StorageFactory & factory)
         StorageInMemoryMetadata metadata;
         metadata.setColumns(args.columns);
         metadata.setConstraints(args.constraints);
+        metadata.setComment(args.comment);
 
         if (args.mode <= LoadingStrictnessLevel::CREATE
             && !args.getLocalContext()->getSettingsRef().allow_experimental_materialized_postgresql_table)
diff --git a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp
index f3d2aff68c8..9e3c40071b5 100644
--- a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp
+++ b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp
@@ -70,6 +70,7 @@ StorageRabbitMQ::StorageRabbitMQ(
         const StorageID & table_id_,
         ContextPtr context_,
         const ColumnsDescription & columns_,
+        const String & comment,
         std::unique_ptr<RabbitMQSettings> rabbitmq_settings_,
         LoadingStrictnessLevel mode)
         : IStorage(table_id_)
@@ -145,6 +146,7 @@ StorageRabbitMQ::StorageRabbitMQ(
 
     StorageInMemoryMetadata storage_metadata;
     storage_metadata.setColumns(columns_);
+    storage_metadata.setComment(comment);
     setInMemoryMetadata(storage_metadata);
     setVirtuals(createVirtuals(rabbitmq_settings->rabbitmq_handle_error_mode));
 
@@ -1288,7 +1290,7 @@ void registerStorageRabbitMQ(StorageFactory & factory)
         if (!rabbitmq_settings->rabbitmq_format.changed)
             throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "You must specify `rabbitmq_format` setting");
 
-        return std::make_shared<StorageRabbitMQ>(args.table_id, args.getContext(), args.columns, std::move(rabbitmq_settings), args.mode);
+        return std::make_shared<StorageRabbitMQ>(args.table_id, args.getContext(), args.columns, args.comment, std::move(rabbitmq_settings), args.mode);
     };
 
     factory.registerStorage("RabbitMQ", creator_fn, StorageFactory::StorageFeatures{ .supports_settings = true, });
diff --git a/src/Storages/RabbitMQ/StorageRabbitMQ.h b/src/Storages/RabbitMQ/StorageRabbitMQ.h
index b8fab5825e4..fed80a4357b 100644
--- a/src/Storages/RabbitMQ/StorageRabbitMQ.h
+++ b/src/Storages/RabbitMQ/StorageRabbitMQ.h
@@ -26,6 +26,7 @@ public:
             const StorageID & table_id_,
             ContextPtr context_,
             const ColumnsDescription & columns_,
+            const String & comment,
             std::unique_ptr<RabbitMQSettings> rabbitmq_settings_,
             LoadingStrictnessLevel mode);
 
diff --git a/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp b/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp
index 409703c84c6..fafc72da04e 100644
--- a/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp
+++ b/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp
@@ -691,6 +691,7 @@ static StoragePtr create(const StorageFactory::Arguments & args)
     StorageInMemoryMetadata metadata;
     metadata.setColumns(args.columns);
     metadata.setConstraints(args.constraints);
+    metadata.setComment(args.comment);
 
     if (!args.storage_def->primary_key)
         throw Exception(ErrorCodes::BAD_ARGUMENTS, "StorageEmbeddedRocksDB must require one column in primary key");
diff --git a/src/Storages/StorageExecutable.cpp b/src/Storages/StorageExecutable.cpp
index 381c20c616d..0094723e3fd 100644
--- a/src/Storages/StorageExecutable.cpp
+++ b/src/Storages/StorageExecutable.cpp
@@ -77,7 +77,8 @@ StorageExecutable::StorageExecutable(
     const ExecutableSettings & settings_,
     const std::vector<ASTPtr> & input_queries_,
     const ColumnsDescription & columns,
-    const ConstraintsDescription & constraints)
+    const ConstraintsDescription & constraints,
+    const String & comment)
     : IStorage(table_id_)
     , settings(settings_)
     , input_queries(input_queries_)
@@ -86,6 +87,7 @@ StorageExecutable::StorageExecutable(
     StorageInMemoryMetadata storage_metadata;
     storage_metadata.setColumns(columns);
     storage_metadata.setConstraints(constraints);
+    storage_metadata.setComment(comment);
     setInMemoryMetadata(storage_metadata);
 
     ShellCommandSourceCoordinator::Configuration configuration
@@ -237,7 +239,7 @@ void registerStorageExecutable(StorageFactory & factory)
             settings.loadFromQuery(*args.storage_def);
 
         auto global_context = args.getContext()->getGlobalContext();
-        return std::make_shared<StorageExecutable>(args.table_id, format, settings, input_queries, columns, constraints);
+        return std::make_shared<StorageExecutable>(args.table_id, format, settings, input_queries, columns, constraints, args.comment);
     };
 
     StorageFactory::StorageFeatures storage_features;
@@ -255,4 +257,3 @@ void registerStorageExecutable(StorageFactory & factory)
 }
 
 }
-
diff --git a/src/Storages/StorageExecutable.h b/src/Storages/StorageExecutable.h
index 2be2a84ab49..6748bb3223e 100644
--- a/src/Storages/StorageExecutable.h
+++ b/src/Storages/StorageExecutable.h
@@ -22,7 +22,8 @@ public:
         const ExecutableSettings & settings,
         const std::vector<ASTPtr> & input_queries,
         const ColumnsDescription & columns,
-        const ConstraintsDescription & constraints);
+        const ConstraintsDescription & constraints,
+        const String & comment);
 
     String getName() const override
     {
diff --git a/src/Storages/StorageKeeperMap.cpp b/src/Storages/StorageKeeperMap.cpp
index 587cb621362..16caf01955e 100644
--- a/src/Storages/StorageKeeperMap.cpp
+++ b/src/Storages/StorageKeeperMap.cpp
@@ -1280,6 +1280,7 @@ StoragePtr create(const StorageFactory::Arguments & args)
     StorageInMemoryMetadata metadata;
     metadata.setColumns(args.columns);
     metadata.setConstraints(args.constraints);
+    metadata.setComment(args.comment);
 
     if (!args.storage_def->primary_key)
         throw Exception(ErrorCodes::BAD_ARGUMENTS, "StorageKeeperMap requires one column in primary key");
diff --git a/src/Storages/StorageSQLite.cpp b/src/Storages/StorageSQLite.cpp
index 85417a2f2a4..b90b15f3b99 100644
--- a/src/Storages/StorageSQLite.cpp
+++ b/src/Storages/StorageSQLite.cpp
@@ -50,6 +50,7 @@ StorageSQLite::StorageSQLite(
     const String & remote_table_name_,
     const ColumnsDescription & columns_,
     const ConstraintsDescription & constraints_,
+    const String & comment,
     ContextPtr context_)
     : IStorage(table_id_)
     , WithContext(context_->getGlobalContext())
@@ -71,6 +72,7 @@ StorageSQLite::StorageSQLite(
 
     storage_metadata.setConstraints(constraints_);
     setInMemoryMetadata(storage_metadata);
+    storage_metadata.setComment(comment);
 }
 
 
@@ -211,7 +213,7 @@ void registerStorageSQLite(StorageFactory & factory)
         auto sqlite_db = openSQLiteDB(database_path, args.getContext(), /* throw_on_error */ args.mode <= LoadingStrictnessLevel::CREATE);
 
         return std::make_shared<StorageSQLite>(args.table_id, sqlite_db, database_path,
-                                     table_name, args.columns, args.constraints, args.getContext());
+                                     table_name, args.columns, args.constraints, args.comment, args.getContext());
     },
     {
         .supports_schema_inference = true,
diff --git a/src/Storages/StorageSQLite.h b/src/Storages/StorageSQLite.h
index ed673123fe0..97638ac04cb 100644
--- a/src/Storages/StorageSQLite.h
+++ b/src/Storages/StorageSQLite.h
@@ -27,6 +27,7 @@ public:
         const String & remote_table_name_,
         const ColumnsDescription & columns_,
         const ConstraintsDescription & constraints_,
+        const String & comment,
         ContextPtr context_);
 
     std::string getName() const override { return "SQLite"; }
diff --git a/src/Storages/WindowView/StorageWindowView.cpp b/src/Storages/WindowView/StorageWindowView.cpp
index e15da0074d5..7e1bca7d0d6 100644
--- a/src/Storages/WindowView/StorageWindowView.cpp
+++ b/src/Storages/WindowView/StorageWindowView.cpp
@@ -1188,6 +1188,7 @@ StorageWindowView::StorageWindowView(
     ContextPtr context_,
     const ASTCreateQuery & query,
     const ColumnsDescription & columns_,
+    const String & comment,
     LoadingStrictnessLevel mode)
     : IStorage(table_id_)
     , WithContext(context_->getGlobalContext())
@@ -1206,6 +1207,7 @@ StorageWindowView::StorageWindowView(
 
     StorageInMemoryMetadata storage_metadata;
     storage_metadata.setColumns(columns_);
+    storage_metadata.setComment(comment);
     setInMemoryMetadata(storage_metadata);
 
     /// If the target table is not set, use inner target table
@@ -1761,7 +1763,7 @@ void registerStorageWindowView(StorageFactory & factory)
                             "Experimental WINDOW VIEW feature "
                             "is not enabled (the setting 'allow_experimental_window_view')");
 
-        return std::make_shared<StorageWindowView>(args.table_id, args.getLocalContext(), args.query, args.columns, args.mode);
+        return std::make_shared<StorageWindowView>(args.table_id, args.getLocalContext(), args.query, args.columns, args.comment, args.mode);
     });
 }
 
diff --git a/src/Storages/WindowView/StorageWindowView.h b/src/Storages/WindowView/StorageWindowView.h
index 14ac65091d3..38fca512ed9 100644
--- a/src/Storages/WindowView/StorageWindowView.h
+++ b/src/Storages/WindowView/StorageWindowView.h
@@ -111,6 +111,7 @@ public:
         ContextPtr context_,
         const ASTCreateQuery & query,
         const ColumnsDescription & columns_,
+        const String & comment,
         LoadingStrictnessLevel mode);
 
     String getName() const override { return "WindowView"; }
diff --git a/src/TableFunctions/TableFunctionExecutable.cpp b/src/TableFunctions/TableFunctionExecutable.cpp
index 2c3802e8667..cccd3587bc7 100644
--- a/src/TableFunctions/TableFunctionExecutable.cpp
+++ b/src/TableFunctions/TableFunctionExecutable.cpp
@@ -170,7 +170,14 @@ StoragePtr TableFunctionExecutable::executeImpl(const ASTPtr & /*ast_function*/,
     if (settings_query != nullptr)
         settings.applyChanges(settings_query->as<ASTSetQuery>()->changes);
 
-    auto storage = std::make_shared<StorageExecutable>(storage_id, format, settings, input_queries, getActualTableStructure(context, is_insert_query), ConstraintsDescription{});
+    auto storage = std::make_shared<StorageExecutable>(
+        storage_id,
+        format,
+        settings,
+        input_queries,
+        getActualTableStructure(context, is_insert_query),
+        ConstraintsDescription{},
+        /* comment = */ "");
     storage->startup();
     return storage;
 }
diff --git a/src/TableFunctions/TableFunctionSQLite.cpp b/src/TableFunctions/TableFunctionSQLite.cpp
index e367e05bf73..87353025d1d 100644
--- a/src/TableFunctions/TableFunctionSQLite.cpp
+++ b/src/TableFunctions/TableFunctionSQLite.cpp
@@ -57,7 +57,7 @@ StoragePtr TableFunctionSQLite::executeImpl(const ASTPtr & /*ast_function*/,
                                          sqlite_db,
                                          database_path,
                                          remote_table_name,
-                                         cached_columns, ConstraintsDescription{}, context);
+                                         cached_columns, ConstraintsDescription{}, /* comment = */ "", context);
 
     storage->startup();
     return storage;

From dd9fe61d1a973a7fa528259b507218cf264548fe Mon Sep 17 00:00:00 2001
From: Max K <maxkaynov@gmail.com>
Date: Sun, 21 Jul 2024 17:44:32 +0200
Subject: [PATCH 0537/1488] CI: New Release workflow updates and fixes

---
 .github/actions/release/action.yml | 27 +++++-------
 tests/ci/artifactory.py            | 56 ++++++++++++++-----------
 tests/ci/ci_utils.py               |  3 +-
 tests/ci/create_release.py         | 66 +++++++++++++++++++-----------
 4 files changed, 87 insertions(+), 65 deletions(-)

diff --git a/.github/actions/release/action.yml b/.github/actions/release/action.yml
index 99ec02662f6..c3897682a33 100644
--- a/.github/actions/release/action.yml
+++ b/.github/actions/release/action.yml
@@ -62,8 +62,8 @@ runs:
       if: ${{ inputs.type == 'patch' }}
       shell: bash
       run: |
-        python3 ./tests/ci/create_release.py --set-progress-started --progress "update ChangeLog"
-        [ "$(git branch --show-current)" != "master" ] && echo "not on the master" && exit 1
+        git checkout master
+        python3 ./tests/ci/create_release.py --set-progress-started --progress "update changelog, docker version, security"
         echo "List versions"
         ./utils/list-versions/list-versions.sh > ./utils/list-versions/version_date.tsv
         echo "Update docker version"
@@ -96,17 +96,13 @@ runs:
           Update version_date.tsv and changelogs after ${{ env.RELEASE_TAG }}
           ### Changelog category (leave one):
           - Not for changelog (changelog entry is not required)
-    - name: Reset changes if Dry-run
-      if: ${{ inputs.dry-run }}
+    - name: Complete previous steps and Restore git state
+      if: ${{ inputs.type == 'patch' }}
       shell: bash
       run: |
-        git reset --hard HEAD
-    - name: Checkout back to GITHUB_REF
-      shell: bash
-      run: |
-        git checkout "$GITHUB_REF_NAME"
-        # set current progress to OK
         python3 ./tests/ci/create_release.py --set-progress-completed
+        git reset --hard HEAD
+        git checkout "$GITHUB_REF_NAME"
     - name: Create GH Release
       shell: bash
       if: ${{ inputs.type == 'patch' }}
@@ -146,24 +142,23 @@ runs:
       if: ${{ inputs.type == 'patch' }}
       shell: bash
       run: |
-        python3 ./tests/ci/create_release.py --set-progress-started --progress "docker server release"
         cd "./tests/ci"
+        python3 ./create_release.py --set-progress-started --progress "docker server release"
         export CHECK_NAME="Docker server image"
         python3 docker_server.py --release-type auto --version ${{ env.RELEASE_TAG }} --check-name "$CHECK_NAME" --sha ${{ env.COMMIT_SHA }} ${{ ! inputs.dry-run && '--push' || '' }}
-        python3 ./tests/ci/create_release.py --set-progress-completed
+        python3 ./create_release.py --set-progress-completed
     - name: Docker clickhouse/clickhouse-keeper building
       if: ${{ inputs.type == 'patch' }}
       shell: bash
       run: |
-        python3 ./tests/ci/create_release.py --set-progress-started --progress "docker keeper release"
         cd "./tests/ci"
+        python3 ./create_release.py --set-progress-started --progress "docker keeper release"
         export CHECK_NAME="Docker keeper image"
         python3 docker_server.py --release-type auto --version ${{ env.RELEASE_TAG }} --check-name "$CHECK_NAME" --sha ${{ env.COMMIT_SHA }} ${{ ! inputs.dry-run && '--push'  || '' }}
-        python3 ./tests/ci/create_release.py --set-progress-completed
-    - name: Set Release progress completed
+        python3 ./create_release.py --set-progress-completed
+    - name: Set current Release progress to Completed with OK
       shell: bash
       run: |
-        # If we here - set completed status, to post proper Slack OK or FAIL message in the next step
         python3 ./tests/ci/create_release.py --set-progress-started --progress "completed"
         python3 ./tests/ci/create_release.py --set-progress-completed
     - name: Post Slack Message
diff --git a/tests/ci/artifactory.py b/tests/ci/artifactory.py
index 98a0345c6bd..86dcaf79854 100644
--- a/tests/ci/artifactory.py
+++ b/tests/ci/artifactory.py
@@ -43,7 +43,6 @@ class R2MountPoint:
             self.bucket_name = self._PROD_BUCKET_NAME
 
         self.aux_mount_options = ""
-        self.async_mount = False
         if self.app == MountPointApp.S3FS:
             self.cache_dir = "/home/ubuntu/s3fs_cache"
             # self.aux_mount_options += "-o nomodtime " if self.NOMODTIME else "" not for s3fs
@@ -57,7 +56,6 @@ class R2MountPoint:
             self.mount_cmd = f"s3fs {self.bucket_name} {self.MOUNT_POINT} -o url={self.API_ENDPOINT} -o use_path_request_style -o umask=0000 -o nomultipart -o logfile={self.LOG_FILE} {self.aux_mount_options}"
         elif self.app == MountPointApp.RCLONE:
             # run rclone mount process asynchronously, otherwise subprocess.run(daemonized command) will not return
-            self.async_mount = True
             self.cache_dir = "/home/ubuntu/rclone_cache"
             self.aux_mount_options += "--no-modtime " if self.NOMODTIME else ""
             self.aux_mount_options += "-v " if self.DEBUG else ""  # -vv too verbose
@@ -85,10 +83,12 @@ class R2MountPoint:
         Shell.run(_UNMOUNT_CMD)
         Shell.run(_MKDIR_CMD)
         Shell.run(_MKDIR_FOR_CACHE)
-        # didn't manage to use simple run() and not block or fail
-        Shell.run_as_daemon(self.mount_cmd)
-        if self.async_mount:
-            time.sleep(3)
+        if self.app == MountPointApp.S3FS:
+            Shell.run(self.mount_cmd, check=True)
+        else:
+            # didn't manage to use simple run() and without blocking or failure
+            Shell.run_as_daemon(self.mount_cmd)
+        time.sleep(3)
         Shell.run(_TEST_MOUNT_CMD, check=True)
 
     @classmethod
@@ -107,6 +107,7 @@ class DebianArtifactory:
     _PROD_REPO_URL = "https://packages.clickhouse.com/deb"
 
     def __init__(self, release_info: ReleaseInfo, dry_run: bool):
+        self.release_info = release_info
         self.codename = release_info.codename
         self.version = release_info.version
         if dry_run:
@@ -154,9 +155,8 @@ class DebianArtifactory:
         print("Running test command:")
         print(f"  {cmd}")
         Shell.run(cmd, check=True)
-        release_info = ReleaseInfo.from_file()
-        release_info.debian_command = debian_command
-        release_info.dump()
+        self.release_info.debian_command = debian_command
+        self.release_info.dump()
 
 
 def _copy_if_not_exists(src: Path, dst: Path) -> Path:
@@ -177,6 +177,7 @@ class RpmArtifactory:
     _SIGN_KEY = "885E2BDCF96B0B45ABF058453E4AD4719DDE9A38"
 
     def __init__(self, release_info: ReleaseInfo, dry_run: bool):
+        self.release_info = release_info
         self.codename = release_info.codename
         self.version = release_info.version
         if dry_run:
@@ -230,9 +231,8 @@ class RpmArtifactory:
         print("Running test command:")
         print(f"  {cmd}")
         Shell.run(cmd, check=True)
-        release_info = ReleaseInfo.from_file()
-        release_info.rpm_command = rpm_command
-        release_info.dump()
+        self.release_info.rpm_command = rpm_command
+        self.release_info.dump()
 
 
 class TgzArtifactory:
@@ -240,6 +240,7 @@ class TgzArtifactory:
     _PROD_REPO_URL = "https://packages.clickhouse.com/tgz"
 
     def __init__(self, release_info: ReleaseInfo, dry_run: bool):
+        self.release_info = release_info
         self.codename = release_info.codename
         self.version = release_info.version
         if dry_run:
@@ -290,9 +291,8 @@ class TgzArtifactory:
             expected_checksum == actual_checksum
         ), f"[{actual_checksum} != {expected_checksum}]"
         Shell.run("rm /tmp/tmp.tgz*")
-        release_info = ReleaseInfo.from_file()
-        release_info.tgz_command = cmd
-        release_info.dump()
+        self.release_info.tgz_command = cmd
+        self.release_info.dump()
 
 
 def parse_args() -> argparse.Namespace:
@@ -340,9 +340,7 @@ def parse_args() -> argparse.Namespace:
 
 if __name__ == "__main__":
     args = parse_args()
-    assert args.dry_run
 
-    release_info = ReleaseInfo.from_file()
     """
     Use S3FS. RCLONE has some errors with r2 remote which I didn't figure out how to resolve:
            ERROR : IO error: NotImplemented: versionId not implemented
@@ -350,26 +348,38 @@ if __name__ == "__main__":
     """
     mp = R2MountPoint(MountPointApp.S3FS, dry_run=args.dry_run)
     if args.export_debian:
-        with ReleaseContextManager(release_progress=ReleaseProgress.EXPORT_DEB) as _:
+        with ReleaseContextManager(
+            release_progress=ReleaseProgress.EXPORT_DEB
+        ) as release_info:
             mp.init()
             DebianArtifactory(release_info, dry_run=args.dry_run).export_packages()
             mp.teardown()
     if args.export_rpm:
-        with ReleaseContextManager(release_progress=ReleaseProgress.EXPORT_RPM) as _:
+        with ReleaseContextManager(
+            release_progress=ReleaseProgress.EXPORT_RPM
+        ) as release_info:
             mp.init()
             RpmArtifactory(release_info, dry_run=args.dry_run).export_packages()
             mp.teardown()
     if args.export_tgz:
-        with ReleaseContextManager(release_progress=ReleaseProgress.EXPORT_TGZ) as _:
+        with ReleaseContextManager(
+            release_progress=ReleaseProgress.EXPORT_TGZ
+        ) as release_info:
             mp.init()
             TgzArtifactory(release_info, dry_run=args.dry_run).export_packages()
             mp.teardown()
     if args.test_debian:
-        with ReleaseContextManager(release_progress=ReleaseProgress.TEST_DEB) as _:
+        with ReleaseContextManager(
+            release_progress=ReleaseProgress.TEST_DEB
+        ) as release_info:
             DebianArtifactory(release_info, dry_run=args.dry_run).test_packages()
     if args.test_tgz:
-        with ReleaseContextManager(release_progress=ReleaseProgress.TEST_TGZ) as _:
+        with ReleaseContextManager(
+            release_progress=ReleaseProgress.TEST_TGZ
+        ) as release_info:
             TgzArtifactory(release_info, dry_run=args.dry_run).test_packages()
     if args.test_rpm:
-        with ReleaseContextManager(release_progress=ReleaseProgress.TEST_RPM) as _:
+        with ReleaseContextManager(
+            release_progress=ReleaseProgress.TEST_RPM
+        ) as release_info:
             RpmArtifactory(release_info, dry_run=args.dry_run).test_packages()
diff --git a/tests/ci/ci_utils.py b/tests/ci/ci_utils.py
index efbf014cd52..9a1b12af310 100644
--- a/tests/ci/ci_utils.py
+++ b/tests/ci/ci_utils.py
@@ -182,10 +182,11 @@ class Shell:
             check=False,
         )
         if result.returncode == 0:
+            print(f"stdout: {result.stdout.strip()}")
             res = result.stdout
         else:
             print(
-                f"ERROR: stdout {result.stdout.strip()}, stderr {result.stderr.strip()}"
+                f"ERROR: stdout: {result.stdout.strip()}, stderr: {result.stderr.strip()}"
             )
             if check:
                 assert result.returncode == 0
diff --git a/tests/ci/create_release.py b/tests/ci/create_release.py
index 4347cfebb54..a0b4083b673 100755
--- a/tests/ci/create_release.py
+++ b/tests/ci/create_release.py
@@ -43,6 +43,7 @@ class ReleaseProgress:
     TEST_TGZ = "test TGZ packages"
     TEST_RPM = "test RPM packages"
     TEST_DEB = "test DEB packages"
+    COMPLETED = "completed"
 
 
 class ReleaseProgressDescription:
@@ -108,6 +109,12 @@ class ReleaseInfo:
     release_progress: str = ""
     progress_description: str = ""
 
+    def is_patch(self):
+        return self.release_branch != "master"
+
+    def is_new_release_branch(self):
+        return self.release_branch == "master"
+
     @staticmethod
     def from_file() -> "ReleaseInfo":
         with open(RELEASE_INFO_FILE, "r", encoding="utf-8") as json_file:
@@ -126,12 +133,12 @@ class ReleaseInfo:
         release_tag = None
         previous_release_tag = None
         previous_release_sha = None
-        codename = None
+        codename = ""
         assert release_type in ("patch", "new")
         if release_type == "new":
             # check commit_ref is right and on a right branch
             Shell.run(
-                f"git merge-base --is-ancestor origin/{commit_ref} origin/master",
+                f"git merge-base --is-ancestor {commit_ref} origin/master",
                 check=True,
             )
             with checkout(commit_ref):
@@ -146,9 +153,6 @@ class ReleaseInfo:
                     git.latest_tag == expected_prev_tag
                 ), f"BUG: latest tag [{git.latest_tag}], expected [{expected_prev_tag}]"
                 release_tag = version.describe
-                codename = (
-                    VersionType.STABLE
-                )  # dummy value (artifactory won't be updated for new release)
                 previous_release_tag = expected_prev_tag
                 previous_release_sha = Shell.run_strict(
                     f"git rev-parse {previous_release_tag}"
@@ -205,7 +209,7 @@ class ReleaseInfo:
             and commit_sha
             and release_tag
             and version
-            and codename in ("lts", "stable")
+            and (codename in ("lts", "stable") or release_type == "new")
         )
 
         self.release_branch = release_branch
@@ -320,24 +324,27 @@ class ReleaseInfo:
                     Shell.run(
                         f"{GIT_PREFIX} checkout '{CMAKE_PATH}' '{CONTRIBUTORS_PATH}'"
                     )
-                self.version_bump_pr = GHActions.get_pr_url_by_branch(
-                    repo=GITHUB_REPOSITORY, branch=branch_upd_version_contributors
-                )
+                    self.version_bump_pr = "dry-run"
+                else:
+                    self.version_bump_pr = GHActions.get_pr_url_by_branch(
+                        repo=GITHUB_REPOSITORY, branch=branch_upd_version_contributors
+                    )
 
     def update_release_info(self, dry_run: bool) -> "ReleaseInfo":
-        branch = f"auto/{release_info.release_tag}"
-        if not dry_run:
-            url = GHActions.get_pr_url_by_branch(repo=GITHUB_REPOSITORY, branch=branch)
-        else:
-            url = "dry-run"
-
-        print(f"ChangeLog PR url [{url}]")
-        self.changelog_pr = url
-        print(f"Release url [{url}]")
-        self.release_url = (
-            f"https://github.com/{GITHUB_REPOSITORY}/releases/tag/{self.release_tag}"
-        )
-        self.docker_command = f"docker run --rm clickhouse/clickhouse:{self.release_branch} clickhouse --version"
+        if self.release_branch != "master":
+            branch = f"auto/{release_info.release_tag}"
+            if not dry_run:
+                url = GHActions.get_pr_url_by_branch(
+                    repo=GITHUB_REPOSITORY, branch=branch
+                )
+            else:
+                url = "dry-run"
+            print(f"ChangeLog PR url [{url}]")
+            self.changelog_pr = url
+            print(f"Release url [{url}]")
+            self.release_url = f"https://github.com/{GITHUB_REPOSITORY}/releases/tag/{self.release_tag}"
+            if self.release_progress == ReleaseProgress.COMPLETED:
+                self.docker_command = f"docker run --rm clickhouse/clickhouse:{self.version} clickhouse --version"
         self.dump()
         return self
 
@@ -712,13 +719,22 @@ if __name__ == "__main__":
     if args.post_status:
         release_info = ReleaseInfo.from_file()
         release_info.update_release_info(dry_run=args.dry_run)
-        if release_info.debian_command:
+        if release_info.is_new_release_branch():
+            title = "New release branch"
+        else:
+            title = "New release"
+        if (
+            release_info.progress_description == ReleaseProgressDescription.OK
+            and release_info.release_progress == ReleaseProgress.COMPLETED
+        ):
+            title = "Completed: " + title
             CIBuddy(dry_run=args.dry_run).post_done(
-                f"New release issued", dataclasses.asdict(release_info)
+                title, dataclasses.asdict(release_info)
             )
         else:
+            title = "Failed: " + title
             CIBuddy(dry_run=args.dry_run).post_critical(
-                f"Failed to issue new release", dataclasses.asdict(release_info)
+                title, dataclasses.asdict(release_info)
             )
 
     if args.set_progress_started:

From 20d4b16fc1d92632b714368c85ac6701723ef096 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 21 Jul 2024 22:19:30 +0200
Subject: [PATCH 0538/1488] Update
 03207_composite_expressions_lambda_consistent_formatting.sql

---
 .../03207_composite_expressions_lambda_consistent_formatting.sql | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/queries/0_stateless/03207_composite_expressions_lambda_consistent_formatting.sql b/tests/queries/0_stateless/03207_composite_expressions_lambda_consistent_formatting.sql
index 42c823cf476..2e2f5510876 100644
--- a/tests/queries/0_stateless/03207_composite_expressions_lambda_consistent_formatting.sql
+++ b/tests/queries/0_stateless/03207_composite_expressions_lambda_consistent_formatting.sql
@@ -1,3 +1,4 @@
+SET allow_experimental_analyzer = 1;
 SELECT [1, (x -> 1)]; -- { serverError ILLEGAL_TYPE_OF_ARGUMENT }
 SELECT (1, (x -> 1)); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT }
 SELECT map(1, (x -> 1)); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT }

From 2be21fe05c6b952735fc6895c0286b177864dde5 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 22 Jul 2024 05:53:12 +0200
Subject: [PATCH 0539/1488] Fix RocksDB bs

---
 tests/queries/0_stateless/02241_join_rocksdb_bs.sql.j2 | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/queries/0_stateless/02241_join_rocksdb_bs.sql.j2 b/tests/queries/0_stateless/02241_join_rocksdb_bs.sql.j2
index 6121db6d6a2..4ab98201eed 100644
--- a/tests/queries/0_stateless/02241_join_rocksdb_bs.sql.j2
+++ b/tests/queries/0_stateless/02241_join_rocksdb_bs.sql.j2
@@ -1,4 +1,4 @@
--- Tags: use-rocksdb, long
+-- Tags: use-rocksdb, long, no-s3-storage, no-random-settings, no-random-merge-tree-settings
 
 SET join_algorithm = 'direct';
 
@@ -41,4 +41,3 @@ ON rdb.key == t1.k;
 {% for table_size in [10, 65555, 100000] -%}
 DROP TABLE IF EXISTS rdb_{{ table_size }};
 {% endfor -%}
-

From 7643b99dc5a892f404d756e92c4a17cf689c720b Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 22 Jul 2024 06:08:00 +0200
Subject: [PATCH 0540/1488] Split a test for index

---
 .../0_stateless/02995_index_1.reference       | 15 +++++++
 tests/queries/0_stateless/02995_index_1.sh    | 44 +++++++++++++++++++
 .../0_stateless/02995_index_10.reference      | 13 ++++++
 tests/queries/0_stateless/02995_index_10.sh   | 44 +++++++++++++++++++
 .../0_stateless/02995_index_2.reference       | 13 ++++++
 tests/queries/0_stateless/02995_index_2.sh    | 44 +++++++++++++++++++
 .../0_stateless/02995_index_3.reference       | 16 +++++++
 tests/queries/0_stateless/02995_index_3.sh    | 44 +++++++++++++++++++
 .../0_stateless/02995_index_4.reference       | 16 +++++++
 tests/queries/0_stateless/02995_index_4.sh    | 44 +++++++++++++++++++
 .../0_stateless/02995_index_5.reference       |  9 ++++
 tests/queries/0_stateless/02995_index_5.sh    | 44 +++++++++++++++++++
 .../0_stateless/02995_index_6.reference       | 14 ++++++
 tests/queries/0_stateless/02995_index_6.sh    | 44 +++++++++++++++++++
 .../0_stateless/02995_index_7.reference       |  9 ++++
 tests/queries/0_stateless/02995_index_7.sh    | 44 +++++++++++++++++++
 .../0_stateless/02995_index_8.reference       | 12 +++++
 tests/queries/0_stateless/02995_index_8.sh    | 44 +++++++++++++++++++
 .../0_stateless/02995_index_9.reference       |  9 ++++
 tests/queries/0_stateless/02995_index_9.sh    | 44 +++++++++++++++++++
 20 files changed, 566 insertions(+)
 create mode 100644 tests/queries/0_stateless/02995_index_1.reference
 create mode 100755 tests/queries/0_stateless/02995_index_1.sh
 create mode 100644 tests/queries/0_stateless/02995_index_10.reference
 create mode 100755 tests/queries/0_stateless/02995_index_10.sh
 create mode 100644 tests/queries/0_stateless/02995_index_2.reference
 create mode 100755 tests/queries/0_stateless/02995_index_2.sh
 create mode 100644 tests/queries/0_stateless/02995_index_3.reference
 create mode 100755 tests/queries/0_stateless/02995_index_3.sh
 create mode 100644 tests/queries/0_stateless/02995_index_4.reference
 create mode 100755 tests/queries/0_stateless/02995_index_4.sh
 create mode 100644 tests/queries/0_stateless/02995_index_5.reference
 create mode 100755 tests/queries/0_stateless/02995_index_5.sh
 create mode 100644 tests/queries/0_stateless/02995_index_6.reference
 create mode 100755 tests/queries/0_stateless/02995_index_6.sh
 create mode 100644 tests/queries/0_stateless/02995_index_7.reference
 create mode 100755 tests/queries/0_stateless/02995_index_7.sh
 create mode 100644 tests/queries/0_stateless/02995_index_8.reference
 create mode 100755 tests/queries/0_stateless/02995_index_8.sh
 create mode 100644 tests/queries/0_stateless/02995_index_9.reference
 create mode 100755 tests/queries/0_stateless/02995_index_9.sh

diff --git a/tests/queries/0_stateless/02995_index_1.reference b/tests/queries/0_stateless/02995_index_1.reference
new file mode 100644
index 00000000000..6c3b1230db6
--- /dev/null
+++ b/tests/queries/0_stateless/02995_index_1.reference
@@ -0,0 +1,15 @@
+12	4	21722	2209341	4	1415	2333	4	61	64	3
+21	1134	11363	58484	1106	1458	1592	136	26	62	32
+22	210	4504	5729	196	291	767	124	47	54	8
+26	196	1327684	5221	195	4140	5661	161	28	49	19
+28	5	2034378	7102	5	325	3255	2	53	60	4
+29	53	45041	45189	45	1580	211	31	55	84	18
+38	424	1600675	4653	424	562	5944	244	60	65	6
+45	17	62743	674873	17	6239	6494	17	65	76	8
+72	1862	1210073	6200	1677	2498	528	859	51	61	11
+79	2	2255228	2255293	2	5495	7057	2	65	65	1
+85	459	1051571	1829638	459	6402	7131	334	32	61	25
+86	10	1748130	1754217	10	4374	7003	10	56	59	4
+91	165	5718	5802	75	282	7113	112	41	63	22
+94	20	1231916	2050003	20	4802	4917	19	53	59	7
+99	2	3665	36667	2	497	697	2	70	71	2
diff --git a/tests/queries/0_stateless/02995_index_1.sh b/tests/queries/0_stateless/02995_index_1.sh
new file mode 100755
index 00000000000..a5f1b30c2e8
--- /dev/null
+++ b/tests/queries/0_stateless/02995_index_1.sh
@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+# Tags: long, no-debug, no-asan, no-tsan, no-msan, no-ubsan, no-sanitize-coverage
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+${CLICKHOUSE_CLIENT} --multiquery "
+
+DROP TABLE IF EXISTS test;
+CREATE TABLE test (a String, b String, c String) ENGINE = MergeTree ORDER BY (a, b, c) SETTINGS index_granularity = 11;
+
+INSERT INTO test
+SELECT round(pow(sipHash64(1, number), 1/4)), round(pow(sipHash64(2, number), 1/6)), round(pow(sipHash64(3, number), 1/10))
+FROM numbers(100000);
+
+INSERT INTO test
+SELECT round(pow(sipHash64(1, number), 1/3)), round(pow(sipHash64(2, number), 1/5)), round(pow(sipHash64(3, number), 1/10))
+FROM numbers(100000);
+
+INSERT INTO test
+SELECT round(pow(sipHash64(1, number), 1/5)), round(pow(sipHash64(2, number), 1/7)), round(pow(sipHash64(3, number), 1/10))
+FROM numbers(100000);
+
+DETACH TABLE test;
+ATTACH TABLE test;
+"
+
+for i in {1..100}
+do
+    echo "
+WITH ${i} AS try
+SELECT try, count(), min(a), max(a), uniqExact(a), min(b), max(b), uniqExact(b), min(c), max(c), uniqExact(c) FROM test
+WHERE a >= (round(pow(sipHash64(1, try), 1 / (3 + sipHash64(2, try) % 8))) AS a1)::String
+  AND a <= (a1 + round(pow(sipHash64(3, try), 1 / (3 + sipHash64(4, try) % 8))))::String
+  AND b >= (round(pow(sipHash64(5, try), 1 / (3 + sipHash64(6, try) % 8))) AS b1)::String
+  AND b <= (b1 + round(pow(sipHash64(7, try), 1 / (3 + sipHash64(8, try) % 8))))::String
+  AND c >= (round(pow(sipHash64(9, try), 1 / (3 + sipHash64(10, try) % 8))) AS c1)::String
+  AND c <= (c1 + round(pow(sipHash64(11, try), 1 / (3 + sipHash64(12, try) % 8))))::String
+HAVING count() > 0;
+"
+done | ${CLICKHOUSE_CLIENT} --multiquery
+
+${CLICKHOUSE_CLIENT} --multiquery "DROP TABLE test"
diff --git a/tests/queries/0_stateless/02995_index_10.reference b/tests/queries/0_stateless/02995_index_10.reference
new file mode 100644
index 00000000000..bfa38d03801
--- /dev/null
+++ b/tests/queries/0_stateless/02995_index_10.reference
@@ -0,0 +1,13 @@
+912	146	1322641	2238040	146	1366	6354	143	59	59	1
+913	82	5495	6870	78	350	565	67	24	43	15
+921	763	1580790	416881	763	6191	7131	509	63	64	2
+925	318	2500952	5025	309	476	6114	182	32	56	21
+931	12	4277	4809	12	238	256	9	63	83	9
+942	954	1331	2228193	952	1121	5047	788	65	70	6
+948	14	1785593	2600431	14	6550	6598	13	34	49	9
+956	5	5755	6023	5	359	411	5	43	48	4
+963	4	3812	3835	4	444	537	4	47	53	4
+978	5	51632	58212	5	1127	1556	5	24	32	5
+980	53	47201	59744	53	1537	1625	36	41	49	9
+987	6033	2020131	763444	4306	256	792	1832	60	64	5
+993	4	1615159	1718339	4	1570	3093	4	62	63	2
diff --git a/tests/queries/0_stateless/02995_index_10.sh b/tests/queries/0_stateless/02995_index_10.sh
new file mode 100755
index 00000000000..d72c7c72705
--- /dev/null
+++ b/tests/queries/0_stateless/02995_index_10.sh
@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+# Tags: long, no-debug, no-asan, no-tsan, no-msan, no-ubsan, no-sanitize-coverage
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+${CLICKHOUSE_CLIENT} --multiquery "
+
+DROP TABLE IF EXISTS test;
+CREATE TABLE test (a String, b String, c String) ENGINE = MergeTree ORDER BY (a, b, c) SETTINGS index_granularity = 11;
+
+INSERT INTO test
+SELECT round(pow(sipHash64(1, number), 1/4)), round(pow(sipHash64(2, number), 1/6)), round(pow(sipHash64(3, number), 1/10))
+FROM numbers(100000);
+
+INSERT INTO test
+SELECT round(pow(sipHash64(1, number), 1/3)), round(pow(sipHash64(2, number), 1/5)), round(pow(sipHash64(3, number), 1/10))
+FROM numbers(100000);
+
+INSERT INTO test
+SELECT round(pow(sipHash64(1, number), 1/5)), round(pow(sipHash64(2, number), 1/7)), round(pow(sipHash64(3, number), 1/10))
+FROM numbers(100000);
+
+DETACH TABLE test;
+ATTACH TABLE test;
+"
+
+for i in {901..1000}
+do
+    echo "
+WITH ${i} AS try
+SELECT try, count(), min(a), max(a), uniqExact(a), min(b), max(b), uniqExact(b), min(c), max(c), uniqExact(c) FROM test
+WHERE a >= (round(pow(sipHash64(1, try), 1 / (3 + sipHash64(2, try) % 8))) AS a1)::String
+  AND a <= (a1 + round(pow(sipHash64(3, try), 1 / (3 + sipHash64(4, try) % 8))))::String
+  AND b >= (round(pow(sipHash64(5, try), 1 / (3 + sipHash64(6, try) % 8))) AS b1)::String
+  AND b <= (b1 + round(pow(sipHash64(7, try), 1 / (3 + sipHash64(8, try) % 8))))::String
+  AND c >= (round(pow(sipHash64(9, try), 1 / (3 + sipHash64(10, try) % 8))) AS c1)::String
+  AND c <= (c1 + round(pow(sipHash64(11, try), 1 / (3 + sipHash64(12, try) % 8))))::String
+HAVING count() > 0;
+"
+done | ${CLICKHOUSE_CLIENT} --multiquery
+
+${CLICKHOUSE_CLIENT} --multiquery "DROP TABLE test"
diff --git a/tests/queries/0_stateless/02995_index_2.reference b/tests/queries/0_stateless/02995_index_2.reference
new file mode 100644
index 00000000000..098292a289f
--- /dev/null
+++ b/tests/queries/0_stateless/02995_index_2.reference
@@ -0,0 +1,13 @@
+103	1	2446615	2446615	1	2498	2498	1	58	58	1
+106	72	6149	6699	67	527	826	40	61	61	1
+111	43	2273186	5272	43	492	4923	4	54	72	15
+120	3129	45117	6735	2868	1030	1625	561	59	64	6
+138	2	49243	49374	2	1428	1519	2	47	48	2
+143	100	23321	63639	100	1115	1624	88	51	51	1
+145	1	2447976	2447976	1	6173	6173	1	44	44	1
+153	16	13748	16881	16	1506	1636	16	54	68	9
+159	19952	1525336	7131	12957	1280	6163	2668	24	66	39
+171	5	15042	16698	5	1302	1608	5	65	65	1
+179	6264	1362341	2686	6244	2554	7132	2705	61	67	7
+192	1	1639623	1639623	1	3406	3406	1	32	32	1
+193	1	1429969	1429969	1	7131	7131	1	45	45	1
diff --git a/tests/queries/0_stateless/02995_index_2.sh b/tests/queries/0_stateless/02995_index_2.sh
new file mode 100755
index 00000000000..e7451c7ee4b
--- /dev/null
+++ b/tests/queries/0_stateless/02995_index_2.sh
@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+# Tags: long, no-debug, no-asan, no-tsan, no-msan, no-ubsan, no-sanitize-coverage
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+${CLICKHOUSE_CLIENT} --multiquery "
+
+DROP TABLE IF EXISTS test;
+CREATE TABLE test (a String, b String, c String) ENGINE = MergeTree ORDER BY (a, b, c) SETTINGS index_granularity = 11;
+
+INSERT INTO test
+SELECT round(pow(sipHash64(1, number), 1/4)), round(pow(sipHash64(2, number), 1/6)), round(pow(sipHash64(3, number), 1/10))
+FROM numbers(100000);
+
+INSERT INTO test
+SELECT round(pow(sipHash64(1, number), 1/3)), round(pow(sipHash64(2, number), 1/5)), round(pow(sipHash64(3, number), 1/10))
+FROM numbers(100000);
+
+INSERT INTO test
+SELECT round(pow(sipHash64(1, number), 1/5)), round(pow(sipHash64(2, number), 1/7)), round(pow(sipHash64(3, number), 1/10))
+FROM numbers(100000);
+
+DETACH TABLE test;
+ATTACH TABLE test;
+"
+
+for i in {101..200}
+do
+    echo "
+WITH ${i} AS try
+SELECT try, count(), min(a), max(a), uniqExact(a), min(b), max(b), uniqExact(b), min(c), max(c), uniqExact(c) FROM test
+WHERE a >= (round(pow(sipHash64(1, try), 1 / (3 + sipHash64(2, try) % 8))) AS a1)::String
+  AND a <= (a1 + round(pow(sipHash64(3, try), 1 / (3 + sipHash64(4, try) % 8))))::String
+  AND b >= (round(pow(sipHash64(5, try), 1 / (3 + sipHash64(6, try) % 8))) AS b1)::String
+  AND b <= (b1 + round(pow(sipHash64(7, try), 1 / (3 + sipHash64(8, try) % 8))))::String
+  AND c >= (round(pow(sipHash64(9, try), 1 / (3 + sipHash64(10, try) % 8))) AS c1)::String
+  AND c <= (c1 + round(pow(sipHash64(11, try), 1 / (3 + sipHash64(12, try) % 8))))::String
+HAVING count() > 0;
+"
+done | ${CLICKHOUSE_CLIENT} --multiquery
+
+${CLICKHOUSE_CLIENT} --multiquery "DROP TABLE test"
diff --git a/tests/queries/0_stateless/02995_index_3.reference b/tests/queries/0_stateless/02995_index_3.reference
new file mode 100644
index 00000000000..9c2fca9fde6
--- /dev/null
+++ b/tests/queries/0_stateless/02995_index_3.reference
@@ -0,0 +1,16 @@
+207	12	23057	32500	12	1491	1726	12	32	46	7
+221	5081	1366870	6649	3432	4527	5226	687	24	69	39
+228	73	12281	17929	71	1328	2034	63	49	71	18
+229	2	1617478	1723791	2	4590	5578	2	41	42	2
+230	3916	1332729	6949	3668	1330	4703	845	62	65	4
+238	25	2624456	2625673	24	2535	6465	25	58	75	14
+241	154	2554929	2616444	154	2626	7131	148	34	57	17
+248	276	15529	30987	274	1040	1222	136	37	79	27
+254	3018	33966	6635	2837	1057	1622	539	24	60	33
+255	20	1581774	1811334	20	6068	6301	18	33	57	10
+256	5	5145	6841	5	367	376	5	58	58	1
+270	2	2195579	2262119	2	7102	7123	2	33	34	2
+281	32	2379460	616215	32	6042	6086	23	53	64	12
+282	7	1292651	24244	7	1607	2455	6	46	55	5
+286	123	1521935	5269	123	3793	3940	81	40	66	22
+291	21	2419080	3567	21	297	4731	21	54	55	2
diff --git a/tests/queries/0_stateless/02995_index_3.sh b/tests/queries/0_stateless/02995_index_3.sh
new file mode 100755
index 00000000000..506429e2696
--- /dev/null
+++ b/tests/queries/0_stateless/02995_index_3.sh
@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+# Tags: long, no-debug, no-asan, no-tsan, no-msan, no-ubsan, no-sanitize-coverage
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+${CLICKHOUSE_CLIENT} --multiquery "
+
+DROP TABLE IF EXISTS test;
+CREATE TABLE test (a String, b String, c String) ENGINE = MergeTree ORDER BY (a, b, c) SETTINGS index_granularity = 11;
+
+INSERT INTO test
+SELECT round(pow(sipHash64(1, number), 1/4)), round(pow(sipHash64(2, number), 1/6)), round(pow(sipHash64(3, number), 1/10))
+FROM numbers(100000);
+
+INSERT INTO test
+SELECT round(pow(sipHash64(1, number), 1/3)), round(pow(sipHash64(2, number), 1/5)), round(pow(sipHash64(3, number), 1/10))
+FROM numbers(100000);
+
+INSERT INTO test
+SELECT round(pow(sipHash64(1, number), 1/5)), round(pow(sipHash64(2, number), 1/7)), round(pow(sipHash64(3, number), 1/10))
+FROM numbers(100000);
+
+DETACH TABLE test;
+ATTACH TABLE test;
+"
+
+for i in {201..300}
+do
+    echo "
+WITH ${i} AS try
+SELECT try, count(), min(a), max(a), uniqExact(a), min(b), max(b), uniqExact(b), min(c), max(c), uniqExact(c) FROM test
+WHERE a >= (round(pow(sipHash64(1, try), 1 / (3 + sipHash64(2, try) % 8))) AS a1)::String
+  AND a <= (a1 + round(pow(sipHash64(3, try), 1 / (3 + sipHash64(4, try) % 8))))::String
+  AND b >= (round(pow(sipHash64(5, try), 1 / (3 + sipHash64(6, try) % 8))) AS b1)::String
+  AND b <= (b1 + round(pow(sipHash64(7, try), 1 / (3 + sipHash64(8, try) % 8))))::String
+  AND c >= (round(pow(sipHash64(9, try), 1 / (3 + sipHash64(10, try) % 8))) AS c1)::String
+  AND c <= (c1 + round(pow(sipHash64(11, try), 1 / (3 + sipHash64(12, try) % 8))))::String
+HAVING count() > 0;
+"
+done | ${CLICKHOUSE_CLIENT} --multiquery
+
+${CLICKHOUSE_CLIENT} --multiquery "DROP TABLE test"
diff --git a/tests/queries/0_stateless/02995_index_4.reference b/tests/queries/0_stateless/02995_index_4.reference
new file mode 100644
index 00000000000..deff7afaed3
--- /dev/null
+++ b/tests/queries/0_stateless/02995_index_4.reference
@@ -0,0 +1,16 @@
+316	4	5221	5616	4	505	558	4	32	35	3
+319	232	56480	63033	230	1599	313	50	33	64	26
+327	15	51647	51894	14	1292	1585	14	47	57	7
+332	24	23484	54948	24	1609	1726	16	32	49	11
+333	1	14189	14189	1	1550	1550	1	63	63	1
+342	49	2579220	2622432	49	4626	6933	48	34	54	14
+344	1	6486	6486	1	509	509	1	24	24	1
+346	1987	53016	6735	1823	1334	174	294	26	62	32
+358	45	59058	60844	44	6746	722	40	57	84	15
+363	1198	1260033	2568811	1196	5710	5790	82	55	80	26
+384	150	2361175	476024	150	7008	7123	81	38	64	22
+387	277	5200	6553	252	243	521	130	65	65	1
+392	1877	1607428	2030850	1875	1416	7131	1379	54	66	13
+396	8181	1380803	6186	7920	545	798	1743	24	67	39
+398	3	5183	5213	2	291	352	3	53	59	3
+399	62	51494	59203	61	7073	754	42	55	78	18
diff --git a/tests/queries/0_stateless/02995_index_4.sh b/tests/queries/0_stateless/02995_index_4.sh
new file mode 100755
index 00000000000..1a0458728f9
--- /dev/null
+++ b/tests/queries/0_stateless/02995_index_4.sh
@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+# Tags: long, no-debug, no-asan, no-tsan, no-msan, no-ubsan, no-sanitize-coverage
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+${CLICKHOUSE_CLIENT} --multiquery "
+
+DROP TABLE IF EXISTS test;
+CREATE TABLE test (a String, b String, c String) ENGINE = MergeTree ORDER BY (a, b, c) SETTINGS index_granularity = 11;
+
+INSERT INTO test
+SELECT round(pow(sipHash64(1, number), 1/4)), round(pow(sipHash64(2, number), 1/6)), round(pow(sipHash64(3, number), 1/10))
+FROM numbers(100000);
+
+INSERT INTO test
+SELECT round(pow(sipHash64(1, number), 1/3)), round(pow(sipHash64(2, number), 1/5)), round(pow(sipHash64(3, number), 1/10))
+FROM numbers(100000);
+
+INSERT INTO test
+SELECT round(pow(sipHash64(1, number), 1/5)), round(pow(sipHash64(2, number), 1/7)), round(pow(sipHash64(3, number), 1/10))
+FROM numbers(100000);
+
+DETACH TABLE test;
+ATTACH TABLE test;
+"
+
+for i in {301..400}
+do
+    echo "
+WITH ${i} AS try
+SELECT try, count(), min(a), max(a), uniqExact(a), min(b), max(b), uniqExact(b), min(c), max(c), uniqExact(c) FROM test
+WHERE a >= (round(pow(sipHash64(1, try), 1 / (3 + sipHash64(2, try) % 8))) AS a1)::String
+  AND a <= (a1 + round(pow(sipHash64(3, try), 1 / (3 + sipHash64(4, try) % 8))))::String
+  AND b >= (round(pow(sipHash64(5, try), 1 / (3 + sipHash64(6, try) % 8))) AS b1)::String
+  AND b <= (b1 + round(pow(sipHash64(7, try), 1 / (3 + sipHash64(8, try) % 8))))::String
+  AND c >= (round(pow(sipHash64(9, try), 1 / (3 + sipHash64(10, try) % 8))) AS c1)::String
+  AND c <= (c1 + round(pow(sipHash64(11, try), 1 / (3 + sipHash64(12, try) % 8))))::String
+HAVING count() > 0;
+"
+done | ${CLICKHOUSE_CLIENT} --multiquery
+
+${CLICKHOUSE_CLIENT} --multiquery "DROP TABLE test"
diff --git a/tests/queries/0_stateless/02995_index_5.reference b/tests/queries/0_stateless/02995_index_5.reference
new file mode 100644
index 00000000000..c5ab4d2417c
--- /dev/null
+++ b/tests/queries/0_stateless/02995_index_5.reference
@@ -0,0 +1,9 @@
+412	2141	1360120	2189792	2136	2491	5658	1371	71	75	5
+413	2	2036037	2064917	2	3963	4666	2	43	45	2
+431	33	2302331	2348449	33	4425	6516	32	69	69	1
+447	59	25125	33094	59	1176	1817	56	53	58	6
+456	1	53157	53157	1	1556	1556	1	26	26	1
+462	5	5456	6280	5	348	4337	5	28	40	5
+472	1	1443716	1443716	1	6122	6122	1	42	42	1
+491	34	1066102	1183673	34	6606	6822	32	46	67	15
+498	896	2230163	3054	895	537	7131	714	24	59	28
diff --git a/tests/queries/0_stateless/02995_index_5.sh b/tests/queries/0_stateless/02995_index_5.sh
new file mode 100755
index 00000000000..60c12a8146d
--- /dev/null
+++ b/tests/queries/0_stateless/02995_index_5.sh
@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+# Tags: long, no-debug, no-asan, no-tsan, no-msan, no-ubsan, no-sanitize-coverage
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+${CLICKHOUSE_CLIENT} --multiquery "
+
+DROP TABLE IF EXISTS test;
+CREATE TABLE test (a String, b String, c String) ENGINE = MergeTree ORDER BY (a, b, c) SETTINGS index_granularity = 11;
+
+INSERT INTO test
+SELECT round(pow(sipHash64(1, number), 1/4)), round(pow(sipHash64(2, number), 1/6)), round(pow(sipHash64(3, number), 1/10))
+FROM numbers(100000);
+
+INSERT INTO test
+SELECT round(pow(sipHash64(1, number), 1/3)), round(pow(sipHash64(2, number), 1/5)), round(pow(sipHash64(3, number), 1/10))
+FROM numbers(100000);
+
+INSERT INTO test
+SELECT round(pow(sipHash64(1, number), 1/5)), round(pow(sipHash64(2, number), 1/7)), round(pow(sipHash64(3, number), 1/10))
+FROM numbers(100000);
+
+DETACH TABLE test;
+ATTACH TABLE test;
+"
+
+for i in {401..500}
+do
+    echo "
+WITH ${i} AS try
+SELECT try, count(), min(a), max(a), uniqExact(a), min(b), max(b), uniqExact(b), min(c), max(c), uniqExact(c) FROM test
+WHERE a >= (round(pow(sipHash64(1, try), 1 / (3 + sipHash64(2, try) % 8))) AS a1)::String
+  AND a <= (a1 + round(pow(sipHash64(3, try), 1 / (3 + sipHash64(4, try) % 8))))::String
+  AND b >= (round(pow(sipHash64(5, try), 1 / (3 + sipHash64(6, try) % 8))) AS b1)::String
+  AND b <= (b1 + round(pow(sipHash64(7, try), 1 / (3 + sipHash64(8, try) % 8))))::String
+  AND c >= (round(pow(sipHash64(9, try), 1 / (3 + sipHash64(10, try) % 8))) AS c1)::String
+  AND c <= (c1 + round(pow(sipHash64(11, try), 1 / (3 + sipHash64(12, try) % 8))))::String
+HAVING count() > 0;
+"
+done | ${CLICKHOUSE_CLIENT} --multiquery
+
+${CLICKHOUSE_CLIENT} --multiquery "DROP TABLE test"
diff --git a/tests/queries/0_stateless/02995_index_6.reference b/tests/queries/0_stateless/02995_index_6.reference
new file mode 100644
index 00000000000..bac19179bb6
--- /dev/null
+++ b/tests/queries/0_stateless/02995_index_6.reference
@@ -0,0 +1,14 @@
+504	108	12281	25180	108	1318	1784	94	55	66	12
+515	22	1588883	2640809	22	6554	6571	15	46	59	12
+518	1	37743	37743	1	1558	1558	1	72	72	1
+530	1	3033	3033	1	561	561	1	59	59	1
+532	26	5721	6355	25	549	665	14	44	50	7
+546	156	2577874	48517	156	1105	324	133	44	51	8
+554	12	1665194	2640066	12	1817	2951	12	57	57	1
+564	3865	2028049	2083433	3722	1115	985	2203	44	84	41
+566	4432	50605	57509	3217	1191	267	459	26	72	39
+567	8	5221	5893	7	333	558	8	27	35	4
+582	1172	1320619	2019743	1172	5819	7131	757	26	63	30
+584	43100	2500	5594	22561	134	4573	1660	48	84	37
+589	28	6046	6068	19	345	564	27	55	62	8
+595	139	1585165	1683606	138	2231	3598	132	54	84	28
diff --git a/tests/queries/0_stateless/02995_index_6.sh b/tests/queries/0_stateless/02995_index_6.sh
new file mode 100755
index 00000000000..4936f73f36b
--- /dev/null
+++ b/tests/queries/0_stateless/02995_index_6.sh
@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+# Tags: long, no-debug, no-asan, no-tsan, no-msan, no-ubsan, no-sanitize-coverage
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+${CLICKHOUSE_CLIENT} --multiquery "
+
+DROP TABLE IF EXISTS test;
+CREATE TABLE test (a String, b String, c String) ENGINE = MergeTree ORDER BY (a, b, c) SETTINGS index_granularity = 11;
+
+INSERT INTO test
+SELECT round(pow(sipHash64(1, number), 1/4)), round(pow(sipHash64(2, number), 1/6)), round(pow(sipHash64(3, number), 1/10))
+FROM numbers(100000);
+
+INSERT INTO test
+SELECT round(pow(sipHash64(1, number), 1/3)), round(pow(sipHash64(2, number), 1/5)), round(pow(sipHash64(3, number), 1/10))
+FROM numbers(100000);
+
+INSERT INTO test
+SELECT round(pow(sipHash64(1, number), 1/5)), round(pow(sipHash64(2, number), 1/7)), round(pow(sipHash64(3, number), 1/10))
+FROM numbers(100000);
+
+DETACH TABLE test;
+ATTACH TABLE test;
+"
+
+for i in {501..600}
+do
+    echo "
+WITH ${i} AS try
+SELECT try, count(), min(a), max(a), uniqExact(a), min(b), max(b), uniqExact(b), min(c), max(c), uniqExact(c) FROM test
+WHERE a >= (round(pow(sipHash64(1, try), 1 / (3 + sipHash64(2, try) % 8))) AS a1)::String
+  AND a <= (a1 + round(pow(sipHash64(3, try), 1 / (3 + sipHash64(4, try) % 8))))::String
+  AND b >= (round(pow(sipHash64(5, try), 1 / (3 + sipHash64(6, try) % 8))) AS b1)::String
+  AND b <= (b1 + round(pow(sipHash64(7, try), 1 / (3 + sipHash64(8, try) % 8))))::String
+  AND c >= (round(pow(sipHash64(9, try), 1 / (3 + sipHash64(10, try) % 8))) AS c1)::String
+  AND c <= (c1 + round(pow(sipHash64(11, try), 1 / (3 + sipHash64(12, try) % 8))))::String
+HAVING count() > 0;
+"
+done | ${CLICKHOUSE_CLIENT} --multiquery
+
+${CLICKHOUSE_CLIENT} --multiquery "DROP TABLE test"
diff --git a/tests/queries/0_stateless/02995_index_7.reference b/tests/queries/0_stateless/02995_index_7.reference
new file mode 100644
index 00000000000..4f1d28ab37e
--- /dev/null
+++ b/tests/queries/0_stateless/02995_index_7.reference
@@ -0,0 +1,9 @@
+615	3	1056081	1116230	3	5794	5796	2	59	62	3
+619	7	1543114	5241	7	2442	3105	7	41	45	3
+634	2722	1221058	4999	2686	2426	7131	1735	54	60	7
+635	237	2119333	4667	237	561	5999	176	49	60	12
+644	5	1774169	2056171	5	5591	6091	4	33	39	3
+647	8	51632	64403	8	1457	1624	8	26	34	5
+651	1325	1620565	6281	1301	528	792	815	62	63	2
+665	13	4598	4789	13	511	558	11	39	46	7
+679	1560	1613200	25940	1552	1569	3118	781	49	84	35
diff --git a/tests/queries/0_stateless/02995_index_7.sh b/tests/queries/0_stateless/02995_index_7.sh
new file mode 100755
index 00000000000..26be310abce
--- /dev/null
+++ b/tests/queries/0_stateless/02995_index_7.sh
@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+# Tags: long, no-debug, no-asan, no-tsan, no-msan, no-ubsan, no-sanitize-coverage
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+${CLICKHOUSE_CLIENT} --multiquery "
+
+DROP TABLE IF EXISTS test;
+CREATE TABLE test (a String, b String, c String) ENGINE = MergeTree ORDER BY (a, b, c) SETTINGS index_granularity = 11;
+
+INSERT INTO test
+SELECT round(pow(sipHash64(1, number), 1/4)), round(pow(sipHash64(2, number), 1/6)), round(pow(sipHash64(3, number), 1/10))
+FROM numbers(100000);
+
+INSERT INTO test
+SELECT round(pow(sipHash64(1, number), 1/3)), round(pow(sipHash64(2, number), 1/5)), round(pow(sipHash64(3, number), 1/10))
+FROM numbers(100000);
+
+INSERT INTO test
+SELECT round(pow(sipHash64(1, number), 1/5)), round(pow(sipHash64(2, number), 1/7)), round(pow(sipHash64(3, number), 1/10))
+FROM numbers(100000);
+
+DETACH TABLE test;
+ATTACH TABLE test;
+"
+
+for i in {601..700}
+do
+    echo "
+WITH ${i} AS try
+SELECT try, count(), min(a), max(a), uniqExact(a), min(b), max(b), uniqExact(b), min(c), max(c), uniqExact(c) FROM test
+WHERE a >= (round(pow(sipHash64(1, try), 1 / (3 + sipHash64(2, try) % 8))) AS a1)::String
+  AND a <= (a1 + round(pow(sipHash64(3, try), 1 / (3 + sipHash64(4, try) % 8))))::String
+  AND b >= (round(pow(sipHash64(5, try), 1 / (3 + sipHash64(6, try) % 8))) AS b1)::String
+  AND b <= (b1 + round(pow(sipHash64(7, try), 1 / (3 + sipHash64(8, try) % 8))))::String
+  AND c >= (round(pow(sipHash64(9, try), 1 / (3 + sipHash64(10, try) % 8))) AS c1)::String
+  AND c <= (c1 + round(pow(sipHash64(11, try), 1 / (3 + sipHash64(12, try) % 8))))::String
+HAVING count() > 0;
+"
+done | ${CLICKHOUSE_CLIENT} --multiquery
+
+${CLICKHOUSE_CLIENT} --multiquery "DROP TABLE test"
diff --git a/tests/queries/0_stateless/02995_index_8.reference b/tests/queries/0_stateless/02995_index_8.reference
new file mode 100644
index 00000000000..62fbfb2af9a
--- /dev/null
+++ b/tests/queries/0_stateless/02995_index_8.reference
@@ -0,0 +1,12 @@
+704	2	14226	15594	2	1086	1116	2	65	71	2
+715	25	1199352	3490	25	5036	5112	23	34	55	13
+716	1253	61989	6735	1050	1203	1625	397	52	65	14
+730	2584	5560	6170	634	2421	627	293	56	69	14
+736	8	1433153	4941	8	339	4594	8	28	36	5
+749	2	1326176	1339862	2	4339	6213	2	49	50	2
+753	1	53157	53157	1	1556	1556	1	26	26	1
+761	63	1443230	6881	63	3154	3204	26	56	73	14
+762	49	1449596	1968154	49	2437	3753	48	54	62	9
+775	35107	5330	769436	2471	447	6607	656	70	81	12
+789	1	1552458	1552458	1	2441	2441	1	62	62	1
+794	158	5585	6585	155	495	929	67	24	50	20
diff --git a/tests/queries/0_stateless/02995_index_8.sh b/tests/queries/0_stateless/02995_index_8.sh
new file mode 100755
index 00000000000..8c2620b59fd
--- /dev/null
+++ b/tests/queries/0_stateless/02995_index_8.sh
@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+# Tags: long, no-debug, no-asan, no-tsan, no-msan, no-ubsan, no-sanitize-coverage
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+${CLICKHOUSE_CLIENT} --multiquery "
+
+DROP TABLE IF EXISTS test;
+CREATE TABLE test (a String, b String, c String) ENGINE = MergeTree ORDER BY (a, b, c) SETTINGS index_granularity = 11;
+
+INSERT INTO test
+SELECT round(pow(sipHash64(1, number), 1/4)), round(pow(sipHash64(2, number), 1/6)), round(pow(sipHash64(3, number), 1/10))
+FROM numbers(100000);
+
+INSERT INTO test
+SELECT round(pow(sipHash64(1, number), 1/3)), round(pow(sipHash64(2, number), 1/5)), round(pow(sipHash64(3, number), 1/10))
+FROM numbers(100000);
+
+INSERT INTO test
+SELECT round(pow(sipHash64(1, number), 1/5)), round(pow(sipHash64(2, number), 1/7)), round(pow(sipHash64(3, number), 1/10))
+FROM numbers(100000);
+
+DETACH TABLE test;
+ATTACH TABLE test;
+"
+
+for i in {701..800}
+do
+    echo "
+WITH ${i} AS try
+SELECT try, count(), min(a), max(a), uniqExact(a), min(b), max(b), uniqExact(b), min(c), max(c), uniqExact(c) FROM test
+WHERE a >= (round(pow(sipHash64(1, try), 1 / (3 + sipHash64(2, try) % 8))) AS a1)::String
+  AND a <= (a1 + round(pow(sipHash64(3, try), 1 / (3 + sipHash64(4, try) % 8))))::String
+  AND b >= (round(pow(sipHash64(5, try), 1 / (3 + sipHash64(6, try) % 8))) AS b1)::String
+  AND b <= (b1 + round(pow(sipHash64(7, try), 1 / (3 + sipHash64(8, try) % 8))))::String
+  AND c >= (round(pow(sipHash64(9, try), 1 / (3 + sipHash64(10, try) % 8))) AS c1)::String
+  AND c <= (c1 + round(pow(sipHash64(11, try), 1 / (3 + sipHash64(12, try) % 8))))::String
+HAVING count() > 0;
+"
+done | ${CLICKHOUSE_CLIENT} --multiquery
+
+${CLICKHOUSE_CLIENT} --multiquery "DROP TABLE test"
diff --git a/tests/queries/0_stateless/02995_index_9.reference b/tests/queries/0_stateless/02995_index_9.reference
new file mode 100644
index 00000000000..86c7be173bd
--- /dev/null
+++ b/tests/queries/0_stateless/02995_index_9.reference
@@ -0,0 +1,9 @@
+839	9	29223	46530	9	1336	1465	9	52	52	1
+844	5	2377545	2377635	5	5129	6321	5	53	69	5
+846	50	2172273	2589295	50	1582	3053	48	64	68	5
+847	2577	56656	63658	1582	1444	838	474	26	63	33
+861	1333	5570	6909	839	457	489	37	33	70	34
+873	2360	1519811	50487	2248	1310	1784	316	60	68	9
+879	228	6704	6785	79	279	507	121	35	66	24
+889	5130	2070007	39692	5040	1151	6791	2606	44	66	23
+896	4	511246	859452	4	6554	6561	4	67	71	4
diff --git a/tests/queries/0_stateless/02995_index_9.sh b/tests/queries/0_stateless/02995_index_9.sh
new file mode 100755
index 00000000000..76160c62aaa
--- /dev/null
+++ b/tests/queries/0_stateless/02995_index_9.sh
@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+# Tags: long, no-debug, no-asan, no-tsan, no-msan, no-ubsan, no-sanitize-coverage
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+${CLICKHOUSE_CLIENT} --multiquery "
+
+DROP TABLE IF EXISTS test;
+CREATE TABLE test (a String, b String, c String) ENGINE = MergeTree ORDER BY (a, b, c) SETTINGS index_granularity = 11;
+
+INSERT INTO test
+SELECT round(pow(sipHash64(1, number), 1/4)), round(pow(sipHash64(2, number), 1/6)), round(pow(sipHash64(3, number), 1/10))
+FROM numbers(100000);
+
+INSERT INTO test
+SELECT round(pow(sipHash64(1, number), 1/3)), round(pow(sipHash64(2, number), 1/5)), round(pow(sipHash64(3, number), 1/10))
+FROM numbers(100000);
+
+INSERT INTO test
+SELECT round(pow(sipHash64(1, number), 1/5)), round(pow(sipHash64(2, number), 1/7)), round(pow(sipHash64(3, number), 1/10))
+FROM numbers(100000);
+
+DETACH TABLE test;
+ATTACH TABLE test;
+"
+
+for i in {801..900}
+do
+    echo "
+WITH ${i} AS try
+SELECT try, count(), min(a), max(a), uniqExact(a), min(b), max(b), uniqExact(b), min(c), max(c), uniqExact(c) FROM test
+WHERE a >= (round(pow(sipHash64(1, try), 1 / (3 + sipHash64(2, try) % 8))) AS a1)::String
+  AND a <= (a1 + round(pow(sipHash64(3, try), 1 / (3 + sipHash64(4, try) % 8))))::String
+  AND b >= (round(pow(sipHash64(5, try), 1 / (3 + sipHash64(6, try) % 8))) AS b1)::String
+  AND b <= (b1 + round(pow(sipHash64(7, try), 1 / (3 + sipHash64(8, try) % 8))))::String
+  AND c >= (round(pow(sipHash64(9, try), 1 / (3 + sipHash64(10, try) % 8))) AS c1)::String
+  AND c <= (c1 + round(pow(sipHash64(11, try), 1 / (3 + sipHash64(12, try) % 8))))::String
+HAVING count() > 0;
+"
+done | ${CLICKHOUSE_CLIENT} --multiquery
+
+${CLICKHOUSE_CLIENT} --multiquery "DROP TABLE test"

From 70d6320577a483d6011872a5f642927c27d095c1 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 22 Jul 2024 06:08:16 +0200
Subject: [PATCH 0541/1488] Split a test for index

---
 .../queries/0_stateless/02995_index.reference | 126 ------------------
 tests/queries/0_stateless/02995_index.sh      |  44 ------
 2 files changed, 170 deletions(-)
 delete mode 100644 tests/queries/0_stateless/02995_index.reference
 delete mode 100755 tests/queries/0_stateless/02995_index.sh

diff --git a/tests/queries/0_stateless/02995_index.reference b/tests/queries/0_stateless/02995_index.reference
deleted file mode 100644
index 1e8639caa88..00000000000
--- a/tests/queries/0_stateless/02995_index.reference
+++ /dev/null
@@ -1,126 +0,0 @@
-12	4	21722	2209341	4	1415	2333	4	61	64	3
-21	1134	11363	58484	1106	1458	1592	136	26	62	32
-22	210	4504	5729	196	291	767	124	47	54	8
-26	196	1327684	5221	195	4140	5661	161	28	49	19
-28	5	2034378	7102	5	325	3255	2	53	60	4
-29	53	45041	45189	45	1580	211	31	55	84	18
-38	424	1600675	4653	424	562	5944	244	60	65	6
-45	17	62743	674873	17	6239	6494	17	65	76	8
-72	1862	1210073	6200	1677	2498	528	859	51	61	11
-79	2	2255228	2255293	2	5495	7057	2	65	65	1
-85	459	1051571	1829638	459	6402	7131	334	32	61	25
-86	10	1748130	1754217	10	4374	7003	10	56	59	4
-91	165	5718	5802	75	282	7113	112	41	63	22
-94	20	1231916	2050003	20	4802	4917	19	53	59	7
-99	2	3665	36667	2	497	697	2	70	71	2
-103	1	2446615	2446615	1	2498	2498	1	58	58	1
-106	72	6149	6699	67	527	826	40	61	61	1
-111	43	2273186	5272	43	492	4923	4	54	72	15
-120	3129	45117	6735	2868	1030	1625	561	59	64	6
-138	2	49243	49374	2	1428	1519	2	47	48	2
-143	100	23321	63639	100	1115	1624	88	51	51	1
-145	1	2447976	2447976	1	6173	6173	1	44	44	1
-153	16	13748	16881	16	1506	1636	16	54	68	9
-159	19952	1525336	7131	12957	1280	6163	2668	24	66	39
-171	5	15042	16698	5	1302	1608	5	65	65	1
-179	6264	1362341	2686	6244	2554	7132	2705	61	67	7
-192	1	1639623	1639623	1	3406	3406	1	32	32	1
-193	1	1429969	1429969	1	7131	7131	1	45	45	1
-207	12	23057	32500	12	1491	1726	12	32	46	7
-221	5081	1366870	6649	3432	4527	5226	687	24	69	39
-228	73	12281	17929	71	1328	2034	63	49	71	18
-229	2	1617478	1723791	2	4590	5578	2	41	42	2
-230	3916	1332729	6949	3668	1330	4703	845	62	65	4
-238	25	2624456	2625673	24	2535	6465	25	58	75	14
-241	154	2554929	2616444	154	2626	7131	148	34	57	17
-248	276	15529	30987	274	1040	1222	136	37	79	27
-254	3018	33966	6635	2837	1057	1622	539	24	60	33
-255	20	1581774	1811334	20	6068	6301	18	33	57	10
-256	5	5145	6841	5	367	376	5	58	58	1
-270	2	2195579	2262119	2	7102	7123	2	33	34	2
-281	32	2379460	616215	32	6042	6086	23	53	64	12
-282	7	1292651	24244	7	1607	2455	6	46	55	5
-286	123	1521935	5269	123	3793	3940	81	40	66	22
-291	21	2419080	3567	21	297	4731	21	54	55	2
-316	4	5221	5616	4	505	558	4	32	35	3
-319	232	56480	63033	230	1599	313	50	33	64	26
-327	15	51647	51894	14	1292	1585	14	47	57	7
-332	24	23484	54948	24	1609	1726	16	32	49	11
-333	1	14189	14189	1	1550	1550	1	63	63	1
-342	49	2579220	2622432	49	4626	6933	48	34	54	14
-344	1	6486	6486	1	509	509	1	24	24	1
-346	1987	53016	6735	1823	1334	174	294	26	62	32
-358	45	59058	60844	44	6746	722	40	57	84	15
-363	1198	1260033	2568811	1196	5710	5790	82	55	80	26
-384	150	2361175	476024	150	7008	7123	81	38	64	22
-387	277	5200	6553	252	243	521	130	65	65	1
-392	1877	1607428	2030850	1875	1416	7131	1379	54	66	13
-396	8181	1380803	6186	7920	545	798	1743	24	67	39
-398	3	5183	5213	2	291	352	3	53	59	3
-399	62	51494	59203	61	7073	754	42	55	78	18
-412	2141	1360120	2189792	2136	2491	5658	1371	71	75	5
-413	2	2036037	2064917	2	3963	4666	2	43	45	2
-431	33	2302331	2348449	33	4425	6516	32	69	69	1
-447	59	25125	33094	59	1176	1817	56	53	58	6
-456	1	53157	53157	1	1556	1556	1	26	26	1
-462	5	5456	6280	5	348	4337	5	28	40	5
-472	1	1443716	1443716	1	6122	6122	1	42	42	1
-491	34	1066102	1183673	34	6606	6822	32	46	67	15
-498	896	2230163	3054	895	537	7131	714	24	59	28
-504	108	12281	25180	108	1318	1784	94	55	66	12
-515	22	1588883	2640809	22	6554	6571	15	46	59	12
-518	1	37743	37743	1	1558	1558	1	72	72	1
-530	1	3033	3033	1	561	561	1	59	59	1
-532	26	5721	6355	25	549	665	14	44	50	7
-546	156	2577874	48517	156	1105	324	133	44	51	8
-554	12	1665194	2640066	12	1817	2951	12	57	57	1
-564	3865	2028049	2083433	3722	1115	985	2203	44	84	41
-566	4432	50605	57509	3217	1191	267	459	26	72	39
-567	8	5221	5893	7	333	558	8	27	35	4
-582	1172	1320619	2019743	1172	5819	7131	757	26	63	30
-584	43100	2500	5594	22561	134	4573	1660	48	84	37
-589	28	6046	6068	19	345	564	27	55	62	8
-595	139	1585165	1683606	138	2231	3598	132	54	84	28
-615	3	1056081	1116230	3	5794	5796	2	59	62	3
-619	7	1543114	5241	7	2442	3105	7	41	45	3
-634	2722	1221058	4999	2686	2426	7131	1735	54	60	7
-635	237	2119333	4667	237	561	5999	176	49	60	12
-644	5	1774169	2056171	5	5591	6091	4	33	39	3
-647	8	51632	64403	8	1457	1624	8	26	34	5
-651	1325	1620565	6281	1301	528	792	815	62	63	2
-665	13	4598	4789	13	511	558	11	39	46	7
-679	1560	1613200	25940	1552	1569	3118	781	49	84	35
-704	2	14226	15594	2	1086	1116	2	65	71	2
-715	25	1199352	3490	25	5036	5112	23	34	55	13
-716	1253	61989	6735	1050	1203	1625	397	52	65	14
-730	2584	5560	6170	634	2421	627	293	56	69	14
-736	8	1433153	4941	8	339	4594	8	28	36	5
-749	2	1326176	1339862	2	4339	6213	2	49	50	2
-753	1	53157	53157	1	1556	1556	1	26	26	1
-761	63	1443230	6881	63	3154	3204	26	56	73	14
-762	49	1449596	1968154	49	2437	3753	48	54	62	9
-775	35107	5330	769436	2471	447	6607	656	70	81	12
-789	1	1552458	1552458	1	2441	2441	1	62	62	1
-794	158	5585	6585	155	495	929	67	24	50	20
-839	9	29223	46530	9	1336	1465	9	52	52	1
-844	5	2377545	2377635	5	5129	6321	5	53	69	5
-846	50	2172273	2589295	50	1582	3053	48	64	68	5
-847	2577	56656	63658	1582	1444	838	474	26	63	33
-861	1333	5570	6909	839	457	489	37	33	70	34
-873	2360	1519811	50487	2248	1310	1784	316	60	68	9
-879	228	6704	6785	79	279	507	121	35	66	24
-889	5130	2070007	39692	5040	1151	6791	2606	44	66	23
-896	4	511246	859452	4	6554	6561	4	67	71	4
-912	146	1322641	2238040	146	1366	6354	143	59	59	1
-913	82	5495	6870	78	350	565	67	24	43	15
-921	763	1580790	416881	763	6191	7131	509	63	64	2
-925	318	2500952	5025	309	476	6114	182	32	56	21
-931	12	4277	4809	12	238	256	9	63	83	9
-942	954	1331	2228193	952	1121	5047	788	65	70	6
-948	14	1785593	2600431	14	6550	6598	13	34	49	9
-956	5	5755	6023	5	359	411	5	43	48	4
-963	4	3812	3835	4	444	537	4	47	53	4
-978	5	51632	58212	5	1127	1556	5	24	32	5
-980	53	47201	59744	53	1537	1625	36	41	49	9
-987	6033	2020131	763444	4306	256	792	1832	60	64	5
-993	4	1615159	1718339	4	1570	3093	4	62	63	2
diff --git a/tests/queries/0_stateless/02995_index.sh b/tests/queries/0_stateless/02995_index.sh
deleted file mode 100755
index 5125d03904e..00000000000
--- a/tests/queries/0_stateless/02995_index.sh
+++ /dev/null
@@ -1,44 +0,0 @@
-#!/usr/bin/env bash
-# Tags: long, no-debug, no-asan, no-tsan, no-msan, no-ubsan, no-sanitize-coverage
-
-CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# shellcheck source=../shell_config.sh
-. "$CURDIR"/../shell_config.sh
-
-${CLICKHOUSE_CLIENT} --multiquery "
-
-DROP TABLE IF EXISTS test;
-CREATE TABLE test (a String, b String, c String) ENGINE = MergeTree ORDER BY (a, b, c) SETTINGS index_granularity = 11;
-
-INSERT INTO test
-SELECT round(pow(sipHash64(1, number), 1/4)), round(pow(sipHash64(2, number), 1/6)), round(pow(sipHash64(3, number), 1/10))
-FROM numbers(100000);
-
-INSERT INTO test
-SELECT round(pow(sipHash64(1, number), 1/3)), round(pow(sipHash64(2, number), 1/5)), round(pow(sipHash64(3, number), 1/10))
-FROM numbers(100000);
-
-INSERT INTO test
-SELECT round(pow(sipHash64(1, number), 1/5)), round(pow(sipHash64(2, number), 1/7)), round(pow(sipHash64(3, number), 1/10))
-FROM numbers(100000);
-
-DETACH TABLE test;
-ATTACH TABLE test;
-"
-
-for i in {1..1000}
-do
-    echo "
-WITH ${i} AS try
-SELECT try, count(), min(a), max(a), uniqExact(a), min(b), max(b), uniqExact(b), min(c), max(c), uniqExact(c) FROM test
-WHERE a >= (round(pow(sipHash64(1, try), 1 / (3 + sipHash64(2, try) % 8))) AS a1)::String
-  AND a <= (a1 + round(pow(sipHash64(3, try), 1 / (3 + sipHash64(4, try) % 8))))::String
-  AND b >= (round(pow(sipHash64(5, try), 1 / (3 + sipHash64(6, try) % 8))) AS b1)::String
-  AND b <= (b1 + round(pow(sipHash64(7, try), 1 / (3 + sipHash64(8, try) % 8))))::String
-  AND c >= (round(pow(sipHash64(9, try), 1 / (3 + sipHash64(10, try) % 8))) AS c1)::String
-  AND c <= (c1 + round(pow(sipHash64(11, try), 1 / (3 + sipHash64(12, try) % 8))))::String
-HAVING count() > 0;
-"
-done | ${CLICKHOUSE_CLIENT} --multiquery
-
-${CLICKHOUSE_CLIENT} --multiquery "DROP TABLE test"

From 1b606e525a9942ef69b0210760380170291d4f1b Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 22 Jul 2024 06:34:10 +0200
Subject: [PATCH 0542/1488] Fix inconsistent formatting of `NOT ((SELECT ...))`

---
 src/Parsers/ASTFunction.cpp                          | 12 ++++++++----
 ...inconsistent_formatting_of_not_subquery.reference |  1 +
 .../03208_inconsistent_formatting_of_not_subquery.sh |  9 +++++++++
 3 files changed, 18 insertions(+), 4 deletions(-)
 create mode 100644 tests/queries/0_stateless/03208_inconsistent_formatting_of_not_subquery.reference
 create mode 100755 tests/queries/0_stateless/03208_inconsistent_formatting_of_not_subquery.sh

diff --git a/src/Parsers/ASTFunction.cpp b/src/Parsers/ASTFunction.cpp
index b04ec1c22b2..230d4c778e8 100644
--- a/src/Parsers/ASTFunction.cpp
+++ b/src/Parsers/ASTFunction.cpp
@@ -329,19 +329,23 @@ void ASTFunction::formatImplWithoutAlias(const FormatSettings & settings, Format
 
                 const auto * literal = arguments->children[0]->as<ASTLiteral>();
                 const auto * function = arguments->children[0]->as<ASTFunction>();
+                const auto * subquery = arguments->children[0]->as<ASTSubquery>();
                 bool is_tuple = literal && literal->value.getType() == Field::Types::Tuple;
-                // do not add parentheses for tuple literal, otherwise extra parens will be added `-((3, 7, 3), 1)` -> `-(((3, 7, 3), 1))`
+                /// Do not add parentheses for tuple literal, otherwise extra parens will be added `-((3, 7, 3), 1)` -> `-(((3, 7, 3), 1))`
                 bool literal_need_parens = literal && !is_tuple;
 
-                // negate always requires parentheses, otherwise -(-1) will be printed as --1
-                bool inside_parens = name == "negate" && (literal_need_parens || (function && function->name == "negate"));
+                /// Negate always requires parentheses, otherwise -(-1) will be printed as --1
+                /// Also extra parentheses are needed for subqueries, because NOT can be parsed as a function:
+                /// not(SELECT 1) cannot be parsed, while not((SELECT 1)) can.
+                bool inside_parens = (name == "negate" && (literal_need_parens || (function && function->name == "negate")))
+                    || (subquery && name == "not");
 
                 /// We DO need parentheses around a single literal
                 /// For example, SELECT (NOT 0) + (NOT 0) cannot be transformed into SELECT NOT 0 + NOT 0, since
                 /// this is equal to SELECT NOT (0 + NOT 0)
                 bool outside_parens = frame.need_parens && !inside_parens;
 
-                // do not add extra parentheses for functions inside negate, i.e. -(-toUInt64(-(1)))
+                /// Do not add extra parentheses for functions inside negate, i.e. -(-toUInt64(-(1)))
                 if (inside_parens)
                     nested_need_parens.need_parens = false;
 
diff --git a/tests/queries/0_stateless/03208_inconsistent_formatting_of_not_subquery.reference b/tests/queries/0_stateless/03208_inconsistent_formatting_of_not_subquery.reference
new file mode 100644
index 00000000000..a1afeb1ab82
--- /dev/null
+++ b/tests/queries/0_stateless/03208_inconsistent_formatting_of_not_subquery.reference
@@ -0,0 +1 @@
+SELECT NOT ((SELECT 1))
diff --git a/tests/queries/0_stateless/03208_inconsistent_formatting_of_not_subquery.sh b/tests/queries/0_stateless/03208_inconsistent_formatting_of_not_subquery.sh
new file mode 100755
index 00000000000..5f7397015fa
--- /dev/null
+++ b/tests/queries/0_stateless/03208_inconsistent_formatting_of_not_subquery.sh
@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+# Tags: no-fasttest
+# Tag no-fasttest: In fasttest, ENABLE_LIBRARIES=0, so the grpc library is not built
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+$CLICKHOUSE_FORMAT --oneline --query "SELECT NOT((SELECT 1))"

From efb6491cfdc10c2b8ba6268c29677ea298046bed Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 22 Jul 2024 06:34:57 +0200
Subject: [PATCH 0543/1488] Update
 03208_inconsistent_formatting_of_not_subquery.sh

---
 .../03208_inconsistent_formatting_of_not_subquery.sh            | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tests/queries/0_stateless/03208_inconsistent_formatting_of_not_subquery.sh b/tests/queries/0_stateless/03208_inconsistent_formatting_of_not_subquery.sh
index 5f7397015fa..594d316b621 100755
--- a/tests/queries/0_stateless/03208_inconsistent_formatting_of_not_subquery.sh
+++ b/tests/queries/0_stateless/03208_inconsistent_formatting_of_not_subquery.sh
@@ -1,6 +1,4 @@
 #!/usr/bin/env bash
-# Tags: no-fasttest
-# Tag no-fasttest: In fasttest, ENABLE_LIBRARIES=0, so the grpc library is not built
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh

From 989476d5234bffd223988bf8aa88e2021e999574 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 22 Jul 2024 06:41:52 +0200
Subject: [PATCH 0544/1488] Make test `01592_long_window_functions1` lighter

---
 .../0_stateless/01592_long_window_functions1.sql   | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/tests/queries/0_stateless/01592_long_window_functions1.sql b/tests/queries/0_stateless/01592_long_window_functions1.sql
index d2d32e24eaa..671245599cc 100644
--- a/tests/queries/0_stateless/01592_long_window_functions1.sql
+++ b/tests/queries/0_stateless/01592_long_window_functions1.sql
@@ -8,14 +8,14 @@ drop table if exists stack;
 set max_insert_threads = 4;
 
 create table stack(item_id Int64, brand_id Int64, rack_id Int64, dt DateTime, expiration_dt DateTime, quantity UInt64)
-Engine = MergeTree 
-partition by toYYYYMM(dt) 
+Engine = MergeTree
+partition by toYYYYMM(dt)
 order by (brand_id, toStartOfHour(dt)) SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';
 
-insert into stack 
-select number%99991, number%11, number%1111, toDateTime('2020-01-01 00:00:00')+number/100, 
+insert into stack
+select number%99991, number%11, number%1111, toDateTime('2020-01-01 00:00:00')+number/100,
    toDateTime('2020-02-01 00:00:00')+number/10, intDiv(number,100)+1
-from numbers_mt(10000000);
+from numbers_mt(1000000);
 
 select '---- arrays ----';
 
@@ -32,8 +32,8 @@ select '---- window f ----';
 select cityHash64( toString( groupArray (tuple(*) ) )) from (
     select brand_id, rack_id,  quantity from
        ( select brand_id, rack_id, quantity, row_number() over (partition by brand_id, rack_id order by quantity) rn
-         from stack ) as t0 
-    where rn <= 2  
+         from stack ) as t0
+    where rn <= 2
     order by brand_id, rack_id, quantity
 ) t;
 

From db549c93a18f49540676ae53bc04e75b85705ddb Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 22 Jul 2024 07:34:34 +0200
Subject: [PATCH 0545/1488] Fix error

---
 src/IO/ReadWriteBufferFromHTTP.cpp | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/src/IO/ReadWriteBufferFromHTTP.cpp b/src/IO/ReadWriteBufferFromHTTP.cpp
index 4d27a78c8dc..cea1a272401 100644
--- a/src/IO/ReadWriteBufferFromHTTP.cpp
+++ b/src/IO/ReadWriteBufferFromHTTP.cpp
@@ -123,7 +123,16 @@ void ReadWriteBufferFromHTTP::prepareRequest(Poco::Net::HTTPRequest & request, s
 std::optional<size_t> ReadWriteBufferFromHTTP::tryGetFileSize()
 {
     if (!file_info)
-        file_info = getFileInfo();
+    {
+        try
+        {
+            file_info = getFileInfo();
+        }
+        catch (const HTTPException & e)
+        {
+            return std::nullopt;
+        }
+    }
 
     return file_info->file_size;
 }
@@ -679,7 +688,7 @@ std::optional<time_t> ReadWriteBufferFromHTTP::tryGetLastModificationTime()
         {
             file_info = getFileInfo();
         }
-        catch (...)
+        catch (const HTTPException & e)
         {
             return std::nullopt;
         }
@@ -700,7 +709,7 @@ ReadWriteBufferFromHTTP::HTTPFileInfo ReadWriteBufferFromHTTP::getFileInfo()
     {
         getHeadResponse(response);
     }
-    catch (HTTPException & e)
+    catch (const HTTPException & e)
     {
         /// Maybe the web server doesn't support HEAD requests.
         /// E.g. webhdfs reports status 400.

From 32f624eebaa560f4c9d6bf9145931270098e8db1 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 22 Jul 2024 07:35:10 +0200
Subject: [PATCH 0546/1488] Fix error

---
 src/IO/ReadWriteBufferFromHTTP.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/IO/ReadWriteBufferFromHTTP.cpp b/src/IO/ReadWriteBufferFromHTTP.cpp
index cea1a272401..961e8dd6425 100644
--- a/src/IO/ReadWriteBufferFromHTTP.cpp
+++ b/src/IO/ReadWriteBufferFromHTTP.cpp
@@ -128,7 +128,7 @@ std::optional<size_t> ReadWriteBufferFromHTTP::tryGetFileSize()
         {
             file_info = getFileInfo();
         }
-        catch (const HTTPException & e)
+        catch (const HTTPException &)
         {
             return std::nullopt;
         }

From dc601dc7455895574143f5baf345731d437bf8d3 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 22 Jul 2024 07:37:15 +0200
Subject: [PATCH 0547/1488] Fix error

---
 src/IO/ReadWriteBufferFromHTTP.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/IO/ReadWriteBufferFromHTTP.cpp b/src/IO/ReadWriteBufferFromHTTP.cpp
index 961e8dd6425..85230957b3f 100644
--- a/src/IO/ReadWriteBufferFromHTTP.cpp
+++ b/src/IO/ReadWriteBufferFromHTTP.cpp
@@ -688,7 +688,7 @@ std::optional<time_t> ReadWriteBufferFromHTTP::tryGetLastModificationTime()
         {
             file_info = getFileInfo();
         }
-        catch (const HTTPException & e)
+        catch (const HTTPException &)
         {
             return std::nullopt;
         }

From de4a97a9a732945c3b3df4cc903a1dea30fffc5a Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 22 Jul 2024 07:41:15 +0200
Subject: [PATCH 0548/1488] Remove support for -WithDictionary suffix for data
 types

---
 src/DataTypes/DataTypeFactory.cpp | 17 -----------------
 1 file changed, 17 deletions(-)

diff --git a/src/DataTypes/DataTypeFactory.cpp b/src/DataTypes/DataTypeFactory.cpp
index af37cde2846..eb3bc973857 100644
--- a/src/DataTypes/DataTypeFactory.cpp
+++ b/src/DataTypes/DataTypeFactory.cpp
@@ -125,23 +125,6 @@ DataTypePtr DataTypeFactory::getImpl(const String & family_name_param, const AST
 {
     String family_name = getAliasToOrName(family_name_param);
 
-    if (endsWith(family_name, "WithDictionary"))
-    {
-        ASTPtr low_cardinality_params = std::make_shared<ASTExpressionList>();
-        String param_name = family_name.substr(0, family_name.size() - strlen("WithDictionary"));
-        if (parameters)
-        {
-            auto func = std::make_shared<ASTFunction>();
-            func->name = param_name;
-            func->arguments = parameters;
-            low_cardinality_params->children.push_back(func);
-        }
-        else
-            low_cardinality_params->children.push_back(std::make_shared<ASTIdentifier>(param_name));
-
-        return getImpl<nullptr_on_error>("LowCardinality", low_cardinality_params);
-    }
-
     const auto * creator = findCreatorByName<nullptr_on_error>(family_name);
     if constexpr (nullptr_on_error)
     {

From f0f3bcfee972ba28928a9f33afd374d369d9babf Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 22 Jul 2024 07:45:57 +0200
Subject: [PATCH 0549/1488] Update tests

---
 ...cardinality_dictionary_deserialization.sql |  2 +-
 .../0_stateless/00688_low_cardinality_in.sql  |  2 +-
 .../00688_low_cardinality_prewhere.sql        |  2 +-
 .../00688_low_cardinality_serialization.sql   |  4 +--
 .../00688_low_cardinality_syntax.reference    |  6 -----
 .../00688_low_cardinality_syntax.sql          | 26 +------------------
 .../00717_low_cardinaliry_group_by.sql        |  2 +-
 .../00718_low_cardinaliry_alter.sql           |  2 +-
 .../00752_low_cardinality_mv_1.sql            |  3 +--
 .../02235_remote_fs_cache_stress.sh           |  2 +-
 10 files changed, 10 insertions(+), 41 deletions(-)

diff --git a/tests/queries/0_stateless/00688_low_cardinality_dictionary_deserialization.sql b/tests/queries/0_stateless/00688_low_cardinality_dictionary_deserialization.sql
index c4613acf5f3..d359efd8d42 100644
--- a/tests/queries/0_stateless/00688_low_cardinality_dictionary_deserialization.sql
+++ b/tests/queries/0_stateless/00688_low_cardinality_dictionary_deserialization.sql
@@ -1,5 +1,5 @@
 drop table if exists lc_dict_reading;
-create table lc_dict_reading (val UInt64, str StringWithDictionary, pat String) engine = MergeTree order by val SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';
+create table lc_dict_reading (val UInt64, str LowCardinality(String), pat String) engine = MergeTree order by val SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';
 insert into lc_dict_reading select number, if(number < 8192 * 4, number % 100, number) as s, s from system.numbers limit 1000000;
 select sum(toUInt64(str)), sum(toUInt64(pat)) from lc_dict_reading where val < 8129 or val > 8192 * 4;
 drop table if exists lc_dict_reading;
diff --git a/tests/queries/0_stateless/00688_low_cardinality_in.sql b/tests/queries/0_stateless/00688_low_cardinality_in.sql
index cb57fad51a4..c39fdb37160 100644
--- a/tests/queries/0_stateless/00688_low_cardinality_in.sql
+++ b/tests/queries/0_stateless/00688_low_cardinality_in.sql
@@ -1,6 +1,6 @@
 set allow_suspicious_low_cardinality_types = 1;
 drop table if exists lc_00688;
-create table lc_00688 (str StringWithDictionary, val UInt8WithDictionary) engine = MergeTree order by tuple();
+create table lc_00688 (str LowCardinality(String), val LowCardinality(UInt8)) engine = MergeTree order by tuple();
 insert into lc_00688 values ('a', 1), ('b', 2);
 select str, str in ('a', 'd') from lc_00688;
 select val, val in (1, 3) from lc_00688;
diff --git a/tests/queries/0_stateless/00688_low_cardinality_prewhere.sql b/tests/queries/0_stateless/00688_low_cardinality_prewhere.sql
index a15b2540fe6..17c74b7ca05 100644
--- a/tests/queries/0_stateless/00688_low_cardinality_prewhere.sql
+++ b/tests/queries/0_stateless/00688_low_cardinality_prewhere.sql
@@ -1,5 +1,5 @@
 drop table if exists lc_prewhere;
-create table lc_prewhere (key UInt64, val UInt64, str StringWithDictionary, s String) engine = MergeTree order by key settings index_granularity = 8192;
+create table lc_prewhere (key UInt64, val UInt64, str LowCardinality(String), s String) engine = MergeTree order by key settings index_granularity = 8192;
 insert into lc_prewhere select number, if(number < 10 or number > 8192 * 9, 1, 0), toString(number) as s, s from system.numbers limit 100000;
 select sum(toUInt64(str)), sum(toUInt64(s)) from lc_prewhere prewhere val == 1;
 drop table if exists lc_prewhere;
diff --git a/tests/queries/0_stateless/00688_low_cardinality_serialization.sql b/tests/queries/0_stateless/00688_low_cardinality_serialization.sql
index b4fe4b29200..1e4de3f3d3e 100644
--- a/tests/queries/0_stateless/00688_low_cardinality_serialization.sql
+++ b/tests/queries/0_stateless/00688_low_cardinality_serialization.sql
@@ -8,8 +8,8 @@ select 'MergeTree';
 drop table if exists lc_small_dict;
 drop table if exists lc_big_dict;
 
-create table lc_small_dict (str StringWithDictionary) engine = MergeTree order by str SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';
-create table lc_big_dict (str StringWithDictionary) engine = MergeTree order by str SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';
+create table lc_small_dict (str LowCardinality(String)) engine = MergeTree order by str SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';
+create table lc_big_dict (str LowCardinality(String)) engine = MergeTree order by str SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';
 
 insert into lc_small_dict select toString(number % 1000) from system.numbers limit 1000000;
 insert into lc_big_dict select toString(number) from system.numbers limit 1000000;
diff --git a/tests/queries/0_stateless/00688_low_cardinality_syntax.reference b/tests/queries/0_stateless/00688_low_cardinality_syntax.reference
index ca27069a7df..b06beb0d6f0 100644
--- a/tests/queries/0_stateless/00688_low_cardinality_syntax.reference
+++ b/tests/queries/0_stateless/00688_low_cardinality_syntax.reference
@@ -1,13 +1,7 @@
 a
 a
-a
-a
 1
 1
-1
-1
-ab
-ab
 ab
 ab
 -
diff --git a/tests/queries/0_stateless/00688_low_cardinality_syntax.sql b/tests/queries/0_stateless/00688_low_cardinality_syntax.sql
index a11d9e2d9fe..dccdac1d95d 100644
--- a/tests/queries/0_stateless/00688_low_cardinality_syntax.sql
+++ b/tests/queries/0_stateless/00688_low_cardinality_syntax.sql
@@ -13,56 +13,32 @@ drop table if exists lc_null_fix_str_0;
 drop table if exists lc_null_fix_str_1;
 
 create table lc_str_0 (str LowCardinality(String)) engine = Memory;
-create table lc_str_1 (str StringWithDictionary) engine = Memory;
 create table lc_null_str_0 (str LowCardinality(Nullable(String))) engine = Memory;
-create table lc_null_str_1 (str NullableWithDictionary(String)) engine = Memory;
 create table lc_int8_0 (val LowCardinality(Int8)) engine = Memory;
-create table lc_int8_1 (val Int8WithDictionary) engine = Memory;
 create table lc_null_int8_0 (val LowCardinality(Nullable(Int8))) engine = Memory;
-create table lc_null_int8_1 (val NullableWithDictionary(Int8)) engine = Memory;
 create table lc_fix_str_0 (str LowCardinality(FixedString(2))) engine = Memory;
-create table lc_fix_str_1 (str FixedStringWithDictionary(2)) engine = Memory;
 create table lc_null_fix_str_0 (str LowCardinality(Nullable(FixedString(2)))) engine = Memory;
-create table lc_null_fix_str_1 (str NullableWithDictionary(FixedString(2))) engine = Memory;
 
 insert into lc_str_0 select 'a';
-insert into lc_str_1 select 'a';
 insert into lc_null_str_0 select 'a';
-insert into lc_null_str_1 select 'a';
 insert into lc_int8_0 select 1;
-insert into lc_int8_1 select 1;
 insert into lc_null_int8_0 select 1;
-insert into lc_null_int8_1 select 1;
 insert into lc_fix_str_0 select 'ab';
-insert into lc_fix_str_1 select 'ab';
 insert into lc_null_fix_str_0 select 'ab';
-insert into lc_null_fix_str_1 select 'ab';
 
 select str from lc_str_0;
-select str from lc_str_1;
 select str from lc_null_str_0;
-select str from lc_null_str_1;
 select val from lc_int8_0;
-select val from lc_int8_1;
 select val from lc_null_int8_0;
-select val from lc_null_int8_1;
 select str from lc_fix_str_0;
-select str from lc_fix_str_1;
 select str from lc_null_fix_str_0;
-select str from lc_null_fix_str_1;
 
 drop table if exists lc_str_0;
-drop table if exists lc_str_1;
 drop table if exists lc_null_str_0;
-drop table if exists lc_null_str_1;
 drop table if exists lc_int8_0;
-drop table if exists lc_int8_1;
 drop table if exists lc_null_int8_0;
-drop table if exists lc_null_int8_1;
 drop table if exists lc_fix_str_0;
-drop table if exists lc_fix_str_1;
 drop table if exists lc_null_fix_str_0;
-drop table if exists lc_null_fix_str_1;
 
 select '-';
 SELECT toLowCardinality('a') AS s, toTypeName(s), toTypeName(length(s)) from system.one;
@@ -73,7 +49,7 @@ select (toLowCardinality(z) as val) || 'b'  from (select arrayJoin(['c', 'd']) a
 
 select '-';
 drop table if exists lc_str_uuid;
-create table lc_str_uuid(str1 String, str2 LowCardinality(String), str3 StringWithDictionary) ENGINE=Memory;
+create table lc_str_uuid(str1 String, str2 LowCardinality(String), str3 LowCardinality(String)) ENGINE=Memory;
 select toUUID(str1), toUUID(str2), toUUID(str3) from lc_str_uuid;
 select toUUID(str1, '', NULL), toUUID(str2, '', NULL), toUUID(str3, '', NULL) from lc_str_uuid;
 insert into lc_str_uuid values ('61f0c404-5cb3-11e7-907b-a6006ad3dba0', '61f0c404-5cb3-11e7-907b-a6006ad3dba0', '61f0c404-5cb3-11e7-907b-a6006ad3dba0');
diff --git a/tests/queries/0_stateless/00717_low_cardinaliry_group_by.sql b/tests/queries/0_stateless/00717_low_cardinaliry_group_by.sql
index 02915d4e611..3115ab508fe 100644
--- a/tests/queries/0_stateless/00717_low_cardinaliry_group_by.sql
+++ b/tests/queries/0_stateless/00717_low_cardinaliry_group_by.sql
@@ -1,5 +1,5 @@
 drop table if exists tab_00717;
-create table tab_00717 (a String, b StringWithDictionary) engine = MergeTree order by a;
+create table tab_00717 (a String, b LowCardinality(String)) engine = MergeTree order by a;
 insert into tab_00717 values ('a_1', 'b_1'), ('a_2', 'b_2');
 select count() from tab_00717;
 select a from tab_00717 group by a order by a;
diff --git a/tests/queries/0_stateless/00718_low_cardinaliry_alter.sql b/tests/queries/0_stateless/00718_low_cardinaliry_alter.sql
index 591ff952132..524e396bcb1 100644
--- a/tests/queries/0_stateless/00718_low_cardinaliry_alter.sql
+++ b/tests/queries/0_stateless/00718_low_cardinaliry_alter.sql
@@ -7,7 +7,7 @@ alter table tab_00718 modify column b UInt32;
 select *, toTypeName(b) from tab_00718;
 alter table tab_00718 modify column b LowCardinality(UInt32);
 select *, toTypeName(b) from tab_00718;
-alter table tab_00718 modify column b StringWithDictionary;
+alter table tab_00718 modify column b LowCardinality(String);
 select *, toTypeName(b) from tab_00718;
 alter table tab_00718 modify column b LowCardinality(UInt32);
 select *, toTypeName(b) from tab_00718;
diff --git a/tests/queries/0_stateless/00752_low_cardinality_mv_1.sql b/tests/queries/0_stateless/00752_low_cardinality_mv_1.sql
index 60cc30ec2c3..4fdd0a2496e 100644
--- a/tests/queries/0_stateless/00752_low_cardinality_mv_1.sql
+++ b/tests/queries/0_stateless/00752_low_cardinality_mv_1.sql
@@ -1,7 +1,7 @@
 drop table if exists lc_00752;
 drop table if exists lc_mv_00752;
 
-create table lc_00752 (str StringWithDictionary) engine = MergeTree order by tuple();
+create table lc_00752 (str LowCardinality(String)) engine = MergeTree order by tuple();
 
 insert into lc_00752 values ('a'), ('bbb'), ('ab'), ('accccc'), ('baasddas'), ('bcde');
 
@@ -12,4 +12,3 @@ select * from lc_mv_00752 order by letter;
 
 drop table if exists lc_00752;
 drop table if exists lc_mv_00752;
-
diff --git a/tests/queries/0_stateless/02235_remote_fs_cache_stress.sh b/tests/queries/0_stateless/02235_remote_fs_cache_stress.sh
index 0b6b9f461b0..ffc38c0c1bd 100755
--- a/tests/queries/0_stateless/02235_remote_fs_cache_stress.sh
+++ b/tests/queries/0_stateless/02235_remote_fs_cache_stress.sh
@@ -28,7 +28,7 @@ ORDER BY tuple();
 
 INSERT INTO t_01411_num (num) SELECT number % 1000 FROM numbers(100000);
 
-create table lc_dict_reading (val UInt64, str StringWithDictionary, pat String) engine = MergeTree order by val;
+create table lc_dict_reading (val UInt64, str LowCardinality(String), pat String) engine = MergeTree order by val;
 insert into lc_dict_reading select number, if(number < 8192 * 4, number % 100, number) as s, s from system.numbers limit 100000;
 """
 

From 049056e55aa974ecad8786fb1ec738c96a191118 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Mon, 22 Jul 2024 05:48:40 +0000
Subject: [PATCH 0550/1488] Fix style

---
 src/Functions/dateDiff.cpp | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/Functions/dateDiff.cpp b/src/Functions/dateDiff.cpp
index a39cbae4e30..faab42817ba 100644
--- a/src/Functions/dateDiff.cpp
+++ b/src/Functions/dateDiff.cpp
@@ -26,8 +26,6 @@ namespace DB
 
 namespace ErrorCodes
 {
-    extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
-    extern const int ILLEGAL_TYPE_OF_ARGUMENT;
     extern const int ILLEGAL_COLUMN;
     extern const int BAD_ARGUMENTS;
 }

From a564f70b66367ee4363d46e45eb5a9c66f131fec Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 22 Jul 2024 08:09:39 +0200
Subject: [PATCH 0551/1488] Fix error

---
 src/Parsers/ParserAlterQuery.cpp       | 2 --
 src/Parsers/ParserCreateIndexQuery.cpp | 4 ++--
 src/Parsers/ParserCreateQuery.cpp      | 4 ++--
 3 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/src/Parsers/ParserAlterQuery.cpp b/src/Parsers/ParserAlterQuery.cpp
index 28dbf781011..dbefb0cb966 100644
--- a/src/Parsers/ParserAlterQuery.cpp
+++ b/src/Parsers/ParserAlterQuery.cpp
@@ -9,8 +9,6 @@
 #include <Parsers/ParserRefreshStrategy.h>
 #include <Parsers/ParserSelectWithUnionQuery.h>
 #include <Parsers/ParserSetQuery.h>
-#include <Parsers/ASTIdentifier.h>
-#include <Parsers/ASTIndexDeclaration.h>
 #include <Parsers/ASTAlterQuery.h>
 #include <Parsers/ASTLiteral.h>
 #include <Parsers/parseDatabaseAndTableName.h>
diff --git a/src/Parsers/ParserCreateIndexQuery.cpp b/src/Parsers/ParserCreateIndexQuery.cpp
index 2fa34696c58..ddefb3d37fb 100644
--- a/src/Parsers/ParserCreateIndexQuery.cpp
+++ b/src/Parsers/ParserCreateIndexQuery.cpp
@@ -21,7 +21,7 @@ bool ParserCreateIndexDeclaration::parseImpl(Pos & pos, ASTPtr & node, Expected
     ParserToken close_p(TokenType::ClosingRoundBracket);
     ParserOrderByExpressionList order_list_p;
 
-    ParserDataType data_type_p;
+    ParserFunction type_p;
     ParserExpression expression_p;
     ParserUnsignedInteger granularity_p;
 
@@ -68,7 +68,7 @@ bool ParserCreateIndexDeclaration::parseImpl(Pos & pos, ASTPtr & node, Expected
 
     if (s_type.ignore(pos, expected))
     {
-        if (!data_type_p.parse(pos, type, expected))
+        if (!type_p.parse(pos, type, expected))
             return false;
     }
 
diff --git a/src/Parsers/ParserCreateQuery.cpp b/src/Parsers/ParserCreateQuery.cpp
index fa232954cd6..3dba58546af 100644
--- a/src/Parsers/ParserCreateQuery.cpp
+++ b/src/Parsers/ParserCreateQuery.cpp
@@ -179,7 +179,7 @@ bool ParserIndexDeclaration::parseImpl(Pos & pos, ASTPtr & node, Expected & expe
     ParserKeyword s_granularity(Keyword::GRANULARITY);
 
     ParserIdentifier name_p;
-    ParserDataType data_type_p;
+    ParserFunction type_p;
     ParserExpression expression_p;
     ParserUnsignedInteger granularity_p;
 
@@ -197,7 +197,7 @@ bool ParserIndexDeclaration::parseImpl(Pos & pos, ASTPtr & node, Expected & expe
     if (!s_type.ignore(pos, expected))
         return false;
 
-    if (!data_type_p.parse(pos, type, expected))
+    if (!type_p.parse(pos, type, expected))
         return false;
 
     if (s_granularity.ignore(pos, expected))

From 8d7471f8bd2e0c6dc242231c4358448787e6c56f Mon Sep 17 00:00:00 2001
From: Alexander Gololobov <davenger@clickhouse.com>
Date: Sat, 20 Jul 2024 00:03:40 +0200
Subject: [PATCH 0552/1488] Fix for deadlock in getDDLWorker

---
 src/Interpreters/Context.cpp | 24 ++++++++++++++----------
 1 file changed, 14 insertions(+), 10 deletions(-)

diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp
index 94bcb88ed53..48878733a00 100644
--- a/src/Interpreters/Context.cpp
+++ b/src/Interpreters/Context.cpp
@@ -3490,18 +3490,22 @@ DDLWorker & Context::getDDLWorker() const
     if (shared->ddl_worker_startup_task)
         waitLoad(shared->ddl_worker_startup_task); // Just wait and do not prioritize, because it depends on all load and startup tasks
 
-    SharedLockGuard lock(shared->mutex);
-    if (!shared->ddl_worker)
     {
-        if (!hasZooKeeper())
-            throw Exception(ErrorCodes::NO_ELEMENTS_IN_CONFIG, "There is no Zookeeper configuration in server config");
-
-        if (!hasDistributedDDL())
-            throw Exception(ErrorCodes::NO_ELEMENTS_IN_CONFIG, "There is no DistributedDDL configuration in server config");
-
-        throw Exception(ErrorCodes::NO_ELEMENTS_IN_CONFIG, "DDL background thread is not initialized");
+        /// Only acquire the lock for reading ddl_worker field.
+        /// hasZooKeeper() and hasDistributedDDL() acquire the same lock as well and double acquisition of the lock in shared mode can lead
+        /// to a deadlock if an exclusive lock attempt is made in the meantime by another thread.
+        SharedLockGuard lock(shared->mutex);
+        if (shared->ddl_worker)
+            return *shared->ddl_worker;
     }
-    return *shared->ddl_worker;
+
+    if (!hasZooKeeper())
+        throw Exception(ErrorCodes::NO_ELEMENTS_IN_CONFIG, "There is no Zookeeper configuration in server config");
+
+    if (!hasDistributedDDL())
+        throw Exception(ErrorCodes::NO_ELEMENTS_IN_CONFIG, "There is no DistributedDDL configuration in server config");
+
+    throw Exception(ErrorCodes::NO_ELEMENTS_IN_CONFIG, "DDL background thread is not initialized");
 }
 
 zkutil::ZooKeeperPtr Context::getZooKeeper() const

From bbbf2fec88a331281bf51d3a02bd3f476e3bf6ab Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 22 Jul 2024 08:37:54 +0200
Subject: [PATCH 0553/1488] Fix error

---
 src/Parsers/ParserCreateIndexQuery.cpp | 9 +++++++--
 src/Parsers/ParserCreateQuery.cpp      | 7 ++++++-
 2 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/src/Parsers/ParserCreateIndexQuery.cpp b/src/Parsers/ParserCreateIndexQuery.cpp
index ddefb3d37fb..8a4c1c0b17a 100644
--- a/src/Parsers/ParserCreateIndexQuery.cpp
+++ b/src/Parsers/ParserCreateIndexQuery.cpp
@@ -7,9 +7,9 @@
 #include <Parsers/ASTLiteral.h>
 #include <Parsers/CommonParsers.h>
 #include <Parsers/ExpressionListParsers.h>
-#include <Parsers/ParserDataType.h>
 #include <Parsers/parseDatabaseAndTableName.h>
 
+
 namespace DB
 {
 
@@ -69,7 +69,12 @@ bool ParserCreateIndexDeclaration::parseImpl(Pos & pos, ASTPtr & node, Expected
     if (s_type.ignore(pos, expected))
     {
         if (!type_p.parse(pos, type, expected))
-            return false;
+        {
+            if (ParserIdentifier().parse(pos, type, expected))
+                type = makeASTFunction(type->as<ASTIdentifier &>().name());
+            else
+                return false;
+        }
     }
 
     if (s_granularity.ignore(pos, expected))
diff --git a/src/Parsers/ParserCreateQuery.cpp b/src/Parsers/ParserCreateQuery.cpp
index 3dba58546af..bff5da4a536 100644
--- a/src/Parsers/ParserCreateQuery.cpp
+++ b/src/Parsers/ParserCreateQuery.cpp
@@ -198,7 +198,12 @@ bool ParserIndexDeclaration::parseImpl(Pos & pos, ASTPtr & node, Expected & expe
         return false;
 
     if (!type_p.parse(pos, type, expected))
-        return false;
+    {
+        if (name_p.parse(pos, type, expected))
+            type = makeASTFunction(type->as<ASTIdentifier &>().name());
+        else
+            return false;
+    }
 
     if (s_granularity.ignore(pos, expected))
     {

From ed02246e0c2019f9ee661e9636f166b955a672de Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 22 Jul 2024 08:46:41 +0200
Subject: [PATCH 0554/1488] What if I will change this test?

---
 tests/integration/test_ssl_cert_authentication/test.py | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/tests/integration/test_ssl_cert_authentication/test.py b/tests/integration/test_ssl_cert_authentication/test.py
index 756a1e1996c..3af88759e82 100644
--- a/tests/integration/test_ssl_cert_authentication/test.py
+++ b/tests/integration/test_ssl_cert_authentication/test.py
@@ -43,15 +43,10 @@ def started_cluster():
 config = """<clickhouse>
     <openSSL>
         <client>
-            <verificationMode>none</verificationMode>
-
+            <verificationMode>strict</verificationMode>
             <certificateFile>{certificateFile}</certificateFile>
             <privateKeyFile>{privateKeyFile}</privateKeyFile>
             <caConfig>{caConfig}</caConfig>
-
-            <invalidCertificateHandler>
-                <name>AcceptCertificateHandler</name>
-            </invalidCertificateHandler>
         </client>
     </openSSL>
 </clickhouse>"""

From 3d4604a64ad01c70c80924ea11154514bab4e424 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 22 Jul 2024 08:47:08 +0200
Subject: [PATCH 0555/1488] Revert "What if I will change this test?"

This reverts commit ed02246e0c2019f9ee661e9636f166b955a672de.
---
 tests/integration/test_ssl_cert_authentication/test.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/tests/integration/test_ssl_cert_authentication/test.py b/tests/integration/test_ssl_cert_authentication/test.py
index 3af88759e82..756a1e1996c 100644
--- a/tests/integration/test_ssl_cert_authentication/test.py
+++ b/tests/integration/test_ssl_cert_authentication/test.py
@@ -43,10 +43,15 @@ def started_cluster():
 config = """<clickhouse>
     <openSSL>
         <client>
-            <verificationMode>strict</verificationMode>
+            <verificationMode>none</verificationMode>
+
             <certificateFile>{certificateFile}</certificateFile>
             <privateKeyFile>{privateKeyFile}</privateKeyFile>
             <caConfig>{caConfig}</caConfig>
+
+            <invalidCertificateHandler>
+                <name>AcceptCertificateHandler</name>
+            </invalidCertificateHandler>
         </client>
     </openSSL>
 </clickhouse>"""

From 1a3559fbc3c5257c4e0f5ec16eda3d09d8ebcca0 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 22 Jul 2024 08:46:41 +0200
Subject: [PATCH 0556/1488] What if I will change this test?

---
 tests/integration/test_ssl_cert_authentication/test.py | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/tests/integration/test_ssl_cert_authentication/test.py b/tests/integration/test_ssl_cert_authentication/test.py
index 756a1e1996c..3af88759e82 100644
--- a/tests/integration/test_ssl_cert_authentication/test.py
+++ b/tests/integration/test_ssl_cert_authentication/test.py
@@ -43,15 +43,10 @@ def started_cluster():
 config = """<clickhouse>
     <openSSL>
         <client>
-            <verificationMode>none</verificationMode>
-
+            <verificationMode>strict</verificationMode>
             <certificateFile>{certificateFile}</certificateFile>
             <privateKeyFile>{privateKeyFile}</privateKeyFile>
             <caConfig>{caConfig}</caConfig>
-
-            <invalidCertificateHandler>
-                <name>AcceptCertificateHandler</name>
-            </invalidCertificateHandler>
         </client>
     </openSSL>
 </clickhouse>"""

From 9d2dac90ed30643bd8c4fb49280060432677905f Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Mon, 22 Jul 2024 07:27:46 +0000
Subject: [PATCH 0557/1488] Functions [a-g]: Use input_rows_count where
 appropriate

---
 src/Functions/acosh.cpp                       | 11 ++++---
 src/Functions/addMicroseconds.cpp             |  1 +
 src/Functions/addMilliseconds.cpp             |  1 +
 src/Functions/addNanoseconds.cpp              |  1 +
 src/Functions/aes_encrypt_mysql.cpp           |  1 -
 src/Functions/appendTrailingCharIfAbsent.cpp  |  9 +++---
 src/Functions/ascii.cpp                       |  2 --
 src/Functions/asinh.cpp                       | 11 ++++---
 src/Functions/atan2.cpp                       | 11 ++++---
 src/Functions/atanh.cpp                       | 11 ++++---
 src/Functions/base58Encode.cpp                |  2 ++
 src/Functions/base64Decode.cpp                |  2 ++
 src/Functions/base64Encode.cpp                |  2 ++
 src/Functions/base64URLDecode.cpp             |  2 ++
 src/Functions/base64URLEncode.cpp             |  2 ++
 src/Functions/byteSize.cpp                    |  8 ++---
 src/Functions/byteSwap.cpp                    |  1 +
 src/Functions/caseWithExpression.cpp          |  3 +-
 src/Functions/convertCharset.cpp              | 12 +++----
 src/Functions/cosh.cpp                        | 11 ++++---
 .../countSubstringsCaseInsensitiveUTF8.cpp    |  3 +-
 src/Functions/dateName.cpp                    | 16 +++++-----
 src/Functions/degrees.cpp                     | 22 +++++++------
 src/Functions/filesystem.cpp                  |  2 +-
 src/Functions/formatDateTime.cpp              | 32 +++++++++----------
 src/Functions/formatQuery.cpp                 | 10 +++---
 src/Functions/formatReadable.h                | 31 +++++++++---------
 src/Functions/geohashDecode.cpp               | 16 ++++------
 src/Functions/geohashEncode.cpp               | 14 ++++----
 src/Functions/transform.cpp                   |  3 +-
 30 files changed, 131 insertions(+), 122 deletions(-)

diff --git a/src/Functions/acosh.cpp b/src/Functions/acosh.cpp
index 5b071da9c40..2bab84c77af 100644
--- a/src/Functions/acosh.cpp
+++ b/src/Functions/acosh.cpp
@@ -5,11 +5,12 @@ namespace DB
 {
 namespace
 {
-    struct AcoshName
-    {
-        static constexpr auto name = "acosh";
-    };
-    using FunctionAcosh = FunctionMathUnary<UnaryFunctionVectorized<AcoshName, acosh>>;
+
+struct AcoshName
+{
+    static constexpr auto name = "acosh";
+};
+using FunctionAcosh = FunctionMathUnary<UnaryFunctionVectorized<AcoshName, acosh>>;
 
 }
 
diff --git a/src/Functions/addMicroseconds.cpp b/src/Functions/addMicroseconds.cpp
index 0dcd6b4452f..8c0ae06dcd0 100644
--- a/src/Functions/addMicroseconds.cpp
+++ b/src/Functions/addMicroseconds.cpp
@@ -6,6 +6,7 @@ namespace DB
 {
 
 using FunctionAddMicroseconds = FunctionDateOrDateTimeAddInterval<AddMicrosecondsImpl>;
+
 REGISTER_FUNCTION(AddMicroseconds)
 {
     factory.registerFunction<FunctionAddMicroseconds>();
diff --git a/src/Functions/addMilliseconds.cpp b/src/Functions/addMilliseconds.cpp
index 0e2b696d367..83e1f96ec4b 100644
--- a/src/Functions/addMilliseconds.cpp
+++ b/src/Functions/addMilliseconds.cpp
@@ -6,6 +6,7 @@ namespace DB
 {
 
 using FunctionAddMilliseconds = FunctionDateOrDateTimeAddInterval<AddMillisecondsImpl>;
+
 REGISTER_FUNCTION(AddMilliseconds)
 {
     factory.registerFunction<FunctionAddMilliseconds>();
diff --git a/src/Functions/addNanoseconds.cpp b/src/Functions/addNanoseconds.cpp
index 93eadc814d9..8f9a54752b9 100644
--- a/src/Functions/addNanoseconds.cpp
+++ b/src/Functions/addNanoseconds.cpp
@@ -6,6 +6,7 @@ namespace DB
 {
 
 using FunctionAddNanoseconds = FunctionDateOrDateTimeAddInterval<AddNanosecondsImpl>;
+
 REGISTER_FUNCTION(AddNanoseconds)
 {
     factory.registerFunction<FunctionAddNanoseconds>();
diff --git a/src/Functions/aes_encrypt_mysql.cpp b/src/Functions/aes_encrypt_mysql.cpp
index fb120151c25..33733f92b27 100644
--- a/src/Functions/aes_encrypt_mysql.cpp
+++ b/src/Functions/aes_encrypt_mysql.cpp
@@ -7,7 +7,6 @@
 
 namespace DB
 {
-
 namespace
 {
 
diff --git a/src/Functions/appendTrailingCharIfAbsent.cpp b/src/Functions/appendTrailingCharIfAbsent.cpp
index a5554171aaa..0e57d5c55ce 100644
--- a/src/Functions/appendTrailingCharIfAbsent.cpp
+++ b/src/Functions/appendTrailingCharIfAbsent.cpp
@@ -57,7 +57,7 @@ private:
     bool useDefaultImplementationForConstants() const override { return true; }
     ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1}; }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
         const auto & column = arguments[0].column;
         const auto & column_char = arguments[1].column;
@@ -80,14 +80,13 @@ private:
             auto & dst_data = col_res->getChars();
             auto & dst_offsets = col_res->getOffsets();
 
-            const auto size = src_offsets.size();
-            dst_data.resize(src_data.size() + size);
-            dst_offsets.resize(size);
+            dst_data.resize(src_data.size() + input_rows_count);
+            dst_offsets.resize(input_rows_count);
 
             ColumnString::Offset src_offset{};
             ColumnString::Offset dst_offset{};
 
-            for (const auto i : collections::range(0, size))
+            for (size_t i = 0; i < input_rows_count; ++i)
             {
                 const auto src_length = src_offsets[i] - src_offset;
                 memcpySmallAllowReadWriteOverflow15(&dst_data[dst_offset], &src_data[src_offset], src_length);
diff --git a/src/Functions/ascii.cpp b/src/Functions/ascii.cpp
index 7c8158b53d4..0d50e5d203b 100644
--- a/src/Functions/ascii.cpp
+++ b/src/Functions/ascii.cpp
@@ -45,9 +45,7 @@ struct AsciiImpl
         size_t size = data.size() / n;
 
         for (size_t i = 0; i < size; ++i)
-        {
             res[i] = doAscii(data, i * n, n);
-        }
     }
 
     [[noreturn]] static void array(const ColumnString::Offsets & /*offsets*/, PaddedPODArray<ReturnType> & /*res*/)
diff --git a/src/Functions/asinh.cpp b/src/Functions/asinh.cpp
index 6af832ae07c..b5e3626148f 100644
--- a/src/Functions/asinh.cpp
+++ b/src/Functions/asinh.cpp
@@ -5,11 +5,12 @@ namespace DB
 {
 namespace
 {
-    struct AsinhName
-    {
-        static constexpr auto name = "asinh";
-    };
-    using FunctionAsinh = FunctionMathUnary<UnaryFunctionVectorized<AsinhName, asinh>>;
+
+struct AsinhName
+{
+    static constexpr auto name = "asinh";
+};
+using FunctionAsinh = FunctionMathUnary<UnaryFunctionVectorized<AsinhName, asinh>>;
 
 }
 
diff --git a/src/Functions/atan2.cpp b/src/Functions/atan2.cpp
index 42294e11458..218f4c5406f 100644
--- a/src/Functions/atan2.cpp
+++ b/src/Functions/atan2.cpp
@@ -5,11 +5,12 @@ namespace DB
 {
 namespace
 {
-    struct Atan2Name
-    {
-        static constexpr auto name = "atan2";
-    };
-    using FunctionAtan2 = FunctionMathBinaryFloat64<BinaryFunctionVectorized<Atan2Name, atan2>>;
+
+struct Atan2Name
+{
+    static constexpr auto name = "atan2";
+};
+using FunctionAtan2 = FunctionMathBinaryFloat64<BinaryFunctionVectorized<Atan2Name, atan2>>;
 
 }
 
diff --git a/src/Functions/atanh.cpp b/src/Functions/atanh.cpp
index fab25414725..a36f5bcbcf0 100644
--- a/src/Functions/atanh.cpp
+++ b/src/Functions/atanh.cpp
@@ -5,11 +5,12 @@ namespace DB
 {
 namespace
 {
-    struct AtanhName
-    {
-        static constexpr auto name = "atanh";
-    };
-    using FunctionAtanh = FunctionMathUnary<UnaryFunctionVectorized<AtanhName, atanh>>;
+
+struct AtanhName
+{
+    static constexpr auto name = "atanh";
+};
+using FunctionAtanh = FunctionMathUnary<UnaryFunctionVectorized<AtanhName, atanh>>;
 
 }
 
diff --git a/src/Functions/base58Encode.cpp b/src/Functions/base58Encode.cpp
index cf790ebddab..3ae2fb12c5e 100644
--- a/src/Functions/base58Encode.cpp
+++ b/src/Functions/base58Encode.cpp
@@ -3,8 +3,10 @@
 
 namespace DB
 {
+
 REGISTER_FUNCTION(Base58Encode)
 {
     factory.registerFunction<FunctionBase58Conversion<Base58Encode>>();
 }
+
 }
diff --git a/src/Functions/base64Decode.cpp b/src/Functions/base64Decode.cpp
index 4d06ac99d6f..349475af3f0 100644
--- a/src/Functions/base64Decode.cpp
+++ b/src/Functions/base64Decode.cpp
@@ -5,6 +5,7 @@
 
 namespace DB
 {
+
 REGISTER_FUNCTION(Base64Decode)
 {
     FunctionDocumentation::Description description = R"(Accepts a String and decodes it from base64, according to RFC 4648 (https://datatracker.ietf.org/doc/html/rfc4648#section-4). Throws an exception in case of an error. Alias: FROM_BASE64.)";
@@ -19,6 +20,7 @@ REGISTER_FUNCTION(Base64Decode)
     /// MySQL compatibility alias.
     factory.registerAlias("FROM_BASE64", "base64Decode", FunctionFactory::Case::Insensitive);
 }
+
 }
 
 #endif
diff --git a/src/Functions/base64Encode.cpp b/src/Functions/base64Encode.cpp
index 64142995552..fe0fa642599 100644
--- a/src/Functions/base64Encode.cpp
+++ b/src/Functions/base64Encode.cpp
@@ -5,6 +5,7 @@
 
 namespace DB
 {
+
 REGISTER_FUNCTION(Base64Encode)
 {
     FunctionDocumentation::Description description = R"(Encodes a String as base64, according to RFC 4648 (https://datatracker.ietf.org/doc/html/rfc4648#section-4). Alias: TO_BASE64.)";
@@ -19,6 +20,7 @@ REGISTER_FUNCTION(Base64Encode)
     /// MySQL compatibility alias.
     factory.registerAlias("TO_BASE64", "base64Encode", FunctionFactory::Case::Insensitive);
 }
+
 }
 
 #endif
diff --git a/src/Functions/base64URLDecode.cpp b/src/Functions/base64URLDecode.cpp
index f5766dc60bd..f256e111619 100644
--- a/src/Functions/base64URLDecode.cpp
+++ b/src/Functions/base64URLDecode.cpp
@@ -5,6 +5,7 @@
 
 namespace DB
 {
+
 REGISTER_FUNCTION(Base64URLDecode)
 {
     FunctionDocumentation::Description description = R"(Accepts a base64-encoded URL and decodes it from base64 with URL-specific modifications, according to RFC 4648 (https://datatracker.ietf.org/doc/html/rfc4648#section-5).)";
@@ -16,6 +17,7 @@ REGISTER_FUNCTION(Base64URLDecode)
 
     factory.registerFunction<FunctionBase64Conversion<Base64Decode<Base64Variant::URL>>>({description, syntax, arguments, returned_value, examples, categories});
 }
+
 }
 
 #endif
diff --git a/src/Functions/base64URLEncode.cpp b/src/Functions/base64URLEncode.cpp
index 73a465a30c5..215712f7586 100644
--- a/src/Functions/base64URLEncode.cpp
+++ b/src/Functions/base64URLEncode.cpp
@@ -5,6 +5,7 @@
 
 namespace DB
 {
+
 REGISTER_FUNCTION(Base64URLEncode)
 {
     FunctionDocumentation::Description description = R"(Encodes an URL (String or FixedString) as base64 with URL-specific modifications, according to RFC 4648 (https://datatracker.ietf.org/doc/html/rfc4648#section-5).)";
@@ -16,6 +17,7 @@ REGISTER_FUNCTION(Base64URLEncode)
 
     factory.registerFunction<FunctionBase64Conversion<Base64Encode<Base64Variant::URL>>>({description, syntax, arguments, returned_value, examples, categories});
 }
+
 }
 
 #endif
diff --git a/src/Functions/byteSize.cpp b/src/Functions/byteSize.cpp
index 93a3a86641a..d366a1b2e12 100644
--- a/src/Functions/byteSize.cpp
+++ b/src/Functions/byteSize.cpp
@@ -67,11 +67,11 @@ public:
             const IColumn * column = arguments[arg_num].column.get();
 
             if (arg_num == 0)
-                for (size_t row_num = 0; row_num < input_rows_count; ++row_num)
-                    vec_res[row_num] = column->byteSizeAt(row_num);
+                for (size_t row = 0; row < input_rows_count; ++row)
+                    vec_res[row] = column->byteSizeAt(row);
             else
-                for (size_t row_num = 0; row_num < input_rows_count; ++row_num)
-                    vec_res[row_num] += column->byteSizeAt(row_num);
+                for (size_t row = 0; row < input_rows_count; ++row)
+                    vec_res[row] += column->byteSizeAt(row);
         }
 
         return result_col;
diff --git a/src/Functions/byteSwap.cpp b/src/Functions/byteSwap.cpp
index 6c824b851b0..2094ec4fa1a 100644
--- a/src/Functions/byteSwap.cpp
+++ b/src/Functions/byteSwap.cpp
@@ -10,6 +10,7 @@ extern const int NOT_IMPLEMENTED;
 
 namespace
 {
+
 template <typename T>
 requires std::is_integral_v<T>
 T byteSwap(T x)
diff --git a/src/Functions/caseWithExpression.cpp b/src/Functions/caseWithExpression.cpp
index 71fccc8436e..f0a620489ef 100644
--- a/src/Functions/caseWithExpression.cpp
+++ b/src/Functions/caseWithExpression.cpp
@@ -98,8 +98,7 @@ public:
 
         /// Execute transform.
         ColumnsWithTypeAndName transform_args{args.front(), src_array_col, dst_array_col, args.back()};
-        return FunctionFactory::instance().get("transform", context)->build(transform_args)
-            ->execute(transform_args, result_type, input_rows_count);
+        return FunctionFactory::instance().get("transform", context)->build(transform_args)->execute(transform_args, result_type, input_rows_count);
     }
 
 private:
diff --git a/src/Functions/convertCharset.cpp b/src/Functions/convertCharset.cpp
index b3b7394acb9..d998e88e7c2 100644
--- a/src/Functions/convertCharset.cpp
+++ b/src/Functions/convertCharset.cpp
@@ -88,7 +88,8 @@ private:
 
     static void convert(const String & from_charset, const String & to_charset,
         const ColumnString::Chars & from_chars, const ColumnString::Offsets & from_offsets,
-        ColumnString::Chars & to_chars, ColumnString::Offsets & to_offsets)
+        ColumnString::Chars & to_chars, ColumnString::Offsets & to_offsets,
+        size_t input_rows_count)
     {
         auto converter_from = getConverter(from_charset);
         auto converter_to = getConverter(to_charset);
@@ -96,12 +97,11 @@ private:
         ColumnString::Offset current_from_offset = 0;
         ColumnString::Offset current_to_offset = 0;
 
-        size_t size = from_offsets.size();
-        to_offsets.resize(size);
+        to_offsets.resize(input_rows_count);
 
         PODArray<UChar> uchars;
 
-        for (size_t i = 0; i < size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             size_t from_string_size = from_offsets[i] - current_from_offset - 1;
 
@@ -184,7 +184,7 @@ public:
     bool useDefaultImplementationForConstants() const override { return true; }
     ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1, 2}; }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
         const ColumnWithTypeAndName & arg_from = arguments[0];
         const ColumnWithTypeAndName & arg_charset_from = arguments[1];
@@ -204,7 +204,7 @@ public:
         if (const ColumnString * col_from = checkAndGetColumn<ColumnString>(arg_from.column.get()))
         {
             auto col_to = ColumnString::create();
-            convert(charset_from, charset_to, col_from->getChars(), col_from->getOffsets(), col_to->getChars(), col_to->getOffsets());
+            convert(charset_from, charset_to, col_from->getChars(), col_from->getOffsets(), col_to->getChars(), col_to->getOffsets(), input_rows_count);
             return col_to;
         }
         else
diff --git a/src/Functions/cosh.cpp b/src/Functions/cosh.cpp
index 54b52051aab..f4302292303 100644
--- a/src/Functions/cosh.cpp
+++ b/src/Functions/cosh.cpp
@@ -5,11 +5,12 @@ namespace DB
 {
 namespace
 {
-    struct CoshName
-    {
-        static constexpr auto name = "cosh";
-    };
-    using FunctionCosh = FunctionMathUnary<UnaryFunctionVectorized<CoshName, cosh>>;
+
+struct CoshName
+{
+    static constexpr auto name = "cosh";
+};
+using FunctionCosh = FunctionMathUnary<UnaryFunctionVectorized<CoshName, cosh>>;
 
 }
 
diff --git a/src/Functions/countSubstringsCaseInsensitiveUTF8.cpp b/src/Functions/countSubstringsCaseInsensitiveUTF8.cpp
index 3f71bca63d2..99ae4f1927e 100644
--- a/src/Functions/countSubstringsCaseInsensitiveUTF8.cpp
+++ b/src/Functions/countSubstringsCaseInsensitiveUTF8.cpp
@@ -13,8 +13,7 @@ struct NameCountSubstringsCaseInsensitiveUTF8
     static constexpr auto name = "countSubstringsCaseInsensitiveUTF8";
 };
 
-using FunctionCountSubstringsCaseInsensitiveUTF8 = FunctionsStringSearch<
-        CountSubstringsImpl<NameCountSubstringsCaseInsensitiveUTF8, PositionCaseInsensitiveUTF8>>;
+using FunctionCountSubstringsCaseInsensitiveUTF8 = FunctionsStringSearch<CountSubstringsImpl<NameCountSubstringsCaseInsensitiveUTF8, PositionCaseInsensitiveUTF8>>;
 
 }
 
diff --git a/src/Functions/dateName.cpp b/src/Functions/dateName.cpp
index 8165ea1b8d3..846cb87f1ee 100644
--- a/src/Functions/dateName.cpp
+++ b/src/Functions/dateName.cpp
@@ -109,14 +109,14 @@ public:
     ColumnPtr executeImpl(
         const ColumnsWithTypeAndName & arguments,
         const DataTypePtr & result_type,
-        [[maybe_unused]] size_t input_rows_count) const override
+        size_t input_rows_count) const override
     {
         ColumnPtr res;
 
-        if (!((res = executeType<DataTypeDate>(arguments, result_type))
-            || (res = executeType<DataTypeDate32>(arguments, result_type))
-            || (res = executeType<DataTypeDateTime>(arguments, result_type))
-            || (res = executeType<DataTypeDateTime64>(arguments, result_type))))
+        if (!((res = executeType<DataTypeDate>(arguments, result_type, input_rows_count))
+            || (res = executeType<DataTypeDate32>(arguments, result_type, input_rows_count))
+            || (res = executeType<DataTypeDateTime>(arguments, result_type, input_rows_count))
+            || (res = executeType<DataTypeDateTime64>(arguments, result_type, input_rows_count))))
             throw Exception(
                 ErrorCodes::ILLEGAL_COLUMN,
                 "Illegal column {} of function {}, must be Date or DateTime.",
@@ -127,7 +127,7 @@ public:
     }
 
     template <typename DataType>
-    ColumnPtr executeType(const ColumnsWithTypeAndName & arguments, const DataTypePtr &) const
+    ColumnPtr executeType(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const
     {
         auto * times = checkAndGetColumn<typename DataType::ColumnType>(arguments[1].column.get());
         if (!times)
@@ -144,7 +144,7 @@ public:
         String date_part = date_part_column->getValue<String>();
 
         const DateLUTImpl * time_zone_tmp;
-        if (std::is_same_v<DataType, DataTypeDateTime64> || std::is_same_v<DataType, DataTypeDateTime>)
+        if constexpr (std::is_same_v<DataType, DataTypeDateTime64> || std::is_same_v<DataType, DataTypeDateTime>)
             time_zone_tmp = &extractTimeZoneFromFunctionArguments(arguments, 2, 1);
         else
             time_zone_tmp = &DateLUT::instance();
@@ -175,7 +175,7 @@ public:
         using TimeType = DateTypeToTimeType<DataType>;
         callOnDatePartWriter<TimeType>(date_part, [&](const auto & writer)
         {
-            for (size_t i = 0; i < times_data.size(); ++i)
+            for (size_t i = 0; i < input_rows_count; ++i)
             {
                 if constexpr (std::is_same_v<DataType, DataTypeDateTime64>)
                 {
diff --git a/src/Functions/degrees.cpp b/src/Functions/degrees.cpp
index 8646eb54d9a..94b5ce3682c 100644
--- a/src/Functions/degrees.cpp
+++ b/src/Functions/degrees.cpp
@@ -7,18 +7,20 @@ namespace DB
 {
 namespace
 {
-    struct DegreesName
-    {
-        static constexpr auto name = "degrees";
-    };
 
-    Float64 degrees(Float64 r)
-    {
-        Float64 degrees = r * (180 / M_PI);
-        return degrees;
-    }
+struct DegreesName
+{
+    static constexpr auto name = "degrees";
+};
+
+Float64 degrees(Float64 r)
+{
+    Float64 degrees = r * (180 / M_PI);
+    return degrees;
+}
+
+using FunctionDegrees = FunctionMathUnary<UnaryFunctionVectorized<DegreesName, degrees>>;
 
-    using FunctionDegrees = FunctionMathUnary<UnaryFunctionVectorized<DegreesName, degrees>>;
 }
 
 REGISTER_FUNCTION(Degrees)
diff --git a/src/Functions/filesystem.cpp b/src/Functions/filesystem.cpp
index 9fbf9b0cbe7..9b168f3f088 100644
--- a/src/Functions/filesystem.cpp
+++ b/src/Functions/filesystem.cpp
@@ -91,7 +91,7 @@ public:
 
                 auto col_res = ColumnVector<UInt64>::create(col_str->size());
                 auto & data = col_res->getData();
-                for (size_t i = 0; i < col_str->size(); ++i)
+                for (size_t i = 0; i < input_rows_count; ++i)
                 {
                     auto disk_name = col_str->getDataAt(i).toString();
                     if (auto it = disk_map.find(disk_name); it != disk_map.end())
diff --git a/src/Functions/formatDateTime.cpp b/src/Functions/formatDateTime.cpp
index f89afd67e78..f33b7849a43 100644
--- a/src/Functions/formatDateTime.cpp
+++ b/src/Functions/formatDateTime.cpp
@@ -848,7 +848,7 @@ public:
         return std::make_shared<DataTypeString>();
     }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, [[maybe_unused]] size_t input_rows_count) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override
     {
         ColumnPtr res;
         if constexpr (support_integer == SupportInteger::Yes)
@@ -862,17 +862,17 @@ public:
                 if (!castType(arguments[0].type.get(), [&](const auto & type)
                     {
                         using FromDataType = std::decay_t<decltype(type)>;
-                        if (!(res = executeType<FromDataType>(arguments, result_type)))
+                        if (!(res = executeType<FromDataType>(arguments, result_type, input_rows_count)))
                             throw Exception(ErrorCodes::ILLEGAL_COLUMN,
                                 "Illegal column {} of function {}, must be Integer, Date, Date32, DateTime or DateTime64.",
                                 arguments[0].column->getName(), getName());
                         return true;
                     }))
                 {
-                    if (!((res = executeType<DataTypeDate>(arguments, result_type))
-                        || (res = executeType<DataTypeDate32>(arguments, result_type))
-                        || (res = executeType<DataTypeDateTime>(arguments, result_type))
-                        || (res = executeType<DataTypeDateTime64>(arguments, result_type))))
+                    if (!((res = executeType<DataTypeDate>(arguments, result_type, input_rows_count))
+                        || (res = executeType<DataTypeDate32>(arguments, result_type, input_rows_count))
+                        || (res = executeType<DataTypeDateTime>(arguments, result_type, input_rows_count))
+                        || (res = executeType<DataTypeDateTime64>(arguments, result_type, input_rows_count))))
                         throw Exception(ErrorCodes::ILLEGAL_COLUMN,
                             "Illegal column {} of function {}, must be Integer or DateTime.",
                             arguments[0].column->getName(), getName());
@@ -881,10 +881,10 @@ public:
         }
         else
         {
-            if (!((res = executeType<DataTypeDate>(arguments, result_type))
-                || (res = executeType<DataTypeDate32>(arguments, result_type))
-                || (res = executeType<DataTypeDateTime>(arguments, result_type))
-                || (res = executeType<DataTypeDateTime64>(arguments, result_type))))
+            if (!((res = executeType<DataTypeDate>(arguments, result_type, input_rows_count))
+                || (res = executeType<DataTypeDate32>(arguments, result_type, input_rows_count))
+                || (res = executeType<DataTypeDateTime>(arguments, result_type, input_rows_count))
+                || (res = executeType<DataTypeDateTime64>(arguments, result_type, input_rows_count))))
                 throw Exception(ErrorCodes::ILLEGAL_COLUMN,
                     "Illegal column {} of function {}, must be Date or DateTime.",
                     arguments[0].column->getName(), getName());
@@ -894,7 +894,7 @@ public:
     }
 
     template <typename DataType>
-    ColumnPtr executeType(const ColumnsWithTypeAndName & arguments, const DataTypePtr &) const
+    ColumnPtr executeType(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const
     {
         auto non_const_datetime = arguments[0].column->convertToFullColumnIfConst();
         auto * times = checkAndGetColumn<typename DataType::ColumnType>(non_const_datetime.get());
@@ -955,13 +955,11 @@ public:
         else
             time_zone = &DateLUT::instance();
 
-        const auto & vec = times->getData();
-
         auto col_res = ColumnString::create();
         auto & res_data = col_res->getChars();
         auto & res_offsets = col_res->getOffsets();
-        res_data.resize(vec.size() * (out_template_size + 1));
-        res_offsets.resize(vec.size());
+        res_data.resize(input_rows_count * (out_template_size + 1));
+        res_offsets.resize(input_rows_count);
 
         if constexpr (format_syntax == FormatSyntax::MySQL)
         {
@@ -990,9 +988,11 @@ public:
             }
         }
 
+        const auto & vec = times->getData();
+
         auto * begin = reinterpret_cast<char *>(res_data.data());
         auto * pos = begin;
-        for (size_t i = 0; i < vec.size(); ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             if (!const_time_zone_column && arguments.size() > 2)
             {
diff --git a/src/Functions/formatQuery.cpp b/src/Functions/formatQuery.cpp
index d10b3f9a5b7..4e3f302ce36 100644
--- a/src/Functions/formatQuery.cpp
+++ b/src/Functions/formatQuery.cpp
@@ -75,7 +75,7 @@ public:
         if (const ColumnString * col_query_string = checkAndGetColumn<ColumnString>(col_query.get()))
         {
             auto col_res = ColumnString::create();
-            formatVector(col_query_string->getChars(), col_query_string->getOffsets(), col_res->getChars(), col_res->getOffsets(), col_null_map);
+            formatVector(col_query_string->getChars(), col_query_string->getOffsets(), col_res->getChars(), col_res->getOffsets(), col_null_map, input_rows_count);
 
             if (error_handling == ErrorHandling::Null)
                 return ColumnNullable::create(std::move(col_res), std::move(col_null_map));
@@ -92,16 +92,16 @@ private:
         const ColumnString::Offsets & offsets,
         ColumnString::Chars & res_data,
         ColumnString::Offsets & res_offsets,
-        ColumnUInt8::MutablePtr & res_null_map) const
+        ColumnUInt8::MutablePtr & res_null_map,
+        size_t input_rows_count) const
     {
-        const size_t size = offsets.size();
-        res_offsets.resize(size);
+        res_offsets.resize(input_rows_count);
         res_data.resize(data.size());
 
         size_t prev_offset = 0;
         size_t res_data_size = 0;
 
-        for (size_t i = 0; i < size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             const char * begin = reinterpret_cast<const char *>(&data[prev_offset]);
             const char * end = begin + offsets[i] - prev_offset - 1;
diff --git a/src/Functions/formatReadable.h b/src/Functions/formatReadable.h
index 487ec9d79d0..9161ab43e28 100644
--- a/src/Functions/formatReadable.h
+++ b/src/Functions/formatReadable.h
@@ -55,19 +55,19 @@ public:
 
     bool useDefaultImplementationForConstants() const override { return true; }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
         ColumnPtr res;
-        if (!((res = executeType<UInt8>(arguments))
-            || (res = executeType<UInt16>(arguments))
-            || (res = executeType<UInt32>(arguments))
-            || (res = executeType<UInt64>(arguments))
-            || (res = executeType<Int8>(arguments))
-            || (res = executeType<Int16>(arguments))
-            || (res = executeType<Int32>(arguments))
-            || (res = executeType<Int64>(arguments))
-            || (res = executeType<Float32>(arguments))
-            || (res = executeType<Float64>(arguments))))
+        if (!((res = executeType<UInt8>(arguments, input_rows_count))
+            || (res = executeType<UInt16>(arguments, input_rows_count))
+            || (res = executeType<UInt32>(arguments, input_rows_count))
+            || (res = executeType<UInt64>(arguments, input_rows_count))
+            || (res = executeType<Int8>(arguments, input_rows_count))
+            || (res = executeType<Int16>(arguments, input_rows_count))
+            || (res = executeType<Int32>(arguments, input_rows_count))
+            || (res = executeType<Int64>(arguments, input_rows_count))
+            || (res = executeType<Float32>(arguments, input_rows_count))
+            || (res = executeType<Float64>(arguments, input_rows_count))))
             throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of argument of function {}",
                             arguments[0].column->getName(), getName());
 
@@ -76,7 +76,7 @@ public:
 
 private:
     template <typename T>
-    ColumnPtr executeType(const ColumnsWithTypeAndName & arguments) const
+    ColumnPtr executeType(const ColumnsWithTypeAndName & arguments, size_t input_rows_count) const
     {
         if (const ColumnVector<T> * col_from = checkAndGetColumn<ColumnVector<T>>(arguments[0].column.get()))
         {
@@ -85,13 +85,12 @@ private:
             const typename ColumnVector<T>::Container & vec_from = col_from->getData();
             ColumnString::Chars & data_to = col_to->getChars();
             ColumnString::Offsets & offsets_to = col_to->getOffsets();
-            size_t size = vec_from.size();
-            data_to.resize(size * 2);
-            offsets_to.resize(size);
+            data_to.resize(input_rows_count * 2);
+            offsets_to.resize(input_rows_count);
 
             WriteBufferFromVector<ColumnString::Chars> buf_to(data_to);
 
-            for (size_t i = 0; i < size; ++i)
+            for (size_t i = 0; i < input_rows_count; ++i)
             {
                 Impl::format(static_cast<double>(vec_from[i]), buf_to);
                 writeChar(0, buf_to);
diff --git a/src/Functions/geohashDecode.cpp b/src/Functions/geohashDecode.cpp
index 96ad7dacfc4..cace6c09fec 100644
--- a/src/Functions/geohashDecode.cpp
+++ b/src/Functions/geohashDecode.cpp
@@ -51,21 +51,19 @@ public:
     }
 
     template <typename ColumnTypeEncoded>
-    bool tryExecute(const IColumn * encoded_column, ColumnPtr & result_column) const
+    bool tryExecute(const IColumn * encoded_column, ColumnPtr & result_column, size_t input_rows_count) const
     {
         const auto * encoded = checkAndGetColumn<ColumnTypeEncoded>(encoded_column);
         if (!encoded)
             return false;
 
-        const size_t count = encoded->size();
-
-        auto latitude = ColumnFloat64::create(count);
-        auto longitude = ColumnFloat64::create(count);
+        auto latitude = ColumnFloat64::create(input_rows_count);
+        auto longitude = ColumnFloat64::create(input_rows_count);
 
         ColumnFloat64::Container & lon_data = longitude->getData();
         ColumnFloat64::Container & lat_data = latitude->getData();
 
-        for (size_t i = 0; i < count; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             std::string_view encoded_string = encoded->getDataAt(i).toView();
             geohashDecode(encoded_string.data(), encoded_string.size(), &lon_data[i], &lat_data[i]);
@@ -79,13 +77,13 @@ public:
         return true;
     }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
         const IColumn * encoded = arguments[0].column.get();
         ColumnPtr res_column;
 
-        if (tryExecute<ColumnString>(encoded, res_column) ||
-            tryExecute<ColumnFixedString>(encoded, res_column))
+        if (tryExecute<ColumnString>(encoded, res_column, input_rows_count) ||
+            tryExecute<ColumnFixedString>(encoded, res_column, input_rows_count))
             return res_column;
 
         throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Unsupported argument type:{} of argument of function {}",
diff --git a/src/Functions/geohashEncode.cpp b/src/Functions/geohashEncode.cpp
index 034c8188b63..c49acddd81f 100644
--- a/src/Functions/geohashEncode.cpp
+++ b/src/Functions/geohashEncode.cpp
@@ -53,7 +53,7 @@ public:
         return std::make_shared<DataTypeString>();
     }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
         const IColumn * longitude = arguments[0].column.get();
         const IColumn * latitude = arguments[1].column.get();
@@ -65,26 +65,24 @@ public:
             precision = arguments[2].column;
 
         ColumnPtr res_column;
-        vector(longitude, latitude, precision.get(), res_column);
+        vector(longitude, latitude, precision.get(), res_column, input_rows_count);
         return res_column;
     }
 
 private:
-    void vector(const IColumn * lon_column, const IColumn * lat_column, const IColumn * precision_column, ColumnPtr & result) const
+    void vector(const IColumn * lon_column, const IColumn * lat_column, const IColumn * precision_column, ColumnPtr & result, size_t input_rows_count) const
     {
         auto col_str = ColumnString::create();
         ColumnString::Chars & out_vec = col_str->getChars();
         ColumnString::Offsets & out_offsets = col_str->getOffsets();
 
-        const size_t size = lat_column->size();
-
-        out_offsets.resize(size);
-        out_vec.resize(size * (GEOHASH_MAX_TEXT_LENGTH + 1));
+        out_offsets.resize(input_rows_count);
+        out_vec.resize(input_rows_count * (GEOHASH_MAX_TEXT_LENGTH + 1));
 
         char * begin = reinterpret_cast<char *>(out_vec.data());
         char * pos = begin;
 
-        for (size_t i = 0; i < size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             const Float64 longitude_value = lon_column->getFloat64(i);
             const Float64 latitude_value = lat_column->getFloat64(i);
diff --git a/src/Functions/transform.cpp b/src/Functions/transform.cpp
index 0dbc9946710..68500779f93 100644
--- a/src/Functions/transform.cpp
+++ b/src/Functions/transform.cpp
@@ -138,8 +138,7 @@ namespace
             }
         }
 
-        ColumnPtr executeImpl(
-            const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override
+        ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override
         {
             std::call_once(once, [&] { initialize(arguments, result_type); });
 

From 885acd3aa80d421e82f75150b4152e227ca0fba4 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 22 Jul 2024 09:42:20 +0200
Subject: [PATCH 0558/1488] Compatibility

---
 src/Parsers/ParserCreateIndexQuery.cpp | 3 +++
 src/Parsers/ParserCreateQuery.cpp      | 3 +++
 2 files changed, 6 insertions(+)

diff --git a/src/Parsers/ParserCreateIndexQuery.cpp b/src/Parsers/ParserCreateIndexQuery.cpp
index 8a4c1c0b17a..2761c99738b 100644
--- a/src/Parsers/ParserCreateIndexQuery.cpp
+++ b/src/Parsers/ParserCreateIndexQuery.cpp
@@ -71,7 +71,10 @@ bool ParserCreateIndexDeclaration::parseImpl(Pos & pos, ASTPtr & node, Expected
         if (!type_p.parse(pos, type, expected))
         {
             if (ParserIdentifier().parse(pos, type, expected))
+            {
                 type = makeASTFunction(type->as<ASTIdentifier &>().name());
+                type->as<ASTFunction &>().no_empty_args = true;
+            }
             else
                 return false;
         }
diff --git a/src/Parsers/ParserCreateQuery.cpp b/src/Parsers/ParserCreateQuery.cpp
index bff5da4a536..9aaecd84f59 100644
--- a/src/Parsers/ParserCreateQuery.cpp
+++ b/src/Parsers/ParserCreateQuery.cpp
@@ -200,7 +200,10 @@ bool ParserIndexDeclaration::parseImpl(Pos & pos, ASTPtr & node, Expected & expe
     if (!type_p.parse(pos, type, expected))
     {
         if (name_p.parse(pos, type, expected))
+        {
             type = makeASTFunction(type->as<ASTIdentifier &>().name());
+            type->as<ASTFunction &>().no_empty_args = true;
+        }
         else
             return false;
     }

From 8217dcccc1438ec6186e8db53e17429a7060183f Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Mon, 22 Jul 2024 09:53:56 +0200
Subject: [PATCH 0559/1488] Stop ignoring SIGSEGV in GDB

---
 docker/test/fuzzer/run-fuzzer.sh     | 1 -
 docker/test/stateless/attach_gdb.lib | 1 -
 2 files changed, 2 deletions(-)

diff --git a/docker/test/fuzzer/run-fuzzer.sh b/docker/test/fuzzer/run-fuzzer.sh
index 6191aeaf304..b8f967ed9c2 100755
--- a/docker/test/fuzzer/run-fuzzer.sh
+++ b/docker/test/fuzzer/run-fuzzer.sh
@@ -208,7 +208,6 @@ handle SIGPIPE nostop noprint pass
 handle SIGTERM nostop noprint pass
 handle SIGUSR1 nostop noprint pass
 handle SIGUSR2 nostop noprint pass
-handle SIGSEGV nostop pass
 handle SIG$RTMIN nostop noprint pass
 info signals
 continue
diff --git a/docker/test/stateless/attach_gdb.lib b/docker/test/stateless/attach_gdb.lib
index eb54f920b98..d288288bb17 100644
--- a/docker/test/stateless/attach_gdb.lib
+++ b/docker/test/stateless/attach_gdb.lib
@@ -20,7 +20,6 @@ handle SIGPIPE nostop noprint pass
 handle SIGTERM nostop noprint pass
 handle SIGUSR1 nostop noprint pass
 handle SIGUSR2 nostop noprint pass
-handle SIGSEGV nostop pass
 handle SIG$RTMIN nostop noprint pass
 info signals
 continue

From c2ac13291f3bf201f7189bd36f2c9be7c06aa886 Mon Sep 17 00:00:00 2001
From: Michael Stetsyuk <mike.stetsyuk@gmail.com>
Date: Mon, 22 Jul 2024 09:06:13 +0100
Subject: [PATCH 0560/1488] fix tests

---
 tests/integration/test_recovery_time_metric/test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/test_recovery_time_metric/test.py b/tests/integration/test_recovery_time_metric/test.py
index 90155f81ba2..e4a44103b76 100644
--- a/tests/integration/test_recovery_time_metric/test.py
+++ b/tests/integration/test_recovery_time_metric/test.py
@@ -3,7 +3,7 @@ from helpers.cluster import ClickHouseCluster
 
 cluster = ClickHouseCluster(__file__)
 node = cluster.add_instance(
-    "node", main_configs=["configs/config.xml"], with_zookeeper=True
+    "node", main_configs=["configs/config.xml"], with_zookeeper=True, stay_alive=True,
 )
 
 

From d040e436f3a1f8594070b04ec10cbf7391f6994a Mon Sep 17 00:00:00 2001
From: robot-clickhouse <robot-clickhouse@users.noreply.github.com>
Date: Mon, 22 Jul 2024 08:18:58 +0000
Subject: [PATCH 0561/1488] Automatic style fix

---
 tests/integration/test_recovery_time_metric/test.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tests/integration/test_recovery_time_metric/test.py b/tests/integration/test_recovery_time_metric/test.py
index e4a44103b76..4dad844b950 100644
--- a/tests/integration/test_recovery_time_metric/test.py
+++ b/tests/integration/test_recovery_time_metric/test.py
@@ -3,7 +3,10 @@ from helpers.cluster import ClickHouseCluster
 
 cluster = ClickHouseCluster(__file__)
 node = cluster.add_instance(
-    "node", main_configs=["configs/config.xml"], with_zookeeper=True, stay_alive=True,
+    "node",
+    main_configs=["configs/config.xml"],
+    with_zookeeper=True,
+    stay_alive=True,
 )
 
 

From 0f327869132940b3bae932730cb2ce2f9c394163 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Mon, 22 Jul 2024 10:46:42 +0200
Subject: [PATCH 0562/1488] Better random and queries

---
 tests/queries/0_stateless/01194_http_query_id.sh | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/tests/queries/0_stateless/01194_http_query_id.sh b/tests/queries/0_stateless/01194_http_query_id.sh
index fac17cca3c6..42321112185 100755
--- a/tests/queries/0_stateless/01194_http_query_id.sh
+++ b/tests/queries/0_stateless/01194_http_query_id.sh
@@ -4,14 +4,22 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
-rnd=$RANDOM
-url="${CLICKHOUSE_URL}&session_id=test_01194_$RANDOM"
+rnd="$CLICKHOUSE_DATABASE"
+url="${CLICKHOUSE_URL}&session_id=test_01194_${CLICKHOUSE_DATABASE}"
 
 ${CLICKHOUSE_CURL} -sS "$url&query=SELECT+'test_01194',$rnd,1" > /dev/null
 ${CLICKHOUSE_CURL} -sS "$url&query=SELECT+'test_01194',$rnd,2" > /dev/null
 ${CLICKHOUSE_CURL} -sS "$url" --data "SELECT 'test_01194',$rnd,3" > /dev/null
 ${CLICKHOUSE_CURL} -sS "$url" --data "SELECT 'test_01194',$rnd,4" > /dev/null
 
-${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" --data "SYSTEM FLUSH LOGS"
+$CLICKHOUSE_CLIENT -q "SYSTEM FLUSH LOGS"
 
-${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" --data "SELECT count(DISTINCT query_id) FROM system.query_log WHERE current_database = currentDatabase() AND event_date >= yesterday() AND query LIKE 'SELECT ''test_01194'',$rnd%'"
+$CLICKHOUSE_CLIENT -q "
+  SELECT
+    count(DISTINCT query_id)
+  FROM system.query_log
+  WHERE
+        current_database = currentDatabase()
+    AND event_date >= yesterday()
+    AND query LIKE 'SELECT ''test_01194'',$rnd%'
+    AND query_id != queryID()"

From 1be54641c16569e7ee3398fe0770a9b532342643 Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Mon, 22 Jul 2024 11:03:19 +0200
Subject: [PATCH 0563/1488] Revert libunwind patch

---
 contrib/libunwind | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contrib/libunwind b/contrib/libunwind
index 8f28e64d158..fe854449e24 160000
--- a/contrib/libunwind
+++ b/contrib/libunwind
@@ -1 +1 @@
-Subproject commit 8f28e64d15819d2d096badd598c7d85bebddb1f2
+Subproject commit fe854449e24bedfa26e38465b84374312dbd587f

From 255dcec501e7506291cee44f5c9eb30b5eec7e99 Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Mon, 22 Jul 2024 11:10:34 +0200
Subject: [PATCH 0564/1488] Fix 02241_join_rocksdb_bs

---
 .../02241_join_rocksdb_bs.reference           | 56 -------------------
 .../0_stateless/02241_join_rocksdb_bs.sql.j2  | 20 +++----
 2 files changed, 7 insertions(+), 69 deletions(-)

diff --git a/tests/queries/0_stateless/02241_join_rocksdb_bs.reference b/tests/queries/0_stateless/02241_join_rocksdb_bs.reference
index 8416a2991c1..4dff9ef38ef 100644
--- a/tests/queries/0_stateless/02241_join_rocksdb_bs.reference
+++ b/tests/queries/0_stateless/02241_join_rocksdb_bs.reference
@@ -10,59 +10,3 @@
 1
 1
 1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
-1
diff --git a/tests/queries/0_stateless/02241_join_rocksdb_bs.sql.j2 b/tests/queries/0_stateless/02241_join_rocksdb_bs.sql.j2
index 4ab98201eed..e5703f99d62 100644
--- a/tests/queries/0_stateless/02241_join_rocksdb_bs.sql.j2
+++ b/tests/queries/0_stateless/02241_join_rocksdb_bs.sql.j2
@@ -1,4 +1,4 @@
--- Tags: use-rocksdb, long, no-s3-storage, no-random-settings, no-random-merge-tree-settings
+-- Tags: use-rocksdb, long, no-s3-storage
 
 SET join_algorithm = 'direct';
 
@@ -13,27 +13,21 @@ INSERT INTO rdb_{{ table_size }}
     SELECT (sipHash64(number) % {{ table_size }}) as key, ('val' || toString(key)) AS value
     FROM numbers_mt({{ table_size }});
 
-{% for block_size in [10, 11, 128, 129, 65505, 65506, 70000] -%}
-
-{% if block_size * 5000 > table_size -%}
-
-SET max_block_size = {{ block_size }};
-
 {% for right_size in [table_size // 2, table_size + table_size // 4 + 1] -%}
 
 SELECT count() == (SELECT count() FROM rdb_{{ table_size }} WHERE key < {{ right_size }})
 FROM (SELECT number as k FROM numbers_mt({{ right_size }})) as t1
 INNER JOIN rdb_{{ table_size }} as rdb
-ON rdb.key == t1.k;
+ON rdb.key == t1.k
+{% if table_size < 100 %}SETTINGS max_block_size = 1{% endif -%}
+;
 
 SELECT count() == {{ right_size }} and countIf(value != '') == (SELECT count() FROM rdb_{{ table_size }} WHERE key < {{ right_size }})
 FROM (SELECT number as k FROM numbers_mt({{ right_size }})) as t1
 LEFT JOIN rdb_{{ table_size }} as rdb
-ON rdb.key == t1.k;
-
-{% endfor -%}
-
-{% endif -%}
+ON rdb.key == t1.k
+{% if table_size < 100 %}SETTINGS max_block_size = 1{% endif -%}
+;
 
 {% endfor -%}
 {% endfor -%}

From 2f1818b8d5d49747faa8b496eb6c86d1d78b3ec7 Mon Sep 17 00:00:00 2001
From: Yohann Jardin <yohann.jardin@telecomnancy.net>
Date: Mon, 22 Jul 2024 11:08:07 +0200
Subject: [PATCH 0565/1488] update error codes related to number of accepted
 arguments

---
 src/Functions/FunctionBase58Conversion.h      |  3 +-
 src/Functions/FunctionChar.cpp                |  3 +-
 src/Functions/hilbertEncode.cpp               |  4 +--
 src/Functions/mortonEncode.cpp                |  4 +--
 src/Functions/randDistribution.cpp            |  3 +-
 src/Processors/Transforms/WindowTransform.cpp | 31 ++++++++++---------
 .../test_functions.py                         | 13 ++++++--
 .../0_stateless/02560_window_ntile.reference  |  2 ++
 .../0_stateless/02560_window_ntile.sql        |  2 ++
 .../0_stateless/03131_hilbert_coding.sql      |  1 +
 10 files changed, 42 insertions(+), 24 deletions(-)

diff --git a/src/Functions/FunctionBase58Conversion.h b/src/Functions/FunctionBase58Conversion.h
index e519f9768cc..a34a36d8b81 100644
--- a/src/Functions/FunctionBase58Conversion.h
+++ b/src/Functions/FunctionBase58Conversion.h
@@ -18,6 +18,7 @@ namespace ErrorCodes
     extern const int ILLEGAL_COLUMN;
     extern const int ILLEGAL_TYPE_OF_ARGUMENT;
     extern const int BAD_ARGUMENTS;
+    extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
 }
 
 struct Base58Encode
@@ -135,7 +136,7 @@ public:
     DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
     {
         if (arguments.size() != 1)
-            throw Exception(ErrorCodes::BAD_ARGUMENTS, "Wrong number of arguments for function {}: 1 expected.", getName());
+            throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Wrong number of arguments for function {}: 1 expected.", getName());
 
         if (!isString(arguments[0].type))
             throw Exception(
diff --git a/src/Functions/FunctionChar.cpp b/src/Functions/FunctionChar.cpp
index 0ebe1442f08..79e346a3ea4 100644
--- a/src/Functions/FunctionChar.cpp
+++ b/src/Functions/FunctionChar.cpp
@@ -15,6 +15,7 @@ namespace ErrorCodes
 {
     extern const int ILLEGAL_TYPE_OF_ARGUMENT;
     extern const int ILLEGAL_COLUMN;
+    extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION;
 }
 
 class FunctionChar : public IFunction
@@ -36,7 +37,7 @@ public:
     DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
     {
         if (arguments.empty())
-            throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
+            throw Exception(ErrorCodes::TOO_FEW_ARGUMENTS_FOR_FUNCTION,
                             "Number of arguments for function {} can't be {}, should be at least 1",
                             getName(), arguments.size());
 
diff --git a/src/Functions/hilbertEncode.cpp b/src/Functions/hilbertEncode.cpp
index 13512d0d36c..a4e3cc59b76 100644
--- a/src/Functions/hilbertEncode.cpp
+++ b/src/Functions/hilbertEncode.cpp
@@ -11,8 +11,8 @@ namespace DB
 
 namespace ErrorCodes
 {
-    extern const int ILLEGAL_TYPE_OF_ARGUMENT;
     extern const int ARGUMENT_OUT_OF_BOUND;
+    extern const int TOO_MANY_ARGUMENTS_FOR_FUNCTION;
 }
 
 
@@ -87,7 +87,7 @@ public:
             return col_res;
         }
 
-        throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
+        throw Exception(ErrorCodes::TOO_MANY_ARGUMENTS_FOR_FUNCTION,
                         "Illegal number of UInt arguments of function {}: should be not more than 2 dimensions",
                         getName());
     }
diff --git a/src/Functions/mortonEncode.cpp b/src/Functions/mortonEncode.cpp
index 0c19c7c3134..8cf0f18dbfe 100644
--- a/src/Functions/mortonEncode.cpp
+++ b/src/Functions/mortonEncode.cpp
@@ -16,8 +16,8 @@ namespace DB
 
 namespace ErrorCodes
 {
-    extern const int ILLEGAL_TYPE_OF_ARGUMENT;
     extern const int ARGUMENT_OUT_OF_BOUND;
+    extern const int TOO_MANY_ARGUMENTS_FOR_FUNCTION;
 }
 
 #define EXTRACT_VECTOR(INDEX) \
@@ -130,7 +130,7 @@ namespace ErrorCodes
            MASK(8, 6, col6->getUInt(i)), \
            MASK(8, 7, col7->getUInt(i))) \
      \
-    throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, \
+    throw Exception(ErrorCodes::TOO_MANY_ARGUMENTS_FOR_FUNCTION, \
                     "Illegal number of UInt arguments of function {}, max: 8", \
                     getName()); \
 
diff --git a/src/Functions/randDistribution.cpp b/src/Functions/randDistribution.cpp
index 4e616ada697..6a3dac748c1 100644
--- a/src/Functions/randDistribution.cpp
+++ b/src/Functions/randDistribution.cpp
@@ -24,6 +24,7 @@ namespace ErrorCodes
     extern const int ILLEGAL_COLUMN;
     extern const int BAD_ARGUMENTS;
     extern const int LOGICAL_ERROR;
+    extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
 }
 
 namespace
@@ -246,7 +247,7 @@ public:
     {
         auto desired = Distribution::getNumberOfArguments();
         if (arguments.size() != desired && arguments.size() != desired + 1)
-            throw Exception(ErrorCodes::BAD_ARGUMENTS,
+            throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
                             "Wrong number of arguments for function {}. Should be {} or {}",
                             getName(), desired, desired + 1);
 
diff --git a/src/Processors/Transforms/WindowTransform.cpp b/src/Processors/Transforms/WindowTransform.cpp
index 86421adf4fb..2b255c5120e 100644
--- a/src/Processors/Transforms/WindowTransform.cpp
+++ b/src/Processors/Transforms/WindowTransform.cpp
@@ -56,7 +56,10 @@ namespace ErrorCodes
 {
     extern const int BAD_ARGUMENTS;
     extern const int NOT_IMPLEMENTED;
+    extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
     extern const int ILLEGAL_TYPE_OF_ARGUMENT;
+    extern const int TOO_FEW_ARGUMENTS_FOR_FUNCTION;
+    extern const int TOO_MANY_ARGUMENTS_FOR_FUNCTION;
 }
 
 // Interface for true window functions. It's not much of an interface, they just
@@ -1710,7 +1713,7 @@ struct WindowFunctionExponentialTimeDecayedSum final : public StatefulWindowFunc
     {
         if (parameters_.size() != 1)
         {
-            throw Exception(ErrorCodes::BAD_ARGUMENTS,
+            throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
                 "Function {} takes exactly one parameter", name_);
         }
         return applyVisitor(FieldVisitorConvertToNumber<Float64>(), parameters_[0]);
@@ -1723,7 +1726,7 @@ struct WindowFunctionExponentialTimeDecayedSum final : public StatefulWindowFunc
     {
         if (argument_types.size() != 2)
         {
-            throw Exception(ErrorCodes::BAD_ARGUMENTS,
+            throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
                 "Function {} takes exactly two arguments", name_);
         }
 
@@ -1807,7 +1810,7 @@ struct WindowFunctionExponentialTimeDecayedMax final : public WindowFunction
     {
         if (parameters_.size() != 1)
         {
-            throw Exception(ErrorCodes::BAD_ARGUMENTS,
+            throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
                 "Function {} takes exactly one parameter", name_);
         }
         return applyVisitor(FieldVisitorConvertToNumber<Float64>(), parameters_[0]);
@@ -1820,7 +1823,7 @@ struct WindowFunctionExponentialTimeDecayedMax final : public WindowFunction
     {
         if (argument_types.size() != 2)
         {
-            throw Exception(ErrorCodes::BAD_ARGUMENTS,
+            throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
                 "Function {} takes exactly two arguments", name_);
         }
 
@@ -1882,7 +1885,7 @@ struct WindowFunctionExponentialTimeDecayedCount final : public StatefulWindowFu
     {
         if (parameters_.size() != 1)
         {
-            throw Exception(ErrorCodes::BAD_ARGUMENTS,
+            throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
                 "Function {} takes exactly one parameter", name_);
         }
         return applyVisitor(FieldVisitorConvertToNumber<Float64>(), parameters_[0]);
@@ -1895,7 +1898,7 @@ struct WindowFunctionExponentialTimeDecayedCount final : public StatefulWindowFu
     {
         if (argument_types.size() != 1)
         {
-            throw Exception(ErrorCodes::BAD_ARGUMENTS,
+            throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
                 "Function {} takes exactly one argument", name_);
         }
 
@@ -1968,7 +1971,7 @@ struct WindowFunctionExponentialTimeDecayedAvg final : public StatefulWindowFunc
     {
         if (parameters_.size() != 1)
         {
-            throw Exception(ErrorCodes::BAD_ARGUMENTS,
+            throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
                 "Function {} takes exactly one parameter", name_);
         }
         return applyVisitor(FieldVisitorConvertToNumber<Float64>(), parameters_[0]);
@@ -1981,7 +1984,7 @@ struct WindowFunctionExponentialTimeDecayedAvg final : public StatefulWindowFunc
     {
         if (argument_types.size() != 2)
         {
-            throw Exception(ErrorCodes::BAD_ARGUMENTS,
+            throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
                 "Function {} takes exactly two arguments", name_);
         }
 
@@ -2116,7 +2119,7 @@ struct WindowFunctionNtile final : public StatefulWindowFunction<NtileState>
         : StatefulWindowFunction<NtileState>(name_, argument_types_, parameters_, std::make_shared<DataTypeUInt64>())
     {
         if (argument_types.size() != 1)
-            throw Exception(ErrorCodes::BAD_ARGUMENTS, "Function {} takes exactly one argument", name_);
+            throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {} takes exactly one argument", name_);
 
         auto type_id = argument_types[0]->getTypeId();
         if (type_id != TypeIndex::UInt8 && type_id != TypeIndex::UInt16 && type_id != TypeIndex::UInt32 && type_id != TypeIndex::UInt64)
@@ -2191,7 +2194,7 @@ namespace
 
             if (!buckets)
             {
-                throw Exception(ErrorCodes::BAD_ARGUMENTS, "Argument of 'ntile' funtcion must be greater than zero");
+                throw Exception(ErrorCodes::BAD_ARGUMENTS, "Argument of 'ntile' function must be greater than zero");
             }
         }
         // new partition
@@ -2404,7 +2407,7 @@ struct WindowFunctionLagLeadInFrame final : public WindowFunction
 
         if (argument_types.size() > 3)
         {
-            throw Exception(ErrorCodes::BAD_ARGUMENTS,
+            throw Exception(ErrorCodes::TOO_MANY_ARGUMENTS_FOR_FUNCTION,
                 "Function '{}' accepts at most 3 arguments, {} given",
                 name, argument_types.size());
         }
@@ -2414,7 +2417,7 @@ struct WindowFunctionLagLeadInFrame final : public WindowFunction
     {
         if (argument_types_.empty())
         {
-            throw Exception(ErrorCodes::BAD_ARGUMENTS,
+            throw Exception(ErrorCodes::TOO_FEW_ARGUMENTS_FOR_FUNCTION,
                 "Function {} takes at least one argument", name_);
         }
 
@@ -2504,7 +2507,7 @@ struct WindowFunctionNthValue final : public WindowFunction
     {
         if (argument_types_.size() != 2)
         {
-            throw Exception(ErrorCodes::BAD_ARGUMENTS,
+            throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
                 "Function {} takes exactly two arguments", name_);
         }
 
@@ -2578,7 +2581,7 @@ struct NonNegativeDerivativeParams
 
         if (argument_types.size() != 2 && argument_types.size() != 3)
         {
-            throw Exception(ErrorCodes::BAD_ARGUMENTS,
+            throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
                             "Function {} takes 2 or 3 arguments", name_);
         }
 
diff --git a/tests/integration/test_backward_compatibility/test_functions.py b/tests/integration/test_backward_compatibility/test_functions.py
index fc03a77030e..3231fb87f33 100644
--- a/tests/integration/test_backward_compatibility/test_functions.py
+++ b/tests/integration/test_backward_compatibility/test_functions.py
@@ -75,11 +75,15 @@ def test_aggregate_states(start_cluster):
         except QueryRuntimeException as e:
             error_message = str(e)
             allowed_errors = [
-                "NUMBER_OF_ARGUMENTS_DOESNT_MATCH",
                 "ILLEGAL_TYPE_OF_ARGUMENT",
                 # sequenceNextNode() and friends
                 "UNKNOWN_AGGREGATE_FUNCTION",
                 # Function X takes exactly one parameter:
+                "NUMBER_OF_ARGUMENTS_DOESNT_MATCH",
+                # Function X takes at least one argument
+                "TOO_FEW_ARGUMENTS_FOR_FUNCTION",
+                # Function X accepts at most 3 arguments, Y given
+                "TOO_MANY_ARGUMENTS_FOR_FUNCTION",
                 # The function 'X' can only be used as a window function
                 "BAD_ARGUMENTS",
                 # aggThrow
@@ -196,9 +200,7 @@ def test_string_functions(start_cluster):
                 "Should start with ",  # POINT/POLYGON/...
                 "Cannot read input: expected a digit but got something else:",
                 # ErrorCodes
-                "NUMBER_OF_ARGUMENTS_DOESNT_MATCH",
                 "ILLEGAL_TYPE_OF_ARGUMENT",
-                "TOO_FEW_ARGUMENTS_FOR_FUNCTION",
                 "DICTIONARIES_WAS_NOT_LOADED",
                 "CANNOT_PARSE_UUID",
                 "CANNOT_PARSE_DOMAIN_VALUE_FROM_STRING",
@@ -218,6 +220,11 @@ def test_string_functions(start_cluster):
                 "CANNOT_PARSE_TEXT",
                 "CANNOT_PARSE_DATETIME",
                 # Function X takes exactly one parameter:
+                "NUMBER_OF_ARGUMENTS_DOESNT_MATCH",
+                # Function X takes at least one argument
+                "TOO_FEW_ARGUMENTS_FOR_FUNCTION",
+                # Function X accepts at most 3 arguments, Y given
+                "TOO_MANY_ARGUMENTS_FOR_FUNCTION",
                 # The function 'X' can only be used as a window function
                 "BAD_ARGUMENTS",
                 # String foo is obviously not a valid IP address.
diff --git a/tests/queries/0_stateless/02560_window_ntile.reference b/tests/queries/0_stateless/02560_window_ntile.reference
index d877b2034cb..5f01832d075 100644
--- a/tests/queries/0_stateless/02560_window_ntile.reference
+++ b/tests/queries/0_stateless/02560_window_ntile.reference
@@ -213,6 +213,8 @@ select a, b, ntile('2') over (partition by a order by b) from(select intDiv(numb
 select a, b, ntile(0) over (partition by a order by b) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError BAD_ARGUMENTS }
 select a, b, ntile(-2) over (partition by a order by b) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError BAD_ARGUMENTS }
 select a, b, ntile(b + 1) over (partition by a order by b) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError BAD_ARGUMENTS }
+select a, b, ntile() over (partition by a order by b) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH }
+select a, b, ntile(3, 2) over (partition by a order by b) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH }
 -- Bad window type
 select a, b, ntile(2) over (partition by a) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError BAD_ARGUMENTS }
 select a, b, ntile(2) over (partition by a order by b rows between 4 preceding and unbounded following) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError BAD_ARGUMENTS }
diff --git a/tests/queries/0_stateless/02560_window_ntile.sql b/tests/queries/0_stateless/02560_window_ntile.sql
index 44e9b865052..d0e4d557e58 100644
--- a/tests/queries/0_stateless/02560_window_ntile.sql
+++ b/tests/queries/0_stateless/02560_window_ntile.sql
@@ -16,6 +16,8 @@ select a, b, ntile('2') over (partition by a order by b) from(select intDiv(numb
 select a, b, ntile(0) over (partition by a order by b) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError BAD_ARGUMENTS }
 select a, b, ntile(-2) over (partition by a order by b) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError BAD_ARGUMENTS }
 select a, b, ntile(b + 1) over (partition by a order by b) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError BAD_ARGUMENTS }
+select a, b, ntile() over (partition by a order by b) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH }
+select a, b, ntile(3, 2) over (partition by a order by b) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH }
 
 -- Bad window type
 select a, b, ntile(2) over (partition by a) from(select intDiv(number,10) as a, number%10 as b from numbers(20)); -- { serverError BAD_ARGUMENTS }
diff --git a/tests/queries/0_stateless/03131_hilbert_coding.sql b/tests/queries/0_stateless/03131_hilbert_coding.sql
index ed293dc6910..b16a0efad5d 100644
--- a/tests/queries/0_stateless/03131_hilbert_coding.sql
+++ b/tests/queries/0_stateless/03131_hilbert_coding.sql
@@ -46,6 +46,7 @@ drop table if exists hilbert_numbers_1_03131;
 
 select '----- ERRORS -----';
 select hilbertEncode(); -- { serverError TOO_FEW_ARGUMENTS_FOR_FUNCTION }
+select hilbertEncode(1, 2, 3); -- { serverError TOO_MANY_ARGUMENTS_FOR_FUNCTION }
 select hilbertDecode(); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH }
 select hilbertEncode('text'); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT }
 select hilbertDecode('text', 'text'); -- { serverError ILLEGAL_COLUMN }

From 41218ad01889cce72569a840461b54dbf9f4b832 Mon Sep 17 00:00:00 2001
From: Nikita Fomichev <nikita.fomichev@clickhouse.com>
Date: Mon, 22 Jul 2024 12:03:21 +0200
Subject: [PATCH 0566/1488] Stateless tests: add no-parallel tag for high-load
 tests

---
 .../01076_cache_dictionary_datarace_exception_ptr.sh            | 2 +-
 .../0_stateless/01171_mv_select_insert_isolation_long.sh        | 2 +-
 .../0_stateless/01301_aggregate_state_exception_memory_leak.sh  | 2 +-
 .../0_stateless/01302_aggregate_state_exception_memory_leak.sh  | 2 +-
 tests/queries/0_stateless/02372_data_race_in_avro.sh            | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/tests/queries/0_stateless/01076_cache_dictionary_datarace_exception_ptr.sh b/tests/queries/0_stateless/01076_cache_dictionary_datarace_exception_ptr.sh
index dcd15718416..e003d2a26da 100755
--- a/tests/queries/0_stateless/01076_cache_dictionary_datarace_exception_ptr.sh
+++ b/tests/queries/0_stateless/01076_cache_dictionary_datarace_exception_ptr.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Tags: race
+# Tags: race, no-parallel
 
 # This is a monkey test used to trigger sanitizers.
 
diff --git a/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh b/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh
index 2ab7f883367..8344bb6f426 100755
--- a/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh
+++ b/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Tags: long, no-ordinary-database, no-debug
+# Tags: long, no-parallel, no-ordinary-database, no-debug
 # Test is too heavy, avoid parallel run in Flaky Check
 # shellcheck disable=SC2119
 
diff --git a/tests/queries/0_stateless/01301_aggregate_state_exception_memory_leak.sh b/tests/queries/0_stateless/01301_aggregate_state_exception_memory_leak.sh
index 47fe7a9c7d9..d74092d828d 100755
--- a/tests/queries/0_stateless/01301_aggregate_state_exception_memory_leak.sh
+++ b/tests/queries/0_stateless/01301_aggregate_state_exception_memory_leak.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Tags: no-fasttest
+# Tags: no-fasttest, no-parallel
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
diff --git a/tests/queries/0_stateless/01302_aggregate_state_exception_memory_leak.sh b/tests/queries/0_stateless/01302_aggregate_state_exception_memory_leak.sh
index a521accb082..bbf2fd9177a 100755
--- a/tests/queries/0_stateless/01302_aggregate_state_exception_memory_leak.sh
+++ b/tests/queries/0_stateless/01302_aggregate_state_exception_memory_leak.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Tags: no-fasttest
+# Tags: no-fasttest, no-parallel
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
diff --git a/tests/queries/0_stateless/02372_data_race_in_avro.sh b/tests/queries/0_stateless/02372_data_race_in_avro.sh
index 49c34e31923..50a7ae1e3c5 100755
--- a/tests/queries/0_stateless/02372_data_race_in_avro.sh
+++ b/tests/queries/0_stateless/02372_data_race_in_avro.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Tags: no-fasttest
+# Tags: no-fasttest, no-parallel
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh

From bb0b29f6e50e74098fcc8a9b83150998f1bc2601 Mon Sep 17 00:00:00 2001
From: Alexander Gololobov <davenger@clickhouse.com>
Date: Mon, 22 Jul 2024 12:11:56 +0200
Subject: [PATCH 0567/1488] Set writer_thread_id earlier, when new exclusive
 owener is waiting for existing readers to finish

---
 src/Common/SharedMutex.cpp | 6 ++++--
 src/Common/SharedMutex.h   | 2 +-
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/src/Common/SharedMutex.cpp b/src/Common/SharedMutex.cpp
index 7b00ef0b28b..2d63f8542b0 100644
--- a/src/Common/SharedMutex.cpp
+++ b/src/Common/SharedMutex.cpp
@@ -31,11 +31,13 @@ void SharedMutex::lock()
             break;
     }
 
+    /// The first step of acquiring the exclusive ownership is finished.
+    /// Now we just wait until all readers release the shared ownership.
+    writer_thread_id.store(getThreadId());
+
     value |= writers;
     while (value & readers)
         futexWaitLowerFetch(state, value);
-
-    writer_thread_id.store(getThreadId());
 }
 
 bool SharedMutex::try_lock()
diff --git a/src/Common/SharedMutex.h b/src/Common/SharedMutex.h
index c77c8765885..d2947645eca 100644
--- a/src/Common/SharedMutex.h
+++ b/src/Common/SharedMutex.h
@@ -38,7 +38,7 @@ private:
 
     alignas(64) std::atomic<UInt64> state;
     std::atomic<UInt32> waiters;
-    /// Is set while the lock is held in exclusive mode only to facilitate debugging
+    /// Is set while the lock is held (or is in the process of being acquired) in exclusive mode only to facilitate debugging
     std::atomic<UInt64> writer_thread_id;
 };
 

From c36dde51031e817b20db6773df47a42092f6616b Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Mon, 22 Jul 2024 12:17:29 +0200
Subject: [PATCH 0568/1488] Fix KeeperMap create after incomplete drop

---
 src/Common/FailPoint.cpp                      |  3 ++-
 src/Storages/StorageKeeperMap.cpp             | 27 +++++++++++--------
 ...03208_keepermap_failed_data_drop.reference |  0
 .../03208_keepermap_failed_data_drop.sql      |  7 +++++
 4 files changed, 25 insertions(+), 12 deletions(-)
 create mode 100644 tests/queries/0_stateless/03208_keepermap_failed_data_drop.reference
 create mode 100644 tests/queries/0_stateless/03208_keepermap_failed_data_drop.sql

diff --git a/src/Common/FailPoint.cpp b/src/Common/FailPoint.cpp
index 5454cba8e2e..b952e9725e3 100644
--- a/src/Common/FailPoint.cpp
+++ b/src/Common/FailPoint.cpp
@@ -57,7 +57,8 @@ static struct InitFiu
     PAUSEABLE_ONCE(finish_clean_quorum_failed_parts) \
     PAUSEABLE(dummy_pausable_failpoint) \
     ONCE(execute_query_calling_empty_set_result_func_on_exception) \
-    ONCE(receive_timeout_on_table_status_response)
+    ONCE(receive_timeout_on_table_status_response) \
+    ONCE(keepermap_fail_drop_data) \
 
 
 namespace FailPoints
diff --git a/src/Storages/StorageKeeperMap.cpp b/src/Storages/StorageKeeperMap.cpp
index 587cb621362..1c1de245d10 100644
--- a/src/Storages/StorageKeeperMap.cpp
+++ b/src/Storages/StorageKeeperMap.cpp
@@ -37,6 +37,7 @@
 
 #include <Common/Base64.h>
 #include <Common/Exception.h>
+#include <Common/FailPoint.h>
 #include <Common/ZooKeeper/IKeeper.h>
 #include <Common/ZooKeeper/KeeperException.h>
 #include <Common/ZooKeeper/Types.h>
@@ -64,6 +65,11 @@
 namespace DB
 {
 
+namespace FailPoints
+{
+    extern const char keepermap_fail_drop_data[];
+}
+
 namespace ErrorCodes
 {
     extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
@@ -411,18 +417,13 @@ StorageKeeperMap::StorageKeeperMap(
 
             auto code = client->tryCreate(zk_table_path, "", zkutil::CreateMode::Persistent);
 
-            // tables_path was removed with drop
-            if (code == Coordination::Error::ZNONODE)
-            {
-                LOG_INFO(log, "Metadata nodes were removed by another server, will retry");
-                continue;
-            }
-            else if (code != Coordination::Error::ZOK)
-            {
-                throw zkutil::KeeperException(code, "Failed to create table on path {} because a table with same UUID already exists", zk_root_path);
-            }
+            if (code == Coordination::Error::ZOK)
+                return;
 
-            return;
+            if (code != Coordination::Error::ZNONODE)
+                throw zkutil::KeeperException(code, "Failed to create table on path {} because a table with same UUID already exists", zk_root_path);
+
+            /// ZNONODE means we dropped zk_tables_path but didn't finish drop completely
         }
 
         if (client->exists(zk_dropped_path))
@@ -561,6 +562,10 @@ void StorageKeeperMap::truncate(const ASTPtr &, const StorageMetadataPtr &, Cont
 
 bool StorageKeeperMap::dropTable(zkutil::ZooKeeperPtr zookeeper, const zkutil::EphemeralNodeHolder::Ptr & metadata_drop_lock)
 {
+    fiu_do_on(FailPoints::keepermap_fail_drop_data,
+    {
+        throw zkutil::KeeperException(Coordination::Error::ZOPERATIONTIMEOUT, "Manually triggered operation timeout");
+    });
     zookeeper->removeChildrenRecursive(zk_data_path);
 
     bool completely_removed = false;
diff --git a/tests/queries/0_stateless/03208_keepermap_failed_data_drop.reference b/tests/queries/0_stateless/03208_keepermap_failed_data_drop.reference
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/queries/0_stateless/03208_keepermap_failed_data_drop.sql b/tests/queries/0_stateless/03208_keepermap_failed_data_drop.sql
new file mode 100644
index 00000000000..ad0603f12a9
--- /dev/null
+++ b/tests/queries/0_stateless/03208_keepermap_failed_data_drop.sql
@@ -0,0 +1,7 @@
+DROP TABLE IF EXISTS 03208_keepermap_test SYNC;
+
+CREATE TABLE 03208_keepermap_test (key UInt64, value UInt64) Engine=KeeperMap('/' || currentDatabase() || '/test03208') PRIMARY KEY(key);
+INSERT INTO 03208_keepermap_test VALUES (1, 11);
+SYSTEM ENABLE FAILPOINT keepermap_fail_drop_data;
+DROP TABLE 03208_keepermap_test;
+CREATE TABLE 03208_keepermap_test_another (key UInt64, value UInt64) Engine=KeeperMap('/' || currentDatabase() || '/test03208') PRIMARY KEY(key);

From 1ea05e55d25e144ff2294265a4d70c420174004a Mon Sep 17 00:00:00 2001
From: avogar <avogar@clickhouse.com>
Date: Mon, 22 Jul 2024 10:19:54 +0000
Subject: [PATCH 0569/1488] Split test 03038_nested_dynamic_merges to avoid
 timeouts

---
 .../03038_nested_dynamic_merges.reference     | 92 -------------------
 .../03038_nested_dynamic_merges.sh            | 53 -----------
 ...ynamic_merges_compact_horizontal.reference | 21 +++++
 ...ested_dynamic_merges_compact_horizontal.sh | 32 +++++++
 ..._dynamic_merges_compact_vertical.reference | 21 +++++
 ..._nested_dynamic_merges_compact_vertical.sh | 32 +++++++
 ...d_dynamic_merges_wide_horizontal.reference | 21 +++++
 ...8_nested_dynamic_merges_wide_horizontal.sh | 32 +++++++
 ...ted_dynamic_merges_wide_vertical.reference | 21 +++++
 ...038_nested_dynamic_merges_wide_vertical.sh | 32 +++++++
 10 files changed, 212 insertions(+), 145 deletions(-)
 delete mode 100644 tests/queries/0_stateless/03038_nested_dynamic_merges.reference
 delete mode 100755 tests/queries/0_stateless/03038_nested_dynamic_merges.sh
 create mode 100644 tests/queries/0_stateless/03038_nested_dynamic_merges_compact_horizontal.reference
 create mode 100755 tests/queries/0_stateless/03038_nested_dynamic_merges_compact_horizontal.sh
 create mode 100644 tests/queries/0_stateless/03038_nested_dynamic_merges_compact_vertical.reference
 create mode 100755 tests/queries/0_stateless/03038_nested_dynamic_merges_compact_vertical.sh
 create mode 100644 tests/queries/0_stateless/03038_nested_dynamic_merges_wide_horizontal.reference
 create mode 100755 tests/queries/0_stateless/03038_nested_dynamic_merges_wide_horizontal.sh
 create mode 100644 tests/queries/0_stateless/03038_nested_dynamic_merges_wide_vertical.reference
 create mode 100755 tests/queries/0_stateless/03038_nested_dynamic_merges_wide_vertical.sh

diff --git a/tests/queries/0_stateless/03038_nested_dynamic_merges.reference b/tests/queries/0_stateless/03038_nested_dynamic_merges.reference
deleted file mode 100644
index 65034647775..00000000000
--- a/tests/queries/0_stateless/03038_nested_dynamic_merges.reference
+++ /dev/null
@@ -1,92 +0,0 @@
-MergeTree compact + horizontal merge
-test
-16667	Tuple(a Dynamic(max_types=3)):Date
-33333	Tuple(a Dynamic(max_types=3)):Array(UInt8)
-50000	Tuple(a Dynamic(max_types=3)):String
-50000	Tuple(a Dynamic(max_types=3)):UInt64
-100000	UInt64:None
-33333	Tuple(a Dynamic(max_types=3)):Array(UInt8)
-50000	Tuple(a Dynamic(max_types=3)):UInt64
-66667	Tuple(a Dynamic(max_types=3)):String
-100000	UInt64:None
-16667	Tuple(a Dynamic(max_types=3)):DateTime
-33333	Tuple(a Dynamic(max_types=3)):Array(UInt8)
-50000	Tuple(a Dynamic(max_types=3)):UInt64
-66667	Tuple(a Dynamic(max_types=3)):String
-100000	Tuple(a Dynamic(max_types=3)):Tuple(UInt64)
-100000	UInt64:None
-133333	Tuple(a Dynamic(max_types=3)):None
-50000	Tuple(a Dynamic(max_types=3)):UInt64
-100000	Tuple(a Dynamic(max_types=3)):Tuple(UInt64)
-100000	UInt64:None
-116667	Tuple(a Dynamic(max_types=3)):String
-133333	Tuple(a Dynamic(max_types=3)):None
-MergeTree wide + horizontal merge
-test
-16667	Tuple(a Dynamic(max_types=3)):Date
-33333	Tuple(a Dynamic(max_types=3)):Array(UInt8)
-50000	Tuple(a Dynamic(max_types=3)):String
-50000	Tuple(a Dynamic(max_types=3)):UInt64
-100000	UInt64:None
-33333	Tuple(a Dynamic(max_types=3)):Array(UInt8)
-50000	Tuple(a Dynamic(max_types=3)):UInt64
-66667	Tuple(a Dynamic(max_types=3)):String
-100000	UInt64:None
-16667	Tuple(a Dynamic(max_types=3)):DateTime
-33333	Tuple(a Dynamic(max_types=3)):Array(UInt8)
-50000	Tuple(a Dynamic(max_types=3)):UInt64
-66667	Tuple(a Dynamic(max_types=3)):String
-100000	Tuple(a Dynamic(max_types=3)):Tuple(UInt64)
-100000	UInt64:None
-133333	Tuple(a Dynamic(max_types=3)):None
-50000	Tuple(a Dynamic(max_types=3)):UInt64
-100000	Tuple(a Dynamic(max_types=3)):Tuple(UInt64)
-100000	UInt64:None
-116667	Tuple(a Dynamic(max_types=3)):String
-133333	Tuple(a Dynamic(max_types=3)):None
-MergeTree compact + vertical merge
-test
-16667	Tuple(a Dynamic(max_types=3)):Date
-33333	Tuple(a Dynamic(max_types=3)):Array(UInt8)
-50000	Tuple(a Dynamic(max_types=3)):String
-50000	Tuple(a Dynamic(max_types=3)):UInt64
-100000	UInt64:None
-33333	Tuple(a Dynamic(max_types=3)):Array(UInt8)
-50000	Tuple(a Dynamic(max_types=3)):UInt64
-66667	Tuple(a Dynamic(max_types=3)):String
-100000	UInt64:None
-16667	Tuple(a Dynamic(max_types=3)):DateTime
-33333	Tuple(a Dynamic(max_types=3)):Array(UInt8)
-50000	Tuple(a Dynamic(max_types=3)):UInt64
-66667	Tuple(a Dynamic(max_types=3)):String
-100000	Tuple(a Dynamic(max_types=3)):Tuple(UInt64)
-100000	UInt64:None
-133333	Tuple(a Dynamic(max_types=3)):None
-50000	Tuple(a Dynamic(max_types=3)):UInt64
-100000	Tuple(a Dynamic(max_types=3)):Tuple(UInt64)
-100000	UInt64:None
-116667	Tuple(a Dynamic(max_types=3)):String
-133333	Tuple(a Dynamic(max_types=3)):None
-MergeTree wide + vertical merge
-test
-16667	Tuple(a Dynamic(max_types=3)):Date
-33333	Tuple(a Dynamic(max_types=3)):Array(UInt8)
-50000	Tuple(a Dynamic(max_types=3)):String
-50000	Tuple(a Dynamic(max_types=3)):UInt64
-100000	UInt64:None
-33333	Tuple(a Dynamic(max_types=3)):Array(UInt8)
-50000	Tuple(a Dynamic(max_types=3)):UInt64
-66667	Tuple(a Dynamic(max_types=3)):String
-100000	UInt64:None
-16667	Tuple(a Dynamic(max_types=3)):DateTime
-33333	Tuple(a Dynamic(max_types=3)):Array(UInt8)
-50000	Tuple(a Dynamic(max_types=3)):UInt64
-66667	Tuple(a Dynamic(max_types=3)):String
-100000	Tuple(a Dynamic(max_types=3)):Tuple(UInt64)
-100000	UInt64:None
-133333	Tuple(a Dynamic(max_types=3)):None
-50000	Tuple(a Dynamic(max_types=3)):UInt64
-100000	Tuple(a Dynamic(max_types=3)):Tuple(UInt64)
-100000	UInt64:None
-116667	Tuple(a Dynamic(max_types=3)):String
-133333	Tuple(a Dynamic(max_types=3)):None
diff --git a/tests/queries/0_stateless/03038_nested_dynamic_merges.sh b/tests/queries/0_stateless/03038_nested_dynamic_merges.sh
deleted file mode 100755
index b82ddb3813e..00000000000
--- a/tests/queries/0_stateless/03038_nested_dynamic_merges.sh
+++ /dev/null
@@ -1,53 +0,0 @@
-#!/usr/bin/env bash
-# Tags: long
-
-CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# reset --log_comment
-CLICKHOUSE_LOG_COMMENT=
-# shellcheck source=../shell_config.sh
-. "$CUR_DIR"/../shell_config.sh
-
-CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --use_variant_as_common_type=1 --allow_experimental_dynamic_type=1"
-
-
-function test()
-{
-    echo "test"
-    $CH_CLIENT -q "system stop merges test"
-    $CH_CLIENT -q "insert into test select number, number from numbers(100000)"
-    $CH_CLIENT -q "insert into test select number, tuple(if(number % 2 == 0, number, 'str_' || toString(number)))::Tuple(a Dynamic(max_types=3)) from numbers(100000)"
-    $CH_CLIENT -q "insert into test select number, tuple(if(number % 3 == 0, toDate(number), range(number % 10)))::Tuple(a Dynamic(max_types=3)) from numbers(50000)"
-
-    $CH_CLIENT -q "select count(), dynamicType(d) || ':' || dynamicType(d.\`Tuple(a Dynamic(max_types=3))\`.a) as type from test group by type order by count(), type"
-    $CH_CLIENT -nm -q "system start merges test; optimize table test final;"
-    $CH_CLIENT -q "select count(), dynamicType(d) || ':' || dynamicType(d.\`Tuple(a Dynamic(max_types=3))\`.a) as type from test group by type order by count(), type"
-
-    $CH_CLIENT -q "insert into test select number, tuple(if(number % 3 == 0, toDateTime(number), NULL))::Tuple(a Dynamic(max_types=3)) from numbers(50000)"
-    $CH_CLIENT -q "insert into test select number, tuple(if(number % 2 == 0, tuple(number), NULL))::Tuple(a Dynamic(max_types=3)) from numbers(200000)"
-
-    $CH_CLIENT -q "select count(), dynamicType(d) || ':' || dynamicType(d.\`Tuple(a Dynamic(max_types=3))\`.a) as type from test group by type order by count(), type"
-    $CH_CLIENT -nm -q "system start merges test; optimize table test final;"
-    $CH_CLIENT -q "select count(), dynamicType(d) || ':' || dynamicType(d.\`Tuple(a Dynamic(max_types=3))\`.a) as type from test group by type order by count(), type"
-}
-
-$CH_CLIENT -q "drop table if exists test;"
-
-echo "MergeTree compact + horizontal merge"
-$CH_CLIENT -q "create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000;"
-test
-$CH_CLIENT -q "drop table test;"
-
-echo "MergeTree wide + horizontal merge"
-$CH_CLIENT -q "create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1;"
-test
-$CH_CLIENT -q "drop table test;"
-
-echo "MergeTree compact + vertical merge"
-$CH_CLIENT -q "create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1;"
-test
-$CH_CLIENT -q "drop table test;"
-
-echo "MergeTree wide + vertical merge"
-$CH_CLIENT -q "create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1;"
-test
-$CH_CLIENT -q "drop table test;"
diff --git a/tests/queries/0_stateless/03038_nested_dynamic_merges_compact_horizontal.reference b/tests/queries/0_stateless/03038_nested_dynamic_merges_compact_horizontal.reference
new file mode 100644
index 00000000000..27ed336a035
--- /dev/null
+++ b/tests/queries/0_stateless/03038_nested_dynamic_merges_compact_horizontal.reference
@@ -0,0 +1,21 @@
+16667	Tuple(a Dynamic(max_types=3)):Date
+33333	Tuple(a Dynamic(max_types=3)):Array(UInt8)
+50000	Tuple(a Dynamic(max_types=3)):String
+50000	Tuple(a Dynamic(max_types=3)):UInt64
+100000	UInt64:None
+33333	Tuple(a Dynamic(max_types=3)):Array(UInt8)
+50000	Tuple(a Dynamic(max_types=3)):UInt64
+66667	Tuple(a Dynamic(max_types=3)):String
+100000	UInt64:None
+16667	Tuple(a Dynamic(max_types=3)):DateTime
+33333	Tuple(a Dynamic(max_types=3)):Array(UInt8)
+50000	Tuple(a Dynamic(max_types=3)):UInt64
+66667	Tuple(a Dynamic(max_types=3)):String
+100000	Tuple(a Dynamic(max_types=3)):Tuple(number UInt64)
+100000	UInt64:None
+133333	Tuple(a Dynamic(max_types=3)):None
+50000	Tuple(a Dynamic(max_types=3)):UInt64
+100000	Tuple(a Dynamic(max_types=3)):Tuple(number UInt64)
+100000	UInt64:None
+116667	Tuple(a Dynamic(max_types=3)):String
+133333	Tuple(a Dynamic(max_types=3)):None
diff --git a/tests/queries/0_stateless/03038_nested_dynamic_merges_compact_horizontal.sh b/tests/queries/0_stateless/03038_nested_dynamic_merges_compact_horizontal.sh
new file mode 100755
index 00000000000..cb8a8fad6ba
--- /dev/null
+++ b/tests/queries/0_stateless/03038_nested_dynamic_merges_compact_horizontal.sh
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+# Tags: long
+
+CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# reset --log_comment
+CLICKHOUSE_LOG_COMMENT=
+# shellcheck source=../shell_config.sh
+. "$CUR_DIR"/../shell_config.sh
+
+CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --use_variant_as_common_type=1 --allow_experimental_dynamic_type=1"
+
+$CH_CLIENT -q "drop table if exists test;"
+$CH_CLIENT -q "create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000;"
+
+$CH_CLIENT -q "system stop merges test"
+$CH_CLIENT -q "insert into test select number, number from numbers(100000)"
+$CH_CLIENT -q "insert into test select number, tuple(if(number % 2 == 0, number, 'str_' || toString(number)))::Tuple(a Dynamic(max_types=3)) from numbers(100000)"
+$CH_CLIENT -q "insert into test select number, tuple(if(number % 3 == 0, toDate(number), range(number % 10)))::Tuple(a Dynamic(max_types=3)) from numbers(50000)"
+
+$CH_CLIENT -q "select count(), dynamicType(d) || ':' || dynamicType(d.\`Tuple(a Dynamic(max_types=3))\`.a) as type from test group by type order by count(), type"
+$CH_CLIENT -nm -q "system start merges test; optimize table test final;"
+$CH_CLIENT -q "select count(), dynamicType(d) || ':' || dynamicType(d.\`Tuple(a Dynamic(max_types=3))\`.a) as type from test group by type order by count(), type"
+
+$CH_CLIENT -q "insert into test select number, tuple(if(number % 3 == 0, toDateTime(number), NULL))::Tuple(a Dynamic(max_types=3)) from numbers(50000)"
+$CH_CLIENT -q "insert into test select number, tuple(if(number % 2 == 0, tuple(number), NULL))::Tuple(a Dynamic(max_types=3)) from numbers(200000)"
+
+$CH_CLIENT -q "select count(), dynamicType(d) || ':' || dynamicType(d.\`Tuple(a Dynamic(max_types=3))\`.a) as type from test group by type order by count(), type"
+$CH_CLIENT -nm -q "system start merges test; optimize table test final;"
+$CH_CLIENT -q "select count(), dynamicType(d) || ':' || dynamicType(d.\`Tuple(a Dynamic(max_types=3))\`.a) as type from test group by type order by count(), type"
+
+$CH_CLIENT -q "drop table test;"
+
diff --git a/tests/queries/0_stateless/03038_nested_dynamic_merges_compact_vertical.reference b/tests/queries/0_stateless/03038_nested_dynamic_merges_compact_vertical.reference
new file mode 100644
index 00000000000..27ed336a035
--- /dev/null
+++ b/tests/queries/0_stateless/03038_nested_dynamic_merges_compact_vertical.reference
@@ -0,0 +1,21 @@
+16667	Tuple(a Dynamic(max_types=3)):Date
+33333	Tuple(a Dynamic(max_types=3)):Array(UInt8)
+50000	Tuple(a Dynamic(max_types=3)):String
+50000	Tuple(a Dynamic(max_types=3)):UInt64
+100000	UInt64:None
+33333	Tuple(a Dynamic(max_types=3)):Array(UInt8)
+50000	Tuple(a Dynamic(max_types=3)):UInt64
+66667	Tuple(a Dynamic(max_types=3)):String
+100000	UInt64:None
+16667	Tuple(a Dynamic(max_types=3)):DateTime
+33333	Tuple(a Dynamic(max_types=3)):Array(UInt8)
+50000	Tuple(a Dynamic(max_types=3)):UInt64
+66667	Tuple(a Dynamic(max_types=3)):String
+100000	Tuple(a Dynamic(max_types=3)):Tuple(number UInt64)
+100000	UInt64:None
+133333	Tuple(a Dynamic(max_types=3)):None
+50000	Tuple(a Dynamic(max_types=3)):UInt64
+100000	Tuple(a Dynamic(max_types=3)):Tuple(number UInt64)
+100000	UInt64:None
+116667	Tuple(a Dynamic(max_types=3)):String
+133333	Tuple(a Dynamic(max_types=3)):None
diff --git a/tests/queries/0_stateless/03038_nested_dynamic_merges_compact_vertical.sh b/tests/queries/0_stateless/03038_nested_dynamic_merges_compact_vertical.sh
new file mode 100755
index 00000000000..0627244a02d
--- /dev/null
+++ b/tests/queries/0_stateless/03038_nested_dynamic_merges_compact_vertical.sh
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+# Tags: long
+
+CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# reset --log_comment
+CLICKHOUSE_LOG_COMMENT=
+# shellcheck source=../shell_config.sh
+. "$CUR_DIR"/../shell_config.sh
+
+CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --use_variant_as_common_type=1 --allow_experimental_dynamic_type=1"
+
+$CH_CLIENT -q "drop table if exists test;"
+$CH_CLIENT -q "create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1;"
+
+$CH_CLIENT -q "system stop merges test"
+$CH_CLIENT -q "insert into test select number, number from numbers(100000)"
+$CH_CLIENT -q "insert into test select number, tuple(if(number % 2 == 0, number, 'str_' || toString(number)))::Tuple(a Dynamic(max_types=3)) from numbers(100000)"
+$CH_CLIENT -q "insert into test select number, tuple(if(number % 3 == 0, toDate(number), range(number % 10)))::Tuple(a Dynamic(max_types=3)) from numbers(50000)"
+
+$CH_CLIENT -q "select count(), dynamicType(d) || ':' || dynamicType(d.\`Tuple(a Dynamic(max_types=3))\`.a) as type from test group by type order by count(), type"
+$CH_CLIENT -nm -q "system start merges test; optimize table test final;"
+$CH_CLIENT -q "select count(), dynamicType(d) || ':' || dynamicType(d.\`Tuple(a Dynamic(max_types=3))\`.a) as type from test group by type order by count(), type"
+
+$CH_CLIENT -q "insert into test select number, tuple(if(number % 3 == 0, toDateTime(number), NULL))::Tuple(a Dynamic(max_types=3)) from numbers(50000)"
+$CH_CLIENT -q "insert into test select number, tuple(if(number % 2 == 0, tuple(number), NULL))::Tuple(a Dynamic(max_types=3)) from numbers(200000)"
+
+$CH_CLIENT -q "select count(), dynamicType(d) || ':' || dynamicType(d.\`Tuple(a Dynamic(max_types=3))\`.a) as type from test group by type order by count(), type"
+$CH_CLIENT -nm -q "system start merges test; optimize table test final;"
+$CH_CLIENT -q "select count(), dynamicType(d) || ':' || dynamicType(d.\`Tuple(a Dynamic(max_types=3))\`.a) as type from test group by type order by count(), type"
+
+$CH_CLIENT -q "drop table test;"
+
diff --git a/tests/queries/0_stateless/03038_nested_dynamic_merges_wide_horizontal.reference b/tests/queries/0_stateless/03038_nested_dynamic_merges_wide_horizontal.reference
new file mode 100644
index 00000000000..27ed336a035
--- /dev/null
+++ b/tests/queries/0_stateless/03038_nested_dynamic_merges_wide_horizontal.reference
@@ -0,0 +1,21 @@
+16667	Tuple(a Dynamic(max_types=3)):Date
+33333	Tuple(a Dynamic(max_types=3)):Array(UInt8)
+50000	Tuple(a Dynamic(max_types=3)):String
+50000	Tuple(a Dynamic(max_types=3)):UInt64
+100000	UInt64:None
+33333	Tuple(a Dynamic(max_types=3)):Array(UInt8)
+50000	Tuple(a Dynamic(max_types=3)):UInt64
+66667	Tuple(a Dynamic(max_types=3)):String
+100000	UInt64:None
+16667	Tuple(a Dynamic(max_types=3)):DateTime
+33333	Tuple(a Dynamic(max_types=3)):Array(UInt8)
+50000	Tuple(a Dynamic(max_types=3)):UInt64
+66667	Tuple(a Dynamic(max_types=3)):String
+100000	Tuple(a Dynamic(max_types=3)):Tuple(number UInt64)
+100000	UInt64:None
+133333	Tuple(a Dynamic(max_types=3)):None
+50000	Tuple(a Dynamic(max_types=3)):UInt64
+100000	Tuple(a Dynamic(max_types=3)):Tuple(number UInt64)
+100000	UInt64:None
+116667	Tuple(a Dynamic(max_types=3)):String
+133333	Tuple(a Dynamic(max_types=3)):None
diff --git a/tests/queries/0_stateless/03038_nested_dynamic_merges_wide_horizontal.sh b/tests/queries/0_stateless/03038_nested_dynamic_merges_wide_horizontal.sh
new file mode 100755
index 00000000000..f305fac071c
--- /dev/null
+++ b/tests/queries/0_stateless/03038_nested_dynamic_merges_wide_horizontal.sh
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+# Tags: long
+
+CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# reset --log_comment
+CLICKHOUSE_LOG_COMMENT=
+# shellcheck source=../shell_config.sh
+. "$CUR_DIR"/../shell_config.sh
+
+CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --use_variant_as_common_type=1 --allow_experimental_dynamic_type=1"
+
+$CH_CLIENT -q "drop table if exists test;"
+$CH_CLIENT -q "create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1;"
+
+$CH_CLIENT -q "system stop merges test"
+$CH_CLIENT -q "insert into test select number, number from numbers(100000)"
+$CH_CLIENT -q "insert into test select number, tuple(if(number % 2 == 0, number, 'str_' || toString(number)))::Tuple(a Dynamic(max_types=3)) from numbers(100000)"
+$CH_CLIENT -q "insert into test select number, tuple(if(number % 3 == 0, toDate(number), range(number % 10)))::Tuple(a Dynamic(max_types=3)) from numbers(50000)"
+
+$CH_CLIENT -q "select count(), dynamicType(d) || ':' || dynamicType(d.\`Tuple(a Dynamic(max_types=3))\`.a) as type from test group by type order by count(), type"
+$CH_CLIENT -nm -q "system start merges test; optimize table test final;"
+$CH_CLIENT -q "select count(), dynamicType(d) || ':' || dynamicType(d.\`Tuple(a Dynamic(max_types=3))\`.a) as type from test group by type order by count(), type"
+
+$CH_CLIENT -q "insert into test select number, tuple(if(number % 3 == 0, toDateTime(number), NULL))::Tuple(a Dynamic(max_types=3)) from numbers(50000)"
+$CH_CLIENT -q "insert into test select number, tuple(if(number % 2 == 0, tuple(number), NULL))::Tuple(a Dynamic(max_types=3)) from numbers(200000)"
+
+$CH_CLIENT -q "select count(), dynamicType(d) || ':' || dynamicType(d.\`Tuple(a Dynamic(max_types=3))\`.a) as type from test group by type order by count(), type"
+$CH_CLIENT -nm -q "system start merges test; optimize table test final;"
+$CH_CLIENT -q "select count(), dynamicType(d) || ':' || dynamicType(d.\`Tuple(a Dynamic(max_types=3))\`.a) as type from test group by type order by count(), type"
+
+$CH_CLIENT -q "drop table test;"
+
diff --git a/tests/queries/0_stateless/03038_nested_dynamic_merges_wide_vertical.reference b/tests/queries/0_stateless/03038_nested_dynamic_merges_wide_vertical.reference
new file mode 100644
index 00000000000..27ed336a035
--- /dev/null
+++ b/tests/queries/0_stateless/03038_nested_dynamic_merges_wide_vertical.reference
@@ -0,0 +1,21 @@
+16667	Tuple(a Dynamic(max_types=3)):Date
+33333	Tuple(a Dynamic(max_types=3)):Array(UInt8)
+50000	Tuple(a Dynamic(max_types=3)):String
+50000	Tuple(a Dynamic(max_types=3)):UInt64
+100000	UInt64:None
+33333	Tuple(a Dynamic(max_types=3)):Array(UInt8)
+50000	Tuple(a Dynamic(max_types=3)):UInt64
+66667	Tuple(a Dynamic(max_types=3)):String
+100000	UInt64:None
+16667	Tuple(a Dynamic(max_types=3)):DateTime
+33333	Tuple(a Dynamic(max_types=3)):Array(UInt8)
+50000	Tuple(a Dynamic(max_types=3)):UInt64
+66667	Tuple(a Dynamic(max_types=3)):String
+100000	Tuple(a Dynamic(max_types=3)):Tuple(number UInt64)
+100000	UInt64:None
+133333	Tuple(a Dynamic(max_types=3)):None
+50000	Tuple(a Dynamic(max_types=3)):UInt64
+100000	Tuple(a Dynamic(max_types=3)):Tuple(number UInt64)
+100000	UInt64:None
+116667	Tuple(a Dynamic(max_types=3)):String
+133333	Tuple(a Dynamic(max_types=3)):None
diff --git a/tests/queries/0_stateless/03038_nested_dynamic_merges_wide_vertical.sh b/tests/queries/0_stateless/03038_nested_dynamic_merges_wide_vertical.sh
new file mode 100755
index 00000000000..0627244a02d
--- /dev/null
+++ b/tests/queries/0_stateless/03038_nested_dynamic_merges_wide_vertical.sh
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+# Tags: long
+
+CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# reset --log_comment
+CLICKHOUSE_LOG_COMMENT=
+# shellcheck source=../shell_config.sh
+. "$CUR_DIR"/../shell_config.sh
+
+CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --use_variant_as_common_type=1 --allow_experimental_dynamic_type=1"
+
+$CH_CLIENT -q "drop table if exists test;"
+$CH_CLIENT -q "create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1;"
+
+$CH_CLIENT -q "system stop merges test"
+$CH_CLIENT -q "insert into test select number, number from numbers(100000)"
+$CH_CLIENT -q "insert into test select number, tuple(if(number % 2 == 0, number, 'str_' || toString(number)))::Tuple(a Dynamic(max_types=3)) from numbers(100000)"
+$CH_CLIENT -q "insert into test select number, tuple(if(number % 3 == 0, toDate(number), range(number % 10)))::Tuple(a Dynamic(max_types=3)) from numbers(50000)"
+
+$CH_CLIENT -q "select count(), dynamicType(d) || ':' || dynamicType(d.\`Tuple(a Dynamic(max_types=3))\`.a) as type from test group by type order by count(), type"
+$CH_CLIENT -nm -q "system start merges test; optimize table test final;"
+$CH_CLIENT -q "select count(), dynamicType(d) || ':' || dynamicType(d.\`Tuple(a Dynamic(max_types=3))\`.a) as type from test group by type order by count(), type"
+
+$CH_CLIENT -q "insert into test select number, tuple(if(number % 3 == 0, toDateTime(number), NULL))::Tuple(a Dynamic(max_types=3)) from numbers(50000)"
+$CH_CLIENT -q "insert into test select number, tuple(if(number % 2 == 0, tuple(number), NULL))::Tuple(a Dynamic(max_types=3)) from numbers(200000)"
+
+$CH_CLIENT -q "select count(), dynamicType(d) || ':' || dynamicType(d.\`Tuple(a Dynamic(max_types=3))\`.a) as type from test group by type order by count(), type"
+$CH_CLIENT -nm -q "system start merges test; optimize table test final;"
+$CH_CLIENT -q "select count(), dynamicType(d) || ':' || dynamicType(d.\`Tuple(a Dynamic(max_types=3))\`.a) as type from test group by type order by count(), type"
+
+$CH_CLIENT -q "drop table test;"
+

From 378502a331c60cc023e0c965611237cb5a3cfb47 Mon Sep 17 00:00:00 2001
From: Nikita Fomichev <nikita.fomichev@clickhouse.com>
Date: Mon, 22 Jul 2024 12:54:05 +0200
Subject: [PATCH 0570/1488] Stateless tests: sync tests with private

---
 .../01651_lc_insert_tiny_log.reference        |  12 +-
 .../0_stateless/01651_lc_insert_tiny_log.sql  |   6 +-
 .../01753_direct_dictionary_simple_key.sql    |   2 +-
 .../0_stateless/02372_analyzer_join.reference | 688 +++++++++---------
 .../0_stateless/02372_analyzer_join.sql.j2    |  44 +-
 .../02373_analyzer_join_use_nulls.reference   |  16 +-
 .../02373_analyzer_join_use_nulls.sql         |  16 +-
 .../02992_all_columns_should_have_comment.sql |   6 +-
 8 files changed, 397 insertions(+), 393 deletions(-)

diff --git a/tests/queries/0_stateless/01651_lc_insert_tiny_log.reference b/tests/queries/0_stateless/01651_lc_insert_tiny_log.reference
index 3da44c57b27..5cc8909b6c8 100644
--- a/tests/queries/0_stateless/01651_lc_insert_tiny_log.reference
+++ b/tests/queries/0_stateless/01651_lc_insert_tiny_log.reference
@@ -1,12 +1,12 @@
 10000000
 10000000	1274991808
-20000000
-20000000	2549983616
+30000000
+30000000	3824991808
 10000000
 10000000	1274991808
-20000000
-20000000	2549983616
+30000000
+30000000	3824991808
 10000000
 10000000	1274991808
-20000000
-20000000	2549983616
+30000000
+30000000	3824991808
diff --git a/tests/queries/0_stateless/01651_lc_insert_tiny_log.sql b/tests/queries/0_stateless/01651_lc_insert_tiny_log.sql
index d405bb01fd9..d11c9120c61 100644
--- a/tests/queries/0_stateless/01651_lc_insert_tiny_log.sql
+++ b/tests/queries/0_stateless/01651_lc_insert_tiny_log.sql
@@ -8,7 +8,7 @@ INSERT INTO perf_lc_num (num) SELECT toUInt8(number) FROM numbers(10000000);
 select sum(length(arr)) from perf_lc_num;
 select sum(length(arr)), sum(num) from perf_lc_num;
 
-INSERT INTO perf_lc_num (num) SELECT toUInt8(number) FROM numbers(10000000);
+INSERT INTO perf_lc_num (num) SELECT toUInt8(number) FROM numbers(10000000, 20000000);
 
 select sum(length(arr)) from perf_lc_num;
 select sum(length(arr)), sum(num) from perf_lc_num;
@@ -23,7 +23,7 @@ INSERT INTO perf_lc_num (num) SELECT toUInt8(number) FROM numbers(10000000);
 select sum(length(arr)) from perf_lc_num;
 select sum(length(arr)), sum(num) from perf_lc_num;
 
-INSERT INTO perf_lc_num (num) SELECT toUInt8(number) FROM numbers(10000000);
+INSERT INTO perf_lc_num (num) SELECT toUInt8(number) FROM numbers(10000000, 20000000);
 
 select sum(length(arr)) from perf_lc_num;
 select sum(length(arr)), sum(num) from perf_lc_num;
@@ -38,7 +38,7 @@ INSERT INTO perf_lc_num (num) SELECT toUInt8(number) FROM numbers(10000000);
 select sum(length(arr)) from perf_lc_num;
 select sum(length(arr)), sum(num) from perf_lc_num;
 
-INSERT INTO perf_lc_num (num) SELECT toUInt8(number) FROM numbers(10000000);
+INSERT INTO perf_lc_num (num) SELECT toUInt8(number) FROM numbers(10000000, 20000000);
 
 select sum(length(arr)) from perf_lc_num;
 select sum(length(arr)), sum(num) from perf_lc_num;
diff --git a/tests/queries/0_stateless/01753_direct_dictionary_simple_key.sql b/tests/queries/0_stateless/01753_direct_dictionary_simple_key.sql
index 86af09f391d..93ed3f93c4e 100644
--- a/tests/queries/0_stateless/01753_direct_dictionary_simple_key.sql
+++ b/tests/queries/0_stateless/01753_direct_dictionary_simple_key.sql
@@ -41,7 +41,7 @@ SELECT dictGetOrDefault('01753_dictionary_db.direct_dictionary_simple_key_simple
 SELECT 'dictHas';
 SELECT dictHas('01753_dictionary_db.direct_dictionary_simple_key_simple_attributes', number) FROM system.numbers LIMIT 4;
 SELECT 'select all values as input stream';
-SELECT * FROM 01753_dictionary_db.direct_dictionary_simple_key_simple_attributes;
+SELECT * FROM 01753_dictionary_db.direct_dictionary_simple_key_simple_attributes ORDER BY ALL;
 
 DROP DICTIONARY 01753_dictionary_db.direct_dictionary_simple_key_simple_attributes;
 DROP TABLE 01753_dictionary_db.simple_key_simple_attributes_source_table;
diff --git a/tests/queries/0_stateless/02372_analyzer_join.reference b/tests/queries/0_stateless/02372_analyzer_join.reference
index eefcb1e50dc..9204dded262 100644
--- a/tests/queries/0_stateless/02372_analyzer_join.reference
+++ b/tests/queries/0_stateless/02372_analyzer_join.reference
@@ -5,63 +5,63 @@ JOIN INNER
 SELECT 'JOIN ON without conditions';
 JOIN ON without conditions
 SELECT test_table_join_1.id, test_table_join_1.value, test_table_join_2.id, test_table_join_2.value
-FROM test_table_join_1 INNER JOIN test_table_join_2 ON test_table_join_1.id = test_table_join_2.id;
+FROM test_table_join_1 INNER JOIN test_table_join_2 ON test_table_join_1.id = test_table_join_2.id ORDER BY ALL;
 0	Join_1_Value_0	0	Join_2_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1
 SELECT '--';
 --
 SELECT t1.id, t1.value, t2.id, t2.value
-FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON t1.id = t2.id;
+FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON t1.id = t2.id ORDER BY ALL;
 0	Join_1_Value_0	0	Join_2_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1
 SELECT '--';
 --
 SELECT t1.id, test_table_join_1.id, t1.value, test_table_join_1.value, t2.id, test_table_join_2.id, t2.value, test_table_join_2.value
-FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON t1.id = t2.id;
+FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON t1.id = t2.id ORDER BY ALL;
 0	0	Join_1_Value_0	Join_1_Value_0	0	0	Join_2_Value_0	Join_2_Value_0
 1	1	Join_1_Value_1	Join_1_Value_1	1	1	Join_2_Value_1	Join_2_Value_1
 SELECT '--';
 --
 SELECT t1.value, t2.value
-FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON t1.id = t2.id;
+FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON t1.id = t2.id ORDER BY ALL;
 Join_1_Value_0	Join_2_Value_0
 Join_1_Value_1	Join_2_Value_1
-SELECT id FROM test_table_join_1 INNER JOIN test_table_join_2 ON test_table_join_1.id = test_table_join_2.id; -- { serverError AMBIGUOUS_IDENTIFIER }
-SELECT value FROM test_table_join_1 INNER JOIN test_table_join_2 ON test_table_join_1.id = test_table_join_2.id; -- { serverError AMBIGUOUS_IDENTIFIER }
+SELECT id FROM test_table_join_1 INNER JOIN test_table_join_2 ON test_table_join_1.id = test_table_join_2.id ORDER BY ALL; -- { serverError AMBIGUOUS_IDENTIFIER }
+SELECT value FROM test_table_join_1 INNER JOIN test_table_join_2 ON test_table_join_1.id = test_table_join_2.id ORDER BY ALL; -- { serverError AMBIGUOUS_IDENTIFIER }
 SELECT 'JOIN ON with conditions';
 JOIN ON with conditions
 SELECT t1.id, t1.value, t2.id, t2.value
-FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t1.value = 'Join_1_Value_0';
+FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t1.value = 'Join_1_Value_0' ORDER BY ALL;
 0	Join_1_Value_0	0	Join_2_Value_0
 SELECT '--';
 --
 SELECT t1.id, t1.value, t2.id, t2.value
-FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t2.value = 'Join_2_Value_0';
+FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t2.value = 'Join_2_Value_0' ORDER BY ALL;
 0	Join_1_Value_0	0	Join_2_Value_0
 SELECT '--';
 --
 SELECT t1.id, t1.value, t2.id, t2.value
-FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t1.value = 'Join_1_Value_0' AND t2.value = 'Join_2_Value_0';
+FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t1.value = 'Join_1_Value_0' AND t2.value = 'Join_2_Value_0' ORDER BY ALL;
 0	Join_1_Value_0	0	Join_2_Value_0
 SELECT '--';
 --
 SELECT t1.id, t1.value, t2.id, t2.value
-FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON toString(t1.id) = toString(t2.id) AND t1.value = 'Join_1_Value_0' AND t2.value = 'Join_2_Value_0';
+FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON toString(t1.id) = toString(t2.id) AND t1.value = 'Join_1_Value_0' AND t2.value = 'Join_2_Value_0' ORDER BY ALL;
 0	Join_1_Value_0	0	Join_2_Value_0
 SELECT 'JOIN multiple clauses';
 JOIN multiple clauses
 SELECT t1.id, t1.value, t2.id, t2.value
-FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON t1.id = t2.id OR t1.id = t2.id;
+FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON t1.id = t2.id OR t1.id = t2.id ORDER BY ALL;
 0	Join_1_Value_0	0	Join_2_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1
 SELECT 'JOIN expression aliases';
 JOIN expression aliases
-SELECT t1_id, t1.value, t2_id, t2.value FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON (t1.id AS t1_id) = (t2.id AS t2_id);
+SELECT t1_id, t1.value, t2_id, t2.value FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON (t1.id AS t1_id) = (t2.id AS t2_id) ORDER BY ALL;
 0	Join_1_Value_0	0	Join_2_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1
 SELECT '--';
 --
-SELECT t1.id AS t1_id, t1.value, t2.id AS t2_id, t2.value FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON t1_id = t2_id;
+SELECT t1.id AS t1_id, t1.value, t2.id AS t2_id, t2.value FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON t1_id = t2_id ORDER BY ALL;
 0	Join_1_Value_0	0	Join_2_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1
 SELECT 'JOIN LEFT';
@@ -69,75 +69,75 @@ JOIN LEFT
 SELECT 'JOIN ON without conditions';
 JOIN ON without conditions
 SELECT test_table_join_1.id, test_table_join_1.value, test_table_join_2.id, test_table_join_2.value
-FROM test_table_join_1 LEFT JOIN test_table_join_2 ON test_table_join_1.id = test_table_join_2.id;
+FROM test_table_join_1 LEFT JOIN test_table_join_2 ON test_table_join_1.id = test_table_join_2.id ORDER BY ALL;
 0	Join_1_Value_0	0	Join_2_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1
 2	Join_1_Value_2	0	
 SELECT '--';
 --
 SELECT t1.id, t1.value, t2.id, t2.value
-FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 ON t1.id = t2.id;
+FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 ON t1.id = t2.id ORDER BY ALL;
 0	Join_1_Value_0	0	Join_2_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1
 2	Join_1_Value_2	0	
 SELECT '--';
 --
 SELECT t1.id, test_table_join_1.id, t1.value, test_table_join_1.value, t2.id, test_table_join_2.id, t2.value, test_table_join_2.value
-FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 ON t1.id = t2.id;
+FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 ON t1.id = t2.id ORDER BY ALL;
 0	0	Join_1_Value_0	Join_1_Value_0	0	0	Join_2_Value_0	Join_2_Value_0
 1	1	Join_1_Value_1	Join_1_Value_1	1	1	Join_2_Value_1	Join_2_Value_1
 2	2	Join_1_Value_2	Join_1_Value_2	0	0		
 SELECT '--';
 --
 SELECT t1.value, t2.value
-FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 ON t1.id = t2.id;
+FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 ON t1.id = t2.id ORDER BY ALL;
 Join_1_Value_0	Join_2_Value_0
 Join_1_Value_1	Join_2_Value_1
 Join_1_Value_2	
-SELECT id FROM test_table_join_1 LEFT JOIN test_table_join_2 ON test_table_join_1.id = test_table_join_2.id; -- { serverError AMBIGUOUS_IDENTIFIER }
-SELECT value FROM test_table_join_1 LEFT JOIN test_table_join_2 ON test_table_join_1.id = test_table_join_2.id; -- { serverError AMBIGUOUS_IDENTIFIER }
+SELECT id FROM test_table_join_1 LEFT JOIN test_table_join_2 ON test_table_join_1.id = test_table_join_2.id ORDER BY ALL; -- { serverError AMBIGUOUS_IDENTIFIER }
+SELECT value FROM test_table_join_1 LEFT JOIN test_table_join_2 ON test_table_join_1.id = test_table_join_2.id ORDER BY ALL; -- { serverError AMBIGUOUS_IDENTIFIER }
 SELECT 'JOIN ON with conditions';
 JOIN ON with conditions
 SELECT t1.id, t1.value, t2.id, t2.value
-FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t1.value = 'Join_1_Value_0';
+FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t1.value = 'Join_1_Value_0' ORDER BY ALL;
 0	Join_1_Value_0	0	Join_2_Value_0
 SELECT '--';
 --
 SELECT t1.id, t1.value, t2.id, t2.value
-FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t2.value = 'Join_2_Value_0';
+FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t2.value = 'Join_2_Value_0' ORDER BY ALL;
 0	Join_1_Value_0	0	Join_2_Value_0
 1	Join_1_Value_1	0	
 2	Join_1_Value_2	0	
 SELECT '--';
 --
 SELECT t1.id, t1.value, t2.id, t2.value
-FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t1.value = 'Join_1_Value_0' AND t2.value = 'Join_2_Value_0';
+FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t1.value = 'Join_1_Value_0' AND t2.value = 'Join_2_Value_0' ORDER BY ALL;
 0	Join_1_Value_0	0	Join_2_Value_0
 1	Join_1_Value_1	0	
 2	Join_1_Value_2	0	
 SELECT '--';
 --
 SELECT t1.id, t1.value, t2.id, t2.value
-FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 ON toString(t1.id) = toString(t2.id) AND t1.value = 'Join_1_Value_0' AND t2.value = 'Join_2_Value_0';
+FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 ON toString(t1.id) = toString(t2.id) AND t1.value = 'Join_1_Value_0' AND t2.value = 'Join_2_Value_0' ORDER BY ALL;
 0	Join_1_Value_0	0	Join_2_Value_0
 1	Join_1_Value_1	0	
 2	Join_1_Value_2	0	
 SELECT 'JOIN multiple clauses';
 JOIN multiple clauses
 SELECT t1.id, t1.value, t2.id, t2.value
-FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 ON t1.id = t2.id OR t1.id = t2.id;
+FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 ON t1.id = t2.id OR t1.id = t2.id ORDER BY ALL;
 0	Join_1_Value_0	0	Join_2_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1
 2	Join_1_Value_2	0	
 SELECT 'JOIN expression aliases';
 JOIN expression aliases
-SELECT t1_id, t1.value, t2_id, t2.value FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 ON (t1.id AS t1_id) = (t2.id AS t2_id);
+SELECT t1_id, t1.value, t2_id, t2.value FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 ON (t1.id AS t1_id) = (t2.id AS t2_id) ORDER BY ALL;
 0	Join_1_Value_0	0	Join_2_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1
 2	Join_1_Value_2	0	
 SELECT '--';
 --
-SELECT t1.id AS t1_id, t1.value, t2.id AS t2_id, t2.value FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 ON t1_id = t2_id;
+SELECT t1.id AS t1_id, t1.value, t2.id AS t2_id, t2.value FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 ON t1_id = t2_id ORDER BY ALL;
 0	Join_1_Value_0	0	Join_2_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1
 2	Join_1_Value_2	0	
@@ -146,182 +146,182 @@ JOIN RIGHT
 SELECT 'JOIN ON without conditions';
 JOIN ON without conditions
 SELECT test_table_join_1.id, test_table_join_1.value, test_table_join_2.id, test_table_join_2.value
-FROM test_table_join_1 RIGHT JOIN test_table_join_2 ON test_table_join_1.id = test_table_join_2.id;
+FROM test_table_join_1 RIGHT JOIN test_table_join_2 ON test_table_join_1.id = test_table_join_2.id ORDER BY ALL;
+0		3	Join_2_Value_3
 0	Join_1_Value_0	0	Join_2_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1
-0		3	Join_2_Value_3
 SELECT '--';
 --
 SELECT t1.id, t1.value, t2.id, t2.value
-FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 ON t1.id = t2.id;
+FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 ON t1.id = t2.id ORDER BY ALL;
+0		3	Join_2_Value_3
 0	Join_1_Value_0	0	Join_2_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1
-0		3	Join_2_Value_3
 SELECT '--';
 --
 SELECT t1.id, test_table_join_1.id, t1.value, test_table_join_1.value, t2.id, test_table_join_2.id, t2.value, test_table_join_2.value
-FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 ON t1.id = t2.id;
+FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 ON t1.id = t2.id ORDER BY ALL;
+0	0			3	3	Join_2_Value_3	Join_2_Value_3
 0	0	Join_1_Value_0	Join_1_Value_0	0	0	Join_2_Value_0	Join_2_Value_0
 1	1	Join_1_Value_1	Join_1_Value_1	1	1	Join_2_Value_1	Join_2_Value_1
-0	0			3	3	Join_2_Value_3	Join_2_Value_3
 SELECT '--';
 --
 SELECT t1.value, t2.value
-FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 ON t1.id = t2.id;
+FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 ON t1.id = t2.id ORDER BY ALL;
+	Join_2_Value_3
 Join_1_Value_0	Join_2_Value_0
 Join_1_Value_1	Join_2_Value_1
-	Join_2_Value_3
-SELECT id FROM test_table_join_1 RIGHT JOIN test_table_join_2 ON test_table_join_1.id = test_table_join_2.id; -- { serverError AMBIGUOUS_IDENTIFIER }
-SELECT value FROM test_table_join_1 RIGHT JOIN test_table_join_2 ON test_table_join_1.id = test_table_join_2.id; -- { serverError AMBIGUOUS_IDENTIFIER }
+SELECT id FROM test_table_join_1 RIGHT JOIN test_table_join_2 ON test_table_join_1.id = test_table_join_2.id ORDER BY ALL; -- { serverError AMBIGUOUS_IDENTIFIER }
+SELECT value FROM test_table_join_1 RIGHT JOIN test_table_join_2 ON test_table_join_1.id = test_table_join_2.id ORDER BY ALL; -- { serverError AMBIGUOUS_IDENTIFIER }
 SELECT 'JOIN ON with conditions';
 JOIN ON with conditions
 SELECT t1.id, t1.value, t2.id, t2.value
-FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t1.value = 'Join_1_Value_0';
+FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t1.value = 'Join_1_Value_0' ORDER BY ALL;
 0	Join_1_Value_0	0	Join_2_Value_0
 SELECT '--';
 --
 SELECT t1.id, t1.value, t2.id, t2.value
-FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t2.value = 'Join_2_Value_0';
-0	Join_1_Value_0	0	Join_2_Value_0
+FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t2.value = 'Join_2_Value_0' ORDER BY ALL;
 0		1	Join_2_Value_1
 0		3	Join_2_Value_3
+0	Join_1_Value_0	0	Join_2_Value_0
 SELECT '--';
 --
 SELECT t1.id, t1.value, t2.id, t2.value
-FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t1.value = 'Join_1_Value_0' AND t2.value = 'Join_2_Value_0';
-0	Join_1_Value_0	0	Join_2_Value_0
+FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t1.value = 'Join_1_Value_0' AND t2.value = 'Join_2_Value_0' ORDER BY ALL;
 0		1	Join_2_Value_1
 0		3	Join_2_Value_3
+0	Join_1_Value_0	0	Join_2_Value_0
 SELECT '--';
 --
 SELECT t1.id, t1.value, t2.id, t2.value
-FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 ON toString(t1.id) = toString(t2.id) AND t1.value = 'Join_1_Value_0' AND t2.value = 'Join_2_Value_0';
-0	Join_1_Value_0	0	Join_2_Value_0
+FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 ON toString(t1.id) = toString(t2.id) AND t1.value = 'Join_1_Value_0' AND t2.value = 'Join_2_Value_0' ORDER BY ALL;
 0		1	Join_2_Value_1
 0		3	Join_2_Value_3
+0	Join_1_Value_0	0	Join_2_Value_0
 SELECT 'JOIN multiple clauses';
 JOIN multiple clauses
 SELECT t1.id, t1.value, t2.id, t2.value
-FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 ON t1.id = t2.id OR t1.id = t2.id;
+FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 ON t1.id = t2.id OR t1.id = t2.id ORDER BY ALL;
+0		3	Join_2_Value_3
 0	Join_1_Value_0	0	Join_2_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1
-0		3	Join_2_Value_3
 SELECT 'JOIN expression aliases';
 JOIN expression aliases
-SELECT t1_id, t1.value, t2_id, t2.value FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 ON (t1.id AS t1_id) = (t2.id AS t2_id);
+SELECT t1_id, t1.value, t2_id, t2.value FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 ON (t1.id AS t1_id) = (t2.id AS t2_id) ORDER BY ALL;
+0		3	Join_2_Value_3
 0	Join_1_Value_0	0	Join_2_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1
-0		3	Join_2_Value_3
 SELECT '--';
 --
-SELECT t1.id AS t1_id, t1.value, t2.id AS t2_id, t2.value FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 ON t1_id = t2_id;
+SELECT t1.id AS t1_id, t1.value, t2.id AS t2_id, t2.value FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 ON t1_id = t2_id ORDER BY ALL;
+0		3	Join_2_Value_3
 0	Join_1_Value_0	0	Join_2_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1
-0		3	Join_2_Value_3
 SELECT 'JOIN FULL';
 JOIN FULL
 SELECT 'JOIN ON without conditions';
 JOIN ON without conditions
 SELECT test_table_join_1.id, test_table_join_1.value, test_table_join_2.id, test_table_join_2.value
-FROM test_table_join_1 FULL JOIN test_table_join_2 ON test_table_join_1.id = test_table_join_2.id;
+FROM test_table_join_1 FULL JOIN test_table_join_2 ON test_table_join_1.id = test_table_join_2.id ORDER BY ALL;
+0		3	Join_2_Value_3
 0	Join_1_Value_0	0	Join_2_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1
 2	Join_1_Value_2	0	
-0		3	Join_2_Value_3
 SELECT '--';
 --
 SELECT t1.id, t1.value, t2.id, t2.value
-FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 ON t1.id = t2.id;
+FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 ON t1.id = t2.id ORDER BY ALL;
+0		3	Join_2_Value_3
 0	Join_1_Value_0	0	Join_2_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1
 2	Join_1_Value_2	0	
-0		3	Join_2_Value_3
 SELECT '--';
 --
 SELECT t1.id, test_table_join_1.id, t1.value, test_table_join_1.value, t2.id, test_table_join_2.id, t2.value, test_table_join_2.value
-FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 ON t1.id = t2.id;
+FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 ON t1.id = t2.id ORDER BY ALL;
+0	0			3	3	Join_2_Value_3	Join_2_Value_3
 0	0	Join_1_Value_0	Join_1_Value_0	0	0	Join_2_Value_0	Join_2_Value_0
 1	1	Join_1_Value_1	Join_1_Value_1	1	1	Join_2_Value_1	Join_2_Value_1
 2	2	Join_1_Value_2	Join_1_Value_2	0	0		
-0	0			3	3	Join_2_Value_3	Join_2_Value_3
 SELECT '--';
 --
 SELECT t1.value, t2.value
-FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 ON t1.id = t2.id;
+FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 ON t1.id = t2.id ORDER BY ALL;
+	Join_2_Value_3
 Join_1_Value_0	Join_2_Value_0
 Join_1_Value_1	Join_2_Value_1
 Join_1_Value_2	
-	Join_2_Value_3
-SELECT id FROM test_table_join_1 FULL JOIN test_table_join_2 ON test_table_join_1.id = test_table_join_2.id; -- { serverError AMBIGUOUS_IDENTIFIER }
-SELECT value FROM test_table_join_1 FULL JOIN test_table_join_2 ON test_table_join_1.id = test_table_join_2.id; -- { serverError AMBIGUOUS_IDENTIFIER }
+SELECT id FROM test_table_join_1 FULL JOIN test_table_join_2 ON test_table_join_1.id = test_table_join_2.id ORDER BY ALL; -- { serverError AMBIGUOUS_IDENTIFIER }
+SELECT value FROM test_table_join_1 FULL JOIN test_table_join_2 ON test_table_join_1.id = test_table_join_2.id ORDER BY ALL; -- { serverError AMBIGUOUS_IDENTIFIER }
 SELECT 'JOIN ON with conditions';
 JOIN ON with conditions
 SELECT t1.id, t1.value, t2.id, t2.value
-FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t1.value = 'Join_1_Value_0';
+FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t1.value = 'Join_1_Value_0' ORDER BY ALL;
 0	Join_1_Value_0	0	Join_2_Value_0
 SELECT '--';
 --
 SELECT t1.id, t1.value, t2.id, t2.value
-FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t2.value = 'Join_2_Value_0';
+FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t2.value = 'Join_2_Value_0' ORDER BY ALL;
+0		1	Join_2_Value_1
+0		3	Join_2_Value_3
 0	Join_1_Value_0	0	Join_2_Value_0
 1	Join_1_Value_1	0	
 2	Join_1_Value_2	0	
-0		1	Join_2_Value_1
-0		3	Join_2_Value_3
 SELECT '--';
 --
 SELECT t1.id, t1.value, t2.id, t2.value
-FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t1.value = 'Join_1_Value_0' AND t2.value = 'Join_2_Value_0';
+FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t1.value = 'Join_1_Value_0' AND t2.value = 'Join_2_Value_0' ORDER BY ALL;
+0		1	Join_2_Value_1
+0		3	Join_2_Value_3
 0	Join_1_Value_0	0	Join_2_Value_0
 1	Join_1_Value_1	0	
 2	Join_1_Value_2	0	
-0		1	Join_2_Value_1
-0		3	Join_2_Value_3
 SELECT '--';
 --
 SELECT t1.id, t1.value, t2.id, t2.value
-FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 ON toString(t1.id) = toString(t2.id) AND t1.value = 'Join_1_Value_0' AND t2.value = 'Join_2_Value_0';
+FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 ON toString(t1.id) = toString(t2.id) AND t1.value = 'Join_1_Value_0' AND t2.value = 'Join_2_Value_0' ORDER BY ALL;
+0		1	Join_2_Value_1
+0		3	Join_2_Value_3
 0	Join_1_Value_0	0	Join_2_Value_0
 1	Join_1_Value_1	0	
 2	Join_1_Value_2	0	
-0		1	Join_2_Value_1
-0		3	Join_2_Value_3
 SELECT 'JOIN multiple clauses';
 JOIN multiple clauses
 SELECT t1.id, t1.value, t2.id, t2.value
-FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 ON t1.id = t2.id OR t1.id = t2.id;
+FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 ON t1.id = t2.id OR t1.id = t2.id ORDER BY ALL;
+0		3	Join_2_Value_3
 0	Join_1_Value_0	0	Join_2_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1
 2	Join_1_Value_2	0	
-0		3	Join_2_Value_3
 SELECT 'JOIN expression aliases';
 JOIN expression aliases
-SELECT t1_id, t1.value, t2_id, t2.value FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 ON (t1.id AS t1_id) = (t2.id AS t2_id);
+SELECT t1_id, t1.value, t2_id, t2.value FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 ON (t1.id AS t1_id) = (t2.id AS t2_id) ORDER BY ALL;
+0		3	Join_2_Value_3
 0	Join_1_Value_0	0	Join_2_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1
 2	Join_1_Value_2	0	
-0		3	Join_2_Value_3
 SELECT '--';
 --
-SELECT t1.id AS t1_id, t1.value, t2.id AS t2_id, t2.value FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 ON t1_id = t2_id;
+SELECT t1.id AS t1_id, t1.value, t2.id AS t2_id, t2.value FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 ON t1_id = t2_id ORDER BY ALL;
+0		3	Join_2_Value_3
 0	Join_1_Value_0	0	Join_2_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1
 2	Join_1_Value_2	0	
-0		3	Join_2_Value_3
 SELECT 'First JOIN INNER second JOIN INNER';
 First JOIN INNER second JOIN INNER
 SELECT 'JOIN ON without conditions';
 JOIN ON without conditions
 SELECT test_table_join_1.id, test_table_join_1.value, test_table_join_2.id, test_table_join_2.value, test_table_join_3.id, test_table_join_3.value
 FROM test_table_join_1 INNER JOIN test_table_join_2 ON test_table_join_1.id = test_table_join_2.id
-INNER JOIN test_table_join_3 ON test_table_join_2.id = test_table_join_3.id;
+INNER JOIN test_table_join_3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
 SELECT '--';
 --
 SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value
 FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON test_table_join_1.id = test_table_join_2.id
-INNER JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id;
+INNER JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
 SELECT '--';
@@ -329,48 +329,48 @@ SELECT '--';
 SELECT t1.id, test_table_join_1.id, t1.value, test_table_join_1.value, t2.id, test_table_join_2.id, t2.value, test_table_join_2.value,
 t3.id, test_table_join_3.id, t3.value, test_table_join_3.value
 FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON test_table_join_1.id = test_table_join_2.id
-INNER JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id;
+INNER JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
 0	0	Join_1_Value_0	Join_1_Value_0	0	0	Join_2_Value_0	Join_2_Value_0	0	0	Join_3_Value_0	Join_3_Value_0
 1	1	Join_1_Value_1	Join_1_Value_1	1	1	Join_2_Value_1	Join_2_Value_1	1	1	Join_3_Value_1	Join_3_Value_1
 SELECT '--';
 --
 SELECT t1.value, t2.value, t3.value
 FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON test_table_join_1.id = test_table_join_2.id
-INNER JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id;
+INNER JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
 Join_1_Value_0	Join_2_Value_0	Join_3_Value_0
 Join_1_Value_1	Join_2_Value_1	Join_3_Value_1
 SELECT 'JOIN ON with conditions';
 JOIN ON with conditions
 SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value
 FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t1.value == 'Join_1_Value_0'
-INNER JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0';
+INNER JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0' ORDER BY ALL;
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 SELECT '--';
 --
 SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value
 FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t1.value == 'Join_1_Value_0' AND t2.value == 'Join_2_Value_0'
-INNER JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0' AND t3.value == 'Join_3_Value_0';
+INNER JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0' AND t3.value == 'Join_3_Value_0' ORDER BY ALL;
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 SELECT 'JOIN multiple clauses';
 JOIN multiple clauses
 SELECT t1.value, t2.value, t3.value
 FROM test_table_join_1 AS t1
 INNER JOIN test_table_join_2 AS t2 ON t1.id = t2.id OR t1.id = t2.id
-INNER JOIN test_table_join_3 AS t3 ON t2.id = t3.id OR t3.id = t2.id;
+INNER JOIN test_table_join_3 AS t3 ON t2.id = t3.id OR t3.id = t2.id ORDER BY ALL;
 Join_1_Value_0	Join_2_Value_0	Join_3_Value_0
 Join_1_Value_1	Join_2_Value_1	Join_3_Value_1
 SELECT 'JOIN expression aliases';
 JOIN expression aliases
 SELECT t1_id, t1.value, t2_id, t2.value, t3_id, t3.value
 FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON (t1.id AS t1_id) = (t2.id AS t2_id)
-INNER JOIN test_table_join_3 AS t3 ON t2_id = (t3.id AS t3_id);
+INNER JOIN test_table_join_3 AS t3 ON t2_id = (t3.id AS t3_id) ORDER BY ALL;
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
 SELECT '--';
 --
 SELECT t1.id AS t1_id, t1.value, t2.id AS t2_id, t2.value, t3.id AS t3_id, t3.value
 FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON t1_id = t2_id
-INNER JOIN test_table_join_3 AS t3 ON t2_id = t3_id;
+INNER JOIN test_table_join_3 AS t3 ON t2_id = t3_id ORDER BY ALL;
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
 SELECT 'First JOIN INNER second JOIN LEFT';
@@ -379,14 +379,14 @@ SELECT 'JOIN ON without conditions';
 JOIN ON without conditions
 SELECT test_table_join_1.id, test_table_join_1.value, test_table_join_2.id, test_table_join_2.value, test_table_join_3.id, test_table_join_3.value
 FROM test_table_join_1 INNER JOIN test_table_join_2 ON test_table_join_1.id = test_table_join_2.id
-LEFT JOIN test_table_join_3 ON test_table_join_2.id = test_table_join_3.id;
+LEFT JOIN test_table_join_3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
 SELECT '--';
 --
 SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value
 FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON test_table_join_1.id = test_table_join_2.id
-LEFT JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id;
+LEFT JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
 SELECT '--';
@@ -394,48 +394,48 @@ SELECT '--';
 SELECT t1.id, test_table_join_1.id, t1.value, test_table_join_1.value, t2.id, test_table_join_2.id, t2.value, test_table_join_2.value,
 t3.id, test_table_join_3.id, t3.value, test_table_join_3.value
 FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON test_table_join_1.id = test_table_join_2.id
-LEFT JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id;
+LEFT JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
 0	0	Join_1_Value_0	Join_1_Value_0	0	0	Join_2_Value_0	Join_2_Value_0	0	0	Join_3_Value_0	Join_3_Value_0
 1	1	Join_1_Value_1	Join_1_Value_1	1	1	Join_2_Value_1	Join_2_Value_1	1	1	Join_3_Value_1	Join_3_Value_1
 SELECT '--';
 --
 SELECT t1.value, t2.value, t3.value
 FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON test_table_join_1.id = test_table_join_2.id
-LEFT JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id;
+LEFT JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
 Join_1_Value_0	Join_2_Value_0	Join_3_Value_0
 Join_1_Value_1	Join_2_Value_1	Join_3_Value_1
 SELECT 'JOIN ON with conditions';
 JOIN ON with conditions
 SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value
 FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t1.value == 'Join_1_Value_0'
-LEFT JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0';
+LEFT JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0' ORDER BY ALL;
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 SELECT '--';
 --
 SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value
 FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t1.value == 'Join_1_Value_0' AND t2.value == 'Join_2_Value_0'
-LEFT JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0' AND t3.value == 'Join_3_Value_0';
+LEFT JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0' AND t3.value == 'Join_3_Value_0' ORDER BY ALL;
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 SELECT 'JOIN multiple clauses';
 JOIN multiple clauses
 SELECT t1.value, t2.value, t3.value
 FROM test_table_join_1 AS t1
 INNER JOIN test_table_join_2 AS t2 ON t1.id = t2.id OR t1.id = t2.id
-LEFT JOIN test_table_join_3 AS t3 ON t2.id = t3.id OR t3.id = t2.id;
+LEFT JOIN test_table_join_3 AS t3 ON t2.id = t3.id OR t3.id = t2.id ORDER BY ALL;
 Join_1_Value_0	Join_2_Value_0	Join_3_Value_0
 Join_1_Value_1	Join_2_Value_1	Join_3_Value_1
 SELECT 'JOIN expression aliases';
 JOIN expression aliases
 SELECT t1_id, t1.value, t2_id, t2.value, t3_id, t3.value
 FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON (t1.id AS t1_id) = (t2.id AS t2_id)
-LEFT JOIN test_table_join_3 AS t3 ON t2_id = (t3.id AS t3_id);
+LEFT JOIN test_table_join_3 AS t3 ON t2_id = (t3.id AS t3_id) ORDER BY ALL;
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
 SELECT '--';
 --
 SELECT t1.id AS t1_id, t1.value, t2.id AS t2_id, t2.value, t3.id AS t3_id, t3.value
 FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON t1_id = t2_id
-LEFT JOIN test_table_join_3 AS t3 ON t2_id = t3_id;
+LEFT JOIN test_table_join_3 AS t3 ON t2_id = t3_id ORDER BY ALL;
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
 SELECT 'First JOIN INNER second JOIN RIGHT';
@@ -444,159 +444,159 @@ SELECT 'JOIN ON without conditions';
 JOIN ON without conditions
 SELECT test_table_join_1.id, test_table_join_1.value, test_table_join_2.id, test_table_join_2.value, test_table_join_3.id, test_table_join_3.value
 FROM test_table_join_1 INNER JOIN test_table_join_2 ON test_table_join_1.id = test_table_join_2.id
-RIGHT JOIN test_table_join_3 ON test_table_join_2.id = test_table_join_3.id;
+RIGHT JOIN test_table_join_3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
+0		0		4	Join_3_Value_4
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
-0		0		4	Join_3_Value_4
 SELECT '--';
 --
 SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value
 FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON test_table_join_1.id = test_table_join_2.id
-RIGHT JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id;
+RIGHT JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
+0		0		4	Join_3_Value_4
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
-0		0		4	Join_3_Value_4
 SELECT '--';
 --
 SELECT t1.id, test_table_join_1.id, t1.value, test_table_join_1.value, t2.id, test_table_join_2.id, t2.value, test_table_join_2.value,
 t3.id, test_table_join_3.id, t3.value, test_table_join_3.value
 FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON test_table_join_1.id = test_table_join_2.id
-RIGHT JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id;
+RIGHT JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
+0	0			0	0			4	4	Join_3_Value_4	Join_3_Value_4
 0	0	Join_1_Value_0	Join_1_Value_0	0	0	Join_2_Value_0	Join_2_Value_0	0	0	Join_3_Value_0	Join_3_Value_0
 1	1	Join_1_Value_1	Join_1_Value_1	1	1	Join_2_Value_1	Join_2_Value_1	1	1	Join_3_Value_1	Join_3_Value_1
-0	0			0	0			4	4	Join_3_Value_4	Join_3_Value_4
 SELECT '--';
 --
 SELECT t1.value, t2.value, t3.value
 FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON test_table_join_1.id = test_table_join_2.id
-RIGHT JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id;
+RIGHT JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
+		Join_3_Value_4
 Join_1_Value_0	Join_2_Value_0	Join_3_Value_0
 Join_1_Value_1	Join_2_Value_1	Join_3_Value_1
-		Join_3_Value_4
 SELECT 'JOIN ON with conditions';
 JOIN ON with conditions
 SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value
 FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t1.value == 'Join_1_Value_0'
-RIGHT JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0';
-0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
-0		0		4	Join_3_Value_4
+RIGHT JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0' ORDER BY ALL;
 0		0		1	Join_3_Value_1
+0		0		4	Join_3_Value_4
+0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 SELECT '--';
 --
 SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value
 FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t1.value == 'Join_1_Value_0' AND t2.value == 'Join_2_Value_0'
-RIGHT JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0' AND t3.value == 'Join_3_Value_0';
-0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
+RIGHT JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0' AND t3.value == 'Join_3_Value_0' ORDER BY ALL;
 0		0		1	Join_3_Value_1
 0		0		4	Join_3_Value_4
+0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 SELECT 'JOIN multiple clauses';
 JOIN multiple clauses
 SELECT t1.value, t2.value, t3.value
 FROM test_table_join_1 AS t1
 INNER JOIN test_table_join_2 AS t2 ON t1.id = t2.id OR t1.id = t2.id
-RIGHT JOIN test_table_join_3 AS t3 ON t2.id = t3.id OR t3.id = t2.id;
+RIGHT JOIN test_table_join_3 AS t3 ON t2.id = t3.id OR t3.id = t2.id ORDER BY ALL;
+		Join_3_Value_4
 Join_1_Value_0	Join_2_Value_0	Join_3_Value_0
 Join_1_Value_1	Join_2_Value_1	Join_3_Value_1
-		Join_3_Value_4
 SELECT 'JOIN expression aliases';
 JOIN expression aliases
 SELECT t1_id, t1.value, t2_id, t2.value, t3_id, t3.value
 FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON (t1.id AS t1_id) = (t2.id AS t2_id)
-RIGHT JOIN test_table_join_3 AS t3 ON t2_id = (t3.id AS t3_id);
+RIGHT JOIN test_table_join_3 AS t3 ON t2_id = (t3.id AS t3_id) ORDER BY ALL;
+0		0		4	Join_3_Value_4
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
-0		0		4	Join_3_Value_4
 SELECT '--';
 --
 SELECT t1.id AS t1_id, t1.value, t2.id AS t2_id, t2.value, t3.id AS t3_id, t3.value
 FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON t1_id = t2_id
-RIGHT JOIN test_table_join_3 AS t3 ON t2_id = t3_id;
+RIGHT JOIN test_table_join_3 AS t3 ON t2_id = t3_id ORDER BY ALL;
+0		0		4	Join_3_Value_4
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
-0		0		4	Join_3_Value_4
 SELECT 'First JOIN INNER second JOIN FULL';
 First JOIN INNER second JOIN FULL
 SELECT 'JOIN ON without conditions';
 JOIN ON without conditions
 SELECT test_table_join_1.id, test_table_join_1.value, test_table_join_2.id, test_table_join_2.value, test_table_join_3.id, test_table_join_3.value
 FROM test_table_join_1 INNER JOIN test_table_join_2 ON test_table_join_1.id = test_table_join_2.id
-FULL JOIN test_table_join_3 ON test_table_join_2.id = test_table_join_3.id;
+FULL JOIN test_table_join_3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
+0		0		4	Join_3_Value_4
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
-0		0		4	Join_3_Value_4
 SELECT '--';
 --
 SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value
 FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON test_table_join_1.id = test_table_join_2.id
-FULL JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id;
+FULL JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
+0		0		4	Join_3_Value_4
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
-0		0		4	Join_3_Value_4
 SELECT '--';
 --
 SELECT t1.id, test_table_join_1.id, t1.value, test_table_join_1.value, t2.id, test_table_join_2.id, t2.value, test_table_join_2.value,
 t3.id, test_table_join_3.id, t3.value, test_table_join_3.value
 FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON test_table_join_1.id = test_table_join_2.id
-FULL JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id;
+FULL JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
+0	0			0	0			4	4	Join_3_Value_4	Join_3_Value_4
 0	0	Join_1_Value_0	Join_1_Value_0	0	0	Join_2_Value_0	Join_2_Value_0	0	0	Join_3_Value_0	Join_3_Value_0
 1	1	Join_1_Value_1	Join_1_Value_1	1	1	Join_2_Value_1	Join_2_Value_1	1	1	Join_3_Value_1	Join_3_Value_1
-0	0			0	0			4	4	Join_3_Value_4	Join_3_Value_4
 SELECT '--';
 --
 SELECT t1.value, t2.value, t3.value
 FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON test_table_join_1.id = test_table_join_2.id
-FULL JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id;
+FULL JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
+		Join_3_Value_4
 Join_1_Value_0	Join_2_Value_0	Join_3_Value_0
 Join_1_Value_1	Join_2_Value_1	Join_3_Value_1
-		Join_3_Value_4
 SELECT 'JOIN ON with conditions';
 JOIN ON with conditions
 SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value
 FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t1.value == 'Join_1_Value_0'
-FULL JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0';
-0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
-0		0		4	Join_3_Value_4
+FULL JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0' ORDER BY ALL;
 0		0		1	Join_3_Value_1
+0		0		4	Join_3_Value_4
+0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 SELECT '--';
 --
 SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value
 FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t1.value == 'Join_1_Value_0' AND t2.value == 'Join_2_Value_0'
-FULL JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0' AND t3.value == 'Join_3_Value_0';
-0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
+FULL JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0' AND t3.value == 'Join_3_Value_0' ORDER BY ALL;
 0		0		1	Join_3_Value_1
 0		0		4	Join_3_Value_4
+0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 SELECT 'JOIN multiple clauses';
 JOIN multiple clauses
 SELECT t1.value, t2.value, t3.value
 FROM test_table_join_1 AS t1
 INNER JOIN test_table_join_2 AS t2 ON t1.id = t2.id OR t1.id = t2.id
-FULL JOIN test_table_join_3 AS t3 ON t2.id = t3.id OR t3.id = t2.id;
+FULL JOIN test_table_join_3 AS t3 ON t2.id = t3.id OR t3.id = t2.id ORDER BY ALL;
+		Join_3_Value_4
 Join_1_Value_0	Join_2_Value_0	Join_3_Value_0
 Join_1_Value_1	Join_2_Value_1	Join_3_Value_1
-		Join_3_Value_4
 SELECT 'JOIN expression aliases';
 JOIN expression aliases
 SELECT t1_id, t1.value, t2_id, t2.value, t3_id, t3.value
 FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON (t1.id AS t1_id) = (t2.id AS t2_id)
-FULL JOIN test_table_join_3 AS t3 ON t2_id = (t3.id AS t3_id);
+FULL JOIN test_table_join_3 AS t3 ON t2_id = (t3.id AS t3_id) ORDER BY ALL;
+0		0		4	Join_3_Value_4
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
-0		0		4	Join_3_Value_4
 SELECT '--';
 --
 SELECT t1.id AS t1_id, t1.value, t2.id AS t2_id, t2.value, t3.id AS t3_id, t3.value
 FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON t1_id = t2_id
-FULL JOIN test_table_join_3 AS t3 ON t2_id = t3_id;
+FULL JOIN test_table_join_3 AS t3 ON t2_id = t3_id ORDER BY ALL;
+0		0		4	Join_3_Value_4
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
-0		0		4	Join_3_Value_4
 SELECT 'First JOIN LEFT second JOIN INNER';
 First JOIN LEFT second JOIN INNER
 SELECT 'JOIN ON without conditions';
 JOIN ON without conditions
 SELECT test_table_join_1.id, test_table_join_1.value, test_table_join_2.id, test_table_join_2.value, test_table_join_3.id, test_table_join_3.value
 FROM test_table_join_1 LEFT JOIN test_table_join_2 ON test_table_join_1.id = test_table_join_2.id
-INNER JOIN test_table_join_3 ON test_table_join_2.id = test_table_join_3.id;
+INNER JOIN test_table_join_3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
 2	Join_1_Value_2	0		0	Join_3_Value_0
@@ -604,7 +604,7 @@ SELECT '--';
 --
 SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value
 FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 ON test_table_join_1.id = test_table_join_2.id
-INNER JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id;
+INNER JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
 2	Join_1_Value_2	0		0	Join_3_Value_0
@@ -613,7 +613,7 @@ SELECT '--';
 SELECT t1.id, test_table_join_1.id, t1.value, test_table_join_1.value, t2.id, test_table_join_2.id, t2.value, test_table_join_2.value,
 t3.id, test_table_join_3.id, t3.value, test_table_join_3.value
 FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 ON test_table_join_1.id = test_table_join_2.id
-INNER JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id;
+INNER JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
 0	0	Join_1_Value_0	Join_1_Value_0	0	0	Join_2_Value_0	Join_2_Value_0	0	0	Join_3_Value_0	Join_3_Value_0
 1	1	Join_1_Value_1	Join_1_Value_1	1	1	Join_2_Value_1	Join_2_Value_1	1	1	Join_3_Value_1	Join_3_Value_1
 2	2	Join_1_Value_2	Join_1_Value_2	0	0			0	0	Join_3_Value_0	Join_3_Value_0
@@ -621,7 +621,7 @@ SELECT '--';
 --
 SELECT t1.value, t2.value, t3.value
 FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 ON test_table_join_1.id = test_table_join_2.id
-INNER JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id;
+INNER JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
 Join_1_Value_0	Join_2_Value_0	Join_3_Value_0
 Join_1_Value_1	Join_2_Value_1	Join_3_Value_1
 Join_1_Value_2		Join_3_Value_0
@@ -629,20 +629,20 @@ SELECT 'JOIN ON with conditions';
 JOIN ON with conditions
 SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value
 FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t1.value == 'Join_1_Value_0'
-INNER JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0';
+INNER JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0' ORDER BY ALL;
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 SELECT '--';
 --
 SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value
 FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t1.value == 'Join_1_Value_0' AND t2.value == 'Join_2_Value_0'
-INNER JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0' AND t3.value == 'Join_3_Value_0';
+INNER JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0' AND t3.value == 'Join_3_Value_0' ORDER BY ALL;
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 SELECT 'JOIN multiple clauses';
 JOIN multiple clauses
 SELECT t1.value, t2.value, t3.value
 FROM test_table_join_1 AS t1
 LEFT JOIN test_table_join_2 AS t2 ON t1.id = t2.id OR t1.id = t2.id
-INNER JOIN test_table_join_3 AS t3 ON t2.id = t3.id OR t3.id = t2.id;
+INNER JOIN test_table_join_3 AS t3 ON t2.id = t3.id OR t3.id = t2.id ORDER BY ALL;
 Join_1_Value_0	Join_2_Value_0	Join_3_Value_0
 Join_1_Value_1	Join_2_Value_1	Join_3_Value_1
 Join_1_Value_2		Join_3_Value_0
@@ -650,7 +650,7 @@ SELECT 'JOIN expression aliases';
 JOIN expression aliases
 SELECT t1_id, t1.value, t2_id, t2.value, t3_id, t3.value
 FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 ON (t1.id AS t1_id) = (t2.id AS t2_id)
-INNER JOIN test_table_join_3 AS t3 ON t2_id = (t3.id AS t3_id);
+INNER JOIN test_table_join_3 AS t3 ON t2_id = (t3.id AS t3_id) ORDER BY ALL;
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
 2	Join_1_Value_2	0		0	Join_3_Value_0
@@ -658,7 +658,7 @@ SELECT '--';
 --
 SELECT t1.id AS t1_id, t1.value, t2.id AS t2_id, t2.value, t3.id AS t3_id, t3.value
 FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 ON t1_id = t2_id
-INNER JOIN test_table_join_3 AS t3 ON t2_id = t3_id;
+INNER JOIN test_table_join_3 AS t3 ON t2_id = t3_id ORDER BY ALL;
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
 2	Join_1_Value_2	0		0	Join_3_Value_0
@@ -668,7 +668,7 @@ SELECT 'JOIN ON without conditions';
 JOIN ON without conditions
 SELECT test_table_join_1.id, test_table_join_1.value, test_table_join_2.id, test_table_join_2.value, test_table_join_3.id, test_table_join_3.value
 FROM test_table_join_1 LEFT JOIN test_table_join_2 ON test_table_join_1.id = test_table_join_2.id
-LEFT JOIN test_table_join_3 ON test_table_join_2.id = test_table_join_3.id;
+LEFT JOIN test_table_join_3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
 2	Join_1_Value_2	0		0	Join_3_Value_0
@@ -676,7 +676,7 @@ SELECT '--';
 --
 SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value
 FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 ON test_table_join_1.id = test_table_join_2.id
-LEFT JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id;
+LEFT JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
 2	Join_1_Value_2	0		0	Join_3_Value_0
@@ -685,7 +685,7 @@ SELECT '--';
 SELECT t1.id, test_table_join_1.id, t1.value, test_table_join_1.value, t2.id, test_table_join_2.id, t2.value, test_table_join_2.value,
 t3.id, test_table_join_3.id, t3.value, test_table_join_3.value
 FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 ON test_table_join_1.id = test_table_join_2.id
-LEFT JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id;
+LEFT JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
 0	0	Join_1_Value_0	Join_1_Value_0	0	0	Join_2_Value_0	Join_2_Value_0	0	0	Join_3_Value_0	Join_3_Value_0
 1	1	Join_1_Value_1	Join_1_Value_1	1	1	Join_2_Value_1	Join_2_Value_1	1	1	Join_3_Value_1	Join_3_Value_1
 2	2	Join_1_Value_2	Join_1_Value_2	0	0			0	0	Join_3_Value_0	Join_3_Value_0
@@ -693,7 +693,7 @@ SELECT '--';
 --
 SELECT t1.value, t2.value, t3.value
 FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 ON test_table_join_1.id = test_table_join_2.id
-LEFT JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id;
+LEFT JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
 Join_1_Value_0	Join_2_Value_0	Join_3_Value_0
 Join_1_Value_1	Join_2_Value_1	Join_3_Value_1
 Join_1_Value_2		Join_3_Value_0
@@ -701,7 +701,7 @@ SELECT 'JOIN ON with conditions';
 JOIN ON with conditions
 SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value
 FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t1.value == 'Join_1_Value_0'
-LEFT JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0';
+LEFT JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0' ORDER BY ALL;
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	0		0	
 2	Join_1_Value_2	0		0	
@@ -709,7 +709,7 @@ SELECT '--';
 --
 SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value
 FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t1.value == 'Join_1_Value_0' AND t2.value == 'Join_2_Value_0'
-LEFT JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0' AND t3.value == 'Join_3_Value_0';
+LEFT JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0' AND t3.value == 'Join_3_Value_0' ORDER BY ALL;
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	0		0	
 2	Join_1_Value_2	0		0	
@@ -718,7 +718,7 @@ JOIN multiple clauses
 SELECT t1.value, t2.value, t3.value
 FROM test_table_join_1 AS t1
 LEFT JOIN test_table_join_2 AS t2 ON t1.id = t2.id OR t1.id = t2.id
-LEFT JOIN test_table_join_3 AS t3 ON t2.id = t3.id OR t3.id = t2.id;
+LEFT JOIN test_table_join_3 AS t3 ON t2.id = t3.id OR t3.id = t2.id ORDER BY ALL;
 Join_1_Value_0	Join_2_Value_0	Join_3_Value_0
 Join_1_Value_1	Join_2_Value_1	Join_3_Value_1
 Join_1_Value_2		Join_3_Value_0
@@ -726,7 +726,7 @@ SELECT 'JOIN expression aliases';
 JOIN expression aliases
 SELECT t1_id, t1.value, t2_id, t2.value, t3_id, t3.value
 FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 ON (t1.id AS t1_id) = (t2.id AS t2_id)
-LEFT JOIN test_table_join_3 AS t3 ON t2_id = (t3.id AS t3_id);
+LEFT JOIN test_table_join_3 AS t3 ON t2_id = (t3.id AS t3_id) ORDER BY ALL;
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
 2	Join_1_Value_2	0		0	Join_3_Value_0
@@ -734,7 +734,7 @@ SELECT '--';
 --
 SELECT t1.id AS t1_id, t1.value, t2.id AS t2_id, t2.value, t3.id AS t3_id, t3.value
 FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 ON t1_id = t2_id
-LEFT JOIN test_table_join_3 AS t3 ON t2_id = t3_id;
+LEFT JOIN test_table_join_3 AS t3 ON t2_id = t3_id ORDER BY ALL;
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
 2	Join_1_Value_2	0		0	Join_3_Value_0
@@ -744,184 +744,184 @@ SELECT 'JOIN ON without conditions';
 JOIN ON without conditions
 SELECT test_table_join_1.id, test_table_join_1.value, test_table_join_2.id, test_table_join_2.value, test_table_join_3.id, test_table_join_3.value
 FROM test_table_join_1 LEFT JOIN test_table_join_2 ON test_table_join_1.id = test_table_join_2.id
-RIGHT JOIN test_table_join_3 ON test_table_join_2.id = test_table_join_3.id;
+RIGHT JOIN test_table_join_3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
+0		0		4	Join_3_Value_4
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
 2	Join_1_Value_2	0		0	Join_3_Value_0
-0		0		4	Join_3_Value_4
 SELECT '--';
 --
 SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value
 FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 ON test_table_join_1.id = test_table_join_2.id
-RIGHT JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id;
+RIGHT JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
+0		0		4	Join_3_Value_4
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
 2	Join_1_Value_2	0		0	Join_3_Value_0
-0		0		4	Join_3_Value_4
 SELECT '--';
 --
 SELECT t1.id, test_table_join_1.id, t1.value, test_table_join_1.value, t2.id, test_table_join_2.id, t2.value, test_table_join_2.value,
 t3.id, test_table_join_3.id, t3.value, test_table_join_3.value
 FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 ON test_table_join_1.id = test_table_join_2.id
-RIGHT JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id;
+RIGHT JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
+0	0			0	0			4	4	Join_3_Value_4	Join_3_Value_4
 0	0	Join_1_Value_0	Join_1_Value_0	0	0	Join_2_Value_0	Join_2_Value_0	0	0	Join_3_Value_0	Join_3_Value_0
 1	1	Join_1_Value_1	Join_1_Value_1	1	1	Join_2_Value_1	Join_2_Value_1	1	1	Join_3_Value_1	Join_3_Value_1
 2	2	Join_1_Value_2	Join_1_Value_2	0	0			0	0	Join_3_Value_0	Join_3_Value_0
-0	0			0	0			4	4	Join_3_Value_4	Join_3_Value_4
 SELECT '--';
 --
 SELECT t1.value, t2.value, t3.value
 FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 ON test_table_join_1.id = test_table_join_2.id
-RIGHT JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id;
+RIGHT JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
+		Join_3_Value_4
 Join_1_Value_0	Join_2_Value_0	Join_3_Value_0
 Join_1_Value_1	Join_2_Value_1	Join_3_Value_1
 Join_1_Value_2		Join_3_Value_0
-		Join_3_Value_4
 SELECT 'JOIN ON with conditions';
 JOIN ON with conditions
 SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value
 FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t1.value == 'Join_1_Value_0'
-RIGHT JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0';
-0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
-0		0		4	Join_3_Value_4
+RIGHT JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0' ORDER BY ALL;
 0		0		1	Join_3_Value_1
+0		0		4	Join_3_Value_4
+0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 SELECT '--';
 --
 SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value
 FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t1.value == 'Join_1_Value_0' AND t2.value == 'Join_2_Value_0'
-RIGHT JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0' AND t3.value == 'Join_3_Value_0';
-0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
+RIGHT JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0' AND t3.value == 'Join_3_Value_0' ORDER BY ALL;
 0		0		1	Join_3_Value_1
 0		0		4	Join_3_Value_4
+0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 SELECT 'JOIN multiple clauses';
 JOIN multiple clauses
 SELECT t1.value, t2.value, t3.value
 FROM test_table_join_1 AS t1
 LEFT JOIN test_table_join_2 AS t2 ON t1.id = t2.id OR t1.id = t2.id
-RIGHT JOIN test_table_join_3 AS t3 ON t2.id = t3.id OR t3.id = t2.id;
+RIGHT JOIN test_table_join_3 AS t3 ON t2.id = t3.id OR t3.id = t2.id ORDER BY ALL;
+		Join_3_Value_4
 Join_1_Value_0	Join_2_Value_0	Join_3_Value_0
 Join_1_Value_1	Join_2_Value_1	Join_3_Value_1
 Join_1_Value_2		Join_3_Value_0
-		Join_3_Value_4
 SELECT 'JOIN expression aliases';
 JOIN expression aliases
 SELECT t1_id, t1.value, t2_id, t2.value, t3_id, t3.value
 FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 ON (t1.id AS t1_id) = (t2.id AS t2_id)
-RIGHT JOIN test_table_join_3 AS t3 ON t2_id = (t3.id AS t3_id);
+RIGHT JOIN test_table_join_3 AS t3 ON t2_id = (t3.id AS t3_id) ORDER BY ALL;
+0		0		4	Join_3_Value_4
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
 2	Join_1_Value_2	0		0	Join_3_Value_0
-0		0		4	Join_3_Value_4
 SELECT '--';
 --
 SELECT t1.id AS t1_id, t1.value, t2.id AS t2_id, t2.value, t3.id AS t3_id, t3.value
 FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 ON t1_id = t2_id
-RIGHT JOIN test_table_join_3 AS t3 ON t2_id = t3_id;
+RIGHT JOIN test_table_join_3 AS t3 ON t2_id = t3_id ORDER BY ALL;
+0		0		4	Join_3_Value_4
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
 2	Join_1_Value_2	0		0	Join_3_Value_0
-0		0		4	Join_3_Value_4
 SELECT 'First JOIN LEFT second JOIN FULL';
 First JOIN LEFT second JOIN FULL
 SELECT 'JOIN ON without conditions';
 JOIN ON without conditions
 SELECT test_table_join_1.id, test_table_join_1.value, test_table_join_2.id, test_table_join_2.value, test_table_join_3.id, test_table_join_3.value
 FROM test_table_join_1 LEFT JOIN test_table_join_2 ON test_table_join_1.id = test_table_join_2.id
-FULL JOIN test_table_join_3 ON test_table_join_2.id = test_table_join_3.id;
+FULL JOIN test_table_join_3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
+0		0		4	Join_3_Value_4
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
 2	Join_1_Value_2	0		0	Join_3_Value_0
-0		0		4	Join_3_Value_4
 SELECT '--';
 --
 SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value
 FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 ON test_table_join_1.id = test_table_join_2.id
-FULL JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id;
+FULL JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
+0		0		4	Join_3_Value_4
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
 2	Join_1_Value_2	0		0	Join_3_Value_0
-0		0		4	Join_3_Value_4
 SELECT '--';
 --
 SELECT t1.id, test_table_join_1.id, t1.value, test_table_join_1.value, t2.id, test_table_join_2.id, t2.value, test_table_join_2.value,
 t3.id, test_table_join_3.id, t3.value, test_table_join_3.value
 FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 ON test_table_join_1.id = test_table_join_2.id
-FULL JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id;
+FULL JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
+0	0			0	0			4	4	Join_3_Value_4	Join_3_Value_4
 0	0	Join_1_Value_0	Join_1_Value_0	0	0	Join_2_Value_0	Join_2_Value_0	0	0	Join_3_Value_0	Join_3_Value_0
 1	1	Join_1_Value_1	Join_1_Value_1	1	1	Join_2_Value_1	Join_2_Value_1	1	1	Join_3_Value_1	Join_3_Value_1
 2	2	Join_1_Value_2	Join_1_Value_2	0	0			0	0	Join_3_Value_0	Join_3_Value_0
-0	0			0	0			4	4	Join_3_Value_4	Join_3_Value_4
 SELECT '--';
 --
 SELECT t1.value, t2.value, t3.value
 FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 ON test_table_join_1.id = test_table_join_2.id
-FULL JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id;
+FULL JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
+		Join_3_Value_4
 Join_1_Value_0	Join_2_Value_0	Join_3_Value_0
 Join_1_Value_1	Join_2_Value_1	Join_3_Value_1
 Join_1_Value_2		Join_3_Value_0
-		Join_3_Value_4
 SELECT 'JOIN ON with conditions';
 JOIN ON with conditions
 SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value
 FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t1.value == 'Join_1_Value_0'
-FULL JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0';
+FULL JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0' ORDER BY ALL;
+0		0		1	Join_3_Value_1
+0		0		4	Join_3_Value_4
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	0		0	
 2	Join_1_Value_2	0		0	
-0		0		4	Join_3_Value_4
-0		0		1	Join_3_Value_1
 SELECT '--';
 --
 SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value
 FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t1.value == 'Join_1_Value_0' AND t2.value == 'Join_2_Value_0'
-FULL JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0' AND t3.value == 'Join_3_Value_0';
+FULL JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0' AND t3.value == 'Join_3_Value_0' ORDER BY ALL;
+0		0		1	Join_3_Value_1
+0		0		4	Join_3_Value_4
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	0		0	
 2	Join_1_Value_2	0		0	
-0		0		1	Join_3_Value_1
-0		0		4	Join_3_Value_4
 SELECT 'JOIN multiple clauses';
 JOIN multiple clauses
 SELECT t1.value, t2.value, t3.value
 FROM test_table_join_1 AS t1
 LEFT JOIN test_table_join_2 AS t2 ON t1.id = t2.id OR t1.id = t2.id
-FULL JOIN test_table_join_3 AS t3 ON t2.id = t3.id OR t3.id = t2.id;
+FULL JOIN test_table_join_3 AS t3 ON t2.id = t3.id OR t3.id = t2.id ORDER BY ALL;
+		Join_3_Value_4
 Join_1_Value_0	Join_2_Value_0	Join_3_Value_0
 Join_1_Value_1	Join_2_Value_1	Join_3_Value_1
 Join_1_Value_2		Join_3_Value_0
-		Join_3_Value_4
 SELECT 'JOIN expression aliases';
 JOIN expression aliases
 SELECT t1_id, t1.value, t2_id, t2.value, t3_id, t3.value
 FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 ON (t1.id AS t1_id) = (t2.id AS t2_id)
-FULL JOIN test_table_join_3 AS t3 ON t2_id = (t3.id AS t3_id);
+FULL JOIN test_table_join_3 AS t3 ON t2_id = (t3.id AS t3_id) ORDER BY ALL;
+0		0		4	Join_3_Value_4
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
 2	Join_1_Value_2	0		0	Join_3_Value_0
-0		0		4	Join_3_Value_4
 SELECT '--';
 --
 SELECT t1.id AS t1_id, t1.value, t2.id AS t2_id, t2.value, t3.id AS t3_id, t3.value
 FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 ON t1_id = t2_id
-FULL JOIN test_table_join_3 AS t3 ON t2_id = t3_id;
+FULL JOIN test_table_join_3 AS t3 ON t2_id = t3_id ORDER BY ALL;
+0		0		4	Join_3_Value_4
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
 2	Join_1_Value_2	0		0	Join_3_Value_0
-0		0		4	Join_3_Value_4
 SELECT 'First JOIN RIGHT second JOIN INNER';
 First JOIN RIGHT second JOIN INNER
 SELECT 'JOIN ON without conditions';
 JOIN ON without conditions
 SELECT test_table_join_1.id, test_table_join_1.value, test_table_join_2.id, test_table_join_2.value, test_table_join_3.id, test_table_join_3.value
 FROM test_table_join_1 RIGHT JOIN test_table_join_2 ON test_table_join_1.id = test_table_join_2.id
-INNER JOIN test_table_join_3 ON test_table_join_2.id = test_table_join_3.id;
+INNER JOIN test_table_join_3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
 SELECT '--';
 --
 SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value
 FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 ON test_table_join_1.id = test_table_join_2.id
-INNER JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id;
+INNER JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
 SELECT '--';
@@ -929,48 +929,48 @@ SELECT '--';
 SELECT t1.id, test_table_join_1.id, t1.value, test_table_join_1.value, t2.id, test_table_join_2.id, t2.value, test_table_join_2.value,
 t3.id, test_table_join_3.id, t3.value, test_table_join_3.value
 FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 ON test_table_join_1.id = test_table_join_2.id
-INNER JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id;
+INNER JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
 0	0	Join_1_Value_0	Join_1_Value_0	0	0	Join_2_Value_0	Join_2_Value_0	0	0	Join_3_Value_0	Join_3_Value_0
 1	1	Join_1_Value_1	Join_1_Value_1	1	1	Join_2_Value_1	Join_2_Value_1	1	1	Join_3_Value_1	Join_3_Value_1
 SELECT '--';
 --
 SELECT t1.value, t2.value, t3.value
 FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 ON test_table_join_1.id = test_table_join_2.id
-INNER JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id;
+INNER JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
 Join_1_Value_0	Join_2_Value_0	Join_3_Value_0
 Join_1_Value_1	Join_2_Value_1	Join_3_Value_1
 SELECT 'JOIN ON with conditions';
 JOIN ON with conditions
 SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value
 FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t1.value == 'Join_1_Value_0'
-INNER JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0';
+INNER JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0' ORDER BY ALL;
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 SELECT '--';
 --
 SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value
 FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t1.value == 'Join_1_Value_0' AND t2.value == 'Join_2_Value_0'
-INNER JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0' AND t3.value == 'Join_3_Value_0';
+INNER JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0' AND t3.value == 'Join_3_Value_0' ORDER BY ALL;
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 SELECT 'JOIN multiple clauses';
 JOIN multiple clauses
 SELECT t1.value, t2.value, t3.value
 FROM test_table_join_1 AS t1
 RIGHT JOIN test_table_join_2 AS t2 ON t1.id = t2.id OR t1.id = t2.id
-INNER JOIN test_table_join_3 AS t3 ON t2.id = t3.id OR t3.id = t2.id;
+INNER JOIN test_table_join_3 AS t3 ON t2.id = t3.id OR t3.id = t2.id ORDER BY ALL;
 Join_1_Value_0	Join_2_Value_0	Join_3_Value_0
 Join_1_Value_1	Join_2_Value_1	Join_3_Value_1
 SELECT 'JOIN expression aliases';
 JOIN expression aliases
 SELECT t1_id, t1.value, t2_id, t2.value, t3_id, t3.value
 FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 ON (t1.id AS t1_id) = (t2.id AS t2_id)
-INNER JOIN test_table_join_3 AS t3 ON t2_id = (t3.id AS t3_id);
+INNER JOIN test_table_join_3 AS t3 ON t2_id = (t3.id AS t3_id) ORDER BY ALL;
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
 SELECT '--';
 --
 SELECT t1.id AS t1_id, t1.value, t2.id AS t2_id, t2.value, t3.id AS t3_id, t3.value
 FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 ON t1_id = t2_id
-INNER JOIN test_table_join_3 AS t3 ON t2_id = t3_id;
+INNER JOIN test_table_join_3 AS t3 ON t2_id = t3_id ORDER BY ALL;
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
 SELECT 'First JOIN RIGHT second JOIN LEFT';
@@ -979,246 +979,246 @@ SELECT 'JOIN ON without conditions';
 JOIN ON without conditions
 SELECT test_table_join_1.id, test_table_join_1.value, test_table_join_2.id, test_table_join_2.value, test_table_join_3.id, test_table_join_3.value
 FROM test_table_join_1 RIGHT JOIN test_table_join_2 ON test_table_join_1.id = test_table_join_2.id
-LEFT JOIN test_table_join_3 ON test_table_join_2.id = test_table_join_3.id;
+LEFT JOIN test_table_join_3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
+0		3	Join_2_Value_3	0	
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
-0		3	Join_2_Value_3	0	
 SELECT '--';
 --
 SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value
 FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 ON test_table_join_1.id = test_table_join_2.id
-LEFT JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id;
+LEFT JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
+0		3	Join_2_Value_3	0	
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
-0		3	Join_2_Value_3	0	
 SELECT '--';
 --
 SELECT t1.id, test_table_join_1.id, t1.value, test_table_join_1.value, t2.id, test_table_join_2.id, t2.value, test_table_join_2.value,
 t3.id, test_table_join_3.id, t3.value, test_table_join_3.value
 FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 ON test_table_join_1.id = test_table_join_2.id
-LEFT JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id;
+LEFT JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
+0	0			3	3	Join_2_Value_3	Join_2_Value_3	0	0		
 0	0	Join_1_Value_0	Join_1_Value_0	0	0	Join_2_Value_0	Join_2_Value_0	0	0	Join_3_Value_0	Join_3_Value_0
 1	1	Join_1_Value_1	Join_1_Value_1	1	1	Join_2_Value_1	Join_2_Value_1	1	1	Join_3_Value_1	Join_3_Value_1
-0	0			3	3	Join_2_Value_3	Join_2_Value_3	0	0		
 SELECT '--';
 --
 SELECT t1.value, t2.value, t3.value
 FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 ON test_table_join_1.id = test_table_join_2.id
-LEFT JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id;
+LEFT JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
+	Join_2_Value_3	
 Join_1_Value_0	Join_2_Value_0	Join_3_Value_0
 Join_1_Value_1	Join_2_Value_1	Join_3_Value_1
-	Join_2_Value_3	
 SELECT 'JOIN ON with conditions';
 JOIN ON with conditions
 SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value
 FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t1.value == 'Join_1_Value_0'
-LEFT JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0';
-0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
-0		3	Join_2_Value_3	0	
+LEFT JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0' ORDER BY ALL;
 0		1	Join_2_Value_1	0	
+0		3	Join_2_Value_3	0	
+0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 SELECT '--';
 --
 SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value
 FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t1.value == 'Join_1_Value_0' AND t2.value == 'Join_2_Value_0'
-LEFT JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0' AND t3.value == 'Join_3_Value_0';
-0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
+LEFT JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0' AND t3.value == 'Join_3_Value_0' ORDER BY ALL;
 0		1	Join_2_Value_1	0	
 0		3	Join_2_Value_3	0	
+0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 SELECT 'JOIN multiple clauses';
 JOIN multiple clauses
 SELECT t1.value, t2.value, t3.value
 FROM test_table_join_1 AS t1
 RIGHT JOIN test_table_join_2 AS t2 ON t1.id = t2.id OR t1.id = t2.id
-LEFT JOIN test_table_join_3 AS t3 ON t2.id = t3.id OR t3.id = t2.id;
+LEFT JOIN test_table_join_3 AS t3 ON t2.id = t3.id OR t3.id = t2.id ORDER BY ALL;
+	Join_2_Value_3	
 Join_1_Value_0	Join_2_Value_0	Join_3_Value_0
 Join_1_Value_1	Join_2_Value_1	Join_3_Value_1
-	Join_2_Value_3	
 SELECT 'JOIN expression aliases';
 JOIN expression aliases
 SELECT t1_id, t1.value, t2_id, t2.value, t3_id, t3.value
 FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 ON (t1.id AS t1_id) = (t2.id AS t2_id)
-LEFT JOIN test_table_join_3 AS t3 ON t2_id = (t3.id AS t3_id);
+LEFT JOIN test_table_join_3 AS t3 ON t2_id = (t3.id AS t3_id) ORDER BY ALL;
+0		3	Join_2_Value_3	0	
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
-0		3	Join_2_Value_3	0	
 SELECT '--';
 --
 SELECT t1.id AS t1_id, t1.value, t2.id AS t2_id, t2.value, t3.id AS t3_id, t3.value
 FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 ON t1_id = t2_id
-LEFT JOIN test_table_join_3 AS t3 ON t2_id = t3_id;
+LEFT JOIN test_table_join_3 AS t3 ON t2_id = t3_id ORDER BY ALL;
+0		3	Join_2_Value_3	0	
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
-0		3	Join_2_Value_3	0	
 SELECT 'First JOIN RIGHT second JOIN RIGHT';
 First JOIN RIGHT second JOIN RIGHT
 SELECT 'JOIN ON without conditions';
 JOIN ON without conditions
 SELECT test_table_join_1.id, test_table_join_1.value, test_table_join_2.id, test_table_join_2.value, test_table_join_3.id, test_table_join_3.value
 FROM test_table_join_1 RIGHT JOIN test_table_join_2 ON test_table_join_1.id = test_table_join_2.id
-RIGHT JOIN test_table_join_3 ON test_table_join_2.id = test_table_join_3.id;
+RIGHT JOIN test_table_join_3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
+0		0		4	Join_3_Value_4
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
-0		0		4	Join_3_Value_4
 SELECT '--';
 --
 SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value
 FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 ON test_table_join_1.id = test_table_join_2.id
-RIGHT JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id;
+RIGHT JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
+0		0		4	Join_3_Value_4
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
-0		0		4	Join_3_Value_4
 SELECT '--';
 --
 SELECT t1.id, test_table_join_1.id, t1.value, test_table_join_1.value, t2.id, test_table_join_2.id, t2.value, test_table_join_2.value,
 t3.id, test_table_join_3.id, t3.value, test_table_join_3.value
 FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 ON test_table_join_1.id = test_table_join_2.id
-RIGHT JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id;
+RIGHT JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
+0	0			0	0			4	4	Join_3_Value_4	Join_3_Value_4
 0	0	Join_1_Value_0	Join_1_Value_0	0	0	Join_2_Value_0	Join_2_Value_0	0	0	Join_3_Value_0	Join_3_Value_0
 1	1	Join_1_Value_1	Join_1_Value_1	1	1	Join_2_Value_1	Join_2_Value_1	1	1	Join_3_Value_1	Join_3_Value_1
-0	0			0	0			4	4	Join_3_Value_4	Join_3_Value_4
 SELECT '--';
 --
 SELECT t1.value, t2.value, t3.value
 FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 ON test_table_join_1.id = test_table_join_2.id
-RIGHT JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id;
+RIGHT JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
+		Join_3_Value_4
 Join_1_Value_0	Join_2_Value_0	Join_3_Value_0
 Join_1_Value_1	Join_2_Value_1	Join_3_Value_1
-		Join_3_Value_4
 SELECT 'JOIN ON with conditions';
 JOIN ON with conditions
 SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value
 FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t1.value == 'Join_1_Value_0'
-RIGHT JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0';
-0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
-0		0		4	Join_3_Value_4
+RIGHT JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0' ORDER BY ALL;
 0		0		1	Join_3_Value_1
+0		0		4	Join_3_Value_4
+0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 SELECT '--';
 --
 SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value
 FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t1.value == 'Join_1_Value_0' AND t2.value == 'Join_2_Value_0'
-RIGHT JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0' AND t3.value == 'Join_3_Value_0';
-0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
+RIGHT JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0' AND t3.value == 'Join_3_Value_0' ORDER BY ALL;
 0		0		1	Join_3_Value_1
 0		0		4	Join_3_Value_4
+0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 SELECT 'JOIN multiple clauses';
 JOIN multiple clauses
 SELECT t1.value, t2.value, t3.value
 FROM test_table_join_1 AS t1
 RIGHT JOIN test_table_join_2 AS t2 ON t1.id = t2.id OR t1.id = t2.id
-RIGHT JOIN test_table_join_3 AS t3 ON t2.id = t3.id OR t3.id = t2.id;
+RIGHT JOIN test_table_join_3 AS t3 ON t2.id = t3.id OR t3.id = t2.id ORDER BY ALL;
+		Join_3_Value_4
 Join_1_Value_0	Join_2_Value_0	Join_3_Value_0
 Join_1_Value_1	Join_2_Value_1	Join_3_Value_1
-		Join_3_Value_4
 SELECT 'JOIN expression aliases';
 JOIN expression aliases
 SELECT t1_id, t1.value, t2_id, t2.value, t3_id, t3.value
 FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 ON (t1.id AS t1_id) = (t2.id AS t2_id)
-RIGHT JOIN test_table_join_3 AS t3 ON t2_id = (t3.id AS t3_id);
+RIGHT JOIN test_table_join_3 AS t3 ON t2_id = (t3.id AS t3_id) ORDER BY ALL;
+0		0		4	Join_3_Value_4
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
-0		0		4	Join_3_Value_4
 SELECT '--';
 --
 SELECT t1.id AS t1_id, t1.value, t2.id AS t2_id, t2.value, t3.id AS t3_id, t3.value
 FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 ON t1_id = t2_id
-RIGHT JOIN test_table_join_3 AS t3 ON t2_id = t3_id;
+RIGHT JOIN test_table_join_3 AS t3 ON t2_id = t3_id ORDER BY ALL;
+0		0		4	Join_3_Value_4
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
-0		0		4	Join_3_Value_4
 SELECT 'First JOIN RIGHT second JOIN FULL';
 First JOIN RIGHT second JOIN FULL
 SELECT 'JOIN ON without conditions';
 JOIN ON without conditions
 SELECT test_table_join_1.id, test_table_join_1.value, test_table_join_2.id, test_table_join_2.value, test_table_join_3.id, test_table_join_3.value
 FROM test_table_join_1 RIGHT JOIN test_table_join_2 ON test_table_join_1.id = test_table_join_2.id
-FULL JOIN test_table_join_3 ON test_table_join_2.id = test_table_join_3.id;
+FULL JOIN test_table_join_3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
+0		0		4	Join_3_Value_4
+0		3	Join_2_Value_3	0	
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
-0		3	Join_2_Value_3	0	
-0		0		4	Join_3_Value_4
 SELECT '--';
 --
 SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value
 FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 ON test_table_join_1.id = test_table_join_2.id
-FULL JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id;
+FULL JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
+0		0		4	Join_3_Value_4
+0		3	Join_2_Value_3	0	
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
-0		3	Join_2_Value_3	0	
-0		0		4	Join_3_Value_4
 SELECT '--';
 --
 SELECT t1.id, test_table_join_1.id, t1.value, test_table_join_1.value, t2.id, test_table_join_2.id, t2.value, test_table_join_2.value,
 t3.id, test_table_join_3.id, t3.value, test_table_join_3.value
 FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 ON test_table_join_1.id = test_table_join_2.id
-FULL JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id;
+FULL JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
+0	0			0	0			4	4	Join_3_Value_4	Join_3_Value_4
+0	0			3	3	Join_2_Value_3	Join_2_Value_3	0	0		
 0	0	Join_1_Value_0	Join_1_Value_0	0	0	Join_2_Value_0	Join_2_Value_0	0	0	Join_3_Value_0	Join_3_Value_0
 1	1	Join_1_Value_1	Join_1_Value_1	1	1	Join_2_Value_1	Join_2_Value_1	1	1	Join_3_Value_1	Join_3_Value_1
-0	0			3	3	Join_2_Value_3	Join_2_Value_3	0	0		
-0	0			0	0			4	4	Join_3_Value_4	Join_3_Value_4
 SELECT '--';
 --
 SELECT t1.value, t2.value, t3.value
 FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 ON test_table_join_1.id = test_table_join_2.id
-FULL JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id;
+FULL JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
+		Join_3_Value_4
+	Join_2_Value_3	
 Join_1_Value_0	Join_2_Value_0	Join_3_Value_0
 Join_1_Value_1	Join_2_Value_1	Join_3_Value_1
-	Join_2_Value_3	
-		Join_3_Value_4
 SELECT 'JOIN ON with conditions';
 JOIN ON with conditions
 SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value
 FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t1.value == 'Join_1_Value_0'
-FULL JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0';
-0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
-0		3	Join_2_Value_3	0	
-0		1	Join_2_Value_1	0	
-0		0		4	Join_3_Value_4
+FULL JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0' ORDER BY ALL;
 0		0		1	Join_3_Value_1
+0		0		4	Join_3_Value_4
+0		1	Join_2_Value_1	0	
+0		3	Join_2_Value_3	0	
+0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 SELECT '--';
 --
 SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value
 FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t1.value == 'Join_1_Value_0' AND t2.value == 'Join_2_Value_0'
-FULL JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0' AND t3.value == 'Join_3_Value_0';
-0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
-0		1	Join_2_Value_1	0	
-0		3	Join_2_Value_3	0	
+FULL JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0' AND t3.value == 'Join_3_Value_0' ORDER BY ALL;
 0		0		1	Join_3_Value_1
 0		0		4	Join_3_Value_4
+0		1	Join_2_Value_1	0	
+0		3	Join_2_Value_3	0	
+0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 SELECT 'JOIN multiple clauses';
 JOIN multiple clauses
 SELECT t1.value, t2.value, t3.value
 FROM test_table_join_1 AS t1
 RIGHT JOIN test_table_join_2 AS t2 ON t1.id = t2.id OR t1.id = t2.id
-FULL JOIN test_table_join_3 AS t3 ON t2.id = t3.id OR t3.id = t2.id;
+FULL JOIN test_table_join_3 AS t3 ON t2.id = t3.id OR t3.id = t2.id ORDER BY ALL;
+		Join_3_Value_4
+	Join_2_Value_3	
 Join_1_Value_0	Join_2_Value_0	Join_3_Value_0
 Join_1_Value_1	Join_2_Value_1	Join_3_Value_1
-	Join_2_Value_3	
-		Join_3_Value_4
 SELECT 'JOIN expression aliases';
 JOIN expression aliases
 SELECT t1_id, t1.value, t2_id, t2.value, t3_id, t3.value
 FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 ON (t1.id AS t1_id) = (t2.id AS t2_id)
-FULL JOIN test_table_join_3 AS t3 ON t2_id = (t3.id AS t3_id);
+FULL JOIN test_table_join_3 AS t3 ON t2_id = (t3.id AS t3_id) ORDER BY ALL;
+0		0		4	Join_3_Value_4
+0		3	Join_2_Value_3	0	
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
-0		3	Join_2_Value_3	0	
-0		0		4	Join_3_Value_4
 SELECT '--';
 --
 SELECT t1.id AS t1_id, t1.value, t2.id AS t2_id, t2.value, t3.id AS t3_id, t3.value
 FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 ON t1_id = t2_id
-FULL JOIN test_table_join_3 AS t3 ON t2_id = t3_id;
+FULL JOIN test_table_join_3 AS t3 ON t2_id = t3_id ORDER BY ALL;
+0		0		4	Join_3_Value_4
+0		3	Join_2_Value_3	0	
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
-0		3	Join_2_Value_3	0	
-0		0		4	Join_3_Value_4
 SELECT 'First JOIN FULL second JOIN INNER';
 First JOIN FULL second JOIN INNER
 SELECT 'JOIN ON without conditions';
 JOIN ON without conditions
 SELECT test_table_join_1.id, test_table_join_1.value, test_table_join_2.id, test_table_join_2.value, test_table_join_3.id, test_table_join_3.value
 FROM test_table_join_1 FULL JOIN test_table_join_2 ON test_table_join_1.id = test_table_join_2.id
-INNER JOIN test_table_join_3 ON test_table_join_2.id = test_table_join_3.id;
+INNER JOIN test_table_join_3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
 2	Join_1_Value_2	0		0	Join_3_Value_0
@@ -1226,7 +1226,7 @@ SELECT '--';
 --
 SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value
 FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 ON test_table_join_1.id = test_table_join_2.id
-INNER JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id;
+INNER JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
 2	Join_1_Value_2	0		0	Join_3_Value_0
@@ -1235,7 +1235,7 @@ SELECT '--';
 SELECT t1.id, test_table_join_1.id, t1.value, test_table_join_1.value, t2.id, test_table_join_2.id, t2.value, test_table_join_2.value,
 t3.id, test_table_join_3.id, t3.value, test_table_join_3.value
 FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 ON test_table_join_1.id = test_table_join_2.id
-INNER JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id;
+INNER JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
 0	0	Join_1_Value_0	Join_1_Value_0	0	0	Join_2_Value_0	Join_2_Value_0	0	0	Join_3_Value_0	Join_3_Value_0
 1	1	Join_1_Value_1	Join_1_Value_1	1	1	Join_2_Value_1	Join_2_Value_1	1	1	Join_3_Value_1	Join_3_Value_1
 2	2	Join_1_Value_2	Join_1_Value_2	0	0			0	0	Join_3_Value_0	Join_3_Value_0
@@ -1243,7 +1243,7 @@ SELECT '--';
 --
 SELECT t1.value, t2.value, t3.value
 FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 ON test_table_join_1.id = test_table_join_2.id
-INNER JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id;
+INNER JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
 Join_1_Value_0	Join_2_Value_0	Join_3_Value_0
 Join_1_Value_1	Join_2_Value_1	Join_3_Value_1
 Join_1_Value_2		Join_3_Value_0
@@ -1251,20 +1251,20 @@ SELECT 'JOIN ON with conditions';
 JOIN ON with conditions
 SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value
 FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t1.value == 'Join_1_Value_0'
-INNER JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0';
+INNER JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0' ORDER BY ALL;
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 SELECT '--';
 --
 SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value
 FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t1.value == 'Join_1_Value_0' AND t2.value == 'Join_2_Value_0'
-INNER JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0' AND t3.value == 'Join_3_Value_0';
+INNER JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0' AND t3.value == 'Join_3_Value_0' ORDER BY ALL;
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 SELECT 'JOIN multiple clauses';
 JOIN multiple clauses
 SELECT t1.value, t2.value, t3.value
 FROM test_table_join_1 AS t1
 FULL JOIN test_table_join_2 AS t2 ON t1.id = t2.id OR t1.id = t2.id
-INNER JOIN test_table_join_3 AS t3 ON t2.id = t3.id OR t3.id = t2.id;
+INNER JOIN test_table_join_3 AS t3 ON t2.id = t3.id OR t3.id = t2.id ORDER BY ALL;
 Join_1_Value_0	Join_2_Value_0	Join_3_Value_0
 Join_1_Value_1	Join_2_Value_1	Join_3_Value_1
 Join_1_Value_2		Join_3_Value_0
@@ -1272,7 +1272,7 @@ SELECT 'JOIN expression aliases';
 JOIN expression aliases
 SELECT t1_id, t1.value, t2_id, t2.value, t3_id, t3.value
 FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 ON (t1.id AS t1_id) = (t2.id AS t2_id)
-INNER JOIN test_table_join_3 AS t3 ON t2_id = (t3.id AS t3_id);
+INNER JOIN test_table_join_3 AS t3 ON t2_id = (t3.id AS t3_id) ORDER BY ALL;
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
 2	Join_1_Value_2	0		0	Join_3_Value_0
@@ -1280,7 +1280,7 @@ SELECT '--';
 --
 SELECT t1.id AS t1_id, t1.value, t2.id AS t2_id, t2.value, t3.id AS t3_id, t3.value
 FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 ON t1_id = t2_id
-INNER JOIN test_table_join_3 AS t3 ON t2_id = t3_id;
+INNER JOIN test_table_join_3 AS t3 ON t2_id = t3_id ORDER BY ALL;
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
 2	Join_1_Value_2	0		0	Join_3_Value_0
@@ -1290,265 +1290,265 @@ SELECT 'JOIN ON without conditions';
 JOIN ON without conditions
 SELECT test_table_join_1.id, test_table_join_1.value, test_table_join_2.id, test_table_join_2.value, test_table_join_3.id, test_table_join_3.value
 FROM test_table_join_1 FULL JOIN test_table_join_2 ON test_table_join_1.id = test_table_join_2.id
-LEFT JOIN test_table_join_3 ON test_table_join_2.id = test_table_join_3.id;
+LEFT JOIN test_table_join_3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
+0		3	Join_2_Value_3	0	
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
 2	Join_1_Value_2	0		0	Join_3_Value_0
-0		3	Join_2_Value_3	0	
 SELECT '--';
 --
 SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value
 FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 ON test_table_join_1.id = test_table_join_2.id
-LEFT JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id;
+LEFT JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
+0		3	Join_2_Value_3	0	
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
 2	Join_1_Value_2	0		0	Join_3_Value_0
-0		3	Join_2_Value_3	0	
 SELECT '--';
 --
 SELECT t1.id, test_table_join_1.id, t1.value, test_table_join_1.value, t2.id, test_table_join_2.id, t2.value, test_table_join_2.value,
 t3.id, test_table_join_3.id, t3.value, test_table_join_3.value
 FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 ON test_table_join_1.id = test_table_join_2.id
-LEFT JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id;
+LEFT JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
+0	0			3	3	Join_2_Value_3	Join_2_Value_3	0	0		
 0	0	Join_1_Value_0	Join_1_Value_0	0	0	Join_2_Value_0	Join_2_Value_0	0	0	Join_3_Value_0	Join_3_Value_0
 1	1	Join_1_Value_1	Join_1_Value_1	1	1	Join_2_Value_1	Join_2_Value_1	1	1	Join_3_Value_1	Join_3_Value_1
 2	2	Join_1_Value_2	Join_1_Value_2	0	0			0	0	Join_3_Value_0	Join_3_Value_0
-0	0			3	3	Join_2_Value_3	Join_2_Value_3	0	0		
 SELECT '--';
 --
 SELECT t1.value, t2.value, t3.value
 FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 ON test_table_join_1.id = test_table_join_2.id
-LEFT JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id;
+LEFT JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
+	Join_2_Value_3	
 Join_1_Value_0	Join_2_Value_0	Join_3_Value_0
 Join_1_Value_1	Join_2_Value_1	Join_3_Value_1
 Join_1_Value_2		Join_3_Value_0
-	Join_2_Value_3	
 SELECT 'JOIN ON with conditions';
 JOIN ON with conditions
 SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value
 FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t1.value == 'Join_1_Value_0'
-LEFT JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0';
+LEFT JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0' ORDER BY ALL;
+0		1	Join_2_Value_1	0	
+0		3	Join_2_Value_3	0	
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	0		0	
 2	Join_1_Value_2	0		0	
-0		3	Join_2_Value_3	0	
-0		1	Join_2_Value_1	0	
 SELECT '--';
 --
 SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value
 FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t1.value == 'Join_1_Value_0' AND t2.value == 'Join_2_Value_0'
-LEFT JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0' AND t3.value == 'Join_3_Value_0';
+LEFT JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0' AND t3.value == 'Join_3_Value_0' ORDER BY ALL;
+0		1	Join_2_Value_1	0	
+0		3	Join_2_Value_3	0	
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	0		0	
 2	Join_1_Value_2	0		0	
-0		1	Join_2_Value_1	0	
-0		3	Join_2_Value_3	0	
 SELECT 'JOIN multiple clauses';
 JOIN multiple clauses
 SELECT t1.value, t2.value, t3.value
 FROM test_table_join_1 AS t1
 FULL JOIN test_table_join_2 AS t2 ON t1.id = t2.id OR t1.id = t2.id
-LEFT JOIN test_table_join_3 AS t3 ON t2.id = t3.id OR t3.id = t2.id;
+LEFT JOIN test_table_join_3 AS t3 ON t2.id = t3.id OR t3.id = t2.id ORDER BY ALL;
+	Join_2_Value_3	
 Join_1_Value_0	Join_2_Value_0	Join_3_Value_0
 Join_1_Value_1	Join_2_Value_1	Join_3_Value_1
 Join_1_Value_2		Join_3_Value_0
-	Join_2_Value_3	
 SELECT 'JOIN expression aliases';
 JOIN expression aliases
 SELECT t1_id, t1.value, t2_id, t2.value, t3_id, t3.value
 FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 ON (t1.id AS t1_id) = (t2.id AS t2_id)
-LEFT JOIN test_table_join_3 AS t3 ON t2_id = (t3.id AS t3_id);
+LEFT JOIN test_table_join_3 AS t3 ON t2_id = (t3.id AS t3_id) ORDER BY ALL;
+0		3	Join_2_Value_3	0	
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
 2	Join_1_Value_2	0		0	Join_3_Value_0
-0		3	Join_2_Value_3	0	
 SELECT '--';
 --
 SELECT t1.id AS t1_id, t1.value, t2.id AS t2_id, t2.value, t3.id AS t3_id, t3.value
 FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 ON t1_id = t2_id
-LEFT JOIN test_table_join_3 AS t3 ON t2_id = t3_id;
+LEFT JOIN test_table_join_3 AS t3 ON t2_id = t3_id ORDER BY ALL;
+0		3	Join_2_Value_3	0	
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
 2	Join_1_Value_2	0		0	Join_3_Value_0
-0		3	Join_2_Value_3	0	
 SELECT 'First JOIN FULL second JOIN RIGHT';
 First JOIN FULL second JOIN RIGHT
 SELECT 'JOIN ON without conditions';
 JOIN ON without conditions
 SELECT test_table_join_1.id, test_table_join_1.value, test_table_join_2.id, test_table_join_2.value, test_table_join_3.id, test_table_join_3.value
 FROM test_table_join_1 FULL JOIN test_table_join_2 ON test_table_join_1.id = test_table_join_2.id
-RIGHT JOIN test_table_join_3 ON test_table_join_2.id = test_table_join_3.id;
+RIGHT JOIN test_table_join_3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
+0		0		4	Join_3_Value_4
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
 2	Join_1_Value_2	0		0	Join_3_Value_0
-0		0		4	Join_3_Value_4
 SELECT '--';
 --
 SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value
 FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 ON test_table_join_1.id = test_table_join_2.id
-RIGHT JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id;
+RIGHT JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
+0		0		4	Join_3_Value_4
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
 2	Join_1_Value_2	0		0	Join_3_Value_0
-0		0		4	Join_3_Value_4
 SELECT '--';
 --
 SELECT t1.id, test_table_join_1.id, t1.value, test_table_join_1.value, t2.id, test_table_join_2.id, t2.value, test_table_join_2.value,
 t3.id, test_table_join_3.id, t3.value, test_table_join_3.value
 FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 ON test_table_join_1.id = test_table_join_2.id
-RIGHT JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id;
+RIGHT JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
+0	0			0	0			4	4	Join_3_Value_4	Join_3_Value_4
 0	0	Join_1_Value_0	Join_1_Value_0	0	0	Join_2_Value_0	Join_2_Value_0	0	0	Join_3_Value_0	Join_3_Value_0
 1	1	Join_1_Value_1	Join_1_Value_1	1	1	Join_2_Value_1	Join_2_Value_1	1	1	Join_3_Value_1	Join_3_Value_1
 2	2	Join_1_Value_2	Join_1_Value_2	0	0			0	0	Join_3_Value_0	Join_3_Value_0
-0	0			0	0			4	4	Join_3_Value_4	Join_3_Value_4
 SELECT '--';
 --
 SELECT t1.value, t2.value, t3.value
 FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 ON test_table_join_1.id = test_table_join_2.id
-RIGHT JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id;
+RIGHT JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
+		Join_3_Value_4
 Join_1_Value_0	Join_2_Value_0	Join_3_Value_0
 Join_1_Value_1	Join_2_Value_1	Join_3_Value_1
 Join_1_Value_2		Join_3_Value_0
-		Join_3_Value_4
 SELECT 'JOIN ON with conditions';
 JOIN ON with conditions
 SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value
 FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t1.value == 'Join_1_Value_0'
-RIGHT JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0';
-0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
-0		0		4	Join_3_Value_4
+RIGHT JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0' ORDER BY ALL;
 0		0		1	Join_3_Value_1
+0		0		4	Join_3_Value_4
+0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 SELECT '--';
 --
 SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value
 FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t1.value == 'Join_1_Value_0' AND t2.value == 'Join_2_Value_0'
-RIGHT JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0' AND t3.value == 'Join_3_Value_0';
-0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
+RIGHT JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0' AND t3.value == 'Join_3_Value_0' ORDER BY ALL;
 0		0		1	Join_3_Value_1
 0		0		4	Join_3_Value_4
+0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 SELECT 'JOIN multiple clauses';
 JOIN multiple clauses
 SELECT t1.value, t2.value, t3.value
 FROM test_table_join_1 AS t1
 FULL JOIN test_table_join_2 AS t2 ON t1.id = t2.id OR t1.id = t2.id
-RIGHT JOIN test_table_join_3 AS t3 ON t2.id = t3.id OR t3.id = t2.id;
+RIGHT JOIN test_table_join_3 AS t3 ON t2.id = t3.id OR t3.id = t2.id ORDER BY ALL;
+		Join_3_Value_4
 Join_1_Value_0	Join_2_Value_0	Join_3_Value_0
 Join_1_Value_1	Join_2_Value_1	Join_3_Value_1
 Join_1_Value_2		Join_3_Value_0
-		Join_3_Value_4
 SELECT 'JOIN expression aliases';
 JOIN expression aliases
 SELECT t1_id, t1.value, t2_id, t2.value, t3_id, t3.value
 FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 ON (t1.id AS t1_id) = (t2.id AS t2_id)
-RIGHT JOIN test_table_join_3 AS t3 ON t2_id = (t3.id AS t3_id);
+RIGHT JOIN test_table_join_3 AS t3 ON t2_id = (t3.id AS t3_id) ORDER BY ALL;
+0		0		4	Join_3_Value_4
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
 2	Join_1_Value_2	0		0	Join_3_Value_0
-0		0		4	Join_3_Value_4
 SELECT '--';
 --
 SELECT t1.id AS t1_id, t1.value, t2.id AS t2_id, t2.value, t3.id AS t3_id, t3.value
 FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 ON t1_id = t2_id
-RIGHT JOIN test_table_join_3 AS t3 ON t2_id = t3_id;
+RIGHT JOIN test_table_join_3 AS t3 ON t2_id = t3_id ORDER BY ALL;
+0		0		4	Join_3_Value_4
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
 2	Join_1_Value_2	0		0	Join_3_Value_0
-0		0		4	Join_3_Value_4
 SELECT 'First JOIN FULL second JOIN FULL';
 First JOIN FULL second JOIN FULL
 SELECT 'JOIN ON without conditions';
 JOIN ON without conditions
 SELECT test_table_join_1.id, test_table_join_1.value, test_table_join_2.id, test_table_join_2.value, test_table_join_3.id, test_table_join_3.value
 FROM test_table_join_1 FULL JOIN test_table_join_2 ON test_table_join_1.id = test_table_join_2.id
-FULL JOIN test_table_join_3 ON test_table_join_2.id = test_table_join_3.id;
+FULL JOIN test_table_join_3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
+0		0		4	Join_3_Value_4
+0		3	Join_2_Value_3	0	
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
 2	Join_1_Value_2	0		0	Join_3_Value_0
-0		3	Join_2_Value_3	0	
-0		0		4	Join_3_Value_4
 SELECT '--';
 --
 SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value
 FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 ON test_table_join_1.id = test_table_join_2.id
-FULL JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id;
+FULL JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
+0		0		4	Join_3_Value_4
+0		3	Join_2_Value_3	0	
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
 2	Join_1_Value_2	0		0	Join_3_Value_0
-0		3	Join_2_Value_3	0	
-0		0		4	Join_3_Value_4
 SELECT '--';
 --
 SELECT t1.id, test_table_join_1.id, t1.value, test_table_join_1.value, t2.id, test_table_join_2.id, t2.value, test_table_join_2.value,
 t3.id, test_table_join_3.id, t3.value, test_table_join_3.value
 FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 ON test_table_join_1.id = test_table_join_2.id
-FULL JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id;
+FULL JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
+0	0			0	0			4	4	Join_3_Value_4	Join_3_Value_4
+0	0			3	3	Join_2_Value_3	Join_2_Value_3	0	0		
 0	0	Join_1_Value_0	Join_1_Value_0	0	0	Join_2_Value_0	Join_2_Value_0	0	0	Join_3_Value_0	Join_3_Value_0
 1	1	Join_1_Value_1	Join_1_Value_1	1	1	Join_2_Value_1	Join_2_Value_1	1	1	Join_3_Value_1	Join_3_Value_1
 2	2	Join_1_Value_2	Join_1_Value_2	0	0			0	0	Join_3_Value_0	Join_3_Value_0
-0	0			3	3	Join_2_Value_3	Join_2_Value_3	0	0		
-0	0			0	0			4	4	Join_3_Value_4	Join_3_Value_4
 SELECT '--';
 --
 SELECT t1.value, t2.value, t3.value
 FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 ON test_table_join_1.id = test_table_join_2.id
-FULL JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id;
+FULL JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
+		Join_3_Value_4
+	Join_2_Value_3	
 Join_1_Value_0	Join_2_Value_0	Join_3_Value_0
 Join_1_Value_1	Join_2_Value_1	Join_3_Value_1
 Join_1_Value_2		Join_3_Value_0
-	Join_2_Value_3	
-		Join_3_Value_4
 SELECT 'JOIN ON with conditions';
 JOIN ON with conditions
 SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value
 FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t1.value == 'Join_1_Value_0'
-FULL JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0';
+FULL JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0' ORDER BY ALL;
+0		0		1	Join_3_Value_1
+0		0		4	Join_3_Value_4
+0		1	Join_2_Value_1	0	
+0		3	Join_2_Value_3	0	
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	0		0	
 2	Join_1_Value_2	0		0	
-0		3	Join_2_Value_3	0	
-0		1	Join_2_Value_1	0	
-0		0		4	Join_3_Value_4
-0		0		1	Join_3_Value_1
 SELECT '--';
 --
 SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value
 FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t1.value == 'Join_1_Value_0' AND t2.value == 'Join_2_Value_0'
-FULL JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0' AND t3.value == 'Join_3_Value_0';
+FULL JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0' AND t3.value == 'Join_3_Value_0' ORDER BY ALL;
+0		0		1	Join_3_Value_1
+0		0		4	Join_3_Value_4
+0		1	Join_2_Value_1	0	
+0		3	Join_2_Value_3	0	
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	0		0	
 2	Join_1_Value_2	0		0	
-0		1	Join_2_Value_1	0	
-0		3	Join_2_Value_3	0	
-0		0		1	Join_3_Value_1
-0		0		4	Join_3_Value_4
 SELECT 'JOIN multiple clauses';
 JOIN multiple clauses
 SELECT t1.value, t2.value, t3.value
 FROM test_table_join_1 AS t1
 FULL JOIN test_table_join_2 AS t2 ON t1.id = t2.id OR t1.id = t2.id
-FULL JOIN test_table_join_3 AS t3 ON t2.id = t3.id OR t3.id = t2.id;
+FULL JOIN test_table_join_3 AS t3 ON t2.id = t3.id OR t3.id = t2.id ORDER BY ALL;
+		Join_3_Value_4
+	Join_2_Value_3	
 Join_1_Value_0	Join_2_Value_0	Join_3_Value_0
 Join_1_Value_1	Join_2_Value_1	Join_3_Value_1
 Join_1_Value_2		Join_3_Value_0
-	Join_2_Value_3	
-		Join_3_Value_4
 SELECT 'JOIN expression aliases';
 JOIN expression aliases
 SELECT t1_id, t1.value, t2_id, t2.value, t3_id, t3.value
 FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 ON (t1.id AS t1_id) = (t2.id AS t2_id)
-FULL JOIN test_table_join_3 AS t3 ON t2_id = (t3.id AS t3_id);
+FULL JOIN test_table_join_3 AS t3 ON t2_id = (t3.id AS t3_id) ORDER BY ALL;
+0		0		4	Join_3_Value_4
+0		3	Join_2_Value_3	0	
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
 2	Join_1_Value_2	0		0	Join_3_Value_0
-0		3	Join_2_Value_3	0	
-0		0		4	Join_3_Value_4
 SELECT '--';
 --
 SELECT t1.id AS t1_id, t1.value, t2.id AS t2_id, t2.value, t3.id AS t3_id, t3.value
 FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 ON t1_id = t2_id
-FULL JOIN test_table_join_3 AS t3 ON t2_id = t3_id;
+FULL JOIN test_table_join_3 AS t3 ON t2_id = t3_id ORDER BY ALL;
+0		0		4	Join_3_Value_4
+0		3	Join_2_Value_3	0	
 0	Join_1_Value_0	0	Join_2_Value_0	0	Join_3_Value_0
 1	Join_1_Value_1	1	Join_2_Value_1	1	Join_3_Value_1
 2	Join_1_Value_2	0		0	Join_3_Value_0
-0		3	Join_2_Value_3	0	
-0		0		4	Join_3_Value_4
diff --git a/tests/queries/0_stateless/02372_analyzer_join.sql.j2 b/tests/queries/0_stateless/02372_analyzer_join.sql.j2
index facf4dc018b..45ae63b9a49 100644
--- a/tests/queries/0_stateless/02372_analyzer_join.sql.j2
+++ b/tests/queries/0_stateless/02372_analyzer_join.sql.j2
@@ -45,59 +45,59 @@ SELECT 'JOIN {{ join_type }}';
 SELECT 'JOIN ON without conditions';
 
 SELECT test_table_join_1.id, test_table_join_1.value, test_table_join_2.id, test_table_join_2.value
-FROM test_table_join_1 {{ join_type }} JOIN test_table_join_2 ON test_table_join_1.id = test_table_join_2.id;
+FROM test_table_join_1 {{ join_type }} JOIN test_table_join_2 ON test_table_join_1.id = test_table_join_2.id ORDER BY ALL;
 
 SELECT '--';
 
 SELECT t1.id, t1.value, t2.id, t2.value
-FROM test_table_join_1 AS t1 {{ join_type }} JOIN test_table_join_2 AS t2 ON t1.id = t2.id;
+FROM test_table_join_1 AS t1 {{ join_type }} JOIN test_table_join_2 AS t2 ON t1.id = t2.id ORDER BY ALL;
 
 SELECT '--';
 
 SELECT t1.id, test_table_join_1.id, t1.value, test_table_join_1.value, t2.id, test_table_join_2.id, t2.value, test_table_join_2.value
-FROM test_table_join_1 AS t1 {{ join_type }} JOIN test_table_join_2 AS t2 ON t1.id = t2.id;
+FROM test_table_join_1 AS t1 {{ join_type }} JOIN test_table_join_2 AS t2 ON t1.id = t2.id ORDER BY ALL;
 
 SELECT '--';
 
 SELECT t1.value, t2.value
-FROM test_table_join_1 AS t1 {{ join_type }} JOIN test_table_join_2 AS t2 ON t1.id = t2.id;
+FROM test_table_join_1 AS t1 {{ join_type }} JOIN test_table_join_2 AS t2 ON t1.id = t2.id ORDER BY ALL;
 
-SELECT id FROM test_table_join_1 {{ join_type }} JOIN test_table_join_2 ON test_table_join_1.id = test_table_join_2.id; -- { serverError AMBIGUOUS_IDENTIFIER }
+SELECT id FROM test_table_join_1 {{ join_type }} JOIN test_table_join_2 ON test_table_join_1.id = test_table_join_2.id ORDER BY ALL; -- { serverError AMBIGUOUS_IDENTIFIER }
 
-SELECT value FROM test_table_join_1 {{ join_type }} JOIN test_table_join_2 ON test_table_join_1.id = test_table_join_2.id; -- { serverError AMBIGUOUS_IDENTIFIER }
+SELECT value FROM test_table_join_1 {{ join_type }} JOIN test_table_join_2 ON test_table_join_1.id = test_table_join_2.id ORDER BY ALL; -- { serverError AMBIGUOUS_IDENTIFIER }
 
 SELECT 'JOIN ON with conditions';
 
 SELECT t1.id, t1.value, t2.id, t2.value
-FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t1.value = 'Join_1_Value_0';
+FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t1.value = 'Join_1_Value_0' ORDER BY ALL;
 
 SELECT '--';
 
 SELECT t1.id, t1.value, t2.id, t2.value
-FROM test_table_join_1 AS t1 {{ join_type }} JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t2.value = 'Join_2_Value_0';
+FROM test_table_join_1 AS t1 {{ join_type }} JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t2.value = 'Join_2_Value_0' ORDER BY ALL;
 
 SELECT '--';
 
 SELECT t1.id, t1.value, t2.id, t2.value
-FROM test_table_join_1 AS t1 {{ join_type }} JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t1.value = 'Join_1_Value_0' AND t2.value = 'Join_2_Value_0';
+FROM test_table_join_1 AS t1 {{ join_type }} JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t1.value = 'Join_1_Value_0' AND t2.value = 'Join_2_Value_0' ORDER BY ALL;
 
 SELECT '--';
 
 SELECT t1.id, t1.value, t2.id, t2.value
-FROM test_table_join_1 AS t1 {{ join_type }} JOIN test_table_join_2 AS t2 ON toString(t1.id) = toString(t2.id) AND t1.value = 'Join_1_Value_0' AND t2.value = 'Join_2_Value_0';
+FROM test_table_join_1 AS t1 {{ join_type }} JOIN test_table_join_2 AS t2 ON toString(t1.id) = toString(t2.id) AND t1.value = 'Join_1_Value_0' AND t2.value = 'Join_2_Value_0' ORDER BY ALL;
 
 SELECT 'JOIN multiple clauses';
 
 SELECT t1.id, t1.value, t2.id, t2.value
-FROM test_table_join_1 AS t1 {{ join_type }} JOIN test_table_join_2 AS t2 ON t1.id = t2.id OR t1.id = t2.id;
+FROM test_table_join_1 AS t1 {{ join_type }} JOIN test_table_join_2 AS t2 ON t1.id = t2.id OR t1.id = t2.id ORDER BY ALL;
 
 SELECT 'JOIN expression aliases';
 
-SELECT t1_id, t1.value, t2_id, t2.value FROM test_table_join_1 AS t1 {{ join_type }} JOIN test_table_join_2 AS t2 ON (t1.id AS t1_id) = (t2.id AS t2_id);
+SELECT t1_id, t1.value, t2_id, t2.value FROM test_table_join_1 AS t1 {{ join_type }} JOIN test_table_join_2 AS t2 ON (t1.id AS t1_id) = (t2.id AS t2_id) ORDER BY ALL;
 
 SELECT '--';
 
-SELECT t1.id AS t1_id, t1.value, t2.id AS t2_id, t2.value FROM test_table_join_1 AS t1 {{ join_type }} JOIN test_table_join_2 AS t2 ON t1_id = t2_id;
+SELECT t1.id AS t1_id, t1.value, t2.id AS t2_id, t2.value FROM test_table_join_1 AS t1 {{ join_type }} JOIN test_table_join_2 AS t2 ON t1_id = t2_id ORDER BY ALL;
 
 {% endfor %}
 
@@ -110,56 +110,56 @@ SELECT 'JOIN ON without conditions';
 
 SELECT test_table_join_1.id, test_table_join_1.value, test_table_join_2.id, test_table_join_2.value, test_table_join_3.id, test_table_join_3.value
 FROM test_table_join_1 {{ first_join_type }} JOIN test_table_join_2 ON test_table_join_1.id = test_table_join_2.id
-{{ second_join_type }} JOIN test_table_join_3 ON test_table_join_2.id = test_table_join_3.id;
+{{ second_join_type }} JOIN test_table_join_3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
 
 SELECT '--';
 
 SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value
 FROM test_table_join_1 AS t1 {{ first_join_type }} JOIN test_table_join_2 AS t2 ON test_table_join_1.id = test_table_join_2.id
-{{ second_join_type }} JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id;
+{{ second_join_type }} JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
 
 SELECT '--';
 
 SELECT t1.id, test_table_join_1.id, t1.value, test_table_join_1.value, t2.id, test_table_join_2.id, t2.value, test_table_join_2.value,
 t3.id, test_table_join_3.id, t3.value, test_table_join_3.value
 FROM test_table_join_1 AS t1 {{ first_join_type }} JOIN test_table_join_2 AS t2 ON test_table_join_1.id = test_table_join_2.id
-{{ second_join_type }} JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id;
+{{ second_join_type }} JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
 
 SELECT '--';
 SELECT t1.value, t2.value, t3.value
 FROM test_table_join_1 AS t1 {{ first_join_type }} JOIN test_table_join_2 AS t2 ON test_table_join_1.id = test_table_join_2.id
-{{ second_join_type }} JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id;
+{{ second_join_type }} JOIN test_table_join_3 AS t3 ON test_table_join_2.id = test_table_join_3.id  ORDER BY ALL;
 
 SELECT 'JOIN ON with conditions';
 
 SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value
 FROM test_table_join_1 AS t1 {{ first_join_type }} JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t1.value == 'Join_1_Value_0'
-{{ second_join_type }} JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0';
+{{ second_join_type }} JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0' ORDER BY ALL;
 
 SELECT '--';
 
 SELECT t1.id, t1.value, t2.id, t2.value, t3.id, t3.value
 FROM test_table_join_1 AS t1 {{ first_join_type }} JOIN test_table_join_2 AS t2 ON t1.id = t2.id AND t1.value == 'Join_1_Value_0' AND t2.value == 'Join_2_Value_0'
-{{ second_join_type }} JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0' AND t3.value == 'Join_3_Value_0';
+{{ second_join_type }} JOIN test_table_join_3 AS t3 ON t2.id = t3.id AND t2.value == 'Join_2_Value_0' AND t3.value == 'Join_3_Value_0' ORDER BY ALL;
 
 SELECT 'JOIN multiple clauses';
 
 SELECT t1.value, t2.value, t3.value
 FROM test_table_join_1 AS t1
 {{ first_join_type }} JOIN test_table_join_2 AS t2 ON t1.id = t2.id OR t1.id = t2.id
-{{ second_join_type }} JOIN test_table_join_3 AS t3 ON t2.id = t3.id OR t3.id = t2.id;
+{{ second_join_type }} JOIN test_table_join_3 AS t3 ON t2.id = t3.id OR t3.id = t2.id ORDER BY ALL;
 
 SELECT 'JOIN expression aliases';
 
 SELECT t1_id, t1.value, t2_id, t2.value, t3_id, t3.value
 FROM test_table_join_1 AS t1 {{ first_join_type }} JOIN test_table_join_2 AS t2 ON (t1.id AS t1_id) = (t2.id AS t2_id)
-{{ second_join_type }} JOIN test_table_join_3 AS t3 ON t2_id = (t3.id AS t3_id);
+{{ second_join_type }} JOIN test_table_join_3 AS t3 ON t2_id = (t3.id AS t3_id) ORDER BY ALL;
 
 SELECT '--';
 
 SELECT t1.id AS t1_id, t1.value, t2.id AS t2_id, t2.value, t3.id AS t3_id, t3.value
 FROM test_table_join_1 AS t1 {{ first_join_type }} JOIN test_table_join_2 AS t2 ON t1_id = t2_id
-{{ second_join_type }} JOIN test_table_join_3 AS t3 ON t2_id = t3_id;
+{{ second_join_type }} JOIN test_table_join_3 AS t3 ON t2_id = t3_id ORDER BY ALL;
 
 {% endfor %}
 {% endfor %}
diff --git a/tests/queries/0_stateless/02373_analyzer_join_use_nulls.reference b/tests/queries/0_stateless/02373_analyzer_join_use_nulls.reference
index 3722c23e4a0..2b3671e1ea6 100644
--- a/tests/queries/0_stateless/02373_analyzer_join_use_nulls.reference
+++ b/tests/queries/0_stateless/02373_analyzer_join_use_nulls.reference
@@ -1,27 +1,27 @@
 -- { echoOn }
 
 SELECT t1.id AS t1_id, toTypeName(t1_id), t1.value AS t1_value, toTypeName(t1_value), t2.id AS t2_id, toTypeName(t2_id), t2.value AS t2_value, toTypeName(t2_value)
-FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON t1.id = t2.id;
+FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON t1.id = t2.id ORDER BY ALL;
 0	UInt64	Join_1_Value_0	String	0	UInt64	Join_2_Value_0	String
 1	UInt64	Join_1_Value_1	String	1	UInt64	Join_2_Value_1	String
 SELECT '--';
 --
 SELECT t1.id AS t1_id, toTypeName(t1_id), t1.value AS t1_value, toTypeName(t1_value), t2.id AS t2_id, toTypeName(t2_id), t2.value AS t2_value, toTypeName(t2_value)
-FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 ON t1.id = t2.id;
+FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 ON t1.id = t2.id ORDER BY ALL;
 0	UInt64	Join_1_Value_0	String	0	Nullable(UInt64)	Join_2_Value_0	Nullable(String)
 1	UInt64	Join_1_Value_1	String	1	Nullable(UInt64)	Join_2_Value_1	Nullable(String)
 2	UInt64	Join_1_Value_2	String	\N	Nullable(UInt64)	\N	Nullable(String)
 SELECT '--';
 --
 SELECT t1.id AS t1_id, toTypeName(t1_id), t1.value AS t1_value, toTypeName(t1_value), t2.id AS t2_id, toTypeName(t2_id), t2.value AS t2_value, toTypeName(t2_value)
-FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 ON t1.id = t2.id;
+FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 ON t1.id = t2.id ORDER BY ALL;
 0	Nullable(UInt64)	Join_1_Value_0	Nullable(String)	0	UInt64	Join_2_Value_0	String
 1	Nullable(UInt64)	Join_1_Value_1	Nullable(String)	1	UInt64	Join_2_Value_1	String
 \N	Nullable(UInt64)	\N	Nullable(String)	3	UInt64	Join_2_Value_3	String
 SELECT '--';
 --
 SELECT t1.id AS t1_id, toTypeName(t1_id), t1.value AS t1_value, toTypeName(t1_value), t2.id AS t2_id, toTypeName(t2_id), t2.value AS t2_value, toTypeName(t2_value)
-FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 ON t1.id = t2.id;
+FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 ON t1.id = t2.id ORDER BY ALL;
 0	Nullable(UInt64)	Join_1_Value_0	Nullable(String)	0	Nullable(UInt64)	Join_2_Value_0	Nullable(String)
 1	Nullable(UInt64)	Join_1_Value_1	Nullable(String)	1	Nullable(UInt64)	Join_2_Value_1	Nullable(String)
 2	Nullable(UInt64)	Join_1_Value_2	Nullable(String)	\N	Nullable(UInt64)	\N	Nullable(String)
@@ -30,14 +30,14 @@ SELECT '--';
 --
 SELECT id AS using_id, toTypeName(using_id), t1.id AS t1_id, toTypeName(t1_id), t1.value AS t1_value, toTypeName(t1_value),
 t2.id AS t2_id, toTypeName(t2_id), t2.value AS t2_value, toTypeName(t2_value)
-FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 USING (id);
+FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 USING (id) ORDER BY ALL;
 0	UInt64	0	UInt64	Join_1_Value_0	String	0	UInt64	Join_2_Value_0	String
 1	UInt64	1	UInt64	Join_1_Value_1	String	1	UInt64	Join_2_Value_1	String
 SELECT '--';
 --
 SELECT id AS using_id, toTypeName(using_id), t1.id AS t1_id, toTypeName(t1_id), t1.value AS t1_value, toTypeName(t1_value),
 t2.id AS t2_id, toTypeName(t2_id), t2.value AS t2_value, toTypeName(t2_value)
-FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 USING (id);
+FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 USING (id) ORDER BY ALL;
 0	UInt64	0	UInt64	Join_1_Value_0	String	0	Nullable(UInt64)	Join_2_Value_0	Nullable(String)
 1	UInt64	1	UInt64	Join_1_Value_1	String	1	Nullable(UInt64)	Join_2_Value_1	Nullable(String)
 2	UInt64	2	UInt64	Join_1_Value_2	String	\N	Nullable(UInt64)	\N	Nullable(String)
@@ -45,7 +45,7 @@ SELECT '--';
 --
 SELECT id AS using_id, toTypeName(using_id), t1.id AS t1_id, toTypeName(t1_id), t1.value AS t1_value, toTypeName(t1_value),
 t2.id AS t2_id, toTypeName(t2_id), t2.value AS t2_value, toTypeName(t2_value)
-FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 USING (id);
+FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 USING (id) ORDER BY ALL;
 0	UInt64	0	Nullable(UInt64)	Join_1_Value_0	Nullable(String)	0	UInt64	Join_2_Value_0	String
 1	UInt64	1	Nullable(UInt64)	Join_1_Value_1	Nullable(String)	1	UInt64	Join_2_Value_1	String
 3	UInt64	\N	Nullable(UInt64)	\N	Nullable(String)	3	UInt64	Join_2_Value_3	String
@@ -53,7 +53,7 @@ SELECT '--';
 --
 SELECT id AS using_id, toTypeName(using_id), t1.id AS t1_id, toTypeName(t1_id), t1.value AS t1_value, toTypeName(t1_value),
 t2.id AS t2_id, toTypeName(t2_id), t2.value AS t2_value, toTypeName(t2_value)
-FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 USING (id);
+FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 USING (id) ORDER BY ALL;
 0	Nullable(UInt64)	0	Nullable(UInt64)	Join_1_Value_0	Nullable(String)	0	Nullable(UInt64)	Join_2_Value_0	Nullable(String)
 1	Nullable(UInt64)	1	Nullable(UInt64)	Join_1_Value_1	Nullable(String)	1	Nullable(UInt64)	Join_2_Value_1	Nullable(String)
 2	Nullable(UInt64)	2	Nullable(UInt64)	Join_1_Value_2	Nullable(String)	\N	Nullable(UInt64)	\N	Nullable(String)
diff --git a/tests/queries/0_stateless/02373_analyzer_join_use_nulls.sql b/tests/queries/0_stateless/02373_analyzer_join_use_nulls.sql
index db7895084e8..bcec6d178a8 100644
--- a/tests/queries/0_stateless/02373_analyzer_join_use_nulls.sql
+++ b/tests/queries/0_stateless/02373_analyzer_join_use_nulls.sql
@@ -26,46 +26,46 @@ INSERT INTO test_table_join_2 VALUES (3, 'Join_2_Value_3');
 -- { echoOn }
 
 SELECT t1.id AS t1_id, toTypeName(t1_id), t1.value AS t1_value, toTypeName(t1_value), t2.id AS t2_id, toTypeName(t2_id), t2.value AS t2_value, toTypeName(t2_value)
-FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON t1.id = t2.id;
+FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 ON t1.id = t2.id ORDER BY ALL;
 
 SELECT '--';
 
 SELECT t1.id AS t1_id, toTypeName(t1_id), t1.value AS t1_value, toTypeName(t1_value), t2.id AS t2_id, toTypeName(t2_id), t2.value AS t2_value, toTypeName(t2_value)
-FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 ON t1.id = t2.id;
+FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 ON t1.id = t2.id ORDER BY ALL;
 
 SELECT '--';
 
 SELECT t1.id AS t1_id, toTypeName(t1_id), t1.value AS t1_value, toTypeName(t1_value), t2.id AS t2_id, toTypeName(t2_id), t2.value AS t2_value, toTypeName(t2_value)
-FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 ON t1.id = t2.id;
+FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 ON t1.id = t2.id ORDER BY ALL;
 
 SELECT '--';
 
 SELECT t1.id AS t1_id, toTypeName(t1_id), t1.value AS t1_value, toTypeName(t1_value), t2.id AS t2_id, toTypeName(t2_id), t2.value AS t2_value, toTypeName(t2_value)
-FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 ON t1.id = t2.id;
+FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 ON t1.id = t2.id ORDER BY ALL;
 
 SELECT '--';
 
 SELECT id AS using_id, toTypeName(using_id), t1.id AS t1_id, toTypeName(t1_id), t1.value AS t1_value, toTypeName(t1_value),
 t2.id AS t2_id, toTypeName(t2_id), t2.value AS t2_value, toTypeName(t2_value)
-FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 USING (id);
+FROM test_table_join_1 AS t1 INNER JOIN test_table_join_2 AS t2 USING (id) ORDER BY ALL;
 
 SELECT '--';
 
 SELECT id AS using_id, toTypeName(using_id), t1.id AS t1_id, toTypeName(t1_id), t1.value AS t1_value, toTypeName(t1_value),
 t2.id AS t2_id, toTypeName(t2_id), t2.value AS t2_value, toTypeName(t2_value)
-FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 USING (id);
+FROM test_table_join_1 AS t1 LEFT JOIN test_table_join_2 AS t2 USING (id) ORDER BY ALL;
 
 SELECT '--';
 
 SELECT id AS using_id, toTypeName(using_id), t1.id AS t1_id, toTypeName(t1_id), t1.value AS t1_value, toTypeName(t1_value),
 t2.id AS t2_id, toTypeName(t2_id), t2.value AS t2_value, toTypeName(t2_value)
-FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 USING (id);
+FROM test_table_join_1 AS t1 RIGHT JOIN test_table_join_2 AS t2 USING (id) ORDER BY ALL;
 
 SELECT '--';
 
 SELECT id AS using_id, toTypeName(using_id), t1.id AS t1_id, toTypeName(t1_id), t1.value AS t1_value, toTypeName(t1_value),
 t2.id AS t2_id, toTypeName(t2_id), t2.value AS t2_value, toTypeName(t2_value)
-FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 USING (id);
+FROM test_table_join_1 AS t1 FULL JOIN test_table_join_2 AS t2 USING (id) ORDER BY ALL;
 
 -- { echoOff }
 
diff --git a/tests/queries/0_stateless/02992_all_columns_should_have_comment.sql b/tests/queries/0_stateless/02992_all_columns_should_have_comment.sql
index 127c6fee07d..dcb7c09a973 100644
--- a/tests/queries/0_stateless/02992_all_columns_should_have_comment.sql
+++ b/tests/queries/0_stateless/02992_all_columns_should_have_comment.sql
@@ -1,4 +1,8 @@
 SYSTEM FLUSH LOGS;
 SELECT 'Column ' || name || ' from table ' || concat(database, '.', table) || ' should have a comment'
 FROM system.columns
-WHERE (database = 'system') AND (comment = '') AND (table NOT ILIKE '%_log_%') AND (table NOT IN ('numbers', 'numbers_mt', 'one', 'generate_series', 'generateSeries', 'coverage_log')) AND (default_kind != 'ALIAS');
+WHERE (database = 'system') AND
+      (comment = '') AND
+      (table NOT ILIKE '%_log_%') AND
+      (table NOT IN ('numbers', 'numbers_mt', 'one', 'generate_series', 'generateSeries', 'coverage_log', 'filesystem_read_prefetches_log')) AND
+      (default_kind != 'ALIAS');

From d0053b505b7d05379644d5d0f47ec704d7496294 Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Mon, 22 Jul 2024 12:56:35 +0200
Subject: [PATCH 0571/1488] Rename test

---
 ...p.reference => 03208_keepermap_incomplete_data_drop.reference} | 0
 ...led_data_drop.sql => 03208_keepermap_incomplete_data_drop.sql} | 0
 2 files changed, 0 insertions(+), 0 deletions(-)
 rename tests/queries/0_stateless/{03208_keepermap_failed_data_drop.reference => 03208_keepermap_incomplete_data_drop.reference} (100%)
 rename tests/queries/0_stateless/{03208_keepermap_failed_data_drop.sql => 03208_keepermap_incomplete_data_drop.sql} (100%)

diff --git a/tests/queries/0_stateless/03208_keepermap_failed_data_drop.reference b/tests/queries/0_stateless/03208_keepermap_incomplete_data_drop.reference
similarity index 100%
rename from tests/queries/0_stateless/03208_keepermap_failed_data_drop.reference
rename to tests/queries/0_stateless/03208_keepermap_incomplete_data_drop.reference
diff --git a/tests/queries/0_stateless/03208_keepermap_failed_data_drop.sql b/tests/queries/0_stateless/03208_keepermap_incomplete_data_drop.sql
similarity index 100%
rename from tests/queries/0_stateless/03208_keepermap_failed_data_drop.sql
rename to tests/queries/0_stateless/03208_keepermap_incomplete_data_drop.sql

From e708219f6aecfe9934827f453665e03142ad5112 Mon Sep 17 00:00:00 2001
From: Max K <maxkaynov@gmail.com>
Date: Mon, 22 Jul 2024 13:01:27 +0200
Subject: [PATCH 0572/1488] CI: Print instance info in runner's init script

---
 tests/ci/worker/init_runner.sh | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/tests/ci/worker/init_runner.sh b/tests/ci/worker/init_runner.sh
index d6cdb6d9c57..1bfeeb38c15 100644
--- a/tests/ci/worker/init_runner.sh
+++ b/tests/ci/worker/init_runner.sh
@@ -50,7 +50,7 @@ set -uo pipefail
 # set accordingly to a runner role #
 ####################################
 
-echo "Running init v1"
+echo "Running init v1.1"
 export DEBIAN_FRONTEND=noninteractive
 export RUNNER_HOME=/home/ubuntu/actions-runner
 
@@ -66,6 +66,11 @@ bash /usr/local/share/scripts/init-network.sh
 RUNNER_TYPE=$(/usr/local/bin/aws ec2 describe-tags --filters "Name=resource-id,Values=$INSTANCE_ID" --query "Tags[?Key=='github:runner-type'].Value" --output text)
 LABELS="self-hosted,Linux,$(uname -m),$RUNNER_TYPE"
 export LABELS
+echo "Instance Labels: $LABELS"
+
+LIFE_CYCLE=$(curl -s --fail http://169.254.169.254/latest/meta-data/instance-life-cycle)
+export LIFE_CYCLE
+echo "Instance lifecycle: $LIFE_CYCLE"
 
 # Refresh CloudWatch agent config
 aws ssm get-parameter --region us-east-1 --name AmazonCloudWatch-github-runners --query 'Parameter.Value' --output text > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json
@@ -124,10 +129,6 @@ terminate_decrease_and_exit() {
 declare -f terminate_and_exit >> /tmp/actions-hooks/common.sh
 
 check_spot_instance_is_old() {
-    # This function should be executed ONLY BETWEEN runnings.
-    # It's unsafe to execute while the runner is working!
-    local LIFE_CYCLE
-    LIFE_CYCLE=$(curl -s --fail http://169.254.169.254/latest/meta-data/instance-life-cycle)
     if [ "$LIFE_CYCLE" == "spot" ]; then
         local UPTIME
         UPTIME=$(< /proc/uptime)

From a493e5c8e7e885cc8b66626ebf2911a6e3387b78 Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Mon, 22 Jul 2024 11:05:37 +0000
Subject: [PATCH 0573/1488] Followup #66725

---
 .../Passes/LogicalExpressionOptimizerPass.cpp         | 11 +++++++++--
 .../02911_join_on_nullsafe_optimization.reference     |  2 ++
 .../02911_join_on_nullsafe_optimization.sql           |  2 ++
 3 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp
index 5c68bca3a6e..e136440556f 100644
--- a/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp
+++ b/src/Analyzer/Passes/LogicalExpressionOptimizerPass.cpp
@@ -68,10 +68,13 @@ QueryTreeNodePtr findEqualsFunction(const QueryTreeNodes & nodes)
     return nullptr;
 }
 
-/// Checks if the node is combination of isNull and notEquals functions of two the same arguments
+/// Checks if the node is combination of isNull and notEquals functions of two the same arguments:
+/// [ (a <> b AND) ] (a IS NULL) AND (b IS NULL)
 bool matchIsNullOfTwoArgs(const QueryTreeNodes & nodes, QueryTreeNodePtr & lhs, QueryTreeNodePtr & rhs)
 {
     QueryTreeNodePtrWithHashSet all_arguments;
+    QueryTreeNodePtrWithHashSet is_null_arguments;
+
     for (const auto & node : nodes)
     {
         const auto * func_node = node->as<FunctionNode>();
@@ -80,7 +83,11 @@ bool matchIsNullOfTwoArgs(const QueryTreeNodes & nodes, QueryTreeNodePtr & lhs,
 
         const auto & arguments = func_node->getArguments().getNodes();
         if (func_node->getFunctionName() == "isNull" && arguments.size() == 1)
+        {
             all_arguments.insert(QueryTreeNodePtrWithHash(arguments[0]));
+            is_null_arguments.insert(QueryTreeNodePtrWithHash(arguments[0]));
+        }
+
         else if (func_node->getFunctionName() == "notEquals" && arguments.size() == 2)
         {
             if (arguments[0]->isEqual(*arguments[1]))
@@ -95,7 +102,7 @@ bool matchIsNullOfTwoArgs(const QueryTreeNodes & nodes, QueryTreeNodePtr & lhs,
             return false;
     }
 
-    if (all_arguments.size() != 2)
+    if (all_arguments.size() != 2 || is_null_arguments.size() != 2)
         return false;
 
     lhs = all_arguments.begin()->node;
diff --git a/tests/queries/0_stateless/02911_join_on_nullsafe_optimization.reference b/tests/queries/0_stateless/02911_join_on_nullsafe_optimization.reference
index 31a1cda18e7..8f194b4ffde 100644
--- a/tests/queries/0_stateless/02911_join_on_nullsafe_optimization.reference
+++ b/tests/queries/0_stateless/02911_join_on_nullsafe_optimization.reference
@@ -39,6 +39,8 @@ SELECT * FROM t1 JOIN t2 ON (t1.x == t2.x AND ((t2.x IS NOT NULL) AND (t1.x IS N
 2	2	2	2
 3	3	3	33
 \N	\N	\N	\N
+SELECT * FROM t1 JOIN t2 ON (t1.x == t2.x AND ((t2.x IS NOT NULL) AND (t1.x IS NOT NULL)) ) OR ( t2.x <> t1.x AND t2.x <> t1.x ) ORDER BY t1.x NULLS LAST; -- { serverError INVALID_JOIN_ON_EXPRESSION }
+SELECT * FROM t1 JOIN t2 ON (t1.x == t2.x AND ((t2.x IS NOT NULL) AND (t1.x IS NOT NULL)) ) OR ( t2.x <> t1.x AND (t2.x IS NULL) AND (t2.x IS NULL) ) ORDER BY t1.x NULLS LAST; -- { serverError INVALID_JOIN_ON_EXPRESSION }
 -- aliases defined in the join condition are valid
 -- FIXME(@vdimir) broken query formatting for the following queries:
 -- SELECT *, e, e2 FROM t1 FULL JOIN t2 ON ( ( ((t1.x == t2.x) AS e) AND ((t2.x IS NOT NULL) AND (t1.x IS NOT NULL)) ) OR ( (t2.x IS NULL) AND (t1.x IS NULL) ) AS e2 ) ORDER BY t1.x NULLS LAST, t2.x NULLS LAST;
diff --git a/tests/queries/0_stateless/02911_join_on_nullsafe_optimization.sql b/tests/queries/0_stateless/02911_join_on_nullsafe_optimization.sql
index f739259caf9..18cb303a54a 100644
--- a/tests/queries/0_stateless/02911_join_on_nullsafe_optimization.sql
+++ b/tests/queries/0_stateless/02911_join_on_nullsafe_optimization.sql
@@ -35,6 +35,8 @@ SELECT x = y OR (x IS NULL AND y IS NULL) FROM t1 ORDER BY x NULLS LAST;
 
 SELECT * FROM t1 JOIN t2 ON (t1.x == t2.x AND ((t2.x IS NOT NULL) AND (t1.x IS NOT NULL)) ) OR ( (t2.x IS NULL) AND (t1.x IS NULL) ) ORDER BY t1.x NULLS LAST;
 SELECT * FROM t1 JOIN t2 ON (t1.x == t2.x AND ((t2.x IS NOT NULL) AND (t1.x IS NOT NULL)) ) OR ( t2.x <> t1.x AND (t2.x IS NULL) AND (t1.x IS NULL) ) ORDER BY t1.x NULLS LAST;
+SELECT * FROM t1 JOIN t2 ON (t1.x == t2.x AND ((t2.x IS NOT NULL) AND (t1.x IS NOT NULL)) ) OR ( t2.x <> t1.x AND t2.x <> t1.x ) ORDER BY t1.x NULLS LAST; -- { serverError INVALID_JOIN_ON_EXPRESSION }
+SELECT * FROM t1 JOIN t2 ON (t1.x == t2.x AND ((t2.x IS NOT NULL) AND (t1.x IS NOT NULL)) ) OR ( t2.x <> t1.x AND (t2.x IS NULL) AND (t2.x IS NULL) ) ORDER BY t1.x NULLS LAST; -- { serverError INVALID_JOIN_ON_EXPRESSION }
 
 -- aliases defined in the join condition are valid
 -- FIXME(@vdimir) broken query formatting for the following queries:

From cd700c59b1d0b894a7a8461a33d994732b7864f0 Mon Sep 17 00:00:00 2001
From: Max K <maxkaynov@gmail.com>
Date: Mon, 22 Jul 2024 13:17:25 +0200
Subject: [PATCH 0574/1488] minor fix

---
 tests/ci/auto_release.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/ci/auto_release.py b/tests/ci/auto_release.py
index 39ab3156c80..f2386fe207f 100644
--- a/tests/ci/auto_release.py
+++ b/tests/ci/auto_release.py
@@ -191,7 +191,7 @@ def main():
                     title=f"Auto Release Status for {release_info.release_branch}",
                     body=release_info.to_dict(),
                 )
-    if args.post_auto_release_complete:
+    elif args.post_auto_release_complete:
         assert args.wf_status, "--wf-status Required with --post-auto-release-complete"
         if args.wf_status != SUCCESS:
             CIBuddy(dry_run=False).post_job_error(

From 8246614f5e5cdec99941f048f51cb28b9eee03a5 Mon Sep 17 00:00:00 2001
From: zoomxi <419486879@qq.com>
Date: Mon, 22 Jul 2024 17:57:30 +0800
Subject: [PATCH 0575/1488] throw if can't connect to any participating
 replicas

---
 src/Processors/QueryPlan/ReadFromRemote.cpp   |  3 +-
 .../ParallelReplicasReadingCoordinator.cpp    |  4 +--
 .../ParallelReplicasReadingCoordinator.h      |  4 +++
 .../test.py                                   | 35 +++++++++++++++++++
 4 files changed, 43 insertions(+), 3 deletions(-)

diff --git a/src/Processors/QueryPlan/ReadFromRemote.cpp b/src/Processors/QueryPlan/ReadFromRemote.cpp
index e27515a62a4..29e12c1e613 100644
--- a/src/Processors/QueryPlan/ReadFromRemote.cpp
+++ b/src/Processors/QueryPlan/ReadFromRemote.cpp
@@ -21,7 +21,7 @@
 #include <Client/ConnectionPoolWithFailover.h>
 #include <QueryPipeline/QueryPipelineBuilder.h>
 #include <Parsers/ASTFunction.h>
-
+#include <Storages/MergeTree/ParallelReplicasReadingCoordinator.h>
 #include <boost/algorithm/string/join.hpp>
 
 namespace DB
@@ -429,6 +429,7 @@ void ReadFromParallelRemoteReplicasStep::initializePipeline(QueryPipelineBuilder
     {
         shuffled_pool = shard.pool->getShuffledPools(current_settings);
         shuffled_pool.resize(max_replicas_to_use);
+        coordinator->adjustParticipatingReplicasCount(max_replicas_to_use);
     }
     else
     {
diff --git a/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp b/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp
index f46b4de10b7..2ba66256116 100644
--- a/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp
+++ b/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp
@@ -1031,7 +1031,7 @@ void ParallelReplicasReadingCoordinator::markReplicaAsUnavailable(size_t replica
     if (!pimpl)
     {
         unavailable_nodes_registered_before_initialization.push_back(replica_number);
-        if (unavailable_nodes_registered_before_initialization.size() == replicas_count)
+        if (unavailable_nodes_registered_before_initialization.size() == participating_replicas_count)
             throw Exception(ErrorCodes::ALL_CONNECTION_TRIES_FAILED, "Can't connect to any replica chosen for query execution");
     }
     else
@@ -1061,7 +1061,7 @@ void ParallelReplicasReadingCoordinator::initialize(CoordinationMode mode)
 }
 
 ParallelReplicasReadingCoordinator::ParallelReplicasReadingCoordinator(size_t replicas_count_, size_t mark_segment_size_)
-    : replicas_count(replicas_count_), mark_segment_size(mark_segment_size_)
+    : replicas_count(replicas_count_), participating_replicas_count(replicas_count_), mark_segment_size(mark_segment_size_)
 {
 }
 
diff --git a/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.h b/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.h
index 8b463fda395..c06ef6ef01a 100644
--- a/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.h
+++ b/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.h
@@ -30,11 +30,15 @@ public:
     /// needed to report total rows to read
     void setProgressCallback(ProgressCallback callback);
 
+    /// Participating replicas count may be less than replicas count in a shard.
+    void adjustParticipatingReplicasCount(size_t count) { participating_replicas_count = count; }
+
 private:
     void initialize(CoordinationMode mode);
 
     std::mutex mutex;
     const size_t replicas_count{0};
+    size_t participating_replicas_count{0};
     size_t mark_segment_size{0};
     std::unique_ptr<ImplInterface> pimpl;
     ProgressCallback progress_callback; // store the callback only to bypass it to coordinator implementation
diff --git a/tests/integration/test_parallel_replicas_no_replicas/test.py b/tests/integration/test_parallel_replicas_no_replicas/test.py
index 9f716459643..b77da338554 100644
--- a/tests/integration/test_parallel_replicas_no_replicas/test.py
+++ b/tests/integration/test_parallel_replicas_no_replicas/test.py
@@ -48,3 +48,38 @@ def test_skip_all_replicas(start_cluster, skip_unavailable_shards):
                 "skip_unavailable_shards": skip_unavailable_shards,
             },
         )
+
+@pytest.mark.parametrize("skip_unavailable_shards", [1, 0])
+def test_skip_all_participating_replicas1(start_cluster, skip_unavailable_shards):
+    cluster_name = "test_1_shard_3_unavaliable_replicas"
+    table_name = "tt1"
+    create_tables(cluster_name, table_name)
+
+    with pytest.raises(QueryRuntimeException):
+        initiator.query(
+            f"SELECT key, count() FROM {table_name}  GROUP BY key ORDER BY key",
+            settings={
+                "allow_experimental_parallel_reading_from_replicas": 2,
+                "max_parallel_replicas": 3,
+                "cluster_for_parallel_replicas": cluster_name,
+                "skip_unavailable_shards": skip_unavailable_shards,
+                "parallel_replicas_min_number_of_rows_per_replica": 500,
+            },
+        )
+
+@pytest.mark.parametrize("skip_unavailable_shards", [1, 0])
+def test_skip_all_participating_replicas2(start_cluster, skip_unavailable_shards):
+    cluster_name = "test_1_shard_3_unavaliable_replicas"
+    table_name = "tt2"
+    create_tables(cluster_name, table_name)
+
+    with pytest.raises(QueryRuntimeException):
+        initiator.query(
+            f"SELECT key, count() FROM {table_name}  GROUP BY key ORDER BY key",
+            settings={
+                "allow_experimental_parallel_reading_from_replicas": 2,
+                "max_parallel_replicas": 2,
+                "cluster_for_parallel_replicas": cluster_name,
+                "skip_unavailable_shards": skip_unavailable_shards,
+            },
+        )

From 2dc264928f311e2f4d10001044d070b6a6a05471 Mon Sep 17 00:00:00 2001
From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com>
Date: Mon, 22 Jul 2024 11:33:51 +0000
Subject: [PATCH 0576/1488] Added tests, rewritten logic which engines and
 table functions to allow, added replace for create table ... AS
 table_function() syntax.

---
 docs/en/operations/settings/settings.md       |  12 ++
 src/Core/Settings.h                           |   3 +-
 src/Core/SettingsChangesHistory.cpp           |   3 +-
 src/Interpreters/InterpreterCreateQuery.cpp   |  50 +++---
 .../test_restore_external_engines/__init__.py |   0
 .../configs/backups_disk.xml                  |  14 ++
 .../configs/remote_servers.xml                |  21 +++
 .../test_restore_external_engines/test.py     | 143 ++++++++++++++++++
 8 files changed, 217 insertions(+), 29 deletions(-)
 create mode 100644 tests/integration/test_restore_external_engines/__init__.py
 create mode 100644 tests/integration/test_restore_external_engines/configs/backups_disk.xml
 create mode 100644 tests/integration/test_restore_external_engines/configs/remote_servers.xml
 create mode 100644 tests/integration/test_restore_external_engines/test.py

diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md
index c3f697c3bdc..65b8df7a9e2 100644
--- a/docs/en/operations/settings/settings.md
+++ b/docs/en/operations/settings/settings.md
@@ -5608,3 +5608,15 @@ Default value: `10000000`.
 Minimal size of block to compress in CROSS JOIN. Zero value means - disable this threshold. This block is compressed when any of the two thresholds (by rows or by bytes) are reached.
 
 Default value: `1GiB`.
+
+## restore_replace_external_engines_to_null
+
+For testing purposes. Replaces all external engines to Null to not initiate external connections.
+
+Default value: `False`
+
+## restore_replace_external_table_functions_to_null
+
+For testing purposes. Replaces all external engines to Null to not initiate external connections.
+
+Default value: `False`
\ No newline at end of file
diff --git a/src/Core/Settings.h b/src/Core/Settings.h
index 7bf97896357..e6d2cac359b 100644
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@@ -891,7 +891,8 @@ class IColumn;
     M(Bool, optimize_distinct_in_order, true, "Enable DISTINCT optimization if some columns in DISTINCT form a prefix of sorting. For example, prefix of sorting key in merge tree or ORDER BY statement", 0) \
     M(Bool, keeper_map_strict_mode, false, "Enforce additional checks during operations on KeeperMap. E.g. throw an exception on an insert for already existing key", 0) \
     M(UInt64, extract_key_value_pairs_max_pairs_per_row, 1000, "Max number of pairs that can be produced by the `extractKeyValuePairs` function. Used as a safeguard against consuming too much memory.", 0) ALIAS(extract_kvp_max_pairs_per_row) \
-    M(Bool, restore_replace_external_engine_to_null, false, "Replace all the External table engines to Null on restore. Useful for testing purposes", 0) \
+    M(Bool, restore_replace_external_engines_to_null, false, "Replace all the External table engines to Null on restore. Useful for testing purposes", 0) \
+    M(Bool, restore_replace_external_table_functions_to_null, false, "Replace all table functions to Null on restore. Useful for testing purposes", 0) \
     \
     \
     /* ###################################### */ \
diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp
index a23d9d17da2..0abcfb0cfb9 100644
--- a/src/Core/SettingsChangesHistory.cpp
+++ b/src/Core/SettingsChangesHistory.cpp
@@ -79,7 +79,8 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
               {"ignore_on_cluster_for_replicated_named_collections_queries", false, false, "Ignore ON CLUSTER clause for replicated named collections management queries."},
               {"postgresql_connection_attempt_timeout", 2, 2, "Allow to control 'connect_timeout' parameter of PostgreSQL connection."},
               {"postgresql_connection_pool_retries", 2, 2, "Allow to control the number of retries in PostgreSQL connection pool."},
-              {"restore_replace_external_engine_to_null", false, false, "New setting."}
+              {"restore_replace_external_table_functions_to_null", false, false, "New setting."},
+              {"restore_replace_external_engins_to_null", false, false, "New setting."}
               }},
     {"24.6", {{"materialize_skip_indexes_on_insert", true, true, "Added new setting to allow to disable materialization of skip indexes on insert"},
               {"materialize_statistics_on_insert", true, true, "Added new setting to allow to disable materialization of statistics on insert"},
diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp
index 3b23c6899e9..70f2a7d01e5 100644
--- a/src/Interpreters/InterpreterCreateQuery.cpp
+++ b/src/Interpreters/InterpreterCreateQuery.cpp
@@ -968,12 +968,33 @@ namespace
         engine_ast->no_empty_args = true;
         storage.set(storage.engine, engine_ast);
     }
+
 }
 
 void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const
 {
+
     if (create.as_table_function)
+    {
+        if (getContext()->getSettingsRef().restore_replace_external_table_functions_to_null)
+        {
+            const auto & factory = TableFunctionFactory::instance();
+
+            auto properties = factory.tryGetProperties(create.as_table_function->as<ASTFunction>()->name);
+            if (properties && properties->allow_readonly)
+                return;
+            if (!create.storage)
+            {
+                auto storage_ast = std::make_shared<ASTStorage>();
+                create.set(create.storage, storage_ast);
+            }
+            else
+                throw Exception(ErrorCodes::LOGICAL_ERROR, "Storage should not be created yet, it's a bug.");
+            create.as_table_function = nullptr;
+            setNullTableEngine(*create.storage);
+        }
         return;
+    }
 
     if (create.is_dictionary || create.is_ordinary_view || create.is_live_view || create.is_window_view)
         return;
@@ -1010,34 +1031,9 @@ void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const
             setDefaultTableEngine(*create.storage, getContext()->getSettingsRef().default_table_engine.value);
         /// For exrternal tables with restore_replace_external_engine_to_null setting we replace external engines to
         /// Null table engine.
-        else if (create.storage->engine->name == "AzureBlobStorage" ||
-            create.storage->engine->name == "AzureQueue" ||
-            create.storage->engine->name == "COSN" ||
-            create.storage->engine->name == "DeltaLake" ||
-            create.storage->engine->name == "Dictionary" ||
-            create.storage->engine->name == "Executable" ||
-            create.storage->engine->name == "ExecutablePool" ||
-            create.storage->engine->name == "ExternalDistributed" ||
-            create.storage->engine->name == "File" ||
-            create.storage->engine->name == "Hudi" ||
-            create.storage->engine->name == "Iceberg" ||
-            create.storage->engine->name == "JDBC" ||
-            create.storage->engine->name == "Kafka" ||
-            create.storage->engine->name == "MaterializedPostgreSQL" ||
-            create.storage->engine->name == "MongoDB" ||
-            create.storage->engine->name == "MySQL" ||
-            create.storage->engine->name == "NATS" ||
-            create.storage->engine->name == "ODBC" ||
-            create.storage->engine->name == "OSS" ||
-            create.storage->engine->name == "PostgreSQL" ||
-            create.storage->engine->name == "RabbitMQ" ||
-            create.storage->engine->name == "Redis" ||
-            create.storage->engine->name == "S3" ||
-            create.storage->engine->name == "S3Queue" ||
-            create.storage->engine->name == "TinyLog" ||
-            create.storage->engine->name == "URL")
+        else if (getContext()->getSettingsRef().restore_replace_external_engines_to_null)
         {
-            if (getContext()->getSettingsRef().restore_replace_external_engine_to_null)
+            if (StorageFactory::instance().getStorageFeatures(create.storage->engine->name).source_access_type != AccessType::NONE)
                 setNullTableEngine(*create.storage);
         }
         return;
diff --git a/tests/integration/test_restore_external_engines/__init__.py b/tests/integration/test_restore_external_engines/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/integration/test_restore_external_engines/configs/backups_disk.xml b/tests/integration/test_restore_external_engines/configs/backups_disk.xml
new file mode 100644
index 00000000000..f7d666c6542
--- /dev/null
+++ b/tests/integration/test_restore_external_engines/configs/backups_disk.xml
@@ -0,0 +1,14 @@
+<clickhouse>
+    <storage_configuration>
+        <disks>
+            <backups>
+                <type>local</type>
+                <path>/backups/</path>
+            </backups>
+        </disks>
+    </storage_configuration>
+    <backups>
+        <allowed_disk>backups</allowed_disk>
+        <allowed_path>/backups/</allowed_path>
+    </backups>
+</clickhouse>
diff --git a/tests/integration/test_restore_external_engines/configs/remote_servers.xml b/tests/integration/test_restore_external_engines/configs/remote_servers.xml
new file mode 100644
index 00000000000..76ad3618339
--- /dev/null
+++ b/tests/integration/test_restore_external_engines/configs/remote_servers.xml
@@ -0,0 +1,21 @@
+<clickhouse>
+    <remote_servers>
+        <test_cluster>
+            <shard>
+                <internal_replication>true</internal_replication>
+                <replica>
+                    <host>replica1</host>
+                    <port>9000</port>
+                </replica>
+                <replica>
+                    <host>replica2</host>
+                    <port>9000</port>
+                </replica>
+                 <replica>
+                    <host>replica3</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+        </test_cluster>
+    </remote_servers>
+</clickhouse>
diff --git a/tests/integration/test_restore_external_engines/test.py b/tests/integration/test_restore_external_engines/test.py
new file mode 100644
index 00000000000..cde4b0deb00
--- /dev/null
+++ b/tests/integration/test_restore_external_engines/test.py
@@ -0,0 +1,143 @@
+import pytest
+
+import pymysql.cursors
+import pytest
+from helpers.cluster import ClickHouseCluster
+
+cluster = ClickHouseCluster(__file__)
+configs = ["configs/remote_servers.xml", "configs/backups_disk.xml"]
+
+node1 = cluster.add_instance("replica1", with_zookeeper=True, with_mysql8=True, main_configs=configs, external_dirs=["/backups/"])
+node2 = cluster.add_instance("replica2", with_zookeeper=True, with_mysql8=True, main_configs=configs, external_dirs=["/backups/"])
+node3 = cluster.add_instance("replica3", with_zookeeper=True, with_mysql8=True, main_configs=configs, external_dirs=["/backups/"])
+nodes = [node1, node2, node3]
+
+backup_id_counter = 0
+
+def new_backup_name():
+    global backup_id_counter
+    backup_id_counter += 1
+    return f"Disk('backups', '{backup_id_counter}/')"
+
+def cleanup_nodes(nodes, dbname):
+    for node in nodes:
+        node.query(f"DROP DATABASE IF EXISTS {dbname} SYNC")
+
+def fill_nodes(nodes, dbname):
+    cleanup_nodes(nodes, dbname)
+    for node in nodes:
+        node.query(f"CREATE DATABASE {dbname} ENGINE = Replicated('/clickhouse/databases/{dbname}', 'default', '{node.name}')")
+
+def drop_mysql_table(conn, tableName):
+    with conn.cursor() as cursor:
+        cursor.execute(f"DROP TABLE IF EXISTS `clickhouse`.`{tableName}`")
+
+def get_mysql_conn(cluster):
+    conn = pymysql.connect(
+        user="root", password="clickhouse", host=cluster.mysql8_ip, port=cluster.mysql8_port
+    )
+    return conn
+
+def fill_tables(cluster, dbname):
+    fill_nodes(nodes, dbname)
+
+    conn = get_mysql_conn(cluster)
+
+    with conn.cursor() as cursor:
+        cursor.execute(
+            "DROP DATABASE IF EXISTS clickhouse"
+        )
+        cursor.execute(
+            "CREATE DATABASE clickhouse"
+        )
+        cursor.execute(
+            "DROP TABLE IF EXISTS clickhouse.inference_table"
+        )
+        cursor.execute(
+            "CREATE TABLE clickhouse.inference_table (id INT PRIMARY KEY, data BINARY(16) NOT NULL)"
+        )
+        cursor.execute(
+            "INSERT INTO clickhouse.inference_table VALUES (100, X'9fad5e9eefdfb449')"
+        )
+        conn.commit()
+
+    parameters = "'mysql80:3306', 'clickhouse', 'inference_table', 'root', 'clickhouse'"
+
+    node1.query(
+        f"CREATE TABLE {dbname}.mysql_schema_inference_engine ENGINE=MySQL({parameters})"
+    )
+    node1.query(f"CREATE TABLE {dbname}.mysql_schema_inference_function AS mysql({parameters})")
+
+    node1.query(f"CREATE TABLE {dbname}.merge_tree (id UInt64, b String) ORDER BY id")
+    node1.query(f"INSERT INTO {dbname}.merge_tree VALUES (100, 'abc')")
+
+    expected = "id\tInt32\t\t\t\t\t\ndata\tFixedString(16)\t\t\t\t\t\n"
+    assert node1.query(f"DESCRIBE TABLE {dbname}.mysql_schema_inference_engine") == expected
+    assert node1.query(f"DESCRIBE TABLE {dbname}.mysql_schema_inference_function") == expected
+    assert node1.query(f"SELECT id FROM mysql({parameters})") == "100\n"
+    assert node1.query(f"SELECT id FROM {dbname}.mysql_schema_inference_engine") == "100\n"
+    assert node1.query(f"SELECT id FROM {dbname}.mysql_schema_inference_function") == "100\n"
+    assert node1.query(f"SELECT id FROM {dbname}.merge_tree") == "100\n"
+
+
+@pytest.fixture(scope="module")
+def start_cluster():
+    try:
+        cluster.start()
+        yield cluster
+
+    except Exception as ex:
+        print(ex)
+
+    finally:
+        cluster.shutdown()
+
+def test_restore_table(start_cluster):
+    fill_tables(cluster, "replicated")
+    backup_name = new_backup_name()
+    node2.query(f"SYSTEM SYNC DATABASE REPLICA replicated;")
+
+    node2.query(f"BACKUP DATABASE replicated TO {backup_name}")
+
+    node2.query("DROP TABLE replicated.mysql_schema_inference_engine")
+    node2.query("DROP TABLE replicated.mysql_schema_inference_function")
+
+    node3.query(f"SYSTEM SYNC DATABASE REPLICA replicated;")
+
+    assert node3.query("EXISTS replicated.mysql_schema_inference_engine") == "0\n"
+    assert node3.query("EXISTS replicated.mysql_schema_inference_function") == "0\n"
+
+    node3.query(f"RESTORE DATABASE replicated FROM {backup_name} SETTINGS allow_different_database_def=true")
+    node1.query(f"SYSTEM SYNC DATABASE REPLICA replicated;")
+
+    assert node1.query("SELECT count(), sum(id) FROM replicated.mysql_schema_inference_engine") == "1\t100\n"
+    assert node1.query("SELECT count(), sum(id) FROM replicated.mysql_schema_inference_function") == "1\t100\n"
+    assert node1.query("SELECT count(), sum(id) FROM replicated.merge_tree") == "1\t100\n"
+    cleanup_nodes(nodes, "replicated")
+
+
+def test_restore_table_null(start_cluster):
+    fill_tables(cluster, "replicated2")
+
+    backup_name = new_backup_name()
+    node2.query(f"SYSTEM SYNC DATABASE REPLICA replicated2;")
+
+    node2.query(f"BACKUP DATABASE replicated2 TO {backup_name}")
+
+    node2.query("DROP TABLE replicated2.mysql_schema_inference_engine")
+    node2.query("DROP TABLE replicated2.mysql_schema_inference_function")
+
+    node3.query(f"SYSTEM SYNC DATABASE REPLICA replicated2;")
+
+    assert node3.query("EXISTS replicated2.mysql_schema_inference_engine") == "0\n"
+    assert node3.query("EXISTS replicated2.mysql_schema_inference_function") == "0\n"
+
+    node3.query(f"RESTORE DATABASE replicated2 FROM {backup_name} SETTINGS allow_different_database_def=1, allow_different_table_def=1 SETTINGS restore_replace_external_engine_to_null=1, restore_replace_external_table_functions_to_null=1")
+    node1.query(f"SYSTEM SYNC DATABASE REPLICA replicated2;")
+
+    assert node1.query("SELECT count(), sum(id) FROM replicated2.mysql_schema_inference_engine") == "0\t0\n"
+    assert node1.query("SELECT count(), sum(id) FROM replicated2.mysql_schema_inference_function") == "0\t0\n"
+    assert node1.query("SELECT count(), sum(id) FROM replicated2.merge_tree") == "1\t100\n"
+    assert node1.query("SELECT engine FROM system.tables where database = 'replicated2' and name like '%mysql%'") == "Null\nNull\n"
+    assert node1.query("SELECT engine FROM system.tables where database = 'replicated2' and name like '%merge_tree%'") == "MergeTree\n"
+    cleanup_nodes(nodes, "replicated2")

From 69ad8feb9078421aca99709ffb839e2f3b923427 Mon Sep 17 00:00:00 2001
From: Max K <maxkaynov@gmail.com>
Date: Mon, 22 Jul 2024 13:38:10 +0200
Subject: [PATCH 0577/1488] add instance type

---
 tests/ci/worker/init_runner.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/ci/worker/init_runner.sh b/tests/ci/worker/init_runner.sh
index 1bfeeb38c15..5177e112edd 100644
--- a/tests/ci/worker/init_runner.sh
+++ b/tests/ci/worker/init_runner.sh
@@ -72,6 +72,9 @@ LIFE_CYCLE=$(curl -s --fail http://169.254.169.254/latest/meta-data/instance-lif
 export LIFE_CYCLE
 echo "Instance lifecycle: $LIFE_CYCLE"
 
+INSTANCE_TYPE=$(ec2metadata --instance-type)
+echo "Instance type: $INSTANCE_TYPE"
+
 # Refresh CloudWatch agent config
 aws ssm get-parameter --region us-east-1 --name AmazonCloudWatch-github-runners --query 'Parameter.Value' --output text > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json
 systemctl restart amazon-cloudwatch-agent.service

From af4c2fa8a405c53d7de6d9ed41d63988caf22b04 Mon Sep 17 00:00:00 2001
From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com>
Date: Mon, 22 Jul 2024 12:27:24 +0000
Subject: [PATCH 0578/1488] fix

---
 tests/integration/test_restore_external_engines/test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/test_restore_external_engines/test.py b/tests/integration/test_restore_external_engines/test.py
index cde4b0deb00..be2cae334e2 100644
--- a/tests/integration/test_restore_external_engines/test.py
+++ b/tests/integration/test_restore_external_engines/test.py
@@ -132,7 +132,7 @@ def test_restore_table_null(start_cluster):
     assert node3.query("EXISTS replicated2.mysql_schema_inference_engine") == "0\n"
     assert node3.query("EXISTS replicated2.mysql_schema_inference_function") == "0\n"
 
-    node3.query(f"RESTORE DATABASE replicated2 FROM {backup_name} SETTINGS allow_different_database_def=1, allow_different_table_def=1 SETTINGS restore_replace_external_engine_to_null=1, restore_replace_external_table_functions_to_null=1")
+    node3.query(f"RESTORE DATABASE replicated2 FROM {backup_name} SETTINGS allow_different_database_def=1, allow_different_table_def=1 SETTINGS restore_replace_external_engines_to_null=1, restore_replace_external_table_functions_to_null=1")
     node1.query(f"SYSTEM SYNC DATABASE REPLICA replicated2;")
 
     assert node1.query("SELECT count(), sum(id) FROM replicated2.mysql_schema_inference_engine") == "0\t0\n"

From 9f86b22d302c2feef5d666e75d11d09fe6ee1c6c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Mon, 22 Jul 2024 14:42:30 +0200
Subject: [PATCH 0579/1488] Speed up stateful tests table setup

---
 docker/test/stateful/run.sh                              | 6 +++---
 docker/test/stress/run.sh                                | 6 +++---
 tests/queries/1_stateful/00162_mmap_compression_none.sql | 2 +-
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/docker/test/stateful/run.sh b/docker/test/stateful/run.sh
index 857385f4715..df85c047649 100755
--- a/docker/test/stateful/run.sh
+++ b/docker/test/stateful/run.sh
@@ -191,8 +191,8 @@ else
             ENGINE = CollapsingMergeTree(Sign) PARTITION BY toYYYYMM(StartDate) ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID)
             SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192, storage_policy='s3_cache'"
 
-        clickhouse-client --query "INSERT INTO test.hits SELECT * FROM datasets.hits_v1 SETTINGS enable_filesystem_cache_on_write_operations=0"
-        clickhouse-client --query "INSERT INTO test.visits SELECT * FROM datasets.visits_v1 SETTINGS enable_filesystem_cache_on_write_operations=0"
+        clickhouse-client --query "INSERT INTO test.hits SELECT * FROM datasets.hits_v1 SETTINGS enable_filesystem_cache_on_write_operations=0, max_insert_threads=16"
+        clickhouse-client --query "INSERT INTO test.visits SELECT * FROM datasets.visits_v1 SETTINGS enable_filesystem_cache_on_write_operations=0, max_insert_threads=16"
         clickhouse-client --query "DROP TABLE datasets.visits_v1 SYNC"
         clickhouse-client --query "DROP TABLE datasets.hits_v1 SYNC"
     else
@@ -200,7 +200,7 @@ else
         clickhouse-client --query "RENAME TABLE datasets.visits_v1 TO test.visits"
     fi
     clickhouse-client --query "CREATE TABLE test.hits_s3  (WatchID UInt64, JavaEnable UInt8, Title String, GoodEvent Int16, EventTime DateTime, EventDate Date, CounterID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RegionID UInt32, UserID UInt64, CounterClass Int8, OS UInt8, UserAgent UInt8, URL String, Referer String, URLDomain String, RefererDomain String, Refresh UInt8, IsRobot UInt8, RefererCategories Array(UInt16), URLCategories Array(UInt16), URLRegions Array(UInt32), RefererRegions Array(UInt32), ResolutionWidth UInt16, ResolutionHeight UInt16, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, FlashMinor2 String, NetMajor UInt8, NetMinor UInt8, UserAgentMajor UInt16, UserAgentMinor FixedString(2), CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, MobilePhone UInt8, MobilePhoneModel String, Params String, IPNetworkID UInt32, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, IsArtifical UInt8, WindowClientWidth UInt16, WindowClientHeight UInt16, ClientTimeZone Int16, ClientEventTime DateTime, SilverlightVersion1 UInt8, SilverlightVersion2 UInt8, SilverlightVersion3 UInt32, SilverlightVersion4 UInt16, PageCharset String, CodeVersion UInt32, IsLink UInt8, IsDownload UInt8, IsNotBounce UInt8, FUniqID UInt64, HID UInt32, IsOldCounter UInt8, IsEvent UInt8, IsParameter UInt8, DontCountHits UInt8, WithHash UInt8, HitColor FixedString(1), UTCEventTime DateTime, Age UInt8, Sex UInt8, Income UInt8, Interests UInt16, Robotness UInt8, GeneralInterests Array(UInt16), RemoteIP UInt32, RemoteIP6 FixedString(16), WindowName Int32, OpenerName Int32, HistoryLength Int16, BrowserLanguage FixedString(2), BrowserCountry FixedString(2), SocialNetwork String, SocialAction String, HTTPError UInt16, SendTiming Int32, DNSTiming Int32, ConnectTiming Int32, ResponseStartTiming Int32, ResponseEndTiming Int32, FetchTiming Int32, RedirectTiming Int32, DOMInteractiveTiming Int32, DOMContentLoadedTiming Int32, DOMCompleteTiming Int32, LoadEventStartTiming Int32, LoadEventEndTiming Int32, NSToDOMContentLoadedTiming Int32, FirstPaintTiming Int32, RedirectCount Int8, SocialSourceNetworkID UInt8, SocialSourcePage String, ParamPrice Int64, ParamOrderID String, ParamCurrency FixedString(3), ParamCurrencyID UInt16, GoalsReached Array(UInt32), OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, RefererHash UInt64, URLHash UInt64, CLID UInt32, YCLID UInt64, ShareService String, ShareURL String, ShareTitle String, ParsedParams Nested(Key1 String, Key2 String, Key3 String, Key4 String, Key5 String, ValueDouble Float64), IslandID FixedString(16), RequestNum UInt32, RequestTry UInt8) ENGINE = MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192, storage_policy='s3_cache'"
-    clickhouse-client --query "INSERT INTO test.hits_s3 SELECT * FROM test.hits SETTINGS enable_filesystem_cache_on_write_operations=0"
+    clickhouse-client --query "INSERT INTO test.hits_s3 SELECT * FROM test.hits SETTINGS enable_filesystem_cache_on_write_operations=0, max_insert_threads=16"
 fi
 
 clickhouse-client --query "SHOW TABLES FROM test"
diff --git a/docker/test/stress/run.sh b/docker/test/stress/run.sh
index 86467394513..b21114e456f 100644
--- a/docker/test/stress/run.sh
+++ b/docker/test/stress/run.sh
@@ -209,9 +209,9 @@ clickhouse-client --query "CREATE TABLE test.visits (CounterID UInt32,  StartDat
     ENGINE = CollapsingMergeTree(Sign) PARTITION BY toYYYYMM(StartDate) ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID)
     SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192, storage_policy='$TEMP_POLICY'"
 
-clickhouse-client --query "INSERT INTO test.hits_s3 SELECT * FROM datasets.hits_v1 SETTINGS enable_filesystem_cache_on_write_operations=0"
-clickhouse-client --query "INSERT INTO test.hits SELECT * FROM datasets.hits_v1 SETTINGS enable_filesystem_cache_on_write_operations=0"
-clickhouse-client --query "INSERT INTO test.visits SELECT * FROM datasets.visits_v1 SETTINGS enable_filesystem_cache_on_write_operations=0"
+clickhouse-client --query "INSERT INTO test.hits_s3 SELECT * FROM datasets.hits_v1 SETTINGS enable_filesystem_cache_on_write_operations=0, max_insert_threads=16"
+clickhouse-client --query "INSERT INTO test.hits SELECT * FROM datasets.hits_v1 SETTINGS enable_filesystem_cache_on_write_operations=0, max_insert_threads=16"
+clickhouse-client --query "INSERT INTO test.visits SELECT * FROM datasets.visits_v1 SETTINGS enable_filesystem_cache_on_write_operations=0, max_insert_threads=16"
 
 clickhouse-client --query "DROP TABLE datasets.visits_v1 SYNC"
 clickhouse-client --query "DROP TABLE datasets.hits_v1 SYNC"
diff --git a/tests/queries/1_stateful/00162_mmap_compression_none.sql b/tests/queries/1_stateful/00162_mmap_compression_none.sql
index d2cbcea8aaa..48d6ada821e 100644
--- a/tests/queries/1_stateful/00162_mmap_compression_none.sql
+++ b/tests/queries/1_stateful/00162_mmap_compression_none.sql
@@ -1,6 +1,6 @@
 DROP TABLE IF EXISTS hits_none;
 CREATE TABLE hits_none (Title String CODEC(NONE)) ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';
-INSERT INTO hits_none SELECT Title FROM test.hits;
+INSERT INTO hits_none SELECT Title FROM test.hits SETTINGS enable_filesystem_cache_on_write_operations=0, max_insert_threads=16;
 
 SET min_bytes_to_use_mmap_io = 1;
 SELECT sum(length(Title)) FROM hits_none;

From ae5eccbf20b7198d6a3cc908e0186a384aba038a Mon Sep 17 00:00:00 2001
From: jsc0218 <jsc0218@gmail.com>
Date: Mon, 22 Jul 2024 13:39:48 +0000
Subject: [PATCH 0580/1488] just a commit to trigger CI

---
 .../test_grant_and_revoke/test_without_table_engine_grant.py     | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/integration/test_grant_and_revoke/test_without_table_engine_grant.py b/tests/integration/test_grant_and_revoke/test_without_table_engine_grant.py
index 210bb8ec465..4a5dfb83f79 100644
--- a/tests/integration/test_grant_and_revoke/test_without_table_engine_grant.py
+++ b/tests/integration/test_grant_and_revoke/test_without_table_engine_grant.py
@@ -60,6 +60,7 @@ def test_table_engine_and_source_grant():
     )
 
     # expecting grant POSTGRES instead of grant PostgreSQL due to discrepancy between source access type and table engine
+    # similarily, other sources should also use their own defined name instead of the name of table engine
     assert "grant POSTGRES ON *.*" in instance.query_and_get_error(
         """
         CREATE TABLE test.table1(a Integer)

From 85241b3b8e869879718d74c01ab5071a10d66c06 Mon Sep 17 00:00:00 2001
From: Aleksandr Musorin <aleksandr.musorin@semrush.com>
Date: Fri, 12 Jul 2024 18:35:01 +0200
Subject: [PATCH 0581/1488] Allow run query instantly in play

Automatically execute the query after the page loads if the
`play_now=true` parameter is present. By default, the query does not
execute automatically.

Reason:

While it might be acceptable to click `Run` once or twice, it becomes
tedious when using the play service frequently as a simple frontend to
generate and open multiple links. This change eliminates the need to
click `Run` every time.
---
 programs/server/play.html | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/programs/server/play.html b/programs/server/play.html
index 9590a65524c..a4c01237abd 100644
--- a/programs/server/play.html
+++ b/programs/server/play.html
@@ -522,6 +522,9 @@
     const current_url = new URL(window.location);
     const opened_locally = location.protocol == 'file:';
 
+    /// Run query instantly after page is loaded
+    const play_now = current_url.searchParams.get("play_now");
+
     const server_address = current_url.searchParams.get('url');
     if (server_address) {
         document.getElementById('url').value = server_address;
@@ -599,6 +602,9 @@
                     const title = "ClickHouse Query: " + query;
 
                     let history_url = window.location.pathname + '?user=' + encodeURIComponent(user);
+                    if (play_now) {
+                        history_url += "&play_now=" + encodeURIComponent(play_now);
+                    }
                     if (server_address != location.origin) {
                         /// Save server's address in URL if it's not identical to the address of the play UI.
                         history_url += '&url=' + encodeURIComponent(server_address);
@@ -1160,6 +1166,10 @@
         });
     }
 
+    if (play_now === 'true') {
+        post();
+    }
+
     document.getElementById('toggle-light').onclick = function() {
         setColorTheme('light', true);
     }

From 85afb1757e92b279dfc7171c409e09d5b6873e8b Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 12 Jul 2024 21:21:35 +0200
Subject: [PATCH 0582/1488] Rename variables

---
 programs/server/play.html | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/programs/server/play.html b/programs/server/play.html
index a4c01237abd..b1da7408b58 100644
--- a/programs/server/play.html
+++ b/programs/server/play.html
@@ -523,7 +523,7 @@
     const opened_locally = location.protocol == 'file:';
 
     /// Run query instantly after page is loaded
-    const play_now = current_url.searchParams.get("play_now");
+    const run_immediately = current_url.searchParams.get("run");
 
     const server_address = current_url.searchParams.get('url');
     if (server_address) {
@@ -602,8 +602,8 @@
                     const title = "ClickHouse Query: " + query;
 
                     let history_url = window.location.pathname + '?user=' + encodeURIComponent(user);
-                    if (play_now) {
-                        history_url += "&play_now=" + encodeURIComponent(play_now);
+                    if (run_immediately) {
+                        history_url += "&run=" + encodeURIComponent(run_immediately);
                     }
                     if (server_address != location.origin) {
                         /// Save server's address in URL if it's not identical to the address of the play UI.
@@ -1166,7 +1166,7 @@
         });
     }
 
-    if (play_now === 'true') {
+    if (run_immediately === 'true') {
         post();
     }
 

From 11aee643adc49cf14e73f3ea0a8fbdfe284dd2ef Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 12 Jul 2024 21:23:18 +0200
Subject: [PATCH 0583/1488] Convert to bool

---
 programs/server/play.html | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/programs/server/play.html b/programs/server/play.html
index b1da7408b58..d4fc1446fa0 100644
--- a/programs/server/play.html
+++ b/programs/server/play.html
@@ -523,7 +523,7 @@
     const opened_locally = location.protocol == 'file:';
 
     /// Run query instantly after page is loaded
-    const run_immediately = current_url.searchParams.get("run");
+    const run_immediately = !!current_url.searchParams.get("run");
 
     const server_address = current_url.searchParams.get('url');
     if (server_address) {
@@ -603,7 +603,7 @@
 
                     let history_url = window.location.pathname + '?user=' + encodeURIComponent(user);
                     if (run_immediately) {
-                        history_url += "&run=" + encodeURIComponent(run_immediately);
+                        history_url += "&run=" + run_immediately;
                     }
                     if (server_address != location.origin) {
                         /// Save server's address in URL if it's not identical to the address of the play UI.

From eb4cddd139c5ccdc43a3a0b8168220395303a276 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 12 Jul 2024 21:28:39 +0200
Subject: [PATCH 0584/1488] Allow `run=1` to save space

---
 programs/server/play.html | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/programs/server/play.html b/programs/server/play.html
index d4fc1446fa0..eb861172c02 100644
--- a/programs/server/play.html
+++ b/programs/server/play.html
@@ -523,7 +523,7 @@
     const opened_locally = location.protocol == 'file:';
 
     /// Run query instantly after page is loaded
-    const run_immediately = !!current_url.searchParams.get("run");
+    const run_immediately = current_url.searchParams.get("run") == 1;
 
     const server_address = current_url.searchParams.get('url');
     if (server_address) {
@@ -603,7 +603,7 @@
 
                     let history_url = window.location.pathname + '?user=' + encodeURIComponent(user);
                     if (run_immediately) {
-                        history_url += "&run=" + run_immediately;
+                        history_url += "&run=" + (run_immediately ? 1 : 0);
                     }
                     if (server_address != location.origin) {
                         /// Save server's address in URL if it's not identical to the address of the play UI.
@@ -1166,7 +1166,7 @@
         });
     }
 
-    if (run_immediately === 'true') {
+    if (run_immediately) {
         post();
     }
 

From 02704bc5cc8213d6bd56183f613bda960a69b93a Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 12 Jul 2024 21:32:14 +0200
Subject: [PATCH 0585/1488] Different

---
 programs/server/play.html | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/programs/server/play.html b/programs/server/play.html
index eb861172c02..e30b4ac3450 100644
--- a/programs/server/play.html
+++ b/programs/server/play.html
@@ -522,8 +522,8 @@
     const current_url = new URL(window.location);
     const opened_locally = location.protocol == 'file:';
 
-    /// Run query instantly after page is loaded
-    const run_immediately = current_url.searchParams.get("run") == 1;
+    /// Run query instantly after page is loaded if the run parameter is present.
+    const run_immediately = !!current_url.searchParams.get("run");
 
     const server_address = current_url.searchParams.get('url');
     if (server_address) {

From af53fd96a7a7adbbe6a6660a37d30b4acaf9fe40 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 12 Jul 2024 21:34:38 +0200
Subject: [PATCH 0586/1488] JavaScript Programming

---
 programs/server/play.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/programs/server/play.html b/programs/server/play.html
index e30b4ac3450..8f2fab36df4 100644
--- a/programs/server/play.html
+++ b/programs/server/play.html
@@ -523,7 +523,7 @@
     const opened_locally = location.protocol == 'file:';
 
     /// Run query instantly after page is loaded if the run parameter is present.
-    const run_immediately = !!current_url.searchParams.get("run");
+    const run_immediately = current_url.searchParams.has("run");
 
     const server_address = current_url.searchParams.get('url');
     if (server_address) {

From 0d2459f0de566dfa13eecf3bd176e59ec11e0239 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 12 Jul 2024 21:35:20 +0200
Subject: [PATCH 0587/1488] JavaScript

---
 programs/server/play.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/programs/server/play.html b/programs/server/play.html
index 8f2fab36df4..0d76a01cf7e 100644
--- a/programs/server/play.html
+++ b/programs/server/play.html
@@ -603,7 +603,7 @@
 
                     let history_url = window.location.pathname + '?user=' + encodeURIComponent(user);
                     if (run_immediately) {
-                        history_url += "&run=" + (run_immediately ? 1 : 0);
+                        history_url += "&run=1";
                     }
                     if (server_address != location.origin) {
                         /// Save server's address in URL if it's not identical to the address of the play UI.

From a3dbf87df6a7b3105fdbc79cafe0b2ec2fa547d5 Mon Sep 17 00:00:00 2001
From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com>
Date: Mon, 22 Jul 2024 16:19:16 +0200
Subject: [PATCH 0588/1488] Update convertFieldToType.cpp

---
 src/Interpreters/convertFieldToType.cpp | 38 ++++++++++++-------------
 1 file changed, 19 insertions(+), 19 deletions(-)

diff --git a/src/Interpreters/convertFieldToType.cpp b/src/Interpreters/convertFieldToType.cpp
index 9ee214f4415..b92cbae7b09 100644
--- a/src/Interpreters/convertFieldToType.cpp
+++ b/src/Interpreters/convertFieldToType.cpp
@@ -384,25 +384,25 @@ Field convertFieldToTypeImpl(const Field & src, const IDataType & type, const ID
             {
                 const auto & element_type = *(type_tuple->getElements()[i]);
                 res[i] = convertFieldToType(src_tuple[i], element_type);
-                if (!res[i].isNull() || canContainNull(element_type))
-                    continue;
-
-                /*
-                 * Either the source element was Null, or the conversion did not
-                 * succeed, because the source and the requested types of the
-                 * element are compatible, but the value is not convertible
-                 * (e.g. trying to convert -1 from Int8 to UInt8). In these
-                 * cases, consider the whole tuple also compatible but not
-                 * convertible. According to the specification of this function,
-                 * we must return Null in this case.
-                 *
-                 * The following elements might be not even compatible, so it
-                 * makes sense to check them to detect user errors. Remember
-                 * that there is an unconvertible element, and try to process
-                 * the remaining ones. The convertFieldToType for each element
-                 * will throw if it detects incompatibility.
-                 */
-                have_unconvertible_element = true;
+                if (res[i].isNull() && !canContainNull(element_type))
+                {
+                    /*
+                     * Either the source element was Null, or the conversion did not
+                     * succeed, because the source and the requested types of the
+                     * element are compatible, but the value is not convertible
+                     * (e.g. trying to convert -1 from Int8 to UInt8). In these
+                     * cases, consider the whole tuple also compatible but not
+                     * convertible. According to the specification of this function,
+                     * we must return Null in this case.
+                     *
+                     * The following elements might be not even compatible, so it
+                     * makes sense to check them to detect user errors. Remember
+                     * that there is an unconvertible element, and try to process
+                     * the remaining ones. The convertFieldToType for each element
+                     * will throw if it detects incompatibility.
+                     */
+                    have_unconvertible_element = true;
+                }
             }
 
             return have_unconvertible_element ? Field(Null()) : Field(res);

From f5710beebf9394d954a1fe5dadb9df87b2b8ebaa Mon Sep 17 00:00:00 2001
From: Nikita Taranov <nikita.taranov@clickhouse.com>
Date: Mon, 22 Jul 2024 15:22:31 +0100
Subject: [PATCH 0589/1488] fix build

---
 src/Common/tests/gtest_cgroups_reader.cpp | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/Common/tests/gtest_cgroups_reader.cpp b/src/Common/tests/gtest_cgroups_reader.cpp
index 38e56401401..2de25bb42ce 100644
--- a/src/Common/tests/gtest_cgroups_reader.cpp
+++ b/src/Common/tests/gtest_cgroups_reader.cpp
@@ -1,12 +1,13 @@
+#if defined(OS_LINUX)
+
+#include <gtest/gtest.h>
+
 #include <cstdint>
 #include <filesystem>
-#include <stdint.h>
-#include <gtest/gtest.h>
 
 #include <IO/WriteBufferFromFile.h>
 #include <Common/CgroupsMemoryUsageObserver.h>
 #include <Common/filesystemHelpers.h>
-#include "IO/WriteBufferFromFileBase.h"
 
 using namespace DB;
 
@@ -173,3 +174,5 @@ INSTANTIATE_TEST_SUITE_P(
     CgroupsMemoryUsageObserverTests,
     CgroupsMemoryUsageObserverFixture,
     ::testing::Values(CgroupsMemoryUsageObserver::CgroupsVersion::V1, CgroupsMemoryUsageObserver::CgroupsVersion::V2));
+
+#endif

From 023276b6f6296a68f0e36004c9e40696bad73742 Mon Sep 17 00:00:00 2001
From: avogar <avogar@clickhouse.com>
Date: Mon, 22 Jul 2024 14:32:04 +0000
Subject: [PATCH 0590/1488] Fix tests reference

---
 .../03038_nested_dynamic_merges_compact_horizontal.reference  | 4 ++--
 .../03038_nested_dynamic_merges_compact_vertical.reference    | 4 ++--
 .../03038_nested_dynamic_merges_wide_horizontal.reference     | 4 ++--
 .../03038_nested_dynamic_merges_wide_vertical.reference       | 4 ++--
 4 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/tests/queries/0_stateless/03038_nested_dynamic_merges_compact_horizontal.reference b/tests/queries/0_stateless/03038_nested_dynamic_merges_compact_horizontal.reference
index 27ed336a035..4be740f6050 100644
--- a/tests/queries/0_stateless/03038_nested_dynamic_merges_compact_horizontal.reference
+++ b/tests/queries/0_stateless/03038_nested_dynamic_merges_compact_horizontal.reference
@@ -11,11 +11,11 @@
 33333	Tuple(a Dynamic(max_types=3)):Array(UInt8)
 50000	Tuple(a Dynamic(max_types=3)):UInt64
 66667	Tuple(a Dynamic(max_types=3)):String
-100000	Tuple(a Dynamic(max_types=3)):Tuple(number UInt64)
+100000	Tuple(a Dynamic(max_types=3)):Tuple(UInt64)
 100000	UInt64:None
 133333	Tuple(a Dynamic(max_types=3)):None
 50000	Tuple(a Dynamic(max_types=3)):UInt64
-100000	Tuple(a Dynamic(max_types=3)):Tuple(number UInt64)
+100000	Tuple(a Dynamic(max_types=3)):Tuple(UInt64)
 100000	UInt64:None
 116667	Tuple(a Dynamic(max_types=3)):String
 133333	Tuple(a Dynamic(max_types=3)):None
diff --git a/tests/queries/0_stateless/03038_nested_dynamic_merges_compact_vertical.reference b/tests/queries/0_stateless/03038_nested_dynamic_merges_compact_vertical.reference
index 27ed336a035..4be740f6050 100644
--- a/tests/queries/0_stateless/03038_nested_dynamic_merges_compact_vertical.reference
+++ b/tests/queries/0_stateless/03038_nested_dynamic_merges_compact_vertical.reference
@@ -11,11 +11,11 @@
 33333	Tuple(a Dynamic(max_types=3)):Array(UInt8)
 50000	Tuple(a Dynamic(max_types=3)):UInt64
 66667	Tuple(a Dynamic(max_types=3)):String
-100000	Tuple(a Dynamic(max_types=3)):Tuple(number UInt64)
+100000	Tuple(a Dynamic(max_types=3)):Tuple(UInt64)
 100000	UInt64:None
 133333	Tuple(a Dynamic(max_types=3)):None
 50000	Tuple(a Dynamic(max_types=3)):UInt64
-100000	Tuple(a Dynamic(max_types=3)):Tuple(number UInt64)
+100000	Tuple(a Dynamic(max_types=3)):Tuple(UInt64)
 100000	UInt64:None
 116667	Tuple(a Dynamic(max_types=3)):String
 133333	Tuple(a Dynamic(max_types=3)):None
diff --git a/tests/queries/0_stateless/03038_nested_dynamic_merges_wide_horizontal.reference b/tests/queries/0_stateless/03038_nested_dynamic_merges_wide_horizontal.reference
index 27ed336a035..4be740f6050 100644
--- a/tests/queries/0_stateless/03038_nested_dynamic_merges_wide_horizontal.reference
+++ b/tests/queries/0_stateless/03038_nested_dynamic_merges_wide_horizontal.reference
@@ -11,11 +11,11 @@
 33333	Tuple(a Dynamic(max_types=3)):Array(UInt8)
 50000	Tuple(a Dynamic(max_types=3)):UInt64
 66667	Tuple(a Dynamic(max_types=3)):String
-100000	Tuple(a Dynamic(max_types=3)):Tuple(number UInt64)
+100000	Tuple(a Dynamic(max_types=3)):Tuple(UInt64)
 100000	UInt64:None
 133333	Tuple(a Dynamic(max_types=3)):None
 50000	Tuple(a Dynamic(max_types=3)):UInt64
-100000	Tuple(a Dynamic(max_types=3)):Tuple(number UInt64)
+100000	Tuple(a Dynamic(max_types=3)):Tuple(UInt64)
 100000	UInt64:None
 116667	Tuple(a Dynamic(max_types=3)):String
 133333	Tuple(a Dynamic(max_types=3)):None
diff --git a/tests/queries/0_stateless/03038_nested_dynamic_merges_wide_vertical.reference b/tests/queries/0_stateless/03038_nested_dynamic_merges_wide_vertical.reference
index 27ed336a035..4be740f6050 100644
--- a/tests/queries/0_stateless/03038_nested_dynamic_merges_wide_vertical.reference
+++ b/tests/queries/0_stateless/03038_nested_dynamic_merges_wide_vertical.reference
@@ -11,11 +11,11 @@
 33333	Tuple(a Dynamic(max_types=3)):Array(UInt8)
 50000	Tuple(a Dynamic(max_types=3)):UInt64
 66667	Tuple(a Dynamic(max_types=3)):String
-100000	Tuple(a Dynamic(max_types=3)):Tuple(number UInt64)
+100000	Tuple(a Dynamic(max_types=3)):Tuple(UInt64)
 100000	UInt64:None
 133333	Tuple(a Dynamic(max_types=3)):None
 50000	Tuple(a Dynamic(max_types=3)):UInt64
-100000	Tuple(a Dynamic(max_types=3)):Tuple(number UInt64)
+100000	Tuple(a Dynamic(max_types=3)):Tuple(UInt64)
 100000	UInt64:None
 116667	Tuple(a Dynamic(max_types=3)):String
 133333	Tuple(a Dynamic(max_types=3)):None

From 660530c611000f5eb8875c640d5aed196315a187 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 22 Jul 2024 17:10:39 +0200
Subject: [PATCH 0591/1488] Fix tidy

---
 src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp b/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp
index a36a8b031b4..377f6b36888 100644
--- a/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp
+++ b/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp
@@ -258,10 +258,9 @@ std::optional<size_t> ReadBufferFromAzureBlobStorage::tryGetFileSize()
     if (!blob_client)
         blob_client = std::make_unique<Azure::Storage::Blobs::BlobClient>(blob_container_client->GetBlobClient(path));
 
-    if (file_size.has_value())
-        return *file_size;
+    if (!file_size)
+        file_size = blob_client->GetProperties().Value.BlobSize;
 
-    file_size = blob_client->GetProperties().Value.BlobSize;
     return *file_size;
 }
 

From eb519c501622af6a6df6f3370b6209c92c2d4c20 Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitlibar@clickhouse.com>
Date: Sat, 20 Jul 2024 15:01:47 +0200
Subject: [PATCH 0592/1488] Change paths of temporary part directories during
 RESTORE use
 "/var/lib/clickhouse/data/test/table/tmp_restore_all_0_1_1_0-XXXXXXXX"
 instead of "/tmp/XXXXXXXX/data/test/table/0_1_1_0" (because directories can
 only be renamed in s3_plain_rewritable, and not moved to another parent
 directory).

---
 src/Disks/TemporaryFileOnDisk.cpp        |  3 +-
 src/Disks/TemporaryFileOnDisk.h          |  7 +++++
 src/Storages/MergeTree/MergeTreeData.cpp | 36 ++++++++++++++----------
 src/Storages/MergeTree/MergeTreeData.h   |  2 +-
 4 files changed, 31 insertions(+), 17 deletions(-)

diff --git a/src/Disks/TemporaryFileOnDisk.cpp b/src/Disks/TemporaryFileOnDisk.cpp
index 91eb214d941..88674e068d9 100644
--- a/src/Disks/TemporaryFileOnDisk.cpp
+++ b/src/Disks/TemporaryFileOnDisk.cpp
@@ -58,7 +58,8 @@ TemporaryFileOnDisk::~TemporaryFileOnDisk()
 
         if (!disk->exists(relative_path))
         {
-            LOG_WARNING(getLogger("TemporaryFileOnDisk"), "Temporary path '{}' does not exist in '{}'", relative_path, disk->getPath());
+            if (show_warning_if_removed)
+                LOG_WARNING(getLogger("TemporaryFileOnDisk"), "Temporary path '{}' does not exist in '{}'", relative_path, disk->getPath());
             return;
         }
 
diff --git a/src/Disks/TemporaryFileOnDisk.h b/src/Disks/TemporaryFileOnDisk.h
index cccfc82cf9e..d0ff44c6f03 100644
--- a/src/Disks/TemporaryFileOnDisk.h
+++ b/src/Disks/TemporaryFileOnDisk.h
@@ -27,12 +27,19 @@ public:
     /// Return relative path (without disk)
     const String & getRelativePath() const { return relative_path; }
 
+    /// Sets whether the destructor should show a warning if the temporary file has been already removed.
+    /// By default a warning is shown.
+    void setShowWarningIfRemoved(bool show_warning_if_removed_) { show_warning_if_removed = show_warning_if_removed_; }
+
 private:
     DiskPtr disk;
 
     /// Relative path in disk to the temporary file or directory
     String relative_path;
 
+    /// Whether the destructor should show a warning if the temporary file has been already removed.
+    bool show_warning_if_removed = true;
+
     CurrentMetrics::Increment metric_increment;
 
     /// Specified if we know what for file is used (sort/aggregate/join).
diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index 12a4effe33c..642edf1cfb8 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -5549,12 +5549,17 @@ public:
         attachIfAllPartsRestored();
     }
 
-    String getTemporaryDirectory(const DiskPtr & disk)
+    String getTemporaryDirectory(const DiskPtr & disk, const String & part_name)
     {
         std::lock_guard lock{mutex};
-        auto it = temp_dirs.find(disk);
-        if (it == temp_dirs.end())
-            it = temp_dirs.emplace(disk, std::make_shared<TemporaryFileOnDisk>(disk, "tmp/")).first;
+        auto it = temp_part_dirs.find(part_name);
+        if (it == temp_part_dirs.end())
+        {
+            auto temp_part_dir = std::make_shared<TemporaryFileOnDisk>(disk, fs::path{storage->getRelativeDataPath()} / ("tmp_restore_" + part_name + "-"));
+            /// Attaching parts will rename them so it's expected for a temporary part directory not to exist anymore in the end.
+            temp_part_dir->setShowWarningIfRemoved(false);
+            it = temp_part_dirs.emplace(part_name, temp_part_dir).first;
+        }
         return it->second->getRelativePath();
     }
 
@@ -5572,7 +5577,7 @@ private:
 
         storage->attachRestoredParts(std::move(parts));
         parts.clear();
-        temp_dirs.clear();
+        temp_part_dirs.clear();
         num_parts = 0;
     }
 
@@ -5581,7 +5586,7 @@ private:
     size_t num_parts = 0;
     size_t num_broken_parts = 0;
     MutableDataPartsVector parts;
-    std::map<DiskPtr, std::shared_ptr<TemporaryFileOnDisk>> temp_dirs;
+    std::map<String /* part_name*/, std::shared_ptr<TemporaryFileOnDisk>> temp_part_dirs;
     mutable std::mutex mutex;
 };
 
@@ -5648,11 +5653,9 @@ void MergeTreeData::restorePartFromBackup(std::shared_ptr<RestoredPartsHolder> r
     /// Calculate paths, for example:
     /// part_name = 0_1_1_0
     /// part_path_in_backup = /data/test/table/0_1_1_0
-    /// tmp_dir = tmp/1aaaaaa
-    /// tmp_part_dir = tmp/1aaaaaa/data/test/table/0_1_1_0
+    /// temp_part_dir = /var/lib/clickhouse/data/test/table/tmp_restore_all_0_1_1_0-XXXXXXXX
     auto disk = reservation->getDisk();
-    fs::path temp_dir = restored_parts_holder->getTemporaryDirectory(disk);
-    fs::path temp_part_dir = temp_dir / part_path_in_backup_fs.relative_path();
+    fs::path temp_part_dir = restored_parts_holder->getTemporaryDirectory(disk, part_name);
 
     /// Subdirectories in the part's directory. It's used to restore projections.
     std::unordered_set<String> subdirs;
@@ -5679,22 +5682,25 @@ void MergeTreeData::restorePartFromBackup(std::shared_ptr<RestoredPartsHolder> r
         reservation->update(reservation->getSize() - file_size);
     }
 
-    if (auto part = loadPartRestoredFromBackup(disk, temp_part_dir.parent_path(), part_name, detach_if_broken))
+    if (auto part = loadPartRestoredFromBackup(part_name, disk, temp_part_dir, detach_if_broken))
         restored_parts_holder->addPart(part);
     else
         restored_parts_holder->increaseNumBrokenParts();
 }
 
-MergeTreeData::MutableDataPartPtr MergeTreeData::loadPartRestoredFromBackup(const DiskPtr & disk, const String & temp_dir, const String & part_name, bool detach_if_broken) const
+MergeTreeData::MutableDataPartPtr MergeTreeData::loadPartRestoredFromBackup(const String & part_name, const DiskPtr & disk, const String & temp_part_dir, bool detach_if_broken) const
 {
     MutableDataPartPtr part;
 
     auto single_disk_volume = std::make_shared<SingleDiskVolume>(disk->getName(), disk, 0);
+    fs::path full_part_dir{temp_part_dir};
+    String parent_part_dir = full_part_dir.parent_path();
+    String part_dir_name = full_part_dir.filename();
 
-    /// Load this part from the directory `tmp_part_dir`.
+    /// Load this part from the directory `temp_part_dir`.
     auto load_part = [&]
     {
-        MergeTreeDataPartBuilder builder(*this, part_name, single_disk_volume, temp_dir, part_name);
+        MergeTreeDataPartBuilder builder(*this, part_name, single_disk_volume, parent_part_dir, part_dir_name);
         builder.withPartFormatFromDisk();
         part = std::move(builder).build();
         part->version.setCreationTID(Tx::PrehistoricTID, nullptr);
@@ -5709,7 +5715,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::loadPartRestoredFromBackup(cons
         if (!part)
         {
             /// Make a fake data part only to copy its files to /detached/.
-            part = MergeTreeDataPartBuilder{*this, part_name, single_disk_volume, temp_dir, part_name}
+            part = MergeTreeDataPartBuilder{*this, part_name, single_disk_volume, parent_part_dir, part_dir_name}
                        .withPartStorageType(MergeTreeDataPartStorageType::Full)
                        .withPartType(MergeTreeDataPartType::Wide)
                        .build();
diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h
index 7076b680521..d185a5262fd 100644
--- a/src/Storages/MergeTree/MergeTreeData.h
+++ b/src/Storages/MergeTree/MergeTreeData.h
@@ -1473,7 +1473,7 @@ protected:
     /// Restores the parts of this table from backup.
     void restorePartsFromBackup(RestorerFromBackup & restorer, const String & data_path_in_backup, const std::optional<ASTs> & partitions);
     void restorePartFromBackup(std::shared_ptr<RestoredPartsHolder> restored_parts_holder, const MergeTreePartInfo & part_info, const String & part_path_in_backup, bool detach_if_broken) const;
-    MutableDataPartPtr loadPartRestoredFromBackup(const DiskPtr & disk, const String & temp_dir, const String & part_name, bool detach_if_broken) const;
+    MutableDataPartPtr loadPartRestoredFromBackup(const String & part_name, const DiskPtr & disk, const String & temp_part_dir, bool detach_if_broken) const;
 
     /// Attaches restored parts to the storage.
     virtual void attachRestoredParts(MutableDataPartsVector && parts) = 0;

From 5fa2db8e4828b004ac10d625df62efcc8711dc98 Mon Sep 17 00:00:00 2001
From: Michael Stetsyuk <mike.stetsyuk@gmail.com>
Date: Mon, 22 Jul 2024 16:59:17 +0100
Subject: [PATCH 0593/1488] fix 01293_show_clusters stateless test

---
 tests/queries/0_stateless/01293_show_clusters.reference | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/queries/0_stateless/01293_show_clusters.reference b/tests/queries/0_stateless/01293_show_clusters.reference
index e140f207022..9569fcf2e37 100644
--- a/tests/queries/0_stateless/01293_show_clusters.reference
+++ b/tests/queries/0_stateless/01293_show_clusters.reference
@@ -1,3 +1,3 @@
 test_shard_localhost
-test_cluster_one_shard_two_replicas	1	1	0	1	127.0.0.1	127.0.0.1	9000	1	default
-test_cluster_one_shard_two_replicas	1	1	0	2	127.0.0.2	127.0.0.2	9000	0	default
+test_cluster_one_shard_two_replicas	1	1	0	1	127.0.0.1	127.0.0.1	9000	1	default	0	NULL
+test_cluster_one_shard_two_replicas	1	1	0	2	127.0.0.2	127.0.0.2	9000	0	default	0	NULL

From fa0f760fd9afd652dee50b89fd8d176d4e031174 Mon Sep 17 00:00:00 2001
From: Max K <maxkaynov@gmail.com>
Date: Mon, 22 Jul 2024 15:46:27 +0200
Subject: [PATCH 0594/1488] CI: CI Buddy to notify about fatal workflow
 failures

---
 .github/workflows/backport_branches.yml | 14 ++++++++-
 .github/workflows/master.yml            | 38 +++++++------------------
 .github/workflows/merge_queue.yml       | 13 ++++++++-
 .github/workflows/nightly.yml           | 17 +++++++++++
 .github/workflows/pull_request.yml      | 13 ++++++++-
 .github/workflows/release_branches.yml  | 14 ++++++++-
 .yamllint                               |  6 ++++
 tests/ci/ci_buddy.py                    | 37 +++++++++++++++++++++---
 tests/ci/ci_utils.py                    | 27 ++++++++++++++++++
 9 files changed, 143 insertions(+), 36 deletions(-)

diff --git a/.github/workflows/backport_branches.yml b/.github/workflows/backport_branches.yml
index 9645d0e46de..50f4f503f5d 100644
--- a/.github/workflows/backport_branches.yml
+++ b/.github/workflows/backport_branches.yml
@@ -241,8 +241,9 @@ jobs:
       runner_type: stress-tester
       data: ${{ needs.RunConfig.outputs.data }}
   FinishCheck:
-    if: ${{ !failure() && !cancelled() }}
+    if: ${{ !cancelled() }}
     needs:
+      - RunConfig
       - Builds_Report
       - FunctionalStatelessTestAsan
       - FunctionalStatefulTestDebug
@@ -257,6 +258,7 @@ jobs:
         with:
           clear-repository: true
       - name: Finish label
+        if: ${{ !failure() }}
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
           # update mergeable check
@@ -264,3 +266,13 @@ jobs:
           # update overall ci report
           python3 finish_check.py --wf-status ${{ contains(needs.*.result, 'failure') && 'failure' || 'success' }}
           python3 merge_pr.py
+      - name: Check Workflow results
+        run: |
+          export WORKFLOW_RESULT_FILE="/tmp/workflow_results.json"
+          cat >> "$WORKFLOW_RESULT_FILE" << 'EOF'
+          ${{ toJson(needs) }}
+          EOF
+          echo "::group::Workflow results"
+          python3 -m json.tool "$WORKFLOW_RESULT_FILE"
+          echo "::endgroup::"
+          python3 ./tests/ci/ci_buddy.py --check-wf-status
diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index 09acef5eb8b..b28d87ee31f 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -121,34 +121,6 @@ jobs:
       runner_type: style-checker-aarch64
       data: ${{ needs.RunConfig.outputs.data }}
 
-  MarkReleaseReady:
-    if: ${{ !failure() && !cancelled() }}
-    needs: [RunConfig, Builds_1, Builds_2]
-    runs-on: [self-hosted, style-checker-aarch64]
-    steps:
-      - name: Debug
-        run: |
-          echo need with different filters
-          cat << 'EOF'
-          ${{ toJSON(needs) }}
-          ${{ toJSON(needs.*.result) }}
-          no failures ${{ !contains(needs.*.result, 'failure') }}
-          no skips ${{ !contains(needs.*.result, 'skipped') }}
-          no both ${{ !(contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure')) }}
-          EOF
-      - name: Not ready
-        # fail the job to be able to restart it
-        if: ${{ contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure') }}
-        run: exit 1
-      - name: Check out repository code
-        if: ${{ ! (contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure')) }}
-        uses: ClickHouse/checkout@v1
-      - name: Mark Commit Release Ready
-        if: ${{ ! (contains(needs.*.result, 'skipped') || contains(needs.*.result, 'failure')) }}
-        run: |
-          cd "$GITHUB_WORKSPACE/tests/ci"
-          python3 mark_release_ready.py
-
   FinishCheck:
     if: ${{ !cancelled() }}
     needs: [RunConfig, Builds_1, Builds_2, Builds_Report, Tests_1, Tests_2, Tests_3]
@@ -160,3 +132,13 @@ jobs:
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
           python3 finish_check.py --wf-status ${{ contains(needs.*.result, 'failure') && 'failure' || 'success' }}
+      - name: Check Workflow results
+        run: |
+          export WORKFLOW_RESULT_FILE="/tmp/workflow_results.json"
+          cat >> "$WORKFLOW_RESULT_FILE" << 'EOF'
+          ${{ toJson(needs) }}
+          EOF
+          echo "::group::Workflow results"
+          python3 -m json.tool "$WORKFLOW_RESULT_FILE"
+          echo "::endgroup::"
+          python3 ./tests/ci/ci_buddy.py --check-wf-status
diff --git a/.github/workflows/merge_queue.yml b/.github/workflows/merge_queue.yml
index 31a65ac3d15..db89825a99a 100644
--- a/.github/workflows/merge_queue.yml
+++ b/.github/workflows/merge_queue.yml
@@ -93,7 +93,7 @@ jobs:
       data: ${{ needs.RunConfig.outputs.data }}
 
   CheckReadyForMerge:
-    if: ${{ !cancelled() && needs.StyleCheck.result == 'success' }}
+    if: ${{ !cancelled() }}
     # Test_2 or Test_3 must not have jobs required for Mergeable check
     needs: [RunConfig, BuildDockers, StyleCheck, FastTest, Builds_1, Tests_1]
     runs-on: [self-hosted, style-checker-aarch64]
@@ -101,6 +101,17 @@ jobs:
       - name: Check out repository code
         uses: ClickHouse/checkout@v1
       - name: Check and set merge status
+        if: ${{ needs.StyleCheck.result == 'success' }}
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
           python3 merge_pr.py --set-ci-status --wf-status ${{ contains(needs.*.result, 'failure') && 'failure' || 'success' }}
+      - name: Check Workflow results
+        run: |
+          export WORKFLOW_RESULT_FILE="/tmp/workflow_results.json"
+          cat >> "$WORKFLOW_RESULT_FILE" << 'EOF'
+          ${{ toJson(needs) }}
+          EOF
+          echo "::group::Workflow results"
+          python3 -m json.tool "$WORKFLOW_RESULT_FILE"
+          echo "::endgroup::"
+          python3 ./tests/ci/ci_buddy.py --check-wf-status
diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
index bffe5b4c1bf..fd5b5eefcc4 100644
--- a/.github/workflows/nightly.yml
+++ b/.github/workflows/nightly.yml
@@ -44,3 +44,20 @@ jobs:
     with:
       data: "${{ needs.RunConfig.outputs.data }}"
       set_latest: true
+  CheckWorkflow:
+    if: ${{ !cancelled() }}
+    needs: [RunConfig, BuildDockers]
+    runs-on: [self-hosted, style-checker-aarch64]
+    steps:
+      - name: Check out repository code
+        uses: ClickHouse/checkout@v1
+      - name: Check Workflow results
+        run: |
+          export WORKFLOW_RESULT_FILE="/tmp/workflow_results.json"
+          cat >> "$WORKFLOW_RESULT_FILE" << 'EOF'
+          ${{ toJson(needs) }}
+          EOF
+          echo "::group::Workflow results"
+          python3 -m json.tool "$WORKFLOW_RESULT_FILE"
+          echo "::endgroup::"
+          python3 ./tests/ci/ci_buddy.py --check-wf-status
diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
index 5124e4dba2c..9930cf6dde4 100644
--- a/.github/workflows/pull_request.yml
+++ b/.github/workflows/pull_request.yml
@@ -151,7 +151,7 @@ jobs:
       data: ${{ needs.RunConfig.outputs.data }}
 
   CheckReadyForMerge:
-    if: ${{ !cancelled() && needs.StyleCheck.result == 'success' }}
+    if: ${{ !cancelled() }}
     # Test_2 or Test_3 must not have jobs required for Mergeable check
     needs: [RunConfig, BuildDockers, StyleCheck, FastTest, Builds_1, Builds_2, Builds_Report, Tests_1]
     runs-on: [self-hosted, style-checker-aarch64]
@@ -161,9 +161,20 @@ jobs:
         with:
           filter: tree:0
       - name: Check and set merge status
+        if: ${{ needs.StyleCheck.result == 'success' }}
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
           python3 merge_pr.py --set-ci-status --wf-status ${{ contains(needs.*.result, 'failure') && 'failure' || 'success' }}
+      - name: Check Workflow results
+        run: |
+          export WORKFLOW_RESULT_FILE="/tmp/workflow_results.json"
+          cat >> "$WORKFLOW_RESULT_FILE" << 'EOF'
+          ${{ toJson(needs) }}
+          EOF
+          echo "::group::Workflow results"
+          python3 -m json.tool "$WORKFLOW_RESULT_FILE"
+          echo "::endgroup::"
+          python3 ./tests/ci/ci_buddy.py --check-wf-status
 
   ################################# Stage Final #################################
   #
diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml
index 6a18999d74e..50565112825 100644
--- a/.github/workflows/release_branches.yml
+++ b/.github/workflows/release_branches.yml
@@ -441,8 +441,9 @@ jobs:
       runner_type: stress-tester
       data: ${{ needs.RunConfig.outputs.data }}
   FinishCheck:
-    if: ${{ !failure() && !cancelled() }}
+    if: ${{ !cancelled() }}
     needs:
+      - RunConfig
       - DockerServerImage
       - DockerKeeperImage
       - Builds_Report
@@ -478,9 +479,20 @@ jobs:
         with:
           clear-repository: true
       - name: Finish label
+        if: ${{ !failure() }}
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
           # update mergeable check
           python3 merge_pr.py --set-ci-status --wf-status ${{ contains(needs.*.result, 'failure') && 'failure' || 'success' }}
           # update overall ci report
           python3 finish_check.py --wf-status ${{ contains(needs.*.result, 'failure') && 'failure' || 'success' }}
+      - name: Check Workflow results
+        run: |
+          export WORKFLOW_RESULT_FILE="/tmp/workflow_results.json"
+          cat >> "$WORKFLOW_RESULT_FILE" << 'EOF'
+          ${{ toJson(needs) }}
+          EOF
+          echo "::group::Workflow results"
+          python3 -m json.tool "$WORKFLOW_RESULT_FILE"
+          echo "::endgroup::"
+          python3 ./tests/ci/ci_buddy.py --check-wf-status
diff --git a/.yamllint b/.yamllint
index f144e2d47b1..7fb741ec9f4 100644
--- a/.yamllint
+++ b/.yamllint
@@ -14,3 +14,9 @@ rules:
     comments:
         min-spaces-from-content: 1
     document-start: disable
+    colons: disable
+    indentation: disable
+    line-length: disable
+    trailing-spaces: disable
+    truthy: disable
+    new-line-at-end-of-file: disable
diff --git a/tests/ci/ci_buddy.py b/tests/ci/ci_buddy.py
index 3eba5532e66..688c7d59988 100644
--- a/tests/ci/ci_buddy.py
+++ b/tests/ci/ci_buddy.py
@@ -1,3 +1,4 @@
+import argparse
 import json
 import os
 from typing import Union, Dict
@@ -7,7 +8,7 @@ import requests
 from botocore.exceptions import ClientError
 
 from pr_info import PRInfo
-from ci_utils import Shell
+from ci_utils import Shell, GHActions
 
 
 class CIBuddy:
@@ -29,6 +30,11 @@ class CIBuddy:
         self.commit_url = pr_info.commit_html_url
         self.sha = pr_info.sha[:10]
 
+    def check_workflow(self):
+        res = GHActions.get_workflow_job_result(GHActions.ActionsNames.RunConfig)
+        if res != GHActions.ActionStatuses.SUCCESS:
+            self.post_job_error("Workflow Configuration Failed", critical=True)
+
     @staticmethod
     def _get_webhooks():
         name = "ci_buddy_web_hooks"
@@ -139,7 +145,30 @@ class CIBuddy:
         self.post(message)
 
 
+def parse_args():
+    parser = argparse.ArgumentParser("CI Buddy bot notifies about CI events")
+    parser.add_argument(
+        "--check-wf-status",
+        action="store_true",
+        help="Checks workflow status",
+    )
+    parser.add_argument(
+        "--test",
+        action="store_true",
+        help="for test and debug",
+    )
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="dry run mode",
+    )
+    return parser.parse_args(), parser
+
+
 if __name__ == "__main__":
-    # test
-    buddy = CIBuddy(dry_run=True)
-    buddy.post_job_error("TEst")
+    args, parser = parse_args()
+
+    if args.test:
+        CIBuddy(dry_run=True).post_job_error("TEst")
+    elif args.check_wf_status:
+        CIBuddy(dry_run=args.dry_run).check_workflow()
diff --git a/tests/ci/ci_utils.py b/tests/ci/ci_utils.py
index 9a1b12af310..1963e3f39d0 100644
--- a/tests/ci/ci_utils.py
+++ b/tests/ci/ci_utils.py
@@ -1,3 +1,4 @@
+import json
 import os
 import re
 import subprocess
@@ -11,6 +12,9 @@ import requests
 
 class Envs:
     GITHUB_REPOSITORY = os.getenv("GITHUB_REPOSITORY", "ClickHouse/ClickHouse")
+    WORKFLOW_RESULT_FILE = os.getenv(
+        "WORKFLOW_RESULT_FILE", "/tmp/workflow_results.json"
+    )
 
 
 LABEL_CATEGORIES = {
@@ -79,6 +83,29 @@ def normalize_string(string: str) -> str:
 
 
 class GHActions:
+    class ActionsNames:
+        RunConfig = "RunConfig"
+
+    class ActionStatuses:
+        ERROR = "error"
+        FAILURE = "failure"
+        PENDING = "pending"
+        SUCCESS = "success"
+
+    @staticmethod
+    def get_workflow_job_result(wf_job_name: str) -> Optional[str]:
+        if not Path(Envs.WORKFLOW_RESULT_FILE).exists():
+            print(
+                f"ERROR: Failed to get workflow results from file [{Envs.WORKFLOW_RESULT_FILE}]"
+            )
+            return None
+        with open(Envs.WORKFLOW_RESULT_FILE, "r", encoding="utf-8") as json_file:
+            res = json.load(json_file)
+        if wf_job_name in res:
+            return res[wf_job_name]["result"]  # type: ignore
+        else:
+            return None
+
     @staticmethod
     def print_in_group(group_name: str, lines: Union[Any, List[Any]]) -> None:
         lines = list(lines)

From 2a893ed8cd10e6e3b7506f43b644b5037f96c49a Mon Sep 17 00:00:00 2001
From: Michael Stetsyuk <mike.stetsyuk@gmail.com>
Date: Mon, 22 Jul 2024 17:03:41 +0100
Subject: [PATCH 0595/1488] fxs

---
 src/IO/S3/Client.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/IO/S3/Client.cpp b/src/IO/S3/Client.cpp
index db20420db9f..325c820f8bd 100644
--- a/src/IO/S3/Client.cpp
+++ b/src/IO/S3/Client.cpp
@@ -388,7 +388,8 @@ Model::HeadObjectOutcome Client::HeadObject(HeadObjectRequest & request) const
     if (isClientForDisk())
         CurrentMetrics::add(CurrentMetrics::S3DiskNoKeyErrors);
 
-    return enrichErrorMessage(std::move(result));
+    return enrichErrorMessage(
+        HeadObject(static_cast<const Model::HeadObjectRequest&>(request)));
 }
 
 /// For each request, we wrap the request functions from Aws::S3::Client with doRequest

From 69be00cbf00b1be867760c996244093020da9034 Mon Sep 17 00:00:00 2001
From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com>
Date: Mon, 22 Jul 2024 15:47:53 +0000
Subject: [PATCH 0596/1488] Improve

---
 tests/integration/test_storage_s3/test.py | 78 +++++++++++------------
 1 file changed, 38 insertions(+), 40 deletions(-)

diff --git a/tests/integration/test_storage_s3/test.py b/tests/integration/test_storage_s3/test.py
index 40cbf4b44a6..65a5cdcad29 100644
--- a/tests/integration/test_storage_s3/test.py
+++ b/tests/integration/test_storage_s3/test.py
@@ -154,6 +154,7 @@ def test_put(started_cluster, maybe_auth, positive, compression):
 
 
 def test_partition_by(started_cluster):
+    id = uuid.uuid4()
     bucket = started_cluster.minio_bucket
     instance = started_cluster.instances["dummy"]  # type: ClickHouseInstance
     table_format = "column1 UInt32, column2 UInt32, column3 UInt32"
@@ -161,26 +162,37 @@ def test_partition_by(started_cluster):
     values = "(1, 2, 3), (3, 2, 1), (78, 43, 45)"
     filename = "test_{_partition_id}.csv"
     put_query = f"""INSERT INTO TABLE FUNCTION
-        s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{filename}', 'CSV', '{table_format}')
+        s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{id}/{filename}', 'CSV', '{table_format}')
         PARTITION BY {partition_by} VALUES {values}"""
 
     run_query(instance, put_query)
 
-    assert "1,2,3\n" == get_s3_file_content(started_cluster, bucket, "test_3.csv")
-    assert "3,2,1\n" == get_s3_file_content(started_cluster, bucket, "test_1.csv")
-    assert "78,43,45\n" == get_s3_file_content(started_cluster, bucket, "test_45.csv")
+    assert "1,2,3\n" == get_s3_file_content(started_cluster, bucket, f"{id}/test_3.csv")
+    assert "3,2,1\n" == get_s3_file_content(started_cluster, bucket, f"{id}/test_1.csv")
+    assert "78,43,45\n" == get_s3_file_content(
+        started_cluster, bucket, f"{id}/test_45.csv"
+    )
 
     filename = "test2_{_partition_id}.csv"
     instance.query(
-        f"create table p ({table_format}) engine=S3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{filename}', 'CSV') partition by column3"
+        f"create table p ({table_format}) engine=S3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{id}/{filename}', 'CSV') partition by column3"
     )
     instance.query(f"insert into p values {values}")
-    assert "1,2,3\n" == get_s3_file_content(started_cluster, bucket, "test2_3.csv")
-    assert "3,2,1\n" == get_s3_file_content(started_cluster, bucket, "test2_1.csv")
-    assert "78,43,45\n" == get_s3_file_content(started_cluster, bucket, "test2_45.csv")
+    assert "1,2,3\n" == get_s3_file_content(
+        started_cluster, bucket, f"{id}/test2_3.csv"
+    )
+    assert "3,2,1\n" == get_s3_file_content(
+        started_cluster, bucket, f"{id}/test2_1.csv"
+    )
+    assert "78,43,45\n" == get_s3_file_content(
+        started_cluster, bucket, f"{id}/test2_45.csv"
+    )
+
+    instance.query("drop table p")
 
 
 def test_partition_by_string_column(started_cluster):
+    id = uuid.uuid4()
     bucket = started_cluster.minio_bucket
     instance = started_cluster.instances["dummy"]  # type: ClickHouseInstance
     table_format = "col_num UInt32, col_str String"
@@ -188,21 +200,20 @@ def test_partition_by_string_column(started_cluster):
     values = "(1, 'foo/bar'), (3, 'йцук'), (78, '你好')"
     filename = "test_{_partition_id}.csv"
     put_query = f"""INSERT INTO TABLE FUNCTION
-        s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{filename}', 'CSV', '{table_format}')
+        s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{id}/{filename}', 'CSV', '{table_format}')
         PARTITION BY {partition_by} VALUES {values}"""
 
     run_query(instance, put_query)
 
     assert '1,"foo/bar"\n' == get_s3_file_content(
-        started_cluster, bucket, "test_foo/bar.csv"
-    )
-    assert '3,"йцук"\n' == get_s3_file_content(started_cluster, bucket, "test_йцук.csv")
-    assert '78,"你好"\n' == get_s3_file_content(
-        started_cluster, bucket, "test_你好.csv"
+        started_cluster, bucket, f"{id}/test_foo/bar.csv"
     )
+    assert '3,"йцук"\n' == get_s3_file_content(started_cluster, bucket, f"{id}/test_йцук.csv")
+    assert '78,"你好"\n' == get_s3_file_content(started_cluster, bucket, f"{id}/test_你好.csv")
 
 
 def test_partition_by_const_column(started_cluster):
+    id = uuid.uuid4()
     bucket = started_cluster.minio_bucket
     instance = started_cluster.instances["dummy"]  # type: ClickHouseInstance
     table_format = "column1 UInt32, column2 UInt32, column3 UInt32"
@@ -211,12 +222,14 @@ def test_partition_by_const_column(started_cluster):
     values_csv = "1,2,3\n3,2,1\n78,43,45\n"
     filename = "test_{_partition_id}.csv"
     put_query = f"""INSERT INTO TABLE FUNCTION
-        s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{filename}', 'CSV', '{table_format}')
+        s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{id}/{filename}', 'CSV', '{table_format}')
         PARTITION BY {partition_by} VALUES {values}"""
 
     run_query(instance, put_query)
 
-    assert values_csv == get_s3_file_content(started_cluster, bucket, "test_88.csv")
+    assert values_csv == get_s3_file_content(
+        started_cluster, bucket, f"{id}/test_88.csv"
+    )
 
 
 @pytest.mark.parametrize("special", ["space", "plus"])
@@ -276,46 +289,31 @@ def test_get_path_with_special(started_cluster, special):
 @pytest.mark.parametrize("auth", [pytest.param("'minio','minio123',", id="minio")])
 def test_empty_put(started_cluster, auth):
     # type: (ClickHouseCluster, str) -> None
-
+    id = uuid.uuid4()
     bucket = started_cluster.minio_bucket
     instance = started_cluster.instances["dummy"]  # type: ClickHouseInstance
     table_format = "column1 UInt32, column2 UInt32, column3 UInt32"
 
     drop_empty_table_query = "DROP TABLE IF EXISTS empty_table"
-    create_empty_table_query = """
-        CREATE TABLE empty_table (
-        {}
-        ) ENGINE = Null()
-    """.format(
-        table_format
+    create_empty_table_query = (
+        f"CREATE TABLE empty_table ({table_format}) ENGINE = Null()"
     )
 
     run_query(instance, drop_empty_table_query)
     run_query(instance, create_empty_table_query)
 
     filename = "empty_put_test.csv"
-    put_query = "insert into table function s3('http://{}:{}/{}/{}', {}'CSV', '{}') select * from empty_table".format(
-        started_cluster.minio_ip,
-        MINIO_INTERNAL_PORT,
-        bucket,
-        filename,
-        auth,
-        table_format,
-    )
+    put_query = f"""insert into table function
+        s3('http://{started_cluster.minio_ip}:{MINIO_INTERNAL_PORT}/{bucket}/{id}/{filename}', {auth} 'CSV', '{table_format}')
+        select * from empty_table"""
 
     run_query(instance, put_query)
 
     assert (
         run_query(
             instance,
-            "select count(*) from s3('http://{}:{}/{}/{}', {}'CSV', '{}')".format(
-                started_cluster.minio_ip,
-                MINIO_INTERNAL_PORT,
-                bucket,
-                filename,
-                auth,
-                table_format,
-            ),
+            f"""select count(*) from
+            s3('http://{started_cluster.minio_ip}:{MINIO_INTERNAL_PORT}/{bucket}/{id}/{filename}', {auth} 'CSV', '{table_format}')""",
         )
         == "0\n"
     )
@@ -881,7 +879,7 @@ def test_storage_s3_get_unstable(started_cluster):
     bucket = started_cluster.minio_bucket
     instance = started_cluster.instances["dummy"]
     table_format = "column1 Int64, column2 Int64, column3 Int64, column4 Int64"
-    get_query = f"SELECT count(), sum(column3), sum(column4) FROM s3('http://resolver:8081/{started_cluster.minio_bucket}/test.csv', 'CSV', '{table_format}') FORMAT CSV"
+    get_query = f"SELECT count(), sum(column3), sum(column4) FROM s3('http://resolver:8081/{started_cluster.minio_bucket}/test.csv', 'CSV', '{table_format}') SETTINGS s3_max_single_read_retries=30 FORMAT CSV"
     result = run_query(instance, get_query)
     assert result.splitlines() == ["500001,500000,0"]
 

From ac679892ab656788d9c9a23362ad82082e8620d4 Mon Sep 17 00:00:00 2001
From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com>
Date: Mon, 22 Jul 2024 16:12:11 +0000
Subject: [PATCH 0597/1488] fix

---
 tests/integration/test_storage_s3/test.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/tests/integration/test_storage_s3/test.py b/tests/integration/test_storage_s3/test.py
index 65a5cdcad29..84c887be388 100644
--- a/tests/integration/test_storage_s3/test.py
+++ b/tests/integration/test_storage_s3/test.py
@@ -497,6 +497,7 @@ def test_put_get_with_globs(started_cluster):
 def test_multipart(started_cluster, maybe_auth, positive):
     # type: (ClickHouseCluster, str, bool) -> None
 
+    id = uuid.uuid4()
     bucket = (
         started_cluster.minio_bucket
         if not maybe_auth
@@ -519,7 +520,7 @@ def test_multipart(started_cluster, maybe_auth, positive):
 
     assert len(csv_data) > min_part_size_bytes
 
-    filename = "test_multipart.csv"
+    filename = f"{id}/test_multipart.csv"
     put_query = "insert into table function s3('http://{}:{}/{}/{}', {}'CSV', '{}') format CSV".format(
         started_cluster.minio_redirect_host,
         started_cluster.minio_redirect_port,
@@ -691,7 +692,7 @@ def test_s3_glob_many_objects_under_selection(started_cluster):
         def create_files(thread_num):
             for f_num in range(thread_num * 63, thread_num * 63 + 63):
                 path = f"folder1/file{f_num}.csv"
-                query = "insert into table function s3('http://{}:{}/{}/{}', 'CSV', '{}') values {}".format(
+                query = "insert into table function s3('http://{}:{}/{}/{}', 'CSV', '{}') settings s3_truncate_on_insert=1 values {}".format(
                     started_cluster.minio_ip,
                     MINIO_INTERNAL_PORT,
                     bucket,
@@ -704,7 +705,7 @@ def test_s3_glob_many_objects_under_selection(started_cluster):
         jobs.append(threading.Thread(target=create_files, args=(thread_num,)))
         jobs[-1].start()
 
-    query = "insert into table function s3('http://{}:{}/{}/{}', 'CSV', '{}') values {}".format(
+    query = "insert into table function s3('http://{}:{}/{}/{}', 'CSV', '{}') settings s3_truncate_on_insert=1 values {}".format(
         started_cluster.minio_ip,
         MINIO_INTERNAL_PORT,
         bucket,

From 1d7fcade729b68276dd3c58bec4d81983ca476b1 Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Mon, 22 Jul 2024 17:28:12 +0200
Subject: [PATCH 0598/1488] Fixes

---
 src/Common/FailPoint.cpp                                      | 2 +-
 src/Storages/StorageKeeperMap.cpp                             | 4 ++++
 .../0_stateless/03208_keepermap_incomplete_data_drop.sql      | 3 ++-
 3 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/Common/FailPoint.cpp b/src/Common/FailPoint.cpp
index b952e9725e3..7b8b5036af0 100644
--- a/src/Common/FailPoint.cpp
+++ b/src/Common/FailPoint.cpp
@@ -58,7 +58,7 @@ static struct InitFiu
     PAUSEABLE(dummy_pausable_failpoint) \
     ONCE(execute_query_calling_empty_set_result_func_on_exception) \
     ONCE(receive_timeout_on_table_status_response) \
-    ONCE(keepermap_fail_drop_data) \
+    REGULAR(keepermap_fail_drop_data) \
 
 
 namespace FailPoints
diff --git a/src/Storages/StorageKeeperMap.cpp b/src/Storages/StorageKeeperMap.cpp
index 1c1de245d10..b32a2d302a7 100644
--- a/src/Storages/StorageKeeperMap.cpp
+++ b/src/Storages/StorageKeeperMap.cpp
@@ -417,6 +417,9 @@ StorageKeeperMap::StorageKeeperMap(
 
             auto code = client->tryCreate(zk_table_path, "", zkutil::CreateMode::Persistent);
 
+            /// A table on the same Keeper path already exists, we just appended our table id to subscribe as a new replica
+            /// We still don't know if the table matches the expected metadata so table_is_valid is not changed
+            /// It will be checked lazily on the first operation
             if (code == Coordination::Error::ZOK)
                 return;
 
@@ -474,6 +477,7 @@ StorageKeeperMap::StorageKeeperMap(
 
 
         table_is_valid = true;
+        /// we are the first table created for the specified Keeper path, i.e. we are the first replica
         return;
     }
 
diff --git a/tests/queries/0_stateless/03208_keepermap_incomplete_data_drop.sql b/tests/queries/0_stateless/03208_keepermap_incomplete_data_drop.sql
index ad0603f12a9..e8d4a5bc298 100644
--- a/tests/queries/0_stateless/03208_keepermap_incomplete_data_drop.sql
+++ b/tests/queries/0_stateless/03208_keepermap_incomplete_data_drop.sql
@@ -3,5 +3,6 @@ DROP TABLE IF EXISTS 03208_keepermap_test SYNC;
 CREATE TABLE 03208_keepermap_test (key UInt64, value UInt64) Engine=KeeperMap('/' || currentDatabase() || '/test03208') PRIMARY KEY(key);
 INSERT INTO 03208_keepermap_test VALUES (1, 11);
 SYSTEM ENABLE FAILPOINT keepermap_fail_drop_data;
-DROP TABLE 03208_keepermap_test;
+DROP TABLE 03208_keepermap_test SYNC; -- { KEEPER_EXCEPTION }
+SYSTEM DISABLE FAILPOINT keepermap_fail_drop_data;
 CREATE TABLE 03208_keepermap_test_another (key UInt64, value UInt64) Engine=KeeperMap('/' || currentDatabase() || '/test03208') PRIMARY KEY(key);

From 4b0b5b7f2dd66ecfd9cb0b533166a229e501cd52 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Mon, 22 Jul 2024 18:53:26 +0200
Subject: [PATCH 0599/1488] groupArrayIntersect: Fix internal name

---
 .../AggregateFunctionGroupArrayIntersect.cpp                  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/AggregateFunctions/AggregateFunctionGroupArrayIntersect.cpp b/src/AggregateFunctions/AggregateFunctionGroupArrayIntersect.cpp
index 20acda213da..591422adc57 100644
--- a/src/AggregateFunctions/AggregateFunctionGroupArrayIntersect.cpp
+++ b/src/AggregateFunctions/AggregateFunctionGroupArrayIntersect.cpp
@@ -69,7 +69,7 @@ public:
         : IAggregateFunctionDataHelper<AggregateFunctionGroupArrayIntersectData<T>,
           AggregateFunctionGroupArrayIntersect<T>>({argument_type}, parameters_, result_type_) {}
 
-    String getName() const override { return "GroupArrayIntersect"; }
+    String getName() const override { return "groupArrayIntersect"; }
 
     bool allocatesMemoryInArena() const override { return false; }
 
@@ -213,7 +213,7 @@ public:
         : IAggregateFunctionDataHelper<AggregateFunctionGroupArrayIntersectGenericData, AggregateFunctionGroupArrayIntersectGeneric<is_plain_column>>({input_data_type_}, parameters_, result_type_)
         , input_data_type(result_type_) {}
 
-    String getName() const override { return "GroupArrayIntersect"; }
+    String getName() const override { return "groupArrayIntersect"; }
 
     bool allocatesMemoryInArena() const override { return true; }
 

From 468bd551c64057fb056ed5434e68c26ded062a84 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Mon, 22 Jul 2024 19:02:20 +0200
Subject: [PATCH 0600/1488] Fix includes

---
 .../AggregateFunctionGroupArrayIntersect.cpp  | 24 ++++++++-----------
 1 file changed, 10 insertions(+), 14 deletions(-)

diff --git a/src/AggregateFunctions/AggregateFunctionGroupArrayIntersect.cpp b/src/AggregateFunctions/AggregateFunctionGroupArrayIntersect.cpp
index 591422adc57..1529cd5899a 100644
--- a/src/AggregateFunctions/AggregateFunctionGroupArrayIntersect.cpp
+++ b/src/AggregateFunctions/AggregateFunctionGroupArrayIntersect.cpp
@@ -1,12 +1,12 @@
-#include <cassert>
-#include <memory>
-
 #include <IO/WriteHelpers.h>
 #include <IO/ReadHelpers.h>
 #include <IO/ReadHelpersArena.h>
 
 #include <DataTypes/DataTypeArray.h>
-#include <DataTypes/DataTypesNumber.h>
+#include <DataTypes/DataTypeDate.h>
+#include <DataTypes/DataTypeDate32.h>
+#include <DataTypes/DataTypeDateTime.h>
+#include <DataTypes/DataTypeDateTime64.h>
 #include <DataTypes/DataTypeString.h>
 
 #include <Columns/ColumnArray.h>
@@ -15,18 +15,14 @@
 #include <Common/HashTable/HashTableKeyHolder.h>
 #include <Common/assert_cast.h>
 
-#include <AggregateFunctions/IAggregateFunction.h>
-#include <AggregateFunctions/KeyHolderHelpers.h>
-
 #include <Core/Field.h>
 
 #include <AggregateFunctions/AggregateFunctionFactory.h>
-#include <AggregateFunctions/Helpers.h>
 #include <AggregateFunctions/FactoryHelpers.h>
-#include <DataTypes/DataTypeDate.h>
-#include <DataTypes/DataTypeDate32.h>
-#include <DataTypes/DataTypeDateTime.h>
-#include <DataTypes/DataTypeDateTime64.h>
+#include <AggregateFunctions/Helpers.h>
+#include <AggregateFunctions/IAggregateFunction.h>
+
+#include <memory>
 
 
 namespace DB
@@ -240,7 +236,7 @@ public:
                 {
                     const char * begin = nullptr;
                     StringRef serialized = data_column->serializeValueIntoArena(offset + i, *arena, begin);
-                    assert(serialized.data != nullptr);
+                    chassert(serialized.data != nullptr);
                     set.emplace(SerializedKeyHolder{serialized, *arena}, it, inserted);
                 }
             }
@@ -260,7 +256,7 @@ public:
                 {
                     const char * begin = nullptr;
                     StringRef serialized = data_column->serializeValueIntoArena(offset + i, *arena, begin);
-                    assert(serialized.data != nullptr);
+                    chassert(serialized.data != nullptr);
                     it = set.find(serialized);
 
                     if (it != nullptr)

From fd2d56d58a6ebfd7c995bfc88fafde75dd12381f Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Mon, 22 Jul 2024 19:05:49 +0200
Subject: [PATCH 0601/1488] Better processing of broken parts and their
 projections

---
 src/Storages/MergeTree/DataPartsExchange.cpp  | 10 ++++++--
 .../MergeTree/MergeTreeSequentialSource.cpp   |  3 ++-
 src/Storages/MergeTree/checkDataPart.cpp      | 25 ++++++++++++-------
 3 files changed, 26 insertions(+), 12 deletions(-)

diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp
index 8e73021d3e7..14c503d0420 100644
--- a/src/Storages/MergeTree/DataPartsExchange.cpp
+++ b/src/Storages/MergeTree/DataPartsExchange.cpp
@@ -16,6 +16,7 @@
 #include <Storages/MergeTree/DataPartStorageOnDiskFull.h>
 #include <Storages/MergeTree/MergeTreeData.h>
 #include <Storages/MergeTree/MergeTreeSettings.h>
+#include <Storages/MergeTree/checkDataPart.h>
 #include <Common/CurrentMetrics.h>
 #include <Common/NetException.h>
 #include <Common/randomDelay.h>
@@ -224,14 +225,19 @@ void Service::processQuery(const HTMLForm & params, ReadBuffer & /*body*/, Write
     }
     catch (const Exception & e)
     {
-        if (e.code() != ErrorCodes::ABORTED && e.code() != ErrorCodes::CANNOT_WRITE_TO_OSTREAM)
+        if (e.code() != ErrorCodes::ABORTED
+            && e.code() != ErrorCodes::CANNOT_WRITE_TO_OSTREAM
+            && !isRetryableException(std::current_exception()))
+        {
             report_broken_part();
+        }
 
         throw;
     }
     catch (...)
     {
-        report_broken_part();
+        if (!isRetryableException(std::current_exception()))
+            report_broken_part();
         throw;
     }
 }
diff --git a/src/Storages/MergeTree/MergeTreeSequentialSource.cpp b/src/Storages/MergeTree/MergeTreeSequentialSource.cpp
index 07476e8b2e9..6eec03c4f18 100644
--- a/src/Storages/MergeTree/MergeTreeSequentialSource.cpp
+++ b/src/Storages/MergeTree/MergeTreeSequentialSource.cpp
@@ -15,6 +15,7 @@
 #include <Processors/QueryPlan/FilterStep.h>
 #include <Common/logger_useful.h>
 #include <Processors/Merges/Algorithms/MergeTreePartLevelInfo.h>
+#include <Storages/MergeTree/checkDataPart.h>
 
 namespace DB
 {
@@ -281,7 +282,7 @@ try
 catch (...)
 {
     /// Suspicion of the broken part. A part is added to the queue for verification.
-    if (getCurrentExceptionCode() != ErrorCodes::MEMORY_LIMIT_EXCEEDED)
+    if (!isRetryableException(std::current_exception()))
         storage.reportBrokenPart(data_part);
     throw;
 }
diff --git a/src/Storages/MergeTree/checkDataPart.cpp b/src/Storages/MergeTree/checkDataPart.cpp
index 774fd95ebc6..922e6e8fb5d 100644
--- a/src/Storages/MergeTree/checkDataPart.cpp
+++ b/src/Storages/MergeTree/checkDataPart.cpp
@@ -36,6 +36,7 @@ namespace ErrorCodes
     extern const int CANNOT_ALLOCATE_MEMORY;
     extern const int CANNOT_MUNMAP;
     extern const int CANNOT_MREMAP;
+    extern const int CANNOT_SCHEDULE_TASK;
     extern const int UNEXPECTED_FILE_IN_DATA_PART;
     extern const int NO_FILE_IN_DATA_PART;
     extern const int NETWORK_ERROR;
@@ -85,7 +86,8 @@ bool isRetryableException(std::exception_ptr exception_ptr)
     {
         return isNotEnoughMemoryErrorCode(e.code())
             || e.code() == ErrorCodes::NETWORK_ERROR
-            || e.code() == ErrorCodes::SOCKET_TIMEOUT;
+            || e.code() == ErrorCodes::SOCKET_TIMEOUT
+            || e.code() == ErrorCodes::CANNOT_SCHEDULE_TASK;
     }
     catch (const Poco::Net::NetException &)
     {
@@ -329,16 +331,21 @@ static IMergeTreeDataPart::Checksums checkDataPart(
         projections_on_disk.erase(projection_file);
     }
 
-    if (throw_on_broken_projection && !broken_projections_message.empty())
+    if (throw_on_broken_projection)
     {
-        throw Exception(ErrorCodes::BROKEN_PROJECTION, "{}", broken_projections_message);
-    }
+        if (!broken_projections_message.empty())
+        {
+            throw Exception(ErrorCodes::BROKEN_PROJECTION, "{}", broken_projections_message);
+        }
 
-    if (require_checksums && !projections_on_disk.empty())
-    {
-        throw Exception(ErrorCodes::UNEXPECTED_FILE_IN_DATA_PART,
-            "Found unexpected projection directories: {}",
-            fmt::join(projections_on_disk, ","));
+        /// This one is actually not broken, just redundant files on disk which
+        /// MergeTree will never use.
+        if (require_checksums && !projections_on_disk.empty())
+        {
+            throw Exception(ErrorCodes::UNEXPECTED_FILE_IN_DATA_PART,
+                "Found unexpected projection directories: {}",
+                fmt::join(projections_on_disk, ","));
+        }
     }
 
     if (is_cancelled())

From 078bddd65e0978c3feb2f8d77008f3c23bbc136f Mon Sep 17 00:00:00 2001
From: joelynch <joelynch112@gmail.com>
Date: Mon, 22 Jul 2024 18:55:23 +0200
Subject: [PATCH 0602/1488] Let window view take comments

---
 src/Parsers/ParserCreateQuery.cpp | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/Parsers/ParserCreateQuery.cpp b/src/Parsers/ParserCreateQuery.cpp
index 014dc7bd3bf..3e908ea3602 100644
--- a/src/Parsers/ParserCreateQuery.cpp
+++ b/src/Parsers/ParserCreateQuery.cpp
@@ -1176,6 +1176,7 @@ bool ParserCreateWindowViewQuery::parseImpl(Pos & pos, ASTPtr & node, Expected &
     if (!select_p.parse(pos, select, expected))
         return false;
 
+    auto comment = parseComment(pos, expected);
 
     auto query = std::make_shared<ASTCreateQuery>();
     node = query;
@@ -1194,6 +1195,8 @@ bool ParserCreateWindowViewQuery::parseImpl(Pos & pos, ASTPtr & node, Expected &
         query->children.push_back(query->database);
     if (query->table)
         query->children.push_back(query->table);
+    if (comment)
+        query->set(query->comment, comment);
 
     if (to_table)
         query->to_table_id = to_table->as<ASTTableIdentifier>()->getTableId();

From 240f04561e1ebca2fb1823a6ee34d4f2c15d1e66 Mon Sep 17 00:00:00 2001
From: joelynch <joelynch112@gmail.com>
Date: Mon, 22 Jul 2024 19:07:46 +0200
Subject: [PATCH 0603/1488] Fix docs for COMMENT clause

---
 docs/en/sql-reference/statements/create/table.md | 7 +------
 docs/en/sql-reference/statements/create/view.md  | 8 +++++++-
 2 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/docs/en/sql-reference/statements/create/table.md b/docs/en/sql-reference/statements/create/table.md
index b866d0b9f5f..9c8984d698f 100644
--- a/docs/en/sql-reference/statements/create/table.md
+++ b/docs/en/sql-reference/statements/create/table.md
@@ -21,7 +21,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster]
     name2 [type2] [NULL|NOT NULL] [DEFAULT|MATERIALIZED|EPHEMERAL|ALIAS expr2] [COMMENT 'comment for column'] [compression_codec] [TTL expr2],
     ...
 ) ENGINE = engine
-  COMMENT 'comment for table'
+  [COMMENT 'comment for table']
 ```
 
 Creates a table named `table_name` in the `db` database or the current database if `db` is not set, with the structure specified in brackets and the `engine` engine.
@@ -626,11 +626,6 @@ SELECT * FROM base.t1;
 
 You can add a comment to the table when you creating it.
 
-:::note
-The comment clause is supported by all table engines except [Kafka](../../../engines/table-engines/integrations/kafka.md), [RabbitMQ](../../../engines/table-engines/integrations/rabbitmq.md) and [EmbeddedRocksDB](../../../engines/table-engines/integrations/embedded-rocksdb.md).
-:::
-
-
 **Syntax**
 
 ``` sql
diff --git a/docs/en/sql-reference/statements/create/view.md b/docs/en/sql-reference/statements/create/view.md
index 1fabb6d8cc7..2931f7020fb 100644
--- a/docs/en/sql-reference/statements/create/view.md
+++ b/docs/en/sql-reference/statements/create/view.md
@@ -16,6 +16,7 @@ Syntax:
 CREATE [OR REPLACE] VIEW [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster_name] 
 [DEFINER = { user | CURRENT_USER }] [SQL SECURITY { DEFINER | INVOKER | NONE }] 
 AS SELECT ...
+[COMMENT 'comment']
 ```
 
 Normal views do not store any data. They just perform a read from another table on each access. In other words, a normal view is nothing more than a saved query. When reading from a view, this saved query is used as a subquery in the [FROM](../../../sql-reference/statements/select/from.md) clause.
@@ -57,6 +58,7 @@ SELECT * FROM view(column1=value1, column2=value2 ...)
 CREATE MATERIALIZED VIEW [IF NOT EXISTS] [db.]table_name [ON CLUSTER] [TO[db.]name] [ENGINE = engine] [POPULATE] 
 [DEFINER = { user | CURRENT_USER }] [SQL SECURITY { DEFINER | INVOKER | NONE }] 
 AS SELECT ...
+[COMMENT 'comment']
 ```
 
 :::tip
@@ -161,6 +163,7 @@ RANDOMIZE FOR interval
 DEPENDS ON [db.]name [, [db.]name [, ...]]
 [TO[db.]name] [(columns)] [ENGINE = engine] [EMPTY]
 AS SELECT ...
+[COMMENT 'comment']
 ```
 where `interval` is a sequence of simple intervals:
 ```sql
@@ -267,7 +270,10 @@ This is an experimental feature that may change in backwards-incompatible ways i
 :::
 
 ``` sql
-CREATE WINDOW VIEW [IF NOT EXISTS] [db.]table_name [TO [db.]table_name] [INNER ENGINE engine] [ENGINE engine] [WATERMARK strategy] [ALLOWED_LATENESS interval_function] [POPULATE] AS SELECT ... GROUP BY time_window_function
+CREATE WINDOW VIEW [IF NOT EXISTS] [db.]table_name [TO [db.]table_name] [INNER ENGINE engine] [ENGINE engine] [WATERMARK strategy] [ALLOWED_LATENESS interval_function] [POPULATE]
+AS SELECT ...
+GROUP BY time_window_function
+[COMMENT 'comment']
 ```
 
 Window view can aggregate data by time window and output the results when the window is ready to fire. It stores the partial aggregation results in an inner(or specified) table to reduce latency and can push the processing result to a specified table or push notifications using the WATCH query.

From 4a2708658d6d9b3ab0de9eab41d4e97bf04c3523 Mon Sep 17 00:00:00 2001
From: joelynch <joelynch112@gmail.com>
Date: Mon, 22 Jul 2024 19:08:04 +0200
Subject: [PATCH 0604/1488] Add more tables to table comment test

---
 .../0_stateless/01821_table_comment.reference |  8 ++-
 .../0_stateless/01821_table_comment.sql       | 53 +++++++++++++++++--
 2 files changed, 54 insertions(+), 7 deletions(-)

diff --git a/tests/queries/0_stateless/01821_table_comment.reference b/tests/queries/0_stateless/01821_table_comment.reference
index 05acabae3d4..cdd87df43d2 100644
--- a/tests/queries/0_stateless/01821_table_comment.reference
+++ b/tests/queries/0_stateless/01821_table_comment.reference
@@ -1,4 +1,8 @@
-t1	this is a temtorary table
+t1	this is a temporary table
 t2	this is a MergeTree table
 t3	this is a Log table
-CREATE TABLE default.t1\n(\n    `n` Int8\n)\nENGINE = Memory\nCOMMENT \'this is a temtorary table\'
+t4	this is a Kafka table
+t5	this is a EmbeddedRocksDB table
+t6	this is a Executable table
+t7	this is a WindowView table
+CREATE TABLE default.t1\n(\n    `n` Int8\n)\nENGINE = Memory\nCOMMENT \'this is a temporary table\'
diff --git a/tests/queries/0_stateless/01821_table_comment.sql b/tests/queries/0_stateless/01821_table_comment.sql
index 4bd71d3e278..32b89af0750 100644
--- a/tests/queries/0_stateless/01821_table_comment.sql
+++ b/tests/queries/0_stateless/01821_table_comment.sql
@@ -9,7 +9,7 @@ CREATE TABLE t1
     `n` Int8
 )
 ENGINE = Memory
-COMMENT 'this is a temtorary table';
+COMMENT 'this is a temporary table';
 
 CREATE TABLE t2
 (
@@ -26,14 +26,57 @@ CREATE TABLE t3
 ENGINE = Log
 COMMENT 'this is a Log table';
 
+CREATE TABLE t4
+(
+    `n` Int8
+)
+ENGINE = Kafka
+SETTINGS
+    kafka_broker_list = 'localhost:10000',
+    kafka_topic_list = 'test',
+    kafka_group_name = 'test',
+    kafka_format = 'JSONEachRow'
+COMMENT 'this is a Kafka table';
+
+CREATE TABLE t5
+(
+    `n` Int8
+)
+ENGINE = EmbeddedRocksDB
+PRIMARY KEY n
+COMMENT 'this is a EmbeddedRocksDB table';
+
+CREATE TABLE t6
+(
+    `n` Int8
+)
+ENGINE = Executable('script.py', TabSeparated)
+COMMENT 'this is a Executable table';
+
+SET allow_experimental_window_view = 1;
+-- New analyzer doesn't support WindowView tables
+SET allow_experimental_analyzer = 0;
+
+CREATE WINDOW VIEW t7
+(
+    `n` Int8
+)
+ENGINE MergeTree
+ORDER BY n
+AS SELECT 1
+GROUP BY tumble(now(), toIntervalDay('1'))
+COMMENT 'this is a WindowView table';
+
+SET allow_experimental_analyzer = 1;
+
 SELECT
     name,
     comment
 FROM system.tables
-WHERE name IN ('t1', 't2', 't3') AND database = currentDatabase() order by name;
+WHERE name IN ('t1', 't2', 't3', 't4', 't5', 't6', 't7')
+    AND database = currentDatabase() order by name;
 
 SHOW CREATE TABLE t1;
 
-DROP TABLE t1;
-DROP TABLE t2;
-DROP TABLE t3;
+DROP TABLE t1, t2, t3, t4, t5, t6;
+DROP VIEW t7;

From 454eb1af7ed69d00617e6bfa3daece508fcb565b Mon Sep 17 00:00:00 2001
From: avogar <avogar@clickhouse.com>
Date: Mon, 22 Jul 2024 17:15:05 +0000
Subject: [PATCH 0605/1488] Fix invalid format detection in schema inference

tmp
---
 src/Storages/ObjectStorage/ReadBufferIterator.cpp      |  4 +++-
 src/Storages/StorageFile.cpp                           |  8 ++++++--
 src/Storages/StorageURL.cpp                            |  4 +++-
 .../03023_invalid_format_detection.reference           |  1 +
 .../0_stateless/03023_invalid_format_detection.sh      | 10 ++++++++++
 5 files changed, 23 insertions(+), 4 deletions(-)
 create mode 100644 tests/queries/0_stateless/03023_invalid_format_detection.reference
 create mode 100755 tests/queries/0_stateless/03023_invalid_format_detection.sh

diff --git a/src/Storages/ObjectStorage/ReadBufferIterator.cpp b/src/Storages/ObjectStorage/ReadBufferIterator.cpp
index 78cdc442f64..3d428073e88 100644
--- a/src/Storages/ObjectStorage/ReadBufferIterator.cpp
+++ b/src/Storages/ObjectStorage/ReadBufferIterator.cpp
@@ -162,7 +162,9 @@ ReadBufferIterator::Data ReadBufferIterator::next()
         {
             for (const auto & object_info : read_keys)
             {
-                if (auto format_from_file_name = FormatFactory::instance().tryGetFormatFromFileName(object_info->getFileName()))
+                auto format_from_file_name = FormatFactory::instance().tryGetFormatFromFileName(object_info->getFileName());
+                /// Use this format only if we have a schema reader for it.
+                if (format_from_file_name && FormatFactory::instance().checkIfFormatHasAnySchemaReader(*format_from_file_name))
                 {
                     format = format_from_file_name;
                     break;
diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp
index 7f39ff615f0..b4c03fa86c6 100644
--- a/src/Storages/StorageFile.cpp
+++ b/src/Storages/StorageFile.cpp
@@ -416,7 +416,9 @@ namespace
                 {
                     for (const auto & path : paths)
                     {
-                        if (auto format_from_path = FormatFactory::instance().tryGetFormatFromFileName(path))
+                        auto format_from_path = FormatFactory::instance().tryGetFormatFromFileName(path);
+                        /// Use this format only if we have a schema reader for it.
+                        if (format_from_path && FormatFactory::instance().checkIfFormatHasAnySchemaReader(*format_from_path))
                         {
                             format = format_from_path;
                             break;
@@ -705,7 +707,9 @@ namespace
                     /// If format is unknown we can try to determine it by the file name.
                     if (!format)
                     {
-                        if (auto format_from_file = FormatFactory::instance().tryGetFormatFromFileName(*filename))
+                        auto format_from_file = FormatFactory::instance().tryGetFormatFromFileName(*filename);
+                        /// Use this format only if we have a schema reader for it.
+                        if (format_from_file && FormatFactory::instance().checkIfFormatHasAnySchemaReader(*format_from_file))
                             format = format_from_file;
                     }
 
diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp
index 895da028fc2..4d82c2bfadb 100644
--- a/src/Storages/StorageURL.cpp
+++ b/src/Storages/StorageURL.cpp
@@ -735,7 +735,9 @@ namespace
                     {
                         for (const auto & url : options)
                         {
-                            if (auto format_from_file_name = FormatFactory::instance().tryGetFormatFromFileName(url))
+                            auto format_from_file_name = FormatFactory::instance().tryGetFormatFromFileName(url);
+                            /// Use this format only if we have a schema reader for it.
+                            if (format_from_file_name && FormatFactory::instance().checkIfFormatHasAnySchemaReader(*format_from_file_name))
                             {
                                 format = format_from_file_name;
                                 break;
diff --git a/tests/queries/0_stateless/03023_invalid_format_detection.reference b/tests/queries/0_stateless/03023_invalid_format_detection.reference
new file mode 100644
index 00000000000..d00491fd7e5
--- /dev/null
+++ b/tests/queries/0_stateless/03023_invalid_format_detection.reference
@@ -0,0 +1 @@
+1
diff --git a/tests/queries/0_stateless/03023_invalid_format_detection.sh b/tests/queries/0_stateless/03023_invalid_format_detection.sh
new file mode 100755
index 00000000000..984a42aa218
--- /dev/null
+++ b/tests/queries/0_stateless/03023_invalid_format_detection.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+touch $CLICKHOUSE_TEST_UNIQUE_NAME.xml
+$CLICKHOUSE_LOCAL -q "select * from file('$CLICKHOUSE_TEST_UNIQUE_NAME.*')" 2>&1 | grep -c "CANNOT_DETECT_FORMAT"
+rm $CLICKHOUSE_TEST_UNIQUE_NAME.xml
+

From ef4cb8b28324f1599c082b70e9c9c516c6c519d4 Mon Sep 17 00:00:00 2001
From: avogar <avogar@clickhouse.com>
Date: Mon, 22 Jul 2024 17:18:20 +0000
Subject: [PATCH 0606/1488] Add missing check

---
 src/Storages/ObjectStorage/ReadBufferIterator.cpp | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/Storages/ObjectStorage/ReadBufferIterator.cpp b/src/Storages/ObjectStorage/ReadBufferIterator.cpp
index 3d428073e88..df78f128c80 100644
--- a/src/Storages/ObjectStorage/ReadBufferIterator.cpp
+++ b/src/Storages/ObjectStorage/ReadBufferIterator.cpp
@@ -1,6 +1,7 @@
 #include <Storages/ObjectStorage/ReadBufferIterator.h>
 #include <Storages/ObjectStorage/StorageObjectStorage.h>
 #include <Storages/ObjectStorage/StorageObjectStorageSource.h>
+#include <Core/Settings.h>
 #include <IO/ReadBufferFromFileBase.h>
 
 
@@ -222,7 +223,9 @@ ReadBufferIterator::Data ReadBufferIterator::next()
             {
                 for (auto it = read_keys.begin() + prev_read_keys_size; it != read_keys.end(); ++it)
                 {
-                    if (auto format_from_file_name = FormatFactory::instance().tryGetFormatFromFileName((*it)->getFileName()))
+                    auto format_from_file_name = FormatFactory::instance().tryGetFormatFromFileName((*it)->getFileName());
+                    /// Use this format only if we have a schema reader for it.
+                    if (format_from_file_name && FormatFactory::instance().checkIfFormatHasAnySchemaReader(*format_from_file_name))
                     {
                         format = format_from_file_name;
                         break;

From ea78ae85ed38e757f8dddd99d2018f01f3996f25 Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Mon, 22 Jul 2024 19:23:45 +0200
Subject: [PATCH 0607/1488] Fix style

---
 src/Storages/MergeTree/MergeTreeSequentialSource.cpp | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/src/Storages/MergeTree/MergeTreeSequentialSource.cpp b/src/Storages/MergeTree/MergeTreeSequentialSource.cpp
index 6eec03c4f18..311720728e7 100644
--- a/src/Storages/MergeTree/MergeTreeSequentialSource.cpp
+++ b/src/Storages/MergeTree/MergeTreeSequentialSource.cpp
@@ -20,12 +20,6 @@
 namespace DB
 {
 
-namespace ErrorCodes
-{
-    extern const int MEMORY_LIMIT_EXCEEDED;
-}
-
-
 /// Lightweight (in terms of logic) stream for reading single part from
 /// MergeTree, used for merges and mutations.
 ///

From 406addb2dcd0e82a71db06eb8ebaaa34bf1a543d Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Mon, 22 Jul 2024 19:39:37 +0200
Subject: [PATCH 0608/1488] Add aborted to retryable errors

---
 src/Storages/MergeTree/DataPartsExchange.cpp | 3 +--
 src/Storages/MergeTree/checkDataPart.cpp     | 4 +++-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp
index 14c503d0420..061ee356203 100644
--- a/src/Storages/MergeTree/DataPartsExchange.cpp
+++ b/src/Storages/MergeTree/DataPartsExchange.cpp
@@ -225,8 +225,7 @@ void Service::processQuery(const HTMLForm & params, ReadBuffer & /*body*/, Write
     }
     catch (const Exception & e)
     {
-        if (e.code() != ErrorCodes::ABORTED
-            && e.code() != ErrorCodes::CANNOT_WRITE_TO_OSTREAM
+        if (e.code() != ErrorCodes::CANNOT_WRITE_TO_OSTREAM
             && !isRetryableException(std::current_exception()))
         {
             report_broken_part();
diff --git a/src/Storages/MergeTree/checkDataPart.cpp b/src/Storages/MergeTree/checkDataPart.cpp
index 922e6e8fb5d..fb86d9e7603 100644
--- a/src/Storages/MergeTree/checkDataPart.cpp
+++ b/src/Storages/MergeTree/checkDataPart.cpp
@@ -42,6 +42,7 @@ namespace ErrorCodes
     extern const int NETWORK_ERROR;
     extern const int SOCKET_TIMEOUT;
     extern const int BROKEN_PROJECTION;
+    extern const int ABORTED;
 }
 
 
@@ -87,7 +88,8 @@ bool isRetryableException(std::exception_ptr exception_ptr)
         return isNotEnoughMemoryErrorCode(e.code())
             || e.code() == ErrorCodes::NETWORK_ERROR
             || e.code() == ErrorCodes::SOCKET_TIMEOUT
-            || e.code() == ErrorCodes::CANNOT_SCHEDULE_TASK;
+            || e.code() == ErrorCodes::CANNOT_SCHEDULE_TASK
+            || e.code() == ErrorCodes::ABORTED;
     }
     catch (const Poco::Net::NetException &)
     {

From b1029fbd671310a4c8d48070f87d84f33f8842fd Mon Sep 17 00:00:00 2001
From: xogoodnow <xo4ever9999@gmail.com>
Date: Mon, 22 Jul 2024 22:02:54 +0330
Subject: [PATCH 0609/1488] Fixed the style issue

---
 src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp b/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp
index e217d93975d..f51a7a913b8 100644
--- a/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp
+++ b/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp
@@ -37,6 +37,7 @@ namespace ErrorCodes
     extern const int LOGICAL_ERROR;
     extern const int BAD_ARGUMENTS;
     extern const int BAD_QUERY_PARAMETER;
+    extern const int QUERY_NOT_ALLOWED;
 }
 
 namespace

From 71cdf82643fb17b5b68003df314c54ebbca0842f Mon Sep 17 00:00:00 2001
From: Igor Nikonov <igor@clickhouse.com>
Date: Mon, 22 Jul 2024 18:41:30 +0000
Subject: [PATCH 0610/1488] Fix: reset is_async_state flag

---
 src/Processors/Sources/RemoteSource.cpp | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/Processors/Sources/RemoteSource.cpp b/src/Processors/Sources/RemoteSource.cpp
index 46c27676e12..2f9a30296be 100644
--- a/src/Processors/Sources/RemoteSource.cpp
+++ b/src/Processors/Sources/RemoteSource.cpp
@@ -115,7 +115,10 @@ void RemoteSource::onAsyncJobReady()
     if (!was_query_sent)
         return;
 
+    chassert(!preprocessed_packet);
     preprocessed_packet = query_executor->processParallelReplicaPacketIfAny();
+    if (preprocessed_packet)
+        is_async_state = false;
 }
 
 std::optional<Chunk> RemoteSource::tryGenerate()

From 155b28227972fe2f33dac98d3c471c555637d246 Mon Sep 17 00:00:00 2001
From: joelynch <joelynch112@gmail.com>
Date: Mon, 22 Jul 2024 21:30:40 +0200
Subject: [PATCH 0611/1488] This cannot be fasttest because it uses Kafka
 engine

---
 tests/queries/0_stateless/01821_table_comment.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/01821_table_comment.sql b/tests/queries/0_stateless/01821_table_comment.sql
index 32b89af0750..4946e46d37a 100644
--- a/tests/queries/0_stateless/01821_table_comment.sql
+++ b/tests/queries/0_stateless/01821_table_comment.sql
@@ -1,4 +1,4 @@
--- Tags: no-parallel
+-- Tags: no-parallel, no-fasttest
 
 DROP TABLE IF EXISTS t1;
 DROP TABLE IF EXISTS t2;

From 7f5c58f599d34f690c4a04e4223a2f86a433d0e9 Mon Sep 17 00:00:00 2001
From: Michael Stetsyuk <mike.stetsyuk@gmail.com>
Date: Mon, 22 Jul 2024 20:58:56 +0100
Subject: [PATCH 0612/1488] fxs

---
 tests/queries/0_stateless/01293_show_clusters.reference | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/queries/0_stateless/01293_show_clusters.reference b/tests/queries/0_stateless/01293_show_clusters.reference
index 9569fcf2e37..e140f207022 100644
--- a/tests/queries/0_stateless/01293_show_clusters.reference
+++ b/tests/queries/0_stateless/01293_show_clusters.reference
@@ -1,3 +1,3 @@
 test_shard_localhost
-test_cluster_one_shard_two_replicas	1	1	0	1	127.0.0.1	127.0.0.1	9000	1	default	0	NULL
-test_cluster_one_shard_two_replicas	1	1	0	2	127.0.0.2	127.0.0.2	9000	0	default	0	NULL
+test_cluster_one_shard_two_replicas	1	1	0	1	127.0.0.1	127.0.0.1	9000	1	default
+test_cluster_one_shard_two_replicas	1	1	0	2	127.0.0.2	127.0.0.2	9000	0	default

From 72f4919fdad5217f48bd83e51ce2d1f3f083087b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Mi=D1=81hael=20Stetsyuk?=
 <59827607+mstetsyuk@users.noreply.github.com>
Date: Mon, 22 Jul 2024 21:33:47 +0100
Subject: [PATCH 0613/1488] Update src/Common/CurrentMetrics.cpp

Co-authored-by: Sema Checherinda <104093494+CheSema@users.noreply.github.com>
---
 src/Common/CurrentMetrics.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Common/CurrentMetrics.cpp b/src/Common/CurrentMetrics.cpp
index 2fedba0175b..39198147794 100644
--- a/src/Common/CurrentMetrics.cpp
+++ b/src/Common/CurrentMetrics.cpp
@@ -307,7 +307,7 @@
     M(FilteringMarksWithPrimaryKey, "Number of threads currently doing filtering of mark ranges by the primary key") \
     M(FilteringMarksWithSecondaryKeys, "Number of threads currently doing filtering of mark ranges by secondary keys") \
     \
-    M(S3DiskNoKeyErrors, "Number of no-key S3 disk errors") \
+    M(S3DiskNoKeyErrors, "The number of `NoSuchKey` errors that occur when reading data from S3 cloud storage through ClickHouse disks.") \
 
 #ifdef APPLY_FOR_EXTERNAL_METRICS
     #define APPLY_FOR_METRICS(M) APPLY_FOR_BUILTIN_METRICS(M) APPLY_FOR_EXTERNAL_METRICS(M)

From 7353064d1c97231d02aab2ffade3e4c63a3ebcb1 Mon Sep 17 00:00:00 2001
From: Alexander Tokmakov <tavplubix@clickhouse.com>
Date: Mon, 22 Jul 2024 23:12:22 +0200
Subject: [PATCH 0614/1488] fix

---
 tests/queries/0_stateless/replication.lib | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/replication.lib b/tests/queries/0_stateless/replication.lib
index b2fcc422508..fe867537000 100755
--- a/tests/queries/0_stateless/replication.lib
+++ b/tests/queries/0_stateless/replication.lib
@@ -52,7 +52,12 @@ function check_replication_consistency()
     check_query_part=$2
 
     # Try to kill some mutations because sometimes tests run too much (it's not guarenteed to kill all mutations, see below)
-    ${CLICKHOUSE_CLIENT} -q "KILL MUTATION WHERE database=currentDatabase() AND table like '$table_name_prefix%'" > /dev/null
+    # Try multiple replicas, because queries are not finished yet, and "global" KILL MUTATION may fail due to another query (like DROP TABLE)
+    readarray -t tables_arr < <(${CLICKHOUSE_CLIENT} -q "SELECT name FROM system.tables WHERE database=currentDatabase() AND name like '$table_name_prefix%'")
+    for t in "${tables_arr[@]}"
+    do
+        ${CLICKHOUSE_CLIENT} -q "KILL MUTATION WHERE database=currentDatabase() AND table='$t'" > /dev/null 2>/dev/null
+    done
 
     # Wait for all queries to finish (query may still be running if thread is killed by timeout)
     num_tries=0

From abb6025e727d8b825ed62b07d8684dc0f7eddb7d Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Tue, 23 Jul 2024 02:30:45 +0200
Subject: [PATCH 0615/1488] Fix performance test about the generateRandom table
 function, supposedly

---
 tests/performance/generate_table_function.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/performance/generate_table_function.xml b/tests/performance/generate_table_function.xml
index c219d73b6cf..d56c585188d 100644
--- a/tests/performance/generate_table_function.xml
+++ b/tests/performance/generate_table_function.xml
@@ -7,7 +7,7 @@
     <query>SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('d Date, dt DateTime, dtm DateTime(\'Asia/Istanbul\')', 0, 10, 10) LIMIT 1000000000);</query>
     <query>SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('dt64 DateTime64, dts64 DateTime64(6), dtms64 DateTime64(6 ,\'Asia/Istanbul\')', 0, 10, 10) LIMIT 100000000);</query>
     <query>SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('f32 Float32, f64 Float64', 0, 10, 10) LIMIT 1000000000);</query>
-    <query>SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('d32 Decimal32(4), d64 Decimal64(8), d128 Decimal128(16)', 0, 10, 10) LIMIT 1000000000);</query>
+    <query>SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('d32 Decimal32(4), d64 Decimal64(8), d128 Decimal128(16)', 0, 10, 10) LIMIT 100000000);</query>
     <query>SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('i Tuple(Int32, Int64)', 0, 10, 10) LIMIT 1000000000);</query>
     <query>SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('i Array(Int8)', 0, 10, 10) LIMIT 100000000);</query>
     <query>SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('i Array(Nullable(Int32))', 0, 10, 10) LIMIT 100000000);</query>

From 07f642f1cac5253fda7eaa928ca5e659159b6b16 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Tue, 23 Jul 2024 03:13:52 +0200
Subject: [PATCH 0616/1488] Fix bad tests `share_big_sets`, CC @davenger

---
 ...en_multiple_mutations_tasks_long.reference |  1 +
 ..._between_multiple_mutations_tasks_long.sql | 13 +++++++----
 ...sets_between_mutation_tasks_long.reference |  1 +
 ...e_big_sets_between_mutation_tasks_long.sql | 23 +++++++++++--------
 4 files changed, 23 insertions(+), 15 deletions(-)

diff --git a/tests/queries/0_stateless/02581_share_big_sets_between_multiple_mutations_tasks_long.reference b/tests/queries/0_stateless/02581_share_big_sets_between_multiple_mutations_tasks_long.reference
index 3a92fcf283d..9cb32105006 100644
--- a/tests/queries/0_stateless/02581_share_big_sets_between_multiple_mutations_tasks_long.reference
+++ b/tests/queries/0_stateless/02581_share_big_sets_between_multiple_mutations_tasks_long.reference
@@ -7,3 +7,4 @@ all_4_4_0
 5000	all_2_2_0_9
 5000	all_3_3_0_9
 5000	all_4_4_0_9
+Ok
diff --git a/tests/queries/0_stateless/02581_share_big_sets_between_multiple_mutations_tasks_long.sql b/tests/queries/0_stateless/02581_share_big_sets_between_multiple_mutations_tasks_long.sql
index 741d0177971..631fd9cb2cc 100644
--- a/tests/queries/0_stateless/02581_share_big_sets_between_multiple_mutations_tasks_long.sql
+++ b/tests/queries/0_stateless/02581_share_big_sets_between_multiple_mutations_tasks_long.sql
@@ -18,12 +18,15 @@ SELECT name FROM system.parts WHERE database=currentDatabase() AND table = '0258
 
 -- Start multiple mutations simultaneously
 SYSTEM STOP MERGES 02581_trips;
-ALTER TABLE 02581_trips UPDATE description='5' WHERE id IN (SELECT (number*10 + 5)::UInt32 FROM numbers(200000000)) SETTINGS mutations_sync=0;
-ALTER TABLE 02581_trips UPDATE description='6' WHERE id IN (SELECT (number*10 + 6)::UInt32 FROM numbers(200000000)) SETTINGS mutations_sync=0;
-ALTER TABLE 02581_trips DELETE WHERE id IN (SELECT (number*10 + 7)::UInt32 FROM numbers(200000000)) SETTINGS mutations_sync=0;
-ALTER TABLE 02581_trips UPDATE description='8' WHERE id IN (SELECT (number*10 + 8)::UInt32 FROM numbers(200000000)) SETTINGS mutations_sync=0;
+ALTER TABLE 02581_trips UPDATE description='5' WHERE id IN (SELECT (number*10 + 5)::UInt32 FROM numbers(10000000)) SETTINGS mutations_sync=0;
+ALTER TABLE 02581_trips UPDATE description='6' WHERE id IN (SELECT (number*10 + 6)::UInt32 FROM numbers(10000000)) SETTINGS mutations_sync=0;
+ALTER TABLE 02581_trips DELETE                 WHERE id IN (SELECT (number*10 + 7)::UInt32 FROM numbers(10000000)) SETTINGS mutations_sync=0;
+ALTER TABLE 02581_trips UPDATE description='8' WHERE id IN (SELECT (number*10 + 8)::UInt32 FROM numbers(10000000)) SETTINGS mutations_sync=0;
 SYSTEM START MERGES 02581_trips;
-DELETE FROM 02581_trips WHERE id IN (SELECT (number*10 + 9)::UInt32 FROM numbers(200000000));
+DELETE FROM 02581_trips                        WHERE id IN (SELECT (number*10 + 9)::UInt32 FROM numbers(10000000));
 SELECT count(), _part from 02581_trips WHERE description = '' GROUP BY _part ORDER BY _part;
 
+SYSTEM FLUSH LOGS;
+SELECT DISTINCT peak_memory_usage < 2000000000 ? 'Ok' : toString(tuple(*)) FROM system.part_log WHERE database = currentDatabase() AND event_date >= yesterday() AND table = '02581_trips' AND event_type = 'MutatePart';
+
 DROP TABLE 02581_trips;
diff --git a/tests/queries/0_stateless/02581_share_big_sets_between_mutation_tasks_long.reference b/tests/queries/0_stateless/02581_share_big_sets_between_mutation_tasks_long.reference
index 3a7410d925f..d21598bc12e 100644
--- a/tests/queries/0_stateless/02581_share_big_sets_between_mutation_tasks_long.reference
+++ b/tests/queries/0_stateless/02581_share_big_sets_between_mutation_tasks_long.reference
@@ -10,3 +10,4 @@ all_4_4_0
 20000
 16000
 12000
+Ok
diff --git a/tests/queries/0_stateless/02581_share_big_sets_between_mutation_tasks_long.sql b/tests/queries/0_stateless/02581_share_big_sets_between_mutation_tasks_long.sql
index b7314c8fa47..062f22357e8 100644
--- a/tests/queries/0_stateless/02581_share_big_sets_between_mutation_tasks_long.sql
+++ b/tests/queries/0_stateless/02581_share_big_sets_between_mutation_tasks_long.sql
@@ -18,42 +18,45 @@ SELECT count() from 02581_trips WHERE description = '';
 SELECT name FROM system.parts WHERE database=currentDatabase() AND table = '02581_trips' AND active ORDER BY name;
 
 -- Run mutation with `id` a 'IN big subquery'
-ALTER TABLE 02581_trips UPDATE description='a' WHERE id IN (SELECT (number*10)::UInt32 FROM numbers(200000000)) SETTINGS mutations_sync=2;
+ALTER TABLE 02581_trips UPDATE description='a' WHERE id IN (SELECT (number*10)::UInt32 FROM numbers(10000000)) SETTINGS mutations_sync=2;
 SELECT count() from 02581_trips WHERE description = '';
 
-ALTER TABLE 02581_trips UPDATE description='a' WHERE id IN (SELECT (number*10 + 1)::UInt32 FROM numbers(200000000)) SETTINGS mutations_sync=2, max_rows_in_set=1000;
+ALTER TABLE 02581_trips UPDATE description='a' WHERE id IN (SELECT (number*10 + 1)::UInt32 FROM numbers(10000000)) SETTINGS mutations_sync=2, max_rows_in_set=1000;
 SELECT count() from 02581_trips WHERE description = '';
 
 -- Run mutation with func(`id`) IN big subquery
-ALTER TABLE 02581_trips UPDATE description='b' WHERE id::UInt64 IN (SELECT (number*10 + 2)::UInt32 FROM numbers(200000000)) SETTINGS mutations_sync=2;
+ALTER TABLE 02581_trips UPDATE description='b' WHERE id::UInt64 IN (SELECT (number*10 + 2)::UInt32 FROM numbers(10000000)) SETTINGS mutations_sync=2;
 SELECT count() from 02581_trips WHERE description = '';
 
 -- Run mutation with non-PK `id2` IN big subquery
-ALTER TABLE 02581_trips UPDATE description='c' WHERE id2 IN (SELECT (number*10 + 3)::UInt32 FROM numbers(200000000)) SETTINGS mutations_sync=2;
+ALTER TABLE 02581_trips UPDATE description='c' WHERE id2 IN (SELECT (number*10 + 3)::UInt32 FROM numbers(10000000)) SETTINGS mutations_sync=2;
 SELECT count() from 02581_trips WHERE description = '';
 
 -- Run mutation with PK and non-PK IN big subquery
 ALTER TABLE 02581_trips UPDATE description='c'
 WHERE
-    (id IN (SELECT (number*10 + 4)::UInt32 FROM numbers(200000000))) OR
-    (id2 IN (SELECT (number*10 + 4)::UInt32 FROM numbers(200000000)))
+    (id IN (SELECT (number*10 + 4)::UInt32 FROM numbers(10000000))) OR
+    (id2 IN (SELECT (number*10 + 4)::UInt32 FROM numbers(10000000)))
 SETTINGS mutations_sync=2;
 SELECT count() from 02581_trips WHERE description = '';
 
 -- Run mutation with PK and non-PK IN big subquery
 ALTER TABLE 02581_trips UPDATE description='c'
 WHERE
-    (id::UInt64 IN (SELECT (number*10 + 5)::UInt32 FROM numbers(200000000))) OR
-    (id2::UInt64 IN (SELECT (number*10 + 5)::UInt32 FROM numbers(200000000)))
+    (id::UInt64 IN (SELECT (number*10 + 5)::UInt32 FROM numbers(10000000))) OR
+    (id2::UInt64 IN (SELECT (number*10 + 5)::UInt32 FROM numbers(10000000)))
 SETTINGS mutations_sync=2;
 SELECT count() from 02581_trips WHERE description = '';
 
 -- Run mutation with PK and non-PK IN big subquery
 ALTER TABLE 02581_trips UPDATE description='c'
 WHERE
-    (id::UInt32 IN (SELECT (number*10 + 6)::UInt32 FROM numbers(200000000))) OR
-    ((id2+1)::String IN (SELECT (number*10 + 6)::UInt32 FROM numbers(200000000)))
+    (id::UInt32 IN (SELECT (number*10 + 6)::UInt32 FROM numbers(10000000))) OR
+    ((id2+1)::String IN (SELECT (number*10 + 6)::UInt32 FROM numbers(10000000)))
 SETTINGS mutations_sync=2;
 SELECT count() from 02581_trips WHERE description = '';
 
+SYSTEM FLUSH LOGS;
+SELECT DISTINCT peak_memory_usage < 2000000000 ? 'Ok' : toString(tuple(*)) FROM system.part_log WHERE database = currentDatabase() AND event_date >= yesterday() AND table = '02581_trips' AND event_type = 'MutatePart';
+
 DROP TABLE 02581_trips;

From 4cbb8bab56c288a8a2d5bd5d0bc5ec102516c621 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Tue, 23 Jul 2024 04:09:06 +0200
Subject: [PATCH 0617/1488] Speed up mutations for non-replicated MergeTree a
 bit

---
 src/Storages/StorageMergeTree.cpp | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp
index 444b3fbae4c..45573925c02 100644
--- a/src/Storages/StorageMergeTree.cpp
+++ b/src/Storages/StorageMergeTree.cpp
@@ -506,13 +506,14 @@ Int64 StorageMergeTree::startMutation(const MutationCommands & commands, Context
     }
 
     Int64 version;
+    String mutation_id;
     {
         std::lock_guard lock(currently_processing_in_background_mutex);
 
         MergeTreeMutationEntry entry(commands, disk, relative_data_path, insert_increment.get(), current_tid, getContext()->getWriteSettings());
         version = increment.get();
         entry.commit(version);
-        String mutation_id = entry.file_name;
+        mutation_id = entry.file_name;
         if (txn)
             txn->addMutation(shared_from_this(), mutation_id);
 
@@ -527,9 +528,9 @@ Int64 StorageMergeTree::startMutation(const MutationCommands & commands, Context
             }
             throw Exception(ErrorCodes::LOGICAL_ERROR, "Mutation {} already exists, it's a bug", version);
         }
-
-        LOG_INFO(log, "Added mutation: {}{}", mutation_id, additional_info);
     }
+
+    LOG_INFO(log, "Added mutation: {}{}", mutation_id, additional_info);
     background_operations_assignee.trigger();
     return version;
 }

From 8899d4242895893c0485efb2de237bca008d1a8d Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Tue, 23 Jul 2024 04:29:46 +0200
Subject: [PATCH 0618/1488] Speed up mutations for non-replicated MergeTree
 significantly

---
 src/Storages/StorageMergeTree.cpp | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp
index 45573925c02..527872d701e 100644
--- a/src/Storages/StorageMergeTree.cpp
+++ b/src/Storages/StorageMergeTree.cpp
@@ -505,19 +505,18 @@ Int64 StorageMergeTree::startMutation(const MutationCommands & commands, Context
         additional_info = fmt::format(" (TID: {}; TIDH: {})", current_tid, current_tid.getHash());
     }
 
-    Int64 version;
-    String mutation_id;
+    MergeTreeMutationEntry entry(commands, disk, relative_data_path, insert_increment.get(), current_tid, getContext()->getWriteSettings());
+    Int64 version = increment.get();
+    entry.commit(version);
+    String mutation_id = entry.file_name;
+    if (txn)
+        txn->addMutation(shared_from_this(), mutation_id);
+
+    bool alter_conversions_mutations_updated = updateAlterConversionsMutations(entry.commands, alter_conversions_mutations, /* remove= */ false);
+
     {
         std::lock_guard lock(currently_processing_in_background_mutex);
 
-        MergeTreeMutationEntry entry(commands, disk, relative_data_path, insert_increment.get(), current_tid, getContext()->getWriteSettings());
-        version = increment.get();
-        entry.commit(version);
-        mutation_id = entry.file_name;
-        if (txn)
-            txn->addMutation(shared_from_this(), mutation_id);
-
-        bool alter_conversions_mutations_updated = updateAlterConversionsMutations(entry.commands, alter_conversions_mutations, /* remove= */ false);
         bool inserted = current_mutations_by_version.try_emplace(version, std::move(entry)).second;
         if (!inserted)
         {

From dee5790b22d06e3916f4030937eb9384247a6baa Mon Sep 17 00:00:00 2001
From: zoomxi <419486879@qq.com>
Date: Tue, 23 Jul 2024 10:49:37 +0800
Subject: [PATCH 0619/1488] =?UTF-8?q?According=20to=20the=20suggestions=20?=
 =?UTF-8?q?from=20cr=EF=BC=8C=20modify=20test=20code?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../test.py                                   | 26 +++----------------
 1 file changed, 4 insertions(+), 22 deletions(-)

diff --git a/tests/integration/test_parallel_replicas_no_replicas/test.py b/tests/integration/test_parallel_replicas_no_replicas/test.py
index b77da338554..e05f72316d0 100644
--- a/tests/integration/test_parallel_replicas_no_replicas/test.py
+++ b/tests/integration/test_parallel_replicas_no_replicas/test.py
@@ -49,8 +49,8 @@ def test_skip_all_replicas(start_cluster, skip_unavailable_shards):
             },
         )
 
-@pytest.mark.parametrize("skip_unavailable_shards", [1, 0])
-def test_skip_all_participating_replicas1(start_cluster, skip_unavailable_shards):
+@pytest.mark.parametrize("max_parallel_replicas", [2, 3, 100])
+def test_skip_all_participating_replicas(start_cluster, max_parallel_replicas):
     cluster_name = "test_1_shard_3_unavaliable_replicas"
     table_name = "tt1"
     create_tables(cluster_name, table_name)
@@ -60,26 +60,8 @@ def test_skip_all_participating_replicas1(start_cluster, skip_unavailable_shards
             f"SELECT key, count() FROM {table_name}  GROUP BY key ORDER BY key",
             settings={
                 "allow_experimental_parallel_reading_from_replicas": 2,
-                "max_parallel_replicas": 3,
+                "max_parallel_replicas": max_parallel_replicas,
                 "cluster_for_parallel_replicas": cluster_name,
-                "skip_unavailable_shards": skip_unavailable_shards,
-                "parallel_replicas_min_number_of_rows_per_replica": 500,
-            },
-        )
-
-@pytest.mark.parametrize("skip_unavailable_shards", [1, 0])
-def test_skip_all_participating_replicas2(start_cluster, skip_unavailable_shards):
-    cluster_name = "test_1_shard_3_unavaliable_replicas"
-    table_name = "tt2"
-    create_tables(cluster_name, table_name)
-
-    with pytest.raises(QueryRuntimeException):
-        initiator.query(
-            f"SELECT key, count() FROM {table_name}  GROUP BY key ORDER BY key",
-            settings={
-                "allow_experimental_parallel_reading_from_replicas": 2,
-                "max_parallel_replicas": 2,
-                "cluster_for_parallel_replicas": cluster_name,
-                "skip_unavailable_shards": skip_unavailable_shards,
+                "skip_unavailable_shards": 1,
             },
         )

From feaa7ede979a624dfbe67facefe66395cfc021b4 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Tue, 23 Jul 2024 06:34:29 +0200
Subject: [PATCH 0620/1488] Fix views over distributed tables with Analyzer

---
 src/Analyzer/IQueryTreeNode.h       |  2 +-
 src/Storages/StorageDistributed.cpp | 37 ++++++++++-------------------
 2 files changed, 14 insertions(+), 25 deletions(-)

diff --git a/src/Analyzer/IQueryTreeNode.h b/src/Analyzer/IQueryTreeNode.h
index df3687f8fd9..b36c1401798 100644
--- a/src/Analyzer/IQueryTreeNode.h
+++ b/src/Analyzer/IQueryTreeNode.h
@@ -49,7 +49,7 @@ enum class QueryTreeNodeType : uint8_t
 /// Convert query tree node type to string
 const char * toString(QueryTreeNodeType type);
 
-/** Query tree is semantical representation of query.
+/** Query tree is a semantic representation of query.
   * Query tree node represent node in query tree.
   * IQueryTreeNode is base class for all query tree nodes.
   *
diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp
index 65323b4bb52..07892971ec2 100644
--- a/src/Storages/StorageDistributed.cpp
+++ b/src/Storages/StorageDistributed.cpp
@@ -43,7 +43,6 @@
 #include <Parsers/parseQuery.h>
 #include <Parsers/IAST.h>
 
-#include <Analyzer/Utils.h>
 #include <Analyzer/ColumnNode.h>
 #include <Analyzer/FunctionNode.h>
 #include <Analyzer/TableNode.h>
@@ -61,26 +60,20 @@
 #include <Interpreters/ClusterProxy/SelectStreamFactory.h>
 #include <Interpreters/ClusterProxy/executeQuery.h>
 #include <Interpreters/Cluster.h>
-#include <Interpreters/DatabaseAndTableWithAlias.h>
 #include <Interpreters/ExpressionAnalyzer.h>
 #include <Interpreters/InterpreterSelectQuery.h>
 #include <Interpreters/InterpreterSelectQueryAnalyzer.h>
 #include <Interpreters/InterpreterInsertQuery.h>
 #include <Interpreters/JoinedTables.h>
-#include <Interpreters/TranslateQualifiedNamesVisitor.h>
 #include <Interpreters/AddDefaultDatabaseVisitor.h>
 #include <Interpreters/TreeRewriter.h>
 #include <Interpreters/Context.h>
 #include <Interpreters/createBlockSelector.h>
 #include <Interpreters/evaluateConstantExpression.h>
 #include <Interpreters/getClusterName.h>
-#include <Interpreters/getTableExpressions.h>
 #include <Interpreters/RequiredSourceColumnsVisitor.h>
-#include <Interpreters/getCustomKeyFilterForParallelReplicas.h>
 #include <Interpreters/getHeaderForProcessingStage.h>
 
-#include <Functions/IFunction.h>
-#include <Functions/FunctionFactory.h>
 #include <TableFunctions/TableFunctionView.h>
 #include <TableFunctions/TableFunctionFactory.h>
 
@@ -90,7 +83,6 @@
 #include <Processors/Executors/PushingPipelineExecutor.h>
 #include <Processors/Executors/CompletedPipelineExecutor.h>
 #include <Processors/QueryPlan/QueryPlan.h>
-#include <Processors/QueryPlan/BuildQueryPipelineSettings.h>
 #include <Processors/QueryPlan/ReadFromPreparedSource.h>
 #include <Processors/QueryPlan/ExpressionStep.h>
 #include <Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h>
@@ -496,7 +488,7 @@ QueryProcessingStage::Enum StorageDistributed::getQueryProcessingStage(
     }
 
     std::optional<QueryProcessingStage::Enum> optimized_stage;
-    if (settings.allow_experimental_analyzer)
+    if (query_info.query_tree)
         optimized_stage = getOptimizedQueryProcessingStageAnalyzer(query_info, settings);
     else
         optimized_stage = getOptimizedQueryProcessingStage(query_info, settings);
@@ -860,31 +852,28 @@ void StorageDistributed::read(
         modified_query_info.query = queryNodeToDistributedSelectQuery(query_tree_distributed);
 
         modified_query_info.query_tree = std::move(query_tree_distributed);
+
+        /// Return directly (with correct header) if no shard to query.
+        if (modified_query_info.getCluster()->getShardsInfo().empty())
+            return;
     }
     else
     {
         header = InterpreterSelectQuery(modified_query_info.query, local_context, SelectQueryOptions(processed_stage).analyze()).getSampleBlock();
-    }
 
-    if (!settings.allow_experimental_analyzer)
-    {
         modified_query_info.query = ClusterProxy::rewriteSelectQuery(
             local_context, modified_query_info.query,
             remote_database, remote_table, remote_table_function_ptr);
-    }
 
-    /// Return directly (with correct header) if no shard to query.
-    if (modified_query_info.getCluster()->getShardsInfo().empty())
-    {
-        if (settings.allow_experimental_analyzer)
+        if (modified_query_info.getCluster()->getShardsInfo().empty())
+        {
+            Pipe pipe(std::make_shared<NullSource>(header));
+            auto read_from_pipe = std::make_unique<ReadFromPreparedSource>(std::move(pipe));
+            read_from_pipe->setStepDescription("Read from NullSource (Distributed)");
+            query_plan.addStep(std::move(read_from_pipe));
+
             return;
-
-        Pipe pipe(std::make_shared<NullSource>(header));
-        auto read_from_pipe = std::make_unique<ReadFromPreparedSource>(std::move(pipe));
-        read_from_pipe->setStepDescription("Read from NullSource (Distributed)");
-        query_plan.addStep(std::move(read_from_pipe));
-
-        return;
+        }
     }
 
     const auto & snapshot_data = assert_cast<const SnapshotData &>(*storage_snapshot->data);

From ac13983ebb8f2a75ed44f1e2616e1aac006ce69d Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Tue, 23 Jul 2024 06:52:19 +0200
Subject: [PATCH 0621/1488] Better

---
 .../IInterpreterUnionOrSelectQuery.cpp        | 18 +++++++++++------
 src/Interpreters/InterpreterSelectQuery.cpp   | 20 ++++++++-----------
 src/Interpreters/InterpreterSelectQuery.h     |  1 -
 src/Storages/TTLDescription.cpp               |  2 +-
 4 files changed, 21 insertions(+), 20 deletions(-)

diff --git a/src/Interpreters/IInterpreterUnionOrSelectQuery.cpp b/src/Interpreters/IInterpreterUnionOrSelectQuery.cpp
index d8f6df05ca4..bc7b07ca18d 100644
--- a/src/Interpreters/IInterpreterUnionOrSelectQuery.cpp
+++ b/src/Interpreters/IInterpreterUnionOrSelectQuery.cpp
@@ -17,13 +17,19 @@
 namespace DB
 {
 
-IInterpreterUnionOrSelectQuery::IInterpreterUnionOrSelectQuery(const DB::ASTPtr& query_ptr_,
-        const DB::ContextMutablePtr& context_, const DB::SelectQueryOptions& options_)
-        : query_ptr(query_ptr_)
-        , context(context_)
-        , options(options_)
-        , max_streams(context->getSettingsRef().max_threads)
+IInterpreterUnionOrSelectQuery::IInterpreterUnionOrSelectQuery(const ASTPtr & query_ptr_,
+    const ContextMutablePtr & context_, const SelectQueryOptions & options_)
+    : query_ptr(query_ptr_)
+    , context(context_)
+    , options(options_)
+    , max_streams(context->getSettingsRef().max_threads)
 {
+    /// FIXME All code here will work with the old analyzer, however for views over  Distributed tables
+    /// it's possible that new analyzer will be enabled in ::getQueryProcessingStage method
+    /// of the underlying storage when all other parts of infrastructure are not ready for it
+    /// (built with old analyzer).
+    context->setSetting("allow_experimental_analyzer", false);
+
     if (options.shard_num)
         context->addSpecialScalar(
                 "_shard_num",
diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp
index 7bee497f6da..0f24888cb79 100644
--- a/src/Interpreters/InterpreterSelectQuery.cpp
+++ b/src/Interpreters/InterpreterSelectQuery.cpp
@@ -75,7 +75,6 @@
 
 #include <Storages/MergeTree/MergeTreeWhereOptimizer.h>
 #include <Storages/StorageDistributed.h>
-#include <Storages/StorageDummy.h>
 #include <Storages/StorageMerge.h>
 #include <Storages/StorageValues.h>
 #include <Storages/StorageView.h>
@@ -214,11 +213,11 @@ InterpreterSelectQuery::InterpreterSelectQuery(
 {}
 
 InterpreterSelectQuery::InterpreterSelectQuery(
-        const ASTPtr & query_ptr_,
-        const ContextPtr & context_,
-        Pipe input_pipe_,
-        const SelectQueryOptions & options_)
-        : InterpreterSelectQuery(query_ptr_, context_, std::move(input_pipe_), nullptr, options_.copy().noSubquery())
+    const ASTPtr & query_ptr_,
+    const ContextPtr & context_,
+    Pipe input_pipe_,
+    const SelectQueryOptions & options_)
+    : InterpreterSelectQuery(query_ptr_, context_, std::move(input_pipe_), nullptr, options_.copy().noSubquery())
 {}
 
 InterpreterSelectQuery::InterpreterSelectQuery(
@@ -227,18 +226,15 @@ InterpreterSelectQuery::InterpreterSelectQuery(
     const StoragePtr & storage_,
     const StorageMetadataPtr & metadata_snapshot_,
     const SelectQueryOptions & options_)
-    : InterpreterSelectQuery(
-        query_ptr_, context_, std::nullopt, storage_, options_.copy().noSubquery(), {}, metadata_snapshot_)
-{
-}
+    : InterpreterSelectQuery(query_ptr_, context_, std::nullopt, storage_, options_.copy().noSubquery(), {}, metadata_snapshot_)
+{}
 
 InterpreterSelectQuery::InterpreterSelectQuery(
     const ASTPtr & query_ptr_,
     const ContextPtr & context_,
     const SelectQueryOptions & options_,
     PreparedSetsPtr prepared_sets_)
-    : InterpreterSelectQuery(
-        query_ptr_, context_, std::nullopt, nullptr, options_, {}, {}, prepared_sets_)
+    : InterpreterSelectQuery(query_ptr_, context_, std::nullopt, nullptr, options_, {}, {}, prepared_sets_)
 {}
 
 InterpreterSelectQuery::~InterpreterSelectQuery() = default;
diff --git a/src/Interpreters/InterpreterSelectQuery.h b/src/Interpreters/InterpreterSelectQuery.h
index d4ed19d45ea..ac1230a6eba 100644
--- a/src/Interpreters/InterpreterSelectQuery.h
+++ b/src/Interpreters/InterpreterSelectQuery.h
@@ -26,7 +26,6 @@ class Logger;
 namespace DB
 {
 
-class SubqueryForSet;
 class InterpreterSelectWithUnionQuery;
 class Context;
 class QueryPlan;
diff --git a/src/Storages/TTLDescription.cpp b/src/Storages/TTLDescription.cpp
index ac091e7cf3c..92f6f17583d 100644
--- a/src/Storages/TTLDescription.cpp
+++ b/src/Storages/TTLDescription.cpp
@@ -172,7 +172,7 @@ static ExpressionAndSets buildExpressionAndSets(ASTPtr & ast, const NamesAndType
     /// with subqueries it's possible that new analyzer will be enabled in ::read method
     /// of underlying storage when all other parts of infra are not ready for it
     /// (built with old analyzer).
-    context_copy->setSetting("allow_experimental_analyzer", Field{0});
+    context_copy->setSetting("allow_experimental_analyzer", false);
     auto syntax_analyzer_result = TreeRewriter(context_copy).analyze(ast, columns);
     ExpressionAnalyzer analyzer(ast, syntax_analyzer_result, context_copy);
     auto dag = analyzer.getActionsDAG(false);

From d50742eadc1234bf75fa76e7b21aec0f8763851e Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Tue, 23 Jul 2024 06:52:34 +0200
Subject: [PATCH 0622/1488] Better

---
 src/Interpreters/IInterpreterUnionOrSelectQuery.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Interpreters/IInterpreterUnionOrSelectQuery.cpp b/src/Interpreters/IInterpreterUnionOrSelectQuery.cpp
index bc7b07ca18d..f64b9540dbb 100644
--- a/src/Interpreters/IInterpreterUnionOrSelectQuery.cpp
+++ b/src/Interpreters/IInterpreterUnionOrSelectQuery.cpp
@@ -24,7 +24,7 @@ IInterpreterUnionOrSelectQuery::IInterpreterUnionOrSelectQuery(const ASTPtr & qu
     , options(options_)
     , max_streams(context->getSettingsRef().max_threads)
 {
-    /// FIXME All code here will work with the old analyzer, however for views over  Distributed tables
+    /// FIXME All code here will work with the old analyzer, however for views over Distributed tables
     /// it's possible that new analyzer will be enabled in ::getQueryProcessingStage method
     /// of the underlying storage when all other parts of infrastructure are not ready for it
     /// (built with old analyzer).

From 133e734d73b87707ccba17bd488cf60e1895abe5 Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Mon, 22 Jul 2024 21:53:22 +0200
Subject: [PATCH 0623/1488] Move to integration test

---
 tests/integration/test_keeper_map/test.py     | 21 +++++++++++++++++++
 ...8_keepermap_incomplete_data_drop.reference |  0
 .../03208_keepermap_incomplete_data_drop.sql  |  8 -------
 3 files changed, 21 insertions(+), 8 deletions(-)
 delete mode 100644 tests/queries/0_stateless/03208_keepermap_incomplete_data_drop.reference
 delete mode 100644 tests/queries/0_stateless/03208_keepermap_incomplete_data_drop.sql

diff --git a/tests/integration/test_keeper_map/test.py b/tests/integration/test_keeper_map/test.py
index d7b4230d872..31316af7b1e 100644
--- a/tests/integration/test_keeper_map/test.py
+++ b/tests/integration/test_keeper_map/test.py
@@ -104,3 +104,24 @@ def test_keeper_map_without_zk(started_cluster):
     node.query("DETACH TABLE test_keeper_map_without_zk")
 
     client.stop()
+
+
+def test_keeper_map_with_failed_drop(started_cluster):
+    run_query(
+        "CREATE TABLE test_keeper_map_with_failed_drop (key UInt64, value UInt64) ENGINE = KeeperMap('/test_keeper_map_with_failed_drop') PRIMARY KEY(key);"
+    )
+
+    run_query("INSERT INTO test_keeper_map_with_failed_drop VALUES (1, 11)")
+    run_query("SYSTEM ENABLE FAILPOINT keepermap_fail_drop_data")
+    node.query("DROP TABLE test_keeper_map_with_failed_drop SYNC")
+
+    zk_client = get_genuine_zk()
+    assert (
+        zk_client.get("/test_keeper_map/test_keeper_map_with_failed_drop/data")
+        is not None
+    )
+
+    run_query("SYSTEM DISABLE FAILPOINT keepermap_fail_drop_data")
+    run_query(
+        "CREATE TABLE test_keeper_map_with_failed_drop_another (key UInt64, value UInt64) ENGINE = KeeperMap('/test_keeper_map_with_failed_drop') PRIMARY KEY(key);"
+    )
diff --git a/tests/queries/0_stateless/03208_keepermap_incomplete_data_drop.reference b/tests/queries/0_stateless/03208_keepermap_incomplete_data_drop.reference
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/tests/queries/0_stateless/03208_keepermap_incomplete_data_drop.sql b/tests/queries/0_stateless/03208_keepermap_incomplete_data_drop.sql
deleted file mode 100644
index e8d4a5bc298..00000000000
--- a/tests/queries/0_stateless/03208_keepermap_incomplete_data_drop.sql
+++ /dev/null
@@ -1,8 +0,0 @@
-DROP TABLE IF EXISTS 03208_keepermap_test SYNC;
-
-CREATE TABLE 03208_keepermap_test (key UInt64, value UInt64) Engine=KeeperMap('/' || currentDatabase() || '/test03208') PRIMARY KEY(key);
-INSERT INTO 03208_keepermap_test VALUES (1, 11);
-SYSTEM ENABLE FAILPOINT keepermap_fail_drop_data;
-DROP TABLE 03208_keepermap_test SYNC; -- { KEEPER_EXCEPTION }
-SYSTEM DISABLE FAILPOINT keepermap_fail_drop_data;
-CREATE TABLE 03208_keepermap_test_another (key UInt64, value UInt64) Engine=KeeperMap('/' || currentDatabase() || '/test03208') PRIMARY KEY(key);

From 9d7bb2a430b142e0761cc30efcf78ea1bbeb6871 Mon Sep 17 00:00:00 2001
From: JackyWoo <wuchienchao@qq.com>
Date: Tue, 23 Jul 2024 15:07:18 +0800
Subject: [PATCH 0624/1488] Move unit tests to gtest_convertFieldToType.cpp

---
 .../tests/gtest_convertFieldToType.cpp        | 83 ++++++++++++++++++-
 src/Storages/Statistics/Statistics.cpp        | 12 +--
 .../Statistics/StatisticsCountMinSketch.cpp   | 17 ++--
 .../Statistics/StatisticsCountMinSketch.h     |  4 +-
 src/Storages/Statistics/StatisticsTDigest.cpp |  6 +-
 src/Storages/Statistics/StatisticsTDigest.h   |  4 +-
 src/Storages/Statistics/StatisticsUniq.cpp    |  4 +-
 src/Storages/Statistics/StatisticsUniq.h      |  4 +-
 src/Storages/Statistics/tests/gtest_stats.cpp | 47 -----------
 9 files changed, 109 insertions(+), 72 deletions(-)

diff --git a/src/Interpreters/tests/gtest_convertFieldToType.cpp b/src/Interpreters/tests/gtest_convertFieldToType.cpp
index c8a9d5aa2c0..0cac9a3b59d 100644
--- a/src/Interpreters/tests/gtest_convertFieldToType.cpp
+++ b/src/Interpreters/tests/gtest_convertFieldToType.cpp
@@ -147,7 +147,7 @@ INSTANTIATE_TEST_SUITE_P(
             DecimalField(DateTime64(123 * Day * 1'000'000), 6)
         }
     })
-    );
+);
 
 INSTANTIATE_TEST_SUITE_P(
     DateTimeToDateTime64,
@@ -179,3 +179,84 @@ INSTANTIATE_TEST_SUITE_P(
         },
     })
 );
+
+INSTANTIATE_TEST_SUITE_P(
+    StringToNumber,
+    ConvertFieldToTypeTest,
+    ::testing::ValuesIn(std::initializer_list<ConvertFieldToTypeTestParams>{
+        {
+            "String",
+            Field("1"),
+            "Int8",
+            Field(1)
+        },
+        {
+            "String",
+            Field("256"),
+            "Int8",
+            Field()
+        },
+        {
+            "String",
+            Field("not a number"),
+            "Int8",
+            {}
+        },
+        {
+            "String",
+            Field("1.1"),
+            "Int8",
+            {} /// we can not convert '1.1' to Int8
+        },
+        {
+            "String",
+            Field("1.1"),
+            "Float64",
+            Field(1.1)
+        },
+    })
+);
+
+INSTANTIATE_TEST_SUITE_P(
+    NumberToString,
+    ConvertFieldToTypeTest,
+    ::testing::ValuesIn(std::initializer_list<ConvertFieldToTypeTestParams>{
+        {
+            "Int8",
+            Field(1),
+            "String",
+            Field("1")
+        },
+        {
+            "Int8",
+            Field(-1),
+            "String",
+            Field("-1")
+        },
+        {
+            "Float64",
+            Field(1.1),
+            "String",
+            Field("1.1")
+        },
+    })
+);
+
+INSTANTIATE_TEST_SUITE_P(
+    StringToDate,
+    ConvertFieldToTypeTest,
+    ::testing::ValuesIn(std::initializer_list<ConvertFieldToTypeTestParams>{
+        {
+            "String",
+            Field("2024-07-12"),
+            "Date",
+            Field(static_cast<UInt16>(19916))
+        },
+        {
+            "String",
+            Field("not a date"),
+            "Date",
+            {}
+        },
+    })
+);
diff --git a/src/Storages/Statistics/Statistics.cpp b/src/Storages/Statistics/Statistics.cpp
index 588e20e801f..ade3326288a 100644
--- a/src/Storages/Statistics/Statistics.cpp
+++ b/src/Storages/Statistics/Statistics.cpp
@@ -204,15 +204,15 @@ void MergeTreeStatisticsFactory::registerValidator(StatisticsType stats_type, Va
 
 MergeTreeStatisticsFactory::MergeTreeStatisticsFactory()
 {
-    registerValidator(StatisticsType::TDigest, TDigestValidator);
-    registerCreator(StatisticsType::TDigest, TDigestCreator);
+    registerValidator(StatisticsType::TDigest, tdigestValidator);
+    registerCreator(StatisticsType::TDigest, tdigestCreator);
 
-    registerValidator(StatisticsType::Uniq, UniqValidator);
-    registerCreator(StatisticsType::Uniq, UniqCreator);
+    registerValidator(StatisticsType::Uniq, uniqValidator);
+    registerCreator(StatisticsType::Uniq, uniqCreator);
 
 #if USE_DATASKETCHES
-    registerValidator(StatisticsType::CountMinSketch, CountMinSketchValidator);
-    registerCreator(StatisticsType::CountMinSketch, CountMinSketchCreator);
+    registerValidator(StatisticsType::CountMinSketch, countMinSketchValidator);
+    registerCreator(StatisticsType::CountMinSketch, countMinSketchCreator);
 #endif
 }
 
diff --git a/src/Storages/Statistics/StatisticsCountMinSketch.cpp b/src/Storages/Statistics/StatisticsCountMinSketch.cpp
index 95a8ceda8c8..e69bbc1515b 100644
--- a/src/Storages/Statistics/StatisticsCountMinSketch.cpp
+++ b/src/Storages/Statistics/StatisticsCountMinSketch.cpp
@@ -17,13 +17,18 @@ extern const int LOGICAL_ERROR;
 extern const int ILLEGAL_STATISTICS;
 }
 
-/// Constants chosen based on rolling dices, which provides an error tolerance of 0.1% (ε = 0.001) and a confidence level of 99.9% (δ = 0.001).
+/// Constants chosen based on rolling dices.
+/// The values provides:
+///     1. an error tolerance of 0.1% (ε = 0.001)
+///     2. a confidence level of 99.9% (δ = 0.001).
 /// And sketch the size is 152kb.
 static constexpr auto num_hashes = 7uz;
 static constexpr auto num_buckets = 2718uz;
 
 StatisticsCountMinSketch::StatisticsCountMinSketch(const SingleStatisticsDescription & stat_, DataTypePtr data_type_)
-    : IStatistics(stat_), sketch(num_hashes, num_buckets), data_type(data_type_)
+    : IStatistics(stat_)
+    , sketch(num_hashes, num_buckets)
+    , data_type(data_type_)
 {
 }
 
@@ -50,8 +55,7 @@ Float64 StatisticsCountMinSketch::estimateEqual(const Field & val) const
 
 void StatisticsCountMinSketch::update(const ColumnPtr & column)
 {
-    size_t size = column->size();
-    for (size_t row = 0; row < size; ++row)
+    for (size_t row = 0; row < column->size(); ++row)
     {
         if (column->isNullAt(row))
             continue;
@@ -80,16 +84,15 @@ void StatisticsCountMinSketch::deserialize(ReadBuffer & buf)
 }
 
 
-void CountMinSketchValidator(const SingleStatisticsDescription &, DataTypePtr data_type)
+void countMinSketchValidator(const SingleStatisticsDescription &, DataTypePtr data_type)
 {
     data_type = removeNullable(data_type);
     data_type = removeLowCardinalityAndNullable(data_type);
-    /// Data types of Numeric, String family, IPv4, IPv6, Date family, Enum family are supported.
     if (!data_type->isValueRepresentedByNumber() && !isStringOrFixedString(data_type))
         throw Exception(ErrorCodes::ILLEGAL_STATISTICS, "Statistics of type 'count_min' does not support type {}", data_type->getName());
 }
 
-StatisticsPtr CountMinSketchCreator(const SingleStatisticsDescription & stat, DataTypePtr data_type)
+StatisticsPtr countMinSketchCreator(const SingleStatisticsDescription & stat, DataTypePtr data_type)
 {
     return std::make_shared<StatisticsCountMinSketch>(stat, data_type);
 }
diff --git a/src/Storages/Statistics/StatisticsCountMinSketch.h b/src/Storages/Statistics/StatisticsCountMinSketch.h
index aa71c643c05..6c8b74f8c35 100644
--- a/src/Storages/Statistics/StatisticsCountMinSketch.h
+++ b/src/Storages/Statistics/StatisticsCountMinSketch.h
@@ -31,8 +31,8 @@ private:
 };
 
 
-void CountMinSketchValidator(const SingleStatisticsDescription &, DataTypePtr data_type);
-StatisticsPtr CountMinSketchCreator(const SingleStatisticsDescription & stat, DataTypePtr);
+void countMinSketchValidator(const SingleStatisticsDescription &, DataTypePtr data_type);
+StatisticsPtr countMinSketchCreator(const SingleStatisticsDescription & stat, DataTypePtr);
 
 }
 
diff --git a/src/Storages/Statistics/StatisticsTDigest.cpp b/src/Storages/Statistics/StatisticsTDigest.cpp
index e3a59f3251a..66150e00fdb 100644
--- a/src/Storages/Statistics/StatisticsTDigest.cpp
+++ b/src/Storages/Statistics/StatisticsTDigest.cpp
@@ -46,7 +46,7 @@ Float64 StatisticsTDigest::estimateLess(const Field & val) const
     auto val_as_float = StatisticsUtils::tryConvertToFloat64(val);
     if (val_as_float)
         return t_digest.getCountLessThan(*val_as_float);
-    throw Exception(ErrorCodes::LOGICAL_ERROR, "Statistics 'tdigest' does not support estimate value of type {}", val.getTypeName());
+    throw Exception(ErrorCodes::LOGICAL_ERROR, "Statistics 'tdigest' does not support estimating value of type {}", val.getTypeName());
 }
 
 Float64 StatisticsTDigest::estimateEqual(const Field & val) const
@@ -57,7 +57,7 @@ Float64 StatisticsTDigest::estimateEqual(const Field & val) const
     throw Exception(ErrorCodes::LOGICAL_ERROR, "Statistics 'tdigest' does not support estimating value of type {}", val.getTypeName());
 }
 
-void TDigestValidator(const SingleStatisticsDescription &, DataTypePtr data_type)
+void tdigestValidator(const SingleStatisticsDescription &, DataTypePtr data_type)
 {
     data_type = removeNullable(data_type);
     data_type = removeLowCardinalityAndNullable(data_type);
@@ -65,7 +65,7 @@ void TDigestValidator(const SingleStatisticsDescription &, DataTypePtr data_type
         throw Exception(ErrorCodes::ILLEGAL_STATISTICS, "Statistics of type 'tdigest' do not support type {}", data_type->getName());
 }
 
-StatisticsPtr TDigestCreator(const SingleStatisticsDescription & stat, DataTypePtr)
+StatisticsPtr tdigestCreator(const SingleStatisticsDescription & stat, DataTypePtr)
 {
     return std::make_shared<StatisticsTDigest>(stat);
 }
diff --git a/src/Storages/Statistics/StatisticsTDigest.h b/src/Storages/Statistics/StatisticsTDigest.h
index 801d0787eaf..614973e5d8b 100644
--- a/src/Storages/Statistics/StatisticsTDigest.h
+++ b/src/Storages/Statistics/StatisticsTDigest.h
@@ -23,7 +23,7 @@ private:
     QuantileTDigest<Float64> t_digest;
 };
 
-void TDigestValidator(const SingleStatisticsDescription &, DataTypePtr data_type);
-StatisticsPtr TDigestCreator(const SingleStatisticsDescription & stat, DataTypePtr);
+void tdigestValidator(const SingleStatisticsDescription &, DataTypePtr data_type);
+StatisticsPtr tdigestCreator(const SingleStatisticsDescription & stat, DataTypePtr);
 
 }
diff --git a/src/Storages/Statistics/StatisticsUniq.cpp b/src/Storages/Statistics/StatisticsUniq.cpp
index 9eea1dec39b..8f60ffcf0b5 100644
--- a/src/Storages/Statistics/StatisticsUniq.cpp
+++ b/src/Storages/Statistics/StatisticsUniq.cpp
@@ -52,7 +52,7 @@ UInt64 StatisticsUniq::estimateCardinality() const
     return column->getUInt(0);
 }
 
-void UniqValidator(const SingleStatisticsDescription &, DataTypePtr data_type)
+void uniqValidator(const SingleStatisticsDescription &, DataTypePtr data_type)
 {
     data_type = removeNullable(data_type);
     data_type = removeLowCardinalityAndNullable(data_type);
@@ -60,7 +60,7 @@ void UniqValidator(const SingleStatisticsDescription &, DataTypePtr data_type)
         throw Exception(ErrorCodes::ILLEGAL_STATISTICS, "Statistics of type 'uniq' do not support type {}", data_type->getName());
 }
 
-StatisticsPtr UniqCreator(const SingleStatisticsDescription & stat, DataTypePtr data_type)
+StatisticsPtr uniqCreator(const SingleStatisticsDescription & stat, DataTypePtr data_type)
 {
     return std::make_shared<StatisticsUniq>(stat, data_type);
 }
diff --git a/src/Storages/Statistics/StatisticsUniq.h b/src/Storages/Statistics/StatisticsUniq.h
index 5290585bd94..faabde8d47c 100644
--- a/src/Storages/Statistics/StatisticsUniq.h
+++ b/src/Storages/Statistics/StatisticsUniq.h
@@ -27,7 +27,7 @@ private:
 
 };
 
-void UniqValidator(const SingleStatisticsDescription &, DataTypePtr data_type);
-StatisticsPtr UniqCreator(const SingleStatisticsDescription & stat, DataTypePtr data_type);
+void uniqValidator(const SingleStatisticsDescription &, DataTypePtr data_type);
+StatisticsPtr uniqCreator(const SingleStatisticsDescription & stat, DataTypePtr data_type);
 
 }
diff --git a/src/Storages/Statistics/tests/gtest_stats.cpp b/src/Storages/Statistics/tests/gtest_stats.cpp
index 9b9fae83109..e55c52c49f3 100644
--- a/src/Storages/Statistics/tests/gtest_stats.cpp
+++ b/src/Storages/Statistics/tests/gtest_stats.cpp
@@ -44,50 +44,3 @@ TEST(Statistics, TDigestLessThan)
     std::reverse(data.begin(), data.end());
     test_less_than(data, {-1, 1e9, 50000.0, 3000.0, 30.0}, {0, 100000, 50000, 3000, 30}, {0, 0, 0.001, 0.001, 0.001});
 }
-
-using Fields = std::vector<Field>;
-
-template <typename T>
-void testConvertFieldToDataType(const DataTypePtr & data_type, const Fields & fields, const T & expected_value, bool convert_failed = false)
-{
-    for (const auto & field : fields)
-    {
-        Field converted_value;
-        try
-        {
-            converted_value = convertFieldToType(field, *data_type);
-        }
-        catch(...)
-        {
-            ASSERT_TRUE(convert_failed);
-        }
-        if (convert_failed)
-            ASSERT_TRUE(converted_value.isNull());
-        else
-            ASSERT_EQ(converted_value.template get<T>(), expected_value);
-    }
-}
-
-TEST(Statistics, convertFieldToType)
-{
-    Fields fields;
-
-    auto data_type_int8 = DataTypeFactory::instance().get("Int8");
-    fields = {1, 1.0, "1"};
-    testConvertFieldToDataType(data_type_int8, fields, static_cast<Int8>(1));
-
-    fields = {256, 1.1, "not a number"};
-    testConvertFieldToDataType(data_type_int8, fields, static_cast<Int8>(1), true);
-
-    auto data_type_float64 = DataTypeFactory::instance().get("Float64");
-    fields = {1, 1.0, "1.0"};
-    testConvertFieldToDataType(data_type_float64, fields, static_cast<Float64>(1.0));
-
-    auto data_type_string = DataTypeFactory::instance().get("String");
-    fields = {1, "1"};
-    testConvertFieldToDataType(data_type_string, fields, static_cast<String>("1"));
-
-    auto data_type_date = DataTypeFactory::instance().get("Date");
-    fields = {"2024-07-12", 19916};
-    testConvertFieldToDataType(data_type_date, fields, static_cast<UInt64>(19916));
-}

From 8fb560d2575c121b252ab1e6d8e13f9486dc2b38 Mon Sep 17 00:00:00 2001
From: Andrey Zvonov <azvonov@altinity.com>
Date: Tue, 23 Jul 2024 07:08:58 +0000
Subject: [PATCH 0625/1488] add replica sync

---
 tests/integration/test_broken_projections/test.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/integration/test_broken_projections/test.py b/tests/integration/test_broken_projections/test.py
index e198f98e4c5..a565f47449f 100644
--- a/tests/integration/test_broken_projections/test.py
+++ b/tests/integration/test_broken_projections/test.py
@@ -433,6 +433,7 @@ def test_broken_ignored_replicated(cluster):
     check(node, table_name, 1)
 
     create_table(node, table_name2, 2, table_name)
+    node.query(f"system sync replica {table_name}")
     check(node, table_name2, 1)
 
     break_projection(node, table_name, "proj1", "all_0_0_0", "data")

From d7ffbab7c4eab820b303bd80f6b52e856f4e1d47 Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Tue, 23 Jul 2024 09:23:18 +0200
Subject: [PATCH 0626/1488] Add test from #66378

Co-authored-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 src/Parsers/ASTTablesInSelectQuery.cpp            |  1 +
 .../0_stateless/03204_format_join_on.reference    |  4 ++++
 tests/queries/0_stateless/03204_format_join_on.sh | 15 +++++++++++++++
 3 files changed, 20 insertions(+)
 create mode 100644 tests/queries/0_stateless/03204_format_join_on.reference
 create mode 100644 tests/queries/0_stateless/03204_format_join_on.sh

diff --git a/src/Parsers/ASTTablesInSelectQuery.cpp b/src/Parsers/ASTTablesInSelectQuery.cpp
index dbb2a008bae..b6d42513aa7 100644
--- a/src/Parsers/ASTTablesInSelectQuery.cpp
+++ b/src/Parsers/ASTTablesInSelectQuery.cpp
@@ -235,6 +235,7 @@ void ASTTableJoin::formatImplAfterTable(const FormatSettings & settings, FormatS
     else if (on_expression)
     {
         settings.ostr << (settings.hilite ? hilite_keyword : "") << " ON " << (settings.hilite ? hilite_none : "");
+        /// If there is an alias for the whole expression parens should be added, otherwise it will be invalid syntax
         bool on_has_alias = !on_expression->tryGetAlias().empty();
         if (on_has_alias)
             settings.ostr << "(";
diff --git a/tests/queries/0_stateless/03204_format_join_on.reference b/tests/queries/0_stateless/03204_format_join_on.reference
new file mode 100644
index 00000000000..846f36fcca4
--- /dev/null
+++ b/tests/queries/0_stateless/03204_format_join_on.reference
@@ -0,0 +1,4 @@
+SELECT * FROM t1 INNER JOIN t2 ON ((t1.x = t2.x) AND (t1.x IS NULL) AS e2)
+SELECT * FROM t1 INNER JOIN t2 ON ((t1.x = t2.x) AND (t1.x IS NULL) AS e2)
+SELECT * FROM t1 INNER JOIN t2 ON (t1.x = t2.x) AND ((t1.x IS NULL) AS e2)
+SELECT * FROM t1 INNER JOIN t2 ON t1.x = t2.x
diff --git a/tests/queries/0_stateless/03204_format_join_on.sh b/tests/queries/0_stateless/03204_format_join_on.sh
new file mode 100644
index 00000000000..87b0afac042
--- /dev/null
+++ b/tests/queries/0_stateless/03204_format_join_on.sh
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CUR_DIR"/../shell_config.sh
+
+# regression for the JOIN ON alias for the whole expression
+phase1="$($CLICKHOUSE_FORMAT --oneline --query "SELECT * FROM t1 JOIN t2 ON ((t1.x = t2.x) AND (t1.x IS NULL) AS e2)")"
+echo "$phase1"
+# phase 2
+$CLICKHOUSE_FORMAT --oneline --query "$phase1"
+
+# other test cases
+$CLICKHOUSE_FORMAT --oneline --query "SELECT * FROM t1 JOIN t2 ON (t1.x = t2.x) AND (t1.x IS NULL AS e2)"
+$CLICKHOUSE_FORMAT --oneline --query "SELECT * FROM t1 JOIN t2 ON t1.x = t2.x"

From 2b56cbc1fdff42e31a9fee80f931726a5123e675 Mon Sep 17 00:00:00 2001
From: Max K <maxkaynov@gmail.com>
Date: Tue, 23 Jul 2024 09:42:58 +0200
Subject: [PATCH 0627/1488] CI: Add ec2 instance lifecycle metadata to CIDB

---
 tests/ci/clickhouse_helper.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tests/ci/clickhouse_helper.py b/tests/ci/clickhouse_helper.py
index 91ea5c6d5d3..287970cce9a 100644
--- a/tests/ci/clickhouse_helper.py
+++ b/tests/ci/clickhouse_helper.py
@@ -197,6 +197,10 @@ def get_instance_id():
     return _query_imds("latest/meta-data/instance-id")
 
 
+def get_instance_lifecycle():
+    return _query_imds("latest/meta-data/instance-life-cycle")
+
+
 def prepare_tests_results_for_clickhouse(
     pr_info: PRInfo,
     test_results: TestResults,
@@ -233,7 +237,7 @@ def prepare_tests_results_for_clickhouse(
         "head_ref": head_ref,
         "head_repo": head_repo,
         "task_url": pr_info.task_url,
-        "instance_type": get_instance_type(),
+        "instance_type": ",".join([get_instance_type(), get_instance_lifecycle()]),
         "instance_id": get_instance_id(),
     }
 

From 223eee3f46b07c38de3223fb56575f9ecbc5bea7 Mon Sep 17 00:00:00 2001
From: Igor Nikonov <igor@clickhouse.com>
Date: Tue, 23 Jul 2024 07:57:03 +0000
Subject: [PATCH 0628/1488] Comment to new IProcessor method

---
 src/Processors/IProcessor.h | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/src/Processors/IProcessor.h b/src/Processors/IProcessor.h
index 0776921a814..94e93595f4e 100644
--- a/src/Processors/IProcessor.h
+++ b/src/Processors/IProcessor.h
@@ -221,6 +221,21 @@ public:
         throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method 'schedule' is not implemented for {} processor", getName());
     }
 
+    /* The method is called right after asynchronous job is done
+     * i.e. when file descriptor returned by schedule() is readable.
+     * The sequence of method calls:
+     * ... prepare() -> schedule() -> onAsyncJobReady() -> work() ...
+     * See also comment to schedule() method
+     *
+     * It allows doing some preprocessing immediately after asynchronous job is done.
+     * The implementation should return control quickly, to avoid blocking another asynchronous completed jobs
+     * created by the same pipeline.
+     *
+     * Example, scheduling tasks for remote workers (file descriptor in this case is a socket)
+     * When the remote worker asks for the next task, doing it in onAsyncJobReady() we can provide it immediately.
+     * Otherwise, the returning of the next task for the remote worker can be delayed by current work done in the pipeline
+     * i.e. processor->work(), which will create unnecessary latency in query processing by remote workers   Not Committed Yet
+     */
     virtual void onAsyncJobReady() {}
 
     /** You must call this method if 'prepare' returned ExpandPipeline.

From b6ad57aa37f01ed4d101bd059b04222f361245ff Mon Sep 17 00:00:00 2001
From: Nikita Fomichev <nikita.fomichev@clickhouse.com>
Date: Mon, 22 Jul 2024 19:25:34 +0200
Subject: [PATCH 0629/1488] Stateless tests: change status for failed tests in
 case of server crash

---
 .../util/process_functional_tests_result.py   | 32 ++++++++++++-------
 1 file changed, 20 insertions(+), 12 deletions(-)

diff --git a/docker/test/util/process_functional_tests_result.py b/docker/test/util/process_functional_tests_result.py
index 4442c9d7d9e..dbe50eeade0 100755
--- a/docker/test/util/process_functional_tests_result.py
+++ b/docker/test/util/process_functional_tests_result.py
@@ -12,6 +12,7 @@ UNKNOWN_SIGN = "[ UNKNOWN "
 SKIPPED_SIGN = "[ SKIPPED "
 HUNG_SIGN = "Found hung queries in processlist"
 SERVER_DIED_SIGN = "Server died, terminating all processes"
+SERVER_DIED_SIGN2 = "Server does not respond to health check"
 DATABASE_SIGN = "Database: "
 
 SUCCESS_FINISH_SIGNS = ["All tests have finished", "No tests were run"]
@@ -43,7 +44,7 @@ def process_test_log(log_path, broken_tests):
             if HUNG_SIGN in line:
                 hung = True
                 break
-            if SERVER_DIED_SIGN in line:
+            if SERVER_DIED_SIGN in line or SERVER_DIED_SIGN2 in line:
                 server_died = True
             if RETRIES_SIGN in line:
                 retries = True
@@ -111,12 +112,12 @@ def process_test_log(log_path, broken_tests):
     # Python does not support TSV, so we have to escape '\t' and '\n' manually
     # and hope that complex escape sequences will not break anything
     test_results = [
-        (
+        [
             test[0],
             test[1],
             test[2],
             "".join(test[3])[:4096].replace("\t", "\\t").replace("\n", "\\n"),
-        )
+        ]
         for test in test_results
     ]
 
@@ -170,18 +171,24 @@ def process_result(result_path, broken_tests):
         if hung:
             description = "Some queries hung, "
             state = "failure"
-            test_results.append(("Some queries hung", "FAIL", "0", ""))
+            test_results.append(["Some queries hung", "FAIL", "0", ""])
         elif server_died:
             description = "Server died, "
             state = "failure"
-            test_results.append(("Server died", "FAIL", "0", ""))
+            # When ClickHouse server crashes, some tests are still running
+            # and fail because they cannot connect to server
+            for result in test_results:
+                if result[1] == "FAIL":
+                    result[1] = "SERVER_DIED"
+
+            test_results.append(["Server died", "FAIL", "0", ""])
         elif not success_finish:
             description = "Tests are not finished, "
             state = "failure"
-            test_results.append(("Tests are not finished", "FAIL", "0", ""))
+            test_results.append(["Tests are not finished", "FAIL", "0", ""])
         elif retries:
             description = "Some tests restarted, "
-            test_results.append(("Some tests restarted", "SKIPPED", "0", ""))
+            test_results.append(["Some tests restarted", "SKIPPED", "0", ""])
         else:
             description = ""
 
@@ -233,11 +240,12 @@ if __name__ == "__main__":
         # sort by status then by check name
         order = {
             "FAIL": 0,
-            "Timeout": 1,
-            "NOT_FAILED": 2,
-            "BROKEN": 3,
-            "OK": 4,
-            "SKIPPED": 5,
+            "SERVER_DIED": 1,
+            "Timeout": 2,
+            "NOT_FAILED": 3,
+            "BROKEN": 4,
+            "OK": 5,
+            "SKIPPED": 6,
         }
         return order.get(item[1], 10), str(item[0]), item[1]
 

From 492dab5e5d4fb775d8f3551e990fc078929c2bd8 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Tue, 23 Jul 2024 01:52:01 +0200
Subject: [PATCH 0630/1488] Update
 tests/queries/0_stateless/02992_all_columns_should_have_comment.sql

---
 .../0_stateless/02992_all_columns_should_have_comment.sql       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/02992_all_columns_should_have_comment.sql b/tests/queries/0_stateless/02992_all_columns_should_have_comment.sql
index dcb7c09a973..0d34b033354 100644
--- a/tests/queries/0_stateless/02992_all_columns_should_have_comment.sql
+++ b/tests/queries/0_stateless/02992_all_columns_should_have_comment.sql
@@ -3,6 +3,6 @@ SELECT 'Column ' || name || ' from table ' || concat(database, '.', table) || '
 FROM system.columns
 WHERE (database = 'system') AND
       (comment = '') AND
-      (table NOT ILIKE '%_log_%') AND
+      (table NOT ILIKE '%\_log\_%') AND
       (table NOT IN ('numbers', 'numbers_mt', 'one', 'generate_series', 'generateSeries', 'coverage_log', 'filesystem_read_prefetches_log')) AND
       (default_kind != 'ALIAS');

From b164014bf99278f6e1bac67d3066d77aefc229c2 Mon Sep 17 00:00:00 2001
From: Max K <maxkaynov@gmail.com>
Date: Tue, 23 Jul 2024 11:00:05 +0200
Subject: [PATCH 0631/1488] CI: Remove ci runners scripts from oss

---
 tests/ci/worker/.gitignore            |   1 -
 tests/ci/worker/deploy-runner-init.sh |  87 ------
 tests/ci/worker/init_runner.sh        | 406 --------------------------
 3 files changed, 494 deletions(-)
 delete mode 100644 tests/ci/worker/.gitignore
 delete mode 100755 tests/ci/worker/deploy-runner-init.sh
 delete mode 100644 tests/ci/worker/init_runner.sh

diff --git a/tests/ci/worker/.gitignore b/tests/ci/worker/.gitignore
deleted file mode 100644
index 4ed18989e78..00000000000
--- a/tests/ci/worker/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-generated_*init_runner.sh
diff --git a/tests/ci/worker/deploy-runner-init.sh b/tests/ci/worker/deploy-runner-init.sh
deleted file mode 100755
index 96fbd82a99c..00000000000
--- a/tests/ci/worker/deploy-runner-init.sh
+++ /dev/null
@@ -1,87 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-usage() {
-  echo "Usage: $0 ENVIRONMENT" >&2
-  echo "Valid values for ENVIRONMENT: staging, production" >&2
-  exit 1
-}
-
-case "$1" in
-  staging|production)
-    ENVIRONMENT="$1" ;;
-  --help)
-    usage ;;
-  *)
-    echo "Invalid argument" >&2
-    usage ;;
-esac
-
-cd "$(dirname "$0")" || exit 1
-SOURCE_SCRIPT='init_runner.sh'
-
-check_response() {
-  # Are we even in the interactive shell?
-  [ -t 1 ] || return 1
-  local request
-  request="$1"
-  read -rp "$request (y/N): " response
-  case "$response" in
-    [Yy])
-      return 0
-      # Your code to continue goes here
-      ;;
-    *)
-      return 1
-      ;;
-  esac
-}
-
-check_dirty() {
-  if [ -n "$(git status --porcelain=v2 "$SOURCE_SCRIPT")" ]; then
-    echo "The $SOURCE_SCRIPT has uncommited changes, won't deploy it" >&2
-    exit 1
-  fi
-}
-GIT_HASH=$(git log -1 --format=format:%H)
-
-header() {
-  cat << EOF
-#!/usr/bin/env bash
-
-echo 'The $ENVIRONMENT script is generated from $SOURCE_SCRIPT, commit $GIT_HASH'
-
-EOF
-}
-
-body() {
-  local first_line
-  first_line=$(sed -n '/^# THE SCRIPT START$/{=;q;}' "$SOURCE_SCRIPT")
-  if [ -z "$first_line" ]; then
-    echo "The pattern '# THE SCRIPT START' is not found in $SOURCE_SCRIPT" >&2
-    exit 1
-  fi
-  tail "+$first_line" "$SOURCE_SCRIPT"
-}
-
-GENERATED_FILE="generated_${ENVIRONMENT}_${SOURCE_SCRIPT}"
-
-{ header && body; } > "$GENERATED_FILE"
-
-echo "The file $GENERATED_FILE is generated"
-
-if check_response "Display the content of $GENERATED_FILE?"; then
-  if [ -z "$PAGER" ]; then
-    less "$GENERATED_FILE"
-  else
-    $PAGER "$GENERATED_FILE"
-  fi
-fi
-
-check_dirty
-
-S3_OBJECT=${S3_OBJECT:-s3://github-runners-data/cloud-init/${ENVIRONMENT}.sh}
-if check_response "Deploy the generated script to $S3_OBJECT?"; then
-  aws s3 mv "$GENERATED_FILE" "$S3_OBJECT"
-fi
diff --git a/tests/ci/worker/init_runner.sh b/tests/ci/worker/init_runner.sh
deleted file mode 100644
index 5177e112edd..00000000000
--- a/tests/ci/worker/init_runner.sh
+++ /dev/null
@@ -1,406 +0,0 @@
-#!/usr/bin/env bash
-
-cat > /dev/null << 'EOF'
-The following content is embedded into the s3 object via the script
-deploy-runner-init.sh {staging,production}
-with additional helping information
-
-In the `user data` you should define as the following text
-between `### COPY BELOW` and `### COPY ABOVE`
-
-### COPY BELOW
-Content-Type: multipart/mixed; boundary="//"
-MIME-Version: 1.0
-
---//
-Content-Type: text/cloud-config; charset="us-ascii"
-MIME-Version: 1.0
-Content-Transfer-Encoding: 7bit
-Content-Disposition: attachment; filename="cloud-config.txt"
-
-#cloud-config
-cloud_final_modules:
-- [scripts-user, always]
-
---//
-Content-Type: text/x-shellscript; charset="us-ascii"
-MIME-Version: 1.0
-Content-Transfer-Encoding: 7bit
-Content-Disposition: attachment; filename="userdata.txt"
-
-#!/bin/bash
-INSTANCE_ID=$(ec2metadata --instance-id)
-INIT_ENVIRONMENT=$(/usr/local/bin/aws ec2 describe-tags --filters "Name=resource-id,Values=$INSTANCE_ID" --query "Tags[?Key=='github:init-environment'].Value" --output text)
-echo "Downloading and using $INIT_ENVIRONMENT cloud-init.sh"
-aws s3 cp "s3://github-runners-data/cloud-init/${INIT_ENVIRONMENT:-production}.sh" /tmp/cloud-init.sh
-chmod 0700 /tmp/cloud-init.sh
-exec bash /tmp/cloud-init.sh
---//
-### COPY ABOVE
-EOF
-
-# THE SCRIPT START
-
-set -uo pipefail
-
-####################################
-#            IMPORTANT!            #
-# EC2 instance should have         #
-# `github:runner-type` tag         #
-# set accordingly to a runner role #
-####################################
-
-echo "Running init v1.1"
-export DEBIAN_FRONTEND=noninteractive
-export RUNNER_HOME=/home/ubuntu/actions-runner
-
-export RUNNER_ORG="ClickHouse"
-export RUNNER_URL="https://github.com/${RUNNER_ORG}"
-# Funny fact, but metadata service has fixed IP
-INSTANCE_ID=$(ec2metadata --instance-id)
-export INSTANCE_ID
-
-bash /usr/local/share/scripts/init-network.sh
-
-# combine labels
-RUNNER_TYPE=$(/usr/local/bin/aws ec2 describe-tags --filters "Name=resource-id,Values=$INSTANCE_ID" --query "Tags[?Key=='github:runner-type'].Value" --output text)
-LABELS="self-hosted,Linux,$(uname -m),$RUNNER_TYPE"
-export LABELS
-echo "Instance Labels: $LABELS"
-
-LIFE_CYCLE=$(curl -s --fail http://169.254.169.254/latest/meta-data/instance-life-cycle)
-export LIFE_CYCLE
-echo "Instance lifecycle: $LIFE_CYCLE"
-
-INSTANCE_TYPE=$(ec2metadata --instance-type)
-echo "Instance type: $INSTANCE_TYPE"
-
-# Refresh CloudWatch agent config
-aws ssm get-parameter --region us-east-1 --name AmazonCloudWatch-github-runners --query 'Parameter.Value' --output text > /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json
-systemctl restart amazon-cloudwatch-agent.service
-
-# Refresh teams ssh keys
-TEAM_KEYS_URL=$(aws ssm get-parameter --region us-east-1 --name team-keys-url --query 'Parameter.Value' --output=text)
-curl -s "${TEAM_KEYS_URL}" > /home/ubuntu/.ssh/authorized_keys2
-chown ubuntu: /home/ubuntu/.ssh -R
-
-
-# Create a pre-run script that will provide diagnostics info
-mkdir -p /tmp/actions-hooks
-cat > /tmp/actions-hooks/common.sh << 'EOF'
-#!/bin/bash
-EOF
-
-terminate_delayed() {
-    # The function for post hook to gracefully finish the job and then tear down
-    # The very specific sleep time is used later to determine in the main loop if
-    # the instance is tearing down
-    # IF `sleep` IS CHANGED, CHANGE ANOTHER VALUE IN `pgrep`
-    sleep=13.14159265358979323846
-    echo "Going to terminate the runner's instance in $sleep seconds"
-    # We execute it with `at` to not have it as an orphan process, but launched independently
-    # GH Runners kill all remain processes
-    echo "sleep '$sleep'; aws ec2 terminate-instances --instance-ids $INSTANCE_ID" | at now || \
-        aws ec2 terminate-instances --instance-ids "$INSTANCE_ID"  # workaround for complete out of space or non-installed `at`
-    exit 0
-}
-
-detect_delayed_termination() {
-    # The function look for very specific sleep with pi
-    if pgrep 'sleep 13.14159265358979323846'; then
-        echo 'The instance has delayed termination, sleep the same time to wait if it goes down'
-        sleep 14
-    fi
-}
-
-declare -f terminate_delayed >> /tmp/actions-hooks/common.sh
-
-terminate_and_exit() {
-    # Terminate instance and exit from the script instantly
-    echo "Going to terminate the runner's instance"
-    aws ec2 terminate-instances --instance-ids "$INSTANCE_ID"
-    exit 0
-}
-
-terminate_decrease_and_exit() {
-    # Terminate instance and exit from the script instantly
-    echo "Going to terminate the runner's instance and decrease asg capacity"
-    aws autoscaling terminate-instance-in-auto-scaling-group --instance-id "$INSTANCE_ID" --should-decrement-desired-capacity
-    exit 0
-}
-
-declare -f terminate_and_exit >> /tmp/actions-hooks/common.sh
-
-check_spot_instance_is_old() {
-    if [ "$LIFE_CYCLE" == "spot" ]; then
-        local UPTIME
-        UPTIME=$(< /proc/uptime)
-        UPTIME=${UPTIME%%.*}
-        if (( 3600 < UPTIME )); then
-            echo "The spot instance has uptime $UPTIME, it's time to shut it down"
-            return 0
-        fi
-    fi
-    return 1
-}
-
-check_proceed_spot_termination() {
-    # The function checks and proceeds spot instance termination if exists
-    # The event for spot instance termination
-    local FORCE
-    FORCE=${1:-}
-    if TERMINATION_DATA=$(curl -s --fail http://169.254.169.254/latest/meta-data/spot/instance-action); then
-        # https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-instance-termination-notices.html#instance-action-metadata
-        _action=$(jq '.action' -r <<< "$TERMINATION_DATA")
-        _time=$(jq '.time | fromdate' <<< "$TERMINATION_DATA")
-        _until_action=$((_time - $(date +%s)))
-        echo "Received the '$_action' event that will be effective in $_until_action seconds"
-        if (( _until_action <= 30 )) || [ "$FORCE" == "force" ]; then
-            echo "The action $_action will be done in $_until_action, killing the runner and exit"
-            local runner_pid
-            runner_pid=$(pgrep Runner.Listener)
-            if [ -n "$runner_pid" ]; then
-                # Kill the runner to not allow it cancelling the job
-                # shellcheck disable=SC2046
-                kill -9 "$runner_pid" $(list_children "$runner_pid")
-            fi
-            sudo -u ubuntu ./config.sh remove --token "$(get_runner_token)"
-            terminate_and_exit
-        fi
-    fi
-}
-
-no_terminating_metadata() {
-    # The function check that instance could continue work
-    # Returns 1 if any of termination events are received
-
-    # The event for rebalance recommendation. Not strict, so we have some room to make a decision here
-    if curl -s --fail http://169.254.169.254/latest/meta-data/events/recommendations/rebalance; then
-        echo 'Received recommendation to rebalance, checking the uptime'
-        local UPTIME
-        UPTIME=$(< /proc/uptime)
-        UPTIME=${UPTIME%%.*}
-        # We don't shutdown the instances younger than 30m
-        if (( 1800 < UPTIME )); then
-            # To not shutdown everything at once, use the 66% to survive
-            if (( $((RANDOM % 3)) == 0 )); then
-                echo 'The instance is older than 30m and won the roulette'
-                return 1
-            fi
-            echo 'The instance is older than 30m, but is not chosen for rebalance'
-        else
-            echo 'The instance is younger than 30m, do not shut it down'
-        fi
-    fi
-
-    # Checks if the ASG in a lifecycle hook state
-    local ASG_STATUS
-    ASG_STATUS=$(curl -s http://169.254.169.254/latest/meta-data/autoscaling/target-lifecycle-state)
-    if [ "$ASG_STATUS" == "Terminated" ]; then
-        echo 'The instance in ASG status Terminating:Wait'
-        return 1
-    fi
-}
-
-terminate_on_event() {
-    # If there is a rebalance event, then the instance could die soon
-    # Let's don't wait for it and terminate proactively
-    if curl -s --fail http://169.254.169.254/latest/meta-data/events/recommendations/rebalance; then
-        terminate_and_exit
-    fi
-
-    # Here we check if the autoscaling group marked the instance for termination, and it's wait for the job to finish
-    ASG_STATUS=$(curl -s http://169.254.169.254/latest/meta-data/autoscaling/target-lifecycle-state)
-    if [ "$ASG_STATUS" == "Terminated" ]; then
-        INSTANCE_ID=$(ec2metadata --instance-id)
-        ASG_NAME=$(aws ec2 describe-tags --filters "Name=resource-id,Values=$INSTANCE_ID" --query "Tags[?Key=='aws:autoscaling:groupName'].Value" --output text)
-        LIFECYCLE_HOOKS=$(aws autoscaling describe-lifecycle-hooks --auto-scaling-group-name "$ASG_NAME" --query "LifecycleHooks[].LifecycleHookName" --output text)
-        for LCH in $LIFECYCLE_HOOKS; do
-            aws autoscaling complete-lifecycle-action --lifecycle-action-result CONTINUE \
-                --lifecycle-hook-name "$LCH" --auto-scaling-group-name "$ASG_NAME" \
-                --instance-id "$INSTANCE_ID"
-            true  # autoformat issue
-        done
-        echo 'The runner is marked as "Terminated" by the autoscaling group, we are terminating'
-        terminate_and_exit
-    fi
-}
-
-cat > /tmp/actions-hooks/pre-run.sh << EOF
-#!/bin/bash
-set -uo pipefail
-
-echo "Runner's public DNS: $(ec2metadata --public-hostname)"
-echo "Runner's labels: ${LABELS}"
-echo "Runner's instance type: $(ec2metadata --instance-type)"
-EOF
-
-# Create a post-run script that will restart docker daemon before the job started
-cat > /tmp/actions-hooks/post-run.sh << 'EOF'
-#!/bin/bash
-set -xuo pipefail
-
-source /tmp/actions-hooks/common.sh
-
-# Free KiB, free percents
-ROOT_STAT=($(df / | awk '/\// {print $4 " " int($4/$2 * 100)}'))
-if [[ ${ROOT_STAT[0]} -lt 3000000 ]] || [[ ${ROOT_STAT[1]} -lt 5 ]]; then
-  echo "The runner has ${ROOT_STAT[0]}KiB and ${ROOT_STAT[1]}% of free space on /"
-  terminate_delayed
-fi
-
-# shellcheck disable=SC2046
-docker ps --quiet | xargs --no-run-if-empty docker kill ||:
-# shellcheck disable=SC2046
-docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
-
-# If we have hanged containers after the previous commands, than we have a hanged one
-# and should restart the daemon
-if [ "$(docker ps --all --quiet)" ]; then
-  # Systemd service of docker has StartLimitBurst=3 and StartLimitInterval=60s,
-  # that's why we try restarting it for long
-  for i in {1..25};
-  do
-    sudo systemctl restart docker && break || sleep 5
-  done
-
-  for i in {1..10}
-  do
-    docker info && break || sleep 2
-  done
-  # Last chance, otherwise we have to terminate poor instance
-  docker info 1>/dev/null || { echo Docker unable to start; terminate_delayed ; }
-fi
-EOF
-
-get_runner_token() {
-    /usr/local/bin/aws ssm  get-parameter --name github_runner_registration_token --with-decryption --output text --query Parameter.Value
-}
-
-is_job_assigned() {
-    local runner_pid
-    runner_pid=$(pgrep Runner.Listener)
-    if [ -z "$runner_pid" ]; then
-        # if runner has finished, it's fine
-        return 0
-    fi
-    local log_file
-    log_file=$(lsof -p "$runner_pid" 2>/dev/null | grep -o "$RUNNER_HOME/_diag/Runner.*log")
-    if [ -z "$log_file" ]; then
-        # assume, the process is over or just started
-        return 0
-    fi
-    # So far it's the only solid way to determine that the job is starting
-    grep -q 'Terminal] .* Running job:' "$log_file" \
-        && return 0 \
-        || return 1
-}
-
-list_children () {
-    local children
-    children=$(ps --ppid "$1" -o pid=)
-    if [ -z "$children" ]; then
-        return
-    fi
-
-    for pid in $children; do
-        list_children "$pid"
-    done
-    echo "$children"
-}
-
-# There's possibility that it fails because the runner's version is outdated,
-# so after the first failure we'll try to launch it with enabled autoupdate.
-#
-# We'll fail and terminate after 10 consequent failures.
-ATTEMPT=0
-# In `kill` 0 means "all processes in process group", -1 is "all but PID 1"
-# We use `-2` to get an error
-RUNNER_PID=-2
-
-while true; do
-    # Does not send signal, but checks that the process $RUNNER_PID is running
-    if kill -0 -- $RUNNER_PID; then
-        ATTEMPT=0
-        echo "Runner is working with pid $RUNNER_PID, checking the metadata in background"
-        check_proceed_spot_termination
-
-        if ! is_job_assigned; then
-            RUNNER_AGE=$(( $(date +%s) - $(stat -c +%Y /proc/"$RUNNER_PID" 2>/dev/null || date +%s) ))
-            echo "The runner is launched $RUNNER_AGE seconds ago and still hasn't received a job"
-            if (( 60 < RUNNER_AGE )); then
-                echo "Attempt to delete the runner for a graceful shutdown"
-                sudo -u ubuntu ./config.sh remove --token "$(get_runner_token)" \
-                    || continue
-                echo "Runner didn't launch or have assigned jobs after ${RUNNER_AGE} seconds, shutting down"
-                terminate_decrease_and_exit
-            fi
-        fi
-    else
-        if [ "$RUNNER_PID" != "-2" ]; then
-            wait $RUNNER_PID \
-                && echo "Runner with PID $RUNNER_PID successfully finished" \
-                || echo "Attempt $((++ATTEMPT)) to start the runner"
-        fi
-        if (( ATTEMPT > 10 )); then
-            echo "The runner has failed to start after $ATTEMPT attempt. Give up and terminate it"
-            terminate_and_exit
-        fi
-
-        cd $RUNNER_HOME || terminate_and_exit
-        detect_delayed_termination
-        # If runner is not active, check that it needs to terminate itself
-        echo "Checking if the instance suppose to terminate"
-        no_terminating_metadata || terminate_on_event
-        check_spot_instance_is_old && terminate_and_exit
-        check_proceed_spot_termination force
-
-        echo "Going to configure runner"
-        token_args=(--token "$(get_runner_token)")
-        config_args=(
-            "${token_args[@]}" --url "$RUNNER_URL"
-            --ephemeral --unattended --replace --runnergroup Default
-            --labels "$LABELS" --work _work --name "$INSTANCE_ID"
-        )
-        if (( ATTEMPT > 1 )); then
-            echo 'The runner failed to start at least once. Removing it and then configuring with autoupdate enabled.'
-            sudo -u ubuntu ./config.sh remove "${token_args[@]}"
-            sudo -u ubuntu ./config.sh "${config_args[@]}"
-        else
-            echo "Configure runner with disabled autoupdate"
-            config_args+=("--disableupdate")
-            sudo -u ubuntu ./config.sh "${config_args[@]}"
-        fi
-
-        echo "Another one check to avoid race between runner and infrastructure"
-        no_terminating_metadata || terminate_on_event
-        check_spot_instance_is_old && terminate_and_exit
-        check_proceed_spot_termination force
-
-        # There were some failures to start the Job because of trash in _work
-        rm -rf _work
-
-        # https://github.com/actions/runner/issues/3266
-        # We're unable to know if the runner is failed to start.
-        echo 'Monkey-patching run helpers to get genuine exit code of the runner'
-        for script in run.sh run-helper.sh.template; do
-            # shellcheck disable=SC2016
-            grep -q 'exit 0$' "$script" && \
-                sed 's/exit 0/exit $returnCode/' -i "$script" && \
-                echo "Script $script is patched"
-        done
-
-        echo "Run"
-        sudo -u ubuntu \
-          ACTIONS_RUNNER_HOOK_JOB_STARTED=/tmp/actions-hooks/pre-run.sh \
-          ACTIONS_RUNNER_HOOK_JOB_COMPLETED=/tmp/actions-hooks/post-run.sh \
-          ./run.sh &
-        RUNNER_PID=$!
-
-        sleep 10
-    fi
-
-    sleep 5
-done
-
-# vim:ts=4:sw=4

From 1c723e7fd7f85cb8bdd4b6103a158229912af164 Mon Sep 17 00:00:00 2001
From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com>
Date: Tue, 23 Jul 2024 09:10:47 +0000
Subject: [PATCH 0632/1488] black

---
 tests/integration/test_storage_s3/test.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/tests/integration/test_storage_s3/test.py b/tests/integration/test_storage_s3/test.py
index 84c887be388..5453ad5a796 100644
--- a/tests/integration/test_storage_s3/test.py
+++ b/tests/integration/test_storage_s3/test.py
@@ -208,8 +208,12 @@ def test_partition_by_string_column(started_cluster):
     assert '1,"foo/bar"\n' == get_s3_file_content(
         started_cluster, bucket, f"{id}/test_foo/bar.csv"
     )
-    assert '3,"йцук"\n' == get_s3_file_content(started_cluster, bucket, f"{id}/test_йцук.csv")
-    assert '78,"你好"\n' == get_s3_file_content(started_cluster, bucket, f"{id}/test_你好.csv")
+    assert '3,"йцук"\n' == get_s3_file_content(
+        started_cluster, bucket, f"{id}/test_йцук.csv"
+    )
+    assert '78,"你好"\n' == get_s3_file_content(
+        started_cluster, bucket, f"{id}/test_你好.csv"
+    )
 
 
 def test_partition_by_const_column(started_cluster):
@@ -895,9 +899,10 @@ def test_storage_s3_get_slow(started_cluster):
 
 
 def test_storage_s3_put_uncompressed(started_cluster):
+    id = uuid.uuid4()
     bucket = started_cluster.minio_bucket
     instance = started_cluster.instances["dummy"]
-    filename = "test_put_uncompressed.bin"
+    filename = f"{id}/test_put_uncompressed.bin"
     name = "test_put_uncompressed"
     data = [
         "'Gloria Thompson',99",
@@ -949,6 +954,7 @@ def test_storage_s3_put_uncompressed(started_cluster):
     r = result.strip().split("\t")
     assert int(r[0]) >= 1, blob_storage_log
     assert all(col == r[0] for col in r), blob_storage_log
+    run_query(instance, f"DROP TABLE {name}")
 
 
 @pytest.mark.parametrize(

From a000f8f8a13598d8c4dd24043d5029d7a0158ace Mon Sep 17 00:00:00 2001
From: Yohann Jardin <yohann.jardin@telecomnancy.net>
Date: Tue, 23 Jul 2024 11:27:16 +0200
Subject: [PATCH 0633/1488] fix stateless test 02346_non_negative_derivative

---
 tests/queries/0_stateless/02346_non_negative_derivative.sql | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/queries/0_stateless/02346_non_negative_derivative.sql b/tests/queries/0_stateless/02346_non_negative_derivative.sql
index 704241da16c..ab648f2ee73 100644
--- a/tests/queries/0_stateless/02346_non_negative_derivative.sql
+++ b/tests/queries/0_stateless/02346_non_negative_derivative.sql
@@ -58,8 +58,8 @@ SELECT ts, metric, nonNegativeDerivative(metric, ts, INTERVAL 13 YEAR) OVER (PAR
 -- test against wrong arguments/types
 SELECT ts, metric, nonNegativeDerivative(metric, 1, INTERVAL 3 NANOSECOND) OVER (PARTITION BY metric ORDER BY ts, metric ASC Rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS deriv FROM nnd; -- { serverError BAD_ARGUMENTS }
 SELECT ts, metric, nonNegativeDerivative('string not datetime', ts, INTERVAL 3 NANOSECOND) OVER (PARTITION BY metric ORDER BY ts, metric ASC Rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS deriv FROM nnd; -- { serverError BAD_ARGUMENTS }
-SELECT ts, metric, nonNegativeDerivative(metric, ts, INTERVAL 3 NANOSECOND, id) OVER (PARTITION BY metric ORDER BY ts, metric ASC Rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS deriv FROM nnd; -- { serverError BAD_ARGUMENTS }
-SELECT ts, metric, nonNegativeDerivative(metric) OVER (PARTITION BY metric ORDER BY ts, metric ASC Rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS deriv FROM nnd; -- { serverError BAD_ARGUMENTS }
+SELECT ts, metric, nonNegativeDerivative(metric, ts, INTERVAL 3 NANOSECOND, id) OVER (PARTITION BY metric ORDER BY ts, metric ASC Rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS deriv FROM nnd; -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH }
+SELECT ts, metric, nonNegativeDerivative(metric) OVER (PARTITION BY metric ORDER BY ts, metric ASC Rows BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING) AS deriv FROM nnd; -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH }
 
 -- cleanup
 DROP TABLE IF EXISTS nnd;

From 7750914f0b85e324dbf5d49d565312f117c02a2a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?=
 <benjamin.antal@clickhouse.com>
Date: Tue, 23 Jul 2024 09:33:58 +0000
Subject: [PATCH 0634/1488] Decrease rate limit

In slow tests (sanitizer, debug) it can take 7-8 seconds to start up `clickhouse-client`. In those cases  the rate limit doesn't make the wait time longer, because the input will be already in the input of clickhouse-client when it starts to execute the query.
---
 .../0_stateless/01923_network_receive_time_metric_insert.sh     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/01923_network_receive_time_metric_insert.sh b/tests/queries/0_stateless/01923_network_receive_time_metric_insert.sh
index 4d7e79fae52..a6b83eba27d 100755
--- a/tests/queries/0_stateless/01923_network_receive_time_metric_insert.sh
+++ b/tests/queries/0_stateless/01923_network_receive_time_metric_insert.sh
@@ -9,7 +9,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 ${CLICKHOUSE_CLIENT} --multiquery --query "DROP TABLE IF EXISTS t; CREATE TABLE t (x UInt64) ENGINE = Memory;"
 
 # Rate limit is chosen for operation to spent more than one second.
-seq 1 1000 | pv --quiet --rate-limit 500 | ${CLICKHOUSE_CLIENT} --query "INSERT INTO t FORMAT TSV"
+seq 1 1000 | pv --quiet --rate-limit 400 | ${CLICKHOUSE_CLIENT} --query "INSERT INTO t FORMAT TSV"
 
 # We check that the value of NetworkReceiveElapsedMicroseconds correctly includes the time spent waiting data from the client.
 ${CLICKHOUSE_CLIENT} --multiquery --query "SYSTEM FLUSH LOGS;

From bb28a65e9814302307c52d9e9b7ea6d052624fad Mon Sep 17 00:00:00 2001
From: Max K <maxkaynov@gmail.com>
Date: Tue, 23 Jul 2024 11:25:19 +0200
Subject: [PATCH 0635/1488] minor fix for wf status reporting

---
 .github/workflows/backport_branches.yml |  3 ---
 .github/workflows/master.yml            |  3 ---
 .github/workflows/merge_queue.yml       |  3 ---
 .github/workflows/nightly.yml           |  3 ---
 .github/workflows/pull_request.yml      |  3 ---
 .github/workflows/release_branches.yml  |  4 +---
 tests/ci/ci_buddy.py                    |  1 +
 tests/ci/ci_utils.py                    | 23 +++++++++++++++++++----
 8 files changed, 21 insertions(+), 22 deletions(-)

diff --git a/.github/workflows/backport_branches.yml b/.github/workflows/backport_branches.yml
index 50f4f503f5d..c602a46d23c 100644
--- a/.github/workflows/backport_branches.yml
+++ b/.github/workflows/backport_branches.yml
@@ -272,7 +272,4 @@ jobs:
           cat >> "$WORKFLOW_RESULT_FILE" << 'EOF'
           ${{ toJson(needs) }}
           EOF
-          echo "::group::Workflow results"
-          python3 -m json.tool "$WORKFLOW_RESULT_FILE"
-          echo "::endgroup::"
           python3 ./tests/ci/ci_buddy.py --check-wf-status
diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index b28d87ee31f..7c319da6045 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -138,7 +138,4 @@ jobs:
           cat >> "$WORKFLOW_RESULT_FILE" << 'EOF'
           ${{ toJson(needs) }}
           EOF
-          echo "::group::Workflow results"
-          python3 -m json.tool "$WORKFLOW_RESULT_FILE"
-          echo "::endgroup::"
           python3 ./tests/ci/ci_buddy.py --check-wf-status
diff --git a/.github/workflows/merge_queue.yml b/.github/workflows/merge_queue.yml
index db89825a99a..4b186241a0e 100644
--- a/.github/workflows/merge_queue.yml
+++ b/.github/workflows/merge_queue.yml
@@ -111,7 +111,4 @@ jobs:
           cat >> "$WORKFLOW_RESULT_FILE" << 'EOF'
           ${{ toJson(needs) }}
           EOF
-          echo "::group::Workflow results"
-          python3 -m json.tool "$WORKFLOW_RESULT_FILE"
-          echo "::endgroup::"
           python3 ./tests/ci/ci_buddy.py --check-wf-status
diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
index fd5b5eefcc4..84db3338065 100644
--- a/.github/workflows/nightly.yml
+++ b/.github/workflows/nightly.yml
@@ -57,7 +57,4 @@ jobs:
           cat >> "$WORKFLOW_RESULT_FILE" << 'EOF'
           ${{ toJson(needs) }}
           EOF
-          echo "::group::Workflow results"
-          python3 -m json.tool "$WORKFLOW_RESULT_FILE"
-          echo "::endgroup::"
           python3 ./tests/ci/ci_buddy.py --check-wf-status
diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
index 9930cf6dde4..c7d7b28af38 100644
--- a/.github/workflows/pull_request.yml
+++ b/.github/workflows/pull_request.yml
@@ -171,9 +171,6 @@ jobs:
           cat >> "$WORKFLOW_RESULT_FILE" << 'EOF'
           ${{ toJson(needs) }}
           EOF
-          echo "::group::Workflow results"
-          python3 -m json.tool "$WORKFLOW_RESULT_FILE"
-          echo "::endgroup::"
           python3 ./tests/ci/ci_buddy.py --check-wf-status
 
   ################################# Stage Final #################################
diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml
index 50565112825..bca9ff33cd0 100644
--- a/.github/workflows/release_branches.yml
+++ b/.github/workflows/release_branches.yml
@@ -492,7 +492,5 @@ jobs:
           cat >> "$WORKFLOW_RESULT_FILE" << 'EOF'
           ${{ toJson(needs) }}
           EOF
-          echo "::group::Workflow results"
-          python3 -m json.tool "$WORKFLOW_RESULT_FILE"
-          echo "::endgroup::"
+
           python3 ./tests/ci/ci_buddy.py --check-wf-status
diff --git a/tests/ci/ci_buddy.py b/tests/ci/ci_buddy.py
index 688c7d59988..dfb5885270a 100644
--- a/tests/ci/ci_buddy.py
+++ b/tests/ci/ci_buddy.py
@@ -31,6 +31,7 @@ class CIBuddy:
         self.sha = pr_info.sha[:10]
 
     def check_workflow(self):
+        GHActions.print_workflow_results()
         res = GHActions.get_workflow_job_result(GHActions.ActionsNames.RunConfig)
         if res != GHActions.ActionStatuses.SUCCESS:
             self.post_job_error("Workflow Configuration Failed", critical=True)
diff --git a/tests/ci/ci_utils.py b/tests/ci/ci_utils.py
index 1963e3f39d0..3b12fe6974f 100644
--- a/tests/ci/ci_utils.py
+++ b/tests/ci/ci_utils.py
@@ -92,15 +92,30 @@ class GHActions:
         PENDING = "pending"
         SUCCESS = "success"
 
-    @staticmethod
-    def get_workflow_job_result(wf_job_name: str) -> Optional[str]:
+    @classmethod
+    def _get_workflow_results(cls):
         if not Path(Envs.WORKFLOW_RESULT_FILE).exists():
             print(
                 f"ERROR: Failed to get workflow results from file [{Envs.WORKFLOW_RESULT_FILE}]"
             )
-            return None
+            return {}
         with open(Envs.WORKFLOW_RESULT_FILE, "r", encoding="utf-8") as json_file:
-            res = json.load(json_file)
+            try:
+                res = json.load(json_file)
+            except json.JSONDecodeError as e:
+                print(f"ERROR: json decoder exception {e}")
+                return {}
+        return res
+
+    @classmethod
+    def print_workflow_results(cls):
+        res = cls._get_workflow_results()
+        results = [f"{job}: {data['result']}" for job, data in res.items()]
+        cls.print_in_group("Workflow results", results)
+
+    @classmethod
+    def get_workflow_job_result(cls, wf_job_name: str) -> Optional[str]:
+        res = cls._get_workflow_results()
         if wf_job_name in res:
             return res[wf_job_name]["result"]  # type: ignore
         else:

From 932033fca9bdacbfdb544fac5389e03fa7732eeb Mon Sep 17 00:00:00 2001
From: Michael Stetsyuk <mike.stetsyuk@gmail.com>
Date: Tue, 23 Jul 2024 10:55:45 +0100
Subject: [PATCH 0636/1488] use atomic to avoid data race

---
 src/Databases/DatabaseReplicatedWorker.cpp | 12 +++++++++---
 src/Databases/DatabaseReplicatedWorker.h   |  5 +++--
 2 files changed, 12 insertions(+), 5 deletions(-)

diff --git a/src/Databases/DatabaseReplicatedWorker.cpp b/src/Databases/DatabaseReplicatedWorker.cpp
index cea2d123f87..a9a74c5f56a 100644
--- a/src/Databases/DatabaseReplicatedWorker.cpp
+++ b/src/Databases/DatabaseReplicatedWorker.cpp
@@ -32,7 +32,8 @@ DatabaseReplicatedDDLWorker::DatabaseReplicatedDDLWorker(DatabaseReplicated * db
 
 bool DatabaseReplicatedDDLWorker::initializeMainThread()
 {
-    initialization_duration_timer.emplace();
+    initialization_duration_timer.restart();
+    initializing.store(true, std::memory_order_release);
 
     while (!stop_flag)
     {
@@ -71,7 +72,7 @@ bool DatabaseReplicatedDDLWorker::initializeMainThread()
 
             initializeReplication();
             initialized = true;
-            initialization_duration_timer.reset();
+            initializing.store(false, std::memory_order_relaxed);
             return true;
         }
         catch (...)
@@ -81,7 +82,7 @@ bool DatabaseReplicatedDDLWorker::initializeMainThread()
         }
     }
 
-    initialization_duration_timer.reset();
+    initializing.store(false, std::memory_order_relaxed);
     return false;
 }
 
@@ -463,4 +464,9 @@ UInt32 DatabaseReplicatedDDLWorker::getLogPointer() const
     return max_id.load();
 }
 
+UInt64 DatabaseReplicatedDDLWorker::getCurrentInitializationDurationMs() const
+{
+    return initializing.load(std::memory_order_acquire) ? initialization_duration_timer.elapsedMilliseconds() : 0;
+}
+
 }
diff --git a/src/Databases/DatabaseReplicatedWorker.h b/src/Databases/DatabaseReplicatedWorker.h
index aea3b71173d..3e5887be825 100644
--- a/src/Databases/DatabaseReplicatedWorker.h
+++ b/src/Databases/DatabaseReplicatedWorker.h
@@ -37,7 +37,7 @@ public:
 
     UInt32 getLogPointer() const;
 
-    UInt64 getCurrentInitializationDurationMs() const { return initialization_duration_timer ? initialization_duration_timer->elapsedMilliseconds() : 0; }
+    UInt64 getCurrentInitializationDurationMs() const;
 private:
     bool initializeMainThread() override;
     void initializeReplication();
@@ -59,7 +59,8 @@ private:
     /// It will remove "active" node when database is detached
     zkutil::EphemeralNodeHolderPtr active_node_holder;
 
-    std::optional<Stopwatch> initialization_duration_timer;
+    Stopwatch initialization_duration_timer;
+    std::atomic<bool> initializing = false;
 };
 
 }

From 342efff2edd9bcbecd28f5196fb11dcabdb629e1 Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Tue, 23 Jul 2024 12:49:50 +0200
Subject: [PATCH 0637/1488] Bump


From 86ff4e8b73d99daa5239104a0223271411949b4a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Tue, 23 Jul 2024 12:39:24 +0200
Subject: [PATCH 0638/1488] groupArrayIntersect: Fix serialization bug

---
 .../AggregateFunctionGroupArrayIntersect.cpp  |  4 +-
 ...roupArrayIntersect_serialization.reference | 12 ++++++
 ...3208_groupArrayIntersect_serialization.sql | 41 +++++++++++++++++++
 3 files changed, 55 insertions(+), 2 deletions(-)
 create mode 100644 tests/queries/0_stateless/03208_groupArrayIntersect_serialization.reference
 create mode 100644 tests/queries/0_stateless/03208_groupArrayIntersect_serialization.sql

diff --git a/src/AggregateFunctions/AggregateFunctionGroupArrayIntersect.cpp b/src/AggregateFunctions/AggregateFunctionGroupArrayIntersect.cpp
index 1529cd5899a..38f2fcb9fb9 100644
--- a/src/AggregateFunctions/AggregateFunctionGroupArrayIntersect.cpp
+++ b/src/AggregateFunctions/AggregateFunctionGroupArrayIntersect.cpp
@@ -47,7 +47,7 @@ struct AggregateFunctionGroupArrayIntersectData
 };
 
 
-/// Puts all values to the hash set. Returns an array of unique values. Implemented for numeric types.
+/// Puts all values to the hash set. Returns an array of unique values present in all inputs. Implemented for numeric types.
 template <typename T>
 class AggregateFunctionGroupArrayIntersect
     : public IAggregateFunctionDataHelper<AggregateFunctionGroupArrayIntersectData<T>, AggregateFunctionGroupArrayIntersect<T>>
@@ -154,7 +154,7 @@ public:
         set.reserve(size);
         for (size_t i = 0; i < size; ++i)
         {
-            int key;
+            T key;
             readIntBinary(key, buf);
             set.insert(key);
         }
diff --git a/tests/queries/0_stateless/03208_groupArrayIntersect_serialization.reference b/tests/queries/0_stateless/03208_groupArrayIntersect_serialization.reference
new file mode 100644
index 00000000000..c3b6e0cd5b7
--- /dev/null
+++ b/tests/queries/0_stateless/03208_groupArrayIntersect_serialization.reference
@@ -0,0 +1,12 @@
+010101	AggregateFunction(groupArrayIntersect, Array(UInt8))
+[1]
+1	[2,4,6,8,10]
+2	[2,4,6,8,10]
+3	[2,4,6,8,10]
+5	[2,6,10]
+6	[10]
+7	[]
+a	[(['2','4','6','8','10'])]
+b	[(['2','4','6','8','10'])]
+c	[(['2','4','6','8','10'])]
+d	[]
diff --git a/tests/queries/0_stateless/03208_groupArrayIntersect_serialization.sql b/tests/queries/0_stateless/03208_groupArrayIntersect_serialization.sql
new file mode 100644
index 00000000000..e05f78a4051
--- /dev/null
+++ b/tests/queries/0_stateless/03208_groupArrayIntersect_serialization.sql
@@ -0,0 +1,41 @@
+SELECT hex(groupArrayIntersectState([1]) AS a), toTypeName(a);
+SELECT finalizeAggregation(CAST(unhex('010101'), 'AggregateFunction(groupArrayIntersect, Array(UInt8))'));
+
+DROP TABLE IF EXISTS grouparray;
+CREATE TABLE grouparray
+(
+    `v` AggregateFunction(groupArrayIntersect, Array(UInt8))
+)
+ENGINE = Log;
+
+INSERT INTO grouparray Select groupArrayIntersectState([2, 4, 6, 8, 10]::Array(UInt8));
+SELECT '1', arraySort(groupArrayIntersectMerge(v)) FROM grouparray;
+INSERT INTO grouparray Select groupArrayIntersectState([2, 4, 6, 8, 10]::Array(UInt8));
+SELECT '2', arraySort(groupArrayIntersectMerge(v)) FROM grouparray;
+INSERT INTO grouparray Select groupArrayIntersectState([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]::Array(UInt8));
+SELECT '3', arraySort(groupArrayIntersectMerge(v)) FROM grouparray;
+INSERT INTO grouparray Select groupArrayIntersectState([2, 6, 10]::Array(UInt8));
+SELECT '5', arraySort(groupArrayIntersectMerge(v)) FROM grouparray;
+INSERT INTO grouparray Select groupArrayIntersectState([10]::Array(UInt8));
+SELECT '6', arraySort(groupArrayIntersectMerge(v)) FROM grouparray;
+INSERT INTO grouparray Select groupArrayIntersectState([]::Array(UInt8));
+SELECT '7', arraySort(groupArrayIntersectMerge(v)) FROM grouparray;
+
+DROP TABLE IF EXISTS grouparray;
+
+
+DROP TABLE IF EXISTS grouparray_string;
+CREATE TABLE grouparray_string
+(
+    `v` AggregateFunction(groupArrayIntersect, Array(Tuple(Array(String))))
+)
+ENGINE = Log;
+
+INSERT INTO grouparray_string Select groupArrayIntersectState([tuple(['2', '4', '6', '8', '10'])]);
+SELECT 'a', arraySort(groupArrayIntersectMerge(v)) FROM grouparray_string;
+INSERT INTO grouparray_string Select groupArrayIntersectState([tuple(['2', '4', '6', '8', '10']), tuple(['2', '4', '6', '8', '10'])]);
+SELECT 'b', arraySort(groupArrayIntersectMerge(v)) FROM grouparray_string;
+INSERT INTO grouparray_string Select groupArrayIntersectState([tuple(['2', '4', '6', '8', '10']), tuple(['2', '4', '6', '8', '10', '14'])]);
+SELECT 'c', arraySort(groupArrayIntersectMerge(v)) FROM grouparray_string;
+INSERT INTO grouparray_string Select groupArrayIntersectState([tuple(['2', '4', '6', '8', '10', '20']), tuple(['2', '4', '6', '8', '10', '14'])]);
+SELECT 'd', arraySort(groupArrayIntersectMerge(v)) FROM grouparray_string;

From 0256dba672bd23302b845f7d78f4663e3c633140 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Tue, 23 Jul 2024 13:20:41 +0200
Subject: [PATCH 0639/1488] Make 02987_group_array_intersect smaller

---
 .../0_stateless/02987_group_array_intersect.reference  |  8 ++++----
 .../0_stateless/02987_group_array_intersect.sql        | 10 +++++-----
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/tests/queries/0_stateless/02987_group_array_intersect.reference b/tests/queries/0_stateless/02987_group_array_intersect.reference
index 7ec64a889f5..ec4d07742af 100644
--- a/tests/queries/0_stateless/02987_group_array_intersect.reference
+++ b/tests/queries/0_stateless/02987_group_array_intersect.reference
@@ -8,12 +8,12 @@
 [1,4,5]
 []
 []
-1000000
-999999
+100000
+99999
 [9]
 ['a','c']
-1000000
-999999
+50000
+49999
 ['1']
 []
 ['2023-01-01 00:00:00']
diff --git a/tests/queries/0_stateless/02987_group_array_intersect.sql b/tests/queries/0_stateless/02987_group_array_intersect.sql
index 321e860b0a8..15acd0ca900 100644
--- a/tests/queries/0_stateless/02987_group_array_intersect.sql
+++ b/tests/queries/0_stateless/02987_group_array_intersect.sql
@@ -39,15 +39,15 @@ DROP TABLE test_numbers;
 
 DROP TABLE IF EXISTS test_big_numbers_sep;
 CREATE TABLE test_big_numbers_sep (a Array(Int64)) engine=MergeTree ORDER BY a;
-INSERT INTO test_big_numbers_sep SELECT array(number) FROM numbers_mt(1000000);
+INSERT INTO test_big_numbers_sep SELECT array(number) FROM numbers_mt(100000);
 SELECT groupArrayIntersect(*) FROM test_big_numbers_sep;
 DROP TABLE test_big_numbers_sep;
 
 DROP TABLE IF EXISTS test_big_numbers;
 CREATE TABLE test_big_numbers (a Array(Int64)) engine=MergeTree ORDER BY a;
-INSERT INTO test_big_numbers SELECT range(1000000);
+INSERT INTO test_big_numbers SELECT range(100000);
 SELECT length(groupArrayIntersect(*)) FROM test_big_numbers;
-INSERT INTO test_big_numbers SELECT range(999999);
+INSERT INTO test_big_numbers SELECT range(99999);
 SELECT length(groupArrayIntersect(*)) FROM test_big_numbers;
 INSERT INTO test_big_numbers VALUES ([9]);
 SELECT groupArrayIntersect(*) FROM test_big_numbers;
@@ -63,9 +63,9 @@ DROP TABLE test_string;
 
 DROP TABLE IF EXISTS test_big_string;
 CREATE TABLE test_big_string (a Array(String)) engine=MergeTree ORDER BY a;
-INSERT INTO test_big_string SELECT groupArray(toString(number)) FROM numbers_mt(1000000);
+INSERT INTO test_big_string SELECT groupArray(toString(number)) FROM numbers_mt(50000);
 SELECT length(groupArrayIntersect(*)) FROM test_big_string;
-INSERT INTO test_big_string SELECT groupArray(toString(number)) FROM numbers_mt(999999);
+INSERT INTO test_big_string SELECT groupArray(toString(number)) FROM numbers_mt(49999);
 SELECT length(groupArrayIntersect(*)) FROM test_big_string;
 INSERT INTO test_big_string VALUES (['1']);
 SELECT groupArrayIntersect(*) FROM test_big_string;

From 53c7bf680b401dc4948353dec2d94a428327b1f9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Tue, 23 Jul 2024 11:46:48 +0000
Subject: [PATCH 0640/1488] Update autogenerated version to 24.8.1.1 and
 contributors

---
 cmake/autogenerated_versions.txt              | 10 +++----
 .../StorageSystemContributors.generated.cpp   | 27 +++++++++++++++++++
 2 files changed, 32 insertions(+), 5 deletions(-)

diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt
index bb776fa9506..d69646d3694 100644
--- a/cmake/autogenerated_versions.txt
+++ b/cmake/autogenerated_versions.txt
@@ -2,11 +2,11 @@
 
 # NOTE: VERSION_REVISION has nothing common with DBMS_TCP_PROTOCOL_VERSION,
 # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes.
-SET(VERSION_REVISION 54488)
+SET(VERSION_REVISION 54489)
 SET(VERSION_MAJOR 24)
-SET(VERSION_MINOR 7)
+SET(VERSION_MINOR 8)
 SET(VERSION_PATCH 1)
-SET(VERSION_GITHASH aa023477a9265e403982fca5ee29a714db5133d9)
-SET(VERSION_DESCRIBE v24.7.1.1-testing)
-SET(VERSION_STRING 24.7.1.1)
+SET(VERSION_GITHASH 3f8b27d7accd2b5ec4afe7d0dd459115323304af)
+SET(VERSION_DESCRIBE v24.8.1.1-testing)
+SET(VERSION_STRING 24.8.1.1)
 # end of autochange
diff --git a/src/Storages/System/StorageSystemContributors.generated.cpp b/src/Storages/System/StorageSystemContributors.generated.cpp
index 9201eef185f..35b9c0008c6 100644
--- a/src/Storages/System/StorageSystemContributors.generated.cpp
+++ b/src/Storages/System/StorageSystemContributors.generated.cpp
@@ -1,5 +1,6 @@
 // autogenerated by tests/ci/version_helper.py
 const char * auto_contributors[] {
+    "0x01f",
     "0xflotus",
     "13DaGGeR",
     "1lann",
@@ -167,6 +168,7 @@ const char * auto_contributors[] {
     "AnneClickHouse",
     "Anselmo D. Adams",
     "Anthony N. Simon",
+    "AntiTopQuark",
     "Anton Ivashkin",
     "Anton Kobzev",
     "Anton Kozlov",
@@ -299,6 +301,7 @@ const char * auto_contributors[] {
     "Dan Wu",
     "DanRoscigno",
     "Dani Pozo",
+    "Daniel Anugerah",
     "Daniel Bershatsky",
     "Daniel Byta",
     "Daniel Dao",
@@ -370,6 +373,7 @@ const char * auto_contributors[] {
     "Elena",
     "Elena Baskakova",
     "Elena Torró",
+    "Elena Torró Martínez",
     "Elghazal Ahmed",
     "Eliot Hautefeuille",
     "Elizaveta Mironyuk",
@@ -415,6 +419,7 @@ const char * auto_contributors[] {
     "FgoDt",
     "Filatenkov Artur",
     "Filipe Caixeta",
+    "Filipp Bakanov",
     "Filipp Ozinov",
     "Filippov Denis",
     "Fille",
@@ -451,6 +456,7 @@ const char * auto_contributors[] {
     "Gleb Novikov",
     "Gleb-Tretyakov",
     "GoGoWen2021",
+    "Gosha Letov",
     "Gregory",
     "Grigorii Sokolik",
     "Grigory",
@@ -461,6 +467,7 @@ const char * auto_contributors[] {
     "Guillaume Tassery",
     "Guo Wangyang",
     "Guo Wei (William)",
+    "Guspan Tanadi",
     "Haavard Kvaalen",
     "Habibullah Oladepo",
     "HaiBo Li",
@@ -474,6 +481,7 @@ const char * auto_contributors[] {
     "HarryLeeIBM",
     "Hasitha Kanchana",
     "Hasnat",
+    "Haydn",
     "Heena Bansal",
     "HeenaBansal2009",
     "Hendrik M",
@@ -606,6 +614,7 @@ const char * auto_contributors[] {
     "Kevin Chiang",
     "Kevin Michel",
     "Kevin Mingtarja",
+    "Kevin Song",
     "Kevin Zhang",
     "KevinyhZou",
     "KinderRiven",
@@ -661,6 +670,7 @@ const char * auto_contributors[] {
     "Lewinma",
     "Li Shuai",
     "Li Yin",
+    "Linh Giang",
     "Lino Uruñuela",
     "Lirikl",
     "Liu Cong",
@@ -690,6 +700,7 @@ const char * auto_contributors[] {
     "Maksim Alekseev",
     "Maksim Buren",
     "Maksim Fedotov",
+    "Maksim Galkin",
     "Maksim Kita",
     "Maksym Sobolyev",
     "Mal Curtis",
@@ -724,6 +735,7 @@ const char * auto_contributors[] {
     "Max Akhmedov",
     "Max Bruce",
     "Max K",
+    "Max K.",
     "Max Kainov",
     "Max Vetrov",
     "MaxTheHuman",
@@ -811,6 +823,7 @@ const char * auto_contributors[] {
     "Nataly Merezhuk",
     "Natalya Chizhonkova",
     "Natasha Murashkina",
+    "Nathan Clevenger",
     "NeZeD [Mac Pro]",
     "Neeke Gao",
     "Neng Liu",
@@ -946,6 +959,7 @@ const char * auto_contributors[] {
     "Robert Coelho",
     "Robert Hodges",
     "Robert Schulze",
+    "Rodolphe Dugé de Bernonville",
     "RogerYK",
     "Rohit Agarwal",
     "Romain Neutron",
@@ -1107,6 +1121,7 @@ const char * auto_contributors[] {
     "Timur Solodovnikov",
     "TiunovNN",
     "Tobias Adamson",
+    "Tobias Florek",
     "Tobias Lins",
     "Tom Bombadil",
     "Tom Risse",
@@ -1231,11 +1246,13 @@ const char * auto_contributors[] {
     "Yingchun Lai",
     "Yingfan Chen",
     "Yinzheng-Sun",
+    "Yinzuo Jiang",
     "Yiğit Konur",
     "Yohann Jardin",
     "Yong Wang",
     "Yong-Hao Zou",
     "Youenn Lebras",
+    "Your Name",
     "Yu, Peng",
     "Yuko Takagi",
     "Yuntao Wu",
@@ -1250,6 +1267,7 @@ const char * auto_contributors[] {
     "Yury Stankevich",
     "Yusuke Tanaka",
     "Zach Naimon",
+    "Zawa-II",
     "Zheng Miao",
     "ZhiHong Zhang",
     "ZhiYong Wang",
@@ -1380,6 +1398,7 @@ const char * auto_contributors[] {
     "conicliu",
     "copperybean",
     "coraxster",
+    "cw5121",
     "cwkyaoyao",
     "d.v.semenov",
     "dalei2019",
@@ -1460,12 +1479,14 @@ const char * auto_contributors[] {
     "fuzzERot",
     "fyu",
     "g-arslan",
+    "gabrielmcg44",
     "ggerogery",
     "giordyb",
     "glockbender",
     "glushkovds",
     "grantovsky",
     "gulige",
+    "gun9nir",
     "guoleiyi",
     "guomaolin",
     "guov100",
@@ -1527,6 +1548,7 @@ const char * auto_contributors[] {
     "jferroal",
     "jiahui-97",
     "jianmei zhang",
+    "jiaosenvip",
     "jinjunzh",
     "jiyoungyoooo",
     "jktng",
@@ -1541,6 +1563,7 @@ const char * auto_contributors[] {
     "jun won",
     "jus1096",
     "justindeguzman",
+    "jwoodhead",
     "jyz0309",
     "karnevil13",
     "kashwy",
@@ -1633,10 +1656,12 @@ const char * auto_contributors[] {
     "mateng0915",
     "mateng915",
     "mauidude",
+    "max-vostrikov",
     "maxim",
     "maxim-babenko",
     "maxkuzn",
     "maxulan",
+    "maxvostrikov",
     "mayamika",
     "mehanizm",
     "melin",
@@ -1677,6 +1702,7 @@ const char * auto_contributors[] {
     "nathanbegbie",
     "nauta",
     "nautaa",
+    "nauu",
     "ndchikin",
     "nellicus",
     "nemonlou",
@@ -1975,6 +2001,7 @@ const char * auto_contributors[] {
     "张健",
     "张风啸",
     "徐炘",
+    "忒休斯~Theseus",
     "曲正鹏",
     "木木夕120",
     "未来星___费",

From ba3979275b53a44d9a9c097f047188b2d44ffa8b Mon Sep 17 00:00:00 2001
From: Alexander Gololobov <davenger@clickhouse.com>
Date: Tue, 23 Jul 2024 13:54:18 +0200
Subject: [PATCH 0641/1488] Test that big set is shared between parts by
 checking read_rows from system.part_log

---
 ...en_multiple_mutations_tasks_long.reference |  5 +++-
 ..._between_multiple_mutations_tasks_long.sql | 26 ++++++++++++++++---
 ...sets_between_mutation_tasks_long.reference |  9 ++++++-
 ...e_big_sets_between_mutation_tasks_long.sql | 19 +++++++++++++-
 4 files changed, 53 insertions(+), 6 deletions(-)

diff --git a/tests/queries/0_stateless/02581_share_big_sets_between_multiple_mutations_tasks_long.reference b/tests/queries/0_stateless/02581_share_big_sets_between_multiple_mutations_tasks_long.reference
index 9cb32105006..d3e7e113ffe 100644
--- a/tests/queries/0_stateless/02581_share_big_sets_between_multiple_mutations_tasks_long.reference
+++ b/tests/queries/0_stateless/02581_share_big_sets_between_multiple_mutations_tasks_long.reference
@@ -3,8 +3,11 @@ all_1_1_0
 all_2_2_0
 all_3_3_0
 all_4_4_0
+40000
 5000	all_1_1_0_9
 5000	all_2_2_0_9
 5000	all_3_3_0_9
 5000	all_4_4_0_9
-Ok
+mutation_version	has_parts_for_which_set_was_built	has_parts_that_shared_set
+8	1	1
+9	1	1
diff --git a/tests/queries/0_stateless/02581_share_big_sets_between_multiple_mutations_tasks_long.sql b/tests/queries/0_stateless/02581_share_big_sets_between_multiple_mutations_tasks_long.sql
index 631fd9cb2cc..40f7800fee1 100644
--- a/tests/queries/0_stateless/02581_share_big_sets_between_multiple_mutations_tasks_long.sql
+++ b/tests/queries/0_stateless/02581_share_big_sets_between_multiple_mutations_tasks_long.sql
@@ -23,10 +23,30 @@ ALTER TABLE 02581_trips UPDATE description='6' WHERE id IN (SELECT (number*10 +
 ALTER TABLE 02581_trips DELETE                 WHERE id IN (SELECT (number*10 + 7)::UInt32 FROM numbers(10000000)) SETTINGS mutations_sync=0;
 ALTER TABLE 02581_trips UPDATE description='8' WHERE id IN (SELECT (number*10 + 8)::UInt32 FROM numbers(10000000)) SETTINGS mutations_sync=0;
 SYSTEM START MERGES 02581_trips;
-DELETE FROM 02581_trips                        WHERE id IN (SELECT (number*10 + 9)::UInt32 FROM numbers(10000000));
-SELECT count(), _part from 02581_trips WHERE description = '' GROUP BY _part ORDER BY _part;
+
+-- Wait for mutations to finish
+SELECT count() FROM 02581_trips SETTINGS select_sequential_consistency = 1;
+
+DELETE FROM 02581_trips                        WHERE id IN (SELECT (number*10 + 9)::UInt32 FROM numbers(10000000)) SETTINGS lightweight_deletes_sync = 2;
+SELECT count(), _part from 02581_trips WHERE description = '' GROUP BY _part ORDER BY _part SETTINGS select_sequential_consistency=1;
 
 SYSTEM FLUSH LOGS;
-SELECT DISTINCT peak_memory_usage < 2000000000 ? 'Ok' : toString(tuple(*)) FROM system.part_log WHERE database = currentDatabase() AND event_date >= yesterday() AND table = '02581_trips' AND event_type = 'MutatePart';
+
+-- Check that in every mutation there were parts where selected rows count then the size of big sets which will mean that sets were shared
+-- Also check that there was at least one part that read more rows then the size of set which will mean that the 
+WITH 10000000 AS rows_in_set
+SELECT
+    mutation_version,
+    countIf(read_rows >= rows_in_set) >= 1 as has_parts_for_which_set_was_built,
+    countIf(read_rows <= rows_in_set) >= 1 as has_parts_that_shared_set
+FROM
+(
+    SELECT
+        CAST(splitByChar('_', part_name)[5], 'UInt64') AS mutation_version,
+        read_rows
+    FROM system.part_log
+    WHERE database = currentDatabase() and (event_date >= yesterday()) AND (`table` = '02581_trips') AND (event_type = 'MutatePart')
+)
+GROUP BY mutation_version ORDER BY mutation_version FORMAT TSVWithNames;
 
 DROP TABLE 02581_trips;
diff --git a/tests/queries/0_stateless/02581_share_big_sets_between_mutation_tasks_long.reference b/tests/queries/0_stateless/02581_share_big_sets_between_mutation_tasks_long.reference
index d21598bc12e..eecd768eb5e 100644
--- a/tests/queries/0_stateless/02581_share_big_sets_between_mutation_tasks_long.reference
+++ b/tests/queries/0_stateless/02581_share_big_sets_between_mutation_tasks_long.reference
@@ -10,4 +10,11 @@ all_4_4_0
 20000
 16000
 12000
-Ok
+mutation_version	has_parts_for_which_set_was_built	has_parts_that_shared_set
+5	1	1
+6	1	1
+7	1	1
+8	1	1
+9	1	1
+10	1	1
+11	1	1
diff --git a/tests/queries/0_stateless/02581_share_big_sets_between_mutation_tasks_long.sql b/tests/queries/0_stateless/02581_share_big_sets_between_mutation_tasks_long.sql
index 062f22357e8..603c7cb7db0 100644
--- a/tests/queries/0_stateless/02581_share_big_sets_between_mutation_tasks_long.sql
+++ b/tests/queries/0_stateless/02581_share_big_sets_between_mutation_tasks_long.sql
@@ -29,6 +29,8 @@ ALTER TABLE 02581_trips UPDATE description='b' WHERE id::UInt64 IN (SELECT (numb
 SELECT count() from 02581_trips WHERE description = '';
 
 -- Run mutation with non-PK `id2` IN big subquery
+--SELECT count(), _part FROM 02581_trips WHERE id2 IN (SELECT (number*10 + 3)::UInt32 FROM numbers(10000000)) GROUP BY _part ORDER BY _part;
+--EXPLAIN SELECT (), _part FROM 02581_trips WHERE id2 IN (SELECT (number*10 + 3)::UInt32 FROM numbers(10000000));
 ALTER TABLE 02581_trips UPDATE description='c' WHERE id2 IN (SELECT (number*10 + 3)::UInt32 FROM numbers(10000000)) SETTINGS mutations_sync=2;
 SELECT count() from 02581_trips WHERE description = '';
 
@@ -57,6 +59,21 @@ SETTINGS mutations_sync=2;
 SELECT count() from 02581_trips WHERE description = '';
 
 SYSTEM FLUSH LOGS;
-SELECT DISTINCT peak_memory_usage < 2000000000 ? 'Ok' : toString(tuple(*)) FROM system.part_log WHERE database = currentDatabase() AND event_date >= yesterday() AND table = '02581_trips' AND event_type = 'MutatePart';
+-- Check that in every mutation there were parts where selected rows count then the size of big sets which will mean that sets were shared
+-- Also check that there was at least one part that read more rows then the size of set which will mean that the 
+WITH 10000000 AS rows_in_set
+SELECT
+    mutation_version,
+    countIf(read_rows >= rows_in_set) >= 1 as has_parts_for_which_set_was_built,
+    countIf(read_rows <= rows_in_set) >= 1 as has_parts_that_shared_set
+FROM
+(
+    SELECT
+        CAST(splitByChar('_', part_name)[5], 'UInt64') AS mutation_version,
+        read_rows
+    FROM system.part_log
+    WHERE database = currentDatabase() and (event_date >= yesterday()) AND (`table` = '02581_trips') AND (event_type = 'MutatePart')
+)
+GROUP BY mutation_version ORDER BY mutation_version FORMAT TSVWithNames;
 
 DROP TABLE 02581_trips;

From aaf603035e31874d6d5bcd024d0f4040715baa72 Mon Sep 17 00:00:00 2001
From: Michael Stetsyuk <mike.stetsyuk@gmail.com>
Date: Tue, 23 Jul 2024 13:35:37 +0100
Subject: [PATCH 0642/1488] check error type

---
 src/IO/S3/Client.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/IO/S3/Client.cpp b/src/IO/S3/Client.cpp
index 325c820f8bd..7196dfa9bdc 100644
--- a/src/IO/S3/Client.cpp
+++ b/src/IO/S3/Client.cpp
@@ -385,7 +385,7 @@ Model::HeadObjectOutcome Client::HeadObject(HeadObjectRequest & request) const
 
     request.overrideURI(std::move(*bucket_uri));
 
-    if (isClientForDisk())
+    if (isClientForDisk() && error.GetErrorType() == Aws::S3::S3Errors::NO_SUCH_KEY)
         CurrentMetrics::add(CurrentMetrics::S3DiskNoKeyErrors);
 
     return enrichErrorMessage(
@@ -410,7 +410,7 @@ Model::ListObjectsOutcome Client::ListObjects(ListObjectsRequest & request) cons
 Model::GetObjectOutcome Client::GetObject(GetObjectRequest & request) const
 {
     auto resp = doRequest(request, [this](const Model::GetObjectRequest & req) { return GetObject(req); });
-    if (!resp.IsSuccess() && isClientForDisk())
+    if (!resp.IsSuccess() && isClientForDisk() && resp.GetError().GetErrorType() == Aws::S3::S3Errors::NO_SUCH_KEY)
         CurrentMetrics::add(CurrentMetrics::S3DiskNoKeyErrors);
 
     return enrichErrorMessage(std::move(resp));

From ea61af961acd3d8fb9dd3ee2b48900e4f1ee7937 Mon Sep 17 00:00:00 2001
From: Nikita Taranov <nikita.taranov@clickhouse.com>
Date: Tue, 23 Jul 2024 00:14:19 +0100
Subject: [PATCH 0643/1488] impl

---
 src/Server/TCPHandler.cpp | 73 +++++++++++++++++++++++----------------
 1 file changed, 43 insertions(+), 30 deletions(-)

diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp
index 07366d7cc07..2a2e2e22538 100644
--- a/src/Server/TCPHandler.cpp
+++ b/src/Server/TCPHandler.cpp
@@ -1,48 +1,48 @@
-#include <Interpreters/AsynchronousInsertQueue.h>
-#include <Interpreters/Squashing.h>
-#include <Parsers/ASTInsertQuery.h>
 #include <algorithm>
 #include <exception>
 #include <memory>
 #include <mutex>
-#include <vector>
 #include <string_view>
-#include <Poco/Net/NetException.h>
-#include <Poco/Net/SocketAddress.h>
-#include <Poco/Util/LayeredConfiguration.h>
-#include <Common/CurrentThread.h>
-#include <Common/Stopwatch.h>
-#include <Common/NetException.h>
-#include <Common/setThreadName.h>
-#include <Common/OpenSSLHelpers.h>
-#include <IO/Progress.h>
+#include <vector>
+#include <Access/AccessControl.h>
+#include <Access/Credentials.h>
 #include <Compression/CompressedReadBuffer.h>
 #include <Compression/CompressedWriteBuffer.h>
-#include <IO/ReadBufferFromPocoSocket.h>
-#include <IO/WriteBufferFromPocoSocket.h>
-#include <IO/LimitReadBuffer.h>
-#include <IO/ReadHelpers.h>
-#include <IO/WriteHelpers.h>
+#include <Compression/CompressionFactory.h>
+#include <Core/ExternalTable.h>
+#include <Core/ServerSettings.h>
 #include <Formats/NativeReader.h>
 #include <Formats/NativeWriter.h>
-#include <Interpreters/executeQuery.h>
-#include <Interpreters/TablesStatus.h>
+#include <IO/LimitReadBuffer.h>
+#include <IO/Progress.h>
+#include <IO/ReadBufferFromPocoSocket.h>
+#include <IO/ReadHelpers.h>
+#include <IO/WriteBufferFromPocoSocket.h>
+#include <IO/WriteHelpers.h>
+#include <Interpreters/AsynchronousInsertQueue.h>
 #include <Interpreters/InternalTextLogsQueue.h>
 #include <Interpreters/OpenTelemetrySpanLog.h>
 #include <Interpreters/Session.h>
+#include <Interpreters/Squashing.h>
+#include <Interpreters/TablesStatus.h>
+#include <Interpreters/executeQuery.h>
+#include <Parsers/ASTInsertQuery.h>
 #include <Server/TCPServer.h>
-#include <Storages/StorageReplicatedMergeTree.h>
 #include <Storages/MergeTree/MergeTreeDataPartUUID.h>
 #include <Storages/ObjectStorage/StorageObjectStorageCluster.h>
-#include <Core/ExternalTable.h>
-#include <Core/ServerSettings.h>
-#include <Access/AccessControl.h>
-#include <Access/Credentials.h>
-#include <Compression/CompressionFactory.h>
-#include <Common/logger_useful.h>
+#include <Storages/StorageReplicatedMergeTree.h>
+#include <Poco/Net/NetException.h>
+#include <Poco/Net/SocketAddress.h>
+#include <Poco/Util/LayeredConfiguration.h>
 #include <Common/CurrentMetrics.h>
+#include <Common/CurrentThread.h>
+#include <Common/NetException.h>
+#include <Common/OpenSSLHelpers.h>
+#include <Common/Stopwatch.h>
+#include <Common/logger_useful.h>
+#include <Common/scope_guard_safe.h>
+#include <Common/setThreadName.h>
 #include <Common/thread_local_rng.h>
-#include <fmt/format.h>
 
 #include <Processors/Executors/PullingAsyncPipelineExecutor.h>
 #include <Processors/Executors/PushingPipelineExecutor.h>
@@ -61,6 +61,8 @@
 
 #include <Common/config_version.h>
 
+#include <fmt/format.h>
+
 using namespace std::literals;
 using namespace DB;
 
@@ -1036,6 +1038,17 @@ void TCPHandler::processOrdinaryQuery()
         PullingAsyncPipelineExecutor executor(pipeline);
         CurrentMetrics::Increment query_thread_metric_increment{CurrentMetrics::QueryThread};
 
+        /// The following may happen:
+        /// * current thread is holding the lock
+        /// * because of the exception we unwind the stack and call the destructor of `executor`
+        /// * the destructor calls cancel() and waits for all query threads to finish
+        /// * at the same time one of the query threads is trying to acquire the lock, e.g. inside `merge_tree_read_task_callback`
+        /// * deadlock
+        SCOPE_EXIT({
+            if (out_lock.owns_lock())
+                out_lock.unlock();
+        });
+
         Block block;
         while (executor.pull(block, interactive_delay / 1000))
         {
@@ -1079,8 +1092,7 @@ void TCPHandler::processOrdinaryQuery()
         }
 
         /// This lock wasn't acquired before and we make .lock() call here
-        /// so everything under this line is covered even together
-        /// with sendProgress() out of the scope
+        /// so everything under this line is covered.
         out_lock.lock();
 
         /** If data has run out, we will send the profiling data and total values to
@@ -1107,6 +1119,7 @@ void TCPHandler::processOrdinaryQuery()
         last_sent_snapshots.clear();
     }
 
+    out_lock.lock();
     sendProgress();
 }
 

From 496c1fbf8413db3b3a3b410750adf3b5ecddabf3 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Tue, 23 Jul 2024 12:46:09 +0000
Subject: [PATCH 0644/1488] Un-flake test_runtime_configurable_cache_size

---
 ..._query_cache.xml => empty_query_cache.xml} |  2 +-
 .../test.py                                   | 60 +++++++++----------
 ...query_cache_asynchronous_metrics.reference |  2 +
 ...02494_query_cache_asynchronous_metrics.sql | 13 ++++
 4 files changed, 45 insertions(+), 32 deletions(-)
 rename tests/integration/test_runtime_configurable_cache_size/configs/{smaller_query_cache.xml => empty_query_cache.xml} (64%)
 create mode 100644 tests/queries/0_stateless/02494_query_cache_asynchronous_metrics.reference
 create mode 100644 tests/queries/0_stateless/02494_query_cache_asynchronous_metrics.sql

diff --git a/tests/integration/test_runtime_configurable_cache_size/configs/smaller_query_cache.xml b/tests/integration/test_runtime_configurable_cache_size/configs/empty_query_cache.xml
similarity index 64%
rename from tests/integration/test_runtime_configurable_cache_size/configs/smaller_query_cache.xml
rename to tests/integration/test_runtime_configurable_cache_size/configs/empty_query_cache.xml
index 6f2de0fa8f5..c4872a0ce41 100644
--- a/tests/integration/test_runtime_configurable_cache_size/configs/smaller_query_cache.xml
+++ b/tests/integration/test_runtime_configurable_cache_size/configs/empty_query_cache.xml
@@ -1,7 +1,7 @@
 <clickhouse>
 
     <query_cache>
-        <max_entries>1</max_entries>
+        <max_entries>0</max_entries>
     </query_cache>
 
 </clickhouse>
diff --git a/tests/integration/test_runtime_configurable_cache_size/test.py b/tests/integration/test_runtime_configurable_cache_size/test.py
index f761005f297..26676009e19 100644
--- a/tests/integration/test_runtime_configurable_cache_size/test.py
+++ b/tests/integration/test_runtime_configurable_cache_size/test.py
@@ -94,51 +94,49 @@ CONFIG_DIR = os.path.join(SCRIPT_DIR, "configs")
 
 
 def test_query_cache_size_is_runtime_configurable(start_cluster):
-    # the initial config specifies the maximum query cache size as 2, run 3 queries, expect 2 cache entries
     node.query("SYSTEM DROP QUERY CACHE")
+
+    # the initial config allows at most two query cache entries but we don't mind
     node.query("SELECT 1 SETTINGS use_query_cache = 1, query_cache_ttl = 1")
-    node.query("SELECT 2 SETTINGS use_query_cache = 1, query_cache_ttl = 1")
-    node.query("SELECT 3 SETTINGS use_query_cache = 1, query_cache_ttl = 1")
 
     time.sleep(2)
-    node.query("SYSTEM RELOAD ASYNCHRONOUS METRICS")
-    res = node.query(
-        "SELECT value FROM system.asynchronous_metrics WHERE metric = 'QueryCacheEntries'",
-    )
-    assert res == "2\n"
+    # at this point, the query cache contains one entry and it is stale
 
-    # switch to a config with a maximum query cache size of 1
+    res = node.query(
+        "SELECT count(*) FROM system.query_cache",
+    )
+    assert res == "1\n"
+
+    # switch to a config with a maximum query cache size of _0_
     node.copy_file_to_container(
-        os.path.join(CONFIG_DIR, "smaller_query_cache.xml"),
+        os.path.join(CONFIG_DIR, "empty_query_cache.xml"),
         "/etc/clickhouse-server/config.d/default.xml",
     )
 
     node.query("SYSTEM RELOAD CONFIG")
 
-    # check that eviction worked as expected
-    time.sleep(2)
-    node.query("SYSTEM RELOAD ASYNCHRONOUS METRICS")
     res = node.query(
-        "SELECT value FROM system.asynchronous_metrics WHERE metric = 'QueryCacheEntries'",
-    )
-    assert (
-        res == "2\n"
-    )  # "Why not 1?", you think. Reason is that QC uses the TTLCachePolicy that evicts lazily only upon insert.
-    # Not a real issue, can be changed later, at least there's a test now.
-
-    # Also, you may also wonder "why query_cache_ttl = 1"? Reason is that TTLCachePolicy only removes *stale* entries. With the default TTL
-    # (60 sec), no entries would be removed at all. Again: not a real issue, can be changed later and there's at least a test now.
-
-    # check that the new query cache maximum size is respected when more queries run
-    node.query("SELECT 4 SETTINGS use_query_cache = 1, query_cache_ttl = 1")
-    node.query("SELECT 5 SETTINGS use_query_cache = 1, query_cache_ttl = 1")
-
-    time.sleep(2)
-    node.query("SYSTEM RELOAD ASYNCHRONOUS METRICS")
-    res = node.query(
-        "SELECT value FROM system.asynchronous_metrics WHERE metric = 'QueryCacheEntries'",
+        "SELECT count(*) FROM system.query_cache",
     )
     assert res == "1\n"
+    # "Why not 0?", I hear you say. Reason is that QC uses the TTLCachePolicy that evicts lazily only upon insert.
+    # Not a real issue, can be changed later, at least there's a test now.
+
+    # The next SELECT will find a single stale entry which is one entry too much according to the new config.
+    # This triggers the eviction of all stale entries, in this case the 'SELECT 1' result.
+    # Then, it tries to insert the 'SELECT 2' result but it also cannot be added according to the config.
+    node.query("SELECT 2 SETTINGS use_query_cache = 1, query_cache_ttl = 1")
+    res = node.query(
+        "SELECT count(*) FROM system.query_cache",
+    )
+    assert res == "0\n"
+
+    # The new maximum cache size is respected when more queries run
+    node.query("SELECT 3 SETTINGS use_query_cache = 1, query_cache_ttl = 1")
+    res = node.query(
+        "SELECT count(*) FROM system.query_cache",
+    )
+    assert res == "0\n"
 
     # restore the original config
     node.copy_file_to_container(
diff --git a/tests/queries/0_stateless/02494_query_cache_asynchronous_metrics.reference b/tests/queries/0_stateless/02494_query_cache_asynchronous_metrics.reference
new file mode 100644
index 00000000000..6ed281c757a
--- /dev/null
+++ b/tests/queries/0_stateless/02494_query_cache_asynchronous_metrics.reference
@@ -0,0 +1,2 @@
+1
+1
diff --git a/tests/queries/0_stateless/02494_query_cache_asynchronous_metrics.sql b/tests/queries/0_stateless/02494_query_cache_asynchronous_metrics.sql
new file mode 100644
index 00000000000..d8de4facb38
--- /dev/null
+++ b/tests/queries/0_stateless/02494_query_cache_asynchronous_metrics.sql
@@ -0,0 +1,13 @@
+-- Tags: no-parallel
+-- Tag no-parallel: Messes with internal cache
+
+SYSTEM DROP QUERY CACHE;
+
+-- Create an entry in the query cache
+SELECT 1 SETTINGS use_query_cache = true;
+
+-- Asynchronous metrics must know about the entry
+SYSTEM RELOAD ASYNCHRONOUS METRICS;
+SELECT value FROM system.asynchronous_metrics WHERE metric = 'QueryCacheEntries';
+
+SYSTEM DROP QUERY CACHE;

From d74dc587d7a183225b7cf0846b85e8213dcb7fc0 Mon Sep 17 00:00:00 2001
From: Igor Nikonov <igor@clickhouse.com>
Date: Tue, 23 Jul 2024 13:06:58 +0000
Subject: [PATCH 0645/1488] Fix comment

---
 src/Processors/IProcessor.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Processors/IProcessor.h b/src/Processors/IProcessor.h
index 94e93595f4e..4fd00d5e164 100644
--- a/src/Processors/IProcessor.h
+++ b/src/Processors/IProcessor.h
@@ -234,7 +234,7 @@ public:
      * Example, scheduling tasks for remote workers (file descriptor in this case is a socket)
      * When the remote worker asks for the next task, doing it in onAsyncJobReady() we can provide it immediately.
      * Otherwise, the returning of the next task for the remote worker can be delayed by current work done in the pipeline
-     * i.e. processor->work(), which will create unnecessary latency in query processing by remote workers   Not Committed Yet
+     * (by other processors), which will create unnecessary latency in query processing by remote workers
      */
     virtual void onAsyncJobReady() {}
 

From 919005c4f70b044ecd9cc1bbce5dc5e276e11929 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Tue, 23 Jul 2024 13:09:34 +0000
Subject: [PATCH 0646/1488] Incorporate review feedback

---
 .../test_runtime_configurable_cache_size/test.py  | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/tests/integration/test_runtime_configurable_cache_size/test.py b/tests/integration/test_runtime_configurable_cache_size/test.py
index 26676009e19..beaf83ea754 100644
--- a/tests/integration/test_runtime_configurable_cache_size/test.py
+++ b/tests/integration/test_runtime_configurable_cache_size/test.py
@@ -96,11 +96,11 @@ CONFIG_DIR = os.path.join(SCRIPT_DIR, "configs")
 def test_query_cache_size_is_runtime_configurable(start_cluster):
     node.query("SYSTEM DROP QUERY CACHE")
 
-    # the initial config allows at most two query cache entries but we don't mind
+    # The initial config allows at most two query cache entries but we don't mind
     node.query("SELECT 1 SETTINGS use_query_cache = 1, query_cache_ttl = 1")
 
     time.sleep(2)
-    # at this point, the query cache contains one entry and it is stale
+    # At this point, the query cache contains one entry and it is stale
 
     res = node.query(
         "SELECT count(*) FROM system.query_cache",
@@ -138,8 +138,17 @@ def test_query_cache_size_is_runtime_configurable(start_cluster):
     )
     assert res == "0\n"
 
-    # restore the original config
+    # Restore the original config
     node.copy_file_to_container(
         os.path.join(CONFIG_DIR, "default.xml"),
         "/etc/clickhouse-server/config.d/default.xml",
     )
+
+    node.query("SYSTEM RELOAD CONFIG")
+
+    # It is possible to insert entries again
+    node.query("SELECT 4 SETTINGS use_query_cache = 1, query_cache_ttl = 1")
+    res = node.query(
+        "SELECT count(*) FROM system.query_cache",
+    )
+    assert res == "1\n"

From 17b306dee522bf40b6020df5e664fcd9cc66dacf Mon Sep 17 00:00:00 2001
From: Sema Checherinda <Sema.Checherinda@clickhouse.com>
Date: Tue, 23 Jul 2024 16:20:12 +0200
Subject: [PATCH 0647/1488] fix tests with timeouts

---
 src/Common/tests/gtest_resolve_pool.cpp | 29 +++++++++++++++----------
 1 file changed, 18 insertions(+), 11 deletions(-)

diff --git a/src/Common/tests/gtest_resolve_pool.cpp b/src/Common/tests/gtest_resolve_pool.cpp
index b760b9b1524..c443e961cc7 100644
--- a/src/Common/tests/gtest_resolve_pool.cpp
+++ b/src/Common/tests/gtest_resolve_pool.cpp
@@ -33,7 +33,7 @@ size_t toMilliseconds(auto duration)
    return std::chrono::duration_cast<std::chrono::milliseconds>(duration).count();
 }
 
-const auto epsilon = 500us;
+const auto epsilon = 1ms;
 
 class ResolvePoolMock : public DB::HostResolver
 {
@@ -358,53 +358,59 @@ void check_no_failed_address(size_t iteration, auto & resolver, auto & addresses
 
 TEST_F(ResolvePoolTest, BannedForConsiquenceFail)
 {
-    auto history = 5ms;
+    auto history = 10ms;
     auto resolver = make_resolver(toMilliseconds(history));
 
     auto failed_addr = resolver->resolve();
     ASSERT_TRUE(addresses.contains(*failed_addr));
 
-    auto start_at = now();
 
     failed_addr.setFail();
+    auto start_at = now();
+
     ASSERT_EQ(3, CurrentMetrics::get(metrics.active_count));
     ASSERT_EQ(1, CurrentMetrics::get(metrics.banned_count));
     check_no_failed_address(1, resolver, addresses, failed_addr, metrics, start_at + history - epsilon);
 
     sleep_until(start_at + history + epsilon);
-    start_at = now();
 
     resolver->update();
     ASSERT_EQ(3, CurrentMetrics::get(metrics.active_count));
     ASSERT_EQ(0, CurrentMetrics::get(metrics.banned_count));
 
     failed_addr.setFail();
+    start_at = now();
+
     check_no_failed_address(2, resolver, addresses, failed_addr, metrics, start_at + history - epsilon);
 
     sleep_until(start_at + history + epsilon);
-    start_at = now();
 
     resolver->update();
+
+    // too much time has passed
+    if (now() > start_at + 2*history - epsilon)
+        return;
+
     ASSERT_EQ(3, CurrentMetrics::get(metrics.active_count));
     ASSERT_EQ(1, CurrentMetrics::get(metrics.banned_count));
 
     // ip still banned adter history_ms + update, because it was his second consiquent fail
-    check_no_failed_address(2, resolver, addresses, failed_addr, metrics, start_at + history - epsilon);
+    check_no_failed_address(2, resolver, addresses, failed_addr, metrics, start_at + 2*history - epsilon);
 }
 
 TEST_F(ResolvePoolTest, NoAditionalBannForConcurrentFail)
 {
-    auto history = 5ms;
+    auto history = 10ms;
     auto resolver = make_resolver(toMilliseconds(history));
 
     auto failed_addr = resolver->resolve();
     ASSERT_TRUE(addresses.contains(*failed_addr));
 
-    auto start_at = now();
+    failed_addr.setFail();
+    failed_addr.setFail();
+    failed_addr.setFail();
 
-    failed_addr.setFail();
-    failed_addr.setFail();
-    failed_addr.setFail();
+    auto start_at = now();
 
     ASSERT_EQ(3, CurrentMetrics::get(metrics.active_count));
     ASSERT_EQ(1, CurrentMetrics::get(metrics.banned_count));
@@ -413,6 +419,7 @@ TEST_F(ResolvePoolTest, NoAditionalBannForConcurrentFail)
     sleep_until(start_at + history + epsilon);
 
     resolver->update();
+
     // ip is cleared after just 1 history_ms interval.
     ASSERT_EQ(3, CurrentMetrics::get(metrics.active_count));
     ASSERT_EQ(0, CurrentMetrics::get(metrics.banned_count));

From 094eac641b0ff72e35b735a36eb5d70b6b52f3f1 Mon Sep 17 00:00:00 2001
From: Nikita Taranov <nikita.taranov@clickhouse.com>
Date: Tue, 23 Jul 2024 15:25:23 +0100
Subject: [PATCH 0648/1488] impl

---
 .../03036_dynamic_read_subcolumns_1.reference | 19 +++++++++++++++++
 .../03036_dynamic_read_subcolumns_1.sh        | 21 +++++++++++++++++++
 .../03036_dynamic_read_subcolumns_2.reference | 19 +++++++++++++++++
 .../03036_dynamic_read_subcolumns_2.sh        | 21 +++++++++++++++++++
 .../03036_dynamic_read_subcolumns_3.reference | 19 +++++++++++++++++
 .../03036_dynamic_read_subcolumns_3.sh        | 21 +++++++++++++++++++
 6 files changed, 120 insertions(+)
 create mode 100644 tests/queries/0_stateless/03036_dynamic_read_subcolumns_1.reference
 create mode 100755 tests/queries/0_stateless/03036_dynamic_read_subcolumns_1.sh
 create mode 100644 tests/queries/0_stateless/03036_dynamic_read_subcolumns_2.reference
 create mode 100755 tests/queries/0_stateless/03036_dynamic_read_subcolumns_2.sh
 create mode 100644 tests/queries/0_stateless/03036_dynamic_read_subcolumns_3.reference
 create mode 100755 tests/queries/0_stateless/03036_dynamic_read_subcolumns_3.sh

diff --git a/tests/queries/0_stateless/03036_dynamic_read_subcolumns_1.reference b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_1.reference
new file mode 100644
index 00000000000..0d51ecfac3b
--- /dev/null
+++ b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_1.reference
@@ -0,0 +1,19 @@
+Memory
+test
+Array(Array(Dynamic))
+Array(Variant(String, UInt64))
+None
+String
+UInt64
+200000
+200000
+200000
+200000
+0
+0
+200000
+200000
+100000
+100000
+200000
+0
diff --git a/tests/queries/0_stateless/03036_dynamic_read_subcolumns_1.sh b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_1.sh
new file mode 100755
index 00000000000..aabba731816
--- /dev/null
+++ b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_1.sh
@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+# Tags: long
+
+CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# reset --log_comment
+CLICKHOUSE_LOG_COMMENT=
+# shellcheck source=../shell_config.sh
+. "$CUR_DIR"/../shell_config.sh
+
+
+# shellcheck source=./03036_dynamic_read_subcolumns.lib
+. "$CUR_DIR"/03036_dynamic_read_subcolumns.lib
+
+CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --use_variant_as_common_type=1 --allow_experimental_dynamic_type=1"
+
+$CH_CLIENT -q "drop table if exists test;"
+
+echo "Memory"
+$CH_CLIENT -q "create table test (id UInt64, d Dynamic) engine=Memory"
+test
+$CH_CLIENT -q "drop table test;"
diff --git a/tests/queries/0_stateless/03036_dynamic_read_subcolumns_2.reference b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_2.reference
new file mode 100644
index 00000000000..099b7574566
--- /dev/null
+++ b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_2.reference
@@ -0,0 +1,19 @@
+MergeTree compact
+test
+Array(Array(Dynamic))
+Array(Variant(String, UInt64))
+None
+String
+UInt64
+200000
+200000
+200000
+200000
+0
+0
+200000
+200000
+100000
+100000
+200000
+0
diff --git a/tests/queries/0_stateless/03036_dynamic_read_subcolumns_2.sh b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_2.sh
new file mode 100755
index 00000000000..872f4c20a98
--- /dev/null
+++ b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_2.sh
@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+# Tags: long
+
+CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# reset --log_comment
+CLICKHOUSE_LOG_COMMENT=
+# shellcheck source=../shell_config.sh
+. "$CUR_DIR"/../shell_config.sh
+
+
+# shellcheck source=./03036_dynamic_read_subcolumns.lib
+. "$CUR_DIR"/03036_dynamic_read_subcolumns.lib
+
+CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --use_variant_as_common_type=1 --allow_experimental_dynamic_type=1"
+
+$CH_CLIENT -q "drop table if exists test;"
+
+echo "MergeTree compact"
+$CH_CLIENT -q "create table test (id UInt64, d Dynamic) engine=MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000;"
+test
+$CH_CLIENT -q "drop table test;"
diff --git a/tests/queries/0_stateless/03036_dynamic_read_subcolumns_3.reference b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_3.reference
new file mode 100644
index 00000000000..35db4a22b4c
--- /dev/null
+++ b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_3.reference
@@ -0,0 +1,19 @@
+MergeTree wide
+test
+Array(Array(Dynamic))
+Array(Variant(String, UInt64))
+None
+String
+UInt64
+200000
+200000
+200000
+200000
+0
+0
+200000
+200000
+100000
+100000
+200000
+0
diff --git a/tests/queries/0_stateless/03036_dynamic_read_subcolumns_3.sh b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_3.sh
new file mode 100755
index 00000000000..96276c96add
--- /dev/null
+++ b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_3.sh
@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+# Tags: long
+
+CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# reset --log_comment
+CLICKHOUSE_LOG_COMMENT=
+# shellcheck source=../shell_config.sh
+. "$CUR_DIR"/../shell_config.sh
+
+
+# shellcheck source=./03036_dynamic_read_subcolumns.lib
+. "$CUR_DIR"/03036_dynamic_read_subcolumns.lib
+
+CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --use_variant_as_common_type=1 --allow_experimental_dynamic_type=1"
+
+$CH_CLIENT -q "drop table if exists test;"
+
+echo "MergeTree wide"
+$CH_CLIENT -q "create table test (id UInt64, d Dynamic) engine=MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1;"
+test
+$CH_CLIENT -q "drop table test;"

From a7baafc0221178099ef4d683a1e4f6d81af160f2 Mon Sep 17 00:00:00 2001
From: Nikita Taranov <nikita.taranov@clickhouse.com>
Date: Tue, 23 Jul 2024 15:25:52 +0100
Subject: [PATCH 0649/1488] remove old

---
 .../03036_dynamic_read_subcolumns.reference   | 57 -----------------
 .../03036_dynamic_read_subcolumns.sh          | 62 -------------------
 2 files changed, 119 deletions(-)
 delete mode 100644 tests/queries/0_stateless/03036_dynamic_read_subcolumns.reference
 delete mode 100755 tests/queries/0_stateless/03036_dynamic_read_subcolumns.sh

diff --git a/tests/queries/0_stateless/03036_dynamic_read_subcolumns.reference b/tests/queries/0_stateless/03036_dynamic_read_subcolumns.reference
deleted file mode 100644
index 36984bc8b9b..00000000000
--- a/tests/queries/0_stateless/03036_dynamic_read_subcolumns.reference
+++ /dev/null
@@ -1,57 +0,0 @@
-Memory
-test
-Array(Array(Dynamic))
-Array(Variant(String, UInt64))
-None
-String
-UInt64
-200000
-200000
-200000
-200000
-0
-0
-200000
-200000
-100000
-100000
-200000
-0
-MergeTree compact
-test
-Array(Array(Dynamic))
-Array(Variant(String, UInt64))
-None
-String
-UInt64
-200000
-200000
-200000
-200000
-0
-0
-200000
-200000
-100000
-100000
-200000
-0
-MergeTree wide
-test
-Array(Array(Dynamic))
-Array(Variant(String, UInt64))
-None
-String
-UInt64
-200000
-200000
-200000
-200000
-0
-0
-200000
-200000
-100000
-100000
-200000
-0
diff --git a/tests/queries/0_stateless/03036_dynamic_read_subcolumns.sh b/tests/queries/0_stateless/03036_dynamic_read_subcolumns.sh
deleted file mode 100755
index 65517061b99..00000000000
--- a/tests/queries/0_stateless/03036_dynamic_read_subcolumns.sh
+++ /dev/null
@@ -1,62 +0,0 @@
-#!/usr/bin/env bash
-# Tags: long
-
-CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# reset --log_comment
-CLICKHOUSE_LOG_COMMENT=
-# shellcheck source=../shell_config.sh
-. "$CUR_DIR"/../shell_config.sh
-
-CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --use_variant_as_common_type=1 --allow_experimental_dynamic_type=1"
-
-
-function test()
-{
-    echo "test"
-    $CH_CLIENT -q "insert into test select number, number from numbers(100000) settings min_insert_block_size_rows=50000"
-    $CH_CLIENT -q "insert into test select number, 'str_' || toString(number) from numbers(100000, 100000) settings min_insert_block_size_rows=50000"
-    $CH_CLIENT -q "insert into test select number, arrayMap(x -> multiIf(number % 9 == 0, NULL, number % 9 == 3, 'str_' || toString(number), number), range(number % 10 + 1)) from numbers(200000, 100000) settings min_insert_block_size_rows=50000"
-    $CH_CLIENT -q "insert into test select number, NULL from numbers(300000, 100000) settings min_insert_block_size_rows=50000"
-    $CH_CLIENT -q "insert into test select number, multiIf(number % 4 == 3, 'str_' || toString(number), number % 4 == 2, NULL, number % 4 == 1, number, arrayMap(x -> multiIf(number % 9 == 0, NULL, number % 9 == 3, 'str_' || toString(number), number), range(number % 10 + 1))) from numbers(400000, 400000) settings min_insert_block_size_rows=50000"
-    $CH_CLIENT -q "insert into test select number, [range((number % 10 + 1)::UInt64)]::Array(Array(Dynamic)) from numbers(100000, 100000) settings min_insert_block_size_rows=50000"
-
-    $CH_CLIENT -q "select distinct dynamicType(d) as type from test order by type"
-    $CH_CLIENT -q "select count() from test where dynamicType(d) == 'UInt64'"
-    $CH_CLIENT -q "select count() from test where d.UInt64 is not NULL"
-    $CH_CLIENT -q "select count() from test where dynamicType(d) == 'String'"
-    $CH_CLIENT -q "select count() from test where d.String is not NULL"
-    $CH_CLIENT -q "select count() from test where dynamicType(d) == 'Date'"
-    $CH_CLIENT -q "select count() from test where d.Date is not NULL"
-    $CH_CLIENT -q "select count() from test where dynamicType(d) == 'Array(Variant(String, UInt64))'"
-    $CH_CLIENT -q "select count() from test where not empty(d.\`Array(Variant(String, UInt64))\`)"
-    $CH_CLIENT -q "select count() from test where dynamicType(d) == 'Array(Array(Dynamic))'"
-    $CH_CLIENT -q "select count() from test where not empty(d.\`Array(Array(Dynamic))\`)"
-    $CH_CLIENT -q "select count() from test where d is NULL"
-    $CH_CLIENT -q "select count() from test where not empty(d.\`Tuple(a Array(Dynamic))\`.a.String)"
-
-    $CH_CLIENT -q "select d, d.UInt64, d.String, d.\`Array(Variant(String, UInt64))\` from test format Null"
-    $CH_CLIENT -q "select d.UInt64, d.String, d.\`Array(Variant(String, UInt64))\` from test format Null"
-    $CH_CLIENT -q "select d.Int8, d.Date, d.\`Array(String)\` from test format Null"
-    $CH_CLIENT -q "select d, d.UInt64, d.Date, d.\`Array(Variant(String, UInt64))\`, d.\`Array(Variant(String, UInt64))\`.size0, d.\`Array(Variant(String, UInt64))\`.UInt64 from test format Null"
-    $CH_CLIENT -q "select d.UInt64, d.Date, d.\`Array(Variant(String, UInt64))\`, d.\`Array(Variant(String, UInt64))\`.size0, d.\`Array(Variant(String, UInt64))\`.UInt64, d.\`Array(Variant(String, UInt64))\`.String from test format Null"
-    $CH_CLIENT -q "select d, d.\`Tuple(a UInt64, b String)\`.a, d.\`Array(Dynamic)\`.\`Variant(String, UInt64)\`.UInt64, d.\`Array(Variant(String, UInt64))\`.UInt64 from test format Null"
-    $CH_CLIENT -q "select d.\`Array(Dynamic)\`.\`Variant(String, UInt64)\`.UInt64, d.\`Array(Dynamic)\`.size0, d.\`Array(Variant(String, UInt64))\`.UInt64 from test format Null"
-    $CH_CLIENT -q "select d.\`Array(Array(Dynamic))\`.size1, d.\`Array(Array(Dynamic))\`.UInt64, d.\`Array(Array(Dynamic))\`.\`Map(String, Tuple(a UInt64))\`.values.a from test format Null"
-}
-
-$CH_CLIENT -q "drop table if exists test;"
-
-echo "Memory"
-$CH_CLIENT -q "create table test (id UInt64, d Dynamic) engine=Memory"
-test
-$CH_CLIENT -q "drop table test;"
-
-echo "MergeTree compact"
-$CH_CLIENT -q "create table test (id UInt64, d Dynamic) engine=MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000;"
-test
-$CH_CLIENT -q "drop table test;"
-
-echo "MergeTree wide"
-$CH_CLIENT -q "create table test (id UInt64, d Dynamic) engine=MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1;"
-test
-$CH_CLIENT -q "drop table test;"

From e671d4c55d520187f0fb2c5abb4ac301926d9651 Mon Sep 17 00:00:00 2001
From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com>
Date: Tue, 23 Jul 2024 14:43:49 +0000
Subject: [PATCH 0650/1488] more changes

---
 tests/integration/test_storage_s3/test.py | 137 ++++++++++++++--------
 1 file changed, 86 insertions(+), 51 deletions(-)

diff --git a/tests/integration/test_storage_s3/test.py b/tests/integration/test_storage_s3/test.py
index 5453ad5a796..d13605170ec 100644
--- a/tests/integration/test_storage_s3/test.py
+++ b/tests/integration/test_storage_s3/test.py
@@ -962,9 +962,10 @@ def test_storage_s3_put_uncompressed(started_cluster):
     [pytest.param("bin", "gzip", id="bin"), pytest.param("gz", "auto", id="gz")],
 )
 def test_storage_s3_put_gzip(started_cluster, extension, method):
+    id = uuid.uuid4()
     bucket = started_cluster.minio_bucket
     instance = started_cluster.instances["dummy"]
-    filename = f"test_put_gzip.{extension}"
+    filename = f"{id}/test_put_gzip.{extension}"
     name = f"test_put_gzip_{extension}"
     data = [
         "'Joseph Tomlinson',5",
@@ -1001,6 +1002,7 @@ def test_storage_s3_put_gzip(started_cluster, extension, method):
     f = gzip.GzipFile(fileobj=buf, mode="rb")
     uncompressed_content = f.read().decode()
     assert sum([int(i.split(",")[1]) for i in uncompressed_content.splitlines()]) == 708
+    run_query(instance, f"DROP TABLE {name}")
 
 
 def test_truncate_table(started_cluster):
@@ -1026,14 +1028,24 @@ def test_truncate_table(started_cluster):
             len(list(minio.list_objects(started_cluster.minio_bucket, "truncate/")))
             == 0
         ):
-            return
+            break
         timeout -= 1
         time.sleep(1)
     assert len(list(minio.list_objects(started_cluster.minio_bucket, "truncate/"))) == 0
-    assert instance.query("SELECT * FROM {}".format(name)) == ""
+    # FIXME: there was a bug in test and it was never checked.
+    # Currently read from truncated table fails with
+    # DB::Exception: Failed to get object info: No response body..
+    # HTTP response code: 404: while reading truncate: While executing S3Source
+    # assert instance.query("SELECT * FROM {}".format(name)) == ""
+    instance.query(f"DROP TABLE {name} SYNC")
+    assert (
+        instance.query(f"SELECT count() FROM system.tables where name='{name}'")
+        == "0\n"
+    )
 
 
 def test_predefined_connection_configuration(started_cluster):
+    id = uuid.uuid4()
     bucket = started_cluster.minio_bucket
     instance = started_cluster.instances[
         "dummy_without_named_collections"
@@ -1061,7 +1073,9 @@ def test_predefined_connection_configuration(started_cluster):
         user="user",
     )
 
-    instance.query(f"INSERT INTO {name} SELECT number FROM numbers(10)")
+    instance.query(
+        f"INSERT INTO {name} SELECT number FROM numbers(10) SETTINGS s3_truncate_on_insert=1"
+    )
     result = instance.query(f"SELECT * FROM {name}")
     assert result == instance.query("SELECT number FROM numbers(10)")
 
@@ -1075,9 +1089,11 @@ def test_predefined_connection_configuration(started_cluster):
         "To execute this query, it's necessary to have the grant NAMED COLLECTION ON no_collection"
         in error
     )
-    instance = started_cluster.instances["dummy"]  # has named collection access
-    error = instance.query_and_get_error("SELECT * FROM s3(no_collection)")
+    instance2 = started_cluster.instances["dummy"]  # has named collection access
+    error = instance2.query_and_get_error("SELECT * FROM s3(no_collection)")
     assert "There is no named collection `no_collection`" in error
+    instance.query("DROP USER user")
+    instance.query(f"DROP TABLE {name}")
 
 
 result = ""
@@ -1227,7 +1243,7 @@ def test_s3_schema_inference(started_cluster):
     instance = started_cluster.instances["dummy"]
 
     instance.query(
-        f"insert into table function s3(s3_native, structure='a Int32, b String', format='Native') select number, randomString(100) from numbers(5000000)"
+        f"insert into table function s3(s3_native, structure='a Int32, b String', format='Native') select number, randomString(100) from numbers(5000000) SETTINGS s3_truncate_on_insert=1"
     )
     result = instance.query(f"desc s3(s3_native, format='Native')")
     assert result == "a\tInt32\t\t\t\t\t\nb\tString\t\t\t\t\t\n"
@@ -1267,6 +1283,9 @@ def test_s3_schema_inference(started_cluster):
     result = instance.query(f"select count(*) from {table_function}")
     assert int(result) == 5000000
 
+    instance.query("drop table schema_inference")
+    instance.query("drop table schema_inference_2")
+
 
 def test_empty_file(started_cluster):
     bucket = started_cluster.minio_bucket
@@ -1302,6 +1321,7 @@ def test_overwrite(started_cluster):
 
     result = instance.query(f"select count() from test_overwrite")
     assert int(result) == 200
+    instance.query(f"drop table test_overwrite")
 
 
 def test_create_new_files_on_insert(started_cluster):
@@ -1343,6 +1363,7 @@ def test_create_new_files_on_insert(started_cluster):
 
     result = instance.query(f"select count() from test_multiple_inserts")
     assert int(result) == 60
+    instance.query("drop table test_multiple_inserts")
 
 
 def test_format_detection(started_cluster):
@@ -1350,7 +1371,9 @@ def test_format_detection(started_cluster):
     instance = started_cluster.instances["dummy"]
 
     instance.query(f"create table arrow_table_s3 (x UInt64) engine=S3(s3_arrow)")
-    instance.query(f"insert into arrow_table_s3 select 1")
+    instance.query(
+        f"insert into arrow_table_s3 select 1 settings s3_truncate_on_insert=1"
+    )
     result = instance.query(f"select * from s3(s3_arrow)")
     assert int(result) == 1
 
@@ -1365,7 +1388,9 @@ def test_format_detection(started_cluster):
     assert int(result) == 1
 
     instance.query(f"create table parquet_table_s3 (x UInt64) engine=S3(s3_parquet2)")
-    instance.query(f"insert into parquet_table_s3 select 1")
+    instance.query(
+        f"insert into parquet_table_s3 select 1 settings s3_truncate_on_insert=1"
+    )
     result = instance.query(f"select * from s3(s3_parquet2)")
     assert int(result) == 1
 
@@ -1378,64 +1403,67 @@ def test_format_detection(started_cluster):
         f"select * from s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test.parquet')"
     )
     assert int(result) == 1
+    instance.query(f"drop table arrow_table_s3")
+    instance.query(f"drop table parquet_table_s3")
 
 
 def test_schema_inference_from_globs(started_cluster):
+    id = uuid.uuid4()
     bucket = started_cluster.minio_bucket
     instance = started_cluster.instances["dummy"]
 
     instance.query(
-        f"insert into table function s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test1.jsoncompacteachrow', 'JSONCompactEachRow', 'x Nullable(UInt32)') select NULL"
+        f"insert into table function s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{id}/test1.jsoncompacteachrow', 'JSONCompactEachRow', 'x Nullable(UInt32)') select NULL"
     )
     instance.query(
-        f"insert into table function s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test2.jsoncompacteachrow', 'JSONCompactEachRow', 'x Nullable(UInt32)') select 0"
+        f"insert into table function s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{id}/test2.jsoncompacteachrow', 'JSONCompactEachRow', 'x Nullable(UInt32)') select 0"
     )
 
     url_filename = "test{1,2}.jsoncompacteachrow"
     result = instance.query(
-        f"desc url('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{url_filename}') settings input_format_json_infer_incomplete_types_as_strings=0"
+        f"desc url('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{id}/{url_filename}') settings input_format_json_infer_incomplete_types_as_strings=0"
     )
     assert result.strip() == "c1\tNullable(Int64)"
 
     result = instance.query(
-        f"select * from url('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{url_filename}') settings input_format_json_infer_incomplete_types_as_strings=0"
+        f"select * from url('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{id}/{url_filename}') settings input_format_json_infer_incomplete_types_as_strings=0"
     )
     assert sorted(result.split()) == ["0", "\\N"]
 
     result = instance.query(
-        f"desc s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test*.jsoncompacteachrow') settings input_format_json_infer_incomplete_types_as_strings=0"
+        f"desc s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{id}/test*.jsoncompacteachrow') settings input_format_json_infer_incomplete_types_as_strings=0"
     )
     assert result.strip() == "c1\tNullable(Int64)"
 
     result = instance.query(
-        f"select * from s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test*.jsoncompacteachrow') settings input_format_json_infer_incomplete_types_as_strings=0"
+        f"select * from s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{id}/test*.jsoncompacteachrow') settings input_format_json_infer_incomplete_types_as_strings=0"
     )
     assert sorted(result.split()) == ["0", "\\N"]
 
     instance.query(
-        f"insert into table function s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test3.jsoncompacteachrow', 'JSONCompactEachRow', 'x Nullable(UInt32)') select NULL"
+        f"insert into table function s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{id}/test3.jsoncompacteachrow', 'JSONCompactEachRow', 'x Nullable(UInt32)') select NULL"
     )
 
     url_filename = "test{1,3}.jsoncompacteachrow"
 
     result = instance.query_and_get_error(
-        f"desc s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{url_filename}') settings schema_inference_use_cache_for_s3=0, input_format_json_infer_incomplete_types_as_strings=0"
+        f"desc s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{id}/{url_filename}') settings schema_inference_use_cache_for_s3=0, input_format_json_infer_incomplete_types_as_strings=0"
     )
 
     assert "All attempts to extract table structure from files failed" in result
 
     result = instance.query_and_get_error(
-        f"desc url('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{url_filename}') settings schema_inference_use_cache_for_url=0, input_format_json_infer_incomplete_types_as_strings=0"
+        f"desc url('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{id}/{url_filename}') settings schema_inference_use_cache_for_url=0, input_format_json_infer_incomplete_types_as_strings=0"
     )
 
     assert "All attempts to extract table structure from files failed" in result
 
     instance.query(
-        f"insert into table function s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test0.jsoncompacteachrow', 'TSV', 'x String') select '[123;]'"
+        f"insert into table function s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{id}/test0.jsoncompacteachrow', 'TSV', 'x String') select '[123;]'"
     )
 
     result = instance.query_and_get_error(
-        f"desc s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test*.jsoncompacteachrow') settings schema_inference_use_cache_for_s3=0, input_format_json_infer_incomplete_types_as_strings=0"
+        f"desc s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{id}/test*.jsoncompacteachrow') settings schema_inference_use_cache_for_s3=0, input_format_json_infer_incomplete_types_as_strings=0"
     )
 
     assert "CANNOT_EXTRACT_TABLE_STRUCTURE" in result
@@ -1443,7 +1471,7 @@ def test_schema_inference_from_globs(started_cluster):
     url_filename = "test{0,1,2,3}.jsoncompacteachrow"
 
     result = instance.query_and_get_error(
-        f"desc url('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{url_filename}') settings schema_inference_use_cache_for_url=0, input_format_json_infer_incomplete_types_as_strings=0"
+        f"desc url('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{id}/{url_filename}') settings schema_inference_use_cache_for_url=0, input_format_json_infer_incomplete_types_as_strings=0"
     )
 
     assert "CANNOT_EXTRACT_TABLE_STRUCTURE" in result
@@ -1503,9 +1531,12 @@ def test_signatures(started_cluster):
     )
     assert "S3_ERROR" in error
 
+    instance.query(f"drop table test_signatures")
+
 
 def test_select_columns(started_cluster):
     bucket = started_cluster.minio_bucket
+    id = uuid.uuid4()
     instance = started_cluster.instances["dummy"]
     name = "test_table2"
     structure = "id UInt32, value1 Int32, value2 Int32"
@@ -1519,36 +1550,37 @@ def test_select_columns(started_cluster):
     instance.query(
         f"INSERT INTO {name} SELECT * FROM generateRandom('{structure}') LIMIT {limit} SETTINGS s3_truncate_on_insert=1"
     )
-    instance.query(f"SELECT value2 FROM {name}")
+    instance.query(f"SELECT value2, '{id}' FROM {name}")
 
     instance.query("SYSTEM FLUSH LOGS")
     result1 = instance.query(
-        f"SELECT ProfileEvents['ReadBufferFromS3Bytes'] FROM system.query_log WHERE type='QueryFinish' and query LIKE 'SELECT value2 FROM {name}'"
+        f"SELECT ProfileEvents['ReadBufferFromS3Bytes'] FROM system.query_log WHERE type='QueryFinish' and query LIKE 'SELECT value2, ''{id}'' FROM {name}'"
     )
 
-    instance.query(f"SELECT * FROM {name}")
+    instance.query(f"SELECT *, '{id}' FROM {name}")
     instance.query("SYSTEM FLUSH LOGS")
     result2 = instance.query(
-        f"SELECT ProfileEvents['ReadBufferFromS3Bytes'] FROM system.query_log WHERE type='QueryFinish' and query LIKE 'SELECT * FROM {name}'"
+        f"SELECT ProfileEvents['ReadBufferFromS3Bytes'] FROM system.query_log WHERE type='QueryFinish' and query LIKE 'SELECT *, ''{id}'' FROM {name}'"
     )
 
     assert round(int(result2) / int(result1)) == 3
 
 
 def test_insert_select_schema_inference(started_cluster):
+    id = uuid.uuid4()
     bucket = started_cluster.minio_bucket
     instance = started_cluster.instances["dummy"]
 
     instance.query(
-        f"insert into function s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test_insert_select.native') select toUInt64(1) as x"
+        f"insert into function s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{id}/test_insert_select.native') select toUInt64(1) as x"
     )
     result = instance.query(
-        f"desc s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test_insert_select.native')"
+        f"desc s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{id}/test_insert_select.native')"
     )
     assert result.strip() == "x\tUInt64"
 
     result = instance.query(
-        f"select * from s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test_insert_select.native')"
+        f"select * from s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{id}/test_insert_select.native')"
     )
     assert int(result) == 1
 
@@ -1558,7 +1590,7 @@ def test_parallel_reading_with_memory_limit(started_cluster):
     instance = started_cluster.instances["dummy"]
 
     instance.query(
-        f"insert into function s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test_memory_limit.native') select * from numbers(1000000)"
+        f"insert into function s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test_memory_limit.native') select * from numbers(1000000) SETTINGS s3_truncate_on_insert=1"
     )
 
     result = instance.query_and_get_error(
@@ -1579,7 +1611,7 @@ def test_wrong_format_usage(started_cluster):
     instance = started_cluster.instances["dummy"]
 
     instance.query(
-        f"insert into function s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test_wrong_format.native') select * from numbers(10e6)"
+        f"insert into function s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test_wrong_format.native') select * from numbers(10e6) SETTINGS s3_truncate_on_insert=1"
     )
     # size(test_wrong_format.native) = 10e6*8+16(header) ~= 76MiB
 
@@ -2102,11 +2134,11 @@ def test_read_subcolumns(started_cluster):
     instance = started_cluster.instances["dummy"]
 
     instance.query(
-        f"insert into function s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test_subcolumns.tsv', auto, 'a Tuple(b Tuple(c UInt32, d UInt32), e UInt32)') select  ((1, 2), 3)"
+        f"insert into function s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test_subcolumns.tsv', auto, 'a Tuple(b Tuple(c UInt32, d UInt32), e UInt32)') select  ((1, 2), 3) SETTINGS s3_truncate_on_insert=1"
     )
 
     instance.query(
-        f"insert into function s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test_subcolumns.jsonl', auto, 'a Tuple(b Tuple(c UInt32, d UInt32), e UInt32)') select  ((1, 2), 3)"
+        f"insert into function s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test_subcolumns.jsonl', auto, 'a Tuple(b Tuple(c UInt32, d UInt32), e UInt32)') select  ((1, 2), 3)  SETTINGS s3_truncate_on_insert=1"
     )
 
     res = instance.query(
@@ -2165,7 +2197,7 @@ def test_read_subcolumn_time(started_cluster):
     instance = started_cluster.instances["dummy"]
 
     instance.query(
-        f"insert into function s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test_subcolumn_time.tsv', auto, 'a UInt32') select  (42)"
+        f"insert into function s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test_subcolumn_time.tsv', auto, 'a UInt32') select  (42)  SETTINGS s3_truncate_on_insert=1"
     )
 
     res = instance.query(
@@ -2176,29 +2208,30 @@ def test_read_subcolumn_time(started_cluster):
 
 
 def test_filtering_by_file_or_path(started_cluster):
+    id = uuid.uuid4()
     bucket = started_cluster.minio_bucket
     instance = started_cluster.instances["dummy"]
 
     instance.query(
-        f"insert into function s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test_filter1.tsv', auto, 'x UInt64') select 1"
+        f"insert into function s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test_filter1.tsv', auto, 'x UInt64') select 1 SETTINGS s3_truncate_on_insert=1"
     )
 
     instance.query(
-        f"insert into function s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test_filter2.tsv', auto, 'x UInt64') select 2"
+        f"insert into function s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test_filter2.tsv', auto, 'x UInt64') select 2 SETTINGS s3_truncate_on_insert=1"
     )
 
     instance.query(
-        f"insert into function s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test_filter3.tsv', auto, 'x UInt64') select 3"
+        f"insert into function s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test_filter3.tsv', auto, 'x UInt64') select 3 SETTINGS s3_truncate_on_insert=1"
     )
 
     instance.query(
-        f"select count() from s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test_filter*.tsv') where _file = 'test_filter1.tsv'"
+        f"select count(), '{id}' from s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test_filter*.tsv') where _file = 'test_filter1.tsv'"
     )
 
     instance.query("SYSTEM FLUSH LOGS")
 
     result = instance.query(
-        f"SELECT ProfileEvents['EngineFileLikeReadFiles'] FROM system.query_log WHERE query like '%select%s3%test_filter%' AND type='QueryFinish'"
+        f"SELECT ProfileEvents['EngineFileLikeReadFiles'] FROM system.query_log WHERE query like '%{id}%' AND type='QueryFinish'"
     )
 
     assert int(result) == 1
@@ -2211,54 +2244,56 @@ def test_filtering_by_file_or_path(started_cluster):
 
 
 def test_union_schema_inference_mode(started_cluster):
+    id = uuid.uuid4()
     bucket = started_cluster.minio_bucket
     instance = started_cluster.instances["s3_non_default"]
+    file_name_prefix = f"test_union_schema_inference_{id}_"
 
     instance.query(
-        f"insert into function s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test_union_schema_inference1.jsonl') select 1 as a"
+        f"insert into function s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{file_name_prefix}1.jsonl') select 1 as a SETTINGS s3_truncate_on_insert=1"
     )
 
     instance.query(
-        f"insert into function s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test_union_schema_inference2.jsonl') select 2 as b"
+        f"insert into function s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{file_name_prefix}2.jsonl') select 2 as b SETTINGS s3_truncate_on_insert=1"
     )
 
     instance.query(
-        f"insert into function s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test_union_schema_inference3.jsonl') select 2 as c"
+        f"insert into function s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{file_name_prefix}3.jsonl') select 2 as c SETTINGS s3_truncate_on_insert=1"
     )
 
     instance.query(
-        f"insert into function s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test_union_schema_inference4.jsonl', TSV) select 'Error'"
+        f"insert into function s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{file_name_prefix}4.jsonl', TSV) select 'Error' SETTINGS s3_truncate_on_insert=1"
     )
 
     for engine in ["s3", "url"]:
         instance.query("system drop schema cache for s3")
 
         result = instance.query(
-            f"desc {engine}('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test_union_schema_inference{{1,2,3}}.jsonl') settings schema_inference_mode='union', describe_compact_output=1 format TSV"
+            f"desc {engine}('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{file_name_prefix}{{1,2,3}}.jsonl') settings schema_inference_mode='union', describe_compact_output=1 format TSV"
         )
         assert result == "a\tNullable(Int64)\nb\tNullable(Int64)\nc\tNullable(Int64)\n"
 
         result = instance.query(
-            "select schema_inference_mode, splitByChar('/', source)[-1] as file, schema from system.schema_inference_cache where source like '%test_union_schema_inference%' order by file format TSV"
+            f"select schema_inference_mode, splitByChar('/', source)[-1] as file, schema from system.schema_inference_cache where source like '%{file_name_prefix}%' order by file format TSV"
         )
         assert (
-            result == "UNION\ttest_union_schema_inference1.jsonl\ta Nullable(Int64)\n"
-            "UNION\ttest_union_schema_inference2.jsonl\tb Nullable(Int64)\n"
-            "UNION\ttest_union_schema_inference3.jsonl\tc Nullable(Int64)\n"
+            result == f"UNION\t{file_name_prefix}1.jsonl\ta Nullable(Int64)\n"
+            f"UNION\t{file_name_prefix}2.jsonl\tb Nullable(Int64)\n"
+            f"UNION\t{file_name_prefix}3.jsonl\tc Nullable(Int64)\n"
         )
         result = instance.query(
-            f"select * from {engine}('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test_union_schema_inference{{1,2,3}}.jsonl') order by tuple(*) settings schema_inference_mode='union', describe_compact_output=1 format TSV"
+            f"select * from {engine}('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{file_name_prefix}{{1,2,3}}.jsonl') order by tuple(*) settings schema_inference_mode='union', describe_compact_output=1 format TSV"
         )
         assert result == "1\t\\N\t\\N\n" "\\N\t2\t\\N\n" "\\N\t\\N\t2\n"
 
         instance.query(f"system drop schema cache for {engine}")
         result = instance.query(
-            f"desc {engine}('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test_union_schema_inference2.jsonl') settings schema_inference_mode='union', describe_compact_output=1 format TSV"
+            f"desc {engine}('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{file_name_prefix}2.jsonl') settings schema_inference_mode='union', describe_compact_output=1 format TSV"
         )
         assert result == "b\tNullable(Int64)\n"
 
         result = instance.query(
-            f"desc {engine}('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test_union_schema_inference{{1,2,3}}.jsonl') settings schema_inference_mode='union', describe_compact_output=1 format TSV"
+            f"desc {engine}('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{file_name_prefix}{{1,2,3}}.jsonl') settings schema_inference_mode='union', describe_compact_output=1 format TSV"
         )
         assert (
             result == "a\tNullable(Int64)\n"
@@ -2267,7 +2302,7 @@ def test_union_schema_inference_mode(started_cluster):
         )
 
         error = instance.query_and_get_error(
-            f"desc {engine}('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test_union_schema_inference{{1,2,3,4}}.jsonl') settings schema_inference_mode='union', describe_compact_output=1 format TSV"
+            f"desc {engine}('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/{file_name_prefix}{{1,2,3,4}}.jsonl') settings schema_inference_mode='union', describe_compact_output=1 format TSV"
         )
         assert "CANNOT_EXTRACT_TABLE_STRUCTURE" in error
 

From bc9c462155bed43eeb72415660917900ee4e7b58 Mon Sep 17 00:00:00 2001
From: Nikita Taranov <nikita.taranov@clickhouse.com>
Date: Tue, 23 Jul 2024 15:47:55 +0100
Subject: [PATCH 0651/1488] more comments

---
 src/Storages/MergeTree/MergeTreeReadPoolBase.cpp | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/Storages/MergeTree/MergeTreeReadPoolBase.cpp b/src/Storages/MergeTree/MergeTreeReadPoolBase.cpp
index 0e713150625..46482bc0959 100644
--- a/src/Storages/MergeTree/MergeTreeReadPoolBase.cpp
+++ b/src/Storages/MergeTree/MergeTreeReadPoolBase.cpp
@@ -61,6 +61,9 @@ static size_t calculateMinMarksPerTask(
     const size_t part_marks_count = part.getMarksCount();
     if (part_marks_count && part.data_part->isStoredOnRemoteDisk())
     {
+        /// We assume that most of the time prewhere does it's job good meaning that lion's share of the rows is filtered out.
+        /// Which means in turn that for most of the rows we will read only the columns from prewhere clause.
+        /// So it makes sense to use only them for the estimation.
         const auto & columns = settings.merge_tree_determine_task_size_by_prewhere_columns && prewhere_info
             ? prewhere_info->prewhere_actions->getRequiredColumnsNames()
             : columns_to_read;
@@ -69,6 +72,7 @@ static size_t calculateMinMarksPerTask(
         const auto avg_mark_bytes = std::max<size_t>(part_compressed_bytes / part_marks_count, 1);
         const auto min_bytes_per_task = settings.merge_tree_min_bytes_per_task_for_remote_reading;
         /// We're taking min here because number of tasks shouldn't be too low - it will make task stealing impossible.
+        /// We also create at least two tasks per thread to have something to steal from a slow thread.
         const auto heuristic_min_marks
             = std::min<size_t>(pool_settings.sum_marks / pool_settings.threads / 2, min_bytes_per_task / avg_mark_bytes);
         if (heuristic_min_marks > min_marks_per_task)

From e0a7aa80af298465292a5227e65fe21c931b1b9b Mon Sep 17 00:00:00 2001
From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com>
Date: Tue, 23 Jul 2024 16:57:07 +0200
Subject: [PATCH 0652/1488] Enable repeat for flaky check for more tests
 hardening

---
 tests/ci/integration_tests_runner.py | 18 ++++++++++++++----
 tests/integration/runner             |  6 +++++-
 2 files changed, 19 insertions(+), 5 deletions(-)

diff --git a/tests/ci/integration_tests_runner.py b/tests/ci/integration_tests_runner.py
index 21f16d995a4..2b348be8b51 100755
--- a/tests/ci/integration_tests_runner.py
+++ b/tests/ci/integration_tests_runner.py
@@ -29,7 +29,8 @@ CLICKHOUSE_BINARY_PATH = "usr/bin/clickhouse"
 CLICKHOUSE_ODBC_BRIDGE_BINARY_PATH = "usr/bin/clickhouse-odbc-bridge"
 CLICKHOUSE_LIBRARY_BRIDGE_BINARY_PATH = "usr/bin/clickhouse-library-bridge"
 
-FLAKY_TRIES_COUNT = 10
+FLAKY_TRIES_COUNT = 10  # run whole pytest several times
+FLAKY_REPEAT_COUNT = 5  # runs test case in single module several times
 MAX_TIME_SECONDS = 3600
 
 MAX_TIME_IN_SANDBOX = 20 * 60  # 20 minutes
@@ -568,6 +569,7 @@ class ClickhouseIntegrationTestsRunner:
         tests_in_group,
         num_tries,
         num_workers,
+        repeat_count,
     ):
         try:
             return self.run_test_group(
@@ -576,6 +578,7 @@ class ClickhouseIntegrationTestsRunner:
                 tests_in_group,
                 num_tries,
                 num_workers,
+                repeat_count,
             )
         except Exception as e:
             logging.info("Failed to run %s:\n%s", test_group, e)
@@ -598,6 +601,7 @@ class ClickhouseIntegrationTestsRunner:
         tests_in_group,
         num_tries,
         num_workers,
+        repeat_count,
     ):
         counters = {
             "ERROR": [],
@@ -639,6 +643,7 @@ class ClickhouseIntegrationTestsRunner:
 
             test_cmd = " ".join([shlex.quote(test) for test in sorted(test_names)])
             parallel_cmd = f" --parallel {num_workers} " if num_workers > 0 else ""
+            repeat_cmd = f" --count {repeat_count} " if repeat_count > 0 else ""
             # -r -- show extra test summary:
             # -f -- (f)ailed
             # -E -- (E)rror
@@ -647,7 +652,7 @@ class ClickhouseIntegrationTestsRunner:
             cmd = (
                 f"cd {repo_path}/tests/integration && "
                 f"timeout --signal=KILL 1h ./runner {self._get_runner_opts()} "
-                f"{image_cmd} -t {test_cmd} {parallel_cmd} -- -rfEps --run-id={i} "
+                f"{image_cmd} -t {test_cmd} {parallel_cmd} {repeat_cmd} -- -rfEps --run-id={i} "
                 f"--color=no --durations=0 {_get_deselect_option(self.should_skip_tests())} "
                 f"| tee {info_path}"
             )
@@ -784,7 +789,12 @@ class ClickhouseIntegrationTestsRunner:
             final_retry += 1
             logging.info("Running tests for the %s time", i)
             counters, tests_times, log_paths = self.try_run_test_group(
-                repo_path, "bugfix" if should_fail else "flaky", tests_to_run, 1, 1
+                repo_path,
+                "bugfix" if should_fail else "flaky",
+                tests_to_run,
+                1,
+                1,
+                FLAKY_REPEAT_COUNT,
             )
             logs += log_paths
             if counters["FAILED"]:
@@ -919,7 +929,7 @@ class ClickhouseIntegrationTestsRunner:
         for group, tests in items_to_run:
             logging.info("Running test group %s containing %s tests", group, len(tests))
             group_counters, group_test_times, log_paths = self.try_run_test_group(
-                repo_path, group, tests, MAX_RETRY, NUM_WORKERS
+                repo_path, group, tests, MAX_RETRY, NUM_WORKERS, 0
             )
             total_tests = 0
             for counter, value in group_counters.items():
diff --git a/tests/integration/runner b/tests/integration/runner
index a583d7fe897..fc13cb9807a 100755
--- a/tests/integration/runner
+++ b/tests/integration/runner
@@ -243,6 +243,8 @@ if __name__ == "__main__":
         "-n", "--parallel", action="store", dest="parallel", help="Parallelism"
     )
 
+    parser.add_argument("--count", action="store", dest="count", help="Repeat count")
+
     parser.add_argument(
         "--no-random",
         action="store",
@@ -318,6 +320,8 @@ if __name__ == "__main__":
         parallel_args += "--dist=loadfile"
         parallel_args += f" -n {args.parallel}".format()
 
+    repeat_args = f" --count {args.count}" if args.count > 0 else ""
+
     rand_args = ""
     # if not args.no_random:
     #     rand_args += f"--random-seed={os.getpid()}"
@@ -409,7 +413,7 @@ if __name__ == "__main__":
         f"--volume={args.utils_dir}/grpc-client/pb2:/ClickHouse/utils/grpc-client/pb2 "
         f"--volume=/run:/run/host:ro {dockerd_internal_volume} {env_tags} {env_cleanup} "
         f"-e DOCKER_CLIENT_TIMEOUT=300 -e COMPOSE_HTTP_TIMEOUT=600 {use_old_analyzer} -e PYTHONUNBUFFERED=1 "
-        f'-e PYTEST_ADDOPTS="{parallel_args} {pytest_opts} {tests_list} {rand_args} -vvv"'
+        f'-e PYTEST_ADDOPTS="{parallel_args} {repeat_args} {pytest_opts} {tests_list} {rand_args} -vvv"'
         f" {DIND_INTEGRATION_TESTS_IMAGE_NAME}:{args.docker_image_version}"
     )
 

From 2f8b28a17cd885581b4cb177643b952d7975d30c Mon Sep 17 00:00:00 2001
From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com>
Date: Tue, 23 Jul 2024 17:05:23 +0200
Subject: [PATCH 0653/1488] Touch test to trigger flaky check

---
 tests/integration/test_backup_restore_new/test.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/integration/test_backup_restore_new/test.py b/tests/integration/test_backup_restore_new/test.py
index d8662fad011..fbdc64725d5 100644
--- a/tests/integration/test_backup_restore_new/test.py
+++ b/tests/integration/test_backup_restore_new/test.py
@@ -1,8 +1,8 @@
-import pytest
 import glob
-import re
-import random
 import os.path
+import pytest
+import random
+import re
 import sys
 from collections import namedtuple
 from helpers.cluster import ClickHouseCluster

From c7427e6572f278d1a72289ca9ef50fbb701c1115 Mon Sep 17 00:00:00 2001
From: Max K <maxkaynov@gmail.com>
Date: Tue, 23 Jul 2024 16:36:30 +0200
Subject: [PATCH 0654/1488] CI: Fixes docker server build for release branches

---
 tests/ci/docker_server.py | 27 ++++++++++++---------------
 1 file changed, 12 insertions(+), 15 deletions(-)

diff --git a/tests/ci/docker_server.py b/tests/ci/docker_server.py
index 21fc02ce02a..413c35cbebe 100644
--- a/tests/ci/docker_server.py
+++ b/tests/ci/docker_server.py
@@ -21,7 +21,7 @@ from env_helper import (
     TEMP_PATH,
 )
 from git_helper import Git
-from pr_info import PRInfo, EventType
+from pr_info import PRInfo
 from report import FAILURE, SUCCESS, JobReport, TestResult, TestResults
 from stopwatch import Stopwatch
 from tee_popen import TeePopen
@@ -375,25 +375,23 @@ def main():
     tags = gen_tags(args.version, args.release_type)
     repo_urls = {}
     direct_urls: Dict[str, List[str]] = {}
-    if pr_info.event_type == EventType.PULL_REQUEST:
-        release_or_pr = str(pr_info.number)
-        sha = pr_info.sha
-    elif pr_info.event_type == EventType.PUSH and pr_info.is_master:
-        release_or_pr = str(0)
-        sha = pr_info.sha
-    else:
-        release_or_pr = f"{args.version.major}.{args.version.minor}"
-        sha = args.sha
-        assert sha
 
     for arch, build_name in zip(ARCH, ("package_release", "package_aarch64")):
-        if not args.bucket_prefix:
+        if args.bucket_prefix:
+            assert not args.allow_build_reuse
+            repo_urls[arch] = f"{args.bucket_prefix}/{build_name}"
+        elif args.sha:
+            # CreateRelease workflow only. TODO
+            version = args.version
             repo_urls[arch] = (
                 f"{S3_DOWNLOAD}/{S3_BUILDS_BUCKET}/"
-                f"{release_or_pr}/{sha}/{build_name}"
+                f"{version.major}.{version.minor}/{args.sha}/{build_name}"
             )
         else:
-            repo_urls[arch] = f"{args.bucket_prefix}/{build_name}"
+            # In all other cases urls must be fetched from build reports. TODO: script needs refactoring
+            repo_urls[arch] = ""
+            assert args.allow_build_reuse
+
         if args.allow_build_reuse:
             # read s3 urls from pre-downloaded build reports
             if "clickhouse-server" in image_repo:
@@ -431,7 +429,6 @@ def main():
             )
             if test_results[-1].status != "OK":
                 status = FAILURE
-    pr_info = pr_info or PRInfo()
 
     description = f"Processed tags: {', '.join(tags)}"
     JobReport(

From c7400fbd508af022f9fdace728094444e03c1d28 Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Tue, 23 Jul 2024 18:08:07 +0200
Subject: [PATCH 0655/1488] Add more diagnostics in case of digest mismatch for
 DatabaseReplicated

---
 src/Databases/DatabaseReplicated.cpp | 249 +++++++++++++++++++--------
 src/Databases/DatabaseReplicated.h   |  10 +-
 2 files changed, 185 insertions(+), 74 deletions(-)

diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp
index 4c079ae5300..d0648faf07b 100644
--- a/src/Databases/DatabaseReplicated.cpp
+++ b/src/Databases/DatabaseReplicated.cpp
@@ -671,92 +671,96 @@ void DatabaseReplicated::stopLoading()
     DatabaseAtomic::stopLoading();
 }
 
-bool DatabaseReplicated::checkDigestValid(const ContextPtr & local_context, bool debug_check /* = true */) const
+void DatabaseReplicated::dumpLocalTablesForDebugOnly(const ContextPtr & local_context) const
 {
-    if (debug_check)
+    auto table_names = getAllTableNames(context.lock());
+    for (const auto & table_name : table_names)
     {
-        /// Reduce number of debug checks
-        if (thread_local_rng() % 16)
-            return true;
+        auto ast_ptr = tryGetCreateTableQuery(table_name, local_context);
+        if (ast_ptr)
+            LOG_DEBUG(log, "[local] Table {} create query is {}", table_name, queryToString(ast_ptr));
+        else
+            LOG_DEBUG(log, "[local] Table {} has no create query", table_name);
     }
-
-    LOG_TEST(log, "Current in-memory metadata digest: {}", tables_metadata_digest);
-
-    /// Database is probably being dropped
-    if (!local_context->getZooKeeperMetadataTransaction() && (!ddl_worker || !ddl_worker->isCurrentlyActive()))
-        return true;
-
-    UInt64 local_digest = 0;
-    {
-        std::lock_guard lock{mutex};
-        for (const auto & table : TSA_SUPPRESS_WARNING_FOR_READ(tables))
-            local_digest += getMetadataHash(table.first);
-    }
-
-    if (local_digest != tables_metadata_digest)
-    {
-        LOG_ERROR(log, "Digest of local metadata ({}) is not equal to in-memory digest ({})", local_digest, tables_metadata_digest);
-        return false;
-    }
-
-    /// Do not check digest in Keeper after internal subquery, it's probably not committed yet
-    if (local_context->isInternalSubquery())
-        return true;
-
-    /// Check does not make sense to check digest in Keeper during recovering
-    if (is_recovering)
-        return true;
-
-    String zk_digest = getZooKeeper()->get(replica_path + "/digest");
-    String local_digest_str = toString(local_digest);
-    if (zk_digest != local_digest_str)
-    {
-        LOG_ERROR(log, "Digest of local metadata ({}) is not equal to digest in Keeper ({})", local_digest_str, zk_digest);
-        return false;
-    }
-
-    return true;
 }
 
-void DatabaseReplicated::checkQueryValid(const ASTPtr & query, ContextPtr query_context) const
+void DatabaseReplicated::dumpTablesInZooKeeperForDebugOnly() const
 {
-    /// Replicas will set correct name of current database in query context (database name can be different on replicas)
-    if (auto * ddl_query = dynamic_cast<ASTQueryWithTableAndOutput *>(query.get()))
+    UInt32 max_log_ptr;
+    auto table_name_to_metadata = tryGetConsistentMetadataSnapshot(getZooKeeper(), max_log_ptr);
+    for (const auto & [table_name, create_table_query] : table_name_to_metadata)
     {
-        if (ddl_query->getDatabase() != getDatabaseName())
-            throw Exception(ErrorCodes::UNKNOWN_DATABASE, "Database was renamed");
-        ddl_query->database.reset();
-
-        if (auto * create = query->as<ASTCreateQuery>())
+        auto query_ast = parseQueryFromMetadataInZooKeeper(table_name, create_table_query);
+        if (query_ast)
         {
-            if (create->storage)
-                checkTableEngine(*create, *create->storage, query_context);
+            LOG_DEBUG(log, "[zookeeper] Table {} create query is {}", table_name, queryToString(query_ast));
+        }
+        else
+        {
+            LOG_DEBUG(log, "[zookeeper] Table {} has no create query", table_name);
+        }
+    }
+}
 
-            if (create->targets)
+void DatabaseReplicated::tryCompareLocalAndZooKeeperTablesAndDumpDiffForDebugOnly(const ContextPtr & local_context) const
+{
+    UInt32 max_log_ptr;
+    auto table_name_to_metadata_in_zk = tryGetConsistentMetadataSnapshot(getZooKeeper(), max_log_ptr);
+    auto table_names_local = getAllTableNames(local_context);
+
+    if (table_name_to_metadata_in_zk.size() != table_names_local.size())
+        LOG_DEBUG(log, "Amount of tables in zk {} locally {}", table_name_to_metadata_in_zk.size(), table_names_local.size());
+
+    std::unordered_set<std::string> checked_tables;
+
+    for (const auto & table_name : table_names_local)
+    {
+        auto local_ast_ptr = tryGetCreateTableQuery(table_name, local_context);
+        if (table_name_to_metadata_in_zk.contains(table_name))
+        {
+            checked_tables.insert(table_name);
+            auto create_table_query_in_zk = table_name_to_metadata_in_zk[table_name];
+            auto zk_ast_ptr = parseQueryFromMetadataInZooKeeper(table_name, create_table_query_in_zk);
+
+            if (local_ast_ptr == nullptr && zk_ast_ptr == nullptr)
             {
-                for (const auto & inner_table_engine : create->targets->getInnerEngines())
-                    checkTableEngine(*create, *inner_table_engine, query_context);
+                LOG_DEBUG(log, "AST for table {} is the same (nullptr) in local and ZK", table_name);
+            }
+            else if (local_ast_ptr != nullptr && zk_ast_ptr != nullptr && queryToString(local_ast_ptr) != queryToString(zk_ast_ptr))
+            {
+                LOG_DEBUG(log, "AST differs for table {}, local {}, in zookeeper {}", table_name, queryToString(local_ast_ptr), queryToString(zk_ast_ptr));
+            }
+            else if (local_ast_ptr == nullptr)
+            {
+                LOG_DEBUG(log, "AST differs for table {}, local nullptr, in zookeeper {}", table_name, queryToString(zk_ast_ptr));
+            }
+            else if (zk_ast_ptr == nullptr)
+            {
+                LOG_DEBUG(log, "AST differs for table {}, local {}, in zookeeper nullptr", table_name, queryToString(local_ast_ptr));
+            }
+            else
+            {
+                LOG_DEBUG(log, "AST for table {} is the same in local and ZK", table_name);
             }
         }
-    }
-
-    if (const auto * query_alter = query->as<ASTAlterQuery>())
-    {
-        for (const auto & command : query_alter->command_list->children)
+        else
         {
-            if (!isSupportedAlterTypeForOnClusterDDLQuery(command->as<ASTAlterCommand&>().type))
-                throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Unsupported type of ALTER query");
+            if (local_ast_ptr == nullptr)
+                LOG_DEBUG(log, "Table {} exists locally, but missing in ZK", table_name);
+            else
+                LOG_DEBUG(log, "Table {} exists locally with AST {}, but missing in ZK", table_name, queryToString(local_ast_ptr));
         }
     }
-
-    if (auto * query_drop = query->as<ASTDropQuery>())
+    for (const auto & [table_name, table_metadata] : table_name_to_metadata_in_zk)
     {
-        if (query_drop->kind == ASTDropQuery::Kind::Detach && query_context->getSettingsRef().database_replicated_always_detach_permanently)
-            query_drop->permanently = true;
-        if (query_drop->kind == ASTDropQuery::Kind::Detach && !query_drop->permanently)
-            throw Exception(ErrorCodes::INCORRECT_QUERY, "DETACH TABLE is not allowed for Replicated databases. "
-                                                         "Use DETACH TABLE PERMANENTLY or SYSTEM RESTART REPLICA or set "
-                                                         "database_replicated_always_detach_permanently to 1");
+        if (!checked_tables.contains(table_name))
+        {
+            auto zk_ast_ptr = parseQueryFromMetadataInZooKeeper(table_name, table_metadata);
+            if (zk_ast_ptr == nullptr)
+                LOG_DEBUG(log, "Table {} exists in ZK with AST {}, but missing locally", table_name, queryToString(zk_ast_ptr));
+            else
+                LOG_DEBUG(log, "Table {} exists in ZK, but missing locally", table_name);
+        }
     }
 }
 
@@ -839,6 +843,107 @@ void DatabaseReplicated::checkTableEngine(const ASTCreateQuery & query, ASTStora
                     "to distinguish different shards and replicas");
 }
 
+bool DatabaseReplicated::checkDigestValid(const ContextPtr & local_context, bool debug_check /* = true */) const
+{
+    if (debug_check)
+    {
+        /// Reduce number of debug checks
+        if (thread_local_rng() % 16)
+            return true;
+    }
+
+    LOG_TEST(log, "Current in-memory metadata digest: {}", tables_metadata_digest);
+
+    /// Database is probably being dropped
+    if (!local_context->getZooKeeperMetadataTransaction() && (!ddl_worker || !ddl_worker->isCurrentlyActive()))
+        return true;
+
+    UInt64 local_digest = 0;
+    {
+        std::lock_guard lock{mutex};
+        for (const auto & table : TSA_SUPPRESS_WARNING_FOR_READ(tables))
+            local_digest += getMetadataHash(table.first);
+    }
+
+    if (local_digest != tables_metadata_digest)
+    {
+        LOG_ERROR(log, "Digest of local metadata ({}) is not equal to in-memory digest ({})", local_digest, tables_metadata_digest);
+
+#ifndef NDEBUG
+        dumpLocalTablesForDebugOnly(local_context);
+        dumpTablesInZooKeeperForDebugOnly();
+        tryCompareLocalAndZooKeeperTablesAndDumpDiffForDebugOnly(local_context);
+#endif
+
+        return false;
+    }
+
+    /// Do not check digest in Keeper after internal subquery, it's probably not committed yet
+    if (local_context->isInternalSubquery())
+        return true;
+
+    /// Check does not make sense to check digest in Keeper during recovering
+    if (is_recovering)
+        return true;
+
+    String zk_digest = getZooKeeper()->get(replica_path + "/digest");
+    String local_digest_str = toString(local_digest);
+    if (zk_digest != local_digest_str)
+    {
+        LOG_ERROR(log, "Digest of local metadata ({}) is not equal to digest in Keeper ({})", local_digest_str, zk_digest);
+#ifndef NDEBUG
+        dumpLocalTablesForDebugOnly(local_context);
+        dumpTablesInZooKeeperForDebugOnly();
+        tryCompareLocalAndZooKeeperTablesAndDumpDiffForDebugOnly(local_context);
+#endif
+        return false;
+    }
+
+    return true;
+}
+
+void DatabaseReplicated::checkQueryValid(const ASTPtr & query, ContextPtr query_context) const
+{
+    /// Replicas will set correct name of current database in query context (database name can be different on replicas)
+    if (auto * ddl_query = dynamic_cast<ASTQueryWithTableAndOutput *>(query.get()))
+    {
+        if (ddl_query->getDatabase() != getDatabaseName())
+            throw Exception(ErrorCodes::UNKNOWN_DATABASE, "Database was renamed");
+        ddl_query->database.reset();
+
+        if (auto * create = query->as<ASTCreateQuery>())
+        {
+            if (create->storage)
+                checkTableEngine(*create, *create->storage, query_context);
+
+            if (create->targets)
+            {
+                for (const auto & inner_table_engine : create->targets->getInnerEngines())
+                    checkTableEngine(*create, *inner_table_engine, query_context);
+            }
+        }
+    }
+
+    if (const auto * query_alter = query->as<ASTAlterQuery>())
+    {
+        for (const auto & command : query_alter->command_list->children)
+        {
+            if (!isSupportedAlterTypeForOnClusterDDLQuery(command->as<ASTAlterCommand&>().type))
+                throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Unsupported type of ALTER query");
+        }
+    }
+
+    if (auto * query_drop = query->as<ASTDropQuery>())
+    {
+        if (query_drop->kind == ASTDropQuery::Kind::Detach && query_context->getSettingsRef().database_replicated_always_detach_permanently)
+            query_drop->permanently = true;
+        if (query_drop->kind == ASTDropQuery::Kind::Detach && !query_drop->permanently)
+            throw Exception(ErrorCodes::INCORRECT_QUERY, "DETACH TABLE is not allowed for Replicated databases. "
+                                                         "Use DETACH TABLE PERMANENTLY or SYSTEM RESTART REPLICA or set "
+                                                         "database_replicated_always_detach_permanently to 1");
+    }
+}
+
 BlockIO DatabaseReplicated::tryEnqueueReplicatedDDL(const ASTPtr & query, ContextPtr query_context, QueryFlags flags)
 {
     waitDatabaseStarted();
@@ -1253,7 +1358,7 @@ void DatabaseReplicated::recoverLostReplica(const ZooKeeperPtr & current_zookeep
     current_zookeeper->set(replica_path + "/digest", toString(tables_metadata_digest));
 }
 
-std::map<String, String> DatabaseReplicated::tryGetConsistentMetadataSnapshot(const ZooKeeperPtr & zookeeper, UInt32 & max_log_ptr)
+std::map<String, String> DatabaseReplicated::tryGetConsistentMetadataSnapshot(const ZooKeeperPtr & zookeeper, UInt32 & max_log_ptr) const
 {
     return getConsistentMetadataSnapshotImpl(zookeeper, {}, /* max_retries= */ 10, max_log_ptr);
 }
@@ -1314,7 +1419,7 @@ std::map<String, String> DatabaseReplicated::getConsistentMetadataSnapshotImpl(
     return table_name_to_metadata;
 }
 
-ASTPtr DatabaseReplicated::parseQueryFromMetadataInZooKeeper(const String & node_name, const String & query)
+ASTPtr DatabaseReplicated::parseQueryFromMetadataInZooKeeper(const String & node_name, const String & query) const
 {
     ParserCreateQuery parser;
     String description = "in ZooKeeper " + zookeeper_path + "/metadata/" + node_name;
diff --git a/src/Databases/DatabaseReplicated.h b/src/Databases/DatabaseReplicated.h
index 8c3fa7c87f6..2d96695f7cc 100644
--- a/src/Databases/DatabaseReplicated.h
+++ b/src/Databases/DatabaseReplicated.h
@@ -109,14 +109,15 @@ private:
     void checkQueryValid(const ASTPtr & query, ContextPtr query_context) const;
     void checkTableEngine(const ASTCreateQuery & query, ASTStorage & storage, ContextPtr query_context) const;
 
+
     void recoverLostReplica(const ZooKeeperPtr & current_zookeeper, UInt32 our_log_ptr, UInt32 & max_log_ptr);
 
-    std::map<String, String> tryGetConsistentMetadataSnapshot(const ZooKeeperPtr & zookeeper, UInt32 & max_log_ptr);
+    std::map<String, String> tryGetConsistentMetadataSnapshot(const ZooKeeperPtr & zookeeper, UInt32 & max_log_ptr) const;
 
     std::map<String, String> getConsistentMetadataSnapshotImpl(const ZooKeeperPtr & zookeeper, const FilterByNameFunction & filter_by_table_name,
                                                                size_t max_retries, UInt32 & max_log_ptr) const;
 
-    ASTPtr parseQueryFromMetadataInZooKeeper(const String & node_name, const String & query);
+    ASTPtr parseQueryFromMetadataInZooKeeper(const String & node_name, const String & query) const;
     String readMetadataFile(const String & table_name) const;
 
     ClusterPtr getClusterImpl(bool all_groups = false) const;
@@ -132,6 +133,11 @@ private:
     UInt64 getMetadataHash(const String & table_name) const;
     bool checkDigestValid(const ContextPtr & local_context, bool debug_check = true) const TSA_REQUIRES(metadata_mutex);
 
+    /// For debug purposes only, don't use in production code
+    void dumpLocalTablesForDebugOnly(const ContextPtr & local_context) const;
+    void dumpTablesInZooKeeperForDebugOnly() const;
+    void tryCompareLocalAndZooKeeperTablesAndDumpDiffForDebugOnly(const ContextPtr & local_context) const;
+
     void waitDatabaseStarted() const override;
     void stopLoading() override;
 

From 7a65eb08ed571037206640cbdfe305a383e52c9b Mon Sep 17 00:00:00 2001
From: Nikita Taranov <nikita.taranov@clickhouse.com>
Date: Tue, 23 Jul 2024 18:00:33 +0100
Subject: [PATCH 0656/1488] add forgotten file

---
 .../03036_dynamic_read_subcolumns.lib         | 35 +++++++++++++++++++
 1 file changed, 35 insertions(+)
 create mode 100755 tests/queries/0_stateless/03036_dynamic_read_subcolumns.lib

diff --git a/tests/queries/0_stateless/03036_dynamic_read_subcolumns.lib b/tests/queries/0_stateless/03036_dynamic_read_subcolumns.lib
new file mode 100755
index 00000000000..4914051db82
--- /dev/null
+++ b/tests/queries/0_stateless/03036_dynamic_read_subcolumns.lib
@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+function test()
+{
+    echo "test"
+    $CH_CLIENT -q "insert into test select number, number from numbers(100000) settings min_insert_block_size_rows=50000"
+    $CH_CLIENT -q "insert into test select number, 'str_' || toString(number) from numbers(100000, 100000) settings min_insert_block_size_rows=50000"
+    $CH_CLIENT -q "insert into test select number, arrayMap(x -> multiIf(number % 9 == 0, NULL, number % 9 == 3, 'str_' || toString(number), number), range(number % 10 + 1)) from numbers(200000, 100000) settings min_insert_block_size_rows=50000"
+    $CH_CLIENT -q "insert into test select number, NULL from numbers(300000, 100000) settings min_insert_block_size_rows=50000"
+    $CH_CLIENT -q "insert into test select number, multiIf(number % 4 == 3, 'str_' || toString(number), number % 4 == 2, NULL, number % 4 == 1, number, arrayMap(x -> multiIf(number % 9 == 0, NULL, number % 9 == 3, 'str_' || toString(number), number), range(number % 10 + 1))) from numbers(400000, 400000) settings min_insert_block_size_rows=50000"
+    $CH_CLIENT -q "insert into test select number, [range((number % 10 + 1)::UInt64)]::Array(Array(Dynamic)) from numbers(100000, 100000) settings min_insert_block_size_rows=50000"
+
+    $CH_CLIENT -q "select distinct dynamicType(d) as type from test order by type"
+    $CH_CLIENT -q "select count() from test where dynamicType(d) == 'UInt64'"
+    $CH_CLIENT -q "select count() from test where d.UInt64 is not NULL"
+    $CH_CLIENT -q "select count() from test where dynamicType(d) == 'String'"
+    $CH_CLIENT -q "select count() from test where d.String is not NULL"
+    $CH_CLIENT -q "select count() from test where dynamicType(d) == 'Date'"
+    $CH_CLIENT -q "select count() from test where d.Date is not NULL"
+    $CH_CLIENT -q "select count() from test where dynamicType(d) == 'Array(Variant(String, UInt64))'"
+    $CH_CLIENT -q "select count() from test where not empty(d.\`Array(Variant(String, UInt64))\`)"
+    $CH_CLIENT -q "select count() from test where dynamicType(d) == 'Array(Array(Dynamic))'"
+    $CH_CLIENT -q "select count() from test where not empty(d.\`Array(Array(Dynamic))\`)"
+    $CH_CLIENT -q "select count() from test where d is NULL"
+    $CH_CLIENT -q "select count() from test where not empty(d.\`Tuple(a Array(Dynamic))\`.a.String)"
+
+    $CH_CLIENT -q "select d, d.UInt64, d.String, d.\`Array(Variant(String, UInt64))\` from test format Null"
+    $CH_CLIENT -q "select d.UInt64, d.String, d.\`Array(Variant(String, UInt64))\` from test format Null"
+    $CH_CLIENT -q "select d.Int8, d.Date, d.\`Array(String)\` from test format Null"
+    $CH_CLIENT -q "select d, d.UInt64, d.Date, d.\`Array(Variant(String, UInt64))\`, d.\`Array(Variant(String, UInt64))\`.size0, d.\`Array(Variant(String, UInt64))\`.UInt64 from test format Null"
+    $CH_CLIENT -q "select d.UInt64, d.Date, d.\`Array(Variant(String, UInt64))\`, d.\`Array(Variant(String, UInt64))\`.size0, d.\`Array(Variant(String, UInt64))\`.UInt64, d.\`Array(Variant(String, UInt64))\`.String from test format Null"
+    $CH_CLIENT -q "select d, d.\`Tuple(a UInt64, b String)\`.a, d.\`Array(Dynamic)\`.\`Variant(String, UInt64)\`.UInt64, d.\`Array(Variant(String, UInt64))\`.UInt64 from test format Null"
+    $CH_CLIENT -q "select d.\`Array(Dynamic)\`.\`Variant(String, UInt64)\`.UInt64, d.\`Array(Dynamic)\`.size0, d.\`Array(Variant(String, UInt64))\`.UInt64 from test format Null"
+    $CH_CLIENT -q "select d.\`Array(Array(Dynamic))\`.size1, d.\`Array(Array(Dynamic))\`.UInt64, d.\`Array(Array(Dynamic))\`.\`Map(String, Tuple(a UInt64))\`.values.a from test format Null"
+}

From 3df8faf640945ca73a292fe8da6c8b4eed6398b3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Tue, 23 Jul 2024 19:07:19 +0200
Subject: [PATCH 0657/1488] Add initial 24.7 changelog

---
 CHANGELOG.md | 179 +++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 179 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index e2eb65e2967..4fddd7d7685 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,4 +1,5 @@
 ### Table of Contents
+**[ClickHouse release v24.7, 2024-07-30](#247)**<br/>
 **[ClickHouse release v24.6, 2024-07-01](#246)**<br/>
 **[ClickHouse release v24.5, 2024-05-30](#245)**<br/>
 **[ClickHouse release v24.4, 2024-04-30](#244)**<br/>
@@ -9,6 +10,184 @@
 
 # 2024 Changelog
 
+### <a id="247"></a> ClickHouse release 24.7, 2024-07-30
+
+#### Backward Incompatible Change
+* Forbid `CRATE MATERIALIZED VIEW ... ENGINE Replicated*MergeTree POPULATE AS SELECT ...` with Replicated databases. [#63963](https://github.com/ClickHouse/ClickHouse/pull/63963) ([vdimir](https://github.com/vdimir)).
+* `clickhouse-keeper-client` will only accept paths in string literals, such as `ls '/hello/world'`, not bare strings such as `ls /hello/world`. [#65494](https://github.com/ClickHouse/ClickHouse/pull/65494) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Metric `KeeperOutstandingRequets` was renamed to `KeeperOutstandingRequests`. [#66206](https://github.com/ClickHouse/ClickHouse/pull/66206) ([Robert Schulze](https://github.com/rschu1ze)).
+* Remove `is_deterministic` field from the `system.functions` table. [#66630](https://github.com/ClickHouse/ClickHouse/pull/66630) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+
+#### New Feature
+* Extend function `tuple` to construct named tuples in query. Introduce function `tupleNames` to extract names from tuples. [#54881](https://github.com/ClickHouse/ClickHouse/pull/54881) ([Amos Bird](https://github.com/amosbird)).
+* Add `ASOF JOIN` support for `full_sorting_join` algorithm. [#55051](https://github.com/ClickHouse/ClickHouse/pull/55051) ([vdimir](https://github.com/vdimir)).
+* A new table function, `fuzzQuery,` was added. This function allows you to modify a given query string with random variations. Example: `SELECT query FROM fuzzQuery('SELECT 1');`. [#62103](https://github.com/ClickHouse/ClickHouse/pull/62103) ([pufit](https://github.com/pufit)).
+* Add new window function `percent_rank`. [#62747](https://github.com/ClickHouse/ClickHouse/pull/62747) ([lgbo](https://github.com/lgbo-ustc)).
+* Support JWT authentication in `clickhouse-client`. [#62829](https://github.com/ClickHouse/ClickHouse/pull/62829) ([Konstantin Bogdanov](https://github.com/thevar1able)).
+* Add SQL functions `changeYear`, `changeMonth`, `changeDay`, `changeHour`, `changeMinute`, `changeSecond`. For example, `SELECT changeMonth(toDate('2024-06-14'), 7)` returns date `2024-07-14`. [#63186](https://github.com/ClickHouse/ClickHouse/pull/63186) ([cucumber95](https://github.com/cucumber95)).
+* Introduce startup scripts, which allow the execution of preconfigured queries at the startup stage. [#64889](https://github.com/ClickHouse/ClickHouse/pull/64889) ([pufit](https://github.com/pufit)).
+* Support accept_invalid_certificate in client's config in order to allow for client to connect over secure TCP to a server running with self-signed certificate - can be used as a shorthand for corresponding `openSSL` client settings `verificationMode=none` + `invalidCertificateHandler.name=AcceptCertificateHandler`. [#65238](https://github.com/ClickHouse/ClickHouse/pull/65238) ([peacewalker122](https://github.com/peacewalker122)).
+* Add system.error_log which contains history of error values from table system.errors, periodically flushed to disk. [#65381](https://github.com/ClickHouse/ClickHouse/pull/65381) ([Pablo Marcos](https://github.com/pamarcos)).
+* Add aggregate function `groupConcat`. About the same as `arrayStringConcat( groupArray(column), ',')` Can receive 2 parameters: a string delimiter and the number of elements to be processed. [#65451](https://github.com/ClickHouse/ClickHouse/pull/65451) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
+* Add AzureQueue storage. [#65458](https://github.com/ClickHouse/ClickHouse/pull/65458) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Add a new setting to disable/enable writing page index into parquet files. [#65475](https://github.com/ClickHouse/ClickHouse/pull/65475) ([lgbo](https://github.com/lgbo-ustc)).
+* Allow system administrators to configure `logger.console_log_level`. [#65559](https://github.com/ClickHouse/ClickHouse/pull/65559) ([Azat Khuzhin](https://github.com/azat)).
+* Automatically append a wildcard `*` to the end of a directory path with table function `file`. [#66019](https://github.com/ClickHouse/ClickHouse/pull/66019) ([Zhidong (David) Guo](https://github.com/Gun9niR)).
+* Add `--memory-usage` option to client in non interactive mode. [#66393](https://github.com/ClickHouse/ClickHouse/pull/66393) ([vdimir](https://github.com/vdimir)).
+
+#### Experimental Feature
+* Change binary serialization of Variant data type: add `compact` mode to avoid writing the same discriminator multiple times for granules with single variant or with only NULL values. Add MergeTree setting `use_compact_variant_discriminators_serialization` that is enabled by default. Note that Variant type is still experimental and backward-incompatible change in serialization is ok. [#62774](https://github.com/ClickHouse/ClickHouse/pull/62774) ([Kruglov Pavel](https://github.com/Avogar)).
+* Support rocksdb as backend storage of keeper. [#56626](https://github.com/ClickHouse/ClickHouse/pull/56626) ([Han Fei](https://github.com/hanfei1991)).
+* Refactor JSONExtract functions, support more types including experimental Dynamic type. [#66046](https://github.com/ClickHouse/ClickHouse/pull/66046) ([Kruglov Pavel](https://github.com/Avogar)).
+* Support null map subcolumn for Variant and Dynamic subcolumns. [#66178](https://github.com/ClickHouse/ClickHouse/pull/66178) ([Kruglov Pavel](https://github.com/Avogar)).
+
+#### Performance Improvement
+* Enable `optimize_functions_to_subcolumns` by default. [#58661](https://github.com/ClickHouse/ClickHouse/pull/58661) ([Anton Popov](https://github.com/CurtizJ)).
+* Replace int to string algorithm with a faster one (from a modified amdn/itoa to a modified jeaiii/itoa). [#61661](https://github.com/ClickHouse/ClickHouse/pull/61661) ([Raúl Marín](https://github.com/Algunenano)).
+* Sizes of hash tables created by join (`parallel_hash` algorithm) is collected and cached now. This information will be used to preallocate space in hash tables for subsequent query executions and save time on hash table resizes. [#64553](https://github.com/ClickHouse/ClickHouse/pull/64553) ([Nikita Taranov](https://github.com/nickitat)).
+* Optimized queries with `ORDER BY` primary key and `WHERE` that have a condition with high selectivity by using of buffering. It is controlled by setting `read_in_order_use_buffering` (enabled by default) and can increase memory usage of query. [#64607](https://github.com/ClickHouse/ClickHouse/pull/64607) ([Anton Popov](https://github.com/CurtizJ)).
+* Improve performance of loading `plain_rewritable` metadata. [#65634](https://github.com/ClickHouse/ClickHouse/pull/65634) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Attaching tables on read-only disks will use fewer resources by not loading outdated parts. [#65635](https://github.com/ClickHouse/ClickHouse/pull/65635) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Support minmax hyperrectangle for Set indices. [#65676](https://github.com/ClickHouse/ClickHouse/pull/65676) ([AntiTopQuark](https://github.com/AntiTopQuark)).
+* Unload primary index of outdated parts to reduce total memory usage. [#65852](https://github.com/ClickHouse/ClickHouse/pull/65852) ([Anton Popov](https://github.com/CurtizJ)).
+* Functions `replaceRegexpAll` and `replaceRegexpOne` are now significantly faster if the pattern is trivial, i.e. contains no metacharacters, pattern classes, flags, grouping characters etc. (Thanks to Taiyang Li). [#66185](https://github.com/ClickHouse/ClickHouse/pull/66185) ([Robert Schulze](https://github.com/rschu1ze)).
+
+#### Improvement
+* The setting `optimize_trivial_insert_select` is disabled by default. In most cases, it should be beneficial. Nevertheless, if you are seeing slower INSERT SELECT or increased memory usage, you can enable it back or `SET compatibility = '24.6'`. [#58970](https://github.com/ClickHouse/ClickHouse/pull/58970) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Print stacktrace and diagnostic info if `clickhouse-client` or `clickhouse-local` crashes. [#61109](https://github.com/ClickHouse/ClickHouse/pull/61109) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* The result of `SHOW INDEX | INDEXES | INDICES | KEYS` was previously sorted by the primary key column names. Since this was unintuitive, the result is now sorted by the position of the primary key columns within the primary key. [#61131](https://github.com/ClickHouse/ClickHouse/pull/61131) ([Robert Schulze](https://github.com/rschu1ze)).
+* Change how deduplication for Materialized Views works. Fixed a lot of cases like: - on destination table: data is split for 2 or more blocks and that blocks is considered as duplicate when that block is inserted in parallel. - on MV destination table: the equal blocks are deduplicated, that happens when MV often produces equal data as a result for different input data due to performing aggregation. - on MV destination table: the equal blocks which comes from different MV are deduplicated. [#61601](https://github.com/ClickHouse/ClickHouse/pull/61601) ([Sema Checherinda](https://github.com/CheSema)).
+* Allow matching column names in a case insensitive manner when reading json files (`input_format_json_ignore_key_case`). [#61750](https://github.com/ClickHouse/ClickHouse/pull/61750) ([kevinyhzou](https://github.com/KevinyhZou)).
+* Support reading partitioned data DeltaLake data. Infer DeltaLake schema by reading metadata instead of data. [#63201](https://github.com/ClickHouse/ClickHouse/pull/63201) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* In composable protocols TLS layer accepted only `certificateFile` and `privateKeyFile` parameters. https://clickhouse.com/docs/en/operations/settings/composable-protocols. [#63985](https://github.com/ClickHouse/ClickHouse/pull/63985) ([Anton Ivashkin](https://github.com/ianton-ru)).
+* Make an interactive client for clickhouse-disks, add local disk from the local directory. [#64446](https://github.com/ClickHouse/ClickHouse/pull/64446) ([Daniil Ivanik](https://github.com/divanik)).
+* Added profile event `SelectQueriesWithPrimaryKeyUsage` which indicates how many SELECT queries use the primary key to evaluate the WHERE clause. [#64492](https://github.com/ClickHouse/ClickHouse/pull/64492) ([0x01f](https://github.com/0xfei)).
+* `StorageS3Queue` related fixes and improvements. Deduce a default value of `s3queue_processing_threads_num` according to the number of physical cpu cores on the server (instead of the previous default value as 1). Set default value of `s3queue_loading_retries` to 10. Fix possible vague "Uncaught exception" in exception column of `system.s3queue`. Do not increment retry count on `MEMORY_LIMIT_EXCEEDED` exception. Move files commit to a stage after insertion into table fully finished to avoid files being commited while not inserted. Add settings `s3queue_max_processed_files_before_commit`, `s3queue_max_processed_rows_before_commit`, `s3queue_max_processed_bytes_before_commit`, `s3queue_max_processing_time_sec_before_commit`, to better control commit and flush time. [#65046](https://github.com/ClickHouse/ClickHouse/pull/65046) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fixed broken multiple columns aggregation on s390x. [#65062](https://github.com/ClickHouse/ClickHouse/pull/65062) ([Harry Lee](https://github.com/HarryLeeIBM)).
+* Support aliases in parametrized view function (only new analyzer). [#65190](https://github.com/ClickHouse/ClickHouse/pull/65190) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* s3 requests: Reduce retry time for queries, increase retries count for backups. 8.5 minutes and 100 retires for queries, 1.2 hours and 1000 retries for backup restore. [#65232](https://github.com/ClickHouse/ClickHouse/pull/65232) ([Sema Checherinda](https://github.com/CheSema)).
+* Updated to mask account key in logs in azureBlobStorage. [#65273](https://github.com/ClickHouse/ClickHouse/pull/65273) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)).
+* Partition pruning for `IN` predicates when filter expression is a part of `PARTITION BY` expression. [#65335](https://github.com/ClickHouse/ClickHouse/pull/65335) ([Eduard Karacharov](https://github.com/korowa)).
+* Add system tables with main information about all detached tables. [#65400](https://github.com/ClickHouse/ClickHouse/pull/65400) ([Konstantin Morozov](https://github.com/k-morozov)).
+* Add support for `cluster_for_parallel_replicas` when using custom key parallel replicas. It allows you to use parallel replicas with custom key with MergeTree tables. [#65453](https://github.com/ClickHouse/ClickHouse/pull/65453) ([Antonio Andelic](https://github.com/antonio2368)).
+* Support query plan LIMIT optimization. Support LIMIT pushdown for PostgreSQL storage and table function. [#65454](https://github.com/ClickHouse/ClickHouse/pull/65454) ([Maksim Kita](https://github.com/kitaisreal)).
+* `arrayMin`/`arrayMax` can be applicable to all data types that are comparable. [#65455](https://github.com/ClickHouse/ClickHouse/pull/65455) ([pn](https://github.com/chloro-pn)).
+* Improved memory accounting for cgroups v2 to exclude the amount occupied by the page cache. [#65470](https://github.com/ClickHouse/ClickHouse/pull/65470) ([Nikita Taranov](https://github.com/nickitat)).
+* Do not create format settings for each row when serializing chunks to insert to EmbeddedRocksDB table. [#65474](https://github.com/ClickHouse/ClickHouse/pull/65474) ([Duc Canh Le](https://github.com/canhld94)).
+* Fixed out-of-range exception in parsing Dwarf5 on s390x. [#65501](https://github.com/ClickHouse/ClickHouse/pull/65501) ([Harry Lee](https://github.com/HarryLeeIBM)).
+* Reduce `clickhouse-local` prompt to just `:)`. `getFQDNOrHostName()` takes too long on macOS, and we don't want a hostname in the prompt for `clickhouse-local` anyway. [#65510](https://github.com/ClickHouse/ClickHouse/pull/65510) ([Konstantin Bogdanov](https://github.com/thevar1able)).
+* Avoid printing a message from jemalloc about per-CPU arenas on low-end virtual machines. [#65532](https://github.com/ClickHouse/ClickHouse/pull/65532) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Disable filesystem cache background download by default. It will be enabled back when we fix the issue with possible "Memory limit exceeded" because memory deallocation is done outside of query context (while buffer is allocated inside of query context) if we use background download threads. Plus we need to add a separate setting to define max size to download for background workers (currently it is limited by max_file_segment_size, which might be too big). [#65534](https://github.com/ClickHouse/ClickHouse/pull/65534) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Add new option to config `<config_reload_interval_ms>` which allow to specify how often clickhouse will reload config. [#65545](https://github.com/ClickHouse/ClickHouse/pull/65545) ([alesapin](https://github.com/alesapin)).
+* Implement binary encoding for ClickHouse data types and add its specification in docs. Use it in Dynamic binary serialization, allow to use it in RowBinaryWithNamesAndTypes and Native formats under settings. [#65546](https://github.com/ClickHouse/ClickHouse/pull/65546) ([Kruglov Pavel](https://github.com/Avogar)).
+* Improved ZooKeeper load balancing. The current session doesn't expire until the optimal nodes become available despite `fallback_session_lifetime`. Added support for AZ-aware balancing. [#65570](https://github.com/ClickHouse/ClickHouse/pull/65570) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Server settings `compiled_expression_cache_size` and `compiled_expression_cache_elements_size` are now shown in `system.server_settings`. [#65584](https://github.com/ClickHouse/ClickHouse/pull/65584) ([Robert Schulze](https://github.com/rschu1ze)).
+* When lightweight delete happens on a table with projection(s), users have choices either throw an exception (by default) or drop the projection [#65594](https://github.com/ClickHouse/ClickHouse/pull/65594) ([jsc0218](https://github.com/jsc0218)).
+* Add support for user identification based on x509 SubjectAltName extension. [#65626](https://github.com/ClickHouse/ClickHouse/pull/65626) ([Anton Kozlov](https://github.com/tonickkozlov)).
+* `clickhouse-local` will respect the `max_server_memory_usage` and `max_server_memory_usage_to_ram_ratio` from the configuration file. It will also set the max memory usage to 90% of the system memory by default, like `clickhouse-server` does. [#65697](https://github.com/ClickHouse/ClickHouse/pull/65697) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Add a script to backup your files to ClickHouse. [#65699](https://github.com/ClickHouse/ClickHouse/pull/65699) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* PostgreSQL source support cancel. [#65722](https://github.com/ClickHouse/ClickHouse/pull/65722) ([Maksim Kita](https://github.com/kitaisreal)).
+* Make allow_experimental_analyzer be controlled by the initiator for distributed queries. This ensures compatibility and correctness during operations in mixed version clusters. [#65777](https://github.com/ClickHouse/ClickHouse/pull/65777) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Respect cgroup CPU limit in Keeper. [#65819](https://github.com/ClickHouse/ClickHouse/pull/65819) ([Antonio Andelic](https://github.com/antonio2368)).
+* Allow to use `concat` function with empty arguments ``` sql :) select concat();. [#65887](https://github.com/ClickHouse/ClickHouse/pull/65887) ([李扬](https://github.com/taiyang-li)).
+* Allow controlling named collections in clickhouse-local. [#65973](https://github.com/ClickHouse/ClickHouse/pull/65973) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Improve Azure profile events. [#65999](https://github.com/ClickHouse/ClickHouse/pull/65999) ([alesapin](https://github.com/alesapin)).
+* Support ORC file read by writer time zone. [#66025](https://github.com/ClickHouse/ClickHouse/pull/66025) ([kevinyhzou](https://github.com/KevinyhZou)).
+* DatabaseCatalog drops tables faster by using up to database_catalog_drop_table_concurrency threads. [#66065](https://github.com/ClickHouse/ClickHouse/pull/66065) ([Sema Checherinda](https://github.com/CheSema)).
+* Add settings to control connection to the PostgreSQL. * Setting `postgresql_connection_attempt_timeout` specifies the value passed to `connect_timeout` parameter of connection URL. * Setting `postgresql_connection_pool_retries` specifies the number of retries to establish a connection to the PostgreSQL end-point. [#66232](https://github.com/ClickHouse/ClickHouse/pull/66232) ([Dmitry Novik](https://github.com/novikd)).
+* Reduce inaccuracy of input_wait_elapsed_us/input_wait_elapsed_us/elapsed_us. [#66239](https://github.com/ClickHouse/ClickHouse/pull/66239) ([Azat Khuzhin](https://github.com/azat)).
+* Improve FilesystemCache ProfileEvents. [#66249](https://github.com/ClickHouse/ClickHouse/pull/66249) ([zhukai](https://github.com/nauu)).
+* Add settings to ignore ON CLUSTER clause in queries for named collection management with replicated storage. [#66288](https://github.com/ClickHouse/ClickHouse/pull/66288) ([MikhailBurdukov](https://github.com/MikhailBurdukov)).
+* Function `generateSnowflakeID` now allows to specify a machine ID as a parameter to prevent collisions in large clusters. [#66374](https://github.com/ClickHouse/ClickHouse/pull/66374) ([ZAWA_ll](https://github.com/Zawa-ll)).
+* Disable suspending on Ctrl+Z in interactive mode. This is a common trap and is not expected behavior for almost all users. I imagine only a few extreme power users could appreciate suspending terminal applications to the background, but I don't know any. [#66511](https://github.com/ClickHouse/ClickHouse/pull/66511) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Add option for validating the Primary key type in Dictionaries. Without this option for simple layouts any column type will be implicitly converted to UInt64. ### Documentation entry for user-facing changes. [#66595](https://github.com/ClickHouse/ClickHouse/pull/66595) ([MikhailBurdukov](https://github.com/MikhailBurdukov)).
+
+#### Critical Bug Fix (crash, LOGICAL_ERROR, data loss, RBAC)
+* Fix unexpeced size of low cardinality column in function calls. [#65298](https://github.com/ClickHouse/ClickHouse/pull/65298) ([Raúl Marín](https://github.com/Algunenano)).
+* Check cyclic dependencies on CREATE/REPLACE/RENAME/EXCHANGE queries and throw an exception if there is a cyclic dependency. Previously such cyclic dependencies could lead to a deadlock during server startup. Also fix some bugs in dependencies creation. [#65405](https://github.com/ClickHouse/ClickHouse/pull/65405) ([Kruglov Pavel](https://github.com/Avogar)).
+* Fix crash in maxIntersections. [#65689](https://github.com/ClickHouse/ClickHouse/pull/65689) ([Raúl Marín](https://github.com/Algunenano)).
+* Fix the VALID UNTIL clause in the user definition resetting after a restart. [#66409](https://github.com/ClickHouse/ClickHouse/pull/66409) ([Nikolay Degterinsky](https://github.com/evillique)).
+
+#### Bug Fix (user-visible misbehavior in an official stable release)
+* Fix SHOW MERGES remaining time. [#66735](https://github.com/ClickHouse/ClickHouse/pull/66735) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* `Query was cancelled` might have been printed twice in clickhouse-client. This behaviour is fixed. [#66005](https://github.com/ClickHouse/ClickHouse/pull/66005) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Fixed crash while using MaterializedMySQL with TABLE OVERRIDE that maps MySQL NULL field into ClickHouse not NULL field. [#54649](https://github.com/ClickHouse/ClickHouse/pull/54649) ([Filipp Ozinov](https://github.com/bakwc)).
+* Fix logical error when PREWHERE expression read no columns and table has no adaptive index granularity (very old table). [#59173](https://github.com/ClickHouse/ClickHouse/pull/59173) ([Alexander Gololobov](https://github.com/davenger)).
+* Fix bug with cancelation buffer when canceling a query. [#64478](https://github.com/ClickHouse/ClickHouse/pull/64478) ([Sema Checherinda](https://github.com/CheSema)).
+* Fix filling parts columns from metadata (when columns.txt does not exists). [#64757](https://github.com/ClickHouse/ClickHouse/pull/64757) ([Azat Khuzhin](https://github.com/azat)).
+* Fix crash for `ALTER TABLE ... ON CLUSTER ... MODIFY SQL SECURITY`. [#64957](https://github.com/ClickHouse/ClickHouse/pull/64957) ([pufit](https://github.com/pufit)).
+* Fix crash on destroying AccessControl: add explicit shutdown. [#64993](https://github.com/ClickHouse/ClickHouse/pull/64993) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Eliminate injective function in argument of functions `uniq*` recursively. This used to work correctly but was broken in the new analyzer. [#65140](https://github.com/ClickHouse/ClickHouse/pull/65140) ([Duc Canh Le](https://github.com/canhld94)).
+* Fix unexpected projection name when query with CTE. [#65267](https://github.com/ClickHouse/ClickHouse/pull/65267) ([wudidapaopao](https://github.com/wudidapaopao)).
+* Require `dictGet` privilege when accessing dictionaries via direct query or the `Dictionary` table engine. [#65359](https://github.com/ClickHouse/ClickHouse/pull/65359) ([Joe Lynch](https://github.com/joelynch)).
+* Fix user-specific S3 auth with incremental backups. [#65481](https://github.com/ClickHouse/ClickHouse/pull/65481) ([Antonio Andelic](https://github.com/antonio2368)).
+* Disable `non-intersecting-parts` optimization for queries with `FINAL` in case of `read-in-order` optimization was enabled. This could lead to an incorrect query result. As a workaround, disable `do_not_merge_across_partitions_select_final` and `split_parts_ranges_into_intersecting_and_non_intersecting_final` before this fix is merged. [#65505](https://github.com/ClickHouse/ClickHouse/pull/65505) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix getting exception `Index out of bound for blob metadata` in case all files from list batch were filtered out. [#65523](https://github.com/ClickHouse/ClickHouse/pull/65523) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fix NOT_FOUND_COLUMN_IN_BLOCK for deduplicate merge of projection. [#65573](https://github.com/ClickHouse/ClickHouse/pull/65573) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
+* Fixed bug in MergeJoin. Column in sparse serialisation might be treated as a column of its nested type though the required conversion wasn't performed. [#65632](https://github.com/ClickHouse/ClickHouse/pull/65632) ([Nikita Taranov](https://github.com/nickitat)).
+* Fixed a bug that compatibility level '23.4' was not properly applied. [#65737](https://github.com/ClickHouse/ClickHouse/pull/65737) ([cw5121](https://github.com/cw5121)).
+* Fix odbc table with nullable fields. [#65738](https://github.com/ClickHouse/ClickHouse/pull/65738) ([Rodolphe Dugé de Bernonville](https://github.com/RodolpheDuge)).
+* Fix data race in `TCPHandler`, which could happen on fatal error. [#65744](https://github.com/ClickHouse/ClickHouse/pull/65744) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fix invalid exceptions in function `parseDateTime` with `%F` and `%D` placeholders. [#65768](https://github.com/ClickHouse/ClickHouse/pull/65768) ([Antonio Andelic](https://github.com/antonio2368)).
+* For queries that read from `PostgreSQL`, cancel the internal `PostgreSQL` query if the ClickHouse query is finished. Otherwise, `ClickHouse` query cannot be canceled until the internal `PostgreSQL` query is finished. [#65771](https://github.com/ClickHouse/ClickHouse/pull/65771) ([Maksim Kita](https://github.com/kitaisreal)).
+* Fix a bug in short circuit logic when old analyzer and dictGetOrDefault is used. [#65802](https://github.com/ClickHouse/ClickHouse/pull/65802) ([jsc0218](https://github.com/jsc0218)).
+* Fix a bug leads to EmbeddedRocksDB with TTL write corrupted SST files. [#65816](https://github.com/ClickHouse/ClickHouse/pull/65816) ([Duc Canh Le](https://github.com/canhld94)).
+* Functions `bitTest`, `bitTestAll`, and `bitTestAny` now return an error if the specified bit index is out-of-bounds [#65818](https://github.com/ClickHouse/ClickHouse/pull/65818) ([Pablo Marcos](https://github.com/pamarcos)).
+* Setting `join_any_take_last_row` is supported in any query with hash join. [#65820](https://github.com/ClickHouse/ClickHouse/pull/65820) ([vdimir](https://github.com/vdimir)).
+* Better handling of join conditions involving `IS NULL` checks (for example `ON (a = b AND (a IS NOT NULL) AND (b IS NOT NULL) ) OR ( (a IS NULL) AND (b IS NULL) )` is rewritten to `ON a <=> b`), fix incorrect optimization when condition other then `IS NULL` are present. [#65835](https://github.com/ClickHouse/ClickHouse/pull/65835) ([vdimir](https://github.com/vdimir)).
+* Functions `bitShiftLeft` and `bitShitfRight` return an error for out of bounds shift positions [#65838](https://github.com/ClickHouse/ClickHouse/pull/65838) ([Pablo Marcos](https://github.com/pamarcos)).
+* Fix growing memory usage in S3Queue. [#65839](https://github.com/ClickHouse/ClickHouse/pull/65839) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fix tie handling in `arrayAUC` to match sklearn. [#65840](https://github.com/ClickHouse/ClickHouse/pull/65840) ([gabrielmcg44](https://github.com/gabrielmcg44)).
+* Fix possible issues with MySQL server protocol TLS connections. [#65917](https://github.com/ClickHouse/ClickHouse/pull/65917) ([Azat Khuzhin](https://github.com/azat)).
+* Fix possible issues with MySQL client protocol TLS connections. [#65938](https://github.com/ClickHouse/ClickHouse/pull/65938) ([Azat Khuzhin](https://github.com/azat)).
+* Fix handling of `SSL_ERROR_WANT_READ`/`SSL_ERROR_WANT_WRITE` with zero timeout. [#65941](https://github.com/ClickHouse/ClickHouse/pull/65941) ([Azat Khuzhin](https://github.com/azat)).
+* Add missing settings `input_format_csv_skip_first_lines/input_format_tsv_skip_first_lines/input_format_csv_try_infer_numbers_from_strings/input_format_csv_try_infer_strings_from_quoted_tuples` in schema inference cache because they can change the resulting schema. It prevents from incorrect result of schema inference with these settings changed. [#65980](https://github.com/ClickHouse/ClickHouse/pull/65980) ([Kruglov Pavel](https://github.com/Avogar)).
+* Column _size in s3 engine and s3 table function denotes the size of a file inside the archive, not a size of the archive itself. [#65993](https://github.com/ClickHouse/ClickHouse/pull/65993) ([Daniil Ivanik](https://github.com/divanik)).
+* Fix resolving dynamic subcolumns in analyzer, avoid reading the whole column on dynamic subcolumn reading. [#66004](https://github.com/ClickHouse/ClickHouse/pull/66004) ([Kruglov Pavel](https://github.com/Avogar)).
+* Fix config merging for from_env with replace overrides. [#66034](https://github.com/ClickHouse/ClickHouse/pull/66034) ([Azat Khuzhin](https://github.com/azat)).
+* Fix a possible hanging in `GRPCServer` during shutdown. [#66061](https://github.com/ClickHouse/ClickHouse/pull/66061) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Fix reading dynamic subcolumns from altered Memory table. Previously if `max_types` peremeter of a Dynamic type was changed in Memory table via alter, further subcolumns reading can return wrong result. [#66066](https://github.com/ClickHouse/ClickHouse/pull/66066) ([Kruglov Pavel](https://github.com/Avogar)).
+* Fixed several cases in function `has` with non-constant `LowCardinality` arguments. [#66088](https://github.com/ClickHouse/ClickHouse/pull/66088) ([Anton Popov](https://github.com/CurtizJ)).
+* Fix for `groupArrayIntersect`. It had incorrect behavior in the `merge()` function. Also, fixed behavior in `deserialise()` for numeric and general data. [#66103](https://github.com/ClickHouse/ClickHouse/pull/66103) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
+* Fixed buffer overflow bug in `unbin`/`unhex` implementation. [#66106](https://github.com/ClickHouse/ClickHouse/pull/66106) ([Nikita Taranov](https://github.com/nickitat)).
+* Disable the `merge-filters` optimization introduced in [#64760](https://github.com/ClickHouse/ClickHouse/issues/64760). It may cause an exception if optimization merges two filter expressions and does not apply a short-circuit evaluation. [#66126](https://github.com/ClickHouse/ClickHouse/pull/66126) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fixed the issue when the server failed to parse Avro files with negative block size arrays encoded, which is now allowed by the Avro specification. [#66130](https://github.com/ClickHouse/ClickHouse/pull/66130) ([Serge Klochkov](https://github.com/slvrtrn)).
+* Fixed a bug in ZooKeeper client: a session could get stuck in unusable state after receiving a hardware error from ZooKeeper. For example, this might happen due to "soft memory limit" in ClickHouse Keeper. [#66140](https://github.com/ClickHouse/ClickHouse/pull/66140) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Fix issue in SumIfToCountIfVisitor and signed integers. [#66146](https://github.com/ClickHouse/ClickHouse/pull/66146) ([Raúl Marín](https://github.com/Algunenano)).
+* Fix rare case with missing data in the result of distributed query. [#66174](https://github.com/ClickHouse/ClickHouse/pull/66174) ([vdimir](https://github.com/vdimir)).
+* Fix order of parsing metadata fields in StorageDeltaLake. [#66211](https://github.com/ClickHouse/ClickHouse/pull/66211) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Don't throw `TIMEOUT_EXCEEDED` for `none_only_active` mode of `distributed_ddl_output_mode`. [#66218](https://github.com/ClickHouse/ClickHouse/pull/66218) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Fix handling limit for `system.numbers_mt` when no index can be used. [#66231](https://github.com/ClickHouse/ClickHouse/pull/66231) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
+* Fixed how the ClickHouse server detects the maximum number of usable CPU cores as specified by cgroups v2 if the server runs in a container such as Docker. In more detail, containers often run their process in the root cgroup which has an empty name. In that case, ClickHouse ignored the CPU limits set by cgroups v2. [#66237](https://github.com/ClickHouse/ClickHouse/pull/66237) ([filimonov](https://github.com/filimonov)).
+* Fix the `Not-ready set` error when a subquery with `IN` is used in the constraint. [#66261](https://github.com/ClickHouse/ClickHouse/pull/66261) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix error reporting while copying to S3 or AzureBlobStorage. [#66295](https://github.com/ClickHouse/ClickHouse/pull/66295) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Prevent watchdog from keeping descriptors of unlinked(rotated) log files. [#66334](https://github.com/ClickHouse/ClickHouse/pull/66334) ([Aleksei Filatov](https://github.com/aalexfvk)).
+* Fix the bug that logicalexpressionoptimizerpass lost logical type of constant. [#66344](https://github.com/ClickHouse/ClickHouse/pull/66344) ([pn](https://github.com/chloro-pn)).
+* Fix `Column identifier is already registered` error with `group_by_use_nulls=true` and new analyzer. [#66400](https://github.com/ClickHouse/ClickHouse/pull/66400) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix possible incorrect result for queries joining and filtering table external engine (like PostgreSQL), due to too aggressive filter pushdown. Since now, conditions from where section won't be send to external database in case of outer join with external table. [#66402](https://github.com/ClickHouse/ClickHouse/pull/66402) ([vdimir](https://github.com/vdimir)).
+* Added missing column materialization for cross join. [#66413](https://github.com/ClickHouse/ClickHouse/pull/66413) ([lgbo](https://github.com/lgbo-ustc)).
+* Fix `Cannot find column` error for queries with constant expression in `GROUP BY` key and new analyzer enabled. [#66433](https://github.com/ClickHouse/ClickHouse/pull/66433) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Avoid possible logical error during import from Npy format in case of bad array nesting level, fix testing of other kinds of errors. [#66461](https://github.com/ClickHouse/ClickHouse/pull/66461) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
+* Fix wrong count() result when there is non-deterministic function in predicate. [#66510](https://github.com/ClickHouse/ClickHouse/pull/66510) ([Duc Canh Le](https://github.com/canhld94)).
+* Correctly track memory for `Allocator::realloc`. [#66548](https://github.com/ClickHouse/ClickHouse/pull/66548) ([Antonio Andelic](https://github.com/antonio2368)).
+* Fix reading of uninitialized memory when hashing empty tuples. [#66562](https://github.com/ClickHouse/ClickHouse/pull/66562) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix an invalid result for queries with `WINDOW`. This could happen when `PARTITION` columns have sparse serialization and window functions are executed in parallel. [#66579](https://github.com/ClickHouse/ClickHouse/pull/66579) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix removing named collections in local storage. [#66599](https://github.com/ClickHouse/ClickHouse/pull/66599) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
+* Fix `column_length` is not updated in `ColumnTuple::insertManyFrom`. [#66626](https://github.com/ClickHouse/ClickHouse/pull/66626) ([lgbo](https://github.com/lgbo-ustc)).
+* Fix `Unknown identifier` and `Column is not under aggregate function` errors for queries with the expression `(column IS NULL).` The bug was triggered by [#65088](https://github.com/ClickHouse/ClickHouse/issues/65088), with the disabled analyzer only. [#66654](https://github.com/ClickHouse/ClickHouse/pull/66654) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix `Method getResultType is not supported for QUERY query node` error when scalar subquery was used as the first argument of IN (with new analyzer). [#66655](https://github.com/ClickHouse/ClickHouse/pull/66655) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix possible PARAMETER_OUT_OF_BOUND error during reading variant subcolumn. [#66659](https://github.com/ClickHouse/ClickHouse/pull/66659) ([Kruglov Pavel](https://github.com/Avogar)).
+* Fix rare case of stuck merge after drop column. [#66707](https://github.com/ClickHouse/ClickHouse/pull/66707) ([Raúl Marín](https://github.com/Algunenano)).
+* Fix assertion `isUniqTypes` when insert select from remote sources. [#66722](https://github.com/ClickHouse/ClickHouse/pull/66722) ([Sema Checherinda](https://github.com/CheSema)).
+* Fix logical error in PrometheusRequestHandler. [#66621](https://github.com/ClickHouse/ClickHouse/pull/66621) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Fix `indexHint` function case found by fuzzer. [#66286](https://github.com/ClickHouse/ClickHouse/pull/66286) ([Anton Popov](https://github.com/CurtizJ)).
+* Fix AST formatting of 'create table b empty as a'. [#64951](https://github.com/ClickHouse/ClickHouse/pull/64951) ([Michael Kolupaev](https://github.com/al13n321)).
+
+#### Build/Testing/Packaging Improvement
+* Instantiate template methods ahead in different .cpp files, avoid too large translation units during compiling. [#64818](https://github.com/ClickHouse/ClickHouse/pull/64818) ([lgbo](https://github.com/lgbo-ustc)).
+* Upgraded `pocketfft` dependency to the recent commit https://github.com/mreineck/pocketfft/commit/f4c1aa8aa9ce79ad39e80f2c9c41b92ead90fda3. [#66291](https://github.com/ClickHouse/ClickHouse/pull/66291) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Upgraded `azure-sdk-for-cpp` to the recent commit https://github.com/ClickHouse/azure-sdk-for-cpp/commit/ea3e19a7be08519134c643177d56c7484dfec884. [#66292](https://github.com/ClickHouse/ClickHouse/pull/66292) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+
 ### <a id="246"></a> ClickHouse release 24.6, 2024-07-01
 
 #### Backward Incompatible Change

From 73ab9a1197f0d7e71be7c8014621de7226ad4da0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Tue, 23 Jul 2024 19:11:22 +0200
Subject: [PATCH 0658/1488] Revert "Merge pull request #65298 from
 Algunenano/low_cardinality_wtf"

This reverts commit d3a269c61dfa784414245b01ec4d8ccccfeb8b3a, reversing
changes made to 023cf118dc77cc316b158a543a5be51ae67b65f9.
---
 src/Analyzer/Resolve/QueryAnalyzer.cpp        |   4 +-
 src/Functions/IFunction.cpp                   | 101 ++++++------------
 src/Planner/Planner.cpp                       |   7 +-
 src/Planner/PlannerActionsVisitor.cpp         |  61 +++--------
 src/Planner/PlannerActionsVisitor.h           |   9 +-
 ..._no_aggregates_and_constant_keys.reference |   4 +-
 ...nality_group_by_distributed_plan.reference |  55 ----------
 ..._cardinality_group_by_distributed_plan.sql |  80 --------------
 8 files changed, 56 insertions(+), 265 deletions(-)
 delete mode 100644 tests/queries/0_stateless/03174_low_cardinality_group_by_distributed_plan.reference
 delete mode 100644 tests/queries/0_stateless/03174_low_cardinality_group_by_distributed_plan.sql

diff --git a/src/Analyzer/Resolve/QueryAnalyzer.cpp b/src/Analyzer/Resolve/QueryAnalyzer.cpp
index a43e3e62ebc..b1fe2554988 100644
--- a/src/Analyzer/Resolve/QueryAnalyzer.cpp
+++ b/src/Analyzer/Resolve/QueryAnalyzer.cpp
@@ -4124,9 +4124,7 @@ void QueryAnalyzer::resolveInterpolateColumnsNodeList(QueryTreeNodePtr & interpo
 
         auto * column_to_interpolate = interpolate_node_typed.getExpression()->as<IdentifierNode>();
         if (!column_to_interpolate)
-            throw Exception(
-                ErrorCodes::LOGICAL_ERROR,
-                "INTERPOLATE can work only for identifiers, but {} is found",
+            throw Exception(ErrorCodes::LOGICAL_ERROR, "INTERPOLATE can work only for indentifiers, but {} is found",
                 interpolate_node_typed.getExpression()->formatASTForErrorMessage());
         auto column_to_interpolate_name = column_to_interpolate->getIdentifier().getFullName();
 
diff --git a/src/Functions/IFunction.cpp b/src/Functions/IFunction.cpp
index 8b092ba9b6e..31695fc95d5 100644
--- a/src/Functions/IFunction.cpp
+++ b/src/Functions/IFunction.cpp
@@ -47,85 +47,54 @@ bool allArgumentsAreConstants(const ColumnsWithTypeAndName & args)
     return true;
 }
 
-/// Replaces single low cardinality column in a function call by its dictionary
-/// This can only happen after the arguments have been adapted in IFunctionOverloadResolver::getReturnType
-/// as it's only possible if there is one low cardinality column and, optionally, const columns
 ColumnPtr replaceLowCardinalityColumnsByNestedAndGetDictionaryIndexes(
     ColumnsWithTypeAndName & args, bool can_be_executed_on_default_arguments, size_t input_rows_count)
 {
-    /// We return the LC indexes so the LC can be reconstructed with the function result
+    size_t num_rows = input_rows_count;
     ColumnPtr indexes;
 
-    size_t number_low_cardinality_columns = 0;
-    size_t last_low_cardinality = 0;
-    size_t number_const_columns = 0;
-    size_t number_full_columns = 0;
-
-    for (size_t i = 0; i < args.size(); i++)
+    /// Find first LowCardinality column and replace it to nested dictionary.
+    for (auto & column : args)
     {
-        auto const & arg = args[i];
-        if (checkAndGetColumn<ColumnLowCardinality>(arg.column.get()))
+        if (const auto * low_cardinality_column = checkAndGetColumn<ColumnLowCardinality>(column.column.get()))
         {
-            number_low_cardinality_columns++;
-            last_low_cardinality = i;
+            /// Single LowCardinality column is supported now.
+            if (indexes)
+                throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected single dictionary argument for function.");
+
+            const auto * low_cardinality_type = checkAndGetDataType<DataTypeLowCardinality>(column.type.get());
+
+            if (!low_cardinality_type)
+                throw Exception(ErrorCodes::LOGICAL_ERROR,
+                    "Incompatible type for LowCardinality column: {}",
+                    column.type->getName());
+
+            if (can_be_executed_on_default_arguments)
+            {
+                /// Normal case, when function can be executed on values' default.
+                column.column = low_cardinality_column->getDictionary().getNestedColumn();
+                indexes = low_cardinality_column->getIndexesPtr();
+            }
+            else
+            {
+                /// Special case when default value can't be used. Example: 1 % LowCardinality(Int).
+                /// LowCardinality always contains default, so 1 % 0 will throw exception in normal case.
+                auto dict_encoded = low_cardinality_column->getMinimalDictionaryEncodedColumn(0, low_cardinality_column->size());
+                column.column = dict_encoded.dictionary;
+                indexes = dict_encoded.indexes;
+            }
+
+            num_rows = column.column->size();
+            column.type = low_cardinality_type->getDictionaryType();
         }
-        else if (checkAndGetColumn<ColumnConst>(arg.column.get()))
-            number_const_columns++;
-        else
-            number_full_columns++;
     }
 
-    if (!number_low_cardinality_columns && !number_const_columns)
-        return nullptr;
-
-    if (number_full_columns > 0 || number_low_cardinality_columns > 1)
-    {
-        /// This should not be possible but currently there are multiple tests in CI failing because of it
-        /// TODO: Fix those cases, then enable this exception
-#if 0
-        throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected low cardinality types found. Low cardinality: {}. Full {}. Const {}",
-                number_low_cardinality_columns, number_full_columns, number_const_columns);
-#else
-        return nullptr;
-#endif
-    }
-    else if (number_low_cardinality_columns == 1)
-    {
-        auto & lc_arg = args[last_low_cardinality];
-
-        const auto * low_cardinality_type = checkAndGetDataType<DataTypeLowCardinality>(lc_arg.type.get());
-        if (!low_cardinality_type)
-            throw Exception(ErrorCodes::LOGICAL_ERROR, "Incompatible type for LowCardinality column: {}", lc_arg.type->getName());
-
-        const auto * low_cardinality_column = checkAndGetColumn<ColumnLowCardinality>(lc_arg.column.get());
-        chassert(low_cardinality_column);
-
-        if (can_be_executed_on_default_arguments)
-        {
-            /// Normal case, when function can be executed on values' default.
-            lc_arg.column = low_cardinality_column->getDictionary().getNestedColumn();
-            indexes = low_cardinality_column->getIndexesPtr();
-        }
-        else
-        {
-            /// Special case when default value can't be used. Example: 1 % LowCardinality(Int).
-            /// LowCardinality always contains default, so 1 % 0 will throw exception in normal case.
-            auto dict_encoded = low_cardinality_column->getMinimalDictionaryEncodedColumn(0, low_cardinality_column->size());
-            lc_arg.column = dict_encoded.dictionary;
-            indexes = dict_encoded.indexes;
-        }
-
-        /// The new column will have a different number of rows, normally less but occasionally it might be more (NULL)
-        input_rows_count = lc_arg.column->size();
-        lc_arg.type = low_cardinality_type->getDictionaryType();
-    }
-
-    /// Change size of constants
+    /// Change size of constants.
     for (auto & column : args)
     {
         if (const auto * column_const = checkAndGetColumn<ColumnConst>(column.column.get()))
         {
-            column.column = ColumnConst::create(recursiveRemoveLowCardinality(column_const->getDataColumnPtr()), input_rows_count);
+            column.column = ColumnConst::create(recursiveRemoveLowCardinality(column_const->getDataColumnPtr()), num_rows);
             column.type = recursiveRemoveLowCardinality(column.type);
         }
     }
@@ -301,8 +270,6 @@ ColumnPtr IExecutableFunction::executeWithoutSparseColumns(const ColumnsWithType
             bool can_be_executed_on_default_arguments = canBeExecutedOnDefaultArguments();
 
             const auto & dictionary_type = res_low_cardinality_type->getDictionaryType();
-            /// The arguments should have been adapted in IFunctionOverloadResolver::getReturnType
-            /// So there is only one low cardinality column (and optionally some const columns) and no full column
             ColumnPtr indexes = replaceLowCardinalityColumnsByNestedAndGetDictionaryIndexes(
                     columns_without_low_cardinality, can_be_executed_on_default_arguments, input_rows_count);
 
diff --git a/src/Planner/Planner.cpp b/src/Planner/Planner.cpp
index a35ba69d459..5640d84731f 100644
--- a/src/Planner/Planner.cpp
+++ b/src/Planner/Planner.cpp
@@ -745,12 +745,7 @@ void addWithFillStepIfNeeded(QueryPlan & query_plan,
             {
                 auto & interpolate_node_typed = interpolate_node->as<InterpolateNode &>();
 
-                PlannerActionsVisitor planner_actions_visitor(
-                    planner_context,
-                    /* use_column_identifier_as_action_node_name_, (default value)*/ true,
-                    /// Prefer the INPUT to CONSTANT nodes (actions must be non constant)
-                    /* always_use_const_column_for_constant_nodes */ false);
-
+                PlannerActionsVisitor planner_actions_visitor(planner_context);
                 auto expression_to_interpolate_expression_nodes = planner_actions_visitor.visit(*interpolate_actions_dag,
                     interpolate_node_typed.getExpression());
                 if (expression_to_interpolate_expression_nodes.size() != 1)
diff --git a/src/Planner/PlannerActionsVisitor.cpp b/src/Planner/PlannerActionsVisitor.cpp
index 6fd37df11c6..a0c7f9b2516 100644
--- a/src/Planner/PlannerActionsVisitor.cpp
+++ b/src/Planner/PlannerActionsVisitor.cpp
@@ -487,33 +487,16 @@ public:
         return node;
     }
 
-    [[nodiscard]] String addConstantIfNecessary(
-        const std::string & node_name, const ColumnWithTypeAndName & column, bool always_use_const_column_for_constant_nodes)
+    const ActionsDAG::Node * addConstantIfNecessary(const std::string & node_name, const ColumnWithTypeAndName & column)
     {
-        chassert(column.column != nullptr);
         auto it = node_name_to_node.find(node_name);
-        if (it != node_name_to_node.end() && (!always_use_const_column_for_constant_nodes || it->second->column))
-            return {node_name};
-
         if (it != node_name_to_node.end())
-        {
-            /// There is a node with this name, but it doesn't have a column
-            /// This likely happens because we executed the query until WithMergeableState with a const node in the
-            /// WHERE clause and, as the results of headers are materialized, the column was removed
-            /// Let's add a new column and keep this
-            String dupped_name{node_name + "_dupped"};
-            if (node_name_to_node.find(dupped_name) != node_name_to_node.end())
-                return dupped_name;
-
-            const auto * node = &actions_dag.addColumn(column);
-            node_name_to_node[dupped_name] = node;
-            return dupped_name;
-        }
+            return it->second;
 
         const auto * node = &actions_dag.addColumn(column);
         node_name_to_node[node->result_name] = node;
 
-        return {node_name};
+        return node;
     }
 
     template <typename FunctionOrOverloadResolver>
@@ -542,7 +525,7 @@ public:
     }
 
 private:
-    std::unordered_map<String, const ActionsDAG::Node *> node_name_to_node;
+    std::unordered_map<std::string_view, const ActionsDAG::Node *> node_name_to_node;
     ActionsDAG & actions_dag;
     QueryTreeNodePtr scope_node;
 };
@@ -550,11 +533,9 @@ private:
 class PlannerActionsVisitorImpl
 {
 public:
-    PlannerActionsVisitorImpl(
-        ActionsDAG & actions_dag,
+    PlannerActionsVisitorImpl(ActionsDAG & actions_dag,
         const PlannerContextPtr & planner_context_,
-        bool use_column_identifier_as_action_node_name_,
-        bool always_use_const_column_for_constant_nodes_);
+        bool use_column_identifier_as_action_node_name_);
 
     ActionsDAG::NodeRawConstPtrs visit(QueryTreeNodePtr expression_node);
 
@@ -614,18 +595,14 @@ private:
     const PlannerContextPtr planner_context;
     ActionNodeNameHelper action_node_name_helper;
     bool use_column_identifier_as_action_node_name;
-    bool always_use_const_column_for_constant_nodes;
 };
 
-PlannerActionsVisitorImpl::PlannerActionsVisitorImpl(
-    ActionsDAG & actions_dag,
+PlannerActionsVisitorImpl::PlannerActionsVisitorImpl(ActionsDAG & actions_dag,
     const PlannerContextPtr & planner_context_,
-    bool use_column_identifier_as_action_node_name_,
-    bool always_use_const_column_for_constant_nodes_)
+    bool use_column_identifier_as_action_node_name_)
     : planner_context(planner_context_)
     , action_node_name_helper(node_to_node_name, *planner_context, use_column_identifier_as_action_node_name_)
     , use_column_identifier_as_action_node_name(use_column_identifier_as_action_node_name_)
-    , always_use_const_column_for_constant_nodes(always_use_const_column_for_constant_nodes_)
 {
     actions_stack.emplace_back(actions_dag, nullptr);
 }
@@ -748,16 +725,17 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::vi
     column.type = constant_type;
     column.column = column.type->createColumnConst(1, constant_literal);
 
-    String final_name = actions_stack[0].addConstantIfNecessary(constant_node_name, column, always_use_const_column_for_constant_nodes);
+    actions_stack[0].addConstantIfNecessary(constant_node_name, column);
 
     size_t actions_stack_size = actions_stack.size();
     for (size_t i = 1; i < actions_stack_size; ++i)
     {
         auto & actions_stack_node = actions_stack[i];
-        actions_stack_node.addInputConstantColumnIfNecessary(final_name, column);
+        actions_stack_node.addInputConstantColumnIfNecessary(constant_node_name, column);
     }
 
-    return {final_name, Levels(0)};
+    return {constant_node_name, Levels(0)};
+
 }
 
 PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::visitLambda(const QueryTreeNodePtr & node)
@@ -886,16 +864,16 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::ma
     else
         column.column = std::move(column_set);
 
-    String final_name = actions_stack[0].addConstantIfNecessary(column.name, column, always_use_const_column_for_constant_nodes);
+    actions_stack[0].addConstantIfNecessary(column.name, column);
 
     size_t actions_stack_size = actions_stack.size();
     for (size_t i = 1; i < actions_stack_size; ++i)
     {
         auto & actions_stack_node = actions_stack[i];
-        actions_stack_node.addInputConstantColumnIfNecessary(final_name, column);
+        actions_stack_node.addInputConstantColumnIfNecessary(column.name, column);
     }
 
-    return {final_name, Levels(0)};
+    return {column.name, Levels(0)};
 }
 
 PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::visitIndexHintFunction(const QueryTreeNodePtr & node)
@@ -1032,19 +1010,14 @@ PlannerActionsVisitorImpl::NodeNameAndNodeMinLevel PlannerActionsVisitorImpl::vi
 
 }
 
-PlannerActionsVisitor::PlannerActionsVisitor(
-    const PlannerContextPtr & planner_context_,
-    bool use_column_identifier_as_action_node_name_,
-    bool always_use_const_column_for_constant_nodes_)
+PlannerActionsVisitor::PlannerActionsVisitor(const PlannerContextPtr & planner_context_, bool use_column_identifier_as_action_node_name_)
     : planner_context(planner_context_)
     , use_column_identifier_as_action_node_name(use_column_identifier_as_action_node_name_)
-    , always_use_const_column_for_constant_nodes(always_use_const_column_for_constant_nodes_)
 {}
 
 ActionsDAG::NodeRawConstPtrs PlannerActionsVisitor::visit(ActionsDAG & actions_dag, QueryTreeNodePtr expression_node)
 {
-    PlannerActionsVisitorImpl actions_visitor_impl(
-        actions_dag, planner_context, use_column_identifier_as_action_node_name, always_use_const_column_for_constant_nodes);
+    PlannerActionsVisitorImpl actions_visitor_impl(actions_dag, planner_context, use_column_identifier_as_action_node_name);
     return actions_visitor_impl.visit(expression_node);
 }
 
diff --git a/src/Planner/PlannerActionsVisitor.h b/src/Planner/PlannerActionsVisitor.h
index 1dbd149bc4b..6bb32047327 100644
--- a/src/Planner/PlannerActionsVisitor.h
+++ b/src/Planner/PlannerActionsVisitor.h
@@ -27,17 +27,11 @@ using PlannerContextPtr = std::shared_ptr<PlannerContext>;
   * During actions build, there is special handling for following functions:
   * 1. Aggregate functions are added in actions dag as INPUT nodes. Aggregate functions arguments are not added.
   * 2. For function `in` and its variants, already collected sets from planner context are used.
-  * 3. When building actions that use CONSTANT nodes, by default we ignore pre-existing INPUTs if those don't have
-  * a column (a const column always has a column). This is for compatibility with previous headers. We disable this
-  * behaviour when we explicitly want to override CONSTANT nodes with the input (resolving InterpolateNode for example)
   */
 class PlannerActionsVisitor
 {
 public:
-    explicit PlannerActionsVisitor(
-        const PlannerContextPtr & planner_context_,
-        bool use_column_identifier_as_action_node_name_ = true,
-        bool always_use_const_column_for_constant_nodes_ = true);
+    explicit PlannerActionsVisitor(const PlannerContextPtr & planner_context_, bool use_column_identifier_as_action_node_name_ = true);
 
     /** Add actions necessary to calculate expression node into expression dag.
       * Necessary actions are not added in actions dag output.
@@ -48,7 +42,6 @@ public:
 private:
     const PlannerContextPtr planner_context;
     bool use_column_identifier_as_action_node_name = true;
-    bool always_use_const_column_for_constant_nodes = true;
 };
 
 /** Calculate query tree expression node action dag name and add them into node to name map.
diff --git a/tests/queries/0_stateless/00257_shard_no_aggregates_and_constant_keys.reference b/tests/queries/0_stateless/00257_shard_no_aggregates_and_constant_keys.reference
index fc77ed8a241..63b8a9d14fc 100644
--- a/tests/queries/0_stateless/00257_shard_no_aggregates_and_constant_keys.reference
+++ b/tests/queries/0_stateless/00257_shard_no_aggregates_and_constant_keys.reference
@@ -8,13 +8,13 @@
 40
 41
 
-41
+0
 2	42
 
 2	42
 43
 
-43
+0
 11
 
 11
diff --git a/tests/queries/0_stateless/03174_low_cardinality_group_by_distributed_plan.reference b/tests/queries/0_stateless/03174_low_cardinality_group_by_distributed_plan.reference
deleted file mode 100644
index 1508c24f410..00000000000
--- a/tests/queries/0_stateless/03174_low_cardinality_group_by_distributed_plan.reference
+++ /dev/null
@@ -1,55 +0,0 @@
--- { echoOn }
-SELECT concatWithSeparator('.', toUInt128(6), '666' as b, materialize(toLowCardinality(8)))
-FROM system.one
-GROUP BY '666';
-6.666.8
-SELECT concatWithSeparator('.', toUInt128(6), '666' as b, materialize(toLowCardinality(8)))
-FROM remote('127.0.0.{1,1}', 'system.one')
-GROUP BY '666';
-6.666.8
--- https://github.com/ClickHouse/ClickHouse/issues/63006
-SELECT
-    6,
-    concat(' World', toUInt128(6), 6, 6, 6, toNullable(6), materialize(toLowCardinality(toNullable(toUInt128(6))))) AS a,
-    concat(concat(' World', 6, toLowCardinality(6), ' World', toUInt256(6), materialize(6), 6, toNullable(6), 6, 6, NULL, 6, 6), ' World', 6, 6, 6, 6, toUInt256(6), NULL, 6, 6) AS b
-FROM system.one
-GROUP BY toNullable(6)
-    WITH ROLLUP
-WITH TOTALS;
-6	 World666666	\N
-6	 World666666	\N
-
-6	 World666666	\N
-SELECT
-    6,
-    concat(' World', toUInt128(6), 6, 6, 6, toNullable(6), materialize(toLowCardinality(toNullable(toUInt128(6))))) AS a,
-    concat(concat(' World', 6, toLowCardinality(6), ' World', toUInt256(6), materialize(6), 6, toNullable(6), 6, 6, NULL, 6, 6), ' World', 6, 6, 6, 6, toUInt256(6), NULL, 6, 6) AS b
-FROM remote('127.0.0.1')
-GROUP BY toNullable(6)
-    WITH ROLLUP
-    WITH TOTALS;
-6	 World666666	\N
-6	 World666666	\N
-
-6	 World666666	\N
--- { echoOn }
-SELECT
-    '%',
-    tuple(concat('%', 1, toLowCardinality(toLowCardinality(toNullable(materialize(1)))), currentDatabase(), 101., toNullable(13), '%AS%id_02%', toNullable(toNullable(10)), toLowCardinality(toNullable(10)), 10, 10)),
-    (toDecimal128(99.67, 6), 36, 61, 14)
-FROM dist_03174
-WHERE dummy IN (0, '255')
-GROUP BY
-    toNullable(13),
-    (99.67, 61, toLowCardinality(14));
-%	('%11default10113%AS%id_02%10101010')	(99.67,36,61,14)
--- { echoOn }
-SELECT
-    38,
-    concat(position(concat(concat(position(concat(toUInt256(3)), 'ca', 2), 3), NULLIF(1, materialize(toLowCardinality(1)))), toLowCardinality(toNullable('ca'))), concat(NULLIF(1, 1), concat(3), toNullable(3)))
-FROM set_index_not__fuzz_0
-GROUP BY
-    toNullable(3),
-    concat(concat(CAST(NULL, 'Nullable(Int8)'), toNullable(3)))
-FORMAT Null
-SETTINGS max_threads = 1, allow_experimental_analyzer = 1, cluster_for_parallel_replicas = 'parallel_replicas', max_parallel_replicas = 3, allow_experimental_parallel_reading_from_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, max_threads = 1;
diff --git a/tests/queries/0_stateless/03174_low_cardinality_group_by_distributed_plan.sql b/tests/queries/0_stateless/03174_low_cardinality_group_by_distributed_plan.sql
deleted file mode 100644
index d397d30e285..00000000000
--- a/tests/queries/0_stateless/03174_low_cardinality_group_by_distributed_plan.sql
+++ /dev/null
@@ -1,80 +0,0 @@
--- There are various tests that check that group by keys don't propagate into functions replacing const arguments
--- by full (empty) columns
-
-DROP TABLE IF EXISTS dist_03174;
-DROP TABLE IF EXISTS set_index_not__fuzz_0;
-
--- https://github.com/ClickHouse/ClickHouse/issues/63006
-
-SET allow_experimental_analyzer=1;
-
--- { echoOn }
-SELECT concatWithSeparator('.', toUInt128(6), '666' as b, materialize(toLowCardinality(8)))
-FROM system.one
-GROUP BY '666';
-
-SELECT concatWithSeparator('.', toUInt128(6), '666' as b, materialize(toLowCardinality(8)))
-FROM remote('127.0.0.{1,1}', 'system.one')
-GROUP BY '666';
-
--- https://github.com/ClickHouse/ClickHouse/issues/63006
-SELECT
-    6,
-    concat(' World', toUInt128(6), 6, 6, 6, toNullable(6), materialize(toLowCardinality(toNullable(toUInt128(6))))) AS a,
-    concat(concat(' World', 6, toLowCardinality(6), ' World', toUInt256(6), materialize(6), 6, toNullable(6), 6, 6, NULL, 6, 6), ' World', 6, 6, 6, 6, toUInt256(6), NULL, 6, 6) AS b
-FROM system.one
-GROUP BY toNullable(6)
-    WITH ROLLUP
-WITH TOTALS;
-
-SELECT
-    6,
-    concat(' World', toUInt128(6), 6, 6, 6, toNullable(6), materialize(toLowCardinality(toNullable(toUInt128(6))))) AS a,
-    concat(concat(' World', 6, toLowCardinality(6), ' World', toUInt256(6), materialize(6), 6, toNullable(6), 6, 6, NULL, 6, 6), ' World', 6, 6, 6, 6, toUInt256(6), NULL, 6, 6) AS b
-FROM remote('127.0.0.1')
-GROUP BY toNullable(6)
-    WITH ROLLUP
-    WITH TOTALS;
-
--- https://github.com/ClickHouse/ClickHouse/issues/64945
--- { echoOff }
-CREATE TABLE dist_03174 AS system.one ENGINE = Distributed(test_cluster_two_shards, system, one, dummy);
-
--- { echoOn }
-SELECT
-    '%',
-    tuple(concat('%', 1, toLowCardinality(toLowCardinality(toNullable(materialize(1)))), currentDatabase(), 101., toNullable(13), '%AS%id_02%', toNullable(toNullable(10)), toLowCardinality(toNullable(10)), 10, 10)),
-    (toDecimal128(99.67, 6), 36, 61, 14)
-FROM dist_03174
-WHERE dummy IN (0, '255')
-GROUP BY
-    toNullable(13),
-    (99.67, 61, toLowCardinality(14));
-
--- Parallel replicas
--- { echoOff }
-CREATE TABLE set_index_not__fuzz_0
-(
-    `name` String,
-    `status` Enum8('alive' = 0, 'rip' = 1),
-    INDEX idx_status status TYPE set(2) GRANULARITY 1
-)
-ENGINE = MergeTree()
-ORDER BY name;
-
-INSERT INTO set_index_not__fuzz_0 SELECT * FROM generateRandom() LIMIT 10;
-
--- { echoOn }
-SELECT
-    38,
-    concat(position(concat(concat(position(concat(toUInt256(3)), 'ca', 2), 3), NULLIF(1, materialize(toLowCardinality(1)))), toLowCardinality(toNullable('ca'))), concat(NULLIF(1, 1), concat(3), toNullable(3)))
-FROM set_index_not__fuzz_0
-GROUP BY
-    toNullable(3),
-    concat(concat(CAST(NULL, 'Nullable(Int8)'), toNullable(3)))
-FORMAT Null
-SETTINGS max_threads = 1, allow_experimental_analyzer = 1, cluster_for_parallel_replicas = 'parallel_replicas', max_parallel_replicas = 3, allow_experimental_parallel_reading_from_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, max_threads = 1;
-
--- { echoOff }
-DROP TABLE IF EXISTS dist_03174;
-DROP TABLE IF EXISTS set_index_not__fuzz_0;

From 104c3be7b691a8e2c7b8c71789d60868aa062e01 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Mon, 22 Jul 2024 16:23:51 +0000
Subject: [PATCH 0659/1488] Functions [h-r]*: Iterate over input_rows_count
 where appropriate

---
 src/Functions/FunctionStringToString.h        |  6 ++--
 src/Functions/LowerUpperImpl.h                |  8 +++--
 src/Functions/LowerUpperUTF8Impl.h            |  7 +++--
 src/Functions/StringHelpers.h                 | 28 +++++++++--------
 .../URL/ExtractFirstSignificantSubdomain.h    |  4 +--
 src/Functions/URL/cutFragment.cpp             |  2 +-
 src/Functions/URL/cutQueryString.cpp          |  2 +-
 .../URL/cutQueryStringAndFragment.cpp         |  2 +-
 .../URL/cutToFirstSignificantSubdomain.cpp    |  2 +-
 .../cutToFirstSignificantSubdomainCustom.cpp  |  4 +--
 src/Functions/URL/cutWWW.cpp                  |  2 +-
 src/Functions/URL/decodeURLComponent.cpp      | 15 +++++-----
 src/Functions/URL/domain.cpp                  |  3 +-
 src/Functions/URL/domain.h                    |  7 +++--
 src/Functions/URL/domainWithoutWWW.cpp        |  2 +-
 .../URL/firstSignificantSubdomain.cpp         |  2 +-
 .../URL/firstSignificantSubdomainCustom.cpp   |  4 +--
 src/Functions/URL/fragment.cpp                |  2 +-
 src/Functions/URL/path.cpp                    |  2 +-
 src/Functions/URL/pathFull.cpp                |  2 +-
 src/Functions/URL/port.cpp                    |  2 +-
 src/Functions/URL/protocol.cpp                |  2 +-
 src/Functions/URL/queryString.cpp             |  2 +-
 src/Functions/URL/queryStringAndFragment.cpp  |  2 +-
 src/Functions/URL/topLevelDomain.cpp          |  2 +-
 src/Functions/decodeHTMLComponent.cpp         | 11 ++++---
 src/Functions/decodeXMLComponent.cpp          | 10 +++----
 src/Functions/encodeXMLComponent.cpp          | 10 +++----
 src/Functions/idna.cpp                        | 20 ++++++-------
 src/Functions/initcap.cpp                     |  8 +++--
 src/Functions/initcapUTF8.cpp                 |  5 ++--
 src/Functions/normalizeQuery.cpp              | 14 +++++----
 src/Functions/normalizeString.cpp             | 10 +++----
 src/Functions/normalizedQueryHash.cpp         | 14 ++++-----
 src/Functions/pointInEllipses.cpp             | 10 ++-----
 src/Functions/punycode.cpp                    | 30 +++++++++----------
 src/Functions/reverse.cpp                     |  6 ++--
 src/Functions/reverse.h                       | 18 ++++++-----
 src/Functions/reverseUTF8.cpp                 | 14 ++++-----
 src/Functions/soundex.cpp                     | 12 ++++----
 src/Functions/toValidUTF8.cpp                 | 10 +++----
 src/Functions/trim.cpp                        | 10 +++----
 42 files changed, 169 insertions(+), 159 deletions(-)

diff --git a/src/Functions/FunctionStringToString.h b/src/Functions/FunctionStringToString.h
index 62d7b09f79e..e0e64e47b49 100644
--- a/src/Functions/FunctionStringToString.h
+++ b/src/Functions/FunctionStringToString.h
@@ -59,19 +59,19 @@ public:
 
     bool useDefaultImplementationForConstants() const override { return true; }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
         const ColumnPtr column = arguments[0].column;
         if (const ColumnString * col = checkAndGetColumn<ColumnString>(column.get()))
         {
             auto col_res = ColumnString::create();
-            Impl::vector(col->getChars(), col->getOffsets(), col_res->getChars(), col_res->getOffsets());
+            Impl::vector(col->getChars(), col->getOffsets(), col_res->getChars(), col_res->getOffsets(), input_rows_count);
             return col_res;
         }
         else if (const ColumnFixedString * col_fixed = checkAndGetColumn<ColumnFixedString>(column.get()))
         {
             auto col_res = ColumnFixedString::create(col_fixed->getN());
-            Impl::vectorFixed(col_fixed->getChars(), col_fixed->getN(), col_res->getChars());
+            Impl::vectorFixed(col_fixed->getChars(), col_fixed->getN(), col_res->getChars(), input_rows_count);
             return col_res;
         }
         else
diff --git a/src/Functions/LowerUpperImpl.h b/src/Functions/LowerUpperImpl.h
index 72b3ce1ca34..d463ef96e16 100644
--- a/src/Functions/LowerUpperImpl.h
+++ b/src/Functions/LowerUpperImpl.h
@@ -8,17 +8,19 @@ namespace DB
 template <char not_case_lower_bound, char not_case_upper_bound>
 struct LowerUpperImpl
 {
-    static void vector(const ColumnString::Chars & data,
+    static void vector(
+        const ColumnString::Chars & data,
         const ColumnString::Offsets & offsets,
         ColumnString::Chars & res_data,
-        ColumnString::Offsets & res_offsets)
+        ColumnString::Offsets & res_offsets,
+        size_t /*input_rows_count*/)
     {
         res_data.resize_exact(data.size());
         res_offsets.assign(offsets);
         array(data.data(), data.data() + data.size(), res_data.data());
     }
 
-    static void vectorFixed(const ColumnString::Chars & data, size_t /*n*/, ColumnString::Chars & res_data)
+    static void vectorFixed(const ColumnString::Chars & data, size_t /*n*/, ColumnString::Chars & res_data, size_t /*input_rows_count*/)
     {
         res_data.resize_exact(data.size());
         array(data.data(), data.data() + data.size(), res_data.data());
diff --git a/src/Functions/LowerUpperUTF8Impl.h b/src/Functions/LowerUpperUTF8Impl.h
index eebba7b9d5f..eedabca5b22 100644
--- a/src/Functions/LowerUpperUTF8Impl.h
+++ b/src/Functions/LowerUpperUTF8Impl.h
@@ -90,7 +90,8 @@ struct LowerUpperUTF8Impl
         const ColumnString::Chars & data,
         const ColumnString::Offsets & offsets,
         ColumnString::Chars & res_data,
-        ColumnString::Offsets & res_offsets)
+        ColumnString::Offsets & res_offsets,
+        size_t input_rows_count)
     {
         if (data.empty())
             return;
@@ -98,7 +99,7 @@ struct LowerUpperUTF8Impl
         bool all_ascii = isAllASCII(data.data(), data.size());
         if (all_ascii)
         {
-            LowerUpperImpl<not_case_lower_bound, not_case_upper_bound>::vector(data, offsets, res_data, res_offsets);
+            LowerUpperImpl<not_case_lower_bound, not_case_upper_bound>::vector(data, offsets, res_data, res_offsets, input_rows_count);
             return;
         }
 
@@ -107,7 +108,7 @@ struct LowerUpperUTF8Impl
         array(data.data(), data.data() + data.size(), offsets, res_data.data());
     }
 
-    static void vectorFixed(const ColumnString::Chars &, size_t, ColumnString::Chars &)
+    static void vectorFixed(const ColumnString::Chars &, size_t, ColumnString::Chars &, size_t)
     {
         throw Exception(ErrorCodes::BAD_ARGUMENTS, "Functions lowerUTF8 and upperUTF8 cannot work with FixedString argument");
     }
diff --git a/src/Functions/StringHelpers.h b/src/Functions/StringHelpers.h
index 8f3a87d5d0e..24d8fe86c62 100644
--- a/src/Functions/StringHelpers.h
+++ b/src/Functions/StringHelpers.h
@@ -62,12 +62,13 @@ using Pos = const char *;
 template <typename Extractor>
 struct ExtractSubstringImpl
 {
-    static void vector(const ColumnString::Chars & data, const ColumnString::Offsets & offsets,
-        ColumnString::Chars & res_data, ColumnString::Offsets & res_offsets)
+    static void vector(
+        const ColumnString::Chars & data, const ColumnString::Offsets & offsets,
+        ColumnString::Chars & res_data, ColumnString::Offsets & res_offsets,
+        size_t input_rows_count)
     {
-        size_t size = offsets.size();
-        res_offsets.resize(size);
-        res_data.reserve(size * Extractor::getReserveLengthForElement());
+        res_offsets.resize(input_rows_count);
+        res_data.reserve(input_rows_count * Extractor::getReserveLengthForElement());
 
         size_t prev_offset = 0;
         size_t res_offset = 0;
@@ -76,7 +77,7 @@ struct ExtractSubstringImpl
         Pos start;
         size_t length;
 
-        for (size_t i = 0; i < size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             Extractor::execute(reinterpret_cast<const char *>(&data[prev_offset]), offsets[i] - prev_offset - 1, start, length);
 
@@ -99,7 +100,7 @@ struct ExtractSubstringImpl
         res_data.assign(start, length);
     }
 
-    static void vectorFixed(const ColumnString::Chars &, size_t, ColumnString::Chars &)
+    static void vectorFixed(const ColumnString::Chars &, size_t, ColumnString::Chars &, size_t)
     {
         throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Column of type FixedString is not supported by this function");
     }
@@ -111,12 +112,13 @@ struct ExtractSubstringImpl
 template <typename Extractor>
 struct CutSubstringImpl
 {
-    static void vector(const ColumnString::Chars & data, const ColumnString::Offsets & offsets,
-        ColumnString::Chars & res_data, ColumnString::Offsets & res_offsets)
+    static void vector(
+        const ColumnString::Chars & data, const ColumnString::Offsets & offsets,
+        ColumnString::Chars & res_data, ColumnString::Offsets & res_offsets,
+        size_t input_rows_count)
     {
         res_data.reserve(data.size());
-        size_t size = offsets.size();
-        res_offsets.resize(size);
+        res_offsets.resize(input_rows_count);
 
         size_t prev_offset = 0;
         size_t res_offset = 0;
@@ -125,7 +127,7 @@ struct CutSubstringImpl
         Pos start;
         size_t length;
 
-        for (size_t i = 0; i < size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             const char * current = reinterpret_cast<const char *>(&data[prev_offset]);
             Extractor::execute(current, offsets[i] - prev_offset - 1, start, length);
@@ -154,7 +156,7 @@ struct CutSubstringImpl
         res_data.append(start + length, data.data() + data.size());
     }
 
-    static void vectorFixed(const ColumnString::Chars &, size_t, ColumnString::Chars &)
+    static void vectorFixed(const ColumnString::Chars &, size_t, ColumnString::Chars &, size_t)
     {
         throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Column of type FixedString is not supported by this function");
     }
diff --git a/src/Functions/URL/ExtractFirstSignificantSubdomain.h b/src/Functions/URL/ExtractFirstSignificantSubdomain.h
index 0d1b1cac8ef..4316fd2fc3a 100644
--- a/src/Functions/URL/ExtractFirstSignificantSubdomain.h
+++ b/src/Functions/URL/ExtractFirstSignificantSubdomain.h
@@ -1,8 +1,8 @@
 #pragma once
 
 #include <base/find_symbols.h>
-#include "domain.h"
-#include "tldLookup.h"
+#include <Functions/URL/domain.h>
+#include <Functions/URL/tldLookup.h>
 #include <Common/TLDListsHolder.h> /// TLDType
 
 namespace DB
diff --git a/src/Functions/URL/cutFragment.cpp b/src/Functions/URL/cutFragment.cpp
index 3b99edf61a3..b32c4410190 100644
--- a/src/Functions/URL/cutFragment.cpp
+++ b/src/Functions/URL/cutFragment.cpp
@@ -1,6 +1,6 @@
 #include <Functions/FunctionFactory.h>
-#include "fragment.h"
 #include <Functions/FunctionStringToString.h>
+#include <Functions/URL/fragment.h>
 
 namespace DB
 {
diff --git a/src/Functions/URL/cutQueryString.cpp b/src/Functions/URL/cutQueryString.cpp
index 2886adc2e36..d2fa4a004f2 100644
--- a/src/Functions/URL/cutQueryString.cpp
+++ b/src/Functions/URL/cutQueryString.cpp
@@ -1,6 +1,6 @@
 #include <Functions/FunctionFactory.h>
-#include "queryString.h"
 #include <Functions/FunctionStringToString.h>
+#include <Functions/URL/queryString.h>
 
 namespace DB
 {
diff --git a/src/Functions/URL/cutQueryStringAndFragment.cpp b/src/Functions/URL/cutQueryStringAndFragment.cpp
index 4116b352542..ff427beb277 100644
--- a/src/Functions/URL/cutQueryStringAndFragment.cpp
+++ b/src/Functions/URL/cutQueryStringAndFragment.cpp
@@ -1,6 +1,6 @@
 #include <Functions/FunctionFactory.h>
-#include "queryStringAndFragment.h"
 #include <Functions/FunctionStringToString.h>
+#include <Functions/URL/queryStringAndFragment.h>
 
 namespace DB
 {
diff --git a/src/Functions/URL/cutToFirstSignificantSubdomain.cpp b/src/Functions/URL/cutToFirstSignificantSubdomain.cpp
index 6e64b0b6ab8..454a241c54f 100644
--- a/src/Functions/URL/cutToFirstSignificantSubdomain.cpp
+++ b/src/Functions/URL/cutToFirstSignificantSubdomain.cpp
@@ -1,6 +1,6 @@
 #include <Functions/FunctionFactory.h>
 #include <Functions/FunctionStringToString.h>
-#include "ExtractFirstSignificantSubdomain.h"
+#include <Functions/URL/ExtractFirstSignificantSubdomain.h>
 
 
 namespace DB
diff --git a/src/Functions/URL/cutToFirstSignificantSubdomainCustom.cpp b/src/Functions/URL/cutToFirstSignificantSubdomainCustom.cpp
index 77f40e465a6..7612b6ea4af 100644
--- a/src/Functions/URL/cutToFirstSignificantSubdomainCustom.cpp
+++ b/src/Functions/URL/cutToFirstSignificantSubdomainCustom.cpp
@@ -1,6 +1,6 @@
 #include <Functions/FunctionFactory.h>
-#include "ExtractFirstSignificantSubdomain.h"
-#include "FirstSignificantSubdomainCustomImpl.h"
+#include <Functions/URL/ExtractFirstSignificantSubdomain.h>
+#include <Functions/URL/FirstSignificantSubdomainCustomImpl.h>
 
 namespace DB
 {
diff --git a/src/Functions/URL/cutWWW.cpp b/src/Functions/URL/cutWWW.cpp
index 992d5128440..ab3fae6b094 100644
--- a/src/Functions/URL/cutWWW.cpp
+++ b/src/Functions/URL/cutWWW.cpp
@@ -1,6 +1,6 @@
 #include <Functions/FunctionFactory.h>
 #include <Functions/FunctionStringToString.h>
-#include "protocol.h"
+#include <Functions/URL/protocol.h>
 #include <base/find_symbols.h>
 
 
diff --git a/src/Functions/URL/decodeURLComponent.cpp b/src/Functions/URL/decodeURLComponent.cpp
index 05e3fbea3fd..bf4aaa6d5e3 100644
--- a/src/Functions/URL/decodeURLComponent.cpp
+++ b/src/Functions/URL/decodeURLComponent.cpp
@@ -1,7 +1,7 @@
-#include <base/hex.h>
 #include <Functions/FunctionFactory.h>
 #include <Functions/FunctionStringToString.h>
 #include <base/find_symbols.h>
+#include <base/hex.h>
 
 
 namespace DB
@@ -121,8 +121,10 @@ enum URLCodeStrategy
 template <URLCodeStrategy code_strategy, bool space_as_plus>
 struct CodeURLComponentImpl
 {
-    static void vector(const ColumnString::Chars & data, const ColumnString::Offsets & offsets,
-        ColumnString::Chars & res_data, ColumnString::Offsets & res_offsets)
+    static void vector(
+        const ColumnString::Chars & data, const ColumnString::Offsets & offsets,
+        ColumnString::Chars & res_data, ColumnString::Offsets & res_offsets,
+        size_t input_rows_count)
     {
         if (code_strategy == encode)
         {
@@ -134,13 +136,12 @@ struct CodeURLComponentImpl
             res_data.resize(data.size());
         }
 
-        size_t size = offsets.size();
-        res_offsets.resize(size);
+        res_offsets.resize(input_rows_count);
 
         size_t prev_offset = 0;
         size_t res_offset = 0;
 
-        for (size_t i = 0; i < size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             const char * src_data = reinterpret_cast<const char *>(&data[prev_offset]);
             size_t src_size = offsets[i] - prev_offset;
@@ -165,7 +166,7 @@ struct CodeURLComponentImpl
         res_data.resize(res_offset);
     }
 
-    [[noreturn]] static void vectorFixed(const ColumnString::Chars &, size_t, ColumnString::Chars &)
+    [[noreturn]] static void vectorFixed(const ColumnString::Chars &, size_t, ColumnString::Chars &, size_t)
     {
         throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Column of type FixedString is not supported by URL functions");
     }
diff --git a/src/Functions/URL/domain.cpp b/src/Functions/URL/domain.cpp
index 443a3323075..68724b57a70 100644
--- a/src/Functions/URL/domain.cpp
+++ b/src/Functions/URL/domain.cpp
@@ -1,5 +1,4 @@
-#include "domain.h"
-
+#include <Functions/URL/domain.h>
 #include <Functions/FunctionFactory.h>
 #include <Functions/FunctionStringToString.h>
 
diff --git a/src/Functions/URL/domain.h b/src/Functions/URL/domain.h
index 936fb9d5f00..328df76b570 100644
--- a/src/Functions/URL/domain.h
+++ b/src/Functions/URL/domain.h
@@ -1,9 +1,10 @@
 #pragma once
 
-#include "protocol.h"
-#include <base/find_symbols.h>
-#include <cstring>
 #include <Common/StringUtils.h>
+#include <Functions/URL/protocol.h>
+#include <base/find_symbols.h>
+
+#include <cstring>
 
 namespace DB
 {
diff --git a/src/Functions/URL/domainWithoutWWW.cpp b/src/Functions/URL/domainWithoutWWW.cpp
index f6c8b5c84fc..436aad01b82 100644
--- a/src/Functions/URL/domainWithoutWWW.cpp
+++ b/src/Functions/URL/domainWithoutWWW.cpp
@@ -1,6 +1,6 @@
 #include <Functions/FunctionFactory.h>
 #include <Functions/FunctionStringToString.h>
-#include "domain.h"
+#include <Functions/URL/domain.h>
 
 namespace DB
 {
diff --git a/src/Functions/URL/firstSignificantSubdomain.cpp b/src/Functions/URL/firstSignificantSubdomain.cpp
index b04f6d882ef..e929aefbc27 100644
--- a/src/Functions/URL/firstSignificantSubdomain.cpp
+++ b/src/Functions/URL/firstSignificantSubdomain.cpp
@@ -1,6 +1,6 @@
 #include <Functions/FunctionFactory.h>
 #include <Functions/FunctionStringToString.h>
-#include "ExtractFirstSignificantSubdomain.h"
+#include <Functions/URL/ExtractFirstSignificantSubdomain.h>
 
 
 namespace DB
diff --git a/src/Functions/URL/firstSignificantSubdomainCustom.cpp b/src/Functions/URL/firstSignificantSubdomainCustom.cpp
index c07aa2b3ac8..95e5142667b 100644
--- a/src/Functions/URL/firstSignificantSubdomainCustom.cpp
+++ b/src/Functions/URL/firstSignificantSubdomainCustom.cpp
@@ -1,6 +1,6 @@
 #include <Functions/FunctionFactory.h>
-#include "ExtractFirstSignificantSubdomain.h"
-#include "FirstSignificantSubdomainCustomImpl.h"
+#include <Functions/URL/ExtractFirstSignificantSubdomain.h>
+#include <Functions/URL/FirstSignificantSubdomainCustomImpl.h>
 
 
 namespace DB
diff --git a/src/Functions/URL/fragment.cpp b/src/Functions/URL/fragment.cpp
index 262c1a4c7a6..66da5529d84 100644
--- a/src/Functions/URL/fragment.cpp
+++ b/src/Functions/URL/fragment.cpp
@@ -1,6 +1,6 @@
 #include <Functions/FunctionFactory.h>
 #include <Functions/FunctionStringToString.h>
-#include "fragment.h"
+#include <Functions/URL/fragment.h>
 
 namespace DB
 {
diff --git a/src/Functions/URL/path.cpp b/src/Functions/URL/path.cpp
index 8d609f43191..6de8e4fbf95 100644
--- a/src/Functions/URL/path.cpp
+++ b/src/Functions/URL/path.cpp
@@ -1,7 +1,7 @@
 #include <Functions/FunctionFactory.h>
 #include <Functions/FunctionStringToString.h>
 #include <Functions/StringHelpers.h>
-#include "path.h"
+#include <Functions/URL/path.h>
 #include <base/find_symbols.h>
 
 
diff --git a/src/Functions/URL/pathFull.cpp b/src/Functions/URL/pathFull.cpp
index 9aacee21fed..deea617eb29 100644
--- a/src/Functions/URL/pathFull.cpp
+++ b/src/Functions/URL/pathFull.cpp
@@ -1,7 +1,7 @@
 #include <Functions/FunctionFactory.h>
 #include <Functions/FunctionStringToString.h>
 #include <Functions/StringHelpers.h>
-#include "path.h"
+#include <Functions/URL/path.h>
 #include <base/find_symbols.h>
 
 namespace DB
diff --git a/src/Functions/URL/port.cpp b/src/Functions/URL/port.cpp
index c8f50f10a56..fac46281604 100644
--- a/src/Functions/URL/port.cpp
+++ b/src/Functions/URL/port.cpp
@@ -5,7 +5,7 @@
 #include <Columns/ColumnsNumber.h>
 #include <Columns/ColumnArray.h>
 #include <Columns/ColumnConst.h>
-#include "domain.h"
+#include <Functions/URL/domain.h>
 
 
 namespace DB
diff --git a/src/Functions/URL/protocol.cpp b/src/Functions/URL/protocol.cpp
index 6bed71207f6..1ac395dc554 100644
--- a/src/Functions/URL/protocol.cpp
+++ b/src/Functions/URL/protocol.cpp
@@ -1,6 +1,6 @@
 #include <Functions/FunctionFactory.h>
 #include <Functions/FunctionStringToString.h>
-#include "protocol.h"
+#include <Functions/URL/protocol.h>
 
 
 namespace DB
diff --git a/src/Functions/URL/queryString.cpp b/src/Functions/URL/queryString.cpp
index 7069ce46b38..f6aaf40fc96 100644
--- a/src/Functions/URL/queryString.cpp
+++ b/src/Functions/URL/queryString.cpp
@@ -1,6 +1,6 @@
 #include <Functions/FunctionFactory.h>
 #include <Functions/FunctionStringToString.h>
-#include "queryString.h"
+#include <Functions/URL/queryString.h>
 
 namespace DB
 {
diff --git a/src/Functions/URL/queryStringAndFragment.cpp b/src/Functions/URL/queryStringAndFragment.cpp
index 367a95acbdc..94f1dfa41e3 100644
--- a/src/Functions/URL/queryStringAndFragment.cpp
+++ b/src/Functions/URL/queryStringAndFragment.cpp
@@ -1,6 +1,6 @@
 #include <Functions/FunctionFactory.h>
 #include <Functions/FunctionStringToString.h>
-#include "queryStringAndFragment.h"
+#include <Functions/URL/queryStringAndFragment.h>
 
 namespace DB
 {
diff --git a/src/Functions/URL/topLevelDomain.cpp b/src/Functions/URL/topLevelDomain.cpp
index 25e9f383f60..b3e88832350 100644
--- a/src/Functions/URL/topLevelDomain.cpp
+++ b/src/Functions/URL/topLevelDomain.cpp
@@ -1,6 +1,6 @@
 #include <Functions/FunctionFactory.h>
 #include <Functions/FunctionStringToString.h>
-#include "domain.h"
+#include <Functions/URL/domain.h>
 
 namespace DB
 {
diff --git a/src/Functions/decodeHTMLComponent.cpp b/src/Functions/decodeHTMLComponent.cpp
index 00a601b77a6..d36bee534a8 100644
--- a/src/Functions/decodeHTMLComponent.cpp
+++ b/src/Functions/decodeHTMLComponent.cpp
@@ -28,20 +28,20 @@ namespace
             const ColumnString::Chars & data,
             const ColumnString::Offsets & offsets,
             ColumnString::Chars & res_data,
-            ColumnString::Offsets & res_offsets)
+            ColumnString::Offsets & res_offsets,
+            size_t input_rows_count)
         {
             /// The size of result is always not more than the size of source.
             /// Because entities decodes to the shorter byte sequence.
             /// Example: &#xx... &#xx... will decode to UTF-8 byte sequence not longer than 4 bytes.
             res_data.resize(data.size());
 
-            size_t size = offsets.size();
-            res_offsets.resize(size);
+            res_offsets.resize(input_rows_count);
 
             size_t prev_offset = 0;
             size_t res_offset = 0;
 
-            for (size_t i = 0; i < size; ++i)
+            for (size_t i = 0; i < input_rows_count; ++i)
             {
                 const char * src_data = reinterpret_cast<const char *>(&data[prev_offset]);
                 size_t src_size = offsets[i] - prev_offset;
@@ -55,7 +55,7 @@ namespace
             res_data.resize(res_offset);
         }
 
-        [[noreturn]] static void vectorFixed(const ColumnString::Chars &, size_t, ColumnString::Chars &)
+        [[noreturn]] static void vectorFixed(const ColumnString::Chars &, size_t, ColumnString::Chars &, size_t)
         {
             throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Function decodeHTMLComponent cannot work with FixedString argument");
         }
@@ -64,7 +64,6 @@ namespace
         static const int max_legal_unicode_value = 0x10FFFF;
         static const int max_decimal_length_of_unicode_point = 7; /// 1114111
 
-
         static size_t execute(const char * src, size_t src_size, char * dst)
         {
             const char * src_pos = src;
diff --git a/src/Functions/decodeXMLComponent.cpp b/src/Functions/decodeXMLComponent.cpp
index cbbe46fcb8c..8743aa4759d 100644
--- a/src/Functions/decodeXMLComponent.cpp
+++ b/src/Functions/decodeXMLComponent.cpp
@@ -27,20 +27,20 @@ namespace
             const ColumnString::Chars & data,
             const ColumnString::Offsets & offsets,
             ColumnString::Chars & res_data,
-            ColumnString::Offsets & res_offsets)
+            ColumnString::Offsets & res_offsets,
+            size_t input_rows_count)
         {
             /// The size of result is always not more than the size of source.
             /// Because entities decodes to the shorter byte sequence.
             /// Example: &#xx... &#xx... will decode to UTF-8 byte sequence not longer than 4 bytes.
             res_data.resize(data.size());
 
-            size_t size = offsets.size();
-            res_offsets.resize(size);
+            res_offsets.resize(input_rows_count);
 
             size_t prev_offset = 0;
             size_t res_offset = 0;
 
-            for (size_t i = 0; i < size; ++i)
+            for (size_t i = 0; i < input_rows_count; ++i)
             {
                 const char * src_data = reinterpret_cast<const char *>(&data[prev_offset]);
                 size_t src_size = offsets[i] - prev_offset;
@@ -54,7 +54,7 @@ namespace
             res_data.resize(res_offset);
         }
 
-        [[noreturn]] static void vectorFixed(const ColumnString::Chars &, size_t, ColumnString::Chars &)
+        [[noreturn]] static void vectorFixed(const ColumnString::Chars &, size_t, ColumnString::Chars &, size_t)
         {
             throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Function decodeXMLComponent cannot work with FixedString argument");
         }
diff --git a/src/Functions/encodeXMLComponent.cpp b/src/Functions/encodeXMLComponent.cpp
index 64d85ecaeb8..a22917838b7 100644
--- a/src/Functions/encodeXMLComponent.cpp
+++ b/src/Functions/encodeXMLComponent.cpp
@@ -25,17 +25,17 @@ namespace
             const ColumnString::Chars & data,
             const ColumnString::Offsets & offsets,
             ColumnString::Chars & res_data,
-            ColumnString::Offsets & res_offsets)
+            ColumnString::Offsets & res_offsets,
+            size_t input_rows_count)
         {
             /// 6 is the maximum size amplification (the maximum length of encoded entity: &quot;)
             res_data.resize(data.size() * 6);
-            size_t size = offsets.size();
-            res_offsets.resize(size);
+            res_offsets.resize(input_rows_count);
 
             size_t prev_offset = 0;
             size_t res_offset = 0;
 
-            for (size_t i = 0; i < size; ++i)
+            for (size_t i = 0; i < input_rows_count; ++i)
             {
                 const char * src_data = reinterpret_cast<const char *>(&data[prev_offset]);
                 size_t src_size = offsets[i] - prev_offset;
@@ -49,7 +49,7 @@ namespace
             res_data.resize(res_offset);
         }
 
-        [[noreturn]] static void vectorFixed(const ColumnString::Chars &, size_t, ColumnString::Chars &)
+        [[noreturn]] static void vectorFixed(const ColumnString::Chars &, size_t, ColumnString::Chars &, size_t)
         {
             throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Function encodeXML cannot work with FixedString argument");
         }
diff --git a/src/Functions/idna.cpp b/src/Functions/idna.cpp
index 5a7ae3485ba..cf9e855c912 100644
--- a/src/Functions/idna.cpp
+++ b/src/Functions/idna.cpp
@@ -44,15 +44,15 @@ struct IdnaEncode
         const ColumnString::Chars & data,
         const ColumnString::Offsets & offsets,
         ColumnString::Chars & res_data,
-        ColumnString::Offsets & res_offsets)
+        ColumnString::Offsets & res_offsets,
+        size_t input_rows_count)
     {
-        const size_t rows = offsets.size();
         res_data.reserve(data.size()); /// just a guess, assuming the input is all-ASCII
-        res_offsets.reserve(rows);
+        res_offsets.reserve(input_rows_count);
 
         size_t prev_offset = 0;
         std::string ascii;
-        for (size_t row = 0; row < rows; ++row)
+        for (size_t row = 0; row < input_rows_count; ++row)
         {
             const char * value = reinterpret_cast<const char *>(&data[prev_offset]);
             const size_t value_length = offsets[row] - prev_offset - 1;
@@ -85,7 +85,7 @@ struct IdnaEncode
         }
     }
 
-    [[noreturn]] static void vectorFixed(const ColumnString::Chars &, size_t, ColumnString::Chars &)
+    [[noreturn]] static void vectorFixed(const ColumnString::Chars &, size_t, ColumnString::Chars &, size_t)
     {
         throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Arguments of type FixedString are not allowed");
     }
@@ -99,15 +99,15 @@ struct IdnaDecode
         const ColumnString::Chars & data,
         const ColumnString::Offsets & offsets,
         ColumnString::Chars & res_data,
-        ColumnString::Offsets & res_offsets)
+        ColumnString::Offsets & res_offsets,
+        size_t input_rows_count)
     {
-        const size_t rows = offsets.size();
         res_data.reserve(data.size()); /// just a guess, assuming the input is all-ASCII
-        res_offsets.reserve(rows);
+        res_offsets.reserve(input_rows_count);
 
         size_t prev_offset = 0;
         std::string unicode;
-        for (size_t row = 0; row < rows; ++row)
+        for (size_t row = 0; row < input_rows_count; ++row)
         {
             const char * ascii = reinterpret_cast<const char *>(&data[prev_offset]);
             const size_t ascii_length = offsets[row] - prev_offset - 1;
@@ -124,7 +124,7 @@ struct IdnaDecode
         }
     }
 
-    [[noreturn]] static void vectorFixed(const ColumnString::Chars &, size_t, ColumnString::Chars &)
+    [[noreturn]] static void vectorFixed(const ColumnString::Chars &, size_t, ColumnString::Chars &, size_t)
     {
         throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Arguments of type FixedString are not allowed");
     }
diff --git a/src/Functions/initcap.cpp b/src/Functions/initcap.cpp
index 4661ce117c0..00414b22344 100644
--- a/src/Functions/initcap.cpp
+++ b/src/Functions/initcap.cpp
@@ -9,10 +9,12 @@ namespace
 
 struct InitcapImpl
 {
-    static void vector(const ColumnString::Chars & data,
+    static void vector(
+        const ColumnString::Chars & data,
         const ColumnString::Offsets & offsets,
         ColumnString::Chars & res_data,
-        ColumnString::Offsets & res_offsets)
+        ColumnString::Offsets & res_offsets,
+        size_t /*input_rows_count*/)
     {
         if (data.empty())
             return;
@@ -21,7 +23,7 @@ struct InitcapImpl
         array(data.data(), data.data() + data.size(), res_data.data());
     }
 
-    static void vectorFixed(const ColumnString::Chars & data, size_t /*n*/, ColumnString::Chars & res_data)
+    static void vectorFixed(const ColumnString::Chars & data, size_t /*n*/, ColumnString::Chars & res_data, size_t)
     {
         res_data.resize(data.size());
         array(data.data(), data.data() + data.size(), res_data.data());
diff --git a/src/Functions/initcapUTF8.cpp b/src/Functions/initcapUTF8.cpp
index 076dcff6622..282d846094e 100644
--- a/src/Functions/initcapUTF8.cpp
+++ b/src/Functions/initcapUTF8.cpp
@@ -22,7 +22,8 @@ struct InitcapUTF8Impl
         const ColumnString::Chars & data,
         const ColumnString::Offsets & offsets,
         ColumnString::Chars & res_data,
-        ColumnString::Offsets & res_offsets)
+        ColumnString::Offsets & res_offsets,
+        size_t /*input_rows_count*/)
     {
         if (data.empty())
             return;
@@ -31,7 +32,7 @@ struct InitcapUTF8Impl
         array(data.data(), data.data() + data.size(), offsets, res_data.data());
     }
 
-    [[noreturn]] static void vectorFixed(const ColumnString::Chars &, size_t, ColumnString::Chars &)
+    [[noreturn]] static void vectorFixed(const ColumnString::Chars &, size_t, ColumnString::Chars &, size_t)
     {
         throw Exception(ErrorCodes::BAD_ARGUMENTS, "Function initcapUTF8 cannot work with FixedString argument");
     }
diff --git a/src/Functions/normalizeQuery.cpp b/src/Functions/normalizeQuery.cpp
index ad9a8903733..1a78bce7d29 100644
--- a/src/Functions/normalizeQuery.cpp
+++ b/src/Functions/normalizeQuery.cpp
@@ -19,17 +19,19 @@ template <bool keep_names>
 struct Impl
 {
     static constexpr auto name = keep_names ? "normalizeQueryKeepNames" : "normalizeQuery";
-    static void vector(const ColumnString::Chars & data,
+
+    static void vector(
+        const ColumnString::Chars & data,
         const ColumnString::Offsets & offsets,
         ColumnString::Chars & res_data,
-        ColumnString::Offsets & res_offsets)
+        ColumnString::Offsets & res_offsets,
+        size_t input_rows_count)
     {
-        size_t size = offsets.size();
-        res_offsets.resize(size);
+        res_offsets.resize(input_rows_count);
         res_data.reserve(data.size());
 
         ColumnString::Offset prev_src_offset = 0;
-        for (size_t i = 0; i < size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             ColumnString::Offset curr_src_offset = offsets[i];
 
@@ -43,7 +45,7 @@ struct Impl
         }
     }
 
-    [[noreturn]] static void vectorFixed(const ColumnString::Chars &, size_t, ColumnString::Chars &)
+    [[noreturn]] static void vectorFixed(const ColumnString::Chars &, size_t, ColumnString::Chars &, size_t)
     {
         throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Cannot apply function normalizeQuery to fixed string.");
     }
diff --git a/src/Functions/normalizeString.cpp b/src/Functions/normalizeString.cpp
index 92411490eaa..c56508ced0e 100644
--- a/src/Functions/normalizeString.cpp
+++ b/src/Functions/normalizeString.cpp
@@ -84,7 +84,8 @@ struct NormalizeUTF8Impl
     static void vector(const ColumnString::Chars & data,
         const ColumnString::Offsets & offsets,
         ColumnString::Chars & res_data,
-        ColumnString::Offsets & res_offsets)
+        ColumnString::Offsets & res_offsets,
+        size_t input_rows_count)
     {
         UErrorCode err = U_ZERO_ERROR;
 
@@ -92,8 +93,7 @@ struct NormalizeUTF8Impl
         if (U_FAILURE(err))
             throw Exception(ErrorCodes::CANNOT_NORMALIZE_STRING, "Normalization failed (getNormalizer): {}", u_errorName(err));
 
-        size_t size = offsets.size();
-        res_offsets.resize(size);
+        res_offsets.resize(input_rows_count);
 
         res_data.reserve(data.size() * 2);
 
@@ -103,7 +103,7 @@ struct NormalizeUTF8Impl
         PODArray<UChar> from_uchars;
         PODArray<UChar> to_uchars;
 
-        for (size_t i = 0; i < size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             size_t from_size = offsets[i] - current_from_offset - 1;
 
@@ -157,7 +157,7 @@ struct NormalizeUTF8Impl
         res_data.resize(current_to_offset);
     }
 
-    [[noreturn]] static void vectorFixed(const ColumnString::Chars &, size_t, ColumnString::Chars &)
+    [[noreturn]] static void vectorFixed(const ColumnString::Chars &, size_t, ColumnString::Chars &, size_t)
     {
         throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Cannot apply function normalizeUTF8 to fixed string.");
     }
diff --git a/src/Functions/normalizedQueryHash.cpp b/src/Functions/normalizedQueryHash.cpp
index 63218f28af5..3dbe3ff9847 100644
--- a/src/Functions/normalizedQueryHash.cpp
+++ b/src/Functions/normalizedQueryHash.cpp
@@ -27,13 +27,13 @@ struct Impl
     static void vector(
         const ColumnString::Chars & data,
         const ColumnString::Offsets & offsets,
-        PaddedPODArray<UInt64> & res_data)
+        PaddedPODArray<UInt64> & res_data,
+        size_t input_rows_count)
     {
-        size_t size = offsets.size();
-        res_data.resize(size);
+        res_data.resize(input_rows_count);
 
         ColumnString::Offset prev_src_offset = 0;
-        for (size_t i = 0; i < size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             ColumnString::Offset curr_src_offset = offsets[i];
             res_data[i] = normalizedQueryHash(
@@ -77,15 +77,15 @@ public:
 
     bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
         const ColumnPtr column = arguments[0].column;
         if (const ColumnString * col = checkAndGetColumn<ColumnString>(column.get()))
         {
             auto col_res = ColumnUInt64::create();
             typename ColumnUInt64::Container & vec_res = col_res->getData();
-            vec_res.resize(col->size());
-            Impl<keep_names>::vector(col->getChars(), col->getOffsets(), vec_res);
+            vec_res.resize(input_rows_count);
+            Impl<keep_names>::vector(col->getChars(), col->getOffsets(), vec_res, input_rows_count);
             return col_res;
         }
         else
diff --git a/src/Functions/pointInEllipses.cpp b/src/Functions/pointInEllipses.cpp
index 2147428cee3..ac7a8cc4204 100644
--- a/src/Functions/pointInEllipses.cpp
+++ b/src/Functions/pointInEllipses.cpp
@@ -91,8 +91,6 @@ private:
 
     ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
-        const auto size = input_rows_count;
-
         /// Prepare array of ellipses.
         size_t ellipses_count = (arguments.size() - 2) / 4;
         std::vector<Ellipse> ellipses(ellipses_count);
@@ -141,13 +139,11 @@ private:
 
                 auto dst = ColumnVector<UInt8>::create();
                 auto & dst_data = dst->getData();
-                dst_data.resize(size);
+                dst_data.resize(input_rows_count);
 
                 size_t start_index = 0;
-                for (const auto row : collections::range(0, size))
-                {
+                for (size_t row = 0; row < input_rows_count; ++row)
                     dst_data[row] = isPointInEllipses(col_vec_x->getData()[row], col_vec_y->getData()[row], ellipses.data(), ellipses_count, start_index);
-                }
 
                 return dst;
         }
@@ -157,7 +153,7 @@ private:
             const auto * col_const_y = assert_cast<const ColumnConst *> (col_y);
             size_t start_index = 0;
             UInt8 res = isPointInEllipses(col_const_x->getValue<Float64>(), col_const_y->getValue<Float64>(), ellipses.data(), ellipses_count, start_index);
-            return DataTypeUInt8().createColumnConst(size, res);
+            return DataTypeUInt8().createColumnConst(input_rows_count, res);
         }
         else
         {
diff --git a/src/Functions/punycode.cpp b/src/Functions/punycode.cpp
index 8004e3731b5..ec1fcfd0a70 100644
--- a/src/Functions/punycode.cpp
+++ b/src/Functions/punycode.cpp
@@ -6,11 +6,11 @@
 #include <Functions/FunctionFactory.h>
 #include <Functions/FunctionStringToString.h>
 
-#    pragma clang diagnostic push
-#    pragma clang diagnostic ignored "-Wnewline-eof"
-#    include <ada/idna/punycode.h>
-#    include <ada/idna/unicode_transcoding.h>
-#    pragma clang diagnostic pop
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wnewline-eof"
+#include <ada/idna/punycode.h>
+#include <ada/idna/unicode_transcoding.h>
+#pragma clang diagnostic pop
 
 namespace DB
 {
@@ -38,16 +38,16 @@ struct PunycodeEncode
         const ColumnString::Chars & data,
         const ColumnString::Offsets & offsets,
         ColumnString::Chars & res_data,
-        ColumnString::Offsets & res_offsets)
+        ColumnString::Offsets & res_offsets,
+        size_t input_rows_count)
     {
-        const size_t rows = offsets.size();
         res_data.reserve(data.size()); /// just a guess, assuming the input is all-ASCII
-        res_offsets.reserve(rows);
+        res_offsets.reserve(input_rows_count);
 
         size_t prev_offset = 0;
         std::u32string value_utf32;
         std::string value_puny;
-        for (size_t row = 0; row < rows; ++row)
+        for (size_t row = 0; row < input_rows_count; ++row)
         {
             const char * value = reinterpret_cast<const char *>(&data[prev_offset]);
             const size_t value_length = offsets[row] - prev_offset - 1;
@@ -72,7 +72,7 @@ struct PunycodeEncode
         }
     }
 
-    [[noreturn]] static void vectorFixed(const ColumnString::Chars &, size_t, ColumnString::Chars &)
+    [[noreturn]] static void vectorFixed(const ColumnString::Chars &, size_t, ColumnString::Chars &, size_t)
     {
         throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Arguments of type FixedString are not allowed");
     }
@@ -86,16 +86,16 @@ struct PunycodeDecode
         const ColumnString::Chars & data,
         const ColumnString::Offsets & offsets,
         ColumnString::Chars & res_data,
-        ColumnString::Offsets & res_offsets)
+        ColumnString::Offsets & res_offsets,
+        size_t input_rows_count)
     {
-        const size_t rows = offsets.size();
         res_data.reserve(data.size()); /// just a guess, assuming the input is all-ASCII
-        res_offsets.reserve(rows);
+        res_offsets.reserve(input_rows_count);
 
         size_t prev_offset = 0;
         std::u32string value_utf32;
         std::string value_utf8;
-        for (size_t row = 0; row < rows; ++row)
+        for (size_t row = 0; row < input_rows_count; ++row)
         {
             const char * value = reinterpret_cast<const char *>(&data[prev_offset]);
             const size_t value_length = offsets[row] - prev_offset - 1;
@@ -129,7 +129,7 @@ struct PunycodeDecode
         }
     }
 
-    [[noreturn]] static void vectorFixed(const ColumnString::Chars &, size_t, ColumnString::Chars &)
+    [[noreturn]] static void vectorFixed(const ColumnString::Chars &, size_t, ColumnString::Chars &, size_t)
     {
         throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Arguments of type FixedString are not allowed");
     }
diff --git a/src/Functions/reverse.cpp b/src/Functions/reverse.cpp
index d23e48b8d42..94b6634ffbd 100644
--- a/src/Functions/reverse.cpp
+++ b/src/Functions/reverse.cpp
@@ -55,19 +55,19 @@ public:
 
     bool useDefaultImplementationForConstants() const override { return true; }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
         const ColumnPtr column = arguments[0].column;
         if (const ColumnString * col = checkAndGetColumn<ColumnString>(column.get()))
         {
             auto col_res = ColumnString::create();
-            ReverseImpl::vector(col->getChars(), col->getOffsets(), col_res->getChars(), col_res->getOffsets());
+            ReverseImpl::vector(col->getChars(), col->getOffsets(), col_res->getChars(), col_res->getOffsets(), input_rows_count);
             return col_res;
         }
         else if (const ColumnFixedString * col_fixed = checkAndGetColumn<ColumnFixedString>(column.get()))
         {
             auto col_res = ColumnFixedString::create(col_fixed->getN());
-            ReverseImpl::vectorFixed(col_fixed->getChars(), col_fixed->getN(), col_res->getChars());
+            ReverseImpl::vectorFixed(col_fixed->getChars(), col_fixed->getN(), col_res->getChars(), input_rows_count);
             return col_res;
         }
         else
diff --git a/src/Functions/reverse.h b/src/Functions/reverse.h
index 5f999af4297..7c18d72769a 100644
--- a/src/Functions/reverse.h
+++ b/src/Functions/reverse.h
@@ -9,17 +9,18 @@ namespace DB
   */
 struct ReverseImpl
 {
-    static void vector(const ColumnString::Chars & data,
+    static void vector(
+        const ColumnString::Chars & data,
         const ColumnString::Offsets & offsets,
         ColumnString::Chars & res_data,
-        ColumnString::Offsets & res_offsets)
+        ColumnString::Offsets & res_offsets,
+        size_t input_rows_count)
     {
         res_data.resize_exact(data.size());
         res_offsets.assign(offsets);
-        size_t size = offsets.size();
 
         ColumnString::Offset prev_offset = 0;
-        for (size_t i = 0; i < size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             for (size_t j = prev_offset; j < offsets[i] - 1; ++j)
                 res_data[j] = data[offsets[i] + prev_offset - 2 - j];
@@ -28,12 +29,15 @@ struct ReverseImpl
         }
     }
 
-    static void vectorFixed(const ColumnString::Chars & data, size_t n, ColumnString::Chars & res_data)
+    static void vectorFixed(
+        const ColumnString::Chars & data,
+        size_t n,
+        ColumnString::Chars & res_data,
+        size_t input_rows_count)
     {
         res_data.resize_exact(data.size());
-        size_t size = data.size() / n;
 
-        for (size_t i = 0; i < size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
             for (size_t j = i * n; j < (i + 1) * n; ++j)
                 res_data[j] = data[(i * 2 + 1) * n - j - 1];
     }
diff --git a/src/Functions/reverseUTF8.cpp b/src/Functions/reverseUTF8.cpp
index 1aee349fa8d..ca57d946b19 100644
--- a/src/Functions/reverseUTF8.cpp
+++ b/src/Functions/reverseUTF8.cpp
@@ -23,25 +23,25 @@ namespace
   */
 struct ReverseUTF8Impl
 {
-    static void vector(const ColumnString::Chars & data,
+    static void vector(
+        const ColumnString::Chars & data,
         const ColumnString::Offsets & offsets,
         ColumnString::Chars & res_data,
-        ColumnString::Offsets & res_offsets)
+        ColumnString::Offsets & res_offsets,
+        size_t input_rows_count)
     {
         bool all_ascii = isAllASCII(data.data(), data.size());
         if (all_ascii)
         {
-            ReverseImpl::vector(data, offsets, res_data, res_offsets);
+            ReverseImpl::vector(data, offsets, res_data, res_offsets, input_rows_count);
             return;
         }
 
         res_data.resize(data.size());
         res_offsets.assign(offsets);
-        size_t size = offsets.size();
-
 
         ColumnString::Offset prev_offset = 0;
-        for (size_t i = 0; i < size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             ColumnString::Offset j = prev_offset;
             while (j < offsets[i] - 1)
@@ -73,7 +73,7 @@ struct ReverseUTF8Impl
         }
     }
 
-    [[noreturn]] static void vectorFixed(const ColumnString::Chars &, size_t, ColumnString::Chars &)
+    [[noreturn]] static void vectorFixed(const ColumnString::Chars &, size_t, ColumnString::Chars &, size_t)
     {
         throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Cannot apply function reverseUTF8 to fixed string.");
     }
diff --git a/src/Functions/soundex.cpp b/src/Functions/soundex.cpp
index fcf1523d1a3..e8bd1b664e3 100644
--- a/src/Functions/soundex.cpp
+++ b/src/Functions/soundex.cpp
@@ -79,14 +79,14 @@ struct SoundexImpl
         const ColumnString::Chars & data,
         const ColumnString::Offsets & offsets,
         ColumnString::Chars & res_data,
-        ColumnString::Offsets & res_offsets)
+        ColumnString::Offsets & res_offsets,
+        size_t input_rows_count)
     {
-        const size_t size = offsets.size();
-        res_data.resize(size * (length + 1));
-        res_offsets.resize(size);
+        res_data.resize(input_rows_count * (length + 1));
+        res_offsets.resize(input_rows_count);
 
         size_t prev_offset = 0;
-        for (size_t i = 0; i < size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             const char * value = reinterpret_cast<const char *>(&data[prev_offset]);
             const size_t value_length = offsets[i] - prev_offset - 1;
@@ -98,7 +98,7 @@ struct SoundexImpl
         }
     }
 
-    [[noreturn]] static void vectorFixed(const ColumnString::Chars &, size_t, ColumnString::Chars &)
+    [[noreturn]] static void vectorFixed(const ColumnString::Chars &, size_t, ColumnString::Chars &, size_t)
     {
         throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Column of type FixedString is not supported by soundex function");
     }
diff --git a/src/Functions/toValidUTF8.cpp b/src/Functions/toValidUTF8.cpp
index 41d29d9c494..376732256b0 100644
--- a/src/Functions/toValidUTF8.cpp
+++ b/src/Functions/toValidUTF8.cpp
@@ -128,16 +128,16 @@ struct ToValidUTF8Impl
         const ColumnString::Chars & data,
         const ColumnString::Offsets & offsets,
         ColumnString::Chars & res_data,
-        ColumnString::Offsets & res_offsets)
+        ColumnString::Offsets & res_offsets,
+        size_t input_rows_count)
     {
-        const size_t offsets_size = offsets.size();
         /// It can be larger than that, but we believe it is unlikely to happen.
         res_data.resize(data.size());
-        res_offsets.resize(offsets_size);
+        res_offsets.resize(input_rows_count);
 
         size_t prev_offset = 0;
         WriteBufferFromVector<ColumnString::Chars> write_buffer(res_data);
-        for (size_t i = 0; i < offsets_size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             const char * haystack_data = reinterpret_cast<const char *>(&data[prev_offset]);
             const size_t haystack_size = offsets[i] - prev_offset - 1;
@@ -149,7 +149,7 @@ struct ToValidUTF8Impl
         write_buffer.finalize();
     }
 
-    [[noreturn]] static void vectorFixed(const ColumnString::Chars &, size_t, ColumnString::Chars &)
+    [[noreturn]] static void vectorFixed(const ColumnString::Chars &, size_t, ColumnString::Chars &, size_t)
     {
         throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Column of type FixedString is not supported by toValidUTF8 function");
     }
diff --git a/src/Functions/trim.cpp b/src/Functions/trim.cpp
index 1f0011b8e99..5703e871423 100644
--- a/src/Functions/trim.cpp
+++ b/src/Functions/trim.cpp
@@ -43,10 +43,10 @@ public:
         const ColumnString::Chars & data,
         const ColumnString::Offsets & offsets,
         ColumnString::Chars & res_data,
-        ColumnString::Offsets & res_offsets)
+        ColumnString::Offsets & res_offsets,
+        size_t input_rows_count)
     {
-        size_t size = offsets.size();
-        res_offsets.resize_exact(size);
+        res_offsets.resize_exact(input_rows_count);
         res_data.reserve_exact(data.size());
 
         size_t prev_offset = 0;
@@ -55,7 +55,7 @@ public:
         const UInt8 * start;
         size_t length;
 
-        for (size_t i = 0; i < size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             execute(reinterpret_cast<const UInt8 *>(&data[prev_offset]), offsets[i] - prev_offset - 1, start, length);
 
@@ -69,7 +69,7 @@ public:
         }
     }
 
-    static void vectorFixed(const ColumnString::Chars &, size_t, ColumnString::Chars &)
+    static void vectorFixed(const ColumnString::Chars &, size_t, ColumnString::Chars &, size_t)
     {
         throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Functions trimLeft, trimRight and trimBoth cannot work with FixedString argument");
     }

From 54a503910299c13dac305049a98e8136ab989fbb Mon Sep 17 00:00:00 2001
From: Alexander Gololobov <davenger@clickhouse.com>
Date: Tue, 23 Jul 2024 19:25:38 +0200
Subject: [PATCH 0660/1488] Count log messages for building set and reusing set
 from cache

---
 ..._between_multiple_mutations_tasks_long.sql | 30 +++++++++----------
 ...e_big_sets_between_mutation_tasks_long.sql | 29 +++++++++---------
 2 files changed, 30 insertions(+), 29 deletions(-)

diff --git a/tests/queries/0_stateless/02581_share_big_sets_between_multiple_mutations_tasks_long.sql b/tests/queries/0_stateless/02581_share_big_sets_between_multiple_mutations_tasks_long.sql
index 40f7800fee1..6b0677a80ae 100644
--- a/tests/queries/0_stateless/02581_share_big_sets_between_multiple_mutations_tasks_long.sql
+++ b/tests/queries/0_stateless/02581_share_big_sets_between_multiple_mutations_tasks_long.sql
@@ -31,22 +31,22 @@ DELETE FROM 02581_trips                        WHERE id IN (SELECT (number*10 +
 SELECT count(), _part from 02581_trips WHERE description = '' GROUP BY _part ORDER BY _part SETTINGS select_sequential_consistency=1;
 
 SYSTEM FLUSH LOGS;
-
--- Check that in every mutation there were parts where selected rows count then the size of big sets which will mean that sets were shared
--- Also check that there was at least one part that read more rows then the size of set which will mean that the 
-WITH 10000000 AS rows_in_set
+-- Check that in every mutation there were parts that built sets (log messages like 'Created Set with 10000000 entries from 10000000 rows in 0.388989187 sec.' )
+-- and parts that shared sets (log messages like 'Got set from cache in 0.388930505 sec.' )
+WITH (
+        SELECT uuid
+        FROM system.tables
+        WHERE (database = currentDatabase()) AND (name = '02581_trips')
+    ) AS table_uuid
 SELECT
-    mutation_version,
-    countIf(read_rows >= rows_in_set) >= 1 as has_parts_for_which_set_was_built,
-    countIf(read_rows <= rows_in_set) >= 1 as has_parts_that_shared_set
-FROM
-(
-    SELECT
-        CAST(splitByChar('_', part_name)[5], 'UInt64') AS mutation_version,
-        read_rows
-    FROM system.part_log
-    WHERE database = currentDatabase() and (event_date >= yesterday()) AND (`table` = '02581_trips') AND (event_type = 'MutatePart')
-)
+    CAST(splitByChar('_', query_id)[5], 'UInt64') AS mutation_version, -- '5521485f-8a40-4aba-87a2-00342c369563::all_3_3_0_6'
+    sum(message LIKE 'Created Set with % entries%') >= 1  AS has_parts_for_which_set_was_built,
+    sum(message LIKE 'Got set from cache%') >= 1 AS has_parts_that_shared_set
+FROM system.text_log
+WHERE
+    query_id LIKE concat(CAST(table_uuid, 'String'), '::all\\_%')
+    AND (event_date >= yesterday())
+    AND (message LIKE 'Created Set with % entries%' OR message LIKE 'Got set from cache%')
 GROUP BY mutation_version ORDER BY mutation_version FORMAT TSVWithNames;
 
 DROP TABLE 02581_trips;
diff --git a/tests/queries/0_stateless/02581_share_big_sets_between_mutation_tasks_long.sql b/tests/queries/0_stateless/02581_share_big_sets_between_mutation_tasks_long.sql
index 603c7cb7db0..091a9c8171d 100644
--- a/tests/queries/0_stateless/02581_share_big_sets_between_mutation_tasks_long.sql
+++ b/tests/queries/0_stateless/02581_share_big_sets_between_mutation_tasks_long.sql
@@ -59,21 +59,22 @@ SETTINGS mutations_sync=2;
 SELECT count() from 02581_trips WHERE description = '';
 
 SYSTEM FLUSH LOGS;
--- Check that in every mutation there were parts where selected rows count then the size of big sets which will mean that sets were shared
--- Also check that there was at least one part that read more rows then the size of set which will mean that the 
-WITH 10000000 AS rows_in_set
+-- Check that in every mutation there were parts that built sets (log messages like 'Created Set with 10000000 entries from 10000000 rows in 0.388989187 sec.' )
+-- and parts that shared sets (log messages like 'Got set from cache in 0.388930505 sec.' )
+WITH (
+        SELECT uuid
+        FROM system.tables
+        WHERE (database = currentDatabase()) AND (name = '02581_trips')
+    ) AS table_uuid
 SELECT
-    mutation_version,
-    countIf(read_rows >= rows_in_set) >= 1 as has_parts_for_which_set_was_built,
-    countIf(read_rows <= rows_in_set) >= 1 as has_parts_that_shared_set
-FROM
-(
-    SELECT
-        CAST(splitByChar('_', part_name)[5], 'UInt64') AS mutation_version,
-        read_rows
-    FROM system.part_log
-    WHERE database = currentDatabase() and (event_date >= yesterday()) AND (`table` = '02581_trips') AND (event_type = 'MutatePart')
-)
+    CAST(splitByChar('_', query_id)[5], 'UInt64') AS mutation_version, -- '5521485f-8a40-4aba-87a2-00342c369563::all_3_3_0_6'
+    sum(message LIKE 'Created Set with % entries%') >= 1  AS has_parts_for_which_set_was_built,
+    sum(message LIKE 'Got set from cache%') >= 1 AS has_parts_that_shared_set
+FROM system.text_log
+WHERE
+    query_id LIKE concat(CAST(table_uuid, 'String'), '::all\\_%')
+    AND (event_date >= yesterday())
+    AND (message LIKE 'Created Set with % entries%' OR message LIKE 'Got set from cache%')
 GROUP BY mutation_version ORDER BY mutation_version FORMAT TSVWithNames;
 
 DROP TABLE 02581_trips;

From a9d7abbb928d43664f057dde88058c449a9a521c Mon Sep 17 00:00:00 2001
From: jsc0218 <jsc0218@gmail.com>
Date: Tue, 23 Jul 2024 17:27:05 +0000
Subject: [PATCH 0661/1488] add drop option

---
 src/Core/SettingsEnums.cpp                           |  2 +-
 src/Storages/MergeTree/MergeTask.cpp                 |  9 +++++++++
 .../03206_projection_merge_special_mergetree.sql     | 12 ++++++++++++
 3 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/src/Core/SettingsEnums.cpp b/src/Core/SettingsEnums.cpp
index 74b6c793849..b53a882de4e 100644
--- a/src/Core/SettingsEnums.cpp
+++ b/src/Core/SettingsEnums.cpp
@@ -180,7 +180,7 @@ IMPLEMENT_SETTING_ENUM(LightweightMutationProjectionMode, ErrorCodes::BAD_ARGUME
 IMPLEMENT_SETTING_ENUM(DeduplicateMergeProjectionMode, ErrorCodes::BAD_ARGUMENTS,
     {{"throw", DeduplicateMergeProjectionMode::THROW},
      {"drop", DeduplicateMergeProjectionMode::DROP},
-     {"rebuild", DeduplicateMergeProjectionMode::THROW}})
+     {"rebuild", DeduplicateMergeProjectionMode::REBUILD}})
 
 IMPLEMENT_SETTING_AUTO_ENUM(LocalFSReadMethod, ErrorCodes::BAD_ARGUMENTS)
 
diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp
index fc64fae9a58..0b358c0fd7c 100644
--- a/src/Storages/MergeTree/MergeTask.cpp
+++ b/src/Storages/MergeTree/MergeTask.cpp
@@ -797,6 +797,15 @@ bool MergeTask::MergeProjectionsStage::mergeMinMaxIndexAndPrepareProjections() c
     }
 
 
+    const auto mode = global_ctx->data->getSettings()->deduplicate_merge_projection_mode;
+    /// Under throw mode, we still choose to drop projections due to backward compatibility since some
+    /// users might have projections before this change.
+    if (mode == DeduplicateMergeProjectionMode::THROW || mode == DeduplicateMergeProjectionMode::DROP)
+    {
+        ctx->projections_iterator = ctx->tasks_for_projections.begin();
+        return false;
+    }
+
     const auto & projections = global_ctx->metadata_snapshot->getProjections();
 
     for (const auto & projection : projections)
diff --git a/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql b/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql
index c8945fd784c..749f906569e 100644
--- a/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql
+++ b/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql
@@ -20,6 +20,18 @@ CREATE TABLE tp (
 ) engine = ReplacingMergeTree order by type
 SETTINGS deduplicate_merge_projection_mode = 'drop';
 
+INSERT INTO tp SELECT number%3, 1 FROM numbers(3);
+
+OPTIMIZE TABLE tp FINAL;
+
+-- expecting no projection
+SYSTEM FLUSH LOGS;
+SELECT
+    name,
+    part_name
+FROM system.projection_parts
+WHERE (database = currentDatabase()) AND (`table` = 'tp') AND (active = 1);
+
 ALTER TABLE tp MODIFY SETTING deduplicate_merge_projection_mode = 'throw';
 
 OPTIMIZE TABLE tp DEDUPLICATE;  -- { serverError NOT_IMPLEMENTED }

From 1cbbbd107712c066bef673e5d4692a6950d9e85e Mon Sep 17 00:00:00 2001
From: Michael Kolupaev <michael.kolupaev@clickhouse.com>
Date: Tue, 23 Jul 2024 17:35:10 +0000
Subject: [PATCH 0662/1488] Apply libunwind fix

---
 contrib/libunwind            | 2 +-
 src/Common/QueryProfiler.cpp | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/contrib/libunwind b/contrib/libunwind
index 8f28e64d158..9b1f47ad8a6 160000
--- a/contrib/libunwind
+++ b/contrib/libunwind
@@ -1 +1 @@
-Subproject commit 8f28e64d15819d2d096badd598c7d85bebddb1f2
+Subproject commit 9b1f47ad8a6fcecbeaaead93bd87756ccf658071
diff --git a/src/Common/QueryProfiler.cpp b/src/Common/QueryProfiler.cpp
index 746010b5462..a7717a4288a 100644
--- a/src/Common/QueryProfiler.cpp
+++ b/src/Common/QueryProfiler.cpp
@@ -33,7 +33,7 @@ namespace DB
 namespace
 {
 #if defined(OS_LINUX)
-    thread_local size_t write_trace_iteration = 0;
+    //thread_local size_t write_trace_iteration = 0;
 #endif
     /// Even after timer_delete() the signal can be delivered,
     /// since it does not do anything with pending signals.
@@ -57,7 +57,7 @@ namespace
 
         auto saved_errno = errno;   /// We must restore previous value of errno in signal handler.
 
-#if defined(OS_LINUX)
+#if defined(OS_LINUX) && false //asdqwe
         if (info)
         {
             int overrun_count = info->si_overrun;
@@ -92,7 +92,7 @@ namespace
         constexpr bool sanitizer = false;
 #endif
 
-        asynchronous_stack_unwinding = true;
+        //asdqwe asynchronous_stack_unwinding = true;
         if (sanitizer || 0 == sigsetjmp(asynchronous_stack_unwinding_signal_jump_buffer, 1))
         {
             stack_trace.emplace(signal_context);

From 2551f141418a9625678ea6295426bc1d10517482 Mon Sep 17 00:00:00 2001
From: Max K <maxkaynov@gmail.com>
Date: Tue, 23 Jul 2024 20:42:21 +0200
Subject: [PATCH 0663/1488] CI: Add messages fro debugging

---
 tests/ci/ci_utils.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/ci/ci_utils.py b/tests/ci/ci_utils.py
index 3b12fe6974f..d42091fb0da 100644
--- a/tests/ci/ci_utils.py
+++ b/tests/ci/ci_utils.py
@@ -104,6 +104,9 @@ class GHActions:
                 res = json.load(json_file)
             except json.JSONDecodeError as e:
                 print(f"ERROR: json decoder exception {e}")
+                json_file.seek(0)
+                print("    File content:")
+                print(json_file.read())
                 return {}
         return res
 

From 9d55553225c4c5e253e32fb0de9944a2e29b7bcf Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Tue, 23 Jul 2024 18:52:50 +0000
Subject: [PATCH 0664/1488] Forbid create as select even when
 database_replicated_allow_heavy_create is set

---
 src/Interpreters/InterpreterCreateQuery.cpp      | 16 ++++++++++++----
 ...ed_database_forbid_create_as_select.reference |  2 ++
 ...eplicated_database_forbid_create_as_select.sh |  8 ++++++--
 3 files changed, 20 insertions(+), 6 deletions(-)

diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp
index ea10ad59db4..2f837fe4d2b 100644
--- a/src/Interpreters/InterpreterCreateQuery.cpp
+++ b/src/Interpreters/InterpreterCreateQuery.cpp
@@ -1329,8 +1329,8 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create)
     if (need_add_to_database)
         database = DatabaseCatalog::instance().tryGetDatabase(database_name);
 
-    bool allow_heavy_create = getContext()->getSettingsRef().database_replicated_allow_heavy_create;
-    if (!allow_heavy_create && database && database->getEngineName() == "Replicated" && (create.select || create.is_populate))
+    bool allow_heavy_populate = getContext()->getSettingsRef().database_replicated_allow_heavy_create && create.is_populate;
+    if (!allow_heavy_populate && database && database->getEngineName() == "Replicated" && (create.select || create.is_populate))
     {
         bool is_storage_replicated = false;
         if (create.storage && create.storage->engine)
@@ -1342,10 +1342,18 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create)
 
         const bool allow_create_select_for_replicated = (create.isView() && !create.is_populate) || create.is_create_empty || !is_storage_replicated;
         if (!allow_create_select_for_replicated)
+        {
+            /// POPULATE can be enabled with setting, provide hint in error message
+            if (create.is_populate)
+                throw Exception(
+                    ErrorCodes::SUPPORT_IS_DISABLED,
+                    "CREATE with POPULATE is not supported with Replicated databases. Consider using separate CREATE and INSERT queries. "
+                    "Alternatively, you can enable 'database_replicated_allow_heavy_create' setting to allow this operation, use with caution");
+
             throw Exception(
                 ErrorCodes::SUPPORT_IS_DISABLED,
-                "CREATE AS SELECT and POPULATE is not supported with Replicated databases. Consider using separate CREATE and INSERT queries. "
-                "Alternatively, you can enable 'database_replicated_allow_heavy_create' setting to allow this operation, use with caution");
+                "CREATE AS SELECT is not supported with Replicated databases. Consider using separate CREATE and INSERT queries.");
+        }
     }
 
     if (database && database->shouldReplicateQuery(getContext(), query_ptr))
diff --git a/tests/queries/0_stateless/02933_replicated_database_forbid_create_as_select.reference b/tests/queries/0_stateless/02933_replicated_database_forbid_create_as_select.reference
index 6ed281c757a..98fb6a68656 100644
--- a/tests/queries/0_stateless/02933_replicated_database_forbid_create_as_select.reference
+++ b/tests/queries/0_stateless/02933_replicated_database_forbid_create_as_select.reference
@@ -1,2 +1,4 @@
 1
 1
+1
+1
diff --git a/tests/queries/0_stateless/02933_replicated_database_forbid_create_as_select.sh b/tests/queries/0_stateless/02933_replicated_database_forbid_create_as_select.sh
index 15f169d880f..b587549cb60 100755
--- a/tests/queries/0_stateless/02933_replicated_database_forbid_create_as_select.sh
+++ b/tests/queries/0_stateless/02933_replicated_database_forbid_create_as_select.sh
@@ -18,8 +18,12 @@ ${CLICKHOUSE_CLIENT} --distributed_ddl_output_mode=none --query "CREATE MATERIAL
 ${CLICKHOUSE_CLIENT} --distributed_ddl_output_mode=none --query "CREATE TABLE ${CLICKHOUSE_DATABASE}_db.test2 (id UInt64) ENGINE = ReplicatedMergeTree ORDER BY id AS SELECT 1" |& grep -cm1 "SUPPORT_IS_DISABLED"
 ${CLICKHOUSE_CLIENT} --distributed_ddl_output_mode=none --query "CREATE MATERIALIZED VIEW ${CLICKHOUSE_DATABASE}_db.test_mv2 (id UInt64) ENGINE = ReplicatedMergeTree ORDER BY id POPULATE AS SELECT 1" |& grep -cm1 "SUPPORT_IS_DISABLED"
 
-# But it is allowed with the special setting
-${CLICKHOUSE_CLIENT} --distributed_ddl_output_mode=none --query "CREATE TABLE ${CLICKHOUSE_DATABASE}_db.test2 (id UInt64) ENGINE = ReplicatedMergeTree ORDER BY id AS SELECT 1" --database_replicated_allow_heavy_create=1
+# POPULATE is allowed with the special setting
 ${CLICKHOUSE_CLIENT} --distributed_ddl_output_mode=none --query "CREATE MATERIALIZED VIEW ${CLICKHOUSE_DATABASE}_db.test_mv2 (id UInt64) ENGINE = ReplicatedMergeTree ORDER BY id POPULATE AS SELECT 1" --database_replicated_allow_heavy_create=1
+${CLICKHOUSE_CLIENT} --distributed_ddl_output_mode=none --query "CREATE MATERIALIZED VIEW ${CLICKHOUSE_DATABASE}_db.test_mv3 (id UInt64) ENGINE = ReplicatedMergeTree ORDER BY id POPULATE AS SELECT 1" --compatibility='24.6'
+
+# AS SELECT is forbidden even with the setting
+${CLICKHOUSE_CLIENT} --distributed_ddl_output_mode=none --query "CREATE TABLE ${CLICKHOUSE_DATABASE}_db.test2 (id UInt64) ENGINE = ReplicatedMergeTree ORDER BY id AS SELECT 1" --database_replicated_allow_heavy_create=1  |& grep -cm1 "SUPPORT_IS_DISABLED"
+${CLICKHOUSE_CLIENT} --distributed_ddl_output_mode=none --query "CREATE TABLE ${CLICKHOUSE_DATABASE}_db.test2 (id UInt64) ENGINE = ReplicatedMergeTree ORDER BY id AS SELECT 1" --compatibility='24.6'  |& grep -cm1 "SUPPORT_IS_DISABLED"
 
 ${CLICKHOUSE_CLIENT} --query "DROP DATABASE ${CLICKHOUSE_DATABASE}_db"

From 032486951316e518aa31e2548e6a501289bb0b6e Mon Sep 17 00:00:00 2001
From: avogar <avogar@clickhouse.com>
Date: Tue, 23 Jul 2024 19:07:56 +0000
Subject: [PATCH 0665/1488] Split dynamic tests and rewrite them from sh to sql
 to speed up

---
 .../03036_dynamic_read_subcolumns.reference   |  57 --
 .../03036_dynamic_read_subcolumns.sh          |  62 ---
 ...ad_subcolumns_compact_merge_tree.reference |  17 +
 ...mic_read_subcolumns_compact_merge_tree.sql |  40 ++
 ...6_dynamic_read_subcolumns_memory.reference |  17 +
 .../03036_dynamic_read_subcolumns_memory.sql  |  40 ++
 ..._read_subcolumns_wide_merge_tree.reference |  17 +
 ...ynamic_read_subcolumns_wide_merge_tree.sql |  40 ++
 ...3037_dynamic_merges_1_horizontal.reference |  60 --
 .../03037_dynamic_merges_1_horizontal.sh      |  52 --
 ..._1_horizontal_compact_merge_tree.reference |  28 +
 ...merges_1_horizontal_compact_merge_tree.sql |  33 ++
 ...s_1_horizontal_compact_wide_tree.reference |  28 +
 ..._merges_1_horizontal_compact_wide_tree.sql |  33 ++
 .../03037_dynamic_merges_1_vertical.reference |  60 --
 .../03037_dynamic_merges_1_vertical.sh        |  51 --
 ...es_1_vertical_compact_merge_tree.reference |  28 +
 ...c_merges_1_vertical_compact_merge_tree.sql |  33 ++
 ...erges_1_vertical_wide_merge_tree.reference |  28 +
 ...amic_merges_1_vertical_wide_merge_tree.sql |  33 ++
 .../03037_dynamic_merges_2.reference          |  20 -
 .../0_stateless/03037_dynamic_merges_2.sh     |  45 --
 ..._2_horizontal_compact_merge_tree.reference |   3 +
 ...merges_2_horizontal_compact_merge_tree.sql |  14 +
 ...ges_2_horizontal_wide_merge_tree.reference |   3 +
 ...ic_merges_2_horizontal_wide_merge_tree.sql |  14 +
 ...es_2_vertical_compact_merge_tree.reference |   3 +
 ...c_merges_2_vertical_compact_merge_tree.sql |  14 +
 ...erges_2_vertical_wide_merge_tree.reference |   3 +
 ...amic_merges_2_vertical_wide_merge_tree.sql |  14 +
 ...ested_dynamic_merges_compact_horizontal.sh |  32 --
 ...sted_dynamic_merges_compact_horizontal.sql |  29 +
 ..._nested_dynamic_merges_compact_vertical.sh |  32 --
 ...nested_dynamic_merges_compact_vertical.sql |  29 +
 ...8_nested_dynamic_merges_wide_horizontal.sh |  32 --
 ..._nested_dynamic_merges_wide_horizontal.sql |  29 +
 ...038_nested_dynamic_merges_wide_vertical.sh |  32 --
 ...38_nested_dynamic_merges_wide_vertical.sql |  29 +
 .../03040_dynamic_type_alters_1.reference     | 526 ------------------
 .../03040_dynamic_type_alters_1.sh            |  77 ---
 ...type_alters_1_compact_merge_tree.reference | 174 ++++++
 ...namic_type_alters_1_compact_merge_tree.sql |  53 ++
 ...040_dynamic_type_alters_1_memory.reference | 175 ++++++
 .../03040_dynamic_type_alters_1_memory.sql    |  53 ++
 ...ic_type_alters_1_wide_merge_tree.reference | 174 ++++++
 ..._dynamic_type_alters_1_wide_merge_tree.sql |  53 ++
 .../03040_dynamic_type_alters_2.reference     | 182 ------
 .../03040_dynamic_type_alters_2.sh            |  57 --
 ...type_alters_2_compact_merge_tree.reference |  90 +++
 ...namic_type_alters_2_compact_merge_tree.sql |  39 ++
 ...ic_type_alters_2_wide_merge_tree.reference |  90 +++
 ..._dynamic_type_alters_2_wide_merge_tree.sql |  39 ++
 52 files changed, 1539 insertions(+), 1377 deletions(-)
 delete mode 100644 tests/queries/0_stateless/03036_dynamic_read_subcolumns.reference
 delete mode 100755 tests/queries/0_stateless/03036_dynamic_read_subcolumns.sh
 create mode 100644 tests/queries/0_stateless/03036_dynamic_read_subcolumns_compact_merge_tree.reference
 create mode 100644 tests/queries/0_stateless/03036_dynamic_read_subcolumns_compact_merge_tree.sql
 create mode 100644 tests/queries/0_stateless/03036_dynamic_read_subcolumns_memory.reference
 create mode 100644 tests/queries/0_stateless/03036_dynamic_read_subcolumns_memory.sql
 create mode 100644 tests/queries/0_stateless/03036_dynamic_read_subcolumns_wide_merge_tree.reference
 create mode 100644 tests/queries/0_stateless/03036_dynamic_read_subcolumns_wide_merge_tree.sql
 delete mode 100644 tests/queries/0_stateless/03037_dynamic_merges_1_horizontal.reference
 delete mode 100755 tests/queries/0_stateless/03037_dynamic_merges_1_horizontal.sh
 create mode 100644 tests/queries/0_stateless/03037_dynamic_merges_1_horizontal_compact_merge_tree.reference
 create mode 100644 tests/queries/0_stateless/03037_dynamic_merges_1_horizontal_compact_merge_tree.sql
 create mode 100644 tests/queries/0_stateless/03037_dynamic_merges_1_horizontal_compact_wide_tree.reference
 create mode 100644 tests/queries/0_stateless/03037_dynamic_merges_1_horizontal_compact_wide_tree.sql
 delete mode 100644 tests/queries/0_stateless/03037_dynamic_merges_1_vertical.reference
 delete mode 100755 tests/queries/0_stateless/03037_dynamic_merges_1_vertical.sh
 create mode 100644 tests/queries/0_stateless/03037_dynamic_merges_1_vertical_compact_merge_tree.reference
 create mode 100644 tests/queries/0_stateless/03037_dynamic_merges_1_vertical_compact_merge_tree.sql
 create mode 100644 tests/queries/0_stateless/03037_dynamic_merges_1_vertical_wide_merge_tree.reference
 create mode 100644 tests/queries/0_stateless/03037_dynamic_merges_1_vertical_wide_merge_tree.sql
 delete mode 100644 tests/queries/0_stateless/03037_dynamic_merges_2.reference
 delete mode 100755 tests/queries/0_stateless/03037_dynamic_merges_2.sh
 create mode 100644 tests/queries/0_stateless/03037_dynamic_merges_2_horizontal_compact_merge_tree.reference
 create mode 100644 tests/queries/0_stateless/03037_dynamic_merges_2_horizontal_compact_merge_tree.sql
 create mode 100644 tests/queries/0_stateless/03037_dynamic_merges_2_horizontal_wide_merge_tree.reference
 create mode 100644 tests/queries/0_stateless/03037_dynamic_merges_2_horizontal_wide_merge_tree.sql
 create mode 100644 tests/queries/0_stateless/03037_dynamic_merges_2_vertical_compact_merge_tree.reference
 create mode 100644 tests/queries/0_stateless/03037_dynamic_merges_2_vertical_compact_merge_tree.sql
 create mode 100644 tests/queries/0_stateless/03037_dynamic_merges_2_vertical_wide_merge_tree.reference
 create mode 100644 tests/queries/0_stateless/03037_dynamic_merges_2_vertical_wide_merge_tree.sql
 delete mode 100755 tests/queries/0_stateless/03038_nested_dynamic_merges_compact_horizontal.sh
 create mode 100644 tests/queries/0_stateless/03038_nested_dynamic_merges_compact_horizontal.sql
 delete mode 100755 tests/queries/0_stateless/03038_nested_dynamic_merges_compact_vertical.sh
 create mode 100644 tests/queries/0_stateless/03038_nested_dynamic_merges_compact_vertical.sql
 delete mode 100755 tests/queries/0_stateless/03038_nested_dynamic_merges_wide_horizontal.sh
 create mode 100644 tests/queries/0_stateless/03038_nested_dynamic_merges_wide_horizontal.sql
 delete mode 100755 tests/queries/0_stateless/03038_nested_dynamic_merges_wide_vertical.sh
 create mode 100644 tests/queries/0_stateless/03038_nested_dynamic_merges_wide_vertical.sql
 delete mode 100644 tests/queries/0_stateless/03040_dynamic_type_alters_1.reference
 delete mode 100755 tests/queries/0_stateless/03040_dynamic_type_alters_1.sh
 create mode 100644 tests/queries/0_stateless/03040_dynamic_type_alters_1_compact_merge_tree.reference
 create mode 100644 tests/queries/0_stateless/03040_dynamic_type_alters_1_compact_merge_tree.sql
 create mode 100644 tests/queries/0_stateless/03040_dynamic_type_alters_1_memory.reference
 create mode 100644 tests/queries/0_stateless/03040_dynamic_type_alters_1_memory.sql
 create mode 100644 tests/queries/0_stateless/03040_dynamic_type_alters_1_wide_merge_tree.reference
 create mode 100644 tests/queries/0_stateless/03040_dynamic_type_alters_1_wide_merge_tree.sql
 delete mode 100644 tests/queries/0_stateless/03040_dynamic_type_alters_2.reference
 delete mode 100755 tests/queries/0_stateless/03040_dynamic_type_alters_2.sh
 create mode 100644 tests/queries/0_stateless/03040_dynamic_type_alters_2_compact_merge_tree.reference
 create mode 100644 tests/queries/0_stateless/03040_dynamic_type_alters_2_compact_merge_tree.sql
 create mode 100644 tests/queries/0_stateless/03040_dynamic_type_alters_2_wide_merge_tree.reference
 create mode 100644 tests/queries/0_stateless/03040_dynamic_type_alters_2_wide_merge_tree.sql

diff --git a/tests/queries/0_stateless/03036_dynamic_read_subcolumns.reference b/tests/queries/0_stateless/03036_dynamic_read_subcolumns.reference
deleted file mode 100644
index 36984bc8b9b..00000000000
--- a/tests/queries/0_stateless/03036_dynamic_read_subcolumns.reference
+++ /dev/null
@@ -1,57 +0,0 @@
-Memory
-test
-Array(Array(Dynamic))
-Array(Variant(String, UInt64))
-None
-String
-UInt64
-200000
-200000
-200000
-200000
-0
-0
-200000
-200000
-100000
-100000
-200000
-0
-MergeTree compact
-test
-Array(Array(Dynamic))
-Array(Variant(String, UInt64))
-None
-String
-UInt64
-200000
-200000
-200000
-200000
-0
-0
-200000
-200000
-100000
-100000
-200000
-0
-MergeTree wide
-test
-Array(Array(Dynamic))
-Array(Variant(String, UInt64))
-None
-String
-UInt64
-200000
-200000
-200000
-200000
-0
-0
-200000
-200000
-100000
-100000
-200000
-0
diff --git a/tests/queries/0_stateless/03036_dynamic_read_subcolumns.sh b/tests/queries/0_stateless/03036_dynamic_read_subcolumns.sh
deleted file mode 100755
index 65517061b99..00000000000
--- a/tests/queries/0_stateless/03036_dynamic_read_subcolumns.sh
+++ /dev/null
@@ -1,62 +0,0 @@
-#!/usr/bin/env bash
-# Tags: long
-
-CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# reset --log_comment
-CLICKHOUSE_LOG_COMMENT=
-# shellcheck source=../shell_config.sh
-. "$CUR_DIR"/../shell_config.sh
-
-CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --use_variant_as_common_type=1 --allow_experimental_dynamic_type=1"
-
-
-function test()
-{
-    echo "test"
-    $CH_CLIENT -q "insert into test select number, number from numbers(100000) settings min_insert_block_size_rows=50000"
-    $CH_CLIENT -q "insert into test select number, 'str_' || toString(number) from numbers(100000, 100000) settings min_insert_block_size_rows=50000"
-    $CH_CLIENT -q "insert into test select number, arrayMap(x -> multiIf(number % 9 == 0, NULL, number % 9 == 3, 'str_' || toString(number), number), range(number % 10 + 1)) from numbers(200000, 100000) settings min_insert_block_size_rows=50000"
-    $CH_CLIENT -q "insert into test select number, NULL from numbers(300000, 100000) settings min_insert_block_size_rows=50000"
-    $CH_CLIENT -q "insert into test select number, multiIf(number % 4 == 3, 'str_' || toString(number), number % 4 == 2, NULL, number % 4 == 1, number, arrayMap(x -> multiIf(number % 9 == 0, NULL, number % 9 == 3, 'str_' || toString(number), number), range(number % 10 + 1))) from numbers(400000, 400000) settings min_insert_block_size_rows=50000"
-    $CH_CLIENT -q "insert into test select number, [range((number % 10 + 1)::UInt64)]::Array(Array(Dynamic)) from numbers(100000, 100000) settings min_insert_block_size_rows=50000"
-
-    $CH_CLIENT -q "select distinct dynamicType(d) as type from test order by type"
-    $CH_CLIENT -q "select count() from test where dynamicType(d) == 'UInt64'"
-    $CH_CLIENT -q "select count() from test where d.UInt64 is not NULL"
-    $CH_CLIENT -q "select count() from test where dynamicType(d) == 'String'"
-    $CH_CLIENT -q "select count() from test where d.String is not NULL"
-    $CH_CLIENT -q "select count() from test where dynamicType(d) == 'Date'"
-    $CH_CLIENT -q "select count() from test where d.Date is not NULL"
-    $CH_CLIENT -q "select count() from test where dynamicType(d) == 'Array(Variant(String, UInt64))'"
-    $CH_CLIENT -q "select count() from test where not empty(d.\`Array(Variant(String, UInt64))\`)"
-    $CH_CLIENT -q "select count() from test where dynamicType(d) == 'Array(Array(Dynamic))'"
-    $CH_CLIENT -q "select count() from test where not empty(d.\`Array(Array(Dynamic))\`)"
-    $CH_CLIENT -q "select count() from test where d is NULL"
-    $CH_CLIENT -q "select count() from test where not empty(d.\`Tuple(a Array(Dynamic))\`.a.String)"
-
-    $CH_CLIENT -q "select d, d.UInt64, d.String, d.\`Array(Variant(String, UInt64))\` from test format Null"
-    $CH_CLIENT -q "select d.UInt64, d.String, d.\`Array(Variant(String, UInt64))\` from test format Null"
-    $CH_CLIENT -q "select d.Int8, d.Date, d.\`Array(String)\` from test format Null"
-    $CH_CLIENT -q "select d, d.UInt64, d.Date, d.\`Array(Variant(String, UInt64))\`, d.\`Array(Variant(String, UInt64))\`.size0, d.\`Array(Variant(String, UInt64))\`.UInt64 from test format Null"
-    $CH_CLIENT -q "select d.UInt64, d.Date, d.\`Array(Variant(String, UInt64))\`, d.\`Array(Variant(String, UInt64))\`.size0, d.\`Array(Variant(String, UInt64))\`.UInt64, d.\`Array(Variant(String, UInt64))\`.String from test format Null"
-    $CH_CLIENT -q "select d, d.\`Tuple(a UInt64, b String)\`.a, d.\`Array(Dynamic)\`.\`Variant(String, UInt64)\`.UInt64, d.\`Array(Variant(String, UInt64))\`.UInt64 from test format Null"
-    $CH_CLIENT -q "select d.\`Array(Dynamic)\`.\`Variant(String, UInt64)\`.UInt64, d.\`Array(Dynamic)\`.size0, d.\`Array(Variant(String, UInt64))\`.UInt64 from test format Null"
-    $CH_CLIENT -q "select d.\`Array(Array(Dynamic))\`.size1, d.\`Array(Array(Dynamic))\`.UInt64, d.\`Array(Array(Dynamic))\`.\`Map(String, Tuple(a UInt64))\`.values.a from test format Null"
-}
-
-$CH_CLIENT -q "drop table if exists test;"
-
-echo "Memory"
-$CH_CLIENT -q "create table test (id UInt64, d Dynamic) engine=Memory"
-test
-$CH_CLIENT -q "drop table test;"
-
-echo "MergeTree compact"
-$CH_CLIENT -q "create table test (id UInt64, d Dynamic) engine=MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000;"
-test
-$CH_CLIENT -q "drop table test;"
-
-echo "MergeTree wide"
-$CH_CLIENT -q "create table test (id UInt64, d Dynamic) engine=MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1;"
-test
-$CH_CLIENT -q "drop table test;"
diff --git a/tests/queries/0_stateless/03036_dynamic_read_subcolumns_compact_merge_tree.reference b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_compact_merge_tree.reference
new file mode 100644
index 00000000000..d75d75896f7
--- /dev/null
+++ b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_compact_merge_tree.reference
@@ -0,0 +1,17 @@
+Array(Array(Dynamic))
+Array(Variant(String, UInt64))
+None
+String
+UInt64
+200000
+200000
+200000
+200000
+0
+0
+200000
+200000
+100000
+100000
+200000
+0
diff --git a/tests/queries/0_stateless/03036_dynamic_read_subcolumns_compact_merge_tree.sql b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_compact_merge_tree.sql
new file mode 100644
index 00000000000..66fbf006a8c
--- /dev/null
+++ b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_compact_merge_tree.sql
@@ -0,0 +1,40 @@
+-- Tags: long
+
+set allow_experimental_variant_type = 1;
+set use_variant_as_common_type = 1;
+set allow_experimental_dynamic_type = 1;
+
+drop table if exists test;
+create table test (id UInt64, d Dynamic) engine=MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000;
+
+insert into test select number, number from numbers(100000) settings min_insert_block_size_rows=50000;
+insert into test select number, 'str_' || toString(number) from numbers(100000, 100000) settings min_insert_block_size_rows=50000;
+insert into test select number, arrayMap(x -> multiIf(number % 9 == 0, NULL, number % 9 == 3, 'str_' || toString(number), number), range(number % 10 + 1)) from numbers(200000, 100000) settings min_insert_block_size_rows=50000;
+insert into test select number, NULL from numbers(300000, 100000) settings min_insert_block_size_rows=50000;
+insert into test select number, multiIf(number % 4 == 3, 'str_' || toString(number), number % 4 == 2, NULL, number % 4 == 1, number, arrayMap(x -> multiIf(number % 9 == 0, NULL, number % 9 == 3, 'str_' || toString(number), number), range(number % 10 + 1))) from numbers(400000, 400000) settings min_insert_block_size_rows=50000;
+insert into test select number, [range((number % 10 + 1)::UInt64)]::Array(Array(Dynamic)) from numbers(100000, 100000) settings min_insert_block_size_rows=50000;
+
+select distinct dynamicType(d) as type from test order by type;
+select count() from test where dynamicType(d) == 'UInt64';
+select count() from test where d.UInt64 is not NULL;
+select count() from test where dynamicType(d) == 'String';
+select count() from test where d.String is not NULL;
+select count() from test where dynamicType(d) == 'Date';
+select count() from test where d.Date is not NULL;
+select count() from test where dynamicType(d) == 'Array(Variant(String, UInt64))';
+select count() from test where not empty(d.`Array(Variant(String, UInt64))`);
+select count() from test where dynamicType(d) == 'Array(Array(Dynamic))';
+select count() from test where not empty(d.`Array(Array(Dynamic))`);
+select count() from test where d is NULL;
+select count() from test where not empty(d.`Tuple(a Array(Dynamic))`.a.String);
+
+select d, d.UInt64, d.String, d.`Array(Variant(String, UInt64))` from test format Null;
+select d.UInt64, d.String, d.`Array(Variant(String, UInt64))` from test format Null;
+select d.Int8, d.Date, d.`Array(String)` from test format Null;
+select d, d.UInt64, d.Date, d.`Array(Variant(String, UInt64))`, d.`Array(Variant(String, UInt64))`.size0, d.`Array(Variant(String, UInt64))`.UInt64 from test format Null;
+select d.UInt64, d.Date, d.`Array(Variant(String, UInt64))`, d.`Array(Variant(String, UInt64))`.size0, d.`Array(Variant(String, UInt64))`.UInt64, d.`Array(Variant(String, UInt64))`.String from test format Null;
+select d, d.`Tuple(a UInt64, b String)`.a, d.`Array(Dynamic)`.`Variant(String, UInt64)`.UInt64, d.`Array(Variant(String, UInt64))`.UInt64 from test format Null;
+select d.`Array(Dynamic)`.`Variant(String, UInt64)`.UInt64, d.`Array(Dynamic)`.size0, d.`Array(Variant(String, UInt64))`.UInt64 from test format Null;
+select d.`Array(Array(Dynamic))`.size1, d.`Array(Array(Dynamic))`.UInt64, d.`Array(Array(Dynamic))`.`Map(String, Tuple(a UInt64))`.values.a from test format Null;
+    
+drop table test;
diff --git a/tests/queries/0_stateless/03036_dynamic_read_subcolumns_memory.reference b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_memory.reference
new file mode 100644
index 00000000000..d75d75896f7
--- /dev/null
+++ b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_memory.reference
@@ -0,0 +1,17 @@
+Array(Array(Dynamic))
+Array(Variant(String, UInt64))
+None
+String
+UInt64
+200000
+200000
+200000
+200000
+0
+0
+200000
+200000
+100000
+100000
+200000
+0
diff --git a/tests/queries/0_stateless/03036_dynamic_read_subcolumns_memory.sql b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_memory.sql
new file mode 100644
index 00000000000..bb03bdef704
--- /dev/null
+++ b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_memory.sql
@@ -0,0 +1,40 @@
+-- Tags: long
+
+set allow_experimental_variant_type = 1;
+set use_variant_as_common_type = 1;
+set allow_experimental_dynamic_type = 1;
+
+drop table if exists test;
+create table test (id UInt64, d Dynamic) engine=Memory;
+
+insert into test select number, number from numbers(100000) settings min_insert_block_size_rows=50000;
+insert into test select number, 'str_' || toString(number) from numbers(100000, 100000) settings min_insert_block_size_rows=50000;
+insert into test select number, arrayMap(x -> multiIf(number % 9 == 0, NULL, number % 9 == 3, 'str_' || toString(number), number), range(number % 10 + 1)) from numbers(200000, 100000) settings min_insert_block_size_rows=50000;
+insert into test select number, NULL from numbers(300000, 100000) settings min_insert_block_size_rows=50000;
+insert into test select number, multiIf(number % 4 == 3, 'str_' || toString(number), number % 4 == 2, NULL, number % 4 == 1, number, arrayMap(x -> multiIf(number % 9 == 0, NULL, number % 9 == 3, 'str_' || toString(number), number), range(number % 10 + 1))) from numbers(400000, 400000) settings min_insert_block_size_rows=50000;
+insert into test select number, [range((number % 10 + 1)::UInt64)]::Array(Array(Dynamic)) from numbers(100000, 100000) settings min_insert_block_size_rows=50000;
+
+select distinct dynamicType(d) as type from test order by type;
+select count() from test where dynamicType(d) == 'UInt64';
+select count() from test where d.UInt64 is not NULL;
+select count() from test where dynamicType(d) == 'String';
+select count() from test where d.String is not NULL;
+select count() from test where dynamicType(d) == 'Date';
+select count() from test where d.Date is not NULL;
+select count() from test where dynamicType(d) == 'Array(Variant(String, UInt64))';
+select count() from test where not empty(d.`Array(Variant(String, UInt64))`);
+select count() from test where dynamicType(d) == 'Array(Array(Dynamic))';
+select count() from test where not empty(d.`Array(Array(Dynamic))`);
+select count() from test where d is NULL;
+select count() from test where not empty(d.`Tuple(a Array(Dynamic))`.a.String);
+
+select d, d.UInt64, d.String, d.`Array(Variant(String, UInt64))` from test format Null;
+select d.UInt64, d.String, d.`Array(Variant(String, UInt64))` from test format Null;
+select d.Int8, d.Date, d.`Array(String)` from test format Null;
+select d, d.UInt64, d.Date, d.`Array(Variant(String, UInt64))`, d.`Array(Variant(String, UInt64))`.size0, d.`Array(Variant(String, UInt64))`.UInt64 from test format Null;
+select d.UInt64, d.Date, d.`Array(Variant(String, UInt64))`, d.`Array(Variant(String, UInt64))`.size0, d.`Array(Variant(String, UInt64))`.UInt64, d.`Array(Variant(String, UInt64))`.String from test format Null;
+select d, d.`Tuple(a UInt64, b String)`.a, d.`Array(Dynamic)`.`Variant(String, UInt64)`.UInt64, d.`Array(Variant(String, UInt64))`.UInt64 from test format Null;
+select d.`Array(Dynamic)`.`Variant(String, UInt64)`.UInt64, d.`Array(Dynamic)`.size0, d.`Array(Variant(String, UInt64))`.UInt64 from test format Null;
+select d.`Array(Array(Dynamic))`.size1, d.`Array(Array(Dynamic))`.UInt64, d.`Array(Array(Dynamic))`.`Map(String, Tuple(a UInt64))`.values.a from test format Null;
+    
+drop table test;
diff --git a/tests/queries/0_stateless/03036_dynamic_read_subcolumns_wide_merge_tree.reference b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_wide_merge_tree.reference
new file mode 100644
index 00000000000..d75d75896f7
--- /dev/null
+++ b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_wide_merge_tree.reference
@@ -0,0 +1,17 @@
+Array(Array(Dynamic))
+Array(Variant(String, UInt64))
+None
+String
+UInt64
+200000
+200000
+200000
+200000
+0
+0
+200000
+200000
+100000
+100000
+200000
+0
diff --git a/tests/queries/0_stateless/03036_dynamic_read_subcolumns_wide_merge_tree.sql b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_wide_merge_tree.sql
new file mode 100644
index 00000000000..00aba3a57b6
--- /dev/null
+++ b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_wide_merge_tree.sql
@@ -0,0 +1,40 @@
+-- Tags: long
+
+set allow_experimental_variant_type = 1;
+set use_variant_as_common_type = 1;
+set allow_experimental_dynamic_type = 1;
+
+drop table if exists test;
+create table test (id UInt64, d Dynamic) engine=MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1;
+
+insert into test select number, number from numbers(100000) settings min_insert_block_size_rows=50000;
+insert into test select number, 'str_' || toString(number) from numbers(100000, 100000) settings min_insert_block_size_rows=50000;
+insert into test select number, arrayMap(x -> multiIf(number % 9 == 0, NULL, number % 9 == 3, 'str_' || toString(number), number), range(number % 10 + 1)) from numbers(200000, 100000) settings min_insert_block_size_rows=50000;
+insert into test select number, NULL from numbers(300000, 100000) settings min_insert_block_size_rows=50000;
+insert into test select number, multiIf(number % 4 == 3, 'str_' || toString(number), number % 4 == 2, NULL, number % 4 == 1, number, arrayMap(x -> multiIf(number % 9 == 0, NULL, number % 9 == 3, 'str_' || toString(number), number), range(number % 10 + 1))) from numbers(400000, 400000) settings min_insert_block_size_rows=50000;
+insert into test select number, [range((number % 10 + 1)::UInt64)]::Array(Array(Dynamic)) from numbers(100000, 100000) settings min_insert_block_size_rows=50000;
+
+select distinct dynamicType(d) as type from test order by type;
+select count() from test where dynamicType(d) == 'UInt64';
+select count() from test where d.UInt64 is not NULL;
+select count() from test where dynamicType(d) == 'String';
+select count() from test where d.String is not NULL;
+select count() from test where dynamicType(d) == 'Date';
+select count() from test where d.Date is not NULL;
+select count() from test where dynamicType(d) == 'Array(Variant(String, UInt64))';
+select count() from test where not empty(d.`Array(Variant(String, UInt64))`);
+select count() from test where dynamicType(d) == 'Array(Array(Dynamic))';
+select count() from test where not empty(d.`Array(Array(Dynamic))`);
+select count() from test where d is NULL;
+select count() from test where not empty(d.`Tuple(a Array(Dynamic))`.a.String);
+
+select d, d.UInt64, d.String, d.`Array(Variant(String, UInt64))` from test format Null;
+select d.UInt64, d.String, d.`Array(Variant(String, UInt64))` from test format Null;
+select d.Int8, d.Date, d.`Array(String)` from test format Null;
+select d, d.UInt64, d.Date, d.`Array(Variant(String, UInt64))`, d.`Array(Variant(String, UInt64))`.size0, d.`Array(Variant(String, UInt64))`.UInt64 from test format Null;
+select d.UInt64, d.Date, d.`Array(Variant(String, UInt64))`, d.`Array(Variant(String, UInt64))`.size0, d.`Array(Variant(String, UInt64))`.UInt64, d.`Array(Variant(String, UInt64))`.String from test format Null;
+select d, d.`Tuple(a UInt64, b String)`.a, d.`Array(Dynamic)`.`Variant(String, UInt64)`.UInt64, d.`Array(Variant(String, UInt64))`.UInt64 from test format Null;
+select d.`Array(Dynamic)`.`Variant(String, UInt64)`.UInt64, d.`Array(Dynamic)`.size0, d.`Array(Variant(String, UInt64))`.UInt64 from test format Null;
+select d.`Array(Array(Dynamic))`.size1, d.`Array(Array(Dynamic))`.UInt64, d.`Array(Array(Dynamic))`.`Map(String, Tuple(a UInt64))`.values.a from test format Null;
+    
+drop table test;
diff --git a/tests/queries/0_stateless/03037_dynamic_merges_1_horizontal.reference b/tests/queries/0_stateless/03037_dynamic_merges_1_horizontal.reference
deleted file mode 100644
index 59297e46330..00000000000
--- a/tests/queries/0_stateless/03037_dynamic_merges_1_horizontal.reference
+++ /dev/null
@@ -1,60 +0,0 @@
-MergeTree compact
-test
-50000	DateTime
-60000	Date
-70000	Array(UInt16)
-80000	String
-100000	None
-100000	UInt64
-70000	Array(UInt16)
-100000	None
-100000	UInt64
-190000	String
-70000	Array(UInt16)
-100000	None
-100000	UInt64
-190000	String
-200000	Map(UInt64, UInt64)
-100000	None
-100000	UInt64
-200000	Map(UInt64, UInt64)
-260000	String
-10000	Tuple(UInt64, UInt64)
-100000	None
-100000	UInt64
-200000	Map(UInt64, UInt64)
-260000	String
-100000	None
-100000	UInt64
-200000	Map(UInt64, UInt64)
-270000	String
-MergeTree wide
-test
-50000	DateTime
-60000	Date
-70000	Array(UInt16)
-80000	String
-100000	None
-100000	UInt64
-70000	Array(UInt16)
-100000	None
-100000	UInt64
-190000	String
-70000	Array(UInt16)
-100000	None
-100000	UInt64
-190000	String
-200000	Map(UInt64, UInt64)
-100000	None
-100000	UInt64
-200000	Map(UInt64, UInt64)
-260000	String
-10000	Tuple(UInt64, UInt64)
-100000	None
-100000	UInt64
-200000	Map(UInt64, UInt64)
-260000	String
-100000	None
-100000	UInt64
-200000	Map(UInt64, UInt64)
-270000	String
diff --git a/tests/queries/0_stateless/03037_dynamic_merges_1_horizontal.sh b/tests/queries/0_stateless/03037_dynamic_merges_1_horizontal.sh
deleted file mode 100755
index 887b2ed94d7..00000000000
--- a/tests/queries/0_stateless/03037_dynamic_merges_1_horizontal.sh
+++ /dev/null
@@ -1,52 +0,0 @@
-#!/usr/bin/env bash
-# Tags: long
-
-CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# reset --log_comment
-CLICKHOUSE_LOG_COMMENT=
-# shellcheck source=../shell_config.sh
-. "$CUR_DIR"/../shell_config.sh
-
-
-CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_dynamic_type=1"
-
-function test()
-{
-    echo "test"
-    $CH_CLIENT -q "system stop merges test"
-    $CH_CLIENT -q "insert into test select number, number from numbers(100000)"
-    $CH_CLIENT -q "insert into test select number, 'str_' || toString(number) from numbers(80000)"
-    $CH_CLIENT -q "insert into test select number, range(number % 10 + 1) from numbers(70000)"
-    $CH_CLIENT -q "insert into test select number, toDate(number) from numbers(60000)"
-    $CH_CLIENT -q "insert into test select number, toDateTime(number) from numbers(50000)"
-    $CH_CLIENT -q "insert into test select number, NULL from numbers(100000)"
-
-    $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)"
-    $CH_CLIENT -nm -q "system start merges test; optimize table test final;"
-    $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)"
-
-    $CH_CLIENT -q "system stop merges test"
-    $CH_CLIENT -q "insert into test select number, map(number, number) from numbers(200000)"
-    $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)"
-    $CH_CLIENT -nm -q "system start merges test; optimize table test final;"
-    $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)"
-
-    $CH_CLIENT -q "system stop merges test"
-    $CH_CLIENT -q "insert into test select number, tuple(number, number) from numbers(10000)"
-    $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)"
-    $CH_CLIENT -nm -q "system start merges test; optimize table test final;"
-    $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)"
-}
-
-$CH_CLIENT -q "drop table if exists test;"
-
-echo "MergeTree compact"
-$CH_CLIENT -q "create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000, vertical_merge_algorithm_min_columns_to_activate=10, index_granularity_bytes=10485760, index_granularity=8192, merge_max_block_size=8192, merge_max_block_size_bytes=10485760;"
-test
-$CH_CLIENT -q "drop table test;"
-
-echo "MergeTree wide"
-$CH_CLIENT -q "create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1, vertical_merge_algorithm_min_columns_to_activate=10, index_granularity_bytes=10485760, index_granularity=8192, merge_max_block_size=8192, merge_max_block_size_bytes=10485760;"
-test
-$CH_CLIENT -q "drop table test;"
-
diff --git a/tests/queries/0_stateless/03037_dynamic_merges_1_horizontal_compact_merge_tree.reference b/tests/queries/0_stateless/03037_dynamic_merges_1_horizontal_compact_merge_tree.reference
new file mode 100644
index 00000000000..d0d777a5a38
--- /dev/null
+++ b/tests/queries/0_stateless/03037_dynamic_merges_1_horizontal_compact_merge_tree.reference
@@ -0,0 +1,28 @@
+50000	DateTime
+60000	Date
+70000	Array(UInt16)
+80000	String
+100000	None
+100000	UInt64
+70000	Array(UInt16)
+100000	None
+100000	UInt64
+190000	String
+70000	Array(UInt16)
+100000	None
+100000	UInt64
+190000	String
+200000	Map(UInt64, UInt64)
+100000	None
+100000	UInt64
+200000	Map(UInt64, UInt64)
+260000	String
+10000	Tuple(UInt64, UInt64)
+100000	None
+100000	UInt64
+200000	Map(UInt64, UInt64)
+260000	String
+100000	None
+100000	UInt64
+200000	Map(UInt64, UInt64)
+270000	String
diff --git a/tests/queries/0_stateless/03037_dynamic_merges_1_horizontal_compact_merge_tree.sql b/tests/queries/0_stateless/03037_dynamic_merges_1_horizontal_compact_merge_tree.sql
new file mode 100644
index 00000000000..b66fe5e2187
--- /dev/null
+++ b/tests/queries/0_stateless/03037_dynamic_merges_1_horizontal_compact_merge_tree.sql
@@ -0,0 +1,33 @@
+-- Tags: long
+set allow_experimental_dynamic_type=1;
+
+drop table if exists test;
+create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000, vertical_merge_algorithm_min_columns_to_activate=10, index_granularity_bytes=10485760, index_granularity=8192, merge_max_block_size=8192, merge_max_block_size_bytes=10485760;
+
+system stop merges test;
+insert into test select number, number from numbers(100000);
+insert into test select number, 'str_' || toString(number) from numbers(80000);
+insert into test select number, range(number % 10 + 1) from numbers(70000);
+insert into test select number, toDate(number) from numbers(60000);
+insert into test select number, toDateTime(number) from numbers(50000);
+insert into test select number, NULL from numbers(100000);
+
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+system start merges test; optimize table test final;;
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+
+system stop merges test;
+insert into test select number, map(number, number) from numbers(200000);
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+system start merges test;
+optimize table test final;
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+
+system stop merges test;
+insert into test select number, tuple(number, number) from numbers(10000);
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+system start merges test;
+optimize table test final;
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+
+drop table test;
\ No newline at end of file
diff --git a/tests/queries/0_stateless/03037_dynamic_merges_1_horizontal_compact_wide_tree.reference b/tests/queries/0_stateless/03037_dynamic_merges_1_horizontal_compact_wide_tree.reference
new file mode 100644
index 00000000000..d0d777a5a38
--- /dev/null
+++ b/tests/queries/0_stateless/03037_dynamic_merges_1_horizontal_compact_wide_tree.reference
@@ -0,0 +1,28 @@
+50000	DateTime
+60000	Date
+70000	Array(UInt16)
+80000	String
+100000	None
+100000	UInt64
+70000	Array(UInt16)
+100000	None
+100000	UInt64
+190000	String
+70000	Array(UInt16)
+100000	None
+100000	UInt64
+190000	String
+200000	Map(UInt64, UInt64)
+100000	None
+100000	UInt64
+200000	Map(UInt64, UInt64)
+260000	String
+10000	Tuple(UInt64, UInt64)
+100000	None
+100000	UInt64
+200000	Map(UInt64, UInt64)
+260000	String
+100000	None
+100000	UInt64
+200000	Map(UInt64, UInt64)
+270000	String
diff --git a/tests/queries/0_stateless/03037_dynamic_merges_1_horizontal_compact_wide_tree.sql b/tests/queries/0_stateless/03037_dynamic_merges_1_horizontal_compact_wide_tree.sql
new file mode 100644
index 00000000000..8a376b6d7d7
--- /dev/null
+++ b/tests/queries/0_stateless/03037_dynamic_merges_1_horizontal_compact_wide_tree.sql
@@ -0,0 +1,33 @@
+-- Tags: long
+set allow_experimental_dynamic_type=1;
+
+drop table if exists test;
+create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1, vertical_merge_algorithm_min_columns_to_activate=10, index_granularity_bytes=10485760, index_granularity=8192, merge_max_block_size=8192, merge_max_block_size_bytes=10485760;
+
+system stop merges test;
+insert into test select number, number from numbers(100000);
+insert into test select number, 'str_' || toString(number) from numbers(80000);
+insert into test select number, range(number % 10 + 1) from numbers(70000);
+insert into test select number, toDate(number) from numbers(60000);
+insert into test select number, toDateTime(number) from numbers(50000);
+insert into test select number, NULL from numbers(100000);
+
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+system start merges test; optimize table test final;;
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+
+system stop merges test;
+insert into test select number, map(number, number) from numbers(200000);
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+system start merges test;
+optimize table test final;
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+
+system stop merges test;
+insert into test select number, tuple(number, number) from numbers(10000);
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+system start merges test;
+optimize table test final;
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+
+drop table test;
\ No newline at end of file
diff --git a/tests/queries/0_stateless/03037_dynamic_merges_1_vertical.reference b/tests/queries/0_stateless/03037_dynamic_merges_1_vertical.reference
deleted file mode 100644
index 59297e46330..00000000000
--- a/tests/queries/0_stateless/03037_dynamic_merges_1_vertical.reference
+++ /dev/null
@@ -1,60 +0,0 @@
-MergeTree compact
-test
-50000	DateTime
-60000	Date
-70000	Array(UInt16)
-80000	String
-100000	None
-100000	UInt64
-70000	Array(UInt16)
-100000	None
-100000	UInt64
-190000	String
-70000	Array(UInt16)
-100000	None
-100000	UInt64
-190000	String
-200000	Map(UInt64, UInt64)
-100000	None
-100000	UInt64
-200000	Map(UInt64, UInt64)
-260000	String
-10000	Tuple(UInt64, UInt64)
-100000	None
-100000	UInt64
-200000	Map(UInt64, UInt64)
-260000	String
-100000	None
-100000	UInt64
-200000	Map(UInt64, UInt64)
-270000	String
-MergeTree wide
-test
-50000	DateTime
-60000	Date
-70000	Array(UInt16)
-80000	String
-100000	None
-100000	UInt64
-70000	Array(UInt16)
-100000	None
-100000	UInt64
-190000	String
-70000	Array(UInt16)
-100000	None
-100000	UInt64
-190000	String
-200000	Map(UInt64, UInt64)
-100000	None
-100000	UInt64
-200000	Map(UInt64, UInt64)
-260000	String
-10000	Tuple(UInt64, UInt64)
-100000	None
-100000	UInt64
-200000	Map(UInt64, UInt64)
-260000	String
-100000	None
-100000	UInt64
-200000	Map(UInt64, UInt64)
-270000	String
diff --git a/tests/queries/0_stateless/03037_dynamic_merges_1_vertical.sh b/tests/queries/0_stateless/03037_dynamic_merges_1_vertical.sh
deleted file mode 100755
index 371ae87c2ef..00000000000
--- a/tests/queries/0_stateless/03037_dynamic_merges_1_vertical.sh
+++ /dev/null
@@ -1,51 +0,0 @@
-#!/usr/bin/env bash
-# Tags: long
-
-CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# reset --log_comment
-CLICKHOUSE_LOG_COMMENT=
-# shellcheck source=../shell_config.sh
-. "$CUR_DIR"/../shell_config.sh
-
-
-
-CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_dynamic_type=1"
-function test()
-{
-    echo "test"
-    $CH_CLIENT -q "system stop merges test"
-    $CH_CLIENT -q "insert into test select number, number from numbers(100000)"
-    $CH_CLIENT -q "insert into test select number, 'str_' || toString(number) from numbers(80000)"
-    $CH_CLIENT -q "insert into test select number, range(number % 10 + 1) from numbers(70000)"
-    $CH_CLIENT -q "insert into test select number, toDate(number) from numbers(60000)"
-    $CH_CLIENT -q "insert into test select number, toDateTime(number) from numbers(50000)"
-    $CH_CLIENT -q "insert into test select number, NULL from numbers(100000)"
-
-    $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)"
-    $CH_CLIENT -nm -q "system start merges test; optimize table test final;"
-    $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)"
-
-    $CH_CLIENT -q "system stop merges test"
-    $CH_CLIENT -q "insert into test select number, map(number, number) from numbers(200000)"
-    $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)"
-    $CH_CLIENT -nm -q "system start merges test; optimize table test final;"
-    $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)"
-
-    $CH_CLIENT -q "system stop merges test"
-    $CH_CLIENT -q "insert into test select number, tuple(number, number) from numbers(10000)"
-    $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)"
-    $CH_CLIENT -nm -q "system start merges test; optimize table test final;"
-    $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)"
-}
-
-$CH_CLIENT -q "drop table if exists test;"
-
-echo "MergeTree compact"
-$CH_CLIENT -q "create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1, index_granularity_bytes=10485760, index_granularity=8192, merge_max_block_size=8192, merge_max_block_size_bytes=10485760;"
-test
-$CH_CLIENT -q "drop table test;"
-
-echo "MergeTree wide"
-$CH_CLIENT -q "create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1, index_granularity_bytes=10485760, index_granularity=8192, merge_max_block_size=8192, merge_max_block_size_bytes=10485760;"
-test
-$CH_CLIENT -q "drop table test;"
diff --git a/tests/queries/0_stateless/03037_dynamic_merges_1_vertical_compact_merge_tree.reference b/tests/queries/0_stateless/03037_dynamic_merges_1_vertical_compact_merge_tree.reference
new file mode 100644
index 00000000000..d0d777a5a38
--- /dev/null
+++ b/tests/queries/0_stateless/03037_dynamic_merges_1_vertical_compact_merge_tree.reference
@@ -0,0 +1,28 @@
+50000	DateTime
+60000	Date
+70000	Array(UInt16)
+80000	String
+100000	None
+100000	UInt64
+70000	Array(UInt16)
+100000	None
+100000	UInt64
+190000	String
+70000	Array(UInt16)
+100000	None
+100000	UInt64
+190000	String
+200000	Map(UInt64, UInt64)
+100000	None
+100000	UInt64
+200000	Map(UInt64, UInt64)
+260000	String
+10000	Tuple(UInt64, UInt64)
+100000	None
+100000	UInt64
+200000	Map(UInt64, UInt64)
+260000	String
+100000	None
+100000	UInt64
+200000	Map(UInt64, UInt64)
+270000	String
diff --git a/tests/queries/0_stateless/03037_dynamic_merges_1_vertical_compact_merge_tree.sql b/tests/queries/0_stateless/03037_dynamic_merges_1_vertical_compact_merge_tree.sql
new file mode 100644
index 00000000000..127b56e727c
--- /dev/null
+++ b/tests/queries/0_stateless/03037_dynamic_merges_1_vertical_compact_merge_tree.sql
@@ -0,0 +1,33 @@
+-- Tags: long
+set allow_experimental_dynamic_type=1;
+
+drop table if exists test;
+create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1, index_granularity_bytes=10485760, index_granularity=8192, merge_max_block_size=8192, merge_max_block_size_bytes=10485760;
+
+system stop merges test;
+insert into test select number, number from numbers(100000);
+insert into test select number, 'str_' || toString(number) from numbers(80000);
+insert into test select number, range(number % 10 + 1) from numbers(70000);
+insert into test select number, toDate(number) from numbers(60000);
+insert into test select number, toDateTime(number) from numbers(50000);
+insert into test select number, NULL from numbers(100000);
+
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+system start merges test; optimize table test final;;
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+
+system stop merges test;
+insert into test select number, map(number, number) from numbers(200000);
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+system start merges test;
+optimize table test final;
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+
+system stop merges test;
+insert into test select number, tuple(number, number) from numbers(10000);
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+system start merges test;
+optimize table test final;
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+
+drop table test;
\ No newline at end of file
diff --git a/tests/queries/0_stateless/03037_dynamic_merges_1_vertical_wide_merge_tree.reference b/tests/queries/0_stateless/03037_dynamic_merges_1_vertical_wide_merge_tree.reference
new file mode 100644
index 00000000000..d0d777a5a38
--- /dev/null
+++ b/tests/queries/0_stateless/03037_dynamic_merges_1_vertical_wide_merge_tree.reference
@@ -0,0 +1,28 @@
+50000	DateTime
+60000	Date
+70000	Array(UInt16)
+80000	String
+100000	None
+100000	UInt64
+70000	Array(UInt16)
+100000	None
+100000	UInt64
+190000	String
+70000	Array(UInt16)
+100000	None
+100000	UInt64
+190000	String
+200000	Map(UInt64, UInt64)
+100000	None
+100000	UInt64
+200000	Map(UInt64, UInt64)
+260000	String
+10000	Tuple(UInt64, UInt64)
+100000	None
+100000	UInt64
+200000	Map(UInt64, UInt64)
+260000	String
+100000	None
+100000	UInt64
+200000	Map(UInt64, UInt64)
+270000	String
diff --git a/tests/queries/0_stateless/03037_dynamic_merges_1_vertical_wide_merge_tree.sql b/tests/queries/0_stateless/03037_dynamic_merges_1_vertical_wide_merge_tree.sql
new file mode 100644
index 00000000000..e5c273cb592
--- /dev/null
+++ b/tests/queries/0_stateless/03037_dynamic_merges_1_vertical_wide_merge_tree.sql
@@ -0,0 +1,33 @@
+-- Tags: long
+set allow_experimental_dynamic_type=1;
+
+drop table if exists test;
+create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1, index_granularity_bytes=10485760, index_granularity=8192, merge_max_block_size=8192, merge_max_block_size_bytes=10485760;
+
+system stop merges test;
+insert into test select number, number from numbers(100000);
+insert into test select number, 'str_' || toString(number) from numbers(80000);
+insert into test select number, range(number % 10 + 1) from numbers(70000);
+insert into test select number, toDate(number) from numbers(60000);
+insert into test select number, toDateTime(number) from numbers(50000);
+insert into test select number, NULL from numbers(100000);
+
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+system start merges test; optimize table test final;;
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+
+system stop merges test;
+insert into test select number, map(number, number) from numbers(200000);
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+system start merges test;
+optimize table test final;
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+
+system stop merges test;
+insert into test select number, tuple(number, number) from numbers(10000);
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+system start merges test;
+optimize table test final;
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+
+drop table test;
\ No newline at end of file
diff --git a/tests/queries/0_stateless/03037_dynamic_merges_2.reference b/tests/queries/0_stateless/03037_dynamic_merges_2.reference
deleted file mode 100644
index 420b8185b16..00000000000
--- a/tests/queries/0_stateless/03037_dynamic_merges_2.reference
+++ /dev/null
@@ -1,20 +0,0 @@
-MergeTree compact + horizontal merge
-test
-1000000	Array(UInt16)
-1000000	String
-1000000	UInt64
-MergeTree wide + horizontal merge
-test
-1000000	Array(UInt16)
-1000000	String
-1000000	UInt64
-MergeTree compact + vertical merge
-test
-1000000	Array(UInt16)
-1000000	String
-1000000	UInt64
-MergeTree wide + vertical merge
-test
-1000000	Array(UInt16)
-1000000	String
-1000000	UInt64
diff --git a/tests/queries/0_stateless/03037_dynamic_merges_2.sh b/tests/queries/0_stateless/03037_dynamic_merges_2.sh
deleted file mode 100755
index 40adbdd4262..00000000000
--- a/tests/queries/0_stateless/03037_dynamic_merges_2.sh
+++ /dev/null
@@ -1,45 +0,0 @@
-#!/usr/bin/env bash
-# Tags: long
-
-CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# reset --log_comment
-CLICKHOUSE_LOG_COMMENT=
-# shellcheck source=../shell_config.sh
-. "$CUR_DIR"/../shell_config.sh
-
-CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_dynamic_type=1"
-
-
-function test()
-{
-    echo "test"
-    $CH_CLIENT -q "system stop merges test"
-    $CH_CLIENT -q "insert into test select number, number from numbers(1000000)"
-    $CH_CLIENT -q "insert into test select number, 'str_' || toString(number) from numbers(1000000, 1000000)"
-    $CH_CLIENT -q "insert into test select number, range(number % 10 + 1) from numbers(2000000, 1000000)"
-
-    $CH_CLIENT -nm -q "system start merges test; optimize table test final;"
-    $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)"
-}
-
-$CH_CLIENT -q "drop table if exists test;"
-
-echo "MergeTree compact + horizontal merge"
-$CH_CLIENT -q "create table test (id UInt64, d Dynamic) engine=MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000;"
-test
-$CH_CLIENT -q "drop table test;"
-
-echo "MergeTree wide + horizontal merge"
-$CH_CLIENT -q "create table test (id UInt64, d Dynamic) engine=MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1;"
-test
-$CH_CLIENT -q "drop table test;"
-
-echo "MergeTree compact + vertical merge"
-$CH_CLIENT -q "create table test (id UInt64, d Dynamic) engine=MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1;"
-test
-$CH_CLIENT -q "drop table test;"
-
-echo "MergeTree wide + vertical merge"
-$CH_CLIENT -q "create table test (id UInt64, d Dynamic) engine=MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1;"
-test
-$CH_CLIENT -q "drop table test;"
diff --git a/tests/queries/0_stateless/03037_dynamic_merges_2_horizontal_compact_merge_tree.reference b/tests/queries/0_stateless/03037_dynamic_merges_2_horizontal_compact_merge_tree.reference
new file mode 100644
index 00000000000..afd392002e5
--- /dev/null
+++ b/tests/queries/0_stateless/03037_dynamic_merges_2_horizontal_compact_merge_tree.reference
@@ -0,0 +1,3 @@
+1000000	Array(UInt16)
+1000000	String
+1000000	UInt64
diff --git a/tests/queries/0_stateless/03037_dynamic_merges_2_horizontal_compact_merge_tree.sql b/tests/queries/0_stateless/03037_dynamic_merges_2_horizontal_compact_merge_tree.sql
new file mode 100644
index 00000000000..6d7a0dd8c18
--- /dev/null
+++ b/tests/queries/0_stateless/03037_dynamic_merges_2_horizontal_compact_merge_tree.sql
@@ -0,0 +1,14 @@
+-- Tags: long
+
+set allow_experimental_dynamic_type = 1;
+
+drop table if exists test;
+create table test (id UInt64, d Dynamic) engine=MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000;
+system stop merges test;
+insert into test select number, number from numbers(1000000);
+insert into test select number, 'str_' || toString(number) from numbers(1000000, 1000000);
+insert into test select number, range(number % 10 + 1) from numbers(2000000, 1000000);
+system start merges test;
+optimize table test final;
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+drop table test;
diff --git a/tests/queries/0_stateless/03037_dynamic_merges_2_horizontal_wide_merge_tree.reference b/tests/queries/0_stateless/03037_dynamic_merges_2_horizontal_wide_merge_tree.reference
new file mode 100644
index 00000000000..afd392002e5
--- /dev/null
+++ b/tests/queries/0_stateless/03037_dynamic_merges_2_horizontal_wide_merge_tree.reference
@@ -0,0 +1,3 @@
+1000000	Array(UInt16)
+1000000	String
+1000000	UInt64
diff --git a/tests/queries/0_stateless/03037_dynamic_merges_2_horizontal_wide_merge_tree.sql b/tests/queries/0_stateless/03037_dynamic_merges_2_horizontal_wide_merge_tree.sql
new file mode 100644
index 00000000000..011d54d2360
--- /dev/null
+++ b/tests/queries/0_stateless/03037_dynamic_merges_2_horizontal_wide_merge_tree.sql
@@ -0,0 +1,14 @@
+-- Tags: long
+
+set allow_experimental_dynamic_type = 1;
+
+drop table if exists test;
+create table test (id UInt64, d Dynamic) engine=MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1;
+system stop merges test;
+insert into test select number, number from numbers(1000000);
+insert into test select number, 'str_' || toString(number) from numbers(1000000, 1000000);
+insert into test select number, range(number % 10 + 1) from numbers(2000000, 1000000);
+system start merges test;
+optimize table test final;
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+drop table test;
diff --git a/tests/queries/0_stateless/03037_dynamic_merges_2_vertical_compact_merge_tree.reference b/tests/queries/0_stateless/03037_dynamic_merges_2_vertical_compact_merge_tree.reference
new file mode 100644
index 00000000000..afd392002e5
--- /dev/null
+++ b/tests/queries/0_stateless/03037_dynamic_merges_2_vertical_compact_merge_tree.reference
@@ -0,0 +1,3 @@
+1000000	Array(UInt16)
+1000000	String
+1000000	UInt64
diff --git a/tests/queries/0_stateless/03037_dynamic_merges_2_vertical_compact_merge_tree.sql b/tests/queries/0_stateless/03037_dynamic_merges_2_vertical_compact_merge_tree.sql
new file mode 100644
index 00000000000..1a74f9e5417
--- /dev/null
+++ b/tests/queries/0_stateless/03037_dynamic_merges_2_vertical_compact_merge_tree.sql
@@ -0,0 +1,14 @@
+-- Tags: long
+
+set allow_experimental_dynamic_type = 1;
+
+drop table if exists test;
+create table test (id UInt64, d Dynamic) engine=MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1;
+system stop merges test;
+insert into test select number, number from numbers(1000000);
+insert into test select number, 'str_' || toString(number) from numbers(1000000, 1000000);
+insert into test select number, range(number % 10 + 1) from numbers(2000000, 1000000);
+system start merges test;
+optimize table test final;
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+drop table test;
diff --git a/tests/queries/0_stateless/03037_dynamic_merges_2_vertical_wide_merge_tree.reference b/tests/queries/0_stateless/03037_dynamic_merges_2_vertical_wide_merge_tree.reference
new file mode 100644
index 00000000000..afd392002e5
--- /dev/null
+++ b/tests/queries/0_stateless/03037_dynamic_merges_2_vertical_wide_merge_tree.reference
@@ -0,0 +1,3 @@
+1000000	Array(UInt16)
+1000000	String
+1000000	UInt64
diff --git a/tests/queries/0_stateless/03037_dynamic_merges_2_vertical_wide_merge_tree.sql b/tests/queries/0_stateless/03037_dynamic_merges_2_vertical_wide_merge_tree.sql
new file mode 100644
index 00000000000..cbc834e9660
--- /dev/null
+++ b/tests/queries/0_stateless/03037_dynamic_merges_2_vertical_wide_merge_tree.sql
@@ -0,0 +1,14 @@
+-- Tags: long
+
+set allow_experimental_dynamic_type = 1;
+
+drop table if exists test;
+create table test (id UInt64, d Dynamic) engine=MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1;
+system stop merges test;
+insert into test select number, number from numbers(1000000);
+insert into test select number, 'str_' || toString(number) from numbers(1000000, 1000000);
+insert into test select number, range(number % 10 + 1) from numbers(2000000, 1000000);
+system start merges test;
+optimize table test final;
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+drop table test;
diff --git a/tests/queries/0_stateless/03038_nested_dynamic_merges_compact_horizontal.sh b/tests/queries/0_stateless/03038_nested_dynamic_merges_compact_horizontal.sh
deleted file mode 100755
index d4b6d1f4b63..00000000000
--- a/tests/queries/0_stateless/03038_nested_dynamic_merges_compact_horizontal.sh
+++ /dev/null
@@ -1,32 +0,0 @@
-#!/usr/bin/env bash
-# Tags: long
-
-CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# reset --log_comment
-CLICKHOUSE_LOG_COMMENT=
-# shellcheck source=../shell_config.sh
-. "$CUR_DIR"/../shell_config.sh
-
-CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --use_variant_as_common_type=1 --allow_experimental_dynamic_type=1 --enable_named_columns_in_function_tuple=0"
-
-$CH_CLIENT -q "drop table if exists test;"
-$CH_CLIENT -q "create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000;"
-
-$CH_CLIENT -q "system stop merges test"
-$CH_CLIENT -q "insert into test select number, number from numbers(100000)"
-$CH_CLIENT -q "insert into test select number, tuple(if(number % 2 == 0, number, 'str_' || toString(number)))::Tuple(a Dynamic(max_types=3)) from numbers(100000)"
-$CH_CLIENT -q "insert into test select number, tuple(if(number % 3 == 0, toDate(number), range(number % 10)))::Tuple(a Dynamic(max_types=3)) from numbers(50000)"
-
-$CH_CLIENT -q "select count(), dynamicType(d) || ':' || dynamicType(d.\`Tuple(a Dynamic(max_types=3))\`.a) as type from test group by type order by count(), type"
-$CH_CLIENT -nm -q "system start merges test; optimize table test final;"
-$CH_CLIENT -q "select count(), dynamicType(d) || ':' || dynamicType(d.\`Tuple(a Dynamic(max_types=3))\`.a) as type from test group by type order by count(), type"
-
-$CH_CLIENT -q "insert into test select number, tuple(if(number % 3 == 0, toDateTime(number), NULL))::Tuple(a Dynamic(max_types=3)) from numbers(50000)"
-$CH_CLIENT -q "insert into test select number, tuple(if(number % 2 == 0, tuple(number), NULL))::Tuple(a Dynamic(max_types=3)) from numbers(200000)"
-
-$CH_CLIENT -q "select count(), dynamicType(d) || ':' || dynamicType(d.\`Tuple(a Dynamic(max_types=3))\`.a) as type from test group by type order by count(), type"
-$CH_CLIENT -nm -q "system start merges test; optimize table test final;"
-$CH_CLIENT -q "select count(), dynamicType(d) || ':' || dynamicType(d.\`Tuple(a Dynamic(max_types=3))\`.a) as type from test group by type order by count(), type"
-
-$CH_CLIENT -q "drop table test;"
-
diff --git a/tests/queries/0_stateless/03038_nested_dynamic_merges_compact_horizontal.sql b/tests/queries/0_stateless/03038_nested_dynamic_merges_compact_horizontal.sql
new file mode 100644
index 00000000000..ff1dc5e7ded
--- /dev/null
+++ b/tests/queries/0_stateless/03038_nested_dynamic_merges_compact_horizontal.sql
@@ -0,0 +1,29 @@
+-- Tags: long
+
+set allow_experimental_variant_type = 1;
+set use_variant_as_common_type = 1;
+set allow_experimental_dynamic_type = 1;
+set enable_named_columns_in_function_tuple = 0;
+
+drop table if exists test;;
+create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000;;
+
+system stop merges test;
+insert into test select number, number from numbers(100000);
+insert into test select number, tuple(if(number % 2 == 0, number, 'str_' || toString(number)))::Tuple(a Dynamic(max_types=3)) from numbers(100000);
+insert into test select number, tuple(if(number % 3 == 0, toDate(number), range(number % 10)))::Tuple(a Dynamic(max_types=3)) from numbers(50000);
+
+select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=3))`.a) as type from test group by type order by count(), type;
+system start merges test;
+optimize table test final;
+select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=3))`.a) as type from test group by type order by count(), type;
+
+insert into test select number, tuple(if(number % 3 == 0, toDateTime(number), NULL))::Tuple(a Dynamic(max_types=3)) from numbers(50000);
+insert into test select number, tuple(if(number % 2 == 0, tuple(number), NULL))::Tuple(a Dynamic(max_types=3)) from numbers(200000);
+
+select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=3))`.a) as type from test group by type order by count(), type;
+system start merges test;
+optimize table test final;
+select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=3))`.a) as type from test group by type order by count(), type;
+
+drop table test;
diff --git a/tests/queries/0_stateless/03038_nested_dynamic_merges_compact_vertical.sh b/tests/queries/0_stateless/03038_nested_dynamic_merges_compact_vertical.sh
deleted file mode 100755
index 39671a297cf..00000000000
--- a/tests/queries/0_stateless/03038_nested_dynamic_merges_compact_vertical.sh
+++ /dev/null
@@ -1,32 +0,0 @@
-#!/usr/bin/env bash
-# Tags: long
-
-CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# reset --log_comment
-CLICKHOUSE_LOG_COMMENT=
-# shellcheck source=../shell_config.sh
-. "$CUR_DIR"/../shell_config.sh
-
-CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --use_variant_as_common_type=1 --allow_experimental_dynamic_type=1 --enable_named_columns_in_function_tuple=0"
-
-$CH_CLIENT -q "drop table if exists test;"
-$CH_CLIENT -q "create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1;"
-
-$CH_CLIENT -q "system stop merges test"
-$CH_CLIENT -q "insert into test select number, number from numbers(100000)"
-$CH_CLIENT -q "insert into test select number, tuple(if(number % 2 == 0, number, 'str_' || toString(number)))::Tuple(a Dynamic(max_types=3)) from numbers(100000)"
-$CH_CLIENT -q "insert into test select number, tuple(if(number % 3 == 0, toDate(number), range(number % 10)))::Tuple(a Dynamic(max_types=3)) from numbers(50000)"
-
-$CH_CLIENT -q "select count(), dynamicType(d) || ':' || dynamicType(d.\`Tuple(a Dynamic(max_types=3))\`.a) as type from test group by type order by count(), type"
-$CH_CLIENT -nm -q "system start merges test; optimize table test final;"
-$CH_CLIENT -q "select count(), dynamicType(d) || ':' || dynamicType(d.\`Tuple(a Dynamic(max_types=3))\`.a) as type from test group by type order by count(), type"
-
-$CH_CLIENT -q "insert into test select number, tuple(if(number % 3 == 0, toDateTime(number), NULL))::Tuple(a Dynamic(max_types=3)) from numbers(50000)"
-$CH_CLIENT -q "insert into test select number, tuple(if(number % 2 == 0, tuple(number), NULL))::Tuple(a Dynamic(max_types=3)) from numbers(200000)"
-
-$CH_CLIENT -q "select count(), dynamicType(d) || ':' || dynamicType(d.\`Tuple(a Dynamic(max_types=3))\`.a) as type from test group by type order by count(), type"
-$CH_CLIENT -nm -q "system start merges test; optimize table test final;"
-$CH_CLIENT -q "select count(), dynamicType(d) || ':' || dynamicType(d.\`Tuple(a Dynamic(max_types=3))\`.a) as type from test group by type order by count(), type"
-
-$CH_CLIENT -q "drop table test;"
-
diff --git a/tests/queries/0_stateless/03038_nested_dynamic_merges_compact_vertical.sql b/tests/queries/0_stateless/03038_nested_dynamic_merges_compact_vertical.sql
new file mode 100644
index 00000000000..f9b0101cb87
--- /dev/null
+++ b/tests/queries/0_stateless/03038_nested_dynamic_merges_compact_vertical.sql
@@ -0,0 +1,29 @@
+-- Tags: long
+
+set allow_experimental_variant_type = 1;
+set use_variant_as_common_type = 1;
+set allow_experimental_dynamic_type = 1;
+set enable_named_columns_in_function_tuple = 0;
+
+drop table if exists test;;
+create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1;
+
+system stop merges test;
+insert into test select number, number from numbers(100000);
+insert into test select number, tuple(if(number % 2 == 0, number, 'str_' || toString(number)))::Tuple(a Dynamic(max_types=3)) from numbers(100000);
+insert into test select number, tuple(if(number % 3 == 0, toDate(number), range(number % 10)))::Tuple(a Dynamic(max_types=3)) from numbers(50000);
+
+select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=3))`.a) as type from test group by type order by count(), type;
+system start merges test;
+optimize table test final;
+select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=3))`.a) as type from test group by type order by count(), type;
+
+insert into test select number, tuple(if(number % 3 == 0, toDateTime(number), NULL))::Tuple(a Dynamic(max_types=3)) from numbers(50000);
+insert into test select number, tuple(if(number % 2 == 0, tuple(number), NULL))::Tuple(a Dynamic(max_types=3)) from numbers(200000);
+
+select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=3))`.a) as type from test group by type order by count(), type;
+system start merges test;
+optimize table test final;
+select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=3))`.a) as type from test group by type order by count(), type;
+
+drop table test;
diff --git a/tests/queries/0_stateless/03038_nested_dynamic_merges_wide_horizontal.sh b/tests/queries/0_stateless/03038_nested_dynamic_merges_wide_horizontal.sh
deleted file mode 100755
index d58545c0b13..00000000000
--- a/tests/queries/0_stateless/03038_nested_dynamic_merges_wide_horizontal.sh
+++ /dev/null
@@ -1,32 +0,0 @@
-#!/usr/bin/env bash
-# Tags: long
-
-CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# reset --log_comment
-CLICKHOUSE_LOG_COMMENT=
-# shellcheck source=../shell_config.sh
-. "$CUR_DIR"/../shell_config.sh
-
-CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --use_variant_as_common_type=1 --allow_experimental_dynamic_type=1 --enable_named_columns_in_function_tuple=0"
-
-$CH_CLIENT -q "drop table if exists test;"
-$CH_CLIENT -q "create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1;"
-
-$CH_CLIENT -q "system stop merges test"
-$CH_CLIENT -q "insert into test select number, number from numbers(100000)"
-$CH_CLIENT -q "insert into test select number, tuple(if(number % 2 == 0, number, 'str_' || toString(number)))::Tuple(a Dynamic(max_types=3)) from numbers(100000)"
-$CH_CLIENT -q "insert into test select number, tuple(if(number % 3 == 0, toDate(number), range(number % 10)))::Tuple(a Dynamic(max_types=3)) from numbers(50000)"
-
-$CH_CLIENT -q "select count(), dynamicType(d) || ':' || dynamicType(d.\`Tuple(a Dynamic(max_types=3))\`.a) as type from test group by type order by count(), type"
-$CH_CLIENT -nm -q "system start merges test; optimize table test final;"
-$CH_CLIENT -q "select count(), dynamicType(d) || ':' || dynamicType(d.\`Tuple(a Dynamic(max_types=3))\`.a) as type from test group by type order by count(), type"
-
-$CH_CLIENT -q "insert into test select number, tuple(if(number % 3 == 0, toDateTime(number), NULL))::Tuple(a Dynamic(max_types=3)) from numbers(50000)"
-$CH_CLIENT -q "insert into test select number, tuple(if(number % 2 == 0, tuple(number), NULL))::Tuple(a Dynamic(max_types=3)) from numbers(200000)"
-
-$CH_CLIENT -q "select count(), dynamicType(d) || ':' || dynamicType(d.\`Tuple(a Dynamic(max_types=3))\`.a) as type from test group by type order by count(), type"
-$CH_CLIENT -nm -q "system start merges test; optimize table test final;"
-$CH_CLIENT -q "select count(), dynamicType(d) || ':' || dynamicType(d.\`Tuple(a Dynamic(max_types=3))\`.a) as type from test group by type order by count(), type"
-
-$CH_CLIENT -q "drop table test;"
-
diff --git a/tests/queries/0_stateless/03038_nested_dynamic_merges_wide_horizontal.sql b/tests/queries/0_stateless/03038_nested_dynamic_merges_wide_horizontal.sql
new file mode 100644
index 00000000000..5f373d41c7d
--- /dev/null
+++ b/tests/queries/0_stateless/03038_nested_dynamic_merges_wide_horizontal.sql
@@ -0,0 +1,29 @@
+-- Tags: long
+
+set allow_experimental_variant_type = 1;
+set use_variant_as_common_type = 1;
+set allow_experimental_dynamic_type = 1;
+set enable_named_columns_in_function_tuple = 0;
+
+drop table if exists test;;
+create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1;
+
+system stop merges test;
+insert into test select number, number from numbers(100000);
+insert into test select number, tuple(if(number % 2 == 0, number, 'str_' || toString(number)))::Tuple(a Dynamic(max_types=3)) from numbers(100000);
+insert into test select number, tuple(if(number % 3 == 0, toDate(number), range(number % 10)))::Tuple(a Dynamic(max_types=3)) from numbers(50000);
+
+select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=3))`.a) as type from test group by type order by count(), type;
+system start merges test;
+optimize table test final;
+select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=3))`.a) as type from test group by type order by count(), type;
+
+insert into test select number, tuple(if(number % 3 == 0, toDateTime(number), NULL))::Tuple(a Dynamic(max_types=3)) from numbers(50000);
+insert into test select number, tuple(if(number % 2 == 0, tuple(number), NULL))::Tuple(a Dynamic(max_types=3)) from numbers(200000);
+
+select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=3))`.a) as type from test group by type order by count(), type;
+system start merges test;
+optimize table test final;
+select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=3))`.a) as type from test group by type order by count(), type;
+
+drop table test;
diff --git a/tests/queries/0_stateless/03038_nested_dynamic_merges_wide_vertical.sh b/tests/queries/0_stateless/03038_nested_dynamic_merges_wide_vertical.sh
deleted file mode 100755
index 39671a297cf..00000000000
--- a/tests/queries/0_stateless/03038_nested_dynamic_merges_wide_vertical.sh
+++ /dev/null
@@ -1,32 +0,0 @@
-#!/usr/bin/env bash
-# Tags: long
-
-CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# reset --log_comment
-CLICKHOUSE_LOG_COMMENT=
-# shellcheck source=../shell_config.sh
-. "$CUR_DIR"/../shell_config.sh
-
-CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --use_variant_as_common_type=1 --allow_experimental_dynamic_type=1 --enable_named_columns_in_function_tuple=0"
-
-$CH_CLIENT -q "drop table if exists test;"
-$CH_CLIENT -q "create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1;"
-
-$CH_CLIENT -q "system stop merges test"
-$CH_CLIENT -q "insert into test select number, number from numbers(100000)"
-$CH_CLIENT -q "insert into test select number, tuple(if(number % 2 == 0, number, 'str_' || toString(number)))::Tuple(a Dynamic(max_types=3)) from numbers(100000)"
-$CH_CLIENT -q "insert into test select number, tuple(if(number % 3 == 0, toDate(number), range(number % 10)))::Tuple(a Dynamic(max_types=3)) from numbers(50000)"
-
-$CH_CLIENT -q "select count(), dynamicType(d) || ':' || dynamicType(d.\`Tuple(a Dynamic(max_types=3))\`.a) as type from test group by type order by count(), type"
-$CH_CLIENT -nm -q "system start merges test; optimize table test final;"
-$CH_CLIENT -q "select count(), dynamicType(d) || ':' || dynamicType(d.\`Tuple(a Dynamic(max_types=3))\`.a) as type from test group by type order by count(), type"
-
-$CH_CLIENT -q "insert into test select number, tuple(if(number % 3 == 0, toDateTime(number), NULL))::Tuple(a Dynamic(max_types=3)) from numbers(50000)"
-$CH_CLIENT -q "insert into test select number, tuple(if(number % 2 == 0, tuple(number), NULL))::Tuple(a Dynamic(max_types=3)) from numbers(200000)"
-
-$CH_CLIENT -q "select count(), dynamicType(d) || ':' || dynamicType(d.\`Tuple(a Dynamic(max_types=3))\`.a) as type from test group by type order by count(), type"
-$CH_CLIENT -nm -q "system start merges test; optimize table test final;"
-$CH_CLIENT -q "select count(), dynamicType(d) || ':' || dynamicType(d.\`Tuple(a Dynamic(max_types=3))\`.a) as type from test group by type order by count(), type"
-
-$CH_CLIENT -q "drop table test;"
-
diff --git a/tests/queries/0_stateless/03038_nested_dynamic_merges_wide_vertical.sql b/tests/queries/0_stateless/03038_nested_dynamic_merges_wide_vertical.sql
new file mode 100644
index 00000000000..36bbc76b8cb
--- /dev/null
+++ b/tests/queries/0_stateless/03038_nested_dynamic_merges_wide_vertical.sql
@@ -0,0 +1,29 @@
+-- Tags: long
+
+set allow_experimental_variant_type = 1;
+set use_variant_as_common_type = 1;
+set allow_experimental_dynamic_type = 1;
+set enable_named_columns_in_function_tuple = 0;
+
+drop table if exists test;;
+create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1;
+
+system stop merges test;
+insert into test select number, number from numbers(100000);
+insert into test select number, tuple(if(number % 2 == 0, number, 'str_' || toString(number)))::Tuple(a Dynamic(max_types=3)) from numbers(100000);
+insert into test select number, tuple(if(number % 3 == 0, toDate(number), range(number % 10)))::Tuple(a Dynamic(max_types=3)) from numbers(50000);
+
+select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=3))`.a) as type from test group by type order by count(), type;
+system start merges test;
+optimize table test final;
+select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=3))`.a) as type from test group by type order by count(), type;
+
+insert into test select number, tuple(if(number % 3 == 0, toDateTime(number), NULL))::Tuple(a Dynamic(max_types=3)) from numbers(50000);
+insert into test select number, tuple(if(number % 2 == 0, tuple(number), NULL))::Tuple(a Dynamic(max_types=3)) from numbers(200000);
+
+select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=3))`.a) as type from test group by type order by count(), type;
+system start merges test;
+optimize table test final;
+select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=3))`.a) as type from test group by type order by count(), type;
+
+drop table test;
diff --git a/tests/queries/0_stateless/03040_dynamic_type_alters_1.reference b/tests/queries/0_stateless/03040_dynamic_type_alters_1.reference
deleted file mode 100644
index a9c785d1e48..00000000000
--- a/tests/queries/0_stateless/03040_dynamic_type_alters_1.reference
+++ /dev/null
@@ -1,526 +0,0 @@
-Memory
-initial insert
-alter add column 1
-3	None
-0	0	\N	\N	\N	0
-1	1	\N	\N	\N	0
-2	2	\N	\N	\N	0
-insert after alter add column 1
-4	String
-4	UInt64
-7	None
-0	0	\N	\N	\N	\N	0
-1	1	\N	\N	\N	\N	0
-2	2	\N	\N	\N	\N	0
-3	3	3	\N	3	\N	0
-4	4	4	\N	4	\N	0
-5	5	5	\N	5	\N	0
-6	6	str_6	str_6	\N	\N	0
-7	7	str_7	str_7	\N	\N	0
-8	8	str_8	str_8	\N	\N	0
-9	9	\N	\N	\N	\N	0
-10	10	\N	\N	\N	\N	0
-11	11	\N	\N	\N	\N	0
-12	12	12	\N	12	\N	0
-13	13	str_13	str_13	\N	\N	0
-14	14	\N	\N	\N	\N	0
-alter modify column 1
-7	None
-8	String
-0	0	\N	\N	\N	\N	0
-1	1	\N	\N	\N	\N	0
-2	2	\N	\N	\N	\N	0
-3	3	3	3	\N	\N	0
-4	4	4	4	\N	\N	0
-5	5	5	5	\N	\N	0
-6	6	str_6	str_6	\N	\N	0
-7	7	str_7	str_7	\N	\N	0
-8	8	str_8	str_8	\N	\N	0
-9	9	\N	\N	\N	\N	0
-10	10	\N	\N	\N	\N	0
-11	11	\N	\N	\N	\N	0
-12	12	12	12	\N	\N	0
-13	13	str_13	str_13	\N	\N	0
-14	14	\N	\N	\N	\N	0
-insert after alter modify column 1
-8	None
-11	String
-0	0	\N	\N	\N	\N	0
-1	1	\N	\N	\N	\N	0
-2	2	\N	\N	\N	\N	0
-3	3	3	3	\N	\N	0
-4	4	4	4	\N	\N	0
-5	5	5	5	\N	\N	0
-6	6	str_6	str_6	\N	\N	0
-7	7	str_7	str_7	\N	\N	0
-8	8	str_8	str_8	\N	\N	0
-9	9	\N	\N	\N	\N	0
-10	10	\N	\N	\N	\N	0
-11	11	\N	\N	\N	\N	0
-12	12	12	12	\N	\N	0
-13	13	str_13	str_13	\N	\N	0
-14	14	\N	\N	\N	\N	0
-15	15	\N	\N	\N	\N	0
-16	16	16	16	\N	\N	0
-17	17	str_17	str_17	\N	\N	0
-18	18	1970-01-19	1970-01-19	\N	\N	0
-alter modify column 2
-4	UInt64
-7	String
-8	None
-0	0	\N	\N	\N	\N	0
-1	1	\N	\N	\N	\N	0
-2	2	\N	\N	\N	\N	0
-3	3	3	\N	3	\N	0
-4	4	4	\N	4	\N	0
-5	5	5	\N	5	\N	0
-6	6	str_6	str_6	\N	\N	0
-7	7	str_7	str_7	\N	\N	0
-8	8	str_8	str_8	\N	\N	0
-9	9	\N	\N	\N	\N	0
-10	10	\N	\N	\N	\N	0
-11	11	\N	\N	\N	\N	0
-12	12	12	\N	12	\N	0
-13	13	str_13	str_13	\N	\N	0
-14	14	\N	\N	\N	\N	0
-15	15	\N	\N	\N	\N	0
-16	16	16	16	\N	\N	0
-17	17	str_17	str_17	\N	\N	0
-18	18	1970-01-19	1970-01-19	\N	\N	0
-insert after alter modify column 2
-1	Date
-5	UInt64
-8	String
-9	None
-0	0	\N	\N	\N	\N	0
-1	1	\N	\N	\N	\N	0
-2	2	\N	\N	\N	\N	0
-3	3	3	\N	3	\N	0
-4	4	4	\N	4	\N	0
-5	5	5	\N	5	\N	0
-6	6	str_6	str_6	\N	\N	0
-7	7	str_7	str_7	\N	\N	0
-8	8	str_8	str_8	\N	\N	0
-9	9	\N	\N	\N	\N	0
-10	10	\N	\N	\N	\N	0
-11	11	\N	\N	\N	\N	0
-12	12	12	\N	12	\N	0
-13	13	str_13	str_13	\N	\N	0
-14	14	\N	\N	\N	\N	0
-15	15	\N	\N	\N	\N	0
-16	16	16	16	\N	\N	0
-17	17	str_17	str_17	\N	\N	0
-18	18	1970-01-19	1970-01-19	\N	\N	0
-19	19	\N	\N	\N	\N	0
-20	20	20	\N	20	\N	0
-21	21	str_21	str_21	\N	\N	0
-22	22	1970-01-23	\N	\N	1970-01-23	0
-alter modify column 3
-1	Date
-5	UInt64
-8	String
-9	None
-0	0	0	\N	0	\N	\N	\N	0
-1	1	1	\N	0	\N	\N	\N	0
-2	2	2	\N	0	\N	\N	\N	0
-3	3	3	\N	0	\N	3	\N	0
-4	4	4	\N	0	\N	4	\N	0
-5	5	5	\N	0	\N	5	\N	0
-6	6	6	\N	0	str_6	\N	\N	0
-7	7	7	\N	0	str_7	\N	\N	0
-8	8	8	\N	0	str_8	\N	\N	0
-9	9	9	\N	0	\N	\N	\N	0
-10	10	10	\N	0	\N	\N	\N	0
-11	11	11	\N	0	\N	\N	\N	0
-12	12	12	\N	0	\N	12	\N	0
-13	13	13	\N	0	str_13	\N	\N	0
-14	14	14	\N	0	\N	\N	\N	0
-15	15	15	\N	0	\N	\N	\N	0
-16	16	16	\N	0	16	\N	\N	0
-17	17	17	\N	0	str_17	\N	\N	0
-18	18	18	\N	0	1970-01-19	\N	\N	0
-19	19	19	\N	0	\N	\N	\N	0
-20	20	20	\N	0	\N	20	\N	0
-21	21	21	\N	0	str_21	\N	\N	0
-22	22	22	\N	0	\N	\N	1970-01-23	0
-insert after alter modify column 3
-1	Date
-5	UInt64
-8	String
-12	None
-0	0	0	\N	0	\N	\N	\N	0
-1	1	1	\N	0	\N	\N	\N	0
-2	2	2	\N	0	\N	\N	\N	0
-3	3	3	\N	0	\N	3	\N	0
-4	4	4	\N	0	\N	4	\N	0
-5	5	5	\N	0	\N	5	\N	0
-6	6	6	\N	0	str_6	\N	\N	0
-7	7	7	\N	0	str_7	\N	\N	0
-8	8	8	\N	0	str_8	\N	\N	0
-9	9	9	\N	0	\N	\N	\N	0
-10	10	10	\N	0	\N	\N	\N	0
-11	11	11	\N	0	\N	\N	\N	0
-12	12	12	\N	0	\N	12	\N	0
-13	13	13	\N	0	str_13	\N	\N	0
-14	14	14	\N	0	\N	\N	\N	0
-15	15	15	\N	0	\N	\N	\N	0
-16	16	16	\N	0	16	\N	\N	0
-17	17	17	\N	0	str_17	\N	\N	0
-18	18	18	\N	0	1970-01-19	\N	\N	0
-19	19	19	\N	0	\N	\N	\N	0
-20	20	20	\N	0	\N	20	\N	0
-21	21	21	\N	0	str_21	\N	\N	0
-22	22	22	\N	0	\N	\N	1970-01-23	0
-23	\N	\N	\N	0	\N	\N	\N	0
-24	24	24	\N	0	\N	\N	\N	0
-25	str_25	\N	str_25	0	\N	\N	\N	0
-MergeTree compact
-initial insert
-alter add column 1
-3	None
-0	0	\N	\N	\N	0
-1	1	\N	\N	\N	0
-2	2	\N	\N	\N	0
-insert after alter add column 1
-4	String
-4	UInt64
-7	None
-0	0	\N	\N	\N	\N	0
-1	1	\N	\N	\N	\N	0
-2	2	\N	\N	\N	\N	0
-3	3	3	\N	3	\N	0
-4	4	4	\N	4	\N	0
-5	5	5	\N	5	\N	0
-6	6	str_6	str_6	\N	\N	0
-7	7	str_7	str_7	\N	\N	0
-8	8	str_8	str_8	\N	\N	0
-9	9	\N	\N	\N	\N	0
-10	10	\N	\N	\N	\N	0
-11	11	\N	\N	\N	\N	0
-12	12	12	\N	12	\N	0
-13	13	str_13	str_13	\N	\N	0
-14	14	\N	\N	\N	\N	0
-alter modify column 1
-7	None
-8	String
-0	0	\N	\N	\N	\N	0
-1	1	\N	\N	\N	\N	0
-2	2	\N	\N	\N	\N	0
-3	3	3	3	\N	\N	0
-4	4	4	4	\N	\N	0
-5	5	5	5	\N	\N	0
-6	6	str_6	str_6	\N	\N	0
-7	7	str_7	str_7	\N	\N	0
-8	8	str_8	str_8	\N	\N	0
-9	9	\N	\N	\N	\N	0
-10	10	\N	\N	\N	\N	0
-11	11	\N	\N	\N	\N	0
-12	12	12	12	\N	\N	0
-13	13	str_13	str_13	\N	\N	0
-14	14	\N	\N	\N	\N	0
-insert after alter modify column 1
-8	None
-11	String
-0	0	\N	\N	\N	\N	0
-1	1	\N	\N	\N	\N	0
-2	2	\N	\N	\N	\N	0
-3	3	3	3	\N	\N	0
-4	4	4	4	\N	\N	0
-5	5	5	5	\N	\N	0
-6	6	str_6	str_6	\N	\N	0
-7	7	str_7	str_7	\N	\N	0
-8	8	str_8	str_8	\N	\N	0
-9	9	\N	\N	\N	\N	0
-10	10	\N	\N	\N	\N	0
-11	11	\N	\N	\N	\N	0
-12	12	12	12	\N	\N	0
-13	13	str_13	str_13	\N	\N	0
-14	14	\N	\N	\N	\N	0
-15	15	\N	\N	\N	\N	0
-16	16	16	16	\N	\N	0
-17	17	str_17	str_17	\N	\N	0
-18	18	1970-01-19	1970-01-19	\N	\N	0
-alter modify column 2
-8	None
-11	String
-0	0	\N	\N	\N	\N	0
-1	1	\N	\N	\N	\N	0
-2	2	\N	\N	\N	\N	0
-3	3	3	3	\N	\N	0
-4	4	4	4	\N	\N	0
-5	5	5	5	\N	\N	0
-6	6	str_6	str_6	\N	\N	0
-7	7	str_7	str_7	\N	\N	0
-8	8	str_8	str_8	\N	\N	0
-9	9	\N	\N	\N	\N	0
-10	10	\N	\N	\N	\N	0
-11	11	\N	\N	\N	\N	0
-12	12	12	12	\N	\N	0
-13	13	str_13	str_13	\N	\N	0
-14	14	\N	\N	\N	\N	0
-15	15	\N	\N	\N	\N	0
-16	16	16	16	\N	\N	0
-17	17	str_17	str_17	\N	\N	0
-18	18	1970-01-19	1970-01-19	\N	\N	0
-insert after alter modify column 2
-1	Date
-1	UInt64
-9	None
-12	String
-0	0	\N	\N	\N	\N	0
-1	1	\N	\N	\N	\N	0
-2	2	\N	\N	\N	\N	0
-3	3	3	3	\N	\N	0
-4	4	4	4	\N	\N	0
-5	5	5	5	\N	\N	0
-6	6	str_6	str_6	\N	\N	0
-7	7	str_7	str_7	\N	\N	0
-8	8	str_8	str_8	\N	\N	0
-9	9	\N	\N	\N	\N	0
-10	10	\N	\N	\N	\N	0
-11	11	\N	\N	\N	\N	0
-12	12	12	12	\N	\N	0
-13	13	str_13	str_13	\N	\N	0
-14	14	\N	\N	\N	\N	0
-15	15	\N	\N	\N	\N	0
-16	16	16	16	\N	\N	0
-17	17	str_17	str_17	\N	\N	0
-18	18	1970-01-19	1970-01-19	\N	\N	0
-19	19	\N	\N	\N	\N	0
-20	20	20	\N	20	\N	0
-21	21	str_21	str_21	\N	\N	0
-22	22	1970-01-23	\N	\N	1970-01-23	0
-alter modify column 3
-1	Date
-1	UInt64
-9	None
-12	String
-0	0	0	\N	0	\N	\N	\N	0
-1	1	1	\N	0	\N	\N	\N	0
-2	2	2	\N	0	\N	\N	\N	0
-3	3	3	\N	0	3	\N	\N	0
-4	4	4	\N	0	4	\N	\N	0
-5	5	5	\N	0	5	\N	\N	0
-6	6	6	\N	0	str_6	\N	\N	0
-7	7	7	\N	0	str_7	\N	\N	0
-8	8	8	\N	0	str_8	\N	\N	0
-9	9	9	\N	0	\N	\N	\N	0
-10	10	10	\N	0	\N	\N	\N	0
-11	11	11	\N	0	\N	\N	\N	0
-12	12	12	\N	0	12	\N	\N	0
-13	13	13	\N	0	str_13	\N	\N	0
-14	14	14	\N	0	\N	\N	\N	0
-15	15	15	\N	0	\N	\N	\N	0
-16	16	16	\N	0	16	\N	\N	0
-17	17	17	\N	0	str_17	\N	\N	0
-18	18	18	\N	0	1970-01-19	\N	\N	0
-19	19	19	\N	0	\N	\N	\N	0
-20	20	20	\N	0	\N	20	\N	0
-21	21	21	\N	0	str_21	\N	\N	0
-22	22	22	\N	0	\N	\N	1970-01-23	0
-insert after alter modify column 3
-1	Date
-1	UInt64
-12	None
-12	String
-0	0	0	\N	0	\N	\N	\N	0
-1	1	1	\N	0	\N	\N	\N	0
-2	2	2	\N	0	\N	\N	\N	0
-3	3	3	\N	0	3	\N	\N	0
-4	4	4	\N	0	4	\N	\N	0
-5	5	5	\N	0	5	\N	\N	0
-6	6	6	\N	0	str_6	\N	\N	0
-7	7	7	\N	0	str_7	\N	\N	0
-8	8	8	\N	0	str_8	\N	\N	0
-9	9	9	\N	0	\N	\N	\N	0
-10	10	10	\N	0	\N	\N	\N	0
-11	11	11	\N	0	\N	\N	\N	0
-12	12	12	\N	0	12	\N	\N	0
-13	13	13	\N	0	str_13	\N	\N	0
-14	14	14	\N	0	\N	\N	\N	0
-15	15	15	\N	0	\N	\N	\N	0
-16	16	16	\N	0	16	\N	\N	0
-17	17	17	\N	0	str_17	\N	\N	0
-18	18	18	\N	0	1970-01-19	\N	\N	0
-19	19	19	\N	0	\N	\N	\N	0
-20	20	20	\N	0	\N	20	\N	0
-21	21	21	\N	0	str_21	\N	\N	0
-22	22	22	\N	0	\N	\N	1970-01-23	0
-23	\N	\N	\N	0	\N	\N	\N	0
-24	24	24	\N	0	\N	\N	\N	0
-25	str_25	\N	str_25	0	\N	\N	\N	0
-MergeTree wide
-initial insert
-alter add column 1
-3	None
-0	0	\N	\N	\N	0
-1	1	\N	\N	\N	0
-2	2	\N	\N	\N	0
-insert after alter add column 1
-4	String
-4	UInt64
-7	None
-0	0	\N	\N	\N	\N	0
-1	1	\N	\N	\N	\N	0
-2	2	\N	\N	\N	\N	0
-3	3	3	\N	3	\N	0
-4	4	4	\N	4	\N	0
-5	5	5	\N	5	\N	0
-6	6	str_6	str_6	\N	\N	0
-7	7	str_7	str_7	\N	\N	0
-8	8	str_8	str_8	\N	\N	0
-9	9	\N	\N	\N	\N	0
-10	10	\N	\N	\N	\N	0
-11	11	\N	\N	\N	\N	0
-12	12	12	\N	12	\N	0
-13	13	str_13	str_13	\N	\N	0
-14	14	\N	\N	\N	\N	0
-alter modify column 1
-7	None
-8	String
-0	0	\N	\N	\N	\N	0
-1	1	\N	\N	\N	\N	0
-2	2	\N	\N	\N	\N	0
-3	3	3	3	\N	\N	0
-4	4	4	4	\N	\N	0
-5	5	5	5	\N	\N	0
-6	6	str_6	str_6	\N	\N	0
-7	7	str_7	str_7	\N	\N	0
-8	8	str_8	str_8	\N	\N	0
-9	9	\N	\N	\N	\N	0
-10	10	\N	\N	\N	\N	0
-11	11	\N	\N	\N	\N	0
-12	12	12	12	\N	\N	0
-13	13	str_13	str_13	\N	\N	0
-14	14	\N	\N	\N	\N	0
-insert after alter modify column 1
-8	None
-11	String
-0	0	\N	\N	\N	\N	0
-1	1	\N	\N	\N	\N	0
-2	2	\N	\N	\N	\N	0
-3	3	3	3	\N	\N	0
-4	4	4	4	\N	\N	0
-5	5	5	5	\N	\N	0
-6	6	str_6	str_6	\N	\N	0
-7	7	str_7	str_7	\N	\N	0
-8	8	str_8	str_8	\N	\N	0
-9	9	\N	\N	\N	\N	0
-10	10	\N	\N	\N	\N	0
-11	11	\N	\N	\N	\N	0
-12	12	12	12	\N	\N	0
-13	13	str_13	str_13	\N	\N	0
-14	14	\N	\N	\N	\N	0
-15	15	\N	\N	\N	\N	0
-16	16	16	16	\N	\N	0
-17	17	str_17	str_17	\N	\N	0
-18	18	1970-01-19	1970-01-19	\N	\N	0
-alter modify column 2
-8	None
-11	String
-0	0	\N	\N	\N	\N	0
-1	1	\N	\N	\N	\N	0
-2	2	\N	\N	\N	\N	0
-3	3	3	3	\N	\N	0
-4	4	4	4	\N	\N	0
-5	5	5	5	\N	\N	0
-6	6	str_6	str_6	\N	\N	0
-7	7	str_7	str_7	\N	\N	0
-8	8	str_8	str_8	\N	\N	0
-9	9	\N	\N	\N	\N	0
-10	10	\N	\N	\N	\N	0
-11	11	\N	\N	\N	\N	0
-12	12	12	12	\N	\N	0
-13	13	str_13	str_13	\N	\N	0
-14	14	\N	\N	\N	\N	0
-15	15	\N	\N	\N	\N	0
-16	16	16	16	\N	\N	0
-17	17	str_17	str_17	\N	\N	0
-18	18	1970-01-19	1970-01-19	\N	\N	0
-insert after alter modify column 2
-1	Date
-1	UInt64
-9	None
-12	String
-0	0	\N	\N	\N	\N	0
-1	1	\N	\N	\N	\N	0
-2	2	\N	\N	\N	\N	0
-3	3	3	3	\N	\N	0
-4	4	4	4	\N	\N	0
-5	5	5	5	\N	\N	0
-6	6	str_6	str_6	\N	\N	0
-7	7	str_7	str_7	\N	\N	0
-8	8	str_8	str_8	\N	\N	0
-9	9	\N	\N	\N	\N	0
-10	10	\N	\N	\N	\N	0
-11	11	\N	\N	\N	\N	0
-12	12	12	12	\N	\N	0
-13	13	str_13	str_13	\N	\N	0
-14	14	\N	\N	\N	\N	0
-15	15	\N	\N	\N	\N	0
-16	16	16	16	\N	\N	0
-17	17	str_17	str_17	\N	\N	0
-18	18	1970-01-19	1970-01-19	\N	\N	0
-19	19	\N	\N	\N	\N	0
-20	20	20	\N	20	\N	0
-21	21	str_21	str_21	\N	\N	0
-22	22	1970-01-23	\N	\N	1970-01-23	0
-alter modify column 3
-1	Date
-1	UInt64
-9	None
-12	String
-0	0	0	\N	0	\N	\N	\N	0
-1	1	1	\N	0	\N	\N	\N	0
-2	2	2	\N	0	\N	\N	\N	0
-3	3	3	\N	0	3	\N	\N	0
-4	4	4	\N	0	4	\N	\N	0
-5	5	5	\N	0	5	\N	\N	0
-6	6	6	\N	0	str_6	\N	\N	0
-7	7	7	\N	0	str_7	\N	\N	0
-8	8	8	\N	0	str_8	\N	\N	0
-9	9	9	\N	0	\N	\N	\N	0
-10	10	10	\N	0	\N	\N	\N	0
-11	11	11	\N	0	\N	\N	\N	0
-12	12	12	\N	0	12	\N	\N	0
-13	13	13	\N	0	str_13	\N	\N	0
-14	14	14	\N	0	\N	\N	\N	0
-15	15	15	\N	0	\N	\N	\N	0
-16	16	16	\N	0	16	\N	\N	0
-17	17	17	\N	0	str_17	\N	\N	0
-18	18	18	\N	0	1970-01-19	\N	\N	0
-19	19	19	\N	0	\N	\N	\N	0
-20	20	20	\N	0	\N	20	\N	0
-21	21	21	\N	0	str_21	\N	\N	0
-22	22	22	\N	0	\N	\N	1970-01-23	0
-insert after alter modify column 3
-1	Date
-1	UInt64
-12	None
-12	String
-0	0	0	\N	0	\N	\N	\N	0
-1	1	1	\N	0	\N	\N	\N	0
-2	2	2	\N	0	\N	\N	\N	0
-3	3	3	\N	0	3	\N	\N	0
-4	4	4	\N	0	4	\N	\N	0
-5	5	5	\N	0	5	\N	\N	0
-6	6	6	\N	0	str_6	\N	\N	0
-7	7	7	\N	0	str_7	\N	\N	0
-8	8	8	\N	0	str_8	\N	\N	0
-9	9	9	\N	0	\N	\N	\N	0
-10	10	10	\N	0	\N	\N	\N	0
-11	11	11	\N	0	\N	\N	\N	0
-12	12	12	\N	0	12	\N	\N	0
-13	13	13	\N	0	str_13	\N	\N	0
-14	14	14	\N	0	\N	\N	\N	0
-15	15	15	\N	0	\N	\N	\N	0
-16	16	16	\N	0	16	\N	\N	0
-17	17	17	\N	0	str_17	\N	\N	0
-18	18	18	\N	0	1970-01-19	\N	\N	0
-19	19	19	\N	0	\N	\N	\N	0
-20	20	20	\N	0	\N	20	\N	0
-21	21	21	\N	0	str_21	\N	\N	0
-22	22	22	\N	0	\N	\N	1970-01-23	0
-23	\N	\N	\N	0	\N	\N	\N	0
-24	24	24	\N	0	\N	\N	\N	0
-25	str_25	\N	str_25	0	\N	\N	\N	0
diff --git a/tests/queries/0_stateless/03040_dynamic_type_alters_1.sh b/tests/queries/0_stateless/03040_dynamic_type_alters_1.sh
deleted file mode 100755
index 1f2a6a31ad7..00000000000
--- a/tests/queries/0_stateless/03040_dynamic_type_alters_1.sh
+++ /dev/null
@@ -1,77 +0,0 @@
-#!/usr/bin/env bash
-# Tags: long
-
-CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# reset --log_comment
-CLICKHOUSE_LOG_COMMENT=
-# shellcheck source=../shell_config.sh
-. "$CUR_DIR"/../shell_config.sh
-
-CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_dynamic_type=1 --allow_experimental_variant_type=1 --use_variant_as_common_type=1"
-
-function run()
-{
-    echo "initial insert"
-    $CH_CLIENT -q "insert into test select number, number from numbers(3)"
-
-    echo "alter add column 1"
-    $CH_CLIENT -q "alter table test add column d Dynamic(max_types=3) settings mutations_sync=1"
-    $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)"
-    $CH_CLIENT -q "select x, y, d, d.String, d.UInt64, d.\`Tuple(a UInt64)\`.a from test order by x"
-
-    echo "insert after alter add column 1"
-    $CH_CLIENT -q "insert into test select number, number, number from numbers(3, 3)"
-    $CH_CLIENT -q "insert into test select number, number, 'str_' || toString(number) from numbers(6, 3)"
-    $CH_CLIENT -q "insert into test select number, number, NULL from numbers(9, 3)"
-    $CH_CLIENT -q "insert into test select number, number, multiIf(number % 3 == 0, number, number % 3 == 1, 'str_' || toString(number), NULL) from numbers(12, 3)"
-    $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)"
-    $CH_CLIENT -q "select x, y, d, d.String, d.UInt64, d.Date, d.\`Tuple(a UInt64)\`.a from test order by x"
-
-    echo "alter modify column 1"
-    $CH_CLIENT -q "alter table test modify column d Dynamic(max_types=1) settings mutations_sync=1"
-    $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)"
-    $CH_CLIENT -q "select x, y, d, d.String, d.UInt64, d.Date, d.\`Tuple(a UInt64)\`.a from test order by x"
-
-    echo "insert after alter modify column 1"
-    $CH_CLIENT -q "insert into test select number, number, multiIf(number % 4 == 0, number, number % 4 == 1, 'str_' || toString(number), number % 4 == 2, toDate(number), NULL) from numbers(15, 4)"
-    $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)"
-    $CH_CLIENT -q "select x, y, d, d.String, d.UInt64, d.Date, d.\`Tuple(a UInt64)\`.a from test order by x"
-
-    echo "alter modify column 2"
-    $CH_CLIENT -q "alter table test modify column d Dynamic(max_types=3) settings mutations_sync=1"
-    $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)"
-    $CH_CLIENT -q "select x, y, d, d.String, d.UInt64, d.Date, d.\`Tuple(a UInt64)\`.a from test order by x"
-
-    echo "insert after alter modify column 2"
-    $CH_CLIENT -q "insert into test select number, number, multiIf(number % 4 == 0, number, number % 4 == 1, 'str_' || toString(number), number % 4 == 2, toDate(number), NULL) from numbers(19, 4)"
-    $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)"
-    $CH_CLIENT -q "select x, y, d, d.String, d.UInt64, d.Date, d.\`Tuple(a UInt64)\`.a from test order by x"
-
-    echo "alter modify column 3"
-    $CH_CLIENT -q "alter table test modify column y Dynamic settings mutations_sync=1"
-    $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)"
-    $CH_CLIENT -q "select x, y, y.UInt64, y.String, y.\`Tuple(a UInt64)\`.a, d.String, d.UInt64, d.Date, d.\`Tuple(a UInt64)\`.a from test order by x"
-
-    echo "insert after alter modify column 3"
-    $CH_CLIENT -q "insert into test select number, multiIf(number % 3 == 0, number, number % 3 == 1, 'str_' || toString(number), NULL), NULL from numbers(23, 3)"
-    $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)"
-    $CH_CLIENT -q "select x, y, y.UInt64, y.String, y.\`Tuple(a UInt64)\`.a, d.String, d.UInt64, d.Date, d.\`Tuple(a UInt64)\`.a from test order by x"
-}
-
-$CH_CLIENT -q "drop table if exists test;"
-
-echo "Memory"
-$CH_CLIENT -q "create table test (x UInt64, y UInt64) engine=Memory"
-run
-$CH_CLIENT -q "drop table test;"
-
-echo "MergeTree compact"
-$CH_CLIENT -q "create table test (x UInt64, y UInt64) engine=MergeTree order by x settings min_rows_for_wide_part=100000000, min_bytes_for_wide_part=1000000000;"
-run
-$CH_CLIENT -q "drop table test;"
-
-echo "MergeTree wide"
-$CH_CLIENT -q "create table test (x UInt64, y UInt64 ) engine=MergeTree order by x settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1;"
-run
-$CH_CLIENT -q "drop table test;"
-
diff --git a/tests/queries/0_stateless/03040_dynamic_type_alters_1_compact_merge_tree.reference b/tests/queries/0_stateless/03040_dynamic_type_alters_1_compact_merge_tree.reference
new file mode 100644
index 00000000000..2ec301b747b
--- /dev/null
+++ b/tests/queries/0_stateless/03040_dynamic_type_alters_1_compact_merge_tree.reference
@@ -0,0 +1,174 @@
+initial insert
+alter add column 1
+3	None
+0	0	\N	\N	\N	0
+1	1	\N	\N	\N	0
+2	2	\N	\N	\N	0
+insert after alter add column 1
+4	String
+4	UInt64
+7	None
+0	0	\N	\N	\N	\N	0
+1	1	\N	\N	\N	\N	0
+2	2	\N	\N	\N	\N	0
+3	3	3	\N	3	\N	0
+4	4	4	\N	4	\N	0
+5	5	5	\N	5	\N	0
+6	6	str_6	str_6	\N	\N	0
+7	7	str_7	str_7	\N	\N	0
+8	8	str_8	str_8	\N	\N	0
+9	9	\N	\N	\N	\N	0
+10	10	\N	\N	\N	\N	0
+11	11	\N	\N	\N	\N	0
+12	12	12	\N	12	\N	0
+13	13	str_13	str_13	\N	\N	0
+14	14	\N	\N	\N	\N	0
+alter modify column 1
+7	None
+8	String
+0	0	\N	\N	\N	\N	0
+1	1	\N	\N	\N	\N	0
+2	2	\N	\N	\N	\N	0
+3	3	3	3	\N	\N	0
+4	4	4	4	\N	\N	0
+5	5	5	5	\N	\N	0
+6	6	str_6	str_6	\N	\N	0
+7	7	str_7	str_7	\N	\N	0
+8	8	str_8	str_8	\N	\N	0
+9	9	\N	\N	\N	\N	0
+10	10	\N	\N	\N	\N	0
+11	11	\N	\N	\N	\N	0
+12	12	12	12	\N	\N	0
+13	13	str_13	str_13	\N	\N	0
+14	14	\N	\N	\N	\N	0
+insert after alter modify column 1
+8	None
+11	String
+0	0	\N	\N	\N	\N	0
+1	1	\N	\N	\N	\N	0
+2	2	\N	\N	\N	\N	0
+3	3	3	3	\N	\N	0
+4	4	4	4	\N	\N	0
+5	5	5	5	\N	\N	0
+6	6	str_6	str_6	\N	\N	0
+7	7	str_7	str_7	\N	\N	0
+8	8	str_8	str_8	\N	\N	0
+9	9	\N	\N	\N	\N	0
+10	10	\N	\N	\N	\N	0
+11	11	\N	\N	\N	\N	0
+12	12	12	12	\N	\N	0
+13	13	str_13	str_13	\N	\N	0
+14	14	\N	\N	\N	\N	0
+15	15	\N	\N	\N	\N	0
+16	16	16	16	\N	\N	0
+17	17	str_17	str_17	\N	\N	0
+18	18	1970-01-19	1970-01-19	\N	\N	0
+alter modify column 2
+8	None
+11	String
+0	0	\N	\N	\N	\N	0
+1	1	\N	\N	\N	\N	0
+2	2	\N	\N	\N	\N	0
+3	3	3	3	\N	\N	0
+4	4	4	4	\N	\N	0
+5	5	5	5	\N	\N	0
+6	6	str_6	str_6	\N	\N	0
+7	7	str_7	str_7	\N	\N	0
+8	8	str_8	str_8	\N	\N	0
+9	9	\N	\N	\N	\N	0
+10	10	\N	\N	\N	\N	0
+11	11	\N	\N	\N	\N	0
+12	12	12	12	\N	\N	0
+13	13	str_13	str_13	\N	\N	0
+14	14	\N	\N	\N	\N	0
+15	15	\N	\N	\N	\N	0
+16	16	16	16	\N	\N	0
+17	17	str_17	str_17	\N	\N	0
+18	18	1970-01-19	1970-01-19	\N	\N	0
+insert after alter modify column 2
+1	Date
+1	UInt64
+9	None
+12	String
+0	0	\N	\N	\N	\N	0
+1	1	\N	\N	\N	\N	0
+2	2	\N	\N	\N	\N	0
+3	3	3	3	\N	\N	0
+4	4	4	4	\N	\N	0
+5	5	5	5	\N	\N	0
+6	6	str_6	str_6	\N	\N	0
+7	7	str_7	str_7	\N	\N	0
+8	8	str_8	str_8	\N	\N	0
+9	9	\N	\N	\N	\N	0
+10	10	\N	\N	\N	\N	0
+11	11	\N	\N	\N	\N	0
+12	12	12	12	\N	\N	0
+13	13	str_13	str_13	\N	\N	0
+14	14	\N	\N	\N	\N	0
+15	15	\N	\N	\N	\N	0
+16	16	16	16	\N	\N	0
+17	17	str_17	str_17	\N	\N	0
+18	18	1970-01-19	1970-01-19	\N	\N	0
+19	19	\N	\N	\N	\N	0
+20	20	20	\N	20	\N	0
+21	21	str_21	str_21	\N	\N	0
+22	22	1970-01-23	\N	\N	1970-01-23	0
+alter modify column 3
+1	Date
+1	UInt64
+9	None
+12	String
+0	0	0	\N	0	\N	\N	\N	0
+1	1	1	\N	0	\N	\N	\N	0
+2	2	2	\N	0	\N	\N	\N	0
+3	3	3	\N	0	3	\N	\N	0
+4	4	4	\N	0	4	\N	\N	0
+5	5	5	\N	0	5	\N	\N	0
+6	6	6	\N	0	str_6	\N	\N	0
+7	7	7	\N	0	str_7	\N	\N	0
+8	8	8	\N	0	str_8	\N	\N	0
+9	9	9	\N	0	\N	\N	\N	0
+10	10	10	\N	0	\N	\N	\N	0
+11	11	11	\N	0	\N	\N	\N	0
+12	12	12	\N	0	12	\N	\N	0
+13	13	13	\N	0	str_13	\N	\N	0
+14	14	14	\N	0	\N	\N	\N	0
+15	15	15	\N	0	\N	\N	\N	0
+16	16	16	\N	0	16	\N	\N	0
+17	17	17	\N	0	str_17	\N	\N	0
+18	18	18	\N	0	1970-01-19	\N	\N	0
+19	19	19	\N	0	\N	\N	\N	0
+20	20	20	\N	0	\N	20	\N	0
+21	21	21	\N	0	str_21	\N	\N	0
+22	22	22	\N	0	\N	\N	1970-01-23	0
+insert after alter modify column 3
+1	Date
+1	UInt64
+12	None
+12	String
+0	0	0	\N	0	\N	\N	\N	0
+1	1	1	\N	0	\N	\N	\N	0
+2	2	2	\N	0	\N	\N	\N	0
+3	3	3	\N	0	3	\N	\N	0
+4	4	4	\N	0	4	\N	\N	0
+5	5	5	\N	0	5	\N	\N	0
+6	6	6	\N	0	str_6	\N	\N	0
+7	7	7	\N	0	str_7	\N	\N	0
+8	8	8	\N	0	str_8	\N	\N	0
+9	9	9	\N	0	\N	\N	\N	0
+10	10	10	\N	0	\N	\N	\N	0
+11	11	11	\N	0	\N	\N	\N	0
+12	12	12	\N	0	12	\N	\N	0
+13	13	13	\N	0	str_13	\N	\N	0
+14	14	14	\N	0	\N	\N	\N	0
+15	15	15	\N	0	\N	\N	\N	0
+16	16	16	\N	0	16	\N	\N	0
+17	17	17	\N	0	str_17	\N	\N	0
+18	18	18	\N	0	1970-01-19	\N	\N	0
+19	19	19	\N	0	\N	\N	\N	0
+20	20	20	\N	0	\N	20	\N	0
+21	21	21	\N	0	str_21	\N	\N	0
+22	22	22	\N	0	\N	\N	1970-01-23	0
+23	\N	\N	\N	0	\N	\N	\N	0
+24	24	24	\N	0	\N	\N	\N	0
+25	str_25	\N	str_25	0	\N	\N	\N	0
diff --git a/tests/queries/0_stateless/03040_dynamic_type_alters_1_compact_merge_tree.sql b/tests/queries/0_stateless/03040_dynamic_type_alters_1_compact_merge_tree.sql
new file mode 100644
index 00000000000..4ab700306d4
--- /dev/null
+++ b/tests/queries/0_stateless/03040_dynamic_type_alters_1_compact_merge_tree.sql
@@ -0,0 +1,53 @@
+set allow_experimental_dynamic_type = 1;
+set allow_experimental_variant_type = 1;
+set use_variant_as_common_type = 1;
+
+drop table if exists test;
+create table test (x UInt64, y UInt64) engine=MergeTree order by x settings min_rows_for_wide_part=100000000, min_bytes_for_wide_part=1000000000;
+select 'initial insert';
+insert into test select number, number from numbers(3);
+
+select 'alter add column 1';
+alter table test add column d Dynamic(max_types=3) settings mutations_sync=1;
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+select x, y, d, d.String, d.UInt64, d.`Tuple(a UInt64)`.a from test order by x;
+
+select 'insert after alter add column 1';
+insert into test select number, number, number from numbers(3, 3);
+insert into test select number, number, 'str_' || toString(number) from numbers(6, 3);
+insert into test select number, number, NULL from numbers(9, 3);
+insert into test select number, number, multiIf(number % 3 == 0, number, number % 3 == 1, 'str_' || toString(number), NULL) from numbers(12, 3);
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+select x, y, d, d.String, d.UInt64, d.Date, d.`Tuple(a UInt64)`.a from test order by x;
+
+select 'alter modify column 1';
+alter table test modify column d Dynamic(max_types=1) settings mutations_sync=1;
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+select x, y, d, d.String, d.UInt64, d.Date, d.`Tuple(a UInt64)`.a from test order by x;
+
+select 'insert after alter modify column 1';
+insert into test select number, number, multiIf(number % 4 == 0, number, number % 4 == 1, 'str_' || toString(number), number % 4 == 2, toDate(number), NULL) from numbers(15, 4);
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+select x, y, d, d.String, d.UInt64, d.Date, d.`Tuple(a UInt64)`.a from test order by x;
+
+select 'alter modify column 2';
+alter table test modify column d Dynamic(max_types=3) settings mutations_sync=1;
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+select x, y, d, d.String, d.UInt64, d.Date, d.`Tuple(a UInt64)`.a from test order by x;
+
+select 'insert after alter modify column 2';
+insert into test select number, number, multiIf(number % 4 == 0, number, number % 4 == 1, 'str_' || toString(number), number % 4 == 2, toDate(number), NULL) from numbers(19, 4);
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+select x, y, d, d.String, d.UInt64, d.Date, d.`Tuple(a UInt64)`.a from test order by x;
+
+select 'alter modify column 3';
+alter table test modify column y Dynamic settings mutations_sync=1;
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+select x, y, y.UInt64, y.String, y.`Tuple(a UInt64)`.a, d.String, d.UInt64, d.Date, d.`Tuple(a UInt64)`.a from test order by x;
+
+select 'insert after alter modify column 3';
+insert into test select number, multiIf(number % 3 == 0, number, number % 3 == 1, 'str_' || toString(number), NULL), NULL from numbers(23, 3);
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+select x, y, y.UInt64, y.String, y.`Tuple(a UInt64)`.a, d.String, d.UInt64, d.Date, d.`Tuple(a UInt64)`.a from test order by x;
+
+drop table test;
\ No newline at end of file
diff --git a/tests/queries/0_stateless/03040_dynamic_type_alters_1_memory.reference b/tests/queries/0_stateless/03040_dynamic_type_alters_1_memory.reference
new file mode 100644
index 00000000000..c592528c3cd
--- /dev/null
+++ b/tests/queries/0_stateless/03040_dynamic_type_alters_1_memory.reference
@@ -0,0 +1,175 @@
+initial insert
+alter add column 1
+3	None
+0	0	\N	\N	\N	0
+1	1	\N	\N	\N	0
+2	2	\N	\N	\N	0
+insert after alter add column 1
+4	String
+4	UInt64
+7	None
+0	0	\N	\N	\N	\N	0
+1	1	\N	\N	\N	\N	0
+2	2	\N	\N	\N	\N	0
+3	3	3	\N	3	\N	0
+4	4	4	\N	4	\N	0
+5	5	5	\N	5	\N	0
+6	6	str_6	str_6	\N	\N	0
+7	7	str_7	str_7	\N	\N	0
+8	8	str_8	str_8	\N	\N	0
+9	9	\N	\N	\N	\N	0
+10	10	\N	\N	\N	\N	0
+11	11	\N	\N	\N	\N	0
+12	12	12	\N	12	\N	0
+13	13	str_13	str_13	\N	\N	0
+14	14	\N	\N	\N	\N	0
+alter modify column 1
+7	None
+8	String
+0	0	\N	\N	\N	\N	0
+1	1	\N	\N	\N	\N	0
+2	2	\N	\N	\N	\N	0
+3	3	3	3	\N	\N	0
+4	4	4	4	\N	\N	0
+5	5	5	5	\N	\N	0
+6	6	str_6	str_6	\N	\N	0
+7	7	str_7	str_7	\N	\N	0
+8	8	str_8	str_8	\N	\N	0
+9	9	\N	\N	\N	\N	0
+10	10	\N	\N	\N	\N	0
+11	11	\N	\N	\N	\N	0
+12	12	12	12	\N	\N	0
+13	13	str_13	str_13	\N	\N	0
+14	14	\N	\N	\N	\N	0
+insert after alter modify column 1
+8	None
+11	String
+0	0	\N	\N	\N	\N	0
+1	1	\N	\N	\N	\N	0
+2	2	\N	\N	\N	\N	0
+3	3	3	3	\N	\N	0
+4	4	4	4	\N	\N	0
+5	5	5	5	\N	\N	0
+6	6	str_6	str_6	\N	\N	0
+7	7	str_7	str_7	\N	\N	0
+8	8	str_8	str_8	\N	\N	0
+9	9	\N	\N	\N	\N	0
+10	10	\N	\N	\N	\N	0
+11	11	\N	\N	\N	\N	0
+12	12	12	12	\N	\N	0
+13	13	str_13	str_13	\N	\N	0
+14	14	\N	\N	\N	\N	0
+15	15	\N	\N	\N	\N	0
+16	16	16	16	\N	\N	0
+17	17	str_17	str_17	\N	\N	0
+18	18	1970-01-19	1970-01-19	\N	\N	0
+alter modify column 2
+4	UInt64
+7	String
+8	None
+0	0	\N	\N	\N	\N	0
+1	1	\N	\N	\N	\N	0
+2	2	\N	\N	\N	\N	0
+3	3	3	\N	3	\N	0
+4	4	4	\N	4	\N	0
+5	5	5	\N	5	\N	0
+6	6	str_6	str_6	\N	\N	0
+7	7	str_7	str_7	\N	\N	0
+8	8	str_8	str_8	\N	\N	0
+9	9	\N	\N	\N	\N	0
+10	10	\N	\N	\N	\N	0
+11	11	\N	\N	\N	\N	0
+12	12	12	\N	12	\N	0
+13	13	str_13	str_13	\N	\N	0
+14	14	\N	\N	\N	\N	0
+15	15	\N	\N	\N	\N	0
+16	16	16	16	\N	\N	0
+17	17	str_17	str_17	\N	\N	0
+18	18	1970-01-19	1970-01-19	\N	\N	0
+insert after alter modify column 2
+1	Date
+5	UInt64
+8	String
+9	None
+0	0	\N	\N	\N	\N	0
+1	1	\N	\N	\N	\N	0
+2	2	\N	\N	\N	\N	0
+3	3	3	\N	3	\N	0
+4	4	4	\N	4	\N	0
+5	5	5	\N	5	\N	0
+6	6	str_6	str_6	\N	\N	0
+7	7	str_7	str_7	\N	\N	0
+8	8	str_8	str_8	\N	\N	0
+9	9	\N	\N	\N	\N	0
+10	10	\N	\N	\N	\N	0
+11	11	\N	\N	\N	\N	0
+12	12	12	\N	12	\N	0
+13	13	str_13	str_13	\N	\N	0
+14	14	\N	\N	\N	\N	0
+15	15	\N	\N	\N	\N	0
+16	16	16	16	\N	\N	0
+17	17	str_17	str_17	\N	\N	0
+18	18	1970-01-19	1970-01-19	\N	\N	0
+19	19	\N	\N	\N	\N	0
+20	20	20	\N	20	\N	0
+21	21	str_21	str_21	\N	\N	0
+22	22	1970-01-23	\N	\N	1970-01-23	0
+alter modify column 3
+1	Date
+5	UInt64
+8	String
+9	None
+0	0	0	\N	0	\N	\N	\N	0
+1	1	1	\N	0	\N	\N	\N	0
+2	2	2	\N	0	\N	\N	\N	0
+3	3	3	\N	0	\N	3	\N	0
+4	4	4	\N	0	\N	4	\N	0
+5	5	5	\N	0	\N	5	\N	0
+6	6	6	\N	0	str_6	\N	\N	0
+7	7	7	\N	0	str_7	\N	\N	0
+8	8	8	\N	0	str_8	\N	\N	0
+9	9	9	\N	0	\N	\N	\N	0
+10	10	10	\N	0	\N	\N	\N	0
+11	11	11	\N	0	\N	\N	\N	0
+12	12	12	\N	0	\N	12	\N	0
+13	13	13	\N	0	str_13	\N	\N	0
+14	14	14	\N	0	\N	\N	\N	0
+15	15	15	\N	0	\N	\N	\N	0
+16	16	16	\N	0	16	\N	\N	0
+17	17	17	\N	0	str_17	\N	\N	0
+18	18	18	\N	0	1970-01-19	\N	\N	0
+19	19	19	\N	0	\N	\N	\N	0
+20	20	20	\N	0	\N	20	\N	0
+21	21	21	\N	0	str_21	\N	\N	0
+22	22	22	\N	0	\N	\N	1970-01-23	0
+insert after alter modify column 3
+1	Date
+5	UInt64
+8	String
+12	None
+0	0	0	\N	0	\N	\N	\N	0
+1	1	1	\N	0	\N	\N	\N	0
+2	2	2	\N	0	\N	\N	\N	0
+3	3	3	\N	0	\N	3	\N	0
+4	4	4	\N	0	\N	4	\N	0
+5	5	5	\N	0	\N	5	\N	0
+6	6	6	\N	0	str_6	\N	\N	0
+7	7	7	\N	0	str_7	\N	\N	0
+8	8	8	\N	0	str_8	\N	\N	0
+9	9	9	\N	0	\N	\N	\N	0
+10	10	10	\N	0	\N	\N	\N	0
+11	11	11	\N	0	\N	\N	\N	0
+12	12	12	\N	0	\N	12	\N	0
+13	13	13	\N	0	str_13	\N	\N	0
+14	14	14	\N	0	\N	\N	\N	0
+15	15	15	\N	0	\N	\N	\N	0
+16	16	16	\N	0	16	\N	\N	0
+17	17	17	\N	0	str_17	\N	\N	0
+18	18	18	\N	0	1970-01-19	\N	\N	0
+19	19	19	\N	0	\N	\N	\N	0
+20	20	20	\N	0	\N	20	\N	0
+21	21	21	\N	0	str_21	\N	\N	0
+22	22	22	\N	0	\N	\N	1970-01-23	0
+23	\N	\N	\N	0	\N	\N	\N	0
+24	24	24	\N	0	\N	\N	\N	0
+25	str_25	\N	str_25	0	\N	\N	\N	0
diff --git a/tests/queries/0_stateless/03040_dynamic_type_alters_1_memory.sql b/tests/queries/0_stateless/03040_dynamic_type_alters_1_memory.sql
new file mode 100644
index 00000000000..e802fd034ce
--- /dev/null
+++ b/tests/queries/0_stateless/03040_dynamic_type_alters_1_memory.sql
@@ -0,0 +1,53 @@
+set allow_experimental_dynamic_type = 1;
+set allow_experimental_variant_type = 1;
+set use_variant_as_common_type = 1;
+
+drop table if exists test;
+create table test (x UInt64, y UInt64) engine=Memory;
+select 'initial insert';
+insert into test select number, number from numbers(3);
+
+select 'alter add column 1';
+alter table test add column d Dynamic(max_types=3) settings mutations_sync=1;
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+select x, y, d, d.String, d.UInt64, d.`Tuple(a UInt64)`.a from test order by x;
+
+select 'insert after alter add column 1';
+insert into test select number, number, number from numbers(3, 3);
+insert into test select number, number, 'str_' || toString(number) from numbers(6, 3);
+insert into test select number, number, NULL from numbers(9, 3);
+insert into test select number, number, multiIf(number % 3 == 0, number, number % 3 == 1, 'str_' || toString(number), NULL) from numbers(12, 3);
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+select x, y, d, d.String, d.UInt64, d.Date, d.`Tuple(a UInt64)`.a from test order by x;
+
+select 'alter modify column 1';
+alter table test modify column d Dynamic(max_types=1) settings mutations_sync=1;
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+select x, y, d, d.String, d.UInt64, d.Date, d.`Tuple(a UInt64)`.a from test order by x;
+
+select 'insert after alter modify column 1';
+insert into test select number, number, multiIf(number % 4 == 0, number, number % 4 == 1, 'str_' || toString(number), number % 4 == 2, toDate(number), NULL) from numbers(15, 4);
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+select x, y, d, d.String, d.UInt64, d.Date, d.`Tuple(a UInt64)`.a from test order by x;
+
+select 'alter modify column 2';
+alter table test modify column d Dynamic(max_types=3) settings mutations_sync=1;
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+select x, y, d, d.String, d.UInt64, d.Date, d.`Tuple(a UInt64)`.a from test order by x;
+
+select 'insert after alter modify column 2';
+insert into test select number, number, multiIf(number % 4 == 0, number, number % 4 == 1, 'str_' || toString(number), number % 4 == 2, toDate(number), NULL) from numbers(19, 4);
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+select x, y, d, d.String, d.UInt64, d.Date, d.`Tuple(a UInt64)`.a from test order by x;
+
+select 'alter modify column 3';
+alter table test modify column y Dynamic settings mutations_sync=1;
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+select x, y, y.UInt64, y.String, y.`Tuple(a UInt64)`.a, d.String, d.UInt64, d.Date, d.`Tuple(a UInt64)`.a from test order by x;
+
+select 'insert after alter modify column 3';
+insert into test select number, multiIf(number % 3 == 0, number, number % 3 == 1, 'str_' || toString(number), NULL), NULL from numbers(23, 3);
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+select x, y, y.UInt64, y.String, y.`Tuple(a UInt64)`.a, d.String, d.UInt64, d.Date, d.`Tuple(a UInt64)`.a from test order by x;
+
+drop table test;
\ No newline at end of file
diff --git a/tests/queries/0_stateless/03040_dynamic_type_alters_1_wide_merge_tree.reference b/tests/queries/0_stateless/03040_dynamic_type_alters_1_wide_merge_tree.reference
new file mode 100644
index 00000000000..2ec301b747b
--- /dev/null
+++ b/tests/queries/0_stateless/03040_dynamic_type_alters_1_wide_merge_tree.reference
@@ -0,0 +1,174 @@
+initial insert
+alter add column 1
+3	None
+0	0	\N	\N	\N	0
+1	1	\N	\N	\N	0
+2	2	\N	\N	\N	0
+insert after alter add column 1
+4	String
+4	UInt64
+7	None
+0	0	\N	\N	\N	\N	0
+1	1	\N	\N	\N	\N	0
+2	2	\N	\N	\N	\N	0
+3	3	3	\N	3	\N	0
+4	4	4	\N	4	\N	0
+5	5	5	\N	5	\N	0
+6	6	str_6	str_6	\N	\N	0
+7	7	str_7	str_7	\N	\N	0
+8	8	str_8	str_8	\N	\N	0
+9	9	\N	\N	\N	\N	0
+10	10	\N	\N	\N	\N	0
+11	11	\N	\N	\N	\N	0
+12	12	12	\N	12	\N	0
+13	13	str_13	str_13	\N	\N	0
+14	14	\N	\N	\N	\N	0
+alter modify column 1
+7	None
+8	String
+0	0	\N	\N	\N	\N	0
+1	1	\N	\N	\N	\N	0
+2	2	\N	\N	\N	\N	0
+3	3	3	3	\N	\N	0
+4	4	4	4	\N	\N	0
+5	5	5	5	\N	\N	0
+6	6	str_6	str_6	\N	\N	0
+7	7	str_7	str_7	\N	\N	0
+8	8	str_8	str_8	\N	\N	0
+9	9	\N	\N	\N	\N	0
+10	10	\N	\N	\N	\N	0
+11	11	\N	\N	\N	\N	0
+12	12	12	12	\N	\N	0
+13	13	str_13	str_13	\N	\N	0
+14	14	\N	\N	\N	\N	0
+insert after alter modify column 1
+8	None
+11	String
+0	0	\N	\N	\N	\N	0
+1	1	\N	\N	\N	\N	0
+2	2	\N	\N	\N	\N	0
+3	3	3	3	\N	\N	0
+4	4	4	4	\N	\N	0
+5	5	5	5	\N	\N	0
+6	6	str_6	str_6	\N	\N	0
+7	7	str_7	str_7	\N	\N	0
+8	8	str_8	str_8	\N	\N	0
+9	9	\N	\N	\N	\N	0
+10	10	\N	\N	\N	\N	0
+11	11	\N	\N	\N	\N	0
+12	12	12	12	\N	\N	0
+13	13	str_13	str_13	\N	\N	0
+14	14	\N	\N	\N	\N	0
+15	15	\N	\N	\N	\N	0
+16	16	16	16	\N	\N	0
+17	17	str_17	str_17	\N	\N	0
+18	18	1970-01-19	1970-01-19	\N	\N	0
+alter modify column 2
+8	None
+11	String
+0	0	\N	\N	\N	\N	0
+1	1	\N	\N	\N	\N	0
+2	2	\N	\N	\N	\N	0
+3	3	3	3	\N	\N	0
+4	4	4	4	\N	\N	0
+5	5	5	5	\N	\N	0
+6	6	str_6	str_6	\N	\N	0
+7	7	str_7	str_7	\N	\N	0
+8	8	str_8	str_8	\N	\N	0
+9	9	\N	\N	\N	\N	0
+10	10	\N	\N	\N	\N	0
+11	11	\N	\N	\N	\N	0
+12	12	12	12	\N	\N	0
+13	13	str_13	str_13	\N	\N	0
+14	14	\N	\N	\N	\N	0
+15	15	\N	\N	\N	\N	0
+16	16	16	16	\N	\N	0
+17	17	str_17	str_17	\N	\N	0
+18	18	1970-01-19	1970-01-19	\N	\N	0
+insert after alter modify column 2
+1	Date
+1	UInt64
+9	None
+12	String
+0	0	\N	\N	\N	\N	0
+1	1	\N	\N	\N	\N	0
+2	2	\N	\N	\N	\N	0
+3	3	3	3	\N	\N	0
+4	4	4	4	\N	\N	0
+5	5	5	5	\N	\N	0
+6	6	str_6	str_6	\N	\N	0
+7	7	str_7	str_7	\N	\N	0
+8	8	str_8	str_8	\N	\N	0
+9	9	\N	\N	\N	\N	0
+10	10	\N	\N	\N	\N	0
+11	11	\N	\N	\N	\N	0
+12	12	12	12	\N	\N	0
+13	13	str_13	str_13	\N	\N	0
+14	14	\N	\N	\N	\N	0
+15	15	\N	\N	\N	\N	0
+16	16	16	16	\N	\N	0
+17	17	str_17	str_17	\N	\N	0
+18	18	1970-01-19	1970-01-19	\N	\N	0
+19	19	\N	\N	\N	\N	0
+20	20	20	\N	20	\N	0
+21	21	str_21	str_21	\N	\N	0
+22	22	1970-01-23	\N	\N	1970-01-23	0
+alter modify column 3
+1	Date
+1	UInt64
+9	None
+12	String
+0	0	0	\N	0	\N	\N	\N	0
+1	1	1	\N	0	\N	\N	\N	0
+2	2	2	\N	0	\N	\N	\N	0
+3	3	3	\N	0	3	\N	\N	0
+4	4	4	\N	0	4	\N	\N	0
+5	5	5	\N	0	5	\N	\N	0
+6	6	6	\N	0	str_6	\N	\N	0
+7	7	7	\N	0	str_7	\N	\N	0
+8	8	8	\N	0	str_8	\N	\N	0
+9	9	9	\N	0	\N	\N	\N	0
+10	10	10	\N	0	\N	\N	\N	0
+11	11	11	\N	0	\N	\N	\N	0
+12	12	12	\N	0	12	\N	\N	0
+13	13	13	\N	0	str_13	\N	\N	0
+14	14	14	\N	0	\N	\N	\N	0
+15	15	15	\N	0	\N	\N	\N	0
+16	16	16	\N	0	16	\N	\N	0
+17	17	17	\N	0	str_17	\N	\N	0
+18	18	18	\N	0	1970-01-19	\N	\N	0
+19	19	19	\N	0	\N	\N	\N	0
+20	20	20	\N	0	\N	20	\N	0
+21	21	21	\N	0	str_21	\N	\N	0
+22	22	22	\N	0	\N	\N	1970-01-23	0
+insert after alter modify column 3
+1	Date
+1	UInt64
+12	None
+12	String
+0	0	0	\N	0	\N	\N	\N	0
+1	1	1	\N	0	\N	\N	\N	0
+2	2	2	\N	0	\N	\N	\N	0
+3	3	3	\N	0	3	\N	\N	0
+4	4	4	\N	0	4	\N	\N	0
+5	5	5	\N	0	5	\N	\N	0
+6	6	6	\N	0	str_6	\N	\N	0
+7	7	7	\N	0	str_7	\N	\N	0
+8	8	8	\N	0	str_8	\N	\N	0
+9	9	9	\N	0	\N	\N	\N	0
+10	10	10	\N	0	\N	\N	\N	0
+11	11	11	\N	0	\N	\N	\N	0
+12	12	12	\N	0	12	\N	\N	0
+13	13	13	\N	0	str_13	\N	\N	0
+14	14	14	\N	0	\N	\N	\N	0
+15	15	15	\N	0	\N	\N	\N	0
+16	16	16	\N	0	16	\N	\N	0
+17	17	17	\N	0	str_17	\N	\N	0
+18	18	18	\N	0	1970-01-19	\N	\N	0
+19	19	19	\N	0	\N	\N	\N	0
+20	20	20	\N	0	\N	20	\N	0
+21	21	21	\N	0	str_21	\N	\N	0
+22	22	22	\N	0	\N	\N	1970-01-23	0
+23	\N	\N	\N	0	\N	\N	\N	0
+24	24	24	\N	0	\N	\N	\N	0
+25	str_25	\N	str_25	0	\N	\N	\N	0
diff --git a/tests/queries/0_stateless/03040_dynamic_type_alters_1_wide_merge_tree.sql b/tests/queries/0_stateless/03040_dynamic_type_alters_1_wide_merge_tree.sql
new file mode 100644
index 00000000000..55c4f0b5f0c
--- /dev/null
+++ b/tests/queries/0_stateless/03040_dynamic_type_alters_1_wide_merge_tree.sql
@@ -0,0 +1,53 @@
+set allow_experimental_dynamic_type = 1;
+set allow_experimental_variant_type = 1;
+set use_variant_as_common_type = 1;
+
+drop table if exists test;
+create table test (x UInt64, y UInt64) engine=MergeTree order by x settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1;
+select 'initial insert';
+insert into test select number, number from numbers(3);
+
+select 'alter add column 1';
+alter table test add column d Dynamic(max_types=3) settings mutations_sync=1;
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+select x, y, d, d.String, d.UInt64, d.`Tuple(a UInt64)`.a from test order by x;
+
+select 'insert after alter add column 1';
+insert into test select number, number, number from numbers(3, 3);
+insert into test select number, number, 'str_' || toString(number) from numbers(6, 3);
+insert into test select number, number, NULL from numbers(9, 3);
+insert into test select number, number, multiIf(number % 3 == 0, number, number % 3 == 1, 'str_' || toString(number), NULL) from numbers(12, 3);
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+select x, y, d, d.String, d.UInt64, d.Date, d.`Tuple(a UInt64)`.a from test order by x;
+
+select 'alter modify column 1';
+alter table test modify column d Dynamic(max_types=1) settings mutations_sync=1;
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+select x, y, d, d.String, d.UInt64, d.Date, d.`Tuple(a UInt64)`.a from test order by x;
+
+select 'insert after alter modify column 1';
+insert into test select number, number, multiIf(number % 4 == 0, number, number % 4 == 1, 'str_' || toString(number), number % 4 == 2, toDate(number), NULL) from numbers(15, 4);
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+select x, y, d, d.String, d.UInt64, d.Date, d.`Tuple(a UInt64)`.a from test order by x;
+
+select 'alter modify column 2';
+alter table test modify column d Dynamic(max_types=3) settings mutations_sync=1;
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+select x, y, d, d.String, d.UInt64, d.Date, d.`Tuple(a UInt64)`.a from test order by x;
+
+select 'insert after alter modify column 2';
+insert into test select number, number, multiIf(number % 4 == 0, number, number % 4 == 1, 'str_' || toString(number), number % 4 == 2, toDate(number), NULL) from numbers(19, 4);
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+select x, y, d, d.String, d.UInt64, d.Date, d.`Tuple(a UInt64)`.a from test order by x;
+
+select 'alter modify column 3';
+alter table test modify column y Dynamic settings mutations_sync=1;
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+select x, y, y.UInt64, y.String, y.`Tuple(a UInt64)`.a, d.String, d.UInt64, d.Date, d.`Tuple(a UInt64)`.a from test order by x;
+
+select 'insert after alter modify column 3';
+insert into test select number, multiIf(number % 3 == 0, number, number % 3 == 1, 'str_' || toString(number), NULL), NULL from numbers(23, 3);
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+select x, y, y.UInt64, y.String, y.`Tuple(a UInt64)`.a, d.String, d.UInt64, d.Date, d.`Tuple(a UInt64)`.a from test order by x;
+
+drop table test;
\ No newline at end of file
diff --git a/tests/queries/0_stateless/03040_dynamic_type_alters_2.reference b/tests/queries/0_stateless/03040_dynamic_type_alters_2.reference
deleted file mode 100644
index f7c00bd8c44..00000000000
--- a/tests/queries/0_stateless/03040_dynamic_type_alters_2.reference
+++ /dev/null
@@ -1,182 +0,0 @@
-MergeTree compact
-initial insert
-alter add column
-3	None
-0	0	\N	\N	\N	0
-1	1	\N	\N	\N	0
-2	2	\N	\N	\N	0
-insert after alter add column 1
-4	String
-4	UInt64
-7	None
-0	0	\N	\N	\N	\N	0
-1	1	\N	\N	\N	\N	0
-2	2	\N	\N	\N	\N	0
-3	3	3	\N	3	\N	0
-4	4	4	\N	4	\N	0
-5	5	5	\N	5	\N	0
-6	6	str_6	str_6	\N	\N	0
-7	7	str_7	str_7	\N	\N	0
-8	8	str_8	str_8	\N	\N	0
-9	9	\N	\N	\N	\N	0
-10	10	\N	\N	\N	\N	0
-11	11	\N	\N	\N	\N	0
-12	12	12	\N	12	\N	0
-13	13	str_13	str_13	\N	\N	0
-14	14	\N	\N	\N	\N	0
-alter rename column 1
-4	String
-4	UInt64
-7	None
-0	0	\N	\N	\N	\N	0
-1	1	\N	\N	\N	\N	0
-2	2	\N	\N	\N	\N	0
-3	3	3	\N	3	\N	0
-4	4	4	\N	4	\N	0
-5	5	5	\N	5	\N	0
-6	6	str_6	str_6	\N	\N	0
-7	7	str_7	str_7	\N	\N	0
-8	8	str_8	str_8	\N	\N	0
-9	9	\N	\N	\N	\N	0
-10	10	\N	\N	\N	\N	0
-11	11	\N	\N	\N	\N	0
-12	12	12	\N	12	\N	0
-13	13	str_13	str_13	\N	\N	0
-14	14	\N	\N	\N	\N	0
-insert nested dynamic
-3	Array(Dynamic)
-4	String
-4	UInt64
-7	None
-0	0	\N	\N	\N	\N	0	[]	[]	[]
-1	1	\N	\N	\N	\N	0	[]	[]	[]
-2	2	\N	\N	\N	\N	0	[]	[]	[]
-3	3	3	\N	3	\N	0	[]	[]	[]
-4	4	4	\N	4	\N	0	[]	[]	[]
-5	5	5	\N	5	\N	0	[]	[]	[]
-6	6	str_6	str_6	\N	\N	0	[]	[]	[]
-7	7	str_7	str_7	\N	\N	0	[]	[]	[]
-8	8	str_8	str_8	\N	\N	0	[]	[]	[]
-9	9	\N	\N	\N	\N	0	[]	[]	[]
-10	10	\N	\N	\N	\N	0	[]	[]	[]
-11	11	\N	\N	\N	\N	0	[]	[]	[]
-12	12	12	\N	12	\N	0	[]	[]	[]
-13	13	str_13	str_13	\N	\N	0	[]	[]	[]
-14	14	\N	\N	\N	\N	0	[]	[]	[]
-15	15	[15]	\N	\N	\N	0	[15]	[NULL]	[NULL]
-16	16	['str_16']	\N	\N	\N	0	[NULL]	['str_16']	[NULL]
-17	17	[17]	\N	\N	\N	0	[17]	[NULL]	[NULL]
-alter rename column 2
-3	Array(Dynamic)
-4	String
-4	UInt64
-7	None
-0	0	\N	\N	\N	\N	0	[]	[]	[]
-1	1	\N	\N	\N	\N	0	[]	[]	[]
-2	2	\N	\N	\N	\N	0	[]	[]	[]
-3	3	3	\N	3	\N	0	[]	[]	[]
-4	4	4	\N	4	\N	0	[]	[]	[]
-5	5	5	\N	5	\N	0	[]	[]	[]
-6	6	str_6	str_6	\N	\N	0	[]	[]	[]
-7	7	str_7	str_7	\N	\N	0	[]	[]	[]
-8	8	str_8	str_8	\N	\N	0	[]	[]	[]
-9	9	\N	\N	\N	\N	0	[]	[]	[]
-10	10	\N	\N	\N	\N	0	[]	[]	[]
-11	11	\N	\N	\N	\N	0	[]	[]	[]
-12	12	12	\N	12	\N	0	[]	[]	[]
-13	13	str_13	str_13	\N	\N	0	[]	[]	[]
-14	14	\N	\N	\N	\N	0	[]	[]	[]
-15	15	[15]	\N	\N	\N	0	[15]	[NULL]	[NULL]
-16	16	['str_16']	\N	\N	\N	0	[NULL]	['str_16']	[NULL]
-17	17	[17]	\N	\N	\N	0	[17]	[NULL]	[NULL]
-MergeTree wide
-initial insert
-alter add column
-3	None
-0	0	\N	\N	\N	0
-1	1	\N	\N	\N	0
-2	2	\N	\N	\N	0
-insert after alter add column 1
-4	String
-4	UInt64
-7	None
-0	0	\N	\N	\N	\N	0
-1	1	\N	\N	\N	\N	0
-2	2	\N	\N	\N	\N	0
-3	3	3	\N	3	\N	0
-4	4	4	\N	4	\N	0
-5	5	5	\N	5	\N	0
-6	6	str_6	str_6	\N	\N	0
-7	7	str_7	str_7	\N	\N	0
-8	8	str_8	str_8	\N	\N	0
-9	9	\N	\N	\N	\N	0
-10	10	\N	\N	\N	\N	0
-11	11	\N	\N	\N	\N	0
-12	12	12	\N	12	\N	0
-13	13	str_13	str_13	\N	\N	0
-14	14	\N	\N	\N	\N	0
-alter rename column 1
-4	String
-4	UInt64
-7	None
-0	0	\N	\N	\N	\N	0
-1	1	\N	\N	\N	\N	0
-2	2	\N	\N	\N	\N	0
-3	3	3	\N	3	\N	0
-4	4	4	\N	4	\N	0
-5	5	5	\N	5	\N	0
-6	6	str_6	str_6	\N	\N	0
-7	7	str_7	str_7	\N	\N	0
-8	8	str_8	str_8	\N	\N	0
-9	9	\N	\N	\N	\N	0
-10	10	\N	\N	\N	\N	0
-11	11	\N	\N	\N	\N	0
-12	12	12	\N	12	\N	0
-13	13	str_13	str_13	\N	\N	0
-14	14	\N	\N	\N	\N	0
-insert nested dynamic
-3	Array(Dynamic)
-4	String
-4	UInt64
-7	None
-0	0	\N	\N	\N	\N	0	[]	[]	[]
-1	1	\N	\N	\N	\N	0	[]	[]	[]
-2	2	\N	\N	\N	\N	0	[]	[]	[]
-3	3	3	\N	3	\N	0	[]	[]	[]
-4	4	4	\N	4	\N	0	[]	[]	[]
-5	5	5	\N	5	\N	0	[]	[]	[]
-6	6	str_6	str_6	\N	\N	0	[]	[]	[]
-7	7	str_7	str_7	\N	\N	0	[]	[]	[]
-8	8	str_8	str_8	\N	\N	0	[]	[]	[]
-9	9	\N	\N	\N	\N	0	[]	[]	[]
-10	10	\N	\N	\N	\N	0	[]	[]	[]
-11	11	\N	\N	\N	\N	0	[]	[]	[]
-12	12	12	\N	12	\N	0	[]	[]	[]
-13	13	str_13	str_13	\N	\N	0	[]	[]	[]
-14	14	\N	\N	\N	\N	0	[]	[]	[]
-15	15	[15]	\N	\N	\N	0	[15]	[NULL]	[NULL]
-16	16	['str_16']	\N	\N	\N	0	[NULL]	['str_16']	[NULL]
-17	17	[17]	\N	\N	\N	0	[17]	[NULL]	[NULL]
-alter rename column 2
-3	Array(Dynamic)
-4	String
-4	UInt64
-7	None
-0	0	\N	\N	\N	\N	0	[]	[]	[]
-1	1	\N	\N	\N	\N	0	[]	[]	[]
-2	2	\N	\N	\N	\N	0	[]	[]	[]
-3	3	3	\N	3	\N	0	[]	[]	[]
-4	4	4	\N	4	\N	0	[]	[]	[]
-5	5	5	\N	5	\N	0	[]	[]	[]
-6	6	str_6	str_6	\N	\N	0	[]	[]	[]
-7	7	str_7	str_7	\N	\N	0	[]	[]	[]
-8	8	str_8	str_8	\N	\N	0	[]	[]	[]
-9	9	\N	\N	\N	\N	0	[]	[]	[]
-10	10	\N	\N	\N	\N	0	[]	[]	[]
-11	11	\N	\N	\N	\N	0	[]	[]	[]
-12	12	12	\N	12	\N	0	[]	[]	[]
-13	13	str_13	str_13	\N	\N	0	[]	[]	[]
-14	14	\N	\N	\N	\N	0	[]	[]	[]
-15	15	[15]	\N	\N	\N	0	[15]	[NULL]	[NULL]
-16	16	['str_16']	\N	\N	\N	0	[NULL]	['str_16']	[NULL]
-17	17	[17]	\N	\N	\N	0	[17]	[NULL]	[NULL]
diff --git a/tests/queries/0_stateless/03040_dynamic_type_alters_2.sh b/tests/queries/0_stateless/03040_dynamic_type_alters_2.sh
deleted file mode 100755
index 6491e64372f..00000000000
--- a/tests/queries/0_stateless/03040_dynamic_type_alters_2.sh
+++ /dev/null
@@ -1,57 +0,0 @@
-#!/usr/bin/env bash
-# Tags: long
-
-CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# reset --log_comment
-CLICKHOUSE_LOG_COMMENT=
-# shellcheck source=../shell_config.sh
-. "$CUR_DIR"/../shell_config.sh
-
-CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_dynamic_type=1 --allow_experimental_variant_type=1 --use_variant_as_common_type=1"
-
-function run()
-{
-    echo "initial insert"
-    $CH_CLIENT -q "insert into test select number, number from numbers(3)"
-
-    echo "alter add column"
-    $CH_CLIENT -q "alter table test add column d Dynamic settings mutations_sync=1"
-    $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)"
-    $CH_CLIENT -q "select x, y, d, d.String, d.UInt64, d.\`Tuple(a UInt64)\`.a from test order by x"
-
-    echo "insert after alter add column 1"
-    $CH_CLIENT -q "insert into test select number, number, number from numbers(3, 3)"
-    $CH_CLIENT -q "insert into test select number, number, 'str_' || toString(number) from numbers(6, 3)"
-    $CH_CLIENT -q "insert into test select number, number, NULL from numbers(9, 3)"
-    $CH_CLIENT -q "insert into test select number, number, multiIf(number % 3 == 0, number, number % 3 == 1, 'str_' || toString(number), NULL) from numbers(12, 3)"
-    $CH_CLIENT -q "select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d)"
-    $CH_CLIENT -q "select x, y, d, d.String, d.UInt64, d.Date, d.\`Tuple(a UInt64)\`.a from test order by x"
-
-    echo "alter rename column 1"
-    $CH_CLIENT -q "alter table test rename column d to d1 settings mutations_sync=1"
-    $CH_CLIENT -q "select count(), dynamicType(d1) from test group by dynamicType(d1) order by count(), dynamicType(d1)"
-    $CH_CLIENT -q "select x, y, d1, d1.String, d1.UInt64, d1.Date, d1.\`Tuple(a UInt64)\`.a from test order by x"
-
-    echo "insert nested dynamic"
-    $CH_CLIENT -q "insert into test select number, number, [number % 2 ? number : 'str_' || toString(number)]::Array(Dynamic) from numbers(15, 3)"
-    $CH_CLIENT -q "select count(), dynamicType(d1) from test group by dynamicType(d1) order by count(), dynamicType(d1)"
-    $CH_CLIENT -q "select x, y, d1, d1.String, d1.UInt64, d1.Date, d1.\`Tuple(a UInt64)\`.a, d1.\`Array(Dynamic)\`.UInt64, d1.\`Array(Dynamic)\`.String, d1.\`Array(Dynamic)\`.Date from test order by x"
-
-    echo "alter rename column 2"
-    $CH_CLIENT -q "alter table test rename column d1 to d2 settings mutations_sync=1"
-    $CH_CLIENT -q "select count(), dynamicType(d2) from test group by dynamicType(d2) order by count(), dynamicType(d2)"
-    $CH_CLIENT -q "select x, y, d2, d2.String, d2.UInt64, d2.Date, d2.\`Tuple(a UInt64)\`.a, d2.\`Array(Dynamic)\`.UInt64, d2.\`Array(Dynamic)\`.String, d2.\`Array(Dynamic)\`.Date, from test order by x"
-}
-
-$CH_CLIENT -q "drop table if exists test;"
-
-echo "MergeTree compact"
-$CH_CLIENT -q "create table test (x UInt64, y UInt64) engine=MergeTree order by x settings min_rows_for_wide_part=100000000, min_bytes_for_wide_part=1000000000;"
-run
-$CH_CLIENT -q "drop table test;"
-
-echo "MergeTree wide"
-$CH_CLIENT -q "create table test (x UInt64, y UInt64 ) engine=MergeTree order by x settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1;"
-run
-$CH_CLIENT -q "drop table test;"
-
diff --git a/tests/queries/0_stateless/03040_dynamic_type_alters_2_compact_merge_tree.reference b/tests/queries/0_stateless/03040_dynamic_type_alters_2_compact_merge_tree.reference
new file mode 100644
index 00000000000..a2f2a19805d
--- /dev/null
+++ b/tests/queries/0_stateless/03040_dynamic_type_alters_2_compact_merge_tree.reference
@@ -0,0 +1,90 @@
+initial insert
+alter add column
+3	None
+0	0	\N	\N	\N	0
+1	1	\N	\N	\N	0
+2	2	\N	\N	\N	0
+insert after alter add column 1
+4	String
+4	UInt64
+7	None
+0	0	\N	\N	\N	\N	0
+1	1	\N	\N	\N	\N	0
+2	2	\N	\N	\N	\N	0
+3	3	3	\N	3	\N	0
+4	4	4	\N	4	\N	0
+5	5	5	\N	5	\N	0
+6	6	str_6	str_6	\N	\N	0
+7	7	str_7	str_7	\N	\N	0
+8	8	str_8	str_8	\N	\N	0
+9	9	\N	\N	\N	\N	0
+10	10	\N	\N	\N	\N	0
+11	11	\N	\N	\N	\N	0
+12	12	12	\N	12	\N	0
+13	13	str_13	str_13	\N	\N	0
+14	14	\N	\N	\N	\N	0
+alter rename column 1
+4	String
+4	UInt64
+7	None
+0	0	\N	\N	\N	\N	0
+1	1	\N	\N	\N	\N	0
+2	2	\N	\N	\N	\N	0
+3	3	3	\N	3	\N	0
+4	4	4	\N	4	\N	0
+5	5	5	\N	5	\N	0
+6	6	str_6	str_6	\N	\N	0
+7	7	str_7	str_7	\N	\N	0
+8	8	str_8	str_8	\N	\N	0
+9	9	\N	\N	\N	\N	0
+10	10	\N	\N	\N	\N	0
+11	11	\N	\N	\N	\N	0
+12	12	12	\N	12	\N	0
+13	13	str_13	str_13	\N	\N	0
+14	14	\N	\N	\N	\N	0
+insert nested dynamic
+3	Array(Dynamic)
+4	String
+4	UInt64
+7	None
+0	0	\N	\N	\N	\N	0	[]	[]	[]
+1	1	\N	\N	\N	\N	0	[]	[]	[]
+2	2	\N	\N	\N	\N	0	[]	[]	[]
+3	3	3	\N	3	\N	0	[]	[]	[]
+4	4	4	\N	4	\N	0	[]	[]	[]
+5	5	5	\N	5	\N	0	[]	[]	[]
+6	6	str_6	str_6	\N	\N	0	[]	[]	[]
+7	7	str_7	str_7	\N	\N	0	[]	[]	[]
+8	8	str_8	str_8	\N	\N	0	[]	[]	[]
+9	9	\N	\N	\N	\N	0	[]	[]	[]
+10	10	\N	\N	\N	\N	0	[]	[]	[]
+11	11	\N	\N	\N	\N	0	[]	[]	[]
+12	12	12	\N	12	\N	0	[]	[]	[]
+13	13	str_13	str_13	\N	\N	0	[]	[]	[]
+14	14	\N	\N	\N	\N	0	[]	[]	[]
+15	15	[15]	\N	\N	\N	0	[15]	[NULL]	[NULL]
+16	16	['str_16']	\N	\N	\N	0	[NULL]	['str_16']	[NULL]
+17	17	[17]	\N	\N	\N	0	[17]	[NULL]	[NULL]
+alter rename column 2
+3	Array(Dynamic)
+4	String
+4	UInt64
+7	None
+0	0	\N	\N	\N	\N	0	[]	[]	[]
+1	1	\N	\N	\N	\N	0	[]	[]	[]
+2	2	\N	\N	\N	\N	0	[]	[]	[]
+3	3	3	\N	3	\N	0	[]	[]	[]
+4	4	4	\N	4	\N	0	[]	[]	[]
+5	5	5	\N	5	\N	0	[]	[]	[]
+6	6	str_6	str_6	\N	\N	0	[]	[]	[]
+7	7	str_7	str_7	\N	\N	0	[]	[]	[]
+8	8	str_8	str_8	\N	\N	0	[]	[]	[]
+9	9	\N	\N	\N	\N	0	[]	[]	[]
+10	10	\N	\N	\N	\N	0	[]	[]	[]
+11	11	\N	\N	\N	\N	0	[]	[]	[]
+12	12	12	\N	12	\N	0	[]	[]	[]
+13	13	str_13	str_13	\N	\N	0	[]	[]	[]
+14	14	\N	\N	\N	\N	0	[]	[]	[]
+15	15	[15]	\N	\N	\N	0	[15]	[NULL]	[NULL]
+16	16	['str_16']	\N	\N	\N	0	[NULL]	['str_16']	[NULL]
+17	17	[17]	\N	\N	\N	0	[17]	[NULL]	[NULL]
diff --git a/tests/queries/0_stateless/03040_dynamic_type_alters_2_compact_merge_tree.sql b/tests/queries/0_stateless/03040_dynamic_type_alters_2_compact_merge_tree.sql
new file mode 100644
index 00000000000..cead110dd7d
--- /dev/null
+++ b/tests/queries/0_stateless/03040_dynamic_type_alters_2_compact_merge_tree.sql
@@ -0,0 +1,39 @@
+set allow_experimental_dynamic_type = 1;
+set allow_experimental_variant_type = 1;
+set use_variant_as_common_type = 1;
+
+drop table if exists test;
+create table test (x UInt64, y UInt64) engine=MergeTree order by x settings min_rows_for_wide_part=100000000, min_bytes_for_wide_part=1000000000;
+
+select 'initial insert';
+insert into test select number, number from numbers(3);
+
+select 'alter add column';
+alter table test add column d Dynamic settings mutations_sync=1;
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+select x, y, d, d.String, d.UInt64, d.`Tuple(a UInt64)`.a from test order by x;
+
+select 'insert after alter add column 1';
+insert into test select number, number, number from numbers(3, 3);
+insert into test select number, number, 'str_' || toString(number) from numbers(6, 3);
+insert into test select number, number, NULL from numbers(9, 3);
+insert into test select number, number, multiIf(number % 3 == 0, number, number % 3 == 1, 'str_' || toString(number), NULL) from numbers(12, 3);
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+select x, y, d, d.String, d.UInt64, d.Date, d.`Tuple(a UInt64)`.a from test order by x;
+
+select 'alter rename column 1';
+alter table test rename column d to d1 settings mutations_sync=1;
+select count(), dynamicType(d1) from test group by dynamicType(d1) order by count(), dynamicType(d1);
+select x, y, d1, d1.String, d1.UInt64, d1.Date, d1.`Tuple(a UInt64)`.a from test order by x;
+
+select 'insert nested dynamic';
+insert into test select number, number, [number % 2 ? number : 'str_' || toString(number)]::Array(Dynamic) from numbers(15, 3);
+select count(), dynamicType(d1) from test group by dynamicType(d1) order by count(), dynamicType(d1);
+select x, y, d1, d1.String, d1.UInt64, d1.Date, d1.`Tuple(a UInt64)`.a, d1.`Array(Dynamic)`.UInt64, d1.`Array(Dynamic)`.String, d1.`Array(Dynamic)`.Date from test order by x;
+
+select 'alter rename column 2';
+alter table test rename column d1 to d2 settings mutations_sync=1;
+select count(), dynamicType(d2) from test group by dynamicType(d2) order by count(), dynamicType(d2);
+select x, y, d2, d2.String, d2.UInt64, d2.Date, d2.`Tuple(a UInt64)`.a, d2.`Array(Dynamic)`.UInt64, d2.`Array(Dynamic)`.String, d2.`Array(Dynamic)`.Date, from test order by x;
+
+drop table test;
diff --git a/tests/queries/0_stateless/03040_dynamic_type_alters_2_wide_merge_tree.reference b/tests/queries/0_stateless/03040_dynamic_type_alters_2_wide_merge_tree.reference
new file mode 100644
index 00000000000..a2f2a19805d
--- /dev/null
+++ b/tests/queries/0_stateless/03040_dynamic_type_alters_2_wide_merge_tree.reference
@@ -0,0 +1,90 @@
+initial insert
+alter add column
+3	None
+0	0	\N	\N	\N	0
+1	1	\N	\N	\N	0
+2	2	\N	\N	\N	0
+insert after alter add column 1
+4	String
+4	UInt64
+7	None
+0	0	\N	\N	\N	\N	0
+1	1	\N	\N	\N	\N	0
+2	2	\N	\N	\N	\N	0
+3	3	3	\N	3	\N	0
+4	4	4	\N	4	\N	0
+5	5	5	\N	5	\N	0
+6	6	str_6	str_6	\N	\N	0
+7	7	str_7	str_7	\N	\N	0
+8	8	str_8	str_8	\N	\N	0
+9	9	\N	\N	\N	\N	0
+10	10	\N	\N	\N	\N	0
+11	11	\N	\N	\N	\N	0
+12	12	12	\N	12	\N	0
+13	13	str_13	str_13	\N	\N	0
+14	14	\N	\N	\N	\N	0
+alter rename column 1
+4	String
+4	UInt64
+7	None
+0	0	\N	\N	\N	\N	0
+1	1	\N	\N	\N	\N	0
+2	2	\N	\N	\N	\N	0
+3	3	3	\N	3	\N	0
+4	4	4	\N	4	\N	0
+5	5	5	\N	5	\N	0
+6	6	str_6	str_6	\N	\N	0
+7	7	str_7	str_7	\N	\N	0
+8	8	str_8	str_8	\N	\N	0
+9	9	\N	\N	\N	\N	0
+10	10	\N	\N	\N	\N	0
+11	11	\N	\N	\N	\N	0
+12	12	12	\N	12	\N	0
+13	13	str_13	str_13	\N	\N	0
+14	14	\N	\N	\N	\N	0
+insert nested dynamic
+3	Array(Dynamic)
+4	String
+4	UInt64
+7	None
+0	0	\N	\N	\N	\N	0	[]	[]	[]
+1	1	\N	\N	\N	\N	0	[]	[]	[]
+2	2	\N	\N	\N	\N	0	[]	[]	[]
+3	3	3	\N	3	\N	0	[]	[]	[]
+4	4	4	\N	4	\N	0	[]	[]	[]
+5	5	5	\N	5	\N	0	[]	[]	[]
+6	6	str_6	str_6	\N	\N	0	[]	[]	[]
+7	7	str_7	str_7	\N	\N	0	[]	[]	[]
+8	8	str_8	str_8	\N	\N	0	[]	[]	[]
+9	9	\N	\N	\N	\N	0	[]	[]	[]
+10	10	\N	\N	\N	\N	0	[]	[]	[]
+11	11	\N	\N	\N	\N	0	[]	[]	[]
+12	12	12	\N	12	\N	0	[]	[]	[]
+13	13	str_13	str_13	\N	\N	0	[]	[]	[]
+14	14	\N	\N	\N	\N	0	[]	[]	[]
+15	15	[15]	\N	\N	\N	0	[15]	[NULL]	[NULL]
+16	16	['str_16']	\N	\N	\N	0	[NULL]	['str_16']	[NULL]
+17	17	[17]	\N	\N	\N	0	[17]	[NULL]	[NULL]
+alter rename column 2
+3	Array(Dynamic)
+4	String
+4	UInt64
+7	None
+0	0	\N	\N	\N	\N	0	[]	[]	[]
+1	1	\N	\N	\N	\N	0	[]	[]	[]
+2	2	\N	\N	\N	\N	0	[]	[]	[]
+3	3	3	\N	3	\N	0	[]	[]	[]
+4	4	4	\N	4	\N	0	[]	[]	[]
+5	5	5	\N	5	\N	0	[]	[]	[]
+6	6	str_6	str_6	\N	\N	0	[]	[]	[]
+7	7	str_7	str_7	\N	\N	0	[]	[]	[]
+8	8	str_8	str_8	\N	\N	0	[]	[]	[]
+9	9	\N	\N	\N	\N	0	[]	[]	[]
+10	10	\N	\N	\N	\N	0	[]	[]	[]
+11	11	\N	\N	\N	\N	0	[]	[]	[]
+12	12	12	\N	12	\N	0	[]	[]	[]
+13	13	str_13	str_13	\N	\N	0	[]	[]	[]
+14	14	\N	\N	\N	\N	0	[]	[]	[]
+15	15	[15]	\N	\N	\N	0	[15]	[NULL]	[NULL]
+16	16	['str_16']	\N	\N	\N	0	[NULL]	['str_16']	[NULL]
+17	17	[17]	\N	\N	\N	0	[17]	[NULL]	[NULL]
diff --git a/tests/queries/0_stateless/03040_dynamic_type_alters_2_wide_merge_tree.sql b/tests/queries/0_stateless/03040_dynamic_type_alters_2_wide_merge_tree.sql
new file mode 100644
index 00000000000..f58599b1d61
--- /dev/null
+++ b/tests/queries/0_stateless/03040_dynamic_type_alters_2_wide_merge_tree.sql
@@ -0,0 +1,39 @@
+set allow_experimental_dynamic_type = 1;
+set allow_experimental_variant_type = 1;
+set use_variant_as_common_type = 1;
+
+drop table if exists test;
+create table test (x UInt64, y UInt64) engine=MergeTree order by x settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1;
+
+select 'initial insert';
+insert into test select number, number from numbers(3);
+
+select 'alter add column';
+alter table test add column d Dynamic settings mutations_sync=1;
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+select x, y, d, d.String, d.UInt64, d.`Tuple(a UInt64)`.a from test order by x;
+
+select 'insert after alter add column 1';
+insert into test select number, number, number from numbers(3, 3);
+insert into test select number, number, 'str_' || toString(number) from numbers(6, 3);
+insert into test select number, number, NULL from numbers(9, 3);
+insert into test select number, number, multiIf(number % 3 == 0, number, number % 3 == 1, 'str_' || toString(number), NULL) from numbers(12, 3);
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+select x, y, d, d.String, d.UInt64, d.Date, d.`Tuple(a UInt64)`.a from test order by x;
+
+select 'alter rename column 1';
+alter table test rename column d to d1 settings mutations_sync=1;
+select count(), dynamicType(d1) from test group by dynamicType(d1) order by count(), dynamicType(d1);
+select x, y, d1, d1.String, d1.UInt64, d1.Date, d1.`Tuple(a UInt64)`.a from test order by x;
+
+select 'insert nested dynamic';
+insert into test select number, number, [number % 2 ? number : 'str_' || toString(number)]::Array(Dynamic) from numbers(15, 3);
+select count(), dynamicType(d1) from test group by dynamicType(d1) order by count(), dynamicType(d1);
+select x, y, d1, d1.String, d1.UInt64, d1.Date, d1.`Tuple(a UInt64)`.a, d1.`Array(Dynamic)`.UInt64, d1.`Array(Dynamic)`.String, d1.`Array(Dynamic)`.Date from test order by x;
+
+select 'alter rename column 2';
+alter table test rename column d1 to d2 settings mutations_sync=1;
+select count(), dynamicType(d2) from test group by dynamicType(d2) order by count(), dynamicType(d2);
+select x, y, d2, d2.String, d2.UInt64, d2.Date, d2.`Tuple(a UInt64)`.a, d2.`Array(Dynamic)`.UInt64, d2.`Array(Dynamic)`.String, d2.`Array(Dynamic)`.Date, from test order by x;
+
+drop table test;

From 6a3f49ada4d8319edf34fdf91d5eee007df48e26 Mon Sep 17 00:00:00 2001
From: Linh Giang <165205637+linhgiang24@users.noreply.github.com>
Date: Tue, 23 Jul 2024 13:23:35 -0600
Subject: [PATCH 0666/1488] Included "ON CLUSTER cluster_name" syntax for
 RELOAD DICTIONARIES command

Included "ON CLUSTER cluster_name" syntax for RELOAD DICTIONARY and RELOAD DICTIONARIES commands to ensure execution on all replicas.  Also to conform with the rest of the document.
---
 docs/en/sql-reference/statements/system.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/docs/en/sql-reference/statements/system.md b/docs/en/sql-reference/statements/system.md
index e6d3439d2b9..f4780fd41c1 100644
--- a/docs/en/sql-reference/statements/system.md
+++ b/docs/en/sql-reference/statements/system.md
@@ -18,6 +18,12 @@ Reloads all dictionaries that have been successfully loaded before.
 By default, dictionaries are loaded lazily (see [dictionaries_lazy_load](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-dictionaries_lazy_load)), so instead of being loaded automatically at startup, they are initialized on first access through dictGet function or SELECT from tables with ENGINE = Dictionary. The `SYSTEM RELOAD DICTIONARIES` query reloads such dictionaries (LOADED).
 Always returns `Ok.` regardless of the result of the dictionary update.
 
+**Syntax**
+
+```sql
+SYSTEM RELOAD DICTIONARIES [ON CLUSTER cluster_name]
+```
+
 ## RELOAD DICTIONARY
 
 Completely reloads a dictionary `dictionary_name`, regardless of the state of the dictionary (LOADED / NOT_LOADED / FAILED).
@@ -25,6 +31,8 @@ Always returns `Ok.` regardless of the result of updating the dictionary.
 The status of the dictionary can be checked by querying the `system.dictionaries` table.
 
 ``` sql
+SYSTEM RELOAD DICTIONARY [ON CLUSTER cluster_name] dictionary_name
+
 SELECT name, status FROM system.dictionaries;
 ```
 

From 190421aafa83bf54c3921ae17c91cbad8c28bd1d Mon Sep 17 00:00:00 2001
From: Nikita Taranov <nikita.taranov@clickhouse.com>
Date: Tue, 23 Jul 2024 20:38:54 +0100
Subject: [PATCH 0667/1488] impl

---
 .../01508_partition_pruning_long.sh           |  30 -----
 ... 01508_partition_pruning_long_1.reference} | 119 ------------------
 .../01508_partition_pruning_long_1.sh         |  79 ++++++++++++
 .../01508_partition_pruning_long_2.reference  | 119 ++++++++++++++++++
 ...ries => 01508_partition_pruning_long_2.sh} |  80 +++++-------
 5 files changed, 228 insertions(+), 199 deletions(-)
 delete mode 100755 tests/queries/0_stateless/01508_partition_pruning_long.sh
 rename tests/queries/0_stateless/{01508_partition_pruning_long.reference => 01508_partition_pruning_long_1.reference} (50%)
 create mode 100755 tests/queries/0_stateless/01508_partition_pruning_long_1.sh
 create mode 100644 tests/queries/0_stateless/01508_partition_pruning_long_2.reference
 rename tests/queries/0_stateless/{01508_partition_pruning_long.queries => 01508_partition_pruning_long_2.sh} (58%)
 mode change 100644 => 100755

diff --git a/tests/queries/0_stateless/01508_partition_pruning_long.sh b/tests/queries/0_stateless/01508_partition_pruning_long.sh
deleted file mode 100755
index 7b56d8bbf03..00000000000
--- a/tests/queries/0_stateless/01508_partition_pruning_long.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/usr/bin/env bash
-# Tags: long, no-polymorphic-parts, no-random-settings, no-random-merge-tree-settings, no-debug
-
-# Description of test result:
-# Test the correctness of the partition pruning
-#
-# Script executes queries from a file 01508_partition_pruning_long.queries (1 line = 1 query)
-# Queries are started with 'select' (but NOT with 'SELECT') are executed with log_level=debug
-
-CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# shellcheck source=../shell_config.sh
-. "$CURDIR"/../shell_config.sh
-
-
-queries="${CURDIR}/01508_partition_pruning_long.queries"
-while IFS= read -r sql
-do
-  [ -z "$sql" ] && continue
-  if [[ "$sql" == select* ]] ;
-  then
-    echo "$sql"
-    ${CLICKHOUSE_CLIENT} --query "$sql"
-    CLICKHOUSE_CLIENT=$(echo ${CLICKHOUSE_CLIENT} | sed 's/'"--send_logs_level=${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL}"'/--send_logs_level=debug/g')
-    ${CLICKHOUSE_CLIENT} --query "$sql" 2>&1 | grep -oh "Selected .* parts by partition key, *. parts by primary key, .* marks by primary key, .* marks to read from .* ranges.*$"
-    CLICKHOUSE_CLIENT=$(echo ${CLICKHOUSE_CLIENT} | sed 's/--send_logs_level=debug/'"--send_logs_level=${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL}"'/g')
-    echo ""
-  else
-    ${CLICKHOUSE_CLIENT} --query "$sql"
-  fi
-done < "$queries"
diff --git a/tests/queries/0_stateless/01508_partition_pruning_long.reference b/tests/queries/0_stateless/01508_partition_pruning_long_1.reference
similarity index 50%
rename from tests/queries/0_stateless/01508_partition_pruning_long.reference
rename to tests/queries/0_stateless/01508_partition_pruning_long_1.reference
index afdb4257505..3ea4cc4f6ee 100644
--- a/tests/queries/0_stateless/01508_partition_pruning_long.reference
+++ b/tests/queries/0_stateless/01508_partition_pruning_long_1.reference
@@ -123,122 +123,3 @@ select uniqExact(_part), count() from tMM where toYYYYMM(d) between 202009 and 2
 2	20000
 Selected 2/3 parts by partition key, 2 parts by primary key, 2/2 marks by primary key, 2 marks to read from 2 ranges
 
---------- tDD ----------------------------
-select uniqExact(_part), count() from tDD where toDate(d)=toDate('2020-09-24');
-1	10000
-Selected 1/4 parts by partition key, 1 parts by primary key, 1/1 marks by primary key, 1 marks to read from 1 ranges
-
-select uniqExact(_part), count() FROM tDD WHERE toDate(d) = toDate('2020-09-24');
-1	10000
-Selected 1/4 parts by partition key, 1 parts by primary key, 1/1 marks by primary key, 1 marks to read from 1 ranges
-
-select uniqExact(_part), count() FROM tDD WHERE toDate(d) = '2020-09-24';
-1	10000
-Selected 1/4 parts by partition key, 1 parts by primary key, 1/1 marks by primary key, 1 marks to read from 1 ranges
-
-select uniqExact(_part), count() FROM tDD WHERE toDate(d) >= '2020-09-23' and toDate(d) <= '2020-09-26';
-3	40000
-Selected 3/4 parts by partition key, 3 parts by primary key, 4/4 marks by primary key, 4 marks to read from 3 ranges
-
-select uniqExact(_part), count() FROM tDD WHERE toYYYYMMDD(d) >= 20200923 and toDate(d) <= '2020-09-26';
-3	40000
-Selected 3/4 parts by partition key, 3 parts by primary key, 4/4 marks by primary key, 4 marks to read from 3 ranges
-
---------- sDD ----------------------------
-select uniqExact(_part), count() from sDD;
-6	30000
-Selected 6/6 parts by partition key, 6 parts by primary key, 6/6 marks by primary key, 6 marks to read from 6 ranges
-
-select uniqExact(_part), count() from sDD where toYYYYMM(toDateTime(intDiv(d,1000),'UTC')-1)+1 = 202010;
-3	9999
-Selected 3/6 parts by partition key, 3 parts by primary key, 3/3 marks by primary key, 3 marks to read from 3 ranges
-
-select uniqExact(_part), count() from sDD where toYYYYMM(toDateTime(intDiv(d,1000),'UTC')-1) = 202010;
-2	9999
-Selected 2/6 parts by partition key, 2 parts by primary key, 2/2 marks by primary key, 2 marks to read from 2 ranges
-
-select uniqExact(_part), count() from sDD where toYYYYMM(toDateTime(intDiv(d,1000),'UTC')-1) = 202110;
-0	0
-Selected 0/6 parts by partition key, 0 parts by primary key, 0/0 marks by primary key, 0 marks to read from 0 ranges
-
-select uniqExact(_part), count() from sDD where toYYYYMM(toDateTime(intDiv(d,1000),'UTC'))+1 > 202009 and toStartOfDay(toDateTime(intDiv(d,1000),'UTC')) < toDateTime('2020-10-02 00:00:00','UTC');
-3	11440
-Selected 3/6 parts by partition key, 3 parts by primary key, 3/3 marks by primary key, 3 marks to read from 3 ranges
-
-select uniqExact(_part), count() from sDD where toYYYYMM(toDateTime(intDiv(d,1000),'UTC'))+1 > 202009 and toDateTime(intDiv(d,1000),'UTC') < toDateTime('2020-10-01 00:00:00','UTC');
-2	10000
-Selected 2/6 parts by partition key, 2 parts by primary key, 2/2 marks by primary key, 2 marks to read from 2 ranges
-
-select uniqExact(_part), count() from sDD where d >= 1598918400000;
-4	20000
-Selected 4/6 parts by partition key, 4 parts by primary key, 4/4 marks by primary key, 4 marks to read from 4 ranges
-
-select uniqExact(_part), count() from sDD where d >= 1598918400000 and toYYYYMM(toDateTime(intDiv(d,1000),'UTC')-1) < 202010;
-3	10001
-Selected 3/6 parts by partition key, 3 parts by primary key, 3/3 marks by primary key, 3 marks to read from 3 ranges
-
---------- xMM ----------------------------
-select uniqExact(_part), count() from xMM where toStartOfDay(d) >= '2020-10-01 00:00:00';
-2	10000
-Selected 2/6 parts by partition key, 2 parts by primary key, 2/2 marks by primary key, 2 marks to read from 2 ranges
-
-select uniqExact(_part), count() from xMM where d >= '2020-09-01 00:00:00' and d <= '2020-10-01 00:00:00';
-3	10001
-Selected 3/6 parts by partition key, 3 parts by primary key, 3/3 marks by primary key, 3 marks to read from 3 ranges
-
-select uniqExact(_part), count() from xMM where d >= '2020-09-01 00:00:00' and d < '2020-10-01 00:00:00';
-2	10000
-Selected 2/6 parts by partition key, 2 parts by primary key, 2/2 marks by primary key, 2 marks to read from 2 ranges
-
-select uniqExact(_part), count() from xMM where d >= '2020-09-01 00:00:00' and d <= '2020-10-01 00:00:00' and a=1;
-1	1
-Selected 1/6 parts by partition key, 1 parts by primary key, 1/1 marks by primary key, 1 marks to read from 1 ranges
-
-select uniqExact(_part), count() from xMM where d >= '2020-09-01 00:00:00' and d <= '2020-10-01 00:00:00' and a<>3;
-2	5001
-Selected 2/6 parts by partition key, 2 parts by primary key, 2/2 marks by primary key, 2 marks to read from 2 ranges
-
-select uniqExact(_part), count() from xMM where d >= '2020-09-01 00:00:00' and d < '2020-10-01 00:00:00' and a<>3;
-1	5000
-Selected 1/6 parts by partition key, 1 parts by primary key, 1/1 marks by primary key, 1 marks to read from 1 ranges
-
-select uniqExact(_part), count() from xMM where d >= '2020-09-01 00:00:00' and d < '2020-11-01 00:00:00' and a = 1;
-2	10000
-Selected 2/6 parts by partition key, 2 parts by primary key, 2/2 marks by primary key, 2 marks to read from 2 ranges
-
-select uniqExact(_part), count() from xMM where a = 1;
-3	15000
-Selected 3/6 parts by partition key, 3 parts by primary key, 3/3 marks by primary key, 3 marks to read from 3 ranges
-
-select uniqExact(_part), count() from xMM where a = 66;
-0	0
-Selected 0/6 parts by partition key, 0 parts by primary key, 0/0 marks by primary key, 0 marks to read from 0 ranges
-
-select uniqExact(_part), count() from xMM where a <> 66;
-6	30000
-Selected 6/6 parts by partition key, 6 parts by primary key, 6/6 marks by primary key, 6 marks to read from 6 ranges
-
-select uniqExact(_part), count() from xMM where a = 2;
-2	10000
-Selected 2/6 parts by partition key, 2 parts by primary key, 2/2 marks by primary key, 2 marks to read from 2 ranges
-
-select uniqExact(_part), count() from xMM where a = 1;
-2	15000
-Selected 2/5 parts by partition key, 2 parts by primary key, 2/2 marks by primary key, 2 marks to read from 2 ranges
-
-select uniqExact(_part), count() from xMM where toStartOfDay(d) >= '2020-10-01 00:00:00';
-1	10000
-Selected 1/5 parts by partition key, 1 parts by primary key, 1/1 marks by primary key, 1 marks to read from 1 ranges
-
-select uniqExact(_part), count() from xMM where a <> 66;
-5	30000
-Selected 5/5 parts by partition key, 5 parts by primary key, 5/5 marks by primary key, 5 marks to read from 5 ranges
-
-select uniqExact(_part), count() from xMM where d >= '2020-09-01 00:00:00' and d <= '2020-10-01 00:00:00' and a<>3;
-2	5001
-Selected 2/5 parts by partition key, 2 parts by primary key, 2/2 marks by primary key, 2 marks to read from 2 ranges
-
-select uniqExact(_part), count() from xMM where d >= '2020-09-01 00:00:00' and d < '2020-10-01 00:00:00' and a<>3;
-1	5000
-Selected 1/5 parts by partition key, 1 parts by primary key, 1/1 marks by primary key, 1 marks to read from 1 ranges
-
diff --git a/tests/queries/0_stateless/01508_partition_pruning_long_1.sh b/tests/queries/0_stateless/01508_partition_pruning_long_1.sh
new file mode 100755
index 00000000000..512cf8f5265
--- /dev/null
+++ b/tests/queries/0_stateless/01508_partition_pruning_long_1.sh
@@ -0,0 +1,79 @@
+#!/usr/bin/env bash
+# Tags: long, no-polymorphic-parts, no-random-settings, no-random-merge-tree-settings, no-debug
+
+# Description of test result:
+# Test the correctness of the partition pruning
+#
+# Script executes queries from a file 01508_partition_pruning_long.queries (1 line = 1 query)
+# Queries are started with 'select' (but NOT with 'SELECT') are executed with log_level=debug
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+
+while IFS= read -r sql
+do
+  [ -z "$sql" ] && continue
+  if [[ "$sql" == select* ]] ;
+  then
+    echo "$sql"
+    ${CLICKHOUSE_CLIENT} --query "$sql"
+    CLICKHOUSE_CLIENT=$(echo ${CLICKHOUSE_CLIENT} | sed 's/'"--send_logs_level=${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL}"'/--send_logs_level=debug/g')
+    ${CLICKHOUSE_CLIENT} --query "$sql" 2>&1 | grep -oh "Selected .* parts by partition key, *. parts by primary key, .* marks by primary key, .* marks to read from .* ranges.*$"
+    CLICKHOUSE_CLIENT=$(echo ${CLICKHOUSE_CLIENT} | sed 's/--send_logs_level=debug/'"--send_logs_level=${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL}"'/g')
+    echo ""
+  else
+    ${CLICKHOUSE_CLIENT} --query "$sql"
+  fi
+done <<< "
+DROP TABLE IF EXISTS tMM;
+
+CREATE TABLE tMM(d DateTime('Asia/Istanbul'), a Int64) ENGINE = MergeTree PARTITION BY toYYYYMM(d) ORDER BY tuple() SETTINGS index_granularity = 8192;
+SYSTEM STOP MERGES tMM;
+INSERT INTO tMM SELECT toDateTime('2020-08-16 00:00:00', 'Asia/Istanbul') + number*60, number FROM numbers(5000);
+INSERT INTO tMM SELECT toDateTime('2020-08-16 00:00:00', 'Asia/Istanbul') + number*60, number FROM numbers(5000);
+INSERT INTO tMM SELECT toDateTime('2020-09-01 00:00:00', 'Asia/Istanbul') + number*60, number FROM numbers(5000);
+INSERT INTO tMM SELECT toDateTime('2020-09-01 00:00:00', 'Asia/Istanbul') + number*60, number FROM numbers(5000);
+INSERT INTO tMM SELECT toDateTime('2020-10-01 00:00:00', 'Asia/Istanbul') + number*60, number FROM numbers(5000);
+INSERT INTO tMM SELECT toDateTime('2020-10-15 00:00:00', 'Asia/Istanbul') + number*60, number FROM numbers(5000);
+
+SELECT '--------- tMM ----------------------------';
+select uniqExact(_part), count() from tMM where toDate(d)=toDate('2020-09-15');
+select uniqExact(_part), count() from tMM where toDate(d)=toDate('2020-09-01');
+select uniqExact(_part), count() from tMM where toDate(d)=toDate('2020-10-15');
+select uniqExact(_part), count() from tMM where toDate(d)='2020-09-15';
+select uniqExact(_part), count() from tMM where toYYYYMM(d)=202009;
+select uniqExact(_part), count() from tMM where toYYYYMMDD(d)=20200816;
+select uniqExact(_part), count() from tMM where toYYYYMMDD(d)=20201015;
+select uniqExact(_part), count() from tMM where toDate(d)='2020-10-15';
+select uniqExact(_part), count() from tMM where d >= '2020-09-01 00:00:00' and d<'2020-10-15 00:00:00';
+select uniqExact(_part), count() from tMM where d >= '2020-01-16 00:00:00' and d < toDateTime('2021-08-17 00:00:00', 'Asia/Istanbul');
+select uniqExact(_part), count() from tMM where d >= '2020-09-16 00:00:00' and d < toDateTime('2020-10-01 00:00:00', 'Asia/Istanbul');
+select uniqExact(_part), count() from tMM where d >= '2020-09-12 00:00:00' and d < '2020-10-16 00:00:00';
+select uniqExact(_part), count() from tMM where toStartOfDay(d) >= '2020-09-12 00:00:00';
+select uniqExact(_part), count() from tMM where toStartOfDay(d) = '2020-09-01 00:00:00';
+select uniqExact(_part), count() from tMM where toStartOfDay(d) = '2020-10-01 00:00:00';
+select uniqExact(_part), count() from tMM where toStartOfDay(d) >= '2020-09-15 00:00:00' and d < '2020-10-16 00:00:00';
+select uniqExact(_part), count() from tMM where toYYYYMM(d) between 202009 and 202010;
+select uniqExact(_part), count() from tMM where toYYYYMM(d) between 202009 and 202009;
+select uniqExact(_part), count() from tMM where toYYYYMM(d) between 202009 and 202010 and toStartOfDay(d) = '2020-10-01 00:00:00';
+select uniqExact(_part), count() from tMM where toYYYYMM(d) >= 202009 and toStartOfDay(d) < '2020-10-02 00:00:00';
+select uniqExact(_part), count() from tMM where toYYYYMM(d) > 202009 and toStartOfDay(d) < '2020-10-02 00:00:00';
+select uniqExact(_part), count() from tMM where toYYYYMM(d)+1 > 202009 and toStartOfDay(d) < '2020-10-02 00:00:00';
+select uniqExact(_part), count() from tMM where toYYYYMM(d)+1 > 202010 and toStartOfDay(d) < '2020-10-02 00:00:00';
+select uniqExact(_part), count() from tMM where toYYYYMM(d)+1 > 202010;
+select uniqExact(_part), count() from tMM where toYYYYMM(d-1)+1 = 202010;
+select uniqExact(_part), count() from tMM where toStartOfMonth(d) >= '2020-09-15';
+select uniqExact(_part), count() from tMM where toStartOfMonth(d) >= '2020-09-01';
+select uniqExact(_part), count() from tMM where toStartOfMonth(d) >= '2020-09-01' and toStartOfMonth(d) < '2020-10-01';
+
+SYSTEM START MERGES tMM;
+OPTIMIZE TABLE tMM FINAL;
+
+select uniqExact(_part), count() from tMM where toYYYYMM(d-1)+1 = 202010;
+select uniqExact(_part), count() from tMM where toYYYYMM(d)+1 > 202010;
+select uniqExact(_part), count() from tMM where toYYYYMM(d) between 202009 and 202010;
+
+DROP TABLE tMM;
+"
diff --git a/tests/queries/0_stateless/01508_partition_pruning_long_2.reference b/tests/queries/0_stateless/01508_partition_pruning_long_2.reference
new file mode 100644
index 00000000000..bc767f17031
--- /dev/null
+++ b/tests/queries/0_stateless/01508_partition_pruning_long_2.reference
@@ -0,0 +1,119 @@
+--------- tDD ----------------------------
+select uniqExact(_part), count() from tDD where toDate(d)=toDate('2020-09-24');
+1	10000
+Selected 1/4 parts by partition key, 1 parts by primary key, 1/1 marks by primary key, 1 marks to read from 1 ranges
+
+select uniqExact(_part), count() FROM tDD WHERE toDate(d) = toDate('2020-09-24');
+1	10000
+Selected 1/4 parts by partition key, 1 parts by primary key, 1/1 marks by primary key, 1 marks to read from 1 ranges
+
+select uniqExact(_part), count() FROM tDD WHERE toDate(d) = '2020-09-24';
+1	10000
+Selected 1/4 parts by partition key, 1 parts by primary key, 1/1 marks by primary key, 1 marks to read from 1 ranges
+
+select uniqExact(_part), count() FROM tDD WHERE toDate(d) >= '2020-09-23' and toDate(d) <= '2020-09-26';
+3	40000
+Selected 3/4 parts by partition key, 3 parts by primary key, 4/4 marks by primary key, 4 marks to read from 3 ranges
+
+select uniqExact(_part), count() FROM tDD WHERE toYYYYMMDD(d) >= 20200923 and toDate(d) <= '2020-09-26';
+3	40000
+Selected 3/4 parts by partition key, 3 parts by primary key, 4/4 marks by primary key, 4 marks to read from 3 ranges
+
+--------- sDD ----------------------------
+select uniqExact(_part), count() from sDD;
+6	30000
+Selected 6/6 parts by partition key, 6 parts by primary key, 6/6 marks by primary key, 6 marks to read from 6 ranges
+
+select uniqExact(_part), count() from sDD where toYYYYMM(toDateTime(intDiv(d,1000),'UTC')-1)+1 = 202010;
+3	9999
+Selected 3/6 parts by partition key, 3 parts by primary key, 3/3 marks by primary key, 3 marks to read from 3 ranges
+
+select uniqExact(_part), count() from sDD where toYYYYMM(toDateTime(intDiv(d,1000),'UTC')-1) = 202010;
+2	9999
+Selected 2/6 parts by partition key, 2 parts by primary key, 2/2 marks by primary key, 2 marks to read from 2 ranges
+
+select uniqExact(_part), count() from sDD where toYYYYMM(toDateTime(intDiv(d,1000),'UTC')-1) = 202110;
+0	0
+Selected 0/6 parts by partition key, 0 parts by primary key, 0/0 marks by primary key, 0 marks to read from 0 ranges
+
+select uniqExact(_part), count() from sDD where toYYYYMM(toDateTime(intDiv(d,1000),'UTC'))+1 > 202009 and toStartOfDay(toDateTime(intDiv(d,1000),'UTC')) < toDateTime('2020-10-02 00:00:00','UTC');
+3	11440
+Selected 3/6 parts by partition key, 3 parts by primary key, 3/3 marks by primary key, 3 marks to read from 3 ranges
+
+select uniqExact(_part), count() from sDD where toYYYYMM(toDateTime(intDiv(d,1000),'UTC'))+1 > 202009 and toDateTime(intDiv(d,1000),'UTC') < toDateTime('2020-10-01 00:00:00','UTC');
+2	10000
+Selected 2/6 parts by partition key, 2 parts by primary key, 2/2 marks by primary key, 2 marks to read from 2 ranges
+
+select uniqExact(_part), count() from sDD where d >= 1598918400000;
+4	20000
+Selected 4/6 parts by partition key, 4 parts by primary key, 4/4 marks by primary key, 4 marks to read from 4 ranges
+
+select uniqExact(_part), count() from sDD where d >= 1598918400000 and toYYYYMM(toDateTime(intDiv(d,1000),'UTC')-1) < 202010;
+3	10001
+Selected 3/6 parts by partition key, 3 parts by primary key, 3/3 marks by primary key, 3 marks to read from 3 ranges
+
+--------- xMM ----------------------------
+select uniqExact(_part), count() from xMM where toStartOfDay(d) >= '2020-10-01 00:00:00';
+2	10000
+Selected 2/6 parts by partition key, 2 parts by primary key, 2/2 marks by primary key, 2 marks to read from 2 ranges
+
+select uniqExact(_part), count() from xMM where d >= '2020-09-01 00:00:00' and d <= '2020-10-01 00:00:00';
+3	10001
+Selected 3/6 parts by partition key, 3 parts by primary key, 3/3 marks by primary key, 3 marks to read from 3 ranges
+
+select uniqExact(_part), count() from xMM where d >= '2020-09-01 00:00:00' and d < '2020-10-01 00:00:00';
+2	10000
+Selected 2/6 parts by partition key, 2 parts by primary key, 2/2 marks by primary key, 2 marks to read from 2 ranges
+
+select uniqExact(_part), count() from xMM where d >= '2020-09-01 00:00:00' and d <= '2020-10-01 00:00:00' and a=1;
+1	1
+Selected 1/6 parts by partition key, 1 parts by primary key, 1/1 marks by primary key, 1 marks to read from 1 ranges
+
+select uniqExact(_part), count() from xMM where d >= '2020-09-01 00:00:00' and d <= '2020-10-01 00:00:00' and a<>3;
+2	5001
+Selected 2/6 parts by partition key, 2 parts by primary key, 2/2 marks by primary key, 2 marks to read from 2 ranges
+
+select uniqExact(_part), count() from xMM where d >= '2020-09-01 00:00:00' and d < '2020-10-01 00:00:00' and a<>3;
+1	5000
+Selected 1/6 parts by partition key, 1 parts by primary key, 1/1 marks by primary key, 1 marks to read from 1 ranges
+
+select uniqExact(_part), count() from xMM where d >= '2020-09-01 00:00:00' and d < '2020-11-01 00:00:00' and a = 1;
+2	10000
+Selected 2/6 parts by partition key, 2 parts by primary key, 2/2 marks by primary key, 2 marks to read from 2 ranges
+
+select uniqExact(_part), count() from xMM where a = 1;
+3	15000
+Selected 3/6 parts by partition key, 3 parts by primary key, 3/3 marks by primary key, 3 marks to read from 3 ranges
+
+select uniqExact(_part), count() from xMM where a = 66;
+0	0
+Selected 0/6 parts by partition key, 0 parts by primary key, 0/0 marks by primary key, 0 marks to read from 0 ranges
+
+select uniqExact(_part), count() from xMM where a <> 66;
+6	30000
+Selected 6/6 parts by partition key, 6 parts by primary key, 6/6 marks by primary key, 6 marks to read from 6 ranges
+
+select uniqExact(_part), count() from xMM where a = 2;
+2	10000
+Selected 2/6 parts by partition key, 2 parts by primary key, 2/2 marks by primary key, 2 marks to read from 2 ranges
+
+select uniqExact(_part), count() from xMM where a = 1;
+2	15000
+Selected 2/5 parts by partition key, 2 parts by primary key, 2/2 marks by primary key, 2 marks to read from 2 ranges
+
+select uniqExact(_part), count() from xMM where toStartOfDay(d) >= '2020-10-01 00:00:00';
+1	10000
+Selected 1/5 parts by partition key, 1 parts by primary key, 1/1 marks by primary key, 1 marks to read from 1 ranges
+
+select uniqExact(_part), count() from xMM where a <> 66;
+5	30000
+Selected 5/5 parts by partition key, 5 parts by primary key, 5/5 marks by primary key, 5 marks to read from 5 ranges
+
+select uniqExact(_part), count() from xMM where d >= '2020-09-01 00:00:00' and d <= '2020-10-01 00:00:00' and a<>3;
+2	5001
+Selected 2/5 parts by partition key, 2 parts by primary key, 2/2 marks by primary key, 2 marks to read from 2 ranges
+
+select uniqExact(_part), count() from xMM where d >= '2020-09-01 00:00:00' and d < '2020-10-01 00:00:00' and a<>3;
+1	5000
+Selected 1/5 parts by partition key, 1 parts by primary key, 1/1 marks by primary key, 1 marks to read from 1 ranges
+
diff --git a/tests/queries/0_stateless/01508_partition_pruning_long.queries b/tests/queries/0_stateless/01508_partition_pruning_long_2.sh
old mode 100644
new mode 100755
similarity index 58%
rename from tests/queries/0_stateless/01508_partition_pruning_long.queries
rename to tests/queries/0_stateless/01508_partition_pruning_long_2.sh
index 0d64fc05f0f..3f8a89bdb20
--- a/tests/queries/0_stateless/01508_partition_pruning_long.queries
+++ b/tests/queries/0_stateless/01508_partition_pruning_long_2.sh
@@ -1,15 +1,35 @@
-DROP TABLE IF EXISTS tMM;
+#!/usr/bin/env bash
+# Tags: long, no-polymorphic-parts, no-random-settings, no-random-merge-tree-settings, no-debug
+
+# Description of test result:
+# Test the correctness of the partition pruning
+#
+# Script executes queries from a file 01508_partition_pruning_long.queries (1 line = 1 query)
+# Queries are started with 'select' (but NOT with 'SELECT') are executed with log_level=debug
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+
+while IFS= read -r sql
+do
+  [ -z "$sql" ] && continue
+  if [[ "$sql" == select* ]] ;
+  then
+    echo "$sql"
+    ${CLICKHOUSE_CLIENT} --query "$sql"
+    CLICKHOUSE_CLIENT=$(echo ${CLICKHOUSE_CLIENT} | sed 's/'"--send_logs_level=${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL}"'/--send_logs_level=debug/g')
+    ${CLICKHOUSE_CLIENT} --query "$sql" 2>&1 | grep -oh "Selected .* parts by partition key, *. parts by primary key, .* marks by primary key, .* marks to read from .* ranges.*$"
+    CLICKHOUSE_CLIENT=$(echo ${CLICKHOUSE_CLIENT} | sed 's/--send_logs_level=debug/'"--send_logs_level=${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL}"'/g')
+    echo ""
+  else
+    ${CLICKHOUSE_CLIENT} --query "$sql"
+  fi
+done <<< "
 DROP TABLE IF EXISTS tDD;
 DROP TABLE IF EXISTS sDD;
 DROP TABLE IF EXISTS xMM;
-CREATE TABLE tMM(d DateTime('Asia/Istanbul'), a Int64) ENGINE = MergeTree PARTITION BY toYYYYMM(d) ORDER BY tuple() SETTINGS index_granularity = 8192;
-SYSTEM STOP MERGES tMM;
-INSERT INTO tMM SELECT toDateTime('2020-08-16 00:00:00', 'Asia/Istanbul') + number*60, number FROM numbers(5000);
-INSERT INTO tMM SELECT toDateTime('2020-08-16 00:00:00', 'Asia/Istanbul') + number*60, number FROM numbers(5000);
-INSERT INTO tMM SELECT toDateTime('2020-09-01 00:00:00', 'Asia/Istanbul') + number*60, number FROM numbers(5000);
-INSERT INTO tMM SELECT toDateTime('2020-09-01 00:00:00', 'Asia/Istanbul') + number*60, number FROM numbers(5000);
-INSERT INTO tMM SELECT toDateTime('2020-10-01 00:00:00', 'Asia/Istanbul') + number*60, number FROM numbers(5000);
-INSERT INTO tMM SELECT toDateTime('2020-10-15 00:00:00', 'Asia/Istanbul') + number*60, number FROM numbers(5000);
 
 CREATE TABLE tDD(d DateTime('Asia/Istanbul'),a Int) ENGINE = MergeTree PARTITION BY toYYYYMMDD(d) ORDER BY tuple() SETTINGS index_granularity = 8192;
 SYSTEM STOP MERGES tDD;
@@ -34,44 +54,6 @@ INSERT INTO xMM SELECT toDateTime('2020-10-01 00:00:00', 'Asia/Istanbul') + numb
 INSERT INTO xMM SELECT toDateTime('2020-10-15 00:00:00', 'Asia/Istanbul') + number*60, 1, number FROM numbers(5000);
 
 
-SELECT '--------- tMM ----------------------------';
-select uniqExact(_part), count() from tMM where toDate(d)=toDate('2020-09-15');
-select uniqExact(_part), count() from tMM where toDate(d)=toDate('2020-09-01');
-select uniqExact(_part), count() from tMM where toDate(d)=toDate('2020-10-15');
-select uniqExact(_part), count() from tMM where toDate(d)='2020-09-15';
-select uniqExact(_part), count() from tMM where toYYYYMM(d)=202009;
-select uniqExact(_part), count() from tMM where toYYYYMMDD(d)=20200816;
-select uniqExact(_part), count() from tMM where toYYYYMMDD(d)=20201015;
-select uniqExact(_part), count() from tMM where toDate(d)='2020-10-15';
-select uniqExact(_part), count() from tMM where d >= '2020-09-01 00:00:00' and d<'2020-10-15 00:00:00';
-select uniqExact(_part), count() from tMM where d >= '2020-01-16 00:00:00' and d < toDateTime('2021-08-17 00:00:00', 'Asia/Istanbul');
-select uniqExact(_part), count() from tMM where d >= '2020-09-16 00:00:00' and d < toDateTime('2020-10-01 00:00:00', 'Asia/Istanbul');
-select uniqExact(_part), count() from tMM where d >= '2020-09-12 00:00:00' and d < '2020-10-16 00:00:00';
-select uniqExact(_part), count() from tMM where toStartOfDay(d) >= '2020-09-12 00:00:00';
-select uniqExact(_part), count() from tMM where toStartOfDay(d) = '2020-09-01 00:00:00';
-select uniqExact(_part), count() from tMM where toStartOfDay(d) = '2020-10-01 00:00:00';
-select uniqExact(_part), count() from tMM where toStartOfDay(d) >= '2020-09-15 00:00:00' and d < '2020-10-16 00:00:00';
-select uniqExact(_part), count() from tMM where toYYYYMM(d) between 202009 and 202010;
-select uniqExact(_part), count() from tMM where toYYYYMM(d) between 202009 and 202009;
-select uniqExact(_part), count() from tMM where toYYYYMM(d) between 202009 and 202010 and toStartOfDay(d) = '2020-10-01 00:00:00';
-select uniqExact(_part), count() from tMM where toYYYYMM(d) >= 202009 and toStartOfDay(d) < '2020-10-02 00:00:00';
-select uniqExact(_part), count() from tMM where toYYYYMM(d) > 202009 and toStartOfDay(d) < '2020-10-02 00:00:00';
-select uniqExact(_part), count() from tMM where toYYYYMM(d)+1 > 202009 and toStartOfDay(d) < '2020-10-02 00:00:00';
-select uniqExact(_part), count() from tMM where toYYYYMM(d)+1 > 202010 and toStartOfDay(d) < '2020-10-02 00:00:00';
-select uniqExact(_part), count() from tMM where toYYYYMM(d)+1 > 202010;
-select uniqExact(_part), count() from tMM where toYYYYMM(d-1)+1 = 202010;
-select uniqExact(_part), count() from tMM where toStartOfMonth(d) >= '2020-09-15';
-select uniqExact(_part), count() from tMM where toStartOfMonth(d) >= '2020-09-01';
-select uniqExact(_part), count() from tMM where toStartOfMonth(d) >= '2020-09-01' and toStartOfMonth(d) < '2020-10-01';
-
-SYSTEM START MERGES tMM;
-OPTIMIZE TABLE tMM FINAL;
-
-select uniqExact(_part), count() from tMM where toYYYYMM(d-1)+1 = 202010;
-select uniqExact(_part), count() from tMM where toYYYYMM(d)+1 > 202010;
-select uniqExact(_part), count() from tMM where toYYYYMM(d) between 202009 and 202010;
-
-
 SELECT '--------- tDD ----------------------------';
 SYSTEM START MERGES tDD;
 OPTIMIZE TABLE tDD FINAL;
@@ -116,9 +98,7 @@ select uniqExact(_part), count() from xMM where a <> 66;
 select uniqExact(_part), count() from xMM where d >= '2020-09-01 00:00:00' and d <= '2020-10-01 00:00:00' and a<>3;
 select uniqExact(_part), count() from xMM where d >= '2020-09-01 00:00:00' and d < '2020-10-01 00:00:00' and a<>3;
 
-DROP TABLE tMM;
 DROP TABLE tDD;
 DROP TABLE sDD;
 DROP TABLE xMM;
-
-
+"

From beed31a7e38a6484aec156754106570a34dcb5a9 Mon Sep 17 00:00:00 2001
From: avogar <avogar@clickhouse.com>
Date: Tue, 23 Jul 2024 20:07:47 +0000
Subject: [PATCH 0668/1488] Fix use-of-uninitialized-value in JSONExtract*
 numeric functions

---
 src/Functions/FunctionsJSON.cpp                                | 3 ++-
 .../03209_functions_json_msan_fuzzer_issue.reference           | 1 +
 .../0_stateless/03209_functions_json_msan_fuzzer_issue.sql     | 2 ++
 3 files changed, 5 insertions(+), 1 deletion(-)
 create mode 100644 tests/queries/0_stateless/03209_functions_json_msan_fuzzer_issue.reference
 create mode 100644 tests/queries/0_stateless/03209_functions_json_msan_fuzzer_issue.sql

diff --git a/src/Functions/FunctionsJSON.cpp b/src/Functions/FunctionsJSON.cpp
index 47040545677..848856c500f 100644
--- a/src/Functions/FunctionsJSON.cpp
+++ b/src/Functions/FunctionsJSON.cpp
@@ -739,7 +739,8 @@ public:
     {
         NumberType value;
 
-        tryGetNumericValueFromJSONElement<JSONParser, NumberType>(value, element, convert_bool_to_integer, error);
+        if (!tryGetNumericValueFromJSONElement<JSONParser, NumberType>(value, element, convert_bool_to_integer, error))
+            return false;
         auto & col_vec = assert_cast<ColumnVector<NumberType> &>(dest);
         col_vec.insertValue(value);
         return true;
diff --git a/tests/queries/0_stateless/03209_functions_json_msan_fuzzer_issue.reference b/tests/queries/0_stateless/03209_functions_json_msan_fuzzer_issue.reference
new file mode 100644
index 00000000000..e02f3666d40
--- /dev/null
+++ b/tests/queries/0_stateless/03209_functions_json_msan_fuzzer_issue.reference
@@ -0,0 +1 @@
+0	0	0	1.1
diff --git a/tests/queries/0_stateless/03209_functions_json_msan_fuzzer_issue.sql b/tests/queries/0_stateless/03209_functions_json_msan_fuzzer_issue.sql
new file mode 100644
index 00000000000..a05b07d5971
--- /dev/null
+++ b/tests/queries/0_stateless/03209_functions_json_msan_fuzzer_issue.sql
@@ -0,0 +1,2 @@
+WITH '{ "v":1.1}' AS raw SELECT JSONExtract(raw, 'float') AS float32_1, JSONExtract(concat(tuple('1970-01-05', 10, materialize(10), 10, 10, 10, toUInt256(10), 10, toNullable(10), 10, 10), materialize(toUInt256(0)), ', ', 2, 2, toLowCardinality(toLowCardinality(2))), raw, toLowCardinality('v'), 'Float32') AS float32_2, JSONExtractFloat(raw) AS float64_1, JSONExtract(raw, 'v', 'double') AS float64_2;
+

From 7a6233340e5a141a43e3a1ac7a1d33ea1c4d5a8b Mon Sep 17 00:00:00 2001
From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com>
Date: Tue, 23 Jul 2024 21:57:01 +0000
Subject: [PATCH 0669/1488] Hardening of test_backup_restore_new test

---
 .../test_backup_restore_new/test.py           | 46 ++++++++++++++-----
 1 file changed, 34 insertions(+), 12 deletions(-)

diff --git a/tests/integration/test_backup_restore_new/test.py b/tests/integration/test_backup_restore_new/test.py
index d8662fad011..5cbabe35b03 100644
--- a/tests/integration/test_backup_restore_new/test.py
+++ b/tests/integration/test_backup_restore_new/test.py
@@ -4,6 +4,7 @@ import re
 import random
 import os.path
 import sys
+import uuid
 from collections import namedtuple
 from helpers.cluster import ClickHouseCluster
 from helpers.test_tools import assert_eq_with_retry, TSV
@@ -538,7 +539,8 @@ def test_backup_not_found_or_already_exists():
 
 
 def test_file_engine():
-    backup_name = f"File('/backups/file/')"
+    id = uuid.uuid4()
+    backup_name = f"File('/backups/file/{id}/')"
     create_and_fill_table()
 
     assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n"
@@ -549,6 +551,7 @@ def test_file_engine():
 
     instance.query(f"RESTORE TABLE test.table FROM {backup_name}")
     assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n"
+    instance.query("DROP TABLE test.table")
 
 
 def test_database():
@@ -565,7 +568,8 @@ def test_database():
 
 
 def test_zip_archive():
-    backup_name = f"Disk('backups', 'archive.zip')"
+    id = uuid.uuid4()
+    backup_name = f"Disk('backups', 'archive_{id}.zip')"
     create_and_fill_table()
 
     assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n"
@@ -578,10 +582,12 @@ def test_zip_archive():
 
     instance.query(f"RESTORE TABLE test.table FROM {backup_name}")
     assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n"
+    instance.query("DROP TABLE test.table")
 
 
 def test_zip_archive_with_settings():
-    backup_name = f"Disk('backups', 'archive_with_settings.zip')"
+    id = uuid.uuid4()
+    backup_name = f"Disk('backups', 'archive_with_settings_{id}.zip')"
     create_and_fill_table()
 
     assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n"
@@ -596,10 +602,12 @@ def test_zip_archive_with_settings():
         f"RESTORE TABLE test.table FROM {backup_name} SETTINGS password='qwerty'"
     )
     assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n"
+    instance.query("DROP TABLE test.table")
 
 
 def test_zip_archive_with_bad_compression_method():
-    backup_name = f"Disk('backups', 'archive_with_bad_compression_method.zip')"
+    id = uuid.uuid4()
+    backup_name = f"Disk('backups', 'archive_with_bad_compression_method_{id}.zip')"
     create_and_fill_table()
 
     assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n"
@@ -617,7 +625,8 @@ def test_zip_archive_with_bad_compression_method():
 
 
 def test_tar_archive():
-    backup_name = f"Disk('backups', 'archive.tar')"
+    id = uuid.uuid4()
+    backup_name = f"Disk('backups', 'archive_{id}.tar')"
     create_and_fill_table()
 
     assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n"
@@ -630,10 +639,12 @@ def test_tar_archive():
 
     instance.query(f"RESTORE TABLE test.table FROM {backup_name}")
     assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n"
+    instance.query("DROP TABLE test.table")
 
 
 def test_tar_bz2_archive():
-    backup_name = f"Disk('backups', 'archive.tar.bz2')"
+    id = uuid.uuid4()
+    backup_name = f"Disk('backups', 'archive_{id}.tar.bz2')"
     create_and_fill_table()
 
     assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n"
@@ -649,7 +660,8 @@ def test_tar_bz2_archive():
 
 
 def test_tar_gz_archive():
-    backup_name = f"Disk('backups', 'archive.tar.gz')"
+    id = uuid.uuid4()
+    backup_name = f"Disk('backups', 'archive_{id}.tar.gz')"
     create_and_fill_table()
 
     assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n"
@@ -665,7 +677,8 @@ def test_tar_gz_archive():
 
 
 def test_tar_lzma_archive():
-    backup_name = f"Disk('backups', 'archive.tar.lzma')"
+    id = uuid.uuid4()
+    backup_name = f"Disk('backups', 'archive_{id}.tar.lzma')"
     create_and_fill_table()
 
     assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n"
@@ -681,7 +694,8 @@ def test_tar_lzma_archive():
 
 
 def test_tar_zst_archive():
-    backup_name = f"Disk('backups', 'archive.tar.zst')"
+    id = uuid.uuid4()
+    backup_name = f"Disk('backups', 'archive_{id}.tar.zst')"
     create_and_fill_table()
 
     assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n"
@@ -697,7 +711,8 @@ def test_tar_zst_archive():
 
 
 def test_tar_xz_archive():
-    backup_name = f"Disk('backups', 'archive.tar.xz')"
+    id = uuid.uuid4()
+    backup_name = f"Disk('backups', 'archive_{id}.tar.xz')"
     create_and_fill_table()
 
     assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n"
@@ -713,7 +728,8 @@ def test_tar_xz_archive():
 
 
 def test_tar_archive_with_password():
-    backup_name = f"Disk('backups', 'archive_with_password.tar')"
+    id = uuid.uuid4()
+    backup_name = f"Disk('backups', 'archive_with_password_{id}.tar')"
     create_and_fill_table()
 
     assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n"
@@ -731,7 +747,8 @@ def test_tar_archive_with_password():
 
 
 def test_tar_archive_with_bad_compression_method():
-    backup_name = f"Disk('backups', 'archive_with_bad_compression_method.tar')"
+    id = uuid.uuid4()
+    backup_name = f"Disk('backups', 'archive_with_bad_compression_method_{id}.tar')"
     create_and_fill_table()
 
     assert instance.query("SELECT count(), sum(x) FROM test.table") == "100\t4950\n"
@@ -1220,6 +1237,9 @@ def test_system_users_required_privileges():
     assert instance.query("SHOW CREATE ROLE r1") == "CREATE ROLE r1\n"
     assert instance.query("SHOW GRANTS FOR r1") == ""
 
+    instance.query("DROP USER u1")
+    instance.query("DROP ROLE r1")
+
 
 def test_system_users_async():
     instance.query("CREATE USER u1 IDENTIFIED BY 'qwe123' SETTINGS custom_c = 3")
@@ -1412,6 +1432,8 @@ def test_system_functions():
     assert instance.query("SELECT number, parity_str(number) FROM numbers(3)") == TSV(
         [[0, "even"], [1, "odd"], [2, "even"]]
     )
+    instance.query("DROP FUNCTION linear_equation")
+    instance.query("DROP FUNCTION parity_str")
 
 
 def test_backup_partition():

From d04ca9eea617a406be715fdf5d541e8d650b23d0 Mon Sep 17 00:00:00 2001
From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com>
Date: Wed, 24 Jul 2024 00:08:32 +0200
Subject: [PATCH 0670/1488] fix

---
 tests/integration/runner | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/tests/integration/runner b/tests/integration/runner
index fc13cb9807a..0667541b196 100755
--- a/tests/integration/runner
+++ b/tests/integration/runner
@@ -243,7 +243,9 @@ if __name__ == "__main__":
         "-n", "--parallel", action="store", dest="parallel", help="Parallelism"
     )
 
-    parser.add_argument("--count", action="store", dest="count", help="Repeat count")
+    parser.add_argument(
+        "--count", action="store", type=int, dest="count", help="Repeat count"
+    )
 
     parser.add_argument(
         "--no-random",
@@ -320,7 +322,9 @@ if __name__ == "__main__":
         parallel_args += "--dist=loadfile"
         parallel_args += f" -n {args.parallel}".format()
 
-    repeat_args = f" --count {args.count}" if args.count > 0 else ""
+    repeat_args = (
+        f" --count {args.count}" if args.count is not None and args.count > 0 else ""
+    )
 
     rand_args = ""
     # if not args.no_random:

From 8925060366e18e2fcb88d5d3c173097e524ce613 Mon Sep 17 00:00:00 2001
From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com>
Date: Wed, 24 Jul 2024 00:18:12 +0200
Subject: [PATCH 0671/1488] Touch dockerfile to forcefully rebuild image

---
 docker/test/sqlancer/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/test/sqlancer/Dockerfile b/docker/test/sqlancer/Dockerfile
index 82fc2598397..3b919ffb3e3 100644
--- a/docker/test/sqlancer/Dockerfile
+++ b/docker/test/sqlancer/Dockerfile
@@ -6,7 +6,7 @@ ARG apt_archive="http://archive.ubuntu.com"
 RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
 
 RUN apt-get update --yes \
-    && env DEBIAN_FRONTEND=noninteractive apt-get install wget git default-jdk maven python3 --yes --no-install-recommends  \
+    && env DEBIAN_FRONTEND=noninteractive apt-get install wget git python3 default-jdk maven --yes --no-install-recommends  \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/*
 

From d460fa9f3618cdba64315bce579a93329463b7ad Mon Sep 17 00:00:00 2001
From: zoomxi <419486879@qq.com>
Date: Wed, 24 Jul 2024 09:57:00 +0800
Subject: [PATCH 0672/1488] create coordinator after the number of replicas to
 use for the query is determined

---
 .../ClusterProxy/executeQuery.cpp             |  3 ---
 src/Processors/QueryPlan/ReadFromRemote.cpp   |  6 ++---
 src/Processors/QueryPlan/ReadFromRemote.h     |  1 -
 .../ParallelReplicasReadingCoordinator.cpp    |  4 ++--
 .../ParallelReplicasReadingCoordinator.h      |  4 ----
 .../test.py                                   | 22 +++----------------
 6 files changed, 8 insertions(+), 32 deletions(-)

diff --git a/src/Interpreters/ClusterProxy/executeQuery.cpp b/src/Interpreters/ClusterProxy/executeQuery.cpp
index 5d56ef09127..59f095f7487 100644
--- a/src/Interpreters/ClusterProxy/executeQuery.cpp
+++ b/src/Interpreters/ClusterProxy/executeQuery.cpp
@@ -517,14 +517,11 @@ void executeQueryWithParallelReplicas(
                 "`cluster_for_parallel_replicas` setting refers to cluster with several shards. Expected a cluster with one shard");
     }
 
-    auto coordinator = std::make_shared<ParallelReplicasReadingCoordinator>(
-        new_cluster->getShardsInfo().begin()->getAllNodeCount(), settings.parallel_replicas_mark_segment_size);
     auto external_tables = new_context->getExternalTables();
     auto read_from_remote = std::make_unique<ReadFromParallelRemoteReplicasStep>(
         query_ast,
         new_cluster,
         storage_id,
-        std::move(coordinator),
         header,
         processed_stage,
         new_context,
diff --git a/src/Processors/QueryPlan/ReadFromRemote.cpp b/src/Processors/QueryPlan/ReadFromRemote.cpp
index 29e12c1e613..cf11052cd59 100644
--- a/src/Processors/QueryPlan/ReadFromRemote.cpp
+++ b/src/Processors/QueryPlan/ReadFromRemote.cpp
@@ -362,7 +362,6 @@ ReadFromParallelRemoteReplicasStep::ReadFromParallelRemoteReplicasStep(
     ASTPtr query_ast_,
     ClusterPtr cluster_,
     const StorageID & storage_id_,
-    ParallelReplicasReadingCoordinatorPtr coordinator_,
     Block header_,
     QueryProcessingStage::Enum stage_,
     ContextMutablePtr context_,
@@ -375,7 +374,6 @@ ReadFromParallelRemoteReplicasStep::ReadFromParallelRemoteReplicasStep(
     , cluster(cluster_)
     , query_ast(query_ast_)
     , storage_id(storage_id_)
-    , coordinator(std::move(coordinator_))
     , stage(std::move(stage_))
     , context(context_)
     , throttler(throttler_)
@@ -429,7 +427,6 @@ void ReadFromParallelRemoteReplicasStep::initializePipeline(QueryPipelineBuilder
     {
         shuffled_pool = shard.pool->getShuffledPools(current_settings);
         shuffled_pool.resize(max_replicas_to_use);
-        coordinator->adjustParticipatingReplicasCount(max_replicas_to_use);
     }
     else
     {
@@ -439,6 +436,9 @@ void ReadFromParallelRemoteReplicasStep::initializePipeline(QueryPipelineBuilder
         shuffled_pool = shard.pool->getShuffledPools(current_settings, priority_func);
     }
 
+    coordinator
+        = std::make_shared<ParallelReplicasReadingCoordinator>(max_replicas_to_use, current_settings.parallel_replicas_mark_segment_size);
+
     for (size_t i=0; i < max_replicas_to_use; ++i)
     {
         IConnections::ReplicaInfo replica_info
diff --git a/src/Processors/QueryPlan/ReadFromRemote.h b/src/Processors/QueryPlan/ReadFromRemote.h
index eb15269155a..1adb26b2915 100644
--- a/src/Processors/QueryPlan/ReadFromRemote.h
+++ b/src/Processors/QueryPlan/ReadFromRemote.h
@@ -70,7 +70,6 @@ public:
         ASTPtr query_ast_,
         ClusterPtr cluster_,
         const StorageID & storage_id_,
-        ParallelReplicasReadingCoordinatorPtr coordinator_,
         Block header_,
         QueryProcessingStage::Enum stage_,
         ContextMutablePtr context_,
diff --git a/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp b/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp
index 2ba66256116..f46b4de10b7 100644
--- a/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp
+++ b/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.cpp
@@ -1031,7 +1031,7 @@ void ParallelReplicasReadingCoordinator::markReplicaAsUnavailable(size_t replica
     if (!pimpl)
     {
         unavailable_nodes_registered_before_initialization.push_back(replica_number);
-        if (unavailable_nodes_registered_before_initialization.size() == participating_replicas_count)
+        if (unavailable_nodes_registered_before_initialization.size() == replicas_count)
             throw Exception(ErrorCodes::ALL_CONNECTION_TRIES_FAILED, "Can't connect to any replica chosen for query execution");
     }
     else
@@ -1061,7 +1061,7 @@ void ParallelReplicasReadingCoordinator::initialize(CoordinationMode mode)
 }
 
 ParallelReplicasReadingCoordinator::ParallelReplicasReadingCoordinator(size_t replicas_count_, size_t mark_segment_size_)
-    : replicas_count(replicas_count_), participating_replicas_count(replicas_count_), mark_segment_size(mark_segment_size_)
+    : replicas_count(replicas_count_), mark_segment_size(mark_segment_size_)
 {
 }
 
diff --git a/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.h b/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.h
index c06ef6ef01a..8b463fda395 100644
--- a/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.h
+++ b/src/Storages/MergeTree/ParallelReplicasReadingCoordinator.h
@@ -30,15 +30,11 @@ public:
     /// needed to report total rows to read
     void setProgressCallback(ProgressCallback callback);
 
-    /// Participating replicas count may be less than replicas count in a shard.
-    void adjustParticipatingReplicasCount(size_t count) { participating_replicas_count = count; }
-
 private:
     void initialize(CoordinationMode mode);
 
     std::mutex mutex;
     const size_t replicas_count{0};
-    size_t participating_replicas_count{0};
     size_t mark_segment_size{0};
     std::unique_ptr<ImplInterface> pimpl;
     ProgressCallback progress_callback; // store the callback only to bypass it to coordinator implementation
diff --git a/tests/integration/test_parallel_replicas_no_replicas/test.py b/tests/integration/test_parallel_replicas_no_replicas/test.py
index e05f72316d0..04e3a54e581 100644
--- a/tests/integration/test_parallel_replicas_no_replicas/test.py
+++ b/tests/integration/test_parallel_replicas_no_replicas/test.py
@@ -33,7 +33,8 @@ def create_tables(cluster, table_name):
 
 
 @pytest.mark.parametrize("skip_unavailable_shards", [1, 0])
-def test_skip_all_replicas(start_cluster, skip_unavailable_shards):
+@pytest.mark.parametrize("max_parallel_replicas", [2, 3, 100])
+def test_skip_all_replicas(start_cluster, skip_unavailable_shards, max_parallel_replicas):
     cluster_name = "test_1_shard_3_unavaliable_replicas"
     table_name = "tt"
     create_tables(cluster_name, table_name)
@@ -43,25 +44,8 @@ def test_skip_all_replicas(start_cluster, skip_unavailable_shards):
             f"SELECT key, count() FROM {table_name}  GROUP BY key ORDER BY key",
             settings={
                 "allow_experimental_parallel_reading_from_replicas": 2,
-                "max_parallel_replicas": 3,
+                "max_parallel_replicas": max_parallel_replicas,
                 "cluster_for_parallel_replicas": cluster_name,
                 "skip_unavailable_shards": skip_unavailable_shards,
             },
         )
-
-@pytest.mark.parametrize("max_parallel_replicas", [2, 3, 100])
-def test_skip_all_participating_replicas(start_cluster, max_parallel_replicas):
-    cluster_name = "test_1_shard_3_unavaliable_replicas"
-    table_name = "tt1"
-    create_tables(cluster_name, table_name)
-
-    with pytest.raises(QueryRuntimeException):
-        initiator.query(
-            f"SELECT key, count() FROM {table_name}  GROUP BY key ORDER BY key",
-            settings={
-                "allow_experimental_parallel_reading_from_replicas": 2,
-                "max_parallel_replicas": max_parallel_replicas,
-                "cluster_for_parallel_replicas": cluster_name,
-                "skip_unavailable_shards": 1,
-            },
-        )

From e855ce793783bb50f95339608f688f6c27e83c36 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 24 Jul 2024 04:59:57 +0200
Subject: [PATCH 0673/1488] Revert "Fix for 992 and friends"

---
 src/Storages/StorageReplicatedMergeTree.cpp            |  2 +-
 ...00992_system_parts_race_condition_zookeeper_long.sh |  2 +-
 tests/queries/0_stateless/replication.lib              | 10 +---------
 3 files changed, 3 insertions(+), 11 deletions(-)

diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp
index da379a466af..3f02486ed15 100644
--- a/src/Storages/StorageReplicatedMergeTree.cpp
+++ b/src/Storages/StorageReplicatedMergeTree.cpp
@@ -3940,7 +3940,7 @@ void StorageReplicatedMergeTree::mergeSelectingTask()
         merge_selecting_task->schedule();
     else
     {
-        LOG_TRACE(log, "Scheduling next merge selecting task after {}ms, current attempt status: {}", merge_selecting_sleep_ms, result);
+        LOG_TRACE(log, "Scheduling next merge selecting task after {}ms", merge_selecting_sleep_ms);
         merge_selecting_task->scheduleAfter(merge_selecting_sleep_ms);
     }
 }
diff --git a/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper_long.sh b/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper_long.sh
index 02a739ece4a..4887c409844 100755
--- a/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper_long.sh
+++ b/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper_long.sh
@@ -41,7 +41,7 @@ function thread3()
 
 function thread4()
 {
-    while true; do $CLICKHOUSE_CLIENT --receive_timeout=1 -q "OPTIMIZE TABLE alter_table0 FINAL" | grep -Fv "Timeout exceeded while receiving data from server"; done
+    while true; do $CLICKHOUSE_CLIENT --receive_timeout=3 -q "OPTIMIZE TABLE alter_table0 FINAL" | grep -Fv "Timeout exceeded while receiving data from server"; done
 }
 
 function thread5()
diff --git a/tests/queries/0_stateless/replication.lib b/tests/queries/0_stateless/replication.lib
index fe867537000..1a86cd9f8db 100755
--- a/tests/queries/0_stateless/replication.lib
+++ b/tests/queries/0_stateless/replication.lib
@@ -51,14 +51,6 @@ function check_replication_consistency()
     table_name_prefix=$1
     check_query_part=$2
 
-    # Try to kill some mutations because sometimes tests run too much (it's not guarenteed to kill all mutations, see below)
-    # Try multiple replicas, because queries are not finished yet, and "global" KILL MUTATION may fail due to another query (like DROP TABLE)
-    readarray -t tables_arr < <(${CLICKHOUSE_CLIENT} -q "SELECT name FROM system.tables WHERE database=currentDatabase() AND name like '$table_name_prefix%'")
-    for t in "${tables_arr[@]}"
-    do
-        ${CLICKHOUSE_CLIENT} -q "KILL MUTATION WHERE database=currentDatabase() AND table='$t'" > /dev/null 2>/dev/null
-    done
-
     # Wait for all queries to finish (query may still be running if thread is killed by timeout)
     num_tries=0
     while [[ $($CLICKHOUSE_CLIENT -q "SELECT count() FROM system.processes WHERE current_database=currentDatabase() AND query LIKE '%$table_name_prefix%'") -ne 1 ]]; do
@@ -104,7 +96,7 @@ function check_replication_consistency()
     some_table=$($CLICKHOUSE_CLIENT -q "SELECT name FROM system.tables WHERE database=currentDatabase() AND name like '$table_name_prefix%' ORDER BY rand() LIMIT 1")
     $CLICKHOUSE_CLIENT -q "SYSTEM SYNC REPLICA $some_table PULL" 1>/dev/null 2>/dev/null ||:
 
-    # Forcefully cancel mutations to avoid waiting for them to finish. Kills the remaining mutations
+    # Forcefully cancel mutations to avoid waiting for them to finish
     ${CLICKHOUSE_CLIENT} -q "KILL MUTATION WHERE database=currentDatabase() AND table like '$table_name_prefix%'" > /dev/null
 
     # SYNC REPLICA is not enough if some MUTATE_PARTs are not assigned yet

From 385dabaa19a41a703d549925aeb6c9ecc9c0a493 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 24 Jul 2024 07:14:32 +0200
Subject: [PATCH 0674/1488] Fix CLion, which is one of the worst examples of
 software ever created

---
 src/Core/Settings.h | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/src/Core/Settings.h b/src/Core/Settings.h
index 3d181e33001..8d3a11dac98 100644
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@@ -1,5 +1,8 @@
 #pragma once
 
+/// CLion freezes for a minute on every keypress in any file including this.
+#if !defined(__CLION_IDE__)
+
 #include <Common/NamePrompter.h>
 #include <Core/BaseSettings.h>
 #include <Core/SettingsEnums.h>
@@ -1348,3 +1351,5 @@ struct FormatFactorySettings : public BaseSettings<FormatFactorySettingsTraits>
 };
 
 }
+
+#endif

From bc899ead26f593f2e09f6f303151539b5b615560 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 24 Jul 2024 07:45:24 +0200
Subject: [PATCH 0675/1488] Fix the estimation of total rows for the numbers
 data source

---
 src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp
index b070bbe739b..4136e2d58b4 100644
--- a/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp
+++ b/src/Processors/QueryPlan/ReadFromSystemNumbersStep.cpp
@@ -596,12 +596,12 @@ Pipe ReadFromSystemNumbersStep::makePipe()
             numbers_storage.step,
             step_between_chunks);
 
-        if (numbers_storage.limit && i == 0)
+        if (end && i == 0)
         {
-            auto rows_appr = itemCountInRange(numbers_storage.offset, *numbers_storage.limit, numbers_storage.step);
-            if (limit > 0 && limit < rows_appr)
-                rows_appr = query_info_limit;
-            source->addTotalRowsApprox(rows_appr);
+            UInt64 rows_approx = itemCountInRange(numbers_storage.offset, *end, numbers_storage.step);
+            if (limit > 0 && limit < rows_approx)
+                rows_approx = query_info_limit;
+            source->addTotalRowsApprox(rows_approx);
         }
 
         pipe.addSource(std::move(source));

From 0744a761cd86d43e8746bf71619987b70e34238b Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 24 Jul 2024 07:46:23 +0200
Subject: [PATCH 0676/1488] Add a test

---
 .../0_stateless/03208_numbers_total_rows_approx.reference        | 1 +
 tests/queries/0_stateless/03208_numbers_total_rows_approx.sql    | 1 +
 2 files changed, 2 insertions(+)
 create mode 100644 tests/queries/0_stateless/03208_numbers_total_rows_approx.reference
 create mode 100644 tests/queries/0_stateless/03208_numbers_total_rows_approx.sql

diff --git a/tests/queries/0_stateless/03208_numbers_total_rows_approx.reference b/tests/queries/0_stateless/03208_numbers_total_rows_approx.reference
new file mode 100644
index 00000000000..0cfbf08886f
--- /dev/null
+++ b/tests/queries/0_stateless/03208_numbers_total_rows_approx.reference
@@ -0,0 +1 @@
+2
diff --git a/tests/queries/0_stateless/03208_numbers_total_rows_approx.sql b/tests/queries/0_stateless/03208_numbers_total_rows_approx.sql
new file mode 100644
index 00000000000..7855dfb6207
--- /dev/null
+++ b/tests/queries/0_stateless/03208_numbers_total_rows_approx.sql
@@ -0,0 +1 @@
+SELECT number FROM numbers(2, 1) WHERE number % 2 = 0 SETTINGS max_rows_to_read = 10;

From e156db49225d273abe97eef84612f564a7547be9 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 24 Jul 2024 07:56:11 +0200
Subject: [PATCH 0677/1488] Remove the support for Kerberized HDFS

---
 .../test_storage_kerberized_hdfs/__init__.py  |   0
 .../configs/hdfs.xml                          |  13 -
 .../hdfs_configs/bootstrap.sh                 | 280 ------------------
 .../kerberos_image_config.sh                  | 140 ---------
 .../secrets/krb.conf                          |  25 --
 .../secrets/krb_long.conf                     |  24 --
 .../test_storage_kerberized_hdfs/test.py      | 155 ----------
 7 files changed, 637 deletions(-)
 delete mode 100644 tests/integration/test_storage_kerberized_hdfs/__init__.py
 delete mode 100644 tests/integration/test_storage_kerberized_hdfs/configs/hdfs.xml
 delete mode 100755 tests/integration/test_storage_kerberized_hdfs/hdfs_configs/bootstrap.sh
 delete mode 100644 tests/integration/test_storage_kerberized_hdfs/kerberos_image_config.sh
 delete mode 100644 tests/integration/test_storage_kerberized_hdfs/secrets/krb.conf
 delete mode 100644 tests/integration/test_storage_kerberized_hdfs/secrets/krb_long.conf
 delete mode 100644 tests/integration/test_storage_kerberized_hdfs/test.py

diff --git a/tests/integration/test_storage_kerberized_hdfs/__init__.py b/tests/integration/test_storage_kerberized_hdfs/__init__.py
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/tests/integration/test_storage_kerberized_hdfs/configs/hdfs.xml b/tests/integration/test_storage_kerberized_hdfs/configs/hdfs.xml
deleted file mode 100644
index 2e9dc6cc06c..00000000000
--- a/tests/integration/test_storage_kerberized_hdfs/configs/hdfs.xml
+++ /dev/null
@@ -1,13 +0,0 @@
-<clickhouse>
-  <hdfs>
-		<hadoop_kerberos_keytab>/tmp/keytab/clickhouse.keytab</hadoop_kerberos_keytab>
-		<hadoop_kerberos_principal>root@TEST.CLICKHOUSE.TECH</hadoop_kerberos_principal>
-		<hadoop_security_authentication>kerberos</hadoop_security_authentication>
-  </hdfs>
-  <hdfs_suser>
-		<hadoop_kerberos_principal>specuser@TEST.CLICKHOUSE.TECH</hadoop_kerberos_principal>
-  </hdfs_suser>
-  <hdfs_dedicatedcachepath>
-		<hadoop_security_kerberos_ticket_cache_path>/tmp/kerb_cache</hadoop_security_kerberos_ticket_cache_path>
-  </hdfs_dedicatedcachepath>
-</clickhouse>
diff --git a/tests/integration/test_storage_kerberized_hdfs/hdfs_configs/bootstrap.sh b/tests/integration/test_storage_kerberized_hdfs/hdfs_configs/bootstrap.sh
deleted file mode 100755
index db6921bc1c8..00000000000
--- a/tests/integration/test_storage_kerberized_hdfs/hdfs_configs/bootstrap.sh
+++ /dev/null
@@ -1,280 +0,0 @@
-#!/bin/bash
-
-: "${HADOOP_PREFIX:=/usr/local/hadoop}"
-
-cat >> $HADOOP_PREFIX/etc/hadoop/hadoop-env.sh <<EOF
-export HADOOP_SECURE_DN_USER=hdfs
-export HADOOP_SECURE_DN_PID_DIR=$HADOOP_PREFIX/pid
-export HADOOP_SECURE_DN_LOG_DIR=$HADOOP_PREFIX/logs/hdfs
-export JSVC_HOME=$HADOOP_PREFIX/sbin
-EOF
-$HADOOP_PREFIX/etc/hadoop/hadoop-env.sh
-
-mkdir -p "${HADOOP_SECURE_DN_PID_DIR}"
-mkdir -p "${HADOOP_SECURE_DN_LOG_DIR}"
-
-rm /tmp/*.pid
-
-# installing libraries if any - (resource urls added comma separated to the ACP system variable)
-cd "${HADOOP_PREFIX}/share/hadoop/common" || exit
-for cp in ${ACP//,/ }; do  echo "== ${cp}"; curl -LO "${cp}" ; done;
-cd - || exit
-
-# altering the core-site configuration
-sed "s/HOSTNAME/${HOSTNAME}/" /usr/local/hadoop/etc/hadoop/core-site.xml.template | grep -v '/configuration' > /usr/local/hadoop/etc/hadoop/core-site.xml
-
-cat >> /usr/local/hadoop/etc/hadoop/core-site.xml << EOF
-  <property>
-    <name>hadoop.security.authentication</name>
-    <value>kerberos</value> <!-- A value of "simple" would disable security. -->
-  </property>
-   <property>
-     <name>hadoop.security.authorization</name>
-     <value>true</value>
-   </property>
-  <property>
-    <name>fs.defaultFS</name>
-    <value>hdfs://kerberizedhdfs1:9010</value>
-  </property>
-  <property>
-    <name>fs.default.name</name>
-    <value>hdfs://kerberizedhdfs1:9010</value>
-  </property>
-  <!--
-  <property>
-      <name>hadoop.rpc.protection</name>
-      <value>privacy</value>
-  </property>
-  -->
-</configuration>
-EOF
-
-
-cat > /usr/local/hadoop/etc/hadoop/hdfs-site.xml << EOF
-<configuration>
-<!--
-<property>
-	<name>dfs.permissions.enabled</name>
-	<value>false</value>
-</property>
--->
-<property>
-  <name>dfs.replication</name>
-  <value>1</value>
-</property>
-<!-- General HDFS security config -->
-<property>
-  <name>dfs.block.access.token.enable</name>
-  <value>true</value>
-</property>
-<!-- NameNode security config -->
-<property>
-  <name>dfs.namenode.keytab.file</name>
-  <value>/usr/local/hadoop/etc/hadoop/conf/hdfs.keytab</value> <!-- path to the HDFS keytab -->
-</property>
-<property>
-  <name>dfs.namenode.kerberos.principal</name>
-  <value>hdfs/_HOST@TEST.CLICKHOUSE.TECH</value>
-</property>
-<property>
-  <name>dfs.namenode.kerberos.internal.spnego.principal</name>
-  <value>HTTP/_HOST@TEST.CLICKHOUSE.TECH</value>
-</property>
-<!-- Secondary NameNode security config -->
-<property>
-  <name>dfs.secondary.namenode.keytab.file</name>
-  <value>/usr/local/hadoop/etc/hadoop/conf/hdfs.keytab</value> <!-- path to the HDFS keytab -->
-</property>
-<property>
-  <name>dfs.secondary.namenode.kerberos.principal</name>
-  <value>hdfs/_HOST@TEST.CLICKHOUSE.TECH</value>
-</property>
-<property>
-  <name>dfs.secondary.namenode.kerberos.internal.spnego.principal</name>
-  <value>HTTP/_HOST@TEST.CLICKHOUSE.TECH</value>
-</property>
-<!-- DataNode security config
-<property>
-   <name>dfs.data.transfer.protectionл</name>
-   <value>integrity</value>
-</property>
--->
-<property>
-  <name>dfs.datanode.data.dir.perm</name>
-  <value>700</value>
-</property>
-<property>
-  <name>dfs.datanode.address</name>
-  <value>0.0.0.0:1004</value>
-</property>
-<property>
-  <name>dfs.datanode.http.address</name>
-  <value>0.0.0.0:1006</value>
-</property>
-<!-- If the port is 0 then the server will start on a free port. -->
-<property>
-  <name>dfs.datanode.ipc.address</name>
-  <value>0.0.0.0:0</value>
-</property>
-<property>
-  <name>dfs.namenode.secondary.http-address</name>
-  <value>0.0.0.0:0</value>
-</property>
-<property>
-  <name>dfs.namenode.backup.address</name>
-  <value>0.0.0.0:0</value>
-</property>
-<property>
-  <name>dfs.namenode.backup.http-address</name>
-  <value>0.0.0.0:0</value>
-</property>
-<!--
-<property>
-  <name>dfs.http.policy</name>
-  <value>HTTPS_ONLY</value>
-</property>
--->
-
-
-<property>
-  <name>dfs.datanode.keytab.file</name>
-  <value>/usr/local/hadoop/etc/hadoop/conf/hdfs.keytab</value> <!-- path to the HDFS keytab -->
-</property>
-<property>
-  <name>dfs.datanode.kerberos.principal</name>
-  <value>hdfs/_HOST@TEST.CLICKHOUSE.TECH</value>
-</property>
-
-<!-- Web Authentication config -->
-<property>
-    <name>dfs.webhdfs.enabled</name>
-    <value>true</value>
-</property>
-<property>
-<name>dfs.encrypt.data.transfer</name>
-<value>false</value>
-</property>
-<property>
-  <name>dfs.web.authentication.kerberos.principal</name>
-  <value>HTTP/_HOST@TEST.CLICKHOUSE.TECH</value>
-</property>
-<property>
-    <name>dfs.web.authentication.kerberos.keytab</name>
-  <value>/usr/local/hadoop/etc/hadoop/conf/hdfs.keytab</value> <!-- path to the HDFS keytab -->
-</property>
-</configuration>
-EOF
-
-
-
-# cat > /usr/local/hadoop/etc/hadoop/ssl-server.xml << EOF
-# <configuration>
-# <property>
-#   <name>ssl.server.truststore.location</name>
-#   <value>/usr/local/hadoop/etc/hadoop/conf/hdfs.jks</value>
-# </property>
-# <property>
-#   <name>ssl.server.truststore.password</name>
-#   <value>masterkey</value>
-# </property>
-# <property>
-#   <name>ssl.server.keystore.location</name>
-#   <value>/usr/local/hadoop/etc/hadoop/conf/hdfs.jks</value>
-# </property>
-# <property>
-#   <name>ssl.server.keystore.password</name>
-#   <value>masterkey</value>
-# </property>
-# <property>
-#   <name>ssl.server.keystore.keypassword</name>
-#   <value>masterkey</value>
-# </property>
-# </configuration>
-# EOF
-
-cat > /usr/local/hadoop/etc/hadoop/log4j.properties << EOF
-# Set everything to be logged to the console
-log4j.rootCategory=DEBUG, console
-log4j.appender.console=org.apache.log4j.ConsoleAppender
-log4j.appender.console.target=System.err
-log4j.appender.console.layout=org.apache.log4j.PatternLayout
-log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n
-
-# Set the default spark-shell log level to WARN. When running the spark-shell, the
-# log level for this class is used to overwrite the root logger's log level, so that
-# the user can have different defaults for the shell and regular Spark apps.
-log4j.logger.org.apache.spark.repl.Main=INFO
-
-# Settings to quiet third party logs that are too verbose
-log4j.logger.org.spark_project.jetty=DEBUG
-log4j.logger.org.spark_project.jetty.util.component.AbstractLifeCycle=ERROR
-log4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=INFO
-log4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=INFO
-log4j.logger.org.apache.parquet=ERROR
-log4j.logger.parquet=ERROR
-
-# SPARK-9183: Settings to avoid annoying messages when looking up nonexistent UDFs in SparkSQL with Hive support
-log4j.logger.org.apache.hadoop.hive.metastore.RetryingHMSHandler=FATAL
-log4j.logger.org.apache.hadoop.hive.ql.exec.FunctionRegistry=ERROR
-
-log4j.logger.org.apache.spark.deploy=DEBUG
-log4j.logger.org.apache.spark.executor=DEBUG
-log4j.logger.org.apache.spark.scheduler=DEBUG
-EOF
-
-useradd -u 1098 hdfs
-
-# keytool -genkey -alias kerberized_hdfs1.test.clickhouse.com -keyalg rsa -keysize 1024 -dname "CN=kerberized_hdfs1.test.clickhouse.com" -keypass masterkey -keystore /usr/local/hadoop/etc/hadoop/conf/hdfs.jks -storepass masterkey
-keytool -genkey -alias kerberizedhdfs1 -keyalg rsa -keysize 1024 -dname "CN=kerberizedhdfs1" -keypass masterkey -keystore /usr/local/hadoop/etc/hadoop/conf/hdfs.jks -storepass masterkey
-
-chmod g+r /usr/local/hadoop/etc/hadoop/conf/hdfs.jks
-
-
-service sshd start
-
-# yum --quiet --assumeyes install krb5-workstation.x86_64
-# yum --quiet --assumeyes install tcpdump
-
-# cd /tmp
-# curl http://archive.apache.org/dist/commons/daemon/source/commons-daemon-1.0.15-src.tar.gz   -o  commons-daemon-1.0.15-src.tar.gz
-# tar xzf commons-daemon-1.0.15-src.tar.gz
-# cd commons-daemon-1.0.15-src/src/native/unix
-# ./configure && make
-# cp ./jsvc /usr/local/hadoop/sbin
-
-
-until kinit -kt /usr/local/hadoop/etc/hadoop/conf/hdfs.keytab hdfs/kerberizedhdfs1@TEST.CLICKHOUSE.TECH; do sleep 2; done
-echo "KDC is up and ready to go... starting up"
-
-$HADOOP_PREFIX/sbin/start-dfs.sh
-$HADOOP_PREFIX/sbin/start-yarn.sh
-$HADOOP_PREFIX/sbin/mr-jobhistory-daemon.sh start historyserver
-
-chmod a+r /usr/local/hadoop/etc/hadoop/conf/hdfs.keytab   # create dedicated keytab for hdfsuser
-
-$HADOOP_PREFIX/sbin/start-secure-dns.sh
-sleep 3
-
-/usr/local/hadoop/bin/hdfs dfsadmin -safemode leave
-
-/usr/local/hadoop/bin/hdfs dfs -mkdir /user/specuser
-/usr/local/hadoop/bin/hdfs dfs -chown specuser /user/specuser
-echo "chown_completed" | /usr/local/hadoop/bin/hdfs dfs -appendToFile - /preparations_done_marker
-
-kdestroy
-
-
-
-# adduser --groups hdfs hdfsuser
-
-# /usr/local/hadoop/sbin/hadoop-daemon.sh --config /usr/local/hadoop/etc/hadoop/ --script /usr/local/hadoop/sbin/hdfs start namenode
-# /usr/local/hadoop/sbin/hadoop-daemon.sh --config /usr/local/hadoop/etc/hadoop/ --script /usr/local/hadoop/sbin/hdfs start datanode
-
-
-if [[ $1 == "-d" ]]; then
-  while true; do sleep 1000; done
-fi
-
-if [[ $1 == "-bash" ]]; then
-  /bin/bash
-fi
diff --git a/tests/integration/test_storage_kerberized_hdfs/kerberos_image_config.sh b/tests/integration/test_storage_kerberized_hdfs/kerberos_image_config.sh
deleted file mode 100644
index 45fb93792e0..00000000000
--- a/tests/integration/test_storage_kerberized_hdfs/kerberos_image_config.sh
+++ /dev/null
@@ -1,140 +0,0 @@
-#!/bin/bash
-
-
-set -x  #  trace
-
-: "${REALM:=TEST.CLICKHOUSE.TECH}"
-: "${DOMAIN_REALM:=test.clickhouse.com}"
-: "${KERB_MASTER_KEY:=masterkey}"
-: "${KERB_ADMIN_USER:=admin}"
-: "${KERB_ADMIN_PASS:=admin}"
-
-create_config() {
-  : "${KDC_ADDRESS:=$(hostname -f)}"
-
-  cat>/etc/krb5.conf<<EOF
-[logging]
- default = FILE:/var/log/kerberos/krb5libs.log
- kdc = FILE:/var/log/kerberos/krb5kdc.log
- admin_server = FILE:/var/log/kerberos/kadmind.log
-
-[libdefaults]
- default_realm = $REALM
- dns_lookup_realm = false
- dns_lookup_kdc = false
- ticket_lifetime = 15d
- renew_lifetime = 15d
- forwardable = true
- # WARNING: We use weaker key types to simplify testing as stronger key types
- # require the enhanced security JCE policy file to be installed. You should
- # NOT run with this configuration in production or any real environment. You
- # have been warned.
- default_tkt_enctypes = des-cbc-md5 des-cbc-crc des3-cbc-sha1
- default_tgs_enctypes = des-cbc-md5 des-cbc-crc des3-cbc-sha1
- permitted_enctypes = des-cbc-md5 des-cbc-crc des3-cbc-sha1
-
-[realms]
- $REALM = {
-  kdc = $KDC_ADDRESS
-  admin_server = $KDC_ADDRESS
- }
-
-[domain_realm]
- .$DOMAIN_REALM = $REALM
- $DOMAIN_REALM = $REALM
-EOF
-
-cat>/var/kerberos/krb5kdc/kdc.conf<<EOF
-[kdcdefaults]
- kdc_ports = 88
- kdc_tcp_ports = 88
-
-[realms]
- $REALM = {
-  acl_file = /var/kerberos/krb5kdc/kadm5.acl
-  dict_file = /usr/share/dict/words
-  admin_keytab = /var/kerberos/krb5kdc/kadm5.keytab
-  # WARNING: We use weaker key types to simplify testing as stronger key types
-  # require the enhanced security JCE policy file to be installed. You should
-  # NOT run with this configuration in production or any real environment. You
-  # have been warned.
-  master_key_type = des3-hmac-sha1
-  supported_enctypes = arcfour-hmac:normal des3-hmac-sha1:normal des-cbc-crc:normal des:normal des:v4 des:norealm des:onlyrealm des:afs3
-  default_principal_flags = +preauth
- }
-EOF
-}
-
-create_db() {
-  /usr/sbin/kdb5_util -P $KERB_MASTER_KEY -r $REALM create -s
-}
-
-start_kdc() {
-  mkdir -p /var/log/kerberos
-
-  /etc/rc.d/init.d/krb5kdc start
-  /etc/rc.d/init.d/kadmin start
-
-  chkconfig krb5kdc on
-  chkconfig kadmin on
-}
-
-restart_kdc() {
-  /etc/rc.d/init.d/krb5kdc restart
-  /etc/rc.d/init.d/kadmin restart
-}
-
-create_admin_user() {
-  kadmin.local -q "addprinc -pw $KERB_ADMIN_PASS $KERB_ADMIN_USER/admin"
-  echo "*/admin@$REALM *" > /var/kerberos/krb5kdc/kadm5.acl
-}
-
-create_keytabs() {
-  rm /tmp/keytab/*.keytab
-
-
-  # kadmin.local -q "addprinc -randkey hdfs/kerberizedhdfs1.${DOMAIN_REALM}@${REALM}"
-  # kadmin.local -q "ktadd -norandkey -k /tmp/keytab/hdfs.keytab hdfs/kerberizedhdfs1.${DOMAIN_REALM}@${REALM}"
-
-  # kadmin.local -q "addprinc -randkey HTTP/kerberizedhdfs1.${DOMAIN_REALM}@${REALM}"
-  # kadmin.local -q "ktadd -norandkey -k /tmp/keytab/hdfs.keytab HTTP/kerberizedhdfs1.${DOMAIN_REALM}@${REALM}"
-  kadmin.local -q "addprinc -randkey hdfs/kerberizedhdfs1@${REALM}"
-  kadmin.local -q "ktadd -norandkey -k /tmp/keytab/hdfs.keytab hdfs/kerberizedhdfs1@${REALM}"
-
-  kadmin.local -q "addprinc -randkey HTTP/kerberizedhdfs1@${REALM}"
-  kadmin.local -q "ktadd -norandkey -k /tmp/keytab/hdfs.keytab HTTP/kerberizedhdfs1@${REALM}"
-
-  kadmin.local -q "addprinc -randkey hdfsuser/node1@${REALM}"
-  kadmin.local -q "ktadd -norandkey -k /tmp/keytab/clickhouse.keytab hdfsuser/node1@${REALM}"
-  kadmin.local -q "addprinc -randkey hdfsuser@${REALM}"
-  kadmin.local -q "ktadd -norandkey -k /tmp/keytab/clickhouse.keytab hdfsuser@${REALM}"
-  kadmin.local -q "addprinc -randkey root@${REALM}"
-  kadmin.local -q "ktadd -norandkey -k /tmp/keytab/clickhouse.keytab root@${REALM}"
-  kadmin.local -q "addprinc -randkey specuser@${REALM}"
-  kadmin.local -q "ktadd -norandkey -k /tmp/keytab/clickhouse.keytab specuser@${REALM}"
-
-  chmod g+r /tmp/keytab/clickhouse.keytab
-}
-
-main() {
-
-  if [ ! -f /kerberos_initialized ]; then
-    create_config
-    create_db
-    create_admin_user
-    start_kdc
-
-    touch /kerberos_initialized
-  fi
-
-  if [ ! -f /var/kerberos/krb5kdc/principal ]; then
-    while true; do sleep 1000; done
-  else
-    start_kdc
-		create_keytabs
-    tail -F /var/log/kerberos/krb5kdc.log
-  fi
-
-}
-
-[[ "$0" == "${BASH_SOURCE[0]}" ]] && main "$@"
diff --git a/tests/integration/test_storage_kerberized_hdfs/secrets/krb.conf b/tests/integration/test_storage_kerberized_hdfs/secrets/krb.conf
deleted file mode 100644
index dffdcaebe81..00000000000
--- a/tests/integration/test_storage_kerberized_hdfs/secrets/krb.conf
+++ /dev/null
@@ -1,25 +0,0 @@
-[logging]
- default = FILE:/var/log/krb5libs.log
- kdc = FILE:/var/log/krb5kdc.log
- admin_server = FILE:/var/log/kadmind.log
-
-[libdefaults]
- default_realm = TEST.CLICKHOUSE.TECH
- dns_lookup_realm = false
- dns_lookup_kdc = false
- ticket_lifetime = 5s
- forwardable = true
- rdns = false
- default_tgs_enctypes = des3-hmac-sha1
- default_tkt_enctypes = des3-hmac-sha1
- permitted_enctypes = des3-hmac-sha1
-
-[realms]
- TEST.CLICKHOUSE.TECH = {
-  kdc = hdfskerberos
-  admin_server = hdfskerberos
- }
-
-[domain_realm]
- .test.clickhouse.com = TEST.CLICKHOUSE.TECH
- test.clickhouse.com = TEST.CLICKHOUSE.TECH
diff --git a/tests/integration/test_storage_kerberized_hdfs/secrets/krb_long.conf b/tests/integration/test_storage_kerberized_hdfs/secrets/krb_long.conf
deleted file mode 100644
index 43c009d2e98..00000000000
--- a/tests/integration/test_storage_kerberized_hdfs/secrets/krb_long.conf
+++ /dev/null
@@ -1,24 +0,0 @@
-[logging]
- default = FILE:/var/log/krb5libs.log
- kdc = FILE:/var/log/krb5kdc.log
- admin_server = FILE:/var/log/kadmind.log
-
-[libdefaults]
- default_realm = TEST.CLICKHOUSE.TECH
- dns_lookup_realm = false
- dns_lookup_kdc = false
- ticket_lifetime = 15d
- forwardable = true
- default_tgs_enctypes = des3-hmac-sha1
- default_tkt_enctypes = des3-hmac-sha1
- permitted_enctypes = des3-hmac-sha1
-
-[realms]
- TEST.CLICKHOUSE.TECH = {
-  kdc = hdfskerberos
-  admin_server = hdfskerberos
- }
-
-[domain_realm]
- .test.clickhouse.com = TEST.CLICKHOUSE.TECH
- test.clickhouse.com = TEST.CLICKHOUSE.TECH
diff --git a/tests/integration/test_storage_kerberized_hdfs/test.py b/tests/integration/test_storage_kerberized_hdfs/test.py
deleted file mode 100644
index ddfc1f6483d..00000000000
--- a/tests/integration/test_storage_kerberized_hdfs/test.py
+++ /dev/null
@@ -1,155 +0,0 @@
-import time
-import pytest
-
-import os
-
-from helpers.cluster import ClickHouseCluster, is_arm
-import subprocess
-
-if is_arm():
-    pytestmark = pytest.mark.skip
-
-cluster = ClickHouseCluster(__file__)
-node1 = cluster.add_instance(
-    "node1",
-    with_kerberized_hdfs=True,
-    user_configs=[],
-    main_configs=["configs/hdfs.xml"],
-)
-
-
-@pytest.fixture(scope="module")
-def started_cluster():
-    try:
-        cluster.start()
-
-        yield cluster
-
-    except Exception as ex:
-        print(ex)
-        raise ex
-    finally:
-        cluster.shutdown()
-
-
-def test_read_table(started_cluster):
-    hdfs_api = started_cluster.hdfs_api
-
-    data = "1\tSerialize\t555.222\n2\tData\t777.333\n"
-    hdfs_api.write_data("/simple_table_function", data)
-
-    api_read = hdfs_api.read_data("/simple_table_function")
-    assert api_read == data
-
-    select_read = node1.query(
-        "select * from hdfs('hdfs://kerberizedhdfs1:9010/simple_table_function', 'TSV', 'id UInt64, text String, number Float64')"
-    )
-    assert select_read == data
-
-
-def test_read_write_storage(started_cluster):
-    hdfs_api = started_cluster.hdfs_api
-
-    node1.query(
-        "create table SimpleHDFSStorage2 (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://kerberizedhdfs1:9010/simple_storage1', 'TSV')"
-    )
-    node1.query("insert into SimpleHDFSStorage2 values (1, 'Mark', 72.53)")
-
-    api_read = hdfs_api.read_data("/simple_storage1")
-    assert api_read == "1\tMark\t72.53\n"
-
-    select_read = node1.query("select * from SimpleHDFSStorage2")
-    assert select_read == "1\tMark\t72.53\n"
-
-
-def test_write_storage_not_expired(started_cluster):
-    hdfs_api = started_cluster.hdfs_api
-
-    node1.query(
-        "create table SimpleHDFSStorageNotExpired (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://kerberizedhdfs1:9010/simple_storage_not_expired', 'TSV')"
-    )
-
-    time.sleep(15)  # wait for ticket expiration
-    node1.query("insert into SimpleHDFSStorageNotExpired values (1, 'Mark', 72.53)")
-
-    api_read = hdfs_api.read_data("/simple_storage_not_expired")
-    assert api_read == "1\tMark\t72.53\n"
-
-    select_read = node1.query("select * from SimpleHDFSStorageNotExpired")
-    assert select_read == "1\tMark\t72.53\n"
-
-
-def test_two_users(started_cluster):
-    hdfs_api = started_cluster.hdfs_api
-
-    node1.query(
-        "create table HDFSStorOne (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://kerberizedhdfs1:9010/storage_user_one', 'TSV')"
-    )
-    node1.query("insert into HDFSStorOne values (1, 'Real', 86.00)")
-
-    node1.query(
-        "create table HDFSStorTwo (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://suser@kerberizedhdfs1:9010/user/specuser/storage_user_two', 'TSV')"
-    )
-    node1.query("insert into HDFSStorTwo values (1, 'Ideal', 74.00)")
-
-    select_read_1 = node1.query(
-        "select * from hdfs('hdfs://kerberizedhdfs1:9010/user/specuser/storage_user_two', 'TSV', 'id UInt64, text String, number Float64')"
-    )
-
-    select_read_2 = node1.query(
-        "select * from hdfs('hdfs://suser@kerberizedhdfs1:9010/storage_user_one', 'TSV', 'id UInt64, text String, number Float64')"
-    )
-
-
-def test_read_table_expired(started_cluster):
-    hdfs_api = started_cluster.hdfs_api
-
-    data = "1\tSerialize\t555.222\n2\tData\t777.333\n"
-    hdfs_api.write_data("/simple_table_function_relogin", data)
-
-    started_cluster.pause_container("hdfskerberos")
-    time.sleep(15)
-
-    try:
-        select_read = node1.query(
-            "select * from hdfs('hdfs://reloginuser&kerberizedhdfs1:9010/simple_table_function', 'TSV', 'id UInt64, text String, number Float64')"
-        )
-        assert False, "Exception have to be thrown"
-    except Exception as ex:
-        assert "DB::Exception: KerberosInit failure:" in str(ex)
-
-    started_cluster.unpause_container("hdfskerberos")
-
-
-def test_prohibited(started_cluster):
-    node1.query(
-        "create table HDFSStorTwoProhibited (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://suser@kerberizedhdfs1:9010/storage_user_two_prohibited', 'TSV')"
-    )
-    try:
-        node1.query("insert into HDFSStorTwoProhibited values (1, 'SomeOne', 74.00)")
-        assert False, "Exception have to be thrown"
-    except Exception as ex:
-        assert (
-            "Unable to open HDFS file: /storage_user_two_prohibited (hdfs://suser@kerberizedhdfs1:9010/storage_user_two_prohibited) error: Permission denied: user=specuser, access=WRITE"
-            in str(ex)
-        )
-
-
-def test_cache_path(started_cluster):
-    node1.query(
-        "create table HDFSStorCachePath (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://dedicatedcachepath@kerberizedhdfs1:9010/storage_dedicated_cache_path', 'TSV')"
-    )
-    try:
-        node1.query("insert into HDFSStorCachePath values (1, 'FatMark', 92.53)")
-        assert False, "Exception have to be thrown"
-    except Exception as ex:
-        assert (
-            "DB::Exception: hadoop.security.kerberos.ticket.cache.path cannot be set per user"
-            in str(ex)
-        )
-
-
-if __name__ == "__main__":
-    cluster.start()
-    input("Cluster created, press any key to destroy...")
-    cluster.shutdown()

From c850fac65276342e0b8694fa00d44dd3269d1abc Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 24 Jul 2024 09:05:06 +0200
Subject: [PATCH 0678/1488] Fix error

---
 src/Parsers/MySQL/tests/gtest_column_parser.cpp | 11 ++++++-----
 src/Parsers/ParserCreateQuery.h                 |  7 ++++---
 2 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/src/Parsers/MySQL/tests/gtest_column_parser.cpp b/src/Parsers/MySQL/tests/gtest_column_parser.cpp
index 21c37e4ee2e..3a9a0690f06 100644
--- a/src/Parsers/MySQL/tests/gtest_column_parser.cpp
+++ b/src/Parsers/MySQL/tests/gtest_column_parser.cpp
@@ -1,13 +1,14 @@
 #include <gtest/gtest.h>
 #include <Parsers/parseQuery.h>
 #include <Parsers/ASTLiteral.h>
-#include <Parsers/ASTFunction.h>
+#include <Parsers/ASTDataType.h>
 #include <Parsers/ASTIdentifier.h>
 #include <Parsers/MySQL/ASTDeclareColumn.h>
 #include <Parsers/MySQL/ASTDeclareOption.h>
 #include <Parsers/MySQL/ASTDeclareReference.h>
 #include <Parsers/MySQL/ASTDeclareConstraint.h>
 
+
 using namespace DB;
 using namespace DB::MySQLParser;
 
@@ -19,8 +20,8 @@ TEST(ParserColumn, AllNonGeneratedColumnOption)
                    "COLUMN_FORMAT FIXED STORAGE MEMORY REFERENCES tbl_name (col_01) CHECK 1";
     ASTPtr ast = parseQuery(p_column, input.data(), input.data() + input.size(), "", 0, 0, 0);
     EXPECT_EQ(ast->as<ASTDeclareColumn>()->name, "col_01");
-    EXPECT_EQ(ast->as<ASTDeclareColumn>()->data_type->as<ASTFunction>()->name, "VARCHAR");
-    EXPECT_EQ(ast->as<ASTDeclareColumn>()->data_type->as<ASTFunction>()->arguments->children[0]->as<ASTLiteral>()->value.safeGet<UInt64>(), 100);
+    EXPECT_EQ(ast->as<ASTDeclareColumn>()->data_type->as<ASTDataType>()->name, "VARCHAR");
+    EXPECT_EQ(ast->as<ASTDeclareColumn>()->data_type->as<ASTDataType>()->arguments->children[0]->as<ASTLiteral>()->value.safeGet<UInt64>(), 100);
 
     ASTDeclareOptions * declare_options = ast->as<ASTDeclareColumn>()->column_options->as<ASTDeclareOptions>();
     EXPECT_EQ(declare_options->changes["is_null"]->as<ASTLiteral>()->value.safeGet<UInt64>(), 0);
@@ -44,8 +45,8 @@ TEST(ParserColumn, AllGeneratedColumnOption)
                    "REFERENCES tbl_name (col_01) CHECK 1 GENERATED ALWAYS AS (1) STORED";
     ASTPtr ast = parseQuery(p_column, input.data(), input.data() + input.size(), "", 0, 0, 0);
     EXPECT_EQ(ast->as<ASTDeclareColumn>()->name, "col_01");
-    EXPECT_EQ(ast->as<ASTDeclareColumn>()->data_type->as<ASTFunction>()->name, "VARCHAR");
-    EXPECT_EQ(ast->as<ASTDeclareColumn>()->data_type->as<ASTFunction>()->arguments->children[0]->as<ASTLiteral>()->value.safeGet<UInt64>(), 100);
+    EXPECT_EQ(ast->as<ASTDeclareColumn>()->data_type->as<ASTDataType>()->name, "VARCHAR");
+    EXPECT_EQ(ast->as<ASTDeclareColumn>()->data_type->as<ASTDataType>()->arguments->children[0]->as<ASTLiteral>()->value.safeGet<UInt64>(), 100);
 
     ASTDeclareOptions * declare_options = ast->as<ASTDeclareColumn>()->column_options->as<ASTDeclareOptions>();
     EXPECT_EQ(declare_options->changes["is_null"]->as<ASTLiteral>()->value.safeGet<UInt64>(), 1);
diff --git a/src/Parsers/ParserCreateQuery.h b/src/Parsers/ParserCreateQuery.h
index 7bd1d1bf588..53a62deb22b 100644
--- a/src/Parsers/ParserCreateQuery.h
+++ b/src/Parsers/ParserCreateQuery.h
@@ -1,6 +1,7 @@
 #pragma once
 
 #include <Parsers/ASTFunction.h>
+#include <Parsers/ASTDataType.h>
 #include <Parsers/ASTColumnDeclaration.h>
 #include <Parsers/ASTIdentifier_fwd.h>
 #include <Parsers/ASTLiteral.h>
@@ -13,6 +14,7 @@
 #include <Parsers/ParserSetQuery.h>
 #include <Poco/String.h>
 
+
 namespace DB
 {
 
@@ -268,9 +270,8 @@ bool IParserColumnDeclaration<NameParser>::parseImpl(Pos & pos, ASTPtr & node, E
             auto default_function = std::make_shared<ASTFunction>();
             default_function->name = "defaultValueOfTypeName";
             default_function->arguments = std::make_shared<ASTExpressionList>();
-            // Ephemeral columns don't really have secrets but we need to format
-            // into a String, hence the strange call
-            default_function->arguments->children.emplace_back(std::make_shared<ASTLiteral>(type->as<ASTFunction>()->formatForLogging()));
+            /// Ephemeral columns don't really have secrets but we need to format into a String, hence the strange call
+            default_function->arguments->children.emplace_back(std::make_shared<ASTLiteral>(type->as<ASTDataType>()->formatForLogging()));
             default_expression = default_function;
         }
 

From c3204fb89577e50ec7ef2c7ddd3c62f913e084f2 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 24 Jul 2024 09:25:22 +0200
Subject: [PATCH 0679/1488] Fix error

---
 src/Parsers/ExpressionElementParsers.cpp |  3 +--
 src/Parsers/ExpressionListParsers.cpp    | 17 +++++++++++++++++
 src/Parsers/ExpressionListParsers.h      | 10 ++++++++++
 src/Parsers/ParserCreateIndexQuery.cpp   |  2 +-
 src/Parsers/ParserCreateQuery.cpp        |  4 ++--
 5 files changed, 31 insertions(+), 5 deletions(-)

diff --git a/src/Parsers/ExpressionElementParsers.cpp b/src/Parsers/ExpressionElementParsers.cpp
index d4fc9a4bc4d..865d07faaa7 100644
--- a/src/Parsers/ExpressionElementParsers.cpp
+++ b/src/Parsers/ExpressionElementParsers.cpp
@@ -9,8 +9,8 @@
 #include <Common/PODArray.h>
 #include <Common/StringUtils.h>
 #include <Common/typeid_cast.h>
-#include "Parsers/CommonParsers.h"
 
+#include <Parsers/CommonParsers.h>
 #include <Parsers/DumpASTNode.h>
 #include <Parsers/ASTAsterisk.h>
 #include <Parsers/ASTCollation.h>
@@ -725,7 +725,6 @@ bool ParserStatisticsType::parseImpl(Pos & pos, ASTPtr & node, Expected & expect
     function_node->name = "STATISTICS";
     function_node->arguments = stat_type;
     function_node->children.push_back(function_node->arguments);
-
     node = function_node;
     return true;
 }
diff --git a/src/Parsers/ExpressionListParsers.cpp b/src/Parsers/ExpressionListParsers.cpp
index f97c042e91e..66817fafa5e 100644
--- a/src/Parsers/ExpressionListParsers.cpp
+++ b/src/Parsers/ExpressionListParsers.cpp
@@ -2388,6 +2388,23 @@ bool ParserFunction::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
     }
 }
 
+bool ParserExpressionWithOptionalArguments::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
+{
+    ParserIdentifier id_p;
+    ParserFunction func_p;
+
+    if (ParserFunction(false, false).parse(pos, node, expected))
+        return true;
+
+    if (ParserIdentifier().parse(pos, node, expected))
+    {
+        node = makeASTFunction(node->as<ASTIdentifier>()->name());
+        return true;
+    }
+
+    return false;
+}
+
 const std::vector<std::pair<std::string_view, Operator>> ParserExpressionImpl::operators_table
 {
     {"->",            Operator("lambda",          1,  2, OperatorType::Lambda)},
diff --git a/src/Parsers/ExpressionListParsers.h b/src/Parsers/ExpressionListParsers.h
index 235d5782630..6ab38416f32 100644
--- a/src/Parsers/ExpressionListParsers.h
+++ b/src/Parsers/ExpressionListParsers.h
@@ -144,6 +144,16 @@ protected:
 };
 
 
+/** Similar to ParserFunction (and yields ASTFunction), but can also parse identifiers without braces.
+  */
+class ParserExpressionWithOptionalArguments : public IParserBase
+{
+protected:
+    const char * getName() const override { return "expression with optional parameters"; }
+    bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override;
+};
+
+
 /** An expression with an infix binary left-associative operator.
   * For example, a + b - c + d.
   */
diff --git a/src/Parsers/ParserCreateIndexQuery.cpp b/src/Parsers/ParserCreateIndexQuery.cpp
index 2761c99738b..b815ba60bab 100644
--- a/src/Parsers/ParserCreateIndexQuery.cpp
+++ b/src/Parsers/ParserCreateIndexQuery.cpp
@@ -21,7 +21,7 @@ bool ParserCreateIndexDeclaration::parseImpl(Pos & pos, ASTPtr & node, Expected
     ParserToken close_p(TokenType::ClosingRoundBracket);
     ParserOrderByExpressionList order_list_p;
 
-    ParserFunction type_p;
+    ParserExpressionWithOptionalArguments type_p;
     ParserExpression expression_p;
     ParserUnsignedInteger granularity_p;
 
diff --git a/src/Parsers/ParserCreateQuery.cpp b/src/Parsers/ParserCreateQuery.cpp
index 92c0e7b2558..5da6c3a2510 100644
--- a/src/Parsers/ParserCreateQuery.cpp
+++ b/src/Parsers/ParserCreateQuery.cpp
@@ -180,7 +180,7 @@ bool ParserIndexDeclaration::parseImpl(Pos & pos, ASTPtr & node, Expected & expe
     ParserKeyword s_granularity(Keyword::GRANULARITY);
 
     ParserIdentifier name_p;
-    ParserFunction type_p;
+    ParserExpressionWithOptionalArguments type_p;
     ParserExpression expression_p;
     ParserUnsignedInteger granularity_p;
 
@@ -240,7 +240,7 @@ bool ParserStatisticsDeclaration::parseImpl(Pos & pos, ASTPtr & node, Expected &
     ParserKeyword s_type(Keyword::TYPE);
 
     ParserList columns_p(std::make_unique<ParserIdentifier>(), std::make_unique<ParserToken>(TokenType::Comma), false);
-    ParserList types_p(std::make_unique<ParserDataType>(), std::make_unique<ParserToken>(TokenType::Comma), false);
+    ParserList types_p(std::make_unique<ParserExpressionWithOptionalArguments>(), std::make_unique<ParserToken>(TokenType::Comma), false);
 
     ASTPtr columns;
     ASTPtr types;

From 3a69e1555530f59cc83cb274c7dfa1dcaab14a2d Mon Sep 17 00:00:00 2001
From: Max K <maxkaynov@gmail.com>
Date: Wed, 24 Jul 2024 09:32:51 +0200
Subject: [PATCH 0680/1488] CI: Fix for workflow results parsing

---
 .github/workflows/backport_branches.yml | 2 +-
 .github/workflows/master.yml            | 2 +-
 .github/workflows/merge_queue.yml       | 2 +-
 .github/workflows/nightly.yml           | 2 +-
 .github/workflows/pull_request.yml      | 2 +-
 .github/workflows/release_branches.yml  | 2 +-
 6 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/backport_branches.yml b/.github/workflows/backport_branches.yml
index c602a46d23c..322946ac77b 100644
--- a/.github/workflows/backport_branches.yml
+++ b/.github/workflows/backport_branches.yml
@@ -269,7 +269,7 @@ jobs:
       - name: Check Workflow results
         run: |
           export WORKFLOW_RESULT_FILE="/tmp/workflow_results.json"
-          cat >> "$WORKFLOW_RESULT_FILE" << 'EOF'
+          cat > "$WORKFLOW_RESULT_FILE" << 'EOF'
           ${{ toJson(needs) }}
           EOF
           python3 ./tests/ci/ci_buddy.py --check-wf-status
diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index 7c319da6045..acd7511d520 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -135,7 +135,7 @@ jobs:
       - name: Check Workflow results
         run: |
           export WORKFLOW_RESULT_FILE="/tmp/workflow_results.json"
-          cat >> "$WORKFLOW_RESULT_FILE" << 'EOF'
+          cat > "$WORKFLOW_RESULT_FILE" << 'EOF'
           ${{ toJson(needs) }}
           EOF
           python3 ./tests/ci/ci_buddy.py --check-wf-status
diff --git a/.github/workflows/merge_queue.yml b/.github/workflows/merge_queue.yml
index 4b186241a0e..64083668719 100644
--- a/.github/workflows/merge_queue.yml
+++ b/.github/workflows/merge_queue.yml
@@ -108,7 +108,7 @@ jobs:
       - name: Check Workflow results
         run: |
           export WORKFLOW_RESULT_FILE="/tmp/workflow_results.json"
-          cat >> "$WORKFLOW_RESULT_FILE" << 'EOF'
+          cat > "$WORKFLOW_RESULT_FILE" << 'EOF'
           ${{ toJson(needs) }}
           EOF
           python3 ./tests/ci/ci_buddy.py --check-wf-status
diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
index 84db3338065..ea9c125db70 100644
--- a/.github/workflows/nightly.yml
+++ b/.github/workflows/nightly.yml
@@ -54,7 +54,7 @@ jobs:
       - name: Check Workflow results
         run: |
           export WORKFLOW_RESULT_FILE="/tmp/workflow_results.json"
-          cat >> "$WORKFLOW_RESULT_FILE" << 'EOF'
+          cat > "$WORKFLOW_RESULT_FILE" << 'EOF'
           ${{ toJson(needs) }}
           EOF
           python3 ./tests/ci/ci_buddy.py --check-wf-status
diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
index c7d7b28af38..63b2bd87dc9 100644
--- a/.github/workflows/pull_request.yml
+++ b/.github/workflows/pull_request.yml
@@ -168,7 +168,7 @@ jobs:
       - name: Check Workflow results
         run: |
           export WORKFLOW_RESULT_FILE="/tmp/workflow_results.json"
-          cat >> "$WORKFLOW_RESULT_FILE" << 'EOF'
+          cat > "$WORKFLOW_RESULT_FILE" << 'EOF'
           ${{ toJson(needs) }}
           EOF
           python3 ./tests/ci/ci_buddy.py --check-wf-status
diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml
index bca9ff33cd0..b79208b03a6 100644
--- a/.github/workflows/release_branches.yml
+++ b/.github/workflows/release_branches.yml
@@ -489,7 +489,7 @@ jobs:
       - name: Check Workflow results
         run: |
           export WORKFLOW_RESULT_FILE="/tmp/workflow_results.json"
-          cat >> "$WORKFLOW_RESULT_FILE" << 'EOF'
+          cat > "$WORKFLOW_RESULT_FILE" << 'EOF'
           ${{ toJson(needs) }}
           EOF
 

From f39c18d4d0e97771d919c2f4d511b8f8f87dae24 Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Wed, 24 Jul 2024 10:03:36 +0200
Subject: [PATCH 0681/1488] Fix flaky 01454_storagememory_data_race_challenge

---
 .../01454_storagememory_data_race_challenge.sh      | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/tests/queries/0_stateless/01454_storagememory_data_race_challenge.sh b/tests/queries/0_stateless/01454_storagememory_data_race_challenge.sh
index fa9238041b1..ec9c5134059 100755
--- a/tests/queries/0_stateless/01454_storagememory_data_race_challenge.sh
+++ b/tests/queries/0_stateless/01454_storagememory_data_race_challenge.sh
@@ -11,12 +11,17 @@ $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS mem"
 $CLICKHOUSE_CLIENT -q "CREATE TABLE mem (x UInt64) engine = Memory"
 
 function f {
+  local TIMELIMIT=$((SECONDS+$1))
   for _ in $(seq 1 300); do
     $CLICKHOUSE_CLIENT -q "SELECT count() FROM (SELECT * FROM mem SETTINGS max_threads=2) FORMAT Null;"
+    if [ $SECONDS -ge "$TIMELIMIT" ]; then
+        break
+    fi
   done
 }
 
 function g {
+  local TIMELIMIT=$((SECONDS+$1))
   for _ in $(seq 1 100); do
     $CLICKHOUSE_CLIENT -n -q "
         INSERT INTO mem SELECT number FROM numbers(1000000);
@@ -30,14 +35,18 @@ function g {
         INSERT INTO mem VALUES (1);
         TRUNCATE TABLE mem;
     "
+    if [ $SECONDS -ge "$TIMELIMIT" ]; then
+        break
+    fi
   done
 }
 
 export -f f;
 export -f g;
 
-timeout 20 bash -c f > /dev/null &
-timeout 20 bash -c g > /dev/null &
+TIMEOUT=20
+f $TIMEOUT &
+g $TIMEOUT &
 wait
 
 $CLICKHOUSE_CLIENT -q "DROP TABLE mem"

From 5fe78d47bc855867f6431ad06e019b3e0278d0ae Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 24 Jul 2024 10:24:13 +0200
Subject: [PATCH 0682/1488] Compatibility

---
 src/Parsers/ExpressionListParsers.cpp  |  1 +
 src/Parsers/ParserCreateIndexQuery.cpp | 10 +---------
 src/Parsers/ParserCreateQuery.cpp      | 10 +---------
 3 files changed, 3 insertions(+), 18 deletions(-)

diff --git a/src/Parsers/ExpressionListParsers.cpp b/src/Parsers/ExpressionListParsers.cpp
index 66817fafa5e..a9715cec81e 100644
--- a/src/Parsers/ExpressionListParsers.cpp
+++ b/src/Parsers/ExpressionListParsers.cpp
@@ -2399,6 +2399,7 @@ bool ParserExpressionWithOptionalArguments::parseImpl(Pos & pos, ASTPtr & node,
     if (ParserIdentifier().parse(pos, node, expected))
     {
         node = makeASTFunction(node->as<ASTIdentifier>()->name());
+        node->as<ASTFunction>().no_empty_args = true;
         return true;
     }
 
diff --git a/src/Parsers/ParserCreateIndexQuery.cpp b/src/Parsers/ParserCreateIndexQuery.cpp
index b815ba60bab..9ebee4cc852 100644
--- a/src/Parsers/ParserCreateIndexQuery.cpp
+++ b/src/Parsers/ParserCreateIndexQuery.cpp
@@ -69,15 +69,7 @@ bool ParserCreateIndexDeclaration::parseImpl(Pos & pos, ASTPtr & node, Expected
     if (s_type.ignore(pos, expected))
     {
         if (!type_p.parse(pos, type, expected))
-        {
-            if (ParserIdentifier().parse(pos, type, expected))
-            {
-                type = makeASTFunction(type->as<ASTIdentifier &>().name());
-                type->as<ASTFunction &>().no_empty_args = true;
-            }
-            else
-                return false;
-        }
+            return false;
     }
 
     if (s_granularity.ignore(pos, expected))
diff --git a/src/Parsers/ParserCreateQuery.cpp b/src/Parsers/ParserCreateQuery.cpp
index 5da6c3a2510..bf5523152ac 100644
--- a/src/Parsers/ParserCreateQuery.cpp
+++ b/src/Parsers/ParserCreateQuery.cpp
@@ -199,15 +199,7 @@ bool ParserIndexDeclaration::parseImpl(Pos & pos, ASTPtr & node, Expected & expe
         return false;
 
     if (!type_p.parse(pos, type, expected))
-    {
-        if (name_p.parse(pos, type, expected))
-        {
-            type = makeASTFunction(type->as<ASTIdentifier &>().name());
-            type->as<ASTFunction &>().no_empty_args = true;
-        }
-        else
-            return false;
-    }
+        return false;
 
     if (s_granularity.ignore(pos, expected))
     {

From e5bb485a006d93a9e00736dc37ad90a0a0a47673 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 24 Jul 2024 10:25:23 +0200
Subject: [PATCH 0683/1488] Compatibility

---
 src/Parsers/ExpressionListParsers.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Parsers/ExpressionListParsers.cpp b/src/Parsers/ExpressionListParsers.cpp
index a9715cec81e..d38dc6d5f37 100644
--- a/src/Parsers/ExpressionListParsers.cpp
+++ b/src/Parsers/ExpressionListParsers.cpp
@@ -2399,7 +2399,7 @@ bool ParserExpressionWithOptionalArguments::parseImpl(Pos & pos, ASTPtr & node,
     if (ParserIdentifier().parse(pos, node, expected))
     {
         node = makeASTFunction(node->as<ASTIdentifier>()->name());
-        node->as<ASTFunction>().no_empty_args = true;
+        node->as<ASTFunction &>().no_empty_args = true;
         return true;
     }
 

From 73fc5c266f3bc254db3882bfa2f9f42db6b2bc87 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 24 Jul 2024 10:37:19 +0200
Subject: [PATCH 0684/1488] Fix error

---
 src/DataTypes/DataTypeObject.cpp | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/DataTypes/DataTypeObject.cpp b/src/DataTypes/DataTypeObject.cpp
index 5636a46373f..91b9bfcb2a5 100644
--- a/src/DataTypes/DataTypeObject.cpp
+++ b/src/DataTypes/DataTypeObject.cpp
@@ -4,9 +4,10 @@
 
 #include <Parsers/IAST.h>
 #include <Parsers/ASTLiteral.h>
-#include <Parsers/ASTFunction.h>
+#include <Parsers/ASTDataType.h>
 #include <IO/Operators.h>
 
+
 namespace DB
 {
 
@@ -53,13 +54,13 @@ static DataTypePtr create(const ASTPtr & arguments)
     ASTPtr schema_argument = arguments->children[0];
     bool is_nullable = false;
 
-    if (const auto * func = schema_argument->as<ASTFunction>())
+    if (const auto * type = schema_argument->as<ASTDataType>())
     {
-        if (func->name != "Nullable" || func->arguments->children.size() != 1)
+        if (type->name != "Nullable" || type->arguments->children.size() != 1)
             throw Exception(ErrorCodes::UNEXPECTED_AST_STRUCTURE,
-                "Expected 'Nullable(<schema_name>)' as parameter for type Object (function: {})", func->name);
+                "Expected 'Nullable(<schema_name>)' as parameter for type Object (function: {})", type->name);
 
-        schema_argument = func->arguments->children[0];
+        schema_argument = type->arguments->children[0];
         is_nullable = true;
     }
 

From 42fe58f0466cf5a74f88c1a48bee9e96540f01e5 Mon Sep 17 00:00:00 2001
From: Xu Jia <jia.xu@clickhouse.com>
Date: Wed, 24 Jul 2024 08:33:00 +0000
Subject: [PATCH 0685/1488] bugfix AttachedTable counting not symmetry, and
 adding some test logs on attaching and detaching tables

---
 src/Databases/DatabasesCommon.cpp | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/src/Databases/DatabasesCommon.cpp b/src/Databases/DatabasesCommon.cpp
index 6ccaf811764..851ba8aef35 100644
--- a/src/Databases/DatabasesCommon.cpp
+++ b/src/Databases/DatabasesCommon.cpp
@@ -289,8 +289,12 @@ StoragePtr DatabaseWithOwnTablesBase::detachTableUnlocked(const String & table_n
     tables.erase(it);
     table_storage->is_detached = true;
 
-    if (!table_storage->isSystemStorage() && database_name != DatabaseCatalog::SYSTEM_DATABASE)
+    if (!table_storage->isSystemStorage()
+        && database_name != DatabaseCatalog::SYSTEM_DATABASE
+        && database_name != DatabaseCatalog::TEMPORARY_DATABASE) {
+        LOG_TEST(log, "Counting detached table {} to database {}", table_name, database_name);
         CurrentMetrics::sub(getAttachedCounterForStorage(table_storage));
+    }
 
     auto table_id = table_storage->getStorageID();
     if (table_id.hasUUID())
@@ -334,8 +338,12 @@ void DatabaseWithOwnTablesBase::attachTableUnlocked(const String & table_name, c
     /// non-Atomic database the is_detached is set to true before RENAME.
     table->is_detached = false;
 
-    if (!table->isSystemStorage() && table_id.database_name != DatabaseCatalog::SYSTEM_DATABASE)
+    if (!table->isSystemStorage()
+        && database_name != DatabaseCatalog::SYSTEM_DATABASE
+        && database_name != DatabaseCatalog::TEMPORARY_DATABASE) {
+        LOG_TEST(log, "Counting attached table {} to database {}", table_name, database_name);
         CurrentMetrics::add(getAttachedCounterForStorage(table));
+    }
 }
 
 void DatabaseWithOwnTablesBase::shutdown()

From a43c4f4f7ff5e0f0f63c870a01e6c1c28ee6f267 Mon Sep 17 00:00:00 2001
From: Max K <maxkaynov@gmail.com>
Date: Wed, 24 Jul 2024 10:54:16 +0200
Subject: [PATCH 0686/1488] CI: Automerge when required and non-required checks
 completed

---
 .github/workflows/pull_request.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
index 63b2bd87dc9..34bf51871d2 100644
--- a/.github/workflows/pull_request.yml
+++ b/.github/workflows/pull_request.yml
@@ -152,8 +152,9 @@ jobs:
 
   CheckReadyForMerge:
     if: ${{ !cancelled() }}
-    # Test_2 or Test_3 must not have jobs required for Mergeable check
-    needs: [RunConfig, BuildDockers, StyleCheck, FastTest, Builds_1, Builds_2, Builds_Report, Tests_1]
+    # Test_2 or Test_3 do not have the jobs required for Mergeable check,
+    #  however, set them as "needs" to get all checks results before the automatic merge occurs.
+    needs: [RunConfig, BuildDockers, StyleCheck, FastTest, Builds_1, Builds_2, Builds_Report, Tests_1, Tests_2, Tests_3]
     runs-on: [self-hosted, style-checker-aarch64]
     steps:
       - name: Check out repository code

From e46d400f7beba9479dfcd0d63f025268081f625c Mon Sep 17 00:00:00 2001
From: Xu Jia <jia.xu@clickhouse.com>
Date: Wed, 24 Jul 2024 09:03:05 +0000
Subject: [PATCH 0687/1488] fix style

---
 src/Databases/DatabasesCommon.cpp | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/Databases/DatabasesCommon.cpp b/src/Databases/DatabasesCommon.cpp
index 851ba8aef35..d2926c64f29 100644
--- a/src/Databases/DatabasesCommon.cpp
+++ b/src/Databases/DatabasesCommon.cpp
@@ -291,7 +291,8 @@ StoragePtr DatabaseWithOwnTablesBase::detachTableUnlocked(const String & table_n
 
     if (!table_storage->isSystemStorage()
         && database_name != DatabaseCatalog::SYSTEM_DATABASE
-        && database_name != DatabaseCatalog::TEMPORARY_DATABASE) {
+        && database_name != DatabaseCatalog::TEMPORARY_DATABASE)
+    {
         LOG_TEST(log, "Counting detached table {} to database {}", table_name, database_name);
         CurrentMetrics::sub(getAttachedCounterForStorage(table_storage));
     }
@@ -340,7 +341,8 @@ void DatabaseWithOwnTablesBase::attachTableUnlocked(const String & table_name, c
 
     if (!table->isSystemStorage()
         && database_name != DatabaseCatalog::SYSTEM_DATABASE
-        && database_name != DatabaseCatalog::TEMPORARY_DATABASE) {
+        && database_name != DatabaseCatalog::TEMPORARY_DATABASE)
+    {
         LOG_TEST(log, "Counting attached table {} to database {}", table_name, database_name);
         CurrentMetrics::add(getAttachedCounterForStorage(table));
     }

From 57a6d281000f0a49116db82e8b0b364990e61970 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 24 Jul 2024 11:17:43 +0200
Subject: [PATCH 0688/1488] Fix error

---
 src/IO/ReadWriteBufferFromHTTP.cpp | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/IO/ReadWriteBufferFromHTTP.cpp b/src/IO/ReadWriteBufferFromHTTP.cpp
index 85230957b3f..17a5ed385d4 100644
--- a/src/IO/ReadWriteBufferFromHTTP.cpp
+++ b/src/IO/ReadWriteBufferFromHTTP.cpp
@@ -132,6 +132,14 @@ std::optional<size_t> ReadWriteBufferFromHTTP::tryGetFileSize()
         {
             return std::nullopt;
         }
+        catch (const NetException &)
+        {
+            return std::nullopt;
+        }
+        catch (const Poco::Net::NetException &)
+        {
+            return std::nullopt;
+        }
     }
 
     return file_info->file_size;

From 0f9ee5c37d68f0877c0bc982c6bb59bf4803f98e Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 24 Jul 2024 11:25:08 +0200
Subject: [PATCH 0689/1488] Fix test "very_long_arrays"

---
 tests/queries/0_stateless/00186_very_long_arrays.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/00186_very_long_arrays.sh b/tests/queries/0_stateless/00186_very_long_arrays.sh
index 739b17ccc99..086303279fa 100755
--- a/tests/queries/0_stateless/00186_very_long_arrays.sh
+++ b/tests/queries/0_stateless/00186_very_long_arrays.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Tags: long
+# Tags: long, no-tsan
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh

From 48b30081265f85502b83c7bc5cc4beb9bfeaa3a1 Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Wed, 24 Jul 2024 12:04:28 +0200
Subject: [PATCH 0690/1488] Handle better static destructors

---
 src/Client/ClientBase.cpp                     | 13 +++++++---
 src/Common/FailPoint.cpp                      |  1 +
 src/Common/PipeFDs.cpp                        | 13 ++++++++--
 src/Common/PipeFDs.h                          |  3 ---
 src/Common/SignalHandlers.cpp                 |  9 ++++++-
 src/Common/StackTrace.cpp                     | 23 ++++++++++++++++-
 src/Daemon/BaseDaemon.cpp                     | 17 ++++++++++---
 src/Loggers/OwnSplitChannel.cpp               | 10 ++++++++
 src/Loggers/OwnSplitChannel.h                 |  2 ++
 .../__init__.py                               |  0
 .../test.py                                   | 25 +++++++++++++++++++
 11 files changed, 102 insertions(+), 14 deletions(-)
 create mode 100644 tests/integration/test_shutdown_static_destructor_failure/__init__.py
 create mode 100644 tests/integration/test_shutdown_static_destructor_failure/test.py

diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp
index 7af199131b6..7867cf32d24 100644
--- a/src/Client/ClientBase.cpp
+++ b/src/Client/ClientBase.cpp
@@ -309,9 +309,16 @@ public:
 
 ClientBase::~ClientBase()
 {
-    writeSignalIDtoSignalPipe(SignalListener::StopThread);
-    signal_listener_thread.join();
-    HandledSignals::instance().reset();
+    try
+    {
+        writeSignalIDtoSignalPipe(SignalListener::StopThread);
+        signal_listener_thread.join();
+        HandledSignals::instance().reset();
+    }
+    catch (...)
+    {
+        tryLogCurrentException(__PRETTY_FUNCTION__);
+    }
 }
 
 ClientBase::ClientBase(
diff --git a/src/Common/FailPoint.cpp b/src/Common/FailPoint.cpp
index 7b8b5036af0..f5ec8cf0356 100644
--- a/src/Common/FailPoint.cpp
+++ b/src/Common/FailPoint.cpp
@@ -59,6 +59,7 @@ static struct InitFiu
     ONCE(execute_query_calling_empty_set_result_func_on_exception) \
     ONCE(receive_timeout_on_table_status_response) \
     REGULAR(keepermap_fail_drop_data) \
+    REGULAR(lazy_pipe_fds_fail_close) \
 
 
 namespace FailPoints
diff --git a/src/Common/PipeFDs.cpp b/src/Common/PipeFDs.cpp
index ceadbb2f983..50eeda1bbe2 100644
--- a/src/Common/PipeFDs.cpp
+++ b/src/Common/PipeFDs.cpp
@@ -1,19 +1,23 @@
 #include <Common/PipeFDs.h>
 #include <Common/Exception.h>
 #include <Common/formatReadable.h>
+#include <Common/FailPoint.h>
 
 #include <Common/logger_useful.h>
 #include <base/errnoToString.h>
 
 #include <unistd.h>
 #include <fcntl.h>
-#include <string>
 #include <algorithm>
 
-
 namespace DB
 {
 
+namespace FailPoints
+{
+    extern const char lazy_pipe_fds_fail_close[];
+}
+
 namespace ErrorCodes
 {
     extern const int CANNOT_PIPE;
@@ -42,6 +46,11 @@ void LazyPipeFDs::open()
 
 void LazyPipeFDs::close()
 {
+    fiu_do_on(FailPoints::lazy_pipe_fds_fail_close,
+    {
+        throw Exception(ErrorCodes::CANNOT_PIPE, "Manually triggered exception on close");
+    });
+
     for (int & fd : fds_rw)
     {
         if (fd < 0)
diff --git a/src/Common/PipeFDs.h b/src/Common/PipeFDs.h
index 20bd847c077..b651176ee26 100644
--- a/src/Common/PipeFDs.h
+++ b/src/Common/PipeFDs.h
@@ -1,8 +1,5 @@
 #pragma once
 
-#include <cstddef>
-
-
 namespace DB
 {
 
diff --git a/src/Common/SignalHandlers.cpp b/src/Common/SignalHandlers.cpp
index a4b7784df5c..52c83d80121 100644
--- a/src/Common/SignalHandlers.cpp
+++ b/src/Common/SignalHandlers.cpp
@@ -605,7 +605,14 @@ void HandledSignals::reset()
 
 HandledSignals::~HandledSignals()
 {
-    reset();
+    try
+    {
+        reset();
+    }
+    catch (...)
+    {
+        tryLogCurrentException(__PRETTY_FUNCTION__);
+    }
 };
 
 HandledSignals & HandledSignals::instance()
diff --git a/src/Common/StackTrace.cpp b/src/Common/StackTrace.cpp
index 34f6f0b7535..b2ece02e2f0 100644
--- a/src/Common/StackTrace.cpp
+++ b/src/Common/StackTrace.cpp
@@ -4,6 +4,7 @@
 #include <base/constexpr_helpers.h>
 #include <base/demangle.h>
 
+#include "Common/PipeFDs.h"
 #include <Common/scope_guard_safe.h>
 #include <Common/Dwarf.h>
 #include <Common/Elf.h>
@@ -489,11 +490,24 @@ struct CacheEntry
 
 using CacheEntryPtr = std::shared_ptr<CacheEntry>;
 
-using StackTraceCache = std::map<StackTraceTriple, CacheEntryPtr, std::less<>>;
+static constinit std::atomic<bool> can_use_cache = false;
+
+using StackTraceCacheBase = std::map<StackTraceTriple, CacheEntryPtr, std::less<>>;
+
+struct StackTraceCache : public StackTraceCacheBase
+{
+    using StackTraceCacheBase::StackTraceCacheBase;
+
+    ~StackTraceCache()
+    {
+        can_use_cache = false;
+    }
+};
 
 static StackTraceCache & cacheInstance()
 {
     static StackTraceCache cache;
+    can_use_cache = true;
     return cache;
 }
 
@@ -503,6 +517,13 @@ String toStringCached(const StackTrace::FramePointers & pointers, size_t offset,
 {
     const StackTraceRefTriple key{pointers, offset, size};
 
+    if (!can_use_cache)
+    {
+        DB::WriteBufferFromOwnString out;
+        toStringEveryLineImpl(false, key, [&](std::string_view str) { out << str << '\n'; });
+        return out.str();
+    }
+
     /// Calculation of stack trace text is extremely slow.
     /// We use cache because otherwise the server could be overloaded by trash queries.
     /// Note that this cache can grow unconditionally, but practically it should be small.
diff --git a/src/Daemon/BaseDaemon.cpp b/src/Daemon/BaseDaemon.cpp
index d77878415db..366aad00376 100644
--- a/src/Daemon/BaseDaemon.cpp
+++ b/src/Daemon/BaseDaemon.cpp
@@ -146,10 +146,19 @@ BaseDaemon::BaseDaemon() = default;
 
 BaseDaemon::~BaseDaemon()
 {
-    writeSignalIDtoSignalPipe(SignalListener::StopThread);
-    signal_listener_thread.join();
-    HandledSignals::instance().reset();
-    SentryWriter::resetInstance();
+    try
+    {
+        writeSignalIDtoSignalPipe(SignalListener::StopThread);
+        signal_listener_thread.join();
+        HandledSignals::instance().reset();
+        SentryWriter::resetInstance();
+    }
+    catch (...)
+    {
+        tryLogCurrentException(&logger());
+    }
+
+    OwnSplitChannel::disableLogging();
 }
 
 
diff --git a/src/Loggers/OwnSplitChannel.cpp b/src/Loggers/OwnSplitChannel.cpp
index dc51a13e01f..c0e8514c62a 100644
--- a/src/Loggers/OwnSplitChannel.cpp
+++ b/src/Loggers/OwnSplitChannel.cpp
@@ -16,8 +16,18 @@
 namespace DB
 {
 
+static constinit std::atomic<bool> allow_logging{true};
+
+void OwnSplitChannel::disableLogging()
+{
+    allow_logging = false;
+}
+
 void OwnSplitChannel::log(const Poco::Message & msg)
 {
+    if (!allow_logging)
+        return;
+
 #ifndef WITHOUT_TEXT_LOG
     auto logs_queue = CurrentThread::getInternalTextLogsQueue();
 
diff --git a/src/Loggers/OwnSplitChannel.h b/src/Loggers/OwnSplitChannel.h
index 88bb6b9ce76..9872a4fb558 100644
--- a/src/Loggers/OwnSplitChannel.h
+++ b/src/Loggers/OwnSplitChannel.h
@@ -39,6 +39,8 @@ public:
 
     void setLevel(const std::string & name, int level);
 
+    static void disableLogging();
+
 private:
     void logSplit(const Poco::Message & msg);
     void tryLogSplit(const Poco::Message & msg);
diff --git a/tests/integration/test_shutdown_static_destructor_failure/__init__.py b/tests/integration/test_shutdown_static_destructor_failure/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/integration/test_shutdown_static_destructor_failure/test.py b/tests/integration/test_shutdown_static_destructor_failure/test.py
new file mode 100644
index 00000000000..8f1d4423238
--- /dev/null
+++ b/tests/integration/test_shutdown_static_destructor_failure/test.py
@@ -0,0 +1,25 @@
+#!/usr/bin/env python3
+import pytest
+from helpers.cluster import ClickHouseCluster
+
+cluster = ClickHouseCluster(__file__)
+
+node = cluster.add_instance(
+    "node",
+    main_configs=[],
+    stay_alive=True
+)
+
+
+@pytest.fixture(scope="module", autouse=True)
+def start_cluster():
+    try:
+        cluster.start()
+        yield cluster
+    finally:
+        cluster.shutdown()
+
+
+def test_shutdown():
+    node.query("SYSTEM ENABLE FAILPOINT lazy_pipe_fds_fail_close")
+    node.stop_clickhouse()

From e7b5c0ea19a64157f7bc743f81cf148cf3fdb4eb Mon Sep 17 00:00:00 2001
From: robot-clickhouse <robot-clickhouse@users.noreply.github.com>
Date: Wed, 24 Jul 2024 10:14:25 +0000
Subject: [PATCH 0691/1488] Automatic style fix

---
 .../test_shutdown_static_destructor_failure/test.py         | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/tests/integration/test_shutdown_static_destructor_failure/test.py b/tests/integration/test_shutdown_static_destructor_failure/test.py
index 8f1d4423238..b1d925cc432 100644
--- a/tests/integration/test_shutdown_static_destructor_failure/test.py
+++ b/tests/integration/test_shutdown_static_destructor_failure/test.py
@@ -4,11 +4,7 @@ from helpers.cluster import ClickHouseCluster
 
 cluster = ClickHouseCluster(__file__)
 
-node = cluster.add_instance(
-    "node",
-    main_configs=[],
-    stay_alive=True
-)
+node = cluster.add_instance("node", main_configs=[], stay_alive=True)
 
 
 @pytest.fixture(scope="module", autouse=True)

From 8c19d502f861ca844501fa505481a1b64684a8ec Mon Sep 17 00:00:00 2001
From: divanik <dan.ivanik@clickhouse.com>
Date: Wed, 24 Jul 2024 10:44:06 +0000
Subject: [PATCH 0692/1488] Add decorator and retries for azurite

---
 tests/integration/helpers/cluster.py         | 22 ++++++------
 tests/integration/helpers/retry_decorator.py | 36 ++++++++++++++++++++
 2 files changed, 48 insertions(+), 10 deletions(-)
 create mode 100644 tests/integration/helpers/retry_decorator.py

diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py
index 548b58a17e8..3914192f9af 100644
--- a/tests/integration/helpers/cluster.py
+++ b/tests/integration/helpers/cluster.py
@@ -52,6 +52,7 @@ from helpers.client import QueryRuntimeException
 import docker
 
 from .client import Client
+from .retry_decorator import retry
 
 from .config_cluster import *
 
@@ -2690,15 +2691,12 @@ class ClickHouseCluster:
 
             images_pull_cmd = self.base_cmd + ["pull"]
             # sometimes dockerhub/proxy can be flaky
-            for i in range(5):
-                try:
-                    run_and_check(images_pull_cmd)
-                    break
-                except Exception as ex:
-                    if i == 4:
-                        raise ex
-                    logging.info("Got exception pulling images: %s", ex)
-                    time.sleep(i * 3)
+
+            retry(
+                log_function=lambda exception: logging.info(
+                    "Got exception pulling images: %s", exception
+                ),
+            )(run_and_check)(images_pull_cmd)
 
             if self.with_zookeeper_secure and self.base_zookeeper_cmd:
                 logging.debug("Setup ZooKeeper Secure")
@@ -2971,7 +2969,11 @@ class ClickHouseCluster:
                     "Trying to create Azurite instance by command %s",
                     " ".join(map(str, azurite_start_cmd)),
                 )
-                run_and_check(azurite_start_cmd)
+                retry(
+                    log_function=lambda exception: logginnfo(
+                        f"Azurite initialization failed with error: {exception}"
+                    ),
+                )(run_and_check)(azurite_start_cmd)
                 self.up_called = True
                 logging.info("Trying to connect to Azurite")
                 self.wait_azurite_to_start()
diff --git a/tests/integration/helpers/retry_decorator.py b/tests/integration/helpers/retry_decorator.py
new file mode 100644
index 00000000000..aaa040464c2
--- /dev/null
+++ b/tests/integration/helpers/retry_decorator.py
@@ -0,0 +1,36 @@
+import time
+import random
+from typing import Type, List
+
+
+def retry(
+    retries: int = 5,
+    delay: float = 1,
+    backoff: float = 1.5,
+    jitter: float = 2,
+    log_function=lambda *args, **kwargs: None,
+    retriable_expections_list: List[Type[BaseException]] = [Exception],
+):
+    def inner(func):
+        def wrapper(*args, **kwargs):
+            current_delay = delay
+            for retry in range(retries):
+                try:
+                    func(*args, **kwargs)
+                    break
+                except Exception as e:
+                    should_retry = False
+                    for retriable_exception in retriable_expections_list:
+                        if isinstance(e, retriable_exception):
+                            should_retry = True
+                            break
+                    if not should_retry or (retry == retries - 1):
+                        raise e
+                    log_function(retry=retry, exception=e)
+                    sleep_time = current_delay + random.uniform(0, jitter)
+                    time.sleep(sleep_time)
+                    current_delay *= backoff
+
+        return wrapper
+
+    return inner

From 1b95a68dc08266c50b2dafb5c852b84431723112 Mon Sep 17 00:00:00 2001
From: divanik <dan.ivanik@clickhouse.com>
Date: Wed, 24 Jul 2024 10:48:18 +0000
Subject: [PATCH 0693/1488] Fix typo

---
 tests/integration/helpers/cluster.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py
index 3914192f9af..dfdeddb75cf 100644
--- a/tests/integration/helpers/cluster.py
+++ b/tests/integration/helpers/cluster.py
@@ -2969,8 +2969,8 @@ class ClickHouseCluster:
                     "Trying to create Azurite instance by command %s",
                     " ".join(map(str, azurite_start_cmd)),
                 )
-                retry(
-                    log_function=lambda exception: logginnfo(
+                retry_decorator.retry(
+                    log_function=lambda exception: logging.info(
                         f"Azurite initialization failed with error: {exception}"
                     ),
                 )(run_and_check)(azurite_start_cmd)

From f21a9e5d0892315ba79d8f10bd9bb6ef271a3aa9 Mon Sep 17 00:00:00 2001
From: divanik <dan.ivanik@clickhouse.com>
Date: Wed, 24 Jul 2024 11:00:49 +0000
Subject: [PATCH 0694/1488] Fix bug

---
 tests/integration/helpers/cluster.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py
index dfdeddb75cf..7d80fbe90f8 100644
--- a/tests/integration/helpers/cluster.py
+++ b/tests/integration/helpers/cluster.py
@@ -2969,7 +2969,7 @@ class ClickHouseCluster:
                     "Trying to create Azurite instance by command %s",
                     " ".join(map(str, azurite_start_cmd)),
                 )
-                retry_decorator.retry(
+                retry(
                     log_function=lambda exception: logging.info(
                         f"Azurite initialization failed with error: {exception}"
                     ),

From 2ea10d99940132828c61457d62a54b77a7a66af2 Mon Sep 17 00:00:00 2001
From: Nikita Fomichev <nikita.fomichev@clickhouse.com>
Date: Wed, 24 Jul 2024 13:13:58 +0200
Subject: [PATCH 0695/1488] Integration tests: fix flaky tests

---
 .../test_concurrency.py                               | 11 +++--------
 tests/integration/test_manipulate_statistics/test.py  |  6 ++++--
 2 files changed, 7 insertions(+), 10 deletions(-)

diff --git a/tests/integration/test_backup_restore_on_cluster/test_concurrency.py b/tests/integration/test_backup_restore_on_cluster/test_concurrency.py
index c08f3c9c242..d3caadd0b46 100644
--- a/tests/integration/test_backup_restore_on_cluster/test_concurrency.py
+++ b/tests/integration/test_backup_restore_on_cluster/test_concurrency.py
@@ -276,15 +276,10 @@ def test_create_or_drop_tables_during_backup(db_engine, table_engine):
     for node in nodes:
         assert_eq_with_retry(
             node,
-            f"SELECT status from system.backups WHERE id IN {ids_list} AND (status == 'CREATING_BACKUP')",
-            "",
-        )
-
-    for node in nodes:
-        assert_eq_with_retry(
-            node,
-            f"SELECT status, error from system.backups WHERE id IN {ids_list} AND (status == 'BACKUP_FAILED')",
+            f"SELECT status from system.backups "
+            f"WHERE id IN {ids_list} AND ((status == 'CREATING_BACKUP') OR (status == 'BACKUP_FAILED'))",
             "",
+            retry_count=100,
         )
 
     backup_names = {}
diff --git a/tests/integration/test_manipulate_statistics/test.py b/tests/integration/test_manipulate_statistics/test.py
index a602cce63df..2541c9b946f 100644
--- a/tests/integration/test_manipulate_statistics/test.py
+++ b/tests/integration/test_manipulate_statistics/test.py
@@ -167,7 +167,8 @@ def test_replicated_table_ddl(started_cluster):
     check_stat_file_on_disk(node2, "test_stat", "all_0_0_0_1", "c", False)
     check_stat_file_on_disk(node2, "test_stat", "all_0_0_0_1", "d", True)
     node1.query(
-        "ALTER TABLE test_stat CLEAR STATISTICS d", settings={"alter_sync": "2"}
+        "ALTER TABLE test_stat CLEAR STATISTICS d",
+        settings={"alter_sync": "2", "mutations_sync": 2},
     )
     node1.query(
         "ALTER TABLE test_stat ADD STATISTICS b type tdigest",
@@ -177,7 +178,8 @@ def test_replicated_table_ddl(started_cluster):
     check_stat_file_on_disk(node2, "test_stat", "all_0_0_0_2", "b", False)
     check_stat_file_on_disk(node2, "test_stat", "all_0_0_0_2", "d", False)
     node1.query(
-        "ALTER TABLE test_stat MATERIALIZE STATISTICS b", settings={"alter_sync": "2"}
+        "ALTER TABLE test_stat MATERIALIZE STATISTICS b",
+        settings={"alter_sync": "2", "mutations_sync": 2},
     )
     check_stat_file_on_disk(node2, "test_stat", "all_0_0_0_3", "a", True)
     check_stat_file_on_disk(node2, "test_stat", "all_0_0_0_3", "b", True)

From d507adf9a7a524706b8247026acfbb29d8467e2b Mon Sep 17 00:00:00 2001
From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com>
Date: Wed, 24 Jul 2024 11:33:31 +0000
Subject: [PATCH 0696/1488] fix

---
 tests/integration/test_backup_restore_new/test.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/integration/test_backup_restore_new/test.py b/tests/integration/test_backup_restore_new/test.py
index a632e18a650..4806625f3f0 100644
--- a/tests/integration/test_backup_restore_new/test.py
+++ b/tests/integration/test_backup_restore_new/test.py
@@ -1239,6 +1239,7 @@ def test_system_users_required_privileges():
 
     instance.query("DROP USER u1")
     instance.query("DROP ROLE r1")
+    instance.query("DROP USER u2")
 
 
 def test_system_users_async():

From 595cf9a6a81f034a552d2b0e365bda87e0a7a9f5 Mon Sep 17 00:00:00 2001
From: Alexander Tokmakov <tavplubix@gmail.com>
Date: Wed, 24 Jul 2024 13:42:32 +0200
Subject: [PATCH 0697/1488] Revert "Revert "Fix for 992 and friends""

---
 src/Storages/StorageReplicatedMergeTree.cpp            |  2 +-
 ...00992_system_parts_race_condition_zookeeper_long.sh |  2 +-
 tests/queries/0_stateless/replication.lib              | 10 +++++++++-
 3 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp
index 3f02486ed15..da379a466af 100644
--- a/src/Storages/StorageReplicatedMergeTree.cpp
+++ b/src/Storages/StorageReplicatedMergeTree.cpp
@@ -3940,7 +3940,7 @@ void StorageReplicatedMergeTree::mergeSelectingTask()
         merge_selecting_task->schedule();
     else
     {
-        LOG_TRACE(log, "Scheduling next merge selecting task after {}ms", merge_selecting_sleep_ms);
+        LOG_TRACE(log, "Scheduling next merge selecting task after {}ms, current attempt status: {}", merge_selecting_sleep_ms, result);
         merge_selecting_task->scheduleAfter(merge_selecting_sleep_ms);
     }
 }
diff --git a/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper_long.sh b/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper_long.sh
index 4887c409844..02a739ece4a 100755
--- a/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper_long.sh
+++ b/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper_long.sh
@@ -41,7 +41,7 @@ function thread3()
 
 function thread4()
 {
-    while true; do $CLICKHOUSE_CLIENT --receive_timeout=3 -q "OPTIMIZE TABLE alter_table0 FINAL" | grep -Fv "Timeout exceeded while receiving data from server"; done
+    while true; do $CLICKHOUSE_CLIENT --receive_timeout=1 -q "OPTIMIZE TABLE alter_table0 FINAL" | grep -Fv "Timeout exceeded while receiving data from server"; done
 }
 
 function thread5()
diff --git a/tests/queries/0_stateless/replication.lib b/tests/queries/0_stateless/replication.lib
index 1a86cd9f8db..fe867537000 100755
--- a/tests/queries/0_stateless/replication.lib
+++ b/tests/queries/0_stateless/replication.lib
@@ -51,6 +51,14 @@ function check_replication_consistency()
     table_name_prefix=$1
     check_query_part=$2
 
+    # Try to kill some mutations because sometimes tests run too much (it's not guarenteed to kill all mutations, see below)
+    # Try multiple replicas, because queries are not finished yet, and "global" KILL MUTATION may fail due to another query (like DROP TABLE)
+    readarray -t tables_arr < <(${CLICKHOUSE_CLIENT} -q "SELECT name FROM system.tables WHERE database=currentDatabase() AND name like '$table_name_prefix%'")
+    for t in "${tables_arr[@]}"
+    do
+        ${CLICKHOUSE_CLIENT} -q "KILL MUTATION WHERE database=currentDatabase() AND table='$t'" > /dev/null 2>/dev/null
+    done
+
     # Wait for all queries to finish (query may still be running if thread is killed by timeout)
     num_tries=0
     while [[ $($CLICKHOUSE_CLIENT -q "SELECT count() FROM system.processes WHERE current_database=currentDatabase() AND query LIKE '%$table_name_prefix%'") -ne 1 ]]; do
@@ -96,7 +104,7 @@ function check_replication_consistency()
     some_table=$($CLICKHOUSE_CLIENT -q "SELECT name FROM system.tables WHERE database=currentDatabase() AND name like '$table_name_prefix%' ORDER BY rand() LIMIT 1")
     $CLICKHOUSE_CLIENT -q "SYSTEM SYNC REPLICA $some_table PULL" 1>/dev/null 2>/dev/null ||:
 
-    # Forcefully cancel mutations to avoid waiting for them to finish
+    # Forcefully cancel mutations to avoid waiting for them to finish. Kills the remaining mutations
     ${CLICKHOUSE_CLIENT} -q "KILL MUTATION WHERE database=currentDatabase() AND table like '$table_name_prefix%'" > /dev/null
 
     # SYNC REPLICA is not enough if some MUTATE_PARTs are not assigned yet

From b25cad23ed3b90dc8c0903710dba0714bac7219c Mon Sep 17 00:00:00 2001
From: Pablo Marcos <pablo.marcos.oltra@clickhouse.com>
Date: Wed, 24 Jul 2024 11:42:28 +0000
Subject: [PATCH 0698/1488] Use unique names for tables and files

---
 .../integration/test_storage_s3_queue/test.py | 29 ++++++++++---------
 1 file changed, 16 insertions(+), 13 deletions(-)

diff --git a/tests/integration/test_storage_s3_queue/test.py b/tests/integration/test_storage_s3_queue/test.py
index bf3c28c5429..cf24e91f36b 100644
--- a/tests/integration/test_storage_s3_queue/test.py
+++ b/tests/integration/test_storage_s3_queue/test.py
@@ -7,6 +7,7 @@ import pytest
 from helpers.client import QueryRuntimeException
 from helpers.cluster import ClickHouseCluster, ClickHouseInstance
 import json
+from uuid import uuid4
 
 
 AVAILABLE_MODES = ["unordered", "ordered"]
@@ -822,7 +823,7 @@ def test_multiple_tables_streaming_sync_distributed(started_cluster, mode):
 
 def test_max_set_age(started_cluster):
     node = started_cluster.instances["instance"]
-    table_name = f"max_set_age"
+    table_name = f"max_set_age_{uuid4().hex}"
     dst_table_name = f"{table_name}_dst"
     keeper_path = f"/clickhouse/test_{table_name}"
     files_path = f"{table_name}_data"
@@ -847,11 +848,11 @@ def test_max_set_age(started_cluster):
     )
     create_mv(node, table_name, dst_table_name)
 
-    total_values = generate_random_files(
+    _ = generate_random_files(
         started_cluster, files_path, files_to_generate, row_num=1
     )
 
-    expected_rows = 10
+    expected_rows = files_to_generate
 
     node.wait_for_log_line("Checking node limits")
     node.wait_for_log_line("Node limits check finished")
@@ -865,11 +866,11 @@ def test_max_set_age(started_cluster):
         time.sleep(1)
 
     assert expected_rows == get_count()
-    assert 10 == int(node.query(f"SELECT uniq(_path) from {dst_table_name}"))
+    assert files_to_generate == int(node.query(f"SELECT uniq(_path) from {dst_table_name}"))
 
     time.sleep(max_age + 5)
 
-    expected_rows = 20
+    expected_rows *= 2
 
     for _ in range(20):
         if expected_rows == get_count():
@@ -877,7 +878,7 @@ def test_max_set_age(started_cluster):
         time.sleep(1)
 
     assert expected_rows == get_count()
-    assert 10 == int(node.query(f"SELECT uniq(_path) from {dst_table_name}"))
+    assert files_to_generate == int(node.query(f"SELECT uniq(_path) from {dst_table_name}"))
 
     paths_count = [
         int(x)
@@ -885,7 +886,7 @@ def test_max_set_age(started_cluster):
             f"SELECT count() from {dst_table_name} GROUP BY _path"
         ).splitlines()
     ]
-    assert 10 == len(paths_count)
+    assert files_to_generate == len(paths_count)
     for path_count in paths_count:
         assert 2 == path_count
 
@@ -901,7 +902,8 @@ def test_max_set_age(started_cluster):
     values_csv = (
         "\n".join((",".join(map(str, row)) for row in values)) + "\n"
     ).encode()
-    put_s3_file_content(started_cluster, f"{files_path}/fff.csv", values_csv)
+    file_with_error = f"fff_{uuid4().hex}.csv"
+    put_s3_file_content(started_cluster, f"{files_path}/{file_with_error}", values_csv)
 
     for _ in range(30):
         if failed_count + 1 == int(
@@ -920,16 +922,17 @@ def test_max_set_age(started_cluster):
 
     node.query("SYSTEM FLUSH LOGS")
     assert "Cannot parse input" in node.query(
-        "SELECT exception FROM system.s3queue WHERE file_name ilike '%fff.csv'"
+        f"SELECT exception FROM system.s3queue WHERE file_name ilike '%{file_with_error}'"
     )
+
     assert 1 == int(
         node.query(
-            "SELECT count() FROM system.s3queue_log WHERE file_name ilike '%fff.csv'"
+            f"SELECT count() FROM system.s3queue_log WHERE file_name ilike '%{file_with_error}'"
         )
     )
     assert 1 == int(
         node.query(
-            "SELECT count() FROM system.s3queue_log WHERE file_name ilike '%fff.csv' AND notEmpty(exception)"
+            f"SELECT count() FROM system.s3queue_log WHERE file_name ilike '%{file_with_error}' AND notEmpty(exception)"
         )
     )
 
@@ -943,11 +946,11 @@ def test_max_set_age(started_cluster):
 
     node.query("SYSTEM FLUSH LOGS")
     assert "Cannot parse input" in node.query(
-        "SELECT exception FROM system.s3queue WHERE file_name ilike '%fff.csv' ORDER BY processing_end_time DESC LIMIT 1"
+        f"SELECT exception FROM system.s3queue WHERE file_name ilike '%{file_with_error}' ORDER BY processing_end_time DESC LIMIT 1"
     )
     assert 1 < int(
         node.query(
-            "SELECT count() FROM system.s3queue_log WHERE file_name ilike '%fff.csv' AND notEmpty(exception)"
+            f"SELECT count() FROM system.s3queue_log WHERE file_name ilike '%{file_with_error}' AND notEmpty(exception)"
         )
     )
 

From 3f992e4e08c326857c6dec7c5d45fb46154280b7 Mon Sep 17 00:00:00 2001
From: Alexander Tokmakov <tavplubix@clickhouse.com>
Date: Wed, 24 Jul 2024 13:43:13 +0200
Subject: [PATCH 0699/1488] Update replication.lib

---
 tests/queries/0_stateless/replication.lib | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/replication.lib b/tests/queries/0_stateless/replication.lib
index fe867537000..05651531fba 100755
--- a/tests/queries/0_stateless/replication.lib
+++ b/tests/queries/0_stateless/replication.lib
@@ -56,7 +56,7 @@ function check_replication_consistency()
     readarray -t tables_arr < <(${CLICKHOUSE_CLIENT} -q "SELECT name FROM system.tables WHERE database=currentDatabase() AND name like '$table_name_prefix%'")
     for t in "${tables_arr[@]}"
     do
-        ${CLICKHOUSE_CLIENT} -q "KILL MUTATION WHERE database=currentDatabase() AND table='$t'" > /dev/null 2>/dev/null
+        ${CLICKHOUSE_CLIENT} -q "KILL MUTATION WHERE database=currentDatabase() AND table='$t'" > /dev/null 2>/dev/null ||:
     done
 
     # Wait for all queries to finish (query may still be running if thread is killed by timeout)

From 9aaf806021873f010b5d6fda9061ad836de54e36 Mon Sep 17 00:00:00 2001
From: Nikita Fomichev <nikita.fomichev@clickhouse.com>
Date: Wed, 24 Jul 2024 14:03:40 +0200
Subject: [PATCH 0700/1488] Integration tests: fix flaky tests 2

---
 .../test_backup_restore_on_cluster/test_concurrency.py          | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/test_backup_restore_on_cluster/test_concurrency.py b/tests/integration/test_backup_restore_on_cluster/test_concurrency.py
index d3caadd0b46..0e381d48fb1 100644
--- a/tests/integration/test_backup_restore_on_cluster/test_concurrency.py
+++ b/tests/integration/test_backup_restore_on_cluster/test_concurrency.py
@@ -276,7 +276,7 @@ def test_create_or_drop_tables_during_backup(db_engine, table_engine):
     for node in nodes:
         assert_eq_with_retry(
             node,
-            f"SELECT status from system.backups "
+            f"SELECT status, error from system.backups "
             f"WHERE id IN {ids_list} AND ((status == 'CREATING_BACKUP') OR (status == 'BACKUP_FAILED'))",
             "",
             retry_count=100,

From 09cb2d822a0960f9917f28d36c47c1e67e861178 Mon Sep 17 00:00:00 2001
From: Nikita Fomichev <nikita.fomichev@clickhouse.com>
Date: Wed, 24 Jul 2024 14:05:50 +0200
Subject: [PATCH 0701/1488] Change description for setting:
 `throw_if_deduplication_in_dependent_materialized_views_enabled_with_async_insert`

---
 src/Core/SettingsChangesHistory.cpp | 2 +-
 src/Interpreters/executeQuery.cpp   | 6 +++---
 src/Server/TCPHandler.cpp           | 4 ++--
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp
index de4725dc350..38ad88cfa1d 100644
--- a/src/Core/SettingsChangesHistory.cpp
+++ b/src/Core/SettingsChangesHistory.cpp
@@ -147,7 +147,7 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
               {"default_table_engine", "None", "MergeTree", "Set default table engine to MergeTree for better usability"},
               {"input_format_json_use_string_type_for_ambiguous_paths_in_named_tuples_inference_from_objects", false, false, "Allow to use String type for ambiguous paths during named tuple inference from JSON objects"},
               {"traverse_shadow_remote_data_paths", false, false, "Traverse shadow directory when query system.remote_data_paths."},
-              {"throw_if_deduplication_in_dependent_materialized_views_enabled_with_async_insert", false, true, "Deduplication is dependent materialized view cannot work together with async inserts."},
+              {"throw_if_deduplication_in_dependent_materialized_views_enabled_with_async_insert", false, true, "Deduplication in dependent materialized view cannot work together with async inserts."},
               {"parallel_replicas_allow_in_with_subquery", false, true, "If true, subquery for IN will be executed on every follower replica"},
               {"log_processors_profiles", false, true, "Enable by default"},
               {"function_locate_has_mysql_compatible_argument_order", false, true, "Increase compatibility with MySQL's locate function."},
diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp
index d9d3ba58160..ba6fc0f14a0 100644
--- a/src/Interpreters/executeQuery.cpp
+++ b/src/Interpreters/executeQuery.cpp
@@ -1048,14 +1048,14 @@ static std::tuple<ASTPtr, BlockIO> executeQueryImpl(
             /// In case when the client had to retry some mini-INSERTs then they will be properly deduplicated
             /// by the source tables. This functionality is controlled by a setting `async_insert_deduplicate`.
             /// But then they will be glued together into a block and pushed through a chain of Materialized Views if any.
-            /// The process of forming such blocks is not deteministic so each time we retry mini-INSERTs the resulting
+            /// The process of forming such blocks is not deterministic so each time we retry mini-INSERTs the resulting
             /// block may be concatenated differently.
             /// That's why deduplication in dependent Materialized Views doesn't make sense in presence of async INSERTs.
             if (settings.throw_if_deduplication_in_dependent_materialized_views_enabled_with_async_insert &&
                 settings.deduplicate_blocks_in_dependent_materialized_views)
                 throw Exception(ErrorCodes::SUPPORT_IS_DISABLED,
-                        "Deduplication is dependent materialized view cannot work together with async inserts. "\
-                        "Please disable eiher `deduplicate_blocks_in_dependent_materialized_views` or `async_insert` setting.");
+                        "Deduplication in dependent materialized view cannot work together with async inserts. "\
+                        "Please disable either `deduplicate_blocks_in_dependent_materialized_views` or `async_insert` setting.");
 
             quota = context->getQuota();
             if (quota)
diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp
index 5bc2d09df35..42dc7f5a78f 100644
--- a/src/Server/TCPHandler.cpp
+++ b/src/Server/TCPHandler.cpp
@@ -961,8 +961,8 @@ void TCPHandler::processInsertQuery()
         if (settings.throw_if_deduplication_in_dependent_materialized_views_enabled_with_async_insert &&
             settings.deduplicate_blocks_in_dependent_materialized_views)
             throw Exception(ErrorCodes::SUPPORT_IS_DISABLED,
-                    "Deduplication is dependent materialized view cannot work together with async inserts. "\
-                    "Please disable eiher `deduplicate_blocks_in_dependent_materialized_views` or `async_insert` setting.");
+                    "Deduplication in dependent materialized view cannot work together with async inserts. "\
+                    "Please disable either `deduplicate_blocks_in_dependent_materialized_views` or `async_insert` setting.");
 
         auto result = processAsyncInsertQuery(*insert_queue);
         if (result.status == AsynchronousInsertQueue::PushResult::OK)

From 91b7001df6e827f801bd792e7bd9d96cdd947946 Mon Sep 17 00:00:00 2001
From: Pablo Marcos <pablo.marcos.oltra@clickhouse.com>
Date: Wed, 24 Jul 2024 12:08:31 +0000
Subject: [PATCH 0702/1488] Refactor test to improve it

- Create wait_for_condition that checks greedily for a period of time
- Remove redundant checks
- Allow other tests running in parallel to have `ObjectStorageQueueFailedFiles` errors
---
 .../integration/test_storage_s3_queue/test.py | 56 ++++++-------------
 1 file changed, 16 insertions(+), 40 deletions(-)

diff --git a/tests/integration/test_storage_s3_queue/test.py b/tests/integration/test_storage_s3_queue/test.py
index cf24e91f36b..e178b3b6608 100644
--- a/tests/integration/test_storage_s3_queue/test.py
+++ b/tests/integration/test_storage_s3_queue/test.py
@@ -860,24 +860,21 @@ def test_max_set_age(started_cluster):
     def get_count():
         return int(node.query(f"SELECT count() FROM {dst_table_name}"))
 
-    for _ in range(20):
-        if expected_rows == get_count():
-            break
-        time.sleep(1)
+    def wait_for_condition(check_function, max_wait_time=30):
+        before = time.time()
+        while time.time() - before < max_wait_time:
+            if check_function():
+                return
+            time.sleep(0.1)
+        assert False
 
-    assert expected_rows == get_count()
+    wait_for_condition(lambda: get_count() == expected_rows)
     assert files_to_generate == int(node.query(f"SELECT uniq(_path) from {dst_table_name}"))
 
     time.sleep(max_age + 5)
 
     expected_rows *= 2
-
-    for _ in range(20):
-        if expected_rows == get_count():
-            break
-        time.sleep(1)
-
-    assert expected_rows == get_count()
+    wait_for_condition(lambda: get_count() == expected_rows)
     assert files_to_generate == int(node.query(f"SELECT uniq(_path) from {dst_table_name}"))
 
     paths_count = [
@@ -890,11 +887,12 @@ def test_max_set_age(started_cluster):
     for path_count in paths_count:
         assert 2 == path_count
 
-    failed_count = int(
-        node.query(
+    def get_object_storage_failures():
+        return int(node.query(
             "SELECT value FROM system.events WHERE name = 'ObjectStorageQueueFailedFiles' SETTINGS system_events_show_zero_values=1"
-        )
-    )
+        ))
+
+    failed_count = get_object_storage_failures()
 
     values = [
         ["failed", 1, 1],
@@ -905,31 +903,13 @@ def test_max_set_age(started_cluster):
     file_with_error = f"fff_{uuid4().hex}.csv"
     put_s3_file_content(started_cluster, f"{files_path}/{file_with_error}", values_csv)
 
-    for _ in range(30):
-        if failed_count + 1 == int(
-            node.query(
-                "SELECT value FROM system.events WHERE name = 'ObjectStorageQueueFailedFiles' SETTINGS system_events_show_zero_values=1"
-            )
-        ):
-            break
-        time.sleep(1)
-
-    assert failed_count + 1 == int(
-        node.query(
-            "SELECT value FROM system.events WHERE name = 'ObjectStorageQueueFailedFiles' SETTINGS system_events_show_zero_values=1"
-        )
-    )
+    wait_for_condition(lambda: failed_count + 1 <= get_object_storage_failures())
 
     node.query("SYSTEM FLUSH LOGS")
     assert "Cannot parse input" in node.query(
         f"SELECT exception FROM system.s3queue WHERE file_name ilike '%{file_with_error}'"
     )
 
-    assert 1 == int(
-        node.query(
-            f"SELECT count() FROM system.s3queue_log WHERE file_name ilike '%{file_with_error}'"
-        )
-    )
     assert 1 == int(
         node.query(
             f"SELECT count() FROM system.s3queue_log WHERE file_name ilike '%{file_with_error}' AND notEmpty(exception)"
@@ -938,11 +918,7 @@ def test_max_set_age(started_cluster):
 
     time.sleep(max_age + 1)
 
-    assert failed_count + 2 == int(
-        node.query(
-            "SELECT value FROM system.events WHERE name = 'ObjectStorageQueueFailedFiles'"
-        )
-    )
+    assert failed_count + 2 <= get_object_storage_failures()
 
     node.query("SYSTEM FLUSH LOGS")
     assert "Cannot parse input" in node.query(

From d5ea07be2ef2e61edf95408dec70a8e1bc66cafa Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Wed, 24 Jul 2024 14:20:03 +0200
Subject: [PATCH 0703/1488] Bump


From 01ce22049a76995dc00974618c94af9ccbcc30db Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 24 Jul 2024 14:29:14 +0200
Subject: [PATCH 0704/1488] Fix tests

---
 src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp  | 11 +++++------
 .../MySQL/tests/gtest_create_rewritten.cpp            |  6 ++----
 .../test_postgresql_replica_database_engine_2/test.py |  2 +-
 3 files changed, 8 insertions(+), 11 deletions(-)

diff --git a/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp b/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp
index f73965cfcc8..3917ffb8823 100644
--- a/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp
+++ b/src/Interpreters/MySQL/InterpretersMySQLDDLQuery.cpp
@@ -17,7 +17,6 @@
 #include <Parsers/MySQL/ASTCreateDefines.h>
 
 #include <DataTypes/DataTypeFactory.h>
-#include <DataTypes/DataTypesNumber.h>
 #include <DataTypes/DataTypeNullable.h>
 #include <Parsers/MySQL/ASTDeclareIndex.h>
 #include <Common/quoteString.h>
@@ -158,7 +157,7 @@ static ColumnsDescription createColumnsDescription(const NamesAndTypesList & col
     /// (see git blame for details).
     auto column_name_and_type = columns_name_and_type.begin();
     const auto * declare_column_ast = columns_definition->children.begin();
-    for (; column_name_and_type != columns_name_and_type.end(); column_name_and_type++, declare_column_ast++)
+    for (; column_name_and_type != columns_name_and_type.end(); ++column_name_and_type, ++declare_column_ast)
     {
         const auto & declare_column = (*declare_column_ast)->as<MySQLParser::ASTDeclareColumn>();
         String comment;
@@ -177,7 +176,7 @@ static ColumnsDescription createColumnsDescription(const NamesAndTypesList & col
     return columns_description;
 }
 
-static NamesAndTypesList getNames(const ASTDataType & expr, ContextPtr context, const NamesAndTypesList & columns)
+static NamesAndTypesList getNames(const ASTFunction & expr, ContextPtr context, const NamesAndTypesList & columns)
 {
     if (expr.arguments->children.empty())
         return NamesAndTypesList{};
@@ -221,9 +220,9 @@ static std::tuple<NamesAndTypesList, NamesAndTypesList, NamesAndTypesList, NameS
     ASTExpressionList * columns_definition, ASTExpressionList * indices_define, ContextPtr context, NamesAndTypesList & columns)
 {
     NameSet increment_columns;
-    auto keys = makeASTDataType("tuple");
-    auto unique_keys = makeASTDataType("tuple");
-    auto primary_keys = makeASTDataType("tuple");
+    auto keys = makeASTFunction("tuple");
+    auto unique_keys = makeASTFunction("tuple");
+    auto primary_keys = makeASTFunction("tuple");
 
     if (indices_define && !indices_define->children.empty())
     {
diff --git a/src/Interpreters/MySQL/tests/gtest_create_rewritten.cpp b/src/Interpreters/MySQL/tests/gtest_create_rewritten.cpp
index 6d6077a0295..81e6e6a8761 100644
--- a/src/Interpreters/MySQL/tests/gtest_create_rewritten.cpp
+++ b/src/Interpreters/MySQL/tests/gtest_create_rewritten.cpp
@@ -2,12 +2,10 @@
 
 #include <gtest/gtest.h>
 
-#include <Parsers/IAST.h>
 #include <Parsers/queryToString.h>
 #include <Parsers/ASTExternalDDLQuery.h>
 #include <Parsers/ParserExternalDDLQuery.h>
 #include <Parsers/parseQuery.h>
-#include <Interpreters/Context.h>
 #include <Interpreters/MySQL/InterpretersMySQLDDLQuery.h>
 #include <Common/tests/gtest_global_context.h>
 #include <Common/tests/gtest_global_register.h>
@@ -26,8 +24,8 @@ static inline ASTPtr tryRewrittenCreateQuery(const String & query, ContextPtr co
         context, "test_database", "test_database")[0];
 }
 
-static const char MATERIALIZEDMYSQL_TABLE_COLUMNS[] = ", `_sign` Int8() MATERIALIZED 1"
-                                                     ", `_version` UInt64() MATERIALIZED 1"
+static const char MATERIALIZEDMYSQL_TABLE_COLUMNS[] = ", `_sign` Int8 MATERIALIZED 1"
+                                                     ", `_version` UInt64 MATERIALIZED 1"
                                                      ", INDEX _version _version TYPE minmax GRANULARITY 1";
 
 TEST(MySQLCreateRewritten, ColumnsDataType)
diff --git a/tests/integration/test_postgresql_replica_database_engine_2/test.py b/tests/integration/test_postgresql_replica_database_engine_2/test.py
index 5e04c9e4d12..406b50bc486 100644
--- a/tests/integration/test_postgresql_replica_database_engine_2/test.py
+++ b/tests/integration/test_postgresql_replica_database_engine_2/test.py
@@ -654,7 +654,7 @@ def test_table_override(started_cluster):
         instance.query(f"SELECT count() FROM {materialized_database}.{table_name}")
     )
 
-    expected = "CREATE TABLE test_database.table_override\\n(\\n    `key` Int32,\\n    `value` String,\\n    `_sign` Int8() MATERIALIZED 1,\\n    `_version` UInt64() MATERIALIZED 1\\n)\\nENGINE = ReplacingMergeTree(_version)\\nPARTITION BY key\\nORDER BY tuple(key)"
+    expected = "CREATE TABLE test_database.table_override\\n(\\n    `key` Int32,\\n    `value` String,\\n    `_sign` Int8 MATERIALIZED 1,\\n    `_version` UInt64 MATERIALIZED 1\\n)\\nENGINE = ReplacingMergeTree(_version)\\nPARTITION BY key\\nORDER BY tuple(key)"
     assert (
         expected
         == instance.query(

From 81c19b02e78ea7f2980531732bbaf304622e81fb Mon Sep 17 00:00:00 2001
From: Mark Needham <m.h.needham@gmail.com>
Date: Wed, 24 Jul 2024 13:33:32 +0100
Subject: [PATCH 0705/1488] Update json.md

---
 docs/en/sql-reference/data-types/json.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/en/sql-reference/data-types/json.md b/docs/en/sql-reference/data-types/json.md
index f2eac12594d..f218c8d0339 100644
--- a/docs/en/sql-reference/data-types/json.md
+++ b/docs/en/sql-reference/data-types/json.md
@@ -7,7 +7,7 @@ keywords: [object, data type]
 
 # Object Data Type (deprecated)
 
-**This feature is not production-ready and is now deprecated.** If you need to work with JSON documents, consider using [this guide](/docs/en/integrations/data-ingestion/data-formats/json) instead. A new implementation to support JSON object is in progress and can be tracked [here](https://github.com/ClickHouse/ClickHouse/issues/54864).
+**This feature is not production-ready and is now deprecated.** If you need to work with JSON documents, consider using [this guide](/docs/en/integrations/data-formats/json/overview) instead. A new implementation to support JSON object is in progress and can be tracked [here](https://github.com/ClickHouse/ClickHouse/issues/54864).
 
 <hr />
 

From 6efd29144558ded7fb95b36c6c19ee50aee0071f Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 24 Jul 2024 14:37:05 +0200
Subject: [PATCH 0706/1488] Add a test

---
 .../03210_inconsistent_formatting_of_data_types.reference  | 1 +
 .../03210_inconsistent_formatting_of_data_types.sh         | 7 +++++++
 2 files changed, 8 insertions(+)
 create mode 100644 tests/queries/0_stateless/03210_inconsistent_formatting_of_data_types.reference
 create mode 100755 tests/queries/0_stateless/03210_inconsistent_formatting_of_data_types.sh

diff --git a/tests/queries/0_stateless/03210_inconsistent_formatting_of_data_types.reference b/tests/queries/0_stateless/03210_inconsistent_formatting_of_data_types.reference
new file mode 100644
index 00000000000..ccb445a0573
--- /dev/null
+++ b/tests/queries/0_stateless/03210_inconsistent_formatting_of_data_types.reference
@@ -0,0 +1 @@
+ALTER TABLE columns_with_multiple_streams MODIFY COLUMN `field1` Nullable(tupleElement(x, 2), UInt8)
diff --git a/tests/queries/0_stateless/03210_inconsistent_formatting_of_data_types.sh b/tests/queries/0_stateless/03210_inconsistent_formatting_of_data_types.sh
new file mode 100755
index 00000000000..6cb2d083d71
--- /dev/null
+++ b/tests/queries/0_stateless/03210_inconsistent_formatting_of_data_types.sh
@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+$CLICKHOUSE_FORMAT --oneline --query "ALTER TABLE columns_with_multiple_streams MODIFY COLUMN field1 Nullable(tupleElement(x, 2), UInt8)" | $CLICKHOUSE_FORMAT --oneline

From cb7fafd1e6f04a2f29cd77036bb29042b4cfe3f6 Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Wed, 24 Jul 2024 14:39:26 +0200
Subject: [PATCH 0707/1488] Fix

---
 src/Common/StackTrace.cpp | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/Common/StackTrace.cpp b/src/Common/StackTrace.cpp
index b2ece02e2f0..59a58ac027a 100644
--- a/src/Common/StackTrace.cpp
+++ b/src/Common/StackTrace.cpp
@@ -4,7 +4,6 @@
 #include <base/constexpr_helpers.h>
 #include <base/demangle.h>
 
-#include "Common/PipeFDs.h"
 #include <Common/scope_guard_safe.h>
 #include <Common/Dwarf.h>
 #include <Common/Elf.h>
@@ -496,8 +495,6 @@ using StackTraceCacheBase = std::map<StackTraceTriple, CacheEntryPtr, std::less<
 
 struct StackTraceCache : public StackTraceCacheBase
 {
-    using StackTraceCacheBase::StackTraceCacheBase;
-
     ~StackTraceCache()
     {
         can_use_cache = false;

From 3631bf48729bf40401bb607ba9b54f8f1445860b Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 24 Jul 2024 14:45:58 +0200
Subject: [PATCH 0708/1488] Fix tidy

---
 programs/local/LocalServer.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp
index 5879fd50872..b96de397e96 100644
--- a/programs/local/LocalServer.cpp
+++ b/programs/local/LocalServer.cpp
@@ -743,7 +743,7 @@ void LocalServer::processConfig()
         DatabasePtr database = createClickHouseLocalDatabaseOverlay(default_database, global_context);
         if (UUID uuid = database->getUUID(); uuid != UUIDHelpers::Nil)
             DatabaseCatalog::instance().addUUIDMapping(uuid);
-        DatabaseCatalog::instance().attachDatabase(default_database, std::move(database));
+        DatabaseCatalog::instance().attachDatabase(default_database, database);
     }
     global_context->setCurrentDatabase(default_database);
 

From 009da21694539f4c35983898fb34d4a492c5a692 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Wed, 24 Jul 2024 14:46:44 +0200
Subject: [PATCH 0709/1488] 24.7 changelog improvements

---
 CHANGELOG.md | 34 ++++++++++++++--------------------
 1 file changed, 14 insertions(+), 20 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4fddd7d7685..a0933bd6544 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -17,13 +17,12 @@
 * `clickhouse-keeper-client` will only accept paths in string literals, such as `ls '/hello/world'`, not bare strings such as `ls /hello/world`. [#65494](https://github.com/ClickHouse/ClickHouse/pull/65494) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
 * Metric `KeeperOutstandingRequets` was renamed to `KeeperOutstandingRequests`. [#66206](https://github.com/ClickHouse/ClickHouse/pull/66206) ([Robert Schulze](https://github.com/rschu1ze)).
 * Remove `is_deterministic` field from the `system.functions` table. [#66630](https://github.com/ClickHouse/ClickHouse/pull/66630) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Function `tuple` will now try to construct named tuples in query (controlled by `enable_named_columns_in_function_tuple`). Introduce function `tupleNames` to extract names from tuples. [#54881](https://github.com/ClickHouse/ClickHouse/pull/54881) ([Amos Bird](https://github.com/amosbird)).
 
 #### New Feature
-* Extend function `tuple` to construct named tuples in query. Introduce function `tupleNames` to extract names from tuples. [#54881](https://github.com/ClickHouse/ClickHouse/pull/54881) ([Amos Bird](https://github.com/amosbird)).
 * Add `ASOF JOIN` support for `full_sorting_join` algorithm. [#55051](https://github.com/ClickHouse/ClickHouse/pull/55051) ([vdimir](https://github.com/vdimir)).
-* A new table function, `fuzzQuery,` was added. This function allows you to modify a given query string with random variations. Example: `SELECT query FROM fuzzQuery('SELECT 1');`. [#62103](https://github.com/ClickHouse/ClickHouse/pull/62103) ([pufit](https://github.com/pufit)).
 * Add new window function `percent_rank`. [#62747](https://github.com/ClickHouse/ClickHouse/pull/62747) ([lgbo](https://github.com/lgbo-ustc)).
-* Support JWT authentication in `clickhouse-client`. [#62829](https://github.com/ClickHouse/ClickHouse/pull/62829) ([Konstantin Bogdanov](https://github.com/thevar1able)).
+* Support JWT authentication in `clickhouse-client` (will be available only in ClickHouse Cloud). [#62829](https://github.com/ClickHouse/ClickHouse/pull/62829) ([Konstantin Bogdanov](https://github.com/thevar1able)).
 * Add SQL functions `changeYear`, `changeMonth`, `changeDay`, `changeHour`, `changeMinute`, `changeSecond`. For example, `SELECT changeMonth(toDate('2024-06-14'), 7)` returns date `2024-07-14`. [#63186](https://github.com/ClickHouse/ClickHouse/pull/63186) ([cucumber95](https://github.com/cucumber95)).
 * Introduce startup scripts, which allow the execution of preconfigured queries at the startup stage. [#64889](https://github.com/ClickHouse/ClickHouse/pull/64889) ([pufit](https://github.com/pufit)).
 * Support accept_invalid_certificate in client's config in order to allow for client to connect over secure TCP to a server running with self-signed certificate - can be used as a shorthand for corresponding `openSSL` client settings `verificationMode=none` + `invalidCertificateHandler.name=AcceptCertificateHandler`. [#65238](https://github.com/ClickHouse/ClickHouse/pull/65238) ([peacewalker122](https://github.com/peacewalker122)).
@@ -31,15 +30,19 @@
 * Add aggregate function `groupConcat`. About the same as `arrayStringConcat( groupArray(column), ',')` Can receive 2 parameters: a string delimiter and the number of elements to be processed. [#65451](https://github.com/ClickHouse/ClickHouse/pull/65451) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
 * Add AzureQueue storage. [#65458](https://github.com/ClickHouse/ClickHouse/pull/65458) ([Kseniia Sumarokova](https://github.com/kssenii)).
 * Add a new setting to disable/enable writing page index into parquet files. [#65475](https://github.com/ClickHouse/ClickHouse/pull/65475) ([lgbo](https://github.com/lgbo-ustc)).
-* Allow system administrators to configure `logger.console_log_level`. [#65559](https://github.com/ClickHouse/ClickHouse/pull/65559) ([Azat Khuzhin](https://github.com/azat)).
+* Introduce `logger.console_log_level` server config to control the log level to the console (if enabled). [#65559](https://github.com/ClickHouse/ClickHouse/pull/65559) ([Azat Khuzhin](https://github.com/azat)).
 * Automatically append a wildcard `*` to the end of a directory path with table function `file`. [#66019](https://github.com/ClickHouse/ClickHouse/pull/66019) ([Zhidong (David) Guo](https://github.com/Gun9niR)).
 * Add `--memory-usage` option to client in non interactive mode. [#66393](https://github.com/ClickHouse/ClickHouse/pull/66393) ([vdimir](https://github.com/vdimir)).
+* Make an interactive client for clickhouse-disks, add local disk from the local directory. [#64446](https://github.com/ClickHouse/ClickHouse/pull/64446) ([Daniil Ivanik](https://github.com/divanik)).
+* When lightweight delete happens on a table with projection(s), users have choices either throw an exception (by default) or drop the projection [#65594](https://github.com/ClickHouse/ClickHouse/pull/65594) ([jsc0218](https://github.com/jsc0218)).
 
 #### Experimental Feature
 * Change binary serialization of Variant data type: add `compact` mode to avoid writing the same discriminator multiple times for granules with single variant or with only NULL values. Add MergeTree setting `use_compact_variant_discriminators_serialization` that is enabled by default. Note that Variant type is still experimental and backward-incompatible change in serialization is ok. [#62774](https://github.com/ClickHouse/ClickHouse/pull/62774) ([Kruglov Pavel](https://github.com/Avogar)).
 * Support rocksdb as backend storage of keeper. [#56626](https://github.com/ClickHouse/ClickHouse/pull/56626) ([Han Fei](https://github.com/hanfei1991)).
 * Refactor JSONExtract functions, support more types including experimental Dynamic type. [#66046](https://github.com/ClickHouse/ClickHouse/pull/66046) ([Kruglov Pavel](https://github.com/Avogar)).
 * Support null map subcolumn for Variant and Dynamic subcolumns. [#66178](https://github.com/ClickHouse/ClickHouse/pull/66178) ([Kruglov Pavel](https://github.com/Avogar)).
+* Fix reading dynamic subcolumns from altered Memory table. Previously if `max_types` parameter of a Dynamic type was changed in Memory table via alter, further subcolumns reading can return wrong result. [#66066](https://github.com/ClickHouse/ClickHouse/pull/66066) ([Kruglov Pavel](https://github.com/Avogar)).
+* Add support for `cluster_for_parallel_replicas` when using custom key parallel replicas. It allows you to use parallel replicas with custom key with MergeTree tables. [#65453](https://github.com/ClickHouse/ClickHouse/pull/65453) ([Antonio Andelic](https://github.com/antonio2368)).
 
 #### Performance Improvement
 * Enable `optimize_functions_to_subcolumns` by default. [#58661](https://github.com/ClickHouse/ClickHouse/pull/58661) ([Anton Popov](https://github.com/CurtizJ)).
@@ -51,6 +54,10 @@
 * Support minmax hyperrectangle for Set indices. [#65676](https://github.com/ClickHouse/ClickHouse/pull/65676) ([AntiTopQuark](https://github.com/AntiTopQuark)).
 * Unload primary index of outdated parts to reduce total memory usage. [#65852](https://github.com/ClickHouse/ClickHouse/pull/65852) ([Anton Popov](https://github.com/CurtizJ)).
 * Functions `replaceRegexpAll` and `replaceRegexpOne` are now significantly faster if the pattern is trivial, i.e. contains no metacharacters, pattern classes, flags, grouping characters etc. (Thanks to Taiyang Li). [#66185](https://github.com/ClickHouse/ClickHouse/pull/66185) ([Robert Schulze](https://github.com/rschu1ze)).
+* s3 requests: Reduce retry time for queries, increase retries count for backups. 8.5 minutes and 100 retires for queries, 1.2 hours and 1000 retries for backup restore. [#65232](https://github.com/ClickHouse/ClickHouse/pull/65232) ([Sema Checherinda](https://github.com/CheSema)).
+* Support query plan LIMIT optimization. Support LIMIT pushdown for PostgreSQL storage and table function. [#65454](https://github.com/ClickHouse/ClickHouse/pull/65454) ([Maksim Kita](https://github.com/kitaisreal)).
+* Improved ZooKeeper load balancing. The current session doesn't expire until the optimal nodes become available despite `fallback_session_lifetime`. Added support for AZ-aware balancing. [#65570](https://github.com/ClickHouse/ClickHouse/pull/65570) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* DatabaseCatalog drops tables faster by using up to database_catalog_drop_table_concurrency threads. [#66065](https://github.com/ClickHouse/ClickHouse/pull/66065) ([Sema Checherinda](https://github.com/CheSema)).
 
 #### Improvement
 * The setting `optimize_trivial_insert_select` is disabled by default. In most cases, it should be beneficial. Nevertheless, if you are seeing slower INSERT SELECT or increased memory usage, you can enable it back or `SET compatibility = '24.6'`. [#58970](https://github.com/ClickHouse/ClickHouse/pull/58970) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
@@ -60,21 +67,15 @@
 * Allow matching column names in a case insensitive manner when reading json files (`input_format_json_ignore_key_case`). [#61750](https://github.com/ClickHouse/ClickHouse/pull/61750) ([kevinyhzou](https://github.com/KevinyhZou)).
 * Support reading partitioned data DeltaLake data. Infer DeltaLake schema by reading metadata instead of data. [#63201](https://github.com/ClickHouse/ClickHouse/pull/63201) ([Kseniia Sumarokova](https://github.com/kssenii)).
 * In composable protocols TLS layer accepted only `certificateFile` and `privateKeyFile` parameters. https://clickhouse.com/docs/en/operations/settings/composable-protocols. [#63985](https://github.com/ClickHouse/ClickHouse/pull/63985) ([Anton Ivashkin](https://github.com/ianton-ru)).
-* Make an interactive client for clickhouse-disks, add local disk from the local directory. [#64446](https://github.com/ClickHouse/ClickHouse/pull/64446) ([Daniil Ivanik](https://github.com/divanik)).
 * Added profile event `SelectQueriesWithPrimaryKeyUsage` which indicates how many SELECT queries use the primary key to evaluate the WHERE clause. [#64492](https://github.com/ClickHouse/ClickHouse/pull/64492) ([0x01f](https://github.com/0xfei)).
 * `StorageS3Queue` related fixes and improvements. Deduce a default value of `s3queue_processing_threads_num` according to the number of physical cpu cores on the server (instead of the previous default value as 1). Set default value of `s3queue_loading_retries` to 10. Fix possible vague "Uncaught exception" in exception column of `system.s3queue`. Do not increment retry count on `MEMORY_LIMIT_EXCEEDED` exception. Move files commit to a stage after insertion into table fully finished to avoid files being commited while not inserted. Add settings `s3queue_max_processed_files_before_commit`, `s3queue_max_processed_rows_before_commit`, `s3queue_max_processed_bytes_before_commit`, `s3queue_max_processing_time_sec_before_commit`, to better control commit and flush time. [#65046](https://github.com/ClickHouse/ClickHouse/pull/65046) ([Kseniia Sumarokova](https://github.com/kssenii)).
-* Fixed broken multiple columns aggregation on s390x. [#65062](https://github.com/ClickHouse/ClickHouse/pull/65062) ([Harry Lee](https://github.com/HarryLeeIBM)).
 * Support aliases in parametrized view function (only new analyzer). [#65190](https://github.com/ClickHouse/ClickHouse/pull/65190) ([Kseniia Sumarokova](https://github.com/kssenii)).
-* s3 requests: Reduce retry time for queries, increase retries count for backups. 8.5 minutes and 100 retires for queries, 1.2 hours and 1000 retries for backup restore. [#65232](https://github.com/ClickHouse/ClickHouse/pull/65232) ([Sema Checherinda](https://github.com/CheSema)).
 * Updated to mask account key in logs in azureBlobStorage. [#65273](https://github.com/ClickHouse/ClickHouse/pull/65273) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)).
 * Partition pruning for `IN` predicates when filter expression is a part of `PARTITION BY` expression. [#65335](https://github.com/ClickHouse/ClickHouse/pull/65335) ([Eduard Karacharov](https://github.com/korowa)).
 * Add system tables with main information about all detached tables. [#65400](https://github.com/ClickHouse/ClickHouse/pull/65400) ([Konstantin Morozov](https://github.com/k-morozov)).
-* Add support for `cluster_for_parallel_replicas` when using custom key parallel replicas. It allows you to use parallel replicas with custom key with MergeTree tables. [#65453](https://github.com/ClickHouse/ClickHouse/pull/65453) ([Antonio Andelic](https://github.com/antonio2368)).
-* Support query plan LIMIT optimization. Support LIMIT pushdown for PostgreSQL storage and table function. [#65454](https://github.com/ClickHouse/ClickHouse/pull/65454) ([Maksim Kita](https://github.com/kitaisreal)).
 * `arrayMin`/`arrayMax` can be applicable to all data types that are comparable. [#65455](https://github.com/ClickHouse/ClickHouse/pull/65455) ([pn](https://github.com/chloro-pn)).
 * Improved memory accounting for cgroups v2 to exclude the amount occupied by the page cache. [#65470](https://github.com/ClickHouse/ClickHouse/pull/65470) ([Nikita Taranov](https://github.com/nickitat)).
 * Do not create format settings for each row when serializing chunks to insert to EmbeddedRocksDB table. [#65474](https://github.com/ClickHouse/ClickHouse/pull/65474) ([Duc Canh Le](https://github.com/canhld94)).
-* Fixed out-of-range exception in parsing Dwarf5 on s390x. [#65501](https://github.com/ClickHouse/ClickHouse/pull/65501) ([Harry Lee](https://github.com/HarryLeeIBM)).
 * Reduce `clickhouse-local` prompt to just `:)`. `getFQDNOrHostName()` takes too long on macOS, and we don't want a hostname in the prompt for `clickhouse-local` anyway. [#65510](https://github.com/ClickHouse/ClickHouse/pull/65510) ([Konstantin Bogdanov](https://github.com/thevar1able)).
 * Avoid printing a message from jemalloc about per-CPU arenas on low-end virtual machines. [#65532](https://github.com/ClickHouse/ClickHouse/pull/65532) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
 * Disable filesystem cache background download by default. It will be enabled back when we fix the issue with possible "Memory limit exceeded" because memory deallocation is done outside of query context (while buffer is allocated inside of query context) if we use background download threads. Plus we need to add a separate setting to define max size to download for background workers (currently it is limited by max_file_segment_size, which might be too big). [#65534](https://github.com/ClickHouse/ClickHouse/pull/65534) ([Kseniia Sumarokova](https://github.com/kssenii)).
@@ -82,7 +83,6 @@
 * Implement binary encoding for ClickHouse data types and add its specification in docs. Use it in Dynamic binary serialization, allow to use it in RowBinaryWithNamesAndTypes and Native formats under settings. [#65546](https://github.com/ClickHouse/ClickHouse/pull/65546) ([Kruglov Pavel](https://github.com/Avogar)).
 * Improved ZooKeeper load balancing. The current session doesn't expire until the optimal nodes become available despite `fallback_session_lifetime`. Added support for AZ-aware balancing. [#65570](https://github.com/ClickHouse/ClickHouse/pull/65570) ([Alexander Tokmakov](https://github.com/tavplubix)).
 * Server settings `compiled_expression_cache_size` and `compiled_expression_cache_elements_size` are now shown in `system.server_settings`. [#65584](https://github.com/ClickHouse/ClickHouse/pull/65584) ([Robert Schulze](https://github.com/rschu1ze)).
-* When lightweight delete happens on a table with projection(s), users have choices either throw an exception (by default) or drop the projection [#65594](https://github.com/ClickHouse/ClickHouse/pull/65594) ([jsc0218](https://github.com/jsc0218)).
 * Add support for user identification based on x509 SubjectAltName extension. [#65626](https://github.com/ClickHouse/ClickHouse/pull/65626) ([Anton Kozlov](https://github.com/tonickkozlov)).
 * `clickhouse-local` will respect the `max_server_memory_usage` and `max_server_memory_usage_to_ram_ratio` from the configuration file. It will also set the max memory usage to 90% of the system memory by default, like `clickhouse-server` does. [#65697](https://github.com/ClickHouse/ClickHouse/pull/65697) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
 * Add a script to backup your files to ClickHouse. [#65699](https://github.com/ClickHouse/ClickHouse/pull/65699) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
@@ -93,7 +93,6 @@
 * Allow controlling named collections in clickhouse-local. [#65973](https://github.com/ClickHouse/ClickHouse/pull/65973) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
 * Improve Azure profile events. [#65999](https://github.com/ClickHouse/ClickHouse/pull/65999) ([alesapin](https://github.com/alesapin)).
 * Support ORC file read by writer time zone. [#66025](https://github.com/ClickHouse/ClickHouse/pull/66025) ([kevinyhzou](https://github.com/KevinyhZou)).
-* DatabaseCatalog drops tables faster by using up to database_catalog_drop_table_concurrency threads. [#66065](https://github.com/ClickHouse/ClickHouse/pull/66065) ([Sema Checherinda](https://github.com/CheSema)).
 * Add settings to control connection to the PostgreSQL. * Setting `postgresql_connection_attempt_timeout` specifies the value passed to `connect_timeout` parameter of connection URL. * Setting `postgresql_connection_pool_retries` specifies the number of retries to establish a connection to the PostgreSQL end-point. [#66232](https://github.com/ClickHouse/ClickHouse/pull/66232) ([Dmitry Novik](https://github.com/novikd)).
 * Reduce inaccuracy of input_wait_elapsed_us/input_wait_elapsed_us/elapsed_us. [#66239](https://github.com/ClickHouse/ClickHouse/pull/66239) ([Azat Khuzhin](https://github.com/azat)).
 * Improve FilesystemCache ProfileEvents. [#66249](https://github.com/ClickHouse/ClickHouse/pull/66249) ([zhukai](https://github.com/nauu)).
@@ -102,18 +101,16 @@
 * Disable suspending on Ctrl+Z in interactive mode. This is a common trap and is not expected behavior for almost all users. I imagine only a few extreme power users could appreciate suspending terminal applications to the background, but I don't know any. [#66511](https://github.com/ClickHouse/ClickHouse/pull/66511) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
 * Add option for validating the Primary key type in Dictionaries. Without this option for simple layouts any column type will be implicitly converted to UInt64. ### Documentation entry for user-facing changes. [#66595](https://github.com/ClickHouse/ClickHouse/pull/66595) ([MikhailBurdukov](https://github.com/MikhailBurdukov)).
 
-#### Critical Bug Fix (crash, LOGICAL_ERROR, data loss, RBAC)
-* Fix unexpeced size of low cardinality column in function calls. [#65298](https://github.com/ClickHouse/ClickHouse/pull/65298) ([Raúl Marín](https://github.com/Algunenano)).
+#### Bug Fix (user-visible misbehavior in an official stable release)
+* Fix unexpected size of low cardinality column in function calls. [#65298](https://github.com/ClickHouse/ClickHouse/pull/65298) ([Raúl Marín](https://github.com/Algunenano)).
 * Check cyclic dependencies on CREATE/REPLACE/RENAME/EXCHANGE queries and throw an exception if there is a cyclic dependency. Previously such cyclic dependencies could lead to a deadlock during server startup. Also fix some bugs in dependencies creation. [#65405](https://github.com/ClickHouse/ClickHouse/pull/65405) ([Kruglov Pavel](https://github.com/Avogar)).
 * Fix crash in maxIntersections. [#65689](https://github.com/ClickHouse/ClickHouse/pull/65689) ([Raúl Marín](https://github.com/Algunenano)).
 * Fix the VALID UNTIL clause in the user definition resetting after a restart. [#66409](https://github.com/ClickHouse/ClickHouse/pull/66409) ([Nikolay Degterinsky](https://github.com/evillique)).
-
-#### Bug Fix (user-visible misbehavior in an official stable release)
 * Fix SHOW MERGES remaining time. [#66735](https://github.com/ClickHouse/ClickHouse/pull/66735) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
 * `Query was cancelled` might have been printed twice in clickhouse-client. This behaviour is fixed. [#66005](https://github.com/ClickHouse/ClickHouse/pull/66005) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
 * Fixed crash while using MaterializedMySQL with TABLE OVERRIDE that maps MySQL NULL field into ClickHouse not NULL field. [#54649](https://github.com/ClickHouse/ClickHouse/pull/54649) ([Filipp Ozinov](https://github.com/bakwc)).
 * Fix logical error when PREWHERE expression read no columns and table has no adaptive index granularity (very old table). [#59173](https://github.com/ClickHouse/ClickHouse/pull/59173) ([Alexander Gololobov](https://github.com/davenger)).
-* Fix bug with cancelation buffer when canceling a query. [#64478](https://github.com/ClickHouse/ClickHouse/pull/64478) ([Sema Checherinda](https://github.com/CheSema)).
+* Fix bug with cancellation buffer when canceling a query. [#64478](https://github.com/ClickHouse/ClickHouse/pull/64478) ([Sema Checherinda](https://github.com/CheSema)).
 * Fix filling parts columns from metadata (when columns.txt does not exists). [#64757](https://github.com/ClickHouse/ClickHouse/pull/64757) ([Azat Khuzhin](https://github.com/azat)).
 * Fix crash for `ALTER TABLE ... ON CLUSTER ... MODIFY SQL SECURITY`. [#64957](https://github.com/ClickHouse/ClickHouse/pull/64957) ([pufit](https://github.com/pufit)).
 * Fix crash on destroying AccessControl: add explicit shutdown. [#64993](https://github.com/ClickHouse/ClickHouse/pull/64993) ([Vitaly Baranov](https://github.com/vitlibar)).
@@ -146,7 +143,6 @@
 * Fix resolving dynamic subcolumns in analyzer, avoid reading the whole column on dynamic subcolumn reading. [#66004](https://github.com/ClickHouse/ClickHouse/pull/66004) ([Kruglov Pavel](https://github.com/Avogar)).
 * Fix config merging for from_env with replace overrides. [#66034](https://github.com/ClickHouse/ClickHouse/pull/66034) ([Azat Khuzhin](https://github.com/azat)).
 * Fix a possible hanging in `GRPCServer` during shutdown. [#66061](https://github.com/ClickHouse/ClickHouse/pull/66061) ([Vitaly Baranov](https://github.com/vitlibar)).
-* Fix reading dynamic subcolumns from altered Memory table. Previously if `max_types` peremeter of a Dynamic type was changed in Memory table via alter, further subcolumns reading can return wrong result. [#66066](https://github.com/ClickHouse/ClickHouse/pull/66066) ([Kruglov Pavel](https://github.com/Avogar)).
 * Fixed several cases in function `has` with non-constant `LowCardinality` arguments. [#66088](https://github.com/ClickHouse/ClickHouse/pull/66088) ([Anton Popov](https://github.com/CurtizJ)).
 * Fix for `groupArrayIntersect`. It had incorrect behavior in the `merge()` function. Also, fixed behavior in `deserialise()` for numeric and general data. [#66103](https://github.com/ClickHouse/ClickHouse/pull/66103) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
 * Fixed buffer overflow bug in `unbin`/`unhex` implementation. [#66106](https://github.com/ClickHouse/ClickHouse/pull/66106) ([Nikita Taranov](https://github.com/nickitat)).
@@ -185,8 +181,6 @@
 
 #### Build/Testing/Packaging Improvement
 * Instantiate template methods ahead in different .cpp files, avoid too large translation units during compiling. [#64818](https://github.com/ClickHouse/ClickHouse/pull/64818) ([lgbo](https://github.com/lgbo-ustc)).
-* Upgraded `pocketfft` dependency to the recent commit https://github.com/mreineck/pocketfft/commit/f4c1aa8aa9ce79ad39e80f2c9c41b92ead90fda3. [#66291](https://github.com/ClickHouse/ClickHouse/pull/66291) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
-* Upgraded `azure-sdk-for-cpp` to the recent commit https://github.com/ClickHouse/azure-sdk-for-cpp/commit/ea3e19a7be08519134c643177d56c7484dfec884. [#66292](https://github.com/ClickHouse/ClickHouse/pull/66292) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
 
 ### <a id="246"></a> ClickHouse release 24.6, 2024-07-01
 

From ff44b2066195c8f72897794ff1051a695c56cbfd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Wed, 24 Jul 2024 14:51:48 +0200
Subject: [PATCH 0710/1488] Revert "FuzzQuery table function"

---
 .../table-functions/fuzzQuery.md              |  36 ----
 programs/client/Client.h                      |   5 +-
 src/Client/ClientBase.h                       |   2 +-
 src/{Common => Client}/QueryFuzzer.cpp        |  50 ++----
 src/{Common => Client}/QueryFuzzer.h          |  35 ++--
 src/Storages/StorageFuzzQuery.cpp             | 169 ------------------
 src/Storages/StorageFuzzQuery.h               |  88 ---------
 src/Storages/registerStorages.cpp             |   2 -
 src/TableFunctions/TableFunctionFuzzQuery.cpp |  54 ------
 src/TableFunctions/TableFunctionFuzzQuery.h   |  42 -----
 src/TableFunctions/registerTableFunctions.cpp |   1 -
 src/TableFunctions/registerTableFunctions.h   |   1 -
 .../03031_table_function_fuzzquery.reference  |   2 -
 .../03031_table_function_fuzzquery.sql        |  18 --
 14 files changed, 31 insertions(+), 474 deletions(-)
 delete mode 100644 docs/en/sql-reference/table-functions/fuzzQuery.md
 rename src/{Common => Client}/QueryFuzzer.cpp (97%)
 rename src/{Common => Client}/QueryFuzzer.h (91%)
 delete mode 100644 src/Storages/StorageFuzzQuery.cpp
 delete mode 100644 src/Storages/StorageFuzzQuery.h
 delete mode 100644 src/TableFunctions/TableFunctionFuzzQuery.cpp
 delete mode 100644 src/TableFunctions/TableFunctionFuzzQuery.h
 delete mode 100644 tests/queries/0_stateless/03031_table_function_fuzzquery.reference
 delete mode 100644 tests/queries/0_stateless/03031_table_function_fuzzquery.sql

diff --git a/docs/en/sql-reference/table-functions/fuzzQuery.md b/docs/en/sql-reference/table-functions/fuzzQuery.md
deleted file mode 100644
index e15f8a40156..00000000000
--- a/docs/en/sql-reference/table-functions/fuzzQuery.md
+++ /dev/null
@@ -1,36 +0,0 @@
----
-slug: /en/sql-reference/table-functions/fuzzQuery
-sidebar_position: 75
-sidebar_label: fuzzQuery
----
-
-# fuzzQuery
-
-Perturbs the given query string with random variations.
-
-``` sql
-fuzzQuery(query[, max_query_length[, random_seed]])
-```
-
-**Arguments**
-
-- `query` (String) - The source query to perform the fuzzing on.
-- `max_query_length` (UInt64) - A maximum length the query can get during the fuzzing process.
-- `random_seed` (UInt64) - A random seed for producing stable results.
-
-**Returned Value**
-
-A table object with a single column containing perturbed query strings.
-
-## Usage Example
-
-``` sql
-SELECT * FROM fuzzQuery('SELECT materialize(\'a\' AS key) GROUP BY key') LIMIT 2;
-```
-
-```
-   ┌─query──────────────────────────────────────────────────────────┐
-1. │ SELECT 'a' AS key GROUP BY key                                 │
-2. │ EXPLAIN PIPELINE compact = true SELECT 'a' AS key GROUP BY key │
-   └────────────────────────────────────────────────────────────────┘
-```
diff --git a/programs/client/Client.h b/programs/client/Client.h
index 6d57a6ea648..229608f787d 100644
--- a/programs/client/Client.h
+++ b/programs/client/Client.h
@@ -9,10 +9,7 @@ namespace DB
 class Client : public ClientBase
 {
 public:
-    Client()
-    {
-        fuzzer = QueryFuzzer(randomSeed(), &std::cout, &std::cerr);
-    }
+    Client() = default;
 
     void initialize(Poco::Util::Application & self) override;
 
diff --git a/src/Client/ClientBase.h b/src/Client/ClientBase.h
index 4f500a4c45d..986990aecaa 100644
--- a/src/Client/ClientBase.h
+++ b/src/Client/ClientBase.h
@@ -6,7 +6,6 @@
 #include <Common/ProgressIndication.h>
 #include <Common/InterruptListener.h>
 #include <Common/ShellCommand.h>
-#include <Common/QueryFuzzer.h>
 #include <Common/Stopwatch.h>
 #include <Common/DNSResolver.h>
 #include <Core/ExternalTable.h>
@@ -17,6 +16,7 @@
 #include <Poco/SplitterChannel.h>
 #include <Interpreters/Context.h>
 #include <Client/Suggest.h>
+#include <Client/QueryFuzzer.h>
 #include <boost/program_options.hpp>
 #include <Storages/StorageFile.h>
 #include <Storages/SelectQueryInfo.h>
diff --git a/src/Common/QueryFuzzer.cpp b/src/Client/QueryFuzzer.cpp
similarity index 97%
rename from src/Common/QueryFuzzer.cpp
rename to src/Client/QueryFuzzer.cpp
index 161c38f20e0..f5b700ea529 100644
--- a/src/Common/QueryFuzzer.cpp
+++ b/src/Client/QueryFuzzer.cpp
@@ -68,21 +68,22 @@ Field QueryFuzzer::getRandomField(int type)
     {
     case 0:
     {
-        return bad_int64_values[fuzz_rand() % std::size(bad_int64_values)];
+        return bad_int64_values[fuzz_rand() % (sizeof(bad_int64_values)
+                / sizeof(*bad_int64_values))];
     }
     case 1:
     {
         static constexpr double values[]
                 = {NAN, INFINITY, -INFINITY, 0., -0., 0.0001, 0.5, 0.9999,
                    1., 1.0001, 2., 10.0001, 100.0001, 1000.0001, 1e10, 1e20,
-                  FLT_MIN, FLT_MIN + FLT_EPSILON, FLT_MAX, FLT_MAX + FLT_EPSILON}; return values[fuzz_rand() % std::size(values)];
+                  FLT_MIN, FLT_MIN + FLT_EPSILON, FLT_MAX, FLT_MAX + FLT_EPSILON}; return values[fuzz_rand() % (sizeof(values) / sizeof(*values))];
     }
     case 2:
     {
         static constexpr UInt64 scales[] = {0, 1, 2, 10};
         return DecimalField<Decimal64>(
-            bad_int64_values[fuzz_rand() % std::size(bad_int64_values)],
-            static_cast<UInt32>(scales[fuzz_rand() % std::size(scales)])
+            bad_int64_values[fuzz_rand() % (sizeof(bad_int64_values) / sizeof(*bad_int64_values))],
+            static_cast<UInt32>(scales[fuzz_rand() % (sizeof(scales) / sizeof(*scales))])
         );
     }
     default:
@@ -164,8 +165,7 @@ Field QueryFuzzer::fuzzField(Field field)
         {
             size_t pos = fuzz_rand() % arr.size();
             arr.erase(arr.begin() + pos);
-            if (debug_stream)
-                *debug_stream << "erased\n";
+            std::cerr << "erased\n";
         }
 
         if (fuzz_rand() % 5 == 0)
@@ -174,14 +174,12 @@ Field QueryFuzzer::fuzzField(Field field)
             {
                 size_t pos = fuzz_rand() % arr.size();
                 arr.insert(arr.begin() + pos, fuzzField(arr[pos]));
-                if (debug_stream)
-                    *debug_stream << fmt::format("inserted (pos {})\n", pos);
+                std::cerr << fmt::format("inserted (pos {})\n", pos);
             }
             else
             {
                 arr.insert(arr.begin(), getRandomField(0));
-                if (debug_stream)
-                    *debug_stream << "inserted (0)\n";
+                std::cerr << "inserted (0)\n";
             }
 
         }
@@ -199,9 +197,7 @@ Field QueryFuzzer::fuzzField(Field field)
         {
             size_t pos = fuzz_rand() % arr.size();
             arr.erase(arr.begin() + pos);
-
-            if (debug_stream)
-                *debug_stream << "erased\n";
+            std::cerr << "erased\n";
         }
 
         if (fuzz_rand() % 5 == 0)
@@ -210,16 +206,12 @@ Field QueryFuzzer::fuzzField(Field field)
             {
                 size_t pos = fuzz_rand() % arr.size();
                 arr.insert(arr.begin() + pos, fuzzField(arr[pos]));
-
-                if (debug_stream)
-                    *debug_stream << fmt::format("inserted (pos {})\n", pos);
+                std::cerr << fmt::format("inserted (pos {})\n", pos);
             }
             else
             {
                 arr.insert(arr.begin(), getRandomField(0));
-
-                if (debug_stream)
-                    *debug_stream << "inserted (0)\n";
+                std::cerr << "inserted (0)\n";
             }
 
         }
@@ -352,8 +344,7 @@ void QueryFuzzer::fuzzOrderByList(IAST * ast)
         }
         else
         {
-            if (debug_stream)
-                *debug_stream << "No random column.\n";
+            std::cerr << "No random column.\n";
         }
     }
 
@@ -387,8 +378,7 @@ void QueryFuzzer::fuzzColumnLikeExpressionList(IAST * ast)
         if (col)
             impl->children.insert(pos, col);
         else
-            if (debug_stream)
-                *debug_stream << "No random column.\n";
+            std::cerr << "No random column.\n";
     }
 
     // We don't have to recurse here to fuzz the children, this is handled by
@@ -1371,15 +1361,11 @@ void QueryFuzzer::fuzzMain(ASTPtr & ast)
     collectFuzzInfoMain(ast);
     fuzz(ast);
 
-    if (out_stream)
-    {
-        *out_stream << std::endl;
-
-        WriteBufferFromOStream ast_buf(*out_stream, 4096);
-        formatAST(*ast, ast_buf, false /*highlight*/);
-        ast_buf.finalize();
-        *out_stream << std::endl << std::endl;
-    }
+    std::cout << std::endl;
+    WriteBufferFromOStream ast_buf(std::cout, 4096);
+    formatAST(*ast, ast_buf, false /*highlight*/);
+    ast_buf.finalize();
+    std::cout << std::endl << std::endl;
 }
 
 }
diff --git a/src/Common/QueryFuzzer.h b/src/Client/QueryFuzzer.h
similarity index 91%
rename from src/Common/QueryFuzzer.h
rename to src/Client/QueryFuzzer.h
index 35d088809f2..6165e589cae 100644
--- a/src/Common/QueryFuzzer.h
+++ b/src/Client/QueryFuzzer.h
@@ -35,31 +35,9 @@ struct ASTWindowDefinition;
  * queries, so you want to feed it a lot of queries to get some interesting mix
  * of them. Normally we feed SQL regression tests to it.
  */
-class QueryFuzzer
+struct QueryFuzzer
 {
-public:
-    explicit QueryFuzzer(pcg64 fuzz_rand_ = randomSeed(), std::ostream * out_stream_ = nullptr, std::ostream * debug_stream_ = nullptr)
-        : fuzz_rand(fuzz_rand_)
-        , out_stream(out_stream_)
-        , debug_stream(debug_stream_)
-    {
-    }
-
-    // This is the only function you have to call -- it will modify the passed
-    // ASTPtr to point to new AST with some random changes.
-    void fuzzMain(ASTPtr & ast);
-
-    ASTs getInsertQueriesForFuzzedTables(const String & full_query);
-    ASTs getDropQueriesForFuzzedTables(const ASTDropQuery & drop_query);
-    void notifyQueryFailed(ASTPtr ast);
-
-    static bool isSuitableForFuzzing(const ASTCreateQuery & create);
-
-private:
-    pcg64 fuzz_rand;
-
-    std::ostream * out_stream = nullptr;
-    std::ostream * debug_stream = nullptr;
+    pcg64 fuzz_rand{randomSeed()};
 
     // We add elements to expression lists with fixed probability. Some elements
     // are so large, that the expected number of elements we add to them is
@@ -88,6 +66,10 @@ private:
     std::unordered_map<std::string, size_t> index_of_fuzzed_table;
     std::set<IAST::Hash> created_tables_hashes;
 
+    // This is the only function you have to call -- it will modify the passed
+    // ASTPtr to point to new AST with some random changes.
+    void fuzzMain(ASTPtr & ast);
+
     // Various helper functions follow, normally you shouldn't have to call them.
     Field getRandomField(int type);
     Field fuzzField(Field field);
@@ -95,6 +77,9 @@ private:
     ASTPtr getRandomExpressionList();
     DataTypePtr fuzzDataType(DataTypePtr type);
     DataTypePtr getRandomType();
+    ASTs getInsertQueriesForFuzzedTables(const String & full_query);
+    ASTs getDropQueriesForFuzzedTables(const ASTDropQuery & drop_query);
+    void notifyQueryFailed(ASTPtr ast);
     void replaceWithColumnLike(ASTPtr & ast);
     void replaceWithTableLike(ASTPtr & ast);
     void fuzzOrderByElement(ASTOrderByElement * elem);
@@ -117,6 +102,8 @@ private:
     void addTableLike(ASTPtr ast);
     void addColumnLike(ASTPtr ast);
     void collectFuzzInfoRecurse(ASTPtr ast);
+
+    static bool isSuitableForFuzzing(const ASTCreateQuery & create);
 };
 
 }
diff --git a/src/Storages/StorageFuzzQuery.cpp b/src/Storages/StorageFuzzQuery.cpp
deleted file mode 100644
index 6e8f425f8dc..00000000000
--- a/src/Storages/StorageFuzzQuery.cpp
+++ /dev/null
@@ -1,169 +0,0 @@
-#include <Storages/StorageFuzzQuery.h>
-
-#include <optional>
-#include <unordered_set>
-#include <Columns/ColumnString.h>
-#include <Interpreters/evaluateConstantExpression.h>
-#include <Storages/NamedCollectionsHelpers.h>
-#include <Storages/StorageFactory.h>
-#include <Storages/checkAndGetLiteralArgument.h>
-#include <Parsers/ParserQuery.h>
-#include <Parsers/parseQuery.h>
-
-namespace DB
-{
-
-namespace ErrorCodes
-{
-    extern const int BAD_ARGUMENTS;
-    extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
-}
-
-ColumnPtr FuzzQuerySource::createColumn()
-{
-    auto column = ColumnString::create();
-    ColumnString::Chars & data_to = column->getChars();
-    ColumnString::Offsets & offsets_to = column->getOffsets();
-
-    offsets_to.resize(block_size);
-    IColumn::Offset offset = 0;
-
-    auto fuzz_base = query;
-    size_t row_num = 0;
-
-    while (row_num < block_size)
-    {
-        ASTPtr new_query = fuzz_base->clone();
-
-        auto base_before_fuzz = fuzz_base->formatForErrorMessage();
-        fuzzer.fuzzMain(new_query);
-        auto fuzzed_text = new_query->formatForErrorMessage();
-
-        if (base_before_fuzz == fuzzed_text)
-            continue;
-
-        /// AST is too long, will start from the original query.
-        if (config.max_query_length > 500)
-        {
-            fuzz_base = query;
-            continue;
-        }
-
-        IColumn::Offset next_offset = offset + fuzzed_text.size() + 1;
-        data_to.resize(next_offset);
-
-        std::copy(fuzzed_text.begin(), fuzzed_text.end(), &data_to[offset]);
-
-        data_to[offset + fuzzed_text.size()] = 0;
-        offsets_to[row_num] = next_offset;
-
-        offset = next_offset;
-        fuzz_base = new_query;
-        ++row_num;
-    }
-
-    return column;
-}
-
-StorageFuzzQuery::StorageFuzzQuery(
-    const StorageID & table_id_, const ColumnsDescription & columns_, const String & comment_, const Configuration & config_)
-    : IStorage(table_id_), config(config_)
-{
-    StorageInMemoryMetadata storage_metadata;
-    storage_metadata.setColumns(columns_);
-    storage_metadata.setComment(comment_);
-    setInMemoryMetadata(storage_metadata);
-}
-
-Pipe StorageFuzzQuery::read(
-    const Names & column_names,
-    const StorageSnapshotPtr & storage_snapshot,
-    SelectQueryInfo & /*query_info*/,
-    ContextPtr /*context*/,
-    QueryProcessingStage::Enum /*processed_stage*/,
-    size_t max_block_size,
-    size_t num_streams)
-{
-    storage_snapshot->check(column_names);
-
-    Pipes pipes;
-    pipes.reserve(num_streams);
-
-    const ColumnsDescription & our_columns = storage_snapshot->metadata->getColumns();
-    Block block_header;
-    for (const auto & name : column_names)
-    {
-        const auto & name_type = our_columns.get(name);
-        MutableColumnPtr column = name_type.type->createColumn();
-        block_header.insert({std::move(column), name_type.type, name_type.name});
-    }
-
-    const char * begin = config.query.data();
-    const char * end = begin + config.query.size();
-
-    ParserQuery parser(end, false);
-    auto query = parseQuery(parser, begin, end, "", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS);
-
-    for (UInt64 i = 0; i < num_streams; ++i)
-        pipes.emplace_back(std::make_shared<FuzzQuerySource>(max_block_size, block_header, config, query));
-
-    return Pipe::unitePipes(std::move(pipes));
-}
-
-StorageFuzzQuery::Configuration StorageFuzzQuery::getConfiguration(ASTs & engine_args, ContextPtr local_context)
-{
-    StorageFuzzQuery::Configuration configuration{};
-
-    // Supported signatures:
-    //
-    // FuzzQuery(query)
-    // FuzzQuery(query, max_query_length)
-    // FuzzQuery(query, max_query_length, random_seed)
-    if (engine_args.empty() || engine_args.size() > 3)
-        throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "FuzzQuery requires 1 to 3 arguments: query, max_query_length, random_seed");
-
-    for (auto & engine_arg : engine_args)
-        engine_arg = evaluateConstantExpressionOrIdentifierAsLiteral(engine_arg, local_context);
-
-    auto first_arg = checkAndGetLiteralArgument<String>(engine_args[0], "query");
-    configuration.query = std::move(first_arg);
-
-    if (engine_args.size() >= 2)
-    {
-        const auto & literal = engine_args[1]->as<const ASTLiteral &>();
-        if (!literal.value.isNull())
-            configuration.max_query_length = checkAndGetLiteralArgument<UInt64>(literal, "max_query_length");
-    }
-
-    if (engine_args.size() == 3)
-    {
-        const auto & literal = engine_args[2]->as<const ASTLiteral &>();
-        if (!literal.value.isNull())
-            configuration.random_seed = checkAndGetLiteralArgument<UInt64>(literal, "random_seed");
-    }
-
-    return configuration;
-}
-
-void registerStorageFuzzQuery(StorageFactory & factory)
-{
-    factory.registerStorage(
-        "FuzzQuery",
-        [](const StorageFactory::Arguments & args) -> std::shared_ptr<StorageFuzzQuery>
-        {
-            ASTs & engine_args = args.engine_args;
-
-            if (engine_args.empty())
-                throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Storage FuzzQuery must have arguments.");
-
-            StorageFuzzQuery::Configuration configuration = StorageFuzzQuery::getConfiguration(engine_args, args.getLocalContext());
-
-            for (const auto& col : args.columns)
-                if (col.type->getTypeId() != TypeIndex::String)
-                    throw Exception(ErrorCodes::BAD_ARGUMENTS, "'StorageFuzzQuery' supports only columns of String type, got {}.", col.type->getName());
-
-            return std::make_shared<StorageFuzzQuery>(args.table_id, args.columns, args.comment, configuration);
-        });
-}
-
-}
diff --git a/src/Storages/StorageFuzzQuery.h b/src/Storages/StorageFuzzQuery.h
deleted file mode 100644
index 125ef960e74..00000000000
--- a/src/Storages/StorageFuzzQuery.h
+++ /dev/null
@@ -1,88 +0,0 @@
-#pragma once
-
-#include <Storages/IStorage.h>
-#include <Storages/StorageConfiguration.h>
-#include <Common/randomSeed.h>
-#include <Common/QueryFuzzer.h>
-
-#include "config.h"
-
-namespace DB
-{
-
-class NamedCollection;
-
-class StorageFuzzQuery final : public IStorage
-{
-public:
-    struct Configuration : public StatelessTableEngineConfiguration
-    {
-        String query;
-        UInt64 max_query_length = 500;
-        UInt64 random_seed = randomSeed();
-    };
-
-    StorageFuzzQuery(
-        const StorageID & table_id_, const ColumnsDescription & columns_, const String & comment_, const Configuration & config_);
-
-    std::string getName() const override { return "FuzzQuery"; }
-
-    Pipe read(
-        const Names & column_names,
-        const StorageSnapshotPtr & storage_snapshot,
-        SelectQueryInfo & query_info,
-        ContextPtr context,
-        QueryProcessingStage::Enum processed_stage,
-        size_t max_block_size,
-        size_t num_streams) override;
-
-    static StorageFuzzQuery::Configuration getConfiguration(ASTs & engine_args, ContextPtr local_context);
-
-private:
-    const Configuration config;
-};
-
-
-class FuzzQuerySource : public ISource
-{
-public:
-    FuzzQuerySource(
-        UInt64 block_size_, Block block_header_, const StorageFuzzQuery::Configuration & config_, ASTPtr query_)
-        : ISource(block_header_)
-        , block_size(block_size_)
-        , block_header(std::move(block_header_))
-        , config(config_)
-        , query(query_)
-        , fuzzer(config_.random_seed)
-    {
-    }
-
-    String getName() const override { return "FuzzQuery"; }
-
-protected:
-    Chunk generate() override
-    {
-        Columns columns;
-        columns.reserve(block_header.columns());
-        for (const auto & col : block_header)
-        {
-            chassert(col.type->getTypeId() == TypeIndex::String);
-            columns.emplace_back(createColumn());
-        }
-
-        return {std::move(columns), block_size};
-    }
-
-private:
-    ColumnPtr createColumn();
-
-    UInt64 block_size;
-    Block block_header;
-
-    StorageFuzzQuery::Configuration config;
-    ASTPtr query;
-
-    QueryFuzzer fuzzer;
-};
-
-}
diff --git a/src/Storages/registerStorages.cpp b/src/Storages/registerStorages.cpp
index adc1074b1fe..8f33314397c 100644
--- a/src/Storages/registerStorages.cpp
+++ b/src/Storages/registerStorages.cpp
@@ -26,7 +26,6 @@ void registerStorageGenerateRandom(StorageFactory & factory);
 void registerStorageExecutable(StorageFactory & factory);
 void registerStorageWindowView(StorageFactory & factory);
 void registerStorageLoop(StorageFactory & factory);
-void registerStorageFuzzQuery(StorageFactory & factory);
 #if USE_RAPIDJSON || USE_SIMDJSON
 void registerStorageFuzzJSON(StorageFactory & factory);
 #endif
@@ -127,7 +126,6 @@ void registerStorages()
     registerStorageExecutable(factory);
     registerStorageWindowView(factory);
     registerStorageLoop(factory);
-    registerStorageFuzzQuery(factory);
 #if USE_RAPIDJSON || USE_SIMDJSON
     registerStorageFuzzJSON(factory);
 #endif
diff --git a/src/TableFunctions/TableFunctionFuzzQuery.cpp b/src/TableFunctions/TableFunctionFuzzQuery.cpp
deleted file mode 100644
index 224f6666556..00000000000
--- a/src/TableFunctions/TableFunctionFuzzQuery.cpp
+++ /dev/null
@@ -1,54 +0,0 @@
-#include <TableFunctions/TableFunctionFuzzQuery.h>
-
-#include <DataTypes/DataTypeString.h>
-#include <Storages/checkAndGetLiteralArgument.h>
-#include <TableFunctions/TableFunctionFactory.h>
-#include <TableFunctions/registerTableFunctions.h>
-
-namespace DB
-{
-
-
-namespace ErrorCodes
-{
-    extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
-}
-
-void TableFunctionFuzzQuery::parseArguments(const ASTPtr & ast_function, ContextPtr context)
-{
-    ASTs & args_func = ast_function->children;
-
-    if (args_func.size() != 1)
-        throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Table function '{}' must have arguments", getName());
-
-    auto args = args_func.at(0)->children;
-    configuration = StorageFuzzQuery::getConfiguration(args, context);
-}
-
-StoragePtr TableFunctionFuzzQuery::executeImpl(
-    const ASTPtr & /*ast_function*/,
-    ContextPtr context,
-    const std::string & table_name,
-    ColumnsDescription /*cached_columns*/,
-    bool is_insert_query) const
-{
-    ColumnsDescription columns = getActualTableStructure(context, is_insert_query);
-    auto res = std::make_shared<StorageFuzzQuery>(
-        StorageID(getDatabaseName(), table_name),
-        columns,
-        /* comment */ String{},
-        configuration);
-    res->startup();
-    return res;
-}
-
-void registerTableFunctionFuzzQuery(TableFunctionFactory & factory)
-{
-    factory.registerFunction<TableFunctionFuzzQuery>(
-        {.documentation
-         = {.description = "Perturbs a query string with random variations.",
-            .returned_value = "A table object with a single column containing perturbed query strings."},
-         .allow_readonly = true});
-}
-
-}
diff --git a/src/TableFunctions/TableFunctionFuzzQuery.h b/src/TableFunctions/TableFunctionFuzzQuery.h
deleted file mode 100644
index 22d10341c4d..00000000000
--- a/src/TableFunctions/TableFunctionFuzzQuery.h
+++ /dev/null
@@ -1,42 +0,0 @@
-#pragma once
-
-#include <optional>
-
-#include <TableFunctions/ITableFunction.h>
-#include <DataTypes/DataTypeString.h>
-#include <Storages/StorageFuzzQuery.h>
-
-#include "config.h"
-
-namespace DB
-{
-
-class TableFunctionFuzzQuery : public ITableFunction
-{
-public:
-    static constexpr auto name = "fuzzQuery";
-    std::string getName() const override { return name; }
-
-    void parseArguments(const ASTPtr & ast_function, ContextPtr context) override;
-
-    ColumnsDescription getActualTableStructure(ContextPtr /* context */, bool /* is_insert_query */) const override
-    {
-        return ColumnsDescription{{"query", std::make_shared<DataTypeString>()}};
-    }
-
-private:
-    StoragePtr executeImpl(
-        const ASTPtr & ast_function,
-        ContextPtr context,
-        const std::string & table_name,
-        ColumnsDescription cached_columns,
-        bool is_insert_query) const override;
-
-    const char * getStorageTypeName() const override { return "fuzzQuery"; }
-
-    String source;
-    std::optional<UInt64> random_seed;
-    StorageFuzzQuery::Configuration configuration;
-};
-
-}
diff --git a/src/TableFunctions/registerTableFunctions.cpp b/src/TableFunctions/registerTableFunctions.cpp
index a6c90872f12..ca4913898f9 100644
--- a/src/TableFunctions/registerTableFunctions.cpp
+++ b/src/TableFunctions/registerTableFunctions.cpp
@@ -26,7 +26,6 @@ void registerTableFunctions()
     registerTableFunctionMongoDB(factory);
     registerTableFunctionRedis(factory);
     registerTableFunctionMergeTreeIndex(factory);
-    registerTableFunctionFuzzQuery(factory);
 #if USE_RAPIDJSON || USE_SIMDJSON
     registerTableFunctionFuzzJSON(factory);
 #endif
diff --git a/src/TableFunctions/registerTableFunctions.h b/src/TableFunctions/registerTableFunctions.h
index 2a8864a9bfd..efde4d6dcdc 100644
--- a/src/TableFunctions/registerTableFunctions.h
+++ b/src/TableFunctions/registerTableFunctions.h
@@ -23,7 +23,6 @@ void registerTableFunctionGenerate(TableFunctionFactory & factory);
 void registerTableFunctionMongoDB(TableFunctionFactory & factory);
 void registerTableFunctionRedis(TableFunctionFactory & factory);
 void registerTableFunctionMergeTreeIndex(TableFunctionFactory & factory);
-void registerTableFunctionFuzzQuery(TableFunctionFactory & factory);
 #if USE_RAPIDJSON || USE_SIMDJSON
 void registerTableFunctionFuzzJSON(TableFunctionFactory & factory);
 #endif
diff --git a/tests/queries/0_stateless/03031_table_function_fuzzquery.reference b/tests/queries/0_stateless/03031_table_function_fuzzquery.reference
deleted file mode 100644
index 202e4557a33..00000000000
--- a/tests/queries/0_stateless/03031_table_function_fuzzquery.reference
+++ /dev/null
@@ -1,2 +0,0 @@
-query
-String
diff --git a/tests/queries/0_stateless/03031_table_function_fuzzquery.sql b/tests/queries/0_stateless/03031_table_function_fuzzquery.sql
deleted file mode 100644
index b26096f7f0e..00000000000
--- a/tests/queries/0_stateless/03031_table_function_fuzzquery.sql
+++ /dev/null
@@ -1,18 +0,0 @@
-
-SELECT * FROM fuzzQuery('SELECT 1', 500, 8956) LIMIT 0 FORMAT TSVWithNamesAndTypes;
-
-SELECT * FROM fuzzQuery('SELECT *
-FROM (
-  SELECT
-    ([toString(number % 2)] :: Array(LowCardinality(String))) AS item_id,
-    count()
-  FROM numbers(3)
-  GROUP BY item_id WITH TOTALS
-) AS l FULL JOIN (
-  SELECT
-    ([toString((number % 2) * 2)] :: Array(String)) AS item_id
-  FROM numbers(3)
-) AS r
-ON l.item_id = r.item_id
-ORDER BY 1,2,3;
-', 500, 8956) LIMIT 10 FORMAT NULL;

From e42e04c5230560f652c75c2c3b43efc3567cb31c Mon Sep 17 00:00:00 2001
From: Igor Nikonov <igor@clickhouse.com>
Date: Wed, 24 Jul 2024 12:53:52 +0000
Subject: [PATCH 0711/1488] Test

---
 ...llel_replicas_alter_select_ubsan.reference |  0
 ...5_parallel_replicas_alter_select_ubsan.sql | 35 +++++++++++++++++++
 2 files changed, 35 insertions(+)
 create mode 100644 tests/queries/0_stateless/03205_parallel_replicas_alter_select_ubsan.reference
 create mode 100644 tests/queries/0_stateless/03205_parallel_replicas_alter_select_ubsan.sql

diff --git a/tests/queries/0_stateless/03205_parallel_replicas_alter_select_ubsan.reference b/tests/queries/0_stateless/03205_parallel_replicas_alter_select_ubsan.reference
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/queries/0_stateless/03205_parallel_replicas_alter_select_ubsan.sql b/tests/queries/0_stateless/03205_parallel_replicas_alter_select_ubsan.sql
new file mode 100644
index 00000000000..2ec9368327d
--- /dev/null
+++ b/tests/queries/0_stateless/03205_parallel_replicas_alter_select_ubsan.sql
@@ -0,0 +1,35 @@
+SET alter_sync = 2;
+SET max_parallel_replicas = 3, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_for_non_replicated_merge_tree = true;
+
+DROP TABLE IF EXISTS t1;
+DROP TABLE IF EXISTS t1__fuzz_26;
+
+CREATE TABLE t1__fuzz_26 (`a` Nullable(Float64), `b` Nullable(Float32), `pk` Int64) ENGINE = MergeTree ORDER BY pk;
+CREATE TABLE t1 ( a Float64, b Int64, pk String) Engine = MergeTree() ORDER BY pk;
+
+ALTER TABLE t1
+    (MODIFY COLUMN `a` Float64 TTL toDateTime(b) + toIntervalMonth(viewExplain('EXPLAIN', 'actions = 1', (
+        SELECT
+            toIntervalMonth(1),
+            2
+        FROM t1__fuzz_26
+        GROUP BY
+            toFixedString('%Prewhere%', 10),
+            toNullable(12)
+            WITH ROLLUP
+    )), 1)) settings allow_experimental_parallel_reading_from_replicas = 1; -- { serverError INCORRECT_RESULT_OF_SCALAR_SUBQUERY }
+
+ALTER TABLE t1
+    (MODIFY COLUMN `a` Float64 TTL toDateTime(b) + toIntervalMonth(viewExplain('EXPLAIN', 'actions = 1', (
+        SELECT
+            toIntervalMonth(1),
+            2
+        FROM t1__fuzz_26
+        GROUP BY
+            toFixedString('%Prewhere%', 10),
+            toNullable(12)
+            WITH ROLLUP
+    )), 1)) settings allow_experimental_parallel_reading_from_replicas = 0; -- { serverError INCORRECT_RESULT_OF_SCALAR_SUBQUERY }
+
+DROP TABLE t1;
+DROP TABLE t1__fuzz_26;

From 80304b59626c2cf2d9cbe3ba39058b9dfd22ddd9 Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Wed, 24 Jul 2024 15:13:10 +0200
Subject: [PATCH 0712/1488] Less memory-intensive fill queries for decimal
 aggregates

---
 tests/performance/decimal_aggregates.xml | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/tests/performance/decimal_aggregates.xml b/tests/performance/decimal_aggregates.xml
index ec88be0124f..724d0c5d0e6 100644
--- a/tests/performance/decimal_aggregates.xml
+++ b/tests/performance/decimal_aggregates.xml
@@ -4,8 +4,13 @@
     </settings>
 
     <create_query>CREATE TABLE t (x UInt64, d32 Decimal32(3), d64 Decimal64(4), d128 Decimal128(5)) ENGINE = Memory</create_query>
-    <!-- use less threads to save memory -->
-    <fill_query>INSERT INTO t SELECT number AS x, x % 1000000 AS d32, x AS d64, x d128 FROM numbers_mt(500000000) SETTINGS max_threads = 8</fill_query>
+    <!-- use less threads and several queries to save memory -->
+    <fill_query>INSERT INTO t SELECT number AS x, x % 1000000 AS d32, x AS d64, x d128 FROM numbers_mt(100000000) SETTINGS max_threads = 2</fill_query>
+    <fill_query>INSERT INTO t SELECT number AS x, x % 1000000 AS d32, x AS d64, x d128 FROM numbers_mt(100000000, 100000000) SETTINGS max_threads = 2</fill_query>
+    <fill_query>INSERT INTO t SELECT number AS x, x % 1000000 AS d32, x AS d64, x d128 FROM numbers_mt(200000000, 100000000) SETTINGS max_threads = 2</fill_query>
+    <fill_query>INSERT INTO t SELECT number AS x, x % 1000000 AS d32, x AS d64, x d128 FROM numbers_mt(300000000, 100000000) SETTINGS max_threads = 2</fill_query>
+    <fill_query>INSERT INTO t SELECT number AS x, x % 1000000 AS d32, x AS d64, x d128 FROM numbers_mt(400000000, 100000000) SETTINGS max_threads = 2</fill_query>
+
     <drop_query>DROP TABLE IF EXISTS t</drop_query>
 
     <query>SELECT min(d32), max(d32), argMin(x, d32), argMax(x, d32) FROM t</query>

From a03a59a3bfd2dc55f1f04197cad36d147edb3ee1 Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Wed, 24 Jul 2024 15:14:02 +0200
Subject: [PATCH 0713/1488] Use less data overall

---
 tests/performance/decimal_aggregates.xml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tests/performance/decimal_aggregates.xml b/tests/performance/decimal_aggregates.xml
index 724d0c5d0e6..9fc94f01a4d 100644
--- a/tests/performance/decimal_aggregates.xml
+++ b/tests/performance/decimal_aggregates.xml
@@ -8,8 +8,6 @@
     <fill_query>INSERT INTO t SELECT number AS x, x % 1000000 AS d32, x AS d64, x d128 FROM numbers_mt(100000000) SETTINGS max_threads = 2</fill_query>
     <fill_query>INSERT INTO t SELECT number AS x, x % 1000000 AS d32, x AS d64, x d128 FROM numbers_mt(100000000, 100000000) SETTINGS max_threads = 2</fill_query>
     <fill_query>INSERT INTO t SELECT number AS x, x % 1000000 AS d32, x AS d64, x d128 FROM numbers_mt(200000000, 100000000) SETTINGS max_threads = 2</fill_query>
-    <fill_query>INSERT INTO t SELECT number AS x, x % 1000000 AS d32, x AS d64, x d128 FROM numbers_mt(300000000, 100000000) SETTINGS max_threads = 2</fill_query>
-    <fill_query>INSERT INTO t SELECT number AS x, x % 1000000 AS d32, x AS d64, x d128 FROM numbers_mt(400000000, 100000000) SETTINGS max_threads = 2</fill_query>
 
     <drop_query>DROP TABLE IF EXISTS t</drop_query>
 

From 3c6c9303c3046d780ef01ce0fcb87f03b004162a Mon Sep 17 00:00:00 2001
From: Anton Popov <anton@clickhouse.com>
Date: Wed, 24 Jul 2024 13:14:43 +0000
Subject: [PATCH 0714/1488] disable setting optimize_functions_to_subcolumns

---
 src/Core/Settings.h   | 2 +-
 tests/clickhouse-test | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/Core/Settings.h b/src/Core/Settings.h
index 3d181e33001..fbb7663b612 100644
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@@ -602,7 +602,7 @@ class IColumn;
     M(Bool, optimize_if_chain_to_multiif, false, "Replace if(cond1, then1, if(cond2, ...)) chains to multiIf. Currently it's not beneficial for numeric types.", 0) \
     M(Bool, optimize_multiif_to_if, true, "Replace 'multiIf' with only one condition to 'if'.", 0) \
     M(Bool, optimize_if_transform_strings_to_enum, false, "Replaces string-type arguments in If and Transform to enum. Disabled by default cause it could make inconsistent change in distributed query that would lead to its fail.", 0) \
-    M(Bool, optimize_functions_to_subcolumns, true, "Transform functions to subcolumns, if possible, to reduce amount of read data. E.g. 'length(arr)' -> 'arr.size0', 'col IS NULL' -> 'col.null' ", 0) \
+    M(Bool, optimize_functions_to_subcolumns, false, "Transform functions to subcolumns, if possible, to reduce amount of read data. E.g. 'length(arr)' -> 'arr.size0', 'col IS NULL' -> 'col.null' ", 0) \
     M(Bool, optimize_using_constraints, false, "Use constraints for query optimization", 0)                                                                                                                                           \
     M(Bool, optimize_substitute_columns, false, "Use constraints for column substitution", 0)                                                                                                                                         \
     M(Bool, optimize_append_index, false, "Use constraints in order to append index condition (indexHint)", 0) \
diff --git a/tests/clickhouse-test b/tests/clickhouse-test
index 0c04d8fb2c3..a29c786e998 100755
--- a/tests/clickhouse-test
+++ b/tests/clickhouse-test
@@ -835,6 +835,7 @@ class SettingsRandomizer:
         "cross_join_min_bytes_to_compress": lambda: random.choice([0, 1, 100000000]),
         "min_external_table_block_size_bytes": lambda: random.choice([0, 1, 100000000]),
         "max_parsing_threads": lambda: random.choice([0, 1, 10]),
+        "optimize_functions_to_subcolumns": lambda: random.randint(0, 1),
     }
 
     @staticmethod

From 5d14d823e3882887ab8238c6ee5778f552cce835 Mon Sep 17 00:00:00 2001
From: Anton Popov <anton@clickhouse.com>
Date: Wed, 24 Jul 2024 13:27:19 +0000
Subject: [PATCH 0715/1488] remove setting from changes history

---
 src/Core/SettingsChangesHistory.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp
index de4725dc350..c395bfdc815 100644
--- a/src/Core/SettingsChangesHistory.cpp
+++ b/src/Core/SettingsChangesHistory.cpp
@@ -63,7 +63,6 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
               {"output_format_native_encode_types_in_binary_format", false, false, "Added new setting to allow to write type names in binary format in Native output format"},
               {"input_format_native_decode_types_in_binary_format", false, false, "Added new setting to allow to read type names in binary format in Native output format"},
               {"read_in_order_use_buffering", false, true, "Use buffering before merging while reading in order of primary key"},
-              {"optimize_functions_to_subcolumns", false, true, "Enable optimization by default"},
               {"enable_named_columns_in_function_tuple", false, true, "Generate named tuples in function tuple() when all names are unique and can be treated as unquoted identifiers."},
               {"input_format_json_ignore_key_case", false, false, "Ignore json key case while read json field from string."},
               {"optimize_trivial_insert_select", true, false, "The optimization does not make sense in many cases."},

From 63f9f65ab1d2508c21d124b715ac5e397d91c65b Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Wed, 24 Jul 2024 15:33:59 +0200
Subject: [PATCH 0716/1488] Bump


From 535c872b39b7f593e9e63e614a78d39cde5dd4d9 Mon Sep 17 00:00:00 2001
From: Yohann Jardin <yohann.jardin@telecomnancy.net>
Date: Wed, 24 Jul 2024 15:46:48 +0200
Subject: [PATCH 0717/1488] empty comit


From c5d262c23b4ee289a374ae5d817a101bd44154c4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?=
 <benjamin.antal@clickhouse.com>
Date: Wed, 24 Jul 2024 13:55:59 +0000
Subject: [PATCH 0718/1488] Increase max allocation size for sanitizers

---
 docker/test/base/Dockerfile | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/docker/test/base/Dockerfile b/docker/test/base/Dockerfile
index 2317f84e0cb..79ed871b822 100644
--- a/docker/test/base/Dockerfile
+++ b/docker/test/base/Dockerfile
@@ -23,15 +23,16 @@ RUN apt-get update \
 # and MEMORY_LIMIT_EXCEEDED exceptions in Functional tests (total memory limit in Functional tests is ~55.24 GiB).
 # TSAN will flush shadow memory when reaching this limit.
 # It may cause false-negatives, but it's better than OOM.
-RUN echo "TSAN_OPTIONS='verbosity=1000 halt_on_error=1 abort_on_error=1 history_size=7 memory_limit_mb=46080 second_deadlock_stack=1'" >> /etc/environment
-RUN echo "UBSAN_OPTIONS='print_stacktrace=1'" >> /etc/environment
-RUN echo "MSAN_OPTIONS='abort_on_error=1 poison_in_dtor=1'" >> /etc/environment
-RUN echo "LSAN_OPTIONS='suppressions=/usr/share/clickhouse-test/config/lsan_suppressions.txt'" >> /etc/environment
+#  max_allocation_size_mb is set to 32GB, so we have much bigger chance to run into memory limit than the limitation of the sanitizers
+RUN echo "TSAN_OPTIONS='verbosity=1000 halt_on_error=1 abort_on_error=1 history_size=7 memory_limit_mb=46080 second_deadlock_stack=1 max_allocation_size_mb=32768'" >> /etc/environment
+RUN echo "UBSAN_OPTIONS='print_stacktrace=1 max_allocation_size_mb=32768'" >> /etc/environment
+RUN echo "MSAN_OPTIONS='abort_on_error=1 poison_in_dtor=1 max_allocation_size_mb=32768'" >> /etc/environment
+RUN echo "LSAN_OPTIONS='suppressions=/usr/share/clickhouse-test/config/lsan_suppressions.txt max_allocation_size_mb=32768'" >> /etc/environment
 # Sanitizer options for current shell (not current, but the one that will be spawned on "docker run")
 # (but w/o verbosity for TSAN, otherwise test.reference will not match)
-ENV TSAN_OPTIONS='halt_on_error=1 abort_on_error=1 history_size=7 memory_limit_mb=46080 second_deadlock_stack=1'
-ENV UBSAN_OPTIONS='print_stacktrace=1'
-ENV MSAN_OPTIONS='abort_on_error=1 poison_in_dtor=1'
+ENV TSAN_OPTIONS='halt_on_error=1 abort_on_error=1 history_size=7 memory_limit_mb=46080 second_deadlock_stack=1 max_allocation_size_mb=32768'
+ENV UBSAN_OPTIONS='print_stacktrace=1 max_allocation_size_mb=32768'
+ENV MSAN_OPTIONS='abort_on_error=1 poison_in_dtor=1 max_allocation_size_mb=32768'
 
 # for external_symbolizer_path
 RUN ln -s /usr/bin/llvm-symbolizer-${LLVM_VERSION} /usr/bin/llvm-symbolizer

From e498b86f50dbba8ac8e50f832547850c7cc29c6a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?=
 <benjamin.antal@clickhouse.com>
Date: Wed, 24 Jul 2024 14:00:28 +0000
Subject: [PATCH 0719/1488] Add option for LSAN in shell also

---
 docker/test/base/Dockerfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docker/test/base/Dockerfile b/docker/test/base/Dockerfile
index 79ed871b822..a81826ed6b5 100644
--- a/docker/test/base/Dockerfile
+++ b/docker/test/base/Dockerfile
@@ -33,6 +33,7 @@ RUN echo "LSAN_OPTIONS='suppressions=/usr/share/clickhouse-test/config/lsan_supp
 ENV TSAN_OPTIONS='halt_on_error=1 abort_on_error=1 history_size=7 memory_limit_mb=46080 second_deadlock_stack=1 max_allocation_size_mb=32768'
 ENV UBSAN_OPTIONS='print_stacktrace=1 max_allocation_size_mb=32768'
 ENV MSAN_OPTIONS='abort_on_error=1 poison_in_dtor=1 max_allocation_size_mb=32768'
+ENV LSAN_OPTIONS='max_allocation_size_mb=32768'
 
 # for external_symbolizer_path
 RUN ln -s /usr/bin/llvm-symbolizer-${LLVM_VERSION} /usr/bin/llvm-symbolizer

From 37c345bb4925095da3e82e3fc3ed27072786d7e7 Mon Sep 17 00:00:00 2001
From: Sema Checherinda <Sema.Checherinda@clickhouse.com>
Date: Wed, 24 Jul 2024 16:01:21 +0200
Subject: [PATCH 0720/1488] rewrite 01171 test

---
 ..._mv_select_insert_isolation_long.reference |   4 -
 .../01171_mv_select_insert_isolation_long.sh  | 229 ++++++++++++------
 2 files changed, 152 insertions(+), 81 deletions(-)

diff --git a/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.reference b/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.reference
index d8bb9e310e6..e69de29bb2d 100644
--- a/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.reference
+++ b/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.reference
@@ -1,4 +0,0 @@
-275	0	138	136	0
-275	0
-275	0	138	136	0
-275	0
diff --git a/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh b/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh
index 2ab7f883367..f6850864be5 100755
--- a/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh
+++ b/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Tags: long, no-ordinary-database, no-debug
+# Tags: long, no-ordinary-database
 # Test is too heavy, avoid parallel run in Flaky Check
 # shellcheck disable=SC2119
 
@@ -7,82 +7,125 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
-set -e
+set -ue
 
 $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS src";
 $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS dst";
 $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS mv";
 $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS tmp";
-$CLICKHOUSE_CLIENT --query "CREATE TABLE src (n Int8, m Int8, CONSTRAINT c CHECK xxHash32(n+m) % 8 != 0) ENGINE=MergeTree ORDER BY n PARTITION BY 0 < n SETTINGS old_parts_lifetime=0";
-$CLICKHOUSE_CLIENT --query "CREATE TABLE dst (nm Int16, CONSTRAINT c CHECK xxHash32(nm) % 8 != 0) ENGINE=MergeTree ORDER BY nm SETTINGS old_parts_lifetime=0";
-$CLICKHOUSE_CLIENT --query "CREATE MATERIALIZED VIEW mv TO dst (nm Int16) AS SELECT n*m AS nm FROM src";
 
-$CLICKHOUSE_CLIENT --query "CREATE TABLE tmp (x UInt8, nm Int16) ENGINE=MergeTree ORDER BY (x, nm) SETTINGS old_parts_lifetime=0"
+$CLICKHOUSE_CLIENT --query "CREATE TABLE src (n Int32, m Int32, CONSTRAINT c CHECK xxHash32(n+m) % 8 != 0) ENGINE=MergeTree ORDER BY n PARTITION BY 0 < n SETTINGS old_parts_lifetime=0";
+$CLICKHOUSE_CLIENT --query "CREATE TABLE dst (nm Int32, CONSTRAINT c CHECK xxHash32(nm) % 8 != 0) ENGINE=MergeTree ORDER BY nm SETTINGS old_parts_lifetime=0";
+$CLICKHOUSE_CLIENT --query "CREATE MATERIALIZED VIEW mv TO dst (nm Int32) AS SELECT n*m AS nm FROM src";
+
+$CLICKHOUSE_CLIENT --query "CREATE TABLE tmp (x UInt32, nm Int32) ENGINE=MergeTree ORDER BY (x, nm) SETTINGS old_parts_lifetime=0"
 
 $CLICKHOUSE_CLIENT --query "INSERT INTO src VALUES (0, 0)"
 
-# some transactions will fail due to constraint
-function thread_insert_commit()
+function get_now()
 {
-    set -e
-    for i in {1..100}; do
-        $CLICKHOUSE_CLIENT --multiquery --query "
-        BEGIN TRANSACTION;
-        INSERT INTO src VALUES /* ($i, $1) */ ($i, $1);
-        SELECT throwIf((SELECT sum(nm) FROM mv) != $(($i * $1))) FORMAT Null;
-        INSERT INTO src VALUES /* (-$i, $1) */ (-$i, $1);
-        COMMIT;" 2>&1| grep -Fv "is violated at row" | grep -Fv "Transaction is not in RUNNING state" | grep -F "Received from " ||:
-    done
+    date +%s
 }
 
-function thread_insert_rollback()
+is_pid_exist()
+{
+    local pid=$1
+    ps -p $pid > /dev/null
+}
+
+function run_until_deadline_and_at_least_times()
 {
     set -e
-    for _ in {1..100}; do
-        $CLICKHOUSE_CLIENT --multiquery --query "
-        BEGIN TRANSACTION;
-        INSERT INTO src VALUES /* (42, $1) */ (42, $1);
-        SELECT throwIf((SELECT count() FROM src WHERE n=42 AND m=$1) != 1) FORMAT Null;
-        ROLLBACK;"
+
+    local deadline=$1; shift
+    local min_iterations=$1; shift
+    local function_to_run=$1; shift
+
+    local started_time=$(get_now)
+    local i=0
+
+    while true
+    do
+        $function_to_run $i $@
+
+        [[ $(get_now) -lt $deadline ]] || break
+
+        i=$(($i + 1))
     done
+
+    [[ $i -gt $min_iterations ]] || echo "$i/$min_iterations : not enough iterations of $function_to_run has been made from $started_time until $deadline" >&2
+}
+
+function insert_commit_action()
+{
+    set -e
+
+    local i=$1; shift
+    local tag=$1; shift
+
+    # some transactions will fail due to constraint
+    $CLICKHOUSE_CLIENT --multiquery --query "
+        BEGIN TRANSACTION;
+        INSERT INTO src VALUES /* ($i, $tag) */ ($i, $tag);
+        SELECT throwIf((SELECT sum(nm) FROM mv) != $(($i * $tag))) /* ($i, $tag) */ FORMAT Null;
+        INSERT INTO src VALUES /* (-$i, $tag) */ (-$i, $tag);
+        COMMIT;
+    " 2>&1 \
+    | grep -Fv "is violated at row" | grep -Fv "Transaction is not in RUNNING state" | grep -F "Received from " ||:
+}
+
+
+function insert_rollback_action()
+{
+    set -e
+
+    local i=$1; shift
+    local tag=$1; shift
+
+    $CLICKHOUSE_CLIENT --multiquery --query "
+        BEGIN TRANSACTION;
+        INSERT INTO src VALUES /* (42, $tag) */ (42, $tag);
+        SELECT throwIf((SELECT count() FROM src WHERE n=42 AND m=$tag) != 1) FORMAT Null;
+        ROLLBACK;"
 }
 
 # make merges more aggressive
-function thread_optimize()
+function optimize_action()
 {
     set -e
-    while true; do
-        optimize_query="OPTIMIZE TABLE src"
-        partition_id=$(( RANDOM % 2 ))
-        if (( RANDOM % 2 )); then
-            optimize_query="OPTIMIZE TABLE dst"
-            partition_id="all"
-        fi
-        if (( RANDOM % 2 )); then
-            optimize_query="$optimize_query PARTITION ID '$partition_id'"
-        fi
-        if (( RANDOM % 2 )); then
-            optimize_query="$optimize_query FINAL"
-        fi
-        action="COMMIT"
-        if (( RANDOM % 4 )); then
-            action="ROLLBACK"
-        fi
 
-        $CLICKHOUSE_CLIENT --multiquery --query "
+    optimize_query="OPTIMIZE TABLE src"
+    partition_id=$(( RANDOM % 2 ))
+    if (( RANDOM % 2 )); then
+        optimize_query="OPTIMIZE TABLE dst"
+        partition_id="all"
+    fi
+    if (( RANDOM % 2 )); then
+        optimize_query="$optimize_query PARTITION ID '$partition_id'"
+    fi
+    if (( RANDOM % 2 )); then
+        optimize_query="$optimize_query FINAL"
+    fi
+    action="COMMIT"
+    if (( RANDOM % 4 )); then
+        action="ROLLBACK"
+    fi
+
+    $CLICKHOUSE_CLIENT --multiquery --query "
         BEGIN TRANSACTION;
-        $optimize_query;
+            $optimize_query;
         $action;
-        " 2>&1| grep -Fv "already exists, but it will be deleted soon" | grep -F "Received from " ||:
-        sleep 0.$RANDOM;
-    done
+    " 2>&1 \
+    | grep -Fv "already exists, but it will be deleted soon" | grep -F "Received from " ||:
+
+    sleep 0.$RANDOM;
 }
 
-function thread_select()
+function select_action()
 {
     set -e
-    while true; do
-        $CLICKHOUSE_CLIENT --multiquery --query "
+
+    $CLICKHOUSE_CLIENT --multiquery --query "
         BEGIN TRANSACTION;
         SELECT throwIf((SELECT (sum(n), count() % 2) FROM src) != (0, 1)) FORMAT Null;
         SELECT throwIf((SELECT (sum(nm), count() % 2) FROM mv) != (0, 1)) FORMAT Null;
@@ -90,14 +133,13 @@ function thread_select()
         SELECT throwIf((SELECT arraySort(groupArray(nm)) FROM mv) != (SELECT arraySort(groupArray(nm)) FROM dst)) FORMAT Null;
         SELECT throwIf((SELECT arraySort(groupArray(nm)) FROM mv) != (SELECT arraySort(groupArray(n*m)) FROM src)) FORMAT Null;
         COMMIT;"
-    done
 }
 
-function thread_select_insert()
+function select_insert_action()
 {
     set -e
-    while true; do
-        $CLICKHOUSE_CLIENT --multiquery --query "
+
+    $CLICKHOUSE_CLIENT --multiquery --query "
         BEGIN TRANSACTION;
         SELECT throwIf((SELECT count() FROM tmp) != 0) FORMAT Null;
         INSERT INTO tmp SELECT 1, n*m FROM src;
@@ -110,36 +152,69 @@ function thread_select_insert()
         SELECT throwIf(1 != (SELECT countDistinct(arr) FROM (SELECT x, arraySort(groupArray(nm)) AS arr FROM tmp WHERE x!=4 GROUP BY x))) FORMAT Null;
         SELECT throwIf((SELECT count(), sum(nm) FROM tmp WHERE x=4) != (SELECT count(), sum(nm) FROM tmp WHERE x!=4)) FORMAT Null;
         ROLLBACK;"
-    done
 }
 
-thread_insert_commit 1 & PID_1=$!
-thread_insert_commit 2 & PID_2=$!
-thread_insert_rollback 3 & PID_3=$!
+MAIN_TIME_PART=400
+SECOND_TIME_PART=30
+WAIT_FINISH=60
+LAST_TIME_GAP=10
 
-thread_optimize & PID_4=$!
-thread_select & PID_5=$!
-thread_select_insert & PID_6=$!
-sleep 0.$RANDOM;
-thread_select & PID_7=$!
-thread_select_insert & PID_8=$!
+if [[ $((MAIN_TIME_PART + SECOND_TIME_PART + WAIT_FINISH + LAST_TIME_GAP)) -ge  600 ]]; then
+    echo "time sttings are wrong" 2>&1
+    exit 1
+fi
 
-wait $PID_1 && wait $PID_2 && wait $PID_3
-kill -TERM $PID_4
-kill -TERM $PID_5
-kill -TERM $PID_6
-kill -TERM $PID_7
-kill -TERM $PID_8
-wait
-wait_for_queries_to_finish 40
+START_TIME=$(get_now)
+STOP_TIME=$((START_TIME + MAIN_TIME_PART))
+SECOND_STOP_TIME=$((STOP_TIME + SECOND_TIME_PART))
+MIN_ITERATIONS=50
+
+run_until_deadline_and_at_least_times $STOP_TIME $MIN_ITERATIONS insert_commit_action 1   & PID_1=$!
+run_until_deadline_and_at_least_times $STOP_TIME $MIN_ITERATIONS insert_commit_action 2   & PID_2=$!
+run_until_deadline_and_at_least_times $STOP_TIME $MIN_ITERATIONS insert_rollback_action 3 & PID_3=$!
+
+run_until_deadline_and_at_least_times $SECOND_STOP_TIME $MIN_ITERATIONS optimize_action      & PID_4=$!
+run_until_deadline_and_at_least_times $SECOND_STOP_TIME $MIN_ITERATIONS select_action        & PID_5=$!
+run_until_deadline_and_at_least_times $SECOND_STOP_TIME $MIN_ITERATIONS select_insert_action & PID_6=$!
+sleep 0.$RANDOM
+run_until_deadline_and_at_least_times $SECOND_STOP_TIME $MIN_ITERATIONS select_action        & PID_7=$!
+run_until_deadline_and_at_least_times $SECOND_STOP_TIME $MIN_ITERATIONS select_insert_action & PID_8=$!
+
+wait $PID_1 || echo "insert_commit_action has failed with status $?" 2>&1
+wait $PID_2 || echo "second insert_commit_action has failed with status $?" 2>&1
+wait $PID_3 || echo "insert_rollback_action has failed with status $?" 2>&1
+
+is_pid_exist $PID_4 || echo "optimize_action is not running" 2>&1
+is_pid_exist $PID_5 || echo "select_action is not running" 2>&1
+is_pid_exist $PID_6 || echo "select_insert_action is not running" 2>&1
+is_pid_exist $PID_7 || echo "second select_action is not running" 2>&1
+is_pid_exist $PID_8 || echo "second select_insert_action is not running" 2>&1
+
+wait $PID_4 || echo "optimize_action has failed with status $?" 2>&1
+wait $PID_5 || echo "select_action has failed with status $?" 2>&1
+wait $PID_6 || echo "select_insert_action has failed with status $?" 2>&1
+wait $PID_7 || echo "second select_action has failed with status $?" 2>&1
+wait $PID_8 || echo "second select_insert_action has failed with status $?" 2>&1
+
+wait_for_queries_to_finish $WAIT_FINISH
 
 $CLICKHOUSE_CLIENT --multiquery --query "
-BEGIN TRANSACTION;
-SELECT count(), sum(n), sum(m=1), sum(m=2), sum(m=3) FROM src;
-SELECT count(), sum(nm) FROM mv";
+    BEGIN TRANSACTION;
+        SELECT throwIf((SELECT (sum(n), count() % 2) FROM src) != (0, 1)) FORMAT Null;
+        SELECT throwIf((SELECT (sum(nm), count() % 2) FROM mv) != (0, 1)) FORMAT Null;
+        SELECT throwIf((SELECT (sum(nm), count() % 2) FROM dst) != (0, 1)) FORMAT Null;
+        SELECT throwIf((SELECT arraySort(groupArray(nm)) FROM mv) != (SELECT arraySort(groupArray(nm)) FROM dst)) FORMAT Null;
+        SELECT throwIf((SELECT arraySort(groupArray(nm)) FROM mv) != (SELECT arraySort(groupArray(n*m)) FROM src)) FORMAT Null;
+    COMMIT;
+"
 
-$CLICKHOUSE_CLIENT --query "SELECT count(), sum(n), sum(m=1), sum(m=2), sum(m=3) FROM src"
-$CLICKHOUSE_CLIENT --query "SELECT count(), sum(nm) FROM mv"
+$CLICKHOUSE_CLIENT --multiquery --query  "
+    SELECT throwIf((SELECT (sum(n), count() % 2) FROM src) != (0, 1)) FORMAT Null;
+    SELECT throwIf((SELECT (sum(nm), count() % 2) FROM mv) != (0, 1)) FORMAT Null;
+    SELECT throwIf((SELECT (sum(nm), count() % 2) FROM dst) != (0, 1)) FORMAT Null;
+    SELECT throwIf((SELECT arraySort(groupArray(nm)) FROM mv) != (SELECT arraySort(groupArray(nm)) FROM dst)) FORMAT Null;
+    SELECT throwIf((SELECT arraySort(groupArray(nm)) FROM mv) != (SELECT arraySort(groupArray(n*m)) FROM src)) FORMAT Null;
+"
 
 $CLICKHOUSE_CLIENT --query "DROP TABLE src";
 $CLICKHOUSE_CLIENT --query "DROP TABLE dst";

From 62d956f8a882f74a930340cce1650babc13bf7a1 Mon Sep 17 00:00:00 2001
From: zoomxi <419486879@qq.com>
Date: Wed, 24 Jul 2024 22:39:30 +0800
Subject: [PATCH 0721/1488] remove unused file
 ParallelReplicasReadingCoordinator.h

---
 src/Interpreters/ClusterProxy/executeQuery.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/Interpreters/ClusterProxy/executeQuery.cpp b/src/Interpreters/ClusterProxy/executeQuery.cpp
index 59f095f7487..d04a73e384e 100644
--- a/src/Interpreters/ClusterProxy/executeQuery.cpp
+++ b/src/Interpreters/ClusterProxy/executeQuery.cpp
@@ -24,7 +24,6 @@
 #include <Processors/Sources/NullSource.h>
 #include <QueryPipeline/Pipe.h>
 #include <Storages/Distributed/DistributedSettings.h>
-#include <Storages/MergeTree/ParallelReplicasReadingCoordinator.h>
 #include <Storages/SelectQueryInfo.h>
 #include <Storages/StorageReplicatedMergeTree.h>
 #include <Storages/StorageSnapshot.h>

From 4ab4b6729224806e7d34f1bee91a21dcd1818e8f Mon Sep 17 00:00:00 2001
From: Justin de Guzman <justin@justindeguzman.net>
Date: Wed, 24 Jul 2024 07:59:34 -0700
Subject: [PATCH 0722/1488] Update link for JSON schema inference

---
 docs/en/getting-started/example-datasets/nypd_complaint_data.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/en/getting-started/example-datasets/nypd_complaint_data.md b/docs/en/getting-started/example-datasets/nypd_complaint_data.md
index a178fe456a6..1caa1ab6a9a 100644
--- a/docs/en/getting-started/example-datasets/nypd_complaint_data.md
+++ b/docs/en/getting-started/example-datasets/nypd_complaint_data.md
@@ -55,7 +55,7 @@ CMPLNT_FR_TM                Nullable(String)
 ```
 
 :::tip
-Most of the time the above command will let you know which fields in the input data are numeric, and which are strings, and which are tuples.  This is not always the case.  Because ClickHouse is routineley used with datasets containing billions of records there is a default number (100) of rows examined to [infer the schema](/docs/en/integrations/data-ingestion/data-formats/json.md#relying-on-schema-inference) in order to avoid parsing billions of rows to infer the schema. The response below may not match what you see, as the dataset is updated several times each year. Looking at the Data Dictionary you can see that CMPLNT_NUM is specified as text, and not numeric.  By overriding the default of 100 rows for inference with the setting `SETTINGS input_format_max_rows_to_read_for_schema_inference=2000`
+Most of the time the above command will let you know which fields in the input data are numeric, and which are strings, and which are tuples.  This is not always the case.  Because ClickHouse is routineley used with datasets containing billions of records there is a default number (100) of rows examined to [infer the schema](/en/integrations/data-formats/json/inference) in order to avoid parsing billions of rows to infer the schema. The response below may not match what you see, as the dataset is updated several times each year. Looking at the Data Dictionary you can see that CMPLNT_NUM is specified as text, and not numeric.  By overriding the default of 100 rows for inference with the setting `SETTINGS input_format_max_rows_to_read_for_schema_inference=2000`
 you can get a better idea of the content.
 
 Note: as of version 22.5 the default is now 25,000 rows for inferring the schema, so only change the setting if you are on an older version or if you need more than 25,000 rows to be sampled.

From f03d4bb7d5d40203bba68c4f8958d584f27ae881 Mon Sep 17 00:00:00 2001
From: Pablo Marcos <pablo.marcos.oltra@clickhouse.com>
Date: Wed, 24 Jul 2024 15:07:53 +0000
Subject: [PATCH 0723/1488] Format with black

---
 .../integration/test_storage_s3_queue/test.py | 24 ++++++++++++-------
 1 file changed, 15 insertions(+), 9 deletions(-)

diff --git a/tests/integration/test_storage_s3_queue/test.py b/tests/integration/test_storage_s3_queue/test.py
index e178b3b6608..4348857acd3 100644
--- a/tests/integration/test_storage_s3_queue/test.py
+++ b/tests/integration/test_storage_s3_queue/test.py
@@ -823,7 +823,7 @@ def test_multiple_tables_streaming_sync_distributed(started_cluster, mode):
 
 def test_max_set_age(started_cluster):
     node = started_cluster.instances["instance"]
-    table_name = f"max_set_age_{uuid4().hex}"
+    table_name = "max_set_age"
     dst_table_name = f"{table_name}_dst"
     keeper_path = f"/clickhouse/test_{table_name}"
     files_path = f"{table_name}_data"
@@ -848,9 +848,7 @@ def test_max_set_age(started_cluster):
     )
     create_mv(node, table_name, dst_table_name)
 
-    _ = generate_random_files(
-        started_cluster, files_path, files_to_generate, row_num=1
-    )
+    _ = generate_random_files(started_cluster, files_path, files_to_generate, row_num=1)
 
     expected_rows = files_to_generate
 
@@ -869,13 +867,17 @@ def test_max_set_age(started_cluster):
         assert False
 
     wait_for_condition(lambda: get_count() == expected_rows)
-    assert files_to_generate == int(node.query(f"SELECT uniq(_path) from {dst_table_name}"))
+    assert files_to_generate == int(
+        node.query(f"SELECT uniq(_path) from {dst_table_name}")
+    )
 
     time.sleep(max_age + 5)
 
     expected_rows *= 2
     wait_for_condition(lambda: get_count() == expected_rows)
-    assert files_to_generate == int(node.query(f"SELECT uniq(_path) from {dst_table_name}"))
+    assert files_to_generate == int(
+        node.query(f"SELECT uniq(_path) from {dst_table_name}")
+    )
 
     paths_count = [
         int(x)
@@ -888,9 +890,11 @@ def test_max_set_age(started_cluster):
         assert 2 == path_count
 
     def get_object_storage_failures():
-        return int(node.query(
-            "SELECT value FROM system.events WHERE name = 'ObjectStorageQueueFailedFiles' SETTINGS system_events_show_zero_values=1"
-        ))
+        return int(
+            node.query(
+                "SELECT value FROM system.events WHERE name = 'ObjectStorageQueueFailedFiles' SETTINGS system_events_show_zero_values=1"
+            )
+        )
 
     failed_count = get_object_storage_failures()
 
@@ -900,6 +904,8 @@ def test_max_set_age(started_cluster):
     values_csv = (
         "\n".join((",".join(map(str, row)) for row in values)) + "\n"
     ).encode()
+
+    # use a different filename for each test to allow running a bunch of them sequentially with --count
     file_with_error = f"fff_{uuid4().hex}.csv"
     put_s3_file_content(started_cluster, f"{files_path}/{file_with_error}", values_csv)
 

From f990f235da10b72e1625d007563761da52067753 Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Wed, 24 Jul 2024 17:20:19 +0200
Subject: [PATCH 0724/1488] Increase timeout for test_broken_part_during_merge

---
 tests/integration/test_broken_part_during_merge/test.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/integration/test_broken_part_during_merge/test.py b/tests/integration/test_broken_part_during_merge/test.py
index 19c22201fb0..0ba7beeb1fd 100644
--- a/tests/integration/test_broken_part_during_merge/test.py
+++ b/tests/integration/test_broken_part_during_merge/test.py
@@ -54,7 +54,7 @@ def test_merge_and_part_corruption(started_cluster):
     with Pool(1) as p:
 
         def optimize_with_delay(x):
-            node1.query("OPTIMIZE TABLE replicated_mt FINAL", timeout=30)
+            node1.query("OPTIMIZE TABLE replicated_mt FINAL", timeout=120)
 
         # corrupt part after merge already assigned, but not started
         res_opt = p.apply_async(optimize_with_delay, (1,))
@@ -70,7 +70,7 @@ def test_merge_and_part_corruption(started_cluster):
         node1.query(
             "ALTER TABLE replicated_mt UPDATE value = 7 WHERE 1",
             settings={"mutations_sync": 2},
-            timeout=30,
+            timeout=120,
         )
         assert node1.query("SELECT sum(value) FROM replicated_mt") == "2100000\n"
 

From 1cca1d4ba3c9f020f21e684423ace943d863bb0f Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Wed, 24 Jul 2024 17:27:18 +0200
Subject: [PATCH 0725/1488] bump


From 8d4b919bf4d02c09399296e5213525f78bd68a21 Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitlibar@clickhouse.com>
Date: Wed, 24 Jul 2024 17:45:40 +0200
Subject: [PATCH 0726/1488] Fix detaching broken parts from backup.

---
 .../MergeTree/DataPartStorageOnDiskBase.cpp   | 37 ++++++++++++++++---
 .../MergeTree/DataPartStorageOnDiskBase.h     |  3 ++
 2 files changed, 34 insertions(+), 6 deletions(-)

diff --git a/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp b/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp
index 378a1944396..df151e8478f 100644
--- a/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp
+++ b/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp
@@ -73,12 +73,7 @@ std::optional<String> DataPartStorageOnDiskBase::getRelativePathForPrefix(Logger
 
     for (int try_no = 0; try_no < 10; ++try_no)
     {
-        if (prefix.empty())
-            res = part_dir + (try_no ? "_try" + DB::toString(try_no) : "");
-        else if (prefix.ends_with("_"))
-            res = prefix + part_dir + (try_no ? "_try" + DB::toString(try_no) : "");
-        else
-            res = prefix + "_" + part_dir + (try_no ? "_try" + DB::toString(try_no) : "");
+        res = getPartDirForPrefix(prefix, detached, try_no);
 
         if (!volume->getDisk()->exists(full_relative_path / res))
             return res;
@@ -101,6 +96,36 @@ std::optional<String> DataPartStorageOnDiskBase::getRelativePathForPrefix(Logger
     return res;
 }
 
+String DataPartStorageOnDiskBase::getPartDirForPrefix(const String & prefix, bool detached, int try_no) const
+{
+    /// This function joins `prefix` and the part name and an attempt number returning something like "<prefix>_<part_name>_<tryN>".
+    String res = prefix;
+    if (!prefix.empty() && !prefix.ends_with("_"))
+        res += "_";
+
+    /// During RESTORE temporary part directories are created with names like "tmp_restore_all_2_2_0-XXXXXXXX".
+    /// To detach such a directory we need to rename it replacing "tmp_restore_" with a specified prefix,
+    /// and a random suffix with an attempt number.
+    String part_name;
+    if (detached && part_dir.starts_with("tmp_restore_"))
+    {
+        part_name = part_dir.substr(strlen("tmp_restore_"));
+        size_t endpos = part_name.find('-');
+        if (endpos != String::npos)
+            part_name.erase(endpos, String::npos);
+    }
+
+    if (!part_name.empty())
+        res += part_name;
+    else
+        res += part_dir;
+
+    if (try_no)
+        res += "_try" + DB::toString(try_no);
+
+    return res;
+}
+
 bool DataPartStorageOnDiskBase::looksLikeBrokenDetachedPartHasTheSameContent(const String & detached_part_path,
                                                                          std::optional<String> & original_checksums_content,
                                                                          std::optional<Strings> & original_files_list) const
diff --git a/src/Storages/MergeTree/DataPartStorageOnDiskBase.h b/src/Storages/MergeTree/DataPartStorageOnDiskBase.h
index 81353d4e20b..1707efc2d4d 100644
--- a/src/Storages/MergeTree/DataPartStorageOnDiskBase.h
+++ b/src/Storages/MergeTree/DataPartStorageOnDiskBase.h
@@ -148,6 +148,9 @@ private:
     /// Actual file name may be the same as expected
     /// or be the name of the file with packed data.
     virtual NameSet getActualFileNamesOnDisk(const NameSet & file_names) const = 0;
+
+    /// Returns the destination path for the part directory while copying a detached part.
+    String getPartDirForPrefix(const String & prefix, bool detached, int try_no) const;
 };
 
 }

From c3620391b0befaf30eea0eab9001cc98fd5eeecc Mon Sep 17 00:00:00 2001
From: Sema Checherinda <Sema.Checherinda@clickhouse.com>
Date: Wed, 24 Jul 2024 17:48:07 +0200
Subject: [PATCH 0727/1488] fix style

---
 .../0_stateless/01171_mv_select_insert_isolation_long.sh     | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh b/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh
index f6850864be5..718017bca3d 100755
--- a/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh
+++ b/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh
@@ -41,12 +41,13 @@ function run_until_deadline_and_at_least_times()
     local min_iterations=$1; shift
     local function_to_run=$1; shift
 
-    local started_time=$(get_now)
+    local started_time
+    started_time=$(get_now)
     local i=0
 
     while true
     do
-        $function_to_run $i $@
+        $function_to_run $i "$@"
 
         [[ $(get_now) -lt $deadline ]] || break
 

From fa48ff0ca1561f20eb5abb285f400a8faf664d0b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?=
 <benjamin.antal@clickhouse.com>
Date: Wed, 24 Jul 2024 15:49:17 +0000
Subject: [PATCH 0728/1488] Separate tests

---
 tests/integration/test_storage_s3/test.py | 52 ++++++++++-------------
 1 file changed, 22 insertions(+), 30 deletions(-)

diff --git a/tests/integration/test_storage_s3/test.py b/tests/integration/test_storage_s3/test.py
index d13605170ec..dae98daf0cf 100644
--- a/tests/integration/test_storage_s3/test.py
+++ b/tests/integration/test_storage_s3/test.py
@@ -1148,31 +1148,27 @@ def test_url_reconnect_in_the_middle(started_cluster):
         assert result == "1000000\t3914219105369203805\n"
 
 
-def test_seekable_formats(started_cluster):
-    bucket = started_cluster.minio_bucket
+
+# At the time of writing the actual read bytes are respectively 148 and 169, so -10% to not be flaky
+@pytest.mark.parametrize("format_name,expected_bytes_read", [("Parquet", 133), ("ORC", 150)])
+def test_seekable_formats(started_cluster, format_name, expected_bytes_read):
+    expected_lines=1500000
     instance = started_cluster.instances["dummy"]  # type: ClickHouseInstance
 
-    table_function = f"s3(s3_parquet, structure='a Int32, b String', format='Parquet')"
+    table_function = f"s3(s3_{format_name.lower()}, structure='a Int32, b String', format='{format_name}')"
     exec_query_with_retry(
         instance,
-        f"insert into table function {table_function} SELECT number, randomString(100) FROM numbers(1000000) settings s3_truncate_on_insert=1",
-        timeout=100,
+        f"INSERT INTO TABLE FUNCTION {table_function} SELECT number, randomString(100) FROM numbers({expected_lines}) settings s3_truncate_on_insert=1",
+        timeout=300,
     )
 
     result = instance.query(f"SELECT count() FROM {table_function}")
-    assert int(result) == 1000000
-
-    table_function = f"s3(s3_orc, structure='a Int32, b String', format='ORC')"
-    exec_query_with_retry(
-        instance,
-        f"insert into table function {table_function} SELECT number, randomString(100) FROM numbers(1500000) settings s3_truncate_on_insert=1",
-        timeout=100,
-    )
+    assert int(result) == expected_lines
 
     result = instance.query(
         f"SELECT count() FROM {table_function} SETTINGS max_memory_usage='60M', max_download_threads=1"
     )
-    assert int(result) == 1500000
+    assert int(result) == expected_lines
 
     instance.query(f"SELECT * FROM {table_function} FORMAT Null")
 
@@ -1183,35 +1179,31 @@ def test_seekable_formats(started_cluster):
     result = result.strip()
     assert result.endswith("MiB")
     result = result[: result.index(".")]
-    assert int(result) > 150
+    assert int(result) > 140
 
 
-def test_seekable_formats_url(started_cluster):
+@pytest.mark.parametrize("format_name", ["Parquet", "ORC"])
+def test_seekable_formats_url(started_cluster, format_name):
     bucket = started_cluster.minio_bucket
+    expected_lines=1500000
     instance = started_cluster.instances["dummy"]  # type: ClickHouseInstance
 
-    table_function = f"s3(s3_parquet, structure='a Int32, b String', format='Parquet')"
+    format_name_lower=format_name.lower()
+    table_function = f"s3(s3_{format_name_lower}, structure='a Int32, b String', format='{format_name}')"
     exec_query_with_retry(
         instance,
-        f"insert into table function {table_function} SELECT number, randomString(100) FROM numbers(1500000) settings s3_truncate_on_insert=1",
-        timeout=100,
+        f"INSERT INTO TABLE FUNCTION {table_function} SELECT number, randomString(100) FROM numbers({expected_lines}) settings s3_truncate_on_insert=1",
+        timeout=300,
     )
 
     result = instance.query(f"SELECT count() FROM {table_function}")
-    assert int(result) == 1500000
+    assert int(result) == expected_lines
 
-    table_function = f"s3(s3_orc, structure='a Int32, b String', format='ORC')"
-    exec_query_with_retry(
-        instance,
-        f"insert into table function {table_function} SELECT number, randomString(100) FROM numbers(1500000) settings s3_truncate_on_insert=1",
-        timeout=100,
-    )
-
-    table_function = f"url('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test_parquet', 'Parquet', 'a Int32, b String')"
+    url_function = f"url('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test_{format_name_lower}', '{format_name}', 'a Int32, b String')"
     result = instance.query(
-        f"SELECT count() FROM {table_function} SETTINGS max_memory_usage='60M'"
+        f"SELECT count() FROM {url_function} SETTINGS max_memory_usage='60M'"
     )
-    assert int(result) == 1500000
+    assert int(result) == expected_lines
 
 
 def test_empty_file(started_cluster):

From 68c53764f271463bf7d05639c7d31975717e8d6f Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@clickhouse.com>
Date: Wed, 24 Jul 2024 17:50:42 +0200
Subject: [PATCH 0729/1488] Update tests/performance/decimal_aggregates.xml

---
 tests/performance/decimal_aggregates.xml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/performance/decimal_aggregates.xml b/tests/performance/decimal_aggregates.xml
index 9fc94f01a4d..b204dddea6e 100644
--- a/tests/performance/decimal_aggregates.xml
+++ b/tests/performance/decimal_aggregates.xml
@@ -8,6 +8,7 @@
     <fill_query>INSERT INTO t SELECT number AS x, x % 1000000 AS d32, x AS d64, x d128 FROM numbers_mt(100000000) SETTINGS max_threads = 2</fill_query>
     <fill_query>INSERT INTO t SELECT number AS x, x % 1000000 AS d32, x AS d64, x d128 FROM numbers_mt(100000000, 100000000) SETTINGS max_threads = 2</fill_query>
     <fill_query>INSERT INTO t SELECT number AS x, x % 1000000 AS d32, x AS d64, x d128 FROM numbers_mt(200000000, 100000000) SETTINGS max_threads = 2</fill_query>
+    <fill_query>INSERT INTO t SELECT number AS x, x % 1000000 AS d32, x AS d64, x d128 FROM numbers_mt(300000000, 100000000) SETTINGS max_threads = 2</fill_query>
 
     <drop_query>DROP TABLE IF EXISTS t</drop_query>
 

From 2e25808e586d46baf52029d626d02fdcc6ff53d2 Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitlibar@clickhouse.com>
Date: Wed, 24 Jul 2024 18:03:48 +0200
Subject: [PATCH 0730/1488] Fix test test_backup_restore_s3.

---
 tests/integration/test_backup_restore_s3/test.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/integration/test_backup_restore_s3/test.py b/tests/integration/test_backup_restore_s3/test.py
index 4840f5afc66..381268ce7fe 100644
--- a/tests/integration/test_backup_restore_s3/test.py
+++ b/tests/integration/test_backup_restore_s3/test.py
@@ -254,6 +254,7 @@ def check_system_tables(backup_query_id=None):
         ("disk_s3_cache", "ObjectStorage", "S3", "Local"),
         ("disk_s3_other_bucket", "ObjectStorage", "S3", "Local"),
         ("disk_s3_plain", "ObjectStorage", "S3", "Plain"),
+        ("disk_s3_plain_rewritable", "ObjectStorage", "S3", "PlainRewritable"),
         ("disk_s3_restricted_user", "ObjectStorage", "S3", "Local"),
     )
     assert len(expected_disks) == len(disks)

From ce153c9b54045959877e0b400971db352fbeb916 Mon Sep 17 00:00:00 2001
From: robot-clickhouse <robot-clickhouse@users.noreply.github.com>
Date: Wed, 24 Jul 2024 16:05:10 +0000
Subject: [PATCH 0731/1488] Automatic style fix

---
 tests/integration/test_storage_s3/test.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/tests/integration/test_storage_s3/test.py b/tests/integration/test_storage_s3/test.py
index dae98daf0cf..ab327afe90b 100644
--- a/tests/integration/test_storage_s3/test.py
+++ b/tests/integration/test_storage_s3/test.py
@@ -1148,11 +1148,12 @@ def test_url_reconnect_in_the_middle(started_cluster):
         assert result == "1000000\t3914219105369203805\n"
 
 
-
 # At the time of writing the actual read bytes are respectively 148 and 169, so -10% to not be flaky
-@pytest.mark.parametrize("format_name,expected_bytes_read", [("Parquet", 133), ("ORC", 150)])
+@pytest.mark.parametrize(
+    "format_name,expected_bytes_read", [("Parquet", 133), ("ORC", 150)]
+)
 def test_seekable_formats(started_cluster, format_name, expected_bytes_read):
-    expected_lines=1500000
+    expected_lines = 1500000
     instance = started_cluster.instances["dummy"]  # type: ClickHouseInstance
 
     table_function = f"s3(s3_{format_name.lower()}, structure='a Int32, b String', format='{format_name}')"
@@ -1185,10 +1186,10 @@ def test_seekable_formats(started_cluster, format_name, expected_bytes_read):
 @pytest.mark.parametrize("format_name", ["Parquet", "ORC"])
 def test_seekable_formats_url(started_cluster, format_name):
     bucket = started_cluster.minio_bucket
-    expected_lines=1500000
+    expected_lines = 1500000
     instance = started_cluster.instances["dummy"]  # type: ClickHouseInstance
 
-    format_name_lower=format_name.lower()
+    format_name_lower = format_name.lower()
     table_function = f"s3(s3_{format_name_lower}, structure='a Int32, b String', format='{format_name}')"
     exec_query_with_retry(
         instance,

From 689e31b47e1c85f1ae9721b3928de658eaf9a6ff Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 24 Jul 2024 18:31:40 +0200
Subject: [PATCH 0732/1488] More tests

---
 ...03210_inconsistent_formatting_of_data_types.reference | 6 ++++++
 .../03210_inconsistent_formatting_of_data_types.sh       | 9 +++++++++
 2 files changed, 15 insertions(+)

diff --git a/tests/queries/0_stateless/03210_inconsistent_formatting_of_data_types.reference b/tests/queries/0_stateless/03210_inconsistent_formatting_of_data_types.reference
index ccb445a0573..836b526905a 100644
--- a/tests/queries/0_stateless/03210_inconsistent_formatting_of_data_types.reference
+++ b/tests/queries/0_stateless/03210_inconsistent_formatting_of_data_types.reference
@@ -1 +1,7 @@
 ALTER TABLE columns_with_multiple_streams MODIFY COLUMN `field1` Nullable(tupleElement(x, 2), UInt8)
+ALTER TABLE t_update_empty_nested ADD COLUMN `nested.arr2` Array(tuple('- ON NULL -', toLowCardinality(11), 11, 11, toLowCardinality(11), 11), UInt64)
+ALTER TABLE t ADD COLUMN `x` Array(tuple(1), UInt8)
+ALTER TABLE enum_alter_issue MODIFY COLUMN `a` Enum8(equals('one', timeSlots(timeSlots(arrayEnumerateDense(tuple('0.2147483646', toLowCardinality(toUInt128)), NULL), 4, 12.34, materialize(73), 2)), 1))
+ALTER TABLE t_sparse_mutations_3 MODIFY COLUMN `s` Tuple(Nullable(tupleElement(s, 1), UInt64), Nullable(UInt64), Nullable(UInt64), Nullable(UInt64), Nullable(String))
+Syntax error
+Syntax error
diff --git a/tests/queries/0_stateless/03210_inconsistent_formatting_of_data_types.sh b/tests/queries/0_stateless/03210_inconsistent_formatting_of_data_types.sh
index 6cb2d083d71..86c7a5469ca 100755
--- a/tests/queries/0_stateless/03210_inconsistent_formatting_of_data_types.sh
+++ b/tests/queries/0_stateless/03210_inconsistent_formatting_of_data_types.sh
@@ -4,4 +4,13 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
+# Ensure that these (possibly incorrect) queries can at least be parsed back after formatting.
 $CLICKHOUSE_FORMAT --oneline --query "ALTER TABLE columns_with_multiple_streams MODIFY COLUMN field1 Nullable(tupleElement(x, 2), UInt8)" | $CLICKHOUSE_FORMAT --oneline
+$CLICKHOUSE_FORMAT --oneline --query "ALTER TABLE t_update_empty_nested ADD COLUMN \`nested.arr2\` Array(tuple('- ON NULL -', toLowCardinality(11), 11, 11, toLowCardinality(11), 11), UInt64)" | $CLICKHOUSE_FORMAT --oneline
+$CLICKHOUSE_FORMAT --oneline --query "ALTER TABLE t ADD COLUMN x Array((1), UInt8)" | $CLICKHOUSE_FORMAT --oneline
+$CLICKHOUSE_FORMAT --oneline --query "ALTER TABLE enum_alter_issue (MODIFY COLUMN a Enum8(equals('one', timeSlots(timeSlots(arrayEnumerateDense(tuple('0.2147483646', toLowCardinality(toUInt128(12))), NULL), 4, 12.34, materialize(73), 2)), 1)))" | $CLICKHOUSE_FORMAT --oneline
+$CLICKHOUSE_FORMAT --oneline --query "ALTER TABLE t_sparse_mutations_3 MODIFY COLUMN s Tuple(Nullable(tupleElement(s, 1), UInt64), Nullable(UInt64), Nullable(UInt64), Nullable(UInt64), Nullable(String))" | $CLICKHOUSE_FORMAT --oneline
+
+# These invalid queries don't parse and this is normal.
+$CLICKHOUSE_FORMAT --oneline --query "ALTER TABLE alter_compression_codec1 MODIFY COLUMN alter_column CODEC((2 + ignore(1, toUInt128(materialize(2)), 2 + toNullable(toNullable(3))), 3), NONE)" 2>&1 | grep -o -F 'Syntax error'
+$CLICKHOUSE_FORMAT --oneline --query "ALTER TABLE test_table ADD COLUMN \`array\` Array(('110', 3, toLowCardinality(3), 3, toNullable(3), toLowCardinality(toNullable(3)), 3), UInt8) DEFAULT [1, 2, 3]" 2>&1 | grep -o -F 'Syntax error'

From 5d88f6fc8c4b8fcce4e7a5da073f5d11a86cd3cb Mon Sep 17 00:00:00 2001
From: Sema Checherinda <Sema.Checherinda@clickhouse.com>
Date: Wed, 24 Jul 2024 19:32:20 +0200
Subject: [PATCH 0733/1488] fix MIN_ITERATIONS

---
 .../0_stateless/01171_mv_select_insert_isolation_long.sh        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh b/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh
index 718017bca3d..d79ab27d8b2 100755
--- a/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh
+++ b/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh
@@ -168,7 +168,7 @@ fi
 START_TIME=$(get_now)
 STOP_TIME=$((START_TIME + MAIN_TIME_PART))
 SECOND_STOP_TIME=$((STOP_TIME + SECOND_TIME_PART))
-MIN_ITERATIONS=50
+MIN_ITERATIONS=30
 
 run_until_deadline_and_at_least_times $STOP_TIME $MIN_ITERATIONS insert_commit_action 1   & PID_1=$!
 run_until_deadline_and_at_least_times $STOP_TIME $MIN_ITERATIONS insert_commit_action 2   & PID_2=$!

From 67567fcff481793cf7828808c094221ad2ec7389 Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Wed, 24 Jul 2024 19:51:22 +0200
Subject: [PATCH 0734/1488] Fix build

---
 programs/odbc-bridge/tests/CMakeLists.txt |  2 +-
 src/Common/examples/CMakeLists.txt        | 38 ++++++++---------
 src/Common/mysqlxx/tests/CMakeLists.txt   |  2 +-
 src/Compression/examples/CMakeLists.txt   |  2 +-
 src/Core/examples/CMakeLists.txt          |  4 +-
 src/IO/examples/CMakeLists.txt            | 50 +++++++++++------------
 src/Interpreters/examples/CMakeLists.txt  | 20 ++++-----
 src/Parsers/examples/CMakeLists.txt       |  2 +-
 utils/corrector_utf8/CMakeLists.txt       |  2 +-
 9 files changed, 61 insertions(+), 61 deletions(-)

diff --git a/programs/odbc-bridge/tests/CMakeLists.txt b/programs/odbc-bridge/tests/CMakeLists.txt
index edf364ea192..f1411dbb554 100644
--- a/programs/odbc-bridge/tests/CMakeLists.txt
+++ b/programs/odbc-bridge/tests/CMakeLists.txt
@@ -1,2 +1,2 @@
 clickhouse_add_executable (validate-odbc-connection-string validate-odbc-connection-string.cpp ../validateODBCConnectionString.cpp)
-target_link_libraries (validate-odbc-connection-string PRIVATE clickhouse_common_io)
+target_link_libraries (validate-odbc-connection-string PRIVATE clickhouse_common_io clickhouse_common_config)
diff --git a/src/Common/examples/CMakeLists.txt b/src/Common/examples/CMakeLists.txt
index c133e9f5617..69580d4ad0e 100644
--- a/src/Common/examples/CMakeLists.txt
+++ b/src/Common/examples/CMakeLists.txt
@@ -1,14 +1,14 @@
 clickhouse_add_executable (hashes_test hashes_test.cpp)
-target_link_libraries (hashes_test PRIVATE clickhouse_common_io ch_contrib::cityhash)
+target_link_libraries (hashes_test PRIVATE clickhouse_common_io clickhouse_common_config ch_contrib::cityhash)
 if (TARGET OpenSSL::Crypto)
     target_link_libraries (hashes_test PRIVATE OpenSSL::Crypto)
 endif()
 
 clickhouse_add_executable (sip_hash_perf sip_hash_perf.cpp)
-target_link_libraries (sip_hash_perf PRIVATE clickhouse_common_io)
+target_link_libraries (sip_hash_perf PRIVATE clickhouse_common_io clickhouse_common_config)
 
 clickhouse_add_executable (small_table small_table.cpp)
-target_link_libraries (small_table PRIVATE clickhouse_common_io)
+target_link_libraries (small_table PRIVATE clickhouse_common_io clickhouse_common_config)
 
 clickhouse_add_executable (parallel_aggregation parallel_aggregation.cpp)
 target_link_libraries (parallel_aggregation PRIVATE dbms clickhouse_functions)
@@ -17,13 +17,13 @@ clickhouse_add_executable (parallel_aggregation2 parallel_aggregation2.cpp)
 target_link_libraries (parallel_aggregation2 PRIVATE dbms clickhouse_functions)
 
 clickhouse_add_executable (int_hashes_perf int_hashes_perf.cpp)
-target_link_libraries (int_hashes_perf PRIVATE clickhouse_common_io)
+target_link_libraries (int_hashes_perf PRIVATE clickhouse_common_io clickhouse_common_config)
 
 clickhouse_add_executable (compact_array compact_array.cpp)
-target_link_libraries (compact_array PRIVATE clickhouse_common_io)
+target_link_libraries (compact_array PRIVATE clickhouse_common_io clickhouse_common_config)
 
 clickhouse_add_executable (radix_sort radix_sort.cpp)
-target_link_libraries (radix_sort PRIVATE clickhouse_common_io ch_contrib::pdqsort)
+target_link_libraries (radix_sort PRIVATE clickhouse_common_io clickhouse_common_config ch_contrib::pdqsort)
 
 clickhouse_add_executable (arena_with_free_lists arena_with_free_lists.cpp)
 target_link_libraries (arena_with_free_lists PRIVATE dbms)
@@ -33,46 +33,46 @@ target_link_libraries (lru_hash_map_perf PRIVATE dbms)
 
 if (OS_LINUX)
     clickhouse_add_executable (thread_creation_latency thread_creation_latency.cpp)
-    target_link_libraries (thread_creation_latency PRIVATE clickhouse_common_io)
+    target_link_libraries (thread_creation_latency PRIVATE clickhouse_common_io clickhouse_common_config)
 endif()
 
 clickhouse_add_executable (array_cache array_cache.cpp)
-target_link_libraries (array_cache PRIVATE clickhouse_common_io)
+target_link_libraries (array_cache PRIVATE clickhouse_common_io clickhouse_common_config)
 
 clickhouse_add_executable (space_saving space_saving.cpp)
-target_link_libraries (space_saving PRIVATE clickhouse_common_io)
+target_link_libraries (space_saving PRIVATE clickhouse_common_io clickhouse_common_config)
 
 clickhouse_add_executable (integer_hash_tables_benchmark integer_hash_tables_benchmark.cpp)
 target_link_libraries (integer_hash_tables_benchmark PRIVATE dbms ch_contrib::abseil_swiss_tables ch_contrib::sparsehash)
 
 clickhouse_add_executable (cow_columns cow_columns.cpp)
-target_link_libraries (cow_columns PRIVATE clickhouse_common_io)
+target_link_libraries (cow_columns PRIVATE clickhouse_common_io clickhouse_common_config)
 
 clickhouse_add_executable (cow_compositions cow_compositions.cpp)
-target_link_libraries (cow_compositions PRIVATE clickhouse_common_io)
+target_link_libraries (cow_compositions PRIVATE clickhouse_common_io clickhouse_common_config)
 
 clickhouse_add_executable (stopwatch stopwatch.cpp)
-target_link_libraries (stopwatch PRIVATE clickhouse_common_io)
+target_link_libraries (stopwatch PRIVATE clickhouse_common_io clickhouse_common_config)
 
 clickhouse_add_executable (symbol_index symbol_index.cpp)
-target_link_libraries (symbol_index PRIVATE clickhouse_common_io)
+target_link_libraries (symbol_index PRIVATE clickhouse_common_io clickhouse_common_config)
 
 clickhouse_add_executable (chaos_sanitizer chaos_sanitizer.cpp)
-target_link_libraries (chaos_sanitizer PRIVATE clickhouse_common_io)
+target_link_libraries (chaos_sanitizer PRIVATE clickhouse_common_io clickhouse_common_config)
 
 if (OS_LINUX)
     clickhouse_add_executable (memory_statistics_os_perf memory_statistics_os_perf.cpp)
-    target_link_libraries (memory_statistics_os_perf PRIVATE clickhouse_common_io)
+    target_link_libraries (memory_statistics_os_perf PRIVATE clickhouse_common_io clickhouse_common_config)
 endif()
 
 clickhouse_add_executable (procfs_metrics_provider_perf procfs_metrics_provider_perf.cpp)
-target_link_libraries (procfs_metrics_provider_perf PRIVATE clickhouse_common_io)
+target_link_libraries (procfs_metrics_provider_perf PRIVATE clickhouse_common_io clickhouse_common_config)
 
 clickhouse_add_executable (average average.cpp)
-target_link_libraries (average PRIVATE clickhouse_common_io)
+target_link_libraries (average PRIVATE clickhouse_common_io clickhouse_common_config)
 
 clickhouse_add_executable (shell_command_inout shell_command_inout.cpp)
-target_link_libraries (shell_command_inout PRIVATE clickhouse_common_io)
+target_link_libraries (shell_command_inout PRIVATE clickhouse_common_io clickhouse_common_config)
 
 clickhouse_add_executable (executable_udf executable_udf.cpp)
 target_link_libraries (executable_udf PRIVATE dbms)
@@ -91,4 +91,4 @@ if (ENABLE_SSL)
 endif()
 
 clickhouse_add_executable (check_pointer_valid check_pointer_valid.cpp)
-target_link_libraries (check_pointer_valid PRIVATE clickhouse_common_io)
+target_link_libraries (check_pointer_valid PRIVATE clickhouse_common_io clickhouse_common_config)
diff --git a/src/Common/mysqlxx/tests/CMakeLists.txt b/src/Common/mysqlxx/tests/CMakeLists.txt
index c560d0e3153..f62908ddcaf 100644
--- a/src/Common/mysqlxx/tests/CMakeLists.txt
+++ b/src/Common/mysqlxx/tests/CMakeLists.txt
@@ -1,2 +1,2 @@
 clickhouse_add_executable (mysqlxx_pool_test mysqlxx_pool_test.cpp)
-target_link_libraries (mysqlxx_pool_test PRIVATE mysqlxx)
+target_link_libraries (mysqlxx_pool_test PRIVATE mysqlxx clickhouse_common_config)
diff --git a/src/Compression/examples/CMakeLists.txt b/src/Compression/examples/CMakeLists.txt
index a7cc6bebf42..fee8cf89942 100644
--- a/src/Compression/examples/CMakeLists.txt
+++ b/src/Compression/examples/CMakeLists.txt
@@ -1,2 +1,2 @@
 clickhouse_add_executable (compressed_buffer compressed_buffer.cpp)
-target_link_libraries (compressed_buffer PRIVATE clickhouse_common_io clickhouse_compression)
+target_link_libraries (compressed_buffer PRIVATE clickhouse_common_io clickhouse_common_config clickhouse_compression)
diff --git a/src/Core/examples/CMakeLists.txt b/src/Core/examples/CMakeLists.txt
index f30ee25491f..97e5e8e67e6 100644
--- a/src/Core/examples/CMakeLists.txt
+++ b/src/Core/examples/CMakeLists.txt
@@ -1,8 +1,8 @@
 clickhouse_add_executable (string_pool string_pool.cpp)
-target_link_libraries (string_pool PRIVATE clickhouse_common_io ch_contrib::sparsehash)
+target_link_libraries (string_pool PRIVATE clickhouse_common_io clickhouse_common_config ch_contrib::sparsehash)
 
 clickhouse_add_executable (field field.cpp)
 target_link_libraries (field PRIVATE dbms)
 
 clickhouse_add_executable (string_ref_hash string_ref_hash.cpp)
-target_link_libraries (string_ref_hash PRIVATE clickhouse_common_io)
+target_link_libraries (string_ref_hash PRIVATE clickhouse_common_io clickhouse_common_config)
diff --git a/src/IO/examples/CMakeLists.txt b/src/IO/examples/CMakeLists.txt
index fc9d9c7dcd1..bfd171a3d00 100644
--- a/src/IO/examples/CMakeLists.txt
+++ b/src/IO/examples/CMakeLists.txt
@@ -1,77 +1,77 @@
 clickhouse_add_executable (read_buffer read_buffer.cpp)
-target_link_libraries (read_buffer PRIVATE clickhouse_common_io)
+target_link_libraries (read_buffer PRIVATE clickhouse_common_io clickhouse_common_config)
 
 clickhouse_add_executable (read_buffer_perf read_buffer_perf.cpp)
-target_link_libraries (read_buffer_perf PRIVATE clickhouse_common_io)
+target_link_libraries (read_buffer_perf PRIVATE clickhouse_common_io clickhouse_common_config)
 
 clickhouse_add_executable (read_float_perf read_float_perf.cpp)
-target_link_libraries (read_float_perf PRIVATE clickhouse_common_io)
+target_link_libraries (read_float_perf PRIVATE clickhouse_common_io clickhouse_common_config)
 
 clickhouse_add_executable (write_buffer write_buffer.cpp)
-target_link_libraries (write_buffer PRIVATE clickhouse_common_io)
+target_link_libraries (write_buffer PRIVATE clickhouse_common_io clickhouse_common_config)
 
 clickhouse_add_executable (write_buffer_perf write_buffer_perf.cpp)
-target_link_libraries (write_buffer_perf PRIVATE clickhouse_common_io)
+target_link_libraries (write_buffer_perf PRIVATE clickhouse_common_io clickhouse_common_config)
 
 clickhouse_add_executable (valid_utf8_perf valid_utf8_perf.cpp)
-target_link_libraries (valid_utf8_perf PRIVATE clickhouse_common_io)
+target_link_libraries (valid_utf8_perf PRIVATE clickhouse_common_io clickhouse_common_config)
 
 clickhouse_add_executable (valid_utf8 valid_utf8.cpp)
-target_link_libraries (valid_utf8 PRIVATE clickhouse_common_io)
+target_link_libraries (valid_utf8 PRIVATE clickhouse_common_io clickhouse_common_config)
 
 clickhouse_add_executable (var_uint var_uint.cpp)
-target_link_libraries (var_uint PRIVATE clickhouse_common_io)
+target_link_libraries (var_uint PRIVATE clickhouse_common_io clickhouse_common_config)
 
 clickhouse_add_executable (read_escaped_string read_escaped_string.cpp)
-target_link_libraries (read_escaped_string PRIVATE clickhouse_common_io)
+target_link_libraries (read_escaped_string PRIVATE clickhouse_common_io clickhouse_common_config)
 
 clickhouse_add_executable (parse_int_perf parse_int_perf.cpp)
-target_link_libraries (parse_int_perf PRIVATE clickhouse_common_io)
+target_link_libraries (parse_int_perf PRIVATE clickhouse_common_io clickhouse_common_config)
 
 clickhouse_add_executable (parse_int_perf2 parse_int_perf2.cpp)
-target_link_libraries (parse_int_perf2 PRIVATE clickhouse_common_io)
+target_link_libraries (parse_int_perf2 PRIVATE clickhouse_common_io clickhouse_common_config)
 
 clickhouse_add_executable (read_write_int read_write_int.cpp)
-target_link_libraries (read_write_int PRIVATE clickhouse_common_io)
+target_link_libraries (read_write_int PRIVATE clickhouse_common_io clickhouse_common_config)
 
 clickhouse_add_executable (o_direct_and_dirty_pages o_direct_and_dirty_pages.cpp)
-target_link_libraries (o_direct_and_dirty_pages PRIVATE clickhouse_common_io)
+target_link_libraries (o_direct_and_dirty_pages PRIVATE clickhouse_common_io clickhouse_common_config)
 
 clickhouse_add_executable (io_operators io_operators.cpp)
-target_link_libraries (io_operators PRIVATE clickhouse_common_io)
+target_link_libraries (io_operators PRIVATE clickhouse_common_io clickhouse_common_config)
 
 clickhouse_add_executable (write_int write_int.cpp)
-target_link_libraries (write_int PRIVATE clickhouse_common_io)
+target_link_libraries (write_int PRIVATE clickhouse_common_io clickhouse_common_config)
 
 clickhouse_add_executable (zlib_buffers zlib_buffers.cpp)
-target_link_libraries (zlib_buffers PRIVATE clickhouse_common_io)
+target_link_libraries (zlib_buffers PRIVATE clickhouse_common_io clickhouse_common_config)
 
 clickhouse_add_executable (lzma_buffers lzma_buffers.cpp)
-target_link_libraries (lzma_buffers PRIVATE clickhouse_common_io)
+target_link_libraries (lzma_buffers PRIVATE clickhouse_common_io clickhouse_common_config)
 
 clickhouse_add_executable (limit_read_buffer limit_read_buffer.cpp)
-target_link_libraries (limit_read_buffer PRIVATE clickhouse_common_io)
+target_link_libraries (limit_read_buffer PRIVATE clickhouse_common_io clickhouse_common_config)
 
 clickhouse_add_executable (limit_read_buffer2 limit_read_buffer2.cpp)
-target_link_libraries (limit_read_buffer2 PRIVATE clickhouse_common_io)
+target_link_libraries (limit_read_buffer2 PRIVATE clickhouse_common_io clickhouse_common_config)
 
 clickhouse_add_executable (parse_date_time_best_effort parse_date_time_best_effort.cpp)
-target_link_libraries (parse_date_time_best_effort PRIVATE clickhouse_common_io)
+target_link_libraries (parse_date_time_best_effort PRIVATE clickhouse_common_io clickhouse_common_config)
 
 clickhouse_add_executable (zlib_ng_bug zlib_ng_bug.cpp)
-target_link_libraries (zlib_ng_bug PRIVATE ch_contrib::zlib clickhouse_common_io)
+target_link_libraries (zlib_ng_bug PRIVATE ch_contrib::zlib clickhouse_common_io clickhouse_common_config)
 
 clickhouse_add_executable (dragonbox_test dragonbox_test.cpp)
-target_link_libraries (dragonbox_test PRIVATE ch_contrib::dragonbox_to_chars clickhouse_common_io)
+target_link_libraries (dragonbox_test PRIVATE ch_contrib::dragonbox_to_chars clickhouse_common_io clickhouse_common_config)
 
 clickhouse_add_executable (zstd_buffers zstd_buffers.cpp)
-target_link_libraries (zstd_buffers PRIVATE clickhouse_common_io)
+target_link_libraries (zstd_buffers PRIVATE clickhouse_common_io clickhouse_common_config)
 
 clickhouse_add_executable (snappy_read_buffer snappy_read_buffer.cpp)
-target_link_libraries (snappy_read_buffer PRIVATE clickhouse_common_io)
+target_link_libraries (snappy_read_buffer PRIVATE clickhouse_common_io clickhouse_common_config)
 
 clickhouse_add_executable (hadoop_snappy_read_buffer hadoop_snappy_read_buffer.cpp)
-target_link_libraries (hadoop_snappy_read_buffer PRIVATE clickhouse_common_io)
+target_link_libraries (hadoop_snappy_read_buffer PRIVATE clickhouse_common_io clickhouse_common_config)
 
 if (TARGET ch_contrib::hdfs)
     clickhouse_add_executable (read_buffer_from_hdfs read_buffer_from_hdfs.cpp)
diff --git a/src/Interpreters/examples/CMakeLists.txt b/src/Interpreters/examples/CMakeLists.txt
index 8bb7f9eeb98..4b1ec970b26 100644
--- a/src/Interpreters/examples/CMakeLists.txt
+++ b/src/Interpreters/examples/CMakeLists.txt
@@ -2,34 +2,34 @@ clickhouse_add_executable (hash_map hash_map.cpp)
 target_link_libraries (hash_map PRIVATE dbms clickhouse_functions ch_contrib::sparsehash)
 
 clickhouse_add_executable (hash_map_lookup hash_map_lookup.cpp)
-target_link_libraries (hash_map_lookup PRIVATE clickhouse_common_io clickhouse_compression)
+target_link_libraries (hash_map_lookup PRIVATE clickhouse_common_io clickhouse_common_config clickhouse_compression)
 
 clickhouse_add_executable (hash_map3 hash_map3.cpp)
-target_link_libraries (hash_map3 PRIVATE clickhouse_common_io clickhouse_compression ch_contrib::farmhash ch_contrib::metrohash)
+target_link_libraries (hash_map3 PRIVATE clickhouse_common_io clickhouse_common_config clickhouse_compression ch_contrib::farmhash ch_contrib::metrohash)
 
 clickhouse_add_executable (hash_map_string hash_map_string.cpp)
-target_link_libraries (hash_map_string PRIVATE clickhouse_common_io clickhouse_compression ch_contrib::sparsehash)
+target_link_libraries (hash_map_string PRIVATE clickhouse_common_io clickhouse_common_config clickhouse_compression ch_contrib::sparsehash)
 
 clickhouse_add_executable (hash_map_string_2 hash_map_string_2.cpp)
-target_link_libraries (hash_map_string_2 PRIVATE clickhouse_common_io clickhouse_compression)
+target_link_libraries (hash_map_string_2 PRIVATE clickhouse_common_io clickhouse_common_config clickhouse_compression)
 
 clickhouse_add_executable (hash_map_string_3 hash_map_string_3.cpp)
-target_link_libraries (hash_map_string_3 PRIVATE clickhouse_common_io clickhouse_compression ch_contrib::farmhash ch_contrib::metrohash)
+target_link_libraries (hash_map_string_3 PRIVATE clickhouse_common_io clickhouse_common_config clickhouse_compression ch_contrib::farmhash ch_contrib::metrohash)
 
 clickhouse_add_executable (hash_map_string_small hash_map_string_small.cpp)
-target_link_libraries (hash_map_string_small PRIVATE clickhouse_common_io clickhouse_compression ch_contrib::sparsehash)
+target_link_libraries (hash_map_string_small PRIVATE clickhouse_common_io clickhouse_common_config clickhouse_compression ch_contrib::sparsehash)
 
 clickhouse_add_executable (string_hash_map string_hash_map.cpp)
-target_link_libraries (string_hash_map PRIVATE clickhouse_common_io clickhouse_compression ch_contrib::sparsehash)
+target_link_libraries (string_hash_map PRIVATE clickhouse_common_io clickhouse_common_config clickhouse_compression ch_contrib::sparsehash)
 
 clickhouse_add_executable (string_hash_map_aggregation string_hash_map.cpp)
-target_link_libraries (string_hash_map_aggregation PRIVATE clickhouse_common_io clickhouse_compression)
+target_link_libraries (string_hash_map_aggregation PRIVATE clickhouse_common_io clickhouse_common_config clickhouse_compression)
 
 clickhouse_add_executable (string_hash_set string_hash_set.cpp)
-target_link_libraries (string_hash_set PRIVATE clickhouse_common_io clickhouse_compression)
+target_link_libraries (string_hash_set PRIVATE clickhouse_common_io clickhouse_common_config clickhouse_compression)
 
 clickhouse_add_executable (two_level_hash_map two_level_hash_map.cpp)
-target_link_libraries (two_level_hash_map PRIVATE clickhouse_common_io clickhouse_compression ch_contrib::sparsehash)
+target_link_libraries (two_level_hash_map PRIVATE clickhouse_common_io clickhouse_common_config clickhouse_compression ch_contrib::sparsehash)
 
 clickhouse_add_executable (jit_example jit_example.cpp)
 target_link_libraries (jit_example PRIVATE dbms)
diff --git a/src/Parsers/examples/CMakeLists.txt b/src/Parsers/examples/CMakeLists.txt
index 261f234081c..07f60601acd 100644
--- a/src/Parsers/examples/CMakeLists.txt
+++ b/src/Parsers/examples/CMakeLists.txt
@@ -1,7 +1,7 @@
 set(SRCS)
 
 clickhouse_add_executable(lexer lexer.cpp ${SRCS})
-target_link_libraries(lexer PRIVATE clickhouse_parsers)
+target_link_libraries(lexer PRIVATE clickhouse_parsers clickhouse_common_config)
 
 clickhouse_add_executable(select_parser select_parser.cpp ${SRCS} "../../Server/ServerType.cpp")
 target_link_libraries(select_parser PRIVATE dbms)
diff --git a/utils/corrector_utf8/CMakeLists.txt b/utils/corrector_utf8/CMakeLists.txt
index 4744dd5e0a5..17f047a8cff 100644
--- a/utils/corrector_utf8/CMakeLists.txt
+++ b/utils/corrector_utf8/CMakeLists.txt
@@ -1,2 +1,2 @@
 clickhouse_add_executable(corrector_utf8 corrector_utf8.cpp)
-target_link_libraries(corrector_utf8 PRIVATE clickhouse_common_io)
+target_link_libraries(corrector_utf8 PRIVATE clickhouse_common_io clickhouse_common_config)

From 9de43325e4771f54e29ccc43722c5f8c104f129a Mon Sep 17 00:00:00 2001
From: Max K <maxkaynov@gmail.com>
Date: Wed, 24 Jul 2024 19:51:34 +0200
Subject: [PATCH 0735/1488] CI: Jepsen Workflow with CI buddy

---
 .github/workflows/jepsen.yml | 69 +++++++++++++++++++++++++++++-------
 tests/ci/ci.py               | 40 +++++++++++++++------
 tests/ci/ci_config.py        | 26 ++++++++++++--
 tests/ci/ci_definitions.py   |  8 +++++
 tests/ci/ci_settings.py      |  7 ----
 tests/ci/jepsen_check.py     | 48 ++++++-------------------
 tests/ci/pr_info.py          |  4 +++
 tests/ci/test_ci_config.py   | 26 ++++++++++++++
 8 files changed, 159 insertions(+), 69 deletions(-)

diff --git a/.github/workflows/jepsen.yml b/.github/workflows/jepsen.yml
index db837ac1ec7..035ba2e5b98 100644
--- a/.github/workflows/jepsen.yml
+++ b/.github/workflows/jepsen.yml
@@ -9,19 +9,64 @@ on: # yamllint disable-line rule:truthy
     - cron: '0 */6 * * *'
   workflow_dispatch:
 jobs:
+  RunConfig:
+    runs-on: [self-hosted, style-checker-aarch64]
+    outputs:
+      data: ${{ steps.runconfig.outputs.CI_DATA }}
+    steps:
+      - name: DebugInfo
+        uses: hmarr/debug-action@f7318c783045ac39ed9bb497e22ce835fdafbfe6
+      - name: Check out repository code
+        uses: ClickHouse/checkout@v1
+        with:
+          clear-repository: true # to ensure correct digests
+          fetch-depth: 0 # to get version
+          filter: tree:0
+      - name: PrepareRunConfig
+        id: runconfig
+        run: |
+          echo "::group::configure CI run"
+          python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --configure --workflow "$GITHUB_WORKFLOW" --outfile ${{ runner.temp }}/ci_run_data.json
+          echo "::endgroup::"
+          
+          echo "::group::CI run configure results"
+          python3 -m json.tool ${{ runner.temp }}/ci_run_data.json
+          echo "::endgroup::"
+          {
+            echo 'CI_DATA<<EOF'
+            cat  ${{ runner.temp }}/ci_run_data.json
+            echo 'EOF'
+          } >> "$GITHUB_OUTPUT"
   KeeperJepsenRelease:
-    uses: ./.github/workflows/reusable_simple_job.yml
+    needs: [RunConfig]
+    uses: ./.github/workflows/reusable_test.yml
     with:
-      test_name: Jepsen keeper check
-      runner_type: style-checker
-      report_required: true
+      test_name: ClickHouse Keeper Jepsen
+      runner_type: style-checker-aarch64
+      data: ${{ needs.RunConfig.outputs.data }}
       run_command: |
         python3 jepsen_check.py keeper
-  # ServerJepsenRelease:
-  #   uses: ./.github/workflows/reusable_simple_job.yml
-  #   with:
-  #     test_name: Jepsen server check
-  #     runner_type: style-checker
-  #     run_command: |
-  #       cd "$REPO_COPY/tests/ci"
-  #       python3 jepsen_check.py server
+  ServerJepsenRelease:
+    if: false  # skip for server
+    needs: [RunConfig]
+    uses: ./.github/workflows/reusable_test.yml
+    with:
+      test_name: ClickHouse Server Jepsen
+      runner_type: style-checker-aarch64
+      data: ${{ needs.RunConfig.outputs.data }}
+      run_command: |
+        python3 jepsen_check.py server
+  CheckWorkflow:
+    if: ${{ !cancelled() }}
+    needs: [RunConfig, ServerJepsenRelease, KeeperJepsenRelease]
+    runs-on: [self-hosted, style-checker-aarch64]
+    steps:
+      - name: Check out repository code
+        uses: ClickHouse/checkout@v1
+      - name: Check Workflow results
+        run: |
+          export WORKFLOW_RESULT_FILE="/tmp/workflow_results.json"
+          cat >> "$WORKFLOW_RESULT_FILE" << 'EOF'
+          ${{ toJson(needs) }}
+          EOF
+          python3 ./tests/ci/ci_buddy.py --check-wf-status
diff --git a/tests/ci/ci.py b/tests/ci/ci.py
index 171819e2632..e30062c32ff 100644
--- a/tests/ci/ci.py
+++ b/tests/ci/ci.py
@@ -95,6 +95,12 @@ def parse_args(parser: argparse.ArgumentParser) -> argparse.Namespace:
         action="store_true",
         help="Action that configures ci run. Calculates digests, checks job to be executed, generates json output",
     )
+    parser.add_argument(
+        "--workflow",
+        default="",
+        type=str,
+        help="Workflow Name, to be provided with --configure for workflow-specific CI runs",
+    )
     parser.add_argument(
         "--update-gh-statuses",
         action="store_true",
@@ -287,7 +293,10 @@ def _pre_action(s3, job_name, batch, indata, pr_info):
     # for release/master branches reports must be from the same branch
     report_prefix = ""
     if pr_info.is_master or pr_info.is_release:
-        report_prefix = normalize_string(pr_info.head_ref)
+        # do not set report prefix for scheduled or dispatched wf (in case it started from feature branch while
+        #   testing), otherwise reports won't be found
+        if not (pr_info.is_scheduled or pr_info.is_dispatched):
+            report_prefix = normalize_string(pr_info.head_ref)
     print(
         f"Use report prefix [{report_prefix}], pr_num [{pr_info.number}], head_ref [{pr_info.head_ref}]"
     )
@@ -520,6 +529,7 @@ def _configure_jobs(
     pr_info: PRInfo,
     ci_settings: CiSettings,
     skip_jobs: bool,
+    workflow_name: str = "",
     dry_run: bool = False,
 ) -> CiCache:
     """
@@ -537,18 +547,27 @@ def _configure_jobs(
             is_docs_only=pr_info.has_changes_in_documentation_only(),
             is_master=pr_info.is_master,
             is_pr=pr_info.is_pr,
+            workflow_name=workflow_name,
         )
     else:
         job_configs = {}
 
-    # filter jobs in accordance with ci settings
-    job_configs = ci_settings.apply(
-        job_configs,
-        pr_info.is_release,
-        is_pr=pr_info.is_pr,
-        is_mq=pr_info.is_merge_queue,
-        labels=pr_info.labels,
-    )
+    if not workflow_name:
+        # filter jobs in accordance with ci settings
+        job_configs = ci_settings.apply(
+            job_configs,
+            pr_info.is_release,
+            is_pr=pr_info.is_pr,
+            is_mq=pr_info.is_merge_queue,
+            labels=pr_info.labels,
+        )
+
+    # add all job batches to job's to_do batches
+    for _job, job_config in job_configs.items():
+        batches = []
+        for batch in range(job_config.num_batches):
+            batches.append(batch)
+        job_config.batches = batches
 
     # check jobs in ci cache
     ci_cache = CiCache.calc_digests_and_create(
@@ -1102,6 +1121,7 @@ def main() -> int:
             pr_info,
             ci_settings,
             args.skip_jobs,
+            args.workflow,
         )
 
         ci_cache.print_status()
@@ -1111,7 +1131,7 @@ def main() -> int:
 
         if IS_CI and not pr_info.is_merge_queue:
 
-            if pr_info.is_release:
+            if pr_info.is_release and pr_info.is_push_event:
                 print("Release/master: CI Cache add pending records for all todo jobs")
                 ci_cache.push_pending_all(pr_info.is_release)
 
diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py
index a44b15f34c1..a9bdb639835 100644
--- a/tests/ci/ci_config.py
+++ b/tests/ci/ci_config.py
@@ -37,11 +37,22 @@ class CI:
     from ci_utils import GHActions as GHActions
     from ci_definitions import Labels as Labels
     from ci_definitions import TRUSTED_CONTRIBUTORS as TRUSTED_CONTRIBUTORS
+    from ci_definitions import WorkFlowNames as WorkFlowNames
     from ci_utils import CATEGORY_TO_LABEL as CATEGORY_TO_LABEL
 
     # Jobs that run for doc related updates
     _DOCS_CHECK_JOBS = [JobNames.DOCS_CHECK, JobNames.STYLE_CHECK]
 
+    WORKFLOW_CONFIGS = {
+        WorkFlowNames.JEPSEN: LabelConfig(
+            run_jobs=[
+                BuildNames.BINARY_RELEASE,
+                JobNames.JEPSEN_KEEPER,
+                JobNames.JEPSEN_SERVER,
+            ]
+        )
+    }  # type: Dict[str, LabelConfig]
+
     TAG_CONFIGS = {
         Tags.DO_NOT_TEST_LABEL: LabelConfig(run_jobs=[JobNames.STYLE_CHECK]),
         Tags.CI_SET_ARM: LabelConfig(
@@ -68,7 +79,7 @@ class CI:
                 JobNames.STATEFUL_TEST_ASAN,
             ]
         ),
-    }
+    }  # type: Dict[str, LabelConfig]
 
     JOB_CONFIGS: Dict[str, JobConfig] = {
         BuildNames.PACKAGE_RELEASE: CommonJobConfigs.BUILD.with_properties(
@@ -599,16 +610,25 @@ class CI:
 
     @classmethod
     def get_workflow_jobs_with_configs(
-        cls, is_mq: bool, is_docs_only: bool, is_master: bool, is_pr: bool
+        cls,
+        is_mq: bool,
+        is_docs_only: bool,
+        is_master: bool,
+        is_pr: bool,
+        workflow_name: str,
     ) -> Dict[str, JobConfig]:
         """
         get a list of all jobs for a workflow with configs
         """
-        jobs = []
         if is_mq:
             jobs = MQ_JOBS
         elif is_docs_only:
             jobs = cls._DOCS_CHECK_JOBS
+        elif workflow_name:
+            assert (
+                workflow_name in cls.WORKFLOW_CONFIGS
+            ), "Workflow name if provided must be configured in WORKFLOW_CONFIGS"
+            jobs = list(cls.WORKFLOW_CONFIGS[workflow_name].run_jobs)
         else:
             # add all jobs
             jobs = list(cls.JOB_CONFIGS)
diff --git a/tests/ci/ci_definitions.py b/tests/ci/ci_definitions.py
index a8d9793f1d3..149177ecba5 100644
--- a/tests/ci/ci_definitions.py
+++ b/tests/ci/ci_definitions.py
@@ -106,6 +106,14 @@ class Tags(metaclass=WithIter):
     libFuzzer = "libFuzzer"
 
 
+class WorkFlowNames(metaclass=WithIter):
+    """
+    CI WorkFlow Names for custom CI runs
+    """
+
+    JEPSEN = "JepsenWorkflow"
+
+
 class BuildNames(metaclass=WithIter):
     """
     Build' job names
diff --git a/tests/ci/ci_settings.py b/tests/ci/ci_settings.py
index c29c5777dba..d6e9765ceb7 100644
--- a/tests/ci/ci_settings.py
+++ b/tests/ci/ci_settings.py
@@ -234,11 +234,4 @@ class CiSettings:
         for job in add_parents:
             res[job] = job_configs[job]
 
-        for job, job_config in res.items():
-            batches = []
-            for batch in range(job_config.num_batches):
-                if not self.job_batches or batch in self.job_batches:
-                    batches.append(batch)
-            job_config.batches = batches
-
         return res
diff --git a/tests/ci/jepsen_check.py b/tests/ci/jepsen_check.py
index f91a3f080c0..772467d4245 100644
--- a/tests/ci/jepsen_check.py
+++ b/tests/ci/jepsen_check.py
@@ -9,16 +9,13 @@ from pathlib import Path
 from typing import Any, List
 
 import boto3  # type: ignore
-import requests
 
 from build_download_helper import (
-    download_build_with_progress,
     read_build_urls,
 )
 from compress_files import compress_fast
-from env_helper import REPO_COPY, REPORT_PATH, S3_BUILDS_BUCKET, S3_URL, TEMP_PATH
+from env_helper import REPO_COPY, REPORT_PATH, TEMP_PATH
 from get_robot_token import get_parameter_from_ssm
-from git_helper import git_runner
 from pr_info import PRInfo
 from report import FAILURE, SUCCESS, JobReport, TestResult, TestResults
 from ssh import SSHKey
@@ -32,11 +29,10 @@ KEEPER_DESIRED_INSTANCE_COUNT = 3
 SERVER_DESIRED_INSTANCE_COUNT = 4
 
 KEEPER_IMAGE_NAME = "clickhouse/keeper-jepsen-test"
-KEEPER_CHECK_NAME = "ClickHouse Keeper Jepsen"
+KEEPER_CHECK_NAME = CI.JobNames.JEPSEN_KEEPER
 
 SERVER_IMAGE_NAME = "clickhouse/server-jepsen-test"
-SERVER_CHECK_NAME = "ClickHouse Server Jepsen"
-
+SERVER_CHECK_NAME = CI.JobNames.JEPSEN_SERVER
 
 SUCCESSFUL_TESTS_ANCHOR = "# Successful tests"
 INTERMINATE_TESTS_ANCHOR = "# Indeterminate tests"
@@ -201,36 +197,14 @@ def main():
     # always use latest
     docker_image = KEEPER_IMAGE_NAME if args.program == "keeper" else SERVER_IMAGE_NAME
 
-    if pr_info.is_scheduled or pr_info.is_dispatched:
-        # get latest clickhouse by the static link for latest master buit - get its version and provide permanent url for this version to the jepsen
-        build_url = f"{S3_URL}/{S3_BUILDS_BUCKET}/master/amd64/clickhouse"
-        download_build_with_progress(build_url, Path(TEMP_PATH) / "clickhouse")
-        git_runner.run(f"chmod +x {TEMP_PATH}/clickhouse")
-        sha = git_runner.run(
-            f"{TEMP_PATH}/clickhouse local -q \"select value from system.build_options where name='GIT_HASH'\""
-        )
-        version_full = git_runner.run(
-            f'{TEMP_PATH}/clickhouse local -q "select version()"'
-        )
-        version = ".".join(version_full.split(".")[0:2])
-        assert len(sha) == 40, f"failed to fetch sha from the binary. result: {sha}"
-        assert (
-            version
-        ), f"failed to fetch version from the binary. result: {version_full}"
-        build_url = (
-            f"{S3_URL}/{S3_BUILDS_BUCKET}/{version}/{sha}/binary_release/clickhouse"
-        )
-        print(f"Clickhouse version: [{version_full}], sha: [{sha}], url: [{build_url}]")
-        head = requests.head(build_url, timeout=60)
-        assert head.status_code == 200, f"Clickhouse binary not found: {build_url}"
-    else:
-        build_name = CI.get_required_build_name(check_name)
-        urls = read_build_urls(build_name, REPORT_PATH)
-        build_url = None
-        for url in urls:
-            if url.endswith("clickhouse"):
-                build_url = url
-        assert build_url, "No build url found in the report"
+    # binary_release assumed to be always ready on the master as it's part of the merge queue workflow
+    build_name = CI.get_required_build_name(check_name)
+    urls = read_build_urls(build_name, REPORT_PATH)
+    build_url = None
+    for url in urls:
+        if url.endswith("clickhouse"):
+            build_url = url
+    assert build_url, "No build url found in the report"
 
     extra_args = ""
     if args.program == "server":
diff --git a/tests/ci/pr_info.py b/tests/ci/pr_info.py
index 59806a2a8fa..2c8ada7b983 100644
--- a/tests/ci/pr_info.py
+++ b/tests/ci/pr_info.py
@@ -337,6 +337,10 @@ class PRInfo:
             return True
         return False
 
+    @property
+    def is_push_event(self) -> bool:
+        return self.event_type == EventType.PUSH
+
     @property
     def is_scheduled(self) -> bool:
         return self.event_type == EventType.SCHEDULE
diff --git a/tests/ci/test_ci_config.py b/tests/ci/test_ci_config.py
index 44142050821..4a2bd606d0e 100644
--- a/tests/ci/test_ci_config.py
+++ b/tests/ci/test_ci_config.py
@@ -419,6 +419,32 @@ class TestCIConfig(unittest.TestCase):
         ]
         self.assertCountEqual(expected_jobs_to_do, actual_jobs_to_do)
 
+    def test_ci_py_for_specific_workflow(self):
+        """
+        checks ci.py job configuration
+        """
+        settings = CiSettings()
+        settings.no_ci_cache = True
+        pr_info = PRInfo(github_event=_TEST_EVENT_JSON)
+        # make it merge_queue
+        pr_info.event_type = EventType.SCHEDULE
+        assert pr_info.number == 0 and not pr_info.is_merge_queue and not pr_info.is_pr
+        ci_cache = CIPY._configure_jobs(
+            S3Helper(),
+            pr_info,
+            settings,
+            skip_jobs=False,
+            dry_run=True,
+            workflow_name=CI.WorkFlowNames.JEPSEN,
+        )
+        actual_jobs_to_do = list(ci_cache.jobs_to_do)
+        expected_jobs_to_do = [
+            CI.BuildNames.BINARY_RELEASE,
+            CI.JobNames.JEPSEN_KEEPER,
+            CI.JobNames.JEPSEN_SERVER,
+        ]
+        self.assertCountEqual(expected_jobs_to_do, actual_jobs_to_do)
+
     def test_ci_py_await(self):
         """
         checks ci.py job configuration

From 687c99e39a3ece073239517ffbcecf4612721995 Mon Sep 17 00:00:00 2001
From: jsc0218 <jsc0218@gmail.com>
Date: Wed, 24 Jul 2024 18:37:25 +0000
Subject: [PATCH 0736/1488] try to fix

---
 .../0_stateless/02680_mysql_ast_logical_err.sql        | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/tests/queries/0_stateless/02680_mysql_ast_logical_err.sql b/tests/queries/0_stateless/02680_mysql_ast_logical_err.sql
index bde91df83ca..78ce1b68b0d 100644
--- a/tests/queries/0_stateless/02680_mysql_ast_logical_err.sql
+++ b/tests/queries/0_stateless/02680_mysql_ast_logical_err.sql
@@ -1,4 +1,10 @@
 CREATE TABLE foo (key UInt32, a String, b Int64, c String) ENGINE = TinyLog;
 
-SELECT count() FROM mysql(mysql('127.0.0.1:9004', currentDatabase(), 'foo', 'default', ''), '127.0.0.1:9004', currentDatabase(), 'foo', '', ''); -- { serverError UNKNOWN_FUNCTION }
-SELECT count() FROM mysql(mysql('127.0.0.1:9004', currentDatabase(), 'foo', 'default', '', SETTINGS connection_pool_size = 1), '127.0.0.1:9004', currentDatabase(), 'foo', '', ''); -- { serverError UNKNOWN_FUNCTION, UNSUPPORTED_METHOD }
+SELECT count() FROM mysql(
+    mysql('127.0.0.1:9004', currentDatabase(), 'foo', 'default', ''),
+    '127.0.0.1:9004', currentDatabase(), 'foo', '', '',
+    SETTINGS connect_timeout = 100, connection_wait_timeout = 100, read_write_timeout = 300); -- { serverError UNKNOWN_FUNCTION }
+SELECT count() FROM mysql(
+    mysql('127.0.0.1:9004', currentDatabase(), 'foo', 'default', '', SETTINGS connection_pool_size = 1),
+    '127.0.0.1:9004', currentDatabase(), 'foo', '', '',
+    SETTINGS connect_timeout = 100, connection_wait_timeout = 100, read_write_timeout = 300); -- { serverError UNKNOWN_FUNCTION, UNSUPPORTED_METHOD }

From e4b50c18c2c1918905bf44a8e1183f0cddd5a811 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Wed, 24 Jul 2024 22:26:46 +0200
Subject: [PATCH 0737/1488] getauxval: Avoid crash under sanitizer re-exec due
 to high ASLR entropy

---
 base/glibc-compatibility/musl/getauxval.c | 38 +++++++++++++++++++++++
 1 file changed, 38 insertions(+)

diff --git a/base/glibc-compatibility/musl/getauxval.c b/base/glibc-compatibility/musl/getauxval.c
index ea5cff9fc11..86f9a546ee4 100644
--- a/base/glibc-compatibility/musl/getauxval.c
+++ b/base/glibc-compatibility/musl/getauxval.c
@@ -75,6 +75,44 @@ unsigned long NO_SANITIZE_THREAD __getauxval_procfs(unsigned long type)
 }
 static unsigned long NO_SANITIZE_THREAD __auxv_init_procfs(unsigned long type)
 {
+#if defined(__x86_64__) && defined(__has_feature)
+#    if __has_feature(memory_sanitizer) || __has_feature(thread_sanitizer)
+    /// Sanitizers are not compatible with high ASLR entropy, which is the default on modern Linux distributions, and
+    /// to workaround this limitation, TSAN and MSAN (couldn't see other sanitizers doing the same), re-exec the binary
+    /// without ASLR (see https://github.com/llvm/llvm-project/commit/0784b1eefa36d4acbb0dacd2d18796e26313b6c5)
+
+    /// The problem we face is that, in order to re-exec, the sanitizer wants to use the original pathname in the call
+    /// and to get its value it uses getauxval (https://github.com/llvm/llvm-project/blob/20eff684203287828d6722fc860b9d3621429542/compiler-rt/lib/sanitizer_common/sanitizer_linux_libcdep.cpp#L985-L988).
+    /// Since we provide getauxval ourselves (to minimize the version dependency on runtime glibc), we are the ones
+    // being called and we fail horribly:
+    ///
+    ///    ==301455==ERROR: MemorySanitizer: SEGV on unknown address 0x2ffc6d721550 (pc 0x5622c1cc0073 bp 0x000000000003 sp 0x7ffc6d721530 T301455)
+    ///    ==301455==The signal is caused by a WRITE memory access.
+    /// #0 0x5622c1cc0073 in __auxv_init_procfs ./ClickHouse/base/glibc-compatibility/musl/getauxval.c:129:5
+    /// #1 0x5622c1cbffe9 in getauxval ./ClickHouse/base/glibc-compatibility/musl/getauxval.c:240:12
+    /// #2 0x5622c0d7bfb4 in __sanitizer::ReExec() crtstuff.c
+    /// #3 0x5622c0df7bfc in __msan::InitShadowWithReExec(bool) crtstuff.c
+    /// #4 0x5622c0d95356 in __msan_init (./ClickHouse/build_msan/contrib/google-protobuf-cmake/protoc+0x256356) (BuildId: 6411d3c88b898ba3f7d49760555977d3e61f0741)
+    /// #5 0x5622c0dfe878 in msan.module_ctor main.cc
+    /// #6 0x5622c1cc156c in __libc_csu_init (./ClickHouse/build_msan/contrib/google-protobuf-cmake/protoc+0x118256c) (BuildId: 6411d3c88b898ba3f7d49760555977d3e61f0741)
+    /// #7 0x73dc05dd7ea3 in __libc_start_main /usr/src/debug/glibc/glibc/csu/../csu/libc-start.c:343:6
+    /// #8 0x5622c0d6b7cd in _start (./ClickHouse/build_msan/contrib/google-protobuf-cmake/protoc+0x22c7cd) (BuildId: 6411d3c88b898ba3f7d49760555977d3e61f0741)
+
+    /// The source of the issue above is that, at this point in time during __msan_init, we can't really do much as
+    /// most global variables aren't initialized or available yet, so we we can't initiate the auxiliar vector.
+    /// Normal glibc / musl getauxval doesn't have this problem since they initiate their auxval vector at the very
+    /// start of __libc_start_main (just keeping track of argv+argc+1), but we don't have such option (otherwise
+    // this complexity of reading "/proc/self/auxv" or using __environ would not be necessary).
+
+    /// To avoid this crashes on the re-exec call (see above how it would fail when creating `aux`, and it we used
+    /// __auxv_init_environ then it would SIGSEV on READing `__environ`) we capture this call for `AT_EXECFN` and
+    /// unconditionally return "/proc/self/exe" without any preparation. Theoretically this should be fine in
+    /// our case, as we don't load any libraries. That's the theory at least.
+    if (type == AT_EXECFN)
+        return (unsigned long)"/proc/self/exe";
+#    endif
+#endif
+
     // For debugging:
     // - od -t dL /proc/self/auxv
     // - LD_SHOW_AUX= ls

From fda11dc62d81b717b9ab06c8adc8554c827764bf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Wed, 24 Jul 2024 22:51:26 +0200
Subject: [PATCH 0738/1488] Typo

---
 base/glibc-compatibility/musl/getauxval.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/base/glibc-compatibility/musl/getauxval.c b/base/glibc-compatibility/musl/getauxval.c
index 86f9a546ee4..b5bd2f114c2 100644
--- a/base/glibc-compatibility/musl/getauxval.c
+++ b/base/glibc-compatibility/musl/getauxval.c
@@ -99,7 +99,7 @@ static unsigned long NO_SANITIZE_THREAD __auxv_init_procfs(unsigned long type)
     /// #8 0x5622c0d6b7cd in _start (./ClickHouse/build_msan/contrib/google-protobuf-cmake/protoc+0x22c7cd) (BuildId: 6411d3c88b898ba3f7d49760555977d3e61f0741)
 
     /// The source of the issue above is that, at this point in time during __msan_init, we can't really do much as
-    /// most global variables aren't initialized or available yet, so we we can't initiate the auxiliar vector.
+    /// most global variables aren't initialized or available yet, so we we can't initiate the auxiliary vector.
     /// Normal glibc / musl getauxval doesn't have this problem since they initiate their auxval vector at the very
     /// start of __libc_start_main (just keeping track of argv+argc+1), but we don't have such option (otherwise
     // this complexity of reading "/proc/self/auxv" or using __environ would not be necessary).

From c847d2f63fdacf1ce5d636a9af8812d543547cfe Mon Sep 17 00:00:00 2001
From: jsc0218 <jsc0218@gmail.com>
Date: Wed, 24 Jul 2024 20:52:35 +0000
Subject: [PATCH 0739/1488] fix

---
 src/Interpreters/InterpreterCreateQuery.cpp | 25 ++++++++++++---------
 1 file changed, 14 insertions(+), 11 deletions(-)

diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp
index 05df26b0d31..a5f374ba71c 100644
--- a/src/Interpreters/InterpreterCreateQuery.cpp
+++ b/src/Interpreters/InterpreterCreateQuery.cpp
@@ -1282,24 +1282,27 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create)
     TableProperties properties = getTablePropertiesAndNormalizeCreateQuery(create, mode);
 
     /// Projection is only supported in (Replictaed)MergeTree.
-    if (std::string_view engine_name(create.storage->engine->name);
-        !properties.projections.empty() && engine_name != "MergeTree" && engine_name != "ReplicatedMergeTree")
+    if (create.storage && create.storage->engine)
     {
-        bool projection_support = false;
-        if (auto * setting = create.storage->settings; setting != nullptr)
+        if (std::string_view engine_name(create.storage->engine->name);
+            !properties.projections.empty() && engine_name != "MergeTree" && engine_name != "ReplicatedMergeTree")
         {
-            for (const auto & change : setting->changes)
+            bool projection_support = false;
+            if (auto * setting = create.storage->settings; setting != nullptr)
             {
-                if (change.name == "deduplicate_merge_projection_mode" && change.value != Field("throw"))
+                for (const auto & change : setting->changes)
                 {
-                    projection_support = true;
-                    break;
+                    if (change.name == "deduplicate_merge_projection_mode" && change.value != Field("throw"))
+                    {
+                        projection_support = true;
+                        break;
+                    }
                 }
             }
+            if (!projection_support)
+                throw Exception(ErrorCodes::NOT_IMPLEMENTED,
+                    "Projection is only supported in (Replictaed)MergeTree. Consider drop or rebuild option of deduplicate_merge_projection_mode.");
         }
-        if (!projection_support)
-            throw Exception(ErrorCodes::NOT_IMPLEMENTED,
-                "Projection is only supported in (Replictaed)MergeTree. Consider drop or rebuild option of deduplicate_merge_projection_mode.");
     }
 
     /// Check type compatible for materialized dest table and select columns

From a6a9b8c27204f96e373c9625145dc1609cb7ca8f Mon Sep 17 00:00:00 2001
From: Nikolay Degterinsky <43110995+evillique@users.noreply.github.com>
Date: Thu, 25 Jul 2024 00:49:28 +0200
Subject: [PATCH 0740/1488] Fix flaky 02447_drop_replica test

---
 tests/queries/0_stateless/02447_drop_database_replica.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tests/queries/0_stateless/02447_drop_database_replica.sh b/tests/queries/0_stateless/02447_drop_database_replica.sh
index 93a5fcee8e2..c6bf298f944 100755
--- a/tests/queries/0_stateless/02447_drop_database_replica.sh
+++ b/tests/queries/0_stateless/02447_drop_database_replica.sh
@@ -1,5 +1,9 @@
 #!/usr/bin/env bash
 
+# Tags: no-parallel
+# no-parallel: This test is not parallel because when we execute system-wide SYSTEM DROP REPLICA,
+#  other tests might shut down the storage in parallel and the test will fail.
+
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh

From 8df648b3c8bbc22cee9657145b825e9d991e3c8e Mon Sep 17 00:00:00 2001
From: Alexander Tokmakov <tavplubix@clickhouse.com>
Date: Thu, 25 Jul 2024 00:56:41 +0200
Subject: [PATCH 0741/1488] fix a test, add retries for sql tests

---
 src/Client/ClientBase.cpp                     | 12 ++++-
 src/Client/TestHint.cpp                       | 52 ++++++++++++++++++-
 src/Client/TestHint.h                         |  6 +++
 .../02446_parent_zero_copy_locks.sql          | 14 +++--
 4 files changed, 76 insertions(+), 8 deletions(-)

diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp
index 13dce05cabc..149e1899ac3 100644
--- a/src/Client/ClientBase.cpp
+++ b/src/Client/ClientBase.cpp
@@ -2230,6 +2230,8 @@ bool ClientBase::executeMultiQuery(const String & all_queries_text)
     ASTPtr parsed_query;
     std::unique_ptr<Exception> current_exception;
 
+    size_t retries_count = 0;
+
     while (true)
     {
         auto stage = analyzeMultiQueryText(this_query_begin, this_query_end, all_queries_end,
@@ -2310,7 +2312,12 @@ bool ClientBase::executeMultiQuery(const String & all_queries_text)
                 // Check whether the error (or its absence) matches the test hints
                 // (or their absence).
                 bool error_matches_hint = true;
-                if (have_error)
+                bool need_retry = test_hint.needRetry(server_exception, &retries_count);
+                if (need_retry)
+                {
+                    std::this_thread::sleep_for(std::chrono::seconds(1));
+                }
+                else if (have_error)
                 {
                     if (test_hint.hasServerErrors())
                     {
@@ -2404,7 +2411,8 @@ bool ClientBase::executeMultiQuery(const String & all_queries_text)
                 if (have_error && !ignore_error)
                     return is_interactive;
 
-                this_query_begin = this_query_end;
+                if (!need_retry)
+                    this_query_begin = this_query_end;
                 break;
             }
         }
diff --git a/src/Client/TestHint.cpp b/src/Client/TestHint.cpp
index b64882577ee..74c65009a73 100644
--- a/src/Client/TestHint.cpp
+++ b/src/Client/TestHint.cpp
@@ -10,6 +10,7 @@
 namespace DB::ErrorCodes
 {
     extern const int CANNOT_PARSE_TEXT;
+    extern const int OK;
 }
 
 namespace DB
@@ -62,9 +63,28 @@ bool TestHint::hasExpectedServerError(int error)
     return std::find(server_errors.begin(), server_errors.end(), error) != server_errors.end();
 }
 
+bool TestHint::needRetry(const std::unique_ptr<Exception> & server_exception, size_t * retries_counter)
+{
+    chassert(retries_counter);
+    if (max_retries <= *retries_counter)
+        return false;
+
+    ++*retries_counter;
+
+    int error = ErrorCodes::OK;
+    if (server_exception)
+        error = server_exception->code();
+
+
+    if (retry_until)
+        return !hasExpectedServerError(error);  /// retry until we get the expected error
+    else
+        return hasExpectedServerError(error);   /// retry while we have the expected error
+}
+
 void TestHint::parse(Lexer & comment_lexer, bool is_leading_hint)
 {
-    std::unordered_set<std::string_view> commands{"echo", "echoOn", "echoOff"};
+    std::unordered_set<std::string_view> commands{"echo", "echoOn", "echoOff", "retry"};
 
     std::unordered_set<std::string_view> command_errors{
         "serverError",
@@ -73,6 +93,9 @@ void TestHint::parse(Lexer & comment_lexer, bool is_leading_hint)
 
     for (Token token = comment_lexer.nextToken(); !token.isEnd(); token = comment_lexer.nextToken())
     {
+        if (token.type == TokenType::Whitespace)
+            continue;
+
         String item = String(token.begin, token.end);
         if (token.type == TokenType::BareWord && commands.contains(item))
         {
@@ -82,6 +105,30 @@ void TestHint::parse(Lexer & comment_lexer, bool is_leading_hint)
                 echo.emplace(true);
             if (item == "echoOff")
                 echo.emplace(false);
+
+            if (item == "retry")
+            {
+                token = comment_lexer.nextToken();
+                while (token.type == TokenType::Whitespace)
+                    token = comment_lexer.nextToken();
+
+                if (token.type != TokenType::Number)
+                    throw DB::Exception(DB::ErrorCodes::CANNOT_PARSE_TEXT, "Could not parse the number of retries: {}",
+                                        std::string_view(token.begin, token.end));
+
+                max_retries = std::stoul(std::string(token.begin, token.end));
+
+                token = comment_lexer.nextToken();
+                while (token.type == TokenType::Whitespace)
+                    token = comment_lexer.nextToken();
+
+                if (token.type != TokenType::BareWord ||
+                    (std::string_view(token.begin, token.end) != "until" &&
+                    std::string_view(token.begin, token.end) != "while"))
+                    throw DB::Exception(DB::ErrorCodes::CANNOT_PARSE_TEXT, "Expected 'until' or 'while' after the number of retries, got: {}",
+                                        std::string_view(token.begin, token.end));
+                retry_until = std::string_view(token.begin, token.end) == "until";
+            }
         }
         else if (!is_leading_hint && token.type == TokenType::BareWord && command_errors.contains(item))
         {
@@ -133,6 +180,9 @@ void TestHint::parse(Lexer & comment_lexer, bool is_leading_hint)
             break;
         }
     }
+
+    if (max_retries && server_errors.size() != 1)
+        throw DB::Exception(DB::ErrorCodes::CANNOT_PARSE_TEXT, "Expected one serverError after the 'retry N while|until' command");
 }
 
 }
diff --git a/src/Client/TestHint.h b/src/Client/TestHint.h
index b76c4245df4..bbe7873c08b 100644
--- a/src/Client/TestHint.h
+++ b/src/Client/TestHint.h
@@ -6,6 +6,7 @@
 #include <fmt/format.h>
 
 #include <Core/Types.h>
+#include <Common/Exception.h>
 
 
 namespace DB
@@ -65,12 +66,17 @@ public:
     bool hasExpectedClientError(int error);
     bool hasExpectedServerError(int error);
 
+    bool needRetry(const std::unique_ptr<Exception> & server_exception, size_t * retries_counter);
+
 private:
     const String & query;
     ErrorVector server_errors{};
     ErrorVector client_errors{};
     std::optional<bool> echo;
 
+    size_t max_retries = 0;
+    bool retry_until = false;
+
     void parse(Lexer & comment_lexer, bool is_leading_hint);
 
     bool allErrorsExpected(int actual_server_error, int actual_client_error) const
diff --git a/tests/queries/0_stateless/02446_parent_zero_copy_locks.sql b/tests/queries/0_stateless/02446_parent_zero_copy_locks.sql
index 86eda526c72..1cae8ae0237 100644
--- a/tests/queries/0_stateless/02446_parent_zero_copy_locks.sql
+++ b/tests/queries/0_stateless/02446_parent_zero_copy_locks.sql
@@ -7,7 +7,7 @@ create table rmt2 (n int, m int, k int) engine=ReplicatedMergeTree('/test/02446/
     settings storage_policy='s3_cache', allow_remote_fs_zero_copy_replication=1, old_parts_lifetime=0, cleanup_delay_period=0, max_cleanup_delay_period=1, cleanup_delay_period_random_add=1, min_bytes_for_wide_part=0;
 
 -- FIXME zero-copy locks may remain in ZooKeeper forever if we failed to insert a part.
--- Probably that's why we have to replace repsistent lock with ephemeral sometimes.
+-- Probably that's why we have to replace persistent lock with ephemeral sometimes.
 -- See also "Replacing persistent lock with ephemeral for path {}. It can happen only in case of local part loss"
 -- in StorageReplicatedMergeTree::createZeroCopyLockNode
 set insert_keeper_fault_injection_probability=0;
@@ -23,6 +23,10 @@ select sleepEachRow(0.5) as test_does_not_rely_on_this;
 insert into rmt1 values(5, 5, 5);
 alter table rmt2 update m = m * 10 where 1 settings mutations_sync=2;
 
+-- wait for parts to be merged
+select throwIf(name = 'all_0_5_1_6') from system.parts where database=currentDatabase() and table like 'rmt%' and active
+format Null; -- { retry 30 until serverError FUNCTION_THROW_IF_VALUE_IS_NON_ZERO }
+
 system sync replica rmt2;
 set optimize_throw_if_noop=1;
 optimize table rmt2 final;
@@ -32,10 +36,10 @@ select 1, * from rmt1 order by n;
 system sync replica rmt1;
 select 2, * from rmt2 order by n;
 
--- a funny way to wait for outdated parts to be removed
-select sleep(1), sleepEachRow(0.1) from url('http://localhost:8123/?param_tries={1..10}&query=' || encodeURLComponent(
-            'select *, _state from system.parts where database=''' || currentDatabase() || ''' and table like ''rmt%'' and active=0'
-    ), 'LineAsString', 's String') settings max_threads=1 format Null;
+-- wait for outdated parts to be removed
+select throwIf(count() = 0) from (
+select *, _state from system.parts where database=currentDatabase() and table like 'rmt%' and active=0
+) format Null; -- { retry 30 until serverError FUNCTION_THROW_IF_VALUE_IS_NON_ZERO }
 
 select *, _state from system.parts where database=currentDatabase() and table like 'rmt%' and active=0;
 

From 23c3fa73266cb9f32c7f30a2c83437815de89291 Mon Sep 17 00:00:00 2001
From: jsc0218 <jsc0218@gmail.com>
Date: Thu, 25 Jul 2024 01:03:59 +0000
Subject: [PATCH 0742/1488] fix

---
 .../01710_aggregate_projection_with_normalized_states.sql      | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/01710_aggregate_projection_with_normalized_states.sql b/tests/queries/0_stateless/01710_aggregate_projection_with_normalized_states.sql
index e023c0991b3..5375823aa8e 100644
--- a/tests/queries/0_stateless/01710_aggregate_projection_with_normalized_states.sql
+++ b/tests/queries/0_stateless/01710_aggregate_projection_with_normalized_states.sql
@@ -9,7 +9,8 @@ CREATE TABLE r (
      s Int64,
      PROJECTION p
          (SELECT a, quantilesTimingMerge(0.5, 0.95, 0.99)(q), sum(s) GROUP BY a)
-) Engine=SummingMergeTree order by (x, a);
+) Engine=SummingMergeTree order by (x, a)
+SETTINGS deduplicate_merge_projection_mode = 'drop';  -- should set it to rebuild once projection is supported with SummingMergeTree
 
 insert into r
 select number%100 x,

From fa437b34ec16a0f7fe52f0e4261f964fef2ed606 Mon Sep 17 00:00:00 2001
From: Anton Popov <anton@clickhouse.com>
Date: Thu, 25 Jul 2024 03:16:20 +0200
Subject: [PATCH 0743/1488] Increase timeout for curl in tests

---
 tests/queries/shell_config.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/shell_config.sh b/tests/queries/shell_config.sh
index ef2d89f0218..f7017958635 100644
--- a/tests/queries/shell_config.sh
+++ b/tests/queries/shell_config.sh
@@ -132,7 +132,7 @@ export CLICKHOUSE_URL_INTERSERVER=${CLICKHOUSE_URL_INTERSERVER:="${CLICKHOUSE_PO
 export CLICKHOUSE_CURL_COMMAND=${CLICKHOUSE_CURL_COMMAND:="curl"}
 # The queries in CI are prone to sudden delays, and we often don't check for curl
 # errors, so it makes sense to set a relatively generous timeout.
-export CLICKHOUSE_CURL_TIMEOUT=${CLICKHOUSE_CURL_TIMEOUT:="60"}
+export CLICKHOUSE_CURL_TIMEOUT=${CLICKHOUSE_CURL_TIMEOUT:="120"}
 export CLICKHOUSE_CURL=${CLICKHOUSE_CURL:="${CLICKHOUSE_CURL_COMMAND} -q -s --max-time ${CLICKHOUSE_CURL_TIMEOUT}"}
 export CLICKHOUSE_TMP=${CLICKHOUSE_TMP:="."}
 mkdir -p ${CLICKHOUSE_TMP}

From 82959ce5b3e3709f12cb5cf8d50f8ca81858c7ed Mon Sep 17 00:00:00 2001
From: zoomxi <419486879@qq.com>
Date: Thu, 25 Jul 2024 09:55:23 +0800
Subject: [PATCH 0744/1488] format test.py

---
 tests/integration/test_parallel_replicas_no_replicas/test.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/integration/test_parallel_replicas_no_replicas/test.py b/tests/integration/test_parallel_replicas_no_replicas/test.py
index 04e3a54e581..62d4b005d94 100644
--- a/tests/integration/test_parallel_replicas_no_replicas/test.py
+++ b/tests/integration/test_parallel_replicas_no_replicas/test.py
@@ -34,7 +34,9 @@ def create_tables(cluster, table_name):
 
 @pytest.mark.parametrize("skip_unavailable_shards", [1, 0])
 @pytest.mark.parametrize("max_parallel_replicas", [2, 3, 100])
-def test_skip_all_replicas(start_cluster, skip_unavailable_shards, max_parallel_replicas):
+def test_skip_all_replicas(
+    start_cluster, skip_unavailable_shards, max_parallel_replicas
+):
     cluster_name = "test_1_shard_3_unavaliable_replicas"
     table_name = "tt"
     create_tables(cluster_name, table_name)

From 36e59a1b7083f646b5509e6ef379d1c24f23ad0b Mon Sep 17 00:00:00 2001
From: Michael Kolupaev <michael.kolupaev@clickhouse.com>
Date: Thu, 25 Jul 2024 01:58:27 +0000
Subject: [PATCH 0745/1488] Final commit hash

---
 contrib/libunwind | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contrib/libunwind b/contrib/libunwind
index 9b1f47ad8a6..a89d904befe 160000
--- a/contrib/libunwind
+++ b/contrib/libunwind
@@ -1 +1 @@
-Subproject commit 9b1f47ad8a6fcecbeaaead93bd87756ccf658071
+Subproject commit a89d904befea07814628c6ce0b44083c4e149c62

From f239dd67eec9cded889d8aee9e4895da5745541f Mon Sep 17 00:00:00 2001
From: Konstantin Bogdanov <konstantin@clickhouse.com>
Date: Wed, 24 Jul 2024 03:30:04 +0200
Subject: [PATCH 0746/1488] Fix

---
 src/Databases/DatabaseReplicated.cpp | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp
index 4c079ae5300..706d91d6c9e 100644
--- a/src/Databases/DatabaseReplicated.cpp
+++ b/src/Databases/DatabaseReplicated.cpp
@@ -641,7 +641,10 @@ LoadTaskPtr DatabaseReplicated::startupDatabaseAsync(AsyncLoader & async_loader,
             if (is_probably_dropped)
                 return;
 
-            ddl_worker = std::make_unique<DatabaseReplicatedDDLWorker>(this, getContext());
+            {
+                std::lock_guard lock{mutex};
+                ddl_worker = std::make_unique<DatabaseReplicatedDDLWorker>(this, getContext());
+            }
             ddl_worker->startup();
             ddl_worker_initialized = true;
         });

From aa9908f6da6ee7bf932a649c2884615856b408bb Mon Sep 17 00:00:00 2001
From: Konstantin Bogdanov <konstantin@clickhouse.com>
Date: Wed, 24 Jul 2024 04:33:47 +0200
Subject: [PATCH 0747/1488] Fix harder

---
 src/Databases/DatabaseReplicated.cpp | 2 +-
 src/Databases/DatabaseReplicated.h   | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp
index 706d91d6c9e..dd524e305a1 100644
--- a/src/Databases/DatabaseReplicated.cpp
+++ b/src/Databases/DatabaseReplicated.cpp
@@ -642,7 +642,7 @@ LoadTaskPtr DatabaseReplicated::startupDatabaseAsync(AsyncLoader & async_loader,
                 return;
 
             {
-                std::lock_guard lock{mutex};
+                std::lock_guard lock{ddl_worker_mutex};
                 ddl_worker = std::make_unique<DatabaseReplicatedDDLWorker>(this, getContext());
             }
             ddl_worker->startup();
diff --git a/src/Databases/DatabaseReplicated.h b/src/Databases/DatabaseReplicated.h
index 8c3fa7c87f6..9edba0eeb2b 100644
--- a/src/Databases/DatabaseReplicated.h
+++ b/src/Databases/DatabaseReplicated.h
@@ -149,6 +149,7 @@ private:
     std::atomic_bool is_recovering = false;
     std::atomic_bool ddl_worker_initialized = false;
     std::unique_ptr<DatabaseReplicatedDDLWorker> ddl_worker;
+    std::mutex ddl_worker_mutex;
     UInt32 max_log_ptr_at_creation = 0;
 
     /// Usually operation with metadata are single-threaded because of the way replication works,

From cea3bbc45b530742ea5d0374093278494ec5a9b8 Mon Sep 17 00:00:00 2001
From: Konstantin Bogdanov <konstantin@clickhouse.com>
Date: Wed, 24 Jul 2024 07:06:08 +0200
Subject: [PATCH 0748/1488] More checks

---
 src/Databases/DatabaseReplicated.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp
index dd524e305a1..97a9b65f6b4 100644
--- a/src/Databases/DatabaseReplicated.cpp
+++ b/src/Databases/DatabaseReplicated.cpp
@@ -559,7 +559,7 @@ void DatabaseReplicated::createEmptyLogEntry(const ZooKeeperPtr & current_zookee
 
 bool DatabaseReplicated::waitForReplicaToProcessAllEntries(UInt64 timeout_ms)
 {
-    if (!ddl_worker || is_probably_dropped)
+    if (!ddl_worker_initialized || !ddl_worker || is_probably_dropped)
         return false;
     return ddl_worker->waitForReplicaToProcessAllEntries(timeout_ms);
 }
@@ -686,7 +686,7 @@ bool DatabaseReplicated::checkDigestValid(const ContextPtr & local_context, bool
     LOG_TEST(log, "Current in-memory metadata digest: {}", tables_metadata_digest);
 
     /// Database is probably being dropped
-    if (!local_context->getZooKeeperMetadataTransaction() && (!ddl_worker || !ddl_worker->isCurrentlyActive()))
+    if (!local_context->getZooKeeperMetadataTransaction() && (!ddl_worker_initialized || !ddl_worker || !ddl_worker->isCurrentlyActive()))
         return true;
 
     UInt64 local_digest = 0;
@@ -1414,7 +1414,7 @@ void DatabaseReplicated::renameDatabase(ContextPtr query_context, const String &
 
 void DatabaseReplicated::stopReplication()
 {
-    if (ddl_worker)
+    if (ddl_worker_initialized && ddl_worker)
         ddl_worker->shutdown();
 }
 

From 80b9b13771bb08a1d8d713aabd55ac0471292b1a Mon Sep 17 00:00:00 2001
From: Konstantin Bogdanov <konstantin@clickhouse.com>
Date: Wed, 24 Jul 2024 07:15:35 +0200
Subject: [PATCH 0749/1488] Less checks, more locks

---
 src/Databases/DatabaseReplicated.cpp | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp
index 97a9b65f6b4..c90f4ea4719 100644
--- a/src/Databases/DatabaseReplicated.cpp
+++ b/src/Databases/DatabaseReplicated.cpp
@@ -559,8 +559,11 @@ void DatabaseReplicated::createEmptyLogEntry(const ZooKeeperPtr & current_zookee
 
 bool DatabaseReplicated::waitForReplicaToProcessAllEntries(UInt64 timeout_ms)
 {
-    if (!ddl_worker_initialized || !ddl_worker || is_probably_dropped)
-        return false;
+    {
+        std::lock_guard lock{ddl_worker_mutex};
+        if (!ddl_worker || is_probably_dropped)
+            return false;
+    }
     return ddl_worker->waitForReplicaToProcessAllEntries(timeout_ms);
 }
 
@@ -686,8 +689,11 @@ bool DatabaseReplicated::checkDigestValid(const ContextPtr & local_context, bool
     LOG_TEST(log, "Current in-memory metadata digest: {}", tables_metadata_digest);
 
     /// Database is probably being dropped
-    if (!local_context->getZooKeeperMetadataTransaction() && (!ddl_worker_initialized || !ddl_worker || !ddl_worker->isCurrentlyActive()))
-        return true;
+    {
+        std::lock_guard lock{ddl_worker_mutex};
+        if (!local_context->getZooKeeperMetadataTransaction() && (!ddl_worker || !ddl_worker->isCurrentlyActive()))
+            return true;
+    }
 
     UInt64 local_digest = 0;
     {
@@ -1414,7 +1420,8 @@ void DatabaseReplicated::renameDatabase(ContextPtr query_context, const String &
 
 void DatabaseReplicated::stopReplication()
 {
-    if (ddl_worker_initialized && ddl_worker)
+    std::lock_guard lock{ddl_worker_mutex};
+    if (ddl_worker)
         ddl_worker->shutdown();
 }
 

From e21d23d04daf2577db63e26cee22bc686e10833e Mon Sep 17 00:00:00 2001
From: Konstantin Bogdanov <konstantin@clickhouse.com>
Date: Wed, 24 Jul 2024 07:55:36 +0200
Subject: [PATCH 0750/1488] Less locks

---
 src/Databases/DatabaseReplicated.cpp | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp
index c90f4ea4719..c737ece02ec 100644
--- a/src/Databases/DatabaseReplicated.cpp
+++ b/src/Databases/DatabaseReplicated.cpp
@@ -689,11 +689,8 @@ bool DatabaseReplicated::checkDigestValid(const ContextPtr & local_context, bool
     LOG_TEST(log, "Current in-memory metadata digest: {}", tables_metadata_digest);
 
     /// Database is probably being dropped
-    {
-        std::lock_guard lock{ddl_worker_mutex};
-        if (!local_context->getZooKeeperMetadataTransaction() && (!ddl_worker || !ddl_worker->isCurrentlyActive()))
-            return true;
-    }
+    if (!local_context->getZooKeeperMetadataTransaction() && (!ddl_worker || !ddl_worker->isCurrentlyActive()))
+        return true;
 
     UInt64 local_digest = 0;
     {

From 7612060d232a24dbd721597c8e33cd1f556cddd6 Mon Sep 17 00:00:00 2001
From: Yakov Olkhovskiy <yakov@clickhouse.com>
Date: Thu, 25 Jul 2024 06:40:51 +0000
Subject: [PATCH 0751/1488] allow only equal types in lagInFrame and
 leadInFrame

---
 src/Processors/Transforms/WindowTransform.cpp | 15 ++------------
 .../03210_lag_lead_inframe_types.reference    | 20 +++++++++++++++++++
 .../03210_lag_lead_inframe_types.sql          |  4 ++++
 3 files changed, 26 insertions(+), 13 deletions(-)
 create mode 100644 tests/queries/0_stateless/03210_lag_lead_inframe_types.reference
 create mode 100644 tests/queries/0_stateless/03210_lag_lead_inframe_types.sql

diff --git a/src/Processors/Transforms/WindowTransform.cpp b/src/Processors/Transforms/WindowTransform.cpp
index 86421adf4fb..06ae2bfb25e 100644
--- a/src/Processors/Transforms/WindowTransform.cpp
+++ b/src/Processors/Transforms/WindowTransform.cpp
@@ -2385,22 +2385,11 @@ struct WindowFunctionLagLeadInFrame final : public WindowFunction
             return;
         }
 
-        const auto supertype = getLeastSupertype(DataTypes{argument_types[0], argument_types[2]});
-        if (!supertype)
-        {
+        if (!argument_types[0]->equals(*argument_types[2]))
             throw Exception(ErrorCodes::BAD_ARGUMENTS,
-                "There is no supertype for the argument type '{}' and the default value type '{}'",
+                "Argument type '{}' and the default value type '{}' are different",
                 argument_types[0]->getName(),
                 argument_types[2]->getName());
-        }
-        if (!argument_types[0]->equals(*supertype))
-        {
-            throw Exception(ErrorCodes::BAD_ARGUMENTS,
-                "The supertype '{}' for the argument type '{}' and the default value type '{}' is not the same as the argument type",
-                supertype->getName(),
-                argument_types[0]->getName(),
-                argument_types[2]->getName());
-        }
 
         if (argument_types.size() > 3)
         {
diff --git a/tests/queries/0_stateless/03210_lag_lead_inframe_types.reference b/tests/queries/0_stateless/03210_lag_lead_inframe_types.reference
new file mode 100644
index 00000000000..cc3b9a096b9
--- /dev/null
+++ b/tests/queries/0_stateless/03210_lag_lead_inframe_types.reference
@@ -0,0 +1,20 @@
+0
+1
+2
+2
+2
+2
+2
+2
+2
+2
+0
+1
+2
+3
+4
+5
+6
+7
+8
+9
diff --git a/tests/queries/0_stateless/03210_lag_lead_inframe_types.sql b/tests/queries/0_stateless/03210_lag_lead_inframe_types.sql
new file mode 100644
index 00000000000..5466cfe0fad
--- /dev/null
+++ b/tests/queries/0_stateless/03210_lag_lead_inframe_types.sql
@@ -0,0 +1,4 @@
+SELECT lagInFrame(2::UInt128, 2, number) OVER w FROM numbers(10) WINDOW w AS (ORDER BY number); -- { serverError BAD_ARGUMENTS }
+SELECT leadInFrame(2::UInt128, 2, number) OVER w FROM numbers(10) WINDOW w AS (ORDER BY number); -- { serverError BAD_ARGUMENTS }
+SELECT lagInFrame(2::UInt64, 2, number) OVER w FROM numbers(10) WINDOW w AS (ORDER BY number);
+SELECT leadInFrame(2::UInt64, 2, number) OVER w FROM numbers(10) WINDOW w AS (ORDER BY number);

From c6a643f981505d0293358c912723f1aece480c7c Mon Sep 17 00:00:00 2001
From: Nikita Fomichev <nikita.fomichev@clickhouse.com>
Date: Thu, 25 Jul 2024 10:31:38 +0200
Subject: [PATCH 0752/1488] Update
 tests/queries/0_stateless/02992_all_columns_should_have_comment.sql

---
 .../0_stateless/02992_all_columns_should_have_comment.sql       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/02992_all_columns_should_have_comment.sql b/tests/queries/0_stateless/02992_all_columns_should_have_comment.sql
index 0d34b033354..ad056384bfd 100644
--- a/tests/queries/0_stateless/02992_all_columns_should_have_comment.sql
+++ b/tests/queries/0_stateless/02992_all_columns_should_have_comment.sql
@@ -3,6 +3,6 @@ SELECT 'Column ' || name || ' from table ' || concat(database, '.', table) || '
 FROM system.columns
 WHERE (database = 'system') AND
       (comment = '') AND
-      (table NOT ILIKE '%\_log\_%') AND
+      (table NOT ILIKE '%log%') AND
       (table NOT IN ('numbers', 'numbers_mt', 'one', 'generate_series', 'generateSeries', 'coverage_log', 'filesystem_read_prefetches_log')) AND
       (default_kind != 'ALIAS');

From b5171df7798323761b366f01d401c0559ff4c736 Mon Sep 17 00:00:00 2001
From: Nikita Fomichev <nikita.fomichev@clickhouse.com>
Date: Thu, 25 Jul 2024 10:32:52 +0200
Subject: [PATCH 0753/1488] Update test 03198_table_function_directory_path.sql

---
 .../0_stateless/03198_table_function_directory_path.reference   | 1 +
 .../queries/0_stateless/03198_table_function_directory_path.sql | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/tests/queries/0_stateless/03198_table_function_directory_path.reference b/tests/queries/0_stateless/03198_table_function_directory_path.reference
index 19920de3d3c..74cd8c6d31f 100644
--- a/tests/queries/0_stateless/03198_table_function_directory_path.reference
+++ b/tests/queries/0_stateless/03198_table_function_directory_path.reference
@@ -1,3 +1,4 @@
 2
 2
 1
+1
diff --git a/tests/queries/0_stateless/03198_table_function_directory_path.sql b/tests/queries/0_stateless/03198_table_function_directory_path.sql
index 9e2791847af..90f687ed6a3 100644
--- a/tests/queries/0_stateless/03198_table_function_directory_path.sql
+++ b/tests/queries/0_stateless/03198_table_function_directory_path.sql
@@ -1,5 +1,6 @@
 -- Tags: no-parallel
 
+INSERT INTO FUNCTION file('data_03198_table_function_directory_path.csv', 'csv') SELECT '1.csv' SETTINGS engine_file_truncate_on_insert=1;
 INSERT INTO FUNCTION file('data_03198_table_function_directory_path/1.csv', 'csv') SELECT '1.csv' SETTINGS engine_file_truncate_on_insert=1;
 INSERT INTO FUNCTION file('data_03198_table_function_directory_path/2.csv', 'csv') SELECT '2.csv' SETTINGS engine_file_truncate_on_insert=1;
 INSERT INTO FUNCTION file('data_03198_table_function_directory_path/dir/3.csv', 'csv') SELECT '3.csv' SETTINGS engine_file_truncate_on_insert=1;
@@ -11,3 +12,4 @@ SELECT COUNT(*) FROM file('data_03198_table_function_directory_path/');
 SELECT COUNT(*) FROM file('data_03198_table_function_directory_path/dir');
 SELECT COUNT(*) FROM file('data_03198_table_function_directory_path/*/dir', 'csv'); -- { serverError 74, 636 }
 SELECT COUNT(*) FROM file('data_03198_table_function_directory_pat'); -- { serverError 400 }
+SELECT COUNT(*) FROM file('data_03198_table_function_directory_path.csv');

From 60f529f667069c15fa49296ac1f59a33d94d3f31 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Thu, 25 Jul 2024 11:06:00 +0200
Subject: [PATCH 0754/1488] The most precise way of tracking flushing time in
 01246_buffer_flush

Right now there are couple of issues with the test:

- it does not takes into account INSERT time
- it does not takes into account SELECT time, which can be significant
  from time to time, for instance here [1] it takes 3.3 seconds (and due
  to tsan build it is not possible to find out why)

      2024.07.23 20:52:18.238844 [ 13045 ] {d903650b-ab87-44f3-b7c3-4145e02f1301} <Debug> executeQuery: (from [::1]:39430) (comment: 01246_buffer_flush.sh) select count() from data_01256; (stage: Complete)
      2024.07.23 20:52:21.588183 [ 13045 ] {d903650b-ab87-44f3-b7c3-4145e02f1301} <Debug> TCPHandler: Processed in 3.354887498 sec.

  [1]: https://s3.amazonaws.com/clickhouse-test-reports/66934/919005c4f70b044ecd9cc1bbce5dc5e276e11929/stateless_tests__tsan__s3_storage__[4_4].html

Anyway all of this can be fixed by using QueryStart-insert into data
table time.

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 .../queries/0_stateless/01246_buffer_flush.sh | 49 ++++++++++++-------
 1 file changed, 31 insertions(+), 18 deletions(-)

diff --git a/tests/queries/0_stateless/01246_buffer_flush.sh b/tests/queries/0_stateless/01246_buffer_flush.sh
index 1ca953c80d9..27c3f01f216 100755
--- a/tests/queries/0_stateless/01246_buffer_flush.sh
+++ b/tests/queries/0_stateless/01246_buffer_flush.sh
@@ -5,59 +5,72 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CUR_DIR"/../shell_config.sh
 
-function elapsed_sec()
+set -e
+
+function wait_until()
 {
     local expr=$1 && shift
-    local start end
-    start=$(date +%s.%N)
     while ! eval "$expr"; do
         sleep 0.5
     done
-    end=$(date +%s.%N)
-    $CLICKHOUSE_LOCAL -q "select floor($end-$start)"
+}
+function get_buffer_delay()
+{
+    local buffer_insert_id=$1 && shift
+    $CLICKHOUSE_CLIENT -nm -q "
+    SYSTEM FLUSH LOGS;
+    WITH
+        (SELECT event_time_microseconds FROM system.query_log WHERE current_database = currentDatabase() AND type = 'QueryStart' AND query_id = '$buffer_insert_id') AS begin_,
+        (SELECT max(event_time) FROM data_01256) AS end_
+    SELECT dateDiff('seconds', begin_, end_)::UInt64;
+    "
 }
 
 $CLICKHOUSE_CLIENT -nm -q "
     drop table if exists data_01256;
     drop table if exists buffer_01256;
 
-    create table data_01256 as system.numbers Engine=Memory();
+    create table data_01256 (key UInt64, event_time DateTime(6) MATERIALIZED now64(6)) Engine=Memory();
 "
 
 echo "min"
-$CLICKHOUSE_CLIENT -nm -q "
-    create table buffer_01256 as system.numbers Engine=Buffer(currentDatabase(), data_01256, 1,
+$CLICKHOUSE_CLIENT -q "
+    create table buffer_01256 (key UInt64) Engine=Buffer(currentDatabase(), data_01256, 1,
         2, 100, /* time */
         4, 100, /* rows */
         1, 1e6  /* bytes */
-    );
-    insert into buffer_01256 select * from system.numbers limit 5;
-    select count() from data_01256;
+    )
 "
-sec=$(elapsed_sec '[[ $($CLICKHOUSE_CLIENT -q "select count() from data_01256") -eq 5 ]]')
+min_query_id=$(random_str 10)
+$CLICKHOUSE_CLIENT --query_id="$min_query_id" -q "insert into buffer_01256 select * from system.numbers limit 5"
+$CLICKHOUSE_CLIENT -q "select count() from data_01256"
+wait_until '[[ $($CLICKHOUSE_CLIENT -q "select count() from data_01256") -eq 5 ]]'
+sec=$(get_buffer_delay "$min_query_id")
 [[ $sec -ge 2 ]] || echo "Buffer flushed too early, min_time=2, flushed after $sec sec"
 [[ $sec -lt 100 ]] || echo "Buffer flushed too late, max_time=100, flushed after $sec sec"
 $CLICKHOUSE_CLIENT -q "select count() from data_01256"
 $CLICKHOUSE_CLIENT -q "drop table buffer_01256"
 
 echo "max"
-$CLICKHOUSE_CLIENT -nm -q "
-    create table buffer_01256 as system.numbers Engine=Buffer(currentDatabase(), data_01256, 1,
+$CLICKHOUSE_CLIENT -q "
+    create table buffer_01256 (key UInt64) Engine=Buffer(currentDatabase(), data_01256, 1,
         100, 2,   /* time */
         0,   100, /* rows */
         0,   1e6  /* bytes */
     );
-    insert into buffer_01256 select * from system.numbers limit 5;
-    select count() from data_01256;
 "
-sec=$(elapsed_sec '[[ $($CLICKHOUSE_CLIENT -q "select count() from data_01256") -eq 10 ]]')
+max_query_id=$(random_str 10)
+$CLICKHOUSE_CLIENT --query_id="$max_query_id" -q "insert into buffer_01256 select * from system.numbers limit 5"
+$CLICKHOUSE_CLIENT -q "select count() from data_01256"
+wait_until '[[ $($CLICKHOUSE_CLIENT -q "select count() from data_01256") -eq 10 ]]'
+sec=$(get_buffer_delay "$max_query_id")
 [[ $sec -ge 2 ]] || echo "Buffer flushed too early, max_time=2, flushed after $sec sec"
 $CLICKHOUSE_CLIENT -q "select count() from data_01256"
 $CLICKHOUSE_CLIENT -q "drop table buffer_01256"
 
 echo "direct"
 $CLICKHOUSE_CLIENT -nm -q "
-    create table buffer_01256 as system.numbers Engine=Buffer(currentDatabase(), data_01256, 1,
+    create table buffer_01256 (key UInt64) Engine=Buffer(currentDatabase(), data_01256, 1,
         100, 100, /* time */
         0,   9,   /* rows */
         0,   1e6  /* bytes */

From c2f85c6fd062dde095ee34178450dc94c245e691 Mon Sep 17 00:00:00 2001
From: taiyang-li <654010905@qq.com>
Date: Thu, 25 Jul 2024 17:43:02 +0800
Subject: [PATCH 0755/1488] support map type as first argument type

---
 .../functions/tuple-map-functions.md          |   2 +-
 src/Functions/map.cpp                         | 131 +++++++++++-------
 .../0_stateless/01651_map_functions.reference |   3 +
 .../0_stateless/01651_map_functions.sql       |   9 +-
 4 files changed, 89 insertions(+), 56 deletions(-)

diff --git a/docs/en/sql-reference/functions/tuple-map-functions.md b/docs/en/sql-reference/functions/tuple-map-functions.md
index 24b356eca87..ae23387f6e5 100644
--- a/docs/en/sql-reference/functions/tuple-map-functions.md
+++ b/docs/en/sql-reference/functions/tuple-map-functions.md
@@ -62,7 +62,7 @@ Alias: `MAP_FROM_ARRAYS(keys, values)`
 
 **Arguments**
 
-- `keys` —  Array of keys to create the map from. [Array(T)](../data-types/array.md) where `T` can be any type supported by [Map](../data-types/map.md) as key type.
+- `keys` —  Array or map of keys to create the map from. [Array(T)](../data-types/array.md) where `T` can be any type supported by [Map](../data-types/map.md) as key type.
 - `values`  - Array or map of values to create the map from. [Array](../data-types/array.md) or [Map](../data-types/map.md).
 
 **Returned value**
diff --git a/src/Functions/map.cpp b/src/Functions/map.cpp
index 66cd10a3f0b..5319390fb70 100644
--- a/src/Functions/map.cpp
+++ b/src/Functions/map.cpp
@@ -1,14 +1,17 @@
-#include <Functions/IFunction.h>
+#include <Columns/ColumnLowCardinality.h>
+#include <Columns/ColumnMap.h>
+#include <Columns/ColumnNullable.h>
+#include <Columns/ColumnsCommon.h>
+#include <DataTypes/DataTypeArray.h>
+#include <DataTypes/DataTypeMap.h>
+#include <DataTypes/DataTypeTuple.h>
+#include <DataTypes/DataTypesNumber.h>
+#include <DataTypes/getLeastSupertype.h>
 #include <Functions/FunctionFactory.h>
 #include <Functions/FunctionHelpers.h>
-#include <DataTypes/DataTypeMap.h>
-#include <DataTypes/DataTypesNumber.h>
-#include <DataTypes/DataTypeArray.h>
-#include <DataTypes/DataTypeTuple.h>
-#include <DataTypes/getLeastSupertype.h>
-#include <Columns/ColumnMap.h>
-#include <Interpreters/castColumn.h>
+#include <Functions/IFunction.h>
 #include <Interpreters/Context.h>
+#include <Interpreters/castColumn.h>
 #include <Common/HashTable/HashSet.h>
 
 
@@ -178,22 +181,28 @@ public:
                 getName(),
                 arguments.size());
 
-        /// The first argument should always be Array.
-        /// Because key type can not be nested type of Map, which is Tuple
-        DataTypePtr key_type;
-        if (const auto * keys_type = checkAndGetDataType<DataTypeArray>(arguments[0].get()))
-            key_type = keys_type->getNestedType();
-        else
-            throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "First argument for function {} must be an Array", getName());
+        auto get_nested_type = [this](const DataTypePtr & type) -> DataTypePtr
+        {
+            DataTypePtr nested;
+            if (const auto * array_type = checkAndGetDataType<DataTypeArray>(type.get()))
+                nested = array_type->getNestedType();
+            else if (const auto * map_type = checkAndGetDataType<DataTypeMap>(type.get()))
+                nested = std::make_shared<DataTypeTuple>(map_type->getKeyValueTypes());
+            else
+                throw Exception(
+                    ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
+                    "Argument types of function {} must be Array or Map, but {} is given",
+                    getName(),
+                    type->getName());
 
-        DataTypePtr value_type;
-        if (const auto * value_array_type = checkAndGetDataType<DataTypeArray>(arguments[1].get()))
-            value_type = value_array_type->getNestedType();
-        else if (const auto * value_map_type = checkAndGetDataType<DataTypeMap>(arguments[1].get()))
-            value_type = std::make_shared<DataTypeTuple>(value_map_type->getKeyValueTypes());
-        else
-            throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Second argument for function {} must be Array or Map", getName());
+            return nested;
+        };
 
+        auto key_type = get_nested_type(arguments[0]);
+        auto value_type = get_nested_type(arguments[1]);
+
+        /// Remove Nullable from key_type if needed for map key must not be Nullable
+        key_type = removeNullableOrLowCardinalityNullable(key_type);
         DataTypes key_value_types{key_type, value_type};
         return std::make_shared<DataTypeMap>(key_value_types);
     }
@@ -201,44 +210,62 @@ public:
     ColumnPtr executeImpl(
         const ColumnsWithTypeAndName & arguments, const DataTypePtr & /* result_type */, size_t /* input_rows_count */) const override
     {
-        bool is_keys_const = isColumnConst(*arguments[0].column);
-        ColumnPtr holder_keys;
-        const ColumnArray * col_keys;
-        if (is_keys_const)
+        auto get_array_column = [this](const ColumnPtr & column) -> std::pair<const ColumnArray *, ColumnPtr>
         {
-            holder_keys = arguments[0].column->convertToFullColumnIfConst();
-            col_keys = checkAndGetColumn<ColumnArray>(holder_keys.get());
-        }
-        else
+            bool is_const = isColumnConst(*column);
+            ColumnPtr holder = is_const ? column->convertToFullColumnIfConst() : column;
+
+            const ColumnArray * col_res = nullptr;
+            if (const auto * col_array = checkAndGetColumn<ColumnArray>(holder.get()))
+                col_res = col_array;
+            else if (const auto * col_map = checkAndGetColumn<ColumnMap>(holder.get()))
+                col_res = &col_map->getNestedColumn();
+            else
+                throw Exception(
+                    ErrorCodes::ILLEGAL_COLUMN,
+                    "Argument columns of function {} must be Array or Map, but {} is given",
+                    getName(),
+                    holder->getName());
+
+            return {col_res, holder};
+        };
+
+        auto [col_keys, key_holder] = get_array_column(arguments[0].column);
+
+        /// Check if nested column of first argument contains NULL value in case its nested type is Nullable(T) type.
+        ColumnPtr data_keys = col_keys->getDataPtr();
+        if (isColumnNullableOrLowCardinalityNullable(*data_keys))
         {
-            col_keys = checkAndGetColumn<ColumnArray>(arguments[0].column.get());
+            std::cout << "data keys is nullable" << std::endl;
+            const NullMap * null_map = nullptr;
+            if (const auto * nullable = checkAndGetColumn<ColumnNullable>(data_keys.get()))
+            {
+                null_map = &nullable->getNullMapData();
+                data_keys = nullable->getNestedColumnPtr();
+            }
+            else if (const auto * low_cardinality = checkAndGetColumn<ColumnLowCardinality>(data_keys.get()))
+            {
+                if (const auto * nullable_dict = checkAndGetColumn<ColumnNullable>(low_cardinality->getDictionaryPtr().get()))
+                {
+                    null_map = &nullable_dict->getNullMapData();
+                    data_keys = ColumnLowCardinality::create(nullable_dict->getNestedColumnPtr(), low_cardinality->getIndexesPtr());
+                }
+            }
+
+            if (null_map && !memoryIsZero(null_map->data(), 0, null_map->size()))
+                throw Exception(
+                    ErrorCodes::ILLEGAL_COLUMN, "The nested column of first argument in function {} must not contain NULLs", getName());
         }
 
-        if (!col_keys)
-            throw Exception(ErrorCodes::ILLEGAL_COLUMN, "The first argument of function {} must be Array", getName());
-
-        bool is_values_const = isColumnConst(*arguments[1].column);
-        ColumnPtr holder_values;
-        if (is_values_const)
-            holder_values = arguments[1].column->convertToFullColumnIfConst();
-        else
-            holder_values = arguments[1].column;
-
-        const ColumnArray * col_values;
-        if (const auto * col_values_array = checkAndGetColumn<ColumnArray>(holder_values.get()))
-            col_values = col_values_array;
-        else if (const auto * col_values_map = checkAndGetColumn<ColumnMap>(holder_values.get()))
-            col_values = &col_values_map->getNestedColumn();
-        else
-            throw Exception(ErrorCodes::ILLEGAL_COLUMN, "The second arguments of function {} must be Array or Map", getName());
-
+        auto [col_values, values_holder] = get_array_column(arguments[1].column);
         if (!col_keys->hasEqualOffsets(*col_values))
-            throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DONT_MATCH, "Two arguments for function {} must have equal sizes", getName());
+            throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DONT_MATCH, "Two arguments of function {} must have equal sizes", getName());
 
-        const auto & data_keys = col_keys->getDataPtr();
         const auto & data_values = col_values->getDataPtr();
         const auto & offsets = col_keys->getOffsetsPtr();
-        auto nested_column = ColumnArray::create(ColumnTuple::create(Columns{data_keys, data_values}), offsets);
+        std::cout << "before create array:" << "offsets:" << offsets->getName() << std::endl;
+        auto nested_column = ColumnArray::create(ColumnTuple::create(Columns{std::move(data_keys), data_values}), offsets);
+        std::cout << "after create array:" << "offsets:" << offsets->getName() << std::endl;
         return ColumnMap::create(nested_column);
     }
 };
diff --git a/tests/queries/0_stateless/01651_map_functions.reference b/tests/queries/0_stateless/01651_map_functions.reference
index 471da5586b7..9114aa419b1 100644
--- a/tests/queries/0_stateless/01651_map_functions.reference
+++ b/tests/queries/0_stateless/01651_map_functions.reference
@@ -52,3 +52,6 @@
 {1:4,2:5}
 {1:4,2:5}
 {1:4,2:5}
+{1:3,2:4}
+{1:3,2:4}
+{1:3,2:4}	{(1,3):'a',(2,4):'b'}
diff --git a/tests/queries/0_stateless/01651_map_functions.sql b/tests/queries/0_stateless/01651_map_functions.sql
index cf2460fce2c..4604ddd6db1 100644
--- a/tests/queries/0_stateless/01651_map_functions.sql
+++ b/tests/queries/0_stateless/01651_map_functions.sql
@@ -67,12 +67,15 @@ select mapFromArrays(['aa', 'bb'], [4, 5, 6]); -- { serverError SIZES_OF_ARRAYS_
 select mapFromArrays([[1,2], [3,4]], [4, 5, 6]); -- { serverError SIZES_OF_ARRAYS_DONT_MATCH }
 select mapFromArrays(['a', 2], [4, 5]); -- { serverError NO_COMMON_TYPE}
 select mapFromArrays([1, 2], [4, 'a']); -- { serverError NO_COMMON_TYPE}
+select mapFromArrays(['aa', 'bb'], map('a', 4)); -- { serverError SIZES_OF_ARRAYS_DONT_MATCH }
+select mapFromArrays([1,null]::Array(Nullable(UInt8)), [3,4]); -- { serverError ILLEGAL_COLUMN }
 
 select mapFromArrays(['aa', 'bb'], map('a', 4, 'b', 5));
 select mapFromArrays(['aa', 'bb'], materialize(map('a', 4, 'b', 5))) from numbers(2);
 
-select mapFromArrays(map('a', 4, 'b', 4), ['aa', 'bb']) from numbers(2); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT }
-select mapFromArrays(['aa', 'bb'], map('a', 4)); -- { serverError SIZES_OF_ARRAYS_DONT_MATCH }
-
 select mapFromArrays([toLowCardinality(1), toLowCardinality(2)], [4, 5]);
 select mapFromArrays([toLowCardinality(1), toLowCardinality(2)], materialize([4, 5])) from numbers(2);
+
+select mapFromArrays([1,2], [3,4]);
+select mapFromArrays([1,2]::Array(Nullable(UInt8)), [3,4]);
+select mapFromArrays([1,2], [3,4]) as x, mapFromArrays(x, ['a', 'b']);

From 65573871485c2e8ca45d791551856fd2f8622cf9 Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Thu, 25 Jul 2024 11:05:36 +0200
Subject: [PATCH 0756/1488] Try calcualting memory with numactl if membind used

---
 .gitmodules                            |  3 +
 base/base/CMakeLists.txt               |  4 ++
 base/base/getMemoryAmount.cpp          | 26 +++++++-
 contrib/CMakeLists.txt                 |  2 +
 contrib/numactl                        |  1 +
 contrib/numactl-cmake/CMakeLists.txt   | 20 +++++++
 contrib/numactl-cmake/include/config.h | 82 ++++++++++++++++++++++++++
 programs/server/Server.cpp             | 27 +++++++++
 src/Common/config.h.in                 |  1 +
 src/configure_config.cmake             |  3 +
 10 files changed, 168 insertions(+), 1 deletion(-)
 create mode 160000 contrib/numactl
 create mode 100644 contrib/numactl-cmake/CMakeLists.txt
 create mode 100644 contrib/numactl-cmake/include/config.h

diff --git a/.gitmodules b/.gitmodules
index 12d865307d8..b5d7e1e56b3 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -372,3 +372,6 @@
 [submodule "contrib/double-conversion"]
 	path = contrib/double-conversion
 	url = https://github.com/ClickHouse/double-conversion.git
+[submodule "contrib/numactl"]
+	path = contrib/numactl
+	url = https://github.com/numactl/numactl.git
diff --git a/base/base/CMakeLists.txt b/base/base/CMakeLists.txt
index 159502c9735..451a6eb5e8b 100644
--- a/base/base/CMakeLists.txt
+++ b/base/base/CMakeLists.txt
@@ -46,6 +46,10 @@ if (TARGET ch_contrib::crc32_s390x)
     target_link_libraries(common PUBLIC ch_contrib::crc32_s390x)
 endif()
 
+if (TARGET ch_contrib::numactl)
+    target_link_libraries(common PUBLIC ch_contrib::numactl)
+endif()
+
 target_include_directories(common PUBLIC .. "${CMAKE_CURRENT_BINARY_DIR}/..")
 
 target_link_libraries (common
diff --git a/base/base/getMemoryAmount.cpp b/base/base/getMemoryAmount.cpp
index afdb6ba068a..b8162146496 100644
--- a/base/base/getMemoryAmount.cpp
+++ b/base/base/getMemoryAmount.cpp
@@ -4,12 +4,17 @@
 #include <base/getPageSize.h>
 
 #include <fstream>
-#include <stdexcept>
 
 #include <unistd.h>
 #include <sys/types.h>
 #include <sys/param.h>
 
+#include "config.h"
+
+#if USE_NUMACTL
+#include <numa.h>
+#endif
+
 
 namespace
 {
@@ -63,6 +68,25 @@ uint64_t getMemoryAmountOrZero()
 
     uint64_t memory_amount = num_pages * page_size;
 
+#if USE_NUMACTL
+    if (numa_available() != -1)
+    {
+        auto * membind = numa_get_membind();
+        if (!numa_bitmask_equal(membind, numa_all_nodes_ptr))
+        {
+            uint64_t total_numa_memory = 0;
+            auto max_node = numa_max_node();
+            for (int i = 0; i <= max_node; ++i)
+            {
+                if (numa_bitmask_isbitset(membind, i))
+                    total_numa_memory += numa_node_size(i, nullptr);
+            }
+
+            memory_amount = total_numa_memory;
+        }
+    }
+#endif
+
     /// Respect the memory limit set by cgroups v2.
     auto limit_v2 = getCgroupsV2MemoryLimit();
     if (limit_v2.has_value() && *limit_v2 < memory_amount)
diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt
index 90ae5981a21..977efda15ff 100644
--- a/contrib/CMakeLists.txt
+++ b/contrib/CMakeLists.txt
@@ -230,6 +230,8 @@ add_contrib (libssh-cmake libssh)
 
 add_contrib (prometheus-protobufs-cmake prometheus-protobufs prometheus-protobufs-gogo)
 
+add_contrib(numactl-cmake numactl)
+
 # Put all targets defined here and in subdirectories under "contrib/<immediate-subdir>" folders in GUI-based IDEs.
 # Some of third-party projects may override CMAKE_FOLDER or FOLDER property of their targets, so they would not appear
 # in "contrib/..." as originally planned, so we workaround this by fixing FOLDER properties of all targets manually,
diff --git a/contrib/numactl b/contrib/numactl
new file mode 160000
index 00000000000..3871b1c42fc
--- /dev/null
+++ b/contrib/numactl
@@ -0,0 +1 @@
+Subproject commit 3871b1c42fc71bceadafd745d2eff5dddfc2d67e
diff --git a/contrib/numactl-cmake/CMakeLists.txt b/contrib/numactl-cmake/CMakeLists.txt
new file mode 100644
index 00000000000..5d086366c7f
--- /dev/null
+++ b/contrib/numactl-cmake/CMakeLists.txt
@@ -0,0 +1,20 @@
+option (ENABLE_NUMACTL "Enable numactl" ${ENABLE_LIBRARIES})
+
+if (NOT ENABLE_NUMACTL)
+    message (STATUS "Not using numactl")
+    return()
+endif ()
+
+set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/numactl")
+
+set (SRCS
+    "${LIBRARY_DIR}/libnuma.c"
+    "${LIBRARY_DIR}/syscall.c"
+)
+
+add_library(_numactl ${SRCS})
+
+target_include_directories(_numactl SYSTEM PRIVATE include)
+target_include_directories(_numactl SYSTEM PUBLIC "${LIBRARY_DIR}")
+
+add_library(ch_contrib::numactl ALIAS _numactl)
diff --git a/contrib/numactl-cmake/include/config.h b/contrib/numactl-cmake/include/config.h
new file mode 100644
index 00000000000..a304db38e53
--- /dev/null
+++ b/contrib/numactl-cmake/include/config.h
@@ -0,0 +1,82 @@
+/* config.h.  Generated from config.h.in by configure.  */
+/* config.h.in.  Generated from configure.ac by autoheader.  */
+
+/* Checking for symver attribute */
+#define HAVE_ATTRIBUTE_SYMVER 0
+
+/* Define to 1 if you have the <dlfcn.h> header file. */
+#define HAVE_DLFCN_H 1
+
+/* Define to 1 if you have the <inttypes.h> header file. */
+#define HAVE_INTTYPES_H 1
+
+/* Define to 1 if you have the <stdint.h> header file. */
+#define HAVE_STDINT_H 1
+
+/* Define to 1 if you have the <stdio.h> header file. */
+#define HAVE_STDIO_H 1
+
+/* Define to 1 if you have the <stdlib.h> header file. */
+#define HAVE_STDLIB_H 1
+
+/* Define to 1 if you have the <strings.h> header file. */
+#define HAVE_STRINGS_H 1
+
+/* Define to 1 if you have the <string.h> header file. */
+#define HAVE_STRING_H 1
+
+/* Define to 1 if you have the <sys/stat.h> header file. */
+#define HAVE_SYS_STAT_H 1
+
+/* Define to 1 if you have the <sys/types.h> header file. */
+#define HAVE_SYS_TYPES_H 1
+
+/* Define to 1 if you have the <unistd.h> header file. */
+#define HAVE_UNISTD_H 1
+
+/* Define to the sub-directory where libtool stores uninstalled libraries. */
+#define LT_OBJDIR ".libs/"
+
+/* Name of package */
+#define PACKAGE "numactl"
+
+/* Define to the address where bug reports for this package should be sent. */
+#define PACKAGE_BUGREPORT ""
+
+/* Define to the full name of this package. */
+#define PACKAGE_NAME "numactl"
+
+/* Define to the full name and version of this package. */
+#define PACKAGE_STRING "numactl 2.1"
+
+/* Define to the one symbol short name of this package. */
+#define PACKAGE_TARNAME "numactl"
+
+/* Define to the home page for this package. */
+#define PACKAGE_URL ""
+
+/* Define to the version of this package. */
+#define PACKAGE_VERSION "2.1"
+
+/* Define to 1 if all of the C89 standard headers exist (not just the ones
+   required in a freestanding environment). This macro is provided for
+   backward compatibility; new code need not use it. */
+#define STDC_HEADERS 1
+
+/* If the compiler supports a TLS storage class define it to that here */
+#define TLS __thread
+
+/* Version number of package */
+#define VERSION "2.1"
+
+/* Number of bits in a file offset, on hosts where this is settable. */
+/* #undef _FILE_OFFSET_BITS */
+
+/* Define to 1 on platforms where this makes off_t a 64-bit type. */
+/* #undef _LARGE_FILES */
+
+/* Number of bits in time_t, on hosts where this is settable. */
+/* #undef _TIME_BITS */
+
+/* Define to 1 on platforms where this makes time_t a 64-bit type. */
+/* #undef __MINGW_USE_VC2005_COMPAT */
diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp
index 16888015f8b..619a72ff200 100644
--- a/programs/server/Server.cpp
+++ b/programs/server/Server.cpp
@@ -140,6 +140,11 @@
 #   include <azure/core/diagnostics/logger.hpp>
 #endif
 
+#if USE_NUMACTL
+#include <numa.h>
+#endif
+
+
 #include <incbin.h>
 /// A minimal file used when the server is run without installation
 INCBIN(resource_embedded_xml, SOURCE_DIR "/programs/server/embedded.xml");
@@ -754,6 +759,28 @@ try
         setenv("OPENSSL_CONF", config_dir.c_str(), true); /// NOLINT
     }
 
+#if USE_NUMACTL
+    if (numa_available() != -1)
+    {
+        auto * membind = numa_get_membind();
+        if (!numa_bitmask_equal(membind, numa_all_nodes_ptr))
+        {
+            uint64_t total_numa_memory = 0;
+            auto max_node = numa_max_node();
+            for (int i = 0; i <= max_node; ++i)
+            {
+                if (numa_bitmask_isbitset(membind, i))
+                    total_numa_memory += numa_node_size(i, nullptr);
+            }
+
+            LOG_INFO(
+                log,
+                "ClickHouse is bound to a subset of NUMA nodes. Total memory of all available nodes {}",
+                ReadableSize(total_numa_memory));
+        }
+    }
+#endif
+
     registerInterpreters();
     registerFunctions();
     registerAggregateFunctions();
diff --git a/src/Common/config.h.in b/src/Common/config.h.in
index f68701d5d10..6a0090130a3 100644
--- a/src/Common/config.h.in
+++ b/src/Common/config.h.in
@@ -64,6 +64,7 @@
 #cmakedefine01 USE_LIBARCHIVE
 #cmakedefine01 USE_POCKETFFT
 #cmakedefine01 USE_PROMETHEUS_PROTOBUFS
+#cmakedefine01 USE_NUMACTL
 
 /// This is needed for .incbin in assembly. For some reason, include paths don't work there in presence of LTO.
 /// That's why we use absolute paths.
diff --git a/src/configure_config.cmake b/src/configure_config.cmake
index 75f61baa854..d22bf674df4 100644
--- a/src/configure_config.cmake
+++ b/src/configure_config.cmake
@@ -173,5 +173,8 @@ endif()
 if (TARGET ch_contrib::prometheus_protobufs)
     set(USE_PROMETHEUS_PROTOBUFS 1)
 endif()
+if (TARGET ch_contrib::numactl)
+    set(USE_NUMACTL 1)
+endif()
 
 set(SOURCE_DIR ${PROJECT_SOURCE_DIR})

From e181ccd0173c46d31867097532f64df0be3944da Mon Sep 17 00:00:00 2001
From: taiyang-li <654010905@qq.com>
Date: Thu, 25 Jul 2024 17:53:51 +0800
Subject: [PATCH 0757/1488] update doc

---
 docs/en/sql-reference/functions/tuple-map-functions.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/en/sql-reference/functions/tuple-map-functions.md b/docs/en/sql-reference/functions/tuple-map-functions.md
index ae23387f6e5..db66188b1f5 100644
--- a/docs/en/sql-reference/functions/tuple-map-functions.md
+++ b/docs/en/sql-reference/functions/tuple-map-functions.md
@@ -62,7 +62,7 @@ Alias: `MAP_FROM_ARRAYS(keys, values)`
 
 **Arguments**
 
-- `keys` —  Array or map of keys to create the map from. [Array(T)](../data-types/array.md) where `T` can be any type supported by [Map](../data-types/map.md) as key type.
+- `keys` —  Array or map of keys to create the map from. [Array(T)](../data-types/array.md) where `T` can be any type supported by [Map](../data-types/map.md) as key type, or [Map](../data-types/map.md).
 - `values`  - Array or map of values to create the map from. [Array](../data-types/array.md) or [Map](../data-types/map.md).
 
 **Returned value**

From 6968945373b2a73c135b0025cf892e21a2af4dbf Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Thu, 25 Jul 2024 09:58:32 +0000
Subject: [PATCH 0758/1488] Functions [s-t]*: Iterate over input_rows_count
 where appropriate

---
 src/Functions/FunctionTokens.h          |   9 +-
 src/Functions/seriesDecomposeSTL.cpp    |   4 +-
 src/Functions/space.cpp                 |  50 +++----
 src/Functions/stem.cpp                  |   9 +-
 src/Functions/stringCutToZero.cpp       |  21 ++-
 src/Functions/substringIndex.cpp        |  22 ++--
 src/Functions/subtractNanoseconds.cpp   |   1 +
 src/Functions/throwIf.cpp               |   2 +-
 src/Functions/timeSlots.cpp             |  80 ++++++------
 src/Functions/toDecimalString.cpp       |  64 ++++-----
 src/Functions/toStartOfInterval.cpp     |  44 +++----
 src/Functions/tokenExtractors.cpp       |  18 +--
 src/Functions/transform.cpp             | 167 ++++++++++++------------
 src/Functions/translate.cpp             |  16 ++-
 src/Functions/tupleToNameValuePairs.cpp |   6 +-
 15 files changed, 251 insertions(+), 262 deletions(-)

diff --git a/src/Functions/FunctionTokens.h b/src/Functions/FunctionTokens.h
index f1435ca5651..b6d8e9ee589 100644
--- a/src/Functions/FunctionTokens.h
+++ b/src/Functions/FunctionTokens.h
@@ -84,7 +84,7 @@ public:
         return std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>());
     }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t /*input_rows_count*/) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override
     {
         Generator generator;
         generator.init(arguments, max_substrings_includes_remaining_string);
@@ -107,18 +107,17 @@ public:
             const ColumnString::Chars & src_chars = col_str->getChars();
             const ColumnString::Offsets & src_offsets = col_str->getOffsets();
 
-            res_offsets.reserve(src_offsets.size());
-            res_strings_offsets.reserve(src_offsets.size() * 5);    /// Constant 5 - at random.
+            res_offsets.reserve(input_rows_count);
+            res_strings_offsets.reserve(input_rows_count * 5);    /// Constant 5 - at random.
             res_strings_chars.reserve(src_chars.size());
 
             Pos token_begin = nullptr;
             Pos token_end = nullptr;
 
-            size_t size = src_offsets.size();
             ColumnString::Offset current_src_offset = 0;
             ColumnArray::Offset current_dst_offset = 0;
             ColumnString::Offset current_dst_strings_offset = 0;
-            for (size_t i = 0; i < size; ++i)
+            for (size_t i = 0; i < input_rows_count; ++i)
             {
                 Pos pos = reinterpret_cast<Pos>(&src_chars[current_src_offset]);
                 current_src_offset = src_offsets[i];
diff --git a/src/Functions/seriesDecomposeSTL.cpp b/src/Functions/seriesDecomposeSTL.cpp
index 720aa1e0799..1e1c41cafad 100644
--- a/src/Functions/seriesDecomposeSTL.cpp
+++ b/src/Functions/seriesDecomposeSTL.cpp
@@ -50,7 +50,7 @@ public:
         return std::make_shared<DataTypeArray>(std::make_shared<DataTypeArray>(std::make_shared<DataTypeFloat32>()));
     }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
         ColumnPtr array_ptr = arguments[0].column;
         const ColumnArray * array = checkAndGetColumn<ColumnArray>(array_ptr.get());
@@ -79,7 +79,7 @@ public:
 
         ColumnArray::Offset prev_src_offset = 0;
 
-        for (size_t i = 0; i < src_offsets.size(); ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             UInt64 period;
             auto period_ptr = arguments[1].column->convertToFullColumnIfConst();
diff --git a/src/Functions/space.cpp b/src/Functions/space.cpp
index cd6ca73c088..cf1634e0319 100644
--- a/src/Functions/space.cpp
+++ b/src/Functions/space.cpp
@@ -55,7 +55,7 @@ public:
 
 
     template <typename DataType>
-    bool executeConstant(ColumnPtr col_times, ColumnString::Offsets & res_offsets, ColumnString::Chars & res_chars) const
+    bool executeConstant(ColumnPtr col_times, ColumnString::Offsets & res_offsets, ColumnString::Chars & res_chars, size_t input_rows_count) const
     {
         const ColumnConst & col_times_const = checkAndGetColumn<ColumnConst>(*col_times);
 
@@ -71,12 +71,12 @@ public:
 
         checkRepeatTime(times);
 
-        res_offsets.resize(col_times->size());
-        res_chars.resize(col_times->size() * (times + 1));
+        res_offsets.resize(input_rows_count);
+        res_chars.resize(input_rows_count * (times + 1));
 
         size_t pos = 0;
 
-        for (size_t i = 0; i < col_times->size(); ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             memset(res_chars.begin() + pos, space, times);
             pos += times;
@@ -92,20 +92,20 @@ public:
 
 
     template <typename DataType>
-    bool executeVector(ColumnPtr col_times_, ColumnString::Offsets & res_offsets, ColumnString::Chars & res_chars) const
+    bool executeVector(ColumnPtr col_times_, ColumnString::Offsets & res_offsets, ColumnString::Chars & res_chars, size_t input_rows_count) const
     {
         auto * col_times = checkAndGetColumn<typename DataType::ColumnType>(col_times_.get());
         if (!col_times)
             return false;
 
-        res_offsets.resize(col_times->size());
-        res_chars.resize(col_times->size() * 10); /// heuristic
+        res_offsets.resize(input_rows_count);
+        res_chars.resize(input_rows_count * 10); /// heuristic
 
         const PaddedPODArray<typename DataType::FieldType> & times_data = col_times->getData();
 
         size_t pos = 0;
 
-        for (size_t i = 0; i < col_times->size(); ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             typename DataType::FieldType times = times_data[i];
 
@@ -132,7 +132,7 @@ public:
     }
 
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
         const auto & col_num = arguments[0].column;
 
@@ -143,26 +143,26 @@ public:
 
         if (const ColumnConst * col_num_const = checkAndGetColumn<ColumnConst>(col_num.get()))
         {
-            if ((executeConstant<DataTypeUInt8>(col_num, res_offsets, res_chars))
-                || (executeConstant<DataTypeUInt16>(col_num, res_offsets, res_chars))
-                || (executeConstant<DataTypeUInt32>(col_num, res_offsets, res_chars))
-                || (executeConstant<DataTypeUInt64>(col_num, res_offsets, res_chars))
-                || (executeConstant<DataTypeInt8>(col_num, res_offsets, res_chars))
-                || (executeConstant<DataTypeInt16>(col_num, res_offsets, res_chars))
-                || (executeConstant<DataTypeInt32>(col_num, res_offsets, res_chars))
-                || (executeConstant<DataTypeInt64>(col_num, res_offsets, res_chars)))
+            if ((executeConstant<DataTypeUInt8>(col_num, res_offsets, res_chars, input_rows_count))
+                || (executeConstant<DataTypeUInt16>(col_num, res_offsets, res_chars, input_rows_count))
+                || (executeConstant<DataTypeUInt32>(col_num, res_offsets, res_chars, input_rows_count))
+                || (executeConstant<DataTypeUInt64>(col_num, res_offsets, res_chars, input_rows_count))
+                || (executeConstant<DataTypeInt8>(col_num, res_offsets, res_chars, input_rows_count))
+                || (executeConstant<DataTypeInt16>(col_num, res_offsets, res_chars, input_rows_count))
+                || (executeConstant<DataTypeInt32>(col_num, res_offsets, res_chars, input_rows_count))
+                || (executeConstant<DataTypeInt64>(col_num, res_offsets, res_chars, input_rows_count)))
                 return col_res;
         }
         else
         {
-            if ((executeVector<DataTypeUInt8>(col_num, res_offsets, res_chars))
-                || (executeVector<DataTypeUInt16>(col_num, res_offsets, res_chars))
-                || (executeVector<DataTypeUInt32>(col_num, res_offsets, res_chars))
-                || (executeVector<DataTypeUInt64>(col_num, res_offsets, res_chars))
-                || (executeVector<DataTypeInt8>(col_num, res_offsets, res_chars))
-                || (executeVector<DataTypeInt16>(col_num, res_offsets, res_chars))
-                || (executeVector<DataTypeInt32>(col_num, res_offsets, res_chars))
-                || (executeVector<DataTypeInt64>(col_num, res_offsets, res_chars)))
+            if ((executeVector<DataTypeUInt8>(col_num, res_offsets, res_chars, input_rows_count))
+                || (executeVector<DataTypeUInt16>(col_num, res_offsets, res_chars, input_rows_count))
+                || (executeVector<DataTypeUInt32>(col_num, res_offsets, res_chars, input_rows_count))
+                || (executeVector<DataTypeUInt64>(col_num, res_offsets, res_chars, input_rows_count))
+                || (executeVector<DataTypeInt8>(col_num, res_offsets, res_chars, input_rows_count))
+                || (executeVector<DataTypeInt16>(col_num, res_offsets, res_chars, input_rows_count))
+                || (executeVector<DataTypeInt32>(col_num, res_offsets, res_chars, input_rows_count))
+                || (executeVector<DataTypeInt64>(col_num, res_offsets, res_chars, input_rows_count)))
                 return col_res;
         }
 
diff --git a/src/Functions/stem.cpp b/src/Functions/stem.cpp
index 5b845cf332b..b3be40f4022 100644
--- a/src/Functions/stem.cpp
+++ b/src/Functions/stem.cpp
@@ -32,7 +32,8 @@ struct StemImpl
         const ColumnString::Offsets & offsets,
         ColumnString::Chars & res_data,
         ColumnString::Offsets & res_offsets,
-        const String & language)
+        const String & language,
+        size_t input_rows_count)
     {
         sb_stemmer * stemmer = sb_stemmer_new(language.data(), "UTF_8");
 
@@ -45,7 +46,7 @@ struct StemImpl
         res_offsets.assign(offsets);
 
         UInt64 data_size = 0;
-        for (UInt64 i = 0; i < offsets.size(); ++i)
+        for (UInt64 i = 0; i < input_rows_count; ++i)
         {
             /// Note that accessing -1th element is valid for PaddedPODArray.
             size_t original_size = offsets[i] - offsets[i - 1];
@@ -101,7 +102,7 @@ public:
 
     ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {0}; }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
         const auto & langcolumn = arguments[0].column;
         const auto & strcolumn = arguments[1].column;
@@ -119,7 +120,7 @@ public:
         String language = lang_col->getValue<String>();
 
         auto col_res = ColumnString::create();
-        StemImpl::vector(words_col->getChars(), words_col->getOffsets(), col_res->getChars(), col_res->getOffsets(), language);
+        StemImpl::vector(words_col->getChars(), words_col->getOffsets(), col_res->getChars(), col_res->getOffsets(), language, input_rows_count);
         return col_res;
     }
 };
diff --git a/src/Functions/stringCutToZero.cpp b/src/Functions/stringCutToZero.cpp
index b9f742cd8bc..16e57d741fa 100644
--- a/src/Functions/stringCutToZero.cpp
+++ b/src/Functions/stringCutToZero.cpp
@@ -40,7 +40,7 @@ public:
 
     bool useDefaultImplementationForConstants() const override { return true; }
 
-    static bool tryExecuteString(const IColumn * col, ColumnPtr & col_res)
+    static bool tryExecuteString(const IColumn * col, ColumnPtr & col_res, size_t input_rows_count)
     {
         const ColumnString * col_str_in = checkAndGetColumn<ColumnString>(col);
 
@@ -53,8 +53,7 @@ public:
             const ColumnString::Chars & in_vec = col_str_in->getChars();
             const ColumnString::Offsets & in_offsets = col_str_in->getOffsets();
 
-            size_t size = in_offsets.size();
-            out_offsets.resize(size);
+            out_offsets.resize(input_rows_count);
             out_vec.resize(in_vec.size());
 
             char * begin = reinterpret_cast<char *>(out_vec.data());
@@ -62,7 +61,7 @@ public:
 
             ColumnString::Offset current_in_offset = 0;
 
-            for (size_t i = 0; i < size; ++i)
+            for (size_t i = 0; i < input_rows_count; ++i)
             {
                 const char * pos_in = reinterpret_cast<const char *>(&in_vec[current_in_offset]);
                 size_t current_size = strlen(pos_in);
@@ -87,7 +86,7 @@ public:
         }
     }
 
-    static bool tryExecuteFixedString(const IColumn * col, ColumnPtr & col_res)
+    static bool tryExecuteFixedString(const IColumn * col, ColumnPtr & col_res, size_t input_rows_count)
     {
         const ColumnFixedString * col_fstr_in = checkAndGetColumn<ColumnFixedString>(col);
 
@@ -99,10 +98,8 @@ public:
 
             const ColumnString::Chars & in_vec = col_fstr_in->getChars();
 
-            size_t size = col_fstr_in->size();
-
-            out_offsets.resize(size);
-            out_vec.resize(in_vec.size() + size);
+            out_offsets.resize(input_rows_count);
+            out_vec.resize(in_vec.size() + input_rows_count);
 
             char * begin = reinterpret_cast<char *>(out_vec.data());
             char * pos = begin;
@@ -110,7 +107,7 @@ public:
 
             size_t n = col_fstr_in->getN();
 
-            for (size_t i = 0; i < size; ++i)
+            for (size_t i = 0; i < input_rows_count; ++i)
             {
                 size_t current_size = strnlen(pos_in, n);
                 memcpySmallAllowReadWriteOverflow15(pos, pos_in, current_size);
@@ -133,12 +130,12 @@ public:
         }
     }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
         const IColumn * column = arguments[0].column.get();
         ColumnPtr res_column;
 
-        if (tryExecuteFixedString(column, res_column) || tryExecuteString(column, res_column))
+        if (tryExecuteFixedString(column, res_column, input_rows_count) || tryExecuteString(column, res_column, input_rows_count))
             return res_column;
 
         throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of argument of function {}",
diff --git a/src/Functions/substringIndex.cpp b/src/Functions/substringIndex.cpp
index eccd849059b..dc12ae193ff 100644
--- a/src/Functions/substringIndex.cpp
+++ b/src/Functions/substringIndex.cpp
@@ -68,7 +68,7 @@ namespace
             return std::make_shared<DataTypeString>();
         }
 
-        ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
+        ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
         {
             ColumnPtr column_string = arguments[0].column;
             ColumnPtr column_delim = arguments[1].column;
@@ -110,10 +110,10 @@ namespace
                 if (is_count_const)
                 {
                     Int64 count = column_count->getInt(0);
-                    vectorConstant(col_str, delim, count, vec_res, offsets_res);
+                    vectorConstant(col_str, delim, count, vec_res, offsets_res, input_rows_count);
                 }
                 else
-                    vectorVector(col_str, delim, column_count.get(), vec_res, offsets_res);
+                    vectorVector(col_str, delim, column_count.get(), vec_res, offsets_res, input_rows_count);
             }
             return column_res;
         }
@@ -124,18 +124,18 @@ namespace
             const String & delim,
             const IColumn * count_column,
             ColumnString::Chars & res_data,
-            ColumnString::Offsets & res_offsets)
+            ColumnString::Offsets & res_offsets,
+            size_t input_rows_count)
         {
-            size_t rows = str_column->size();
             res_data.reserve(str_column->getChars().size() / 2);
-            res_offsets.reserve(rows);
+            res_offsets.reserve(input_rows_count);
 
             bool all_ascii = isAllASCII(str_column->getChars().data(), str_column->getChars().size())
                 && isAllASCII(reinterpret_cast<const UInt8 *>(delim.data()), delim.size());
             std::unique_ptr<PositionCaseSensitiveUTF8::SearcherInBigHaystack> searcher
                 = !is_utf8 || all_ascii ? nullptr : std::make_unique<PositionCaseSensitiveUTF8::SearcherInBigHaystack>(delim.data(), delim.size());
 
-            for (size_t i = 0; i < rows; ++i)
+            for (size_t i = 0; i < input_rows_count; ++i)
             {
                 StringRef str_ref = str_column->getDataAt(i);
                 Int64 count = count_column->getInt(i);
@@ -157,18 +157,18 @@ namespace
             const String & delim,
             Int64 count,
             ColumnString::Chars & res_data,
-            ColumnString::Offsets & res_offsets)
+            ColumnString::Offsets & res_offsets,
+            size_t input_rows_count)
         {
-            size_t rows = str_column->size();
             res_data.reserve(str_column->getChars().size() / 2);
-            res_offsets.reserve(rows);
+            res_offsets.reserve(input_rows_count);
 
             bool all_ascii = isAllASCII(str_column->getChars().data(), str_column->getChars().size())
                 && isAllASCII(reinterpret_cast<const UInt8 *>(delim.data()), delim.size());
             std::unique_ptr<PositionCaseSensitiveUTF8::SearcherInBigHaystack> searcher
                 = !is_utf8 || all_ascii ? nullptr : std::make_unique<PositionCaseSensitiveUTF8::SearcherInBigHaystack>(delim.data(), delim.size());
 
-            for (size_t i = 0; i < rows; ++i)
+            for (size_t i = 0; i < input_rows_count; ++i)
             {
                 StringRef str_ref = str_column->getDataAt(i);
 
diff --git a/src/Functions/subtractNanoseconds.cpp b/src/Functions/subtractNanoseconds.cpp
index fffb4eae37a..360c5ecd9cb 100644
--- a/src/Functions/subtractNanoseconds.cpp
+++ b/src/Functions/subtractNanoseconds.cpp
@@ -6,6 +6,7 @@ namespace DB
 {
 
 using FunctionSubtractNanoseconds = FunctionDateOrDateTimeAddInterval<SubtractNanosecondsImpl>;
+
 REGISTER_FUNCTION(SubtractNanoseconds)
 {
     factory.registerFunction<FunctionSubtractNanoseconds>();
diff --git a/src/Functions/throwIf.cpp b/src/Functions/throwIf.cpp
index becc6d2f772..e317c65c622 100644
--- a/src/Functions/throwIf.cpp
+++ b/src/Functions/throwIf.cpp
@@ -152,7 +152,7 @@ private:
         return nullptr;
     }
 
-    bool allow_custom_error_code_argument;
+    const bool allow_custom_error_code_argument;
 };
 
 }
diff --git a/src/Functions/timeSlots.cpp b/src/Functions/timeSlots.cpp
index 040495ab023..b62bb20c64e 100644
--- a/src/Functions/timeSlots.cpp
+++ b/src/Functions/timeSlots.cpp
@@ -41,18 +41,17 @@ struct TimeSlotsImpl
     /// The following three methods process DateTime type
     static void vectorVector(
         const PaddedPODArray<UInt32> & starts, const PaddedPODArray<UInt32> & durations, UInt32 time_slot_size,
-        PaddedPODArray<UInt32> & result_values, ColumnArray::Offsets & result_offsets)
+        PaddedPODArray<UInt32> & result_values, ColumnArray::Offsets & result_offsets,
+        size_t input_rows_count)
     {
         if (time_slot_size == 0)
             throw Exception(ErrorCodes::BAD_ARGUMENTS, "Time slot size cannot be zero");
 
-        size_t size = starts.size();
-
-        result_offsets.resize(size);
-        result_values.reserve(size);
+        result_offsets.resize(input_rows_count);
+        result_values.reserve(input_rows_count);
 
         ColumnArray::Offset current_offset = 0;
-        for (size_t i = 0; i < size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             for (UInt32 value = starts[i] / time_slot_size, end = (starts[i] + durations[i]) / time_slot_size; value <= end; ++value)
             {
@@ -66,18 +65,17 @@ struct TimeSlotsImpl
 
     static void vectorConstant(
         const PaddedPODArray<UInt32> & starts, UInt32 duration, UInt32 time_slot_size,
-        PaddedPODArray<UInt32> & result_values, ColumnArray::Offsets & result_offsets)
+        PaddedPODArray<UInt32> & result_values, ColumnArray::Offsets & result_offsets,
+        size_t input_rows_count)
     {
         if (time_slot_size == 0)
             throw Exception(ErrorCodes::BAD_ARGUMENTS, "Time slot size cannot be zero");
 
-        size_t size = starts.size();
-
-        result_offsets.resize(size);
-        result_values.reserve(size);
+        result_offsets.resize(input_rows_count);
+        result_values.reserve(input_rows_count);
 
         ColumnArray::Offset current_offset = 0;
-        for (size_t i = 0; i < size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             for (UInt32 value = starts[i] / time_slot_size, end = (starts[i] + duration) / time_slot_size; value <= end; ++value)
             {
@@ -91,18 +89,17 @@ struct TimeSlotsImpl
 
     static void constantVector(
         UInt32 start, const PaddedPODArray<UInt32> & durations, UInt32 time_slot_size,
-        PaddedPODArray<UInt32> & result_values, ColumnArray::Offsets & result_offsets)
+        PaddedPODArray<UInt32> & result_values, ColumnArray::Offsets & result_offsets,
+        size_t input_rows_count)
     {
         if (time_slot_size == 0)
             throw Exception(ErrorCodes::BAD_ARGUMENTS, "Time slot size cannot be zero");
 
-        size_t size = durations.size();
-
-        result_offsets.resize(size);
-        result_values.reserve(size);
+        result_offsets.resize(input_rows_count);
+        result_values.reserve(input_rows_count);
 
         ColumnArray::Offset current_offset = 0;
-        for (size_t i = 0; i < size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             for (UInt32 value = start / time_slot_size, end = (start + durations[i]) / time_slot_size; value <= end; ++value)
             {
@@ -120,12 +117,11 @@ struct TimeSlotsImpl
     */
     static NO_SANITIZE_UNDEFINED void vectorVector(
         const PaddedPODArray<DateTime64> & starts, const PaddedPODArray<Decimal64> & durations, Decimal64 time_slot_size,
-        PaddedPODArray<DateTime64> & result_values, ColumnArray::Offsets & result_offsets, UInt16 dt_scale, UInt16 duration_scale, UInt16 time_slot_scale)
+        PaddedPODArray<DateTime64> & result_values, ColumnArray::Offsets & result_offsets, UInt16 dt_scale, UInt16 duration_scale, UInt16 time_slot_scale,
+        size_t input_rows_count)
     {
-        size_t size = starts.size();
-
-        result_offsets.resize(size);
-        result_values.reserve(size);
+        result_offsets.resize(input_rows_count);
+        result_values.reserve(input_rows_count);
 
         /// Modify all units to have same scale
         UInt16 max_scale = std::max({dt_scale, duration_scale, time_slot_scale});
@@ -139,7 +135,7 @@ struct TimeSlotsImpl
         if (time_slot_size == 0)
             throw Exception(ErrorCodes::BAD_ARGUMENTS, "Time slot size cannot be zero");
 
-        for (size_t i = 0; i < size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             for (DateTime64 value = (starts[i] * dt_multiplier) / time_slot_size, end = (starts[i] * dt_multiplier + durations[i] * dur_multiplier) / time_slot_size; value <= end; value += 1)
             {
@@ -152,12 +148,11 @@ struct TimeSlotsImpl
 
     static NO_SANITIZE_UNDEFINED void vectorConstant(
         const PaddedPODArray<DateTime64> & starts, Decimal64 duration, Decimal64 time_slot_size,
-        PaddedPODArray<DateTime64> & result_values, ColumnArray::Offsets & result_offsets, UInt16 dt_scale, UInt16 duration_scale, UInt16 time_slot_scale)
+        PaddedPODArray<DateTime64> & result_values, ColumnArray::Offsets & result_offsets, UInt16 dt_scale, UInt16 duration_scale, UInt16 time_slot_scale,
+        size_t input_rows_count)
     {
-        size_t size = starts.size();
-
-        result_offsets.resize(size);
-        result_values.reserve(size);
+        result_offsets.resize(input_rows_count);
+        result_values.reserve(input_rows_count);
 
         /// Modify all units to have same scale
         UInt16 max_scale = std::max({dt_scale, duration_scale, time_slot_scale});
@@ -172,7 +167,7 @@ struct TimeSlotsImpl
         if (time_slot_size == 0)
             throw Exception(ErrorCodes::BAD_ARGUMENTS, "Time slot size cannot be zero");
 
-        for (size_t i = 0; i < size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             for (DateTime64 value = (starts[i] * dt_multiplier) / time_slot_size, end = (starts[i] * dt_multiplier + duration) / time_slot_size; value <= end; value += 1)
             {
@@ -185,12 +180,11 @@ struct TimeSlotsImpl
 
     static NO_SANITIZE_UNDEFINED void constantVector(
         DateTime64 start, const PaddedPODArray<Decimal64> & durations, Decimal64 time_slot_size,
-        PaddedPODArray<DateTime64> & result_values, ColumnArray::Offsets & result_offsets, UInt16 dt_scale, UInt16 duration_scale, UInt16 time_slot_scale)
+        PaddedPODArray<DateTime64> & result_values, ColumnArray::Offsets & result_offsets, UInt16 dt_scale, UInt16 duration_scale, UInt16 time_slot_scale,
+        size_t input_rows_count)
     {
-        size_t size = durations.size();
-
-        result_offsets.resize(size);
-        result_values.reserve(size);
+        result_offsets.resize(input_rows_count);
+        result_values.reserve(input_rows_count);
 
         /// Modify all units to have same scale
         UInt16 max_scale = std::max({dt_scale, duration_scale, time_slot_scale});
@@ -205,7 +199,7 @@ struct TimeSlotsImpl
         if (time_slot_size == 0)
             throw Exception(ErrorCodes::BAD_ARGUMENTS, "Time slot size cannot be zero");
 
-        for (size_t i = 0; i < size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             for (DateTime64 value = start / time_slot_size, end = (start + durations[i] * dur_multiplier) / time_slot_size; value <= end; value += 1)
             {
@@ -282,7 +276,7 @@ public:
 
     }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
         if (WhichDataType(arguments[0].type).isDateTime())
         {
@@ -308,17 +302,17 @@ public:
 
             if (dt_starts && durations)
             {
-                TimeSlotsImpl::vectorVector(dt_starts->getData(), durations->getData(), time_slot_size, res_values, res->getOffsets());
+                TimeSlotsImpl::vectorVector(dt_starts->getData(), durations->getData(), time_slot_size, res_values, res->getOffsets(), input_rows_count);
                 return res;
             }
             else if (dt_starts && const_durations)
             {
-                TimeSlotsImpl::vectorConstant(dt_starts->getData(), const_durations->getValue<UInt32>(), time_slot_size, res_values, res->getOffsets());
+                TimeSlotsImpl::vectorConstant(dt_starts->getData(), const_durations->getValue<UInt32>(), time_slot_size, res_values, res->getOffsets(), input_rows_count);
                 return res;
             }
             else if (dt_const_starts && durations)
             {
-                TimeSlotsImpl::constantVector(dt_const_starts->getValue<UInt32>(), durations->getData(), time_slot_size, res_values, res->getOffsets());
+                TimeSlotsImpl::constantVector(dt_const_starts->getValue<UInt32>(), durations->getData(), time_slot_size, res_values, res->getOffsets(), input_rows_count);
                 return res;
             }
         }
@@ -353,21 +347,21 @@ public:
             if (starts && durations)
             {
                 TimeSlotsImpl::vectorVector(starts->getData(), durations->getData(), time_slot_size, res_values, res->getOffsets(),
-                    start_time_scale, duration_scale, time_slot_scale);
+                    start_time_scale, duration_scale, time_slot_scale, input_rows_count);
                 return res;
             }
             else if (starts && const_durations)
             {
                 TimeSlotsImpl::vectorConstant(
                     starts->getData(), const_durations->getValue<Decimal64>(), time_slot_size, res_values, res->getOffsets(),
-                    start_time_scale, duration_scale, time_slot_scale);
+                    start_time_scale, duration_scale, time_slot_scale, input_rows_count);
                 return res;
             }
             else if (const_starts && durations)
             {
                 TimeSlotsImpl::constantVector(
                     const_starts->getValue<DateTime64>(), durations->getData(), time_slot_size, res_values, res->getOffsets(),
-                    start_time_scale, duration_scale, time_slot_scale);
+                    start_time_scale, duration_scale, time_slot_scale, input_rows_count);
                 return res;
             }
         }
diff --git a/src/Functions/toDecimalString.cpp b/src/Functions/toDecimalString.cpp
index 523948a5396..3566ebc93ad 100644
--- a/src/Functions/toDecimalString.cpp
+++ b/src/Functions/toDecimalString.cpp
@@ -54,9 +54,9 @@ private:
     /// For operations with Integer/Float
     template <typename FromVectorType>
     void vectorConstant(const FromVectorType & vec_from, UInt8 precision,
-                        ColumnString::Chars & vec_to, ColumnString::Offsets & result_offsets) const
+                        ColumnString::Chars & vec_to, ColumnString::Offsets & result_offsets,
+                        size_t input_rows_count) const
     {
-        size_t input_rows_count = vec_from.size();
         result_offsets.resize(input_rows_count);
 
         /// Buffer is used here and in functions below because resulting size cannot be precisely anticipated,
@@ -74,9 +74,9 @@ private:
 
     template <typename FirstArgVectorType>
     void vectorVector(const FirstArgVectorType & vec_from, const ColumnVector<UInt8>::Container & vec_precision,
-                      ColumnString::Chars & vec_to, ColumnString::Offsets & result_offsets) const
+                      ColumnString::Chars & vec_to, ColumnString::Offsets & result_offsets,
+                      size_t input_rows_count) const
     {
-        size_t input_rows_count = vec_from.size();
         result_offsets.resize(input_rows_count);
 
         WriteBufferFromVector<ColumnString::Chars> buf_to(vec_to);
@@ -98,7 +98,8 @@ private:
     /// For operations with Decimal
     template <typename FirstArgVectorType>
     void vectorConstant(const FirstArgVectorType & vec_from, UInt8 precision,
-                        ColumnString::Chars & vec_to, ColumnString::Offsets & result_offsets, UInt8 from_scale) const
+                        ColumnString::Chars & vec_to, ColumnString::Offsets & result_offsets, UInt8 from_scale,
+                        size_t input_rows_count) const
     {
         /// There are no more than 77 meaning digits (as it is the max length of UInt256). So we can limit it with 77.
         constexpr size_t max_digits = std::numeric_limits<UInt256>::digits10;
@@ -107,7 +108,6 @@ private:
                                 "Too many fractional digits requested for Decimal, must not be more than {}", max_digits);
 
         WriteBufferFromVector<ColumnString::Chars> buf_to(vec_to);
-        size_t input_rows_count = vec_from.size();
         result_offsets.resize(input_rows_count);
 
         for (size_t i = 0; i < input_rows_count; ++i)
@@ -121,9 +121,9 @@ private:
 
     template <typename FirstArgVectorType>
     void vectorVector(const FirstArgVectorType & vec_from, const ColumnVector<UInt8>::Container & vec_precision,
-                      ColumnString::Chars & vec_to, ColumnString::Offsets & result_offsets, UInt8 from_scale) const
+                      ColumnString::Chars & vec_to, ColumnString::Offsets & result_offsets, UInt8 from_scale,
+                      size_t input_rows_count) const
     {
-        size_t input_rows_count = vec_from.size();
         result_offsets.resize(input_rows_count);
 
         WriteBufferFromVector<ColumnString::Chars> buf_to(vec_to);
@@ -182,28 +182,28 @@ private:
     }
 
 public:
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
         switch (arguments[0].type->getTypeId())
         {
-            case TypeIndex::UInt8:      return executeType<UInt8>(arguments);
-            case TypeIndex::UInt16:     return executeType<UInt16>(arguments);
-            case TypeIndex::UInt32:     return executeType<UInt32>(arguments);
-            case TypeIndex::UInt64:     return executeType<UInt64>(arguments);
-            case TypeIndex::UInt128:    return executeType<UInt128>(arguments);
-            case TypeIndex::UInt256:    return executeType<UInt256>(arguments);
-            case TypeIndex::Int8:       return executeType<Int8>(arguments);
-            case TypeIndex::Int16:      return executeType<Int16>(arguments);
-            case TypeIndex::Int32:      return executeType<Int32>(arguments);
-            case TypeIndex::Int64:      return executeType<Int64>(arguments);
-            case TypeIndex::Int128:     return executeType<Int128>(arguments);
-            case TypeIndex::Int256:     return executeType<Int256>(arguments);
-            case TypeIndex::Float32:    return executeType<Float32>(arguments);
-            case TypeIndex::Float64:    return executeType<Float64>(arguments);
-            case TypeIndex::Decimal32:  return executeType<Decimal32>(arguments);
-            case TypeIndex::Decimal64:  return executeType<Decimal64>(arguments);
-            case TypeIndex::Decimal128: return executeType<Decimal128>(arguments);
-            case TypeIndex::Decimal256: return executeType<Decimal256>(arguments);
+            case TypeIndex::UInt8:      return executeType<UInt8>(arguments, input_rows_count);
+            case TypeIndex::UInt16:     return executeType<UInt16>(arguments, input_rows_count);
+            case TypeIndex::UInt32:     return executeType<UInt32>(arguments, input_rows_count);
+            case TypeIndex::UInt64:     return executeType<UInt64>(arguments, input_rows_count);
+            case TypeIndex::UInt128:    return executeType<UInt128>(arguments, input_rows_count);
+            case TypeIndex::UInt256:    return executeType<UInt256>(arguments, input_rows_count);
+            case TypeIndex::Int8:       return executeType<Int8>(arguments, input_rows_count);
+            case TypeIndex::Int16:      return executeType<Int16>(arguments, input_rows_count);
+            case TypeIndex::Int32:      return executeType<Int32>(arguments, input_rows_count);
+            case TypeIndex::Int64:      return executeType<Int64>(arguments, input_rows_count);
+            case TypeIndex::Int128:     return executeType<Int128>(arguments, input_rows_count);
+            case TypeIndex::Int256:     return executeType<Int256>(arguments, input_rows_count);
+            case TypeIndex::Float32:    return executeType<Float32>(arguments, input_rows_count);
+            case TypeIndex::Float64:    return executeType<Float64>(arguments, input_rows_count);
+            case TypeIndex::Decimal32:  return executeType<Decimal32>(arguments, input_rows_count);
+            case TypeIndex::Decimal64:  return executeType<Decimal64>(arguments, input_rows_count);
+            case TypeIndex::Decimal128: return executeType<Decimal128>(arguments, input_rows_count);
+            case TypeIndex::Decimal256: return executeType<Decimal256>(arguments, input_rows_count);
             default:
                 throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of argument of function {}",
                                 arguments[0].column->getName(), getName());
@@ -212,7 +212,7 @@ public:
 
 private:
     template <typename T>
-    ColumnPtr executeType(const ColumnsWithTypeAndName & arguments) const
+    ColumnPtr executeType(const ColumnsWithTypeAndName & arguments, size_t input_rows_count) const
     {
         const auto * precision_col = checkAndGetColumn<ColumnVector<UInt8>>(arguments[1].column.get());
         const auto * precision_col_const = checkAndGetColumnConst<ColumnVector<UInt8>>(arguments[1].column.get());
@@ -230,9 +230,9 @@ private:
             {
                 UInt8 from_scale = from_col->getScale();
                 if (precision_col_const)
-                    vectorConstant(from_col->getData(), precision_col_const->template getValue<UInt8>(), result_chars, result_offsets, from_scale);
+                    vectorConstant(from_col->getData(), precision_col_const->template getValue<UInt8>(), result_chars, result_offsets, from_scale, input_rows_count);
                 else if (precision_col)
-                    vectorVector(from_col->getData(), precision_col->getData(), result_chars, result_offsets, from_scale);
+                    vectorVector(from_col->getData(), precision_col->getData(), result_chars, result_offsets, from_scale, input_rows_count);
                 else
                     throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of second argument of function formatDecimal", arguments[1].column->getName());
             }
@@ -245,9 +245,9 @@ private:
             if (from_col)
             {
                 if (precision_col_const)
-                    vectorConstant(from_col->getData(), precision_col_const->template getValue<UInt8>(), result_chars, result_offsets);
+                    vectorConstant(from_col->getData(), precision_col_const->template getValue<UInt8>(), result_chars, result_offsets, input_rows_count);
                 else if (precision_col)
-                    vectorVector(from_col->getData(), precision_col->getData(), result_chars, result_offsets);
+                    vectorVector(from_col->getData(), precision_col->getData(), result_chars, result_offsets, input_rows_count);
                 else
                     throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of second argument of function formatDecimal", arguments[1].column->getName());
 
diff --git a/src/Functions/toStartOfInterval.cpp b/src/Functions/toStartOfInterval.cpp
index 50442d1b448..21b7cf895d2 100644
--- a/src/Functions/toStartOfInterval.cpp
+++ b/src/Functions/toStartOfInterval.cpp
@@ -147,19 +147,20 @@ public:
         std::unreachable();
     }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t /* input_rows_count */) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override
     {
         const auto & time_column = arguments[0];
         const auto & interval_column = arguments[1];
         const auto & time_zone = extractTimeZoneFromFunctionArguments(arguments, 2, 0);
-        auto result_column = dispatchForTimeColumn(time_column, interval_column, result_type, time_zone);
+        auto result_column = dispatchForTimeColumn(time_column, interval_column, result_type, time_zone, input_rows_count);
         return result_column;
     }
 
 private:
     ColumnPtr dispatchForTimeColumn(
         const ColumnWithTypeAndName & time_column, const ColumnWithTypeAndName & interval_column,
-        const DataTypePtr & result_type, const DateLUTImpl & time_zone) const
+        const DataTypePtr & result_type, const DateLUTImpl & time_zone,
+        size_t input_rows_count) const
     {
         const auto & time_column_type = *time_column.type.get();
         const auto & time_column_col = *time_column.column.get();
@@ -170,19 +171,19 @@ private:
             auto scale = assert_cast<const DataTypeDateTime64 &>(time_column_type).getScale();
 
             if (time_column_vec)
-                return dispatchForIntervalColumn(assert_cast<const DataTypeDateTime64 &>(time_column_type), *time_column_vec, interval_column, result_type, time_zone, scale);
+                return dispatchForIntervalColumn(assert_cast<const DataTypeDateTime64 &>(time_column_type), *time_column_vec, interval_column, result_type, time_zone, input_rows_count, scale);
         }
         else if (isDateTime(time_column_type))
         {
             const auto * time_column_vec = checkAndGetColumn<ColumnDateTime>(&time_column_col);
             if (time_column_vec)
-                return dispatchForIntervalColumn(assert_cast<const DataTypeDateTime &>(time_column_type), *time_column_vec, interval_column, result_type, time_zone);
+                return dispatchForIntervalColumn(assert_cast<const DataTypeDateTime &>(time_column_type), *time_column_vec, interval_column, result_type, time_zone, input_rows_count);
         }
         else if (isDate(time_column_type))
         {
             const auto * time_column_vec = checkAndGetColumn<ColumnDate>(&time_column_col);
             if (time_column_vec)
-                return dispatchForIntervalColumn(assert_cast<const DataTypeDate &>(time_column_type), *time_column_vec, interval_column, result_type, time_zone);
+                return dispatchForIntervalColumn(assert_cast<const DataTypeDate &>(time_column_type), *time_column_vec, interval_column, result_type, time_zone, input_rows_count);
         }
         throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal column for 1st argument of function {}, expected a Date, DateTime or DateTime64", getName());
     }
@@ -190,7 +191,7 @@ private:
     template <typename TimeDataType, typename TimeColumnType>
     ColumnPtr dispatchForIntervalColumn(
         const TimeDataType & time_data_type, const TimeColumnType & time_column, const ColumnWithTypeAndName & interval_column,
-        const DataTypePtr & result_type, const DateLUTImpl & time_zone, UInt16 scale = 1) const
+        const DataTypePtr & result_type, const DateLUTImpl & time_zone, size_t input_rows_count, UInt16 scale = 1) const
     {
         const auto * interval_type = checkAndGetDataType<DataTypeInterval>(interval_column.type.get());
         if (!interval_type)
@@ -207,27 +208,27 @@ private:
         switch (interval_type->getKind()) // NOLINT(bugprone-switch-missing-default-case)
         {
             case IntervalKind::Kind::Nanosecond:
-                return execute<TimeDataType, TimeColumnType, DataTypeDateTime64, IntervalKind::Kind::Nanosecond>(time_data_type, time_column, num_units, result_type, time_zone, scale);
+                return execute<TimeDataType, TimeColumnType, DataTypeDateTime64, IntervalKind::Kind::Nanosecond>(time_data_type, time_column, num_units, result_type, time_zone, input_rows_count, scale);
             case IntervalKind::Kind::Microsecond:
-                return execute<TimeDataType, TimeColumnType, DataTypeDateTime64, IntervalKind::Kind::Microsecond>(time_data_type, time_column, num_units, result_type, time_zone, scale);
+                return execute<TimeDataType, TimeColumnType, DataTypeDateTime64, IntervalKind::Kind::Microsecond>(time_data_type, time_column, num_units, result_type, time_zone, input_rows_count, scale);
             case IntervalKind::Kind::Millisecond:
-                return execute<TimeDataType, TimeColumnType, DataTypeDateTime64, IntervalKind::Kind::Millisecond>(time_data_type, time_column, num_units, result_type, time_zone, scale);
+                return execute<TimeDataType, TimeColumnType, DataTypeDateTime64, IntervalKind::Kind::Millisecond>(time_data_type, time_column, num_units, result_type, time_zone, input_rows_count, scale);
             case IntervalKind::Kind::Second:
-                return execute<TimeDataType, TimeColumnType, DataTypeDateTime, IntervalKind::Kind::Second>(time_data_type, time_column, num_units, result_type, time_zone, scale);
+                return execute<TimeDataType, TimeColumnType, DataTypeDateTime, IntervalKind::Kind::Second>(time_data_type, time_column, num_units, result_type, time_zone, input_rows_count, scale);
             case IntervalKind::Kind::Minute:
-                return execute<TimeDataType, TimeColumnType, DataTypeDateTime, IntervalKind::Kind::Minute>(time_data_type, time_column, num_units, result_type, time_zone, scale);
+                return execute<TimeDataType, TimeColumnType, DataTypeDateTime, IntervalKind::Kind::Minute>(time_data_type, time_column, num_units, result_type, time_zone, input_rows_count, scale);
             case IntervalKind::Kind::Hour:
-                return execute<TimeDataType, TimeColumnType, DataTypeDateTime, IntervalKind::Kind::Hour>(time_data_type, time_column, num_units, result_type, time_zone, scale);
+                return execute<TimeDataType, TimeColumnType, DataTypeDateTime, IntervalKind::Kind::Hour>(time_data_type, time_column, num_units, result_type, time_zone, input_rows_count, scale);
             case IntervalKind::Kind::Day:
-                return execute<TimeDataType, TimeColumnType, DataTypeDateTime, IntervalKind::Kind::Day>(time_data_type, time_column, num_units, result_type, time_zone, scale);
+                return execute<TimeDataType, TimeColumnType, DataTypeDateTime, IntervalKind::Kind::Day>(time_data_type, time_column, num_units, result_type, time_zone, input_rows_count, scale);
             case IntervalKind::Kind::Week:
-                return execute<TimeDataType, TimeColumnType, DataTypeDate, IntervalKind::Kind::Week>(time_data_type, time_column, num_units, result_type, time_zone, scale);
+                return execute<TimeDataType, TimeColumnType, DataTypeDate, IntervalKind::Kind::Week>(time_data_type, time_column, num_units, result_type, time_zone, input_rows_count, scale);
             case IntervalKind::Kind::Month:
-                return execute<TimeDataType, TimeColumnType, DataTypeDate, IntervalKind::Kind::Month>(time_data_type, time_column, num_units, result_type, time_zone, scale);
+                return execute<TimeDataType, TimeColumnType, DataTypeDate, IntervalKind::Kind::Month>(time_data_type, time_column, num_units, result_type, time_zone, input_rows_count, scale);
             case IntervalKind::Kind::Quarter:
-                return execute<TimeDataType, TimeColumnType, DataTypeDate, IntervalKind::Kind::Quarter>(time_data_type, time_column, num_units, result_type, time_zone, scale);
+                return execute<TimeDataType, TimeColumnType, DataTypeDate, IntervalKind::Kind::Quarter>(time_data_type, time_column, num_units, result_type, time_zone, input_rows_count, scale);
             case IntervalKind::Kind::Year:
-                return execute<TimeDataType, TimeColumnType, DataTypeDate, IntervalKind::Kind::Year>(time_data_type, time_column, num_units, result_type, time_zone, scale);
+                return execute<TimeDataType, TimeColumnType, DataTypeDate, IntervalKind::Kind::Year>(time_data_type, time_column, num_units, result_type, time_zone, input_rows_count, scale);
         }
 
         std::unreachable();
@@ -236,22 +237,21 @@ private:
     template <typename TimeDataType, typename TimeColumnType, typename ResultDataType, IntervalKind::Kind unit>
     ColumnPtr execute(
         const TimeDataType &, const TimeColumnType & time_column_type, Int64 num_units,
-        const DataTypePtr & result_type, const DateLUTImpl & time_zone, UInt16 scale) const
+        const DataTypePtr & result_type, const DateLUTImpl & time_zone, size_t input_rows_count, UInt16 scale) const
     {
         using ResultColumnType = typename ResultDataType::ColumnType;
         using ResultFieldType = typename ResultDataType::FieldType;
 
         const auto & time_data = time_column_type.getData();
-        size_t size = time_data.size();
 
         auto result_col = result_type->createColumn();
         auto * col_to = assert_cast<ResultColumnType *>(result_col.get());
         auto & result_data = col_to->getData();
-        result_data.resize(size);
+        result_data.resize(input_rows_count);
 
         Int64 scale_multiplier = DecimalUtils::scaleMultiplier<DateTime64>(scale);
 
-        for (size_t i = 0; i != size; ++i)
+        for (size_t i = 0; i != input_rows_count; ++i)
             result_data[i] = static_cast<ResultFieldType>(ToStartOfInterval<unit>::execute(time_data[i], num_units, time_zone, scale_multiplier));
 
         return result_col;
diff --git a/src/Functions/tokenExtractors.cpp b/src/Functions/tokenExtractors.cpp
index e7dcb5cced3..1bbf313fbae 100644
--- a/src/Functions/tokenExtractors.cpp
+++ b/src/Functions/tokenExtractors.cpp
@@ -73,7 +73,7 @@ public:
         return std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>());
     }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
         auto column_offsets = ColumnArray::ColumnOffsets::create();
 
@@ -90,9 +90,9 @@ public:
             auto input_column = arguments[0].column;
 
             if (const auto * column_string = checkAndGetColumn<ColumnString>(input_column.get()))
-                executeImpl(extractor, *column_string, *result_column_string, *column_offsets);
+                executeImpl(extractor, *column_string, *result_column_string, *column_offsets, input_rows_count);
             else if (const auto * column_fixed_string = checkAndGetColumn<ColumnFixedString>(input_column.get()))
-                executeImpl(extractor, *column_fixed_string, *result_column_string, *column_offsets);
+                executeImpl(extractor, *column_fixed_string, *result_column_string, *column_offsets, input_rows_count);
 
             return ColumnArray::create(std::move(result_column_string), std::move(column_offsets));
         }
@@ -105,9 +105,9 @@ public:
             auto input_column = arguments[0].column;
 
             if (const auto * column_string = checkAndGetColumn<ColumnString>(input_column.get()))
-                executeImpl(extractor, *column_string, *result_column_string, *column_offsets);
+                executeImpl(extractor, *column_string, *result_column_string, *column_offsets, input_rows_count);
             else if (const auto * column_fixed_string = checkAndGetColumn<ColumnFixedString>(input_column.get()))
-                executeImpl(extractor, *column_fixed_string, *result_column_string, *column_offsets);
+                executeImpl(extractor, *column_fixed_string, *result_column_string, *column_offsets, input_rows_count);
 
             return ColumnArray::create(std::move(result_column_string), std::move(column_offsets));
         }
@@ -120,15 +120,15 @@ private:
         const ExtractorType & extractor,
         StringColumnType & input_data_column,
         ResultStringColumnType & result_data_column,
-        ColumnArray::ColumnOffsets & offsets_column) const
+        ColumnArray::ColumnOffsets & offsets_column,
+        size_t input_rows_count) const
     {
         size_t current_tokens_size = 0;
         auto & offsets_data = offsets_column.getData();
 
-        size_t column_size = input_data_column.size();
-        offsets_data.resize(column_size);
+        offsets_data.resize(input_rows_count);
 
-        for (size_t i = 0; i < column_size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             auto data = input_data_column.getDataAt(i);
 
diff --git a/src/Functions/transform.cpp b/src/Functions/transform.cpp
index 68500779f93..0dfc9197845 100644
--- a/src/Functions/transform.cpp
+++ b/src/Functions/transform.cpp
@@ -173,30 +173,30 @@ namespace
             }
             else if (cache.table_num_to_idx)
             {
-                if (!executeNum<ColumnVector<UInt8>>(in, *column_result, default_non_const, *in_casted)
-                    && !executeNum<ColumnVector<UInt16>>(in, *column_result, default_non_const, *in_casted)
-                    && !executeNum<ColumnVector<UInt32>>(in, *column_result, default_non_const, *in_casted)
-                    && !executeNum<ColumnVector<UInt64>>(in, *column_result, default_non_const, *in_casted)
-                    && !executeNum<ColumnVector<Int8>>(in, *column_result, default_non_const, *in_casted)
-                    && !executeNum<ColumnVector<Int16>>(in, *column_result, default_non_const, *in_casted)
-                    && !executeNum<ColumnVector<Int32>>(in, *column_result, default_non_const, *in_casted)
-                    && !executeNum<ColumnVector<Int64>>(in, *column_result, default_non_const, *in_casted)
-                    && !executeNum<ColumnVector<Float32>>(in, *column_result, default_non_const, *in_casted)
-                    && !executeNum<ColumnVector<Float64>>(in, *column_result, default_non_const, *in_casted)
-                    && !executeNum<ColumnDecimal<Decimal32>>(in, *column_result, default_non_const, *in_casted)
-                    && !executeNum<ColumnDecimal<Decimal64>>(in, *column_result, default_non_const, *in_casted))
+                if (!executeNum<ColumnVector<UInt8>>(in, *column_result, default_non_const, *in_casted, input_rows_count)
+                    && !executeNum<ColumnVector<UInt16>>(in, *column_result, default_non_const, *in_casted, input_rows_count)
+                    && !executeNum<ColumnVector<UInt32>>(in, *column_result, default_non_const, *in_casted, input_rows_count)
+                    && !executeNum<ColumnVector<UInt64>>(in, *column_result, default_non_const, *in_casted, input_rows_count)
+                    && !executeNum<ColumnVector<Int8>>(in, *column_result, default_non_const, *in_casted, input_rows_count)
+                    && !executeNum<ColumnVector<Int16>>(in, *column_result, default_non_const, *in_casted, input_rows_count)
+                    && !executeNum<ColumnVector<Int32>>(in, *column_result, default_non_const, *in_casted, input_rows_count)
+                    && !executeNum<ColumnVector<Int64>>(in, *column_result, default_non_const, *in_casted, input_rows_count)
+                    && !executeNum<ColumnVector<Float32>>(in, *column_result, default_non_const, *in_casted, input_rows_count)
+                    && !executeNum<ColumnVector<Float64>>(in, *column_result, default_non_const, *in_casted, input_rows_count)
+                    && !executeNum<ColumnDecimal<Decimal32>>(in, *column_result, default_non_const, *in_casted, input_rows_count)
+                    && !executeNum<ColumnDecimal<Decimal64>>(in, *column_result, default_non_const, *in_casted, input_rows_count))
                 {
                     throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of first argument of function {}", in->getName(), getName());
                 }
             }
             else if (cache.table_string_to_idx)
             {
-                if (!executeString(in, *column_result, default_non_const, *in_casted))
-                    executeContiguous(in, *column_result, default_non_const, *in_casted);
+                if (!executeString(in, *column_result, default_non_const, *in_casted, input_rows_count))
+                    executeContiguous(in, *column_result, default_non_const, *in_casted, input_rows_count);
             }
             else if (cache.table_anything_to_idx)
             {
-                executeAnything(in, *column_result, default_non_const, *in_casted);
+                executeAnything(in, *column_result, default_non_const, *in_casted, input_rows_count);
             }
             else
                 throw Exception(ErrorCodes::LOGICAL_ERROR, "State of the function `transform` is not initialized");
@@ -217,12 +217,11 @@ namespace
             return impl->execute(args, result_type, input_rows_count);
         }
 
-        void executeAnything(const IColumn * in, IColumn & column_result, const ColumnPtr default_non_const, const IColumn & in_casted) const
+        void executeAnything(const IColumn * in, IColumn & column_result, const ColumnPtr default_non_const, const IColumn & in_casted, size_t input_rows_count) const
         {
-            const size_t size = in->size();
             const auto & table = *cache.table_anything_to_idx;
-            column_result.reserve(size);
-            for (size_t i = 0; i < size; ++i)
+            column_result.reserve(input_rows_count);
+            for (size_t i = 0; i < input_rows_count; ++i)
             {
                 SipHash hash;
                 in->updateHashWithValue(i, hash);
@@ -239,12 +238,11 @@ namespace
             }
         }
 
-        void executeContiguous(const IColumn * in, IColumn & column_result, const ColumnPtr default_non_const, const IColumn & in_casted) const
+        void executeContiguous(const IColumn * in, IColumn & column_result, const ColumnPtr default_non_const, const IColumn & in_casted, size_t input_rows_count) const
         {
-            const size_t size = in->size();
             const auto & table = *cache.table_string_to_idx;
-            column_result.reserve(size);
-            for (size_t i = 0; i < size; ++i)
+            column_result.reserve(input_rows_count);
+            for (size_t i = 0; i < input_rows_count; ++i)
             {
                 const auto * it = table.find(in->getDataAt(i));
                 if (it)
@@ -259,7 +257,7 @@ namespace
         }
 
         template <typename T>
-        bool executeNum(const IColumn * in_untyped, IColumn & column_result, const ColumnPtr default_non_const, const IColumn & in_casted) const
+        bool executeNum(const IColumn * in_untyped, IColumn & column_result, const ColumnPtr default_non_const, const IColumn & in_casted, size_t input_rows_count) const
         {
             const auto * const in = checkAndGetColumn<T>(in_untyped);
             if (!in)
@@ -269,24 +267,23 @@ namespace
             if constexpr (std::is_same_v<ColumnDecimal<Decimal32>, T> || std::is_same_v<ColumnDecimal<Decimal64>, T>)
                 in_scale = in->getScale();
 
-            if (!executeNumToString(pod, column_result, default_non_const)
-                && !executeNumToNum<ColumnVector<UInt8>>(pod, column_result, default_non_const, in_scale)
-                && !executeNumToNum<ColumnVector<UInt16>>(pod, column_result, default_non_const, in_scale)
-                && !executeNumToNum<ColumnVector<UInt32>>(pod, column_result, default_non_const, in_scale)
-                && !executeNumToNum<ColumnVector<UInt64>>(pod, column_result, default_non_const, in_scale)
-                && !executeNumToNum<ColumnVector<Int8>>(pod, column_result, default_non_const, in_scale)
-                && !executeNumToNum<ColumnVector<Int16>>(pod, column_result, default_non_const, in_scale)
-                && !executeNumToNum<ColumnVector<Int32>>(pod, column_result, default_non_const, in_scale)
-                && !executeNumToNum<ColumnVector<Int64>>(pod, column_result, default_non_const, in_scale)
-                && !executeNumToNum<ColumnVector<Float32>>(pod, column_result, default_non_const, in_scale)
-                && !executeNumToNum<ColumnVector<Float64>>(pod, column_result, default_non_const, in_scale)
-                && !executeNumToNum<ColumnDecimal<Decimal32>>(pod, column_result, default_non_const, in_scale)
-                && !executeNumToNum<ColumnDecimal<Decimal64>>(pod, column_result, default_non_const, in_scale))
+            if (!executeNumToString(pod, column_result, default_non_const, input_rows_count)
+                && !executeNumToNum<ColumnVector<UInt8>>(pod, column_result, default_non_const, in_scale, input_rows_count)
+                && !executeNumToNum<ColumnVector<UInt16>>(pod, column_result, default_non_const, in_scale, input_rows_count)
+                && !executeNumToNum<ColumnVector<UInt32>>(pod, column_result, default_non_const, in_scale, input_rows_count)
+                && !executeNumToNum<ColumnVector<UInt64>>(pod, column_result, default_non_const, in_scale, input_rows_count)
+                && !executeNumToNum<ColumnVector<Int8>>(pod, column_result, default_non_const, in_scale, input_rows_count)
+                && !executeNumToNum<ColumnVector<Int16>>(pod, column_result, default_non_const, in_scale, input_rows_count)
+                && !executeNumToNum<ColumnVector<Int32>>(pod, column_result, default_non_const, in_scale, input_rows_count)
+                && !executeNumToNum<ColumnVector<Int64>>(pod, column_result, default_non_const, in_scale, input_rows_count)
+                && !executeNumToNum<ColumnVector<Float32>>(pod, column_result, default_non_const, in_scale, input_rows_count)
+                && !executeNumToNum<ColumnVector<Float64>>(pod, column_result, default_non_const, in_scale, input_rows_count)
+                && !executeNumToNum<ColumnDecimal<Decimal32>>(pod, column_result, default_non_const, in_scale, input_rows_count)
+                && !executeNumToNum<ColumnDecimal<Decimal64>>(pod, column_result, default_non_const, in_scale, input_rows_count))
             {
-                const size_t size = pod.size();
                 const auto & table = *cache.table_num_to_idx;
-                column_result.reserve(size);
-                for (size_t i = 0; i < size; ++i)
+                column_result.reserve(input_rows_count);
+                for (size_t i = 0; i < input_rows_count; ++i)
                 {
                     const auto * it = table.find(bit_cast<UInt64>(pod[i]));
                     if (it)
@@ -303,14 +300,13 @@ namespace
         }
 
         template <typename T>
-        bool executeNumToString(const PaddedPODArray<T> & pod, IColumn & column_result, const ColumnPtr default_non_const) const
+        bool executeNumToString(const PaddedPODArray<T> & pod, IColumn & column_result, const ColumnPtr default_non_const, size_t input_rows_count) const
         {
             auto * out = typeid_cast<ColumnString *>(&column_result);
             if (!out)
                 return false;
             auto & out_offs = out->getOffsets();
-            const size_t size = pod.size();
-            out_offs.resize(size);
+            out_offs.resize(input_rows_count);
             auto & out_chars = out->getChars();
 
             const auto * to_col = assert_cast<const ColumnString *>(cache.to_column.get());
@@ -325,14 +321,14 @@ namespace
                 const auto & def_offs = def->getOffsets();
                 const auto * def_data = def_chars.data();
                 auto def_size = def_offs[0];
-                executeNumToStringHelper(table, pod, out_chars, out_offs, to_chars, to_offs, def_data, def_size, size);
+                executeNumToStringHelper(table, pod, out_chars, out_offs, to_chars, to_offs, def_data, def_size, input_rows_count);
             }
             else
             {
                 const auto * def = assert_cast<const ColumnString *>(default_non_const.get());
                 const auto & def_chars = def->getChars();
                 const auto & def_offs = def->getOffsets();
-                executeNumToStringHelper(table, pod, out_chars, out_offs, to_chars, to_offs, def_chars, def_offs, size);
+                executeNumToStringHelper(table, pod, out_chars, out_offs, to_chars, to_offs, def_chars, def_offs, input_rows_count);
             }
             return true;
         }
@@ -347,10 +343,10 @@ namespace
             const ColumnString::Offsets & to_offsets,
             const DefData & def_data,
             const DefOffs & def_offsets,
-            const size_t size) const
+            size_t input_rows_count) const
         {
             size_t out_cur_off = 0;
-            for (size_t i = 0; i < size; ++i)
+            for (size_t i = 0; i < input_rows_count; ++i)
             {
                 const char8_t * to = nullptr;
                 size_t to_size = 0;
@@ -382,14 +378,13 @@ namespace
 
         template <typename T, typename U>
         bool executeNumToNum(
-            const PaddedPODArray<U> & pod, IColumn & column_result, const ColumnPtr default_non_const, const UInt32 in_scale) const
+            const PaddedPODArray<U> & pod, IColumn & column_result, ColumnPtr default_non_const, UInt32 in_scale, size_t input_rows_count) const
         {
             auto * out = typeid_cast<T *>(&column_result);
             if (!out)
                 return false;
             auto & out_pod = out->getData();
-            const size_t size = pod.size();
-            out_pod.resize(size);
+            out_pod.resize(input_rows_count);
             UInt32 out_scale = 0;
             if constexpr (std::is_same_v<ColumnDecimal<Decimal32>, T> || std::is_same_v<ColumnDecimal<Decimal64>, T>)
                 out_scale = out->getScale();
@@ -399,15 +394,15 @@ namespace
             if (cache.default_column)
             {
                 const auto const_def = assert_cast<const T *>(cache.default_column.get())->getData()[0];
-                executeNumToNumHelper(table, pod, out_pod, to_pod, const_def, size, out_scale, out_scale);
+                executeNumToNumHelper(table, pod, out_pod, to_pod, const_def, input_rows_count, out_scale, out_scale);
             }
             else if (default_non_const)
             {
                 const auto & nconst_def = assert_cast<const T *>(default_non_const.get())->getData();
-                executeNumToNumHelper(table, pod, out_pod, to_pod, nconst_def, size, out_scale, out_scale);
+                executeNumToNumHelper(table, pod, out_pod, to_pod, nconst_def, input_rows_count, out_scale, out_scale);
             }
             else
-                executeNumToNumHelper(table, pod, out_pod, to_pod, pod, size, out_scale, in_scale);
+                executeNumToNumHelper(table, pod, out_pod, to_pod, pod, input_rows_count, out_scale, in_scale);
             return true;
         }
 
@@ -418,11 +413,11 @@ namespace
             PaddedPODArray<Out> & out_pod,
             const PaddedPODArray<Out> & to_pod,
             const Def & def,
-            const size_t size,
-            const UInt32 out_scale,
-            const UInt32 def_scale) const
+            size_t input_rows_count,
+            UInt32 out_scale,
+            UInt32 def_scale) const
         {
-            for (size_t i = 0; i < size; ++i)
+            for (size_t i = 0; i < input_rows_count; ++i)
             {
                 const auto * it = table.find(bit_cast<UInt64>(pod[i]));
                 if (it)
@@ -450,7 +445,7 @@ namespace
             }
         }
 
-        bool executeString(const IColumn * in_untyped, IColumn & column_result, const ColumnPtr default_non_const, const IColumn & in_casted) const
+        bool executeString(const IColumn * in_untyped, IColumn & column_result, const ColumnPtr default_non_const, const IColumn & in_casted, size_t input_rows_count) const
         {
             const auto * const in = checkAndGetColumn<ColumnString>(in_untyped);
             if (!in)
@@ -458,19 +453,19 @@ namespace
             const auto & data = in->getChars();
             const auto & offsets = in->getOffsets();
 
-            if (!executeStringToString(data, offsets, column_result, default_non_const)
-                && !executeStringToNum<ColumnVector<UInt8>>(data, offsets, column_result, default_non_const)
-                && !executeStringToNum<ColumnVector<UInt16>>(data, offsets, column_result, default_non_const)
-                && !executeStringToNum<ColumnVector<UInt32>>(data, offsets, column_result, default_non_const)
-                && !executeStringToNum<ColumnVector<UInt64>>(data, offsets, column_result, default_non_const)
-                && !executeStringToNum<ColumnVector<Int8>>(data, offsets, column_result, default_non_const)
-                && !executeStringToNum<ColumnVector<Int16>>(data, offsets, column_result, default_non_const)
-                && !executeStringToNum<ColumnVector<Int32>>(data, offsets, column_result, default_non_const)
-                && !executeStringToNum<ColumnVector<Int64>>(data, offsets, column_result, default_non_const)
-                && !executeStringToNum<ColumnVector<Float32>>(data, offsets, column_result, default_non_const)
-                && !executeStringToNum<ColumnVector<Float64>>(data, offsets, column_result, default_non_const)
-                && !executeStringToNum<ColumnDecimal<Decimal32>>(data, offsets, column_result, default_non_const)
-                && !executeStringToNum<ColumnDecimal<Decimal64>>(data, offsets, column_result, default_non_const))
+            if (!executeStringToString(data, offsets, column_result, default_non_const, input_rows_count)
+                && !executeStringToNum<ColumnVector<UInt8>>(data, offsets, column_result, default_non_const, input_rows_count)
+                && !executeStringToNum<ColumnVector<UInt16>>(data, offsets, column_result, default_non_const, input_rows_count)
+                && !executeStringToNum<ColumnVector<UInt32>>(data, offsets, column_result, default_non_const, input_rows_count)
+                && !executeStringToNum<ColumnVector<UInt64>>(data, offsets, column_result, default_non_const, input_rows_count)
+                && !executeStringToNum<ColumnVector<Int8>>(data, offsets, column_result, default_non_const, input_rows_count)
+                && !executeStringToNum<ColumnVector<Int16>>(data, offsets, column_result, default_non_const, input_rows_count)
+                && !executeStringToNum<ColumnVector<Int32>>(data, offsets, column_result, default_non_const, input_rows_count)
+                && !executeStringToNum<ColumnVector<Int64>>(data, offsets, column_result, default_non_const, input_rows_count)
+                && !executeStringToNum<ColumnVector<Float32>>(data, offsets, column_result, default_non_const, input_rows_count)
+                && !executeStringToNum<ColumnVector<Float64>>(data, offsets, column_result, default_non_const, input_rows_count)
+                && !executeStringToNum<ColumnDecimal<Decimal32>>(data, offsets, column_result, default_non_const, input_rows_count)
+                && !executeStringToNum<ColumnDecimal<Decimal64>>(data, offsets, column_result, default_non_const, input_rows_count))
             {
                 const size_t size = offsets.size();
                 const auto & table = *cache.table_string_to_idx;
@@ -497,14 +492,14 @@ namespace
             const ColumnString::Chars & data,
             const ColumnString::Offsets & offsets,
             IColumn & column_result,
-            const ColumnPtr default_non_const) const
+            const ColumnPtr default_non_const,
+            size_t input_rows_count) const
         {
             auto * out = typeid_cast<ColumnString *>(&column_result);
             if (!out)
                 return false;
             auto & out_offs = out->getOffsets();
-            const size_t size = offsets.size();
-            out_offs.resize(size);
+            out_offs.resize(input_rows_count);
             auto & out_chars = out->getChars();
 
             const auto * to_col = assert_cast<const ColumnString *>(cache.to_column.get());
@@ -519,18 +514,18 @@ namespace
                 const auto & def_offs = def->getOffsets();
                 const auto * def_data = def_chars.data();
                 auto def_size = def_offs[0];
-                executeStringToStringHelper(table, data, offsets, out_chars, out_offs, to_chars, to_offs, def_data, def_size, size);
+                executeStringToStringHelper(table, data, offsets, out_chars, out_offs, to_chars, to_offs, def_data, def_size, input_rows_count);
             }
             else if (default_non_const)
             {
                 const auto * def = assert_cast<const ColumnString *>(default_non_const.get());
                 const auto & def_chars = def->getChars();
                 const auto & def_offs = def->getOffsets();
-                executeStringToStringHelper(table, data, offsets, out_chars, out_offs, to_chars, to_offs, def_chars, def_offs, size);
+                executeStringToStringHelper(table, data, offsets, out_chars, out_offs, to_chars, to_offs, def_chars, def_offs, input_rows_count);
             }
             else
             {
-                executeStringToStringHelper(table, data, offsets, out_chars, out_offs, to_chars, to_offs, data, offsets, size);
+                executeStringToStringHelper(table, data, offsets, out_chars, out_offs, to_chars, to_offs, data, offsets, input_rows_count);
             }
             return true;
         }
@@ -546,11 +541,11 @@ namespace
             const ColumnString::Offsets & to_offsets,
             const DefData & def_data,
             const DefOffs & def_offsets,
-            const size_t size) const
+            size_t input_rows_count) const
         {
             ColumnString::Offset current_offset = 0;
             size_t out_cur_off = 0;
-            for (size_t i = 0; i < size; ++i)
+            for (size_t i = 0; i < input_rows_count; ++i)
             {
                 const char8_t * to = nullptr;
                 size_t to_size = 0;
@@ -587,26 +582,26 @@ namespace
             const ColumnString::Chars & data,
             const ColumnString::Offsets & offsets,
             IColumn & column_result,
-            const ColumnPtr default_non_const) const
+            const ColumnPtr default_non_const,
+            size_t input_rows_count) const
         {
             auto * out = typeid_cast<T *>(&column_result);
             if (!out)
                 return false;
             auto & out_pod = out->getData();
-            const size_t size = offsets.size();
-            out_pod.resize(size);
+            out_pod.resize(input_rows_count);
 
             const auto & to_pod = assert_cast<const T *>(cache.to_column.get())->getData();
             const auto & table = *cache.table_string_to_idx;
             if (cache.default_column)
             {
                 const auto const_def = assert_cast<const T *>(cache.default_column.get())->getData()[0];
-                executeStringToNumHelper(table, data, offsets, out_pod, to_pod, const_def, size);
+                executeStringToNumHelper(table, data, offsets, out_pod, to_pod, const_def, input_rows_count);
             }
             else
             {
                 const auto & nconst_def = assert_cast<const T *>(default_non_const.get())->getData();
-                executeStringToNumHelper(table, data, offsets, out_pod, to_pod, nconst_def, size);
+                executeStringToNumHelper(table, data, offsets, out_pod, to_pod, nconst_def, input_rows_count);
             }
             return true;
         }
@@ -619,10 +614,10 @@ namespace
             PaddedPODArray<Out> & out_pod,
             const PaddedPODArray<Out> & to_pod,
             const Def & def,
-            const size_t size) const
+            size_t input_rows_count) const
         {
             ColumnString::Offset current_offset = 0;
-            for (size_t i = 0; i < size; ++i)
+            for (size_t i = 0; i < input_rows_count; ++i)
             {
                 const StringRef ref{&data[current_offset], offsets[i] - current_offset - 1};
                 current_offset = offsets[i];
diff --git a/src/Functions/translate.cpp b/src/Functions/translate.cpp
index 2df08a5664e..366640d7d20 100644
--- a/src/Functions/translate.cpp
+++ b/src/Functions/translate.cpp
@@ -52,7 +52,8 @@ struct TranslateImpl
         const std::string & map_from,
         const std::string & map_to,
         ColumnString::Chars & res_data,
-        ColumnString::Offsets & res_offsets)
+        ColumnString::Offsets & res_offsets,
+        size_t input_rows_count)
     {
         Map map;
         fillMapWithValues(map, map_from, map_to);
@@ -62,7 +63,7 @@ struct TranslateImpl
 
         UInt8 * dst = res_data.data();
 
-        for (UInt64 i = 0; i < offsets.size(); ++i)
+        for (UInt64 i = 0; i < input_rows_count; ++i)
         {
             const UInt8 * src = data.data() + offsets[i - 1];
             const UInt8 * src_end = data.data() + offsets[i] - 1;
@@ -175,19 +176,20 @@ struct TranslateUTF8Impl
         const std::string & map_from,
         const std::string & map_to,
         ColumnString::Chars & res_data,
-        ColumnString::Offsets & res_offsets)
+        ColumnString::Offsets & res_offsets,
+        size_t input_rows_count)
     {
         MapASCII map_ascii;
         MapUTF8 map;
         fillMapWithValues(map_ascii, map, map_from, map_to);
 
         res_data.resize(data.size());
-        res_offsets.resize(offsets.size());
+        res_offsets.resize(input_rows_count);
 
         UInt8 * dst = res_data.data();
         UInt64 data_size = 0;
 
-        for (UInt64 i = 0; i < offsets.size(); ++i)
+        for (UInt64 i = 0; i < input_rows_count; ++i)
         {
             const UInt8 * src = data.data() + offsets[i - 1];
             const UInt8 * src_end = data.data() + offsets[i] - 1;
@@ -311,7 +313,7 @@ public:
         }
     }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
         const ColumnPtr column_src = arguments[0].column;
         const ColumnPtr column_map_from = arguments[1].column;
@@ -330,7 +332,7 @@ public:
         if (const ColumnString * col = checkAndGetColumn<ColumnString>(column_src.get()))
         {
             auto col_res = ColumnString::create();
-            Impl::vector(col->getChars(), col->getOffsets(), map_from, map_to, col_res->getChars(), col_res->getOffsets());
+            Impl::vector(col->getChars(), col->getOffsets(), map_from, map_to, col_res->getChars(), col_res->getOffsets(), input_rows_count);
             return col_res;
         }
         else if (const ColumnFixedString * col_fixed = checkAndGetColumn<ColumnFixedString>(column_src.get()))
diff --git a/src/Functions/tupleToNameValuePairs.cpp b/src/Functions/tupleToNameValuePairs.cpp
index 998e0da4f0c..92734d3d1fc 100644
--- a/src/Functions/tupleToNameValuePairs.cpp
+++ b/src/Functions/tupleToNameValuePairs.cpp
@@ -99,16 +99,16 @@ public:
         return std::make_shared<DataTypeArray>(item_data_type);
     }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
         const IColumn * tuple_col = arguments[0].column.get();
         const DataTypeTuple * tuple = checkAndGetDataType<DataTypeTuple>(arguments[0].type.get());
-        const auto * tuple_col_concrete = assert_cast<const ColumnTuple*>(tuple_col);
+        const auto * tuple_col_concrete = assert_cast<const ColumnTuple *>(tuple_col);
 
         auto keys = ColumnString::create();
         MutableColumnPtr values = tuple_col_concrete->getColumn(0).cloneEmpty();
         auto offsets = ColumnVector<UInt64>::create();
-        for (size_t row = 0; row < tuple_col_concrete->size(); ++row)
+        for (size_t row = 0; row < input_rows_count; ++row)
         {
             for (size_t col = 0; col < tuple_col_concrete->tupleSize(); ++col)
             {

From dc2c3fb1ca4653ee006c8cbbbfa32688f19f1992 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Thu, 25 Jul 2024 11:57:14 +0200
Subject: [PATCH 0759/1488] Revert "Merge pull request #66563 from
 ClickHouse/delete-bad-test"

This reverts commit d0753c8bb60dacfbd99687906fe4efb7665b20fa, reversing
changes made to b8202e19baf7ad171e232a431c8a4c3f1c86e63e.

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 .../0_stateless/02805_distributed_queries_timeouts.reference   | 0
 .../queries/0_stateless/02805_distributed_queries_timeouts.sql | 3 +++
 2 files changed, 3 insertions(+)
 create mode 100644 tests/queries/0_stateless/02805_distributed_queries_timeouts.reference
 create mode 100644 tests/queries/0_stateless/02805_distributed_queries_timeouts.sql

diff --git a/tests/queries/0_stateless/02805_distributed_queries_timeouts.reference b/tests/queries/0_stateless/02805_distributed_queries_timeouts.reference
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/queries/0_stateless/02805_distributed_queries_timeouts.sql b/tests/queries/0_stateless/02805_distributed_queries_timeouts.sql
new file mode 100644
index 00000000000..0b7337d1255
--- /dev/null
+++ b/tests/queries/0_stateless/02805_distributed_queries_timeouts.sql
@@ -0,0 +1,3 @@
+select * from remote('127.2', view(select sleep(3) from system.one)) settings receive_timeout=1, async_socket_for_remote=0, use_hedged_requests=1 format Null;
+select * from remote('127.2', view(select sleep(3) from system.one)) settings receive_timeout=1, async_socket_for_remote=1, use_hedged_requests=0 format Null;
+select * from remote('127.2', view(select sleep(3) from system.one)) settings receive_timeout=1, async_socket_for_remote=0, use_hedged_requests=0 format Null;

From 1a4730f1f390e468dab2849bd1b2770e0fb2cbe6 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Thu, 25 Jul 2024 12:03:50 +0200
Subject: [PATCH 0760/1488] Use Distributed table to avoid extra DESC queries

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 .../0_stateless/02805_distributed_queries_timeouts.sql     | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/tests/queries/0_stateless/02805_distributed_queries_timeouts.sql b/tests/queries/0_stateless/02805_distributed_queries_timeouts.sql
index 0b7337d1255..bfa39cd78ee 100644
--- a/tests/queries/0_stateless/02805_distributed_queries_timeouts.sql
+++ b/tests/queries/0_stateless/02805_distributed_queries_timeouts.sql
@@ -1,3 +1,4 @@
-select * from remote('127.2', view(select sleep(3) from system.one)) settings receive_timeout=1, async_socket_for_remote=0, use_hedged_requests=1 format Null;
-select * from remote('127.2', view(select sleep(3) from system.one)) settings receive_timeout=1, async_socket_for_remote=1, use_hedged_requests=0 format Null;
-select * from remote('127.2', view(select sleep(3) from system.one)) settings receive_timeout=1, async_socket_for_remote=0, use_hedged_requests=0 format Null;
+create table dist as system.one engine=Distributed(test_shard_localhost, system, one);
+select sleep(3) from dist settings prefer_localhost_replica=0, receive_timeout=1, async_socket_for_remote=0, use_hedged_requests=1 format Null;
+select sleep(3) from dist settings prefer_localhost_replica=0, receive_timeout=1, async_socket_for_remote=1, use_hedged_requests=0 format Null;
+select sleep(3) from dist settings prefer_localhost_replica=0, receive_timeout=1, async_socket_for_remote=0, use_hedged_requests=0 format Null;

From 49732f2966cd793e32234068cf0b87cea9e3eed6 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Thu, 25 Jul 2024 12:06:10 +0200
Subject: [PATCH 0761/1488] Tune sleep duration/receive_timeout in
 02805_distributed_queries_timeouts

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 .../0_stateless/02805_distributed_queries_timeouts.sql      | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/queries/0_stateless/02805_distributed_queries_timeouts.sql b/tests/queries/0_stateless/02805_distributed_queries_timeouts.sql
index bfa39cd78ee..f6bccc99977 100644
--- a/tests/queries/0_stateless/02805_distributed_queries_timeouts.sql
+++ b/tests/queries/0_stateless/02805_distributed_queries_timeouts.sql
@@ -1,4 +1,4 @@
 create table dist as system.one engine=Distributed(test_shard_localhost, system, one);
-select sleep(3) from dist settings prefer_localhost_replica=0, receive_timeout=1, async_socket_for_remote=0, use_hedged_requests=1 format Null;
-select sleep(3) from dist settings prefer_localhost_replica=0, receive_timeout=1, async_socket_for_remote=1, use_hedged_requests=0 format Null;
-select sleep(3) from dist settings prefer_localhost_replica=0, receive_timeout=1, async_socket_for_remote=0, use_hedged_requests=0 format Null;
+select sleep(8) from dist settings function_sleep_max_microseconds_per_block=8e9, prefer_localhost_replica=0, receive_timeout=7, async_socket_for_remote=0, use_hedged_requests=1 format Null;
+select sleep(8) from dist settings function_sleep_max_microseconds_per_block=8e9, prefer_localhost_replica=0, receive_timeout=7, async_socket_for_remote=1, use_hedged_requests=0 format Null;
+select sleep(8) from dist settings function_sleep_max_microseconds_per_block=8e9, prefer_localhost_replica=0, receive_timeout=7, async_socket_for_remote=0, use_hedged_requests=0 format Null;

From 4ee409094d4b1b9a8c8f4da5f1cf55d78a43b11f Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@clickhouse.com>
Date: Thu, 25 Jul 2024 12:12:23 +0200
Subject: [PATCH 0762/1488] Update tests/performance/decimal_aggregates.xml

---
 tests/performance/decimal_aggregates.xml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/performance/decimal_aggregates.xml b/tests/performance/decimal_aggregates.xml
index b204dddea6e..724d0c5d0e6 100644
--- a/tests/performance/decimal_aggregates.xml
+++ b/tests/performance/decimal_aggregates.xml
@@ -9,6 +9,7 @@
     <fill_query>INSERT INTO t SELECT number AS x, x % 1000000 AS d32, x AS d64, x d128 FROM numbers_mt(100000000, 100000000) SETTINGS max_threads = 2</fill_query>
     <fill_query>INSERT INTO t SELECT number AS x, x % 1000000 AS d32, x AS d64, x d128 FROM numbers_mt(200000000, 100000000) SETTINGS max_threads = 2</fill_query>
     <fill_query>INSERT INTO t SELECT number AS x, x % 1000000 AS d32, x AS d64, x d128 FROM numbers_mt(300000000, 100000000) SETTINGS max_threads = 2</fill_query>
+    <fill_query>INSERT INTO t SELECT number AS x, x % 1000000 AS d32, x AS d64, x d128 FROM numbers_mt(400000000, 100000000) SETTINGS max_threads = 2</fill_query>
 
     <drop_query>DROP TABLE IF EXISTS t</drop_query>
 

From 86e23b346fbce791794e38ad7ae77d8af964988a Mon Sep 17 00:00:00 2001
From: Max K <maxkaynov@gmail.com>
Date: Thu, 25 Jul 2024 12:12:37 +0200
Subject: [PATCH 0763/1488] rename test stages

---
 .github/workflows/master.yml       | 21 +++++++++++----------
 .github/workflows/pull_request.yml | 20 ++++++++++----------
 tests/ci/ci_config.py              |  6 +++---
 tests/ci/ci_definitions.py         |  6 +++---
 tests/ci/test_ci_config.py         |  4 ++--
 5 files changed, 29 insertions(+), 28 deletions(-)

diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml
index acd7511d520..2ce1124404f 100644
--- a/.github/workflows/master.yml
+++ b/.github/workflows/master.yml
@@ -93,21 +93,21 @@ jobs:
     with:
       stage: Builds_2
       data: ${{ needs.RunConfig.outputs.data }}
-  Tests_2:
+  Tests_2_ww:
     needs: [RunConfig, Builds_2]
+    if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Tests_2_ww') }}
+    uses: ./.github/workflows/reusable_test_stage.yml
+    with:
+      stage: Tests_2_ww
+      data: ${{ needs.RunConfig.outputs.data }}
+  Tests_2:
+    # Test_3 should not wait for Test_1/Test_2 and should not be blocked by them on master branch since all jobs need to run there.
+    needs: [RunConfig, Builds_1]
     if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Tests_2') }}
     uses: ./.github/workflows/reusable_test_stage.yml
     with:
       stage: Tests_2
       data: ${{ needs.RunConfig.outputs.data }}
-  Tests_3:
-    # Test_3 should not wait for Test_1/Test_2 and should not be blocked by them on master branch since all jobs need to run there.
-    needs: [RunConfig, Builds_1]
-    if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Tests_3') }}
-    uses: ./.github/workflows/reusable_test_stage.yml
-    with:
-      stage: Tests_3
-      data: ${{ needs.RunConfig.outputs.data }}
 
   ################################# Reports #################################
   # Reports should run even if Builds_1/2 fail - run them separately, not in Tests_1/2/3
@@ -123,7 +123,7 @@ jobs:
 
   FinishCheck:
     if: ${{ !cancelled() }}
-    needs: [RunConfig, Builds_1, Builds_2, Builds_Report, Tests_1, Tests_2, Tests_3]
+    needs: [RunConfig, Builds_1, Builds_2, Builds_Report, Tests_1, Tests_2_ww, Tests_2]
     runs-on: [self-hosted, style-checker-aarch64]
     steps:
       - name: Check out repository code
@@ -133,6 +133,7 @@ jobs:
           cd "$GITHUB_WORKSPACE/tests/ci"
           python3 finish_check.py --wf-status ${{ contains(needs.*.result, 'failure') && 'failure' || 'success' }}
       - name: Check Workflow results
+        if: ${{ !cancelled() }}
         run: |
           export WORKFLOW_RESULT_FILE="/tmp/workflow_results.json"
           cat > "$WORKFLOW_RESULT_FILE" << 'EOF'
diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
index 34bf51871d2..854dff530e7 100644
--- a/.github/workflows/pull_request.yml
+++ b/.github/workflows/pull_request.yml
@@ -123,20 +123,20 @@ jobs:
       stage: Builds_2
       data: ${{ needs.RunConfig.outputs.data }}
   # stage for running non-required checks without being blocked by required checks (Test_1) if corresponding settings is selected
-  Tests_2:
+  Tests_2_ww:
     needs: [RunConfig, Builds_1]
+    if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Tests_2_ww') }}
+    uses: ./.github/workflows/reusable_test_stage.yml
+    with:
+      stage: Tests_2_ww
+      data: ${{ needs.RunConfig.outputs.data }}
+  Tests_2:
+    needs: [RunConfig, Builds_1, Tests_1]
     if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Tests_2') }}
     uses: ./.github/workflows/reusable_test_stage.yml
     with:
       stage: Tests_2
       data: ${{ needs.RunConfig.outputs.data }}
-  Tests_3:
-    needs: [RunConfig, Builds_1, Tests_1]
-    if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Tests_3') }}
-    uses: ./.github/workflows/reusable_test_stage.yml
-    with:
-      stage: Tests_3
-      data: ${{ needs.RunConfig.outputs.data }}
 
   ################################# Reports #################################
   # Reports should run even if Builds_1/2 fail - run them separately (not in Tests_1/2/3)
@@ -154,7 +154,7 @@ jobs:
     if: ${{ !cancelled() }}
     # Test_2 or Test_3 do not have the jobs required for Mergeable check,
     #  however, set them as "needs" to get all checks results before the automatic merge occurs.
-    needs: [RunConfig, BuildDockers, StyleCheck, FastTest, Builds_1, Builds_2, Builds_Report, Tests_1, Tests_2, Tests_3]
+    needs: [RunConfig, BuildDockers, StyleCheck, FastTest, Builds_1, Builds_2, Builds_Report, Tests_1, Tests_2_ww, Tests_2]
     runs-on: [self-hosted, style-checker-aarch64]
     steps:
       - name: Check out repository code
@@ -178,7 +178,7 @@ jobs:
   #
   FinishCheck:
     if: ${{ !failure() && !cancelled() }}
-    needs: [RunConfig, BuildDockers, StyleCheck, FastTest, Builds_1, Builds_2, Builds_Report, Tests_1, Tests_2, Tests_3]
+    needs: [RunConfig, BuildDockers, StyleCheck, FastTest, Builds_1, Builds_2, Builds_Report, Tests_1, Tests_2_ww, Tests_2]
     runs-on: [self-hosted, style-checker-aarch64]
     steps:
       - name: Check out repository code
diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py
index a9bdb639835..0e295b2339d 100644
--- a/tests/ci/ci_config.py
+++ b/tests/ci/ci_config.py
@@ -587,10 +587,10 @@ class CI:
                 if job_name in REQUIRED_CHECKS:
                     stage_type = WorkflowStages.TESTS_1
                 else:
-                    stage_type = WorkflowStages.TESTS_3
+                    stage_type = WorkflowStages.TESTS_2
         assert stage_type, f"BUG [{job_name}]"
-        if non_blocking_ci and stage_type == WorkflowStages.TESTS_3:
-            stage_type = WorkflowStages.TESTS_2
+        if non_blocking_ci and stage_type == WorkflowStages.TESTS_2:
+            stage_type = WorkflowStages.TESTS_2_WW
         return stage_type
 
     @classmethod
diff --git a/tests/ci/ci_definitions.py b/tests/ci/ci_definitions.py
index 149177ecba5..054b554b8fa 100644
--- a/tests/ci/ci_definitions.py
+++ b/tests/ci/ci_definitions.py
@@ -67,10 +67,10 @@ class WorkflowStages(metaclass=WithIter):
     BUILDS_2 = "Builds_2"
     # all tests required for merge
     TESTS_1 = "Tests_1"
-    # not used atm
-    TESTS_2 = "Tests_2"
+    # used in woolenwolfdog mode
+    TESTS_2_WW = "Tests_2_ww"
     # all tests not required for merge
-    TESTS_3 = "Tests_3"
+    TESTS_2 = "Tests_2"
 
 
 class Runners(metaclass=WithIter):
diff --git a/tests/ci/test_ci_config.py b/tests/ci/test_ci_config.py
index 4a2bd606d0e..be540413b3c 100644
--- a/tests/ci/test_ci_config.py
+++ b/tests/ci/test_ci_config.py
@@ -211,7 +211,7 @@ class TestCIConfig(unittest.TestCase):
                 else:
                     self.assertTrue(
                         CI.get_job_ci_stage(job)
-                        in (CI.WorkflowStages.TESTS_1, CI.WorkflowStages.TESTS_3),
+                        in (CI.WorkflowStages.TESTS_1, CI.WorkflowStages.TESTS_2),
                         msg=f"Stage for [{job}] is not correct",
                     )
 
@@ -242,7 +242,7 @@ class TestCIConfig(unittest.TestCase):
                 else:
                     self.assertTrue(
                         CI.get_job_ci_stage(job, non_blocking_ci=True)
-                        in (CI.WorkflowStages.TESTS_1, CI.WorkflowStages.TESTS_2),
+                        in (CI.WorkflowStages.TESTS_1, CI.WorkflowStages.TESTS_2_WW),
                         msg=f"Stage for [{job}] is not correct",
                     )
 

From a32c702caa142d15bc3e5bc51ca90240d5d010a9 Mon Sep 17 00:00:00 2001
From: taiyang-li <654010905@qq.com>
Date: Thu, 25 Jul 2024 18:23:47 +0800
Subject: [PATCH 0764/1488] fix style

---
 src/Functions/map.cpp | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/Functions/map.cpp b/src/Functions/map.cpp
index 5319390fb70..a8e5f7ad90e 100644
--- a/src/Functions/map.cpp
+++ b/src/Functions/map.cpp
@@ -236,7 +236,6 @@ public:
         ColumnPtr data_keys = col_keys->getDataPtr();
         if (isColumnNullableOrLowCardinalityNullable(*data_keys))
         {
-            std::cout << "data keys is nullable" << std::endl;
             const NullMap * null_map = nullptr;
             if (const auto * nullable = checkAndGetColumn<ColumnNullable>(data_keys.get()))
             {
@@ -263,9 +262,7 @@ public:
 
         const auto & data_values = col_values->getDataPtr();
         const auto & offsets = col_keys->getOffsetsPtr();
-        std::cout << "before create array:" << "offsets:" << offsets->getName() << std::endl;
         auto nested_column = ColumnArray::create(ColumnTuple::create(Columns{std::move(data_keys), data_values}), offsets);
-        std::cout << "after create array:" << "offsets:" << offsets->getName() << std::endl;
         return ColumnMap::create(nested_column);
     }
 };

From 7fedc0ffbee9d04e0352037021a127cea93cbbfa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Thu, 25 Jul 2024 12:26:37 +0200
Subject: [PATCH 0765/1488] Update base/glibc-compatibility/musl/getauxval.c

Co-authored-by: Alexander Gololobov <davenger@clickhouse.com>
---
 base/glibc-compatibility/musl/getauxval.c | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/base/glibc-compatibility/musl/getauxval.c b/base/glibc-compatibility/musl/getauxval.c
index b5bd2f114c2..28cb0f8d005 100644
--- a/base/glibc-compatibility/musl/getauxval.c
+++ b/base/glibc-compatibility/musl/getauxval.c
@@ -99,12 +99,12 @@ static unsigned long NO_SANITIZE_THREAD __auxv_init_procfs(unsigned long type)
     /// #8 0x5622c0d6b7cd in _start (./ClickHouse/build_msan/contrib/google-protobuf-cmake/protoc+0x22c7cd) (BuildId: 6411d3c88b898ba3f7d49760555977d3e61f0741)
 
     /// The source of the issue above is that, at this point in time during __msan_init, we can't really do much as
-    /// most global variables aren't initialized or available yet, so we we can't initiate the auxiliary vector.
+    /// most global variables aren't initialized or available yet, so we can't initiate the auxiliary vector.
     /// Normal glibc / musl getauxval doesn't have this problem since they initiate their auxval vector at the very
     /// start of __libc_start_main (just keeping track of argv+argc+1), but we don't have such option (otherwise
     // this complexity of reading "/proc/self/auxv" or using __environ would not be necessary).
 
-    /// To avoid this crashes on the re-exec call (see above how it would fail when creating `aux`, and it we used
+    /// To avoid this crashes on the re-exec call (see above how it would fail when creating `aux`, and if we used
     /// __auxv_init_environ then it would SIGSEV on READing `__environ`) we capture this call for `AT_EXECFN` and
     /// unconditionally return "/proc/self/exe" without any preparation. Theoretically this should be fine in
     /// our case, as we don't load any libraries. That's the theory at least.

From beb506a5b8179f5c88a6f5fc90d62b8e74bf0d35 Mon Sep 17 00:00:00 2001
From: maxvostrikov <max.vostrikov@clickhouse.com>
Date: Thu, 25 Jul 2024 12:37:05 +0200
Subject: [PATCH 0766/1488] added somme tests in relation with
 https://github.com/ClickHouse/ClickHouse/pull/54881 with new behaviour when
 enable_named_columns_in_function_tuple=1 (default value)

---
 .../0_stateless/00307_format_xml.reference    |  41 ++++++++++++++++++
 .../queries/0_stateless/00307_format_xml.sql  |   3 ++
 .../0_stateless/00309_formats.reference       | Bin 18537 -> 18736 bytes
 tests/queries/0_stateless/00309_formats.sql   |   5 +++
 4 files changed, 49 insertions(+)

diff --git a/tests/queries/0_stateless/00307_format_xml.reference b/tests/queries/0_stateless/00307_format_xml.reference
index 2d9badc5a3e..14e74653d4f 100644
--- a/tests/queries/0_stateless/00307_format_xml.reference
+++ b/tests/queries/0_stateless/00307_format_xml.reference
@@ -1,3 +1,4 @@
+unnamed columns in tuple
 <?xml version='1.0' encoding='UTF-8' ?>
 <result>
 	<meta>
@@ -54,3 +55,43 @@
 	</extremes>
 	<rows>1</rows>
 </result>
+named columns in tuple
+<?xml version='1.0' encoding='UTF-8' ?>
+<result>
+	<meta>
+		<columns>
+			<column>
+				<name>s</name>
+				<type>String</type>
+			</column>
+			<column>
+				<name>time</name>
+				<type>DateTime</type>
+			</column>
+			<column>
+				<name>tpl</name>
+				<type>Tuple(String, DateTime)</type>
+			</column>
+		</columns>
+	</meta>
+	<data>
+		<row>
+			<s>Hello &amp; world</s>
+			<time>2001-02-03 04:05:06</time>
+			<tpl><tuple><elem>Hello &amp; world</elem><elem>2001-02-03 04:05:06</elem></tuple></tpl>
+		</row>
+	</data>
+	<extremes>
+		<min>
+			<s>Hello &amp; world</s>
+			<time>2001-02-03 04:05:06</time>
+			<tpl><tuple><elem>Hello &amp; world</elem><elem>2001-02-03 04:05:06</elem></tuple></tpl>
+		</min>
+		<max>
+			<s>Hello &amp; world</s>
+			<time>2001-02-03 04:05:06</time>
+			<tpl><tuple><elem>Hello &amp; world</elem><elem>2001-02-03 04:05:06</elem></tuple></tpl>
+		</max>
+	</extremes>
+	<rows>1</rows>
+</result>
diff --git a/tests/queries/0_stateless/00307_format_xml.sql b/tests/queries/0_stateless/00307_format_xml.sql
index 29c733bb186..22566112bc7 100644
--- a/tests/queries/0_stateless/00307_format_xml.sql
+++ b/tests/queries/0_stateless/00307_format_xml.sql
@@ -1,2 +1,5 @@
 SET output_format_write_statistics = 0;
+SELECT 'unnamed columns in tuple';
 SELECT 'Hello & world' AS s, 'Hello\n<World>', toDateTime('2001-02-03 04:05:06') AS time, arrayMap(x -> toString(x), range(10)) AS arr, (s, time) AS tpl SETTINGS extremes = 1, enable_named_columns_in_function_tuple = 0 FORMAT XML;
+SELECT 'named columns in tuple';
+SELECT 'Hello & world' AS s, toDateTime('2001-02-03 04:05:06') AS time, (s, time) AS tpl SETTINGS extremes = 1, enable_named_columns_in_function_tuple = 0 FORMAT XML;`
diff --git a/tests/queries/0_stateless/00309_formats.reference b/tests/queries/0_stateless/00309_formats.reference
index e637ee0363a7b35152a155ae3fa73a4f451d5148..a63720618ba54c6cc456f3356512449322dc2e80 100644
GIT binary patch
delta 149
zcmaDkfpNnm#toSsmduQKj47-or3E>uY@wccC1xfpE{P?n;vparjXVX2n2th<0!Ubs
ziva?zYfU!f5;0_8gmBMl=W(WRf=%Iq=->n?=fp5&@&sYg$;X8ig*A*!&2-d_3=9l(
M4J>sHjny@|06s7$QUCw|

delta 9
QcmdlmiSgwG#toSs02V|99{>OV

diff --git a/tests/queries/0_stateless/00309_formats.sql b/tests/queries/0_stateless/00309_formats.sql
index b0939c00a10..691fc6e7ab6 100644
--- a/tests/queries/0_stateless/00309_formats.sql
+++ b/tests/queries/0_stateless/00309_formats.sql
@@ -9,3 +9,8 @@ SELECT number * 246 + 10 AS n, toDate('2000-01-01') + n AS d, range(n) AS arr, a
 SELECT number * 246 + 10 AS n, toDate('2000-01-01') + n AS d, range(n) AS arr, arrayStringConcat(arrayMap(x -> reinterpretAsString(x), arr)) AS s, (n, d) AS tuple FROM system.numbers LIMIT 2 FORMAT JSON;
 SELECT number * 246 + 10 AS n, toDate('2000-01-01') + n AS d, range(n) AS arr, arrayStringConcat(arrayMap(x -> reinterpretAsString(x), arr)) AS s, (n, d) AS tuple FROM system.numbers LIMIT 2 FORMAT JSONCompact;
 SELECT number * 246 + 10 AS n, toDate('2000-01-01') + n AS d, range(n) AS arr, arrayStringConcat(arrayMap(x -> reinterpretAsString(x), arr)) AS s, (n, d) AS tuple FROM system.numbers LIMIT 2 FORMAT XML;
+
+SET enable_named_columns_in_function_tuple = 1;
+
+SELECT number * 246 + 10 AS n, toDate('2000-01-01') + n AS d, (n, d) AS tuple FROM system.numbers LIMIT 2 FORMAT RowBinaryWithNamesAndTypes;
+SELECT number * 246 + 10 AS n, toDate('2000-01-01') + n AS d, (n, d) AS tuple FROM system.numbers LIMIT 2 FORMAT TabSeparatedWithNamesAndTypes;

From 0dc67aae97d4b964cb0f9c389cbf3ce91cb76fb7 Mon Sep 17 00:00:00 2001
From: Sema Checherinda <Sema.Checherinda@clickhouse.com>
Date: Thu, 25 Jul 2024 13:52:30 +0200
Subject: [PATCH 0767/1488] fix MIN_ITERATIONS 2

---
 .../0_stateless/01171_mv_select_insert_isolation_long.sh        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh b/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh
index d79ab27d8b2..620281ee972 100755
--- a/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh
+++ b/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh
@@ -168,7 +168,7 @@ fi
 START_TIME=$(get_now)
 STOP_TIME=$((START_TIME + MAIN_TIME_PART))
 SECOND_STOP_TIME=$((STOP_TIME + SECOND_TIME_PART))
-MIN_ITERATIONS=30
+MIN_ITERATIONS=25
 
 run_until_deadline_and_at_least_times $STOP_TIME $MIN_ITERATIONS insert_commit_action 1   & PID_1=$!
 run_until_deadline_and_at_least_times $STOP_TIME $MIN_ITERATIONS insert_commit_action 2   & PID_2=$!

From 2988e13050f0ab8a06e36ec8fe745386a214141b Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Thu, 25 Jul 2024 13:55:01 +0200
Subject: [PATCH 0768/1488] Free bitmask

---
 base/base/getMemoryAmount.cpp | 1 +
 programs/server/Server.cpp    | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/base/base/getMemoryAmount.cpp b/base/base/getMemoryAmount.cpp
index b8162146496..56cddbfd628 100644
--- a/base/base/getMemoryAmount.cpp
+++ b/base/base/getMemoryAmount.cpp
@@ -84,6 +84,7 @@ uint64_t getMemoryAmountOrZero()
 
             memory_amount = total_numa_memory;
         }
+        numa_bitmask_free(membind);
     }
 #endif
 
diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp
index 619a72ff200..b9a7c298f00 100644
--- a/programs/server/Server.cpp
+++ b/programs/server/Server.cpp
@@ -778,6 +778,8 @@ try
                 "ClickHouse is bound to a subset of NUMA nodes. Total memory of all available nodes {}",
                 ReadableSize(total_numa_memory));
         }
+
+        numa_bitmask_free(membind);
     }
 #endif
 

From b80305ba981ca1f862084d3316144efcba17466b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Thu, 25 Jul 2024 14:31:00 +0200
Subject: [PATCH 0769/1488] Improve backport script

---
 tests/ci/cherry_pick.py | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/tests/ci/cherry_pick.py b/tests/ci/cherry_pick.py
index a7fc6d02853..623a816148e 100644
--- a/tests/ci/cherry_pick.py
+++ b/tests/ci/cherry_pick.py
@@ -420,7 +420,8 @@ class Backport:
             fetch_release_prs = self.gh.get_release_pulls(self._fetch_from)
             fetch_release_branches = [pr.head.ref for pr in fetch_release_prs]
             self.labels_to_backport = [
-                f"v{branch}-must-backport" for branch in fetch_release_branches
+                f"v{branch if self._repo_name == "ClickHouse/ClickHouse" else branch.replace('release/','')}-must-backport"
+                for branch in fetch_release_branches
             ]
 
             logging.info("Fetching from %s", self._fetch_from)
@@ -490,17 +491,23 @@ class Backport:
     def process_pr(self, pr: PullRequest) -> None:
         pr_labels = [label.name for label in pr.labels]
 
-        if (
-            any(label in pr_labels for label in self.must_create_backport_labels)
-            or self._repo_name != self._fetch_from
-        ):
+        if any(label in pr_labels for label in self.must_create_backport_labels):
             branches = [
                 ReleaseBranch(br, pr, self.repo, self.backport_created_label)
                 for br in self.release_branches
             ]  # type: List[ReleaseBranch]
         else:
             branches = [
-                ReleaseBranch(br, pr, self.repo, self.backport_created_label)
+                ReleaseBranch(
+                    (
+                        br
+                        if self._repo_name == "ClickHouse/clickhouse"
+                        else f"release/{br}"
+                    ),
+                    pr,
+                    self.repo,
+                    self.backport_created_label,
+                )
                 for br in [
                     label.split("-", 1)[0][1:]  # v21.8-must-backport
                     for label in pr_labels

From 2c83a39503255f0b2233b511a63a262cb8749a53 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Thu, 25 Jul 2024 12:53:16 +0000
Subject: [PATCH 0770/1488] Fixes

---
 tests/ci/cherry_pick.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tests/ci/cherry_pick.py b/tests/ci/cherry_pick.py
index 623a816148e..c2f567e5f15 100644
--- a/tests/ci/cherry_pick.py
+++ b/tests/ci/cherry_pick.py
@@ -420,7 +420,9 @@ class Backport:
             fetch_release_prs = self.gh.get_release_pulls(self._fetch_from)
             fetch_release_branches = [pr.head.ref for pr in fetch_release_prs]
             self.labels_to_backport = [
-                f"v{branch if self._repo_name == "ClickHouse/ClickHouse" else branch.replace('release/','')}-must-backport"
+                f"v{branch}-must-backport"
+                if self._repo_name == "ClickHouse/ClickHouse"
+                else f"v{branch.replace('release/','')}-must-backport"
                 for branch in fetch_release_branches
             ]
 
@@ -501,7 +503,7 @@ class Backport:
                 ReleaseBranch(
                     (
                         br
-                        if self._repo_name == "ClickHouse/clickhouse"
+                        if self._repo_name == "ClickHouse/Clickhouse"
                         else f"release/{br}"
                     ),
                     pr,

From 92cca8e65dec9f46d5a248c10e748088c9437cb6 Mon Sep 17 00:00:00 2001
From: Konstantin Bogdanov <konstantin@clickhouse.com>
Date: Thu, 25 Jul 2024 14:51:53 +0200
Subject: [PATCH 0771/1488] Fix

---
 src/AggregateFunctions/SingleValueData.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/AggregateFunctions/SingleValueData.cpp b/src/AggregateFunctions/SingleValueData.cpp
index a14caf00f73..996e64b22e0 100644
--- a/src/AggregateFunctions/SingleValueData.cpp
+++ b/src/AggregateFunctions/SingleValueData.cpp
@@ -1191,7 +1191,7 @@ bool SingleValueDataString::isEqualTo(const DB::IColumn & column, size_t row_num
 bool SingleValueDataString::isEqualTo(const SingleValueDataBase & other) const
 {
     auto const & to = assert_cast<const Self &>(other);
-    return has() && to.getStringRef() == getStringRef();
+    return has() && to.has() && to.getStringRef() == getStringRef();
 }
 
 void SingleValueDataString::set(const IColumn & column, size_t row_num, Arena * arena)

From cd06945a03df0f8dbec6ff82332236caf86fbff3 Mon Sep 17 00:00:00 2001
From: avogar <avogar@clickhouse.com>
Date: Thu, 25 Jul 2024 13:05:25 +0000
Subject: [PATCH 0772/1488] Fix crash with Variant + AggregateFunction type

---
 src/Columns/ColumnAggregateFunction.cpp       |  6 +-
 ...ant_with_aggregate_function_type.reference |  6 ++
 ...0_variant_with_aggregate_function_type.sql | 60 +++++++++++++++++++
 3 files changed, 71 insertions(+), 1 deletion(-)
 create mode 100644 tests/queries/0_stateless/03210_variant_with_aggregate_function_type.reference
 create mode 100644 tests/queries/0_stateless/03210_variant_with_aggregate_function_type.sql

diff --git a/src/Columns/ColumnAggregateFunction.cpp b/src/Columns/ColumnAggregateFunction.cpp
index e26fe790a8e..9934970c868 100644
--- a/src/Columns/ColumnAggregateFunction.cpp
+++ b/src/Columns/ColumnAggregateFunction.cpp
@@ -330,7 +330,11 @@ ColumnPtr ColumnAggregateFunction::filter(const Filter & filter, ssize_t result_
 
 void ColumnAggregateFunction::expand(const Filter & mask, bool inverted)
 {
-    expandDataByMask<char *>(data, mask, inverted);
+    ensureOwnership();
+    Arena & arena = createOrGetArena();
+    char * default_ptr = arena.alignedAlloc(func->sizeOfData(), func->alignOfData());
+    func->create(default_ptr);
+    expandDataByMask<char *>(data, mask, inverted, default_ptr);
 }
 
 ColumnPtr ColumnAggregateFunction::permute(const Permutation & perm, size_t limit) const
diff --git a/tests/queries/0_stateless/03210_variant_with_aggregate_function_type.reference b/tests/queries/0_stateless/03210_variant_with_aggregate_function_type.reference
new file mode 100644
index 00000000000..105e8e7d8bd
--- /dev/null
+++ b/tests/queries/0_stateless/03210_variant_with_aggregate_function_type.reference
@@ -0,0 +1,6 @@
+	500
+fail	500
+	499
+fail	500
+	500	499
+fail	500	500
diff --git a/tests/queries/0_stateless/03210_variant_with_aggregate_function_type.sql b/tests/queries/0_stateless/03210_variant_with_aggregate_function_type.sql
new file mode 100644
index 00000000000..cb9cdb0b456
--- /dev/null
+++ b/tests/queries/0_stateless/03210_variant_with_aggregate_function_type.sql
@@ -0,0 +1,60 @@
+SET allow_experimental_variant_type = 1;
+
+DROP TABLE IF EXISTS source;
+CREATE TABLE source 
+(
+   Name String,
+   Value Int64
+
+) ENGINE = MergeTree ORDER BY ();
+
+INSERT INTO source SELECT ['fail', 'success'][number % 2] as Name, number AS Value FROM numbers(1000);
+
+DROP TABLE IF EXISTS test_agg_variant;
+CREATE TABLE test_agg_variant
+(
+   Name String,
+   Value Variant(AggregateFunction(uniqExact, Int64), AggregateFunction(avg, Int64))
+)
+ENGINE = MergeTree
+ORDER BY (Name);
+
+INSERT INTO test_agg_variant
+SELECT
+    Name,
+    t AS Value
+FROM
+(
+    SELECT
+        Name,
+        arrayJoin([
+            uniqExactState(Value)::Variant(AggregateFunction(uniqExact, Int64), AggregateFunction(avg, Int64)), 
+            avgState(Value)::Variant(AggregateFunction(uniqExact, Int64), AggregateFunction(avg, Int64))
+        ]) AS t
+    FROM source
+    GROUP BY Name 
+);
+
+SELECT
+    Name,
+    uniqExactMerge(Value.`AggregateFunction(uniqExact, Int64)`) AS Value
+FROM test_agg_variant
+GROUP BY Name;
+
+SELECT
+    Name,
+    avgMerge(Value.`AggregateFunction(avg, Int64)`) AS Value
+FROM test_agg_variant
+GROUP BY Name;
+
+SELECT
+    Name,
+    uniqExactMerge(Value.`AggregateFunction(uniqExact, Int64)`) AS ValueUniq,
+    avgMerge(Value.`AggregateFunction(avg, Int64)`) AS ValueAvg
+FROM test_agg_variant
+GROUP BY Name;
+
+
+DROP TABLE test_agg_variant;
+DROP TABLE source;
+

From b23ce171c3620568829201d80789f314fc27499a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Thu, 25 Jul 2024 13:11:32 +0000
Subject: [PATCH 0773/1488] My black version said this was ok

---
 tests/ci/cherry_pick.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/tests/ci/cherry_pick.py b/tests/ci/cherry_pick.py
index c2f567e5f15..0b2aa9a2d35 100644
--- a/tests/ci/cherry_pick.py
+++ b/tests/ci/cherry_pick.py
@@ -420,9 +420,11 @@ class Backport:
             fetch_release_prs = self.gh.get_release_pulls(self._fetch_from)
             fetch_release_branches = [pr.head.ref for pr in fetch_release_prs]
             self.labels_to_backport = [
-                f"v{branch}-must-backport"
-                if self._repo_name == "ClickHouse/ClickHouse"
-                else f"v{branch.replace('release/','')}-must-backport"
+                (
+                    f"v{branch}-must-backport"
+                    if self._repo_name == "ClickHouse/ClickHouse"
+                    else f"v{branch.replace('release/','')}-must-backport"
+                )
                 for branch in fetch_release_branches
             ]
 

From c5164fede8665b61c10ec0d7b6873a7cf04aab12 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Thu, 25 Jul 2024 13:17:21 +0000
Subject: [PATCH 0774/1488] Fix some test.

---
 src/Interpreters/ExpressionAnalyzer.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp
index 6b5b129085d..5972d89bddd 100644
--- a/src/Interpreters/ExpressionAnalyzer.cpp
+++ b/src/Interpreters/ExpressionAnalyzer.cpp
@@ -1944,7 +1944,7 @@ ExpressionAnalysisResult::ExpressionAnalysisResult(
                 Block before_prewhere_sample = source_header;
                 if (sanitizeBlock(before_prewhere_sample))
                 {
-                    prewhere_dag_and_flags->dag.updateHeader(before_prewhere_sample);
+                    before_prewhere_sample = prewhere_dag_and_flags->dag.updateHeader(before_prewhere_sample);
                     auto & column_elem = before_prewhere_sample.getByName(query.prewhere()->getColumnName());
                     /// If the filter column is a constant, record it.
                     if (column_elem.column)
@@ -1976,7 +1976,7 @@ ExpressionAnalysisResult::ExpressionAnalysisResult(
                     before_where_sample = source_header;
                 if (sanitizeBlock(before_where_sample))
                 {
-                    before_where->dag.updateHeader(before_where_sample);
+                    before_where_sample = before_where->dag.updateHeader(before_where_sample);
 
                     auto & column_elem
                         = before_where_sample.getByName(query.where()->getColumnName());

From ae75c99e3fad02a3716c9d520c3a680f4d9d28e4 Mon Sep 17 00:00:00 2001
From: Konstantin Bogdanov <konstantin@clickhouse.com>
Date: Thu, 25 Jul 2024 15:17:59 +0200
Subject: [PATCH 0775/1488] Add a test

---
 .../0_stateless/03210_fix-single-value-data-assertion.reference  | 0
 .../0_stateless/03210_fix-single-value-data-assertion.sql        | 1 +
 2 files changed, 1 insertion(+)
 create mode 100644 tests/queries/0_stateless/03210_fix-single-value-data-assertion.reference
 create mode 100644 tests/queries/0_stateless/03210_fix-single-value-data-assertion.sql

diff --git a/tests/queries/0_stateless/03210_fix-single-value-data-assertion.reference b/tests/queries/0_stateless/03210_fix-single-value-data-assertion.reference
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/queries/0_stateless/03210_fix-single-value-data-assertion.sql b/tests/queries/0_stateless/03210_fix-single-value-data-assertion.sql
new file mode 100644
index 00000000000..66e62377d6b
--- /dev/null
+++ b/tests/queries/0_stateless/03210_fix-single-value-data-assertion.sql
@@ -0,0 +1 @@
+SELECT intDiv(number, 2) AS k, count(toFixedString(toFixedString('hello', 5), 5)) IGNORE NULLS, sumArgMax(number, toString(number % 20)), argMax(toString(number), number) FROM (SELECT number FROM system.numbers LIMIT 65537) WHERE toLowCardinality(toLowCardinality(toNullable(21))) GROUP BY k WITH TOTALS ORDER BY k ASC NULLS FIRST LIMIT 255 SETTINGS group_by_overflow_mode = 'any', totals_mode = 'before_having', max_rows_to_group_by = 100000 FORMAT Null

From ad44fb1ba4759434ecb4353a7878aea6162f8fef Mon Sep 17 00:00:00 2001
From: avogar <avogar@clickhouse.com>
Date: Thu, 25 Jul 2024 13:22:19 +0000
Subject: [PATCH 0776/1488] Increase
 lock_acquire_timeout_for_background_operations setting in dynamic merges
 tests

---
 .../03037_dynamic_merges_1_horizontal_compact_merge_tree.sql | 2 +-
 .../03037_dynamic_merges_1_horizontal_compact_wide_tree.sql  | 2 +-
 .../03037_dynamic_merges_1_vertical_compact_merge_tree.sql   | 5 +++--
 .../03037_dynamic_merges_1_vertical_wide_merge_tree.sql      | 2 +-
 .../03037_dynamic_merges_2_horizontal_compact_merge_tree.sql | 2 +-
 .../03037_dynamic_merges_2_horizontal_wide_merge_tree.sql    | 2 +-
 .../03037_dynamic_merges_2_vertical_compact_merge_tree.sql   | 2 +-
 .../03037_dynamic_merges_2_vertical_wide_merge_tree.sql      | 2 +-
 .../03038_nested_dynamic_merges_compact_horizontal.sql       | 2 +-
 .../03038_nested_dynamic_merges_compact_vertical.sql         | 2 +-
 .../03038_nested_dynamic_merges_wide_horizontal.sql          | 2 +-
 .../03038_nested_dynamic_merges_wide_vertical.sql            | 2 +-
 12 files changed, 14 insertions(+), 13 deletions(-)

diff --git a/tests/queries/0_stateless/03037_dynamic_merges_1_horizontal_compact_merge_tree.sql b/tests/queries/0_stateless/03037_dynamic_merges_1_horizontal_compact_merge_tree.sql
index b66fe5e2187..07371ee099b 100644
--- a/tests/queries/0_stateless/03037_dynamic_merges_1_horizontal_compact_merge_tree.sql
+++ b/tests/queries/0_stateless/03037_dynamic_merges_1_horizontal_compact_merge_tree.sql
@@ -2,7 +2,7 @@
 set allow_experimental_dynamic_type=1;
 
 drop table if exists test;
-create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000, vertical_merge_algorithm_min_columns_to_activate=10, index_granularity_bytes=10485760, index_granularity=8192, merge_max_block_size=8192, merge_max_block_size_bytes=10485760;
+create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000, vertical_merge_algorithm_min_columns_to_activate=10, index_granularity_bytes=10485760, index_granularity=8192, merge_max_block_size=8192, merge_max_block_size_bytes=10485760, lock_acquire_timeout_for_background_operations=600;
 
 system stop merges test;
 insert into test select number, number from numbers(100000);
diff --git a/tests/queries/0_stateless/03037_dynamic_merges_1_horizontal_compact_wide_tree.sql b/tests/queries/0_stateless/03037_dynamic_merges_1_horizontal_compact_wide_tree.sql
index 8a376b6d7d7..2b55a31e937 100644
--- a/tests/queries/0_stateless/03037_dynamic_merges_1_horizontal_compact_wide_tree.sql
+++ b/tests/queries/0_stateless/03037_dynamic_merges_1_horizontal_compact_wide_tree.sql
@@ -2,7 +2,7 @@
 set allow_experimental_dynamic_type=1;
 
 drop table if exists test;
-create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1, vertical_merge_algorithm_min_columns_to_activate=10, index_granularity_bytes=10485760, index_granularity=8192, merge_max_block_size=8192, merge_max_block_size_bytes=10485760;
+create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1, vertical_merge_algorithm_min_columns_to_activate=10, index_granularity_bytes=10485760, index_granularity=8192, merge_max_block_size=8192, merge_max_block_size_bytes=10485760, lock_acquire_timeout_for_background_operations=600;
 
 system stop merges test;
 insert into test select number, number from numbers(100000);
diff --git a/tests/queries/0_stateless/03037_dynamic_merges_1_vertical_compact_merge_tree.sql b/tests/queries/0_stateless/03037_dynamic_merges_1_vertical_compact_merge_tree.sql
index 127b56e727c..ea7295a9eab 100644
--- a/tests/queries/0_stateless/03037_dynamic_merges_1_vertical_compact_merge_tree.sql
+++ b/tests/queries/0_stateless/03037_dynamic_merges_1_vertical_compact_merge_tree.sql
@@ -2,7 +2,7 @@
 set allow_experimental_dynamic_type=1;
 
 drop table if exists test;
-create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1, index_granularity_bytes=10485760, index_granularity=8192, merge_max_block_size=8192, merge_max_block_size_bytes=10485760;
+create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1, index_granularity_bytes=10485760, index_granularity=8192, merge_max_block_size=8192, merge_max_block_size_bytes=10485760, lock_acquire_timeout_for_background_operations=600;
 
 system stop merges test;
 insert into test select number, number from numbers(100000);
@@ -13,7 +13,8 @@ insert into test select number, toDateTime(number) from numbers(50000);
 insert into test select number, NULL from numbers(100000);
 
 select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
-system start merges test; optimize table test final;;
+system start merges test;
+optimize table test final;
 select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
 
 system stop merges test;
diff --git a/tests/queries/0_stateless/03037_dynamic_merges_1_vertical_wide_merge_tree.sql b/tests/queries/0_stateless/03037_dynamic_merges_1_vertical_wide_merge_tree.sql
index e5c273cb592..e888a14b323 100644
--- a/tests/queries/0_stateless/03037_dynamic_merges_1_vertical_wide_merge_tree.sql
+++ b/tests/queries/0_stateless/03037_dynamic_merges_1_vertical_wide_merge_tree.sql
@@ -2,7 +2,7 @@
 set allow_experimental_dynamic_type=1;
 
 drop table if exists test;
-create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1, index_granularity_bytes=10485760, index_granularity=8192, merge_max_block_size=8192, merge_max_block_size_bytes=10485760;
+create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1, index_granularity_bytes=10485760, index_granularity=8192, merge_max_block_size=8192, merge_max_block_size_bytes=10485760, lock_acquire_timeout_for_background_operations=600;
 
 system stop merges test;
 insert into test select number, number from numbers(100000);
diff --git a/tests/queries/0_stateless/03037_dynamic_merges_2_horizontal_compact_merge_tree.sql b/tests/queries/0_stateless/03037_dynamic_merges_2_horizontal_compact_merge_tree.sql
index 6d7a0dd8c18..e633b277ebd 100644
--- a/tests/queries/0_stateless/03037_dynamic_merges_2_horizontal_compact_merge_tree.sql
+++ b/tests/queries/0_stateless/03037_dynamic_merges_2_horizontal_compact_merge_tree.sql
@@ -3,7 +3,7 @@
 set allow_experimental_dynamic_type = 1;
 
 drop table if exists test;
-create table test (id UInt64, d Dynamic) engine=MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000;
+create table test (id UInt64, d Dynamic) engine=MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000, lock_acquire_timeout_for_background_operations=600;
 system stop merges test;
 insert into test select number, number from numbers(1000000);
 insert into test select number, 'str_' || toString(number) from numbers(1000000, 1000000);
diff --git a/tests/queries/0_stateless/03037_dynamic_merges_2_horizontal_wide_merge_tree.sql b/tests/queries/0_stateless/03037_dynamic_merges_2_horizontal_wide_merge_tree.sql
index 011d54d2360..90dbc2d84f5 100644
--- a/tests/queries/0_stateless/03037_dynamic_merges_2_horizontal_wide_merge_tree.sql
+++ b/tests/queries/0_stateless/03037_dynamic_merges_2_horizontal_wide_merge_tree.sql
@@ -3,7 +3,7 @@
 set allow_experimental_dynamic_type = 1;
 
 drop table if exists test;
-create table test (id UInt64, d Dynamic) engine=MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1;
+create table test (id UInt64, d Dynamic) engine=MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1, lock_acquire_timeout_for_background_operations=600;
 system stop merges test;
 insert into test select number, number from numbers(1000000);
 insert into test select number, 'str_' || toString(number) from numbers(1000000, 1000000);
diff --git a/tests/queries/0_stateless/03037_dynamic_merges_2_vertical_compact_merge_tree.sql b/tests/queries/0_stateless/03037_dynamic_merges_2_vertical_compact_merge_tree.sql
index 1a74f9e5417..ffd2618ee51 100644
--- a/tests/queries/0_stateless/03037_dynamic_merges_2_vertical_compact_merge_tree.sql
+++ b/tests/queries/0_stateless/03037_dynamic_merges_2_vertical_compact_merge_tree.sql
@@ -3,7 +3,7 @@
 set allow_experimental_dynamic_type = 1;
 
 drop table if exists test;
-create table test (id UInt64, d Dynamic) engine=MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1;
+create table test (id UInt64, d Dynamic) engine=MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1, lock_acquire_timeout_for_background_operations=600;
 system stop merges test;
 insert into test select number, number from numbers(1000000);
 insert into test select number, 'str_' || toString(number) from numbers(1000000, 1000000);
diff --git a/tests/queries/0_stateless/03037_dynamic_merges_2_vertical_wide_merge_tree.sql b/tests/queries/0_stateless/03037_dynamic_merges_2_vertical_wide_merge_tree.sql
index cbc834e9660..36dff88751b 100644
--- a/tests/queries/0_stateless/03037_dynamic_merges_2_vertical_wide_merge_tree.sql
+++ b/tests/queries/0_stateless/03037_dynamic_merges_2_vertical_wide_merge_tree.sql
@@ -3,7 +3,7 @@
 set allow_experimental_dynamic_type = 1;
 
 drop table if exists test;
-create table test (id UInt64, d Dynamic) engine=MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1;
+create table test (id UInt64, d Dynamic) engine=MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1, lock_acquire_timeout_for_background_operations=600;
 system stop merges test;
 insert into test select number, number from numbers(1000000);
 insert into test select number, 'str_' || toString(number) from numbers(1000000, 1000000);
diff --git a/tests/queries/0_stateless/03038_nested_dynamic_merges_compact_horizontal.sql b/tests/queries/0_stateless/03038_nested_dynamic_merges_compact_horizontal.sql
index ff1dc5e7ded..1d5c63dcdf1 100644
--- a/tests/queries/0_stateless/03038_nested_dynamic_merges_compact_horizontal.sql
+++ b/tests/queries/0_stateless/03038_nested_dynamic_merges_compact_horizontal.sql
@@ -6,7 +6,7 @@ set allow_experimental_dynamic_type = 1;
 set enable_named_columns_in_function_tuple = 0;
 
 drop table if exists test;;
-create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000;;
+create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000, lock_acquire_timeout_for_background_operations=600;
 
 system stop merges test;
 insert into test select number, number from numbers(100000);
diff --git a/tests/queries/0_stateless/03038_nested_dynamic_merges_compact_vertical.sql b/tests/queries/0_stateless/03038_nested_dynamic_merges_compact_vertical.sql
index f9b0101cb87..2bffe35c577 100644
--- a/tests/queries/0_stateless/03038_nested_dynamic_merges_compact_vertical.sql
+++ b/tests/queries/0_stateless/03038_nested_dynamic_merges_compact_vertical.sql
@@ -6,7 +6,7 @@ set allow_experimental_dynamic_type = 1;
 set enable_named_columns_in_function_tuple = 0;
 
 drop table if exists test;;
-create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1;
+create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1, lock_acquire_timeout_for_background_operations=600;
 
 system stop merges test;
 insert into test select number, number from numbers(100000);
diff --git a/tests/queries/0_stateless/03038_nested_dynamic_merges_wide_horizontal.sql b/tests/queries/0_stateless/03038_nested_dynamic_merges_wide_horizontal.sql
index 5f373d41c7d..fb686091ebb 100644
--- a/tests/queries/0_stateless/03038_nested_dynamic_merges_wide_horizontal.sql
+++ b/tests/queries/0_stateless/03038_nested_dynamic_merges_wide_horizontal.sql
@@ -6,7 +6,7 @@ set allow_experimental_dynamic_type = 1;
 set enable_named_columns_in_function_tuple = 0;
 
 drop table if exists test;;
-create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1;
+create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1, lock_acquire_timeout_for_background_operations=600;
 
 system stop merges test;
 insert into test select number, number from numbers(100000);
diff --git a/tests/queries/0_stateless/03038_nested_dynamic_merges_wide_vertical.sql b/tests/queries/0_stateless/03038_nested_dynamic_merges_wide_vertical.sql
index 36bbc76b8cb..ed195452d56 100644
--- a/tests/queries/0_stateless/03038_nested_dynamic_merges_wide_vertical.sql
+++ b/tests/queries/0_stateless/03038_nested_dynamic_merges_wide_vertical.sql
@@ -6,7 +6,7 @@ set allow_experimental_dynamic_type = 1;
 set enable_named_columns_in_function_tuple = 0;
 
 drop table if exists test;;
-create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1;
+create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1, lock_acquire_timeout_for_background_operations=600;
 
 system stop merges test;
 insert into test select number, number from numbers(100000);

From e0b125368855e57733132046de5cd383ccc9b7d2 Mon Sep 17 00:00:00 2001
From: Konstantin Bogdanov <konstantin@clickhouse.com>
Date: Thu, 25 Jul 2024 15:25:33 +0200
Subject: [PATCH 0777/1488] Fix harder

---
 src/AggregateFunctions/SingleValueData.cpp | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/AggregateFunctions/SingleValueData.cpp b/src/AggregateFunctions/SingleValueData.cpp
index 996e64b22e0..566b40253a3 100644
--- a/src/AggregateFunctions/SingleValueData.cpp
+++ b/src/AggregateFunctions/SingleValueData.cpp
@@ -195,7 +195,7 @@ bool SingleValueDataFixed<T>::isEqualTo(const IColumn & column, size_t index) co
 template <typename T>
 bool SingleValueDataFixed<T>::isEqualTo(const SingleValueDataFixed<T> & to) const
 {
-    return has() && to.value == value;
+    return has() && to.has() && to.value == value;
 }
 
 template <typename T>
@@ -905,7 +905,7 @@ template <typename T>
 bool SingleValueDataNumeric<T>::isEqualTo(const DB::SingleValueDataBase & to) const
 {
     auto const & other = assert_cast<const Self &>(to);
-    return memory.get().isEqualTo(other.memory.get());
+    return to.has() && memory.get().isEqualTo(other.memory.get());
 }
 
 template <typename T>
@@ -1291,7 +1291,7 @@ bool SingleValueDataGeneric::isEqualTo(const IColumn & column, size_t row_num) c
 bool SingleValueDataGeneric::isEqualTo(const DB::SingleValueDataBase & other) const
 {
     auto const & to = assert_cast<const Self &>(other);
-    return has() && to.value == value;
+    return has() && to.has() && to.value == value;
 }
 
 void SingleValueDataGeneric::set(const IColumn & column, size_t row_num, Arena *)

From 64eeece5331c36c4f2b1c00c2bc229e40cd74d63 Mon Sep 17 00:00:00 2001
From: Alexander Tokmakov <tavplubix@clickhouse.com>
Date: Thu, 25 Jul 2024 15:49:59 +0200
Subject: [PATCH 0778/1488] fix

---
 tests/queries/0_stateless/02446_parent_zero_copy_locks.sql | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/queries/0_stateless/02446_parent_zero_copy_locks.sql b/tests/queries/0_stateless/02446_parent_zero_copy_locks.sql
index 1cae8ae0237..a44322e02cf 100644
--- a/tests/queries/0_stateless/02446_parent_zero_copy_locks.sql
+++ b/tests/queries/0_stateless/02446_parent_zero_copy_locks.sql
@@ -36,8 +36,8 @@ select 1, * from rmt1 order by n;
 system sync replica rmt1;
 select 2, * from rmt2 order by n;
 
--- wait for outdated parts to be removed
-select throwIf(count() = 0) from (
+-- wait for outdated parts to be removed (do not ignore _state column, so it will count Deleting parts as well)
+select throwIf(count() = 0), groupArray(_state) from (
 select *, _state from system.parts where database=currentDatabase() and table like 'rmt%' and active=0
 ) format Null; -- { retry 30 until serverError FUNCTION_THROW_IF_VALUE_IS_NON_ZERO }
 

From f654db215ffb961010763c0daf6484fa75e4fd6b Mon Sep 17 00:00:00 2001
From: Konstantin Bogdanov <konstantin@clickhouse.com>
Date: Thu, 25 Jul 2024 15:56:29 +0200
Subject: [PATCH 0779/1488] Fix naming

---
 ....reference => 03210_fix_single_value_data_assertion.reference} | 0
 ...ta-assertion.sql => 03210_fix_single_value_data_assertion.sql} | 0
 2 files changed, 0 insertions(+), 0 deletions(-)
 rename tests/queries/0_stateless/{03210_fix-single-value-data-assertion.reference => 03210_fix_single_value_data_assertion.reference} (100%)
 rename tests/queries/0_stateless/{03210_fix-single-value-data-assertion.sql => 03210_fix_single_value_data_assertion.sql} (100%)

diff --git a/tests/queries/0_stateless/03210_fix-single-value-data-assertion.reference b/tests/queries/0_stateless/03210_fix_single_value_data_assertion.reference
similarity index 100%
rename from tests/queries/0_stateless/03210_fix-single-value-data-assertion.reference
rename to tests/queries/0_stateless/03210_fix_single_value_data_assertion.reference
diff --git a/tests/queries/0_stateless/03210_fix-single-value-data-assertion.sql b/tests/queries/0_stateless/03210_fix_single_value_data_assertion.sql
similarity index 100%
rename from tests/queries/0_stateless/03210_fix-single-value-data-assertion.sql
rename to tests/queries/0_stateless/03210_fix_single_value_data_assertion.sql

From 4b505badd3566cf2b47681c667ee134699cf2764 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Thu, 25 Jul 2024 14:11:41 +0000
Subject: [PATCH 0780/1488] Attempt to fix flakiness of some window view tests

---
 .../queries/0_stateless/01052_window_view_proc_tumble_to_now.sh  | 1 +
 tests/queries/0_stateless/01053_window_view_proc_hop_to_now.sh   | 1 +
 tests/queries/0_stateless/01054_window_view_proc_tumble_to.sh    | 1 +
 tests/queries/0_stateless/01055_window_view_proc_hop_to.sh       | 1 +
 .../0_stateless/01075_window_view_proc_tumble_to_now_populate.sh | 1 +
 5 files changed, 5 insertions(+)

diff --git a/tests/queries/0_stateless/01052_window_view_proc_tumble_to_now.sh b/tests/queries/0_stateless/01052_window_view_proc_tumble_to_now.sh
index 4325ebeed24..5c70806ea7b 100755
--- a/tests/queries/0_stateless/01052_window_view_proc_tumble_to_now.sh
+++ b/tests/queries/0_stateless/01052_window_view_proc_tumble_to_now.sh
@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
+# Tags: no-random-settings, no-parallel
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
diff --git a/tests/queries/0_stateless/01053_window_view_proc_hop_to_now.sh b/tests/queries/0_stateless/01053_window_view_proc_hop_to_now.sh
index 8e28995980f..32c9c52ab09 100755
--- a/tests/queries/0_stateless/01053_window_view_proc_hop_to_now.sh
+++ b/tests/queries/0_stateless/01053_window_view_proc_hop_to_now.sh
@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
+# Tags: no-random-settings, no-parallel
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
diff --git a/tests/queries/0_stateless/01054_window_view_proc_tumble_to.sh b/tests/queries/0_stateless/01054_window_view_proc_tumble_to.sh
index ee11b265ecd..ba566bb4ae6 100755
--- a/tests/queries/0_stateless/01054_window_view_proc_tumble_to.sh
+++ b/tests/queries/0_stateless/01054_window_view_proc_tumble_to.sh
@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
+# Tags: no-random-settings, no-parallel
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
diff --git a/tests/queries/0_stateless/01055_window_view_proc_hop_to.sh b/tests/queries/0_stateless/01055_window_view_proc_hop_to.sh
index ea8ad372617..0db4173b3dc 100755
--- a/tests/queries/0_stateless/01055_window_view_proc_hop_to.sh
+++ b/tests/queries/0_stateless/01055_window_view_proc_hop_to.sh
@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
+# Tags: no-random-settings, no-parallel
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
diff --git a/tests/queries/0_stateless/01075_window_view_proc_tumble_to_now_populate.sh b/tests/queries/0_stateless/01075_window_view_proc_tumble_to_now_populate.sh
index f7842af4dad..67c249a9d0e 100755
--- a/tests/queries/0_stateless/01075_window_view_proc_tumble_to_now_populate.sh
+++ b/tests/queries/0_stateless/01075_window_view_proc_tumble_to_now_populate.sh
@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
+# Tags: no-random-settings, no-parallel
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh

From baee31c12a68cd6e8f906a9224a39eea446a8f2b Mon Sep 17 00:00:00 2001
From: Alexander Tokmakov <tavplubix@clickhouse.com>
Date: Thu, 25 Jul 2024 16:15:37 +0200
Subject: [PATCH 0781/1488] fix truncate database

---
 src/Interpreters/InterpreterDropQuery.cpp                   | 3 +--
 tests/queries/0_stateless/02842_truncate_database.reference | 2 ++
 tests/queries/0_stateless/02842_truncate_database.sql       | 4 ++++
 3 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/Interpreters/InterpreterDropQuery.cpp b/src/Interpreters/InterpreterDropQuery.cpp
index b68b3ddcd48..bad3e5277db 100644
--- a/src/Interpreters/InterpreterDropQuery.cpp
+++ b/src/Interpreters/InterpreterDropQuery.cpp
@@ -399,10 +399,9 @@ BlockIO InterpreterDropQuery::executeToDatabaseImpl(const ASTDropQuery & query,
             if (query.if_empty)
                 throw Exception(ErrorCodes::NOT_IMPLEMENTED, "DROP IF EMPTY is not implemented for databases");
 
-            if (database->hasReplicationThread())
+            if (!truncate && database->hasReplicationThread())
                 database->stopReplication();
 
-
             if (database->shouldBeEmptyOnDetach())
             {
                 /// Cancel restarting replicas in that database, wait for remaining RESTART queries to finish.
diff --git a/tests/queries/0_stateless/02842_truncate_database.reference b/tests/queries/0_stateless/02842_truncate_database.reference
index 71f52bcd1da..bc8c0210d27 100644
--- a/tests/queries/0_stateless/02842_truncate_database.reference
+++ b/tests/queries/0_stateless/02842_truncate_database.reference
@@ -20,3 +20,5 @@ source_table_stripe_log
 source_table_tiny_log
 === DICTIONARIES IN test_truncate_database ===
 dest_dictionary
+new tables
+new_table
diff --git a/tests/queries/0_stateless/02842_truncate_database.sql b/tests/queries/0_stateless/02842_truncate_database.sql
index 09ac844cfe2..be92108ccb8 100644
--- a/tests/queries/0_stateless/02842_truncate_database.sql
+++ b/tests/queries/0_stateless/02842_truncate_database.sql
@@ -73,4 +73,8 @@ SELECT * FROM dest_dictionary; -- {serverError UNKNOWN_TABLE}
 SHOW TABLES FROM test_truncate_database;
 SHOW DICTIONARIES FROM test_truncate_database;
 
+CREATE TABLE new_table (x UInt16) ENGINE = ReplicatedMergeTree ORDER BY x;
+select 'new tables';
+SHOW TABLES FROM test_truncate_database;
+
 DROP DATABASE test_truncate_database;

From 0642ed19b7c67e443be110f2a0f2d1f032ddd8d5 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Thu, 25 Jul 2024 14:17:45 +0000
Subject: [PATCH 0782/1488] Fixing more tests.

---
 src/Processors/QueryPlan/ReadFromMergeTree.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp
index bdb90abd326..e5aeb9686be 100644
--- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp
+++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp
@@ -1525,7 +1525,7 @@ void ReadFromMergeTree::applyFilters(ActionDAGNodes added_filter_nodes)
         /// TODO: Get rid of filter_actions_dag in query_info after we move analysis of
         /// parallel replicas and unused shards into optimization, similar to projection analysis.
         if (filter_actions_dag)
-            query_info.filter_actions_dag = std::make_shared<const ActionsDAG>(std::move(*filter_actions_dag));
+            query_info.filter_actions_dag = std::make_shared<const ActionsDAG>(filter_actions_dag->clone());
 
         buildIndexes(
             indexes,

From bd721950b0401f94be652c11015bd1985c283f3a Mon Sep 17 00:00:00 2001
From: maxvostrikov <max.vostrikov@clickhouse.com>
Date: Thu, 25 Jul 2024 16:24:17 +0200
Subject: [PATCH 0783/1488] squash! added somme tests in relation with
 https://github.com/ClickHouse/ClickHouse/pull/54881 with new behaviour when
 enable_named_columns_in_function_tuple=1 (default value)

---
 .../0_stateless/00309_formats.reference       | Bin 18736 -> 18666 bytes
 tests/queries/0_stateless/00309_formats.sql   |   4 ++--
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/queries/0_stateless/00309_formats.reference b/tests/queries/0_stateless/00309_formats.reference
index a63720618ba54c6cc456f3356512449322dc2e80..f3ea45520bb50fb936caf6724e9fedf3cdd00b75 100644
GIT binary patch
delta 54
zcmdlmiSgA$#tkJN8myt7c_kJsE{P?nVj-mkIjI_X3Sc1}g%kylsHVzCEg+hFz{3v!
D3NRA?

delta 99
zcmaDgk#WN$#tkJNdTgPdc_n5hEG~&9sp27}1v#l2c?u9Q9fcGHkgz5f0|Z>xnrz4=
o!o|P{;hxo=JjcUt@=alWVGScwGaYp!0|NtH14~^)V|7g~0E0RhwEzGB

diff --git a/tests/queries/0_stateless/00309_formats.sql b/tests/queries/0_stateless/00309_formats.sql
index 691fc6e7ab6..0366cdeea5c 100644
--- a/tests/queries/0_stateless/00309_formats.sql
+++ b/tests/queries/0_stateless/00309_formats.sql
@@ -12,5 +12,5 @@ SELECT number * 246 + 10 AS n, toDate('2000-01-01') + n AS d, range(n) AS arr, a
 
 SET enable_named_columns_in_function_tuple = 1;
 
-SELECT number * 246 + 10 AS n, toDate('2000-01-01') + n AS d, (n, d) AS tuple FROM system.numbers LIMIT 2 FORMAT RowBinaryWithNamesAndTypes;
-SELECT number * 246 + 10 AS n, toDate('2000-01-01') + n AS d, (n, d) AS tuple FROM system.numbers LIMIT 2 FORMAT TabSeparatedWithNamesAndTypes;
+SELECT 36 AS n, toDate('2000-01-01') + n AS d, (n, d) AS tuple FROM system.numbers LIMIT 1 FORMAT RowBinaryWithNamesAndTypes;
+SELECT number * 246 + 10 AS n, toDate('2000-01-01') + n AS d, (n, d) AS tuple FROM system.numbers LIMIT 1 FORMAT TabSeparatedWithNamesAndTypes;

From 59f9c125044b6e56a3ded8034478eff79e930018 Mon Sep 17 00:00:00 2001
From: Pablo Marcos <pablo.marcos.oltra@clickhouse.com>
Date: Thu, 25 Jul 2024 14:37:47 +0000
Subject: [PATCH 0784/1488] Increase sleep time make sure there is a new
 failure

The previous sleep was already adding +5s to make sure
the TTL was properly applied, so we'd rather use the
same value here instead of just 1s.
---
 tests/integration/test_storage_s3_queue/test.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/integration/test_storage_s3_queue/test.py b/tests/integration/test_storage_s3_queue/test.py
index 4348857acd3..2e339a9b5c9 100644
--- a/tests/integration/test_storage_s3_queue/test.py
+++ b/tests/integration/test_storage_s3_queue/test.py
@@ -871,7 +871,7 @@ def test_max_set_age(started_cluster):
         node.query(f"SELECT uniq(_path) from {dst_table_name}")
     )
 
-    time.sleep(max_age + 5)
+    time.sleep(max_age + max_age / 2)
 
     expected_rows *= 2
     wait_for_condition(lambda: get_count() == expected_rows)
@@ -922,7 +922,7 @@ def test_max_set_age(started_cluster):
         )
     )
 
-    time.sleep(max_age + 1)
+    time.sleep(max_age + max_age / 2)
 
     assert failed_count + 2 <= get_object_storage_failures()
 

From 1973458ae07a5cd519b7069451d2be5822a89bf7 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Thu, 25 Jul 2024 15:00:37 +0000
Subject: [PATCH 0785/1488] Update PlannerWindowFunctions

---
 src/Planner/Planner.cpp                | 13 ++++++-------
 src/Planner/PlannerWindowFunctions.cpp | 15 ++-------------
 src/Planner/PlannerWindowFunctions.h   |  2 +-
 3 files changed, 9 insertions(+), 21 deletions(-)

diff --git a/src/Planner/Planner.cpp b/src/Planner/Planner.cpp
index fb721069e6e..968642dc9de 100644
--- a/src/Planner/Planner.cpp
+++ b/src/Planner/Planner.cpp
@@ -933,19 +933,19 @@ void addPreliminarySortOrDistinctOrLimitStepsIfNeeded(QueryPlan & query_plan,
 
 void addWindowSteps(QueryPlan & query_plan,
     const PlannerContextPtr & planner_context,
-    const WindowAnalysisResult & window_analysis_result)
+    WindowAnalysisResult & window_analysis_result)
 {
     const auto & query_context = planner_context->getQueryContext();
     const auto & settings = query_context->getSettingsRef();
 
-    const auto & window_descriptions = window_analysis_result.window_descriptions;
-    auto perm = sortWindowDescriptions(window_descriptions);
+    auto & window_descriptions = window_analysis_result.window_descriptions;
+    sortWindowDescriptions(window_descriptions);
 
     size_t window_descriptions_size = window_descriptions.size();
 
     for (size_t i = 0; i < window_descriptions_size; ++i)
     {
-        const auto & window_description = window_descriptions[perm[i]];
+        const auto & window_description = window_descriptions[i];
 
         /** We don't need to sort again if the input from previous window already
           * has suitable sorting. Also don't create sort steps when there are no
@@ -958,9 +958,8 @@ void addWindowSteps(QueryPlan & query_plan,
         bool need_sort = !window_description.full_sort_description.empty();
         if (need_sort && i != 0)
         {
-            auto prev = perm[i - 1];
-            need_sort = !sortDescriptionIsPrefix(window_description.full_sort_description, window_descriptions[prev].full_sort_description)
-                        || (settings.max_threads != 1 && window_description.partition_by.size() != window_descriptions[prev].partition_by.size());
+            need_sort = !sortDescriptionIsPrefix(window_description.full_sort_description, window_descriptions[i - 1].full_sort_description)
+                        || (settings.max_threads != 1 && window_description.partition_by.size() != window_descriptions[i - 1].partition_by.size());
         }
         if (need_sort)
         {
diff --git a/src/Planner/PlannerWindowFunctions.cpp b/src/Planner/PlannerWindowFunctions.cpp
index ba0e11df76b..225852de5a7 100644
--- a/src/Planner/PlannerWindowFunctions.cpp
+++ b/src/Planner/PlannerWindowFunctions.cpp
@@ -122,7 +122,7 @@ std::vector<WindowDescription> extractWindowDescriptions(const QueryTreeNodes &
     return result;
 }
 
-std::vector<size_t> sortWindowDescriptions(const std::vector<WindowDescription> & window_descriptions)
+void sortWindowDescriptions(std::vector<WindowDescription> & window_descriptions)
 {
     auto window_description_comparator = [](const WindowDescription & lhs, const WindowDescription & rhs)
     {
@@ -153,18 +153,7 @@ std::vector<size_t> sortWindowDescriptions(const std::vector<WindowDescription>
         return left.size() > right.size();
     };
 
-    auto comparator = [&](size_t lhs, size_t rhs)
-    {
-        return window_description_comparator(window_descriptions[lhs], window_descriptions[rhs]);
-    };
-
-    std::vector<size_t> perm(window_descriptions.size());
-    for (size_t i = 0; i < perm.size(); ++i)
-        perm[i] = i;
-
-    ::sort(perm.begin(), perm.end(), comparator);
-
-    return perm;
+    ::sort(window_descriptions.begin(), window_descriptions.end(), window_description_comparator);
 }
 
 }
diff --git a/src/Planner/PlannerWindowFunctions.h b/src/Planner/PlannerWindowFunctions.h
index 3039ecefc4b..1552ef5a71f 100644
--- a/src/Planner/PlannerWindowFunctions.h
+++ b/src/Planner/PlannerWindowFunctions.h
@@ -15,6 +15,6 @@ std::vector<WindowDescription> extractWindowDescriptions(const QueryTreeNodes &
 /** Try to sort window descriptions in such an order that the window with the longest
   * sort description goes first, and all window that use its prefixes follow.
   */
-std::vector<size_t> sortWindowDescriptions(const std::vector<WindowDescription> & window_descriptions);
+void sortWindowDescriptions(std::vector<WindowDescription> & window_descriptions);
 
 }

From 93a2dbf85c446cd5efa802dfe8fc99e2a82f394f Mon Sep 17 00:00:00 2001
From: Dmitry Novik <mrnovikd@gmail.com>
Date: Thu, 25 Jul 2024 17:02:11 +0200
Subject: [PATCH 0786/1488] Move syntax part on top

---
 docs/en/sql-reference/statements/system.md | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/docs/en/sql-reference/statements/system.md b/docs/en/sql-reference/statements/system.md
index f4780fd41c1..35f2f15dd80 100644
--- a/docs/en/sql-reference/statements/system.md
+++ b/docs/en/sql-reference/statements/system.md
@@ -28,11 +28,14 @@ SYSTEM RELOAD DICTIONARIES [ON CLUSTER cluster_name]
 
 Completely reloads a dictionary `dictionary_name`, regardless of the state of the dictionary (LOADED / NOT_LOADED / FAILED).
 Always returns `Ok.` regardless of the result of updating the dictionary.
-The status of the dictionary can be checked by querying the `system.dictionaries` table.
 
 ``` sql
 SYSTEM RELOAD DICTIONARY [ON CLUSTER cluster_name] dictionary_name
+```
 
+The status of the dictionary can be checked by querying the `system.dictionaries` table.
+
+``` sql
 SELECT name, status FROM system.dictionaries;
 ```
 

From 5ea867231bafc01b4512989f351106b7afcc14af Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Thu, 25 Jul 2024 16:55:23 +0200
Subject: [PATCH 0787/1488] Read configuration for clickhouse-local from
 ~/.clickhouse-local

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 programs/local/LocalServer.cpp           | 16 +++++++--
 src/Common/Config/CMakeLists.txt         |  1 +
 src/Common/Config/getLocalConfigPath.cpp | 46 ++++++++++++++++++++++++
 src/Common/Config/getLocalConfigPath.h   | 12 +++++++
 4 files changed, 73 insertions(+), 2 deletions(-)
 create mode 100644 src/Common/Config/getLocalConfigPath.cpp
 create mode 100644 src/Common/Config/getLocalConfigPath.h

diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp
index 48e0cca7b73..ade4e0f49df 100644
--- a/programs/local/LocalServer.cpp
+++ b/programs/local/LocalServer.cpp
@@ -1,6 +1,7 @@
 #include "LocalServer.h"
 
 #include <sys/resource.h>
+#include <Common/Config/getLocalConfigPath.h>
 #include <Common/logger_useful.h>
 #include <Common/formatReadable.h>
 #include <Core/UUID.h>
@@ -127,10 +128,21 @@ void LocalServer::initialize(Poco::Util::Application & self)
 {
     Poco::Util::Application::initialize(self);
 
+    const char * home_path_cstr = getenv("HOME"); // NOLINT(concurrency-mt-unsafe)
+    if (home_path_cstr)
+        home_path = home_path_cstr;
+
     /// Load config files if exists
-    if (getClientConfiguration().has("config-file") || fs::exists("config.xml"))
+    std::string config_path;
+    if (getClientConfiguration().has("config-file"))
+        config_path = getClientConfiguration().getString("config-file");
+    else if (config_path.empty() && fs::exists("config.xml"))
+        config_path = "config.xml";
+    else if (config_path.empty())
+        config_path = getLocalConfigPath(home_path).value_or("");
+
+    if (fs::exists(config_path))
     {
-        const auto config_path = getClientConfiguration().getString("config-file", "config.xml");
         ConfigProcessor config_processor(config_path, false, true);
         ConfigProcessor::setConfigPath(fs::path(config_path).parent_path());
         auto loaded_config = config_processor.loadConfig();
diff --git a/src/Common/Config/CMakeLists.txt b/src/Common/Config/CMakeLists.txt
index 09095ef5acc..2bd32b98bda 100644
--- a/src/Common/Config/CMakeLists.txt
+++ b/src/Common/Config/CMakeLists.txt
@@ -2,6 +2,7 @@ set (SRCS
     AbstractConfigurationComparison.cpp
     ConfigProcessor.cpp
     getClientConfigPath.cpp
+    getLocalConfigPath.cpp
     ConfigReloader.cpp
     YAMLParser.cpp
     ConfigHelper.cpp
diff --git a/src/Common/Config/getLocalConfigPath.cpp b/src/Common/Config/getLocalConfigPath.cpp
new file mode 100644
index 00000000000..afaa7f79026
--- /dev/null
+++ b/src/Common/Config/getLocalConfigPath.cpp
@@ -0,0 +1,46 @@
+#include <Common/Config/getLocalConfigPath.h>
+
+#include <filesystem>
+#include <vector>
+
+
+namespace fs = std::filesystem;
+
+namespace DB
+{
+
+std::optional<std::string> getLocalConfigPath(const std::string & home_path)
+{
+    std::string config_path;
+    bool found = false;
+
+    std::vector<std::string> names;
+    names.emplace_back("./clickhouse-local");
+    if (!home_path.empty())
+        names.emplace_back(home_path + "/.clickhouse-local/config");
+    names.emplace_back("/etc/clickhouse-local/config");
+
+    for (const auto & name : names)
+    {
+        for (const auto & extension : {".xml", ".yaml", ".yml"})
+        {
+            config_path = name + extension;
+
+            std::error_code ec;
+            if (fs::exists(config_path, ec))
+            {
+                found = true;
+                break;
+            }
+        }
+        if (found)
+            break;
+    }
+
+    if (found)
+        return config_path;
+
+    return std::nullopt;
+}
+
+}
diff --git a/src/Common/Config/getLocalConfigPath.h b/src/Common/Config/getLocalConfigPath.h
new file mode 100644
index 00000000000..14625571d6c
--- /dev/null
+++ b/src/Common/Config/getLocalConfigPath.h
@@ -0,0 +1,12 @@
+#pragma once
+
+#include <string>
+#include <optional>
+
+namespace DB
+{
+
+/// Return path to existing configuration file.
+std::optional<std::string> getLocalConfigPath(const std::string & home_path);
+
+}

From fb271436a1efe969f4de09b14aec942baa145cb9 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Thu, 25 Jul 2024 15:37:13 +0000
Subject: [PATCH 0788/1488] Remove ActionsDAGPtr completely.

---
 src/Interpreters/ActionsDAG.h                      |  3 ---
 src/Interpreters/evaluateConstantExpression.cpp    |  4 ++--
 .../optimizePrimaryKeyConditionAndLimit.cpp        |  6 +++---
 src/Processors/QueryPlan/SourceStepWithFilter.h    |  6 +++---
 src/Processors/QueryPlan/TotalsHavingStep.h        |  3 ---
 src/Processors/QueryPlan/WindowStep.h              |  3 ---
 src/Storages/MergeTree/KeyCondition.cpp            | 14 +++++++-------
 src/Storages/MergeTree/KeyCondition.h              |  2 +-
 .../MergeTreeSplitPrewhereIntoReadSteps.cpp        |  3 +++
 src/Storages/SelectQueryInfo.h                     |  3 ---
 src/Storages/StorageMerge.cpp                      |  2 +-
 11 files changed, 20 insertions(+), 29 deletions(-)

diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h
index 76cc9327530..43c1b41a240 100644
--- a/src/Interpreters/ActionsDAG.h
+++ b/src/Interpreters/ActionsDAG.h
@@ -11,9 +11,6 @@
 namespace DB
 {
 
-class ActionsDAG;
-using ActionsDAGPtr = std::unique_ptr<ActionsDAG>;
-
 class IExecutableFunction;
 using ExecutableFunctionPtr = std::shared_ptr<IExecutableFunction>;
 
diff --git a/src/Interpreters/evaluateConstantExpression.cpp b/src/Interpreters/evaluateConstantExpression.cpp
index 42d6f3d3037..4bfc80af1fe 100644
--- a/src/Interpreters/evaluateConstantExpression.cpp
+++ b/src/Interpreters/evaluateConstantExpression.cpp
@@ -679,9 +679,9 @@ std::optional<ConstantVariants> evaluateExpressionOverConstantCondition(
     size_t max_elements)
 {
     auto inverted_dag = KeyCondition::cloneASTWithInversionPushDown({predicate}, context);
-    auto matches = matchTrees(expr, *inverted_dag, false);
+    auto matches = matchTrees(expr, inverted_dag, false);
 
-    auto predicates = analyze(inverted_dag->getOutputs().at(0), matches, context, max_elements);
+    auto predicates = analyze(inverted_dag.getOutputs().at(0), matches, context, max_elements);
 
     if (!predicates)
         return {};
diff --git a/src/Processors/QueryPlan/Optimizations/optimizePrimaryKeyConditionAndLimit.cpp b/src/Processors/QueryPlan/Optimizations/optimizePrimaryKeyConditionAndLimit.cpp
index 63b4e019066..f53212407d2 100644
--- a/src/Processors/QueryPlan/Optimizations/optimizePrimaryKeyConditionAndLimit.cpp
+++ b/src/Processors/QueryPlan/Optimizations/optimizePrimaryKeyConditionAndLimit.cpp
@@ -18,16 +18,16 @@ void optimizePrimaryKeyConditionAndLimit(const Stack & stack)
     const auto & storage_prewhere_info = source_step_with_filter->getPrewhereInfo();
     if (storage_prewhere_info)
     {
-        source_step_with_filter->addFilter(std::make_unique<ActionsDAG>(storage_prewhere_info->prewhere_actions->clone()), storage_prewhere_info->prewhere_column_name);
+        source_step_with_filter->addFilter(storage_prewhere_info->prewhere_actions->clone(), storage_prewhere_info->prewhere_column_name);
         if (storage_prewhere_info->row_level_filter)
-            source_step_with_filter->addFilter(std::make_unique<ActionsDAG>(storage_prewhere_info->row_level_filter->clone()), storage_prewhere_info->row_level_column_name);
+            source_step_with_filter->addFilter(storage_prewhere_info->row_level_filter->clone(), storage_prewhere_info->row_level_column_name);
     }
 
     for (auto iter = stack.rbegin() + 1; iter != stack.rend(); ++iter)
     {
         if (auto * filter_step = typeid_cast<FilterStep *>(iter->node->step.get()))
         {
-            source_step_with_filter->addFilter(std::make_unique<ActionsDAG>(filter_step->getExpression().clone()), filter_step->getFilterColumnName());
+            source_step_with_filter->addFilter(filter_step->getExpression().clone(), filter_step->getFilterColumnName());
         }
         else if (auto * limit_step = typeid_cast<LimitStep *>(iter->node->step.get()))
         {
diff --git a/src/Processors/QueryPlan/SourceStepWithFilter.h b/src/Processors/QueryPlan/SourceStepWithFilter.h
index f7a030c0628..6cea5fd7245 100644
--- a/src/Processors/QueryPlan/SourceStepWithFilter.h
+++ b/src/Processors/QueryPlan/SourceStepWithFilter.h
@@ -45,9 +45,9 @@ public:
 
     const Names & requiredSourceColumns() const { return required_source_columns; }
 
-    void addFilter(ActionsDAGPtr filter_dag, std::string column_name)
+    void addFilter(ActionsDAG filter_dag, std::string column_name)
     {
-        filter_nodes.nodes.push_back(&filter_dag->findInOutputs(column_name));
+        filter_nodes.nodes.push_back(&filter_dag.findInOutputs(column_name));
         filter_dags.push_back(std::move(filter_dag));
     }
 
@@ -86,7 +86,7 @@ protected:
 private:
     /// Will be cleared after applyFilters() is called.
     ActionDAGNodes filter_nodes;
-    std::vector<ActionsDAGPtr> filter_dags;
+    std::vector<ActionsDAG> filter_dags;
 };
 
 }
diff --git a/src/Processors/QueryPlan/TotalsHavingStep.h b/src/Processors/QueryPlan/TotalsHavingStep.h
index 927b8d99de3..4b414d41c57 100644
--- a/src/Processors/QueryPlan/TotalsHavingStep.h
+++ b/src/Processors/QueryPlan/TotalsHavingStep.h
@@ -6,9 +6,6 @@
 namespace DB
 {
 
-class ActionsDAG;
-using ActionsDAGPtr = std::unique_ptr<ActionsDAG>;
-
 enum class TotalsMode : uint8_t;
 
 /// Execute HAVING and calculate totals. See TotalsHavingTransform.
diff --git a/src/Processors/QueryPlan/WindowStep.h b/src/Processors/QueryPlan/WindowStep.h
index 47883e5edf6..d79cd7fd45e 100644
--- a/src/Processors/QueryPlan/WindowStep.h
+++ b/src/Processors/QueryPlan/WindowStep.h
@@ -6,9 +6,6 @@
 namespace DB
 {
 
-class ActionsDAG;
-using ActionsDAGPtr = std::unique_ptr<ActionsDAG>;
-
 class WindowTransform;
 
 class WindowStep : public ITransformingStep
diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp
index 0eb59a47cae..69bffac9160 100644
--- a/src/Storages/MergeTree/KeyCondition.cpp
+++ b/src/Storages/MergeTree/KeyCondition.cpp
@@ -696,22 +696,22 @@ const std::unordered_map<String, KeyCondition::SpaceFillingCurveType> KeyConditi
     {"hilbertEncode", SpaceFillingCurveType::Hilbert}
 };
 
-ActionsDAGPtr KeyCondition::cloneASTWithInversionPushDown(ActionsDAG::NodeRawConstPtrs nodes, const ContextPtr & context)
+ActionsDAG KeyCondition::cloneASTWithInversionPushDown(ActionsDAG::NodeRawConstPtrs nodes, const ContextPtr & context)
 {
-    auto res = std::make_unique<ActionsDAG>();
+    ActionsDAG res;
 
     std::unordered_map<const ActionsDAG::Node *, const ActionsDAG::Node *> to_inverted;
 
     for (auto & node : nodes)
-        node = &DB::cloneASTWithInversionPushDown(*node, *res, to_inverted, context, false);
+        node = &DB::cloneASTWithInversionPushDown(*node, res, to_inverted, context, false);
 
     if (nodes.size() > 1)
     {
         auto function_builder = FunctionFactory::instance().get("and", context);
-        nodes = {&res->addFunction(function_builder, std::move(nodes), "")};
+        nodes = {&res.addFunction(function_builder, std::move(nodes), "")};
     }
 
-    res->getOutputs().swap(nodes);
+    res.getOutputs().swap(nodes);
 
     return res;
 }
@@ -826,9 +826,9 @@ KeyCondition::KeyCondition(
       * are pushed down and applied (when possible) to leaf nodes.
       */
     auto inverted_dag = cloneASTWithInversionPushDown({filter_dag->getOutputs().at(0)}, context);
-    assert(inverted_dag->getOutputs().size() == 1);
+    assert(inverted_dag.getOutputs().size() == 1);
 
-    const auto * inverted_dag_filter_node = inverted_dag->getOutputs()[0];
+    const auto * inverted_dag_filter_node = inverted_dag.getOutputs()[0];
 
     RPNBuilder<RPNElement> builder(inverted_dag_filter_node, context, [&](const RPNBuilderTreeNode & node, RPNElement & out)
     {
diff --git a/src/Storages/MergeTree/KeyCondition.h b/src/Storages/MergeTree/KeyCondition.h
index a9e1a589ba5..e9343ec08ea 100644
--- a/src/Storages/MergeTree/KeyCondition.h
+++ b/src/Storages/MergeTree/KeyCondition.h
@@ -134,7 +134,7 @@ public:
         DataTypePtr current_type,
         bool single_point = false);
 
-    static ActionsDAGPtr cloneASTWithInversionPushDown(ActionsDAG::NodeRawConstPtrs nodes, const ContextPtr & context);
+    static ActionsDAG cloneASTWithInversionPushDown(ActionsDAG::NodeRawConstPtrs nodes, const ContextPtr & context);
 
     bool matchesExactContinuousRange() const;
 
diff --git a/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp b/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp
index 116edf5b9cb..1d0569e0df6 100644
--- a/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp
+++ b/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp
@@ -10,6 +10,9 @@
 namespace DB
 {
 
+class ActionsDAG;
+using ActionsDAGPtr = std::unique_ptr<ActionsDAG>;
+
 namespace ErrorCodes
 {
     extern const int LOGICAL_ERROR;
diff --git a/src/Storages/SelectQueryInfo.h b/src/Storages/SelectQueryInfo.h
index 60f103fdb70..1c4cb7d92d8 100644
--- a/src/Storages/SelectQueryInfo.h
+++ b/src/Storages/SelectQueryInfo.h
@@ -18,9 +18,6 @@ namespace DB
 class ExpressionActions;
 using ExpressionActionsPtr = std::shared_ptr<ExpressionActions>;
 
-class ActionsDAG;
-using ActionsDAGPtr = std::unique_ptr<ActionsDAG>;
-
 struct PrewhereInfo;
 using PrewhereInfoPtr = std::shared_ptr<PrewhereInfo>;
 
diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp
index 0e1568c8e79..e5de15c1d21 100644
--- a/src/Storages/StorageMerge.cpp
+++ b/src/Storages/StorageMerge.cpp
@@ -1279,7 +1279,7 @@ void ReadFromMerge::RowPolicyData::extendNames(Names & names) const
 
 void ReadFromMerge::RowPolicyData::addStorageFilter(SourceStepWithFilter * step) const
 {
-    step->addFilter(std::make_unique<ActionsDAG>(actions_dag.clone()), filter_column_name);
+    step->addFilter(actions_dag.clone(), filter_column_name);
 }
 
 void ReadFromMerge::RowPolicyData::addFilterTransform(QueryPlan & plan) const

From ccd92d20821903123d4748027cc2248095b34efa Mon Sep 17 00:00:00 2001
From: kssenii <sumarokovakseniia@mail.ru>
Date: Thu, 25 Jul 2024 17:44:26 +0200
Subject: [PATCH 0789/1488] Update chassert in cache

---
 src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp
index 198f6c0ea04..c928d25c7b8 100644
--- a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp
+++ b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp
@@ -810,6 +810,7 @@ bool CachedOnDiskReadBufferFromFile::nextImplStep()
 {
     last_caller_id = FileSegment::getCallerId();
 
+    chassert(file_offset_of_buffer_end <= read_until_position);
     if (file_offset_of_buffer_end == read_until_position)
         return false;
 
@@ -1051,7 +1052,11 @@ bool CachedOnDiskReadBufferFromFile::nextImplStep()
 
         if (download_current_segment && download_current_segment_succeeded)
             chassert(file_segment.getCurrentWriteOffset() >= file_offset_of_buffer_end);
-        chassert(file_offset_of_buffer_end <= read_until_position);
+
+        chassert(
+            file_offset_of_buffer_end <= read_until_position,
+            fmt::format("Expected {} <= {} (size: {}, read range: {})",
+                        file_offset_of_buffer_end, read_until_position, size, current_read_range.toString()));
     }
 
     swap(*implementation_buffer);

From e199fbaeaadd05b28e9dee1265fc813b081071f1 Mon Sep 17 00:00:00 2001
From: Sema Checherinda <104093494+CheSema@users.noreply.github.com>
Date: Thu, 25 Jul 2024 17:55:32 +0200
Subject: [PATCH 0790/1488] Update
 tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh

---
 .../0_stateless/01171_mv_select_insert_isolation_long.sh        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh b/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh
index 620281ee972..13aa64d3cbe 100755
--- a/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh
+++ b/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh
@@ -168,7 +168,7 @@ fi
 START_TIME=$(get_now)
 STOP_TIME=$((START_TIME + MAIN_TIME_PART))
 SECOND_STOP_TIME=$((STOP_TIME + SECOND_TIME_PART))
-MIN_ITERATIONS=25
+MIN_ITERATIONS=20
 
 run_until_deadline_and_at_least_times $STOP_TIME $MIN_ITERATIONS insert_commit_action 1   & PID_1=$!
 run_until_deadline_and_at_least_times $STOP_TIME $MIN_ITERATIONS insert_commit_action 2   & PID_2=$!

From 869f6a6f105f50aa4d0e71e6440646b78539f0ff Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Thu, 25 Jul 2024 16:33:12 +0000
Subject: [PATCH 0791/1488] Updating PrewhereInfo

---
 src/Interpreters/ExpressionAnalyzer.cpp          |  9 ++++-----
 src/Interpreters/InterpreterSelectQuery.cpp      | 16 ++++++++--------
 src/Planner/PlannerJoinTree.cpp                  |  8 +++-----
 .../QueryPlan/Optimizations/optimizePrewhere.cpp |  8 ++++----
 .../optimizePrimaryKeyConditionAndLimit.cpp      |  2 +-
 .../Optimizations/optimizeReadInOrder.cpp        | 12 +++++-------
 .../Optimizations/projectionsCommon.cpp          | 15 ++++++---------
 src/Processors/QueryPlan/ReadFromMergeTree.cpp   | 12 ++++--------
 .../QueryPlan/SourceStepWithFilter.cpp           |  9 +++------
 src/Storages/IStorage.cpp                        |  3 +--
 .../MergeTree/MergeTreePrefetchedReadPool.cpp    |  2 +-
 src/Storages/MergeTree/MergeTreeReadPoolBase.cpp |  2 +-
 .../MergeTree/MergeTreeSelectProcessor.cpp       |  4 ++--
 .../MergeTreeSplitPrewhereIntoReadSteps.cpp      |  6 +++---
 src/Storages/SelectQueryInfo.h                   |  7 +++----
 src/Storages/StorageBuffer.cpp                   |  7 +++----
 16 files changed, 52 insertions(+), 70 deletions(-)

diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp
index 5972d89bddd..d25434a515d 100644
--- a/src/Interpreters/ExpressionAnalyzer.cpp
+++ b/src/Interpreters/ExpressionAnalyzer.cpp
@@ -2230,12 +2230,11 @@ void ExpressionAnalysisResult::checkActions() const
     /// Check that PREWHERE doesn't contain unusual actions. Unusual actions are that can change number of rows.
     if (hasPrewhere())
     {
-        auto check_actions = [](const std::optional<ActionsDAG> & actions)
+        auto check_actions = [](ActionsDAG & actions)
         {
-            if (actions)
-                for (const auto & node : actions->getNodes())
-                    if (node.type == ActionsDAG::ActionType::ARRAY_JOIN)
-                        throw Exception(ErrorCodes::ILLEGAL_PREWHERE, "PREWHERE cannot contain ARRAY JOIN action");
+            for (const auto & node : actions.getNodes())
+                if (node.type == ActionsDAG::ActionType::ARRAY_JOIN)
+                    throw Exception(ErrorCodes::ILLEGAL_PREWHERE, "PREWHERE cannot contain ARRAY JOIN action");
         };
 
         check_actions(prewhere_info->prewhere_actions);
diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp
index 174e6b5b0e0..4fd6f7a2900 100644
--- a/src/Interpreters/InterpreterSelectQuery.cpp
+++ b/src/Interpreters/InterpreterSelectQuery.cpp
@@ -937,7 +937,7 @@ bool InterpreterSelectQuery::adjustParallelReplicasAfterAnalysis()
     {
         {
             const auto & node
-                = query_info_copy.prewhere_info->prewhere_actions->findInOutputs(query_info_copy.prewhere_info->prewhere_column_name);
+                = query_info_copy.prewhere_info->prewhere_actions.findInOutputs(query_info_copy.prewhere_info->prewhere_column_name);
             added_filter_nodes.nodes.push_back(&node);
         }
 
@@ -1058,7 +1058,7 @@ Block InterpreterSelectQuery::getSampleBlockImpl()
 
         if (analysis_result.prewhere_info)
         {
-            header = analysis_result.prewhere_info->prewhere_actions->updateHeader(header);
+            header = analysis_result.prewhere_info->prewhere_actions.updateHeader(header);
             if (analysis_result.prewhere_info->remove_prewhere_column)
                 header.erase(analysis_result.prewhere_info->prewhere_column_name);
         }
@@ -1521,7 +1521,7 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional<P
 
             auto prewhere_step = std::make_unique<FilterStep>(
                 query_plan.getCurrentDataStream(),
-                expressions.prewhere_info->prewhere_actions->clone(),
+                expressions.prewhere_info->prewhere_actions.clone(),
                 expressions.prewhere_info->prewhere_column_name,
                 expressions.prewhere_info->remove_prewhere_column);
 
@@ -2066,7 +2066,7 @@ void InterpreterSelectQuery::addEmptySourceToQueryPlan(QueryPlan & query_plan, c
             });
         }
 
-        auto filter_actions = std::make_shared<ExpressionActions>(prewhere_info.prewhere_actions->clone());
+        auto filter_actions = std::make_shared<ExpressionActions>(prewhere_info.prewhere_actions.clone());
         pipe.addSimpleTransform([&](const Block & header)
         {
             return std::make_shared<FilterTransform>(
@@ -2157,7 +2157,7 @@ void InterpreterSelectQuery::addPrewhereAliasActions()
         if (prewhere_info)
         {
             /// Get some columns directly from PREWHERE expression actions
-            auto prewhere_required_columns = prewhere_info->prewhere_actions->getRequiredColumns().getNames();
+            auto prewhere_required_columns = prewhere_info->prewhere_actions.getRequiredColumns().getNames();
             columns.insert(prewhere_required_columns.begin(), prewhere_required_columns.end());
 
             if (prewhere_info->row_level_filter)
@@ -2229,7 +2229,7 @@ void InterpreterSelectQuery::addPrewhereAliasActions()
         if (prewhere_info)
         {
             NameSet columns_to_remove(columns_to_remove_after_prewhere.begin(), columns_to_remove_after_prewhere.end());
-            Block prewhere_actions_result = prewhere_info->prewhere_actions->getResultColumns();
+            Block prewhere_actions_result = prewhere_info->prewhere_actions.getResultColumns();
 
             /// Populate required columns with the columns, added by PREWHERE actions and not removed afterwards.
             /// XXX: looks hacky that we already know which columns after PREWHERE we won't need for sure.
@@ -2268,7 +2268,7 @@ void InterpreterSelectQuery::addPrewhereAliasActions()
         {
             /// Don't remove columns which are needed to be aliased.
             for (const auto & name : required_columns)
-                prewhere_info->prewhere_actions->tryRestoreColumn(name);
+                prewhere_info->prewhere_actions.tryRestoreColumn(name);
 
             /// Add physical columns required by prewhere actions.
             for (const auto & column : required_columns_from_prewhere)
@@ -2326,7 +2326,7 @@ std::optional<UInt64> InterpreterSelectQuery::getTrivialCount(UInt64 max_paralle
         if (analysis_result.hasPrewhere())
         {
             auto & prewhere_info = analysis_result.prewhere_info;
-            filter_nodes.push_back(&prewhere_info->prewhere_actions->findInOutputs(prewhere_info->prewhere_column_name));
+            filter_nodes.push_back(&prewhere_info->prewhere_actions.findInOutputs(prewhere_info->prewhere_column_name));
 
             if (prewhere_info->row_level_filter)
                 filter_nodes.push_back(&prewhere_info->row_level_filter->findInOutputs(prewhere_info->row_level_column_name));
diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp
index e9f886ab162..a3db0395ccc 100644
--- a/src/Planner/PlannerJoinTree.cpp
+++ b/src/Planner/PlannerJoinTree.cpp
@@ -437,7 +437,7 @@ void updatePrewhereOutputsIfNeeded(SelectQueryInfo & table_expression_query_info
 
     std::unordered_set<const ActionsDAG::Node *> required_output_nodes;
 
-    for (const auto * input : prewhere_actions->getInputs())
+    for (const auto * input : prewhere_actions.getInputs())
     {
         if (required_columns.contains(input->result_name))
             required_output_nodes.insert(input);
@@ -446,7 +446,7 @@ void updatePrewhereOutputsIfNeeded(SelectQueryInfo & table_expression_query_info
     if (required_output_nodes.empty())
         return;
 
-    auto & prewhere_outputs = prewhere_actions->getOutputs();
+    auto & prewhere_outputs = prewhere_actions.getOutputs();
     for (const auto & output : prewhere_outputs)
     {
         auto required_output_node_it = required_output_nodes.find(output);
@@ -801,10 +801,8 @@ JoinTreeQueryPlan buildQueryPlanForTableExpression(QueryTreeNodePtr table_expres
                     if (storage->canMoveConditionsToPrewhere() && optimize_move_to_prewhere && (!supported_prewhere_columns || supported_prewhere_columns->contains(filter_info.column_name)))
                     {
                         if (!prewhere_info)
-                            prewhere_info = std::make_shared<PrewhereInfo>();
-
-                        if (!prewhere_info->prewhere_actions)
                         {
+                            prewhere_info = std::make_shared<PrewhereInfo>();
                             prewhere_info->prewhere_actions = std::move(filter_info.actions);
                             prewhere_info->prewhere_column_name = filter_info.column_name;
                             prewhere_info->remove_prewhere_column = filter_info.do_remove_column;
diff --git a/src/Processors/QueryPlan/Optimizations/optimizePrewhere.cpp b/src/Processors/QueryPlan/Optimizations/optimizePrewhere.cpp
index 5711189136c..dc73521210a 100644
--- a/src/Processors/QueryPlan/Optimizations/optimizePrewhere.cpp
+++ b/src/Processors/QueryPlan/Optimizations/optimizePrewhere.cpp
@@ -56,7 +56,7 @@ void optimizePrewhere(Stack & stack, QueryPlan::Nodes &)
         return;
 
     const auto & storage_prewhere_info = source_step_with_filter->getPrewhereInfo();
-    if (storage_prewhere_info && storage_prewhere_info->prewhere_actions)
+    if (storage_prewhere_info)
         return;
 
     /// TODO: We can also check for UnionStep, such as StorageBuffer and local distributed plans.
@@ -165,16 +165,16 @@ void optimizePrewhere(Stack & stack, QueryPlan::Nodes &)
     {
         prewhere_info->prewhere_column_name = conditions.front()->result_name;
         if (prewhere_info->remove_prewhere_column)
-            prewhere_info->prewhere_actions->getOutputs().push_back(conditions.front());
+            prewhere_info->prewhere_actions.getOutputs().push_back(conditions.front());
     }
     else
     {
         prewhere_info->remove_prewhere_column = true;
 
         FunctionOverloadResolverPtr func_builder_and = std::make_unique<FunctionToOverloadResolverAdaptor>(std::make_shared<FunctionAnd>());
-        const auto * node = &prewhere_info->prewhere_actions->addFunction(func_builder_and, std::move(conditions), {});
+        const auto * node = &prewhere_info->prewhere_actions.addFunction(func_builder_and, std::move(conditions), {});
         prewhere_info->prewhere_column_name = node->result_name;
-        prewhere_info->prewhere_actions->getOutputs().push_back(node);
+        prewhere_info->prewhere_actions.getOutputs().push_back(node);
     }
 
     source_step_with_filter->updatePrewhereInfo(prewhere_info);
diff --git a/src/Processors/QueryPlan/Optimizations/optimizePrimaryKeyConditionAndLimit.cpp b/src/Processors/QueryPlan/Optimizations/optimizePrimaryKeyConditionAndLimit.cpp
index f53212407d2..490b79fbf8d 100644
--- a/src/Processors/QueryPlan/Optimizations/optimizePrimaryKeyConditionAndLimit.cpp
+++ b/src/Processors/QueryPlan/Optimizations/optimizePrimaryKeyConditionAndLimit.cpp
@@ -18,7 +18,7 @@ void optimizePrimaryKeyConditionAndLimit(const Stack & stack)
     const auto & storage_prewhere_info = source_step_with_filter->getPrewhereInfo();
     if (storage_prewhere_info)
     {
-        source_step_with_filter->addFilter(storage_prewhere_info->prewhere_actions->clone(), storage_prewhere_info->prewhere_column_name);
+        source_step_with_filter->addFilter(storage_prewhere_info->prewhere_actions.clone(), storage_prewhere_info->prewhere_column_name);
         if (storage_prewhere_info->row_level_filter)
             source_step_with_filter->addFilter(storage_prewhere_info->row_level_filter->clone(), storage_prewhere_info->row_level_column_name);
     }
diff --git a/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp b/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp
index 252420e19fe..99df6da263f 100644
--- a/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp
+++ b/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp
@@ -191,13 +191,11 @@ void buildSortingDAG(QueryPlan::Node & node, std::optional<ActionsDAG> & dag, Fi
             /// Should ignore limit if there is filtering.
             limit = 0;
 
-            if (prewhere_info->prewhere_actions)
-            {
-                //std::cerr << "====== Adding prewhere " << std::endl;
-                appendExpression(dag, *prewhere_info->prewhere_actions);
-                if (const auto * filter_expression = dag->tryFindInOutputs(prewhere_info->prewhere_column_name))
-                    appendFixedColumnsFromFilterExpression(*filter_expression, fixed_columns);
-            }
+            //std::cerr << "====== Adding prewhere " << std::endl;
+            appendExpression(dag, prewhere_info->prewhere_actions);
+            if (const auto * filter_expression = dag->tryFindInOutputs(prewhere_info->prewhere_column_name))
+                appendFixedColumnsFromFilterExpression(*filter_expression, fixed_columns);
+
         }
         return;
     }
diff --git a/src/Processors/QueryPlan/Optimizations/projectionsCommon.cpp b/src/Processors/QueryPlan/Optimizations/projectionsCommon.cpp
index 571d1dd0cc1..7414d479cc9 100644
--- a/src/Processors/QueryPlan/Optimizations/projectionsCommon.cpp
+++ b/src/Processors/QueryPlan/Optimizations/projectionsCommon.cpp
@@ -128,15 +128,12 @@ bool QueryDAG::buildImpl(QueryPlan::Node & node, ActionsDAG::NodeRawConstPtrs &
                     return false;
             }
 
-            if (prewhere_info->prewhere_actions)
-            {
-                appendExpression(*prewhere_info->prewhere_actions);
-                if (const auto * filter_expression
-                    = findInOutputs(*dag, prewhere_info->prewhere_column_name, prewhere_info->remove_prewhere_column))
-                    filter_nodes.push_back(filter_expression);
-                else
-                    return false;
-            }
+            appendExpression(prewhere_info->prewhere_actions);
+            if (const auto * filter_expression
+                = findInOutputs(*dag, prewhere_info->prewhere_column_name, prewhere_info->remove_prewhere_column))
+                filter_nodes.push_back(filter_expression);
+            else
+                return false;
         }
         return true;
     }
diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp
index e5aeb9686be..483876dd293 100644
--- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp
+++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp
@@ -109,8 +109,7 @@ bool restorePrewhereInputs(PrewhereInfo & info, const NameSet & inputs)
     if (info.row_level_filter)
         added = added || restoreDAGInputs(*info.row_level_filter, inputs);
 
-    if (info.prewhere_actions)
-        added = added || restoreDAGInputs(*info.prewhere_actions, inputs);
+    added = added || restoreDAGInputs(info.prewhere_actions, inputs);
 
     return added;
 }
@@ -175,9 +174,8 @@ static void updateSortDescriptionForOutputStream(
     Block original_header = output_stream.header.cloneEmpty();
     if (prewhere_info)
     {
-        if (prewhere_info->prewhere_actions)
         {
-            FindOriginalNodeForOutputName original_column_finder(*prewhere_info->prewhere_actions);
+            FindOriginalNodeForOutputName original_column_finder(prewhere_info->prewhere_actions);
             for (auto & column : original_header)
             {
                 const auto * original_node = original_column_finder.find(column.name);
@@ -2131,7 +2129,6 @@ void ReadFromMergeTree::describeActions(FormatSettings & format_settings) const
         prefix.push_back(format_settings.indent_char);
         prefix.push_back(format_settings.indent_char);
 
-        if (prewhere_info->prewhere_actions)
         {
             format_settings.out << prefix << "Prewhere filter" << '\n';
             format_settings.out << prefix << "Prewhere filter column: " << prewhere_info->prewhere_column_name;
@@ -2139,7 +2136,7 @@ void ReadFromMergeTree::describeActions(FormatSettings & format_settings) const
                format_settings.out << " (removed)";
             format_settings.out << '\n';
 
-            auto expression = std::make_shared<ExpressionActions>(prewhere_info->prewhere_actions->clone());
+            auto expression = std::make_shared<ExpressionActions>(prewhere_info->prewhere_actions.clone());
             expression->describeActions(format_settings.out, prefix);
         }
 
@@ -2169,12 +2166,11 @@ void ReadFromMergeTree::describeActions(JSONBuilder::JSONMap & map) const
         std::unique_ptr<JSONBuilder::JSONMap> prewhere_info_map = std::make_unique<JSONBuilder::JSONMap>();
         prewhere_info_map->add("Need filter", prewhere_info->need_filter);
 
-        if (prewhere_info->prewhere_actions)
         {
             std::unique_ptr<JSONBuilder::JSONMap> prewhere_filter_map = std::make_unique<JSONBuilder::JSONMap>();
             prewhere_filter_map->add("Prewhere filter column", prewhere_info->prewhere_column_name);
             prewhere_filter_map->add("Prewhere filter remove filter column", prewhere_info->remove_prewhere_column);
-            auto expression = std::make_shared<ExpressionActions>(prewhere_info->prewhere_actions->clone());
+            auto expression = std::make_shared<ExpressionActions>(prewhere_info->prewhere_actions.clone());
             prewhere_filter_map->add("Prewhere filter expression", expression->toTree());
 
             prewhere_info_map->add("Prewhere filter", std::move(prewhere_filter_map));
diff --git a/src/Processors/QueryPlan/SourceStepWithFilter.cpp b/src/Processors/QueryPlan/SourceStepWithFilter.cpp
index b91debc8239..3de9ae37db0 100644
--- a/src/Processors/QueryPlan/SourceStepWithFilter.cpp
+++ b/src/Processors/QueryPlan/SourceStepWithFilter.cpp
@@ -34,9 +34,8 @@ Block SourceStepWithFilter::applyPrewhereActions(Block block, const PrewhereInfo
             block.erase(prewhere_info->row_level_column_name);
         }
 
-        if (prewhere_info->prewhere_actions)
         {
-            block = prewhere_info->prewhere_actions->updateHeader(block);
+            block = prewhere_info->prewhere_actions.updateHeader(block);
 
             auto & prewhere_column = block.getByName(prewhere_info->prewhere_column_name);
             if (!prewhere_column.type->canBeUsedInBooleanContext())
@@ -102,7 +101,6 @@ void SourceStepWithFilter::describeActions(FormatSettings & format_settings) con
         prefix.push_back(format_settings.indent_char);
         prefix.push_back(format_settings.indent_char);
 
-        if (prewhere_info->prewhere_actions)
         {
             format_settings.out << prefix << "Prewhere filter" << '\n';
             format_settings.out << prefix << "Prewhere filter column: " << prewhere_info->prewhere_column_name;
@@ -110,7 +108,7 @@ void SourceStepWithFilter::describeActions(FormatSettings & format_settings) con
                 format_settings.out << " (removed)";
             format_settings.out << '\n';
 
-            auto expression = std::make_shared<ExpressionActions>(prewhere_info->prewhere_actions->clone());
+            auto expression = std::make_shared<ExpressionActions>(prewhere_info->prewhere_actions.clone());
             expression->describeActions(format_settings.out, prefix);
         }
 
@@ -132,12 +130,11 @@ void SourceStepWithFilter::describeActions(JSONBuilder::JSONMap & map) const
         std::unique_ptr<JSONBuilder::JSONMap> prewhere_info_map = std::make_unique<JSONBuilder::JSONMap>();
         prewhere_info_map->add("Need filter", prewhere_info->need_filter);
 
-        if (prewhere_info->prewhere_actions)
         {
             std::unique_ptr<JSONBuilder::JSONMap> prewhere_filter_map = std::make_unique<JSONBuilder::JSONMap>();
             prewhere_filter_map->add("Prewhere filter column", prewhere_info->prewhere_column_name);
             prewhere_filter_map->add("Prewhere filter remove filter column", prewhere_info->remove_prewhere_column);
-            auto expression = std::make_shared<ExpressionActions>(prewhere_info->prewhere_actions->clone());
+            auto expression = std::make_shared<ExpressionActions>(prewhere_info->prewhere_actions.clone());
             prewhere_filter_map->add("Prewhere filter expression", expression->toTree());
 
             prewhere_info_map->add("Prewhere filter", std::move(prewhere_filter_map));
diff --git a/src/Storages/IStorage.cpp b/src/Storages/IStorage.cpp
index 823a6ae1cbc..755d71df531 100644
--- a/src/Storages/IStorage.cpp
+++ b/src/Storages/IStorage.cpp
@@ -325,9 +325,8 @@ std::string PrewhereInfo::dump() const
         ss << "row_level_filter " << row_level_filter->dumpDAG() << "\n";
     }
 
-    if (prewhere_actions)
     {
-        ss << "prewhere_actions " << prewhere_actions->dumpDAG() << "\n";
+        ss << "prewhere_actions " << prewhere_actions.dumpDAG() << "\n";
     }
 
     ss << "remove_prewhere_column " << remove_prewhere_column
diff --git a/src/Storages/MergeTree/MergeTreePrefetchedReadPool.cpp b/src/Storages/MergeTree/MergeTreePrefetchedReadPool.cpp
index 26595fbb36d..a9b77fb6c03 100644
--- a/src/Storages/MergeTree/MergeTreePrefetchedReadPool.cpp
+++ b/src/Storages/MergeTree/MergeTreePrefetchedReadPool.cpp
@@ -329,7 +329,7 @@ void MergeTreePrefetchedReadPool::fillPerPartStatistics()
             part_stat.sum_marks += range.end - range.begin;
 
         const auto & columns = settings.merge_tree_determine_task_size_by_prewhere_columns && prewhere_info
-            ? prewhere_info->prewhere_actions->getRequiredColumnsNames()
+            ? prewhere_info->prewhere_actions.getRequiredColumnsNames()
             : column_names;
 
         part_stat.approx_size_of_mark = getApproximateSizeOfGranule(*read_info.data_part, columns);
diff --git a/src/Storages/MergeTree/MergeTreeReadPoolBase.cpp b/src/Storages/MergeTree/MergeTreeReadPoolBase.cpp
index 46482bc0959..6d2560bc9c7 100644
--- a/src/Storages/MergeTree/MergeTreeReadPoolBase.cpp
+++ b/src/Storages/MergeTree/MergeTreeReadPoolBase.cpp
@@ -65,7 +65,7 @@ static size_t calculateMinMarksPerTask(
         /// Which means in turn that for most of the rows we will read only the columns from prewhere clause.
         /// So it makes sense to use only them for the estimation.
         const auto & columns = settings.merge_tree_determine_task_size_by_prewhere_columns && prewhere_info
-            ? prewhere_info->prewhere_actions->getRequiredColumnsNames()
+            ? prewhere_info->prewhere_actions.getRequiredColumnsNames()
             : columns_to_read;
         const size_t part_compressed_bytes = getApproxSizeOfPart(*part.data_part, columns);
 
diff --git a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp
index f1df9e231c4..1a0709faf1c 100644
--- a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp
+++ b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp
@@ -59,7 +59,7 @@ MergeTreeSelectProcessor::MergeTreeSelectProcessor(
 
     if (prewhere_info)
         LOG_TEST(log, "Original PREWHERE DAG:\n{}\nPREWHERE actions:\n{}",
-            (prewhere_info->prewhere_actions ? prewhere_info->prewhere_actions->dumpDAG(): std::string("<nullptr>")),
+            prewhere_info->prewhere_actions.dumpDAG(),
             (!prewhere_actions.steps.empty() ? prewhere_actions.dump() : std::string("<nullptr>")));
 }
 
@@ -96,7 +96,7 @@ PrewhereExprInfo MergeTreeSelectProcessor::getPrewhereActions(PrewhereInfoPtr pr
             PrewhereExprStep prewhere_step
             {
                 .type = PrewhereExprStep::Filter,
-                .actions = std::make_shared<ExpressionActions>(prewhere_info->prewhere_actions->clone(), actions_settings),
+                .actions = std::make_shared<ExpressionActions>(prewhere_info->prewhere_actions.clone(), actions_settings),
                 .filter_column_name = prewhere_info->prewhere_column_name,
                 .remove_filter_column = prewhere_info->remove_prewhere_column,
                 .need_filter = prewhere_info->need_filter,
diff --git a/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp b/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp
index 1d0569e0df6..36ff6c0a4bd 100644
--- a/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp
+++ b/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp
@@ -216,11 +216,11 @@ const ActionsDAG::Node & addAndTrue(
 /// 8. Add computation of the remaining outputs to the last step with the procedure similar to 4
 bool tryBuildPrewhereSteps(PrewhereInfoPtr prewhere_info, const ExpressionActionsSettings & actions_settings, PrewhereExprInfo & prewhere)
 {
-    if (!prewhere_info || !prewhere_info->prewhere_actions)
+    if (!prewhere_info)
         return true;
 
     /// 1. List all condition nodes that are combined with AND into PREWHERE condition
-    const auto & condition_root = prewhere_info->prewhere_actions->findInOutputs(prewhere_info->prewhere_column_name);
+    const auto & condition_root = prewhere_info->prewhere_actions.findInOutputs(prewhere_info->prewhere_column_name);
     const bool is_conjunction = (condition_root.type == ActionsDAG::ActionType::FUNCTION && condition_root.function_base->getName() == "and");
     if (!is_conjunction)
         return false;
@@ -306,7 +306,7 @@ bool tryBuildPrewhereSteps(PrewhereInfoPtr prewhere_info, const ExpressionAction
     }
 
     /// 6. Find all outputs of the original DAG
-    auto original_outputs = prewhere_info->prewhere_actions->getOutputs();
+    auto original_outputs = prewhere_info->prewhere_actions.getOutputs();
     /// 7. Find all outputs that were computed in the already built DAGs, mark these nodes as outputs in the steps where they were computed
     /// 8. Add computation of the remaining outputs to the last step with the procedure similar to 4
     NameSet all_output_names;
diff --git a/src/Storages/SelectQueryInfo.h b/src/Storages/SelectQueryInfo.h
index 1c4cb7d92d8..7ad6a733c6f 100644
--- a/src/Storages/SelectQueryInfo.h
+++ b/src/Storages/SelectQueryInfo.h
@@ -45,7 +45,7 @@ struct PrewhereInfo
     /// This actions are separate because prewhere condition should not be executed over filtered rows.
     std::optional<ActionsDAG> row_level_filter;
     /// Actions which are executed on block in order to get filter column for prewhere step.
-    std::optional<ActionsDAG> prewhere_actions;
+    ActionsDAG prewhere_actions;
     String row_level_column_name;
     String prewhere_column_name;
     bool remove_prewhere_column = false;
@@ -53,7 +53,7 @@ struct PrewhereInfo
     bool generated_by_optimizer = false;
 
     PrewhereInfo() = default;
-    explicit PrewhereInfo(std::optional<ActionsDAG> prewhere_actions_, String prewhere_column_name_)
+    explicit PrewhereInfo(ActionsDAG prewhere_actions_, String prewhere_column_name_)
             : prewhere_actions(std::move(prewhere_actions_)), prewhere_column_name(std::move(prewhere_column_name_)) {}
 
     std::string dump() const;
@@ -65,8 +65,7 @@ struct PrewhereInfo
         if (row_level_filter)
             prewhere_info->row_level_filter = row_level_filter->clone();
 
-        if (prewhere_actions)
-            prewhere_info->prewhere_actions = prewhere_actions->clone();
+        prewhere_info->prewhere_actions = prewhere_actions.clone();
 
         prewhere_info->row_level_column_name = row_level_column_name;
         prewhere_info->prewhere_column_name = prewhere_column_name;
diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp
index aee4e4683ad..04e6d6676d1 100644
--- a/src/Storages/StorageBuffer.cpp
+++ b/src/Storages/StorageBuffer.cpp
@@ -319,13 +319,12 @@ void StorageBuffer::read(
                         src_table_query_info.prewhere_info->row_level_filter->removeUnusedActions();
                     }
 
-                    if (src_table_query_info.prewhere_info->prewhere_actions)
                     {
                         src_table_query_info.prewhere_info->prewhere_actions = ActionsDAG::merge(
                             actions_dag.clone(),
-                            std::move(*src_table_query_info.prewhere_info->prewhere_actions));
+                            std::move(src_table_query_info.prewhere_info->prewhere_actions));
 
-                        src_table_query_info.prewhere_info->prewhere_actions->removeUnusedActions();
+                        src_table_query_info.prewhere_info->prewhere_actions.removeUnusedActions();
                     }
                 }
 
@@ -440,7 +439,7 @@ void StorageBuffer::read(
                 });
             }
 
-            auto actions = std::make_shared<ExpressionActions>(query_info.prewhere_info->prewhere_actions->clone(), actions_settings);
+            auto actions = std::make_shared<ExpressionActions>(query_info.prewhere_info->prewhere_actions.clone(), actions_settings);
             pipe_from_buffers.addSimpleTransform([&](const Block & header)
             {
                 return std::make_shared<FilterTransform>(

From 638d4640959f93924cec00b172d1cc1837d9ac10 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Thu, 25 Jul 2024 18:42:16 +0200
Subject: [PATCH 0792/1488] Fix test `00673_subquery_prepared_set_performance`

---
 .../0_stateless/00673_subquery_prepared_set_performance.sql     | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/queries/0_stateless/00673_subquery_prepared_set_performance.sql b/tests/queries/0_stateless/00673_subquery_prepared_set_performance.sql
index 0591592344c..b938d54c646 100644
--- a/tests/queries/0_stateless/00673_subquery_prepared_set_performance.sql
+++ b/tests/queries/0_stateless/00673_subquery_prepared_set_performance.sql
@@ -1,3 +1,5 @@
+-- Tags: no-tsan
+
 DROP TABLE IF EXISTS mergetree_00673;
 
 CREATE TABLE mergetree_00673 (x UInt64) ENGINE = MergeTree ORDER BY x;

From 7a003237befaa8d58cb6a77bb47e11fd1493e277 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Thu, 25 Jul 2024 18:43:11 +0200
Subject: [PATCH 0793/1488] Fix test `00673_subquery_prepared_set_performance`

---
 .../0_stateless/00673_subquery_prepared_set_performance.sql | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/tests/queries/0_stateless/00673_subquery_prepared_set_performance.sql b/tests/queries/0_stateless/00673_subquery_prepared_set_performance.sql
index b938d54c646..98c0802ffbc 100644
--- a/tests/queries/0_stateless/00673_subquery_prepared_set_performance.sql
+++ b/tests/queries/0_stateless/00673_subquery_prepared_set_performance.sql
@@ -1,14 +1,12 @@
--- Tags: no-tsan
-
 DROP TABLE IF EXISTS mergetree_00673;
 
 CREATE TABLE mergetree_00673 (x UInt64) ENGINE = MergeTree ORDER BY x;
 INSERT INTO mergetree_00673 VALUES (1);
 
-SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM mergetree_00673 WHERE x IN (SELECT * FROM numbers(10000000))))))))))));
+SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM mergetree_00673 WHERE x IN (SELECT * FROM numbers(1000000))))))))))))))))))))));
 
 SET force_primary_key = 1;
 
-SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM mergetree_00673 WHERE x IN (SELECT * FROM numbers(10000000))))))))))));
+SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM mergetree_00673 WHERE x IN (SELECT * FROM numbers(1000000))))))))))))))))))))));
 
 DROP TABLE mergetree_00673;

From 21f3a08ba7d626b967d99f694b1fde93da022ab9 Mon Sep 17 00:00:00 2001
From: Alexander Tokmakov <tavplubix@clickhouse.com>
Date: Thu, 25 Jul 2024 18:54:51 +0200
Subject: [PATCH 0794/1488] fix flaky test

---
 .../queries/0_stateless/03145_non_loaded_projection_backup.sh | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/tests/queries/0_stateless/03145_non_loaded_projection_backup.sh b/tests/queries/0_stateless/03145_non_loaded_projection_backup.sh
index 7df2118ad0c..95aef9bbc5b 100755
--- a/tests/queries/0_stateless/03145_non_loaded_projection_backup.sh
+++ b/tests/queries/0_stateless/03145_non_loaded_projection_backup.sh
@@ -6,8 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
 $CLICKHOUSE_CLIENT -nm -q "
 drop table if exists tp_1;
-create table tp_1 (x Int32, y Int32, projection p (select x, y order by x)) engine = MergeTree order by y partition by intDiv(y, 100);
-system stop merges tp_1;
+create table tp_1 (x Int32, y Int32, projection p (select x, y order by x)) engine = MergeTree order by y partition by intDiv(y, 100) settings max_parts_to_merge_at_once=1;
 insert into tp_1 select number, number from numbers(3);
 
 set mutations_sync = 2;
@@ -39,7 +38,6 @@ $CLICKHOUSE_CLIENT -nm -q "
 set send_logs_level='fatal';
 drop table tp_1;
 restore table tp_1 from Disk('backups', '$backup_id');
-system stop merges tp_1;
 " | grep -o "RESTORED"
 
 $CLICKHOUSE_CLIENT -q "select count() from tp_1;"

From f4b943f9f82bd4d297574774173e45abb2ee42d0 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Thu, 25 Jul 2024 19:05:41 +0200
Subject: [PATCH 0795/1488] Fix tidy

---
 src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp b/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp
index 377f6b36888..ba864035777 100644
--- a/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp
+++ b/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp
@@ -261,7 +261,7 @@ std::optional<size_t> ReadBufferFromAzureBlobStorage::tryGetFileSize()
     if (!file_size)
         file_size = blob_client->GetProperties().Value.BlobSize;
 
-    return *file_size;
+    return file_size;
 }
 
 size_t ReadBufferFromAzureBlobStorage::readBigAt(char * to, size_t n, size_t range_begin, const std::function<bool(size_t)> & /*progress_callback*/) const

From a06df0729ea398642b715bfd2b121b1db0c5dd6d Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Thu, 25 Jul 2024 17:10:59 +0000
Subject: [PATCH 0796/1488] Remove the comment.

---
 src/Interpreters/ActionsDAG.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp
index 85b2b38da17..4aaecc491e0 100644
--- a/src/Interpreters/ActionsDAG.cpp
+++ b/src/Interpreters/ActionsDAG.cpp
@@ -3111,7 +3111,6 @@ ActionsDAG::NodeRawConstPtrs ActionsDAG::filterNodesByAllowedInputs(
 }
 
 FindOriginalNodeForOutputName::FindOriginalNodeForOutputName(const ActionsDAG & actions_)
-    //: actions(actions_)
 {
     const auto & actions_outputs = actions_.getOutputs();
     for (const auto * output_node : actions_outputs)

From 257be35365b8e0fd6163af027bbc02288ce8910b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Thu, 25 Jul 2024 19:21:31 +0200
Subject: [PATCH 0797/1488] Minor tweaks and extra type tests

---
 src/AggregateFunctions/SingleValueData.cpp    |  6 +++++-
 ..._fix_single_value_data_assertion.reference | 12 +++++++++++
 .../03210_fix_single_value_data_assertion.sql | 20 ++++++++++++++++++-
 3 files changed, 36 insertions(+), 2 deletions(-)

diff --git a/src/AggregateFunctions/SingleValueData.cpp b/src/AggregateFunctions/SingleValueData.cpp
index 566b40253a3..11931acbbc8 100644
--- a/src/AggregateFunctions/SingleValueData.cpp
+++ b/src/AggregateFunctions/SingleValueData.cpp
@@ -904,8 +904,9 @@ bool SingleValueDataNumeric<T>::isEqualTo(const DB::IColumn & column, size_t ind
 template <typename T>
 bool SingleValueDataNumeric<T>::isEqualTo(const DB::SingleValueDataBase & to) const
 {
+    /// to.has() is checked in memory.get().isEqualTo
     auto const & other = assert_cast<const Self &>(to);
-    return to.has() && memory.get().isEqualTo(other.memory.get());
+    return memory.get().isEqualTo(other.memory.get());
 }
 
 template <typename T>
@@ -917,6 +918,7 @@ void SingleValueDataNumeric<T>::set(const DB::IColumn & column, size_t row_num,
 template <typename T>
 void SingleValueDataNumeric<T>::set(const DB::SingleValueDataBase & to, DB::Arena * arena)
 {
+    /// to.has() is checked in memory.get().set
     auto const & other = assert_cast<const Self &>(to);
     return memory.get().set(other.memory.get(), arena);
 }
@@ -924,6 +926,7 @@ void SingleValueDataNumeric<T>::set(const DB::SingleValueDataBase & to, DB::Aren
 template <typename T>
 bool SingleValueDataNumeric<T>::setIfSmaller(const DB::SingleValueDataBase & to, DB::Arena * arena)
 {
+    /// to.has() is checked in memory.get().setIfSmaller
     auto const & other = assert_cast<const Self &>(to);
     return memory.get().setIfSmaller(other.memory.get(), arena);
 }
@@ -931,6 +934,7 @@ bool SingleValueDataNumeric<T>::setIfSmaller(const DB::SingleValueDataBase & to,
 template <typename T>
 bool SingleValueDataNumeric<T>::setIfGreater(const DB::SingleValueDataBase & to, DB::Arena * arena)
 {
+    /// to.has() is checked in memory.get().setIfGreater
     auto const & other = assert_cast<const Self &>(to);
     return memory.get().setIfGreater(other.memory.get(), arena);
 }
diff --git a/tests/queries/0_stateless/03210_fix_single_value_data_assertion.reference b/tests/queries/0_stateless/03210_fix_single_value_data_assertion.reference
index e69de29bb2d..d8f7e13db55 100644
--- a/tests/queries/0_stateless/03210_fix_single_value_data_assertion.reference
+++ b/tests/queries/0_stateless/03210_fix_single_value_data_assertion.reference
@@ -0,0 +1,12 @@
+0	1	1	1	0	0	0
+1	3	3	3	2	2	2
+2	5	5	5	4	4	4
+3	7	7	7	6	6	6
+4	9	9	9	8	8	8
+5	11	11	11	10	10	10
+6	13	13	13	12	12	12
+7	15	15	15	14	14	14
+8	17	17	17	16	16	16
+9	19	19	19	18	18	18
+
+0	107351244	107351244	107351244	107354520	107354520	107354520
diff --git a/tests/queries/0_stateless/03210_fix_single_value_data_assertion.sql b/tests/queries/0_stateless/03210_fix_single_value_data_assertion.sql
index 66e62377d6b..a1243ef0b25 100644
--- a/tests/queries/0_stateless/03210_fix_single_value_data_assertion.sql
+++ b/tests/queries/0_stateless/03210_fix_single_value_data_assertion.sql
@@ -1 +1,19 @@
-SELECT intDiv(number, 2) AS k, count(toFixedString(toFixedString('hello', 5), 5)) IGNORE NULLS, sumArgMax(number, toString(number % 20)), argMax(toString(number), number) FROM (SELECT number FROM system.numbers LIMIT 65537) WHERE toLowCardinality(toLowCardinality(toNullable(21))) GROUP BY k WITH TOTALS ORDER BY k ASC NULLS FIRST LIMIT 255 SETTINGS group_by_overflow_mode = 'any', totals_mode = 'before_having', max_rows_to_group_by = 100000 FORMAT Null
+SELECT
+    intDiv(number, 2) AS k,
+    sumArgMax(number, number % 20),
+    sumArgMax(number, leftPad(toString(number % 20), 5, '0')), -- Pad with 0 to preserve number ordering
+    sumArgMax(number, [number % 20, number % 20]),
+    sumArgMin(number, number % 20),
+    sumArgMin(number, leftPad(toString(number % 20), 5, '0')),
+    sumArgMin(number, [number % 20, number % 20]),
+FROM
+(
+    SELECT number
+    FROM system.numbers
+    LIMIT 65537
+)
+GROUP BY k
+    WITH TOTALS
+ORDER BY k ASC
+    LIMIT 10
+SETTINGS group_by_overflow_mode = 'any', totals_mode = 'before_having', max_rows_to_group_by = 100000;

From ee193ffa019fc0f6104c1c5010ba00cd1993c8bd Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Thu, 25 Jul 2024 19:25:58 +0200
Subject: [PATCH 0798/1488] Lower max allocation size in query fuzzer

---
 docker/test/fuzzer/query-fuzzer-tweaks-users.xml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/test/fuzzer/query-fuzzer-tweaks-users.xml b/docker/test/fuzzer/query-fuzzer-tweaks-users.xml
index e2a4976b385..d5b876a4c85 100644
--- a/docker/test/fuzzer/query-fuzzer-tweaks-users.xml
+++ b/docker/test/fuzzer/query-fuzzer-tweaks-users.xml
@@ -20,7 +20,7 @@
                 </max_execution_time>
 
                 <max_memory_usage>
-                    <max>10G</max>
+                    <max>5G</max>
                 </max_memory_usage>
 
                 <table_function_remote_max_addresses>

From 738d659e3bd8e222ff947e206d03d516c7053052 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Thu, 25 Jul 2024 17:26:16 +0000
Subject: [PATCH 0799/1488] Do not remove constants from Distributed header if
 query is executed up to Complete.

---
 src/Storages/StorageDistributed.cpp           |  7 ++-
 .../02563_analyzer_merge.reference            |  1 +
 .../0_stateless/02563_analyzer_merge.sql      | 45 +++++++++++++++++++
 3 files changed, 51 insertions(+), 2 deletions(-)

diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp
index 07892971ec2..9b417cda177 100644
--- a/src/Storages/StorageDistributed.cpp
+++ b/src/Storages/StorageDistributed.cpp
@@ -847,8 +847,11 @@ void StorageDistributed::read(
         /** For distributed tables we do not need constants in header, since we don't send them to remote servers.
           * Moreover, constants can break some functions like `hostName` that are constants only for local queries.
           */
-        for (auto & column : header)
-            column.column = column.column->convertToFullColumnIfConst();
+        if (processed_stage != QueryProcessingStage::Complete)
+        {
+            for (auto & column : header)
+                column.column = column.column->convertToFullColumnIfConst();
+        }
         modified_query_info.query = queryNodeToDistributedSelectQuery(query_tree_distributed);
 
         modified_query_info.query_tree = std::move(query_tree_distributed);
diff --git a/tests/queries/0_stateless/02563_analyzer_merge.reference b/tests/queries/0_stateless/02563_analyzer_merge.reference
index 8be01c88d6f..2b3cc2d5dfb 100644
--- a/tests/queries/0_stateless/02563_analyzer_merge.reference
+++ b/tests/queries/0_stateless/02563_analyzer_merge.reference
@@ -1,2 +1,3 @@
 0	Value_0	02563_db	test_merge_table_1
 1	Value_1	02563_db	test_merge_table_2
+91138316-5127-45ac-9c25-4ad8779777b4	160
diff --git a/tests/queries/0_stateless/02563_analyzer_merge.sql b/tests/queries/0_stateless/02563_analyzer_merge.sql
index c90f7dcb2a5..217fb7019c4 100644
--- a/tests/queries/0_stateless/02563_analyzer_merge.sql
+++ b/tests/queries/0_stateless/02563_analyzer_merge.sql
@@ -35,4 +35,49 @@ SELECT id, value, _database, _table FROM 02563_db.test_merge_table ORDER BY id;
 DROP TABLE 02563_db.test_merge_table;
 DROP TABLE 02563_db.test_merge_table_1;
 DROP TABLE 02563_db.test_merge_table_2;
+
+CREATE TABLE 02563_db.t_1
+(
+    timestamp DateTime64(9),
+    a String,
+    b String
+)
+ENGINE = MergeTree
+PARTITION BY formatDateTime(toStartOfMinute(timestamp), '%Y%m%d%H', 'UTC')
+ORDER BY (timestamp, a, b);
+
+CREATE TABLE 02563_db.dist_t_1 (timestamp DateTime64(9), a String, b String) ENGINE = Distributed('test_shard_localhost', '02563_db', 't_1');
+
+CREATE TABLE 02563_db.m ENGINE = Merge('02563_db', '^dist_');
+
+INSERT INTO 02563_db.t_1 (timestamp, a, b)
+select
+    addMinutes(toDateTime64('2024-07-13 22:00:00', 9, 'UTC'), number),
+    randomString(5),
+    randomString(5)
+from numbers(30);
+
+INSERT INTO 02563_db.t_1 (timestamp, a, b)
+select
+    addMinutes(toDateTime64('2024-07-13 23:00:00', 9, 'UTC'), number),
+    randomString(5),
+    randomString(5)
+from numbers(30);
+
+INSERT INTO 02563_db.t_1 (timestamp, a, b)
+select
+    addMinutes(toDateTime64('2024-07-14 00:00:00', 9, 'UTC'), number),
+    randomString(5),
+    randomString(5)
+from numbers(100);
+
+
+SELECT '91138316-5127-45ac-9c25-4ad8779777b4',
+  count()
+FROM 02563_db.m;
+
+DROP TABLE 02563_db.t_1;
+DROP TABLE 02563_db.dist_t_1;
+DROP TABLE 02563_db.m;
+
 DROP DATABASE 02563_db;

From a3d5b2d29014bb3894982cdb1cadd65448ecdf63 Mon Sep 17 00:00:00 2001
From: Alexander Tokmakov <tavplubix@clickhouse.com>
Date: Thu, 25 Jul 2024 19:39:20 +0200
Subject: [PATCH 0800/1488] Update ZooKeeperImpl.cpp

---
 src/Common/ZooKeeper/ZooKeeperImpl.cpp | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/src/Common/ZooKeeper/ZooKeeperImpl.cpp
index 2728f953bea..d01fc341a63 100644
--- a/src/Common/ZooKeeper/ZooKeeperImpl.cpp
+++ b/src/Common/ZooKeeper/ZooKeeperImpl.cpp
@@ -1014,9 +1014,6 @@ void ZooKeeper::finalize(bool error_send, bool error_receive, const String & rea
     LOG_INFO(log, "Finalizing session {}. finalization_started: {}, queue_finished: {}, reason: '{}'",
              session_id, already_started, requests_queue.isFinished(), reason);
 
-    /// Reset the original index.
-    original_index = -1;
-
     auto expire_session_if_not_expired = [&]
     {
         /// No new requests will appear in queue after finish()

From f32a0716b9bb42a09ece308a3ca64626099bfb1e Mon Sep 17 00:00:00 2001
From: Alexander Tokmakov <tavplubix@clickhouse.com>
Date: Thu, 25 Jul 2024 19:45:06 +0200
Subject: [PATCH 0801/1488] Update 02842_truncate_database.sql

---
 tests/queries/0_stateless/02842_truncate_database.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/02842_truncate_database.sql b/tests/queries/0_stateless/02842_truncate_database.sql
index be92108ccb8..bcd818f55ba 100644
--- a/tests/queries/0_stateless/02842_truncate_database.sql
+++ b/tests/queries/0_stateless/02842_truncate_database.sql
@@ -73,7 +73,7 @@ SELECT * FROM dest_dictionary; -- {serverError UNKNOWN_TABLE}
 SHOW TABLES FROM test_truncate_database;
 SHOW DICTIONARIES FROM test_truncate_database;
 
-CREATE TABLE new_table (x UInt16) ENGINE = ReplicatedMergeTree ORDER BY x;
+CREATE TABLE new_table (x UInt16) ENGINE = MergeTree ORDER BY x;
 select 'new tables';
 SHOW TABLES FROM test_truncate_database;
 

From e46512a3bed4cd260042acbde4fcbef5cb83e032 Mon Sep 17 00:00:00 2001
From: Julia Kartseva <yulia.kartseva@gmail.com>
Date: Thu, 25 Jul 2024 16:59:10 +0000
Subject: [PATCH 0802/1488] Try fix 02481_async_insert_race_long

https://github.com/ClickHouse/clickhouse-private/issues/13101
My assumption is that for high --insert_keeper_fault_injection_probability
values, entries don't leave the sink, causing the queue memory consumption
to grow, eventually hitting the memory limit and the MEMORY_LIMIT_EXCEEDED
error.

Later, we may raise the insert_keeper_fault_injection_probability
to a small positive value.
---
 tests/queries/0_stateless/02481_async_insert_race_long.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/queries/0_stateless/02481_async_insert_race_long.sh b/tests/queries/0_stateless/02481_async_insert_race_long.sh
index d8153967e9a..b0088017d32 100755
--- a/tests/queries/0_stateless/02481_async_insert_race_long.sh
+++ b/tests/queries/0_stateless/02481_async_insert_race_long.sh
@@ -13,7 +13,7 @@ function insert1()
 {
     local TIMELIMIT=$((SECONDS+$1))
     while [ $SECONDS -lt "$TIMELIMIT" ]; do
-        ${MY_CLICKHOUSE_CLIENT} --wait_for_async_insert 0 -q 'INSERT INTO async_inserts_race FORMAT CSV 1,"a"'
+        ${MY_CLICKHOUSE_CLIENT} --insert_keeper_fault_injection_probability=0 --wait_for_async_insert 0 -q 'INSERT INTO async_inserts_race FORMAT CSV 1,"a"'
     done
 }
 
@@ -21,7 +21,7 @@ function insert2()
 {
     local TIMELIMIT=$((SECONDS+$1))
     while [ $SECONDS -lt "$TIMELIMIT" ]; do
-        ${MY_CLICKHOUSE_CLIENT} --wait_for_async_insert 0 -q 'INSERT INTO async_inserts_race FORMAT JSONEachRow {"id": 5, "s": "e"} {"id": 6, "s": "f"}'
+        ${MY_CLICKHOUSE_CLIENT} --insert_keeper_fault_injection_probability=0 --wait_for_async_insert 0 -q 'INSERT INTO async_inserts_race FORMAT JSONEachRow {"id": 5, "s": "e"} {"id": 6, "s": "f"}'
     done
 }
 
@@ -29,7 +29,7 @@ function insert3()
 {
     local TIMELIMIT=$((SECONDS+$1))
     while [ $SECONDS -lt "$TIMELIMIT" ]; do
-        ${MY_CLICKHOUSE_CLIENT} --wait_for_async_insert 1 -q "INSERT INTO async_inserts_race VALUES (7, 'g') (8, 'h')" &
+        ${MY_CLICKHOUSE_CLIENT} --insert_keeper_fault_injection_probability=0 --wait_for_async_insert 1 -q "INSERT INTO async_inserts_race VALUES (7, 'g') (8, 'h')" &
         sleep 0.05
     done
 

From eb4ec0912ad3a1e89ea7aec424366bc268262e11 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Thu, 25 Jul 2024 20:21:37 +0200
Subject: [PATCH 0803/1488] Rename bad setting

---
 CHANGELOG.md                                              | 2 +-
 src/Core/Settings.h                                       | 2 +-
 src/Core/SettingsChangesHistory.cpp                       | 2 +-
 src/Formats/FormatFactory.cpp                             | 2 +-
 src/Formats/FormatSettings.h                              | 2 +-
 src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp | 4 ++--
 tests/queries/0_stateless/03013_json_key_ignore_case.sh   | 4 ++--
 7 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index a0933bd6544..07b37835dda 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -64,7 +64,7 @@
 * Print stacktrace and diagnostic info if `clickhouse-client` or `clickhouse-local` crashes. [#61109](https://github.com/ClickHouse/ClickHouse/pull/61109) ([Alexander Tokmakov](https://github.com/tavplubix)).
 * The result of `SHOW INDEX | INDEXES | INDICES | KEYS` was previously sorted by the primary key column names. Since this was unintuitive, the result is now sorted by the position of the primary key columns within the primary key. [#61131](https://github.com/ClickHouse/ClickHouse/pull/61131) ([Robert Schulze](https://github.com/rschu1ze)).
 * Change how deduplication for Materialized Views works. Fixed a lot of cases like: - on destination table: data is split for 2 or more blocks and that blocks is considered as duplicate when that block is inserted in parallel. - on MV destination table: the equal blocks are deduplicated, that happens when MV often produces equal data as a result for different input data due to performing aggregation. - on MV destination table: the equal blocks which comes from different MV are deduplicated. [#61601](https://github.com/ClickHouse/ClickHouse/pull/61601) ([Sema Checherinda](https://github.com/CheSema)).
-* Allow matching column names in a case insensitive manner when reading json files (`input_format_json_ignore_key_case`). [#61750](https://github.com/ClickHouse/ClickHouse/pull/61750) ([kevinyhzou](https://github.com/KevinyhZou)).
+* Allow matching column names in a case insensitive manner when reading json files (`input_format_json_case_insensitive_column_matching`). [#61750](https://github.com/ClickHouse/ClickHouse/pull/61750) ([kevinyhzou](https://github.com/KevinyhZou)).
 * Support reading partitioned data DeltaLake data. Infer DeltaLake schema by reading metadata instead of data. [#63201](https://github.com/ClickHouse/ClickHouse/pull/63201) ([Kseniia Sumarokova](https://github.com/kssenii)).
 * In composable protocols TLS layer accepted only `certificateFile` and `privateKeyFile` parameters. https://clickhouse.com/docs/en/operations/settings/composable-protocols. [#63985](https://github.com/ClickHouse/ClickHouse/pull/63985) ([Anton Ivashkin](https://github.com/ianton-ru)).
 * Added profile event `SelectQueriesWithPrimaryKeyUsage` which indicates how many SELECT queries use the primary key to evaluate the WHERE clause. [#64492](https://github.com/ClickHouse/ClickHouse/pull/64492) ([0x01f](https://github.com/0xfei)).
diff --git a/src/Core/Settings.h b/src/Core/Settings.h
index 3f1ecc47f79..e10cf3fd745 100644
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@@ -1122,7 +1122,7 @@ class IColumn;
     M(Bool, input_format_json_defaults_for_missing_elements_in_named_tuple, true, "Insert default value in named tuple element if it's missing in json object", 0) \
     M(Bool, input_format_json_throw_on_bad_escape_sequence, true, "Throw an exception if JSON string contains bad escape sequence in JSON input formats. If disabled, bad escape sequences will remain as is in the data", 0) \
     M(Bool, input_format_json_ignore_unnecessary_fields, true, "Ignore unnecessary fields and not parse them. Enabling this may not throw exceptions on json strings of invalid format or with duplicated fields", 0) \
-    M(Bool, input_format_json_ignore_key_case, false, "Ignore json key case while read json field from string", 0) \
+    M(Bool, input_format_json_case_insensitive_column_matching, false, "Ignore case when matching JSON keys with CH columns", 0) \
     M(Bool, input_format_try_infer_integers, true, "Try to infer integers instead of floats while schema inference in text formats", 0) \
     M(Bool, input_format_try_infer_dates, true, "Try to infer dates from string fields while schema inference in text formats", 0) \
     M(Bool, input_format_try_infer_datetimes, true, "Try to infer datetimes from string fields while schema inference in text formats", 0) \
diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp
index acd119c159b..9faf77e9087 100644
--- a/src/Core/SettingsChangesHistory.cpp
+++ b/src/Core/SettingsChangesHistory.cpp
@@ -64,7 +64,7 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
               {"input_format_native_decode_types_in_binary_format", false, false, "Added new setting to allow to read type names in binary format in Native output format"},
               {"read_in_order_use_buffering", false, true, "Use buffering before merging while reading in order of primary key"},
               {"enable_named_columns_in_function_tuple", false, true, "Generate named tuples in function tuple() when all names are unique and can be treated as unquoted identifiers."},
-              {"input_format_json_ignore_key_case", false, false, "Ignore json key case while read json field from string."},
+              {"input_format_json_case_insensitive_column_matching", false, false, "Ignore case when matching JSON keys with CH columns."},
               {"optimize_trivial_insert_select", true, false, "The optimization does not make sense in many cases."},
               {"dictionary_validate_primary_key_type", false, false, "Validate primary key type for dictionaries. By default id type for simple layouts will be implicitly converted to UInt64."},
               {"collect_hash_table_stats_during_joins", false, true, "New setting."},
diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp
index 1271cdfb7ad..501343d2ca9 100644
--- a/src/Formats/FormatFactory.cpp
+++ b/src/Formats/FormatFactory.cpp
@@ -150,7 +150,7 @@ FormatSettings getFormatSettings(const ContextPtr & context, const Settings & se
     format_settings.json.try_infer_objects_as_tuples = settings.input_format_json_try_infer_named_tuples_from_objects;
     format_settings.json.throw_on_bad_escape_sequence = settings.input_format_json_throw_on_bad_escape_sequence;
     format_settings.json.ignore_unnecessary_fields = settings.input_format_json_ignore_unnecessary_fields;
-    format_settings.json.ignore_key_case = settings.input_format_json_ignore_key_case;
+    format_settings.json.case_insensitive_column_matching = settings.input_format_json_case_insensitive_column_matching;
     format_settings.null_as_default = settings.input_format_null_as_default;
     format_settings.force_null_for_omitted_fields = settings.input_format_force_null_for_omitted_fields;
     format_settings.decimal_trailing_zeros = settings.output_format_decimal_trailing_zeros;
diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h
index a736bb1a558..72d1515b7e1 100644
--- a/src/Formats/FormatSettings.h
+++ b/src/Formats/FormatSettings.h
@@ -233,7 +233,7 @@ struct FormatSettings
         bool infer_incomplete_types_as_strings = true;
         bool throw_on_bad_escape_sequence = true;
         bool ignore_unnecessary_fields = true;
-        bool ignore_key_case = false;
+        bool case_insensitive_column_matching = false;
     } json{};
 
     struct
diff --git a/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp b/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp
index be4e9430c34..0e3972e8c45 100644
--- a/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp
+++ b/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp
@@ -46,7 +46,7 @@ JSONEachRowRowInputFormat::JSONEachRowRowInputFormat(
 {
     const auto & header = getPort().getHeader();
     name_map = header.getNamesToIndexesMap();
-    if (format_settings_.json.ignore_key_case)
+    if (format_settings_.json.case_insensitive_column_matching)
     {
         for (auto & it : name_map)
         {
@@ -181,7 +181,7 @@ void JSONEachRowRowInputFormat::readJSONObject(MutableColumns & columns)
             continue;
         }
         size_t column_index = 0;
-        if (format_settings.json.ignore_key_case)
+        if (format_settings.json.case_insensitive_column_matching)
         {
             String lower_case_name = transformFieldNameToLowerCase(name_ref);
             StringRef field_name_ref = lower_case_name_map[lower_case_name];
diff --git a/tests/queries/0_stateless/03013_json_key_ignore_case.sh b/tests/queries/0_stateless/03013_json_key_ignore_case.sh
index 807e743b22a..7882a29ae4f 100755
--- a/tests/queries/0_stateless/03013_json_key_ignore_case.sh
+++ b/tests/queries/0_stateless/03013_json_key_ignore_case.sh
@@ -13,6 +13,6 @@ cp "$CURDIR"/data_json/key_ignore_case.json $USER_FILES_PATH/
 
 $CLICKHOUSE_CLIENT -q "drop table if exists test_tbl"
 $CLICKHOUSE_CLIENT -q "create table test_tbl (id UInt16, reqid UInt32, name String) engine=MergeTree order by id"
-$CLICKHOUSE_CLIENT -q "INSERT INTO test_tbl SELECT * FROM file('key_ignore_case.json', 'JSONEachRow') SETTINGS input_format_json_ignore_key_case=true"
+$CLICKHOUSE_CLIENT -q "INSERT INTO test_tbl SELECT * FROM file('key_ignore_case.json', 'JSONEachRow') SETTINGS input_format_json_case_insensitive_column_matching=true"
 $CLICKHOUSE_CLIENT -q "select * from test_tbl"
-$CLICKHOUSE_CLIENT -q "drop table test_tbl"
\ No newline at end of file
+$CLICKHOUSE_CLIENT -q "drop table test_tbl"

From 8da2ef8287b8fd74f143897d8bb3ce72270e292d Mon Sep 17 00:00:00 2001
From: Igor Nikonov <igor@clickhouse.com>
Date: Thu, 25 Jul 2024 19:17:38 +0000
Subject: [PATCH 0804/1488] Fix: order by all with parallel replicas

---
 src/Analyzer/QueryTreeBuilder.cpp              |  7 ++++++-
 ...09_parallel_replicas_order_by_all.reference | 12 ++++++++++++
 .../03209_parallel_replicas_order_by_all.sql   | 18 ++++++++++++++++++
 3 files changed, 36 insertions(+), 1 deletion(-)
 create mode 100644 tests/queries/0_stateless/03209_parallel_replicas_order_by_all.reference
 create mode 100644 tests/queries/0_stateless/03209_parallel_replicas_order_by_all.sql

diff --git a/src/Analyzer/QueryTreeBuilder.cpp b/src/Analyzer/QueryTreeBuilder.cpp
index a62b6e56ac5..ed1227b0f00 100644
--- a/src/Analyzer/QueryTreeBuilder.cpp
+++ b/src/Analyzer/QueryTreeBuilder.cpp
@@ -268,6 +268,8 @@ QueryTreeNodePtr QueryTreeBuilder::buildSelectExpression(const ASTPtr & select_q
         }
     }
 
+    const auto enable_order_by_all = updated_context->getSettingsRef().enable_order_by_all;
+
     auto current_query_tree = std::make_shared<QueryNode>(std::move(updated_context), std::move(settings_changes));
 
     current_query_tree->setIsSubquery(is_subquery);
@@ -281,7 +283,10 @@ QueryTreeNodePtr QueryTreeBuilder::buildSelectExpression(const ASTPtr & select_q
     current_query_tree->setIsGroupByWithRollup(select_query_typed.group_by_with_rollup);
     current_query_tree->setIsGroupByWithGroupingSets(select_query_typed.group_by_with_grouping_sets);
     current_query_tree->setIsGroupByAll(select_query_typed.group_by_all);
-    current_query_tree->setIsOrderByAll(select_query_typed.order_by_all);
+    /// order_by_all flag in AST is set w/o consideration of `enable_order_by_all` setting
+    /// since SETTINGS section has not been parsed yet, - so, check the setting here
+    if (enable_order_by_all)
+        current_query_tree->setIsOrderByAll(select_query_typed.order_by_all);
     current_query_tree->setOriginalAST(select_query);
 
     auto current_context = current_query_tree->getContext();
diff --git a/tests/queries/0_stateless/03209_parallel_replicas_order_by_all.reference b/tests/queries/0_stateless/03209_parallel_replicas_order_by_all.reference
new file mode 100644
index 00000000000..fd453d088a6
--- /dev/null
+++ b/tests/queries/0_stateless/03209_parallel_replicas_order_by_all.reference
@@ -0,0 +1,12 @@
+-- { echoOn }
+SELECT a, b, all FROM order_by_all ORDER BY all SETTINGS enable_order_by_all = 0, allow_experimental_parallel_reading_from_replicas=0;
+B	3	10
+D	1	20
+A	2	30
+C	\N	40
+SELECT a, b, all FROM order_by_all ORDER BY all SETTINGS enable_order_by_all = 0, allow_experimental_parallel_reading_from_replicas=1;
+B	3	10
+D	1	20
+A	2	30
+C	\N	40
+DROP TABLE order_by_all SYNC;
diff --git a/tests/queries/0_stateless/03209_parallel_replicas_order_by_all.sql b/tests/queries/0_stateless/03209_parallel_replicas_order_by_all.sql
new file mode 100644
index 00000000000..46a3ab4d171
--- /dev/null
+++ b/tests/queries/0_stateless/03209_parallel_replicas_order_by_all.sql
@@ -0,0 +1,18 @@
+DROP TABLE IF EXISTS order_by_all SYNC;
+CREATE TABLE order_by_all
+(
+    a String,
+    b Nullable(Int32),
+    all UInt64
+)
+ENGINE = ReplicatedMergeTree('/clickhouse/{database}/test_03210', 'r1') ORDER BY tuple();
+
+INSERT INTO order_by_all VALUES ('B', 3, 10), ('C', NULL, 40), ('D', 1, 20), ('A', 2, 30);
+
+SET allow_experimental_parallel_reading_from_replicas=1, max_parallel_replicas=3, cluster_for_parallel_replicas='parallel_replicas';
+SET enable_order_by_all = 0;
+-- { echoOn }
+SELECT a, b, all FROM order_by_all ORDER BY all SETTINGS enable_order_by_all = 0, allow_experimental_parallel_reading_from_replicas=0;
+SELECT a, b, all FROM order_by_all ORDER BY all SETTINGS enable_order_by_all = 0, allow_experimental_parallel_reading_from_replicas=1;
+
+DROP TABLE order_by_all SYNC;

From 1ba4790511e1a06af8fb85e01767ce95866ee2a8 Mon Sep 17 00:00:00 2001
From: Blargian <shaunstruwig.ss@gmail.com>
Date: Thu, 25 Jul 2024 21:18:48 +0200
Subject: [PATCH 0805/1488] Review changes

---
 .../functions/type-conversion-functions.md    | 550 ++++++++++--------
 1 file changed, 310 insertions(+), 240 deletions(-)

diff --git a/docs/en/sql-reference/functions/type-conversion-functions.md b/docs/en/sql-reference/functions/type-conversion-functions.md
index 057083d317f..844d957d538 100644
--- a/docs/en/sql-reference/functions/type-conversion-functions.md
+++ b/docs/en/sql-reference/functions/type-conversion-functions.md
@@ -51,7 +51,7 @@ SETTINGS cast_keep_nullable = 1
 
 ## toInt8
 
-Converts an input value to a value of type `Int8`.
+Converts an input value to a value of type [`Int8`](../data-types/int-uint.md). Throws an exception in case of an error.
 
 **Syntax**
 
@@ -61,10 +61,20 @@ toInt8(expr)
 
 **Arguments**
 
-- `expr` — Expression returning a number or a string with the decimal representation of a number. [Expression](../syntax.md/#syntax-expressions).
+- `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions).
+
+Supported types:
+- (U)Int8/16/32/64/128/256
+- Float*
+- String representations of (U)Int8/16/32/128/256
+
+Unsupported types:
+- Float values `NaN` and `Inf` throw an exception.
+- String representations of binary and hexadecimal values, e.g. `SELECT toInt8('0xc0fe');`
 
 :::note
-Binary, octal, and hexadecimal representations of numbers are not supported. Leading zeroes are stripped.
+If the input value cannot be represented within the bounds of [Int8](../data-types/int-uint.md), the result over or under flows. This is not considered an error.  
+For example: `SELECT toInt8(128) == -128;`, `SELECT toInt8(128.0) == -128;`,  `SELECT toInt8('128') == -128;`.
 :::
 
 **Returned value**
@@ -72,11 +82,7 @@ Binary, octal, and hexadecimal representations of numbers are not supported. Lea
 - 8-bit integer value. [Int8](../data-types/int-uint.md).
 
 :::note
-Function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
-:::
-
-:::danger
-An exception is thrown for [NaN and Inf](../data-types/float.md/#data_type-float-nan-inf) arguments. Keep in mind [numeric conversions issues](#common-issues-with-data-conversion), when using this function.
+The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
 :::
 
 **Example**
@@ -106,32 +112,33 @@ Result:
 
 ## toInt8OrZero
 
-Like [`toInt8`](#toint8), it takes an argument of type [String](../data-types/string.md) and tries to parse it to type `Int8`. If unsuccessful, returns `0`.
+Like [`toInt8`](#toint8), this function converts an input value to a value of type [Int8](../data-types/int-uint.md) but returns `0` in case of an error.
 
 **Syntax**
 
 ```sql
-toInt8OrZero(expr)
+toInt8OrZero(x)
 ```
 
 **Arguments**
 
-- `expr` — Expression returning a number or a string with the decimal representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
+- `x` — A String representation of a number. [String](../data-types/string.md).
 
-:::note
-Binary, octal, and hexadecimal representations of numbers are not supported. Leading zeroes are stripped.
-:::
+Supported types:
+- String representations of (U)Int8/16/32/128/256
+
+Types for which `0` is returned:
+- String representations of ordinary Float32/64 values.
+- String representations of Float values `NaN` and `Inf`.
+- String representations of binary and hexadecimal values, e.g. `SELECT toInt8OrZero('0xc0fe');`.
+- If the input value cannot be represented within the bounds of [toInt16](../data-types/int-uint.md), and the result over or under flows.
 
 **Returned value**
 
 - 8-bit integer value if successful, otherwise `0`. [Int8](../data-types/int-uint.md).
 
 :::note
-Function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers. 
-:::
-
-:::danger
-An exception is thrown for [NaN and Inf](../data-types/float.md/#data_type-float-nan-inf) arguments. Keep in mind [numeric conversions issues](#common-issues-with-data-conversion), when using this function.
+The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers. 
 :::
 
 **Example**
@@ -160,32 +167,33 @@ Result:
 
 ## toInt8OrNull
 
-Like [`toInt8`](#toint8), takes an argument of type [String](../data-types/string.md) and tries to parse it to type `Int8`. If unsuccessful, returns `NULL`.
+Like [`toInt8`](#toint8), takes an argument of type [String](../data-types/string.md) and tries to parse it to type [`Int8`](../data-types/int-uint.md). If unsuccessful, returns [`NULL`](../data-types/nullable.md).
 
 **Syntax**
 
 ```sql
-toInt8OrNull(expr)
+toInt8OrNull(x)
 ```
 
 **Arguments**
 
-- `expr` — Expression returning a number or a string with the decimal representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
+- `x` — A String representation of a number. [String](../data-types/string.md).
 
-:::note
-Binary, octal, and hexadecimal representations of numbers are not supported. Leading zeroes are stripped.
-:::
+Supported types:
+- String representations of (U)Int8/16/32/128/256
+
+Types for which `\N` is returned:
+- String representations of ordinary Float32/64 values.
+- String representations of Float values `NaN` and `Inf`.
+- String representations of binary and hexadecimal values, e.g. `SELECT toInt8OrNull('0xc0fe');`.
+- If the input value cannot be represented within the bounds of [Int16](../data-types/int-uint.md), and the result over or under flows.
 
 **Returned value**
 
 - 8-bit integer value if successful, otherwise `NULL`. [Int8](../data-types/int-uint.md) / [NULL](../data-types/nullable.md).
 
 :::note
-Function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
-:::
-
-:::danger
-An exception is thrown for [NaN and Inf](../data-types/float.md/#data_type-float-nan-inf) arguments. Keep in mind [numeric conversions issues](#common-issues-with-data-conversion), when using this function.
+The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
 :::
 
 **Example**
@@ -212,7 +220,7 @@ Result:
 
 ## toInt8OrDefault
 
-Like [`toInt8`](#toint8), takes an argument of type [String](../data-types/string.md) and tries to parse it to type `Int8`. If unsuccessful, returns the default type value.
+Like [`toInt8`](#toint8), takes an argument of type [String](../data-types/string.md) and tries to parse it to type [`Int8`](../data-types/int-uint.md). If unsuccessful, returns the default type value.
 
 **Syntax**
 
@@ -222,26 +230,28 @@ toInt8OrDefault(expr, def)
 
 **Arguments**
 
-- `expr` — Expression returning a number or a string with the decimal representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
+- `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
 - `def` — The default value to return if parsing to type `Int8` is unsuccessful. [Int8](../data-types/int-uint.md).
 
-:::note
-Binary, octal, and hexadecimal representations of numbers are not supported. Leading zeroes are stripped.
-:::
+Supported types:
+- (U)Int8/16/32/64/128/256
+- Float*
+- String representations of (U)Int8/16/32/128/256
+
+Types for which the default value is returned:
+- Float values `NaN` and `Inf` return the default value.
+- String representations of binary and hexadecimal values, e.g. `SELECT toInt8OrDefault('0xc0fe', CAST('-1', 'Int8'));`
+- If the input value cannot be represented within the bounds of [Int8](../data-types/int-uint.md) and the result over or under flows.
 
 **Returned value**
 
 - 8-bit integer value if successful, otherwise returns the default value. [Int8](../data-types/int-uint.md).
 
 :::note
-- Function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
+- The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
 - The default value type should be the same as the cast type.
 :::
 
-:::danger
-An exception is thrown for [NaN and Inf](../data-types/float.md/#data_type-float-nan-inf) arguments. Keep in mind [numeric conversions issues](#common-issues-with-data-conversion), when using this function.
-:::
-
 **Example**
 
 Query:
@@ -268,7 +278,7 @@ Result:
 
 ## toInt16
 
-Converts an input value to a value of type `Int16`.
+Converts an input value to a value of type [`Int16`](../data-types/int-uint.md). Throws an exception in case of an error.
 
 **Syntax**
 
@@ -278,10 +288,20 @@ toInt16(expr)
 
 **Arguments**
 
-- `expr` — Expression returning a number or a string with the decimal representation of a number. [Expression](../syntax.md/#syntax-expressions).
+- `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions).
+
+Supported types:
+- (U)Int8/16/32/64/128/256
+- Float*
+- String representations of (U)Int8/16/32/128/256
+
+Unsupported types:
+- Float values `NaN` and `Inf` throw an exception.
+- String representations of binary and hexadecimal values, e.g. `SELECT toInt16('0xc0fe');`
 
 :::note
-Binary, octal, and hexadecimal representations of numbers are not supported. Leading zeroes are stripped.
+If the input value cannot be represented within the bounds of [toInt16](../data-types/int-uint.md), the result over or under flows. This is not considered an error.  
+For example: `SELECT toInt16(32768) == -32768;`, `SELECT toInt16(32768) == -32768;`,  `SELECT toInt16('32768') == -32768;`.
 :::
 
 **Returned value**
@@ -289,11 +309,7 @@ Binary, octal, and hexadecimal representations of numbers are not supported. Lea
 - 16-bit integer value. [Int16](../data-types/int-uint.md).
 
 :::note
-Function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
-:::
-
-:::danger
-An exception is thrown for [NaN and Inf](../data-types/float.md/#data_type-float-nan-inf) arguments. Keep in mind [numeric conversions issues](#common-issues-with-data-conversion), when using this function.
+The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
 :::
 
 **Example**
@@ -323,32 +339,33 @@ Result:
 
 ## toInt16OrZero
 
-Like [`toInt16`](#toint16), takes an argument of type [String](../data-types/string.md) and tries to parse it to type `Int16`. If unsuccessful, returns `0`.
+Like [`toInt16`](#toint16), this function converts an input value to a value of type [Int16](../data-types/int-uint.md) but returns `0` in case of an error.
 
 **Syntax**
 
 ```sql
-toInt16OrZero(expr)
+toInt16OrZero(x)
 ```
 
 **Arguments**
 
-- `expr` — Expression returning a number or a string with the decimal representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
+- `x` — A String representation of a number. [String](../data-types/string.md).
 
-:::note
-Binary, octal, and hexadecimal representations of numbers are not supported. Leading zeroes are stripped.
-:::
+Supported types:
+- String representations of (U)Int8/16/32/128/256
+
+Types for which `0` is returned:
+- String representations of ordinary Float32/64 values.
+- String representations of Float values `NaN` and `Inf`.
+- String representations of binary and hexadecimal values, e.g. `SELECT toInt16OrZero('0xc0fe');`.
+- If the input value cannot be represented within the bounds of [Int16](../data-types/int-uint.md) and the result over or under flows.
 
 **Returned value**
 
 - 16-bit integer value if successful, otherwise `0`. [Int16](../data-types/int-uint.md).
 
 :::note
-Function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
-:::
-
-:::danger
-An exception is thrown for [NaN and Inf](../data-types/float.md/#data_type-float-nan-inf) arguments. Keep in mind [numeric conversions issues](#common-issues-with-data-conversion), when using this function.
+The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
 :::
 
 **Example**
@@ -377,32 +394,33 @@ Result:
 
 ## toInt16OrNull
 
-Like [`toInt16`](#toint16), takes an argument of type [String](../data-types/string.md) and tries to parse it to type `Int16`. If unsuccessful, returns `NULL`.
+Like [`toInt16`](#toint16), takes an argument of type [String](../data-types/string.md) and tries to parse it to type [`Int16`](../data-types/int-uint.md). If unsuccessful, returns [`NULL`](../data-types/nullable.md).
 
 **Syntax**
 
 ```sql
-toInt16OrNull(expr)
+toInt16OrNull(x)
 ```
 
 **Arguments**
 
-- `expr` — Expression returning a number or a string with the decimal representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
+- `x` — A String representation of a number. [String](../data-types/string.md).
 
-:::note
-Binary, octal, and hexadecimal representations of numbers are not supported. Leading zeroes are stripped.
-:::
+Supported types:
+- String representations of (U)Int8/16/32/128/256
+
+Types for which `\N` is returned:
+- String representations of ordinary Float32/64 values.
+- String representations of Float values `NaN` and `Inf`.
+- String representations of binary and hexadecimal values, e.g. `SELECT toInt16OrNull('0xc0fe');`.
+- If the input value cannot be represented within the bounds of [Int16](../data-types/int-uint.md) and the result over or under flows.
 
 **Returned value**
 
 - 16-bit integer value if successful, otherwise `NULL`. [Int16](../data-types/int-uint.md) / [NULL](../data-types/nullable.md).
 
 :::note
-Function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
-:::
-
-:::danger
-An exception is thrown for [NaN and Inf](../data-types/float.md/#data_type-float-nan-inf) arguments. Keep in mind [numeric conversions issues](#common-issues-with-data-conversion), when using this function.
+The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
 :::
 
 **Example**
@@ -431,7 +449,7 @@ Result:
 
 ## toInt16OrDefault
 
-Like [`toInt16`](#toint16), takes an argument of type [String](../data-types/string.md) and tries to parse it to type `Int16`. If unsuccessful, returns the default type value.
+Like [`toInt16`](#toint16), takes an argument of type [String](../data-types/string.md) and tries to parse it to type [`Int16`](../data-types/int-uint.md). If unsuccessful, returns the default type value.
 
 **Syntax**
 
@@ -441,26 +459,28 @@ toInt16OrDefault(expr, def)
 
 **Arguments**
 
-- `expr` — Expression returning a number or a string with the decimal representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
+- `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
 - `def` — The default value to return if parsing to type `Int16` is unsuccessful. [Int8](../data-types/int-uint.md).
 
-:::note
-Binary, octal, and hexadecimal representations of numbers are not supported. Leading zeroes are stripped.
-:::
+Supported types:
+- (U)Int8/16/32/64/128/256
+- Float*
+- String representations of (U)Int8/16/32/128/256
+
+Types for which the default value is returned:
+- Float values `NaN` and `Inf` return the default value.
+- String representations of binary and hexadecimal values, e.g. `SELECT toInt16OrDefault('0xc0fe', CAST('-1', 'Int16'));`
+- If the input value cannot be represented within the bounds of [Int16](../data-types/int-uint.md) and the result over or under flows.
 
 **Returned value**
 
 - 16-bit integer value if successful, otherwise returns the default value. [Int16](../data-types/int-uint.md).
 
 :::note
-- Function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
+- The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
 - The default value type should be the same as the cast type.
 :::
 
-:::danger
-An exception is thrown for [NaN and Inf](../data-types/float.md/#data_type-float-nan-inf) arguments. Keep in mind [numeric conversions issues](#common-issues-with-data-conversion), when using this function.
-:::
-
 **Example**
 
 Query:
@@ -485,7 +505,7 @@ Result:
 
 ## toInt32
 
-Converts an input value to a value of type `Int32`.
+Converts an input value to a value of type [`Int32`](../data-types/int-uint.md). Throws an exception in case of an error.
 
 **Syntax**
 
@@ -495,10 +515,25 @@ toInt32(expr)
 
 **Arguments**
 
-- `expr` — Expression returning a number or a string with the decimal representation of a number. [Expression](../syntax.md/#syntax-expressions).
+- `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions).
+
+Supported types:
+- (U)Int8/16/32/64/128/256
+- Float*
+- String representations of (U)Int8/16/32/128/256
+
+Unsupported types:
+- Float values `NaN` and `Inf` throw an exception.
+- String representations of binary and hexadecimal values, e.g. `SELECT toInt32('0xc0fe');`
 
 :::note
-Binary, octal, and hexadecimal representations of numbers are not supported. Leading zeroes are stripped.
+If the input value cannot be represented within the bounds of [toInt16](../data-types/int-uint.md), the result over or under flows. This is not considered an error.  
+For example: 
+```
+SELECT toInt32(2147483648) == -2147483648;
+SELECT toInt32(2147483648.0) == -2147483648;
+SELECT toInt32('2147483648') == -2147483648;
+```
 :::
 
 **Returned value**
@@ -506,11 +541,7 @@ Binary, octal, and hexadecimal representations of numbers are not supported. Lea
 - 32-bit integer value. [Int32](../data-types/int-uint.md).
 
 :::note
-Function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
-:::
-
-:::danger
-An exception is thrown for [NaN and Inf](../data-types/float.md/#data_type-float-nan-inf) arguments. Keep in mind [numeric conversions issues](#common-issues-with-data-conversion), when using this function.
+The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
 :::
 
 **Example**
@@ -540,32 +571,34 @@ Result:
 
 ## toInt32OrZero
 
-Like [`toInt32`](#toint32), takes an argument of type [String](../data-types/string.md) and tries to parse it to type `Int32`. If unsuccessful, returns `0`.
+Like [`toInt32`](#toint32), this function converts an input value to a value of type [Int8](../data-types/int-uint.md) but returns `0` in case of an error.
 
 **Syntax**
 
 ```sql
-toInt32OrZero(expr)
+toInt32OrZero(x)
 ```
 
 **Arguments**
 
-- `expr` — Expression returning a number or a string with the decimal representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
+- `x` — A String representation of a number. [String](../data-types/string.md).
+
+Supported types:
+- String representations of (U)Int8/16/32/128/256
+
+Types for which `0` is returned:
+- String representations of ordinary Float32/64 values.
+- String representations of Float values `NaN` and `Inf`.
+- String representations of binary and hexadecimal values, e.g. `SELECT toInt32OrZero('0xc0fe');`.
+- If the input value cannot be represented within the bounds of [Int32](../data-types/int-uint.md) and the result over or under flows.
 
-:::note
-Binary, octal, and hexadecimal representations of numbers are not supported. Leading zeroes are stripped.
-:::
 
 **Returned value**
 
 - 32-bit integer value if successful, otherwise `0`. [Int32](../data-types/int-uint.md)
 
 :::note
-Function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncate fractional digits of numbers.
-:::
-
-:::danger
-An exception is thrown for [NaN and Inf](../data-types/float.md/#data_type-float-nan-inf) arguments. Keep in mind [numeric conversions issues](#common-issues-with-data-conversion), when using this function.
+The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncate fractional digits of numbers.
 :::
 
 **Example**
@@ -588,35 +621,36 @@ Result:
 - [`toInt32`](#toint32).
 - [`toInt32OrNull`](#toint32ornull).
 - [`toInt32OrDefault`](#toint32ordefault).
-- 
+
 ## toInt32OrNull
 
-Like [`toInt32`](#toint32), takes an argument of type [String](../data-types/string.md) and tries to parse it to type `Int32`. If unsuccessful, returns `NULL`.
+Like [`toInt32`](#toint32), takes an argument of type [String](../data-types/string.md) and tries to parse it to type [`Int32`](../data-types/int-uint.md). If unsuccessful, returns [`NULL`](../data-types/nullable.md).
 
 **Syntax**
 
 ```sql
-toInt32OrNull(expr)
+toInt32OrNull(x)
 ```
 
 **Arguments**
 
-- `expr` — Expression returning a number or a string with the decimal representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
+- `x` — A String representation of a number. [String](../data-types/string.md).
 
-:::note
-Binary, octal, and hexadecimal representations of numbers are not supported. Leading zeroes are stripped.
-:::
+Supported types:
+- String representations of (U)Int8/16/32/128/256
+
+Types for which `\N` is returned:
+- String representations of ordinary Float32/64 values.
+- String representations of Float values `NaN` and `Inf`.
+- String representations of binary and hexadecimal values, e.g. `SELECT toInt32OrNull('0xc0fe');`.
+- If the input value cannot be represented within the bounds of [Int32](../data-types/int-uint.md) and the result over or under flows.
 
 **Returned value**
 
 - 32-bit integer value if successful, otherwise `NULL`. [Int32](../data-types/int-uint.md) / [NULL](../data-types/nullable.md).
 
 :::note
-Function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
-:::
-
-:::danger
-An exception is thrown for [NaN and Inf](../data-types/float.md/#data_type-float-nan-inf) arguments. Keep in mind [numeric conversions issues](#common-issues-with-data-conversion), when using this function.
+The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
 :::
 
 **Example**
@@ -643,7 +677,7 @@ Result:
 
 ## toInt32OrDefault
 
-Like [`toInt32`](#toint32), takes an argument of type [String](../data-types/string.md) and tries to parse it to type `Int32`. If unsuccessful, returns the default type value.
+Like [`toInt32`](#toint32), takes an argument of type [String](../data-types/string.md) and tries to parse it to type [`Int32`](../data-types/int-uint.md). If unsuccessful, returns the default type value.
 
 **Syntax**
 
@@ -653,24 +687,26 @@ toInt32OrDefault(expr, def)
 
 **Arguments**
 
-- `expr` — Expression returning a number or a string with the decimal representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
+- `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
 - `def` — The default value to return if parsing to type `Int32` is unsuccessful. [Int32](../data-types/int-uint.md).
 
-:::note
-Binary, octal, and hexadecimal representations of numbers are not supported. Leading zeroes are stripped.
-:::
+Supported types:
+- (U)Int8/16/32/64/128/256
+- Float*
+- String representations of (U)Int8/16/32/128/256
+
+Types for which the default value is returned:
+- Float values `NaN` and `Inf` return the default value.
+- String representations of binary and hexadecimal values, e.g. `SELECT toInt32OrDefault('0xc0fe', CAST('-1', 'Int32'));`
+- If the input value cannot be represented within the bounds of [Int32](../data-types/int-uint.md) and the result over or under flows.
 
 **Returned value**
 
 - 32-bit integer value if successful, otherwise returns the default value. [Int32](../data-types/int-uint.md).
 
 :::note
-- Function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
+- The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
 - The default value type should be the same as the cast type.
-  :::
-
-:::danger
-An exception is thrown for [NaN and Inf](../data-types/float.md/#data_type-float-nan-inf) arguments. Keep in mind [numeric conversions issues](#common-issues-with-data-conversion), when using this function.
 :::
 
 **Example**
@@ -697,7 +733,7 @@ Result:
 
 ## toInt64
 
-Converts an input value to a value of type `Int64`.
+Converts an input value to a value of type [`Int64`](../data-types/int-uint.md). Throws an exception in case of an error.
 
 **Syntax**
 
@@ -707,10 +743,26 @@ toInt64(expr)
 
 **Arguments**
 
-- `expr` — Expression returning a number or a string with the decimal representation of a number. [Expression](../syntax.md/#syntax-expressions).
+- `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions).
+
+Supported types:
+- (U)Int8/16/32/64/128/256
+- Float*
+- String representations of (U)Int8/16/32/128/256
+
+Unsupported types:
+- Float values `NaN` and `Inf` throw an exception.
+- String representations of binary and hexadecimal values, e.g. `SELECT toInt64('0xc0fe');`
 
 :::note
-Binary, octal, and hexadecimal representations of numbers are not supported. Leading zeroes are stripped.
+If the input value cannot be represented within the bounds of [Int16](../data-types/int-uint.md), the result over or under flows. This is not considered an error.  
+For example: 
+
+```
+SELECT toInt64(9223372036854775808) == -9223372036854775808;
+SELECT toInt64(9223372036854775808.0) == -9223372036854775808;
+SELECT toInt64('9223372036854775808') == --9223372036854775808;
+```
 :::
 
 **Returned value**
@@ -718,11 +770,7 @@ Binary, octal, and hexadecimal representations of numbers are not supported. Lea
 - 64-bit integer value. [Int64](../data-types/int-uint.md). [Int64](../data-types/int-uint.md).
 
 :::note
-Function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
-:::
-
-:::danger
-An exception is thrown for [NaN and Inf](../data-types/float.md/#data_type-float-nan-inf) arguments. Keep in mind [numeric conversions issues](#common-issues-with-data-conversion), when using this function.
+The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
 :::
 
 **Example**
@@ -752,32 +800,33 @@ Result:
 
 ## toInt64OrZero
 
-Like [`toInt64`](#toint64), takes an argument of type [String](../data-types/string.md) and tries to parse it to type `Int64`. If unsuccessful, returns `0`.
+Like [`toInt64`](#toint64), this function converts an input value to a value of type [Int8](../data-types/int-uint.md) but returns `0` in case of an error.
 
 **Syntax**
 
 ```sql
-toInt64OrZero(expr)
+toInt64OrZero(x)
 ```
 
 **Arguments**
 
-- `expr` — Expression returning a number or a string with the decimal representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
+- `x` — A String representation of a number. [String](../data-types/string.md).
 
-:::note
-Binary, octal, and hexadecimal representations of numbers are not supported. Leading zeroes are stripped.
-:::
+Supported types:
+- String representations of (U)Int8/16/32/128/256
+
+Types for which `0` is returned:
+- String representations of ordinary Float32/64 values.
+- String representations of Float values `NaN` and `Inf`.
+- String representations of binary and hexadecimal values, e.g. `SELECT toInt64OrZero('0xc0fe');`.
+- If the input value cannot be represented within the bounds of [Int64](../data-types/int-uint.md) and the result over or under flows.
 
 **Returned value**
 
 - 64-bit integer value if successful, otherwise `0`. [Int64](../data-types/int-uint.md).
 
 :::note
-Function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
-:::
-
-:::danger
-An exception is thrown for [NaN and Inf](../data-types/float.md/#data_type-float-nan-inf) arguments. Keep in mind [numeric conversions issues](#common-issues-with-data-conversion), when using this function.
+The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
 :::
 
 **Example**
@@ -806,32 +855,33 @@ Result:
 
 ## toInt64OrNull
 
-Like [`toInt64`], takes an argument of type [String](../data-types/string.md) and tries to parse it to type `Int64`. If unsuccessful, returns `NULL`.
+Like [`toInt64`], takes an argument of type [String](../data-types/string.md) and tries to parse it to type [`Int64`](../data-types/nullable.md). If unsuccessful, returns [`NULL`](../data-types/nullable.md).
 
 **Syntax**
 
 ```sql
-toInt64OrNull(expr)
+toInt64OrNull(x)
 ```
 
 **Arguments**
 
-- `expr` — Expression returning a number or a string with the decimal representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
+- `x` — A String representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
 
-:::note
-Binary, octal, and hexadecimal representations of numbers are not supported. Leading zeroes are stripped.
-:::
+Supported types:
+- String representations of (U)Int8/16/32/128/256
+
+Types for which `\N` is returned:
+- String representations of ordinary Float32/64 values.
+- String representations of Float values `NaN` and `Inf`.
+- String representations of binary and hexadecimal values, e.g. `SELECT toInt64OrNull('0xc0fe');`.
+- If the input value cannot be represented within the bounds of [Int64](../data-types/int-uint.md) and the result over or under flows.
 
 **Returned value**
 
 - Integer value of type `Int64` if successful, otherwise `NULL`. [Int64](../data-types/int-uint.md) / [NULL](../data-types/nullable.md).
 
 :::note
-Function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
-:::
-
-:::danger
-An exception is thrown for [NaN and Inf](../data-types/float.md/#data_type-float-nan-inf) arguments. Keep in mind [numeric conversions issues](#common-issues-with-data-conversion), when using this function.
+The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
 :::
 
 **Example**
@@ -860,7 +910,7 @@ Result:
 
 ## toInt64OrDefault
 
-Like [`toInt64`](#toint64), takes an argument of type [String](../data-types/string.md) and tries to parse it to type `Int64`. If unsuccessful, returns the default type value.
+Like [`toInt64`](#toint64), takes an argument of type [String](../data-types/string.md) and tries to parse it to type [`Int64`](../data-types/nullable.md). If unsuccessful, returns the default type value.
 
 **Syntax**
 
@@ -870,24 +920,26 @@ toInt64OrDefault(expr, def)
 
 **Arguments**
 
-- `expr` — Expression returning a number or a string with the decimal representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
+- `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
 - `def` — The default value to return if parsing to type `Int64` is unsuccessful. [Int64](../data-types/int-uint.md).
 
-:::note
-Binary, octal, and hexadecimal representations of numbers are not supported. Leading zeroes are stripped.
-:::
+Supported types:
+- (U)Int8/16/32/64/128/256
+- Float*
+- String representations of (U)Int8/16/32/128/256
+
+Types for which the default value is returned:
+- Float values `NaN` and `Inf` return the default value.
+- String representations of binary and hexadecimal values, e.g. `SELECT toInt64OrDefault('0xc0fe', CAST('-1', 'Int64'));`
+- If the input value cannot be represented within the bounds of [Int64](../data-types/int-uint.md) and the result over or under flows.
 
 **Returned value**
 
 - Integer value of type `Int64` if successful, otherwise returns the default value. [Int64](../data-types/int-uint.md).
 
 :::note
-- Function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
+- The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
 - The default value type should be the same as the cast type.
-  :::
-
-:::danger
-An exception is thrown for [NaN and Inf](../data-types/float.md/#data_type-float-nan-inf) arguments. Keep in mind [numeric conversions issues](#common-issues-with-data-conversion), when using this function.
 :::
 
 **Example**
@@ -916,7 +968,7 @@ Result:
 
 ## toInt128
 
-Converts an input value to a value of type `Int128`.
+Converts an input value to a value of type [`Int128`](../data-types/int-uint.md). Throws an exception in case of an error.
 
 **Syntax**
 
@@ -926,10 +978,19 @@ toInt128(expr)
 
 **Arguments**
 
-- `expr` — Expression returning a number or a string with the decimal representation of a number. [Expression](../syntax.md/#syntax-expressions).
+- `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions).
+
+Supported types:
+- (U)Int8/16/32/64/128/256
+- Float*
+- String representations of (U)Int8/16/32/128/256
+
+Unsupported types:
+- Float values `NaN` and `Inf` throw an exception.
+- String representations of binary and hexadecimal values, e.g. `SELECT toInt128('0xc0fe');`
 
 :::note
-Binary, octal, and hexadecimal representations of numbers are not supported. Leading zeroes are stripped.
+If the input value cannot be represented within the bounds of [Int128](../data-types/int-uint.md), the result over or under flows. This is not considered an error.
 :::
 
 **Returned value**
@@ -937,11 +998,7 @@ Binary, octal, and hexadecimal representations of numbers are not supported. Lea
 - 128-bit integer value. [Int128](../data-types/int-uint.md).
 
 :::note
-Function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
-:::
-
-:::danger
-An exception is thrown for [NaN and Inf](../data-types/float.md/#data_type-float-nan-inf) arguments. Keep in mind [numeric conversions issues](#common-issues-with-data-conversion), when using this function.
+The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
 :::
 
 **Example**
@@ -971,7 +1028,7 @@ Result:
 
 ## toInt128OrZero
 
-Like [`toInt128`](#toint128), takes an argument of type [String](../data-types/string.md) and tries to parse it to type `Int128`. If unsuccessful, returns `0`.
+Like [`toInt128`](#toint128), this function converts an input value to a value of type [Int8](../data-types/int-uint.md) but returns `0` in case of an error.
 
 **Syntax**
 
@@ -981,22 +1038,23 @@ toInt128OrZero(expr)
 
 **Arguments**
 
-- `expr` — Expression returning a number or a string with the decimal representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
+- `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
 
-:::note
-Binary, octal, and hexadecimal representations of numbers are not supported. Leading zeroes are stripped.
-:::
+Supported types:
+- String representations of (U)Int8/16/32/128/256
+
+Types for which `0` is returned:
+- String representations of ordinary Float32/64 values.
+- String representations of Float values `NaN` and `Inf`.
+- String representations of binary and hexadecimal values, e.g. `SELECT toInt128OrZero('0xc0fe');`.
+- If the input value cannot be represented within the bounds of [Int128](../data-types/int-uint.md) and the result over or under flows.
 
 **Returned value**
 
 - 128-bit integer value if successful, otherwise `0`. [Int128](../data-types/int-uint.md).
 
 :::note
-Function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
-:::
-
-:::danger
-An exception is thrown for [NaN and Inf](../data-types/float.md/#data_type-float-nan-inf) arguments. Keep in mind [numeric conversions issues](#common-issues-with-data-conversion), when using this function.
+The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
 :::
 
 **Example**
@@ -1025,32 +1083,33 @@ Result:
 
 ## toInt128OrNull
 
-Like [`toInt128`](#toint128), takes an argument of type [String](../data-types/string.md) and tries to parse it to type `Int128`. If unsuccessful, returns `NULL`.
+Like [`toInt128`](#toint128), takes an argument of type [String](../data-types/string.md) and tries to parse it to type [`Int128`](../data-types/int-uint.md). If unsuccessful, returns [`NULL`](../data-types/nullable.md).
 
 **Syntax**
 
 ```sql
-toInt128OrNull(expr)
+toInt128OrNull(x)
 ```
 
 **Arguments**
 
-- `expr` — Expression returning a number or a string with the decimal representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
+- `x` — A String representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
 
-:::note
-Binary, octal, and hexadecimal representations of numbers are not supported. Leading zeroes are stripped.
-:::
+Supported types:
+- String representations of (U)Int8/16/32/128/256
+
+Types for which `\N` is returned:
+- String representations of ordinary Float32/64 values.
+- String representations of Float values `NaN` and `Inf`.
+- String representations of binary and hexadecimal values, e.g. `SELECT toInt128OrNull('0xc0fe');`.
+- If the input value cannot be represented within the bounds of [Int128](../data-types/int-uint.md) and the result over or under flows.
 
 **Returned value**
 
 - 128-bit integer value if successful, otherwise `NULL`. [Int128](../data-types/int-uint.md) / [NULL](../data-types/nullable.md).
 
 :::note
-Function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
-:::
-
-:::danger
-An exception is thrown for [NaN and Inf](../data-types/float.md/#data_type-float-nan-inf) arguments. Keep in mind [numeric conversions issues](#common-issues-with-data-conversion), when using this function.
+The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
 :::
 
 **Example**
@@ -1079,7 +1138,7 @@ Result:
 
 ## toInt128OrDefault
 
-Like [`toInt128`](#toint128), takes an argument of type [String](../data-types/string.md) and tries to parse it to type `Int128`. If unsuccessful, returns the default type value.
+Like [`toInt128`](#toint128), takes an argument of type [String](../data-types/string.md) and tries to parse it to type [`Int128`](../data-types/int-uint.md). If unsuccessful, returns the default type value.
 
 **Syntax**
 
@@ -1089,26 +1148,28 @@ toInt128OrDefault(expr, def)
 
 **Arguments**
 
-- `expr` — Expression returning a number or a string with the decimal representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
+- `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
 - `def` — The default value to return if parsing to type `Int128` is unsuccessful. [Int128](../data-types/int-uint.md).
 
-:::note
-Binary, octal, and hexadecimal representations of numbers are not supported. Leading zeroes are stripped.
-:::
+Supported types:
+- (U)Int8/16/32/64/128/256
+- Float*
+- String representations of (U)Int8/16/32/128/256
+
+Types for which the default value is returned:
+- Float values `NaN` and `Inf` return the default value.
+- String representations of binary and hexadecimal values, e.g. `SELECT toInt128OrDefault('0xc0fe', CAST('-1', 'Int128'));`
+- If the input value cannot be represented within the bounds of [Int128](../data-types/int-uint.md) and the result over or under flows.
 
 **Returned value**
 
 - 128-bit integer value if successful, otherwise returns the default value. [Int128](../data-types/int-uint.md).
 
 :::note
-- Function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
+- The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
 - The default value type should be the same as the cast type.
 :::
 
-:::danger
-An exception is thrown for [NaN and Inf](../data-types/float.md/#data_type-float-nan-inf) arguments. Keep in mind [numeric conversions issues](#common-issues-with-data-conversion), when using this function.
-:::
-
 **Example**
 
 Query:
@@ -1135,7 +1196,7 @@ Result:
 
 ## toInt256
 
-Converts an input value to a value of type `Int256`.
+Converts an input value to a value of type [`Int256`](../data-types/int-uint.md). Throws an exception in case of an error.
 
 **Syntax**
 
@@ -1145,10 +1206,19 @@ toInt256(expr)
 
 **Arguments**
 
-- `expr` — Expression returning a number or a string with the decimal representation of a number. [Expression](../syntax.md/#syntax-expressions).
+- `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions).
+
+Supported types:
+- (U)Int8/16/32/64/128/256
+- Float*
+- String representations of (U)Int8/16/32/128/256
+
+Unsupported types:
+- Float values `NaN` and `Inf` throw an exception.
+- String representations of binary and hexadecimal values, e.g. `SELECT toInt256('0xc0fe');`
 
 :::note
-Binary, octal, and hexadecimal representations of numbers are not supported. Leading zeroes are stripped.
+If the input value cannot be represented within the bounds of [Int256](../data-types/int-uint.md), the result over or under flows. This is not considered an error.
 :::
 
 **Returned value**
@@ -1156,11 +1226,7 @@ Binary, octal, and hexadecimal representations of numbers are not supported. Lea
 - 256-bit integer value. [Int256](../data-types/int-uint.md).
 
 :::note
-Function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
-:::
-
-:::danger
-An exception is thrown for [NaN and Inf](../data-types/float.md/#data_type-float-nan-inf) arguments. Keep in mind [numeric conversions issues](#common-issues-with-data-conversion), when using this function.
+The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
 :::
 
 **Example**
@@ -1190,32 +1256,33 @@ Result:
 
 ## toInt256OrZero
 
-Like [`toInt256`](#toint256), takes an argument of type [String](../data-types/string.md) and tries to parse it to type `Int256`. If unsuccessful, returns `0`.
+Like [`toInt256`](#toint256), this function converts an input value to a value of type [Int8](../data-types/int-uint.md) but returns `0` in case of an error.
 
 **Syntax**
 
 ```sql
-toInt256OrZero(expr)
+toInt256OrZero(x)
 ```
 
 **Arguments**
 
-- `expr` — Expression returning a number or a string with the decimal representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
+- `x` — A String representation of a number. [String](../data-types/string.md).
 
-:::note
-Binary, octal, and hexadecimal representations of numbers are not supported. Leading zeroes are stripped.
-:::
+Supported types:
+- String representations of (U)Int8/16/32/128/256
+
+Types for which `0` is returned:
+- String representations of ordinary Float32/64 values.
+- String representations of Float values `NaN` and `Inf`.
+- String representations of binary and hexadecimal values, e.g. `SELECT toInt256OrZero('0xc0fe');`.
+- If the input value cannot be represented within the bounds of [Int256](../data-types/int-uint.md) and the result over or under flows.
 
 **Returned value**
 
 - 256-bit integer value if successful, otherwise `0`. [Int256](../data-types/int-uint.md).
 
 :::note
-Functions uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
-:::
-
-:::danger
-An exception is thrown for [NaN and Inf](../data-types/float.md/#data_type-float-nan-inf) arguments. Keep in mind [numeric conversions issues](#common-issues-with-data-conversion), when using this function.
+The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
 :::
 
 **Example**
@@ -1244,32 +1311,33 @@ Result:
 
 ## toInt256OrNull
 
-Like [`toInt256`](#toint256), takes an argument of type [String](../data-types/string.md) and tries to parse it to type `Int256`. If unsuccessful, returns `NULL`.
+Like [`toInt256`](#toint256), takes an argument of type [String](../data-types/string.md) and tries to parse it to type [`Int256`](../data-types/int-uint.md). If unsuccessful, returns [`NULL`](../data-types/nullable.md).
 
 **Syntax**
 
 ```sql
-toInt256OrNull(expr)
+toInt256OrNull(x)
 ```
 
 **Arguments**
 
-- `expr` — Expression returning a number or a string with the decimal representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
+- `x` — A String representation of a number. [String](../data-types/string.md).
 
-:::note
-Binary, octal, and hexadecimal representations of numbers are not supported. Leading zeroes are stripped.
-:::
+Supported types:
+- String representations of (U)Int8/16/32/128/256
+
+Types for which `\N` is returned:
+- String representations of ordinary Float32/64 values.
+- String representations of Float values `NaN` and `Inf`.
+- String representations of binary and hexadecimal values, e.g. `SELECT toInt256OrNull('0xc0fe');`.
+- If the input value cannot be represented within the bounds of [Int256](../data-types/int-uint.md) and the result over or under flows.
 
 **Returned value**
 
 - 256-bit integer value if successful, otherwise `NULL`. [Int256](../data-types/int-uint.md) / [NULL](../data-types/nullable.md).
 
 :::note
-Function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
-:::
-
-:::danger
-An exception is thrown for [NaN and Inf](../data-types/float.md/#data_type-float-nan-inf) arguments. Keep in mind [numeric conversions issues](#common-issues-with-data-conversion), when using this function.
+The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
 :::
 
 **Example**
@@ -1298,7 +1366,7 @@ Result:
 
 ## toInt256OrDefault
 
-Like [`toInt256`](#toint256), takes an argument of type [String](../data-types/string.md) and tries to parse it to type `Int256`. If unsuccessful, returns the default type value.
+Like [`toInt256`](#toint256), takes an argument of type [String](../data-types/string.md) and tries to parse it to type [`Int256`](../data-types/int-uint.md). If unsuccessful, returns the default type value.
 
 **Syntax**
 
@@ -1308,26 +1376,28 @@ toInt256OrDefault(expr, def)
 
 **Arguments**
 
-- `expr` — Expression returning a number or a string with the decimal representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
+- `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
 - `def` — The default value to return if parsing to type `Int256` is unsuccessful. [Int256](../data-types/int-uint.md).
 
-:::note
-Binary, octal, and hexadecimal representations of numbers are not supported. Leading zeroes are stripped.
-:::
+Supported types:
+- (U)Int8/16/32/64/128/256
+- Float*
+- String representations of (U)Int8/16/32/128/256
+
+Types for which the default value is returned:
+- Float values `NaN` and `Inf` return the default value.
+- String representations of binary and hexadecimal values, e.g. `SELECT toInt128OrDefault('0xc0fe', CAST('-1', 'Int256'));`
+- If the input value cannot be represented within the bounds of [Int256](../data-types/int-uint.md) and the result over or under flows.
 
 **Returned value**
 
 - 256-bit integer value if successful, otherwise returns the default value. [Int256](../data-types/int-uint.md).
 
 :::note
-- Function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
+- The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
 - The default value type should be the same as the cast type.
 :::
 
-:::danger
-An exception is thrown for [NaN and Inf](../data-types/float.md/#data_type-float-nan-inf) arguments. Keep in mind [numeric conversions issues](#common-issues-with-data-conversion), when using this function.
-:::
-
 **Example**
 
 Query:

From 3c1004aee4a3b1f3e1b0bd91a1b02c6c9e16c832 Mon Sep 17 00:00:00 2001
From: Blargian <shaunstruwig.ss@gmail.com>
Date: Thu, 25 Jul 2024 21:26:14 +0200
Subject: [PATCH 0806/1488] Fix typo

---
 .../functions/type-conversion-functions.md           | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/docs/en/sql-reference/functions/type-conversion-functions.md b/docs/en/sql-reference/functions/type-conversion-functions.md
index 844d957d538..4326753216e 100644
--- a/docs/en/sql-reference/functions/type-conversion-functions.md
+++ b/docs/en/sql-reference/functions/type-conversion-functions.md
@@ -131,7 +131,7 @@ Types for which `0` is returned:
 - String representations of ordinary Float32/64 values.
 - String representations of Float values `NaN` and `Inf`.
 - String representations of binary and hexadecimal values, e.g. `SELECT toInt8OrZero('0xc0fe');`.
-- If the input value cannot be represented within the bounds of [toInt16](../data-types/int-uint.md), and the result over or under flows.
+- If the input value cannot be represented within the bounds of [Int8](../data-types/int-uint.md), and the result over or under flows.
 
 **Returned value**
 
@@ -186,7 +186,7 @@ Types for which `\N` is returned:
 - String representations of ordinary Float32/64 values.
 - String representations of Float values `NaN` and `Inf`.
 - String representations of binary and hexadecimal values, e.g. `SELECT toInt8OrNull('0xc0fe');`.
-- If the input value cannot be represented within the bounds of [Int16](../data-types/int-uint.md), and the result over or under flows.
+- If the input value cannot be represented within the bounds of [Int8](../data-types/int-uint.md), and the result over or under flows.
 
 **Returned value**
 
@@ -300,7 +300,7 @@ Unsupported types:
 - String representations of binary and hexadecimal values, e.g. `SELECT toInt16('0xc0fe');`
 
 :::note
-If the input value cannot be represented within the bounds of [toInt16](../data-types/int-uint.md), the result over or under flows. This is not considered an error.  
+If the input value cannot be represented within the bounds of [Int16](../data-types/int-uint.md), the result over or under flows. This is not considered an error.  
 For example: `SELECT toInt16(32768) == -32768;`, `SELECT toInt16(32768) == -32768;`,  `SELECT toInt16('32768') == -32768;`.
 :::
 
@@ -527,7 +527,7 @@ Unsupported types:
 - String representations of binary and hexadecimal values, e.g. `SELECT toInt32('0xc0fe');`
 
 :::note
-If the input value cannot be represented within the bounds of [toInt16](../data-types/int-uint.md), the result over or under flows. This is not considered an error.  
+If the input value cannot be represented within the bounds of [Int32](../data-types/int-uint.md), the result over or under flows. This is not considered an error.  
 For example: 
 ```
 SELECT toInt32(2147483648) == -2147483648;
@@ -755,7 +755,7 @@ Unsupported types:
 - String representations of binary and hexadecimal values, e.g. `SELECT toInt64('0xc0fe');`
 
 :::note
-If the input value cannot be represented within the bounds of [Int16](../data-types/int-uint.md), the result over or under flows. This is not considered an error.  
+If the input value cannot be represented within the bounds of [Int64](../data-types/int-uint.md), the result over or under flows. This is not considered an error.  
 For example: 
 
 ```
@@ -1386,7 +1386,7 @@ Supported types:
 
 Types for which the default value is returned:
 - Float values `NaN` and `Inf` return the default value.
-- String representations of binary and hexadecimal values, e.g. `SELECT toInt128OrDefault('0xc0fe', CAST('-1', 'Int256'));`
+- String representations of binary and hexadecimal values, e.g. `SELECT toInt256OrDefault('0xc0fe', CAST('-1', 'Int256'));`
 - If the input value cannot be represented within the bounds of [Int256](../data-types/int-uint.md) and the result over or under flows.
 
 **Returned value**

From 7b8c41818bcec1c567c85b15d916fd2e064b7482 Mon Sep 17 00:00:00 2001
From: Michael Kolupaev <michael.kolupaev@clickhouse.com>
Date: Thu, 25 Jul 2024 19:34:41 +0000
Subject: [PATCH 0807/1488] Uncomment accidentally commented out code in
 QueryProfiler

---
 src/Common/QueryProfiler.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Common/QueryProfiler.cpp b/src/Common/QueryProfiler.cpp
index a7717a4288a..f6524088102 100644
--- a/src/Common/QueryProfiler.cpp
+++ b/src/Common/QueryProfiler.cpp
@@ -57,7 +57,7 @@ namespace
 
         auto saved_errno = errno;   /// We must restore previous value of errno in signal handler.
 
-#if defined(OS_LINUX) && false //asdqwe
+#if defined(OS_LINUX)
         if (info)
         {
             int overrun_count = info->si_overrun;
@@ -92,7 +92,7 @@ namespace
         constexpr bool sanitizer = false;
 #endif
 
-        //asdqwe asynchronous_stack_unwinding = true;
+        asynchronous_stack_unwinding = true;
         if (sanitizer || 0 == sigsetjmp(asynchronous_stack_unwinding_signal_jump_buffer, 1))
         {
             stack_trace.emplace(signal_context);

From f2e83f092d1f677c4e0240e749f96766ff6e205c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Thu, 25 Jul 2024 21:56:42 +0200
Subject: [PATCH 0808/1488] Patch getauxval for tsan re-exec

---
 base/glibc-compatibility/CMakeLists.txt   | 10 ++++++++++
 base/glibc-compatibility/musl/getauxval.c |  4 ++--
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/base/glibc-compatibility/CMakeLists.txt b/base/glibc-compatibility/CMakeLists.txt
index c967fa5b11b..8948e25cb8e 100644
--- a/base/glibc-compatibility/CMakeLists.txt
+++ b/base/glibc-compatibility/CMakeLists.txt
@@ -18,6 +18,16 @@ if (GLIBC_COMPATIBILITY)
         message (FATAL_ERROR "glibc_compatibility can only be used on x86_64 or aarch64.")
     endif ()
 
+    if (SANITIZE STREQUAL thread)
+        # Disable TSAN instrumentation that conflicts with re-exec due to high ASLR entropy using getauxval
+        # See longer comment in __auxv_init_procfs
+        # In the case of tsan we need to make sure getauxval is not instrumented as that would introduce tsan
+        # internal calls to functions that depend on a state that isn't initialized yet
+        set_source_files_properties(
+                musl/getauxval.c
+                PROPERTIES COMPILE_FLAGS "-mllvm -tsan-instrument-func-entry-exit=false")
+    endif()
+
     # Need to omit frame pointers to match the performance of glibc
     set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fomit-frame-pointer")
 
diff --git a/base/glibc-compatibility/musl/getauxval.c b/base/glibc-compatibility/musl/getauxval.c
index 28cb0f8d005..ec2cce1e4aa 100644
--- a/base/glibc-compatibility/musl/getauxval.c
+++ b/base/glibc-compatibility/musl/getauxval.c
@@ -102,7 +102,7 @@ static unsigned long NO_SANITIZE_THREAD __auxv_init_procfs(unsigned long type)
     /// most global variables aren't initialized or available yet, so we can't initiate the auxiliary vector.
     /// Normal glibc / musl getauxval doesn't have this problem since they initiate their auxval vector at the very
     /// start of __libc_start_main (just keeping track of argv+argc+1), but we don't have such option (otherwise
-    // this complexity of reading "/proc/self/auxv" or using __environ would not be necessary).
+    /// this complexity of reading "/proc/self/auxv" or using __environ would not be necessary).
 
     /// To avoid this crashes on the re-exec call (see above how it would fail when creating `aux`, and if we used
     /// __auxv_init_environ then it would SIGSEV on READing `__environ`) we capture this call for `AT_EXECFN` and
@@ -237,7 +237,7 @@ static unsigned long NO_SANITIZE_THREAD __auxv_init_environ(unsigned long type)
 // - __auxv_init_procfs -> __auxv_init_environ -> __getauxval_environ
 static void * volatile getauxval_func = (void *)__auxv_init_procfs;
 
-unsigned long getauxval(unsigned long type)
+unsigned long NO_SANITIZE_THREAD getauxval(unsigned long type)
 {
     return ((unsigned long (*)(unsigned long))getauxval_func)(type);
 }

From 992a2764855d61ea8c8a75c727e6e0fd33054c1a Mon Sep 17 00:00:00 2001
From: Michael Kolupaev <michael.kolupaev@clickhouse.com>
Date: Thu, 25 Jul 2024 20:23:45 +0000
Subject: [PATCH 0809/1488] uncomment also write_trace_iteration

---
 src/Common/QueryProfiler.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Common/QueryProfiler.cpp b/src/Common/QueryProfiler.cpp
index f6524088102..746010b5462 100644
--- a/src/Common/QueryProfiler.cpp
+++ b/src/Common/QueryProfiler.cpp
@@ -33,7 +33,7 @@ namespace DB
 namespace
 {
 #if defined(OS_LINUX)
-    //thread_local size_t write_trace_iteration = 0;
+    thread_local size_t write_trace_iteration = 0;
 #endif
     /// Even after timer_delete() the signal can be delivered,
     /// since it does not do anything with pending signals.

From 3f70977cd660e4617d9bbd68cc229020adc57f98 Mon Sep 17 00:00:00 2001
From: jsc0218 <jsc0218@gmail.com>
Date: Thu, 25 Jul 2024 21:02:30 +0000
Subject: [PATCH 0810/1488] try to fix

---
 ...2572_query_views_log_background_thread.reference | 13 +++++++++----
 .../02572_query_views_log_background_thread.sql     |  8 ++++++--
 2 files changed, 15 insertions(+), 6 deletions(-)

diff --git a/tests/queries/0_stateless/02572_query_views_log_background_thread.reference b/tests/queries/0_stateless/02572_query_views_log_background_thread.reference
index 22dfaf93781..f867fd0d085 100644
--- a/tests/queries/0_stateless/02572_query_views_log_background_thread.reference
+++ b/tests/queries/0_stateless/02572_query_views_log_background_thread.reference
@@ -1,15 +1,14 @@
 -- { echoOn }
 insert into buffer_02572 values (1);
 -- ensure that the flush was not direct
+select * from buffer_02572;
+1
 select * from data_02572;
 select * from copy_02572;
 -- we cannot use OPTIMIZE, this will attach query context, so let's wait
 SET function_sleep_max_microseconds_per_block = 6000000;
 select sleepEachRow(1) from numbers(3*2) format Null;
-select * from data_02572;
-1
-select * from copy_02572;
-1
+select sleepEachRow(1) from numbers(3*2) format Null;
 system flush logs;
 select count() > 0, lower(status::String), errorCodeToName(exception_code)
     from system.query_views_log where
@@ -18,3 +17,9 @@ select count() > 0, lower(status::String), errorCodeToName(exception_code)
     group by 2, 3
 ;
 1	queryfinish	OK
+select * from buffer_02572;
+1
+select * from data_02572;
+1
+select * from copy_02572;
+1
diff --git a/tests/queries/0_stateless/02572_query_views_log_background_thread.sql b/tests/queries/0_stateless/02572_query_views_log_background_thread.sql
index 939c189c5fe..2e9a62b71da 100644
--- a/tests/queries/0_stateless/02572_query_views_log_background_thread.sql
+++ b/tests/queries/0_stateless/02572_query_views_log_background_thread.sql
@@ -19,13 +19,13 @@ create materialized view mv_02572 to copy_02572 as select * from data_02572;
 -- { echoOn }
 insert into buffer_02572 values (1);
 -- ensure that the flush was not direct
+select * from buffer_02572;
 select * from data_02572;
 select * from copy_02572;
 -- we cannot use OPTIMIZE, this will attach query context, so let's wait
 SET function_sleep_max_microseconds_per_block = 6000000;
 select sleepEachRow(1) from numbers(3*2) format Null;
-select * from data_02572;
-select * from copy_02572;
+select sleepEachRow(1) from numbers(3*2) format Null;
 
 system flush logs;
 select count() > 0, lower(status::String), errorCodeToName(exception_code)
@@ -34,3 +34,7 @@ select count() > 0, lower(status::String), errorCodeToName(exception_code)
     view_target = concatWithSeparator('.', currentDatabase(), 'copy_02572')
     group by 2, 3
 ;
+
+select * from buffer_02572;
+select * from data_02572;
+select * from copy_02572;
\ No newline at end of file

From f0faa111d73c8512c1f88009f0ecfd1a804de45c Mon Sep 17 00:00:00 2001
From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com>
Date: Thu, 25 Jul 2024 23:19:58 +0200
Subject: [PATCH 0811/1488] Fix wrong usage of
 input_format_max_bytes_to_read_for_schema_inference

---
 src/Formats/FormatFactory.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp
index 1271cdfb7ad..e8956159714 100644
--- a/src/Formats/FormatFactory.cpp
+++ b/src/Formats/FormatFactory.cpp
@@ -253,7 +253,7 @@ FormatSettings getFormatSettings(const ContextPtr & context, const Settings & se
     format_settings.msgpack.number_of_columns = settings.input_format_msgpack_number_of_columns;
     format_settings.msgpack.output_uuid_representation = settings.output_format_msgpack_uuid_representation;
     format_settings.max_rows_to_read_for_schema_inference = settings.input_format_max_rows_to_read_for_schema_inference;
-    format_settings.max_bytes_to_read_for_schema_inference = settings.input_format_max_rows_to_read_for_schema_inference;
+    format_settings.max_bytes_to_read_for_schema_inference = settings.input_format_max_bytes_to_read_for_schema_inference;
     format_settings.column_names_for_schema_inference = settings.column_names_for_schema_inference;
     format_settings.schema_inference_hints = settings.schema_inference_hints;
     format_settings.schema_inference_make_columns_nullable = settings.schema_inference_make_columns_nullable;

From 18fb7396f941fd5a7e3872788ab07a18731dc943 Mon Sep 17 00:00:00 2001
From: xc0derx <11428624+xc0derx@users.noreply.github.com>
Date: Thu, 25 Jul 2024 23:21:30 +0200
Subject: [PATCH 0812/1488] fix broken links (compression codecs)

---
 docs/en/sql-reference/statements/alter/column.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/en/sql-reference/statements/alter/column.md b/docs/en/sql-reference/statements/alter/column.md
index aa6f132e08e..2e9b0cf3080 100644
--- a/docs/en/sql-reference/statements/alter/column.md
+++ b/docs/en/sql-reference/statements/alter/column.md
@@ -36,7 +36,7 @@ These actions are described in detail below.
 ADD COLUMN [IF NOT EXISTS] name [type] [default_expr] [codec] [AFTER name_after | FIRST]
 ```
 
-Adds a new column to the table with the specified `name`, `type`, [`codec`](../create/table.md/#codecs) and `default_expr` (see the section [Default expressions](/docs/en/sql-reference/statements/create/table.md/#create-default-values)).
+Adds a new column to the table with the specified `name`, `type`, [`codec`](../create/table.md/#column_compression_codec) and `default_expr` (see the section [Default expressions](/docs/en/sql-reference/statements/create/table.md/#create-default-values)).
 
 If the `IF NOT EXISTS` clause is included, the query won’t return an error if the column already exists. If you specify `AFTER name_after` (the name of another column), the column is added after the specified one in the list of table columns. If you want to add a column to the beginning of the table use the `FIRST` clause. Otherwise, the column is added to the end of the table. For a chain of actions, `name_after` can be the name of a column that is added in one of the previous actions.
 
@@ -155,7 +155,7 @@ This query changes the `name` column properties:
 
 - Column-level Settings
 
-For examples of columns compression CODECS modifying, see [Column Compression Codecs](../create/table.md/#codecs).
+For examples of columns compression CODECS modifying, see [Column Compression Codecs](../create/table.md/#column_compression_codec).
 
 For examples of columns TTL modifying, see [Column TTL](/docs/en/engines/table-engines/mergetree-family/mergetree.md/#mergetree-column-ttl).
 

From d0c4c4151c5e4bcb86b9417f4ab8cc71316404b5 Mon Sep 17 00:00:00 2001
From: Shri Bodas <shri.bodas@clickhouse.com>
Date: Thu, 25 Jul 2024 14:24:28 -0700
Subject: [PATCH 0813/1488] Update keepermap.md

Needs quotes around keeper path
---
 docs/en/engines/table-engines/special/keepermap.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/en/engines/table-engines/special/keepermap.md b/docs/en/engines/table-engines/special/keepermap.md
index 5559cc2c648..04a9a4b0d4e 100644
--- a/docs/en/engines/table-engines/special/keepermap.md
+++ b/docs/en/engines/table-engines/special/keepermap.md
@@ -54,7 +54,7 @@ CREATE TABLE keeper_map_table
     `v2` String,
     `v3` Float32
 )
-ENGINE = KeeperMap(/keeper_map_table, 4)
+ENGINE = KeeperMap('/keeper_map_table', 4)
 PRIMARY KEY key
 ```
 

From 321766d0b8161a794f11835b4650d30b3723835b Mon Sep 17 00:00:00 2001
From: robot-clickhouse <robot-clickhouse@users.noreply.github.com>
Date: Thu, 25 Jul 2024 22:51:14 +0000
Subject: [PATCH 0814/1488] Automatic style fix

---
 tests/performance/scripts/perf.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/tests/performance/scripts/perf.py b/tests/performance/scripts/perf.py
index f89784a0e0b..83d66997677 100755
--- a/tests/performance/scripts/perf.py
+++ b/tests/performance/scripts/perf.py
@@ -349,9 +349,7 @@ for query_index in queries_to_run:
         try:
             c.execute("SYSTEM JEMALLOC PURGE")
 
-            print(
-                f"purging jemalloc arenas\t{conn_index}\t{c.last_query.elapsed}"
-            )
+            print(f"purging jemalloc arenas\t{conn_index}\t{c.last_query.elapsed}")
         except KeyboardInterrupt:
             raise
         except:

From f3c88ff66707a50523ccef6e964f2fe78a711ace Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 26 Jul 2024 03:56:02 +0200
Subject: [PATCH 0815/1488] Fix benign data race in ZooKeeper

---
 src/Common/ZooKeeper/IKeeper.h                |  2 +-
 src/Common/ZooKeeper/TestKeeper.h             |  2 +-
 src/Common/ZooKeeper/ZooKeeper.cpp            | 27 ++++++++---------
 src/Common/ZooKeeper/ZooKeeper.h              |  2 +-
 src/Common/ZooKeeper/ZooKeeperImpl.cpp        | 29 +++++++++++++++++--
 src/Common/ZooKeeper/ZooKeeperImpl.h          |  9 +++---
 .../StorageSystemZooKeeperConnection.cpp      | 10 +++++--
 7 files changed, 54 insertions(+), 27 deletions(-)

diff --git a/src/Common/ZooKeeper/IKeeper.h b/src/Common/ZooKeeper/IKeeper.h
index 2c6cbc4a5d5..ce7489a33e5 100644
--- a/src/Common/ZooKeeper/IKeeper.h
+++ b/src/Common/ZooKeeper/IKeeper.h
@@ -548,7 +548,7 @@ public:
     virtual bool isExpired() const = 0;
 
     /// Get the current connected node idx.
-    virtual Int8 getConnectedNodeIdx() const = 0;
+    virtual std::optional<int8_t> getConnectedNodeIdx() const = 0;
 
     /// Get the current connected host and port.
     virtual String getConnectedHostPort() const = 0;
diff --git a/src/Common/ZooKeeper/TestKeeper.h b/src/Common/ZooKeeper/TestKeeper.h
index 2194ad015bf..562c313ac0e 100644
--- a/src/Common/ZooKeeper/TestKeeper.h
+++ b/src/Common/ZooKeeper/TestKeeper.h
@@ -39,7 +39,7 @@ public:
     ~TestKeeper() override;
 
     bool isExpired() const override { return expired; }
-    Int8 getConnectedNodeIdx() const override { return 0; }
+    std::optional<int8_t> getConnectedNodeIdx() const override { return 0; }
     String getConnectedHostPort() const override { return "TestKeeper:0000"; }
     int32_t getConnectionXid() const override { return 0; }
     int64_t getSessionID() const override { return 0; }
diff --git a/src/Common/ZooKeeper/ZooKeeper.cpp b/src/Common/ZooKeeper/ZooKeeper.cpp
index 01bb508da95..1250e1273b9 100644
--- a/src/Common/ZooKeeper/ZooKeeper.cpp
+++ b/src/Common/ZooKeeper/ZooKeeper.cpp
@@ -128,16 +128,15 @@ void ZooKeeper::init(ZooKeeperArgs args_, std::unique_ptr<Coordination::IKeeper>
         ShuffleHosts shuffled_hosts = shuffleHosts();
 
         impl = std::make_unique<Coordination::ZooKeeper>(shuffled_hosts, args, zk_log);
-        Int8 node_idx = impl->getConnectedNodeIdx();
+        auto node_idx = impl->getConnectedNodeIdx();
 
         if (args.chroot.empty())
             LOG_TRACE(log, "Initialized, hosts: {}", fmt::join(args.hosts, ","));
         else
             LOG_TRACE(log, "Initialized, hosts: {}, chroot: {}", fmt::join(args.hosts, ","), args.chroot);
 
-
         /// If the balancing strategy has an optimal node then it will be the first in the list
-        bool connected_to_suboptimal_node = node_idx != shuffled_hosts[0].original_index;
+        bool connected_to_suboptimal_node = node_idx && *node_idx != shuffled_hosts[0].original_index;
         bool respect_az = args.prefer_local_availability_zone && !args.client_availability_zone.empty();
         bool may_benefit_from_reconnecting = respect_az || args.get_priority_load_balancing.hasOptimalNode();
         if (connected_to_suboptimal_node && may_benefit_from_reconnecting)
@@ -145,7 +144,7 @@ void ZooKeeper::init(ZooKeeperArgs args_, std::unique_ptr<Coordination::IKeeper>
             auto reconnect_timeout_sec = getSecondsUntilReconnect(args);
             LOG_DEBUG(log, "Connected to a suboptimal ZooKeeper host ({}, index {})."
                            " To preserve balance in ZooKeeper usage, this ZooKeeper session will expire in {} seconds",
-                      impl->getConnectedHostPort(), node_idx, reconnect_timeout_sec);
+                      impl->getConnectedHostPort(), *node_idx, reconnect_timeout_sec);
 
             auto reconnect_task_holder = DB::Context::getGlobalContextInstance()->getSchedulePool().createTask("ZKReconnect", [this, optimal_host = shuffled_hosts[0]]()
             {
@@ -154,13 +153,15 @@ void ZooKeeper::init(ZooKeeperArgs args_, std::unique_ptr<Coordination::IKeeper>
                     LOG_DEBUG(log, "Trying to connect to a more optimal node {}", optimal_host.host);
                     ShuffleHosts node{optimal_host};
                     std::unique_ptr<Coordination::IKeeper> new_impl = std::make_unique<Coordination::ZooKeeper>(node, args, zk_log);
-                    Int8 new_node_idx = new_impl->getConnectedNodeIdx();
 
-                    /// Maybe the node was unavailable when getting AZs first time, update just in case
-                    if (args.availability_zone_autodetect && availability_zones[new_node_idx].empty())
+                    if (auto new_node_idx = new_impl->getConnectedNodeIdx(); new_node_idx)
                     {
-                        availability_zones[new_node_idx] = new_impl->tryGetAvailabilityZone();
-                        LOG_DEBUG(log, "Got availability zone for {}: {}", optimal_host.host, availability_zones[new_node_idx]);
+                        /// Maybe the node was unavailable when getting AZs first time, update just in case
+                        if (args.availability_zone_autodetect && availability_zones[*new_node_idx].empty())
+                        {
+                            availability_zones[*new_node_idx] = new_impl->tryGetAvailabilityZone();
+                            LOG_DEBUG(log, "Got availability zone for {}: {}", optimal_host.host, availability_zones[*new_node_idx]);
+                        }
                     }
 
                     optimal_impl = std::move(new_impl);
@@ -1525,7 +1526,7 @@ void ZooKeeper::setServerCompletelyStarted()
         zk->setServerCompletelyStarted();
 }
 
-Int8 ZooKeeper::getConnectedHostIdx() const
+std::optional<int8_t> ZooKeeper::getConnectedHostIdx() const
 {
     return impl->getConnectedNodeIdx();
 }
@@ -1544,10 +1545,10 @@ String ZooKeeper::getConnectedHostAvailabilityZone() const
 {
     if (args.implementation != "zookeeper" || !impl)
         return "";
-    Int8 idx = impl->getConnectedNodeIdx();
-    if (idx < 0)
+    std::optional<int8_t> idx = impl->getConnectedNodeIdx();
+    if (!idx)
         return "";     /// session expired
-    return availability_zones.at(idx);
+    return availability_zones.at(*idx);
 }
 
 size_t getFailedOpIndex(Coordination::Error exception_code, const Coordination::Responses & responses)
diff --git a/src/Common/ZooKeeper/ZooKeeper.h b/src/Common/ZooKeeper/ZooKeeper.h
index 4ae2cfa6096..657c9cb2c03 100644
--- a/src/Common/ZooKeeper/ZooKeeper.h
+++ b/src/Common/ZooKeeper/ZooKeeper.h
@@ -620,7 +620,7 @@ public:
 
     void setServerCompletelyStarted();
 
-    Int8 getConnectedHostIdx() const;
+    std::optional<int8_t> getConnectedHostIdx() const;
     String getConnectedHostPort() const;
     int32_t getConnectionXid() const;
 
diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.cpp b/src/Common/ZooKeeper/ZooKeeperImpl.cpp
index 2728f953bea..53c7a5728aa 100644
--- a/src/Common/ZooKeeper/ZooKeeperImpl.cpp
+++ b/src/Common/ZooKeeper/ZooKeeperImpl.cpp
@@ -536,7 +536,7 @@ void ZooKeeper::connect(
                     compressed_out.emplace(*out, CompressionCodecFactory::instance().get("LZ4", {}));
                 }
 
-                original_index = static_cast<Int8>(node.original_index);
+                original_index.store(node.original_index);
                 break;
             }
             catch (...)
@@ -1014,8 +1014,7 @@ void ZooKeeper::finalize(bool error_send, bool error_receive, const String & rea
     LOG_INFO(log, "Finalizing session {}. finalization_started: {}, queue_finished: {}, reason: '{}'",
              session_id, already_started, requests_queue.isFinished(), reason);
 
-    /// Reset the original index.
-    original_index = -1;
+    original_index.store(-1);
 
     auto expire_session_if_not_expired = [&]
     {
@@ -1534,6 +1533,30 @@ void ZooKeeper::close()
 }
 
 
+std::optional<int8_t> ZooKeeper::getConnectedNodeIdx() const
+{
+    int8_t res = original_index.load();
+    if (res == -1)
+        return std::nullopt;
+    else
+        return res;
+}
+
+String ZooKeeper::getConnectedHostPort() const
+{
+    auto idx = getConnectedNodeIdx();
+    if (idx)
+        return args.hosts[*idx];
+    else
+        return "";
+}
+
+int32_t ZooKeeper::getConnectionXid() const
+{
+    return next_xid.load();
+}
+
+
 void ZooKeeper::setZooKeeperLog(std::shared_ptr<DB::ZooKeeperLog> zk_log_)
 {
     /// logOperationIfNeeded(...) uses zk_log and can be called from different threads, so we have to use atomic shared_ptr
diff --git a/src/Common/ZooKeeper/ZooKeeperImpl.h b/src/Common/ZooKeeper/ZooKeeperImpl.h
index 0c88c35b381..39082cd14c1 100644
--- a/src/Common/ZooKeeper/ZooKeeperImpl.h
+++ b/src/Common/ZooKeeper/ZooKeeperImpl.h
@@ -114,13 +114,12 @@ public:
 
     ~ZooKeeper() override;
 
-
     /// If expired, you can only destroy the object. All other methods will throw exception.
     bool isExpired() const override { return requests_queue.isFinished(); }
 
-    Int8 getConnectedNodeIdx() const override { return original_index; }
-    String getConnectedHostPort() const override { return (original_index == -1) ? "" : args.hosts[original_index]; }
-    int32_t getConnectionXid() const override { return next_xid.load(); }
+    std::optional<int8_t> getConnectedNodeIdx() const override;
+    String getConnectedHostPort() const override;
+    int32_t getConnectionXid() const override;
 
     String tryGetAvailabilityZone() override;
 
@@ -219,7 +218,7 @@ private:
     ACLs default_acls;
 
     zkutil::ZooKeeperArgs args;
-    Int8 original_index = -1;
+    std::atomic<int8_t> original_index{-1};
 
     /// Fault injection
     void maybeInjectSendFault();
diff --git a/src/Storages/System/StorageSystemZooKeeperConnection.cpp b/src/Storages/System/StorageSystemZooKeeperConnection.cpp
index ec29b84dac3..72a7ba38429 100644
--- a/src/Storages/System/StorageSystemZooKeeperConnection.cpp
+++ b/src/Storages/System/StorageSystemZooKeeperConnection.cpp
@@ -4,6 +4,7 @@
 #include <DataTypes/DataTypeString.h>
 #include <DataTypes/DataTypesNumber.h>
 #include <DataTypes/DataTypeDateTime.h>
+#include <DataTypes/DataTypeNullable.h>
 #include <Common/ZooKeeper/ZooKeeper.h>
 #include <Coordination/KeeperFeatureFlags.h>
 #include <Storages/System/StorageSystemZooKeeperConnection.h>
@@ -27,7 +28,7 @@ ColumnsDescription StorageSystemZooKeeperConnection::getColumnsDescription()
         /* 0 */ {"name", std::make_shared<DataTypeString>(), "ZooKeeper cluster's name."},
         /* 1 */ {"host", std::make_shared<DataTypeString>(), "The hostname/IP of the ZooKeeper node that ClickHouse connected to."},
         /* 2 */ {"port", std::make_shared<DataTypeUInt16>(), "The port of the ZooKeeper node that ClickHouse connected to."},
-        /* 3 */ {"index", std::make_shared<DataTypeUInt8>(), "The index of the ZooKeeper node that ClickHouse connected to. The index is from ZooKeeper config."},
+        /* 3 */ {"index", std::make_shared<DataTypeNullable>(std::make_shared<DataTypeUInt8>()), "The index of the ZooKeeper node that ClickHouse connected to. The index is from ZooKeeper config. If not connected, this column is NULL."},
         /* 4 */ {"connected_time", std::make_shared<DataTypeDateTime>(), "When the connection was established."},
         /* 5 */ {"session_uptime_elapsed_seconds", std::make_shared<DataTypeUInt64>(), "Seconds elapsed since the connection was established."},
         /* 6 */ {"is_expired", std::make_shared<DataTypeUInt8>(), "Is the current connection expired."},
@@ -64,7 +65,7 @@ void StorageSystemZooKeeperConnection::fillData(MutableColumns & res_columns, Co
     /// For read-only snapshot type functionality, it's acceptable even though 'getZooKeeper' may cause data inconsistency.
     auto fill_data = [&](const String & name, const zkutil::ZooKeeperPtr zookeeper, MutableColumns & columns)
     {
-        Int8 index = zookeeper->getConnectedHostIdx();
+        auto index = zookeeper->getConnectedHostIdx();
         String host_port = zookeeper->getConnectedHostPort();
         if (index != -1 && !host_port.empty())
         {
@@ -78,7 +79,10 @@ void StorageSystemZooKeeperConnection::fillData(MutableColumns & res_columns, Co
             columns[0]->insert(name);
             columns[1]->insert(host);
             columns[2]->insert(port);
-            columns[3]->insert(index);
+            if (index)
+                columns[3]->insert(*index);
+            else
+                columns[3]->insertDefault();
             columns[4]->insert(connected_time);
             columns[5]->insert(uptime);
             columns[6]->insert(zookeeper->expired());

From d6fdf29679ece887567cba6fa43aee4c22c7d6f7 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 26 Jul 2024 04:48:41 +0200
Subject: [PATCH 0816/1488] Remove too long unit test

---
 .../tests/gtest_archive_reader_and_writer.cpp | 42 -------------------
 1 file changed, 42 deletions(-)

diff --git a/src/IO/tests/gtest_archive_reader_and_writer.cpp b/src/IO/tests/gtest_archive_reader_and_writer.cpp
index 898c7017e7d..06f8f53546b 100644
--- a/src/IO/tests/gtest_archive_reader_and_writer.cpp
+++ b/src/IO/tests/gtest_archive_reader_and_writer.cpp
@@ -492,48 +492,6 @@ TEST_P(ArchiveReaderAndWriterTest, ManyFilesOnDisk)
     }
 }
 
-TEST_P(ArchiveReaderAndWriterTest, LargeFile)
-{
-    /// Make an archive.
-    std::string_view contents = "The contents of a.txt\n";
-    int times = 10000000;
-    {
-        auto writer = createArchiveWriter(getPathToArchive());
-        {
-            auto out = writer->writeFile("a.txt", times * contents.size());
-            for (int i = 0; i < times; i++)
-                writeString(contents, *out);
-            out->finalize();
-        }
-        writer->finalize();
-    }
-
-    /// Read the archive.
-    auto reader = createArchiveReader(getPathToArchive());
-
-    ASSERT_TRUE(reader->fileExists("a.txt"));
-
-    auto file_info = reader->getFileInfo("a.txt");
-    EXPECT_EQ(file_info.uncompressed_size, contents.size() * times);
-    EXPECT_GT(file_info.compressed_size, 0);
-
-    {
-        auto in = reader->readFile("a.txt", /*throw_on_not_found=*/true);
-        for (int i = 0; i < times; i++)
-            ASSERT_TRUE(checkString(String(contents), *in));
-    }
-
-    {
-        /// Use an enumerator.
-        auto enumerator = reader->firstFile();
-        ASSERT_NE(enumerator, nullptr);
-        EXPECT_EQ(enumerator->getFileName(), "a.txt");
-        EXPECT_EQ(enumerator->getFileInfo().uncompressed_size, contents.size() * times);
-        EXPECT_GT(enumerator->getFileInfo().compressed_size, 0);
-        EXPECT_FALSE(enumerator->nextFile());
-    }
-}
-
 TEST(TarArchiveReaderTest, FileExists)
 {
     String archive_path = "archive.tar";

From 9c6026965d985ca0ffcf0ab789d09946bd37c569 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 26 Jul 2024 04:55:53 +0200
Subject: [PATCH 0817/1488] Fix error

---
 src/IO/ReadWriteBufferFromHTTP.cpp | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/src/IO/ReadWriteBufferFromHTTP.cpp b/src/IO/ReadWriteBufferFromHTTP.cpp
index 17a5ed385d4..a62f22d4bd9 100644
--- a/src/IO/ReadWriteBufferFromHTTP.cpp
+++ b/src/IO/ReadWriteBufferFromHTTP.cpp
@@ -700,6 +700,14 @@ std::optional<time_t> ReadWriteBufferFromHTTP::tryGetLastModificationTime()
         {
             return std::nullopt;
         }
+        catch (const NetException &)
+        {
+            return std::nullopt;
+        }
+        catch (const Poco::Net::NetException &)
+        {
+            return std::nullopt;
+        }
     }
 
     return file_info->last_modified;

From 64ff5d7bc443cdb15fd0a5eec391d449a617b3f9 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 26 Jul 2024 05:07:56 +0200
Subject: [PATCH 0818/1488] Fix `00705_drop_create_merge_tree`

---
 tests/queries/0_stateless/00705_drop_create_merge_tree.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/queries/0_stateless/00705_drop_create_merge_tree.sh b/tests/queries/0_stateless/00705_drop_create_merge_tree.sh
index d7754091290..ea8b9d02e49 100755
--- a/tests/queries/0_stateless/00705_drop_create_merge_tree.sh
+++ b/tests/queries/0_stateless/00705_drop_create_merge_tree.sh
@@ -5,8 +5,8 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
-yes 'CREATE TABLE IF NOT EXISTS table (x UInt8) ENGINE = MergeTree ORDER BY tuple();' | head -n 1000 | $CLICKHOUSE_CLIENT --ignore-error -nm 2>/dev/null &
-yes 'DROP TABLE table;' | head -n 1000 | $CLICKHOUSE_CLIENT --ignore-error -nm 2>/dev/null &
+yes 'CREATE TABLE IF NOT EXISTS table (x UInt8) ENGINE = MergeTree ORDER BY tuple();' | head -n 1000 | $CLICKHOUSE_CLIENT --multiquery &
+yes 'DROP TABLE IF EXISTS table;' | head -n 1000 | $CLICKHOUSE_CLIENT --multiquery &
 wait
 
 ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS table"

From 3f483e547572c47d137d0f2664bd82c0b978ac7b Mon Sep 17 00:00:00 2001
From: heguangnan <heguangnan@bytedance.com>
Date: Fri, 26 Jul 2024 12:09:03 +0800
Subject: [PATCH 0819/1488] fix memory leak when exception happend during count
 distinct for null key

---
 src/Interpreters/Aggregator.cpp | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp
index e073b7a49b6..543fd8a0bf2 100644
--- a/src/Interpreters/Aggregator.cpp
+++ b/src/Interpreters/Aggregator.cpp
@@ -3300,6 +3300,17 @@ void NO_INLINE Aggregator::destroyImpl(Table & table) const
 
         data = nullptr;
     });
+
+    if constexpr (Method::low_cardinality_optimization || Method::one_key_nullable_optimization)
+    {
+        if (table.getNullKeyData() != nullptr)
+        {
+            for (size_t i = 0; i < params.aggregates_size; ++i)
+                aggregate_functions[i]->destroy(table.getNullKeyData() + offsets_of_aggregate_states[i]);
+
+            table.getNullKeyData() = nullptr;
+        }
+    }
 }
 
 

From 287cce7d211b9386895a4fa898f87405eccb3e96 Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Fri, 26 Jul 2024 09:20:15 +0200
Subject: [PATCH 0820/1488] Fixes

---
 .gitmodules                               | 2 +-
 contrib/numactl                           | 2 +-
 docker/test/performance-comparison/run.sh | 1 +
 programs/server/Server.cpp                | 9 ++++++++-
 4 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/.gitmodules b/.gitmodules
index b5d7e1e56b3..7e0b4df4ad1 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -374,4 +374,4 @@
 	url = https://github.com/ClickHouse/double-conversion.git
 [submodule "contrib/numactl"]
 	path = contrib/numactl
-	url = https://github.com/numactl/numactl.git
+	url = https://github.com/ClickHouse/numactl.git
diff --git a/contrib/numactl b/contrib/numactl
index 3871b1c42fc..8d13d63a05f 160000
--- a/contrib/numactl
+++ b/contrib/numactl
@@ -1 +1 @@
-Subproject commit 3871b1c42fc71bceadafd745d2eff5dddfc2d67e
+Subproject commit 8d13d63a05f0c3cd88bf777cbb61541202b7da08
diff --git a/docker/test/performance-comparison/run.sh b/docker/test/performance-comparison/run.sh
index 7afb5da59b1..6ef781fa4c8 100644
--- a/docker/test/performance-comparison/run.sh
+++ b/docker/test/performance-comparison/run.sh
@@ -13,6 +13,7 @@ entry="/usr/share/clickhouse-test/performance/scripts/entrypoint.sh"
 # https://www.kernel.org/doc/Documentation/filesystems/tmpfs.txt
 # Double-escaped backslashes are a tribute to the engineering wonder of docker --
 # it gives '/bin/sh: 1: [bash,: not found' otherwise.
+numactl --hardware
 node=$(( RANDOM % $(numactl --hardware | sed -n 's/^.*available:\(.*\)nodes.*$/\1/p') ));
 echo Will bind to NUMA node $node;
 numactl --cpunodebind=$node --membind=$node $entry
diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp
index b9a7c298f00..b818ff1f3a2 100644
--- a/programs/server/Server.cpp
+++ b/programs/server/Server.cpp
@@ -775,9 +775,16 @@ try
 
             LOG_INFO(
                 log,
-                "ClickHouse is bound to a subset of NUMA nodes. Total memory of all available nodes {}",
+                "ClickHouse is bound to a subset of NUMA nodes. Total memory of all available nodes: {}",
                 ReadableSize(total_numa_memory));
         }
+        else
+        {
+            LOG_TRACE(
+                log,
+                "All NUMA nodes are used. Detected NUMA nodes: {}",
+                numa_num_configured_nodes());
+        }
 
         numa_bitmask_free(membind);
     }

From 400f8e5b2116ab585312e578eee4d783b9d6783b Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Fri, 26 Jul 2024 09:33:46 +0200
Subject: [PATCH 0821/1488] Fix stacktrace cache

---
 src/Common/StackTrace.cpp | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/src/Common/StackTrace.cpp b/src/Common/StackTrace.cpp
index 59a58ac027a..ff8765c9727 100644
--- a/src/Common/StackTrace.cpp
+++ b/src/Common/StackTrace.cpp
@@ -495,18 +495,19 @@ using StackTraceCacheBase = std::map<StackTraceTriple, CacheEntryPtr, std::less<
 
 struct StackTraceCache : public StackTraceCacheBase
 {
+    StackTraceCache()
+        : StackTraceCacheBase()
+    {
+        can_use_cache = true;
+    }
+
     ~StackTraceCache()
     {
         can_use_cache = false;
     }
 };
 
-static StackTraceCache & cacheInstance()
-{
-    static StackTraceCache cache;
-    can_use_cache = true;
-    return cache;
-}
+static StackTraceCache cache;
 
 static DB::SharedMutex stacktrace_cache_mutex;
 
@@ -524,7 +525,6 @@ String toStringCached(const StackTrace::FramePointers & pointers, size_t offset,
     /// Calculation of stack trace text is extremely slow.
     /// We use cache because otherwise the server could be overloaded by trash queries.
     /// Note that this cache can grow unconditionally, but practically it should be small.
-    StackTraceCache & cache = cacheInstance();
     CacheEntryPtr cache_entry;
 
     // Optimistic try for cache hit to avoid any contention whatsoever, should be the main hot code route
@@ -576,7 +576,7 @@ std::string StackTrace::toString(void * const * frame_pointers_raw, size_t offse
 void StackTrace::dropCache()
 {
     std::lock_guard lock{stacktrace_cache_mutex};
-    cacheInstance().clear();
+    cache.clear();
 }
 
 

From 10b36c09e9ffd96f53d2fdada725f9c999241dd1 Mon Sep 17 00:00:00 2001
From: kevinyhzou <kevinyunhe8@gmail.com>
Date: Fri, 26 Jul 2024 15:38:26 +0800
Subject: [PATCH 0822/1488] support set orc reader time zone name

---
 src/Core/Settings.h                                    |  2 +-
 src/Core/SettingsChangesHistory.cpp                    |  2 +-
 src/Formats/FormatFactory.cpp                          |  2 +-
 src/Formats/FormatSettings.h                           |  2 +-
 .../Formats/Impl/NativeORCBlockInputFormat.cpp         |  6 +-----
 tests/queries/0_stateless/03198_orc_read_time_zone.sh  | 10 +++++-----
 6 files changed, 10 insertions(+), 14 deletions(-)

diff --git a/src/Core/Settings.h b/src/Core/Settings.h
index 2a665326afc..a5220c3017c 100644
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@@ -1073,7 +1073,7 @@ class IColumn;
     M(Bool, input_format_orc_allow_missing_columns, true, "Allow missing columns while reading ORC input formats", 0) \
     M(Bool, input_format_orc_use_fast_decoder, true, "Use a faster ORC decoder implementation.", 0) \
     M(Bool, input_format_orc_filter_push_down, true, "When reading ORC files, skip whole stripes or row groups based on the WHERE/PREWHERE expressions, min/max statistics or bloom filter in the ORC metadata.", 0) \
-    M(Bool, input_format_orc_read_use_writer_time_zone, false, "Whether use the writer's time zone in ORC stripe for ORC row reader, the default ORC row reader's time zone is GMT.", 0) \
+    M(String, input_format_orc_reader_time_zone_name, "GMT", "The time zone name for ORC row reader, the default ORC row reader's time zone is GMT.", 0) \
     M(Bool, input_format_parquet_allow_missing_columns, true, "Allow missing columns while reading Parquet input formats", 0) \
     M(UInt64, input_format_parquet_local_file_min_bytes_for_seek, 8192, "Min bytes required for local read (file) to do seek, instead of read with ignore in Parquet input format", 0) \
     M(Bool, input_format_arrow_allow_missing_columns, true, "Allow missing columns while reading Arrow input formats", 0) \
diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp
index acd119c159b..457caa76bb6 100644
--- a/src/Core/SettingsChangesHistory.cpp
+++ b/src/Core/SettingsChangesHistory.cpp
@@ -69,7 +69,7 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
               {"dictionary_validate_primary_key_type", false, false, "Validate primary key type for dictionaries. By default id type for simple layouts will be implicitly converted to UInt64."},
               {"collect_hash_table_stats_during_joins", false, true, "New setting."},
               {"max_size_to_preallocate_for_joins", 0, 100'000'000, "New setting."},
-              {"input_format_orc_read_use_writer_time_zone", false, false, "Whether use the writer's time zone in ORC stripe for ORC row reader, the default ORC row reader's time zone is GMT."},
+              {"input_format_orc_reader_time_zone_name", "GMT", "GMT", "The time zone name for ORC row reader, the default ORC row reader's time zone is GMT."},
               {"lightweight_mutation_projection_mode", "throw", "throw", "When lightweight delete happens on a table with projection(s), the possible operations include throw the exception as projection exists, or drop all projection related to this table then do lightweight delete."},
               {"database_replicated_allow_heavy_create", true, false, "Long-running DDL queries (CREATE AS SELECT and POPULATE) for Replicated database engine was forbidden"},
               {"query_plan_merge_filters", false, false, "Allow to merge filters in the query plan"},
diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp
index 1271cdfb7ad..e61bd16b537 100644
--- a/src/Formats/FormatFactory.cpp
+++ b/src/Formats/FormatFactory.cpp
@@ -244,7 +244,7 @@ FormatSettings getFormatSettings(const ContextPtr & context, const Settings & se
     format_settings.orc.output_row_index_stride = settings.output_format_orc_row_index_stride;
     format_settings.orc.use_fast_decoder = settings.input_format_orc_use_fast_decoder;
     format_settings.orc.filter_push_down = settings.input_format_orc_filter_push_down;
-    format_settings.orc.read_use_writer_time_zone = settings.input_format_orc_read_use_writer_time_zone;
+    format_settings.orc.reader_time_zone_name = settings.input_format_orc_reader_time_zone_name;
     format_settings.defaults_for_omitted_fields = settings.input_format_defaults_for_omitted_fields;
     format_settings.capn_proto.enum_comparing_mode = settings.format_capn_proto_enum_comparising_mode;
     format_settings.capn_proto.skip_fields_with_unsupported_types_in_schema_inference = settings.input_format_capn_proto_skip_fields_with_unsupported_types_in_schema_inference;
diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h
index a736bb1a558..2df8c9ca171 100644
--- a/src/Formats/FormatSettings.h
+++ b/src/Formats/FormatSettings.h
@@ -409,7 +409,7 @@ struct FormatSettings
         bool use_fast_decoder = true;
         bool filter_push_down = true;
         UInt64 output_row_index_stride = 10'000;
-        bool read_use_writer_time_zone = false;
+        String reader_time_zone_name = "GMT";
     } orc{};
 
     /// For capnProto format we should determine how to
diff --git a/src/Processors/Formats/Impl/NativeORCBlockInputFormat.cpp b/src/Processors/Formats/Impl/NativeORCBlockInputFormat.cpp
index c10969b02b7..649721f28bf 100644
--- a/src/Processors/Formats/Impl/NativeORCBlockInputFormat.cpp
+++ b/src/Processors/Formats/Impl/NativeORCBlockInputFormat.cpp
@@ -900,11 +900,7 @@ bool NativeORCBlockInputFormat::prepareStripeReader()
 
     orc::RowReaderOptions row_reader_options;
     row_reader_options.includeTypes(include_indices);
-    if (format_settings.orc.read_use_writer_time_zone)
-    {
-        String writer_time_zone = current_stripe_info->getWriterTimezone();
-        row_reader_options.setTimezoneName(writer_time_zone);
-    }
+    row_reader_options.setTimezoneName(format_settings.orc.reader_time_zone_name);
     row_reader_options.range(current_stripe_info->getOffset(), current_stripe_info->getLength());
     if (format_settings.orc.filter_push_down && sarg)
     {
diff --git a/tests/queries/0_stateless/03198_orc_read_time_zone.sh b/tests/queries/0_stateless/03198_orc_read_time_zone.sh
index 27530c06237..7e931e16e48 100755
--- a/tests/queries/0_stateless/03198_orc_read_time_zone.sh
+++ b/tests/queries/0_stateless/03198_orc_read_time_zone.sh
@@ -5,8 +5,8 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
-$CLICKHOUSE_CLIENT -q "drop table if exists test"
-$CLICKHOUSE_CLIENT -q "create table test(id UInt64, t DateTime64) Engine=MergeTree order by id"
-$CLICKHOUSE_CLIENT -q "insert into test from infile '$CURDIR/data_orc/test_reader_time_zone.snappy.orc' SETTINGS input_format_orc_read_use_writer_time_zone=true FORMAT ORC"
-$CLICKHOUSE_CLIENT -q "select * from test SETTINGS session_timezone='Asia/Shanghai'"
-$CLICKHOUSE_CLIENT -q "drop table test"
\ No newline at end of file
+$CLICKHOUSE_CLIENT -q "drop table if exists test_orc_read_timezone"
+$CLICKHOUSE_CLIENT -q "create table test_orc_read_timezone(id UInt64, t DateTime64) Engine=MergeTree order by id"
+$CLICKHOUSE_CLIENT -q "insert into test_orc_read_timezone from infile '$CURDIR/data_orc/test_reader_time_zone.snappy.orc' SETTINGS input_format_orc_reader_time_zone_name='Asia/Shanghai' FORMAT ORC"
+$CLICKHOUSE_CLIENT -q "select * from test_orc_read_timezone"
+$CLICKHOUSE_CLIENT -q "drop table test_orc_read_timezone"
\ No newline at end of file

From c66a9f2d365e101798b2eee6cde7acc903a2fb46 Mon Sep 17 00:00:00 2001
From: Konstantin Bogdanov <konstantin@clickhouse.com>
Date: Fri, 26 Jul 2024 09:39:31 +0200
Subject: [PATCH 0823/1488] Fix

---
 .../01676_clickhouse_client_autocomplete.python       | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/tests/queries/0_stateless/01676_clickhouse_client_autocomplete.python b/tests/queries/0_stateless/01676_clickhouse_client_autocomplete.python
index 9072dfeb09f..0f35d259c7c 100644
--- a/tests/queries/0_stateless/01676_clickhouse_client_autocomplete.python
+++ b/tests/queries/0_stateless/01676_clickhouse_client_autocomplete.python
@@ -59,13 +59,14 @@ def test_completion(program, argv, comp_word):
             output = output_b.decode()
             debug_log_fd.write(repr(output_b) + "\n")
             debug_log_fd.flush()
-            # fail fast if there is a bell character in the output,
-            # meaning no concise completion is found
-            if "\x07" in output:
-                print(f"{comp_word}: FAIL")
-                return
 
             while not comp_word in output:
+                # fail fast if there is a bell character in the output,
+                # meaning no concise completion is found
+                if "\x07" in output:
+                    print(f"{comp_word}: FAIL")
+                    return
+
                 output_b = os.read(master, 4096)
                 output += output_b.decode()
                 debug_log_fd.write(repr(output_b) + "\n")

From 83dba7194f3467dc0f6e5499d65bda8a66fa8206 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Fri, 26 Jul 2024 09:55:09 +0200
Subject: [PATCH 0824/1488] Add deserialization of empty state

---
 .../03208_groupArrayIntersect_serialization.reference           | 1 +
 .../0_stateless/03208_groupArrayIntersect_serialization.sql     | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/tests/queries/0_stateless/03208_groupArrayIntersect_serialization.reference b/tests/queries/0_stateless/03208_groupArrayIntersect_serialization.reference
index c3b6e0cd5b7..e84856c90fd 100644
--- a/tests/queries/0_stateless/03208_groupArrayIntersect_serialization.reference
+++ b/tests/queries/0_stateless/03208_groupArrayIntersect_serialization.reference
@@ -10,3 +10,4 @@ a	[(['2','4','6','8','10'])]
 b	[(['2','4','6','8','10'])]
 c	[(['2','4','6','8','10'])]
 d	[]
+e	[]
diff --git a/tests/queries/0_stateless/03208_groupArrayIntersect_serialization.sql b/tests/queries/0_stateless/03208_groupArrayIntersect_serialization.sql
index e05f78a4051..1b3d48ce0c3 100644
--- a/tests/queries/0_stateless/03208_groupArrayIntersect_serialization.sql
+++ b/tests/queries/0_stateless/03208_groupArrayIntersect_serialization.sql
@@ -39,3 +39,5 @@ INSERT INTO grouparray_string Select groupArrayIntersectState([tuple(['2', '4',
 SELECT 'c', arraySort(groupArrayIntersectMerge(v)) FROM grouparray_string;
 INSERT INTO grouparray_string Select groupArrayIntersectState([tuple(['2', '4', '6', '8', '10', '20']), tuple(['2', '4', '6', '8', '10', '14'])]);
 SELECT 'd', arraySort(groupArrayIntersectMerge(v)) FROM grouparray_string;
+INSERT INTO grouparray_string Select groupArrayIntersectState([]::Array(Tuple(Array(String))));
+SELECT 'e', arraySort(groupArrayIntersectMerge(v)) FROM grouparray_string;

From ad4e807cf4bb3633616b01e3616844fe2108d59f Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Fri, 26 Jul 2024 08:12:01 +0000
Subject: [PATCH 0825/1488] Fix stupid crash.

---
 src/Interpreters/InterpreterSelectQuery.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp
index 4fd6f7a2900..41306a79198 100644
--- a/src/Interpreters/InterpreterSelectQuery.cpp
+++ b/src/Interpreters/InterpreterSelectQuery.cpp
@@ -2055,9 +2055,9 @@ void InterpreterSelectQuery::addEmptySourceToQueryPlan(QueryPlan & query_plan, c
     {
         auto & prewhere_info = *query_info.prewhere_info;
 
-        auto row_level_actions = std::make_shared<ExpressionActions>(prewhere_info.row_level_filter->clone());
         if (prewhere_info.row_level_filter)
         {
+            auto row_level_actions = std::make_shared<ExpressionActions>(prewhere_info.row_level_filter->clone());
             pipe.addSimpleTransform([&](const Block & header)
             {
                 return std::make_shared<FilterTransform>(header,

From de84f4f045f5ece627ca8295a09a5f2cf1eab6aa Mon Sep 17 00:00:00 2001
From: Yakov Olkhovskiy <yakov@clickhouse.com>
Date: Fri, 26 Jul 2024 08:13:21 +0000
Subject: [PATCH 0826/1488] add proper cast to lagInFrame/leadInFrame

---
 src/Processors/Transforms/WindowTransform.cpp | 86 ++++++++++++++++---
 src/Processors/Transforms/WindowTransform.h   |  1 +
 2 files changed, 75 insertions(+), 12 deletions(-)

diff --git a/src/Processors/Transforms/WindowTransform.cpp b/src/Processors/Transforms/WindowTransform.cpp
index 06ae2bfb25e..006593edeaa 100644
--- a/src/Processors/Transforms/WindowTransform.cpp
+++ b/src/Processors/Transforms/WindowTransform.cpp
@@ -16,6 +16,9 @@
 #include <Common/Arena.h>
 #include <Common/FieldVisitorConvertToNumber.h>
 #include <Common/FieldVisitorsAccurateComparison.h>
+#include <Functions/CastOverloadResolver.h>
+#include <Functions/IFunction.h>
+#include <DataTypes/DataTypeString.h>
 
 #include <Poco/Logger.h>
 #include <Common/logger_useful.h>
@@ -75,6 +78,8 @@ public:
 
     virtual std::optional<WindowFrame> getDefaultFrame() const { return {}; }
 
+    virtual ColumnPtr castColumn(const Columns &, const std::vector<size_t> &) { return nullptr; }
+
     /// Is the frame type supported by this function.
     virtual bool checkWindowFrameType(const WindowTransform * /*transform*/) const { return true; }
 };
@@ -1171,6 +1176,9 @@ void WindowTransform::appendChunk(Chunk & chunk)
         // Initialize output columns.
         for (auto & ws : workspaces)
         {
+            if (ws.window_function_impl)
+                block.casted_columns.push_back(ws.window_function_impl->castColumn(block.input_columns, ws.argument_column_indices));
+
             block.output_columns.push_back(ws.aggregate_function->getResultType()
                 ->createColumn());
             block.output_columns.back()->reserve(block.rows);
@@ -2358,6 +2366,8 @@ public:
 template <bool is_lead>
 struct WindowFunctionLagLeadInFrame final : public WindowFunction
 {
+    FunctionBasePtr func_cast = nullptr;
+
     WindowFunctionLagLeadInFrame(const std::string & name_,
             const DataTypes & argument_types_, const Array & parameters_)
         : WindowFunction(name_, argument_types_, parameters_, createResultType(argument_types_, name_))
@@ -2385,18 +2395,71 @@ struct WindowFunctionLagLeadInFrame final : public WindowFunction
             return;
         }
 
-        if (!argument_types[0]->equals(*argument_types[2]))
-            throw Exception(ErrorCodes::BAD_ARGUMENTS,
-                "Argument type '{}' and the default value type '{}' are different",
-                argument_types[0]->getName(),
-                argument_types[2]->getName());
-
         if (argument_types.size() > 3)
         {
             throw Exception(ErrorCodes::BAD_ARGUMENTS,
                 "Function '{}' accepts at most 3 arguments, {} given",
                 name, argument_types.size());
         }
+
+        if (argument_types[0]->equals(*argument_types[2]))
+            return;
+
+        const auto supertype = getLeastSupertype(DataTypes{argument_types[0], argument_types[2]});
+        if (!supertype)
+        {
+            throw Exception(ErrorCodes::BAD_ARGUMENTS,
+                "There is no supertype for the argument type '{}' and the default value type '{}'",
+                argument_types[0]->getName(),
+                argument_types[2]->getName());
+        }
+        if (!argument_types[0]->equals(*supertype))
+        {
+            throw Exception(ErrorCodes::BAD_ARGUMENTS,
+                "The supertype '{}' for the argument type '{}' and the default value type '{}' is not the same as the argument type",
+                supertype->getName(),
+                argument_types[0]->getName(),
+                argument_types[2]->getName());
+        }
+    
+        const auto from_name = argument_types[2]->getName();
+        const auto to_name = argument_types[0]->getName();
+        ColumnsWithTypeAndName arguments
+        {
+            { argument_types[2], "" },
+            {
+                DataTypeString().createColumnConst(0, to_name),
+                std::make_shared<DataTypeString>(),
+                ""
+            }
+        };
+
+        auto get_cast_func = [&arguments]
+        {
+            FunctionOverloadResolverPtr func_builder_cast = createInternalCastOverloadResolver(CastType::accurate, {});
+            return func_builder_cast->build(arguments);
+        };
+
+        func_cast = get_cast_func();
+
+    }
+
+    ColumnPtr castColumn(const Columns & columns, const std::vector<size_t> & idx) override
+    {
+        if (!func_cast)
+            return nullptr;
+
+        ColumnsWithTypeAndName arguments
+        {
+            { columns[idx[2]], argument_types[2], "" },
+            {
+                DataTypeString().createColumnConst(columns[idx[2]]->size(), argument_types[0]->getName()),
+                std::make_shared<DataTypeString>(),
+                ""
+            }
+        };
+
+        return func_cast->execute(arguments, argument_types[0], columns[idx[2]]->size());
     }
 
     static DataTypePtr createResultType(const DataTypes & argument_types_, const std::string & name_)
@@ -2446,12 +2509,11 @@ struct WindowFunctionLagLeadInFrame final : public WindowFunction
             if (argument_types.size() > 2)
             {
                 // Column with default values is specified.
-                // The conversion through Field is inefficient, but we accept
-                // subtypes of the argument type as a default value (for convenience),
-                // and it's a pain to write conversion that respects ColumnNothing
-                // and ColumnConst and so on.
-                const IColumn & default_column = *current_block.input_columns[
-                    workspace.argument_column_indices[2]].get();
+                const IColumn & default_column =
+                    current_block.casted_columns[function_index] ?
+                        *current_block.casted_columns[function_index].get() :
+                        *current_block.input_columns[workspace.argument_column_indices[2]].get();
+
                 to.insert(default_column[transform->current_row.row]);
             }
             else
diff --git a/src/Processors/Transforms/WindowTransform.h b/src/Processors/Transforms/WindowTransform.h
index 43fa6b28019..fe4f79e997c 100644
--- a/src/Processors/Transforms/WindowTransform.h
+++ b/src/Processors/Transforms/WindowTransform.h
@@ -50,6 +50,7 @@ struct WindowTransformBlock
 {
     Columns original_input_columns;
     Columns input_columns;
+    Columns casted_columns;
     MutableColumns output_columns;
 
     size_t rows = 0;

From 498ae4358647dbff5fde2861a7113a9c9597930a Mon Sep 17 00:00:00 2001
From: Alexander Gololobov <davenger@clickhouse.com>
Date: Fri, 26 Jul 2024 10:42:23 +0200
Subject: [PATCH 0827/1488] Make 02908_many_requests_to_system_replicas less
 stressful

---
 ...08_many_requests_to_system_replicas.reference | 14 +++++++-------
 .../02908_many_requests_to_system_replicas.sh    | 16 ++++++++--------
 2 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/tests/queries/0_stateless/02908_many_requests_to_system_replicas.reference b/tests/queries/0_stateless/02908_many_requests_to_system_replicas.reference
index f376bb87044..fdefd2e3466 100644
--- a/tests/queries/0_stateless/02908_many_requests_to_system_replicas.reference
+++ b/tests/queries/0_stateless/02908_many_requests_to_system_replicas.reference
@@ -1,10 +1,10 @@
-Creating 300 tables
-900	1	1
-900	1	1
-900	1	1
-900	1	1
-Making 200 requests to system.replicas
+Creating 50 tables
+150	1	1
+150	1	1
+150	1	1
+150	1	1
+Making 100 requests to system.replicas
 Query system.replicas while waiting for other concurrent requests to finish
 0
-900
+150
 1
diff --git a/tests/queries/0_stateless/02908_many_requests_to_system_replicas.sh b/tests/queries/0_stateless/02908_many_requests_to_system_replicas.sh
index a247c99a818..81ba59fc591 100755
--- a/tests/queries/0_stateless/02908_many_requests_to_system_replicas.sh
+++ b/tests/queries/0_stateless/02908_many_requests_to_system_replicas.sh
@@ -7,8 +7,8 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
 set -e
 
-NUM_TABLES=300
-CONCURRENCY=200
+NUM_TABLES=50
+CONCURRENCY=100
 
 echo "Creating $NUM_TABLES tables"
 
@@ -46,10 +46,10 @@ wait;
 
 
 # Check results with different max_block_size
-$CLICKHOUSE_CLIENT -q 'SELECT count(), sum(total_replicas) >= 2700, sum(active_replicas) >= 2700 FROM system.replicas WHERE database=currentDatabase()'
-$CLICKHOUSE_CLIENT -q 'SELECT count(), sum(total_replicas) >= 2700, sum(active_replicas) >= 2700 FROM system.replicas WHERE database=currentDatabase() SETTINGS max_block_size=1'
-$CLICKHOUSE_CLIENT -q 'SELECT count(), sum(total_replicas) >= 2700, sum(active_replicas) >= 2700 FROM system.replicas WHERE database=currentDatabase() SETTINGS max_block_size=77'
-$CLICKHOUSE_CLIENT -q 'SELECT count(), sum(total_replicas) >= 2700, sum(active_replicas) >= 2700 FROM system.replicas WHERE database=currentDatabase() SETTINGS max_block_size=11111'
+$CLICKHOUSE_CLIENT -q 'SELECT count() as c, sum(total_replicas) >= 3*c, sum(active_replicas) >= 3*c FROM system.replicas WHERE database=currentDatabase()'
+$CLICKHOUSE_CLIENT -q 'SELECT count() as c, sum(total_replicas) >= 3*c, sum(active_replicas) >= 3*c FROM system.replicas WHERE database=currentDatabase() SETTINGS max_block_size=1'
+$CLICKHOUSE_CLIENT -q 'SELECT count() as c, sum(total_replicas) >= 3*c, sum(active_replicas) >= 3*c FROM system.replicas WHERE database=currentDatabase() SETTINGS max_block_size=77'
+$CLICKHOUSE_CLIENT -q 'SELECT count() as c, sum(total_replicas) >= 3*c, sum(active_replicas) >= 3*c FROM system.replicas WHERE database=currentDatabase() SETTINGS max_block_size=11111'
 
 
 echo "Making $CONCURRENCY requests to system.replicas"
@@ -70,8 +70,8 @@ wait;
 $CLICKHOUSE_CLIENT -nq "
 SYSTEM FLUSH LOGS;
 
--- without optimisation there are ~350K zk requests
-SELECT sum(ProfileEvents['ZooKeeperTransactions']) < 30000
+-- Check that number of ZK request is less then a half of (total replicas * concurrency)
+SELECT sum(ProfileEvents['ZooKeeperTransactions']) < (${NUM_TABLES} * 3 * ${CONCURRENCY} / 2)
   FROM system.query_log
  WHERE current_database=currentDatabase() AND log_comment='02908_many_requests';
 "

From 5bf89a433128985944cb5dd6ad6ef40a9658ff52 Mon Sep 17 00:00:00 2001
From: Pablo Marcos <pablo.marcos.oltra@clickhouse.com>
Date: Fri, 26 Jul 2024 08:42:52 +0000
Subject: [PATCH 0828/1488] Set a different instance dir when using
 pytest-xdist

This allows executing in integration tests in parallel
without directory clashes.
---
 tests/integration/helpers/cluster.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py
index 548b58a17e8..0c8278048bf 100644
--- a/tests/integration/helpers/cluster.py
+++ b/tests/integration/helpers/cluster.py
@@ -434,6 +434,11 @@ class ClickHouseCluster:
         # docker-compose removes everything non-alphanumeric from project names so we do it too.
         self.project_name = re.sub(r"[^a-z0-9]", "", project_name.lower())
         self.instances_dir_name = get_instances_dir(self.name)
+        xdist_worker = os.getenv("PYTEST_XDIST_WORKER")
+        if xdist_worker:
+            self.project_name += f"_{xdist_worker}"
+            self.instances_dir_name += f"_{xdist_worker}"
+
         self.instances_dir = p.join(self.base_dir, self.instances_dir_name)
         self.docker_logs_path = p.join(self.instances_dir, "docker.log")
         self.env_file = p.join(self.instances_dir, DEFAULT_ENV_NAME)

From 0c5c23e78477636560cd09f17b91db79e420680f Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Fri, 26 Jul 2024 10:21:36 +0200
Subject: [PATCH 0829/1488] More fixes

---
 src/Common/Exception.cpp        | 42 +++++++++++++++++++++++++++++----
 src/Common/Exception.h          | 31 ++++++++++++++++++------
 src/Common/SignalHandlers.cpp   |  2 +-
 src/Common/StackTrace.cpp       |  2 +-
 src/Common/ThreadPool.cpp       |  4 ++--
 src/Loggers/OwnSplitChannel.cpp |  7 +++++-
 src/Loggers/OwnSplitChannel.h   |  2 ++
 7 files changed, 73 insertions(+), 17 deletions(-)

diff --git a/src/Common/Exception.cpp b/src/Common/Exception.cpp
index 33befa64946..c4bd4fbd943 100644
--- a/src/Common/Exception.cpp
+++ b/src/Common/Exception.cpp
@@ -3,6 +3,7 @@
 #include <IO/ReadBufferFromString.h>
 #include <IO/ReadHelpers.h>
 #include <IO/WriteHelpers.h>
+#include <Loggers/OwnSplitChannel.h>
 #include <base/demangle.h>
 #include <Common/AtomicLogger.h>
 #include <Common/ErrorCodes.h>
@@ -100,7 +101,7 @@ Exception::Exception(const MessageMasked & msg_masked, int code, bool remote_)
 {
     if (terminate_on_any_exception)
         std::_Exit(terminate_status_code);
-    capture_thread_frame_pointers = thread_frame_pointers;
+    capture_thread_frame_pointers = getThreadFramePointers();
     handle_error_code(msg_masked.msg, code, remote, getStackFramePointers());
 }
 
@@ -110,7 +111,7 @@ Exception::Exception(MessageMasked && msg_masked, int code, bool remote_)
 {
     if (terminate_on_any_exception)
         std::_Exit(terminate_status_code);
-    capture_thread_frame_pointers = thread_frame_pointers;
+    capture_thread_frame_pointers = getThreadFramePointers();
     handle_error_code(message(), code, remote, getStackFramePointers());
 }
 
@@ -119,7 +120,7 @@ Exception::Exception(CreateFromPocoTag, const Poco::Exception & exc)
 {
     if (terminate_on_any_exception)
         std::_Exit(terminate_status_code);
-    capture_thread_frame_pointers = thread_frame_pointers;
+    capture_thread_frame_pointers = getThreadFramePointers();
 #ifdef STD_EXCEPTION_HAS_STACK_TRACE
     auto * stack_trace_frames = exc.get_stack_trace_frames();
     auto stack_trace_size = exc.get_stack_trace_size();
@@ -133,7 +134,7 @@ Exception::Exception(CreateFromSTDTag, const std::exception & exc)
 {
     if (terminate_on_any_exception)
         std::_Exit(terminate_status_code);
-    capture_thread_frame_pointers = thread_frame_pointers;
+    capture_thread_frame_pointers = getThreadFramePointers();
 #ifdef STD_EXCEPTION_HAS_STACK_TRACE
     auto * stack_trace_frames = exc.get_stack_trace_frames();
     auto stack_trace_size = exc.get_stack_trace_size();
@@ -223,10 +224,38 @@ Exception::FramePointers Exception::getStackFramePointers() const
 }
 
 thread_local bool Exception::enable_job_stack_trace = false;
-thread_local std::vector<StackTrace::FramePointers> Exception::thread_frame_pointers = {};
+thread_local bool Exception::can_use_thread_frame_pointers = false;
+thread_local Exception::ThreadFramePointers Exception::thread_frame_pointers;
+
+Exception::ThreadFramePointers::ThreadFramePointers()
+{
+    can_use_thread_frame_pointers = true;
+}
+
+Exception::ThreadFramePointers::~ThreadFramePointers()
+{
+    can_use_thread_frame_pointers = false;
+}
+
+Exception::ThreadFramePointersBase Exception::getThreadFramePointers()
+{
+    if (can_use_thread_frame_pointers)
+        return thread_frame_pointers.frame_pointers;
+
+    return {};
+}
+
+void Exception::setThreadFramePointers(ThreadFramePointersBase frame_pointers)
+{
+    if (can_use_thread_frame_pointers)
+        thread_frame_pointers.frame_pointers = std::move(frame_pointers);
+}
 
 static void tryLogCurrentExceptionImpl(Poco::Logger * logger, const std::string & start_of_message)
 {
+    if (!OwnSplitChannel::isLoggingEnabled())
+        return;
+
     try
     {
         PreformattedMessage message = getCurrentExceptionMessageAndPattern(true);
@@ -242,6 +271,9 @@ static void tryLogCurrentExceptionImpl(Poco::Logger * logger, const std::string
 
 void tryLogCurrentException(const char * log_name, const std::string & start_of_message)
 {
+    if (!OwnSplitChannel::isLoggingEnabled())
+        return;
+
     /// Under high memory pressure, new allocations throw a
     /// MEMORY_LIMIT_EXCEEDED exception.
     ///
diff --git a/src/Common/Exception.h b/src/Common/Exception.h
index 4e54c411bf1..a4f55f41caa 100644
--- a/src/Common/Exception.h
+++ b/src/Common/Exception.h
@@ -10,7 +10,6 @@
 
 #include <cerrno>
 #include <exception>
-#include <memory>
 #include <vector>
 
 #include <fmt/core.h>
@@ -49,14 +48,14 @@ public:
     {
         if (terminate_on_any_exception)
             std::terminate();
-        capture_thread_frame_pointers = thread_frame_pointers;
+        capture_thread_frame_pointers = getThreadFramePointers();
     }
 
     Exception(const PreformattedMessage & msg, int code): Exception(msg.text, code)
     {
         if (terminate_on_any_exception)
             std::terminate();
-        capture_thread_frame_pointers = thread_frame_pointers;
+        capture_thread_frame_pointers = getThreadFramePointers();
         message_format_string = msg.format_string;
         message_format_string_args = msg.format_string_args;
     }
@@ -65,18 +64,36 @@ public:
     {
         if (terminate_on_any_exception)
             std::terminate();
-        capture_thread_frame_pointers = thread_frame_pointers;
+        capture_thread_frame_pointers = getThreadFramePointers();
         message_format_string = msg.format_string;
         message_format_string_args = msg.format_string_args;
     }
 
     /// Collect call stacks of all previous jobs' schedulings leading to this thread job's execution
     static thread_local bool enable_job_stack_trace;
-    static thread_local std::vector<StackTrace::FramePointers> thread_frame_pointers;
+    static thread_local bool can_use_thread_frame_pointers;
+    /// Because of unknown order of static destructor calls,
+    /// thread_frame_pointers can already be uninitialized when a different destructor generates an exception.
+    /// To prevent such scenarios, a wrapper class is created and a function that will return empty vector
+    /// if its destructor is already called
+    using ThreadFramePointersBase = std::vector<StackTrace::FramePointers>;
+    struct ThreadFramePointers
+    {
+        ThreadFramePointers();
+        ~ThreadFramePointers();
+
+        ThreadFramePointersBase frame_pointers;
+    };
+
+    static ThreadFramePointersBase getThreadFramePointers();
+    static void setThreadFramePointers(ThreadFramePointersBase frame_pointers);
+
     /// Callback for any exception
     static std::function<void(const std::string & msg, int code, bool remote, const Exception::FramePointers & trace)> callback;
 
 protected:
+    static thread_local ThreadFramePointers thread_frame_pointers;
+
     // used to remove the sensitive information from exceptions if query_masking_rules is configured
     struct MessageMasked
     {
@@ -178,7 +195,7 @@ class ErrnoException : public Exception
 public:
     ErrnoException(std::string && msg, int code, int with_errno) : Exception(msg, code), saved_errno(with_errno)
     {
-        capture_thread_frame_pointers = thread_frame_pointers;
+        capture_thread_frame_pointers = getThreadFramePointers();
         addMessage(", {}", errnoToString(saved_errno));
     }
 
@@ -187,7 +204,7 @@ public:
     requires std::is_convertible_v<T, String>
     ErrnoException(int code, T && message) : Exception(message, code), saved_errno(errno)
     {
-        capture_thread_frame_pointers = thread_frame_pointers;
+        capture_thread_frame_pointers = getThreadFramePointers();
         addMessage(", {}", errnoToString(saved_errno));
     }
 
diff --git a/src/Common/SignalHandlers.cpp b/src/Common/SignalHandlers.cpp
index 52c83d80121..e025e49e0a3 100644
--- a/src/Common/SignalHandlers.cpp
+++ b/src/Common/SignalHandlers.cpp
@@ -89,7 +89,7 @@ void signalHandler(int sig, siginfo_t * info, void * context)
     writePODBinary(*info, out);
     writePODBinary(signal_context, out);
     writePODBinary(stack_trace, out);
-    writeVectorBinary(Exception::enable_job_stack_trace ? Exception::thread_frame_pointers : std::vector<StackTrace::FramePointers>{}, out);
+    writeVectorBinary(Exception::enable_job_stack_trace ? Exception::getThreadFramePointers() : std::vector<StackTrace::FramePointers>{}, out);
     writeBinary(static_cast<UInt32>(getThreadId()), out);
     writePODBinary(current_thread, out);
 
diff --git a/src/Common/StackTrace.cpp b/src/Common/StackTrace.cpp
index ff8765c9727..76277cbc993 100644
--- a/src/Common/StackTrace.cpp
+++ b/src/Common/StackTrace.cpp
@@ -489,7 +489,7 @@ struct CacheEntry
 
 using CacheEntryPtr = std::shared_ptr<CacheEntry>;
 
-static constinit std::atomic<bool> can_use_cache = false;
+static constinit bool can_use_cache = false;
 
 using StackTraceCacheBase = std::map<StackTraceTriple, CacheEntryPtr, std::less<>>;
 
diff --git a/src/Common/ThreadPool.cpp b/src/Common/ThreadPool.cpp
index 01f561d573f..c8f1ae99969 100644
--- a/src/Common/ThreadPool.cpp
+++ b/src/Common/ThreadPool.cpp
@@ -51,7 +51,7 @@ public:
         if (!capture_frame_pointers)
             return;
         /// Save all previous jobs call stacks and append with current
-        frame_pointers = DB::Exception::thread_frame_pointers;
+        frame_pointers = DB::Exception::getThreadFramePointers();
         frame_pointers.push_back(StackTrace().getFramePointers());
     }
 
@@ -455,7 +455,7 @@ void ThreadPoolImpl<Thread>::worker(typename std::list<Thread>::iterator thread_
         try
         {
             if (DB::Exception::enable_job_stack_trace)
-                DB::Exception::thread_frame_pointers = std::move(job_data->frame_pointers);
+                DB::Exception::setThreadFramePointers(std::move(job_data->frame_pointers));
 
             CurrentMetrics::Increment metric_active_pool_threads(metric_active_threads);
 
diff --git a/src/Loggers/OwnSplitChannel.cpp b/src/Loggers/OwnSplitChannel.cpp
index c0e8514c62a..e29d2a1e0aa 100644
--- a/src/Loggers/OwnSplitChannel.cpp
+++ b/src/Loggers/OwnSplitChannel.cpp
@@ -18,6 +18,11 @@ namespace DB
 
 static constinit std::atomic<bool> allow_logging{true};
 
+bool OwnSplitChannel::isLoggingEnabled()
+{
+    return allow_logging;
+}
+
 void OwnSplitChannel::disableLogging()
 {
     allow_logging = false;
@@ -25,7 +30,7 @@ void OwnSplitChannel::disableLogging()
 
 void OwnSplitChannel::log(const Poco::Message & msg)
 {
-    if (!allow_logging)
+    if (!isLoggingEnabled())
         return;
 
 #ifndef WITHOUT_TEXT_LOG
diff --git a/src/Loggers/OwnSplitChannel.h b/src/Loggers/OwnSplitChannel.h
index 9872a4fb558..9de55f330be 100644
--- a/src/Loggers/OwnSplitChannel.h
+++ b/src/Loggers/OwnSplitChannel.h
@@ -41,6 +41,8 @@ public:
 
     static void disableLogging();
 
+    static bool isLoggingEnabled();
+
 private:
     void logSplit(const Poco::Message & msg);
     void tryLogSplit(const Poco::Message & msg);

From c7330252cf581441b95c51b47977f597eb41734e Mon Sep 17 00:00:00 2001
From: Maksim Kita <kitaetoya@gmail.com>
Date: Fri, 26 Jul 2024 11:47:49 +0300
Subject: [PATCH 0830/1488] Disable convert OUTER JOIN to INNER JOIN
 optimization for non ALL JOIN strictness

---
 .../QueryPlan/Optimizations/convertOuterJoinToInnerJoin.cpp  | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/Processors/QueryPlan/Optimizations/convertOuterJoinToInnerJoin.cpp b/src/Processors/QueryPlan/Optimizations/convertOuterJoinToInnerJoin.cpp
index d90f0e152e7..d9296f10a98 100644
--- a/src/Processors/QueryPlan/Optimizations/convertOuterJoinToInnerJoin.cpp
+++ b/src/Processors/QueryPlan/Optimizations/convertOuterJoinToInnerJoin.cpp
@@ -23,7 +23,10 @@ size_t tryConvertOuterJoinToInnerJoin(QueryPlan::Node * parent_node, QueryPlan::
         return 0;
 
     const auto & table_join = join->getJoin()->getTableJoin();
-    if (table_join.strictness() == JoinStrictness::Asof)
+
+    /// Any JOIN issue https://github.com/ClickHouse/ClickHouse/issues/66447
+    /// Anti JOIN issue https://github.com/ClickHouse/ClickHouse/issues/67156
+    if (table_join.strictness() != JoinStrictness::All)
         return 0;
 
     /// TODO: Support join_use_nulls

From aaa25454b31d854338200b335d7ac6442e959af4 Mon Sep 17 00:00:00 2001
From: Blargian <shaunstruwig.ss@gmail.com>
Date: Fri, 26 Jul 2024 10:58:45 +0200
Subject: [PATCH 0831/1488] Additional formatting fixes

---
 .../functions/type-conversion-functions.md         | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/docs/en/sql-reference/functions/type-conversion-functions.md b/docs/en/sql-reference/functions/type-conversion-functions.md
index 4326753216e..87d824ec5bb 100644
--- a/docs/en/sql-reference/functions/type-conversion-functions.md
+++ b/docs/en/sql-reference/functions/type-conversion-functions.md
@@ -571,7 +571,7 @@ Result:
 
 ## toInt32OrZero
 
-Like [`toInt32`](#toint32), this function converts an input value to a value of type [Int8](../data-types/int-uint.md) but returns `0` in case of an error.
+Like [`toInt32`](#toint32), this function converts an input value to a value of type [Int32](../data-types/int-uint.md) but returns `0` in case of an error.
 
 **Syntax**
 
@@ -767,7 +767,7 @@ SELECT toInt64('9223372036854775808') == --9223372036854775808;
 
 **Returned value**
 
-- 64-bit integer value. [Int64](../data-types/int-uint.md). [Int64](../data-types/int-uint.md).
+- 64-bit integer value. [Int64](../data-types/int-uint.md).
 
 :::note
 The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
@@ -800,7 +800,7 @@ Result:
 
 ## toInt64OrZero
 
-Like [`toInt64`](#toint64), this function converts an input value to a value of type [Int8](../data-types/int-uint.md) but returns `0` in case of an error.
+Like [`toInt64`](#toint64), this function converts an input value to a value of type [Int64](../data-types/int-uint.md) but returns `0` in case of an error.
 
 **Syntax**
 
@@ -878,7 +878,7 @@ Types for which `\N` is returned:
 
 **Returned value**
 
-- Integer value of type `Int64` if successful, otherwise `NULL`. [Int64](../data-types/int-uint.md) / [NULL](../data-types/nullable.md).
+- 64-bit integer value if successful, otherwise `NULL`. [Int64](../data-types/int-uint.md) / [NULL](../data-types/nullable.md).
 
 :::note
 The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
@@ -935,7 +935,7 @@ Types for which the default value is returned:
 
 **Returned value**
 
-- Integer value of type `Int64` if successful, otherwise returns the default value. [Int64](../data-types/int-uint.md).
+- 64-bit integer value if successful, otherwise returns the default value. [Int64](../data-types/int-uint.md).
 
 :::note
 - The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
@@ -1028,7 +1028,7 @@ Result:
 
 ## toInt128OrZero
 
-Like [`toInt128`](#toint128), this function converts an input value to a value of type [Int8](../data-types/int-uint.md) but returns `0` in case of an error.
+Like [`toInt128`](#toint128), this function converts an input value to a value of type [Int128](../data-types/int-uint.md) but returns `0` in case of an error.
 
 **Syntax**
 
@@ -1256,7 +1256,7 @@ Result:
 
 ## toInt256OrZero
 
-Like [`toInt256`](#toint256), this function converts an input value to a value of type [Int8](../data-types/int-uint.md) but returns `0` in case of an error.
+Like [`toInt256`](#toint256), this function converts an input value to a value of type [Int256](../data-types/int-uint.md) but returns `0` in case of an error.
 
 **Syntax**
 

From 338685cc79a5358246977f2ba039230a615c6ea6 Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Fri, 26 Jul 2024 10:59:17 +0200
Subject: [PATCH 0832/1488] Fix build

---
 programs/odbc-bridge/tests/CMakeLists.txt | 2 +-
 src/CMakeLists.txt                        | 1 +
 src/Common/mysqlxx/tests/CMakeLists.txt   | 2 +-
 3 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/programs/odbc-bridge/tests/CMakeLists.txt b/programs/odbc-bridge/tests/CMakeLists.txt
index f1411dbb554..2f63aed7942 100644
--- a/programs/odbc-bridge/tests/CMakeLists.txt
+++ b/programs/odbc-bridge/tests/CMakeLists.txt
@@ -1,2 +1,2 @@
 clickhouse_add_executable (validate-odbc-connection-string validate-odbc-connection-string.cpp ../validateODBCConnectionString.cpp)
-target_link_libraries (validate-odbc-connection-string PRIVATE clickhouse_common_io clickhouse_common_config)
+target_link_libraries (validate-odbc-connection-string PRIVATE clickhouse_common_io clickhouse_common_config loggers_no_text_log)
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 0f84dd35320..fede7d69105 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -638,6 +638,7 @@ if (ENABLE_TESTS)
         dbms
         clickhouse_common_config
         clickhouse_common_zookeeper
+        loggers
         hilite_comparator)
 
     if (TARGET ch_contrib::simdjson)
diff --git a/src/Common/mysqlxx/tests/CMakeLists.txt b/src/Common/mysqlxx/tests/CMakeLists.txt
index f62908ddcaf..53bee778470 100644
--- a/src/Common/mysqlxx/tests/CMakeLists.txt
+++ b/src/Common/mysqlxx/tests/CMakeLists.txt
@@ -1,2 +1,2 @@
 clickhouse_add_executable (mysqlxx_pool_test mysqlxx_pool_test.cpp)
-target_link_libraries (mysqlxx_pool_test PRIVATE mysqlxx clickhouse_common_config)
+target_link_libraries (mysqlxx_pool_test PRIVATE mysqlxx clickhouse_common_config loggers_no_text_log)

From 8d13461fb74fc991b73382d04a9bc7a9fd3425fa Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Fri, 26 Jul 2024 09:03:37 +0000
Subject: [PATCH 0833/1488] Another fix.

---
 src/Storages/StorageDistributed.cpp | 7 ++-----
 src/Storages/StorageMerge.cpp       | 8 ++++++++
 2 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp
index 9b417cda177..07892971ec2 100644
--- a/src/Storages/StorageDistributed.cpp
+++ b/src/Storages/StorageDistributed.cpp
@@ -847,11 +847,8 @@ void StorageDistributed::read(
         /** For distributed tables we do not need constants in header, since we don't send them to remote servers.
           * Moreover, constants can break some functions like `hostName` that are constants only for local queries.
           */
-        if (processed_stage != QueryProcessingStage::Complete)
-        {
-            for (auto & column : header)
-                column.column = column.column->convertToFullColumnIfConst();
-        }
+        for (auto & column : header)
+            column.column = column.column->convertToFullColumnIfConst();
         modified_query_info.query = queryNodeToDistributedSelectQuery(query_tree_distributed);
 
         modified_query_info.query_tree = std::move(query_tree_distributed);
diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp
index f5bc183931f..9962da3d6de 100644
--- a/src/Storages/StorageMerge.cpp
+++ b/src/Storages/StorageMerge.cpp
@@ -369,6 +369,14 @@ void StorageMerge::read(
     /// What will be result structure depending on query processed stage in source tables?
     Block common_header = getHeaderForProcessingStage(column_names, storage_snapshot, query_info, local_context, processed_stage);
 
+    if (local_context->getSettingsRef().allow_experimental_analyzer && processed_stage == QueryProcessingStage::Complete)
+    {
+        /// Remove constants.
+        /// For StorageDistributed some functions like `hostName` that are constants only for local queries.
+        for (auto & column : common_header)
+            column.column = column.column->convertToFullColumnIfConst();
+    }
+
     auto step = std::make_unique<ReadFromMerge>(
         column_names,
         query_info,

From 02bfe82192fa4aa6ebb3e7b9192ec6f334fbfc56 Mon Sep 17 00:00:00 2001
From: Blargian <shaunstruwig.ss@gmail.com>
Date: Fri, 26 Jul 2024 11:19:46 +0200
Subject: [PATCH 0834/1488] rename filesystemFree to fiilesystemUnreserved

---
 docs/en/sql-reference/functions/other-functions.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/en/sql-reference/functions/other-functions.md b/docs/en/sql-reference/functions/other-functions.md
index b7e4094f30e..79bffe00d01 100644
--- a/docs/en/sql-reference/functions/other-functions.md
+++ b/docs/en/sql-reference/functions/other-functions.md
@@ -2102,14 +2102,14 @@ Result:
 └─────────────────┘
 ```
 
-## filesystemFree
+## filesystemUnreserved
 
-Returns the total amount of the free space on the filesystem hosting the database persistence. See also `filesystemAvailable`
+Returns the total amount of the free space on the filesystem hosting the database persistence. (previously `filesystemFree`). See also [`filesystemAvailable`](#filesystemavailable).
 
 **Syntax**
 
 ```sql
-filesystemFree()
+filesystemUnreserved()
 ```
 
 **Returned value**
@@ -2121,7 +2121,7 @@ filesystemFree()
 Query:
 
 ```sql
-SELECT formatReadableSize(filesystemFree()) AS "Free space";
+SELECT formatReadableSize(filesystemUnreserved()) AS "Free space";
 ```
 
 Result:

From 434571d496a6ca6fc1b0038ead560572d0553ee5 Mon Sep 17 00:00:00 2001
From: Maksim Kita <kitaetoya@gmail.com>
Date: Fri, 26 Jul 2024 12:40:20 +0300
Subject: [PATCH 0835/1488] Added tests

---
 ...uter_join_to_inner_join_any_join.reference |  3 ++
 ...vert_outer_join_to_inner_join_any_join.sql | 33 ++++++++++++++
 ...ter_join_to_inner_join_anti_join.reference | 19 ++++++++
 ...ert_outer_join_to_inner_join_anti_join.sql | 45 +++++++++++++++++++
 4 files changed, 100 insertions(+)
 create mode 100644 tests/queries/0_stateless/03210_convert_outer_join_to_inner_join_any_join.reference
 create mode 100644 tests/queries/0_stateless/03210_convert_outer_join_to_inner_join_any_join.sql
 create mode 100644 tests/queries/0_stateless/03211_convert_outer_join_to_inner_join_anti_join.reference
 create mode 100644 tests/queries/0_stateless/03211_convert_outer_join_to_inner_join_anti_join.sql

diff --git a/tests/queries/0_stateless/03210_convert_outer_join_to_inner_join_any_join.reference b/tests/queries/0_stateless/03210_convert_outer_join_to_inner_join_any_join.reference
new file mode 100644
index 00000000000..3d6a23045fb
--- /dev/null
+++ b/tests/queries/0_stateless/03210_convert_outer_join_to_inner_join_any_join.reference
@@ -0,0 +1,3 @@
+1	tx1	US
+1	tx2	US
+1	tx3	US
diff --git a/tests/queries/0_stateless/03210_convert_outer_join_to_inner_join_any_join.sql b/tests/queries/0_stateless/03210_convert_outer_join_to_inner_join_any_join.sql
new file mode 100644
index 00000000000..599875e90cf
--- /dev/null
+++ b/tests/queries/0_stateless/03210_convert_outer_join_to_inner_join_any_join.sql
@@ -0,0 +1,33 @@
+DROP TABLE IF EXISTS user_country;
+DROP TABLE IF EXISTS user_transactions;
+
+CREATE TABLE user_country (
+    user_id UInt64,
+    country String
+)
+ENGINE = ReplacingMergeTree
+ORDER BY user_id;
+
+CREATE TABLE user_transactions (
+    user_id UInt64,
+    transaction_id String
+)
+ENGINE = MergeTree
+ORDER BY user_id;
+
+INSERT INTO user_country (user_id, country) VALUES (1, 'US');
+INSERT INTO user_transactions (user_id, transaction_id) VALUES (1, 'tx1'), (1, 'tx2'), (1, 'tx3'), (2, 'tx1');
+
+-- Expected 3 rows, got only 1. Removing 'ANY' and adding 'FINAL' fixes
+-- the issue (but it is not always possible). Moving filter by 'country' to
+-- an outer query doesn't help. Query without filter by 'country' works
+-- as expected (returns 3 rows).
+SELECT * FROM user_transactions
+ANY LEFT JOIN user_country USING (user_id)
+WHERE
+    user_id = 1
+    AND country = 'US'
+ORDER BY ALL;
+
+DROP TABLE user_country;
+DROP TABLE user_transactions;
diff --git a/tests/queries/0_stateless/03211_convert_outer_join_to_inner_join_anti_join.reference b/tests/queries/0_stateless/03211_convert_outer_join_to_inner_join_anti_join.reference
new file mode 100644
index 00000000000..d717a29ab23
--- /dev/null
+++ b/tests/queries/0_stateless/03211_convert_outer_join_to_inner_join_anti_join.reference
@@ -0,0 +1,19 @@
+DATA
+   ┏━━━━━━━━━━━┳━━━━━━━━━━━┳━━━━┓
+   ┃        c0 ┃        c1 ┃ c2 ┃
+   ┡━━━━━━━━━━━╇━━━━━━━━━━━╇━━━━┩
+1. │ 826636805 │         0 │    │
+   ├───────────┼───────────┼────┤
+2. │         0 │ 150808457 │    │
+   └───────────┴───────────┴────┘
+NUMBER OF ROWS IN FIRST SHOULD BE EQUAL TO SECOND
+FISRT
+
+SECOND
+1
+TO DEBUG I TOOK JUST A SUBQUERY AND IT HAS 1 ROW
+THIRD
+1
+AND I ADDED SINGLE CONDITION THAT CONDITION <>0 THAT IS 1 IN THIRD QUERY AND IT HAS NO RESULT!!!
+FOURTH
+1
diff --git a/tests/queries/0_stateless/03211_convert_outer_join_to_inner_join_anti_join.sql b/tests/queries/0_stateless/03211_convert_outer_join_to_inner_join_anti_join.sql
new file mode 100644
index 00000000000..77b1d52dd18
--- /dev/null
+++ b/tests/queries/0_stateless/03211_convert_outer_join_to_inner_join_anti_join.sql
@@ -0,0 +1,45 @@
+DROP TABLE IF EXISTS t0;
+
+CREATE TABLE t0 (c0 Int32, c1 Int32, c2 String) ENGINE = Log() ;
+INSERT INTO t0(c0, c1, c2) VALUES (826636805,0, ''), (0, 150808457, '');
+
+SELECT 'DATA';
+SELECT * FROM t0 FORMAT PrettyMonoBlock;
+
+SELECT 'NUMBER OF ROWS IN FIRST SHOULD BE EQUAL TO SECOND';
+
+
+SELECT 'FISRT';
+SELECT left.c2 FROM t0 AS left
+LEFT ANTI JOIN t0 AS right_0 ON ((left.c0)=(right_0.c1))
+WHERE (abs ((- ((sign (right_0.c1))))));
+
+SELECT 'SECOND';
+SELECT SUM(check <> 0)
+FROM
+(
+  SELECT (abs ((- ((sign (right_0.c1)))))) AS `check`
+  FROM t0 AS left
+  LEFT ANTI JOIN t0 AS right_0 ON ((left.c0)=(right_0.c1))
+);
+
+
+SELECT 'TO DEBUG I TOOK JUST A SUBQUERY AND IT HAS 1 ROW';
+
+SELECT 'THIRD';
+
+SELECT (abs ((- ((sign (right_0.c1)))))) AS `check`
+FROM t0 AS left
+LEFT ANTI JOIN t0 AS right_0 ON ((left.c0)=(right_0.c1));
+
+
+SELECT 'AND I ADDED SINGLE CONDITION THAT CONDITION <>0 THAT IS 1 IN THIRD QUERY AND IT HAS NO RESULT!!!';
+
+
+SELECT 'FOURTH';
+SELECT (abs ((- ((sign (right_0.c1)))))) AS `check`
+FROM t0 AS left
+LEFT ANTI JOIN t0 AS right_0 ON ((left.c0)=(right_0.c1))
+WHERE check <> 0;
+
+DROP TABLE t0;

From b3828b038dbcc9c5cf71b99d58f06497c2af3bd6 Mon Sep 17 00:00:00 2001
From: Shaun Struwig <41984034+Blargian@users.noreply.github.com>
Date: Fri, 26 Jul 2024 11:49:04 +0200
Subject: [PATCH 0836/1488] add `filesystemUnreserved`

---
 utils/check-style/aspell-ignore/en/aspell-dict.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt
index 8e4e4fafe29..1a324b98ff4 100644
--- a/utils/check-style/aspell-ignore/en/aspell-dict.txt
+++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt
@@ -1622,6 +1622,7 @@ filesystem
 filesystemAvailable
 filesystemCapacity
 filesystemFree
+filesystemUnreserved
 filesystems
 finalizeAggregation
 fips

From ca9bf2c67c8ac16d4fd18f2def6e4d3dfea62971 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 26 Jul 2024 11:53:48 +0200
Subject: [PATCH 0837/1488] Fix tidy

---
 src/Common/ZooKeeper/ZooKeeper.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Common/ZooKeeper/ZooKeeper.cpp b/src/Common/ZooKeeper/ZooKeeper.cpp
index 1250e1273b9..7448d73cbbc 100644
--- a/src/Common/ZooKeeper/ZooKeeper.cpp
+++ b/src/Common/ZooKeeper/ZooKeeper.cpp
@@ -136,7 +136,7 @@ void ZooKeeper::init(ZooKeeperArgs args_, std::unique_ptr<Coordination::IKeeper>
             LOG_TRACE(log, "Initialized, hosts: {}, chroot: {}", fmt::join(args.hosts, ","), args.chroot);
 
         /// If the balancing strategy has an optimal node then it will be the first in the list
-        bool connected_to_suboptimal_node = node_idx && *node_idx != shuffled_hosts[0].original_index;
+        bool connected_to_suboptimal_node = node_idx && static_cast<UInt8>(*node_idx) != shuffled_hosts[0].original_index;
         bool respect_az = args.prefer_local_availability_zone && !args.client_availability_zone.empty();
         bool may_benefit_from_reconnecting = respect_az || args.get_priority_load_balancing.hasOptimalNode();
         if (connected_to_suboptimal_node && may_benefit_from_reconnecting)

From 2519f9ed4252020c6a9fb21ef1410c87f4053200 Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Fri, 26 Jul 2024 12:08:16 +0200
Subject: [PATCH 0838/1488] Only support archs

---
 contrib/numactl-cmake/CMakeLists.txt | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/contrib/numactl-cmake/CMakeLists.txt b/contrib/numactl-cmake/CMakeLists.txt
index 5d086366c7f..a72ff11e485 100644
--- a/contrib/numactl-cmake/CMakeLists.txt
+++ b/contrib/numactl-cmake/CMakeLists.txt
@@ -1,4 +1,14 @@
-option (ENABLE_NUMACTL "Enable numactl" ${ENABLE_LIBRARIES})
+if (NOT (
+    OS_LINUX AND (ARCH_AMD64 OR ARCH_AARCH64 OR ARCH_LOONGARCH64))
+)
+    if (ENABLE_NUMACTL)
+        message (${RECONFIGURE_MESSAGE_LEVEL}
+            "numactl is disabled implicitly because the OS or architecture is not supported. Use -DENABLE_NUMACTL=0")
+    endif ()
+    set (ENABLE_NUMACTL OFF)
+else()
+    option (ENABLE_NUMACTL "Enable numactl" ${ENABLE_LIBRARIES})
+endif()
 
 if (NOT ENABLE_NUMACTL)
     message (STATUS "Not using numactl")

From 72ebff825c0752e66cefa4f367ce43ff23d77703 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Fri, 26 Jul 2024 12:19:09 +0200
Subject: [PATCH 0839/1488] Reduce max time of
 00763_long_lock_buffer_alter_destination_table

---
 ...ong_lock_buffer_alter_destination_table.sh | 33 +++++++++++++++----
 1 file changed, 27 insertions(+), 6 deletions(-)

diff --git a/tests/queries/0_stateless/00763_long_lock_buffer_alter_destination_table.sh b/tests/queries/0_stateless/00763_long_lock_buffer_alter_destination_table.sh
index 7e2384cfc52..c12b4426740 100755
--- a/tests/queries/0_stateless/00763_long_lock_buffer_alter_destination_table.sh
+++ b/tests/queries/0_stateless/00763_long_lock_buffer_alter_destination_table.sh
@@ -16,18 +16,39 @@ ${CLICKHOUSE_CLIENT} --query="CREATE TABLE buffer_00763_1 (s String) ENGINE = Bu
 ${CLICKHOUSE_CLIENT} --query="CREATE TABLE mt_00763_1 (x UInt32, s String) ENGINE = MergeTree ORDER BY x"
 ${CLICKHOUSE_CLIENT} --query="INSERT INTO mt_00763_1 VALUES (1, '1'), (2, '2'), (3, '3')"
 
-function thread1()
+function thread_alter()
 {
-    seq 1 300 | sed -r -e 's/.+/ALTER TABLE mt_00763_1 MODIFY column s UInt32; ALTER TABLE mt_00763_1 MODIFY column s String;/' | ${CLICKHOUSE_CLIENT} --multiquery --ignore-error ||:
+    local TIMELIMIT=$((SECONDS+$1))
+    local it=0
+    while [ $SECONDS -lt "$TIMELIMIT" ] && [ $it -lt 300 ];
+    do
+        it=$((it+1))
+        $CLICKHOUSE_CLIENT --multiquery --ignore-error -q "
+            ALTER TABLE mt_00763_1 MODIFY column s UInt32;
+            ALTER TABLE mt_00763_1 MODIFY column s String;
+        " ||:
+    done
 }
 
-function thread2()
+function thread_query()
 {
-    seq 1 2000 | sed -r -e 's/.+/SELECT sum(length(s)) FROM buffer_00763_1;/' | ${CLICKHOUSE_CLIENT} --multiquery --ignore-error 2>&1 | grep -vP '(^3$|^Received exception from server|^Code: 473)'
+    local TIMELIMIT=$((SECONDS+$1))
+    local it=0
+    while [ $SECONDS -lt "$TIMELIMIT" ] && [ $it -lt 2000 ];
+    do
+        it=$((it+1))
+        $CLICKHOUSE_CLIENT --multiquery --ignore-error -q "
+            SELECT sum(length(s)) FROM buffer_00763_1;
+        " 2>&1 | grep -vP '(^3$|^Received exception from server|^Code: 473)'
+    done
 }
 
-thread1 &
-thread2 &
+export -f thread_alter
+export -f thread_query
+
+TIMEOUT=30
+thread_alter $TIMEOUT &
+thread_query $TIMEOUT &
 
 wait
 

From 98418120cd3167983b5436834d7c568cb42865af Mon Sep 17 00:00:00 2001
From: Pablo Marcos <pablo.marcos.oltra@clickhouse.com>
Date: Fri, 26 Jul 2024 10:21:43 +0000
Subject: [PATCH 0840/1488] Add parallel integration test execution to doc

---
 tests/integration/README.md | 67 ++++++++++++++++++++++++++++++++++---
 1 file changed, 62 insertions(+), 5 deletions(-)

diff --git a/tests/integration/README.md b/tests/integration/README.md
index cde4cb05aec..5d4fa407e3f 100644
--- a/tests/integration/README.md
+++ b/tests/integration/README.md
@@ -16,7 +16,7 @@ Don't use Docker from your system repository.
 * [py.test](https://docs.pytest.org/) testing framework. To install: `sudo -H pip install pytest`
 * [docker-compose](https://docs.docker.com/compose/) and additional python libraries. To install:
 
-```
+```bash
 sudo -H pip install \
     PyMySQL \
     avro \
@@ -78,7 +78,7 @@ Notes:
 * Some tests maybe require a lot of resources (CPU, RAM, etc.). Better not try large tests like `test_distributed_ddl*` on your laptop.
 
 You can run tests via `./runner` script and pass pytest arguments as last arg:
-```
+```bash
 $ ./runner --binary $HOME/ClickHouse/programs/clickhouse  --odbc-bridge-binary $HOME/ClickHouse/programs/clickhouse-odbc-bridge --base-configs-dir $HOME/ClickHouse/programs/server/ 'test_ssl_cert_authentication -ss'
 Start tests
 ====================================================================================================== test session starts ======================================================================================================
@@ -102,7 +102,7 @@ test_ssl_cert_authentication/test.py::test_create_user PASSED
 ```
 
 Path to binary and configs maybe specified via env variables:
-```
+```bash
 $ export CLICKHOUSE_TESTS_BASE_CONFIG_DIR=$HOME/ClickHouse/programs/server/
 $ export CLICKHOUSE_TESTS_SERVER_BIN_PATH=$HOME/ClickHouse/programs/clickhouse
 $ export CLICKHOUSE_TESTS_ODBC_BRIDGE_BIN_PATH=$HOME/ClickHouse/programs/clickhouse-odbc-bridge
@@ -121,6 +121,63 @@ test_odbc_interaction/test.py ......                                     [100%]
 You can just open shell inside a container by overwritting the command:
 ./runner --command=bash
 
+### Parallel test execution
+
+On the CI, we run a number of parallel runners (5 at the time of this writing), each on its own
+Docker container. These runner containers spawn more containers for the services needed such as
+ZooKeeper, MySQL, PostgreSQL and minio, among others. Within each runner, tests are parallelized
+using [pytest-xdist](https://pytest-xdist.readthedocs.io/en/stable/). We're using `--dist=loadfile`
+to [distribute the load](https://pytest-xdist.readthedocs.io/en/stable/distribution.html). In other
+words: tests are grouped by module for test functions and by class for test methods. This means that
+any test within the same module (or any class) will never execute their tests in parallel. They'll
+be executed on the same worker one after the other.
+
+If the test supports parallel and repeated execution, you can run a bunch of them in parallel to
+look for flakiness. We use [pytest-repeat](https://pypi.org/project/pytest-repeat/) to set the
+number of times we want to execute a test through the `--count` argument. Then, `-n` sets the number
+of parallel workers for `pytest-xdist`.
+
+```bash
+$ export CLICKHOUSE_TESTS_BASE_CONFIG_DIR=$HOME/ClickHouse/programs/server/
+$ export CLICKHOUSE_TESTS_SERVER_BIN_PATH=$HOME/ClickHouse/programs/clickhouse
+$ export CLICKHOUSE_TESTS_ODBC_BRIDGE_BIN_PATH=$HOME/ClickHouse/programs/clickhouse-odbc-bridge
+$ ./runner 'test_storage_s3_queue/test.py::test_max_set_age -- --count 10 -n 5'
+Start tests
+=============================================================================== test session starts ================================================================================
+platform linux -- Python 3.10.12, pytest-7.4.4, pluggy-1.5.0 -- /usr/bin/python3
+cachedir: .pytest_cache
+rootdir: /ClickHouse/tests/integration
+configfile: pytest.ini
+plugins: reportlog-0.4.0, xdist-3.5.0, random-0.2, repeat-0.9.3, order-1.0.0, timeout-2.2.0
+timeout: 900.0s
+timeout method: signal
+timeout func_only: False
+5 workers [10 items]
+scheduling tests via LoadScheduling
+
+test_storage_s3_queue/test.py::test_max_set_age[9-10]
+test_storage_s3_queue/test.py::test_max_set_age[7-10]
+test_storage_s3_queue/test.py::test_max_set_age[5-10]
+test_storage_s3_queue/test.py::test_max_set_age[1-10]
+test_storage_s3_queue/test.py::test_max_set_age[3-10]
+[gw3] [ 10%] PASSED test_storage_s3_queue/test.py::test_max_set_age[7-10]
+test_storage_s3_queue/test.py::test_max_set_age[8-10]
+[gw4] [ 20%] PASSED test_storage_s3_queue/test.py::test_max_set_age[9-10]
+test_storage_s3_queue/test.py::test_max_set_age[10-10]
+[gw0] [ 30%] PASSED test_storage_s3_queue/test.py::test_max_set_age[1-10]
+test_storage_s3_queue/test.py::test_max_set_age[2-10]
+[gw1] [ 40%] PASSED test_storage_s3_queue/test.py::test_max_set_age[3-10]
+test_storage_s3_queue/test.py::test_max_set_age[4-10]
+[gw2] [ 50%] PASSED test_storage_s3_queue/test.py::test_max_set_age[5-10]
+test_storage_s3_queue/test.py::test_max_set_age[6-10]
+[gw3] [ 60%] PASSED test_storage_s3_queue/test.py::test_max_set_age[8-10]
+[gw4] [ 70%] PASSED test_storage_s3_queue/test.py::test_max_set_age[10-10]
+[gw0] [ 80%] PASSED test_storage_s3_queue/test.py::test_max_set_age[2-10]
+[gw1] [ 90%] PASSED test_storage_s3_queue/test.py::test_max_set_age[4-10]
+[gw2] [100%] PASSED test_storage_s3_queue/test.py::test_max_set_age[6-10]
+========================================================================== 10 passed in 120.65s (0:02:00) ==========================================================================
+```
+
 ### Rebuilding the docker containers
 
 The main container used for integration tests lives in `docker/test/integration/base/Dockerfile`. Rebuild it with
@@ -149,7 +206,7 @@ will automagically detect the types of variables and only the small diff of two
 
 If tests failing for mysterious reasons, this may help:
 
-```
+```bash
 sudo service docker stop
 sudo bash -c 'rm -rf /var/lib/docker/*'
 sudo service docker start
@@ -159,6 +216,6 @@ sudo service docker start
 
 On Ubuntu 20.10 and later in host network mode (default) one may encounter problem with nested containers not seeing each other. It happens because legacy and nftables rules are out of sync. Problem can be solved by:
 
-```
+```bash
 sudo iptables -P FORWARD ACCEPT
 ```

From 7f80dab6927316f5c6c56e51ba439d01161f7567 Mon Sep 17 00:00:00 2001
From: Nikita Fomichev <nikita.fomichev@clickhouse.com>
Date: Fri, 26 Jul 2024 12:34:36 +0200
Subject: [PATCH 0841/1488] CI push

---
 docker/test/util/process_functional_tests_result.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/docker/test/util/process_functional_tests_result.py b/docker/test/util/process_functional_tests_result.py
index dbe50eeade0..3da1a8f3674 100755
--- a/docker/test/util/process_functional_tests_result.py
+++ b/docker/test/util/process_functional_tests_result.py
@@ -180,7 +180,6 @@ def process_result(result_path, broken_tests):
             for result in test_results:
                 if result[1] == "FAIL":
                     result[1] = "SERVER_DIED"
-
             test_results.append(["Server died", "FAIL", "0", ""])
         elif not success_finish:
             description = "Tests are not finished, "

From 60cca77c8a415142fe6f181b25aaed84232ea3c0 Mon Sep 17 00:00:00 2001
From: Blargian <shaunstruwig.ss@gmail.com>
Date: Fri, 26 Jul 2024 12:39:35 +0200
Subject: [PATCH 0842/1488] add example for materialize function

---
 .../functions/other-functions.md              | 32 ++++++++++++++++++-
 1 file changed, 31 insertions(+), 1 deletion(-)

diff --git a/docs/en/sql-reference/functions/other-functions.md b/docs/en/sql-reference/functions/other-functions.md
index b7e4094f30e..797607e552a 100644
--- a/docs/en/sql-reference/functions/other-functions.md
+++ b/docs/en/sql-reference/functions/other-functions.md
@@ -346,7 +346,9 @@ Result:
 ## materialize
 
 Turns a constant into a full column containing a single value.
-Full columns and constants are represented differently in memory. Functions usually execute different code for normal and constant arguments, although the result should typically be the same. This function can be used to debug this behavior.
+Full columns and constants are represented differently in memory.
+Functions usually execute different code for normal and constant arguments, although the result should typically be the same. 
+This function can be used to debug this behavior.
 
 **Syntax**
 
@@ -354,6 +356,34 @@ Full columns and constants are represented differently in memory. Functions usua
 materialize(x)
 ```
 
+**Parameters**
+
+- `x` — A constant. [Constant](../functions/index.md/#constants).
+
+**Returned value**
+
+- A column containing a single value `x`.
+
+**Example**
+
+In the example below the `countMatches` function expects a constant second argument. 
+This behaviour can be debugged by using the `materialize` function to turn a constant into a full column, 
+verifying that the function throws an error for a non-constant argument.
+
+Query:
+
+```sql
+SELECT countMatches('foobarfoo', 'foo');
+SELECT countMatches('foobarfoo', materialize('foo'));
+```
+
+Result:
+
+```response
+2
+Code: 44. DB::Exception: Received from localhost:9000. DB::Exception: Illegal type of argument #2 'pattern' of function countMatches, expected constant String, got String
+```
+
 ## ignore
 
 Accepts any arguments, including `NULL` and does nothing. Always returns 0.

From 1ebafccc13ea69ba06e2450014fd15d39facdcaa Mon Sep 17 00:00:00 2001
From: Shaun Struwig <41984034+Blargian@users.noreply.github.com>
Date: Fri, 26 Jul 2024 12:42:07 +0200
Subject: [PATCH 0843/1488] add `joinGetOrNull`

---
 utils/check-style/aspell-ignore/en/aspell-dict.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt
index 7de065cc589..182e1d2cb33 100644
--- a/utils/check-style/aspell-ignore/en/aspell-dict.txt
+++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt
@@ -1862,6 +1862,7 @@ jdbc
 jemalloc
 jeprof
 joinGet
+joinGetOrNull
 json
 jsonMergePatch
 jsonasstring

From 3f1dbdfce978bab2b2ce2aedecdbb5afbf54c4a0 Mon Sep 17 00:00:00 2001
From: Pablo Marcos <pablo.marcos.oltra@clickhouse.com>
Date: Fri, 26 Jul 2024 11:01:10 +0000
Subject: [PATCH 0844/1488] Clarify documentation

---
 tests/integration/README.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/integration/README.md b/tests/integration/README.md
index 5d4fa407e3f..ab984b7bd04 100644
--- a/tests/integration/README.md
+++ b/tests/integration/README.md
@@ -127,10 +127,10 @@ On the CI, we run a number of parallel runners (5 at the time of this writing),
 Docker container. These runner containers spawn more containers for the services needed such as
 ZooKeeper, MySQL, PostgreSQL and minio, among others. Within each runner, tests are parallelized
 using [pytest-xdist](https://pytest-xdist.readthedocs.io/en/stable/). We're using `--dist=loadfile`
-to [distribute the load](https://pytest-xdist.readthedocs.io/en/stable/distribution.html). In other
-words: tests are grouped by module for test functions and by class for test methods. This means that
-any test within the same module (or any class) will never execute their tests in parallel. They'll
-be executed on the same worker one after the other.
+to [distribute the load](https://pytest-xdist.readthedocs.io/en/stable/distribution.html). In the
+documentation words: this guarantees that all tests in a file run in the same worker. This means
+that any test within the same file will never execute their tests in parallel. They'll be executed
+on the same worker one after the other.
 
 If the test supports parallel and repeated execution, you can run a bunch of them in parallel to
 look for flakiness. We use [pytest-repeat](https://pypi.org/project/pytest-repeat/) to set the

From 1225d50508ad0885dca3367b08c15f54c65b02f6 Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Fri, 26 Jul 2024 11:09:48 +0000
Subject: [PATCH 0845/1488] Do not count AttachedTable for tables in
 information schema databases

---
 src/Databases/DatabasesCommon.cpp | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/src/Databases/DatabasesCommon.cpp b/src/Databases/DatabasesCommon.cpp
index d2926c64f29..b8e9231f5c6 100644
--- a/src/Databases/DatabasesCommon.cpp
+++ b/src/Databases/DatabasesCommon.cpp
@@ -289,9 +289,7 @@ StoragePtr DatabaseWithOwnTablesBase::detachTableUnlocked(const String & table_n
     tables.erase(it);
     table_storage->is_detached = true;
 
-    if (!table_storage->isSystemStorage()
-        && database_name != DatabaseCatalog::SYSTEM_DATABASE
-        && database_name != DatabaseCatalog::TEMPORARY_DATABASE)
+    if (!table_storage->isSystemStorage() && !DatabaseCatalog::isPredefinedDatabase(database_name))
     {
         LOG_TEST(log, "Counting detached table {} to database {}", table_name, database_name);
         CurrentMetrics::sub(getAttachedCounterForStorage(table_storage));
@@ -339,9 +337,7 @@ void DatabaseWithOwnTablesBase::attachTableUnlocked(const String & table_name, c
     /// non-Atomic database the is_detached is set to true before RENAME.
     table->is_detached = false;
 
-    if (!table->isSystemStorage()
-        && database_name != DatabaseCatalog::SYSTEM_DATABASE
-        && database_name != DatabaseCatalog::TEMPORARY_DATABASE)
+    if (!table->isSystemStorage() && !DatabaseCatalog::isPredefinedDatabase(database_name))
     {
         LOG_TEST(log, "Counting attached table {} to database {}", table_name, database_name);
         CurrentMetrics::add(getAttachedCounterForStorage(table));

From 0cf0437196dfe4ee0f489ecc040b71e42e1f1a22 Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitlibar@clickhouse.com>
Date: Thu, 25 Jul 2024 16:36:32 +0200
Subject: [PATCH 0846/1488] Use separate client context in clickhouse-local

---
 programs/client/Client.cpp     |  29 ++++-----
 programs/client/Client.h       |   1 -
 programs/local/LocalServer.cpp |  28 +++++---
 programs/local/LocalServer.h   |   4 +-
 src/Client/ClientBase.cpp      | 113 ++++++++++++++++++---------------
 src/Client/ClientBase.h        |   6 ++
 6 files changed, 102 insertions(+), 79 deletions(-)

diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp
index 887c5cb86bc..f2919db0308 100644
--- a/programs/client/Client.cpp
+++ b/programs/client/Client.cpp
@@ -209,8 +209,8 @@ std::vector<String> Client::loadWarningMessages()
                           {} /* query_parameters */,
                           "" /* query_id */,
                           QueryProcessingStage::Complete,
-                          &global_context->getSettingsRef(),
-                          &global_context->getClientInfo(), false, {});
+                          &client_context->getSettingsRef(),
+                          &client_context->getClientInfo(), false, {});
     while (true)
     {
         Packet packet = connection->receivePacket();
@@ -306,9 +306,6 @@ void Client::initialize(Poco::Util::Application & self)
     if (env_password && !config().has("password"))
         config().setString("password", env_password);
 
-    // global_context->setApplicationType(Context::ApplicationType::CLIENT);
-    global_context->setQueryParameters(query_parameters);
-
     /// settings and limits could be specified in config file, but passed settings has higher priority
     for (const auto & setting : global_context->getSettingsRef().allUnchanged())
     {
@@ -382,7 +379,7 @@ try
         showWarnings();
 
     /// Set user password complexity rules
-    auto & access_control = global_context->getAccessControl();
+    auto & access_control = client_context->getAccessControl();
     access_control.setPasswordComplexityRules(connection->getPasswordComplexityRules());
 
     if (is_interactive && !delayed_interactive)
@@ -459,7 +456,7 @@ void Client::connect()
                           << connection_parameters.host << ":" << connection_parameters.port
                           << (!connection_parameters.user.empty() ? " as user " + connection_parameters.user : "") << "." << std::endl;
 
-            connection = Connection::createConnection(connection_parameters, global_context);
+            connection = Connection::createConnection(connection_parameters, client_context);
 
             if (max_client_network_bandwidth)
             {
@@ -528,7 +525,7 @@ void Client::connect()
         }
     }
 
-    if (!global_context->getSettingsRef().use_client_time_zone)
+    if (!client_context->getSettingsRef().use_client_time_zone)
     {
         const auto & time_zone = connection->getServerTimezone(connection_parameters.timeouts);
         if (!time_zone.empty())
@@ -611,7 +608,7 @@ void Client::printChangedSettings() const
         }
     };
 
-    print_changes(global_context->getSettingsRef().changes(), "settings");
+    print_changes(client_context->getSettingsRef().changes(), "settings");
     print_changes(cmd_merge_tree_settings.changes(), "MergeTree settings");
 }
 
@@ -709,7 +706,7 @@ bool Client::processWithFuzzing(const String & full_query)
     {
         const char * begin = full_query.data();
         orig_ast = parseQuery(begin, begin + full_query.size(),
-            global_context->getSettingsRef(),
+            client_context->getSettingsRef(),
             /*allow_multi_statements=*/ true);
     }
     catch (const Exception & e)
@@ -733,7 +730,7 @@ bool Client::processWithFuzzing(const String & full_query)
     }
 
     // Kusto is not a subject for fuzzing (yet)
-    if (global_context->getSettingsRef().dialect == DB::Dialect::kusto)
+    if (client_context->getSettingsRef().dialect == DB::Dialect::kusto)
     {
         return true;
     }
@@ -1072,6 +1069,11 @@ void Client::processOptions(const OptionsDescription & options_description,
     global_context->makeGlobalContext();
     global_context->setApplicationType(Context::ApplicationType::CLIENT);
 
+    /// In case of clickhouse-client the `client_context` can be just an alias for the `global_context`.
+    /// (There is no need to copy the context because clickhouse-client has no background tasks so it won't use that context in parallel.)
+    client_context = global_context;
+    initClientContext();
+
     global_context->setSettings(cmd_settings);
 
     /// Copy settings-related program options to config.
@@ -1205,11 +1207,6 @@ void Client::processConfig()
     pager = config().getString("pager", "");
 
     setDefaultFormatsAndCompressionFromConfiguration();
-
-    global_context->setClientName(std::string(DEFAULT_CLIENT_NAME));
-    global_context->setQueryKindInitial();
-    global_context->setQuotaClientKey(config().getString("quota_key", ""));
-    global_context->setQueryKind(query_kind);
 }
 
 
diff --git a/programs/client/Client.h b/programs/client/Client.h
index 6d57a6ea648..ff71b36dbf3 100644
--- a/programs/client/Client.h
+++ b/programs/client/Client.h
@@ -19,7 +19,6 @@ public:
     int main(const std::vector<String> & /*args*/) override;
 
 protected:
-
     Poco::Util::LayeredConfiguration & getClientConfiguration() override;
 
     bool processWithFuzzing(const String & full_query) override;
diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp
index 48e0cca7b73..e60c8ef6085 100644
--- a/programs/local/LocalServer.cpp
+++ b/programs/local/LocalServer.cpp
@@ -295,6 +295,8 @@ void LocalServer::cleanup()
         if (suggest)
             suggest.reset();
 
+        client_context.reset();
+
         if (global_context)
         {
             global_context->shutdown();
@@ -436,7 +438,7 @@ void LocalServer::connect()
         in = input.get();
     }
     connection = LocalConnection::createConnection(
-        connection_parameters, global_context, in, need_render_progress, need_render_profile_events, server_display_name);
+        connection_parameters, client_context, in, need_render_progress, need_render_profile_events, server_display_name);
 }
 
 
@@ -497,8 +499,6 @@ try
     initTTYBuffer(toProgressOption(getClientConfiguration().getString("progress", "default")));
     ASTAlterCommand::setFormatAlterCommandsWithParentheses(true);
 
-    applyCmdSettings(global_context);
-
     /// try to load user defined executable functions, throw on error and die
     try
     {
@@ -510,6 +510,11 @@ try
         throw;
     }
 
+    /// Must be called after we stopped initializing the global context and changing its settings.
+    /// After this point the global context must be stayed almost unchanged till shutdown,
+    /// and all necessary changes must be made to the client context instead.
+    createClientContext();
+
     if (is_interactive)
     {
         clearTerminal();
@@ -730,11 +735,12 @@ void LocalServer::processConfig()
     /// there is separate context for Buffer tables).
     adjustSettings();
     applySettingsOverridesForLocal(global_context);
-    applyCmdOptions(global_context);
 
     /// Load global settings from default_profile and system_profile.
     global_context->setDefaultProfiles(getClientConfiguration());
 
+    applyCmdOptions(global_context);
+
     /// We load temporary database first, because projections need it.
     DatabaseCatalog::instance().initializeAndLoadTemporaryDatabase();
 
@@ -778,10 +784,6 @@ void LocalServer::processConfig()
 
     server_display_name = getClientConfiguration().getString("display_name", "");
     prompt_by_server_display_name = getClientConfiguration().getRawString("prompt_by_server_display_name.default", ":) ");
-
-    global_context->setQueryKindInitial();
-    global_context->setQueryKind(query_kind);
-    global_context->setQueryParameters(query_parameters);
 }
 
 
@@ -860,6 +862,16 @@ void LocalServer::applyCmdOptions(ContextMutablePtr context)
 }
 
 
+void LocalServer::createClientContext()
+{
+    /// In case of clickhouse-local it's necessary to use a separate context for client-related purposes.
+    /// We can't just change the global context because it is used in background tasks (for example, in merges)
+    /// which don't expect that the global context can suddenly change.
+    client_context = Context::createCopy(global_context);
+    initClientContext();
+}
+
+
 void LocalServer::processOptions(const OptionsDescription &, const CommandLineOptions & options, const std::vector<Arguments> &, const std::vector<Arguments> &)
 {
     if (options.count("table"))
diff --git a/programs/local/LocalServer.h b/programs/local/LocalServer.h
index 0715f358313..ae9980311e1 100644
--- a/programs/local/LocalServer.h
+++ b/programs/local/LocalServer.h
@@ -31,7 +31,6 @@ public:
     int main(const std::vector<String> & /*args*/) override;
 
 protected:
-
     Poco::Util::LayeredConfiguration & getClientConfiguration() override;
 
     void connect() override;
@@ -50,7 +49,6 @@ protected:
     void processConfig() override;
     void readArguments(int argc, char ** argv, Arguments & common_arguments, std::vector<Arguments> &, std::vector<Arguments> &) override;
 
-
     void updateLoggerLevel(const String & logs_level) override;
 
 private:
@@ -67,6 +65,8 @@ private:
     void applyCmdOptions(ContextMutablePtr context);
     void applyCmdSettings(ContextMutablePtr context);
 
+    void createClientContext();
+
     ServerSettings server_settings;
 
     std::optional<StatusFile> status;
diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp
index 13dce05cabc..50cc6b98b81 100644
--- a/src/Client/ClientBase.cpp
+++ b/src/Client/ClientBase.cpp
@@ -467,7 +467,7 @@ void ClientBase::sendExternalTables(ASTPtr parsed_query)
 
     std::vector<ExternalTableDataPtr> data;
     for (auto & table : external_tables)
-        data.emplace_back(table.getData(global_context));
+        data.emplace_back(table.getData(client_context));
 
     connection->sendExternalTablesData(data);
 }
@@ -680,10 +680,10 @@ try
         /// intermixed with data with parallel formatting.
         /// It may increase code complexity significantly.
         if (!extras_into_stdout || select_only_into_file)
-            output_format = global_context->getOutputFormatParallelIfPossible(
+            output_format = client_context->getOutputFormatParallelIfPossible(
                 current_format, out_file_buf ? *out_file_buf : *out_buf, block);
         else
-            output_format = global_context->getOutputFormat(
+            output_format = client_context->getOutputFormat(
                 current_format, out_file_buf ? *out_file_buf : *out_buf, block);
 
         output_format->setAutoFlush();
@@ -762,6 +762,15 @@ void ClientBase::adjustSettings()
     global_context->setSettings(settings);
 }
 
+void ClientBase::initClientContext()
+{
+    client_context->setClientName(std::string(DEFAULT_CLIENT_NAME));
+    client_context->setQuotaClientKey(getClientConfiguration().getString("quota_key", ""));
+    client_context->setQueryKindInitial();
+    client_context->setQueryKind(query_kind);
+    client_context->setQueryParameters(query_parameters);
+}
+
 bool ClientBase::isRegularFile(int fd)
 {
     struct stat file_stat;
@@ -952,7 +961,7 @@ void ClientBase::processTextAsSingleQuery(const String & full_query)
     /// client-side. Thus we need to parse the query.
     const char * begin = full_query.data();
     auto parsed_query = parseQuery(begin, begin + full_query.size(),
-        global_context->getSettingsRef(),
+        client_context->getSettingsRef(),
         /*allow_multi_statements=*/ false);
 
     if (!parsed_query)
@@ -975,7 +984,7 @@ void ClientBase::processTextAsSingleQuery(const String & full_query)
     /// But for asynchronous inserts we don't extract data, because it's needed
     /// to be done on server side in that case (for coalescing the data from multiple inserts on server side).
     const auto * insert = parsed_query->as<ASTInsertQuery>();
-    if (insert && isSyncInsertWithData(*insert, global_context))
+    if (insert && isSyncInsertWithData(*insert, client_context))
         query_to_execute = full_query.substr(0, insert->data - full_query.data());
     else
         query_to_execute = full_query;
@@ -1093,7 +1102,7 @@ void ClientBase::processOrdinaryQuery(const String & query_to_execute, ASTPtr pa
         }
     }
 
-    const auto & settings = global_context->getSettingsRef();
+    const auto & settings = client_context->getSettingsRef();
     const Int32 signals_before_stop = settings.partial_result_on_first_cancel ? 2 : 1;
 
     int retries_left = 10;
@@ -1108,10 +1117,10 @@ void ClientBase::processOrdinaryQuery(const String & query_to_execute, ASTPtr pa
                 connection_parameters.timeouts,
                 query,
                 query_parameters,
-                global_context->getCurrentQueryId(),
+                client_context->getCurrentQueryId(),
                 query_processing_stage,
-                &global_context->getSettingsRef(),
-                &global_context->getClientInfo(),
+                &client_context->getSettingsRef(),
+                &client_context->getClientInfo(),
                 true,
                 [&](const Progress & progress) { onProgress(progress); });
 
@@ -1298,7 +1307,7 @@ void ClientBase::onProgress(const Progress & value)
 
 void ClientBase::onTimezoneUpdate(const String & tz)
 {
-    global_context->setSetting("session_timezone", tz);
+    client_context->setSetting("session_timezone", tz);
 }
 
 
@@ -1494,13 +1503,13 @@ bool ClientBase::receiveSampleBlock(Block & out, ColumnsDescription & columns_de
 
 void ClientBase::setInsertionTable(const ASTInsertQuery & insert_query)
 {
-    if (!global_context->hasInsertionTable() && insert_query.table)
+    if (!client_context->hasInsertionTable() && insert_query.table)
     {
         String table = insert_query.table->as<ASTIdentifier &>().shortName();
         if (!table.empty())
         {
             String database = insert_query.database ? insert_query.database->as<ASTIdentifier &>().shortName() : "";
-            global_context->setInsertionTable(StorageID(database, table));
+            client_context->setInsertionTable(StorageID(database, table));
         }
     }
 }
@@ -1551,7 +1560,7 @@ void ClientBase::processInsertQuery(const String & query_to_execute, ASTPtr pars
     const auto & parsed_insert_query = parsed_query->as<ASTInsertQuery &>();
     if ((!parsed_insert_query.data && !parsed_insert_query.infile) && (is_interactive || (!stdin_is_a_tty && !isStdinNotEmptyAndValid(std_in))))
     {
-        const auto & settings = global_context->getSettingsRef();
+        const auto & settings = client_context->getSettingsRef();
         if (settings.throw_if_no_data_to_insert)
             throw Exception(ErrorCodes::NO_DATA_TO_INSERT, "No data to insert");
         else
@@ -1565,10 +1574,10 @@ void ClientBase::processInsertQuery(const String & query_to_execute, ASTPtr pars
         connection_parameters.timeouts,
         query,
         query_parameters,
-        global_context->getCurrentQueryId(),
+        client_context->getCurrentQueryId(),
         query_processing_stage,
-        &global_context->getSettingsRef(),
-        &global_context->getClientInfo(),
+        &client_context->getSettingsRef(),
+        &client_context->getClientInfo(),
         true,
         [&](const Progress & progress) { onProgress(progress); });
 
@@ -1616,7 +1625,7 @@ void ClientBase::sendData(Block & sample, const ColumnsDescription & columns_des
 
         /// Set callback to be called on file progress.
         if (tty_buf)
-            progress_indication.setFileProgressCallback(global_context, *tty_buf);
+            progress_indication.setFileProgressCallback(client_context, *tty_buf);
     }
 
     /// If data fetched from file (maybe compressed file)
@@ -1650,10 +1659,10 @@ void ClientBase::sendData(Block & sample, const ColumnsDescription & columns_des
         }
 
         StorageFile::CommonArguments args{
-            WithContext(global_context),
+            WithContext(client_context),
             parsed_insert_query->table_id,
             current_format,
-            getFormatSettings(global_context),
+            getFormatSettings(client_context),
             compression_method,
             columns_for_storage_file,
             ConstraintsDescription{},
@@ -1661,7 +1670,7 @@ void ClientBase::sendData(Block & sample, const ColumnsDescription & columns_des
             {},
             String{},
         };
-        StoragePtr storage = std::make_shared<StorageFile>(in_file, global_context->getUserFilesPath(), args);
+        StoragePtr storage = std::make_shared<StorageFile>(in_file, client_context->getUserFilesPath(), args);
         storage->startup();
         SelectQueryInfo query_info;
 
@@ -1672,16 +1681,16 @@ void ClientBase::sendData(Block & sample, const ColumnsDescription & columns_des
             storage->read(
                     plan,
                     sample.getNames(),
-                    storage->getStorageSnapshot(metadata, global_context),
+                    storage->getStorageSnapshot(metadata, client_context),
                     query_info,
-                    global_context,
+                    client_context,
                     {},
-                    global_context->getSettingsRef().max_block_size,
+                    client_context->getSettingsRef().max_block_size,
                     getNumberOfPhysicalCPUCores());
 
             auto builder = plan.buildQueryPipeline(
-                QueryPlanOptimizationSettings::fromContext(global_context),
-                BuildQueryPipelineSettings::fromContext(global_context));
+                QueryPlanOptimizationSettings::fromContext(client_context),
+                BuildQueryPipelineSettings::fromContext(client_context));
 
             QueryPlanResourceHolder resources;
             auto pipe = QueryPipelineBuilder::getPipe(std::move(*builder), resources);
@@ -1742,14 +1751,14 @@ void ClientBase::sendDataFrom(ReadBuffer & buf, Block & sample, const ColumnsDes
             current_format = insert->format;
     }
 
-    auto source = global_context->getInputFormat(current_format, buf, sample, insert_format_max_block_size);
+    auto source = client_context->getInputFormat(current_format, buf, sample, insert_format_max_block_size);
     Pipe pipe(source);
 
     if (columns_description.hasDefaults())
     {
         pipe.addSimpleTransform([&](const Block & header)
         {
-            return std::make_shared<AddingDefaultsTransform>(header, columns_description, *source, global_context);
+            return std::make_shared<AddingDefaultsTransform>(header, columns_description, *source, client_context);
         });
     }
 
@@ -1911,12 +1920,12 @@ void ClientBase::processParsedSingleQuery(const String & full_query, const Strin
 
     if (is_interactive)
     {
-        global_context->setCurrentQueryId("");
+        client_context->setCurrentQueryId("");
         // Generate a new query_id
         for (const auto & query_id_format : query_id_formats)
         {
             writeString(query_id_format.first, std_out);
-            writeString(fmt::format(fmt::runtime(query_id_format.second), fmt::arg("query_id", global_context->getCurrentQueryId())), std_out);
+            writeString(fmt::format(fmt::runtime(query_id_format.second), fmt::arg("query_id", client_context->getCurrentQueryId())), std_out);
             writeChar('\n', std_out);
             std_out.next();
         }
@@ -1943,7 +1952,7 @@ void ClientBase::processParsedSingleQuery(const String & full_query, const Strin
                 auto password = auth_data->getPassword();
 
                 if (password)
-                    global_context->getAccessControl().checkPasswordComplexityRules(*password);
+                    client_context->getAccessControl().checkPasswordComplexityRules(*password);
             }
         }
     }
@@ -1958,15 +1967,15 @@ void ClientBase::processParsedSingleQuery(const String & full_query, const Strin
         std::optional<Settings> old_settings;
         SCOPE_EXIT_SAFE({
             if (old_settings)
-                global_context->setSettings(*old_settings);
+                client_context->setSettings(*old_settings);
         });
 
         auto apply_query_settings = [&](const IAST & settings_ast)
         {
             if (!old_settings)
-                old_settings.emplace(global_context->getSettingsRef());
-            global_context->applySettingsChanges(settings_ast.as<ASTSetQuery>()->changes);
-            global_context->resetSettingsToDefaultValue(settings_ast.as<ASTSetQuery>()->default_settings);
+                old_settings.emplace(client_context->getSettingsRef());
+            client_context->applySettingsChanges(settings_ast.as<ASTSetQuery>()->changes);
+            client_context->resetSettingsToDefaultValue(settings_ast.as<ASTSetQuery>()->default_settings);
         };
 
         const auto * insert = parsed_query->as<ASTInsertQuery>();
@@ -1999,7 +2008,7 @@ void ClientBase::processParsedSingleQuery(const String & full_query, const Strin
         if (insert && insert->select)
             insert->tryFindInputFunction(input_function);
 
-        bool is_async_insert_with_inlined_data = global_context->getSettingsRef().async_insert && insert && insert->hasInlinedData();
+        bool is_async_insert_with_inlined_data = client_context->getSettingsRef().async_insert && insert && insert->hasInlinedData();
 
         if (is_async_insert_with_inlined_data)
         {
@@ -2034,9 +2043,9 @@ void ClientBase::processParsedSingleQuery(const String & full_query, const Strin
                 if (change.name == "profile")
                     current_profile = change.value.safeGet<String>();
                 else
-                    global_context->applySettingChange(change);
+                    client_context->applySettingChange(change);
             }
-            global_context->resetSettingsToDefaultValue(set_query->default_settings);
+            client_context->resetSettingsToDefaultValue(set_query->default_settings);
 
             /// Query parameters inside SET queries should be also saved on the client side
             ///  to override their previous definitions set with --param_* arguments
@@ -2044,7 +2053,7 @@ void ClientBase::processParsedSingleQuery(const String & full_query, const Strin
             for (const auto & [name, value] : set_query->query_parameters)
                 query_parameters.insert_or_assign(name, value);
 
-            global_context->addQueryParameters(NameToNameMap{set_query->query_parameters.begin(), set_query->query_parameters.end()});
+            client_context->addQueryParameters(NameToNameMap{set_query->query_parameters.begin(), set_query->query_parameters.end()});
         }
         if (const auto * use_query = parsed_query->as<ASTUseQuery>())
         {
@@ -2121,8 +2130,8 @@ MultiQueryProcessingStage ClientBase::analyzeMultiQueryText(
     if (this_query_begin >= all_queries_end)
         return MultiQueryProcessingStage::QUERIES_END;
 
-    unsigned max_parser_depth = static_cast<unsigned>(global_context->getSettingsRef().max_parser_depth);
-    unsigned max_parser_backtracks = static_cast<unsigned>(global_context->getSettingsRef().max_parser_backtracks);
+    unsigned max_parser_depth = static_cast<unsigned>(client_context->getSettingsRef().max_parser_depth);
+    unsigned max_parser_backtracks = static_cast<unsigned>(client_context->getSettingsRef().max_parser_backtracks);
 
     // If there are only comments left until the end of file, we just
     // stop. The parser can't handle this situation because it always
@@ -2142,7 +2151,7 @@ MultiQueryProcessingStage ClientBase::analyzeMultiQueryText(
     try
     {
         parsed_query = parseQuery(this_query_end, all_queries_end,
-            global_context->getSettingsRef(),
+            client_context->getSettingsRef(),
             /*allow_multi_statements=*/ true);
     }
     catch (const Exception & e)
@@ -2185,7 +2194,7 @@ MultiQueryProcessingStage ClientBase::analyzeMultiQueryText(
     {
         this_query_end = find_first_symbols<'\n'>(insert_ast->data, all_queries_end);
         insert_ast->end = this_query_end;
-        query_to_execute_end = isSyncInsertWithData(*insert_ast, global_context) ? insert_ast->data : this_query_end;
+        query_to_execute_end = isSyncInsertWithData(*insert_ast, client_context) ? insert_ast->data : this_query_end;
     }
 
     query_to_execute = all_queries_text.substr(this_query_begin - all_queries_text.data(), query_to_execute_end - this_query_begin);
@@ -2387,13 +2396,13 @@ bool ClientBase::executeMultiQuery(const String & all_queries_text)
                 // , where the inline data is delimited by semicolon and not by a
                 // newline.
                 auto * insert_ast = parsed_query->as<ASTInsertQuery>();
-                if (insert_ast && isSyncInsertWithData(*insert_ast, global_context))
+                if (insert_ast && isSyncInsertWithData(*insert_ast, client_context))
                 {
                     this_query_end = insert_ast->end;
                     adjustQueryEnd(
                         this_query_end, all_queries_end,
-                        static_cast<unsigned>(global_context->getSettingsRef().max_parser_depth),
-                        static_cast<unsigned>(global_context->getSettingsRef().max_parser_backtracks));
+                        static_cast<unsigned>(client_context->getSettingsRef().max_parser_depth),
+                        static_cast<unsigned>(client_context->getSettingsRef().max_parser_backtracks));
                 }
 
                 // Report error.
@@ -2523,10 +2532,10 @@ void ClientBase::runInteractive()
     if (load_suggestions)
     {
         /// Load suggestion data from the server.
-        if (global_context->getApplicationType() == Context::ApplicationType::CLIENT)
-            suggest->load<Connection>(global_context, connection_parameters, getClientConfiguration().getInt("suggestion_limit"), wait_for_suggestions_to_load);
-        else if (global_context->getApplicationType() == Context::ApplicationType::LOCAL)
-            suggest->load<LocalConnection>(global_context, connection_parameters, getClientConfiguration().getInt("suggestion_limit"), wait_for_suggestions_to_load);
+        if (client_context->getApplicationType() == Context::ApplicationType::CLIENT)
+            suggest->load<Connection>(client_context, connection_parameters, getClientConfiguration().getInt("suggestion_limit"), wait_for_suggestions_to_load);
+        else if (client_context->getApplicationType() == Context::ApplicationType::LOCAL)
+            suggest->load<LocalConnection>(client_context, connection_parameters, getClientConfiguration().getInt("suggestion_limit"), wait_for_suggestions_to_load);
     }
 
     if (home_path.empty())
@@ -2664,7 +2673,7 @@ void ClientBase::runInteractive()
         {
             // If a separate connection loading suggestions failed to open a new session,
             // use the main session to receive them.
-            suggest->load(*connection, connection_parameters.timeouts, getClientConfiguration().getInt("suggestion_limit"), global_context->getClientInfo());
+            suggest->load(*connection, connection_parameters.timeouts, getClientConfiguration().getInt("suggestion_limit"), client_context->getClientInfo());
         }
 
         try
@@ -2713,10 +2722,10 @@ bool ClientBase::processMultiQueryFromFile(const String & file_name)
 
     if (!getClientConfiguration().has("log_comment"))
     {
-        Settings settings = global_context->getSettings();
+        Settings settings = client_context->getSettings();
         /// NOTE: cannot use even weakly_canonical() since it fails for /dev/stdin due to resolving of "pipe:[X]"
         settings.log_comment = fs::absolute(fs::path(file_name));
-        global_context->setSettings(settings);
+        client_context->setSettings(settings);
     }
 
     return executeMultiQuery(queries_from_file);
diff --git a/src/Client/ClientBase.h b/src/Client/ClientBase.h
index 4f500a4c45d..be74090b84d 100644
--- a/src/Client/ClientBase.h
+++ b/src/Client/ClientBase.h
@@ -206,6 +206,9 @@ protected:
     /// Adjust some settings after command line options and config had been processed.
     void adjustSettings();
 
+    /// Initializes the client context.
+    void initClientContext();
+
     void setDefaultFormatsAndCompressionFromConfiguration();
 
     void initTTYBuffer(ProgressOption progress);
@@ -215,6 +218,9 @@ protected:
     SharedContextHolder shared_context;
     ContextMutablePtr global_context;
 
+    /// Client context is a context used only by the client to parse queries, process query parameters and to connect to clickhouse-server.
+    ContextMutablePtr client_context;
+
     LoggerPtr fatal_log;
     Poco::AutoPtr<Poco::SplitterChannel> fatal_channel_ptr;
     Poco::AutoPtr<Poco::Channel> fatal_console_channel_ptr;

From 3cf2ec36ca31964a5a57717d5645f5e5a287dd00 Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Fri, 26 Jul 2024 11:36:54 +0000
Subject: [PATCH 0847/1488] Verbose output for 03203_client_benchmark_options

---
 .../03203_client_benchmark_options.sh         | 27 ++++++++++++++-----
 1 file changed, 20 insertions(+), 7 deletions(-)

diff --git a/tests/queries/0_stateless/03203_client_benchmark_options.sh b/tests/queries/0_stateless/03203_client_benchmark_options.sh
index a9b9d69822b..475309cebb9 100755
--- a/tests/queries/0_stateless/03203_client_benchmark_options.sh
+++ b/tests/queries/0_stateless/03203_client_benchmark_options.sh
@@ -4,10 +4,23 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CUR_DIR"/../shell_config.sh
 
-${CLICKHOUSE_CLIENT} -t -q "SELECT sleepEachRow(2) FORMAT Null" 2>&1 | grep -q "^2\." && echo "Ok" || echo "Fail"
-${CLICKHOUSE_CLIENT} --time -q "SELECT sleepEachRow(2) FORMAT Null" 2>&1 | grep -q "^2\." && echo "Ok" || echo "Fail"
-${CLICKHOUSE_CLIENT} --memory-usage -q "SELECT sum(number) FROM numbers(10_000) FORMAT Null" 2>&1 | grep -q "^[0-9]\+$" && echo "Ok" || echo "Fail"
-${CLICKHOUSE_CLIENT} --memory-usage=none -q "SELECT sum(number) FROM numbers(10_000) FORMAT Null" # expected no output
-${CLICKHOUSE_CLIENT} --memory-usage=default -q "SELECT sum(number) FROM numbers(10_000) FORMAT Null" 2>&1 | grep -q "^[0-9]\+$" && echo "Ok" || echo "Fail"
-${CLICKHOUSE_CLIENT} --memory-usage=readable -q "SELECT sum(number) FROM numbers(10_000) FORMAT Null" 2>&1 | grep -q "^[0-9].*B$" && echo "Ok" || echo "Fail"
-${CLICKHOUSE_CLIENT} --memory-usage=unknown -q "SELECT sum(number) FROM numbers(10_000) FORMAT Null" 2>&1 | grep -q "BAD_ARGUMENTS" && echo "Ok" || echo "Fail"
+output=$(${CLICKHOUSE_CLIENT} -t -q "SELECT sleepEachRow(2) FORMAT Null" 2>&1)
+echo "$output" | grep -q "^2\." && echo "Ok" || { echo "Fail"; echo "$output"; }
+
+output=$(${CLICKHOUSE_CLIENT} --time -q "SELECT sleepEachRow(2) FORMAT Null" 2>&1)
+echo "$output" | grep -q "^2\." && echo "Ok" || { echo "Fail"; echo "$output"; }
+
+output=$(${CLICKHOUSE_CLIENT} --memory-usage -q "SELECT sum(number) FROM numbers(10_000) FORMAT Null" 2>&1)
+echo "$output" | grep -q "^[0-9]\+$" && echo "Ok" || { echo "Fail"; echo "$output"; }
+
+output=$(${CLICKHOUSE_CLIENT} --memory-usage=none -q "SELECT sum(number) FROM numbers(10_000) FORMAT Null" 2>&1)
+echo -n "$output" # expected no output
+
+output=$(${CLICKHOUSE_CLIENT} --memory-usage=default -q "SELECT sum(number) FROM numbers(10_000) FORMAT Null" 2>&1)
+echo "$output" | grep -q "^[0-9]\+$" && echo "Ok" || { echo "Fail"; echo "$output"; }
+
+output=$(${CLICKHOUSE_CLIENT} --memory-usage=readable -q "SELECT sum(number) FROM numbers(10_000) FORMAT Null" 2>&1)
+echo "$output" | grep -q "^[0-9].*B$" && echo "Ok" || { echo "Fail"; echo "$output"; }
+
+output=$(${CLICKHOUSE_CLIENT} --memory-usage=unknown -q "SELECT sum(number) FROM numbers(10_000) FORMAT Null" 2>&1)
+echo "$output" | grep -q "BAD_ARGUMENTS" && echo "Ok" || { echo "Fail"; echo "$output"; }

From 28e991708be1facd87c3760f7929cd5ddc299805 Mon Sep 17 00:00:00 2001
From: maxvostrikov <max.vostrikov@clickhouse.com>
Date: Fri, 26 Jul 2024 13:45:57 +0200
Subject: [PATCH 0848/1488] squash! added somme tests in relation with
 https://github.com/ClickHouse/ClickHouse/pull/54881 with new behaviour when
 enable_named_columns_in_function_tuple=1 (default value)

---
 .../0_stateless/00309_formats.reference       | Bin 18666 -> 20353 bytes
 tests/queries/0_stateless/00309_formats.sql   |   5 -----
 2 files changed, 5 deletions(-)

diff --git a/tests/queries/0_stateless/00309_formats.reference b/tests/queries/0_stateless/00309_formats.reference
index f3ea45520bb50fb936caf6724e9fedf3cdd00b75..cab311692be229716b58af39079275d3942b01cc 100644
GIT binary patch
literal 20353
zcmeHP3wIP%5{6aK9zDL_&p18;axq=E-!Ke<i0?P}N)Q7X6ip_W3@R%6%l7O~+P|=0
zRo6hOl6Z`+M|N!w(_i=PTYbO&s&ea2b&tJW?SKFN*FT@{>He<Wy7Ldkn-p(W+@rWx
z@fO8f6_Y9M{>M!>@7a6Ht;zJ#Jk0d+*gt^JbdYKDu4%*X9d$a^ZOY<y#cBH;dVHti
zU5a-r-lKT0;(dzuD?Xt3pyES{4=X;R_^9GzijONkq4=cYjN(&@eZ{90pHX~P@j1m=
z#eItV6`xl;pcoVnD!!mNr<f{|VpPl&bHzfjRIC(h#YXXv;$g*s;t|E6;!(vH6^|(%
zSA0qFgyPGJCly~&d{yx^#n%<zP<&JIEycGL-%)&5@jb=&6+cisrFdHLL&c92KUVxi
z@l(Zl#RbJPik~T-Rs39WQSqGOdBr8gk>Ul#FBHF2yr}q<;@67HiYtm^#Z|>iir*-%
zDXuF{6fY}&tN5Mb_lj2(e^C5U@h8Qria#s<qKG-5+v_yg)y2WCP6NyYf5SM~eSM*H
z+wHwO?!4>nd+xpO{s$g>=;22meeCfko}78A|MWA@J~z8>|MLgJ!58LIidkNiRoxsq
zJUB8u`r@(UFP(V#<SVbf_WB!dzV-Gy@4ol`2d7Sd_|eCod^*2y=CiY(FP=NUG`jG`
zmlwbKdU<8Mdg+_B^~vRLzx)2m4?q5N_2*w8dH3%xRA9RH1%(59c8_;2-L$y2wtIcg
z<npEE(caUqjVEP(^NGdDsCR5_ZSl&!wq^g`4<>6D$6wrgio0oJnBji=r;A-&v0K--
z%ZqMZ-|En<>suYVb$zQtx30$|*-G(uTzhnP-g&=^d;Asr39~@AuK#;~vDmn-A16x*
zR`tnxBCGmdyM|AAwVs$&ebTP#ldV?ulqNAv(}O7;BueJfKp}=K<e>;<s6ri@Gl_ne
zTsXHdN%m11X0Kb!?(fB-)Jrr6DckZA0Y*fDVxTNg9;gUZ2C4$pftnE05HV~FF$*yd
zu?VpYu?n#cu?blkG71@3Wg*K$R)nk!SrxK6WKGD^kW<KG$U!;}c@gq5<W<P)kT;=7
zLqVa4p~ymkdn!UvhN22Z9f~HDX(%a_F_c*-^HAcJ%TQLKtV7v^Dh(BdDuyZxRUWD$
zR8XV}RUN7()M=<G)G^dqsPj-4p)NxW!Rk;qp-DqSp^2f%LX(H42u&H9Dm2iuff|q_
zg&rvcNufvzNm6K%LX;G$q>v>=KamanSU6UWrDN?_JXVk8gMI`72?`P<#P@i7&<`pS
zWF+WF5R#xIK}v#_1ThI}667T4Nf4BvC_z$!rUX$5suE-+=t>Zlpe#XJg0=*43F;E$
zCFn~Kn4mC0VuHp5kqIgjWG3iL5SpMgL28251hENf6XYi7O%R-*I6-oP<^<6RsuN@<
z=uQxxpgci(g7yUQ3F;H%C+JUb0Ko$U7Z7|va00;#1UC@;KyU=X69iWfd_iyq!5aj3
z5d48~#s?7ye?<5r!XFX-i10^*KO+1Q;g1M^M8t7~KO+1Q;g1M^MEE1Z9})hD9M2rk
z64B_#G0j2EQO#k^am|4Ze?<5r!XFX-i0}tWF!-P!{1M@g2!BNQBf=jM{)q5Lgg+wu
z5#f&re?<5r!XFX-i10^*KO+1Q;g1M^#G){NzzgHs7A`ETZCepq_<bdzt|o+Je}n%^
z_F<}?PTo9MWAfg?^nYv8P5#~F-%bAiqmw^gndqJb+~70xpFwZySwHlzui^Wlzva3g
z?bn_n^|2r|HoWYgHgGQ4<|1s?t^B){f4B1gOIH3H|Ae=l{<|l^H~Px`kK~0uz5?2r
zy7yx*>933?i~U2rWWJXqI8KuOcyVRaKQz-H_k$cyu3VDiwye}g+f_bcJ1@D;TzuN5
zD@s+avnf(-W3}#V+EUgwouv-u47MnGiFGeI6M7J0X>pSftOBKaoh3fnm@GvczL`Ta
z^L_N{6Fw0i&Ve~T1wJJ{6+SgSO&=q%!s~pgh07?nk!+)EGuh^{Eo58Dw&Hr*^k(>Z
zybQ>^Hqtrc{32`%3@;;YAm1kG-$5QXD0j)XNqnD@?1}G`_*RMUmH1|f@0J>DqOcTU
z*dnqtku4%yM7D@*5!oWLMaC8xTV$doTV!mJu|>uf8Czs*k+DV27CBqwY>|unY>~4?
z&K5aa<ZO|%MZp#YTNG?jutmWZ1zWIa{emqDwkX)5WQ&q5O13E3qGXGbElRd1*`j2N
zk}WE>sMw-ni;68Owy4;mVvC9`Dz>QDqGpSlEo!!?*`j8Pnk{O!sM(@si<&JOwrJR*
zVT*<>8n$TIqG5}MEgH6H*a8(t*|H6)j<RJdR32r^cBnqemMu|%l&#pR`^B~mfHnf!
z5NKnd4T3fb+AxqfKxLAz6U8>lxDpGfP)fUuE8&1TrL@Sn5)r6Ya*M<SDweV@oFyz!
zx0Dv)ERligrL+iV2@cdSxkbVQbxhfpaV0`f&6E}ySAqmJO>U7eL0wb!Wn76AR5ztX
z#+6_}jZ<1=T!|M{I=Mw62Gvg4mvJR%Q1g@)8CT*4l}~AraV2z6|Kt`49@Ic(U&fXA
zK_ygLWLya$)I+&Nf(SKH*_Ux8j!+qu78zGU3H4EFk#QxOP$lISi6>M_WnadXkV3sw
zT4Y>_DpXCSg~!FbC|e#E)l=EBEh?z8Wm{BHWy|BDk}6xVZKpC4TZt>wRQasL73!+8
zFSZg_sIE$j*tQdl#8s5ILY<Y*wv$Xd%e2!>JI}Nejl>nIuG~}N3YAyc7h8!d)L*4V
z#+A526;@hgT!|~xV!1`)3iVjom;05tLRD5;<bEZtP@Clzi7V7+WnadXxI&dyT8clA
zCKkq$GLCa;SN2|(?MdEJDqBf8Bq@g^<&dNtl9WS|a;TD?wQL*N9+T~H*}f#(6S9@-
z)5)2q7R>5#VP`KA{>G~DrX+|J4NE4^tt78W7My1Ewbk!9(?&_}d~o*M=)&sS=<M>v
zm5Y<Ji<7g<qs8?Jcj;a27DK#%m`b;CF*J`X&<}R)vx61S&R06PcsnIrdmh<8BP*wQ
z{O!QuQ!4FB<~LKp+$Iwo|J`-#7c5hFy6M(0eLPL&i0IZYvfRaUT2`Z5zoe4SyY-8F
zcfylpM8$7kzwC6<|9@Y%AW3art6;IxmL7JtP#GNker0*)+tJ$k#nthVS$ZLzof(bK
zuP)(*{UfudPn|qi&(0hj_6BRC^~=kXVJ{hQkq4Lz&aW<CUKy|R{`5fhlL1$KhU3AU
zAfHYKT=*HbRiD9}9EjOmdy(y~v9$RbTnyU56;M;zd<|10+QG%vH(#nP9qr&6Qcl`@
zoh>R#+gx;@nz8VMIrYYX7g?<00RC^)91khQIPJmG@OrgrFt^k;LYv_Lo5d@`!DxB3
zGDLsD_K3_;G)I{^%FR(>j!JV>nWNeqHPfRx<;2)BT_0Y1Xf&A9h!Y`)0|nvDH@x8n
zHr{gEJg2=CS+6-2Y@=}a^V*~wFA#5d<+f6RtRKlbl5c8*WMo}Q)|X_RN!FWW-AUK{
zDc6tAH<nejtfFNVEvsl*MawE$R?)JGmQ`k1WtLTDS!I@0W?5yHRc2XbmQ`k1WtLTL
zS>={hZdv7)Rc=}3mQ`+9<(5@$S>={hVObTHRbg2bmQ`U{6_yqMxorr-!m=tXtHQD>
zEvwS9DlMzhvMMdB(y}TotJ1P6EvwS9sw}I@vZ^er%Cf2~tID#fEUU`0sw}I@vZ^hs
z+On!GtJ<=vEvwqHsx7P9vZ^hs+OldatH!cwEUU({YAmb9vT7`=#<FTGtH!cIb=w3b
z-mP`VmKff-b>}TLyo>A3TXJ|u*PXZY@b0cX?~)z)dEQU-ex~<Ry`Ss-WbbEtKi#|e
z9#wkFH#+^?l#h3eU4N&aoAvP?vg_~ka}z(_Rkr=z)Q@KZ?!5D_oBi>gv+M8t>k<Io
zg?9a&e_am1`_i_*%K~@;;m$k#Tq3}`)~>(P&*cKVkL~(9{aiZ0vklwdB?P?7?an*>
zTu#9I-mbsX&!q*_8f<@;81VeWop<`V+<^DVU4N&aOAmPG-1T?*xeS5#(rtg2Bk;7w
zop<`VG=X>8U4N&a%M^I;-Sv0+xnzOoK(@b27kFphop<`VjDh#+U4N&aOB#6h-u8EC
z1J9b=d8ePt9C&Zv^>_NY<biklU4N&a%OCjiVfy2@0PfiG!*2xKdCL#K9dPHZe)vs+
zJ8${nw+8mS%OCg+f;;c{x%`1AYOcTI=kf=B%i#JuelCCD_YStd%O7|`=gvF*{Fero
zKk&N=XYcsA{DI$A*#0hm;Mt!$@A$d=f!}4g{!TxaKk(ZP*Wc;q@&|t3Vf(xMfhUdb
zywlI+5BxU7^>_NY{DCT;>+ke)`2)`|ZGV?P@Y@r2-syKefBb6wVB0s=_<vopxN_<6
zmcLbbZh3O}zw>`E`L+6m)Ezc_5xV{`a(e6A7cfwpXiGp`$XFl3xq_$>F>d{|a@uQ~
Lvprlx%YOd>&)TR8

delta 3052
zcmeIy$4}c(00wZIgtksdZ-w^I3JD}eL2?j6{R4XG1$t-?y;Q0c4wb5?2YL>Bj|8*#
z-Xp>6y?59U_TGDMNaN%a?SDX3#j^bK`<|bk?ZfAlN^UDiN<bJ6jY24SuCXUb!%+R!
z(?lWq$P}8rBt@#-Sf0!i4|+jw=mUMBAM}R-Fc1d8U>E{JVHgaD5e8+19DxExLIy^`
zXcz-yVH}Ky2`~{R!DN^MQ}xOSA`L-0%z&9N3ueO{m<#h@J}iKRum~2z5?IQq+n-P(
zP{A_D!g5#vD`6F^hBdGj*1>w%02^VGPHumqnY-wY76U{pY=iBv19rkL*bRGNFYJT;
zPz?v*AoeGQ5Ddc+I10z$IGlhQI0>iVG@OC6a1PGH1;h6f#3F(vxC~d|DqMr>a070_
zEw~ML;4a*Q`|v>j|8;{y1V``~p1@Oh2G8LIyo6Wq8s5NLcn9yHg%&qjk~HD*O`kk{
zMhS#wBF4@26;n)ok$HM~`}q3#2LuKMhlGZON5~bCOjLACY+QUoVp4KSYFc_mW>$7i
zZeD&tVNr2OsZv$OmRD3(RoB$k)i*RYHMg|3wRd!Ob@%l4^{WR4hlWQ+$Hpf#lT*_(
zvvczci%ZKZt842Un_JsEyL<ZwheyXJr)TFEmsi&}w|Dmf?L&1j3gH`4#3+R{X>2uP
zs~KC(*lPbltFhlLRQ$(oSyIc$t;|{F>ph;XVgHfpC$+5H%8ajnp%+Y<&DY2OORU_k
z(V7&oZmv$Gj7LgIiK)l0pFiA9KYx&VyyL(B=HYJsi3@C{mioxSQew%acGgGsx>HI?
tDE$HfH|yd|adXt)D_oo(mI%Z&<wVo8BQ16miz!<%?O^j~Y-3_4_zmv{|2Y5v

diff --git a/tests/queries/0_stateless/00309_formats.sql b/tests/queries/0_stateless/00309_formats.sql
index 0366cdeea5c..b0939c00a10 100644
--- a/tests/queries/0_stateless/00309_formats.sql
+++ b/tests/queries/0_stateless/00309_formats.sql
@@ -9,8 +9,3 @@ SELECT number * 246 + 10 AS n, toDate('2000-01-01') + n AS d, range(n) AS arr, a
 SELECT number * 246 + 10 AS n, toDate('2000-01-01') + n AS d, range(n) AS arr, arrayStringConcat(arrayMap(x -> reinterpretAsString(x), arr)) AS s, (n, d) AS tuple FROM system.numbers LIMIT 2 FORMAT JSON;
 SELECT number * 246 + 10 AS n, toDate('2000-01-01') + n AS d, range(n) AS arr, arrayStringConcat(arrayMap(x -> reinterpretAsString(x), arr)) AS s, (n, d) AS tuple FROM system.numbers LIMIT 2 FORMAT JSONCompact;
 SELECT number * 246 + 10 AS n, toDate('2000-01-01') + n AS d, range(n) AS arr, arrayStringConcat(arrayMap(x -> reinterpretAsString(x), arr)) AS s, (n, d) AS tuple FROM system.numbers LIMIT 2 FORMAT XML;
-
-SET enable_named_columns_in_function_tuple = 1;
-
-SELECT 36 AS n, toDate('2000-01-01') + n AS d, (n, d) AS tuple FROM system.numbers LIMIT 1 FORMAT RowBinaryWithNamesAndTypes;
-SELECT number * 246 + 10 AS n, toDate('2000-01-01') + n AS d, (n, d) AS tuple FROM system.numbers LIMIT 1 FORMAT TabSeparatedWithNamesAndTypes;

From d041df80aa112920f28d74ed26a0c8381808dafc Mon Sep 17 00:00:00 2001
From: avogar <avogar@clickhouse.com>
Date: Fri, 26 Jul 2024 12:14:26 +0000
Subject: [PATCH 0849/1488] Add test

---
 ..._to_read_for_schema_inference_in_cache.reference |  2 ++
 ...x_bytes_to_read_for_schema_inference_in_cache.sh | 13 +++++++++++++
 2 files changed, 15 insertions(+)
 create mode 100644 tests/queries/0_stateless/03212_max_bytes_to_read_for_schema_inference_in_cache.reference
 create mode 100755 tests/queries/0_stateless/03212_max_bytes_to_read_for_schema_inference_in_cache.sh

diff --git a/tests/queries/0_stateless/03212_max_bytes_to_read_for_schema_inference_in_cache.reference b/tests/queries/0_stateless/03212_max_bytes_to_read_for_schema_inference_in_cache.reference
new file mode 100644
index 00000000000..cd109daac52
--- /dev/null
+++ b/tests/queries/0_stateless/03212_max_bytes_to_read_for_schema_inference_in_cache.reference
@@ -0,0 +1,2 @@
+x	Nullable(Int64)					
+schema_inference_hints=, max_rows_to_read_for_schema_inference=25000, max_bytes_to_read_for_schema_inference=1000, schema_inference_make_columns_nullable=true, try_infer_integers=true, try_infer_dates=true, try_infer_datetimes=true, try_infer_numbers_from_strings=false, read_bools_as_numbers=true, read_bools_as_strings=true, read_objects_as_strings=true, read_numbers_as_strings=true, read_arrays_as_strings=true, try_infer_objects_as_tuples=true, infer_incomplete_types_as_strings=true, try_infer_objects=false, use_string_type_for_ambiguous_paths_in_named_tuples_inference_from_objects=false
diff --git a/tests/queries/0_stateless/03212_max_bytes_to_read_for_schema_inference_in_cache.sh b/tests/queries/0_stateless/03212_max_bytes_to_read_for_schema_inference_in_cache.sh
new file mode 100755
index 00000000000..8a77538f592
--- /dev/null
+++ b/tests/queries/0_stateless/03212_max_bytes_to_read_for_schema_inference_in_cache.sh
@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CUR_DIR"/../shell_config.sh
+
+echo '{"x" : 42}' > $CLICKHOUSE_TEST_UNIQUE_NAME.json
+$CLICKHOUSE_LOCAL -nm -q "
+DESC file('$CLICKHOUSE_TEST_UNIQUE_NAME.json') SETTINGS input_format_max_bytes_to_read_for_schema_inference=1000; 
+SELECT additional_format_info from system.schema_inference_cache"
+
+rm $CLICKHOUSE_TEST_UNIQUE_NAME.json
+

From 0299475202b59a4d1a54f13f02b7cc9ff44f38cc Mon Sep 17 00:00:00 2001
From: Nikita Taranov <nikita.taranov@clickhouse.com>
Date: Fri, 26 Jul 2024 14:02:37 +0100
Subject: [PATCH 0850/1488] impl

---
 ...eplicas_join_algo_and_analyzer_1.reference |  30 +++++
 ...allel_replicas_join_algo_and_analyzer_1.sh |  51 ++++++++
 ...eplicas_join_algo_and_analyzer_2.reference |  57 +++++++++
 ...allel_replicas_join_algo_and_analyzer_2.sh | 103 +++++++++++++++
 ...plicas_join_algo_and_analyzer_3.reference} |  87 -------------
 ...llel_replicas_join_algo_and_analyzer_3.sh} | 119 ------------------
 6 files changed, 241 insertions(+), 206 deletions(-)
 create mode 100644 tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_1.reference
 create mode 100755 tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_1.sh
 create mode 100644 tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_2.reference
 create mode 100755 tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_2.sh
 rename tests/queries/0_stateless/{02967_parallel_replicas_join_algo_and_analyzer.reference => 02967_parallel_replicas_join_algo_and_analyzer_3.reference} (55%)
 rename tests/queries/0_stateless/{02967_parallel_replicas_join_algo_and_analyzer.sh => 02967_parallel_replicas_join_algo_and_analyzer_3.sh} (58%)

diff --git a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_1.reference b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_1.reference
new file mode 100644
index 00000000000..e1bf9c27a81
--- /dev/null
+++ b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_1.reference
@@ -0,0 +1,30 @@
+
+simple join with analyzer
+4200000	4200000	4200000	-1400000
+4200006	4200006	4200006	-1400002
+4200012	4200012	4200012	-1400004
+4200018	4200018	4200018	-1400006
+4200024	4200024	4200024	-1400008
+4200030	4200030	4200030	-1400010
+4200036	4200036	4200036	-1400012
+4200042	4200042	4200042	-1400014
+4200048	4200048	4200048	-1400016
+4200054	4200054	4200054	-1400018
+
+simple (global) join with analyzer and parallel replicas
+4200000	4200000	4200000	-1400000
+4200006	4200006	4200006	-1400002
+4200012	4200012	4200012	-1400004
+4200018	4200018	4200018	-1400006
+4200024	4200024	4200024	-1400008
+4200030	4200030	4200030	-1400010
+4200036	4200036	4200036	-1400012
+4200042	4200042	4200042	-1400014
+4200048	4200048	4200048	-1400016
+4200054	4200054	4200054	-1400018
+SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` (stage: WithMergeableState)
+SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` (stage: WithMergeableState)
+<Debug> DefaultCoordinator: Coordination done
+SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(700000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState)
+SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(700000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState)
+<Debug> DefaultCoordinator: Coordination done
diff --git a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_1.sh b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_1.sh
new file mode 100755
index 00000000000..1089eb4051f
--- /dev/null
+++ b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_1.sh
@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+# Tags: long, no-random-settings, no-random-merge-tree-settings
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+
+$CLICKHOUSE_CLIENT -nm -q "
+drop table if exists num_1;
+drop table if exists num_2;
+
+create table num_1 (key UInt64, value String) engine = MergeTree order by key;
+create table num_2 (key UInt64, value Int64) engine = MergeTree order by key;
+
+insert into num_1 select number * 2, toString(number * 2) from numbers(1e7);
+insert into num_2 select number * 3, -number from numbers(1.5e6);
+"
+
+##############
+echo
+echo "simple join with analyzer"
+
+$CLICKHOUSE_CLIENT -q "
+select * from (select key, value from num_1) l
+inner join (select key, value from num_2) r on l.key = r.key
+order by l.key limit 10 offset 700000
+SETTINGS allow_experimental_analyzer=1"
+
+##############
+echo
+echo "simple (global) join with analyzer and parallel replicas"
+
+$CLICKHOUSE_CLIENT -q "
+select * from (select key, value from num_1) l
+inner join (select key, value from num_2) r on l.key = r.key
+order by l.key limit 10 offset 700000
+SETTINGS allow_experimental_analyzer=1, allow_experimental_parallel_reading_from_replicas = 2,
+max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1,
+cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join=0"
+
+$CLICKHOUSE_CLIENT -q "
+select * from (select key, value from num_1) l
+inner join (select key, value from num_2) r on l.key = r.key
+order by l.key limit 10 offset 700000
+SETTINGS allow_experimental_analyzer=1, allow_experimental_parallel_reading_from_replicas = 2, send_logs_level='trace',
+max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1,
+cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join=0" 2>&1 |
+grep "executeQuery\|<Debug>.*Coordinator: Coordination done" |
+grep -o "SELECT.*WithMergeableState)\|<Debug>.*Coordinator: Coordination done" |
+sed -re 's/_data_[[:digit:]]+_[[:digit:]]+/_data_/g'
diff --git a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_2.reference b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_2.reference
new file mode 100644
index 00000000000..297ec311f3e
--- /dev/null
+++ b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_2.reference
@@ -0,0 +1,57 @@
+
+simple (local) join with analyzer and parallel replicas
+4200000	4200000	4200000	-1400000
+4200006	4200006	4200006	-1400002
+4200012	4200012	4200012	-1400004
+4200018	4200018	4200018	-1400006
+4200024	4200024	4200024	-1400008
+4200030	4200030	4200030	-1400010
+4200036	4200036	4200036	-1400012
+4200042	4200042	4200042	-1400014
+4200048	4200048	4200048	-1400016
+4200054	4200054	4200054	-1400018
+SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` ALL INNER JOIN (SELECT `__table4`.`key` AS `key`, `__table4`.`value` AS `value` FROM `default`.`num_2` AS `__table4`) AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(700000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState)
+SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` ALL INNER JOIN (SELECT `__table4`.`key` AS `key`, `__table4`.`value` AS `value` FROM `default`.`num_2` AS `__table4`) AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(700000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState)
+<Debug> DefaultCoordinator: Coordination done
+
+simple (local) join with analyzer and parallel replicas and full sorting merge join
+4200000	4200000	4200000	-1400000
+4200006	4200006	4200006	-1400002
+4200012	4200012	4200012	-1400004
+4200018	4200018	4200018	-1400006
+4200024	4200024	4200024	-1400008
+4200030	4200030	4200030	-1400010
+4200036	4200036	4200036	-1400012
+4200042	4200042	4200042	-1400014
+4200048	4200048	4200048	-1400016
+4200054	4200054	4200054	-1400018
+SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` ALL INNER JOIN (SELECT `__table4`.`key` AS `key`, `__table4`.`value` AS `value` FROM `default`.`num_2` AS `__table4`) AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(700000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState)
+SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` ALL INNER JOIN (SELECT `__table4`.`key` AS `key`, `__table4`.`value` AS `value` FROM `default`.`num_2` AS `__table4`) AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(700000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState)
+<Debug> WithOrderCoordinator: Coordination done
+
+nested join with analyzer
+420000	420000	420000	-140000
+420042	420042	420042	-140014
+420084	420084	420084	-140028
+420126	420126	420126	-140042
+420168	420168	420168	-140056
+420210	420210	420210	-140070
+420252	420252	420252	-140084
+420294	420294	420294	-140098
+420336	420336	420336	-140112
+420378	420378	420378	-140126
+
+nested join with analyzer and parallel replicas, both local
+420000	420000	420000	-140000
+420042	420042	420042	-140014
+420084	420084	420084	-140028
+420126	420126	420126	-140042
+420168	420168	420168	-140056
+420210	420210	420210	-140070
+420252	420252	420252	-140084
+420294	420294	420294	-140098
+420336	420336	420336	-140112
+420378	420378	420378	-140126
+SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` ALL INNER JOIN (SELECT `__table4`.`key` AS `key`, `__table4`.`value` AS `value` FROM `default`.`num_2` AS `__table4` ALL INNER JOIN (SELECT `__table6`.`number` * 7 AS `key` FROM numbers(100000.) AS `__table6`) AS `__table5` ON `__table4`.`key` = `__table5`.`key` SETTINGS parallel_replicas_prefer_local_join = 1) AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState)
+SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` ALL INNER JOIN (SELECT `__table4`.`key` AS `key`, `__table4`.`value` AS `value` FROM `default`.`num_2` AS `__table4` ALL INNER JOIN (SELECT `__table6`.`number` * 7 AS `key` FROM numbers(100000.) AS `__table6`) AS `__table5` ON `__table4`.`key` = `__table5`.`key` SETTINGS parallel_replicas_prefer_local_join = 1) AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState)
+<Debug> WithOrderCoordinator: Coordination done
diff --git a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_2.sh b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_2.sh
new file mode 100755
index 00000000000..7a0e2d9bfdb
--- /dev/null
+++ b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_2.sh
@@ -0,0 +1,103 @@
+#!/usr/bin/env bash
+# Tags: long, no-random-settings, no-random-merge-tree-settings
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+
+$CLICKHOUSE_CLIENT -nm -q "
+drop table if exists num_1;
+drop table if exists num_2;
+
+create table num_1 (key UInt64, value String) engine = MergeTree order by key;
+create table num_2 (key UInt64, value Int64) engine = MergeTree order by key;
+
+insert into num_1 select number * 2, toString(number * 2) from numbers(1e7);
+insert into num_2 select number * 3, -number from numbers(1.5e6);
+"
+
+##############
+echo
+echo "simple (local) join with analyzer and parallel replicas"
+
+$CLICKHOUSE_CLIENT -q "
+select * from (select key, value from num_1) l
+inner join (select key, value from num_2) r on l.key = r.key
+order by l.key limit 10 offset 700000
+SETTINGS allow_experimental_analyzer=1,
+allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1,
+cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join=1"
+
+$CLICKHOUSE_CLIENT -q "
+select * from (select key, value from num_1) l
+inner join (select key, value from num_2) r on l.key = r.key
+order by l.key limit 10 offset 700000
+SETTINGS allow_experimental_analyzer=1, send_logs_level='trace',
+allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1,
+cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join=1" 2>&1 |
+grep "executeQuery\|<Debug>.*Coordinator: Coordination done" |
+grep -o "SELECT.*WithMergeableState)\|<Debug>.*Coordinator: Coordination done" |
+sed -re 's/_data_[[:digit:]]+_[[:digit:]]+/_data_/g'
+
+
+##############
+echo
+echo "simple (local) join with analyzer and parallel replicas and full sorting merge join"
+
+$CLICKHOUSE_CLIENT -q "
+select * from (select key, value from num_1) l
+inner join (select key, value from num_2) r on l.key = r.key
+order by l.key limit 10 offset 700000
+SETTINGS allow_experimental_analyzer=1, join_algorithm='full_sorting_merge',
+allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1,
+cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join=1"
+
+$CLICKHOUSE_CLIENT -q "
+select * from (select key, value from num_1) l
+inner join (select key, value from num_2) r on l.key = r.key
+order by l.key limit 10 offset 700000
+SETTINGS allow_experimental_analyzer=1, join_algorithm='full_sorting_merge', send_logs_level='trace',
+allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1,
+cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join=1" 2>&1 |
+grep "executeQuery\|<Debug>.*Coordinator: Coordination done" |
+grep -o "SELECT.*WithMergeableState)\|<Debug>.*Coordinator: Coordination done" |
+sed -re 's/_data_[[:digit:]]+_[[:digit:]]+/_data_/g'
+
+
+##############
+echo
+echo "nested join with analyzer"
+
+$CLICKHOUSE_CLIENT -q "
+select * from (select key, value from num_1) l
+inner join (select key, value from num_2 inner join
+  (select number * 7 as key from numbers(1e5)) as nn on num_2.key = nn.key settings parallel_replicas_prefer_local_join=1) r
+on l.key = r.key order by l.key limit 10 offset 10000
+SETTINGS allow_experimental_analyzer=1"
+
+
+##############
+echo
+echo "nested join with analyzer and parallel replicas, both local"
+
+$CLICKHOUSE_CLIENT -q "
+select * from (select key, value from num_1) l
+inner join (select key, value from num_2 inner join
+  (select number * 7 as key from numbers(1e5)) as nn on num_2.key = nn.key settings parallel_replicas_prefer_local_join=1) r
+on l.key = r.key order by l.key limit 10 offset 10000
+SETTINGS allow_experimental_analyzer=1,
+allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1,
+cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join=1"
+
+$CLICKHOUSE_CLIENT -q "
+select * from (select key, value from num_1) l
+inner join (select key, value from num_2 inner join
+  (select number * 7 as key from numbers(1e5)) as nn on num_2.key = nn.key settings parallel_replicas_prefer_local_join=1) r
+on l.key = r.key order by l.key limit 10 offset 10000
+SETTINGS allow_experimental_analyzer=1, join_algorithm='full_sorting_merge', send_logs_level='trace',
+allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1,
+cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join=1" 2>&1 |
+grep "executeQuery\|<Debug>.*Coordinator: Coordination done" |
+grep -o "SELECT.*WithMergeableState)\|<Debug>.*Coordinator: Coordination done" |
+sed -re 's/_data_[[:digit:]]+_[[:digit:]]+/_data_/g'
diff --git a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer.reference b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_3.reference
similarity index 55%
rename from tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer.reference
rename to tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_3.reference
index d7fa419aeab..c0485b817c4 100644
--- a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer.reference
+++ b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_3.reference
@@ -1,91 +1,4 @@
 
-simple join with analyzer
-4200000	4200000	4200000	-1400000
-4200006	4200006	4200006	-1400002
-4200012	4200012	4200012	-1400004
-4200018	4200018	4200018	-1400006
-4200024	4200024	4200024	-1400008
-4200030	4200030	4200030	-1400010
-4200036	4200036	4200036	-1400012
-4200042	4200042	4200042	-1400014
-4200048	4200048	4200048	-1400016
-4200054	4200054	4200054	-1400018
-
-simple (global) join with analyzer and parallel replicas
-4200000	4200000	4200000	-1400000
-4200006	4200006	4200006	-1400002
-4200012	4200012	4200012	-1400004
-4200018	4200018	4200018	-1400006
-4200024	4200024	4200024	-1400008
-4200030	4200030	4200030	-1400010
-4200036	4200036	4200036	-1400012
-4200042	4200042	4200042	-1400014
-4200048	4200048	4200048	-1400016
-4200054	4200054	4200054	-1400018
-SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` (stage: WithMergeableState)
-SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value` FROM `default`.`num_2` AS `__table1` (stage: WithMergeableState)
-<Debug> DefaultCoordinator: Coordination done
-SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(700000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState)
-SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` GLOBAL ALL INNER JOIN `_data_` AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(700000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState)
-<Debug> DefaultCoordinator: Coordination done
-
-simple (local) join with analyzer and parallel replicas
-4200000	4200000	4200000	-1400000
-4200006	4200006	4200006	-1400002
-4200012	4200012	4200012	-1400004
-4200018	4200018	4200018	-1400006
-4200024	4200024	4200024	-1400008
-4200030	4200030	4200030	-1400010
-4200036	4200036	4200036	-1400012
-4200042	4200042	4200042	-1400014
-4200048	4200048	4200048	-1400016
-4200054	4200054	4200054	-1400018
-SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` ALL INNER JOIN (SELECT `__table4`.`key` AS `key`, `__table4`.`value` AS `value` FROM `default`.`num_2` AS `__table4`) AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(700000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState)
-SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` ALL INNER JOIN (SELECT `__table4`.`key` AS `key`, `__table4`.`value` AS `value` FROM `default`.`num_2` AS `__table4`) AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(700000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState)
-<Debug> DefaultCoordinator: Coordination done
-
-simple (local) join with analyzer and parallel replicas and full sorting merge join
-4200000	4200000	4200000	-1400000
-4200006	4200006	4200006	-1400002
-4200012	4200012	4200012	-1400004
-4200018	4200018	4200018	-1400006
-4200024	4200024	4200024	-1400008
-4200030	4200030	4200030	-1400010
-4200036	4200036	4200036	-1400012
-4200042	4200042	4200042	-1400014
-4200048	4200048	4200048	-1400016
-4200054	4200054	4200054	-1400018
-SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` ALL INNER JOIN (SELECT `__table4`.`key` AS `key`, `__table4`.`value` AS `value` FROM `default`.`num_2` AS `__table4`) AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(700000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState)
-SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` ALL INNER JOIN (SELECT `__table4`.`key` AS `key`, `__table4`.`value` AS `value` FROM `default`.`num_2` AS `__table4`) AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(700000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState)
-<Debug> WithOrderCoordinator: Coordination done
-
-nested join with analyzer
-420000	420000	420000	-140000
-420042	420042	420042	-140014
-420084	420084	420084	-140028
-420126	420126	420126	-140042
-420168	420168	420168	-140056
-420210	420210	420210	-140070
-420252	420252	420252	-140084
-420294	420294	420294	-140098
-420336	420336	420336	-140112
-420378	420378	420378	-140126
-
-nested join with analyzer and parallel replicas, both local
-420000	420000	420000	-140000
-420042	420042	420042	-140014
-420084	420084	420084	-140028
-420126	420126	420126	-140042
-420168	420168	420168	-140056
-420210	420210	420210	-140070
-420252	420252	420252	-140084
-420294	420294	420294	-140098
-420336	420336	420336	-140112
-420378	420378	420378	-140126
-SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` ALL INNER JOIN (SELECT `__table4`.`key` AS `key`, `__table4`.`value` AS `value` FROM `default`.`num_2` AS `__table4` ALL INNER JOIN (SELECT `__table6`.`number` * 7 AS `key` FROM numbers(100000.) AS `__table6`) AS `__table5` ON `__table4`.`key` = `__table5`.`key` SETTINGS parallel_replicas_prefer_local_join = 1) AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState)
-SELECT `__table1`.`key` AS `key`, `__table1`.`value` AS `value`, `__table3`.`key` AS `r.key`, `__table3`.`value` AS `r.value` FROM (SELECT `__table2`.`key` AS `key`, `__table2`.`value` AS `value` FROM `default`.`num_1` AS `__table2`) AS `__table1` ALL INNER JOIN (SELECT `__table4`.`key` AS `key`, `__table4`.`value` AS `value` FROM `default`.`num_2` AS `__table4` ALL INNER JOIN (SELECT `__table6`.`number` * 7 AS `key` FROM numbers(100000.) AS `__table6`) AS `__table5` ON `__table4`.`key` = `__table5`.`key` SETTINGS parallel_replicas_prefer_local_join = 1) AS `__table3` ON `__table1`.`key` = `__table3`.`key` ORDER BY `__table1`.`key` ASC LIMIT _CAST(10000, 'UInt64'), _CAST(10, 'UInt64') (stage: WithMergeableState)
-<Debug> WithOrderCoordinator: Coordination done
-
 nested join with analyzer and parallel replicas, both global
 420000	420000	420000	-140000
 420042	420042	420042	-140014
diff --git a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer.sh b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_3.sh
similarity index 58%
rename from tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer.sh
rename to tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_3.sh
index 2840482da6d..e49a340ab67 100755
--- a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer.sh
+++ b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_3.sh
@@ -17,125 +17,6 @@ insert into num_1 select number * 2, toString(number * 2) from numbers(1e7);
 insert into num_2 select number * 3, -number from numbers(1.5e6);
 "
 
-##############
-echo
-echo "simple join with analyzer"
-
-$CLICKHOUSE_CLIENT -q "
-select * from (select key, value from num_1) l
-inner join (select key, value from num_2) r on l.key = r.key
-order by l.key limit 10 offset 700000
-SETTINGS allow_experimental_analyzer=1"
-
-##############
-echo
-echo "simple (global) join with analyzer and parallel replicas"
-
-$CLICKHOUSE_CLIENT -q "
-select * from (select key, value from num_1) l
-inner join (select key, value from num_2) r on l.key = r.key
-order by l.key limit 10 offset 700000
-SETTINGS allow_experimental_analyzer=1, allow_experimental_parallel_reading_from_replicas = 2,
-max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1,
-cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join=0"
-
-$CLICKHOUSE_CLIENT -q "
-select * from (select key, value from num_1) l
-inner join (select key, value from num_2) r on l.key = r.key
-order by l.key limit 10 offset 700000
-SETTINGS allow_experimental_analyzer=1, allow_experimental_parallel_reading_from_replicas = 2, send_logs_level='trace',
-max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1,
-cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join=0" 2>&1 |
-grep "executeQuery\|<Debug>.*Coordinator: Coordination done" |
-grep -o "SELECT.*WithMergeableState)\|<Debug>.*Coordinator: Coordination done" |
-sed -re 's/_data_[[:digit:]]+_[[:digit:]]+/_data_/g'
-
-##############
-echo
-echo "simple (local) join with analyzer and parallel replicas"
-
-$CLICKHOUSE_CLIENT -q "
-select * from (select key, value from num_1) l
-inner join (select key, value from num_2) r on l.key = r.key
-order by l.key limit 10 offset 700000
-SETTINGS allow_experimental_analyzer=1,
-allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1,
-cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join=1"
-
-$CLICKHOUSE_CLIENT -q "
-select * from (select key, value from num_1) l
-inner join (select key, value from num_2) r on l.key = r.key
-order by l.key limit 10 offset 700000
-SETTINGS allow_experimental_analyzer=1, send_logs_level='trace',
-allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1,
-cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join=1" 2>&1 |
-grep "executeQuery\|<Debug>.*Coordinator: Coordination done" |
-grep -o "SELECT.*WithMergeableState)\|<Debug>.*Coordinator: Coordination done" |
-sed -re 's/_data_[[:digit:]]+_[[:digit:]]+/_data_/g'
-
-
-##############
-echo
-echo "simple (local) join with analyzer and parallel replicas and full sorting merge join"
-
-$CLICKHOUSE_CLIENT -q "
-select * from (select key, value from num_1) l
-inner join (select key, value from num_2) r on l.key = r.key
-order by l.key limit 10 offset 700000
-SETTINGS allow_experimental_analyzer=1, join_algorithm='full_sorting_merge',
-allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1,
-cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join=1"
-
-$CLICKHOUSE_CLIENT -q "
-select * from (select key, value from num_1) l
-inner join (select key, value from num_2) r on l.key = r.key
-order by l.key limit 10 offset 700000
-SETTINGS allow_experimental_analyzer=1, join_algorithm='full_sorting_merge', send_logs_level='trace',
-allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1,
-cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join=1" 2>&1 |
-grep "executeQuery\|<Debug>.*Coordinator: Coordination done" |
-grep -o "SELECT.*WithMergeableState)\|<Debug>.*Coordinator: Coordination done" |
-sed -re 's/_data_[[:digit:]]+_[[:digit:]]+/_data_/g'
-
-
-##############
-echo
-echo "nested join with analyzer"
-
-$CLICKHOUSE_CLIENT -q "
-select * from (select key, value from num_1) l
-inner join (select key, value from num_2 inner join
-  (select number * 7 as key from numbers(1e5)) as nn on num_2.key = nn.key settings parallel_replicas_prefer_local_join=1) r
-on l.key = r.key order by l.key limit 10 offset 10000
-SETTINGS allow_experimental_analyzer=1"
-
-
-##############
-echo
-echo "nested join with analyzer and parallel replicas, both local"
-
-$CLICKHOUSE_CLIENT -q "
-select * from (select key, value from num_1) l
-inner join (select key, value from num_2 inner join
-  (select number * 7 as key from numbers(1e5)) as nn on num_2.key = nn.key settings parallel_replicas_prefer_local_join=1) r
-on l.key = r.key order by l.key limit 10 offset 10000
-SETTINGS allow_experimental_analyzer=1,
-allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1,
-cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join=1"
-
-$CLICKHOUSE_CLIENT -q "
-select * from (select key, value from num_1) l
-inner join (select key, value from num_2 inner join
-  (select number * 7 as key from numbers(1e5)) as nn on num_2.key = nn.key settings parallel_replicas_prefer_local_join=1) r
-on l.key = r.key order by l.key limit 10 offset 10000
-SETTINGS allow_experimental_analyzer=1, join_algorithm='full_sorting_merge', send_logs_level='trace',
-allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1,
-cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join=1" 2>&1 |
-grep "executeQuery\|<Debug>.*Coordinator: Coordination done" |
-grep -o "SELECT.*WithMergeableState)\|<Debug>.*Coordinator: Coordination done" |
-sed -re 's/_data_[[:digit:]]+_[[:digit:]]+/_data_/g'
-
-
 ##############
 echo
 echo "nested join with analyzer and parallel replicas, both global"

From aec7848525d2d6bd1cc9e7c573c25bd3b4ac79e7 Mon Sep 17 00:00:00 2001
From: Yakov Olkhovskiy <yakov@clickhouse.com>
Date: Fri, 26 Jul 2024 13:04:44 +0000
Subject: [PATCH 0851/1488] fix

---
 src/Processors/Transforms/WindowTransform.cpp |  2 +-
 .../03210_lag_lead_inframe_types.reference    | 20 +++++++++++++++++++
 .../03210_lag_lead_inframe_types.sql          |  4 ++--
 3 files changed, 23 insertions(+), 3 deletions(-)

diff --git a/src/Processors/Transforms/WindowTransform.cpp b/src/Processors/Transforms/WindowTransform.cpp
index 61be0c32a7d..1eac08780e9 100644
--- a/src/Processors/Transforms/WindowTransform.cpp
+++ b/src/Processors/Transforms/WindowTransform.cpp
@@ -2424,7 +2424,7 @@ struct WindowFunctionLagLeadInFrame final : public WindowFunction
                 argument_types[0]->getName(),
                 argument_types[2]->getName());
         }
-    
+
         const auto from_name = argument_types[2]->getName();
         const auto to_name = argument_types[0]->getName();
         ColumnsWithTypeAndName arguments
diff --git a/tests/queries/0_stateless/03210_lag_lead_inframe_types.reference b/tests/queries/0_stateless/03210_lag_lead_inframe_types.reference
index cc3b9a096b9..d4734a85e72 100644
--- a/tests/queries/0_stateless/03210_lag_lead_inframe_types.reference
+++ b/tests/queries/0_stateless/03210_lag_lead_inframe_types.reference
@@ -18,3 +18,23 @@
 7
 8
 9
+0
+1
+2
+2
+2
+2
+2
+2
+2
+2
+0
+1
+2
+3
+4
+5
+6
+7
+8
+9
diff --git a/tests/queries/0_stateless/03210_lag_lead_inframe_types.sql b/tests/queries/0_stateless/03210_lag_lead_inframe_types.sql
index 5466cfe0fad..f6017ee6690 100644
--- a/tests/queries/0_stateless/03210_lag_lead_inframe_types.sql
+++ b/tests/queries/0_stateless/03210_lag_lead_inframe_types.sql
@@ -1,4 +1,4 @@
-SELECT lagInFrame(2::UInt128, 2, number) OVER w FROM numbers(10) WINDOW w AS (ORDER BY number); -- { serverError BAD_ARGUMENTS }
-SELECT leadInFrame(2::UInt128, 2, number) OVER w FROM numbers(10) WINDOW w AS (ORDER BY number); -- { serverError BAD_ARGUMENTS }
+SELECT lagInFrame(2::UInt128, 2, number) OVER w FROM numbers(10) WINDOW w AS (ORDER BY number);
+SELECT leadInFrame(2::UInt128, 2, number) OVER w FROM numbers(10) WINDOW w AS (ORDER BY number);
 SELECT lagInFrame(2::UInt64, 2, number) OVER w FROM numbers(10) WINDOW w AS (ORDER BY number);
 SELECT leadInFrame(2::UInt64, 2, number) OVER w FROM numbers(10) WINDOW w AS (ORDER BY number);

From d2b3be2fb8345436422e6214f7652545696be6ea Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <nikitamikhaylov@clickhouse.com>
Date: Fri, 26 Jul 2024 15:05:03 +0200
Subject: [PATCH 0852/1488] Apply optimizations for a single file

---
 src/CMakeLists.txt                      |   3 +
 src/Client/ClientBase.cpp               | 163 ----------------------
 src/Client/ClientBaseOptimizedParts.cpp | 178 ++++++++++++++++++++++++
 3 files changed, 181 insertions(+), 163 deletions(-)
 create mode 100644 src/Client/ClientBaseOptimizedParts.cpp

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 0f84dd35320..8c133971785 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -226,6 +226,9 @@ add_object_library(clickhouse_storages_windowview Storages/WindowView)
 add_object_library(clickhouse_storages_s3queue Storages/ObjectStorageQueue)
 add_object_library(clickhouse_storages_materializedview Storages/MaterializedView)
 add_object_library(clickhouse_client Client)
+# Always compile this file with the highest possible level of optimizations, even in Debug builds.
+# https://github.com/ClickHouse/ClickHouse/issues/65745
+set_source_files_properties(Client/ClientBaseOptimizedParts.cpp PROPERTIES COMPILE_FLAGS "-O3")
 add_object_library(clickhouse_bridge BridgeHelper)
 add_object_library(clickhouse_server Server)
 add_object_library(clickhouse_server_http Server/HTTP)
diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp
index 1e1917e1ca1..04af9db7afe 100644
--- a/src/Client/ClientBase.cpp
+++ b/src/Client/ClientBase.cpp
@@ -108,7 +108,6 @@ namespace ErrorCodes
     extern const int UNEXPECTED_PACKET_FROM_SERVER;
     extern const int INVALID_USAGE_OF_INPUT;
     extern const int CANNOT_SET_SIGNAL_HANDLER;
-    extern const int UNRECOGNIZED_ARGUMENTS;
     extern const int LOGICAL_ERROR;
     extern const int CANNOT_OPEN_FILE;
     extern const int FILE_ALREADY_EXISTS;
@@ -2848,168 +2847,6 @@ void ClientBase::showClientVersion()
     output_stream << VERSION_NAME << " " + getName() + " version " << VERSION_STRING << VERSION_OFFICIAL << "." << std::endl;
 }
 
-namespace
-{
-
-/// Define transparent hash to we can use
-/// std::string_view with the containers
-struct TransparentStringHash
-{
-    using is_transparent = void;
-    size_t operator()(std::string_view txt) const
-    {
-        return std::hash<std::string_view>{}(txt);
-    }
-};
-
-/*
- * This functor is used to parse command line arguments and replace dashes with underscores,
- * allowing options to be specified using either dashes or underscores.
- */
-class OptionsAliasParser
-{
-public:
-    explicit OptionsAliasParser(const boost::program_options::options_description& options)
-    {
-        options_names.reserve(options.options().size());
-        for (const auto& option : options.options())
-            options_names.insert(option->long_name());
-    }
-
-    /*
-     * Parses arguments by replacing dashes with underscores, and matches the resulting name with known options
-     * Implements boost::program_options::ext_parser logic
-     */
-    std::pair<std::string, std::string> operator()(const std::string & token) const
-    {
-        if (!token.starts_with("--"))
-            return {};
-        std::string arg = token.substr(2);
-
-        // divide token by '=' to separate key and value if options style=long_allow_adjacent
-        auto pos_eq = arg.find('=');
-        std::string key = arg.substr(0, pos_eq);
-
-        if (options_names.contains(key))
-            // option does not require any changes, because it is already correct
-            return {};
-
-        std::replace(key.begin(), key.end(), '-', '_');
-        if (!options_names.contains(key))
-            // after replacing '-' with '_' argument is still unknown
-            return {};
-
-        std::string value;
-        if (pos_eq != std::string::npos && pos_eq < arg.size())
-            value = arg.substr(pos_eq + 1);
-
-        return {key, value};
-    }
-
-private:
-    std::unordered_set<std::string> options_names;
-};
-
-}
-
-/// Enable optimizations even in debug builds because otherwise options parsing becomes extremely slow affecting .sh tests
-#if defined(__clang__)
-#pragma clang optimize on
-#endif
-void ClientBase::parseAndCheckOptions(OptionsDescription & options_description, po::variables_map & options, Arguments & arguments)
-{
-    if (allow_repeated_settings)
-        addProgramOptionsAsMultitokens(cmd_settings, options_description.main_description.value());
-    else
-        addProgramOptions(cmd_settings, options_description.main_description.value());
-
-    if (allow_merge_tree_settings)
-    {
-        /// Add merge tree settings manually, because names of some settings
-        /// may clash. Query settings have higher priority and we just
-        /// skip ambiguous merge tree settings.
-        auto & main_options = options_description.main_description.value();
-
-        std::unordered_set<std::string, TransparentStringHash, std::equal_to<>> main_option_names;
-        for (const auto & option : main_options.options())
-            main_option_names.insert(option->long_name());
-
-        for (const auto & setting : cmd_merge_tree_settings.all())
-        {
-            const auto add_setting = [&](const std::string_view name)
-            {
-                if (auto it = main_option_names.find(name); it != main_option_names.end())
-                    return;
-
-                if (allow_repeated_settings)
-                    addProgramOptionAsMultitoken(cmd_merge_tree_settings, main_options, name, setting);
-                else
-                    addProgramOption(cmd_merge_tree_settings, main_options, name, setting);
-            };
-
-            const auto & setting_name = setting.getName();
-
-            add_setting(setting_name);
-
-            const auto & settings_to_aliases = MergeTreeSettings::Traits::settingsToAliases();
-            if (auto it = settings_to_aliases.find(setting_name); it != settings_to_aliases.end())
-            {
-                for (const auto alias : it->second)
-                {
-                    add_setting(alias);
-                }
-            }
-        }
-    }
-
-    /// Parse main commandline options.
-    auto parser = po::command_line_parser(arguments)
-                      .options(options_description.main_description.value())
-                      .extra_parser(OptionsAliasParser(options_description.main_description.value()))
-                      .allow_unregistered();
-    po::parsed_options parsed = parser.run();
-
-    /// Check unrecognized options without positional options.
-    auto unrecognized_options = po::collect_unrecognized(parsed.options, po::collect_unrecognized_mode::exclude_positional);
-    if (!unrecognized_options.empty())
-    {
-        auto hints = this->getHints(unrecognized_options[0]);
-        if (!hints.empty())
-            throw Exception(ErrorCodes::UNRECOGNIZED_ARGUMENTS, "Unrecognized option '{}'. Maybe you meant {}",
-                            unrecognized_options[0], toString(hints));
-
-        throw Exception(ErrorCodes::UNRECOGNIZED_ARGUMENTS, "Unrecognized option '{}'", unrecognized_options[0]);
-    }
-
-    /// Check positional options.
-    for (const auto & op : parsed.options)
-    {
-        if (!op.unregistered && op.string_key.empty() && !op.original_tokens[0].starts_with("--")
-            && !op.original_tokens[0].empty() && !op.value.empty())
-        {
-            /// Two special cases for better usability:
-            /// - if the option contains a whitespace, it might be a query: clickhouse "SELECT 1"
-            /// These are relevant for interactive usage - user-friendly, but questionable in general.
-            /// In case of ambiguity or for scripts, prefer using proper options.
-
-            const auto & token = op.original_tokens[0];
-            po::variable_value value(boost::any(op.value), false);
-
-            const char * option;
-            if (token.contains(' '))
-                option = "query";
-            else
-                throw Exception(ErrorCodes::BAD_ARGUMENTS, "Positional option `{}` is not supported.", token);
-
-            if (!options.emplace(option, value).second)
-                throw Exception(ErrorCodes::BAD_ARGUMENTS, "Positional option `{}` is not supported.", token);
-        }
-    }
-
-    po::store(parsed, options);
-}
-
-
 void ClientBase::init(int argc, char ** argv)
 {
     namespace po = boost::program_options;
diff --git a/src/Client/ClientBaseOptimizedParts.cpp b/src/Client/ClientBaseOptimizedParts.cpp
new file mode 100644
index 00000000000..31614d301b6
--- /dev/null
+++ b/src/Client/ClientBaseOptimizedParts.cpp
@@ -0,0 +1,178 @@
+#include <Client/ClientBase.h>
+
+#include <Core/BaseSettingsProgramOptions.h>
+
+namespace DB
+{
+
+/**
+ * Program ptions parsing is very slow in debug builds and it affects .sh tests
+ * causing them to timeout sporadically.
+ * It seems impossible to enable optimizations for a single function (only to disable them), so
+ * instead we extract the code to a separate source file and compile it with different options.
+ */
+
+/// 
+namespace ErrorCodes
+{
+    extern const int UNRECOGNIZED_ARGUMENTS;
+}
+
+namespace
+{
+
+/// Define transparent hash to we can use
+/// std::string_view with the containers
+struct TransparentStringHash
+{
+    using is_transparent = void;
+    size_t operator()(std::string_view txt) const
+    {
+        return std::hash<std::string_view>{}(txt);
+    }
+};
+
+/*
+ * This functor is used to parse command line arguments and replace dashes with underscores,
+ * allowing options to be specified using either dashes or underscores.
+ */
+class OptionsAliasParser
+{
+public:
+    explicit OptionsAliasParser(const boost::program_options::options_description& options)
+    {
+        options_names.reserve(options.options().size());
+        for (const auto& option : options.options())
+            options_names.insert(option->long_name());
+    }
+
+    /*
+     * Parses arguments by replacing dashes with underscores, and matches the resulting name with known options
+     * Implements boost::program_options::ext_parser logic
+     */
+    std::pair<std::string, std::string> operator()(const std::string & token) const
+    {
+        if (!token.starts_with("--"))
+            return {};
+        std::string arg = token.substr(2);
+
+        // divide token by '=' to separate key and value if options style=long_allow_adjacent
+        auto pos_eq = arg.find('=');
+        std::string key = arg.substr(0, pos_eq);
+
+        if (options_names.contains(key))
+            // option does not require any changes, because it is already correct
+            return {};
+
+        std::replace(key.begin(), key.end(), '-', '_');
+        if (!options_names.contains(key))
+            // after replacing '-' with '_' argument is still unknown
+            return {};
+
+        std::string value;
+        if (pos_eq != std::string::npos && pos_eq < arg.size())
+            value = arg.substr(pos_eq + 1);
+
+        return {key, value};
+    }
+
+private:
+    std::unordered_set<std::string> options_names;
+};
+
+}
+
+void ClientBase::parseAndCheckOptions(OptionsDescription & options_description, po::variables_map & options, Arguments & arguments)
+{
+    if (allow_repeated_settings)
+        addProgramOptionsAsMultitokens(cmd_settings, options_description.main_description.value());
+    else
+        addProgramOptions(cmd_settings, options_description.main_description.value());
+
+    if (allow_merge_tree_settings)
+    {
+        /// Add merge tree settings manually, because names of some settings
+        /// may clash. Query settings have higher priority and we just
+        /// skip ambiguous merge tree settings.
+        auto & main_options = options_description.main_description.value();
+
+        std::unordered_set<std::string, TransparentStringHash, std::equal_to<>> main_option_names;
+        for (const auto & option : main_options.options())
+            main_option_names.insert(option->long_name());
+
+        for (const auto & setting : cmd_merge_tree_settings.all())
+        {
+            const auto add_setting = [&](const std::string_view name)
+            {
+                if (auto it = main_option_names.find(name); it != main_option_names.end())
+                    return;
+
+                if (allow_repeated_settings)
+                    addProgramOptionAsMultitoken(cmd_merge_tree_settings, main_options, name, setting);
+                else
+                    addProgramOption(cmd_merge_tree_settings, main_options, name, setting);
+            };
+
+            const auto & setting_name = setting.getName();
+
+            add_setting(setting_name);
+
+            const auto & settings_to_aliases = MergeTreeSettings::Traits::settingsToAliases();
+            if (auto it = settings_to_aliases.find(setting_name); it != settings_to_aliases.end())
+            {
+                for (const auto alias : it->second)
+                {
+                    add_setting(alias);
+                }
+            }
+        }
+    }
+
+    /// Parse main commandline options.
+    auto parser = po::command_line_parser(arguments)
+                      .options(options_description.main_description.value())
+                      .extra_parser(OptionsAliasParser(options_description.main_description.value()))
+                      .allow_unregistered();
+    po::parsed_options parsed = parser.run();
+
+    /// Check unrecognized options without positional options.
+    auto unrecognized_options = po::collect_unrecognized(parsed.options, po::collect_unrecognized_mode::exclude_positional);
+    if (!unrecognized_options.empty())
+    {
+        auto hints = this->getHints(unrecognized_options[0]);
+        if (!hints.empty())
+            throw Exception(ErrorCodes::UNRECOGNIZED_ARGUMENTS, "Unrecognized option '{}'. Maybe you meant {}",
+                            unrecognized_options[0], toString(hints));
+
+        throw Exception(ErrorCodes::UNRECOGNIZED_ARGUMENTS, "Unrecognized option '{}'", unrecognized_options[0]);
+    }
+
+    /// Check positional options.
+    for (const auto & op : parsed.options)
+    {
+        if (!op.unregistered && op.string_key.empty() && !op.original_tokens[0].starts_with("--")
+            && !op.original_tokens[0].empty() && !op.value.empty())
+        {
+            /// Two special cases for better usability:
+            /// - if the option contains a whitespace, it might be a query: clickhouse "SELECT 1"
+            /// These are relevant for interactive usage - user-friendly, but questionable in general.
+            /// In case of ambiguity or for scripts, prefer using proper options.
+
+            const auto & token = op.original_tokens[0];
+            po::variable_value value(boost::any(op.value), false);
+
+            const char * option;
+            if (token.contains(' '))
+                option = "query";
+            else
+                throw Exception(ErrorCodes::BAD_ARGUMENTS, "Positional option `{}` is not supported.", token);
+
+            if (!options.emplace(option, value).second)
+                throw Exception(ErrorCodes::BAD_ARGUMENTS, "Positional option `{}` is not supported.", token);
+        }
+    }
+
+    po::store(parsed, options);
+}
+
+}

From 42384af0ef38dd326337e8cf18327871924f7359 Mon Sep 17 00:00:00 2001
From: Pablo Marcos <pablo.marcos.oltra@clickhouse.com>
Date: Thu, 25 Jul 2024 15:42:50 +0000
Subject: [PATCH 0853/1488] Fix crash when the connection is empty

---
 src/Storages/Distributed/DistributedAsyncInsertBatch.cpp    | 6 ++++++
 .../Distributed/DistributedAsyncInsertDirectoryQueue.cpp    | 3 +++
 2 files changed, 9 insertions(+)

diff --git a/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp b/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp
index e1facec5b40..31779a32c1f 100644
--- a/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp
+++ b/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp
@@ -245,6 +245,9 @@ void DistributedAsyncInsertBatch::sendBatch(const SettingsChanges & settings_cha
                 connection = std::move(result.front().entry);
                 compression_expected = connection->getCompression() == Protocol::Compression::Enable;
 
+                if (connection.isNull())
+                    throw Exception(ErrorCodes::LOGICAL_ERROR, "Empty connection");
+
                 LOG_DEBUG(parent.log, "Sending a batch of {} files to {} ({} rows, {} bytes).",
                     files.size(),
                     connection->getDescription(),
@@ -303,6 +306,9 @@ void DistributedAsyncInsertBatch::sendSeparateFiles(const SettingsChanges & sett
             auto connection = std::move(result.front().entry);
             bool compression_expected = connection->getCompression() == Protocol::Compression::Enable;
 
+            if (connection.isNull())
+                throw Exception(ErrorCodes::LOGICAL_ERROR, "Empty connection");
+
             RemoteInserter remote(*connection, timeouts,
                 distributed_header.insert_query,
                 insert_settings,
diff --git a/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp b/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp
index d471c67553d..15998776d27 100644
--- a/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp
+++ b/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp
@@ -415,6 +415,9 @@ void DistributedAsyncInsertDirectoryQueue::processFile(std::string & file_path,
         auto result = pool->getManyCheckedForInsert(timeouts, insert_settings, PoolMode::GET_ONE, storage.remote_storage.getQualifiedName());
         auto connection = std::move(result.front().entry);
 
+        if (connection.isNull())
+            throw Exception(ErrorCodes::LOGICAL_ERROR, "Empty connection");
+
         LOG_DEBUG(log, "Sending `{}` to {} ({} rows, {} bytes)",
             file_path,
             connection->getDescription(),

From 031b435e3ad35a57d82ff98ad4e6f79d47d1cbc3 Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <nikitamikhaylov@clickhouse.com>
Date: Fri, 26 Jul 2024 15:21:11 +0200
Subject: [PATCH 0854/1488] Style

---
 src/Client/ClientBaseOptimizedParts.cpp | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/Client/ClientBaseOptimizedParts.cpp b/src/Client/ClientBaseOptimizedParts.cpp
index 31614d301b6..421843a0e79 100644
--- a/src/Client/ClientBaseOptimizedParts.cpp
+++ b/src/Client/ClientBaseOptimizedParts.cpp
@@ -1,20 +1,18 @@
 #include <Client/ClientBase.h>
-
 #include <Core/BaseSettingsProgramOptions.h>
 
 namespace DB
 {
 
 /**
- * Program ptions parsing is very slow in debug builds and it affects .sh tests
+ * Program options parsing is very slow in debug builds and it affects .sh tests
  * causing them to timeout sporadically.
  * It seems impossible to enable optimizations for a single function (only to disable them), so
  * instead we extract the code to a separate source file and compile it with different options.
  */
-
-/// 
 namespace ErrorCodes
 {
+    extern const int BAD_ARGUMENTS;
     extern const int UNRECOGNIZED_ARGUMENTS;
 }
 

From 414ebf035d9e2f47c16ee93d7ff0d21fbee89bff Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 26 Jul 2024 15:32:05 +0200
Subject: [PATCH 0855/1488] Fix error

---
 src/IO/ReadWriteBufferFromHTTP.cpp | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/src/IO/ReadWriteBufferFromHTTP.cpp b/src/IO/ReadWriteBufferFromHTTP.cpp
index a62f22d4bd9..4b2e6580f9b 100644
--- a/src/IO/ReadWriteBufferFromHTTP.cpp
+++ b/src/IO/ReadWriteBufferFromHTTP.cpp
@@ -140,6 +140,10 @@ std::optional<size_t> ReadWriteBufferFromHTTP::tryGetFileSize()
         {
             return std::nullopt;
         }
+        catch (const Poco::IOException &)
+        {
+            return std::nullopt;
+        }
     }
 
     return file_info->file_size;
@@ -324,12 +328,12 @@ void ReadWriteBufferFromHTTP::doWithRetries(std::function<void()> && callable,
             error_message = e.displayText();
             exception = std::current_exception();
         }
-        catch (DB::NetException & e)
+        catch (NetException & e)
         {
             error_message = e.displayText();
             exception = std::current_exception();
         }
-        catch (DB::HTTPException & e)
+        catch (HTTPException & e)
         {
             if (!isRetriableError(e.getHTTPStatus()))
                 is_retriable = false;
@@ -337,7 +341,7 @@ void ReadWriteBufferFromHTTP::doWithRetries(std::function<void()> && callable,
             error_message = e.displayText();
             exception = std::current_exception();
         }
-        catch (DB::Exception & e)
+        catch (Exception & e)
         {
             is_retriable = false;
 
@@ -708,6 +712,10 @@ std::optional<time_t> ReadWriteBufferFromHTTP::tryGetLastModificationTime()
         {
             return std::nullopt;
         }
+        catch (const Poco::IOException &)
+        {
+            return std::nullopt;
+        }
     }
 
     return file_info->last_modified;

From 981135bfb104b5ecfa0f1da5533e3d12f6850838 Mon Sep 17 00:00:00 2001
From: Pablo Marcos <pablo.marcos.oltra@clickhouse.com>
Date: Fri, 26 Jul 2024 13:38:42 +0000
Subject: [PATCH 0856/1488] Fix style check

---
 src/Storages/Distributed/DistributedAsyncInsertBatch.cpp     | 5 +++--
 .../Distributed/DistributedAsyncInsertDirectoryQueue.cpp     | 2 +-
 src/Storages/Distributed/DistributedSink.cpp                 | 2 ++
 3 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp b/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp
index 31779a32c1f..5e7b4b979c7 100644
--- a/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp
+++ b/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp
@@ -28,6 +28,7 @@ namespace ErrorCodes
     extern const int TOO_MANY_PARTITIONS;
     extern const int DISTRIBUTED_TOO_MANY_PENDING_BYTES;
     extern const int ARGUMENT_OUT_OF_BOUND;
+    extern const int LOGICAL_ERROR;
 }
 
 /// Can the batch be split and send files from batch one-by-one instead?
@@ -246,7 +247,7 @@ void DistributedAsyncInsertBatch::sendBatch(const SettingsChanges & settings_cha
                 compression_expected = connection->getCompression() == Protocol::Compression::Enable;
 
                 if (connection.isNull())
-                    throw Exception(ErrorCodes::LOGICAL_ERROR, "Empty connection");
+                    throw Exception(ErrorCodes::LOGICAL_ERROR, "Got empty connection");
 
                 LOG_DEBUG(parent.log, "Sending a batch of {} files to {} ({} rows, {} bytes).",
                     files.size(),
@@ -307,7 +308,7 @@ void DistributedAsyncInsertBatch::sendSeparateFiles(const SettingsChanges & sett
             bool compression_expected = connection->getCompression() == Protocol::Compression::Enable;
 
             if (connection.isNull())
-                throw Exception(ErrorCodes::LOGICAL_ERROR, "Empty connection");
+                throw Exception(ErrorCodes::LOGICAL_ERROR, "Got empty connection");
 
             RemoteInserter remote(*connection, timeouts,
                 distributed_header.insert_query,
diff --git a/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp b/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp
index 15998776d27..2bb0e720c72 100644
--- a/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp
+++ b/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp
@@ -416,7 +416,7 @@ void DistributedAsyncInsertDirectoryQueue::processFile(std::string & file_path,
         auto connection = std::move(result.front().entry);
 
         if (connection.isNull())
-            throw Exception(ErrorCodes::LOGICAL_ERROR, "Empty connection");
+            throw Exception(ErrorCodes::LOGICAL_ERROR, "Got empty connection");
 
         LOG_DEBUG(log, "Sending `{}` to {} ({} rows, {} bytes)",
             file_path,
diff --git a/src/Storages/Distributed/DistributedSink.cpp b/src/Storages/Distributed/DistributedSink.cpp
index b2ce62caf0a..f8bbc081e55 100644
--- a/src/Storages/Distributed/DistributedSink.cpp
+++ b/src/Storages/Distributed/DistributedSink.cpp
@@ -378,6 +378,8 @@ DistributedSink::runWritingJob(JobReplica & job, const Block & current_block, si
                     /// (anyway fallback_to_stale_replicas_for_distributed_queries=true by default)
                     auto results = shard_info.pool->getManyCheckedForInsert(timeouts, settings, PoolMode::GET_ONE, storage.remote_storage.getQualifiedName());
                     job.connection_entry = std::move(results.front().entry);
+                    if (job.connection_entry.isNull())
+                        throw Exception(ErrorCodes::LOGICAL_ERROR, "Got empty connection");
                 }
                 else
                 {

From 503dc25d1021eb1b598ac52efc0370cfd15c57c6 Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Fri, 26 Jul 2024 15:53:03 +0200
Subject: [PATCH 0857/1488] Fix linking

---
 programs/odbc-bridge/tests/CMakeLists.txt |  2 +-
 src/CMakeLists.txt                        |  1 -
 src/Common/Exception.cpp                  |  6 +++---
 src/Common/Logger.cpp                     | 12 ++++++++++++
 src/Common/Logger.h                       |  4 ++++
 src/Daemon/BaseDaemon.cpp                 |  2 +-
 src/Loggers/OwnSplitChannel.cpp           | 12 ------------
 src/Loggers/OwnSplitChannel.h             |  4 ----
 8 files changed, 21 insertions(+), 22 deletions(-)

diff --git a/programs/odbc-bridge/tests/CMakeLists.txt b/programs/odbc-bridge/tests/CMakeLists.txt
index 2f63aed7942..f1411dbb554 100644
--- a/programs/odbc-bridge/tests/CMakeLists.txt
+++ b/programs/odbc-bridge/tests/CMakeLists.txt
@@ -1,2 +1,2 @@
 clickhouse_add_executable (validate-odbc-connection-string validate-odbc-connection-string.cpp ../validateODBCConnectionString.cpp)
-target_link_libraries (validate-odbc-connection-string PRIVATE clickhouse_common_io clickhouse_common_config loggers_no_text_log)
+target_link_libraries (validate-odbc-connection-string PRIVATE clickhouse_common_io clickhouse_common_config)
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index fede7d69105..0f84dd35320 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -638,7 +638,6 @@ if (ENABLE_TESTS)
         dbms
         clickhouse_common_config
         clickhouse_common_zookeeper
-        loggers
         hilite_comparator)
 
     if (TARGET ch_contrib::simdjson)
diff --git a/src/Common/Exception.cpp b/src/Common/Exception.cpp
index c4bd4fbd943..d68537513da 100644
--- a/src/Common/Exception.cpp
+++ b/src/Common/Exception.cpp
@@ -3,12 +3,12 @@
 #include <IO/ReadBufferFromString.h>
 #include <IO/ReadHelpers.h>
 #include <IO/WriteHelpers.h>
-#include <Loggers/OwnSplitChannel.h>
 #include <base/demangle.h>
 #include <Common/AtomicLogger.h>
 #include <Common/ErrorCodes.h>
 #include <Common/Exception.h>
 #include <Common/LockMemoryExceptionInThread.h>
+#include <Common/Logger.h>
 #include <Common/MemorySanitizer.h>
 #include <Common/SensitiveDataMasker.h>
 #include <Common/config_version.h>
@@ -253,7 +253,7 @@ void Exception::setThreadFramePointers(ThreadFramePointersBase frame_pointers)
 
 static void tryLogCurrentExceptionImpl(Poco::Logger * logger, const std::string & start_of_message)
 {
-    if (!OwnSplitChannel::isLoggingEnabled())
+    if (!isLoggingEnabled())
         return;
 
     try
@@ -271,7 +271,7 @@ static void tryLogCurrentExceptionImpl(Poco::Logger * logger, const std::string
 
 void tryLogCurrentException(const char * log_name, const std::string & start_of_message)
 {
-    if (!OwnSplitChannel::isLoggingEnabled())
+    if (!isLoggingEnabled())
         return;
 
     /// Under high memory pressure, new allocations throw a
diff --git a/src/Common/Logger.cpp b/src/Common/Logger.cpp
index c8d557bc3a3..bd848abe353 100644
--- a/src/Common/Logger.cpp
+++ b/src/Common/Logger.cpp
@@ -25,3 +25,15 @@ bool hasLogger(const std::string & name)
 {
     return Poco::Logger::has(name);
 }
+
+static constinit std::atomic<bool> allow_logging{true};
+
+bool isLoggingEnabled()
+{
+    return allow_logging;
+}
+
+void disableLogging()
+{
+    allow_logging = false;
+}
diff --git a/src/Common/Logger.h b/src/Common/Logger.h
index b54ccd33e72..7471e3dff9b 100644
--- a/src/Common/Logger.h
+++ b/src/Common/Logger.h
@@ -64,3 +64,7 @@ LoggerRawPtr createRawLogger(const std::string & name, Poco::Channel * channel,
   * Otherwise, returns false.
   */
 bool hasLogger(const std::string & name);
+
+void disableLogging();
+
+bool isLoggingEnabled();
diff --git a/src/Daemon/BaseDaemon.cpp b/src/Daemon/BaseDaemon.cpp
index 366aad00376..e7ae8ea5a1d 100644
--- a/src/Daemon/BaseDaemon.cpp
+++ b/src/Daemon/BaseDaemon.cpp
@@ -158,7 +158,7 @@ BaseDaemon::~BaseDaemon()
         tryLogCurrentException(&logger());
     }
 
-    OwnSplitChannel::disableLogging();
+    disableLogging();
 }
 
 
diff --git a/src/Loggers/OwnSplitChannel.cpp b/src/Loggers/OwnSplitChannel.cpp
index e29d2a1e0aa..c1594361b2c 100644
--- a/src/Loggers/OwnSplitChannel.cpp
+++ b/src/Loggers/OwnSplitChannel.cpp
@@ -16,18 +16,6 @@
 namespace DB
 {
 
-static constinit std::atomic<bool> allow_logging{true};
-
-bool OwnSplitChannel::isLoggingEnabled()
-{
-    return allow_logging;
-}
-
-void OwnSplitChannel::disableLogging()
-{
-    allow_logging = false;
-}
-
 void OwnSplitChannel::log(const Poco::Message & msg)
 {
     if (!isLoggingEnabled())
diff --git a/src/Loggers/OwnSplitChannel.h b/src/Loggers/OwnSplitChannel.h
index 9de55f330be..88bb6b9ce76 100644
--- a/src/Loggers/OwnSplitChannel.h
+++ b/src/Loggers/OwnSplitChannel.h
@@ -39,10 +39,6 @@ public:
 
     void setLevel(const std::string & name, int level);
 
-    static void disableLogging();
-
-    static bool isLoggingEnabled();
-
 private:
     void logSplit(const Poco::Message & msg);
     void tryLogSplit(const Poco::Message & msg);

From 1e12ac577a4ed4f64d4de4feb8110cd794d4ce90 Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Fri, 26 Jul 2024 14:26:37 +0000
Subject: [PATCH 0858/1488] Fix flaky `test_pkill_query_log` (tsan)

---
 tests/integration/test_crash_log/test.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/tests/integration/test_crash_log/test.py b/tests/integration/test_crash_log/test.py
index a5b82039a84..5a63e6ca6a7 100644
--- a/tests/integration/test_crash_log/test.py
+++ b/tests/integration/test_crash_log/test.py
@@ -60,6 +60,13 @@ def test_pkill(started_node):
 
 
 def test_pkill_query_log(started_node):
+    if (
+        started_node.is_built_with_thread_sanitizer()
+        or started_node.is_built_with_address_sanitizer()
+        or started_node.is_built_with_memory_sanitizer()
+    ):
+        pytest.skip("doesn't fit in timeouts for stacktrace generation")
+
     for signal in ["SEGV", "4"]:
         # force create query_log if it was not created
         started_node.query("SYSTEM FLUSH LOGS")

From eeb012357196db988b01a70b33798fce99bb5deb Mon Sep 17 00:00:00 2001
From: Anton Popov <anton@clickhouse.com>
Date: Fri, 26 Jul 2024 14:48:52 +0000
Subject: [PATCH 0859/1488] fix filling of multilevel Nested

---
 src/Interpreters/inplaceBlockConversions.cpp | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/Interpreters/inplaceBlockConversions.cpp b/src/Interpreters/inplaceBlockConversions.cpp
index ce3f25d16f8..945cc62754d 100644
--- a/src/Interpreters/inplaceBlockConversions.cpp
+++ b/src/Interpreters/inplaceBlockConversions.cpp
@@ -316,7 +316,7 @@ static String removeTupleElementsFromSubcolumn(String subcolumn_name, const Name
     {
         auto pos = subcolumn_name.find(elem + ".");
         if (pos != std::string::npos)
-            subcolumn_name.erase(pos, elem.size());
+            subcolumn_name.erase(pos, elem.size() + 1);
     }
 
     if (subcolumn_name.ends_with("."))
@@ -395,7 +395,6 @@ void fillMissingColumns(
 
         if (!current_offsets.empty())
         {
-
             Names tuple_elements;
             auto serialization = IDataType::getSerialization(*requested_column);
 

From d8318fc428e2f5b847415886782fd8e25bca401b Mon Sep 17 00:00:00 2001
From: Konstantin Bogdanov <konstantin@clickhouse.com>
Date: Fri, 26 Jul 2024 17:09:22 +0200
Subject: [PATCH 0860/1488] Wrap in retries

---
 ...1676_clickhouse_client_autocomplete.python | 38 +++++++++++++++----
 1 file changed, 31 insertions(+), 7 deletions(-)

diff --git a/tests/queries/0_stateless/01676_clickhouse_client_autocomplete.python b/tests/queries/0_stateless/01676_clickhouse_client_autocomplete.python
index 0f35d259c7c..fe08a07c214 100644
--- a/tests/queries/0_stateless/01676_clickhouse_client_autocomplete.python
+++ b/tests/queries/0_stateless/01676_clickhouse_client_autocomplete.python
@@ -10,18 +10,36 @@ DEBUG_LOG = os.path.join(
     os.path.basename(os.path.abspath(__file__)).strip(".python") + ".debuglog",
 )
 
+STATE_MAP = {
+    -1: "process did not start",
+    0: "completion was found",
+    1: "process started and said ':)'",
+    2: "completion search was started",
+    3: "completion is missing",
+}
+
 
 def run_with_timeout(func, args, timeout):
-    process = multiprocessing.Process(target=func, args=args)
-    process.start()
-    process.join(timeout)
+    for _ in range(5):
+        state = multiprocessing.Value("i", -1)
+        process = multiprocessing.Process(target=func, args=args, kwargs={"state": state})
+        process.start()
+        process.join(timeout)
 
-    if process.is_alive():
-        process.terminate()
-        print("Timeout")
+        if state.value in (0, 3):
+            return
+
+        if process.is_alive():
+            process.terminate()
+
+            if state.value == -1:
+                continue
+
+            print(f"Timeout, state: {STATE_MAP[state.value]}")
+            return
 
 
-def test_completion(program, argv, comp_word):
+def test_completion(program, argv, comp_word, state=None):
     comp_begin = comp_word[:-3]
 
     shell_pid, master = pty.fork()
@@ -41,6 +59,8 @@ def test_completion(program, argv, comp_word):
                 debug_log_fd.write(repr(output_b) + "\n")
                 debug_log_fd.flush()
 
+            state.value = 1
+
             os.write(master, b"SET " + bytes(comp_begin.encode()))
             output_b = os.read(master, 4096)
             output = output_b.decode()
@@ -55,6 +75,8 @@ def test_completion(program, argv, comp_word):
             time.sleep(0.01)
             os.write(master, b"\t")
 
+            state.value = 2
+
             output_b = os.read(master, 4096)
             output = output_b.decode()
             debug_log_fd.write(repr(output_b) + "\n")
@@ -65,6 +87,7 @@ def test_completion(program, argv, comp_word):
                 # meaning no concise completion is found
                 if "\x07" in output:
                     print(f"{comp_word}: FAIL")
+                    state.value = 3
                     return
 
                 output_b = os.read(master, 4096)
@@ -73,6 +96,7 @@ def test_completion(program, argv, comp_word):
                 debug_log_fd.flush()
 
             print(f"{comp_word}: OK")
+            state.value = 0
         finally:
             os.close(master)
             debug_log_fd.close()

From ff5cd2051fc8bfd609a9040ffba02697283e69af Mon Sep 17 00:00:00 2001
From: maxvostrikov <max.vostrikov@clickhouse.com>
Date: Fri, 26 Jul 2024 17:10:39 +0200
Subject: [PATCH 0861/1488] squash! added somme tests in relation with
 https://github.com/ClickHouse/ClickHouse/pull/54881 with new behaviour when
 enable_named_columns_in_function_tuple=1 (default value)

---
 tests/queries/0_stateless/00307_format_xml.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/00307_format_xml.sql b/tests/queries/0_stateless/00307_format_xml.sql
index 22566112bc7..a7e0e628945 100644
--- a/tests/queries/0_stateless/00307_format_xml.sql
+++ b/tests/queries/0_stateless/00307_format_xml.sql
@@ -2,4 +2,4 @@ SET output_format_write_statistics = 0;
 SELECT 'unnamed columns in tuple';
 SELECT 'Hello & world' AS s, 'Hello\n<World>', toDateTime('2001-02-03 04:05:06') AS time, arrayMap(x -> toString(x), range(10)) AS arr, (s, time) AS tpl SETTINGS extremes = 1, enable_named_columns_in_function_tuple = 0 FORMAT XML;
 SELECT 'named columns in tuple';
-SELECT 'Hello & world' AS s, toDateTime('2001-02-03 04:05:06') AS time, (s, time) AS tpl SETTINGS extremes = 1, enable_named_columns_in_function_tuple = 0 FORMAT XML;`
+SELECT 'Hello & world' AS s, toDateTime('2001-02-03 04:05:06') AS time, (s, time) AS tpl SETTINGS extremes = 1, enable_named_columns_in_function_tuple = 0 FORMAT XML;

From 7b4951990edc08a1230c3569339c44ac22036eed Mon Sep 17 00:00:00 2001
From: jsc0218 <jsc0218@gmail.com>
Date: Fri, 26 Jul 2024 15:19:47 +0000
Subject: [PATCH 0862/1488] better place to check and add more test

---
 src/Interpreters/InterpreterCreateQuery.cpp   | 24 --------------
 src/Storages/StorageFactory.cpp               | 28 ++++++++++++++++
 ...206_projection_merge_special_mergetree.sql | 32 +++++++++++++++++++
 3 files changed, 60 insertions(+), 24 deletions(-)

diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp
index a5f374ba71c..ea10ad59db4 100644
--- a/src/Interpreters/InterpreterCreateQuery.cpp
+++ b/src/Interpreters/InterpreterCreateQuery.cpp
@@ -1281,30 +1281,6 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create)
     /// Set and retrieve list of columns, indices and constraints. Set table engine if needed. Rewrite query in canonical way.
     TableProperties properties = getTablePropertiesAndNormalizeCreateQuery(create, mode);
 
-    /// Projection is only supported in (Replictaed)MergeTree.
-    if (create.storage && create.storage->engine)
-    {
-        if (std::string_view engine_name(create.storage->engine->name);
-            !properties.projections.empty() && engine_name != "MergeTree" && engine_name != "ReplicatedMergeTree")
-        {
-            bool projection_support = false;
-            if (auto * setting = create.storage->settings; setting != nullptr)
-            {
-                for (const auto & change : setting->changes)
-                {
-                    if (change.name == "deduplicate_merge_projection_mode" && change.value != Field("throw"))
-                    {
-                        projection_support = true;
-                        break;
-                    }
-                }
-            }
-            if (!projection_support)
-                throw Exception(ErrorCodes::NOT_IMPLEMENTED,
-                    "Projection is only supported in (Replictaed)MergeTree. Consider drop or rebuild option of deduplicate_merge_projection_mode.");
-        }
-    }
-
     /// Check type compatible for materialized dest table and select columns
     if (create.select && create.is_materialized_view && create.to_table_id && mode <= LoadingStrictnessLevel::CREATE)
     {
diff --git a/src/Storages/StorageFactory.cpp b/src/Storages/StorageFactory.cpp
index 060b271d8f4..71f70a807a8 100644
--- a/src/Storages/StorageFactory.cpp
+++ b/src/Storages/StorageFactory.cpp
@@ -20,6 +20,7 @@ namespace ErrorCodes
     extern const int FUNCTION_CANNOT_HAVE_PARAMETERS;
     extern const int BAD_ARGUMENTS;
     extern const int DATA_TYPE_CANNOT_BE_USED_IN_TABLES;
+    extern const int NOT_IMPLEMENTED;
 }
 
 
@@ -196,9 +197,36 @@ StoragePtr StorageFactory::get(
                     [](StorageFeatures features) { return features.supports_skipping_indices; });
 
             if (query.columns_list && query.columns_list->projections && !query.columns_list->projections->children.empty())
+            {
                 check_feature(
                     "projections",
                     [](StorageFeatures features) { return features.supports_projections; });
+
+                /// Now let's handle the merge tree family, projection is fully supported in (Replictaed)MergeTree,
+                /// but also allowed in non-throw mode with other mergetree family members.
+                chassert(query.storage->engine);
+                if (std::string_view engine_name(query.storage->engine->name);
+                    engine_name != "MergeTree" && engine_name != "ReplicatedMergeTree")
+                {
+                    /// default throw mode in deduplicate_merge_projection_mode
+                    bool projection_allowed = false;
+                    if (auto * setting = query.storage->settings; setting != nullptr)
+                    {
+                        for (const auto & change : setting->changes)
+                        {
+                            if (change.name == "deduplicate_merge_projection_mode" && change.value != Field("throw"))
+                            {
+                                projection_allowed = true;
+                                break;
+                            }
+                        }
+                    }
+                    if (!projection_allowed)
+                        throw Exception(ErrorCodes::NOT_IMPLEMENTED,
+                            "Projection is fully supported in (Replictaed)MergeTree, but also allowed in non-throw mode with other"
+                            " mergetree family members. Consider drop or rebuild option of deduplicate_merge_projection_mode.");
+                }
+            }
         }
     }
 
diff --git a/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql b/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql
index 749f906569e..25517fbba30 100644
--- a/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql
+++ b/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql
@@ -1,5 +1,37 @@
 DROP TABLE IF EXISTS tp;
 
+-- test regular merge tree
+CREATE TABLE tp (
+    type Int32,
+    eventcnt UInt64,
+    PROJECTION p (select sum(eventcnt), type group by type)
+) engine = MergeTree order by type;
+
+INSERT INTO tp SELECT number%3, 1 FROM numbers(3);
+
+OPTIMIZE TABLE tp DEDUPLICATE;  -- { serverError NOT_IMPLEMENTED }
+
+DROP TABLE tp;
+
+CREATE TABLE tp (
+    type Int32,
+    eventcnt UInt64,
+    PROJECTION p (select sum(eventcnt), type group by type)
+) engine = MergeTree order by type
+SETTINGS deduplicate_merge_projection_mode = 'drop';
+
+INSERT INTO tp SELECT number%3, 1 FROM numbers(3);
+
+OPTIMIZE TABLE tp DEDUPLICATE;
+
+ALTER TABLE tp MODIFY SETTING deduplicate_merge_projection_mode = 'throw';
+
+OPTIMIZE TABLE tp DEDUPLICATE;  -- { serverError NOT_IMPLEMENTED }
+
+DROP TABLE tp;
+
+
+-- test irregular merge tree
 CREATE TABLE tp (
     type Int32,
     eventcnt UInt64,

From a59036e5152aac2d44b07e0f62ab0ae1a066bb5b Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Fri, 26 Jul 2024 15:36:15 +0000
Subject: [PATCH 0863/1488] chmod +x
 ./tests/queries/0_stateless/03204_format_join_on.sh

---
 tests/queries/0_stateless/03204_format_join_on.sh | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 mode change 100644 => 100755 tests/queries/0_stateless/03204_format_join_on.sh

diff --git a/tests/queries/0_stateless/03204_format_join_on.sh b/tests/queries/0_stateless/03204_format_join_on.sh
old mode 100644
new mode 100755

From d42fa0690d1b6ec19755b64740d83327e71a914a Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Fri, 26 Jul 2024 15:59:23 +0000
Subject: [PATCH 0864/1488] Remove filterBlockWithDAG.

---
 src/Storages/MergeTree/MergeTreeData.cpp                | 2 +-
 src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp  | 2 +-
 src/Storages/System/StorageSystemDroppedTablesParts.cpp | 2 +-
 src/Storages/System/StorageSystemPartsBase.cpp          | 4 ++--
 src/Storages/System/StorageSystemTables.cpp             | 2 +-
 src/Storages/VirtualColumnUtils.cpp                     | 7 +++----
 src/Storages/VirtualColumnUtils.h                       | 2 +-
 7 files changed, 10 insertions(+), 11 deletions(-)

diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index ecd25e3cf71..d9ab2894dc4 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -1164,7 +1164,7 @@ std::optional<UInt64> MergeTreeData::totalRowsByPartitionPredicateImpl(
     if (valid)
     {
         virtual_columns_block = getBlockWithVirtualsForFilter(metadata_snapshot, parts);
-        VirtualColumnUtils::filterBlockWithDAG(std::move(*filter_dag), virtual_columns_block, local_context);
+        VirtualColumnUtils::filterBlockWithExpression(VirtualColumnUtils::buildFilterExpression(std::move(*filter_dag), local_context), virtual_columns_block);
         part_values = VirtualColumnUtils::extractSingleValueFromBlock<String>(virtual_columns_block, "_part");
         if (part_values.empty())
             return 0;
diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp
index a37dbfa554c..a6ef0063069 100644
--- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp
+++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp
@@ -488,7 +488,7 @@ std::optional<std::unordered_set<String>> MergeTreeDataSelectExecutor::filterPar
         return {};
 
     auto virtual_columns_block = data.getBlockWithVirtualsForFilter(metadata_snapshot, parts);
-    VirtualColumnUtils::filterBlockWithDAG(std::move(*dag), virtual_columns_block, context);
+    VirtualColumnUtils::filterBlockWithExpression(VirtualColumnUtils::buildFilterExpression(std::move(*dag), context), virtual_columns_block);
     return VirtualColumnUtils::extractSingleValueFromBlock<String>(virtual_columns_block, "_part");
 }
 
diff --git a/src/Storages/System/StorageSystemDroppedTablesParts.cpp b/src/Storages/System/StorageSystemDroppedTablesParts.cpp
index defc4ec2d2a..c2601b8ebe3 100644
--- a/src/Storages/System/StorageSystemDroppedTablesParts.cpp
+++ b/src/Storages/System/StorageSystemDroppedTablesParts.cpp
@@ -75,7 +75,7 @@ StoragesDroppedInfoStream::StoragesDroppedInfoStream(std::optional<ActionsDAG> f
     {
         /// Filter block_to_filter with columns 'database', 'table', 'engine', 'active'.
         if (filter)
-            VirtualColumnUtils::filterBlockWithDAG(std::move(*filter), block_to_filter, context);
+            VirtualColumnUtils::filterBlockWithExpression(VirtualColumnUtils::buildFilterExpression(std::move(*filter), context), block_to_filter);
         rows = block_to_filter.rows();
     }
 
diff --git a/src/Storages/System/StorageSystemPartsBase.cpp b/src/Storages/System/StorageSystemPartsBase.cpp
index a0c9a5c61bd..7ace8ee24aa 100644
--- a/src/Storages/System/StorageSystemPartsBase.cpp
+++ b/src/Storages/System/StorageSystemPartsBase.cpp
@@ -124,7 +124,7 @@ StoragesInfoStream::StoragesInfoStream(std::optional<ActionsDAG> filter_by_datab
 
         /// Filter block_to_filter with column 'database'.
         if (filter_by_database)
-            VirtualColumnUtils::filterBlockWithDAG(std::move(*filter_by_database), block_to_filter, context);
+            VirtualColumnUtils::filterBlockWithExpression(VirtualColumnUtils::buildFilterExpression(std::move(*filter_by_database), context), block_to_filter);
         rows = block_to_filter.rows();
 
         /// Block contains new columns, update database_column.
@@ -204,7 +204,7 @@ StoragesInfoStream::StoragesInfoStream(std::optional<ActionsDAG> filter_by_datab
     {
         /// Filter block_to_filter with columns 'database', 'table', 'engine', 'active'.
         if (filter_by_other_columns)
-            VirtualColumnUtils::filterBlockWithDAG(std::move(*filter_by_other_columns), block_to_filter, context);
+            VirtualColumnUtils::filterBlockWithExpression(VirtualColumnUtils::buildFilterExpression(std::move(*filter_by_other_columns), context), block_to_filter);
         rows = block_to_filter.rows();
     }
 
diff --git a/src/Storages/System/StorageSystemTables.cpp b/src/Storages/System/StorageSystemTables.cpp
index 9ae21ded9ba..943ce9c317a 100644
--- a/src/Storages/System/StorageSystemTables.cpp
+++ b/src/Storages/System/StorageSystemTables.cpp
@@ -125,7 +125,7 @@ ColumnPtr getFilteredTables(
         block.insert(ColumnWithTypeAndName(std::move(engine_column), std::make_shared<DataTypeString>(), "engine"));
 
     if (dag)
-        VirtualColumnUtils::filterBlockWithDAG(std::move(*dag), block, context);
+        VirtualColumnUtils::filterBlockWithExpression(VirtualColumnUtils::buildFilterExpression(std::move(*dag), context), block);
 
     return block.getByPosition(0).column;
 }
diff --git a/src/Storages/VirtualColumnUtils.cpp b/src/Storages/VirtualColumnUtils.cpp
index a25b7b5ca49..90c2c7f93c1 100644
--- a/src/Storages/VirtualColumnUtils.cpp
+++ b/src/Storages/VirtualColumnUtils.cpp
@@ -77,11 +77,10 @@ void buildSetsForDAG(const ActionsDAG & dag, const ContextPtr & context)
     }
 }
 
-void filterBlockWithDAG(ActionsDAG dag, Block & block, ContextPtr context)
+ExpressionActionsPtr buildFilterExpression(ActionsDAG dag, ContextPtr context)
 {
     buildSetsForDAG(dag, context);
-    auto actions = std::make_shared<ExpressionActions>(std::move(dag));
-    filterBlockWithExpression(actions, block);
+    return std::make_shared<ExpressionActions>(std::move(dag));
 }
 
 void filterBlockWithExpression(const ExpressionActionsPtr & actions, Block & block)
@@ -384,7 +383,7 @@ void filterBlockWithPredicate(const ActionsDAG::Node * predicate, Block & block,
 {
     auto dag = splitFilterDagForAllowedInputs(predicate, &block,  /*allow_non_deterministic_functions=*/ false);
     if (dag)
-        filterBlockWithDAG(std::move(*dag), block, context);
+        filterBlockWithExpression(buildFilterExpression(std::move(*dag), context), block);
 }
 
 }
diff --git a/src/Storages/VirtualColumnUtils.h b/src/Storages/VirtualColumnUtils.h
index 640f9db2fb8..73b7908b75c 100644
--- a/src/Storages/VirtualColumnUtils.h
+++ b/src/Storages/VirtualColumnUtils.h
@@ -23,7 +23,7 @@ namespace VirtualColumnUtils
 void filterBlockWithPredicate(const ActionsDAG::Node * predicate, Block & block, ContextPtr context);
 
 /// Just filters block. Block should contain all the required columns.
-void filterBlockWithDAG(ActionsDAG dag, Block & block, ContextPtr context);
+ExpressionActionsPtr buildFilterExpression(ActionsDAG dag, ContextPtr context);
 void filterBlockWithExpression(const ExpressionActionsPtr & actions, Block & block);
 
 /// Builds sets used by ActionsDAG inplace.

From 454353215736a4c6da635e777b571be0f1bd1831 Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Fri, 26 Jul 2024 18:33:48 +0200
Subject: [PATCH 0865/1488] Fix ShellCheck

---
 .../0_stateless/03203_client_benchmark_options.sh    | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/tests/queries/0_stateless/03203_client_benchmark_options.sh b/tests/queries/0_stateless/03203_client_benchmark_options.sh
index 475309cebb9..cbbd8aab382 100755
--- a/tests/queries/0_stateless/03203_client_benchmark_options.sh
+++ b/tests/queries/0_stateless/03203_client_benchmark_options.sh
@@ -5,22 +5,22 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CUR_DIR"/../shell_config.sh
 
 output=$(${CLICKHOUSE_CLIENT} -t -q "SELECT sleepEachRow(2) FORMAT Null" 2>&1)
-echo "$output" | grep -q "^2\." && echo "Ok" || { echo "Fail"; echo "$output"; }
+{ echo "$output" | grep -q "^2\." && echo "Ok"; } || { echo "Fail"; echo "$output"; }
 
 output=$(${CLICKHOUSE_CLIENT} --time -q "SELECT sleepEachRow(2) FORMAT Null" 2>&1)
-echo "$output" | grep -q "^2\." && echo "Ok" || { echo "Fail"; echo "$output"; }
+{ echo "$output" | grep -q "^2\." && echo "Ok"; } || { echo "Fail"; echo "$output"; }
 
 output=$(${CLICKHOUSE_CLIENT} --memory-usage -q "SELECT sum(number) FROM numbers(10_000) FORMAT Null" 2>&1)
-echo "$output" | grep -q "^[0-9]\+$" && echo "Ok" || { echo "Fail"; echo "$output"; }
+{ echo "$output" | grep -q "^[0-9]\+$" && echo "Ok"; } || { echo "Fail"; echo "$output"; }
 
 output=$(${CLICKHOUSE_CLIENT} --memory-usage=none -q "SELECT sum(number) FROM numbers(10_000) FORMAT Null" 2>&1)
 echo -n "$output" # expected no output
 
 output=$(${CLICKHOUSE_CLIENT} --memory-usage=default -q "SELECT sum(number) FROM numbers(10_000) FORMAT Null" 2>&1)
-echo "$output" | grep -q "^[0-9]\+$" && echo "Ok" || { echo "Fail"; echo "$output"; }
+{ echo "$output" | grep -q "^[0-9]\+$" && echo "Ok"; } || { echo "Fail"; echo "$output"; }
 
 output=$(${CLICKHOUSE_CLIENT} --memory-usage=readable -q "SELECT sum(number) FROM numbers(10_000) FORMAT Null" 2>&1)
-echo "$output" | grep -q "^[0-9].*B$" && echo "Ok" || { echo "Fail"; echo "$output"; }
+{ echo "$output" | grep -q "^[0-9].*B$" && echo "Ok"; } || { echo "Fail"; echo "$output"; }
 
 output=$(${CLICKHOUSE_CLIENT} --memory-usage=unknown -q "SELECT sum(number) FROM numbers(10_000) FORMAT Null" 2>&1)
-echo "$output" | grep -q "BAD_ARGUMENTS" && echo "Ok" || { echo "Fail"; echo "$output"; }
+{ echo "$output" | grep -q "BAD_ARGUMENTS" && echo "Ok"; } || { echo "Fail"; echo "$output"; }

From d153a1cf93e157acb7fadb5ca8b4f30fbd08bad5 Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Fri, 26 Jul 2024 18:37:30 +0200
Subject: [PATCH 0866/1488] add quotes

---
 .../0_stateless/03203_client_benchmark_options.sh    | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/tests/queries/0_stateless/03203_client_benchmark_options.sh b/tests/queries/0_stateless/03203_client_benchmark_options.sh
index cbbd8aab382..37a1f2cd3ac 100755
--- a/tests/queries/0_stateless/03203_client_benchmark_options.sh
+++ b/tests/queries/0_stateless/03203_client_benchmark_options.sh
@@ -5,22 +5,22 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CUR_DIR"/../shell_config.sh
 
 output=$(${CLICKHOUSE_CLIENT} -t -q "SELECT sleepEachRow(2) FORMAT Null" 2>&1)
-{ echo "$output" | grep -q "^2\." && echo "Ok"; } || { echo "Fail"; echo "$output"; }
+{ echo "$output" | grep -q "^2\." && echo "Ok"; } || { echo "Fail"; echo "'$output'"; }
 
 output=$(${CLICKHOUSE_CLIENT} --time -q "SELECT sleepEachRow(2) FORMAT Null" 2>&1)
-{ echo "$output" | grep -q "^2\." && echo "Ok"; } || { echo "Fail"; echo "$output"; }
+{ echo "$output" | grep -q "^2\." && echo "Ok"; } || { echo "Fail"; echo "'$output'"; }
 
 output=$(${CLICKHOUSE_CLIENT} --memory-usage -q "SELECT sum(number) FROM numbers(10_000) FORMAT Null" 2>&1)
-{ echo "$output" | grep -q "^[0-9]\+$" && echo "Ok"; } || { echo "Fail"; echo "$output"; }
+{ echo "$output" | grep -q "^[0-9]\+$" && echo "Ok"; } || { echo "Fail"; echo "'$output'"; }
 
 output=$(${CLICKHOUSE_CLIENT} --memory-usage=none -q "SELECT sum(number) FROM numbers(10_000) FORMAT Null" 2>&1)
 echo -n "$output" # expected no output
 
 output=$(${CLICKHOUSE_CLIENT} --memory-usage=default -q "SELECT sum(number) FROM numbers(10_000) FORMAT Null" 2>&1)
-{ echo "$output" | grep -q "^[0-9]\+$" && echo "Ok"; } || { echo "Fail"; echo "$output"; }
+{ echo "$output" | grep -q "^[0-9]\+$" && echo "Ok"; } || { echo "Fail"; echo "'$output'"; }
 
 output=$(${CLICKHOUSE_CLIENT} --memory-usage=readable -q "SELECT sum(number) FROM numbers(10_000) FORMAT Null" 2>&1)
-{ echo "$output" | grep -q "^[0-9].*B$" && echo "Ok"; } || { echo "Fail"; echo "$output"; }
+{ echo "$output" | grep -q "^[0-9].*B$" && echo "Ok"; } || { echo "Fail"; echo "'$output'"; }
 
 output=$(${CLICKHOUSE_CLIENT} --memory-usage=unknown -q "SELECT sum(number) FROM numbers(10_000) FORMAT Null" 2>&1)
-{ echo "$output" | grep -q "BAD_ARGUMENTS" && echo "Ok"; } || { echo "Fail"; echo "$output"; }
+{ echo "$output" | grep -q "BAD_ARGUMENTS" && echo "Ok"; } || { echo "Fail"; echo "'$output'"; }

From f276be829bebd8e704e33565127034f3e258cc31 Mon Sep 17 00:00:00 2001
From: robot-clickhouse <robot-clickhouse@users.noreply.github.com>
Date: Fri, 26 Jul 2024 16:59:41 +0000
Subject: [PATCH 0867/1488] Automatic style fix

---
 .../0_stateless/01676_clickhouse_client_autocomplete.python   | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/01676_clickhouse_client_autocomplete.python b/tests/queries/0_stateless/01676_clickhouse_client_autocomplete.python
index fe08a07c214..f363cb64018 100644
--- a/tests/queries/0_stateless/01676_clickhouse_client_autocomplete.python
+++ b/tests/queries/0_stateless/01676_clickhouse_client_autocomplete.python
@@ -22,7 +22,9 @@ STATE_MAP = {
 def run_with_timeout(func, args, timeout):
     for _ in range(5):
         state = multiprocessing.Value("i", -1)
-        process = multiprocessing.Process(target=func, args=args, kwargs={"state": state})
+        process = multiprocessing.Process(
+            target=func, args=args, kwargs={"state": state}
+        )
         process.start()
         process.join(timeout)
 

From 343f1fa4bae219f7c287cb314ed6e04feb9a0de4 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Fri, 26 Jul 2024 17:42:06 +0000
Subject: [PATCH 0868/1488] Check type after
 optimize_rewrite_aggregate_function_with_if.

---
 .../RewriteAggregateFunctionWithIfPass.cpp    | 32 ++++++++++++++++---
 src/Analyzer/Resolve/QueryAnalyzer.cpp        |  4 ++-
 2 files changed, 30 insertions(+), 6 deletions(-)

diff --git a/src/Analyzer/Passes/RewriteAggregateFunctionWithIfPass.cpp b/src/Analyzer/Passes/RewriteAggregateFunctionWithIfPass.cpp
index c1adf05ac76..a48e88132a6 100644
--- a/src/Analyzer/Passes/RewriteAggregateFunctionWithIfPass.cpp
+++ b/src/Analyzer/Passes/RewriteAggregateFunctionWithIfPass.cpp
@@ -2,6 +2,7 @@
 
 #include <DataTypes/DataTypesNumber.h>
 #include <DataTypes/DataTypeNullable.h>
+#include <DataTypes/DataTypeAggregateFunction.h>
 
 #include <AggregateFunctions/AggregateFunctionFactory.h>
 #include <AggregateFunctions/IAggregateFunction.h>
@@ -42,7 +43,7 @@ public:
         if (lower_name.ends_with("if"))
             return;
 
-        auto & function_arguments_nodes = function_node->getArguments().getNodes();
+        const auto & function_arguments_nodes = function_node->getArguments().getNodes();
         if (function_arguments_nodes.size() != 1)
             return;
 
@@ -50,6 +51,8 @@ public:
         if (!if_node || if_node->getFunctionName() != "if")
             return;
 
+        FunctionNodePtr replaced_node;
+
         auto if_arguments_nodes = if_node->getArguments().getNodes();
         auto * first_const_node = if_arguments_nodes[1]->as<ConstantNode>();
         auto * second_const_node = if_arguments_nodes[2]->as<ConstantNode>();
@@ -75,8 +78,11 @@ public:
                     new_arguments[0] = std::move(if_arguments_nodes[1]);
 
                 new_arguments[1] = std::move(if_arguments_nodes[0]);
-                function_arguments_nodes = std::move(new_arguments);
-                resolveAggregateFunctionNodeByName(*function_node, function_node->getFunctionName() + "If");
+
+                replaced_node = std::make_shared<FunctionNode>(function_node->getFunctionName() + "If");
+                replaced_node->getArguments().getNodes() = std::move(new_arguments);
+                replaced_node->getParameters().getNodes() = function_node->getParameters().getNodes();
+                resolveAggregateFunctionNodeByName(*replaced_node, replaced_node->getFunctionName());
             }
         }
         else if (first_const_node)
@@ -104,10 +110,26 @@ public:
                     FunctionFactory::instance().get("not", getContext())->build(not_function->getArgumentColumns()));
                 new_arguments[1] = std::move(not_function);
 
-                function_arguments_nodes = std::move(new_arguments);
-                resolveAggregateFunctionNodeByName(*function_node, function_node->getFunctionName() + "If");
+                replaced_node = std::make_shared<FunctionNode>(function_node->getFunctionName() + "If");
+                replaced_node->getArguments().getNodes() = std::move(new_arguments);
+                replaced_node->getParameters().getNodes() = function_node->getParameters().getNodes();
+                resolveAggregateFunctionNodeByName(*replaced_node, replaced_node->getFunctionName());
             }
         }
+
+        if (!replaced_node)
+            return;
+
+        auto prev_type = function_node->getResultType();
+        auto curr_type = replaced_node->getResultType();
+        if (!prev_type->equals(*curr_type))
+            return;
+
+        /// Just in case, CAST compatible aggregate function states.
+        if (WhichDataType(prev_type).isAggregateFunction() && !DataTypeAggregateFunction::strictEquals(prev_type, curr_type))
+            node = createCastFunction(std::move(replaced_node), prev_type, getContext());
+        else
+            node = std::move(replaced_node);
     }
 };
 
diff --git a/src/Analyzer/Resolve/QueryAnalyzer.cpp b/src/Analyzer/Resolve/QueryAnalyzer.cpp
index b1fe2554988..b1603bb18dd 100644
--- a/src/Analyzer/Resolve/QueryAnalyzer.cpp
+++ b/src/Analyzer/Resolve/QueryAnalyzer.cpp
@@ -3239,11 +3239,13 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi
 
         auto action = function_node_ptr->getNullsAction();
         std::string aggregate_function_name = rewriteAggregateFunctionNameIfNeeded(function_name, action, scope.context);
+        std::cerr << "==================== " << function_name << " -> " << aggregate_function_name << std::endl;
+
 
         AggregateFunctionProperties properties;
         auto aggregate_function
             = AggregateFunctionFactory::instance().get(aggregate_function_name, action, argument_types, parameters, properties);
-
+        std::cerr << aggregate_function->getName()  << ' ' << aggregate_function->getResultType()->getName() << std::endl;
         function_node.resolveAsAggregateFunction(std::move(aggregate_function));
 
         return result_projection_names;

From 4833b46a1a86bb1847d2520ea12ea4650c497abc Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Fri, 26 Jul 2024 17:43:30 +0000
Subject: [PATCH 0869/1488] Remove debug code

---
 src/Analyzer/Resolve/QueryAnalyzer.cpp | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/Analyzer/Resolve/QueryAnalyzer.cpp b/src/Analyzer/Resolve/QueryAnalyzer.cpp
index b1603bb18dd..b1fe2554988 100644
--- a/src/Analyzer/Resolve/QueryAnalyzer.cpp
+++ b/src/Analyzer/Resolve/QueryAnalyzer.cpp
@@ -3239,13 +3239,11 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi
 
         auto action = function_node_ptr->getNullsAction();
         std::string aggregate_function_name = rewriteAggregateFunctionNameIfNeeded(function_name, action, scope.context);
-        std::cerr << "==================== " << function_name << " -> " << aggregate_function_name << std::endl;
-
 
         AggregateFunctionProperties properties;
         auto aggregate_function
             = AggregateFunctionFactory::instance().get(aggregate_function_name, action, argument_types, parameters, properties);
-        std::cerr << aggregate_function->getName()  << ' ' << aggregate_function->getResultType()->getName() << std::endl;
+
         function_node.resolveAsAggregateFunction(std::move(aggregate_function));
 
         return result_projection_names;

From 1ba44252cd20ab660d374970257a1ceb438236dd Mon Sep 17 00:00:00 2001
From: jsc0218 <jsc0218@gmail.com>
Date: Fri, 26 Jul 2024 18:33:50 +0000
Subject: [PATCH 0870/1488] turn sql to bash

---
 ...uery_views_log_background_thread.reference | 25 +-----------
 ...02572_query_views_log_background_thread.sh | 38 ++++++++++++++++++
 ...2572_query_views_log_background_thread.sql | 40 -------------------
 3 files changed, 40 insertions(+), 63 deletions(-)
 create mode 100755 tests/queries/0_stateless/02572_query_views_log_background_thread.sh
 delete mode 100644 tests/queries/0_stateless/02572_query_views_log_background_thread.sql

diff --git a/tests/queries/0_stateless/02572_query_views_log_background_thread.reference b/tests/queries/0_stateless/02572_query_views_log_background_thread.reference
index f867fd0d085..d7f2272f5b4 100644
--- a/tests/queries/0_stateless/02572_query_views_log_background_thread.reference
+++ b/tests/queries/0_stateless/02572_query_views_log_background_thread.reference
@@ -1,25 +1,4 @@
--- { echoOn }
-insert into buffer_02572 values (1);
--- ensure that the flush was not direct
-select * from buffer_02572;
+OK
+1
 1
-select * from data_02572;
-select * from copy_02572;
--- we cannot use OPTIMIZE, this will attach query context, so let's wait
-SET function_sleep_max_microseconds_per_block = 6000000;
-select sleepEachRow(1) from numbers(3*2) format Null;
-select sleepEachRow(1) from numbers(3*2) format Null;
-system flush logs;
-select count() > 0, lower(status::String), errorCodeToName(exception_code)
-    from system.query_views_log where
-    view_name = concatWithSeparator('.', currentDatabase(), 'mv_02572') and
-    view_target = concatWithSeparator('.', currentDatabase(), 'copy_02572')
-    group by 2, 3
-;
 1	queryfinish	OK
-select * from buffer_02572;
-1
-select * from data_02572;
-1
-select * from copy_02572;
-1
diff --git a/tests/queries/0_stateless/02572_query_views_log_background_thread.sh b/tests/queries/0_stateless/02572_query_views_log_background_thread.sh
new file mode 100755
index 00000000000..a3e428e75c8
--- /dev/null
+++ b/tests/queries/0_stateless/02572_query_views_log_background_thread.sh
@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+# INSERT buffer_02572 -> data_02572 -> copy_02572
+#                                   ^^
+#                             push to system.query_views_log
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+${CLICKHOUSE_CLIENT} --ignore-error --multiquery --query "drop table if exists buffer_02572;
+    drop table if exists data_02572; drop table if exists copy_02572; drop table if exists mv_02572;"
+
+${CLICKHOUSE_CLIENT} --query="create table copy_02572 (key Int) engine=Memory();"
+${CLICKHOUSE_CLIENT} --query="create table data_02572 (key Int) engine=Memory();"
+${CLICKHOUSE_CLIENT} --query="create table buffer_02572 (key Int) engine=Buffer(currentDatabase(), data_02572, 1, 3, 3, 1, 1e9, 1, 1e9);"
+${CLICKHOUSE_CLIENT} --query="create materialized view mv_02572 to copy_02572 as select * from data_02572;"
+
+${CLICKHOUSE_CLIENT} --query="insert into buffer_02572 values (1);"
+
+# ensure that the flush was not direct
+${CLICKHOUSE_CLIENT} --ignore-error --multiquery --query "select * from data_02572; select * from copy_02572;"
+
+# we cannot use OPTIMIZE, this will attach query context, so let's wait
+for _ in {1..100}; do
+    $CLICKHOUSE_CLIENT -q "select * from data_02572;" | grep -q "1" && echo 'OK' && break
+    sleep 0.5
+done
+
+
+${CLICKHOUSE_CLIENT} --ignore-error --multiquery --query "select * from data_02572; select * from copy_02572;"
+
+${CLICKHOUSE_CLIENT} --query="system flush logs;"
+${CLICKHOUSE_CLIENT} --query="select count() > 0, lower(status::String), errorCodeToName(exception_code)
+    from system.query_views_log where
+    view_name = concatWithSeparator('.', currentDatabase(), 'mv_02572') and
+    view_target = concatWithSeparator('.', currentDatabase(), 'copy_02572')
+    group by 2, 3;"
\ No newline at end of file
diff --git a/tests/queries/0_stateless/02572_query_views_log_background_thread.sql b/tests/queries/0_stateless/02572_query_views_log_background_thread.sql
deleted file mode 100644
index 2e9a62b71da..00000000000
--- a/tests/queries/0_stateless/02572_query_views_log_background_thread.sql
+++ /dev/null
@@ -1,40 +0,0 @@
--- INSERT buffer_02572 -> data_02572 -> copy_02572
---                                   ^^
---                             push to system.query_views_log
-
-drop table if exists buffer_02572;
-drop table if exists data_02572;
-drop table if exists copy_02572;
-drop table if exists mv_02572;
-
-create table copy_02572 (key Int) engine=Memory();
-create table data_02572 (key Int) engine=Memory();
-create table buffer_02572 (key Int) engine=Buffer(currentDatabase(), data_02572, 1,
-    /* never direct flush for flush from background thread */
-    /* min_time= */ 3, 3,
-    1, 1e9,
-    1, 1e9);
-create materialized view mv_02572 to copy_02572 as select * from data_02572;
-
--- { echoOn }
-insert into buffer_02572 values (1);
--- ensure that the flush was not direct
-select * from buffer_02572;
-select * from data_02572;
-select * from copy_02572;
--- we cannot use OPTIMIZE, this will attach query context, so let's wait
-SET function_sleep_max_microseconds_per_block = 6000000;
-select sleepEachRow(1) from numbers(3*2) format Null;
-select sleepEachRow(1) from numbers(3*2) format Null;
-
-system flush logs;
-select count() > 0, lower(status::String), errorCodeToName(exception_code)
-    from system.query_views_log where
-    view_name = concatWithSeparator('.', currentDatabase(), 'mv_02572') and
-    view_target = concatWithSeparator('.', currentDatabase(), 'copy_02572')
-    group by 2, 3
-;
-
-select * from buffer_02572;
-select * from data_02572;
-select * from copy_02572;
\ No newline at end of file

From 870ec237bb427243388acbe5bca770241eeb7fbb Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Sat, 22 Jun 2024 14:14:11 +0200
Subject: [PATCH 0871/1488] Add ability to load dashboards for
 system.dashboards from config

One of the obvious reasons is to allow rendering them with readonly
user, which is not possible right now, due to usage of merge() function
there.

Another one, is to add some custom metrics.

Note, that once set, they overrides the default dashboards preset.

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 programs/server/Server.cpp                    |  2 +
 programs/server/config.xml                    | 25 ++++++++++
 src/Interpreters/Context.cpp                  | 49 +++++++++++++++++++
 src/Interpreters/Context.h                    |  4 ++
 .../System/StorageSystemDashboards.cpp        | 26 +++++++---
 src/Storages/System/StorageSystemDashboards.h |  2 +-
 .../test_custom_dashboards/__init__.py        |  0
 .../configs/config.d/overrides.xml            | 15 ++++++
 .../test_custom_dashboards/test.py            | 35 +++++++++++++
 9 files changed, 149 insertions(+), 9 deletions(-)
 create mode 100644 tests/integration/test_custom_dashboards/__init__.py
 create mode 100644 tests/integration/test_custom_dashboards/configs/config.d/overrides.xml
 create mode 100644 tests/integration/test_custom_dashboards/test.py

diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp
index 16888015f8b..f8aea3ad10c 100644
--- a/programs/server/Server.cpp
+++ b/programs/server/Server.cpp
@@ -1582,6 +1582,8 @@ try
             global_context->setMacros(std::make_unique<Macros>(*config, "macros", log));
             global_context->setExternalAuthenticatorsConfig(*config);
 
+            global_context->setDashboardsConfig(config);
+
             if (global_context->isServerCompletelyStarted())
             {
                 /// It does not make sense to reload anything before server has started.
diff --git a/programs/server/config.xml b/programs/server/config.xml
index 94825a55f67..5dedd78ff2a 100644
--- a/programs/server/config.xml
+++ b/programs/server/config.xml
@@ -1312,6 +1312,31 @@
         <ttl>event_date + INTERVAL 30 DAY</ttl>
     </blob_storage_log>
 
+    <!-- Configure system.dashboards for dashboard.html.
+
+         Could have any query parameters, for which there will be an input on the page.
+         For instance an example from comments have the following:
+         - seconds
+         - rounding
+
+         NOTE: All default dashboards will be overwritten if it was set here. -->
+    <!-- Here is an example without merge() function, to make it work with readonly user -->
+    <!--
+    <dashboards>
+        <dashboard>
+            <dashboard>Overview</dashboard>
+            <title>Queries/second</title>
+            <query>
+                SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(ProfileEvent_Query)
+                FROM system.metric_log
+                WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32}
+                GROUP BY t
+                ORDER BY t WITH FILL STEP {rounding:UInt32}
+            </query>
+        </dashboard>
+    </dashboards>
+    -->
+
     <!-- <top_level_domains_path>/var/lib/clickhouse/top_level_domains/</top_level_domains_path> -->
     <!-- Custom TLD lists.
          Format: <name>/path/to/file</name>
diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp
index 94bcb88ed53..104f90fdb42 100644
--- a/src/Interpreters/Context.cpp
+++ b/src/Interpreters/Context.cpp
@@ -379,6 +379,10 @@ struct ContextSharedPart : boost::noncopyable
     ActionLocksManagerPtr action_locks_manager;             /// Set of storages' action lockers
     OnceFlag system_logs_initialized;
     std::unique_ptr<SystemLogs> system_logs TSA_GUARDED_BY(mutex);                /// Used to log queries and operations on parts
+
+    mutable std::mutex dashboard_mutex;
+    std::optional<Context::Dashboards> dashboards;
+
     std::optional<S3SettingsByEndpoint> storage_s3_settings TSA_GUARDED_BY(mutex);   /// Settings of S3 storage
     std::vector<String> warnings TSA_GUARDED_BY(mutex);                           /// Store warning messages about server configuration.
 
@@ -4313,6 +4317,51 @@ std::vector<ISystemLog *> Context::getSystemLogs() const
     return shared->system_logs->logs;
 }
 
+std::optional<Context::Dashboards> Context::getDashboards() const
+{
+    std::lock_guard lock(shared->dashboard_mutex);
+
+    if (!shared->dashboards)
+        return {};
+    return shared->dashboards;
+}
+
+namespace
+{
+
+String trim(const String & text)
+{
+    std::string_view view(text);
+    ::trim(view, '\n');
+    return String(view);
+}
+
+}
+
+void Context::setDashboardsConfig(const ConfigurationPtr & config)
+{
+    Poco::Util::AbstractConfiguration::Keys keys;
+    config->keys("dashboards", keys);
+
+    Dashboards dashboards;
+    for (const auto & key : keys)
+    {
+        const auto & prefix = "dashboards." + key + ".";
+        dashboards.push_back({
+            { "dashboard", config->getString(prefix + "dashboard") },
+            { "title",     config->getString(prefix + "title") },
+            { "query",     trim(config->getString(prefix + "query")) },
+        });
+    }
+
+    {
+        std::lock_guard lock(shared->dashboard_mutex);
+        if (!dashboards.empty())
+            shared->dashboards.emplace(std::move(dashboards));
+        else
+            shared->dashboards.reset();
+    }
+}
 
 CompressionCodecPtr Context::chooseCompressionCodec(size_t part_size, double part_size_ratio) const
 {
diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h
index cb553d07513..e3cc96ccfc9 100644
--- a/src/Interpreters/Context.h
+++ b/src/Interpreters/Context.h
@@ -1153,6 +1153,10 @@ public:
 
     std::vector<ISystemLog *> getSystemLogs() const;
 
+    using Dashboards = std::vector<std::map<String, String>>;
+    std::optional<Dashboards> getDashboards() const;
+    void setDashboardsConfig(const ConfigurationPtr & config);
+
     /// Returns an object used to log operations with parts if it possible.
     /// Provide table name to make required checks.
     std::shared_ptr<PartLog> getPartLog(const String & part_database) const;
diff --git a/src/Storages/System/StorageSystemDashboards.cpp b/src/Storages/System/StorageSystemDashboards.cpp
index 5faa37d951e..96ba7e59cf2 100644
--- a/src/Storages/System/StorageSystemDashboards.cpp
+++ b/src/Storages/System/StorageSystemDashboards.cpp
@@ -1,6 +1,7 @@
 #include <string_view>
 #include <Storages/System/StorageSystemDashboards.h>
 #include <Common/StringUtils.h>
+#include <Interpreters/Context.h>
 
 namespace DB
 {
@@ -22,9 +23,9 @@ String trim(const char * text)
     return String(view);
 }
 
-void StorageSystemDashboards::fillData(MutableColumns & res_columns, ContextPtr, const ActionsDAG::Node *, std::vector<UInt8>) const
+void StorageSystemDashboards::fillData(MutableColumns & res_columns, ContextPtr context, const ActionsDAG::Node *, std::vector<UInt8>) const
 {
-    static const std::vector<std::map<String, String>> dashboards
+    static const std::vector<std::map<String, String>> default_dashboards
     {
         /// Default dashboard for self-managed ClickHouse
         {
@@ -371,13 +372,22 @@ ORDER BY t WITH FILL STEP {rounding:UInt32}
         }
     };
 
-    for (const auto & row : dashboards)
+    auto add_dashboards = [&](const auto & dashboards)
     {
-        size_t i = 0;
-        res_columns[i++]->insert(row.at("dashboard"));
-        res_columns[i++]->insert(row.at("title"));
-        res_columns[i++]->insert(row.at("query"));
-    }
+        for (const auto & row : dashboards)
+        {
+            size_t i = 0;
+            res_columns[i++]->insert(row.at("dashboard"));
+            res_columns[i++]->insert(row.at("title"));
+            res_columns[i++]->insert(row.at("query"));
+        }
+    };
+
+    const auto & context_dashboards = context->getDashboards();
+    if (context_dashboards.has_value())
+        add_dashboards(*context_dashboards);
+    else
+        add_dashboards(default_dashboards);
 }
 
 }
diff --git a/src/Storages/System/StorageSystemDashboards.h b/src/Storages/System/StorageSystemDashboards.h
index f3e957e06c5..1862d53849b 100644
--- a/src/Storages/System/StorageSystemDashboards.h
+++ b/src/Storages/System/StorageSystemDashboards.h
@@ -22,7 +22,7 @@ public:
 protected:
     using IStorageSystemOneBlock::IStorageSystemOneBlock;
 
-    void fillData(MutableColumns & res_columns, ContextPtr, const ActionsDAG::Node *, std::vector<UInt8>) const override;
+    void fillData(MutableColumns & res_columns, ContextPtr context, const ActionsDAG::Node *, std::vector<UInt8>) const override;
 };
 
 }
diff --git a/tests/integration/test_custom_dashboards/__init__.py b/tests/integration/test_custom_dashboards/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/integration/test_custom_dashboards/configs/config.d/overrides.xml b/tests/integration/test_custom_dashboards/configs/config.d/overrides.xml
new file mode 100644
index 00000000000..c3825784785
--- /dev/null
+++ b/tests/integration/test_custom_dashboards/configs/config.d/overrides.xml
@@ -0,0 +1,15 @@
+<clickhouse>
+    <dashboards>
+        <dashboard>
+            <dashboard>Overview</dashboard>
+            <title>Queries/second</title>
+            <query>
+                SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(ProfileEvent_Query)
+                FROM system.metric_log
+                WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32}
+                GROUP BY t
+                ORDER BY t WITH FILL STEP {rounding:UInt32}
+            </query>
+        </dashboard>
+    </dashboards>
+</clickhouse>
diff --git a/tests/integration/test_custom_dashboards/test.py b/tests/integration/test_custom_dashboards/test.py
new file mode 100644
index 00000000000..7b360f58dc7
--- /dev/null
+++ b/tests/integration/test_custom_dashboards/test.py
@@ -0,0 +1,35 @@
+import pytest
+from helpers.cluster import ClickHouseCluster
+
+
+cluster = ClickHouseCluster(__file__)
+
+default = cluster.add_instance("default")
+custom = cluster.add_instance("custom", main_configs=["configs/config.d/overrides.xml"])
+
+
+@pytest.fixture(scope="module", autouse=True)
+def start_cluster():
+    try:
+        cluster.start()
+        yield cluster
+    finally:
+        cluster.shutdown()
+
+
+def test_custom_dashboards():
+    assert int(default.query("select count()>10 from system.dashboards")) == 1
+    assert int(custom.query("select count() from system.dashboards")) == 1
+    assert (
+        default.query(
+            "select normalizeQuery(query) from system.dashboards where dashboard = 'Overview' and title = 'Queries/second'"
+        ).strip()
+        == """
+SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(ProfileEvent_Query) FROM merge(?..) WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32} GROUP BY t ORDER BY t WITH FILL STEP {rounding:UInt32}
+    """.strip()
+    )
+    custom.query(
+        "select normalizeQuery(query) from system.dashboards where dashboard = 'Overview' and title = 'Queries/second'"
+    ).strip == """
+SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(ProfileEvent_Query) FROM system.metric_log WHERE event_date >= toDate(now() - {seconds:UInt32}) AND event_time >= now() - {seconds:UInt32} GROUP BY t ORDER BY t WITH FILL STEP {rounding:UInt32}
+    """.strip()

From 5be75ee349139acb3b70b9b897060c34c170a8cf Mon Sep 17 00:00:00 2001
From: Igor Nikonov <igor@clickhouse.com>
Date: Fri, 26 Jul 2024 19:43:54 +0000
Subject: [PATCH 0872/1488] Fix is done only for the new analyzer

---
 .../0_stateless/03209_parallel_replicas_order_by_all.sql        | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/queries/0_stateless/03209_parallel_replicas_order_by_all.sql b/tests/queries/0_stateless/03209_parallel_replicas_order_by_all.sql
index 46a3ab4d171..eb3cddbbe07 100644
--- a/tests/queries/0_stateless/03209_parallel_replicas_order_by_all.sql
+++ b/tests/queries/0_stateless/03209_parallel_replicas_order_by_all.sql
@@ -10,7 +10,9 @@ ENGINE = ReplicatedMergeTree('/clickhouse/{database}/test_03210', 'r1') ORDER BY
 INSERT INTO order_by_all VALUES ('B', 3, 10), ('C', NULL, 40), ('D', 1, 20), ('A', 2, 30);
 
 SET allow_experimental_parallel_reading_from_replicas=1, max_parallel_replicas=3, cluster_for_parallel_replicas='parallel_replicas';
+SET allow_experimental_analyzer=1; -- fix has been done only for the analyzer
 SET enable_order_by_all = 0;
+
 -- { echoOn }
 SELECT a, b, all FROM order_by_all ORDER BY all SETTINGS enable_order_by_all = 0, allow_experimental_parallel_reading_from_replicas=0;
 SELECT a, b, all FROM order_by_all ORDER BY all SETTINGS enable_order_by_all = 0, allow_experimental_parallel_reading_from_replicas=1;

From 109bd52296550f506b14134b934c6eb8fd3ec094 Mon Sep 17 00:00:00 2001
From: Nikita Taranov <nikita.taranov@clickhouse.com>
Date: Fri, 26 Jul 2024 23:05:56 +0100
Subject: [PATCH 0873/1488] impl

---
 cmake/sanitize.cmake                                           | 2 +-
 contrib/libprotobuf-mutator                                    | 2 +-
 src/AggregateFunctions/fuzzers/CMakeLists.txt                  | 2 +-
 .../aggregate_function_state_deserialization_fuzzer.cpp        | 1 +
 src/Common/MemoryTracker.cpp                                   | 3 +++
 src/Core/SettingsFields.cpp                                    | 3 +++
 src/Core/fuzzers/CMakeLists.txt                                | 2 +-
 src/DataTypes/fuzzers/CMakeLists.txt                           | 2 +-
 src/Formats/fuzzers/CMakeLists.txt                             | 2 +-
 src/Functions/DateTimeTransforms.h                             | 3 +++
 src/Functions/parseReadableSize.cpp                            | 3 +++
 src/Parsers/fuzzers/codegen_fuzzer/codegen_select_fuzzer.cpp   | 3 ++-
 src/Parsers/fuzzers/create_parser_fuzzer.cpp                   | 2 +-
 src/Storages/fuzzers/CMakeLists.txt                            | 2 +-
 src/Storages/fuzzers/columns_description_fuzzer.cpp            | 2 ++
 15 files changed, 25 insertions(+), 9 deletions(-)

diff --git a/cmake/sanitize.cmake b/cmake/sanitize.cmake
index 08716c1196b..3f111bc65c4 100644
--- a/cmake/sanitize.cmake
+++ b/cmake/sanitize.cmake
@@ -57,7 +57,7 @@ option(WITH_COVERAGE "Instrumentation for code coverage with default implementat
 
 if (WITH_COVERAGE)
     message (STATUS "Enabled instrumentation for code coverage")
-    set(COVERAGE_FLAGS "-fprofile-instr-generate -fcoverage-mapping")
+    set(COVERAGE_FLAGS "SHELL:-fprofile-instr-generate -fcoverage-mapping")
 endif()
 
 option (SANITIZE_COVERAGE "Instrumentation for code coverage with custom callbacks" OFF)
diff --git a/contrib/libprotobuf-mutator b/contrib/libprotobuf-mutator
index a304ec48dcf..1f95f808306 160000
--- a/contrib/libprotobuf-mutator
+++ b/contrib/libprotobuf-mutator
@@ -1 +1 @@
-Subproject commit a304ec48dcf15d942607032151f7e9ee504b5dcf
+Subproject commit 1f95f8083066f5b38fd2db172e7e7f9aa7c49d2d
diff --git a/src/AggregateFunctions/fuzzers/CMakeLists.txt b/src/AggregateFunctions/fuzzers/CMakeLists.txt
index 907a275b4b3..3ff7c86a00c 100644
--- a/src/AggregateFunctions/fuzzers/CMakeLists.txt
+++ b/src/AggregateFunctions/fuzzers/CMakeLists.txt
@@ -1,2 +1,2 @@
 clickhouse_add_executable(aggregate_function_state_deserialization_fuzzer aggregate_function_state_deserialization_fuzzer.cpp ${SRCS})
-target_link_libraries(aggregate_function_state_deserialization_fuzzer PRIVATE dbms clickhouse_aggregate_functions)
+target_link_libraries(aggregate_function_state_deserialization_fuzzer PRIVATE dbms clickhouse_aggregate_functions clickhouse_functions)
diff --git a/src/AggregateFunctions/fuzzers/aggregate_function_state_deserialization_fuzzer.cpp b/src/AggregateFunctions/fuzzers/aggregate_function_state_deserialization_fuzzer.cpp
index 425364efb9c..31fc93e4288 100644
--- a/src/AggregateFunctions/fuzzers/aggregate_function_state_deserialization_fuzzer.cpp
+++ b/src/AggregateFunctions/fuzzers/aggregate_function_state_deserialization_fuzzer.cpp
@@ -12,6 +12,7 @@
 
 #include <Interpreters/Context.h>
 
+#include <AggregateFunctions/IAggregateFunction.h>
 #include <AggregateFunctions/registerAggregateFunctions.h>
 
 #include <base/scope_guard.h>
diff --git a/src/Common/MemoryTracker.cpp b/src/Common/MemoryTracker.cpp
index ac412684cf7..7c0115467c6 100644
--- a/src/Common/MemoryTracker.cpp
+++ b/src/Common/MemoryTracker.cpp
@@ -86,7 +86,10 @@ inline std::string_view toDescription(OvercommitResult result)
 
 bool shouldTrackAllocation(Float64 probability, void * ptr)
 {
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wimplicit-const-int-float-conversion"
     return intHash64(uintptr_t(ptr)) < std::numeric_limits<uint64_t>::max() * probability;
+#pragma clang diagnostic pop
 }
 
 }
diff --git a/src/Core/SettingsFields.cpp b/src/Core/SettingsFields.cpp
index 7d094e2a107..68304a29a23 100644
--- a/src/Core/SettingsFields.cpp
+++ b/src/Core/SettingsFields.cpp
@@ -271,9 +271,12 @@ namespace
         if (d != 0.0 && !std::isnormal(d))
             throw Exception(
                 ErrorCodes::CANNOT_PARSE_NUMBER, "A setting's value in seconds must be a normal floating point number or zero. Got {}", d);
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wimplicit-const-int-float-conversion"
         if (d * 1000000 > std::numeric_limits<Poco::Timespan::TimeDiff>::max() || d * 1000000 < std::numeric_limits<Poco::Timespan::TimeDiff>::min())
             throw Exception(
                 ErrorCodes::BAD_ARGUMENTS, "Cannot convert seconds to microseconds: the setting's value in seconds is too big: {}", d);
+#pragma clang diagnostic pop
 
         return static_cast<Poco::Timespan::TimeDiff>(d * 1000000);
     }
diff --git a/src/Core/fuzzers/CMakeLists.txt b/src/Core/fuzzers/CMakeLists.txt
index 51db6fa0b53..3c5c0eed4e6 100644
--- a/src/Core/fuzzers/CMakeLists.txt
+++ b/src/Core/fuzzers/CMakeLists.txt
@@ -1,2 +1,2 @@
 clickhouse_add_executable (names_and_types_fuzzer names_and_types_fuzzer.cpp)
-target_link_libraries (names_and_types_fuzzer PRIVATE dbms)
+target_link_libraries (names_and_types_fuzzer PRIVATE dbms clickhouse_functions)
diff --git a/src/DataTypes/fuzzers/CMakeLists.txt b/src/DataTypes/fuzzers/CMakeLists.txt
index 939bf5f5e3f..bc640358673 100644
--- a/src/DataTypes/fuzzers/CMakeLists.txt
+++ b/src/DataTypes/fuzzers/CMakeLists.txt
@@ -1,2 +1,2 @@
 clickhouse_add_executable(data_type_deserialization_fuzzer data_type_deserialization_fuzzer.cpp ${SRCS})
-target_link_libraries(data_type_deserialization_fuzzer PRIVATE dbms clickhouse_aggregate_functions)
+target_link_libraries(data_type_deserialization_fuzzer PRIVATE dbms clickhouse_aggregate_functions clickhouse_functions)
diff --git a/src/Formats/fuzzers/CMakeLists.txt b/src/Formats/fuzzers/CMakeLists.txt
index 38009aeec1d..99e57bfbca3 100644
--- a/src/Formats/fuzzers/CMakeLists.txt
+++ b/src/Formats/fuzzers/CMakeLists.txt
@@ -1,2 +1,2 @@
 clickhouse_add_executable(format_fuzzer format_fuzzer.cpp ${SRCS})
-target_link_libraries(format_fuzzer PRIVATE dbms clickhouse_aggregate_functions)
+target_link_libraries(format_fuzzer PRIVATE dbms clickhouse_aggregate_functions clickhouse_functions)
diff --git a/src/Functions/DateTimeTransforms.h b/src/Functions/DateTimeTransforms.h
index 5f745f3ccad..9d9acd2483b 100644
--- a/src/Functions/DateTimeTransforms.h
+++ b/src/Functions/DateTimeTransforms.h
@@ -2147,7 +2147,10 @@ struct Transformer
                 if constexpr (std::is_same_v<Additions, DateTimeAccurateConvertStrategyAdditions>
                     || std::is_same_v<Additions, DateTimeAccurateOrNullConvertStrategyAdditions>)
                 {
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wimplicit-const-int-float-conversion"
                     bool is_valid_input = vec_from[i] >= 0 && vec_from[i] <= 0xFFFFFFFFL;
+#pragma clang diagnostic pop
                     if (!is_valid_input)
                     {
                         if constexpr (std::is_same_v<Additions, DateTimeAccurateOrNullConvertStrategyAdditions>)
diff --git a/src/Functions/parseReadableSize.cpp b/src/Functions/parseReadableSize.cpp
index 1abcf7f164f..4f6afb939a5 100644
--- a/src/Functions/parseReadableSize.cpp
+++ b/src/Functions/parseReadableSize.cpp
@@ -217,7 +217,10 @@ private:
         }
 
         Float64 num_bytes_with_decimals = base * iter->second;
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wimplicit-const-int-float-conversion"
         if (num_bytes_with_decimals > std::numeric_limits<UInt64>::max())
+#pragma clang diagnostic pop
         {
             throw Exception(
                 ErrorCodes::BAD_ARGUMENTS,
diff --git a/src/Parsers/fuzzers/codegen_fuzzer/codegen_select_fuzzer.cpp b/src/Parsers/fuzzers/codegen_fuzzer/codegen_select_fuzzer.cpp
index 9310d7d59f7..09af67c337c 100644
--- a/src/Parsers/fuzzers/codegen_fuzzer/codegen_select_fuzzer.cpp
+++ b/src/Parsers/fuzzers/codegen_fuzzer/codegen_select_fuzzer.cpp
@@ -27,7 +27,8 @@ DEFINE_BINARY_PROTO_FUZZER(const Sentence& main)
         DB::ParserQueryWithOutput parser(input.data() + input.size());
         try
         {
-            DB::ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "", 0, 0);
+            DB::ASTPtr ast
+                = parseQuery(parser, input.data(), input.data() + input.size(), "", 0, 0, DB::DBMS_DEFAULT_MAX_PARSER_BACKTRACKS);
 
             DB::WriteBufferFromOStream out(std::cerr, 4096);
             DB::formatAST(*ast, out);
diff --git a/src/Parsers/fuzzers/create_parser_fuzzer.cpp b/src/Parsers/fuzzers/create_parser_fuzzer.cpp
index 854885ad33b..bab8db5671d 100644
--- a/src/Parsers/fuzzers/create_parser_fuzzer.cpp
+++ b/src/Parsers/fuzzers/create_parser_fuzzer.cpp
@@ -14,7 +14,7 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t * data, size_t size)
         std::string input = std::string(reinterpret_cast<const char*>(data), size);
 
         DB::ParserCreateQuery parser;
-        DB::ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "", 0, 1000);
+        DB::ASTPtr ast = parseQuery(parser, input.data(), input.data() + input.size(), "", 0, 1000, DB::DBMS_DEFAULT_MAX_PARSER_BACKTRACKS);
 
         const UInt64 max_ast_depth = 1000;
         ast->checkDepth(max_ast_depth);
diff --git a/src/Storages/fuzzers/CMakeLists.txt b/src/Storages/fuzzers/CMakeLists.txt
index 719b9b77cd9..e36fccec8df 100644
--- a/src/Storages/fuzzers/CMakeLists.txt
+++ b/src/Storages/fuzzers/CMakeLists.txt
@@ -4,4 +4,4 @@ clickhouse_add_executable (mergetree_checksum_fuzzer mergetree_checksum_fuzzer.c
 target_link_libraries (mergetree_checksum_fuzzer PRIVATE dbms)
 
 clickhouse_add_executable (columns_description_fuzzer columns_description_fuzzer.cpp)
-target_link_libraries (columns_description_fuzzer PRIVATE dbms)
+target_link_libraries (columns_description_fuzzer PRIVATE dbms clickhouse_functions)
diff --git a/src/Storages/fuzzers/columns_description_fuzzer.cpp b/src/Storages/fuzzers/columns_description_fuzzer.cpp
index b703a1e7051..469bfe2fb38 100644
--- a/src/Storages/fuzzers/columns_description_fuzzer.cpp
+++ b/src/Storages/fuzzers/columns_description_fuzzer.cpp
@@ -1,5 +1,7 @@
 #include <Storages/ColumnsDescription.h>
 
+#include <iostream>
+
 
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t * data, size_t size)
 {

From 0f4274ac9659efda64e9f362fc74383c0853ba24 Mon Sep 17 00:00:00 2001
From: Yakov Olkhovskiy <99031427+yakov-olkhovskiy@users.noreply.github.com>
Date: Fri, 26 Jul 2024 18:39:38 -0400
Subject: [PATCH 0874/1488] increase 
 filesystem_cache_reserve_space_wait_lock_timeout_milliseconds to 2000

---
 .../00180_no_seek_avoiding_when_reading_from_cache.sh       | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/queries/1_stateful/00180_no_seek_avoiding_when_reading_from_cache.sh b/tests/queries/1_stateful/00180_no_seek_avoiding_when_reading_from_cache.sh
index 2e1b807c496..f9fea2c1dad 100755
--- a/tests/queries/1_stateful/00180_no_seek_avoiding_when_reading_from_cache.sh
+++ b/tests/queries/1_stateful/00180_no_seek_avoiding_when_reading_from_cache.sh
@@ -17,11 +17,11 @@ $CLICKHOUSE_CLIENT -q "OPTIMIZE TABLE hits_s3_sampled FINAL"
 $CLICKHOUSE_CLIENT -q "SYSTEM DROP FILESYSTEM CACHE"
 
 # Warm up the cache
-$CLICKHOUSE_CLIENT -q "SELECT * FROM hits_s3_sampled WHERE URL LIKE '%google%' ORDER BY EventTime LIMIT 10 FORMAT Null"
-$CLICKHOUSE_CLIENT -q "SELECT * FROM hits_s3_sampled WHERE URL LIKE '%google%' ORDER BY EventTime LIMIT 10 FORMAT Null"
+$CLICKHOUSE_CLIENT -q "SELECT * FROM hits_s3_sampled WHERE URL LIKE '%google%' ORDER BY EventTime LIMIT 10 FORMAT Null SETTINGS filesystem_cache_reserve_space_wait_lock_timeout_milliseconds=2000"
+$CLICKHOUSE_CLIENT -q "SELECT * FROM hits_s3_sampled WHERE URL LIKE '%google%' ORDER BY EventTime LIMIT 10 FORMAT Null SETTINGS filesystem_cache_reserve_space_wait_lock_timeout_milliseconds=2000"
 
 query_id=02906_read_from_cache_$RANDOM
-$CLICKHOUSE_CLIENT --query_id ${query_id} -q "SELECT * FROM hits_s3_sampled WHERE URL LIKE '%google%' ORDER BY EventTime LIMIT 10 FORMAT Null"
+$CLICKHOUSE_CLIENT --query_id ${query_id} -q "SELECT * FROM hits_s3_sampled WHERE URL LIKE '%google%' ORDER BY EventTime LIMIT 10 FORMAT Null SETTINGS filesystem_cache_reserve_space_wait_lock_timeout_milliseconds=2000"
 
 $CLICKHOUSE_CLIENT -nq "
   SYSTEM FLUSH LOGS;

From 04775ec4fb1375ac1aa7c650233a3e03d44a59bb Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 27 Jul 2024 02:52:34 +0200
Subject: [PATCH 0875/1488] English

---
 src/Analyzer/Resolve/QueryAnalyzer.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Analyzer/Resolve/QueryAnalyzer.cpp b/src/Analyzer/Resolve/QueryAnalyzer.cpp
index b1fe2554988..51fe5ee6ec2 100644
--- a/src/Analyzer/Resolve/QueryAnalyzer.cpp
+++ b/src/Analyzer/Resolve/QueryAnalyzer.cpp
@@ -1740,7 +1740,7 @@ QueryAnalyzer::QueryTreeNodesWithNames QueryAnalyzer::resolveQualifiedMatcher(Qu
         const auto * tuple_data_type = typeid_cast<const DataTypeTuple *>(result_type.get());
         if (!tuple_data_type)
             throw Exception(ErrorCodes::UNSUPPORTED_METHOD,
-                "Qualified matcher {} find non compound expression {} with type {}. Expected tuple or array of tuples. In scope {}",
+                "Qualified matcher {} found a non-compound expression {} with type {}. Expected a tuple or an array of tuples. In scope {}",
                 matcher_node->formatASTForErrorMessage(),
                 expression_query_tree_node->formatASTForErrorMessage(),
                 expression_query_tree_node->getResultType()->getName(),

From f187163fa652d59abf75b8e8bbf1cdb85efffb92 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 27 Jul 2024 02:58:00 +0200
Subject: [PATCH 0876/1488] Fix English

---
 src/Interpreters/SubstituteColumnOptimizer.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Interpreters/SubstituteColumnOptimizer.h b/src/Interpreters/SubstituteColumnOptimizer.h
index 28aa8be0801..ecb65cd7707 100644
--- a/src/Interpreters/SubstituteColumnOptimizer.h
+++ b/src/Interpreters/SubstituteColumnOptimizer.h
@@ -15,7 +15,7 @@ struct StorageInMemoryMetadata;
 using StorageMetadataPtr = std::shared_ptr<const StorageInMemoryMetadata>;
 
 /// Optimizer that tries to replace columns to equal columns (according to constraints)
-/// with lower size (according to compressed and uncomressed size).
+/// with lower size (according to compressed and uncompressed sizes).
 class SubstituteColumnOptimizer
 {
 public:

From 0ed2c7e4a00d447c99823aa1b707e392de18c2db Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 27 Jul 2024 03:16:30 +0200
Subject: [PATCH 0877/1488] Sync with private

---
 src/IO/ReadBufferFromEmptyFile.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/IO/ReadBufferFromEmptyFile.h b/src/IO/ReadBufferFromEmptyFile.h
index b15299dafee..7808ef62fd9 100644
--- a/src/IO/ReadBufferFromEmptyFile.h
+++ b/src/IO/ReadBufferFromEmptyFile.h
@@ -20,6 +20,7 @@ private:
     off_t seek(off_t /*off*/, int /*whence*/) override { return 0; }
     off_t getPosition() override { return 0; }
     std::optional<size_t> tryGetFileSize() override { return 0; }
+    size_t getFileOffsetOfBufferEnd() const override { return 0; }
 };
 
 }

From c78aff235c4f21ceaeae6e94be06ddd5e20db0fa Mon Sep 17 00:00:00 2001
From: jsc0218 <jsc0218@gmail.com>
Date: Sat, 27 Jul 2024 01:32:09 +0000
Subject: [PATCH 0878/1488] ignore setting for regular mergetree

---
 src/Storages/MergeTree/MergeTask.cpp       | 3 ++-
 src/Storages/MergeTree/MergeTreeSettings.h | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp
index 0b358c0fd7c..ce06adf110c 100644
--- a/src/Storages/MergeTree/MergeTask.cpp
+++ b/src/Storages/MergeTree/MergeTask.cpp
@@ -800,7 +800,8 @@ bool MergeTask::MergeProjectionsStage::mergeMinMaxIndexAndPrepareProjections() c
     const auto mode = global_ctx->data->getSettings()->deduplicate_merge_projection_mode;
     /// Under throw mode, we still choose to drop projections due to backward compatibility since some
     /// users might have projections before this change.
-    if (mode == DeduplicateMergeProjectionMode::THROW || mode == DeduplicateMergeProjectionMode::DROP)
+    if (global_ctx->data->merging_params.mode != MergeTreeData::MergingParams::Ordinary
+        && (mode == DeduplicateMergeProjectionMode::THROW || mode == DeduplicateMergeProjectionMode::DROP))
     {
         ctx->projections_iterator = ctx->tasks_for_projections.begin();
         return false;
diff --git a/src/Storages/MergeTree/MergeTreeSettings.h b/src/Storages/MergeTree/MergeTreeSettings.h
index 166a18a7bab..5ba1988cc5d 100644
--- a/src/Storages/MergeTree/MergeTreeSettings.h
+++ b/src/Storages/MergeTree/MergeTreeSettings.h
@@ -215,7 +215,7 @@ struct Settings;
     M(Float, primary_key_ratio_of_unique_prefix_values_to_skip_suffix_columns, 0.9f, "If the value of a column of the primary key in data part changes at least in this ratio of times, skip loading next columns in memory. This allows to save memory usage by not loading useless columns of the primary key.", 0) \
     /** Projection settings. */ \
     M(UInt64, max_projections, 25, "The maximum number of merge tree projections.", 0) \
-    M(DeduplicateMergeProjectionMode, deduplicate_merge_projection_mode, DeduplicateMergeProjectionMode::THROW, "Whether to allow create projection for the table with non-classic MergeTree, if allowed, what is the next action.", 0) \
+    M(DeduplicateMergeProjectionMode, deduplicate_merge_projection_mode, DeduplicateMergeProjectionMode::THROW, "Whether to allow create projection for the table with non-classic MergeTree, if allowed, what is the action when merge, drop or rebuild.", 0) \
 
 #define MAKE_OBSOLETE_MERGE_TREE_SETTING(M, TYPE, NAME, DEFAULT) \
     M(TYPE, NAME, DEFAULT, "Obsolete setting, does nothing.", BaseSettingsHelpers::Flags::OBSOLETE)

From 90605127c248ec2995c84045fd6a443bff772903 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 27 Jul 2024 03:35:17 +0200
Subject: [PATCH 0879/1488] Better exception message

---
 src/Parsers/IAST.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Parsers/IAST.h b/src/Parsers/IAST.h
index ee70fed0f07..4f8edac8597 100644
--- a/src/Parsers/IAST.h
+++ b/src/Parsers/IAST.h
@@ -66,7 +66,7 @@ public:
     /** Set the alias. */
     virtual void setAlias(const String & /*to*/)
     {
-        throw Exception(ErrorCodes::LOGICAL_ERROR, "Can't set alias of {}", getColumnName());
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "Can't set alias of {} of {}", getColumnName(), getID());
     }
 
     /** Get the text that identifies this element. */

From 10dc9232a11d9733965b508dbacb84c4df6f6637 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 27 Jul 2024 03:35:25 +0200
Subject: [PATCH 0880/1488] Remove strange code

---
 src/Interpreters/SubstituteColumnOptimizer.cpp | 11 -----------
 1 file changed, 11 deletions(-)

diff --git a/src/Interpreters/SubstituteColumnOptimizer.cpp b/src/Interpreters/SubstituteColumnOptimizer.cpp
index c4aef89fed2..1a7929c1857 100644
--- a/src/Interpreters/SubstituteColumnOptimizer.cpp
+++ b/src/Interpreters/SubstituteColumnOptimizer.cpp
@@ -237,17 +237,6 @@ void SubstituteColumnOptimizer::perform()
 
     const auto & compare_graph = metadata_snapshot->getConstraints().getGraph();
 
-    // Fill aliases
-    if (select_query->select())
-    {
-        auto * list = select_query->refSelect()->as<ASTExpressionList>();
-        if (!list)
-            throw Exception(ErrorCodes::LOGICAL_ERROR, "List of selected columns must be ASTExpressionList");
-
-        for (ASTPtr & ast : list->children)
-            ast->setAlias(ast->getAliasOrColumnName());
-    }
-
     auto run_for_all = [&](const auto func)
     {
         if (select_query->where())

From 4174726d0dd8e450a2ffd009c95c1d39d2de7060 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 27 Jul 2024 03:38:31 +0200
Subject: [PATCH 0881/1488] Add a test

---
 ...ptimize_with_constraints_logical_error.reference |  0
 ...3212_optimize_with_constraints_logical_error.sql | 13 +++++++++++++
 2 files changed, 13 insertions(+)
 create mode 100644 tests/queries/0_stateless/03212_optimize_with_constraints_logical_error.reference
 create mode 100644 tests/queries/0_stateless/03212_optimize_with_constraints_logical_error.sql

diff --git a/tests/queries/0_stateless/03212_optimize_with_constraints_logical_error.reference b/tests/queries/0_stateless/03212_optimize_with_constraints_logical_error.reference
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/queries/0_stateless/03212_optimize_with_constraints_logical_error.sql b/tests/queries/0_stateless/03212_optimize_with_constraints_logical_error.sql
new file mode 100644
index 00000000000..16a27af986b
--- /dev/null
+++ b/tests/queries/0_stateless/03212_optimize_with_constraints_logical_error.sql
@@ -0,0 +1,13 @@
+DROP TABLE IF EXISTS test_table;
+CREATE TABLE test_table
+(
+    id UInt64,
+    value String
+) ENGINE=TinyLog;
+
+EXPLAIN SYNTAX
+WITH 1 AS compound_value SELECT * APPLY (x -> compound_value.*)
+FROM test_table WHERE x > 0
+SETTINGS convert_query_to_cnf = true, optimize_using_constraints = true, optimize_substitute_columns = true; -- { serverError UNKNOWN_IDENTIFIER }
+
+DROP TABLE test_table;

From 9969026e4636f1e94abc61816f355ed5d43a1fce Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 27 Jul 2024 03:39:01 +0200
Subject: [PATCH 0882/1488] Further enhancement

---
 src/Interpreters/SubstituteColumnOptimizer.cpp | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/Interpreters/SubstituteColumnOptimizer.cpp b/src/Interpreters/SubstituteColumnOptimizer.cpp
index 1a7929c1857..ec51db56f14 100644
--- a/src/Interpreters/SubstituteColumnOptimizer.cpp
+++ b/src/Interpreters/SubstituteColumnOptimizer.cpp
@@ -237,6 +237,9 @@ void SubstituteColumnOptimizer::perform()
 
     const auto & compare_graph = metadata_snapshot->getConstraints().getGraph();
 
+    if (compare_graph.getNumOfComponents() == 0)
+        return;
+
     auto run_for_all = [&](const auto func)
     {
         if (select_query->where())

From bf16b18f50f6b9baa038ddde3bb5200d4745cfd7 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 27 Jul 2024 03:47:29 +0200
Subject: [PATCH 0883/1488] Update SubstituteColumnOptimizer.cpp

---
 src/Interpreters/SubstituteColumnOptimizer.cpp | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/src/Interpreters/SubstituteColumnOptimizer.cpp b/src/Interpreters/SubstituteColumnOptimizer.cpp
index ec51db56f14..925ded15857 100644
--- a/src/Interpreters/SubstituteColumnOptimizer.cpp
+++ b/src/Interpreters/SubstituteColumnOptimizer.cpp
@@ -13,10 +13,6 @@
 
 namespace DB
 {
-namespace ErrorCodes
-{
-    extern const int LOGICAL_ERROR;
-}
 
 namespace
 {

From 410125bf8db6a672150c927cef960a181fa39a08 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 27 Jul 2024 03:48:28 +0200
Subject: [PATCH 0884/1488] Fix MSan report in DatabaseReplicated

---
 src/Databases/DatabaseReplicated.cpp | 15 ++++++++++-----
 src/Databases/DatabaseReplicated.h   |  2 +-
 2 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp
index 21218e095bf..f127ccbc224 100644
--- a/src/Databases/DatabaseReplicated.cpp
+++ b/src/Databases/DatabaseReplicated.cpp
@@ -647,12 +647,13 @@ LoadTaskPtr DatabaseReplicated::startupDatabaseAsync(AsyncLoader & async_loader,
             {
                 std::lock_guard lock{ddl_worker_mutex};
                 ddl_worker = std::make_unique<DatabaseReplicatedDDLWorker>(this, getContext());
+                ddl_worker->startup();
+                ddl_worker_initialized = true;
             }
-            ddl_worker->startup();
-            ddl_worker_initialized = true;
         });
     std::scoped_lock lock(mutex);
-    return startup_replicated_database_task = makeLoadTask(async_loader, {job});
+    startup_replicated_database_task = makeLoadTask(async_loader, {job});
+    return startup_replicated_database_task;
 }
 
 void DatabaseReplicated::waitDatabaseStarted() const
@@ -1530,8 +1531,11 @@ void DatabaseReplicated::stopReplication()
 void DatabaseReplicated::shutdown()
 {
     stopReplication();
-    ddl_worker_initialized = false;
-    ddl_worker = nullptr;
+    {
+        std::lock_guard lock{ddl_worker_mutex};
+        ddl_worker_initialized = false;
+        ddl_worker = nullptr;
+    }
     DatabaseAtomic::shutdown();
 }
 
@@ -1679,6 +1683,7 @@ bool DatabaseReplicated::canExecuteReplicatedMetadataAlter() const
     /// It may update the metadata digest (both locally and in ZooKeeper)
     /// before DatabaseReplicatedDDLWorker::initializeReplication() has finished.
     /// We should not update metadata until the database is initialized.
+    std::lock_guard lock{ddl_worker_mutex};
     return ddl_worker_initialized && ddl_worker->isCurrentlyActive();
 }
 
diff --git a/src/Databases/DatabaseReplicated.h b/src/Databases/DatabaseReplicated.h
index 0f901538c33..27ab262d1f1 100644
--- a/src/Databases/DatabaseReplicated.h
+++ b/src/Databases/DatabaseReplicated.h
@@ -155,7 +155,7 @@ private:
     std::atomic_bool is_recovering = false;
     std::atomic_bool ddl_worker_initialized = false;
     std::unique_ptr<DatabaseReplicatedDDLWorker> ddl_worker;
-    std::mutex ddl_worker_mutex;
+    mutable std::mutex ddl_worker_mutex;
     UInt32 max_log_ptr_at_creation = 0;
 
     /// Usually operation with metadata are single-threaded because of the way replication works,

From 10d003b1bf69575c32f55489a2659e7d71d9ad58 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 27 Jul 2024 03:54:00 +0200
Subject: [PATCH 0885/1488] Fix test
 `02310_clickhouse_local_INSERT_progress_profile_events`

---
 ...02310_clickhouse_local_INSERT_progress_profile_events.expect | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/queries/0_stateless/02310_clickhouse_local_INSERT_progress_profile_events.expect b/tests/queries/0_stateless/02310_clickhouse_local_INSERT_progress_profile_events.expect
index bde00b306cf..d5b2a278180 100755
--- a/tests/queries/0_stateless/02310_clickhouse_local_INSERT_progress_profile_events.expect
+++ b/tests/queries/0_stateless/02310_clickhouse_local_INSERT_progress_profile_events.expect
@@ -1,4 +1,6 @@
 #!/usr/bin/expect -f
+# Tags: no-debug, no-tsan, no-msan, no-asan, no-ubsan, no-s3-storage
+# ^ it can be slower than 60 seconds
 
 # This is the regression for the concurrent access in ProgressIndication,
 # so it is important to read enough rows here (10e6).

From 2e4cd203668671ac70d54b72f371e7cf361a9d76 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 27 Jul 2024 04:05:49 +0200
Subject: [PATCH 0886/1488] Remove test `02982_aggregation_states_destruction`

---
 .../02982_aggregation_states_destruction.reference |  1 -
 .../02982_aggregation_states_destruction.sh        | 14 --------------
 2 files changed, 15 deletions(-)
 delete mode 100644 tests/queries/0_stateless/02982_aggregation_states_destruction.reference
 delete mode 100755 tests/queries/0_stateless/02982_aggregation_states_destruction.sh

diff --git a/tests/queries/0_stateless/02982_aggregation_states_destruction.reference b/tests/queries/0_stateless/02982_aggregation_states_destruction.reference
deleted file mode 100644
index d00491fd7e5..00000000000
--- a/tests/queries/0_stateless/02982_aggregation_states_destruction.reference
+++ /dev/null
@@ -1 +0,0 @@
-1
diff --git a/tests/queries/0_stateless/02982_aggregation_states_destruction.sh b/tests/queries/0_stateless/02982_aggregation_states_destruction.sh
deleted file mode 100755
index 84183606d48..00000000000
--- a/tests/queries/0_stateless/02982_aggregation_states_destruction.sh
+++ /dev/null
@@ -1,14 +0,0 @@
-#!/usr/bin/env bash
-# Tags: no-random-settings
-
-CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# shellcheck source=../shell_config.sh
-. "$CURDIR"/../shell_config.sh
-
-
-query_id="02982_$RANDOM"
-$CLICKHOUSE_CLIENT --query_id $query_id --log_query_threads 1 --query="select number, uniq(number) from numbers_mt(1e7) group by number limit 100 format Null;"
-
-$CLICKHOUSE_CLIENT -q "system flush logs;"
-
-$CLICKHOUSE_CLIENT -q "select count() > 0 from system.query_thread_log where query_id = '$query_id' and current_database = currentDatabase() and thread_name = 'AggregDestruct';"

From 51a01fd57660742f3287dea9d9416f0620143233 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 27 Jul 2024 04:33:55 +0200
Subject: [PATCH 0887/1488] Fix error

---
 src/Parsers/ASTDataType.h | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/Parsers/ASTDataType.h b/src/Parsers/ASTDataType.h
index d9427c2fd9e..71d3aeaa4eb 100644
--- a/src/Parsers/ASTDataType.h
+++ b/src/Parsers/ASTDataType.h
@@ -23,12 +23,14 @@ template <typename... Args>
 std::shared_ptr<ASTDataType> makeASTDataType(const String & name, Args &&... args)
 {
     auto data_type = std::make_shared<ASTDataType>();
-
     data_type->name = name;
-    data_type->arguments = std::make_shared<ASTExpressionList>();
-    data_type->children.push_back(data_type->arguments);
 
-    data_type->arguments->children = { std::forward<Args>(args)... };
+    if constexpr (sizeof...(args))
+    {
+        data_type->arguments = std::make_shared<ASTExpressionList>();
+        data_type->children.push_back(data_type->arguments);
+        data_type->arguments->children = { std::forward<Args>(args)... };
+    }
 
     return data_type;
 }

From c2dae64df3946fd45fc4a0c863dd506329ffbb93 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 13 Jul 2024 23:46:06 +0200
Subject: [PATCH 0888/1488] Challenge how optimistic was Maksim Kita

---
 src/Core/Settings.h                 | 2 +-
 src/Core/SettingsChangesHistory.cpp | 3 ++-
 tests/clickhouse-test               | 2 +-
 3 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/Core/Settings.h b/src/Core/Settings.h
index 86e04b2ab4e..c7a1a7e2739 100644
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@@ -186,7 +186,7 @@ class IColumn;
     M(Bool, allow_suspicious_ttl_expressions, false, "Reject TTL expressions that don't depend on any of table's columns. It indicates a user error most of the time.", 0) \
     M(Bool, allow_suspicious_variant_types, false, "In CREATE TABLE statement allows specifying Variant type with similar variant types (for example, with different numeric or date types). Enabling this setting may introduce some ambiguity when working with values with similar types.", 0) \
     M(Bool, allow_suspicious_primary_key, false, "Forbid suspicious PRIMARY KEY/ORDER BY for MergeTree (i.e. SimpleAggregateFunction)", 0) \
-    M(Bool, compile_expressions, false, "Compile some scalar functions and operators to native code.", 0) \
+    M(Bool, compile_expressions, true, "Compile some scalar functions and operators to native code.", 0) \
     M(UInt64, min_count_to_compile_expression, 3, "The number of identical expressions before they are JIT-compiled", 0) \
     M(Bool, compile_aggregate_expressions, true, "Compile aggregate functions to native code.", 0) \
     M(UInt64, min_count_to_compile_aggregate_expression, 3, "The number of identical aggregate expressions before they are JIT-compiled", 0) \
diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp
index 9faf77e9087..0105e69a5e9 100644
--- a/src/Core/SettingsChangesHistory.cpp
+++ b/src/Core/SettingsChangesHistory.cpp
@@ -80,7 +80,8 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
               {"ignore_on_cluster_for_replicated_named_collections_queries", false, false, "Ignore ON CLUSTER clause for replicated named collections management queries."},
               {"backup_restore_s3_retry_attempts", 1000,1000, "Setting for Aws::Client::RetryStrategy, Aws::Client does retries itself, 0 means no retries. It takes place only for backup/restore."},
               {"postgresql_connection_attempt_timeout", 2, 2, "Allow to control 'connect_timeout' parameter of PostgreSQL connection."},
-              {"postgresql_connection_pool_retries", 2, 2, "Allow to control the number of retries in PostgreSQL connection pool."}
+              {"postgresql_connection_pool_retries", 2, 2, "Allow to control the number of retries in PostgreSQL connection pool."},
+              {"compile_expressions", false, true, "We believe that the LLVM infrastructure behind the JIT compiler is stable enough to enable this setting by default."},
               }},
     {"24.6", {{"materialize_skip_indexes_on_insert", true, true, "Added new setting to allow to disable materialization of skip indexes on insert"},
               {"materialize_statistics_on_insert", true, true, "Added new setting to allow to disable materialization of statistics on insert"},
diff --git a/tests/clickhouse-test b/tests/clickhouse-test
index a29c786e998..55ce88917f4 100755
--- a/tests/clickhouse-test
+++ b/tests/clickhouse-test
@@ -779,7 +779,7 @@ class SettingsRandomizer:
         "filesystem_prefetch_step_bytes": lambda: random.choice(
             [0, "100Mi"]
         ),  # 0 means 'auto'
-        # "compile_expressions": lambda: random.randint(0, 1), - this setting has a bug: https://github.com/ClickHouse/ClickHouse/issues/51264
+        "compile_expressions": lambda: random.randint(0, 1),
         "compile_aggregate_expressions": lambda: random.randint(0, 1),
         "compile_sort_description": lambda: random.randint(0, 1),
         "merge_tree_coarse_index_granularity": lambda: random.randint(2, 32),

From 754967c1e30c768bcec2ea727fd91ef024dc4fc3 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Thu, 25 Jul 2024 19:36:48 +0200
Subject: [PATCH 0889/1488] Update setting changes history

---
 src/Core/SettingsChangesHistory.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp
index 0105e69a5e9..87eaeff0ca9 100644
--- a/src/Core/SettingsChangesHistory.cpp
+++ b/src/Core/SettingsChangesHistory.cpp
@@ -57,6 +57,7 @@ String ClickHouseVersion::toString() const
 /// Note: please check if the key already exists to prevent duplicate entries.
 static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory::SettingsChanges>> settings_changes_history_initializer =
 {
+    {"24.8", {{"compile_expressions", false, true, "We believe that the LLVM infrastructure behind the JIT compiler is stable enough to enable this setting by default."}}},
     {"24.7", {{"output_format_parquet_write_page_index", false, true, "Add a possibility to write page index into parquet files."},
               {"output_format_binary_encode_types_in_binary_format", false, false, "Added new setting to allow to write type names in binary format in RowBinaryWithNamesAndTypes output format"},
               {"input_format_binary_decode_types_in_binary_format", false, false, "Added new setting to allow to read type names in binary format in RowBinaryWithNamesAndTypes input format"},
@@ -81,7 +82,6 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
               {"backup_restore_s3_retry_attempts", 1000,1000, "Setting for Aws::Client::RetryStrategy, Aws::Client does retries itself, 0 means no retries. It takes place only for backup/restore."},
               {"postgresql_connection_attempt_timeout", 2, 2, "Allow to control 'connect_timeout' parameter of PostgreSQL connection."},
               {"postgresql_connection_pool_retries", 2, 2, "Allow to control the number of retries in PostgreSQL connection pool."},
-              {"compile_expressions", false, true, "We believe that the LLVM infrastructure behind the JIT compiler is stable enough to enable this setting by default."},
               }},
     {"24.6", {{"materialize_skip_indexes_on_insert", true, true, "Added new setting to allow to disable materialization of skip indexes on insert"},
               {"materialize_statistics_on_insert", true, true, "Added new setting to allow to disable materialization of statistics on insert"},

From a9b09cf426225bfc5f284d05fcc23d84f1a2b1ff Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 27 Jul 2024 04:41:46 +0200
Subject: [PATCH 0890/1488] Fix OOM in test runs

---
 .../02481_parquet_list_monotonically_increasing_offsets.sh     | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/02481_parquet_list_monotonically_increasing_offsets.sh b/tests/queries/0_stateless/02481_parquet_list_monotonically_increasing_offsets.sh
index 55e6ac2f758..6fd6da69b70 100755
--- a/tests/queries/0_stateless/02481_parquet_list_monotonically_increasing_offsets.sh
+++ b/tests/queries/0_stateless/02481_parquet_list_monotonically_increasing_offsets.sh
@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
-# Tags: no-ubsan, no-fasttest, no-tsan
+# Tags: no-ubsan, no-fasttest, no-tsan, no-msan, no-asan
 # It is too slow under TSan
+# It eats too much memory under ASan or MSan
 
 CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh

From ab9270c31a728473ffcea68de05b869f2afbfd5b Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 27 Jul 2024 04:47:13 +0200
Subject: [PATCH 0891/1488] Update test

---
 ..._constraints_simple_optimization.reference |  8 +++---
 .../01623_constraints_column_swap.reference   | 26 +++++++++----------
 2 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/tests/queries/0_stateless/01622_constraints_simple_optimization.reference b/tests/queries/0_stateless/01622_constraints_simple_optimization.reference
index d267df2237f..84c872856ff 100644
--- a/tests/queries/0_stateless/01622_constraints_simple_optimization.reference
+++ b/tests/queries/0_stateless/01622_constraints_simple_optimization.reference
@@ -32,10 +32,10 @@
 1
 1
 0
-SELECT count() AS `count()`
+SELECT count()
 FROM constraint_test_constants
 WHERE (b > 100) OR (c > 100)
-SELECT count() AS `count()`
+SELECT count()
 FROM constraint_test_constants
 WHERE c > 100
 QUERY id: 0
@@ -53,7 +53,7 @@ QUERY id: 0
           COLUMN id: 6, column_name: c, result_type: Int64, source_id: 3
           CONSTANT id: 7, constant_value: UInt64_100, constant_value_type: UInt8
   SETTINGS allow_experimental_analyzer=1
-SELECT count() AS `count()`
+SELECT count()
 FROM constraint_test_constants
 WHERE c > 100
 QUERY id: 0
@@ -71,7 +71,7 @@ QUERY id: 0
           COLUMN id: 6, column_name: c, result_type: Int64, source_id: 3
           CONSTANT id: 7, constant_value: UInt64_100, constant_value_type: UInt8
   SETTINGS allow_experimental_analyzer=1
-SELECT count() AS `count()`
+SELECT count()
 FROM constraint_test_constants
 QUERY id: 0
   PROJECTION COLUMNS
diff --git a/tests/queries/0_stateless/01623_constraints_column_swap.reference b/tests/queries/0_stateless/01623_constraints_column_swap.reference
index 555a4c93f70..d504a86365b 100644
--- a/tests/queries/0_stateless/01623_constraints_column_swap.reference
+++ b/tests/queries/0_stateless/01623_constraints_column_swap.reference
@@ -1,6 +1,6 @@
 SELECT
-    (b AS `cityHash64(a)`) + 10 AS `plus(cityHash64(a), 10)`,
-    (b AS b) + 3 AS `plus(b, 3)`
+    (b AS `cityHash64(a)`) + 10,
+    (b AS b) + 3
 FROM column_swap_test_test
 WHERE b = 1
 QUERY id: 0
@@ -59,8 +59,8 @@ QUERY id: 0
           CONSTANT id: 14, constant_value: UInt64_1, constant_value_type: UInt8
   SETTINGS allow_experimental_analyzer=1
 SELECT
-    (b AS `cityHash64(a)`) + 10 AS `plus(cityHash64(a), 10)`,
-    (b AS b) + 3 AS `plus(b, 3)`
+    (b AS `cityHash64(a)`) + 10,
+    (b AS b) + 3
 FROM column_swap_test_test
 WHERE b = 0
 QUERY id: 0
@@ -89,8 +89,8 @@ QUERY id: 0
           CONSTANT id: 14, constant_value: UInt64_0, constant_value_type: UInt8
   SETTINGS allow_experimental_analyzer=1
 SELECT
-    (b AS `cityHash64(a)`) + 10 AS `plus(cityHash64(a), 10)`,
-    (b AS b) + 3 AS `plus(b, 3)`
+    (b AS `cityHash64(a)`) + 10,
+    (b AS b) + 3
 FROM column_swap_test_test
 WHERE b = 0
 QUERY id: 0
@@ -119,8 +119,8 @@ QUERY id: 0
           CONSTANT id: 14, constant_value: UInt64_0, constant_value_type: UInt8
   SETTINGS allow_experimental_analyzer=1
 SELECT
-    (b AS `cityHash64(a)`) + 10 AS `plus(cityHash64(a), 10)`,
-    (b AS b) + 3 AS `plus(b, 3)`
+    (b AS `cityHash64(a)`) + 10,
+    (b AS b) + 3
 FROM column_swap_test_test
 WHERE b = 1
 QUERY id: 0
@@ -148,7 +148,7 @@ QUERY id: 0
           COLUMN id: 13, column_name: b, result_type: UInt64, source_id: 5
           CONSTANT id: 14, constant_value: UInt64_1, constant_value_type: UInt8
   SETTINGS allow_experimental_analyzer=1
-SELECT (b AS `cityHash64(a)`) + 10 AS `plus(cityHash64(a), 10)`
+SELECT (b AS `cityHash64(a)`) + 10
 FROM column_swap_test_test
 WHERE b = 0
 QUERY id: 0
@@ -171,8 +171,8 @@ QUERY id: 0
           CONSTANT id: 10, constant_value: UInt64_0, constant_value_type: UInt8
   SETTINGS allow_experimental_analyzer=1
 SELECT
-    (cityHash64(a) AS `cityHash64(a)`) + 10 AS `plus(cityHash64(a), 10)`,
-    a AS a
+    (cityHash64(a) AS `cityHash64(a)`) + 10,
+    a
 FROM column_swap_test_test
 WHERE cityHash64(a) = 0
 QUERY id: 0
@@ -203,8 +203,8 @@ QUERY id: 0
           CONSTANT id: 15, constant_value: UInt64_0, constant_value_type: UInt8
   SETTINGS allow_experimental_analyzer=1
 SELECT
-    (cityHash64(a) AS b) + 10 AS `plus(b, 10)`,
-    a AS a
+    (cityHash64(a) AS b) + 10,
+    a
 FROM column_swap_test_test
 WHERE cityHash64(a) = 0
 QUERY id: 0

From 46218b68ff47e079d5636be63f99a1deb5ad180b Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitlibar@clickhouse.com>
Date: Sat, 27 Jul 2024 06:32:26 +0200
Subject: [PATCH 0892/1488] Initialize the client_context after quota_key in
 clickhouse-client.

---
 programs/client/Client.cpp | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp
index f2919db0308..3e613532f3a 100644
--- a/programs/client/Client.cpp
+++ b/programs/client/Client.cpp
@@ -1069,11 +1069,6 @@ void Client::processOptions(const OptionsDescription & options_description,
     global_context->makeGlobalContext();
     global_context->setApplicationType(Context::ApplicationType::CLIENT);
 
-    /// In case of clickhouse-client the `client_context` can be just an alias for the `global_context`.
-    /// (There is no need to copy the context because clickhouse-client has no background tasks so it won't use that context in parallel.)
-    client_context = global_context;
-    initClientContext();
-
     global_context->setSettings(cmd_settings);
 
     /// Copy settings-related program options to config.
@@ -1168,6 +1163,11 @@ void Client::processOptions(const OptionsDescription & options_description,
 
     if (options.count("opentelemetry-tracestate"))
         global_context->getClientTraceContext().tracestate = options["opentelemetry-tracestate"].as<std::string>();
+
+    /// In case of clickhouse-client the `client_context` can be just an alias for the `global_context`.
+    /// (There is no need to copy the context because clickhouse-client has no background tasks so it won't use that context in parallel.)
+    client_context = global_context;
+    initClientContext();
 }
 
 

From 85249b47f18526d59791ffe65f1d247640f9854e Mon Sep 17 00:00:00 2001
From: Blargian <shaunstruwig.ss@gmail.com>
Date: Sat, 27 Jul 2024 11:56:20 +0200
Subject: [PATCH 0893/1488] Review changes

---
 .../functions/type-conversion-functions.md    | 692 ++++++++++--------
 1 file changed, 405 insertions(+), 287 deletions(-)

diff --git a/docs/en/sql-reference/functions/type-conversion-functions.md b/docs/en/sql-reference/functions/type-conversion-functions.md
index 87d824ec5bb..ee253e46014 100644
--- a/docs/en/sql-reference/functions/type-conversion-functions.md
+++ b/docs/en/sql-reference/functions/type-conversion-functions.md
@@ -63,18 +63,19 @@ toInt8(expr)
 
 - `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions).
 
-Supported types:
-- (U)Int8/16/32/64/128/256
-- Float*
-- String representations of (U)Int8/16/32/128/256
+Supported arguments:
+- Values of type (U)Int8/16/32/64/128/256.
+- Values of type Float32/64.
+- String representations of (U)Int8/16/32/128/256.
 
-Unsupported types:
-- Float values `NaN` and `Inf` throw an exception.
-- String representations of binary and hexadecimal values, e.g. `SELECT toInt8('0xc0fe');`
+Unsupported arguments:
+- String representations of Float32/64 values, including `NaN` and `Inf`.
+- String representations of binary and hexadecimal values, e.g. `SELECT toInt8('0xc0fe');`.
 
 :::note
-If the input value cannot be represented within the bounds of [Int8](../data-types/int-uint.md), the result over or under flows. This is not considered an error.  
-For example: `SELECT toInt8(128) == -128;`, `SELECT toInt8(128.0) == -128;`,  `SELECT toInt8('128') == -128;`.
+If the input value cannot be represented within the bounds of [Int8](../data-types/int-uint.md), overflow or underflow of the result occurs. 
+This is not considered an error.  
+For example: `SELECT toInt8(128) == -128;`.
 :::
 
 **Returned value**
@@ -93,15 +94,18 @@ Query:
 SELECT
     toInt8(-8),
     toInt8(-8.8),
-    toInt8('-8');
+    toInt8('-8')
+FORMAT vertical;
 ```
 
 Result:
 
 ```response
-   ┌─toInt8(-8)─┬─toInt8(-8.8)─┬─toInt8('-8')─┐
-1. │         -8 │           -8 │           -8 │
-   └────────────┴──────────────┴──────────────┘
+Row 1:
+──────
+toInt8(-8):   -8
+toInt8(-8.8): -8
+toInt8('-8'): -8
 ```
 
 **See also**
@@ -124,14 +128,17 @@ toInt8OrZero(x)
 
 - `x` — A String representation of a number. [String](../data-types/string.md).
 
-Supported types:
-- String representations of (U)Int8/16/32/128/256
+Supported arguments:
+- String representations of (U)Int8/16/32/128/256.
 
-Types for which `0` is returned:
-- String representations of ordinary Float32/64 values.
-- String representations of Float values `NaN` and `Inf`.
+Unsupported arguments (return `0`):
+- String representations of ordinary Float32/64 values, including `NaN` and `Inf`.
 - String representations of binary and hexadecimal values, e.g. `SELECT toInt8OrZero('0xc0fe');`.
-- If the input value cannot be represented within the bounds of [Int8](../data-types/int-uint.md), and the result over or under flows.
+
+:::note
+If the input value cannot be represented within the bounds of [Int8](../data-types/int-uint.md), overflow or underflow of the result occurs.
+This is not considered an error.
+:::
 
 **Returned value**
 
@@ -148,15 +155,17 @@ Query:
 ``` sql
 SELECT
     toInt8OrZero('-8'),
-    toInt8OrZero('abc');
+    toInt8OrZero('abc')
+FORMAT vertical;
 ```
 
 Result:
 
 ```response
-   ┌─toInt8OrZero('-8')─┬─toInt8OrZero('abc')─┐
-1. │                 -8 │                   0 │
-   └────────────────────┴─────────────────────┘
+Row 1:
+──────
+toInt8OrZero('-8'):  -8
+toInt8OrZero('abc'): 0
 ```
 
 **See also**
@@ -167,7 +176,7 @@ Result:
 
 ## toInt8OrNull
 
-Like [`toInt8`](#toint8), takes an argument of type [String](../data-types/string.md) and tries to parse it to type [`Int8`](../data-types/int-uint.md). If unsuccessful, returns [`NULL`](../data-types/nullable.md).
+Like [`toInt8`](#toint8), this function converts an input value to a value of type [Int8](../data-types/int-uint.md) but returns `NULL` in case of an error.
 
 **Syntax**
 
@@ -179,14 +188,17 @@ toInt8OrNull(x)
 
 - `x` — A String representation of a number. [String](../data-types/string.md).
 
-Supported types:
-- String representations of (U)Int8/16/32/128/256
+Supported arguments:
+- String representations of (U)Int8/16/32/128/256.
 
-Types for which `\N` is returned:
-- String representations of ordinary Float32/64 values.
-- String representations of Float values `NaN` and `Inf`.
+Unsupported arguments (return `\N`)
+- String representations of Float32/64 values, including `NaN` and `Inf`.
 - String representations of binary and hexadecimal values, e.g. `SELECT toInt8OrNull('0xc0fe');`.
-- If the input value cannot be represented within the bounds of [Int8](../data-types/int-uint.md), and the result over or under flows.
+
+:::note
+If the input value cannot be represented within the bounds of [Int8](../data-types/int-uint.md), overflow or underflow of the result occurs.
+This is not considered an error.
+:::
 
 **Returned value**
 
@@ -201,15 +213,19 @@ The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding
 Query:
 
 ``` sql
-SELECT toInt8OrNull('-8'), toInt8OrNull('abc');
+SELECT
+    toInt8OrNull('-8'),
+    toInt8OrNull('abc')
+FORMAT vertical;
 ```
 
 Result:
 
 ```response
-   ┌─toInt8OrNull('-8')─┬─toInt8OrNull('abc')─┐
-1. │                 -8 │                ᴺᵁᴸᴸ │
-   └────────────────────┴─────────────────────┘
+Row 1:
+──────
+toInt8OrNull('-8'):  -8
+toInt8OrNull('abc'): ᴺᵁᴸᴸ
 ```
 
 **See also**
@@ -233,15 +249,19 @@ toInt8OrDefault(expr, def)
 - `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
 - `def` — The default value to return if parsing to type `Int8` is unsuccessful. [Int8](../data-types/int-uint.md).
 
-Supported types:
-- (U)Int8/16/32/64/128/256
-- Float*
-- String representations of (U)Int8/16/32/128/256
+Supported arguments:
+- Values of type (U)Int8/16/32/64/128/256.
+- Values of type Float32/64.
+- String representations of (U)Int8/16/32/128/256.
 
-Types for which the default value is returned:
-- Float values `NaN` and `Inf` return the default value.
-- String representations of binary and hexadecimal values, e.g. `SELECT toInt8OrDefault('0xc0fe', CAST('-1', 'Int8'));`
-- If the input value cannot be represented within the bounds of [Int8](../data-types/int-uint.md) and the result over or under flows.
+Arguments for which the default value is returned:
+- String representations of Float32/64 values, including `NaN` and `Inf`.
+- String representations of binary and hexadecimal values, e.g. `SELECT toInt8OrDefault('0xc0fe', CAST('-1', 'Int8'));`.
+
+:::note
+If the input value cannot be represented within the bounds of [Int8](../data-types/int-uint.md), overflow or underflow of the result occurs.
+This is not considered an error.
+:::
 
 **Returned value**
 
@@ -259,15 +279,17 @@ Query:
 ``` sql
 SELECT
     toInt8OrDefault('-8', CAST('-1', 'Int8')),
-    toInt8OrDefault('abc', CAST('-1', 'Int8'));
+    toInt8OrDefault('abc', CAST('-1', 'Int8'))
+FORMAT vertical;
 ```
 
 Result:
 
 ```response
-   ┌─toInt8OrDefault('-8', CAST('-1', 'Int8'))─┬─toInt8OrDefault('abc', CAST('-1', 'Int8'))─┐
-1. │                                        -8 │                                         -1 │
-   └───────────────────────────────────────────┴────────────────────────────────────────────┘
+Row 1:
+──────
+toInt8OrDefault('-8', CAST('-1', 'Int8')):  -8
+toInt8OrDefault('abc', CAST('-1', 'Int8')): -1
 ```
 
 **See also**
@@ -290,18 +312,19 @@ toInt16(expr)
 
 - `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions).
 
-Supported types:
-- (U)Int8/16/32/64/128/256
-- Float*
-- String representations of (U)Int8/16/32/128/256
+Supported arguments:
+- Values of type (U)Int8/16/32/64/128/256.
+- Values of type Float32/64.
+- String representations of (U)Int8/16/32/128/256.
 
-Unsupported types:
-- Float values `NaN` and `Inf` throw an exception.
-- String representations of binary and hexadecimal values, e.g. `SELECT toInt16('0xc0fe');`
+Unsupported arguments:
+- String representations of Float32/64 values, including `NaN` and `Inf`.
+- String representations of binary and hexadecimal values, e.g. `SELECT toInt16('0xc0fe');`.
 
 :::note
-If the input value cannot be represented within the bounds of [Int16](../data-types/int-uint.md), the result over or under flows. This is not considered an error.  
-For example: `SELECT toInt16(32768) == -32768;`, `SELECT toInt16(32768) == -32768;`,  `SELECT toInt16('32768') == -32768;`.
+If the input value cannot be represented within the bounds of [Int16](../data-types/int-uint.md), overflow or underflow of the result occurs.
+This is not considered an error.  
+For example: `SELECT toInt16(32768) == -32768;`.
 :::
 
 **Returned value**
@@ -320,15 +343,18 @@ Query:
 SELECT
     toInt16(-16),
     toInt16(-16.16),
-    toInt16('-16');
+    toInt16('-16')
+FORMAT vertical;
 ```
 
 Result:
 
 ```response
-   ┌─toInt16(-16)─┬─toInt16(-16.16)─┬─toInt16('-16')─┐
-1. │          -16 │             -16 │            -16 │
-   └──────────────┴─────────────────┴────────────────┘
+Row 1:
+──────
+toInt16(-16):    -16
+toInt16(-16.16): -16
+toInt16('-16'):  -16
 ```
 
 **See also**
@@ -351,14 +377,17 @@ toInt16OrZero(x)
 
 - `x` — A String representation of a number. [String](../data-types/string.md).
 
-Supported types:
-- String representations of (U)Int8/16/32/128/256
+Supported arguments:
+- String representations of (U)Int8/16/32/128/256.
 
-Types for which `0` is returned:
-- String representations of ordinary Float32/64 values.
-- String representations of Float values `NaN` and `Inf`.
+Unsupported arguments (return `0`):
+- String representations of Float32/64 values, including `NaN` and `Inf`.
 - String representations of binary and hexadecimal values, e.g. `SELECT toInt16OrZero('0xc0fe');`.
-- If the input value cannot be represented within the bounds of [Int16](../data-types/int-uint.md) and the result over or under flows.
+
+:::note
+If the input value cannot be represented within the bounds of [Int16](../data-types/int-uint.md), overflow or underflow of the result occurs.
+This is not considered as an error.
+:::
 
 **Returned value**
 
@@ -375,15 +404,17 @@ Query:
 ``` sql
 SELECT
     toInt16OrZero('-16'),
-    toInt16OrZero('abc');
+    toInt16OrZero('abc')
+FORMAT vertical;
 ```
 
 Result:
 
 ```response
-   ┌─toInt16OrZero('-16')─┬─toInt16OrZero('abc')─┐
-1. │                  -16 │                    0 │
-   └──────────────────────┴──────────────────────┘
+Row 1:
+──────
+toInt16OrZero('-16'): -16
+toInt16OrZero('abc'): 0
 ```
 
 **See also**
@@ -394,7 +425,7 @@ Result:
 
 ## toInt16OrNull
 
-Like [`toInt16`](#toint16), takes an argument of type [String](../data-types/string.md) and tries to parse it to type [`Int16`](../data-types/int-uint.md). If unsuccessful, returns [`NULL`](../data-types/nullable.md).
+Like [`toInt16`](#toint16), this function converts an input value to a value of type [Int16](../data-types/int-uint.md) but returns `NULL` in case of an error.
 
 **Syntax**
 
@@ -406,14 +437,17 @@ toInt16OrNull(x)
 
 - `x` — A String representation of a number. [String](../data-types/string.md).
 
-Supported types:
-- String representations of (U)Int8/16/32/128/256
+Supported arguments:
+- String representations of (U)Int8/16/32/128/256.
 
-Types for which `\N` is returned:
-- String representations of ordinary Float32/64 values.
-- String representations of Float values `NaN` and `Inf`.
+Unsupported arguments (return `\N`)
+- String representations of Float32/64 values, including `NaN` and `Inf`.
 - String representations of binary and hexadecimal values, e.g. `SELECT toInt16OrNull('0xc0fe');`.
-- If the input value cannot be represented within the bounds of [Int16](../data-types/int-uint.md) and the result over or under flows.
+
+:::note
+If the input value cannot be represented within the bounds of [Int16](../data-types/int-uint.md), overflow or underflow of the result occurs. 
+This is not considered an error.
+:::
 
 **Returned value**
 
@@ -430,15 +464,17 @@ Query:
 ``` sql
 SELECT
     toInt16OrNull('-16'),
-    toInt16OrNull('abc');
+    toInt16OrNull('abc')
+FORMAT vertical;
 ```
 
 Result:
 
 ```response
-   ┌─toInt16OrNull('-16')─┬─toInt16OrNull('abc')─┐
-1. │                  -16 │                 ᴺᵁᴸᴸ │
-   └──────────────────────┴──────────────────────┘
+Row 1:
+──────
+toInt16OrNull('-16'): -16
+toInt16OrNull('abc'): ᴺᵁᴸᴸ
 ```
 
 **See also**
@@ -449,7 +485,7 @@ Result:
 
 ## toInt16OrDefault
 
-Like [`toInt16`](#toint16), takes an argument of type [String](../data-types/string.md) and tries to parse it to type [`Int16`](../data-types/int-uint.md). If unsuccessful, returns the default type value.
+Like [`toInt16`](#toint16), this function converts an input value to a value of type [Int16](../data-types/int-uint.md) but returns the default value in case of an error.
 
 **Syntax**
 
@@ -462,15 +498,19 @@ toInt16OrDefault(expr, def)
 - `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
 - `def` — The default value to return if parsing to type `Int16` is unsuccessful. [Int8](../data-types/int-uint.md).
 
-Supported types:
-- (U)Int8/16/32/64/128/256
-- Float*
-- String representations of (U)Int8/16/32/128/256
+Supported arguments:
+- Values of type (U)Int8/16/32/64/128/256.
+- Values of type Float32/64.
+- String representations of (U)Int8/16/32/128/256.
 
-Types for which the default value is returned:
-- Float values `NaN` and `Inf` return the default value.
-- String representations of binary and hexadecimal values, e.g. `SELECT toInt16OrDefault('0xc0fe', CAST('-1', 'Int16'));`
-- If the input value cannot be represented within the bounds of [Int16](../data-types/int-uint.md) and the result over or under flows.
+Arguments for which the default value is returned:
+- String representations of Float32/64 values, including `NaN` and `Inf`.
+- String representations of binary and hexadecimal values, e.g. `SELECT toInt16OrDefault('0xc0fe', CAST('-1', 'Int16'));`.
+
+:::note
+If the input value cannot be represented within the bounds of [Int16](../data-types/int-uint.md), overflow or underflow of the result occurs. 
+This is not considered an error.
+:::
 
 **Returned value**
 
@@ -486,15 +526,19 @@ Types for which the default value is returned:
 Query:
 
 ``` sql
-SELECT toInt16OrDefault('-16', cast('-1' as Int16)), toInt16OrDefault('abc', cast('-1' as Int16));
+SELECT
+    toInt16OrDefault('-16', CAST('-1', 'Int16')),
+    toInt16OrDefault('abc', CAST('-1', 'Int16'))
+FORMAT vertical;
 ```
 
 Result:
 
 ```response
-   ┌─toInt16OrDefault('-16', CAST('-1', 'Int16'))─┬─toInt16OrDefault('abc', CAST('-1', 'Int16'))─┐
-1. │                                          -16 │                                           -1 │
-   └──────────────────────────────────────────────┴──────────────────────────────────────────────┘
+Row 1:
+──────
+toInt16OrDefault('-16', CAST('-1', 'Int16')): -16
+toInt16OrDefault('abc', CAST('-1', 'Int16')): -1
 ```
 
 **See also**
@@ -517,23 +561,19 @@ toInt32(expr)
 
 - `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions).
 
-Supported types:
-- (U)Int8/16/32/64/128/256
-- Float*
-- String representations of (U)Int8/16/32/128/256
+Supported arguments:
+- Values of type (U)Int8/16/32/64/128/256.
+- Values of type Float32/64.
+- String representations of (U)Int8/16/32/128/256.
 
-Unsupported types:
-- Float values `NaN` and `Inf` throw an exception.
-- String representations of binary and hexadecimal values, e.g. `SELECT toInt32('0xc0fe');`
+Unsupported arguments:
+- String representations of Float32/64 values, including `NaN` and `Inf`.
+- String representations of binary and hexadecimal values, e.g. `SELECT toInt32('0xc0fe');`.
 
 :::note
-If the input value cannot be represented within the bounds of [Int32](../data-types/int-uint.md), the result over or under flows. This is not considered an error.  
-For example: 
-```
-SELECT toInt32(2147483648) == -2147483648;
-SELECT toInt32(2147483648.0) == -2147483648;
-SELECT toInt32('2147483648') == -2147483648;
-```
+If the input value cannot be represented within the bounds of [Int32](../data-types/int-uint.md), the result over or under flows. 
+This is not considered an error.  
+For example: `SELECT toInt32(2147483648) == -2147483648;`
 :::
 
 **Returned value**
@@ -553,14 +593,17 @@ SELECT
     toInt32(-32),
     toInt32(-32.32),
     toInt32('-32')
+FORMAT vertical;
 ```
 
 Result:
 
 ```response
-   ┌─toInt32(-32)─┬─toInt32(-32.32)─┬─toInt32('-32')─┐
-1. │          -32 │             -32 │            -32 │
-   └──────────────┴─────────────────┴────────────────┘
+Row 1:
+──────
+toInt32(-32):    -32
+toInt32(-32.32): -32
+toInt32('-32'):  -32
 ```
 
 **See also**
@@ -583,15 +626,17 @@ toInt32OrZero(x)
 
 - `x` — A String representation of a number. [String](../data-types/string.md).
 
-Supported types:
-- String representations of (U)Int8/16/32/128/256
+Supported arguments:
+- String representations of (U)Int8/16/32/128/256.
 
-Types for which `0` is returned:
-- String representations of ordinary Float32/64 values.
-- String representations of Float values `NaN` and `Inf`.
+Unsupported arguments (return `0`):
+- String representations of Float32/64 values, including `NaN` and `Inf`.
 - String representations of binary and hexadecimal values, e.g. `SELECT toInt32OrZero('0xc0fe');`.
-- If the input value cannot be represented within the bounds of [Int32](../data-types/int-uint.md) and the result over or under flows.
 
+:::note
+If the input value cannot be represented within the bounds of [Int32](../data-types/int-uint.md), overflow or underflow of the result occurs. 
+This is not considered an error.
+:::
 
 **Returned value**
 
@@ -606,15 +651,19 @@ The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding
 Query:
 
 ``` sql
-SELECT toInt32OrZero('-32'), toInt32OrZero('abc');
+SELECT
+    toInt32OrZero('-32'),
+    toInt32OrZero('abc')
+FORMAT vertical;
 ```
 
 Result:
 
 ```response
-   ┌─toInt32OrZero('-32')─┬─toInt32OrZero('abc')─┐
-1. │                  -32 │                    0 │
-   └──────────────────────┴──────────────────────┘
+Row 1:
+──────
+toInt32OrZero('-32'): -32
+toInt32OrZero('abc'): 0
 ```
 **See also**
 
@@ -624,7 +673,7 @@ Result:
 
 ## toInt32OrNull
 
-Like [`toInt32`](#toint32), takes an argument of type [String](../data-types/string.md) and tries to parse it to type [`Int32`](../data-types/int-uint.md). If unsuccessful, returns [`NULL`](../data-types/nullable.md).
+Like [`toInt32`](#toint32), this function converts an input value to a value of type [Int32](../data-types/int-uint.md) but returns `NULL` in case of an error.
 
 **Syntax**
 
@@ -636,14 +685,17 @@ toInt32OrNull(x)
 
 - `x` — A String representation of a number. [String](../data-types/string.md).
 
-Supported types:
-- String representations of (U)Int8/16/32/128/256
+Supported arguments:
+- String representations of (U)Int8/16/32/128/256.
 
-Types for which `\N` is returned:
-- String representations of ordinary Float32/64 values.
-- String representations of Float values `NaN` and `Inf`.
+Unsupported arguments (return `\N`)
+- String representations of Float32/64 values, including `NaN` and `Inf`.
 - String representations of binary and hexadecimal values, e.g. `SELECT toInt32OrNull('0xc0fe');`.
-- If the input value cannot be represented within the bounds of [Int32](../data-types/int-uint.md) and the result over or under flows.
+
+:::note
+If the input value cannot be represented within the bounds of [Int32](../data-types/int-uint.md), overflow or underflow of the result occurs. 
+This is not considered an error.
+:::
 
 **Returned value**
 
@@ -658,15 +710,19 @@ The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding
 Query:
 
 ``` sql
-SELECT toInt32OrNull('-32'), toInt32OrNull('abc');
+SELECT
+    toInt32OrNull('-32'),
+    toInt32OrNull('abc')
+FORMAT vertical;
 ```
 
 Result:
 
 ```response
-   ┌─toInt32OrNull('-32')─┬─toInt32OrNull('abc')─┐
-1. │                  -32 │                 ᴺᵁᴸᴸ │
-   └──────────────────────┴──────────────────────┘
+Row 1:
+──────
+toInt32OrNull('-32'): -32
+toInt32OrNull('abc'): ᴺᵁᴸᴸ
 ```
 
 **See also**
@@ -677,7 +733,7 @@ Result:
 
 ## toInt32OrDefault
 
-Like [`toInt32`](#toint32), takes an argument of type [String](../data-types/string.md) and tries to parse it to type [`Int32`](../data-types/int-uint.md). If unsuccessful, returns the default type value.
+Like [`toInt32`](#toint32), this function converts an input value to a value of type [Int32](../data-types/int-uint.md) but returns the default value in case of an error.
 
 **Syntax**
 
@@ -690,15 +746,19 @@ toInt32OrDefault(expr, def)
 - `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
 - `def` — The default value to return if parsing to type `Int32` is unsuccessful. [Int32](../data-types/int-uint.md).
 
-Supported types:
-- (U)Int8/16/32/64/128/256
-- Float*
-- String representations of (U)Int8/16/32/128/256
+Supported arguments:
+- Values of type (U)Int8/16/32/64/128/256.
+- Values of type Float32/64.
+- String representations of (U)Int8/16/32/128/256.
 
-Types for which the default value is returned:
-- Float values `NaN` and `Inf` return the default value.
-- String representations of binary and hexadecimal values, e.g. `SELECT toInt32OrDefault('0xc0fe', CAST('-1', 'Int32'));`
-- If the input value cannot be represented within the bounds of [Int32](../data-types/int-uint.md) and the result over or under flows.
+Arguments for which the default value is returned:
+- String representations of Float32/64 values, including `NaN` and `Inf`.
+- String representations of binary and hexadecimal values, e.g. `SELECT toInt32OrDefault('0xc0fe', CAST('-1', 'Int32'));`.
+
+:::note
+If the input value cannot be represented within the bounds of [Int32](../data-types/int-uint.md), overflow or underflow of the result occurs. 
+This is not considered an error.
+:::
 
 **Returned value**
 
@@ -714,15 +774,19 @@ Types for which the default value is returned:
 Query:
 
 ``` sql
-SELECT toInt32OrDefault('-32', cast('-1' as Int32)), toInt32OrDefault('abc', cast('-1' as Int32));
+SELECT
+    toInt32OrDefault('-32', CAST('-1', 'Int32')),
+    toInt32OrDefault('abc', CAST('-1', 'Int32'))
+FORMAT vertical;
 ```
 
 Result:
 
 ```response
-   ┌─toInt32OrDefault('-32', CAST('-1', 'Int32'))─┬─toInt32OrDefault('abc', CAST('-1', 'Int32'))─┐
-1. │                                          -32 │                                           -1 │
-   └──────────────────────────────────────────────┴──────────────────────────────────────────────┘
+Row 1:
+──────
+toInt32OrDefault('-32', CAST('-1', 'Int32')): -32
+toInt32OrDefault('abc', CAST('-1', 'Int32')): -1
 ```
 
 **See also**
@@ -745,24 +809,19 @@ toInt64(expr)
 
 - `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions).
 
-Supported types:
-- (U)Int8/16/32/64/128/256
-- Float*
-- String representations of (U)Int8/16/32/128/256
+Supported arguments:
+- Values of type (U)Int8/16/32/64/128/256.
+- Values of type Float32/64.
+- String representations of (U)Int8/16/32/128/256.
 
 Unsupported types:
-- Float values `NaN` and `Inf` throw an exception.
-- String representations of binary and hexadecimal values, e.g. `SELECT toInt64('0xc0fe');`
+- String representations of Float32/64 values, including `NaN` and `Inf`.
+- String representations of binary and hexadecimal values, e.g. `SELECT toInt64('0xc0fe');`.
 
 :::note
-If the input value cannot be represented within the bounds of [Int64](../data-types/int-uint.md), the result over or under flows. This is not considered an error.  
-For example: 
-
-```
-SELECT toInt64(9223372036854775808) == -9223372036854775808;
-SELECT toInt64(9223372036854775808.0) == -9223372036854775808;
-SELECT toInt64('9223372036854775808') == --9223372036854775808;
-```
+If the input value cannot be represented within the bounds of [Int64](../data-types/int-uint.md), the result over or under flows. 
+This is not considered an error.  
+For example: `SELECT toInt64(9223372036854775808) == -9223372036854775808;`
 :::
 
 **Returned value**
@@ -781,15 +840,18 @@ Query:
 SELECT
     toInt64(-64),
     toInt64(-64.64),
-    toInt64('-64');
+    toInt64('-64')
+FORMAT vertical;
 ```
 
 Result:
 
 ```response
-   ┌─toInt64(-64)─┬─toInt64(-64.64)─┬─toInt64('-64')─┐
-1. │          -64 │             -64 │            -64 │
-   └──────────────┴─────────────────┴────────────────┘
+Row 1:
+──────
+toInt64(-64):    -64
+toInt64(-64.64): -64
+toInt64('-64'):  -64
 ```
 
 **See also**
@@ -812,14 +874,17 @@ toInt64OrZero(x)
 
 - `x` — A String representation of a number. [String](../data-types/string.md).
 
-Supported types:
-- String representations of (U)Int8/16/32/128/256
+Supported arguments:
+- String representations of (U)Int8/16/32/128/256.
 
-Types for which `0` is returned:
-- String representations of ordinary Float32/64 values.
-- String representations of Float values `NaN` and `Inf`.
+Unsupported arguments (return `0`):
+- String representations of Float32/64 values, including `NaN` and `Inf`.
 - String representations of binary and hexadecimal values, e.g. `SELECT toInt64OrZero('0xc0fe');`.
-- If the input value cannot be represented within the bounds of [Int64](../data-types/int-uint.md) and the result over or under flows.
+
+:::note
+If the input value cannot be represented within the bounds of [Int64](../data-types/int-uint.md), overflow or underflow of the result occurs. 
+This is not considered an error.
+:::
 
 **Returned value**
 
@@ -836,15 +901,17 @@ Query:
 ``` sql
 SELECT
     toInt64OrZero('-64'),
-    toInt64OrZero('abc');
+    toInt64OrZero('abc')
+FORMAT vertical;
 ```
 
 Result:
 
 ```response
-   ┌─toInt64OrZero('-64')─┬─toInt64OrZero('abc')─┐
-1. │                  -64 │                    0 │
-   └──────────────────────┴──────────────────────┘
+Row 1:
+──────
+toInt64OrZero('-64'): -64
+toInt64OrZero('abc'): 0
 ```
 
 **See also**
@@ -855,7 +922,7 @@ Result:
 
 ## toInt64OrNull
 
-Like [`toInt64`], takes an argument of type [String](../data-types/string.md) and tries to parse it to type [`Int64`](../data-types/nullable.md). If unsuccessful, returns [`NULL`](../data-types/nullable.md).
+Like [`toInt64`](#toint64), this function converts an input value to a value of type [Int64](../data-types/int-uint.md) but returns `NULL` in case of an error.
 
 **Syntax**
 
@@ -867,14 +934,17 @@ toInt64OrNull(x)
 
 - `x` — A String representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
 
-Supported types:
-- String representations of (U)Int8/16/32/128/256
+Supported arguments:
+- String representations of (U)Int8/16/32/128/256.
 
-Types for which `\N` is returned:
-- String representations of ordinary Float32/64 values.
-- String representations of Float values `NaN` and `Inf`.
+Unsupported arguments (return `\N`)
+- String representations of Float32/64 values, including `NaN` and `Inf`.
 - String representations of binary and hexadecimal values, e.g. `SELECT toInt64OrNull('0xc0fe');`.
-- If the input value cannot be represented within the bounds of [Int64](../data-types/int-uint.md) and the result over or under flows.
+
+:::note
+If the input value cannot be represented within the bounds of [Int64](../data-types/int-uint.md), overflow or underflow of the result occurs. 
+This is not considered an error.
+:::
 
 **Returned value**
 
@@ -891,15 +961,17 @@ Query:
 ``` sql
 SELECT
     toInt64OrNull('-64'),
-    toInt64OrNull('abc');
+    toInt64OrNull('abc')
+FORMAT vertical;
 ```
 
 Result:
 
 ```response
-   ┌─toInt64OrNull('-64')─┬─toInt64OrNull('abc')─┐
-1. │                  -64 │                 ᴺᵁᴸᴸ │
-   └──────────────────────┴──────────────────────┘
+Row 1:
+──────
+toInt64OrNull('-64'): -64
+toInt64OrNull('abc'): ᴺᵁᴸᴸ
 ```
 
 **See also**
@@ -910,7 +982,7 @@ Result:
 
 ## toInt64OrDefault
 
-Like [`toInt64`](#toint64), takes an argument of type [String](../data-types/string.md) and tries to parse it to type [`Int64`](../data-types/nullable.md). If unsuccessful, returns the default type value.
+Like [`toInt64`](#toint64), this function converts an input value to a value of type [Int64](../data-types/int-uint.md) but returns the default value in case of an error.
 
 **Syntax**
 
@@ -923,15 +995,19 @@ toInt64OrDefault(expr, def)
 - `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
 - `def` — The default value to return if parsing to type `Int64` is unsuccessful. [Int64](../data-types/int-uint.md).
 
-Supported types:
-- (U)Int8/16/32/64/128/256
-- Float*
-- String representations of (U)Int8/16/32/128/256
+Supported arguments:
+- Values of type (U)Int8/16/32/64/128/256.
+- Values of type Float32/64.
+- String representations of (U)Int8/16/32/128/256.
 
-Types for which the default value is returned:
-- Float values `NaN` and `Inf` return the default value.
-- String representations of binary and hexadecimal values, e.g. `SELECT toInt64OrDefault('0xc0fe', CAST('-1', 'Int64'));`
-- If the input value cannot be represented within the bounds of [Int64](../data-types/int-uint.md) and the result over or under flows.
+Arguments for which the default value is returned:
+- String representations of Float32/64 values, including `NaN` and `Inf`.
+- String representations of binary and hexadecimal values, e.g. `SELECT toInt64OrDefault('0xc0fe', CAST('-1', 'Int64'));`.
+
+:::note
+If the input value cannot be represented within the bounds of [Int64](../data-types/int-uint.md), overflow or underflow of the result occurs.
+This is not considered an error.
+:::
 
 **Returned value**
 
@@ -949,15 +1025,17 @@ Query:
 ``` sql
 SELECT
     toInt64OrDefault('-64', CAST('-1', 'Int64')),
-    toInt64OrDefault('abc', CAST('-1', 'Int64'));
+    toInt64OrDefault('abc', CAST('-1', 'Int64'))
+FORMAT vertical;
 ```
 
 Result:
 
 ```response
-   ┌─toInt64OrDefault('-64', CAST('-1', 'Int64'))─┬─toInt64OrDefault('abc', CAST('-1', 'Int64'))─┐
-1. │                                          -64 │                                           -1 │
-   └──────────────────────────────────────────────┴──────────────────────────────────────────────┘
+Row 1:
+──────
+toInt64OrDefault('-64', CAST('-1', 'Int64')): -64
+toInt64OrDefault('abc', CAST('-1', 'Int64')): -1
 ```
 
 **See also**
@@ -980,17 +1058,18 @@ toInt128(expr)
 
 - `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions).
 
-Supported types:
-- (U)Int8/16/32/64/128/256
-- Float*
-- String representations of (U)Int8/16/32/128/256
+Supported arguments:
+- Values of type (U)Int8/16/32/64/128/256.
+- Values of type Float32/64.
+- String representations of (U)Int8/16/32/128/256.
 
-Unsupported types:
-- Float values `NaN` and `Inf` throw an exception.
-- String representations of binary and hexadecimal values, e.g. `SELECT toInt128('0xc0fe');`
+Unsupported arguments:
+- String representations of Float32/64 values, including `NaN` and `Inf`.
+- String representations of binary and hexadecimal values, e.g. `SELECT toInt128('0xc0fe');`.
 
 :::note
-If the input value cannot be represented within the bounds of [Int128](../data-types/int-uint.md), the result over or under flows. This is not considered an error.
+If the input value cannot be represented within the bounds of [Int128](../data-types/int-uint.md), the result over or under flows. 
+This is not considered an error.
 :::
 
 **Returned value**
@@ -1009,15 +1088,18 @@ Query:
 SELECT
     toInt128(-128),
     toInt128(-128.8),
-    toInt128('-128'),
+    toInt128('-128')
+FORMAT vertical;
 ```
 
 Result:
 
 ```response
-   ┌─toInt128(-128)─┬─toInt128(-128.8)─┬─toInt128('-128')─┐
-1. │           -128 │             -128 │             -128 │
-   └────────────────┴──────────────────┴──────────────────┘
+Row 1:
+──────
+toInt128(-128):   -128
+toInt128(-128.8): -128
+toInt128('-128'): -128
 ```
 
 **See also**
@@ -1040,14 +1122,17 @@ toInt128OrZero(expr)
 
 - `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
 
-Supported types:
-- String representations of (U)Int8/16/32/128/256
+Supported arguments:
+- String representations of (U)Int8/16/32/128/256.
 
-Types for which `0` is returned:
-- String representations of ordinary Float32/64 values.
-- String representations of Float values `NaN` and `Inf`.
+Unsupported arguments (return `0`):
+- String representations of Float32/64 values, including `NaN` and `Inf`.
 - String representations of binary and hexadecimal values, e.g. `SELECT toInt128OrZero('0xc0fe');`.
-- If the input value cannot be represented within the bounds of [Int128](../data-types/int-uint.md) and the result over or under flows.
+
+:::note
+If the input value cannot be represented within the bounds of [Int128](../data-types/int-uint.md), overflow or underflow of the result occurs.
+This is not considered an error.
+:::
 
 **Returned value**
 
@@ -1064,15 +1149,17 @@ Query:
 ``` sql
 SELECT
     toInt128OrZero('-128'),
-    toInt128OrZero('abc');
+    toInt128OrZero('abc')
+FORMAT vertical;
 ```
 
 Result:
 
 ```response
-   ┌─toInt128OrZero('-128')─┬─toInt128OrZero('abc')─┐
-1. │                   -128 │                     0 │
-   └────────────────────────┴───────────────────────┘
+Row 1:
+──────
+toInt128OrZero('-128'): -128
+toInt128OrZero('abc'):  0
 ```
 
 **See also**
@@ -1083,7 +1170,7 @@ Result:
 
 ## toInt128OrNull
 
-Like [`toInt128`](#toint128), takes an argument of type [String](../data-types/string.md) and tries to parse it to type [`Int128`](../data-types/int-uint.md). If unsuccessful, returns [`NULL`](../data-types/nullable.md).
+Like [`toInt128`](#toint128), this function converts an input value to a value of type [Int128](../data-types/int-uint.md) but returns `NULL` in case of an error.
 
 **Syntax**
 
@@ -1095,14 +1182,17 @@ toInt128OrNull(x)
 
 - `x` — A String representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
 
-Supported types:
-- String representations of (U)Int8/16/32/128/256
+Supported arguments:
+- String representations of (U)Int8/16/32/128/256.
 
-Types for which `\N` is returned:
-- String representations of ordinary Float32/64 values.
-- String representations of Float values `NaN` and `Inf`.
+Unsupported arguments (return `\N`)
+- String representations of Float32/64 values, including `NaN` and `Inf`.
 - String representations of binary and hexadecimal values, e.g. `SELECT toInt128OrNull('0xc0fe');`.
-- If the input value cannot be represented within the bounds of [Int128](../data-types/int-uint.md) and the result over or under flows.
+
+:::note
+If the input value cannot be represented within the bounds of [Int128](../data-types/int-uint.md), overflow or underflow of the result occurs. 
+This is not considered an error.
+:::
 
 **Returned value**
 
@@ -1119,15 +1209,17 @@ Query:
 ``` sql
 SELECT
     toInt128OrNull('-128'),
-    toInt128OrNull('abc');
+    toInt128OrNull('abc')
+FORMAT vertical;
 ```
 
 Result:
 
 ```response
-   ┌─toInt128OrNull('-128')─┬─toInt128OrNull('abc')─┐
-1. │                   -128 │                  ᴺᵁᴸᴸ │
-   └────────────────────────┴───────────────────────┘
+Row 1:
+──────
+toInt128OrNull('-128'): -128
+toInt128OrNull('abc'):  ᴺᵁᴸᴸ
 ```
 
 **See also**
@@ -1138,7 +1230,7 @@ Result:
 
 ## toInt128OrDefault
 
-Like [`toInt128`](#toint128), takes an argument of type [String](../data-types/string.md) and tries to parse it to type [`Int128`](../data-types/int-uint.md). If unsuccessful, returns the default type value.
+Like [`toInt128`](#toint128), this function converts an input value to a value of type [Int128](../data-types/int-uint.md) but returns the default value in case of an error.
 
 **Syntax**
 
@@ -1151,15 +1243,19 @@ toInt128OrDefault(expr, def)
 - `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
 - `def` — The default value to return if parsing to type `Int128` is unsuccessful. [Int128](../data-types/int-uint.md).
 
-Supported types:
-- (U)Int8/16/32/64/128/256
-- Float*
-- String representations of (U)Int8/16/32/128/256
+Supported arguments:
+- (U)Int8/16/32/64/128/256.
+- Float32/64.
+- String representations of (U)Int8/16/32/128/256.
 
-Types for which the default value is returned:
-- Float values `NaN` and `Inf` return the default value.
-- String representations of binary and hexadecimal values, e.g. `SELECT toInt128OrDefault('0xc0fe', CAST('-1', 'Int128'));`
-- If the input value cannot be represented within the bounds of [Int128](../data-types/int-uint.md) and the result over or under flows.
+Arguments for which the default value is returned:
+- String representations of Float32/64 values, including `NaN` and `Inf`.
+- String representations of binary and hexadecimal values, e.g. `SELECT toInt128OrDefault('0xc0fe', CAST('-1', 'Int128'));`.
+
+:::note
+If the input value cannot be represented within the bounds of [Int128](../data-types/int-uint.md), overflow or underflow of the result occurs. 
+This is not considered an error.
+:::
 
 **Returned value**
 
@@ -1177,15 +1273,17 @@ Query:
 ``` sql
 SELECT
     toInt128OrDefault('-128', CAST('-1', 'Int128')),
-    toInt128OrDefault('abc', CAST('-1', 'Int128'));
+    toInt128OrDefault('abc', CAST('-1', 'Int128'))
+FORMAT vertical;
 ```
 
 Result:
 
 ```response
-   ┌─toInt128OrDefault('-128', CAST('-1', 'Int128'))─┬─toInt128OrDefault('abc', CAST('-1', 'Int128'))─┐
-1. │                                            -128 │                                             -1 │
-   └─────────────────────────────────────────────────┴────────────────────────────────────────────────┘
+Row 1:
+──────
+toInt128OrDefault('-128', CAST('-1', 'Int128')): -128
+toInt128OrDefault('abc', CAST('-1', 'Int128')):  -1
 ```
 
 **See also**
@@ -1208,17 +1306,18 @@ toInt256(expr)
 
 - `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions).
 
-Supported types:
-- (U)Int8/16/32/64/128/256
-- Float*
-- String representations of (U)Int8/16/32/128/256
+Supported arguments:
+- Values of type (U)Int8/16/32/64/128/256.
+- Values of type Float32/64.
+- String representations of (U)Int8/16/32/128/256.
 
-Unsupported types:
-- Float values `NaN` and `Inf` throw an exception.
-- String representations of binary and hexadecimal values, e.g. `SELECT toInt256('0xc0fe');`
+Unsupported arguments:
+- String representations of Float32/64 values, including `NaN` and `Inf`.
+- String representations of binary and hexadecimal values, e.g. `SELECT toInt256('0xc0fe');`.
 
 :::note
-If the input value cannot be represented within the bounds of [Int256](../data-types/int-uint.md), the result over or under flows. This is not considered an error.
+If the input value cannot be represented within the bounds of [Int256](../data-types/int-uint.md), the result over or under flows. 
+This is not considered an error.
 :::
 
 **Returned value**
@@ -1237,15 +1336,18 @@ Query:
 SELECT
     toInt256(-256),
     toInt256(-256.256),
-    toInt256('-256');
+    toInt256('-256')
+FORMAT vertical;
 ```
 
 Result:
 
 ```response
-   ┌─toInt256(-256)─┬─toInt256(-256.256)─┬─toInt256('-256')─┐
-1. │           -256 │               -256 │             -256 │
-   └────────────────┴────────────────────┴──────────────────┘
+Row 1:
+──────
+toInt256(-256):     -256
+toInt256(-256.256): -256
+toInt256('-256'):   -256
 ```
 
 **See also**
@@ -1268,14 +1370,17 @@ toInt256OrZero(x)
 
 - `x` — A String representation of a number. [String](../data-types/string.md).
 
-Supported types:
-- String representations of (U)Int8/16/32/128/256
+Supported arguments:
+- String representations of (U)Int8/16/32/128/256.
 
-Types for which `0` is returned:
-- String representations of ordinary Float32/64 values.
-- String representations of Float values `NaN` and `Inf`.
+Unsupported arguments (return `0`):
+- String representations of Float32/64 values, including `NaN` and `Inf`.
 - String representations of binary and hexadecimal values, e.g. `SELECT toInt256OrZero('0xc0fe');`.
-- If the input value cannot be represented within the bounds of [Int256](../data-types/int-uint.md) and the result over or under flows.
+
+:::note
+If the input value cannot be represented within the bounds of [Int256](../data-types/int-uint.md), overflow or underflow of the result occurs. 
+This is not considered an error.
+:::
 
 **Returned value**
 
@@ -1292,15 +1397,17 @@ Query:
 ``` sql
 SELECT
     toInt256OrZero('-256'),
-    toInt256OrZero('abc');
+    toInt256OrZero('abc')
+FORMAT vertical;
 ```
 
 Result:
 
 ```response
-   ┌─toInt256OrZero('-256')─┬─toInt256OrZero('abc')─┐
-1. │                   -256 │                     0 │
-   └────────────────────────┴───────────────────────┘
+Row 1:
+──────
+toInt256OrZero('-256'): -256
+toInt256OrZero('abc'):  0
 ```
 
 **See also**
@@ -1311,7 +1418,7 @@ Result:
 
 ## toInt256OrNull
 
-Like [`toInt256`](#toint256), takes an argument of type [String](../data-types/string.md) and tries to parse it to type [`Int256`](../data-types/int-uint.md). If unsuccessful, returns [`NULL`](../data-types/nullable.md).
+Like [`toInt256`](#toint256), this function converts an input value to a value of type [Int256](../data-types/int-uint.md) but returns `NULL` in case of an error.
 
 **Syntax**
 
@@ -1323,14 +1430,17 @@ toInt256OrNull(x)
 
 - `x` — A String representation of a number. [String](../data-types/string.md).
 
-Supported types:
-- String representations of (U)Int8/16/32/128/256
+Supported arguments:
+- String representations of (U)Int8/16/32/128/256.
 
-Types for which `\N` is returned:
-- String representations of ordinary Float32/64 values.
-- String representations of Float values `NaN` and `Inf`.
+Unsupported arguments (return `\N`)
+- String representations of Float32/64 values, including `NaN` and `Inf`.
 - String representations of binary and hexadecimal values, e.g. `SELECT toInt256OrNull('0xc0fe');`.
-- If the input value cannot be represented within the bounds of [Int256](../data-types/int-uint.md) and the result over or under flows.
+
+:::note
+If the input value cannot be represented within the bounds of [Int256](../data-types/int-uint.md), overflow or underflow of the result occurs. 
+This is not considered an error.
+:::
 
 **Returned value**
 
@@ -1347,15 +1457,17 @@ Query:
 ``` sql
 SELECT
     toInt256OrNull('-256'),
-    toInt256OrNull('abc');
+    toInt256OrNull('abc')
+FORMAT vertical;
 ```
 
 Result:
 
 ```response
-   ┌─toInt256OrNull('-256')─┬─toInt256OrNull('abc')─┐
-1. │                   -256 │                  ᴺᵁᴸᴸ  │
-   └────────────────────────┴───────────────────────┘
+Row 1:
+──────
+toInt256OrNull('-256'): -256
+toInt256OrNull('abc'):  ᴺᵁᴸᴸ
 ```
 
 **See also**
@@ -1366,7 +1478,7 @@ Result:
 
 ## toInt256OrDefault
 
-Like [`toInt256`](#toint256), takes an argument of type [String](../data-types/string.md) and tries to parse it to type [`Int256`](../data-types/int-uint.md). If unsuccessful, returns the default type value.
+Like [`toInt256`](#toint256), this function converts an input value to a value of type [Int256](../data-types/int-uint.md) but returns the default value in case of an error.
 
 **Syntax**
 
@@ -1379,15 +1491,19 @@ toInt256OrDefault(expr, def)
 - `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
 - `def` — The default value to return if parsing to type `Int256` is unsuccessful. [Int256](../data-types/int-uint.md).
 
-Supported types:
-- (U)Int8/16/32/64/128/256
-- Float*
-- String representations of (U)Int8/16/32/128/256
+Supported arguments:
+- Values of type (U)Int8/16/32/64/128/256.
+- Values of type Float32/64.
+- String representations of (U)Int8/16/32/128/256.
 
-Types for which the default value is returned:
-- Float values `NaN` and `Inf` return the default value.
+Arguments for which the default value is returned:
+- String representations of Float32/64 values, including `NaN` and `Inf`
 - String representations of binary and hexadecimal values, e.g. `SELECT toInt256OrDefault('0xc0fe', CAST('-1', 'Int256'));`
-- If the input value cannot be represented within the bounds of [Int256](../data-types/int-uint.md) and the result over or under flows.
+
+:::note
+If the input value cannot be represented within the bounds of [Int256](../data-types/int-uint.md), overflow or underflow of the result occurs.
+This is not considered an error.
+:::
 
 **Returned value**
 
@@ -1405,15 +1521,17 @@ Query:
 ``` sql
 SELECT
     toInt256OrDefault('-256', CAST('-1', 'Int256')),
-    toInt256OrDefault('abc', CAST('-1', 'Int256'));
+    toInt256OrDefault('abc', CAST('-1', 'Int256'))
+FORMAT vertical;
 ```
 
 Result:
 
 ```response
-   ┌─toInt256OrDefault('-256', CAST('-1', 'Int256'))─┬─toInt256OrDefault('abc', CAST('-1', 'Int256'))─┐
-1. │                                            -256 │                                             -1 │
-   └─────────────────────────────────────────────────┴────────────────────────────────────────────────┘
+Row 1:
+──────
+toInt256OrDefault('-256', CAST('-1', 'Int256')): -256
+toInt256OrDefault('abc', CAST('-1', 'Int256')):  -1
 ```
 
 **See also**

From 5f0c40fafcefc8eba63ca1a872a6aa49939dcdaa Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 27 Jul 2024 12:10:41 +0200
Subject: [PATCH 0894/1488] Apply review comments

---
 src/Common/ZooKeeper/ZooKeeper.cpp | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/Common/ZooKeeper/ZooKeeper.cpp b/src/Common/ZooKeeper/ZooKeeper.cpp
index 7448d73cbbc..064ac2261ec 100644
--- a/src/Common/ZooKeeper/ZooKeeper.cpp
+++ b/src/Common/ZooKeeper/ZooKeeper.cpp
@@ -154,14 +154,14 @@ void ZooKeeper::init(ZooKeeperArgs args_, std::unique_ptr<Coordination::IKeeper>
                     ShuffleHosts node{optimal_host};
                     std::unique_ptr<Coordination::IKeeper> new_impl = std::make_unique<Coordination::ZooKeeper>(node, args, zk_log);
 
-                    if (auto new_node_idx = new_impl->getConnectedNodeIdx(); new_node_idx)
+                    auto new_node_idx = new_impl->getConnectedNodeIdx();
+                    chassert(new_node_idx.has_value());
+
+                    /// Maybe the node was unavailable when getting AZs first time, update just in case
+                    if (args.availability_zone_autodetect && availability_zones[*new_node_idx].empty())
                     {
-                        /// Maybe the node was unavailable when getting AZs first time, update just in case
-                        if (args.availability_zone_autodetect && availability_zones[*new_node_idx].empty())
-                        {
-                            availability_zones[*new_node_idx] = new_impl->tryGetAvailabilityZone();
-                            LOG_DEBUG(log, "Got availability zone for {}: {}", optimal_host.host, availability_zones[*new_node_idx]);
-                        }
+                        availability_zones[*new_node_idx] = new_impl->tryGetAvailabilityZone();
+                        LOG_DEBUG(log, "Got availability zone for {}: {}", optimal_host.host, availability_zones[*new_node_idx]);
                     }
 
                     optimal_impl = std::move(new_impl);

From 28b83a38f2671c2b85cfb2b116371ec754f39e75 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 27 Jul 2024 13:08:48 +0200
Subject: [PATCH 0895/1488] Remove harmful stuff from tests

---
 tests/queries/0_stateless/01187_set_profile_as_setting.sh    | 2 --
 tests/queries/0_stateless/02930_client_file_log_comment.sh   | 2 --
 tests/queries/0_stateless/02941_variant_type_1.sh            | 2 --
 tests/queries/0_stateless/02941_variant_type_2.sh            | 2 --
 tests/queries/0_stateless/02941_variant_type_3.sh            | 2 --
 tests/queries/0_stateless/02941_variant_type_4.sh            | 2 --
 tests/queries/0_stateless/02941_variant_type_alters.sh       | 2 --
 tests/queries/0_stateless/02943_variant_read_subcolumns.sh   | 3 ---
 ...943_variant_type_with_different_local_and_global_order.sh | 2 --
 tests/queries/0_stateless/03036_dynamic_read_subcolumns_1.sh | 2 --
 tests/queries/0_stateless/03036_dynamic_read_subcolumns_2.sh | 2 --
 tests/queries/0_stateless/03036_dynamic_read_subcolumns_3.sh | 2 --
 .../0_stateless/03039_dynamic_aggregating_merge_tree.sh      | 2 --
 .../0_stateless/03039_dynamic_collapsing_merge_tree.sh       | 2 --
 .../0_stateless/03039_dynamic_replacing_merge_tree.sh        | 2 --
 .../queries/0_stateless/03039_dynamic_summing_merge_tree.sh  | 2 --
 .../03039_dynamic_versioned_collapsing_merge_tree.sh         | 2 --
 tests/queries/0_stateless/03041_dynamic_type_check_table.sh  | 3 ---
 .../queries/0_stateless/03201_variant_null_map_subcolumn.sh  | 5 +----
 .../queries/0_stateless/03202_dynamic_null_map_subcolumn.sh  | 2 --
 20 files changed, 1 insertion(+), 44 deletions(-)

diff --git a/tests/queries/0_stateless/01187_set_profile_as_setting.sh b/tests/queries/0_stateless/01187_set_profile_as_setting.sh
index dacb609d790..804f51ea8a2 100755
--- a/tests/queries/0_stateless/01187_set_profile_as_setting.sh
+++ b/tests/queries/0_stateless/01187_set_profile_as_setting.sh
@@ -1,8 +1,6 @@
 #!/usr/bin/env bash
 # Tags: no-random-settings
 
-unset CLICKHOUSE_LOG_COMMENT
-
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
diff --git a/tests/queries/0_stateless/02930_client_file_log_comment.sh b/tests/queries/0_stateless/02930_client_file_log_comment.sh
index 3d0df6de9b4..1221c8a6521 100755
--- a/tests/queries/0_stateless/02930_client_file_log_comment.sh
+++ b/tests/queries/0_stateless/02930_client_file_log_comment.sh
@@ -1,8 +1,6 @@
 #!/usr/bin/env bash
 
 CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# reset --log_comment
-CLICKHOUSE_LOG_COMMENT=
 # shellcheck source=../shell_config.sh
 . "$CUR_DIR"/../shell_config.sh
 
diff --git a/tests/queries/0_stateless/02941_variant_type_1.sh b/tests/queries/0_stateless/02941_variant_type_1.sh
index 723de45eaad..c12bece6d54 100755
--- a/tests/queries/0_stateless/02941_variant_type_1.sh
+++ b/tests/queries/0_stateless/02941_variant_type_1.sh
@@ -2,8 +2,6 @@
 # Tags: long
 
 CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# reset --log_comment
-CLICKHOUSE_LOG_COMMENT=
 # shellcheck source=../shell_config.sh
 . "$CUR_DIR"/../shell_config.sh
 
diff --git a/tests/queries/0_stateless/02941_variant_type_2.sh b/tests/queries/0_stateless/02941_variant_type_2.sh
index 8453bce98dc..e93dfac8510 100755
--- a/tests/queries/0_stateless/02941_variant_type_2.sh
+++ b/tests/queries/0_stateless/02941_variant_type_2.sh
@@ -2,8 +2,6 @@
 # Tags: long
 
 CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# reset --log_comment
-CLICKHOUSE_LOG_COMMENT=
 # shellcheck source=../shell_config.sh
 . "$CUR_DIR"/../shell_config.sh
 
diff --git a/tests/queries/0_stateless/02941_variant_type_3.sh b/tests/queries/0_stateless/02941_variant_type_3.sh
index 990eb25b5be..cc0fde5b689 100755
--- a/tests/queries/0_stateless/02941_variant_type_3.sh
+++ b/tests/queries/0_stateless/02941_variant_type_3.sh
@@ -2,8 +2,6 @@
 # Tags: long
 
 CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# reset --log_comment
-CLICKHOUSE_LOG_COMMENT=
 # shellcheck source=../shell_config.sh
 . "$CUR_DIR"/../shell_config.sh
 
diff --git a/tests/queries/0_stateless/02941_variant_type_4.sh b/tests/queries/0_stateless/02941_variant_type_4.sh
index b8f619694b0..93a1770d05e 100755
--- a/tests/queries/0_stateless/02941_variant_type_4.sh
+++ b/tests/queries/0_stateless/02941_variant_type_4.sh
@@ -2,8 +2,6 @@
 # Tags: long
 
 CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# reset --log_comment
-CLICKHOUSE_LOG_COMMENT=
 # shellcheck source=../shell_config.sh
 . "$CUR_DIR"/../shell_config.sh
 
diff --git a/tests/queries/0_stateless/02941_variant_type_alters.sh b/tests/queries/0_stateless/02941_variant_type_alters.sh
index 7c151d1fe9e..0b3ab8e925b 100755
--- a/tests/queries/0_stateless/02941_variant_type_alters.sh
+++ b/tests/queries/0_stateless/02941_variant_type_alters.sh
@@ -2,8 +2,6 @@
 # Tags: long
 
 CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# reset --log_comment
-CLICKHOUSE_LOG_COMMENT=
 # shellcheck source=../shell_config.sh
 . "$CUR_DIR"/../shell_config.sh
 
diff --git a/tests/queries/0_stateless/02943_variant_read_subcolumns.sh b/tests/queries/0_stateless/02943_variant_read_subcolumns.sh
index 5ca8dd5f36f..21ab877eaf3 100755
--- a/tests/queries/0_stateless/02943_variant_read_subcolumns.sh
+++ b/tests/queries/0_stateless/02943_variant_read_subcolumns.sh
@@ -2,8 +2,6 @@
 # Tags: long
 
 CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# reset --log_comment
-CLICKHOUSE_LOG_COMMENT=
 # shellcheck source=../shell_config.sh
 . "$CUR_DIR"/../shell_config.sh
 
@@ -34,4 +32,3 @@ echo "MergeTree wide"
 $CH_CLIENT -q "create table test (id UInt64, v Variant(UInt64, Array(Variant(String, UInt64)))) engine=MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1;"
 test
 $CH_CLIENT -q "drop table test;"
-
diff --git a/tests/queries/0_stateless/02943_variant_type_with_different_local_and_global_order.sh b/tests/queries/0_stateless/02943_variant_type_with_different_local_and_global_order.sh
index 1d88757a5d6..4aa4b11cecb 100755
--- a/tests/queries/0_stateless/02943_variant_type_with_different_local_and_global_order.sh
+++ b/tests/queries/0_stateless/02943_variant_type_with_different_local_and_global_order.sh
@@ -2,8 +2,6 @@
 # Tags: long, no-debug
 
 CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# reset --log_comment
-CLICKHOUSE_LOG_COMMENT=
 # shellcheck source=../shell_config.sh
 . "$CUR_DIR"/../shell_config.sh
 
diff --git a/tests/queries/0_stateless/03036_dynamic_read_subcolumns_1.sh b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_1.sh
index aabba731816..ed548ae74e9 100755
--- a/tests/queries/0_stateless/03036_dynamic_read_subcolumns_1.sh
+++ b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_1.sh
@@ -2,8 +2,6 @@
 # Tags: long
 
 CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# reset --log_comment
-CLICKHOUSE_LOG_COMMENT=
 # shellcheck source=../shell_config.sh
 . "$CUR_DIR"/../shell_config.sh
 
diff --git a/tests/queries/0_stateless/03036_dynamic_read_subcolumns_2.sh b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_2.sh
index 872f4c20a98..95dafcf5832 100755
--- a/tests/queries/0_stateless/03036_dynamic_read_subcolumns_2.sh
+++ b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_2.sh
@@ -2,8 +2,6 @@
 # Tags: long
 
 CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# reset --log_comment
-CLICKHOUSE_LOG_COMMENT=
 # shellcheck source=../shell_config.sh
 . "$CUR_DIR"/../shell_config.sh
 
diff --git a/tests/queries/0_stateless/03036_dynamic_read_subcolumns_3.sh b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_3.sh
index 96276c96add..a3c2d93e568 100755
--- a/tests/queries/0_stateless/03036_dynamic_read_subcolumns_3.sh
+++ b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_3.sh
@@ -2,8 +2,6 @@
 # Tags: long
 
 CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# reset --log_comment
-CLICKHOUSE_LOG_COMMENT=
 # shellcheck source=../shell_config.sh
 . "$CUR_DIR"/../shell_config.sh
 
diff --git a/tests/queries/0_stateless/03039_dynamic_aggregating_merge_tree.sh b/tests/queries/0_stateless/03039_dynamic_aggregating_merge_tree.sh
index b8760ec0e1d..9ea86105a3a 100755
--- a/tests/queries/0_stateless/03039_dynamic_aggregating_merge_tree.sh
+++ b/tests/queries/0_stateless/03039_dynamic_aggregating_merge_tree.sh
@@ -2,8 +2,6 @@
 # Tags: long
 
 CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# reset --log_comment
-CLICKHOUSE_LOG_COMMENT=
 # shellcheck source=../shell_config.sh
 . "$CUR_DIR"/../shell_config.sh
 
diff --git a/tests/queries/0_stateless/03039_dynamic_collapsing_merge_tree.sh b/tests/queries/0_stateless/03039_dynamic_collapsing_merge_tree.sh
index 881c9ec64cc..9a2a6dd957c 100755
--- a/tests/queries/0_stateless/03039_dynamic_collapsing_merge_tree.sh
+++ b/tests/queries/0_stateless/03039_dynamic_collapsing_merge_tree.sh
@@ -2,8 +2,6 @@
 # Tags: long
 
 CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# reset --log_comment
-CLICKHOUSE_LOG_COMMENT=
 # shellcheck source=../shell_config.sh
 . "$CUR_DIR"/../shell_config.sh
 
diff --git a/tests/queries/0_stateless/03039_dynamic_replacing_merge_tree.sh b/tests/queries/0_stateless/03039_dynamic_replacing_merge_tree.sh
index fc9039ac98c..0199035a3df 100755
--- a/tests/queries/0_stateless/03039_dynamic_replacing_merge_tree.sh
+++ b/tests/queries/0_stateless/03039_dynamic_replacing_merge_tree.sh
@@ -2,8 +2,6 @@
 # Tags: long
 
 CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# reset --log_comment
-CLICKHOUSE_LOG_COMMENT=
 # shellcheck source=../shell_config.sh
 . "$CUR_DIR"/../shell_config.sh
 
diff --git a/tests/queries/0_stateless/03039_dynamic_summing_merge_tree.sh b/tests/queries/0_stateless/03039_dynamic_summing_merge_tree.sh
index f9da70e95ca..e2ea5bc3466 100755
--- a/tests/queries/0_stateless/03039_dynamic_summing_merge_tree.sh
+++ b/tests/queries/0_stateless/03039_dynamic_summing_merge_tree.sh
@@ -2,8 +2,6 @@
 # Tags: long
 
 CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# reset --log_comment
-CLICKHOUSE_LOG_COMMENT=
 # shellcheck source=../shell_config.sh
 . "$CUR_DIR"/../shell_config.sh
 
diff --git a/tests/queries/0_stateless/03039_dynamic_versioned_collapsing_merge_tree.sh b/tests/queries/0_stateless/03039_dynamic_versioned_collapsing_merge_tree.sh
index 60248f4453a..43607cf95a8 100755
--- a/tests/queries/0_stateless/03039_dynamic_versioned_collapsing_merge_tree.sh
+++ b/tests/queries/0_stateless/03039_dynamic_versioned_collapsing_merge_tree.sh
@@ -2,8 +2,6 @@
 # Tags: long
 
 CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# reset --log_comment
-CLICKHOUSE_LOG_COMMENT=
 # shellcheck source=../shell_config.sh
 . "$CUR_DIR"/../shell_config.sh
 
diff --git a/tests/queries/0_stateless/03041_dynamic_type_check_table.sh b/tests/queries/0_stateless/03041_dynamic_type_check_table.sh
index 3d802485be3..c8bd533e253 100755
--- a/tests/queries/0_stateless/03041_dynamic_type_check_table.sh
+++ b/tests/queries/0_stateless/03041_dynamic_type_check_table.sh
@@ -2,8 +2,6 @@
 # Tags: long
 
 CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# reset --log_comment
-CLICKHOUSE_LOG_COMMENT=
 # shellcheck source=../shell_config.sh
 . "$CUR_DIR"/../shell_config.sh
 
@@ -42,4 +40,3 @@ echo "MergeTree wide"
 $CH_CLIENT -q "create table test (x UInt64, y UInt64 ) engine=MergeTree order by x settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1;"
 run
 $CH_CLIENT -q "drop table test;"
-
diff --git a/tests/queries/0_stateless/03201_variant_null_map_subcolumn.sh b/tests/queries/0_stateless/03201_variant_null_map_subcolumn.sh
index 8231691e184..050be8359ad 100755
--- a/tests/queries/0_stateless/03201_variant_null_map_subcolumn.sh
+++ b/tests/queries/0_stateless/03201_variant_null_map_subcolumn.sh
@@ -2,8 +2,6 @@
 # Tags: long
 
 CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# reset --log_comment
-CLICKHOUSE_LOG_COMMENT=
 # shellcheck source=../shell_config.sh
 . "$CUR_DIR"/../shell_config.sh
 
@@ -17,7 +15,7 @@ function test()
     $CH_CLIENT -q "select v.UInt64.null, v.\`Array(Variant(String, UInt64))\`.null,  v.\`Array(Variant(String, UInt64))\`.size0, v.\`Array(Variant(String, UInt64))\`.UInt64.null from test order by id"
     $CH_CLIENT -q "select v.\`Array(Variant(String, UInt64))\`.null,  v.\`Array(Variant(String, UInt64))\`.size0, v.\`Array(Variant(String, UInt64))\`.UInt64.null, v.\`Array(Variant(String, UInt64))\`.String.null from test order by id"
     $CH_CLIENT -q "select id from test where v.UInt64 is null order by id"
-    
+
     $CH_CLIENT -q "insert into test select number, multiIf(number % 3 == 2, NULL, number % 3 == 1, number, arrayMap(x -> multiIf(number % 9 == 0, NULL, number % 9 == 3, 'str_' || toString(number), number), range(number % 10))) from numbers(1000000) settings min_insert_block_size_rows=100000"
     $CH_CLIENT -q "select v, v.UInt64.null, v.\`Array(Variant(String, UInt64))\`.null,  v.\`Array(Variant(String, UInt64))\`.size0, v.\`Array(Variant(String, UInt64))\`.UInt64.null from test order by id format Null"
     $CH_CLIENT -q "select v.UInt64.null, v.\`Array(Variant(String, UInt64))\`.null,  v.\`Array(Variant(String, UInt64))\`.size0, v.\`Array(Variant(String, UInt64))\`.UInt64.null from test order by id format Null"
@@ -41,4 +39,3 @@ echo "MergeTree wide"
 $CH_CLIENT -q "create table test (id UInt64, v Variant(UInt64, Array(Variant(String, UInt64)))) engine=MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1;"
 test
 $CH_CLIENT -q "drop table test;"
-
diff --git a/tests/queries/0_stateless/03202_dynamic_null_map_subcolumn.sh b/tests/queries/0_stateless/03202_dynamic_null_map_subcolumn.sh
index 968c9e5271f..bbe25ea72bb 100755
--- a/tests/queries/0_stateless/03202_dynamic_null_map_subcolumn.sh
+++ b/tests/queries/0_stateless/03202_dynamic_null_map_subcolumn.sh
@@ -2,8 +2,6 @@
 # Tags: long
 
 CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# reset --log_comment
-CLICKHOUSE_LOG_COMMENT=
 # shellcheck source=../shell_config.sh
 . "$CUR_DIR"/../shell_config.sh
 

From 8f96858df1be98d4707a2cf5f821c77428134e4f Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 27 Jul 2024 13:11:53 +0200
Subject: [PATCH 0896/1488] Fix test

---
 tests/queries/0_stateless/03201_variant_null_map_subcolumn.sh | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/tests/queries/0_stateless/03201_variant_null_map_subcolumn.sh b/tests/queries/0_stateless/03201_variant_null_map_subcolumn.sh
index 8231691e184..57dc36d8a8f 100755
--- a/tests/queries/0_stateless/03201_variant_null_map_subcolumn.sh
+++ b/tests/queries/0_stateless/03201_variant_null_map_subcolumn.sh
@@ -17,8 +17,7 @@ function test()
     $CH_CLIENT -q "select v.UInt64.null, v.\`Array(Variant(String, UInt64))\`.null,  v.\`Array(Variant(String, UInt64))\`.size0, v.\`Array(Variant(String, UInt64))\`.UInt64.null from test order by id"
     $CH_CLIENT -q "select v.\`Array(Variant(String, UInt64))\`.null,  v.\`Array(Variant(String, UInt64))\`.size0, v.\`Array(Variant(String, UInt64))\`.UInt64.null, v.\`Array(Variant(String, UInt64))\`.String.null from test order by id"
     $CH_CLIENT -q "select id from test where v.UInt64 is null order by id"
-    
-    $CH_CLIENT -q "insert into test select number, multiIf(number % 3 == 2, NULL, number % 3 == 1, number, arrayMap(x -> multiIf(number % 9 == 0, NULL, number % 9 == 3, 'str_' || toString(number), number), range(number % 10))) from numbers(1000000) settings min_insert_block_size_rows=100000"
+    $CH_CLIENT -q "insert into test select number, multiIf(number % 3 == 2, NULL, number % 3 == 1, number, arrayMap(x -> multiIf(number % 9 == 0, NULL, number % 9 == 3, 'str_' || toString(number), number), range(number % 10))) from numbers(250000) settings min_insert_block_size_rows=100000, min_insert_block_size_bytes=0"
     $CH_CLIENT -q "select v, v.UInt64.null, v.\`Array(Variant(String, UInt64))\`.null,  v.\`Array(Variant(String, UInt64))\`.size0, v.\`Array(Variant(String, UInt64))\`.UInt64.null from test order by id format Null"
     $CH_CLIENT -q "select v.UInt64.null, v.\`Array(Variant(String, UInt64))\`.null,  v.\`Array(Variant(String, UInt64))\`.size0, v.\`Array(Variant(String, UInt64))\`.UInt64.null from test order by id format Null"
     $CH_CLIENT -q "select v.\`Array(Variant(String, UInt64))\`.null,  v.\`Array(Variant(String, UInt64))\`.size0, v.\`Array(Variant(String, UInt64))\`.UInt64.null, v.\`Array(Variant(String, UInt64))\`.String.null from test order by id format Null"
@@ -41,4 +40,3 @@ echo "MergeTree wide"
 $CH_CLIENT -q "create table test (id UInt64, v Variant(UInt64, Array(Variant(String, UInt64)))) engine=MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1;"
 test
 $CH_CLIENT -q "drop table test;"
-

From 554cf91f4bb2d8d2272428c3f2cfbb4c3556d4b1 Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitlibar@clickhouse.com>
Date: Sat, 27 Jul 2024 14:42:23 +0200
Subject: [PATCH 0897/1488] Add missing call applyCmdOptions().

---
 programs/local/LocalServer.cpp | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp
index e60c8ef6085..ce0e179939d 100644
--- a/programs/local/LocalServer.cpp
+++ b/programs/local/LocalServer.cpp
@@ -735,11 +735,13 @@ void LocalServer::processConfig()
     /// there is separate context for Buffer tables).
     adjustSettings();
     applySettingsOverridesForLocal(global_context);
+    applyCmdOptions(global_context);
 
     /// Load global settings from default_profile and system_profile.
     global_context->setDefaultProfiles(getClientConfiguration());
 
-    applyCmdOptions(global_context);
+    /// Command-line parameters can override settings from the default profile.
+    applyCmdSettings(global_context);
 
     /// We load temporary database first, because projections need it.
     DatabaseCatalog::instance().initializeAndLoadTemporaryDatabase();

From 6bbd9e6d67800fcfcb7105b386a78ba8c09b709a Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 27 Jul 2024 15:43:09 +0200
Subject: [PATCH 0898/1488] Update 02930_client_file_log_comment.sh

---
 tests/queries/0_stateless/02930_client_file_log_comment.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/queries/0_stateless/02930_client_file_log_comment.sh b/tests/queries/0_stateless/02930_client_file_log_comment.sh
index 1221c8a6521..50cd587e4b5 100755
--- a/tests/queries/0_stateless/02930_client_file_log_comment.sh
+++ b/tests/queries/0_stateless/02930_client_file_log_comment.sh
@@ -1,6 +1,8 @@
 #!/usr/bin/env bash
 
 CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# reset --log_comment, because the test has to set its own
+CLICKHOUSE_LOG_COMMENT=
 # shellcheck source=../shell_config.sh
 . "$CUR_DIR"/../shell_config.sh
 

From b0c2991b84dfeadaae099f102f8ef4d9e58844d4 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 27 Jul 2024 15:43:38 +0200
Subject: [PATCH 0899/1488] Update 01187_set_profile_as_setting.sh

---
 tests/queries/0_stateless/01187_set_profile_as_setting.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/queries/0_stateless/01187_set_profile_as_setting.sh b/tests/queries/0_stateless/01187_set_profile_as_setting.sh
index 804f51ea8a2..42f596c45d6 100755
--- a/tests/queries/0_stateless/01187_set_profile_as_setting.sh
+++ b/tests/queries/0_stateless/01187_set_profile_as_setting.sh
@@ -2,6 +2,8 @@
 # Tags: no-random-settings
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# reset --log_comment, because the test has to use the readonly mode
+CLICKHOUSE_LOG_COMMENT=
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 

From 3c4902d9d327bdf2e9d22606f163cd137edd4b49 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 27 Jul 2024 16:04:08 +0200
Subject: [PATCH 0900/1488] Split `01651_lc_insert_tiny_log`

---
 .../01651_lc_insert_tiny_log.reference        | 12 -----
 .../0_stateless/01651_lc_insert_tiny_log.sql  | 48 -------------------
 .../01651_lc_insert_tiny_log_1.reference      |  4 ++
 .../01651_lc_insert_tiny_log_1.sql            | 16 +++++++
 .../01651_lc_insert_tiny_log_2.reference      |  4 ++
 .../01651_lc_insert_tiny_log_2.sql            | 16 +++++++
 .../01651_lc_insert_tiny_log_3.reference      |  4 ++
 .../01651_lc_insert_tiny_log_3.sql            | 16 +++++++
 8 files changed, 60 insertions(+), 60 deletions(-)
 delete mode 100644 tests/queries/0_stateless/01651_lc_insert_tiny_log.reference
 delete mode 100644 tests/queries/0_stateless/01651_lc_insert_tiny_log.sql
 create mode 100644 tests/queries/0_stateless/01651_lc_insert_tiny_log_1.reference
 create mode 100644 tests/queries/0_stateless/01651_lc_insert_tiny_log_1.sql
 create mode 100644 tests/queries/0_stateless/01651_lc_insert_tiny_log_2.reference
 create mode 100644 tests/queries/0_stateless/01651_lc_insert_tiny_log_2.sql
 create mode 100644 tests/queries/0_stateless/01651_lc_insert_tiny_log_3.reference
 create mode 100644 tests/queries/0_stateless/01651_lc_insert_tiny_log_3.sql

diff --git a/tests/queries/0_stateless/01651_lc_insert_tiny_log.reference b/tests/queries/0_stateless/01651_lc_insert_tiny_log.reference
deleted file mode 100644
index 5cc8909b6c8..00000000000
--- a/tests/queries/0_stateless/01651_lc_insert_tiny_log.reference
+++ /dev/null
@@ -1,12 +0,0 @@
-10000000
-10000000	1274991808
-30000000
-30000000	3824991808
-10000000
-10000000	1274991808
-30000000
-30000000	3824991808
-10000000
-10000000	1274991808
-30000000
-30000000	3824991808
diff --git a/tests/queries/0_stateless/01651_lc_insert_tiny_log.sql b/tests/queries/0_stateless/01651_lc_insert_tiny_log.sql
deleted file mode 100644
index d11c9120c61..00000000000
--- a/tests/queries/0_stateless/01651_lc_insert_tiny_log.sql
+++ /dev/null
@@ -1,48 +0,0 @@
-set allow_suspicious_low_cardinality_types=1;
-drop table if exists perf_lc_num;
-
-CREATE TABLE perf_lc_num(　        num UInt8,　        arr Array(LowCardinality(Int64)) default [num]　        ) ENGINE = TinyLog;
-
-INSERT INTO perf_lc_num (num) SELECT toUInt8(number) FROM numbers(10000000);
-
-select sum(length(arr)) from perf_lc_num;
-select sum(length(arr)), sum(num) from perf_lc_num;
-
-INSERT INTO perf_lc_num (num) SELECT toUInt8(number) FROM numbers(10000000, 20000000);
-
-select sum(length(arr)) from perf_lc_num;
-select sum(length(arr)), sum(num) from perf_lc_num;
-
-drop table if exists perf_lc_num;
-
-
-CREATE TABLE perf_lc_num(　        num UInt8,　        arr Array(LowCardinality(Int64)) default [num]　        ) ENGINE = Log;
-
-INSERT INTO perf_lc_num (num) SELECT toUInt8(number) FROM numbers(10000000);
-
-select sum(length(arr)) from perf_lc_num;
-select sum(length(arr)), sum(num) from perf_lc_num;
-
-INSERT INTO perf_lc_num (num) SELECT toUInt8(number) FROM numbers(10000000, 20000000);
-
-select sum(length(arr)) from perf_lc_num;
-select sum(length(arr)), sum(num) from perf_lc_num;
-
-drop table if exists perf_lc_num;
-
-
-CREATE TABLE perf_lc_num(　        num UInt8,　        arr Array(LowCardinality(Int64)) default [num]　        ) ENGINE = StripeLog;
-
-INSERT INTO perf_lc_num (num) SELECT toUInt8(number) FROM numbers(10000000);
-
-select sum(length(arr)) from perf_lc_num;
-select sum(length(arr)), sum(num) from perf_lc_num;
-
-INSERT INTO perf_lc_num (num) SELECT toUInt8(number) FROM numbers(10000000, 20000000);
-
-select sum(length(arr)) from perf_lc_num;
-select sum(length(arr)), sum(num) from perf_lc_num;
-
-drop table if exists perf_lc_num;
-
-
diff --git a/tests/queries/0_stateless/01651_lc_insert_tiny_log_1.reference b/tests/queries/0_stateless/01651_lc_insert_tiny_log_1.reference
new file mode 100644
index 00000000000..0715a4212ed
--- /dev/null
+++ b/tests/queries/0_stateless/01651_lc_insert_tiny_log_1.reference
@@ -0,0 +1,4 @@
+10000000
+10000000	1274991808
+30000000
+30000000	3824991808
diff --git a/tests/queries/0_stateless/01651_lc_insert_tiny_log_1.sql b/tests/queries/0_stateless/01651_lc_insert_tiny_log_1.sql
new file mode 100644
index 00000000000..142d3b21b47
--- /dev/null
+++ b/tests/queries/0_stateless/01651_lc_insert_tiny_log_1.sql
@@ -0,0 +1,16 @@
+set allow_suspicious_low_cardinality_types = 1, max_rows_to_read = '31M';
+drop table if exists perf_lc_num;
+
+CREATE TABLE perf_lc_num(        num UInt8,        arr Array(LowCardinality(Int64)) default [num]        ) ENGINE = TinyLog;
+
+INSERT INTO perf_lc_num (num) SELECT toUInt8(number) FROM numbers(10000000);
+
+select sum(length(arr)) from perf_lc_num;
+select sum(length(arr)), sum(num) from perf_lc_num;
+
+INSERT INTO perf_lc_num (num) SELECT toUInt8(number) FROM numbers(10000000, 20000000);
+
+select sum(length(arr)) from perf_lc_num;
+select sum(length(arr)), sum(num) from perf_lc_num;
+
+drop table if exists perf_lc_num;
diff --git a/tests/queries/0_stateless/01651_lc_insert_tiny_log_2.reference b/tests/queries/0_stateless/01651_lc_insert_tiny_log_2.reference
new file mode 100644
index 00000000000..0715a4212ed
--- /dev/null
+++ b/tests/queries/0_stateless/01651_lc_insert_tiny_log_2.reference
@@ -0,0 +1,4 @@
+10000000
+10000000	1274991808
+30000000
+30000000	3824991808
diff --git a/tests/queries/0_stateless/01651_lc_insert_tiny_log_2.sql b/tests/queries/0_stateless/01651_lc_insert_tiny_log_2.sql
new file mode 100644
index 00000000000..e6b68e1d682
--- /dev/null
+++ b/tests/queries/0_stateless/01651_lc_insert_tiny_log_2.sql
@@ -0,0 +1,16 @@
+set allow_suspicious_low_cardinality_types = 1, max_rows_to_read = '31M';
+drop table if exists perf_lc_num;
+
+CREATE TABLE perf_lc_num(        num UInt8,        arr Array(LowCardinality(Int64)) default [num]        ) ENGINE = Log;
+
+INSERT INTO perf_lc_num (num) SELECT toUInt8(number) FROM numbers(10000000);
+
+select sum(length(arr)) from perf_lc_num;
+select sum(length(arr)), sum(num) from perf_lc_num;
+
+INSERT INTO perf_lc_num (num) SELECT toUInt8(number) FROM numbers(10000000, 20000000);
+
+select sum(length(arr)) from perf_lc_num;
+select sum(length(arr)), sum(num) from perf_lc_num;
+
+drop table if exists perf_lc_num;
diff --git a/tests/queries/0_stateless/01651_lc_insert_tiny_log_3.reference b/tests/queries/0_stateless/01651_lc_insert_tiny_log_3.reference
new file mode 100644
index 00000000000..0715a4212ed
--- /dev/null
+++ b/tests/queries/0_stateless/01651_lc_insert_tiny_log_3.reference
@@ -0,0 +1,4 @@
+10000000
+10000000	1274991808
+30000000
+30000000	3824991808
diff --git a/tests/queries/0_stateless/01651_lc_insert_tiny_log_3.sql b/tests/queries/0_stateless/01651_lc_insert_tiny_log_3.sql
new file mode 100644
index 00000000000..85a42163897
--- /dev/null
+++ b/tests/queries/0_stateless/01651_lc_insert_tiny_log_3.sql
@@ -0,0 +1,16 @@
+set allow_suspicious_low_cardinality_types = 1, max_rows_to_read = '31M';
+drop table if exists perf_lc_num;
+
+CREATE TABLE perf_lc_num(        num UInt8,        arr Array(LowCardinality(Int64)) default [num]        ) ENGINE = StripeLog;
+
+INSERT INTO perf_lc_num (num) SELECT toUInt8(number) FROM numbers(10000000);
+
+select sum(length(arr)) from perf_lc_num;
+select sum(length(arr)), sum(num) from perf_lc_num;
+
+INSERT INTO perf_lc_num (num) SELECT toUInt8(number) FROM numbers(10000000, 20000000);
+
+select sum(length(arr)) from perf_lc_num;
+select sum(length(arr)), sum(num) from perf_lc_num;
+
+drop table if exists perf_lc_num;

From 0de2b07119b6ab61a25df5f7b641282dda5e63d7 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 27 Jul 2024 16:19:20 +0200
Subject: [PATCH 0901/1488] Debug test `02490_benchmark_max_consecutive_errors`

---
 .../02490_benchmark_max_consecutive_errors.sh            | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/02490_benchmark_max_consecutive_errors.sh b/tests/queries/0_stateless/02490_benchmark_max_consecutive_errors.sh
index 3c046020773..f747b3156a5 100755
--- a/tests/queries/0_stateless/02490_benchmark_max_consecutive_errors.sh
+++ b/tests/queries/0_stateless/02490_benchmark_max_consecutive_errors.sh
@@ -5,4 +5,11 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CUR_DIR"/../shell_config.sh
 
 $CLICKHOUSE_BENCHMARK -q 'select throwIf(1)' |& grep '^An error occurred while processing the query.*Exception:' -c
-$CLICKHOUSE_BENCHMARK --max-consecutive-errors 10 -q 'select throwIf(1)' |& grep '^An error occurred while processing the query.*Exception:' -c
+RES=$($CLICKHOUSE_BENCHMARK --max-consecutive-errors 10 -q 'select throwIf(1)' |& tee "${CLICKHOUSE_TMP}/${CLICKHOUSE_DATABASE}.log" | grep '^An error occurred while processing the query.*Exception:' -c)
+
+if [ "$RES" -eq 10 ]
+then
+    echo "$RES"
+else
+    cat "${CLICKHOUSE_TMP}/${CLICKHOUSE_DATABASE}.log"
+fi

From 38c4251a0eebe285b3f4da3ec21d11421d0e00fa Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Sat, 27 Jul 2024 16:30:18 +0200
Subject: [PATCH 0902/1488] Cleanup get{Local,Client}ConfigPath() helpers

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 src/Common/Config/getClientConfigPath.cpp | 11 +----------
 src/Common/Config/getLocalConfigPath.cpp  | 11 +----------
 2 files changed, 2 insertions(+), 20 deletions(-)

diff --git a/src/Common/Config/getClientConfigPath.cpp b/src/Common/Config/getClientConfigPath.cpp
index a32ad1068bf..5e9bb16bc85 100644
--- a/src/Common/Config/getClientConfigPath.cpp
+++ b/src/Common/Config/getClientConfigPath.cpp
@@ -12,7 +12,6 @@ namespace DB
 std::optional<std::string> getClientConfigPath(const std::string & home_path)
 {
     std::string config_path;
-    bool found = false;
 
     std::vector<std::string> names;
     names.emplace_back("./clickhouse-client");
@@ -28,18 +27,10 @@ std::optional<std::string> getClientConfigPath(const std::string & home_path)
 
             std::error_code ec;
             if (fs::exists(config_path, ec))
-            {
-                found = true;
-                break;
-            }
+                return config_path;
         }
-        if (found)
-            break;
     }
 
-    if (found)
-        return config_path;
-
     return std::nullopt;
 }
 
diff --git a/src/Common/Config/getLocalConfigPath.cpp b/src/Common/Config/getLocalConfigPath.cpp
index afaa7f79026..195de8aed03 100644
--- a/src/Common/Config/getLocalConfigPath.cpp
+++ b/src/Common/Config/getLocalConfigPath.cpp
@@ -12,7 +12,6 @@ namespace DB
 std::optional<std::string> getLocalConfigPath(const std::string & home_path)
 {
     std::string config_path;
-    bool found = false;
 
     std::vector<std::string> names;
     names.emplace_back("./clickhouse-local");
@@ -28,18 +27,10 @@ std::optional<std::string> getLocalConfigPath(const std::string & home_path)
 
             std::error_code ec;
             if (fs::exists(config_path, ec))
-            {
-                found = true;
-                break;
-            }
+                return config_path;
         }
-        if (found)
-            break;
     }
 
-    if (found)
-        return config_path;
-
     return std::nullopt;
 }
 

From 23cd9f2224cea2a52572a6ad4f1ba3f879dacd48 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 27 Jul 2024 17:17:54 +0200
Subject: [PATCH 0903/1488] Fix bad test @Demilivor

---
 programs/server/config.d/session_log.xml      |  1 +
 ...ce => 02833_concurrent_sessions.reference} | 32 ++++++++---------
 ...ssions.sh => 02833_concurrent_sessions.sh} | 34 +++++++++----------
 3 files changed, 34 insertions(+), 33 deletions(-)
 create mode 120000 programs/server/config.d/session_log.xml
 rename tests/queries/0_stateless/{02833_concurrrent_sessions.reference => 02833_concurrent_sessions.reference} (84%)
 rename tests/queries/0_stateless/{02833_concurrrent_sessions.sh => 02833_concurrent_sessions.sh} (88%)

diff --git a/programs/server/config.d/session_log.xml b/programs/server/config.d/session_log.xml
new file mode 120000
index 00000000000..7678cc33129
--- /dev/null
+++ b/programs/server/config.d/session_log.xml
@@ -0,0 +1 @@
+../../../tests/config/config.d/session_log.xml
\ No newline at end of file
diff --git a/tests/queries/0_stateless/02833_concurrrent_sessions.reference b/tests/queries/0_stateless/02833_concurrent_sessions.reference
similarity index 84%
rename from tests/queries/0_stateless/02833_concurrrent_sessions.reference
rename to tests/queries/0_stateless/02833_concurrent_sessions.reference
index bfe507e8eac..904e8cefc6e 100644
--- a/tests/queries/0_stateless/02833_concurrrent_sessions.reference
+++ b/tests/queries/0_stateless/02833_concurrent_sessions.reference
@@ -1,34 +1,34 @@
 sessions:
-150
+45
 port_0_sessions:
 0
 address_0_sessions:
 0
 tcp_sessions
-60
+18
 http_sessions
-30
+9
 http_with_session_id_sessions
-30
-my_sql_sessions
-30
+9
+mysql_sessions
+9
 Corresponding LoginSuccess/Logout
-10
+3
 LoginFailure
-10
+3
 Corresponding LoginSuccess/Logout
-10
+3
 LoginFailure
-10
+3
 Corresponding LoginSuccess/Logout
-10
+3
 LoginFailure
-10
+3
 Corresponding LoginSuccess/Logout
-10
+3
 LoginFailure
-10
+3
 Corresponding LoginSuccess/Logout
-10
+3
 LoginFailure
-10
+3
diff --git a/tests/queries/0_stateless/02833_concurrrent_sessions.sh b/tests/queries/0_stateless/02833_concurrent_sessions.sh
similarity index 88%
rename from tests/queries/0_stateless/02833_concurrrent_sessions.sh
rename to tests/queries/0_stateless/02833_concurrent_sessions.sh
index c5b6204529b..846661cfeed 100755
--- a/tests/queries/0_stateless/02833_concurrrent_sessions.sh
+++ b/tests/queries/0_stateless/02833_concurrent_sessions.sh
@@ -37,11 +37,11 @@ done
 # These functions try to create a session with successful login and logout.
 # Sleep a small, random amount of time to make concurrency more intense.
 # and try to login with an invalid password.
-function tcp_session() 
+function tcp_session()
 {
     local user=$1
     local i=0
-    while (( (i++) < 10 )); do
+    while (( (i++) < 3 )); do
         # login logout
         ${CLICKHOUSE_CLIENT} -q "SELECT 1, sleep(0.01${RANDOM})" --user="${user}" --password="pass"
         # login failure
@@ -49,11 +49,11 @@ function tcp_session()
     done
 }
 
-function http_session() 
+function http_session()
 {
     local user=$1
     local i=0
-    while (( (i++) < 10 )); do
+    while (( (i++) < 3 )); do
         # login logout
         ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&user=${user}&password=pass" -d "SELECT 3, sleep(0.01${RANDOM})"
 
@@ -62,11 +62,11 @@ function http_session()
     done
 }
 
-function http_with_session_id_session() 
+function http_with_session_id_session()
 {
     local user=$1
     local i=0
-    while (( (i++) < 10 )); do
+    while (( (i++) < 3 )); do
         # login logout
         ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&session_id=${user}&user=${user}&password=pass" -d "SELECT 5, sleep 0.01${RANDOM}"
 
@@ -75,11 +75,11 @@ function http_with_session_id_session()
     done
 }
 
-function mysql_session() 
+function mysql_session()
 {
     local user=$1
     local i=0
-    while (( (i++) < 10 )); do
+    while (( (i++) < 3 )); do
         # login logout
         ${CLICKHOUSE_CLIENT} -q "SELECT 1, sleep(0.01${RANDOM}) FROM mysql('127.0.0.1:9004', 'system', 'one', '${user}', 'pass')"
 
@@ -97,29 +97,29 @@ export -f http_with_session_id_session;
 export -f mysql_session;
 
 for user in "${TCP_USERS[@]}"; do
-    timeout 60s bash -c "tcp_session ${user}" >/dev/null 2>&1 &
+    tcp_session ${user} >/dev/null 2>&1 &
 done
 
 for user in "${HTTP_USERS[@]}"; do
-    timeout 60s bash -c "http_session ${user}" >/dev/null 2>&1 &
+    http_session ${user} >/dev/null 2>&1 &
 done
 
 for user in "${HTTP_WITH_SESSION_ID_SESSION_USERS[@]}"; do
-    timeout 60s bash -c "http_with_session_id_session ${user}" >/dev/null 2>&1 &
+    http_with_session_id_session ${user} >/dev/null 2>&1 &
 done
 
 for user in "${MYSQL_USERS[@]}"; do
-    timeout 60s bash -c "mysql_session ${user}" >/dev/null 2>&1 &
+    mysql_session ${user} >/dev/null 2>&1 &
 done
 
 wait
 
 ${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS"
 
-echo "sessions:" 
+echo "sessions:"
 ${CLICKHOUSE_CLIENT} -q "SELECT count(*) FROM system.session_log WHERE user IN (${ALL_USERS_SQL_COLLECTION_STRING})"
 
-echo "port_0_sessions:" 
+echo "port_0_sessions:"
 ${CLICKHOUSE_CLIENT} -q "SELECT count(*) FROM system.session_log WHERE user IN (${ALL_USERS_SQL_COLLECTION_STRING}) AND client_port = 0"
 
 echo "address_0_sessions:"
@@ -131,13 +131,13 @@ echo "http_sessions"
 ${CLICKHOUSE_CLIENT} -q "SELECT count(*) FROM system.session_log WHERE user IN (${HTTP_USERS_SQL_COLLECTION_STRING}) AND interface = 'HTTP'"
 echo "http_with_session_id_sessions"
 ${CLICKHOUSE_CLIENT} -q "SELECT count(*) FROM system.session_log WHERE user IN (${HTTP_WITH_SESSION_ID_USERS_SQL_COLLECTION_STRING}) AND interface = 'HTTP'"
-echo "my_sql_sessions"
+echo "mysql_sessions"
 ${CLICKHOUSE_CLIENT} -q "SELECT count(*) FROM system.session_log WHERE user IN (${MYSQL_USERS_SQL_COLLECTION_STRING}) AND interface = 'MySQL'"
 
 for user in "${ALL_USERS[@]}"; do
     ${CLICKHOUSE_CLIENT} -q "DROP USER ${user}"
-    echo "Corresponding LoginSuccess/Logout" 
+    echo "Corresponding LoginSuccess/Logout"
     ${CLICKHOUSE_CLIENT} -q "SELECT COUNT(*) FROM (SELECT ${SESSION_LOG_MATCHING_FIELDS} FROM system.session_log WHERE user = '${user}' AND type = 'LoginSuccess' INTERSECT SELECT ${SESSION_LOG_MATCHING_FIELDS} FROM system.session_log WHERE user = '${user}' AND type = 'Logout')"
     echo "LoginFailure"
-    ${CLICKHOUSE_CLIENT} -q "SELECT COUNT(*) FROM system.session_log WHERE user = '${user}' AND type = 'LoginFailure'" 
+    ${CLICKHOUSE_CLIENT} -q "SELECT COUNT(*) FROM system.session_log WHERE user = '${user}' AND type = 'LoginFailure'"
  done

From f2a43b62b0af1aa1b41aae2cb23ad20b65fdd2e3 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 27 Jul 2024 17:39:39 +0200
Subject: [PATCH 0904/1488] Add a separate test for exception handling

---
 .../0_stateless/03212_thousand_exceptions.reference      | 1 +
 tests/queries/0_stateless/03212_thousand_exceptions.sh   | 9 +++++++++
 2 files changed, 10 insertions(+)
 create mode 100644 tests/queries/0_stateless/03212_thousand_exceptions.reference
 create mode 100755 tests/queries/0_stateless/03212_thousand_exceptions.sh

diff --git a/tests/queries/0_stateless/03212_thousand_exceptions.reference b/tests/queries/0_stateless/03212_thousand_exceptions.reference
new file mode 100644
index 00000000000..2409ba16fcb
--- /dev/null
+++ b/tests/queries/0_stateless/03212_thousand_exceptions.reference
@@ -0,0 +1 @@
+('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')('.')
\ No newline at end of file
diff --git a/tests/queries/0_stateless/03212_thousand_exceptions.sh b/tests/queries/0_stateless/03212_thousand_exceptions.sh
new file mode 100755
index 00000000000..0a6abf35c10
--- /dev/null
+++ b/tests/queries/0_stateless/03212_thousand_exceptions.sh
@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+# Tags: long
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+# This should not be too slow, even under sanitizers.
+yes "SELECT throwIf(1); SELECT '.' FORMAT Values;" | head -n 1000 | $CLICKHOUSE_CLIENT --multiquery --ignore-error 2>/dev/null

From 7ab1664fe543e64680ac6bea866d9f2c8052ba1d Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 27 Jul 2024 17:46:31 +0200
Subject: [PATCH 0905/1488] Debug test `01600_parts_states_metrics_long`

---
 .../01600_parts_states_metrics_long.sh            | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/tests/queries/0_stateless/01600_parts_states_metrics_long.sh b/tests/queries/0_stateless/01600_parts_states_metrics_long.sh
index 2e47034e528..cee7de3b1db 100755
--- a/tests/queries/0_stateless/01600_parts_states_metrics_long.sh
+++ b/tests/queries/0_stateless/01600_parts_states_metrics_long.sh
@@ -15,13 +15,22 @@ verify_sql="SELECT
 # In case of test failure, this code will do infinite loop and timeout.
 verify()
 {
-    while true
+    for i in {1..100}
     do
         result=$( $CLICKHOUSE_CLIENT -m --query="$verify_sql" )
-        [ "$result" = "1" ] && break
+        [ "$result" = "1" ] && echo "$result" && break
         sleep 0.1
+
+        if [[ $i -eq 100 ]]
+        then
+            $CLICKHOUSE_CLIENT --query "
+              SELECT sumIf(value, metric = 'PartsActive'), sumIf(value, metric = 'PartsOutdated') FROM system.metrics;
+              SELECT sum(active), sum(NOT active) FROM system.parts;
+              SELECT sum(active), sum(NOT active) FROM system.projection_parts;
+              SELECT count() FROM system.dropped_tables_parts;
+            "
+        fi
     done
-    echo 1
 }
 
 $CLICKHOUSE_CLIENT --database_atomic_wait_for_drop_and_detach_synchronously=1 --query="DROP TABLE IF EXISTS test_table"

From 4ec4b1f2f3f1806d479825e1e17d335b811555b4 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 27 Jul 2024 17:57:09 +0200
Subject: [PATCH 0906/1488] Faster test `02231_buffer_aggregate_states_leak`

---
 .../0_stateless/02231_buffer_aggregate_states_leak.sql    | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/queries/0_stateless/02231_buffer_aggregate_states_leak.sql b/tests/queries/0_stateless/02231_buffer_aggregate_states_leak.sql
index 3d6b69fe161..85915e3790e 100644
--- a/tests/queries/0_stateless/02231_buffer_aggregate_states_leak.sql
+++ b/tests/queries/0_stateless/02231_buffer_aggregate_states_leak.sql
@@ -1,4 +1,4 @@
--- Tags: long
+-- Tags: long, no-tsan
 
 drop table if exists buffer_02231;
 drop table if exists out_02231;
@@ -11,8 +11,8 @@ create table buffer_02231
 (
     key Int,
     v1 AggregateFunction(groupArray, String)
-) engine=Buffer(currentDatabase(), 'out_02231',
-    /* layers= */1,
+) engine = Buffer(currentDatabase(), 'out_02231',
+    /* layers= */       1,
     /* min/max time  */ 86400, 86400,
     /* min/max rows  */ 1e9, 1e9,
     /* min/max bytes */ 1e12, 1e12,
@@ -29,7 +29,7 @@ from in_02231
 group by key;
 
 set optimize_trivial_insert_select = 1;
-insert into in_02231 select * from numbers(10e6) settings max_memory_usage='400Mi', max_threads=1;
+insert into in_02231 select * from numbers(5e6) settings max_memory_usage='30Mi', max_threads=1, min_insert_block_size_rows = 10000, min_insert_block_size_bytes = '10Mi';
 
 drop table buffer_02231;
 drop table out_02231;

From 5917210a0247923e4745a6b5b063790fd41202b7 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 27 Jul 2024 19:51:05 +0200
Subject: [PATCH 0907/1488] Update 01600_parts_states_metrics_long.sh

---
 tests/queries/0_stateless/01600_parts_states_metrics_long.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/queries/0_stateless/01600_parts_states_metrics_long.sh b/tests/queries/0_stateless/01600_parts_states_metrics_long.sh
index cee7de3b1db..f535d888446 100755
--- a/tests/queries/0_stateless/01600_parts_states_metrics_long.sh
+++ b/tests/queries/0_stateless/01600_parts_states_metrics_long.sh
@@ -17,13 +17,13 @@ verify()
 {
     for i in {1..100}
     do
-        result=$( $CLICKHOUSE_CLIENT -m --query="$verify_sql" )
+        result=$( $CLICKHOUSE_CLIENT --query="$verify_sql" )
         [ "$result" = "1" ] && echo "$result" && break
         sleep 0.1
 
         if [[ $i -eq 100 ]]
         then
-            $CLICKHOUSE_CLIENT --query "
+            $CLICKHOUSE_CLIENT --multiquery "
               SELECT sumIf(value, metric = 'PartsActive'), sumIf(value, metric = 'PartsOutdated') FROM system.metrics;
               SELECT sum(active), sum(NOT active) FROM system.parts;
               SELECT sum(active), sum(NOT active) FROM system.projection_parts;

From 5ac26d2115c9e3e777990fc932bf03e83a2effee Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 27 Jul 2024 20:36:34 +0200
Subject: [PATCH 0908/1488] Fix curiosities in TimerDescriptor

---
 src/Common/TimerDescriptor.cpp | 44 +++++++++++++++++++++++++---------
 1 file changed, 33 insertions(+), 11 deletions(-)

diff --git a/src/Common/TimerDescriptor.cpp b/src/Common/TimerDescriptor.cpp
index 248febe226e..2890ddec14c 100644
--- a/src/Common/TimerDescriptor.cpp
+++ b/src/Common/TimerDescriptor.cpp
@@ -1,11 +1,12 @@
 #if defined(OS_LINUX)
+
 #include <Common/TimerDescriptor.h>
 #include <Common/Exception.h>
 
 #include <sys/timerfd.h>
-#include <fcntl.h>
 #include <unistd.h>
 
+
 namespace DB
 {
 
@@ -19,15 +20,13 @@ namespace ErrorCodes
 
 TimerDescriptor::TimerDescriptor(int clockid, int flags)
 {
-    timer_fd = timerfd_create(clockid, flags);
+    timer_fd = timerfd_create(clockid, flags | TFD_NONBLOCK);
     if (timer_fd == -1)
-        throw Exception(ErrorCodes::CANNOT_CREATE_TIMER, "Cannot create timer_fd descriptor");
-
-    if (-1 == fcntl(timer_fd, F_SETFL, O_NONBLOCK))
-        throw ErrnoException(ErrorCodes::CANNOT_FCNTL, "Cannot set O_NONBLOCK for timer_fd");
+        throw ErrnoException(ErrorCodes::CANNOT_CREATE_TIMER, "Cannot create timer_fd descriptor");
 }
 
-TimerDescriptor::TimerDescriptor(TimerDescriptor && other) noexcept : timer_fd(other.timer_fd)
+TimerDescriptor::TimerDescriptor(TimerDescriptor && other) noexcept
+    : timer_fd(other.timer_fd)
 {
     other.timer_fd = -1;
 }
@@ -43,13 +42,16 @@ TimerDescriptor::~TimerDescriptor()
     /// Do not check for result cause cannot throw exception.
     if (timer_fd != -1)
     {
-        int err = close(timer_fd);
-        chassert(!err || errno == EINTR);
+        if (0 != ::close(timer_fd))
+            std::terminate();
     }
 }
 
 void TimerDescriptor::reset() const
 {
+    if (timer_fd == -1)
+        return;
+
     itimerspec spec;
     spec.it_interval.tv_nsec = 0;
     spec.it_interval.tv_sec = 0;
@@ -66,25 +68,44 @@ void TimerDescriptor::reset() const
 
 void TimerDescriptor::drain() const
 {
+    if (timer_fd == -1)
+        return;
+
     /// It is expected that socket returns 8 bytes when readable.
     /// Read in loop anyway cause signal may interrupt read call.
+
+    /// man timerfd_create:
+    /// If the timer has already expired one or more times since its settings were last modified using timerfd_settime(),
+    /// or since the last successful read(2), then the buffer given to read(2) returns an unsigned 8-byte integer (uint64_t)
+    /// containing the number of expirations that have occurred.
+    /// (The returned value is in host byte order—that is, the native byte order for integers on the host machine.)
     uint64_t buf;
     while (true)
     {
         ssize_t res = ::read(timer_fd, &buf, sizeof(buf));
         if (res < 0)
         {
+            /// man timerfd_create:
+            /// If no timer expirations have occurred at the time of the read(2),
+            /// then the call either blocks until the next timer expiration, or fails with the error EAGAIN
+            /// if the file descriptor has been made nonblocking
+            /// (via the use of the fcntl(2) F_SETFL operation to set the O_NONBLOCK flag).
             if (errno == EAGAIN)
                 break;
 
-            if (errno != EINTR)
-                throw ErrnoException(ErrorCodes::CANNOT_READ_FROM_SOCKET, "Cannot drain timer_fd");
+            /// A signal happened, need to retry.
+            if (errno == EINTR)
+                continue;
+
+            throw ErrnoException(ErrorCodes::CANNOT_READ_FROM_SOCKET, "Cannot drain timer_fd");
         }
     }
 }
 
 void TimerDescriptor::setRelative(uint64_t usec) const
 {
+    chassert(timer_fd >= 0);
+
     static constexpr uint32_t TIMER_PRECISION = 1e6;
 
     itimerspec spec;
@@ -103,4 +124,5 @@ void TimerDescriptor::setRelative(Poco::Timespan timespan) const
 }
 
 }
+
 #endif

From 299d0fb39eaf3698994a98a65b7f01f191d2b65d Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 27 Jul 2024 20:43:15 +0200
Subject: [PATCH 0909/1488] Fix curiosities in TimerDescriptor

---
 src/Common/TimerDescriptor.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/Common/TimerDescriptor.cpp b/src/Common/TimerDescriptor.cpp
index 2890ddec14c..cb34170191a 100644
--- a/src/Common/TimerDescriptor.cpp
+++ b/src/Common/TimerDescriptor.cpp
@@ -39,7 +39,6 @@ TimerDescriptor & TimerDescriptor::operator=(DB::TimerDescriptor && other) noexc
 
 TimerDescriptor::~TimerDescriptor()
 {
-    /// Do not check for result cause cannot throw exception.
     if (timer_fd != -1)
     {
         if (0 != ::close(timer_fd))

From 377b78119462e3188f6261ae1c3203c999f75159 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 27 Jul 2024 20:47:56 +0200
Subject: [PATCH 0910/1488] Fix curiosities in TimerDescriptor

---
 src/Common/TimerDescriptor.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Common/TimerDescriptor.cpp b/src/Common/TimerDescriptor.cpp
index cb34170191a..ba9f293d147 100644
--- a/src/Common/TimerDescriptor.cpp
+++ b/src/Common/TimerDescriptor.cpp
@@ -18,9 +18,9 @@ namespace ErrorCodes
     extern const int CANNOT_READ_FROM_SOCKET;
 }
 
-TimerDescriptor::TimerDescriptor(int clockid, int flags)
+TimerDescriptor::TimerDescriptor()
 {
-    timer_fd = timerfd_create(clockid, flags | TFD_NONBLOCK);
+    timer_fd = timerfd_create(CLOCK_MONOTONIC, TFD_NONBLOCK | TFD_CLOEXEC);
     if (timer_fd == -1)
         throw ErrnoException(ErrorCodes::CANNOT_CREATE_TIMER, "Cannot create timer_fd descriptor");
 }

From 00b0fa961a225e8d95561d6e0114cac1d86a9a6c Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 27 Jul 2024 20:50:21 +0200
Subject: [PATCH 0911/1488] Removed useless parameters

---
 src/Common/TimerDescriptor.h                       | 2 +-
 src/QueryPipeline/RemoteQueryExecutor.h            | 1 -
 src/QueryPipeline/RemoteQueryExecutorReadContext.h | 2 +-
 3 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/Common/TimerDescriptor.h b/src/Common/TimerDescriptor.h
index 0292f85d770..8d3b22868ac 100644
--- a/src/Common/TimerDescriptor.h
+++ b/src/Common/TimerDescriptor.h
@@ -12,7 +12,7 @@ private:
     int timer_fd;
 
 public:
-    explicit TimerDescriptor(int clockid = CLOCK_MONOTONIC, int flags = 0);
+    TimerDescriptor();
     ~TimerDescriptor();
 
     TimerDescriptor(const TimerDescriptor &) = delete;
diff --git a/src/QueryPipeline/RemoteQueryExecutor.h b/src/QueryPipeline/RemoteQueryExecutor.h
index 04a59cc3b7e..78e0b02c38b 100644
--- a/src/QueryPipeline/RemoteQueryExecutor.h
+++ b/src/QueryPipeline/RemoteQueryExecutor.h
@@ -8,7 +8,6 @@
 #include <Storages/IStorage_fwd.h>
 #include <Interpreters/Context.h>
 #include <Interpreters/StorageID.h>
-#include <Common/TimerDescriptor.h>
 #include <sys/types.h>
 
 
diff --git a/src/QueryPipeline/RemoteQueryExecutorReadContext.h b/src/QueryPipeline/RemoteQueryExecutorReadContext.h
index b8aa8bb9111..fbd55278f86 100644
--- a/src/QueryPipeline/RemoteQueryExecutorReadContext.h
+++ b/src/QueryPipeline/RemoteQueryExecutorReadContext.h
@@ -69,7 +69,7 @@ private:
     /// * timer is a timerfd descriptor to manually check socket timeout
     /// * pipe_fd is a pipe we use to cancel query and socket polling by executor.
     /// We put those descriptors into our own epoll which is used by external executor.
-    TimerDescriptor timer{CLOCK_MONOTONIC, 0};
+    TimerDescriptor timer;
     Poco::Timespan timeout;
     AsyncEventTimeoutType timeout_type;
     std::atomic_bool is_timer_alarmed = false;

From 0967bfadd13c358764cf33d3cd78456a129faf91 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 27 Jul 2024 20:58:53 +0200
Subject: [PATCH 0912/1488] Simpler

---
 src/Common/TimerDescriptor.cpp | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/src/Common/TimerDescriptor.cpp b/src/Common/TimerDescriptor.cpp
index ba9f293d147..dd6ae70511e 100644
--- a/src/Common/TimerDescriptor.cpp
+++ b/src/Common/TimerDescriptor.cpp
@@ -51,11 +51,7 @@ void TimerDescriptor::reset() const
     if (timer_fd == -1)
         return;
 
-    itimerspec spec;
-    spec.it_interval.tv_nsec = 0;
-    spec.it_interval.tv_sec = 0;
-    spec.it_value.tv_sec = 0;
-    spec.it_value.tv_nsec = 0;
+    itimerspec spec{};
 
     if (-1 == timerfd_settime(timer_fd, 0 /*relative timer */, &spec, nullptr))
         throw ErrnoException(ErrorCodes::CANNOT_SET_TIMER_PERIOD, "Cannot reset timer_fd");

From 3dfa32498ce33565deb3d2009c0a5053c46cce98 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 27 Jul 2024 21:06:08 +0200
Subject: [PATCH 0913/1488] Add assert

---
 src/Common/TimerDescriptor.cpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/Common/TimerDescriptor.cpp b/src/Common/TimerDescriptor.cpp
index dd6ae70511e..c7598546799 100644
--- a/src/Common/TimerDescriptor.cpp
+++ b/src/Common/TimerDescriptor.cpp
@@ -94,6 +94,8 @@ void TimerDescriptor::drain() const
 
             throw ErrnoException(ErrorCodes::CANNOT_READ_FROM_SOCKET, "Cannot drain timer_fd");
         }
+
+        chassert(res == sizeof(buf));
     }
 }
 

From e0291e3a06d1f5ccd51f894d2cc9c3582911dc94 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 27 Jul 2024 21:16:11 +0200
Subject: [PATCH 0914/1488] Prevent closing a file twice, even in case of
 errors

---
 src/IO/AsynchronousReadBufferFromFile.cpp | 3 +++
 src/IO/MMapReadBufferFromFile.cpp         | 3 +++
 src/IO/MMappedFile.cpp                    | 3 +++
 src/IO/OpenedFile.cpp                     | 4 +++-
 src/IO/ReadBufferFromFile.cpp             | 3 +++
 src/IO/WriteBufferFromFile.cpp            | 3 +++
 6 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/src/IO/AsynchronousReadBufferFromFile.cpp b/src/IO/AsynchronousReadBufferFromFile.cpp
index c6fe16a7f14..4d148a5c9f9 100644
--- a/src/IO/AsynchronousReadBufferFromFile.cpp
+++ b/src/IO/AsynchronousReadBufferFromFile.cpp
@@ -93,7 +93,10 @@ void AsynchronousReadBufferFromFile::close()
         return;
 
     if (0 != ::close(fd))
+    {
+        fd = -1;
         throw Exception(ErrorCodes::CANNOT_CLOSE_FILE, "Cannot close file");
+    }
 
     fd = -1;
 }
diff --git a/src/IO/MMapReadBufferFromFile.cpp b/src/IO/MMapReadBufferFromFile.cpp
index d3eb11c920d..d0865269f1b 100644
--- a/src/IO/MMapReadBufferFromFile.cpp
+++ b/src/IO/MMapReadBufferFromFile.cpp
@@ -77,7 +77,10 @@ void MMapReadBufferFromFile::close()
     finish();
 
     if (0 != ::close(fd))
+    {
+        fd = -1;
         throw Exception(ErrorCodes::CANNOT_CLOSE_FILE, "Cannot close file");
+    }
 
     fd = -1;
     metric_increment.destroy();
diff --git a/src/IO/MMappedFile.cpp b/src/IO/MMappedFile.cpp
index 7249a25decb..26949061274 100644
--- a/src/IO/MMappedFile.cpp
+++ b/src/IO/MMappedFile.cpp
@@ -69,7 +69,10 @@ void MMappedFile::close()
     finish();
 
     if (0 != ::close(fd))
+    {
+        fd = -1;
         throw Exception(ErrorCodes::CANNOT_CLOSE_FILE, "Cannot close file");
+    }
 
     fd = -1;
     metric_increment.destroy();
diff --git a/src/IO/OpenedFile.cpp b/src/IO/OpenedFile.cpp
index 4677a8259db..c51dc430b35 100644
--- a/src/IO/OpenedFile.cpp
+++ b/src/IO/OpenedFile.cpp
@@ -67,11 +67,13 @@ void OpenedFile::close()
         return;
 
     if (0 != ::close(fd))
+    {
+        fd = -1;
         throw Exception(ErrorCodes::CANNOT_CLOSE_FILE, "Cannot close file");
+    }
 
     fd = -1;
     metric_increment.destroy();
 }
 
 }
-
diff --git a/src/IO/ReadBufferFromFile.cpp b/src/IO/ReadBufferFromFile.cpp
index cb987171bad..2e5b9fcf753 100644
--- a/src/IO/ReadBufferFromFile.cpp
+++ b/src/IO/ReadBufferFromFile.cpp
@@ -88,7 +88,10 @@ void ReadBufferFromFile::close()
         return;
 
     if (0 != ::close(fd))
+    {
+        fd = -1;
         throw Exception(ErrorCodes::CANNOT_CLOSE_FILE, "Cannot close file");
+    }
 
     fd = -1;
     metric_increment.destroy();
diff --git a/src/IO/WriteBufferFromFile.cpp b/src/IO/WriteBufferFromFile.cpp
index 37b1161356f..f1825ce1e22 100644
--- a/src/IO/WriteBufferFromFile.cpp
+++ b/src/IO/WriteBufferFromFile.cpp
@@ -116,7 +116,10 @@ void WriteBufferFromFile::close()
         finalize();
 
     if (0 != ::close(fd))
+    {
+        fd = -1;
         throw Exception(ErrorCodes::CANNOT_CLOSE_FILE, "Cannot close file");
+    }
 
     fd = -1;
     metric_increment.destroy();

From f5576867748c8941cfc13360cc28526f2703e575 Mon Sep 17 00:00:00 2001
From: Nikita Taranov <nikita.taranov@clickhouse.com>
Date: Sat, 27 Jul 2024 21:24:13 +0100
Subject: [PATCH 0915/1488] cherry on top of this cake

---
 cmake/sanitize.cmake | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cmake/sanitize.cmake b/cmake/sanitize.cmake
index 3f111bc65c4..d8d211b9329 100644
--- a/cmake/sanitize.cmake
+++ b/cmake/sanitize.cmake
@@ -58,6 +58,7 @@ option(WITH_COVERAGE "Instrumentation for code coverage with default implementat
 if (WITH_COVERAGE)
     message (STATUS "Enabled instrumentation for code coverage")
     set(COVERAGE_FLAGS "SHELL:-fprofile-instr-generate -fcoverage-mapping")
+    set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fprofile-instr-generate -fcoverage-mapping")
 endif()
 
 option (SANITIZE_COVERAGE "Instrumentation for code coverage with custom callbacks" OFF)

From 408daa0bf44dde0213f9c5bfd722a411e68e6dd2 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 27 Jul 2024 23:00:32 +0200
Subject: [PATCH 0916/1488] Update TimerDescriptor.cpp

---
 src/Common/TimerDescriptor.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/Common/TimerDescriptor.cpp b/src/Common/TimerDescriptor.cpp
index c7598546799..b36ea4059cb 100644
--- a/src/Common/TimerDescriptor.cpp
+++ b/src/Common/TimerDescriptor.cpp
@@ -14,7 +14,6 @@ namespace ErrorCodes
 {
     extern const int CANNOT_CREATE_TIMER;
     extern const int CANNOT_SET_TIMER_PERIOD;
-    extern const int CANNOT_FCNTL;
     extern const int CANNOT_READ_FROM_SOCKET;
 }
 

From 61cc5a1e5440c126098546baee4a29af1a3df9ab Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 27 Jul 2024 23:17:38 +0200
Subject: [PATCH 0917/1488] Add retries to test
 `02911_backup_restore_keeper_map`

---
 programs/server/config.d/backups.xml          | 13 ++++++++++++
 .../server/config.d/enable_keeper_map.xml     |  1 +
 .../02911_backup_restore_keeper_map.sh        | 20 +++++++++++++++----
 3 files changed, 30 insertions(+), 4 deletions(-)
 create mode 100644 programs/server/config.d/backups.xml
 create mode 120000 programs/server/config.d/enable_keeper_map.xml

diff --git a/programs/server/config.d/backups.xml b/programs/server/config.d/backups.xml
new file mode 100644
index 00000000000..382794e1123
--- /dev/null
+++ b/programs/server/config.d/backups.xml
@@ -0,0 +1,13 @@
+<clickhouse>
+    <storage_configuration>
+        <disks>
+            <backups>
+                <type>local</type>
+                <path>/tmp/backups/</path>
+            </backups>
+        </disks>
+    </storage_configuration>
+    <backups>
+        <allowed_disk>backups</allowed_disk>
+    </backups>
+</clickhouse>
diff --git a/programs/server/config.d/enable_keeper_map.xml b/programs/server/config.d/enable_keeper_map.xml
new file mode 120000
index 00000000000..47c36159882
--- /dev/null
+++ b/programs/server/config.d/enable_keeper_map.xml
@@ -0,0 +1 @@
+../../../tests/config/config.d/enable_keeper_map.xml
\ No newline at end of file
diff --git a/tests/queries/0_stateless/02911_backup_restore_keeper_map.sh b/tests/queries/0_stateless/02911_backup_restore_keeper_map.sh
index ccdf52a6e23..385583acbbe 100755
--- a/tests/queries/0_stateless/02911_backup_restore_keeper_map.sh
+++ b/tests/queries/0_stateless/02911_backup_restore_keeper_map.sh
@@ -11,11 +11,23 @@ $CLICKHOUSE_CLIENT -nm -q "
     CREATE TABLE $database_name.02911_backup_restore_keeper_map1 (key UInt64, value String) Engine=KeeperMap('/' || currentDatabase() || '/test02911') PRIMARY KEY key;
     CREATE TABLE $database_name.02911_backup_restore_keeper_map2 (key UInt64, value String) Engine=KeeperMap('/' || currentDatabase() || '/test02911') PRIMARY KEY key; -- table using same Keeper path as 02911_backup_restore_keeper_map1
     CREATE TABLE $database_name.02911_backup_restore_keeper_map3 (key UInt64, value String) Engine=KeeperMap('/' || currentDatabase() || '/test02911_different') PRIMARY KEY key;
-
-    INSERT INTO $database_name.02911_backup_restore_keeper_map2 SELECT number, 'test' || toString(number) FROM system.numbers LIMIT 5000;
-    INSERT INTO $database_name.02911_backup_restore_keeper_map3 SELECT number, 'test' || toString(number) FROM system.numbers LIMIT 3000;
 "
 
+# KeeperMap table engine doesn't have internal retries for interaction with Keeper. Do it on our own, otherwise tests with overloaded server can be flaky.
+while true
+do
+    $CLICKHOUSE_CLIENT -nm -q "INSERT INTO $database_name.02911_backup_restore_keeper_map2 SELECT number, 'test' || toString(number) FROM system.numbers LIMIT 5000;
+    " | grep -q "KEEPER_EXCEPTION" && sleep 1 && continue
+    break
+done
+
+while true
+do
+    $CLICKHOUSE_CLIENT -nm -q "INSERT INTO $database_name.02911_backup_restore_keeper_map3 SELECT number, 'test' || toString(number) FROM system.numbers LIMIT 3000;
+    " | grep -q "KEEPER_EXCEPTION" && sleep 1 && continue
+    break
+done
+
 backup_path="$database_name"
 for i in $(seq 1 3); do
     $CLICKHOUSE_CLIENT -q "SELECT count() FROM $database_name.02911_backup_restore_keeper_map$i;"
@@ -45,4 +57,4 @@ for i in $(seq 1 3); do
     $CLICKHOUSE_CLIENT -q "SELECT count() FROM $database_name.02911_backup_restore_keeper_map$i;"
 done
 
-$CLICKHOUSE_CLIENT -q "DROP DATABASE $database_name SYNC;"
\ No newline at end of file
+$CLICKHOUSE_CLIENT -q "DROP DATABASE $database_name SYNC;"

From dfa5abc754bbbbd46c6cfc5db3ab76d38ef121e1 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 27 Jul 2024 23:22:49 +0200
Subject: [PATCH 0918/1488] Update 02231_buffer_aggregate_states_leak.sql

---
 .../queries/0_stateless/02231_buffer_aggregate_states_leak.sql  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/02231_buffer_aggregate_states_leak.sql b/tests/queries/0_stateless/02231_buffer_aggregate_states_leak.sql
index 85915e3790e..5e6b4393a79 100644
--- a/tests/queries/0_stateless/02231_buffer_aggregate_states_leak.sql
+++ b/tests/queries/0_stateless/02231_buffer_aggregate_states_leak.sql
@@ -29,7 +29,7 @@ from in_02231
 group by key;
 
 set optimize_trivial_insert_select = 1;
-insert into in_02231 select * from numbers(5e6) settings max_memory_usage='30Mi', max_threads=1, min_insert_block_size_rows = 10000, min_insert_block_size_bytes = '10Mi';
+insert into in_02231 select * from numbers(5e6) settings max_memory_usage='40Mi', max_threads=1, min_insert_block_size_rows = 10000, min_insert_block_size_bytes = '10Mi';
 
 drop table buffer_02231;
 drop table out_02231;

From e8c2e1745ee80db01ce164e472c3b0fac31a65b7 Mon Sep 17 00:00:00 2001
From: Shaun Struwig <41984034+Blargian@users.noreply.github.com>
Date: Sun, 28 Jul 2024 00:05:24 +0200
Subject: [PATCH 0919/1488] Fix typo

---
 docs/en/sql-reference/functions/type-conversion-functions.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/en/sql-reference/functions/type-conversion-functions.md b/docs/en/sql-reference/functions/type-conversion-functions.md
index ee253e46014..37541dd4818 100644
--- a/docs/en/sql-reference/functions/type-conversion-functions.md
+++ b/docs/en/sql-reference/functions/type-conversion-functions.md
@@ -496,7 +496,7 @@ toInt16OrDefault(expr, def)
 **Arguments**
 
 - `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
-- `def` — The default value to return if parsing to type `Int16` is unsuccessful. [Int8](../data-types/int-uint.md).
+- `def` — The default value to return if parsing to type `Int16` is unsuccessful. [Int16](../data-types/int-uint.md).
 
 Supported arguments:
 - Values of type (U)Int8/16/32/64/128/256.

From be22d76942d7306bbe125b80ac8397e788dac65a Mon Sep 17 00:00:00 2001
From: Blargian <shaunstruwig.ss@gmail.com>
Date: Sun, 28 Jul 2024 00:38:07 +0200
Subject: [PATCH 0920/1488] add UInt* functions

---
 .../functions/type-conversion-functions.md    | 1484 ++++++++++++++++-
 1 file changed, 1466 insertions(+), 18 deletions(-)

diff --git a/docs/en/sql-reference/functions/type-conversion-functions.md b/docs/en/sql-reference/functions/type-conversion-functions.md
index 61e84ca72d1..bb386092c0f 100644
--- a/docs/en/sql-reference/functions/type-conversion-functions.md
+++ b/docs/en/sql-reference/functions/type-conversion-functions.md
@@ -148,50 +148,1498 @@ Result:
 └─────────────────────────────────────────────────┴──────────────────────────────────────────────────┘
 ```
 
+# toUInt8
 
-## toUInt(8\|16\|32\|64\|256)
+Converts an input value to a value of type [`UInt8`](../data-types/int-uint.md). Throws an exception in case of an error.
 
-Converts an input value to the [UInt](../data-types/int-uint.md) data type. This function family includes:
+**Syntax**
 
-- `toUInt8(expr)` — Converts to a value of data type `UInt8`.
-- `toUInt16(expr)` — Converts to a value of data type `UInt16`.
-- `toUInt32(expr)` — Converts to a value of data type `UInt32`.
-- `toUInt64(expr)` — Converts to a value of data type `UInt64`.
-- `toUInt256(expr)` — Converts to a value of data type `UInt256`.
+```sql
+toUInt8(expr)
+```
 
 **Arguments**
 
-- `expr` — [Expression](../syntax.md/#syntax-expressions) returning a number or a string with the decimal representation of a number. Binary, octal, and hexadecimal representations of numbers are not supported. Leading zeroes are stripped.
+- `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions).
+
+Supported arguments:
+- Values of type (U)Int8/16/32/64/128/256.
+- Values of type Float32/64.
+- String representations of (U)Int8/16/32/128/256.
+
+Unsupported arguments:
+- String representations of Float32/64 values, including `NaN` and `Inf`.
+- String representations of binary and hexadecimal values, e.g. `SELECT toUInt8('0xc0fe');`.
+
+:::note
+If the input value cannot be represented within the bounds of [UInt8](../data-types/int-uint.md), overflow or underflow of the result occurs.
+This is not considered an error.  
+For example: `SELECT toUInt8(256) == 0;`.
+:::
 
 **Returned value**
 
-- Integer value in the `UInt8`, `UInt16`, `UInt32`, `UInt64` or `UInt256` data type.
+- 8-bit unsigned integer value. [UInt8](../data-types/int-uint.md).
 
-Functions use [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning they truncate fractional digits of numbers.
+:::note
+The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
+:::
 
-The behavior of functions for negative arguments and for the [NaN and Inf](../data-types/float.md/#data_type-float-nan-inf) arguments is undefined. If you pass a string with a negative number, for example `'-32'`, ClickHouse raises an exception. Remember about [numeric conversions issues](#common-issues-with-data-conversion), when using the functions.
+**Example**
+
+Query:
+
+```sql
+SELECT
+    toUInt8(8),
+    toUInt8(8.8),
+    toUInt8('8')
+FORMAT vertical;
+```
+
+Result:
+
+```response
+Row 1:
+──────
+toUInt8(8):   8
+toUInt8(8.8): 8
+toUInt8('8'): 8
+```
+
+**See also**
+
+- [`toUInt8OrZero`](#touint8orzero).
+- [`toUInt8OrNull`](#touint8ornull).
+- [`toUInt8OrDefault`](#touint8ordefault).
+
+## toUInt8OrZero
+
+Like [`toUInt8`](#touint8), this function converts an input value to a value of type [UInt8](../data-types/int-uint.md) but returns `0` in case of an error.
+
+**Syntax**
+
+```sql
+toUInt8OrZero(x)
+```
+
+**Arguments**
+
+- `x` — A String representation of a number. [String](../data-types/string.md).
+
+Supported arguments:
+- String representations of (U)Int8/16/32/128/256.
+
+Unsupported arguments (return `0`):
+- String representations of ordinary Float32/64 values, including `NaN` and `Inf`.
+- String representations of binary and hexadecimal values, e.g. `SELECT toUInt8OrZero('0xc0fe');`.
+
+:::note
+If the input value cannot be represented within the bounds of [UInt8](../data-types/int-uint.md), overflow or underflow of the result occurs.
+This is not considered an error.
+:::
+
+**Returned value**
+
+- 8-bit unsigned integer value if successful, otherwise `0`. [UInt8](../data-types/int-uint.md).
+
+:::note
+The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
+:::
 
 **Example**
 
 Query:
 
 ``` sql
-SELECT toUInt64(nan), toUInt32(-32), toUInt16('16'), toUInt8(8.8);
+SELECT
+    toUInt8OrZero('-8'),
+    toUInt8OrZero('abc')
+FORMAT vertical;
 ```
 
 Result:
 
 ```response
-┌───────toUInt64(nan)─┬─toUInt32(-32)─┬─toUInt16('16')─┬─toUInt8(8.8)─┐
-│ 9223372036854775808 │    4294967264 │             16 │            8 │
-└─────────────────────┴───────────────┴────────────────┴──────────────┘
+Row 1:
+──────
+toUInt8OrZero('-8'):  0
+toUInt8OrZero('abc'): 0
 ```
 
-## toUInt(8\|16\|32\|64\|256)OrZero
+**See also**
 
-## toUInt(8\|16\|32\|64\|256)OrNull
+- [`toUInt8`](#touint8).
+- [`toUInt8OrNull`](#touint8ornull).
+- [`toUInt8OrDefault`](#touint8ordefault).
 
-## toUInt(8\|16\|32\|64\|256)OrDefault
+## toUInt8OrNull
+
+Like [`toUInt8`](#touint8), this function converts an input value to a value of type [UInt8](../data-types/int-uint.md) but returns `NULL` in case of an error.
+
+**Syntax**
+
+```sql
+toUInt8OrNull(x)
+```
+
+**Arguments**
+
+- `x` — A String representation of a number. [String](../data-types/string.md).
+
+Supported arguments:
+- String representations of (U)Int8/16/32/128/256.
+
+Unsupported arguments (return `\N`)
+- String representations of Float32/64 values, including `NaN` and `Inf`.
+- String representations of binary and hexadecimal values, e.g. `SELECT toUInt8OrNull('0xc0fe');`.
+
+:::note
+If the input value cannot be represented within the bounds of [UInt8](../data-types/int-uint.md), overflow or underflow of the result occurs.
+This is not considered an error.
+:::
+
+**Returned value**
+
+- 8-bit unsigned integer value if successful, otherwise `NULL`. [UInt8](../data-types/int-uint.md) / [NULL](../data-types/nullable.md).
+
+:::note
+The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
+:::
+
+**Example**
+
+Query:
+
+``` sql
+SELECT
+    toUInt8OrNull('-8'),
+    toUInt8OrNull('abc')
+FORMAT vertical;
+```
+
+Result:
+
+```response
+Row 1:
+──────
+toUInt8OrNull('-8'):  ᴺᵁᴸᴸ
+toUInt8OrNull('abc'): ᴺᵁᴸᴸ
+```
+
+**See also**
+
+- [`toUInt8`](#touint8).
+- [`toUInt8OrZero`](#touint8orzero).
+- [`toUInt8OrDefault`](#touint8ordefault).
+
+## toUInt8OrDefault
+
+Like [`toUInt8`](#touint8), takes an argument of type [String](../data-types/string.md) and tries to parse it to type [`UInt8`](../data-types/int-uint.md). If unsuccessful, returns the default type value.
+
+**Syntax**
+
+```sql
+toUInt8OrDefault(expr, def)
+```
+
+**Arguments**
+
+- `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
+- `def` — The default value to return if parsing to type `UInt8` is unsuccessful. [UInt8](../data-types/int-uint.md).
+
+Supported arguments:
+- Values of type (U)Int8/16/32/64/128/256.
+- Values of type Float32/64.
+- String representations of (U)Int8/16/32/128/256.
+
+Arguments for which the default value is returned:
+- String representations of Float32/64 values, including `NaN` and `Inf`.
+- String representations of binary and hexadecimal values, e.g. `SELECT toUInt8OrDefault('0xc0fe', CAST('-1', 'UInt8'));`.
+
+:::note
+If the input value cannot be represented within the bounds of [UInt8](../data-types/int-uint.md), overflow or underflow of the result occurs.
+This is not considered an error.
+:::
+
+**Returned value**
+
+- 8-bit unsigned integer value if successful, otherwise returns the default value. [UInt8](../data-types/int-uint.md).
+
+:::note
+- The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
+- The default value type should be the same as the cast type.
+:::
+
+**Example**
+
+Query:
+
+``` sql
+SELECT
+    toUInt8OrDefault('8', CAST('0', 'UInt8')),
+    toUInt8OrDefault('abc', CAST('0', 'UInt8'))
+FORMAT vertical;
+```
+
+Result:
+
+```response
+Row 1:
+──────
+toUInt8OrDefault('8', CAST('0', 'UInt8')):   8
+toUInt8OrDefault('abc', CAST('0', 'UInt8')): 0
+```
+
+**See also**
+
+- [`toUInt8`](#touint8).
+- [`toUInt8OrZero`](#touint8orzero).
+- [`toUInt8OrNull`](#touint8orNull).
+
+## toUInt16
+
+Converts an input value to a value of type [`UInt16`](../data-types/int-uint.md). Throws an exception in case of an error.
+
+**Syntax**
+
+```sql
+toUInt16(expr)
+```
+
+**Arguments**
+
+- `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions).
+
+Supported arguments:
+- Values of type (U)Int8/16/32/64/128/256.
+- Values of type Float32/64.
+- String representations of (U)Int8/16/32/128/256.
+
+Unsupported arguments:
+- String representations of Float32/64 values, including `NaN` and `Inf`.
+- String representations of binary and hexadecimal values, e.g. `SELECT toUInt16('0xc0fe');`.
+
+:::note
+If the input value cannot be represented within the bounds of [UInt16](../data-types/int-uint.md), overflow or underflow of the result occurs.
+This is not considered an error.  
+For example: `SELECT toUInt16(65536) == 0;`.
+:::
+
+**Returned value**
+
+- 16-bit unsigned integer value. [UInt16](../data-types/int-uint.md).
+
+:::note
+The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
+:::
+
+**Example**
+
+Query:
+
+```sql
+SELECT
+    toUInt16(16),
+    toUInt16(16.16),
+    toUInt16('16')
+FORMAT vertical;
+```
+
+Result:
+
+```response
+Row 1:
+──────
+toUInt16(16):    16
+toUInt16(16.16): 16
+toUInt16('16'):  16
+```
+
+**See also**
+
+- [`toUInt16OrZero`](#touint16orzero).
+- [`toUInt16OrNull`](#touint16ornull).
+- [`toUInt16OrDefault`](#touint16ordefault).
+
+## toUInt16OrZero
+
+Like [`toUInt16`](#touint16), this function converts an input value to a value of type [UInt16](../data-types/int-uint.md) but returns `0` in case of an error.
+
+**Syntax**
+
+```sql
+toUInt16OrZero(x)
+```
+
+**Arguments**
+
+- `x` — A String representation of a number. [String](../data-types/string.md).
+
+Supported arguments:
+- String representations of (U)Int8/16/32/128/256.
+
+Unsupported arguments (return `0`):
+- String representations of Float32/64 values, including `NaN` and `Inf`.
+- String representations of binary and hexadecimal values, e.g. `SELECT toUInt16OrZero('0xc0fe');`.
+
+:::note
+If the input value cannot be represented within the bounds of [UInt16](../data-types/int-uint.md), overflow or underflow of the result occurs.
+This is not considered as an error.
+:::
+
+**Returned value**
+
+- 16-bit unsigned integer value if successful, otherwise `0`. [UInt16](../data-types/int-uint.md).
+
+:::note
+The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
+:::
+
+**Example**
+
+Query:
+
+``` sql
+SELECT
+    toUInt16OrZero('16'),
+    toUInt16OrZero('abc')
+FORMAT vertical;
+```
+
+Result:
+
+```response
+Row 1:
+──────
+toUInt16OrZero('16'):  16
+toUInt16OrZero('abc'): 0
+```
+
+**See also**
+
+- [`toUInt16`](#touint16).
+- [`toUInt16OrNull`](#touint16ornull).
+- [`toUInt16OrDefault`](#touint16ordefault).
+
+## toUInt16OrNull
+
+Like [`toUInt16`](#touint16), this function converts an input value to a value of type [UInt16](../data-types/int-uint.md) but returns `NULL` in case of an error.
+
+**Syntax**
+
+```sql
+toUInt16OrNull(x)
+```
+
+**Arguments**
+
+- `x` — A String representation of a number. [String](../data-types/string.md).
+
+Supported arguments:
+- String representations of (U)Int8/16/32/128/256.
+
+Unsupported arguments (return `\N`)
+- String representations of Float32/64 values, including `NaN` and `Inf`.
+- String representations of binary and hexadecimal values, e.g. `SELECT toUInt16OrNull('0xc0fe');`.
+
+:::note
+If the input value cannot be represented within the bounds of [UInt16](../data-types/int-uint.md), overflow or underflow of the result occurs.
+This is not considered an error.
+:::
+
+**Returned value**
+
+- 16-bit unsigned integer value if successful, otherwise `NULL`. [UInt16](../data-types/int-uint.md) / [NULL](../data-types/nullable.md).
+
+:::note
+The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
+:::
+
+**Example**
+
+Query:
+
+``` sql
+SELECT
+    toUInt16OrNull('16'),
+    toUInt16OrNull('abc')
+FORMAT vertical;
+```
+
+Result:
+
+```response
+Row 1:
+──────
+toUInt16OrNull('16'):  16
+toUInt16OrNull('abc'): ᴺᵁᴸᴸ
+```
+
+**See also**
+
+- [`toUInt16`](#touint16).
+- [`toUInt16OrZero`](#touint16orzero).
+- [`toUInt16OrDefault`](#touint16ordefault).
+
+## toUInt16OrDefault
+
+Like [`toUInt16`](#touint16), this function converts an input value to a value of type [UInt16](../data-types/int-uint.md) but returns the default value in case of an error.
+
+**Syntax**
+
+```sql
+toInt16OrDefault(expr, def)
+```
+
+**Arguments**
+
+- `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
+- `def` — The default value to return if parsing to type `UInt16` is unsuccessful. [UInt16](../data-types/int-uint.md).
+
+Supported arguments:
+- Values of type (U)Int8/16/32/64/128/256.
+- Values of type Float32/64.
+- String representations of (U)Int8/16/32/128/256.
+
+Arguments for which the default value is returned:
+- String representations of Float32/64 values, including `NaN` and `Inf`.
+- String representations of binary and hexadecimal values, e.g. `SELECT toUInt16OrDefault('0xc0fe', CAST('0', 'UInt16'));`.
+
+:::note
+If the input value cannot be represented within the bounds of [UInt16](../data-types/int-uint.md), overflow or underflow of the result occurs.
+This is not considered an error.
+:::
+
+**Returned value**
+
+- 16-bit unsigned integer value if successful, otherwise returns the default value. [UInt16](../data-types/int-uint.md).
+
+:::note
+- The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
+- The default value type should be the same as the cast type.
+  :::
+
+**Example**
+
+Query:
+
+``` sql
+SELECT
+    toUInt16OrDefault('16', CAST('0', 'UInt16')),
+    toUInt16OrDefault('abc', CAST('0', 'UInt16'))
+FORMAT vertical;
+```
+
+Result:
+
+```response
+Row 1:
+──────
+toUInt16OrDefault('16', CAST('0', 'UInt16')):  16
+toUInt16OrDefault('abc', CAST('0', 'UInt16')): 0
+```
+
+**See also**
+
+- [`toUInt16`](#touint16).
+- [`toUInt16OrZero`](#touint16orzero).
+- [`toUInt16OrNull`](#touint16ornull).
+
+## toUInt32
+
+Converts an input value to a value of type [`UInt32`](../data-types/int-uint.md). Throws an exception in case of an error.
+
+**Syntax**
+
+```sql
+toUInt32(expr)
+```
+
+**Arguments**
+
+- `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions).
+
+Supported arguments:
+- Values of type (U)Int8/16/32/64/128/256.
+- Values of type Float32/64.
+- String representations of (U)Int8/16/32/128/256.
+
+Unsupported arguments:
+- String representations of Float32/64 values, including `NaN` and `Inf`.
+- String representations of binary and hexadecimal values, e.g. `SELECT toUInt32('0xc0fe');`.
+
+:::note
+If the input value cannot be represented within the bounds of [UInt32](../data-types/int-uint.md), the result over or under flows.
+This is not considered an error.  
+For example: `SELECT toUInt32(4294967296) == 0;`
+:::
+
+**Returned value**
+
+- 32-bit unsigned integer value. [Int32](../data-types/int-uint.md).
+
+:::note
+The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
+:::
+
+**Example**
+
+Query:
+
+```sql
+SELECT
+    toUInt32(32),
+    toUInt32(32.32),
+    toUInt32('32')
+FORMAT vertical;
+```
+
+Result:
+
+```response
+Row 1:
+──────
+toUInt32(32):    32
+toUInt32(32.32): 32
+toUInt32('32'):  32
+```
+
+**See also**
+
+- [`toUInt32OrZero`](#touint32orzero).
+- [`toUInt32OrNull`](#touint32ornull).
+- [`toUInt32OrDefault`](#touint32ordefault).
+
+## toUInt32OrZero
+
+Like [`toUInt32`](#touint32), this function converts an input value to a value of type [UInt32](../data-types/int-uint.md) but returns `0` in case of an error.
+
+**Syntax**
+
+```sql
+toUInt32OrZero(x)
+```
+
+**Arguments**
+
+- `x` — A String representation of a number. [String](../data-types/string.md).
+
+Supported arguments:
+- String representations of (U)Int8/16/32/128/256.
+
+Unsupported arguments (return `0`):
+- String representations of Float32/64 values, including `NaN` and `Inf`.
+- String representations of binary and hexadecimal values, e.g. `SELECT toUInt32OrZero('0xc0fe');`.
+
+:::note
+If the input value cannot be represented within the bounds of [UInt32](../data-types/int-uint.md), overflow or underflow of the result occurs.
+This is not considered an error.
+:::
+
+**Returned value**
+
+- 32-bit unsigned integer value if successful, otherwise `0`. [UInt32](../data-types/int-uint.md)
+
+:::note
+The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero)
+, meaning it truncates fractional digits of numbers.
+:::
+
+**Example**
+
+Query:
+
+``` sql
+SELECT
+    toUInt32OrZero('32'),
+    toUInt32OrZero('abc')
+FORMAT vertical;
+```
+
+Result:
+
+```response
+Row 1:
+──────
+toUInt32OrZero('32'):  32
+toUInt32OrZero('abc'): 0
+```
+**See also**
+
+- [`toUInt32`](#touint32).
+- [`toUInt32OrNull`](#touint32ornull).
+- [`toUInt32OrDefault`](#touint32ordefault).
+
+## toUInt32OrNull
+
+Like [`toUInt32`](#touint32), this function converts an input value to a value of type [UInt32](../data-types/int-uint.md) but returns `NULL` in case of an error.
+
+**Syntax**
+
+```sql
+toUInt32OrNull(x)
+```
+
+**Arguments**
+
+- `x` — A String representation of a number. [String](../data-types/string.md).
+
+Supported arguments:
+- String representations of (U)Int8/16/32/128/256.
+
+Unsupported arguments (return `\N`)
+- String representations of Float32/64 values, including `NaN` and `Inf`.
+- String representations of binary and hexadecimal values, e.g. `SELECT toUInt32OrNull('0xc0fe');`.
+
+:::note
+If the input value cannot be represented within the bounds of [UInt32](../data-types/int-uint.md), overflow or underflow of the result occurs.
+This is not considered an error.
+:::
+
+**Returned value**
+
+- 32-bit unsigned integer value if successful, otherwise `NULL`. [UInt32](../data-types/int-uint.md) / [NULL](../data-types/nullable.md).
+
+:::note
+The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero)
+, meaning it truncates fractional digits of numbers.
+:::
+
+**Example**
+
+Query:
+
+``` sql
+SELECT
+    toUInt32OrNull('32'),
+    toUInt32OrNull('abc')
+FORMAT vertical;
+```
+
+Result:
+
+```response
+Row 1:
+──────
+toUInt32OrNull('32'):  32
+toUInt32OrNull('abc'): ᴺᵁᴸᴸ
+```
+
+**See also**
+
+- [`toUInt32`](#touint32).
+- [`toUInt32OrZero`](#touint32orzero).
+- [`toUInt32OrDefault`](#touint32ordefault).
+
+## toUInt32OrDefault
+
+Like [`toUInt32`](#touint32), this function converts an input value to a value of type [UInt32](../data-types/int-uint.md) but returns the default value in case of an error.
+
+**Syntax**
+
+```sql
+toUInt32OrDefault(expr, def)
+```
+
+**Arguments**
+
+- `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
+- `def` — The default value to return if parsing to type `UInt32` is unsuccessful. [UInt32](../data-types/int-uint.md).
+
+Supported arguments:
+- Values of type (U)Int8/16/32/64/128/256.
+- Values of type Float32/64.
+- String representations of (U)Int8/16/32/128/256.
+
+Arguments for which the default value is returned:
+- String representations of Float32/64 values, including `NaN` and `Inf`.
+- String representations of binary and hexadecimal values, e.g. `SELECT toUInt32OrDefault('0xc0fe', CAST('0', 'UInt32'));`.
+
+:::note
+If the input value cannot be represented within the bounds of [UInt32](../data-types/int-uint.md), overflow or underflow of the result occurs.
+This is not considered an error.
+:::
+
+**Returned value**
+
+- 32-bit unsigned integer value if successful, otherwise returns the default value. [UInt32](../data-types/int-uint.md).
+
+:::note
+- The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
+- The default value type should be the same as the cast type.
+:::
+
+**Example**
+
+Query:
+
+``` sql
+SELECT
+    toUInt32OrDefault('32', CAST('0', 'UInt32')),
+    toUInt32OrDefault('abc', CAST('0', 'UInt32'))
+FORMAT vertical;
+```
+
+Result:
+
+```response
+Row 1:
+──────
+toUInt32OrDefault('32', CAST('0', 'UInt32')):  32
+toUInt32OrDefault('abc', CAST('0', 'UInt32')): 0
+```
+
+**See also**
+
+- [`toUInt32`](#touint32).
+- [`toUInt32OrZero`](#touint32orzero).
+- [`toUInt32OrNull`](#touint32ornull).
+
+## toUInt64
+
+Converts an input value to a value of type [`UInt64`](../data-types/int-uint.md). Throws an exception in case of an error.
+
+**Syntax**
+
+```sql
+toUInt64(expr)
+```
+
+**Arguments**
+
+- `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions).
+
+Supported arguments:
+- Values of type (U)Int8/16/32/64/128/256.
+- Values of type Float32/64.
+- String representations of (U)Int8/16/32/128/256.
+
+Unsupported types:
+- String representations of Float32/64 values, including `NaN` and `Inf`.
+- String representations of binary and hexadecimal values, e.g. `SELECT toUInt64('0xc0fe');`.
+
+:::note
+If the input value cannot be represented within the bounds of [UInt64](../data-types/int-uint.md), the result over or under flows.
+This is not considered an error.  
+For example: `SELECT toUInt64(18446744073709551616) == 0;`
+:::
+
+**Returned value**
+
+- 64-bit unsigned integer value. [UInt64](../data-types/int-uint.md).
+
+:::note
+The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
+:::
+
+**Example**
+
+Query:
+
+```sql
+SELECT
+    toUInt64(64),
+    toUInt64(64.64),
+    toUInt64('64')
+FORMAT vertical;
+```
+
+Result:
+
+```response
+Row 1:
+──────
+toUInt64(64):    64
+toUInt64(64.64): 64
+toUInt64('64'):  64
+```
+
+**See also**
+
+- [`toUInt64OrZero`](#touint64orzero).
+- [`toUInt64OrNull`](#touint64ornull).
+- [`toUInt64OrDefault`](#touint64ordefault).
+
+## toInt64OrZero
+
+Like [`toUInt64`](#touint64), this function converts an input value to a value of type [UInt64](../data-types/int-uint.md) but returns `0` in case of an error.
+
+**Syntax**
+
+```sql
+toUInt64OrZero(x)
+```
+
+**Arguments**
+
+- `x` — A String representation of a number. [String](../data-types/string.md).
+
+Supported arguments:
+- String representations of (U)Int8/16/32/128/256.
+
+Unsupported arguments (return `0`):
+- String representations of Float32/64 values, including `NaN` and `Inf`.
+- String representations of binary and hexadecimal values, e.g. `SELECT toUInt64OrZero('0xc0fe');`.
+
+:::note
+If the input value cannot be represented within the bounds of [UInt64](../data-types/int-uint.md), overflow or underflow of the result occurs.
+This is not considered an error.
+:::
+
+**Returned value**
+
+- 64-bit unsigned integer value if successful, otherwise `0`. [UInt64](../data-types/int-uint.md).
+
+:::note
+The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
+:::
+
+**Example**
+
+Query:
+
+``` sql
+SELECT
+    toUInt64OrZero('64'),
+    toUInt64OrZero('abc')
+FORMAT vertical;
+```
+
+Result:
+
+```response
+Row 1:
+──────
+toUInt64OrZero('64'):  64
+toUInt64OrZero('abc'): 0
+```
+
+**See also**
+
+- [`toUInt64`](#touint64).
+- [`toUInt64OrNull`](#touint64ornull).
+- [`toUInt64OrDefault`](#touint64ordefault).
+
+## toUInt64OrNull
+
+Like [`toUInt64`](#touint64), this function converts an input value to a value of type [UInt64](../data-types/int-uint.md) but returns `NULL` in case of an error.
+
+**Syntax**
+
+```sql
+toUInt64OrNull(x)
+```
+
+**Arguments**
+
+- `x` — A String representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
+
+Supported arguments:
+- String representations of (U)Int8/16/32/128/256.
+
+Unsupported arguments (return `\N`)
+- String representations of Float32/64 values, including `NaN` and `Inf`.
+- String representations of binary and hexadecimal values, e.g. `SELECT toUInt64OrNull('0xc0fe');`.
+
+:::note
+If the input value cannot be represented within the bounds of [UInt64](../data-types/int-uint.md), overflow or underflow of the result occurs.
+This is not considered an error.
+:::
+
+**Returned value**
+
+- 64-bit unsigned integer value if successful, otherwise `NULL`. [UInt64](../data-types/int-uint.md) / [NULL](../data-types/nullable.md).
+
+:::note
+The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
+:::
+
+**Example**
+
+Query:
+
+``` sql
+SELECT
+    toUInt64OrNull('64'),
+    toUInt64OrNull('abc')
+FORMAT vertical;
+```
+
+Result:
+
+```response
+Row 1:
+──────
+toUInt64OrNull('64'):  64
+toUInt64OrNull('abc'): ᴺᵁᴸᴸ
+```
+
+**See also**
+
+- [`toUInt64`](#touint64).
+- [`toUInt64OrZero`](#touint64orzero).
+- [`toUInt64OrDefault`](#touint64ordefault).
+
+## toUInt64OrDefault
+
+Like [`toUInt64`](#touint64), this function converts an input value to a value of type [UInt64](../data-types/int-uint.md) but returns the default value in case of an error.
+
+**Syntax**
+
+```sql
+toUInt64OrDefault(expr, def)
+```
+
+**Arguments**
+
+- `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
+- `def` — The default value to return if parsing to type `UInt64` is unsuccessful. [UInt64](../data-types/int-uint.md).
+
+Supported arguments:
+- Values of type (U)Int8/16/32/64/128/256.
+- Values of type Float32/64.
+- String representations of (U)Int8/16/32/128/256.
+
+Arguments for which the default value is returned:
+- String representations of Float32/64 values, including `NaN` and `Inf`.
+- String representations of binary and hexadecimal values, e.g. `SELECT toUInt64OrDefault('0xc0fe', CAST('0', 'UInt64'));`.
+
+:::note
+If the input value cannot be represented within the bounds of [UInt64](../data-types/int-uint.md), overflow or underflow of the result occurs.
+This is not considered an error.
+:::
+
+**Returned value**
+
+- 64-bit unsigned integer value if successful, otherwise returns the default value. [UInt64](../data-types/int-uint.md).
+
+:::note
+- The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
+- The default value type should be the same as the cast type.
+:::
+
+**Example**
+
+Query:
+
+``` sql
+SELECT
+    toUInt64OrDefault('64', CAST('0', 'UInt64')),
+    toUInt64OrDefault('abc', CAST('0', 'UInt64'))
+FORMAT vertical;
+```
+
+Result:
+
+```response
+Row 1:
+──────
+toUInt64OrDefault('64', CAST('0', 'UInt64')):  64
+toUInt64OrDefault('abc', CAST('0', 'UInt64')): 0
+```
+
+**See also**
+
+- [`toUInt64`](#touint64).
+- [`toUInt64OrZero`](#touint64orzero).
+- [`toUInt64OrNull`](#touint64ornull).
+
+## toUInt128
+
+Converts an input value to a value of type [`UInt128`](../data-types/int-uint.md). Throws an exception in case of an error.
+
+**Syntax**
+
+```sql
+toUInt128(expr)
+```
+
+**Arguments**
+
+- `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions).
+
+Supported arguments:
+- Values of type (U)Int8/16/32/64/128/256.
+- Values of type Float32/64.
+- String representations of (U)Int8/16/32/128/256.
+
+Unsupported arguments:
+- String representations of Float32/64 values, including `NaN` and `Inf`.
+- String representations of binary and hexadecimal values, e.g. `SELECT toUInt128('0xc0fe');`.
+
+:::note
+If the input value cannot be represented within the bounds of [UInt128](../data-types/int-uint.md), the result over or under flows.
+This is not considered an error.
+:::
+
+**Returned value**
+
+- 128-bit unsigned integer value. [UInt128](../data-types/int-uint.md).
+
+:::note
+The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
+:::
+
+**Example**
+
+Query:
+
+```sql
+SELECT
+    toUInt128(128),
+    toUInt128(128.8),
+    toUInt128('128')
+FORMAT vertical;
+```
+
+Result:
+
+```response
+Row 1:
+──────
+toUInt128(128):   128
+toUInt128(128.8): 128
+toUInt128('128'): 128
+```
+
+**See also**
+
+- [`toUInt128OrZero`](#touint128orzero).
+- [`toUInt128OrNull`](#touint128ornull).
+- [`toUInt128OrDefault`](#touint128ordefault).
+
+## toUInt128OrZero
+
+Like [`toUInt128`](#touint128), this function converts an input value to a value of type [UInt128](../data-types/int-uint.md) but returns `0` in case of an error.
+
+**Syntax**
+
+```sql
+toUInt128OrZero(expr)
+```
+
+**Arguments**
+
+- `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
+
+Supported arguments:
+- String representations of (U)Int8/16/32/128/256.
+
+Unsupported arguments (return `0`):
+- String representations of Float32/64 values, including `NaN` and `Inf`.
+- String representations of binary and hexadecimal values, e.g. `SELECT toUInt128OrZero('0xc0fe');`.
+
+:::note
+If the input value cannot be represented within the bounds of [UInt128](../data-types/int-uint.md), overflow or underflow of the result occurs.
+This is not considered an error.
+:::
+
+**Returned value**
+
+- 128-bit unsigned integer value if successful, otherwise `0`. [UInt128](../data-types/int-uint.md).
+
+:::note
+The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
+:::
+
+**Example**
+
+Query:
+
+``` sql
+SELECT
+    toUInt128OrZero('128'),
+    toUInt128OrZero('abc')
+FORMAT vertical;
+```
+
+Result:
+
+```response
+Row 1:
+──────
+toUInt128OrZero('128'): 128
+toUInt128OrZero('abc'): 0
+```
+
+**See also**
+
+- [`toUInt128`](#touint128).
+- [`toUInt128OrNull`](#touint128ornull).
+- [`toUInt128OrDefault`](#touint128ordefault).
+
+## toUInt128OrNull
+
+Like [`toUInt128`](#touint128), this function converts an input value to a value of type [UInt128](../data-types/int-uint.md) but returns `NULL` in case of an error.
+
+**Syntax**
+
+```sql
+toUInt128OrNull(x)
+```
+
+**Arguments**
+
+- `x` — A String representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
+
+Supported arguments:
+- String representations of (U)Int8/16/32/128/256.
+
+Unsupported arguments (return `\N`)
+- String representations of Float32/64 values, including `NaN` and `Inf`.
+- String representations of binary and hexadecimal values, e.g. `SELECT toUInt128OrNull('0xc0fe');`.
+
+:::note
+If the input value cannot be represented within the bounds of [UInt128](../data-types/int-uint.md), overflow or underflow of the result occurs.
+This is not considered an error.
+:::
+
+**Returned value**
+
+- 128-bit unsigned integer value if successful, otherwise `NULL`. [UInt128](../data-types/int-uint.md) / [NULL](../data-types/nullable.md).
+
+:::note
+The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
+:::
+
+**Example**
+
+Query:
+
+``` sql
+SELECT
+    toUInt128OrNull('128'),
+    toUInt128OrNull('abc')
+FORMAT vertical;
+```
+
+Result:
+
+```response
+Row 1:
+──────
+toUInt128OrNull('128'): 128
+toUInt128OrNull('abc'): ᴺᵁᴸᴸ
+```
+
+**See also**
+
+- [`toUInt128`](#touint128).
+- [`toUInt128OrZero`](#touint128orzero).
+- [`toUInt128OrDefault`](#touint128ordefault).
+
+## toUInt128OrDefault
+
+Like [`toUInt128`](#toint128), this function converts an input value to a value of type [UInt128](../data-types/int-uint.md) but returns the default value in case of an error.
+
+**Syntax**
+
+```sql
+toUInt128OrDefault(expr, def)
+```
+
+**Arguments**
+
+- `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
+- `def` — The default value to return if parsing to type `UInt128` is unsuccessful. [UInt128](../data-types/int-uint.md).
+
+Supported arguments:
+- (U)Int8/16/32/64/128/256.
+- Float32/64.
+- String representations of (U)Int8/16/32/128/256.
+
+Arguments for which the default value is returned:
+- String representations of Float32/64 values, including `NaN` and `Inf`.
+- String representations of binary and hexadecimal values, e.g. `SELECT toUInt128OrDefault('0xc0fe', CAST('0', 'UInt128'));`.
+
+:::note
+If the input value cannot be represented within the bounds of [UInt128](../data-types/int-uint.md), overflow or underflow of the result occurs.
+This is not considered an error.
+:::
+
+**Returned value**
+
+- 128-bit unsigned integer value if successful, otherwise returns the default value. [UInt128](../data-types/int-uint.md).
+
+:::note
+- The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
+- The default value type should be the same as the cast type.
+:::
+
+**Example**
+
+Query:
+
+``` sql
+SELECT
+    toUInt128OrDefault('128', CAST('0', 'UInt128')),
+    toUInt128OrDefault('abc', CAST('0', 'UInt128'))
+FORMAT vertical;
+```
+
+Result:
+
+```response
+Row 1:
+──────
+toUInt128OrDefault('128', CAST('0', 'UInt128')): 128
+toUInt128OrDefault('abc', CAST('0', 'UInt128')): 0
+```
+
+**See also**
+
+- [`toUInt128`](#touint128).
+- [`toUInt128OrZero`](#touint128orzero).
+- [`toUInt128OrNull`](#touint128ornull).
+
+## toUInt256
+
+Converts an input value to a value of type [`UInt256`](../data-types/int-uint.md). Throws an exception in case of an error.
+
+**Syntax**
+
+```sql
+toUInt256(expr)
+```
+
+**Arguments**
+
+- `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions).
+
+Supported arguments:
+- Values of type (U)Int8/16/32/64/128/256.
+- Values of type Float32/64.
+- String representations of (U)Int8/16/32/128/256.
+
+Unsupported arguments:
+- String representations of Float32/64 values, including `NaN` and `Inf`.
+- String representations of binary and hexadecimal values, e.g. `SELECT toUInt256('0xc0fe');`.
+
+:::note
+If the input value cannot be represented within the bounds of [UInt256](../data-types/int-uint.md), the result over or under flows.
+This is not considered an error.
+:::
+
+**Returned value**
+
+- 256-bit unsigned integer value. [Int256](../data-types/int-uint.md).
+
+:::note
+The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
+:::
+
+**Example**
+
+Query:
+
+```sql
+SELECT
+    toUInt256(256),
+    toUInt256(256.256),
+    toUInt256('256')
+FORMAT vertical;
+```
+
+Result:
+
+```response
+Row 1:
+──────
+toUInt256(256):     256
+toUInt256(256.256): 256
+toUInt256('256'):   256
+```
+
+**See also**
+
+- [`toUInt256OrZero`](#touint256orzero).
+- [`toUInt256OrNull`](#touint256ornull).
+- [`toUInt256OrDefault`](#touint256ordefault).
+
+## toUUInt256OrZero
+
+Like [`toUInt256`](#touint256), this function converts an input value to a value of type [UInt256](../data-types/int-uint.md) but returns `0` in case of an error.
+
+**Syntax**
+
+```sql
+toUInt256OrZero(x)
+```
+
+**Arguments**
+
+- `x` — A String representation of a number. [String](../data-types/string.md).
+
+Supported arguments:
+- String representations of (U)Int8/16/32/128/256.
+
+Unsupported arguments (return `0`):
+- String representations of Float32/64 values, including `NaN` and `Inf`.
+- String representations of binary and hexadecimal values, e.g. `SELECT toUInt256OrZero('0xc0fe');`.
+
+:::note
+If the input value cannot be represented within the bounds of [UInt256](../data-types/int-uint.md), overflow or underflow of the result occurs.
+This is not considered an error.
+:::
+
+**Returned value**
+
+- 256-bit unsigned integer value if successful, otherwise `0`. [UInt256](../data-types/int-uint.md).
+
+:::note
+The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
+:::
+
+**Example**
+
+Query:
+
+``` sql
+SELECT
+    toUInt256OrZero('256'),
+    toUInt256OrZero('abc')
+FORMAT vertical;
+```
+
+Result:
+
+```response
+Row 1:
+──────
+toUInt256OrZero('256'): 256
+toUInt256OrZero('abc'): 0
+```
+
+**See also**
+
+- [`toUInt256`](#touint256).
+- [`toUInt256OrNull`](#touint256ornull).
+- [`toUInt256OrDefault`](#touint256ordefault).
+
+## toUInt256OrNull
+
+Like [`toUInt256`](#touint256), this function converts an input value to a value of type [UInt256](../data-types/int-uint.md) but returns `NULL` in case of an error.
+
+**Syntax**
+
+```sql
+toUInt256OrNull(x)
+```
+
+**Arguments**
+
+- `x` — A String representation of a number. [String](../data-types/string.md).
+
+Supported arguments:
+- String representations of (U)Int8/16/32/128/256.
+
+Unsupported arguments (return `\N`)
+- String representations of Float32/64 values, including `NaN` and `Inf`.
+- String representations of binary and hexadecimal values, e.g. `SELECT toUInt256OrNull('0xc0fe');`.
+
+:::note
+If the input value cannot be represented within the bounds of [UInt256](../data-types/int-uint.md), overflow or underflow of the result occurs.
+This is not considered an error.
+:::
+
+**Returned value**
+
+- 256-bit unsigned integer value if successful, otherwise `NULL`. [UInt256](../data-types/int-uint.md) / [NULL](../data-types/nullable.md).
+
+:::note
+The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
+:::
+
+**Example**
+
+Query:
+
+``` sql
+SELECT
+    toUInt256OrNull('256'),
+    toUInt256OrNull('abc')
+FORMAT vertical;
+```
+
+Result:
+
+```response
+Row 1:
+──────
+toUInt256OrNull('256'): 256
+toUInt256OrNull('abc'): ᴺᵁᴸᴸ
+```
+
+**See also**
+
+- [`toUInt256`](#touint256).
+- [`toUInt256OrZero`](#touint256orzero).
+- [`toUInt256OrDefault`](#touint256ordefault).
+
+## toUInt256OrDefault
+
+Like [`toUInt256`](#touint256), this function converts an input value to a value of type [UInt256](../data-types/int-uint.md) but returns the default value in case of an error.
+
+**Syntax**
+
+```sql
+toUInt256OrDefault(expr, def)
+```
+
+**Arguments**
+
+- `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
+- `def` — The default value to return if parsing to type `UInt256` is unsuccessful. [UInt256](../data-types/int-uint.md).
+
+Supported arguments:
+- Values of type (U)Int8/16/32/64/128/256.
+- Values of type Float32/64.
+- String representations of (U)Int8/16/32/128/256.
+
+Arguments for which the default value is returned:
+- String representations of Float32/64 values, including `NaN` and `Inf`
+- String representations of binary and hexadecimal values, e.g. `SELECT toUInt256OrDefault('0xc0fe', CAST('0', 'UInt256'));`
+
+:::note
+If the input value cannot be represented within the bounds of [UInt256](../data-types/int-uint.md), overflow or underflow of the result occurs.
+This is not considered an error.
+:::
+
+**Returned value**
+
+- 256-bit unsigned integer value if successful, otherwise returns the default value. [UInt256](../data-types/int-uint.md).
+
+:::note
+- The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
+- The default value type should be the same as the cast type.
+:::
+
+**Example**
+
+Query:
+
+``` sql
+SELECT
+    toUInt256OrDefault('-256', CAST('0', 'UInt256')),
+    toUInt256OrDefault('abc', CAST('0', 'UInt256'))
+FORMAT vertical;
+```
+
+Result:
+
+```response
+Row 1:
+──────
+toUInt256OrDefault('-256', CAST('0', 'UInt256')): 0
+toUInt256OrDefault('abc', CAST('0', 'UInt256')):  0
+```
+
+**See also**
+
+- [`toUInt256`](#touint256).
+- [`toUInt256OrZero`](#touint256orzero).
+- [`toUInt256OrNull`](#touint256ornull).
 
 ## toFloat(32\|64)
 

From 99322feabe5b15ebfee7f3699ec4de984725ecda Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitlibar@clickhouse.com>
Date: Fri, 26 Jul 2024 21:06:55 +0200
Subject: [PATCH 0921/1488] Add parameter 'backups.test_randomize_order' to
 backup configuration to make reproducing the issue easier.

---
 src/Backups/BackupsWorker.cpp                    | 16 ++++++++++++++--
 src/Backups/BackupsWorker.h                      |  1 +
 .../test_broken_projections/config.d/backups.xml |  1 +
 3 files changed, 16 insertions(+), 2 deletions(-)

diff --git a/src/Backups/BackupsWorker.cpp b/src/Backups/BackupsWorker.cpp
index 15a7d7c1eca..ab278429f5f 100644
--- a/src/Backups/BackupsWorker.cpp
+++ b/src/Backups/BackupsWorker.cpp
@@ -384,6 +384,7 @@ BackupsWorker::BackupsWorker(ContextMutablePtr global_context, size_t num_backup
     , allow_concurrent_backups(global_context->getConfigRef().getBool("backups.allow_concurrent_backups", true))
     , allow_concurrent_restores(global_context->getConfigRef().getBool("backups.allow_concurrent_restores", true))
     , remove_backup_files_after_failure(global_context->getConfigRef().getBool("backups.remove_backup_files_after_failure", true))
+    , test_randomize_order(global_context->getConfigRef().getBool("backups.test_randomize_order", false))
     , test_inject_sleep(global_context->getConfigRef().getBool("backups.test_inject_sleep", false))
     , log(getLogger("BackupsWorker"))
     , backup_log(global_context->getBackupLog())
@@ -713,14 +714,25 @@ void BackupsWorker::writeBackupEntries(
     bool always_single_threaded = !backup->supportsWritingInMultipleThreads();
     auto & thread_pool = getThreadPool(ThreadPoolId::BACKUP_COPY_FILES);
 
+    std::vector<size_t> writing_order;
+    if (test_randomize_order)
+    {
+        /// Randomize the order in which we write backup entries to the backup.
+        writing_order.resize(backup_entries.size());
+        std::iota(writing_order.begin(), writing_order.end(), 0);
+        std::shuffle(writing_order.begin(), writing_order.end(), thread_local_rng);
+    }
+
     ThreadPoolCallbackRunnerLocal<void> runner(thread_pool, "BackupWorker");
     for (size_t i = 0; i != backup_entries.size(); ++i)
     {
         if (failed)
             break;
 
-        auto & entry = backup_entries[i].second;
-        const auto & file_info = file_infos[i];
+        size_t index = !writing_order.empty() ? writing_order[i] : i;
+
+        auto & entry = backup_entries[index].second;
+        const auto & file_info = file_infos[index];
 
         auto job = [&]()
         {
diff --git a/src/Backups/BackupsWorker.h b/src/Backups/BackupsWorker.h
index e7ace91dace..946562b575f 100644
--- a/src/Backups/BackupsWorker.h
+++ b/src/Backups/BackupsWorker.h
@@ -119,6 +119,7 @@ private:
     const bool allow_concurrent_backups;
     const bool allow_concurrent_restores;
     const bool remove_backup_files_after_failure;
+    const bool test_randomize_order;
     const bool test_inject_sleep;
 
     LoggerPtr log;
diff --git a/tests/integration/test_broken_projections/config.d/backups.xml b/tests/integration/test_broken_projections/config.d/backups.xml
index 4da8edffd67..76068cb01c2 100644
--- a/tests/integration/test_broken_projections/config.d/backups.xml
+++ b/tests/integration/test_broken_projections/config.d/backups.xml
@@ -9,5 +9,6 @@
     </storage_configuration>
     <backups>
         <allowed_disk>backups</allowed_disk>
+        <test_randomize_order>true</test_randomize_order>
     </backups>
 </clickhouse>

From 4606ec96d3a57003d3271124a98ab4194518e3af Mon Sep 17 00:00:00 2001
From: sakulali <sakulali.dev@gmail.com>
Date: Sat, 27 Jul 2024 22:17:53 +0800
Subject: [PATCH 0922/1488] Autodetect configuration file format if is not
 .xml, .yml or .yaml

---
 src/Common/Config/ConfigProcessor.cpp         | 100 ++++++++++--------
 src/Common/Config/ConfigProcessor.h           |   2 +
 ..._clickhouse_client_config_format.reference |   8 +-
 .../01889_clickhouse_client_config_format.sh  |  33 ++++--
 4 files changed, 89 insertions(+), 54 deletions(-)

diff --git a/src/Common/Config/ConfigProcessor.cpp b/src/Common/Config/ConfigProcessor.cpp
index 67d6036aa51..f0b83c035c8 100644
--- a/src/Common/Config/ConfigProcessor.cpp
+++ b/src/Common/Config/ConfigProcessor.cpp
@@ -138,9 +138,14 @@ static Node * getRootNode(Document * document)
     return XMLUtils::getRootNode(document);
 }
 
+static size_t firstNonWhitespacePos(const std::string & s)
+{
+    return s.find_first_not_of(" \t\n\r");
+}
+
 static bool allWhitespace(const std::string & s)
 {
-    return s.find_first_not_of(" \t\n\r") == std::string::npos;
+    return firstNonWhitespacePos(s) == std::string::npos;
 }
 
 static void deleteAttributesRecursive(Node * root)
@@ -622,6 +627,52 @@ ConfigProcessor::Files ConfigProcessor::getConfigMergeFiles(const std::string &
     return files;
 }
 
+XMLDocumentPtr ConfigProcessor::parseConfig(const std::string & config_path)
+{
+    fs::path p(config_path);
+    std::string extension = p.extension();
+    boost::algorithm::to_lower(extension);
+
+    if (extension == ".xml")
+        return dom_parser.parse(config_path);
+    else if (extension == ".yaml" || extension == ".yml")
+        return YAMLParser::parse(config_path);
+    else
+    {
+        /// Suppose non regular file parsed as XML, such as pipe: /dev/fd/X (regardless it has .xml extension or not)
+        if (!fs::is_regular_file(config_path))
+            return dom_parser.parse(config_path);
+
+        /// If the regular file begins with < it might be XML, otherwise it might be YAML.
+        bool maybe_xml = false;
+        {
+            std::ifstream file(config_path);
+            if (!file.is_open())
+                throw Exception(ErrorCodes::CANNOT_LOAD_CONFIG, "Unknown format of '{}' config", config_path);
+
+            std::string line;
+            while (std::getline(file, line))
+            {
+                const size_t pos = firstNonWhitespacePos(line);
+                if (pos == std::string::npos)
+                    continue;
+
+                if (pos < line.size() && '<' == line[pos])
+                {
+                    maybe_xml = true;
+                }
+
+                break;
+            }
+        }
+
+        if (maybe_xml)
+            return dom_parser.parse(config_path);
+        else
+            return YAMLParser::parse(config_path);
+    }
+}
+
 XMLDocumentPtr ConfigProcessor::processConfig(
     bool * has_zk_includes,
     zkutil::ZooKeeperNodeCache * zk_node_cache,
@@ -633,23 +684,7 @@ XMLDocumentPtr ConfigProcessor::processConfig(
 
     if (fs::exists(path))
     {
-        fs::path p(path);
-
-        std::string extension = p.extension();
-        boost::algorithm::to_lower(extension);
-
-        if (extension == ".yaml" || extension == ".yml")
-        {
-            config = YAMLParser::parse(path);
-        }
-        else if (extension == ".xml" || extension == ".conf" || extension.empty())
-        {
-            config = dom_parser.parse(path);
-        }
-        else
-        {
-            throw Exception(ErrorCodes::CANNOT_LOAD_CONFIG, "Unknown format of '{}' config", path);
-        }
+        config = parseConfig(path);
     }
     else
     {
@@ -673,20 +708,7 @@ XMLDocumentPtr ConfigProcessor::processConfig(
             LOG_DEBUG(log, "Merging configuration file '{}'.", merge_file);
 
             XMLDocumentPtr with;
-
-            fs::path p(merge_file);
-            std::string extension = p.extension();
-            boost::algorithm::to_lower(extension);
-
-            if (extension == ".yaml" || extension == ".yml")
-            {
-                with = YAMLParser::parse(merge_file);
-            }
-            else
-            {
-                with = dom_parser.parse(merge_file);
-            }
-
+            with = parseConfig(merge_file);
             if (!merge(config, with))
             {
                 LOG_DEBUG(log, "Merging bypassed - configuration file '{}' doesn't belong to configuration '{}' - merging root node name '{}' doesn't match '{}'",
@@ -730,19 +752,7 @@ XMLDocumentPtr ConfigProcessor::processConfig(
         {
             LOG_DEBUG(log, "Including configuration file '{}'.", include_from_path);
 
-            fs::path p(include_from_path);
-            std::string extension = p.extension();
-            boost::algorithm::to_lower(extension);
-
-            if (extension == ".yaml" || extension == ".yml")
-            {
-                include_from = YAMLParser::parse(include_from_path);
-            }
-            else
-            {
-                include_from = dom_parser.parse(include_from_path);
-            }
-
+            include_from = parseConfig(include_from_path);
             contributing_files.push_back(include_from_path);
         }
 
diff --git a/src/Common/Config/ConfigProcessor.h b/src/Common/Config/ConfigProcessor.h
index 5712c36d737..a9d1325b722 100644
--- a/src/Common/Config/ConfigProcessor.h
+++ b/src/Common/Config/ConfigProcessor.h
@@ -65,6 +65,8 @@ public:
         zkutil::ZooKeeperNodeCache * zk_node_cache = nullptr,
         const zkutil::EventPtr & zk_changed_event = nullptr);
 
+    XMLDocumentPtr parseConfig(const std::string & config_path);
+
     /// These configurations will be used if there is no configuration file.
     static void registerEmbeddedConfig(std::string name, std::string_view content);
 
diff --git a/tests/queries/0_stateless/01889_clickhouse_client_config_format.reference b/tests/queries/0_stateless/01889_clickhouse_client_config_format.reference
index 202e32a583e..afe27ddb063 100644
--- a/tests/queries/0_stateless/01889_clickhouse_client_config_format.reference
+++ b/tests/queries/0_stateless/01889_clickhouse_client_config_format.reference
@@ -12,5 +12,9 @@ yml
 2
 yaml
 2
-ini
-Code: 347. Unknown format of '/config_default.ini' config. (CANNOT_LOAD_CONFIG)
+autodetect xml (with leading whitespaces)
+2
+autodetect xml (non leading whitespaces)
+2
+autodetect yaml
+2
diff --git a/tests/queries/0_stateless/01889_clickhouse_client_config_format.sh b/tests/queries/0_stateless/01889_clickhouse_client_config_format.sh
index 4e970f17d3a..14d67f943f1 100755
--- a/tests/queries/0_stateless/01889_clickhouse_client_config_format.sh
+++ b/tests/queries/0_stateless/01889_clickhouse_client_config_format.sh
@@ -12,7 +12,9 @@ XML_config=$CLICKHOUSE_TMP/config_$CLICKHOUSE_DATABASE.XML
 conf_config=$CLICKHOUSE_TMP/config_$CLICKHOUSE_DATABASE.conf
 yml_config=$CLICKHOUSE_TMP/config_$CLICKHOUSE_DATABASE.yml
 yaml_config=$CLICKHOUSE_TMP/config_$CLICKHOUSE_DATABASE.yaml
-ini_config=$CLICKHOUSE_TMP/config_$CLICKHOUSE_DATABASE.ini
+autodetect_xml_with_leading_whitespace_config=$CLICKHOUSE_TMP/config_$CLICKHOUSE_DATABASE.config
+autodetect_xml_non_leading_whitespace_config=$CLICKHOUSE_TMP/config_$CLICKHOUSE_DATABASE.cfg
+autodetect_yaml_config=$CLICKHOUSE_TMP/config_$CLICKHOUSE_DATABASE.properties
 
 function cleanup()
 {
@@ -22,7 +24,9 @@ function cleanup()
     rm "${conf_config:?}"
     rm "${yml_config:?}"
     rm "${yaml_config:?}"
-    rm "${ini_config:?}"
+    rm "${autodetect_xml_with_leading_whitespace_config:?}"
+    rm "${autodetect_xml_non_leading_whitespace_config:?}"
+    rm "${autodetect_yaml_config:?}"
 }
 trap cleanup EXIT
 
@@ -52,9 +56,19 @@ EOL
 cat > "$yaml_config" <<EOL
 max_threads: 2
 EOL
-cat > "$ini_config" <<EOL
-[config]
-max_threads=2
+cat > "$autodetect_xml_with_leading_whitespace_config" <<EOL
+
+    <config>
+        <max_threads>2</max_threads>
+    </config>
+EOL
+cat > "$autodetect_xml_non_leading_whitespace_config" <<EOL
+<config>
+    <max_threads>2</max_threads>
+</config>
+EOL
+cat > "$autodetect_yaml_config" <<EOL
+max_threads: 2
 EOL
 
 echo 'default'
@@ -74,5 +88,10 @@ echo 'yml'
 $CLICKHOUSE_CLIENT --config "$yml_config" -q "select getSetting('max_threads')"
 echo 'yaml'
 $CLICKHOUSE_CLIENT --config "$yaml_config" -q "select getSetting('max_threads')"
-echo 'ini'
-$CLICKHOUSE_CLIENT --config "$ini_config" -q "select getSetting('max_threads')" 2>&1 |& sed -e "s#$CLICKHOUSE_TMP##" -e "s#DB::Exception: ##"
+
+echo 'autodetect xml (with leading whitespaces)'
+$CLICKHOUSE_CLIENT --config "$autodetect_xml_with_leading_whitespace_config" -q "select getSetting('max_threads')"
+echo 'autodetect xml (non leading whitespaces)'
+$CLICKHOUSE_CLIENT --config "$autodetect_xml_non_leading_whitespace_config" -q "select getSetting('max_threads')"
+echo 'autodetect yaml'
+$CLICKHOUSE_CLIENT --config "$autodetect_yaml_config" -q "select getSetting('max_threads')"

From a9b947d9ac422290cc42b4c29ba5cc1b934244b3 Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Sun, 28 Jul 2024 11:51:09 +0200
Subject: [PATCH 0923/1488] Extract common

---
 base/base/CMakeLists.txt      |  1 +
 base/base/Numa.cpp            | 37 +++++++++++++++++++++++++++++++++++
 base/base/Numa.h              | 12 ++++++++++++
 base/base/getMemoryAmount.cpp | 29 +++------------------------
 programs/keeper/Keeper.cpp    |  7 +++++++
 programs/server/Server.cpp    | 36 ++++------------------------------
 6 files changed, 64 insertions(+), 58 deletions(-)
 create mode 100644 base/base/Numa.cpp
 create mode 100644 base/base/Numa.h

diff --git a/base/base/CMakeLists.txt b/base/base/CMakeLists.txt
index 451a6eb5e8b..341c92d3042 100644
--- a/base/base/CMakeLists.txt
+++ b/base/base/CMakeLists.txt
@@ -32,6 +32,7 @@ set (SRCS
     StringRef.cpp
     safeExit.cpp
     throwError.cpp
+    Numa.cpp
 )
 
 add_library (common ${SRCS})
diff --git a/base/base/Numa.cpp b/base/base/Numa.cpp
new file mode 100644
index 00000000000..0bf56a993b8
--- /dev/null
+++ b/base/base/Numa.cpp
@@ -0,0 +1,37 @@
+#include <base/Numa.h>
+
+#include "config.h"
+
+#if USE_NUMACTL
+#    include <numa.h>
+#endif
+
+namespace DB
+{
+
+std::optional<size_t> getNumaNodesTotalMemory()
+{
+    std::optional<size_t> total_memory;
+#if USE_NUMACTL
+    if (numa_available() != -1)
+    {
+        auto * membind = numa_get_membind();
+        if (!numa_bitmask_equal(membind, numa_all_nodes_ptr))
+        {
+            total_memory.emplace(0);
+            auto max_node = numa_max_node();
+            for (int i = 0; i <= max_node; ++i)
+            {
+                if (numa_bitmask_isbitset(membind, i))
+                    *total_memory += numa_node_size(i, nullptr);
+            }
+        }
+
+        numa_bitmask_free(membind);
+    }
+
+#endif
+    return total_memory;
+}
+
+}
diff --git a/base/base/Numa.h b/base/base/Numa.h
new file mode 100644
index 00000000000..b48ab15766b
--- /dev/null
+++ b/base/base/Numa.h
@@ -0,0 +1,12 @@
+#pragma once
+
+#include <optional>
+
+namespace DB
+{
+
+/// return total memory of NUMA nodes the process is bound to
+/// if NUMA is not supported or process can use all nodes, std::nullopt is returned
+std::optional<size_t> getNumaNodesTotalMemory();
+
+}
diff --git a/base/base/getMemoryAmount.cpp b/base/base/getMemoryAmount.cpp
index 56cddbfd628..03aab1eac72 100644
--- a/base/base/getMemoryAmount.cpp
+++ b/base/base/getMemoryAmount.cpp
@@ -2,6 +2,7 @@
 
 #include <base/cgroupsv2.h>
 #include <base/getPageSize.h>
+#include <base/Numa.h>
 
 #include <fstream>
 
@@ -9,13 +10,6 @@
 #include <sys/types.h>
 #include <sys/param.h>
 
-#include "config.h"
-
-#if USE_NUMACTL
-#include <numa.h>
-#endif
-
-
 namespace
 {
 
@@ -68,25 +62,8 @@ uint64_t getMemoryAmountOrZero()
 
     uint64_t memory_amount = num_pages * page_size;
 
-#if USE_NUMACTL
-    if (numa_available() != -1)
-    {
-        auto * membind = numa_get_membind();
-        if (!numa_bitmask_equal(membind, numa_all_nodes_ptr))
-        {
-            uint64_t total_numa_memory = 0;
-            auto max_node = numa_max_node();
-            for (int i = 0; i <= max_node; ++i)
-            {
-                if (numa_bitmask_isbitset(membind, i))
-                    total_numa_memory += numa_node_size(i, nullptr);
-            }
-
-            memory_amount = total_numa_memory;
-        }
-        numa_bitmask_free(membind);
-    }
-#endif
+    if (auto total_numa_memory = DB::getNumaNodesTotalMemory(); total_numa_memory.has_value())
+        memory_amount = *total_numa_memory;
 
     /// Respect the memory limit set by cgroups v2.
     auto limit_v2 = getCgroupsV2MemoryLimit();
diff --git a/programs/keeper/Keeper.cpp b/programs/keeper/Keeper.cpp
index 8cf1a4d1999..a1793b5d603 100644
--- a/programs/keeper/Keeper.cpp
+++ b/programs/keeper/Keeper.cpp
@@ -19,6 +19,7 @@
 #include <base/getMemoryAmount.h>
 #include <base/scope_guard.h>
 #include <base/safeExit.h>
+#include <base/Numa.h>
 #include <Poco/Net/NetException.h>
 #include <Poco/Net/TCPServerParams.h>
 #include <Poco/Net/TCPServer.h>
@@ -311,6 +312,12 @@ try
 
     MainThreadStatus::getInstance();
 
+    if (auto total_numa_memory = getNumaNodesTotalMemory(); total_numa_memory.has_value())
+    {
+        LOG_INFO(
+            log, "ClickHouse is bound to a subset of NUMA nodes. Total memory of all available nodes: {}", ReadableSize(*total_numa_memory));
+    }
+
 #if !defined(NDEBUG) || !defined(__OPTIMIZE__)
     LOG_WARNING(log, "Keeper was built in debug mode. It will work slowly.");
 #endif
diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp
index b818ff1f3a2..86ebd3a3225 100644
--- a/programs/server/Server.cpp
+++ b/programs/server/Server.cpp
@@ -22,6 +22,7 @@
 #include <base/coverage.h>
 #include <base/getFQDNOrHostName.h>
 #include <base/safeExit.h>
+#include <base/Numa.h>
 #include <Common/PoolId.h>
 #include <Common/MemoryTracker.h>
 #include <Common/ClickHouseRevision.h>
@@ -140,10 +141,6 @@
 #   include <azure/core/diagnostics/logger.hpp>
 #endif
 
-#if USE_NUMACTL
-#include <numa.h>
-#endif
-
 
 #include <incbin.h>
 /// A minimal file used when the server is run without installation
@@ -759,36 +756,11 @@ try
         setenv("OPENSSL_CONF", config_dir.c_str(), true); /// NOLINT
     }
 
-#if USE_NUMACTL
-    if (numa_available() != -1)
+    if (auto total_numa_memory = getNumaNodesTotalMemory(); total_numa_memory.has_value())
     {
-        auto * membind = numa_get_membind();
-        if (!numa_bitmask_equal(membind, numa_all_nodes_ptr))
-        {
-            uint64_t total_numa_memory = 0;
-            auto max_node = numa_max_node();
-            for (int i = 0; i <= max_node; ++i)
-            {
-                if (numa_bitmask_isbitset(membind, i))
-                    total_numa_memory += numa_node_size(i, nullptr);
-            }
-
-            LOG_INFO(
-                log,
-                "ClickHouse is bound to a subset of NUMA nodes. Total memory of all available nodes: {}",
-                ReadableSize(total_numa_memory));
-        }
-        else
-        {
-            LOG_TRACE(
-                log,
-                "All NUMA nodes are used. Detected NUMA nodes: {}",
-                numa_num_configured_nodes());
-        }
-
-        numa_bitmask_free(membind);
+        LOG_INFO(
+            log, "ClickHouse is bound to a subset of NUMA nodes. Total memory of all available nodes: {}", ReadableSize(*total_numa_memory));
     }
-#endif
 
     registerInterpreters();
     registerFunctions();

From 13c93cc90cc35e33cad71095fa0300c5f4f1753e Mon Sep 17 00:00:00 2001
From: Duc Canh Le <duccanh.le@ahrefs.com>
Date: Sun, 28 Jul 2024 10:29:52 +0000
Subject: [PATCH 0924/1488] fix hung query when using empty tuple as lhs of
 function IN

Signed-off-by: Duc Canh Le <duccanh.le@ahrefs.com>
---
 src/Analyzer/SetUtils.cpp                                   | 6 ++++++
 .../0_stateless/03210_empty_tuple_lhs_of_in.reference       | 0
 tests/queries/0_stateless/03210_empty_tuple_lhs_of_in.sql   | 1 +
 3 files changed, 7 insertions(+)
 create mode 100644 tests/queries/0_stateless/03210_empty_tuple_lhs_of_in.reference
 create mode 100644 tests/queries/0_stateless/03210_empty_tuple_lhs_of_in.sql

diff --git a/src/Analyzer/SetUtils.cpp b/src/Analyzer/SetUtils.cpp
index 0ecb3545225..e7d1a22f45a 100644
--- a/src/Analyzer/SetUtils.cpp
+++ b/src/Analyzer/SetUtils.cpp
@@ -43,6 +43,12 @@ size_t getCompoundTypeDepth(const IDataType & type)
             const auto & tuple_elements = assert_cast<const DataTypeTuple &>(*current_type).getElements();
             if (!tuple_elements.empty())
                 current_type = tuple_elements.at(0).get();
+            else
+            {
+                /// Special case: tuple with no element - tuple(). In this case, what's the compound type depth?
+                /// I'm not certain about the theoretical answer, but from experiment, 1 is the most reasonable choice.
+                return 1;
+            }
 
             ++result;
         }
diff --git a/tests/queries/0_stateless/03210_empty_tuple_lhs_of_in.reference b/tests/queries/0_stateless/03210_empty_tuple_lhs_of_in.reference
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/queries/0_stateless/03210_empty_tuple_lhs_of_in.sql b/tests/queries/0_stateless/03210_empty_tuple_lhs_of_in.sql
new file mode 100644
index 00000000000..0fc5f072da5
--- /dev/null
+++ b/tests/queries/0_stateless/03210_empty_tuple_lhs_of_in.sql
@@ -0,0 +1 @@
+SELECT tuple() IN tuple(1) SETTINGS allow_experimental_map_type = 1; -- { serverError INCORRECT_ELEMENT_OF_SET }

From 96eee7469a342648bce942c93ca0c5673234a662 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 28 Jul 2024 13:31:44 +0200
Subject: [PATCH 0925/1488] Update 02231_buffer_aggregate_states_leak.sql

---
 .../queries/0_stateless/02231_buffer_aggregate_states_leak.sql  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/02231_buffer_aggregate_states_leak.sql b/tests/queries/0_stateless/02231_buffer_aggregate_states_leak.sql
index 5e6b4393a79..7cc267b7263 100644
--- a/tests/queries/0_stateless/02231_buffer_aggregate_states_leak.sql
+++ b/tests/queries/0_stateless/02231_buffer_aggregate_states_leak.sql
@@ -29,7 +29,7 @@ from in_02231
 group by key;
 
 set optimize_trivial_insert_select = 1;
-insert into in_02231 select * from numbers(5e6) settings max_memory_usage='40Mi', max_threads=1, min_insert_block_size_rows = 10000, min_insert_block_size_bytes = '10Mi';
+insert into in_02231 select * from numbers(5e6) settings max_memory_usage='400Mi', max_threads=1;
 
 drop table buffer_02231;
 drop table out_02231;

From 28f7eef7a96837dca1ec72fbcfffcfbb7ca965e2 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 28 Jul 2024 13:37:26 +0200
Subject: [PATCH 0926/1488] Double whitespace

---
 tests/queries/0_stateless/01600_parts_states_metrics_long.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/01600_parts_states_metrics_long.sh b/tests/queries/0_stateless/01600_parts_states_metrics_long.sh
index cee7de3b1db..0c4d6988e71 100755
--- a/tests/queries/0_stateless/01600_parts_states_metrics_long.sh
+++ b/tests/queries/0_stateless/01600_parts_states_metrics_long.sh
@@ -34,7 +34,7 @@ verify()
 }
 
 $CLICKHOUSE_CLIENT --database_atomic_wait_for_drop_and_detach_synchronously=1 --query="DROP TABLE IF EXISTS test_table"
-$CLICKHOUSE_CLIENT --query="CREATE TABLE test_table(data Date) ENGINE = MergeTree  PARTITION BY toYear(data) ORDER BY data;"
+$CLICKHOUSE_CLIENT --query="CREATE TABLE test_table (data Date) ENGINE = MergeTree PARTITION BY toYear(data) ORDER BY data;"
 
 $CLICKHOUSE_CLIENT --query="INSERT INTO test_table VALUES ('1992-01-01')"
 verify

From 22211e30aa5762a499ad42e40f565690fb283e5d Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 28 Jul 2024 13:42:45 +0200
Subject: [PATCH 0927/1488] Revert "Enable `compile_expressions` by default."

---
 src/Core/Settings.h                 | 2 +-
 src/Core/SettingsChangesHistory.cpp | 3 +--
 tests/clickhouse-test               | 2 +-
 3 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/Core/Settings.h b/src/Core/Settings.h
index c7a1a7e2739..86e04b2ab4e 100644
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@@ -186,7 +186,7 @@ class IColumn;
     M(Bool, allow_suspicious_ttl_expressions, false, "Reject TTL expressions that don't depend on any of table's columns. It indicates a user error most of the time.", 0) \
     M(Bool, allow_suspicious_variant_types, false, "In CREATE TABLE statement allows specifying Variant type with similar variant types (for example, with different numeric or date types). Enabling this setting may introduce some ambiguity when working with values with similar types.", 0) \
     M(Bool, allow_suspicious_primary_key, false, "Forbid suspicious PRIMARY KEY/ORDER BY for MergeTree (i.e. SimpleAggregateFunction)", 0) \
-    M(Bool, compile_expressions, true, "Compile some scalar functions and operators to native code.", 0) \
+    M(Bool, compile_expressions, false, "Compile some scalar functions and operators to native code.", 0) \
     M(UInt64, min_count_to_compile_expression, 3, "The number of identical expressions before they are JIT-compiled", 0) \
     M(Bool, compile_aggregate_expressions, true, "Compile aggregate functions to native code.", 0) \
     M(UInt64, min_count_to_compile_aggregate_expression, 3, "The number of identical aggregate expressions before they are JIT-compiled", 0) \
diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp
index 87eaeff0ca9..9faf77e9087 100644
--- a/src/Core/SettingsChangesHistory.cpp
+++ b/src/Core/SettingsChangesHistory.cpp
@@ -57,7 +57,6 @@ String ClickHouseVersion::toString() const
 /// Note: please check if the key already exists to prevent duplicate entries.
 static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory::SettingsChanges>> settings_changes_history_initializer =
 {
-    {"24.8", {{"compile_expressions", false, true, "We believe that the LLVM infrastructure behind the JIT compiler is stable enough to enable this setting by default."}}},
     {"24.7", {{"output_format_parquet_write_page_index", false, true, "Add a possibility to write page index into parquet files."},
               {"output_format_binary_encode_types_in_binary_format", false, false, "Added new setting to allow to write type names in binary format in RowBinaryWithNamesAndTypes output format"},
               {"input_format_binary_decode_types_in_binary_format", false, false, "Added new setting to allow to read type names in binary format in RowBinaryWithNamesAndTypes input format"},
@@ -81,7 +80,7 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
               {"ignore_on_cluster_for_replicated_named_collections_queries", false, false, "Ignore ON CLUSTER clause for replicated named collections management queries."},
               {"backup_restore_s3_retry_attempts", 1000,1000, "Setting for Aws::Client::RetryStrategy, Aws::Client does retries itself, 0 means no retries. It takes place only for backup/restore."},
               {"postgresql_connection_attempt_timeout", 2, 2, "Allow to control 'connect_timeout' parameter of PostgreSQL connection."},
-              {"postgresql_connection_pool_retries", 2, 2, "Allow to control the number of retries in PostgreSQL connection pool."},
+              {"postgresql_connection_pool_retries", 2, 2, "Allow to control the number of retries in PostgreSQL connection pool."}
               }},
     {"24.6", {{"materialize_skip_indexes_on_insert", true, true, "Added new setting to allow to disable materialization of skip indexes on insert"},
               {"materialize_statistics_on_insert", true, true, "Added new setting to allow to disable materialization of statistics on insert"},
diff --git a/tests/clickhouse-test b/tests/clickhouse-test
index 55ce88917f4..a29c786e998 100755
--- a/tests/clickhouse-test
+++ b/tests/clickhouse-test
@@ -779,7 +779,7 @@ class SettingsRandomizer:
         "filesystem_prefetch_step_bytes": lambda: random.choice(
             [0, "100Mi"]
         ),  # 0 means 'auto'
-        "compile_expressions": lambda: random.randint(0, 1),
+        # "compile_expressions": lambda: random.randint(0, 1), - this setting has a bug: https://github.com/ClickHouse/ClickHouse/issues/51264
         "compile_aggregate_expressions": lambda: random.randint(0, 1),
         "compile_sort_description": lambda: random.randint(0, 1),
         "merge_tree_coarse_index_granularity": lambda: random.randint(2, 32),

From b74614852e4580f318e48ffc90658954b716f315 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 28 Jul 2024 13:55:23 +0200
Subject: [PATCH 0928/1488] Add documentation for `compile_expressions`

---
 src/Core/Settings.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Core/Settings.h b/src/Core/Settings.h
index 86e04b2ab4e..3349662cb78 100644
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@@ -186,7 +186,7 @@ class IColumn;
     M(Bool, allow_suspicious_ttl_expressions, false, "Reject TTL expressions that don't depend on any of table's columns. It indicates a user error most of the time.", 0) \
     M(Bool, allow_suspicious_variant_types, false, "In CREATE TABLE statement allows specifying Variant type with similar variant types (for example, with different numeric or date types). Enabling this setting may introduce some ambiguity when working with values with similar types.", 0) \
     M(Bool, allow_suspicious_primary_key, false, "Forbid suspicious PRIMARY KEY/ORDER BY for MergeTree (i.e. SimpleAggregateFunction)", 0) \
-    M(Bool, compile_expressions, false, "Compile some scalar functions and operators to native code.", 0) \
+    M(Bool, compile_expressions, false, "Compile some scalar functions and operators to native code. Due to a bug in the LLVM compiler infrastructure, on AArch64 machines, it is known to lead to a nullptr dereference and, consequently, server crash. Do not enable this setting.", 0) \
     M(UInt64, min_count_to_compile_expression, 3, "The number of identical expressions before they are JIT-compiled", 0) \
     M(Bool, compile_aggregate_expressions, true, "Compile aggregate functions to native code.", 0) \
     M(UInt64, min_count_to_compile_aggregate_expression, 3, "The number of identical aggregate expressions before they are JIT-compiled", 0) \

From 29b804adf38401dd47298ed285aee587b732595b Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 28 Jul 2024 14:01:17 +0200
Subject: [PATCH 0929/1488] Wrap log lines in the CI report for functional
 tests

---
 tests/ci/report.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/ci/report.py b/tests/ci/report.py
index 77043dfc8be..48e24f9c870 100644
--- a/tests/ci/report.py
+++ b/tests/ci/report.py
@@ -135,6 +135,7 @@ th {{ cursor: pointer; }}
 tr:hover {{ filter: var(--tr-hover-filter); }}
 .expandable {{ cursor: pointer; }}
 .expandable-content {{ display: none; }}
+pre {{ white-space: pre-wrap; }}
 #fish {{ display: none; float: right; position: relative; top: -20em; right: 2vw; margin-bottom: -20em; width: 30vw; filter: brightness(7%); z-index: -1; }}
 
 .themes {{

From 1ed6ea746917018fb02b00d78c5b0c4593919efd Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Fri, 26 Jul 2024 10:28:01 +0000
Subject: [PATCH 0930/1488] Functions: Iterate over input_rows_count where
 appropriate

---
 src/Functions/CRC.cpp                         |  31 ++--
 src/Functions/CountSubstringsImpl.h           |  17 +-
 src/Functions/CustomWeekTransforms.h          |  15 +-
 src/Functions/DateTimeTransforms.h            |  17 +-
 src/Functions/EmptyImpl.h                     |  23 ++-
 src/Functions/ExtractString.h                 |   2 +-
 src/Functions/FunctionBitTestMany.h           |  49 +++---
 src/Functions/FunctionChar.cpp                |   5 +-
 .../FunctionDateOrDateTimeAddInterval.h       |  61 +++----
 src/Functions/FunctionMathBinaryFloat64.h     |  31 ++--
 src/Functions/FunctionMathUnary.h             |  20 +--
 src/Functions/FunctionNumericPredicate.h      |  30 ++--
 src/Functions/FunctionSpaceFillingCurve.h     |   2 -
 src/Functions/FunctionStringOrArrayToT.h      |  16 +-
 src/Functions/FunctionStringReplace.h         |  17 +-
 src/Functions/FunctionTokens.h                |   9 +-
 .../FunctionsBinaryRepresentation.cpp         |  16 +-
 src/Functions/FunctionsBitToArray.cpp         |  64 ++++----
 src/Functions/FunctionsBitmap.h               |  24 +--
 .../FunctionsCharsetClassification.cpp        |  15 +-
 src/Functions/FunctionsCodingIP.cpp           |  34 ++--
 src/Functions/FunctionsCodingUUID.cpp         |  35 ++--
 src/Functions/FunctionsDecimalArithmetics.h   |  70 ++++----
 src/Functions/FunctionsEmbeddedDictionaries.h |  33 ++--
 src/Functions/FunctionsHashing.h              | 155 +++++++++---------
 src/Functions/FunctionsJSON.cpp               |   4 +-
 .../FunctionsLanguageClassification.cpp       |  11 +-
 src/Functions/FunctionsLogical.cpp            |   2 +-
 .../FunctionsMultiStringFuzzySearch.h         |   8 +-
 src/Functions/FunctionsMultiStringSearch.h    |   8 +-
 .../FunctionsProgrammingClassification.cpp    |   9 +-
 src/Functions/FunctionsRandom.h               |   3 +-
 src/Functions/FunctionsStringDistance.cpp     |  17 +-
 src/Functions/FunctionsStringHash.cpp         |   9 +-
 src/Functions/FunctionsStringHash.h           |  14 +-
 .../FunctionsStringHashFixedString.cpp        |  17 +-
 src/Functions/FunctionsStringSearch.h         |  17 +-
 src/Functions/FunctionsStringSearchToString.h |   4 +-
 src/Functions/FunctionsStringSimilarity.cpp   |  27 +--
 src/Functions/FunctionsStringSimilarity.h     |  11 +-
 src/Functions/FunctionsTextClassification.h   |  10 +-
 src/Functions/FunctionsTimeWindow.cpp         | 113 +++++++------
 .../FunctionsTonalityClassification.cpp       |   8 +-
 src/Functions/FunctionsVisitParam.h           |  10 +-
 src/Functions/HasTokenImpl.h                  |   5 +-
 src/Functions/Kusto/KqlArraySort.cpp          |  24 ++-
 src/Functions/MatchImpl.h                     |  44 +++--
 src/Functions/MultiMatchAllIndicesImpl.h      |  25 +--
 src/Functions/MultiMatchAnyImpl.h             |  29 ++--
 src/Functions/MultiSearchFirstIndexImpl.h     |  22 +--
 src/Functions/MultiSearchFirstPositionImpl.h  |  20 +--
 src/Functions/MultiSearchImpl.h               |  18 +-
 src/Functions/PolygonUtils.h                  |   8 +-
 src/Functions/PositionImpl.h                  |  22 ++-
 src/Functions/ReplaceRegexpImpl.h             |  42 ++---
 src/Functions/ReplaceStringImpl.h             |  43 +++--
 .../URL/FirstSignificantSubdomainCustomImpl.h |  14 +-
 src/Functions/URL/cutURLParameter.cpp         |   9 +-
 src/Functions/URL/extractURLParameter.cpp     |   7 +-
 src/Functions/URL/port.cpp                    |  10 +-
 src/Functions/UTCTimestampTransform.cpp       |  12 +-
 src/Functions/array/arrayAUC.cpp              |  13 +-
 src/Functions/array/arrayConcat.cpp           |   3 +-
 src/Functions/array/arrayEnumerateRanked.h    |   2 +-
 src/Functions/array/length.cpp                |  20 +--
 src/Functions/ascii.cpp                       |  22 +--
 src/Functions/extract.cpp                     |   7 +-
 src/Functions/isValidUTF8.cpp                 |  24 +--
 .../keyvaluepair/extractKeyValuePairs.cpp     |   8 +-
 src/Functions/lengthUTF8.cpp                  |  24 +--
 src/Functions/reinterpretAs.cpp               |  44 +++--
 src/Functions/visibleWidth.cpp                |   5 +-
 src/Functions/widthBucket.cpp                 |  10 +-
 73 files changed, 778 insertions(+), 851 deletions(-)

diff --git a/src/Functions/CRC.cpp b/src/Functions/CRC.cpp
index 0ba976669a3..73318d523b5 100644
--- a/src/Functions/CRC.cpp
+++ b/src/Functions/CRC.cpp
@@ -81,46 +81,43 @@ struct CRCFunctionWrapper
     static constexpr auto is_fixed_to_constant = true;
     using ReturnType = typename Impl::ReturnType;
 
-    static void vector(const ColumnString::Chars & data, const ColumnString::Offsets & offsets, PaddedPODArray<ReturnType> & res)
+    static void vector(const ColumnString::Chars & data, const ColumnString::Offsets & offsets, PaddedPODArray<ReturnType> & res, size_t input_rows_count)
     {
-        size_t size = offsets.size();
-
         ColumnString::Offset prev_offset = 0;
-        for (size_t i = 0; i < size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             res[i] = doCRC(data, prev_offset, offsets[i] - prev_offset - 1);
             prev_offset = offsets[i];
         }
     }
 
-    static void vectorFixedToConstant(const ColumnString::Chars & data, size_t n, ReturnType & res) { res = doCRC(data, 0, n); }
-
-    static void vectorFixedToVector(const ColumnString::Chars & data, size_t n, PaddedPODArray<ReturnType> & res)
+    static void vectorFixedToConstant(const ColumnString::Chars & data, size_t n, ReturnType & res, size_t)
     {
-        size_t size = data.size() / n;
-
-        for (size_t i = 0; i < size; ++i)
-        {
-            res[i] = doCRC(data, i * n, n);
-        }
+        res = doCRC(data, 0, n);
     }
 
-    [[noreturn]] static void array(const ColumnString::Offsets & /*offsets*/, PaddedPODArray<ReturnType> & /*res*/)
+    static void vectorFixedToVector(const ColumnString::Chars & data, size_t n, PaddedPODArray<ReturnType> & res, size_t input_rows_count)
+    {
+        for (size_t i = 0; i < input_rows_count; ++i)
+            res[i] = doCRC(data, i * n, n);
+    }
+
+    [[noreturn]] static void array(const ColumnString::Offsets &, PaddedPODArray<ReturnType> &, size_t)
     {
         throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Cannot apply function {} to Array argument", std::string(Impl::name));
     }
 
-    [[noreturn]] static void uuid(const ColumnUUID::Container & /*offsets*/, size_t /*n*/, PaddedPODArray<ReturnType> & /*res*/)
+    [[noreturn]] static void uuid(const ColumnUUID::Container &, size_t, PaddedPODArray<ReturnType> &, size_t)
     {
         throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Cannot apply function {} to UUID argument", std::string(Impl::name));
     }
 
-    [[noreturn]] static void ipv6(const ColumnIPv6::Container & /*offsets*/, size_t /*n*/, PaddedPODArray<ReturnType> & /*res*/)
+    [[noreturn]] static void ipv6(const ColumnIPv6::Container &, size_t, PaddedPODArray<ReturnType> &, size_t)
     {
         throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Cannot apply function {} to IPv6 argument", std::string(Impl::name));
     }
 
-    [[noreturn]] static void ipv4(const ColumnIPv4::Container & /*offsets*/, size_t /*n*/, PaddedPODArray<ReturnType> & /*res*/)
+    [[noreturn]] static void ipv4(const ColumnIPv4::Container &, size_t, PaddedPODArray<ReturnType> &, size_t)
     {
         throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Cannot apply function {} to IPv4 argument", std::string(Impl::name));
     }
diff --git a/src/Functions/CountSubstringsImpl.h b/src/Functions/CountSubstringsImpl.h
index 9ff3e4e1f2a..8e91bc3aeb4 100644
--- a/src/Functions/CountSubstringsImpl.h
+++ b/src/Functions/CountSubstringsImpl.h
@@ -37,7 +37,8 @@ struct CountSubstringsImpl
         const std::string & needle,
         const ColumnPtr & start_pos,
         PaddedPODArray<UInt64> & res,
-        [[maybe_unused]] ColumnUInt8 * res_null)
+        [[maybe_unused]] ColumnUInt8 * res_null,
+        size_t /*input_rows_count*/)
     {
         /// `res_null` serves as an output parameter for implementing an XYZOrNull variant.
         assert(!res_null);
@@ -150,7 +151,8 @@ struct CountSubstringsImpl
         const ColumnString::Offsets & needle_offsets,
         const ColumnPtr & start_pos,
         PaddedPODArray<UInt64> & res,
-        [[maybe_unused]] ColumnUInt8 * res_null)
+        [[maybe_unused]] ColumnUInt8 * res_null,
+        size_t input_rows_count)
     {
         /// `res_null` serves as an output parameter for implementing an XYZOrNull variant.
         assert(!res_null);
@@ -158,9 +160,7 @@ struct CountSubstringsImpl
         ColumnString::Offset prev_haystack_offset = 0;
         ColumnString::Offset prev_needle_offset = 0;
 
-        size_t size = haystack_offsets.size();
-
-        for (size_t i = 0; i < size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             size_t needle_size = needle_offsets[i] - prev_needle_offset - 1;
             size_t haystack_size = haystack_offsets[i] - prev_haystack_offset - 1;
@@ -207,7 +207,8 @@ struct CountSubstringsImpl
         const ColumnString::Offsets & needle_offsets,
         const ColumnPtr & start_pos,
         PaddedPODArray<UInt64> & res,
-        [[maybe_unused]] ColumnUInt8 * res_null)
+        [[maybe_unused]] ColumnUInt8 * res_null,
+        size_t input_rows_count)
     {
         /// `res_null` serves as an output parameter for implementing an XYZOrNull variant.
         assert(!res_null);
@@ -215,9 +216,7 @@ struct CountSubstringsImpl
         /// NOTE You could use haystack indexing. But this is a rare case.
         ColumnString::Offset prev_needle_offset = 0;
 
-        size_t size = needle_offsets.size();
-
-        for (size_t i = 0; i < size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             res[i] = 0;
             auto start = start_pos != nullptr ? std::max(start_pos->getUInt(i), UInt64(1)) : UInt64(1);
diff --git a/src/Functions/CustomWeekTransforms.h b/src/Functions/CustomWeekTransforms.h
index 75fb3c32f16..5b4de765362 100644
--- a/src/Functions/CustomWeekTransforms.h
+++ b/src/Functions/CustomWeekTransforms.h
@@ -32,13 +32,12 @@ struct WeekTransformer
     {}
 
     template <typename FromVectorType, typename ToVectorType>
-    void vector(const FromVectorType & vec_from, ToVectorType & vec_to, UInt8 week_mode, const DateLUTImpl & time_zone) const
+    void vector(const FromVectorType & vec_from, ToVectorType & vec_to, UInt8 week_mode, const DateLUTImpl & time_zone, size_t input_rows_count) const
     {
         using ValueType = typename ToVectorType::value_type;
-        size_t size = vec_from.size();
-        vec_to.resize(size);
+        vec_to.resize(input_rows_count);
 
-        for (size_t i = 0; i < size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             if constexpr (is_extended_result)
                 vec_to[i] = static_cast<ValueType>(transform.executeExtendedResult(vec_from[i], week_mode, time_zone));
@@ -56,7 +55,7 @@ template <typename FromDataType, typename ToDataType, bool is_extended_result =
 struct CustomWeekTransformImpl
 {
     template <typename Transform>
-    static ColumnPtr execute(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/, Transform transform = {})
+    static ColumnPtr execute(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count, Transform transform = {})
     {
         const auto op = WeekTransformer<typename FromDataType::FieldType, typename ToDataType::FieldType, Transform, is_extended_result>{transform};
 
@@ -77,9 +76,9 @@ struct CustomWeekTransformImpl
             const auto * sources = checkAndGetColumn<DataTypeString::ColumnType>(source_col.get());
 
             auto col_to = ToDataType::ColumnType::create();
-            col_to->getData().resize(sources->size());
+            col_to->getData().resize(input_rows_count);
 
-            for (size_t i = 0; i < sources->size(); ++i)
+            for (size_t i = 0; i < input_rows_count; ++i)
             {
                 DateTime64 dt64;
                 ReadBufferFromString buf(sources->getDataAt(i).toView());
@@ -92,7 +91,7 @@ struct CustomWeekTransformImpl
         else if (const auto * sources = checkAndGetColumn<typename FromDataType::ColumnType>(source_col.get()))
         {
             auto col_to = ToDataType::ColumnType::create();
-            op.vector(sources->getData(), col_to->getData(), week_mode, time_zone);
+            op.vector(sources->getData(), col_to->getData(), week_mode, time_zone, input_rows_count);
             return col_to;
         }
         else
diff --git a/src/Functions/DateTimeTransforms.h b/src/Functions/DateTimeTransforms.h
index 5f745f3ccad..a7bd398cdaa 100644
--- a/src/Functions/DateTimeTransforms.h
+++ b/src/Functions/DateTimeTransforms.h
@@ -24,7 +24,7 @@ namespace DB
 
 static constexpr auto millisecond_multiplier = 1'000;
 static constexpr auto microsecond_multiplier = 1'000'000;
-static constexpr auto nanosecond_multiplier = 1'000'000'000;
+static constexpr auto nanosecond_multiplier  = 1'000'000'000;
 
 static constexpr FormatSettings::DateTimeOverflowBehavior default_date_time_overflow_behavior = FormatSettings::DateTimeOverflowBehavior::Ignore;
 
@@ -2134,13 +2134,12 @@ struct Transformer
 {
     template <typename FromTypeVector, typename ToTypeVector>
     static void vector(const FromTypeVector & vec_from, ToTypeVector & vec_to, const DateLUTImpl & time_zone, const Transform & transform,
-        [[maybe_unused]] ColumnUInt8::Container * vec_null_map_to)
+        [[maybe_unused]] ColumnUInt8::Container * vec_null_map_to, size_t input_rows_count)
     {
         using ValueType = typename ToTypeVector::value_type;
-        size_t size = vec_from.size();
-        vec_to.resize(size);
+        vec_to.resize(input_rows_count);
 
-        for (size_t i = 0; i < size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             if constexpr (std::is_same_v<ToType, DataTypeDate> || std::is_same_v<ToType, DataTypeDateTime>)
             {
@@ -2178,7 +2177,7 @@ struct DateTimeTransformImpl
 {
     template <typename Additions = void *>
     static ColumnPtr execute(
-        const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t /*input_rows_count*/, const Transform & transform = {})
+        const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count, const Transform & transform = {})
     {
         using Op = Transformer<FromDataType, ToDataType, Transform, is_extended_result, Additions>;
 
@@ -2200,7 +2199,7 @@ struct DateTimeTransformImpl
             if (result_data_type.isDateTime() || result_data_type.isDateTime64())
             {
                 const auto & time_zone = dynamic_cast<const TimezoneMixin &>(*result_type).getTimeZone();
-                Op::vector(sources->getData(), col_to->getData(), time_zone, transform, vec_null_map_to);
+                Op::vector(sources->getData(), col_to->getData(), time_zone, transform, vec_null_map_to, input_rows_count);
             }
             else
             {
@@ -2209,15 +2208,13 @@ struct DateTimeTransformImpl
                     time_zone_argument_position = 2;
 
                 const DateLUTImpl & time_zone = extractTimeZoneFromFunctionArguments(arguments, time_zone_argument_position, 0);
-                Op::vector(sources->getData(), col_to->getData(), time_zone, transform, vec_null_map_to);
+                Op::vector(sources->getData(), col_to->getData(), time_zone, transform, vec_null_map_to, input_rows_count);
             }
 
             if constexpr (std::is_same_v<Additions, DateTimeAccurateOrNullConvertStrategyAdditions>)
             {
                 if (vec_null_map_to)
-                {
                     return ColumnNullable::create(std::move(mutable_result_col), std::move(col_null_map_to));
-                }
             }
 
             return mutable_result_col;
diff --git a/src/Functions/EmptyImpl.h b/src/Functions/EmptyImpl.h
index d3b2dda024b..03675254eb6 100644
--- a/src/Functions/EmptyImpl.h
+++ b/src/Functions/EmptyImpl.h
@@ -21,11 +21,10 @@ struct EmptyImpl
     /// If the function will return constant value for FixedString data type.
     static constexpr auto is_fixed_to_constant = false;
 
-    static void vector(const ColumnString::Chars & /*data*/, const ColumnString::Offsets & offsets, PaddedPODArray<UInt8> & res)
+    static void vector(const ColumnString::Chars & /*data*/, const ColumnString::Offsets & offsets, PaddedPODArray<UInt8> & res, size_t input_rows_count)
     {
-        size_t size = offsets.size();
         ColumnString::Offset prev_offset = 1;
-        for (size_t i = 0; i < size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             res[i] = negative ^ (offsets[i] == prev_offset);
             prev_offset = offsets[i] + 1;
@@ -33,42 +32,40 @@ struct EmptyImpl
     }
 
     /// Only make sense if is_fixed_to_constant.
-    static void vectorFixedToConstant(const ColumnString::Chars & /*data*/, size_t /*n*/, UInt8 & /*res*/)
+    static void vectorFixedToConstant(const ColumnString::Chars &, size_t, UInt8 &, size_t)
     {
         throw Exception(ErrorCodes::LOGICAL_ERROR, "'vectorFixedToConstant method' is called");
     }
 
-    static void vectorFixedToVector(const ColumnString::Chars & data, size_t n, PaddedPODArray<UInt8> & res)
+    static void vectorFixedToVector(const ColumnString::Chars & data, size_t n, PaddedPODArray<UInt8> & res, size_t input_rows_count)
     {
-        size_t size = data.size() / n;
-        for (size_t i = 0; i < size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
             res[i] = negative ^ memoryIsZeroSmallAllowOverflow15(data.data() + i * n, n);
     }
 
-    static void array(const ColumnString::Offsets & offsets, PaddedPODArray<UInt8> & res)
+    static void array(const ColumnString::Offsets & offsets, PaddedPODArray<UInt8> & res, size_t input_rows_count)
     {
-        size_t size = offsets.size();
         ColumnString::Offset prev_offset = 0;
-        for (size_t i = 0; i < size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             res[i] = negative ^ (offsets[i] == prev_offset);
             prev_offset = offsets[i];
         }
     }
 
-    static void uuid(const ColumnUUID::Container & container, size_t n, PaddedPODArray<UInt8> & res)
+    static void uuid(const ColumnUUID::Container & container, size_t n, PaddedPODArray<UInt8> & res, size_t)
     {
         for (size_t i = 0; i < n; ++i)
             res[i] = negative ^ (container[i].toUnderType() == 0);
     }
 
-    static void ipv6(const ColumnIPv6::Container & container, size_t n, PaddedPODArray<UInt8> & res)
+    static void ipv6(const ColumnIPv6::Container & container, size_t n, PaddedPODArray<UInt8> & res, size_t)
     {
         for (size_t i = 0; i < n; ++i)
             res[i] = negative ^ (container[i].toUnderType() == 0);
     }
 
-    static void ipv4(const ColumnIPv4::Container & container, size_t n, PaddedPODArray<UInt8> & res)
+    static void ipv4(const ColumnIPv4::Container & container, size_t n, PaddedPODArray<UInt8> & res, size_t)
     {
         for (size_t i = 0; i < n; ++i)
             res[i] = negative ^ (container[i].toUnderType() == 0);
diff --git a/src/Functions/ExtractString.h b/src/Functions/ExtractString.h
index 6beb8be830a..9f039f86b18 100644
--- a/src/Functions/ExtractString.h
+++ b/src/Functions/ExtractString.h
@@ -20,7 +20,7 @@ namespace DB
 // includes extracting ASCII ngram, UTF8 ngram, ASCII word and UTF8 word
 struct ExtractStringImpl
 {
-    static ALWAYS_INLINE inline const UInt8 * readOneWord(const UInt8 *& pos, const UInt8 * end)
+    static const UInt8 * readOneWord(const UInt8 *& pos, const UInt8 * end)
     {
         // jump separators
         while (pos < end && isUTF8Sep(*pos))
diff --git a/src/Functions/FunctionBitTestMany.h b/src/Functions/FunctionBitTestMany.h
index 950e4ab4ea8..514b78ce59f 100644
--- a/src/Functions/FunctionBitTestMany.h
+++ b/src/Functions/FunctionBitTestMany.h
@@ -46,7 +46,7 @@ public:
             throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of first argument of function {}", first_arg->getName(), getName());
 
 
-        for (const auto i : collections::range(1, arguments.size()))
+        for (size_t i = 1; i < arguments.size(); ++i)
         {
             const auto & pos_arg = arguments[i];
 
@@ -57,19 +57,19 @@ public:
         return std::make_shared<DataTypeUInt8>();
     }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t /*input_rows_count*/) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override
     {
         const auto * value_col = arguments.front().column.get();
 
         ColumnPtr res;
-        if (!((res = execute<UInt8>(arguments, result_type, value_col))
-            || (res = execute<UInt16>(arguments, result_type, value_col))
-            || (res = execute<UInt32>(arguments, result_type, value_col))
-            || (res = execute<UInt64>(arguments, result_type, value_col))
-            || (res = execute<Int8>(arguments, result_type, value_col))
-            || (res = execute<Int16>(arguments, result_type, value_col))
-            || (res = execute<Int32>(arguments, result_type, value_col))
-            || (res = execute<Int64>(arguments, result_type, value_col))))
+        if (!((res = execute<UInt8>(arguments, result_type, value_col, input_rows_count))
+            || (res = execute<UInt16>(arguments, result_type, value_col, input_rows_count))
+            || (res = execute<UInt32>(arguments, result_type, value_col, input_rows_count))
+            || (res = execute<UInt64>(arguments, result_type, value_col, input_rows_count))
+            || (res = execute<Int8>(arguments, result_type, value_col, input_rows_count))
+            || (res = execute<Int16>(arguments, result_type, value_col, input_rows_count))
+            || (res = execute<Int32>(arguments, result_type, value_col, input_rows_count))
+            || (res = execute<Int64>(arguments, result_type, value_col, input_rows_count))))
             throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of argument of function {}", value_col->getName(), getName());
 
         return res;
@@ -79,28 +79,28 @@ private:
     template <typename T>
     ColumnPtr execute(
             const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type,
-            const IColumn * const value_col_untyped) const
+            const IColumn * const value_col_untyped,
+            size_t input_rows_count) const
     {
         if (const auto value_col = checkAndGetColumn<ColumnVector<T>>(value_col_untyped))
         {
-            const auto size = value_col->size();
             bool is_const;
             const auto const_mask = createConstMaskIfConst<T>(arguments, is_const);
             const auto & val = value_col->getData();
 
-            auto out_col = ColumnVector<UInt8>::create(size);
+            auto out_col = ColumnVector<UInt8>::create(input_rows_count);
             auto & out = out_col->getData();
 
             if (is_const)
             {
-                for (const auto i : collections::range(0, size))
+                for (size_t i = 0; i < input_rows_count; ++i)
                     out[i] = Impl::apply(val[i], const_mask);
             }
             else
             {
-                const auto mask = createMask<T>(size, arguments);
+                const auto mask = createMask<T>(input_rows_count, arguments);
 
-                for (const auto i : collections::range(0, size))
+                for (size_t i = 0; i < input_rows_count; ++i)
                     out[i] = Impl::apply(val[i], mask[i]);
             }
 
@@ -108,23 +108,22 @@ private:
         }
         else if (const auto value_col_const = checkAndGetColumnConst<ColumnVector<T>>(value_col_untyped))
         {
-            const auto size = value_col_const->size();
             bool is_const;
             const auto const_mask = createConstMaskIfConst<T>(arguments, is_const);
             const auto val = value_col_const->template getValue<T>();
 
             if (is_const)
             {
-                return result_type->createColumnConst(size, toField(Impl::apply(val, const_mask)));
+                return result_type->createColumnConst(input_rows_count, toField(Impl::apply(val, const_mask)));
             }
             else
             {
-                const auto mask = createMask<T>(size, arguments);
-                auto out_col = ColumnVector<UInt8>::create(size);
+                const auto mask = createMask<T>(input_rows_count, arguments);
+                auto out_col = ColumnVector<UInt8>::create(input_rows_count);
 
                 auto & out = out_col->getData();
 
-                for (const auto i : collections::range(0, size))
+                for (size_t i = 0; i < input_rows_count; ++i)
                     out[i] = Impl::apply(val, mask[i]);
 
                 return out_col;
@@ -140,7 +139,7 @@ private:
         out_is_const = true;
         ValueType mask = 0;
 
-        for (const auto i : collections::range(1, arguments.size()))
+        for (size_t i = 1; i < arguments.size(); ++i)
         {
             if (auto pos_col_const = checkAndGetColumnConst<ColumnVector<ValueType>>(arguments[i].column.get()))
             {
@@ -166,7 +165,7 @@ private:
     {
         PaddedPODArray<ValueType> mask(size, ValueType{});
 
-        for (const auto i : collections::range(1, arguments.size()))
+        for (size_t i = 1; i < arguments.size(); ++i)
         {
             const auto * pos_col = arguments[i].column.get();
 
@@ -187,7 +186,7 @@ private:
         {
             const auto & pos = pos_col->getData();
 
-            for (const auto i : collections::range(0, mask.size()))
+            for (size_t i = 0; i < mask.size(); ++i)
                 if (pos[i] < 8 * sizeof(ValueType))
                     mask[i] = mask[i] | (ValueType(1) << pos[i]);
                 else
@@ -205,7 +204,7 @@ private:
 
             const auto new_mask = ValueType(1) << pos;
 
-            for (const auto i : collections::range(0, mask.size()))
+            for (size_t i = 0; i < mask.size(); ++i)
                 mask[i] = mask[i] | new_mask;
 
             return true;
diff --git a/src/Functions/FunctionChar.cpp b/src/Functions/FunctionChar.cpp
index 79e346a3ea4..dbdb7692177 100644
--- a/src/Functions/FunctionChar.cpp
+++ b/src/Functions/FunctionChar.cpp
@@ -103,14 +103,11 @@ private:
         const ColumnVector<T> * src_data_concrete = checkAndGetColumn<ColumnVector<T>>(&src_data);
 
         if (!src_data_concrete)
-        {
             return false;
-        }
 
         for (size_t row = 0; row < rows; ++row)
-        {
             out_vec[row * size_per_row + column_idx] = static_cast<char>(src_data_concrete->getInt(row));
-        }
+
         return true;
     }
 };
diff --git a/src/Functions/FunctionDateOrDateTimeAddInterval.h b/src/Functions/FunctionDateOrDateTimeAddInterval.h
index f50b1415622..25231b8887b 100644
--- a/src/Functions/FunctionDateOrDateTimeAddInterval.h
+++ b/src/Functions/FunctionDateOrDateTimeAddInterval.h
@@ -428,19 +428,17 @@ struct Processor
     {}
 
     template <typename FromColumnType, typename ToColumnType>
-    void NO_INLINE vectorConstant(const FromColumnType & col_from, ToColumnType & col_to, Int64 delta, const DateLUTImpl & time_zone, UInt16 scale) const
+    void NO_INLINE vectorConstant(const FromColumnType & col_from, ToColumnType & col_to, Int64 delta, const DateLUTImpl & time_zone, UInt16 scale, size_t input_rows_count) const
     {
         static const DateLUTImpl & utc_time_zone = DateLUT::instance("UTC");
 
         if constexpr (std::is_same_v<FromColumnType, ColumnString>)
         {
-            const auto & offsets_from = col_from.getOffsets();
             auto & vec_to = col_to.getData();
 
-            size_t size = offsets_from.size();
-            vec_to.resize(size);
+            vec_to.resize(input_rows_count);
 
-            for (size_t i = 0 ; i < size; ++i)
+            for (size_t i = 0 ; i < input_rows_count; ++i)
             {
                 std::string_view from = col_from.getDataAt(i).toView();
                 vec_to[i] = transform.execute(from, checkOverflow(delta), time_zone, utc_time_zone, scale);
@@ -451,32 +449,31 @@ struct Processor
             const auto & vec_from = col_from.getData();
             auto & vec_to = col_to.getData();
 
-            size_t size = vec_from.size();
-            vec_to.resize(size);
+            vec_to.resize(input_rows_count);
 
-            for (size_t i = 0; i < size; ++i)
+            for (size_t i = 0; i < input_rows_count; ++i)
                 vec_to[i] = transform.execute(vec_from[i], checkOverflow(delta), time_zone, utc_time_zone, scale);
         }
     }
 
     template <typename FromColumnType, typename ToColumnType>
-    void vectorVector(const FromColumnType & col_from, ToColumnType & col_to, const IColumn & delta, const DateLUTImpl & time_zone, UInt16 scale) const
+    void vectorVector(const FromColumnType & col_from, ToColumnType & col_to, const IColumn & delta, const DateLUTImpl & time_zone, UInt16 scale, size_t input_rows_count) const
     {
         castTypeToEither<
             ColumnUInt8, ColumnUInt16, ColumnUInt32, ColumnUInt64,
             ColumnInt8, ColumnInt16, ColumnInt32, ColumnInt64,
             ColumnFloat32, ColumnFloat64>(
-            &delta, [&](const auto & column){ vectorVector(col_from, col_to, column, time_zone, scale); return true; });
+            &delta, [&](const auto & column){ vectorVector(col_from, col_to, column, time_zone, scale, input_rows_count); return true; });
     }
 
     template <typename FromType, typename ToColumnType>
-    void constantVector(const FromType & from, ToColumnType & col_to, const IColumn & delta, const DateLUTImpl & time_zone, UInt16 scale) const
+    void constantVector(const FromType & from, ToColumnType & col_to, const IColumn & delta, const DateLUTImpl & time_zone, UInt16 scale, size_t input_rows_count) const
     {
         castTypeToEither<
             ColumnUInt8, ColumnUInt16, ColumnUInt32, ColumnUInt64,
             ColumnInt8, ColumnInt16, ColumnInt32, ColumnInt64,
             ColumnFloat32, ColumnFloat64>(
-            &delta, [&](const auto & column){ constantVector(from, col_to, column, time_zone, scale); return true; });
+            &delta, [&](const auto & column){ constantVector(from, col_to, column, time_zone, scale, input_rows_count); return true; });
     }
 
 private:
@@ -491,19 +488,17 @@ private:
 
     template <typename FromColumnType, typename ToColumnType, typename DeltaColumnType>
     NO_INLINE NO_SANITIZE_UNDEFINED void vectorVector(
-        const FromColumnType & col_from, ToColumnType & col_to, const DeltaColumnType & delta, const DateLUTImpl & time_zone, UInt16 scale) const
+        const FromColumnType & col_from, ToColumnType & col_to, const DeltaColumnType & delta, const DateLUTImpl & time_zone, UInt16 scale, size_t input_rows_count) const
     {
         static const DateLUTImpl & utc_time_zone = DateLUT::instance("UTC");
 
         if constexpr (std::is_same_v<FromColumnType, ColumnString>)
         {
-            const auto & offsets_from = col_from.getOffsets();
             auto & vec_to = col_to.getData();
 
-            size_t size = offsets_from.size();
-            vec_to.resize(size);
+            vec_to.resize(input_rows_count);
 
-            for (size_t i = 0 ; i < size; ++i)
+            for (size_t i = 0 ; i < input_rows_count; ++i)
             {
                 std::string_view from = col_from.getDataAt(i).toView();
                 vec_to[i] = transform.execute(from, checkOverflow(delta.getData()[i]), time_zone, utc_time_zone, scale);
@@ -514,26 +509,24 @@ private:
             const auto & vec_from = col_from.getData();
             auto & vec_to = col_to.getData();
 
-            size_t size = vec_from.size();
-            vec_to.resize(size);
+            vec_to.resize(input_rows_count);
 
-            for (size_t i = 0; i < size; ++i)
+            for (size_t i = 0; i < input_rows_count; ++i)
                 vec_to[i] = transform.execute(vec_from[i], checkOverflow(delta.getData()[i]), time_zone, utc_time_zone, scale);
         }
     }
 
     template <typename FromType, typename ToColumnType, typename DeltaColumnType>
     NO_INLINE NO_SANITIZE_UNDEFINED void constantVector(
-        const FromType & from, ToColumnType & col_to, const DeltaColumnType & delta, const DateLUTImpl & time_zone, UInt16 scale) const
+        const FromType & from, ToColumnType & col_to, const DeltaColumnType & delta, const DateLUTImpl & time_zone, UInt16 scale, size_t input_rows_count) const
     {
         static const DateLUTImpl & utc_time_zone = DateLUT::instance("UTC");
 
         auto & vec_to = col_to.getData();
 
-        size_t size = delta.size();
-        vec_to.resize(size);
+        vec_to.resize(input_rows_count);
 
-        for (size_t i = 0; i < size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
             vec_to[i] = transform.execute(from, checkOverflow(delta.getData()[i]), time_zone, utc_time_zone, scale);
     }
 };
@@ -542,7 +535,7 @@ private:
 template <typename FromDataType, typename ToDataType, typename Transform>
 struct DateTimeAddIntervalImpl
 {
-    static ColumnPtr execute(Transform transform, const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, UInt16 scale)
+    static ColumnPtr execute(Transform transform, const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, UInt16 scale, size_t input_rows_count)
     {
         using FromValueType = typename FromDataType::FieldType;
         using FromColumnType = typename FromDataType::ColumnType;
@@ -561,15 +554,15 @@ struct DateTimeAddIntervalImpl
         if (const auto * sources = checkAndGetColumn<FromColumnType>(&source_column))
         {
             if (const auto * delta_const_column = typeid_cast<const ColumnConst *>(&delta_column))
-                processor.vectorConstant(*sources, *col_to, delta_const_column->getInt(0), time_zone, scale);
+                processor.vectorConstant(*sources, *col_to, delta_const_column->getInt(0), time_zone, scale, input_rows_count);
             else
-                processor.vectorVector(*sources, *col_to, delta_column, time_zone, scale);
+                processor.vectorVector(*sources, *col_to, delta_column, time_zone, scale, input_rows_count);
         }
         else if (const auto * sources_const = checkAndGetColumnConst<FromColumnType>(&source_column))
         {
             processor.constantVector(
                 sources_const->template getValue<FromValueType>(),
-                *col_to, delta_column, time_zone, scale);
+                *col_to, delta_column, time_zone, scale, input_rows_count);
         }
         else
         {
@@ -708,25 +701,25 @@ public:
     bool useDefaultImplementationForConstants() const override { return true; }
     ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {2}; }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t /*input_rows_count*/) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override
     {
         const IDataType * from_type = arguments[0].type.get();
         WhichDataType which(from_type);
 
         if (which.isDate())
-            return DateTimeAddIntervalImpl<DataTypeDate, TransformResultDataType<DataTypeDate>, Transform>::execute(Transform{}, arguments, result_type, 0);
+            return DateTimeAddIntervalImpl<DataTypeDate, TransformResultDataType<DataTypeDate>, Transform>::execute(Transform{}, arguments, result_type, 0, input_rows_count);
         else if (which.isDate32())
-            return DateTimeAddIntervalImpl<DataTypeDate32, TransformResultDataType<DataTypeDate32>, Transform>::execute(Transform{}, arguments, result_type, 0);
+            return DateTimeAddIntervalImpl<DataTypeDate32, TransformResultDataType<DataTypeDate32>, Transform>::execute(Transform{}, arguments, result_type, 0, input_rows_count);
         else if (which.isDateTime())
-            return DateTimeAddIntervalImpl<DataTypeDateTime, TransformResultDataType<DataTypeDateTime>, Transform>::execute(Transform{}, arguments, result_type, 0);
+            return DateTimeAddIntervalImpl<DataTypeDateTime, TransformResultDataType<DataTypeDateTime>, Transform>::execute(Transform{}, arguments, result_type, 0, input_rows_count);
         else if (which.isDateTime64())
         {
             const auto * datetime64_type = assert_cast<const DataTypeDateTime64 *>(from_type);
             auto from_scale = datetime64_type->getScale();
-            return DateTimeAddIntervalImpl<DataTypeDateTime64, TransformResultDataType<DataTypeDateTime64>, Transform>::execute(Transform{}, arguments, result_type, from_scale);
+            return DateTimeAddIntervalImpl<DataTypeDateTime64, TransformResultDataType<DataTypeDateTime64>, Transform>::execute(Transform{}, arguments, result_type, from_scale, input_rows_count);
         }
         else if (which.isString())
-            return DateTimeAddIntervalImpl<DataTypeString, DataTypeDateTime64, Transform>::execute(Transform{}, arguments, result_type, 3);
+            return DateTimeAddIntervalImpl<DataTypeString, DataTypeDateTime64, Transform>::execute(Transform{}, arguments, result_type, 3, input_rows_count);
         else
             throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of first argument of function {}", arguments[0].type->getName(), getName());
     }
diff --git a/src/Functions/FunctionMathBinaryFloat64.h b/src/Functions/FunctionMathBinaryFloat64.h
index 1b75ee688f4..82ff6ae7fbf 100644
--- a/src/Functions/FunctionMathBinaryFloat64.h
+++ b/src/Functions/FunctionMathBinaryFloat64.h
@@ -54,7 +54,7 @@ private:
     }
 
     template <typename LeftType, typename RightType>
-    static ColumnPtr executeTyped(const ColumnConst * left_arg, const IColumn * right_arg)
+    static ColumnPtr executeTyped(const ColumnConst * left_arg, const IColumn * right_arg, size_t input_rows_count)
     {
         if (const auto right_arg_typed = checkAndGetColumn<ColumnVector<RightType>>(right_arg))
         {
@@ -63,12 +63,11 @@ private:
             LeftType left_src_data[Impl::rows_per_iteration];
             std::fill(std::begin(left_src_data), std::end(left_src_data), left_arg->template getValue<LeftType>());
             const auto & right_src_data = right_arg_typed->getData();
-            const auto src_size = right_src_data.size();
             auto & dst_data = dst->getData();
-            dst_data.resize(src_size);
+            dst_data.resize(input_rows_count);
 
-            const auto rows_remaining = src_size % Impl::rows_per_iteration;
-            const auto rows_size = src_size - rows_remaining;
+            const auto rows_remaining = input_rows_count % Impl::rows_per_iteration;
+            const auto rows_size = input_rows_count - rows_remaining;
 
             for (size_t i = 0; i < rows_size; i += Impl::rows_per_iteration)
                 Impl::execute(left_src_data, &right_src_data[i], &dst_data[i]);
@@ -92,7 +91,7 @@ private:
     }
 
     template <typename LeftType, typename RightType>
-    static ColumnPtr executeTyped(const ColumnVector<LeftType> * left_arg, const IColumn * right_arg)
+    static ColumnPtr executeTyped(const ColumnVector<LeftType> * left_arg, const IColumn * right_arg, size_t input_rows_count)
     {
         if (const auto right_arg_typed = checkAndGetColumn<ColumnVector<RightType>>(right_arg))
         {
@@ -100,12 +99,11 @@ private:
 
             const auto & left_src_data = left_arg->getData();
             const auto & right_src_data = right_arg_typed->getData();
-            const auto src_size = left_src_data.size();
             auto & dst_data = dst->getData();
-            dst_data.resize(src_size);
+            dst_data.resize(input_rows_count);
 
-            const auto rows_remaining = src_size % Impl::rows_per_iteration;
-            const auto rows_size = src_size - rows_remaining;
+            const auto rows_remaining = input_rows_count % Impl::rows_per_iteration;
+            const auto rows_size = input_rows_count - rows_remaining;
 
             for (size_t i = 0; i < rows_size; i += Impl::rows_per_iteration)
                 Impl::execute(&left_src_data[i], &right_src_data[i], &dst_data[i]);
@@ -136,12 +134,11 @@ private:
             const auto & left_src_data = left_arg->getData();
             RightType right_src_data[Impl::rows_per_iteration];
             std::fill(std::begin(right_src_data), std::end(right_src_data), right_arg_typed->template getValue<RightType>());
-            const auto src_size = left_src_data.size();
             auto & dst_data = dst->getData();
-            dst_data.resize(src_size);
+            dst_data.resize(input_rows_count);
 
-            const auto rows_remaining = src_size % Impl::rows_per_iteration;
-            const auto rows_size = src_size - rows_remaining;
+            const auto rows_remaining = input_rows_count % Impl::rows_per_iteration;
+            const auto rows_size = input_rows_count - rows_remaining;
 
             for (size_t i = 0; i < rows_size; i += Impl::rows_per_iteration)
                 Impl::execute(&left_src_data[i], right_src_data, &dst_data[i]);
@@ -165,7 +162,7 @@ private:
         return nullptr;
     }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
         const ColumnWithTypeAndName & col_left = arguments[0];
         const ColumnWithTypeAndName & col_right = arguments[1];
@@ -202,7 +199,7 @@ private:
 
             if (const auto left_arg_typed = checkAndGetColumn<ColVecLeft>(left_arg))
             {
-                if ((res = executeTyped<LeftType, RightType>(left_arg_typed, right_arg)))
+                if ((res = executeTyped<LeftType, RightType>(left_arg_typed, right_arg, input_rows_count)))
                     return true;
 
                 throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of second argument of function {}",
@@ -210,7 +207,7 @@ private:
             }
             if (const auto left_arg_typed = checkAndGetColumnConst<ColVecLeft>(left_arg))
             {
-                if ((res = executeTyped<LeftType, RightType>(left_arg_typed, right_arg)))
+                if ((res = executeTyped<LeftType, RightType>(left_arg_typed, right_arg, input_rows_count)))
                     return true;
 
                 throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of second argument of function {}",
diff --git a/src/Functions/FunctionMathUnary.h b/src/Functions/FunctionMathUnary.h
index 9f400932356..3cc8bf391b4 100644
--- a/src/Functions/FunctionMathUnary.h
+++ b/src/Functions/FunctionMathUnary.h
@@ -106,42 +106,40 @@ private:
     }
 
     template <typename T, typename ReturnType>
-    static ColumnPtr execute(const ColumnVector<T> * col)
+    static ColumnPtr execute(const ColumnVector<T> * col, size_t input_rows_count)
     {
         const auto & src_data = col->getData();
-        const size_t size = src_data.size();
 
         auto dst = ColumnVector<ReturnType>::create();
         auto & dst_data = dst->getData();
-        dst_data.resize(size);
+        dst_data.resize(input_rows_count);
 
-        executeInIterations(src_data.data(), dst_data.data(), size);
+        executeInIterations(src_data.data(), dst_data.data(), input_rows_count);
 
         return dst;
     }
 
     template <typename T, typename ReturnType>
-    static ColumnPtr execute(const ColumnDecimal<T> * col)
+    static ColumnPtr execute(const ColumnDecimal<T> * col, size_t input_rows_count)
     {
         const auto & src_data = col->getData();
-        const size_t size = src_data.size();
         UInt32 scale = col->getScale();
 
         auto dst = ColumnVector<ReturnType>::create();
         auto & dst_data = dst->getData();
-        dst_data.resize(size);
+        dst_data.resize(input_rows_count);
 
-        for (size_t i = 0; i < size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
             dst_data[i] = DecimalUtils::convertTo<ReturnType>(src_data[i], scale);
 
-        executeInIterations(dst_data.data(), dst_data.data(), size);
+        executeInIterations(dst_data.data(), dst_data.data(), input_rows_count);
 
         return dst;
     }
 
     bool useDefaultImplementationForConstants() const override { return true; }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
         const ColumnWithTypeAndName & col = arguments[0];
         ColumnPtr res;
@@ -156,7 +154,7 @@ private:
             const auto col_vec = checkAndGetColumn<ColVecType>(col.column.get());
             if (col_vec == nullptr)
                 return false;
-            return (res = execute<Type, ReturnType>(col_vec)) != nullptr;
+            return (res = execute<Type, ReturnType>(col_vec, input_rows_count)) != nullptr;
         };
 
         if (!callOnBasicType<void, true, true, true, false>(col.type->getTypeId(), call))
diff --git a/src/Functions/FunctionNumericPredicate.h b/src/Functions/FunctionNumericPredicate.h
index fd495d7e8e7..97a3639734b 100644
--- a/src/Functions/FunctionNumericPredicate.h
+++ b/src/Functions/FunctionNumericPredicate.h
@@ -53,39 +53,37 @@ public:
 
     bool useDefaultImplementationForConstants() const override { return true; }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
         const auto * in = arguments.front().column.get();
 
         ColumnPtr res;
-        if (!((res = execute<UInt8>(in))
-            || (res = execute<UInt16>(in))
-            || (res = execute<UInt32>(in))
-            || (res = execute<UInt64>(in))
-            || (res = execute<Int8>(in))
-            || (res = execute<Int16>(in))
-            || (res = execute<Int32>(in))
-            || (res = execute<Int64>(in))
-            || (res = execute<Float32>(in))
-            || (res = execute<Float64>(in))))
+        if (!((res = execute<UInt8>(in, input_rows_count))
+            || (res = execute<UInt16>(in, input_rows_count))
+            || (res = execute<UInt32>(in, input_rows_count))
+            || (res = execute<UInt64>(in, input_rows_count))
+            || (res = execute<Int8>(in, input_rows_count))
+            || (res = execute<Int16>(in, input_rows_count))
+            || (res = execute<Int32>(in, input_rows_count))
+            || (res = execute<Int64>(in, input_rows_count))
+            || (res = execute<Float32>(in, input_rows_count))
+            || (res = execute<Float64>(in, input_rows_count))))
             throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of first argument of function {}", in->getName(), getName());
 
         return res;
     }
 
     template <typename T>
-    ColumnPtr execute(const IColumn * in_untyped) const
+    ColumnPtr execute(const IColumn * in_untyped, size_t input_rows_count) const
     {
         if (const auto in = checkAndGetColumn<ColumnVector<T>>(in_untyped))
         {
-            const auto size = in->size();
-
-            auto out = ColumnUInt8::create(size);
+            auto out = ColumnUInt8::create(input_rows_count);
 
             const auto & in_data = in->getData();
             auto & out_data = out->getData();
 
-            for (const auto i : collections::range(0, size))
+            for (size_t i = 0; i < input_rows_count; ++i)
                 out_data[i] = Impl::execute(in_data[i]);
 
             return out;
diff --git a/src/Functions/FunctionSpaceFillingCurve.h b/src/Functions/FunctionSpaceFillingCurve.h
index ac9215f88e1..76f6678e847 100644
--- a/src/Functions/FunctionSpaceFillingCurve.h
+++ b/src/Functions/FunctionSpaceFillingCurve.h
@@ -132,9 +132,7 @@ public:
         }
         DataTypes types(tuple_size);
         for (size_t i = 0; i < tuple_size; i++)
-        {
             types[i] = std::make_shared<DataTypeUInt64>();
-        }
         return std::make_shared<DataTypeTuple>(types);
     }
 };
diff --git a/src/Functions/FunctionStringOrArrayToT.h b/src/Functions/FunctionStringOrArrayToT.h
index 26c740f1fac..40f780d82a8 100644
--- a/src/Functions/FunctionStringOrArrayToT.h
+++ b/src/Functions/FunctionStringOrArrayToT.h
@@ -71,7 +71,7 @@ public:
 
             typename ColumnVector<ResultType>::Container & vec_res = col_res->getData();
             vec_res.resize(col->size());
-            Impl::vector(col->getChars(), col->getOffsets(), vec_res);
+            Impl::vector(col->getChars(), col->getOffsets(), vec_res, input_rows_count);
 
             return col_res;
         }
@@ -80,7 +80,7 @@ public:
             if (Impl::is_fixed_to_constant)
             {
                 ResultType res = 0;
-                Impl::vectorFixedToConstant(col_fixed->getChars(), col_fixed->getN(), res);
+                Impl::vectorFixedToConstant(col_fixed->getChars(), col_fixed->getN(), res, input_rows_count);
 
                 return result_type->createColumnConst(col_fixed->size(), toField(res));
             }
@@ -90,7 +90,7 @@ public:
 
                 typename ColumnVector<ResultType>::Container & vec_res = col_res->getData();
                 vec_res.resize(col_fixed->size());
-                Impl::vectorFixedToVector(col_fixed->getChars(), col_fixed->getN(), vec_res);
+                Impl::vectorFixedToVector(col_fixed->getChars(), col_fixed->getN(), vec_res, input_rows_count);
 
                 return col_res;
             }
@@ -101,7 +101,7 @@ public:
 
             typename ColumnVector<ResultType>::Container & vec_res = col_res->getData();
             vec_res.resize(col_arr->size());
-            Impl::array(col_arr->getOffsets(), vec_res);
+            Impl::array(col_arr->getOffsets(), vec_res, input_rows_count);
 
             return col_res;
         }
@@ -112,7 +112,7 @@ public:
             vec_res.resize(col_map->size());
             const auto & col_nested = col_map->getNestedColumn();
 
-            Impl::array(col_nested.getOffsets(), vec_res);
+            Impl::array(col_nested.getOffsets(), vec_res, input_rows_count);
             return col_res;
         }
         else if (const ColumnUUID * col_uuid = checkAndGetColumn<ColumnUUID>(column.get()))
@@ -120,7 +120,7 @@ public:
             auto col_res = ColumnVector<ResultType>::create();
             typename ColumnVector<ResultType>::Container & vec_res = col_res->getData();
             vec_res.resize(col_uuid->size());
-            Impl::uuid(col_uuid->getData(), input_rows_count, vec_res);
+            Impl::uuid(col_uuid->getData(), input_rows_count, vec_res, input_rows_count);
             return col_res;
         }
         else if (const ColumnIPv6 * col_ipv6 = checkAndGetColumn<ColumnIPv6>(column.get()))
@@ -128,7 +128,7 @@ public:
             auto col_res = ColumnVector<ResultType>::create();
             typename ColumnVector<ResultType>::Container & vec_res = col_res->getData();
             vec_res.resize(col_ipv6->size());
-            Impl::ipv6(col_ipv6->getData(), input_rows_count, vec_res);
+            Impl::ipv6(col_ipv6->getData(), input_rows_count, vec_res, input_rows_count);
             return col_res;
         }
         else if (const ColumnIPv4 * col_ipv4 = checkAndGetColumn<ColumnIPv4>(column.get()))
@@ -136,7 +136,7 @@ public:
             auto col_res = ColumnVector<ResultType>::create();
             typename ColumnVector<ResultType>::Container & vec_res = col_res->getData();
             vec_res.resize(col_ipv4->size());
-            Impl::ipv4(col_ipv4->getData(), input_rows_count, vec_res);
+            Impl::ipv4(col_ipv4->getData(), input_rows_count, vec_res, input_rows_count);
             return col_res;
         }
         else
diff --git a/src/Functions/FunctionStringReplace.h b/src/Functions/FunctionStringReplace.h
index b4bcfa514a8..432e03bfe9e 100644
--- a/src/Functions/FunctionStringReplace.h
+++ b/src/Functions/FunctionStringReplace.h
@@ -45,7 +45,7 @@ public:
         return std::make_shared<DataTypeString>();
     }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
         ColumnPtr column_haystack = arguments[0].column;
         column_haystack = column_haystack->convertToFullColumnIfConst();
@@ -70,7 +70,8 @@ public:
                 col_haystack->getChars(), col_haystack->getOffsets(),
                 col_needle_const->getValue<String>(),
                 col_replacement_const->getValue<String>(),
-                col_res->getChars(), col_res->getOffsets());
+                col_res->getChars(), col_res->getOffsets(),
+                input_rows_count);
             return col_res;
         }
         else if (col_haystack && col_needle_vector && col_replacement_const)
@@ -79,7 +80,8 @@ public:
                 col_haystack->getChars(), col_haystack->getOffsets(),
                 col_needle_vector->getChars(), col_needle_vector->getOffsets(),
                 col_replacement_const->getValue<String>(),
-                col_res->getChars(), col_res->getOffsets());
+                col_res->getChars(), col_res->getOffsets(),
+                input_rows_count);
             return col_res;
         }
         else if (col_haystack && col_needle_const && col_replacement_vector)
@@ -88,7 +90,8 @@ public:
                 col_haystack->getChars(), col_haystack->getOffsets(),
                 col_needle_const->getValue<String>(),
                 col_replacement_vector->getChars(), col_replacement_vector->getOffsets(),
-                col_res->getChars(), col_res->getOffsets());
+                col_res->getChars(), col_res->getOffsets(),
+                input_rows_count);
             return col_res;
         }
         else if (col_haystack && col_needle_vector && col_replacement_vector)
@@ -97,7 +100,8 @@ public:
                 col_haystack->getChars(), col_haystack->getOffsets(),
                 col_needle_vector->getChars(), col_needle_vector->getOffsets(),
                 col_replacement_vector->getChars(), col_replacement_vector->getOffsets(),
-                col_res->getChars(), col_res->getOffsets());
+                col_res->getChars(), col_res->getOffsets(),
+                input_rows_count);
             return col_res;
         }
         else if (col_haystack_fixed && col_needle_const && col_replacement_const)
@@ -106,7 +110,8 @@ public:
                 col_haystack_fixed->getChars(), col_haystack_fixed->getN(),
                 col_needle_const->getValue<String>(),
                 col_replacement_const->getValue<String>(),
-                col_res->getChars(), col_res->getOffsets());
+                col_res->getChars(), col_res->getOffsets(),
+                input_rows_count);
             return col_res;
         }
         else
diff --git a/src/Functions/FunctionTokens.h b/src/Functions/FunctionTokens.h
index f1435ca5651..b6d8e9ee589 100644
--- a/src/Functions/FunctionTokens.h
+++ b/src/Functions/FunctionTokens.h
@@ -84,7 +84,7 @@ public:
         return std::make_shared<DataTypeArray>(std::make_shared<DataTypeString>());
     }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t /*input_rows_count*/) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override
     {
         Generator generator;
         generator.init(arguments, max_substrings_includes_remaining_string);
@@ -107,18 +107,17 @@ public:
             const ColumnString::Chars & src_chars = col_str->getChars();
             const ColumnString::Offsets & src_offsets = col_str->getOffsets();
 
-            res_offsets.reserve(src_offsets.size());
-            res_strings_offsets.reserve(src_offsets.size() * 5);    /// Constant 5 - at random.
+            res_offsets.reserve(input_rows_count);
+            res_strings_offsets.reserve(input_rows_count * 5);    /// Constant 5 - at random.
             res_strings_chars.reserve(src_chars.size());
 
             Pos token_begin = nullptr;
             Pos token_end = nullptr;
 
-            size_t size = src_offsets.size();
             ColumnString::Offset current_src_offset = 0;
             ColumnArray::Offset current_dst_offset = 0;
             ColumnString::Offset current_dst_strings_offset = 0;
-            for (size_t i = 0; i < size; ++i)
+            for (size_t i = 0; i < input_rows_count; ++i)
             {
                 Pos pos = reinterpret_cast<Pos>(&src_chars[current_src_offset]);
                 current_src_offset = src_offsets[i];
diff --git a/src/Functions/FunctionsBinaryRepresentation.cpp b/src/Functions/FunctionsBinaryRepresentation.cpp
index f77d2f1f350..b3861f0394d 100644
--- a/src/Functions/FunctionsBinaryRepresentation.cpp
+++ b/src/Functions/FunctionsBinaryRepresentation.cpp
@@ -632,7 +632,7 @@ public:
 
     bool useDefaultImplementationForConstants() const override { return true; }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
         const ColumnPtr & column = arguments[0].column;
 
@@ -646,11 +646,10 @@ public:
             const ColumnString::Chars & in_vec = col->getChars();
             const ColumnString::Offsets & in_offsets = col->getOffsets();
 
-            size_t size = in_offsets.size();
-            out_offsets.resize(size);
+            out_offsets.resize(input_rows_count);
 
             size_t max_out_len = 0;
-            for (size_t i = 0; i < in_offsets.size(); ++i)
+            for (size_t i = 0; i < input_rows_count; ++i)
             {
                 const size_t len = in_offsets[i] - (i == 0 ? 0 : in_offsets[i - 1])
                     - /* trailing zero symbol that is always added in ColumnString and that is ignored while decoding */ 1;
@@ -662,7 +661,7 @@ public:
             char * pos = begin;
             size_t prev_offset = 0;
 
-            for (size_t i = 0; i < size; ++i)
+            for (size_t i = 0; i < input_rows_count; ++i)
             {
                 size_t new_offset = in_offsets[i];
 
@@ -691,15 +690,14 @@ public:
             const ColumnString::Chars & in_vec = col_fix_string->getChars();
             const size_t n = col_fix_string->getN();
 
-            size_t size = col_fix_string->size();
-            out_offsets.resize(size);
-            out_vec.resize(((n + word_size - 1) / word_size + /* trailing zero symbol that is always added by Impl::decode */ 1) * size);
+            out_offsets.resize(input_rows_count);
+            out_vec.resize(((n + word_size - 1) / word_size + /* trailing zero symbol that is always added by Impl::decode */ 1) * input_rows_count);
 
             char * begin = reinterpret_cast<char *>(out_vec.data());
             char * pos = begin;
             size_t prev_offset = 0;
 
-            for (size_t i = 0; i < size; ++i)
+            for (size_t i = 0; i < input_rows_count; ++i)
             {
                 size_t new_offset = prev_offset + n;
 
diff --git a/src/Functions/FunctionsBitToArray.cpp b/src/Functions/FunctionsBitToArray.cpp
index beaaccad6db..0a14516268a 100644
--- a/src/Functions/FunctionsBitToArray.cpp
+++ b/src/Functions/FunctionsBitToArray.cpp
@@ -60,17 +60,17 @@ public:
 
     bool useDefaultImplementationForConstants() const override { return true; }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
         ColumnPtr res;
-        if (!((res = executeType<UInt8>(arguments))
-            || (res = executeType<UInt16>(arguments))
-            || (res = executeType<UInt32>(arguments))
-            || (res = executeType<UInt64>(arguments))
-            || (res = executeType<Int8>(arguments))
-            || (res = executeType<Int16>(arguments))
-            || (res = executeType<Int32>(arguments))
-            || (res = executeType<Int64>(arguments))))
+        if (!((res = executeType<UInt8>(arguments, input_rows_count))
+            || (res = executeType<UInt16>(arguments, input_rows_count))
+            || (res = executeType<UInt32>(arguments, input_rows_count))
+            || (res = executeType<UInt64>(arguments, input_rows_count))
+            || (res = executeType<Int8>(arguments, input_rows_count))
+            || (res = executeType<Int16>(arguments, input_rows_count))
+            || (res = executeType<Int32>(arguments, input_rows_count))
+            || (res = executeType<Int64>(arguments, input_rows_count))))
             throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of argument of function {}",
                             arguments[0].column->getName(), getName());
 
@@ -98,7 +98,7 @@ private:
     }
 
     template <typename T>
-    ColumnPtr executeType(const ColumnsWithTypeAndName & columns) const
+    ColumnPtr executeType(const ColumnsWithTypeAndName & columns, size_t input_rows_count) const
     {
         if (const ColumnVector<T> * col_from = checkAndGetColumn<ColumnVector<T>>(columns[0].column.get()))
         {
@@ -107,13 +107,12 @@ private:
             const typename ColumnVector<T>::Container & vec_from = col_from->getData();
             ColumnString::Chars & data_to = col_to->getChars();
             ColumnString::Offsets & offsets_to = col_to->getOffsets();
-            size_t size = vec_from.size();
-            data_to.resize(size * 2);
-            offsets_to.resize(size);
+            data_to.resize(input_rows_count * 2);
+            offsets_to.resize(input_rows_count);
 
             WriteBufferFromVector<ColumnString::Chars> buf_to(data_to);
 
-            for (size_t i = 0; i < size; ++i)
+            for (size_t i = 0; i < input_rows_count; ++i)
             {
                 writeBitmask<T>(vec_from[i], buf_to);
                 writeChar(0, buf_to);
@@ -244,7 +243,7 @@ public:
     bool useDefaultImplementationForConstants() const override { return true; }
 
     template <typename T>
-    ColumnPtr executeType(const IColumn * column) const
+    ColumnPtr executeType(const IColumn * column, size_t input_rows_count) const
     {
         const ColumnVector<T> * col_from = checkAndGetColumn<ColumnVector<T>>(column);
         if (!col_from)
@@ -257,13 +256,12 @@ public:
         auto & result_array_offsets_data = result_array_offsets->getData();
 
         auto & vec_from = col_from->getData();
-        size_t size = vec_from.size();
-        result_array_offsets_data.resize(size);
-        result_array_values_data.reserve(size * 2);
+        result_array_offsets_data.resize(input_rows_count);
+        result_array_values_data.reserve(input_rows_count * 2);
 
         using UnsignedType = make_unsigned_t<T>;
 
-        for (size_t row = 0; row < size; ++row)
+        for (size_t row = 0; row < input_rows_count; ++row)
         {
             UnsignedType x = static_cast<UnsignedType>(vec_from[row]);
 
@@ -302,24 +300,24 @@ public:
         return result_column;
     }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
         const IColumn * in_column = arguments[0].column.get();
         ColumnPtr result_column;
 
-        if (!((result_column = executeType<UInt8>(in_column))
-              || (result_column = executeType<UInt16>(in_column))
-              || (result_column = executeType<UInt32>(in_column))
-              || (result_column = executeType<UInt32>(in_column))
-              || (result_column = executeType<UInt64>(in_column))
-              || (result_column = executeType<UInt128>(in_column))
-              || (result_column = executeType<UInt256>(in_column))
-              || (result_column = executeType<Int8>(in_column))
-              || (result_column = executeType<Int16>(in_column))
-              || (result_column = executeType<Int32>(in_column))
-              || (result_column = executeType<Int64>(in_column))
-              || (result_column = executeType<Int128>(in_column))
-              || (result_column = executeType<Int256>(in_column))))
+        if (!((result_column = executeType<UInt8>(in_column, input_rows_count))
+              || (result_column = executeType<UInt16>(in_column, input_rows_count))
+              || (result_column = executeType<UInt32>(in_column, input_rows_count))
+              || (result_column = executeType<UInt32>(in_column, input_rows_count))
+              || (result_column = executeType<UInt64>(in_column, input_rows_count))
+              || (result_column = executeType<UInt128>(in_column, input_rows_count))
+              || (result_column = executeType<UInt256>(in_column, input_rows_count))
+              || (result_column = executeType<Int8>(in_column, input_rows_count))
+              || (result_column = executeType<Int16>(in_column, input_rows_count))
+              || (result_column = executeType<Int32>(in_column, input_rows_count))
+              || (result_column = executeType<Int64>(in_column, input_rows_count))
+              || (result_column = executeType<Int128>(in_column, input_rows_count))
+              || (result_column = executeType<Int256>(in_column, input_rows_count))))
         {
             throw Exception(ErrorCodes::ILLEGAL_COLUMN,
                             "Illegal column {} of first argument of function {}",
diff --git a/src/Functions/FunctionsBitmap.h b/src/Functions/FunctionsBitmap.h
index 92ec71a3118..12b2b1a662a 100644
--- a/src/Functions/FunctionsBitmap.h
+++ b/src/Functions/FunctionsBitmap.h
@@ -155,7 +155,7 @@ public:
 
     bool useDefaultImplementationForConstants() const override { return true; }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /* input_rows_count */) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
         const IDataType * from_type = arguments[0].type.get();
         const auto * array_type = typeid_cast<const DataTypeArray *>(from_type);
@@ -165,21 +165,21 @@ public:
 
         WhichDataType which(nested_type);
         if (which.isUInt8())
-            return executeBitmapData<UInt8>(argument_types, arguments);
+            return executeBitmapData<UInt8>(argument_types, arguments, input_rows_count);
         else if (which.isUInt16())
-            return executeBitmapData<UInt16>(argument_types, arguments);
+            return executeBitmapData<UInt16>(argument_types, arguments, input_rows_count);
         else if (which.isUInt32())
-            return executeBitmapData<UInt32>(argument_types, arguments);
+            return executeBitmapData<UInt32>(argument_types, arguments, input_rows_count);
         else if (which.isUInt64())
-            return executeBitmapData<UInt64>(argument_types, arguments);
+            return executeBitmapData<UInt64>(argument_types, arguments, input_rows_count);
         else if (which.isInt8())
-            return executeBitmapData<Int8>(argument_types, arguments);
+            return executeBitmapData<Int8>(argument_types, arguments, input_rows_count);
         else if (which.isInt16())
-            return executeBitmapData<Int16>(argument_types, arguments);
+            return executeBitmapData<Int16>(argument_types, arguments, input_rows_count);
         else if (which.isInt32())
-            return executeBitmapData<Int32>(argument_types, arguments);
+            return executeBitmapData<Int32>(argument_types, arguments, input_rows_count);
         else if (which.isInt64())
-            return executeBitmapData<Int64>(argument_types, arguments);
+            return executeBitmapData<Int64>(argument_types, arguments, input_rows_count);
         else
             throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Unexpected type {} of argument of function {}",
                 from_type->getName(), getName());
@@ -187,7 +187,7 @@ public:
 
 private:
     template <typename T>
-    ColumnPtr executeBitmapData(DataTypes & argument_types, const ColumnsWithTypeAndName & arguments) const
+    ColumnPtr executeBitmapData(DataTypes & argument_types, const ColumnsWithTypeAndName & arguments, size_t input_rows_count) const
     {
         // input data
         const ColumnArray * array = typeid_cast<const ColumnArray *>(arguments[0].column.get());
@@ -203,10 +203,10 @@ private:
         AggregateFunctionPtr bitmap_function = AggregateFunctionFactory::instance().get(
             AggregateFunctionGroupBitmapData<T>::name(), action, argument_types, params_row, properties);
         auto col_to = ColumnAggregateFunction::create(bitmap_function);
-        col_to->reserve(offsets.size());
+        col_to->reserve(input_rows_count);
 
         size_t pos = 0;
-        for (size_t i = 0; i < offsets.size(); ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             col_to->insertDefault();
             AggregateFunctionGroupBitmapData<T> & bitmap_data
diff --git a/src/Functions/FunctionsCharsetClassification.cpp b/src/Functions/FunctionsCharsetClassification.cpp
index 0a332ab70a9..f01ede64cc0 100644
--- a/src/Functions/FunctionsCharsetClassification.cpp
+++ b/src/Functions/FunctionsCharsetClassification.cpp
@@ -23,7 +23,7 @@ namespace
     constexpr size_t max_string_size = 1UL << 15;
 
     template <typename ModelMap>
-    ALWAYS_INLINE inline Float64 naiveBayes(
+    Float64 naiveBayes(
         const FrequencyHolder::EncodingMap & standard,
         const ModelMap & model,
         Float64 max_result)
@@ -51,7 +51,7 @@ namespace
 
     /// Count how many times each bigram occurs in the text.
     template <typename ModelMap>
-    ALWAYS_INLINE inline void calculateStats(
+    void calculateStats(
         const UInt8 * data,
         const size_t size,
         ModelMap & model)
@@ -77,24 +77,25 @@ struct CharsetClassificationImpl
         const ColumnString::Chars & data,
         const ColumnString::Offsets & offsets,
         ColumnString::Chars & res_data,
-        ColumnString::Offsets & res_offsets)
+        ColumnString::Offsets & res_offsets,
+        size_t input_rows_count)
     {
         const auto & encodings_freq = FrequencyHolder::getInstance().getEncodingsFrequency();
 
         if constexpr (detect_language)
             /// 2 chars for ISO code + 1 zero byte
-            res_data.reserve(offsets.size() * 3);
+            res_data.reserve(input_rows_count * 3);
         else
             /// Mean charset length is 8
-            res_data.reserve(offsets.size() * 8);
+            res_data.reserve(input_rows_count * 8);
 
-        res_offsets.resize(offsets.size());
+        res_offsets.resize(input_rows_count);
 
         size_t current_result_offset = 0;
 
         double zero_frequency_log = log(zero_frequency);
 
-        for (size_t i = 0; i < offsets.size(); ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             const UInt8 * str = data.data() + offsets[i - 1];
             const size_t str_len = offsets[i] - offsets[i - 1] - 1;
diff --git a/src/Functions/FunctionsCodingIP.cpp b/src/Functions/FunctionsCodingIP.cpp
index 0a97d029f84..0416df8f462 100644
--- a/src/Functions/FunctionsCodingIP.cpp
+++ b/src/Functions/FunctionsCodingIP.cpp
@@ -341,7 +341,7 @@ class FunctionIPv4NumToString : public IFunction
 {
 private:
     template <typename ArgType>
-    ColumnPtr executeTyped(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const
+    ColumnPtr executeTyped(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const
     {
         using ColumnType = ColumnVector<ArgType>;
 
@@ -356,12 +356,12 @@ private:
             ColumnString::Chars & vec_res = col_res->getChars();
             ColumnString::Offsets & offsets_res = col_res->getOffsets();
 
-            vec_res.resize(vec_in.size() * (IPV4_MAX_TEXT_LENGTH + 1)); /// the longest value is: 255.255.255.255\0
-            offsets_res.resize(vec_in.size());
+            vec_res.resize(input_rows_count * (IPV4_MAX_TEXT_LENGTH + 1)); /// the longest value is: 255.255.255.255\0
+            offsets_res.resize(input_rows_count);
             char * begin = reinterpret_cast<char *>(vec_res.data());
             char * pos = begin;
 
-            for (size_t i = 0; i < vec_in.size(); ++i)
+            for (size_t i = 0; i < input_rows_count; ++i)
             {
                 DB::formatIPv4(reinterpret_cast<const unsigned char*>(&vec_in[i]), sizeof(ArgType), pos, mask_tail_octets, "xxx");
                 offsets_res[i] = pos - begin;
@@ -532,7 +532,7 @@ public:
 
     bool useDefaultImplementationForConstants() const override { return true; }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
         const auto & col_type_name = arguments[0];
         const ColumnPtr & column = col_type_name.column;
@@ -542,11 +542,11 @@ public:
             auto col_res = ColumnIPv6::create();
 
             auto & vec_res = col_res->getData();
-            vec_res.resize(col_in->size());
+            vec_res.resize(input_rows_count);
 
             const auto & vec_in = col_in->getData();
 
-            for (size_t i = 0; i < vec_res.size(); ++i)
+            for (size_t i = 0; i < input_rows_count; ++i)
                 mapIPv4ToIPv6(vec_in[i], reinterpret_cast<UInt8 *>(&vec_res[i].toUnderType()));
 
             return col_res;
@@ -557,7 +557,7 @@ public:
             auto col_res = ColumnFixedString::create(IPV6_BINARY_LENGTH);
 
             auto & vec_res = col_res->getChars();
-            vec_res.resize(col_in->size() * IPV6_BINARY_LENGTH);
+            vec_res.resize(input_rows_count * IPV6_BINARY_LENGTH);
 
             const auto & vec_in = col_in->getData();
 
@@ -742,7 +742,7 @@ public:
 
     bool useDefaultImplementationForConstants() const override { return true; }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
         const ColumnPtr & column = arguments[0].column;
 
@@ -751,13 +751,13 @@ public:
             auto col_res = ColumnUInt64::create();
 
             ColumnUInt64::Container & vec_res = col_res->getData();
-            vec_res.resize(col->size());
+            vec_res.resize(input_rows_count);
 
             const ColumnString::Chars & vec_src = col->getChars();
             const ColumnString::Offsets & offsets_src = col->getOffsets();
             size_t prev_offset = 0;
 
-            for (size_t i = 0; i < vec_res.size(); ++i)
+            for (size_t i = 0; i < input_rows_count; ++i)
             {
                 size_t current_offset = offsets_src[i];
                 size_t string_size = current_offset - prev_offset - 1; /// mind the terminating zero byte
@@ -1054,7 +1054,7 @@ public:
         return std::make_shared<DataTypeUInt8>();
     }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
         const ColumnString * input_column = checkAndGetColumn<ColumnString>(arguments[0].column.get());
 
@@ -1067,14 +1067,14 @@ public:
         auto col_res = ColumnUInt8::create();
 
         ColumnUInt8::Container & vec_res = col_res->getData();
-        vec_res.resize(input_column->size());
+        vec_res.resize(input_rows_count);
 
         const ColumnString::Chars & vec_src = input_column->getChars();
         const ColumnString::Offsets & offsets_src = input_column->getOffsets();
         size_t prev_offset = 0;
         UInt32 result = 0;
 
-        for (size_t i = 0; i < vec_res.size(); ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             vec_res[i] = DB::parseIPv4whole(reinterpret_cast<const char *>(&vec_src[prev_offset]), reinterpret_cast<unsigned char *>(&result));
             prev_offset = offsets_src[i];
@@ -1110,7 +1110,7 @@ public:
         return std::make_shared<DataTypeUInt8>();
     }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
         const ColumnString * input_column = checkAndGetColumn<ColumnString>(arguments[0].column.get());
         if (!input_column)
@@ -1122,14 +1122,14 @@ public:
         auto col_res = ColumnUInt8::create();
 
         ColumnUInt8::Container & vec_res = col_res->getData();
-        vec_res.resize(input_column->size());
+        vec_res.resize(input_rows_count);
 
         const ColumnString::Chars & vec_src = input_column->getChars();
         const ColumnString::Offsets & offsets_src = input_column->getOffsets();
         size_t prev_offset = 0;
         char buffer[IPV6_BINARY_LENGTH];
 
-        for (size_t i = 0; i < vec_res.size(); ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             vec_res[i] = DB::parseIPv6whole(reinterpret_cast<const char *>(&vec_src[prev_offset]),
                                             reinterpret_cast<const char *>(&vec_src[offsets_src[i] - 1]),
diff --git a/src/Functions/FunctionsCodingUUID.cpp b/src/Functions/FunctionsCodingUUID.cpp
index 83fdcbc4af9..179ba1bf97a 100644
--- a/src/Functions/FunctionsCodingUUID.cpp
+++ b/src/Functions/FunctionsCodingUUID.cpp
@@ -177,7 +177,7 @@ public:
     bool useDefaultImplementationForConstants() const override { return true; }
     ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1}; }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
         const ColumnWithTypeAndName & col_type_name = arguments[0];
         const ColumnPtr & column = col_type_name.column;
@@ -189,21 +189,20 @@ public:
                                 "Illegal type {} of column {} argument of function {}, expected FixedString({})",
                                 col_type_name.type->getName(), col_in->getName(), getName(), uuid_bytes_length);
 
-            const auto size = col_in->size();
             const auto & vec_in = col_in->getChars();
 
             auto col_res = ColumnString::create();
 
             ColumnString::Chars & vec_res = col_res->getChars();
             ColumnString::Offsets & offsets_res = col_res->getOffsets();
-            vec_res.resize(size * (uuid_text_length + 1));
-            offsets_res.resize(size);
+            vec_res.resize(input_rows_count * (uuid_text_length + 1));
+            offsets_res.resize(input_rows_count);
 
             size_t src_offset = 0;
             size_t dst_offset = 0;
 
             const UUIDSerializer uuid_serializer(variant);
-            for (size_t i = 0; i < size; ++i)
+            for (size_t i = 0; i < input_rows_count; ++i)
             {
                 uuid_serializer.deserialize(&vec_in[src_offset], &vec_res[dst_offset]);
                 src_offset += uuid_bytes_length;
@@ -256,7 +255,7 @@ public:
     bool useDefaultImplementationForConstants() const override { return true; }
     ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {1}; }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
         const ColumnWithTypeAndName & col_type_name = arguments[0];
         const ColumnPtr & column = col_type_name.column;
@@ -266,17 +265,16 @@ public:
         {
             const auto & vec_in = col_in->getChars();
             const auto & offsets_in = col_in->getOffsets();
-            const size_t size = offsets_in.size();
 
             auto col_res = ColumnFixedString::create(uuid_bytes_length);
 
             ColumnString::Chars & vec_res = col_res->getChars();
-            vec_res.resize(size * uuid_bytes_length);
+            vec_res.resize(input_rows_count * uuid_bytes_length);
 
             size_t src_offset = 0;
             size_t dst_offset = 0;
 
-            for (size_t i = 0; i < size; ++i)
+            for (size_t i = 0; i < input_rows_count; ++i)
             {
                 /// If string has incorrect length - then return zero UUID.
                 /// If string has correct length but contains something not like UUID - implementation specific behaviour.
@@ -300,18 +298,17 @@ public:
                                 "Illegal type {} of column {} argument of function {}, expected FixedString({})",
                                 col_type_name.type->getName(), col_in_fixed->getName(), getName(), uuid_text_length);
 
-            const auto size = col_in_fixed->size();
             const auto & vec_in = col_in_fixed->getChars();
 
             auto col_res = ColumnFixedString::create(uuid_bytes_length);
 
             ColumnString::Chars & vec_res = col_res->getChars();
-            vec_res.resize(size * uuid_bytes_length);
+            vec_res.resize(input_rows_count * uuid_bytes_length);
 
             size_t src_offset = 0;
             size_t dst_offset = 0;
 
-            for (size_t i = 0; i < size; ++i)
+            for (size_t i = 0; i < input_rows_count; ++i)
             {
                 uuid_serializer.serialize(&vec_in[src_offset], &vec_res[dst_offset]);
                 src_offset += uuid_text_length;
@@ -359,7 +356,7 @@ public:
         return std::make_shared<DataTypeFixedString>(uuid_bytes_length);
     }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
         const ColumnWithTypeAndName & col_type_name = arguments[0];
         const ColumnPtr & column = col_type_name.column;
@@ -370,16 +367,15 @@ public:
         {
             const auto & vec_in = col_in->getData();
             const UUID * uuids = vec_in.data();
-            const size_t size = vec_in.size();
 
             auto col_res = ColumnFixedString::create(uuid_bytes_length);
 
             ColumnString::Chars & vec_res = col_res->getChars();
-            vec_res.resize(size * uuid_bytes_length);
+            vec_res.resize(input_rows_count * uuid_bytes_length);
 
             size_t dst_offset = 0;
 
-            for (size_t i = 0; i < size; ++i)
+            for (size_t i = 0; i < input_rows_count; ++i)
             {
                 uint64_t hiBytes = DB::UUIDHelpers::getHighBytes(uuids[i]);
                 uint64_t loBytes = DB::UUIDHelpers::getLowBytes(uuids[i]);
@@ -448,7 +444,7 @@ public:
         return std::make_shared<DataTypeDateTime64>(datetime_scale, timezone);
     }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
         const ColumnWithTypeAndName & col_type_name = arguments[0];
         const ColumnPtr & column = col_type_name.column;
@@ -457,12 +453,11 @@ public:
         {
             const auto & vec_in = col_in->getData();
             const UUID * uuids = vec_in.data();
-            const size_t size = vec_in.size();
 
-            auto col_res = ColumnDateTime64::create(size, datetime_scale);
+            auto col_res = ColumnDateTime64::create(input_rows_count, datetime_scale);
             auto & vec_res = col_res->getData();
 
-            for (size_t i = 0; i < size; ++i)
+            for (size_t i = 0; i < input_rows_count; ++i)
             {
                 const uint64_t hiBytes = DB::UUIDHelpers::getHighBytes(uuids[i]);
                 const uint64_t ms = ((hiBytes & 0xf000) == 0x7000) ? (hiBytes >> 16) : 0;
diff --git a/src/Functions/FunctionsDecimalArithmetics.h b/src/Functions/FunctionsDecimalArithmetics.h
index e26ad7362b3..9b9045f7c69 100644
--- a/src/Functions/FunctionsDecimalArithmetics.h
+++ b/src/Functions/FunctionsDecimalArithmetics.h
@@ -151,36 +151,36 @@ struct Processor
     template <typename FirstArgVectorType, typename SecondArgType>
     void NO_INLINE
     vectorConstant(const FirstArgVectorType & vec_first, const SecondArgType second_value,
-                   PaddedPODArray<typename ResultType::FieldType> & vec_to, UInt16 scale_a, UInt16 scale_b, UInt16 result_scale) const
+                   PaddedPODArray<typename ResultType::FieldType> & vec_to, UInt16 scale_a, UInt16 scale_b, UInt16 result_scale,
+                   size_t input_rows_count) const
     {
-        size_t size = vec_first.size();
-        vec_to.resize(size);
+        vec_to.resize(input_rows_count);
 
-        for (size_t i = 0; i < size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
             vec_to[i] = transform.execute(vec_first[i], second_value, scale_a, scale_b, result_scale);
     }
 
     template <typename FirstArgVectorType, typename SecondArgVectorType>
     void NO_INLINE
     vectorVector(const FirstArgVectorType & vec_first, const SecondArgVectorType & vec_second,
-                 PaddedPODArray<typename ResultType::FieldType> & vec_to, UInt16 scale_a, UInt16 scale_b, UInt16 result_scale) const
+                 PaddedPODArray<typename ResultType::FieldType> & vec_to, UInt16 scale_a, UInt16 scale_b, UInt16 result_scale,
+                 size_t input_rows_count) const
     {
-        size_t size = vec_first.size();
-        vec_to.resize(size);
+        vec_to.resize(input_rows_count);
 
-        for (size_t i = 0; i < size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
             vec_to[i] = transform.execute(vec_first[i], vec_second[i], scale_a, scale_b, result_scale);
     }
 
     template <typename FirstArgType, typename SecondArgVectorType>
     void NO_INLINE
     constantVector(const FirstArgType & first_value, const SecondArgVectorType & vec_second,
-                   PaddedPODArray<typename ResultType::FieldType> & vec_to, UInt16 scale_a, UInt16 scale_b, UInt16 result_scale) const
+                   PaddedPODArray<typename ResultType::FieldType> & vec_to, UInt16 scale_a, UInt16 scale_b, UInt16 result_scale,
+                   size_t input_rows_count) const
     {
-        size_t size = vec_second.size();
-        vec_to.resize(size);
+        vec_to.resize(input_rows_count);
 
-        for (size_t i = 0; i < size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
             vec_to[i] = transform.execute(first_value, vec_second[i], scale_a, scale_b, result_scale);
     }
 };
@@ -189,7 +189,7 @@ struct Processor
 template <typename FirstArgType, typename SecondArgType, typename ResultType, typename Transform>
 struct DecimalArithmeticsImpl
 {
-    static ColumnPtr execute(Transform transform, const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type)
+    static ColumnPtr execute(Transform transform, const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count)
     {
         using FirstArgValueType = typename FirstArgType::FieldType;
         using FirstArgColumnType = typename FirstArgType::ColumnType;
@@ -214,13 +214,13 @@ struct DecimalArithmeticsImpl
         if (first_col)
         {
             if (second_col_const)
-                op.vectorConstant(first_col->getData(), second_col_const->template getValue<SecondArgValueType>(), col_to->getData(), scale_a, scale_b, result_scale);
+                op.vectorConstant(first_col->getData(), second_col_const->template getValue<SecondArgValueType>(), col_to->getData(), scale_a, scale_b, result_scale, input_rows_count);
             else
-                op.vectorVector(first_col->getData(), second_col->getData(), col_to->getData(), scale_a, scale_b, result_scale);
+                op.vectorVector(first_col->getData(), second_col->getData(), col_to->getData(), scale_a, scale_b, result_scale, input_rows_count);
         }
         else if (first_col_const)
         {
-            op.constantVector(first_col_const->template getValue<FirstArgValueType>(), second_col->getData(), col_to->getData(), scale_a, scale_b, result_scale);
+            op.constantVector(first_col_const->template getValue<FirstArgValueType>(), second_col->getData(), col_to->getData(), scale_a, scale_b, result_scale, input_rows_count);
         }
         else
         {
@@ -293,14 +293,14 @@ public:
     bool useDefaultImplementationForConstants() const override { return true; }
     ColumnNumbers getArgumentsThatAreAlwaysConstant() const override { return {2}; }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t /*input_rows_count*/) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override
     {
-        return resolveOverload(arguments, result_type);
+        return resolveOverload(arguments, result_type, input_rows_count);
     }
 
 private:
     // long resolver to call proper templated func
-    ColumnPtr resolveOverload(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type) const
+    ColumnPtr resolveOverload(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const
     {
         WhichDataType which_dividend(arguments[0].type.get());
         WhichDataType which_divisor(arguments[1].type.get());
@@ -309,26 +309,26 @@ private:
         {
             using DividendType = DataTypeDecimal32;
             if (which_divisor.isDecimal32())
-                return DecimalArithmeticsImpl<DividendType, DataTypeDecimal32, DataTypeDecimal256, Transform>::execute(Transform{}, arguments, result_type);
+                return DecimalArithmeticsImpl<DividendType, DataTypeDecimal32, DataTypeDecimal256, Transform>::execute(Transform{}, arguments, result_type, input_rows_count);
             else if (which_divisor.isDecimal64())
-                return DecimalArithmeticsImpl<DividendType, DataTypeDecimal64, DataTypeDecimal256, Transform>::execute(Transform{}, arguments, result_type);
+                return DecimalArithmeticsImpl<DividendType, DataTypeDecimal64, DataTypeDecimal256, Transform>::execute(Transform{}, arguments, result_type, input_rows_count);
             else if (which_divisor.isDecimal128())
-                return DecimalArithmeticsImpl<DividendType, DataTypeDecimal128, DataTypeDecimal256, Transform>::execute(Transform{}, arguments, result_type);
+                return DecimalArithmeticsImpl<DividendType, DataTypeDecimal128, DataTypeDecimal256, Transform>::execute(Transform{}, arguments, result_type, input_rows_count);
             else if (which_divisor.isDecimal256())
-                return DecimalArithmeticsImpl<DividendType, DataTypeDecimal256, DataTypeDecimal256, Transform>::execute(Transform{}, arguments, result_type);
+                return DecimalArithmeticsImpl<DividendType, DataTypeDecimal256, DataTypeDecimal256, Transform>::execute(Transform{}, arguments, result_type, input_rows_count);
         }
 
         else if (which_dividend.isDecimal64())
         {
             using DividendType = DataTypeDecimal64;
             if (which_divisor.isDecimal32())
-                return DecimalArithmeticsImpl<DividendType, DataTypeDecimal32, DataTypeDecimal256, Transform>::execute(Transform{}, arguments, result_type);
+                return DecimalArithmeticsImpl<DividendType, DataTypeDecimal32, DataTypeDecimal256, Transform>::execute(Transform{}, arguments, result_type, input_rows_count);
             else if (which_divisor.isDecimal64())
-                return DecimalArithmeticsImpl<DividendType, DataTypeDecimal64, DataTypeDecimal256, Transform>::execute(Transform{}, arguments, result_type);
+                return DecimalArithmeticsImpl<DividendType, DataTypeDecimal64, DataTypeDecimal256, Transform>::execute(Transform{}, arguments, result_type, input_rows_count);
             else if (which_divisor.isDecimal128())
-                return DecimalArithmeticsImpl<DividendType, DataTypeDecimal128, DataTypeDecimal256, Transform>::execute(Transform{}, arguments, result_type);
+                return DecimalArithmeticsImpl<DividendType, DataTypeDecimal128, DataTypeDecimal256, Transform>::execute(Transform{}, arguments, result_type, input_rows_count);
             else if (which_divisor.isDecimal256())
-                return DecimalArithmeticsImpl<DividendType, DataTypeDecimal256, DataTypeDecimal256, Transform>::execute(Transform{}, arguments, result_type);
+                return DecimalArithmeticsImpl<DividendType, DataTypeDecimal256, DataTypeDecimal256, Transform>::execute(Transform{}, arguments, result_type, input_rows_count);
 
         }
 
@@ -336,13 +336,13 @@ private:
         {
             using DividendType = DataTypeDecimal128;
             if (which_divisor.isDecimal32())
-                return DecimalArithmeticsImpl<DividendType, DataTypeDecimal32, DataTypeDecimal256, Transform>::execute(Transform{}, arguments, result_type);
+                return DecimalArithmeticsImpl<DividendType, DataTypeDecimal32, DataTypeDecimal256, Transform>::execute(Transform{}, arguments, result_type, input_rows_count);
             else if (which_divisor.isDecimal64())
-                return DecimalArithmeticsImpl<DividendType, DataTypeDecimal64, DataTypeDecimal256, Transform>::execute(Transform{}, arguments, result_type);
+                return DecimalArithmeticsImpl<DividendType, DataTypeDecimal64, DataTypeDecimal256, Transform>::execute(Transform{}, arguments, result_type, input_rows_count);
             else if (which_divisor.isDecimal128())
-                return DecimalArithmeticsImpl<DividendType, DataTypeDecimal128, DataTypeDecimal256, Transform>::execute(Transform{}, arguments, result_type);
+                return DecimalArithmeticsImpl<DividendType, DataTypeDecimal128, DataTypeDecimal256, Transform>::execute(Transform{}, arguments, result_type, input_rows_count);
             else if (which_divisor.isDecimal256())
-                return DecimalArithmeticsImpl<DividendType, DataTypeDecimal256, DataTypeDecimal256, Transform>::execute(Transform{}, arguments, result_type);
+                return DecimalArithmeticsImpl<DividendType, DataTypeDecimal256, DataTypeDecimal256, Transform>::execute(Transform{}, arguments, result_type, input_rows_count);
 
         }
 
@@ -350,13 +350,13 @@ private:
         {
             using DividendType = DataTypeDecimal256;
             if (which_divisor.isDecimal32())
-                return DecimalArithmeticsImpl<DividendType, DataTypeDecimal32, DataTypeDecimal256, Transform>::execute(Transform{}, arguments, result_type);
+                return DecimalArithmeticsImpl<DividendType, DataTypeDecimal32, DataTypeDecimal256, Transform>::execute(Transform{}, arguments, result_type, input_rows_count);
             else if (which_divisor.isDecimal64())
-                return DecimalArithmeticsImpl<DividendType, DataTypeDecimal64, DataTypeDecimal256, Transform>::execute(Transform{}, arguments, result_type);
+                return DecimalArithmeticsImpl<DividendType, DataTypeDecimal64, DataTypeDecimal256, Transform>::execute(Transform{}, arguments, result_type, input_rows_count);
             else if (which_divisor.isDecimal128())
-                return DecimalArithmeticsImpl<DividendType, DataTypeDecimal128, DataTypeDecimal256, Transform>::execute(Transform{}, arguments, result_type);
+                return DecimalArithmeticsImpl<DividendType, DataTypeDecimal128, DataTypeDecimal256, Transform>::execute(Transform{}, arguments, result_type, input_rows_count);
             else if (which_divisor.isDecimal256())
-                return DecimalArithmeticsImpl<DividendType, DataTypeDecimal256, DataTypeDecimal256, Transform>::execute(Transform{}, arguments, result_type);
+                return DecimalArithmeticsImpl<DividendType, DataTypeDecimal256, DataTypeDecimal256, Transform>::execute(Transform{}, arguments, result_type, input_rows_count);
 
         }
 
diff --git a/src/Functions/FunctionsEmbeddedDictionaries.h b/src/Functions/FunctionsEmbeddedDictionaries.h
index 2f270bf999a..f27934ce5a9 100644
--- a/src/Functions/FunctionsEmbeddedDictionaries.h
+++ b/src/Functions/FunctionsEmbeddedDictionaries.h
@@ -181,7 +181,7 @@ public:
 
     bool isDeterministic() const override { return false; }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
         /// The dictionary key that defines the "point of view".
         std::string dict_key;
@@ -205,10 +205,9 @@ public:
 
             const typename ColumnVector<T>::Container & vec_from = col_from->getData();
             typename ColumnVector<T>::Container & vec_to = col_to->getData();
-            size_t size = vec_from.size();
-            vec_to.resize(size);
+            vec_to.resize(input_rows_count);
 
-            for (size_t i = 0; i < size; ++i)
+            for (size_t i = 0; i < input_rows_count; ++i)
                 vec_to[i] = Transform::apply(vec_from[i], dict);
 
             return col_to;
@@ -273,7 +272,7 @@ public:
 
     bool isDeterministic() const override { return false; }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
         /// The dictionary key that defines the "point of view".
         std::string dict_key;
@@ -303,10 +302,9 @@ public:
             const typename ColumnVector<T>::Container & vec_from1 = col_vec1->getData();
             const typename ColumnVector<T>::Container & vec_from2 = col_vec2->getData();
             typename ColumnUInt8::Container & vec_to = col_to->getData();
-            size_t size = vec_from1.size();
-            vec_to.resize(size);
+            vec_to.resize(input_rows_count);
 
-            for (size_t i = 0; i < size; ++i)
+            for (size_t i = 0; i < input_rows_count; ++i)
                 vec_to[i] = Transform::apply(vec_from1[i], vec_from2[i], dict);
 
             return col_to;
@@ -318,10 +316,9 @@ public:
             const typename ColumnVector<T>::Container & vec_from1 = col_vec1->getData();
             const T const_from2 = col_const2->template getValue<T>();
             typename ColumnUInt8::Container & vec_to = col_to->getData();
-            size_t size = vec_from1.size();
-            vec_to.resize(size);
+            vec_to.resize(input_rows_count);
 
-            for (size_t i = 0; i < size; ++i)
+            for (size_t i = 0; i < input_rows_count; ++i)
                 vec_to[i] = Transform::apply(vec_from1[i], const_from2, dict);
 
             return col_to;
@@ -333,10 +330,9 @@ public:
             const T const_from1 = col_const1->template getValue<T>();
             const typename ColumnVector<T>::Container & vec_from2 = col_vec2->getData();
             typename ColumnUInt8::Container & vec_to = col_to->getData();
-            size_t size = vec_from2.size();
-            vec_to.resize(size);
+            vec_to.resize(input_rows_count);
 
-            for (size_t i = 0; i < size; ++i)
+            for (size_t i = 0; i < input_rows_count; ++i)
                 vec_to[i] = Transform::apply(const_from1, vec_from2[i], dict);
 
             return col_to;
@@ -405,7 +401,7 @@ public:
 
     bool isDeterministic() const override { return false; }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
         /// The dictionary key that defines the "point of view".
         std::string dict_key;
@@ -432,11 +428,10 @@ public:
             auto & res_values = col_values->getData();
 
             const typename ColumnVector<T>::Container & vec_from = col_from->getData();
-            size_t size = vec_from.size();
-            res_offsets.resize(size);
-            res_values.reserve(size * 4);
+            res_offsets.resize(input_rows_count);
+            res_values.reserve(input_rows_count * 4);
 
-            for (size_t i = 0; i < size; ++i)
+            for (size_t i = 0; i < input_rows_count; ++i)
             {
                 T cur = vec_from[i];
                 for (size_t depth = 0; cur && depth < DBMS_HIERARCHICAL_DICTIONARY_MAX_DEPTH; ++depth)
diff --git a/src/Functions/FunctionsHashing.h b/src/Functions/FunctionsHashing.h
index 95c54ac9528..91e49bf5035 100644
--- a/src/Functions/FunctionsHashing.h
+++ b/src/Functions/FunctionsHashing.h
@@ -715,7 +715,7 @@ private:
     using ToType = typename Impl::ReturnType;
 
     template <typename FromType>
-    ColumnPtr executeType(const ColumnsWithTypeAndName & arguments) const
+    ColumnPtr executeType(const ColumnsWithTypeAndName & arguments, size_t input_rows_count) const
     {
         using ColVecType = ColumnVectorOrDecimal<FromType>;
 
@@ -726,9 +726,8 @@ private:
             const typename ColVecType::Container & vec_from = col_from->getData();
             typename ColumnVector<ToType>::Container & vec_to = col_to->getData();
 
-            size_t size = vec_from.size();
-            vec_to.resize(size);
-            for (size_t i = 0; i < size; ++i)
+            vec_to.resize(input_rows_count);
+            for (size_t i = 0; i < input_rows_count; ++i)
                 vec_to[i] = Impl::apply(vec_from[i]);
 
             return col_to;
@@ -759,39 +758,39 @@ public:
 
     bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
         const IDataType * from_type = arguments[0].type.get();
         WhichDataType which(from_type);
 
         if (which.isUInt8())
-            return executeType<UInt8>(arguments);
+            return executeType<UInt8>(arguments, input_rows_count);
         else if (which.isUInt16())
-            return executeType<UInt16>(arguments);
+            return executeType<UInt16>(arguments, input_rows_count);
         else if (which.isUInt32())
-            return executeType<UInt32>(arguments);
+            return executeType<UInt32>(arguments, input_rows_count);
         else if (which.isUInt64())
-            return executeType<UInt64>(arguments);
+            return executeType<UInt64>(arguments, input_rows_count);
         else if (which.isInt8())
-            return executeType<Int8>(arguments);
+            return executeType<Int8>(arguments, input_rows_count);
         else if (which.isInt16())
-            return executeType<Int16>(arguments);
+            return executeType<Int16>(arguments, input_rows_count);
         else if (which.isInt32())
-            return executeType<Int32>(arguments);
+            return executeType<Int32>(arguments, input_rows_count);
         else if (which.isInt64())
-            return executeType<Int64>(arguments);
+            return executeType<Int64>(arguments, input_rows_count);
         else if (which.isDate())
-            return executeType<UInt16>(arguments);
+            return executeType<UInt16>(arguments, input_rows_count);
         else if (which.isDate32())
-            return executeType<Int32>(arguments);
+            return executeType<Int32>(arguments, input_rows_count);
         else if (which.isDateTime())
-            return executeType<UInt32>(arguments);
+            return executeType<UInt32>(arguments, input_rows_count);
         else if (which.isDecimal32())
-            return executeType<Decimal32>(arguments);
+            return executeType<Decimal32>(arguments, input_rows_count);
         else if (which.isDecimal64())
-            return executeType<Decimal64>(arguments);
+            return executeType<Decimal64>(arguments, input_rows_count);
         else if (which.isIPv4())
-            return executeType<IPv4>(arguments);
+            return executeType<IPv4>(arguments, input_rows_count);
         else
             throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}",
                 arguments[0].type->getName(), getName());
@@ -843,7 +842,7 @@ private:
     using ToType = typename Impl::ReturnType;
 
     template <typename FromType, bool first>
-    void executeIntType(const KeyColumnsType & key_cols, const IColumn * column, typename ColumnVector<ToType>::Container & vec_to) const
+    void executeIntType(const KeyColumnsType & key_cols, const IColumn * column, typename ColumnVector<ToType>::Container & vec_to, size_t input_rows_count) const
     {
         using ColVecType = ColumnVectorOrDecimal<FromType>;
         KeyType key{};
@@ -853,8 +852,7 @@ private:
         if (const ColVecType * col_from = checkAndGetColumn<ColVecType>(column))
         {
             const typename ColVecType::Container & vec_from = col_from->getData();
-            const size_t size = vec_from.size();
-            for (size_t i = 0; i < size; ++i)
+            for (size_t i = 0; i < input_rows_count; ++i)
             {
                 ToType hash;
 
@@ -894,7 +892,7 @@ private:
                 if (!key_cols.is_const)
                 {
                     ColumnPtr full_column = col_from_const->convertToFullColumn();
-                    return executeIntType<FromType, first>(key_cols, full_column.get(), vec_to);
+                    return executeIntType<FromType, first>(key_cols, full_column.get(), vec_to, input_rows_count);
                 }
             }
             auto value = col_from_const->template getValue<FromType>();
@@ -938,7 +936,7 @@ private:
     }
 
     template <typename FromType, bool first>
-    void executeBigIntType(const KeyColumnsType & key_cols, const IColumn * column, typename ColumnVector<ToType>::Container & vec_to) const
+    void executeBigIntType(const KeyColumnsType & key_cols, const IColumn * column, typename ColumnVector<ToType>::Container & vec_to, size_t input_rows_count) const
     {
         using ColVecType = ColumnVectorOrDecimal<FromType>;
         KeyType key{};
@@ -958,8 +956,7 @@ private:
         if (const ColVecType * col_from = checkAndGetColumn<ColVecType>(column))
         {
             const typename ColVecType::Container & vec_from = col_from->getData();
-            size_t size = vec_from.size();
-            for (size_t i = 0; i < size; ++i)
+            for (size_t i = 0; i < input_rows_count; ++i)
             {
                 ToType hash;
                 if constexpr (Keyed)
@@ -987,7 +984,7 @@ private:
                 if (!key_cols.is_const)
                 {
                     ColumnPtr full_column = col_from_const->convertToFullColumn();
-                    return executeBigIntType<FromType, first>(key_cols, full_column.get(), vec_to);
+                    return executeBigIntType<FromType, first>(key_cols, full_column.get(), vec_to, input_rows_count);
                 }
             }
             auto value = col_from_const->template getValue<FromType>();
@@ -1014,12 +1011,12 @@ private:
     }
 
     template <bool first>
-    void executeGeneric(const KeyColumnsType & key_cols, const IColumn * column, typename ColumnVector<ToType>::Container & vec_to) const
+    void executeGeneric(const KeyColumnsType & key_cols, const IColumn * column, typename ColumnVector<ToType>::Container & vec_to, size_t input_rows_count) const
     {
         KeyType key{};
         if constexpr (Keyed)
             key = Impl::getKey(key_cols, 0);
-        for (size_t i = 0, size = column->size(); i < size; ++i)
+        for (size_t i = 0, size = input_rows_count; i < size; ++i)
         {
             if constexpr (Keyed)
                 if (!key_cols.is_const && i != 0)
@@ -1034,7 +1031,7 @@ private:
     }
 
     template <bool first>
-    void executeString(const KeyColumnsType & key_cols, const IColumn * column, typename ColumnVector<ToType>::Container & vec_to) const
+    void executeString(const KeyColumnsType & key_cols, const IColumn * column, typename ColumnVector<ToType>::Container & vec_to, size_t input_rows_count) const
     {
         KeyType key{};
         if constexpr (Keyed)
@@ -1043,10 +1040,9 @@ private:
         {
             const typename ColumnString::Chars & data = col_from->getChars();
             const typename ColumnString::Offsets & offsets = col_from->getOffsets();
-            size_t size = offsets.size();
 
             ColumnString::Offset current_offset = 0;
-            for (size_t i = 0; i < size; ++i)
+            for (size_t i = 0; i < input_rows_count; ++i)
             {
                 if constexpr (Keyed)
                     if (!key_cols.is_const && i != 0)
@@ -1067,9 +1063,8 @@ private:
         {
             const typename ColumnString::Chars & data = col_from_fixed->getChars();
             size_t n = col_from_fixed->getN();
-            size_t size = data.size() / n;
 
-            for (size_t i = 0; i < size; ++i)
+            for (size_t i = 0; i < input_rows_count; ++i)
             {
                 if constexpr (Keyed)
                     if (!key_cols.is_const && i != 0)
@@ -1088,7 +1083,7 @@ private:
                 if (!key_cols.is_const)
                 {
                     ColumnPtr full_column = col_from_const->convertToFullColumn();
-                    return executeString<first>(key_cols, full_column.get(), vec_to);
+                    return executeString<first>(key_cols, full_column.get(), vec_to, input_rows_count);
                 }
             }
             String value = col_from_const->getValue<String>();
@@ -1114,7 +1109,7 @@ private:
     }
 
     template <bool first>
-    void executeArray(const KeyColumnsType & key_cols, const IDataType * type, const IColumn * column, typename ColumnVector<ToType>::Container & vec_to) const
+    void executeArray(const KeyColumnsType & key_cols, const IDataType * type, const IColumn * column, typename ColumnVector<ToType>::Container & vec_to, size_t input_rows_count) const
     {
         const IDataType * nested_type = typeid_cast<const DataTypeArray &>(*type).getNestedType().get();
 
@@ -1131,18 +1126,16 @@ private:
             {
                 KeyColumnsType key_cols_tmp{key_cols};
                 key_cols_tmp.offsets = &offsets;
-                executeForArgument(key_cols_tmp, nested_type, nested_column, vec_temp, nested_is_first);
+                executeForArgument(key_cols_tmp, nested_type, nested_column, vec_temp, nested_is_first, input_rows_count);
             }
             else
-                executeForArgument(key_cols, nested_type, nested_column, vec_temp, nested_is_first);
-
-            const size_t size = offsets.size();
+                executeForArgument(key_cols, nested_type, nested_column, vec_temp, nested_is_first, input_rows_count);
 
             ColumnArray::Offset current_offset = 0;
             KeyType key{};
             if constexpr (Keyed)
                 key = Impl::getKey(key_cols, 0);
-            for (size_t i = 0; i < size; ++i)
+            for (size_t i = 0; i < input_rows_count; ++i)
             {
                 if constexpr (Keyed)
                     if (!key_cols.is_const && i != 0)
@@ -1170,7 +1163,7 @@ private:
         {
             /// NOTE: here, of course, you can do without the materialization of the column.
             ColumnPtr full_column = col_from_const->convertToFullColumn();
-            executeArray<first>(key_cols, type, full_column.get(), vec_to);
+            executeArray<first>(key_cols, type, full_column.get(), vec_to, input_rows_count);
         }
         else
             throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of first argument of function {}",
@@ -1178,7 +1171,7 @@ private:
     }
 
     template <bool first>
-    void executeAny(const KeyColumnsType & key_cols, const IDataType * from_type, const IColumn * icolumn, typename ColumnVector<ToType>::Container & vec_to) const
+    void executeAny(const KeyColumnsType & key_cols, const IDataType * from_type, const IColumn * icolumn, typename ColumnVector<ToType>::Container & vec_to, size_t input_rows_count) const
     {
         WhichDataType which(from_type);
 
@@ -1190,43 +1183,43 @@ private:
             if (key_cols.size() != vec_to.size() && key_cols.size() != 1)
                 throw Exception(ErrorCodes::LOGICAL_ERROR, "Key column size {} doesn't match result column size {} of function {}", key_cols.size(), vec_to.size(), getName());
 
-        if      (which.isUInt8()) executeIntType<UInt8, first>(key_cols, icolumn, vec_to);
-        else if (which.isUInt16()) executeIntType<UInt16, first>(key_cols, icolumn, vec_to);
-        else if (which.isUInt32()) executeIntType<UInt32, first>(key_cols, icolumn, vec_to);
-        else if (which.isUInt64()) executeIntType<UInt64, first>(key_cols, icolumn, vec_to);
-        else if (which.isUInt128()) executeBigIntType<UInt128, first>(key_cols, icolumn, vec_to);
-        else if (which.isUInt256()) executeBigIntType<UInt256, first>(key_cols, icolumn, vec_to);
-        else if (which.isInt8()) executeIntType<Int8, first>(key_cols, icolumn, vec_to);
-        else if (which.isInt16()) executeIntType<Int16, first>(key_cols, icolumn, vec_to);
-        else if (which.isInt32()) executeIntType<Int32, first>(key_cols, icolumn, vec_to);
-        else if (which.isInt64()) executeIntType<Int64, first>(key_cols, icolumn, vec_to);
-        else if (which.isInt128()) executeBigIntType<Int128, first>(key_cols, icolumn, vec_to);
-        else if (which.isInt256()) executeBigIntType<Int256, first>(key_cols, icolumn, vec_to);
-        else if (which.isUUID()) executeBigIntType<UUID, first>(key_cols, icolumn, vec_to);
-        else if (which.isIPv4()) executeIntType<IPv4, first>(key_cols, icolumn, vec_to);
-        else if (which.isIPv6()) executeBigIntType<IPv6, first>(key_cols, icolumn, vec_to);
-        else if (which.isEnum8()) executeIntType<Int8, first>(key_cols, icolumn, vec_to);
-        else if (which.isEnum16()) executeIntType<Int16, first>(key_cols, icolumn, vec_to);
-        else if (which.isDate()) executeIntType<UInt16, first>(key_cols, icolumn, vec_to);
-        else if (which.isDate32()) executeIntType<Int32, first>(key_cols, icolumn, vec_to);
-        else if (which.isDateTime()) executeIntType<UInt32, first>(key_cols, icolumn, vec_to);
+        if      (which.isUInt8()) executeIntType<UInt8, first>(key_cols, icolumn, vec_to, input_rows_count);
+        else if (which.isUInt16()) executeIntType<UInt16, first>(key_cols, icolumn, vec_to, input_rows_count);
+        else if (which.isUInt32()) executeIntType<UInt32, first>(key_cols, icolumn, vec_to, input_rows_count);
+        else if (which.isUInt64()) executeIntType<UInt64, first>(key_cols, icolumn, vec_to, input_rows_count);
+        else if (which.isUInt128()) executeBigIntType<UInt128, first>(key_cols, icolumn, vec_to, input_rows_count);
+        else if (which.isUInt256()) executeBigIntType<UInt256, first>(key_cols, icolumn, vec_to, input_rows_count);
+        else if (which.isInt8()) executeIntType<Int8, first>(key_cols, icolumn, vec_to, input_rows_count);
+        else if (which.isInt16()) executeIntType<Int16, first>(key_cols, icolumn, vec_to, input_rows_count);
+        else if (which.isInt32()) executeIntType<Int32, first>(key_cols, icolumn, vec_to, input_rows_count);
+        else if (which.isInt64()) executeIntType<Int64, first>(key_cols, icolumn, vec_to, input_rows_count);
+        else if (which.isInt128()) executeBigIntType<Int128, first>(key_cols, icolumn, vec_to, input_rows_count);
+        else if (which.isInt256()) executeBigIntType<Int256, first>(key_cols, icolumn, vec_to, input_rows_count);
+        else if (which.isUUID()) executeBigIntType<UUID, first>(key_cols, icolumn, vec_to, input_rows_count);
+        else if (which.isIPv4()) executeIntType<IPv4, first>(key_cols, icolumn, vec_to, input_rows_count);
+        else if (which.isIPv6()) executeBigIntType<IPv6, first>(key_cols, icolumn, vec_to, input_rows_count);
+        else if (which.isEnum8()) executeIntType<Int8, first>(key_cols, icolumn, vec_to, input_rows_count);
+        else if (which.isEnum16()) executeIntType<Int16, first>(key_cols, icolumn, vec_to, input_rows_count);
+        else if (which.isDate()) executeIntType<UInt16, first>(key_cols, icolumn, vec_to, input_rows_count);
+        else if (which.isDate32()) executeIntType<Int32, first>(key_cols, icolumn, vec_to, input_rows_count);
+        else if (which.isDateTime()) executeIntType<UInt32, first>(key_cols, icolumn, vec_to, input_rows_count);
         /// TODO: executeIntType() for Decimal32/64 leads to incompatible result
-        else if (which.isDecimal32()) executeBigIntType<Decimal32, first>(key_cols, icolumn, vec_to);
-        else if (which.isDecimal64()) executeBigIntType<Decimal64, first>(key_cols, icolumn, vec_to);
-        else if (which.isDecimal128()) executeBigIntType<Decimal128, first>(key_cols, icolumn, vec_to);
-        else if (which.isDecimal256()) executeBigIntType<Decimal256, first>(key_cols, icolumn, vec_to);
-        else if (which.isFloat32()) executeIntType<Float32, first>(key_cols, icolumn, vec_to);
-        else if (which.isFloat64()) executeIntType<Float64, first>(key_cols, icolumn, vec_to);
-        else if (which.isString()) executeString<first>(key_cols, icolumn, vec_to);
-        else if (which.isFixedString()) executeString<first>(key_cols, icolumn, vec_to);
-        else if (which.isArray()) executeArray<first>(key_cols, from_type, icolumn, vec_to);
-        else executeGeneric<first>(key_cols, icolumn, vec_to);
+        else if (which.isDecimal32()) executeBigIntType<Decimal32, first>(key_cols, icolumn, vec_to, input_rows_count);
+        else if (which.isDecimal64()) executeBigIntType<Decimal64, first>(key_cols, icolumn, vec_to, input_rows_count);
+        else if (which.isDecimal128()) executeBigIntType<Decimal128, first>(key_cols, icolumn, vec_to, input_rows_count);
+        else if (which.isDecimal256()) executeBigIntType<Decimal256, first>(key_cols, icolumn, vec_to, input_rows_count);
+        else if (which.isFloat32()) executeIntType<Float32, first>(key_cols, icolumn, vec_to, input_rows_count);
+        else if (which.isFloat64()) executeIntType<Float64, first>(key_cols, icolumn, vec_to, input_rows_count);
+        else if (which.isString()) executeString<first>(key_cols, icolumn, vec_to, input_rows_count);
+        else if (which.isFixedString()) executeString<first>(key_cols, icolumn, vec_to, input_rows_count);
+        else if (which.isArray()) executeArray<first>(key_cols, from_type, icolumn, vec_to, input_rows_count);
+        else executeGeneric<first>(key_cols, icolumn, vec_to, input_rows_count);
     }
 
     /// Return a fixed random-looking magic number when input is empty.
     static constexpr auto filler = 0xe28dbde7fe22e41c;
 
-    void executeForArgument(const KeyColumnsType & key_cols, const IDataType * type, const IColumn * column, typename ColumnVector<ToType>::Container & vec_to, bool & is_first) const
+    void executeForArgument(const KeyColumnsType & key_cols, const IDataType * type, const IColumn * column, typename ColumnVector<ToType>::Container & vec_to, bool & is_first, size_t input_rows_count) const
     {
         /// Flattening of tuples.
         if (const ColumnTuple * tuple = typeid_cast<const ColumnTuple *>(column))
@@ -1240,7 +1233,7 @@ private:
                     hash = static_cast<ToType>(filler);
 
             for (size_t i = 0; i < tuple_size; ++i)
-                executeForArgument(key_cols, tuple_types[i].get(), tuple_columns[i].get(), vec_to, is_first);
+                executeForArgument(key_cols, tuple_types[i].get(), tuple_columns[i].get(), vec_to, is_first, input_rows_count);
         }
         else if (const ColumnTuple * tuple_const = checkAndGetColumnConstData<ColumnTuple>(column))
         {
@@ -1255,24 +1248,24 @@ private:
             for (size_t i = 0; i < tuple_size; ++i)
             {
                 auto tmp = ColumnConst::create(tuple_columns[i], column->size());
-                executeForArgument(key_cols, tuple_types[i].get(), tmp.get(), vec_to, is_first);
+                executeForArgument(key_cols, tuple_types[i].get(), tmp.get(), vec_to, is_first, input_rows_count);
             }
         }
         else if (const auto * map = checkAndGetColumn<ColumnMap>(column))
         {
             const auto & type_map = assert_cast<const DataTypeMap &>(*type);
-            executeForArgument(key_cols, type_map.getNestedType().get(), map->getNestedColumnPtr().get(), vec_to, is_first);
+            executeForArgument(key_cols, type_map.getNestedType().get(), map->getNestedColumnPtr().get(), vec_to, is_first, input_rows_count);
         }
         else if (const auto * const_map = checkAndGetColumnConst<ColumnMap>(column))
         {
-            executeForArgument(key_cols, type, const_map->convertToFullColumnIfConst().get(), vec_to, is_first);
+            executeForArgument(key_cols, type, const_map->convertToFullColumnIfConst().get(), vec_to, is_first, input_rows_count);
         }
         else
         {
             if (is_first)
-                executeAny<true>(key_cols, type, column, vec_to);
+                executeAny<true>(key_cols, type, column, vec_to, input_rows_count);
             else
-                executeAny<false>(key_cols, type, column, vec_to);
+                executeAny<false>(key_cols, type, column, vec_to, input_rows_count);
         }
 
         is_first = false;
@@ -1325,7 +1318,7 @@ public:
             for (size_t i = first_data_argument; i < arguments.size(); ++i)
             {
                 const auto & col = arguments[i];
-                executeForArgument(key_cols, col.type.get(), col.column.get(), vec_to, is_first_argument);
+                executeForArgument(key_cols, col.type.get(), col.column.get(), vec_to, is_first_argument, input_rows_count);
             }
         }
 
diff --git a/src/Functions/FunctionsJSON.cpp b/src/Functions/FunctionsJSON.cpp
index 848856c500f..c35df8ba72d 100644
--- a/src/Functions/FunctionsJSON.cpp
+++ b/src/Functions/FunctionsJSON.cpp
@@ -125,7 +125,7 @@ public:
             }
 
             String error;
-            for (const auto i : collections::range(0, input_rows_count))
+            for (size_t i = 0; i < input_rows_count; ++i)
             {
                 if (!col_json_const)
                 {
@@ -314,7 +314,7 @@ private:
     static size_t calculateMaxSize(const ColumnString::Offsets & offsets)
     {
         size_t max_size = 0;
-        for (const auto i : collections::range(0, offsets.size()))
+        for (size_t i = 0; i < offsets.size(); ++i)
         {
             size_t size = offsets[i] - offsets[i - 1];
             max_size = std::max(max_size, size);
diff --git a/src/Functions/FunctionsLanguageClassification.cpp b/src/Functions/FunctionsLanguageClassification.cpp
index 55485d41ce0..410eea0f437 100644
--- a/src/Functions/FunctionsLanguageClassification.cpp
+++ b/src/Functions/FunctionsLanguageClassification.cpp
@@ -31,7 +31,7 @@ extern const int SUPPORT_IS_DISABLED;
 
 struct FunctionDetectLanguageImpl
 {
-    static ALWAYS_INLINE inline std::string_view codeISO(std::string_view code_string)
+    static std::string_view codeISO(std::string_view code_string)
     {
         if (code_string.ends_with("-Latn"))
             code_string.remove_suffix(code_string.size() - 5);
@@ -63,16 +63,17 @@ struct FunctionDetectLanguageImpl
         const ColumnString::Chars & data,
         const ColumnString::Offsets & offsets,
         ColumnString::Chars & res_data,
-        ColumnString::Offsets & res_offsets)
+        ColumnString::Offsets & res_offsets,
+        size_t input_rows_count)
     {
         /// Constant 3 is based on the fact that in general we need 2 characters for ISO code + 1 zero byte
-        res_data.reserve(offsets.size() * 3);
-        res_offsets.resize(offsets.size());
+        res_data.reserve(input_rows_count * 3);
+        res_offsets.resize(input_rows_count);
 
         bool is_reliable;
         size_t res_offset = 0;
 
-        for (size_t i = 0; i < offsets.size(); ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             const UInt8 * str = data.data() + offsets[i - 1];
             const size_t str_len = offsets[i] - offsets[i - 1] - 1;
diff --git a/src/Functions/FunctionsLogical.cpp b/src/Functions/FunctionsLogical.cpp
index e1814150da6..65d7473b945 100644
--- a/src/Functions/FunctionsLogical.cpp
+++ b/src/Functions/FunctionsLogical.cpp
@@ -48,7 +48,7 @@ using UInt8Container = ColumnUInt8::Container;
 using UInt8ColumnPtrs = std::vector<const ColumnUInt8 *>;
 
 
-MutableColumnPtr buildColumnFromTernaryData(const UInt8Container & ternary_data, const bool make_nullable)
+MutableColumnPtr buildColumnFromTernaryData(const UInt8Container & ternary_data, bool make_nullable)
 {
     const size_t rows_count = ternary_data.size();
 
diff --git a/src/Functions/FunctionsMultiStringFuzzySearch.h b/src/Functions/FunctionsMultiStringFuzzySearch.h
index a92a6570279..8346380c35d 100644
--- a/src/Functions/FunctionsMultiStringFuzzySearch.h
+++ b/src/Functions/FunctionsMultiStringFuzzySearch.h
@@ -71,7 +71,7 @@ public:
         return Impl::getReturnType();
     }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
         const ColumnPtr & haystack_ptr = arguments[0].column;
         const ColumnPtr & edit_distance_ptr = arguments[1].column;
@@ -114,14 +114,16 @@ public:
                 col_needles_const->getValue<Array>(),
                 vec_res, offsets_res,
                 edit_distance,
-                allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps);
+                allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps,
+                input_rows_count);
         else
             Impl::vectorVector(
                 col_haystack_vector->getChars(), col_haystack_vector->getOffsets(),
                 col_needles_vector->getData(), col_needles_vector->getOffsets(),
                 vec_res, offsets_res,
                 edit_distance,
-                allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps);
+                allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps,
+                input_rows_count);
 
         // the combination of const haystack + const needle is not implemented because
         // useDefaultImplementationForConstants() == true makes upper layers convert both to
diff --git a/src/Functions/FunctionsMultiStringSearch.h b/src/Functions/FunctionsMultiStringSearch.h
index 03db2651fd0..6bcc8581a38 100644
--- a/src/Functions/FunctionsMultiStringSearch.h
+++ b/src/Functions/FunctionsMultiStringSearch.h
@@ -81,7 +81,7 @@ public:
         return Impl::getReturnType();
     }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
         const ColumnPtr & haystack_ptr = arguments[0].column;
         const ColumnPtr & needles_ptr = arguments[1].column;
@@ -110,13 +110,15 @@ public:
                 col_haystack_vector->getChars(), col_haystack_vector->getOffsets(),
                 col_needles_const->getValue<Array>(),
                 vec_res, offsets_res,
-                allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps);
+                allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps,
+                input_rows_count);
         else
             Impl::vectorVector(
                 col_haystack_vector->getChars(), col_haystack_vector->getOffsets(),
                 col_needles_vector->getData(), col_needles_vector->getOffsets(),
                 vec_res, offsets_res,
-                allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps);
+                allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps,
+                input_rows_count);
 
         // the combination of const haystack + const needle is not implemented because
         // useDefaultImplementationForConstants() == true makes upper layers convert both to
diff --git a/src/Functions/FunctionsProgrammingClassification.cpp b/src/Functions/FunctionsProgrammingClassification.cpp
index c01e47ad0d7..bbedef024d5 100644
--- a/src/Functions/FunctionsProgrammingClassification.cpp
+++ b/src/Functions/FunctionsProgrammingClassification.cpp
@@ -40,17 +40,18 @@ struct FunctionDetectProgrammingLanguageImpl
         const ColumnString::Chars & data,
         const ColumnString::Offsets & offsets,
         ColumnString::Chars & res_data,
-        ColumnString::Offsets & res_offsets)
+        ColumnString::Offsets & res_offsets,
+        size_t input_rows_count)
     {
         const auto & programming_freq = FrequencyHolder::getInstance().getProgrammingFrequency();
 
         /// Constant 5 is arbitrary
-        res_data.reserve(offsets.size() * 5);
-        res_offsets.resize(offsets.size());
+        res_data.reserve(input_rows_count * 5);
+        res_offsets.resize(input_rows_count);
 
         size_t res_offset = 0;
 
-        for (size_t i = 0; i < offsets.size(); ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             const UInt8 * str = data.data() + offsets[i - 1];
             const size_t str_len = offsets[i] - offsets[i - 1] - 1;
diff --git a/src/Functions/FunctionsRandom.h b/src/Functions/FunctionsRandom.h
index 36448c6f689..83075ca01cb 100644
--- a/src/Functions/FunctionsRandom.h
+++ b/src/Functions/FunctionsRandom.h
@@ -80,8 +80,7 @@ public:
         auto col_to = ColumnVector<ToType>::create();
         typename ColumnVector<ToType>::Container & vec_to = col_to->getData();
 
-        size_t size = input_rows_count;
-        vec_to.resize(size);
+        vec_to.resize(input_rows_count);
         RandImpl::execute(reinterpret_cast<char *>(vec_to.data()), vec_to.size() * sizeof(ToType));
 
         return col_to;
diff --git a/src/Functions/FunctionsStringDistance.cpp b/src/Functions/FunctionsStringDistance.cpp
index 48f4aaf4e09..5aae92a8141 100644
--- a/src/Functions/FunctionsStringDistance.cpp
+++ b/src/Functions/FunctionsStringDistance.cpp
@@ -37,12 +37,12 @@ struct FunctionStringDistanceImpl
         const ColumnString::Offsets & haystack_offsets,
         const ColumnString::Chars & needle_data,
         const ColumnString::Offsets & needle_offsets,
-        PaddedPODArray<ResultType> & res)
+        PaddedPODArray<ResultType> & res,
+        size_t input_rows_count)
     {
-        size_t size = res.size();
         const char * haystack = reinterpret_cast<const char *>(haystack_data.data());
         const char * needle = reinterpret_cast<const char *>(needle_data.data());
-        for (size_t i = 0; i < size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             res[i] = Op::process(
                 haystack + haystack_offsets[i - 1],
@@ -56,13 +56,13 @@ struct FunctionStringDistanceImpl
         const String & haystack,
         const ColumnString::Chars & needle_data,
         const ColumnString::Offsets & needle_offsets,
-        PaddedPODArray<ResultType> & res)
+        PaddedPODArray<ResultType> & res,
+        size_t input_rows_count)
     {
         const char * haystack_data = haystack.data();
         size_t haystack_size = haystack.size();
         const char * needle = reinterpret_cast<const char *>(needle_data.data());
-        size_t size = res.size();
-        for (size_t i = 0; i < size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             res[i] = Op::process(haystack_data, haystack_size,
                 needle + needle_offsets[i - 1], needle_offsets[i] - needle_offsets[i - 1] - 1);
@@ -73,9 +73,10 @@ struct FunctionStringDistanceImpl
         const ColumnString::Chars & data,
         const ColumnString::Offsets & offsets,
         const String & needle,
-        PaddedPODArray<ResultType> & res)
+        PaddedPODArray<ResultType> & res,
+        size_t input_rows_count)
     {
-        constantVector(needle, data, offsets, res);
+        constantVector(needle, data, offsets, res, input_rows_count);
     }
 
 };
diff --git a/src/Functions/FunctionsStringHash.cpp b/src/Functions/FunctionsStringHash.cpp
index 0bf6e39e651..bd7d45d781a 100644
--- a/src/Functions/FunctionsStringHash.cpp
+++ b/src/Functions/FunctionsStringHash.cpp
@@ -315,9 +315,9 @@ struct SimHashImpl
         return getSimHash(finger_vec);
     }
 
-    static void apply(const ColumnString::Chars & data, const ColumnString::Offsets & offsets, size_t shingle_size, PaddedPODArray<UInt64> & res)
+    static void apply(const ColumnString::Chars & data, const ColumnString::Offsets & offsets, size_t shingle_size, PaddedPODArray<UInt64> & res, size_t input_rows_count)
     {
-        for (size_t i = 0; i < offsets.size(); ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             const UInt8 * one_data = &data[offsets[i - 1]];
             const size_t data_size = offsets[i] - offsets[i - 1] - 1;
@@ -543,12 +543,13 @@ struct MinHashImpl
         PaddedPODArray<UInt64> * res1,
         PaddedPODArray<UInt64> * res2,
         ColumnTuple * res1_strings,
-        ColumnTuple * res2_strings)
+        ColumnTuple * res2_strings,
+        size_t input_rows_count)
     {
         MinHeap min_heap;
         MaxHeap max_heap;
 
-        for (size_t i = 0; i < offsets.size(); ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             const UInt8 * one_data = &data[offsets[i - 1]];
             const size_t data_size = offsets[i] - offsets[i - 1] - 1;
diff --git a/src/Functions/FunctionsStringHash.h b/src/Functions/FunctionsStringHash.h
index fcd4c970a47..f790c660e21 100644
--- a/src/Functions/FunctionsStringHash.h
+++ b/src/Functions/FunctionsStringHash.h
@@ -135,7 +135,7 @@ public:
 
     bool useDefaultImplementationForConstants() const override { return true; }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
         const ColumnPtr & column = arguments[0].column;
 
@@ -152,9 +152,9 @@ public:
         {
             auto col_res = ColumnVector<UInt64>::create();
             auto & vec_res = col_res->getData();
-            vec_res.resize(column->size());
+            vec_res.resize(input_rows_count);
             const ColumnString & col_str_vector = checkAndGetColumn<ColumnString>(*column);
-            Impl::apply(col_str_vector.getChars(), col_str_vector.getOffsets(), shingle_size, vec_res);
+            Impl::apply(col_str_vector.getChars(), col_str_vector.getOffsets(), shingle_size, vec_res, input_rows_count);
             return col_res;
         }
         else if constexpr (is_arg) // Min hash arg
@@ -171,7 +171,7 @@ public:
             auto max_tuple = ColumnTuple::create(std::move(max_columns));
 
             const ColumnString & col_str_vector = checkAndGetColumn<ColumnString>(*column);
-            Impl::apply(col_str_vector.getChars(), col_str_vector.getOffsets(), shingle_size, num_hashes, nullptr, nullptr, min_tuple.get(), max_tuple.get());
+            Impl::apply(col_str_vector.getChars(), col_str_vector.getOffsets(), shingle_size, num_hashes, nullptr, nullptr, min_tuple.get(), max_tuple.get(), input_rows_count);
 
             MutableColumns tuple_columns;
             tuple_columns.emplace_back(std::move(min_tuple));
@@ -184,10 +184,10 @@ public:
             auto col_h2 = ColumnVector<UInt64>::create();
             auto & vec_h1 = col_h1->getData();
             auto & vec_h2 = col_h2->getData();
-            vec_h1.resize(column->size());
-            vec_h2.resize(column->size());
+            vec_h1.resize(input_rows_count);
+            vec_h2.resize(input_rows_count);
             const ColumnString & col_str_vector = checkAndGetColumn<ColumnString>(*column);
-            Impl::apply(col_str_vector.getChars(), col_str_vector.getOffsets(), shingle_size, num_hashes, &vec_h1, &vec_h2, nullptr, nullptr);
+            Impl::apply(col_str_vector.getChars(), col_str_vector.getOffsets(), shingle_size, num_hashes, &vec_h1, &vec_h2, nullptr, nullptr, input_rows_count);
             MutableColumns tuple_columns;
             tuple_columns.emplace_back(std::move(col_h1));
             tuple_columns.emplace_back(std::move(col_h2));
diff --git a/src/Functions/FunctionsStringHashFixedString.cpp b/src/Functions/FunctionsStringHashFixedString.cpp
index 01e989a7f2c..9474fe2629e 100644
--- a/src/Functions/FunctionsStringHashFixedString.cpp
+++ b/src/Functions/FunctionsStringHashFixedString.cpp
@@ -224,7 +224,7 @@ public:
 
     bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
         if (const ColumnString * col_from = checkAndGetColumn<ColumnString>(arguments[0].column.get()))
         {
@@ -233,11 +233,10 @@ public:
             const typename ColumnString::Chars & data = col_from->getChars();
             const typename ColumnString::Offsets & offsets = col_from->getOffsets();
             auto & chars_to = col_to->getChars();
-            const auto size = offsets.size();
-            chars_to.resize(size * Impl::length);
+            chars_to.resize(input_rows_count * Impl::length);
 
             ColumnString::Offset current_offset = 0;
-            for (size_t i = 0; i < size; ++i)
+            for (size_t i = 0; i < input_rows_count; ++i)
             {
                 Impl::apply(
                     reinterpret_cast<const char *>(&data[current_offset]),
@@ -253,11 +252,10 @@ public:
         {
             auto col_to = ColumnFixedString::create(Impl::length);
             const typename ColumnFixedString::Chars & data = col_from_fix->getChars();
-            const auto size = col_from_fix->size();
             auto & chars_to = col_to->getChars();
             const auto length = col_from_fix->getN();
-            chars_to.resize(size * Impl::length);
-            for (size_t i = 0; i < size; ++i)
+            chars_to.resize(input_rows_count * Impl::length);
+            for (size_t i = 0; i < input_rows_count; ++i)
             {
                 Impl::apply(
                     reinterpret_cast<const char *>(&data[i * length]), length, reinterpret_cast<uint8_t *>(&chars_to[i * Impl::length]));
@@ -268,11 +266,10 @@ public:
         {
             auto col_to = ColumnFixedString::create(Impl::length);
             const typename ColumnIPv6::Container & data = col_from_ip->getData();
-            const auto size = col_from_ip->size();
             auto & chars_to = col_to->getChars();
             const auto length = sizeof(IPv6::UnderlyingType);
-            chars_to.resize(size * Impl::length);
-            for (size_t i = 0; i < size; ++i)
+            chars_to.resize(input_rows_count * Impl::length);
+            for (size_t i = 0; i < input_rows_count; ++i)
             {
                 Impl::apply(
                     reinterpret_cast<const char *>(&data[i]), length, reinterpret_cast<uint8_t *>(&chars_to[i * Impl::length]));
diff --git a/src/Functions/FunctionsStringSearch.h b/src/Functions/FunctionsStringSearch.h
index fba6336ebff..7ec0076e395 100644
--- a/src/Functions/FunctionsStringSearch.h
+++ b/src/Functions/FunctionsStringSearch.h
@@ -163,7 +163,7 @@ public:
         return return_type;
     }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t /*input_rows_count*/) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override
     {
         const ColumnPtr & column_haystack = (argument_order == ArgumentOrder::HaystackNeedle) ? arguments[0].column : arguments[1].column;
         const ColumnPtr & column_needle = (argument_order == ArgumentOrder::HaystackNeedle) ? arguments[1].column : arguments[0].column;
@@ -236,7 +236,8 @@ public:
                 col_needle_vector->getOffsets(),
                 column_start_pos,
                 vec_res,
-                null_map.get());
+                null_map.get(),
+                input_rows_count);
         else if (col_haystack_vector && col_needle_const)
             Impl::vectorConstant(
                 col_haystack_vector->getChars(),
@@ -244,7 +245,8 @@ public:
                 col_needle_const->getValue<String>(),
                 column_start_pos,
                 vec_res,
-                null_map.get());
+                null_map.get(),
+                input_rows_count);
         else if (col_haystack_vector_fixed && col_needle_vector)
             Impl::vectorFixedVector(
                 col_haystack_vector_fixed->getChars(),
@@ -253,14 +255,16 @@ public:
                 col_needle_vector->getOffsets(),
                 column_start_pos,
                 vec_res,
-                null_map.get());
+                null_map.get(),
+                input_rows_count);
         else if (col_haystack_vector_fixed && col_needle_const)
             Impl::vectorFixedConstant(
                 col_haystack_vector_fixed->getChars(),
                 col_haystack_vector_fixed->getN(),
                 col_needle_const->getValue<String>(),
                 vec_res,
-                null_map.get());
+                null_map.get(),
+                input_rows_count);
         else if (col_haystack_const && col_needle_vector)
             Impl::constantVector(
                 col_haystack_const->getValue<String>(),
@@ -268,7 +272,8 @@ public:
                 col_needle_vector->getOffsets(),
                 column_start_pos,
                 vec_res,
-                null_map.get());
+                null_map.get(),
+                input_rows_count);
         else
             throw Exception(
                 ErrorCodes::ILLEGAL_COLUMN,
diff --git a/src/Functions/FunctionsStringSearchToString.h b/src/Functions/FunctionsStringSearchToString.h
index 978a84de472..c889cf062a3 100644
--- a/src/Functions/FunctionsStringSearchToString.h
+++ b/src/Functions/FunctionsStringSearchToString.h
@@ -60,7 +60,7 @@ public:
         return std::make_shared<DataTypeString>();
     }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
         const ColumnPtr column = arguments[0].column;
         const ColumnPtr column_needle = arguments[1].column;
@@ -75,7 +75,7 @@ public:
 
             ColumnString::Chars & vec_res = col_res->getChars();
             ColumnString::Offsets & offsets_res = col_res->getOffsets();
-            Impl::vector(col->getChars(), col->getOffsets(), col_needle->getValue<String>(), vec_res, offsets_res);
+            Impl::vector(col->getChars(), col->getOffsets(), col_needle->getValue<String>(), vec_res, offsets_res, input_rows_count);
 
             return col_res;
         }
diff --git a/src/Functions/FunctionsStringSimilarity.cpp b/src/Functions/FunctionsStringSimilarity.cpp
index 7b3f2337c89..5e26e4ad482 100644
--- a/src/Functions/FunctionsStringSimilarity.cpp
+++ b/src/Functions/FunctionsStringSimilarity.cpp
@@ -90,7 +90,7 @@ struct NgramDistanceImpl
         ((cont[Offset + I] = std::tolower(cont[Offset + I])), ...);
     }
 
-    static ALWAYS_INLINE size_t readASCIICodePoints(CodePoint * code_points, const char *& pos, const char * end)
+    static size_t readASCIICodePoints(CodePoint * code_points, const char *& pos, const char * end)
     {
         /// Offset before which we copy some data.
         constexpr size_t padding_offset = default_padding - N + 1;
@@ -120,7 +120,7 @@ struct NgramDistanceImpl
         return default_padding;
     }
 
-    static ALWAYS_INLINE size_t readUTF8CodePoints(CodePoint * code_points, const char *& pos, const char * end)
+    static size_t readUTF8CodePoints(CodePoint * code_points, const char *& pos, const char * end)
     {
         /// The same copying as described in the function above.
         memcpy(code_points, code_points + default_padding - N + 1, roundUpToPowerOfTwoOrZero(N - 1) * sizeof(CodePoint));
@@ -195,7 +195,7 @@ struct NgramDistanceImpl
     }
 
     template <bool save_ngrams>
-    static ALWAYS_INLINE inline size_t calculateNeedleStats(
+    static inline size_t calculateNeedleStats(
         const char * data,
         const size_t size,
         NgramCount * ngram_stats,
@@ -228,7 +228,7 @@ struct NgramDistanceImpl
     }
 
     template <bool reuse_stats>
-    static ALWAYS_INLINE inline UInt64 calculateHaystackStatsAndMetric(
+    static inline UInt64 calculateHaystackStatsAndMetric(
         const char * data,
         const size_t size,
         NgramCount * ngram_stats,
@@ -318,9 +318,9 @@ struct NgramDistanceImpl
         const ColumnString::Offsets & haystack_offsets,
         const ColumnString::Chars & needle_data,
         const ColumnString::Offsets & needle_offsets,
-        PaddedPODArray<Float32> & res)
+        PaddedPODArray<Float32> & res,
+        size_t input_rows_count)
     {
-        const size_t haystack_offsets_size = haystack_offsets.size();
         size_t prev_haystack_offset = 0;
         size_t prev_needle_offset = 0;
 
@@ -331,7 +331,7 @@ struct NgramDistanceImpl
         std::unique_ptr<UInt16[]> needle_ngram_storage(new UInt16[max_string_size]);
         std::unique_ptr<UInt16[]> haystack_ngram_storage(new UInt16[max_string_size]);
 
-        for (size_t i = 0; i < haystack_offsets_size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             const char * haystack = reinterpret_cast<const char *>(&haystack_data[prev_haystack_offset]);
             const size_t haystack_size = haystack_offsets[i] - prev_haystack_offset - 1;
@@ -391,12 +391,13 @@ struct NgramDistanceImpl
         std::string haystack,
         const ColumnString::Chars & needle_data,
         const ColumnString::Offsets & needle_offsets,
-        PaddedPODArray<Float32> & res)
+        PaddedPODArray<Float32> & res,
+        size_t input_rows_count)
     {
         /// For symmetric version it is better to use vector_constant
         if constexpr (symmetric)
         {
-            vectorConstant(needle_data, needle_offsets, std::move(haystack), res);
+            vectorConstant(needle_data, needle_offsets, std::move(haystack), res, input_rows_count);
         }
         else
         {
@@ -404,7 +405,6 @@ struct NgramDistanceImpl
             haystack.resize(haystack_size + default_padding);
 
             /// For logic explanation see vector_vector function.
-            const size_t needle_offsets_size = needle_offsets.size();
             size_t prev_offset = 0;
 
             std::unique_ptr<NgramCount[]> common_stats{new NgramCount[map_size]{}};
@@ -412,7 +412,7 @@ struct NgramDistanceImpl
             std::unique_ptr<UInt16[]> needle_ngram_storage(new UInt16[max_string_size]);
             std::unique_ptr<UInt16[]> haystack_ngram_storage(new UInt16[max_string_size]);
 
-            for (size_t i = 0; i < needle_offsets_size; ++i)
+            for (size_t i = 0; i < input_rows_count; ++i)
             {
                 const char * needle = reinterpret_cast<const char *>(&needle_data[prev_offset]);
                 const size_t needle_size = needle_offsets[i] - prev_offset - 1;
@@ -456,7 +456,8 @@ struct NgramDistanceImpl
         const ColumnString::Chars & data,
         const ColumnString::Offsets & offsets,
         std::string needle,
-        PaddedPODArray<Float32> & res)
+        PaddedPODArray<Float32> & res,
+        size_t input_rows_count)
     {
         /// zeroing our map
         std::unique_ptr<NgramCount[]> common_stats{new NgramCount[map_size]{}};
@@ -472,7 +473,7 @@ struct NgramDistanceImpl
 
         size_t distance = needle_stats_size;
         size_t prev_offset = 0;
-        for (size_t i = 0; i < offsets.size(); ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             const UInt8 * haystack = &data[prev_offset];
             const size_t haystack_size = offsets[i] - prev_offset - 1;
diff --git a/src/Functions/FunctionsStringSimilarity.h b/src/Functions/FunctionsStringSimilarity.h
index e148730054d..c2cf7137286 100644
--- a/src/Functions/FunctionsStringSimilarity.h
+++ b/src/Functions/FunctionsStringSimilarity.h
@@ -57,7 +57,7 @@ public:
         return std::make_shared<DataTypeNumber<typename Impl::ResultType>>();
     }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t /*input_rows_count*/) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override
     {
         using ResultType = typename Impl::ResultType;
 
@@ -90,7 +90,7 @@ public:
         auto col_res = ColumnVector<ResultType>::create();
 
         typename ColumnVector<ResultType>::Container & vec_res = col_res->getData();
-        vec_res.resize(column_haystack->size());
+        vec_res.resize(input_rows_count);
 
         const ColumnString * col_haystack_vector = checkAndGetColumn<ColumnString>(&*column_haystack);
         const ColumnString * col_needle_vector = checkAndGetColumn<ColumnString>(&*column_needle);
@@ -110,7 +110,7 @@ public:
                         Impl::max_string_size);
                 }
             }
-            Impl::vectorConstant(col_haystack_vector->getChars(), col_haystack_vector->getOffsets(), needle, vec_res);
+            Impl::vectorConstant(col_haystack_vector->getChars(), col_haystack_vector->getOffsets(), needle, vec_res, input_rows_count);
         }
         else if (col_haystack_vector && col_needle_vector)
         {
@@ -119,7 +119,8 @@ public:
                 col_haystack_vector->getOffsets(),
                 col_needle_vector->getChars(),
                 col_needle_vector->getOffsets(),
-                vec_res);
+                vec_res,
+                input_rows_count);
         }
         else if (col_haystack_const && col_needle_vector)
         {
@@ -136,7 +137,7 @@ public:
                         Impl::max_string_size);
                 }
             }
-            Impl::constantVector(haystack, col_needle_vector->getChars(), col_needle_vector->getOffsets(), vec_res);
+            Impl::constantVector(haystack, col_needle_vector->getChars(), col_needle_vector->getOffsets(), vec_res, input_rows_count);
         }
         else
         {
diff --git a/src/Functions/FunctionsTextClassification.h b/src/Functions/FunctionsTextClassification.h
index 90e8af06ccc..d5cba690f81 100644
--- a/src/Functions/FunctionsTextClassification.h
+++ b/src/Functions/FunctionsTextClassification.h
@@ -55,7 +55,7 @@ public:
         return arguments[0];
     }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t /*input_rows_count*/) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override
     {
         const ColumnPtr & column = arguments[0].column;
         const ColumnString * col = checkAndGetColumn<ColumnString>(column.get());
@@ -65,7 +65,7 @@ public:
                 arguments[0].column->getName(), getName());
 
         auto col_res = ColumnString::create();
-        Impl::vector(col->getChars(), col->getOffsets(), col_res->getChars(), col_res->getOffsets());
+        Impl::vector(col->getChars(), col->getOffsets(), col_res->getChars(), col_res->getOffsets(), input_rows_count);
         return col_res;
     }
 };
@@ -104,7 +104,7 @@ public:
         return std::make_shared<DataTypeFloat32>();
     }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t /*input_rows_count*/) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override
     {
         const ColumnPtr & column = arguments[0].column;
         const ColumnString * col = checkAndGetColumn<ColumnString>(column.get());
@@ -115,9 +115,9 @@ public:
 
         auto col_res = ColumnVector<Float32>::create();
         ColumnVector<Float32>::Container & vec_res = col_res->getData();
-        vec_res.resize(col->size());
+        vec_res.resize(input_rows_count);
 
-        Impl::vector(col->getChars(), col->getOffsets(), vec_res);
+        Impl::vector(col->getChars(), col->getOffsets(), vec_res, input_rows_count);
         return col_res;
     }
 };
diff --git a/src/Functions/FunctionsTimeWindow.cpp b/src/Functions/FunctionsTimeWindow.cpp
index f93a885ee65..77d740803be 100644
--- a/src/Functions/FunctionsTimeWindow.cpp
+++ b/src/Functions/FunctionsTimeWindow.cpp
@@ -130,7 +130,7 @@ struct TimeWindowImpl
 
     static DataTypePtr getReturnType(const ColumnsWithTypeAndName & arguments, const String & function_name);
 
-    static ColumnPtr dispatchForColumns(const ColumnsWithTypeAndName & arguments, const String & function_name);
+    static ColumnPtr dispatchForColumns(const ColumnsWithTypeAndName & arguments, const String & function_name, size_t input_rows_count);
 };
 
 template <TimeWindowFunctionName type>
@@ -196,7 +196,7 @@ struct TimeWindowImpl<TUMBLE>
         return std::make_shared<DataTypeTuple>(DataTypes{data_type, data_type});
     }
 
-    static ColumnPtr dispatchForColumns(const ColumnsWithTypeAndName & arguments, const String & function_name)
+    static ColumnPtr dispatchForColumns(const ColumnsWithTypeAndName & arguments, const String & function_name, size_t input_rows_count)
     {
         const auto & time_column = arguments[0];
         const auto & interval_column = arguments[1];
@@ -214,38 +214,37 @@ struct TimeWindowImpl<TUMBLE>
         {
             /// TODO: add proper support for fractional seconds
             case IntervalKind::Kind::Second:
-                return executeTumble<UInt32, IntervalKind::Kind::Second>(*time_column_vec, std::get<1>(interval), time_zone);
+                return executeTumble<UInt32, IntervalKind::Kind::Second>(*time_column_vec, std::get<1>(interval), time_zone, input_rows_count);
             case IntervalKind::Kind::Minute:
-                return executeTumble<UInt32, IntervalKind::Kind::Minute>(*time_column_vec, std::get<1>(interval), time_zone);
+                return executeTumble<UInt32, IntervalKind::Kind::Minute>(*time_column_vec, std::get<1>(interval), time_zone, input_rows_count);
             case IntervalKind::Kind::Hour:
-                return executeTumble<UInt32, IntervalKind::Kind::Hour>(*time_column_vec, std::get<1>(interval), time_zone);
+                return executeTumble<UInt32, IntervalKind::Kind::Hour>(*time_column_vec, std::get<1>(interval), time_zone, input_rows_count);
             case IntervalKind::Kind::Day:
-                return executeTumble<UInt32, IntervalKind::Kind::Day>(*time_column_vec, std::get<1>(interval), time_zone);
+                return executeTumble<UInt32, IntervalKind::Kind::Day>(*time_column_vec, std::get<1>(interval), time_zone, input_rows_count);
             case IntervalKind::Kind::Week:
-                return executeTumble<UInt16, IntervalKind::Kind::Week>(*time_column_vec, std::get<1>(interval), time_zone);
+                return executeTumble<UInt16, IntervalKind::Kind::Week>(*time_column_vec, std::get<1>(interval), time_zone, input_rows_count);
             case IntervalKind::Kind::Month:
-                return executeTumble<UInt16, IntervalKind::Kind::Month>(*time_column_vec, std::get<1>(interval), time_zone);
+                return executeTumble<UInt16, IntervalKind::Kind::Month>(*time_column_vec, std::get<1>(interval), time_zone, input_rows_count);
             case IntervalKind::Kind::Quarter:
-                return executeTumble<UInt16, IntervalKind::Kind::Quarter>(*time_column_vec, std::get<1>(interval), time_zone);
+                return executeTumble<UInt16, IntervalKind::Kind::Quarter>(*time_column_vec, std::get<1>(interval), time_zone, input_rows_count);
             case IntervalKind::Kind::Year:
-                return executeTumble<UInt16, IntervalKind::Kind::Year>(*time_column_vec, std::get<1>(interval), time_zone);
+                return executeTumble<UInt16, IntervalKind::Kind::Year>(*time_column_vec, std::get<1>(interval), time_zone, input_rows_count);
             default:
                 throw Exception(ErrorCodes::SYNTAX_ERROR, "Fraction seconds are unsupported by windows yet");
         }
     }
 
     template <typename ToType, IntervalKind::Kind unit>
-    static ColumnPtr executeTumble(const ColumnDateTime & time_column, UInt64 num_units, const DateLUTImpl & time_zone)
+    static ColumnPtr executeTumble(const ColumnDateTime & time_column, UInt64 num_units, const DateLUTImpl & time_zone, size_t input_rows_count)
     {
         const auto & time_data = time_column.getData();
-        size_t size = time_column.size();
         auto start = ColumnVector<ToType>::create();
         auto end = ColumnVector<ToType>::create();
         auto & start_data = start->getData();
         auto & end_data = end->getData();
-        start_data.resize(size);
-        end_data.resize(size);
-        for (size_t i = 0; i != size; ++i)
+        start_data.resize(input_rows_count);
+        end_data.resize(input_rows_count);
+        for (size_t i = 0; i != input_rows_count; ++i)
         {
             start_data[i] = ToStartOfTransform<unit>::execute(time_data[i], num_units, time_zone);
             end_data[i] = AddTime<unit>::execute(start_data[i], num_units, time_zone);
@@ -283,7 +282,7 @@ struct TimeWindowImpl<TUMBLE_START>
         }
     }
 
-    [[maybe_unused]] static ColumnPtr dispatchForColumns(const ColumnsWithTypeAndName & arguments, const String & function_name)
+    [[maybe_unused]] static ColumnPtr dispatchForColumns(const ColumnsWithTypeAndName & arguments, const String & function_name, size_t input_rows_count)
     {
         const auto & time_column = arguments[0];
         const auto which_type = WhichDataType(time_column.type);
@@ -296,7 +295,7 @@ struct TimeWindowImpl<TUMBLE_START>
                 result_column = time_column.column;
         }
         else
-            result_column = TimeWindowImpl<TUMBLE>::dispatchForColumns(arguments, function_name);
+            result_column = TimeWindowImpl<TUMBLE>::dispatchForColumns(arguments, function_name, input_rows_count);
         return executeWindowBound(result_column, 0, function_name);
     }
 };
@@ -311,7 +310,7 @@ struct TimeWindowImpl<TUMBLE_END>
         return TimeWindowImpl<TUMBLE_START>::getReturnType(arguments, function_name);
     }
 
-    [[maybe_unused]] static ColumnPtr dispatchForColumns(const ColumnsWithTypeAndName & arguments, const String& function_name)
+    [[maybe_unused]] static ColumnPtr dispatchForColumns(const ColumnsWithTypeAndName & arguments, const String& function_name, size_t input_rows_count)
     {
         const auto & time_column = arguments[0];
         const auto which_type = WhichDataType(time_column.type);
@@ -324,7 +323,7 @@ struct TimeWindowImpl<TUMBLE_END>
                 result_column = time_column.column;
         }
         else
-            result_column = TimeWindowImpl<TUMBLE>::dispatchForColumns(arguments, function_name);
+            result_column = TimeWindowImpl<TUMBLE>::dispatchForColumns(arguments, function_name, input_rows_count);
         return executeWindowBound(result_column, 1, function_name);
     }
 };
@@ -372,7 +371,7 @@ struct TimeWindowImpl<HOP>
         return std::make_shared<DataTypeTuple>(DataTypes{data_type, data_type});
     }
 
-    static ColumnPtr dispatchForColumns(const ColumnsWithTypeAndName & arguments, const String & function_name)
+    static ColumnPtr dispatchForColumns(const ColumnsWithTypeAndName & arguments, const String & function_name, size_t input_rows_count)
     {
         const auto & time_column = arguments[0];
         const auto & hop_interval_column = arguments[1];
@@ -396,28 +395,28 @@ struct TimeWindowImpl<HOP>
             /// TODO: add proper support for fractional seconds
             case IntervalKind::Kind::Second:
                 return executeHop<UInt32, IntervalKind::Kind::Second>(
-                    *time_column_vec, std::get<1>(hop_interval), std::get<1>(window_interval), time_zone);
+                    *time_column_vec, std::get<1>(hop_interval), std::get<1>(window_interval), time_zone, input_rows_count);
             case IntervalKind::Kind::Minute:
                 return executeHop<UInt32, IntervalKind::Kind::Minute>(
-                    *time_column_vec, std::get<1>(hop_interval), std::get<1>(window_interval), time_zone);
+                    *time_column_vec, std::get<1>(hop_interval), std::get<1>(window_interval), time_zone, input_rows_count);
             case IntervalKind::Kind::Hour:
                 return executeHop<UInt32, IntervalKind::Kind::Hour>(
-                    *time_column_vec, std::get<1>(hop_interval), std::get<1>(window_interval), time_zone);
+                    *time_column_vec, std::get<1>(hop_interval), std::get<1>(window_interval), time_zone, input_rows_count);
             case IntervalKind::Kind::Day:
                 return executeHop<UInt32, IntervalKind::Kind::Day>(
-                    *time_column_vec, std::get<1>(hop_interval), std::get<1>(window_interval), time_zone);
+                    *time_column_vec, std::get<1>(hop_interval), std::get<1>(window_interval), time_zone, input_rows_count);
             case IntervalKind::Kind::Week:
                 return executeHop<UInt16, IntervalKind::Kind::Week>(
-                    *time_column_vec, std::get<1>(hop_interval), std::get<1>(window_interval), time_zone);
+                    *time_column_vec, std::get<1>(hop_interval), std::get<1>(window_interval), time_zone, input_rows_count);
             case IntervalKind::Kind::Month:
                 return executeHop<UInt16, IntervalKind::Kind::Month>(
-                    *time_column_vec, std::get<1>(hop_interval), std::get<1>(window_interval), time_zone);
+                    *time_column_vec, std::get<1>(hop_interval), std::get<1>(window_interval), time_zone, input_rows_count);
             case IntervalKind::Kind::Quarter:
                 return executeHop<UInt16, IntervalKind::Kind::Quarter>(
-                    *time_column_vec, std::get<1>(hop_interval), std::get<1>(window_interval), time_zone);
+                    *time_column_vec, std::get<1>(hop_interval), std::get<1>(window_interval), time_zone, input_rows_count);
             case IntervalKind::Kind::Year:
                 return executeHop<UInt16, IntervalKind::Kind::Year>(
-                    *time_column_vec, std::get<1>(hop_interval), std::get<1>(window_interval), time_zone);
+                    *time_column_vec, std::get<1>(hop_interval), std::get<1>(window_interval), time_zone, input_rows_count);
             default:
                 throw Exception(ErrorCodes::SYNTAX_ERROR, "Fraction seconds are unsupported by windows yet");
         }
@@ -425,18 +424,17 @@ struct TimeWindowImpl<HOP>
 
     template <typename ToType, IntervalKind::Kind kind>
     static ColumnPtr
-    executeHop(const ColumnDateTime & time_column, UInt64 hop_num_units, UInt64 window_num_units, const DateLUTImpl & time_zone)
+    executeHop(const ColumnDateTime & time_column, UInt64 hop_num_units, UInt64 window_num_units, const DateLUTImpl & time_zone, size_t input_rows_count)
     {
         const auto & time_data = time_column.getData();
-        size_t size = time_column.size();
         auto start = ColumnVector<ToType>::create();
         auto end = ColumnVector<ToType>::create();
         auto & start_data = start->getData();
         auto & end_data = end->getData();
-        start_data.resize(size);
-        end_data.resize(size);
+        start_data.resize(input_rows_count);
+        end_data.resize(input_rows_count);
 
-        for (size_t i = 0; i < size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             ToType wstart = ToStartOfTransform<kind>::execute(time_data[i], hop_num_units, time_zone);
             ToType wend = AddTime<kind>::execute(wstart, hop_num_units, time_zone);
@@ -509,7 +507,7 @@ struct TimeWindowImpl<WINDOW_ID>
             return std::make_shared<DataTypeUInt32>();
     }
 
-    static ColumnPtr dispatchForHopColumns(const ColumnsWithTypeAndName & arguments, const String & function_name)
+    static ColumnPtr dispatchForHopColumns(const ColumnsWithTypeAndName & arguments, const String & function_name, size_t input_rows_count)
     {
         const auto & time_column = arguments[0];
         const auto & hop_interval_column = arguments[1];
@@ -533,28 +531,28 @@ struct TimeWindowImpl<WINDOW_ID>
             /// TODO: add proper support for fractional seconds
             case IntervalKind::Kind::Second:
                 return executeHopSlice<UInt32, IntervalKind::Kind::Second>(
-                    *time_column_vec, std::get<1>(hop_interval), std::get<1>(window_interval), time_zone);
+                    *time_column_vec, std::get<1>(hop_interval), std::get<1>(window_interval), time_zone, input_rows_count);
             case IntervalKind::Kind::Minute:
                 return executeHopSlice<UInt32, IntervalKind::Kind::Minute>(
-                    *time_column_vec, std::get<1>(hop_interval), std::get<1>(window_interval), time_zone);
+                    *time_column_vec, std::get<1>(hop_interval), std::get<1>(window_interval), time_zone, input_rows_count);
             case IntervalKind::Kind::Hour:
                 return executeHopSlice<UInt32, IntervalKind::Kind::Hour>(
-                    *time_column_vec, std::get<1>(hop_interval), std::get<1>(window_interval), time_zone);
+                    *time_column_vec, std::get<1>(hop_interval), std::get<1>(window_interval), time_zone, input_rows_count);
             case IntervalKind::Kind::Day:
                 return executeHopSlice<UInt32, IntervalKind::Kind::Day>(
-                    *time_column_vec, std::get<1>(hop_interval), std::get<1>(window_interval), time_zone);
+                    *time_column_vec, std::get<1>(hop_interval), std::get<1>(window_interval), time_zone, input_rows_count);
             case IntervalKind::Kind::Week:
                 return executeHopSlice<UInt16, IntervalKind::Kind::Week>(
-                    *time_column_vec, std::get<1>(hop_interval), std::get<1>(window_interval), time_zone);
+                    *time_column_vec, std::get<1>(hop_interval), std::get<1>(window_interval), time_zone, input_rows_count);
             case IntervalKind::Kind::Month:
                 return executeHopSlice<UInt16, IntervalKind::Kind::Month>(
-                    *time_column_vec, std::get<1>(hop_interval), std::get<1>(window_interval), time_zone);
+                    *time_column_vec, std::get<1>(hop_interval), std::get<1>(window_interval), time_zone, input_rows_count);
             case IntervalKind::Kind::Quarter:
                 return executeHopSlice<UInt16, IntervalKind::Kind::Quarter>(
-                    *time_column_vec, std::get<1>(hop_interval), std::get<1>(window_interval), time_zone);
+                    *time_column_vec, std::get<1>(hop_interval), std::get<1>(window_interval), time_zone, input_rows_count);
             case IntervalKind::Kind::Year:
                 return executeHopSlice<UInt16, IntervalKind::Kind::Year>(
-                    *time_column_vec, std::get<1>(hop_interval), std::get<1>(window_interval), time_zone);
+                    *time_column_vec, std::get<1>(hop_interval), std::get<1>(window_interval), time_zone, input_rows_count);
             default:
                 throw Exception(ErrorCodes::SYNTAX_ERROR, "Fraction seconds are unsupported by windows yet");
         }
@@ -563,17 +561,16 @@ struct TimeWindowImpl<WINDOW_ID>
 
     template <typename ToType, IntervalKind::Kind kind>
     static ColumnPtr
-    executeHopSlice(const ColumnDateTime & time_column, UInt64 hop_num_units, UInt64 window_num_units, const DateLUTImpl & time_zone)
+    executeHopSlice(const ColumnDateTime & time_column, UInt64 hop_num_units, UInt64 window_num_units, const DateLUTImpl & time_zone, size_t input_rows_count)
     {
         Int64 gcd_num_units = std::gcd(hop_num_units, window_num_units);
 
         const auto & time_data = time_column.getData();
-        size_t size = time_column.size();
 
         auto end = ColumnVector<ToType>::create();
         auto & end_data = end->getData();
-        end_data.resize(size);
-        for (size_t i = 0; i < size; ++i)
+        end_data.resize(input_rows_count);
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             ToType wstart = ToStartOfTransform<kind>::execute(time_data[i], hop_num_units, time_zone);
             ToType wend = AddTime<kind>::execute(wstart, hop_num_units, time_zone);
@@ -593,23 +590,23 @@ struct TimeWindowImpl<WINDOW_ID>
         return end;
     }
 
-    static ColumnPtr dispatchForTumbleColumns(const ColumnsWithTypeAndName & arguments, const String & function_name)
+    static ColumnPtr dispatchForTumbleColumns(const ColumnsWithTypeAndName & arguments, const String & function_name, size_t input_rows_count)
     {
-        ColumnPtr column = TimeWindowImpl<TUMBLE>::dispatchForColumns(arguments, function_name);
+        ColumnPtr column = TimeWindowImpl<TUMBLE>::dispatchForColumns(arguments, function_name, input_rows_count);
         return executeWindowBound(column, 1, function_name);
     }
 
-    static ColumnPtr dispatchForColumns(const ColumnsWithTypeAndName & arguments, const String & function_name)
+    static ColumnPtr dispatchForColumns(const ColumnsWithTypeAndName & arguments, const String & function_name, size_t input_rows_count)
     {
         if (arguments.size() == 2)
-            return dispatchForTumbleColumns(arguments, function_name);
+            return dispatchForTumbleColumns(arguments, function_name, input_rows_count);
         else
         {
             const auto & third_column = arguments[2];
             if (arguments.size() == 3 && WhichDataType(third_column.type).isString())
-                return dispatchForTumbleColumns(arguments, function_name);
+                return dispatchForTumbleColumns(arguments, function_name, input_rows_count);
             else
-                return dispatchForHopColumns(arguments, function_name);
+                return dispatchForHopColumns(arguments, function_name, input_rows_count);
         }
     }
 };
@@ -639,7 +636,7 @@ struct TimeWindowImpl<HOP_START>
         }
     }
 
-    static ColumnPtr dispatchForColumns(const ColumnsWithTypeAndName & arguments, const String & function_name)
+    static ColumnPtr dispatchForColumns(const ColumnsWithTypeAndName & arguments, const String & function_name, size_t input_rows_count)
     {
         const auto & time_column = arguments[0];
         const auto which_type = WhichDataType(time_column.type);
@@ -652,7 +649,7 @@ struct TimeWindowImpl<HOP_START>
                 result_column = time_column.column;
         }
         else
-            result_column = TimeWindowImpl<HOP>::dispatchForColumns(arguments, function_name);
+            result_column = TimeWindowImpl<HOP>::dispatchForColumns(arguments, function_name, input_rows_count);
         return executeWindowBound(result_column, 0, function_name);
     }
 };
@@ -667,7 +664,7 @@ struct TimeWindowImpl<HOP_END>
         return TimeWindowImpl<HOP_START>::getReturnType(arguments, function_name);
     }
 
-    static ColumnPtr dispatchForColumns(const ColumnsWithTypeAndName & arguments, const String & function_name)
+    static ColumnPtr dispatchForColumns(const ColumnsWithTypeAndName & arguments, const String & function_name, size_t input_rows_count)
     {
         const auto & time_column = arguments[0];
         const auto which_type = WhichDataType(time_column.type);
@@ -680,7 +677,7 @@ struct TimeWindowImpl<HOP_END>
                 result_column = time_column.column;
         }
         else
-            result_column = TimeWindowImpl<HOP>::dispatchForColumns(arguments, function_name);
+            result_column = TimeWindowImpl<HOP>::dispatchForColumns(arguments, function_name, input_rows_count);
 
         return executeWindowBound(result_column, 1, function_name);
     }
@@ -693,9 +690,9 @@ DataTypePtr FunctionTimeWindow<type>::getReturnTypeImpl(const ColumnsWithTypeAnd
 }
 
 template <TimeWindowFunctionName type>
-ColumnPtr FunctionTimeWindow<type>::executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t /*input_rows_count*/) const
+ColumnPtr FunctionTimeWindow<type>::executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const
 {
-    return TimeWindowImpl<type>::dispatchForColumns(arguments, name);
+    return TimeWindowImpl<type>::dispatchForColumns(arguments, name, input_rows_count);
 }
 
 }
diff --git a/src/Functions/FunctionsTonalityClassification.cpp b/src/Functions/FunctionsTonalityClassification.cpp
index a9321819a26..7627c68c057 100644
--- a/src/Functions/FunctionsTonalityClassification.cpp
+++ b/src/Functions/FunctionsTonalityClassification.cpp
@@ -18,7 +18,7 @@ namespace DB
   */
 struct FunctionDetectTonalityImpl
 {
-    static ALWAYS_INLINE inline Float32 detectTonality(
+    static Float32 detectTonality(
         const UInt8 * str,
         const size_t str_len,
         const FrequencyHolder::Map & emotional_dict)
@@ -63,13 +63,13 @@ struct FunctionDetectTonalityImpl
     static void vector(
         const ColumnString::Chars & data,
         const ColumnString::Offsets & offsets,
-        PaddedPODArray<Float32> & res)
+        PaddedPODArray<Float32> & res,
+        size_t input_rows_count)
     {
         const auto & emotional_dict = FrequencyHolder::getInstance().getEmotionalDict();
 
-        size_t size = offsets.size();
         size_t prev_offset = 0;
-        for (size_t i = 0; i < size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             res[i] = detectTonality(data.data() + prev_offset, offsets[i] - 1 - prev_offset, emotional_dict);
             prev_offset = offsets[i];
diff --git a/src/Functions/FunctionsVisitParam.h b/src/Functions/FunctionsVisitParam.h
index 5e13fbbad5c..dbe291fcb93 100644
--- a/src/Functions/FunctionsVisitParam.h
+++ b/src/Functions/FunctionsVisitParam.h
@@ -93,7 +93,8 @@ struct ExtractParamImpl
         std::string needle,
         const ColumnPtr & start_pos,
         PaddedPODArray<ResultType> & res,
-        [[maybe_unused]] ColumnUInt8 * res_null)
+        [[maybe_unused]] ColumnUInt8 * res_null,
+        size_t /*input_rows_count*/)
     {
         /// `res_null` serves as an output parameter for implementing an XYZOrNull variant.
         assert(!res_null);
@@ -168,11 +169,12 @@ struct ExtractParamToStringImpl
 {
     static void vector(const ColumnString::Chars & haystack_data, const ColumnString::Offsets & haystack_offsets,
                        std::string needle,
-                       ColumnString::Chars & res_data, ColumnString::Offsets & res_offsets)
+                       ColumnString::Chars & res_data, ColumnString::Offsets & res_offsets,
+                       size_t input_rows_count)
     {
         /// Constant 5 is taken from a function that performs a similar task FunctionsStringSearch.h::ExtractImpl
-        res_data.reserve(haystack_data.size() / 5);
-        res_offsets.resize(haystack_offsets.size());
+        res_data.reserve(input_rows_count / 5);
+        res_offsets.resize(input_rows_count);
 
         /// We are looking for a parameter simply as a substring of the form "name"
         needle = "\"" + needle + "\":";
diff --git a/src/Functions/HasTokenImpl.h b/src/Functions/HasTokenImpl.h
index a4ff49859cc..4943bf708c5 100644
--- a/src/Functions/HasTokenImpl.h
+++ b/src/Functions/HasTokenImpl.h
@@ -35,12 +35,13 @@ struct HasTokenImpl
         const std::string & pattern,
         const ColumnPtr & start_pos,
         PaddedPODArray<UInt8> & res,
-        ColumnUInt8 * res_null)
+        ColumnUInt8 * res_null,
+        size_t input_rows_count)
     {
         if (start_pos != nullptr)
             throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Function '{}' does not support start_pos argument", name);
 
-        if (haystack_offsets.empty())
+        if (input_rows_count == 0)
             return;
 
         const UInt8 * const begin = haystack_data.data();
diff --git a/src/Functions/Kusto/KqlArraySort.cpp b/src/Functions/Kusto/KqlArraySort.cpp
index 11157aa53e6..fb3e6259ee4 100644
--- a/src/Functions/Kusto/KqlArraySort.cpp
+++ b/src/Functions/Kusto/KqlArraySort.cpp
@@ -73,13 +73,11 @@ public:
         size_t array_count = arguments.size();
         const auto & last_arg = arguments[array_count - 1];
 
-        size_t input_rows_count_local = input_rows_count;
-
         bool null_last = true;
         if (!isArray(last_arg.type))
         {
             --array_count;
-            null_last = check_condition(last_arg, context, input_rows_count_local);
+            null_last = check_condition(last_arg, context, input_rows_count);
         }
 
         ColumnsWithTypeAndName new_args;
@@ -119,11 +117,11 @@ public:
         }
 
         auto zipped
-            = FunctionFactory::instance().get("arrayZip", context)->build(new_args)->execute(new_args, result_type, input_rows_count_local);
+            = FunctionFactory::instance().get("arrayZip", context)->build(new_args)->execute(new_args, result_type, input_rows_count);
 
         ColumnsWithTypeAndName sort_arg({{zipped, std::make_shared<DataTypeArray>(result_type), "zipped"}});
         auto sorted_tuple
-            = FunctionFactory::instance().get(sort_function, context)->build(sort_arg)->execute(sort_arg, result_type, input_rows_count_local);
+            = FunctionFactory::instance().get(sort_function, context)->build(sort_arg)->execute(sort_arg, result_type, input_rows_count);
 
         auto null_type = std::make_shared<DataTypeNullable>(std::make_shared<DataTypeInt8>());
 
@@ -139,10 +137,10 @@ public:
                     = std::make_shared<DataTypeArray>(makeNullable(nested_types[i]));
 
                 ColumnsWithTypeAndName null_array_arg({
-                    {null_type->createColumnConstWithDefaultValue(input_rows_count_local), null_type, "NULL"},
+                    {null_type->createColumnConstWithDefaultValue(input_rows_count), null_type, "NULL"},
                 });
 
-                tuple_columns[i] = fun_array->build(null_array_arg)->execute(null_array_arg, arg_type, input_rows_count_local);
+                tuple_columns[i] = fun_array->build(null_array_arg)->execute(null_array_arg, arg_type, input_rows_count);
                 tuple_columns[i] = tuple_columns[i]->convertToFullColumnIfConst();
             }
             else
@@ -153,7 +151,7 @@ public:
                 auto tuple_coulmn = FunctionFactory::instance()
                                         .get("tupleElement", context)
                                         ->build(untuple_args)
-                                        ->execute(untuple_args, result_type, input_rows_count_local);
+                                        ->execute(untuple_args, result_type, input_rows_count);
 
                 auto out_tmp = ColumnArray::create(nested_types[i]->createColumn());
 
@@ -183,7 +181,7 @@ public:
             auto inside_null_type = nested_types[0];
             ColumnsWithTypeAndName indexof_args({
                 arg_of_index,
-                {inside_null_type->createColumnConstWithDefaultValue(input_rows_count_local), inside_null_type, "NULL"},
+                {inside_null_type->createColumnConstWithDefaultValue(input_rows_count), inside_null_type, "NULL"},
             });
 
             auto null_index_datetype = std::make_shared<DataTypeUInt64>();
@@ -192,7 +190,7 @@ public:
             slice_index.column = FunctionFactory::instance()
                                      .get("indexOf", context)
                                      ->build(indexof_args)
-                                     ->execute(indexof_args, result_type, input_rows_count_local);
+                                     ->execute(indexof_args, result_type, input_rows_count);
 
             auto null_index_in_array = slice_index.column->get64(0);
             if (null_index_in_array > 0)
@@ -220,15 +218,15 @@ public:
                         ColumnsWithTypeAndName slice_args_right(
                             {{ColumnWithTypeAndName(tuple_columns[i], arg_type, "array")}, slice_index});
                         ColumnWithTypeAndName arr_left{
-                            fun_slice->build(slice_args_left)->execute(slice_args_left, arg_type, input_rows_count_local), arg_type, ""};
+                            fun_slice->build(slice_args_left)->execute(slice_args_left, arg_type, input_rows_count), arg_type, ""};
                         ColumnWithTypeAndName arr_right{
-                            fun_slice->build(slice_args_right)->execute(slice_args_right, arg_type, input_rows_count_local), arg_type, ""};
+                            fun_slice->build(slice_args_right)->execute(slice_args_right, arg_type, input_rows_count), arg_type, ""};
 
                         ColumnsWithTypeAndName arr_cancat({arr_right, arr_left});
                         auto out_tmp = FunctionFactory::instance()
                                            .get("arrayConcat", context)
                                            ->build(arr_cancat)
-                                           ->execute(arr_cancat, arg_type, input_rows_count_local);
+                                           ->execute(arr_cancat, arg_type, input_rows_count);
                         adjusted_columns[i] = std::move(out_tmp);
                     }
                 }
diff --git a/src/Functions/MatchImpl.h b/src/Functions/MatchImpl.h
index 55b2fee5400..dd1dec9bdff 100644
--- a/src/Functions/MatchImpl.h
+++ b/src/Functions/MatchImpl.h
@@ -127,14 +127,13 @@ struct MatchImpl
         const String & needle,
         [[maybe_unused]] const ColumnPtr & start_pos_,
         PaddedPODArray<UInt8> & res,
-        [[maybe_unused]] ColumnUInt8 * res_null)
+        [[maybe_unused]] ColumnUInt8 * res_null,
+        size_t input_rows_count)
     {
         /// `res_null` serves as an output parameter for implementing an XYZOrNull variant.
         assert(!res_null);
 
-        const size_t haystack_size = haystack_offsets.size();
-
-        assert(haystack_size == res.size());
+        assert(input_rows_count == res.size());
         assert(start_pos_ == nullptr);
 
         if (haystack_offsets.empty())
@@ -202,11 +201,11 @@ struct MatchImpl
         if (required_substring.empty())
         {
             if (!regexp.getRE2()) /// An empty regexp. Always matches.
-                memset(res.data(), !negate, haystack_size * sizeof(res[0]));
+                memset(res.data(), !negate, input_rows_count * sizeof(res[0]));
             else
             {
                 size_t prev_offset = 0;
-                for (size_t i = 0; i < haystack_size; ++i)
+                for (size_t i = 0; i < input_rows_count; ++i)
                 {
                     const bool match = regexp.getRE2()->Match(
                             {reinterpret_cast<const char *>(&haystack_data[prev_offset]), haystack_offsets[i] - prev_offset - 1},
@@ -291,14 +290,13 @@ struct MatchImpl
         size_t N,
         const String & needle,
         PaddedPODArray<UInt8> & res,
-        [[maybe_unused]] ColumnUInt8 * res_null)
+        [[maybe_unused]] ColumnUInt8 * res_null,
+        size_t input_rows_count)
     {
         /// `res_null` serves as an output parameter for implementing an XYZOrNull variant.
         assert(!res_null);
 
-        const size_t haystack_size = haystack.size() / N;
-
-        assert(haystack_size == res.size());
+        assert(input_rows_count == res.size());
 
         if (haystack.empty())
             return;
@@ -370,11 +368,11 @@ struct MatchImpl
         if (required_substring.empty())
         {
             if (!regexp.getRE2()) /// An empty regexp. Always matches.
-                memset(res.data(), !negate, haystack_size * sizeof(res[0]));
+                memset(res.data(), !negate, input_rows_count * sizeof(res[0]));
             else
             {
                 size_t offset = 0;
-                for (size_t i = 0; i < haystack_size; ++i)
+                for (size_t i = 0; i < input_rows_count; ++i)
                 {
                     const bool match = regexp.getRE2()->Match(
                             {reinterpret_cast<const char *>(&haystack[offset]), N},
@@ -464,15 +462,14 @@ struct MatchImpl
         const ColumnString::Offsets & needle_offset,
         [[maybe_unused]] const ColumnPtr & start_pos_,
         PaddedPODArray<UInt8> & res,
-        [[maybe_unused]] ColumnUInt8 * res_null)
+        [[maybe_unused]] ColumnUInt8 * res_null,
+        size_t input_rows_count)
     {
         /// `res_null` serves as an output parameter for implementing an XYZOrNull variant.
         assert(!res_null);
 
-        const size_t haystack_size = haystack_offsets.size();
-
-        assert(haystack_size == needle_offset.size());
-        assert(haystack_size == res.size());
+        assert(input_rows_count == needle_offset.size());
+        assert(input_rows_count == res.size());
         assert(start_pos_ == nullptr);
 
         if (haystack_offsets.empty())
@@ -488,7 +485,7 @@ struct MatchImpl
         Regexps::LocalCacheTable cache;
         Regexps::RegexpPtr regexp;
 
-        for (size_t i = 0; i < haystack_size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             const auto * const cur_haystack_data = &haystack_data[prev_haystack_offset];
             const size_t cur_haystack_length = haystack_offsets[i] - prev_haystack_offset - 1;
@@ -573,15 +570,14 @@ struct MatchImpl
         const ColumnString::Offsets & needle_offset,
         [[maybe_unused]] const ColumnPtr & start_pos_,
         PaddedPODArray<UInt8> & res,
-        [[maybe_unused]] ColumnUInt8 * res_null)
+        [[maybe_unused]] ColumnUInt8 * res_null,
+        size_t input_rows_count)
     {
         /// `res_null` serves as an output parameter for implementing an XYZOrNull variant.
         assert(!res_null);
 
-        const size_t haystack_size = haystack.size()/N;
-
-        assert(haystack_size == needle_offset.size());
-        assert(haystack_size == res.size());
+        assert(input_rows_count == needle_offset.size());
+        assert(input_rows_count == res.size());
         assert(start_pos_ == nullptr);
 
         if (haystack.empty())
@@ -597,7 +593,7 @@ struct MatchImpl
         Regexps::LocalCacheTable cache;
         Regexps::RegexpPtr regexp;
 
-        for (size_t i = 0; i < haystack_size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             const auto * const cur_haystack_data = &haystack[prev_haystack_offset];
             const size_t cur_haystack_length = N;
diff --git a/src/Functions/MultiMatchAllIndicesImpl.h b/src/Functions/MultiMatchAllIndicesImpl.h
index 3e9c8fba215..17232bbc366 100644
--- a/src/Functions/MultiMatchAllIndicesImpl.h
+++ b/src/Functions/MultiMatchAllIndicesImpl.h
@@ -52,9 +52,10 @@ struct MultiMatchAllIndicesImpl
         bool allow_hyperscan,
         size_t max_hyperscan_regexp_length,
         size_t max_hyperscan_regexp_total_length,
-        bool reject_expensive_hyperscan_regexps)
+        bool reject_expensive_hyperscan_regexps,
+        size_t input_rows_count)
     {
-        vectorConstant(haystack_data, haystack_offsets, needles_arr, res, offsets, std::nullopt, allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps);
+        vectorConstant(haystack_data, haystack_offsets, needles_arr, res, offsets, std::nullopt, allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps, input_rows_count);
     }
 
     static void vectorConstant(
@@ -67,7 +68,8 @@ struct MultiMatchAllIndicesImpl
         bool allow_hyperscan,
         size_t max_hyperscan_regexp_length,
         size_t max_hyperscan_regexp_total_length,
-        bool reject_expensive_hyperscan_regexps)
+        bool reject_expensive_hyperscan_regexps,
+        size_t input_rows_count)
     {
         if (!allow_hyperscan)
             throw Exception(ErrorCodes::FUNCTION_NOT_ALLOWED, "Hyperscan functions are disabled, because setting 'allow_hyperscan' is set to 0");
@@ -87,7 +89,7 @@ struct MultiMatchAllIndicesImpl
                     throw Exception(ErrorCodes::HYPERSCAN_CANNOT_SCAN_TEXT, "Regular expression evaluation in vectorscan will be too slow. To ignore this error, disable setting 'reject_expensive_hyperscan_regexps'.");
         }
 
-        offsets.resize(haystack_offsets.size());
+        offsets.resize(input_rows_count);
 
         if (needles_arr.empty())
         {
@@ -114,9 +116,8 @@ struct MultiMatchAllIndicesImpl
             static_cast<PaddedPODArray<ResultType>*>(context)->push_back(id);
             return 0;
         };
-        const size_t haystack_offsets_size = haystack_offsets.size();
         UInt64 offset = 0;
-        for (size_t i = 0; i < haystack_offsets_size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             UInt64 length = haystack_offsets[i] - offset - 1;
             /// vectorscan restriction.
@@ -160,9 +161,10 @@ struct MultiMatchAllIndicesImpl
         bool allow_hyperscan,
         size_t max_hyperscan_regexp_length,
         size_t max_hyperscan_regexp_total_length,
-        bool reject_expensive_hyperscan_regexps)
+        bool reject_expensive_hyperscan_regexps,
+        size_t input_rows_count)
     {
-        vectorVector(haystack_data, haystack_offsets, needles_data, needles_offsets, res, offsets, std::nullopt, allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps);
+        vectorVector(haystack_data, haystack_offsets, needles_data, needles_offsets, res, offsets, std::nullopt, allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps, input_rows_count);
     }
 
     static void vectorVector(
@@ -176,12 +178,13 @@ struct MultiMatchAllIndicesImpl
         bool allow_hyperscan,
         size_t max_hyperscan_regexp_length,
         size_t max_hyperscan_regexp_total_length,
-        bool reject_expensive_hyperscan_regexps)
+        bool reject_expensive_hyperscan_regexps,
+        size_t input_rows_count)
     {
         if (!allow_hyperscan)
             throw Exception(ErrorCodes::FUNCTION_NOT_ALLOWED, "Hyperscan functions are disabled, because setting 'allow_hyperscan' is set to 0");
 #if USE_VECTORSCAN
-        offsets.resize(haystack_offsets.size());
+        offsets.resize(input_rows_count);
         size_t prev_haystack_offset = 0;
         size_t prev_needles_offset = 0;
 
@@ -189,7 +192,7 @@ struct MultiMatchAllIndicesImpl
 
         std::vector<std::string_view> needles;
 
-        for (size_t i = 0; i < haystack_offsets.size(); ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             needles.reserve(needles_offsets[i] - prev_needles_offset);
 
diff --git a/src/Functions/MultiMatchAnyImpl.h b/src/Functions/MultiMatchAnyImpl.h
index 20b2150048b..b5872579ebb 100644
--- a/src/Functions/MultiMatchAnyImpl.h
+++ b/src/Functions/MultiMatchAnyImpl.h
@@ -66,9 +66,10 @@ struct MultiMatchAnyImpl
         bool allow_hyperscan,
         size_t max_hyperscan_regexp_length,
         size_t max_hyperscan_regexp_total_length,
-        bool reject_expensive_hyperscan_regexps)
+        bool reject_expensive_hyperscan_regexps,
+        size_t input_rows_count)
     {
-        vectorConstant(haystack_data, haystack_offsets, needles_arr, res, offsets, std::nullopt, allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps);
+        vectorConstant(haystack_data, haystack_offsets, needles_arr, res, offsets, std::nullopt, allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps, input_rows_count);
     }
 
     static void vectorConstant(
@@ -81,7 +82,8 @@ struct MultiMatchAnyImpl
         bool allow_hyperscan,
         size_t max_hyperscan_regexp_length,
         size_t max_hyperscan_regexp_total_length,
-        bool reject_expensive_hyperscan_regexps)
+        bool reject_expensive_hyperscan_regexps,
+        size_t input_rows_count)
     {
         if (!allow_hyperscan)
             throw Exception(ErrorCodes::FUNCTION_NOT_ALLOWED, "Hyperscan functions are disabled, because setting 'allow_hyperscan' is set to 0");
@@ -101,7 +103,7 @@ struct MultiMatchAnyImpl
                     throw Exception(ErrorCodes::HYPERSCAN_CANNOT_SCAN_TEXT, "Regular expression evaluation in vectorscan will be too slow. To ignore this error, disable setting 'reject_expensive_hyperscan_regexps'.");
         }
 
-        res.resize(haystack_offsets.size());
+        res.resize(input_rows_count);
 
         if (needles_arr.empty())
         {
@@ -133,9 +135,8 @@ struct MultiMatchAnyImpl
             /// Once we hit the callback, there is no need to search for others.
             return 1;
         };
-        const size_t haystack_offsets_size = haystack_offsets.size();
         UInt64 offset = 0;
-        for (size_t i = 0; i < haystack_offsets_size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             UInt64 length = haystack_offsets[i] - offset - 1;
             /// vectorscan restriction.
@@ -162,7 +163,7 @@ struct MultiMatchAnyImpl
         PaddedPODArray<UInt8> accum(res.size());
         memset(res.data(), 0, res.size() * sizeof(res.front()));
         memset(accum.data(), 0, accum.size());
-        for (size_t j = 0; j < needles.size(); ++j)
+        for (size_t j = 0; j < input_rows_count; ++j)
         {
             MatchImpl<Name, MatchTraits::Syntax::Re2, MatchTraits::Case::Sensitive, MatchTraits::Result::DontNegate>::vectorConstant(haystack_data, haystack_offsets, String(needles[j].data(), needles[j].size()), nullptr, accum, nullptr);
             for (size_t i = 0; i < res.size(); ++i)
@@ -186,9 +187,10 @@ struct MultiMatchAnyImpl
         bool allow_hyperscan,
         size_t max_hyperscan_regexp_length,
         size_t max_hyperscan_regexp_total_length,
-        bool reject_expensive_hyperscan_regexps)
+        bool reject_expensive_hyperscan_regexps,
+        size_t input_rows_count)
     {
-        vectorVector(haystack_data, haystack_offsets, needles_data, needles_offsets, res, offsets, std::nullopt, allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps);
+        vectorVector(haystack_data, haystack_offsets, needles_data, needles_offsets, res, offsets, std::nullopt, allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps, input_rows_count);
     }
 
     static void vectorVector(
@@ -202,12 +204,13 @@ struct MultiMatchAnyImpl
         bool allow_hyperscan,
         size_t max_hyperscan_regexp_length,
         size_t max_hyperscan_regexp_total_length,
-        bool reject_expensive_hyperscan_regexps)
+        bool reject_expensive_hyperscan_regexps,
+        size_t input_rows_count)
     {
         if (!allow_hyperscan)
             throw Exception(ErrorCodes::FUNCTION_NOT_ALLOWED, "Hyperscan functions are disabled, because setting 'allow_hyperscan' is set to 0");
 
-        res.resize(haystack_offsets.size());
+        res.resize(input_rows_count);
 #if USE_VECTORSCAN
         size_t prev_haystack_offset = 0;
         size_t prev_needles_offset = 0;
@@ -216,7 +219,7 @@ struct MultiMatchAnyImpl
 
         std::vector<std::string_view> needles;
 
-        for (size_t i = 0; i < haystack_offsets.size(); ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             needles.reserve(needles_offsets[i] - prev_needles_offset);
 
@@ -306,7 +309,7 @@ struct MultiMatchAnyImpl
 
         std::vector<std::string_view> needles;
 
-        for (size_t i = 0; i < haystack_offsets.size(); ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             const auto * const cur_haystack_data = &haystack_data[prev_haystack_offset];
             const size_t cur_haystack_length = haystack_offsets[i] - prev_haystack_offset - 1;
diff --git a/src/Functions/MultiSearchFirstIndexImpl.h b/src/Functions/MultiSearchFirstIndexImpl.h
index 36a5fd514d9..b80d9d3a124 100644
--- a/src/Functions/MultiSearchFirstIndexImpl.h
+++ b/src/Functions/MultiSearchFirstIndexImpl.h
@@ -33,7 +33,8 @@ struct MultiSearchFirstIndexImpl
         bool /*allow_hyperscan*/,
         size_t /*max_hyperscan_regexp_length*/,
         size_t /*max_hyperscan_regexp_total_length*/,
-        bool /*reject_expensive_hyperscan_regexps*/)
+        bool /*reject_expensive_hyperscan_regexps*/,
+        size_t input_rows_count)
     {
         // For performance of Volnitsky search, it is crucial to save only one byte for pattern number.
         if (needles_arr.size() > std::numeric_limits<UInt8>::max())
@@ -48,14 +49,13 @@ struct MultiSearchFirstIndexImpl
 
         auto searcher = Impl::createMultiSearcherInBigHaystack(needles);
 
-        const size_t haystack_size = haystack_offsets.size();
-        res.resize(haystack_size);
+        res.resize(input_rows_count);
 
         size_t iteration = 0;
         while (searcher.hasMoreToSearch())
         {
             size_t prev_haystack_offset = 0;
-            for (size_t j = 0; j < haystack_size; ++j)
+            for (size_t j = 0; j < input_rows_count; ++j)
             {
                 const auto * haystack = &haystack_data[prev_haystack_offset];
                 const auto * haystack_end = haystack + haystack_offsets[j] - prev_haystack_offset - 1;
@@ -80,10 +80,10 @@ struct MultiSearchFirstIndexImpl
         bool /*allow_hyperscan*/,
         size_t /*max_hyperscan_regexp_length*/,
         size_t /*max_hyperscan_regexp_total_length*/,
-        bool /*reject_expensive_hyperscan_regexps*/)
+        bool /*reject_expensive_hyperscan_regexps*/,
+        size_t input_rows_count)
     {
-        const size_t haystack_size = haystack_offsets.size();
-        res.resize(haystack_size);
+        res.resize(input_rows_count);
 
         size_t prev_haystack_offset = 0;
         size_t prev_needles_offset = 0;
@@ -92,14 +92,12 @@ struct MultiSearchFirstIndexImpl
 
         std::vector<std::string_view> needles;
 
-        for (size_t i = 0; i < haystack_size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             needles.reserve(needles_offsets[i] - prev_needles_offset);
 
             for (size_t j = prev_needles_offset; j < needles_offsets[i]; ++j)
-            {
                 needles.emplace_back(needles_data_string.getDataAt(j).toView());
-            }
 
             auto searcher = Impl::createMultiSearcherInBigHaystack(needles); // sub-optimal
 
@@ -110,15 +108,11 @@ struct MultiSearchFirstIndexImpl
             while (searcher.hasMoreToSearch())
             {
                 if (iteration == 0 || res[i] == 0)
-                {
                     res[i] = searcher.searchOneFirstIndex(haystack, haystack_end);
-                }
                 ++iteration;
             }
             if (iteration == 0)
-            {
                 res[i] = 0;
-            }
 
             prev_haystack_offset = haystack_offsets[i];
             prev_needles_offset = needles_offsets[i];
diff --git a/src/Functions/MultiSearchFirstPositionImpl.h b/src/Functions/MultiSearchFirstPositionImpl.h
index ccdd82a0ee5..cd4e585e99b 100644
--- a/src/Functions/MultiSearchFirstPositionImpl.h
+++ b/src/Functions/MultiSearchFirstPositionImpl.h
@@ -33,7 +33,8 @@ struct MultiSearchFirstPositionImpl
         bool /*allow_hyperscan*/,
         size_t /*max_hyperscan_regexp_length*/,
         size_t /*max_hyperscan_regexp_total_length*/,
-        bool /*reject_expensive_hyperscan_regexps*/)
+        bool /*reject_expensive_hyperscan_regexps*/,
+        size_t input_rows_count)
     {
         // For performance of Volnitsky search, it is crucial to save only one byte for pattern number.
         if (needles_arr.size() > std::numeric_limits<UInt8>::max())
@@ -52,14 +53,13 @@ struct MultiSearchFirstPositionImpl
         };
         auto searcher = Impl::createMultiSearcherInBigHaystack(needles);
 
-        const size_t haystack_size = haystack_offsets.size();
-        res.resize(haystack_size);
+        res.resize(input_rows_count);
 
         size_t iteration = 0;
         while (searcher.hasMoreToSearch())
         {
             size_t prev_haystack_offset = 0;
-            for (size_t j = 0; j < haystack_size; ++j)
+            for (size_t j = 0; j < input_rows_count; ++j)
             {
                 const auto * haystack = &haystack_data[prev_haystack_offset];
                 const auto * haystack_end = haystack + haystack_offsets[j] - prev_haystack_offset - 1;
@@ -89,10 +89,10 @@ struct MultiSearchFirstPositionImpl
         bool /*allow_hyperscan*/,
         size_t /*max_hyperscan_regexp_length*/,
         size_t /*max_hyperscan_regexp_total_length*/,
-        bool /*reject_expensive_hyperscan_regexps*/)
+        bool /*reject_expensive_hyperscan_regexps*/,
+        size_t input_rows_count)
     {
-        const size_t haystack_size = haystack_offsets.size();
-        res.resize(haystack_size);
+        res.resize(input_rows_count);
 
         size_t prev_haystack_offset = 0;
         size_t prev_needles_offset = 0;
@@ -106,14 +106,12 @@ struct MultiSearchFirstPositionImpl
             return 1 + Impl::countChars(reinterpret_cast<const char *>(start), reinterpret_cast<const char *>(end));
         };
 
-        for (size_t i = 0; i < haystack_size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             needles.reserve(needles_offsets[i] - prev_needles_offset);
 
             for (size_t j = prev_needles_offset; j < needles_offsets[i]; ++j)
-            {
                 needles.emplace_back(needles_data_string.getDataAt(j).toView());
-            }
 
             auto searcher = Impl::createMultiSearcherInBigHaystack(needles); // sub-optimal
 
@@ -138,9 +136,7 @@ struct MultiSearchFirstPositionImpl
                 ++iteration;
             }
             if (iteration == 0)
-            {
                 res[i] = 0;
-            }
 
             prev_haystack_offset = haystack_offsets[i];
             prev_needles_offset = needles_offsets[i];
diff --git a/src/Functions/MultiSearchImpl.h b/src/Functions/MultiSearchImpl.h
index 467cc96a95f..909425f5a93 100644
--- a/src/Functions/MultiSearchImpl.h
+++ b/src/Functions/MultiSearchImpl.h
@@ -33,7 +33,8 @@ struct MultiSearchImpl
         bool /*allow_hyperscan*/,
         size_t /*max_hyperscan_regexp_length*/,
         size_t /*max_hyperscan_regexp_total_length*/,
-        bool /*reject_expensive_hyperscan_regexps*/)
+        bool /*reject_expensive_hyperscan_regexps*/,
+        size_t input_rows_count)
     {
         // For performance of Volnitsky search, it is crucial to save only one byte for pattern number.
         if (needles_arr.size() > std::numeric_limits<UInt8>::max())
@@ -48,14 +49,13 @@ struct MultiSearchImpl
 
         auto searcher = Impl::createMultiSearcherInBigHaystack(needles);
 
-        const size_t haystack_size = haystack_offsets.size();
-        res.resize(haystack_size);
+        res.resize(input_rows_count);
 
         size_t iteration = 0;
         while (searcher.hasMoreToSearch())
         {
             size_t prev_haystack_offset = 0;
-            for (size_t j = 0; j < haystack_size; ++j)
+            for (size_t j = 0; j < input_rows_count; ++j)
             {
                 const auto * haystack = &haystack_data[prev_haystack_offset];
                 const auto * haystack_end = haystack + haystack_offsets[j] - prev_haystack_offset - 1;
@@ -79,10 +79,10 @@ struct MultiSearchImpl
         bool /*allow_hyperscan*/,
         size_t /*max_hyperscan_regexp_length*/,
         size_t /*max_hyperscan_regexp_total_length*/,
-        bool /*reject_expensive_hyperscan_regexps*/)
+        bool /*reject_expensive_hyperscan_regexps*/,
+        size_t input_rows_count)
     {
-        const size_t haystack_size = haystack_offsets.size();
-        res.resize(haystack_size);
+        res.resize(input_rows_count);
 
         size_t prev_haystack_offset = 0;
         size_t prev_needles_offset = 0;
@@ -91,14 +91,12 @@ struct MultiSearchImpl
 
         std::vector<std::string_view> needles;
 
-        for (size_t i = 0; i < haystack_size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             needles.reserve(needles_offsets[i] - prev_needles_offset);
 
             for (size_t j = prev_needles_offset; j < needles_offsets[i]; ++j)
-            {
                 needles.emplace_back(needles_data_string.getDataAt(j).toView());
-            }
 
             const auto * const haystack = &haystack_data[prev_haystack_offset];
             const size_t haystack_length = haystack_offsets[i] - prev_haystack_offset - 1;
diff --git a/src/Functions/PolygonUtils.h b/src/Functions/PolygonUtils.h
index 57f1243537d..bf8241774a6 100644
--- a/src/Functions/PolygonUtils.h
+++ b/src/Functions/PolygonUtils.h
@@ -124,7 +124,7 @@ public:
 
     bool hasEmptyBound() const { return has_empty_bound; }
 
-    inline bool ALWAYS_INLINE contains(CoordinateType x, CoordinateType y) const
+    inline bool contains(CoordinateType x, CoordinateType y) const
     {
         Point point(x, y);
 
@@ -167,7 +167,7 @@ public:
 
     UInt64 getAllocatedBytes() const;
 
-    inline bool ALWAYS_INLINE contains(CoordinateType x, CoordinateType y) const;
+    bool contains(CoordinateType x, CoordinateType y) const;
 
 private:
     enum class CellType : uint8_t
@@ -199,7 +199,7 @@ private:
         }
 
         /// Inner part of the HalfPlane is the left side of initialized vector.
-        bool ALWAYS_INLINE contains(CoordinateType x, CoordinateType y) const { return a * x + b * y + c >= 0; }
+        bool contains(CoordinateType x, CoordinateType y) const { return a * x + b * y + c >= 0; }
     };
 
     struct Cell
@@ -233,7 +233,7 @@ private:
     void calcGridAttributes(Box & box);
 
     template <typename T>
-    T ALWAYS_INLINE getCellIndex(T row, T col) const { return row * grid_size + col; }
+    T getCellIndex(T row, T col) const { return row * grid_size + col; }
 
     /// Complex case. Will check intersection directly.
     inline void addComplexPolygonCell(size_t index, const Box & box);
diff --git a/src/Functions/PositionImpl.h b/src/Functions/PositionImpl.h
index eeb9d8b6a59..e525b5fab57 100644
--- a/src/Functions/PositionImpl.h
+++ b/src/Functions/PositionImpl.h
@@ -193,7 +193,8 @@ struct PositionImpl
         const std::string & needle,
         const ColumnPtr & start_pos,
         PaddedPODArray<UInt64> & res,
-        [[maybe_unused]] ColumnUInt8 * res_null)
+        [[maybe_unused]] ColumnUInt8 * res_null,
+        size_t input_rows_count)
     {
         /// `res_null` serves as an output parameter for implementing an XYZOrNull variant.
         assert(!res_null);
@@ -214,13 +215,12 @@ struct PositionImpl
             }
 
             ColumnString::Offset prev_offset = 0;
-            size_t rows = haystack_offsets.size();
 
             if (const ColumnConst * start_pos_const = typeid_cast<const ColumnConst *>(&*start_pos))
             {
                 /// Needle is empty and start_pos is constant
                 UInt64 start = std::max(start_pos_const->getUInt(0), static_cast<UInt64>(1));
-                for (size_t i = 0; i < rows; ++i)
+                for (size_t i = 0; i < input_rows_count; ++i)
                 {
                     size_t haystack_size = Impl::countChars(
                         reinterpret_cast<const char *>(pos), reinterpret_cast<const char *>(pos + haystack_offsets[i] - prev_offset - 1));
@@ -234,7 +234,7 @@ struct PositionImpl
             else
             {
                 /// Needle is empty and start_pos is not constant
-                for (size_t i = 0; i < rows; ++i)
+                for (size_t i = 0; i < input_rows_count; ++i)
                 {
                     size_t haystack_size = Impl::countChars(
                         reinterpret_cast<const char *>(pos), reinterpret_cast<const char *>(pos + haystack_offsets[i] - prev_offset - 1));
@@ -359,7 +359,8 @@ struct PositionImpl
         const ColumnString::Offsets & needle_offsets,
         const ColumnPtr & start_pos,
         PaddedPODArray<UInt64> & res,
-        [[maybe_unused]] ColumnUInt8 * res_null)
+        [[maybe_unused]] ColumnUInt8 * res_null,
+        size_t input_rows_count)
     {
         /// `res_null` serves as an output parameter for implementing an XYZOrNull variant.
         assert(!res_null);
@@ -367,9 +368,7 @@ struct PositionImpl
         ColumnString::Offset prev_haystack_offset = 0;
         ColumnString::Offset prev_needle_offset = 0;
 
-        size_t size = haystack_offsets.size();
-
-        for (size_t i = 0; i < size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             size_t needle_size = needle_offsets[i] - prev_needle_offset - 1;
             size_t haystack_size = haystack_offsets[i] - prev_haystack_offset - 1;
@@ -423,7 +422,8 @@ struct PositionImpl
         const ColumnString::Offsets & needle_offsets,
         const ColumnPtr & start_pos,
         PaddedPODArray<UInt64> & res,
-        [[maybe_unused]] ColumnUInt8 * res_null)
+        [[maybe_unused]] ColumnUInt8 * res_null,
+        size_t input_rows_count)
     {
         /// `res_null` serves as an output parameter for implementing an XYZOrNull variant.
         assert(!res_null);
@@ -431,9 +431,7 @@ struct PositionImpl
         /// NOTE You could use haystack indexing. But this is a rare case.
         ColumnString::Offset prev_needle_offset = 0;
 
-        size_t size = needle_offsets.size();
-
-        for (size_t i = 0; i < size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             size_t needle_size = needle_offsets[i] - prev_needle_offset - 1;
 
diff --git a/src/Functions/ReplaceRegexpImpl.h b/src/Functions/ReplaceRegexpImpl.h
index f5fb08f71d2..14f5a2d7932 100644
--- a/src/Functions/ReplaceRegexpImpl.h
+++ b/src/Functions/ReplaceRegexpImpl.h
@@ -201,15 +201,15 @@ struct ReplaceRegexpImpl
         const String & needle,
         const String & replacement,
         ColumnString::Chars & res_data,
-        ColumnString::Offsets & res_offsets)
+        ColumnString::Offsets & res_offsets,
+        size_t input_rows_count)
     {
         if (needle.empty())
             throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Length of the pattern argument in function {} must be greater than 0.", name);
 
         ColumnString::Offset res_offset = 0;
         res_data.reserve(haystack_data.size());
-        size_t haystack_size = haystack_offsets.size();
-        res_offsets.resize(haystack_size);
+        res_offsets.resize(input_rows_count);
 
         re2::RE2::Options regexp_options;
         regexp_options.set_log_errors(false); /// don't write error messages to stderr
@@ -232,13 +232,13 @@ struct ReplaceRegexpImpl
                     case ReplaceRegexpTraits::Replace::All:   return ReplaceStringTraits::Replace::All;
                 }
             };
-            ReplaceStringImpl<Name, convertTrait(replace)>::vectorConstantConstant(haystack_data, haystack_offsets, needle, replacement, res_data, res_offsets);
+            ReplaceStringImpl<Name, convertTrait(replace)>::vectorConstantConstant(haystack_data, haystack_offsets, needle, replacement, res_data, res_offsets, input_rows_count);
             return;
         }
 
         Instructions instructions = createInstructions(replacement, num_captures);
 
-        for (size_t i = 0; i < haystack_size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             size_t from = i > 0 ? haystack_offsets[i - 1] : 0;
 
@@ -257,19 +257,19 @@ struct ReplaceRegexpImpl
         const ColumnString::Offsets & needle_offsets,
         const String & replacement,
         ColumnString::Chars & res_data,
-        ColumnString::Offsets & res_offsets)
+        ColumnString::Offsets & res_offsets,
+        size_t input_rows_count)
     {
         assert(haystack_offsets.size() == needle_offsets.size());
 
         ColumnString::Offset res_offset = 0;
         res_data.reserve(haystack_data.size());
-        size_t haystack_size = haystack_offsets.size();
-        res_offsets.resize(haystack_size);
+        res_offsets.resize(input_rows_count);
 
         re2::RE2::Options regexp_options;
         regexp_options.set_log_errors(false); /// don't write error messages to stderr
 
-        for (size_t i = 0; i < haystack_size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             size_t hs_from = i > 0 ? haystack_offsets[i - 1] : 0;
             const char * hs_data = reinterpret_cast<const char *>(haystack_data.data() + hs_from);
@@ -302,7 +302,8 @@ struct ReplaceRegexpImpl
         const ColumnString::Chars & replacement_data,
         const ColumnString::Offsets & replacement_offsets,
         ColumnString::Chars & res_data,
-        ColumnString::Offsets & res_offsets)
+        ColumnString::Offsets & res_offsets,
+        size_t input_rows_count)
     {
         assert(haystack_offsets.size() == replacement_offsets.size());
 
@@ -311,8 +312,7 @@ struct ReplaceRegexpImpl
 
         ColumnString::Offset res_offset = 0;
         res_data.reserve(haystack_data.size());
-        size_t haystack_size = haystack_offsets.size();
-        res_offsets.resize(haystack_size);
+        res_offsets.resize(input_rows_count);
 
         re2::RE2::Options regexp_options;
         regexp_options.set_log_errors(false); /// don't write error messages to stderr
@@ -323,7 +323,7 @@ struct ReplaceRegexpImpl
 
         int num_captures = std::min(searcher.NumberOfCapturingGroups() + 1, max_captures);
 
-        for (size_t i = 0; i < haystack_size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             size_t hs_from = i > 0 ? haystack_offsets[i - 1] : 0;
             const char * hs_data = reinterpret_cast<const char *>(haystack_data.data() + hs_from);
@@ -349,20 +349,20 @@ struct ReplaceRegexpImpl
         const ColumnString::Chars & replacement_data,
         const ColumnString::Offsets & replacement_offsets,
         ColumnString::Chars & res_data,
-        ColumnString::Offsets & res_offsets)
+        ColumnString::Offsets & res_offsets,
+        size_t input_rows_count)
     {
         assert(haystack_offsets.size() == needle_offsets.size());
         assert(needle_offsets.size() == replacement_offsets.size());
 
         ColumnString::Offset res_offset = 0;
         res_data.reserve(haystack_data.size());
-        size_t haystack_size = haystack_offsets.size();
-        res_offsets.resize(haystack_size);
+        res_offsets.resize(input_rows_count);
 
         re2::RE2::Options regexp_options;
         regexp_options.set_log_errors(false); /// don't write error messages to stderr
 
-        for (size_t i = 0; i < haystack_size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             size_t hs_from = i > 0 ? haystack_offsets[i - 1] : 0;
             const char * hs_data = reinterpret_cast<const char *>(haystack_data.data() + hs_from);
@@ -399,15 +399,15 @@ struct ReplaceRegexpImpl
         const String & needle,
         const String & replacement,
         ColumnString::Chars & res_data,
-        ColumnString::Offsets & res_offsets)
+        ColumnString::Offsets & res_offsets,
+        size_t input_rows_count)
     {
         if (needle.empty())
             throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Length of the pattern argument in function {} must be greater than 0.", name);
 
         ColumnString::Offset res_offset = 0;
-        size_t haystack_size = haystack_data.size() / n;
         res_data.reserve(haystack_data.size());
-        res_offsets.resize(haystack_size);
+        res_offsets.resize(input_rows_count);
 
         re2::RE2::Options regexp_options;
         regexp_options.set_log_errors(false); /// don't write error messages to stderr
@@ -419,7 +419,7 @@ struct ReplaceRegexpImpl
         int num_captures = std::min(searcher.NumberOfCapturingGroups() + 1, max_captures);
         Instructions instructions = createInstructions(replacement, num_captures);
 
-        for (size_t i = 0; i < haystack_size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             size_t from = i * n;
             const char * hs_data = reinterpret_cast<const char *>(haystack_data.data() + from);
diff --git a/src/Functions/ReplaceStringImpl.h b/src/Functions/ReplaceStringImpl.h
index de3942acbd8..7c56d657b3e 100644
--- a/src/Functions/ReplaceStringImpl.h
+++ b/src/Functions/ReplaceStringImpl.h
@@ -35,7 +35,8 @@ struct ReplaceStringImpl
         const String & needle,
         const String & replacement,
         ColumnString::Chars & res_data,
-        ColumnString::Offsets & res_offsets)
+        ColumnString::Offsets & res_offsets,
+        size_t input_rows_count)
     {
         if (needle.empty())
             throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Length of the pattern argument in function {} must be greater than 0.", name);
@@ -46,8 +47,7 @@ struct ReplaceStringImpl
 
         ColumnString::Offset res_offset = 0;
         res_data.reserve(haystack_data.size());
-        const size_t haystack_size = haystack_offsets.size();
-        res_offsets.resize(haystack_size);
+        res_offsets.resize(input_rows_count);
 
         /// The current index in the array of strings.
         size_t i = 0;
@@ -124,21 +124,20 @@ struct ReplaceStringImpl
         const ColumnString::Offsets & needle_offsets,
         const String & replacement,
         ColumnString::Chars & res_data,
-        ColumnString::Offsets & res_offsets)
+        ColumnString::Offsets & res_offsets,
+        size_t input_rows_count)
     {
         chassert(haystack_offsets.size() == needle_offsets.size());
 
-        const size_t haystack_size = haystack_offsets.size();
-
         res_data.reserve(haystack_data.size());
-        res_offsets.resize(haystack_size);
+        res_offsets.resize(input_rows_count);
 
         ColumnString::Offset res_offset = 0;
 
         size_t prev_haystack_offset = 0;
         size_t prev_needle_offset = 0;
 
-        for (size_t i = 0; i < haystack_size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             const auto * const cur_haystack_data = &haystack_data[prev_haystack_offset];
             const size_t cur_haystack_length = haystack_offsets[i] - prev_haystack_offset - 1;
@@ -195,24 +194,23 @@ struct ReplaceStringImpl
         const ColumnString::Chars & replacement_data,
         const ColumnString::Offsets & replacement_offsets,
         ColumnString::Chars & res_data,
-        ColumnString::Offsets & res_offsets)
+        ColumnString::Offsets & res_offsets,
+        size_t input_rows_count)
     {
         chassert(haystack_offsets.size() == replacement_offsets.size());
 
         if (needle.empty())
             throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Length of the pattern argument in function {} must be greater than 0.", name);
 
-        const size_t haystack_size = haystack_offsets.size();
-
         res_data.reserve(haystack_data.size());
-        res_offsets.resize(haystack_size);
+        res_offsets.resize(input_rows_count);
 
         ColumnString::Offset res_offset = 0;
 
         size_t prev_haystack_offset = 0;
         size_t prev_replacement_offset = 0;
 
-        for (size_t i = 0; i < haystack_size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             const auto * const cur_haystack_data = &haystack_data[prev_haystack_offset];
             const size_t cur_haystack_length = haystack_offsets[i] - prev_haystack_offset - 1;
@@ -267,15 +265,14 @@ struct ReplaceStringImpl
         const ColumnString::Chars & replacement_data,
         const ColumnString::Offsets & replacement_offsets,
         ColumnString::Chars & res_data,
-        ColumnString::Offsets & res_offsets)
+        ColumnString::Offsets & res_offsets,
+        size_t input_rows_count)
     {
         chassert(haystack_offsets.size() == needle_offsets.size());
         chassert(needle_offsets.size() == replacement_offsets.size());
 
-        const size_t haystack_size = haystack_offsets.size();
-
         res_data.reserve(haystack_data.size());
-        res_offsets.resize(haystack_size);
+        res_offsets.resize(input_rows_count);
 
         ColumnString::Offset res_offset = 0;
 
@@ -283,7 +280,7 @@ struct ReplaceStringImpl
         size_t prev_needle_offset = 0;
         size_t prev_replacement_offset = 0;
 
-        for (size_t i = 0; i < haystack_size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             const auto * const cur_haystack_data = &haystack_data[prev_haystack_offset];
             const size_t cur_haystack_length = haystack_offsets[i] - prev_haystack_offset - 1;
@@ -345,7 +342,8 @@ struct ReplaceStringImpl
         const String & needle,
         const String & replacement,
         ColumnString::Chars & res_data,
-        ColumnString::Offsets & res_offsets)
+        ColumnString::Offsets & res_offsets,
+        size_t input_rows_count)
     {
         if (needle.empty())
             throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Length of the pattern argument in function {} must be greater than 0.", name);
@@ -355,9 +353,8 @@ struct ReplaceStringImpl
         const UInt8 * pos = begin;
 
         ColumnString::Offset res_offset = 0;
-        size_t haystack_size = haystack_data.size() / n;
         res_data.reserve(haystack_data.size());
-        res_offsets.resize(haystack_size);
+        res_offsets.resize(input_rows_count);
 
         /// The current index in the string array.
         size_t i = 0;
@@ -384,13 +381,13 @@ struct ReplaceStringImpl
 
             /// Copy skipped strings without any changes but
             /// add zero byte to the end of each string.
-            while (i < haystack_size && begin + n * (i + 1) <= match)
+            while (i < input_rows_count && begin + n * (i + 1) <= match)
             {
                 COPY_REST_OF_CURRENT_STRING();
             }
 
             /// If you have reached the end, it's time to stop
-            if (i == haystack_size)
+            if (i == input_rows_count)
                 break;
 
             /// Copy unchanged part of current string.
diff --git a/src/Functions/URL/FirstSignificantSubdomainCustomImpl.h b/src/Functions/URL/FirstSignificantSubdomainCustomImpl.h
index 68582198ea3..22fec17654c 100644
--- a/src/Functions/URL/FirstSignificantSubdomainCustomImpl.h
+++ b/src/Functions/URL/FirstSignificantSubdomainCustomImpl.h
@@ -64,7 +64,7 @@ public:
         return arguments[0].type;
     }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t /*input_rows_count*/) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & /*result_type*/, size_t input_rows_count) const override
     {
         const ColumnConst * column_tld_list_name = checkAndGetColumnConstStringOrFixedString(arguments[1].column.get());
         FirstSignificantSubdomainCustomLookup tld_lookup(column_tld_list_name->getValue<String>());
@@ -72,7 +72,7 @@ public:
         if (const ColumnString * col = checkAndGetColumn<ColumnString>(&*arguments[0].column))
         {
             auto col_res = ColumnString::create();
-            vector(tld_lookup, col->getChars(), col->getOffsets(), col_res->getChars(), col_res->getOffsets());
+            vector(tld_lookup, col->getChars(), col->getOffsets(), col_res->getChars(), col_res->getOffsets(), input_rows_count);
             return col_res;
         }
         else
@@ -82,11 +82,11 @@ public:
 
     static void vector(FirstSignificantSubdomainCustomLookup & tld_lookup,
         const ColumnString::Chars & data, const ColumnString::Offsets & offsets,
-        ColumnString::Chars & res_data, ColumnString::Offsets & res_offsets)
+        ColumnString::Chars & res_data, ColumnString::Offsets & res_offsets,
+        size_t input_rows_count)
     {
-        size_t size = offsets.size();
-        res_offsets.resize(size);
-        res_data.reserve(size * Extractor::getReserveLengthForElement());
+        res_offsets.resize(input_rows_count);
+        res_data.reserve(input_rows_count * Extractor::getReserveLengthForElement());
 
         size_t prev_offset = 0;
         size_t res_offset = 0;
@@ -95,7 +95,7 @@ public:
         Pos start;
         size_t length;
 
-        for (size_t i = 0; i < size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             Extractor::execute(tld_lookup, reinterpret_cast<const char *>(&data[prev_offset]), offsets[i] - prev_offset - 1, start, length);
 
diff --git a/src/Functions/URL/cutURLParameter.cpp b/src/Functions/URL/cutURLParameter.cpp
index 7a2b96ec874..3ab9cad1ea7 100644
--- a/src/Functions/URL/cutURLParameter.cpp
+++ b/src/Functions/URL/cutURLParameter.cpp
@@ -44,7 +44,7 @@ public:
         return std::make_shared<DataTypeString>();
     }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
         const ColumnPtr column = arguments[0].column;
         const ColumnPtr column_needle = arguments[1].column;
@@ -71,7 +71,7 @@ public:
 
             ColumnString::Chars & vec_res = col_res->getChars();
             ColumnString::Offsets & offsets_res = col_res->getOffsets();
-            vector(col->getChars(), col->getOffsets(), col_needle, col_needle_const_array, vec_res, offsets_res);
+            vector(col->getChars(), col->getOffsets(), col_needle, col_needle_const_array, vec_res, offsets_res, input_rows_count);
             return col_res;
         }
         else
@@ -130,7 +130,8 @@ public:
         const ColumnString::Offsets & offsets,
         const ColumnConst * col_needle,
         const ColumnArray * col_needle_const_array,
-        ColumnString::Chars & res_data, ColumnString::Offsets & res_offsets)
+        ColumnString::Chars & res_data, ColumnString::Offsets & res_offsets,
+        size_t input_rows_count)
     {
         res_data.reserve(data.size());
         res_offsets.resize(offsets.size());
@@ -141,7 +142,7 @@ public:
         size_t res_offset = 0;
         size_t cur_res_offset;
 
-        for (size_t i = 0; i < offsets.size(); ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             cur_offset = offsets[i];
             cur_len = cur_offset - prev_offset;
diff --git a/src/Functions/URL/extractURLParameter.cpp b/src/Functions/URL/extractURLParameter.cpp
index f75875e0200..590c2779d9c 100644
--- a/src/Functions/URL/extractURLParameter.cpp
+++ b/src/Functions/URL/extractURLParameter.cpp
@@ -10,10 +10,11 @@ struct ExtractURLParameterImpl
     static void vector(const ColumnString::Chars & data,
         const ColumnString::Offsets & offsets,
         std::string pattern,
-        ColumnString::Chars & res_data, ColumnString::Offsets & res_offsets)
+        ColumnString::Chars & res_data, ColumnString::Offsets & res_offsets,
+        size_t input_rows_count)
     {
         res_data.reserve(data.size() / 5);
-        res_offsets.resize(offsets.size());
+        res_offsets.resize(input_rows_count);
 
         pattern += '=';
         const char * param_str = pattern.c_str();
@@ -22,7 +23,7 @@ struct ExtractURLParameterImpl
         ColumnString::Offset prev_offset = 0;
         ColumnString::Offset res_offset = 0;
 
-        for (size_t i = 0; i < offsets.size(); ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             ColumnString::Offset cur_offset = offsets[i];
 
diff --git a/src/Functions/URL/port.cpp b/src/Functions/URL/port.cpp
index fac46281604..7492ebcb4e9 100644
--- a/src/Functions/URL/port.cpp
+++ b/src/Functions/URL/port.cpp
@@ -46,7 +46,7 @@ struct FunctionPortImpl : public IFunction
     }
 
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
         UInt16 default_port = 0;
         if (arguments.size() == 2)
@@ -64,7 +64,7 @@ struct FunctionPortImpl : public IFunction
             typename ColumnVector<UInt16>::Container & vec_res = col_res->getData();
             vec_res.resize(url_column->size());
 
-            vector(default_port, url_strs->getChars(), url_strs->getOffsets(), vec_res);
+            vector(default_port, url_strs->getChars(), url_strs->getOffsets(), vec_res, input_rows_count);
             return col_res;
         }
         else
@@ -73,12 +73,10 @@ struct FunctionPortImpl : public IFunction
 }
 
 private:
-    static void vector(UInt16 default_port, const ColumnString::Chars & data, const ColumnString::Offsets & offsets, PaddedPODArray<UInt16> & res)
+    static void vector(UInt16 default_port, const ColumnString::Chars & data, const ColumnString::Offsets & offsets, PaddedPODArray<UInt16> & res, size_t input_rows_count)
     {
-        size_t size = offsets.size();
-
         ColumnString::Offset prev_offset = 0;
-        for (size_t i = 0; i < size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             res[i] = extractPort(default_port, data, prev_offset, offsets[i] - prev_offset - 1);
             prev_offset = offsets[i];
diff --git a/src/Functions/UTCTimestampTransform.cpp b/src/Functions/UTCTimestampTransform.cpp
index 36ec520068f..35015188078 100644
--- a/src/Functions/UTCTimestampTransform.cpp
+++ b/src/Functions/UTCTimestampTransform.cpp
@@ -67,7 +67,7 @@ namespace
             return date_time_type;
         }
 
-        ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t) const override
+        ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
         {
             if (arguments.size() != 2)
                 throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {}'s arguments number must be 2.", name);
@@ -81,11 +81,10 @@ namespace
             if (WhichDataType(arg1.type).isDateTime())
             {
                 const auto & date_time_col = checkAndGetColumn<ColumnDateTime>(*arg1.column);
-                size_t col_size = date_time_col.size();
                 using ColVecTo = DataTypeDateTime::ColumnType;
-                typename ColVecTo::MutablePtr result_column = ColVecTo::create(col_size);
+                typename ColVecTo::MutablePtr result_column = ColVecTo::create(input_rows_count);
                 typename ColVecTo::Container & result_data = result_column->getData();
-                for (size_t i = 0; i < col_size; ++i)
+                for (size_t i = 0; i < input_rows_count; ++i)
                 {
                     UInt32 date_time_val = date_time_col.getElement(i);
                     LocalDateTime date_time(date_time_val, Name::to ? utc_time_zone : DateLUT::instance(time_zone_val));
@@ -97,14 +96,13 @@ namespace
             else if (WhichDataType(arg1.type).isDateTime64())
             {
                 const auto & date_time_col = checkAndGetColumn<ColumnDateTime64>(*arg1.column);
-                size_t col_size = date_time_col.size();
                 const DataTypeDateTime64 * date_time_type = static_cast<const DataTypeDateTime64 *>(arg1.type.get());
                 UInt32 col_scale = date_time_type->getScale();
                 Int64 scale_multiplier = DecimalUtils::scaleMultiplier<Int64>(col_scale);
                 using ColDecimalTo = DataTypeDateTime64::ColumnType;
-                typename ColDecimalTo::MutablePtr result_column = ColDecimalTo::create(col_size, col_scale);
+                typename ColDecimalTo::MutablePtr result_column = ColDecimalTo::create(input_rows_count, col_scale);
                 typename ColDecimalTo::Container & result_data = result_column->getData();
-                for (size_t i = 0; i < col_size; ++i)
+                for (size_t i = 0; i < input_rows_count; ++i)
                 {
                     DateTime64 date_time_val = date_time_col.getElement(i);
                     Int64 seconds = date_time_val.value / scale_multiplier;
diff --git a/src/Functions/array/arrayAUC.cpp b/src/Functions/array/arrayAUC.cpp
index 3e2a3bf6863..7a61c9d368f 100644
--- a/src/Functions/array/arrayAUC.cpp
+++ b/src/Functions/array/arrayAUC.cpp
@@ -143,13 +143,13 @@ private:
         const IColumn & scores,
         const IColumn & labels,
         const ColumnArray::Offsets & offsets,
-        PaddedPODArray<Float64> & result)
+        PaddedPODArray<Float64> & result,
+        size_t input_rows_count)
     {
-        size_t size = offsets.size();
-        result.resize(size);
+        result.resize(input_rows_count);
 
         ColumnArray::Offset current_offset = 0;
-        for (size_t i = 0; i < size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             auto next_offset = offsets[i];
             result[i] = apply(scores, labels, current_offset, next_offset);
@@ -179,7 +179,7 @@ public:
         return std::make_shared<DataTypeFloat64>();
     }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
         ColumnPtr col1 = arguments[0].column->convertToFullColumnIfConst();
         ColumnPtr col2 = arguments[1].column->convertToFullColumnIfConst();
@@ -203,7 +203,8 @@ public:
             col_array1->getData(),
             col_array2->getData(),
             col_array1->getOffsets(),
-            col_res->getData());
+            col_res->getData(),
+            input_rows_count);
 
         return col_res;
     }
diff --git a/src/Functions/array/arrayConcat.cpp b/src/Functions/array/arrayConcat.cpp
index cdb361b73b9..768877bac99 100644
--- a/src/Functions/array/arrayConcat.cpp
+++ b/src/Functions/array/arrayConcat.cpp
@@ -40,7 +40,6 @@ ColumnPtr FunctionArrayConcat::executeImpl(const ColumnsWithTypeAndName & argume
     if (result_type->onlyNull())
         return result_type->createColumnConstWithDefaultValue(input_rows_count);
 
-    size_t rows = input_rows_count;
     size_t num_args = arguments.size();
 
     Columns preprocessed_columns(num_args);
@@ -69,7 +68,7 @@ ColumnPtr FunctionArrayConcat::executeImpl(const ColumnsWithTypeAndName & argume
         }
 
         if (const auto * argument_column_array = typeid_cast<const ColumnArray *>(argument_column.get()))
-            sources.emplace_back(GatherUtils::createArraySource(*argument_column_array, is_const, rows));
+            sources.emplace_back(GatherUtils::createArraySource(*argument_column_array, is_const, input_rows_count));
         else
             throw Exception(ErrorCodes::LOGICAL_ERROR, "Arguments for function {} must be arrays.", getName());
     }
diff --git a/src/Functions/array/arrayEnumerateRanked.h b/src/Functions/array/arrayEnumerateRanked.h
index ad325fe542a..269a7db6e92 100644
--- a/src/Functions/array/arrayEnumerateRanked.h
+++ b/src/Functions/array/arrayEnumerateRanked.h
@@ -132,7 +132,7 @@ private:
 
 
 /// Hash a set of keys into a UInt128 value.
-static inline UInt128 ALWAYS_INLINE hash128depths(const std::vector<size_t> & indices, const ColumnRawPtrs & key_columns)
+static UInt128 hash128depths(const std::vector<size_t> & indices, const ColumnRawPtrs & key_columns)
 {
     SipHash hash;
     for (size_t j = 0, keys_size = key_columns.size(); j < keys_size; ++j)
diff --git a/src/Functions/array/length.cpp b/src/Functions/array/length.cpp
index d81c071b55e..760506194fa 100644
--- a/src/Functions/array/length.cpp
+++ b/src/Functions/array/length.cpp
@@ -16,40 +16,38 @@ struct LengthImpl
 {
     static constexpr auto is_fixed_to_constant = true;
 
-    static void vector(const ColumnString::Chars & /*data*/, const ColumnString::Offsets & offsets, PaddedPODArray<UInt64> & res)
+    static void vector(const ColumnString::Chars & /*data*/, const ColumnString::Offsets & offsets, PaddedPODArray<UInt64> & res, size_t input_rows_count)
     {
-        size_t size = offsets.size();
-        for (size_t i = 0; i < size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
             res[i] = offsets[i] - 1 - offsets[i - 1];
     }
 
-    static void vectorFixedToConstant(const ColumnString::Chars & /*data*/, size_t n, UInt64 & res)
+    static void vectorFixedToConstant(const ColumnString::Chars & /*data*/, size_t n, UInt64 & res, size_t)
     {
         res = n;
     }
 
-    static void vectorFixedToVector(const ColumnString::Chars & /*data*/, size_t /*n*/, PaddedPODArray<UInt64> & /*res*/)
+    static void vectorFixedToVector(const ColumnString::Chars & /*data*/, size_t /*n*/, PaddedPODArray<UInt64> & /*res*/, size_t)
     {
     }
 
-    static void array(const ColumnString::Offsets & offsets, PaddedPODArray<UInt64> & res)
+    static void array(const ColumnString::Offsets & offsets, PaddedPODArray<UInt64> & res, size_t input_rows_count)
     {
-        size_t size = offsets.size();
-        for (size_t i = 0; i < size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
             res[i] = offsets[i] - offsets[i - 1];
     }
 
-    [[noreturn]] static void uuid(const ColumnUUID::Container &, size_t &, PaddedPODArray<UInt64> &)
+    [[noreturn]] static void uuid(const ColumnUUID::Container &, size_t &, PaddedPODArray<UInt64> &, size_t)
     {
         throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Cannot apply function length to UUID argument");
     }
 
-    [[noreturn]] static void ipv6(const ColumnIPv6::Container &, size_t &, PaddedPODArray<UInt64> &)
+    [[noreturn]] static void ipv6(const ColumnIPv6::Container &, size_t &, PaddedPODArray<UInt64> &, size_t)
     {
         throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Cannot apply function length to IPv6 argument");
     }
 
-    [[noreturn]] static void ipv4(const ColumnIPv4::Container &, size_t &, PaddedPODArray<UInt64> &)
+    [[noreturn]] static void ipv4(const ColumnIPv4::Container &, size_t &, PaddedPODArray<UInt64> &, size_t)
     {
         throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Cannot apply function length to IPv4 argument");
     }
diff --git a/src/Functions/ascii.cpp b/src/Functions/ascii.cpp
index 0d50e5d203b..e65996a90e1 100644
--- a/src/Functions/ascii.cpp
+++ b/src/Functions/ascii.cpp
@@ -23,47 +23,43 @@ struct AsciiImpl
     using ReturnType = Int32;
 
 
-    static void vector(const ColumnString::Chars & data, const ColumnString::Offsets & offsets, PaddedPODArray<ReturnType> & res)
+    static void vector(const ColumnString::Chars & data, const ColumnString::Offsets & offsets, PaddedPODArray<ReturnType> & res, size_t input_rows_count)
     {
-        size_t size = offsets.size();
-
         ColumnString::Offset prev_offset = 0;
-        for (size_t i = 0; i < size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             res[i] = doAscii(data, prev_offset, offsets[i] - prev_offset - 1);
             prev_offset = offsets[i];
         }
     }
 
-    [[noreturn]] static void vectorFixedToConstant(const ColumnString::Chars &  /*data*/, size_t  /*n*/, Int32 &  /*res*/)
+    [[noreturn]] static void vectorFixedToConstant(const ColumnString::Chars &, size_t, Int32 &, size_t)
     {
         throw Exception(ErrorCodes::NOT_IMPLEMENTED, "vectorFixedToConstant not implemented for function {}", AsciiName::name);
     }
 
-    static void vectorFixedToVector(const ColumnString::Chars & data, size_t n, PaddedPODArray<ReturnType> & res)
+    static void vectorFixedToVector(const ColumnString::Chars & data, size_t n, PaddedPODArray<ReturnType> & res, size_t input_rows_count)
     {
-        size_t size = data.size() / n;
-
-        for (size_t i = 0; i < size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
             res[i] = doAscii(data, i * n, n);
     }
 
-    [[noreturn]] static void array(const ColumnString::Offsets & /*offsets*/, PaddedPODArray<ReturnType> & /*res*/)
+    [[noreturn]] static void array(const ColumnString::Offsets &, PaddedPODArray<ReturnType> &, size_t)
     {
         throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Cannot apply function {} to Array argument", AsciiName::name);
     }
 
-    [[noreturn]] static void uuid(const ColumnUUID::Container & /*offsets*/, size_t /*n*/, PaddedPODArray<ReturnType> & /*res*/)
+    [[noreturn]] static void uuid(const ColumnUUID::Container &, size_t, PaddedPODArray<ReturnType> &, size_t)
     {
         throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Cannot apply function {} to UUID argument", AsciiName::name);
     }
 
-    [[noreturn]] static void ipv6(const ColumnIPv6::Container & /*offsets*/, size_t /*n*/, PaddedPODArray<ReturnType> & /*res*/)
+    [[noreturn]] static void ipv6(const ColumnIPv6::Container &, size_t, PaddedPODArray<ReturnType> &, size_t)
     {
         throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Cannot apply function {} to IPv6 argument", AsciiName::name);
     }
 
-    [[noreturn]] static void ipv4(const ColumnIPv4::Container & /*offsets*/, size_t /*n*/, PaddedPODArray<ReturnType> & /*res*/)
+    [[noreturn]] static void ipv4(const ColumnIPv4::Container &, size_t, PaddedPODArray<ReturnType> &, size_t)
     {
         throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Cannot apply function {} to IPv4 argument", AsciiName::name);
     }
diff --git a/src/Functions/extract.cpp b/src/Functions/extract.cpp
index 6bbdaff0e3f..c78ee9898b7 100644
--- a/src/Functions/extract.cpp
+++ b/src/Functions/extract.cpp
@@ -16,10 +16,11 @@ struct ExtractImpl
         const ColumnString::Offsets & offsets,
         const std::string & pattern,
         ColumnString::Chars & res_data,
-        ColumnString::Offsets & res_offsets)
+        ColumnString::Offsets & res_offsets,
+        size_t input_rows_count)
     {
         res_data.reserve(data.size() / 5);
-        res_offsets.resize(offsets.size());
+        res_offsets.resize(input_rows_count);
 
         const OptimizedRegularExpression regexp = Regexps::createRegexp<false, false, false>(pattern);
 
@@ -29,7 +30,7 @@ struct ExtractImpl
         size_t prev_offset = 0;
         size_t res_offset = 0;
 
-        for (size_t i = 0; i < offsets.size(); ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             size_t cur_offset = offsets[i];
 
diff --git a/src/Functions/isValidUTF8.cpp b/src/Functions/isValidUTF8.cpp
index d5f5e6a8986..1959502af06 100644
--- a/src/Functions/isValidUTF8.cpp
+++ b/src/Functions/isValidUTF8.cpp
@@ -219,42 +219,42 @@ SOFTWARE.
 
     static constexpr bool is_fixed_to_constant = false;
 
-    static void vector(const ColumnString::Chars & data, const ColumnString::Offsets & offsets, PaddedPODArray<UInt8> & res)
+    static void vector(const ColumnString::Chars & data, const ColumnString::Offsets & offsets, PaddedPODArray<UInt8> & res, size_t input_rows_count)
     {
-        size_t size = offsets.size();
         size_t prev_offset = 0;
-        for (size_t i = 0; i < size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             res[i] = isValidUTF8(data.data() + prev_offset, offsets[i] - 1 - prev_offset);
             prev_offset = offsets[i];
         }
     }
 
-    static void vectorFixedToConstant(const ColumnString::Chars & /*data*/, size_t /*n*/, UInt8 & /*res*/) {}
-
-    static void vectorFixedToVector(const ColumnString::Chars & data, size_t n, PaddedPODArray<UInt8> & res)
+    static void vectorFixedToConstant(const ColumnString::Chars &, size_t, UInt8 &, size_t)
     {
-        size_t size = data.size() / n;
-        for (size_t i = 0; i < size; ++i)
+    }
+
+    static void vectorFixedToVector(const ColumnString::Chars & data, size_t n, PaddedPODArray<UInt8> & res, size_t input_rows_count)
+    {
+        for (size_t i = 0; i < input_rows_count; ++i)
             res[i] = isValidUTF8(data.data() + i * n, n);
     }
 
-    [[noreturn]] static void array(const ColumnString::Offsets &, PaddedPODArray<UInt8> &)
+    [[noreturn]] static void array(const ColumnString::Offsets &, PaddedPODArray<UInt8> &, size_t)
     {
         throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Cannot apply function isValidUTF8 to Array argument");
     }
 
-    [[noreturn]] static void uuid(const ColumnUUID::Container &, size_t &, PaddedPODArray<UInt8> &)
+    [[noreturn]] static void uuid(const ColumnUUID::Container &, size_t &, PaddedPODArray<UInt8> &, size_t)
     {
         throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Cannot apply function isValidUTF8 to UUID argument");
     }
 
-    [[noreturn]] static void ipv6(const ColumnIPv6::Container &, size_t &, PaddedPODArray<UInt8> &)
+    [[noreturn]] static void ipv6(const ColumnIPv6::Container &, size_t &, PaddedPODArray<UInt8> &, size_t)
     {
         throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Cannot apply function isValidUTF8 to IPv6 argument");
     }
 
-    [[noreturn]] static void ipv4(const ColumnIPv4::Container &, size_t &, PaddedPODArray<UInt8> &)
+    [[noreturn]] static void ipv4(const ColumnIPv4::Container &, size_t &, PaddedPODArray<UInt8> &, size_t)
     {
         throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Cannot apply function isValidUTF8 to IPv4 argument");
     }
diff --git a/src/Functions/keyvaluepair/extractKeyValuePairs.cpp b/src/Functions/keyvaluepair/extractKeyValuePairs.cpp
index 1c5164e132d..cc9e57ac186 100644
--- a/src/Functions/keyvaluepair/extractKeyValuePairs.cpp
+++ b/src/Functions/keyvaluepair/extractKeyValuePairs.cpp
@@ -54,7 +54,7 @@ class ExtractKeyValuePairs : public IFunction
         return builder.build();
     }
 
-    ColumnPtr extract(ColumnPtr data_column, std::shared_ptr<KeyValuePairExtractor> extractor) const
+    ColumnPtr extract(ColumnPtr data_column, std::shared_ptr<KeyValuePairExtractor> extractor, size_t input_rows_count) const
     {
         auto offsets = ColumnUInt64::create();
 
@@ -63,7 +63,7 @@ class ExtractKeyValuePairs : public IFunction
 
         uint64_t offset = 0u;
 
-        for (auto i = 0u; i < data_column->size(); i++)
+        for (auto i = 0u; i < input_rows_count; i++)
         {
             auto row = data_column->getDataAt(i).toView();
 
@@ -97,13 +97,13 @@ public:
         return std::make_shared<ExtractKeyValuePairs>(context);
     }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
         auto parsed_arguments = ArgumentExtractor::extract(arguments);
 
         auto extractor = getExtractor(parsed_arguments);
 
-        return extract(parsed_arguments.data_column, extractor);
+        return extract(parsed_arguments.data_column, extractor, input_rows_count);
     }
 
     DataTypePtr getReturnTypeImpl(const DataTypes &) const override
diff --git a/src/Functions/lengthUTF8.cpp b/src/Functions/lengthUTF8.cpp
index 59a0d532602..97a42816674 100644
--- a/src/Functions/lengthUTF8.cpp
+++ b/src/Functions/lengthUTF8.cpp
@@ -23,48 +23,42 @@ struct LengthUTF8Impl
 {
     static constexpr auto is_fixed_to_constant = false;
 
-    static void vector(const ColumnString::Chars & data, const ColumnString::Offsets & offsets, PaddedPODArray<UInt64> & res)
+    static void vector(const ColumnString::Chars & data, const ColumnString::Offsets & offsets, PaddedPODArray<UInt64> & res, size_t input_rows_count)
     {
-        size_t size = offsets.size();
-
         ColumnString::Offset prev_offset = 0;
-        for (size_t i = 0; i < size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             res[i] = UTF8::countCodePoints(&data[prev_offset], offsets[i] - prev_offset - 1);
             prev_offset = offsets[i];
         }
     }
 
-    static void vectorFixedToConstant(const ColumnString::Chars & /*data*/, size_t /*n*/, UInt64 & /*res*/)
+    static void vectorFixedToConstant(const ColumnString::Chars & /*data*/, size_t /*n*/, UInt64 & /*res*/, size_t)
     {
     }
 
-    static void vectorFixedToVector(const ColumnString::Chars & data, size_t n, PaddedPODArray<UInt64> & res)
+    static void vectorFixedToVector(const ColumnString::Chars & data, size_t n, PaddedPODArray<UInt64> & res, size_t input_rows_count)
     {
-        size_t size = data.size() / n;
-
-        for (size_t i = 0; i < size; ++i)
-        {
+        for (size_t i = 0; i < input_rows_count; ++i)
             res[i] = UTF8::countCodePoints(&data[i * n], n);
-        }
     }
 
-    [[noreturn]] static void array(const ColumnString::Offsets &, PaddedPODArray<UInt64> &)
+    [[noreturn]] static void array(const ColumnString::Offsets &, PaddedPODArray<UInt64> &, size_t)
     {
         throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Cannot apply function lengthUTF8 to Array argument");
     }
 
-    [[noreturn]] static void uuid(const ColumnUUID::Container &, size_t &, PaddedPODArray<UInt64> &)
+    [[noreturn]] static void uuid(const ColumnUUID::Container &, size_t &, PaddedPODArray<UInt64> &, size_t)
     {
         throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Cannot apply function lengthUTF8 to UUID argument");
     }
 
-    [[noreturn]] static void ipv6(const ColumnIPv6::Container &, size_t &, PaddedPODArray<UInt64> &)
+    [[noreturn]] static void ipv6(const ColumnIPv6::Container &, size_t &, PaddedPODArray<UInt64> &, size_t)
     {
         throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Cannot apply function lengthUTF8 to IPv6 argument");
     }
 
-    [[noreturn]] static void ipv4(const ColumnIPv4::Container &, size_t &, PaddedPODArray<UInt64> &)
+    [[noreturn]] static void ipv4(const ColumnIPv4::Container &, size_t &, PaddedPODArray<UInt64> &, size_t)
     {
         throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Cannot apply function lengthUTF8 to IPv4 argument");
     }
diff --git a/src/Functions/reinterpretAs.cpp b/src/Functions/reinterpretAs.cpp
index 5293b688678..81c9d20ce82 100644
--- a/src/Functions/reinterpretAs.cpp
+++ b/src/Functions/reinterpretAs.cpp
@@ -114,7 +114,7 @@ public:
         return to_type;
     }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t /*input_rows_count*/) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override
     {
         auto from_type = arguments[0].type;
 
@@ -136,9 +136,9 @@ public:
                 ColumnFixedString * dst_concrete = assert_cast<ColumnFixedString *>(dst.get());
 
                 if (src.isFixedAndContiguous() && src.sizeOfValueIfFixed() == dst_concrete->getN())
-                    executeContiguousToFixedString(src, *dst_concrete, dst_concrete->getN());
+                    executeContiguousToFixedString(src, *dst_concrete, dst_concrete->getN(), input_rows_count);
                 else
-                    executeToFixedString(src, *dst_concrete, dst_concrete->getN());
+                    executeToFixedString(src, *dst_concrete, dst_concrete->getN(), input_rows_count);
 
                 result = std::move(dst);
 
@@ -156,7 +156,7 @@ public:
                 MutableColumnPtr dst = result_type->createColumn();
 
                 ColumnString * dst_concrete = assert_cast<ColumnString *>(dst.get());
-                executeToString(src, *dst_concrete);
+                executeToString(src, *dst_concrete, input_rows_count);
 
                 result = std::move(dst);
 
@@ -174,12 +174,11 @@ public:
 
                     const auto & data_from = col_from->getChars();
                     const auto & offsets_from = col_from->getOffsets();
-                    size_t size = offsets_from.size();
                     auto & vec_res = col_res->getData();
-                    vec_res.resize_fill(size);
+                    vec_res.resize_fill(input_rows_count);
 
                     size_t offset = 0;
-                    for (size_t i = 0; i < size; ++i)
+                    for (size_t i = 0; i < input_rows_count; ++i)
                     {
                         size_t copy_size = std::min(static_cast<UInt64>(sizeof(ToFieldType)), offsets_from[i] - offset - 1);
                         if constexpr (std::endian::native == std::endian::little)
@@ -209,7 +208,6 @@ public:
 
                     const auto& data_from = col_from_fixed->getChars();
                     size_t step = col_from_fixed->getN();
-                    size_t size = data_from.size() / step;
                     auto & vec_res = col_res->getData();
 
                     size_t offset = 0;
@@ -217,11 +215,11 @@ public:
                     size_t index = data_from.size() - copy_size;
 
                     if (sizeof(ToFieldType) <= step)
-                        vec_res.resize(size);
+                        vec_res.resize(input_rows_count);
                     else
-                        vec_res.resize_fill(size);
+                        vec_res.resize_fill(input_rows_count);
 
-                    for (size_t i = 0; i < size; ++i)
+                    for (size_t i = 0; i < input_rows_count; ++i)
                     {
                         if constexpr (std::endian::native == std::endian::little)
                             memcpy(&vec_res[i], &data_from[offset], copy_size);
@@ -251,12 +249,11 @@ public:
                     auto & from = column_from->getData();
                     auto & to = column_to->getData();
 
-                    size_t size = from.size();
-                    to.resize_fill(size);
+                    to.resize_fill(input_rows_count);
 
                     static constexpr size_t copy_size = std::min(sizeof(From), sizeof(To));
 
-                    for (size_t i = 0; i < size; ++i)
+                    for (size_t i = 0; i < input_rows_count; ++i)
                     {
                         if constexpr (std::endian::native == std::endian::little)
                             memcpy(static_cast<void*>(&to[i]), static_cast<const void*>(&from[i]), copy_size);
@@ -307,14 +304,13 @@ private:
             type.isDecimal();
     }
 
-    static void NO_INLINE executeToFixedString(const IColumn & src, ColumnFixedString & dst, size_t n)
+    static void NO_INLINE executeToFixedString(const IColumn & src, ColumnFixedString & dst, size_t n, size_t input_rows_count)
     {
-        size_t rows = src.size();
         ColumnFixedString::Chars & data_to = dst.getChars();
-        data_to.resize_fill(n * rows);
+        data_to.resize_fill(n * input_rows_count);
 
         ColumnFixedString::Offset offset = 0;
-        for (size_t i = 0; i < rows; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             std::string_view data = src.getDataAt(i).toView();
 
@@ -327,11 +323,10 @@ private:
         }
     }
 
-    static void NO_INLINE executeContiguousToFixedString(const IColumn & src, ColumnFixedString & dst, size_t n)
+    static void NO_INLINE executeContiguousToFixedString(const IColumn & src, ColumnFixedString & dst, size_t n, size_t input_rows_count)
     {
-        size_t rows = src.size();
         ColumnFixedString::Chars & data_to = dst.getChars();
-        data_to.resize(n * rows);
+        data_to.resize(n * input_rows_count);
 
 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
         memcpy(data_to.data(), src.getRawData().data(), data_to.size());
@@ -340,15 +335,14 @@ private:
 #endif
     }
 
-    static void NO_INLINE executeToString(const IColumn & src, ColumnString & dst)
+    static void NO_INLINE executeToString(const IColumn & src, ColumnString & dst, size_t input_rows_count)
     {
-        size_t rows = src.size();
         ColumnString::Chars & data_to = dst.getChars();
         ColumnString::Offsets & offsets_to = dst.getOffsets();
-        offsets_to.resize(rows);
+        offsets_to.resize(input_rows_count);
 
         ColumnString::Offset offset = 0;
-        for (size_t i = 0; i < rows; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             StringRef data = src.getDataAt(i);
 
diff --git a/src/Functions/visibleWidth.cpp b/src/Functions/visibleWidth.cpp
index ebd4a1ff713..3e70418a456 100644
--- a/src/Functions/visibleWidth.cpp
+++ b/src/Functions/visibleWidth.cpp
@@ -66,9 +66,8 @@ public:
     ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
         const auto & src = arguments[0];
-        size_t size = input_rows_count;
 
-        auto res_col = ColumnUInt64::create(size);
+        auto res_col = ColumnUInt64::create(input_rows_count);
         auto & res_data = assert_cast<ColumnUInt64 &>(*res_col).getData();
 
         /// For simplicity reasons, the function is implemented by serializing into temporary buffer.
@@ -76,7 +75,7 @@ public:
         String tmp;
         FormatSettings format_settings;
         auto serialization = src.type->getDefaultSerialization();
-        for (size_t i = 0; i < size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             {
                 WriteBufferFromString out(tmp);
diff --git a/src/Functions/widthBucket.cpp b/src/Functions/widthBucket.cpp
index d007cc968f0..ba24362034f 100644
--- a/src/Functions/widthBucket.cpp
+++ b/src/Functions/widthBucket.cpp
@@ -166,12 +166,12 @@ class FunctionWidthBucket : public IFunction
         result_column->reserve(1);
         auto & result_data = result_column->getData();
 
-        for (const auto row_index : collections::range(0, input_rows_count))
+        for (size_t row = 0; row < input_rows_count; ++row)
         {
-            const auto operand = getValue<Float64>(operands_col_const, operands_vec, row_index);
-            const auto low = getValue<Float64>(lows_col_const, lows_vec, row_index);
-            const auto high = getValue<Float64>(highs_col_const, highs_vec, row_index);
-            const auto count = getValue<TCountType>(counts_col_const, counts_vec, row_index);
+            const auto operand = getValue<Float64>(operands_col_const, operands_vec, row);
+            const auto low = getValue<Float64>(lows_col_const, lows_vec, row);
+            const auto high = getValue<Float64>(highs_col_const, highs_vec, row);
+            const auto count = getValue<TCountType>(counts_col_const, counts_vec, row);
             result_data.push_back(calculate<ResultType>(operand, low, high, count));
         }
 

From 4192bcf2b3e9a144756aea204d8dd5ad7036925e Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 28 Jul 2024 15:59:35 +0200
Subject: [PATCH 0931/1488] Update 01600_parts_states_metrics_long.sh

---
 tests/queries/0_stateless/01600_parts_states_metrics_long.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/queries/0_stateless/01600_parts_states_metrics_long.sh b/tests/queries/0_stateless/01600_parts_states_metrics_long.sh
index fff98a9dd20..fa01ca4fbef 100755
--- a/tests/queries/0_stateless/01600_parts_states_metrics_long.sh
+++ b/tests/queries/0_stateless/01600_parts_states_metrics_long.sh
@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
+# Tags: long
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh

From b503bad51d4f1068b17be967c4e5f9b6393d490e Mon Sep 17 00:00:00 2001
From: flynn <fenglv15@mails.ucas.ac.cn>
Date: Sun, 28 Jul 2024 14:42:14 +0000
Subject: [PATCH 0932/1488] Fix crash of uniq and uniqTheta with empty tuple
 argument

---
 src/AggregateFunctions/UniqVariadicHash.h                 | 3 +++
 tests/queries/0_stateless/03208_uniq_with_empty_tuple.sql | 3 +++
 2 files changed, 6 insertions(+)
 create mode 100644 tests/queries/0_stateless/03208_uniq_with_empty_tuple.sql

diff --git a/src/AggregateFunctions/UniqVariadicHash.h b/src/AggregateFunctions/UniqVariadicHash.h
index 5bb245397d4..279feed8bc6 100644
--- a/src/AggregateFunctions/UniqVariadicHash.h
+++ b/src/AggregateFunctions/UniqVariadicHash.h
@@ -67,6 +67,9 @@ struct UniqVariadicHash<false, true>
 {
     static UInt64 apply(size_t num_args, const IColumn ** columns, size_t row_num)
     {
+        if (!num_args)
+            return 0;
+
         UInt64 hash;
 
         const auto & tuple_columns = assert_cast<const ColumnTuple *>(columns[0])->getColumns();
diff --git a/tests/queries/0_stateless/03208_uniq_with_empty_tuple.sql b/tests/queries/0_stateless/03208_uniq_with_empty_tuple.sql
new file mode 100644
index 00000000000..4c5705261f0
--- /dev/null
+++ b/tests/queries/0_stateless/03208_uniq_with_empty_tuple.sql
@@ -0,0 +1,3 @@
+-- https://github.com/ClickHouse/ClickHouse/issues/67303
+SELECT uniqTheta(tuple());
+SELECT uniq(tuple());

From fd4a733d647b094bb4cf07ed914484d8cd656d61 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 28 Jul 2024 18:01:18 +0200
Subject: [PATCH 0933/1488] Fix bad test `02050_client_profile_events`

---
 .../queries/0_stateless/02050_client_profile_events.sh | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/tests/queries/0_stateless/02050_client_profile_events.sh b/tests/queries/0_stateless/02050_client_profile_events.sh
index 05e48de771d..e20590a6d45 100755
--- a/tests/queries/0_stateless/02050_client_profile_events.sh
+++ b/tests/queries/0_stateless/02050_client_profile_events.sh
@@ -21,11 +21,17 @@ echo 'regression test for overlap profile events snapshots between queries (clic
 $CLICKHOUSE_LOCAL --print-profile-events --profile-events-delay-ms=-1 -n -q 'select 1; select 1' |& grep -F -o '[ 0 ] SelectedRows: 1 (increment)'
 
 echo 'print everything'
-profile_events="$($CLICKHOUSE_CLIENT --max_block_size 1 --print-profile-events -q 'select sleep(1) from numbers(2) format Null' |& grep -c 'SelectedRows')"
+profile_events="$(
+  $CLICKHOUSE_CLIENT --max_block_size 1 --print-profile-events -q 'select sleep(1) from numbers(2) format Null' |&
+  $CLICKHOUSE_LOCAL --input-format LineAsString --query "SELECT sum(extract(line, 'SelectedRows: (\\d+)')::UInt64) FROM table WHERE line LIKE '%SelectedRows: %'"
+)"
 test "$profile_events" -gt 1 && echo OK || echo "FAIL ($profile_events)"
 
 echo 'print each 100 ms'
-profile_events="$($CLICKHOUSE_CLIENT --max_block_size 1 --print-profile-events --profile-events-delay-ms=100 -q 'select sleep(0.2) from numbers(10) format Null' |& grep -c 'SelectedRows')"
+profile_events="$(
+  $CLICKHOUSE_CLIENT --max_block_size 1 --print-profile-events --profile-events-delay-ms=100 -q 'select sleep(0.2) from numbers(10) format Null' |&
+  $CLICKHOUSE_LOCAL --input-format LineAsString --query "SELECT sum(extract(line, 'SelectedRows: (\\d+)')::UInt64) FROM table WHERE line LIKE '%SelectedRows: %'"
+)"
 test "$profile_events" -gt 1 && echo OK || echo "FAIL ($profile_events)"
 
 echo 'check that ProfileEvents is new for each query'

From f2b8431867e3f1147d3ed942a3723e9389d3bac2 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 28 Jul 2024 20:23:00 +0200
Subject: [PATCH 0934/1488] I do not want to think about this code

---
 src/Storages/MergeTree/MergeTreeData.cpp | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index 30a4a7caa0f..e5116afa3fb 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -4405,25 +4405,25 @@ bool MergeTreeData::tryRemovePartImmediately(DataPartPtr && part)
 
 size_t MergeTreeData::getTotalActiveSizeInBytes() const
 {
-    return total_active_size_bytes.load(std::memory_order_acquire);
+    return total_active_size_bytes.load();
 }
 
 
 size_t MergeTreeData::getTotalActiveSizeInRows() const
 {
-    return total_active_size_rows.load(std::memory_order_acquire);
+    return total_active_size_rows.load();
 }
 
 
 size_t MergeTreeData::getActivePartsCount() const
 {
-    return total_active_size_parts.load(std::memory_order_acquire);
+    return total_active_size_parts.load();
 }
 
 
 size_t MergeTreeData::getOutdatedPartsCount() const
 {
-    return total_outdated_parts_count.load(std::memory_order_relaxed);
+    return total_outdated_parts_count.load();
 }
 
 size_t MergeTreeData::getNumberOfOutdatedPartsWithExpiredRemovalTime() const
@@ -8184,16 +8184,16 @@ void MergeTreeData::removePartContributionToDataVolume(const DataPartPtr & part)
 
 void MergeTreeData::increaseDataVolume(ssize_t bytes, ssize_t rows, ssize_t parts)
 {
-    total_active_size_bytes.fetch_add(bytes, std::memory_order_acq_rel);
-    total_active_size_rows.fetch_add(rows, std::memory_order_acq_rel);
-    total_active_size_parts.fetch_add(parts, std::memory_order_acq_rel);
+    total_active_size_bytes.fetch_add(bytes);
+    total_active_size_rows.fetch_add(rows);
+    total_active_size_parts.fetch_add(parts);
 }
 
 void MergeTreeData::setDataVolume(size_t bytes, size_t rows, size_t parts)
 {
-    total_active_size_bytes.store(bytes, std::memory_order_release);
-    total_active_size_rows.store(rows, std::memory_order_release);
-    total_active_size_parts.store(parts, std::memory_order_release);
+    total_active_size_bytes.store(bytes);
+    total_active_size_rows.store(rows);
+    total_active_size_parts.store(parts);
 }
 
 bool MergeTreeData::insertQueryIdOrThrow(const String & query_id, size_t max_queries) const

From ce1c965b297c7ea88535cbd53501790bbf1fdbd0 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 28 Jul 2024 20:56:51 +0200
Subject: [PATCH 0935/1488] Fix test `00940_max_parts_in_total`

---
 tests/queries/0_stateless/00940_max_parts_in_total.sql | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/queries/0_stateless/00940_max_parts_in_total.sql b/tests/queries/0_stateless/00940_max_parts_in_total.sql
index e2633c2f0f8..955f593c907 100644
--- a/tests/queries/0_stateless/00940_max_parts_in_total.sql
+++ b/tests/queries/0_stateless/00940_max_parts_in_total.sql
@@ -1,6 +1,7 @@
 drop table if exists max_parts_in_total;
 create table max_parts_in_total (x UInt64) ENGINE = MergeTree PARTITION BY x ORDER BY x SETTINGS max_parts_in_total = 10;
 
+SET max_insert_threads = 1;
 INSERT INTO max_parts_in_total SELECT number FROM numbers(10);
 SELECT 1;
 INSERT INTO max_parts_in_total SELECT 123; -- { serverError TOO_MANY_PARTS }

From 10b3cb92bc0294d12da77a9aac40e56dd1b8f6ab Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 28 Jul 2024 21:14:47 +0200
Subject: [PATCH 0936/1488] Tests for Kafka cannot run in parallel

---
 tests/integration/parallel_skip.json | 70 ++++++++++++++++++++++++++--
 1 file changed, 67 insertions(+), 3 deletions(-)

diff --git a/tests/integration/parallel_skip.json b/tests/integration/parallel_skip.json
index 3c3d1b6cc96..75b8d25ec04 100644
--- a/tests/integration/parallel_skip.json
+++ b/tests/integration/parallel_skip.json
@@ -43,7 +43,6 @@
   "test_replicated_database/test.py::test_startup_without_zk",
   "test_replicated_database/test.py::test_sync_replica",
   "test_replicated_fetches_timeouts/test.py::test_no_stall",
-  "test_storage_kafka/test.py::test_kafka_no_holes_when_write_suffix_failed",
   "test_storage_s3/test.py::test_url_reconnect_in_the_middle",
   "test_system_metrics/test.py::test_readonly_metrics",
   "test_system_replicated_fetches/test.py::test_system_replicated_fetches",
@@ -73,7 +72,7 @@
   "test_keeper_map/test.py::test_keeper_map_without_zk",
 
   "test_replicated_merge_tree_wait_on_shutdown/test.py::test_shutdown_and_wait",
-  
+
   "test_http_failover/test.py::test_url_destination_host_with_multiple_addrs",
   "test_http_failover/test.py::test_url_invalid_hostname",
   "test_http_failover/test.py::test_url_ip_change",
@@ -97,5 +96,70 @@
 
   "test_ttl_move/test.py::TestCancelBackgroundMoving::test_cancel_background_moving_on_stop_moves_query",
   "test_ttl_move/test.py::TestCancelBackgroundMoving::test_cancel_background_moving_on_table_detach",
-  "test_ttl_move/test.py::TestCancelBackgroundMoving::test_cancel_background_moving_on_zookeeper_disconnect"
+  "test_ttl_move/test.py::TestCancelBackgroundMoving::test_cancel_background_moving_on_zookeeper_disconnect",
+
+  "test_storage_kafka/test.py::test_kafka_column_types",
+  "test_storage_kafka/test.py::test_kafka_settings_old_syntax",
+  "test_storage_kafka/test.py::test_kafka_settings_new_syntax",
+  "test_storage_kafka/test.py::test_kafka_settings_predefined_macros",
+  "test_storage_kafka/test.py::test_kafka_json_as_string",
+  "test_storage_kafka/test.py::test_kafka_formats",
+  "test_storage_kafka/test.py::test_kafka_issue11308",
+  "test_storage_kafka/test.py::test_kafka_issue4116",
+  "test_storage_kafka/test.py::test_kafka_consumer_hang",
+  "test_storage_kafka/test.py::test_kafka_consumer_hang2",
+  "test_storage_kafka/test.py::test_kafka_read_consumers_in_parallel",
+  "test_storage_kafka/test.py::test_kafka_csv_with_delimiter",
+  "test_storage_kafka/test.py::test_kafka_tsv_with_delimiter",
+  "test_storage_kafka/test.py::test_kafka_select_empty",
+  "test_storage_kafka/test.py::test_kafka_json_without_delimiter",
+  "test_storage_kafka/test.py::test_kafka_protobuf",
+  "test_storage_kafka/test.py::test_kafka_string_field_on_first_position_in_protobuf",
+  "test_storage_kafka/test.py::test_kafka_protobuf_no_delimiter",
+  "test_storage_kafka/test.py::test_kafka_materialized_view",
+  "test_storage_kafka/test.py::test_kafka_recreate_kafka_table",
+  "test_storage_kafka/test.py::test_librdkafka_compression",
+  "test_storage_kafka/test.py::test_kafka_materialized_view_with_subquery",
+  "test_storage_kafka/test.py::test_kafka_many_materialized_views",
+  "test_storage_kafka/test.py::test_kafka_flush_on_big_message",
+  "test_storage_kafka/test.py::test_kafka_virtual_columns",
+  "test_storage_kafka/test.py::test_kafka_virtual_columns_with_materialized_view",
+  "test_storage_kafka/test.py::test_kafka_insert",
+  "test_storage_kafka/test.py::test_kafka_produce_consume",
+  "test_storage_kafka/test.py::test_kafka_commit_on_block_write",
+  "test_storage_kafka/test.py::test_kafka_virtual_columns2",
+  "test_storage_kafka/test.py::test_kafka_producer_consumer_separate_settings",
+  "test_storage_kafka/test.py::test_kafka_produce_key_timestamp",
+  "test_storage_kafka/test.py::test_kafka_insert_avro",
+  "test_storage_kafka/test.py::test_kafka_produce_consume_avro",
+  "test_storage_kafka/test.py::test_kafka_flush_by_time",
+  "test_storage_kafka/test.py::test_kafka_flush_by_block_size",
+  "test_storage_kafka/test.py::test_kafka_lot_of_partitions_partial_commit_of_bulk",
+  "test_storage_kafka/test.py::test_kafka_rebalance",
+  "test_storage_kafka/test.py::test_kafka_no_holes_when_write_suffix_failed",
+  "test_storage_kafka/test.py::test_exception_from_destructor",
+  "test_storage_kafka/test.py::test_commits_of_unprocessed_messages_on_drop",
+  "test_storage_kafka/test.py::test_bad_reschedule",
+  "test_storage_kafka/test.py::test_kafka_duplicates_when_commit_failed",
+  "test_storage_kafka/test.py::test_premature_flush_on_eof",
+  "test_storage_kafka/test.py::test_kafka_unavailable",
+  "test_storage_kafka/test.py::test_kafka_issue14202",
+  "test_storage_kafka/test.py::test_kafka_csv_with_thread_per_consumer",
+  "test_storage_kafka/test.py::test_kafka_engine_put_errors_to_stream",
+  "test_storage_kafka/test.py::test_kafka_engine_put_errors_to_stream_with_random_malformed_json",
+  "test_storage_kafka/test.py::test_kafka_formats_with_broken_message",
+  "test_storage_kafka/test.py::test_kafka_consumer_failover",
+  "test_storage_kafka/test.py::test_kafka_predefined_configuration",
+  "test_storage_kafka/test.py::test_issue26643",
+  "test_storage_kafka/test.py::test_num_consumers_limit",
+  "test_storage_kafka/test.py::test_format_with_prefix_and_suffix",
+  "test_storage_kafka/test.py::test_max_rows_per_message",
+  "test_storage_kafka/test.py::test_row_based_formats",
+  "test_storage_kafka/test.py::test_block_based_formats_1",
+  "test_storage_kafka/test.py::test_block_based_formats_2",
+  "test_storage_kafka/test.py::test_system_kafka_consumers",
+  "test_storage_kafka/test.py::test_system_kafka_consumers_rebalance",
+  "test_storage_kafka/test.py::test_system_kafka_consumers_rebalance_mv",
+  "test_storage_kafka/test.py::test_formats_errors",
+  "test_storage_kafka/test.py::test_multiple_read_in_materialized_views",
 ]

From 1f784d1862d7b2c60ea890019d8663f8ef7858b6 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 28 Jul 2024 21:15:23 +0200
Subject: [PATCH 0937/1488] Tests for Kafka cannot run in parallel

---
 tests/integration/parallel_skip.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/parallel_skip.json b/tests/integration/parallel_skip.json
index 75b8d25ec04..9b970a326c4 100644
--- a/tests/integration/parallel_skip.json
+++ b/tests/integration/parallel_skip.json
@@ -161,5 +161,5 @@
   "test_storage_kafka/test.py::test_system_kafka_consumers_rebalance",
   "test_storage_kafka/test.py::test_system_kafka_consumers_rebalance_mv",
   "test_storage_kafka/test.py::test_formats_errors",
-  "test_storage_kafka/test.py::test_multiple_read_in_materialized_views",
+  "test_storage_kafka/test.py::test_multiple_read_in_materialized_views"
 ]

From 8115bb7a54e91b993e9d287bb049f128fea489f1 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 29 Jul 2024 01:01:03 +0200
Subject: [PATCH 0938/1488] Update 01600_parts_states_metrics_long.sh

---
 tests/queries/0_stateless/01600_parts_states_metrics_long.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/queries/0_stateless/01600_parts_states_metrics_long.sh b/tests/queries/0_stateless/01600_parts_states_metrics_long.sh
index fa01ca4fbef..7215f270a4c 100755
--- a/tests/queries/0_stateless/01600_parts_states_metrics_long.sh
+++ b/tests/queries/0_stateless/01600_parts_states_metrics_long.sh
@@ -16,13 +16,13 @@ verify_sql="SELECT
 # In case of test failure, this code will do infinite loop and timeout.
 verify()
 {
-    for i in {1..100}
+    for i in {1..5000}
     do
         result=$( $CLICKHOUSE_CLIENT --query="$verify_sql" )
         [ "$result" = "1" ] && echo "$result" && break
         sleep 0.1
 
-        if [[ $i -eq 100 ]]
+        if [[ $i -eq 5000 ]]
         then
             $CLICKHOUSE_CLIENT --multiquery "
               SELECT sumIf(value, metric = 'PartsActive'), sumIf(value, metric = 'PartsOutdated') FROM system.metrics;

From 60acbe7720ce31a55e3f846536ad2e1572f541ea Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 29 Jul 2024 01:40:05 +0200
Subject: [PATCH 0939/1488] Fix stack overflow in too deep JSON

---
 src/Core/Settings.h                           |  1 +
 src/Core/SettingsChangesHistory.cpp           |  1 +
 src/Formats/FormatFactory.cpp                 |  1 +
 src/Formats/FormatSettings.h                  |  1 +
 src/IO/ReadHelpers.cpp                        | 25 +++++++++++++------
 .../0_stateless/03213_deep_json.reference     |  0
 tests/queries/0_stateless/03213_deep_json.sql |  5 ++++
 7 files changed, 27 insertions(+), 7 deletions(-)
 create mode 100644 tests/queries/0_stateless/03213_deep_json.reference
 create mode 100644 tests/queries/0_stateless/03213_deep_json.sql

diff --git a/src/Core/Settings.h b/src/Core/Settings.h
index 86e04b2ab4e..6c8e09f38f9 100644
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@@ -1126,6 +1126,7 @@ class IColumn;
     M(Bool, input_format_json_throw_on_bad_escape_sequence, true, "Throw an exception if JSON string contains bad escape sequence in JSON input formats. If disabled, bad escape sequences will remain as is in the data", 0) \
     M(Bool, input_format_json_ignore_unnecessary_fields, true, "Ignore unnecessary fields and not parse them. Enabling this may not throw exceptions on json strings of invalid format or with duplicated fields", 0) \
     M(Bool, input_format_json_case_insensitive_column_matching, false, "Ignore case when matching JSON keys with CH columns", 0) \
+    M(UInt64, input_format_json_max_depth, 1000, "Maximum depth of a field in JSON. This is not a strict limit, it does not have to be applied precisely.", 0) \
     M(Bool, input_format_try_infer_integers, true, "Try to infer integers instead of floats while schema inference in text formats", 0) \
     M(Bool, input_format_try_infer_dates, true, "Try to infer dates from string fields while schema inference in text formats", 0) \
     M(Bool, input_format_try_infer_datetimes, true, "Try to infer datetimes from string fields while schema inference in text formats", 0) \
diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp
index 9faf77e9087..b340091798c 100644
--- a/src/Core/SettingsChangesHistory.cpp
+++ b/src/Core/SettingsChangesHistory.cpp
@@ -57,6 +57,7 @@ String ClickHouseVersion::toString() const
 /// Note: please check if the key already exists to prevent duplicate entries.
 static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory::SettingsChanges>> settings_changes_history_initializer =
 {
+    {"24.8", {{"input_format_json_max_depth", 1000000, 1000, "It was unlimited in previous versions, but that was unsafe."}}},
     {"24.7", {{"output_format_parquet_write_page_index", false, true, "Add a possibility to write page index into parquet files."},
               {"output_format_binary_encode_types_in_binary_format", false, false, "Added new setting to allow to write type names in binary format in RowBinaryWithNamesAndTypes output format"},
               {"input_format_binary_decode_types_in_binary_format", false, false, "Added new setting to allow to read type names in binary format in RowBinaryWithNamesAndTypes input format"},
diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp
index 501343d2ca9..695f15e3fdb 100644
--- a/src/Formats/FormatFactory.cpp
+++ b/src/Formats/FormatFactory.cpp
@@ -123,6 +123,7 @@ FormatSettings getFormatSettings(const ContextPtr & context, const Settings & se
     format_settings.import_nested_json = settings.input_format_import_nested_json;
     format_settings.input_allow_errors_num = settings.input_format_allow_errors_num;
     format_settings.input_allow_errors_ratio = settings.input_format_allow_errors_ratio;
+    format_settings.json.max_depth = settings.input_format_json_max_depth;
     format_settings.json.array_of_rows = settings.output_format_json_array_of_rows;
     format_settings.json.escape_forward_slashes = settings.output_format_json_escape_forward_slashes;
     format_settings.json.write_named_tuples_as_objects = settings.output_format_json_named_tuples_as_objects;
diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h
index 72d1515b7e1..32676970011 100644
--- a/src/Formats/FormatSettings.h
+++ b/src/Formats/FormatSettings.h
@@ -205,6 +205,7 @@ struct FormatSettings
 
     struct JSON
     {
+        size_t max_depth = 1000;
         bool array_of_rows = false;
         bool quote_64bit_integers = true;
         bool quote_64bit_floats = false;
diff --git a/src/IO/ReadHelpers.cpp b/src/IO/ReadHelpers.cpp
index c771fced73a..bc9c7bb110c 100644
--- a/src/IO/ReadHelpers.cpp
+++ b/src/IO/ReadHelpers.cpp
@@ -3,6 +3,7 @@
 #include <Common/PODArray.h>
 #include <Common/StringUtils.h>
 #include <Common/memcpySmall.h>
+#include <Common/checkStackSize.h>
 #include <Formats/FormatSettings.h>
 #include <IO/WriteBufferFromString.h>
 #include <IO/BufferWithOwnMemory.h>
@@ -1494,10 +1495,20 @@ template bool readDateTimeTextFallback<bool, true>(time_t &, ReadBuffer &, const
 
 
 template <typename ReturnType>
-ReturnType skipJSONFieldImpl(ReadBuffer & buf, StringRef name_of_field, const FormatSettings::JSON & settings)
+ReturnType skipJSONFieldImpl(ReadBuffer & buf, StringRef name_of_field, const FormatSettings::JSON & settings, size_t current_depth)
 {
     static constexpr bool throw_exception = std::is_same_v<ReturnType, void>;
 
+    if (unlikely(current_depth > settings.max_depth))
+    {
+        if constexpr (throw_exception)
+            throw Exception(ErrorCodes::INCORRECT_DATA, "JSON is too deep for key '{}'", name_of_field.toString());
+        return ReturnType(false);
+    }
+
+    if (unlikely(current_depth > 0 && current_depth % 1024 == 0))
+        checkStackSize();
+
     if (buf.eof())
     {
         if constexpr (throw_exception)
@@ -1560,8 +1571,8 @@ ReturnType skipJSONFieldImpl(ReadBuffer & buf, StringRef name_of_field, const Fo
         while (true)
         {
             if constexpr (throw_exception)
-                skipJSONFieldImpl<ReturnType>(buf, name_of_field, settings);
-            else if (!skipJSONFieldImpl<ReturnType>(buf, name_of_field, settings))
+                skipJSONFieldImpl<ReturnType>(buf, name_of_field, settings, current_depth + 1);
+            else if (!skipJSONFieldImpl<ReturnType>(buf, name_of_field, settings, current_depth + 1))
                 return ReturnType(false);
 
             skipWhitespaceIfAny(buf);
@@ -1619,8 +1630,8 @@ ReturnType skipJSONFieldImpl(ReadBuffer & buf, StringRef name_of_field, const Fo
             skipWhitespaceIfAny(buf);
 
             if constexpr (throw_exception)
-                skipJSONFieldImpl<ReturnType>(buf, name_of_field, settings);
-            else if (!skipJSONFieldImpl<ReturnType>(buf, name_of_field, settings))
+                skipJSONFieldImpl<ReturnType>(buf, name_of_field, settings, current_depth + 1);
+            else if (!skipJSONFieldImpl<ReturnType>(buf, name_of_field, settings, current_depth + 1))
                 return ReturnType(false);
 
             skipWhitespaceIfAny(buf);
@@ -1659,12 +1670,12 @@ ReturnType skipJSONFieldImpl(ReadBuffer & buf, StringRef name_of_field, const Fo
 
 void skipJSONField(ReadBuffer & buf, StringRef name_of_field, const FormatSettings::JSON & settings)
 {
-    skipJSONFieldImpl<void>(buf, name_of_field, settings);
+    skipJSONFieldImpl<void>(buf, name_of_field, settings, 0);
 }
 
 bool trySkipJSONField(ReadBuffer & buf, StringRef name_of_field, const FormatSettings::JSON & settings)
 {
-    return skipJSONFieldImpl<bool>(buf, name_of_field, settings);
+    return skipJSONFieldImpl<bool>(buf, name_of_field, settings, 0);
 }
 
 
diff --git a/tests/queries/0_stateless/03213_deep_json.reference b/tests/queries/0_stateless/03213_deep_json.reference
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/queries/0_stateless/03213_deep_json.sql b/tests/queries/0_stateless/03213_deep_json.sql
new file mode 100644
index 00000000000..4f79d99eb26
--- /dev/null
+++ b/tests/queries/0_stateless/03213_deep_json.sql
@@ -0,0 +1,5 @@
+-- The default limit works.
+SELECT * FROM format("JSONCompactEachRow", 'x UInt32, y UInt32', REPEAT('[1,1,', 100000)) SETTINGS input_format_json_compact_allow_variable_number_of_columns = 1; -- { serverError TOO_DEEP_RECURSION }
+-- Even if we relax the limit, it is also safe.
+SET input_format_json_max_depth = 100000;
+SELECT * FROM format("JSONCompactEachRow", 'x UInt32, y UInt32', REPEAT('[1,1,', 100000)) SETTINGS input_format_json_compact_allow_variable_number_of_columns = 1; -- { serverError TOO_DEEP_RECURSION }

From 86f8fa5efbfd0c7bfe3f00ff37b85b2efe17cfe5 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 29 Jul 2024 01:40:16 +0200
Subject: [PATCH 0940/1488] Fix stack overflow in too deep JSON

---
 src/IO/ReadHelpers.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/IO/ReadHelpers.cpp b/src/IO/ReadHelpers.cpp
index bc9c7bb110c..9559462e62b 100644
--- a/src/IO/ReadHelpers.cpp
+++ b/src/IO/ReadHelpers.cpp
@@ -10,7 +10,6 @@
 #include <IO/PeekableReadBuffer.h>
 #include <IO/readFloatText.h>
 #include <IO/Operators.h>
-#include <base/find_symbols.h>
 #include <cstdlib>
 #include <bit>
 
@@ -40,6 +39,7 @@ namespace ErrorCodes
     extern const int ATTEMPT_TO_READ_AFTER_EOF;
     extern const int LOGICAL_ERROR;
     extern const int BAD_ARGUMENTS;
+    extern const int TOO_DEEP_RECURSION;
 }
 
 template <size_t num_bytes, typename IteratorSrc, typename IteratorDst>
@@ -1502,7 +1502,7 @@ ReturnType skipJSONFieldImpl(ReadBuffer & buf, StringRef name_of_field, const Fo
     if (unlikely(current_depth > settings.max_depth))
     {
         if constexpr (throw_exception)
-            throw Exception(ErrorCodes::INCORRECT_DATA, "JSON is too deep for key '{}'", name_of_field.toString());
+            throw Exception(ErrorCodes::TOO_DEEP_RECURSION, "JSON is too deep for key '{}'", name_of_field.toString());
         return ReturnType(false);
     }
 

From 6e31e813b87aafe4b97b18c5cbbce9483402bde1 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 29 Jul 2024 01:45:29 +0200
Subject: [PATCH 0941/1488] Add a test for #66285

---
 tests/queries/0_stateless/03213_distributed_analyzer.reference | 1 +
 tests/queries/0_stateless/03213_distributed_analyzer.sql       | 3 +++
 2 files changed, 4 insertions(+)
 create mode 100644 tests/queries/0_stateless/03213_distributed_analyzer.reference
 create mode 100644 tests/queries/0_stateless/03213_distributed_analyzer.sql

diff --git a/tests/queries/0_stateless/03213_distributed_analyzer.reference b/tests/queries/0_stateless/03213_distributed_analyzer.reference
new file mode 100644
index 00000000000..9d63c0a7a5e
--- /dev/null
+++ b/tests/queries/0_stateless/03213_distributed_analyzer.reference
@@ -0,0 +1 @@
+['digraph','{','  rankdir="LR";','  { node [shape = rect]','        n1 [label="Remote"];','  }','}','digraph','{','  rankdir="LR";','  { node [shape = rect]','        n1 [label="Remote"];','  }','}']
diff --git a/tests/queries/0_stateless/03213_distributed_analyzer.sql b/tests/queries/0_stateless/03213_distributed_analyzer.sql
new file mode 100644
index 00000000000..ae0f13f08f0
--- /dev/null
+++ b/tests/queries/0_stateless/03213_distributed_analyzer.sql
@@ -0,0 +1,3 @@
+-- This triggered a nullptr dereference due to the confusion between old and new analyzers:
+SELECT sum(*) FROM remote('127.0.0.4', currentDatabase(), viewExplain('EXPLAIN PIPELINE', 'graph = 1', (SELECT * FROM remote('127.0.0.4', system, one)))); -- { serverError UNKNOWN_FUNCTION }
+SELECT groupArray(*) FROM cluster(test_cluster_two_shards, viewExplain('EXPLAIN PIPELINE', 'graph = 1', (SELECT * FROM remote('127.0.0.4', system, one))));

From a1ab412fcbbf7867ac833cf1cc0261c65b5a3195 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 29 Jul 2024 02:00:30 +0200
Subject: [PATCH 0942/1488] Prevent misuse of random number distributions

---
 src/Functions/randDistribution.cpp | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/src/Functions/randDistribution.cpp b/src/Functions/randDistribution.cpp
index 6a3dac748c1..dc2ddf2ef24 100644
--- a/src/Functions/randDistribution.cpp
+++ b/src/Functions/randDistribution.cpp
@@ -93,6 +93,9 @@ struct ChiSquaredDistribution
 
     static void generate(Float64 degree_of_freedom, ColumnFloat64::Container & container)
     {
+        if (degree_of_freedom <= 0)
+            throw Exception(ErrorCodes::BAD_ARGUMENTS, "Argument (degrees of freedom) of function {} should be greater than zero", getName());
+
         auto distribution = std::chi_squared_distribution<>(degree_of_freedom);
         for (auto & elem : container)
             elem = distribution(thread_local_rng);
@@ -107,6 +110,9 @@ struct StudentTDistribution
 
     static void generate(Float64 degree_of_freedom, ColumnFloat64::Container & container)
     {
+        if (degree_of_freedom <= 0)
+            throw Exception(ErrorCodes::BAD_ARGUMENTS, "Argument (degrees of freedom) of function {} should be greater than zero", getName());
+
         auto distribution = std::student_t_distribution<>(degree_of_freedom);
         for (auto & elem : container)
             elem = distribution(thread_local_rng);
@@ -121,6 +127,9 @@ struct FisherFDistribution
 
     static void generate(Float64 d1, Float64 d2, ColumnFloat64::Container & container)
     {
+        if (d1 <= 0 || d2 <= 0)
+            throw Exception(ErrorCodes::BAD_ARGUMENTS, "Argument (degrees of freedom) of function {} should be greater than zero", getName());
+
         auto distribution = std::fisher_f_distribution<>(d1, d2);
         for (auto & elem : container)
             elem = distribution(thread_local_rng);
@@ -300,7 +309,7 @@ public:
             }
             else
             {
-                throw Exception(ErrorCodes::BAD_ARGUMENTS, "More than two argument specified for function {}", getName());
+                throw Exception(ErrorCodes::BAD_ARGUMENTS, "More than two arguments specified for function {}", getName());
             }
 
             return res_column;

From 5216a3d4f260e770b8bf780fbba11dfe822bbd4f Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 29 Jul 2024 02:05:25 +0200
Subject: [PATCH 0943/1488] Add a test

---
 tests/queries/0_stateless/03213_rand_dos.reference | 0
 tests/queries/0_stateless/03213_rand_dos.sql       | 5 +++++
 2 files changed, 5 insertions(+)
 create mode 100644 tests/queries/0_stateless/03213_rand_dos.reference
 create mode 100644 tests/queries/0_stateless/03213_rand_dos.sql

diff --git a/tests/queries/0_stateless/03213_rand_dos.reference b/tests/queries/0_stateless/03213_rand_dos.reference
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/queries/0_stateless/03213_rand_dos.sql b/tests/queries/0_stateless/03213_rand_dos.sql
new file mode 100644
index 00000000000..1250995bfe5
--- /dev/null
+++ b/tests/queries/0_stateless/03213_rand_dos.sql
@@ -0,0 +1,5 @@
+SELECT randChiSquared(-0.0000001); -- { serverError BAD_ARGUMENTS }
+SELECT randChiSquared(-0.0); -- { serverError BAD_ARGUMENTS }
+SELECT randStudentT(-0.); -- { serverError BAD_ARGUMENTS }
+SELECT randFisherF(-0., 1); -- { serverError BAD_ARGUMENTS }
+SELECT randFisherF(1, -0.); -- { serverError BAD_ARGUMENTS }

From 39de59a605ffe6ad63990a219b49781816c4aaa8 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 29 Jul 2024 02:11:55 +0200
Subject: [PATCH 0944/1488] Follow-up for #67301

---
 tests/ci/report.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/ci/report.py b/tests/ci/report.py
index 48e24f9c870..f50ed4c1f85 100644
--- a/tests/ci/report.py
+++ b/tests/ci/report.py
@@ -125,7 +125,7 @@ html {{ min-height: 100%; font-family: "DejaVu Sans", "Noto Sans", Arial, sans-s
 h1 {{ margin-left: 10px; }}
 th, td {{ padding: 5px 10px 5px 10px; text-align: left; vertical-align: top; line-height: 1.5; border: 1px solid var(--table-border-color); }}
 td {{ background: var(--td-background); }}
-th {{ background: var(--th-background); }}
+th {{ background: var(--th-background); white-space: nowrap; }}
 a {{ color: var(--link-color); text-decoration: none; }}
 a:hover, a:active {{ color: var(--link-hover-color); text-decoration: none; }}
 table {{ box-shadow: 0 8px 25px -5px rgba(0, 0, 0, var(--shadow-intensity)); border-collapse: collapse; border-spacing: 0; }}

From e274aca937bb1b89e6c8646457b046a827093ecb Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 29 Jul 2024 02:40:33 +0200
Subject: [PATCH 0945/1488] Add a test for #61659

---
 tests/queries/0_stateless/03213_array_element_msan.reference | 5 +++++
 tests/queries/0_stateless/03213_array_element_msan.sql       | 2 ++
 2 files changed, 7 insertions(+)
 create mode 100644 tests/queries/0_stateless/03213_array_element_msan.reference
 create mode 100644 tests/queries/0_stateless/03213_array_element_msan.sql

diff --git a/tests/queries/0_stateless/03213_array_element_msan.reference b/tests/queries/0_stateless/03213_array_element_msan.reference
new file mode 100644
index 00000000000..6844e39d0aa
--- /dev/null
+++ b/tests/queries/0_stateless/03213_array_element_msan.reference
@@ -0,0 +1,5 @@
+10
+-- system.settings_profiles
+-- system.settings_profiles
+-- system.settings_profiles
+-- system.settings_profiles
diff --git a/tests/queries/0_stateless/03213_array_element_msan.sql b/tests/queries/0_stateless/03213_array_element_msan.sql
new file mode 100644
index 00000000000..68955856792
--- /dev/null
+++ b/tests/queries/0_stateless/03213_array_element_msan.sql
@@ -0,0 +1,2 @@
+SELECT [[10, 2, 13, 15][toNullable(toLowCardinality(1))]][materialize(toLowCardinality(1))];
+SELECT '-- system.settings_profiles' GROUP BY [[[[[[[[[[10, toNullable(10)][1], [materialize(toLowCardinality(10)), 2][materialize(toLowCardinality(1))]][1]][materialize(materialize(1))], [10, 2, 1][1]][1]][1], 1][toLowCardinality(1)]][1], 1][1], 10][1], [[10, toLowCardinality(2)][toNullable(toLowCardinality(1))]][materialize(toLowCardinality(1))]][1], [[[[10, 2][1]][1]][1], [10, 2][materialize(1)], [[[2][1]][materialize(1)], 2, 1][1], [2, 10, toNullable(1)][1]] WITH CUBE;

From 19ce50f959b65fa1c1fd82b32c5d59252ed72383 Mon Sep 17 00:00:00 2001
From: flynn <fenglv15@mails.ucas.ac.cn>
Date: Mon, 29 Jul 2024 02:36:28 +0000
Subject: [PATCH 0946/1488] Add missing reference file

---
 tests/queries/0_stateless/03208_uniq_with_empty_tuple.reference | 2 ++
 tests/queries/0_stateless/03208_uniq_with_empty_tuple.sql       | 1 +
 2 files changed, 3 insertions(+)
 create mode 100644 tests/queries/0_stateless/03208_uniq_with_empty_tuple.reference

diff --git a/tests/queries/0_stateless/03208_uniq_with_empty_tuple.reference b/tests/queries/0_stateless/03208_uniq_with_empty_tuple.reference
new file mode 100644
index 00000000000..6ed281c757a
--- /dev/null
+++ b/tests/queries/0_stateless/03208_uniq_with_empty_tuple.reference
@@ -0,0 +1,2 @@
+1
+1
diff --git a/tests/queries/0_stateless/03208_uniq_with_empty_tuple.sql b/tests/queries/0_stateless/03208_uniq_with_empty_tuple.sql
index 4c5705261f0..09eeaf6f370 100644
--- a/tests/queries/0_stateless/03208_uniq_with_empty_tuple.sql
+++ b/tests/queries/0_stateless/03208_uniq_with_empty_tuple.sql
@@ -1,3 +1,4 @@
+-- Tags: no-fasttest
 -- https://github.com/ClickHouse/ClickHouse/issues/67303
 SELECT uniqTheta(tuple());
 SELECT uniq(tuple());

From 56353f7d228fc2ac7942cbc974b1027c57071aff Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitlibar@clickhouse.com>
Date: Sun, 28 Jul 2024 03:47:13 +0200
Subject: [PATCH 0947/1488] Remove test test_broken_projections_in_backups_1

---
 .../test_broken_projections/test.py           | 48 -------------------
 1 file changed, 48 deletions(-)

diff --git a/tests/integration/test_broken_projections/test.py b/tests/integration/test_broken_projections/test.py
index e198f98e4c5..2cbbee5563f 100644
--- a/tests/integration/test_broken_projections/test.py
+++ b/tests/integration/test_broken_projections/test.py
@@ -450,54 +450,6 @@ def get_random_string(string_length=8):
     return "".join((random.choice(alphabet) for _ in range(string_length)))
 
 
-def test_broken_projections_in_backups_1(cluster):
-    node = cluster.instances["node"]
-
-    table_name = "test4"
-    create_table(node, table_name, 1, aggressive_merge=False, data_prefix=table_name)
-
-    node.query("SYSTEM STOP MERGES")
-
-    insert(node, table_name, 0, 5)
-    insert(node, table_name, 5, 5)
-    insert(node, table_name, 10, 5)
-    insert(node, table_name, 15, 5)
-
-    assert ["all_0_0_0", "all_1_1_0", "all_2_2_0", "all_3_3_0"] == get_parts(
-        node, table_name
-    )
-
-    check(node, table_name, 1)
-
-    break_projection(node, table_name, "proj1", "all_2_2_0", "data")
-    check(node, table_name, 0, "proj1", "FILE_DOESNT_EXIST")
-
-    assert "all_2_2_0\tproj1\tNO_FILE_IN_DATA_PART" in get_broken_projections_info(
-        node, table_name
-    )
-
-    backup_name = f"b1-{get_random_string()}"
-    assert "BACKUP_CREATED" in node.query(
-        f"""
-    set backup_restore_keeper_fault_injection_probability=0.0;
-    backup table {table_name} to Disk('backups', '{backup_name}') settings check_projection_parts=false;
-    """
-    )
-
-    assert "RESTORED" in node.query(
-        f"""
-    drop table {table_name} sync;
-    set backup_restore_keeper_fault_injection_probability=0.0;
-    restore table {table_name} from Disk('backups', '{backup_name}');
-    """
-    )
-
-    node.query("SYSTEM STOP MERGES")
-
-    check(node, table_name, 1)
-    assert "" == get_broken_projections_info(node, table_name)
-
-
 def test_broken_projections_in_backups_2(cluster):
     node = cluster.instances["node"]
 

From ca3b21a2be3b798bfad59b2a2bec01f364f8c7e5 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Mon, 29 Jul 2024 06:41:51 +0200
Subject: [PATCH 0948/1488] Faster and less flaky 01246_buffer_flush (by using
 HTTP over clickhouse-client)

clickhouse-client is incredibly slow with sanitizers:

This is two subsequent queries, that should be executed one, after
another:

    2024.07.27 19:18:49.371354 [ 11070 ] {ywjiyfmvjd} <Debug> executeQuery: (from [::1]:47746) (comment: 01246_buffer_flush.sh) insert into buffer_01256 select * from system.numbers limit 5 (stage: Complete)
    2024.07.27 19:18:49.374647 [ 11070 ] {ywjiyfmvjd} <Debug> TCPHandler: Processed in 0.004721391 sec.

    2024.07.27 19:18:54.293488 [ 11070 ] {30d1f5f7-9594-41e3-9d54-18e1ddfe72af} <Debug> executeQuery: (from [::1]:47782) (comment: 01246_buffer_flush.sh) select count() from data_01256 (stage: Complete)

While the delay is 5 seconds between them!

Refs: https://github.com/ClickHouse/ClickHouse/issues/65745
Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 .../queries/0_stateless/01246_buffer_flush.sh | 81 ++++++++++---------
 1 file changed, 43 insertions(+), 38 deletions(-)

diff --git a/tests/queries/0_stateless/01246_buffer_flush.sh b/tests/queries/0_stateless/01246_buffer_flush.sh
index 27c3f01f216..aea91a0bf6b 100755
--- a/tests/queries/0_stateless/01246_buffer_flush.sh
+++ b/tests/queries/0_stateless/01246_buffer_flush.sh
@@ -7,6 +7,16 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
 set -e
 
+function query()
+{
+    local query_id
+    if [[ $1 == --query_id ]]; then
+        query_id="&query_id=$2"
+        shift 2
+    fi
+    ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}$query_id" -d "$*"
+}
+
 function wait_until()
 {
     local expr=$1 && shift
@@ -17,73 +27,68 @@ function wait_until()
 function get_buffer_delay()
 {
     local buffer_insert_id=$1 && shift
-    $CLICKHOUSE_CLIENT -nm -q "
-    SYSTEM FLUSH LOGS;
-    WITH
-        (SELECT event_time_microseconds FROM system.query_log WHERE current_database = currentDatabase() AND type = 'QueryStart' AND query_id = '$buffer_insert_id') AS begin_,
-        (SELECT max(event_time) FROM data_01256) AS end_
-    SELECT dateDiff('seconds', begin_, end_)::UInt64;
+    query "SYSTEM FLUSH LOGS"
+    query "
+        WITH
+            (SELECT event_time_microseconds FROM system.query_log WHERE current_database = '$CLICKHOUSE_DATABASE' AND type = 'QueryStart' AND query_id = '$buffer_insert_id') AS begin_,
+            (SELECT max(event_time) FROM data_01256) AS end_
+        SELECT dateDiff('seconds', begin_, end_)::UInt64
     "
 }
 
-$CLICKHOUSE_CLIENT -nm -q "
-    drop table if exists data_01256;
-    drop table if exists buffer_01256;
-
-    create table data_01256 (key UInt64, event_time DateTime(6) MATERIALIZED now64(6)) Engine=Memory();
-"
+query "drop table if exists data_01256"
+query "drop table if exists buffer_01256"
+query "create table data_01256 (key UInt64, event_time DateTime(6) MATERIALIZED now64(6)) Engine=Memory()"
 
 echo "min"
-$CLICKHOUSE_CLIENT -q "
-    create table buffer_01256 (key UInt64) Engine=Buffer(currentDatabase(), data_01256, 1,
+query "
+    create table buffer_01256 (key UInt64) Engine=Buffer($CLICKHOUSE_DATABASE, data_01256, 1,
         2, 100, /* time */
         4, 100, /* rows */
         1, 1e6  /* bytes */
     )
 "
 min_query_id=$(random_str 10)
-$CLICKHOUSE_CLIENT --query_id="$min_query_id" -q "insert into buffer_01256 select * from system.numbers limit 5"
-$CLICKHOUSE_CLIENT -q "select count() from data_01256"
-wait_until '[[ $($CLICKHOUSE_CLIENT -q "select count() from data_01256") -eq 5 ]]'
+query --query_id "$min_query_id" "insert into buffer_01256 select * from system.numbers limit 5"
+query "select count() from data_01256"
+wait_until '[[ $(query "select count() from data_01256") -eq 5 ]]'
 sec=$(get_buffer_delay "$min_query_id")
 [[ $sec -ge 2 ]] || echo "Buffer flushed too early, min_time=2, flushed after $sec sec"
 [[ $sec -lt 100 ]] || echo "Buffer flushed too late, max_time=100, flushed after $sec sec"
-$CLICKHOUSE_CLIENT -q "select count() from data_01256"
-$CLICKHOUSE_CLIENT -q "drop table buffer_01256"
+query "select count() from data_01256"
+query "drop table buffer_01256"
 
 echo "max"
-$CLICKHOUSE_CLIENT -q "
-    create table buffer_01256 (key UInt64) Engine=Buffer(currentDatabase(), data_01256, 1,
+query "
+    create table buffer_01256 (key UInt64) Engine=Buffer($CLICKHOUSE_DATABASE, data_01256, 1,
         100, 2,   /* time */
         0,   100, /* rows */
         0,   1e6  /* bytes */
-    );
+    )
 "
 max_query_id=$(random_str 10)
-$CLICKHOUSE_CLIENT --query_id="$max_query_id" -q "insert into buffer_01256 select * from system.numbers limit 5"
-$CLICKHOUSE_CLIENT -q "select count() from data_01256"
-wait_until '[[ $($CLICKHOUSE_CLIENT -q "select count() from data_01256") -eq 10 ]]'
+query --query_id "$max_query_id" "insert into buffer_01256 select * from system.numbers limit 5"
+query "select count() from data_01256"
+wait_until '[[ $(query "select count() from data_01256") -eq 10 ]]'
 sec=$(get_buffer_delay "$max_query_id")
 [[ $sec -ge 2 ]] || echo "Buffer flushed too early, max_time=2, flushed after $sec sec"
-$CLICKHOUSE_CLIENT -q "select count() from data_01256"
-$CLICKHOUSE_CLIENT -q "drop table buffer_01256"
+query "select count() from data_01256"
+query "drop table buffer_01256"
 
 echo "direct"
-$CLICKHOUSE_CLIENT -nm -q "
-    create table buffer_01256 (key UInt64) Engine=Buffer(currentDatabase(), data_01256, 1,
+query "
+    create table buffer_01256 (key UInt64) Engine=Buffer($CLICKHOUSE_DATABASE, data_01256, 1,
         100, 100, /* time */
         0,   9,   /* rows */
         0,   1e6  /* bytes */
-    );
-    insert into buffer_01256 select * from system.numbers limit 10;
-    select count() from data_01256;
+    )
 "
+query "insert into buffer_01256 select * from system.numbers limit 10"
+query "select count() from data_01256"
 
 echo "drop"
-$CLICKHOUSE_CLIENT -nm -q "
-    insert into buffer_01256 select * from system.numbers limit 10;
-    drop table if exists buffer_01256;
-    select count() from data_01256;
-"
+query "insert into buffer_01256 select * from system.numbers limit 10"
+query "drop table if exists buffer_01256"
+query "select count() from data_01256"
 
-$CLICKHOUSE_CLIENT -q "drop table data_01256"
+query "drop table data_01256"

From e8c95bb130161bfd02e48b79d00976c3e51cf05b Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Fri, 7 Jun 2024 07:16:39 +0200
Subject: [PATCH 0949/1488] Fix 02444_async_broken_outdated_part_loading
 flakiness

It uses already removed path to detect the absolute path:

    2024.06.06 22:36:55.845743 [ 1055 ] {} <Debug> test_858tcd7j.rmt (8d64bafa-bedf-4015-9673-8911de129a8f): Removing 2 parts from memory: Parts: [all_0_0_0, all_1_1_0]
    2024.06.06 22:36:56.608589 [ 2065 ] {41c3a91d-2ee5-4006-bb79-92e7e8f005bb} <Debug> executeQuery: (from [::1]:48792) (comment: 02444_async_broken_outdated_part_loading.sh) select path from system.parts where database='test_858tcd7j' and table='rmt' and name='all_1_1_0' (stage: Complete)

CI: https://s3.amazonaws.com/clickhouse-test-reports/64856/d10027cc3b7737c524f4cfce262d46753fd03036/stateless_tests__debug__[5_5].html
Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 .../0_stateless/02444_async_broken_outdated_part_loading.sh     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/02444_async_broken_outdated_part_loading.sh b/tests/queries/0_stateless/02444_async_broken_outdated_part_loading.sh
index d24c6afcef3..a1313ba16e5 100755
--- a/tests/queries/0_stateless/02444_async_broken_outdated_part_loading.sh
+++ b/tests/queries/0_stateless/02444_async_broken_outdated_part_loading.sh
@@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CURDIR"/../shell_config.sh
 
 $CLICKHOUSE_CLIENT -q "drop table if exists rmt sync;"
-$CLICKHOUSE_CLIENT -q "create table rmt (n int) engine=ReplicatedMergeTree('/test/02444/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/rmt', '1') order by n"
+$CLICKHOUSE_CLIENT -q "create table rmt (n int) engine=ReplicatedMergeTree('/test/02444/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/rmt', '1') order by n settings old_parts_lifetime=600"
 
 $CLICKHOUSE_CLIENT --insert_keeper_fault_injection_probability=0 -q "insert into rmt values (1);"
 $CLICKHOUSE_CLIENT --insert_keeper_fault_injection_probability=0 -q "insert into rmt values (2);"

From bdcedeb3f0fba764c7471bc4a95a6fafdfaeaf7e Mon Sep 17 00:00:00 2001
From: Max K <maxkaynov@gmail.com>
Date: Mon, 29 Jul 2024 09:49:18 +0200
Subject: [PATCH 0950/1488] CI: Add stdout, err for docker pull command

---
 tests/ci/ci_utils.py             |  3 ++-
 tests/ci/docker_images_helper.py | 24 +++++++-----------------
 2 files changed, 9 insertions(+), 18 deletions(-)

diff --git a/tests/ci/ci_utils.py b/tests/ci/ci_utils.py
index d42091fb0da..447aac74c7f 100644
--- a/tests/ci/ci_utils.py
+++ b/tests/ci/ci_utils.py
@@ -212,7 +212,7 @@ class Shell:
         return res.stdout.strip()
 
     @classmethod
-    def run(cls, command, check=False, dry_run=False):
+    def run(cls, command, check=False, dry_run=False, **kwargs):
         if dry_run:
             print(f"Dry-ryn. Would run command [{command}]")
             return ""
@@ -225,6 +225,7 @@ class Shell:
             stderr=subprocess.PIPE,
             text=True,
             check=False,
+            **kwargs,
         )
         if result.returncode == 0:
             print(f"stdout: {result.stdout.strip()}")
diff --git a/tests/ci/docker_images_helper.py b/tests/ci/docker_images_helper.py
index 6ea679e0597..e6869852c4e 100644
--- a/tests/ci/docker_images_helper.py
+++ b/tests/ci/docker_images_helper.py
@@ -3,12 +3,12 @@
 import json
 import logging
 import os
-import subprocess
 from pathlib import Path
 from typing import Any, Dict, List, Optional
 
 from env_helper import ROOT_DIR, DOCKER_TAG
 from get_robot_token import get_parameter_from_ssm
+from ci_utils import Shell
 
 IMAGES_FILE_PATH = Path("docker/images.json")
 
@@ -16,20 +16,14 @@ ImagesDict = Dict[str, dict]
 
 
 def docker_login(relogin: bool = True) -> None:
-    if (
-        relogin
-        or subprocess.run(  # pylint: disable=unexpected-keyword-arg
-            "docker system info | grep --quiet -E 'Username|Registry'",
-            shell=True,
-            check=False,
-        ).returncode
-        == 1
+    if relogin or not Shell.check(
+        "docker system info | grep --quiet -E 'Username|Registry'"
     ):
-        subprocess.check_output(  # pylint: disable=unexpected-keyword-arg
+        Shell.run(  # pylint: disable=unexpected-keyword-arg
             "docker login --username 'robotclickhouse' --password-stdin",
             input=get_parameter_from_ssm("dockerhub_robot_password"),
             encoding="utf-8",
-            shell=True,
+            check=True,
         )
 
 
@@ -48,14 +42,10 @@ class DockerImage:
 def pull_image(image: DockerImage) -> DockerImage:
     try:
         logging.info("Pulling image %s - start", image)
-        subprocess.check_output(
-            f"docker pull {image}",
-            stderr=subprocess.STDOUT,
-            shell=True,
-        )
+        Shell.run(f"docker pull {image}", check=True)
         logging.info("Pulling image %s - done", image)
     except Exception as ex:
-        logging.info("Got execption pulling docker %s", ex)
+        logging.info("Got exception pulling docker %s", ex)
         raise ex
     return image
 

From e05972d9a6df1aa8853626e5f83a65da45e9561a Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Mon, 29 Jul 2024 08:39:18 +0000
Subject: [PATCH 0951/1488] Followup #67290

---
 tests/queries/0_stateless/02911_backup_restore_keeper_map.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/queries/0_stateless/02911_backup_restore_keeper_map.sh b/tests/queries/0_stateless/02911_backup_restore_keeper_map.sh
index 385583acbbe..ee070b40f6f 100755
--- a/tests/queries/0_stateless/02911_backup_restore_keeper_map.sh
+++ b/tests/queries/0_stateless/02911_backup_restore_keeper_map.sh
@@ -17,14 +17,14 @@ $CLICKHOUSE_CLIENT -nm -q "
 while true
 do
     $CLICKHOUSE_CLIENT -nm -q "INSERT INTO $database_name.02911_backup_restore_keeper_map2 SELECT number, 'test' || toString(number) FROM system.numbers LIMIT 5000;
-    " | grep -q "KEEPER_EXCEPTION" && sleep 1 && continue
+    " 2>&1 | grep -q "KEEPER_EXCEPTION" && sleep 1 && continue
     break
 done
 
 while true
 do
     $CLICKHOUSE_CLIENT -nm -q "INSERT INTO $database_name.02911_backup_restore_keeper_map3 SELECT number, 'test' || toString(number) FROM system.numbers LIMIT 3000;
-    " | grep -q "KEEPER_EXCEPTION" && sleep 1 && continue
+    "  2>&1 | grep -q "KEEPER_EXCEPTION" && sleep 1 && continue
     break
 done
 

From ac001c0c6fe9bde746ffc4e688b74d81de9c1dc8 Mon Sep 17 00:00:00 2001
From: Andrey Zvonov <azvonov@altinity.com>
Date: Mon, 29 Jul 2024 08:41:23 +0000
Subject: [PATCH 0952/1488] no parallel for kerberized kafka

---
 tests/integration/parallel_skip.json | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/tests/integration/parallel_skip.json b/tests/integration/parallel_skip.json
index 9b970a326c4..99fa626bd1e 100644
--- a/tests/integration/parallel_skip.json
+++ b/tests/integration/parallel_skip.json
@@ -161,5 +161,10 @@
   "test_storage_kafka/test.py::test_system_kafka_consumers_rebalance",
   "test_storage_kafka/test.py::test_system_kafka_consumers_rebalance_mv",
   "test_storage_kafka/test.py::test_formats_errors",
-  "test_storage_kafka/test.py::test_multiple_read_in_materialized_views"
+  "test_storage_kafka/test.py::test_multiple_read_in_materialized_views",
+
+  "test_storage_kerberized_kafka/test.py::test_kafka_json_as_string",
+  "test_storage_kerberized_kafka/test.py::test_kafka_json_as_string_request_new_ticket_after_expiration",
+  "test_storage_kerberized_kafka/test.py::test_kafka_json_as_string_no_kdc",
+  "test_storage_kerberized_kafka/test.py::test_kafka_config_from_sql_named_collection"
 ]

From 1ae3a1b1256a2ffda2e7d549189b60a0caf35554 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Mon, 29 Jul 2024 09:03:43 +0000
Subject: [PATCH 0953/1488] Fix build

---
 src/Functions/MultiMatchAllIndicesImpl.h | 2 ++
 src/Functions/MultiMatchAnyImpl.h        | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/Functions/MultiMatchAllIndicesImpl.h b/src/Functions/MultiMatchAllIndicesImpl.h
index 17232bbc366..3aeac808880 100644
--- a/src/Functions/MultiMatchAllIndicesImpl.h
+++ b/src/Functions/MultiMatchAllIndicesImpl.h
@@ -147,6 +147,7 @@ struct MultiMatchAllIndicesImpl
         (void)max_hyperscan_regexp_length;
         (void)max_hyperscan_regexp_total_length;
         (void)reject_expensive_hyperscan_regexps;
+        (void)input_rows_count;
         throw Exception(ErrorCodes::NOT_IMPLEMENTED, "multi-search all indices is not implemented when vectorscan is off");
 #endif // USE_VECTORSCAN
     }
@@ -274,6 +275,7 @@ struct MultiMatchAllIndicesImpl
         (void)max_hyperscan_regexp_length;
         (void)max_hyperscan_regexp_total_length;
         (void)reject_expensive_hyperscan_regexps;
+        (void)input_rows_count;
         throw Exception(ErrorCodes::NOT_IMPLEMENTED, "multi-search all indices is not implemented when vectorscan is off");
 #endif // USE_VECTORSCAN
     }
diff --git a/src/Functions/MultiMatchAnyImpl.h b/src/Functions/MultiMatchAnyImpl.h
index b5872579ebb..03b17321eea 100644
--- a/src/Functions/MultiMatchAnyImpl.h
+++ b/src/Functions/MultiMatchAnyImpl.h
@@ -165,7 +165,7 @@ struct MultiMatchAnyImpl
         memset(accum.data(), 0, accum.size());
         for (size_t j = 0; j < input_rows_count; ++j)
         {
-            MatchImpl<Name, MatchTraits::Syntax::Re2, MatchTraits::Case::Sensitive, MatchTraits::Result::DontNegate>::vectorConstant(haystack_data, haystack_offsets, String(needles[j].data(), needles[j].size()), nullptr, accum, nullptr);
+            MatchImpl<Name, MatchTraits::Syntax::Re2, MatchTraits::Case::Sensitive, MatchTraits::Result::DontNegate>::vectorConstant(haystack_data, haystack_offsets, String(needles[j].data(), needles[j].size()), nullptr, accum, nullptr, input_rows_count);
             for (size_t i = 0; i < res.size(); ++i)
             {
                 if constexpr (FindAny)

From 906d709d84757220355dd1e7b0b24c793a185f96 Mon Sep 17 00:00:00 2001
From: wxybear <wangxingyu0927@outlook.com>
Date: Fri, 26 Jul 2024 01:40:37 +0800
Subject: [PATCH 0954/1488] clickhouse-client/local: make multi-query mode the
 default

---
 programs/client/Client.cpp                    |  14 ---
 programs/local/LocalServer.cpp                |  10 --
 src/Client/ClientBase.cpp                     | 102 ++++++++++--------
 src/Client/ClientBase.h                       |   3 -
 src/Core/Settings.h                           |   1 -
 src/Formats/FormatFactory.cpp                 |   1 -
 .../Formats/Impl/ValuesBlockInputFormat.cpp   |   2 -
 .../00366_multi_statements.reference          |   6 +-
 .../0_stateless/00366_multi_statements.sh     |   4 +-
 .../0_stateless/00499_json_enum_insert.sql    |   3 +
 .../0_stateless/00517_date_parsing.sql        |   1 +
 .../00560_float_leading_plus_in_exponent.sql  |   2 +
 .../0_stateless/00700_decimal_formats.sql     |  16 +--
 .../00760_insert_json_with_defaults.sql       |   9 ++
 .../0_stateless/00861_decimal_quoted_csv.sql  |   9 +-
 tests/queries/0_stateless/00945_ml_test.sql   |   2 +-
 tests/queries/0_stateless/00946_ml_test.sql   |   2 +-
 .../00976_system_stop_ttl_merges.sql          |   2 +-
 .../0_stateless/01034_JSONCompactEachRow.sql  |  13 +++
 .../01060_defaults_all_columns.sql            |   1 +
 ..._json_each_row_data_in_square_brackets.sql |   4 +
 .../01179_insert_values_semicolon.expect      |   4 +-
 .../0_stateless/01231_operator_null_in.sql    |   2 +-
 ..._block_size_rows_for_materialized_views.sh |   2 +-
 .../0_stateless/01299_alter_merge_tree.sql    |   4 +-
 .../0_stateless/01324_insert_tsv_raw.sql      |   1 -
 ...6_low_cardinality_fixed_string_default.sql |   1 +
 .../01448_json_compact_strings_each_row.sql   |  13 +++
 ...nput_format_csv_enum_as_number_setting.sql |   1 +
 ...01514_input_format_json_enum_as_number.sql |   1 +
 ...nput_format_tsv_enum_as_number_setting.sql |   2 +
 .../01558_enum_as_num_in_tsv_csv_input.sql    |   8 ++
 .../01564_test_hint_woes.reference            |   2 +-
 .../queries/0_stateless/01825_type_json_1.sql |   5 +-
 .../0_stateless/01825_type_json_10.sql        |   1 +
 .../0_stateless/01825_type_json_17.sql        |   4 +
 .../0_stateless/01825_type_json_18.sql        |   2 +
 .../queries/0_stateless/01825_type_json_9.sql |   1 +
 .../0_stateless/01825_type_json_in_array.sql  |   2 +
 .../01825_type_json_insert_select.sql         |   8 +-
 ...json_as_string_data_in_square_brackets.sql |   3 +
 .../0_stateless/01943_query_id_check.sql      |   2 +-
 .../0_stateless/02002_parse_map_int_key.sql   |   1 -
 .../02105_backslash_letter_commands.expect    |   1 -
 tests/queries/0_stateless/02114_bool_type.sql |   9 ++
 .../0_stateless/02184_ipv6_select_parsing.sql |   3 +
 .../0_stateless/02192_comment_error.reference |   4 +-
 .../0_stateless/02192_comment_error.sh        |   2 +
 .../02193_async_insert_tcp_client_1.sql       |   4 +-
 ...02234_clickhouse_local_test_mode.reference |   2 +-
 .../02234_clickhouse_local_test_mode.sh       |   4 +-
 .../0_stateless/02242_subcolumns_sizes.sql    |   1 +
 ...ert_select_from_input_schema_inference.sql |   1 +
 .../02267_jsonlines_ndjson_format.sql         |   1 +
 .../0_stateless/02366_kql_summarize.sql       |   2 +-
 .../02387_parse_date_as_datetime.sql          |   6 ++
 .../02435_rollback_cancelled_queries.sh       |   2 +-
 ...458_use_structure_from_insertion_table.sql |  11 +-
 .../02495_analyzer_storage_join.sql           |   2 +-
 .../02504_regexp_dictionary_table_source.sql  |  23 ++--
 ...e_with_nonconst_needle_and_replacement.sql |   4 +-
 .../02553_type_json_attach_partition.sql      |   1 +
 .../02726_async_insert_flush_queue.sql        |   8 +-
 .../02751_multiquery_with_argument.reference  |   2 -
 .../02751_multiquery_with_argument.sh         |   5 +-
 .../02771_multiple_query_arguments.reference  |  10 +-
 .../02771_multiple_query_arguments.sh         |   8 +-
 ...2875_clickhouse_local_multiquery.reference |   6 +-
 .../02875_clickhouse_local_multiquery.sh      |   9 +-
 ...alues_allow_data_after_semicolon.reference |   6 --
 ...ormat_values_allow_data_after_semicolon.sh |  11 +-
 ...900_window_function_with_sparse_column.sql |   2 +-
 .../02932_refreshable_materialized_views.sh   |   3 +-
 .../02950_dictionary_short_circuit.sql        |  12 +--
 .../02974_analyzer_array_join_subcolumn.sql   |   2 +-
 .../03005_input_function_in_join.sql          |   2 +-
 .../03156_default_multiquery_split.reference  |  10 ++
 .../03156_default_multiquery_split.sh         |  59 ++++++++++
 78 files changed, 335 insertions(+), 180 deletions(-)
 create mode 100644 tests/queries/0_stateless/03156_default_multiquery_split.reference
 create mode 100755 tests/queries/0_stateless/03156_default_multiquery_split.sh

diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp
index 3e613532f3a..1d99d223ee9 100644
--- a/programs/client/Client.cpp
+++ b/programs/client/Client.cpp
@@ -1135,8 +1135,6 @@ void Client::processOptions(const OptionsDescription & options_description,
 
     if ((query_fuzzer_runs = options["query-fuzzer-runs"].as<int>()))
     {
-        // Fuzzer implies multiquery.
-        config().setBool("multiquery", true);
         // Ignore errors in parsing queries.
         config().setBool("ignore-error", true);
         ignore_error = true;
@@ -1144,8 +1142,6 @@ void Client::processOptions(const OptionsDescription & options_description,
 
     if ((create_query_fuzzer_runs = options["create-query-fuzzer-runs"].as<int>()))
     {
-        // Fuzzer implies multiquery.
-        config().setBool("multiquery", true);
         // Ignore errors in parsing queries.
         config().setBool("ignore-error", true);
 
@@ -1201,9 +1197,6 @@ void Client::processConfig()
     }
     print_stack_trace = config().getBool("stacktrace", false);
 
-    if (config().has("multiquery"))
-        is_multiquery = true;
-
     pager = config().getString("pager", "");
 
     setDefaultFormatsAndCompressionFromConfiguration();
@@ -1359,13 +1352,6 @@ void Client::readArguments(
                 allow_repeated_settings = true;
             else if (arg == "--allow_merge_tree_settings")
                 allow_merge_tree_settings = true;
-            else if (arg == "--multiquery" && (arg_num + 1) < argc && !std::string_view(argv[arg_num + 1]).starts_with('-'))
-            {
-                /// Transform the abbreviated syntax '--multiquery <SQL>' into the full syntax '--multiquery -q <SQL>'
-                ++arg_num;
-                arg = argv[arg_num];
-                addMultiquery(arg, common_arguments);
-            }
             else if (arg == "--password" && ((arg_num + 1) >= argc || std::string_view(argv[arg_num + 1]).starts_with('-')))
             {
                 common_arguments.emplace_back(arg);
diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp
index ce0e179939d..88d5a0253d1 100644
--- a/programs/local/LocalServer.cpp
+++ b/programs/local/LocalServer.cpp
@@ -569,9 +569,6 @@ void LocalServer::processConfig()
     if (!queries.empty() && getClientConfiguration().has("queries-file"))
         throw Exception(ErrorCodes::BAD_ARGUMENTS, "Options '--query' and '--queries-file' cannot be specified at the same time");
 
-    if (getClientConfiguration().has("multiquery"))
-        is_multiquery = true;
-
     pager = getClientConfiguration().getString("pager", "");
 
     delayed_interactive = getClientConfiguration().has("interactive") && (!queries.empty() || getClientConfiguration().has("queries-file"));
@@ -936,13 +933,6 @@ void LocalServer::readArguments(int argc, char ** argv, Arguments & common_argum
                 query_parameters.emplace(param_continuation.substr(0, equal_pos), param_continuation.substr(equal_pos + 1));
             }
         }
-        else if (arg == "--multiquery" && (arg_num + 1) < argc && !std::string_view(argv[arg_num + 1]).starts_with('-'))
-        {
-            /// Transform the abbreviated syntax '--multiquery <SQL>' into the full syntax '--multiquery -q <SQL>'
-            ++arg_num;
-            arg = argv[arg_num];
-            addMultiquery(arg, common_arguments);
-        }
         else
         {
             common_arguments.emplace_back(arg);
diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp
index 78d8e7e3d6a..91cf737f292 100644
--- a/src/Client/ClientBase.cpp
+++ b/src/Client/ClientBase.cpp
@@ -80,6 +80,8 @@
 
 #include <Common/config_version.h>
 #include "config.h"
+#include <IO/ReadHelpers.h>
+#include <Processors/Formats/Impl/ValuesBlockInputFormat.h>
 
 #if USE_GWP_ASAN
 #    include <Common/GWPAsan.h>
@@ -745,14 +747,6 @@ void ClientBase::adjustSettings()
 
     /// NOTE: Do not forget to set changed=false to avoid sending it to the server (to avoid breakage read only profiles)
 
-    /// In case of multi-query we allow data after semicolon since it will be
-    /// parsed by the client and interpreted as new query
-    if (is_multiquery && !global_context->getSettingsRef().input_format_values_allow_data_after_semicolon.changed)
-    {
-        settings.input_format_values_allow_data_after_semicolon = true;
-        settings.input_format_values_allow_data_after_semicolon.changed = false;
-    }
-
     /// Do not limit pretty format output in case of --pager specified or in case of stdout is not a tty.
     if (!pager.empty() || !stdout_is_a_tty)
     {
@@ -1525,13 +1519,6 @@ void ClientBase::setInsertionTable(const ASTInsertQuery & insert_query)
 }
 
 
-void ClientBase::addMultiquery(std::string_view query, Arguments & common_arguments) const
-{
-    common_arguments.emplace_back("--multiquery");
-    common_arguments.emplace_back("-q");
-    common_arguments.emplace_back(query);
-}
-
 namespace
 {
 bool isStdinNotEmptyAndValid(ReadBufferFromFileDescriptor & std_in)
@@ -2186,23 +2173,48 @@ MultiQueryProcessingStage ClientBase::analyzeMultiQueryText(
         return MultiQueryProcessingStage::PARSING_FAILED;
     }
 
-    // INSERT queries may have the inserted data in the query text
-    // that follow the query itself, e.g. "insert into t format CSV 1;2".
-    // They need special handling. First of all, here we find where the
-    // inserted data ends. In multi-query mode, it is delimited by a
-    // newline.
-    // The VALUES format needs even more handling - we also allow the
-    // data to be delimited by semicolon. This case is handled later by
-    // the format parser itself.
-    // We can't do multiline INSERTs with inline data, because most
-    // row input formats (e.g. TSV) can't tell when the input stops,
-    // unlike VALUES.
+    // INSERT queries may have the inserted data in the query text that follow the query itself, e.g. "insert into t format CSV 1,2". They
+    // need special handling.
+    // - If the INSERT statement FORMAT is VALUES, we use the VALUES format parser to skip the inserted data until we reach the trailing single semicolon.
+    // - Other formats (e.g. FORMAT CSV) are arbitrarily more complex and tricky to parse. For example, we may be unable to distinguish if the semicolon
+    //   is part of the data or ends the statement. In this case, we simply assume that the end of the INSERT statement is determined by \n\n (two newlines).
     auto * insert_ast = parsed_query->as<ASTInsertQuery>();
     const char * query_to_execute_end = this_query_end;
-
     if (insert_ast && insert_ast->data)
     {
-        this_query_end = find_first_symbols<'\n'>(insert_ast->data, all_queries_end);
+        if (insert_ast->format == "Values")
+        {
+            // Invoke the VALUES format parser to skip the inserted data
+            ReadBufferFromMemory data_in(insert_ast->data, all_queries_end - insert_ast->data);
+            skipBOMIfExists(data_in);
+            do
+            {
+                skipWhitespaceIfAny(data_in);
+                if (data_in.eof() || *data_in.position() == ';')
+                    break;
+            }
+            while (ValuesBlockInputFormat::skipToNextRow(&data_in, 1, 0));
+            // Handle the case of a comment followed by a semicolon
+            //   Example: INSERT INTO tab VALUES xx; -- {serverError xx}
+            // If we use this error hint, the next query should not be placed on the same line
+            this_query_end = insert_ast->data + data_in.count();
+            const auto * pos_newline = find_first_symbols<'\n'>(this_query_end, all_queries_end);
+            if (pos_newline != this_query_end)
+            {
+                TestHint hint(String(this_query_end, pos_newline - this_query_end));
+                if (hint.hasClientErrors() || hint.hasServerErrors())
+                    this_query_end = pos_newline;
+            }
+        }
+        else
+        {
+            // Handling of generic formats
+            auto pos_newline = String(insert_ast->data, all_queries_end).find("\n\n");
+            if (pos_newline != std::string::npos)
+                this_query_end = insert_ast->data + pos_newline;
+            else
+                this_query_end = all_queries_end;
+        }
         insert_ast->end = this_query_end;
         query_to_execute_end = isSyncInsertWithData(*insert_ast, client_context) ? insert_ast->data : this_query_end;
     }
@@ -2237,7 +2249,10 @@ bool ClientBase::executeMultiQuery(const String & all_queries_text)
     size_t test_tags_length = getTestTagsLength(all_queries_text);
 
     /// Several queries separated by ';'.
-    /// INSERT data is ended by the end of line, not ';'.
+    /// INSERT data is ended by the empty line (\n\n), not ';'.
+    /// Unnecessary semicolons may cause data to be parsed containing ';'
+    /// e.g. 'insert into xx format csv val;' will insert "val;" instead of "val"
+    ///      'insert into xx format csv val\n;' will insert "val" and ";"
     /// An exception is VALUES format where we also support semicolon in
     /// addition to end of line.
     const char * this_query_begin = all_queries_text.data() + test_tags_length;
@@ -2248,8 +2263,8 @@ bool ClientBase::executeMultiQuery(const String & all_queries_text)
     String query_to_execute;
     ASTPtr parsed_query;
     std::unique_ptr<Exception> current_exception;
-
     size_t retries_count = 0;
+    bool is_first = true;
 
     while (true)
     {
@@ -2258,16 +2273,24 @@ bool ClientBase::executeMultiQuery(const String & all_queries_text)
         switch (stage)
         {
             case MultiQueryProcessingStage::QUERIES_END:
+            {
+                /// Compatible with old version when run interactive, e.g. "", "\ld"
+                if (is_first && is_interactive)
+                    processTextAsSingleQuery(all_queries_text);
+                return true;
+            }
             case MultiQueryProcessingStage::PARSING_FAILED:
             {
                 return true;
             }
             case MultiQueryProcessingStage::CONTINUE_PARSING:
             {
+                is_first = false;
                 continue;
             }
             case MultiQueryProcessingStage::PARSING_EXCEPTION:
             {
+                is_first = false;
                 this_query_end = find_first_symbols<'\n'>(this_query_end, all_queries_end);
 
                 // Try to find test hint for syntax error. We don't know where
@@ -2297,6 +2320,7 @@ bool ClientBase::executeMultiQuery(const String & all_queries_text)
             }
             case MultiQueryProcessingStage::EXECUTE_QUERY:
             {
+                is_first = false;
                 full_query = all_queries_text.substr(this_query_begin - all_queries_text.data(), this_query_end - this_query_begin);
                 if (query_fuzzer_runs)
                 {
@@ -2306,6 +2330,8 @@ bool ClientBase::executeMultiQuery(const String & all_queries_text)
                     this_query_begin = this_query_end;
                     continue;
                 }
+                if (suggest)
+                    updateSuggest(parsed_query);
 
                 // Now we know for sure where the query ends.
                 // Look for the hint in the text of query + insert data + trailing
@@ -2456,14 +2482,6 @@ bool ClientBase::processQueryText(const String & text)
         return processMultiQueryFromFile(file_name);
     }
 
-    if (!is_multiquery)
-    {
-        assert(!query_fuzzer_runs);
-        processTextAsSingleQuery(text);
-
-        return true;
-    }
-
     if (query_fuzzer_runs)
     {
         processWithFuzzing(text);
@@ -2901,9 +2919,9 @@ void ClientBase::init(int argc, char ** argv)
 
         ("config-file,C", po::value<std::string>(), "config-file path")
 
-        ("query,q", po::value<std::vector<std::string>>()->multitoken(), R"(query; can be specified multiple times (--query "SELECT 1" --query "SELECT 2"...))")
+        ("query,q", po::value<std::vector<std::string>>()->multitoken(), R"(Query. Can be specified multiple times (--query "SELECT 1" --query "SELECT 2") or once with multiple comma-separated queries (--query "SELECT 1; SELECT 2;"). In the latter case, INSERT queries with non-VALUE format must be separated by empty lines.)")
         ("queries-file", po::value<std::vector<std::string>>()->multitoken(), "file path with queries to execute; multiple files can be specified (--queries-file file1 file2...)")
-        ("multiquery,n", "If specified, multiple queries separated by semicolons can be listed after --query. For convenience, it is also possible to omit --query and pass the queries directly after --multiquery.")
+        ("multiquery,n", "Obsolete, does nothing")
         ("multiline,m", "If specified, allow multiline queries (do not send the query on Enter)")
         ("database,d", po::value<std::string>(), "database")
         ("query_kind", po::value<std::string>()->default_value("initial_query"), "One of initial_query/secondary_query/no_query")
@@ -2932,7 +2950,7 @@ void ClientBase::init(int argc, char ** argv)
         ("vertical,E", "vertical output format, same as --format=Vertical or FORMAT Vertical or \\G at end of command")
         ("highlight", po::value<bool>()->default_value(true), "enable or disable basic syntax highlight in interactive command line")
 
-        ("ignore-error", "do not stop processing in multiquery mode")
+        ("ignore-error", "do not stop processing when an error occurs")
         ("stacktrace", "print stack traces of exceptions")
         ("hardware-utilization", "print hardware utilization information in progress bar")
         ("print-profile-events", po::value(&profile_events.print)->zero_tokens(), "Printing ProfileEvents packets")
@@ -3025,8 +3043,6 @@ void ClientBase::init(int argc, char ** argv)
         queries_files = options["queries-file"].as<std::vector<std::string>>();
     if (options.count("multiline"))
         getClientConfiguration().setBool("multiline", true);
-    if (options.count("multiquery"))
-        getClientConfiguration().setBool("multiquery", true);
     if (options.count("ignore-error"))
         getClientConfiguration().setBool("ignore-error", true);
     if (options.count("format"))
diff --git a/src/Client/ClientBase.h b/src/Client/ClientBase.h
index e3d5fe8dc97..7689744a373 100644
--- a/src/Client/ClientBase.h
+++ b/src/Client/ClientBase.h
@@ -156,8 +156,6 @@ protected:
 
     void setInsertionTable(const ASTInsertQuery & insert_query);
 
-    void addMultiquery(std::string_view query, Arguments & common_arguments) const;
-
 private:
     void receiveResult(ASTPtr parsed_query, Int32 signals_before_stop, bool partial_result_on_first_cancel);
     bool receiveAndProcessPacket(ASTPtr parsed_query, bool cancelled_);
@@ -229,7 +227,6 @@ protected:
     std::unique_ptr<Poco::Runnable> signal_listener;
 
     bool is_interactive = false; /// Use either interactive line editing interface or batch mode.
-    bool is_multiquery = false;
     bool delayed_interactive = false;
 
     bool echo_queries = false; /// Print queries before execution in batch mode.
diff --git a/src/Core/Settings.h b/src/Core/Settings.h
index 3349662cb78..4fc2034b855 100644
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@@ -1156,7 +1156,6 @@ class IColumn;
     M(Bool, input_format_values_interpret_expressions, true, "For Values format: if the field could not be parsed by streaming parser, run SQL parser and try to interpret it as SQL expression.", 0) \
     M(Bool, input_format_values_deduce_templates_of_expressions, true, "For Values format: if the field could not be parsed by streaming parser, run SQL parser, deduce template of the SQL expression, try to parse all rows using template and then interpret expression for all rows.", 0) \
     M(Bool, input_format_values_accurate_types_of_literals, true, "For Values format: when parsing and interpreting expressions using template, check actual type of literal to avoid possible overflow and precision issues.", 0) \
-    M(Bool, input_format_values_allow_data_after_semicolon, false, "For Values format: allow extra data after semicolon (used by client to interpret comments).", 0) \
     M(Bool, input_format_avro_allow_missing_fields, false, "For Avro/AvroConfluent format: when field is not found in schema use default value instead of error", 0) \
     /** This setting is obsolete and do nothing, left for compatibility reasons. */ \
     M(Bool, input_format_avro_null_as_default, false, "For Avro/AvroConfluent format: insert default in case of null and non Nullable column", 0) \
diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp
index 501343d2ca9..efd467abb37 100644
--- a/src/Formats/FormatFactory.cpp
+++ b/src/Formats/FormatFactory.cpp
@@ -215,7 +215,6 @@ FormatSettings getFormatSettings(const ContextPtr & context, const Settings & se
     format_settings.tsv.allow_variable_number_of_columns = settings.input_format_tsv_allow_variable_number_of_columns;
     format_settings.tsv.crlf_end_of_line_input = settings.input_format_tsv_crlf_end_of_line;
     format_settings.values.accurate_types_of_literals = settings.input_format_values_accurate_types_of_literals;
-    format_settings.values.allow_data_after_semicolon = settings.input_format_values_allow_data_after_semicolon;
     format_settings.values.deduce_templates_of_expressions = settings.input_format_values_deduce_templates_of_expressions;
     format_settings.values.interpret_expressions = settings.input_format_values_interpret_expressions;
     format_settings.values.escape_quote_with_quote = settings.output_format_values_escape_quote_with_quote;
diff --git a/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp b/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp
index c23d717d52f..157b61aa2af 100644
--- a/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp
+++ b/src/Processors/Formats/Impl/ValuesBlockInputFormat.cpp
@@ -625,8 +625,6 @@ void ValuesBlockInputFormat::readSuffix()
         skipWhitespaceIfAny(*buf);
         if (buf->hasUnreadData())
             throw Exception(ErrorCodes::CANNOT_READ_ALL_DATA, "Cannot read data after semicolon");
-        if (!format_settings.values.allow_data_after_semicolon && !buf->eof())
-            throw Exception(ErrorCodes::CANNOT_READ_ALL_DATA, "Cannot read data after semicolon (and input_format_values_allow_data_after_semicolon=0)");
         return;
     }
 
diff --git a/tests/queries/0_stateless/00366_multi_statements.reference b/tests/queries/0_stateless/00366_multi_statements.reference
index 68f9261b937..71ca552522c 100644
--- a/tests/queries/0_stateless/00366_multi_statements.reference
+++ b/tests/queries/0_stateless/00366_multi_statements.reference
@@ -3,8 +3,10 @@
 1
 1
 Syntax error
-Syntax error
-Syntax error
+1
+2
+1
+2
 Syntax error
 Syntax error
 1
diff --git a/tests/queries/0_stateless/00366_multi_statements.sh b/tests/queries/0_stateless/00366_multi_statements.sh
index 9b885bb1b32..0b2e80fe457 100755
--- a/tests/queries/0_stateless/00366_multi_statements.sh
+++ b/tests/queries/0_stateless/00366_multi_statements.sh
@@ -10,8 +10,8 @@ $CLICKHOUSE_CLIENT --query="SELECT 1; "
 $CLICKHOUSE_CLIENT --query="SELECT 1 ; "
 
 $CLICKHOUSE_CLIENT --query="SELECT 1; S" 2>&1 | grep -o 'Syntax error'
-$CLICKHOUSE_CLIENT --query="SELECT 1; SELECT 2" 2>&1 | grep -o 'Syntax error'
-$CLICKHOUSE_CLIENT --query="SELECT 1; SELECT 2;" 2>&1 | grep -o 'Syntax error'
+$CLICKHOUSE_CLIENT --query="SELECT 1; SELECT 2"
+$CLICKHOUSE_CLIENT --query="SELECT 1; SELECT 2;"
 $CLICKHOUSE_CLIENT --query="SELECT 1; SELECT 2; SELECT" 2>&1 | grep -o 'Syntax error'
 
 $CLICKHOUSE_CLIENT -n --query="SELECT 1; S" 2>&1 | grep -o 'Syntax error'
diff --git a/tests/queries/0_stateless/00499_json_enum_insert.sql b/tests/queries/0_stateless/00499_json_enum_insert.sql
index c4988d08243..24eab38c65f 100644
--- a/tests/queries/0_stateless/00499_json_enum_insert.sql
+++ b/tests/queries/0_stateless/00499_json_enum_insert.sql
@@ -4,12 +4,15 @@ DROP TABLE IF EXISTS json;
 CREATE TABLE json (x Enum8('browser' = 1, 'mobile' = 2), y String) ENGINE = Memory;
 
 INSERT INTO json (y) VALUES ('Hello');
+
 SELECT * FROM json ORDER BY y;
 
 INSERT INTO json (y) FORMAT JSONEachRow {"y": "World 1"};
+
 SELECT * FROM json ORDER BY y;
 
 INSERT INTO json (x, y) FORMAT JSONEachRow {"y": "World 2"};
+
 SELECT * FROM json ORDER BY y;
 
 DROP TABLE json;
diff --git a/tests/queries/0_stateless/00517_date_parsing.sql b/tests/queries/0_stateless/00517_date_parsing.sql
index 9067d39ba3e..4cfede7cb2b 100644
--- a/tests/queries/0_stateless/00517_date_parsing.sql
+++ b/tests/queries/0_stateless/00517_date_parsing.sql
@@ -7,6 +7,7 @@ INSERT INTO date VALUES ('2017-01-02'), ('2017-1-02'), ('2017-01-2'), ('2017-1-2
 SELECT * FROM date;
 
 INSERT INTO date FORMAT JSONEachRow {"d": "2017-01-02"}, {"d": "2017-1-02"}, {"d": "2017-01-2"}, {"d": "2017-1-2"}, {"d": "2017/01/02"}, {"d": "2017/1/02"}, {"d": "2017/01/2"}, {"d": "2017/1/2"}, {"d": "2017-11-12"};
+
 SELECT * FROM date ORDER BY d;
 
 DROP TABLE date;
diff --git a/tests/queries/0_stateless/00560_float_leading_plus_in_exponent.sql b/tests/queries/0_stateless/00560_float_leading_plus_in_exponent.sql
index 33e010cdc9c..d2d62510e7b 100644
--- a/tests/queries/0_stateless/00560_float_leading_plus_in_exponent.sql
+++ b/tests/queries/0_stateless/00560_float_leading_plus_in_exponent.sql
@@ -1,3 +1,5 @@
 CREATE TEMPORARY TABLE test_float (x Float64);
+
 INSERT INTO test_float FORMAT TabSeparated 1.075e+06
+
 SELECT * FROM test_float;
diff --git a/tests/queries/0_stateless/00700_decimal_formats.sql b/tests/queries/0_stateless/00700_decimal_formats.sql
index 45b2a5f1078..f008897bf5d 100644
--- a/tests/queries/0_stateless/00700_decimal_formats.sql
+++ b/tests/queries/0_stateless/00700_decimal_formats.sql
@@ -9,22 +9,22 @@ CREATE TABLE IF NOT EXISTS decimal
 
 INSERT INTO decimal (a, b, c) VALUES (42.0, -42.0, 42) (0.42, -0.42, .42) (42.42, -42.42, 42.42);
 INSERT INTO decimal (a, b, c) FORMAT JSONEachRow {"a":1.1, "b":-1.1, "c":1.1} {"a":1.0, "b":-1.0, "c":1} {"a":0.1, "b":-0.1, "c":.1};
+
 INSERT INTO decimal (a, b, c) FORMAT CSV 2.0,-2.0,2
-;
+
 INSERT INTO decimal (a, b, c) FORMAT CSV 0.2 ,-0.2 ,.2
-;
+
 INSERT INTO decimal (a, b, c) FORMAT CSV 2.2 , -2.2 , 2.2
-;
+
 INSERT INTO decimal (a, b, c) FORMAT TabSeparated 3.3	-3.3	3.3
-;
+
 INSERT INTO decimal (a, b, c) FORMAT TabSeparated 3.0	-3.0	3
-;
+
 INSERT INTO decimal (a, b, c) FORMAT TabSeparated 0.3	-0.3	.3
-;
+
 INSERT INTO decimal (a, b, c) FORMAT CSV 4.4E+5,-4E+8,.4E+20
-;
+
 INSERT INTO decimal (a, b, c) FORMAT CSV 5.5e-2, -5e-9 ,.5e-17
-;
 
 SELECT * FROM decimal ORDER BY a FORMAT JSONEachRow;
 SELECT * FROM decimal ORDER BY b DESC FORMAT CSV;
diff --git a/tests/queries/0_stateless/00760_insert_json_with_defaults.sql b/tests/queries/0_stateless/00760_insert_json_with_defaults.sql
index 1430d10cdb0..54987258375 100644
--- a/tests/queries/0_stateless/00760_insert_json_with_defaults.sql
+++ b/tests/queries/0_stateless/00760_insert_json_with_defaults.sql
@@ -15,20 +15,29 @@ CREATE TABLE defaults
 ) ENGINE = MergeTree ORDER BY x;
 
 INSERT INTO defaults FORMAT JSONEachRow {"x":1, "y":1};
+
 INSERT INTO defaults (x, y) SELECT x, y FROM defaults LIMIT 1;
+
 INSERT INTO defaults FORMAT JSONEachRow {"x":2, "y":2, "c":2};
+
 INSERT INTO defaults FORMAT JSONEachRow {"x":3, "y":3, "a":3, "b":3, "c":3};
+
 INSERT INTO defaults FORMAT JSONEachRow {"x":4} {"y":5, "c":5} {"a":6, "b":6, "c":6};
 
 SELECT * FROM defaults ORDER BY (x, y);
 
 ALTER TABLE defaults ADD COLUMN n Nested(a UInt64, b String);
+
 INSERT INTO defaults FORMAT JSONEachRow {"x":7, "y":7, "n.a":[1,2], "n.b":["a","b"]};
+
 SELECT * FROM defaults WHERE x = 7 FORMAT JSONEachRow;
 
 ALTER TABLE defaults ADD COLUMN n.c Array(UInt8) DEFAULT arrayMap(x -> 0, n.a) AFTER n.a;
+
 INSERT INTO defaults FORMAT JSONEachRow {"x":8, "y":8, "n.a":[3,4], "n.b":["c","d"]};
+
 INSERT INTO defaults FORMAT JSONEachRow {"x":9, "y":9};
+
 SELECT * FROM defaults WHERE x > 7 ORDER BY x FORMAT JSONEachRow;
 
 DROP TABLE defaults;
diff --git a/tests/queries/0_stateless/00861_decimal_quoted_csv.sql b/tests/queries/0_stateless/00861_decimal_quoted_csv.sql
index e716c607ef8..d7211c55df7 100644
--- a/tests/queries/0_stateless/00861_decimal_quoted_csv.sql
+++ b/tests/queries/0_stateless/00861_decimal_quoted_csv.sql
@@ -2,15 +2,14 @@ DROP TABLE IF EXISTS test_00861;
 CREATE TABLE test_00861 (key UInt64, d32 Decimal32(2), d64 Decimal64(2), d128 Decimal128(2)) ENGINE = Memory;
 
 INSERT INTO test_00861 FORMAT CSV "1","1","1","1"
-;
+
 INSERT INTO test_00861 FORMAT CSV "2","-1","-1","-1"
-;
+
 INSERT INTO test_00861 FORMAT CSV "3","1.0","1.0","1.0"
-;
+
 INSERT INTO test_00861 FORMAT CSV "4","-0.1","-0.1","-0.1"
-;
+
 INSERT INTO test_00861 FORMAT CSV "5","0.010","0.010","0.010"
-;
 
 SELECT * FROM test_00861 ORDER BY key;
 
diff --git a/tests/queries/0_stateless/00945_ml_test.sql b/tests/queries/0_stateless/00945_ml_test.sql
index 299b550bceb..c3a3a51b98c 100644
--- a/tests/queries/0_stateless/00945_ml_test.sql
+++ b/tests/queries/0_stateless/00945_ml_test.sql
@@ -7,7 +7,7 @@ CREATE TABLE IF NOT EXISTS defaults
     predict1 Float64,
     predict2 Float64
 ) ENGINE = Memory;
-insert into defaults values (1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2)
+insert into defaults values (1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2),(1,2,1,-1,-2),(-1,-2,-1,1,2);
 DROP TABLE IF EXISTS model;
 create table model engine = Memory as select stochasticLogisticRegressionState(0.1, 0.0, 1.0, 'SGD')(target, param1, param2) as state from defaults;
 
diff --git a/tests/queries/0_stateless/00946_ml_test.sql b/tests/queries/0_stateless/00946_ml_test.sql
index b3f4941c2a7..a3da3b7e5c7 100644
--- a/tests/queries/0_stateless/00946_ml_test.sql
+++ b/tests/queries/0_stateless/00946_ml_test.sql
@@ -18,7 +18,7 @@ CREATE TABLE IF NOT EXISTS defaults
     predict7 Float64
 
 ) ENGINE = Memory;
-insert into defaults values (1.76210664421617,1.7469706406568504,0.7988286239230257,1.0938642223599824,1.167321139201246,1.7648182796261376,0.909111664354187,0.92,-0.5940592289464697,-0.7274920231630502,-0.952028633990455,-0.8949798182945309,-1.0614543055744028,-0.6026567095733507,-1.0999748867047898),(0.6276564089897139,-0.06763531281107672,0.7988286239230257,0.5966532121963541,1.167321139201246,0.4551512643242912,0.909111664354187,0.76,-0.5940592289464697,-0.7274920231630502,-0.952028633990455,-0.8949798182945309,-1.0614543055744028,-0.6026567095733507,-1.0999748867047898),(-0.07046681268810527,-0.5625278455750569,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,-1.0056311758200724,0.909111664354187,0.72,-0.5940592289464697,-0.7274920231630502,-0.952028633990455,-0.8949798182945309,-1.0614543055744028,-0.6026567095733507,-1.0999748867047898),(0.4531256035702591,0.4272572199529034,-0.0766000050337147,0.09944220203272576,-1.0614543055744028,0.11933920911869125,0.909111664354187,0.8,-0.5940592289464697,-0.7274920231630502,-0.952028633990455,-0.8949798182945309,-1.0614543055744028,-0.6026567095733507,-1.0999748867047898),(-0.24499761810756004,-0.7274920231630502,-0.952028633990455,-1.3921908284581592,-0.5042604443804907,-0.6530285178541898,-1.0999748867047898,0.65,-0.5940592289464697,-0.7274920231630502,-0.952028633990455,-0.8949798182945309,-1.0614543055744028,-0.6026567095733507,-1.0999748867047898),(1.1512488252480781,1.2520781078928702,1.674257252879766,1.0938642223599824,-0.5042604443804907,1.244309594057455,0.909111664354187,0.9,-0.5940592289464697,-0.7274920231630502,-0.952028633990455,-0.8949798182945309,-1.0614543055744028,-0.6026567095733507,-1.0999748867047898),(0.36586020086053167,0.26229304236491,-0.0766000050337147,-0.3977688081309026,0.6101272780073337,-0.6698191206144725,0.909111664354187,0.75,-0.5940592289464697,-0.7274920231630502,-0.952028633990455,-0.8949798182945309,-1.0614543055744028,-0.6026567095733507,-1.0999748867047898),(-0.7685900343659244,-1.057420378339037,-0.952028633990455,-0.3977688081309026,0.6101272780073337,-1.1735372034228724,-1.0999748867047898,0.68,-0.5940592289464697,-0.7274920231630502,-0.952028633990455,-0.8949798182945309,-1.0614543055744028,-0.6026567095733507,-1.0999748867047898),(-1.2921824506242887,-0.8924562007510436,-1.8274572629471952,-1.3921908284581592,-2.175842027962227,-1.0056311758200724,-1.0999748867047898,0.5,-0.5940592289464697,-0.7274920231630502,-0.952028633990455,-0.8949798182945309,-1.0614543055744028,-0.6026567095733507,-1.0999748867047898),(0.5403910062799865,0.09732886477691666,-0.0766000050337147,0.09944220203272576,-0.5042604443804907,0.0018049897967303734,-1.0999748867047898,0.45,-0.5940592289464697,-0.7274920231630502,-0.952028633990455,-0.8949798182945309,-1.0614543055744028,-0.6026567095733507,-1.0999748867047898),(0.7149218116994412,-0.2325994903990701,-0.0766000050337147,0.09944220203272576,0.6101272780073337,-0.3340070654088696,0.909111664354187,0.52,-0.5940592289464697,-0.7274920231630502,-0.952028633990455,-0.8949798182945309,-1.0614543055744028,-0.6026567095733507,-1.0999748867047898),(0.889452617118896,0.5922213975408968,0.7988286239230257,0.5966532121963541,1.167321139201246,0.6734291002079332,0.909111664354187,0.84,-0.5940592289464697,-0.7274920231630502,-0.952028633990455,-0.8949798182945309,-1.0614543055744028,-0.6026567095733507,-1.0999748867047898),(0.9767180198286235,0.7571855751288902,0.7988286239230257,0.5966532121963541,1.167321139201246,0.8413351278107333,0.909111664354187,0.78,-0.5940592289464697,-0.7274920231630502,-0.952028633990455,-0.8949798182945309,-1.0614543055744028,-0.6026567095733507,-1.0999748867047898),(-0.8558554370756518,0.26229304236491,-0.0766000050337147,0.5966532121963541,-0.5042604443804907,-1.0056311758200724,0.909111664354187,0.62,0.6276564089897139,0.4272572199529034,-0.0766000050337147,0.09944220203272576,-0.5042604443804907,1.0428223609340956,0.909111664354187),(-0.5067938262367422,-0.5625278455750569,-0.0766000050337147,0.09944220203272576,-1.618648166768315,-0.6698191206144725,0.909111664354187,0.61,0.6276564089897139,0.4272572199529034,-0.0766000050337147,0.09944220203272576,-0.5042604443804907,1.0428223609340956,0.909111664354187),(-0.24499761810756004,-0.39756366798706344,-0.0766000050337147,0.09944220203272576,-1.0614543055744028,-0.5019130930116695,-1.0999748867047898,0.54,0.6276564089897139,0.4272572199529034,-0.0766000050337147,0.09944220203272576,-0.5042604443804907,1.0428223609340956,0.909111664354187),(0.016798590021622126,-0.06763531281107672,-0.0766000050337147,0.5966532121963541,-0.5042604443804907,0.16971101739953035,-1.0999748867047898,0.66,0.6276564089897139,0.4272572199529034,-0.0766000050337147,0.09944220203272576,-0.5042604443804907,1.0428223609340956,0.909111664354187),(0.1913293954410769,-0.2325994903990701,-0.0766000050337147,0.5966532121963541,-0.5042604443804907,-1.0056311758200724,0.909111664354187,0.65,0.6276564089897139,0.4272572199529034,-0.0766000050337147,0.09944220203272576,-0.5042604443804907,1.0428223609340956,0.909111664354187),(0.10406399273134952,0.4272572199529034,-0.0766000050337147,0.5966532121963541,-0.5042604443804907,0.3376170450023333,-1.0999748867047898,0.63,0.6276564089897139,0.4272572199529034,-0.0766000050337147,0.09944220203272576,-0.5042604443804907,1.0428223609340956,0.909111664354187),(-1.2049170479145614,-0.8924562007510436,-0.0766000050337147,0.09944220203272576,-0.5042604443804907,-0.1661010378060696,-1.0999748867047898,0.62,0.6276564089897139,0.4272572199529034,-0.0766000050337147,0.09944220203272576,-0.5042604443804907,1.0428223609340956,0.909111664354187),(-0.41952842352701486,-0.06763531281107672,-0.0766000050337147,-0.3977688081309026,-1.618648166768315,-1.1735372034228724,0.909111664354187,0.64,0.6276564089897139,0.4272572199529034,-0.0766000050337147,0.09944220203272576,-0.5042604443804907,1.0428223609340956,0.909111664354187),(0.7149218116994412,1.087113930304877,0.7988286239230257,-0.3977688081309026,-1.618648166768315,-0.3340070654088696,-1.0999748867047898,0.7,0.6276564089897139,0.4272572199529034,-0.0766000050337147,0.09944220203272576,-0.5042604443804907,1.0428223609340956,0.909111664354187),(0.9767180198286235,1.4170422854808635,1.674257252879766,1.5910752325236108,1.724515000395158,1.5129592382219361,0.909111664354187,0.94,0.6276564089897139,0.4272572199529034,-0.0766000050337147,0.09944220203272576,-0.5042604443804907,1.0428223609340956,0.909111664354187),(1.5003104360869879,1.9119348182448437,1.674257252879766,1.5910752325236108,1.167321139201246,1.848771293427536,0.909111664354187,0.95,0.6276564089897139,0.4272572199529034,-0.0766000050337147,0.09944220203272576,-0.5042604443804907,1.0428223609340956,0.909111664354187),(1.6748412415064426,1.9119348182448437,1.674257252879766,0.5966532121963541,0.052933416813421515,2.016677321030339,0.909111664354187,0.97,0.6276564089897139,0.4272572199529034,-0.0766000050337147,0.09944220203272576,-0.5042604443804907,1.0428223609340956,0.909111664354187),(2.023902852345352,2.076898995832837,1.674257252879766,1.0938642223599824,1.167321139201246,1.680865265824736,0.909111664354187,0.94,1.6748412415064426,1.9119348182448437,0.7988286239230257,1.0938642223599824,0.6101272780073337,1.7144464713452956,0.909111664354187),(0.4531256035702591,0.26229304236491,1.674257252879766,1.0938642223599824,0.052933416813421515,0.3376170450023333,-1.0999748867047898,0.76,1.6748412415064426,1.9119348182448437,0.7988286239230257,1.0938642223599824,0.6101272780073337,1.7144464713452956,0.909111664354187),(-1.6412440614631982,-1.5523129111030172,-0.952028633990455,-1.8894018386217877,-1.0614543055744028,-1.8451613138340752,0.909111664354187,0.44,1.6748412415064426,1.9119348182448437,0.7988286239230257,1.0938642223599824,0.6101272780073337,1.7144464713452956,0.909111664354187),(-1.9030402695923805,-2.377133799042984,-1.8274572629471952,-1.3921908284581592,-1.618648166768315,-2.348879396642477,-1.0999748867047898,0.46,1.6748412415064426,1.9119348182448437,0.7988286239230257,1.0938642223599824,0.6101272780073337,1.7144464713452956,0.909111664354187),(-0.5940592289464697,-1.3873487335150236,-0.952028633990455,-1.8894018386217877,-1.618648166768315,-2.180973369039677,-1.0999748867047898,0.54,1.6748412415064426,1.9119348182448437,0.7988286239230257,1.0938642223599824,0.6101272780073337,1.7144464713452956,0.909111664354187),(-1.4667132560437435,-1.7172770886910105,-0.952028633990455,-0.3977688081309026,-0.5042604443804907,-0.8377251482172725,0.909111664354187,0.65,1.6748412415064426,1.9119348182448437,0.7988286239230257,1.0938642223599824,0.6101272780073337,1.7144464713452956,0.909111664354187),(0.889452617118896,-0.7274920231630502,-0.0766000050337147,0.5966532121963541,0.6101272780073337,-0.5019130930116695,0.909111664354187,0.74,1.6748412415064426,1.9119348182448437,0.7988286239230257,1.0938642223599824,0.6101272780073337,1.7144464713452956,0.909111664354187),(1.8493720469258974,1.7469706406568504,0.7988286239230257,-0.3977688081309026,1.167321139201246,1.3450532106191362,0.909111664354187,0.91,1.6748412415064426,1.9119348182448437,0.7988286239230257,1.0938642223599824,0.6101272780073337,1.7144464713452956,0.909111664354187),(2.023902852345352,1.087113930304877,1.674257252879766,0.5966532121963541,0.6101272780073337,1.680865265824736,0.909111664354187,0.9,1.6748412415064426,1.9119348182448437,0.7988286239230257,1.0938642223599824,0.6101272780073337,1.7144464713452956,0.909111664354187),(1.2385142279578056,0.7571855751288902,1.674257252879766,0.5966532121963541,1.724515000395158,2.016677321030339,0.909111664354187,0.94,1.6748412415064426,1.9119348182448437,0.7988286239230257,1.0938642223599824,0.6101272780073337,1.7144464713452956,0.909111664354187),(0.2785947981508043,0.4272572199529034,1.674257252879766,1.5910752325236108,1.724515000395158,1.0092411554135332,0.909111664354187,0.88,1.6748412415064426,1.9119348182448437,0.7988286239230257,1.0938642223599824,0.6101272780073337,1.7144464713452956,0.909111664354187),(-1.553978658753471,-0.2325994903990701,-0.952028633990455,0.5966532121963541,0.6101272780073337,-0.3340070654088696,-1.0999748867047898,0.64,1.6748412415064426,1.9119348182448437,0.7988286239230257,1.0938642223599824,0.6101272780073337,1.7144464713452956,0.909111664354187),(-1.4667132560437435,-0.39756366798706344,-1.8274572629471952,-2.386612848785416,-1.618648166768315,-1.3414432310256739,-1.0999748867047898,0.58,0.36586020086053167,0.26229304236491,-0.0766000050337147,-0.3977688081309026,-0.5042604443804907,-0.0989386267649508,0.909111664354187),(-1.117651645204834,-0.39756366798706344,-1.8274572629471952,-0.3977688081309026,-2.175842027962227,-1.8451613138340752,-1.0999748867047898,0.52,0.36586020086053167,0.26229304236491,-0.0766000050337147,-0.3977688081309026,-0.5042604443804907,-0.0989386267649508,0.909111664354187),(-0.8558554370756518,0.09732886477691666,-0.952028633990455,0.5966532121963541,0.052933416813421515,-1.5093492586284738,-1.0999748867047898,0.48,0.36586020086053167,0.26229304236491,-0.0766000050337147,-0.3977688081309026,-0.5042604443804907,-0.0989386267649508,0.909111664354187),(-0.7685900343659244,0.4272572199529034,-0.0766000050337147,0.09944220203272576,-0.5042604443804907,-1.0056311758200724,0.909111664354187,0.46,0.36586020086053167,0.26229304236491,-0.0766000050337147,-0.3977688081309026,-0.5042604443804907,-0.0989386267649508,0.909111664354187),(-0.07046681268810527,-0.39756366798706344,-0.952028633990455,-0.8949798182945309,-1.0614543055744028,-0.6698191206144725,0.909111664354187,0.49,0.36586020086053167,0.26229304236491,-0.0766000050337147,-0.3977688081309026,-0.5042604443804907,-0.0989386267649508,0.909111664354187),(-0.3322630208172874,-0.06763531281107672,-0.952028633990455,-0.8949798182945309,-1.618648166768315,-0.1661010378060696,0.909111664354187,0.53,0.36586020086053167,0.26229304236491,-0.0766000050337147,-0.3977688081309026,-0.5042604443804907,-0.0989386267649508,0.909111664354187),(1.3257796306675331,1.582006463068857,0.7988286239230257,1.0938642223599824,0.6101272780073337,0.8413351278107333,-1.0999748867047898,0.87,0.36586020086053167,0.26229304236491,-0.0766000050337147,-0.3977688081309026,-0.5042604443804907,-0.0989386267649508,0.909111664354187),(0.8021872144091686,0.9221497527168835,1.674257252879766,1.0938642223599824,0.6101272780073337,1.3450532106191362,0.909111664354187,0.91,0.36586020086053167,0.26229304236491,-0.0766000050337147,-0.3977688081309026,-0.5042604443804907,-0.0989386267649508,0.909111664354187),(0.4531256035702591,0.4272572199529034,1.674257252879766,1.5910752325236108,0.6101272780073337,0.8413351278107333,0.909111664354187,0.88,0.36586020086053167,0.26229304236491,-0.0766000050337147,-0.3977688081309026,-0.5042604443804907,-0.0989386267649508,0.909111664354187),(1.0639834225383509,1.087113930304877,1.674257252879766,0.5966532121963541,1.724515000395158,1.1771471830163363,0.909111664354187,0.86,0.36586020086053167,0.26229304236491,-0.0766000050337147,-0.3977688081309026,-0.5042604443804907,-0.0989386267649508,0.909111664354187),(1.9366374496356247,1.9119348182448437,1.674257252879766,1.0938642223599824,0.6101272780073337,1.848771293427536,-1.0999748867047898,0.89,0.36586020086053167,0.26229304236491,-0.0766000050337147,-0.3977688081309026,-0.5042604443804907,-0.0989386267649508,0.909111664354187),(0.36586020086053167,0.4272572199529034,-0.0766000050337147,0.09944220203272576,1.724515000395158,0.42157005880373183,0.909111664354187,0.82,0.36586020086053167,0.26229304236491,-0.0766000050337147,-0.3977688081309026,-0.5042604443804907,-0.0989386267649508,0.909111664354187),(0.889452617118896,0.5922213975408968,0.7988286239230257,-0.3977688081309026,0.6101272780073337,-0.3340070654088696,0.909111664354187,0.78,0.36586020086053167,0.26229304236491,-0.0766000050337147,-0.3977688081309026,-0.5042604443804907,-0.0989386267649508,0.909111664354187),(-0.3322630208172874,-1.5523129111030172,-0.0766000050337147,-0.8949798182945309,1.167321139201246,-0.5019130930116695,0.909111664354187,0.76,-0.15773221539783266,-0.39756366798706344,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-1.5933022724298738,-1.0999748867047898),(-0.41952842352701486,-1.2223845559270303,-0.952028633990455,-1.8894018386217877,0.052933416813421515,-1.1735372034228724,0.909111664354187,0.56,-0.15773221539783266,-0.39756366798706344,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-1.5933022724298738,-1.0999748867047898),(1.5003104360869879,1.4170422854808635,0.7988286239230257,0.5966532121963541,-0.5042604443804907,-1.0056311758200724,0.909111664354187,0.78,-0.15773221539783266,-0.39756366798706344,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-1.5933022724298738,-1.0999748867047898),(0.6276564089897139,0.7571855751288902,0.7988286239230257,0.5966532121963541,-1.0614543055744028,-0.8377251482172725,0.909111664354187,0.72,-0.15773221539783266,-0.39756366798706344,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-1.5933022724298738,-1.0999748867047898),(0.4531256035702591,0.4272572199529034,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,-1.0056311758200724,-1.0999748867047898,0.7,-0.15773221539783266,-0.39756366798706344,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-1.5933022724298738,-1.0999748867047898),(0.2785947981508043,-0.7274920231630502,-0.0766000050337147,-0.3977688081309026,-0.5042604443804907,-1.5093492586284738,-1.0999748867047898,0.64,-0.15773221539783266,-0.39756366798706344,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-1.5933022724298738,-1.0999748867047898),(-0.07046681268810527,-0.8924562007510436,-0.0766000050337147,-1.3921908284581592,-0.5042604443804907,-2.013067341436875,-1.0999748867047898,0.64,-0.15773221539783266,-0.39756366798706344,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-1.5933022724298738,-1.0999748867047898),(-1.6412440614631982,-1.3873487335150236,-0.952028633990455,0.5966532121963541,-1.618648166768315,-1.6772552862312753,-1.0999748867047898,0.46,-0.15773221539783266,-0.39756366798706344,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-1.5933022724298738,-1.0999748867047898),(-1.4667132560437435,-1.3873487335150236,-1.8274572629471952,-0.3977688081309026,-1.618648166768315,-3.0205035070536796,0.909111664354187,0.36,-0.15773221539783266,-0.39756366798706344,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-1.5933022724298738,-1.0999748867047898),(-0.5067938262367422,-0.5625278455750569,-0.952028633990455,-1.3921908284581592,-1.618648166768315,-0.5019130930116695,-1.0999748867047898,0.42,-0.15773221539783266,-0.39756366798706344,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-1.5933022724298738,-1.0999748867047898),(-0.681324631656197,-1.2223845559270303,-0.952028633990455,-0.3977688081309026,-0.5042604443804907,-0.8377251482172725,-1.0999748867047898,0.48,-0.15773221539783266,-0.39756366798706344,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-1.5933022724298738,-1.0999748867047898),(-0.8558554370756518,-1.057420378339037,-0.0766000050337147,0.5966532121963541,-0.5042604443804907,-0.6698191206144725,-1.0999748867047898,0.47,-0.15773221539783266,-0.39756366798706344,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-1.5933022724298738,-1.0999748867047898),(-1.117651645204834,-0.39756366798706344,-0.952028633990455,-0.3977688081309026,-0.5042604443804907,-0.6698191206144725,0.909111664354187,0.54,-1.117651645204834,-1.057420378339037,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-1.5765116696695942,-1.0999748867047898),(-0.15773221539783266,-0.06763531281107672,-0.952028633990455,0.5966532121963541,-0.5042604443804907,-0.1661010378060696,0.909111664354187,0.56,-1.117651645204834,-1.057420378339037,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-1.5765116696695942,-1.0999748867047898),(0.7149218116994412,0.5922213975408968,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.16971101739953035,-1.0999748867047898,0.52,-1.117651645204834,-1.057420378339037,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-1.5765116696695942,-1.0999748867047898),(0.7149218116994412,0.7571855751288902,0.7988286239230257,0.09944220203272576,0.052933416813421515,0.5391042781256927,-1.0999748867047898,0.55,-1.117651645204834,-1.057420378339037,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-1.5765116696695942,-1.0999748867047898),(0.889452617118896,1.087113930304877,-0.0766000050337147,-0.3977688081309026,-0.5042604443804907,0.7070103057284927,-1.0999748867047898,0.61,-1.117651645204834,-1.057420378339037,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-1.5765116696695942,-1.0999748867047898),(-0.07046681268810527,-0.06763531281107672,-0.952028633990455,0.09944220203272576,0.052933416813421515,0.06896740083785215,0.909111664354187,0.57,-1.117651645204834,-1.057420378339037,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-1.5765116696695942,-1.0999748867047898),(0.10406399273134952,0.26229304236491,-0.0766000050337147,0.09944220203272576,0.6101272780073337,1.0428223609340956,0.909111664354187,0.68,-1.117651645204834,-1.057420378339037,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-1.5765116696695942,-1.0999748867047898),(0.9767180198286235,1.2520781078928702,0.7988286239230257,1.0938642223599824,0.6101272780073337,0.9420787443724145,0.909111664354187,0.78,-1.117651645204834,-1.057420378339037,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-1.5765116696695942,-1.0999748867047898),(1.3257796306675331,1.7469706406568504,1.674257252879766,1.5910752325236108,1.724515000395158,1.7480276768658578,0.909111664354187,0.94,-1.117651645204834,-1.057420378339037,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-1.5765116696695942,-1.0999748867047898),(1.6748412415064426,0.7571855751288902,1.674257252879766,1.5910752325236108,1.724515000395158,1.9495149099892173,0.909111664354187,0.96,-1.117651645204834,-1.057420378339037,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-1.5765116696695942,-1.0999748867047898),(0.36586020086053167,0.5922213975408968,1.674257252879766,1.5910752325236108,1.724515000395158,1.4290062244205346,0.909111664354187,0.93,-1.117651645204834,-1.057420378339037,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-1.5765116696695942,-1.0999748867047898),(-0.24499761810756004,0.09732886477691666,0.7988286239230257,1.0938642223599824,0.6101272780073337,0.7405915112490521,0.909111664354187,0.84,-1.117651645204834,-1.057420378339037,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-1.5765116696695942,-1.0999748867047898),(-0.24499761810756004,-0.2325994903990701,-0.0766000050337147,-0.3977688081309026,1.724515000395158,0.5055230726051333,-1.0999748867047898,0.74,-1.7285094641729257,-1.8822412662790038,-0.952028633990455,-0.8949798182945309,-1.618648166768315,-1.9626955331560363,-1.0999748867047898),(1.0639834225383509,1.087113930304877,-0.952028633990455,-1.3921908284581592,0.6101272780073337,-0.06535742124438843,0.909111664354187,0.72,-1.7285094641729257,-1.8822412662790038,-0.952028633990455,-0.8949798182945309,-1.618648166768315,-1.9626955331560363,-1.0999748867047898),(0.889452617118896,0.7571855751288902,-0.0766000050337147,-0.3977688081309026,-0.5042604443804907,0.20329222292009272,0.909111664354187,0.74,-1.7285094641729257,-1.8822412662790038,-0.952028633990455,-0.8949798182945309,-1.618648166768315,-1.9626955331560363,-1.0999748867047898),(-1.3794478533340162,-1.3873487335150236,-0.952028633990455,-0.3977688081309026,-1.618648166768315,-0.6362379150939101,-1.0999748867047898,0.64,-1.7285094641729257,-1.8822412662790038,-0.952028633990455,-0.8949798182945309,-1.618648166768315,-1.9626955331560363,-1.0999748867047898),(-1.8157748668826532,-2.0472054438669973,-0.952028633990455,-0.3977688081309026,-1.618648166768315,-1.777998902792955,0.909111664354187,0.44,-1.7285094641729257,-1.8822412662790038,-0.952028633990455,-0.8949798182945309,-1.618648166768315,-1.9626955331560363,-1.0999748867047898),(-1.990305672302108,-2.377133799042984,-1.8274572629471952,-1.8894018386217877,-1.618648166768315,-2.0802297524779956,-1.0999748867047898,0.46,-1.7285094641729257,-1.8822412662790038,-0.952028633990455,-0.8949798182945309,-1.618648166768315,-1.9626955331560363,-1.0999748867047898),(-0.41952842352701486,-0.39756366798706344,-0.0766000050337147,-1.3921908284581592,-0.5042604443804907,-0.972049970299513,0.909111664354187,0.5,-1.7285094641729257,-1.8822412662790038,-0.952028633990455,-0.8949798182945309,-1.618648166768315,-1.9626955331560363,-1.0999748867047898),(2.023902852345352,2.076898995832837,0.7988286239230257,1.5910752325236108,1.724515000395158,1.5129592382219361,0.909111664354187,0.96,-1.7285094641729257,-1.8822412662790038,-0.952028633990455,-0.8949798182945309,-1.618648166768315,-1.9626955331560363,-1.0999748867047898),(0.2785947981508043,0.4272572199529034,1.674257252879766,1.5910752325236108,1.167321139201246,1.0428223609340956,0.909111664354187,0.92,-1.7285094641729257,-1.8822412662790038,-0.952028633990455,-0.8949798182945309,-1.618648166768315,-1.9626955331560363,-1.0999748867047898),(0.4531256035702591,1.2520781078928702,1.674257252879766,0.5966532121963541,1.167321139201246,1.2778907995780144,0.909111664354187,0.92,-1.7285094641729257,-1.8822412662790038,-0.952028633990455,-0.8949798182945309,-1.618648166768315,-1.9626955331560363,-1.0999748867047898),(2.023902852345352,1.2520781078928702,1.674257252879766,1.0938642223599824,1.167321139201246,1.4290062244205346,0.909111664354187,0.94,-1.7285094641729257,-1.8822412662790038,-0.952028633990455,-0.8949798182945309,-1.618648166768315,-1.9626955331560363,-1.0999748867047898),(0.1913293954410769,-0.7274920231630502,0.7988286239230257,1.0938642223599824,0.052933416813421515,0.10254860635841155,-1.0999748867047898,0.76,-1.7285094641729257,-1.8822412662790038,-0.952028633990455,-0.8949798182945309,-1.618648166768315,-1.9626955331560363,-1.0999748867047898),(-0.15773221539783266,-0.2325994903990701,-0.0766000050337147,1.0938642223599824,0.052933416813421515,-0.30042585988831016,-1.0999748867047898,0.72,-1.7285094641729257,-1.8822412662790038,-0.952028633990455,-0.8949798182945309,-1.618648166768315,-1.9626955331560363,-1.0999748867047898),(0.016798590021622126,-0.06763531281107672,-0.952028633990455,0.09944220203272576,-0.5042604443804907,-0.535494298532232,-1.0999748867047898,0.66,-2.3393672831410175,-1.2223845559270303,-1.8274572629471952,-1.8894018386217877,-1.618648166768315,-1.7444176972723957,-1.0999748867047898),(-0.24499761810756004,0.09732886477691666,-0.0766000050337147,1.0938642223599824,0.052933416813421515,-0.7705627371761508,-1.0999748867047898,0.64,-2.3393672831410175,-1.2223845559270303,-1.8274572629471952,-1.8894018386217877,-1.618648166768315,-1.7444176972723957,-1.0999748867047898),(-0.07046681268810527,0.26229304236491,0.7988286239230257,1.0938642223599824,0.052933416813421515,0.27045463396121155,0.909111664354187,0.74,-2.3393672831410175,-1.2223845559270303,-1.8274572629471952,-1.8894018386217877,-1.618648166768315,-1.7444176972723957,-1.0999748867047898),(0.10406399273134952,-0.2325994903990701,-0.952028633990455,0.5966532121963541,0.6101272780073337,-1.139955997902313,0.909111664354187,0.64,-2.3393672831410175,-1.2223845559270303,-1.8274572629471952,-1.8894018386217877,-1.618648166768315,-1.7444176972723957,-1.0999748867047898),(-1.553978658753471,-1.7172770886910105,-0.0766000050337147,1.5910752325236108,0.052933416813421515,-1.5765116696695942,-1.0999748867047898,0.38,-2.3393672831410175,-1.2223845559270303,-1.8274572629471952,-1.8894018386217877,-1.618648166768315,-1.7444176972723957,-1.0999748867047898),(-1.6412440614631982,-1.5523129111030172,-0.952028633990455,0.5966532121963541,-0.5042604443804907,-0.9552593675392334,-1.0999748867047898,0.34,-2.3393672831410175,-1.2223845559270303,-1.8274572629471952,-1.8894018386217877,-1.618648166768315,-1.7444176972723957,-1.0999748867047898),(-1.3794478533340162,-1.7172770886910105,-0.952028633990455,-0.3977688081309026,-0.5042604443804907,-1.2071184089434333,0.909111664354187,0.44,-2.3393672831410175,-1.2223845559270303,-1.8274572629471952,-1.8894018386217877,-1.618648166768315,-1.7444176972723957,-1.0999748867047898),(-1.2049170479145614,-1.3873487335150236,-0.0766000050337147,-1.3921908284581592,-1.0614543055744028,-1.5765116696695942,-1.0999748867047898,0.36,-2.3393672831410175,-1.2223845559270303,-1.8274572629471952,-1.8894018386217877,-1.618648166768315,-1.7444176972723957,-1.0999748867047898),(-1.117651645204834,-1.2223845559270303,0.7988286239230257,-1.8894018386217877,-1.0614543055744028,-1.2742808199845537,-1.0999748867047898,0.42,-2.3393672831410175,-1.2223845559270303,-1.8274572629471952,-1.8894018386217877,-1.618648166768315,-1.7444176972723957,-1.0999748867047898),(-0.9431208397853792,-1.2223845559270303,-0.952028633990455,-0.3977688081309026,-0.5042604443804907,-1.0056311758200724,-1.0999748867047898,0.48,-2.3393672831410175,-1.2223845559270303,-1.8274572629471952,-1.8894018386217877,-1.618648166768315,-1.7444176972723957,-1.0999748867047898),(1.2385142279578056,2.076898995832837,-0.0766000050337147,0.5966532121963541,0.6101272780073337,0.6062666891668145,0.909111664354187,0.86,-2.3393672831410175,-1.2223845559270303,-1.8274572629471952,-1.8894018386217877,-1.618648166768315,-1.7444176972723957,-1.0999748867047898),(1.3257796306675331,1.9119348182448437,0.7988286239230257,1.5910752325236108,1.167321139201246,1.076403566454655,0.909111664354187,0.9,-2.3393672831410175,-1.2223845559270303,-1.8274572629471952,-1.8894018386217877,-1.618648166768315,-1.7444176972723957,-1.0999748867047898),(0.5403910062799865,0.9221497527168835,-0.0766000050337147,0.5966532121963541,0.6101272780073337,0.47194186708457386,0.909111664354187,0.79,-1.2049170479145614,-1.5523129111030172,-1.8274572629471952,-1.3921908284581592,-1.0614543055744028,-1.5933022724298738,-1.0999748867047898),(0.4531256035702591,-0.06763531281107672,-0.0766000050337147,0.09944220203272576,0.052933416813421515,-0.2332634488471884,0.909111664354187,0.71,-1.2049170479145614,-1.5523129111030172,-1.8274572629471952,-1.3921908284581592,-1.0614543055744028,-1.5933022724298738,-1.0999748867047898),(-0.41952842352701486,-0.39756366798706344,-0.952028633990455,-0.8949798182945309,-0.5042604443804907,-0.8041439426967131,-1.0999748867047898,0.64,-1.2049170479145614,-1.5523129111030172,-1.8274572629471952,-1.3921908284581592,-1.0614543055744028,-1.5933022724298738,-1.0999748867047898),(-0.24499761810756004,-0.2325994903990701,-0.952028633990455,0.5966532121963541,0.052933416813421515,-0.585866106813071,-1.0999748867047898,0.62,-1.2049170479145614,-1.5523129111030172,-1.8274572629471952,-1.3921908284581592,-1.0614543055744028,-1.5933022724298738,-1.0999748867047898),(0.016798590021622126,-0.5625278455750569,-0.952028633990455,1.0938642223599824,0.6101272780073337,-0.2164728460869087,-1.0999748867047898,0.57,-1.2049170479145614,-1.5523129111030172,-1.8274572629471952,-1.3921908284581592,-1.0614543055744028,-1.5933022724298738,-1.0999748867047898),(0.8021872144091686,0.7571855751288902,-0.0766000050337147,0.09944220203272576,-0.5042604443804907,0.7573821140093348,0.909111664354187,0.74,-1.2049170479145614,-1.5523129111030172,-1.8274572629471952,-1.3921908284581592,-1.0614543055744028,-1.5933022724298738,-1.0999748867047898),(-0.07046681268810527,0.4272572199529034,-0.0766000050337147,0.5966532121963541,1.167321139201246,0.30403583948177093,0.909111664354187,0.69,-1.2049170479145614,-1.5523129111030172,-1.8274572629471952,-1.3921908284581592,-1.0614543055744028,-1.5933022724298738,-1.0999748867047898),(1.0639834225383509,0.5922213975408968,0.7988286239230257,1.0938642223599824,1.167321139201246,0.9756599498929738,0.909111664354187,0.87,-1.2049170479145614,-1.5523129111030172,-1.8274572629471952,-1.3921908284581592,-1.0614543055744028,-1.5933022724298738,-1.0999748867047898),(1.8493720469258974,1.582006463068857,0.7988286239230257,0.09944220203272576,1.167321139201246,1.4457968271808173,0.909111664354187,0.91,-1.2049170479145614,-1.5523129111030172,-1.8274572629471952,-1.3921908284581592,-1.0614543055744028,-1.5933022724298738,-1.0999748867047898),(1.2385142279578056,1.4170422854808635,1.674257252879766,1.5910752325236108,1.724515000395158,1.3114720050985766,0.909111664354187,0.93,-1.2049170479145614,-1.5523129111030172,-1.8274572629471952,-1.3921908284581592,-1.0614543055744028,-1.5933022724298738,-1.0999748867047898),(-1.117651645204834,-0.7274920231630502,1.674257252879766,1.5910752325236108,0.6101272780073337,0.06896740083785215,-1.0999748867047898,0.68,-1.2049170479145614,-1.5523129111030172,-1.8274572629471952,-1.3921908284581592,-1.0614543055744028,-1.5933022724298738,-1.0999748867047898),(-1.0303862424951067,0.09732886477691666,1.674257252879766,-0.3977688081309026,-0.5042604443804907,-0.199682243326629,-1.0999748867047898,0.61,-1.2049170479145614,-1.5523129111030172,-1.8274572629471952,-1.3921908284581592,-1.0614543055744028,-1.5933022724298738,-1.0999748867047898),(0.36586020086053167,0.26229304236491,0.7988286239230257,0.5966532121963541,0.6101272780073337,0.13612981187897094,0.909111664354187,0.69,-0.5067938262367422,-1.3873487335150236,-1.8274572629471952,-0.8949798182945309,-0.5042604443804907,-0.2836352571280305,0.909111664354187),(-1.3794478533340162,-0.06763531281107672,-0.0766000050337147,0.09944220203272576,0.052933416813421515,-0.4347506819705508,0.909111664354187,0.62,-0.5067938262367422,-1.3873487335150236,-1.8274572629471952,-0.8949798182945309,-0.5042604443804907,-0.2836352571280305,0.909111664354187),(0.2785947981508043,0.4272572199529034,-0.952028633990455,0.5966532121963541,0.052933416813421515,-0.06535742124438843,-1.0999748867047898,0.72,-0.5067938262367422,-1.3873487335150236,-1.8274572629471952,-0.8949798182945309,-0.5042604443804907,-0.2836352571280305,0.909111664354187),(-0.5067938262367422,-0.39756366798706344,-0.0766000050337147,0.09944220203272576,-0.5042604443804907,-0.2500540516074711,0.909111664354187,0.59,-0.5067938262367422,-1.3873487335150236,-1.8274572629471952,-0.8949798182945309,-0.5042604443804907,-0.2836352571280305,0.909111664354187),(-0.5940592289464697,-0.2325994903990701,0.7988286239230257,1.0938642223599824,1.167321139201246,0.7405915112490521,0.909111664354187,0.66,-0.5067938262367422,-1.3873487335150236,-1.8274572629471952,-0.8949798182945309,-0.5042604443804907,-0.2836352571280305,0.909111664354187),(-1.553978658753471,-0.8924562007510436,-0.0766000050337147,0.5966532121963541,0.052933416813421515,0.03538619531728977,-1.0999748867047898,0.56,-0.5067938262367422,-1.3873487335150236,-1.8274572629471952,-0.8949798182945309,-0.5042604443804907,-0.2836352571280305,0.909111664354187),(-2.3393672831410175,-0.5625278455750569,0.7988286239230257,-1.3921908284581592,-1.0614543055744028,-1.9123237248751956,-1.0999748867047898,0.45,-0.5067938262367422,-1.3873487335150236,-1.8274572629471952,-0.8949798182945309,-0.5042604443804907,-0.2836352571280305,0.909111664354187),(-1.8157748668826532,-1.3873487335150236,-0.952028633990455,-0.3977688081309026,0.052933416813421515,-2.214554574560236,-1.0999748867047898,0.47,-0.5067938262367422,-1.3873487335150236,-1.8274572629471952,-0.8949798182945309,-0.5042604443804907,-0.2836352571280305,0.909111664354187),(0.889452617118896,-0.5625278455750569,1.674257252879766,-0.3977688081309026,0.052933416813421515,0.4047794560434521,0.909111664354187,0.71,-0.5067938262367422,-1.3873487335150236,-1.8274572629471952,-0.8949798182945309,-0.5042604443804907,-0.2836352571280305,0.909111664354187),(1.5875758387967152,1.582006463068857,1.674257252879766,1.5910752325236108,1.724515000395158,1.6137028547836172,0.909111664354187,0.94,-0.5067938262367422,-1.3873487335150236,-1.8274572629471952,-0.8949798182945309,-0.5042604443804907,-0.2836352571280305,0.909111664354187),(1.5003104360869879,1.9119348182448437,1.674257252879766,1.0938642223599824,1.167321139201246,1.4793780327013768,0.909111664354187,0.94,-0.5067938262367422,-1.3873487335150236,-1.8274572629471952,-0.8949798182945309,-0.5042604443804907,-0.2836352571280305,0.909111664354187),(-0.5940592289464697,-0.2325994903990701,0.7988286239230257,-1.8894018386217877,-1.0614543055744028,-0.40116947644999135,-1.0999748867047898,0.57,-0.5067938262367422,-1.3873487335150236,-1.8274572629471952,-0.8949798182945309,-0.5042604443804907,-0.2836352571280305,0.909111664354187),(-0.7685900343659244,0.09732886477691666,-0.0766000050337147,0.09944220203272576,0.052933416813421515,-0.6362379150939101,-1.0999748867047898,0.61,-0.5067938262367422,-1.3873487335150236,-1.8274572629471952,-0.8949798182945309,-0.5042604443804907,-0.2836352571280305,0.909111664354187),(-1.3794478533340162,-0.2325994903990701,0.7988286239230257,-0.8949798182945309,-0.5042604443804907,-0.2164728460869087,-1.0999748867047898,0.57,0.4531256035702591,-0.5625278455750569,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.4047794560434521,0.909111664354187),(-1.4667132560437435,-1.2223845559270303,-0.0766000050337147,-1.3921908284581592,-0.5042604443804907,0.10254860635841155,0.909111664354187,0.64,0.4531256035702591,-0.5625278455750569,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.4047794560434521,0.909111664354187),(0.5403910062799865,0.9221497527168835,-0.0766000050337147,0.5966532121963541,-0.5042604443804907,1.2107283885368956,0.909111664354187,0.85,0.4531256035702591,-0.5625278455750569,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.4047794560434521,0.909111664354187),(0.1913293954410769,0.7571855751288902,-0.0766000050337147,-0.8949798182945309,-1.618648166768315,0.18650162015981303,0.909111664354187,0.78,0.4531256035702591,-0.5625278455750569,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.4047794560434521,0.909111664354187),(0.8021872144091686,0.7571855751288902,-0.0766000050337147,0.09944220203272576,-0.5042604443804907,0.8413351278107333,0.909111664354187,0.84,0.4531256035702591,-0.5625278455750569,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.4047794560434521,0.909111664354187),(1.4130450333772604,1.7469706406568504,1.674257252879766,1.5910752325236108,1.724515000395158,1.2611001968177347,0.909111664354187,0.92,0.4531256035702591,-0.5625278455750569,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.4047794560434521,0.909111664354187),(1.9366374496356247,1.087113930304877,1.674257252879766,0.5966532121963541,1.167321139201246,1.9495149099892173,0.909111664354187,0.96,0.4531256035702591,-0.5625278455750569,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.4047794560434521,0.909111664354187),(-1.2049170479145614,-0.39756366798706344,1.674257252879766,1.5910752325236108,1.167321139201246,0.08575800359813185,-1.0999748867047898,0.77,0.4531256035702591,-0.5625278455750569,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.4047794560434521,0.909111664354187),(-0.681324631656197,-0.39756366798706344,1.674257252879766,0.09944220203272576,0.052933416813421515,-0.06535742124438843,-1.0999748867047898,0.71,0.4531256035702591,-0.5625278455750569,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.4047794560434521,0.909111664354187),(0.5403910062799865,0.7571855751288902,1.674257252879766,0.5966532121963541,1.167321139201246,0.30403583948177093,-1.0999748867047898,0.79,0.4531256035702591,-0.5625278455750569,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.4047794560434521,0.909111664354187),(1.4130450333772604,0.9221497527168835,1.674257252879766,0.5966532121963541,0.6101272780073337,1.1435659774957738,0.909111664354187,0.89,0.4531256035702591,-0.5625278455750569,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.4047794560434521,0.909111664354187),(-0.24499761810756004,0.26229304236491,0.7988286239230257,0.09944220203272576,0.6101272780073337,0.2872452367214912,0.909111664354187,0.82,0.4531256035702591,-0.5625278455750569,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.4047794560434521,0.909111664354187),(-0.41952842352701486,-0.7274920231630502,-0.0766000050337147,1.5910752325236108,0.6101272780073337,-0.2500540516074711,-1.0999748867047898,0.76,0.1913293954410769,-0.39756366798706344,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.11933920911869125,0.909111664354187),(-0.07046681268810527,-1.2223845559270303,-0.952028633990455,-1.8894018386217877,-0.5042604443804907,-0.7369815316555913,0.909111664354187,0.71,0.1913293954410769,-0.39756366798706344,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.11933920911869125,0.909111664354187),(0.8021872144091686,1.4170422854808635,-0.952028633990455,1.0938642223599824,-0.5042604443804907,0.8077539222901738,0.909111664354187,0.8,0.1913293954410769,-0.39756366798706344,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.11933920911869125,0.909111664354187),(0.10406399273134952,0.26229304236491,-1.8274572629471952,0.09944220203272576,0.052933416813421515,0.8749163333312926,-1.0999748867047898,0.78,0.1913293954410769,-0.39756366798706344,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.11933920911869125,0.909111664354187),(1.0639834225383509,0.4272572199529034,-0.952028633990455,0.5966532121963541,-0.5042604443804907,0.9252881416121347,0.909111664354187,0.84,0.1913293954410769,-0.39756366798706344,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.11933920911869125,0.909111664354187),(1.3257796306675331,1.7469706406568504,-0.952028633990455,1.0938642223599824,0.052933416813421515,1.2778907995780144,0.909111664354187,0.9,0.1913293954410769,-0.39756366798706344,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.11933920911869125,0.909111664354187),(1.2385142279578056,1.2520781078928702,1.674257252879766,0.5966532121963541,0.052933416813421515,1.412215621660255,0.909111664354187,0.92,0.1913293954410769,-0.39756366798706344,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.11933920911869125,0.909111664354187),(2.023902852345352,2.076898995832837,0.7988286239230257,1.0938642223599824,0.6101272780073337,2.2181645541536983,0.909111664354187,0.97,0.1913293954410769,-0.39756366798706344,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.11933920911869125,0.909111664354187),(0.7149218116994412,0.7571855751288902,-0.952028633990455,-0.3977688081309026,0.052933416813421515,0.6062666891668145,0.909111664354187,0.8,0.1913293954410769,-0.39756366798706344,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.11933920911869125,0.909111664354187),(0.2785947981508043,0.9221497527168835,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,0.06896740083785215,0.909111664354187,0.81,0.1913293954410769,-0.39756366798706344,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.11933920911869125,0.909111664354187),(-0.15773221539783266,-0.39756366798706344,-0.0766000050337147,-1.3921908284581592,-1.0614543055744028,-0.199682243326629,-1.0999748867047898,0.75,0.1913293954410769,-0.39756366798706344,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.11933920911869125,0.909111664354187),(0.8021872144091686,1.087113930304877,-0.0766000050337147,-0.3977688081309026,-0.5042604443804907,0.8581257305710129,0.909111664354187,0.83,0.1913293954410769,-0.39756366798706344,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.11933920911869125,0.909111664354187),(1.9366374496356247,1.4170422854808635,0.7988286239230257,0.5966532121963541,0.052933416813421515,2.016677321030339,0.909111664354187,0.96,0.6276564089897139,0.4272572199529034,0.7988286239230257,1.0938642223599824,0.6101272780073337,0.9252881416121347,0.909111664354187),(-0.5067938262367422,-0.2325994903990701,-0.952028633990455,0.09944220203272576,-0.5042604443804907,-0.5690755040527913,0.909111664354187,0.79,0.6276564089897139,0.4272572199529034,0.7988286239230257,1.0938642223599824,0.6101272780073337,0.9252881416121347,0.909111664354187),(1.5003104360869879,1.087113930304877,0.7988286239230257,0.5966532121963541,0.6101272780073337,1.3954250188999753,0.909111664354187,0.93,0.6276564089897139,0.4272572199529034,0.7988286239230257,1.0938642223599824,0.6101272780073337,0.9252881416121347,0.909111664354187),(1.3257796306675331,1.4170422854808635,1.674257252879766,1.5910752325236108,1.724515000395158,1.1435659774957738,0.909111664354187,0.94,0.6276564089897139,0.4272572199529034,0.7988286239230257,1.0938642223599824,0.6101272780073337,0.9252881416121347,0.909111664354187),(0.36586020086053167,0.7571855751288902,1.674257252879766,1.5910752325236108,1.724515000395158,0.7741727167696144,0.909111664354187,0.86,0.6276564089897139,0.4272572199529034,0.7988286239230257,1.0938642223599824,0.6101272780073337,0.9252881416121347,0.909111664354187),(0.6276564089897139,-0.39756366798706344,-0.0766000050337147,-0.3977688081309026,0.6101272780073337,0.25366403120093184,-1.0999748867047898,0.79,0.6276564089897139,0.4272572199529034,0.7988286239230257,1.0938642223599824,0.6101272780073337,0.9252881416121347,0.909111664354187),(0.8021872144091686,0.09732886477691666,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.4887324698448536,-1.0999748867047898,0.8,0.6276564089897139,0.4272572199529034,0.7988286239230257,1.0938642223599824,0.6101272780073337,0.9252881416121347,0.909111664354187),(-0.41952842352701486,0.26229304236491,-0.0766000050337147,-0.3977688081309026,-0.5042604443804907,0.15292041463925066,-1.0999748867047898,0.77,0.6276564089897139,0.4272572199529034,0.7988286239230257,1.0938642223599824,0.6101272780073337,0.9252881416121347,0.909111664354187),(-0.15773221539783266,-0.39756366798706344,-0.0766000050337147,-1.3921908284581592,-1.0614543055744028,-0.4347506819705508,-1.0999748867047898,0.7,0.6276564089897139,0.4272572199529034,0.7988286239230257,1.0938642223599824,0.6101272780073337,0.9252881416121347,0.909111664354187),(-0.681324631656197,-0.5625278455750569,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-0.5690755040527913,-1.0999748867047898,0.65,0.6276564089897139,0.4272572199529034,0.7988286239230257,1.0938642223599824,0.6101272780073337,0.9252881416121347,0.909111664354187),(-0.9431208397853792,-0.2325994903990701,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-0.7705627371761508,-1.0999748867047898,0.61,0.6276564089897139,0.4272572199529034,0.7988286239230257,1.0938642223599824,0.6101272780073337,0.9252881416121347,0.909111664354187),(-1.7285094641729257,-1.2223845559270303,-1.8274572629471952,-1.8894018386217877,-1.618648166768315,-1.1735372034228724,-1.0999748867047898,0.52,0.6276564089897139,0.4272572199529034,0.7988286239230257,1.0938642223599824,0.6101272780073337,0.9252881416121347,0.909111664354187),(-0.15773221539783266,-0.7274920231630502,-1.8274572629471952,-1.8894018386217877,-1.618648166768315,-1.2406996144639928,-1.0999748867047898,0.57,0.6276564089897139,0.4272572199529034,0.7988286239230257,1.0938642223599824,0.6101272780073337,0.9252881416121347,0.909111664354187),(-1.6412440614631982,-1.3873487335150236,-1.8274572629471952,-1.8894018386217877,-0.5042604443804907,-1.9123237248751956,-1.0999748867047898,0.53,-1.4667132560437435,-1.2223845559270303,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,-0.5690755040527913,-1.0999748867047898),(0.10406399273134952,0.26229304236491,-0.0766000050337147,-0.3977688081309026,-0.5042604443804907,-0.1661010378060696,-1.0999748867047898,0.67,-1.4667132560437435,-1.2223845559270303,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,-0.5690755040527913,-1.0999748867047898),(0.016798590021622126,-0.39756366798706344,-0.0766000050337147,0.09944220203272576,-0.5042604443804907,-0.06535742124438843,-1.0999748867047898,0.68,-1.4667132560437435,-1.2223845559270303,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,-0.5690755040527913,-1.0999748867047898),(1.0639834225383509,0.5922213975408968,0.7988286239230257,1.0938642223599824,0.6101272780073337,0.690219702968213,0.909111664354187,0.81,-1.4667132560437435,-1.2223845559270303,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,-0.5690755040527913,-1.0999748867047898),(0.4531256035702591,0.4272572199529034,1.674257252879766,1.0938642223599824,0.6101272780073337,0.6230572919270941,-1.0999748867047898,0.78,-1.4667132560437435,-1.2223845559270303,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,-0.5690755040527913,-1.0999748867047898),(-1.2921824506242887,-0.8924562007510436,-0.0766000050337147,0.09944220203272576,1.724515000395158,-0.45154128473083044,-1.0999748867047898,0.65,-1.4667132560437435,-1.2223845559270303,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,-0.5690755040527913,-1.0999748867047898),(-0.3322630208172874,-0.8924562007510436,-0.0766000050337147,-1.3921908284581592,-0.5042604443804907,-0.5522849012925116,-1.0999748867047898,0.64,-1.4667132560437435,-1.2223845559270303,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,-0.5690755040527913,-1.0999748867047898),(-2.0775710750118352,-1.7172770886910105,-0.952028633990455,-1.3921908284581592,0.6101272780073337,-1.3414432310256739,0.909111664354187,0.64,-1.4667132560437435,-1.2223845559270303,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,-0.5690755040527913,-1.0999748867047898),(-0.5067938262367422,-1.3873487335150236,-0.952028633990455,-0.8949798182945309,-0.5042604443804907,-1.0392123813406318,-1.0999748867047898,0.65,-1.4667132560437435,-1.2223845559270303,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,-0.5690755040527913,-1.0999748867047898),(-0.41952842352701486,-1.057420378339037,-0.952028633990455,-0.8949798182945309,0.052933416813421515,-0.9384687647789537,0.909111664354187,0.68,-1.4667132560437435,-1.2223845559270303,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,-0.5690755040527913,-1.0999748867047898),(1.5003104360869879,1.582006463068857,1.674257252879766,0.5966532121963541,1.167321139201246,0.7909633195298942,0.909111664354187,0.89,-1.4667132560437435,-1.2223845559270303,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,-0.5690755040527913,-1.0999748867047898),(0.4531256035702591,0.4272572199529034,0.7988286239230257,0.5966532121963541,1.724515000395158,0.8917069360915754,0.909111664354187,0.86,-1.4667132560437435,-1.2223845559270303,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,-0.5690755040527913,-1.0999748867047898),(0.5403910062799865,0.9221497527168835,0.7988286239230257,0.5966532121963541,1.167321139201246,1.0596129636943752,0.909111664354187,0.89,2.023902852345352,0.9221497527168835,0.7988286239230257,1.5910752325236108,1.724515000395158,1.915933704468658,0.909111664354187),(0.36586020086053167,0.5922213975408968,0.7988286239230257,0.5966532121963541,0.6101272780073337,0.6230572919270941,0.909111664354187,0.87,2.023902852345352,0.9221497527168835,0.7988286239230257,1.5910752325236108,1.724515000395158,1.915933704468658,0.909111664354187),(0.2785947981508043,0.5922213975408968,0.7988286239230257,1.0938642223599824,0.052933416813421515,0.4551512643242912,0.909111664354187,0.85,2.023902852345352,0.9221497527168835,0.7988286239230257,1.5910752325236108,1.724515000395158,1.915933704468658,0.909111664354187),(1.0639834225383509,1.9119348182448437,0.7988286239230257,1.0938642223599824,1.167321139201246,0.9420787443724145,0.909111664354187,0.9,2.023902852345352,0.9221497527168835,0.7988286239230257,1.5910752325236108,1.724515000395158,1.915933704468658,0.909111664354187),(0.1913293954410769,0.4272572199529034,-0.0766000050337147,0.09944220203272576,0.052933416813421515,0.7405915112490521,-1.0999748867047898,0.82,2.023902852345352,0.9221497527168835,0.7988286239230257,1.5910752325236108,1.724515000395158,1.915933704468658,0.909111664354187),(-0.681324631656197,0.09732886477691666,-0.0766000050337147,-0.8949798182945309,-0.5042604443804907,-0.8041439426967131,-1.0999748867047898,0.72,2.023902852345352,0.9221497527168835,0.7988286239230257,1.5910752325236108,1.724515000395158,1.915933704468658,0.909111664354187),(-0.8558554370756518,-0.8924562007510436,-0.0766000050337147,-0.3977688081309026,-0.5042604443804907,-0.5522849012925116,-1.0999748867047898,0.73,2.023902852345352,0.9221497527168835,0.7988286239230257,1.5910752325236108,1.724515000395158,1.915933704468658,0.909111664354187),(-1.4667132560437435,-0.5625278455750569,-0.0766000050337147,0.09944220203272576,-0.5042604443804907,-0.7369815316555913,-1.0999748867047898,0.71,2.023902852345352,0.9221497527168835,0.7988286239230257,1.5910752325236108,1.724515000395158,1.915933704468658,0.909111664354187),(-1.0303862424951067,-0.06763531281107672,-0.952028633990455,-0.8949798182945309,-1.0614543055744028,-0.30042585988831016,-1.0999748867047898,0.71,2.023902852345352,0.9221497527168835,0.7988286239230257,1.5910752325236108,1.724515000395158,1.915933704468658,0.909111664354187),(-1.553978658753471,-1.2223845559270303,-0.952028633990455,-0.3977688081309026,0.052933416813421515,-1.2071184089434333,-1.0999748867047898,0.68,2.023902852345352,0.9221497527168835,0.7988286239230257,1.5910752325236108,1.724515000395158,1.915933704468658,0.909111664354187),(-0.24499761810756004,0.4272572199529034,-0.0766000050337147,0.5966532121963541,0.6101272780073337,0.3376170450023333,-1.0999748867047898,0.75,2.023902852345352,0.9221497527168835,0.7988286239230257,1.5910752325236108,1.724515000395158,1.915933704468658,0.909111664354187),(-0.07046681268810527,-0.2325994903990701,-0.952028633990455,-0.8949798182945309,0.6101272780073337,-0.46833188749111015,-1.0999748867047898,0.72,2.023902852345352,0.9221497527168835,0.7988286239230257,1.5910752325236108,1.724515000395158,1.915933704468658,0.909111664354187),(0.889452617118896,0.9221497527168835,0.7988286239230257,1.0938642223599824,1.167321139201246,0.8581257305710129,0.909111664354187,0.89,1.5875758387967152,1.582006463068857,1.674257252879766,1.5910752325236108,1.724515000395158,2.0502585265508984,0.909111664354187),(0.016798590021622126,-0.06763531281107672,-0.0766000050337147,0.09944220203272576,-0.5042604443804907,0.13612981187897094,0.909111664354187,0.84,1.5875758387967152,1.582006463068857,1.674257252879766,1.5910752325236108,1.724515000395158,2.0502585265508984,0.909111664354187),(1.5875758387967152,1.7469706406568504,1.674257252879766,1.0938642223599824,0.052933416813421515,1.412215621660255,0.909111664354187,0.93,1.5875758387967152,1.582006463068857,1.674257252879766,1.5910752325236108,1.724515000395158,2.0502585265508984,0.909111664354187),(1.2385142279578056,1.2520781078928702,1.674257252879766,1.0938642223599824,0.052933416813421515,1.2778907995780144,0.909111664354187,0.93,1.5875758387967152,1.582006463068857,1.674257252879766,1.5910752325236108,1.724515000395158,2.0502585265508984,0.909111664354187),(0.6276564089897139,0.7571855751288902,1.674257252879766,1.5910752325236108,1.724515000395158,0.8077539222901738,0.909111664354187,0.88,1.5875758387967152,1.582006463068857,1.674257252879766,1.5910752325236108,1.724515000395158,2.0502585265508984,0.909111664354187),(0.6276564089897139,0.5922213975408968,1.674257252879766,1.0938642223599824,0.6101272780073337,0.9420787443724145,0.909111664354187,0.9,1.5875758387967152,1.582006463068857,1.674257252879766,1.5910752325236108,1.724515000395158,2.0502585265508984,0.909111664354187),(0.5403910062799865,0.4272572199529034,1.674257252879766,0.5966532121963541,1.724515000395158,0.6398478946873739,0.909111664354187,0.87,1.5875758387967152,1.582006463068857,1.674257252879766,1.5910752325236108,1.724515000395158,2.0502585265508984,0.909111664354187),(0.4531256035702591,1.087113930304877,1.674257252879766,1.0938642223599824,0.6101272780073337,0.572685483646252,0.909111664354187,0.86,1.5875758387967152,1.582006463068857,1.674257252879766,1.5910752325236108,1.724515000395158,2.0502585265508984,0.909111664354187),(1.6748412415064426,1.7469706406568504,1.674257252879766,1.0938642223599824,1.724515000395158,1.5633310465027752,0.909111664354187,0.94,1.5875758387967152,1.582006463068857,1.674257252879766,1.5910752325236108,1.724515000395158,2.0502585265508984,0.909111664354187),(-0.07046681268810527,0.26229304236491,-0.0766000050337147,0.09944220203272576,-0.5042604443804907,0.27045463396121155,-1.0999748867047898,0.77,1.5875758387967152,1.582006463068857,1.674257252879766,1.5910752325236108,1.724515000395158,2.0502585265508984,0.909111664354187),(-0.8558554370756518,-0.06763531281107672,-0.952028633990455,-0.3977688081309026,0.052933416813421515,-0.1325198322855102,0.909111664354187,0.78,1.5875758387967152,1.582006463068857,1.674257252879766,1.5910752325236108,1.724515000395158,2.0502585265508984,0.909111664354187),(-0.9431208397853792,-0.39756366798706344,-0.952028633990455,-0.3977688081309026,-1.0614543055744028,-0.5690755040527913,-1.0999748867047898,0.73,1.5875758387967152,1.582006463068857,1.674257252879766,1.5910752325236108,1.724515000395158,2.0502585265508984,0.909111664354187),(-0.5940592289464697,-0.2325994903990701,-0.952028633990455,0.09944220203272576,-1.0614543055744028,-0.45154128473083044,-1.0999748867047898,0.73,1.5875758387967152,1.582006463068857,1.674257252879766,1.5910752325236108,1.724515000395158,2.0502585265508984,0.909111664354187),(-0.5067938262367422,-0.5625278455750569,-0.0766000050337147,1.0938642223599824,1.167321139201246,-0.2836352571280305,-1.0999748867047898,0.7,-1.2921824506242887,-1.057420378339037,-0.952028633990455,-0.8949798182945309,0.052933416813421515,-1.0727935868611929,-1.0999748867047898),(-0.3322630208172874,-0.06763531281107672,-0.0766000050337147,0.5966532121963541,1.167321139201246,0.15292041463925066,-1.0999748867047898,0.72,-1.2921824506242887,-1.057420378339037,-0.952028633990455,-0.8949798182945309,0.052933416813421515,-1.0727935868611929,-1.0999748867047898),(0.016798590021622126,-0.7274920231630502,-0.0766000050337147,-0.8949798182945309,-0.5042604443804907,-0.0989386267649508,0.909111664354187,0.73,-1.2921824506242887,-1.057420378339037,-0.952028633990455,-0.8949798182945309,0.052933416813421515,-1.0727935868611929,-1.0999748867047898),(-0.15773221539783266,0.4272572199529034,-0.952028633990455,0.09944220203272576,-0.5042604443804907,-0.2332634488471884,0.909111664354187,0.72,-1.2921824506242887,-1.057420378339037,-0.952028633990455,-0.8949798182945309,0.052933416813421515,-1.0727935868611929,-1.0999748867047898),(2.023902852345352,2.076898995832837,1.674257252879766,1.0938642223599824,1.167321139201246,2.2013739513934185,0.909111664354187,0.97,-1.2921824506242887,-1.057420378339037,-0.952028633990455,-0.8949798182945309,0.052933416813421515,-1.0727935868611929,-1.0999748867047898),(1.5003104360869879,2.076898995832837,1.674257252879766,0.5966532121963541,1.724515000395158,2.1342115403522968,0.909111664354187,0.97,-1.2921824506242887,-1.057420378339037,-0.952028633990455,-0.8949798182945309,0.052933416813421515,-1.0727935868611929,-1.0999748867047898),(-1.6412440614631982,-0.39756366798706344,-0.0766000050337147,0.09944220203272576,0.6101272780073337,-0.0989386267649508,-1.0999748867047898,0.69,-1.2921824506242887,-1.057420378339037,-0.952028633990455,-0.8949798182945309,0.052933416813421515,-1.0727935868611929,-1.0999748867047898),(-1.9030402695923805,-1.3873487335150236,-0.952028633990455,-0.8949798182945309,-0.5042604443804907,-1.5933022724298738,-1.0999748867047898,0.57,-1.2921824506242887,-1.057420378339037,-0.952028633990455,-0.8949798182945309,0.052933416813421515,-1.0727935868611929,-1.0999748867047898),(-0.15773221539783266,-1.3873487335150236,-0.952028633990455,0.09944220203272576,-0.5042604443804907,-1.1903278061831537,-1.0999748867047898,0.63,-1.2921824506242887,-1.057420378339037,-0.952028633990455,-0.8949798182945309,0.052933416813421515,-1.0727935868611929,-1.0999748867047898),(-0.5940592289464697,-0.8924562007510436,-0.0766000050337147,0.09944220203272576,0.6101272780073337,-0.972049970299513,0.909111664354187,0.66,-1.2921824506242887,-1.057420378339037,-0.952028633990455,-0.8949798182945309,0.052933416813421515,-1.0727935868611929,-1.0999748867047898),(-1.0303862424951067,-0.2325994903990701,-0.952028633990455,-0.3977688081309026,-0.5042604443804907,-0.7369815316555913,-1.0999748867047898,0.64,-1.2921824506242887,-1.057420378339037,-0.952028633990455,-0.8949798182945309,0.052933416813421515,-1.0727935868611929,-1.0999748867047898),(-1.3794478533340162,-0.5625278455750569,-0.0766000050337147,0.09944220203272576,0.6101272780073337,-0.8041439426967131,0.909111664354187,0.68,-1.2921824506242887,-1.057420378339037,-0.952028633990455,-0.8949798182945309,0.052933416813421515,-1.0727935868611929,-1.0999748867047898),(0.7149218116994412,0.09732886477691666,0.7988286239230257,1.0938642223599824,0.6101272780073337,0.7741727167696144,0.909111664354187,0.79,-0.8558554370756518,-0.39756366798706344,-0.952028633990455,-1.3921908284581592,0.052933416813421515,-0.8377251482172725,-1.0999748867047898),(0.9767180198286235,0.4272572199529034,0.7988286239230257,1.5910752325236108,0.6101272780073337,0.908497538851855,0.909111664354187,0.82,-0.8558554370756518,-0.39756366798706344,-0.952028633990455,-1.3921908284581592,0.052933416813421515,-0.8377251482172725,-1.0999748867047898),(1.8493720469258974,2.076898995832837,0.7988286239230257,1.5910752325236108,1.724515000395158,1.7816088823864173,0.909111664354187,0.95,-0.8558554370756518,-0.39756366798706344,-0.952028633990455,-1.3921908284581592,0.052933416813421515,-0.8377251482172725,-1.0999748867047898),(1.4130450333772604,1.9119348182448437,1.674257252879766,1.5910752325236108,1.167321139201246,1.9830961155097766,0.909111664354187,0.96,-0.8558554370756518,-0.39756366798706344,-0.952028633990455,-1.3921908284581592,0.052933416813421515,-0.8377251482172725,-1.0999748867047898),(1.2385142279578056,1.582006463068857,0.7988286239230257,1.0938642223599824,1.724515000395158,1.3786344161396955,0.909111664354187,0.94,-0.8558554370756518,-0.39756366798706344,-0.952028633990455,-1.3921908284581592,0.052933416813421515,-0.8377251482172725,-1.0999748867047898),(1.1512488252480781,1.4170422854808635,1.674257252879766,1.5910752325236108,1.167321139201246,1.2778907995780144,0.909111664354187,0.93,-0.8558554370756518,-0.39756366798706344,-0.952028633990455,-1.3921908284581592,0.052933416813421515,-0.8377251482172725,-1.0999748867047898),(0.4531256035702591,0.7571855751288902,0.7988286239230257,1.0938642223599824,1.167321139201246,1.1099847719752143,0.909111664354187,0.91,-0.8558554370756518,-0.39756366798706344,-0.952028633990455,-1.3921908284581592,0.052933416813421515,-0.8377251482172725,-1.0999748867047898),(0.36586020086053167,0.26229304236491,0.7988286239230257,0.5966532121963541,0.6101272780073337,0.8917069360915754,0.909111664354187,0.85,-0.8558554370756518,-0.39756366798706344,-0.952028633990455,-1.3921908284581592,0.052933416813421515,-0.8377251482172725,-1.0999748867047898),(0.6276564089897139,0.4272572199529034,0.7988286239230257,-0.3977688081309026,0.052933416813421515,0.6230572919270941,0.909111664354187,0.84,-0.8558554370756518,-0.39756366798706344,-0.952028633990455,-1.3921908284581592,0.052933416813421515,-0.8377251482172725,-1.0999748867047898),(-0.41952842352701486,-0.5625278455750569,-0.0766000050337147,0.09944220203272576,0.052933416813421515,-0.30042585988831016,-1.0999748867047898,0.74,-0.8558554370756518,-0.39756366798706344,-0.952028633990455,-1.3921908284581592,0.052933416813421515,-0.8377251482172725,-1.0999748867047898),(-0.3322630208172874,-0.7274920231630502,-0.0766000050337147,0.5966532121963541,0.6101272780073337,0.25366403120093184,-1.0999748867047898,0.76,-0.8558554370756518,-0.39756366798706344,-0.952028633990455,-1.3921908284581592,0.052933416813421515,-0.8377251482172725,-1.0999748867047898),(-0.07046681268810527,0.4272572199529034,-0.0766000050337147,0.09944220203272576,0.6101272780073337,-0.06535742124438843,-1.0999748867047898,0.75,-0.8558554370756518,-0.39756366798706344,-0.952028633990455,-1.3921908284581592,0.052933416813421515,-0.8377251482172725,-1.0999748867047898),(0.6276564089897139,0.9221497527168835,0.7988286239230257,1.0938642223599824,0.6101272780073337,0.32082644224205065,-1.0999748867047898,0.76,-1.8157748668826532,-1.7172770886910105,-0.952028633990455,-1.8894018386217877,-1.618648166768315,-1.3414432310256739,-1.0999748867047898),(-0.7685900343659244,0.26229304236491,-0.952028633990455,-0.3977688081309026,0.6101272780073337,-0.2500540516074711,-1.0999748867047898,0.71,-1.8157748668826532,-1.7172770886910105,-0.952028633990455,-1.8894018386217877,-1.618648166768315,-1.3414432310256739,-1.0999748867047898),(-1.0303862424951067,-0.39756366798706344,-0.952028633990455,-0.3977688081309026,-1.618648166768315,-0.6194473123336305,-1.0999748867047898,0.67,-1.8157748668826532,-1.7172770886910105,-0.952028633990455,-1.8894018386217877,-1.618648166768315,-1.3414432310256739,-1.0999748867047898),(-1.8157748668826532,-1.3873487335150236,-0.952028633990455,-0.8949798182945309,-1.0614543055744028,-0.9552593675392334,-1.0999748867047898,0.61,-1.8157748668826532,-1.7172770886910105,-0.952028633990455,-1.8894018386217877,-1.618648166768315,-1.3414432310256739,-1.0999748867047898),(-0.9431208397853792,0.4272572199529034,-0.952028633990455,0.09944220203272576,0.6101272780073337,-0.2500540516074711,-1.0999748867047898,0.63,-1.8157748668826532,-1.7172770886910105,-0.952028633990455,-1.8894018386217877,-1.618648166768315,-1.3414432310256739,-1.0999748867047898),(-0.41952842352701486,0.4272572199529034,-0.952028633990455,0.09944220203272576,-0.5042604443804907,-0.1157292295252305,-1.0999748867047898,0.64,-1.8157748668826532,-1.7172770886910105,-0.952028633990455,-1.8894018386217877,-1.618648166768315,-1.3414432310256739,-1.0999748867047898),(0.10406399273134952,0.7571855751288902,-0.0766000050337147,0.5966532121963541,0.052933416813421515,0.11933920911869125,-1.0999748867047898,0.71,-1.8157748668826532,-1.7172770886910105,-0.952028633990455,-1.8894018386217877,-1.618648166768315,-1.3414432310256739,-1.0999748867047898),(0.6276564089897139,0.5922213975408968,0.7988286239230257,-0.3977688081309026,-0.5042604443804907,0.690219702968213,0.909111664354187,0.82,-1.8157748668826532,-1.7172770886910105,-0.952028633990455,-1.8894018386217877,-1.618648166768315,-1.3414432310256739,-1.0999748867047898),(-0.3322630208172874,-0.5625278455750569,-0.0766000050337147,0.5966532121963541,1.167321139201246,0.08575800359813185,-1.0999748867047898,0.73,-1.8157748668826532,-1.7172770886910105,-0.952028633990455,-1.8894018386217877,-1.618648166768315,-1.3414432310256739,-1.0999748867047898),(0.1913293954410769,-0.2325994903990701,-0.0766000050337147,0.09944220203272576,-1.0614543055744028,-0.45154128473083044,0.909111664354187,0.74,-1.8157748668826532,-1.7172770886910105,-0.952028633990455,-1.8894018386217877,-1.618648166768315,-1.3414432310256739,-1.0999748867047898),(-0.41952842352701486,-0.06763531281107672,-0.952028633990455,-0.8949798182945309,0.052933416813421515,-0.5522849012925116,-1.0999748867047898,0.69,-1.8157748668826532,-1.7172770886910105,-0.952028633990455,-1.8894018386217877,-1.618648166768315,-1.3414432310256739,-1.0999748867047898),(-1.117651645204834,-1.2223845559270303,-0.952028633990455,-0.8949798182945309,0.052933416813421515,-0.8880969564981116,-1.0999748867047898,0.64,-1.8157748668826532,-1.7172770886910105,-0.952028633990455,-1.8894018386217877,-1.618648166768315,-1.3414432310256739,-1.0999748867047898),(1.1512488252480781,0.9221497527168835,1.674257252879766,1.5910752325236108,0.6101272780073337,1.1939377857766158,0.909111664354187,0.91,-1.8157748668826532,-1.7172770886910105,-0.952028633990455,-1.8894018386217877,-1.618648166768315,-1.3414432310256739,-1.0999748867047898),(0.8021872144091686,0.5922213975408968,1.674257252879766,1.0938642223599824,0.6101272780073337,1.0596129636943752,0.909111664354187,0.88,0.2785947981508043,0.09732886477691666,-0.0766000050337147,0.09944220203272576,0.6101272780073337,-0.2668446543677508,0.909111664354187),(0.7149218116994412,0.7571855751288902,0.7988286239230257,0.5966532121963541,1.167321139201246,0.9588693471326941,0.909111664354187,0.85,0.2785947981508043,0.09732886477691666,-0.0766000050337147,0.09944220203272576,0.6101272780073337,-0.2668446543677508,0.909111664354187),(1.0639834225383509,1.087113930304877,1.674257252879766,1.0938642223599824,1.724515000395158,0.9924505526532535,0.909111664354187,0.86,0.2785947981508043,0.09732886477691666,-0.0766000050337147,0.09944220203272576,0.6101272780073337,-0.2668446543677508,0.909111664354187),(-0.5940592289464697,-0.5625278455750569,-0.0766000050337147,-1.3921908284581592,0.052933416813421515,-0.3843788736897117,-1.0999748867047898,0.7,0.2785947981508043,0.09732886477691666,-0.0766000050337147,0.09944220203272576,0.6101272780073337,-0.2668446543677508,0.909111664354187),(-1.553978658753471,-1.2223845559270303,-1.8274572629471952,-1.8894018386217877,-1.618648166768315,-1.1903278061831537,-1.0999748867047898,0.59,0.2785947981508043,0.09732886477691666,-0.0766000050337147,0.09944220203272576,0.6101272780073337,-0.2668446543677508,0.909111664354187),(-1.8157748668826532,-1.057420378339037,-1.8274572629471952,-0.8949798182945309,-0.5042604443804907,-1.5429304641490347,-1.0999748867047898,0.6,0.2785947981508043,0.09732886477691666,-0.0766000050337147,0.09944220203272576,0.6101272780073337,-0.2668446543677508,0.909111664354187),(0.016798590021622126,-0.7274920231630502,-0.952028633990455,-0.8949798182945309,-1.618648166768315,-0.753772134415871,-1.0999748867047898,0.65,0.2785947981508043,0.09732886477691666,-0.0766000050337147,0.09944220203272576,0.6101272780073337,-0.2668446543677508,0.909111664354187),(0.6276564089897139,1.2520781078928702,-0.0766000050337147,0.09944220203272576,-0.5042604443804907,0.27045463396121155,0.909111664354187,0.7,0.2785947981508043,0.09732886477691666,-0.0766000050337147,0.09944220203272576,0.6101272780073337,-0.2668446543677508,0.909111664354187),(0.7149218116994412,1.087113930304877,-0.0766000050337147,0.09944220203272576,-0.5042604443804907,0.7405915112490521,0.909111664354187,0.76,0.2785947981508043,0.09732886477691666,-0.0766000050337147,0.09944220203272576,0.6101272780073337,-0.2668446543677508,0.909111664354187),(-0.24499761810756004,-0.06763531281107672,-0.952028633990455,-0.8949798182945309,0.6101272780073337,-0.06535742124438843,-1.0999748867047898,0.63,0.2785947981508043,0.09732886477691666,-0.0766000050337147,0.09944220203272576,0.6101272780073337,-0.2668446543677508,0.909111664354187),(0.9767180198286235,0.4272572199529034,0.7988286239230257,0.5966532121963541,-1.0614543055744028,0.7070103057284927,0.909111664354187,0.81,0.2785947981508043,0.09732886477691666,-0.0766000050337147,0.09944220203272576,0.6101272780073337,-0.2668446543677508,0.909111664354187),(-0.07046681268810527,-0.39756366798706344,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.22008282568037243,-1.0999748867047898,0.72,0.2785947981508043,0.09732886477691666,-0.0766000050337147,0.09944220203272576,0.6101272780073337,-0.2668446543677508,0.909111664354187),(-0.5067938262367422,-0.5625278455750569,-0.952028633990455,-0.8949798182945309,0.052933416813421515,-0.199682243326629,-1.0999748867047898,0.71,-0.24499761810756004,-0.8924562007510436,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-0.6026567095733507,-1.0999748867047898),(0.6276564089897139,0.4272572199529034,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.4551512643242912,0.909111664354187,0.8,-0.24499761810756004,-0.8924562007510436,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-0.6026567095733507,-1.0999748867047898),(0.36586020086053167,0.5922213975408968,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.3879888532831724,0.909111664354187,0.77,-0.24499761810756004,-0.8924562007510436,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-0.6026567095733507,-1.0999748867047898),(0.2785947981508043,-0.5625278455750569,-0.0766000050337147,-0.3977688081309026,-1.0614543055744028,-0.04856681848410872,0.909111664354187,0.74,-0.24499761810756004,-0.8924562007510436,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-0.6026567095733507,-1.0999748867047898),(-0.07046681268810527,-1.3873487335150236,-0.952028633990455,-0.8949798182945309,-0.5042604443804907,0.6734291002079332,-1.0999748867047898,0.7,-0.24499761810756004,-0.8924562007510436,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-0.6026567095733507,-1.0999748867047898),(0.10406399273134952,-1.2223845559270303,-0.952028633990455,-0.8949798182945309,0.052933416813421515,-0.0989386267649508,0.909111664354187,0.71,-0.24499761810756004,-0.8924562007510436,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-0.6026567095733507,-1.0999748867047898),(1.5875758387967152,1.2520781078928702,0.7988286239230257,1.0938642223599824,1.167321139201246,1.8151900879069767,0.909111664354187,0.93,-0.24499761810756004,-0.8924562007510436,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-0.6026567095733507,-1.0999748867047898),(0.36586020086053167,1.087113930304877,0.7988286239230257,0.5966532121963541,1.724515000395158,0.8749163333312926,-1.0999748867047898,0.85,-0.24499761810756004,-0.8924562007510436,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-0.6026567095733507,-1.0999748867047898),(-0.8558554370756518,0.4272572199529034,0.7988286239230257,0.5966532121963541,1.167321139201246,-0.3843788736897117,-1.0999748867047898,0.79,-0.24499761810756004,-0.8924562007510436,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-0.6026567095733507,-1.0999748867047898),(-0.681324631656197,-1.3873487335150236,-0.0766000050337147,0.5966532121963541,0.6101272780073337,-0.06535742124438843,-1.0999748867047898,0.76,-0.24499761810756004,-0.8924562007510436,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-0.6026567095733507,-1.0999748867047898),(0.6276564089897139,-1.2223845559270303,-0.0766000050337147,0.5966532121963541,1.724515000395158,0.06896740083785215,0.909111664354187,0.78,-0.24499761810756004,-0.8924562007510436,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-0.6026567095733507,-1.0999748867047898),(0.8021872144091686,-0.8924562007510436,0.7988286239230257,1.5910752325236108,1.724515000395158,0.27045463396121155,0.909111664354187,0.77,-0.24499761810756004,-0.8924562007510436,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-0.6026567095733507,-1.0999748867047898),(1.2385142279578056,1.9119348182448437,0.7988286239230257,1.5910752325236108,1.167321139201246,1.244309594057455,0.909111664354187,0.9,0.10406399273134952,-0.2325994903990701,-0.0766000050337147,-1.3921908284581592,-0.5042604443804907,0.08575800359813185,-1.0999748867047898),(0.889452617118896,0.09732886477691666,1.674257252879766,1.5910752325236108,0.052933416813421515,0.8917069360915754,0.909111664354187,0.87,0.10406399273134952,-0.2325994903990701,-0.0766000050337147,-1.3921908284581592,-0.5042604443804907,0.08575800359813185,-1.0999748867047898),(-0.41952842352701486,-0.5625278455750569,-0.0766000050337147,0.09944220203272576,0.6101272780073337,-0.8545157509775522,-1.0999748867047898,0.71,0.10406399273134952,-0.2325994903990701,-0.0766000050337147,-1.3921908284581592,-0.5042604443804907,0.08575800359813185,-1.0999748867047898),(-0.7685900343659244,-0.7274920231630502,-0.952028633990455,-0.8949798182945309,0.6101272780073337,-0.40116947644999135,0.909111664354187,0.7,0.10406399273134952,-0.2325994903990701,-0.0766000050337147,-1.3921908284581592,-0.5042604443804907,0.08575800359813185,-1.0999748867047898),(0.6276564089897139,0.5922213975408968,-0.0766000050337147,-0.8949798182945309,-2.175842027962227,0.32082644224205065,0.909111664354187,0.7,0.10406399273134952,-0.2325994903990701,-0.0766000050337147,-1.3921908284581592,-0.5042604443804907,0.08575800359813185,-1.0999748867047898),(0.7149218116994412,0.4272572199529034,-0.952028633990455,-0.3977688081309026,-1.0614543055744028,0.27045463396121155,0.909111664354187,0.75,0.10406399273134952,-0.2325994903990701,-0.0766000050337147,-1.3921908284581592,-0.5042604443804907,0.08575800359813185,-1.0999748867047898),(-0.3322630208172874,-0.8924562007510436,-0.0766000050337147,-0.8949798182945309,-1.0614543055744028,0.13612981187897094,-1.0999748867047898,0.71,0.10406399273134952,-0.2325994903990701,-0.0766000050337147,-1.3921908284581592,-0.5042604443804907,0.08575800359813185,-1.0999748867047898),(-0.41952842352701486,-0.39756366798706344,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-0.2500540516074711,-1.0999748867047898,0.72,0.10406399273134952,-0.2325994903990701,-0.0766000050337147,-1.3921908284581592,-0.5042604443804907,0.08575800359813185,-1.0999748867047898),(-0.24499761810756004,-0.06763531281107672,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,-0.7201909288953117,0.909111664354187,0.73,0.10406399273134952,-0.2325994903990701,-0.0766000050337147,-1.3921908284581592,-0.5042604443804907,0.08575800359813185,-1.0999748867047898),(0.889452617118896,0.9221497527168835,0.7988286239230257,1.0938642223599824,1.724515000395158,0.908497538851855,-1.0999748867047898,0.83,0.10406399273134952,-0.2325994903990701,-0.0766000050337147,-1.3921908284581592,-0.5042604443804907,0.08575800359813185,-1.0999748867047898),(-0.7685900343659244,0.09732886477691666,0.7988286239230257,1.0938642223599824,1.724515000395158,-0.4347506819705508,-1.0999748867047898,0.77,0.10406399273134952,-0.2325994903990701,-0.0766000050337147,-1.3921908284581592,-0.5042604443804907,0.08575800359813185,-1.0999748867047898),(-0.9431208397853792,-0.39756366798706344,-0.952028633990455,-0.8949798182945309,-0.5042604443804907,-0.6362379150939101,0.909111664354187,0.72,0.10406399273134952,-0.2325994903990701,-0.0766000050337147,-1.3921908284581592,-0.5042604443804907,0.08575800359813185,-1.0999748867047898),(-1.553978658753471,-1.8822412662790038,-0.952028633990455,-1.8894018386217877,-1.618648166768315,-1.2406996144639928,-1.0999748867047898,0.54,0.10406399273134952,-0.2325994903990701,-0.0766000050337147,-1.3921908284581592,-0.5042604443804907,0.08575800359813185,-1.0999748867047898),(-1.990305672302108,-2.0472054438669973,-1.8274572629471952,-1.8894018386217877,-2.175842027962227,-1.610092875190155,-1.0999748867047898,0.49,0.8021872144091686,0.7571855751288902,0.7988286239230257,0.5966532121963541,0.052933416813421515,0.8749163333312926,0.909111664354187),(-0.41952842352701486,-1.3873487335150236,-1.8274572629471952,-2.386612848785416,-2.175842027962227,-0.9888405730597928,0.909111664354187,0.52,0.8021872144091686,0.7571855751288902,0.7988286239230257,0.5966532121963541,0.052933416813421515,0.8749163333312926,0.909111664354187),(-0.15773221539783266,-1.2223845559270303,-1.8274572629471952,-1.3921908284581592,-1.0614543055744028,-1.0895841896214724,-1.0999748867047898,0.58,0.8021872144091686,0.7571855751288902,0.7988286239230257,0.5966532121963541,0.052933416813421515,0.8749163333312926,0.909111664354187),(0.4531256035702591,0.4272572199529034,-0.0766000050337147,0.09944220203272576,-0.5042604443804907,0.6062666891668145,0.909111664354187,0.78,0.8021872144091686,0.7571855751288902,0.7988286239230257,0.5966532121963541,0.052933416813421515,0.8749163333312926,0.909111664354187),(1.0639834225383509,0.9221497527168835,1.674257252879766,1.5910752325236108,1.167321139201246,1.4290062244205346,0.909111664354187,0.89,0.8021872144091686,0.7571855751288902,0.7988286239230257,0.5966532121963541,0.052933416813421515,0.8749163333312926,0.909111664354187),(0.2785947981508043,-1.057420378339037,-0.952028633990455,-0.8949798182945309,-0.5042604443804907,0.03538619531728977,-1.0999748867047898,0.7,0.8021872144091686,0.7571855751288902,0.7988286239230257,0.5966532121963541,0.052933416813421515,0.8749163333312926,0.909111664354187),(-0.7685900343659244,-0.7274920231630502,-0.952028633990455,-0.3977688081309026,0.052933416813421515,-0.1828916405663493,-1.0999748867047898,0.66,0.8021872144091686,0.7571855751288902,0.7988286239230257,0.5966532121963541,0.052933416813421515,0.8749163333312926,0.909111664354187),(-1.117651645204834,-0.8924562007510436,-0.952028633990455,-0.3977688081309026,0.6101272780073337,0.22008282568037243,-1.0999748867047898,0.67,0.8021872144091686,0.7571855751288902,0.7988286239230257,0.5966532121963541,0.052933416813421515,0.8749163333312926,0.909111664354187),(-0.5067938262367422,-0.8924562007510436,-0.0766000050337147,1.0938642223599824,0.6101272780073337,0.06896740083785215,0.909111664354187,0.68,0.8021872144091686,0.7571855751288902,0.7988286239230257,0.5966532121963541,0.052933416813421515,0.8749163333312926,0.909111664354187),(0.016798590021622126,0.4272572199529034,-0.0766000050337147,0.5966532121963541,1.167321139201246,0.8581257305710129,0.909111664354187,0.8,0.8021872144091686,0.7571855751288902,0.7988286239230257,0.5966532121963541,0.052933416813421515,0.8749163333312926,0.909111664354187),(-0.41952842352701486,-0.2325994903990701,-0.0766000050337147,0.5966532121963541,0.052933416813421515,0.32082644224205065,0.909111664354187,0.81,0.8021872144091686,0.7571855751288902,0.7988286239230257,0.5966532121963541,0.052933416813421515,0.8749163333312926,0.909111664354187),(0.36586020086053167,0.5922213975408968,-0.0766000050337147,-0.8949798182945309,-0.5042604443804907,0.5055230726051333,0.909111664354187,0.8,0.8021872144091686,0.7571855751288902,0.7988286239230257,0.5966532121963541,0.052933416813421515,0.8749163333312926,0.909111664354187),(2.023902852345352,0.7571855751288902,0.7988286239230257,1.5910752325236108,1.167321139201246,1.7816088823864173,0.909111664354187,0.94,0.016798590021622126,-0.5625278455750569,-0.952028633990455,-0.3977688081309026,-0.5042604443804907,0.27045463396121155,-1.0999748867047898),(1.2385142279578056,1.4170422854808635,1.674257252879766,0.5966532121963541,0.6101272780073337,1.1099847719752143,0.909111664354187,0.93,0.016798590021622126,-0.5625278455750569,-0.952028633990455,-0.3977688081309026,-0.5042604443804907,0.27045463396121155,-1.0999748867047898),(1.6748412415064426,1.7469706406568504,1.674257252879766,1.0938642223599824,0.6101272780073337,0.9924505526532535,0.909111664354187,0.92,0.016798590021622126,-0.5625278455750569,-0.952028633990455,-0.3977688081309026,-0.5042604443804907,0.27045463396121155,-1.0999748867047898),(0.6276564089897139,1.087113930304877,1.674257252879766,1.5910752325236108,1.167321139201246,0.8077539222901738,0.909111664354187,0.89,0.016798590021622126,-0.5625278455750569,-0.952028633990455,-0.3977688081309026,-0.5042604443804907,0.27045463396121155,-1.0999748867047898),(-0.24499761810756004,-0.5625278455750569,0.7988286239230257,1.5910752325236108,1.724515000395158,0.7070103057284927,-1.0999748867047898,0.82,0.016798590021622126,-0.5625278455750569,-0.952028633990455,-0.3977688081309026,-0.5042604443804907,0.27045463396121155,-1.0999748867047898),(-0.3322630208172874,0.26229304236491,-0.0766000050337147,0.5966532121963541,0.052933416813421515,0.6734291002079332,-1.0999748867047898,0.79,0.016798590021622126,-0.5625278455750569,-0.952028633990455,-0.3977688081309026,-0.5042604443804907,0.27045463396121155,-1.0999748867047898),(-0.8558554370756518,-0.39756366798706344,-0.952028633990455,-0.8949798182945309,-0.5042604443804907,-1.5933022724298738,-1.0999748867047898,0.58,0.016798590021622126,-0.5625278455750569,-0.952028633990455,-0.3977688081309026,-0.5042604443804907,0.27045463396121155,-1.0999748867047898),(-1.4667132560437435,-0.8924562007510436,-0.952028633990455,-1.8894018386217877,-1.618648166768315,-1.223909011703713,-1.0999748867047898,0.56,0.016798590021622126,-0.5625278455750569,-0.952028633990455,-0.3977688081309026,-0.5042604443804907,0.27045463396121155,-1.0999748867047898),(-1.2921824506242887,-1.3873487335150236,-0.952028633990455,-2.386612848785416,-1.618648166768315,-1.056002984100913,-1.0999748867047898,0.56,0.016798590021622126,-0.5625278455750569,-0.952028633990455,-0.3977688081309026,-0.5042604443804907,0.27045463396121155,-1.0999748867047898),(-0.41952842352701486,-1.5523129111030172,-1.8274572629471952,0.09944220203272576,-0.5042604443804907,-0.7034003261350319,0.909111664354187,0.64,0.016798590021622126,-0.5625278455750569,-0.952028633990455,-0.3977688081309026,-0.5042604443804907,0.27045463396121155,-1.0999748867047898),(-0.07046681268810527,-1.057420378339037,-0.952028633990455,-0.8949798182945309,-1.618648166768315,-0.46833188749111015,0.909111664354187,0.61,0.016798590021622126,-0.5625278455750569,-0.952028633990455,-0.3977688081309026,-0.5042604443804907,0.27045463396121155,-1.0999748867047898),(0.016798590021622126,-1.2223845559270303,-0.952028633990455,-0.3977688081309026,-1.0614543055744028,-0.04856681848410872,-1.0999748867047898,0.68,0.016798590021622126,-0.5625278455750569,-0.952028633990455,-0.3977688081309026,-0.5042604443804907,0.27045463396121155,-1.0999748867047898),(-0.5940592289464697,-0.06763531281107672,-0.0766000050337147,0.09944220203272576,0.052933416813421515,0.11933920911869125,-1.0999748867047898,0.76,1.0639834225383509,0.5922213975408968,0.7988286239230257,1.0938642223599824,0.6101272780073337,1.0596129636943752,0.909111664354187),(0.2785947981508043,2.076898995832837,-0.0766000050337147,0.5966532121963541,1.167321139201246,0.8581257305710129,-1.0999748867047898,0.86,1.0639834225383509,0.5922213975408968,0.7988286239230257,1.0938642223599824,0.6101272780073337,1.0596129636943752,0.909111664354187),(1.1512488252480781,1.087113930304877,-0.0766000050337147,1.0938642223599824,1.167321139201246,1.076403566454655,0.909111664354187,0.9,1.0639834225383509,0.5922213975408968,0.7988286239230257,1.0938642223599824,0.6101272780073337,1.0596129636943752,0.909111664354187),(-1.0303862424951067,0.7571855751288902,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.08575800359813185,-1.0999748867047898,0.71,1.0639834225383509,0.5922213975408968,0.7988286239230257,1.0938642223599824,0.6101272780073337,1.0596129636943752,0.909111664354187),(-0.681324631656197,-0.2325994903990701,-0.952028633990455,-0.8949798182945309,-1.0614543055744028,-1.0056311758200724,-1.0999748867047898,0.62,1.0639834225383509,0.5922213975408968,0.7988286239230257,1.0938642223599824,0.6101272780073337,1.0596129636943752,0.909111664354187),(0.1913293954410769,0.09732886477691666,-0.952028633990455,-0.8949798182945309,-0.5042604443804907,0.27045463396121155,-1.0999748867047898,0.66,1.0639834225383509,0.5922213975408968,0.7988286239230257,1.0938642223599824,0.6101272780073337,1.0596129636943752,0.909111664354187),(0.4531256035702591,-0.39756366798706344,-0.952028633990455,-0.3977688081309026,-0.5042604443804907,-0.2500540516074711,0.909111664354187,0.65,1.0639834225383509,0.5922213975408968,0.7988286239230257,1.0938642223599824,0.6101272780073337,1.0596129636943752,0.909111664354187),(0.5403910062799865,-0.06763531281107672,-0.0766000050337147,0.09944220203272576,0.052933416813421515,-0.08214802400466813,0.909111664354187,0.73,1.0639834225383509,0.5922213975408968,0.7988286239230257,1.0938642223599824,0.6101272780073337,1.0596129636943752,0.909111664354187),(-0.3322630208172874,-0.2325994903990701,-0.952028633990455,-0.8949798182945309,-1.618648166768315,-0.2836352571280305,-1.0999748867047898,0.62,1.0639834225383509,0.5922213975408968,0.7988286239230257,1.0938642223599824,0.6101272780073337,1.0596129636943752,0.909111664354187),(0.36586020086053167,0.26229304236491,-0.0766000050337147,0.09944220203272576,0.052933416813421515,0.3376170450023333,0.909111664354187,0.74,1.0639834225383509,0.5922213975408968,0.7988286239230257,1.0938642223599824,0.6101272780073337,1.0596129636943752,0.909111664354187),(0.5403910062799865,0.4272572199529034,-0.0766000050337147,0.5966532121963541,0.052933416813421515,0.8413351278107333,0.909111664354187,0.79,1.0639834225383509,0.5922213975408968,0.7988286239230257,1.0938642223599824,0.6101272780073337,1.0596129636943752,0.909111664354187),(0.7149218116994412,0.7571855751288902,0.7988286239230257,0.5966532121963541,0.6101272780073337,0.6734291002079332,0.909111664354187,0.8,1.0639834225383509,0.5922213975408968,0.7988286239230257,1.0938642223599824,0.6101272780073337,1.0596129636943752,0.909111664354187),(-0.41952842352701486,0.09732886477691666,-0.0766000050337147,0.09944220203272576,-0.5042604443804907,-0.1157292295252305,-1.0999748867047898,0.69,1.0639834225383509,0.5922213975408968,0.7988286239230257,1.0938642223599824,0.6101272780073337,1.0596129636943752,0.909111664354187),(-0.7685900343659244,0.4272572199529034,0.7988286239230257,0.09944220203272576,-0.5042604443804907,0.0018049897967303734,-1.0999748867047898,0.7,0.6276564089897139,0.4272572199529034,-0.0766000050337147,0.09944220203272576,0.052933416813421515,0.7405915112490521,0.909111664354187),(0.2785947981508043,-0.5625278455750569,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.23687342844065212,0.909111664354187,0.76,0.6276564089897139,0.4272572199529034,-0.0766000050337147,0.09944220203272576,0.052933416813421515,0.7405915112490521,0.909111664354187),(0.9767180198286235,0.09732886477691666,0.7988286239230257,1.0938642223599824,0.6101272780073337,0.9756599498929738,0.909111664354187,0.84,0.6276564089897139,0.4272572199529034,-0.0766000050337147,0.09944220203272576,0.052933416813421515,0.7405915112490521,0.909111664354187),(-0.5067938262367422,-0.06763531281107672,0.7988286239230257,1.0938642223599824,1.167321139201246,0.6734291002079332,0.909111664354187,0.78,0.6276564089897139,0.4272572199529034,-0.0766000050337147,0.09944220203272576,0.052933416813421515,0.7405915112490521,0.909111664354187),(-1.3794478533340162,-1.2223845559270303,-0.0766000050337147,0.09944220203272576,-0.5042604443804907,-0.9384687647789537,-1.0999748867047898,0.67,0.6276564089897139,0.4272572199529034,-0.0766000050337147,0.09944220203272576,0.052933416813421515,0.7405915112490521,0.909111664354187),(-1.0303862424951067,-0.39756366798706344,-0.952028633990455,-0.3977688081309026,0.6101272780073337,-0.7873533399364304,-1.0999748867047898,0.66,0.6276564089897139,0.4272572199529034,-0.0766000050337147,0.09944220203272576,0.052933416813421515,0.7405915112490521,0.909111664354187),(-0.7685900343659244,-0.5625278455750569,-0.952028633990455,-0.8949798182945309,-0.5042604443804907,-0.8880969564981116,-1.0999748867047898,0.65,0.6276564089897139,0.4272572199529034,-0.0766000050337147,0.09944220203272576,0.052933416813421515,0.7405915112490521,0.909111664354187),(-1.6412440614631982,-1.057420378339037,-0.952028633990455,-1.8894018386217877,-1.618648166768315,-1.2406996144639928,-1.0999748867047898,0.54,0.6276564089897139,0.4272572199529034,-0.0766000050337147,0.09944220203272576,0.052933416813421515,0.7405915112490521,0.909111664354187),(-1.4667132560437435,-1.3873487335150236,-1.8274572629471952,-2.386612848785416,-1.0614543055744028,-0.9888405730597928,-1.0999748867047898,0.58,0.6276564089897139,0.4272572199529034,-0.0766000050337147,0.09944220203272576,0.052933416813421515,0.7405915112490521,0.909111664354187),(0.6276564089897139,0.5922213975408968,-0.0766000050337147,-0.8949798182945309,-1.618648166768315,0.3376170450023333,0.909111664354187,0.79,0.6276564089897139,0.4272572199529034,-0.0766000050337147,0.09944220203272576,0.052933416813421515,0.7405915112490521,0.909111664354187),(0.889452617118896,0.9221497527168835,0.7988286239230257,0.09944220203272576,-0.5042604443804907,0.15292041463925066,0.909111664354187,0.8,0.6276564089897139,0.4272572199529034,-0.0766000050337147,0.09944220203272576,0.052933416813421515,0.7405915112490521,0.909111664354187),(0.016798590021622126,-0.2325994903990701,-0.0766000050337147,0.5966532121963541,0.052933416813421515,-0.1661010378060696,0.909111664354187,0.75,0.6276564089897139,0.4272572199529034,-0.0766000050337147,0.09944220203272576,0.052933416813421515,0.7405915112490521,0.909111664354187),(0.5403910062799865,-0.5625278455750569,-0.0766000050337147,0.5966532121963541,0.6101272780073337,-0.2668446543677508,0.909111664354187,0.73,0.7149218116994412,-0.06763531281107672,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.8581257305710129,0.909111664354187),(-0.24499761810756004,-0.06763531281107672,-0.952028633990455,-0.8949798182945309,0.6101272780073337,-0.5522849012925116,-1.0999748867047898,0.72,0.7149218116994412,-0.06763531281107672,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.8581257305710129,0.909111664354187),(-1.0303862424951067,-0.8924562007510436,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-0.7034003261350319,-1.0999748867047898,0.62,0.7149218116994412,-0.06763531281107672,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.8581257305710129,0.909111664354187),(-0.15773221539783266,-0.5625278455750569,-0.0766000050337147,-0.3977688081309026,-1.0614543055744028,-0.45154128473083044,-1.0999748867047898,0.67,0.7149218116994412,-0.06763531281107672,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.8581257305710129,0.909111664354187),(0.8021872144091686,1.4170422854808635,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.908497538851855,0.909111664354187,0.81,0.7149218116994412,-0.06763531281107672,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.8581257305710129,0.909111664354187),(-1.553978658753471,-1.2223845559270303,-0.0766000050337147,-1.3921908284581592,-1.618648166768315,-0.972049970299513,-1.0999748867047898,0.63,0.7149218116994412,-0.06763531281107672,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.8581257305710129,0.909111664354187),(-1.9030402695923805,-1.057420378339037,-0.952028633990455,-0.8949798182945309,-1.618648166768315,-1.2406996144639928,-1.0999748867047898,0.69,0.7149218116994412,-0.06763531281107672,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.8581257305710129,0.909111664354187),(0.6276564089897139,0.7571855751288902,0.7988286239230257,0.5966532121963541,0.052933416813421515,0.2872452367214912,0.909111664354187,0.8,0.7149218116994412,-0.06763531281107672,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.8581257305710129,0.909111664354187),(-1.7285094641729257,-1.8822412662790038,-0.952028633990455,-0.8949798182945309,-2.175842027962227,-1.1903278061831537,-1.0999748867047898,0.43,0.7149218116994412,-0.06763531281107672,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.8581257305710129,0.909111664354187),(0.889452617118896,0.9221497527168835,-0.0766000050337147,0.09944220203272576,-0.5042604443804907,0.10254860635841155,0.909111664354187,0.8,0.7149218116994412,-0.06763531281107672,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.8581257305710129,0.909111664354187),(-0.5067938262367422,-0.39756366798706344,-0.952028633990455,-0.3977688081309026,-1.618648166768315,-0.8041439426967131,0.909111664354187,0.73,0.7149218116994412,-0.06763531281107672,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.8581257305710129,0.909111664354187),(-0.7685900343659244,-0.2325994903990701,-0.0766000050337147,0.09944220203272576,-1.0614543055744028,-0.6530285178541898,0.909111664354187,0.75,0.7149218116994412,-0.06763531281107672,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.8581257305710129,0.909111664354187),(0.1913293954410769,0.09732886477691666,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,-0.0989386267649508,0.909111664354187,0.71,1.1512488252480781,1.4170422854808635,0.7988286239230257,1.5910752325236108,1.167321139201246,1.4290062244205346,0.909111664354187),(-0.41952842352701486,-0.06763531281107672,0.7988286239230257,1.0938642223599824,0.6101272780073337,0.08575800359813185,0.909111664354187,0.73,1.1512488252480781,1.4170422854808635,0.7988286239230257,1.5910752325236108,1.167321139201246,1.4290062244205346,0.909111664354187),(0.7149218116994412,0.5922213975408968,0.7988286239230257,0.5966532121963541,1.167321139201246,0.8581257305710129,0.909111664354187,0.83,1.1512488252480781,1.4170422854808635,0.7988286239230257,1.5910752325236108,1.167321139201246,1.4290062244205346,0.909111664354187),(0.1913293954410769,0.4272572199529034,-0.0766000050337147,-0.3977688081309026,-1.0614543055744028,0.32082644224205065,-1.0999748867047898,0.72,1.1512488252480781,1.4170422854808635,0.7988286239230257,1.5910752325236108,1.167321139201246,1.4290062244205346,0.909111664354187),(1.3257796306675331,1.7469706406568504,1.674257252879766,1.5910752325236108,1.724515000395158,1.462587429941097,0.909111664354187,0.94,1.1512488252480781,1.4170422854808635,0.7988286239230257,1.5910752325236108,1.167321139201246,1.4290062244205346,0.909111664354187),(0.5403910062799865,0.09732886477691666,1.674257252879766,0.5966532121963541,0.6101272780073337,0.23687342844065212,0.909111664354187,0.81,1.1512488252480781,1.4170422854808635,0.7988286239230257,1.5910752325236108,1.167321139201246,1.4290062244205346,0.909111664354187),(0.6276564089897139,-0.06763531281107672,1.674257252879766,0.09944220203272576,0.6101272780073337,0.10254860635841155,0.909111664354187,0.81,1.1512488252480781,1.4170422854808635,0.7988286239230257,1.5910752325236108,1.167321139201246,1.4290062244205346,0.909111664354187),(-0.41952842352701486,-0.06763531281107672,-0.0766000050337147,-0.3977688081309026,-0.5042604443804907,-0.2332634488471884,0.909111664354187,0.75,1.1512488252480781,1.4170422854808635,0.7988286239230257,1.5910752325236108,1.167321139201246,1.4290062244205346,0.909111664354187),(0.8021872144091686,0.4272572199529034,-0.0766000050337147,0.09944220203272576,0.052933416813421515,0.27045463396121155,0.909111664354187,0.79,1.1512488252480781,1.4170422854808635,0.7988286239230257,1.5910752325236108,1.167321139201246,1.4290062244205346,0.909111664354187),(-0.7685900343659244,-0.2325994903990701,-0.0766000050337147,-0.3977688081309026,-0.5042604443804907,-0.6026567095733507,-1.0999748867047898,0.58,1.1512488252480781,1.4170422854808635,0.7988286239230257,1.5910752325236108,1.167321139201246,1.4290062244205346,0.909111664354187),(-1.0303862424951067,-0.7274920231630502,-0.952028633990455,-0.8949798182945309,0.052933416813421515,-0.7873533399364304,-1.0999748867047898,0.59,1.1512488252480781,1.4170422854808635,0.7988286239230257,1.5910752325236108,1.167321139201246,1.4290062244205346,0.909111664354187),(-1.9030402695923805,-1.8822412662790038,-0.952028633990455,-1.8894018386217877,-1.618648166768315,-2.1138109579985565,-1.0999748867047898,0.47,1.1512488252480781,1.4170422854808635,0.7988286239230257,1.5910752325236108,1.167321139201246,1.4290062244205346,0.909111664354187),(-0.07046681268810527,-1.5523129111030172,-1.8274572629471952,-1.8894018386217877,-1.618648166768315,-1.9626955331560363,-1.0999748867047898,0.49,1.1512488252480781,1.4170422854808635,0.7988286239230257,1.5910752325236108,1.167321139201246,1.4290062244205346,0.909111664354187),(-1.117651645204834,-1.7172770886910105,-0.952028633990455,-1.8894018386217877,-1.618648166768315,-1.610092875190155,-1.0999748867047898,0.47,-0.41952842352701486,-0.7274920231630502,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.30403583948177093,-1.0999748867047898),(-1.553978658753471,-2.2121696214549904,-1.8274572629471952,-2.386612848785416,-2.7330358891561395,-2.1138109579985565,-1.0999748867047898,0.42,-0.41952842352701486,-0.7274920231630502,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.30403583948177093,-1.0999748867047898),(-1.2921824506242887,-1.3873487335150236,-1.8274572629471952,-1.3921908284581592,-1.618648166768315,-2.2649263828410766,-1.0999748867047898,0.57,-0.41952842352701486,-0.7274920231630502,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.30403583948177093,-1.0999748867047898),(-0.3322630208172874,-1.057420378339037,-0.0766000050337147,-0.8949798182945309,-0.5042604443804907,-0.9384687647789537,-1.0999748867047898,0.62,-0.41952842352701486,-0.7274920231630502,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.30403583948177093,-1.0999748867047898),(0.10406399273134952,-0.06763531281107672,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,-0.5522849012925116,0.909111664354187,0.74,-0.41952842352701486,-0.7274920231630502,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.30403583948177093,-1.0999748867047898),(0.7149218116994412,0.4272572199529034,0.7988286239230257,0.09944220203272576,0.6101272780073337,0.11933920911869125,0.909111664354187,0.73,-0.41952842352701486,-0.7274920231630502,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.30403583948177093,-1.0999748867047898),(-1.2049170479145614,-1.2223845559270303,-0.952028633990455,-0.3977688081309026,0.052933416813421515,-0.9048875592583913,0.909111664354187,0.64,-0.41952842352701486,-0.7274920231630502,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.30403583948177093,-1.0999748867047898),(-1.4667132560437435,-0.8924562007510436,-0.0766000050337147,0.09944220203272576,-1.0614543055744028,-0.7201909288953117,-1.0999748867047898,0.63,-0.41952842352701486,-0.7274920231630502,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.30403583948177093,-1.0999748867047898),(-1.7285094641729257,-1.5523129111030172,-0.952028633990455,-0.8949798182945309,-0.5042604443804907,-1.5597210669093144,-1.0999748867047898,0.59,-0.41952842352701486,-0.7274920231630502,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.30403583948177093,-1.0999748867047898),(0.016798590021622126,-0.2325994903990701,-0.952028633990455,-1.3921908284581592,0.052933416813421515,-0.8041439426967131,-1.0999748867047898,0.73,-0.41952842352701486,-0.7274920231630502,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.30403583948177093,-1.0999748867047898),(0.889452617118896,0.26229304236491,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.2872452367214912,0.909111664354187,0.79,-0.41952842352701486,-0.7274920231630502,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.30403583948177093,-1.0999748867047898),(-1.3794478533340162,-0.5625278455750569,-0.952028633990455,0.09944220203272576,0.052933416813421515,-1.1903278061831537,0.909111664354187,0.68,-0.41952842352701486,-0.7274920231630502,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.30403583948177093,-1.0999748867047898),(-0.24499761810756004,-0.39756366798706344,-0.952028633990455,-0.8949798182945309,-1.618648166768315,-1.610092875190155,-1.0999748867047898,0.7,1.4130450333772604,1.582006463068857,0.7988286239230257,1.5910752325236108,0.6101272780073337,1.7816088823864173,0.909111664354187),(0.36586020086053167,-0.06763531281107672,-0.952028633990455,-1.3921908284581592,-2.175842027962227,-0.2668446543677508,-1.0999748867047898,0.81,1.4130450333772604,1.582006463068857,0.7988286239230257,1.5910752325236108,0.6101272780073337,1.7816088823864173,0.909111664354187),(0.4531256035702591,0.4272572199529034,-0.0766000050337147,0.5966532121963541,1.724515000395158,0.06896740083785215,0.909111664354187,0.85,1.4130450333772604,1.582006463068857,0.7988286239230257,1.5910752325236108,0.6101272780073337,1.7816088823864173,0.909111664354187),(1.5003104360869879,1.4170422854808635,0.7988286239230257,0.5966532121963541,0.052933416813421515,1.580121649263055,0.909111664354187,0.93,1.4130450333772604,1.582006463068857,0.7988286239230257,1.5910752325236108,0.6101272780073337,1.7816088823864173,0.909111664354187),(1.8493720469258974,1.2520781078928702,1.674257252879766,1.0938642223599824,1.724515000395158,1.0596129636943752,0.909111664354187,0.91,1.4130450333772604,1.582006463068857,0.7988286239230257,1.5910752325236108,0.6101272780073337,1.7816088823864173,0.909111664354187),(-0.9431208397853792,-0.7274920231630502,-0.952028633990455,-0.8949798182945309,-0.5042604443804907,-0.40116947644999135,-1.0999748867047898,0.69,1.4130450333772604,1.582006463068857,0.7988286239230257,1.5910752325236108,0.6101272780073337,1.7816088823864173,0.909111664354187),(-0.3322630208172874,-0.8924562007510436,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.5055230726051333,0.909111664354187,0.77,1.4130450333772604,1.582006463068857,0.7988286239230257,1.5910752325236108,0.6101272780073337,1.7816088823864173,0.909111664354187),(1.1512488252480781,1.087113930304877,0.7988286239230257,1.0938642223599824,-0.5042604443804907,0.9588693471326941,0.909111664354187,0.86,1.4130450333772604,1.582006463068857,0.7988286239230257,1.5910752325236108,0.6101272780073337,1.7816088823864173,0.909111664354187),(0.2785947981508043,-0.5625278455750569,-0.0766000050337147,0.09944220203272576,1.167321139201246,-0.4347506819705508,0.909111664354187,0.74,1.4130450333772604,1.582006463068857,0.7988286239230257,1.5910752325236108,0.6101272780073337,1.7816088823864173,0.909111664354187),(-0.5067938262367422,-1.5523129111030172,-1.8274572629471952,-2.386612848785416,-1.0614543055744028,-1.9123237248751956,-1.0999748867047898,0.57,1.4130450333772604,1.582006463068857,0.7988286239230257,1.5910752325236108,0.6101272780073337,1.7816088823864173,0.909111664354187),(-1.6412440614631982,-2.5420979766309775,-1.8274572629471952,-1.3921908284581592,-1.618648166768315,-1.2071184089434333,-1.0999748867047898,0.51,1.4130450333772604,1.582006463068857,0.7988286239230257,1.5910752325236108,0.6101272780073337,1.7816088823864173,0.909111664354187),(-1.3794478533340162,-1.5523129111030172,-1.8274572629471952,-1.3921908284581592,-0.5042604443804907,-0.9552593675392334,0.909111664354187,0.67,1.4130450333772604,1.582006463068857,0.7988286239230257,1.5910752325236108,0.6101272780073337,1.7816088823864173,0.909111664354187)
+insert into defaults values (1.76210664421617,1.7469706406568504,0.7988286239230257,1.0938642223599824,1.167321139201246,1.7648182796261376,0.909111664354187,0.92,-0.5940592289464697,-0.7274920231630502,-0.952028633990455,-0.8949798182945309,-1.0614543055744028,-0.6026567095733507,-1.0999748867047898),(0.6276564089897139,-0.06763531281107672,0.7988286239230257,0.5966532121963541,1.167321139201246,0.4551512643242912,0.909111664354187,0.76,-0.5940592289464697,-0.7274920231630502,-0.952028633990455,-0.8949798182945309,-1.0614543055744028,-0.6026567095733507,-1.0999748867047898),(-0.07046681268810527,-0.5625278455750569,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,-1.0056311758200724,0.909111664354187,0.72,-0.5940592289464697,-0.7274920231630502,-0.952028633990455,-0.8949798182945309,-1.0614543055744028,-0.6026567095733507,-1.0999748867047898),(0.4531256035702591,0.4272572199529034,-0.0766000050337147,0.09944220203272576,-1.0614543055744028,0.11933920911869125,0.909111664354187,0.8,-0.5940592289464697,-0.7274920231630502,-0.952028633990455,-0.8949798182945309,-1.0614543055744028,-0.6026567095733507,-1.0999748867047898),(-0.24499761810756004,-0.7274920231630502,-0.952028633990455,-1.3921908284581592,-0.5042604443804907,-0.6530285178541898,-1.0999748867047898,0.65,-0.5940592289464697,-0.7274920231630502,-0.952028633990455,-0.8949798182945309,-1.0614543055744028,-0.6026567095733507,-1.0999748867047898),(1.1512488252480781,1.2520781078928702,1.674257252879766,1.0938642223599824,-0.5042604443804907,1.244309594057455,0.909111664354187,0.9,-0.5940592289464697,-0.7274920231630502,-0.952028633990455,-0.8949798182945309,-1.0614543055744028,-0.6026567095733507,-1.0999748867047898),(0.36586020086053167,0.26229304236491,-0.0766000050337147,-0.3977688081309026,0.6101272780073337,-0.6698191206144725,0.909111664354187,0.75,-0.5940592289464697,-0.7274920231630502,-0.952028633990455,-0.8949798182945309,-1.0614543055744028,-0.6026567095733507,-1.0999748867047898),(-0.7685900343659244,-1.057420378339037,-0.952028633990455,-0.3977688081309026,0.6101272780073337,-1.1735372034228724,-1.0999748867047898,0.68,-0.5940592289464697,-0.7274920231630502,-0.952028633990455,-0.8949798182945309,-1.0614543055744028,-0.6026567095733507,-1.0999748867047898),(-1.2921824506242887,-0.8924562007510436,-1.8274572629471952,-1.3921908284581592,-2.175842027962227,-1.0056311758200724,-1.0999748867047898,0.5,-0.5940592289464697,-0.7274920231630502,-0.952028633990455,-0.8949798182945309,-1.0614543055744028,-0.6026567095733507,-1.0999748867047898),(0.5403910062799865,0.09732886477691666,-0.0766000050337147,0.09944220203272576,-0.5042604443804907,0.0018049897967303734,-1.0999748867047898,0.45,-0.5940592289464697,-0.7274920231630502,-0.952028633990455,-0.8949798182945309,-1.0614543055744028,-0.6026567095733507,-1.0999748867047898),(0.7149218116994412,-0.2325994903990701,-0.0766000050337147,0.09944220203272576,0.6101272780073337,-0.3340070654088696,0.909111664354187,0.52,-0.5940592289464697,-0.7274920231630502,-0.952028633990455,-0.8949798182945309,-1.0614543055744028,-0.6026567095733507,-1.0999748867047898),(0.889452617118896,0.5922213975408968,0.7988286239230257,0.5966532121963541,1.167321139201246,0.6734291002079332,0.909111664354187,0.84,-0.5940592289464697,-0.7274920231630502,-0.952028633990455,-0.8949798182945309,-1.0614543055744028,-0.6026567095733507,-1.0999748867047898),(0.9767180198286235,0.7571855751288902,0.7988286239230257,0.5966532121963541,1.167321139201246,0.8413351278107333,0.909111664354187,0.78,-0.5940592289464697,-0.7274920231630502,-0.952028633990455,-0.8949798182945309,-1.0614543055744028,-0.6026567095733507,-1.0999748867047898),(-0.8558554370756518,0.26229304236491,-0.0766000050337147,0.5966532121963541,-0.5042604443804907,-1.0056311758200724,0.909111664354187,0.62,0.6276564089897139,0.4272572199529034,-0.0766000050337147,0.09944220203272576,-0.5042604443804907,1.0428223609340956,0.909111664354187),(-0.5067938262367422,-0.5625278455750569,-0.0766000050337147,0.09944220203272576,-1.618648166768315,-0.6698191206144725,0.909111664354187,0.61,0.6276564089897139,0.4272572199529034,-0.0766000050337147,0.09944220203272576,-0.5042604443804907,1.0428223609340956,0.909111664354187),(-0.24499761810756004,-0.39756366798706344,-0.0766000050337147,0.09944220203272576,-1.0614543055744028,-0.5019130930116695,-1.0999748867047898,0.54,0.6276564089897139,0.4272572199529034,-0.0766000050337147,0.09944220203272576,-0.5042604443804907,1.0428223609340956,0.909111664354187),(0.016798590021622126,-0.06763531281107672,-0.0766000050337147,0.5966532121963541,-0.5042604443804907,0.16971101739953035,-1.0999748867047898,0.66,0.6276564089897139,0.4272572199529034,-0.0766000050337147,0.09944220203272576,-0.5042604443804907,1.0428223609340956,0.909111664354187),(0.1913293954410769,-0.2325994903990701,-0.0766000050337147,0.5966532121963541,-0.5042604443804907,-1.0056311758200724,0.909111664354187,0.65,0.6276564089897139,0.4272572199529034,-0.0766000050337147,0.09944220203272576,-0.5042604443804907,1.0428223609340956,0.909111664354187),(0.10406399273134952,0.4272572199529034,-0.0766000050337147,0.5966532121963541,-0.5042604443804907,0.3376170450023333,-1.0999748867047898,0.63,0.6276564089897139,0.4272572199529034,-0.0766000050337147,0.09944220203272576,-0.5042604443804907,1.0428223609340956,0.909111664354187),(-1.2049170479145614,-0.8924562007510436,-0.0766000050337147,0.09944220203272576,-0.5042604443804907,-0.1661010378060696,-1.0999748867047898,0.62,0.6276564089897139,0.4272572199529034,-0.0766000050337147,0.09944220203272576,-0.5042604443804907,1.0428223609340956,0.909111664354187),(-0.41952842352701486,-0.06763531281107672,-0.0766000050337147,-0.3977688081309026,-1.618648166768315,-1.1735372034228724,0.909111664354187,0.64,0.6276564089897139,0.4272572199529034,-0.0766000050337147,0.09944220203272576,-0.5042604443804907,1.0428223609340956,0.909111664354187),(0.7149218116994412,1.087113930304877,0.7988286239230257,-0.3977688081309026,-1.618648166768315,-0.3340070654088696,-1.0999748867047898,0.7,0.6276564089897139,0.4272572199529034,-0.0766000050337147,0.09944220203272576,-0.5042604443804907,1.0428223609340956,0.909111664354187),(0.9767180198286235,1.4170422854808635,1.674257252879766,1.5910752325236108,1.724515000395158,1.5129592382219361,0.909111664354187,0.94,0.6276564089897139,0.4272572199529034,-0.0766000050337147,0.09944220203272576,-0.5042604443804907,1.0428223609340956,0.909111664354187),(1.5003104360869879,1.9119348182448437,1.674257252879766,1.5910752325236108,1.167321139201246,1.848771293427536,0.909111664354187,0.95,0.6276564089897139,0.4272572199529034,-0.0766000050337147,0.09944220203272576,-0.5042604443804907,1.0428223609340956,0.909111664354187),(1.6748412415064426,1.9119348182448437,1.674257252879766,0.5966532121963541,0.052933416813421515,2.016677321030339,0.909111664354187,0.97,0.6276564089897139,0.4272572199529034,-0.0766000050337147,0.09944220203272576,-0.5042604443804907,1.0428223609340956,0.909111664354187),(2.023902852345352,2.076898995832837,1.674257252879766,1.0938642223599824,1.167321139201246,1.680865265824736,0.909111664354187,0.94,1.6748412415064426,1.9119348182448437,0.7988286239230257,1.0938642223599824,0.6101272780073337,1.7144464713452956,0.909111664354187),(0.4531256035702591,0.26229304236491,1.674257252879766,1.0938642223599824,0.052933416813421515,0.3376170450023333,-1.0999748867047898,0.76,1.6748412415064426,1.9119348182448437,0.7988286239230257,1.0938642223599824,0.6101272780073337,1.7144464713452956,0.909111664354187),(-1.6412440614631982,-1.5523129111030172,-0.952028633990455,-1.8894018386217877,-1.0614543055744028,-1.8451613138340752,0.909111664354187,0.44,1.6748412415064426,1.9119348182448437,0.7988286239230257,1.0938642223599824,0.6101272780073337,1.7144464713452956,0.909111664354187),(-1.9030402695923805,-2.377133799042984,-1.8274572629471952,-1.3921908284581592,-1.618648166768315,-2.348879396642477,-1.0999748867047898,0.46,1.6748412415064426,1.9119348182448437,0.7988286239230257,1.0938642223599824,0.6101272780073337,1.7144464713452956,0.909111664354187),(-0.5940592289464697,-1.3873487335150236,-0.952028633990455,-1.8894018386217877,-1.618648166768315,-2.180973369039677,-1.0999748867047898,0.54,1.6748412415064426,1.9119348182448437,0.7988286239230257,1.0938642223599824,0.6101272780073337,1.7144464713452956,0.909111664354187),(-1.4667132560437435,-1.7172770886910105,-0.952028633990455,-0.3977688081309026,-0.5042604443804907,-0.8377251482172725,0.909111664354187,0.65,1.6748412415064426,1.9119348182448437,0.7988286239230257,1.0938642223599824,0.6101272780073337,1.7144464713452956,0.909111664354187),(0.889452617118896,-0.7274920231630502,-0.0766000050337147,0.5966532121963541,0.6101272780073337,-0.5019130930116695,0.909111664354187,0.74,1.6748412415064426,1.9119348182448437,0.7988286239230257,1.0938642223599824,0.6101272780073337,1.7144464713452956,0.909111664354187),(1.8493720469258974,1.7469706406568504,0.7988286239230257,-0.3977688081309026,1.167321139201246,1.3450532106191362,0.909111664354187,0.91,1.6748412415064426,1.9119348182448437,0.7988286239230257,1.0938642223599824,0.6101272780073337,1.7144464713452956,0.909111664354187),(2.023902852345352,1.087113930304877,1.674257252879766,0.5966532121963541,0.6101272780073337,1.680865265824736,0.909111664354187,0.9,1.6748412415064426,1.9119348182448437,0.7988286239230257,1.0938642223599824,0.6101272780073337,1.7144464713452956,0.909111664354187),(1.2385142279578056,0.7571855751288902,1.674257252879766,0.5966532121963541,1.724515000395158,2.016677321030339,0.909111664354187,0.94,1.6748412415064426,1.9119348182448437,0.7988286239230257,1.0938642223599824,0.6101272780073337,1.7144464713452956,0.909111664354187),(0.2785947981508043,0.4272572199529034,1.674257252879766,1.5910752325236108,1.724515000395158,1.0092411554135332,0.909111664354187,0.88,1.6748412415064426,1.9119348182448437,0.7988286239230257,1.0938642223599824,0.6101272780073337,1.7144464713452956,0.909111664354187),(-1.553978658753471,-0.2325994903990701,-0.952028633990455,0.5966532121963541,0.6101272780073337,-0.3340070654088696,-1.0999748867047898,0.64,1.6748412415064426,1.9119348182448437,0.7988286239230257,1.0938642223599824,0.6101272780073337,1.7144464713452956,0.909111664354187),(-1.4667132560437435,-0.39756366798706344,-1.8274572629471952,-2.386612848785416,-1.618648166768315,-1.3414432310256739,-1.0999748867047898,0.58,0.36586020086053167,0.26229304236491,-0.0766000050337147,-0.3977688081309026,-0.5042604443804907,-0.0989386267649508,0.909111664354187),(-1.117651645204834,-0.39756366798706344,-1.8274572629471952,-0.3977688081309026,-2.175842027962227,-1.8451613138340752,-1.0999748867047898,0.52,0.36586020086053167,0.26229304236491,-0.0766000050337147,-0.3977688081309026,-0.5042604443804907,-0.0989386267649508,0.909111664354187),(-0.8558554370756518,0.09732886477691666,-0.952028633990455,0.5966532121963541,0.052933416813421515,-1.5093492586284738,-1.0999748867047898,0.48,0.36586020086053167,0.26229304236491,-0.0766000050337147,-0.3977688081309026,-0.5042604443804907,-0.0989386267649508,0.909111664354187),(-0.7685900343659244,0.4272572199529034,-0.0766000050337147,0.09944220203272576,-0.5042604443804907,-1.0056311758200724,0.909111664354187,0.46,0.36586020086053167,0.26229304236491,-0.0766000050337147,-0.3977688081309026,-0.5042604443804907,-0.0989386267649508,0.909111664354187),(-0.07046681268810527,-0.39756366798706344,-0.952028633990455,-0.8949798182945309,-1.0614543055744028,-0.6698191206144725,0.909111664354187,0.49,0.36586020086053167,0.26229304236491,-0.0766000050337147,-0.3977688081309026,-0.5042604443804907,-0.0989386267649508,0.909111664354187),(-0.3322630208172874,-0.06763531281107672,-0.952028633990455,-0.8949798182945309,-1.618648166768315,-0.1661010378060696,0.909111664354187,0.53,0.36586020086053167,0.26229304236491,-0.0766000050337147,-0.3977688081309026,-0.5042604443804907,-0.0989386267649508,0.909111664354187),(1.3257796306675331,1.582006463068857,0.7988286239230257,1.0938642223599824,0.6101272780073337,0.8413351278107333,-1.0999748867047898,0.87,0.36586020086053167,0.26229304236491,-0.0766000050337147,-0.3977688081309026,-0.5042604443804907,-0.0989386267649508,0.909111664354187),(0.8021872144091686,0.9221497527168835,1.674257252879766,1.0938642223599824,0.6101272780073337,1.3450532106191362,0.909111664354187,0.91,0.36586020086053167,0.26229304236491,-0.0766000050337147,-0.3977688081309026,-0.5042604443804907,-0.0989386267649508,0.909111664354187),(0.4531256035702591,0.4272572199529034,1.674257252879766,1.5910752325236108,0.6101272780073337,0.8413351278107333,0.909111664354187,0.88,0.36586020086053167,0.26229304236491,-0.0766000050337147,-0.3977688081309026,-0.5042604443804907,-0.0989386267649508,0.909111664354187),(1.0639834225383509,1.087113930304877,1.674257252879766,0.5966532121963541,1.724515000395158,1.1771471830163363,0.909111664354187,0.86,0.36586020086053167,0.26229304236491,-0.0766000050337147,-0.3977688081309026,-0.5042604443804907,-0.0989386267649508,0.909111664354187),(1.9366374496356247,1.9119348182448437,1.674257252879766,1.0938642223599824,0.6101272780073337,1.848771293427536,-1.0999748867047898,0.89,0.36586020086053167,0.26229304236491,-0.0766000050337147,-0.3977688081309026,-0.5042604443804907,-0.0989386267649508,0.909111664354187),(0.36586020086053167,0.4272572199529034,-0.0766000050337147,0.09944220203272576,1.724515000395158,0.42157005880373183,0.909111664354187,0.82,0.36586020086053167,0.26229304236491,-0.0766000050337147,-0.3977688081309026,-0.5042604443804907,-0.0989386267649508,0.909111664354187),(0.889452617118896,0.5922213975408968,0.7988286239230257,-0.3977688081309026,0.6101272780073337,-0.3340070654088696,0.909111664354187,0.78,0.36586020086053167,0.26229304236491,-0.0766000050337147,-0.3977688081309026,-0.5042604443804907,-0.0989386267649508,0.909111664354187),(-0.3322630208172874,-1.5523129111030172,-0.0766000050337147,-0.8949798182945309,1.167321139201246,-0.5019130930116695,0.909111664354187,0.76,-0.15773221539783266,-0.39756366798706344,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-1.5933022724298738,-1.0999748867047898),(-0.41952842352701486,-1.2223845559270303,-0.952028633990455,-1.8894018386217877,0.052933416813421515,-1.1735372034228724,0.909111664354187,0.56,-0.15773221539783266,-0.39756366798706344,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-1.5933022724298738,-1.0999748867047898),(1.5003104360869879,1.4170422854808635,0.7988286239230257,0.5966532121963541,-0.5042604443804907,-1.0056311758200724,0.909111664354187,0.78,-0.15773221539783266,-0.39756366798706344,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-1.5933022724298738,-1.0999748867047898),(0.6276564089897139,0.7571855751288902,0.7988286239230257,0.5966532121963541,-1.0614543055744028,-0.8377251482172725,0.909111664354187,0.72,-0.15773221539783266,-0.39756366798706344,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-1.5933022724298738,-1.0999748867047898),(0.4531256035702591,0.4272572199529034,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,-1.0056311758200724,-1.0999748867047898,0.7,-0.15773221539783266,-0.39756366798706344,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-1.5933022724298738,-1.0999748867047898),(0.2785947981508043,-0.7274920231630502,-0.0766000050337147,-0.3977688081309026,-0.5042604443804907,-1.5093492586284738,-1.0999748867047898,0.64,-0.15773221539783266,-0.39756366798706344,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-1.5933022724298738,-1.0999748867047898),(-0.07046681268810527,-0.8924562007510436,-0.0766000050337147,-1.3921908284581592,-0.5042604443804907,-2.013067341436875,-1.0999748867047898,0.64,-0.15773221539783266,-0.39756366798706344,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-1.5933022724298738,-1.0999748867047898),(-1.6412440614631982,-1.3873487335150236,-0.952028633990455,0.5966532121963541,-1.618648166768315,-1.6772552862312753,-1.0999748867047898,0.46,-0.15773221539783266,-0.39756366798706344,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-1.5933022724298738,-1.0999748867047898),(-1.4667132560437435,-1.3873487335150236,-1.8274572629471952,-0.3977688081309026,-1.618648166768315,-3.0205035070536796,0.909111664354187,0.36,-0.15773221539783266,-0.39756366798706344,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-1.5933022724298738,-1.0999748867047898),(-0.5067938262367422,-0.5625278455750569,-0.952028633990455,-1.3921908284581592,-1.618648166768315,-0.5019130930116695,-1.0999748867047898,0.42,-0.15773221539783266,-0.39756366798706344,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-1.5933022724298738,-1.0999748867047898),(-0.681324631656197,-1.2223845559270303,-0.952028633990455,-0.3977688081309026,-0.5042604443804907,-0.8377251482172725,-1.0999748867047898,0.48,-0.15773221539783266,-0.39756366798706344,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-1.5933022724298738,-1.0999748867047898),(-0.8558554370756518,-1.057420378339037,-0.0766000050337147,0.5966532121963541,-0.5042604443804907,-0.6698191206144725,-1.0999748867047898,0.47,-0.15773221539783266,-0.39756366798706344,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-1.5933022724298738,-1.0999748867047898),(-1.117651645204834,-0.39756366798706344,-0.952028633990455,-0.3977688081309026,-0.5042604443804907,-0.6698191206144725,0.909111664354187,0.54,-1.117651645204834,-1.057420378339037,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-1.5765116696695942,-1.0999748867047898),(-0.15773221539783266,-0.06763531281107672,-0.952028633990455,0.5966532121963541,-0.5042604443804907,-0.1661010378060696,0.909111664354187,0.56,-1.117651645204834,-1.057420378339037,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-1.5765116696695942,-1.0999748867047898),(0.7149218116994412,0.5922213975408968,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.16971101739953035,-1.0999748867047898,0.52,-1.117651645204834,-1.057420378339037,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-1.5765116696695942,-1.0999748867047898),(0.7149218116994412,0.7571855751288902,0.7988286239230257,0.09944220203272576,0.052933416813421515,0.5391042781256927,-1.0999748867047898,0.55,-1.117651645204834,-1.057420378339037,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-1.5765116696695942,-1.0999748867047898),(0.889452617118896,1.087113930304877,-0.0766000050337147,-0.3977688081309026,-0.5042604443804907,0.7070103057284927,-1.0999748867047898,0.61,-1.117651645204834,-1.057420378339037,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-1.5765116696695942,-1.0999748867047898),(-0.07046681268810527,-0.06763531281107672,-0.952028633990455,0.09944220203272576,0.052933416813421515,0.06896740083785215,0.909111664354187,0.57,-1.117651645204834,-1.057420378339037,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-1.5765116696695942,-1.0999748867047898),(0.10406399273134952,0.26229304236491,-0.0766000050337147,0.09944220203272576,0.6101272780073337,1.0428223609340956,0.909111664354187,0.68,-1.117651645204834,-1.057420378339037,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-1.5765116696695942,-1.0999748867047898),(0.9767180198286235,1.2520781078928702,0.7988286239230257,1.0938642223599824,0.6101272780073337,0.9420787443724145,0.909111664354187,0.78,-1.117651645204834,-1.057420378339037,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-1.5765116696695942,-1.0999748867047898),(1.3257796306675331,1.7469706406568504,1.674257252879766,1.5910752325236108,1.724515000395158,1.7480276768658578,0.909111664354187,0.94,-1.117651645204834,-1.057420378339037,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-1.5765116696695942,-1.0999748867047898),(1.6748412415064426,0.7571855751288902,1.674257252879766,1.5910752325236108,1.724515000395158,1.9495149099892173,0.909111664354187,0.96,-1.117651645204834,-1.057420378339037,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-1.5765116696695942,-1.0999748867047898),(0.36586020086053167,0.5922213975408968,1.674257252879766,1.5910752325236108,1.724515000395158,1.4290062244205346,0.909111664354187,0.93,-1.117651645204834,-1.057420378339037,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-1.5765116696695942,-1.0999748867047898),(-0.24499761810756004,0.09732886477691666,0.7988286239230257,1.0938642223599824,0.6101272780073337,0.7405915112490521,0.909111664354187,0.84,-1.117651645204834,-1.057420378339037,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-1.5765116696695942,-1.0999748867047898),(-0.24499761810756004,-0.2325994903990701,-0.0766000050337147,-0.3977688081309026,1.724515000395158,0.5055230726051333,-1.0999748867047898,0.74,-1.7285094641729257,-1.8822412662790038,-0.952028633990455,-0.8949798182945309,-1.618648166768315,-1.9626955331560363,-1.0999748867047898),(1.0639834225383509,1.087113930304877,-0.952028633990455,-1.3921908284581592,0.6101272780073337,-0.06535742124438843,0.909111664354187,0.72,-1.7285094641729257,-1.8822412662790038,-0.952028633990455,-0.8949798182945309,-1.618648166768315,-1.9626955331560363,-1.0999748867047898),(0.889452617118896,0.7571855751288902,-0.0766000050337147,-0.3977688081309026,-0.5042604443804907,0.20329222292009272,0.909111664354187,0.74,-1.7285094641729257,-1.8822412662790038,-0.952028633990455,-0.8949798182945309,-1.618648166768315,-1.9626955331560363,-1.0999748867047898),(-1.3794478533340162,-1.3873487335150236,-0.952028633990455,-0.3977688081309026,-1.618648166768315,-0.6362379150939101,-1.0999748867047898,0.64,-1.7285094641729257,-1.8822412662790038,-0.952028633990455,-0.8949798182945309,-1.618648166768315,-1.9626955331560363,-1.0999748867047898),(-1.8157748668826532,-2.0472054438669973,-0.952028633990455,-0.3977688081309026,-1.618648166768315,-1.777998902792955,0.909111664354187,0.44,-1.7285094641729257,-1.8822412662790038,-0.952028633990455,-0.8949798182945309,-1.618648166768315,-1.9626955331560363,-1.0999748867047898),(-1.990305672302108,-2.377133799042984,-1.8274572629471952,-1.8894018386217877,-1.618648166768315,-2.0802297524779956,-1.0999748867047898,0.46,-1.7285094641729257,-1.8822412662790038,-0.952028633990455,-0.8949798182945309,-1.618648166768315,-1.9626955331560363,-1.0999748867047898),(-0.41952842352701486,-0.39756366798706344,-0.0766000050337147,-1.3921908284581592,-0.5042604443804907,-0.972049970299513,0.909111664354187,0.5,-1.7285094641729257,-1.8822412662790038,-0.952028633990455,-0.8949798182945309,-1.618648166768315,-1.9626955331560363,-1.0999748867047898),(2.023902852345352,2.076898995832837,0.7988286239230257,1.5910752325236108,1.724515000395158,1.5129592382219361,0.909111664354187,0.96,-1.7285094641729257,-1.8822412662790038,-0.952028633990455,-0.8949798182945309,-1.618648166768315,-1.9626955331560363,-1.0999748867047898),(0.2785947981508043,0.4272572199529034,1.674257252879766,1.5910752325236108,1.167321139201246,1.0428223609340956,0.909111664354187,0.92,-1.7285094641729257,-1.8822412662790038,-0.952028633990455,-0.8949798182945309,-1.618648166768315,-1.9626955331560363,-1.0999748867047898),(0.4531256035702591,1.2520781078928702,1.674257252879766,0.5966532121963541,1.167321139201246,1.2778907995780144,0.909111664354187,0.92,-1.7285094641729257,-1.8822412662790038,-0.952028633990455,-0.8949798182945309,-1.618648166768315,-1.9626955331560363,-1.0999748867047898),(2.023902852345352,1.2520781078928702,1.674257252879766,1.0938642223599824,1.167321139201246,1.4290062244205346,0.909111664354187,0.94,-1.7285094641729257,-1.8822412662790038,-0.952028633990455,-0.8949798182945309,-1.618648166768315,-1.9626955331560363,-1.0999748867047898),(0.1913293954410769,-0.7274920231630502,0.7988286239230257,1.0938642223599824,0.052933416813421515,0.10254860635841155,-1.0999748867047898,0.76,-1.7285094641729257,-1.8822412662790038,-0.952028633990455,-0.8949798182945309,-1.618648166768315,-1.9626955331560363,-1.0999748867047898),(-0.15773221539783266,-0.2325994903990701,-0.0766000050337147,1.0938642223599824,0.052933416813421515,-0.30042585988831016,-1.0999748867047898,0.72,-1.7285094641729257,-1.8822412662790038,-0.952028633990455,-0.8949798182945309,-1.618648166768315,-1.9626955331560363,-1.0999748867047898),(0.016798590021622126,-0.06763531281107672,-0.952028633990455,0.09944220203272576,-0.5042604443804907,-0.535494298532232,-1.0999748867047898,0.66,-2.3393672831410175,-1.2223845559270303,-1.8274572629471952,-1.8894018386217877,-1.618648166768315,-1.7444176972723957,-1.0999748867047898),(-0.24499761810756004,0.09732886477691666,-0.0766000050337147,1.0938642223599824,0.052933416813421515,-0.7705627371761508,-1.0999748867047898,0.64,-2.3393672831410175,-1.2223845559270303,-1.8274572629471952,-1.8894018386217877,-1.618648166768315,-1.7444176972723957,-1.0999748867047898),(-0.07046681268810527,0.26229304236491,0.7988286239230257,1.0938642223599824,0.052933416813421515,0.27045463396121155,0.909111664354187,0.74,-2.3393672831410175,-1.2223845559270303,-1.8274572629471952,-1.8894018386217877,-1.618648166768315,-1.7444176972723957,-1.0999748867047898),(0.10406399273134952,-0.2325994903990701,-0.952028633990455,0.5966532121963541,0.6101272780073337,-1.139955997902313,0.909111664354187,0.64,-2.3393672831410175,-1.2223845559270303,-1.8274572629471952,-1.8894018386217877,-1.618648166768315,-1.7444176972723957,-1.0999748867047898),(-1.553978658753471,-1.7172770886910105,-0.0766000050337147,1.5910752325236108,0.052933416813421515,-1.5765116696695942,-1.0999748867047898,0.38,-2.3393672831410175,-1.2223845559270303,-1.8274572629471952,-1.8894018386217877,-1.618648166768315,-1.7444176972723957,-1.0999748867047898),(-1.6412440614631982,-1.5523129111030172,-0.952028633990455,0.5966532121963541,-0.5042604443804907,-0.9552593675392334,-1.0999748867047898,0.34,-2.3393672831410175,-1.2223845559270303,-1.8274572629471952,-1.8894018386217877,-1.618648166768315,-1.7444176972723957,-1.0999748867047898),(-1.3794478533340162,-1.7172770886910105,-0.952028633990455,-0.3977688081309026,-0.5042604443804907,-1.2071184089434333,0.909111664354187,0.44,-2.3393672831410175,-1.2223845559270303,-1.8274572629471952,-1.8894018386217877,-1.618648166768315,-1.7444176972723957,-1.0999748867047898),(-1.2049170479145614,-1.3873487335150236,-0.0766000050337147,-1.3921908284581592,-1.0614543055744028,-1.5765116696695942,-1.0999748867047898,0.36,-2.3393672831410175,-1.2223845559270303,-1.8274572629471952,-1.8894018386217877,-1.618648166768315,-1.7444176972723957,-1.0999748867047898),(-1.117651645204834,-1.2223845559270303,0.7988286239230257,-1.8894018386217877,-1.0614543055744028,-1.2742808199845537,-1.0999748867047898,0.42,-2.3393672831410175,-1.2223845559270303,-1.8274572629471952,-1.8894018386217877,-1.618648166768315,-1.7444176972723957,-1.0999748867047898),(-0.9431208397853792,-1.2223845559270303,-0.952028633990455,-0.3977688081309026,-0.5042604443804907,-1.0056311758200724,-1.0999748867047898,0.48,-2.3393672831410175,-1.2223845559270303,-1.8274572629471952,-1.8894018386217877,-1.618648166768315,-1.7444176972723957,-1.0999748867047898),(1.2385142279578056,2.076898995832837,-0.0766000050337147,0.5966532121963541,0.6101272780073337,0.6062666891668145,0.909111664354187,0.86,-2.3393672831410175,-1.2223845559270303,-1.8274572629471952,-1.8894018386217877,-1.618648166768315,-1.7444176972723957,-1.0999748867047898),(1.3257796306675331,1.9119348182448437,0.7988286239230257,1.5910752325236108,1.167321139201246,1.076403566454655,0.909111664354187,0.9,-2.3393672831410175,-1.2223845559270303,-1.8274572629471952,-1.8894018386217877,-1.618648166768315,-1.7444176972723957,-1.0999748867047898),(0.5403910062799865,0.9221497527168835,-0.0766000050337147,0.5966532121963541,0.6101272780073337,0.47194186708457386,0.909111664354187,0.79,-1.2049170479145614,-1.5523129111030172,-1.8274572629471952,-1.3921908284581592,-1.0614543055744028,-1.5933022724298738,-1.0999748867047898),(0.4531256035702591,-0.06763531281107672,-0.0766000050337147,0.09944220203272576,0.052933416813421515,-0.2332634488471884,0.909111664354187,0.71,-1.2049170479145614,-1.5523129111030172,-1.8274572629471952,-1.3921908284581592,-1.0614543055744028,-1.5933022724298738,-1.0999748867047898),(-0.41952842352701486,-0.39756366798706344,-0.952028633990455,-0.8949798182945309,-0.5042604443804907,-0.8041439426967131,-1.0999748867047898,0.64,-1.2049170479145614,-1.5523129111030172,-1.8274572629471952,-1.3921908284581592,-1.0614543055744028,-1.5933022724298738,-1.0999748867047898),(-0.24499761810756004,-0.2325994903990701,-0.952028633990455,0.5966532121963541,0.052933416813421515,-0.585866106813071,-1.0999748867047898,0.62,-1.2049170479145614,-1.5523129111030172,-1.8274572629471952,-1.3921908284581592,-1.0614543055744028,-1.5933022724298738,-1.0999748867047898),(0.016798590021622126,-0.5625278455750569,-0.952028633990455,1.0938642223599824,0.6101272780073337,-0.2164728460869087,-1.0999748867047898,0.57,-1.2049170479145614,-1.5523129111030172,-1.8274572629471952,-1.3921908284581592,-1.0614543055744028,-1.5933022724298738,-1.0999748867047898),(0.8021872144091686,0.7571855751288902,-0.0766000050337147,0.09944220203272576,-0.5042604443804907,0.7573821140093348,0.909111664354187,0.74,-1.2049170479145614,-1.5523129111030172,-1.8274572629471952,-1.3921908284581592,-1.0614543055744028,-1.5933022724298738,-1.0999748867047898),(-0.07046681268810527,0.4272572199529034,-0.0766000050337147,0.5966532121963541,1.167321139201246,0.30403583948177093,0.909111664354187,0.69,-1.2049170479145614,-1.5523129111030172,-1.8274572629471952,-1.3921908284581592,-1.0614543055744028,-1.5933022724298738,-1.0999748867047898),(1.0639834225383509,0.5922213975408968,0.7988286239230257,1.0938642223599824,1.167321139201246,0.9756599498929738,0.909111664354187,0.87,-1.2049170479145614,-1.5523129111030172,-1.8274572629471952,-1.3921908284581592,-1.0614543055744028,-1.5933022724298738,-1.0999748867047898),(1.8493720469258974,1.582006463068857,0.7988286239230257,0.09944220203272576,1.167321139201246,1.4457968271808173,0.909111664354187,0.91,-1.2049170479145614,-1.5523129111030172,-1.8274572629471952,-1.3921908284581592,-1.0614543055744028,-1.5933022724298738,-1.0999748867047898),(1.2385142279578056,1.4170422854808635,1.674257252879766,1.5910752325236108,1.724515000395158,1.3114720050985766,0.909111664354187,0.93,-1.2049170479145614,-1.5523129111030172,-1.8274572629471952,-1.3921908284581592,-1.0614543055744028,-1.5933022724298738,-1.0999748867047898),(-1.117651645204834,-0.7274920231630502,1.674257252879766,1.5910752325236108,0.6101272780073337,0.06896740083785215,-1.0999748867047898,0.68,-1.2049170479145614,-1.5523129111030172,-1.8274572629471952,-1.3921908284581592,-1.0614543055744028,-1.5933022724298738,-1.0999748867047898),(-1.0303862424951067,0.09732886477691666,1.674257252879766,-0.3977688081309026,-0.5042604443804907,-0.199682243326629,-1.0999748867047898,0.61,-1.2049170479145614,-1.5523129111030172,-1.8274572629471952,-1.3921908284581592,-1.0614543055744028,-1.5933022724298738,-1.0999748867047898),(0.36586020086053167,0.26229304236491,0.7988286239230257,0.5966532121963541,0.6101272780073337,0.13612981187897094,0.909111664354187,0.69,-0.5067938262367422,-1.3873487335150236,-1.8274572629471952,-0.8949798182945309,-0.5042604443804907,-0.2836352571280305,0.909111664354187),(-1.3794478533340162,-0.06763531281107672,-0.0766000050337147,0.09944220203272576,0.052933416813421515,-0.4347506819705508,0.909111664354187,0.62,-0.5067938262367422,-1.3873487335150236,-1.8274572629471952,-0.8949798182945309,-0.5042604443804907,-0.2836352571280305,0.909111664354187),(0.2785947981508043,0.4272572199529034,-0.952028633990455,0.5966532121963541,0.052933416813421515,-0.06535742124438843,-1.0999748867047898,0.72,-0.5067938262367422,-1.3873487335150236,-1.8274572629471952,-0.8949798182945309,-0.5042604443804907,-0.2836352571280305,0.909111664354187),(-0.5067938262367422,-0.39756366798706344,-0.0766000050337147,0.09944220203272576,-0.5042604443804907,-0.2500540516074711,0.909111664354187,0.59,-0.5067938262367422,-1.3873487335150236,-1.8274572629471952,-0.8949798182945309,-0.5042604443804907,-0.2836352571280305,0.909111664354187),(-0.5940592289464697,-0.2325994903990701,0.7988286239230257,1.0938642223599824,1.167321139201246,0.7405915112490521,0.909111664354187,0.66,-0.5067938262367422,-1.3873487335150236,-1.8274572629471952,-0.8949798182945309,-0.5042604443804907,-0.2836352571280305,0.909111664354187),(-1.553978658753471,-0.8924562007510436,-0.0766000050337147,0.5966532121963541,0.052933416813421515,0.03538619531728977,-1.0999748867047898,0.56,-0.5067938262367422,-1.3873487335150236,-1.8274572629471952,-0.8949798182945309,-0.5042604443804907,-0.2836352571280305,0.909111664354187),(-2.3393672831410175,-0.5625278455750569,0.7988286239230257,-1.3921908284581592,-1.0614543055744028,-1.9123237248751956,-1.0999748867047898,0.45,-0.5067938262367422,-1.3873487335150236,-1.8274572629471952,-0.8949798182945309,-0.5042604443804907,-0.2836352571280305,0.909111664354187),(-1.8157748668826532,-1.3873487335150236,-0.952028633990455,-0.3977688081309026,0.052933416813421515,-2.214554574560236,-1.0999748867047898,0.47,-0.5067938262367422,-1.3873487335150236,-1.8274572629471952,-0.8949798182945309,-0.5042604443804907,-0.2836352571280305,0.909111664354187),(0.889452617118896,-0.5625278455750569,1.674257252879766,-0.3977688081309026,0.052933416813421515,0.4047794560434521,0.909111664354187,0.71,-0.5067938262367422,-1.3873487335150236,-1.8274572629471952,-0.8949798182945309,-0.5042604443804907,-0.2836352571280305,0.909111664354187),(1.5875758387967152,1.582006463068857,1.674257252879766,1.5910752325236108,1.724515000395158,1.6137028547836172,0.909111664354187,0.94,-0.5067938262367422,-1.3873487335150236,-1.8274572629471952,-0.8949798182945309,-0.5042604443804907,-0.2836352571280305,0.909111664354187),(1.5003104360869879,1.9119348182448437,1.674257252879766,1.0938642223599824,1.167321139201246,1.4793780327013768,0.909111664354187,0.94,-0.5067938262367422,-1.3873487335150236,-1.8274572629471952,-0.8949798182945309,-0.5042604443804907,-0.2836352571280305,0.909111664354187),(-0.5940592289464697,-0.2325994903990701,0.7988286239230257,-1.8894018386217877,-1.0614543055744028,-0.40116947644999135,-1.0999748867047898,0.57,-0.5067938262367422,-1.3873487335150236,-1.8274572629471952,-0.8949798182945309,-0.5042604443804907,-0.2836352571280305,0.909111664354187),(-0.7685900343659244,0.09732886477691666,-0.0766000050337147,0.09944220203272576,0.052933416813421515,-0.6362379150939101,-1.0999748867047898,0.61,-0.5067938262367422,-1.3873487335150236,-1.8274572629471952,-0.8949798182945309,-0.5042604443804907,-0.2836352571280305,0.909111664354187),(-1.3794478533340162,-0.2325994903990701,0.7988286239230257,-0.8949798182945309,-0.5042604443804907,-0.2164728460869087,-1.0999748867047898,0.57,0.4531256035702591,-0.5625278455750569,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.4047794560434521,0.909111664354187),(-1.4667132560437435,-1.2223845559270303,-0.0766000050337147,-1.3921908284581592,-0.5042604443804907,0.10254860635841155,0.909111664354187,0.64,0.4531256035702591,-0.5625278455750569,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.4047794560434521,0.909111664354187),(0.5403910062799865,0.9221497527168835,-0.0766000050337147,0.5966532121963541,-0.5042604443804907,1.2107283885368956,0.909111664354187,0.85,0.4531256035702591,-0.5625278455750569,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.4047794560434521,0.909111664354187),(0.1913293954410769,0.7571855751288902,-0.0766000050337147,-0.8949798182945309,-1.618648166768315,0.18650162015981303,0.909111664354187,0.78,0.4531256035702591,-0.5625278455750569,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.4047794560434521,0.909111664354187),(0.8021872144091686,0.7571855751288902,-0.0766000050337147,0.09944220203272576,-0.5042604443804907,0.8413351278107333,0.909111664354187,0.84,0.4531256035702591,-0.5625278455750569,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.4047794560434521,0.909111664354187),(1.4130450333772604,1.7469706406568504,1.674257252879766,1.5910752325236108,1.724515000395158,1.2611001968177347,0.909111664354187,0.92,0.4531256035702591,-0.5625278455750569,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.4047794560434521,0.909111664354187),(1.9366374496356247,1.087113930304877,1.674257252879766,0.5966532121963541,1.167321139201246,1.9495149099892173,0.909111664354187,0.96,0.4531256035702591,-0.5625278455750569,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.4047794560434521,0.909111664354187),(-1.2049170479145614,-0.39756366798706344,1.674257252879766,1.5910752325236108,1.167321139201246,0.08575800359813185,-1.0999748867047898,0.77,0.4531256035702591,-0.5625278455750569,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.4047794560434521,0.909111664354187),(-0.681324631656197,-0.39756366798706344,1.674257252879766,0.09944220203272576,0.052933416813421515,-0.06535742124438843,-1.0999748867047898,0.71,0.4531256035702591,-0.5625278455750569,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.4047794560434521,0.909111664354187),(0.5403910062799865,0.7571855751288902,1.674257252879766,0.5966532121963541,1.167321139201246,0.30403583948177093,-1.0999748867047898,0.79,0.4531256035702591,-0.5625278455750569,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.4047794560434521,0.909111664354187),(1.4130450333772604,0.9221497527168835,1.674257252879766,0.5966532121963541,0.6101272780073337,1.1435659774957738,0.909111664354187,0.89,0.4531256035702591,-0.5625278455750569,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.4047794560434521,0.909111664354187),(-0.24499761810756004,0.26229304236491,0.7988286239230257,0.09944220203272576,0.6101272780073337,0.2872452367214912,0.909111664354187,0.82,0.4531256035702591,-0.5625278455750569,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.4047794560434521,0.909111664354187),(-0.41952842352701486,-0.7274920231630502,-0.0766000050337147,1.5910752325236108,0.6101272780073337,-0.2500540516074711,-1.0999748867047898,0.76,0.1913293954410769,-0.39756366798706344,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.11933920911869125,0.909111664354187),(-0.07046681268810527,-1.2223845559270303,-0.952028633990455,-1.8894018386217877,-0.5042604443804907,-0.7369815316555913,0.909111664354187,0.71,0.1913293954410769,-0.39756366798706344,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.11933920911869125,0.909111664354187),(0.8021872144091686,1.4170422854808635,-0.952028633990455,1.0938642223599824,-0.5042604443804907,0.8077539222901738,0.909111664354187,0.8,0.1913293954410769,-0.39756366798706344,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.11933920911869125,0.909111664354187),(0.10406399273134952,0.26229304236491,-1.8274572629471952,0.09944220203272576,0.052933416813421515,0.8749163333312926,-1.0999748867047898,0.78,0.1913293954410769,-0.39756366798706344,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.11933920911869125,0.909111664354187),(1.0639834225383509,0.4272572199529034,-0.952028633990455,0.5966532121963541,-0.5042604443804907,0.9252881416121347,0.909111664354187,0.84,0.1913293954410769,-0.39756366798706344,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.11933920911869125,0.909111664354187),(1.3257796306675331,1.7469706406568504,-0.952028633990455,1.0938642223599824,0.052933416813421515,1.2778907995780144,0.909111664354187,0.9,0.1913293954410769,-0.39756366798706344,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.11933920911869125,0.909111664354187),(1.2385142279578056,1.2520781078928702,1.674257252879766,0.5966532121963541,0.052933416813421515,1.412215621660255,0.909111664354187,0.92,0.1913293954410769,-0.39756366798706344,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.11933920911869125,0.909111664354187),(2.023902852345352,2.076898995832837,0.7988286239230257,1.0938642223599824,0.6101272780073337,2.2181645541536983,0.909111664354187,0.97,0.1913293954410769,-0.39756366798706344,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.11933920911869125,0.909111664354187),(0.7149218116994412,0.7571855751288902,-0.952028633990455,-0.3977688081309026,0.052933416813421515,0.6062666891668145,0.909111664354187,0.8,0.1913293954410769,-0.39756366798706344,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.11933920911869125,0.909111664354187),(0.2785947981508043,0.9221497527168835,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,0.06896740083785215,0.909111664354187,0.81,0.1913293954410769,-0.39756366798706344,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.11933920911869125,0.909111664354187),(-0.15773221539783266,-0.39756366798706344,-0.0766000050337147,-1.3921908284581592,-1.0614543055744028,-0.199682243326629,-1.0999748867047898,0.75,0.1913293954410769,-0.39756366798706344,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.11933920911869125,0.909111664354187),(0.8021872144091686,1.087113930304877,-0.0766000050337147,-0.3977688081309026,-0.5042604443804907,0.8581257305710129,0.909111664354187,0.83,0.1913293954410769,-0.39756366798706344,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.11933920911869125,0.909111664354187),(1.9366374496356247,1.4170422854808635,0.7988286239230257,0.5966532121963541,0.052933416813421515,2.016677321030339,0.909111664354187,0.96,0.6276564089897139,0.4272572199529034,0.7988286239230257,1.0938642223599824,0.6101272780073337,0.9252881416121347,0.909111664354187),(-0.5067938262367422,-0.2325994903990701,-0.952028633990455,0.09944220203272576,-0.5042604443804907,-0.5690755040527913,0.909111664354187,0.79,0.6276564089897139,0.4272572199529034,0.7988286239230257,1.0938642223599824,0.6101272780073337,0.9252881416121347,0.909111664354187),(1.5003104360869879,1.087113930304877,0.7988286239230257,0.5966532121963541,0.6101272780073337,1.3954250188999753,0.909111664354187,0.93,0.6276564089897139,0.4272572199529034,0.7988286239230257,1.0938642223599824,0.6101272780073337,0.9252881416121347,0.909111664354187),(1.3257796306675331,1.4170422854808635,1.674257252879766,1.5910752325236108,1.724515000395158,1.1435659774957738,0.909111664354187,0.94,0.6276564089897139,0.4272572199529034,0.7988286239230257,1.0938642223599824,0.6101272780073337,0.9252881416121347,0.909111664354187),(0.36586020086053167,0.7571855751288902,1.674257252879766,1.5910752325236108,1.724515000395158,0.7741727167696144,0.909111664354187,0.86,0.6276564089897139,0.4272572199529034,0.7988286239230257,1.0938642223599824,0.6101272780073337,0.9252881416121347,0.909111664354187),(0.6276564089897139,-0.39756366798706344,-0.0766000050337147,-0.3977688081309026,0.6101272780073337,0.25366403120093184,-1.0999748867047898,0.79,0.6276564089897139,0.4272572199529034,0.7988286239230257,1.0938642223599824,0.6101272780073337,0.9252881416121347,0.909111664354187),(0.8021872144091686,0.09732886477691666,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.4887324698448536,-1.0999748867047898,0.8,0.6276564089897139,0.4272572199529034,0.7988286239230257,1.0938642223599824,0.6101272780073337,0.9252881416121347,0.909111664354187),(-0.41952842352701486,0.26229304236491,-0.0766000050337147,-0.3977688081309026,-0.5042604443804907,0.15292041463925066,-1.0999748867047898,0.77,0.6276564089897139,0.4272572199529034,0.7988286239230257,1.0938642223599824,0.6101272780073337,0.9252881416121347,0.909111664354187),(-0.15773221539783266,-0.39756366798706344,-0.0766000050337147,-1.3921908284581592,-1.0614543055744028,-0.4347506819705508,-1.0999748867047898,0.7,0.6276564089897139,0.4272572199529034,0.7988286239230257,1.0938642223599824,0.6101272780073337,0.9252881416121347,0.909111664354187),(-0.681324631656197,-0.5625278455750569,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-0.5690755040527913,-1.0999748867047898,0.65,0.6276564089897139,0.4272572199529034,0.7988286239230257,1.0938642223599824,0.6101272780073337,0.9252881416121347,0.909111664354187),(-0.9431208397853792,-0.2325994903990701,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-0.7705627371761508,-1.0999748867047898,0.61,0.6276564089897139,0.4272572199529034,0.7988286239230257,1.0938642223599824,0.6101272780073337,0.9252881416121347,0.909111664354187),(-1.7285094641729257,-1.2223845559270303,-1.8274572629471952,-1.8894018386217877,-1.618648166768315,-1.1735372034228724,-1.0999748867047898,0.52,0.6276564089897139,0.4272572199529034,0.7988286239230257,1.0938642223599824,0.6101272780073337,0.9252881416121347,0.909111664354187),(-0.15773221539783266,-0.7274920231630502,-1.8274572629471952,-1.8894018386217877,-1.618648166768315,-1.2406996144639928,-1.0999748867047898,0.57,0.6276564089897139,0.4272572199529034,0.7988286239230257,1.0938642223599824,0.6101272780073337,0.9252881416121347,0.909111664354187),(-1.6412440614631982,-1.3873487335150236,-1.8274572629471952,-1.8894018386217877,-0.5042604443804907,-1.9123237248751956,-1.0999748867047898,0.53,-1.4667132560437435,-1.2223845559270303,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,-0.5690755040527913,-1.0999748867047898),(0.10406399273134952,0.26229304236491,-0.0766000050337147,-0.3977688081309026,-0.5042604443804907,-0.1661010378060696,-1.0999748867047898,0.67,-1.4667132560437435,-1.2223845559270303,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,-0.5690755040527913,-1.0999748867047898),(0.016798590021622126,-0.39756366798706344,-0.0766000050337147,0.09944220203272576,-0.5042604443804907,-0.06535742124438843,-1.0999748867047898,0.68,-1.4667132560437435,-1.2223845559270303,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,-0.5690755040527913,-1.0999748867047898),(1.0639834225383509,0.5922213975408968,0.7988286239230257,1.0938642223599824,0.6101272780073337,0.690219702968213,0.909111664354187,0.81,-1.4667132560437435,-1.2223845559270303,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,-0.5690755040527913,-1.0999748867047898),(0.4531256035702591,0.4272572199529034,1.674257252879766,1.0938642223599824,0.6101272780073337,0.6230572919270941,-1.0999748867047898,0.78,-1.4667132560437435,-1.2223845559270303,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,-0.5690755040527913,-1.0999748867047898),(-1.2921824506242887,-0.8924562007510436,-0.0766000050337147,0.09944220203272576,1.724515000395158,-0.45154128473083044,-1.0999748867047898,0.65,-1.4667132560437435,-1.2223845559270303,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,-0.5690755040527913,-1.0999748867047898),(-0.3322630208172874,-0.8924562007510436,-0.0766000050337147,-1.3921908284581592,-0.5042604443804907,-0.5522849012925116,-1.0999748867047898,0.64,-1.4667132560437435,-1.2223845559270303,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,-0.5690755040527913,-1.0999748867047898),(-2.0775710750118352,-1.7172770886910105,-0.952028633990455,-1.3921908284581592,0.6101272780073337,-1.3414432310256739,0.909111664354187,0.64,-1.4667132560437435,-1.2223845559270303,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,-0.5690755040527913,-1.0999748867047898),(-0.5067938262367422,-1.3873487335150236,-0.952028633990455,-0.8949798182945309,-0.5042604443804907,-1.0392123813406318,-1.0999748867047898,0.65,-1.4667132560437435,-1.2223845559270303,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,-0.5690755040527913,-1.0999748867047898),(-0.41952842352701486,-1.057420378339037,-0.952028633990455,-0.8949798182945309,0.052933416813421515,-0.9384687647789537,0.909111664354187,0.68,-1.4667132560437435,-1.2223845559270303,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,-0.5690755040527913,-1.0999748867047898),(1.5003104360869879,1.582006463068857,1.674257252879766,0.5966532121963541,1.167321139201246,0.7909633195298942,0.909111664354187,0.89,-1.4667132560437435,-1.2223845559270303,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,-0.5690755040527913,-1.0999748867047898),(0.4531256035702591,0.4272572199529034,0.7988286239230257,0.5966532121963541,1.724515000395158,0.8917069360915754,0.909111664354187,0.86,-1.4667132560437435,-1.2223845559270303,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,-0.5690755040527913,-1.0999748867047898),(0.5403910062799865,0.9221497527168835,0.7988286239230257,0.5966532121963541,1.167321139201246,1.0596129636943752,0.909111664354187,0.89,2.023902852345352,0.9221497527168835,0.7988286239230257,1.5910752325236108,1.724515000395158,1.915933704468658,0.909111664354187),(0.36586020086053167,0.5922213975408968,0.7988286239230257,0.5966532121963541,0.6101272780073337,0.6230572919270941,0.909111664354187,0.87,2.023902852345352,0.9221497527168835,0.7988286239230257,1.5910752325236108,1.724515000395158,1.915933704468658,0.909111664354187),(0.2785947981508043,0.5922213975408968,0.7988286239230257,1.0938642223599824,0.052933416813421515,0.4551512643242912,0.909111664354187,0.85,2.023902852345352,0.9221497527168835,0.7988286239230257,1.5910752325236108,1.724515000395158,1.915933704468658,0.909111664354187),(1.0639834225383509,1.9119348182448437,0.7988286239230257,1.0938642223599824,1.167321139201246,0.9420787443724145,0.909111664354187,0.9,2.023902852345352,0.9221497527168835,0.7988286239230257,1.5910752325236108,1.724515000395158,1.915933704468658,0.909111664354187),(0.1913293954410769,0.4272572199529034,-0.0766000050337147,0.09944220203272576,0.052933416813421515,0.7405915112490521,-1.0999748867047898,0.82,2.023902852345352,0.9221497527168835,0.7988286239230257,1.5910752325236108,1.724515000395158,1.915933704468658,0.909111664354187),(-0.681324631656197,0.09732886477691666,-0.0766000050337147,-0.8949798182945309,-0.5042604443804907,-0.8041439426967131,-1.0999748867047898,0.72,2.023902852345352,0.9221497527168835,0.7988286239230257,1.5910752325236108,1.724515000395158,1.915933704468658,0.909111664354187),(-0.8558554370756518,-0.8924562007510436,-0.0766000050337147,-0.3977688081309026,-0.5042604443804907,-0.5522849012925116,-1.0999748867047898,0.73,2.023902852345352,0.9221497527168835,0.7988286239230257,1.5910752325236108,1.724515000395158,1.915933704468658,0.909111664354187),(-1.4667132560437435,-0.5625278455750569,-0.0766000050337147,0.09944220203272576,-0.5042604443804907,-0.7369815316555913,-1.0999748867047898,0.71,2.023902852345352,0.9221497527168835,0.7988286239230257,1.5910752325236108,1.724515000395158,1.915933704468658,0.909111664354187),(-1.0303862424951067,-0.06763531281107672,-0.952028633990455,-0.8949798182945309,-1.0614543055744028,-0.30042585988831016,-1.0999748867047898,0.71,2.023902852345352,0.9221497527168835,0.7988286239230257,1.5910752325236108,1.724515000395158,1.915933704468658,0.909111664354187),(-1.553978658753471,-1.2223845559270303,-0.952028633990455,-0.3977688081309026,0.052933416813421515,-1.2071184089434333,-1.0999748867047898,0.68,2.023902852345352,0.9221497527168835,0.7988286239230257,1.5910752325236108,1.724515000395158,1.915933704468658,0.909111664354187),(-0.24499761810756004,0.4272572199529034,-0.0766000050337147,0.5966532121963541,0.6101272780073337,0.3376170450023333,-1.0999748867047898,0.75,2.023902852345352,0.9221497527168835,0.7988286239230257,1.5910752325236108,1.724515000395158,1.915933704468658,0.909111664354187),(-0.07046681268810527,-0.2325994903990701,-0.952028633990455,-0.8949798182945309,0.6101272780073337,-0.46833188749111015,-1.0999748867047898,0.72,2.023902852345352,0.9221497527168835,0.7988286239230257,1.5910752325236108,1.724515000395158,1.915933704468658,0.909111664354187),(0.889452617118896,0.9221497527168835,0.7988286239230257,1.0938642223599824,1.167321139201246,0.8581257305710129,0.909111664354187,0.89,1.5875758387967152,1.582006463068857,1.674257252879766,1.5910752325236108,1.724515000395158,2.0502585265508984,0.909111664354187),(0.016798590021622126,-0.06763531281107672,-0.0766000050337147,0.09944220203272576,-0.5042604443804907,0.13612981187897094,0.909111664354187,0.84,1.5875758387967152,1.582006463068857,1.674257252879766,1.5910752325236108,1.724515000395158,2.0502585265508984,0.909111664354187),(1.5875758387967152,1.7469706406568504,1.674257252879766,1.0938642223599824,0.052933416813421515,1.412215621660255,0.909111664354187,0.93,1.5875758387967152,1.582006463068857,1.674257252879766,1.5910752325236108,1.724515000395158,2.0502585265508984,0.909111664354187),(1.2385142279578056,1.2520781078928702,1.674257252879766,1.0938642223599824,0.052933416813421515,1.2778907995780144,0.909111664354187,0.93,1.5875758387967152,1.582006463068857,1.674257252879766,1.5910752325236108,1.724515000395158,2.0502585265508984,0.909111664354187),(0.6276564089897139,0.7571855751288902,1.674257252879766,1.5910752325236108,1.724515000395158,0.8077539222901738,0.909111664354187,0.88,1.5875758387967152,1.582006463068857,1.674257252879766,1.5910752325236108,1.724515000395158,2.0502585265508984,0.909111664354187),(0.6276564089897139,0.5922213975408968,1.674257252879766,1.0938642223599824,0.6101272780073337,0.9420787443724145,0.909111664354187,0.9,1.5875758387967152,1.582006463068857,1.674257252879766,1.5910752325236108,1.724515000395158,2.0502585265508984,0.909111664354187),(0.5403910062799865,0.4272572199529034,1.674257252879766,0.5966532121963541,1.724515000395158,0.6398478946873739,0.909111664354187,0.87,1.5875758387967152,1.582006463068857,1.674257252879766,1.5910752325236108,1.724515000395158,2.0502585265508984,0.909111664354187),(0.4531256035702591,1.087113930304877,1.674257252879766,1.0938642223599824,0.6101272780073337,0.572685483646252,0.909111664354187,0.86,1.5875758387967152,1.582006463068857,1.674257252879766,1.5910752325236108,1.724515000395158,2.0502585265508984,0.909111664354187),(1.6748412415064426,1.7469706406568504,1.674257252879766,1.0938642223599824,1.724515000395158,1.5633310465027752,0.909111664354187,0.94,1.5875758387967152,1.582006463068857,1.674257252879766,1.5910752325236108,1.724515000395158,2.0502585265508984,0.909111664354187),(-0.07046681268810527,0.26229304236491,-0.0766000050337147,0.09944220203272576,-0.5042604443804907,0.27045463396121155,-1.0999748867047898,0.77,1.5875758387967152,1.582006463068857,1.674257252879766,1.5910752325236108,1.724515000395158,2.0502585265508984,0.909111664354187),(-0.8558554370756518,-0.06763531281107672,-0.952028633990455,-0.3977688081309026,0.052933416813421515,-0.1325198322855102,0.909111664354187,0.78,1.5875758387967152,1.582006463068857,1.674257252879766,1.5910752325236108,1.724515000395158,2.0502585265508984,0.909111664354187),(-0.9431208397853792,-0.39756366798706344,-0.952028633990455,-0.3977688081309026,-1.0614543055744028,-0.5690755040527913,-1.0999748867047898,0.73,1.5875758387967152,1.582006463068857,1.674257252879766,1.5910752325236108,1.724515000395158,2.0502585265508984,0.909111664354187),(-0.5940592289464697,-0.2325994903990701,-0.952028633990455,0.09944220203272576,-1.0614543055744028,-0.45154128473083044,-1.0999748867047898,0.73,1.5875758387967152,1.582006463068857,1.674257252879766,1.5910752325236108,1.724515000395158,2.0502585265508984,0.909111664354187),(-0.5067938262367422,-0.5625278455750569,-0.0766000050337147,1.0938642223599824,1.167321139201246,-0.2836352571280305,-1.0999748867047898,0.7,-1.2921824506242887,-1.057420378339037,-0.952028633990455,-0.8949798182945309,0.052933416813421515,-1.0727935868611929,-1.0999748867047898),(-0.3322630208172874,-0.06763531281107672,-0.0766000050337147,0.5966532121963541,1.167321139201246,0.15292041463925066,-1.0999748867047898,0.72,-1.2921824506242887,-1.057420378339037,-0.952028633990455,-0.8949798182945309,0.052933416813421515,-1.0727935868611929,-1.0999748867047898),(0.016798590021622126,-0.7274920231630502,-0.0766000050337147,-0.8949798182945309,-0.5042604443804907,-0.0989386267649508,0.909111664354187,0.73,-1.2921824506242887,-1.057420378339037,-0.952028633990455,-0.8949798182945309,0.052933416813421515,-1.0727935868611929,-1.0999748867047898),(-0.15773221539783266,0.4272572199529034,-0.952028633990455,0.09944220203272576,-0.5042604443804907,-0.2332634488471884,0.909111664354187,0.72,-1.2921824506242887,-1.057420378339037,-0.952028633990455,-0.8949798182945309,0.052933416813421515,-1.0727935868611929,-1.0999748867047898),(2.023902852345352,2.076898995832837,1.674257252879766,1.0938642223599824,1.167321139201246,2.2013739513934185,0.909111664354187,0.97,-1.2921824506242887,-1.057420378339037,-0.952028633990455,-0.8949798182945309,0.052933416813421515,-1.0727935868611929,-1.0999748867047898),(1.5003104360869879,2.076898995832837,1.674257252879766,0.5966532121963541,1.724515000395158,2.1342115403522968,0.909111664354187,0.97,-1.2921824506242887,-1.057420378339037,-0.952028633990455,-0.8949798182945309,0.052933416813421515,-1.0727935868611929,-1.0999748867047898),(-1.6412440614631982,-0.39756366798706344,-0.0766000050337147,0.09944220203272576,0.6101272780073337,-0.0989386267649508,-1.0999748867047898,0.69,-1.2921824506242887,-1.057420378339037,-0.952028633990455,-0.8949798182945309,0.052933416813421515,-1.0727935868611929,-1.0999748867047898),(-1.9030402695923805,-1.3873487335150236,-0.952028633990455,-0.8949798182945309,-0.5042604443804907,-1.5933022724298738,-1.0999748867047898,0.57,-1.2921824506242887,-1.057420378339037,-0.952028633990455,-0.8949798182945309,0.052933416813421515,-1.0727935868611929,-1.0999748867047898),(-0.15773221539783266,-1.3873487335150236,-0.952028633990455,0.09944220203272576,-0.5042604443804907,-1.1903278061831537,-1.0999748867047898,0.63,-1.2921824506242887,-1.057420378339037,-0.952028633990455,-0.8949798182945309,0.052933416813421515,-1.0727935868611929,-1.0999748867047898),(-0.5940592289464697,-0.8924562007510436,-0.0766000050337147,0.09944220203272576,0.6101272780073337,-0.972049970299513,0.909111664354187,0.66,-1.2921824506242887,-1.057420378339037,-0.952028633990455,-0.8949798182945309,0.052933416813421515,-1.0727935868611929,-1.0999748867047898),(-1.0303862424951067,-0.2325994903990701,-0.952028633990455,-0.3977688081309026,-0.5042604443804907,-0.7369815316555913,-1.0999748867047898,0.64,-1.2921824506242887,-1.057420378339037,-0.952028633990455,-0.8949798182945309,0.052933416813421515,-1.0727935868611929,-1.0999748867047898),(-1.3794478533340162,-0.5625278455750569,-0.0766000050337147,0.09944220203272576,0.6101272780073337,-0.8041439426967131,0.909111664354187,0.68,-1.2921824506242887,-1.057420378339037,-0.952028633990455,-0.8949798182945309,0.052933416813421515,-1.0727935868611929,-1.0999748867047898),(0.7149218116994412,0.09732886477691666,0.7988286239230257,1.0938642223599824,0.6101272780073337,0.7741727167696144,0.909111664354187,0.79,-0.8558554370756518,-0.39756366798706344,-0.952028633990455,-1.3921908284581592,0.052933416813421515,-0.8377251482172725,-1.0999748867047898),(0.9767180198286235,0.4272572199529034,0.7988286239230257,1.5910752325236108,0.6101272780073337,0.908497538851855,0.909111664354187,0.82,-0.8558554370756518,-0.39756366798706344,-0.952028633990455,-1.3921908284581592,0.052933416813421515,-0.8377251482172725,-1.0999748867047898),(1.8493720469258974,2.076898995832837,0.7988286239230257,1.5910752325236108,1.724515000395158,1.7816088823864173,0.909111664354187,0.95,-0.8558554370756518,-0.39756366798706344,-0.952028633990455,-1.3921908284581592,0.052933416813421515,-0.8377251482172725,-1.0999748867047898),(1.4130450333772604,1.9119348182448437,1.674257252879766,1.5910752325236108,1.167321139201246,1.9830961155097766,0.909111664354187,0.96,-0.8558554370756518,-0.39756366798706344,-0.952028633990455,-1.3921908284581592,0.052933416813421515,-0.8377251482172725,-1.0999748867047898),(1.2385142279578056,1.582006463068857,0.7988286239230257,1.0938642223599824,1.724515000395158,1.3786344161396955,0.909111664354187,0.94,-0.8558554370756518,-0.39756366798706344,-0.952028633990455,-1.3921908284581592,0.052933416813421515,-0.8377251482172725,-1.0999748867047898),(1.1512488252480781,1.4170422854808635,1.674257252879766,1.5910752325236108,1.167321139201246,1.2778907995780144,0.909111664354187,0.93,-0.8558554370756518,-0.39756366798706344,-0.952028633990455,-1.3921908284581592,0.052933416813421515,-0.8377251482172725,-1.0999748867047898),(0.4531256035702591,0.7571855751288902,0.7988286239230257,1.0938642223599824,1.167321139201246,1.1099847719752143,0.909111664354187,0.91,-0.8558554370756518,-0.39756366798706344,-0.952028633990455,-1.3921908284581592,0.052933416813421515,-0.8377251482172725,-1.0999748867047898),(0.36586020086053167,0.26229304236491,0.7988286239230257,0.5966532121963541,0.6101272780073337,0.8917069360915754,0.909111664354187,0.85,-0.8558554370756518,-0.39756366798706344,-0.952028633990455,-1.3921908284581592,0.052933416813421515,-0.8377251482172725,-1.0999748867047898),(0.6276564089897139,0.4272572199529034,0.7988286239230257,-0.3977688081309026,0.052933416813421515,0.6230572919270941,0.909111664354187,0.84,-0.8558554370756518,-0.39756366798706344,-0.952028633990455,-1.3921908284581592,0.052933416813421515,-0.8377251482172725,-1.0999748867047898),(-0.41952842352701486,-0.5625278455750569,-0.0766000050337147,0.09944220203272576,0.052933416813421515,-0.30042585988831016,-1.0999748867047898,0.74,-0.8558554370756518,-0.39756366798706344,-0.952028633990455,-1.3921908284581592,0.052933416813421515,-0.8377251482172725,-1.0999748867047898),(-0.3322630208172874,-0.7274920231630502,-0.0766000050337147,0.5966532121963541,0.6101272780073337,0.25366403120093184,-1.0999748867047898,0.76,-0.8558554370756518,-0.39756366798706344,-0.952028633990455,-1.3921908284581592,0.052933416813421515,-0.8377251482172725,-1.0999748867047898),(-0.07046681268810527,0.4272572199529034,-0.0766000050337147,0.09944220203272576,0.6101272780073337,-0.06535742124438843,-1.0999748867047898,0.75,-0.8558554370756518,-0.39756366798706344,-0.952028633990455,-1.3921908284581592,0.052933416813421515,-0.8377251482172725,-1.0999748867047898),(0.6276564089897139,0.9221497527168835,0.7988286239230257,1.0938642223599824,0.6101272780073337,0.32082644224205065,-1.0999748867047898,0.76,-1.8157748668826532,-1.7172770886910105,-0.952028633990455,-1.8894018386217877,-1.618648166768315,-1.3414432310256739,-1.0999748867047898),(-0.7685900343659244,0.26229304236491,-0.952028633990455,-0.3977688081309026,0.6101272780073337,-0.2500540516074711,-1.0999748867047898,0.71,-1.8157748668826532,-1.7172770886910105,-0.952028633990455,-1.8894018386217877,-1.618648166768315,-1.3414432310256739,-1.0999748867047898),(-1.0303862424951067,-0.39756366798706344,-0.952028633990455,-0.3977688081309026,-1.618648166768315,-0.6194473123336305,-1.0999748867047898,0.67,-1.8157748668826532,-1.7172770886910105,-0.952028633990455,-1.8894018386217877,-1.618648166768315,-1.3414432310256739,-1.0999748867047898),(-1.8157748668826532,-1.3873487335150236,-0.952028633990455,-0.8949798182945309,-1.0614543055744028,-0.9552593675392334,-1.0999748867047898,0.61,-1.8157748668826532,-1.7172770886910105,-0.952028633990455,-1.8894018386217877,-1.618648166768315,-1.3414432310256739,-1.0999748867047898),(-0.9431208397853792,0.4272572199529034,-0.952028633990455,0.09944220203272576,0.6101272780073337,-0.2500540516074711,-1.0999748867047898,0.63,-1.8157748668826532,-1.7172770886910105,-0.952028633990455,-1.8894018386217877,-1.618648166768315,-1.3414432310256739,-1.0999748867047898),(-0.41952842352701486,0.4272572199529034,-0.952028633990455,0.09944220203272576,-0.5042604443804907,-0.1157292295252305,-1.0999748867047898,0.64,-1.8157748668826532,-1.7172770886910105,-0.952028633990455,-1.8894018386217877,-1.618648166768315,-1.3414432310256739,-1.0999748867047898),(0.10406399273134952,0.7571855751288902,-0.0766000050337147,0.5966532121963541,0.052933416813421515,0.11933920911869125,-1.0999748867047898,0.71,-1.8157748668826532,-1.7172770886910105,-0.952028633990455,-1.8894018386217877,-1.618648166768315,-1.3414432310256739,-1.0999748867047898),(0.6276564089897139,0.5922213975408968,0.7988286239230257,-0.3977688081309026,-0.5042604443804907,0.690219702968213,0.909111664354187,0.82,-1.8157748668826532,-1.7172770886910105,-0.952028633990455,-1.8894018386217877,-1.618648166768315,-1.3414432310256739,-1.0999748867047898),(-0.3322630208172874,-0.5625278455750569,-0.0766000050337147,0.5966532121963541,1.167321139201246,0.08575800359813185,-1.0999748867047898,0.73,-1.8157748668826532,-1.7172770886910105,-0.952028633990455,-1.8894018386217877,-1.618648166768315,-1.3414432310256739,-1.0999748867047898),(0.1913293954410769,-0.2325994903990701,-0.0766000050337147,0.09944220203272576,-1.0614543055744028,-0.45154128473083044,0.909111664354187,0.74,-1.8157748668826532,-1.7172770886910105,-0.952028633990455,-1.8894018386217877,-1.618648166768315,-1.3414432310256739,-1.0999748867047898),(-0.41952842352701486,-0.06763531281107672,-0.952028633990455,-0.8949798182945309,0.052933416813421515,-0.5522849012925116,-1.0999748867047898,0.69,-1.8157748668826532,-1.7172770886910105,-0.952028633990455,-1.8894018386217877,-1.618648166768315,-1.3414432310256739,-1.0999748867047898),(-1.117651645204834,-1.2223845559270303,-0.952028633990455,-0.8949798182945309,0.052933416813421515,-0.8880969564981116,-1.0999748867047898,0.64,-1.8157748668826532,-1.7172770886910105,-0.952028633990455,-1.8894018386217877,-1.618648166768315,-1.3414432310256739,-1.0999748867047898),(1.1512488252480781,0.9221497527168835,1.674257252879766,1.5910752325236108,0.6101272780073337,1.1939377857766158,0.909111664354187,0.91,-1.8157748668826532,-1.7172770886910105,-0.952028633990455,-1.8894018386217877,-1.618648166768315,-1.3414432310256739,-1.0999748867047898),(0.8021872144091686,0.5922213975408968,1.674257252879766,1.0938642223599824,0.6101272780073337,1.0596129636943752,0.909111664354187,0.88,0.2785947981508043,0.09732886477691666,-0.0766000050337147,0.09944220203272576,0.6101272780073337,-0.2668446543677508,0.909111664354187),(0.7149218116994412,0.7571855751288902,0.7988286239230257,0.5966532121963541,1.167321139201246,0.9588693471326941,0.909111664354187,0.85,0.2785947981508043,0.09732886477691666,-0.0766000050337147,0.09944220203272576,0.6101272780073337,-0.2668446543677508,0.909111664354187),(1.0639834225383509,1.087113930304877,1.674257252879766,1.0938642223599824,1.724515000395158,0.9924505526532535,0.909111664354187,0.86,0.2785947981508043,0.09732886477691666,-0.0766000050337147,0.09944220203272576,0.6101272780073337,-0.2668446543677508,0.909111664354187),(-0.5940592289464697,-0.5625278455750569,-0.0766000050337147,-1.3921908284581592,0.052933416813421515,-0.3843788736897117,-1.0999748867047898,0.7,0.2785947981508043,0.09732886477691666,-0.0766000050337147,0.09944220203272576,0.6101272780073337,-0.2668446543677508,0.909111664354187),(-1.553978658753471,-1.2223845559270303,-1.8274572629471952,-1.8894018386217877,-1.618648166768315,-1.1903278061831537,-1.0999748867047898,0.59,0.2785947981508043,0.09732886477691666,-0.0766000050337147,0.09944220203272576,0.6101272780073337,-0.2668446543677508,0.909111664354187),(-1.8157748668826532,-1.057420378339037,-1.8274572629471952,-0.8949798182945309,-0.5042604443804907,-1.5429304641490347,-1.0999748867047898,0.6,0.2785947981508043,0.09732886477691666,-0.0766000050337147,0.09944220203272576,0.6101272780073337,-0.2668446543677508,0.909111664354187),(0.016798590021622126,-0.7274920231630502,-0.952028633990455,-0.8949798182945309,-1.618648166768315,-0.753772134415871,-1.0999748867047898,0.65,0.2785947981508043,0.09732886477691666,-0.0766000050337147,0.09944220203272576,0.6101272780073337,-0.2668446543677508,0.909111664354187),(0.6276564089897139,1.2520781078928702,-0.0766000050337147,0.09944220203272576,-0.5042604443804907,0.27045463396121155,0.909111664354187,0.7,0.2785947981508043,0.09732886477691666,-0.0766000050337147,0.09944220203272576,0.6101272780073337,-0.2668446543677508,0.909111664354187),(0.7149218116994412,1.087113930304877,-0.0766000050337147,0.09944220203272576,-0.5042604443804907,0.7405915112490521,0.909111664354187,0.76,0.2785947981508043,0.09732886477691666,-0.0766000050337147,0.09944220203272576,0.6101272780073337,-0.2668446543677508,0.909111664354187),(-0.24499761810756004,-0.06763531281107672,-0.952028633990455,-0.8949798182945309,0.6101272780073337,-0.06535742124438843,-1.0999748867047898,0.63,0.2785947981508043,0.09732886477691666,-0.0766000050337147,0.09944220203272576,0.6101272780073337,-0.2668446543677508,0.909111664354187),(0.9767180198286235,0.4272572199529034,0.7988286239230257,0.5966532121963541,-1.0614543055744028,0.7070103057284927,0.909111664354187,0.81,0.2785947981508043,0.09732886477691666,-0.0766000050337147,0.09944220203272576,0.6101272780073337,-0.2668446543677508,0.909111664354187),(-0.07046681268810527,-0.39756366798706344,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.22008282568037243,-1.0999748867047898,0.72,0.2785947981508043,0.09732886477691666,-0.0766000050337147,0.09944220203272576,0.6101272780073337,-0.2668446543677508,0.909111664354187),(-0.5067938262367422,-0.5625278455750569,-0.952028633990455,-0.8949798182945309,0.052933416813421515,-0.199682243326629,-1.0999748867047898,0.71,-0.24499761810756004,-0.8924562007510436,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-0.6026567095733507,-1.0999748867047898),(0.6276564089897139,0.4272572199529034,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.4551512643242912,0.909111664354187,0.8,-0.24499761810756004,-0.8924562007510436,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-0.6026567095733507,-1.0999748867047898),(0.36586020086053167,0.5922213975408968,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.3879888532831724,0.909111664354187,0.77,-0.24499761810756004,-0.8924562007510436,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-0.6026567095733507,-1.0999748867047898),(0.2785947981508043,-0.5625278455750569,-0.0766000050337147,-0.3977688081309026,-1.0614543055744028,-0.04856681848410872,0.909111664354187,0.74,-0.24499761810756004,-0.8924562007510436,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-0.6026567095733507,-1.0999748867047898),(-0.07046681268810527,-1.3873487335150236,-0.952028633990455,-0.8949798182945309,-0.5042604443804907,0.6734291002079332,-1.0999748867047898,0.7,-0.24499761810756004,-0.8924562007510436,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-0.6026567095733507,-1.0999748867047898),(0.10406399273134952,-1.2223845559270303,-0.952028633990455,-0.8949798182945309,0.052933416813421515,-0.0989386267649508,0.909111664354187,0.71,-0.24499761810756004,-0.8924562007510436,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-0.6026567095733507,-1.0999748867047898),(1.5875758387967152,1.2520781078928702,0.7988286239230257,1.0938642223599824,1.167321139201246,1.8151900879069767,0.909111664354187,0.93,-0.24499761810756004,-0.8924562007510436,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-0.6026567095733507,-1.0999748867047898),(0.36586020086053167,1.087113930304877,0.7988286239230257,0.5966532121963541,1.724515000395158,0.8749163333312926,-1.0999748867047898,0.85,-0.24499761810756004,-0.8924562007510436,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-0.6026567095733507,-1.0999748867047898),(-0.8558554370756518,0.4272572199529034,0.7988286239230257,0.5966532121963541,1.167321139201246,-0.3843788736897117,-1.0999748867047898,0.79,-0.24499761810756004,-0.8924562007510436,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-0.6026567095733507,-1.0999748867047898),(-0.681324631656197,-1.3873487335150236,-0.0766000050337147,0.5966532121963541,0.6101272780073337,-0.06535742124438843,-1.0999748867047898,0.76,-0.24499761810756004,-0.8924562007510436,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-0.6026567095733507,-1.0999748867047898),(0.6276564089897139,-1.2223845559270303,-0.0766000050337147,0.5966532121963541,1.724515000395158,0.06896740083785215,0.909111664354187,0.78,-0.24499761810756004,-0.8924562007510436,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-0.6026567095733507,-1.0999748867047898),(0.8021872144091686,-0.8924562007510436,0.7988286239230257,1.5910752325236108,1.724515000395158,0.27045463396121155,0.909111664354187,0.77,-0.24499761810756004,-0.8924562007510436,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-0.6026567095733507,-1.0999748867047898),(1.2385142279578056,1.9119348182448437,0.7988286239230257,1.5910752325236108,1.167321139201246,1.244309594057455,0.909111664354187,0.9,0.10406399273134952,-0.2325994903990701,-0.0766000050337147,-1.3921908284581592,-0.5042604443804907,0.08575800359813185,-1.0999748867047898),(0.889452617118896,0.09732886477691666,1.674257252879766,1.5910752325236108,0.052933416813421515,0.8917069360915754,0.909111664354187,0.87,0.10406399273134952,-0.2325994903990701,-0.0766000050337147,-1.3921908284581592,-0.5042604443804907,0.08575800359813185,-1.0999748867047898),(-0.41952842352701486,-0.5625278455750569,-0.0766000050337147,0.09944220203272576,0.6101272780073337,-0.8545157509775522,-1.0999748867047898,0.71,0.10406399273134952,-0.2325994903990701,-0.0766000050337147,-1.3921908284581592,-0.5042604443804907,0.08575800359813185,-1.0999748867047898),(-0.7685900343659244,-0.7274920231630502,-0.952028633990455,-0.8949798182945309,0.6101272780073337,-0.40116947644999135,0.909111664354187,0.7,0.10406399273134952,-0.2325994903990701,-0.0766000050337147,-1.3921908284581592,-0.5042604443804907,0.08575800359813185,-1.0999748867047898),(0.6276564089897139,0.5922213975408968,-0.0766000050337147,-0.8949798182945309,-2.175842027962227,0.32082644224205065,0.909111664354187,0.7,0.10406399273134952,-0.2325994903990701,-0.0766000050337147,-1.3921908284581592,-0.5042604443804907,0.08575800359813185,-1.0999748867047898),(0.7149218116994412,0.4272572199529034,-0.952028633990455,-0.3977688081309026,-1.0614543055744028,0.27045463396121155,0.909111664354187,0.75,0.10406399273134952,-0.2325994903990701,-0.0766000050337147,-1.3921908284581592,-0.5042604443804907,0.08575800359813185,-1.0999748867047898),(-0.3322630208172874,-0.8924562007510436,-0.0766000050337147,-0.8949798182945309,-1.0614543055744028,0.13612981187897094,-1.0999748867047898,0.71,0.10406399273134952,-0.2325994903990701,-0.0766000050337147,-1.3921908284581592,-0.5042604443804907,0.08575800359813185,-1.0999748867047898),(-0.41952842352701486,-0.39756366798706344,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-0.2500540516074711,-1.0999748867047898,0.72,0.10406399273134952,-0.2325994903990701,-0.0766000050337147,-1.3921908284581592,-0.5042604443804907,0.08575800359813185,-1.0999748867047898),(-0.24499761810756004,-0.06763531281107672,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,-0.7201909288953117,0.909111664354187,0.73,0.10406399273134952,-0.2325994903990701,-0.0766000050337147,-1.3921908284581592,-0.5042604443804907,0.08575800359813185,-1.0999748867047898),(0.889452617118896,0.9221497527168835,0.7988286239230257,1.0938642223599824,1.724515000395158,0.908497538851855,-1.0999748867047898,0.83,0.10406399273134952,-0.2325994903990701,-0.0766000050337147,-1.3921908284581592,-0.5042604443804907,0.08575800359813185,-1.0999748867047898),(-0.7685900343659244,0.09732886477691666,0.7988286239230257,1.0938642223599824,1.724515000395158,-0.4347506819705508,-1.0999748867047898,0.77,0.10406399273134952,-0.2325994903990701,-0.0766000050337147,-1.3921908284581592,-0.5042604443804907,0.08575800359813185,-1.0999748867047898),(-0.9431208397853792,-0.39756366798706344,-0.952028633990455,-0.8949798182945309,-0.5042604443804907,-0.6362379150939101,0.909111664354187,0.72,0.10406399273134952,-0.2325994903990701,-0.0766000050337147,-1.3921908284581592,-0.5042604443804907,0.08575800359813185,-1.0999748867047898),(-1.553978658753471,-1.8822412662790038,-0.952028633990455,-1.8894018386217877,-1.618648166768315,-1.2406996144639928,-1.0999748867047898,0.54,0.10406399273134952,-0.2325994903990701,-0.0766000050337147,-1.3921908284581592,-0.5042604443804907,0.08575800359813185,-1.0999748867047898),(-1.990305672302108,-2.0472054438669973,-1.8274572629471952,-1.8894018386217877,-2.175842027962227,-1.610092875190155,-1.0999748867047898,0.49,0.8021872144091686,0.7571855751288902,0.7988286239230257,0.5966532121963541,0.052933416813421515,0.8749163333312926,0.909111664354187),(-0.41952842352701486,-1.3873487335150236,-1.8274572629471952,-2.386612848785416,-2.175842027962227,-0.9888405730597928,0.909111664354187,0.52,0.8021872144091686,0.7571855751288902,0.7988286239230257,0.5966532121963541,0.052933416813421515,0.8749163333312926,0.909111664354187),(-0.15773221539783266,-1.2223845559270303,-1.8274572629471952,-1.3921908284581592,-1.0614543055744028,-1.0895841896214724,-1.0999748867047898,0.58,0.8021872144091686,0.7571855751288902,0.7988286239230257,0.5966532121963541,0.052933416813421515,0.8749163333312926,0.909111664354187),(0.4531256035702591,0.4272572199529034,-0.0766000050337147,0.09944220203272576,-0.5042604443804907,0.6062666891668145,0.909111664354187,0.78,0.8021872144091686,0.7571855751288902,0.7988286239230257,0.5966532121963541,0.052933416813421515,0.8749163333312926,0.909111664354187),(1.0639834225383509,0.9221497527168835,1.674257252879766,1.5910752325236108,1.167321139201246,1.4290062244205346,0.909111664354187,0.89,0.8021872144091686,0.7571855751288902,0.7988286239230257,0.5966532121963541,0.052933416813421515,0.8749163333312926,0.909111664354187),(0.2785947981508043,-1.057420378339037,-0.952028633990455,-0.8949798182945309,-0.5042604443804907,0.03538619531728977,-1.0999748867047898,0.7,0.8021872144091686,0.7571855751288902,0.7988286239230257,0.5966532121963541,0.052933416813421515,0.8749163333312926,0.909111664354187),(-0.7685900343659244,-0.7274920231630502,-0.952028633990455,-0.3977688081309026,0.052933416813421515,-0.1828916405663493,-1.0999748867047898,0.66,0.8021872144091686,0.7571855751288902,0.7988286239230257,0.5966532121963541,0.052933416813421515,0.8749163333312926,0.909111664354187),(-1.117651645204834,-0.8924562007510436,-0.952028633990455,-0.3977688081309026,0.6101272780073337,0.22008282568037243,-1.0999748867047898,0.67,0.8021872144091686,0.7571855751288902,0.7988286239230257,0.5966532121963541,0.052933416813421515,0.8749163333312926,0.909111664354187),(-0.5067938262367422,-0.8924562007510436,-0.0766000050337147,1.0938642223599824,0.6101272780073337,0.06896740083785215,0.909111664354187,0.68,0.8021872144091686,0.7571855751288902,0.7988286239230257,0.5966532121963541,0.052933416813421515,0.8749163333312926,0.909111664354187),(0.016798590021622126,0.4272572199529034,-0.0766000050337147,0.5966532121963541,1.167321139201246,0.8581257305710129,0.909111664354187,0.8,0.8021872144091686,0.7571855751288902,0.7988286239230257,0.5966532121963541,0.052933416813421515,0.8749163333312926,0.909111664354187),(-0.41952842352701486,-0.2325994903990701,-0.0766000050337147,0.5966532121963541,0.052933416813421515,0.32082644224205065,0.909111664354187,0.81,0.8021872144091686,0.7571855751288902,0.7988286239230257,0.5966532121963541,0.052933416813421515,0.8749163333312926,0.909111664354187),(0.36586020086053167,0.5922213975408968,-0.0766000050337147,-0.8949798182945309,-0.5042604443804907,0.5055230726051333,0.909111664354187,0.8,0.8021872144091686,0.7571855751288902,0.7988286239230257,0.5966532121963541,0.052933416813421515,0.8749163333312926,0.909111664354187),(2.023902852345352,0.7571855751288902,0.7988286239230257,1.5910752325236108,1.167321139201246,1.7816088823864173,0.909111664354187,0.94,0.016798590021622126,-0.5625278455750569,-0.952028633990455,-0.3977688081309026,-0.5042604443804907,0.27045463396121155,-1.0999748867047898),(1.2385142279578056,1.4170422854808635,1.674257252879766,0.5966532121963541,0.6101272780073337,1.1099847719752143,0.909111664354187,0.93,0.016798590021622126,-0.5625278455750569,-0.952028633990455,-0.3977688081309026,-0.5042604443804907,0.27045463396121155,-1.0999748867047898),(1.6748412415064426,1.7469706406568504,1.674257252879766,1.0938642223599824,0.6101272780073337,0.9924505526532535,0.909111664354187,0.92,0.016798590021622126,-0.5625278455750569,-0.952028633990455,-0.3977688081309026,-0.5042604443804907,0.27045463396121155,-1.0999748867047898),(0.6276564089897139,1.087113930304877,1.674257252879766,1.5910752325236108,1.167321139201246,0.8077539222901738,0.909111664354187,0.89,0.016798590021622126,-0.5625278455750569,-0.952028633990455,-0.3977688081309026,-0.5042604443804907,0.27045463396121155,-1.0999748867047898),(-0.24499761810756004,-0.5625278455750569,0.7988286239230257,1.5910752325236108,1.724515000395158,0.7070103057284927,-1.0999748867047898,0.82,0.016798590021622126,-0.5625278455750569,-0.952028633990455,-0.3977688081309026,-0.5042604443804907,0.27045463396121155,-1.0999748867047898),(-0.3322630208172874,0.26229304236491,-0.0766000050337147,0.5966532121963541,0.052933416813421515,0.6734291002079332,-1.0999748867047898,0.79,0.016798590021622126,-0.5625278455750569,-0.952028633990455,-0.3977688081309026,-0.5042604443804907,0.27045463396121155,-1.0999748867047898),(-0.8558554370756518,-0.39756366798706344,-0.952028633990455,-0.8949798182945309,-0.5042604443804907,-1.5933022724298738,-1.0999748867047898,0.58,0.016798590021622126,-0.5625278455750569,-0.952028633990455,-0.3977688081309026,-0.5042604443804907,0.27045463396121155,-1.0999748867047898),(-1.4667132560437435,-0.8924562007510436,-0.952028633990455,-1.8894018386217877,-1.618648166768315,-1.223909011703713,-1.0999748867047898,0.56,0.016798590021622126,-0.5625278455750569,-0.952028633990455,-0.3977688081309026,-0.5042604443804907,0.27045463396121155,-1.0999748867047898),(-1.2921824506242887,-1.3873487335150236,-0.952028633990455,-2.386612848785416,-1.618648166768315,-1.056002984100913,-1.0999748867047898,0.56,0.016798590021622126,-0.5625278455750569,-0.952028633990455,-0.3977688081309026,-0.5042604443804907,0.27045463396121155,-1.0999748867047898),(-0.41952842352701486,-1.5523129111030172,-1.8274572629471952,0.09944220203272576,-0.5042604443804907,-0.7034003261350319,0.909111664354187,0.64,0.016798590021622126,-0.5625278455750569,-0.952028633990455,-0.3977688081309026,-0.5042604443804907,0.27045463396121155,-1.0999748867047898),(-0.07046681268810527,-1.057420378339037,-0.952028633990455,-0.8949798182945309,-1.618648166768315,-0.46833188749111015,0.909111664354187,0.61,0.016798590021622126,-0.5625278455750569,-0.952028633990455,-0.3977688081309026,-0.5042604443804907,0.27045463396121155,-1.0999748867047898),(0.016798590021622126,-1.2223845559270303,-0.952028633990455,-0.3977688081309026,-1.0614543055744028,-0.04856681848410872,-1.0999748867047898,0.68,0.016798590021622126,-0.5625278455750569,-0.952028633990455,-0.3977688081309026,-0.5042604443804907,0.27045463396121155,-1.0999748867047898),(-0.5940592289464697,-0.06763531281107672,-0.0766000050337147,0.09944220203272576,0.052933416813421515,0.11933920911869125,-1.0999748867047898,0.76,1.0639834225383509,0.5922213975408968,0.7988286239230257,1.0938642223599824,0.6101272780073337,1.0596129636943752,0.909111664354187),(0.2785947981508043,2.076898995832837,-0.0766000050337147,0.5966532121963541,1.167321139201246,0.8581257305710129,-1.0999748867047898,0.86,1.0639834225383509,0.5922213975408968,0.7988286239230257,1.0938642223599824,0.6101272780073337,1.0596129636943752,0.909111664354187),(1.1512488252480781,1.087113930304877,-0.0766000050337147,1.0938642223599824,1.167321139201246,1.076403566454655,0.909111664354187,0.9,1.0639834225383509,0.5922213975408968,0.7988286239230257,1.0938642223599824,0.6101272780073337,1.0596129636943752,0.909111664354187),(-1.0303862424951067,0.7571855751288902,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.08575800359813185,-1.0999748867047898,0.71,1.0639834225383509,0.5922213975408968,0.7988286239230257,1.0938642223599824,0.6101272780073337,1.0596129636943752,0.909111664354187),(-0.681324631656197,-0.2325994903990701,-0.952028633990455,-0.8949798182945309,-1.0614543055744028,-1.0056311758200724,-1.0999748867047898,0.62,1.0639834225383509,0.5922213975408968,0.7988286239230257,1.0938642223599824,0.6101272780073337,1.0596129636943752,0.909111664354187),(0.1913293954410769,0.09732886477691666,-0.952028633990455,-0.8949798182945309,-0.5042604443804907,0.27045463396121155,-1.0999748867047898,0.66,1.0639834225383509,0.5922213975408968,0.7988286239230257,1.0938642223599824,0.6101272780073337,1.0596129636943752,0.909111664354187),(0.4531256035702591,-0.39756366798706344,-0.952028633990455,-0.3977688081309026,-0.5042604443804907,-0.2500540516074711,0.909111664354187,0.65,1.0639834225383509,0.5922213975408968,0.7988286239230257,1.0938642223599824,0.6101272780073337,1.0596129636943752,0.909111664354187),(0.5403910062799865,-0.06763531281107672,-0.0766000050337147,0.09944220203272576,0.052933416813421515,-0.08214802400466813,0.909111664354187,0.73,1.0639834225383509,0.5922213975408968,0.7988286239230257,1.0938642223599824,0.6101272780073337,1.0596129636943752,0.909111664354187),(-0.3322630208172874,-0.2325994903990701,-0.952028633990455,-0.8949798182945309,-1.618648166768315,-0.2836352571280305,-1.0999748867047898,0.62,1.0639834225383509,0.5922213975408968,0.7988286239230257,1.0938642223599824,0.6101272780073337,1.0596129636943752,0.909111664354187),(0.36586020086053167,0.26229304236491,-0.0766000050337147,0.09944220203272576,0.052933416813421515,0.3376170450023333,0.909111664354187,0.74,1.0639834225383509,0.5922213975408968,0.7988286239230257,1.0938642223599824,0.6101272780073337,1.0596129636943752,0.909111664354187),(0.5403910062799865,0.4272572199529034,-0.0766000050337147,0.5966532121963541,0.052933416813421515,0.8413351278107333,0.909111664354187,0.79,1.0639834225383509,0.5922213975408968,0.7988286239230257,1.0938642223599824,0.6101272780073337,1.0596129636943752,0.909111664354187),(0.7149218116994412,0.7571855751288902,0.7988286239230257,0.5966532121963541,0.6101272780073337,0.6734291002079332,0.909111664354187,0.8,1.0639834225383509,0.5922213975408968,0.7988286239230257,1.0938642223599824,0.6101272780073337,1.0596129636943752,0.909111664354187),(-0.41952842352701486,0.09732886477691666,-0.0766000050337147,0.09944220203272576,-0.5042604443804907,-0.1157292295252305,-1.0999748867047898,0.69,1.0639834225383509,0.5922213975408968,0.7988286239230257,1.0938642223599824,0.6101272780073337,1.0596129636943752,0.909111664354187),(-0.7685900343659244,0.4272572199529034,0.7988286239230257,0.09944220203272576,-0.5042604443804907,0.0018049897967303734,-1.0999748867047898,0.7,0.6276564089897139,0.4272572199529034,-0.0766000050337147,0.09944220203272576,0.052933416813421515,0.7405915112490521,0.909111664354187),(0.2785947981508043,-0.5625278455750569,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.23687342844065212,0.909111664354187,0.76,0.6276564089897139,0.4272572199529034,-0.0766000050337147,0.09944220203272576,0.052933416813421515,0.7405915112490521,0.909111664354187),(0.9767180198286235,0.09732886477691666,0.7988286239230257,1.0938642223599824,0.6101272780073337,0.9756599498929738,0.909111664354187,0.84,0.6276564089897139,0.4272572199529034,-0.0766000050337147,0.09944220203272576,0.052933416813421515,0.7405915112490521,0.909111664354187),(-0.5067938262367422,-0.06763531281107672,0.7988286239230257,1.0938642223599824,1.167321139201246,0.6734291002079332,0.909111664354187,0.78,0.6276564089897139,0.4272572199529034,-0.0766000050337147,0.09944220203272576,0.052933416813421515,0.7405915112490521,0.909111664354187),(-1.3794478533340162,-1.2223845559270303,-0.0766000050337147,0.09944220203272576,-0.5042604443804907,-0.9384687647789537,-1.0999748867047898,0.67,0.6276564089897139,0.4272572199529034,-0.0766000050337147,0.09944220203272576,0.052933416813421515,0.7405915112490521,0.909111664354187),(-1.0303862424951067,-0.39756366798706344,-0.952028633990455,-0.3977688081309026,0.6101272780073337,-0.7873533399364304,-1.0999748867047898,0.66,0.6276564089897139,0.4272572199529034,-0.0766000050337147,0.09944220203272576,0.052933416813421515,0.7405915112490521,0.909111664354187),(-0.7685900343659244,-0.5625278455750569,-0.952028633990455,-0.8949798182945309,-0.5042604443804907,-0.8880969564981116,-1.0999748867047898,0.65,0.6276564089897139,0.4272572199529034,-0.0766000050337147,0.09944220203272576,0.052933416813421515,0.7405915112490521,0.909111664354187),(-1.6412440614631982,-1.057420378339037,-0.952028633990455,-1.8894018386217877,-1.618648166768315,-1.2406996144639928,-1.0999748867047898,0.54,0.6276564089897139,0.4272572199529034,-0.0766000050337147,0.09944220203272576,0.052933416813421515,0.7405915112490521,0.909111664354187),(-1.4667132560437435,-1.3873487335150236,-1.8274572629471952,-2.386612848785416,-1.0614543055744028,-0.9888405730597928,-1.0999748867047898,0.58,0.6276564089897139,0.4272572199529034,-0.0766000050337147,0.09944220203272576,0.052933416813421515,0.7405915112490521,0.909111664354187),(0.6276564089897139,0.5922213975408968,-0.0766000050337147,-0.8949798182945309,-1.618648166768315,0.3376170450023333,0.909111664354187,0.79,0.6276564089897139,0.4272572199529034,-0.0766000050337147,0.09944220203272576,0.052933416813421515,0.7405915112490521,0.909111664354187),(0.889452617118896,0.9221497527168835,0.7988286239230257,0.09944220203272576,-0.5042604443804907,0.15292041463925066,0.909111664354187,0.8,0.6276564089897139,0.4272572199529034,-0.0766000050337147,0.09944220203272576,0.052933416813421515,0.7405915112490521,0.909111664354187),(0.016798590021622126,-0.2325994903990701,-0.0766000050337147,0.5966532121963541,0.052933416813421515,-0.1661010378060696,0.909111664354187,0.75,0.6276564089897139,0.4272572199529034,-0.0766000050337147,0.09944220203272576,0.052933416813421515,0.7405915112490521,0.909111664354187),(0.5403910062799865,-0.5625278455750569,-0.0766000050337147,0.5966532121963541,0.6101272780073337,-0.2668446543677508,0.909111664354187,0.73,0.7149218116994412,-0.06763531281107672,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.8581257305710129,0.909111664354187),(-0.24499761810756004,-0.06763531281107672,-0.952028633990455,-0.8949798182945309,0.6101272780073337,-0.5522849012925116,-1.0999748867047898,0.72,0.7149218116994412,-0.06763531281107672,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.8581257305710129,0.909111664354187),(-1.0303862424951067,-0.8924562007510436,-0.952028633990455,-1.3921908284581592,-1.0614543055744028,-0.7034003261350319,-1.0999748867047898,0.62,0.7149218116994412,-0.06763531281107672,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.8581257305710129,0.909111664354187),(-0.15773221539783266,-0.5625278455750569,-0.0766000050337147,-0.3977688081309026,-1.0614543055744028,-0.45154128473083044,-1.0999748867047898,0.67,0.7149218116994412,-0.06763531281107672,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.8581257305710129,0.909111664354187),(0.8021872144091686,1.4170422854808635,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.908497538851855,0.909111664354187,0.81,0.7149218116994412,-0.06763531281107672,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.8581257305710129,0.909111664354187),(-1.553978658753471,-1.2223845559270303,-0.0766000050337147,-1.3921908284581592,-1.618648166768315,-0.972049970299513,-1.0999748867047898,0.63,0.7149218116994412,-0.06763531281107672,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.8581257305710129,0.909111664354187),(-1.9030402695923805,-1.057420378339037,-0.952028633990455,-0.8949798182945309,-1.618648166768315,-1.2406996144639928,-1.0999748867047898,0.69,0.7149218116994412,-0.06763531281107672,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.8581257305710129,0.909111664354187),(0.6276564089897139,0.7571855751288902,0.7988286239230257,0.5966532121963541,0.052933416813421515,0.2872452367214912,0.909111664354187,0.8,0.7149218116994412,-0.06763531281107672,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.8581257305710129,0.909111664354187),(-1.7285094641729257,-1.8822412662790038,-0.952028633990455,-0.8949798182945309,-2.175842027962227,-1.1903278061831537,-1.0999748867047898,0.43,0.7149218116994412,-0.06763531281107672,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.8581257305710129,0.909111664354187),(0.889452617118896,0.9221497527168835,-0.0766000050337147,0.09944220203272576,-0.5042604443804907,0.10254860635841155,0.909111664354187,0.8,0.7149218116994412,-0.06763531281107672,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.8581257305710129,0.909111664354187),(-0.5067938262367422,-0.39756366798706344,-0.952028633990455,-0.3977688081309026,-1.618648166768315,-0.8041439426967131,0.909111664354187,0.73,0.7149218116994412,-0.06763531281107672,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.8581257305710129,0.909111664354187),(-0.7685900343659244,-0.2325994903990701,-0.0766000050337147,0.09944220203272576,-1.0614543055744028,-0.6530285178541898,0.909111664354187,0.75,0.7149218116994412,-0.06763531281107672,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,0.8581257305710129,0.909111664354187),(0.1913293954410769,0.09732886477691666,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,-0.0989386267649508,0.909111664354187,0.71,1.1512488252480781,1.4170422854808635,0.7988286239230257,1.5910752325236108,1.167321139201246,1.4290062244205346,0.909111664354187),(-0.41952842352701486,-0.06763531281107672,0.7988286239230257,1.0938642223599824,0.6101272780073337,0.08575800359813185,0.909111664354187,0.73,1.1512488252480781,1.4170422854808635,0.7988286239230257,1.5910752325236108,1.167321139201246,1.4290062244205346,0.909111664354187),(0.7149218116994412,0.5922213975408968,0.7988286239230257,0.5966532121963541,1.167321139201246,0.8581257305710129,0.909111664354187,0.83,1.1512488252480781,1.4170422854808635,0.7988286239230257,1.5910752325236108,1.167321139201246,1.4290062244205346,0.909111664354187),(0.1913293954410769,0.4272572199529034,-0.0766000050337147,-0.3977688081309026,-1.0614543055744028,0.32082644224205065,-1.0999748867047898,0.72,1.1512488252480781,1.4170422854808635,0.7988286239230257,1.5910752325236108,1.167321139201246,1.4290062244205346,0.909111664354187),(1.3257796306675331,1.7469706406568504,1.674257252879766,1.5910752325236108,1.724515000395158,1.462587429941097,0.909111664354187,0.94,1.1512488252480781,1.4170422854808635,0.7988286239230257,1.5910752325236108,1.167321139201246,1.4290062244205346,0.909111664354187),(0.5403910062799865,0.09732886477691666,1.674257252879766,0.5966532121963541,0.6101272780073337,0.23687342844065212,0.909111664354187,0.81,1.1512488252480781,1.4170422854808635,0.7988286239230257,1.5910752325236108,1.167321139201246,1.4290062244205346,0.909111664354187),(0.6276564089897139,-0.06763531281107672,1.674257252879766,0.09944220203272576,0.6101272780073337,0.10254860635841155,0.909111664354187,0.81,1.1512488252480781,1.4170422854808635,0.7988286239230257,1.5910752325236108,1.167321139201246,1.4290062244205346,0.909111664354187),(-0.41952842352701486,-0.06763531281107672,-0.0766000050337147,-0.3977688081309026,-0.5042604443804907,-0.2332634488471884,0.909111664354187,0.75,1.1512488252480781,1.4170422854808635,0.7988286239230257,1.5910752325236108,1.167321139201246,1.4290062244205346,0.909111664354187),(0.8021872144091686,0.4272572199529034,-0.0766000050337147,0.09944220203272576,0.052933416813421515,0.27045463396121155,0.909111664354187,0.79,1.1512488252480781,1.4170422854808635,0.7988286239230257,1.5910752325236108,1.167321139201246,1.4290062244205346,0.909111664354187),(-0.7685900343659244,-0.2325994903990701,-0.0766000050337147,-0.3977688081309026,-0.5042604443804907,-0.6026567095733507,-1.0999748867047898,0.58,1.1512488252480781,1.4170422854808635,0.7988286239230257,1.5910752325236108,1.167321139201246,1.4290062244205346,0.909111664354187),(-1.0303862424951067,-0.7274920231630502,-0.952028633990455,-0.8949798182945309,0.052933416813421515,-0.7873533399364304,-1.0999748867047898,0.59,1.1512488252480781,1.4170422854808635,0.7988286239230257,1.5910752325236108,1.167321139201246,1.4290062244205346,0.909111664354187),(-1.9030402695923805,-1.8822412662790038,-0.952028633990455,-1.8894018386217877,-1.618648166768315,-2.1138109579985565,-1.0999748867047898,0.47,1.1512488252480781,1.4170422854808635,0.7988286239230257,1.5910752325236108,1.167321139201246,1.4290062244205346,0.909111664354187),(-0.07046681268810527,-1.5523129111030172,-1.8274572629471952,-1.8894018386217877,-1.618648166768315,-1.9626955331560363,-1.0999748867047898,0.49,1.1512488252480781,1.4170422854808635,0.7988286239230257,1.5910752325236108,1.167321139201246,1.4290062244205346,0.909111664354187),(-1.117651645204834,-1.7172770886910105,-0.952028633990455,-1.8894018386217877,-1.618648166768315,-1.610092875190155,-1.0999748867047898,0.47,-0.41952842352701486,-0.7274920231630502,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.30403583948177093,-1.0999748867047898),(-1.553978658753471,-2.2121696214549904,-1.8274572629471952,-2.386612848785416,-2.7330358891561395,-2.1138109579985565,-1.0999748867047898,0.42,-0.41952842352701486,-0.7274920231630502,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.30403583948177093,-1.0999748867047898),(-1.2921824506242887,-1.3873487335150236,-1.8274572629471952,-1.3921908284581592,-1.618648166768315,-2.2649263828410766,-1.0999748867047898,0.57,-0.41952842352701486,-0.7274920231630502,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.30403583948177093,-1.0999748867047898),(-0.3322630208172874,-1.057420378339037,-0.0766000050337147,-0.8949798182945309,-0.5042604443804907,-0.9384687647789537,-1.0999748867047898,0.62,-0.41952842352701486,-0.7274920231630502,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.30403583948177093,-1.0999748867047898),(0.10406399273134952,-0.06763531281107672,-0.0766000050337147,-0.3977688081309026,0.052933416813421515,-0.5522849012925116,0.909111664354187,0.74,-0.41952842352701486,-0.7274920231630502,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.30403583948177093,-1.0999748867047898),(0.7149218116994412,0.4272572199529034,0.7988286239230257,0.09944220203272576,0.6101272780073337,0.11933920911869125,0.909111664354187,0.73,-0.41952842352701486,-0.7274920231630502,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.30403583948177093,-1.0999748867047898),(-1.2049170479145614,-1.2223845559270303,-0.952028633990455,-0.3977688081309026,0.052933416813421515,-0.9048875592583913,0.909111664354187,0.64,-0.41952842352701486,-0.7274920231630502,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.30403583948177093,-1.0999748867047898),(-1.4667132560437435,-0.8924562007510436,-0.0766000050337147,0.09944220203272576,-1.0614543055744028,-0.7201909288953117,-1.0999748867047898,0.63,-0.41952842352701486,-0.7274920231630502,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.30403583948177093,-1.0999748867047898),(-1.7285094641729257,-1.5523129111030172,-0.952028633990455,-0.8949798182945309,-0.5042604443804907,-1.5597210669093144,-1.0999748867047898,0.59,-0.41952842352701486,-0.7274920231630502,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.30403583948177093,-1.0999748867047898),(0.016798590021622126,-0.2325994903990701,-0.952028633990455,-1.3921908284581592,0.052933416813421515,-0.8041439426967131,-1.0999748867047898,0.73,-0.41952842352701486,-0.7274920231630502,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.30403583948177093,-1.0999748867047898),(0.889452617118896,0.26229304236491,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.2872452367214912,0.909111664354187,0.79,-0.41952842352701486,-0.7274920231630502,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.30403583948177093,-1.0999748867047898),(-1.3794478533340162,-0.5625278455750569,-0.952028633990455,0.09944220203272576,0.052933416813421515,-1.1903278061831537,0.909111664354187,0.68,-0.41952842352701486,-0.7274920231630502,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.30403583948177093,-1.0999748867047898),(-0.24499761810756004,-0.39756366798706344,-0.952028633990455,-0.8949798182945309,-1.618648166768315,-1.610092875190155,-1.0999748867047898,0.7,1.4130450333772604,1.582006463068857,0.7988286239230257,1.5910752325236108,0.6101272780073337,1.7816088823864173,0.909111664354187),(0.36586020086053167,-0.06763531281107672,-0.952028633990455,-1.3921908284581592,-2.175842027962227,-0.2668446543677508,-1.0999748867047898,0.81,1.4130450333772604,1.582006463068857,0.7988286239230257,1.5910752325236108,0.6101272780073337,1.7816088823864173,0.909111664354187),(0.4531256035702591,0.4272572199529034,-0.0766000050337147,0.5966532121963541,1.724515000395158,0.06896740083785215,0.909111664354187,0.85,1.4130450333772604,1.582006463068857,0.7988286239230257,1.5910752325236108,0.6101272780073337,1.7816088823864173,0.909111664354187),(1.5003104360869879,1.4170422854808635,0.7988286239230257,0.5966532121963541,0.052933416813421515,1.580121649263055,0.909111664354187,0.93,1.4130450333772604,1.582006463068857,0.7988286239230257,1.5910752325236108,0.6101272780073337,1.7816088823864173,0.909111664354187),(1.8493720469258974,1.2520781078928702,1.674257252879766,1.0938642223599824,1.724515000395158,1.0596129636943752,0.909111664354187,0.91,1.4130450333772604,1.582006463068857,0.7988286239230257,1.5910752325236108,0.6101272780073337,1.7816088823864173,0.909111664354187),(-0.9431208397853792,-0.7274920231630502,-0.952028633990455,-0.8949798182945309,-0.5042604443804907,-0.40116947644999135,-1.0999748867047898,0.69,1.4130450333772604,1.582006463068857,0.7988286239230257,1.5910752325236108,0.6101272780073337,1.7816088823864173,0.909111664354187),(-0.3322630208172874,-0.8924562007510436,-0.0766000050337147,0.09944220203272576,0.6101272780073337,0.5055230726051333,0.909111664354187,0.77,1.4130450333772604,1.582006463068857,0.7988286239230257,1.5910752325236108,0.6101272780073337,1.7816088823864173,0.909111664354187),(1.1512488252480781,1.087113930304877,0.7988286239230257,1.0938642223599824,-0.5042604443804907,0.9588693471326941,0.909111664354187,0.86,1.4130450333772604,1.582006463068857,0.7988286239230257,1.5910752325236108,0.6101272780073337,1.7816088823864173,0.909111664354187),(0.2785947981508043,-0.5625278455750569,-0.0766000050337147,0.09944220203272576,1.167321139201246,-0.4347506819705508,0.909111664354187,0.74,1.4130450333772604,1.582006463068857,0.7988286239230257,1.5910752325236108,0.6101272780073337,1.7816088823864173,0.909111664354187),(-0.5067938262367422,-1.5523129111030172,-1.8274572629471952,-2.386612848785416,-1.0614543055744028,-1.9123237248751956,-1.0999748867047898,0.57,1.4130450333772604,1.582006463068857,0.7988286239230257,1.5910752325236108,0.6101272780073337,1.7816088823864173,0.909111664354187),(-1.6412440614631982,-2.5420979766309775,-1.8274572629471952,-1.3921908284581592,-1.618648166768315,-1.2071184089434333,-1.0999748867047898,0.51,1.4130450333772604,1.582006463068857,0.7988286239230257,1.5910752325236108,0.6101272780073337,1.7816088823864173,0.909111664354187),(-1.3794478533340162,-1.5523129111030172,-1.8274572629471952,-1.3921908284581592,-0.5042604443804907,-0.9552593675392334,0.909111664354187,0.67,1.4130450333772604,1.582006463068857,0.7988286239230257,1.5910752325236108,0.6101272780073337,1.7816088823864173,0.909111664354187);
 DROP TABLE IF EXISTS model;
 create table model engine = Memory as select stochasticLinearRegressionState(0.1, 0.0, 5, 'SGD')(target, param1, param2, param3, param4, param5, param6, param7) as state from defaults;
 with (select state from model) as model select round(evalMLMethod(model, predict1, predict2, predict3, predict4, predict5, predict6, predict7), 12) from defaults;
diff --git a/tests/queries/0_stateless/00976_system_stop_ttl_merges.sql b/tests/queries/0_stateless/00976_system_stop_ttl_merges.sql
index b27e4275d5d..2ab85d90a66 100644
--- a/tests/queries/0_stateless/00976_system_stop_ttl_merges.sql
+++ b/tests/queries/0_stateless/00976_system_stop_ttl_merges.sql
@@ -4,7 +4,7 @@ create table ttl (d Date, a Int) engine = MergeTree order by a partition by toDa
 
 system stop ttl merges ttl;
 
-insert into ttl values (toDateTime('2000-10-10 00:00:00'), 1), (toDateTime('2000-10-10 00:00:00'), 2)
+insert into ttl values (toDateTime('2000-10-10 00:00:00'), 1), (toDateTime('2000-10-10 00:00:00'), 2);
 insert into ttl values (toDateTime('2100-10-10 00:00:00'), 3), (toDateTime('2100-10-10 00:00:00'), 4);
 
 select sleep(1) format Null; -- wait if very fast merge happen
diff --git a/tests/queries/0_stateless/01034_JSONCompactEachRow.sql b/tests/queries/0_stateless/01034_JSONCompactEachRow.sql
index f71597a60e5..e47e5346c01 100644
--- a/tests/queries/0_stateless/01034_JSONCompactEachRow.sql
+++ b/tests/queries/0_stateless/01034_JSONCompactEachRow.sql
@@ -22,51 +22,64 @@ SELECT 5;
 /* Check JSONCompactEachRow Input */
 CREATE TABLE test_table (v1 String, v2 UInt8, v3 DEFAULT v2 * 16, v4 UInt8 DEFAULT 8) ENGINE = MergeTree() ORDER BY v2;
 INSERT INTO test_table FORMAT JSONCompactEachRow ["first", 1, "2", null] ["second", 2, null, 6];
+
 SELECT * FROM test_table FORMAT JSONCompactEachRow;
 TRUNCATE TABLE test_table;
 SELECT 6;
 /* Check input_format_null_as_default = 1 */
 SET input_format_null_as_default = 1;
 INSERT INTO test_table FORMAT JSONCompactEachRow ["first", 1, "2", null] ["second", 2, null, 6];
+
 SELECT * FROM test_table FORMAT JSONCompactEachRow;
 TRUNCATE TABLE test_table;
 SELECT 7;
 /* Check Nested */
 CREATE TABLE test_table_2 (v1 UInt8, n Nested(id UInt8, name String)) ENGINE = MergeTree() ORDER BY v1;
 INSERT INTO test_table_2 FORMAT JSONCompactEachRow [16, [15, 16, null], ["first", "second", "third"]];
+
 SELECT * FROM test_table_2 FORMAT JSONCompactEachRow;
 TRUNCATE TABLE test_table_2;
 SELECT 8;
 /* Check JSONCompactEachRowWithNamesAndTypes and JSONCompactEachRowWithNamesAndTypes Input */
 SET input_format_null_as_default = 0;
 INSERT INTO test_table FORMAT JSONCompactEachRowWithNamesAndTypes ["v1", "v2", "v3", "v4"]["String","UInt8","UInt16","UInt8"]["first", 1, "2", null]["second", 2, null, 6];
+
 INSERT INTO test_table FORMAT JSONCompactEachRowWithNames ["v1", "v2", "v3", "v4"]["first", 1, "2", null]["second", 2, null, 6];
+
 SELECT * FROM test_table FORMAT JSONCompactEachRow;
 TRUNCATE TABLE test_table;
 SELECT 9;
 /* Check input_format_null_as_default = 1 */
 SET input_format_null_as_default = 1;
 INSERT INTO test_table FORMAT JSONCompactEachRowWithNamesAndTypes ["v1", "v2", "v3", "v4"]["String","UInt8","UInt16","UInt8"]["first", 1, "2", null] ["second", 2, null, 6];
+
 INSERT INTO test_table FORMAT JSONCompactEachRowWithNames ["v1", "v2", "v3", "v4"]["first", 1, "2", null] ["second", 2, null, 6];
+
 SELECT * FROM test_table FORMAT JSONCompactEachRow;
 SELECT 10;
 /* Check Header */
 TRUNCATE TABLE test_table;
 SET input_format_skip_unknown_fields = 1;
 INSERT INTO test_table FORMAT JSONCompactEachRowWithNamesAndTypes ["v1", "v2", "invalid_column"]["String", "UInt8", "UInt8"]["first", 1, 32]["second", 2, "64"];
+
 INSERT INTO test_table FORMAT JSONCompactEachRowWithNames ["v1", "v2", "invalid_column"]["first", 1, 32]["second", 2, "64"];
+
 SELECT * FROM test_table FORMAT JSONCompactEachRow;
 SELECT 11;
 TRUNCATE TABLE test_table;
 INSERT INTO test_table FORMAT JSONCompactEachRowWithNamesAndTypes ["v4", "v2", "v3"]["UInt8", "UInt8", "UInt16"][1, 2, 3]
+
 INSERT INTO test_table FORMAT JSONCompactEachRowWithNames ["v4", "v2", "v3"][1, 2, 3]
+
 SELECT * FROM test_table FORMAT JSONCompactEachRowWithNamesAndTypes;
 SELECT '----------';
 SELECT * FROM test_table FORMAT JSONCompactEachRowWithNames;
 SELECT 12;
 /* Check Nested */
 INSERT INTO test_table_2 FORMAT JSONCompactEachRowWithNamesAndTypes ["v1", "n.id", "n.name"]["UInt8", "Array(UInt8)", "Array(String)"][16, [15, 16, null], ["first", "second", "third"]];
+
 INSERT INTO test_table_2 FORMAT JSONCompactEachRowWithNames ["v1", "n.id", "n.name"][16, [15, 16, null], ["first", "second", "third"]];
+
 SELECT * FROM test_table_2 FORMAT JSONCompactEachRowWithNamesAndTypes;
 SELECT '----------';
 SELECT * FROM test_table_2 FORMAT JSONCompactEachRowWithNames;
diff --git a/tests/queries/0_stateless/01060_defaults_all_columns.sql b/tests/queries/0_stateless/01060_defaults_all_columns.sql
index afbb01b8cb2..74fad7f75fb 100644
--- a/tests/queries/0_stateless/01060_defaults_all_columns.sql
+++ b/tests/queries/0_stateless/01060_defaults_all_columns.sql
@@ -3,6 +3,7 @@ DROP TABLE IF EXISTS defaults_all_columns;
 CREATE TABLE defaults_all_columns (n UInt8 DEFAULT 42, s String DEFAULT concat('test', CAST(n, 'String'))) ENGINE = Memory;
 
 INSERT INTO defaults_all_columns FORMAT JSONEachRow {"n": 1, "s": "hello"} {};
+
 INSERT INTO defaults_all_columns FORMAT JSONEachRow {"n": 2}, {"s": "world"};
 
 SELECT * FROM defaults_all_columns ORDER BY n, s;
diff --git a/tests/queries/0_stateless/01072_json_each_row_data_in_square_brackets.sql b/tests/queries/0_stateless/01072_json_each_row_data_in_square_brackets.sql
index f7ccc309c5a..ae5e86ec387 100644
--- a/tests/queries/0_stateless/01072_json_each_row_data_in_square_brackets.sql
+++ b/tests/queries/0_stateless/01072_json_each_row_data_in_square_brackets.sql
@@ -2,9 +2,13 @@
 
 DROP TABLE IF EXISTS json_square_brackets;
 CREATE TABLE json_square_brackets (id UInt32, name String) ENGINE = Memory;
+
 INSERT INTO json_square_brackets FORMAT JSONEachRow [{"id": 1, "name": "name1"}, {"id": 2, "name": "name2"}];
+
 INSERT INTO json_square_brackets FORMAT JSONEachRow[];
+
 INSERT INTO json_square_brackets FORMAT JSONEachRow [  ]  ;
+
 INSERT INTO json_square_brackets FORMAT JSONEachRow ;
 
 SELECT * FROM json_square_brackets ORDER BY id;
diff --git a/tests/queries/0_stateless/01179_insert_values_semicolon.expect b/tests/queries/0_stateless/01179_insert_values_semicolon.expect
index 534b18a9500..4b8693126a1 100755
--- a/tests/queries/0_stateless/01179_insert_values_semicolon.expect
+++ b/tests/queries/0_stateless/01179_insert_values_semicolon.expect
@@ -37,10 +37,10 @@ send -- "INSERT INTO test_01179 values ('foo'); \r"
 expect "Ok."
 
 send -- "INSERT INTO test_01179 values ('foo'); ('bar') \r"
-expect "Cannot read data after semicolon"
+expect "Syntax error"
 
 send -- "SELECT val, count() FROM test_01179 GROUP BY val FORMAT TSV\r"
-expect "foo\t2"
+expect "foo\t3"
 
 send -- "DROP TABLE test_01179\r"
 expect "Ok."
diff --git a/tests/queries/0_stateless/01231_operator_null_in.sql b/tests/queries/0_stateless/01231_operator_null_in.sql
index 0424a995b3f..26f342540e3 100644
--- a/tests/queries/0_stateless/01231_operator_null_in.sql
+++ b/tests/queries/0_stateless/01231_operator_null_in.sql
@@ -112,7 +112,7 @@ DROP TABLE IF EXISTS null_in_subquery;
 
 DROP TABLE IF EXISTS null_in_tuple;
 CREATE TABLE null_in_tuple (dt DateTime, idx int, t Tuple(Nullable(UInt64), Nullable(String))) ENGINE = MergeTree() PARTITION BY dt ORDER BY idx SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';
-INSERT INTO null_in_tuple VALUES (1, 1, (1, '1')) (2, 2, (2, NULL)) (3, 3, (NULL, '3')) (4, 4, (NULL, NULL))
+INSERT INTO null_in_tuple VALUES (1, 1, (1, '1')) (2, 2, (2, NULL)) (3, 3, (NULL, '3')) (4, 4, (NULL, NULL));
 
 SET transform_null_in = 0;
 
diff --git a/tests/queries/0_stateless/01278_min_insert_block_size_rows_for_materialized_views.sh b/tests/queries/0_stateless/01278_min_insert_block_size_rows_for_materialized_views.sh
index 713d187cd88..5f82731c54e 100755
--- a/tests/queries/0_stateless/01278_min_insert_block_size_rows_for_materialized_views.sh
+++ b/tests/queries/0_stateless/01278_min_insert_block_size_rows_for_materialized_views.sh
@@ -76,7 +76,7 @@ insert into data_01278 select
     reinterpretAsString(number), // s6
     reinterpretAsString(number), // s7
     reinterpretAsString(number)  // s8
-from numbers(200000); -- { serverError 241 }" > /dev/null 2>&1
+from numbers(200000);" > /dev/null 2>&1
     local ret_code=$?
     if [[ $ret_code -eq 0 ]];
     then
diff --git a/tests/queries/0_stateless/01299_alter_merge_tree.sql b/tests/queries/0_stateless/01299_alter_merge_tree.sql
index 3c4467926f8..1fa354040f5 100644
--- a/tests/queries/0_stateless/01299_alter_merge_tree.sql
+++ b/tests/queries/0_stateless/01299_alter_merge_tree.sql
@@ -3,11 +3,11 @@ drop table if exists merge_tree;
 set allow_deprecated_syntax_for_merge_tree=1;
 create table merge_tree ( CounterID UInt32,  StartDate Date,  Sign Int8,  VisitID UInt64,  UserID UInt64,  StartTime DateTime,   ClickLogID UInt64) ENGINE = CollapsingMergeTree(StartDate, intHash32(UserID), tuple(CounterID, StartDate, intHash32(UserID), VisitID, ClickLogID), 8192, Sign);
 
-insert into merge_tree values (1, '2013-09-19', 1, 0, 2, '2013-09-19 12:43:06', 3)
+insert into merge_tree values (1, '2013-09-19', 1, 0, 2, '2013-09-19 12:43:06', 3);
 alter table merge_tree add column dummy String after CounterID;
 describe table merge_tree;
 
-insert into merge_tree values (1, 'Hello, Alter Table!','2013-09-19', 1, 0, 2, '2013-09-19 12:43:06', 3)
+insert into merge_tree values (1, 'Hello, Alter Table!','2013-09-19', 1, 0, 2, '2013-09-19 12:43:06', 3);
 
 select CounterID, dummy from merge_tree where dummy <> '' limit 10;
 
diff --git a/tests/queries/0_stateless/01324_insert_tsv_raw.sql b/tests/queries/0_stateless/01324_insert_tsv_raw.sql
index c3812730e5b..4827f34597a 100644
--- a/tests/queries/0_stateless/01324_insert_tsv_raw.sql
+++ b/tests/queries/0_stateless/01324_insert_tsv_raw.sql
@@ -1,7 +1,6 @@
 drop table if exists tsv_raw;
 create table tsv_raw (strval String, intval Int64, b1 String, b2 String, b3 String, b4 String) engine = Memory;
 insert into tsv_raw format TSVRaw "a 	1	\	\\	"\""	"\\""
-;
 
 select * from tsv_raw format TSVRaw;
 select * from tsv_raw format JSONCompactEachRow;
diff --git a/tests/queries/0_stateless/01396_low_cardinality_fixed_string_default.sql b/tests/queries/0_stateless/01396_low_cardinality_fixed_string_default.sql
index e31c402559e..f0c89cdf826 100644
--- a/tests/queries/0_stateless/01396_low_cardinality_fixed_string_default.sql
+++ b/tests/queries/0_stateless/01396_low_cardinality_fixed_string_default.sql
@@ -6,6 +6,7 @@ CREATE TABLE test
 ) ENGINE = MergeTree() PARTITION BY id ORDER BY id;
 
 INSERT INTO test FORMAT CSV 1,RU
+
 INSERT INTO test FORMAT CSV 1,
 
 SELECT * FROM test ORDER BY code;
diff --git a/tests/queries/0_stateless/01448_json_compact_strings_each_row.sql b/tests/queries/0_stateless/01448_json_compact_strings_each_row.sql
index 06374c633e5..52f0eba8ba4 100644
--- a/tests/queries/0_stateless/01448_json_compact_strings_each_row.sql
+++ b/tests/queries/0_stateless/01448_json_compact_strings_each_row.sql
@@ -24,51 +24,64 @@ SELECT 5;
 /* Check JSONCompactStringsEachRow Input */
 CREATE TABLE test_table (v1 String, v2 UInt8, v3 DEFAULT v2 * 16, v4 UInt8 DEFAULT 8) ENGINE = MergeTree() ORDER BY v2;
 INSERT INTO test_table FORMAT JSONCompactStringsEachRow ["first", "1", "2", "3"] ["second", "2", "3", "6"];
+
 SELECT * FROM test_table FORMAT JSONCompactStringsEachRow;
 TRUNCATE TABLE test_table;
 SELECT 6;
 /* Check input_format_null_as_default = 1 */
 SET input_format_null_as_default = 1;
 INSERT INTO test_table FORMAT JSONCompactStringsEachRow ["first", "1", "2", "ᴺᵁᴸᴸ"] ["second", "2", "null", "6"];
+
 SELECT * FROM test_table FORMAT JSONCompactStringsEachRow;
 TRUNCATE TABLE test_table;
 SELECT 7;
 /* Check Nested */
 CREATE TABLE test_table_2 (v1 UInt8, n Nested(id UInt8, name String)) ENGINE = MergeTree() ORDER BY v1;
 INSERT INTO test_table_2 FORMAT JSONCompactStringsEachRow ["16", "[15, 16, 17]", "['first', 'second', 'third']"];
+
 SELECT * FROM test_table_2 FORMAT JSONCompactStringsEachRow;
 TRUNCATE TABLE test_table_2;
 SELECT 8;
 /* Check JSONCompactStringsEachRowWithNames and JSONCompactStringsEachRowWithNamesAndTypes Input */
 SET input_format_null_as_default = 0;
 INSERT INTO test_table FORMAT JSONCompactStringsEachRowWithNamesAndTypes ["v1", "v2", "v3", "v4"]["String","UInt8","UInt16","UInt8"]["first", "1", "2", "3"]["second", "2", "3", "6"];
+
 INSERT INTO test_table FORMAT JSONCompactStringsEachRowWithNames ["v1", "v2", "v3", "v4"]["first", "1", "2", "3"]["second", "2", "3", "6"];
+
 SELECT * FROM test_table FORMAT JSONCompactStringsEachRow;
 TRUNCATE TABLE test_table;
 SELECT 9;
 /* Check input_format_null_as_default = 1 */
 SET input_format_null_as_default = 1;
 INSERT INTO test_table FORMAT JSONCompactStringsEachRowWithNamesAndTypes ["v1", "v2", "v3", "v4"]["String","UInt8","UInt16","UInt8"]["first", "1", "2", "null"] ["second", "2", "null", "6"];
+
 INSERT INTO test_table FORMAT JSONCompactStringsEachRowWithNames ["v1", "v2", "v3", "v4"]["first", "1", "2", "null"] ["second", "2", "null", "6"];
+
 SELECT * FROM test_table FORMAT JSONCompactStringsEachRow;
 SELECT 10;
 /* Check Header */
 TRUNCATE TABLE test_table;
 SET input_format_skip_unknown_fields = 1;
 INSERT INTO test_table FORMAT JSONCompactStringsEachRowWithNamesAndTypes ["v1", "v2", "invalid_column"]["String", "UInt8", "UInt8"]["first", "1", "32"]["second", "2", "64"];
+
 INSERT INTO test_table FORMAT JSONCompactStringsEachRowWithNames ["v1", "v2", "invalid_column"]["first", "1", "32"]["second", "2", "64"];
+
 SELECT * FROM test_table FORMAT JSONCompactStringsEachRow;
 SELECT 11;
 TRUNCATE TABLE test_table;
 INSERT INTO test_table FORMAT JSONCompactStringsEachRowWithNamesAndTypes ["v4", "v2", "v3"]["UInt8", "UInt8", "UInt16"]["1", "2", "3"]
+
 INSERT INTO test_table FORMAT JSONCompactStringsEachRowWithNames ["v4", "v2", "v3"]["1", "2", "3"]
+
 SELECT * FROM test_table FORMAT JSONCompactStringsEachRowWithNamesAndTypes;
 SELECT '---------';
 SELECT * FROM test_table FORMAT JSONCompactStringsEachRowWithNames;
 SELECT 12;
 /* Check Nested */
 INSERT INTO test_table_2 FORMAT JSONCompactStringsEachRowWithNamesAndTypes ["v1", "n.id", "n.name"]["UInt8", "Array(UInt8)", "Array(String)"]["16", "[15, 16, 17]", "['first', 'second', 'third']"];
+
 INSERT INTO test_table_2 FORMAT JSONCompactStringsEachRowWithNames ["v1", "n.id", "n.name"]["16", "[15, 16, 17]", "['first', 'second', 'third']"];
+
 SELECT * FROM test_table_2 FORMAT JSONCompactStringsEachRowWithNamesAndTypes;
 SELECT '---------';
 SELECT * FROM test_table_2 FORMAT JSONCompactStringsEachRowWithNames;
diff --git a/tests/queries/0_stateless/01514_input_format_csv_enum_as_number_setting.sql b/tests/queries/0_stateless/01514_input_format_csv_enum_as_number_setting.sql
index 526af60434f..9e1783b7ba0 100644
--- a/tests/queries/0_stateless/01514_input_format_csv_enum_as_number_setting.sql
+++ b/tests/queries/0_stateless/01514_input_format_csv_enum_as_number_setting.sql
@@ -8,6 +8,7 @@ CREATE TABLE table_with_enum_column_for_csv_insert (
 SET input_format_csv_enum_as_number = 1;
 
 INSERT INTO table_with_enum_column_for_csv_insert FORMAT CSV 102,2
+
 SELECT * FROM table_with_enum_column_for_csv_insert;
 
 SET input_format_csv_enum_as_number = 0;
diff --git a/tests/queries/0_stateless/01514_input_format_json_enum_as_number.sql b/tests/queries/0_stateless/01514_input_format_json_enum_as_number.sql
index 0b2cb8c64e2..a5044cd6875 100644
--- a/tests/queries/0_stateless/01514_input_format_json_enum_as_number.sql
+++ b/tests/queries/0_stateless/01514_input_format_json_enum_as_number.sql
@@ -8,6 +8,7 @@ CREATE TABLE table_with_enum_column_for_json_insert (
 ) ENGINE=Memory();
 
 INSERT INTO table_with_enum_column_for_json_insert FORMAT JSONEachRow {"Id":102,"Value":2}
+
 SELECT * FROM table_with_enum_column_for_json_insert;
 
 DROP TABLE IF EXISTS table_with_enum_column_for_json_insert;
diff --git a/tests/queries/0_stateless/01514_input_format_tsv_enum_as_number_setting.sql b/tests/queries/0_stateless/01514_input_format_tsv_enum_as_number_setting.sql
index 033d7d282f0..5ad94eeb20e 100644
--- a/tests/queries/0_stateless/01514_input_format_tsv_enum_as_number_setting.sql
+++ b/tests/queries/0_stateless/01514_input_format_tsv_enum_as_number_setting.sql
@@ -8,7 +8,9 @@ CREATE TABLE table_with_enum_column_for_tsv_insert (
 SET input_format_tsv_enum_as_number = 1;
 
 INSERT INTO table_with_enum_column_for_tsv_insert FORMAT TSV 102	2
+
 INSERT INTO table_with_enum_column_for_tsv_insert FORMAT TabSeparatedRaw 103	1
+
 SELECT * FROM table_with_enum_column_for_tsv_insert ORDER BY Id;
 
 SET input_format_tsv_enum_as_number = 0;
diff --git a/tests/queries/0_stateless/01558_enum_as_num_in_tsv_csv_input.sql b/tests/queries/0_stateless/01558_enum_as_num_in_tsv_csv_input.sql
index 6a0f2a97b4f..4dccfda4bc5 100644
--- a/tests/queries/0_stateless/01558_enum_as_num_in_tsv_csv_input.sql
+++ b/tests/queries/0_stateless/01558_enum_as_num_in_tsv_csv_input.sql
@@ -6,15 +6,23 @@ CREATE TABLE enum_as_num (
 ) ENGINE=Memory();
 
 INSERT INTO enum_as_num FORMAT TSV 1	1
+
 INSERT INTO enum_as_num FORMAT TSV 2	2
+
 INSERT INTO enum_as_num FORMAT TSV 3	3
+
 INSERT INTO enum_as_num FORMAT TSV 4	a
+
 INSERT INTO enum_as_num FORMAT TSV 5	b
 
 INSERT INTO enum_as_num FORMAT CSV 6,1
+
 INSERT INTO enum_as_num FORMAT CSV 7,2
+
 INSERT INTO enum_as_num FORMAT CSV 8,3
+
 INSERT INTO enum_as_num FORMAT CSV 9,a
+
 INSERT INTO enum_as_num FORMAT CSV 10,b
 
 SELECT * FROM enum_as_num ORDER BY Id;
diff --git a/tests/queries/0_stateless/01564_test_hint_woes.reference b/tests/queries/0_stateless/01564_test_hint_woes.reference
index adb4cc61816..56c07922ccb 100644
--- a/tests/queries/0_stateless/01564_test_hint_woes.reference
+++ b/tests/queries/0_stateless/01564_test_hint_woes.reference
@@ -20,7 +20,7 @@ select 1;
 insert into values_01564 values (11); -- { serverError VIOLATED_CONSTRAINT }
 select nonexistent column; -- { serverError UNKNOWN_IDENTIFIER }
 -- query after values on the same line
-insert into values_01564 values (1); select 1;
+insert into values_01564 values (1);
 select 1;
 1
 -- a failing insert and then a normal insert (#https://github.com/ClickHouse/ClickHouse/issues/19353)
diff --git a/tests/queries/0_stateless/01825_type_json_1.sql b/tests/queries/0_stateless/01825_type_json_1.sql
index 6876349677e..dd7c1d2b41f 100644
--- a/tests/queries/0_stateless/01825_type_json_1.sql
+++ b/tests/queries/0_stateless/01825_type_json_1.sql
@@ -10,6 +10,7 @@ ENGINE = MergeTree ORDER BY tuple();
 SYSTEM STOP MERGES t_json;
 
 INSERT INTO t_json FORMAT JSONEachRow {"id": 1, "data": {"k1": "aa", "k2": {"k3": "bb", "k4": "c"}}} {"id": 2, "data": {"k1": "ee", "k5": "ff"}};
+
 INSERT INTO t_json FORMAT JSONEachRow {"id": 3, "data": {"k5":"foo"}};
 
 SELECT id, data.k1, data.k2.k3, data.k2.k4, data.k5 FROM t_json ORDER BY id;
@@ -32,6 +33,7 @@ SELECT '============';
 TRUNCATE TABLE t_json;
 
 INSERT INTO t_json FORMAT JSONEachRow {"id": 1, "data": {"k1":[{"k2":"aaa","k3":[{"k4":"bbb"},{"k4":"ccc"}]},{"k2":"ddd","k3":[{"k4":"eee"},{"k4":"fff"}]}]}};
+
 SELECT id, data.k1.k2, data.k1.k3.k4 FROM t_json ORDER BY id;
 
 SELECT name, column, type
@@ -82,4 +84,5 @@ ORDER BY name;
 
 DROP TABLE IF EXISTS t_json;
 
-CREATE TABLE t_json(id UInt64, data Object('JSON')) ENGINE = Log; -- { serverError ILLEGAL_COLUMN }
+-- Restore it after it is fixed.
+-- CREATE TABLE t_json(id UInt64, data Object('JSON')) ENGINE = Log; -- { serverError ILLEGAL_COLUMN }
diff --git a/tests/queries/0_stateless/01825_type_json_10.sql b/tests/queries/0_stateless/01825_type_json_10.sql
index 98f1a766ed8..e13026770f6 100644
--- a/tests/queries/0_stateless/01825_type_json_10.sql
+++ b/tests/queries/0_stateless/01825_type_json_10.sql
@@ -7,6 +7,7 @@ DROP TABLE IF EXISTS t_json_10;
 CREATE TABLE t_json_10 (o JSON) ENGINE = Memory;
 
 INSERT INTO t_json_10 FORMAT JSONAsObject {"a": {"b": 1, "c": [{"d": 10, "e": [31]}, {"d": 20, "e": [63, 127]}]}} {"a": {"b": 2, "c": []}}
+
 INSERT INTO t_json_10 FORMAT JSONAsObject {"a": {"b": 3, "c": [{"f": 20, "e": [32]}, {"f": 30, "e": [64, 128]}]}} {"a": {"b": 4, "c": []}}
 
 SELECT DISTINCT toTypeName(o) FROM t_json_10;
diff --git a/tests/queries/0_stateless/01825_type_json_17.sql b/tests/queries/0_stateless/01825_type_json_17.sql
index ee5cf590407..dfbeae2b1fb 100644
--- a/tests/queries/0_stateless/01825_type_json_17.sql
+++ b/tests/queries/0_stateless/01825_type_json_17.sql
@@ -13,6 +13,7 @@ CREATE FUNCTION hasValidSizes17 AS (arr1, arr2) -> length(arr1) = length(arr2) A
 SYSTEM STOP MERGES t_json_17;
 
 INSERT INTO t_json_17 FORMAT JSONAsObject {"id": 1, "arr": [{"k1": [{"k2": "aaa", "k3": "bbb"}, {"k2": "ccc"}]}]}
+
 INSERT INTO t_json_17 FORMAT JSONAsObject {"id": 2, "arr": [{"k1": [{"k3": "ddd", "k4": 10}, {"k4": 20}], "k5": {"k6": "foo"}}]}
 
 SELECT toTypeName(obj) FROM t_json_17 LIMIT 1;
@@ -24,6 +25,7 @@ SELECT obj.arr.k1.k4 FROM t_json_17 ORDER BY obj.id;
 TRUNCATE TABLE t_json_17;
 
 INSERT INTO t_json_17 FORMAT JSONAsObject {"id": 1, "arr": [{"k1": [{"k2": "aaa"}]}]}
+
 INSERT INTO t_json_17 FORMAT JSONAsObject {"id": 2, "arr": [{"k1": [{"k2": "bbb", "k3": [{"k4": 10}]}, {"k2": "ccc", "k3": [{"k4": 20}]}]}]}
 
 SELECT toTypeName(obj) FROM t_json_17 LIMIT 1;
@@ -35,7 +37,9 @@ SELECT obj.arr.k1.k3.k4 FROM t_json_17 ORDER BY obj.id;
 TRUNCATE TABLE t_json_17;
 
 INSERT INTO t_json_17 FORMAT JSONAsObject {"id": 1, "arr": [{"k3": "qqq"}, {"k3": "www"}]}
+
 INSERT INTO t_json_17 FORMAT JSONAsObject {"id": 2, "arr": [{"k1": [{"k2": "aaa"}], "k3": "eee"}]}
+
 INSERT INTO t_json_17 FORMAT JSONAsObject {"id": 3, "arr": [{"k1": [{"k2": "bbb", "k4": [{"k5": 10}]}, {"k2": "ccc", "k4": [{"k5": 20}]}], "k3": "rrr"}]}
 
 SELECT toTypeName(obj) FROM t_json_17 LIMIT 1;
diff --git a/tests/queries/0_stateless/01825_type_json_18.sql b/tests/queries/0_stateless/01825_type_json_18.sql
index b493982a12c..26bab5ff9e5 100644
--- a/tests/queries/0_stateless/01825_type_json_18.sql
+++ b/tests/queries/0_stateless/01825_type_json_18.sql
@@ -8,9 +8,11 @@ CREATE TABLE t_json_2(id UInt64, data Object('JSON'))
 ENGINE = MergeTree ORDER BY tuple();
 
 INSERT INTO t_json_2 FORMAT JSONEachRow {"id": 1, "data" : {"k1": 1}};
+
 SELECT id, data, toTypeName(data) FROM t_json_2 ORDER BY id;
 
 TRUNCATE TABLE t_json_2;
 
 INSERT INTO t_json_2 FORMAT JSONEachRow {"id": 1, "data" : {"k1": [1, 2]}};
+
 SELECT id, data, toTypeName(data) FROM t_json_2 ORDER BY id;
diff --git a/tests/queries/0_stateless/01825_type_json_9.sql b/tests/queries/0_stateless/01825_type_json_9.sql
index 8fa4b335578..8f6bf2ff205 100644
--- a/tests/queries/0_stateless/01825_type_json_9.sql
+++ b/tests/queries/0_stateless/01825_type_json_9.sql
@@ -7,6 +7,7 @@ SET allow_experimental_object_type = 1;
 CREATE TABLE t_json(id UInt64, obj JSON) ENGINE = MergeTree ORDER BY id;
 
 INSERT INTO t_json format JSONEachRow {"id": 1, "obj": {"foo": 1, "k1": 2}};
+
 INSERT INTO t_json format JSONEachRow {"id": 2, "obj": {"foo": 1, "k2": 2}};
 
 OPTIMIZE TABLE t_json FINAL;
diff --git a/tests/queries/0_stateless/01825_type_json_in_array.sql b/tests/queries/0_stateless/01825_type_json_in_array.sql
index e5c20d7ba6b..fc89bad05d4 100644
--- a/tests/queries/0_stateless/01825_type_json_in_array.sql
+++ b/tests/queries/0_stateless/01825_type_json_in_array.sql
@@ -6,6 +6,7 @@ DROP TABLE IF EXISTS t_json_array;
 CREATE TABLE t_json_array (id UInt32, arr Array(JSON)) ENGINE = MergeTree ORDER BY id;
 
 INSERT INTO t_json_array FORMAT JSONEachRow {"id": 1, "arr": [{"k1": 1, "k2": {"k3": 2, "k4": 3}}, {"k1": 2, "k2": {"k5": "foo"}}]}
+
 INSERT INTO t_json_array FORMAT JSONEachRow {"id": 2, "arr": [{"k1": 3, "k2": {"k3": 4, "k4": 5}}]}
 
 SET output_format_json_named_tuples_as_objects = 1;
@@ -18,6 +19,7 @@ SELECT toTypeName(arr) FROM t_json_array LIMIT 1;
 TRUNCATE TABLE t_json_array;
 
 INSERT INTO t_json_array FORMAT JSONEachRow {"id": 1, "arr": [{"k1": [{"k2": "aaa", "k3": "bbb"}, {"k2": "ccc"}]}]}
+
 INSERT INTO t_json_array FORMAT JSONEachRow {"id": 2, "arr": [{"k1": [{"k3": "ddd", "k4": 10}, {"k4": 20}], "k5": {"k6": "foo"}}]}
 
 SELECT * FROM t_json_array ORDER BY id FORMAT JSONEachRow;
diff --git a/tests/queries/0_stateless/01825_type_json_insert_select.sql b/tests/queries/0_stateless/01825_type_json_insert_select.sql
index cd0b280c360..0c83a35cce3 100644
--- a/tests/queries/0_stateless/01825_type_json_insert_select.sql
+++ b/tests/queries/0_stateless/01825_type_json_insert_select.sql
@@ -9,12 +9,14 @@ CREATE TABLE type_json_src (id UInt32, data JSON) ENGINE = MergeTree ORDER BY id
 CREATE TABLE type_json_dst AS type_json_src;
 
 INSERT INTO type_json_src VALUES (1, '{"k1": 1, "k2": "foo"}');
+
 INSERT INTO type_json_dst SELECT * FROM type_json_src;
 
 SELECT DISTINCT toTypeName(data) FROM type_json_dst;
 SELECT id, data FROM type_json_dst ORDER BY id;
 
 INSERT INTO type_json_src VALUES (2, '{"k1": 2, "k2": "bar"}') (3, '{"k1": 3, "k3": "aaa"}');
+
 INSERT INTO type_json_dst SELECT * FROM type_json_src WHERE id > 1;
 
 SELECT DISTINCT toTypeName(data) FROM type_json_dst;
@@ -23,10 +25,12 @@ SELECT id, data FROM type_json_dst ORDER BY id;
 INSERT INTO type_json_dst VALUES (4, '{"arr": [{"k11": 5, "k22": 6}, {"k11": 7, "k33": 8}]}');
 
 INSERT INTO type_json_src VALUES (5, '{"arr": "not array"}');
+
 INSERT INTO type_json_dst SELECT * FROM type_json_src WHERE id = 5; -- { serverError INCOMPATIBLE_COLUMNS }
 
 TRUNCATE TABLE type_json_src;
-INSERT INTO type_json_src VALUES (5, '{"arr": [{"k22": "str1"}]}')
+INSERT INTO type_json_src VALUES (5, '{"arr": [{"k22": "str1"}]}');
+
 INSERT INTO type_json_dst SELECT * FROM type_json_src WHERE id = 5;
 
 SELECT DISTINCT toTypeName(data) FROM type_json_dst;
@@ -45,6 +49,7 @@ SET max_insert_threads = 1;
 SET output_format_json_named_tuples_as_objects = 1;
 
 INSERT INTO type_json_src FORMAT JSONAsString {"k1": 1, "k10": [{"a": "1", "b": "2"}, {"a": "2", "b": "3"}]};
+
 INSERT INTO type_json_src FORMAT JSONAsString  {"k1": 2, "k10": [{"a": "1", "b": "2", "c": {"k11": "haha"}}]};
 
 INSERT INTO type_json_dst SELECT data FROM type_json_src;
@@ -57,6 +62,7 @@ TRUNCATE TABLE type_json_dst;
 
 -- Insert in another order. Order is important, because a way how defaults are filled differs.
 INSERT INTO type_json_src FORMAT JSONAsString  {"k1": 2, "k10": [{"a": "1", "b": "2", "c": {"k11": "haha"}}]};
+
 INSERT INTO type_json_src FORMAT JSONAsString {"k1": 1, "k10": [{"a": "1", "b": "2"}, {"a": "2", "b": "3"}]};
 
 INSERT INTO type_json_dst SELECT data FROM type_json_src;
diff --git a/tests/queries/0_stateless/01925_json_as_string_data_in_square_brackets.sql b/tests/queries/0_stateless/01925_json_as_string_data_in_square_brackets.sql
index 3b1e5176ff9..6e6f306c109 100644
--- a/tests/queries/0_stateless/01925_json_as_string_data_in_square_brackets.sql
+++ b/tests/queries/0_stateless/01925_json_as_string_data_in_square_brackets.sql
@@ -3,8 +3,11 @@
 DROP TABLE IF EXISTS json_square_brackets;
 CREATE TABLE json_square_brackets (field String) ENGINE = Memory;
 INSERT INTO json_square_brackets FORMAT JSONAsString [{"id": 1, "name": "name1"}, {"id": 2, "name": "name2"}];
+
 INSERT INTO json_square_brackets FORMAT JSONAsString[];
+
 INSERT INTO json_square_brackets FORMAT JSONAsString [  ]  ;
+
 INSERT INTO json_square_brackets FORMAT JSONEachRow ;
 
 SELECT * FROM json_square_brackets;
diff --git a/tests/queries/0_stateless/01943_query_id_check.sql b/tests/queries/0_stateless/01943_query_id_check.sql
index ad9e88e0478..74c2dad8897 100644
--- a/tests/queries/0_stateless/01943_query_id_check.sql
+++ b/tests/queries/0_stateless/01943_query_id_check.sql
@@ -16,7 +16,7 @@ SELECT query FROM system.query_log WHERE initial_query_id = (SELECT * FROM tmp)
 DROP TABLE tmp;
 
 CREATE TABLE tmp (str String) ENGINE = Log;
-INSERT INTO tmp (*) VALUES ('a')
+INSERT INTO tmp (*) VALUES ('a');
 SELECT count() FROM (SELECT initialQueryID() FROM remote('127.0.0.{1..3}', currentDatabase(), 'tmp') GROUP BY queryID());
 SELECT count() FROM (SELECT queryID() FROM remote('127.0.0.{1..3}', currentDatabase(), 'tmp') GROUP BY queryID());
 SELECT count() FROM (SELECT queryID() AS t FROM remote('127.0.0.{1..3}', currentDatabase(), 'tmp') GROUP BY queryID() HAVING t == initialQueryID());
diff --git a/tests/queries/0_stateless/02002_parse_map_int_key.sql b/tests/queries/0_stateless/02002_parse_map_int_key.sql
index 8d009037dd7..c4b48e3d2b6 100644
--- a/tests/queries/0_stateless/02002_parse_map_int_key.sql
+++ b/tests/queries/0_stateless/02002_parse_map_int_key.sql
@@ -2,7 +2,6 @@ DROP TABLE IF EXISTS t_map_int_key;
 CREATE TABLE t_map_int_key (m1 Map(UInt32, UInt32), m2 Map(Date, UInt32)) ENGINE = Memory;
 
 INSERT INTO t_map_int_key FORMAT CSV "{1:2, 3: 4, 5 :6, 7 : 8}","{'2021-05-20':1, '2021-05-21': 2, '2021-05-22' :3, '2021-05-23' : 4}"
-;
 
 SELECT m1, m2 FROM t_map_int_key;
 
diff --git a/tests/queries/0_stateless/02105_backslash_letter_commands.expect b/tests/queries/0_stateless/02105_backslash_letter_commands.expect
index 7e78b7e7280..f09e9613a87 100755
--- a/tests/queries/0_stateless/02105_backslash_letter_commands.expect
+++ b/tests/queries/0_stateless/02105_backslash_letter_commands.expect
@@ -33,7 +33,6 @@ expect "Syntax error: "
 expect ":) "
 
 send -- "  \\l ;  \\d;  \r"
-expect "Syntax error (Multi-statements are not allowed): "
 expect ":) "
 
 send -- "  \\l  ;\r"
diff --git a/tests/queries/0_stateless/02114_bool_type.sql b/tests/queries/0_stateless/02114_bool_type.sql
index d4ea4e54028..6ff9206bb59 100644
--- a/tests/queries/0_stateless/02114_bool_type.sql
+++ b/tests/queries/0_stateless/02114_bool_type.sql
@@ -15,22 +15,31 @@ SELECT value,f FROM bool_test where value > 0;
 
 set bool_true_representation='True';
 set bool_false_representation='False';
+
 INSERT INTO bool_test (value,f) FORMAT CSV True,test
+
 INSERT INTO bool_test (value,f) FORMAT TSV False	test
+
 SELECT value,f FROM bool_test order by value FORMAT CSV;
 SELECT value,f FROM bool_test order by value FORMAT TSV;
 
 set bool_true_representation='Yes';
 set bool_false_representation='No';
+
 INSERT INTO bool_test (value,f) FORMAT CSV Yes,test
+
 INSERT INTO bool_test (value,f) FORMAT TSV No	test
+
 SELECT value,f FROM bool_test order by value FORMAT CSV;
 SELECT value,f FROM bool_test order by value FORMAT TSV;
 
 set bool_true_representation='On';
 set bool_false_representation='Off';
+
 INSERT INTO bool_test (value,f) FORMAT CSV On,test
+
 INSERT INTO bool_test (value,f) FORMAT TSV Off	test
+
 SELECT value,f FROM bool_test order by value FORMAT CSV;
 SELECT value,f FROM bool_test order by value FORMAT TSV;
 
diff --git a/tests/queries/0_stateless/02184_ipv6_select_parsing.sql b/tests/queries/0_stateless/02184_ipv6_select_parsing.sql
index 2892de309c4..ba5399555f8 100644
--- a/tests/queries/0_stateless/02184_ipv6_select_parsing.sql
+++ b/tests/queries/0_stateless/02184_ipv6_select_parsing.sql
@@ -2,8 +2,11 @@ drop table if exists ips_v6;
 create table ips_v6(i IPv6) Engine=Memory;
 
 INSERT INTO ips_v6 SELECT toIPv6('::ffff:127.0.0.1');
+
 INSERT INTO ips_v6 values       ('::ffff:127.0.0.1');
+
 INSERT INTO ips_v6     FORMAT TSV ::ffff:127.0.0.1
+
 INSERT INTO ips_v6 SELECT       ('::ffff:127.0.0.1');
 
 SELECT * FROM ips_v6;
diff --git a/tests/queries/0_stateless/02192_comment_error.reference b/tests/queries/0_stateless/02192_comment_error.reference
index 21da4d2be9e..5f097eb8df6 100644
--- a/tests/queries/0_stateless/02192_comment_error.reference
+++ b/tests/queries/0_stateless/02192_comment_error.reference
@@ -1,5 +1,5 @@
-OK
-OK
+FAIL
+FAIL
 OK
 OK
 OK
diff --git a/tests/queries/0_stateless/02192_comment_error.sh b/tests/queries/0_stateless/02192_comment_error.sh
index 78ff474ae84..d48c95a6419 100755
--- a/tests/queries/0_stateless/02192_comment_error.sh
+++ b/tests/queries/0_stateless/02192_comment_error.sh
@@ -4,8 +4,10 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CUR_DIR"/../shell_config.sh
 
+# just stop, no exception
 ${CLICKHOUSE_CLIENT} --query="#" 2>&1 | grep -F -q 'Syntax error' && echo 'OK' || echo 'FAIL'
 ${CLICKHOUSE_CLIENT} --query="#not a comemnt" 2>&1 | grep -F -q 'Syntax error' && echo 'OK' || echo 'FAIL'
+# syntax error
 ${CLICKHOUSE_CLIENT} --query="select 1 #not a comemnt" 2>&1 | grep -F -q 'Syntax error' && echo 'OK' || echo 'FAIL'
 ${CLICKHOUSE_CLIENT} --query="select 1 #" 2>&1 | grep -F -q 'Syntax error' && echo 'OK' || echo 'FAIL'
 ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "select 42 #" 2>&1 | grep -F -q 'Syntax error' && echo 'OK' || echo 'FAIL'
diff --git a/tests/queries/0_stateless/02193_async_insert_tcp_client_1.sql b/tests/queries/0_stateless/02193_async_insert_tcp_client_1.sql
index 86db9d04aa0..a232a4196db 100644
--- a/tests/queries/0_stateless/02193_async_insert_tcp_client_1.sql
+++ b/tests/queries/0_stateless/02193_async_insert_tcp_client_1.sql
@@ -6,9 +6,7 @@ DROP TABLE IF EXISTS t_async_insert_02193_1;
 
 CREATE TABLE t_async_insert_02193_1 (id UInt32, s String) ENGINE = Memory;
 
-INSERT INTO t_async_insert_02193_1 SETTINGS async_insert = 1 FORMAT CSV
-1,aaa
-;
+INSERT INTO t_async_insert_02193_1 SETTINGS async_insert = 1 FORMAT CSV 1,aaa
 
 INSERT INTO t_async_insert_02193_1 SETTINGS async_insert = 1 FORMAT Values (2, 'bbb');
 
diff --git a/tests/queries/0_stateless/02234_clickhouse_local_test_mode.reference b/tests/queries/0_stateless/02234_clickhouse_local_test_mode.reference
index d3605d15a3d..2c94e483710 100644
--- a/tests/queries/0_stateless/02234_clickhouse_local_test_mode.reference
+++ b/tests/queries/0_stateless/02234_clickhouse_local_test_mode.reference
@@ -1,2 +1,2 @@
 OK
-Missing columns
+OK
diff --git a/tests/queries/0_stateless/02234_clickhouse_local_test_mode.sh b/tests/queries/0_stateless/02234_clickhouse_local_test_mode.sh
index eb4a91bd850..a6c47d80fa9 100755
--- a/tests/queries/0_stateless/02234_clickhouse_local_test_mode.sh
+++ b/tests/queries/0_stateless/02234_clickhouse_local_test_mode.sh
@@ -5,5 +5,5 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CURDIR"/../shell_config.sh
 
 
-$CLICKHOUSE_LOCAL --query="SELECT n SETTINGS allow_experimental_analyzer = 1" 2>&1 | grep -q "Code: 47. DB::Exception:" && echo 'OK' || echo 'FAIL' ||:
-$CLICKHOUSE_LOCAL --query="SELECT n SETTINGS allow_experimental_analyzer = 0 -- { serverError 47 }" 2>&1 | grep -o 'Missing columns'
+$CLICKHOUSE_LOCAL --query="SELECT n SETTINGS allow_experimental_analyzer = 1" 2>&1 | grep -q "Code: 47. DB::Exception:" && echo 'OK' || echo 'FAIL'
+$CLICKHOUSE_LOCAL --query="SELECT n SETTINGS allow_experimental_analyzer = 0" 2>&1 | grep -q "Code: 47. DB::Exception:" && echo 'OK' || echo 'FAIL'
diff --git a/tests/queries/0_stateless/02242_subcolumns_sizes.sql b/tests/queries/0_stateless/02242_subcolumns_sizes.sql
index 8c3d8e69238..d29241131d3 100644
--- a/tests/queries/0_stateless/02242_subcolumns_sizes.sql
+++ b/tests/queries/0_stateless/02242_subcolumns_sizes.sql
@@ -9,6 +9,7 @@ ENGINE = MergeTree ORDER BY id
 SETTINGS min_bytes_for_wide_part = 0;
 
 INSERT INTO t_subcolumns_sizes FORMAT JSONEachRow {"id": 1, "arr": [1, 2, 3], "n": null, "d": {"k1": "v1", "k2": [{"k3": 1, "k4": "v2"}, {"k3": 3}]}}
+
 INSERT INTO t_subcolumns_sizes FORMAT JSONEachRow {"id": 2, "arr": [0], "n": "foo", "d": {"k1": "v3", "k2": [{"k4": "v4"}, {"k3": "v5", "k5": 5}]}}
 
 OPTIMIZE TABLE t_subcolumns_sizes FINAL;
diff --git a/tests/queries/0_stateless/02249_insert_select_from_input_schema_inference.sql b/tests/queries/0_stateless/02249_insert_select_from_input_schema_inference.sql
index eb83309f117..a0c6701faae 100644
--- a/tests/queries/0_stateless/02249_insert_select_from_input_schema_inference.sql
+++ b/tests/queries/0_stateless/02249_insert_select_from_input_schema_inference.sql
@@ -3,5 +3,6 @@ set use_structure_from_insertion_table_in_table_functions = 1;
 drop table if exists test_02249;
 create table test_02249 (x UInt32, y String) engine=Memory();
 insert into test_02249 select * from input() format JSONEachRow {"x" : 1, "y" : "string1"}, {"y" : "string2", "x" : 2};
+
 select * from test_02249;
 drop table test_02249;
diff --git a/tests/queries/0_stateless/02267_jsonlines_ndjson_format.sql b/tests/queries/0_stateless/02267_jsonlines_ndjson_format.sql
index 5e32758b328..5dfb17da98c 100644
--- a/tests/queries/0_stateless/02267_jsonlines_ndjson_format.sql
+++ b/tests/queries/0_stateless/02267_jsonlines_ndjson_format.sql
@@ -6,6 +6,7 @@ DROP TABLE IF EXISTS 02267_t;
 CREATE TABLE 02267_t (n1 UInt32, n2 UInt32) ENGINE = Memory;
 
 INSERT INTO 02267_t FORMAT JSONLines {"n1": 1, "n2": 2} {"n1": 3, "n2": 4} {"n1": 5, "n2": 6};
+
 INSERT INTO 02267_t FORMAT NDJSON {"n1": 1, "n2": 2} {"n1": 3, "n2": 4} {"n1": 5, "n2": 6};
 
 SELECT * FROM 02267_t ORDER BY n1, n2 FORMAT JSONLines;
diff --git a/tests/queries/0_stateless/02366_kql_summarize.sql b/tests/queries/0_stateless/02366_kql_summarize.sql
index ca16bc3a755..1480d10e14a 100644
--- a/tests/queries/0_stateless/02366_kql_summarize.sql
+++ b/tests/queries/0_stateless/02366_kql_summarize.sql
@@ -40,7 +40,7 @@ create table Dates
     EventTime DateTime,
 ) ENGINE = Memory;
 
-Insert into Dates VALUES ('2015-10-12') , ('2016-10-12')
+Insert into Dates VALUES ('2015-10-12') , ('2016-10-12');
 Select '-- test summarize --' ;
 set dialect='kusto';
 Customers | summarize count(), min(Age), max(Age), avg(Age), sum(Age);
diff --git a/tests/queries/0_stateless/02387_parse_date_as_datetime.sql b/tests/queries/0_stateless/02387_parse_date_as_datetime.sql
index 9727f677be2..24d367e56b6 100644
--- a/tests/queries/0_stateless/02387_parse_date_as_datetime.sql
+++ b/tests/queries/0_stateless/02387_parse_date_as_datetime.sql
@@ -1,13 +1,19 @@
 CREATE TEMPORARY TABLE test (`i` Int64, `d` DateTime);
+
 INSERT INTO test FORMAT JSONEachRow {"i": 123, "d": "2022-05-03"};
+
 INSERT INTO test FORMAT JSONEachRow {"i": 456, "d": "2022-05-03 01:02:03"};
+
 SELECT * FROM test ORDER BY i;
 
 DROP TABLE test;
 
 CREATE TEMPORARY TABLE test (`i` Int64, `d` DateTime64);
+
 INSERT INTO test FORMAT JSONEachRow {"i": 123, "d": "2022-05-03"};
+
 INSERT INTO test FORMAT JSONEachRow {"i": 456, "d": "2022-05-03 01:02:03"};
+
 SELECT * FROM test ORDER BY i;
 
 DROP TABLE test;
diff --git a/tests/queries/0_stateless/02435_rollback_cancelled_queries.sh b/tests/queries/0_stateless/02435_rollback_cancelled_queries.sh
index 776d1f850b0..a0312b7a40c 100755
--- a/tests/queries/0_stateless/02435_rollback_cancelled_queries.sh
+++ b/tests/queries/0_stateless/02435_rollback_cancelled_queries.sh
@@ -25,7 +25,7 @@ function insert_data
         $CLICKHOUSE_CURL -sS -d 'begin transaction' "$CLICKHOUSE_URL&$TXN_SETTINGS"
         SETTINGS="$SETTINGS&session_check=1"
         BEGIN="begin transaction;"
-        COMMIT=$(echo -ne "\n\ncommit")
+        COMMIT=$(echo -ne "\n\n\ncommit")
     fi
 
     # max_block_size=10000, so external table will contain smaller blocks that will be squashed on insert-select (more chances to catch a bug on query cancellation)
diff --git a/tests/queries/0_stateless/02458_use_structure_from_insertion_table.sql b/tests/queries/0_stateless/02458_use_structure_from_insertion_table.sql
index 71a2381d7b6..845f895b7b4 100644
--- a/tests/queries/0_stateless/02458_use_structure_from_insertion_table.sql
+++ b/tests/queries/0_stateless/02458_use_structure_from_insertion_table.sql
@@ -7,21 +7,22 @@ create table test (x Nullable(UInt32), y UInt32) engine=Memory();
 
 set use_structure_from_insertion_table_in_table_functions=2;
 set input_format_json_infer_incomplete_types_as_strings=0;
+
 insert into test select * from file(02458_data.jsonl);
 insert into test select x, 1 from file(02458_data.jsonl);
 insert into test select x, y from file(02458_data.jsonl);
 insert into test select x + 1, y from file(02458_data.jsonl); -- {serverError CANNOT_EXTRACT_TABLE_STRUCTURE}
 insert into test select x, z from file(02458_data.jsonl);
-
 insert into test select * from file(02458_data.jsoncompacteachrow);
 insert into test select x, 1 from file(02458_data.jsoncompacteachrow); -- {serverError CANNOT_EXTRACT_TABLE_STRUCTURE}
 insert into test select x, y from file(02458_data.jsoncompacteachrow); -- {serverError CANNOT_EXTRACT_TABLE_STRUCTURE}
 insert into test select x + 1, y from file(02458_data.jsoncompacteachrow); -- {serverError CANNOT_EXTRACT_TABLE_STRUCTURE}
 insert into test select x, z from file(02458_data.jsoncompacteachrow); -- {serverError CANNOT_EXTRACT_TABLE_STRUCTURE}
-
 insert into test select * from input() format CSV 1,2
+
 insert into test select x, y from input() format CSV 1,2 -- {serverError CANNOT_EXTRACT_TABLE_STRUCTURE}
-insert into test select x, y from input() format JSONEachRow {"x" : null, "y" : 42}
+
+insert into test select x, y from input() format JSONEachRow {"x" : null, "y" : 42};
 
 select * from test order by y;
 
@@ -31,9 +32,9 @@ insert into test select * from file(02458_data.jsonl);
 insert into test select x from file(02458_data.jsonl);
 insert into test select y from file(02458_data.jsonl);
 insert into test select y as x from file(02458_data.jsonl);
-
 insert into test select c1 from input() format CSV 1,2; -- {serverError CANNOT_EXTRACT_TABLE_STRUCTURE}
-insert into test select x from input() format JSONEachRow {"x" : null, "y" : 42}
+
+insert into test select x from input() format JSONEachRow {"x" : null, "y" : 42};
 
 select * from test order by x;
 
diff --git a/tests/queries/0_stateless/02495_analyzer_storage_join.sql b/tests/queries/0_stateless/02495_analyzer_storage_join.sql
index aeab15862c3..7e6c03971f9 100644
--- a/tests/queries/0_stateless/02495_analyzer_storage_join.sql
+++ b/tests/queries/0_stateless/02495_analyzer_storage_join.sql
@@ -23,7 +23,7 @@ SELECT x FROM t1 ALL RIGHT JOIN tj ON t1.id1 == tj.key1 AND t1.id2 == tj.key2 OR
 SELECT '--- name clashes ---';
 
 CREATE TABLE t (key2 UInt64, key1 Int64, b UInt64, x UInt64, val UInt64) ENGINE = Memory;
-INSERT INTO t VALUES (1, -1, 11, 111, 1111), (2, -2, 22, 222, 2222), (3, -3, 33, 333, 2222), (4, -4, 44, 444, 4444), (5, -5, 55, 555, 5555)
+INSERT INTO t VALUES (1, -1, 11, 111, 1111), (2, -2, 22, 222, 2222), (3, -3, 33, 333, 2222), (4, -4, 44, 444, 4444), (5, -5, 55, 555, 5555);
 
 SELECT '-- using --';
 
diff --git a/tests/queries/0_stateless/02504_regexp_dictionary_table_source.sql b/tests/queries/0_stateless/02504_regexp_dictionary_table_source.sql
index 487b6e7f58e..604c5b179cf 100644
--- a/tests/queries/0_stateless/02504_regexp_dictionary_table_source.sql
+++ b/tests/queries/0_stateless/02504_regexp_dictionary_table_source.sql
@@ -14,12 +14,12 @@ CREATE TABLE regexp_dictionary_source_table
 
 -- test back reference.
 
-INSERT INTO regexp_dictionary_source_table VALUES (1, 0, 'Linux/(\d+[\.\d]*).+tlinux', ['name', 'version'], ['TencentOS', '\1'])
-INSERT INTO regexp_dictionary_source_table VALUES (2, 0, '(\d+)/tclwebkit(\d+[\.\d]*)', ['name', 'version', 'comment'], ['Android', '$1', 'test $1 and $2'])
-INSERT INTO regexp_dictionary_source_table VALUES (3, 2, '33/tclwebkit', ['version'], ['13'])
-INSERT INTO regexp_dictionary_source_table VALUES (4, 2, '3[12]/tclwebkit', ['version'], ['12'])
-INSERT INTO regexp_dictionary_source_table VALUES (5, 2, '3[12]/tclwebkit', ['version'], ['11'])
-INSERT INTO regexp_dictionary_source_table VALUES (6, 2, '3[12]/tclwebkit', ['version'], ['10'])
+INSERT INTO regexp_dictionary_source_table VALUES (1, 0, 'Linux/(\d+[\.\d]*).+tlinux', ['name', 'version'], ['TencentOS', '\1']);
+INSERT INTO regexp_dictionary_source_table VALUES (2, 0, '(\d+)/tclwebkit(\d+[\.\d]*)', ['name', 'version', 'comment'], ['Android', '$1', 'test $1 and $2']);
+INSERT INTO regexp_dictionary_source_table VALUES (3, 2, '33/tclwebkit', ['version'], ['13']);
+INSERT INTO regexp_dictionary_source_table VALUES (4, 2, '3[12]/tclwebkit', ['version'], ['12']);
+INSERT INTO regexp_dictionary_source_table VALUES (5, 2, '3[12]/tclwebkit', ['version'], ['11']);
+INSERT INTO regexp_dictionary_source_table VALUES (6, 2, '3[12]/tclwebkit', ['version'], ['10']);
 
 create dictionary regexp_dict1
 (
@@ -50,23 +50,24 @@ CREATE TABLE needle_table
 ENGINE=TinyLog;
 
 INSERT INTO needle_table select concat(toString(number + 30), '/tclwebkit', toString(number)) from system.numbers limit 15;
+
 select * from needle_table;
 select dictGet(regexp_dict1, ('name', 'version'), key) from needle_table;
 
 -- test invalid
-INSERT INTO regexp_dictionary_source_table VALUES (6, 2, '3[12]/tclwebkit', ['version'], ['10'])
+INSERT INTO regexp_dictionary_source_table VALUES (6, 2, '3[12]/tclwebkit', ['version'], ['10']);
 SYSTEM RELOAD dictionary regexp_dict1; -- { serverError INCORRECT_DICTIONARY_DEFINITION  }
 
 truncate table regexp_dictionary_source_table;
 
-INSERT INTO regexp_dictionary_source_table VALUES (6, 2, '3[12]/tclwebkit', ['version'], ['10'])
+INSERT INTO regexp_dictionary_source_table VALUES (6, 2, '3[12]/tclwebkit', ['version'], ['10']);
 SYSTEM RELOAD dictionary regexp_dict1; -- { serverError INCORRECT_DICTIONARY_DEFINITION  }
 
 truncate table regexp_dictionary_source_table;
 
-INSERT INTO regexp_dictionary_source_table VALUES (1, 2, 'Linux/(\d+[\.\d]*).+tlinux', ['name', 'version'], ['TencentOS', '\1'])
-INSERT INTO regexp_dictionary_source_table VALUES (2, 3, '(\d+)/tclwebkit(\d+[\.\d]*)', ['name', 'version', 'comment'], ['Android', '$1', 'test $1 and $2'])
-INSERT INTO regexp_dictionary_source_table VALUES (3, 1, '(\d+)/tclwebkit(\d+[\.\d]*)', ['name', 'version', 'comment'], ['Android', '$1', 'test $1 and $2'])
+INSERT INTO regexp_dictionary_source_table VALUES (1, 2, 'Linux/(\d+[\.\d]*).+tlinux', ['name', 'version'], ['TencentOS', '\1']);
+INSERT INTO regexp_dictionary_source_table VALUES (2, 3, '(\d+)/tclwebkit(\d+[\.\d]*)', ['name', 'version', 'comment'], ['Android', '$1', 'test $1 and $2']);
+INSERT INTO regexp_dictionary_source_table VALUES (3, 1, '(\d+)/tclwebkit(\d+[\.\d]*)', ['name', 'version', 'comment'], ['Android', '$1', 'test $1 and $2']);
 SYSTEM RELOAD dictionary regexp_dict1; -- { serverError INCORRECT_DICTIONARY_DEFINITION  }
 
 -- test priority
diff --git a/tests/queries/0_stateless/02536_replace_with_nonconst_needle_and_replacement.sql b/tests/queries/0_stateless/02536_replace_with_nonconst_needle_and_replacement.sql
index 926bde3a74b..b88224a89c1 100644
--- a/tests/queries/0_stateless/02536_replace_with_nonconst_needle_and_replacement.sql
+++ b/tests/queries/0_stateless/02536_replace_with_nonconst_needle_and_replacement.sql
@@ -7,7 +7,7 @@ CREATE TABLE test_tab
   engine = MergeTree()
   ORDER BY id;
 
-INSERT INTO test_tab VALUES (1, 'Hello World', 'l', 'xx') (2, 'Hello World', 'll', 'x') (3, 'Hello World', 'not_found', 'x') (4, 'Hello World', '[eo]', 'x') (5, 'Hello World', '.', 'x')
+INSERT INTO test_tab VALUES (1, 'Hello World', 'l', 'xx') (2, 'Hello World', 'll', 'x') (3, 'Hello World', 'not_found', 'x') (4, 'Hello World', '[eo]', 'x') (5, 'Hello World', '.', 'x');
 
 
 SELECT '** replaceAll() **';
@@ -77,7 +77,7 @@ CREATE TABLE test_tab
   engine = MergeTree()
   ORDER BY id;
 
-INSERT INTO test_tab VALUES (1, 'Hello World', 'l', 'x') (2, 'Hello World', '', 'y')
+INSERT INTO test_tab VALUES (1, 'Hello World', 'l', 'x') (2, 'Hello World', '', 'y');
 
 -- needle: non-const, replacement: const
 SELECT replaceAll(haystack, needle, 'x') FROM test_tab;  -- { serverError ARGUMENT_OUT_OF_BOUND }
diff --git a/tests/queries/0_stateless/02553_type_json_attach_partition.sql b/tests/queries/0_stateless/02553_type_json_attach_partition.sql
index 9225106f767..e77f5885ec3 100644
--- a/tests/queries/0_stateless/02553_type_json_attach_partition.sql
+++ b/tests/queries/0_stateless/02553_type_json_attach_partition.sql
@@ -5,6 +5,7 @@ DROP TABLE IF EXISTS t_json_attach_partition;
 CREATE TABLE t_json_attach_partition(b UInt64, c JSON) ENGINE = MergeTree ORDER BY tuple();
 
 INSERT INTO t_json_attach_partition FORMAT JSONEachRow {"b": 1, "c" : {"k1": 1}};
+
 ALTER TABLE t_json_attach_partition DETACH PARTITION tuple();
 INSERT INTO t_json_attach_partition FORMAT JSONEachRow {"b": 1, "c" : {"k1": [1, 2]}};
 
diff --git a/tests/queries/0_stateless/02726_async_insert_flush_queue.sql b/tests/queries/0_stateless/02726_async_insert_flush_queue.sql
index e393d96fc40..97d644fa4d6 100644
--- a/tests/queries/0_stateless/02726_async_insert_flush_queue.sql
+++ b/tests/queries/0_stateless/02726_async_insert_flush_queue.sql
@@ -11,9 +11,13 @@ SET async_insert_use_adaptive_busy_timeout=0;
 SET async_insert_busy_timeout_max_ms = 10000000;
 
 INSERT INTO t_async_inserts_flush VALUES (1) (2);
+
 INSERT INTO t_async_inserts_flush FORMAT JSONEachRow {"a": 10} {"a": 20};
-INSERT INTO t_async_inserts_flush FORMAT JSONEachRow {"a": "str"}
-INSERT INTO t_async_inserts_flush FORMAT JSONEachRow {"a": 100} {"a": 200}
+
+INSERT INTO t_async_inserts_flush FORMAT JSONEachRow {"a": "str"};
+
+INSERT INTO t_async_inserts_flush FORMAT JSONEachRow {"a": 100} {"a": 200};
+
 INSERT INTO t_async_inserts_flush VALUES (3) (4) (5);
 
 SELECT sleep(1) FORMAT Null;
diff --git a/tests/queries/0_stateless/02751_multiquery_with_argument.reference b/tests/queries/0_stateless/02751_multiquery_with_argument.reference
index 4b0c199ed35..2e55712e49c 100644
--- a/tests/queries/0_stateless/02751_multiquery_with_argument.reference
+++ b/tests/queries/0_stateless/02751_multiquery_with_argument.reference
@@ -17,8 +17,6 @@ Bad arguments
 Bad arguments
 Bad arguments
 Bad arguments
-Bad arguments
-Bad arguments
 320
 317
 Bad arguments
diff --git a/tests/queries/0_stateless/02751_multiquery_with_argument.sh b/tests/queries/0_stateless/02751_multiquery_with_argument.sh
index fea2978c116..7b959a3c3dc 100755
--- a/tests/queries/0_stateless/02751_multiquery_with_argument.sh
+++ b/tests/queries/0_stateless/02751_multiquery_with_argument.sh
@@ -22,14 +22,17 @@ $CLICKHOUSE_CLIENT --queries-file "queries.csv" --multiquery "SELECT 251;" 2>&1
 $CLICKHOUSE_LOCAL -n "SELECT 301"
 $CLICKHOUSE_LOCAL -n "SELECT 302;"
 $CLICKHOUSE_LOCAL -n "SELECT 304;SELECT 305;"
+# --multiquery and -n are obsolete by now and no-ops.
+# The only exception is a single --multiquery "<some_query>"
 $CLICKHOUSE_LOCAL --multiquery --multiquery 2>&1 | grep -o 'Bad arguments'
 $CLICKHOUSE_LOCAL -n --multiquery 2>&1 | grep -o 'Bad arguments'
 $CLICKHOUSE_LOCAL --multiquery -n 2>&1 | grep -o 'Bad arguments'
 $CLICKHOUSE_LOCAL --multiquery --multiquery "SELECT 306; SELECT 307;" 2>&1 | grep -o 'Bad arguments'
 $CLICKHOUSE_LOCAL -n --multiquery "SELECT 307; SELECT 308;" 2>&1 | grep -o 'Bad arguments'
-$CLICKHOUSE_LOCAL --multiquery "SELECT 309; SELECT 310;" --multiquery  2>&1 | grep -o 'Bad arguments'
+$CLICKHOUSE_LOCAL --multiquery "SELECT 309; SELECT 310;" --multiquery 2>&1 | grep -o 'Bad arguments'
 $CLICKHOUSE_LOCAL --multiquery "SELECT 311;" --multiquery "SELECT 312;" 2>&1 | grep -o 'Bad arguments'
 $CLICKHOUSE_LOCAL --multiquery "SELECT 313;" -n "SELECT 314;" 2>&1 | grep -o 'Bad arguments'
 $CLICKHOUSE_LOCAL -n "SELECT 320" --query "SELECT 317;"
+# --query should be followed by SQL
 $CLICKHOUSE_LOCAL --query -n "SELECT 400;" 2>&1 | grep -o 'Bad arguments'
 $CLICKHOUSE_LOCAL --query -n --multiquery "SELECT 401;" 2>&1 | grep -o 'Bad arguments'
diff --git a/tests/queries/0_stateless/02771_multiple_query_arguments.reference b/tests/queries/0_stateless/02771_multiple_query_arguments.reference
index 0c008e96965..b98ad7086a9 100644
--- a/tests/queries/0_stateless/02771_multiple_query_arguments.reference
+++ b/tests/queries/0_stateless/02771_multiple_query_arguments.reference
@@ -2,15 +2,17 @@
 101
 202
 202
-Multi-statements are not allowed
-Empty query
+303
+303
+303
 Bad arguments
 Syntax error
 101
 101
 202
 202
-Multi-statements are not allowed
-Empty query
+303
+303
+303
 Bad arguments
 Syntax error
diff --git a/tests/queries/0_stateless/02771_multiple_query_arguments.sh b/tests/queries/0_stateless/02771_multiple_query_arguments.sh
index 2958c9e9a72..ae6e23eb61a 100755
--- a/tests/queries/0_stateless/02771_multiple_query_arguments.sh
+++ b/tests/queries/0_stateless/02771_multiple_query_arguments.sh
@@ -7,15 +7,15 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # clickhouse-client
 $CLICKHOUSE_CLIENT --query "SELECT 101" --query "SELECT 101"
 $CLICKHOUSE_CLIENT --query "SELECT 202;" --query "SELECT 202;"
-$CLICKHOUSE_CLIENT --query "SELECT 303" --query "SELECT 303; SELECT 303" 2>&1 | grep -o 'Multi-statements are not allowed'
-$CLICKHOUSE_CLIENT --query "" --query "" 2>&1 | grep -o 'Empty query'
+$CLICKHOUSE_CLIENT --query "SELECT 303" --query "SELECT 303; SELECT 303"
+$CLICKHOUSE_CLIENT --query "" --query "" 2>&1
 $CLICKHOUSE_CLIENT --query "SELECT 303" --query 2>&1 | grep -o 'Bad arguments'
 $CLICKHOUSE_CLIENT --query "SELECT 303" --query "SELE" 2>&1 | grep -o 'Syntax error'
 
 # clickhouse-local
 $CLICKHOUSE_LOCAL --query "SELECT 101" --query "SELECT 101"
 $CLICKHOUSE_LOCAL --query "SELECT 202;" --query "SELECT 202;"
-$CLICKHOUSE_LOCAL --query "SELECT 303" --query "SELECT 303; SELECT 303" 2>&1 | grep -o 'Multi-statements are not allowed' # behaves differently than clickhouse-client, TODO make it consistent
-$CLICKHOUSE_LOCAL --query "" --query "" 2>&1 | grep -o 'Empty query' # behaves equally different than clickhouse-client TODO
+$CLICKHOUSE_LOCAL --query "SELECT 303" --query "SELECT 303; SELECT 303"
+$CLICKHOUSE_LOCAL --query "" --query ""
 $CLICKHOUSE_LOCAL --query "SELECT 303" --query 2>&1 | grep -o 'Bad arguments'
 $CLICKHOUSE_LOCAL --query "SELECT 303" --query "SELE" 2>&1 | grep -o 'Syntax error'
diff --git a/tests/queries/0_stateless/02875_clickhouse_local_multiquery.reference b/tests/queries/0_stateless/02875_clickhouse_local_multiquery.reference
index d99881dfa6b..1cc8f10a4cf 100644
--- a/tests/queries/0_stateless/02875_clickhouse_local_multiquery.reference
+++ b/tests/queries/0_stateless/02875_clickhouse_local_multiquery.reference
@@ -1,5 +1,7 @@
-Multi-statements are not allowed
-Multi-statements are not allowed
+1
+2
+1
+2
 1
 2
 1
diff --git a/tests/queries/0_stateless/02875_clickhouse_local_multiquery.sh b/tests/queries/0_stateless/02875_clickhouse_local_multiquery.sh
index e2dab337dc2..3f2b732e71b 100755
--- a/tests/queries/0_stateless/02875_clickhouse_local_multiquery.sh
+++ b/tests/queries/0_stateless/02875_clickhouse_local_multiquery.sh
@@ -4,10 +4,11 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CUR_DIR"/../shell_config.sh
 
-# throw exception
-$CLICKHOUSE_CLIENT -q "select 1; select 2;" 2>&1 | grep -o 'Multi-statements are not allowed'
-$CLICKHOUSE_LOCAL -q "select 1; select 2;" 2>&1 | grep -o 'Multi-statements are not allowed'
-# execute correctly
+# clickhouse-local and clickhouse-client behave the same
+$CLICKHOUSE_CLIENT -q "select 1; select 2;"
+$CLICKHOUSE_LOCAL -q "select 1; select 2;"
+
+# -n is a no-op
 $CLICKHOUSE_CLIENT -n -q "select 1; select 2;"
 $CLICKHOUSE_LOCAL -n -q "select 1; select 2;"
 
diff --git a/tests/queries/0_stateless/02898_input_format_values_allow_data_after_semicolon.reference b/tests/queries/0_stateless/02898_input_format_values_allow_data_after_semicolon.reference
index 250a673a26b..e69de29bb2d 100644
--- a/tests/queries/0_stateless/02898_input_format_values_allow_data_after_semicolon.reference
+++ b/tests/queries/0_stateless/02898_input_format_values_allow_data_after_semicolon.reference
@@ -1,6 +0,0 @@
-client no multiquery
-Cannot read data after semicolon (and input_format_values_allow_data_after_semicolon=0)
-client multiquery
-local no multiquery
-Cannot read data after semicolon (and input_format_values_allow_data_after_semicolon=0)
-local multiquery
diff --git a/tests/queries/0_stateless/02898_input_format_values_allow_data_after_semicolon.sh b/tests/queries/0_stateless/02898_input_format_values_allow_data_after_semicolon.sh
index 8164c91b2ae..2b9ac6bfd54 100755
--- a/tests/queries/0_stateless/02898_input_format_values_allow_data_after_semicolon.sh
+++ b/tests/queries/0_stateless/02898_input_format_values_allow_data_after_semicolon.sh
@@ -4,12 +4,5 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CUR_DIR"/../shell_config.sh
 
-echo "client no multiquery"
-$CLICKHOUSE_CLIENT -q "insert into function null() values (1); -- { foo }" |& grep -F -o "Cannot read data after semicolon (and input_format_values_allow_data_after_semicolon=0)"
-echo "client multiquery"
-$CLICKHOUSE_CLIENT -n -q "insert into function null() values (1); -- { foo }"
-
-echo "local no multiquery"
-$CLICKHOUSE_LOCAL -q "insert into function null() values (1); -- { foo }" |& grep -F -o "Cannot read data after semicolon (and input_format_values_allow_data_after_semicolon=0)"
-echo "local multiquery"
-$CLICKHOUSE_LOCAL -n -q "insert into function null() values (1); -- { foo }"
+$CLICKHOUSE_CLIENT -q "insert into function null() values (1); -- { foo }"
+$CLICKHOUSE_LOCAL  -q "insert into function null() values (1); -- { foo }"
diff --git a/tests/queries/0_stateless/02900_window_function_with_sparse_column.sql b/tests/queries/0_stateless/02900_window_function_with_sparse_column.sql
index 0b053d3e870..6919e23ad1f 100644
--- a/tests/queries/0_stateless/02900_window_function_with_sparse_column.sql
+++ b/tests/queries/0_stateless/02900_window_function_with_sparse_column.sql
@@ -18,7 +18,7 @@ ENGINE = MergeTree
 PARTITION BY toYYYYMM(time)
 ORDER BY (key, id, time);
 
-INSERT INTO test1 VALUES ('id0', now(), 3, false)
+INSERT INTO test1 VALUES ('id0', now(), 3, false);
 
 SELECT last_value(value) OVER (PARTITION BY id ORDER BY time ASC) as last_value
 FROM test1
diff --git a/tests/queries/0_stateless/02932_refreshable_materialized_views.sh b/tests/queries/0_stateless/02932_refreshable_materialized_views.sh
index 9081035579d..6ffc4064413 100755
--- a/tests/queries/0_stateless/02932_refreshable_materialized_views.sh
+++ b/tests/queries/0_stateless/02932_refreshable_materialized_views.sh
@@ -195,7 +195,7 @@ $CLICKHOUSE_CLIENT -nq "
 $CLICKHOUSE_CLIENT -nq "
     drop table d;
     truncate src;
-    insert into src values (1)
+    insert into src values (1);
     create materialized view e refresh every 1 second (x Int64) engine MergeTree order by x empty as select x + sleepEachRow(1) as x from src settings max_block_size = 1;"
 while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result from refreshes -- $LINENO" | xargs`" != 'Finished' ]
 do
@@ -213,6 +213,7 @@ done
 # we wait for a slow refresh, not a previous fast one.)
 $CLICKHOUSE_CLIENT -nq "
     insert into src select * from numbers(1000) settings max_block_size=1;
+    
     system start view e;"
 while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes -- $LINENO" | xargs`" != 'Running' ]
 do
diff --git a/tests/queries/0_stateless/02950_dictionary_short_circuit.sql b/tests/queries/0_stateless/02950_dictionary_short_circuit.sql
index 12c934a8d2d..c613709a975 100644
--- a/tests/queries/0_stateless/02950_dictionary_short_circuit.sql
+++ b/tests/queries/0_stateless/02950_dictionary_short_circuit.sql
@@ -241,12 +241,12 @@ CREATE TABLE regexp_dictionary_source_table
     values Array(String),
 ) ENGINE=TinyLog;
 
-INSERT INTO regexp_dictionary_source_table VALUES (1, 0, 'Linux/(\d+[\.\d]*).+tlinux', ['name', 'version'], ['TencentOS', '\1'])
-INSERT INTO regexp_dictionary_source_table VALUES (2, 0, '(\d+)/tclwebkit(\d+[\.\d]*)', ['name', 'version', 'comment'], ['Android', '$1', 'test $1 and $2'])
-INSERT INTO regexp_dictionary_source_table VALUES (3, 2, '33/tclwebkit', ['version'], ['13'])
-INSERT INTO regexp_dictionary_source_table VALUES (4, 2, '3[12]/tclwebkit', ['version'], ['12'])
-INSERT INTO regexp_dictionary_source_table VALUES (5, 2, '3[12]/tclwebkit', ['version'], ['11'])
-INSERT INTO regexp_dictionary_source_table VALUES (6, 2, '3[12]/tclwebkit', ['version'], ['10'])
+INSERT INTO regexp_dictionary_source_table VALUES (1, 0, 'Linux/(\d+[\.\d]*).+tlinux', ['name', 'version'], ['TencentOS', '\1']);
+INSERT INTO regexp_dictionary_source_table VALUES (2, 0, '(\d+)/tclwebkit(\d+[\.\d]*)', ['name', 'version', 'comment'], ['Android', '$1', 'test $1 and $2']);
+INSERT INTO regexp_dictionary_source_table VALUES (3, 2, '33/tclwebkit', ['version'], ['13']);
+INSERT INTO regexp_dictionary_source_table VALUES (4, 2, '3[12]/tclwebkit', ['version'], ['12']);
+INSERT INTO regexp_dictionary_source_table VALUES (5, 2, '3[12]/tclwebkit', ['version'], ['11']);
+INSERT INTO regexp_dictionary_source_table VALUES (6, 2, '3[12]/tclwebkit', ['version'], ['10']);
 
 DROP DICTIONARY IF EXISTS regexp_dict;
 create dictionary regexp_dict
diff --git a/tests/queries/0_stateless/02974_analyzer_array_join_subcolumn.sql b/tests/queries/0_stateless/02974_analyzer_array_join_subcolumn.sql
index 14823644b96..30fb3c76c1f 100644
--- a/tests/queries/0_stateless/02974_analyzer_array_join_subcolumn.sql
+++ b/tests/queries/0_stateless/02974_analyzer_array_join_subcolumn.sql
@@ -2,7 +2,7 @@ DROP TABLE IF EXISTS t2;
 DROP TABLE IF EXISTS t3;
 
 CREATE TABLE t2 (id Int32, pe Map(String, Tuple(a UInt64, b UInt64))) ENGINE = MergeTree ORDER BY id;
-INSERT INTO t2 VALUES (1, {'a': (1, 2), 'b': (2, 3)}),
+INSERT INTO t2 VALUES (1, {'a': (1, 2), 'b': (2, 3)}),;
 
 CREATE TABLE t3 (id Int32, c Tuple(v String, pe Map(String, Tuple(a UInt64, b UInt64)))) ENGINE = MergeTree ORDER BY id;
 INSERT INTO t3 VALUES (1, ('A', {'a':(1, 2),'b':(2, 3)}));
diff --git a/tests/queries/0_stateless/03005_input_function_in_join.sql b/tests/queries/0_stateless/03005_input_function_in_join.sql
index 8a6b4a48a8d..a6fc27cd8d0 100644
--- a/tests/queries/0_stateless/03005_input_function_in_join.sql
+++ b/tests/queries/0_stateless/03005_input_function_in_join.sql
@@ -8,7 +8,7 @@ SELECT x.number FROM (
 ) AS x
 INNER JOIN input('a UInt64') AS y ON x.number = y.a
 Format CSV 2
-;
+
 select * from test;
 drop table test;
 
diff --git a/tests/queries/0_stateless/03156_default_multiquery_split.reference b/tests/queries/0_stateless/03156_default_multiquery_split.reference
new file mode 100644
index 00000000000..0f3a1baff45
--- /dev/null
+++ b/tests/queries/0_stateless/03156_default_multiquery_split.reference
@@ -0,0 +1,10 @@
+Syntax error
+101
+102
+1
+2;
+3
+4
+7
+8
+9
diff --git a/tests/queries/0_stateless/03156_default_multiquery_split.sh b/tests/queries/0_stateless/03156_default_multiquery_split.sh
new file mode 100755
index 00000000000..ac64c2d093d
--- /dev/null
+++ b/tests/queries/0_stateless/03156_default_multiquery_split.sh
@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+# Tags: no-parallel, no-ordinary-database
+
+CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CUR_DIR"/../shell_config.sh
+
+SQL_FILE_NAME=$"03156_default_multiquery_split_$$.sql"
+
+# The old multiquery implementation uses '\n' to split INSERT query segmentation
+# this case is mainly to test the following situations
+# 1. INSERT(format=Values) query: split by ';'
+# 2. INSERT(format is not Values) query: split by '\n\n' instead of ';', then discard the ramaining data
+# 3. INSERT(format is not Values) query: split by '\n\n', causing the trailing ';'' to be treated as the part of last value
+# 4. The client uses multiquery by default, regardless of whether multiquery option is used.
+
+# create table test1, test2, then
+# 1. insert 101, 102 into test1
+# 2. insert 1, 2; into test2, ';' will be treated as a part of a value
+# 3. insert 3, 4; '6' will be treated as the next query because of the empty line, we use empty line to determine the end of insert query(format IS NOT VALUES)
+#     '6' will cause Syntax error
+cat << EOF > "$SQL_FILE_NAME"
+DROP TABLE IF EXISTS TEST1;
+DROP TABLE IF EXISTS TEST2;
+CREATE TABLE TEST1 (value Float64) ENGINE=MergeTree ORDER BY tuple();
+CREATE TABLE TEST2 (value String)  ENGINE=MergeTree ORDER BY tuple();
+INSERT INTO TEST1 VALUES
+(101),
+(102);
+INSERT INTO TEST2 FORMAT CSV
+1
+2;
+
+INSERT INTO TEST2 FORMAT CSV
+3
+4
+
+6
+EOF
+
+$CLICKHOUSE_CLIENT -m < "$SQL_FILE_NAME" 2>&1 | grep -o 'Syntax error'
+
+# insert 7, 8, 9 into test2, because we use semicolon to determine the end of insert query(format is VALUES)
+# then select all data from test1 and test2
+cat << EOF > "$SQL_FILE_NAME"
+INSERT INTO TEST2 VALUES
+('7'),
+('8'),
+
+('9');
+
+SELECT * FROM TEST1 ORDER BY value;
+SELECT * FROM TEST2 ORDER BY value;
+DROP TABLE TEST1; DROP TABLE TEST2;
+EOF
+
+$CLICKHOUSE_CLIENT -m -n < "$SQL_FILE_NAME"
+
+rm "$SQL_FILE_NAME"

From ba1c191331d3824786cfdac15437dc7bd142cf09 Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio2368@users.noreply.github.com>
Date: Mon, 29 Jul 2024 11:47:57 +0200
Subject: [PATCH 0955/1488] Update programs/keeper/Keeper.cpp
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: János Benjamin Antal <antaljanosbenjamin@users.noreply.github.com>
---
 programs/keeper/Keeper.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/programs/keeper/Keeper.cpp b/programs/keeper/Keeper.cpp
index a1793b5d603..783f60cb8ff 100644
--- a/programs/keeper/Keeper.cpp
+++ b/programs/keeper/Keeper.cpp
@@ -315,7 +315,7 @@ try
     if (auto total_numa_memory = getNumaNodesTotalMemory(); total_numa_memory.has_value())
     {
         LOG_INFO(
-            log, "ClickHouse is bound to a subset of NUMA nodes. Total memory of all available nodes: {}", ReadableSize(*total_numa_memory));
+            log, "Keeper is bound to a subset of NUMA nodes. Total memory of all available nodes: {}", ReadableSize(*total_numa_memory));
     }
 
 #if !defined(NDEBUG) || !defined(__OPTIMIZE__)

From 0f29f89d99694a780f28ba7de4518bb8b0a0ece2 Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Tue, 2 Jul 2024 09:59:43 +0000
Subject: [PATCH 0956/1488] Increase timeout in 02122_join_group_by_timeout for
 tsan build

---
 tests/queries/0_stateless/02122_join_group_by_timeout.sh | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tests/queries/0_stateless/02122_join_group_by_timeout.sh b/tests/queries/0_stateless/02122_join_group_by_timeout.sh
index 59719f75d7c..8380c5dbd0c 100755
--- a/tests/queries/0_stateless/02122_join_group_by_timeout.sh
+++ b/tests/queries/0_stateless/02122_join_group_by_timeout.sh
@@ -12,6 +12,12 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
 MAX_PROCESS_WAIT=5
 
+IS_SANITIZER=$($CLICKHOUSE_CLIENT -q "SELECT count() FROM system.warnings WHERE message like '%built with sanitizer%'")
+if [ "$IS_SANITIZER" -gt 0 ]; then
+    # Query may hang for more than 5 seconds, especially in tsan build
+    MAX_PROCESS_WAIT=15
+fi
+
 # TCP CLIENT: As of today (02/12/21) uses PullingAsyncPipelineExecutor
 ### Should be cancelled after 1 second and return a 159 exception (timeout)
 timeout -s KILL $MAX_PROCESS_WAIT $CLICKHOUSE_CLIENT --max_execution_time 1 -q \

From f7c7d0b79bc4e24428264c33ed9074905ee684ea Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Mon, 29 Jul 2024 10:25:49 +0000
Subject: [PATCH 0957/1488] Fix tests

---
 src/Functions/CountSubstringsImpl.h           |  17 +-
 src/Functions/FunctionsHashing.h              | 155 +++++++++---------
 .../FunctionsMultiStringFuzzySearch.h         |   8 +-
 src/Functions/FunctionsMultiStringSearch.h    |   8 +-
 src/Functions/FunctionsStringSearch.h         |  17 +-
 src/Functions/FunctionsStringSearchToString.h |   4 +-
 src/Functions/FunctionsVisitParam.h           |  10 +-
 src/Functions/HasTokenImpl.h                  |   5 +-
 src/Functions/MatchImpl.h                     |  44 ++---
 src/Functions/MultiMatchAllIndicesImpl.h      |  27 ++-
 src/Functions/MultiMatchAnyImpl.h             |  31 ++--
 src/Functions/MultiSearchFirstIndexImpl.h     |  22 ++-
 src/Functions/MultiSearchFirstPositionImpl.h  |  20 ++-
 src/Functions/MultiSearchImpl.h               |  18 +-
 src/Functions/PositionImpl.h                  |  22 +--
 src/Functions/URL/extractURLParameter.cpp     |   7 +-
 src/Functions/extract.cpp                     |   7 +-
 17 files changed, 213 insertions(+), 209 deletions(-)

diff --git a/src/Functions/CountSubstringsImpl.h b/src/Functions/CountSubstringsImpl.h
index 8e91bc3aeb4..9ff3e4e1f2a 100644
--- a/src/Functions/CountSubstringsImpl.h
+++ b/src/Functions/CountSubstringsImpl.h
@@ -37,8 +37,7 @@ struct CountSubstringsImpl
         const std::string & needle,
         const ColumnPtr & start_pos,
         PaddedPODArray<UInt64> & res,
-        [[maybe_unused]] ColumnUInt8 * res_null,
-        size_t /*input_rows_count*/)
+        [[maybe_unused]] ColumnUInt8 * res_null)
     {
         /// `res_null` serves as an output parameter for implementing an XYZOrNull variant.
         assert(!res_null);
@@ -151,8 +150,7 @@ struct CountSubstringsImpl
         const ColumnString::Offsets & needle_offsets,
         const ColumnPtr & start_pos,
         PaddedPODArray<UInt64> & res,
-        [[maybe_unused]] ColumnUInt8 * res_null,
-        size_t input_rows_count)
+        [[maybe_unused]] ColumnUInt8 * res_null)
     {
         /// `res_null` serves as an output parameter for implementing an XYZOrNull variant.
         assert(!res_null);
@@ -160,7 +158,9 @@ struct CountSubstringsImpl
         ColumnString::Offset prev_haystack_offset = 0;
         ColumnString::Offset prev_needle_offset = 0;
 
-        for (size_t i = 0; i < input_rows_count; ++i)
+        size_t size = haystack_offsets.size();
+
+        for (size_t i = 0; i < size; ++i)
         {
             size_t needle_size = needle_offsets[i] - prev_needle_offset - 1;
             size_t haystack_size = haystack_offsets[i] - prev_haystack_offset - 1;
@@ -207,8 +207,7 @@ struct CountSubstringsImpl
         const ColumnString::Offsets & needle_offsets,
         const ColumnPtr & start_pos,
         PaddedPODArray<UInt64> & res,
-        [[maybe_unused]] ColumnUInt8 * res_null,
-        size_t input_rows_count)
+        [[maybe_unused]] ColumnUInt8 * res_null)
     {
         /// `res_null` serves as an output parameter for implementing an XYZOrNull variant.
         assert(!res_null);
@@ -216,7 +215,9 @@ struct CountSubstringsImpl
         /// NOTE You could use haystack indexing. But this is a rare case.
         ColumnString::Offset prev_needle_offset = 0;
 
-        for (size_t i = 0; i < input_rows_count; ++i)
+        size_t size = needle_offsets.size();
+
+        for (size_t i = 0; i < size; ++i)
         {
             res[i] = 0;
             auto start = start_pos != nullptr ? std::max(start_pos->getUInt(i), UInt64(1)) : UInt64(1);
diff --git a/src/Functions/FunctionsHashing.h b/src/Functions/FunctionsHashing.h
index 91e49bf5035..95c54ac9528 100644
--- a/src/Functions/FunctionsHashing.h
+++ b/src/Functions/FunctionsHashing.h
@@ -715,7 +715,7 @@ private:
     using ToType = typename Impl::ReturnType;
 
     template <typename FromType>
-    ColumnPtr executeType(const ColumnsWithTypeAndName & arguments, size_t input_rows_count) const
+    ColumnPtr executeType(const ColumnsWithTypeAndName & arguments) const
     {
         using ColVecType = ColumnVectorOrDecimal<FromType>;
 
@@ -726,8 +726,9 @@ private:
             const typename ColVecType::Container & vec_from = col_from->getData();
             typename ColumnVector<ToType>::Container & vec_to = col_to->getData();
 
-            vec_to.resize(input_rows_count);
-            for (size_t i = 0; i < input_rows_count; ++i)
+            size_t size = vec_from.size();
+            vec_to.resize(size);
+            for (size_t i = 0; i < size; ++i)
                 vec_to[i] = Impl::apply(vec_from[i]);
 
             return col_to;
@@ -758,39 +759,39 @@ public:
 
     bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
     {
         const IDataType * from_type = arguments[0].type.get();
         WhichDataType which(from_type);
 
         if (which.isUInt8())
-            return executeType<UInt8>(arguments, input_rows_count);
+            return executeType<UInt8>(arguments);
         else if (which.isUInt16())
-            return executeType<UInt16>(arguments, input_rows_count);
+            return executeType<UInt16>(arguments);
         else if (which.isUInt32())
-            return executeType<UInt32>(arguments, input_rows_count);
+            return executeType<UInt32>(arguments);
         else if (which.isUInt64())
-            return executeType<UInt64>(arguments, input_rows_count);
+            return executeType<UInt64>(arguments);
         else if (which.isInt8())
-            return executeType<Int8>(arguments, input_rows_count);
+            return executeType<Int8>(arguments);
         else if (which.isInt16())
-            return executeType<Int16>(arguments, input_rows_count);
+            return executeType<Int16>(arguments);
         else if (which.isInt32())
-            return executeType<Int32>(arguments, input_rows_count);
+            return executeType<Int32>(arguments);
         else if (which.isInt64())
-            return executeType<Int64>(arguments, input_rows_count);
+            return executeType<Int64>(arguments);
         else if (which.isDate())
-            return executeType<UInt16>(arguments, input_rows_count);
+            return executeType<UInt16>(arguments);
         else if (which.isDate32())
-            return executeType<Int32>(arguments, input_rows_count);
+            return executeType<Int32>(arguments);
         else if (which.isDateTime())
-            return executeType<UInt32>(arguments, input_rows_count);
+            return executeType<UInt32>(arguments);
         else if (which.isDecimal32())
-            return executeType<Decimal32>(arguments, input_rows_count);
+            return executeType<Decimal32>(arguments);
         else if (which.isDecimal64())
-            return executeType<Decimal64>(arguments, input_rows_count);
+            return executeType<Decimal64>(arguments);
         else if (which.isIPv4())
-            return executeType<IPv4>(arguments, input_rows_count);
+            return executeType<IPv4>(arguments);
         else
             throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}",
                 arguments[0].type->getName(), getName());
@@ -842,7 +843,7 @@ private:
     using ToType = typename Impl::ReturnType;
 
     template <typename FromType, bool first>
-    void executeIntType(const KeyColumnsType & key_cols, const IColumn * column, typename ColumnVector<ToType>::Container & vec_to, size_t input_rows_count) const
+    void executeIntType(const KeyColumnsType & key_cols, const IColumn * column, typename ColumnVector<ToType>::Container & vec_to) const
     {
         using ColVecType = ColumnVectorOrDecimal<FromType>;
         KeyType key{};
@@ -852,7 +853,8 @@ private:
         if (const ColVecType * col_from = checkAndGetColumn<ColVecType>(column))
         {
             const typename ColVecType::Container & vec_from = col_from->getData();
-            for (size_t i = 0; i < input_rows_count; ++i)
+            const size_t size = vec_from.size();
+            for (size_t i = 0; i < size; ++i)
             {
                 ToType hash;
 
@@ -892,7 +894,7 @@ private:
                 if (!key_cols.is_const)
                 {
                     ColumnPtr full_column = col_from_const->convertToFullColumn();
-                    return executeIntType<FromType, first>(key_cols, full_column.get(), vec_to, input_rows_count);
+                    return executeIntType<FromType, first>(key_cols, full_column.get(), vec_to);
                 }
             }
             auto value = col_from_const->template getValue<FromType>();
@@ -936,7 +938,7 @@ private:
     }
 
     template <typename FromType, bool first>
-    void executeBigIntType(const KeyColumnsType & key_cols, const IColumn * column, typename ColumnVector<ToType>::Container & vec_to, size_t input_rows_count) const
+    void executeBigIntType(const KeyColumnsType & key_cols, const IColumn * column, typename ColumnVector<ToType>::Container & vec_to) const
     {
         using ColVecType = ColumnVectorOrDecimal<FromType>;
         KeyType key{};
@@ -956,7 +958,8 @@ private:
         if (const ColVecType * col_from = checkAndGetColumn<ColVecType>(column))
         {
             const typename ColVecType::Container & vec_from = col_from->getData();
-            for (size_t i = 0; i < input_rows_count; ++i)
+            size_t size = vec_from.size();
+            for (size_t i = 0; i < size; ++i)
             {
                 ToType hash;
                 if constexpr (Keyed)
@@ -984,7 +987,7 @@ private:
                 if (!key_cols.is_const)
                 {
                     ColumnPtr full_column = col_from_const->convertToFullColumn();
-                    return executeBigIntType<FromType, first>(key_cols, full_column.get(), vec_to, input_rows_count);
+                    return executeBigIntType<FromType, first>(key_cols, full_column.get(), vec_to);
                 }
             }
             auto value = col_from_const->template getValue<FromType>();
@@ -1011,12 +1014,12 @@ private:
     }
 
     template <bool first>
-    void executeGeneric(const KeyColumnsType & key_cols, const IColumn * column, typename ColumnVector<ToType>::Container & vec_to, size_t input_rows_count) const
+    void executeGeneric(const KeyColumnsType & key_cols, const IColumn * column, typename ColumnVector<ToType>::Container & vec_to) const
     {
         KeyType key{};
         if constexpr (Keyed)
             key = Impl::getKey(key_cols, 0);
-        for (size_t i = 0, size = input_rows_count; i < size; ++i)
+        for (size_t i = 0, size = column->size(); i < size; ++i)
         {
             if constexpr (Keyed)
                 if (!key_cols.is_const && i != 0)
@@ -1031,7 +1034,7 @@ private:
     }
 
     template <bool first>
-    void executeString(const KeyColumnsType & key_cols, const IColumn * column, typename ColumnVector<ToType>::Container & vec_to, size_t input_rows_count) const
+    void executeString(const KeyColumnsType & key_cols, const IColumn * column, typename ColumnVector<ToType>::Container & vec_to) const
     {
         KeyType key{};
         if constexpr (Keyed)
@@ -1040,9 +1043,10 @@ private:
         {
             const typename ColumnString::Chars & data = col_from->getChars();
             const typename ColumnString::Offsets & offsets = col_from->getOffsets();
+            size_t size = offsets.size();
 
             ColumnString::Offset current_offset = 0;
-            for (size_t i = 0; i < input_rows_count; ++i)
+            for (size_t i = 0; i < size; ++i)
             {
                 if constexpr (Keyed)
                     if (!key_cols.is_const && i != 0)
@@ -1063,8 +1067,9 @@ private:
         {
             const typename ColumnString::Chars & data = col_from_fixed->getChars();
             size_t n = col_from_fixed->getN();
+            size_t size = data.size() / n;
 
-            for (size_t i = 0; i < input_rows_count; ++i)
+            for (size_t i = 0; i < size; ++i)
             {
                 if constexpr (Keyed)
                     if (!key_cols.is_const && i != 0)
@@ -1083,7 +1088,7 @@ private:
                 if (!key_cols.is_const)
                 {
                     ColumnPtr full_column = col_from_const->convertToFullColumn();
-                    return executeString<first>(key_cols, full_column.get(), vec_to, input_rows_count);
+                    return executeString<first>(key_cols, full_column.get(), vec_to);
                 }
             }
             String value = col_from_const->getValue<String>();
@@ -1109,7 +1114,7 @@ private:
     }
 
     template <bool first>
-    void executeArray(const KeyColumnsType & key_cols, const IDataType * type, const IColumn * column, typename ColumnVector<ToType>::Container & vec_to, size_t input_rows_count) const
+    void executeArray(const KeyColumnsType & key_cols, const IDataType * type, const IColumn * column, typename ColumnVector<ToType>::Container & vec_to) const
     {
         const IDataType * nested_type = typeid_cast<const DataTypeArray &>(*type).getNestedType().get();
 
@@ -1126,16 +1131,18 @@ private:
             {
                 KeyColumnsType key_cols_tmp{key_cols};
                 key_cols_tmp.offsets = &offsets;
-                executeForArgument(key_cols_tmp, nested_type, nested_column, vec_temp, nested_is_first, input_rows_count);
+                executeForArgument(key_cols_tmp, nested_type, nested_column, vec_temp, nested_is_first);
             }
             else
-                executeForArgument(key_cols, nested_type, nested_column, vec_temp, nested_is_first, input_rows_count);
+                executeForArgument(key_cols, nested_type, nested_column, vec_temp, nested_is_first);
+
+            const size_t size = offsets.size();
 
             ColumnArray::Offset current_offset = 0;
             KeyType key{};
             if constexpr (Keyed)
                 key = Impl::getKey(key_cols, 0);
-            for (size_t i = 0; i < input_rows_count; ++i)
+            for (size_t i = 0; i < size; ++i)
             {
                 if constexpr (Keyed)
                     if (!key_cols.is_const && i != 0)
@@ -1163,7 +1170,7 @@ private:
         {
             /// NOTE: here, of course, you can do without the materialization of the column.
             ColumnPtr full_column = col_from_const->convertToFullColumn();
-            executeArray<first>(key_cols, type, full_column.get(), vec_to, input_rows_count);
+            executeArray<first>(key_cols, type, full_column.get(), vec_to);
         }
         else
             throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of first argument of function {}",
@@ -1171,7 +1178,7 @@ private:
     }
 
     template <bool first>
-    void executeAny(const KeyColumnsType & key_cols, const IDataType * from_type, const IColumn * icolumn, typename ColumnVector<ToType>::Container & vec_to, size_t input_rows_count) const
+    void executeAny(const KeyColumnsType & key_cols, const IDataType * from_type, const IColumn * icolumn, typename ColumnVector<ToType>::Container & vec_to) const
     {
         WhichDataType which(from_type);
 
@@ -1183,43 +1190,43 @@ private:
             if (key_cols.size() != vec_to.size() && key_cols.size() != 1)
                 throw Exception(ErrorCodes::LOGICAL_ERROR, "Key column size {} doesn't match result column size {} of function {}", key_cols.size(), vec_to.size(), getName());
 
-        if      (which.isUInt8()) executeIntType<UInt8, first>(key_cols, icolumn, vec_to, input_rows_count);
-        else if (which.isUInt16()) executeIntType<UInt16, first>(key_cols, icolumn, vec_to, input_rows_count);
-        else if (which.isUInt32()) executeIntType<UInt32, first>(key_cols, icolumn, vec_to, input_rows_count);
-        else if (which.isUInt64()) executeIntType<UInt64, first>(key_cols, icolumn, vec_to, input_rows_count);
-        else if (which.isUInt128()) executeBigIntType<UInt128, first>(key_cols, icolumn, vec_to, input_rows_count);
-        else if (which.isUInt256()) executeBigIntType<UInt256, first>(key_cols, icolumn, vec_to, input_rows_count);
-        else if (which.isInt8()) executeIntType<Int8, first>(key_cols, icolumn, vec_to, input_rows_count);
-        else if (which.isInt16()) executeIntType<Int16, first>(key_cols, icolumn, vec_to, input_rows_count);
-        else if (which.isInt32()) executeIntType<Int32, first>(key_cols, icolumn, vec_to, input_rows_count);
-        else if (which.isInt64()) executeIntType<Int64, first>(key_cols, icolumn, vec_to, input_rows_count);
-        else if (which.isInt128()) executeBigIntType<Int128, first>(key_cols, icolumn, vec_to, input_rows_count);
-        else if (which.isInt256()) executeBigIntType<Int256, first>(key_cols, icolumn, vec_to, input_rows_count);
-        else if (which.isUUID()) executeBigIntType<UUID, first>(key_cols, icolumn, vec_to, input_rows_count);
-        else if (which.isIPv4()) executeIntType<IPv4, first>(key_cols, icolumn, vec_to, input_rows_count);
-        else if (which.isIPv6()) executeBigIntType<IPv6, first>(key_cols, icolumn, vec_to, input_rows_count);
-        else if (which.isEnum8()) executeIntType<Int8, first>(key_cols, icolumn, vec_to, input_rows_count);
-        else if (which.isEnum16()) executeIntType<Int16, first>(key_cols, icolumn, vec_to, input_rows_count);
-        else if (which.isDate()) executeIntType<UInt16, first>(key_cols, icolumn, vec_to, input_rows_count);
-        else if (which.isDate32()) executeIntType<Int32, first>(key_cols, icolumn, vec_to, input_rows_count);
-        else if (which.isDateTime()) executeIntType<UInt32, first>(key_cols, icolumn, vec_to, input_rows_count);
+        if      (which.isUInt8()) executeIntType<UInt8, first>(key_cols, icolumn, vec_to);
+        else if (which.isUInt16()) executeIntType<UInt16, first>(key_cols, icolumn, vec_to);
+        else if (which.isUInt32()) executeIntType<UInt32, first>(key_cols, icolumn, vec_to);
+        else if (which.isUInt64()) executeIntType<UInt64, first>(key_cols, icolumn, vec_to);
+        else if (which.isUInt128()) executeBigIntType<UInt128, first>(key_cols, icolumn, vec_to);
+        else if (which.isUInt256()) executeBigIntType<UInt256, first>(key_cols, icolumn, vec_to);
+        else if (which.isInt8()) executeIntType<Int8, first>(key_cols, icolumn, vec_to);
+        else if (which.isInt16()) executeIntType<Int16, first>(key_cols, icolumn, vec_to);
+        else if (which.isInt32()) executeIntType<Int32, first>(key_cols, icolumn, vec_to);
+        else if (which.isInt64()) executeIntType<Int64, first>(key_cols, icolumn, vec_to);
+        else if (which.isInt128()) executeBigIntType<Int128, first>(key_cols, icolumn, vec_to);
+        else if (which.isInt256()) executeBigIntType<Int256, first>(key_cols, icolumn, vec_to);
+        else if (which.isUUID()) executeBigIntType<UUID, first>(key_cols, icolumn, vec_to);
+        else if (which.isIPv4()) executeIntType<IPv4, first>(key_cols, icolumn, vec_to);
+        else if (which.isIPv6()) executeBigIntType<IPv6, first>(key_cols, icolumn, vec_to);
+        else if (which.isEnum8()) executeIntType<Int8, first>(key_cols, icolumn, vec_to);
+        else if (which.isEnum16()) executeIntType<Int16, first>(key_cols, icolumn, vec_to);
+        else if (which.isDate()) executeIntType<UInt16, first>(key_cols, icolumn, vec_to);
+        else if (which.isDate32()) executeIntType<Int32, first>(key_cols, icolumn, vec_to);
+        else if (which.isDateTime()) executeIntType<UInt32, first>(key_cols, icolumn, vec_to);
         /// TODO: executeIntType() for Decimal32/64 leads to incompatible result
-        else if (which.isDecimal32()) executeBigIntType<Decimal32, first>(key_cols, icolumn, vec_to, input_rows_count);
-        else if (which.isDecimal64()) executeBigIntType<Decimal64, first>(key_cols, icolumn, vec_to, input_rows_count);
-        else if (which.isDecimal128()) executeBigIntType<Decimal128, first>(key_cols, icolumn, vec_to, input_rows_count);
-        else if (which.isDecimal256()) executeBigIntType<Decimal256, first>(key_cols, icolumn, vec_to, input_rows_count);
-        else if (which.isFloat32()) executeIntType<Float32, first>(key_cols, icolumn, vec_to, input_rows_count);
-        else if (which.isFloat64()) executeIntType<Float64, first>(key_cols, icolumn, vec_to, input_rows_count);
-        else if (which.isString()) executeString<first>(key_cols, icolumn, vec_to, input_rows_count);
-        else if (which.isFixedString()) executeString<first>(key_cols, icolumn, vec_to, input_rows_count);
-        else if (which.isArray()) executeArray<first>(key_cols, from_type, icolumn, vec_to, input_rows_count);
-        else executeGeneric<first>(key_cols, icolumn, vec_to, input_rows_count);
+        else if (which.isDecimal32()) executeBigIntType<Decimal32, first>(key_cols, icolumn, vec_to);
+        else if (which.isDecimal64()) executeBigIntType<Decimal64, first>(key_cols, icolumn, vec_to);
+        else if (which.isDecimal128()) executeBigIntType<Decimal128, first>(key_cols, icolumn, vec_to);
+        else if (which.isDecimal256()) executeBigIntType<Decimal256, first>(key_cols, icolumn, vec_to);
+        else if (which.isFloat32()) executeIntType<Float32, first>(key_cols, icolumn, vec_to);
+        else if (which.isFloat64()) executeIntType<Float64, first>(key_cols, icolumn, vec_to);
+        else if (which.isString()) executeString<first>(key_cols, icolumn, vec_to);
+        else if (which.isFixedString()) executeString<first>(key_cols, icolumn, vec_to);
+        else if (which.isArray()) executeArray<first>(key_cols, from_type, icolumn, vec_to);
+        else executeGeneric<first>(key_cols, icolumn, vec_to);
     }
 
     /// Return a fixed random-looking magic number when input is empty.
     static constexpr auto filler = 0xe28dbde7fe22e41c;
 
-    void executeForArgument(const KeyColumnsType & key_cols, const IDataType * type, const IColumn * column, typename ColumnVector<ToType>::Container & vec_to, bool & is_first, size_t input_rows_count) const
+    void executeForArgument(const KeyColumnsType & key_cols, const IDataType * type, const IColumn * column, typename ColumnVector<ToType>::Container & vec_to, bool & is_first) const
     {
         /// Flattening of tuples.
         if (const ColumnTuple * tuple = typeid_cast<const ColumnTuple *>(column))
@@ -1233,7 +1240,7 @@ private:
                     hash = static_cast<ToType>(filler);
 
             for (size_t i = 0; i < tuple_size; ++i)
-                executeForArgument(key_cols, tuple_types[i].get(), tuple_columns[i].get(), vec_to, is_first, input_rows_count);
+                executeForArgument(key_cols, tuple_types[i].get(), tuple_columns[i].get(), vec_to, is_first);
         }
         else if (const ColumnTuple * tuple_const = checkAndGetColumnConstData<ColumnTuple>(column))
         {
@@ -1248,24 +1255,24 @@ private:
             for (size_t i = 0; i < tuple_size; ++i)
             {
                 auto tmp = ColumnConst::create(tuple_columns[i], column->size());
-                executeForArgument(key_cols, tuple_types[i].get(), tmp.get(), vec_to, is_first, input_rows_count);
+                executeForArgument(key_cols, tuple_types[i].get(), tmp.get(), vec_to, is_first);
             }
         }
         else if (const auto * map = checkAndGetColumn<ColumnMap>(column))
         {
             const auto & type_map = assert_cast<const DataTypeMap &>(*type);
-            executeForArgument(key_cols, type_map.getNestedType().get(), map->getNestedColumnPtr().get(), vec_to, is_first, input_rows_count);
+            executeForArgument(key_cols, type_map.getNestedType().get(), map->getNestedColumnPtr().get(), vec_to, is_first);
         }
         else if (const auto * const_map = checkAndGetColumnConst<ColumnMap>(column))
         {
-            executeForArgument(key_cols, type, const_map->convertToFullColumnIfConst().get(), vec_to, is_first, input_rows_count);
+            executeForArgument(key_cols, type, const_map->convertToFullColumnIfConst().get(), vec_to, is_first);
         }
         else
         {
             if (is_first)
-                executeAny<true>(key_cols, type, column, vec_to, input_rows_count);
+                executeAny<true>(key_cols, type, column, vec_to);
             else
-                executeAny<false>(key_cols, type, column, vec_to, input_rows_count);
+                executeAny<false>(key_cols, type, column, vec_to);
         }
 
         is_first = false;
@@ -1318,7 +1325,7 @@ public:
             for (size_t i = first_data_argument; i < arguments.size(); ++i)
             {
                 const auto & col = arguments[i];
-                executeForArgument(key_cols, col.type.get(), col.column.get(), vec_to, is_first_argument, input_rows_count);
+                executeForArgument(key_cols, col.type.get(), col.column.get(), vec_to, is_first_argument);
             }
         }
 
diff --git a/src/Functions/FunctionsMultiStringFuzzySearch.h b/src/Functions/FunctionsMultiStringFuzzySearch.h
index 8346380c35d..a92a6570279 100644
--- a/src/Functions/FunctionsMultiStringFuzzySearch.h
+++ b/src/Functions/FunctionsMultiStringFuzzySearch.h
@@ -71,7 +71,7 @@ public:
         return Impl::getReturnType();
     }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
     {
         const ColumnPtr & haystack_ptr = arguments[0].column;
         const ColumnPtr & edit_distance_ptr = arguments[1].column;
@@ -114,16 +114,14 @@ public:
                 col_needles_const->getValue<Array>(),
                 vec_res, offsets_res,
                 edit_distance,
-                allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps,
-                input_rows_count);
+                allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps);
         else
             Impl::vectorVector(
                 col_haystack_vector->getChars(), col_haystack_vector->getOffsets(),
                 col_needles_vector->getData(), col_needles_vector->getOffsets(),
                 vec_res, offsets_res,
                 edit_distance,
-                allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps,
-                input_rows_count);
+                allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps);
 
         // the combination of const haystack + const needle is not implemented because
         // useDefaultImplementationForConstants() == true makes upper layers convert both to
diff --git a/src/Functions/FunctionsMultiStringSearch.h b/src/Functions/FunctionsMultiStringSearch.h
index 6bcc8581a38..03db2651fd0 100644
--- a/src/Functions/FunctionsMultiStringSearch.h
+++ b/src/Functions/FunctionsMultiStringSearch.h
@@ -81,7 +81,7 @@ public:
         return Impl::getReturnType();
     }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
     {
         const ColumnPtr & haystack_ptr = arguments[0].column;
         const ColumnPtr & needles_ptr = arguments[1].column;
@@ -110,15 +110,13 @@ public:
                 col_haystack_vector->getChars(), col_haystack_vector->getOffsets(),
                 col_needles_const->getValue<Array>(),
                 vec_res, offsets_res,
-                allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps,
-                input_rows_count);
+                allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps);
         else
             Impl::vectorVector(
                 col_haystack_vector->getChars(), col_haystack_vector->getOffsets(),
                 col_needles_vector->getData(), col_needles_vector->getOffsets(),
                 vec_res, offsets_res,
-                allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps,
-                input_rows_count);
+                allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps);
 
         // the combination of const haystack + const needle is not implemented because
         // useDefaultImplementationForConstants() == true makes upper layers convert both to
diff --git a/src/Functions/FunctionsStringSearch.h b/src/Functions/FunctionsStringSearch.h
index 7ec0076e395..fba6336ebff 100644
--- a/src/Functions/FunctionsStringSearch.h
+++ b/src/Functions/FunctionsStringSearch.h
@@ -163,7 +163,7 @@ public:
         return return_type;
     }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t /*input_rows_count*/) const override
     {
         const ColumnPtr & column_haystack = (argument_order == ArgumentOrder::HaystackNeedle) ? arguments[0].column : arguments[1].column;
         const ColumnPtr & column_needle = (argument_order == ArgumentOrder::HaystackNeedle) ? arguments[1].column : arguments[0].column;
@@ -236,8 +236,7 @@ public:
                 col_needle_vector->getOffsets(),
                 column_start_pos,
                 vec_res,
-                null_map.get(),
-                input_rows_count);
+                null_map.get());
         else if (col_haystack_vector && col_needle_const)
             Impl::vectorConstant(
                 col_haystack_vector->getChars(),
@@ -245,8 +244,7 @@ public:
                 col_needle_const->getValue<String>(),
                 column_start_pos,
                 vec_res,
-                null_map.get(),
-                input_rows_count);
+                null_map.get());
         else if (col_haystack_vector_fixed && col_needle_vector)
             Impl::vectorFixedVector(
                 col_haystack_vector_fixed->getChars(),
@@ -255,16 +253,14 @@ public:
                 col_needle_vector->getOffsets(),
                 column_start_pos,
                 vec_res,
-                null_map.get(),
-                input_rows_count);
+                null_map.get());
         else if (col_haystack_vector_fixed && col_needle_const)
             Impl::vectorFixedConstant(
                 col_haystack_vector_fixed->getChars(),
                 col_haystack_vector_fixed->getN(),
                 col_needle_const->getValue<String>(),
                 vec_res,
-                null_map.get(),
-                input_rows_count);
+                null_map.get());
         else if (col_haystack_const && col_needle_vector)
             Impl::constantVector(
                 col_haystack_const->getValue<String>(),
@@ -272,8 +268,7 @@ public:
                 col_needle_vector->getOffsets(),
                 column_start_pos,
                 vec_res,
-                null_map.get(),
-                input_rows_count);
+                null_map.get());
         else
             throw Exception(
                 ErrorCodes::ILLEGAL_COLUMN,
diff --git a/src/Functions/FunctionsStringSearchToString.h b/src/Functions/FunctionsStringSearchToString.h
index c889cf062a3..978a84de472 100644
--- a/src/Functions/FunctionsStringSearchToString.h
+++ b/src/Functions/FunctionsStringSearchToString.h
@@ -60,7 +60,7 @@ public:
         return std::make_shared<DataTypeString>();
     }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
     {
         const ColumnPtr column = arguments[0].column;
         const ColumnPtr column_needle = arguments[1].column;
@@ -75,7 +75,7 @@ public:
 
             ColumnString::Chars & vec_res = col_res->getChars();
             ColumnString::Offsets & offsets_res = col_res->getOffsets();
-            Impl::vector(col->getChars(), col->getOffsets(), col_needle->getValue<String>(), vec_res, offsets_res, input_rows_count);
+            Impl::vector(col->getChars(), col->getOffsets(), col_needle->getValue<String>(), vec_res, offsets_res);
 
             return col_res;
         }
diff --git a/src/Functions/FunctionsVisitParam.h b/src/Functions/FunctionsVisitParam.h
index dbe291fcb93..5e13fbbad5c 100644
--- a/src/Functions/FunctionsVisitParam.h
+++ b/src/Functions/FunctionsVisitParam.h
@@ -93,8 +93,7 @@ struct ExtractParamImpl
         std::string needle,
         const ColumnPtr & start_pos,
         PaddedPODArray<ResultType> & res,
-        [[maybe_unused]] ColumnUInt8 * res_null,
-        size_t /*input_rows_count*/)
+        [[maybe_unused]] ColumnUInt8 * res_null)
     {
         /// `res_null` serves as an output parameter for implementing an XYZOrNull variant.
         assert(!res_null);
@@ -169,12 +168,11 @@ struct ExtractParamToStringImpl
 {
     static void vector(const ColumnString::Chars & haystack_data, const ColumnString::Offsets & haystack_offsets,
                        std::string needle,
-                       ColumnString::Chars & res_data, ColumnString::Offsets & res_offsets,
-                       size_t input_rows_count)
+                       ColumnString::Chars & res_data, ColumnString::Offsets & res_offsets)
     {
         /// Constant 5 is taken from a function that performs a similar task FunctionsStringSearch.h::ExtractImpl
-        res_data.reserve(input_rows_count / 5);
-        res_offsets.resize(input_rows_count);
+        res_data.reserve(haystack_data.size() / 5);
+        res_offsets.resize(haystack_offsets.size());
 
         /// We are looking for a parameter simply as a substring of the form "name"
         needle = "\"" + needle + "\":";
diff --git a/src/Functions/HasTokenImpl.h b/src/Functions/HasTokenImpl.h
index 4943bf708c5..a4ff49859cc 100644
--- a/src/Functions/HasTokenImpl.h
+++ b/src/Functions/HasTokenImpl.h
@@ -35,13 +35,12 @@ struct HasTokenImpl
         const std::string & pattern,
         const ColumnPtr & start_pos,
         PaddedPODArray<UInt8> & res,
-        ColumnUInt8 * res_null,
-        size_t input_rows_count)
+        ColumnUInt8 * res_null)
     {
         if (start_pos != nullptr)
             throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Function '{}' does not support start_pos argument", name);
 
-        if (input_rows_count == 0)
+        if (haystack_offsets.empty())
             return;
 
         const UInt8 * const begin = haystack_data.data();
diff --git a/src/Functions/MatchImpl.h b/src/Functions/MatchImpl.h
index dd1dec9bdff..55b2fee5400 100644
--- a/src/Functions/MatchImpl.h
+++ b/src/Functions/MatchImpl.h
@@ -127,13 +127,14 @@ struct MatchImpl
         const String & needle,
         [[maybe_unused]] const ColumnPtr & start_pos_,
         PaddedPODArray<UInt8> & res,
-        [[maybe_unused]] ColumnUInt8 * res_null,
-        size_t input_rows_count)
+        [[maybe_unused]] ColumnUInt8 * res_null)
     {
         /// `res_null` serves as an output parameter for implementing an XYZOrNull variant.
         assert(!res_null);
 
-        assert(input_rows_count == res.size());
+        const size_t haystack_size = haystack_offsets.size();
+
+        assert(haystack_size == res.size());
         assert(start_pos_ == nullptr);
 
         if (haystack_offsets.empty())
@@ -201,11 +202,11 @@ struct MatchImpl
         if (required_substring.empty())
         {
             if (!regexp.getRE2()) /// An empty regexp. Always matches.
-                memset(res.data(), !negate, input_rows_count * sizeof(res[0]));
+                memset(res.data(), !negate, haystack_size * sizeof(res[0]));
             else
             {
                 size_t prev_offset = 0;
-                for (size_t i = 0; i < input_rows_count; ++i)
+                for (size_t i = 0; i < haystack_size; ++i)
                 {
                     const bool match = regexp.getRE2()->Match(
                             {reinterpret_cast<const char *>(&haystack_data[prev_offset]), haystack_offsets[i] - prev_offset - 1},
@@ -290,13 +291,14 @@ struct MatchImpl
         size_t N,
         const String & needle,
         PaddedPODArray<UInt8> & res,
-        [[maybe_unused]] ColumnUInt8 * res_null,
-        size_t input_rows_count)
+        [[maybe_unused]] ColumnUInt8 * res_null)
     {
         /// `res_null` serves as an output parameter for implementing an XYZOrNull variant.
         assert(!res_null);
 
-        assert(input_rows_count == res.size());
+        const size_t haystack_size = haystack.size() / N;
+
+        assert(haystack_size == res.size());
 
         if (haystack.empty())
             return;
@@ -368,11 +370,11 @@ struct MatchImpl
         if (required_substring.empty())
         {
             if (!regexp.getRE2()) /// An empty regexp. Always matches.
-                memset(res.data(), !negate, input_rows_count * sizeof(res[0]));
+                memset(res.data(), !negate, haystack_size * sizeof(res[0]));
             else
             {
                 size_t offset = 0;
-                for (size_t i = 0; i < input_rows_count; ++i)
+                for (size_t i = 0; i < haystack_size; ++i)
                 {
                     const bool match = regexp.getRE2()->Match(
                             {reinterpret_cast<const char *>(&haystack[offset]), N},
@@ -462,14 +464,15 @@ struct MatchImpl
         const ColumnString::Offsets & needle_offset,
         [[maybe_unused]] const ColumnPtr & start_pos_,
         PaddedPODArray<UInt8> & res,
-        [[maybe_unused]] ColumnUInt8 * res_null,
-        size_t input_rows_count)
+        [[maybe_unused]] ColumnUInt8 * res_null)
     {
         /// `res_null` serves as an output parameter for implementing an XYZOrNull variant.
         assert(!res_null);
 
-        assert(input_rows_count == needle_offset.size());
-        assert(input_rows_count == res.size());
+        const size_t haystack_size = haystack_offsets.size();
+
+        assert(haystack_size == needle_offset.size());
+        assert(haystack_size == res.size());
         assert(start_pos_ == nullptr);
 
         if (haystack_offsets.empty())
@@ -485,7 +488,7 @@ struct MatchImpl
         Regexps::LocalCacheTable cache;
         Regexps::RegexpPtr regexp;
 
-        for (size_t i = 0; i < input_rows_count; ++i)
+        for (size_t i = 0; i < haystack_size; ++i)
         {
             const auto * const cur_haystack_data = &haystack_data[prev_haystack_offset];
             const size_t cur_haystack_length = haystack_offsets[i] - prev_haystack_offset - 1;
@@ -570,14 +573,15 @@ struct MatchImpl
         const ColumnString::Offsets & needle_offset,
         [[maybe_unused]] const ColumnPtr & start_pos_,
         PaddedPODArray<UInt8> & res,
-        [[maybe_unused]] ColumnUInt8 * res_null,
-        size_t input_rows_count)
+        [[maybe_unused]] ColumnUInt8 * res_null)
     {
         /// `res_null` serves as an output parameter for implementing an XYZOrNull variant.
         assert(!res_null);
 
-        assert(input_rows_count == needle_offset.size());
-        assert(input_rows_count == res.size());
+        const size_t haystack_size = haystack.size()/N;
+
+        assert(haystack_size == needle_offset.size());
+        assert(haystack_size == res.size());
         assert(start_pos_ == nullptr);
 
         if (haystack.empty())
@@ -593,7 +597,7 @@ struct MatchImpl
         Regexps::LocalCacheTable cache;
         Regexps::RegexpPtr regexp;
 
-        for (size_t i = 0; i < input_rows_count; ++i)
+        for (size_t i = 0; i < haystack_size; ++i)
         {
             const auto * const cur_haystack_data = &haystack[prev_haystack_offset];
             const size_t cur_haystack_length = N;
diff --git a/src/Functions/MultiMatchAllIndicesImpl.h b/src/Functions/MultiMatchAllIndicesImpl.h
index 3aeac808880..3e9c8fba215 100644
--- a/src/Functions/MultiMatchAllIndicesImpl.h
+++ b/src/Functions/MultiMatchAllIndicesImpl.h
@@ -52,10 +52,9 @@ struct MultiMatchAllIndicesImpl
         bool allow_hyperscan,
         size_t max_hyperscan_regexp_length,
         size_t max_hyperscan_regexp_total_length,
-        bool reject_expensive_hyperscan_regexps,
-        size_t input_rows_count)
+        bool reject_expensive_hyperscan_regexps)
     {
-        vectorConstant(haystack_data, haystack_offsets, needles_arr, res, offsets, std::nullopt, allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps, input_rows_count);
+        vectorConstant(haystack_data, haystack_offsets, needles_arr, res, offsets, std::nullopt, allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps);
     }
 
     static void vectorConstant(
@@ -68,8 +67,7 @@ struct MultiMatchAllIndicesImpl
         bool allow_hyperscan,
         size_t max_hyperscan_regexp_length,
         size_t max_hyperscan_regexp_total_length,
-        bool reject_expensive_hyperscan_regexps,
-        size_t input_rows_count)
+        bool reject_expensive_hyperscan_regexps)
     {
         if (!allow_hyperscan)
             throw Exception(ErrorCodes::FUNCTION_NOT_ALLOWED, "Hyperscan functions are disabled, because setting 'allow_hyperscan' is set to 0");
@@ -89,7 +87,7 @@ struct MultiMatchAllIndicesImpl
                     throw Exception(ErrorCodes::HYPERSCAN_CANNOT_SCAN_TEXT, "Regular expression evaluation in vectorscan will be too slow. To ignore this error, disable setting 'reject_expensive_hyperscan_regexps'.");
         }
 
-        offsets.resize(input_rows_count);
+        offsets.resize(haystack_offsets.size());
 
         if (needles_arr.empty())
         {
@@ -116,8 +114,9 @@ struct MultiMatchAllIndicesImpl
             static_cast<PaddedPODArray<ResultType>*>(context)->push_back(id);
             return 0;
         };
+        const size_t haystack_offsets_size = haystack_offsets.size();
         UInt64 offset = 0;
-        for (size_t i = 0; i < input_rows_count; ++i)
+        for (size_t i = 0; i < haystack_offsets_size; ++i)
         {
             UInt64 length = haystack_offsets[i] - offset - 1;
             /// vectorscan restriction.
@@ -147,7 +146,6 @@ struct MultiMatchAllIndicesImpl
         (void)max_hyperscan_regexp_length;
         (void)max_hyperscan_regexp_total_length;
         (void)reject_expensive_hyperscan_regexps;
-        (void)input_rows_count;
         throw Exception(ErrorCodes::NOT_IMPLEMENTED, "multi-search all indices is not implemented when vectorscan is off");
 #endif // USE_VECTORSCAN
     }
@@ -162,10 +160,9 @@ struct MultiMatchAllIndicesImpl
         bool allow_hyperscan,
         size_t max_hyperscan_regexp_length,
         size_t max_hyperscan_regexp_total_length,
-        bool reject_expensive_hyperscan_regexps,
-        size_t input_rows_count)
+        bool reject_expensive_hyperscan_regexps)
     {
-        vectorVector(haystack_data, haystack_offsets, needles_data, needles_offsets, res, offsets, std::nullopt, allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps, input_rows_count);
+        vectorVector(haystack_data, haystack_offsets, needles_data, needles_offsets, res, offsets, std::nullopt, allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps);
     }
 
     static void vectorVector(
@@ -179,13 +176,12 @@ struct MultiMatchAllIndicesImpl
         bool allow_hyperscan,
         size_t max_hyperscan_regexp_length,
         size_t max_hyperscan_regexp_total_length,
-        bool reject_expensive_hyperscan_regexps,
-        size_t input_rows_count)
+        bool reject_expensive_hyperscan_regexps)
     {
         if (!allow_hyperscan)
             throw Exception(ErrorCodes::FUNCTION_NOT_ALLOWED, "Hyperscan functions are disabled, because setting 'allow_hyperscan' is set to 0");
 #if USE_VECTORSCAN
-        offsets.resize(input_rows_count);
+        offsets.resize(haystack_offsets.size());
         size_t prev_haystack_offset = 0;
         size_t prev_needles_offset = 0;
 
@@ -193,7 +189,7 @@ struct MultiMatchAllIndicesImpl
 
         std::vector<std::string_view> needles;
 
-        for (size_t i = 0; i < input_rows_count; ++i)
+        for (size_t i = 0; i < haystack_offsets.size(); ++i)
         {
             needles.reserve(needles_offsets[i] - prev_needles_offset);
 
@@ -275,7 +271,6 @@ struct MultiMatchAllIndicesImpl
         (void)max_hyperscan_regexp_length;
         (void)max_hyperscan_regexp_total_length;
         (void)reject_expensive_hyperscan_regexps;
-        (void)input_rows_count;
         throw Exception(ErrorCodes::NOT_IMPLEMENTED, "multi-search all indices is not implemented when vectorscan is off");
 #endif // USE_VECTORSCAN
     }
diff --git a/src/Functions/MultiMatchAnyImpl.h b/src/Functions/MultiMatchAnyImpl.h
index 03b17321eea..20b2150048b 100644
--- a/src/Functions/MultiMatchAnyImpl.h
+++ b/src/Functions/MultiMatchAnyImpl.h
@@ -66,10 +66,9 @@ struct MultiMatchAnyImpl
         bool allow_hyperscan,
         size_t max_hyperscan_regexp_length,
         size_t max_hyperscan_regexp_total_length,
-        bool reject_expensive_hyperscan_regexps,
-        size_t input_rows_count)
+        bool reject_expensive_hyperscan_regexps)
     {
-        vectorConstant(haystack_data, haystack_offsets, needles_arr, res, offsets, std::nullopt, allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps, input_rows_count);
+        vectorConstant(haystack_data, haystack_offsets, needles_arr, res, offsets, std::nullopt, allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps);
     }
 
     static void vectorConstant(
@@ -82,8 +81,7 @@ struct MultiMatchAnyImpl
         bool allow_hyperscan,
         size_t max_hyperscan_regexp_length,
         size_t max_hyperscan_regexp_total_length,
-        bool reject_expensive_hyperscan_regexps,
-        size_t input_rows_count)
+        bool reject_expensive_hyperscan_regexps)
     {
         if (!allow_hyperscan)
             throw Exception(ErrorCodes::FUNCTION_NOT_ALLOWED, "Hyperscan functions are disabled, because setting 'allow_hyperscan' is set to 0");
@@ -103,7 +101,7 @@ struct MultiMatchAnyImpl
                     throw Exception(ErrorCodes::HYPERSCAN_CANNOT_SCAN_TEXT, "Regular expression evaluation in vectorscan will be too slow. To ignore this error, disable setting 'reject_expensive_hyperscan_regexps'.");
         }
 
-        res.resize(input_rows_count);
+        res.resize(haystack_offsets.size());
 
         if (needles_arr.empty())
         {
@@ -135,8 +133,9 @@ struct MultiMatchAnyImpl
             /// Once we hit the callback, there is no need to search for others.
             return 1;
         };
+        const size_t haystack_offsets_size = haystack_offsets.size();
         UInt64 offset = 0;
-        for (size_t i = 0; i < input_rows_count; ++i)
+        for (size_t i = 0; i < haystack_offsets_size; ++i)
         {
             UInt64 length = haystack_offsets[i] - offset - 1;
             /// vectorscan restriction.
@@ -163,9 +162,9 @@ struct MultiMatchAnyImpl
         PaddedPODArray<UInt8> accum(res.size());
         memset(res.data(), 0, res.size() * sizeof(res.front()));
         memset(accum.data(), 0, accum.size());
-        for (size_t j = 0; j < input_rows_count; ++j)
+        for (size_t j = 0; j < needles.size(); ++j)
         {
-            MatchImpl<Name, MatchTraits::Syntax::Re2, MatchTraits::Case::Sensitive, MatchTraits::Result::DontNegate>::vectorConstant(haystack_data, haystack_offsets, String(needles[j].data(), needles[j].size()), nullptr, accum, nullptr, input_rows_count);
+            MatchImpl<Name, MatchTraits::Syntax::Re2, MatchTraits::Case::Sensitive, MatchTraits::Result::DontNegate>::vectorConstant(haystack_data, haystack_offsets, String(needles[j].data(), needles[j].size()), nullptr, accum, nullptr);
             for (size_t i = 0; i < res.size(); ++i)
             {
                 if constexpr (FindAny)
@@ -187,10 +186,9 @@ struct MultiMatchAnyImpl
         bool allow_hyperscan,
         size_t max_hyperscan_regexp_length,
         size_t max_hyperscan_regexp_total_length,
-        bool reject_expensive_hyperscan_regexps,
-        size_t input_rows_count)
+        bool reject_expensive_hyperscan_regexps)
     {
-        vectorVector(haystack_data, haystack_offsets, needles_data, needles_offsets, res, offsets, std::nullopt, allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps, input_rows_count);
+        vectorVector(haystack_data, haystack_offsets, needles_data, needles_offsets, res, offsets, std::nullopt, allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps);
     }
 
     static void vectorVector(
@@ -204,13 +202,12 @@ struct MultiMatchAnyImpl
         bool allow_hyperscan,
         size_t max_hyperscan_regexp_length,
         size_t max_hyperscan_regexp_total_length,
-        bool reject_expensive_hyperscan_regexps,
-        size_t input_rows_count)
+        bool reject_expensive_hyperscan_regexps)
     {
         if (!allow_hyperscan)
             throw Exception(ErrorCodes::FUNCTION_NOT_ALLOWED, "Hyperscan functions are disabled, because setting 'allow_hyperscan' is set to 0");
 
-        res.resize(input_rows_count);
+        res.resize(haystack_offsets.size());
 #if USE_VECTORSCAN
         size_t prev_haystack_offset = 0;
         size_t prev_needles_offset = 0;
@@ -219,7 +216,7 @@ struct MultiMatchAnyImpl
 
         std::vector<std::string_view> needles;
 
-        for (size_t i = 0; i < input_rows_count; ++i)
+        for (size_t i = 0; i < haystack_offsets.size(); ++i)
         {
             needles.reserve(needles_offsets[i] - prev_needles_offset);
 
@@ -309,7 +306,7 @@ struct MultiMatchAnyImpl
 
         std::vector<std::string_view> needles;
 
-        for (size_t i = 0; i < input_rows_count; ++i)
+        for (size_t i = 0; i < haystack_offsets.size(); ++i)
         {
             const auto * const cur_haystack_data = &haystack_data[prev_haystack_offset];
             const size_t cur_haystack_length = haystack_offsets[i] - prev_haystack_offset - 1;
diff --git a/src/Functions/MultiSearchFirstIndexImpl.h b/src/Functions/MultiSearchFirstIndexImpl.h
index b80d9d3a124..36a5fd514d9 100644
--- a/src/Functions/MultiSearchFirstIndexImpl.h
+++ b/src/Functions/MultiSearchFirstIndexImpl.h
@@ -33,8 +33,7 @@ struct MultiSearchFirstIndexImpl
         bool /*allow_hyperscan*/,
         size_t /*max_hyperscan_regexp_length*/,
         size_t /*max_hyperscan_regexp_total_length*/,
-        bool /*reject_expensive_hyperscan_regexps*/,
-        size_t input_rows_count)
+        bool /*reject_expensive_hyperscan_regexps*/)
     {
         // For performance of Volnitsky search, it is crucial to save only one byte for pattern number.
         if (needles_arr.size() > std::numeric_limits<UInt8>::max())
@@ -49,13 +48,14 @@ struct MultiSearchFirstIndexImpl
 
         auto searcher = Impl::createMultiSearcherInBigHaystack(needles);
 
-        res.resize(input_rows_count);
+        const size_t haystack_size = haystack_offsets.size();
+        res.resize(haystack_size);
 
         size_t iteration = 0;
         while (searcher.hasMoreToSearch())
         {
             size_t prev_haystack_offset = 0;
-            for (size_t j = 0; j < input_rows_count; ++j)
+            for (size_t j = 0; j < haystack_size; ++j)
             {
                 const auto * haystack = &haystack_data[prev_haystack_offset];
                 const auto * haystack_end = haystack + haystack_offsets[j] - prev_haystack_offset - 1;
@@ -80,10 +80,10 @@ struct MultiSearchFirstIndexImpl
         bool /*allow_hyperscan*/,
         size_t /*max_hyperscan_regexp_length*/,
         size_t /*max_hyperscan_regexp_total_length*/,
-        bool /*reject_expensive_hyperscan_regexps*/,
-        size_t input_rows_count)
+        bool /*reject_expensive_hyperscan_regexps*/)
     {
-        res.resize(input_rows_count);
+        const size_t haystack_size = haystack_offsets.size();
+        res.resize(haystack_size);
 
         size_t prev_haystack_offset = 0;
         size_t prev_needles_offset = 0;
@@ -92,12 +92,14 @@ struct MultiSearchFirstIndexImpl
 
         std::vector<std::string_view> needles;
 
-        for (size_t i = 0; i < input_rows_count; ++i)
+        for (size_t i = 0; i < haystack_size; ++i)
         {
             needles.reserve(needles_offsets[i] - prev_needles_offset);
 
             for (size_t j = prev_needles_offset; j < needles_offsets[i]; ++j)
+            {
                 needles.emplace_back(needles_data_string.getDataAt(j).toView());
+            }
 
             auto searcher = Impl::createMultiSearcherInBigHaystack(needles); // sub-optimal
 
@@ -108,11 +110,15 @@ struct MultiSearchFirstIndexImpl
             while (searcher.hasMoreToSearch())
             {
                 if (iteration == 0 || res[i] == 0)
+                {
                     res[i] = searcher.searchOneFirstIndex(haystack, haystack_end);
+                }
                 ++iteration;
             }
             if (iteration == 0)
+            {
                 res[i] = 0;
+            }
 
             prev_haystack_offset = haystack_offsets[i];
             prev_needles_offset = needles_offsets[i];
diff --git a/src/Functions/MultiSearchFirstPositionImpl.h b/src/Functions/MultiSearchFirstPositionImpl.h
index cd4e585e99b..ccdd82a0ee5 100644
--- a/src/Functions/MultiSearchFirstPositionImpl.h
+++ b/src/Functions/MultiSearchFirstPositionImpl.h
@@ -33,8 +33,7 @@ struct MultiSearchFirstPositionImpl
         bool /*allow_hyperscan*/,
         size_t /*max_hyperscan_regexp_length*/,
         size_t /*max_hyperscan_regexp_total_length*/,
-        bool /*reject_expensive_hyperscan_regexps*/,
-        size_t input_rows_count)
+        bool /*reject_expensive_hyperscan_regexps*/)
     {
         // For performance of Volnitsky search, it is crucial to save only one byte for pattern number.
         if (needles_arr.size() > std::numeric_limits<UInt8>::max())
@@ -53,13 +52,14 @@ struct MultiSearchFirstPositionImpl
         };
         auto searcher = Impl::createMultiSearcherInBigHaystack(needles);
 
-        res.resize(input_rows_count);
+        const size_t haystack_size = haystack_offsets.size();
+        res.resize(haystack_size);
 
         size_t iteration = 0;
         while (searcher.hasMoreToSearch())
         {
             size_t prev_haystack_offset = 0;
-            for (size_t j = 0; j < input_rows_count; ++j)
+            for (size_t j = 0; j < haystack_size; ++j)
             {
                 const auto * haystack = &haystack_data[prev_haystack_offset];
                 const auto * haystack_end = haystack + haystack_offsets[j] - prev_haystack_offset - 1;
@@ -89,10 +89,10 @@ struct MultiSearchFirstPositionImpl
         bool /*allow_hyperscan*/,
         size_t /*max_hyperscan_regexp_length*/,
         size_t /*max_hyperscan_regexp_total_length*/,
-        bool /*reject_expensive_hyperscan_regexps*/,
-        size_t input_rows_count)
+        bool /*reject_expensive_hyperscan_regexps*/)
     {
-        res.resize(input_rows_count);
+        const size_t haystack_size = haystack_offsets.size();
+        res.resize(haystack_size);
 
         size_t prev_haystack_offset = 0;
         size_t prev_needles_offset = 0;
@@ -106,12 +106,14 @@ struct MultiSearchFirstPositionImpl
             return 1 + Impl::countChars(reinterpret_cast<const char *>(start), reinterpret_cast<const char *>(end));
         };
 
-        for (size_t i = 0; i < input_rows_count; ++i)
+        for (size_t i = 0; i < haystack_size; ++i)
         {
             needles.reserve(needles_offsets[i] - prev_needles_offset);
 
             for (size_t j = prev_needles_offset; j < needles_offsets[i]; ++j)
+            {
                 needles.emplace_back(needles_data_string.getDataAt(j).toView());
+            }
 
             auto searcher = Impl::createMultiSearcherInBigHaystack(needles); // sub-optimal
 
@@ -136,7 +138,9 @@ struct MultiSearchFirstPositionImpl
                 ++iteration;
             }
             if (iteration == 0)
+            {
                 res[i] = 0;
+            }
 
             prev_haystack_offset = haystack_offsets[i];
             prev_needles_offset = needles_offsets[i];
diff --git a/src/Functions/MultiSearchImpl.h b/src/Functions/MultiSearchImpl.h
index 909425f5a93..467cc96a95f 100644
--- a/src/Functions/MultiSearchImpl.h
+++ b/src/Functions/MultiSearchImpl.h
@@ -33,8 +33,7 @@ struct MultiSearchImpl
         bool /*allow_hyperscan*/,
         size_t /*max_hyperscan_regexp_length*/,
         size_t /*max_hyperscan_regexp_total_length*/,
-        bool /*reject_expensive_hyperscan_regexps*/,
-        size_t input_rows_count)
+        bool /*reject_expensive_hyperscan_regexps*/)
     {
         // For performance of Volnitsky search, it is crucial to save only one byte for pattern number.
         if (needles_arr.size() > std::numeric_limits<UInt8>::max())
@@ -49,13 +48,14 @@ struct MultiSearchImpl
 
         auto searcher = Impl::createMultiSearcherInBigHaystack(needles);
 
-        res.resize(input_rows_count);
+        const size_t haystack_size = haystack_offsets.size();
+        res.resize(haystack_size);
 
         size_t iteration = 0;
         while (searcher.hasMoreToSearch())
         {
             size_t prev_haystack_offset = 0;
-            for (size_t j = 0; j < input_rows_count; ++j)
+            for (size_t j = 0; j < haystack_size; ++j)
             {
                 const auto * haystack = &haystack_data[prev_haystack_offset];
                 const auto * haystack_end = haystack + haystack_offsets[j] - prev_haystack_offset - 1;
@@ -79,10 +79,10 @@ struct MultiSearchImpl
         bool /*allow_hyperscan*/,
         size_t /*max_hyperscan_regexp_length*/,
         size_t /*max_hyperscan_regexp_total_length*/,
-        bool /*reject_expensive_hyperscan_regexps*/,
-        size_t input_rows_count)
+        bool /*reject_expensive_hyperscan_regexps*/)
     {
-        res.resize(input_rows_count);
+        const size_t haystack_size = haystack_offsets.size();
+        res.resize(haystack_size);
 
         size_t prev_haystack_offset = 0;
         size_t prev_needles_offset = 0;
@@ -91,12 +91,14 @@ struct MultiSearchImpl
 
         std::vector<std::string_view> needles;
 
-        for (size_t i = 0; i < input_rows_count; ++i)
+        for (size_t i = 0; i < haystack_size; ++i)
         {
             needles.reserve(needles_offsets[i] - prev_needles_offset);
 
             for (size_t j = prev_needles_offset; j < needles_offsets[i]; ++j)
+            {
                 needles.emplace_back(needles_data_string.getDataAt(j).toView());
+            }
 
             const auto * const haystack = &haystack_data[prev_haystack_offset];
             const size_t haystack_length = haystack_offsets[i] - prev_haystack_offset - 1;
diff --git a/src/Functions/PositionImpl.h b/src/Functions/PositionImpl.h
index e525b5fab57..eeb9d8b6a59 100644
--- a/src/Functions/PositionImpl.h
+++ b/src/Functions/PositionImpl.h
@@ -193,8 +193,7 @@ struct PositionImpl
         const std::string & needle,
         const ColumnPtr & start_pos,
         PaddedPODArray<UInt64> & res,
-        [[maybe_unused]] ColumnUInt8 * res_null,
-        size_t input_rows_count)
+        [[maybe_unused]] ColumnUInt8 * res_null)
     {
         /// `res_null` serves as an output parameter for implementing an XYZOrNull variant.
         assert(!res_null);
@@ -215,12 +214,13 @@ struct PositionImpl
             }
 
             ColumnString::Offset prev_offset = 0;
+            size_t rows = haystack_offsets.size();
 
             if (const ColumnConst * start_pos_const = typeid_cast<const ColumnConst *>(&*start_pos))
             {
                 /// Needle is empty and start_pos is constant
                 UInt64 start = std::max(start_pos_const->getUInt(0), static_cast<UInt64>(1));
-                for (size_t i = 0; i < input_rows_count; ++i)
+                for (size_t i = 0; i < rows; ++i)
                 {
                     size_t haystack_size = Impl::countChars(
                         reinterpret_cast<const char *>(pos), reinterpret_cast<const char *>(pos + haystack_offsets[i] - prev_offset - 1));
@@ -234,7 +234,7 @@ struct PositionImpl
             else
             {
                 /// Needle is empty and start_pos is not constant
-                for (size_t i = 0; i < input_rows_count; ++i)
+                for (size_t i = 0; i < rows; ++i)
                 {
                     size_t haystack_size = Impl::countChars(
                         reinterpret_cast<const char *>(pos), reinterpret_cast<const char *>(pos + haystack_offsets[i] - prev_offset - 1));
@@ -359,8 +359,7 @@ struct PositionImpl
         const ColumnString::Offsets & needle_offsets,
         const ColumnPtr & start_pos,
         PaddedPODArray<UInt64> & res,
-        [[maybe_unused]] ColumnUInt8 * res_null,
-        size_t input_rows_count)
+        [[maybe_unused]] ColumnUInt8 * res_null)
     {
         /// `res_null` serves as an output parameter for implementing an XYZOrNull variant.
         assert(!res_null);
@@ -368,7 +367,9 @@ struct PositionImpl
         ColumnString::Offset prev_haystack_offset = 0;
         ColumnString::Offset prev_needle_offset = 0;
 
-        for (size_t i = 0; i < input_rows_count; ++i)
+        size_t size = haystack_offsets.size();
+
+        for (size_t i = 0; i < size; ++i)
         {
             size_t needle_size = needle_offsets[i] - prev_needle_offset - 1;
             size_t haystack_size = haystack_offsets[i] - prev_haystack_offset - 1;
@@ -422,8 +423,7 @@ struct PositionImpl
         const ColumnString::Offsets & needle_offsets,
         const ColumnPtr & start_pos,
         PaddedPODArray<UInt64> & res,
-        [[maybe_unused]] ColumnUInt8 * res_null,
-        size_t input_rows_count)
+        [[maybe_unused]] ColumnUInt8 * res_null)
     {
         /// `res_null` serves as an output parameter for implementing an XYZOrNull variant.
         assert(!res_null);
@@ -431,7 +431,9 @@ struct PositionImpl
         /// NOTE You could use haystack indexing. But this is a rare case.
         ColumnString::Offset prev_needle_offset = 0;
 
-        for (size_t i = 0; i < input_rows_count; ++i)
+        size_t size = needle_offsets.size();
+
+        for (size_t i = 0; i < size; ++i)
         {
             size_t needle_size = needle_offsets[i] - prev_needle_offset - 1;
 
diff --git a/src/Functions/URL/extractURLParameter.cpp b/src/Functions/URL/extractURLParameter.cpp
index 590c2779d9c..f75875e0200 100644
--- a/src/Functions/URL/extractURLParameter.cpp
+++ b/src/Functions/URL/extractURLParameter.cpp
@@ -10,11 +10,10 @@ struct ExtractURLParameterImpl
     static void vector(const ColumnString::Chars & data,
         const ColumnString::Offsets & offsets,
         std::string pattern,
-        ColumnString::Chars & res_data, ColumnString::Offsets & res_offsets,
-        size_t input_rows_count)
+        ColumnString::Chars & res_data, ColumnString::Offsets & res_offsets)
     {
         res_data.reserve(data.size() / 5);
-        res_offsets.resize(input_rows_count);
+        res_offsets.resize(offsets.size());
 
         pattern += '=';
         const char * param_str = pattern.c_str();
@@ -23,7 +22,7 @@ struct ExtractURLParameterImpl
         ColumnString::Offset prev_offset = 0;
         ColumnString::Offset res_offset = 0;
 
-        for (size_t i = 0; i < input_rows_count; ++i)
+        for (size_t i = 0; i < offsets.size(); ++i)
         {
             ColumnString::Offset cur_offset = offsets[i];
 
diff --git a/src/Functions/extract.cpp b/src/Functions/extract.cpp
index c78ee9898b7..6bbdaff0e3f 100644
--- a/src/Functions/extract.cpp
+++ b/src/Functions/extract.cpp
@@ -16,11 +16,10 @@ struct ExtractImpl
         const ColumnString::Offsets & offsets,
         const std::string & pattern,
         ColumnString::Chars & res_data,
-        ColumnString::Offsets & res_offsets,
-        size_t input_rows_count)
+        ColumnString::Offsets & res_offsets)
     {
         res_data.reserve(data.size() / 5);
-        res_offsets.resize(input_rows_count);
+        res_offsets.resize(offsets.size());
 
         const OptimizedRegularExpression regexp = Regexps::createRegexp<false, false, false>(pattern);
 
@@ -30,7 +29,7 @@ struct ExtractImpl
         size_t prev_offset = 0;
         size_t res_offset = 0;
 
-        for (size_t i = 0; i < input_rows_count; ++i)
+        for (size_t i = 0; i < offsets.size(); ++i)
         {
             size_t cur_offset = offsets[i];
 

From 5b9427dc566b1e76a82be9f650b9dbc386567fc4 Mon Sep 17 00:00:00 2001
From: Pablo Marcos <pablo.marcos.oltra@clickhouse.com>
Date: Fri, 26 Jul 2024 13:19:57 +0000
Subject: [PATCH 0958/1488] Fix test flakiness

- Increase from 10s to 20s the max_age. Removes flakiness due to
  the files in minio still present in following execution, even
  if the cleanup stage of the test is clearly removing them.
- Use a busy-loop to check conditions directly, removing the
  unnecessary sleep.
---
 .../integration/test_storage_s3_queue/test.py | 28 +++++++++++--------
 1 file changed, 17 insertions(+), 11 deletions(-)

diff --git a/tests/integration/test_storage_s3_queue/test.py b/tests/integration/test_storage_s3_queue/test.py
index 2e339a9b5c9..963089c3777 100644
--- a/tests/integration/test_storage_s3_queue/test.py
+++ b/tests/integration/test_storage_s3_queue/test.py
@@ -827,7 +827,7 @@ def test_max_set_age(started_cluster):
     dst_table_name = f"{table_name}_dst"
     keeper_path = f"/clickhouse/test_{table_name}"
     files_path = f"{table_name}_data"
-    max_age = 10
+    max_age = 20
     files_to_generate = 10
 
     create_table(
@@ -858,12 +858,17 @@ def test_max_set_age(started_cluster):
     def get_count():
         return int(node.query(f"SELECT count() FROM {dst_table_name}"))
 
-    def wait_for_condition(check_function, max_wait_time=30):
+    import os
+
+    def wait_for_condition(check_function, max_wait_time=1.5 * max_age):
+        logging.debug(f"{os.getenv('PYTEST_XDIST_WORKER')} - PMO: Waiting for condition")
         before = time.time()
         while time.time() - before < max_wait_time:
             if check_function():
+                logging.debug(f"{os.getenv('PYTEST_XDIST_WORKER')} - PMO: Condition verified")
                 return
-            time.sleep(0.1)
+            time.sleep(0.25)
+        logging.debug(f"{os.getenv('PYTEST_XDIST_WORKER')} - PMO: Oh oh... Condition not verified")
         assert False
 
     wait_for_condition(lambda: get_count() == expected_rows)
@@ -871,8 +876,6 @@ def test_max_set_age(started_cluster):
         node.query(f"SELECT uniq(_path) from {dst_table_name}")
     )
 
-    time.sleep(max_age + max_age / 2)
-
     expected_rows *= 2
     wait_for_condition(lambda: get_count() == expected_rows)
     assert files_to_generate == int(
@@ -886,8 +889,12 @@ def test_max_set_age(started_cluster):
         ).splitlines()
     ]
     assert files_to_generate == len(paths_count)
-    for path_count in paths_count:
-        assert 2 == path_count
+    try:
+        for path_count in paths_count:
+            assert 2 == path_count
+    finally:
+        logging.debug(f"{os.getenv('PYTEST_XDIST_WORKER')} - PMO: Got path count {path_count} for paths_count {paths_count}")
+        logging.debug(f"{os.getenv('PYTEST_XDIST_WORKER')} - PMO: Got paths: {node.query('SELECT *, _path from ' + dst_table_name + ' FORMAT JSONEachRow')}")
 
     def get_object_storage_failures():
         return int(
@@ -897,6 +904,7 @@ def test_max_set_age(started_cluster):
         )
 
     failed_count = get_object_storage_failures()
+    logging.debug(f"{os.getenv('PYTEST_XDIST_WORKER')} - PMO: Got failed count {failed_count}")
 
     values = [
         ["failed", 1, 1],
@@ -906,7 +914,7 @@ def test_max_set_age(started_cluster):
     ).encode()
 
     # use a different filename for each test to allow running a bunch of them sequentially with --count
-    file_with_error = f"fff_{uuid4().hex}.csv"
+    file_with_error = f"max_set_age_fail_{uuid4().hex[:8]}.csv"
     put_s3_file_content(started_cluster, f"{files_path}/{file_with_error}", values_csv)
 
     wait_for_condition(lambda: failed_count + 1 <= get_object_storage_failures())
@@ -922,9 +930,7 @@ def test_max_set_age(started_cluster):
         )
     )
 
-    time.sleep(max_age + max_age / 2)
-
-    assert failed_count + 2 <= get_object_storage_failures()
+    wait_for_condition(lambda: failed_count + 2 <= get_object_storage_failures())
 
     node.query("SYSTEM FLUSH LOGS")
     assert "Cannot parse input" in node.query(

From e6c708124553b67eb4f18b6e39c96a24ad1c2dbb Mon Sep 17 00:00:00 2001
From: maxvostrikov <max.vostrikov@clickhouse.com>
Date: Mon, 29 Jul 2024 12:44:33 +0200
Subject: [PATCH 0959/1488] squash! added somme tests in relation with
 https://github.com/ClickHouse/ClickHouse/pull/54881 with new behaviour when
 enable_named_columns_in_function_tuple=1 (default value)

---
 .../0_stateless/00309_formats.reference       | Bin 20353 -> 18666 bytes
 tests/queries/0_stateless/00309_formats.sql   |   5 +++++
 2 files changed, 5 insertions(+)

diff --git a/tests/queries/0_stateless/00309_formats.reference b/tests/queries/0_stateless/00309_formats.reference
index cab311692be229716b58af39079275d3942b01cc..f3ea45520bb50fb936caf6724e9fedf3cdd00b75 100644
GIT binary patch
delta 3052
zcmeIy$4}c(00wZIgtksdZ-w^I3JD}eL2?j6{R4XG1$t-?y;Q0c4wb5?2YL>Bj|8*#
z-Xp>6y?59U_TGDMNaN%a?SDX3#j^bK`<|bk?ZfAlN^UDiN<bJ6jY24SuCXUb!%+R!
z(?lWq$P}8rBt@#-Sf0!i4|+jw=mUMBAM}R-Fc1d8U>E{JVHgaD5e8+19DxExLIy^`
zXcz-yVH}Ky2`~{R!DN^MQ}xOSA`L-0%z&9N3ueO{m<#h@J}iKRum~2z5?IQq+n-P(
zP{A_D!g5#vD`6F^hBdGj*1>w%02^VGPHumqnY-wY76U{pY=iBv19rkL*bRGNFYJT;
zPz?v*AoeGQ5Ddc+I10z$IGlhQI0>iVG@OC6a1PGH1;h6f#3F(vxC~d|DqMr>a070_
zEw~ML;4a*Q`|v>j|8;{y1V``~p1@Oh2G8LIyo6Wq8s5NLcn9yHg%&qjk~HD*O`kk{
zMhS#wBF4@26;n)ok$HM~`}q3#2LuKMhlGZON5~bCOjLACY+QUoVp4KSYFc_mW>$7i
zZeD&tVNr2OsZv$OmRD3(RoB$k)i*RYHMg|3wRd!Ob@%l4^{WR4hlWQ+$Hpf#lT*_(
zvvczci%ZKZt842Un_JsEyL<ZwheyXJr)TFEmsi&}w|Dmf?L&1j3gH`4#3+R{X>2uP
zs~KC(*lPbltFhlLRQ$(oSyIc$t;|{F>ph;XVgHfpC$+5H%8ajnp%+Y<&DY2OORU_k
z(V7&oZmv$Gj7LgIiK)l0pFiA9KYx&VyyL(B=HYJsi3@C{mioxSQew%acGgGsx>HI?
tDE$HfH|yd|adXt)D_oo(mI%Z&<wVo8BQ16miz!<%?O^j~Y-3_4_zmv{|2Y5v

literal 20353
zcmeHP3wIP%5{6aK9zDL_&p18;axq=E-!Ke<i0?P}N)Q7X6ip_W3@R%6%l7O~+P|=0
zRo6hOl6Z`+M|N!w(_i=PTYbO&s&ea2b&tJW?SKFN*FT@{>He<Wy7Ldkn-p(W+@rWx
z@fO8f6_Y9M{>M!>@7a6Ht;zJ#Jk0d+*gt^JbdYKDu4%*X9d$a^ZOY<y#cBH;dVHti
zU5a-r-lKT0;(dzuD?Xt3pyES{4=X;R_^9GzijONkq4=cYjN(&@eZ{90pHX~P@j1m=
z#eItV6`xl;pcoVnD!!mNr<f{|VpPl&bHzfjRIC(h#YXXv;$g*s;t|E6;!(vH6^|(%
zSA0qFgyPGJCly~&d{yx^#n%<zP<&JIEycGL-%)&5@jb=&6+cisrFdHLL&c92KUVxi
z@l(Zl#RbJPik~T-Rs39WQSqGOdBr8gk>Ul#FBHF2yr}q<;@67HiYtm^#Z|>iir*-%
zDXuF{6fY}&tN5Mb_lj2(e^C5U@h8Qria#s<qKG-5+v_yg)y2WCP6NyYf5SM~eSM*H
z+wHwO?!4>nd+xpO{s$g>=;22meeCfko}78A|MWA@J~z8>|MLgJ!58LIidkNiRoxsq
zJUB8u`r@(UFP(V#<SVbf_WB!dzV-Gy@4ol`2d7Sd_|eCod^*2y=CiY(FP=NUG`jG`
zmlwbKdU<8Mdg+_B^~vRLzx)2m4?q5N_2*w8dH3%xRA9RH1%(59c8_;2-L$y2wtIcg
z<npEE(caUqjVEP(^NGdDsCR5_ZSl&!wq^g`4<>6D$6wrgio0oJnBji=r;A-&v0K--
z%ZqMZ-|En<>suYVb$zQtx30$|*-G(uTzhnP-g&=^d;Asr39~@AuK#;~vDmn-A16x*
zR`tnxBCGmdyM|AAwVs$&ebTP#ldV?ulqNAv(}O7;BueJfKp}=K<e>;<s6ri@Gl_ne
zTsXHdN%m11X0Kb!?(fB-)Jrr6DckZA0Y*fDVxTNg9;gUZ2C4$pftnE05HV~FF$*yd
zu?VpYu?n#cu?blkG71@3Wg*K$R)nk!SrxK6WKGD^kW<KG$U!;}c@gq5<W<P)kT;=7
zLqVa4p~ymkdn!UvhN22Z9f~HDX(%a_F_c*-^HAcJ%TQLKtV7v^Dh(BdDuyZxRUWD$
zR8XV}RUN7()M=<G)G^dqsPj-4p)NxW!Rk;qp-DqSp^2f%LX(H42u&H9Dm2iuff|q_
zg&rvcNufvzNm6K%LX;G$q>v>=KamanSU6UWrDN?_JXVk8gMI`72?`P<#P@i7&<`pS
zWF+WF5R#xIK}v#_1ThI}667T4Nf4BvC_z$!rUX$5suE-+=t>Zlpe#XJg0=*43F;E$
zCFn~Kn4mC0VuHp5kqIgjWG3iL5SpMgL28251hENf6XYi7O%R-*I6-oP<^<6RsuN@<
z=uQxxpgci(g7yUQ3F;H%C+JUb0Ko$U7Z7|va00;#1UC@;KyU=X69iWfd_iyq!5aj3
z5d48~#s?7ye?<5r!XFX-i10^*KO+1Q;g1M^M8t7~KO+1Q;g1M^MEE1Z9})hD9M2rk
z64B_#G0j2EQO#k^am|4Ze?<5r!XFX-i0}tWF!-P!{1M@g2!BNQBf=jM{)q5Lgg+wu
z5#f&re?<5r!XFX-i10^*KO+1Q;g1M^#G){NzzgHs7A`ETZCepq_<bdzt|o+Je}n%^
z_F<}?PTo9MWAfg?^nYv8P5#~F-%bAiqmw^gndqJb+~70xpFwZySwHlzui^Wlzva3g
z?bn_n^|2r|HoWYgHgGQ4<|1s?t^B){f4B1gOIH3H|Ae=l{<|l^H~Px`kK~0uz5?2r
zy7yx*>933?i~U2rWWJXqI8KuOcyVRaKQz-H_k$cyu3VDiwye}g+f_bcJ1@D;TzuN5
zD@s+avnf(-W3}#V+EUgwouv-u47MnGiFGeI6M7J0X>pSftOBKaoh3fnm@GvczL`Ta
z^L_N{6Fw0i&Ve~T1wJJ{6+SgSO&=q%!s~pgh07?nk!+)EGuh^{Eo58Dw&Hr*^k(>Z
zybQ>^Hqtrc{32`%3@;;YAm1kG-$5QXD0j)XNqnD@?1}G`_*RMUmH1|f@0J>DqOcTU
z*dnqtku4%yM7D@*5!oWLMaC8xTV$doTV!mJu|>uf8Czs*k+DV27CBqwY>|unY>~4?
z&K5aa<ZO|%MZp#YTNG?jutmWZ1zWIa{emqDwkX)5WQ&q5O13E3qGXGbElRd1*`j2N
zk}WE>sMw-ni;68Owy4;mVvC9`Dz>QDqGpSlEo!!?*`j8Pnk{O!sM(@si<&JOwrJR*
zVT*<>8n$TIqG5}MEgH6H*a8(t*|H6)j<RJdR32r^cBnqemMu|%l&#pR`^B~mfHnf!
z5NKnd4T3fb+AxqfKxLAz6U8>lxDpGfP)fUuE8&1TrL@Sn5)r6Ya*M<SDweV@oFyz!
zx0Dv)ERligrL+iV2@cdSxkbVQbxhfpaV0`f&6E}ySAqmJO>U7eL0wb!Wn76AR5ztX
z#+6_}jZ<1=T!|M{I=Mw62Gvg4mvJR%Q1g@)8CT*4l}~AraV2z6|Kt`49@Ic(U&fXA
zK_ygLWLya$)I+&Nf(SKH*_Ux8j!+qu78zGU3H4EFk#QxOP$lISi6>M_WnadXkV3sw
zT4Y>_DpXCSg~!FbC|e#E)l=EBEh?z8Wm{BHWy|BDk}6xVZKpC4TZt>wRQasL73!+8
zFSZg_sIE$j*tQdl#8s5ILY<Y*wv$Xd%e2!>JI}Nejl>nIuG~}N3YAyc7h8!d)L*4V
z#+A526;@hgT!|~xV!1`)3iVjom;05tLRD5;<bEZtP@Clzi7V7+WnadXxI&dyT8clA
zCKkq$GLCa;SN2|(?MdEJDqBf8Bq@g^<&dNtl9WS|a;TD?wQL*N9+T~H*}f#(6S9@-
z)5)2q7R>5#VP`KA{>G~DrX+|J4NE4^tt78W7My1Ewbk!9(?&_}d~o*M=)&sS=<M>v
zm5Y<Ji<7g<qs8?Jcj;a27DK#%m`b;CF*J`X&<}R)vx61S&R06PcsnIrdmh<8BP*wQ
z{O!QuQ!4FB<~LKp+$Iwo|J`-#7c5hFy6M(0eLPL&i0IZYvfRaUT2`Z5zoe4SyY-8F
zcfylpM8$7kzwC6<|9@Y%AW3art6;IxmL7JtP#GNker0*)+tJ$k#nthVS$ZLzof(bK
zuP)(*{UfudPn|qi&(0hj_6BRC^~=kXVJ{hQkq4Lz&aW<CUKy|R{`5fhlL1$KhU3AU
zAfHYKT=*HbRiD9}9EjOmdy(y~v9$RbTnyU56;M;zd<|10+QG%vH(#nP9qr&6Qcl`@
zoh>R#+gx;@nz8VMIrYYX7g?<00RC^)91khQIPJmG@OrgrFt^k;LYv_Lo5d@`!DxB3
zGDLsD_K3_;G)I{^%FR(>j!JV>nWNeqHPfRx<;2)BT_0Y1Xf&A9h!Y`)0|nvDH@x8n
zHr{gEJg2=CS+6-2Y@=}a^V*~wFA#5d<+f6RtRKlbl5c8*WMo}Q)|X_RN!FWW-AUK{
zDc6tAH<nejtfFNVEvsl*MawE$R?)JGmQ`k1WtLTDS!I@0W?5yHRc2XbmQ`k1WtLTL
zS>={hZdv7)Rc=}3mQ`+9<(5@$S>={hVObTHRbg2bmQ`U{6_yqMxorr-!m=tXtHQD>
zEvwS9DlMzhvMMdB(y}TotJ1P6EvwS9sw}I@vZ^er%Cf2~tID#fEUU`0sw}I@vZ^hs
z+On!GtJ<=vEvwqHsx7P9vZ^hs+OldatH!cwEUU({YAmb9vT7`=#<FTGtH!cIb=w3b
z-mP`VmKff-b>}TLyo>A3TXJ|u*PXZY@b0cX?~)z)dEQU-ex~<Ry`Ss-WbbEtKi#|e
z9#wkFH#+^?l#h3eU4N&aoAvP?vg_~ka}z(_Rkr=z)Q@KZ?!5D_oBi>gv+M8t>k<Io
zg?9a&e_am1`_i_*%K~@;;m$k#Tq3}`)~>(P&*cKVkL~(9{aiZ0vklwdB?P?7?an*>
zTu#9I-mbsX&!q*_8f<@;81VeWop<`V+<^DVU4N&aOAmPG-1T?*xeS5#(rtg2Bk;7w
zop<`VG=X>8U4N&a%M^I;-Sv0+xnzOoK(@b27kFphop<`VjDh#+U4N&aOB#6h-u8EC
z1J9b=d8ePt9C&Zv^>_NY<biklU4N&a%OCjiVfy2@0PfiG!*2xKdCL#K9dPHZe)vs+
zJ8${nw+8mS%OCg+f;;c{x%`1AYOcTI=kf=B%i#JuelCCD_YStd%O7|`=gvF*{Fero
zKk&N=XYcsA{DI$A*#0hm;Mt!$@A$d=f!}4g{!TxaKk(ZP*Wc;q@&|t3Vf(xMfhUdb
zywlI+5BxU7^>_NY{DCT;>+ke)`2)`|ZGV?P@Y@r2-syKefBb6wVB0s=_<vopxN_<6
zmcLbbZh3O}zw>`E`L+6m)Ezc_5xV{`a(e6A7cfwpXiGp`$XFl3xq_$>F>d{|a@uQ~
Lvprlx%YOd>&)TR8

diff --git a/tests/queries/0_stateless/00309_formats.sql b/tests/queries/0_stateless/00309_formats.sql
index b0939c00a10..0366cdeea5c 100644
--- a/tests/queries/0_stateless/00309_formats.sql
+++ b/tests/queries/0_stateless/00309_formats.sql
@@ -9,3 +9,8 @@ SELECT number * 246 + 10 AS n, toDate('2000-01-01') + n AS d, range(n) AS arr, a
 SELECT number * 246 + 10 AS n, toDate('2000-01-01') + n AS d, range(n) AS arr, arrayStringConcat(arrayMap(x -> reinterpretAsString(x), arr)) AS s, (n, d) AS tuple FROM system.numbers LIMIT 2 FORMAT JSON;
 SELECT number * 246 + 10 AS n, toDate('2000-01-01') + n AS d, range(n) AS arr, arrayStringConcat(arrayMap(x -> reinterpretAsString(x), arr)) AS s, (n, d) AS tuple FROM system.numbers LIMIT 2 FORMAT JSONCompact;
 SELECT number * 246 + 10 AS n, toDate('2000-01-01') + n AS d, range(n) AS arr, arrayStringConcat(arrayMap(x -> reinterpretAsString(x), arr)) AS s, (n, d) AS tuple FROM system.numbers LIMIT 2 FORMAT XML;
+
+SET enable_named_columns_in_function_tuple = 1;
+
+SELECT 36 AS n, toDate('2000-01-01') + n AS d, (n, d) AS tuple FROM system.numbers LIMIT 1 FORMAT RowBinaryWithNamesAndTypes;
+SELECT number * 246 + 10 AS n, toDate('2000-01-01') + n AS d, (n, d) AS tuple FROM system.numbers LIMIT 1 FORMAT TabSeparatedWithNamesAndTypes;

From 837b0e053d841bcd00ed15769a22853bb58f1a84 Mon Sep 17 00:00:00 2001
From: avogar <avogar@clickhouse.com>
Date: Mon, 29 Jul 2024 10:50:41 +0000
Subject: [PATCH 0960/1488] Add JSONAsObject format to documentation

---
 docs/en/interfaces/formats.md                 | 62 +++++++++++++++++++
 .../aspell-ignore/en/aspell-dict.txt          |  1 +
 2 files changed, 63 insertions(+)

diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md
index 9c4c082bc3a..8892c6d8d3f 100644
--- a/docs/en/interfaces/formats.md
+++ b/docs/en/interfaces/formats.md
@@ -32,6 +32,7 @@ The supported formats are:
 | [Vertical](#vertical)                                                                     | ✗    | ✔     |
 | [JSON](#json)                                                                             | ✔    | ✔     |
 | [JSONAsString](#jsonasstring)                                                             | ✔    | ✗     |
+| [JSONAsObject](#jsonasobject)                                                             | ✔    | ✗     |
 | [JSONStrings](#jsonstrings)                                                               | ✔    | ✔     |
 | [JSONColumns](#jsoncolumns)                                                               | ✔    | ✔     |
 | [JSONColumnsWithMetadata](#jsoncolumnsmonoblock)                                          | ✔    | ✔     |
@@ -822,6 +823,67 @@ Result:
 └────────────────────────────┘
 ```
 
+## JSONAsObject {#jsonasobject}
+
+In this format, a single JSON object is interpreted as a single [Object('json')](/docs/en/sql-reference/data-types/json.md) value. If the input has several JSON objects (comma separated), they are interpreted as separate rows. If the input data is enclosed in square brackets, it is interpreted as an array of JSONs.
+
+This format can only be parsed for a table with a single field of type [Object('json')](/docs/en/sql-reference/data-types/json.md). The remaining columns must be set to [DEFAULT](/docs/en/sql-reference/statements/create/table.md/#default) or [MATERIALIZED](/docs/en/sql-reference/statements/create/table.md/#materialized).
+
+**Examples**
+
+Query:
+
+``` sql
+SET allow_experimental_object_type = 1;
+CREATE TABLE json_as_object (json Object('json')) ENGINE = Memory;
+INSERT INTO json_as_object (json) FORMAT JSONAsObject {"foo":{"bar":{"x":"y"},"baz":1}},{},{"any json stucture":1}
+SELECT * FROM json_as_object FORMAT JSONEachRow;
+```
+
+Result:
+
+``` response
+{"json":{"any json stucture":0,"foo":{"bar":{"x":"y"},"baz":1}}}
+{"json":{"any json stucture":0,"foo":{"bar":{"x":""},"baz":0}}}
+{"json":{"any json stucture":1,"foo":{"bar":{"x":""},"baz":0}}}
+```
+
+**An array of JSON objects**
+
+Query:
+
+``` sql
+SET allow_experimental_object_type = 1;
+CREATE TABLE json_square_brackets (field Object('json')) ENGINE = Memory;
+INSERT INTO json_square_brackets FORMAT JSONAsObject [{"id": 1, "name": "name1"}, {"id": 2, "name": "name2"}];
+
+SELECT * FROM json_square_brackets FORMAT JSONEachRow;
+```
+
+Result:
+
+```response
+{"field":{"id":1,"name":"name1"}}
+{"field":{"id":2,"name":"name2"}}
+```
+
+**Columns with default values**
+
+```sql
+SET allow_experimental_object_type = 1;
+CREATE TABLE json_as_object (json Object('json'), time DateTime MATERIALIZED now()) ENGINE = Memory;
+INSERT INTO json_as_object (json) FORMAT JSONAsObject {"foo":{"bar":{"x":"y"},"baz":1}};
+INSERT INTO json_as_object (json) FORMAT JSONAsObject {};
+INSERT INTO json_as_object (json) FORMAT JSONAsObject {"any json stucture":1}
+SELECT * FROM json_as_object FORMAT JSONEachRow
+```
+
+```resonse
+{"json":{"any json stucture":0,"foo":{"bar":{"x":"y"},"baz":1}},"time":"2024-07-25 17:02:45"}
+{"json":{"any json stucture":0,"foo":{"bar":{"x":""},"baz":0}},"time":"2024-07-25 17:02:47"}
+{"json":{"any json stucture":1,"foo":{"bar":{"x":""},"baz":0}},"time":"2024-07-25 17:02:50"}
+```
+
 ## JSONCompact {#jsoncompact}
 
 Differs from JSON only in that data rows are output in arrays, not in objects.
diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt
index 8e4e4fafe29..1efb7b23d1b 100644
--- a/utils/check-style/aspell-ignore/en/aspell-dict.txt
+++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt
@@ -1868,6 +1868,7 @@ joinGet
 json
 jsonMergePatch
 jsonasstring
+jsonasobject
 jsoncolumns
 jsoncolumnsmonoblock
 jsoncompact

From c5575e8748cb65a92dcd74256d81d9187e947280 Mon Sep 17 00:00:00 2001
From: avogar <avogar@clickhouse.com>
Date: Mon, 29 Jul 2024 10:58:39 +0000
Subject: [PATCH 0961/1488] Don't use PeekableReadBuffer in JSONAsObject format

---
 .../Impl/JSONAsStringRowInputFormat.cpp       | 64 ++++++++++---------
 .../Formats/Impl/JSONAsStringRowInputFormat.h | 15 +++--
 2 files changed, 41 insertions(+), 38 deletions(-)

diff --git a/src/Processors/Formats/Impl/JSONAsStringRowInputFormat.cpp b/src/Processors/Formats/Impl/JSONAsStringRowInputFormat.cpp
index f57e62adba9..9da5e533324 100644
--- a/src/Processors/Formats/Impl/JSONAsStringRowInputFormat.cpp
+++ b/src/Processors/Formats/Impl/JSONAsStringRowInputFormat.cpp
@@ -15,11 +15,8 @@ namespace ErrorCodes
     extern const int ILLEGAL_COLUMN;
 }
 
-JSONAsRowInputFormat::JSONAsRowInputFormat(const Block & header_, ReadBuffer & in_, Params params_, const FormatSettings & format_settings_)
-    : JSONAsRowInputFormat(header_, std::make_unique<PeekableReadBuffer>(in_), params_, format_settings_) {}
-
-JSONAsRowInputFormat::JSONAsRowInputFormat(const Block & header_, std::unique_ptr<PeekableReadBuffer> buf_, Params params_, const FormatSettings & format_settings_) :
-    JSONEachRowRowInputFormat(*buf_, header_, std::move(params_), format_settings_, false), buf(std::move(buf_))
+JSONAsRowInputFormat::JSONAsRowInputFormat(const Block & header_, ReadBuffer & in_, Params params_, const FormatSettings & format_settings_) :
+    JSONEachRowRowInputFormat(in_, header_, std::move(params_), format_settings_, false)
 {
     if (header_.columns() > 1)
         throw Exception(ErrorCodes::BAD_ARGUMENTS,
@@ -27,19 +24,6 @@ JSONAsRowInputFormat::JSONAsRowInputFormat(const Block & header_, std::unique_pt
             header_.columns());
 }
 
-
-void JSONAsRowInputFormat::setReadBuffer(ReadBuffer & in_)
-{
-    buf = std::make_unique<PeekableReadBuffer>(in_);
-    JSONEachRowRowInputFormat::setReadBuffer(*buf);
-}
-
-void JSONAsRowInputFormat::resetReadBuffer()
-{
-    buf.reset();
-    JSONEachRowRowInputFormat::resetReadBuffer();
-}
-
 bool JSONAsRowInputFormat::readRow(MutableColumns & columns, RowReadExtension &)
 {
     assert(columns.size() == 1);
@@ -48,35 +32,41 @@ bool JSONAsRowInputFormat::readRow(MutableColumns & columns, RowReadExtension &)
     if (!allow_new_rows)
         return false;
 
-    skipWhitespaceIfAny(*buf);
-    if (!buf->eof())
+    skipWhitespaceIfAny(*in);
+    if (!in->eof())
     {
-        if (!data_in_square_brackets && *buf->position() == ';')
+        if (!data_in_square_brackets && *in->position() == ';')
         {
             /// ';' means the end of query, but it cannot be before ']'.
             return allow_new_rows = false;
         }
-        else if (data_in_square_brackets && *buf->position() == ']')
+        else if (data_in_square_brackets && *in->position() == ']')
         {
             /// ']' means the end of query.
             return allow_new_rows = false;
         }
     }
 
-    if (!buf->eof())
+    if (!in->eof())
         readJSONObject(*columns[0]);
 
-    skipWhitespaceIfAny(*buf);
-    if (!buf->eof() && *buf->position() == ',')
-        ++buf->position();
-    skipWhitespaceIfAny(*buf);
+    skipWhitespaceIfAny(*in);
+    if (!in->eof() && *in->position() == ',')
+        ++in->position();
+    skipWhitespaceIfAny(*in);
 
-    return !buf->eof();
+    return !in->eof();
 }
 
 JSONAsStringRowInputFormat::JSONAsStringRowInputFormat(
-    const Block & header_, ReadBuffer & in_, Params params_, const FormatSettings & format_settings_)
-    : JSONAsRowInputFormat(header_, in_, params_, format_settings_)
+    const Block & header_, ReadBuffer & in_, IRowInputFormat::Params params_, const FormatSettings & format_settings_)
+    : JSONAsStringRowInputFormat(header_, std::make_unique<PeekableReadBuffer>(in_), params_, format_settings_)
+{
+}
+
+JSONAsStringRowInputFormat::JSONAsStringRowInputFormat(
+    const Block & header_, std::unique_ptr<PeekableReadBuffer> buf_, Params params_, const FormatSettings & format_settings_)
+    : JSONAsRowInputFormat(header_, *buf_, params_, format_settings_), buf(std::move(buf_))
 {
     if (!isString(removeNullable(removeLowCardinality(header_.getByPosition(0).type))))
         throw Exception(ErrorCodes::BAD_ARGUMENTS,
@@ -84,6 +74,18 @@ JSONAsStringRowInputFormat::JSONAsStringRowInputFormat(
             header_.getByPosition(0).type->getName());
 }
 
+void JSONAsStringRowInputFormat::setReadBuffer(ReadBuffer & in_)
+{
+    buf = std::make_unique<PeekableReadBuffer>(in_);
+    JSONAsRowInputFormat::setReadBuffer(*buf);
+}
+
+void JSONAsStringRowInputFormat::resetReadBuffer()
+{
+    buf.reset();
+    JSONAsRowInputFormat::resetReadBuffer();
+}
+
 void JSONAsStringRowInputFormat::readJSONObject(IColumn & column)
 {
     PeekableReadBufferCheckpoint checkpoint{*buf};
@@ -174,7 +176,7 @@ JSONAsObjectRowInputFormat::JSONAsObjectRowInputFormat(
 
 void JSONAsObjectRowInputFormat::readJSONObject(IColumn & column)
 {
-    serializations[0]->deserializeTextJSON(column, *buf, format_settings);
+    serializations[0]->deserializeTextJSON(column, *in, format_settings);
 }
 
 Chunk JSONAsObjectRowInputFormat::getChunkForCount(size_t rows)
diff --git a/src/Processors/Formats/Impl/JSONAsStringRowInputFormat.h b/src/Processors/Formats/Impl/JSONAsStringRowInputFormat.h
index 16112325a97..5eaa88182b7 100644
--- a/src/Processors/Formats/Impl/JSONAsStringRowInputFormat.h
+++ b/src/Processors/Formats/Impl/JSONAsStringRowInputFormat.h
@@ -18,17 +18,11 @@ class JSONAsRowInputFormat : public JSONEachRowRowInputFormat
 public:
     JSONAsRowInputFormat(const Block & header_, ReadBuffer & in_, Params params_, const FormatSettings & format_settings);
 
-    void setReadBuffer(ReadBuffer & in_) override;
-    void resetReadBuffer() override;
-
 private:
-    JSONAsRowInputFormat(const Block & header_, std::unique_ptr<PeekableReadBuffer> buf_, Params params_, const FormatSettings & format_settings);
-
     bool readRow(MutableColumns & columns, RowReadExtension & ext) override;
 
 protected:
     virtual void readJSONObject(IColumn & column) = 0;
-    std::unique_ptr<PeekableReadBuffer> buf;
 };
 
 /// Each JSON object is parsed as a whole to string.
@@ -36,11 +30,18 @@ protected:
 class JSONAsStringRowInputFormat final : public JSONAsRowInputFormat
 {
 public:
-    JSONAsStringRowInputFormat(const Block & header_, ReadBuffer & in_, Params params_, const FormatSettings & format_settings);
+    JSONAsStringRowInputFormat(const Block & header_, ReadBuffer & in_, Params params_, const FormatSettings & format_settings_);
     String getName() const override { return "JSONAsStringRowInputFormat"; }
 
+    void setReadBuffer(ReadBuffer & in_) override;
+    void resetReadBuffer() override;
+
 private:
+    JSONAsStringRowInputFormat(const Block & header_, std::unique_ptr<PeekableReadBuffer> buf_, Params params_, const FormatSettings & format_settings_);
+
     void readJSONObject(IColumn & column) override;
+
+    std::unique_ptr<PeekableReadBuffer> buf;
 };
 
 

From 5b5e88b7e34567659a6df2d1fc7abec53ff5c3b2 Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <nikitamikhaylov@clickhouse.com>
Date: Mon, 29 Jul 2024 11:00:21 +0000
Subject: [PATCH 0962/1488] Ok

---
 .../queries/0_stateless/01238_http_memory_tracking.sh  | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/tests/queries/0_stateless/01238_http_memory_tracking.sh b/tests/queries/0_stateless/01238_http_memory_tracking.sh
index 26d3dd8acd4..ce1310cf302 100755
--- a/tests/queries/0_stateless/01238_http_memory_tracking.sh
+++ b/tests/queries/0_stateless/01238_http_memory_tracking.sh
@@ -8,18 +8,22 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
 set -o pipefail
 
+MISTER_USER="test_01238_http_$RANDOM"
+# Create another user to check limit on the amount of memory reliabily
+${CLICKHOUSE_CLIENT} --format Null -q "CREATE USER $MISTER_USER"
+
 # This is needed to keep at least one running query for user for the time of test.
 # (1k http queries takes ~1 second, let's run for 5x more to avoid flaps)
-${CLICKHOUSE_CLIENT} --function_sleep_max_microseconds_per_block 5000000 --format Null -n <<<'SELECT sleepEachRow(1) FROM numbers(5)' &
+${CLICKHOUSE_CLIENT} --user ${MISTER_USER} --function_sleep_max_microseconds_per_block 5000000 --format Null -n <<<'SELECT sleepEachRow(1) FROM numbers(5)' &
 
 # ignore "yes: standard output: Broken pipe"
 yes 'SELECT 1' 2>/dev/null | {
     head -n1000
 } | {
-    xargs -I{} ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&wait_end_of_query=1&max_memory_usage_for_user=$((1<<30))" -d '{}'
+    xargs -I{} ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&user=${MISTER_USER}&wait_end_of_query=1&max_memory_usage_for_user=$((1<<30))" -d '{}'
 } | grep -x -c 1
 
 wait
 
 # Reset max_memory_usage_for_user, so it will not affect other tests
-${CLICKHOUSE_CLIENT} --max_memory_usage_for_user=0 -q "SELECT 1 FORMAT Null"
+${CLICKHOUSE_CLIENT} --user ${MISTER_USER} --max_memory_usage_for_user=0 -q "SELECT 1 FORMAT Null"

From a8f36ebded4f2d8919a7713047f675126e7d9131 Mon Sep 17 00:00:00 2001
From: Blargian <shaunstruwig.ss@gmail.com>
Date: Mon, 29 Jul 2024 13:01:47 +0200
Subject: [PATCH 0963/1488] update wording of toInt8OrDefault

---
 docs/en/sql-reference/functions/type-conversion-functions.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/en/sql-reference/functions/type-conversion-functions.md b/docs/en/sql-reference/functions/type-conversion-functions.md
index 37541dd4818..2fb7fa448a2 100644
--- a/docs/en/sql-reference/functions/type-conversion-functions.md
+++ b/docs/en/sql-reference/functions/type-conversion-functions.md
@@ -236,7 +236,7 @@ toInt8OrNull('abc'): ᴺᵁᴸᴸ
 
 ## toInt8OrDefault
 
-Like [`toInt8`](#toint8), takes an argument of type [String](../data-types/string.md) and tries to parse it to type [`Int8`](../data-types/int-uint.md). If unsuccessful, returns the default type value.
+Like [`toInt8`](#toint8), this function converts an input value to a value of type [Int8](../data-types/int-uint.md) but returns the default value in case of an error.
 
 **Syntax**
 

From 8f6a8b7ad6eac57774bf8a05b71a9837c42dca0d Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitlibar@clickhouse.com>
Date: Sun, 28 Jul 2024 16:16:43 +0200
Subject: [PATCH 0964/1488] Fix test
 test_backup_restore_on_cluster/test_disallow_concurrency

---
 .../test_disallow_concurrency.py                   | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/tests/integration/test_backup_restore_on_cluster/test_disallow_concurrency.py b/tests/integration/test_backup_restore_on_cluster/test_disallow_concurrency.py
index cd0f2032559..7ce28cbdc96 100644
--- a/tests/integration/test_backup_restore_on_cluster/test_disallow_concurrency.py
+++ b/tests/integration/test_backup_restore_on_cluster/test_disallow_concurrency.py
@@ -111,10 +111,10 @@ def create_and_fill_table():
         nodes[i].query(f"INSERT INTO tbl SELECT number FROM numbers(40000000)")
 
 
-def wait_for_fail_backup(node, backup_id, backup_name):
+def wait_for_fail_backup(node, backup_id):
     expected_errors = [
         "Concurrent backups not supported",
-        f"Backup {backup_name} already exists",
+        "BACKUP_ALREADY_EXISTS",
     ]
     status = node.query(
         f"SELECT status FROM system.backups WHERE id == '{backup_id}'"
@@ -137,7 +137,7 @@ def wait_for_fail_backup(node, backup_id, backup_name):
         error = node.query(
             f"SELECT error FROM system.backups WHERE id == '{backup_id}'"
         ).rstrip("\n")
-        assert re.search(f"Backup {backup_name} already exists", error)
+        assert any([expected_error in error for expected_error in expected_errors])
         return
     else:
         assert False, "Concurrent backups both passed, when one is expected to fail"
@@ -207,10 +207,10 @@ def test_concurrent_backups_on_same_node():
 
     expected_errors = [
         "Concurrent backups not supported",
-        f"Backup {backup_name} already exists",
+        "BACKUP_ALREADY_EXISTS",
     ]
     if not error:
-        wait_for_fail_backup(nodes[0], id, backup_name)
+        wait_for_fail_backup(nodes[0], id)
 
     assert any([expected_error in error for expected_error in expected_errors])
 
@@ -257,11 +257,11 @@ def test_concurrent_backups_on_different_nodes():
 
     expected_errors = [
         "Concurrent backups not supported",
-        f"Backup {backup_name} already exists",
+        "BACKUP_ALREADY_EXISTS",
     ]
 
     if not error:
-        wait_for_fail_backup(nodes[1], id, backup_name)
+        wait_for_fail_backup(nodes[1], id)
 
     assert any([expected_error in error for expected_error in expected_errors])
 

From 6c104e7c73c454d5c4158365e5625be94ed3c5eb Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitlibar@clickhouse.com>
Date: Sun, 28 Jul 2024 16:16:53 +0200
Subject: [PATCH 0965/1488] Simplify test
 test_backup_restore_on_cluster/test_disallow_concurrency

---
 .../test_disallow_concurrency.py              | 256 ++++++++----------
 1 file changed, 117 insertions(+), 139 deletions(-)

diff --git a/tests/integration/test_backup_restore_on_cluster/test_disallow_concurrency.py b/tests/integration/test_backup_restore_on_cluster/test_disallow_concurrency.py
index 7ce28cbdc96..f3b6993c4fd 100644
--- a/tests/integration/test_backup_restore_on_cluster/test_disallow_concurrency.py
+++ b/tests/integration/test_backup_restore_on_cluster/test_disallow_concurrency.py
@@ -9,13 +9,13 @@ import re
 
 cluster = ClickHouseCluster(__file__)
 
-num_nodes = 10
+num_nodes = 2
 
 
 def generate_cluster_def():
     path = os.path.join(
         os.path.dirname(os.path.realpath(__file__)),
-        "./_gen/cluster_for_concurrency_test.xml",
+        "./_gen/cluster_for_test_disallow_concurrency.xml",
     )
     os.makedirs(os.path.dirname(path), exist_ok=True)
     with open(path, "w") as f:
@@ -111,22 +111,56 @@ def create_and_fill_table():
         nodes[i].query(f"INSERT INTO tbl SELECT number FROM numbers(40000000)")
 
 
-def wait_for_fail_backup(node, backup_id):
+def get_status_and_error(node, backup_or_restore_id):
+    return (
+        node.query(
+            f"SELECT status, error FROM system.backups WHERE id == '{backup_or_restore_id}'"
+        )
+        .rstrip("\n")
+        .split("\t")
+    )
+
+
+def wait_for_backup(node, backup_id):
+    assert_eq_with_retry(
+        node,
+        f"SELECT status FROM system.backups WHERE id = '{backup_id}'",
+        "BACKUP_CREATED",
+        sleep_time=2,
+        retry_count=50,
+    )
+
+
+def wait_for_restore(node, restore_id):
+    assert_eq_with_retry(
+        node,
+        f"SELECT status FROM system.backups WHERE id == '{restore_id}'",
+        "RESTORED",
+        sleep_time=2,
+        retry_count=50,
+    )
+
+
+def check_backup_error(error):
     expected_errors = [
         "Concurrent backups not supported",
         "BACKUP_ALREADY_EXISTS",
     ]
-    status = node.query(
-        f"SELECT status FROM system.backups WHERE id == '{backup_id}'"
-    ).rstrip("\n")
+    assert any([expected_error in error for expected_error in expected_errors])
+
+
+def check_restore_error(error):
+    expected_errors = [
+        "Concurrent restores not supported",
+        "Cannot restore the table default.tbl because it already contains some data",
+    ]
+    assert any([expected_error in error for expected_error in expected_errors])
+
+
+def wait_for_backup_failure(node, backup_id):
+    status, error = get_status_and_error(node, backup_id)
     # It is possible that the second backup was picked up first, and then the async backup
-    if status == "BACKUP_FAILED":
-        error = node.query(
-            f"SELECT error FROM system.backups WHERE id == '{backup_id}'"
-        ).rstrip("\n")
-        assert any([expected_error in error for expected_error in expected_errors])
-        return
-    elif status == "CREATING_BACKUP":
+    if status == "CREATING_BACKUP":
         assert_eq_with_retry(
             node,
             f"SELECT status FROM system.backups WHERE id = '{backup_id}'",
@@ -134,31 +168,17 @@ def wait_for_fail_backup(node, backup_id):
             sleep_time=2,
             retry_count=50,
         )
-        error = node.query(
-            f"SELECT error FROM system.backups WHERE id == '{backup_id}'"
-        ).rstrip("\n")
-        assert any([expected_error in error for expected_error in expected_errors])
-        return
+        status, error = get_status_and_error(node, backup_id)
+    if status == "BACKUP_FAILED":
+        check_backup_error(error)
     else:
         assert False, "Concurrent backups both passed, when one is expected to fail"
 
 
-def wait_for_fail_restore(node, restore_id):
-    expected_errors = [
-        "Concurrent restores not supported",
-        "Cannot restore the table default.tbl because it already contains some data",
-    ]
-    status = node.query(
-        f"SELECT status FROM system.backups WHERE id == '{restore_id}'"
-    ).rstrip("\n")
+def wait_for_restore_failure(node, restore_id):
+    status, error = get_status_and_error(node, restore_id)
     # It is possible that the second backup was picked up first, and then the async backup
-    if status == "RESTORE_FAILED":
-        error = node.query(
-            f"SELECT error FROM system.backups WHERE id == '{restore_id}'"
-        ).rstrip("\n")
-        assert any([expected_error in error for expected_error in expected_errors])
-        return
-    elif status == "RESTORING":
+    if status == "RESTORING":
         assert_eq_with_retry(
             node,
             f"SELECT status FROM system.backups WHERE id = '{restore_id}'",
@@ -166,14 +186,9 @@ def wait_for_fail_restore(node, restore_id):
             sleep_time=2,
             retry_count=50,
         )
-        error = node.query(
-            f"SELECT error FROM system.backups WHERE id == '{restore_id}'"
-        ).rstrip("\n")
-        assert re.search(
-            "Cannot restore the table default.tbl because it already contains some data",
-            error,
-        )
-        return
+        status, error = get_status_and_error(node, restore_id)
+    if status == "RESTORE_FAILED":
+        check_restore_error(error)
     else:
         assert False, "Concurrent restores both passed, when one is expected to fail"
 
@@ -188,39 +203,28 @@ def test_concurrent_backups_on_same_node():
 
     backup_name = new_backup_name()
 
-    id = (
+    # Backup #1.
+    id, status = (
         nodes[0]
         .query(f"BACKUP TABLE tbl ON CLUSTER 'cluster' TO {backup_name} ASYNC")
-        .split("\t")[0]
+        .rstrip("\n")
+        .split("\t")
     )
 
-    status = (
-        nodes[0]
-        .query(f"SELECT status FROM system.backups WHERE id == '{id}'")
-        .rstrip("\n")
-    )
     assert status in ["CREATING_BACKUP", "BACKUP_CREATED"]
 
-    result, error = nodes[0].query_and_get_answer_with_error(
+    # Backup #2.
+    _, error = nodes[0].query_and_get_answer_with_error(
         f"BACKUP TABLE tbl ON CLUSTER 'cluster' TO {backup_name}"
     )
 
-    expected_errors = [
-        "Concurrent backups not supported",
-        "BACKUP_ALREADY_EXISTS",
-    ]
-    if not error:
-        wait_for_fail_backup(nodes[0], id)
-
-    assert any([expected_error in error for expected_error in expected_errors])
-
-    assert_eq_with_retry(
-        nodes[0],
-        f"SELECT status FROM system.backups WHERE id = '{id}'",
-        "BACKUP_CREATED",
-        sleep_time=2,
-        retry_count=50,
-    )
+    if error:
+        # Backup #2 failed, backup #1 should be successful.
+        check_backup_error(error)
+        wait_for_backup(nodes[0], id)
+    else:
+        # Backup #2 was successful, backup #1 should fail.
+        wait_for_backup_failure(nodes[0], id)
 
     # This restore part is added to confirm creating an internal backup & restore work
     # even when a concurrent backup is stopped
@@ -238,40 +242,38 @@ def test_concurrent_backups_on_different_nodes():
 
     backup_name = new_backup_name()
 
-    id = (
-        nodes[1]
+    # Backup #1.
+    id, status = (
+        nodes[0]
         .query(f"BACKUP TABLE tbl ON CLUSTER 'cluster' TO {backup_name} ASYNC")
-        .split("\t")[0]
+        .rstrip("\n")
+        .split("\t")
     )
 
-    status = (
-        nodes[1]
-        .query(f"SELECT status FROM system.backups WHERE id == '{id}'")
-        .rstrip("\n")
-    )
     assert status in ["CREATING_BACKUP", "BACKUP_CREATED"]
 
-    result, error = nodes[0].query_and_get_answer_with_error(
+    # Backup #2.
+    _, error = nodes[1].query_and_get_answer_with_error(
         f"BACKUP TABLE tbl ON CLUSTER 'cluster' TO {backup_name}"
     )
 
-    expected_errors = [
-        "Concurrent backups not supported",
-        "BACKUP_ALREADY_EXISTS",
-    ]
+    if error:
+        # Backup #2 failed, backup #1 should be successful.
+        check_backup_error(error)
+        wait_for_backup(nodes[0], id)
+    else:
+        # Backup #2 was successful, backup #1 should fail.
+        wait_for_backup_failure(nodes[0], id)
 
-    if not error:
-        wait_for_fail_backup(nodes[1], id)
-
-    assert any([expected_error in error for expected_error in expected_errors])
-
-    assert_eq_with_retry(
-        nodes[1],
-        f"SELECT status FROM system.backups WHERE id = '{id}'",
-        "BACKUP_CREATED",
-        sleep_time=2,
-        retry_count=50,
+    # This restore part is added to confirm creating an internal backup & restore work
+    # even when a concurrent backup is stopped
+    nodes[0].query(
+        f"DROP TABLE tbl ON CLUSTER 'cluster' SYNC",
+        settings={
+            "distributed_ddl_task_timeout": 360,
+        },
     )
+    nodes[0].query(f"RESTORE TABLE tbl ON CLUSTER 'cluster' FROM {backup_name}")
 
 
 def test_concurrent_restores_on_same_node():
@@ -288,40 +290,28 @@ def test_concurrent_restores_on_same_node():
         },
     )
 
-    restore_id = (
+    # Restore #1.
+    restore_id, status = (
         nodes[0]
         .query(f"RESTORE TABLE tbl ON CLUSTER 'cluster' FROM {backup_name} ASYNC")
-        .split("\t")[0]
+        .rstrip("\n")
+        .split("\t")
     )
 
-    status = (
-        nodes[0]
-        .query(f"SELECT status FROM system.backups WHERE id == '{restore_id}'")
-        .rstrip("\n")
-    )
     assert status in ["RESTORING", "RESTORED"]
 
-    result, error = nodes[0].query_and_get_answer_with_error(
+    # Restore #2.
+    _, error = nodes[0].query_and_get_answer_with_error(
         f"RESTORE TABLE tbl ON CLUSTER 'cluster' FROM {backup_name}"
     )
 
-    expected_errors = [
-        "Concurrent restores not supported",
-        "Cannot restore the table default.tbl because it already contains some data",
-    ]
-
-    if not error:
-        wait_for_fail_restore(nodes[0], restore_id)
-
-    assert any([expected_error in error for expected_error in expected_errors])
-
-    assert_eq_with_retry(
-        nodes[0],
-        f"SELECT status FROM system.backups WHERE id == '{restore_id}'",
-        "RESTORED",
-        sleep_time=2,
-        retry_count=50,
-    )
+    if error:
+        # Restore #2 failed, restore #1 should be successful.
+        check_restore_error(error)
+        wait_for_restore(nodes[0], restore_id)
+    else:
+        # Restore #2 was successful, restore #1 should fail.
+        wait_for_restore_failure(nodes[0], restore_id)
 
 
 def test_concurrent_restores_on_different_node():
@@ -338,37 +328,25 @@ def test_concurrent_restores_on_different_node():
         },
     )
 
-    restore_id = (
+    # Restore #1.
+    restore_id, status = (
         nodes[0]
         .query(f"RESTORE TABLE tbl ON CLUSTER 'cluster' FROM {backup_name} ASYNC")
-        .split("\t")[0]
+        .rstrip("\n")
+        .split("\t")
     )
 
-    status = (
-        nodes[0]
-        .query(f"SELECT status FROM system.backups WHERE id == '{restore_id}'")
-        .rstrip("\n")
-    )
     assert status in ["RESTORING", "RESTORED"]
 
-    result, error = nodes[1].query_and_get_answer_with_error(
+    # Restore #2.
+    _, error = nodes[1].query_and_get_answer_with_error(
         f"RESTORE TABLE tbl ON CLUSTER 'cluster' FROM {backup_name}"
     )
 
-    expected_errors = [
-        "Concurrent restores not supported",
-        "Cannot restore the table default.tbl because it already contains some data",
-    ]
-
-    if not error:
-        wait_for_fail_restore(nodes[0], restore_id)
-
-    assert any([expected_error in error for expected_error in expected_errors])
-
-    assert_eq_with_retry(
-        nodes[0],
-        f"SELECT status FROM system.backups WHERE id == '{restore_id}'",
-        "RESTORED",
-        sleep_time=2,
-        retry_count=50,
-    )
+    if error:
+        # Restore #2 failed, restore #1 should be successful.
+        check_restore_error(error)
+        wait_for_restore(nodes[0], restore_id)
+    else:
+        # Restore #2 was successful, restore #1 should fail.
+        wait_for_restore_failure(nodes[0], restore_id)

From 4cf7e6a79d2748bd4f055f14d0504959f567574e Mon Sep 17 00:00:00 2001
From: Pablo Marcos <pablo.marcos.oltra@clickhouse.com>
Date: Mon, 29 Jul 2024 11:15:03 +0000
Subject: [PATCH 0966/1488] Remove logging traces to help debugging

---
 tests/integration/test_storage_s3_queue/test.py | 14 ++------------
 1 file changed, 2 insertions(+), 12 deletions(-)

diff --git a/tests/integration/test_storage_s3_queue/test.py b/tests/integration/test_storage_s3_queue/test.py
index 963089c3777..e3445d14cdb 100644
--- a/tests/integration/test_storage_s3_queue/test.py
+++ b/tests/integration/test_storage_s3_queue/test.py
@@ -858,17 +858,12 @@ def test_max_set_age(started_cluster):
     def get_count():
         return int(node.query(f"SELECT count() FROM {dst_table_name}"))
 
-    import os
-
     def wait_for_condition(check_function, max_wait_time=1.5 * max_age):
-        logging.debug(f"{os.getenv('PYTEST_XDIST_WORKER')} - PMO: Waiting for condition")
         before = time.time()
         while time.time() - before < max_wait_time:
             if check_function():
-                logging.debug(f"{os.getenv('PYTEST_XDIST_WORKER')} - PMO: Condition verified")
                 return
             time.sleep(0.25)
-        logging.debug(f"{os.getenv('PYTEST_XDIST_WORKER')} - PMO: Oh oh... Condition not verified")
         assert False
 
     wait_for_condition(lambda: get_count() == expected_rows)
@@ -889,12 +884,8 @@ def test_max_set_age(started_cluster):
         ).splitlines()
     ]
     assert files_to_generate == len(paths_count)
-    try:
-        for path_count in paths_count:
-            assert 2 == path_count
-    finally:
-        logging.debug(f"{os.getenv('PYTEST_XDIST_WORKER')} - PMO: Got path count {path_count} for paths_count {paths_count}")
-        logging.debug(f"{os.getenv('PYTEST_XDIST_WORKER')} - PMO: Got paths: {node.query('SELECT *, _path from ' + dst_table_name + ' FORMAT JSONEachRow')}")
+    for path_count in paths_count:
+        assert 2 == path_count
 
     def get_object_storage_failures():
         return int(
@@ -904,7 +895,6 @@ def test_max_set_age(started_cluster):
         )
 
     failed_count = get_object_storage_failures()
-    logging.debug(f"{os.getenv('PYTEST_XDIST_WORKER')} - PMO: Got failed count {failed_count}")
 
     values = [
         ["failed", 1, 1],

From ceb0f7fe0f09fbb15cd107ba3f832735afca7c71 Mon Sep 17 00:00:00 2001
From: Blargian <shaunstruwig.ss@gmail.com>
Date: Mon, 29 Jul 2024 13:17:27 +0200
Subject: [PATCH 0967/1488] Fix typoo

---
 docs/en/sql-reference/functions/type-conversion-functions.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/en/sql-reference/functions/type-conversion-functions.md b/docs/en/sql-reference/functions/type-conversion-functions.md
index bb386092c0f..339060a1ac4 100644
--- a/docs/en/sql-reference/functions/type-conversion-functions.md
+++ b/docs/en/sql-reference/functions/type-conversion-functions.md
@@ -335,7 +335,7 @@ toUInt8OrNull('abc'): ᴺᵁᴸᴸ
 
 ## toUInt8OrDefault
 
-Like [`toUInt8`](#touint8), takes an argument of type [String](../data-types/string.md) and tries to parse it to type [`UInt8`](../data-types/int-uint.md). If unsuccessful, returns the default type value.
+Like [`toUInt8`](#touint8), this function converts an input value to a value of type [UInt8](../data-types/int-uint.md) but returns the default value in case of an error.
 
 **Syntax**
 
@@ -589,7 +589,7 @@ Like [`toUInt16`](#touint16), this function converts an input value to a value o
 **Syntax**
 
 ```sql
-toInt16OrDefault(expr, def)
+toUInt16OrDefault(expr, def)
 ```
 
 **Arguments**

From 634c513ba710ce3b428641593d7730130a34a338 Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitlibar@clickhouse.com>
Date: Mon, 29 Jul 2024 13:19:24 +0200
Subject: [PATCH 0968/1488] Fix cleanup in
 test_backup_restore_on_cluster/test.py::test_system_functions

---
 tests/integration/test_backup_restore_on_cluster/test.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/integration/test_backup_restore_on_cluster/test.py b/tests/integration/test_backup_restore_on_cluster/test.py
index 700ed6f15f5..1b7f4aaa97d 100644
--- a/tests/integration/test_backup_restore_on_cluster/test.py
+++ b/tests/integration/test_backup_restore_on_cluster/test.py
@@ -776,7 +776,6 @@ def test_system_users():
 
 def test_system_functions():
     node1.query("CREATE FUNCTION linear_equation AS (x, k, b) -> k*x + b;")
-
     node1.query("CREATE FUNCTION parity_str AS (n) -> if(n % 2, 'odd', 'even');")
 
     backup_name = new_backup_name()
@@ -817,6 +816,9 @@ def test_system_functions():
         [[0, "even"], [1, "odd"], [2, "even"]]
     )
 
+    node1.query("DROP FUNCTION linear_equation")
+    node1.query("DROP FUNCTION parity_str")
+
 
 def test_projection():
     node1.query(

From 56135d18cb40506fc500b497068121bc96e3a9ee Mon Sep 17 00:00:00 2001
From: avogar <avogar@clickhouse.com>
Date: Mon, 29 Jul 2024 11:23:01 +0000
Subject: [PATCH 0969/1488] Remove duplicated tests

---
 .../03036_dynamic_read_subcolumns.lib         | 35 -------------------
 .../03036_dynamic_read_subcolumns_1.reference | 19 ----------
 .../03036_dynamic_read_subcolumns_1.sh        | 21 -----------
 .../03036_dynamic_read_subcolumns_2.reference | 19 ----------
 .../03036_dynamic_read_subcolumns_2.sh        | 21 -----------
 .../03036_dynamic_read_subcolumns_3.reference | 19 ----------
 .../03036_dynamic_read_subcolumns_3.sh        | 21 -----------
 7 files changed, 155 deletions(-)
 delete mode 100755 tests/queries/0_stateless/03036_dynamic_read_subcolumns.lib
 delete mode 100644 tests/queries/0_stateless/03036_dynamic_read_subcolumns_1.reference
 delete mode 100755 tests/queries/0_stateless/03036_dynamic_read_subcolumns_1.sh
 delete mode 100644 tests/queries/0_stateless/03036_dynamic_read_subcolumns_2.reference
 delete mode 100755 tests/queries/0_stateless/03036_dynamic_read_subcolumns_2.sh
 delete mode 100644 tests/queries/0_stateless/03036_dynamic_read_subcolumns_3.reference
 delete mode 100755 tests/queries/0_stateless/03036_dynamic_read_subcolumns_3.sh

diff --git a/tests/queries/0_stateless/03036_dynamic_read_subcolumns.lib b/tests/queries/0_stateless/03036_dynamic_read_subcolumns.lib
deleted file mode 100755
index 4914051db82..00000000000
--- a/tests/queries/0_stateless/03036_dynamic_read_subcolumns.lib
+++ /dev/null
@@ -1,35 +0,0 @@
-#!/usr/bin/env bash
-
-function test()
-{
-    echo "test"
-    $CH_CLIENT -q "insert into test select number, number from numbers(100000) settings min_insert_block_size_rows=50000"
-    $CH_CLIENT -q "insert into test select number, 'str_' || toString(number) from numbers(100000, 100000) settings min_insert_block_size_rows=50000"
-    $CH_CLIENT -q "insert into test select number, arrayMap(x -> multiIf(number % 9 == 0, NULL, number % 9 == 3, 'str_' || toString(number), number), range(number % 10 + 1)) from numbers(200000, 100000) settings min_insert_block_size_rows=50000"
-    $CH_CLIENT -q "insert into test select number, NULL from numbers(300000, 100000) settings min_insert_block_size_rows=50000"
-    $CH_CLIENT -q "insert into test select number, multiIf(number % 4 == 3, 'str_' || toString(number), number % 4 == 2, NULL, number % 4 == 1, number, arrayMap(x -> multiIf(number % 9 == 0, NULL, number % 9 == 3, 'str_' || toString(number), number), range(number % 10 + 1))) from numbers(400000, 400000) settings min_insert_block_size_rows=50000"
-    $CH_CLIENT -q "insert into test select number, [range((number % 10 + 1)::UInt64)]::Array(Array(Dynamic)) from numbers(100000, 100000) settings min_insert_block_size_rows=50000"
-
-    $CH_CLIENT -q "select distinct dynamicType(d) as type from test order by type"
-    $CH_CLIENT -q "select count() from test where dynamicType(d) == 'UInt64'"
-    $CH_CLIENT -q "select count() from test where d.UInt64 is not NULL"
-    $CH_CLIENT -q "select count() from test where dynamicType(d) == 'String'"
-    $CH_CLIENT -q "select count() from test where d.String is not NULL"
-    $CH_CLIENT -q "select count() from test where dynamicType(d) == 'Date'"
-    $CH_CLIENT -q "select count() from test where d.Date is not NULL"
-    $CH_CLIENT -q "select count() from test where dynamicType(d) == 'Array(Variant(String, UInt64))'"
-    $CH_CLIENT -q "select count() from test where not empty(d.\`Array(Variant(String, UInt64))\`)"
-    $CH_CLIENT -q "select count() from test where dynamicType(d) == 'Array(Array(Dynamic))'"
-    $CH_CLIENT -q "select count() from test where not empty(d.\`Array(Array(Dynamic))\`)"
-    $CH_CLIENT -q "select count() from test where d is NULL"
-    $CH_CLIENT -q "select count() from test where not empty(d.\`Tuple(a Array(Dynamic))\`.a.String)"
-
-    $CH_CLIENT -q "select d, d.UInt64, d.String, d.\`Array(Variant(String, UInt64))\` from test format Null"
-    $CH_CLIENT -q "select d.UInt64, d.String, d.\`Array(Variant(String, UInt64))\` from test format Null"
-    $CH_CLIENT -q "select d.Int8, d.Date, d.\`Array(String)\` from test format Null"
-    $CH_CLIENT -q "select d, d.UInt64, d.Date, d.\`Array(Variant(String, UInt64))\`, d.\`Array(Variant(String, UInt64))\`.size0, d.\`Array(Variant(String, UInt64))\`.UInt64 from test format Null"
-    $CH_CLIENT -q "select d.UInt64, d.Date, d.\`Array(Variant(String, UInt64))\`, d.\`Array(Variant(String, UInt64))\`.size0, d.\`Array(Variant(String, UInt64))\`.UInt64, d.\`Array(Variant(String, UInt64))\`.String from test format Null"
-    $CH_CLIENT -q "select d, d.\`Tuple(a UInt64, b String)\`.a, d.\`Array(Dynamic)\`.\`Variant(String, UInt64)\`.UInt64, d.\`Array(Variant(String, UInt64))\`.UInt64 from test format Null"
-    $CH_CLIENT -q "select d.\`Array(Dynamic)\`.\`Variant(String, UInt64)\`.UInt64, d.\`Array(Dynamic)\`.size0, d.\`Array(Variant(String, UInt64))\`.UInt64 from test format Null"
-    $CH_CLIENT -q "select d.\`Array(Array(Dynamic))\`.size1, d.\`Array(Array(Dynamic))\`.UInt64, d.\`Array(Array(Dynamic))\`.\`Map(String, Tuple(a UInt64))\`.values.a from test format Null"
-}
diff --git a/tests/queries/0_stateless/03036_dynamic_read_subcolumns_1.reference b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_1.reference
deleted file mode 100644
index 0d51ecfac3b..00000000000
--- a/tests/queries/0_stateless/03036_dynamic_read_subcolumns_1.reference
+++ /dev/null
@@ -1,19 +0,0 @@
-Memory
-test
-Array(Array(Dynamic))
-Array(Variant(String, UInt64))
-None
-String
-UInt64
-200000
-200000
-200000
-200000
-0
-0
-200000
-200000
-100000
-100000
-200000
-0
diff --git a/tests/queries/0_stateless/03036_dynamic_read_subcolumns_1.sh b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_1.sh
deleted file mode 100755
index aabba731816..00000000000
--- a/tests/queries/0_stateless/03036_dynamic_read_subcolumns_1.sh
+++ /dev/null
@@ -1,21 +0,0 @@
-#!/usr/bin/env bash
-# Tags: long
-
-CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# reset --log_comment
-CLICKHOUSE_LOG_COMMENT=
-# shellcheck source=../shell_config.sh
-. "$CUR_DIR"/../shell_config.sh
-
-
-# shellcheck source=./03036_dynamic_read_subcolumns.lib
-. "$CUR_DIR"/03036_dynamic_read_subcolumns.lib
-
-CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --use_variant_as_common_type=1 --allow_experimental_dynamic_type=1"
-
-$CH_CLIENT -q "drop table if exists test;"
-
-echo "Memory"
-$CH_CLIENT -q "create table test (id UInt64, d Dynamic) engine=Memory"
-test
-$CH_CLIENT -q "drop table test;"
diff --git a/tests/queries/0_stateless/03036_dynamic_read_subcolumns_2.reference b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_2.reference
deleted file mode 100644
index 099b7574566..00000000000
--- a/tests/queries/0_stateless/03036_dynamic_read_subcolumns_2.reference
+++ /dev/null
@@ -1,19 +0,0 @@
-MergeTree compact
-test
-Array(Array(Dynamic))
-Array(Variant(String, UInt64))
-None
-String
-UInt64
-200000
-200000
-200000
-200000
-0
-0
-200000
-200000
-100000
-100000
-200000
-0
diff --git a/tests/queries/0_stateless/03036_dynamic_read_subcolumns_2.sh b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_2.sh
deleted file mode 100755
index 872f4c20a98..00000000000
--- a/tests/queries/0_stateless/03036_dynamic_read_subcolumns_2.sh
+++ /dev/null
@@ -1,21 +0,0 @@
-#!/usr/bin/env bash
-# Tags: long
-
-CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# reset --log_comment
-CLICKHOUSE_LOG_COMMENT=
-# shellcheck source=../shell_config.sh
-. "$CUR_DIR"/../shell_config.sh
-
-
-# shellcheck source=./03036_dynamic_read_subcolumns.lib
-. "$CUR_DIR"/03036_dynamic_read_subcolumns.lib
-
-CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --use_variant_as_common_type=1 --allow_experimental_dynamic_type=1"
-
-$CH_CLIENT -q "drop table if exists test;"
-
-echo "MergeTree compact"
-$CH_CLIENT -q "create table test (id UInt64, d Dynamic) engine=MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000;"
-test
-$CH_CLIENT -q "drop table test;"
diff --git a/tests/queries/0_stateless/03036_dynamic_read_subcolumns_3.reference b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_3.reference
deleted file mode 100644
index 35db4a22b4c..00000000000
--- a/tests/queries/0_stateless/03036_dynamic_read_subcolumns_3.reference
+++ /dev/null
@@ -1,19 +0,0 @@
-MergeTree wide
-test
-Array(Array(Dynamic))
-Array(Variant(String, UInt64))
-None
-String
-UInt64
-200000
-200000
-200000
-200000
-0
-0
-200000
-200000
-100000
-100000
-200000
-0
diff --git a/tests/queries/0_stateless/03036_dynamic_read_subcolumns_3.sh b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_3.sh
deleted file mode 100755
index 96276c96add..00000000000
--- a/tests/queries/0_stateless/03036_dynamic_read_subcolumns_3.sh
+++ /dev/null
@@ -1,21 +0,0 @@
-#!/usr/bin/env bash
-# Tags: long
-
-CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# reset --log_comment
-CLICKHOUSE_LOG_COMMENT=
-# shellcheck source=../shell_config.sh
-. "$CUR_DIR"/../shell_config.sh
-
-
-# shellcheck source=./03036_dynamic_read_subcolumns.lib
-. "$CUR_DIR"/03036_dynamic_read_subcolumns.lib
-
-CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --use_variant_as_common_type=1 --allow_experimental_dynamic_type=1"
-
-$CH_CLIENT -q "drop table if exists test;"
-
-echo "MergeTree wide"
-$CH_CLIENT -q "create table test (id UInt64, d Dynamic) engine=MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1;"
-test
-$CH_CLIENT -q "drop table test;"

From 2908cd7284d5b94ca5babe73fbdb4f1f6d7bb2fd Mon Sep 17 00:00:00 2001
From: Blargian <shaunstruwig.ss@gmail.com>
Date: Mon, 29 Jul 2024 13:28:24 +0200
Subject: [PATCH 0970/1488] update toInt*OrDefault to show that the default
 argument is optional

---
 .../functions/type-conversion-functions.md    | 42 +++++++++++--------
 1 file changed, 24 insertions(+), 18 deletions(-)

diff --git a/docs/en/sql-reference/functions/type-conversion-functions.md b/docs/en/sql-reference/functions/type-conversion-functions.md
index 2fb7fa448a2..7cc2c022143 100644
--- a/docs/en/sql-reference/functions/type-conversion-functions.md
+++ b/docs/en/sql-reference/functions/type-conversion-functions.md
@@ -237,17 +237,18 @@ toInt8OrNull('abc'): ᴺᵁᴸᴸ
 ## toInt8OrDefault
 
 Like [`toInt8`](#toint8), this function converts an input value to a value of type [Int8](../data-types/int-uint.md) but returns the default value in case of an error.
+If no `default` value is passed then `0` is returned in case of an error.
 
 **Syntax**
 
 ```sql
-toInt8OrDefault(expr, def)
+toInt8OrDefault(expr[, default])
 ```
 
 **Arguments**
 
 - `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
-- `def` — The default value to return if parsing to type `Int8` is unsuccessful. [Int8](../data-types/int-uint.md).
+- `default` (optional) — The default value to return if parsing to type `Int8` is unsuccessful. [Int8](../data-types/int-uint.md).
 
 Supported arguments:
 - Values of type (U)Int8/16/32/64/128/256.
@@ -265,7 +266,7 @@ This is not considered an error.
 
 **Returned value**
 
-- 8-bit integer value if successful, otherwise returns the default value. [Int8](../data-types/int-uint.md).
+- 8-bit integer value if successful, otherwise returns the default value if passed or `0` if not. [Int8](../data-types/int-uint.md).
 
 :::note
 - The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
@@ -486,17 +487,18 @@ toInt16OrNull('abc'): ᴺᵁᴸᴸ
 ## toInt16OrDefault
 
 Like [`toInt16`](#toint16), this function converts an input value to a value of type [Int16](../data-types/int-uint.md) but returns the default value in case of an error.
+If no `default` value is passed then `0` is returned in case of an error.
 
 **Syntax**
 
 ```sql
-toInt16OrDefault(expr, def)
+toInt16OrDefault(expr[, default])
 ```
 
 **Arguments**
 
 - `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
-- `def` — The default value to return if parsing to type `Int16` is unsuccessful. [Int16](../data-types/int-uint.md).
+- `default` (optional) — The default value to return if parsing to type `Int16` is unsuccessful. [Int16](../data-types/int-uint.md).
 
 Supported arguments:
 - Values of type (U)Int8/16/32/64/128/256.
@@ -514,7 +516,7 @@ This is not considered an error.
 
 **Returned value**
 
-- 16-bit integer value if successful, otherwise returns the default value. [Int16](../data-types/int-uint.md).
+- 16-bit integer value if successful, otherwise returns the default value if passed or `0` if not. [Int16](../data-types/int-uint.md).
 
 :::note
 - The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
@@ -734,17 +736,18 @@ toInt32OrNull('abc'): ᴺᵁᴸᴸ
 ## toInt32OrDefault
 
 Like [`toInt32`](#toint32), this function converts an input value to a value of type [Int32](../data-types/int-uint.md) but returns the default value in case of an error.
+If no `default` value is passed then `0` is returned in case of an error.
 
 **Syntax**
 
 ```sql
-toInt32OrDefault(expr, def)
+toInt32OrDefault(expr[, default])
 ```
 
 **Arguments**
 
 - `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
-- `def` — The default value to return if parsing to type `Int32` is unsuccessful. [Int32](../data-types/int-uint.md).
+- `default` (optional) — The default value to return if parsing to type `Int32` is unsuccessful. [Int32](../data-types/int-uint.md).
 
 Supported arguments:
 - Values of type (U)Int8/16/32/64/128/256.
@@ -762,7 +765,7 @@ This is not considered an error.
 
 **Returned value**
 
-- 32-bit integer value if successful, otherwise returns the default value. [Int32](../data-types/int-uint.md).
+- 32-bit integer value if successful, otherwise returns the default value if passed or `0` if not. [Int32](../data-types/int-uint.md).
 
 :::note
 - The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
@@ -983,17 +986,18 @@ toInt64OrNull('abc'): ᴺᵁᴸᴸ
 ## toInt64OrDefault
 
 Like [`toInt64`](#toint64), this function converts an input value to a value of type [Int64](../data-types/int-uint.md) but returns the default value in case of an error.
+If no `default` value is passed then `0` is returned in case of an error.
 
 **Syntax**
 
 ```sql
-toInt64OrDefault(expr, def)
+toInt64OrDefault(expr[, default])
 ```
 
 **Arguments**
 
 - `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
-- `def` — The default value to return if parsing to type `Int64` is unsuccessful. [Int64](../data-types/int-uint.md).
+- `default` (optional) — The default value to return if parsing to type `Int64` is unsuccessful. [Int64](../data-types/int-uint.md).
 
 Supported arguments:
 - Values of type (U)Int8/16/32/64/128/256.
@@ -1011,7 +1015,7 @@ This is not considered an error.
 
 **Returned value**
 
-- 64-bit integer value if successful, otherwise returns the default value. [Int64](../data-types/int-uint.md).
+- 64-bit integer value if successful, otherwise returns the default value if passed or `0` if not. [Int64](../data-types/int-uint.md).
 
 :::note
 - The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
@@ -1231,17 +1235,18 @@ toInt128OrNull('abc'):  ᴺᵁᴸᴸ
 ## toInt128OrDefault
 
 Like [`toInt128`](#toint128), this function converts an input value to a value of type [Int128](../data-types/int-uint.md) but returns the default value in case of an error.
+If no `default` value is passed then `0` is returned in case of an error.
 
 **Syntax**
 
 ```sql
-toInt128OrDefault(expr, def)
+toInt128OrDefault(expr[, default])
 ```
 
 **Arguments**
 
 - `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
-- `def` — The default value to return if parsing to type `Int128` is unsuccessful. [Int128](../data-types/int-uint.md).
+- `default` (optional) — The default value to return if parsing to type `Int128` is unsuccessful. [Int128](../data-types/int-uint.md).
 
 Supported arguments:
 - (U)Int8/16/32/64/128/256.
@@ -1259,7 +1264,7 @@ This is not considered an error.
 
 **Returned value**
 
-- 128-bit integer value if successful, otherwise returns the default value. [Int128](../data-types/int-uint.md).
+- 128-bit integer value if successful, otherwise returns the default value if passed or `0` if not. [Int128](../data-types/int-uint.md).
 
 :::note
 - The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
@@ -1479,17 +1484,18 @@ toInt256OrNull('abc'):  ᴺᵁᴸᴸ
 ## toInt256OrDefault
 
 Like [`toInt256`](#toint256), this function converts an input value to a value of type [Int256](../data-types/int-uint.md) but returns the default value in case of an error.
+If no `default` value is passed then `0` is returned in case of an error.
 
 **Syntax**
 
 ```sql
-toInt256OrDefault(expr, def)
+toInt256OrDefault(expr[, default])
 ```
 
 **Arguments**
 
 - `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
-- `def` — The default value to return if parsing to type `Int256` is unsuccessful. [Int256](../data-types/int-uint.md).
+- `default` (optional) — The default value to return if parsing to type `Int256` is unsuccessful. [Int256](../data-types/int-uint.md).
 
 Supported arguments:
 - Values of type (U)Int8/16/32/64/128/256.
@@ -1507,7 +1513,7 @@ This is not considered an error.
 
 **Returned value**
 
-- 256-bit integer value if successful, otherwise returns the default value. [Int256](../data-types/int-uint.md).
+- 256-bit integer value if successful, otherwise returns the default value if passed or `0` if not. [Int256](../data-types/int-uint.md).
 
 :::note
 - The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.

From c70bbe1d29288646a4c9897b57e9ff03acbc2052 Mon Sep 17 00:00:00 2001
From: avogar <avogar@clickhouse.com>
Date: Mon, 29 Jul 2024 11:46:00 +0000
Subject: [PATCH 0971/1488] Disable some Dynamic tests under sanitizers,
 rewrite 03202_dynamic_null_map_subcolumn to sql

---
 ...mic_read_subcolumns_compact_merge_tree.sql |  2 +-
 .../03036_dynamic_read_subcolumns_memory.sql  |  2 +-
 ...ynamic_read_subcolumns_wide_merge_tree.sql |  2 +-
 ...merges_1_horizontal_compact_merge_tree.sql |  4 +-
 ..._merges_1_horizontal_compact_wide_tree.sql |  4 +-
 ...c_merges_1_vertical_compact_merge_tree.sql |  4 +-
 ...amic_merges_1_vertical_wide_merge_tree.sql |  4 +-
 ...merges_2_horizontal_compact_merge_tree.sql |  2 +-
 ...ic_merges_2_horizontal_wide_merge_tree.sql |  2 +-
 ...c_merges_2_vertical_compact_merge_tree.sql |  2 +-
 ...amic_merges_2_vertical_wide_merge_tree.sql |  2 +-
 ...2_dynamic_null_map_subcolumn.reference.j2} |  6 +-
 .../03202_dynamic_null_map_subcolumn.sh       | 60 -------------------
 .../03202_dynamic_null_map_subcolumn.sql      | 14 +++++
 .../03202_dynamic_null_map_subcolumn.sql.j2   | 49 +++++++++++++++
 15 files changed, 81 insertions(+), 78 deletions(-)
 rename tests/queries/0_stateless/{03202_dynamic_null_map_subcolumn.reference => 03202_dynamic_null_map_subcolumn.reference.j2} (60%)
 delete mode 100755 tests/queries/0_stateless/03202_dynamic_null_map_subcolumn.sh
 create mode 100644 tests/queries/0_stateless/03202_dynamic_null_map_subcolumn.sql
 create mode 100644 tests/queries/0_stateless/03202_dynamic_null_map_subcolumn.sql.j2

diff --git a/tests/queries/0_stateless/03036_dynamic_read_subcolumns_compact_merge_tree.sql b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_compact_merge_tree.sql
index 66fbf006a8c..ddfba4418bd 100644
--- a/tests/queries/0_stateless/03036_dynamic_read_subcolumns_compact_merge_tree.sql
+++ b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_compact_merge_tree.sql
@@ -1,4 +1,4 @@
--- Tags: long
+-- Tags: long, no-tsan, no-msan, no-ubsan, no-asan
 
 set allow_experimental_variant_type = 1;
 set use_variant_as_common_type = 1;
diff --git a/tests/queries/0_stateless/03036_dynamic_read_subcolumns_memory.sql b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_memory.sql
index bb03bdef704..c446c31fc25 100644
--- a/tests/queries/0_stateless/03036_dynamic_read_subcolumns_memory.sql
+++ b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_memory.sql
@@ -1,4 +1,4 @@
--- Tags: long
+-- Tags: long, no-tsan, no-msan, no-ubsan, no-asan
 
 set allow_experimental_variant_type = 1;
 set use_variant_as_common_type = 1;
diff --git a/tests/queries/0_stateless/03036_dynamic_read_subcolumns_wide_merge_tree.sql b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_wide_merge_tree.sql
index 00aba3a57b6..5aac5f7b72f 100644
--- a/tests/queries/0_stateless/03036_dynamic_read_subcolumns_wide_merge_tree.sql
+++ b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_wide_merge_tree.sql
@@ -1,4 +1,4 @@
--- Tags: long
+-- Tags: long, no-tsan, no-msan, no-ubsan, no-asan
 
 set allow_experimental_variant_type = 1;
 set use_variant_as_common_type = 1;
diff --git a/tests/queries/0_stateless/03037_dynamic_merges_1_horizontal_compact_merge_tree.sql b/tests/queries/0_stateless/03037_dynamic_merges_1_horizontal_compact_merge_tree.sql
index 07371ee099b..d2c787040e5 100644
--- a/tests/queries/0_stateless/03037_dynamic_merges_1_horizontal_compact_merge_tree.sql
+++ b/tests/queries/0_stateless/03037_dynamic_merges_1_horizontal_compact_merge_tree.sql
@@ -1,4 +1,4 @@
--- Tags: long
+-- Tags: long, no-tsan, no-msan, no-ubsan, no-asan
 set allow_experimental_dynamic_type=1;
 
 drop table if exists test;
@@ -30,4 +30,4 @@ system start merges test;
 optimize table test final;
 select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
 
-drop table test;
\ No newline at end of file
+drop table test;
diff --git a/tests/queries/0_stateless/03037_dynamic_merges_1_horizontal_compact_wide_tree.sql b/tests/queries/0_stateless/03037_dynamic_merges_1_horizontal_compact_wide_tree.sql
index 2b55a31e937..f99bf771608 100644
--- a/tests/queries/0_stateless/03037_dynamic_merges_1_horizontal_compact_wide_tree.sql
+++ b/tests/queries/0_stateless/03037_dynamic_merges_1_horizontal_compact_wide_tree.sql
@@ -1,4 +1,4 @@
--- Tags: long
+-- Tags: long, no-tsan, no-msan, no-ubsan, no-asan
 set allow_experimental_dynamic_type=1;
 
 drop table if exists test;
@@ -30,4 +30,4 @@ system start merges test;
 optimize table test final;
 select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
 
-drop table test;
\ No newline at end of file
+drop table test;
diff --git a/tests/queries/0_stateless/03037_dynamic_merges_1_vertical_compact_merge_tree.sql b/tests/queries/0_stateless/03037_dynamic_merges_1_vertical_compact_merge_tree.sql
index ea7295a9eab..be81596d043 100644
--- a/tests/queries/0_stateless/03037_dynamic_merges_1_vertical_compact_merge_tree.sql
+++ b/tests/queries/0_stateless/03037_dynamic_merges_1_vertical_compact_merge_tree.sql
@@ -1,4 +1,4 @@
--- Tags: long
+-- Tags: long, no-tsan, no-msan, no-ubsan, no-asan
 set allow_experimental_dynamic_type=1;
 
 drop table if exists test;
@@ -31,4 +31,4 @@ system start merges test;
 optimize table test final;
 select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
 
-drop table test;
\ No newline at end of file
+drop table test;
diff --git a/tests/queries/0_stateless/03037_dynamic_merges_1_vertical_wide_merge_tree.sql b/tests/queries/0_stateless/03037_dynamic_merges_1_vertical_wide_merge_tree.sql
index e888a14b323..f6396af42a8 100644
--- a/tests/queries/0_stateless/03037_dynamic_merges_1_vertical_wide_merge_tree.sql
+++ b/tests/queries/0_stateless/03037_dynamic_merges_1_vertical_wide_merge_tree.sql
@@ -1,4 +1,4 @@
--- Tags: long
+-- Tags: long, no-tsan, no-msan, no-ubsan, no-asan
 set allow_experimental_dynamic_type=1;
 
 drop table if exists test;
@@ -30,4 +30,4 @@ system start merges test;
 optimize table test final;
 select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
 
-drop table test;
\ No newline at end of file
+drop table test;
diff --git a/tests/queries/0_stateless/03037_dynamic_merges_2_horizontal_compact_merge_tree.sql b/tests/queries/0_stateless/03037_dynamic_merges_2_horizontal_compact_merge_tree.sql
index e633b277ebd..e133ac3001f 100644
--- a/tests/queries/0_stateless/03037_dynamic_merges_2_horizontal_compact_merge_tree.sql
+++ b/tests/queries/0_stateless/03037_dynamic_merges_2_horizontal_compact_merge_tree.sql
@@ -1,4 +1,4 @@
--- Tags: long
+-- Tags: long, no-tsan, no-msan, no-ubsan, no-asan
 
 set allow_experimental_dynamic_type = 1;
 
diff --git a/tests/queries/0_stateless/03037_dynamic_merges_2_horizontal_wide_merge_tree.sql b/tests/queries/0_stateless/03037_dynamic_merges_2_horizontal_wide_merge_tree.sql
index 90dbc2d84f5..d527081b763 100644
--- a/tests/queries/0_stateless/03037_dynamic_merges_2_horizontal_wide_merge_tree.sql
+++ b/tests/queries/0_stateless/03037_dynamic_merges_2_horizontal_wide_merge_tree.sql
@@ -1,4 +1,4 @@
--- Tags: long
+-- Tags: long, no-tsan, no-msan, no-ubsan, no-asan
 
 set allow_experimental_dynamic_type = 1;
 
diff --git a/tests/queries/0_stateless/03037_dynamic_merges_2_vertical_compact_merge_tree.sql b/tests/queries/0_stateless/03037_dynamic_merges_2_vertical_compact_merge_tree.sql
index ffd2618ee51..ebccfb77922 100644
--- a/tests/queries/0_stateless/03037_dynamic_merges_2_vertical_compact_merge_tree.sql
+++ b/tests/queries/0_stateless/03037_dynamic_merges_2_vertical_compact_merge_tree.sql
@@ -1,4 +1,4 @@
--- Tags: long
+-- Tags: long, no-tsan, no-msan, no-ubsan, no-asan
 
 set allow_experimental_dynamic_type = 1;
 
diff --git a/tests/queries/0_stateless/03037_dynamic_merges_2_vertical_wide_merge_tree.sql b/tests/queries/0_stateless/03037_dynamic_merges_2_vertical_wide_merge_tree.sql
index 36dff88751b..104d6018e41 100644
--- a/tests/queries/0_stateless/03037_dynamic_merges_2_vertical_wide_merge_tree.sql
+++ b/tests/queries/0_stateless/03037_dynamic_merges_2_vertical_wide_merge_tree.sql
@@ -1,4 +1,4 @@
--- Tags: long
+-- Tags: long, no-tsan, no-msan, no-ubsan, no-asan
 
 set allow_experimental_dynamic_type = 1;
 
diff --git a/tests/queries/0_stateless/03202_dynamic_null_map_subcolumn.reference b/tests/queries/0_stateless/03202_dynamic_null_map_subcolumn.reference.j2
similarity index 60%
rename from tests/queries/0_stateless/03202_dynamic_null_map_subcolumn.reference
rename to tests/queries/0_stateless/03202_dynamic_null_map_subcolumn.reference.j2
index 8740726c7ef..6684d3736e4 100644
--- a/tests/queries/0_stateless/03202_dynamic_null_map_subcolumn.reference
+++ b/tests/queries/0_stateless/03202_dynamic_null_map_subcolumn.reference.j2
@@ -1,4 +1,4 @@
-Memory
+--- Memory ---
 test
 Array(Array(Dynamic))
 Array(Variant(String, UInt64))
@@ -17,7 +17,7 @@ UInt64
 10
 20
 0
-MergeTree compact
+--- MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000 ---
 test
 Array(Array(Dynamic))
 Array(Variant(String, UInt64))
@@ -36,7 +36,7 @@ UInt64
 10
 20
 0
-MergeTree wide
+--- MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1 ---
 test
 Array(Array(Dynamic))
 Array(Variant(String, UInt64))
diff --git a/tests/queries/0_stateless/03202_dynamic_null_map_subcolumn.sh b/tests/queries/0_stateless/03202_dynamic_null_map_subcolumn.sh
deleted file mode 100755
index bbe25ea72bb..00000000000
--- a/tests/queries/0_stateless/03202_dynamic_null_map_subcolumn.sh
+++ /dev/null
@@ -1,60 +0,0 @@
-#!/usr/bin/env bash
-# Tags: long
-
-CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# shellcheck source=../shell_config.sh
-. "$CUR_DIR"/../shell_config.sh
-
-CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --use_variant_as_common_type=1 --allow_experimental_dynamic_type=1 --optimize_functions_to_subcolumns=0"
-
-
-function test()
-{
-    echo "test"
-    $CH_CLIENT -q "insert into test select number, number from numbers(10) settings min_insert_block_size_rows=50000"
-    $CH_CLIENT -q "insert into test select number, 'str_' || toString(number) from numbers(10, 10) settings min_insert_block_size_rows=50000"
-    $CH_CLIENT -q "insert into test select number, arrayMap(x -> multiIf(number % 9 == 0, NULL, number % 9 == 3, 'str_' || toString(number), number), range(number % 10 + 1)) from numbers(20, 10) settings min_insert_block_size_rows=50000"
-    $CH_CLIENT -q "insert into test select number, NULL from numbers(30, 10) settings min_insert_block_size_rows=50000"
-    $CH_CLIENT -q "insert into test select number, multiIf(number % 4 == 3, 'str_' || toString(number), number % 4 == 2, NULL, number % 4 == 1, number, arrayMap(x -> multiIf(number % 9 == 0, NULL, number % 9 == 3, 'str_' || toString(number), number), range(number % 10 + 1))) from numbers(40, 40) settings min_insert_block_size_rows=50000"
-    $CH_CLIENT -q "insert into test select number, [range((number % 10 + 1)::UInt64)]::Array(Array(Dynamic)) from numbers(10, 10) settings min_insert_block_size_rows=50000"
-
-    $CH_CLIENT -q "select distinct dynamicType(d) as type from test order by type"
-    $CH_CLIENT -q "select count() from test where dynamicType(d) == 'UInt64'"
-    $CH_CLIENT -q "select count() from test where d.UInt64 is not NULL"
-    $CH_CLIENT -q "select count() from test where dynamicType(d) == 'String'"
-    $CH_CLIENT -q "select count() from test where d.String is not NULL"
-    $CH_CLIENT -q "select count() from test where dynamicType(d) == 'Date'"
-    $CH_CLIENT -q "select count() from test where d.Date is not NULL"
-    $CH_CLIENT -q "select count() from test where dynamicType(d) == 'Array(Variant(String, UInt64))'"
-    $CH_CLIENT -q "select count() from test where not empty(d.\`Array(Variant(String, UInt64))\`)"
-    $CH_CLIENT -q "select count() from test where dynamicType(d) == 'Array(Array(Dynamic))'"
-    $CH_CLIENT -q "select count() from test where not empty(d.\`Array(Array(Dynamic))\`)"
-    $CH_CLIENT -q "select count() from test where d is NULL"
-    $CH_CLIENT -q "select count() from test where not empty(d.\`Tuple(a Array(Dynamic))\`.a.String)"
-
-    $CH_CLIENT -q "select d, d.UInt64.null, d.String.null, d.\`Array(Variant(String, UInt64))\`.null from test format Null"
-    $CH_CLIENT -q "select d.UInt64.null, d.String.null, d.\`Array(Variant(String, UInt64))\`.null from test format Null"
-    $CH_CLIENT -q "select d.Int8.null, d.Date.null, d.\`Array(String)\`.null from test format Null"
-    $CH_CLIENT -q "select d, d.UInt64.null, d.Date.null, d.\`Array(Variant(String, UInt64))\`.null, d.\`Array(Variant(String, UInt64))\`.size0, d.\`Array(Variant(String, UInt64))\`.UInt64.null from test format Null"
-    $CH_CLIENT -q "select d.UInt64.null, d.Date.null, d.\`Array(Variant(String, UInt64))\`.null, d.\`Array(Variant(String, UInt64))\`.size0, d.\`Array(Variant(String, UInt64))\`.UInt64.null, d.\`Array(Variant(String, UInt64))\`.String.null from test format Null"
-    $CH_CLIENT -q "select d, d.\`Tuple(a UInt64, b String)\`.a, d.\`Array(Dynamic)\`.\`Variant(String, UInt64)\`.UInt64.null, d.\`Array(Variant(String, UInt64))\`.UInt64.null from test format Null"
-    $CH_CLIENT -q "select d.\`Array(Dynamic)\`.\`Variant(String, UInt64)\`.UInt64.null, d.\`Array(Dynamic)\`.size0, d.\`Array(Variant(String, UInt64))\`.UInt64.null from test format Null"
-    $CH_CLIENT -q "select d.\`Array(Array(Dynamic))\`.size1, d.\`Array(Array(Dynamic))\`.UInt64.null, d.\`Array(Array(Dynamic))\`.\`Map(String, Tuple(a UInt64))\`.values.a from test format Null"
-}
-
-$CH_CLIENT -q "drop table if exists test;"
-
-echo "Memory"
-$CH_CLIENT -q "create table test (id UInt64, d Dynamic) engine=Memory"
-test
-$CH_CLIENT -q "drop table test;"
-
-echo "MergeTree compact"
-$CH_CLIENT -q "create table test (id UInt64, d Dynamic) engine=MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000;"
-test
-$CH_CLIENT -q "drop table test;"
-
-echo "MergeTree wide"
-$CH_CLIENT -q "create table test (id UInt64, d Dynamic) engine=MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1;"
-test
-$CH_CLIENT -q "drop table test;"
diff --git a/tests/queries/0_stateless/03202_dynamic_null_map_subcolumn.sql b/tests/queries/0_stateless/03202_dynamic_null_map_subcolumn.sql
new file mode 100644
index 00000000000..fe9cfc886e2
--- /dev/null
+++ b/tests/queries/0_stateless/03202_dynamic_null_map_subcolumn.sql
@@ -0,0 +1,14 @@
+# Tags: long
+
+set allow_experimental_variant_type = 1;
+set use_variant_as_common_type = 1;
+set allow_experimental_dynamic_type = 1;
+set optimize_functions_to_subcolumns = 0;
+
+{% for engine in ['Memory', 'MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000', 'MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1'] -%}
+
+SELECT '--- {{ engine }} ---';
+
+
+
+{% endfor -%}
diff --git a/tests/queries/0_stateless/03202_dynamic_null_map_subcolumn.sql.j2 b/tests/queries/0_stateless/03202_dynamic_null_map_subcolumn.sql.j2
new file mode 100644
index 00000000000..21bf738dccb
--- /dev/null
+++ b/tests/queries/0_stateless/03202_dynamic_null_map_subcolumn.sql.j2
@@ -0,0 +1,49 @@
+# Tags: long
+
+set allow_experimental_variant_type = 1;
+set use_variant_as_common_type = 1;
+set allow_experimental_dynamic_type = 1;
+set optimize_functions_to_subcolumns = 0;
+
+drop table if exists test;
+
+{% for engine in ['Memory', 'MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000', 'MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1'] -%}
+
+SELECT '--- {{ engine }} ---';
+
+create table test (id UInt64, d Dynamic) engine={{ engine }};
+
+select 'test';
+insert into test select number, number from numbers(10) settings min_insert_block_size_rows=50000;
+insert into test select number, 'str_' || toString(number) from numbers(10, 10) settings min_insert_block_size_rows=50000;
+insert into test select number, arrayMap(x -> multiIf(number % 9 == 0, NULL, number % 9 == 3, 'str_' || toString(number), number), range(number % 10 + 1)) from numbers(20, 10) settings min_insert_block_size_rows=50000;
+insert into test select number, NULL from numbers(30, 10) settings min_insert_block_size_rows=50000;
+insert into test select number, multiIf(number % 4 == 3, 'str_' || toString(number), number % 4 == 2, NULL, number % 4 == 1, number, arrayMap(x -> multiIf(number % 9 == 0, NULL, number % 9 == 3, 'str_' || toString(number), number), range(number % 10 + 1))) from numbers(40, 40) settings min_insert_block_size_rows=50000;
+insert into test select number, [range((number % 10 + 1)::UInt64)]::Array(Array(Dynamic)) from numbers(10, 10) settings min_insert_block_size_rows=50000;
+
+select distinct dynamicType(d) as type from test order by type;
+select count() from test where dynamicType(d) == 'UInt64';
+select count() from test where d.UInt64 is not NULL;
+select count() from test where dynamicType(d) == 'String';
+select count() from test where d.String is not NULL;
+select count() from test where dynamicType(d) == 'Date';
+select count() from test where d.Date is not NULL;
+select count() from test where dynamicType(d) == 'Array(Variant(String, UInt64))';
+select count() from test where not empty(d.`Array(Variant(String, UInt64))`);
+select count() from test where dynamicType(d) == 'Array(Array(Dynamic))';
+select count() from test where not empty(d.`Array(Array(Dynamic))`);
+select count() from test where d is NULL;
+select count() from test where not empty(d.`Tuple(a Array(Dynamic))`.a.String);
+
+select d, d.UInt64.null, d.String.null, d.`Array(Variant(String, UInt64))`.null from test format Null;
+select d.UInt64.null, d.String.null, d.`Array(Variant(String, UInt64))`.null from test format Null;
+select d.Int8.null, d.Date.null, d.`Array(String)`.null from test format Null;
+select d, d.UInt64.null, d.Date.null, d.`Array(Variant(String, UInt64))`.null, d.`Array(Variant(String, UInt64))`.size0, d.`Array(Variant(String, UInt64))`.UInt64.null from test format Null;
+select d.UInt64.null, d.Date.null, d.`Array(Variant(String, UInt64))`.null, d.`Array(Variant(String, UInt64))`.size0, d.`Array(Variant(String, UInt64))`.UInt64.null, d.`Array(Variant(String, UInt64))`.String.null from test format Null;
+select d, d.`Tuple(a UInt64, b String)`.a, d.`Array(Dynamic)`.`Variant(String, UInt64)`.UInt64.null, d.`Array(Variant(String, UInt64))`.UInt64.null from test format Null;
+select d.`Array(Dynamic)`.`Variant(String, UInt64)`.UInt64.null, d.`Array(Dynamic)`.size0, d.`Array(Variant(String, UInt64))`.UInt64.null from test format Null;
+select d.`Array(Array(Dynamic))`.size1, d.`Array(Array(Dynamic))`.UInt64.null, d.`Array(Array(Dynamic))`.`Map(String, Tuple(a UInt64))`.values.a from test format Null;
+
+drop table test;
+
+{% endfor -%}

From 918feaff9dfbae9a8be71ee3a4b69abfb844b52a Mon Sep 17 00:00:00 2001
From: Max K <maxkaynov@gmail.com>
Date: Mon, 29 Jul 2024 13:35:18 +0200
Subject: [PATCH 0972/1488] CI: Fix reuse for release branches

---
 tests/ci/pr_info.py        | 9 ++++++++-
 tests/ci/test_ci_config.py | 1 +
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/tests/ci/pr_info.py b/tests/ci/pr_info.py
index 2c8ada7b983..5c051b093e0 100644
--- a/tests/ci/pr_info.py
+++ b/tests/ci/pr_info.py
@@ -2,6 +2,7 @@
 import json
 import logging
 import os
+import re
 from typing import Dict, List, Set, Union
 from urllib.parse import quote
 
@@ -328,7 +329,13 @@ class PRInfo:
 
     @property
     def is_release(self) -> bool:
-        return self.number == 0 and not self.is_merge_queue
+        return self.is_master or (
+            self.is_push_event
+            and (
+                bool(re.match(r"^2[1-9]\.[1-9][0-9]*$", self.head_ref))
+                or bool(re.match(r"^release/2[1-9]\.[1-9][0-9]*$", self.head_ref))
+            )
+        )
 
     @property
     def is_pr(self):
diff --git a/tests/ci/test_ci_config.py b/tests/ci/test_ci_config.py
index be540413b3c..f376a129e6f 100644
--- a/tests/ci/test_ci_config.py
+++ b/tests/ci/test_ci_config.py
@@ -478,6 +478,7 @@ class TestCIConfig(unittest.TestCase):
         pr_info = PRInfo(github_event=_TEST_EVENT_JSON)
         pr_info.event_type = EventType.PUSH
         pr_info.number = 0
+        pr_info.head_ref = "24.12345"
         assert pr_info.is_release and not pr_info.is_merge_queue
         ci_cache = CIPY._configure_jobs(
             S3Helper(), pr_info, settings, skip_jobs=False, dry_run=True

From aa50bb098767865995d8c7d204e9823213e37764 Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Mon, 29 Jul 2024 13:47:06 +0200
Subject: [PATCH 0973/1488] Fix attach thread after exception

---
 .../MergeTree/ReplicatedMergeTreeRestartingThread.cpp  |  2 +-
 .../configs/fault_injection.xml                        |  7 +++++++
 tests/integration/test_replicated_table_attach/test.py | 10 +++++++++-
 3 files changed, 17 insertions(+), 2 deletions(-)
 create mode 100644 tests/integration/test_replicated_table_attach/configs/fault_injection.xml

diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp
index 268cead2a0e..05fd6f6915b 100644
--- a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp
+++ b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp
@@ -344,7 +344,7 @@ void ReplicatedMergeTreeRestartingThread::partialShutdown(bool part_of_full_shut
 void ReplicatedMergeTreeRestartingThread::shutdown(bool part_of_full_shutdown)
 {
     /// Stop restarting_thread before stopping other tasks - so that it won't restart them again.
-    need_stop = true;
+    need_stop = part_of_full_shutdown;
     task->deactivate();
 
     /// Explicitly set the event, because the restarting thread will not set it again
diff --git a/tests/integration/test_replicated_table_attach/configs/fault_injection.xml b/tests/integration/test_replicated_table_attach/configs/fault_injection.xml
new file mode 100644
index 00000000000..7ee4ab92f37
--- /dev/null
+++ b/tests/integration/test_replicated_table_attach/configs/fault_injection.xml
@@ -0,0 +1,7 @@
+<clickhouse>
+    <zookeeper>
+        <enable_fault_injections_during_startup>1</enable_fault_injections_during_startup>
+        <send_fault_probability>0.001</send_fault_probability>
+        <recv_fault_probability>0.001</recv_fault_probability>
+    </zookeeper>
+</clickhouse>
diff --git a/tests/integration/test_replicated_table_attach/test.py b/tests/integration/test_replicated_table_attach/test.py
index dee2be3fcf7..02ef1ad6353 100644
--- a/tests/integration/test_replicated_table_attach/test.py
+++ b/tests/integration/test_replicated_table_attach/test.py
@@ -3,8 +3,11 @@ import pytest
 from helpers.cluster import ClickHouseCluster
 from helpers.network import PartitionManager
 
+import os
+
 
 cluster = ClickHouseCluster(__file__)
+CONFIG_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "configs")
 
 node = cluster.add_instance(
     "node",
@@ -54,7 +57,12 @@ def test_startup_with_small_bg_pool_partitioned(started_cluster):
     assert_values()
     with PartitionManager() as pm:
         pm.drop_instance_zk_connections(node)
-        node.restart_clickhouse(stop_start_wait_sec=300)
+        node.stop_clickhouse(stop_wait_sec=150)
+        node.copy_file_to_container(
+            os.path.join(CONFIG_DIR, "fault_injection.xml"),
+            "/etc/clickhouse-server/config.d/fault_injection.xml",
+        )
+        node.start_clickhouse(start_wait_sec=150)
         assert_values()
 
     # check that we activate it in the end

From 161e794618a86f39ad4f5e6435306e384c85d01d Mon Sep 17 00:00:00 2001
From: kssenii <sumarokovakseniia@mail.ru>
Date: Mon, 29 Jul 2024 12:57:36 +0200
Subject: [PATCH 0974/1488] Add no-distributed-cache tag

---
 docker/test/stateless/run.sh               |  4 ++++
 tests/clickhouse-test                      | 10 ++++++++++
 tests/queries/0_stateless/02995_index_9.sh |  2 +-
 3 files changed, 15 insertions(+), 1 deletion(-)

diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh
index f9f96c76d59..da17b82d91b 100755
--- a/docker/test/stateless/run.sh
+++ b/docker/test/stateless/run.sh
@@ -212,6 +212,10 @@ function run_tests()
         ADDITIONAL_OPTIONS+=('--shared-catalog')
     fi
 
+    if [[ "$USE_DISTRIBUTED_CACHE" -eq 1 ]]; then
+        ADDITIONAL_OPTIONS+=('--distributed-cache')
+    fi
+
     if [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
         ADDITIONAL_OPTIONS+=('--replicated-database')
         # Too many tests fail for DatabaseReplicated in parallel.
diff --git a/tests/clickhouse-test b/tests/clickhouse-test
index a29c786e998..f810a8714bd 100755
--- a/tests/clickhouse-test
+++ b/tests/clickhouse-test
@@ -681,6 +681,7 @@ class FailureReason(enum.Enum):
     BUILD = "not running for current build"
     NO_PARALLEL_REPLICAS = "smth in not supported with parallel replicas"
     SHARED_MERGE_TREE = "no-shared-merge-tree"
+    DISTRIBUTED_CACHE = "distributed-cache"
 
     # UNKNOWN reasons
     NO_REFERENCE = "no reference file"
@@ -1191,6 +1192,9 @@ class TestCase:
         elif tags and ("no-replicated-database" in tags) and args.replicated_database:
             return FailureReason.REPLICATED_DB
 
+        elif tags and ("no-distributed-cache" in tags) and args.distributed_cache:
+            return FailureReason.DISTRIBUTED_CACHE
+
         elif (
             tags
             and ("atomic-database" in tags)
@@ -3204,6 +3208,12 @@ def parse_args():
         default=False,
         help="Run tests over s3 storage",
     )
+    parser.add_argument(
+        "--distributed-cache",
+        action="store_true",
+        default=False,
+        help="Run tests with enabled distributed cache",
+    )
     parser.add_argument(
         "--azure-blob-storage",
         action="store_true",
diff --git a/tests/queries/0_stateless/02995_index_9.sh b/tests/queries/0_stateless/02995_index_9.sh
index 76160c62aaa..46426f22185 100755
--- a/tests/queries/0_stateless/02995_index_9.sh
+++ b/tests/queries/0_stateless/02995_index_9.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Tags: long, no-debug, no-asan, no-tsan, no-msan, no-ubsan, no-sanitize-coverage
+# Tags: long, no-debug, no-asan, no-tsan, no-msan, no-ubsan, no-sanitize-coverage, no-distributed-cache
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh

From 8affc58c4d77eb490b6b969020d8d797bfb49cb9 Mon Sep 17 00:00:00 2001
From: kssenii <sumarokovakseniia@mail.ru>
Date: Mon, 29 Jul 2024 14:10:19 +0200
Subject: [PATCH 0975/1488] Tag more tests

---
 tests/queries/0_stateless/01603_read_with_backoff_bug.sql       | 2 +-
 .../0_stateless/01605_adaptive_granularity_block_borders.sql    | 2 +-
 tests/queries/0_stateless/02714_local_object_storage.sql        | 2 ++
 tests/queries/0_stateless/02770_async_buffer_ignore.sh          | 2 +-
 4 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/tests/queries/0_stateless/01603_read_with_backoff_bug.sql b/tests/queries/0_stateless/01603_read_with_backoff_bug.sql
index 1cf52c0288b..3d467515b5c 100644
--- a/tests/queries/0_stateless/01603_read_with_backoff_bug.sql
+++ b/tests/queries/0_stateless/01603_read_with_backoff_bug.sql
@@ -1,4 +1,4 @@
--- Tags: no-tsan
+-- Tags: no-tsan, no-distributed-cache
 -- Tag no-tsan: Too long for TSan
 
 set enable_filesystem_cache=0;
diff --git a/tests/queries/0_stateless/01605_adaptive_granularity_block_borders.sql b/tests/queries/0_stateless/01605_adaptive_granularity_block_borders.sql
index 9b96ce3e586..5f09dc423b2 100644
--- a/tests/queries/0_stateless/01605_adaptive_granularity_block_borders.sql
+++ b/tests/queries/0_stateless/01605_adaptive_granularity_block_borders.sql
@@ -1,4 +1,4 @@
--- Tags: no-random-merge-tree-settings, no-tsan, no-debug, no-object-storage
+-- Tags: no-random-merge-tree-settings, no-tsan, no-debug, no-object-storage, no-distributed-cache
 -- no-tsan: too slow
 -- no-object-storage: for remote tables we use thread pool even when reading with one stream, so memory consumption is higher
 
diff --git a/tests/queries/0_stateless/02714_local_object_storage.sql b/tests/queries/0_stateless/02714_local_object_storage.sql
index 914eac1ddba..bfe3162df97 100644
--- a/tests/queries/0_stateless/02714_local_object_storage.sql
+++ b/tests/queries/0_stateless/02714_local_object_storage.sql
@@ -1,3 +1,5 @@
+-- Tags: no-distributed-cache
+
 SET min_bytes_to_use_direct_io='1Gi'; -- It does not work (fixme)
 SET local_filesystem_read_method='pread'; -- ui_uring local_fs_method does not work here (fixme)
 
diff --git a/tests/queries/0_stateless/02770_async_buffer_ignore.sh b/tests/queries/0_stateless/02770_async_buffer_ignore.sh
index 6eaf655324d..37f002767d6 100755
--- a/tests/queries/0_stateless/02770_async_buffer_ignore.sh
+++ b/tests/queries/0_stateless/02770_async_buffer_ignore.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Tags: no-fasttest, no-random-settings
+# Tags: no-fasttest, no-random-settings, no-distributed-cache
 
 CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh

From 9f8c90065e5beff8a6e8d763123bbda672371293 Mon Sep 17 00:00:00 2001
From: Max K <maxkaynov@gmail.com>
Date: Mon, 29 Jul 2024 14:16:20 +0200
Subject: [PATCH 0976/1488] Fix docker jobs dependencies

---
 .github/actions/release/action.yml | 10 ++++--
 tests/ci/ci_config.py              |  4 +--
 tests/ci/docker_server.py          | 53 ++++++++++++------------------
 3 files changed, 31 insertions(+), 36 deletions(-)

diff --git a/.github/actions/release/action.yml b/.github/actions/release/action.yml
index c3897682a33..94f178738e3 100644
--- a/.github/actions/release/action.yml
+++ b/.github/actions/release/action.yml
@@ -145,7 +145,9 @@ runs:
         cd "./tests/ci"
         python3 ./create_release.py --set-progress-started --progress "docker server release"
         export CHECK_NAME="Docker server image"
-        python3 docker_server.py --release-type auto --version ${{ env.RELEASE_TAG }} --check-name "$CHECK_NAME" --sha ${{ env.COMMIT_SHA }} ${{ ! inputs.dry-run && '--push' || '' }}
+        git checkout ${{ env.RELEASE_TAG }}
+        python3 docker_server.py --release-type auto --version ${{ env.RELEASE_TAG }} --check-name "$CHECK_NAME" ${{ ! inputs.dry-run && '--push' || '' }}
+        git checkout -
         python3 ./create_release.py --set-progress-completed
     - name: Docker clickhouse/clickhouse-keeper building
       if: ${{ inputs.type == 'patch' }}
@@ -154,15 +156,19 @@ runs:
         cd "./tests/ci"
         python3 ./create_release.py --set-progress-started --progress "docker keeper release"
         export CHECK_NAME="Docker keeper image"
-        python3 docker_server.py --release-type auto --version ${{ env.RELEASE_TAG }} --check-name "$CHECK_NAME" --sha ${{ env.COMMIT_SHA }} ${{ ! inputs.dry-run && '--push'  || '' }}
+        git checkout ${{ env.RELEASE_TAG }}
+        python3 docker_server.py --release-type auto --version ${{ env.RELEASE_TAG }} --check-name "$CHECK_NAME" ${{ ! inputs.dry-run && '--push'  || '' }}
+        git checkout -
         python3 ./create_release.py --set-progress-completed
     - name: Set current Release progress to Completed with OK
       shell: bash
       run: |
+        git checkout "$GITHUB_REF_NAME"
         python3 ./tests/ci/create_release.py --set-progress-started --progress "completed"
         python3 ./tests/ci/create_release.py --set-progress-completed
     - name: Post Slack Message
       if: ${{ !cancelled() }}
       shell: bash
       run: |
+        git checkout "$GITHUB_REF_NAME"
         python3 ./tests/ci/create_release.py --post-status ${{ inputs.dry-run && '--dry-run' || '' }}
diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py
index a44b15f34c1..72c50886d86 100644
--- a/tests/ci/ci_config.py
+++ b/tests/ci/ci_config.py
@@ -508,10 +508,10 @@ class CI:
             runner_type=Runners.STYLE_CHECKER,
         ),
         JobNames.DOCKER_SERVER: CommonJobConfigs.DOCKER_SERVER.with_properties(
-            required_builds=[BuildNames.PACKAGE_RELEASE]
+            required_builds=[BuildNames.PACKAGE_RELEASE, BuildNames.PACKAGE_AARCH64]
         ),
         JobNames.DOCKER_KEEPER: CommonJobConfigs.DOCKER_SERVER.with_properties(
-            required_builds=[BuildNames.PACKAGE_RELEASE]
+            required_builds=[BuildNames.PACKAGE_RELEASE, BuildNames.PACKAGE_AARCH64]
         ),
         JobNames.DOCS_CHECK: JobConfig(
             digest=DigestConfig(
diff --git a/tests/ci/docker_server.py b/tests/ci/docker_server.py
index 413c35cbebe..8ee0ed84120 100644
--- a/tests/ci/docker_server.py
+++ b/tests/ci/docker_server.py
@@ -62,12 +62,6 @@ def parse_args() -> argparse.Namespace:
         help="a version to build, automaticaly got from version_helper, accepts either "
         "tag ('refs/tags/' is removed automatically) or a normal 22.2.2.2 format",
     )
-    parser.add_argument(
-        "--sha",
-        type=str,
-        default="",
-        help="sha of the commit to use packages from",
-    )
     parser.add_argument(
         "--release-type",
         type=str,
@@ -128,17 +122,9 @@ def parse_args() -> argparse.Namespace:
 
 def retry_popen(cmd: str, log_file: Path) -> int:
     max_retries = 2
+    sleep_seconds = 10
+    retcode = -1
     for retry in range(max_retries):
-        # From time to time docker build may failed. Curl issues, or even push
-        # It will sleep progressively 5, 15, 30 and 50 seconds between retries
-        progressive_sleep = 5 * sum(i + 1 for i in range(retry))
-        if progressive_sleep:
-            logging.warning(
-                "The following command failed, sleep %s before retry: %s",
-                progressive_sleep,
-                cmd,
-            )
-            time.sleep(progressive_sleep)
         with TeePopen(
             cmd,
             log_file=log_file,
@@ -146,7 +132,14 @@ def retry_popen(cmd: str, log_file: Path) -> int:
             retcode = process.wait()
             if retcode == 0:
                 return 0
-
+            else:
+                # From time to time docker build may failed. Curl issues, or even push
+                logging.error(
+                    "The following command failed, sleep %s before retry: %s",
+                    sleep_seconds,
+                    cmd,
+                )
+                time.sleep(sleep_seconds)
     return retcode
 
 
@@ -377,21 +370,6 @@ def main():
     direct_urls: Dict[str, List[str]] = {}
 
     for arch, build_name in zip(ARCH, ("package_release", "package_aarch64")):
-        if args.bucket_prefix:
-            assert not args.allow_build_reuse
-            repo_urls[arch] = f"{args.bucket_prefix}/{build_name}"
-        elif args.sha:
-            # CreateRelease workflow only. TODO
-            version = args.version
-            repo_urls[arch] = (
-                f"{S3_DOWNLOAD}/{S3_BUILDS_BUCKET}/"
-                f"{version.major}.{version.minor}/{args.sha}/{build_name}"
-            )
-        else:
-            # In all other cases urls must be fetched from build reports. TODO: script needs refactoring
-            repo_urls[arch] = ""
-            assert args.allow_build_reuse
-
         if args.allow_build_reuse:
             # read s3 urls from pre-downloaded build reports
             if "clickhouse-server" in image_repo:
@@ -413,6 +391,17 @@ def main():
                 for url in urls
                 if any(package in url for package in PACKAGES) and "-dbg" not in url
             ]
+        elif args.bucket_prefix:
+            assert not args.allow_build_reuse
+            repo_urls[arch] = f"{args.bucket_prefix}/{build_name}"
+            print(f"Bucket prefix is set: Fetching packages from [{repo_urls}]")
+        else:
+            version = args.version
+            repo_urls[arch] = (
+                f"{S3_DOWNLOAD}/{S3_BUILDS_BUCKET}/"
+                f"{version.major}.{version.minor}/{git.sha}/{build_name}"
+            )
+            print(f"Fetching packages from [{repo_urls}]")
 
     if push:
         docker_login()

From 0ce81e7570acc49976a0e3242e7cda3a2f1fa510 Mon Sep 17 00:00:00 2001
From: Max K <maxkaynov@gmail.com>
Date: Mon, 29 Jul 2024 14:19:31 +0200
Subject: [PATCH 0977/1488] Create release workflow to use docker build script
 from master branch

---
 .github/actions/release/action.yml | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/.github/actions/release/action.yml b/.github/actions/release/action.yml
index 94f178738e3..c3897682a33 100644
--- a/.github/actions/release/action.yml
+++ b/.github/actions/release/action.yml
@@ -145,9 +145,7 @@ runs:
         cd "./tests/ci"
         python3 ./create_release.py --set-progress-started --progress "docker server release"
         export CHECK_NAME="Docker server image"
-        git checkout ${{ env.RELEASE_TAG }}
-        python3 docker_server.py --release-type auto --version ${{ env.RELEASE_TAG }} --check-name "$CHECK_NAME" ${{ ! inputs.dry-run && '--push' || '' }}
-        git checkout -
+        python3 docker_server.py --release-type auto --version ${{ env.RELEASE_TAG }} --check-name "$CHECK_NAME" --sha ${{ env.COMMIT_SHA }} ${{ ! inputs.dry-run && '--push' || '' }}
         python3 ./create_release.py --set-progress-completed
     - name: Docker clickhouse/clickhouse-keeper building
       if: ${{ inputs.type == 'patch' }}
@@ -156,19 +154,15 @@ runs:
         cd "./tests/ci"
         python3 ./create_release.py --set-progress-started --progress "docker keeper release"
         export CHECK_NAME="Docker keeper image"
-        git checkout ${{ env.RELEASE_TAG }}
-        python3 docker_server.py --release-type auto --version ${{ env.RELEASE_TAG }} --check-name "$CHECK_NAME" ${{ ! inputs.dry-run && '--push'  || '' }}
-        git checkout -
+        python3 docker_server.py --release-type auto --version ${{ env.RELEASE_TAG }} --check-name "$CHECK_NAME" --sha ${{ env.COMMIT_SHA }} ${{ ! inputs.dry-run && '--push'  || '' }}
         python3 ./create_release.py --set-progress-completed
     - name: Set current Release progress to Completed with OK
       shell: bash
       run: |
-        git checkout "$GITHUB_REF_NAME"
         python3 ./tests/ci/create_release.py --set-progress-started --progress "completed"
         python3 ./tests/ci/create_release.py --set-progress-completed
     - name: Post Slack Message
       if: ${{ !cancelled() }}
       shell: bash
       run: |
-        git checkout "$GITHUB_REF_NAME"
         python3 ./tests/ci/create_release.py --post-status ${{ inputs.dry-run && '--dry-run' || '' }}

From 54d413e09b12d26fb734378b52399d64489f3337 Mon Sep 17 00:00:00 2001
From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com>
Date: Mon, 29 Jul 2024 14:20:35 +0200
Subject: [PATCH 0978/1488] Delete extra file

---
 .../03202_dynamic_null_map_subcolumn.sql           | 14 --------------
 1 file changed, 14 deletions(-)
 delete mode 100644 tests/queries/0_stateless/03202_dynamic_null_map_subcolumn.sql

diff --git a/tests/queries/0_stateless/03202_dynamic_null_map_subcolumn.sql b/tests/queries/0_stateless/03202_dynamic_null_map_subcolumn.sql
deleted file mode 100644
index fe9cfc886e2..00000000000
--- a/tests/queries/0_stateless/03202_dynamic_null_map_subcolumn.sql
+++ /dev/null
@@ -1,14 +0,0 @@
-# Tags: long
-
-set allow_experimental_variant_type = 1;
-set use_variant_as_common_type = 1;
-set allow_experimental_dynamic_type = 1;
-set optimize_functions_to_subcolumns = 0;
-
-{% for engine in ['Memory', 'MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000', 'MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1'] -%}
-
-SELECT '--- {{ engine }} ---';
-
-
-
-{% endfor -%}

From fd2dc474a1073d6d151d6f508d16a2f48296004d Mon Sep 17 00:00:00 2001
From: Max K <maxkaynov@gmail.com>
Date: Mon, 29 Jul 2024 14:26:02 +0200
Subject: [PATCH 0979/1488] docker server fixes

---
 tests/ci/docker_server.py | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/tests/ci/docker_server.py b/tests/ci/docker_server.py
index 8ee0ed84120..479d63f1a3b 100644
--- a/tests/ci/docker_server.py
+++ b/tests/ci/docker_server.py
@@ -62,6 +62,12 @@ def parse_args() -> argparse.Namespace:
         help="a version to build, automaticaly got from version_helper, accepts either "
         "tag ('refs/tags/' is removed automatically) or a normal 22.2.2.2 format",
     )
+    parser.add_argument(
+        "--sha",
+        type=str,
+        default="",
+        help="sha of the commit to use packages from",
+    )
     parser.add_argument(
         "--release-type",
         type=str,
@@ -395,13 +401,17 @@ def main():
             assert not args.allow_build_reuse
             repo_urls[arch] = f"{args.bucket_prefix}/{build_name}"
             print(f"Bucket prefix is set: Fetching packages from [{repo_urls}]")
-        else:
+        elif args.sha:
             version = args.version
             repo_urls[arch] = (
                 f"{S3_DOWNLOAD}/{S3_BUILDS_BUCKET}/"
-                f"{version.major}.{version.minor}/{git.sha}/{build_name}"
+                f"{version.major}.{version.minor}/{args.sha}/{build_name}"
             )
             print(f"Fetching packages from [{repo_urls}]")
+        else:
+            assert (
+                False
+            ), "--sha, --bucket_prefix or --allow-build-reuse (to fetch packages from build report) must be provided"
 
     if push:
         docker_login()

From 82c06920e35238e6d411396e1d67d9fc603d5a04 Mon Sep 17 00:00:00 2001
From: Igor Nikonov <igor@clickhouse.com>
Date: Mon, 29 Jul 2024 12:27:12 +0000
Subject: [PATCH 0980/1488] Try to fix: ALL_CONNECTION_TRIES_FAILED with
 parallel replicas

---
 src/Client/ConnectionEstablisher.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Client/ConnectionEstablisher.cpp b/src/Client/ConnectionEstablisher.cpp
index 4bab643ed31..8cebe7a6183 100644
--- a/src/Client/ConnectionEstablisher.cpp
+++ b/src/Client/ConnectionEstablisher.cpp
@@ -38,7 +38,7 @@ void ConnectionEstablisher::run(ConnectionEstablisher::TryResult & result, std::
     try
     {
         ProfileEvents::increment(ProfileEvents::DistributedConnectionTries);
-        result.entry = pool->get(*timeouts, settings, /* force_connected = */ false);
+        result.entry = pool->get(*timeouts, settings, /* force_connected = */ true);
         AsyncCallbackSetter async_setter(&*result.entry, std::move(async_callback));
 
         UInt64 server_revision = 0;

From e13bc4e47aeb12584c13a7a0fa1e6aa5e59e9770 Mon Sep 17 00:00:00 2001
From: Blargian <shaunstruwig.ss@gmail.com>
Date: Mon, 29 Jul 2024 14:57:30 +0200
Subject: [PATCH 0981/1488] fix formatting issues

---
 .../functions/type-conversion-functions.md    | 50 +++++++++++--------
 1 file changed, 28 insertions(+), 22 deletions(-)

diff --git a/docs/en/sql-reference/functions/type-conversion-functions.md b/docs/en/sql-reference/functions/type-conversion-functions.md
index 339060a1ac4..5f569f1b5c0 100644
--- a/docs/en/sql-reference/functions/type-conversion-functions.md
+++ b/docs/en/sql-reference/functions/type-conversion-functions.md
@@ -313,7 +313,7 @@ Query:
 
 ``` sql
 SELECT
-    toUInt8OrNull('-8'),
+    toUInt8OrNull('8'),
     toUInt8OrNull('abc')
 FORMAT vertical;
 ```
@@ -323,7 +323,7 @@ Result:
 ```response
 Row 1:
 ──────
-toUInt8OrNull('-8'):  ᴺᵁᴸᴸ
+toUInt8OrNull('8'):   8
 toUInt8OrNull('abc'): ᴺᵁᴸᴸ
 ```
 
@@ -336,17 +336,18 @@ toUInt8OrNull('abc'): ᴺᵁᴸᴸ
 ## toUInt8OrDefault
 
 Like [`toUInt8`](#touint8), this function converts an input value to a value of type [UInt8](../data-types/int-uint.md) but returns the default value in case of an error.
+If no `default` value is passed then `0` is returned in case of an error.
 
 **Syntax**
 
 ```sql
-toUInt8OrDefault(expr, def)
+toUInt8OrDefault(expr[, default])
 ```
 
 **Arguments**
 
 - `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
-- `def` — The default value to return if parsing to type `UInt8` is unsuccessful. [UInt8](../data-types/int-uint.md).
+- `default` (optional) — The default value to return if parsing to type `UInt8` is unsuccessful. [UInt8](../data-types/int-uint.md).
 
 Supported arguments:
 - Values of type (U)Int8/16/32/64/128/256.
@@ -355,7 +356,7 @@ Supported arguments:
 
 Arguments for which the default value is returned:
 - String representations of Float32/64 values, including `NaN` and `Inf`.
-- String representations of binary and hexadecimal values, e.g. `SELECT toUInt8OrDefault('0xc0fe', CAST('-1', 'UInt8'));`.
+- String representations of binary and hexadecimal values, e.g. `SELECT toUInt8OrDefault('0xc0fe', CAST('0', 'UInt8'));`.
 
 :::note
 If the input value cannot be represented within the bounds of [UInt8](../data-types/int-uint.md), overflow or underflow of the result occurs.
@@ -364,7 +365,7 @@ This is not considered an error.
 
 **Returned value**
 
-- 8-bit unsigned integer value if successful, otherwise returns the default value. [UInt8](../data-types/int-uint.md).
+- 8-bit unsigned integer value if successful, otherwise returns the default value if passed or `0` if not. [UInt8](../data-types/int-uint.md).
 
 :::note
 - The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
@@ -585,17 +586,18 @@ toUInt16OrNull('abc'): ᴺᵁᴸᴸ
 ## toUInt16OrDefault
 
 Like [`toUInt16`](#touint16), this function converts an input value to a value of type [UInt16](../data-types/int-uint.md) but returns the default value in case of an error.
+If no `default` value is passed then `0` is returned in case of an error.
 
 **Syntax**
 
 ```sql
-toUInt16OrDefault(expr, def)
+toUInt16OrDefault(expr[, default])
 ```
 
 **Arguments**
 
 - `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
-- `def` — The default value to return if parsing to type `UInt16` is unsuccessful. [UInt16](../data-types/int-uint.md).
+- `default` (optional) — The default value to return if parsing to type `UInt16` is unsuccessful. [UInt16](../data-types/int-uint.md).
 
 Supported arguments:
 - Values of type (U)Int8/16/32/64/128/256.
@@ -613,12 +615,12 @@ This is not considered an error.
 
 **Returned value**
 
-- 16-bit unsigned integer value if successful, otherwise returns the default value. [UInt16](../data-types/int-uint.md).
+- 16-bit unsigned integer value if successful, otherwise returns the default value if passed or `0` if not. [UInt16](../data-types/int-uint.md).
 
 :::note
 - The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
 - The default value type should be the same as the cast type.
-  :::
+:::
 
 **Example**
 
@@ -835,17 +837,18 @@ toUInt32OrNull('abc'): ᴺᵁᴸᴸ
 ## toUInt32OrDefault
 
 Like [`toUInt32`](#touint32), this function converts an input value to a value of type [UInt32](../data-types/int-uint.md) but returns the default value in case of an error.
+If no `default` value is passed then `0` is returned in case of an error.
 
 **Syntax**
 
 ```sql
-toUInt32OrDefault(expr, def)
+toUInt32OrDefault(expr[, default])
 ```
 
 **Arguments**
 
 - `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
-- `def` — The default value to return if parsing to type `UInt32` is unsuccessful. [UInt32](../data-types/int-uint.md).
+- `default` (optional) — The default value to return if parsing to type `UInt32` is unsuccessful. [UInt32](../data-types/int-uint.md).
 
 Supported arguments:
 - Values of type (U)Int8/16/32/64/128/256.
@@ -863,7 +866,7 @@ This is not considered an error.
 
 **Returned value**
 
-- 32-bit unsigned integer value if successful, otherwise returns the default value. [UInt32](../data-types/int-uint.md).
+- 32-bit unsigned integer value if successful, otherwise returns the default value if passed or `0` if not. [UInt32](../data-types/int-uint.md).
 
 :::note
 - The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
@@ -1084,17 +1087,18 @@ toUInt64OrNull('abc'): ᴺᵁᴸᴸ
 ## toUInt64OrDefault
 
 Like [`toUInt64`](#touint64), this function converts an input value to a value of type [UInt64](../data-types/int-uint.md) but returns the default value in case of an error.
+If no `default` value is passed then `0` is returned in case of an error.
 
 **Syntax**
 
 ```sql
-toUInt64OrDefault(expr, def)
+toUInt64OrDefault(expr[, default])
 ```
 
 **Arguments**
 
 - `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
-- `def` — The default value to return if parsing to type `UInt64` is unsuccessful. [UInt64](../data-types/int-uint.md).
+- `defauult` (optional) — The default value to return if parsing to type `UInt64` is unsuccessful. [UInt64](../data-types/int-uint.md).
 
 Supported arguments:
 - Values of type (U)Int8/16/32/64/128/256.
@@ -1112,7 +1116,7 @@ This is not considered an error.
 
 **Returned value**
 
-- 64-bit unsigned integer value if successful, otherwise returns the default value. [UInt64](../data-types/int-uint.md).
+- 64-bit unsigned integer value if successful, otherwise returns the default value if passed or `0` if not. [UInt64](../data-types/int-uint.md).
 
 :::note
 - The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
@@ -1332,17 +1336,18 @@ toUInt128OrNull('abc'): ᴺᵁᴸᴸ
 ## toUInt128OrDefault
 
 Like [`toUInt128`](#toint128), this function converts an input value to a value of type [UInt128](../data-types/int-uint.md) but returns the default value in case of an error.
+If no `default` value is passed then `0` is returned in case of an error.
 
 **Syntax**
 
 ```sql
-toUInt128OrDefault(expr, def)
+toUInt128OrDefault(expr[, default])
 ```
 
 **Arguments**
 
 - `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
-- `def` — The default value to return if parsing to type `UInt128` is unsuccessful. [UInt128](../data-types/int-uint.md).
+- `default` (optional) — The default value to return if parsing to type `UInt128` is unsuccessful. [UInt128](../data-types/int-uint.md).
 
 Supported arguments:
 - (U)Int8/16/32/64/128/256.
@@ -1360,7 +1365,7 @@ This is not considered an error.
 
 **Returned value**
 
-- 128-bit unsigned integer value if successful, otherwise returns the default value. [UInt128](../data-types/int-uint.md).
+- 128-bit unsigned integer value if successful, otherwise returns the default value if passed or `0` if not. [UInt128](../data-types/int-uint.md).
 
 :::note
 - The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.
@@ -1580,17 +1585,18 @@ toUInt256OrNull('abc'): ᴺᵁᴸᴸ
 ## toUInt256OrDefault
 
 Like [`toUInt256`](#touint256), this function converts an input value to a value of type [UInt256](../data-types/int-uint.md) but returns the default value in case of an error.
+If no `default` value is passed then `0` is returned in case of an error.
 
 **Syntax**
 
 ```sql
-toUInt256OrDefault(expr, def)
+toUInt256OrDefault(expr[, default])
 ```
 
 **Arguments**
 
 - `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
-- `def` — The default value to return if parsing to type `UInt256` is unsuccessful. [UInt256](../data-types/int-uint.md).
+- `default` (optional) — The default value to return if parsing to type `UInt256` is unsuccessful. [UInt256](../data-types/int-uint.md).
 
 Supported arguments:
 - Values of type (U)Int8/16/32/64/128/256.
@@ -1608,7 +1614,7 @@ This is not considered an error.
 
 **Returned value**
 
-- 256-bit unsigned integer value if successful, otherwise returns the default value. [UInt256](../data-types/int-uint.md).
+- 256-bit unsigned integer value if successful, otherwise returns the default value if passed or `0` if not. [UInt256](../data-types/int-uint.md).
 
 :::note
 - The function uses [rounding towards zero](https://en.wikipedia.org/wiki/Rounding#Rounding_towards_zero), meaning it truncates fractional digits of numbers.

From 327fb40178fcff7051e17ca47d8446e5671b5383 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Mon, 29 Jul 2024 13:21:22 +0000
Subject: [PATCH 0982/1488] Add a test.

---
 ...e_function_with_if_implicit_cast.reference | 32 ++++++++--------
 ...function_with_if_return_type_bug.reference |  9 +++++
 ...egate_function_with_if_return_type_bug.sql | 37 +++++++++++++++++++
 3 files changed, 62 insertions(+), 16 deletions(-)
 create mode 100644 tests/queries/0_stateless/03210_optimize_rewrite_aggregate_function_with_if_return_type_bug.reference
 create mode 100644 tests/queries/0_stateless/03210_optimize_rewrite_aggregate_function_with_if_return_type_bug.sql

diff --git a/tests/queries/0_stateless/03132_rewrite_aggregate_function_with_if_implicit_cast.reference b/tests/queries/0_stateless/03132_rewrite_aggregate_function_with_if_implicit_cast.reference
index 070a6c9e21c..8289ab68ae4 100644
--- a/tests/queries/0_stateless/03132_rewrite_aggregate_function_with_if_implicit_cast.reference
+++ b/tests/queries/0_stateless/03132_rewrite_aggregate_function_with_if_implicit_cast.reference
@@ -30,14 +30,14 @@ QUERY id: 0
       FUNCTION id: 15, function_name: toTypeName, function_type: ordinary, result_type: String
         ARGUMENTS
           LIST id: 16, nodes: 1
-            FUNCTION id: 2, function_name: anyIf, function_type: aggregate, result_type: Nullable(Int128)
+            FUNCTION id: 17, function_name: anyIf, function_type: aggregate, result_type: Nullable(Int128)
               ARGUMENTS
-                LIST id: 3, nodes: 2
-                  FUNCTION id: 4, function_name: _CAST, function_type: ordinary, result_type: Nullable(Int128)
+                LIST id: 18, nodes: 2
+                  FUNCTION id: 19, function_name: _CAST, function_type: ordinary, result_type: Nullable(Int128)
                     ARGUMENTS
-                      LIST id: 5, nodes: 2
+                      LIST id: 20, nodes: 2
                         COLUMN id: 6, column_name: number, result_type: UInt64, source_id: 7
-                        CONSTANT id: 8, constant_value: \'Nullable(Int128)\', constant_value_type: String
+                        CONSTANT id: 21, constant_value: \'Nullable(Int128)\', constant_value_type: String
                   FUNCTION id: 9, function_name: equals, function_type: ordinary, result_type: UInt8
                     ARGUMENTS
                       LIST id: 10, nodes: 2
@@ -50,8 +50,8 @@ QUERY id: 0
   JOIN TREE
     TABLE_FUNCTION id: 7, alias: __table1, table_function_name: numbers
       ARGUMENTS
-        LIST id: 17, nodes: 1
-          CONSTANT id: 18, constant_value: UInt64_100, constant_value_type: UInt8
+        LIST id: 22, nodes: 1
+          CONSTANT id: 23, constant_value: UInt64_100, constant_value_type: UInt8
 SELECT any(if((number % 10) = 5, CAST(NULL, 'Nullable(Int128)'), number)) AS a, toTypeName(a) FROM numbers(100) AS a;
 0	Nullable(Int128)
 EXPLAIN QUERY TREE SELECT any(if((number % 10) = 5, CAST(NULL, 'Nullable(Int128)'), number)) AS a, toTypeName(a) FROM numbers(100);
@@ -84,17 +84,17 @@ QUERY id: 0
       FUNCTION id: 17, function_name: toTypeName, function_type: ordinary, result_type: String
         ARGUMENTS
           LIST id: 18, nodes: 1
-            FUNCTION id: 2, function_name: anyIf, function_type: aggregate, result_type: Nullable(Int128)
+            FUNCTION id: 19, function_name: anyIf, function_type: aggregate, result_type: Nullable(Int128)
               ARGUMENTS
-                LIST id: 3, nodes: 2
-                  FUNCTION id: 4, function_name: _CAST, function_type: ordinary, result_type: Nullable(Int128)
+                LIST id: 20, nodes: 2
+                  FUNCTION id: 21, function_name: _CAST, function_type: ordinary, result_type: Nullable(Int128)
                     ARGUMENTS
-                      LIST id: 5, nodes: 2
+                      LIST id: 22, nodes: 2
                         COLUMN id: 6, column_name: number, result_type: UInt64, source_id: 7
-                        CONSTANT id: 8, constant_value: \'Nullable(Int128)\', constant_value_type: String
-                  FUNCTION id: 9, function_name: not, function_type: ordinary, result_type: UInt8
+                        CONSTANT id: 23, constant_value: \'Nullable(Int128)\', constant_value_type: String
+                  FUNCTION id: 24, function_name: not, function_type: ordinary, result_type: UInt8
                     ARGUMENTS
-                      LIST id: 10, nodes: 1
+                      LIST id: 25, nodes: 1
                         FUNCTION id: 11, function_name: equals, function_type: ordinary, result_type: UInt8
                           ARGUMENTS
                             LIST id: 12, nodes: 2
@@ -107,5 +107,5 @@ QUERY id: 0
   JOIN TREE
     TABLE_FUNCTION id: 7, alias: __table1, table_function_name: numbers
       ARGUMENTS
-        LIST id: 19, nodes: 1
-          CONSTANT id: 20, constant_value: UInt64_100, constant_value_type: UInt8
+        LIST id: 26, nodes: 1
+          CONSTANT id: 27, constant_value: UInt64_100, constant_value_type: UInt8
diff --git a/tests/queries/0_stateless/03210_optimize_rewrite_aggregate_function_with_if_return_type_bug.reference b/tests/queries/0_stateless/03210_optimize_rewrite_aggregate_function_with_if_return_type_bug.reference
new file mode 100644
index 00000000000..0693acb9168
--- /dev/null
+++ b/tests/queries/0_stateless/03210_optimize_rewrite_aggregate_function_with_if_return_type_bug.reference
@@ -0,0 +1,9 @@
+03
+AggregateFunction(count, Nullable(UInt64))
+function_name: _CAST, function_type: ordinary, result_type: AggregateFunction(count, Nullable(UInt64))
+function_name: countStateIf, function_type: aggregate, result_type: AggregateFunction(count, UInt64)
+constant_value: \'AggregateFunction(count, Nullable(UInt64))\', constant_value_type: String
+AggregateFunction(uniq, Nullable(UInt64))
+010003000000007518F0A8E7830665
+function_name: uniqState, function_type: aggregate, result_type: AggregateFunction(uniq, Nullable(UInt64))
+----
diff --git a/tests/queries/0_stateless/03210_optimize_rewrite_aggregate_function_with_if_return_type_bug.sql b/tests/queries/0_stateless/03210_optimize_rewrite_aggregate_function_with_if_return_type_bug.sql
new file mode 100644
index 00000000000..b620a6434bf
--- /dev/null
+++ b/tests/queries/0_stateless/03210_optimize_rewrite_aggregate_function_with_if_return_type_bug.sql
@@ -0,0 +1,37 @@
+-- For function count, rewrite countState to countStateIf changes the type from AggregateFunction(count, Nullable(UInt64)) to AggregateFunction(count, UInt64)
+-- We can cast AggregateFunction(count, UInt64) back to AggregateFunction(count, Nullable(UInt64)) with additional _CAST
+select hex(countState(if(toNullable(number % 2 = 0), number, null))) from numbers(5) settings optimize_rewrite_aggregate_function_with_if=1;
+select toTypeName(countState(if(toNullable(number % 2 = 0), number, null))) from numbers(5) settings optimize_rewrite_aggregate_function_with_if=1;
+select arrayStringConcat(arraySlice(splitByString(', ', trimLeft(explain)), 2), ', ') from (explain query tree select hex(countState(if(toNullable(number % 2 = 0), number, null))) from numbers(5) settings optimize_rewrite_aggregate_function_with_if=1) where explain like '%AggregateFunction%';
+
+-- For function uniq, rewrite uniqState to uniqStateIf changes the type from AggregateFunction(uniq, Nullable(UInt64)) to AggregateFunction(uniq, UInt64)
+-- We can't cast AggregateFunction(uniq, UInt64) back to AggregateFunction(uniq, Nullable(UInt64)) so rewrite is not happening.
+select toTypeName(uniqState(if(toNullable(number % 2 = 0), number, null))) from numbers(5) settings optimize_rewrite_aggregate_function_with_if=1;
+select hex(uniqState(if(toNullable(number % 2 = 0), number, null))) from numbers(5) settings optimize_rewrite_aggregate_function_with_if=1;
+select arrayStringConcat(arraySlice(splitByString(', ', trimLeft(explain)), 2), ', ') from (explain query tree select hex(uniqState(if(toNullable(number % 2 = 0), number, null))) from numbers(5) settings optimize_rewrite_aggregate_function_with_if=1) where explain like '%AggregateFunction%';
+
+select '----';
+
+CREATE TABLE a
+(
+    `a_id` String
+)
+ENGINE = MergeTree
+PARTITION BY tuple()
+ORDER BY tuple();
+
+
+CREATE TABLE b
+(
+    `b_id` AggregateFunction(uniq, Nullable(String))
+)
+ENGINE = AggregatingMergeTree
+PARTITION BY tuple()
+ORDER BY tuple();
+
+CREATE MATERIALIZED VIEW mv TO b
+(
+    `b_id` AggregateFunction(uniq, Nullable(String))
+)
+AS SELECT uniqState(if(a_id != '', a_id, NULL)) AS b_id
+FROM a;

From b271a9dc8ca85ca96ac5f15c583c142f7d63696e Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Mon, 29 Jul 2024 15:44:25 +0200
Subject: [PATCH 0983/1488] Don't detach from thread group if running from main
 thread

---
 src/Interpreters/Aggregator.cpp | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp
index e073b7a49b6..90bbde8be35 100644
--- a/src/Interpreters/Aggregator.cpp
+++ b/src/Interpreters/Aggregator.cpp
@@ -2997,7 +2997,11 @@ void Aggregator::mergeBlocks(BucketToBlocks bucket_to_blocks, AggregatedDataVari
 
         std::unique_ptr<ThreadPool> thread_pool;
         if (max_threads > 1 && total_input_rows > 100000)    /// TODO Make a custom threshold.
-            thread_pool = std::make_unique<ThreadPool>(CurrentMetrics::AggregatorThreads, CurrentMetrics::AggregatorThreadsActive, CurrentMetrics::AggregatorThreadsScheduled, max_threads);
+            thread_pool = std::make_unique<ThreadPool>(
+                CurrentMetrics::AggregatorThreads,
+                CurrentMetrics::AggregatorThreadsActive,
+                CurrentMetrics::AggregatorThreadsScheduled,
+                max_threads);
 
         for (const auto & bucket_blocks : bucket_to_blocks)
         {
@@ -3009,7 +3013,10 @@ void Aggregator::mergeBlocks(BucketToBlocks bucket_to_blocks, AggregatedDataVari
             result.aggregates_pools.push_back(std::make_shared<Arena>());
             Arena * aggregates_pool = result.aggregates_pools.back().get();
 
-            auto task = [group = CurrentThread::getGroup(), bucket, &merge_bucket, aggregates_pool]{ merge_bucket(bucket, aggregates_pool, group); };
+            /// if we don't use thread pool we don't need to attach and definitely don't want to detach from the thread group
+            /// because this thread is already attached
+            auto task = [group = thread_pool != nullptr ? CurrentThread::getGroup() : nullptr, bucket, &merge_bucket, aggregates_pool]
+            { merge_bucket(bucket, aggregates_pool, group); };
 
             if (thread_pool)
                 thread_pool->scheduleOrThrowOnError(task);

From efa218b5342200ff7986d8356ec9d9ea9656e373 Mon Sep 17 00:00:00 2001
From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com>
Date: Mon, 29 Jul 2024 15:54:54 +0200
Subject: [PATCH 0984/1488] address comments

---
 docs/en/operations/settings/settings.md              |  2 +-
 src/Core/Settings.h                                  |  2 +-
 src/Core/SettingsChangesHistory.cpp                  |  2 +-
 src/Interpreters/InterpreterCreateQuery.cpp          | 10 ++++++++--
 .../test_restore_external_engines/test.py            | 12 ++++++------
 5 files changed, 17 insertions(+), 11 deletions(-)

diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md
index 65b8df7a9e2..8739414464e 100644
--- a/docs/en/operations/settings/settings.md
+++ b/docs/en/operations/settings/settings.md
@@ -5617,6 +5617,6 @@ Default value: `False`
 
 ## restore_replace_external_table_functions_to_null
 
-For testing purposes. Replaces all external engines to Null to not initiate external connections.
+For testing purposes. Replaces all external table functions to Null to not initiate external connections.
 
 Default value: `False`
\ No newline at end of file
diff --git a/src/Core/Settings.h b/src/Core/Settings.h
index 36b07a63011..6b090b800fe 100644
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@@ -893,7 +893,7 @@ class IColumn;
     M(Bool, optimize_distinct_in_order, true, "Enable DISTINCT optimization if some columns in DISTINCT form a prefix of sorting. For example, prefix of sorting key in merge tree or ORDER BY statement", 0) \
     M(Bool, keeper_map_strict_mode, false, "Enforce additional checks during operations on KeeperMap. E.g. throw an exception on an insert for already existing key", 0) \
     M(UInt64, extract_key_value_pairs_max_pairs_per_row, 1000, "Max number of pairs that can be produced by the `extractKeyValuePairs` function. Used as a safeguard against consuming too much memory.", 0) ALIAS(extract_kvp_max_pairs_per_row) \
-    M(Bool, restore_replace_external_engines_to_null, false, "Replace all the External table engines to Null on restore. Useful for testing purposes", 0) \
+    M(Bool, restore_replace_external_engines_to_null, false, "Replace all the external table engines to Null on restore. Useful for testing purposes", 0) \
     M(Bool, restore_replace_external_table_functions_to_null, false, "Replace all table functions to Null on restore. Useful for testing purposes", 0) \
     \
     \
diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp
index e79c4cc9bb6..8bea0b1eed3 100644
--- a/src/Core/SettingsChangesHistory.cpp
+++ b/src/Core/SettingsChangesHistory.cpp
@@ -82,7 +82,7 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
               {"postgresql_connection_attempt_timeout", 2, 2, "Allow to control 'connect_timeout' parameter of PostgreSQL connection."},
               {"postgresql_connection_pool_retries", 2, 2, "Allow to control the number of retries in PostgreSQL connection pool."},
               {"restore_replace_external_table_functions_to_null", false, false, "New setting."},
-              {"restore_replace_external_engins_to_null", false, false, "New setting."}
+              {"restore_replace_external_engines_to_null", false, false, "New setting."}
               }},
     {"24.6", {{"materialize_skip_indexes_on_insert", true, true, "Added new setting to allow to disable materialization of skip indexes on insert"},
               {"materialize_statistics_on_insert", true, true, "Added new setting to allow to disable materialization of statistics on insert"},
diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp
index e1cb81612d2..a5eb3a83365 100644
--- a/src/Interpreters/InterpreterCreateQuery.cpp
+++ b/src/Interpreters/InterpreterCreateQuery.cpp
@@ -960,7 +960,7 @@ namespace
         storage.set(storage.engine, engine_ast);
     }
 
-    void setNullTableEngine(ASTStorage &storage)
+    void setNullTableEngine(ASTStorage & storage)
     {
         auto engine_ast = std::make_shared<ASTFunction>();
         engine_ast->name = "Null";
@@ -972,7 +972,6 @@ namespace
 
 void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const
 {
-
     if (create.as_table_function)
     {
         if (getContext()->getSettingsRef().restore_replace_external_table_functions_to_null)
@@ -1032,6 +1031,13 @@ void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const
                 /// Some part of storage definition (such as PARTITION BY) is specified, but ENGINE is not: just set default one.
                 setDefaultTableEngine(*to_engine, getContext()->getSettingsRef().default_table_engine.value);
             }
+            /// For external tables with restore_replace_external_engine_to_null setting we replace external engines to
+            /// Null table engine.
+            else if (getContext()->getSettingsRef().restore_replace_external_engines_to_null)
+            {
+                if (StorageFactory::instance().getStorageFeatures(create.storage->engine->name).source_access_type != AccessType::NONE)
+                    setNullTableEngine(*create.storage);
+            }
             return;
         }
     }
diff --git a/tests/integration/test_restore_external_engines/test.py b/tests/integration/test_restore_external_engines/test.py
index be2cae334e2..eb88da6b61f 100644
--- a/tests/integration/test_restore_external_engines/test.py
+++ b/tests/integration/test_restore_external_engines/test.py
@@ -95,20 +95,20 @@ def start_cluster():
 def test_restore_table(start_cluster):
     fill_tables(cluster, "replicated")
     backup_name = new_backup_name()
-    node2.query(f"SYSTEM SYNC DATABASE REPLICA replicated;")
+    node2.query(f"SYSTEM SYNC DATABASE REPLICA replicated")
 
     node2.query(f"BACKUP DATABASE replicated TO {backup_name}")
 
     node2.query("DROP TABLE replicated.mysql_schema_inference_engine")
     node2.query("DROP TABLE replicated.mysql_schema_inference_function")
 
-    node3.query(f"SYSTEM SYNC DATABASE REPLICA replicated;")
+    node3.query(f"SYSTEM SYNC DATABASE REPLICA replicated")
 
     assert node3.query("EXISTS replicated.mysql_schema_inference_engine") == "0\n"
     assert node3.query("EXISTS replicated.mysql_schema_inference_function") == "0\n"
 
     node3.query(f"RESTORE DATABASE replicated FROM {backup_name} SETTINGS allow_different_database_def=true")
-    node1.query(f"SYSTEM SYNC DATABASE REPLICA replicated;")
+    node1.query(f"SYSTEM SYNC DATABASE REPLICA replicated")
 
     assert node1.query("SELECT count(), sum(id) FROM replicated.mysql_schema_inference_engine") == "1\t100\n"
     assert node1.query("SELECT count(), sum(id) FROM replicated.mysql_schema_inference_function") == "1\t100\n"
@@ -120,20 +120,20 @@ def test_restore_table_null(start_cluster):
     fill_tables(cluster, "replicated2")
 
     backup_name = new_backup_name()
-    node2.query(f"SYSTEM SYNC DATABASE REPLICA replicated2;")
+    node2.query(f"SYSTEM SYNC DATABASE REPLICA replicated2")
 
     node2.query(f"BACKUP DATABASE replicated2 TO {backup_name}")
 
     node2.query("DROP TABLE replicated2.mysql_schema_inference_engine")
     node2.query("DROP TABLE replicated2.mysql_schema_inference_function")
 
-    node3.query(f"SYSTEM SYNC DATABASE REPLICA replicated2;")
+    node3.query(f"SYSTEM SYNC DATABASE REPLICA replicated2")
 
     assert node3.query("EXISTS replicated2.mysql_schema_inference_engine") == "0\n"
     assert node3.query("EXISTS replicated2.mysql_schema_inference_function") == "0\n"
 
     node3.query(f"RESTORE DATABASE replicated2 FROM {backup_name} SETTINGS allow_different_database_def=1, allow_different_table_def=1 SETTINGS restore_replace_external_engines_to_null=1, restore_replace_external_table_functions_to_null=1")
-    node1.query(f"SYSTEM SYNC DATABASE REPLICA replicated2;")
+    node1.query(f"SYSTEM SYNC DATABASE REPLICA replicated2")
 
     assert node1.query("SELECT count(), sum(id) FROM replicated2.mysql_schema_inference_engine") == "0\t0\n"
     assert node1.query("SELECT count(), sum(id) FROM replicated2.mysql_schema_inference_function") == "0\t0\n"

From b7c78465562724bcf036fd5e5f9051fb05702360 Mon Sep 17 00:00:00 2001
From: avogar <avogar@clickhouse.com>
Date: Mon, 29 Jul 2024 13:55:26 +0000
Subject: [PATCH 0985/1488] Add smaller versions of tests

---
 ...dynamic_read_subcolumns_small.reference.j2 | 2211 +++++++++++++++++
 ...03036_dynamic_read_subcolumns_small.sql.j2 |   45 +
 .../03037_dynamic_merges_small.reference.j2   |  112 +
 .../03037_dynamic_merges_small.sql.j2         |   44 +
 4 files changed, 2412 insertions(+)
 create mode 100644 tests/queries/0_stateless/03036_dynamic_read_subcolumns_small.reference.j2
 create mode 100644 tests/queries/0_stateless/03036_dynamic_read_subcolumns_small.sql.j2
 create mode 100644 tests/queries/0_stateless/03037_dynamic_merges_small.reference.j2
 create mode 100644 tests/queries/0_stateless/03037_dynamic_merges_small.sql.j2

diff --git a/tests/queries/0_stateless/03036_dynamic_read_subcolumns_small.reference.j2 b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_small.reference.j2
new file mode 100644
index 00000000000..3d814e1205a
--- /dev/null
+++ b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_small.reference.j2
@@ -0,0 +1,2211 @@
+Array(Array(Dynamic))
+Array(Variant(String, UInt64))
+None
+String
+UInt64
+20
+20
+20
+20
+0
+0
+20
+20
+10
+10
+20
+0
+0	0	\N	[]
+1	1	\N	[]
+2	2	\N	[]
+3	3	\N	[]
+4	4	\N	[]
+5	5	\N	[]
+6	6	\N	[]
+7	7	\N	[]
+8	8	\N	[]
+9	9	\N	[]
+[[0]]	\N	\N	[]
+str_10	\N	str_10	[]
+[[0,1]]	\N	\N	[]
+str_11	\N	str_11	[]
+[[0,1,2]]	\N	\N	[]
+str_12	\N	str_12	[]
+[[0,1,2,3]]	\N	\N	[]
+str_13	\N	str_13	[]
+[[0,1,2,3,4]]	\N	\N	[]
+str_14	\N	str_14	[]
+[[0,1,2,3,4,5]]	\N	\N	[]
+str_15	\N	str_15	[]
+[[0,1,2,3,4,5,6]]	\N	\N	[]
+str_16	\N	str_16	[]
+[[0,1,2,3,4,5,6,7]]	\N	\N	[]
+str_17	\N	str_17	[]
+[[0,1,2,3,4,5,6,7,8]]	\N	\N	[]
+str_18	\N	str_18	[]
+[[0,1,2,3,4,5,6,7,8,9]]	\N	\N	[]
+str_19	\N	str_19	[]
+[20]	\N	\N	[20]
+['str_21','str_21']	\N	\N	['str_21','str_21']
+[22,22,22]	\N	\N	[22,22,22]
+[23,23,23,23]	\N	\N	[23,23,23,23]
+[24,24,24,24,24]	\N	\N	[24,24,24,24,24]
+[25,25,25,25,25,25]	\N	\N	[25,25,25,25,25,25]
+[26,26,26,26,26,26,26]	\N	\N	[26,26,26,26,26,26,26]
+[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]	\N	\N	[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]
+[28,28,28,28,28,28,28,28,28]	\N	\N	[28,28,28,28,28,28,28,28,28]
+[29,29,29,29,29,29,29,29,29,29]	\N	\N	[29,29,29,29,29,29,29,29,29,29]
+\N	\N	\N	[]
+\N	\N	\N	[]
+\N	\N	\N	[]
+\N	\N	\N	[]
+\N	\N	\N	[]
+\N	\N	\N	[]
+\N	\N	\N	[]
+\N	\N	\N	[]
+\N	\N	\N	[]
+\N	\N	\N	[]
+[40]	\N	\N	[40]
+41	41	\N	[]
+\N	\N	\N	[]
+str_43	\N	str_43	[]
+[44,44,44,44,44]	\N	\N	[44,44,44,44,44]
+45	45	\N	[]
+\N	\N	\N	[]
+str_47	\N	str_47	[]
+['str_48','str_48','str_48','str_48','str_48','str_48','str_48','str_48','str_48']	\N	\N	['str_48','str_48','str_48','str_48','str_48','str_48','str_48','str_48','str_48']
+49	49	\N	[]
+\N	\N	\N	[]
+str_51	\N	str_51	[]
+[52,52,52]	\N	\N	[52,52,52]
+53	53	\N	[]
+\N	\N	\N	[]
+str_55	\N	str_55	[]
+[56,56,56,56,56,56,56]	\N	\N	[56,56,56,56,56,56,56]
+57	57	\N	[]
+\N	\N	\N	[]
+str_59	\N	str_59	[]
+[60]	\N	\N	[60]
+61	61	\N	[]
+\N	\N	\N	[]
+str_63	\N	str_63	[]
+[64,64,64,64,64]	\N	\N	[64,64,64,64,64]
+65	65	\N	[]
+\N	\N	\N	[]
+str_67	\N	str_67	[]
+[68,68,68,68,68,68,68,68,68]	\N	\N	[68,68,68,68,68,68,68,68,68]
+69	69	\N	[]
+\N	\N	\N	[]
+str_71	\N	str_71	[]
+[NULL,NULL,NULL]	\N	\N	[NULL,NULL,NULL]
+73	73	\N	[]
+\N	\N	\N	[]
+str_75	\N	str_75	[]
+[76,76,76,76,76,76,76]	\N	\N	[76,76,76,76,76,76,76]
+77	77	\N	[]
+\N	\N	\N	[]
+str_79	\N	str_79	[]
+0	\N	[]
+1	\N	[]
+2	\N	[]
+3	\N	[]
+4	\N	[]
+5	\N	[]
+6	\N	[]
+7	\N	[]
+8	\N	[]
+9	\N	[]
+\N	str_10	[]
+\N	\N	[]
+\N	str_11	[]
+\N	\N	[]
+\N	str_12	[]
+\N	\N	[]
+\N	str_13	[]
+\N	\N	[]
+\N	str_14	[]
+\N	\N	[]
+\N	str_15	[]
+\N	\N	[]
+\N	str_16	[]
+\N	\N	[]
+\N	str_17	[]
+\N	\N	[]
+\N	str_18	[]
+\N	\N	[]
+\N	str_19	[]
+\N	\N	[]
+\N	\N	[20]
+\N	\N	['str_21','str_21']
+\N	\N	[22,22,22]
+\N	\N	[23,23,23,23]
+\N	\N	[24,24,24,24,24]
+\N	\N	[25,25,25,25,25,25]
+\N	\N	[26,26,26,26,26,26,26]
+\N	\N	[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]
+\N	\N	[28,28,28,28,28,28,28,28,28]
+\N	\N	[29,29,29,29,29,29,29,29,29,29]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[40]
+41	\N	[]
+\N	\N	[]
+\N	str_43	[]
+\N	\N	[44,44,44,44,44]
+45	\N	[]
+\N	\N	[]
+\N	str_47	[]
+\N	\N	['str_48','str_48','str_48','str_48','str_48','str_48','str_48','str_48','str_48']
+49	\N	[]
+\N	\N	[]
+\N	str_51	[]
+\N	\N	[52,52,52]
+53	\N	[]
+\N	\N	[]
+\N	str_55	[]
+\N	\N	[56,56,56,56,56,56,56]
+57	\N	[]
+\N	\N	[]
+\N	str_59	[]
+\N	\N	[60]
+61	\N	[]
+\N	\N	[]
+\N	str_63	[]
+\N	\N	[64,64,64,64,64]
+65	\N	[]
+\N	\N	[]
+\N	str_67	[]
+\N	\N	[68,68,68,68,68,68,68,68,68]
+69	\N	[]
+\N	\N	[]
+\N	str_71	[]
+\N	\N	[NULL,NULL,NULL]
+73	\N	[]
+\N	\N	[]
+\N	str_75	[]
+\N	\N	[76,76,76,76,76,76,76]
+77	\N	[]
+\N	\N	[]
+\N	str_79	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+0	0	\N	[]	0	[]
+1	1	\N	[]	0	[]
+2	2	\N	[]	0	[]
+3	3	\N	[]	0	[]
+4	4	\N	[]	0	[]
+5	5	\N	[]	0	[]
+6	6	\N	[]	0	[]
+7	7	\N	[]	0	[]
+8	8	\N	[]	0	[]
+9	9	\N	[]	0	[]
+str_10	\N	\N	[]	0	[]
+[[0]]	\N	\N	[]	0	[]
+str_11	\N	\N	[]	0	[]
+[[0,1]]	\N	\N	[]	0	[]
+str_12	\N	\N	[]	0	[]
+[[0,1,2]]	\N	\N	[]	0	[]
+str_13	\N	\N	[]	0	[]
+[[0,1,2,3]]	\N	\N	[]	0	[]
+str_14	\N	\N	[]	0	[]
+[[0,1,2,3,4]]	\N	\N	[]	0	[]
+str_15	\N	\N	[]	0	[]
+[[0,1,2,3,4,5]]	\N	\N	[]	0	[]
+str_16	\N	\N	[]	0	[]
+[[0,1,2,3,4,5,6]]	\N	\N	[]	0	[]
+str_17	\N	\N	[]	0	[]
+[[0,1,2,3,4,5,6,7]]	\N	\N	[]	0	[]
+str_18	\N	\N	[]	0	[]
+[[0,1,2,3,4,5,6,7,8]]	\N	\N	[]	0	[]
+str_19	\N	\N	[]	0	[]
+[[0,1,2,3,4,5,6,7,8,9]]	\N	\N	[]	0	[]
+[20]	\N	\N	[20]	1	[20]
+['str_21','str_21']	\N	\N	['str_21','str_21']	2	[NULL,NULL]
+[22,22,22]	\N	\N	[22,22,22]	3	[22,22,22]
+[23,23,23,23]	\N	\N	[23,23,23,23]	4	[23,23,23,23]
+[24,24,24,24,24]	\N	\N	[24,24,24,24,24]	5	[24,24,24,24,24]
+[25,25,25,25,25,25]	\N	\N	[25,25,25,25,25,25]	6	[25,25,25,25,25,25]
+[26,26,26,26,26,26,26]	\N	\N	[26,26,26,26,26,26,26]	7	[26,26,26,26,26,26,26]
+[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]	\N	\N	[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]	8	[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]
+[28,28,28,28,28,28,28,28,28]	\N	\N	[28,28,28,28,28,28,28,28,28]	9	[28,28,28,28,28,28,28,28,28]
+[29,29,29,29,29,29,29,29,29,29]	\N	\N	[29,29,29,29,29,29,29,29,29,29]	10	[29,29,29,29,29,29,29,29,29,29]
+\N	\N	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+[40]	\N	\N	[40]	1	[40]
+41	41	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+str_43	\N	\N	[]	0	[]
+[44,44,44,44,44]	\N	\N	[44,44,44,44,44]	5	[44,44,44,44,44]
+45	45	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+str_47	\N	\N	[]	0	[]
+['str_48','str_48','str_48','str_48','str_48','str_48','str_48','str_48','str_48']	\N	\N	['str_48','str_48','str_48','str_48','str_48','str_48','str_48','str_48','str_48']	9	[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]
+49	49	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+str_51	\N	\N	[]	0	[]
+[52,52,52]	\N	\N	[52,52,52]	3	[52,52,52]
+53	53	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+str_55	\N	\N	[]	0	[]
+[56,56,56,56,56,56,56]	\N	\N	[56,56,56,56,56,56,56]	7	[56,56,56,56,56,56,56]
+57	57	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+str_59	\N	\N	[]	0	[]
+[60]	\N	\N	[60]	1	[60]
+61	61	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+str_63	\N	\N	[]	0	[]
+[64,64,64,64,64]	\N	\N	[64,64,64,64,64]	5	[64,64,64,64,64]
+65	65	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+str_67	\N	\N	[]	0	[]
+[68,68,68,68,68,68,68,68,68]	\N	\N	[68,68,68,68,68,68,68,68,68]	9	[68,68,68,68,68,68,68,68,68]
+69	69	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+str_71	\N	\N	[]	0	[]
+[NULL,NULL,NULL]	\N	\N	[NULL,NULL,NULL]	3	[NULL,NULL,NULL]
+73	73	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+str_75	\N	\N	[]	0	[]
+[76,76,76,76,76,76,76]	\N	\N	[76,76,76,76,76,76,76]	7	[76,76,76,76,76,76,76]
+77	77	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+str_79	\N	\N	[]	0	[]
+0	\N	[]	0	[]	[]
+1	\N	[]	0	[]	[]
+2	\N	[]	0	[]	[]
+3	\N	[]	0	[]	[]
+4	\N	[]	0	[]	[]
+5	\N	[]	0	[]	[]
+6	\N	[]	0	[]	[]
+7	\N	[]	0	[]	[]
+8	\N	[]	0	[]	[]
+9	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[20]	1	[20]	[NULL]
+\N	\N	['str_21','str_21']	2	[NULL,NULL]	['str_21','str_21']
+\N	\N	[22,22,22]	3	[22,22,22]	[NULL,NULL,NULL]
+\N	\N	[23,23,23,23]	4	[23,23,23,23]	[NULL,NULL,NULL,NULL]
+\N	\N	[24,24,24,24,24]	5	[24,24,24,24,24]	[NULL,NULL,NULL,NULL,NULL]
+\N	\N	[25,25,25,25,25,25]	6	[25,25,25,25,25,25]	[NULL,NULL,NULL,NULL,NULL,NULL]
+\N	\N	[26,26,26,26,26,26,26]	7	[26,26,26,26,26,26,26]	[NULL,NULL,NULL,NULL,NULL,NULL,NULL]
+\N	\N	[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]	8	[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]	[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]
+\N	\N	[28,28,28,28,28,28,28,28,28]	9	[28,28,28,28,28,28,28,28,28]	[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]
+\N	\N	[29,29,29,29,29,29,29,29,29,29]	10	[29,29,29,29,29,29,29,29,29,29]	[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[40]	1	[40]	[NULL]
+41	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[44,44,44,44,44]	5	[44,44,44,44,44]	[NULL,NULL,NULL,NULL,NULL]
+45	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	['str_48','str_48','str_48','str_48','str_48','str_48','str_48','str_48','str_48']	9	[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]	['str_48','str_48','str_48','str_48','str_48','str_48','str_48','str_48','str_48']
+49	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[52,52,52]	3	[52,52,52]	[NULL,NULL,NULL]
+53	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[56,56,56,56,56,56,56]	7	[56,56,56,56,56,56,56]	[NULL,NULL,NULL,NULL,NULL,NULL,NULL]
+57	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[60]	1	[60]	[NULL]
+61	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[64,64,64,64,64]	5	[64,64,64,64,64]	[NULL,NULL,NULL,NULL,NULL]
+65	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[68,68,68,68,68,68,68,68,68]	9	[68,68,68,68,68,68,68,68,68]	[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]
+69	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[NULL,NULL,NULL]	3	[NULL,NULL,NULL]	[NULL,NULL,NULL]
+73	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[76,76,76,76,76,76,76]	7	[76,76,76,76,76,76,76]	[NULL,NULL,NULL,NULL,NULL,NULL,NULL]
+77	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+0	0	[]	[]
+1	0	[]	[]
+2	0	[]	[]
+3	0	[]	[]
+4	0	[]	[]
+5	0	[]	[]
+6	0	[]	[]
+7	0	[]	[]
+8	0	[]	[]
+9	0	[]	[]
+str_10	0	[]	[]
+[[0]]	0	[]	[]
+str_11	0	[]	[]
+[[0,1]]	0	[]	[]
+str_12	0	[]	[]
+[[0,1,2]]	0	[]	[]
+str_13	0	[]	[]
+[[0,1,2,3]]	0	[]	[]
+str_14	0	[]	[]
+[[0,1,2,3,4]]	0	[]	[]
+str_15	0	[]	[]
+[[0,1,2,3,4,5]]	0	[]	[]
+str_16	0	[]	[]
+[[0,1,2,3,4,5,6]]	0	[]	[]
+str_17	0	[]	[]
+[[0,1,2,3,4,5,6,7]]	0	[]	[]
+str_18	0	[]	[]
+[[0,1,2,3,4,5,6,7,8]]	0	[]	[]
+str_19	0	[]	[]
+[[0,1,2,3,4,5,6,7,8,9]]	0	[]	[]
+[20]	0	[]	[20]
+['str_21','str_21']	0	[]	[NULL,NULL]
+[22,22,22]	0	[]	[22,22,22]
+[23,23,23,23]	0	[]	[23,23,23,23]
+[24,24,24,24,24]	0	[]	[24,24,24,24,24]
+[25,25,25,25,25,25]	0	[]	[25,25,25,25,25,25]
+[26,26,26,26,26,26,26]	0	[]	[26,26,26,26,26,26,26]
+[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]	0	[]	[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]
+[28,28,28,28,28,28,28,28,28]	0	[]	[28,28,28,28,28,28,28,28,28]
+[29,29,29,29,29,29,29,29,29,29]	0	[]	[29,29,29,29,29,29,29,29,29,29]
+\N	0	[]	[]
+\N	0	[]	[]
+\N	0	[]	[]
+\N	0	[]	[]
+\N	0	[]	[]
+\N	0	[]	[]
+\N	0	[]	[]
+\N	0	[]	[]
+\N	0	[]	[]
+\N	0	[]	[]
+[40]	0	[]	[40]
+41	0	[]	[]
+\N	0	[]	[]
+str_43	0	[]	[]
+[44,44,44,44,44]	0	[]	[44,44,44,44,44]
+45	0	[]	[]
+\N	0	[]	[]
+str_47	0	[]	[]
+['str_48','str_48','str_48','str_48','str_48','str_48','str_48','str_48','str_48']	0	[]	[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]
+49	0	[]	[]
+\N	0	[]	[]
+str_51	0	[]	[]
+[52,52,52]	0	[]	[52,52,52]
+53	0	[]	[]
+\N	0	[]	[]
+str_55	0	[]	[]
+[56,56,56,56,56,56,56]	0	[]	[56,56,56,56,56,56,56]
+57	0	[]	[]
+\N	0	[]	[]
+str_59	0	[]	[]
+[60]	0	[]	[60]
+61	0	[]	[]
+\N	0	[]	[]
+str_63	0	[]	[]
+[64,64,64,64,64]	0	[]	[64,64,64,64,64]
+65	0	[]	[]
+\N	0	[]	[]
+str_67	0	[]	[]
+[68,68,68,68,68,68,68,68,68]	0	[]	[68,68,68,68,68,68,68,68,68]
+69	0	[]	[]
+\N	0	[]	[]
+str_71	0	[]	[]
+[NULL,NULL,NULL]	0	[]	[NULL,NULL,NULL]
+73	0	[]	[]
+\N	0	[]	[]
+str_75	0	[]	[]
+[76,76,76,76,76,76,76]	0	[]	[76,76,76,76,76,76,76]
+77	0	[]	[]
+\N	0	[]	[]
+str_79	0	[]	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[20]
+[]	0	[NULL,NULL]
+[]	0	[22,22,22]
+[]	0	[23,23,23,23]
+[]	0	[24,24,24,24,24]
+[]	0	[25,25,25,25,25,25]
+[]	0	[26,26,26,26,26,26,26]
+[]	0	[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]
+[]	0	[28,28,28,28,28,28,28,28,28]
+[]	0	[29,29,29,29,29,29,29,29,29,29]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[40]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[44,44,44,44,44]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[52,52,52]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[56,56,56,56,56,56,56]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[60]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[64,64,64,64,64]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[68,68,68,68,68,68,68,68,68]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[NULL,NULL,NULL]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[76,76,76,76,76,76,76]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[1]	[[0]]	[[[]]]
+[]	[]	[]
+[2]	[[0,1]]	[[[],[]]]
+[]	[]	[]
+[3]	[[0,1,2]]	[[[],[],[]]]
+[]	[]	[]
+[4]	[[0,1,2,3]]	[[[],[],[],[]]]
+[]	[]	[]
+[5]	[[0,1,2,3,4]]	[[[],[],[],[],[]]]
+[]	[]	[]
+[6]	[[0,1,2,3,4,5]]	[[[],[],[],[],[],[]]]
+[]	[]	[]
+[7]	[[0,1,2,3,4,5,6]]	[[[],[],[],[],[],[],[]]]
+[]	[]	[]
+[8]	[[0,1,2,3,4,5,6,7]]	[[[],[],[],[],[],[],[],[]]]
+[]	[]	[]
+[9]	[[0,1,2,3,4,5,6,7,8]]	[[[],[],[],[],[],[],[],[],[]]]
+[]	[]	[]
+[10]	[[0,1,2,3,4,5,6,7,8,9]]	[[[],[],[],[],[],[],[],[],[],[]]]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+Array(Array(Dynamic))
+Array(Variant(String, UInt64))
+None
+String
+UInt64
+20
+20
+20
+20
+0
+0
+20
+20
+10
+10
+20
+0
+0	0	\N	[]
+1	1	\N	[]
+2	2	\N	[]
+3	3	\N	[]
+4	4	\N	[]
+5	5	\N	[]
+6	6	\N	[]
+7	7	\N	[]
+8	8	\N	[]
+9	9	\N	[]
+str_10	\N	str_10	[]
+[[0]]	\N	\N	[]
+str_11	\N	str_11	[]
+[[0,1]]	\N	\N	[]
+str_12	\N	str_12	[]
+[[0,1,2]]	\N	\N	[]
+str_13	\N	str_13	[]
+[[0,1,2,3]]	\N	\N	[]
+str_14	\N	str_14	[]
+[[0,1,2,3,4]]	\N	\N	[]
+str_15	\N	str_15	[]
+[[0,1,2,3,4,5]]	\N	\N	[]
+str_16	\N	str_16	[]
+[[0,1,2,3,4,5,6]]	\N	\N	[]
+str_17	\N	str_17	[]
+[[0,1,2,3,4,5,6,7]]	\N	\N	[]
+str_18	\N	str_18	[]
+[[0,1,2,3,4,5,6,7,8]]	\N	\N	[]
+str_19	\N	str_19	[]
+[[0,1,2,3,4,5,6,7,8,9]]	\N	\N	[]
+[20]	\N	\N	[20]
+['str_21','str_21']	\N	\N	['str_21','str_21']
+[22,22,22]	\N	\N	[22,22,22]
+[23,23,23,23]	\N	\N	[23,23,23,23]
+[24,24,24,24,24]	\N	\N	[24,24,24,24,24]
+[25,25,25,25,25,25]	\N	\N	[25,25,25,25,25,25]
+[26,26,26,26,26,26,26]	\N	\N	[26,26,26,26,26,26,26]
+[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]	\N	\N	[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]
+[28,28,28,28,28,28,28,28,28]	\N	\N	[28,28,28,28,28,28,28,28,28]
+[29,29,29,29,29,29,29,29,29,29]	\N	\N	[29,29,29,29,29,29,29,29,29,29]
+\N	\N	\N	[]
+\N	\N	\N	[]
+\N	\N	\N	[]
+\N	\N	\N	[]
+\N	\N	\N	[]
+\N	\N	\N	[]
+\N	\N	\N	[]
+\N	\N	\N	[]
+\N	\N	\N	[]
+\N	\N	\N	[]
+[40]	\N	\N	[40]
+41	41	\N	[]
+\N	\N	\N	[]
+str_43	\N	str_43	[]
+[44,44,44,44,44]	\N	\N	[44,44,44,44,44]
+45	45	\N	[]
+\N	\N	\N	[]
+str_47	\N	str_47	[]
+['str_48','str_48','str_48','str_48','str_48','str_48','str_48','str_48','str_48']	\N	\N	['str_48','str_48','str_48','str_48','str_48','str_48','str_48','str_48','str_48']
+49	49	\N	[]
+\N	\N	\N	[]
+str_51	\N	str_51	[]
+[52,52,52]	\N	\N	[52,52,52]
+53	53	\N	[]
+\N	\N	\N	[]
+str_55	\N	str_55	[]
+[56,56,56,56,56,56,56]	\N	\N	[56,56,56,56,56,56,56]
+57	57	\N	[]
+\N	\N	\N	[]
+str_59	\N	str_59	[]
+[60]	\N	\N	[60]
+61	61	\N	[]
+\N	\N	\N	[]
+str_63	\N	str_63	[]
+[64,64,64,64,64]	\N	\N	[64,64,64,64,64]
+65	65	\N	[]
+\N	\N	\N	[]
+str_67	\N	str_67	[]
+[68,68,68,68,68,68,68,68,68]	\N	\N	[68,68,68,68,68,68,68,68,68]
+69	69	\N	[]
+\N	\N	\N	[]
+str_71	\N	str_71	[]
+[NULL,NULL,NULL]	\N	\N	[NULL,NULL,NULL]
+73	73	\N	[]
+\N	\N	\N	[]
+str_75	\N	str_75	[]
+[76,76,76,76,76,76,76]	\N	\N	[76,76,76,76,76,76,76]
+77	77	\N	[]
+\N	\N	\N	[]
+str_79	\N	str_79	[]
+0	\N	[]
+1	\N	[]
+2	\N	[]
+3	\N	[]
+4	\N	[]
+5	\N	[]
+6	\N	[]
+7	\N	[]
+8	\N	[]
+9	\N	[]
+\N	str_10	[]
+\N	\N	[]
+\N	str_11	[]
+\N	\N	[]
+\N	str_12	[]
+\N	\N	[]
+\N	str_13	[]
+\N	\N	[]
+\N	str_14	[]
+\N	\N	[]
+\N	str_15	[]
+\N	\N	[]
+\N	str_16	[]
+\N	\N	[]
+\N	str_17	[]
+\N	\N	[]
+\N	str_18	[]
+\N	\N	[]
+\N	str_19	[]
+\N	\N	[]
+\N	\N	[20]
+\N	\N	['str_21','str_21']
+\N	\N	[22,22,22]
+\N	\N	[23,23,23,23]
+\N	\N	[24,24,24,24,24]
+\N	\N	[25,25,25,25,25,25]
+\N	\N	[26,26,26,26,26,26,26]
+\N	\N	[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]
+\N	\N	[28,28,28,28,28,28,28,28,28]
+\N	\N	[29,29,29,29,29,29,29,29,29,29]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[40]
+41	\N	[]
+\N	\N	[]
+\N	str_43	[]
+\N	\N	[44,44,44,44,44]
+45	\N	[]
+\N	\N	[]
+\N	str_47	[]
+\N	\N	['str_48','str_48','str_48','str_48','str_48','str_48','str_48','str_48','str_48']
+49	\N	[]
+\N	\N	[]
+\N	str_51	[]
+\N	\N	[52,52,52]
+53	\N	[]
+\N	\N	[]
+\N	str_55	[]
+\N	\N	[56,56,56,56,56,56,56]
+57	\N	[]
+\N	\N	[]
+\N	str_59	[]
+\N	\N	[60]
+61	\N	[]
+\N	\N	[]
+\N	str_63	[]
+\N	\N	[64,64,64,64,64]
+65	\N	[]
+\N	\N	[]
+\N	str_67	[]
+\N	\N	[68,68,68,68,68,68,68,68,68]
+69	\N	[]
+\N	\N	[]
+\N	str_71	[]
+\N	\N	[NULL,NULL,NULL]
+73	\N	[]
+\N	\N	[]
+\N	str_75	[]
+\N	\N	[76,76,76,76,76,76,76]
+77	\N	[]
+\N	\N	[]
+\N	str_79	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+0	0	\N	[]	0	[]
+1	1	\N	[]	0	[]
+2	2	\N	[]	0	[]
+3	3	\N	[]	0	[]
+4	4	\N	[]	0	[]
+5	5	\N	[]	0	[]
+6	6	\N	[]	0	[]
+7	7	\N	[]	0	[]
+8	8	\N	[]	0	[]
+9	9	\N	[]	0	[]
+str_10	\N	\N	[]	0	[]
+[[0]]	\N	\N	[]	0	[]
+str_11	\N	\N	[]	0	[]
+[[0,1]]	\N	\N	[]	0	[]
+str_12	\N	\N	[]	0	[]
+[[0,1,2]]	\N	\N	[]	0	[]
+str_13	\N	\N	[]	0	[]
+[[0,1,2,3]]	\N	\N	[]	0	[]
+str_14	\N	\N	[]	0	[]
+[[0,1,2,3,4]]	\N	\N	[]	0	[]
+str_15	\N	\N	[]	0	[]
+[[0,1,2,3,4,5]]	\N	\N	[]	0	[]
+str_16	\N	\N	[]	0	[]
+[[0,1,2,3,4,5,6]]	\N	\N	[]	0	[]
+str_17	\N	\N	[]	0	[]
+[[0,1,2,3,4,5,6,7]]	\N	\N	[]	0	[]
+str_18	\N	\N	[]	0	[]
+[[0,1,2,3,4,5,6,7,8]]	\N	\N	[]	0	[]
+str_19	\N	\N	[]	0	[]
+[[0,1,2,3,4,5,6,7,8,9]]	\N	\N	[]	0	[]
+[20]	\N	\N	[20]	1	[20]
+['str_21','str_21']	\N	\N	['str_21','str_21']	2	[NULL,NULL]
+[22,22,22]	\N	\N	[22,22,22]	3	[22,22,22]
+[23,23,23,23]	\N	\N	[23,23,23,23]	4	[23,23,23,23]
+[24,24,24,24,24]	\N	\N	[24,24,24,24,24]	5	[24,24,24,24,24]
+[25,25,25,25,25,25]	\N	\N	[25,25,25,25,25,25]	6	[25,25,25,25,25,25]
+[26,26,26,26,26,26,26]	\N	\N	[26,26,26,26,26,26,26]	7	[26,26,26,26,26,26,26]
+[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]	\N	\N	[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]	8	[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]
+[28,28,28,28,28,28,28,28,28]	\N	\N	[28,28,28,28,28,28,28,28,28]	9	[28,28,28,28,28,28,28,28,28]
+[29,29,29,29,29,29,29,29,29,29]	\N	\N	[29,29,29,29,29,29,29,29,29,29]	10	[29,29,29,29,29,29,29,29,29,29]
+\N	\N	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+[40]	\N	\N	[40]	1	[40]
+41	41	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+str_43	\N	\N	[]	0	[]
+[44,44,44,44,44]	\N	\N	[44,44,44,44,44]	5	[44,44,44,44,44]
+45	45	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+str_47	\N	\N	[]	0	[]
+['str_48','str_48','str_48','str_48','str_48','str_48','str_48','str_48','str_48']	\N	\N	['str_48','str_48','str_48','str_48','str_48','str_48','str_48','str_48','str_48']	9	[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]
+49	49	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+str_51	\N	\N	[]	0	[]
+[52,52,52]	\N	\N	[52,52,52]	3	[52,52,52]
+53	53	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+str_55	\N	\N	[]	0	[]
+[56,56,56,56,56,56,56]	\N	\N	[56,56,56,56,56,56,56]	7	[56,56,56,56,56,56,56]
+57	57	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+str_59	\N	\N	[]	0	[]
+[60]	\N	\N	[60]	1	[60]
+61	61	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+str_63	\N	\N	[]	0	[]
+[64,64,64,64,64]	\N	\N	[64,64,64,64,64]	5	[64,64,64,64,64]
+65	65	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+str_67	\N	\N	[]	0	[]
+[68,68,68,68,68,68,68,68,68]	\N	\N	[68,68,68,68,68,68,68,68,68]	9	[68,68,68,68,68,68,68,68,68]
+69	69	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+str_71	\N	\N	[]	0	[]
+[NULL,NULL,NULL]	\N	\N	[NULL,NULL,NULL]	3	[NULL,NULL,NULL]
+73	73	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+str_75	\N	\N	[]	0	[]
+[76,76,76,76,76,76,76]	\N	\N	[76,76,76,76,76,76,76]	7	[76,76,76,76,76,76,76]
+77	77	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+str_79	\N	\N	[]	0	[]
+0	\N	[]	0	[]	[]
+1	\N	[]	0	[]	[]
+2	\N	[]	0	[]	[]
+3	\N	[]	0	[]	[]
+4	\N	[]	0	[]	[]
+5	\N	[]	0	[]	[]
+6	\N	[]	0	[]	[]
+7	\N	[]	0	[]	[]
+8	\N	[]	0	[]	[]
+9	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[20]	1	[20]	[NULL]
+\N	\N	['str_21','str_21']	2	[NULL,NULL]	['str_21','str_21']
+\N	\N	[22,22,22]	3	[22,22,22]	[NULL,NULL,NULL]
+\N	\N	[23,23,23,23]	4	[23,23,23,23]	[NULL,NULL,NULL,NULL]
+\N	\N	[24,24,24,24,24]	5	[24,24,24,24,24]	[NULL,NULL,NULL,NULL,NULL]
+\N	\N	[25,25,25,25,25,25]	6	[25,25,25,25,25,25]	[NULL,NULL,NULL,NULL,NULL,NULL]
+\N	\N	[26,26,26,26,26,26,26]	7	[26,26,26,26,26,26,26]	[NULL,NULL,NULL,NULL,NULL,NULL,NULL]
+\N	\N	[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]	8	[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]	[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]
+\N	\N	[28,28,28,28,28,28,28,28,28]	9	[28,28,28,28,28,28,28,28,28]	[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]
+\N	\N	[29,29,29,29,29,29,29,29,29,29]	10	[29,29,29,29,29,29,29,29,29,29]	[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[40]	1	[40]	[NULL]
+41	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[44,44,44,44,44]	5	[44,44,44,44,44]	[NULL,NULL,NULL,NULL,NULL]
+45	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	['str_48','str_48','str_48','str_48','str_48','str_48','str_48','str_48','str_48']	9	[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]	['str_48','str_48','str_48','str_48','str_48','str_48','str_48','str_48','str_48']
+49	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[52,52,52]	3	[52,52,52]	[NULL,NULL,NULL]
+53	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[56,56,56,56,56,56,56]	7	[56,56,56,56,56,56,56]	[NULL,NULL,NULL,NULL,NULL,NULL,NULL]
+57	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[60]	1	[60]	[NULL]
+61	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[64,64,64,64,64]	5	[64,64,64,64,64]	[NULL,NULL,NULL,NULL,NULL]
+65	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[68,68,68,68,68,68,68,68,68]	9	[68,68,68,68,68,68,68,68,68]	[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]
+69	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[NULL,NULL,NULL]	3	[NULL,NULL,NULL]	[NULL,NULL,NULL]
+73	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[76,76,76,76,76,76,76]	7	[76,76,76,76,76,76,76]	[NULL,NULL,NULL,NULL,NULL,NULL,NULL]
+77	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+0	0	[]	[]
+1	0	[]	[]
+2	0	[]	[]
+3	0	[]	[]
+4	0	[]	[]
+5	0	[]	[]
+6	0	[]	[]
+7	0	[]	[]
+8	0	[]	[]
+9	0	[]	[]
+str_10	0	[]	[]
+[[0]]	0	[]	[]
+str_11	0	[]	[]
+[[0,1]]	0	[]	[]
+str_12	0	[]	[]
+[[0,1,2]]	0	[]	[]
+str_13	0	[]	[]
+[[0,1,2,3]]	0	[]	[]
+str_14	0	[]	[]
+[[0,1,2,3,4]]	0	[]	[]
+str_15	0	[]	[]
+[[0,1,2,3,4,5]]	0	[]	[]
+str_16	0	[]	[]
+[[0,1,2,3,4,5,6]]	0	[]	[]
+str_17	0	[]	[]
+[[0,1,2,3,4,5,6,7]]	0	[]	[]
+str_18	0	[]	[]
+[[0,1,2,3,4,5,6,7,8]]	0	[]	[]
+str_19	0	[]	[]
+[[0,1,2,3,4,5,6,7,8,9]]	0	[]	[]
+[20]	0	[]	[20]
+['str_21','str_21']	0	[]	[NULL,NULL]
+[22,22,22]	0	[]	[22,22,22]
+[23,23,23,23]	0	[]	[23,23,23,23]
+[24,24,24,24,24]	0	[]	[24,24,24,24,24]
+[25,25,25,25,25,25]	0	[]	[25,25,25,25,25,25]
+[26,26,26,26,26,26,26]	0	[]	[26,26,26,26,26,26,26]
+[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]	0	[]	[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]
+[28,28,28,28,28,28,28,28,28]	0	[]	[28,28,28,28,28,28,28,28,28]
+[29,29,29,29,29,29,29,29,29,29]	0	[]	[29,29,29,29,29,29,29,29,29,29]
+\N	0	[]	[]
+\N	0	[]	[]
+\N	0	[]	[]
+\N	0	[]	[]
+\N	0	[]	[]
+\N	0	[]	[]
+\N	0	[]	[]
+\N	0	[]	[]
+\N	0	[]	[]
+\N	0	[]	[]
+[40]	0	[]	[40]
+41	0	[]	[]
+\N	0	[]	[]
+str_43	0	[]	[]
+[44,44,44,44,44]	0	[]	[44,44,44,44,44]
+45	0	[]	[]
+\N	0	[]	[]
+str_47	0	[]	[]
+['str_48','str_48','str_48','str_48','str_48','str_48','str_48','str_48','str_48']	0	[]	[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]
+49	0	[]	[]
+\N	0	[]	[]
+str_51	0	[]	[]
+[52,52,52]	0	[]	[52,52,52]
+53	0	[]	[]
+\N	0	[]	[]
+str_55	0	[]	[]
+[56,56,56,56,56,56,56]	0	[]	[56,56,56,56,56,56,56]
+57	0	[]	[]
+\N	0	[]	[]
+str_59	0	[]	[]
+[60]	0	[]	[60]
+61	0	[]	[]
+\N	0	[]	[]
+str_63	0	[]	[]
+[64,64,64,64,64]	0	[]	[64,64,64,64,64]
+65	0	[]	[]
+\N	0	[]	[]
+str_67	0	[]	[]
+[68,68,68,68,68,68,68,68,68]	0	[]	[68,68,68,68,68,68,68,68,68]
+69	0	[]	[]
+\N	0	[]	[]
+str_71	0	[]	[]
+[NULL,NULL,NULL]	0	[]	[NULL,NULL,NULL]
+73	0	[]	[]
+\N	0	[]	[]
+str_75	0	[]	[]
+[76,76,76,76,76,76,76]	0	[]	[76,76,76,76,76,76,76]
+77	0	[]	[]
+\N	0	[]	[]
+str_79	0	[]	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[20]
+[]	0	[NULL,NULL]
+[]	0	[22,22,22]
+[]	0	[23,23,23,23]
+[]	0	[24,24,24,24,24]
+[]	0	[25,25,25,25,25,25]
+[]	0	[26,26,26,26,26,26,26]
+[]	0	[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]
+[]	0	[28,28,28,28,28,28,28,28,28]
+[]	0	[29,29,29,29,29,29,29,29,29,29]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[40]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[44,44,44,44,44]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[52,52,52]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[56,56,56,56,56,56,56]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[60]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[64,64,64,64,64]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[68,68,68,68,68,68,68,68,68]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[NULL,NULL,NULL]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[76,76,76,76,76,76,76]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[1]	[[0]]	[[[]]]
+[]	[]	[]
+[2]	[[0,1]]	[[[],[]]]
+[]	[]	[]
+[3]	[[0,1,2]]	[[[],[],[]]]
+[]	[]	[]
+[4]	[[0,1,2,3]]	[[[],[],[],[]]]
+[]	[]	[]
+[5]	[[0,1,2,3,4]]	[[[],[],[],[],[]]]
+[]	[]	[]
+[6]	[[0,1,2,3,4,5]]	[[[],[],[],[],[],[]]]
+[]	[]	[]
+[7]	[[0,1,2,3,4,5,6]]	[[[],[],[],[],[],[],[]]]
+[]	[]	[]
+[8]	[[0,1,2,3,4,5,6,7]]	[[[],[],[],[],[],[],[],[]]]
+[]	[]	[]
+[9]	[[0,1,2,3,4,5,6,7,8]]	[[[],[],[],[],[],[],[],[],[]]]
+[]	[]	[]
+[10]	[[0,1,2,3,4,5,6,7,8,9]]	[[[],[],[],[],[],[],[],[],[],[]]]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+Array(Array(Dynamic))
+Array(Variant(String, UInt64))
+None
+String
+UInt64
+20
+20
+20
+20
+0
+0
+20
+20
+10
+10
+20
+0
+0	0	\N	[]
+1	1	\N	[]
+2	2	\N	[]
+3	3	\N	[]
+4	4	\N	[]
+5	5	\N	[]
+6	6	\N	[]
+7	7	\N	[]
+8	8	\N	[]
+9	9	\N	[]
+str_10	\N	str_10	[]
+[[0]]	\N	\N	[]
+str_11	\N	str_11	[]
+[[0,1]]	\N	\N	[]
+str_12	\N	str_12	[]
+[[0,1,2]]	\N	\N	[]
+str_13	\N	str_13	[]
+[[0,1,2,3]]	\N	\N	[]
+str_14	\N	str_14	[]
+[[0,1,2,3,4]]	\N	\N	[]
+str_15	\N	str_15	[]
+[[0,1,2,3,4,5]]	\N	\N	[]
+str_16	\N	str_16	[]
+[[0,1,2,3,4,5,6]]	\N	\N	[]
+str_17	\N	str_17	[]
+[[0,1,2,3,4,5,6,7]]	\N	\N	[]
+str_18	\N	str_18	[]
+[[0,1,2,3,4,5,6,7,8]]	\N	\N	[]
+str_19	\N	str_19	[]
+[[0,1,2,3,4,5,6,7,8,9]]	\N	\N	[]
+[20]	\N	\N	[20]
+['str_21','str_21']	\N	\N	['str_21','str_21']
+[22,22,22]	\N	\N	[22,22,22]
+[23,23,23,23]	\N	\N	[23,23,23,23]
+[24,24,24,24,24]	\N	\N	[24,24,24,24,24]
+[25,25,25,25,25,25]	\N	\N	[25,25,25,25,25,25]
+[26,26,26,26,26,26,26]	\N	\N	[26,26,26,26,26,26,26]
+[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]	\N	\N	[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]
+[28,28,28,28,28,28,28,28,28]	\N	\N	[28,28,28,28,28,28,28,28,28]
+[29,29,29,29,29,29,29,29,29,29]	\N	\N	[29,29,29,29,29,29,29,29,29,29]
+\N	\N	\N	[]
+\N	\N	\N	[]
+\N	\N	\N	[]
+\N	\N	\N	[]
+\N	\N	\N	[]
+\N	\N	\N	[]
+\N	\N	\N	[]
+\N	\N	\N	[]
+\N	\N	\N	[]
+\N	\N	\N	[]
+[40]	\N	\N	[40]
+41	41	\N	[]
+\N	\N	\N	[]
+str_43	\N	str_43	[]
+[44,44,44,44,44]	\N	\N	[44,44,44,44,44]
+45	45	\N	[]
+\N	\N	\N	[]
+str_47	\N	str_47	[]
+['str_48','str_48','str_48','str_48','str_48','str_48','str_48','str_48','str_48']	\N	\N	['str_48','str_48','str_48','str_48','str_48','str_48','str_48','str_48','str_48']
+49	49	\N	[]
+\N	\N	\N	[]
+str_51	\N	str_51	[]
+[52,52,52]	\N	\N	[52,52,52]
+53	53	\N	[]
+\N	\N	\N	[]
+str_55	\N	str_55	[]
+[56,56,56,56,56,56,56]	\N	\N	[56,56,56,56,56,56,56]
+57	57	\N	[]
+\N	\N	\N	[]
+str_59	\N	str_59	[]
+[60]	\N	\N	[60]
+61	61	\N	[]
+\N	\N	\N	[]
+str_63	\N	str_63	[]
+[64,64,64,64,64]	\N	\N	[64,64,64,64,64]
+65	65	\N	[]
+\N	\N	\N	[]
+str_67	\N	str_67	[]
+[68,68,68,68,68,68,68,68,68]	\N	\N	[68,68,68,68,68,68,68,68,68]
+69	69	\N	[]
+\N	\N	\N	[]
+str_71	\N	str_71	[]
+[NULL,NULL,NULL]	\N	\N	[NULL,NULL,NULL]
+73	73	\N	[]
+\N	\N	\N	[]
+str_75	\N	str_75	[]
+[76,76,76,76,76,76,76]	\N	\N	[76,76,76,76,76,76,76]
+77	77	\N	[]
+\N	\N	\N	[]
+str_79	\N	str_79	[]
+0	\N	[]
+1	\N	[]
+2	\N	[]
+3	\N	[]
+4	\N	[]
+5	\N	[]
+6	\N	[]
+7	\N	[]
+8	\N	[]
+9	\N	[]
+\N	str_10	[]
+\N	\N	[]
+\N	str_11	[]
+\N	\N	[]
+\N	str_12	[]
+\N	\N	[]
+\N	str_13	[]
+\N	\N	[]
+\N	str_14	[]
+\N	\N	[]
+\N	str_15	[]
+\N	\N	[]
+\N	str_16	[]
+\N	\N	[]
+\N	str_17	[]
+\N	\N	[]
+\N	str_18	[]
+\N	\N	[]
+\N	str_19	[]
+\N	\N	[]
+\N	\N	[20]
+\N	\N	['str_21','str_21']
+\N	\N	[22,22,22]
+\N	\N	[23,23,23,23]
+\N	\N	[24,24,24,24,24]
+\N	\N	[25,25,25,25,25,25]
+\N	\N	[26,26,26,26,26,26,26]
+\N	\N	[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]
+\N	\N	[28,28,28,28,28,28,28,28,28]
+\N	\N	[29,29,29,29,29,29,29,29,29,29]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[40]
+41	\N	[]
+\N	\N	[]
+\N	str_43	[]
+\N	\N	[44,44,44,44,44]
+45	\N	[]
+\N	\N	[]
+\N	str_47	[]
+\N	\N	['str_48','str_48','str_48','str_48','str_48','str_48','str_48','str_48','str_48']
+49	\N	[]
+\N	\N	[]
+\N	str_51	[]
+\N	\N	[52,52,52]
+53	\N	[]
+\N	\N	[]
+\N	str_55	[]
+\N	\N	[56,56,56,56,56,56,56]
+57	\N	[]
+\N	\N	[]
+\N	str_59	[]
+\N	\N	[60]
+61	\N	[]
+\N	\N	[]
+\N	str_63	[]
+\N	\N	[64,64,64,64,64]
+65	\N	[]
+\N	\N	[]
+\N	str_67	[]
+\N	\N	[68,68,68,68,68,68,68,68,68]
+69	\N	[]
+\N	\N	[]
+\N	str_71	[]
+\N	\N	[NULL,NULL,NULL]
+73	\N	[]
+\N	\N	[]
+\N	str_75	[]
+\N	\N	[76,76,76,76,76,76,76]
+77	\N	[]
+\N	\N	[]
+\N	str_79	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+\N	\N	[]
+0	0	\N	[]	0	[]
+1	1	\N	[]	0	[]
+2	2	\N	[]	0	[]
+3	3	\N	[]	0	[]
+4	4	\N	[]	0	[]
+5	5	\N	[]	0	[]
+6	6	\N	[]	0	[]
+7	7	\N	[]	0	[]
+8	8	\N	[]	0	[]
+9	9	\N	[]	0	[]
+str_10	\N	\N	[]	0	[]
+[[0]]	\N	\N	[]	0	[]
+str_11	\N	\N	[]	0	[]
+[[0,1]]	\N	\N	[]	0	[]
+str_12	\N	\N	[]	0	[]
+[[0,1,2]]	\N	\N	[]	0	[]
+str_13	\N	\N	[]	0	[]
+[[0,1,2,3]]	\N	\N	[]	0	[]
+str_14	\N	\N	[]	0	[]
+[[0,1,2,3,4]]	\N	\N	[]	0	[]
+str_15	\N	\N	[]	0	[]
+[[0,1,2,3,4,5]]	\N	\N	[]	0	[]
+str_16	\N	\N	[]	0	[]
+[[0,1,2,3,4,5,6]]	\N	\N	[]	0	[]
+str_17	\N	\N	[]	0	[]
+[[0,1,2,3,4,5,6,7]]	\N	\N	[]	0	[]
+str_18	\N	\N	[]	0	[]
+[[0,1,2,3,4,5,6,7,8]]	\N	\N	[]	0	[]
+str_19	\N	\N	[]	0	[]
+[[0,1,2,3,4,5,6,7,8,9]]	\N	\N	[]	0	[]
+[20]	\N	\N	[20]	1	[20]
+['str_21','str_21']	\N	\N	['str_21','str_21']	2	[NULL,NULL]
+[22,22,22]	\N	\N	[22,22,22]	3	[22,22,22]
+[23,23,23,23]	\N	\N	[23,23,23,23]	4	[23,23,23,23]
+[24,24,24,24,24]	\N	\N	[24,24,24,24,24]	5	[24,24,24,24,24]
+[25,25,25,25,25,25]	\N	\N	[25,25,25,25,25,25]	6	[25,25,25,25,25,25]
+[26,26,26,26,26,26,26]	\N	\N	[26,26,26,26,26,26,26]	7	[26,26,26,26,26,26,26]
+[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]	\N	\N	[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]	8	[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]
+[28,28,28,28,28,28,28,28,28]	\N	\N	[28,28,28,28,28,28,28,28,28]	9	[28,28,28,28,28,28,28,28,28]
+[29,29,29,29,29,29,29,29,29,29]	\N	\N	[29,29,29,29,29,29,29,29,29,29]	10	[29,29,29,29,29,29,29,29,29,29]
+\N	\N	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+[40]	\N	\N	[40]	1	[40]
+41	41	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+str_43	\N	\N	[]	0	[]
+[44,44,44,44,44]	\N	\N	[44,44,44,44,44]	5	[44,44,44,44,44]
+45	45	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+str_47	\N	\N	[]	0	[]
+['str_48','str_48','str_48','str_48','str_48','str_48','str_48','str_48','str_48']	\N	\N	['str_48','str_48','str_48','str_48','str_48','str_48','str_48','str_48','str_48']	9	[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]
+49	49	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+str_51	\N	\N	[]	0	[]
+[52,52,52]	\N	\N	[52,52,52]	3	[52,52,52]
+53	53	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+str_55	\N	\N	[]	0	[]
+[56,56,56,56,56,56,56]	\N	\N	[56,56,56,56,56,56,56]	7	[56,56,56,56,56,56,56]
+57	57	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+str_59	\N	\N	[]	0	[]
+[60]	\N	\N	[60]	1	[60]
+61	61	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+str_63	\N	\N	[]	0	[]
+[64,64,64,64,64]	\N	\N	[64,64,64,64,64]	5	[64,64,64,64,64]
+65	65	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+str_67	\N	\N	[]	0	[]
+[68,68,68,68,68,68,68,68,68]	\N	\N	[68,68,68,68,68,68,68,68,68]	9	[68,68,68,68,68,68,68,68,68]
+69	69	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+str_71	\N	\N	[]	0	[]
+[NULL,NULL,NULL]	\N	\N	[NULL,NULL,NULL]	3	[NULL,NULL,NULL]
+73	73	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+str_75	\N	\N	[]	0	[]
+[76,76,76,76,76,76,76]	\N	\N	[76,76,76,76,76,76,76]	7	[76,76,76,76,76,76,76]
+77	77	\N	[]	0	[]
+\N	\N	\N	[]	0	[]
+str_79	\N	\N	[]	0	[]
+0	\N	[]	0	[]	[]
+1	\N	[]	0	[]	[]
+2	\N	[]	0	[]	[]
+3	\N	[]	0	[]	[]
+4	\N	[]	0	[]	[]
+5	\N	[]	0	[]	[]
+6	\N	[]	0	[]	[]
+7	\N	[]	0	[]	[]
+8	\N	[]	0	[]	[]
+9	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[20]	1	[20]	[NULL]
+\N	\N	['str_21','str_21']	2	[NULL,NULL]	['str_21','str_21']
+\N	\N	[22,22,22]	3	[22,22,22]	[NULL,NULL,NULL]
+\N	\N	[23,23,23,23]	4	[23,23,23,23]	[NULL,NULL,NULL,NULL]
+\N	\N	[24,24,24,24,24]	5	[24,24,24,24,24]	[NULL,NULL,NULL,NULL,NULL]
+\N	\N	[25,25,25,25,25,25]	6	[25,25,25,25,25,25]	[NULL,NULL,NULL,NULL,NULL,NULL]
+\N	\N	[26,26,26,26,26,26,26]	7	[26,26,26,26,26,26,26]	[NULL,NULL,NULL,NULL,NULL,NULL,NULL]
+\N	\N	[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]	8	[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]	[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]
+\N	\N	[28,28,28,28,28,28,28,28,28]	9	[28,28,28,28,28,28,28,28,28]	[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]
+\N	\N	[29,29,29,29,29,29,29,29,29,29]	10	[29,29,29,29,29,29,29,29,29,29]	[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[40]	1	[40]	[NULL]
+41	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[44,44,44,44,44]	5	[44,44,44,44,44]	[NULL,NULL,NULL,NULL,NULL]
+45	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	['str_48','str_48','str_48','str_48','str_48','str_48','str_48','str_48','str_48']	9	[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]	['str_48','str_48','str_48','str_48','str_48','str_48','str_48','str_48','str_48']
+49	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[52,52,52]	3	[52,52,52]	[NULL,NULL,NULL]
+53	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[56,56,56,56,56,56,56]	7	[56,56,56,56,56,56,56]	[NULL,NULL,NULL,NULL,NULL,NULL,NULL]
+57	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[60]	1	[60]	[NULL]
+61	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[64,64,64,64,64]	5	[64,64,64,64,64]	[NULL,NULL,NULL,NULL,NULL]
+65	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[68,68,68,68,68,68,68,68,68]	9	[68,68,68,68,68,68,68,68,68]	[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]
+69	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[NULL,NULL,NULL]	3	[NULL,NULL,NULL]	[NULL,NULL,NULL]
+73	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[76,76,76,76,76,76,76]	7	[76,76,76,76,76,76,76]	[NULL,NULL,NULL,NULL,NULL,NULL,NULL]
+77	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+\N	\N	[]	0	[]	[]
+0	0	[]	[]
+1	0	[]	[]
+2	0	[]	[]
+3	0	[]	[]
+4	0	[]	[]
+5	0	[]	[]
+6	0	[]	[]
+7	0	[]	[]
+8	0	[]	[]
+9	0	[]	[]
+str_10	0	[]	[]
+[[0]]	0	[]	[]
+str_11	0	[]	[]
+[[0,1]]	0	[]	[]
+str_12	0	[]	[]
+[[0,1,2]]	0	[]	[]
+str_13	0	[]	[]
+[[0,1,2,3]]	0	[]	[]
+str_14	0	[]	[]
+[[0,1,2,3,4]]	0	[]	[]
+str_15	0	[]	[]
+[[0,1,2,3,4,5]]	0	[]	[]
+str_16	0	[]	[]
+[[0,1,2,3,4,5,6]]	0	[]	[]
+str_17	0	[]	[]
+[[0,1,2,3,4,5,6,7]]	0	[]	[]
+str_18	0	[]	[]
+[[0,1,2,3,4,5,6,7,8]]	0	[]	[]
+str_19	0	[]	[]
+[[0,1,2,3,4,5,6,7,8,9]]	0	[]	[]
+[20]	0	[]	[20]
+['str_21','str_21']	0	[]	[NULL,NULL]
+[22,22,22]	0	[]	[22,22,22]
+[23,23,23,23]	0	[]	[23,23,23,23]
+[24,24,24,24,24]	0	[]	[24,24,24,24,24]
+[25,25,25,25,25,25]	0	[]	[25,25,25,25,25,25]
+[26,26,26,26,26,26,26]	0	[]	[26,26,26,26,26,26,26]
+[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]	0	[]	[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]
+[28,28,28,28,28,28,28,28,28]	0	[]	[28,28,28,28,28,28,28,28,28]
+[29,29,29,29,29,29,29,29,29,29]	0	[]	[29,29,29,29,29,29,29,29,29,29]
+\N	0	[]	[]
+\N	0	[]	[]
+\N	0	[]	[]
+\N	0	[]	[]
+\N	0	[]	[]
+\N	0	[]	[]
+\N	0	[]	[]
+\N	0	[]	[]
+\N	0	[]	[]
+\N	0	[]	[]
+[40]	0	[]	[40]
+41	0	[]	[]
+\N	0	[]	[]
+str_43	0	[]	[]
+[44,44,44,44,44]	0	[]	[44,44,44,44,44]
+45	0	[]	[]
+\N	0	[]	[]
+str_47	0	[]	[]
+['str_48','str_48','str_48','str_48','str_48','str_48','str_48','str_48','str_48']	0	[]	[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]
+49	0	[]	[]
+\N	0	[]	[]
+str_51	0	[]	[]
+[52,52,52]	0	[]	[52,52,52]
+53	0	[]	[]
+\N	0	[]	[]
+str_55	0	[]	[]
+[56,56,56,56,56,56,56]	0	[]	[56,56,56,56,56,56,56]
+57	0	[]	[]
+\N	0	[]	[]
+str_59	0	[]	[]
+[60]	0	[]	[60]
+61	0	[]	[]
+\N	0	[]	[]
+str_63	0	[]	[]
+[64,64,64,64,64]	0	[]	[64,64,64,64,64]
+65	0	[]	[]
+\N	0	[]	[]
+str_67	0	[]	[]
+[68,68,68,68,68,68,68,68,68]	0	[]	[68,68,68,68,68,68,68,68,68]
+69	0	[]	[]
+\N	0	[]	[]
+str_71	0	[]	[]
+[NULL,NULL,NULL]	0	[]	[NULL,NULL,NULL]
+73	0	[]	[]
+\N	0	[]	[]
+str_75	0	[]	[]
+[76,76,76,76,76,76,76]	0	[]	[76,76,76,76,76,76,76]
+77	0	[]	[]
+\N	0	[]	[]
+str_79	0	[]	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[20]
+[]	0	[NULL,NULL]
+[]	0	[22,22,22]
+[]	0	[23,23,23,23]
+[]	0	[24,24,24,24,24]
+[]	0	[25,25,25,25,25,25]
+[]	0	[26,26,26,26,26,26,26]
+[]	0	[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]
+[]	0	[28,28,28,28,28,28,28,28,28]
+[]	0	[29,29,29,29,29,29,29,29,29,29]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[40]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[44,44,44,44,44]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[52,52,52]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[56,56,56,56,56,56,56]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[60]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[64,64,64,64,64]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[68,68,68,68,68,68,68,68,68]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[NULL,NULL,NULL]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	0	[76,76,76,76,76,76,76]
+[]	0	[]
+[]	0	[]
+[]	0	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[1]	[[0]]	[[[]]]
+[]	[]	[]
+[2]	[[0,1]]	[[[],[]]]
+[]	[]	[]
+[3]	[[0,1,2]]	[[[],[],[]]]
+[]	[]	[]
+[4]	[[0,1,2,3]]	[[[],[],[],[]]]
+[]	[]	[]
+[5]	[[0,1,2,3,4]]	[[[],[],[],[],[]]]
+[]	[]	[]
+[6]	[[0,1,2,3,4,5]]	[[[],[],[],[],[],[]]]
+[]	[]	[]
+[7]	[[0,1,2,3,4,5,6]]	[[[],[],[],[],[],[],[]]]
+[]	[]	[]
+[8]	[[0,1,2,3,4,5,6,7]]	[[[],[],[],[],[],[],[],[]]]
+[]	[]	[]
+[9]	[[0,1,2,3,4,5,6,7,8]]	[[[],[],[],[],[],[],[],[],[]]]
+[]	[]	[]
+[10]	[[0,1,2,3,4,5,6,7,8,9]]	[[[],[],[],[],[],[],[],[],[],[]]]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
+[]	[]	[]
diff --git a/tests/queries/0_stateless/03036_dynamic_read_subcolumns_small.sql.j2 b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_small.sql.j2
new file mode 100644
index 00000000000..17d9fe8a992
--- /dev/null
+++ b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_small.sql.j2
@@ -0,0 +1,45 @@
+-- Tags: long
+
+set allow_experimental_variant_type = 1;
+set use_variant_as_common_type = 1;
+set allow_experimental_dynamic_type = 1;
+
+drop table if exists test;
+
+{% for engine in ['Memory', 'MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000', 'MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1'] -%}
+
+create table test (id UInt64, d Dynamic) engine={{ engine }};
+
+insert into test select number, number from numbers(10);
+insert into test select number, 'str_' || toString(number) from numbers(10, 10);
+insert into test select number, arrayMap(x -> multiIf(number % 9 == 0, NULL, number % 9 == 3, 'str_' || toString(number), number), range(number % 10 + 1)) from numbers(20, 10);
+insert into test select number, NULL from numbers(30, 10);
+insert into test select number, multiIf(number % 4 == 3, 'str_' || toString(number), number % 4 == 2, NULL, number % 4 == 1, number, arrayMap(x -> multiIf(number % 9 == 0, NULL, number % 9 == 3, 'str_' || toString(number), number), range(number % 10 + 1))) from numbers(40, 40);
+insert into test select number, [range((number % 10 + 1)::UInt64)]::Array(Array(Dynamic)) from numbers(10, 10);
+
+select distinct dynamicType(d) as type from test order by type;
+select count() from test where dynamicType(d) == 'UInt64';
+select count() from test where d.UInt64 is not NULL;
+select count() from test where dynamicType(d) == 'String';
+select count() from test where d.String is not NULL;
+select count() from test where dynamicType(d) == 'Date';
+select count() from test where d.Date is not NULL;
+select count() from test where dynamicType(d) == 'Array(Variant(String, UInt64))';
+select count() from test where not empty(d.`Array(Variant(String, UInt64))`);
+select count() from test where dynamicType(d) == 'Array(Array(Dynamic))';
+select count() from test where not empty(d.`Array(Array(Dynamic))`);
+select count() from test where d is NULL;
+select count() from test where not empty(d.`Tuple(a Array(Dynamic))`.a.String);
+
+select d, d.UInt64, d.String, d.`Array(Variant(String, UInt64))` from test order by id;
+select d.UInt64, d.String, d.`Array(Variant(String, UInt64))` from test order by id;
+select d.Int8, d.Date, d.`Array(String)` from test order by id;
+select d, d.UInt64, d.Date, d.`Array(Variant(String, UInt64))`, d.`Array(Variant(String, UInt64))`.size0, d.`Array(Variant(String, UInt64))`.UInt64 from test order by id;
+select d.UInt64, d.Date, d.`Array(Variant(String, UInt64))`, d.`Array(Variant(String, UInt64))`.size0, d.`Array(Variant(String, UInt64))`.UInt64, d.`Array(Variant(String, UInt64))`.String from test order by id;
+select d, d.`Tuple(a UInt64, b String)`.a, d.`Array(Dynamic)`.`Variant(String, UInt64)`.UInt64, d.`Array(Variant(String, UInt64))`.UInt64 from test order by id;
+select d.`Array(Dynamic)`.`Variant(String, UInt64)`.UInt64, d.`Array(Dynamic)`.size0, d.`Array(Variant(String, UInt64))`.UInt64 from test order by id;
+select d.`Array(Array(Dynamic))`.size1, d.`Array(Array(Dynamic))`.UInt64, d.`Array(Array(Dynamic))`.`Map(String, Tuple(a UInt64))`.values.a from test order by id;
+
+drop table test;
+
+{% endfor -%}
diff --git a/tests/queries/0_stateless/03037_dynamic_merges_small.reference.j2 b/tests/queries/0_stateless/03037_dynamic_merges_small.reference.j2
new file mode 100644
index 00000000000..96a854630ed
--- /dev/null
+++ b/tests/queries/0_stateless/03037_dynamic_merges_small.reference.j2
@@ -0,0 +1,112 @@
+5	DateTime
+6	Date
+7	Array(UInt16)
+8	String
+10	None
+10	UInt64
+7	Array(UInt16)
+10	None
+10	UInt64
+19	String
+7	Array(UInt16)
+10	None
+10	UInt64
+19	String
+20	Map(UInt64, UInt64)
+10	None
+10	UInt64
+20	Map(UInt64, UInt64)
+26	String
+1	Tuple(UInt64, UInt64)
+10	None
+10	UInt64
+20	Map(UInt64, UInt64)
+26	String
+10	None
+10	UInt64
+20	Map(UInt64, UInt64)
+27	String
+5	DateTime
+6	Date
+7	Array(UInt16)
+8	String
+10	None
+10	UInt64
+7	Array(UInt16)
+10	None
+10	UInt64
+19	String
+7	Array(UInt16)
+10	None
+10	UInt64
+19	String
+20	Map(UInt64, UInt64)
+10	None
+10	UInt64
+20	Map(UInt64, UInt64)
+26	String
+1	Tuple(UInt64, UInt64)
+10	None
+10	UInt64
+20	Map(UInt64, UInt64)
+26	String
+10	None
+10	UInt64
+20	Map(UInt64, UInt64)
+27	String
+5	DateTime
+6	Date
+7	Array(UInt16)
+8	String
+10	None
+10	UInt64
+7	Array(UInt16)
+10	None
+10	UInt64
+19	String
+7	Array(UInt16)
+10	None
+10	UInt64
+19	String
+20	Map(UInt64, UInt64)
+10	None
+10	UInt64
+20	Map(UInt64, UInt64)
+26	String
+1	Tuple(UInt64, UInt64)
+10	None
+10	UInt64
+20	Map(UInt64, UInt64)
+26	String
+10	None
+10	UInt64
+20	Map(UInt64, UInt64)
+27	String
+5	DateTime
+6	Date
+7	Array(UInt16)
+8	String
+10	None
+10	UInt64
+7	Array(UInt16)
+10	None
+10	UInt64
+19	String
+7	Array(UInt16)
+10	None
+10	UInt64
+19	String
+20	Map(UInt64, UInt64)
+10	None
+10	UInt64
+20	Map(UInt64, UInt64)
+26	String
+1	Tuple(UInt64, UInt64)
+10	None
+10	UInt64
+20	Map(UInt64, UInt64)
+26	String
+10	None
+10	UInt64
+20	Map(UInt64, UInt64)
+27	String
diff --git a/tests/queries/0_stateless/03037_dynamic_merges_small.sql.j2 b/tests/queries/0_stateless/03037_dynamic_merges_small.sql.j2
new file mode 100644
index 00000000000..fba349b4697
--- /dev/null
+++ b/tests/queries/0_stateless/03037_dynamic_merges_small.sql.j2
@@ -0,0 +1,44 @@
+-- Tags: long
+
+set allow_experimental_variant_type = 1;
+set use_variant_as_common_type = 1;
+set allow_experimental_dynamic_type = 1;
+
+drop table if exists test;
+
+{% for engine in ['MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000',
+                  'MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1',
+                  'MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1',
+                  'MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1'] -%}
+
+create table test (id UInt64, d Dynamic(max_types=3)) engine={{ engine }};
+
+system stop merges test;
+insert into test select number, number from numbers(10);
+insert into test select number, 'str_' || toString(number) from numbers(8);
+insert into test select number, range(number % 10 + 1) from numbers(7);
+insert into test select number, toDate(number) from numbers(6);
+insert into test select number, toDateTime(number) from numbers(5);
+insert into test select number, NULL from numbers(10);
+
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+system start merges test; optimize table test final;;
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+
+system stop merges test;
+insert into test select number, map(number, number) from numbers(20);
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+system start merges test;
+optimize table test final;
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+
+system stop merges test;
+insert into test select number, tuple(number, number) from numbers(1);
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+system start merges test;
+optimize table test final;
+select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
+
+drop table test;
+
+{% endfor -%}

From c5773def49511550f3505c8afdbf4b6a6f079f0c Mon Sep 17 00:00:00 2001
From: avogar <avogar@clickhouse.com>
Date: Mon, 29 Jul 2024 13:59:17 +0000
Subject: [PATCH 0986/1488] Remove long tag from small test

---
 .../0_stateless/03036_dynamic_read_subcolumns_small.sql.j2      | 2 --
 tests/queries/0_stateless/03037_dynamic_merges_small.sql.j2     | 2 --
 2 files changed, 4 deletions(-)

diff --git a/tests/queries/0_stateless/03036_dynamic_read_subcolumns_small.sql.j2 b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_small.sql.j2
index 17d9fe8a992..1ed836fbeee 100644
--- a/tests/queries/0_stateless/03036_dynamic_read_subcolumns_small.sql.j2
+++ b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_small.sql.j2
@@ -1,5 +1,3 @@
--- Tags: long
-
 set allow_experimental_variant_type = 1;
 set use_variant_as_common_type = 1;
 set allow_experimental_dynamic_type = 1;
diff --git a/tests/queries/0_stateless/03037_dynamic_merges_small.sql.j2 b/tests/queries/0_stateless/03037_dynamic_merges_small.sql.j2
index fba349b4697..263e92be403 100644
--- a/tests/queries/0_stateless/03037_dynamic_merges_small.sql.j2
+++ b/tests/queries/0_stateless/03037_dynamic_merges_small.sql.j2
@@ -1,5 +1,3 @@
--- Tags: long
-
 set allow_experimental_variant_type = 1;
 set use_variant_as_common_type = 1;
 set allow_experimental_dynamic_type = 1;

From 1333fe24645f36f2dd5f45a084d379de834e71a8 Mon Sep 17 00:00:00 2001
From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com>
Date: Mon, 29 Jul 2024 16:09:51 +0200
Subject: [PATCH 0987/1488] Update 01603_read_with_backoff_bug.sql

---
 tests/queries/0_stateless/01603_read_with_backoff_bug.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/01603_read_with_backoff_bug.sql b/tests/queries/0_stateless/01603_read_with_backoff_bug.sql
index 3d467515b5c..8b97f3514b3 100644
--- a/tests/queries/0_stateless/01603_read_with_backoff_bug.sql
+++ b/tests/queries/0_stateless/01603_read_with_backoff_bug.sql
@@ -1,4 +1,4 @@
--- Tags: no-tsan, no-distributed-cache
+-- Tags: long, no-tsan, no-distributed-cache
 -- Tag no-tsan: Too long for TSan
 
 set enable_filesystem_cache=0;

From 4417a9ae6f4d175761bccd01dc15f0648ef25e0b Mon Sep 17 00:00:00 2001
From: Max K <maxkaynov@gmail.com>
Date: Mon, 29 Jul 2024 16:14:45 +0200
Subject: [PATCH 0988/1488] style fix

---
 tests/ci/docker_server.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/ci/docker_server.py b/tests/ci/docker_server.py
index 479d63f1a3b..3e782c079c6 100644
--- a/tests/ci/docker_server.py
+++ b/tests/ci/docker_server.py
@@ -130,7 +130,7 @@ def retry_popen(cmd: str, log_file: Path) -> int:
     max_retries = 2
     sleep_seconds = 10
     retcode = -1
-    for retry in range(max_retries):
+    for _retry in range(max_retries):
         with TeePopen(
             cmd,
             log_file=log_file,

From 985e8ec15ef3cefb7063934ebd9590442faf5045 Mon Sep 17 00:00:00 2001
From: Amos Bird <amosbird@gmail.com>
Date: Sat, 13 Jul 2024 09:25:52 +0800
Subject: [PATCH 0989/1488] Clean up projection inside storage snapshot

---
 .../Passes/ShardNumColumnToFunctionPass.cpp   |  2 +-
 src/Interpreters/TreeRewriter.cpp             |  3 +-
 src/Planner/PlannerJoinTree.cpp               |  6 +--
 .../optimizeUseAggregateProjection.cpp        |  4 +-
 .../optimizeUseNormalProjection.cpp           |  4 +-
 .../QueryPlan/ReadFromMergeTree.cpp           | 49 +++++++++----------
 src/Processors/QueryPlan/ReadFromMergeTree.h  |  4 +-
 src/Storages/MergeTree/IMergeTreeReader.cpp   |  2 +-
 .../MergeTree/MergeTreeBlockReadUtils.cpp     |  3 +-
 src/Storages/MergeTree/MergeTreeData.cpp      |  7 +--
 src/Storages/StorageMerge.cpp                 |  2 +-
 src/Storages/StorageSnapshot.cpp              | 11 ++---
 src/Storages/StorageSnapshot.h                |  8 ---
 13 files changed, 40 insertions(+), 65 deletions(-)

diff --git a/src/Analyzer/Passes/ShardNumColumnToFunctionPass.cpp b/src/Analyzer/Passes/ShardNumColumnToFunctionPass.cpp
index 82e3281121c..c58504064ce 100644
--- a/src/Analyzer/Passes/ShardNumColumnToFunctionPass.cpp
+++ b/src/Analyzer/Passes/ShardNumColumnToFunctionPass.cpp
@@ -46,7 +46,7 @@ public:
             return;
 
         const auto & storage_snapshot = table_node ? table_node->getStorageSnapshot() : table_function_node->getStorageSnapshot();
-        if (!storage->isVirtualColumn(column.name, storage_snapshot->getMetadataForQuery()))
+        if (!storage->isVirtualColumn(column.name, storage_snapshot->metadata))
             return;
 
         auto function_node = std::make_shared<FunctionNode>("shardNum");
diff --git a/src/Interpreters/TreeRewriter.cpp b/src/Interpreters/TreeRewriter.cpp
index 6ce6f5e454e..14094c3cccf 100644
--- a/src/Interpreters/TreeRewriter.cpp
+++ b/src/Interpreters/TreeRewriter.cpp
@@ -1158,7 +1158,8 @@ bool TreeRewriterResult::collectUsedColumns(const ASTPtr & query, bool is_select
             }
         }
 
-        has_virtual_shard_num = is_remote_storage && storage->isVirtualColumn("_shard_num", storage_snapshot->getMetadataForQuery()) && virtuals->has("_shard_num");
+        has_virtual_shard_num
+            = is_remote_storage && storage->isVirtualColumn("_shard_num", storage_snapshot->metadata) && virtuals->has("_shard_num");
     }
 
     /// Collect missed object subcolumns
diff --git a/src/Planner/PlannerJoinTree.cpp b/src/Planner/PlannerJoinTree.cpp
index a3db0395ccc..bc31af32a20 100644
--- a/src/Planner/PlannerJoinTree.cpp
+++ b/src/Planner/PlannerJoinTree.cpp
@@ -417,20 +417,20 @@ void updatePrewhereOutputsIfNeeded(SelectQueryInfo & table_expression_query_info
             /// We evaluate sampling for Merge lazily so we need to get all the columns
             if (storage_snapshot->storage.getName() == "Merge")
             {
-                const auto columns = storage_snapshot->getMetadataForQuery()->getColumns().getAll();
+                const auto columns = storage_snapshot->metadata->getColumns().getAll();
                 for (const auto & column : columns)
                     required_columns.insert(column.name);
             }
             else
             {
-                auto columns_required_for_sampling = storage_snapshot->getMetadataForQuery()->getColumnsRequiredForSampling();
+                auto columns_required_for_sampling = storage_snapshot->metadata->getColumnsRequiredForSampling();
                 required_columns.insert(columns_required_for_sampling.begin(), columns_required_for_sampling.end());
             }
         }
 
         if (table_expression_modifiers->hasFinal())
         {
-            auto columns_required_for_final = storage_snapshot->getMetadataForQuery()->getColumnsRequiredForFinal();
+            auto columns_required_for_final = storage_snapshot->metadata->getColumnsRequiredForFinal();
             required_columns.insert(columns_required_for_final.begin(), columns_required_for_final.end());
         }
     }
diff --git a/src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp b/src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp
index ad89cec5f79..52d1931c51e 100644
--- a/src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp
+++ b/src/Processors/QueryPlan/Optimizations/optimizeUseAggregateProjection.cpp
@@ -757,9 +757,7 @@ std::optional<String> optimizeUseAggregateProjections(QueryPlan::Node & node, Qu
     else
     {
         auto storage_snapshot = reading->getStorageSnapshot();
-        auto proj_snapshot = std::make_shared<StorageSnapshot>(storage_snapshot->storage, storage_snapshot->metadata);
-        proj_snapshot->addProjection(best_candidate->projection);
-
+        auto proj_snapshot = std::make_shared<StorageSnapshot>(storage_snapshot->storage, best_candidate->projection->metadata);
         auto projection_query_info = query_info;
         projection_query_info.prewhere_info = nullptr;
         projection_query_info.filter_actions_dag = nullptr;
diff --git a/src/Processors/QueryPlan/Optimizations/optimizeUseNormalProjection.cpp b/src/Processors/QueryPlan/Optimizations/optimizeUseNormalProjection.cpp
index e9aee65675f..b15f913fc19 100644
--- a/src/Processors/QueryPlan/Optimizations/optimizeUseNormalProjection.cpp
+++ b/src/Processors/QueryPlan/Optimizations/optimizeUseNormalProjection.cpp
@@ -193,9 +193,7 @@ std::optional<String> optimizeUseNormalProjections(Stack & stack, QueryPlan::Nod
     }
 
     auto storage_snapshot = reading->getStorageSnapshot();
-    auto proj_snapshot = std::make_shared<StorageSnapshot>(storage_snapshot->storage, storage_snapshot->metadata);
-    proj_snapshot->addProjection(best_candidate->projection);
-
+    auto proj_snapshot = std::make_shared<StorageSnapshot>(storage_snapshot->storage, best_candidate->projection->metadata);
     auto query_info_copy = query_info;
     query_info_copy.prewhere_info = nullptr;
 
diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp
index 483876dd293..fed94632739 100644
--- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp
+++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp
@@ -283,7 +283,6 @@ ReadFromMergeTree::ReadFromMergeTree(
     , all_column_names(std::move(all_column_names_))
     , data(data_)
     , actions_settings(ExpressionActionsSettings::fromContext(context_))
-    , metadata_for_reading(storage_snapshot->getMetadataForQuery())
     , block_size{
         .max_block_size_rows = max_block_size_,
         .preferred_block_size_bytes = context->getSettingsRef().preferred_block_size_bytes,
@@ -325,7 +324,7 @@ ReadFromMergeTree::ReadFromMergeTree(
 
     updateSortDescriptionForOutputStream(
         *output_stream,
-        storage_snapshot->getMetadataForQuery()->getSortingKeyColumns(),
+        storage_snapshot->metadata->getSortingKeyColumns(),
         getSortDirection(),
         query_info.input_order_info,
         prewhere_info,
@@ -780,7 +779,7 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreams(RangesInDataParts && parts_
         Names in_order_column_names_to_read(column_names);
 
         /// Add columns needed to calculate the sorting expression
-        for (const auto & column_name : metadata_for_reading->getColumnsRequiredForSortingKey())
+        for (const auto & column_name : storage_snapshot->metadata->getColumnsRequiredForSortingKey())
         {
             if (column_names_set.contains(column_name))
                 continue;
@@ -800,10 +799,10 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreams(RangesInDataParts && parts_
                 info.use_uncompressed_cache);
         };
 
-        auto sorting_expr = metadata_for_reading->getSortingKey().expression;
+        auto sorting_expr = storage_snapshot->metadata->getSortingKey().expression;
 
         SplitPartsWithRangesByPrimaryKeyResult split_ranges_result = splitPartsWithRangesByPrimaryKey(
-            metadata_for_reading->getPrimaryKey(),
+            storage_snapshot->metadata->getPrimaryKey(),
             std::move(sorting_expr),
             std::move(parts_with_ranges),
             num_streams,
@@ -881,7 +880,7 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsWithOrder(
     if (prewhere_info)
     {
         NameSet sorting_columns;
-        for (const auto & column : metadata_for_reading->getSortingKey().expression->getRequiredColumnsWithTypes())
+        for (const auto & column : storage_snapshot->metadata->getSortingKey().expression->getRequiredColumnsWithTypes())
             sorting_columns.insert(column.name);
 
         have_input_columns_removed_after_prewhere = restorePrewhereInputs(*prewhere_info, sorting_columns);
@@ -1036,12 +1035,12 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsWithOrder(
     if (need_preliminary_merge || output_each_partition_through_separate_port)
     {
         size_t prefix_size = input_order_info->used_prefix_of_sorting_key_size;
-        auto order_key_prefix_ast = metadata_for_reading->getSortingKey().expression_list_ast->clone();
+        auto order_key_prefix_ast = storage_snapshot->metadata->getSortingKey().expression_list_ast->clone();
         order_key_prefix_ast->children.resize(prefix_size);
 
-        auto syntax_result = TreeRewriter(context).analyze(order_key_prefix_ast, metadata_for_reading->getColumns().getAllPhysical());
+        auto syntax_result = TreeRewriter(context).analyze(order_key_prefix_ast, storage_snapshot->metadata->getColumns().getAllPhysical());
         auto sorting_key_prefix_expr = ExpressionAnalyzer(order_key_prefix_ast, syntax_result, context).getActionsDAG(false);
-        const auto & sorting_columns = metadata_for_reading->getSortingKey().column_names;
+        const auto & sorting_columns = storage_snapshot->metadata->getSortingKey().column_names;
 
         SortDescription sort_description;
         sort_description.compile_sort_description = settings.compile_sort_description;
@@ -1148,7 +1147,7 @@ bool ReadFromMergeTree::doNotMergePartsAcrossPartitionsFinal() const
     if (settings.do_not_merge_across_partitions_select_final.changed)
         return settings.do_not_merge_across_partitions_select_final;
 
-    if (!metadata_for_reading->hasPrimaryKey() || !metadata_for_reading->hasPartitionKey())
+    if (!storage_snapshot->metadata->hasPrimaryKey() || !storage_snapshot->metadata->hasPartitionKey())
         return false;
 
     /** To avoid merging parts across partitions we want result of partition key expression for
@@ -1158,11 +1157,11 @@ bool ReadFromMergeTree::doNotMergePartsAcrossPartitionsFinal() const
       * in primary key, then for same primary key column values, result of partition key expression
       * will be the same.
       */
-    const auto & partition_key_expression = metadata_for_reading->getPartitionKey().expression;
+    const auto & partition_key_expression = storage_snapshot->metadata->getPartitionKey().expression;
     if (partition_key_expression->getActionsDAG().hasNonDeterministic())
         return false;
 
-    const auto & primary_key_columns = metadata_for_reading->getPrimaryKey().column_names;
+    const auto & primary_key_columns = storage_snapshot->metadata->getPrimaryKey().column_names;
     NameSet primary_key_columns_set(primary_key_columns.begin(), primary_key_columns.end());
 
     const auto & partition_key_required_columns = partition_key_expression->getRequiredColumns();
@@ -1215,12 +1214,12 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal(
     /// we will store lonely parts with level > 0 to use parallel select on them.
     RangesInDataParts non_intersecting_parts_by_primary_key;
 
-    auto sorting_expr = metadata_for_reading->getSortingKey().expression;
+    auto sorting_expr = storage_snapshot->metadata->getSortingKey().expression;
 
     if (prewhere_info)
     {
         NameSet sorting_columns;
-        for (const auto & column : metadata_for_reading->getSortingKey().expression->getRequiredColumnsWithTypes())
+        for (const auto & column : storage_snapshot->metadata->getSortingKey().expression->getRequiredColumnsWithTypes())
             sorting_columns.insert(column.name);
         restorePrewhereInputs(*prewhere_info, sorting_columns);
     }
@@ -1251,7 +1250,7 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal(
             if (new_parts.empty())
                 continue;
 
-            if (num_streams > 1 && metadata_for_reading->hasPrimaryKey())
+            if (num_streams > 1 && storage_snapshot->metadata->hasPrimaryKey())
             {
                 // Let's split parts into non intersecting parts ranges and layers to ensure data parallelism of FINAL.
                 auto in_order_reading_step_getter = [this, &column_names, &info](auto parts)
@@ -1271,7 +1270,7 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal(
                     data.merging_params.is_deleted_column.empty() && !reader_settings.read_in_order;
 
                 SplitPartsWithRangesByPrimaryKeyResult split_ranges_result = splitPartsWithRangesByPrimaryKey(
-                    metadata_for_reading->getPrimaryKey(),
+                    storage_snapshot->metadata->getPrimaryKey(),
                     sorting_expr,
                     std::move(new_parts),
                     num_streams,
@@ -1303,7 +1302,7 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal(
         if (pipes.empty())
             continue;
 
-        Names sort_columns = metadata_for_reading->getSortingKeyColumns();
+        Names sort_columns = storage_snapshot->metadata->getSortingKeyColumns();
         SortDescription sort_description;
         sort_description.compile_sort_description = settings.compile_sort_description;
         sort_description.min_count_to_compile_sort_description = settings.min_count_to_compile_sort_description;
@@ -1311,7 +1310,7 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal(
         size_t sort_columns_size = sort_columns.size();
         sort_description.reserve(sort_columns_size);
 
-        Names partition_key_columns = metadata_for_reading->getPartitionKey().column_names;
+        Names partition_key_columns = storage_snapshot->metadata->getPartitionKey().column_names;
 
         for (size_t i = 0; i < sort_columns_size; ++i)
             sort_description.emplace_back(sort_columns[i], 1, 1);
@@ -1368,7 +1367,7 @@ ReadFromMergeTree::AnalysisResultPtr ReadFromMergeTree::selectRangesToRead(
     return selectRangesToRead(
         std::move(parts),
         std::move(alter_conversions),
-        metadata_for_reading,
+        storage_snapshot->metadata,
         query_info,
         context,
         requested_num_streams,
@@ -1532,7 +1531,7 @@ void ReadFromMergeTree::applyFilters(ActionDAGNodes added_filter_nodes)
             prepared_parts,
             context,
             query_info,
-            metadata_for_reading);
+            storage_snapshot->metadata);
     }
 }
 
@@ -1701,7 +1700,7 @@ bool ReadFromMergeTree::requestReadingInOrder(size_t prefix_size, int direction,
 
     /// update sort info for output stream
     SortDescription sort_description;
-    const Names & sorting_key_columns = metadata_for_reading->getSortingKeyColumns();
+    const Names & sorting_key_columns = storage_snapshot->metadata->getSortingKeyColumns();
     const Block & header = output_stream->header;
     const int sort_direction = getSortDirection();
     for (const auto & column_name : sorting_key_columns)
@@ -1743,7 +1742,7 @@ void ReadFromMergeTree::updatePrewhereInfo(const PrewhereInfoPtr & prewhere_info
 
     updateSortDescriptionForOutputStream(
         *output_stream,
-        storage_snapshot->getMetadataForQuery()->getSortingKeyColumns(),
+        storage_snapshot->metadata->getSortingKeyColumns(),
         getSortDirection(),
         query_info.input_order_info,
         prewhere_info,
@@ -1869,7 +1868,7 @@ Pipe ReadFromMergeTree::spreadMarkRanges(
             throw Exception(ErrorCodes::LOGICAL_ERROR, "Optimization isn't supposed to be used for queries with final");
 
         /// Add columns needed to calculate the sorting expression and the sign.
-        for (const auto & column : metadata_for_reading->getColumnsRequiredForSortingKey())
+        for (const auto & column : storage_snapshot->metadata->getColumnsRequiredForSortingKey())
         {
             if (!names.contains(column))
             {
@@ -1963,10 +1962,6 @@ void ReadFromMergeTree::initializePipeline(QueryPipelineBuilder & pipeline, cons
                 fmt::format("{}.{}", data.getStorageID().getFullNameNotQuoted(), part.data_part->info.partition_id));
         }
         context->getQueryContext()->addQueryAccessInfo(partition_names);
-
-        if (storage_snapshot->projection)
-            context->getQueryContext()->addQueryAccessInfo(
-                Context::QualifiedProjectionName{.storage_id = data.getStorageID(), .projection_name = storage_snapshot->projection->name});
     }
 
     ProfileEvents::increment(ProfileEvents::SelectedParts, result.selected_parts);
diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.h b/src/Processors/QueryPlan/ReadFromMergeTree.h
index a12f53924c3..f12da5d10bc 100644
--- a/src/Processors/QueryPlan/ReadFromMergeTree.h
+++ b/src/Processors/QueryPlan/ReadFromMergeTree.h
@@ -171,7 +171,7 @@ public:
 
     AnalysisResultPtr selectRangesToRead(bool find_exact_ranges = false) const;
 
-    StorageMetadataPtr getStorageMetadata() const { return metadata_for_reading; }
+    StorageMetadataPtr getStorageMetadata() const { return storage_snapshot->metadata; }
 
     /// Returns `false` if requested reading cannot be performed.
     bool requestReadingInOrder(size_t prefix_size, int direction, size_t limit);
@@ -216,8 +216,6 @@ private:
     const MergeTreeData & data;
     ExpressionActionsSettings actions_settings;
 
-    StorageMetadataPtr metadata_for_reading;
-
     const MergeTreeReadTask::BlockSizeParams block_size;
 
     size_t requested_num_streams;
diff --git a/src/Storages/MergeTree/IMergeTreeReader.cpp b/src/Storages/MergeTree/IMergeTreeReader.cpp
index 264b2b397f4..b030d4335af 100644
--- a/src/Storages/MergeTree/IMergeTreeReader.cpp
+++ b/src/Storages/MergeTree/IMergeTreeReader.cpp
@@ -75,7 +75,7 @@ void IMergeTreeReader::fillVirtualColumns(Columns & columns, size_t rows) const
         throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Filling of virtual columns is supported only for LoadedMergeTreeDataPartInfoForReader");
 
     const auto & data_part = loaded_part_info->getDataPart();
-    const auto & storage_columns = storage_snapshot->getMetadataForQuery()->getColumns();
+    const auto & storage_columns = storage_snapshot->metadata->getColumns();
     const auto & virtual_columns = storage_snapshot->virtual_columns;
 
     auto it = requested_columns.begin();
diff --git a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp
index aaa4ecd8eee..3dbb9d64f2f 100644
--- a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp
+++ b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp
@@ -71,8 +71,7 @@ bool injectRequiredColumnsRecursively(
 
     /// Column doesn't have default value and don't exist in part
     /// don't need to add to required set.
-    auto metadata_snapshot = storage_snapshot->getMetadataForQuery();
-    const auto column_default = metadata_snapshot->getColumns().getDefault(column_name);
+    const auto column_default = storage_snapshot->metadata->getColumns().getDefault(column_name);
     if (!column_default)
         return false;
 
diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index 2e10f5a0227..2aee3d2ce72 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -7137,12 +7137,7 @@ UInt64 MergeTreeData::estimateNumberOfRowsToRead(
 
     MergeTreeDataSelectExecutor reader(*this);
     auto result_ptr = reader.estimateNumMarksToRead(
-        parts,
-        storage_snapshot->getMetadataForQuery()->getColumns().getAll().getNames(),
-        storage_snapshot->metadata,
-        query_info,
-        query_context,
-        query_context->getSettingsRef().max_threads);
+        parts, {}, storage_snapshot->metadata, query_info, query_context, query_context->getSettingsRef().max_threads);
 
     UInt64 total_rows = result_ptr->selected_rows;
     if (query_info.trivial_limit > 0 && query_info.trivial_limit < total_rows)
diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp
index 7c268d36a7b..e88844e2d31 100644
--- a/src/Storages/StorageMerge.cpp
+++ b/src/Storages/StorageMerge.cpp
@@ -602,7 +602,7 @@ std::vector<ReadFromMerge::ChildPlan> ReadFromMerge::createChildrenPlans(SelectQ
                 ASTPtr required_columns_expr_list = std::make_shared<ASTExpressionList>();
                 ASTPtr column_expr;
 
-                auto sample_block = merge_storage_snapshot->getMetadataForQuery()->getSampleBlock();
+                auto sample_block = merge_storage_snapshot->metadata->getSampleBlock();
 
                 for (const auto & column : real_column_names)
                 {
diff --git a/src/Storages/StorageSnapshot.cpp b/src/Storages/StorageSnapshot.cpp
index aada25168f8..13a46990556 100644
--- a/src/Storages/StorageSnapshot.cpp
+++ b/src/Storages/StorageSnapshot.cpp
@@ -63,7 +63,6 @@ std::shared_ptr<StorageSnapshot> StorageSnapshot::clone(DataPtr data_) const
 {
     auto res = std::make_shared<StorageSnapshot>(storage, metadata, object_columns);
 
-    res->projection = projection;
     res->data = std::move(data_);
 
     return res;
@@ -79,7 +78,7 @@ ColumnsDescription StorageSnapshot::getAllColumnsDescription() const
 
 NamesAndTypesList StorageSnapshot::getColumns(const GetColumnsOptions & options) const
 {
-    auto all_columns = getMetadataForQuery()->getColumns().get(options);
+    auto all_columns = metadata->getColumns().get(options);
 
     if (options.with_extended_objects)
         extendObjectColumns(all_columns, object_columns, options.with_subcolumns);
@@ -113,7 +112,7 @@ NamesAndTypesList StorageSnapshot::getColumnsByNames(const GetColumnsOptions & o
 
 std::optional<NameAndTypePair> StorageSnapshot::tryGetColumn(const GetColumnsOptions & options, const String & column_name) const
 {
-    const auto & columns = getMetadataForQuery()->getColumns();
+    const auto & columns = metadata->getColumns();
     auto column = columns.tryGetColumn(options, column_name);
     if (column && (!column->type->hasDynamicSubcolumnsDeprecated() || !options.with_extended_objects))
         return column;
@@ -189,7 +188,7 @@ Block StorageSnapshot::getSampleBlockForColumns(const Names & column_names) cons
 {
     Block res;
 
-    const auto & columns = getMetadataForQuery()->getColumns();
+    const auto & columns = metadata->getColumns();
     for (const auto & column_name : column_names)
     {
         auto column = columns.tryGetColumnOrSubcolumn(GetColumnsOptions::All, column_name);
@@ -221,7 +220,7 @@ Block StorageSnapshot::getSampleBlockForColumns(const Names & column_names) cons
 ColumnsDescription StorageSnapshot::getDescriptionForColumns(const Names & column_names) const
 {
     ColumnsDescription res;
-    const auto & columns = getMetadataForQuery()->getColumns();
+    const auto & columns = metadata->getColumns();
     for (const auto & name : column_names)
     {
         auto column = columns.tryGetColumnOrSubcolumnDescription(GetColumnsOptions::All, name);
@@ -257,7 +256,7 @@ namespace
 
 void StorageSnapshot::check(const Names & column_names) const
 {
-    const auto & columns = getMetadataForQuery()->getColumns();
+    const auto & columns = metadata->getColumns();
     auto options = GetColumnsOptions(GetColumnsOptions::AllPhysical).withSubcolumns();
 
     if (column_names.empty())
diff --git a/src/Storages/StorageSnapshot.h b/src/Storages/StorageSnapshot.h
index 89e97f2abb8..e94d1fc9a26 100644
--- a/src/Storages/StorageSnapshot.h
+++ b/src/Storages/StorageSnapshot.h
@@ -30,9 +30,6 @@ struct StorageSnapshot
     using DataPtr = std::unique_ptr<Data>;
     DataPtr data;
 
-    /// Projection that is used in query.
-    mutable const ProjectionDescription * projection = nullptr;
-
     StorageSnapshot(
         const IStorage & storage_,
         StorageMetadataPtr metadata_);
@@ -82,11 +79,6 @@ struct StorageSnapshot
     void check(const Names & column_names) const;
 
     DataTypePtr getConcreteType(const String & column_name) const;
-
-    void addProjection(const ProjectionDescription * projection_) const { projection = projection_; }
-
-    /// If we have a projection then we should use its metadata.
-    StorageMetadataPtr getMetadataForQuery() const { return projection ? projection->metadata : metadata; }
 };
 
 using StorageSnapshotPtr = std::shared_ptr<StorageSnapshot>;

From af63af8ff2d5f86aa22300b1e19c37b36307dac9 Mon Sep 17 00:00:00 2001
From: kssenii <sumarokovakseniia@mail.ru>
Date: Mon, 29 Jul 2024 16:44:19 +0200
Subject: [PATCH 0990/1488] Update a few more tests

---
 tests/queries/0_stateless/02150_index_hypothesis_race_long.sh | 1 +
 tests/queries/0_stateless/02995_index_2.sh                    | 2 +-
 tests/queries/0_stateless/02995_index_4.sh                    | 2 +-
 3 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/tests/queries/0_stateless/02150_index_hypothesis_race_long.sh b/tests/queries/0_stateless/02150_index_hypothesis_race_long.sh
index 114f60cc393..be7cfa78492 100755
--- a/tests/queries/0_stateless/02150_index_hypothesis_race_long.sh
+++ b/tests/queries/0_stateless/02150_index_hypothesis_race_long.sh
@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
+# Tags: no-distributed-cache
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
diff --git a/tests/queries/0_stateless/02995_index_2.sh b/tests/queries/0_stateless/02995_index_2.sh
index e7451c7ee4b..f2f466e8ff7 100755
--- a/tests/queries/0_stateless/02995_index_2.sh
+++ b/tests/queries/0_stateless/02995_index_2.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Tags: long, no-debug, no-asan, no-tsan, no-msan, no-ubsan, no-sanitize-coverage
+# Tags: long, no-debug, no-asan, no-tsan, no-msan, no-ubsan, no-sanitize-coverage, no-distributed-cache
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
diff --git a/tests/queries/0_stateless/02995_index_4.sh b/tests/queries/0_stateless/02995_index_4.sh
index 1a0458728f9..59be06ae17a 100755
--- a/tests/queries/0_stateless/02995_index_4.sh
+++ b/tests/queries/0_stateless/02995_index_4.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Tags: long, no-debug, no-asan, no-tsan, no-msan, no-ubsan, no-sanitize-coverage
+# Tags: long, no-debug, no-asan, no-tsan, no-msan, no-ubsan, no-sanitize-coverage, no-distributed-cache
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh

From c48770fdf6f769c5f4678db32ca27245b9fa3158 Mon Sep 17 00:00:00 2001
From: maxvostrikov <max.vostrikov@clickhouse.com>
Date: Mon, 29 Jul 2024 16:46:38 +0200
Subject: [PATCH 0991/1488] squash! added somme tests in relation with
 https://github.com/ClickHouse/ClickHouse/pull/54881 with new behaviour when
 enable_named_columns_in_function_tuple=1 (default value)

---
 .../0_stateless/00309_formats.reference       | Bin 18666 -> 20353 bytes
 tests/queries/0_stateless/00309_formats.sql   |   5 -----
 2 files changed, 5 deletions(-)

diff --git a/tests/queries/0_stateless/00309_formats.reference b/tests/queries/0_stateless/00309_formats.reference
index f3ea45520bb50fb936caf6724e9fedf3cdd00b75..cab311692be229716b58af39079275d3942b01cc 100644
GIT binary patch
literal 20353
zcmeHP3wIP%5{6aK9zDL_&p18;axq=E-!Ke<i0?P}N)Q7X6ip_W3@R%6%l7O~+P|=0
zRo6hOl6Z`+M|N!w(_i=PTYbO&s&ea2b&tJW?SKFN*FT@{>He<Wy7Ldkn-p(W+@rWx
z@fO8f6_Y9M{>M!>@7a6Ht;zJ#Jk0d+*gt^JbdYKDu4%*X9d$a^ZOY<y#cBH;dVHti
zU5a-r-lKT0;(dzuD?Xt3pyES{4=X;R_^9GzijONkq4=cYjN(&@eZ{90pHX~P@j1m=
z#eItV6`xl;pcoVnD!!mNr<f{|VpPl&bHzfjRIC(h#YXXv;$g*s;t|E6;!(vH6^|(%
zSA0qFgyPGJCly~&d{yx^#n%<zP<&JIEycGL-%)&5@jb=&6+cisrFdHLL&c92KUVxi
z@l(Zl#RbJPik~T-Rs39WQSqGOdBr8gk>Ul#FBHF2yr}q<;@67HiYtm^#Z|>iir*-%
zDXuF{6fY}&tN5Mb_lj2(e^C5U@h8Qria#s<qKG-5+v_yg)y2WCP6NyYf5SM~eSM*H
z+wHwO?!4>nd+xpO{s$g>=;22meeCfko}78A|MWA@J~z8>|MLgJ!58LIidkNiRoxsq
zJUB8u`r@(UFP(V#<SVbf_WB!dzV-Gy@4ol`2d7Sd_|eCod^*2y=CiY(FP=NUG`jG`
zmlwbKdU<8Mdg+_B^~vRLzx)2m4?q5N_2*w8dH3%xRA9RH1%(59c8_;2-L$y2wtIcg
z<npEE(caUqjVEP(^NGdDsCR5_ZSl&!wq^g`4<>6D$6wrgio0oJnBji=r;A-&v0K--
z%ZqMZ-|En<>suYVb$zQtx30$|*-G(uTzhnP-g&=^d;Asr39~@AuK#;~vDmn-A16x*
zR`tnxBCGmdyM|AAwVs$&ebTP#ldV?ulqNAv(}O7;BueJfKp}=K<e>;<s6ri@Gl_ne
zTsXHdN%m11X0Kb!?(fB-)Jrr6DckZA0Y*fDVxTNg9;gUZ2C4$pftnE05HV~FF$*yd
zu?VpYu?n#cu?blkG71@3Wg*K$R)nk!SrxK6WKGD^kW<KG$U!;}c@gq5<W<P)kT;=7
zLqVa4p~ymkdn!UvhN22Z9f~HDX(%a_F_c*-^HAcJ%TQLKtV7v^Dh(BdDuyZxRUWD$
zR8XV}RUN7()M=<G)G^dqsPj-4p)NxW!Rk;qp-DqSp^2f%LX(H42u&H9Dm2iuff|q_
zg&rvcNufvzNm6K%LX;G$q>v>=KamanSU6UWrDN?_JXVk8gMI`72?`P<#P@i7&<`pS
zWF+WF5R#xIK}v#_1ThI}667T4Nf4BvC_z$!rUX$5suE-+=t>Zlpe#XJg0=*43F;E$
zCFn~Kn4mC0VuHp5kqIgjWG3iL5SpMgL28251hENf6XYi7O%R-*I6-oP<^<6RsuN@<
z=uQxxpgci(g7yUQ3F;H%C+JUb0Ko$U7Z7|va00;#1UC@;KyU=X69iWfd_iyq!5aj3
z5d48~#s?7ye?<5r!XFX-i10^*KO+1Q;g1M^M8t7~KO+1Q;g1M^MEE1Z9})hD9M2rk
z64B_#G0j2EQO#k^am|4Ze?<5r!XFX-i0}tWF!-P!{1M@g2!BNQBf=jM{)q5Lgg+wu
z5#f&re?<5r!XFX-i10^*KO+1Q;g1M^#G){NzzgHs7A`ETZCepq_<bdzt|o+Je}n%^
z_F<}?PTo9MWAfg?^nYv8P5#~F-%bAiqmw^gndqJb+~70xpFwZySwHlzui^Wlzva3g
z?bn_n^|2r|HoWYgHgGQ4<|1s?t^B){f4B1gOIH3H|Ae=l{<|l^H~Px`kK~0uz5?2r
zy7yx*>933?i~U2rWWJXqI8KuOcyVRaKQz-H_k$cyu3VDiwye}g+f_bcJ1@D;TzuN5
zD@s+avnf(-W3}#V+EUgwouv-u47MnGiFGeI6M7J0X>pSftOBKaoh3fnm@GvczL`Ta
z^L_N{6Fw0i&Ve~T1wJJ{6+SgSO&=q%!s~pgh07?nk!+)EGuh^{Eo58Dw&Hr*^k(>Z
zybQ>^Hqtrc{32`%3@;;YAm1kG-$5QXD0j)XNqnD@?1}G`_*RMUmH1|f@0J>DqOcTU
z*dnqtku4%yM7D@*5!oWLMaC8xTV$doTV!mJu|>uf8Czs*k+DV27CBqwY>|unY>~4?
z&K5aa<ZO|%MZp#YTNG?jutmWZ1zWIa{emqDwkX)5WQ&q5O13E3qGXGbElRd1*`j2N
zk}WE>sMw-ni;68Owy4;mVvC9`Dz>QDqGpSlEo!!?*`j8Pnk{O!sM(@si<&JOwrJR*
zVT*<>8n$TIqG5}MEgH6H*a8(t*|H6)j<RJdR32r^cBnqemMu|%l&#pR`^B~mfHnf!
z5NKnd4T3fb+AxqfKxLAz6U8>lxDpGfP)fUuE8&1TrL@Sn5)r6Ya*M<SDweV@oFyz!
zx0Dv)ERligrL+iV2@cdSxkbVQbxhfpaV0`f&6E}ySAqmJO>U7eL0wb!Wn76AR5ztX
z#+6_}jZ<1=T!|M{I=Mw62Gvg4mvJR%Q1g@)8CT*4l}~AraV2z6|Kt`49@Ic(U&fXA
zK_ygLWLya$)I+&Nf(SKH*_Ux8j!+qu78zGU3H4EFk#QxOP$lISi6>M_WnadXkV3sw
zT4Y>_DpXCSg~!FbC|e#E)l=EBEh?z8Wm{BHWy|BDk}6xVZKpC4TZt>wRQasL73!+8
zFSZg_sIE$j*tQdl#8s5ILY<Y*wv$Xd%e2!>JI}Nejl>nIuG~}N3YAyc7h8!d)L*4V
z#+A526;@hgT!|~xV!1`)3iVjom;05tLRD5;<bEZtP@Clzi7V7+WnadXxI&dyT8clA
zCKkq$GLCa;SN2|(?MdEJDqBf8Bq@g^<&dNtl9WS|a;TD?wQL*N9+T~H*}f#(6S9@-
z)5)2q7R>5#VP`KA{>G~DrX+|J4NE4^tt78W7My1Ewbk!9(?&_}d~o*M=)&sS=<M>v
zm5Y<Ji<7g<qs8?Jcj;a27DK#%m`b;CF*J`X&<}R)vx61S&R06PcsnIrdmh<8BP*wQ
z{O!QuQ!4FB<~LKp+$Iwo|J`-#7c5hFy6M(0eLPL&i0IZYvfRaUT2`Z5zoe4SyY-8F
zcfylpM8$7kzwC6<|9@Y%AW3art6;IxmL7JtP#GNker0*)+tJ$k#nthVS$ZLzof(bK
zuP)(*{UfudPn|qi&(0hj_6BRC^~=kXVJ{hQkq4Lz&aW<CUKy|R{`5fhlL1$KhU3AU
zAfHYKT=*HbRiD9}9EjOmdy(y~v9$RbTnyU56;M;zd<|10+QG%vH(#nP9qr&6Qcl`@
zoh>R#+gx;@nz8VMIrYYX7g?<00RC^)91khQIPJmG@OrgrFt^k;LYv_Lo5d@`!DxB3
zGDLsD_K3_;G)I{^%FR(>j!JV>nWNeqHPfRx<;2)BT_0Y1Xf&A9h!Y`)0|nvDH@x8n
zHr{gEJg2=CS+6-2Y@=}a^V*~wFA#5d<+f6RtRKlbl5c8*WMo}Q)|X_RN!FWW-AUK{
zDc6tAH<nejtfFNVEvsl*MawE$R?)JGmQ`k1WtLTDS!I@0W?5yHRc2XbmQ`k1WtLTL
zS>={hZdv7)Rc=}3mQ`+9<(5@$S>={hVObTHRbg2bmQ`U{6_yqMxorr-!m=tXtHQD>
zEvwS9DlMzhvMMdB(y}TotJ1P6EvwS9sw}I@vZ^er%Cf2~tID#fEUU`0sw}I@vZ^hs
z+On!GtJ<=vEvwqHsx7P9vZ^hs+OldatH!cwEUU({YAmb9vT7`=#<FTGtH!cIb=w3b
z-mP`VmKff-b>}TLyo>A3TXJ|u*PXZY@b0cX?~)z)dEQU-ex~<Ry`Ss-WbbEtKi#|e
z9#wkFH#+^?l#h3eU4N&aoAvP?vg_~ka}z(_Rkr=z)Q@KZ?!5D_oBi>gv+M8t>k<Io
zg?9a&e_am1`_i_*%K~@;;m$k#Tq3}`)~>(P&*cKVkL~(9{aiZ0vklwdB?P?7?an*>
zTu#9I-mbsX&!q*_8f<@;81VeWop<`V+<^DVU4N&aOAmPG-1T?*xeS5#(rtg2Bk;7w
zop<`VG=X>8U4N&a%M^I;-Sv0+xnzOoK(@b27kFphop<`VjDh#+U4N&aOB#6h-u8EC
z1J9b=d8ePt9C&Zv^>_NY<biklU4N&a%OCjiVfy2@0PfiG!*2xKdCL#K9dPHZe)vs+
zJ8${nw+8mS%OCg+f;;c{x%`1AYOcTI=kf=B%i#JuelCCD_YStd%O7|`=gvF*{Fero
zKk&N=XYcsA{DI$A*#0hm;Mt!$@A$d=f!}4g{!TxaKk(ZP*Wc;q@&|t3Vf(xMfhUdb
zywlI+5BxU7^>_NY{DCT;>+ke)`2)`|ZGV?P@Y@r2-syKefBb6wVB0s=_<vopxN_<6
zmcLbbZh3O}zw>`E`L+6m)Ezc_5xV{`a(e6A7cfwpXiGp`$XFl3xq_$>F>d{|a@uQ~
Lvprlx%YOd>&)TR8

delta 3052
zcmeIy$4}c(00wZIgtksdZ-w^I3JD}eL2?j6{R4XG1$t-?y;Q0c4wb5?2YL>Bj|8*#
z-Xp>6y?59U_TGDMNaN%a?SDX3#j^bK`<|bk?ZfAlN^UDiN<bJ6jY24SuCXUb!%+R!
z(?lWq$P}8rBt@#-Sf0!i4|+jw=mUMBAM}R-Fc1d8U>E{JVHgaD5e8+19DxExLIy^`
zXcz-yVH}Ky2`~{R!DN^MQ}xOSA`L-0%z&9N3ueO{m<#h@J}iKRum~2z5?IQq+n-P(
zP{A_D!g5#vD`6F^hBdGj*1>w%02^VGPHumqnY-wY76U{pY=iBv19rkL*bRGNFYJT;
zPz?v*AoeGQ5Ddc+I10z$IGlhQI0>iVG@OC6a1PGH1;h6f#3F(vxC~d|DqMr>a070_
zEw~ML;4a*Q`|v>j|8;{y1V``~p1@Oh2G8LIyo6Wq8s5NLcn9yHg%&qjk~HD*O`kk{
zMhS#wBF4@26;n)ok$HM~`}q3#2LuKMhlGZON5~bCOjLACY+QUoVp4KSYFc_mW>$7i
zZeD&tVNr2OsZv$OmRD3(RoB$k)i*RYHMg|3wRd!Ob@%l4^{WR4hlWQ+$Hpf#lT*_(
zvvczci%ZKZt842Un_JsEyL<ZwheyXJr)TFEmsi&}w|Dmf?L&1j3gH`4#3+R{X>2uP
zs~KC(*lPbltFhlLRQ$(oSyIc$t;|{F>ph;XVgHfpC$+5H%8ajnp%+Y<&DY2OORU_k
z(V7&oZmv$Gj7LgIiK)l0pFiA9KYx&VyyL(B=HYJsi3@C{mioxSQew%acGgGsx>HI?
tDE$HfH|yd|adXt)D_oo(mI%Z&<wVo8BQ16miz!<%?O^j~Y-3_4_zmv{|2Y5v

diff --git a/tests/queries/0_stateless/00309_formats.sql b/tests/queries/0_stateless/00309_formats.sql
index 0366cdeea5c..b0939c00a10 100644
--- a/tests/queries/0_stateless/00309_formats.sql
+++ b/tests/queries/0_stateless/00309_formats.sql
@@ -9,8 +9,3 @@ SELECT number * 246 + 10 AS n, toDate('2000-01-01') + n AS d, range(n) AS arr, a
 SELECT number * 246 + 10 AS n, toDate('2000-01-01') + n AS d, range(n) AS arr, arrayStringConcat(arrayMap(x -> reinterpretAsString(x), arr)) AS s, (n, d) AS tuple FROM system.numbers LIMIT 2 FORMAT JSON;
 SELECT number * 246 + 10 AS n, toDate('2000-01-01') + n AS d, range(n) AS arr, arrayStringConcat(arrayMap(x -> reinterpretAsString(x), arr)) AS s, (n, d) AS tuple FROM system.numbers LIMIT 2 FORMAT JSONCompact;
 SELECT number * 246 + 10 AS n, toDate('2000-01-01') + n AS d, range(n) AS arr, arrayStringConcat(arrayMap(x -> reinterpretAsString(x), arr)) AS s, (n, d) AS tuple FROM system.numbers LIMIT 2 FORMAT XML;
-
-SET enable_named_columns_in_function_tuple = 1;
-
-SELECT 36 AS n, toDate('2000-01-01') + n AS d, (n, d) AS tuple FROM system.numbers LIMIT 1 FORMAT RowBinaryWithNamesAndTypes;
-SELECT number * 246 + 10 AS n, toDate('2000-01-01') + n AS d, (n, d) AS tuple FROM system.numbers LIMIT 1 FORMAT TabSeparatedWithNamesAndTypes;

From d2f89bc7f862073720f10a8fcec1cfb956c0eaae Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 29 Jul 2024 16:57:06 +0200
Subject: [PATCH 0992/1488] Fix bad test
 `01036_no_superfluous_dict_reload_on_create_database`

---
 tests/queries/0_stateless/01018_ddl_dictionaries_select.sql    | 2 +-
 .../01036_no_superfluous_dict_reload_on_create_database.sql    | 3 +++
 tests/queries/0_stateless/01257_dictionary_mismatch_types.sql  | 2 +-
 3 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/tests/queries/0_stateless/01018_ddl_dictionaries_select.sql b/tests/queries/0_stateless/01018_ddl_dictionaries_select.sql
index 4b9b15bee8f..95a5f2a0708 100644
--- a/tests/queries/0_stateless/01018_ddl_dictionaries_select.sql
+++ b/tests/queries/0_stateless/01018_ddl_dictionaries_select.sql
@@ -139,6 +139,6 @@ SELECT {CLICKHOUSE_DATABASE:String} || '.dict3' as n, dictGet(n, 'some_column',
 
 DROP TABLE {CLICKHOUSE_DATABASE:Identifier}.table_for_dict;
 
-SYSTEM RELOAD DICTIONARIES; -- {serverError UNKNOWN_TABLE}
+SYSTEM RELOAD DICTIONARY dict1; -- {serverError UNKNOWN_TABLE}
 
 SELECT dictGetString({CLICKHOUSE_DATABASE:String} || '.dict3', 'some_column', toUInt64(12));
diff --git a/tests/queries/0_stateless/01036_no_superfluous_dict_reload_on_create_database.sql b/tests/queries/0_stateless/01036_no_superfluous_dict_reload_on_create_database.sql
index d0841124706..1334780a5af 100644
--- a/tests/queries/0_stateless/01036_no_superfluous_dict_reload_on_create_database.sql
+++ b/tests/queries/0_stateless/01036_no_superfluous_dict_reload_on_create_database.sql
@@ -1,3 +1,6 @@
+-- Tags: no-parallel
+-- Does not allow if other tests do SYSTEM RELOAD DICTIONARIES at the same time.
+
 CREATE TABLE dict_data (key UInt64, val UInt64) Engine=Memory();
 CREATE DICTIONARY dict
 (
diff --git a/tests/queries/0_stateless/01257_dictionary_mismatch_types.sql b/tests/queries/0_stateless/01257_dictionary_mismatch_types.sql
index a4bb7bf2525..38349d8291b 100644
--- a/tests/queries/0_stateless/01257_dictionary_mismatch_types.sql
+++ b/tests/queries/0_stateless/01257_dictionary_mismatch_types.sql
@@ -98,7 +98,7 @@ ORDER BY (col1, col2, col3, col4, col5);
 SET input_format_null_as_default = 1;
 INSERT INTO test_dict_db.table1 VALUES ('id1',1,'20200127-1',701,'20200127-1-01',0,300,NULL,'N1','Hi','N40',NULL,1,'2020-02-03 10:37:59',NULL,'2020-02-04 11:35:14','2020-02-08 05:32:04',NULL,NULL,'12345'),('id1',1,'20200127-1',701,'20200127-1-01',0,300,NULL,'N1','Hi','N40',NULL,1,'2020-02-03 10:37:59',NULL,'2020-02-04 11:35:14','2020-02-08 05:32:04',NULL,NULL,'12345'),('id1',1,'20200127-1',702,'20200127-1-02',0,300,NULL,'N1','Hi','N40',NULL,1,'2020-02-03 10:37:59',NULL,'2020-02-04 11:35:14','2020-02-08 05:32:04',NULL,NULL,'12345'),('id1',1,'20200127-1',703,'20200127-1-03',0,300,NULL,'N1','Hi','N40',NULL,1,'2020-02-03 10:37:59',NULL,'2020-02-04 11:35:14','2020-02-08 05:32:04',NULL,NULL,'12345'),('id1',1,'20200127-1',704,'20200127-1-04',0,300,NULL,'N1','Hi','N40',NULL,1,'2020-02-03 10:37:59',NULL,'2020-02-04 11:35:14','2020-02-08 05:32:04',NULL,NULL,'12345'),('id1',1,'20200127-1',705,'20200127-1-05',0,300,NULL,'N1','Hi','N40',NULL,1,'2020-02-03 10:37:59',NULL,'2020-02-04 11:35:14','2020-02-08 05:32:04',NULL,NULL,'12345'),('id1',1,'20200202-1',711,'20200202-1-01',0,200,NULL,'C2','Hello','C40',NULL,1,'2020-02-03 11:07:57',NULL,NULL,NULL,'2020-02-03 11:09:23',NULL,NULL),('id1',1,'20200202-2',712,'20200202-2-01',0,0,NULL,'C3','bye','R40',NULL,1,'2020-02-03 14:13:10',NULL,'2020-02-03 16:11:31','2020-02-07 05:32:05','2020-02-07 11:18:15','2020-02-07 11:18:16','123455'),('id1',1,'20200202-2',713,'20200202-2-02',0,0,NULL,'C3','bye','R40',NULL,1,'2020-02-03 14:13:10',NULL,'2020-02-03 16:11:31','2020-02-07 05:32:05','2020-02-07 11:18:15','2020-02-07 11:18:16','123455'),('id1',2,'20200128-1',701,'20200128-1-01',0,0,NULL,'N1','Hi','N40',NULL,2,'2020-02-03 17:07:27',NULL,'2020-02-05 13:33:55','2020-02-13 05:32:04',NULL,NULL,'A123755'),('id1',2,'20200131-1',701,'20200131-1-01',0,0,NULL,'N1','Hi','N40',NULL,1,'2020-02-03 13:07:17',NULL,'2020-02-04 13:47:55','2020-02-12 05:32:04',NULL,NULL,'A123485'),('id1',2,'20200201-1',701,'20200201-1-01',0,0,NULL,'N1','Hi','N40',NULL,1,'2020-02-03 21:07:37',NULL,'2020-02-05 13:40:51','2020-02-13 05:32:04',NULL,NULL,'A123455'),('id1',2,'20200202-1',711,'20200202-1-01',0,0,NULL,'N1','Hi','N40',NULL,1,'2020-02-03 02:06:54',NULL,'2020-02-04 13:36:45','2020-02-12 05:32:04',NULL,NULL,'A123459'),('id1',2,'20200202-1',712,'20200202-1-02',0,0,NULL,'N1','Hi','N40',NULL,1,'2020-02-03 02:06:54',NULL,'2020-02-04 13:36:45','2020-02-12 05:32:04',NULL,NULL,'A123429'),('id2',1,'20200131-1',401,'20200131-1-01',0,210,'2020-02-16 05:22:04','N1','Hi','N40',NULL,1,'2020-02-03 10:11:00',NULL,'2020-02-05 17:30:05','2020-02-09 05:32:05',NULL,NULL,'454545'),('id2',1,'20200131-1',402,'20200131-1-02',0,210,'2020-02-16 05:22:04','N1','Hi','N40',NULL,1,'2020-02-03 10:11:00',NULL,'2020-02-05 17:30:05','2020-02-09 05:32:05',NULL,NULL,'454545'),('id2',1,'20200131-1',403,'20200131-1-03',0,270,'2020-02-16 05:22:04','N1','Hi','N40',NULL,1,'2020-02-03 10:11:00',NULL,'2020-02-05 17:30:05','2020-02-09 05:32:05',NULL,NULL,'454545'),('id2',1,'20200131-1',404,'20200131-1-04',0,270,'2020-02-16 05:22:04','N1','Hi','N40',NULL,1,'2020-02-03 10:11:00',NULL,'2020-02-05 17:30:05','2020-02-09 05:32:05',NULL,NULL,'454545'),('id2',1,'20200131-1',405,'20200131-1-05',0,380,NULL,'N1','Hi','N40',NULL,1,'2020-02-03 10:11:00',NULL,'2020-02-11 16:52:58','2020-02-15 05:32:04',NULL,NULL,'6892144935823'),('id2',1,'20200131-1',406,'20200131-1-06',0,380,NULL,'N1','Hi','N40',NULL,1,'2020-02-03 10:11:00',NULL,'2020-02-11 16:52:58','2020-02-15 05:32:04',NULL,NULL,'6892144935823'),('id2',1,'20200131-1',407,'20200131-1-07',0,280,NULL,'C2','Hello','C40',NULL,1,'2020-02-03 10:11:00',NULL,NULL,NULL,'2020-02-04 11:01:21',NULL,NULL),('id2',1,'20200131-1',408,'20200131-1-08',0,0,NULL,'N1','Hi','N40',NULL,1,'2020-02-03 10:11:00',NULL,'2020-02-05 17:30:05','2020-02-09 05:32:04',NULL,NULL,'454545'),('id2',1,'20200201-1',401,'20200201-1-01',0,190,'2020-02-16 05:22:05','N1','Hi','N40',NULL,1,'2020-02-03 12:06:17',NULL,'2020-02-05 17:30:30','2020-02-09 05:32:03',NULL,NULL,'90071'),('id2',1,'20200201-1',402,'20200201-1-01',0,160,'2020-02-14 05:22:13','N1','Hi','N40',NULL,1,'2020-02-03 06:21:05',NULL,'2020-02-03 17:42:35','2020-02-07 05:32:04',NULL,NULL,'96575'),('id2',1,'20200201-1',403,'20200201-1-02',0,230,'2020-02-14 05:22:13','N1','Hi','N40',NULL,1,'2020-02-03 06:21:05',NULL,'2020-02-03 17:42:35','2020-02-07 05:32:04',NULL,NULL,'96575'),('id2',1,'20200202-1',404,'20200202-1-01',0,130,'2020-02-14 05:22:14','N1','Hi','N40',NULL,1,'2020-02-03 14:00:39',NULL,'2020-02-03 17:42:45','2020-02-07 05:32:04',NULL,NULL,'96850'),('id3',1,'20200130-1',391,'20200130-1-01',0,300,NULL,'N1','Hi','N40',NULL,1,'2020-02-03 10:26:46',NULL,'2020-02-05 15:33:01','2020-02-08 05:32:05',NULL,NULL,'27243'),('id3',1,'20200130-1',392,'20200130-1-02',0,300,NULL,'N1','Hi','N40',NULL,1,'2020-02-03 10:26:46',NULL,'2020-02-10 16:16:11','2020-02-13 05:32:06',NULL,NULL,'92512'),('id3',1,'20200131-1',393,'20200131-1-01',0,0,NULL,'C2','Hello','C40',NULL,1,'2020-02-03 10:24:38',NULL,NULL,NULL,'2020-02-05 14:04:40',NULL,NULL),('id3',1,'20200131-2',391,'20200131-1-01',0,0,NULL,'N1','Hi','N40',NULL,1,'2020-02-03 10:22:08',NULL,'2020-02-06 14:27:06','2020-02-09 05:32:04',NULL,NULL,'46433'),('id3',1,'20200131-2',392,'20200131-1-02',0,0,NULL,'N1','Hi','N40',NULL,1,'2020-02-03 10:22:08',NULL,'2020-02-06 14:27:06','2020-02-09 05:32:02',NULL,NULL,'46433');
 
-SYSTEM RELOAD DICTIONARIES;
+SYSTEM RELOAD DICTIONARY test_dict_db.table1_dict;
 
 SELECT
     dictGet('test_dict_db.table1_dict', 'col6', (col1, col2, col3, col4, col5)),

From c2a40940dbc6a2bac3623bef2ac56bc842d837fd Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@gmail.com>
Date: Mon, 29 Jul 2024 17:02:33 +0200
Subject: [PATCH 0993/1488] Revert "Slightly better calculation of primary
 index"

---
 .../MergeTree/IMergeTreeDataPartWriter.cpp    | 19 +-----
 .../MergeTreeDataPartWriterOnDisk.cpp         | 65 +++++++++----------
 .../MergeTree/MergeTreeDataPartWriterOnDisk.h |  9 ++-
 .../02993_lazy_index_loading.reference        |  2 +-
 ..._system_unload_primary_key_table.reference |  8 +--
 .../03128_system_unload_primary_key.reference |  4 +-
 6 files changed, 45 insertions(+), 62 deletions(-)

diff --git a/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp b/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp
index c87f66b64f3..6152da78395 100644
--- a/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp
+++ b/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp
@@ -1,5 +1,4 @@
 #include <Storages/MergeTree/IMergeTreeDataPartWriter.h>
-#include <Common/MemoryTrackerBlockerInThread.h>
 
 namespace DB
 {
@@ -72,21 +71,9 @@ IMergeTreeDataPartWriter::IMergeTreeDataPartWriter(
 
 Columns IMergeTreeDataPartWriter::releaseIndexColumns()
 {
-    /// The memory for index was allocated without thread memory tracker.
-    /// We need to deallocate it in shrinkToFit without memory tracker as well.
-    MemoryTrackerBlockerInThread temporarily_disable_memory_tracker;
-
-    Columns result;
-    result.reserve(index_columns.size());
-
-    for (auto & column : index_columns)
-    {
-        column->shrinkToFit();
-        result.push_back(std::move(column));
-    }
-
-    index_columns.clear();
-    return result;
+    return Columns(
+        std::make_move_iterator(index_columns.begin()),
+        std::make_move_iterator(index_columns.end()));
 }
 
 SerializationPtr IMergeTreeDataPartWriter::getSerialization(const String & column_name) const
diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp
index 6dc7e649b06..46dd766139a 100644
--- a/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp
+++ b/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp
@@ -255,12 +255,6 @@ void MergeTreeDataPartWriterOnDisk::initPrimaryIndex()
             index_compressor_stream = std::make_unique<CompressedWriteBuffer>(*index_file_hashing_stream, primary_key_compression_codec, settings.primary_key_compress_block_size);
             index_source_hashing_stream = std::make_unique<HashingWriteBuffer>(*index_compressor_stream);
         }
-
-        const auto & primary_key_types = metadata_snapshot->getPrimaryKey().data_types;
-        index_serializations.reserve(primary_key_types.size());
-
-        for (const auto & type : primary_key_types)
-            index_serializations.push_back(type->getDefaultSerialization());
     }
 }
 
@@ -306,33 +300,22 @@ void MergeTreeDataPartWriterOnDisk::initSkipIndices()
             store = std::make_shared<GinIndexStore>(stream_name, data_part_storage, data_part_storage, storage_settings->max_digestion_size_per_segment);
             gin_index_stores[stream_name] = store;
         }
-
         skip_indices_aggregators.push_back(skip_index->createIndexAggregatorForPart(store, settings));
         skip_index_accumulated_marks.push_back(0);
     }
 }
 
-void MergeTreeDataPartWriterOnDisk::calculateAndSerializePrimaryIndexRow(const Block & index_block, size_t row)
-{
-    chassert(index_block.columns() == index_serializations.size());
-    auto & index_stream = compress_primary_key ? *index_source_hashing_stream : *index_file_hashing_stream;
-
-    for (size_t i = 0; i < index_block.columns(); ++i)
-    {
-        const auto & column = index_block.getByPosition(i).column;
-
-        index_columns[i]->insertFrom(*column, row);
-        index_serializations[i]->serializeBinary(*column, row, index_stream, {});
-    }
-}
-
 void MergeTreeDataPartWriterOnDisk::calculateAndSerializePrimaryIndex(const Block & primary_index_block, const Granules & granules_to_write)
 {
-    if (!metadata_snapshot->hasPrimaryKey())
-        return;
-
+    size_t primary_columns_num = primary_index_block.columns();
     if (index_columns.empty())
-        index_columns = primary_index_block.cloneEmptyColumns();
+    {
+        index_types = primary_index_block.getDataTypes();
+        index_columns.resize(primary_columns_num);
+        last_block_index_columns.resize(primary_columns_num);
+        for (size_t i = 0; i < primary_columns_num; ++i)
+            index_columns[i] = primary_index_block.getByPosition(i).column->cloneEmpty();
+    }
 
     {
         /** While filling index (index_columns), disable memory tracker.
@@ -346,14 +329,22 @@ void MergeTreeDataPartWriterOnDisk::calculateAndSerializePrimaryIndex(const Bloc
         /// Write index. The index contains Primary Key value for each `index_granularity` row.
         for (const auto & granule : granules_to_write)
         {
-            if (granule.mark_on_start)
-                calculateAndSerializePrimaryIndexRow(primary_index_block, granule.start_row);
+            if (metadata_snapshot->hasPrimaryKey() && granule.mark_on_start)
+            {
+                for (size_t j = 0; j < primary_columns_num; ++j)
+                {
+                    const auto & primary_column = primary_index_block.getByPosition(j);
+                    index_columns[j]->insertFrom(*primary_column.column, granule.start_row);
+                    primary_column.type->getDefaultSerialization()->serializeBinary(
+                        *primary_column.column, granule.start_row, compress_primary_key ? *index_source_hashing_stream : *index_file_hashing_stream, {});
+                }
+            }
         }
     }
 
-    /// Store block with last index row to write final mark at the end of column
-    if (with_final_mark)
-        last_index_block = primary_index_block;
+    /// store last index row to write final mark at the end of column
+    for (size_t j = 0; j < primary_columns_num; ++j)
+        last_block_index_columns[j] = primary_index_block.getByPosition(j).column;
 }
 
 void MergeTreeDataPartWriterOnDisk::calculateAndSerializeStatistics(const Block & block)
@@ -430,11 +421,17 @@ void MergeTreeDataPartWriterOnDisk::fillPrimaryIndexChecksums(MergeTreeData::Dat
 
     if (index_file_hashing_stream)
     {
-        if (write_final_mark && last_index_block)
+        if (write_final_mark)
         {
-            MemoryTrackerBlockerInThread temporarily_disable_memory_tracker;
-            calculateAndSerializePrimaryIndexRow(last_index_block, last_index_block.rows() - 1);
-            last_index_block.clear();
+            for (size_t j = 0; j < index_columns.size(); ++j)
+            {
+                const auto & column = *last_block_index_columns[j];
+                size_t last_row_number = column.size() - 1;
+                index_columns[j]->insertFrom(column, last_row_number);
+                index_types[j]->getDefaultSerialization()->serializeBinary(
+                    column, last_row_number, compress_primary_key ? *index_source_hashing_stream : *index_file_hashing_stream, {});
+            }
+            last_block_index_columns.clear();
         }
 
         if (compress_primary_key)
diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.h b/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.h
index 8d84442981e..bdf0fdb7f32 100644
--- a/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.h
+++ b/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.h
@@ -173,10 +173,10 @@ protected:
     std::unique_ptr<HashingWriteBuffer> index_source_hashing_stream;
     bool compress_primary_key;
 
-    /// Last block with index columns.
-    /// It's written to index file in the `writeSuffixAndFinalizePart` method.
-    Block last_index_block;
-    Serializations index_serializations;
+    DataTypes index_types;
+    /// Index columns from the last block
+    /// It's written to index file in the `writeSuffixAndFinalizePart` method
+    Columns last_block_index_columns;
 
     bool data_written = false;
 
@@ -193,7 +193,6 @@ private:
     void initStatistics();
 
     virtual void fillIndexGranularity(size_t index_granularity_for_block, size_t rows_in_block) = 0;
-    void calculateAndSerializePrimaryIndexRow(const Block & index_block, size_t row);
 
     struct ExecutionStatistics
     {
diff --git a/tests/queries/0_stateless/02993_lazy_index_loading.reference b/tests/queries/0_stateless/02993_lazy_index_loading.reference
index 08f07a92815..5bc329ae4eb 100644
--- a/tests/queries/0_stateless/02993_lazy_index_loading.reference
+++ b/tests/queries/0_stateless/02993_lazy_index_loading.reference
@@ -1,4 +1,4 @@
-100000000	100000000
+100000000	140000000
 0	0
 1
 100000000	100000000
diff --git a/tests/queries/0_stateless/03127_system_unload_primary_key_table.reference b/tests/queries/0_stateless/03127_system_unload_primary_key_table.reference
index 2d33f7f6683..3ac6127fb21 100644
--- a/tests/queries/0_stateless/03127_system_unload_primary_key_table.reference
+++ b/tests/queries/0_stateless/03127_system_unload_primary_key_table.reference
@@ -1,8 +1,8 @@
-100000000	100000000
-100000000	100000000
-100000000	100000000
+100000000	140000000
+100000000	140000000
+100000000	140000000
 0	0
-100000000	100000000
+100000000	140000000
 0	0
 0	0
 1
diff --git a/tests/queries/0_stateless/03128_system_unload_primary_key.reference b/tests/queries/0_stateless/03128_system_unload_primary_key.reference
index 2646dc7247f..c7b40ae5b06 100644
--- a/tests/queries/0_stateless/03128_system_unload_primary_key.reference
+++ b/tests/queries/0_stateless/03128_system_unload_primary_key.reference
@@ -1,4 +1,4 @@
-100000000	100000000
-100000000	100000000
+100000000	140000000
+100000000	140000000
 0	0
 0	0

From 59c517dd91e996ffadd6d4fe51f8f0cf247467c0 Mon Sep 17 00:00:00 2001
From: Nikita Taranov <nikita.taranov@clickhouse.com>
Date: Mon, 29 Jul 2024 16:08:38 +0100
Subject: [PATCH 0994/1488] fix format_fuzzer

---
 src/Formats/fuzzers/format_fuzzer.cpp | 47 ++++++++++++---------------
 1 file changed, 21 insertions(+), 26 deletions(-)

diff --git a/src/Formats/fuzzers/format_fuzzer.cpp b/src/Formats/fuzzers/format_fuzzer.cpp
index 46661e4828c..408e7218221 100644
--- a/src/Formats/fuzzers/format_fuzzer.cpp
+++ b/src/Formats/fuzzers/format_fuzzer.cpp
@@ -20,37 +20,32 @@
 
 #include <AggregateFunctions/registerAggregateFunctions.h>
 
+using namespace DB;
+
+
+ContextMutablePtr context;
+
+extern "C" int LLVMFuzzerInitialize(int *, char ***)
+{
+    if (context)
+        return true;
+
+    SharedContextHolder shared_context = Context::createShared();
+    context = Context::createGlobal(shared_context.get());
+    context->makeGlobalContext();
+
+    MainThreadStatus::getInstance();
+
+    registerAggregateFunctions();
+    registerFormats();
+
+    return 0;
+}
 
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t * data, size_t size)
 {
     try
     {
-        using namespace DB;
-
-        static SharedContextHolder shared_context;
-        static ContextMutablePtr context;
-
-        auto initialize = [&]() mutable
-        {
-            if (context)
-                return true;
-
-            shared_context = Context::createShared();
-            context = Context::createGlobal(shared_context.get());
-            context->makeGlobalContext();
-            context->setApplicationType(Context::ApplicationType::LOCAL);
-
-            MainThreadStatus::getInstance();
-
-            registerAggregateFunctions();
-            registerFormats();
-
-            return true;
-        };
-
-        static bool initialized = initialize();
-        (void) initialized;
-
         total_memory_tracker.resetCounters();
         total_memory_tracker.setHardLimit(1_GiB);
         CurrentThread::get().memory_tracker.resetCounters();

From 064fe8c3f551a634f50543f1cfadd164f08173d4 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 29 Jul 2024 17:13:38 +0200
Subject: [PATCH 0995/1488] Fix tests

---
 tests/queries/0_stateless/01018_ddl_dictionaries_select.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/01018_ddl_dictionaries_select.sql b/tests/queries/0_stateless/01018_ddl_dictionaries_select.sql
index 95a5f2a0708..4c4bcc440ca 100644
--- a/tests/queries/0_stateless/01018_ddl_dictionaries_select.sql
+++ b/tests/queries/0_stateless/01018_ddl_dictionaries_select.sql
@@ -139,6 +139,6 @@ SELECT {CLICKHOUSE_DATABASE:String} || '.dict3' as n, dictGet(n, 'some_column',
 
 DROP TABLE {CLICKHOUSE_DATABASE:Identifier}.table_for_dict;
 
-SYSTEM RELOAD DICTIONARY dict1; -- {serverError UNKNOWN_TABLE}
+SYSTEM RELOAD DICTIONARY {CLICKHOUSE_DATABASE:Identifier}.dict3; -- {serverError UNKNOWN_TABLE}
 
 SELECT dictGetString({CLICKHOUSE_DATABASE:String} || '.dict3', 'some_column', toUInt64(12));

From 819196cc7469ed02677b7edbc63ebd5dea666456 Mon Sep 17 00:00:00 2001
From: Jordi <jordi@tinybird.co>
Date: Mon, 29 Jul 2024 17:48:08 +0200
Subject: [PATCH 0996/1488] Add total marks and parts to ProfileEvents

---
 src/Common/ProfileEvents.cpp                   | 2 ++
 src/Processors/QueryPlan/ReadFromMergeTree.cpp | 4 ++++
 2 files changed, 6 insertions(+)

diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp
index 2b090136e2e..f8bd84df131 100644
--- a/src/Common/ProfileEvents.cpp
+++ b/src/Common/ProfileEvents.cpp
@@ -193,8 +193,10 @@
     M(ReplicaPartialShutdown, "How many times Replicated table has to deinitialize its state due to session expiration in ZooKeeper. The state is reinitialized every time when ZooKeeper is available again.") \
     \
     M(SelectedParts, "Number of data parts selected to read from a MergeTree table.") \
+    M(SelectedPartsTotal, "Number of total data parts before selecting which ones to read from a MergeTree table.") \
     M(SelectedRanges, "Number of (non-adjacent) ranges in all data parts selected to read from a MergeTree table.") \
     M(SelectedMarks, "Number of marks (index granules) selected to read from a MergeTree table.") \
+    M(SelectedMarksTotal, "Number of total marks (index granules) before selecting which ones to read from a MergeTree table.") \
     M(SelectedRows, "Number of rows SELECTed from all tables.") \
     M(SelectedBytes, "Number of bytes (uncompressed; for columns as they stored in memory) SELECTed from all tables.") \
     M(RowsReadByMainReader, "Number of rows read from MergeTree tables by the main reader (after PREWHERE step).") \
diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp
index a7ce92fe8e0..6e3a71660ad 100644
--- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp
+++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp
@@ -120,8 +120,10 @@ bool restorePrewhereInputs(PrewhereInfo & info, const NameSet & inputs)
 namespace ProfileEvents
 {
     extern const Event SelectedParts;
+    extern const Event SelectedPartsTotal;
     extern const Event SelectedRanges;
     extern const Event SelectedMarks;
+    extern const Event SelectedMarksTotal;
     extern const Event SelectQueriesWithPrimaryKeyUsage;
 }
 
@@ -1971,8 +1973,10 @@ void ReadFromMergeTree::initializePipeline(QueryPipelineBuilder & pipeline, cons
     }
 
     ProfileEvents::increment(ProfileEvents::SelectedParts, result.selected_parts);
+    ProfileEvents::increment(ProfileEvents::SelectedPartsTotal, result.total_parts);
     ProfileEvents::increment(ProfileEvents::SelectedRanges, result.selected_ranges);
     ProfileEvents::increment(ProfileEvents::SelectedMarks, result.selected_marks);
+    ProfileEvents::increment(ProfileEvents::SelectedMarksTotal, result.total_marks_pk);
 
     auto query_id_holder = MergeTreeDataSelectExecutor::checkLimits(data, result, context);
 

From 28ae67187542d7ee3edf9dd30700d5589a2f8389 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?=
 <benjamin.antal@clickhouse.com>
Date: Mon, 29 Jul 2024 15:52:25 +0000
Subject: [PATCH 0997/1488] Print debug info

---
 .../integration/test_storage_s3_queue/test.py | 29 ++++++++++++-------
 1 file changed, 18 insertions(+), 11 deletions(-)

diff --git a/tests/integration/test_storage_s3_queue/test.py b/tests/integration/test_storage_s3_queue/test.py
index bf3c28c5429..93234e635b4 100644
--- a/tests/integration/test_storage_s3_queue/test.py
+++ b/tests/integration/test_storage_s3_queue/test.py
@@ -1352,16 +1352,7 @@ def test_shards_distributed(started_cluster, mode, processing_threads):
     def get_count(node, table_name):
         return int(run_query(node, f"SELECT count() FROM {table_name}"))
 
-    for _ in range(30):
-        if (
-            get_count(node, dst_table_name) + get_count(node_2, dst_table_name)
-        ) == total_rows:
-            break
-        time.sleep(1)
-
-    if (
-        get_count(node, dst_table_name) + get_count(node_2, dst_table_name)
-    ) != total_rows:
+    def print_debug_info():
         processed_files = (
             node.query(
                 f"""
@@ -1388,7 +1379,7 @@ select splitByChar('/', file_name)[-1] as file from system.s3queue where zookeep
         )
 
         count = get_count(node, dst_table_name) + get_count(node_2, dst_table_name)
-        logging.debug(f"Processed rows: {count}/{files_to_generate}")
+        logging.debug(f"Processed rows: {count}/{total_rows}")
 
         info = node.query(
             f"""
@@ -1425,6 +1416,18 @@ select splitByChar('/', file_name)[-1] as file from system.s3queue where zookeep
 
         logging.debug(f"Intersecting files: {intersection(files1, files2)}")
 
+    for _ in range(30):
+        if (
+            get_count(node, dst_table_name) + get_count(node_2, dst_table_name)
+        ) == total_rows:
+            break
+        time.sleep(1)
+
+    if (
+        get_count(node, dst_table_name) + get_count(node_2, dst_table_name)
+    ) != total_rows:
+        print_debug_info()
+
         assert False
 
     get_query = f"SELECT column1, column2, column3 FROM {dst_table_name}"
@@ -1433,6 +1436,10 @@ select splitByChar('/', file_name)[-1] as file from system.s3queue where zookeep
         list(map(int, l.split())) for l in run_query(node_2, get_query).splitlines()
     ]
 
+    if len(res1) + len(res2) != total_rows or len(res1) <= 0 or len(res2) <= 0 or True:
+        logging.debug(f"res1 size: {len(res1)}, res2 size: {len(res2)}, total_rows: {total_rows}")
+        print_debug_info()
+
     assert len(res1) + len(res2) == total_rows
 
     # Checking that all engines have made progress

From f7f2a9872d3a723559779d0be0f2ef915e8cfd31 Mon Sep 17 00:00:00 2001
From: robot-clickhouse <robot-clickhouse@users.noreply.github.com>
Date: Mon, 29 Jul 2024 16:02:06 +0000
Subject: [PATCH 0998/1488] Automatic style fix

---
 tests/integration/test_storage_s3_queue/test.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/integration/test_storage_s3_queue/test.py b/tests/integration/test_storage_s3_queue/test.py
index 93234e635b4..0846872e716 100644
--- a/tests/integration/test_storage_s3_queue/test.py
+++ b/tests/integration/test_storage_s3_queue/test.py
@@ -1437,7 +1437,9 @@ select splitByChar('/', file_name)[-1] as file from system.s3queue where zookeep
     ]
 
     if len(res1) + len(res2) != total_rows or len(res1) <= 0 or len(res2) <= 0 or True:
-        logging.debug(f"res1 size: {len(res1)}, res2 size: {len(res2)}, total_rows: {total_rows}")
+        logging.debug(
+            f"res1 size: {len(res1)}, res2 size: {len(res2)}, total_rows: {total_rows}"
+        )
         print_debug_info()
 
     assert len(res1) + len(res2) == total_rows

From e1677bc1b60be3de954c6305b665732e1d4fe062 Mon Sep 17 00:00:00 2001
From: Nikita Taranov <nikita.taranov@clickhouse.com>
Date: Mon, 29 Jul 2024 17:13:05 +0100
Subject: [PATCH 0999/1488] fix rest

---
 ..._function_state_deserialization_fuzzer.cpp | 45 ++++++++--------
 .../data_type_deserialization_fuzzer.cpp      | 43 +++++++--------
 .../fuzzers/execute_query_fuzzer.cpp          | 54 +++++++++----------
 3 files changed, 65 insertions(+), 77 deletions(-)

diff --git a/src/AggregateFunctions/fuzzers/aggregate_function_state_deserialization_fuzzer.cpp b/src/AggregateFunctions/fuzzers/aggregate_function_state_deserialization_fuzzer.cpp
index 31fc93e4288..87979259c9e 100644
--- a/src/AggregateFunctions/fuzzers/aggregate_function_state_deserialization_fuzzer.cpp
+++ b/src/AggregateFunctions/fuzzers/aggregate_function_state_deserialization_fuzzer.cpp
@@ -17,34 +17,31 @@
 
 #include <base/scope_guard.h>
 
+using namespace DB;
+
+
+ContextMutablePtr context;
+
+extern "C" int LLVMFuzzerInitialize(int *, char ***)
+{
+    if (context)
+        return true;
+
+    SharedContextHolder shared_context = Context::createShared();
+    context = Context::createGlobal(shared_context.get());
+    context->makeGlobalContext();
+
+    MainThreadStatus::getInstance();
+
+    registerAggregateFunctions();
+
+    return 0;
+}
+
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t * data, size_t size)
 {
     try
     {
-        using namespace DB;
-
-        static SharedContextHolder shared_context;
-        static ContextMutablePtr context;
-
-        auto initialize = [&]() mutable
-        {
-            if (context)
-                return true;
-
-            shared_context = Context::createShared();
-            context = Context::createGlobal(shared_context.get());
-            context->makeGlobalContext();
-            context->setApplicationType(Context::ApplicationType::LOCAL);
-
-            MainThreadStatus::getInstance();
-
-            registerAggregateFunctions();
-            return true;
-        };
-
-        static bool initialized = initialize();
-        (void) initialized;
-
         total_memory_tracker.resetCounters();
         total_memory_tracker.setHardLimit(1_GiB);
         CurrentThread::get().memory_tracker.resetCounters();
diff --git a/src/DataTypes/fuzzers/data_type_deserialization_fuzzer.cpp b/src/DataTypes/fuzzers/data_type_deserialization_fuzzer.cpp
index 033a6ea8a4a..f9a733647e1 100644
--- a/src/DataTypes/fuzzers/data_type_deserialization_fuzzer.cpp
+++ b/src/DataTypes/fuzzers/data_type_deserialization_fuzzer.cpp
@@ -12,35 +12,30 @@
 
 #include <AggregateFunctions/registerAggregateFunctions.h>
 
+using namespace DB;
+
+
+ContextMutablePtr context;
+
+extern "C" int LLVMFuzzerInitialize(int *, char ***)
+{
+    if (context)
+        return true;
+
+    SharedContextHolder shared_context = Context::createShared();
+    context = Context::createGlobal(shared_context.get());
+    context->makeGlobalContext();
+
+    MainThreadStatus::getInstance();
+
+    registerAggregateFunctions();
+    return 0;
+}
 
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t * data, size_t size)
 {
     try
     {
-        using namespace DB;
-
-        static SharedContextHolder shared_context;
-        static ContextMutablePtr context;
-
-        auto initialize = [&]() mutable
-        {
-            if (context)
-                return true;
-
-            shared_context = Context::createShared();
-            context = Context::createGlobal(shared_context.get());
-            context->makeGlobalContext();
-            context->setApplicationType(Context::ApplicationType::LOCAL);
-
-            MainThreadStatus::getInstance();
-
-            registerAggregateFunctions();
-            return true;
-        };
-
-        static bool initialized = initialize();
-        (void) initialized;
-
         total_memory_tracker.resetCounters();
         total_memory_tracker.setHardLimit(1_GiB);
         CurrentThread::get().memory_tracker.resetCounters();
diff --git a/src/Interpreters/fuzzers/execute_query_fuzzer.cpp b/src/Interpreters/fuzzers/execute_query_fuzzer.cpp
index a02ce66e6b5..c29efae1e7d 100644
--- a/src/Interpreters/fuzzers/execute_query_fuzzer.cpp
+++ b/src/Interpreters/fuzzers/execute_query_fuzzer.cpp
@@ -14,41 +14,37 @@
 
 using namespace DB;
 
+
+ContextMutablePtr context;
+
+extern "C" int LLVMFuzzerInitialize(int *, char ***)
+{
+    if (context)
+        return true;
+
+    SharedContextHolder shared_context = Context::createShared();
+    context = Context::createGlobal(shared_context.get());
+    context->makeGlobalContext();
+
+    registerInterpreters();
+    registerFunctions();
+    registerAggregateFunctions();
+    registerTableFunctions();
+    registerDatabases();
+    registerStorages();
+    registerDictionaries();
+    registerDisks(/* global_skip_access_check= */ true);
+    registerFormats();
+
+    return 0;
+}
+
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t * data, size_t size)
 {
     try
     {
         std::string input = std::string(reinterpret_cast<const char*>(data), size);
 
-        static SharedContextHolder shared_context;
-        static ContextMutablePtr context;
-
-        auto initialize = [&]() mutable
-        {
-            if (context)
-                return true;
-
-            shared_context = Context::createShared();
-            context = Context::createGlobal(shared_context.get());
-            context->makeGlobalContext();
-            context->setApplicationType(Context::ApplicationType::LOCAL);
-
-            registerInterpreters();
-            registerFunctions();
-            registerAggregateFunctions();
-            registerTableFunctions();
-            registerDatabases();
-            registerStorages();
-            registerDictionaries();
-            registerDisks(/* global_skip_access_check= */ true);
-            registerFormats();
-
-            return true;
-        };
-
-        static bool initialized = initialize();
-        (void) initialized;
-
         auto io = DB::executeQuery(input, context, QueryFlags{ .internal = true }, QueryProcessingStage::Complete).second;
 
         PullingPipelineExecutor executor(io.pipeline);

From f3bc2a1c5a95a2fd1c18b04a29e942bcce9051d3 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Mon, 29 Jul 2024 16:21:55 +0000
Subject: [PATCH 1000/1488] Fix anotehr one case of non-deterministic PK

---
 src/Interpreters/ActionsDAG.cpp               | 96 ++++++++++++-------
 src/Interpreters/ActionsDAG.h                 |  8 +-
 .../01943_non_deterministic_order_key.sql     |  1 +
 3 files changed, 65 insertions(+), 40 deletions(-)

diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp
index 4aaecc491e0..7bf65ee2416 100644
--- a/src/Interpreters/ActionsDAG.cpp
+++ b/src/Interpreters/ActionsDAG.cpp
@@ -100,6 +100,13 @@ bool isConstantFromScalarSubquery(const ActionsDAG::Node * node)
 
 }
 
+bool ActionsDAG::Node::isDeterministic() const
+{
+    bool deterministic_if_func = type != ActionType::FUNCTION || function_base->isDeterministic();
+    bool deterministic_if_const = type != ActionType::COLUMN || is_deterministic_constant;
+    return deterministic_if_func && deterministic_if_const;
+}
+
 void ActionsDAG::Node::toTree(JSONBuilder::JSONMap & map) const
 {
     map.add("Node Type", magic_enum::enum_name(type));
@@ -318,7 +325,6 @@ const ActionsDAG::Node & ActionsDAG::addFunctionImpl(
     node.function_base = function_base;
     node.result_type = result_type;
     node.function = node.function_base->prepare(arguments);
-    node.is_deterministic = node.function_base->isDeterministic();
 
     /// If all arguments are constants, and function is suitable to be executed in 'prepare' stage - execute function.
     if (node.function_base->isSuitableForConstantFolding())
@@ -536,64 +542,80 @@ void ActionsDAG::removeUnusedActions(bool allow_remove_inputs, bool allow_consta
 
 void ActionsDAG::removeUnusedActions(const std::unordered_set<const Node *> & used_inputs, bool allow_constant_folding)
 {
-    std::unordered_set<const Node *> visited_nodes;
-    std::stack<Node *> stack;
-
-    for (const auto * node : outputs)
-    {
-        visited_nodes.insert(node);
-        stack.push(const_cast<Node *>(node));
-    }
+    NodeRawConstPtrs roots;
+    roots.reserve(outputs.size() + used_inputs.size());
+    roots = outputs;
 
     for (auto & node : nodes)
     {
         /// We cannot remove arrayJoin because it changes the number of rows.
-        bool is_array_join = node.type == ActionType::ARRAY_JOIN;
-
-        if (is_array_join && !visited_nodes.contains(&node))
-        {
-            visited_nodes.insert(&node);
-            stack.push(&node);
-        }
+        if (node.type == ActionType::ARRAY_JOIN)
+            roots.push_back(&node);
 
         if (node.type == ActionType::INPUT && used_inputs.contains(&node))
-            visited_nodes.insert(&node);
+            roots.push_back(&node);
     }
 
-    while (!stack.empty())
+    std::unordered_set<const Node *> required_nodes;
+    std::unordered_set<const Node *> non_deterministic_nodes;
+
+    struct Frame
     {
-        auto * node = stack.top();
-        stack.pop();
+        const ActionsDAG::Node * node;
+        size_t next_child_to_visit = 0;
+    };
 
-        /// Constant folding.
-        if (allow_constant_folding && !node->children.empty() && node->column && isColumnConst(*node->column))
+    std::stack<Frame> stack;
+
+    for (const auto * root : roots)
+    {
+        if (!required_nodes.contains(root))
         {
-            node->type = ActionsDAG::ActionType::COLUMN;
+            required_nodes.insert(root);
+            stack.push({.node = root});
+        }
 
-            for (const auto & child : node->children)
+        while (!stack.empty())
+        {
+            auto & frame = stack.top();
+            auto * node = const_cast<Node *>(frame.node);
+
+            while (frame.next_child_to_visit < node->children.size())
             {
-                if (!child->is_deterministic)
+                const auto * child = node->children[frame.next_child_to_visit];
+                ++frame.next_child_to_visit;
+
+                if (!required_nodes.contains(child))
                 {
-                    node->is_deterministic = false;
+                    required_nodes.insert(child);
+                    stack.push({.node = child});
                     break;
                 }
+
+                if (non_deterministic_nodes.contains(child))
+                    non_deterministic_nodes.insert(node);
             }
 
-            node->children.clear();
-        }
+            if (stack.top().node != node)
+                continue;
 
-        for (const auto * child : node->children)
-        {
-            if (!visited_nodes.contains(child))
+            if (!node->isDeterministic())
+                non_deterministic_nodes.insert(node);
+
+            stack.pop();
+
+            /// Constant folding.
+            if (allow_constant_folding && !node->children.empty()
+                && node->column && isColumnConst(*node->column) && !non_deterministic_nodes.contains(node))
             {
-                stack.push(const_cast<Node *>(child));
-                visited_nodes.insert(child);
+                node->type = ActionsDAG::ActionType::COLUMN;
+                node->children.clear();
             }
         }
     }
 
-    std::erase_if(nodes, [&](const Node & node) { return !visited_nodes.contains(&node); });
-    std::erase_if(inputs, [&](const Node * node) { return !visited_nodes.contains(node); });
+    std::erase_if(nodes, [&](const Node & node) { return !required_nodes.contains(&node); });
+    std::erase_if(inputs, [&](const Node * node) { return !required_nodes.contains(node); });
 }
 
 
@@ -1379,7 +1401,7 @@ bool ActionsDAG::trivial() const
 void ActionsDAG::assertDeterministic() const
 {
     for (const auto & node : nodes)
-        if (!node.is_deterministic)
+        if (!node.isDeterministic())
             throw Exception(ErrorCodes::BAD_ARGUMENTS,
                 "Expression must be deterministic but it contains non-deterministic part `{}`", node.result_name);
 }
@@ -1387,7 +1409,7 @@ void ActionsDAG::assertDeterministic() const
 bool ActionsDAG::hasNonDeterministic() const
 {
     for (const auto & node : nodes)
-        if (!node.is_deterministic)
+        if (!node.isDeterministic())
             return true;
     return false;
 }
diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h
index 43c1b41a240..ee2b3fbf4f2 100644
--- a/src/Interpreters/ActionsDAG.h
+++ b/src/Interpreters/ActionsDAG.h
@@ -80,13 +80,15 @@ public:
         ExecutableFunctionPtr function;
         /// If function is a compiled statement.
         bool is_function_compiled = false;
-        /// It is deterministic (See IFunction::isDeterministic).
-        /// This property is kept after constant folding of non-deterministic functions like 'now', 'today'.
-        bool is_deterministic = true;
 
+        /// It is a constant calculated from deterministic functions (See IFunction::isDeterministic).
+        /// This property is kept after constant folding of non-deterministic functions like 'now', 'today'.
+        bool is_deterministic_constant = true;
         /// For COLUMN node and propagated constants.
         ColumnPtr column;
 
+        /// If result of this not is deterministic. Checks only this node, not a subtree.
+        bool isDeterministic() const;
         void toTree(JSONBuilder::JSONMap & map) const;
     };
 
diff --git a/tests/queries/0_stateless/01943_non_deterministic_order_key.sql b/tests/queries/0_stateless/01943_non_deterministic_order_key.sql
index 781ea1b1cc7..94aa423c552 100644
--- a/tests/queries/0_stateless/01943_non_deterministic_order_key.sql
+++ b/tests/queries/0_stateless/01943_non_deterministic_order_key.sql
@@ -1,3 +1,4 @@
 CREATE TABLE a (number UInt64) ENGINE = MergeTree ORDER BY if(now() > toDateTime('2020-06-01 13:31:40'), toInt64(number), -number); -- { serverError BAD_ARGUMENTS }
 CREATE TABLE b (number UInt64) ENGINE = MergeTree ORDER BY now() > toDateTime(number); -- { serverError BAD_ARGUMENTS }
 CREATE TABLE c (number UInt64) ENGINE = MergeTree ORDER BY now(); -- { serverError BAD_ARGUMENTS }
+CREATE TABLE d (number UInt64) ENGINE = MergeTree ORDER BY now() + 1 + 1 + number; -- { serverError BAD_ARGUMENTS }

From eaede913b2ae0d5a4cb02ae82db20b6011270fe6 Mon Sep 17 00:00:00 2001
From: Pablo Marcos <pablo.marcos.oltra@clickhouse.com>
Date: Mon, 29 Jul 2024 16:26:10 +0000
Subject: [PATCH 1001/1488] Ensure there's no logical error using
 PoolWithFailoverBase::getMany

Prevents a segfault in case the connection is null, but also checks
all other conditions to ensure the logic is the expected one.
---
 src/Client/ConnectionPoolWithFailover.h       |  1 +
 src/Common/PoolWithFailoverBase.h             | 11 +++++++++-
 .../DistributedAsyncInsertBatch.cpp           | 22 ++++++++++---------
 .../DistributedAsyncInsertDirectoryQueue.cpp  |  9 ++++----
 src/Storages/Distributed/DistributedSink.cpp  |  8 ++++---
 5 files changed, 33 insertions(+), 18 deletions(-)

diff --git a/src/Client/ConnectionPoolWithFailover.h b/src/Client/ConnectionPoolWithFailover.h
index 7b9f480aa4e..a2dc188eb7d 100644
--- a/src/Client/ConnectionPoolWithFailover.h
+++ b/src/Client/ConnectionPoolWithFailover.h
@@ -42,6 +42,7 @@ public:
             size_t max_error_cap = DBMS_CONNECTION_POOL_WITH_FAILOVER_MAX_ERROR_COUNT);
 
     using Entry = IConnectionPool::Entry;
+    using PoolWithFailoverBase<IConnectionPool>::isTryResultInvalid;
 
     /** Allocates connection to work. */
     Entry get(const ConnectionTimeouts & timeouts) override;
diff --git a/src/Common/PoolWithFailoverBase.h b/src/Common/PoolWithFailoverBase.h
index 3d4de773a36..c44ab7df53a 100644
--- a/src/Common/PoolWithFailoverBase.h
+++ b/src/Common/PoolWithFailoverBase.h
@@ -116,6 +116,12 @@ public:
             const TryGetEntryFunc & try_get_entry,
             const GetPriorityFunc & get_priority);
 
+    // Returns if the TryResult provided is an invalid one that cannot be used. Used to prevent logical errors.
+    bool isTryResultInvalid(const TryResult & result, bool skip_read_only_replicas) const
+    {
+        return result.entry.isNull() || !result.is_usable || (skip_read_only_replicas && result.is_readonly);
+    }
+
     size_t getPoolSize() const { return nested_pools.size(); }
 
 protected:
@@ -300,7 +306,7 @@ PoolWithFailoverBase<TNestedPool>::getMany(
         throw DB::NetException(DB::ErrorCodes::ALL_CONNECTION_TRIES_FAILED,
                 "All connection tries failed. Log: \n\n{}\n", fail_messages);
 
-    std::erase_if(try_results, [&](const TryResult & r) { return r.entry.isNull() || !r.is_usable || (skip_read_only_replicas && r.is_readonly); });
+    std::erase_if(try_results, [&](const TryResult & r) { return isTryResultInvalid(r, skip_read_only_replicas); });
 
     /// Sort so that preferred items are near the beginning.
     std::stable_sort(
@@ -321,6 +327,9 @@ PoolWithFailoverBase<TNestedPool>::getMany(
     }
     else if (up_to_date_count >= min_entries)
     {
+        if (try_results.size() < up_to_date_count)
+            throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Could not find enough connections for up-to-date results. Got: {}, needed: {}", try_results.size(), up_to_date_count);
+
         /// There is enough up-to-date entries.
         try_results.resize(up_to_date_count);
     }
diff --git a/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp b/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp
index 5e7b4b979c7..2cf69b9f6b7 100644
--- a/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp
+++ b/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp
@@ -242,12 +242,13 @@ void DistributedAsyncInsertBatch::sendBatch(const SettingsChanges & settings_cha
                 insert_settings.applyChanges(settings_changes);
 
                 auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(insert_settings);
-                auto result = parent.pool->getManyCheckedForInsert(timeouts, insert_settings, PoolMode::GET_ONE, parent.storage.remote_storage.getQualifiedName());
-                connection = std::move(result.front().entry);
-                compression_expected = connection->getCompression() == Protocol::Compression::Enable;
+                auto results = parent.pool->getManyCheckedForInsert(timeouts, insert_settings, PoolMode::GET_ONE, parent.storage.remote_storage.getQualifiedName());
+                auto result = results.front();
+                if (parent.pool->isTryResultInvalid(result, insert_settings.distributed_insert_skip_read_only_replicas))
+                    throw Exception(ErrorCodes::LOGICAL_ERROR, "Got an invalid connection result");
 
-                if (connection.isNull())
-                    throw Exception(ErrorCodes::LOGICAL_ERROR, "Got empty connection");
+                connection = std::move(result.entry);
+                compression_expected = connection->getCompression() == Protocol::Compression::Enable;
 
                 LOG_DEBUG(parent.log, "Sending a batch of {} files to {} ({} rows, {} bytes).",
                     files.size(),
@@ -303,12 +304,13 @@ void DistributedAsyncInsertBatch::sendSeparateFiles(const SettingsChanges & sett
                 parent.storage.getContext()->getOpenTelemetrySpanLog());
 
             auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(insert_settings);
-            auto result = parent.pool->getManyCheckedForInsert(timeouts, insert_settings, PoolMode::GET_ONE, parent.storage.remote_storage.getQualifiedName());
-            auto connection = std::move(result.front().entry);
-            bool compression_expected = connection->getCompression() == Protocol::Compression::Enable;
+            auto results = parent.pool->getManyCheckedForInsert(timeouts, insert_settings, PoolMode::GET_ONE, parent.storage.remote_storage.getQualifiedName());
+            auto result = results.front();
+            if (parent.pool->isTryResultInvalid(result, insert_settings.distributed_insert_skip_read_only_replicas))
+                throw Exception(ErrorCodes::LOGICAL_ERROR, "Got an invalid connection result");
 
-            if (connection.isNull())
-                throw Exception(ErrorCodes::LOGICAL_ERROR, "Got empty connection");
+            auto connection = std::move(result.entry);
+            bool compression_expected = connection->getCompression() == Protocol::Compression::Enable;
 
             RemoteInserter remote(*connection, timeouts,
                 distributed_header.insert_query,
diff --git a/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp b/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp
index 2bb0e720c72..876eff0021f 100644
--- a/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp
+++ b/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp
@@ -412,11 +412,12 @@ void DistributedAsyncInsertDirectoryQueue::processFile(std::string & file_path,
         insert_settings.applyChanges(settings_changes);
 
         auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(insert_settings);
-        auto result = pool->getManyCheckedForInsert(timeouts, insert_settings, PoolMode::GET_ONE, storage.remote_storage.getQualifiedName());
-        auto connection = std::move(result.front().entry);
+        auto results = pool->getManyCheckedForInsert(timeouts, insert_settings, PoolMode::GET_ONE, storage.remote_storage.getQualifiedName());
+        auto result = results.front();
+        if (pool->isTryResultInvalid(result, insert_settings.distributed_insert_skip_read_only_replicas))
+            throw Exception(ErrorCodes::LOGICAL_ERROR, "Got an invalid connection result");
 
-        if (connection.isNull())
-            throw Exception(ErrorCodes::LOGICAL_ERROR, "Got empty connection");
+        auto connection = std::move(result.entry);
 
         LOG_DEBUG(log, "Sending `{}` to {} ({} rows, {} bytes)",
             file_path,
diff --git a/src/Storages/Distributed/DistributedSink.cpp b/src/Storages/Distributed/DistributedSink.cpp
index f8bbc081e55..e0baefd5838 100644
--- a/src/Storages/Distributed/DistributedSink.cpp
+++ b/src/Storages/Distributed/DistributedSink.cpp
@@ -377,9 +377,11 @@ DistributedSink::runWritingJob(JobReplica & job, const Block & current_block, si
                     /// NOTE: INSERT will also take into account max_replica_delay_for_distributed_queries
                     /// (anyway fallback_to_stale_replicas_for_distributed_queries=true by default)
                     auto results = shard_info.pool->getManyCheckedForInsert(timeouts, settings, PoolMode::GET_ONE, storage.remote_storage.getQualifiedName());
-                    job.connection_entry = std::move(results.front().entry);
-                    if (job.connection_entry.isNull())
-                        throw Exception(ErrorCodes::LOGICAL_ERROR, "Got empty connection");
+                    auto result = results.front();
+                    if (shard_info.pool->isTryResultInvalid(result, settings.distributed_insert_skip_read_only_replicas))
+                        throw Exception(ErrorCodes::LOGICAL_ERROR, "Got an invalid connection result");
+
+                    job.connection_entry = std::move(result.entry);
                 }
                 else
                 {

From c4bdfd418d698bc9229fd7ce5ff34de96e6ffb6e Mon Sep 17 00:00:00 2001
From: Nikita Taranov <nikita.taranov@clickhouse.com>
Date: Mon, 29 Jul 2024 17:48:20 +0100
Subject: [PATCH 1002/1488] impl

---
 .../performance/views_max_insert_threads.xml  | 20 +++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/tests/performance/views_max_insert_threads.xml b/tests/performance/views_max_insert_threads.xml
index 2988984f5d8..473bcd02ab8 100644
--- a/tests/performance/views_max_insert_threads.xml
+++ b/tests/performance/views_max_insert_threads.xml
@@ -1,11 +1,11 @@
 <!-- https://github.com/ClickHouse/ClickHouse/issues/37900 -->
-<test>
-    <create_query>create table views_max_insert_threads_null (a UInt64) Engine = Null</create_query>
-    <create_query>create materialized view views_max_insert_threads_mv Engine = Null AS select now() as ts, max(a) from views_max_insert_threads_null group by ts</create_query>
-
-    <query>insert into views_max_insert_threads_null select * from numbers_mt(3000000000) settings max_threads = 16, max_insert_threads=16</query>
-
-    <drop_query>drop table if exists views_max_insert_threads_null</drop_query>
-    <drop_query>drop table if exists views_max_insert_threads_mv</drop_query>
-</test>
-
+<!-- the test degraded significantly after #66144 to the point when it could be only thrown away -->
+<!-- <test> -->
+<!--     <create_query>create table views_max_insert_threads_null (a UInt64) Engine = Null</create_query> -->
+<!--     <create_query>create materialized view views_max_insert_threads_mv Engine = Null AS select now() as ts, max(a) from views_max_insert_threads_null group by ts</create_query> -->
+<!---->
+<!--     <query>insert into views_max_insert_threads_null select * from numbers_mt(3000000000) settings max_threads = 16, max_insert_threads=16</query> -->
+<!---->
+<!--     <drop_query>drop table if exists views_max_insert_threads_null</drop_query> -->
+<!--     <drop_query>drop table if exists views_max_insert_threads_mv</drop_query> -->
+<!-- </test> -->

From b2d8eaf1e6d67ab76f3e86cd4fd857e9535a9d20 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 29 Jul 2024 18:55:08 +0200
Subject: [PATCH 1003/1488] Debug TimerDescriptor

---
 src/Common/TimerDescriptor.cpp   | 26 ++++++++++++++++++++++++--
 src/IO/MMappedFileDescriptor.cpp |  2 --
 2 files changed, 24 insertions(+), 4 deletions(-)

diff --git a/src/Common/TimerDescriptor.cpp b/src/Common/TimerDescriptor.cpp
index b36ea4059cb..9a171ae9487 100644
--- a/src/Common/TimerDescriptor.cpp
+++ b/src/Common/TimerDescriptor.cpp
@@ -2,9 +2,11 @@
 
 #include <Common/TimerDescriptor.h>
 #include <Common/Exception.h>
+#include <Common/logger_useful.h>
 
 #include <sys/timerfd.h>
 #include <unistd.h>
+#include <fmt/format.h>
 
 
 namespace DB
@@ -89,9 +91,29 @@ void TimerDescriptor::drain() const
 
             /// A signal happened, need to retry.
             if (errno == EINTR)
-                continue;
+            {
+                /** This is to help with debugging.
+                  *
+                  * Sometimes reading from timer_fd blocks, which should not happen, because we opened it in a non-blocking mode.
+                  * But it could be possible if a rogue 3rd-party library closed our file descriptor by mistake
+                  * (for example by double closing due to the lack of exception safety or if it is a crappy code in plain C)
+                  * and then another file descriptor is opened in its place.
+                  *
+                  * Let's try to get a name of this file descriptor and log it.
+                  */
+                LoggerPtr log = getLogger("TimerDescriptor");
 
-            throw ErrnoException(ErrorCodes::CANNOT_READ_FROM_SOCKET, "Cannot drain timer_fd");
+                static constexpr ssize_t max_link_path_length = 256;
+                char link_path[max_link_path_length];
+                ssize_t link_path_length = readlink(fmt::format("/proc/self/fd/{}", timer_fd).c_str(), link_path, max_link_path_length);
+                if (-1 == link_path_length)
+                    throw ErrnoException(ErrorCodes::CANNOT_READ_FROM_SOCKET, "Cannot readlink for a timer_fd {}", timer_fd);
+
+                LOG_TRACE(log, "Received EINTR while trying to drain a TimerDescriptor, fd {}: {}", timer_fd, std::string_view(link_path, link_path_length));
+                continue;
+            }
+
+            throw ErrnoException(ErrorCodes::CANNOT_READ_FROM_SOCKET, "Cannot drain timer_fd {}", timer_fd);
         }
 
         chassert(res == sizeof(buf));
diff --git a/src/IO/MMappedFileDescriptor.cpp b/src/IO/MMappedFileDescriptor.cpp
index a7eb8e4ede5..47f80005c9d 100644
--- a/src/IO/MMappedFileDescriptor.cpp
+++ b/src/IO/MMappedFileDescriptor.cpp
@@ -3,8 +3,6 @@
 #include <sys/stat.h>
 #include <unistd.h>
 
-#include <fmt/format.h>
-
 #include <Common/formatReadable.h>
 #include <Common/Exception.h>
 #include <base/getPageSize.h>

From 45db7c85cf25f9b4b27cedd7464a786c53580d3f Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Mon, 29 Jul 2024 16:57:15 +0000
Subject: [PATCH 1004/1488] Remove has_single_port property from plan stream.

---
 src/Processors/QueryPlan/AggregatingStep.cpp   | 2 --
 src/Processors/QueryPlan/FillingStep.cpp       | 8 +++-----
 src/Processors/QueryPlan/IQueryPlanStep.h      | 6 +-----
 src/Processors/QueryPlan/ITransformingStep.cpp | 3 ---
 src/Processors/QueryPlan/ReadNothingStep.cpp   | 2 +-
 5 files changed, 5 insertions(+), 16 deletions(-)

diff --git a/src/Processors/QueryPlan/AggregatingStep.cpp b/src/Processors/QueryPlan/AggregatingStep.cpp
index f31de80b22d..8a5ed7fde65 100644
--- a/src/Processors/QueryPlan/AggregatingStep.cpp
+++ b/src/Processors/QueryPlan/AggregatingStep.cpp
@@ -134,7 +134,6 @@ AggregatingStep::AggregatingStep(
     {
         output_stream->sort_description = group_by_sort_description;
         output_stream->sort_scope = DataStream::SortScope::Global;
-        output_stream->has_single_port = true;
     }
 }
 
@@ -147,7 +146,6 @@ void AggregatingStep::applyOrder(SortDescription sort_description_for_merging_,
     {
         output_stream->sort_description = group_by_sort_description;
         output_stream->sort_scope = DataStream::SortScope::Global;
-        output_stream->has_single_port = true;
     }
 
     explicit_sorting_required_for_aggregation_in_order = false;
diff --git a/src/Processors/QueryPlan/FillingStep.cpp b/src/Processors/QueryPlan/FillingStep.cpp
index 65c9cf11661..81622389ada 100644
--- a/src/Processors/QueryPlan/FillingStep.cpp
+++ b/src/Processors/QueryPlan/FillingStep.cpp
@@ -39,12 +39,13 @@ FillingStep::FillingStep(
     , interpolate_description(interpolate_description_)
     , use_with_fill_by_sorting_prefix(use_with_fill_by_sorting_prefix_)
 {
-    if (!input_stream_.has_single_port)
-        throw Exception(ErrorCodes::LOGICAL_ERROR, "FillingStep expects single input");
 }
 
 void FillingStep::transformPipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &)
 {
+    if (pipeline.getNumStreams() != 1)
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "FillingStep expects single input");
+
     pipeline.addSimpleTransform([&](const Block & header, QueryPipelineBuilder::StreamType stream_type) -> ProcessorPtr
     {
         if (stream_type == QueryPipelineBuilder::StreamType::Totals)
@@ -69,9 +70,6 @@ void FillingStep::describeActions(JSONBuilder::JSONMap & map) const
 
 void FillingStep::updateOutputStream()
 {
-    if (!input_streams.front().has_single_port)
-        throw Exception(ErrorCodes::LOGICAL_ERROR, "FillingStep expects single input");
-
     output_stream = createOutputStream(
         input_streams.front(), FillingTransform::transformHeader(input_streams.front().header, sort_description), getDataStreamTraits());
 }
diff --git a/src/Processors/QueryPlan/IQueryPlanStep.h b/src/Processors/QueryPlan/IQueryPlanStep.h
index daca88fcceb..44eb7ea0c59 100644
--- a/src/Processors/QueryPlan/IQueryPlanStep.h
+++ b/src/Processors/QueryPlan/IQueryPlanStep.h
@@ -28,9 +28,6 @@ class DataStream
 public:
     Block header;
 
-    /// QueryPipeline has single port. Totals or extremes ports are not counted.
-    bool has_single_port = false;
-
     /// Sorting scope. Please keep the mutual order (more strong mode should have greater value).
     enum class SortScope : uint8_t
     {
@@ -51,8 +48,7 @@ public:
 
     bool hasEqualPropertiesWith(const DataStream & other) const
     {
-        return has_single_port == other.has_single_port
-            && sort_description == other.sort_description
+        return sort_description == other.sort_description
             && (sort_description.empty() || sort_scope == other.sort_scope);
     }
 
diff --git a/src/Processors/QueryPlan/ITransformingStep.cpp b/src/Processors/QueryPlan/ITransformingStep.cpp
index 9ecfdb0af22..3fa9d1b8308 100644
--- a/src/Processors/QueryPlan/ITransformingStep.cpp
+++ b/src/Processors/QueryPlan/ITransformingStep.cpp
@@ -20,9 +20,6 @@ DataStream ITransformingStep::createOutputStream(
 {
     DataStream output_stream{.header = std::move(output_header)};
 
-    output_stream.has_single_port = stream_traits.returns_single_stream
-                                     || (input_stream.has_single_port && stream_traits.preserves_number_of_streams);
-
     if (stream_traits.preserves_sorting)
     {
         output_stream.sort_description = input_stream.sort_description;
diff --git a/src/Processors/QueryPlan/ReadNothingStep.cpp b/src/Processors/QueryPlan/ReadNothingStep.cpp
index 253f3a5b980..3037172bbd4 100644
--- a/src/Processors/QueryPlan/ReadNothingStep.cpp
+++ b/src/Processors/QueryPlan/ReadNothingStep.cpp
@@ -6,7 +6,7 @@ namespace DB
 {
 
 ReadNothingStep::ReadNothingStep(Block output_header)
-    : ISourceStep(DataStream{.header = std::move(output_header), .has_single_port = true})
+    : ISourceStep(DataStream{.header = std::move(output_header)})
 {
 }
 

From 412268bf4e64b5c0df3980e5b8ccd2b078cf2177 Mon Sep 17 00:00:00 2001
From: avogar <avogar@clickhouse.com>
Date: Mon, 29 Jul 2024 17:05:46 +0000
Subject: [PATCH 1005/1488] Update reference

---
 ...dynamic_read_subcolumns_small.reference.j2 | 20 +++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/tests/queries/0_stateless/03036_dynamic_read_subcolumns_small.reference.j2 b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_small.reference.j2
index 3d814e1205a..be3f4e53990 100644
--- a/tests/queries/0_stateless/03036_dynamic_read_subcolumns_small.reference.j2
+++ b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_small.reference.j2
@@ -25,26 +25,26 @@ UInt64
 7	7	\N	[]
 8	8	\N	[]
 9	9	\N	[]
-[[0]]	\N	\N	[]
 str_10	\N	str_10	[]
-[[0,1]]	\N	\N	[]
+[[0]]	\N	\N	[]
 str_11	\N	str_11	[]
-[[0,1,2]]	\N	\N	[]
+[[0,1]]	\N	\N	[]
 str_12	\N	str_12	[]
-[[0,1,2,3]]	\N	\N	[]
+[[0,1,2]]	\N	\N	[]
 str_13	\N	str_13	[]
-[[0,1,2,3,4]]	\N	\N	[]
+[[0,1,2,3]]	\N	\N	[]
 str_14	\N	str_14	[]
-[[0,1,2,3,4,5]]	\N	\N	[]
+[[0,1,2,3,4]]	\N	\N	[]
 str_15	\N	str_15	[]
-[[0,1,2,3,4,5,6]]	\N	\N	[]
+[[0,1,2,3,4,5]]	\N	\N	[]
 str_16	\N	str_16	[]
-[[0,1,2,3,4,5,6,7]]	\N	\N	[]
+[[0,1,2,3,4,5,6]]	\N	\N	[]
 str_17	\N	str_17	[]
-[[0,1,2,3,4,5,6,7,8]]	\N	\N	[]
+[[0,1,2,3,4,5,6,7]]	\N	\N	[]
 str_18	\N	str_18	[]
-[[0,1,2,3,4,5,6,7,8,9]]	\N	\N	[]
+[[0,1,2,3,4,5,6,7,8]]	\N	\N	[]
 str_19	\N	str_19	[]
+[[0,1,2,3,4,5,6,7,8,9]]	\N	\N	[]
 [20]	\N	\N	[20]
 ['str_21','str_21']	\N	\N	['str_21','str_21']
 [22,22,22]	\N	\N	[22,22,22]

From 812a2b929938c293441f6a893adf96a00d469351 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Mon, 29 Jul 2024 17:24:28 +0000
Subject: [PATCH 1006/1488] formatDateTime[InJodaSyntax]: make format string
 optional

---
 .../functions/type-conversion-functions.md    | 12 ++++----
 src/Functions/parseDateTime.cpp               | 30 ++++++++++++-------
 .../02668_parse_datetime.reference            |  6 +++-
 .../0_stateless/02668_parse_datetime.sql      |  7 ++++-
 ...68_parse_datetime_in_joda_syntax.reference |  6 +++-
 .../02668_parse_datetime_in_joda_syntax.sql   |  7 ++++-
 6 files changed, 48 insertions(+), 20 deletions(-)

diff --git a/docs/en/sql-reference/functions/type-conversion-functions.md b/docs/en/sql-reference/functions/type-conversion-functions.md
index 7cc2c022143..dc90697bd20 100644
--- a/docs/en/sql-reference/functions/type-conversion-functions.md
+++ b/docs/en/sql-reference/functions/type-conversion-functions.md
@@ -3467,13 +3467,13 @@ This function is the opposite operation of function [formatDateTime](../function
 **Syntax**
 
 ``` sql
-parseDateTime(str, format[, timezone])
+parseDateTime(str[, format[, timezone]])
 ```
 
 **Arguments**
 
-- `str` — the String to be parsed
-- `format` — the format string
+- `str` — The String to be parsed
+- `format` — The format string. Optional. `%Y-%m-%d %H:%i:%s` if not specified.
 - `timezone` — [Timezone](/docs/en/operations/server-configuration-parameters/settings.md/#server_configuration_parameters-timezone). Optional.
 
 **Returned value(s)**
@@ -3516,13 +3516,13 @@ This function is the opposite operation of function [formatDateTimeInJodaSyntax]
 **Syntax**
 
 ``` sql
-parseDateTimeInJodaSyntax(str, format[, timezone])
+parseDateTimeInJodaSyntax(str[, format[, timezone]])
 ```
 
 **Arguments**
 
-- `str` — the String to be parsed
-- `format` — the format string
+- `str` — The String to be parsed
+- `format` — The format string. Optional. `yyyy-MM-dd HH:mm:ss` if not specified.
 - `timezone` — [Timezone](/docs/en/operations/server-configuration-parameters/settings.md/#server_configuration_parameters-timezone). Optional.
 
 **Returned value(s)**
diff --git a/src/Functions/parseDateTime.cpp b/src/Functions/parseDateTime.cpp
index bdca0151bba..7ca10677be7 100644
--- a/src/Functions/parseDateTime.cpp
+++ b/src/Functions/parseDateTime.cpp
@@ -582,11 +582,11 @@ namespace
         DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
         {
             FunctionArgumentDescriptors mandatory_args{
-                {"time", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"},
-                {"format", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"}
+                {"time", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"}
             };
 
             FunctionArgumentDescriptors optional_args{
+                {"format", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), nullptr, "String"},
                 {"timezone", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isString), &isColumnConst, "const String"}
             };
 
@@ -2029,14 +2029,24 @@ namespace
 
         String getFormat(const ColumnsWithTypeAndName & arguments) const
         {
-            const auto * format_column = checkAndGetColumnConst<ColumnString>(arguments[1].column.get());
-            if (!format_column)
-                throw Exception(
-                    ErrorCodes::ILLEGAL_COLUMN,
-                    "Illegal column {} of second ('format') argument of function {}. Must be constant string.",
-                    arguments[1].column->getName(),
-                    getName());
-            return format_column->getValue<String>();
+            if (arguments.size() == 1)
+            {
+                if constexpr (parse_syntax == ParseSyntax::MySQL)
+                    return "%Y-%m-%d %H:%i:%s";
+                else
+                    return "yyyy-MM-dd HH:mm:ss";
+            }
+            else
+            {
+                const auto * col_format = checkAndGetColumnConst<ColumnString>(arguments[1].column.get());
+                if (!col_format)
+                    throw Exception(
+                        ErrorCodes::ILLEGAL_COLUMN,
+                        "Illegal column {} of second ('format') argument of function {}. Must be constant string.",
+                        arguments[1].column->getName(),
+                        getName());
+                return col_format->getValue<String>();
+            }
         }
 
         const DateLUTImpl & getTimeZone(const ColumnsWithTypeAndName & arguments) const
diff --git a/tests/queries/0_stateless/02668_parse_datetime.reference b/tests/queries/0_stateless/02668_parse_datetime.reference
index d21a51ce70c..b67ca2d8b76 100644
--- a/tests/queries/0_stateless/02668_parse_datetime.reference
+++ b/tests/queries/0_stateless/02668_parse_datetime.reference
@@ -239,7 +239,7 @@ select sTr_To_DaTe('10:04:11 03-07-2019', '%s:%i:%H %d-%m-%Y', 'UTC') = toDateTi
 select str_to_date('10:04:11 invalid 03-07-2019', '%s:%i:%H %d-%m-%Y', 'UTC') IS NULL;
 1
 -- Error handling
-select parseDateTime('12 AM'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH }
+select parseDateTime(); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH }
 select parseDateTime('12 AM', '%h %p', 'UTC', 'a fourth argument'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH }
 -- Fuzzer crash bug #53715
 select parseDateTime('', '', toString(number)) from numbers(13); -- { serverError ILLEGAL_COLUMN }
@@ -270,3 +270,7 @@ select parseDateTime('8 13, 2022, 7:58:32', '%c %e, %G, %k:%i:%s', 'UTC');
 2022-08-13 07:58:32
 select parseDateTime('08 13, 2022, 07:58:32', '%c %e, %G, %k:%i:%s', 'UTC');
 2022-08-13 07:58:32
+-- The format string argument is optional
+set session_timezone = 'UTC'; -- don't randomize the session timezone
+select parseDateTime('2021-01-04 23:12:34') = toDateTime('2021-01-04 23:12:34');
+1
diff --git a/tests/queries/0_stateless/02668_parse_datetime.sql b/tests/queries/0_stateless/02668_parse_datetime.sql
index 02ac0c5f35c..7b3aed60a4a 100644
--- a/tests/queries/0_stateless/02668_parse_datetime.sql
+++ b/tests/queries/0_stateless/02668_parse_datetime.sql
@@ -162,7 +162,7 @@ select sTr_To_DaTe('10:04:11 03-07-2019', '%s:%i:%H %d-%m-%Y', 'UTC') = toDateTi
 select str_to_date('10:04:11 invalid 03-07-2019', '%s:%i:%H %d-%m-%Y', 'UTC') IS NULL;
 
 -- Error handling
-select parseDateTime('12 AM'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH }
+select parseDateTime(); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH }
 select parseDateTime('12 AM', '%h %p', 'UTC', 'a fourth argument'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH }
 
 -- Fuzzer crash bug #53715
@@ -187,4 +187,9 @@ select parseDateTime('08 13, 2022, 07:58:32', '%m %e, %G, %k:%i:%s', 'UTC');
 select parseDateTime('8 13, 2022, 7:58:32', '%c %e, %G, %k:%i:%s', 'UTC');
 select parseDateTime('08 13, 2022, 07:58:32', '%c %e, %G, %k:%i:%s', 'UTC');
 
+-- The format string argument is optional
+set session_timezone = 'UTC'; -- don't randomize the session timezone
+select parseDateTime('2021-01-04 23:12:34') = toDateTime('2021-01-04 23:12:34');
+
+
 -- { echoOff }
diff --git a/tests/queries/0_stateless/02668_parse_datetime_in_joda_syntax.reference b/tests/queries/0_stateless/02668_parse_datetime_in_joda_syntax.reference
index 9fbf105dc41..6f560577ab5 100644
--- a/tests/queries/0_stateless/02668_parse_datetime_in_joda_syntax.reference
+++ b/tests/queries/0_stateless/02668_parse_datetime_in_joda_syntax.reference
@@ -354,5 +354,9 @@ select parseDateTimeInJodaSyntaxOrNull('2001 366 2000', 'yyyy D yyyy', 'UTC') =
 select parseDateTimeInJodaSyntaxOrNull('2001 invalid 366 2000', 'yyyy D yyyy', 'UTC') IS NULL;
 1
 -- Error handling
-select parseDateTimeInJodaSyntax('12 AM'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH }
+select parseDateTimeInJodaSyntax(); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH }
 select parseDateTimeInJodaSyntax('12 AM', 'h a', 'UTC', 'a fourth argument'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH }
+-- The format string argument is optional
+set session_timezone = 'UTC'; -- don't randomize the session timezone
+select parseDateTimeInJodaSyntax('2021-01-04 23:12:34') = toDateTime('2021-01-04 23:12:34');
+1
diff --git a/tests/queries/0_stateless/02668_parse_datetime_in_joda_syntax.sql b/tests/queries/0_stateless/02668_parse_datetime_in_joda_syntax.sql
index f5810d3d4c3..28d14607ba6 100644
--- a/tests/queries/0_stateless/02668_parse_datetime_in_joda_syntax.sql
+++ b/tests/queries/0_stateless/02668_parse_datetime_in_joda_syntax.sql
@@ -239,6 +239,11 @@ select parseDateTimeInJodaSyntaxOrNull('2001 366 2000', 'yyyy D yyyy', 'UTC') =
 select parseDateTimeInJodaSyntaxOrNull('2001 invalid 366 2000', 'yyyy D yyyy', 'UTC') IS NULL;
 
 -- Error handling
-select parseDateTimeInJodaSyntax('12 AM'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH }
+select parseDateTimeInJodaSyntax(); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH }
 select parseDateTimeInJodaSyntax('12 AM', 'h a', 'UTC', 'a fourth argument'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH }
+
+-- The format string argument is optional
+set session_timezone = 'UTC'; -- don't randomize the session timezone
+select parseDateTimeInJodaSyntax('2021-01-04 23:12:34') = toDateTime('2021-01-04 23:12:34');
+
 -- { echoOff }

From 1c9d60ca972eab618b82704e14f1a680daed9c04 Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <nikitamikhaylov@clickhouse.com>
Date: Mon, 15 Jul 2024 16:36:45 +0000
Subject: [PATCH 1007/1488] Refactoring

---
 programs/client/Client.h             |   6 +-
 programs/local/LocalServer.h         |   4 +-
 src/Client/ClientApplicationBase.cpp | 539 +++++++++++++++++++++++++++
 src/Client/ClientApplicationBase.h   |  54 +++
 src/Client/ClientBase.cpp            | 362 +-----------------
 src/Client/ClientBase.h              |  73 +++-
 6 files changed, 659 insertions(+), 379 deletions(-)
 create mode 100644 src/Client/ClientApplicationBase.cpp
 create mode 100644 src/Client/ClientApplicationBase.h

diff --git a/programs/client/Client.h b/programs/client/Client.h
index 9571440d6ba..7fdf77031ab 100644
--- a/programs/client/Client.h
+++ b/programs/client/Client.h
@@ -1,14 +1,16 @@
 #pragma once
 
-#include <Client/ClientBase.h>
+#include <Client/ClientApplicationBase.h>
 
 
 namespace DB
 {
 
-class Client : public ClientBase
+class Client : public ClientApplicationBase
 {
 public:
+    using Arguments = ClientApplicationBase::Arguments;
+
     Client() = default;
 
     void initialize(Poco::Util::Application & self) override;
diff --git a/programs/local/LocalServer.h b/programs/local/LocalServer.h
index ae9980311e1..b18a7a90961 100644
--- a/programs/local/LocalServer.h
+++ b/programs/local/LocalServer.h
@@ -1,6 +1,6 @@
 #pragma once
 
-#include <Client/ClientBase.h>
+#include <Client/ClientApplicationBase.h>
 #include <Client/LocalConnection.h>
 
 #include <Core/ServerSettings.h>
@@ -21,7 +21,7 @@ namespace DB
 /// Lightweight Application for clickhouse-local
 /// No networking, no extra configs and working directories, no pid and status files, no dictionaries, no logging.
 /// Quiet mode by default
-class LocalServer : public ClientBase, public Loggers
+class LocalServer : public ClientApplicationBase, public Loggers
 {
 public:
     LocalServer() = default;
diff --git a/src/Client/ClientApplicationBase.cpp b/src/Client/ClientApplicationBase.cpp
new file mode 100644
index 00000000000..59c98983694
--- /dev/null
+++ b/src/Client/ClientApplicationBase.cpp
@@ -0,0 +1,539 @@
+#include <Client/ClientApplicationBase.h>
+
+#include <base/argsToConfig.h>
+#include <base/safeExit.h>
+#include <Core/BaseSettingsProgramOptions.h>
+#include <Common/clearPasswordFromCommandLine.h>
+#include <Common/TerminalSize.h>
+#include <Common/Exception.h>
+
+#include <Common/config_version.h>
+#include "config.h"
+
+#include <unordered_set>
+#include <string>
+#include <boost/algorithm/string/case_conv.hpp>
+#include <boost/algorithm/string/replace.hpp>
+#include <boost/algorithm/string/split.hpp>
+
+using namespace std::literals;
+
+namespace CurrentMetrics
+{
+    extern const Metric MemoryTracking;
+}
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+    extern const int BAD_ARGUMENTS;
+    extern const int CANNOT_SET_SIGNAL_HANDLER;
+    extern const int UNRECOGNIZED_ARGUMENTS;
+}
+
+static ClientInfo::QueryKind parseQueryKind(const String & query_kind)
+{
+    if (query_kind == "initial_query")
+        return ClientInfo::QueryKind::INITIAL_QUERY;
+    if (query_kind == "secondary_query")
+        return ClientInfo::QueryKind::SECONDARY_QUERY;
+    if (query_kind == "no_query")
+        return ClientInfo::QueryKind::NO_QUERY;
+    throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown query kind {}", query_kind);
+}
+
+/// This signal handler is set only for SIGINT and SIGQUIT.
+void interruptSignalHandler(int signum)
+{
+    if (ClientApplicationBase::getInstance().tryStopQuery())
+        safeExit(128 + signum);
+}
+
+ClientApplicationBase::~ClientApplicationBase() = default;
+ClientApplicationBase::ClientApplicationBase() : ClientBase(STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO, std::cin, std::cout, std::cerr) {}
+
+ClientApplicationBase & ClientApplicationBase::getInstance()
+{
+    return dynamic_cast<ClientApplicationBase&>(Poco::Util::Application::instance());
+}
+
+void ClientApplicationBase::setupSignalHandler()
+{
+    ClientApplicationBase::getInstance().stopQuery();
+
+    struct sigaction new_act;
+    memset(&new_act, 0, sizeof(new_act));
+
+    new_act.sa_handler = interruptSignalHandler;
+    new_act.sa_flags = 0;
+
+#if defined(OS_DARWIN)
+    sigemptyset(&new_act.sa_mask);
+#else
+    if (sigemptyset(&new_act.sa_mask))
+        throw ErrnoException(ErrorCodes::CANNOT_SET_SIGNAL_HANDLER, "Cannot set signal handler");
+#endif
+
+    if (sigaction(SIGINT, &new_act, nullptr))
+        throw ErrnoException(ErrorCodes::CANNOT_SET_SIGNAL_HANDLER, "Cannot set signal handler");
+
+    if (sigaction(SIGQUIT, &new_act, nullptr))
+        throw ErrnoException(ErrorCodes::CANNOT_SET_SIGNAL_HANDLER, "Cannot set signal handler");
+}
+
+
+namespace
+{
+
+/// Define transparent hash to we can use
+/// std::string_view with the containers
+struct TransparentStringHash
+{
+    using is_transparent = void;
+    size_t operator()(std::string_view txt) const
+    {
+        return std::hash<std::string_view>{}(txt);
+    }
+};
+
+/*
+ * This functor is used to parse command line arguments and replace dashes with underscores,
+ * allowing options to be specified using either dashes or underscores.
+ */
+class OptionsAliasParser
+{
+public:
+    explicit OptionsAliasParser(const boost::program_options::options_description& options)
+    {
+        options_names.reserve(options.options().size());
+        for (const auto& option : options.options())
+            options_names.insert(option->long_name());
+    }
+
+    /*
+     * Parses arguments by replacing dashes with underscores, and matches the resulting name with known options
+     * Implements boost::program_options::ext_parser logic
+     */
+    std::pair<std::string, std::string> operator()(const std::string & token) const
+    {
+        if (!token.starts_with("--"))
+            return {};
+        std::string arg = token.substr(2);
+
+        // divide token by '=' to separate key and value if options style=long_allow_adjacent
+        auto pos_eq = arg.find('=');
+        std::string key = arg.substr(0, pos_eq);
+
+        if (options_names.contains(key))
+            // option does not require any changes, because it is already correct
+            return {};
+
+        std::replace(key.begin(), key.end(), '-', '_');
+        if (!options_names.contains(key))
+            // after replacing '-' with '_' argument is still unknown
+            return {};
+
+        std::string value;
+        if (pos_eq != std::string::npos && pos_eq < arg.size())
+            value = arg.substr(pos_eq + 1);
+
+        return {key, value};
+    }
+
+private:
+    std::unordered_set<std::string> options_names;
+};
+
+}
+
+/// Enable optimizations even in debug builds because otherwise options parsing becomes extremely slow affecting .sh tests
+#if defined(__clang__)
+#pragma clang optimize on
+#endif
+void ClientApplicationBase::parseAndCheckOptions(OptionsDescription & options_description, po::variables_map & options, Arguments & arguments)
+{
+    if (allow_repeated_settings)
+        addProgramOptionsAsMultitokens(cmd_settings, options_description.main_description.value());
+    else
+        addProgramOptions(cmd_settings, options_description.main_description.value());
+
+    if (allow_merge_tree_settings)
+    {
+        /// Add merge tree settings manually, because names of some settings
+        /// may clash. Query settings have higher priority and we just
+        /// skip ambiguous merge tree settings.
+        auto & main_options = options_description.main_description.value();
+
+        std::unordered_set<std::string, TransparentStringHash, std::equal_to<>> main_option_names;
+        for (const auto & option : main_options.options())
+            main_option_names.insert(option->long_name());
+
+        for (const auto & setting : cmd_merge_tree_settings.all())
+        {
+            const auto add_setting = [&](const std::string_view name)
+            {
+                if (auto it = main_option_names.find(name); it != main_option_names.end())
+                    return;
+
+                if (allow_repeated_settings)
+                    addProgramOptionAsMultitoken(cmd_merge_tree_settings, main_options, name, setting);
+                else
+                    addProgramOption(cmd_merge_tree_settings, main_options, name, setting);
+            };
+
+            const auto & setting_name = setting.getName();
+
+            add_setting(setting_name);
+
+            const auto & settings_to_aliases = MergeTreeSettings::Traits::settingsToAliases();
+            if (auto it = settings_to_aliases.find(setting_name); it != settings_to_aliases.end())
+            {
+                for (const auto alias : it->second)
+                {
+                    add_setting(alias);
+                }
+            }
+        }
+    }
+
+    /// Parse main commandline options.
+    auto parser = po::command_line_parser(arguments)
+                      .options(options_description.main_description.value())
+                      .extra_parser(OptionsAliasParser(options_description.main_description.value()))
+                      .allow_unregistered();
+    po::parsed_options parsed = parser.run();
+
+    /// Check unrecognized options without positional options.
+    auto unrecognized_options = po::collect_unrecognized(parsed.options, po::collect_unrecognized_mode::exclude_positional);
+    if (!unrecognized_options.empty())
+    {
+        auto hints = this->getHints(unrecognized_options[0]);
+        if (!hints.empty())
+            throw Exception(ErrorCodes::UNRECOGNIZED_ARGUMENTS, "Unrecognized option '{}'. Maybe you meant {}",
+                            unrecognized_options[0], toString(hints));
+
+        throw Exception(ErrorCodes::UNRECOGNIZED_ARGUMENTS, "Unrecognized option '{}'", unrecognized_options[0]);
+    }
+
+    /// Check positional options.
+    for (const auto & op : parsed.options)
+    {
+        if (!op.unregistered && op.string_key.empty() && !op.original_tokens[0].starts_with("--")
+            && !op.original_tokens[0].empty() && !op.value.empty())
+        {
+            /// Two special cases for better usability:
+            /// - if the option contains a whitespace, it might be a query: clickhouse "SELECT 1"
+            /// These are relevant for interactive usage - user-friendly, but questionable in general.
+            /// In case of ambiguity or for scripts, prefer using proper options.
+
+            const auto & token = op.original_tokens[0];
+            po::variable_value value(boost::any(op.value), false);
+
+            const char * option;
+            if (token.contains(' '))
+                option = "query";
+            else
+                throw Exception(ErrorCodes::BAD_ARGUMENTS, "Positional option `{}` is not supported.", token);
+
+            if (!options.emplace(option, value).second)
+                throw Exception(ErrorCodes::BAD_ARGUMENTS, "Positional option `{}` is not supported.", token);
+        }
+    }
+
+    po::store(parsed, options);
+}
+
+void ClientApplicationBase::addMultiquery(std::string_view query, Arguments & common_arguments) const
+{
+    common_arguments.emplace_back("--multiquery");
+    common_arguments.emplace_back("-q");
+    common_arguments.emplace_back(query);
+}
+
+Poco::Util::LayeredConfiguration & ClientApplicationBase::getClientConfiguration()
+{
+    return config();
+}
+
+void ClientApplicationBase::init(int argc, char ** argv)
+{
+    namespace po = boost::program_options;
+
+    /// Don't parse options with Poco library, we prefer neat boost::program_options.
+    stopOptionsProcessing();
+
+    stdin_is_a_tty = isatty(STDIN_FILENO);
+    stdout_is_a_tty = isatty(STDOUT_FILENO);
+    stderr_is_a_tty = isatty(STDERR_FILENO);
+    terminal_width = getTerminalWidth();
+
+    std::vector<Arguments> external_tables_arguments;
+    Arguments common_arguments = {""}; /// 0th argument is ignored.
+    std::vector<Arguments> hosts_and_ports_arguments;
+
+    if (argc)
+        argv0 = argv[0];
+    readArguments(argc, argv, common_arguments, external_tables_arguments, hosts_and_ports_arguments);
+
+    /// Support for Unicode dashes
+    /// Interpret Unicode dashes as default double-hyphen
+    for (auto & arg : common_arguments)
+    {
+        // replace em-dash(U+2014)
+        boost::replace_all(arg, "—", "--");
+        // replace en-dash(U+2013)
+        boost::replace_all(arg, "–", "--");
+        // replace mathematical minus(U+2212)
+        boost::replace_all(arg, "−", "--");
+    }
+
+
+    OptionsDescription options_description;
+    options_description.main_description.emplace(createOptionsDescription("Main options", terminal_width));
+
+    /// Common options for clickhouse-client and clickhouse-local.
+    options_description.main_description->add_options()
+        ("help", "print usage summary, combine with --verbose to display all options")
+        ("verbose", "print query and other debugging info")
+        ("version,V", "print version information and exit")
+        ("version-clean", "print version in machine-readable format and exit")
+
+        ("config-file,C", po::value<std::string>(), "config-file path")
+
+        ("query,q", po::value<std::vector<std::string>>()->multitoken(), R"(Query. Can be specified multiple times (--query "SELECT 1" --query "SELECT 2") or once with multiple comma-separated queries (--query "SELECT 1; SELECT 2;"). In the latter case, INSERT queries with non-VALUE format must be separated by empty lines.)")
+        ("queries-file", po::value<std::vector<std::string>>()->multitoken(), "file path with queries to execute; multiple files can be specified (--queries-file file1 file2...)")
+        ("multiquery,n", "Obsolete, does nothing")
+        ("multiline,m", "If specified, allow multiline queries (do not send the query on Enter)")
+        ("database,d", po::value<std::string>(), "database")
+        ("query_kind", po::value<std::string>()->default_value("initial_query"), "One of initial_query/secondary_query/no_query")
+        ("query_id", po::value<std::string>(), "query_id")
+
+        ("history_file", po::value<std::string>(), "path to history file")
+
+        ("stage", po::value<std::string>()->default_value("complete"), "Request query processing up to specified stage: complete,fetch_columns,with_mergeable_state,with_mergeable_state_after_aggregation,with_mergeable_state_after_aggregation_and_limit")
+        ("progress", po::value<ProgressOption>()->implicit_value(ProgressOption::TTY, "tty")->default_value(ProgressOption::DEFAULT, "default"), "Print progress of queries execution - to TTY: tty|on|1|true|yes; to STDERR non-interactive mode: err; OFF: off|0|false|no; DEFAULT - interactive to TTY, non-interactive is off")
+
+        ("disable_suggestion,A", "Disable loading suggestion data. Note that suggestion data is loaded asynchronously through a second connection to ClickHouse server. Also it is reasonable to disable suggestion if you want to paste a query with TAB characters. Shorthand option -A is for those who get used to mysql client.")
+        ("wait_for_suggestions_to_load", "Load suggestion data synchonously.")
+        ("time,t", "print query execution time to stderr in non-interactive mode (for benchmarks)")
+        ("memory-usage", po::value<std::string>()->implicit_value("default")->default_value("none"), "print memory usage to stderr in non-interactive mode (for benchmarks). Values: 'none', 'default', 'readable'")
+
+        ("echo", "in batch mode, print query before execution")
+
+        ("log-level", po::value<std::string>(), "log level")
+        ("server_logs_file", po::value<std::string>(), "put server logs into specified file")
+
+        ("suggestion_limit", po::value<int>()->default_value(10000), "Suggestion limit for how many databases, tables and columns to fetch.")
+
+        ("format,f", po::value<std::string>(), "default output format (and input format for clickhouse-local)")
+        ("output-format", po::value<std::string>(), "default output format (this option has preference over --format)")
+
+        ("vertical,E", "vertical output format, same as --format=Vertical or FORMAT Vertical or \\G at end of command")
+        ("highlight", po::value<bool>()->default_value(true), "enable or disable basic syntax highlight in interactive command line")
+
+        ("ignore-error", "do not stop processing when an error occurs")
+        ("stacktrace", "print stack traces of exceptions")
+        ("hardware-utilization", "print hardware utilization information in progress bar")
+        ("print-profile-events", po::value(&profile_events.print)->zero_tokens(), "Printing ProfileEvents packets")
+        ("profile-events-delay-ms", po::value<UInt64>()->default_value(profile_events.delay_ms), "Delay between printing `ProfileEvents` packets (-1 - print only totals, 0 - print every single packet)")
+        ("processed-rows", "print the number of locally processed rows")
+
+        ("interactive", "Process queries-file or --query query and start interactive mode")
+        ("pager", po::value<std::string>(), "Pipe all output into this command (less or similar)")
+        ("max_memory_usage_in_client", po::value<std::string>(), "Set memory limit in client/local server")
+
+        ("fuzzer-args", po::value<std::string>(), "Command line arguments for the LLVM's libFuzzer driver. Only relevant if the application is compiled with libFuzzer.")
+
+        ("client_logs_file", po::value<std::string>(), "Path to a file for writing client logs. Currently we only have fatal logs (when the client crashes)")
+    ;
+
+    addOptions(options_description);
+
+    OptionsDescription options_description_non_verbose = options_description;
+
+    auto getter = [](const auto & op)
+    {
+        String op_long_name = op->long_name();
+        return "--" + String(op_long_name);
+    };
+
+    if (options_description.main_description)
+    {
+        const auto & main_options = options_description.main_description->options();
+        std::transform(main_options.begin(), main_options.end(), std::back_inserter(cmd_options), getter);
+    }
+
+    if (options_description.external_description)
+    {
+        const auto & external_options = options_description.external_description->options();
+        std::transform(external_options.begin(), external_options.end(), std::back_inserter(cmd_options), getter);
+    }
+
+    po::variables_map options;
+    parseAndCheckOptions(options_description, options, common_arguments);
+    po::notify(options);
+
+    if (options.count("version") || options.count("V"))
+    {
+        showClientVersion();
+        exit(0); // NOLINT(concurrency-mt-unsafe)
+    }
+
+    if (options.count("version-clean"))
+    {
+        output_stream << VERSION_STRING;
+        exit(0); // NOLINT(concurrency-mt-unsafe)
+    }
+
+    if (options.count("verbose"))
+        getClientConfiguration().setBool("verbose", true);
+
+    /// Output of help message.
+    if (options.count("help")
+        || (options.count("host") && options["host"].as<std::string>() == "elp")) /// If user writes -help instead of --help.
+    {
+        if (getClientConfiguration().getBool("verbose", false))
+            printHelpMessage(options_description, true);
+        else
+            printHelpMessage(options_description_non_verbose, false);
+        exit(0); // NOLINT(concurrency-mt-unsafe)
+    }
+
+    /// Common options for clickhouse-client and clickhouse-local.
+
+    /// Output execution time to stderr in batch mode.
+    if (options.count("time"))
+        getClientConfiguration().setBool("print-time-to-stderr", true);
+    if (options.count("memory-usage"))
+    {
+        const auto & memory_usage_mode = options["memory-usage"].as<std::string>();
+        if (memory_usage_mode != "none" && memory_usage_mode != "default" && memory_usage_mode != "readable")
+            throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown memory-usage mode: {}", memory_usage_mode);
+        getClientConfiguration().setString("print-memory-to-stderr", memory_usage_mode);
+    }
+
+    if (options.count("query"))
+        queries = options["query"].as<std::vector<std::string>>();
+    if (options.count("query_id"))
+        getClientConfiguration().setString("query_id", options["query_id"].as<std::string>());
+    if (options.count("database"))
+        getClientConfiguration().setString("database", options["database"].as<std::string>());
+    if (options.count("config-file"))
+        getClientConfiguration().setString("config-file", options["config-file"].as<std::string>());
+    if (options.count("queries-file"))
+        queries_files = options["queries-file"].as<std::vector<std::string>>();
+    if (options.count("multiline"))
+        getClientConfiguration().setBool("multiline", true);
+    if (options.count("ignore-error"))
+        getClientConfiguration().setBool("ignore-error", true);
+    if (options.count("format"))
+        getClientConfiguration().setString("format", options["format"].as<std::string>());
+    if (options.count("output-format"))
+        getClientConfiguration().setString("output-format", options["output-format"].as<std::string>());
+    if (options.count("vertical"))
+        getClientConfiguration().setBool("vertical", true);
+    if (options.count("stacktrace"))
+        getClientConfiguration().setBool("stacktrace", true);
+    if (options.count("print-profile-events"))
+        getClientConfiguration().setBool("print-profile-events", true);
+    if (options.count("profile-events-delay-ms"))
+        getClientConfiguration().setUInt64("profile-events-delay-ms", options["profile-events-delay-ms"].as<UInt64>());
+    /// Whether to print the number of processed rows at
+    if (options.count("processed-rows"))
+        getClientConfiguration().setBool("print-num-processed-rows", true);
+    if (options.count("progress"))
+    {
+        switch (options["progress"].as<ProgressOption>())
+        {
+            case DEFAULT:
+                getClientConfiguration().setString("progress", "default");
+                break;
+            case OFF:
+                getClientConfiguration().setString("progress", "off");
+                break;
+            case TTY:
+                getClientConfiguration().setString("progress", "tty");
+                break;
+            case ERR:
+                getClientConfiguration().setString("progress", "err");
+                break;
+        }
+    }
+    if (options.count("echo"))
+        getClientConfiguration().setBool("echo", true);
+    if (options.count("disable_suggestion"))
+        getClientConfiguration().setBool("disable_suggestion", true);
+    if (options.count("wait_for_suggestions_to_load"))
+        getClientConfiguration().setBool("wait_for_suggestions_to_load", true);
+    if (options.count("suggestion_limit"))
+        getClientConfiguration().setInt("suggestion_limit", options["suggestion_limit"].as<int>());
+    if (options.count("highlight"))
+        getClientConfiguration().setBool("highlight", options["highlight"].as<bool>());
+    if (options.count("history_file"))
+        getClientConfiguration().setString("history_file", options["history_file"].as<std::string>());
+    if (options.count("interactive"))
+        getClientConfiguration().setBool("interactive", true);
+    if (options.count("pager"))
+        getClientConfiguration().setString("pager", options["pager"].as<std::string>());
+
+    if (options.count("log-level"))
+        Poco::Logger::root().setLevel(options["log-level"].as<std::string>());
+    if (options.count("server_logs_file"))
+        server_logs_file = options["server_logs_file"].as<std::string>();
+
+    query_processing_stage = QueryProcessingStage::fromString(options["stage"].as<std::string>());
+    query_kind = parseQueryKind(options["query_kind"].as<std::string>());
+    profile_events.print = options.count("print-profile-events");
+    profile_events.delay_ms = options["profile-events-delay-ms"].as<UInt64>();
+
+    processOptions(options_description, options, external_tables_arguments, hosts_and_ports_arguments);
+    {
+        std::unordered_set<std::string> alias_names;
+        alias_names.reserve(options_description.main_description->options().size());
+        for (const auto& option : options_description.main_description->options())
+            alias_names.insert(option->long_name());
+        argsToConfig(common_arguments, getClientConfiguration(), 100, &alias_names);
+    }
+
+    clearPasswordFromCommandLine(argc, argv);
+
+    /// Limit on total memory usage
+    std::string max_client_memory_usage = getClientConfiguration().getString("max_memory_usage_in_client", "0" /*default value*/);
+    if (max_client_memory_usage != "0")
+    {
+        UInt64 max_client_memory_usage_int = parseWithSizeSuffix<UInt64>(max_client_memory_usage.c_str(), max_client_memory_usage.length());
+
+        total_memory_tracker.setHardLimit(max_client_memory_usage_int);
+        total_memory_tracker.setDescription("(total)");
+        total_memory_tracker.setMetric(CurrentMetrics::MemoryTracking);
+    }
+
+    /// Print stacktrace in case of crash
+    HandledSignals::instance().setupTerminateHandler();
+    HandledSignals::instance().setupCommonDeadlySignalHandlers();
+    /// We don't setup signal handlers for SIGINT, SIGQUIT, SIGTERM because we don't
+    /// have an option for client to shutdown gracefully.
+
+    fatal_channel_ptr = new Poco::SplitterChannel;
+    fatal_console_channel_ptr = new Poco::ConsoleChannel;
+    fatal_channel_ptr->addChannel(fatal_console_channel_ptr);
+    if (options.count("client_logs_file"))
+    {
+        fatal_file_channel_ptr = new Poco::SimpleFileChannel(options["client_logs_file"].as<std::string>());
+        fatal_channel_ptr->addChannel(fatal_file_channel_ptr);
+    }
+
+    fatal_log = createLogger("ClientBase", fatal_channel_ptr.get(), Poco::Message::PRIO_FATAL);
+    signal_listener = std::make_unique<SignalListener>(nullptr, fatal_log);
+    signal_listener_thread.start(*signal_listener);
+
+#if USE_GWP_ASAN
+    GWPAsan::initFinished();
+#endif
+
+}
+
+
+}
diff --git a/src/Client/ClientApplicationBase.h b/src/Client/ClientApplicationBase.h
new file mode 100644
index 00000000000..217fa29c3f4
--- /dev/null
+++ b/src/Client/ClientApplicationBase.h
@@ -0,0 +1,54 @@
+#pragma once
+
+
+#include <Poco/Util/Application.h>
+#include <Client/ClientBase.h>
+#include <Client/Suggest.h>
+#include <Common/NamePrompter.h>
+#include <Common/QueryFuzzer.h>
+#include <boost/program_options.hpp>
+
+#include <vector>
+
+namespace po = boost::program_options;
+
+namespace DB
+{
+
+void interruptSignalHandler(int signum);
+
+/**
+ * The base class for client appliucations such as
+ * clickhouse-client or clickhouse-local.
+ * The main purpose and responsibility of it is dealing with
+ * application-specific stuff such as command line arguments parsing
+ * and setting up signal handlers, so queries will be cancelled after
+ * Ctrl+C is pressed.
+ */
+class ClientApplicationBase : public ClientBase, public Poco::Util::Application, public IHints<2>
+{
+public:
+    using ClientBase::processOptions;
+    using Arguments = ClientBase::Arguments;
+
+    static ClientApplicationBase & getInstance();
+
+    ClientApplicationBase();
+    ~ClientApplicationBase() override;
+
+    void init(int argc, char ** argv);
+    std::vector<String> getAllRegisteredNames() const override { return cmd_options; }
+
+protected:
+    Poco::Util::LayeredConfiguration & getClientConfiguration() override;
+    void setupSignalHandler() override;
+    void addMultiquery(std::string_view query, Arguments & common_arguments) const;
+
+private:
+    void parseAndCheckOptions(OptionsDescription & options_description, po::variables_map & options, Arguments & arguments);
+
+    std::vector<String> cmd_options;
+};
+
+
+}
diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp
index a88eed25db1..9cf3b955d26 100644
--- a/src/Client/ClientBase.cpp
+++ b/src/Client/ClientBase.cpp
@@ -5,7 +5,6 @@
 #include <Client/InternalTextLogs.h>
 #include <Client/TestTags.h>
 
-#include <base/argsToConfig.h>
 #include <base/safeExit.h>
 #include <Core/Block.h>
 #include <Core/BaseSettingsProgramOptions.h>
@@ -17,7 +16,6 @@
 #include <Common/getNumberOfPhysicalCPUCores.h>
 #include <Common/typeid_cast.h>
 #include <Common/TerminalSize.h>
-#include <Common/clearPasswordFromCommandLine.h>
 #include <Common/StringUtils.h>
 #include <Common/filesystemHelpers.h>
 #include <Common/NetException.h>
@@ -158,17 +156,6 @@ std::istream& operator>> (std::istream & in, ProgressOption & progress)
     return in;
 }
 
-static ClientInfo::QueryKind parseQueryKind(const String & query_kind)
-{
-    if (query_kind == "initial_query")
-        return ClientInfo::QueryKind::INITIAL_QUERY;
-    if (query_kind == "secondary_query")
-        return ClientInfo::QueryKind::SECONDARY_QUERY;
-    if (query_kind == "no_query")
-        return ClientInfo::QueryKind::NO_QUERY;
-    throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown query kind {}", query_kind);
-}
-
 static void incrementProfileEventsBlock(Block & dst, const Block & src)
 {
     if (!dst)
@@ -269,36 +256,6 @@ static void incrementProfileEventsBlock(Block & dst, const Block & src)
     dst.setColumns(std::move(mutable_columns));
 }
 
-
-std::atomic<Int32> exit_after_signals = 0;
-
-class QueryInterruptHandler : private boost::noncopyable
-{
-public:
-    /// Store how much interrupt signals can be before stopping the query
-    /// by default stop after the first interrupt signal.
-    static void start(Int32 signals_before_stop = 1) { exit_after_signals.store(signals_before_stop); }
-
-    /// Set value not greater then 0 to mark the query as stopped.
-    static void stop() { exit_after_signals.store(0); }
-
-    /// Return true if the query was stopped.
-    /// Query was stopped if it received at least "signals_before_stop" interrupt signals.
-    static bool try_stop() { return exit_after_signals.fetch_sub(1) <= 0; }
-    static bool cancelled() { return exit_after_signals.load() <= 0; }
-
-    /// Return how much interrupt signals remain before stop.
-    static Int32 cancelled_status() { return exit_after_signals.load(); }
-};
-
-/// This signal handler is set for SIGINT and SIGQUIT.
-void interruptSignalHandler(int signum)
-{
-    if (QueryInterruptHandler::try_stop())
-        safeExit(128 + signum);
-}
-
-
 /// To cancel the query on local format error.
 class LocalFormatError : public DB::Exception
 {
@@ -345,31 +302,6 @@ ClientBase::ClientBase(
     terminal_width = getTerminalWidth(in_fd, err_fd);
 }
 
-void ClientBase::setupSignalHandler()
-{
-    QueryInterruptHandler::stop();
-
-    struct sigaction new_act;
-    memset(&new_act, 0, sizeof(new_act));
-
-    new_act.sa_handler = interruptSignalHandler;
-    new_act.sa_flags = 0;
-
-#if defined(OS_DARWIN)
-    sigemptyset(&new_act.sa_mask);
-#else
-    if (sigemptyset(&new_act.sa_mask))
-        throw ErrnoException(ErrorCodes::CANNOT_SET_SIGNAL_HANDLER, "Cannot set signal handler");
-#endif
-
-    if (sigaction(SIGINT, &new_act, nullptr))
-        throw ErrnoException(ErrorCodes::CANNOT_SET_SIGNAL_HANDLER, "Cannot set signal handler");
-
-    if (sigaction(SIGQUIT, &new_act, nullptr))
-        throw ErrnoException(ErrorCodes::CANNOT_SET_SIGNAL_HANDLER, "Cannot set signal handler");
-}
-
-
 ASTPtr ClientBase::parseQuery(const char *& pos, const char * end, const Settings & settings, bool allow_multi_statements)
 {
     std::unique_ptr<IParserBase> parser;
@@ -1113,8 +1045,8 @@ void ClientBase::processOrdinaryQuery(const String & query_to_execute, ASTPtr pa
     {
         try
         {
-            QueryInterruptHandler::start(signals_before_stop);
-            SCOPE_EXIT({ QueryInterruptHandler::stop(); });
+            query_interrupt_handler.start(signals_before_stop);
+            SCOPE_EXIT({ query_interrupt_handler.stop(); });
 
             connection->sendQuery(
                 connection_parameters.timeouts,
@@ -1178,13 +1110,13 @@ void ClientBase::receiveResult(ASTPtr parsed_query, Int32 signals_before_stop, b
             /// to avoid losing sync.
             if (!cancelled)
             {
-                if (partial_result_on_first_cancel && QueryInterruptHandler::cancelled_status() == signals_before_stop - 1)
+                if (partial_result_on_first_cancel && query_interrupt_handler.cancelled_status() == signals_before_stop - 1)
                 {
                     connection->sendCancel();
                     /// First cancel reading request was sent. Next requests will only be with a full cancel
                     partial_result_on_first_cancel = false;
                 }
-                else if (QueryInterruptHandler::cancelled())
+                else if (query_interrupt_handler.cancelled())
                 {
                     cancelQuery();
                 }
@@ -1563,8 +1495,8 @@ void ClientBase::processInsertQuery(const String & query_to_execute, ASTPtr pars
             return;
     }
 
-    QueryInterruptHandler::start();
-    SCOPE_EXIT({ QueryInterruptHandler::stop(); });
+    query_interrupt_handler.start();
+    SCOPE_EXIT({ query_interrupt_handler.stop(); });
 
     connection->sendQuery(
         connection_parameters.timeouts,
@@ -1775,7 +1707,7 @@ try
     Block block;
     while (executor.pull(block))
     {
-        if (!cancelled && QueryInterruptHandler::cancelled())
+        if (!cancelled && query_interrupt_handler.cancelled())
         {
             cancelQuery();
             executor.cancel();
@@ -2857,7 +2789,6 @@ void ClientBase::runLibFuzzer()
 void ClientBase::runLibFuzzer() {}
 #endif
 
-
 void ClientBase::clearTerminal()
 {
     /// Clear from cursor until end of screen.
@@ -2867,288 +2798,9 @@ void ClientBase::clearTerminal()
     output_stream << "\033[0J" "\033[?25h";
 }
 
-
 void ClientBase::showClientVersion()
 {
     output_stream << VERSION_NAME << " " + getName() + " version " << VERSION_STRING << VERSION_OFFICIAL << "." << std::endl;
 }
 
-void ClientBase::init(int argc, char ** argv)
-{
-    namespace po = boost::program_options;
-
-    /// Don't parse options with Poco library, we prefer neat boost::program_options.
-    stopOptionsProcessing();
-
-    stdin_is_a_tty = isatty(STDIN_FILENO);
-    stdout_is_a_tty = isatty(STDOUT_FILENO);
-    stderr_is_a_tty = isatty(STDERR_FILENO);
-    terminal_width = getTerminalWidth();
-
-    std::vector<Arguments> external_tables_arguments;
-    Arguments common_arguments = {""}; /// 0th argument is ignored.
-    std::vector<Arguments> hosts_and_ports_arguments;
-
-    if (argc)
-        argv0 = argv[0];
-    readArguments(argc, argv, common_arguments, external_tables_arguments, hosts_and_ports_arguments);
-
-    /// Support for Unicode dashes
-    /// Interpret Unicode dashes as default double-hyphen
-    for (auto & arg : common_arguments)
-    {
-        // replace em-dash(U+2014)
-        boost::replace_all(arg, "—", "--");
-        // replace en-dash(U+2013)
-        boost::replace_all(arg, "–", "--");
-        // replace mathematical minus(U+2212)
-        boost::replace_all(arg, "−", "--");
-    }
-
-
-    OptionsDescription options_description;
-    options_description.main_description.emplace(createOptionsDescription("Main options", terminal_width));
-
-    /// Common options for clickhouse-client and clickhouse-local.
-    options_description.main_description->add_options()
-        ("help", "print usage summary, combine with --verbose to display all options")
-        ("verbose", "print query and other debugging info")
-        ("version,V", "print version information and exit")
-        ("version-clean", "print version in machine-readable format and exit")
-
-        ("config-file,C", po::value<std::string>(), "config-file path")
-
-        ("query,q", po::value<std::vector<std::string>>()->multitoken(), R"(Query. Can be specified multiple times (--query "SELECT 1" --query "SELECT 2") or once with multiple comma-separated queries (--query "SELECT 1; SELECT 2;"). In the latter case, INSERT queries with non-VALUE format must be separated by empty lines.)")
-        ("queries-file", po::value<std::vector<std::string>>()->multitoken(), "file path with queries to execute; multiple files can be specified (--queries-file file1 file2...)")
-        ("multiquery,n", "Obsolete, does nothing")
-        ("multiline,m", "If specified, allow multiline queries (do not send the query on Enter)")
-        ("database,d", po::value<std::string>(), "database")
-        ("query_kind", po::value<std::string>()->default_value("initial_query"), "One of initial_query/secondary_query/no_query")
-        ("query_id", po::value<std::string>(), "query_id")
-
-        ("history_file", po::value<std::string>(), "path to history file")
-
-        ("stage", po::value<std::string>()->default_value("complete"), "Request query processing up to specified stage: complete,fetch_columns,with_mergeable_state,with_mergeable_state_after_aggregation,with_mergeable_state_after_aggregation_and_limit")
-        ("progress", po::value<ProgressOption>()->implicit_value(ProgressOption::TTY, "tty")->default_value(ProgressOption::DEFAULT, "default"), "Print progress of queries execution - to TTY: tty|on|1|true|yes; to STDERR non-interactive mode: err; OFF: off|0|false|no; DEFAULT - interactive to TTY, non-interactive is off")
-
-        ("disable_suggestion,A", "Disable loading suggestion data. Note that suggestion data is loaded asynchronously through a second connection to ClickHouse server. Also it is reasonable to disable suggestion if you want to paste a query with TAB characters. Shorthand option -A is for those who get used to mysql client.")
-        ("wait_for_suggestions_to_load", "Load suggestion data synchonously.")
-        ("time,t", "print query execution time to stderr in non-interactive mode (for benchmarks)")
-        ("memory-usage", po::value<std::string>()->implicit_value("default")->default_value("none"), "print memory usage to stderr in non-interactive mode (for benchmarks). Values: 'none', 'default', 'readable'")
-
-        ("echo", "in batch mode, print query before execution")
-
-        ("log-level", po::value<std::string>(), "log level")
-        ("server_logs_file", po::value<std::string>(), "put server logs into specified file")
-
-        ("suggestion_limit", po::value<int>()->default_value(10000), "Suggestion limit for how many databases, tables and columns to fetch.")
-
-        ("format,f", po::value<std::string>(), "default output format (and input format for clickhouse-local)")
-        ("output-format", po::value<std::string>(), "default output format (this option has preference over --format)")
-
-        ("vertical,E", "vertical output format, same as --format=Vertical or FORMAT Vertical or \\G at end of command")
-        ("highlight", po::value<bool>()->default_value(true), "enable or disable basic syntax highlight in interactive command line")
-
-        ("ignore-error", "do not stop processing when an error occurs")
-        ("stacktrace", "print stack traces of exceptions")
-        ("hardware-utilization", "print hardware utilization information in progress bar")
-        ("print-profile-events", po::value(&profile_events.print)->zero_tokens(), "Printing ProfileEvents packets")
-        ("profile-events-delay-ms", po::value<UInt64>()->default_value(profile_events.delay_ms), "Delay between printing `ProfileEvents` packets (-1 - print only totals, 0 - print every single packet)")
-        ("processed-rows", "print the number of locally processed rows")
-
-        ("interactive", "Process queries-file or --query query and start interactive mode")
-        ("pager", po::value<std::string>(), "Pipe all output into this command (less or similar)")
-        ("max_memory_usage_in_client", po::value<std::string>(), "Set memory limit in client/local server")
-
-        ("fuzzer-args", po::value<std::string>(), "Command line arguments for the LLVM's libFuzzer driver. Only relevant if the application is compiled with libFuzzer.")
-
-        ("client_logs_file", po::value<std::string>(), "Path to a file for writing client logs. Currently we only have fatal logs (when the client crashes)")
-    ;
-
-    addOptions(options_description);
-
-    OptionsDescription options_description_non_verbose = options_description;
-
-    auto getter = [](const auto & op)
-    {
-        String op_long_name = op->long_name();
-        return "--" + String(op_long_name);
-    };
-
-    if (options_description.main_description)
-    {
-        const auto & main_options = options_description.main_description->options();
-        std::transform(main_options.begin(), main_options.end(), std::back_inserter(cmd_options), getter);
-    }
-
-    if (options_description.external_description)
-    {
-        const auto & external_options = options_description.external_description->options();
-        std::transform(external_options.begin(), external_options.end(), std::back_inserter(cmd_options), getter);
-    }
-
-    po::variables_map options;
-    parseAndCheckOptions(options_description, options, common_arguments);
-    po::notify(options);
-
-    if (options.count("version") || options.count("V"))
-    {
-        showClientVersion();
-        exit(0); // NOLINT(concurrency-mt-unsafe)
-    }
-
-    if (options.count("version-clean"))
-    {
-        output_stream << VERSION_STRING;
-        exit(0); // NOLINT(concurrency-mt-unsafe)
-    }
-
-    if (options.count("verbose"))
-        getClientConfiguration().setBool("verbose", true);
-
-    /// Output of help message.
-    if (options.count("help")
-        || (options.count("host") && options["host"].as<std::string>() == "elp")) /// If user writes -help instead of --help.
-    {
-        if (getClientConfiguration().getBool("verbose", false))
-            printHelpMessage(options_description, true);
-        else
-            printHelpMessage(options_description_non_verbose, false);
-        exit(0); // NOLINT(concurrency-mt-unsafe)
-    }
-
-    /// Common options for clickhouse-client and clickhouse-local.
-
-    /// Output execution time to stderr in batch mode.
-    if (options.count("time"))
-        getClientConfiguration().setBool("print-time-to-stderr", true);
-    if (options.count("memory-usage"))
-    {
-        const auto & memory_usage_mode = options["memory-usage"].as<std::string>();
-        if (memory_usage_mode != "none" && memory_usage_mode != "default" && memory_usage_mode != "readable")
-            throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown memory-usage mode: {}", memory_usage_mode);
-        getClientConfiguration().setString("print-memory-to-stderr", memory_usage_mode);
-    }
-
-    if (options.count("query"))
-        queries = options["query"].as<std::vector<std::string>>();
-    if (options.count("query_id"))
-        getClientConfiguration().setString("query_id", options["query_id"].as<std::string>());
-    if (options.count("database"))
-        getClientConfiguration().setString("database", options["database"].as<std::string>());
-    if (options.count("config-file"))
-        getClientConfiguration().setString("config-file", options["config-file"].as<std::string>());
-    if (options.count("queries-file"))
-        queries_files = options["queries-file"].as<std::vector<std::string>>();
-    if (options.count("multiline"))
-        getClientConfiguration().setBool("multiline", true);
-    if (options.count("ignore-error"))
-        getClientConfiguration().setBool("ignore-error", true);
-    if (options.count("format"))
-        getClientConfiguration().setString("format", options["format"].as<std::string>());
-    if (options.count("output-format"))
-        getClientConfiguration().setString("output-format", options["output-format"].as<std::string>());
-    if (options.count("vertical"))
-        getClientConfiguration().setBool("vertical", true);
-    if (options.count("stacktrace"))
-        getClientConfiguration().setBool("stacktrace", true);
-    if (options.count("print-profile-events"))
-        getClientConfiguration().setBool("print-profile-events", true);
-    if (options.count("profile-events-delay-ms"))
-        getClientConfiguration().setUInt64("profile-events-delay-ms", options["profile-events-delay-ms"].as<UInt64>());
-    /// Whether to print the number of processed rows at
-    if (options.count("processed-rows"))
-        getClientConfiguration().setBool("print-num-processed-rows", true);
-    if (options.count("progress"))
-    {
-        switch (options["progress"].as<ProgressOption>())
-        {
-            case DEFAULT:
-                getClientConfiguration().setString("progress", "default");
-                break;
-            case OFF:
-                getClientConfiguration().setString("progress", "off");
-                break;
-            case TTY:
-                getClientConfiguration().setString("progress", "tty");
-                break;
-            case ERR:
-                getClientConfiguration().setString("progress", "err");
-                break;
-        }
-    }
-    if (options.count("echo"))
-        getClientConfiguration().setBool("echo", true);
-    if (options.count("disable_suggestion"))
-        getClientConfiguration().setBool("disable_suggestion", true);
-    if (options.count("wait_for_suggestions_to_load"))
-        getClientConfiguration().setBool("wait_for_suggestions_to_load", true);
-    if (options.count("suggestion_limit"))
-        getClientConfiguration().setInt("suggestion_limit", options["suggestion_limit"].as<int>());
-    if (options.count("highlight"))
-        getClientConfiguration().setBool("highlight", options["highlight"].as<bool>());
-    if (options.count("history_file"))
-        getClientConfiguration().setString("history_file", options["history_file"].as<std::string>());
-    if (options.count("interactive"))
-        getClientConfiguration().setBool("interactive", true);
-    if (options.count("pager"))
-        getClientConfiguration().setString("pager", options["pager"].as<std::string>());
-
-    if (options.count("log-level"))
-        Poco::Logger::root().setLevel(options["log-level"].as<std::string>());
-    if (options.count("server_logs_file"))
-        server_logs_file = options["server_logs_file"].as<std::string>();
-
-    query_processing_stage = QueryProcessingStage::fromString(options["stage"].as<std::string>());
-    query_kind = parseQueryKind(options["query_kind"].as<std::string>());
-    profile_events.print = options.count("print-profile-events");
-    profile_events.delay_ms = options["profile-events-delay-ms"].as<UInt64>();
-
-    processOptions(options_description, options, external_tables_arguments, hosts_and_ports_arguments);
-    {
-        std::unordered_set<std::string> alias_names;
-        alias_names.reserve(options_description.main_description->options().size());
-        for (const auto& option : options_description.main_description->options())
-            alias_names.insert(option->long_name());
-        argsToConfig(common_arguments, getClientConfiguration(), 100, &alias_names);
-    }
-
-    clearPasswordFromCommandLine(argc, argv);
-
-    /// Limit on total memory usage
-    std::string max_client_memory_usage = getClientConfiguration().getString("max_memory_usage_in_client", "0" /*default value*/);
-    if (max_client_memory_usage != "0")
-    {
-        UInt64 max_client_memory_usage_int = parseWithSizeSuffix<UInt64>(max_client_memory_usage.c_str(), max_client_memory_usage.length());
-
-        total_memory_tracker.setHardLimit(max_client_memory_usage_int);
-        total_memory_tracker.setDescription("(total)");
-        total_memory_tracker.setMetric(CurrentMetrics::MemoryTracking);
-    }
-
-    /// Print stacktrace in case of crash
-    HandledSignals::instance().setupTerminateHandler();
-    HandledSignals::instance().setupCommonDeadlySignalHandlers();
-    /// We don't setup signal handlers for SIGINT, SIGQUIT, SIGTERM because we don't
-    /// have an option for client to shutdown gracefully.
-
-    fatal_channel_ptr = new Poco::SplitterChannel;
-    fatal_console_channel_ptr = new Poco::ConsoleChannel;
-    fatal_channel_ptr->addChannel(fatal_console_channel_ptr);
-    if (options.count("client_logs_file"))
-    {
-        fatal_file_channel_ptr = new Poco::SimpleFileChannel(options["client_logs_file"].as<std::string>());
-        fatal_channel_ptr->addChannel(fatal_file_channel_ptr);
-    }
-
-    fatal_log = createLogger("ClientBase", fatal_channel_ptr.get(), Poco::Message::PRIO_FATAL);
-    signal_listener = std::make_unique<SignalListener>(nullptr, fatal_log);
-    signal_listener_thread.start(*signal_listener);
-
-#if USE_GWP_ASAN
-    GWPAsan::initFinished();
-#endif
-
-}
-
 }
diff --git a/src/Client/ClientBase.h b/src/Client/ClientBase.h
index 7689744a373..557ac30d27c 100644
--- a/src/Client/ClientBase.h
+++ b/src/Client/ClientBase.h
@@ -1,26 +1,30 @@
 #pragma once
 
-#include <string_view>
-#include "Common/NamePrompter.h"
-#include <Parsers/ASTCreateQuery.h>
-#include <Common/ProgressIndication.h>
+
+#include <Client/Suggest.h>
+#include <Common/DNSResolver.h>
 #include <Common/InterruptListener.h>
+#include <Common/ProgressIndication.h>
 #include <Common/ShellCommand.h>
 #include <Common/Stopwatch.h>
-#include <Common/DNSResolver.h>
 #include <Core/ExternalTable.h>
 #include <Core/Settings.h>
-#include <Poco/Util/Application.h>
 #include <Poco/ConsoleChannel.h>
 #include <Poco/SimpleFileChannel.h>
 #include <Poco/SplitterChannel.h>
 #include <Interpreters/Context.h>
-#include <Client/Suggest.h>
-#include <Client/QueryFuzzer.h>
-#include <boost/program_options.hpp>
-#include <Storages/StorageFile.h>
-#include <Storages/SelectQueryInfo.h>
+#include <Parsers/ASTCreateQuery.h>
+#include <Poco/Util/Application.h>
 #include <Storages/MergeTree/MergeTreeSettings.h>
+#include <Storages/SelectQueryInfo.h>
+#include <Storages/StorageFile.h>
+
+#include <boost/program_options.hpp>
+
+#include <atomic>
+#include <optional>
+#include <string_view>
+#include <string>
 
 namespace po = boost::program_options;
 
@@ -64,9 +68,16 @@ std::istream& operator>> (std::istream & in, ProgressOption & progress);
 class InternalTextLogs;
 class WriteBufferFromFileDescriptor;
 
-class ClientBase : public Poco::Util::Application, public IHints<2>
+/**
+ * The base class which encapsulates the core functionality of a client.
+ * Can be used in a standalone application (clickhouse-client or clickhouse-local),
+ * or be embedded into server.
+ * Always keep in mind that there can be several instances of this class within
+ * a process. Thus, it cannot keep its state in global shared variables or even use them.
+ * The best example - std::cin, std::cout and std::cerr.
+ */
+class ClientBase
 {
-
 public:
     using Arguments = std::vector<String>;
 
@@ -79,12 +90,11 @@ public:
         std::ostream & output_stream_ = std::cout,
         std::ostream & error_stream_ = std::cerr
     );
+    virtual ~ClientBase();
 
-    ~ClientBase() override;
+    bool tryStopQuery() { return query_interrupt_handler.tryStop(); }
+    void stopQuery() { return query_interrupt_handler.stop(); }
 
-    void init(int argc, char ** argv);
-
-    std::vector<String> getAllRegisteredNames() const override { return cmd_options; }
     ASTPtr parseQuery(const char *& pos, const char * end, const Settings & settings, bool allow_multi_statements);
 
 protected:
@@ -114,7 +124,7 @@ protected:
         ASTPtr parsed_query, std::optional<bool> echo_query_ = {}, bool report_error = false);
 
     static void adjustQueryEnd(const char *& this_query_end, const char * all_queries_end, uint32_t max_parser_depth, uint32_t max_parser_backtracks);
-    static void setupSignalHandler();
+    virtual void setupSignalHandler() = 0;
 
     bool executeMultiQuery(const String & all_queries_text);
     MultiQueryProcessingStage analyzeMultiQueryText(
@@ -188,7 +198,6 @@ private:
     String prompt() const;
 
     void resetOutput();
-    void parseAndCheckOptions(OptionsDescription & options_description, po::variables_map & options, Arguments & arguments);
 
     void updateSuggest(const ASTPtr & ast);
 
@@ -196,6 +205,31 @@ private:
     bool addMergeTreeSettings(ASTCreateQuery & ast_create);
 
 protected:
+
+    class QueryInterruptHandler : private boost::noncopyable
+    {
+    public:
+        /// Store how much interrupt signals can be before stopping the query
+        /// by default stop after the first interrupt signal.
+        void start(Int32 signals_before_stop = 1) { exit_after_signals.store(signals_before_stop); }
+
+        /// Set value not greater then 0 to mark the query as stopped.
+        void stop() { exit_after_signals.store(0); }
+
+        /// Return true if the query was stopped.
+        /// Query was stopped if it received at least "signals_before_stop" interrupt signals.
+        bool tryStop() { return exit_after_signals.fetch_sub(1) <= 0; }
+        bool cancelled() { return exit_after_signals.load() <= 0; }
+
+        /// Return how much interrupt signals remain before stop.
+        Int32 cancelled_status() { return exit_after_signals.load(); }
+
+    private:
+        std::atomic<Int32> exit_after_signals = 0;
+    };
+
+    QueryInterruptHandler query_interrupt_handler;
+
     static bool isSyncInsertWithData(const ASTInsertQuery & insert_query, const ContextPtr & context);
     bool processMultiQueryFromFile(const String & file_name);
 
@@ -239,7 +273,6 @@ protected:
     std::vector<String> queries; /// Queries passed via '--query'
     std::vector<String> queries_files; /// If not empty, queries will be read from these files
     std::vector<String> interleave_queries_files; /// If not empty, run queries from these files before processing every file from 'queries_files'.
-    std::vector<String> cmd_options;
 
     bool stdin_is_a_tty = false; /// stdin is a terminal.
     bool stdout_is_a_tty = false; /// stdout is a terminal.

From 9186e647eb672283b8cb95d2fc152e0994f6df6f Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <nikitamikhaylov@clickhouse.com>
Date: Mon, 15 Jul 2024 17:05:09 +0000
Subject: [PATCH 1008/1488] Fix style

---
 src/Client/ClientBase.cpp | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp
index 9cf3b955d26..56685f9d3f4 100644
--- a/src/Client/ClientBase.cpp
+++ b/src/Client/ClientBase.cpp
@@ -88,12 +88,6 @@
 namespace fs = std::filesystem;
 using namespace std::literals;
 
-
-namespace CurrentMetrics
-{
-    extern const Metric MemoryTracking;
-}
-
 namespace DB
 {
 

From af2c9fcaaf4e2f38c9db105c246ee24b095b256f Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <nikitamikhaylov@clickhouse.com>
Date: Mon, 15 Jul 2024 18:18:42 +0000
Subject: [PATCH 1009/1488] Skip file

---
 utils/check-style/check-style | 1 +
 1 file changed, 1 insertion(+)

diff --git a/utils/check-style/check-style b/utils/check-style/check-style
index 380656cd1ca..3c959617d02 100755
--- a/utils/check-style/check-style
+++ b/utils/check-style/check-style
@@ -318,6 +318,7 @@ std_cerr_cout_excludes=(
     src/Interpreters/Context.cpp
     # IProcessor::dump()
     src/Processors/IProcessor.cpp
+    src/Client/ClientApplicationBase.cpp
     src/Client/ClientBase.cpp
     src/Client/LineReader.cpp
     src/Client/QueryFuzzer.cpp

From 8ba85074e74f403b3d5106f6ef811019075cefb4 Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <nikitamikhaylov@clickhouse.com>
Date: Fri, 19 Jul 2024 12:25:17 +0000
Subject: [PATCH 1010/1488] Fix build

---
 src/Client/ClientApplicationBase.cpp | 16 +++++++++++++++-
 src/Client/ClientApplicationBase.h   | 11 +++++++++++
 src/Client/ClientBase.cpp            | 14 +-------------
 src/Client/ClientBase.h              | 10 ----------
 4 files changed, 27 insertions(+), 24 deletions(-)

diff --git a/src/Client/ClientApplicationBase.cpp b/src/Client/ClientApplicationBase.cpp
index 59c98983694..6b19898ef5c 100644
--- a/src/Client/ClientApplicationBase.cpp
+++ b/src/Client/ClientApplicationBase.cpp
@@ -6,6 +6,7 @@
 #include <Common/clearPasswordFromCommandLine.h>
 #include <Common/TerminalSize.h>
 #include <Common/Exception.h>
+#include <Common/SignalHandlers.h>
 
 #include <Common/config_version.h>
 #include "config.h"
@@ -51,7 +52,20 @@ void interruptSignalHandler(int signum)
         safeExit(128 + signum);
 }
 
-ClientApplicationBase::~ClientApplicationBase() = default;
+ClientApplicationBase::~ClientApplicationBase()
+{
+    try
+    {
+        writeSignalIDtoSignalPipe(SignalListener::StopThread);
+        signal_listener_thread.join();
+        HandledSignals::instance().reset();
+    }
+    catch (...)
+    {
+        tryLogCurrentException(__PRETTY_FUNCTION__);
+    }
+}
+
 ClientApplicationBase::ClientApplicationBase() : ClientBase(STDIN_FILENO, STDOUT_FILENO, STDERR_FILENO, std::cin, std::cout, std::cerr) {}
 
 ClientApplicationBase & ClientApplicationBase::getInstance()
diff --git a/src/Client/ClientApplicationBase.h b/src/Client/ClientApplicationBase.h
index 217fa29c3f4..771bb948cb7 100644
--- a/src/Client/ClientApplicationBase.h
+++ b/src/Client/ClientApplicationBase.h
@@ -6,6 +6,10 @@
 #include <Client/Suggest.h>
 #include <Common/NamePrompter.h>
 #include <Common/QueryFuzzer.h>
+#include <Poco/ConsoleChannel.h>
+#include <Poco/SimpleFileChannel.h>
+#include <Poco/SplitterChannel.h>
+
 #include <boost/program_options.hpp>
 
 #include <vector>
@@ -48,6 +52,13 @@ private:
     void parseAndCheckOptions(OptionsDescription & options_description, po::variables_map & options, Arguments & arguments);
 
     std::vector<String> cmd_options;
+
+    LoggerPtr fatal_log;
+    Poco::AutoPtr<Poco::SplitterChannel> fatal_channel_ptr;
+    Poco::AutoPtr<Poco::Channel> fatal_console_channel_ptr;
+    Poco::AutoPtr<Poco::Channel> fatal_file_channel_ptr;
+    Poco::Thread signal_listener_thread;
+    std::unique_ptr<Poco::Runnable> signal_listener;
 };
 
 
diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp
index 56685f9d3f4..85dfb767e75 100644
--- a/src/Client/ClientBase.cpp
+++ b/src/Client/ClientBase.cpp
@@ -258,19 +258,7 @@ public:
 };
 
 
-ClientBase::~ClientBase()
-{
-    try
-    {
-        writeSignalIDtoSignalPipe(SignalListener::StopThread);
-        signal_listener_thread.join();
-        HandledSignals::instance().reset();
-    }
-    catch (...)
-    {
-        tryLogCurrentException(__PRETTY_FUNCTION__);
-    }
-}
+ClientBase::~ClientBase() = default;
 
 ClientBase::ClientBase(
     int in_fd_,
diff --git a/src/Client/ClientBase.h b/src/Client/ClientBase.h
index 557ac30d27c..304d8c4b890 100644
--- a/src/Client/ClientBase.h
+++ b/src/Client/ClientBase.h
@@ -9,9 +9,6 @@
 #include <Common/Stopwatch.h>
 #include <Core/ExternalTable.h>
 #include <Core/Settings.h>
-#include <Poco/ConsoleChannel.h>
-#include <Poco/SimpleFileChannel.h>
-#include <Poco/SplitterChannel.h>
 #include <Interpreters/Context.h>
 #include <Parsers/ASTCreateQuery.h>
 #include <Poco/Util/Application.h>
@@ -253,13 +250,6 @@ protected:
     /// Client context is a context used only by the client to parse queries, process query parameters and to connect to clickhouse-server.
     ContextMutablePtr client_context;
 
-    LoggerPtr fatal_log;
-    Poco::AutoPtr<Poco::SplitterChannel> fatal_channel_ptr;
-    Poco::AutoPtr<Poco::Channel> fatal_console_channel_ptr;
-    Poco::AutoPtr<Poco::Channel> fatal_file_channel_ptr;
-    Poco::Thread signal_listener_thread;
-    std::unique_ptr<Poco::Runnable> signal_listener;
-
     bool is_interactive = false; /// Use either interactive line editing interface or batch mode.
     bool delayed_interactive = false;
 

From 49dc30d5c28392d361ce0ef1e18f7db73841617f Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <nikitamikhaylov@clickhouse.com>
Date: Wed, 24 Jul 2024 21:42:36 +0000
Subject: [PATCH 1011/1488] Small adjustement

---
 src/Client/ClientApplicationBase.cpp | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/src/Client/ClientApplicationBase.cpp b/src/Client/ClientApplicationBase.cpp
index 6b19898ef5c..0306468d084 100644
--- a/src/Client/ClientApplicationBase.cpp
+++ b/src/Client/ClientApplicationBase.cpp
@@ -48,8 +48,13 @@ static ClientInfo::QueryKind parseQueryKind(const String & query_kind)
 /// This signal handler is set only for SIGINT and SIGQUIT.
 void interruptSignalHandler(int signum)
 {
-    if (ClientApplicationBase::getInstance().tryStopQuery())
-        safeExit(128 + signum);
+    /// Signal handler might be called even before the setup is fully finished
+    /// and client application started to process the query.
+    /// Because of that we have to manually check it.
+    if (auto * instance = ClientApplicationBase::instanceRawPtr(); instance)
+        if (auto * base = dynamic_cast<ClientApplicationBase *>(instance); base)
+            if (base->tryStopQuery())
+                safeExit(128 + signum);
 }
 
 ClientApplicationBase::~ClientApplicationBase()

From c7c1f10720cd194d85de6d81156cbd37304ab52b Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <nikitamikhaylov@clickhouse.com>
Date: Wed, 24 Jul 2024 21:45:16 +0000
Subject: [PATCH 1012/1488] Added new method

---
 base/poco/Util/include/Poco/Util/Application.h | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/base/poco/Util/include/Poco/Util/Application.h b/base/poco/Util/include/Poco/Util/Application.h
index c8d18e1bce9..d1a2021eb67 100644
--- a/base/poco/Util/include/Poco/Util/Application.h
+++ b/base/poco/Util/include/Poco/Util/Application.h
@@ -261,6 +261,11 @@ namespace Util
         ///
         /// Throws a NullPointerException if no Application instance exists.
 
+        static Application * instanceRawPtr();
+        /// Returns a raw pointer to the Application sigleton.
+        ///
+        /// The caller should check whether the result is nullptr.
+
         const Poco::Timestamp & startTime() const;
         /// Returns the application start time (UTC).
 
@@ -448,6 +453,12 @@ namespace Util
     }
 
 
+    inline Application * Application::instanceRawPtr()
+    {
+        return _pInstance;
+    }
+
+
     inline const Poco::Timestamp & Application::startTime() const
     {
         return _startTime;

From 6f068639db627944aaab978c79866ad5a2a234e7 Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <nikitamikhaylov@clickhouse.com>
Date: Thu, 25 Jul 2024 16:38:26 +0000
Subject: [PATCH 1013/1488] Better

---
 src/Client/ClientApplicationBase.h | 1 -
 src/Client/ClientBase.h            | 5 +++++
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/Client/ClientApplicationBase.h b/src/Client/ClientApplicationBase.h
index 771bb948cb7..3663271dd25 100644
--- a/src/Client/ClientApplicationBase.h
+++ b/src/Client/ClientApplicationBase.h
@@ -5,7 +5,6 @@
 #include <Client/ClientBase.h>
 #include <Client/Suggest.h>
 #include <Common/NamePrompter.h>
-#include <Common/QueryFuzzer.h>
 #include <Poco/ConsoleChannel.h>
 #include <Poco/SimpleFileChannel.h>
 #include <Poco/SplitterChannel.h>
diff --git a/src/Client/ClientBase.h b/src/Client/ClientBase.h
index 304d8c4b890..175ebe97075 100644
--- a/src/Client/ClientBase.h
+++ b/src/Client/ClientBase.h
@@ -2,6 +2,7 @@
 
 
 #include <Client/Suggest.h>
+#include <Client/QueryFuzzer.h>
 #include <Common/DNSResolver.h>
 #include <Common/InterruptListener.h>
 #include <Common/ProgressIndication.h>
@@ -9,9 +10,13 @@
 #include <Common/Stopwatch.h>
 #include <Core/ExternalTable.h>
 #include <Core/Settings.h>
+#include <Poco/ConsoleChannel.h>
+#include <Poco/SimpleFileChannel.h>
+#include <Poco/SplitterChannel.h>
 #include <Interpreters/Context.h>
 #include <Parsers/ASTCreateQuery.h>
 #include <Poco/Util/Application.h>
+
 #include <Storages/MergeTree/MergeTreeSettings.h>
 #include <Storages/SelectQueryInfo.h>
 #include <Storages/StorageFile.h>

From a457db34b216ea987a4875a3cbeb5363878d5d5a Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <nikitamikhaylov@clickhouse.com>
Date: Thu, 25 Jul 2024 16:58:41 +0000
Subject: [PATCH 1014/1488] Fixed a typo

---
 base/poco/Util/include/Poco/Util/Application.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/base/poco/Util/include/Poco/Util/Application.h b/base/poco/Util/include/Poco/Util/Application.h
index d1a2021eb67..786e331fe73 100644
--- a/base/poco/Util/include/Poco/Util/Application.h
+++ b/base/poco/Util/include/Poco/Util/Application.h
@@ -262,7 +262,7 @@ namespace Util
         /// Throws a NullPointerException if no Application instance exists.
 
         static Application * instanceRawPtr();
-        /// Returns a raw pointer to the Application sigleton.
+        /// Returns a raw pointer to the Application singleton.
         ///
         /// The caller should check whether the result is nullptr.
 

From 2f255dc68d2cffb8bc17efc153a9e75a9166675d Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <mikhaylovnikitka@gmail.com>
Date: Fri, 26 Jul 2024 01:11:40 +0200
Subject: [PATCH 1015/1488] Fix clang-tidy

---
 src/Client/ClientBase.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Client/ClientBase.h b/src/Client/ClientBase.h
index 175ebe97075..1a23b6b1363 100644
--- a/src/Client/ClientBase.h
+++ b/src/Client/ClientBase.h
@@ -95,7 +95,7 @@ public:
     virtual ~ClientBase();
 
     bool tryStopQuery() { return query_interrupt_handler.tryStop(); }
-    void stopQuery() { return query_interrupt_handler.stop(); }
+    void stopQuery() { query_interrupt_handler.stop(); }
 
     ASTPtr parseQuery(const char *& pos, const char * end, const Settings & settings, bool allow_multi_statements);
 

From 9b4accebb3d19789e69b422ae2f235149e453f94 Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <nikitamikhaylov@clickhouse.com>
Date: Mon, 29 Jul 2024 15:47:52 +0000
Subject: [PATCH 1016/1488] Fix build

---
 src/Client/ClientApplicationBase.cpp    | 162 ------------------------
 src/Client/ClientBaseOptimizedParts.cpp |   4 +-
 2 files changed, 2 insertions(+), 164 deletions(-)

diff --git a/src/Client/ClientApplicationBase.cpp b/src/Client/ClientApplicationBase.cpp
index 0306468d084..4aa8b6c0bbe 100644
--- a/src/Client/ClientApplicationBase.cpp
+++ b/src/Client/ClientApplicationBase.cpp
@@ -102,168 +102,6 @@ void ClientApplicationBase::setupSignalHandler()
         throw ErrnoException(ErrorCodes::CANNOT_SET_SIGNAL_HANDLER, "Cannot set signal handler");
 }
 
-
-namespace
-{
-
-/// Define transparent hash to we can use
-/// std::string_view with the containers
-struct TransparentStringHash
-{
-    using is_transparent = void;
-    size_t operator()(std::string_view txt) const
-    {
-        return std::hash<std::string_view>{}(txt);
-    }
-};
-
-/*
- * This functor is used to parse command line arguments and replace dashes with underscores,
- * allowing options to be specified using either dashes or underscores.
- */
-class OptionsAliasParser
-{
-public:
-    explicit OptionsAliasParser(const boost::program_options::options_description& options)
-    {
-        options_names.reserve(options.options().size());
-        for (const auto& option : options.options())
-            options_names.insert(option->long_name());
-    }
-
-    /*
-     * Parses arguments by replacing dashes with underscores, and matches the resulting name with known options
-     * Implements boost::program_options::ext_parser logic
-     */
-    std::pair<std::string, std::string> operator()(const std::string & token) const
-    {
-        if (!token.starts_with("--"))
-            return {};
-        std::string arg = token.substr(2);
-
-        // divide token by '=' to separate key and value if options style=long_allow_adjacent
-        auto pos_eq = arg.find('=');
-        std::string key = arg.substr(0, pos_eq);
-
-        if (options_names.contains(key))
-            // option does not require any changes, because it is already correct
-            return {};
-
-        std::replace(key.begin(), key.end(), '-', '_');
-        if (!options_names.contains(key))
-            // after replacing '-' with '_' argument is still unknown
-            return {};
-
-        std::string value;
-        if (pos_eq != std::string::npos && pos_eq < arg.size())
-            value = arg.substr(pos_eq + 1);
-
-        return {key, value};
-    }
-
-private:
-    std::unordered_set<std::string> options_names;
-};
-
-}
-
-/// Enable optimizations even in debug builds because otherwise options parsing becomes extremely slow affecting .sh tests
-#if defined(__clang__)
-#pragma clang optimize on
-#endif
-void ClientApplicationBase::parseAndCheckOptions(OptionsDescription & options_description, po::variables_map & options, Arguments & arguments)
-{
-    if (allow_repeated_settings)
-        addProgramOptionsAsMultitokens(cmd_settings, options_description.main_description.value());
-    else
-        addProgramOptions(cmd_settings, options_description.main_description.value());
-
-    if (allow_merge_tree_settings)
-    {
-        /// Add merge tree settings manually, because names of some settings
-        /// may clash. Query settings have higher priority and we just
-        /// skip ambiguous merge tree settings.
-        auto & main_options = options_description.main_description.value();
-
-        std::unordered_set<std::string, TransparentStringHash, std::equal_to<>> main_option_names;
-        for (const auto & option : main_options.options())
-            main_option_names.insert(option->long_name());
-
-        for (const auto & setting : cmd_merge_tree_settings.all())
-        {
-            const auto add_setting = [&](const std::string_view name)
-            {
-                if (auto it = main_option_names.find(name); it != main_option_names.end())
-                    return;
-
-                if (allow_repeated_settings)
-                    addProgramOptionAsMultitoken(cmd_merge_tree_settings, main_options, name, setting);
-                else
-                    addProgramOption(cmd_merge_tree_settings, main_options, name, setting);
-            };
-
-            const auto & setting_name = setting.getName();
-
-            add_setting(setting_name);
-
-            const auto & settings_to_aliases = MergeTreeSettings::Traits::settingsToAliases();
-            if (auto it = settings_to_aliases.find(setting_name); it != settings_to_aliases.end())
-            {
-                for (const auto alias : it->second)
-                {
-                    add_setting(alias);
-                }
-            }
-        }
-    }
-
-    /// Parse main commandline options.
-    auto parser = po::command_line_parser(arguments)
-                      .options(options_description.main_description.value())
-                      .extra_parser(OptionsAliasParser(options_description.main_description.value()))
-                      .allow_unregistered();
-    po::parsed_options parsed = parser.run();
-
-    /// Check unrecognized options without positional options.
-    auto unrecognized_options = po::collect_unrecognized(parsed.options, po::collect_unrecognized_mode::exclude_positional);
-    if (!unrecognized_options.empty())
-    {
-        auto hints = this->getHints(unrecognized_options[0]);
-        if (!hints.empty())
-            throw Exception(ErrorCodes::UNRECOGNIZED_ARGUMENTS, "Unrecognized option '{}'. Maybe you meant {}",
-                            unrecognized_options[0], toString(hints));
-
-        throw Exception(ErrorCodes::UNRECOGNIZED_ARGUMENTS, "Unrecognized option '{}'", unrecognized_options[0]);
-    }
-
-    /// Check positional options.
-    for (const auto & op : parsed.options)
-    {
-        if (!op.unregistered && op.string_key.empty() && !op.original_tokens[0].starts_with("--")
-            && !op.original_tokens[0].empty() && !op.value.empty())
-        {
-            /// Two special cases for better usability:
-            /// - if the option contains a whitespace, it might be a query: clickhouse "SELECT 1"
-            /// These are relevant for interactive usage - user-friendly, but questionable in general.
-            /// In case of ambiguity or for scripts, prefer using proper options.
-
-            const auto & token = op.original_tokens[0];
-            po::variable_value value(boost::any(op.value), false);
-
-            const char * option;
-            if (token.contains(' '))
-                option = "query";
-            else
-                throw Exception(ErrorCodes::BAD_ARGUMENTS, "Positional option `{}` is not supported.", token);
-
-            if (!options.emplace(option, value).second)
-                throw Exception(ErrorCodes::BAD_ARGUMENTS, "Positional option `{}` is not supported.", token);
-        }
-    }
-
-    po::store(parsed, options);
-}
-
 void ClientApplicationBase::addMultiquery(std::string_view query, Arguments & common_arguments) const
 {
     common_arguments.emplace_back("--multiquery");
diff --git a/src/Client/ClientBaseOptimizedParts.cpp b/src/Client/ClientBaseOptimizedParts.cpp
index 421843a0e79..297b8e7ce51 100644
--- a/src/Client/ClientBaseOptimizedParts.cpp
+++ b/src/Client/ClientBaseOptimizedParts.cpp
@@ -1,4 +1,4 @@
-#include <Client/ClientBase.h>
+#include <Client/ClientApplicationBase.h>
 #include <Core/BaseSettingsProgramOptions.h>
 
 namespace DB
@@ -80,7 +80,7 @@ private:
 
 }
 
-void ClientBase::parseAndCheckOptions(OptionsDescription & options_description, po::variables_map & options, Arguments & arguments)
+void ClientApplicationBase::parseAndCheckOptions(OptionsDescription & options_description, po::variables_map & options, Arguments & arguments)
 {
     if (allow_repeated_settings)
         addProgramOptionsAsMultitokens(cmd_settings, options_description.main_description.value());

From 6f16ca02a74a81956b7524958c5f94eb7a2c7bf5 Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <nikitamikhaylov@clickhouse.com>
Date: Mon, 29 Jul 2024 16:14:31 +0000
Subject: [PATCH 1017/1488] Fixed Style Check

---
 src/Client/ClientApplicationBase.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/Client/ClientApplicationBase.cpp b/src/Client/ClientApplicationBase.cpp
index 4aa8b6c0bbe..9f133616d2e 100644
--- a/src/Client/ClientApplicationBase.cpp
+++ b/src/Client/ClientApplicationBase.cpp
@@ -31,7 +31,6 @@ namespace ErrorCodes
 {
     extern const int BAD_ARGUMENTS;
     extern const int CANNOT_SET_SIGNAL_HANDLER;
-    extern const int UNRECOGNIZED_ARGUMENTS;
 }
 
 static ClientInfo::QueryKind parseQueryKind(const String & query_kind)

From 75728ac56d83b85e476162a745686837cb194b73 Mon Sep 17 00:00:00 2001
From: Halersson Paris <142428374+halersson@users.noreply.github.com>
Date: Mon, 29 Jul 2024 14:42:58 -0300
Subject: [PATCH 1018/1488] Fix typo

---
 src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp b/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp
index e837d4d5e20..bc5e8292192 100644
--- a/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp
+++ b/src/Processors/Formats/Impl/ParquetBlockInputFormat.cpp
@@ -420,7 +420,7 @@ void ParquetBlockInputFormat::initializeIfNeeded()
     int num_row_groups = metadata->num_row_groups();
     row_group_batches.reserve(num_row_groups);
 
-    auto adative_chunk_size = [&](int row_group_idx) -> size_t
+    auto adaptive_chunk_size = [&](int row_group_idx) -> size_t
     {
         size_t total_size = 0;
         auto row_group_meta = metadata->RowGroup(row_group_idx);
@@ -457,7 +457,7 @@ void ParquetBlockInputFormat::initializeIfNeeded()
         row_group_batches.back().row_groups_idxs.push_back(row_group);
         row_group_batches.back().total_rows += metadata->RowGroup(row_group)->num_rows();
         row_group_batches.back().total_bytes_compressed += metadata->RowGroup(row_group)->total_compressed_size();
-        auto rows = adative_chunk_size(row_group);
+        auto rows = adaptive_chunk_size(row_group);
         row_group_batches.back().adaptive_chunk_size = rows ? rows : format_settings.parquet.max_block_size;
     }
 }

From 06b3185e57953c43531e4281f22bc03a8cd424e7 Mon Sep 17 00:00:00 2001
From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com>
Date: Mon, 29 Jul 2024 20:05:29 +0200
Subject: [PATCH 1019/1488] fixes

---
 src/Interpreters/InterpreterCreateQuery.cpp   |  14 +-
 .../test_restore_external_engines/test.py     | 133 ++++++++++++++----
 2 files changed, 111 insertions(+), 36 deletions(-)

diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp
index a5eb3a83365..6e689c59c09 100644
--- a/src/Interpreters/InterpreterCreateQuery.cpp
+++ b/src/Interpreters/InterpreterCreateQuery.cpp
@@ -1031,13 +1031,6 @@ void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const
                 /// Some part of storage definition (such as PARTITION BY) is specified, but ENGINE is not: just set default one.
                 setDefaultTableEngine(*to_engine, getContext()->getSettingsRef().default_table_engine.value);
             }
-            /// For external tables with restore_replace_external_engine_to_null setting we replace external engines to
-            /// Null table engine.
-            else if (getContext()->getSettingsRef().restore_replace_external_engines_to_null)
-            {
-                if (StorageFactory::instance().getStorageFeatures(create.storage->engine->name).source_access_type != AccessType::NONE)
-                    setNullTableEngine(*create.storage);
-            }
             return;
         }
     }
@@ -1050,6 +1043,13 @@ void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const
             /// Some part of storage definition (such as PARTITION BY) is specified, but ENGINE is not: just set default one.
             setDefaultTableEngine(*create.storage, getContext()->getSettingsRef().default_table_engine.value);
         }
+        /// For external tables with restore_replace_external_engine_to_null setting we replace external engines to
+        /// Null table engine.
+        else if (getContext()->getSettingsRef().restore_replace_external_engines_to_null)
+        {
+            if (StorageFactory::instance().getStorageFeatures(create.storage->engine->name).source_access_type != AccessType::NONE)
+                setNullTableEngine(*create.storage);
+        }
         return;
     }
 
diff --git a/tests/integration/test_restore_external_engines/test.py b/tests/integration/test_restore_external_engines/test.py
index eb88da6b61f..cf189f2a6ed 100644
--- a/tests/integration/test_restore_external_engines/test.py
+++ b/tests/integration/test_restore_external_engines/test.py
@@ -7,52 +7,75 @@ from helpers.cluster import ClickHouseCluster
 cluster = ClickHouseCluster(__file__)
 configs = ["configs/remote_servers.xml", "configs/backups_disk.xml"]
 
-node1 = cluster.add_instance("replica1", with_zookeeper=True, with_mysql8=True, main_configs=configs, external_dirs=["/backups/"])
-node2 = cluster.add_instance("replica2", with_zookeeper=True, with_mysql8=True, main_configs=configs, external_dirs=["/backups/"])
-node3 = cluster.add_instance("replica3", with_zookeeper=True, with_mysql8=True, main_configs=configs, external_dirs=["/backups/"])
+node1 = cluster.add_instance(
+    "replica1",
+    with_zookeeper=True,
+    with_mysql8=True,
+    main_configs=configs,
+    external_dirs=["/backups/"],
+)
+node2 = cluster.add_instance(
+    "replica2",
+    with_zookeeper=True,
+    with_mysql8=True,
+    main_configs=configs,
+    external_dirs=["/backups/"],
+)
+node3 = cluster.add_instance(
+    "replica3",
+    with_zookeeper=True,
+    with_mysql8=True,
+    main_configs=configs,
+    external_dirs=["/backups/"],
+)
 nodes = [node1, node2, node3]
 
 backup_id_counter = 0
 
+
 def new_backup_name():
     global backup_id_counter
     backup_id_counter += 1
     return f"Disk('backups', '{backup_id_counter}/')"
 
+
 def cleanup_nodes(nodes, dbname):
     for node in nodes:
         node.query(f"DROP DATABASE IF EXISTS {dbname} SYNC")
 
+
 def fill_nodes(nodes, dbname):
     cleanup_nodes(nodes, dbname)
     for node in nodes:
-        node.query(f"CREATE DATABASE {dbname} ENGINE = Replicated('/clickhouse/databases/{dbname}', 'default', '{node.name}')")
+        node.query(
+            f"CREATE DATABASE {dbname} ENGINE = Replicated('/clickhouse/databases/{dbname}', 'default', '{node.name}')"
+        )
+
 
 def drop_mysql_table(conn, tableName):
     with conn.cursor() as cursor:
         cursor.execute(f"DROP TABLE IF EXISTS `clickhouse`.`{tableName}`")
 
+
 def get_mysql_conn(cluster):
     conn = pymysql.connect(
-        user="root", password="clickhouse", host=cluster.mysql8_ip, port=cluster.mysql8_port
+        user="root",
+        password="clickhouse",
+        host=cluster.mysql8_ip,
+        port=cluster.mysql8_port,
     )
     return conn
 
+
 def fill_tables(cluster, dbname):
     fill_nodes(nodes, dbname)
 
     conn = get_mysql_conn(cluster)
 
     with conn.cursor() as cursor:
-        cursor.execute(
-            "DROP DATABASE IF EXISTS clickhouse"
-        )
-        cursor.execute(
-            "CREATE DATABASE clickhouse"
-        )
-        cursor.execute(
-            "DROP TABLE IF EXISTS clickhouse.inference_table"
-        )
+        cursor.execute("DROP DATABASE IF EXISTS clickhouse")
+        cursor.execute("CREATE DATABASE clickhouse")
+        cursor.execute("DROP TABLE IF EXISTS clickhouse.inference_table")
         cursor.execute(
             "CREATE TABLE clickhouse.inference_table (id INT PRIMARY KEY, data BINARY(16) NOT NULL)"
         )
@@ -66,17 +89,30 @@ def fill_tables(cluster, dbname):
     node1.query(
         f"CREATE TABLE {dbname}.mysql_schema_inference_engine ENGINE=MySQL({parameters})"
     )
-    node1.query(f"CREATE TABLE {dbname}.mysql_schema_inference_function AS mysql({parameters})")
+    node1.query(
+        f"CREATE TABLE {dbname}.mysql_schema_inference_function AS mysql({parameters})"
+    )
 
     node1.query(f"CREATE TABLE {dbname}.merge_tree (id UInt64, b String) ORDER BY id")
     node1.query(f"INSERT INTO {dbname}.merge_tree VALUES (100, 'abc')")
 
     expected = "id\tInt32\t\t\t\t\t\ndata\tFixedString(16)\t\t\t\t\t\n"
-    assert node1.query(f"DESCRIBE TABLE {dbname}.mysql_schema_inference_engine") == expected
-    assert node1.query(f"DESCRIBE TABLE {dbname}.mysql_schema_inference_function") == expected
+    assert (
+        node1.query(f"DESCRIBE TABLE {dbname}.mysql_schema_inference_engine")
+        == expected
+    )
+    assert (
+        node1.query(f"DESCRIBE TABLE {dbname}.mysql_schema_inference_function")
+        == expected
+    )
     assert node1.query(f"SELECT id FROM mysql({parameters})") == "100\n"
-    assert node1.query(f"SELECT id FROM {dbname}.mysql_schema_inference_engine") == "100\n"
-    assert node1.query(f"SELECT id FROM {dbname}.mysql_schema_inference_function") == "100\n"
+    assert (
+        node1.query(f"SELECT id FROM {dbname}.mysql_schema_inference_engine") == "100\n"
+    )
+    assert (
+        node1.query(f"SELECT id FROM {dbname}.mysql_schema_inference_function")
+        == "100\n"
+    )
     assert node1.query(f"SELECT id FROM {dbname}.merge_tree") == "100\n"
 
 
@@ -92,6 +128,7 @@ def start_cluster():
     finally:
         cluster.shutdown()
 
+
 def test_restore_table(start_cluster):
     fill_tables(cluster, "replicated")
     backup_name = new_backup_name()
@@ -107,12 +144,26 @@ def test_restore_table(start_cluster):
     assert node3.query("EXISTS replicated.mysql_schema_inference_engine") == "0\n"
     assert node3.query("EXISTS replicated.mysql_schema_inference_function") == "0\n"
 
-    node3.query(f"RESTORE DATABASE replicated FROM {backup_name} SETTINGS allow_different_database_def=true")
+    node3.query(
+        f"RESTORE DATABASE replicated FROM {backup_name} SETTINGS allow_different_database_def=true"
+    )
     node1.query(f"SYSTEM SYNC DATABASE REPLICA replicated")
 
-    assert node1.query("SELECT count(), sum(id) FROM replicated.mysql_schema_inference_engine") == "1\t100\n"
-    assert node1.query("SELECT count(), sum(id) FROM replicated.mysql_schema_inference_function") == "1\t100\n"
-    assert node1.query("SELECT count(), sum(id) FROM replicated.merge_tree") == "1\t100\n"
+    assert (
+        node1.query(
+            "SELECT count(), sum(id) FROM replicated.mysql_schema_inference_engine"
+        )
+        == "1\t100\n"
+    )
+    assert (
+        node1.query(
+            "SELECT count(), sum(id) FROM replicated.mysql_schema_inference_function"
+        )
+        == "1\t100\n"
+    )
+    assert (
+        node1.query("SELECT count(), sum(id) FROM replicated.merge_tree") == "1\t100\n"
+    )
     cleanup_nodes(nodes, "replicated")
 
 
@@ -132,12 +183,36 @@ def test_restore_table_null(start_cluster):
     assert node3.query("EXISTS replicated2.mysql_schema_inference_engine") == "0\n"
     assert node3.query("EXISTS replicated2.mysql_schema_inference_function") == "0\n"
 
-    node3.query(f"RESTORE DATABASE replicated2 FROM {backup_name} SETTINGS allow_different_database_def=1, allow_different_table_def=1 SETTINGS restore_replace_external_engines_to_null=1, restore_replace_external_table_functions_to_null=1")
+    node3.query(
+        f"RESTORE DATABASE replicated2 FROM {backup_name} SETTINGS allow_different_database_def=1, allow_different_table_def=1 SETTINGS restore_replace_external_engines_to_null=1, restore_replace_external_table_functions_to_null=1"
+    )
     node1.query(f"SYSTEM SYNC DATABASE REPLICA replicated2")
 
-    assert node1.query("SELECT count(), sum(id) FROM replicated2.mysql_schema_inference_engine") == "0\t0\n"
-    assert node1.query("SELECT count(), sum(id) FROM replicated2.mysql_schema_inference_function") == "0\t0\n"
-    assert node1.query("SELECT count(), sum(id) FROM replicated2.merge_tree") == "1\t100\n"
-    assert node1.query("SELECT engine FROM system.tables where database = 'replicated2' and name like '%mysql%'") == "Null\nNull\n"
-    assert node1.query("SELECT engine FROM system.tables where database = 'replicated2' and name like '%merge_tree%'") == "MergeTree\n"
+    assert (
+        node1.query(
+            "SELECT count(), sum(id) FROM replicated2.mysql_schema_inference_engine"
+        )
+        == "0\t0\n"
+    )
+    assert (
+        node1.query(
+            "SELECT count(), sum(id) FROM replicated2.mysql_schema_inference_function"
+        )
+        == "0\t0\n"
+    )
+    assert (
+        node1.query("SELECT count(), sum(id) FROM replicated2.merge_tree") == "1\t100\n"
+    )
+    assert (
+        node1.query(
+            "SELECT engine FROM system.tables where database = 'replicated2' and name like '%mysql%'"
+        )
+        == "Null\nNull\n"
+    )
+    assert (
+        node1.query(
+            "SELECT engine FROM system.tables where database = 'replicated2' and name like '%merge_tree%'"
+        )
+        == "MergeTree\n"
+    )
     cleanup_nodes(nodes, "replicated2")

From cb056cf3a5080cbff61f6efd070733ae2061d5b8 Mon Sep 17 00:00:00 2001
From: Peter Nguyen <petern0408@gmail.com>
Date: Mon, 29 Jul 2024 12:05:44 -0700
Subject: [PATCH 1020/1488] Add camelCase aliases for percentRank() and
 denseRank() for percent_rank() and dense_rank()

---
 src/Processors/Transforms/WindowTransform.cpp | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/Processors/Transforms/WindowTransform.cpp b/src/Processors/Transforms/WindowTransform.cpp
index 2b255c5120e..a1b46c8e36c 100644
--- a/src/Processors/Transforms/WindowTransform.cpp
+++ b/src/Processors/Transforms/WindowTransform.cpp
@@ -2721,20 +2721,24 @@ void registerWindowFunctions(AggregateFunctionFactory & factory)
                 parameters);
         }, properties}, AggregateFunctionFactory::Case::Insensitive);
 
-    factory.registerFunction("dense_rank", {[](const std::string & name,
+    factory.registerFunction("denseRank", {[](const std::string & name,
             const DataTypes & argument_types, const Array & parameters, const Settings *)
         {
             return std::make_shared<WindowFunctionDenseRank>(name, argument_types,
                 parameters);
         }, properties}, AggregateFunctionFactory::Case::Insensitive);
 
-    factory.registerFunction("percent_rank", {[](const std::string & name,
+    factory.registerAlias("dense_rank", "denseRank", AggregateFunctionFactory::Case::Sensitive);
+
+    factory.registerFunction("percentRank", {[](const std::string & name,
             const DataTypes & argument_types, const Array & parameters, const Settings *)
         {
             return std::make_shared<WindowFunctionPercentRank>(name, argument_types,
                 parameters);
         }, properties}, AggregateFunctionFactory::Case::Insensitive);
 
+    factory.registerAlias("percent_rank", "percentRank", AggregateFunctionFactory::Case::Sensitive);
+
     factory.registerFunction("row_number", {[](const std::string & name,
             const DataTypes & argument_types, const Array & parameters, const Settings *)
         {

From 9811a2e71b825a55c376edfb38303c817493cd9e Mon Sep 17 00:00:00 2001
From: Peter Nguyen <petern0408@gmail.com>
Date: Sun, 28 Jul 2024 17:32:32 -0700
Subject: [PATCH 1021/1488] Add test 03213_denseRank_percentRank_alias

---
 ...3213_denseRank_percentRank_alias.reference | 45 ++++++++++++++
 .../03213_denseRank_percentRank_alias.sql     | 59 +++++++++++++++++++
 2 files changed, 104 insertions(+)
 create mode 100644 tests/queries/0_stateless/03213_denseRank_percentRank_alias.reference
 create mode 100644 tests/queries/0_stateless/03213_denseRank_percentRank_alias.sql

diff --git a/tests/queries/0_stateless/03213_denseRank_percentRank_alias.reference b/tests/queries/0_stateless/03213_denseRank_percentRank_alias.reference
new file mode 100644
index 00000000000..b49e179df68
--- /dev/null
+++ b/tests/queries/0_stateless/03213_denseRank_percentRank_alias.reference
@@ -0,0 +1,45 @@
+---- denseRank() ----
+0	0	0	1	1	1	1
+3	0	0	2	2	2	2
+1	0	1	3	3	3	3
+4	0	1	4	4	4	4
+2	0	2	5	5	5	5
+6	1	0	1	1	1	1
+9	1	0	2	2	2	2
+7	1	1	3	3	3	3
+5	1	2	4	4	4	4
+8	1	2	5	5	5	5
+12	2	0	1	1	1	1
+10	2	1	2	2	2	2
+13	2	1	3	3	3	3
+11	2	2	4	4	4	4
+14	2	2	5	5	5	5
+15	3	0	1	1	1	1
+18	3	0	2	2	2	2
+16	3	1	3	3	3	3
+19	3	1	4	4	4	4
+17	3	2	5	5	5	5
+21	4	0	1	1	1	1
+24	4	0	2	2	2	2
+22	4	1	3	3	3	3
+20	4	2	4	4	4	4
+23	4	2	5	5	5	5
+27	5	0	1	1	1	1
+25	5	1	2	2	2	2
+28	5	1	3	3	3	3
+26	5	2	4	4	4	4
+29	5	2	5	5	5	5
+30	6	0	1	1	1	1
+---- percentRank() ----
+Lenovo Thinkpad	Laptop	700	1	0
+Sony VAIO	Laptop	700	1	0
+Dell Vostro	Laptop	800	3	0.6666666666666666
+HP Elite	Laptop	1200	4	1
+Microsoft Lumia	Smartphone	200	1	0
+HTC One	Smartphone	400	2	0.3333333333333333
+Nexus	Smartphone	500	3	0.6666666666666666
+iPhone	Smartphone	900	4	1
+Kindle Fire	Tablet	150	1	0
+Samsung Galaxy Tab	Tablet	200	2	0.5
+iPad	Tablet	700	3	1
+Others	Unknow	200	1	0
diff --git a/tests/queries/0_stateless/03213_denseRank_percentRank_alias.sql b/tests/queries/0_stateless/03213_denseRank_percentRank_alias.sql
new file mode 100644
index 00000000000..ff841294eb1
--- /dev/null
+++ b/tests/queries/0_stateless/03213_denseRank_percentRank_alias.sql
@@ -0,0 +1,59 @@
+-- https://github.com/ClickHouse/ClickHouse/issues/67042
+-- Reference generated using percent_rank() and dense_rank()
+
+-- From ClickHouse/tests/queries/0_stateless/01591_window_functions.sql  (for deterministic query)
+SELECT '---- denseRank() ----';
+select number, p, o,
+    count(*) over w,
+    rank() over w,
+    denseRank() over w,
+    row_number() over w
+from (select number, intDiv(number, 5) p, mod(number, 3) o
+    from numbers(31) order by o, number) t
+window w as (partition by p order by o, number)
+order by p, o, number
+settings max_block_size = 2;
+
+-- Modifed from ClickHouse/tests/queries/0_stateless/01592_window_functions.sql (for deterministic query)
+SELECT '---- percentRank() ----';
+
+drop table if exists product_groups;
+drop table if exists products;
+
+CREATE TABLE product_groups (
+	group_id Int64,
+	group_name String
+) Engine = Memory;
+
+CREATE TABLE products (
+	product_id Int64,
+	product_name String,
+	price DECIMAL(11, 2),
+	group_id Int64
+) Engine = Memory;
+
+INSERT INTO product_groups  VALUES	(1, 'Smartphone'),(2, 'Laptop'),(3, 'Tablet');
+INSERT INTO products (product_id,product_name, group_id,price) VALUES (1, 'Microsoft Lumia', 1, 200), (2, 'HTC One', 1, 400), (3, 'Nexus', 1, 500), (4, 'iPhone', 1, 900),(5, 'HP Elite', 2, 1200),(6, 'Lenovo Thinkpad', 2, 700),(7, 'Sony VAIO', 2, 700),(8, 'Dell Vostro', 2, 800),(9, 'iPad', 3, 700),(10, 'Kindle Fire', 3, 150),(11, 'Samsung Galaxy Tab', 3, 200);
+INSERT INTO product_groups  VALUES	(4, 'Unknow');
+INSERT INTO products (product_id,product_name, group_id,price) VALUES (12, 'Others', 4, 200);
+
+
+SELECT *
+FROM
+(
+    SELECT
+        product_name,
+        group_name,
+        price,
+        rank() OVER (PARTITION BY group_name ORDER BY price ASC) AS rank,
+        percentRank() OVER (PARTITION BY group_name ORDER BY price ASC) AS percent
+    FROM products
+    INNER JOIN product_groups USING (group_id)
+) AS t
+ORDER BY
+    group_name ASC,
+    price ASC,
+    product_name ASC;
+
+drop table product_groups;
+drop table products;

From fea03cf46ff29aa398b08d86ae77361fe85d7c40 Mon Sep 17 00:00:00 2001
From: Max K <maxkaynov@gmail.com>
Date: Mon, 29 Jul 2024 21:07:24 +0200
Subject: [PATCH 1022/1488] Build results fix

---
 tests/ci/ci.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/ci/ci.py b/tests/ci/ci.py
index e30062c32ff..935fe472e50 100644
--- a/tests/ci/ci.py
+++ b/tests/ci/ci.py
@@ -766,7 +766,9 @@ def _upload_build_artifacts(
         int(job_report.duration),
         GITHUB_JOB_API_URL(),
         head_ref=pr_info.head_ref,
-        pr_number=pr_info.number,
+        # PRInfo fetches pr number for release branches as well - set pr_number to 0 for release
+        #   so that build results are not mistakenly treated as feature branch builds
+        pr_number=pr_info.number if pr_info.is_pr else 0,
     )
     report_url = ci_cache.upload_build_report(build_result)
     print(f"Report file has been uploaded to [{report_url}]")

From 3df2d88cf13ad552058a6958630741d7cdab9d3c Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 29 Jul 2024 21:09:11 +0200
Subject: [PATCH 1023/1488] Update CHANGELOG.md

---
 CHANGELOG.md | 1 -
 1 file changed, 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 07b37835dda..620b7c99bac 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -45,7 +45,6 @@
 * Add support for `cluster_for_parallel_replicas` when using custom key parallel replicas. It allows you to use parallel replicas with custom key with MergeTree tables. [#65453](https://github.com/ClickHouse/ClickHouse/pull/65453) ([Antonio Andelic](https://github.com/antonio2368)).
 
 #### Performance Improvement
-* Enable `optimize_functions_to_subcolumns` by default. [#58661](https://github.com/ClickHouse/ClickHouse/pull/58661) ([Anton Popov](https://github.com/CurtizJ)).
 * Replace int to string algorithm with a faster one (from a modified amdn/itoa to a modified jeaiii/itoa). [#61661](https://github.com/ClickHouse/ClickHouse/pull/61661) ([Raúl Marín](https://github.com/Algunenano)).
 * Sizes of hash tables created by join (`parallel_hash` algorithm) is collected and cached now. This information will be used to preallocate space in hash tables for subsequent query executions and save time on hash table resizes. [#64553](https://github.com/ClickHouse/ClickHouse/pull/64553) ([Nikita Taranov](https://github.com/nickitat)).
 * Optimized queries with `ORDER BY` primary key and `WHERE` that have a condition with high selectivity by using of buffering. It is controlled by setting `read_in_order_use_buffering` (enabled by default) and can increase memory usage of query. [#64607](https://github.com/ClickHouse/ClickHouse/pull/64607) ([Anton Popov](https://github.com/CurtizJ)).

From 756bde1158c4b3e6e65d324436291d53b9e25fbb Mon Sep 17 00:00:00 2001
From: Nikita Taranov <nikita.taranov@clickhouse.com>
Date: Mon, 29 Jul 2024 20:27:15 +0100
Subject: [PATCH 1024/1488] rm file

---
 tests/performance/views_max_insert_threads.xml | 11 -----------
 1 file changed, 11 deletions(-)
 delete mode 100644 tests/performance/views_max_insert_threads.xml

diff --git a/tests/performance/views_max_insert_threads.xml b/tests/performance/views_max_insert_threads.xml
deleted file mode 100644
index 473bcd02ab8..00000000000
--- a/tests/performance/views_max_insert_threads.xml
+++ /dev/null
@@ -1,11 +0,0 @@
-<!-- https://github.com/ClickHouse/ClickHouse/issues/37900 -->
-<!-- the test degraded significantly after #66144 to the point when it could be only thrown away -->
-<!-- <test> -->
-<!--     <create_query>create table views_max_insert_threads_null (a UInt64) Engine = Null</create_query> -->
-<!--     <create_query>create materialized view views_max_insert_threads_mv Engine = Null AS select now() as ts, max(a) from views_max_insert_threads_null group by ts</create_query> -->
-<!---->
-<!--     <query>insert into views_max_insert_threads_null select * from numbers_mt(3000000000) settings max_threads = 16, max_insert_threads=16</query> -->
-<!---->
-<!--     <drop_query>drop table if exists views_max_insert_threads_null</drop_query> -->
-<!--     <drop_query>drop table if exists views_max_insert_threads_mv</drop_query> -->
-<!-- </test> -->

From 2308a362a0aca716e2e50d3eb5283bdfd575e023 Mon Sep 17 00:00:00 2001
From: Michael Kolupaev <michael.kolupaev@clickhouse.com>
Date: Mon, 29 Jul 2024 19:31:46 +0000
Subject: [PATCH 1025/1488] Disable 02932_refreshable_materialized_views

---
 .../02932_refreshable_materialized_views.sh   | 304 +-----------------
 1 file changed, 1 insertion(+), 303 deletions(-)

diff --git a/tests/queries/0_stateless/02932_refreshable_materialized_views.sh b/tests/queries/0_stateless/02932_refreshable_materialized_views.sh
index 9081035579d..6df3c391ddb 100755
--- a/tests/queries/0_stateless/02932_refreshable_materialized_views.sh
+++ b/tests/queries/0_stateless/02932_refreshable_materialized_views.sh
@@ -1,306 +1,4 @@
 #!/usr/bin/env bash
 # Tags: atomic-database
 
-CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# shellcheck source=../shell_config.sh
-. "$CUR_DIR"/../shell_config.sh
-
-# Set session timezone to UTC to make all DateTime formatting and parsing use UTC, because refresh
-# scheduling is done in UTC.
-CLICKHOUSE_CLIENT="`echo "$CLICKHOUSE_CLIENT" | sed 's/--session_timezone[= ][^ ]*//g'`"
-CLICKHOUSE_CLIENT="`echo "$CLICKHOUSE_CLIENT --allow_experimental_refreshable_materialized_view=1 --session_timezone Etc/UTC"`"
-
-$CLICKHOUSE_CLIENT -nq "create view refreshes as select * from system.view_refreshes where database = '$CLICKHOUSE_DATABASE' order by view"
-
-
-# Basic refreshing.
-$CLICKHOUSE_CLIENT -nq "
-    create materialized view a
-        refresh after 2 second
-        engine Memory
-        empty
-        as select number as x from numbers(2) union all select rand64() as x"
-$CLICKHOUSE_CLIENT -nq "select '<1: created view>', view, remaining_dependencies, exception, last_refresh_result in ('Unknown', 'Finished') from refreshes";
-$CLICKHOUSE_CLIENT -nq "show create a"
-# Wait for any refresh. (xargs trims the string and turns \t and \n into spaces)
-while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result from refreshes -- $LINENO" | xargs`" == 'Unknown' ]
-do
-    sleep 0.1
-done
-start_time="`$CLICKHOUSE_CLIENT -nq "select reinterpret(now64(), 'Int64')"`"
-# Check table contents.
-$CLICKHOUSE_CLIENT -nq "select '<2: refreshed>', count(), sum(x=0), sum(x=1) from a"
-# Wait for table contents to change.
-res1="`$CLICKHOUSE_CLIENT -nq 'select * from a order by x format Values'`"
-while :
-do
-    res2="`$CLICKHOUSE_CLIENT -nq 'select * from a order by x format Values -- $LINENO'`"
-    [ "$res2" == "$res1" ] || break
-    sleep 0.1
-done
-# Wait for another change.
-while :
-do
-    res3="`$CLICKHOUSE_CLIENT -nq 'select * from a order by x format Values -- $LINENO'`"
-    [ "$res3" == "$res2" ] || break
-    sleep 0.1
-done
-# Check that the two changes were at least 1 second apart, in particular that we're not refreshing
-# like crazy. This is potentially flaky, but we need at least one test that uses non-mocked timer
-# to make sure the clock+timer code works at all. If it turns out flaky, increase refresh period above.
-$CLICKHOUSE_CLIENT -nq "
-    select '<3: time difference at least>', min2(reinterpret(now64(), 'Int64') - $start_time, 1000);
-    select '<4: next refresh in>', next_refresh_time-last_refresh_time from refreshes;"
-
-# Create a source table from which views will read.
-$CLICKHOUSE_CLIENT -nq "
-    create table src (x Int8) engine Memory as select 1"
-
-# Switch to fake clock, change refresh schedule, change query.
-$CLICKHOUSE_CLIENT -nq "
-    system test view a set fake time '2050-01-01 00:00:01';"
-while [ "`$CLICKHOUSE_CLIENT -nq "select status, last_refresh_time, next_refresh_time from refreshes -- $LINENO" | xargs`" != 'Scheduled 2050-01-01 00:00:01 2050-01-01 00:00:03' ]
-do
-    sleep 0.1
-done
-$CLICKHOUSE_CLIENT -nq "
-    alter table a modify refresh every 2 year;
-    alter table a modify query select x*2 as x from src;
-    select '<4.5: altered>', status, last_refresh_result, next_refresh_time from refreshes;
-    show create a;"
-# Advance time to trigger the refresh.
-$CLICKHOUSE_CLIENT -nq "
-    select '<5: no refresh>', count() from a;
-    system test view a set fake time '2052-02-03 04:05:06';"
-while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_time from refreshes -- $LINENO" | xargs`" != '2052-02-03 04:05:06' ]
-do
-    sleep 0.1
-done
-$CLICKHOUSE_CLIENT -nq "
-    select '<6: refreshed>', * from a;
-    select '<7: refreshed>', status, last_refresh_result, next_refresh_time from refreshes;"
-
-# Create a dependent view, refresh it once.
-$CLICKHOUSE_CLIENT -nq "
-    create materialized view b refresh every 2 year depends on a (y Int32) engine MergeTree order by y empty as select x*10 as y from a;
-    show create b;
-    system test view b set fake time '2052-11-11 11:11:11';
-    system refresh view b;"
-while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_time from refreshes where view = 'b' -- $LINENO" | xargs`" != '2052-11-11 11:11:11' ]
-do
-    sleep 0.1
-done
-# Next refresh shouldn't start until the dependency refreshes.
-$CLICKHOUSE_CLIENT -nq "
-    select '<8: refreshed>', * from b;
-    select '<9: refreshed>', view, status, last_refresh_result, next_refresh_time from refreshes;
-    system test view b set fake time '2054-01-24 23:22:21';"
-while [ "`$CLICKHOUSE_CLIENT -nq "select status, next_refresh_time from refreshes where view = 'b' -- $LINENO" | xargs`" != 'WaitingForDependencies 2054-01-01 00:00:00' ]
-do
-    sleep 0.1
-done
-# Update source table (by dropping and re-creating it - to test that tables are looked up by name
-# rather than uuid), kick off refresh of the dependency.
-$CLICKHOUSE_CLIENT -nq "
-    select '<10: waiting>', view, status, remaining_dependencies, next_refresh_time from refreshes;
-    drop table src;
-    create table src (x Int16) engine Memory as select 2;
-    system test view a set fake time '2054-01-01 00:00:01';"
-while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes where view = 'b' -- $LINENO" | xargs`" != 'Scheduled' ]
-do
-    sleep 0.1
-done
-# Both tables should've refreshed.
-$CLICKHOUSE_CLIENT -nq "
-    select '<11: chain-refreshed a>', * from a;
-    select '<12: chain-refreshed b>', * from b;
-    select '<13: chain-refreshed>', view, status, remaining_dependencies, last_refresh_result, last_refresh_time, next_refresh_time, exception from refreshes;"
-
-# Make the dependent table run ahead by one refresh cycle, make sure it waits for the dependency to
-# catch up to the same cycle.
-$CLICKHOUSE_CLIENT -nq "
-    system test view b set fake time '2059-01-01 00:00:00';
-    system refresh view b;"
-while [ "`$CLICKHOUSE_CLIENT -nq "select next_refresh_time from refreshes where view = 'b' -- $LINENO" | xargs`" != '2060-01-01 00:00:00' ]
-do
-    sleep 0.1
-done
-$CLICKHOUSE_CLIENT -nq "
-    system test view b set fake time '2061-01-01 00:00:00';
-    system test view a set fake time '2057-01-01 00:00:00';"
-while [ "`$CLICKHOUSE_CLIENT -nq "select status, next_refresh_time from refreshes -- $LINENO" | xargs`" != 'Scheduled 2058-01-01 00:00:00 WaitingForDependencies 2060-01-01 00:00:00' ]
-do
-    sleep 0.1
-done
-
-$CLICKHOUSE_CLIENT -nq "
-    select '<14: waiting for next cycle>', view, status, remaining_dependencies, next_refresh_time from refreshes;
-    truncate src;
-    insert into src values (3);
-    system test view a set fake time '2060-02-02 02:02:02';"
-while [ "`$CLICKHOUSE_CLIENT -nq "select next_refresh_time from refreshes where view = 'b' -- $LINENO" | xargs`" != '2062-01-01 00:00:00' ]
-do
-    sleep 0.1
-done
-$CLICKHOUSE_CLIENT -nq "
-    select '<15: chain-refreshed a>', * from a;
-    select '<16: chain-refreshed b>', * from b;
-    select '<17: chain-refreshed>', view, status, next_refresh_time from refreshes;"
-
-# Get to WaitingForDependencies state and remove the depencency.
-$CLICKHOUSE_CLIENT -nq "
-    system test view b set fake time '2062-03-03 03:03:03'"
-while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes where view = 'b' -- $LINENO" | xargs`" != 'WaitingForDependencies' ]
-do
-    sleep 0.1
-done
-$CLICKHOUSE_CLIENT -nq "
-    alter table b modify refresh every 2 year"
-while [ "`$CLICKHOUSE_CLIENT -nq "select status, last_refresh_time from refreshes where view = 'b' -- $LINENO" | xargs`" != 'Scheduled 2062-03-03 03:03:03' ]
-do
-    sleep 0.1
-done
-$CLICKHOUSE_CLIENT -nq "
-    select '<18: removed dependency>', view, status, remaining_dependencies, last_refresh_time,next_refresh_time, refresh_count from refreshes where view = 'b';
-    show create b;"
-
-# Select from a table that doesn't exist, get an exception.
-$CLICKHOUSE_CLIENT -nq "
-    drop table a;
-    drop table b;
-    create materialized view c refresh every 1 second (x Int64) engine Memory empty as select * from src;
-    drop table src;"
-while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result from refreshes where view = 'c' -- $LINENO" | xargs`" != 'Exception' ]
-do
-    sleep 0.1
-done
-# Check exception, create src, expect successful refresh.
-$CLICKHOUSE_CLIENT -nq "
-    select '<19: exception>', exception ilike '%UNKNOWN_TABLE%' ? '1' : exception from refreshes where view = 'c';
-    create table src (x Int64) engine Memory as select 1;
-    system refresh view c;"
-while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result from refreshes -- $LINENO" | xargs`" != 'Finished' ]
-do
-    sleep 0.1
-done
-# Rename table.
-$CLICKHOUSE_CLIENT -nq "
-    select '<20: unexception>', * from c;
-    rename table c to d;
-    select '<21: rename>', * from d;
-    select '<22: rename>', view, last_refresh_result from refreshes;"
-
-# Do various things during a refresh.
-# First make a nonempty view.
-$CLICKHOUSE_CLIENT -nq "
-    drop table d;
-    truncate src;
-    insert into src values (1)
-    create materialized view e refresh every 1 second (x Int64) engine MergeTree order by x empty as select x + sleepEachRow(1) as x from src settings max_block_size = 1;"
-while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result from refreshes -- $LINENO" | xargs`" != 'Finished' ]
-do
-    sleep 0.1
-done
-# Stop refreshes.
-$CLICKHOUSE_CLIENT -nq "
-    select '<23: simple refresh>', * from e;
-    system stop view e;"
-while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes -- $LINENO" | xargs`" != 'Disabled' ]
-do
-    sleep 0.1
-done
-# Make refreshes slow, wait for a slow refresh to start. (We stopped refreshes first to make sure
-# we wait for a slow refresh, not a previous fast one.)
-$CLICKHOUSE_CLIENT -nq "
-    insert into src select * from numbers(1000) settings max_block_size=1;
-    system start view e;"
-while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes -- $LINENO" | xargs`" != 'Running' ]
-do
-    sleep 0.1
-done
-# Rename.
-$CLICKHOUSE_CLIENT -nq "
-    rename table e to f;
-    select '<24: rename during refresh>', * from f;
-    select '<25: rename during refresh>', view, status from refreshes where view = 'f';
-    alter table f modify refresh after 10 year;"
-
-# Cancel.
-$CLICKHOUSE_CLIENT -nq "
-    system cancel view f;"
-while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result from refreshes where view = 'f' -- $LINENO" | xargs`" != 'Cancelled' ]
-do
-    sleep 0.1
-done
-
-while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes where view = 'f' -- $LINENO" | xargs`" = 'Running' ]
-do
-    sleep 0.1
-done
-
-# Check that another refresh doesn't immediately start after the cancelled one.
-$CLICKHOUSE_CLIENT -nq "
-    select '<27: cancelled>', view, status from refreshes where view = 'f';
-    system refresh view f;"
-while [ "`$CLICKHOUSE_CLIENT -nq "select status from refreshes where view = 'f' -- $LINENO" | xargs`" != 'Running' ]
-do
-    sleep 0.1
-done
-# Drop.
-$CLICKHOUSE_CLIENT -nq "
-    drop table f;
-    select '<28: drop during refresh>', view, status from refreshes;"
-
-# Try OFFSET and RANDOMIZE FOR.
-$CLICKHOUSE_CLIENT -nq "
-    create materialized view g refresh every 1 week offset 3 day 4 hour randomize for 4 day 1 hour (x Int64) engine Memory empty as select 42;
-    show create g;
-    system test view g set fake time '2050-02-03 15:30:13';"
-while [ "`$CLICKHOUSE_CLIENT -nq "select next_refresh_time > '2049-01-01' from refreshes -- $LINENO" | xargs`" != '1' ]
-do
-    sleep 0.1
-done
-$CLICKHOUSE_CLIENT -nq "
-    with '2050-02-10 04:00:00'::DateTime as expected
-    select '<29: randomize>', abs(next_refresh_time::Int64 - expected::Int64) <= 3600*(24*4+1), next_refresh_time != expected from refreshes;"
-
-# Send data 'TO' an existing table.
-$CLICKHOUSE_CLIENT -nq "
-    drop table g;
-    create table dest (x Int64) engine MergeTree order by x;
-    truncate src;
-    insert into src values (1);
-    create materialized view h refresh every 1 second to dest empty as select x*10 as x from src;
-    show create h;"
-while [ "`$CLICKHOUSE_CLIENT -nq "select last_refresh_result from refreshes -- $LINENO" | xargs`" != 'Finished' ]
-do
-    sleep 0.1
-done
-$CLICKHOUSE_CLIENT -nq "
-    select '<30: to existing table>', * from dest;
-    insert into src values (2);"
-while [ "`$CLICKHOUSE_CLIENT -nq "select count() from dest -- $LINENO" | xargs`" != '2' ]
-do
-    sleep 0.1
-done
-$CLICKHOUSE_CLIENT -nq "
-    select '<31: to existing table>', * from dest;
-    drop table dest;
-    drop table src;
-    drop table h;"
-
-# EMPTY
-$CLICKHOUSE_CLIENT -nq "
-    create materialized view i refresh after 1 year engine Memory empty as select number as x from numbers(2);
-    create materialized view j refresh after 1 year engine Memory as select number as x from numbers(2)"
-while [ "`$CLICKHOUSE_CLIENT -nq "select sum(last_success_time is null) from refreshes -- $LINENO" | xargs`" == '2' ]
-do
-    sleep 0.1
-done
-$CLICKHOUSE_CLIENT -nq "
-    select '<32: empty>', view, status, last_refresh_result from refreshes order by view;
-    drop table i;
-    drop table j"
-
-$CLICKHOUSE_CLIENT -nq "
-    drop table refreshes;"
+# TODO: Re-add this test in https://github.com/ClickHouse/ClickHouse/pull/58934

From f3d5859c8bd5d1fb43f0d636d6cff3062e4ca267 Mon Sep 17 00:00:00 2001
From: avogar <avogar@clickhouse.com>
Date: Mon, 29 Jul 2024 19:35:37 +0000
Subject: [PATCH 1026/1488] Fix new test

---
 ...dynamic_read_subcolumns_small.reference.j2 | 192 +++++++++---------
 ...03036_dynamic_read_subcolumns_small.sql.j2 |  16 +-
 2 files changed, 104 insertions(+), 104 deletions(-)

diff --git a/tests/queries/0_stateless/03036_dynamic_read_subcolumns_small.reference.j2 b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_small.reference.j2
index be3f4e53990..d6add681f51 100644
--- a/tests/queries/0_stateless/03036_dynamic_read_subcolumns_small.reference.j2
+++ b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_small.reference.j2
@@ -25,26 +25,26 @@ UInt64
 7	7	\N	[]
 8	8	\N	[]
 9	9	\N	[]
-str_10	\N	str_10	[]
 [[0]]	\N	\N	[]
-str_11	\N	str_11	[]
+str_10	\N	str_10	[]
 [[0,1]]	\N	\N	[]
-str_12	\N	str_12	[]
+str_11	\N	str_11	[]
 [[0,1,2]]	\N	\N	[]
-str_13	\N	str_13	[]
+str_12	\N	str_12	[]
 [[0,1,2,3]]	\N	\N	[]
-str_14	\N	str_14	[]
+str_13	\N	str_13	[]
 [[0,1,2,3,4]]	\N	\N	[]
-str_15	\N	str_15	[]
+str_14	\N	str_14	[]
 [[0,1,2,3,4,5]]	\N	\N	[]
-str_16	\N	str_16	[]
+str_15	\N	str_15	[]
 [[0,1,2,3,4,5,6]]	\N	\N	[]
-str_17	\N	str_17	[]
+str_16	\N	str_16	[]
 [[0,1,2,3,4,5,6,7]]	\N	\N	[]
-str_18	\N	str_18	[]
+str_17	\N	str_17	[]
 [[0,1,2,3,4,5,6,7,8]]	\N	\N	[]
-str_19	\N	str_19	[]
+str_18	\N	str_18	[]
 [[0,1,2,3,4,5,6,7,8,9]]	\N	\N	[]
+str_19	\N	str_19	[]
 [20]	\N	\N	[20]
 ['str_21','str_21']	\N	\N	['str_21','str_21']
 [22,22,22]	\N	\N	[22,22,22]
@@ -115,6 +115,7 @@ str_79	\N	str_79	[]
 7	\N	[]
 8	\N	[]
 9	\N	[]
+\N	\N	[]
 \N	str_10	[]
 \N	\N	[]
 \N	str_11	[]
@@ -134,7 +135,6 @@ str_79	\N	str_79	[]
 \N	str_18	[]
 \N	\N	[]
 \N	str_19	[]
-\N	\N	[]
 \N	\N	[20]
 \N	\N	['str_21','str_21']
 \N	\N	[22,22,22]
@@ -295,26 +295,26 @@ str_79	\N	str_79	[]
 7	7	\N	[]	0	[]
 8	8	\N	[]	0	[]
 9	9	\N	[]	0	[]
-str_10	\N	\N	[]	0	[]
 [[0]]	\N	\N	[]	0	[]
-str_11	\N	\N	[]	0	[]
+str_10	\N	\N	[]	0	[]
 [[0,1]]	\N	\N	[]	0	[]
-str_12	\N	\N	[]	0	[]
+str_11	\N	\N	[]	0	[]
 [[0,1,2]]	\N	\N	[]	0	[]
-str_13	\N	\N	[]	0	[]
+str_12	\N	\N	[]	0	[]
 [[0,1,2,3]]	\N	\N	[]	0	[]
-str_14	\N	\N	[]	0	[]
+str_13	\N	\N	[]	0	[]
 [[0,1,2,3,4]]	\N	\N	[]	0	[]
-str_15	\N	\N	[]	0	[]
+str_14	\N	\N	[]	0	[]
 [[0,1,2,3,4,5]]	\N	\N	[]	0	[]
-str_16	\N	\N	[]	0	[]
+str_15	\N	\N	[]	0	[]
 [[0,1,2,3,4,5,6]]	\N	\N	[]	0	[]
-str_17	\N	\N	[]	0	[]
+str_16	\N	\N	[]	0	[]
 [[0,1,2,3,4,5,6,7]]	\N	\N	[]	0	[]
-str_18	\N	\N	[]	0	[]
+str_17	\N	\N	[]	0	[]
 [[0,1,2,3,4,5,6,7,8]]	\N	\N	[]	0	[]
-str_19	\N	\N	[]	0	[]
+str_18	\N	\N	[]	0	[]
 [[0,1,2,3,4,5,6,7,8,9]]	\N	\N	[]	0	[]
+str_19	\N	\N	[]	0	[]
 [20]	\N	\N	[20]	1	[20]
 ['str_21','str_21']	\N	\N	['str_21','str_21']	2	[NULL,NULL]
 [22,22,22]	\N	\N	[22,22,22]	3	[22,22,22]
@@ -475,26 +475,26 @@ str_79	\N	\N	[]	0	[]
 7	0	[]	[]
 8	0	[]	[]
 9	0	[]	[]
-str_10	0	[]	[]
 [[0]]	0	[]	[]
-str_11	0	[]	[]
+str_10	0	[]	[]
 [[0,1]]	0	[]	[]
-str_12	0	[]	[]
+str_11	0	[]	[]
 [[0,1,2]]	0	[]	[]
-str_13	0	[]	[]
+str_12	0	[]	[]
 [[0,1,2,3]]	0	[]	[]
-str_14	0	[]	[]
+str_13	0	[]	[]
 [[0,1,2,3,4]]	0	[]	[]
-str_15	0	[]	[]
+str_14	0	[]	[]
 [[0,1,2,3,4,5]]	0	[]	[]
-str_16	0	[]	[]
+str_15	0	[]	[]
 [[0,1,2,3,4,5,6]]	0	[]	[]
-str_17	0	[]	[]
+str_16	0	[]	[]
 [[0,1,2,3,4,5,6,7]]	0	[]	[]
-str_18	0	[]	[]
+str_17	0	[]	[]
 [[0,1,2,3,4,5,6,7,8]]	0	[]	[]
-str_19	0	[]	[]
+str_18	0	[]	[]
 [[0,1,2,3,4,5,6,7,8,9]]	0	[]	[]
+str_19	0	[]	[]
 [20]	0	[]	[20]
 ['str_21','str_21']	0	[]	[NULL,NULL]
 [22,22,22]	0	[]	[22,22,22]
@@ -655,7 +655,6 @@ str_79	0	[]	[]
 []	[]	[]
 []	[]	[]
 []	[]	[]
-[]	[]	[]
 [1]	[[0]]	[[[]]]
 []	[]	[]
 [2]	[[0,1]]	[[[],[]]]
@@ -735,6 +734,7 @@ str_79	0	[]	[]
 []	[]	[]
 []	[]	[]
 []	[]	[]
+[]	[]	[]
 Array(Array(Dynamic))
 Array(Variant(String, UInt64))
 None
@@ -762,26 +762,26 @@ UInt64
 7	7	\N	[]
 8	8	\N	[]
 9	9	\N	[]
-str_10	\N	str_10	[]
 [[0]]	\N	\N	[]
-str_11	\N	str_11	[]
+str_10	\N	str_10	[]
 [[0,1]]	\N	\N	[]
-str_12	\N	str_12	[]
+str_11	\N	str_11	[]
 [[0,1,2]]	\N	\N	[]
-str_13	\N	str_13	[]
+str_12	\N	str_12	[]
 [[0,1,2,3]]	\N	\N	[]
-str_14	\N	str_14	[]
+str_13	\N	str_13	[]
 [[0,1,2,3,4]]	\N	\N	[]
-str_15	\N	str_15	[]
+str_14	\N	str_14	[]
 [[0,1,2,3,4,5]]	\N	\N	[]
-str_16	\N	str_16	[]
+str_15	\N	str_15	[]
 [[0,1,2,3,4,5,6]]	\N	\N	[]
-str_17	\N	str_17	[]
+str_16	\N	str_16	[]
 [[0,1,2,3,4,5,6,7]]	\N	\N	[]
-str_18	\N	str_18	[]
+str_17	\N	str_17	[]
 [[0,1,2,3,4,5,6,7,8]]	\N	\N	[]
-str_19	\N	str_19	[]
+str_18	\N	str_18	[]
 [[0,1,2,3,4,5,6,7,8,9]]	\N	\N	[]
+str_19	\N	str_19	[]
 [20]	\N	\N	[20]
 ['str_21','str_21']	\N	\N	['str_21','str_21']
 [22,22,22]	\N	\N	[22,22,22]
@@ -852,6 +852,7 @@ str_79	\N	str_79	[]
 7	\N	[]
 8	\N	[]
 9	\N	[]
+\N	\N	[]
 \N	str_10	[]
 \N	\N	[]
 \N	str_11	[]
@@ -871,7 +872,6 @@ str_79	\N	str_79	[]
 \N	str_18	[]
 \N	\N	[]
 \N	str_19	[]
-\N	\N	[]
 \N	\N	[20]
 \N	\N	['str_21','str_21']
 \N	\N	[22,22,22]
@@ -1032,26 +1032,26 @@ str_79	\N	str_79	[]
 7	7	\N	[]	0	[]
 8	8	\N	[]	0	[]
 9	9	\N	[]	0	[]
-str_10	\N	\N	[]	0	[]
 [[0]]	\N	\N	[]	0	[]
-str_11	\N	\N	[]	0	[]
+str_10	\N	\N	[]	0	[]
 [[0,1]]	\N	\N	[]	0	[]
-str_12	\N	\N	[]	0	[]
+str_11	\N	\N	[]	0	[]
 [[0,1,2]]	\N	\N	[]	0	[]
-str_13	\N	\N	[]	0	[]
+str_12	\N	\N	[]	0	[]
 [[0,1,2,3]]	\N	\N	[]	0	[]
-str_14	\N	\N	[]	0	[]
+str_13	\N	\N	[]	0	[]
 [[0,1,2,3,4]]	\N	\N	[]	0	[]
-str_15	\N	\N	[]	0	[]
+str_14	\N	\N	[]	0	[]
 [[0,1,2,3,4,5]]	\N	\N	[]	0	[]
-str_16	\N	\N	[]	0	[]
+str_15	\N	\N	[]	0	[]
 [[0,1,2,3,4,5,6]]	\N	\N	[]	0	[]
-str_17	\N	\N	[]	0	[]
+str_16	\N	\N	[]	0	[]
 [[0,1,2,3,4,5,6,7]]	\N	\N	[]	0	[]
-str_18	\N	\N	[]	0	[]
+str_17	\N	\N	[]	0	[]
 [[0,1,2,3,4,5,6,7,8]]	\N	\N	[]	0	[]
-str_19	\N	\N	[]	0	[]
+str_18	\N	\N	[]	0	[]
 [[0,1,2,3,4,5,6,7,8,9]]	\N	\N	[]	0	[]
+str_19	\N	\N	[]	0	[]
 [20]	\N	\N	[20]	1	[20]
 ['str_21','str_21']	\N	\N	['str_21','str_21']	2	[NULL,NULL]
 [22,22,22]	\N	\N	[22,22,22]	3	[22,22,22]
@@ -1212,26 +1212,26 @@ str_79	\N	\N	[]	0	[]
 7	0	[]	[]
 8	0	[]	[]
 9	0	[]	[]
-str_10	0	[]	[]
 [[0]]	0	[]	[]
-str_11	0	[]	[]
+str_10	0	[]	[]
 [[0,1]]	0	[]	[]
-str_12	0	[]	[]
+str_11	0	[]	[]
 [[0,1,2]]	0	[]	[]
-str_13	0	[]	[]
+str_12	0	[]	[]
 [[0,1,2,3]]	0	[]	[]
-str_14	0	[]	[]
+str_13	0	[]	[]
 [[0,1,2,3,4]]	0	[]	[]
-str_15	0	[]	[]
+str_14	0	[]	[]
 [[0,1,2,3,4,5]]	0	[]	[]
-str_16	0	[]	[]
+str_15	0	[]	[]
 [[0,1,2,3,4,5,6]]	0	[]	[]
-str_17	0	[]	[]
+str_16	0	[]	[]
 [[0,1,2,3,4,5,6,7]]	0	[]	[]
-str_18	0	[]	[]
+str_17	0	[]	[]
 [[0,1,2,3,4,5,6,7,8]]	0	[]	[]
-str_19	0	[]	[]
+str_18	0	[]	[]
 [[0,1,2,3,4,5,6,7,8,9]]	0	[]	[]
+str_19	0	[]	[]
 [20]	0	[]	[20]
 ['str_21','str_21']	0	[]	[NULL,NULL]
 [22,22,22]	0	[]	[22,22,22]
@@ -1392,7 +1392,6 @@ str_79	0	[]	[]
 []	[]	[]
 []	[]	[]
 []	[]	[]
-[]	[]	[]
 [1]	[[0]]	[[[]]]
 []	[]	[]
 [2]	[[0,1]]	[[[],[]]]
@@ -1472,6 +1471,7 @@ str_79	0	[]	[]
 []	[]	[]
 []	[]	[]
 []	[]	[]
+[]	[]	[]
 Array(Array(Dynamic))
 Array(Variant(String, UInt64))
 None
@@ -1499,26 +1499,26 @@ UInt64
 7	7	\N	[]
 8	8	\N	[]
 9	9	\N	[]
-str_10	\N	str_10	[]
 [[0]]	\N	\N	[]
-str_11	\N	str_11	[]
+str_10	\N	str_10	[]
 [[0,1]]	\N	\N	[]
-str_12	\N	str_12	[]
+str_11	\N	str_11	[]
 [[0,1,2]]	\N	\N	[]
-str_13	\N	str_13	[]
+str_12	\N	str_12	[]
 [[0,1,2,3]]	\N	\N	[]
-str_14	\N	str_14	[]
+str_13	\N	str_13	[]
 [[0,1,2,3,4]]	\N	\N	[]
-str_15	\N	str_15	[]
+str_14	\N	str_14	[]
 [[0,1,2,3,4,5]]	\N	\N	[]
-str_16	\N	str_16	[]
+str_15	\N	str_15	[]
 [[0,1,2,3,4,5,6]]	\N	\N	[]
-str_17	\N	str_17	[]
+str_16	\N	str_16	[]
 [[0,1,2,3,4,5,6,7]]	\N	\N	[]
-str_18	\N	str_18	[]
+str_17	\N	str_17	[]
 [[0,1,2,3,4,5,6,7,8]]	\N	\N	[]
-str_19	\N	str_19	[]
+str_18	\N	str_18	[]
 [[0,1,2,3,4,5,6,7,8,9]]	\N	\N	[]
+str_19	\N	str_19	[]
 [20]	\N	\N	[20]
 ['str_21','str_21']	\N	\N	['str_21','str_21']
 [22,22,22]	\N	\N	[22,22,22]
@@ -1589,6 +1589,7 @@ str_79	\N	str_79	[]
 7	\N	[]
 8	\N	[]
 9	\N	[]
+\N	\N	[]
 \N	str_10	[]
 \N	\N	[]
 \N	str_11	[]
@@ -1608,7 +1609,6 @@ str_79	\N	str_79	[]
 \N	str_18	[]
 \N	\N	[]
 \N	str_19	[]
-\N	\N	[]
 \N	\N	[20]
 \N	\N	['str_21','str_21']
 \N	\N	[22,22,22]
@@ -1769,26 +1769,26 @@ str_79	\N	str_79	[]
 7	7	\N	[]	0	[]
 8	8	\N	[]	0	[]
 9	9	\N	[]	0	[]
-str_10	\N	\N	[]	0	[]
 [[0]]	\N	\N	[]	0	[]
-str_11	\N	\N	[]	0	[]
+str_10	\N	\N	[]	0	[]
 [[0,1]]	\N	\N	[]	0	[]
-str_12	\N	\N	[]	0	[]
+str_11	\N	\N	[]	0	[]
 [[0,1,2]]	\N	\N	[]	0	[]
-str_13	\N	\N	[]	0	[]
+str_12	\N	\N	[]	0	[]
 [[0,1,2,3]]	\N	\N	[]	0	[]
-str_14	\N	\N	[]	0	[]
+str_13	\N	\N	[]	0	[]
 [[0,1,2,3,4]]	\N	\N	[]	0	[]
-str_15	\N	\N	[]	0	[]
+str_14	\N	\N	[]	0	[]
 [[0,1,2,3,4,5]]	\N	\N	[]	0	[]
-str_16	\N	\N	[]	0	[]
+str_15	\N	\N	[]	0	[]
 [[0,1,2,3,4,5,6]]	\N	\N	[]	0	[]
-str_17	\N	\N	[]	0	[]
+str_16	\N	\N	[]	0	[]
 [[0,1,2,3,4,5,6,7]]	\N	\N	[]	0	[]
-str_18	\N	\N	[]	0	[]
+str_17	\N	\N	[]	0	[]
 [[0,1,2,3,4,5,6,7,8]]	\N	\N	[]	0	[]
-str_19	\N	\N	[]	0	[]
+str_18	\N	\N	[]	0	[]
 [[0,1,2,3,4,5,6,7,8,9]]	\N	\N	[]	0	[]
+str_19	\N	\N	[]	0	[]
 [20]	\N	\N	[20]	1	[20]
 ['str_21','str_21']	\N	\N	['str_21','str_21']	2	[NULL,NULL]
 [22,22,22]	\N	\N	[22,22,22]	3	[22,22,22]
@@ -1949,26 +1949,26 @@ str_79	\N	\N	[]	0	[]
 7	0	[]	[]
 8	0	[]	[]
 9	0	[]	[]
-str_10	0	[]	[]
 [[0]]	0	[]	[]
-str_11	0	[]	[]
+str_10	0	[]	[]
 [[0,1]]	0	[]	[]
-str_12	0	[]	[]
+str_11	0	[]	[]
 [[0,1,2]]	0	[]	[]
-str_13	0	[]	[]
+str_12	0	[]	[]
 [[0,1,2,3]]	0	[]	[]
-str_14	0	[]	[]
+str_13	0	[]	[]
 [[0,1,2,3,4]]	0	[]	[]
-str_15	0	[]	[]
+str_14	0	[]	[]
 [[0,1,2,3,4,5]]	0	[]	[]
-str_16	0	[]	[]
+str_15	0	[]	[]
 [[0,1,2,3,4,5,6]]	0	[]	[]
-str_17	0	[]	[]
+str_16	0	[]	[]
 [[0,1,2,3,4,5,6,7]]	0	[]	[]
-str_18	0	[]	[]
+str_17	0	[]	[]
 [[0,1,2,3,4,5,6,7,8]]	0	[]	[]
-str_19	0	[]	[]
+str_18	0	[]	[]
 [[0,1,2,3,4,5,6,7,8,9]]	0	[]	[]
+str_19	0	[]	[]
 [20]	0	[]	[20]
 ['str_21','str_21']	0	[]	[NULL,NULL]
 [22,22,22]	0	[]	[22,22,22]
@@ -2129,7 +2129,6 @@ str_79	0	[]	[]
 []	[]	[]
 []	[]	[]
 []	[]	[]
-[]	[]	[]
 [1]	[[0]]	[[[]]]
 []	[]	[]
 [2]	[[0,1]]	[[[],[]]]
@@ -2209,3 +2208,4 @@ str_79	0	[]	[]
 []	[]	[]
 []	[]	[]
 []	[]	[]
+[]	[]	[]
diff --git a/tests/queries/0_stateless/03036_dynamic_read_subcolumns_small.sql.j2 b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_small.sql.j2
index 1ed836fbeee..3253d7a6c68 100644
--- a/tests/queries/0_stateless/03036_dynamic_read_subcolumns_small.sql.j2
+++ b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_small.sql.j2
@@ -29,14 +29,14 @@ select count() from test where not empty(d.`Array(Array(Dynamic))`);
 select count() from test where d is NULL;
 select count() from test where not empty(d.`Tuple(a Array(Dynamic))`.a.String);
 
-select d, d.UInt64, d.String, d.`Array(Variant(String, UInt64))` from test order by id;
-select d.UInt64, d.String, d.`Array(Variant(String, UInt64))` from test order by id;
-select d.Int8, d.Date, d.`Array(String)` from test order by id;
-select d, d.UInt64, d.Date, d.`Array(Variant(String, UInt64))`, d.`Array(Variant(String, UInt64))`.size0, d.`Array(Variant(String, UInt64))`.UInt64 from test order by id;
-select d.UInt64, d.Date, d.`Array(Variant(String, UInt64))`, d.`Array(Variant(String, UInt64))`.size0, d.`Array(Variant(String, UInt64))`.UInt64, d.`Array(Variant(String, UInt64))`.String from test order by id;
-select d, d.`Tuple(a UInt64, b String)`.a, d.`Array(Dynamic)`.`Variant(String, UInt64)`.UInt64, d.`Array(Variant(String, UInt64))`.UInt64 from test order by id;
-select d.`Array(Dynamic)`.`Variant(String, UInt64)`.UInt64, d.`Array(Dynamic)`.size0, d.`Array(Variant(String, UInt64))`.UInt64 from test order by id;
-select d.`Array(Array(Dynamic))`.size1, d.`Array(Array(Dynamic))`.UInt64, d.`Array(Array(Dynamic))`.`Map(String, Tuple(a UInt64))`.values.a from test order by id;
+select d, d.UInt64, d.String, d.`Array(Variant(String, UInt64))` from test order by id, d;
+select d.UInt64, d.String, d.`Array(Variant(String, UInt64))` from test order by id, d;
+select d.Int8, d.Date, d.`Array(String)` from test order by id, d;
+select d, d.UInt64, d.Date, d.`Array(Variant(String, UInt64))`, d.`Array(Variant(String, UInt64))`.size0, d.`Array(Variant(String, UInt64))`.UInt64 from test order by id, d;
+select d.UInt64, d.Date, d.`Array(Variant(String, UInt64))`, d.`Array(Variant(String, UInt64))`.size0, d.`Array(Variant(String, UInt64))`.UInt64, d.`Array(Variant(String, UInt64))`.String from test order by id, d;
+select d, d.`Tuple(a UInt64, b String)`.a, d.`Array(Dynamic)`.`Variant(String, UInt64)`.UInt64, d.`Array(Variant(String, UInt64))`.UInt64 from test order by id, d;
+select d.`Array(Dynamic)`.`Variant(String, UInt64)`.UInt64, d.`Array(Dynamic)`.size0, d.`Array(Variant(String, UInt64))`.UInt64 from test order by id, d;
+select d.`Array(Array(Dynamic))`.size1, d.`Array(Array(Dynamic))`.UInt64, d.`Array(Array(Dynamic))`.`Map(String, Tuple(a UInt64))`.values.a from test order by id, d;
 
 drop table test;
 

From 36c57ca50bf54180dc1a68bfe097112f3dc13a6f Mon Sep 17 00:00:00 2001
From: jsc0218 <jsc0218@gmail.com>
Date: Mon, 29 Jul 2024 19:50:28 +0000
Subject: [PATCH 1027/1488] only check in the create mode to prevent failure to
 start

---
 src/Storages/StorageFactory.cpp | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/src/Storages/StorageFactory.cpp b/src/Storages/StorageFactory.cpp
index 71f70a807a8..a059d624cd8 100644
--- a/src/Storages/StorageFactory.cpp
+++ b/src/Storages/StorageFactory.cpp
@@ -202,11 +202,12 @@ StoragePtr StorageFactory::get(
                     "projections",
                     [](StorageFeatures features) { return features.supports_projections; });
 
-                /// Now let's handle the merge tree family, projection is fully supported in (Replictaed)MergeTree,
-                /// but also allowed in non-throw mode with other mergetree family members.
+                /// Now let's handle the merge tree family. Note we only handle in the mode of CREATE due to backward compatibility.
+                /// Otherwise, it would fail to start in the case of existing projections with special mergetree.
+                /// Projection is fully supported in (Replictaed)MergeTree, but also allowed in non-throw mode with other mergetree family members.
                 chassert(query.storage->engine);
-                if (std::string_view engine_name(query.storage->engine->name);
-                    engine_name != "MergeTree" && engine_name != "ReplicatedMergeTree")
+                if (std::string_view engine_name(query.storage->engine->name); mode == LoadingStrictnessLevel::CREATE
+                    && engine_name != "MergeTree" && engine_name != "ReplicatedMergeTree")
                 {
                     /// default throw mode in deduplicate_merge_projection_mode
                     bool projection_allowed = false;

From 2cae0cb5ecedc2fd041def829b35bdf4dbb50f2f Mon Sep 17 00:00:00 2001
From: Igor Nikonov <igor@clickhouse.com>
Date: Mon, 29 Jul 2024 20:29:15 +0000
Subject: [PATCH 1028/1488] force_connected flag for connection establisher

---
 src/Client/ConnectionEstablisher.cpp      | 4 ++--
 src/Client/ConnectionEstablisher.h        | 4 +++-
 src/QueryPipeline/RemoteQueryExecutor.cpp | 4 ++--
 3 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/src/Client/ConnectionEstablisher.cpp b/src/Client/ConnectionEstablisher.cpp
index 8cebe7a6183..f96546846c7 100644
--- a/src/Client/ConnectionEstablisher.cpp
+++ b/src/Client/ConnectionEstablisher.cpp
@@ -33,12 +33,12 @@ ConnectionEstablisher::ConnectionEstablisher(
 {
 }
 
-void ConnectionEstablisher::run(ConnectionEstablisher::TryResult & result, std::string & fail_message)
+void ConnectionEstablisher::run(ConnectionEstablisher::TryResult & result, std::string & fail_message, bool force_connected)
 {
     try
     {
         ProfileEvents::increment(ProfileEvents::DistributedConnectionTries);
-        result.entry = pool->get(*timeouts, settings, /* force_connected = */ true);
+        result.entry = pool->get(*timeouts, settings, force_connected);
         AsyncCallbackSetter async_setter(&*result.entry, std::move(async_callback));
 
         UInt64 server_revision = 0;
diff --git a/src/Client/ConnectionEstablisher.h b/src/Client/ConnectionEstablisher.h
index a3a01e63246..304ec4d34b4 100644
--- a/src/Client/ConnectionEstablisher.h
+++ b/src/Client/ConnectionEstablisher.h
@@ -24,7 +24,9 @@ public:
                           const QualifiedTableName * table_to_check = nullptr);
 
     /// Establish connection and save it in result, write possible exception message in fail_message.
-    void run(TryResult & result, std::string & fail_message);
+    /// The connection is returned from the pool, it can be stale. Use force_connected flag
+    /// to ensure that connection is working one
+    void run(TryResult & result, std::string & fail_message, bool force_connected = false);
 
     /// Set async callback that will be called when reading from socket blocks.
     void setAsyncCallback(AsyncCallback async_callback_) { async_callback = std::move(async_callback_); }
diff --git a/src/QueryPipeline/RemoteQueryExecutor.cpp b/src/QueryPipeline/RemoteQueryExecutor.cpp
index b08f2002f64..09ea6a9fb3c 100644
--- a/src/QueryPipeline/RemoteQueryExecutor.cpp
+++ b/src/QueryPipeline/RemoteQueryExecutor.cpp
@@ -89,12 +89,12 @@ RemoteQueryExecutor::RemoteQueryExecutor(
             auto table_name = main_table.getQualifiedName();
 
             ConnectionEstablisher connection_establisher(pool, &timeouts, current_settings, log, &table_name);
-            connection_establisher.run(result, fail_message);
+            connection_establisher.run(result, fail_message, /*force_connected=*/ true);
         }
         else
         {
             ConnectionEstablisher connection_establisher(pool, &timeouts, current_settings, log, nullptr);
-            connection_establisher.run(result, fail_message);
+            connection_establisher.run(result, fail_message, /*force_connected=*/ true);
         }
 
         std::vector<IConnectionPool::Entry> connection_entries;

From 09619e6006f122fb3a8352328a07f42bfd284d17 Mon Sep 17 00:00:00 2001
From: jsc0218 <jsc0218@gmail.com>
Date: Mon, 29 Jul 2024 20:57:21 +0000
Subject: [PATCH 1029/1488] consider the case of alter table add projection

---
 src/Storages/MergeTree/MergeTreeData.cpp      | 10 +++++++
 ...ojection_merge_special_mergetree.reference |  1 +
 ...206_projection_merge_special_mergetree.sql | 26 +++++++++++++++++--
 3 files changed, 35 insertions(+), 2 deletions(-)

diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index 78a551591a6..677c4a92cda 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -3211,6 +3211,16 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, Context
                             queryToString(mutation_commands.ast()));
     }
 
+    /// Block the case of alter table add projection for special merge trees.
+    if (std::any_of(commands.begin(), commands.end(), [](const AlterCommand & c) { return c.type == AlterCommand::ADD_PROJECTION; }))
+    {
+        if (auto storage_name = getName(); storage_name != "MergeTree" && storage_name != "ReplicatedMergeTree"
+            && settings_from_storage->deduplicate_merge_projection_mode == DeduplicateMergeProjectionMode::THROW)
+            throw Exception(ErrorCodes::NOT_IMPLEMENTED,
+                "Projection is fully supported in (Replictaed)MergeTree, but also allowed in non-throw mode with other"
+                " mergetree family members. Consider drop or rebuild option of deduplicate_merge_projection_mode.");
+    }
+
     commands.apply(new_metadata, local_context);
 
     if (AlterCommands::hasFullTextIndex(new_metadata) && !settings.allow_experimental_full_text_index)
diff --git a/tests/queries/0_stateless/03206_projection_merge_special_mergetree.reference b/tests/queries/0_stateless/03206_projection_merge_special_mergetree.reference
index e69de29bb2d..1a9cc2b7fbf 100644
--- a/tests/queries/0_stateless/03206_projection_merge_special_mergetree.reference
+++ b/tests/queries/0_stateless/03206_projection_merge_special_mergetree.reference
@@ -0,0 +1 @@
+p
diff --git a/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql b/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql
index 25517fbba30..e0a4f4f8cec 100644
--- a/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql
+++ b/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql
@@ -59,8 +59,7 @@ OPTIMIZE TABLE tp FINAL;
 -- expecting no projection
 SYSTEM FLUSH LOGS;
 SELECT
-    name,
-    part_name
+    name
 FROM system.projection_parts
 WHERE (database = currentDatabase()) AND (`table` = 'tp') AND (active = 1);
 
@@ -81,4 +80,27 @@ ALTER TABLE tp MODIFY SETTING deduplicate_merge_projection_mode = 'throw';
 
 OPTIMIZE TABLE tp DEDUPLICATE;  -- { serverError NOT_IMPLEMENTED }
 
+DROP TABLE tp;
+
+-- test alter add projection case
+CREATE TABLE tp (
+    type Int32,
+    eventcnt UInt64
+) engine = ReplacingMergeTree order by type;
+
+ALTER TABLE tp ADD PROJECTION p (SELECT sum(eventcnt), type GROUP BY type);  -- { serverError NOT_IMPLEMENTED }
+
+ALTER TABLE tp MODIFY SETTING deduplicate_merge_projection_mode = 'drop';
+
+ALTER TABLE tp ADD PROJECTION p (SELECT sum(eventcnt), type GROUP BY type);
+
+INSERT INTO tp SELECT number%3, 1 FROM numbers(3);
+
+SYSTEM FLUSH LOGS;
+-- expecting projection p
+SELECT
+    name
+FROM system.projection_parts
+WHERE (database = currentDatabase()) AND (`table` = 'tp') AND (active = 1);
+
 DROP TABLE tp;
\ No newline at end of file

From 8dfe4a93f6c1afde8475984a899cd5604f415d78 Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitlibar@clickhouse.com>
Date: Mon, 29 Jul 2024 22:07:10 +0200
Subject: [PATCH 1030/1488] Rewrite function get_broken_projections_info()
 without using system.errors

---
 .../test_broken_projections/test.py           | 86 +++++++++----------
 1 file changed, 42 insertions(+), 44 deletions(-)

diff --git a/tests/integration/test_broken_projections/test.py b/tests/integration/test_broken_projections/test.py
index 2cbbee5563f..9493937d936 100644
--- a/tests/integration/test_broken_projections/test.py
+++ b/tests/integration/test_broken_projections/test.py
@@ -148,23 +148,22 @@ def break_part(node, table, part):
     bash(node, f"rm '{part_path}/columns.txt'")
 
 
-def get_broken_projections_info(node, table, active=True):
+def get_broken_projections_info(node, table, part=None, projection=None, active=True):
+    parent_name_filter = f" AND parent_name = '{part}'" if part else ""
+    name_filter = f" AND name = '{projection}'" if projection else ""
     return node.query(
         f"""
-    SELECT parent_name, name, errors.name FROM
-    (
-        SELECT parent_name, name, exception_code
+    SELECT parent_name, name, exception
         FROM system.projection_parts
         WHERE table='{table}'
         AND database=currentDatabase()
         AND is_broken = 1
         AND active = {active}
-    ) AS parts_info
-    INNER JOIN system.errors AS errors
-    ON parts_info.exception_code = errors.code
+        {parent_name_filter}
+        {name_filter}
     ORDER BY parent_name, name
     """
-    ).strip()
+    )
 
 
 def get_projections_info(node, table):
@@ -312,8 +311,8 @@ def test_broken_ignored(cluster):
 
     # Projection 'proj1' from part all_2_2_0 will now appear in broken parts info
     # because it was marked broken during "check table" query.
-    assert "all_2_2_0\tproj1\tFILE_DOESNT_EXIST" in get_broken_projections_info(
-        node, table_name
+    assert "FILE_DOESNT_EXIST" in get_broken_projections_info(
+        node, table_name, part="all_2_2_0", projection="proj1"
     )
 
     # Check table query will also show a list of parts which have broken projections.
@@ -323,14 +322,14 @@ def test_broken_ignored(cluster):
     break_projection(node, table_name, "proj2", "all_2_2_0", "data")
 
     # It will not yet appear in broken projections info.
-    assert "proj2" not in get_broken_projections_info(node, table_name)
+    assert not get_broken_projections_info(node, table_name, projection="proj2")
 
     # Select now fails with error "File doesn't exist"
     check(node, table_name, 0, "proj2", "FILE_DOESNT_EXIST")
 
     # Projection 'proj2' from part all_2_2_0 will now appear in broken parts info.
-    assert "all_2_2_0\tproj2\tNO_FILE_IN_DATA_PART" in get_broken_projections_info(
-        node, table_name
+    assert "NO_FILE_IN_DATA_PART" in get_broken_projections_info(
+        node, table_name, part="all_2_2_0", projection="proj2"
     )
 
     # Second select works, because projection is now marked as broken.
@@ -340,7 +339,7 @@ def test_broken_ignored(cluster):
     break_projection(node, table_name, "proj2", "all_3_3_0", "data")
 
     # It will not yet appear in broken projections info.
-    assert "all_3_3_0" not in get_broken_projections_info(node, table_name)
+    assert not get_broken_projections_info(node, table_name, part="all_3_3_0")
 
     insert(node, table_name, 20, 5)
     insert(node, table_name, 25, 5)
@@ -371,8 +370,8 @@ def test_broken_ignored(cluster):
         node, table_name
     )
 
-    assert "all_3_3_0" in get_broken_projections_info(node, table_name, active=False)
-    assert "all_2_2_0" in get_broken_projections_info(node, table_name, active=True)
+    assert get_broken_projections_info(node, table_name, part="all_3_3_0", active=False)
+    assert get_broken_projections_info(node, table_name, part="all_2_2_0", active=True)
 
     # 0 because of all_2_2_0
     check(node, table_name, 0)
@@ -396,8 +395,8 @@ def test_materialize_broken_projection(cluster):
     break_projection(node, table_name, "proj1", "all_1_1_0", "metadata")
     reattach(node, table_name)
 
-    assert "all_1_1_0\tproj1\tNO_FILE_IN_DATA_PART" in get_broken_projections_info(
-        node, table_name
+    assert "NO_FILE_IN_DATA_PART" in get_broken_projections_info(
+        node, table_name, part="all_1_1_0", projection="proj1"
     )
     assert "Part all_1_1_0 has a broken projection proj1" in check_table_full(
         node, table_name
@@ -406,8 +405,8 @@ def test_materialize_broken_projection(cluster):
     break_projection(node, table_name, "proj2", "all_1_1_0", "data")
     reattach(node, table_name)
 
-    assert "all_1_1_0\tproj2\tFILE_DOESNT_EXIST" in get_broken_projections_info(
-        node, table_name
+    assert "FILE_DOESNT_EXIST" in get_broken_projections_info(
+        node, table_name, part="all_1_1_0", projection="proj2"
     )
     assert "Part all_1_1_0 has a broken projection proj2" in check_table_full(
         node, table_name
@@ -469,8 +468,8 @@ def test_broken_projections_in_backups_2(cluster):
     break_projection(node, table_name, "proj2", "all_2_2_0", "part")
     check(node, table_name, 0, "proj2", "ErrnoException")
 
-    assert "all_2_2_0\tproj2\tFILE_DOESNT_EXIST" == get_broken_projections_info(
-        node, table_name
+    assert "FILE_DOESNT_EXIST" in get_broken_projections_info(
+        node, table_name, part="all_2_2_0", projection="proj2"
     )
 
     assert "FILE_DOESNT_EXIST" in node.query_and_get_error(
@@ -524,8 +523,8 @@ def test_broken_projections_in_backups_3(cluster):
     assert "Part all_1_1_0 has a broken projection proj1" in check_table_full(
         node, table_name
     )
-    assert "all_1_1_0\tproj1\tFILE_DOESNT_EXIST" == get_broken_projections_info(
-        node, table_name
+    assert "FILE_DOESNT_EXIST" in get_broken_projections_info(
+        node, table_name, part="all_1_1_0", projection="proj1"
     )
 
     backup_name = f"b4-{get_random_string()}"
@@ -545,8 +544,11 @@ def test_broken_projections_in_backups_3(cluster):
     )
 
     check(node, table_name, 0)
-    assert "all_1_1_0\tproj1\tNO_FILE_IN_DATA_PART" == get_broken_projections_info(
-        node, table_name
+    assert (
+        "Projection directory proj1.proj does not exist while loading projections"
+        in get_broken_projections_info(
+            node, table_name, part="all_1_1_0", projection="proj1"
+        )
     )
 
 
@@ -569,7 +571,7 @@ def test_check_part_thread(cluster):
     break_projection(node, table_name, "proj2", "all_2_2_0", "data")
 
     # It will not yet appear in broken projections info.
-    assert "proj2" not in get_broken_projections_info(node, table_name)
+    assert not get_broken_projections_info(node, table_name, projection="proj2")
 
     # Select now fails with error "File doesn't exist"
     check(node, table_name, 0, "proj2", "FILE_DOESNT_EXIST", do_check_command=False)
@@ -606,15 +608,15 @@ def test_broken_on_start(cluster):
     break_projection(node, table_name, "proj2", "all_2_2_0", "data")
 
     # It will not yet appear in broken projections info.
-    assert "proj2" not in get_broken_projections_info(node, table_name)
+    assert not get_broken_projections_info(node, table_name, projection="proj2")
 
     # Select now fails with error "File doesn't exist"
     # We will mark projection as broken.
     check(node, table_name, 0, "proj2", "FILE_DOESNT_EXIST")
 
     # Projection 'proj2' from part all_2_2_0 will now appear in broken parts info.
-    assert "all_2_2_0\tproj2\tNO_FILE_IN_DATA_PART" in get_broken_projections_info(
-        node, table_name
+    assert "NO_FILE_IN_DATA_PART" in get_broken_projections_info(
+        node, table_name, part="all_2_2_0", projection="proj2"
     )
 
     # Second select works, because projection is now marked as broken.
@@ -623,7 +625,7 @@ def test_broken_on_start(cluster):
     node.restart_clickhouse()
 
     # It will not yet appear in broken projections info.
-    assert "proj2" in get_broken_projections_info(node, table_name)
+    assert get_broken_projections_info(node, table_name, projection="proj2")
 
     # Select works
     check(node, table_name, 0)
@@ -654,7 +656,7 @@ def test_mutation_with_broken_projection(cluster):
         node, table_name
     )
 
-    assert "" == get_broken_projections_info(node, table_name)
+    assert not get_broken_projections_info(node, table_name)
 
     check(node, table_name, 1)
 
@@ -662,21 +664,21 @@ def test_mutation_with_broken_projection(cluster):
     break_projection(node, table_name, "proj2", "all_2_2_0_4", "data")
 
     # It will not yet appear in broken projections info.
-    assert "proj2" not in get_broken_projections_info(node, table_name)
+    assert not get_broken_projections_info(node, table_name, projection="proj2")
 
     # Select now fails with error "File doesn't exist"
     # We will mark projection as broken.
     check(node, table_name, 0, "proj2", "FILE_DOESNT_EXIST")
 
     # Projection 'proj2' from part all_2_2_0_4 will now appear in broken parts info.
-    assert "all_2_2_0_4\tproj2\tNO_FILE_IN_DATA_PART" in get_broken_projections_info(
-        node, table_name
+    assert "NO_FILE_IN_DATA_PART" in get_broken_projections_info(
+        node, table_name, part="all_2_2_0_4", projection="proj2"
     )
 
     # Second select works, because projection is now marked as broken.
     check(node, table_name, 0)
 
-    assert "all_2_2_0_4" in get_broken_projections_info(node, table_name)
+    assert get_broken_projections_info(node, table_name, part="all_2_2_0_4")
 
     node.query(
         f"ALTER TABLE {table_name} DELETE WHERE _part == 'all_0_0_0_4' SETTINGS mutations_sync = 1"
@@ -690,14 +692,10 @@ def test_mutation_with_broken_projection(cluster):
 
     # Still broken because it was hardlinked.
     broken = get_broken_projections_info(node, table_name)
-    assert (
-        "all_2_2_0_5" in broken or "" == broken
-    )  # second could be because of a merge.
+    if broken:  # can be not broken because of a merge.
+        assert get_broken_projections_info(node, table_name, part="all_2_2_0_5")
 
-    if "" == broken:
-        check(node, table_name, 1)
-    else:
-        check(node, table_name, 0)
+    check(node, table_name, not broken)
 
     node.query(
         f"ALTER TABLE {table_name} DELETE WHERE c == 13 SETTINGS mutations_sync = 1"
@@ -710,6 +708,6 @@ def test_mutation_with_broken_projection(cluster):
     )
 
     # Not broken anymore.
-    assert "" == get_broken_projections_info(node, table_name)
+    assert not get_broken_projections_info(node, table_name)
 
     check(node, table_name, 1)

From f94bebb0530b7a9fdd3db104ad4261a467fafad3 Mon Sep 17 00:00:00 2001
From: Yakov Olkhovskiy <yakov@clickhouse.com>
Date: Mon, 29 Jul 2024 22:36:59 +0000
Subject: [PATCH 1031/1488] fix

---
 docs/en/sql-reference/window-functions/lagInFrame.md  | 2 +-
 docs/en/sql-reference/window-functions/leadInFrame.md | 2 +-
 src/Processors/Transforms/WindowTransform.cpp         | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/docs/en/sql-reference/window-functions/lagInFrame.md b/docs/en/sql-reference/window-functions/lagInFrame.md
index 049e095c10f..de6e9005baa 100644
--- a/docs/en/sql-reference/window-functions/lagInFrame.md
+++ b/docs/en/sql-reference/window-functions/lagInFrame.md
@@ -23,7 +23,7 @@ For more detail on window function syntax see: [Window Functions - Syntax](./ind
 **Parameters**
 - `x` — Column name. 
 - `offset` — Offset to apply. [(U)Int*](../data-types/int-uint.md). (Optional - `1` by default).
-- `default` — Value to return if calculated row exceeds the boundaries of the window frame. (Optional - `null` by default).
+- `default` — Value to return if calculated row exceeds the boundaries of the window frame. (Optional - default value of column type when omitted).
 
 **Returned value**
 
diff --git a/docs/en/sql-reference/window-functions/leadInFrame.md b/docs/en/sql-reference/window-functions/leadInFrame.md
index fc1b92cc266..4a82c03f6e6 100644
--- a/docs/en/sql-reference/window-functions/leadInFrame.md
+++ b/docs/en/sql-reference/window-functions/leadInFrame.md
@@ -23,7 +23,7 @@ For more detail on window function syntax see: [Window Functions - Syntax](./ind
 **Parameters**
 - `x` — Column name.
 - `offset` — Offset to apply. [(U)Int*](../data-types/int-uint.md). (Optional - `1` by default).
-- `default` — Value to return if calculated row exceeds the boundaries of the window frame. (Optional - `null` by default).
+- `default` — Value to return if calculated row exceeds the boundaries of the window frame. (Optional - default value of column type when omitted).
 
 **Returned value**
 
diff --git a/src/Processors/Transforms/WindowTransform.cpp b/src/Processors/Transforms/WindowTransform.cpp
index 1eac08780e9..f76e2d64368 100644
--- a/src/Processors/Transforms/WindowTransform.cpp
+++ b/src/Processors/Transforms/WindowTransform.cpp
@@ -2408,7 +2408,7 @@ struct WindowFunctionLagLeadInFrame final : public WindowFunction
         if (argument_types[0]->equals(*argument_types[2]))
             return;
 
-        const auto supertype = getLeastSupertype(DataTypes{argument_types[0], argument_types[2]});
+        const auto supertype = tryGetLeastSupertype(DataTypes{argument_types[0], argument_types[2]});
         if (!supertype)
         {
             throw Exception(ErrorCodes::BAD_ARGUMENTS,

From 2aafd711463d30cb7803a054f434268335817db8 Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <nikitamikhaylov@clickhouse.com>
Date: Fri, 26 Jul 2024 19:08:07 +0200
Subject: [PATCH 1032/1488] Moved settings to ServerSettings and made the table
 drop even faster

(cherry picked from commit e1eb542dcc2b9fbc6a470a3cd9a183e79c86d7c7)
---
 programs/local/LocalServer.cpp            |  5 +++
 programs/server/Server.cpp                |  5 +++
 src/Core/ServerSettings.cpp               |  2 +-
 src/Core/ServerSettings.h                 |  9 ++++
 src/IO/SharedThreadPools.cpp              | 10 +++++
 src/IO/SharedThreadPools.h                |  3 ++
 src/Interpreters/DatabaseCatalog.cpp      | 50 ++++++++++-------------
 src/Interpreters/DatabaseCatalog.h        | 15 -------
 src/Interpreters/InterpreterDropQuery.cpp | 19 +++++++--
 9 files changed, 70 insertions(+), 48 deletions(-)

diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp
index 88d5a0253d1..250c5e3b6c8 100644
--- a/programs/local/LocalServer.cpp
+++ b/programs/local/LocalServer.cpp
@@ -184,6 +184,11 @@ void LocalServer::initialize(Poco::Util::Application & self)
         cleanup_threads,
         0, // We don't need any threads one all the parts will be deleted
         cleanup_threads);
+
+    getDatabaseCatalogDropTablesThreadPool().initialize(
+        server_settings.database_catalog_drop_table_concurrency,
+        0, // We don't need any threads if there are no DROP queries.
+        server_settings.database_catalog_drop_table_concurrency);
 }
 
 
diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp
index 16888015f8b..dd56114de0f 100644
--- a/programs/server/Server.cpp
+++ b/programs/server/Server.cpp
@@ -1035,6 +1035,11 @@ try
         0, // We don't need any threads once all the tables will be created
         max_database_replicated_create_table_thread_pool_size);
 
+    getDatabaseCatalogDropTablesThreadPool().initialize(
+        server_settings.database_catalog_drop_table_concurrency,
+        0, // We don't need any threads if there are no DROP queries.
+        server_settings.database_catalog_drop_table_concurrency);
+
     /// Initialize global local cache for remote filesystem.
     if (config().has("local_cache_for_remote_fs"))
     {
diff --git a/src/Core/ServerSettings.cpp b/src/Core/ServerSettings.cpp
index fbf86d3e9ad..6c498014996 100644
--- a/src/Core/ServerSettings.cpp
+++ b/src/Core/ServerSettings.cpp
@@ -1,4 +1,4 @@
-#include "ServerSettings.h"
+#include <Core/ServerSettings.h>
 #include <Poco/Util/AbstractConfiguration.h>
 
 namespace DB
diff --git a/src/Core/ServerSettings.h b/src/Core/ServerSettings.h
index 28b32a6e6a5..f2f78f70e91 100644
--- a/src/Core/ServerSettings.h
+++ b/src/Core/ServerSettings.h
@@ -66,6 +66,15 @@ namespace DB
     M(Bool, async_insert_queue_flush_on_shutdown, true, "If true queue of asynchronous inserts is flushed on graceful shutdown", 0) \
     M(Bool, ignore_empty_sql_security_in_create_view_query, true, "If true, ClickHouse doesn't write defaults for empty SQL security statement in CREATE VIEW queries. This setting is only necessary for the migration period and will become obsolete in 24.4", 0)  \
     \
+    /* Database Catalog */ \
+    M(UInt64, database_atomic_delay_before_drop_table_sec, 8 * 60, "The delay during which a dropped table can be restored using the UNDROP statement. If DROP TABLE ran with a SYNC modifier, the setting is ignored.", 0) \
+    M(UInt64, database_catalog_unused_dir_hide_timeout_sec, 60 * 60, "Parameter of a task that cleans up garbage from store/ directory. If some subdirectory is not used by clickhouse-server and this directory was not modified for last database_catalog_unused_dir_hide_timeout_sec seconds, the task will 'hide' this directory by removing all access rights. It also works for directories that clickhouse-server does not expect to see inside store/. Zero means 'immediately'.", 0) \
+    M(UInt64, database_catalog_unused_dir_rm_timeout_sec, 30 * 24 * 60 * 60, "Parameter of a task that cleans up garbage from store/ directory. If some subdirectory is not used by clickhouse-server and it was previously 'hidden' (see database_catalog_unused_dir_hide_timeout_sec) and this directory was not modified for last database_catalog_unused_dir_rm_timeout_sec seconds, the task will remove this directory. It also works for directories that clickhouse-server does not expect to see inside store/. Zero means 'never'.", 0) \
+    M(UInt64, database_catalog_unused_dir_cleanup_period_sec, 24 * 60 * 60, "Parameter of a task that cleans up garbage from store/ directory. Sets scheduling period of the task. Zero means 'never'.", 0) \
+    M(UInt64, database_catalog_drop_error_cooldown_sec, 5, "In case if drop table failed, ClickHouse will wait for this timeout before retrying the operation.", 0) \
+    M(UInt64, database_catalog_drop_table_concurrency, 16, "The size of the threadpool used for dropping tables.", 0) \
+    \
+    \
     M(UInt64, max_concurrent_queries, 0, "Maximum number of concurrently executed queries. Zero means unlimited.", 0) \
     M(UInt64, max_concurrent_insert_queries, 0, "Maximum number of concurrently INSERT queries. Zero means unlimited.", 0) \
     M(UInt64, max_concurrent_select_queries, 0, "Maximum number of concurrently SELECT queries. Zero means unlimited.", 0) \
diff --git a/src/IO/SharedThreadPools.cpp b/src/IO/SharedThreadPools.cpp
index 3606ddd984c..cda7bc01bbf 100644
--- a/src/IO/SharedThreadPools.cpp
+++ b/src/IO/SharedThreadPools.cpp
@@ -23,6 +23,9 @@ namespace CurrentMetrics
     extern const Metric MergeTreeUnexpectedPartsLoaderThreads;
     extern const Metric MergeTreeUnexpectedPartsLoaderThreadsActive;
     extern const Metric MergeTreeUnexpectedPartsLoaderThreadsScheduled;
+    extern const Metric DatabaseCatalogThreads;
+    extern const Metric DatabaseCatalogThreadsActive;
+    extern const Metric DatabaseCatalogThreadsScheduled;
     extern const Metric DatabaseReplicatedCreateTablesThreads;
     extern const Metric DatabaseReplicatedCreateTablesThreadsActive;
     extern const Metric DatabaseReplicatedCreateTablesThreadsScheduled;
@@ -166,4 +169,11 @@ StaticThreadPool & getDatabaseReplicatedCreateTablesThreadPool()
     return instance;
 }
 
+/// ThreadPool used for dropping tables.
+StaticThreadPool & getDatabaseCatalogDropTablesThreadPool()
+{
+    static StaticThreadPool instance("DropTablesThreadPool", CurrentMetrics::DatabaseCatalogThreads, CurrentMetrics::DatabaseCatalogThreadsActive, CurrentMetrics::DatabaseCatalogThreadsScheduled);
+    return instance;
+}
+
 }
diff --git a/src/IO/SharedThreadPools.h b/src/IO/SharedThreadPools.h
index 50adc70c9a0..06ccebd20b2 100644
--- a/src/IO/SharedThreadPools.h
+++ b/src/IO/SharedThreadPools.h
@@ -69,4 +69,7 @@ StaticThreadPool & getUnexpectedPartsLoadingThreadPool();
 /// ThreadPool used for creating tables in DatabaseReplicated.
 StaticThreadPool & getDatabaseReplicatedCreateTablesThreadPool();
 
+/// ThreadPool used for dropping tables.
+StaticThreadPool & getDatabaseCatalogDropTablesThreadPool();
+
 }
diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp
index bb2dd158710..f64f8a06f38 100644
--- a/src/Interpreters/DatabaseCatalog.cpp
+++ b/src/Interpreters/DatabaseCatalog.cpp
@@ -19,6 +19,8 @@
 #include <IO/ReadHelpers.h>
 #include <Poco/DirectoryIterator.h>
 #include <Poco/Util/AbstractConfiguration.h>
+#include <Core/ServerSettings.h>
+#include <IO/SharedThreadPools.h>
 #include <Common/Exception.h>
 #include <Common/quoteString.h>
 #include <Common/atomicRename.h>
@@ -189,13 +191,6 @@ StoragePtr TemporaryTableHolder::getTable() const
 
 void DatabaseCatalog::initializeAndLoadTemporaryDatabase()
 {
-    drop_delay_sec = getContext()->getConfigRef().getInt("database_atomic_delay_before_drop_table_sec", default_drop_delay_sec);
-    unused_dir_hide_timeout_sec = getContext()->getConfigRef().getInt64("database_catalog_unused_dir_hide_timeout_sec", unused_dir_hide_timeout_sec);
-    unused_dir_rm_timeout_sec = getContext()->getConfigRef().getInt64("database_catalog_unused_dir_rm_timeout_sec", unused_dir_rm_timeout_sec);
-    unused_dir_cleanup_period_sec = getContext()->getConfigRef().getInt64("database_catalog_unused_dir_cleanup_period_sec", unused_dir_cleanup_period_sec);
-    drop_error_cooldown_sec = getContext()->getConfigRef().getInt64("database_catalog_drop_error_cooldown_sec", drop_error_cooldown_sec);
-    drop_table_concurrency = getContext()->getConfigRef().getInt64("database_catalog_drop_table_concurrency", drop_table_concurrency);
-
     auto db_for_temporary_and_external_tables = std::make_shared<DatabaseMemory>(TEMPORARY_DATABASE, getContext());
     attachDatabase(TEMPORARY_DATABASE, db_for_temporary_and_external_tables);
 }
@@ -203,7 +198,7 @@ void DatabaseCatalog::initializeAndLoadTemporaryDatabase()
 void DatabaseCatalog::createBackgroundTasks()
 {
     /// It has to be done before databases are loaded (to avoid a race condition on initialization)
-    if (Context::getGlobalContextInstance()->getApplicationType() == Context::ApplicationType::SERVER && unused_dir_cleanup_period_sec)
+    if (Context::getGlobalContextInstance()->getApplicationType() == Context::ApplicationType::SERVER && getContext()->getServerSettings().database_catalog_unused_dir_cleanup_period_sec)
     {
         auto cleanup_task_holder
             = getContext()->getSchedulePool().createTask("DatabaseCatalogCleanupStoreDirectoryTask", [this]() { this->cleanupStoreDirectoryTask(); });
@@ -224,7 +219,7 @@ void DatabaseCatalog::startupBackgroundTasks()
     {
         (*cleanup_task)->activate();
         /// Do not start task immediately on server startup, it's not urgent.
-        (*cleanup_task)->scheduleAfter(unused_dir_hide_timeout_sec * 1000);
+        (*cleanup_task)->scheduleAfter(static_cast<time_t>(getContext()->getServerSettings().database_catalog_unused_dir_hide_timeout_sec) * 1000);
     }
 
     (*drop_task)->activate();
@@ -1038,15 +1033,12 @@ void DatabaseCatalog::loadMarkedAsDroppedTables()
 
     LOG_INFO(log, "Found {} partially dropped tables. Will load them and retry removal.", dropped_metadata.size());
 
-    ThreadPool pool(CurrentMetrics::DatabaseCatalogThreads, CurrentMetrics::DatabaseCatalogThreadsActive, CurrentMetrics::DatabaseCatalogThreadsScheduled);
+    ThreadPoolCallbackRunnerLocal<void> runner(getDatabaseCatalogDropTablesThreadPool().get(), "DropTables");
     for (const auto & elem : dropped_metadata)
     {
-        pool.scheduleOrThrowOnError([&]()
-        {
-            this->enqueueDroppedTableCleanup(elem.second, nullptr, elem.first);
-        });
+        runner([this, &elem](){ this->enqueueDroppedTableCleanup(elem.second, nullptr, elem.first); });
     }
-    pool.wait();
+    runner.waitForAllToFinishAndRethrowFirstError();
 }
 
 String DatabaseCatalog::getPathForDroppedMetadata(const StorageID & table_id) const
@@ -1135,7 +1127,13 @@ void DatabaseCatalog::enqueueDroppedTableCleanup(StorageID table_id, StoragePtr
     }
     else
     {
-        tables_marked_dropped.push_back({table_id, table, dropped_metadata_path, drop_time + drop_delay_sec});
+        tables_marked_dropped.push_back
+        ({
+            table_id,
+            table,
+            dropped_metadata_path,
+            drop_time + static_cast<time_t>(getContext()->getServerSettings().database_atomic_delay_before_drop_table_sec)
+        });
         if (first_async_drop_in_queue == tables_marked_dropped.end())
             --first_async_drop_in_queue;
     }
@@ -1289,13 +1287,7 @@ void DatabaseCatalog::dropTablesParallel(std::vector<DatabaseCatalog::TablesMark
     if (tables_to_drop.empty())
         return;
 
-    ThreadPool pool(
-        CurrentMetrics::DatabaseCatalogThreads,
-        CurrentMetrics::DatabaseCatalogThreadsActive,
-        CurrentMetrics::DatabaseCatalogThreadsScheduled,
-        /* max_threads */drop_table_concurrency,
-        /* max_free_threads */0,
-        /* queue_size */tables_to_drop.size());
+    ThreadPoolCallbackRunnerLocal<void> runner(getDatabaseCatalogDropTablesThreadPool().get(), "DropTables");
 
     for (const auto & item : tables_to_drop)
     {
@@ -1332,7 +1324,7 @@ void DatabaseCatalog::dropTablesParallel(std::vector<DatabaseCatalog::TablesMark
                         ++first_async_drop_in_queue;
 
                     tables_marked_dropped.splice(tables_marked_dropped.end(), tables_marked_dropped, table_iterator);
-                    table_iterator->drop_time = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()) + drop_error_cooldown_sec;
+                    table_iterator->drop_time = std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()) + getContext()->getServerSettings().database_catalog_drop_error_cooldown_sec;
 
                     if (first_async_drop_in_queue == tables_marked_dropped.end())
                         --first_async_drop_in_queue;
@@ -1342,7 +1334,7 @@ void DatabaseCatalog::dropTablesParallel(std::vector<DatabaseCatalog::TablesMark
 
         try
         {
-            pool.scheduleOrThrowOnError(std::move(job));
+            runner(std::move(job));
         }
         catch (...)
         {
@@ -1353,7 +1345,7 @@ void DatabaseCatalog::dropTablesParallel(std::vector<DatabaseCatalog::TablesMark
 
     try
     {
-        pool.wait();
+        runner.waitForAllToFinishAndRethrowFirstError();
     }
     catch (...)
     {
@@ -1718,7 +1710,7 @@ void DatabaseCatalog::cleanupStoreDirectoryTask()
             LOG_TEST(log, "Nothing to clean up from store/ on disk {}", disk_name);
     }
 
-    (*cleanup_task)->scheduleAfter(unused_dir_cleanup_period_sec * 1000);
+    (*cleanup_task)->scheduleAfter(static_cast<time_t>(getContext()->getServerSettings().database_catalog_unused_dir_cleanup_period_sec) * 1000);
 }
 
 bool DatabaseCatalog::maybeRemoveDirectory(const String & disk_name, const DiskPtr & disk, const String & unused_dir)
@@ -1742,7 +1734,7 @@ bool DatabaseCatalog::maybeRemoveDirectory(const String & disk_name, const DiskP
         time_t current_time = time(nullptr);
         if (st.st_mode & (S_IRWXU | S_IRWXG | S_IRWXO))
         {
-            if (current_time <= max_modification_time + unused_dir_hide_timeout_sec)
+            if (current_time <= max_modification_time + static_cast<time_t>(getContext()->getServerSettings().database_catalog_unused_dir_hide_timeout_sec))
                 return false;
 
             LOG_INFO(log, "Removing access rights for unused directory {} from disk {} (will remove it when timeout exceed)", unused_dir, disk_name);
@@ -1758,6 +1750,8 @@ bool DatabaseCatalog::maybeRemoveDirectory(const String & disk_name, const DiskP
         }
         else
         {
+            auto unused_dir_rm_timeout_sec = static_cast<time_t>(getContext()->getServerSettings().database_catalog_unused_dir_rm_timeout_sec);
+
             if (!unused_dir_rm_timeout_sec)
                 return false;
 
diff --git a/src/Interpreters/DatabaseCatalog.h b/src/Interpreters/DatabaseCatalog.h
index 23e38a6445e..83a302f117d 100644
--- a/src/Interpreters/DatabaseCatalog.h
+++ b/src/Interpreters/DatabaseCatalog.h
@@ -354,23 +354,8 @@ private:
     mutable std::mutex tables_marked_dropped_mutex;
 
     std::unique_ptr<BackgroundSchedulePoolTaskHolder> drop_task;
-    static constexpr time_t default_drop_delay_sec = 8 * 60;
-    time_t drop_delay_sec = default_drop_delay_sec;
     std::condition_variable wait_table_finally_dropped;
-
     std::unique_ptr<BackgroundSchedulePoolTaskHolder> cleanup_task;
-    static constexpr time_t default_unused_dir_hide_timeout_sec = 60 * 60;              /// 1 hour
-    time_t unused_dir_hide_timeout_sec = default_unused_dir_hide_timeout_sec;
-    static constexpr time_t default_unused_dir_rm_timeout_sec = 30 * 24 * 60 * 60;      /// 30 days
-    time_t unused_dir_rm_timeout_sec = default_unused_dir_rm_timeout_sec;
-    static constexpr time_t default_unused_dir_cleanup_period_sec = 24 * 60 * 60;       /// 1 day
-    time_t unused_dir_cleanup_period_sec = default_unused_dir_cleanup_period_sec;
-
-    static constexpr time_t default_drop_error_cooldown_sec = 5;
-    time_t drop_error_cooldown_sec = default_drop_error_cooldown_sec;
-
-    static constexpr size_t default_drop_table_concurrency = 10;
-    size_t drop_table_concurrency = default_drop_table_concurrency;
 
     std::unique_ptr<BackgroundSchedulePoolTaskHolder> reload_disks_task;
     std::mutex reload_disks_mutex;
diff --git a/src/Interpreters/InterpreterDropQuery.cpp b/src/Interpreters/InterpreterDropQuery.cpp
index bad3e5277db..d8056ddd1a3 100644
--- a/src/Interpreters/InterpreterDropQuery.cpp
+++ b/src/Interpreters/InterpreterDropQuery.cpp
@@ -6,6 +6,7 @@
 #include <Interpreters/InterpreterDropQuery.h>
 #include <Interpreters/ExternalDictionariesLoader.h>
 #include <Interpreters/QueryLog.h>
+#include <IO/SharedThreadPools.h>
 #include <Access/Common/AccessRightsElement.h>
 #include <Parsers/ASTDropQuery.h>
 #include <Parsers/ASTIdentifier.h>
@@ -424,18 +425,28 @@ BlockIO InterpreterDropQuery::executeToDatabaseImpl(const ASTDropQuery & query,
                 auto table_context = Context::createCopy(getContext());
                 table_context->setInternalQuery(true);
                 /// Do not hold extra shared pointers to tables
-                std::vector<std::pair<String, bool>> tables_to_drop;
+                std::vector<std::pair<StorageID, bool>> tables_to_drop;
                 // NOTE: This means we wait for all tables to be loaded inside getTablesIterator() call in case of `async_load_databases = true`.
                 for (auto iterator = database->getTablesIterator(table_context); iterator->isValid(); iterator->next())
                 {
                     auto table_ptr = iterator->table();
-                    table_ptr->flushAndPrepareForShutdown();
-                    tables_to_drop.push_back({iterator->name(), table_ptr->isDictionary()});
+                    tables_to_drop.push_back({table_ptr->getStorageID(), table_ptr->isDictionary()});
                 }
 
+                /// Prepare tables for shutdown in parallel.
+                ThreadPoolCallbackRunnerLocal<void> runner(getDatabaseCatalogDropTablesThreadPool().get(), "DropTables");
+                for (const auto & [name, _] : tables_to_drop)
+                {
+                    auto table_ptr = DatabaseCatalog::instance().getTable(name, table_context);
+                    runner([my_table_ptr = std::move(table_ptr)](){
+                        my_table_ptr->flushAndPrepareForShutdown();
+                    });
+                }
+                runner.waitForAllToFinishAndRethrowFirstError();
+
                 for (const auto & table : tables_to_drop)
                 {
-                    query_for_table.setTable(table.first);
+                    query_for_table.setTable(table.first.getTableName());
                     query_for_table.is_dictionary = table.second;
                     DatabasePtr db;
                     UUID table_to_wait = UUIDHelpers::Nil;

From 1427b16689601d7dd29d26de99b233c132905fde Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <nikitamikhaylov@clickhouse.com>
Date: Fri, 26 Jul 2024 21:44:15 +0200
Subject: [PATCH 1033/1488] Fixed style

---
 src/Interpreters/DatabaseCatalog.cpp      | 3 ---
 src/Interpreters/InterpreterDropQuery.cpp | 3 ++-
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp
index f64f8a06f38..30b151eb81d 100644
--- a/src/Interpreters/DatabaseCatalog.cpp
+++ b/src/Interpreters/DatabaseCatalog.cpp
@@ -50,9 +50,6 @@
 namespace CurrentMetrics
 {
     extern const Metric TablesToDropQueueSize;
-    extern const Metric DatabaseCatalogThreads;
-    extern const Metric DatabaseCatalogThreadsActive;
-    extern const Metric DatabaseCatalogThreadsScheduled;
 }
 
 namespace DB
diff --git a/src/Interpreters/InterpreterDropQuery.cpp b/src/Interpreters/InterpreterDropQuery.cpp
index d8056ddd1a3..ef560ec3405 100644
--- a/src/Interpreters/InterpreterDropQuery.cpp
+++ b/src/Interpreters/InterpreterDropQuery.cpp
@@ -438,7 +438,8 @@ BlockIO InterpreterDropQuery::executeToDatabaseImpl(const ASTDropQuery & query,
                 for (const auto & [name, _] : tables_to_drop)
                 {
                     auto table_ptr = DatabaseCatalog::instance().getTable(name, table_context);
-                    runner([my_table_ptr = std::move(table_ptr)](){
+                    runner([my_table_ptr = std::move(table_ptr)]()
+                    {
                         my_table_ptr->flushAndPrepareForShutdown();
                     });
                 }

From 1096a4ff33497c64eb786f6dbb603b18ccf804b1 Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <nikitamikhaylov@clickhouse.com>
Date: Mon, 29 Jul 2024 13:37:36 +0000
Subject: [PATCH 1034/1488] Fixed occasional LOGICAL_ERROR

---
 src/Interpreters/DatabaseCatalog.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp
index 30b151eb81d..eaf8cf1cc82 100644
--- a/src/Interpreters/DatabaseCatalog.cpp
+++ b/src/Interpreters/DatabaseCatalog.cpp
@@ -1336,7 +1336,7 @@ void DatabaseCatalog::dropTablesParallel(std::vector<DatabaseCatalog::TablesMark
         catch (...)
         {
             tryLogCurrentException(log, "Cannot drop tables. Will retry later.");
-            break;
+            throw;
         }
     }
 
@@ -1347,6 +1347,7 @@ void DatabaseCatalog::dropTablesParallel(std::vector<DatabaseCatalog::TablesMark
     catch (...)
     {
         tryLogCurrentException(log, "Cannot drop tables. Will retry later.");
+        throw;
     }
 }
 

From 77f0180ff833c776789e2ef0aeedf224fe3eaadf Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <mikhaylovnikitka@gmail.com>
Date: Tue, 30 Jul 2024 00:39:16 +0200
Subject: [PATCH 1035/1488] Dont throw

---
 src/Interpreters/DatabaseCatalog.cpp | 30 ++++++++++------------------
 1 file changed, 11 insertions(+), 19 deletions(-)

diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp
index eaf8cf1cc82..98526e5c1cd 100644
--- a/src/Interpreters/DatabaseCatalog.cpp
+++ b/src/Interpreters/DatabaseCatalog.cpp
@@ -1329,26 +1329,10 @@ void DatabaseCatalog::dropTablesParallel(std::vector<DatabaseCatalog::TablesMark
             }
         };
 
-        try
-        {
-            runner(std::move(job));
-        }
-        catch (...)
-        {
-            tryLogCurrentException(log, "Cannot drop tables. Will retry later.");
-            throw;
-        }
+        runner(std::move(job));
     }
 
-    try
-    {
-        runner.waitForAllToFinishAndRethrowFirstError();
-    }
-    catch (...)
-    {
-        tryLogCurrentException(log, "Cannot drop tables. Will retry later.");
-        throw;
-    }
+    runner.waitForAllToFinishAndRethrowFirstError();
 }
 
 void DatabaseCatalog::dropTableDataTask()
@@ -1365,7 +1349,15 @@ void DatabaseCatalog::dropTableDataTask()
         LOG_INFO(log, "Have {} tables in drop queue ({} of them are in use), will try drop {} tables",
             drop_tables_count, drop_tables_in_use_count, tables_to_drop.size());
 
-        dropTablesParallel(tables_to_drop);
+        try
+        {
+            dropTablesParallel(tables_to_drop);
+        }
+        catch (...)
+        {
+            /// We don't re-throw expection, because we are in a background pool.
+            tryLogCurrentException(log, "Cannot drop tables. Will retry later.");
+        }
     }
 
     rescheduleDropTableTask();

From 4eb2bf3419d7f3640ec8b77bc0ca80a138cd2584 Mon Sep 17 00:00:00 2001
From: Michael Kolupaev <michael.kolupaev@clickhouse.com>
Date: Mon, 29 Jul 2024 23:10:13 +0000
Subject: [PATCH 1036/1488] Fix Dwarf range list parsing in stack symbolizer

---
 src/Common/Dwarf.cpp | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/Common/Dwarf.cpp b/src/Common/Dwarf.cpp
index 8439c01b22c..1f22e3b05aa 100644
--- a/src/Common/Dwarf.cpp
+++ b/src/Common/Dwarf.cpp
@@ -1559,8 +1559,7 @@ bool Dwarf::isAddrInRangeList(const CompilationUnit & cu,
                     auto sp_start = addr_.substr(*cu.addr_base + index_start * sizeof(uint64_t));
                     auto start = read<uint64_t>(sp_start);
 
-                    auto sp_end = addr_.substr(*cu.addr_base + index_start * sizeof(uint64_t) + length);
-                    auto end = read<uint64_t>(sp_end);
+                    auto end = start + length;
                     if (start != end && address >= start && address < end)
                     {
                         return true;

From 4a42ddc18e57576d119b1416f1da06b9ec292fce Mon Sep 17 00:00:00 2001
From: Michael Kolupaev <michael.kolupaev@clickhouse.com>
Date: Sat, 27 Jul 2024 02:36:54 +0000
Subject: [PATCH 1037/1488] Make Dwarf::findAddress() fallback slow path less
 slow

---
 src/Common/Dwarf.cpp | 42 ++++++++++++++++++++++++++++++++++++++----
 src/Common/Dwarf.h   |  3 ++-
 2 files changed, 40 insertions(+), 5 deletions(-)

diff --git a/src/Common/Dwarf.cpp b/src/Common/Dwarf.cpp
index 8439c01b22c..eda479607b1 100644
--- a/src/Common/Dwarf.cpp
+++ b/src/Common/Dwarf.cpp
@@ -1029,7 +1029,8 @@ bool Dwarf::findLocation(
     const LocationInfoMode mode,
     CompilationUnit & cu,
     LocationInfo & info,
-    std::vector<SymbolizedFrame> & inline_frames) const
+    std::vector<SymbolizedFrame> & inline_frames,
+    bool assume_in_cu_range) const
 {
     Die die = getDieAtOffset(cu, cu.first_die);
     // Partial compilation unit (DW_TAG_partial_unit) is not supported.
@@ -1041,6 +1042,11 @@ bool Dwarf::findLocation(
     std::optional<std::string_view> main_file_name;
     std::optional<uint64_t> base_addr_cu;
 
+    std::optional<uint64_t> low_pc;
+    std::optional<uint64_t> high_pc;
+    std::optional<bool> is_high_pc_addr;
+    std::optional<uint64_t> range_offset;
+
     forEachAttribute(cu, die, [&](const Attribute & attr)
     {
         switch (attr.spec.name) // NOLINT(bugprone-switch-missing-default-case)
@@ -1058,18 +1064,46 @@ bool Dwarf::findLocation(
                 // File name of main file being compiled
                 main_file_name = std::get<std::string_view>(attr.attr_value);
                 break;
-            case DW_AT_low_pc:
             case DW_AT_entry_pc:
                 // 2.17.1: historically DW_AT_low_pc was used. DW_AT_entry_pc was
                 // introduced in DWARF3. Support either to determine the base address of
                 // the CU.
                 base_addr_cu = std::get<uint64_t>(attr.attr_value);
                 break;
+            case DW_AT_ranges:
+                range_offset = std::get<uint64_t>(attr.attr_value);
+                break;
+            case DW_AT_low_pc:
+                low_pc = std::get<uint64_t>(attr.attr_value);
+                base_addr_cu = std::get<uint64_t>(attr.attr_value);
+                break;
+            case DW_AT_high_pc:
+                // The value of the DW_AT_high_pc attribute can be
+                // an address (DW_FORM_addr*) or an offset (DW_FORM_data*).
+                is_high_pc_addr = attr.spec.form == DW_FORM_addr || //
+                    attr.spec.form == DW_FORM_addrx || //
+                    attr.spec.form == DW_FORM_addrx1 || //
+                    attr.spec.form == DW_FORM_addrx2 || //
+                    attr.spec.form == DW_FORM_addrx3 || //
+                    attr.spec.form == DW_FORM_addrx4;
+                high_pc = std::get<uint64_t>(attr.attr_value);
+                break;
         }
         // Iterate through all attributes until find all above.
         return true;
     });
 
+    /// Check if the address falls inside this unit's address ranges.
+    if (!assume_in_cu_range && ((low_pc && high_pc) || range_offset)) {
+        bool pc_match = low_pc && high_pc && is_high_pc_addr && address >= *low_pc
+            && (address < (*is_high_pc_addr ? *high_pc : *low_pc + *high_pc));
+        bool range_match = range_offset && isAddrInRangeList(cu, address, base_addr_cu, range_offset.value(), cu.addr_size);
+        if (!pc_match && !range_match)
+        {
+            return false;
+        }
+    }
+
     if (main_file_name)
     {
         info.has_main_file = true;
@@ -1442,7 +1476,7 @@ bool Dwarf::findAddress(
             {
                 return false;
             }
-            findLocation(address, mode, unit, locationInfo, inline_frames);
+            findLocation(address, mode, unit, locationInfo, inline_frames, /*assume_in_cu_range*/ true);
             return locationInfo.has_file_and_line;
         }
         else if (mode == LocationInfoMode::FAST)
@@ -1471,7 +1505,7 @@ bool Dwarf::findAddress(
         {
             continue;
         }
-        findLocation(address, mode, unit, locationInfo, inline_frames);
+        findLocation(address, mode, unit, locationInfo, inline_frames, /*assume_in_cu_range*/ false);
     }
 
     return locationInfo.has_file_and_line;
diff --git a/src/Common/Dwarf.h b/src/Common/Dwarf.h
index da18b3affa0..d754191bfa9 100644
--- a/src/Common/Dwarf.h
+++ b/src/Common/Dwarf.h
@@ -283,7 +283,8 @@ private:
         LocationInfoMode mode,
         CompilationUnit & cu,
         LocationInfo & info,
-        std::vector<SymbolizedFrame> & inline_frames) const;
+        std::vector<SymbolizedFrame> & inline_frames,
+        bool assume_in_cu_range) const;
 
     /**
      * Finds a subprogram debugging info entry that contains a given address among

From b0629726a05701341e5d96207fe5e3743cd9345b Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Tue, 30 Jul 2024 03:23:19 +0200
Subject: [PATCH 1038/1488] Update src/Common/Dwarf.cpp

---
 src/Common/Dwarf.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/Common/Dwarf.cpp b/src/Common/Dwarf.cpp
index eda479607b1..f17219e9bf5 100644
--- a/src/Common/Dwarf.cpp
+++ b/src/Common/Dwarf.cpp
@@ -1094,7 +1094,8 @@ bool Dwarf::findLocation(
     });
 
     /// Check if the address falls inside this unit's address ranges.
-    if (!assume_in_cu_range && ((low_pc && high_pc) || range_offset)) {
+    if (!assume_in_cu_range && ((low_pc && high_pc) || range_offset))
+    {
         bool pc_match = low_pc && high_pc && is_high_pc_addr && address >= *low_pc
             && (address < (*is_high_pc_addr ? *high_pc : *low_pc + *high_pc));
         bool range_match = range_offset && isAddrInRangeList(cu, address, base_addr_cu, range_offset.value(), cu.addr_size);

From 8f920d064ccca8ed8d9341b10d54ebef7500484c Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Tue, 30 Jul 2024 04:05:43 +0200
Subject: [PATCH 1039/1488] Fix inconsistent formatting of CODEC and STATISTICS

---
 src/Parsers/ASTFunction.cpp                    | 4 +++-
 src/Parsers/ASTFunction.h                      | 2 ++
 src/Parsers/ExpressionElementParsers.cpp       | 2 ++
 src/Parsers/FunctionSecretArgumentsFinderAST.h | 4 +++-
 src/Parsers/IAST.h                             | 1 +
 src/Storages/StatisticsDescription.cpp         | 1 +
 6 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/src/Parsers/ASTFunction.cpp b/src/Parsers/ASTFunction.cpp
index 230d4c778e8..cd9e910d45a 100644
--- a/src/Parsers/ASTFunction.cpp
+++ b/src/Parsers/ASTFunction.cpp
@@ -285,6 +285,8 @@ static bool formatNamedArgWithHiddenValue(IAST * arg, const IAST::FormatSettings
 void ASTFunction::formatImplWithoutAlias(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const
 {
     frame.expression_list_prepend_whitespace = false;
+    if (kind == Kind::CODEC || kind == Kind::STATISTICS || kind == Kind::BACKUP_NAME)
+        frame.allow_operators = false;
     FormatStateStacked nested_need_parens = frame;
     FormatStateStacked nested_dont_need_parens = frame;
     nested_need_parens.need_parens = true;
@@ -308,7 +310,7 @@ void ASTFunction::formatImplWithoutAlias(const FormatSettings & settings, Format
 
     /// Should this function to be written as operator?
     bool written = false;
-    if (arguments && !parameters && nulls_action == NullsAction::EMPTY)
+    if (arguments && !parameters && frame.allow_operators && nulls_action == NullsAction::EMPTY)
     {
         /// Unary prefix operators.
         if (arguments->children.size() == 1)
diff --git a/src/Parsers/ASTFunction.h b/src/Parsers/ASTFunction.h
index be2b6beae54..1b4a5928d1c 100644
--- a/src/Parsers/ASTFunction.h
+++ b/src/Parsers/ASTFunction.h
@@ -58,6 +58,8 @@ public:
         TABLE_ENGINE,
         DATABASE_ENGINE,
         BACKUP_NAME,
+        CODEC,
+        STATISTICS,
     };
     Kind kind = Kind::ORDINARY_FUNCTION;
 
diff --git a/src/Parsers/ExpressionElementParsers.cpp b/src/Parsers/ExpressionElementParsers.cpp
index 865d07faaa7..9927acdcf17 100644
--- a/src/Parsers/ExpressionElementParsers.cpp
+++ b/src/Parsers/ExpressionElementParsers.cpp
@@ -696,6 +696,7 @@ bool ParserCodec::parseImpl(Pos & pos, ASTPtr & node, Expected & expected)
 
     auto function_node = std::make_shared<ASTFunction>();
     function_node->name = "CODEC";
+    function_node->kind = ASTFunction::Kind::CODEC;
     function_node->arguments = expr_list_args;
     function_node->children.push_back(function_node->arguments);
 
@@ -723,6 +724,7 @@ bool ParserStatisticsType::parseImpl(Pos & pos, ASTPtr & node, Expected & expect
 
     auto function_node = std::make_shared<ASTFunction>();
     function_node->name = "STATISTICS";
+    function_node->kind = ASTFunction::Kind::STATISTICS;
     function_node->arguments = stat_type;
     function_node->children.push_back(function_node->arguments);
     node = function_node;
diff --git a/src/Parsers/FunctionSecretArgumentsFinderAST.h b/src/Parsers/FunctionSecretArgumentsFinderAST.h
index 5b77485afb0..94da30922cc 100644
--- a/src/Parsers/FunctionSecretArgumentsFinderAST.h
+++ b/src/Parsers/FunctionSecretArgumentsFinderAST.h
@@ -33,7 +33,9 @@ public:
         {
             case ASTFunction::Kind::ORDINARY_FUNCTION: findOrdinaryFunctionSecretArguments(); break;
             case ASTFunction::Kind::WINDOW_FUNCTION: break;
-            case ASTFunction::Kind::LAMBDA_FUNCTION:  break;
+            case ASTFunction::Kind::LAMBDA_FUNCTION: break;
+            case ASTFunction::Kind::CODEC: break;
+            case ASTFunction::Kind::STATISTICS: break;
             case ASTFunction::Kind::TABLE_ENGINE: findTableEngineSecretArguments(); break;
             case ASTFunction::Kind::DATABASE_ENGINE: findDatabaseEngineSecretArguments(); break;
             case ASTFunction::Kind::BACKUP_NAME: findBackupNameSecretArguments(); break;
diff --git a/src/Parsers/IAST.h b/src/Parsers/IAST.h
index d70c1cd0b6c..e2cf7579667 100644
--- a/src/Parsers/IAST.h
+++ b/src/Parsers/IAST.h
@@ -256,6 +256,7 @@ public:
         bool expression_list_always_start_on_new_line = false;  /// Line feed and indent before expression list even if it's of single element.
         bool expression_list_prepend_whitespace = false; /// Prepend whitespace (if it is required)
         bool surround_each_list_element_with_parens = false;
+        bool allow_operators = true; /// Format some functions, such as "plus", "in", etc. as operators.
         size_t list_element_index = 0;
         const IAST * current_select = nullptr;
     };
diff --git a/src/Storages/StatisticsDescription.cpp b/src/Storages/StatisticsDescription.cpp
index 9c5fd3604b2..63c849e3806 100644
--- a/src/Storages/StatisticsDescription.cpp
+++ b/src/Storages/StatisticsDescription.cpp
@@ -193,6 +193,7 @@ ASTPtr ColumnStatisticsDescription::getAST() const
 {
     auto function_node = std::make_shared<ASTFunction>();
     function_node->name = "STATISTICS";
+    function_node->kind = ASTFunction::Kind::STATISTICS;
     function_node->arguments = std::make_shared<ASTExpressionList>();
     for (const auto & [type, desc] : types_to_desc)
     {

From c1e7b7be89f5c5e39318ab093b00fa6cd8114ff3 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Tue, 30 Jul 2024 04:07:46 +0200
Subject: [PATCH 1040/1488] Add a test

---
 ...nconsistent_formatting_of_codecs_statistics.reference | 2 ++
 ...03214_inconsistent_formatting_of_codecs_statistics.sh | 9 +++++++++
 2 files changed, 11 insertions(+)
 create mode 100644 tests/queries/0_stateless/03214_inconsistent_formatting_of_codecs_statistics.reference
 create mode 100755 tests/queries/0_stateless/03214_inconsistent_formatting_of_codecs_statistics.sh

diff --git a/tests/queries/0_stateless/03214_inconsistent_formatting_of_codecs_statistics.reference b/tests/queries/0_stateless/03214_inconsistent_formatting_of_codecs_statistics.reference
new file mode 100644
index 00000000000..7213baa3e5b
--- /dev/null
+++ b/tests/queries/0_stateless/03214_inconsistent_formatting_of_codecs_statistics.reference
@@ -0,0 +1,2 @@
+ALTER TABLE t MODIFY COLUMN `c` CODEC(in(1, 2))
+ALTER TABLE t MODIFY COLUMN `c` STATISTICS(plus(1, 2))
diff --git a/tests/queries/0_stateless/03214_inconsistent_formatting_of_codecs_statistics.sh b/tests/queries/0_stateless/03214_inconsistent_formatting_of_codecs_statistics.sh
new file mode 100755
index 00000000000..c3f8d89b9a4
--- /dev/null
+++ b/tests/queries/0_stateless/03214_inconsistent_formatting_of_codecs_statistics.sh
@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+# Ensure that these (possibly incorrect) queries can at least be parsed back after formatting.
+$CLICKHOUSE_FORMAT --oneline --query "ALTER TABLE t MODIFY COLUMN c CODEC(in(1, 2))" | $CLICKHOUSE_FORMAT --oneline
+$CLICKHOUSE_FORMAT --oneline --query "ALTER TABLE t MODIFY COLUMN c STATISTICS(plus(1, 2))" | $CLICKHOUSE_FORMAT --oneline

From cb6b6329c8e763f61f70797a95dab8ef24fd47d1 Mon Sep 17 00:00:00 2001
From: kevinyhzou <kevinyunhe8@gmail.com>
Date: Tue, 30 Jul 2024 10:45:36 +0800
Subject: [PATCH 1041/1488] add session timezone settings

---
 tests/queries/0_stateless/03198_orc_read_time_zone.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/03198_orc_read_time_zone.sh b/tests/queries/0_stateless/03198_orc_read_time_zone.sh
index 7e931e16e48..7d1da0c1579 100755
--- a/tests/queries/0_stateless/03198_orc_read_time_zone.sh
+++ b/tests/queries/0_stateless/03198_orc_read_time_zone.sh
@@ -8,5 +8,5 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 $CLICKHOUSE_CLIENT -q "drop table if exists test_orc_read_timezone"
 $CLICKHOUSE_CLIENT -q "create table test_orc_read_timezone(id UInt64, t DateTime64) Engine=MergeTree order by id"
 $CLICKHOUSE_CLIENT -q "insert into test_orc_read_timezone from infile '$CURDIR/data_orc/test_reader_time_zone.snappy.orc' SETTINGS input_format_orc_reader_time_zone_name='Asia/Shanghai' FORMAT ORC"
-$CLICKHOUSE_CLIENT -q "select * from test_orc_read_timezone"
+$CLICKHOUSE_CLIENT -q "select * from test_orc_read_timezone SETTINGS session_timezone='Asia/Shanghai'"
 $CLICKHOUSE_CLIENT -q "drop table test_orc_read_timezone"
\ No newline at end of file

From dd5819ab6ab2df3231737a4808d0445ad5345555 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Tue, 30 Jul 2024 05:16:09 +0200
Subject: [PATCH 1042/1488] Changelog sanity

---
 CHANGELOG.md | 55 +++++++++++++++++++++++++---------------------------
 1 file changed, 26 insertions(+), 29 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 620b7c99bac..06f7bcdd84e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -18,6 +18,7 @@
 * Metric `KeeperOutstandingRequets` was renamed to `KeeperOutstandingRequests`. [#66206](https://github.com/ClickHouse/ClickHouse/pull/66206) ([Robert Schulze](https://github.com/rschu1ze)).
 * Remove `is_deterministic` field from the `system.functions` table. [#66630](https://github.com/ClickHouse/ClickHouse/pull/66630) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
 * Function `tuple` will now try to construct named tuples in query (controlled by `enable_named_columns_in_function_tuple`). Introduce function `tupleNames` to extract names from tuples. [#54881](https://github.com/ClickHouse/ClickHouse/pull/54881) ([Amos Bird](https://github.com/amosbird)).
+* Change how deduplication for Materialized Views works. Fixed a lot of cases like: - on destination table: data is split for 2 or more blocks and that blocks is considered as duplicate when that block is inserted in parallel. - on MV destination table: the equal blocks are deduplicated, that happens when MV often produces equal data as a result for different input data due to performing aggregation. - on MV destination table: the equal blocks which comes from different MV are deduplicated. [#61601](https://github.com/ClickHouse/ClickHouse/pull/61601) ([Sema Checherinda](https://github.com/CheSema)).
 
 #### New Feature
 * Add `ASOF JOIN` support for `full_sorting_join` algorithm. [#55051](https://github.com/ClickHouse/ClickHouse/pull/55051) ([vdimir](https://github.com/vdimir)).
@@ -32,13 +33,14 @@
 * Add a new setting to disable/enable writing page index into parquet files. [#65475](https://github.com/ClickHouse/ClickHouse/pull/65475) ([lgbo](https://github.com/lgbo-ustc)).
 * Introduce `logger.console_log_level` server config to control the log level to the console (if enabled). [#65559](https://github.com/ClickHouse/ClickHouse/pull/65559) ([Azat Khuzhin](https://github.com/azat)).
 * Automatically append a wildcard `*` to the end of a directory path with table function `file`. [#66019](https://github.com/ClickHouse/ClickHouse/pull/66019) ([Zhidong (David) Guo](https://github.com/Gun9niR)).
-* Add `--memory-usage` option to client in non interactive mode. [#66393](https://github.com/ClickHouse/ClickHouse/pull/66393) ([vdimir](https://github.com/vdimir)).
+* Add `--memory-usage` option to client in non-interactive mode. [#66393](https://github.com/ClickHouse/ClickHouse/pull/66393) ([vdimir](https://github.com/vdimir)).
 * Make an interactive client for clickhouse-disks, add local disk from the local directory. [#64446](https://github.com/ClickHouse/ClickHouse/pull/64446) ([Daniil Ivanik](https://github.com/divanik)).
 * When lightweight delete happens on a table with projection(s), users have choices either throw an exception (by default) or drop the projection [#65594](https://github.com/ClickHouse/ClickHouse/pull/65594) ([jsc0218](https://github.com/jsc0218)).
+* Add system tables with main information about all detached tables. [#65400](https://github.com/ClickHouse/ClickHouse/pull/65400) ([Konstantin Morozov](https://github.com/k-morozov)).
 
 #### Experimental Feature
 * Change binary serialization of Variant data type: add `compact` mode to avoid writing the same discriminator multiple times for granules with single variant or with only NULL values. Add MergeTree setting `use_compact_variant_discriminators_serialization` that is enabled by default. Note that Variant type is still experimental and backward-incompatible change in serialization is ok. [#62774](https://github.com/ClickHouse/ClickHouse/pull/62774) ([Kruglov Pavel](https://github.com/Avogar)).
-* Support rocksdb as backend storage of keeper. [#56626](https://github.com/ClickHouse/ClickHouse/pull/56626) ([Han Fei](https://github.com/hanfei1991)).
+* Support rocksdb as backend storage of clickhouse-keeper. [#56626](https://github.com/ClickHouse/ClickHouse/pull/56626) ([Han Fei](https://github.com/hanfei1991)).
 * Refactor JSONExtract functions, support more types including experimental Dynamic type. [#66046](https://github.com/ClickHouse/ClickHouse/pull/66046) ([Kruglov Pavel](https://github.com/Avogar)).
 * Support null map subcolumn for Variant and Dynamic subcolumns. [#66178](https://github.com/ClickHouse/ClickHouse/pull/66178) ([Kruglov Pavel](https://github.com/Avogar)).
 * Fix reading dynamic subcolumns from altered Memory table. Previously if `max_types` parameter of a Dynamic type was changed in Memory table via alter, further subcolumns reading can return wrong result. [#66066](https://github.com/ClickHouse/ClickHouse/pull/66066) ([Kruglov Pavel](https://github.com/Avogar)).
@@ -46,8 +48,8 @@
 
 #### Performance Improvement
 * Replace int to string algorithm with a faster one (from a modified amdn/itoa to a modified jeaiii/itoa). [#61661](https://github.com/ClickHouse/ClickHouse/pull/61661) ([Raúl Marín](https://github.com/Algunenano)).
-* Sizes of hash tables created by join (`parallel_hash` algorithm) is collected and cached now. This information will be used to preallocate space in hash tables for subsequent query executions and save time on hash table resizes. [#64553](https://github.com/ClickHouse/ClickHouse/pull/64553) ([Nikita Taranov](https://github.com/nickitat)).
-* Optimized queries with `ORDER BY` primary key and `WHERE` that have a condition with high selectivity by using of buffering. It is controlled by setting `read_in_order_use_buffering` (enabled by default) and can increase memory usage of query. [#64607](https://github.com/ClickHouse/ClickHouse/pull/64607) ([Anton Popov](https://github.com/CurtizJ)).
+* Sizes of hash tables created by join (`parallel_hash` algorithm) are collected and cached now. This information will be used to preallocate space in hash tables for subsequent query executions and save time on hash table resizes. [#64553](https://github.com/ClickHouse/ClickHouse/pull/64553) ([Nikita Taranov](https://github.com/nickitat)).
+* Optimized queries with `ORDER BY` primary key and `WHERE` that have a condition with high selectivity by using buffering. It is controlled by setting `read_in_order_use_buffering` (enabled by default) and can increase memory usage of query. [#64607](https://github.com/ClickHouse/ClickHouse/pull/64607) ([Anton Popov](https://github.com/CurtizJ)).
 * Improve performance of loading `plain_rewritable` metadata. [#65634](https://github.com/ClickHouse/ClickHouse/pull/65634) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
 * Attaching tables on read-only disks will use fewer resources by not loading outdated parts. [#65635](https://github.com/ClickHouse/ClickHouse/pull/65635) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
 * Support minmax hyperrectangle for Set indices. [#65676](https://github.com/ClickHouse/ClickHouse/pull/65676) ([AntiTopQuark](https://github.com/AntiTopQuark)).
@@ -59,11 +61,11 @@
 * DatabaseCatalog drops tables faster by using up to database_catalog_drop_table_concurrency threads. [#66065](https://github.com/ClickHouse/ClickHouse/pull/66065) ([Sema Checherinda](https://github.com/CheSema)).
 
 #### Improvement
+* Improved ZooKeeper load balancing. The current session doesn't expire until the optimal nodes become available despite `fallback_session_lifetime`. Added support for AZ-aware balancing. [#65570](https://github.com/ClickHouse/ClickHouse/pull/65570) ([Alexander Tokmakov](https://github.com/tavplubix)).
 * The setting `optimize_trivial_insert_select` is disabled by default. In most cases, it should be beneficial. Nevertheless, if you are seeing slower INSERT SELECT or increased memory usage, you can enable it back or `SET compatibility = '24.6'`. [#58970](https://github.com/ClickHouse/ClickHouse/pull/58970) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
 * Print stacktrace and diagnostic info if `clickhouse-client` or `clickhouse-local` crashes. [#61109](https://github.com/ClickHouse/ClickHouse/pull/61109) ([Alexander Tokmakov](https://github.com/tavplubix)).
 * The result of `SHOW INDEX | INDEXES | INDICES | KEYS` was previously sorted by the primary key column names. Since this was unintuitive, the result is now sorted by the position of the primary key columns within the primary key. [#61131](https://github.com/ClickHouse/ClickHouse/pull/61131) ([Robert Schulze](https://github.com/rschu1ze)).
-* Change how deduplication for Materialized Views works. Fixed a lot of cases like: - on destination table: data is split for 2 or more blocks and that blocks is considered as duplicate when that block is inserted in parallel. - on MV destination table: the equal blocks are deduplicated, that happens when MV often produces equal data as a result for different input data due to performing aggregation. - on MV destination table: the equal blocks which comes from different MV are deduplicated. [#61601](https://github.com/ClickHouse/ClickHouse/pull/61601) ([Sema Checherinda](https://github.com/CheSema)).
-* Allow matching column names in a case insensitive manner when reading json files (`input_format_json_case_insensitive_column_matching`). [#61750](https://github.com/ClickHouse/ClickHouse/pull/61750) ([kevinyhzou](https://github.com/KevinyhZou)).
+* Allow matching column names in a case-insensitive manner when reading json files (`input_format_json_case_insensitive_column_matching`). [#61750](https://github.com/ClickHouse/ClickHouse/pull/61750) ([kevinyhzou](https://github.com/KevinyhZou)).
 * Support reading partitioned data DeltaLake data. Infer DeltaLake schema by reading metadata instead of data. [#63201](https://github.com/ClickHouse/ClickHouse/pull/63201) ([Kseniia Sumarokova](https://github.com/kssenii)).
 * In composable protocols TLS layer accepted only `certificateFile` and `privateKeyFile` parameters. https://clickhouse.com/docs/en/operations/settings/composable-protocols. [#63985](https://github.com/ClickHouse/ClickHouse/pull/63985) ([Anton Ivashkin](https://github.com/ianton-ru)).
 * Added profile event `SelectQueriesWithPrimaryKeyUsage` which indicates how many SELECT queries use the primary key to evaluate the WHERE clause. [#64492](https://github.com/ClickHouse/ClickHouse/pull/64492) ([0x01f](https://github.com/0xfei)).
@@ -71,7 +73,6 @@
 * Support aliases in parametrized view function (only new analyzer). [#65190](https://github.com/ClickHouse/ClickHouse/pull/65190) ([Kseniia Sumarokova](https://github.com/kssenii)).
 * Updated to mask account key in logs in azureBlobStorage. [#65273](https://github.com/ClickHouse/ClickHouse/pull/65273) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)).
 * Partition pruning for `IN` predicates when filter expression is a part of `PARTITION BY` expression. [#65335](https://github.com/ClickHouse/ClickHouse/pull/65335) ([Eduard Karacharov](https://github.com/korowa)).
-* Add system tables with main information about all detached tables. [#65400](https://github.com/ClickHouse/ClickHouse/pull/65400) ([Konstantin Morozov](https://github.com/k-morozov)).
 * `arrayMin`/`arrayMax` can be applicable to all data types that are comparable. [#65455](https://github.com/ClickHouse/ClickHouse/pull/65455) ([pn](https://github.com/chloro-pn)).
 * Improved memory accounting for cgroups v2 to exclude the amount occupied by the page cache. [#65470](https://github.com/ClickHouse/ClickHouse/pull/65470) ([Nikita Taranov](https://github.com/nickitat)).
 * Do not create format settings for each row when serializing chunks to insert to EmbeddedRocksDB table. [#65474](https://github.com/ClickHouse/ClickHouse/pull/65474) ([Duc Canh Le](https://github.com/canhld94)).
@@ -80,36 +81,35 @@
 * Disable filesystem cache background download by default. It will be enabled back when we fix the issue with possible "Memory limit exceeded" because memory deallocation is done outside of query context (while buffer is allocated inside of query context) if we use background download threads. Plus we need to add a separate setting to define max size to download for background workers (currently it is limited by max_file_segment_size, which might be too big). [#65534](https://github.com/ClickHouse/ClickHouse/pull/65534) ([Kseniia Sumarokova](https://github.com/kssenii)).
 * Add new option to config `<config_reload_interval_ms>` which allow to specify how often clickhouse will reload config. [#65545](https://github.com/ClickHouse/ClickHouse/pull/65545) ([alesapin](https://github.com/alesapin)).
 * Implement binary encoding for ClickHouse data types and add its specification in docs. Use it in Dynamic binary serialization, allow to use it in RowBinaryWithNamesAndTypes and Native formats under settings. [#65546](https://github.com/ClickHouse/ClickHouse/pull/65546) ([Kruglov Pavel](https://github.com/Avogar)).
-* Improved ZooKeeper load balancing. The current session doesn't expire until the optimal nodes become available despite `fallback_session_lifetime`. Added support for AZ-aware balancing. [#65570](https://github.com/ClickHouse/ClickHouse/pull/65570) ([Alexander Tokmakov](https://github.com/tavplubix)).
 * Server settings `compiled_expression_cache_size` and `compiled_expression_cache_elements_size` are now shown in `system.server_settings`. [#65584](https://github.com/ClickHouse/ClickHouse/pull/65584) ([Robert Schulze](https://github.com/rschu1ze)).
 * Add support for user identification based on x509 SubjectAltName extension. [#65626](https://github.com/ClickHouse/ClickHouse/pull/65626) ([Anton Kozlov](https://github.com/tonickkozlov)).
 * `clickhouse-local` will respect the `max_server_memory_usage` and `max_server_memory_usage_to_ram_ratio` from the configuration file. It will also set the max memory usage to 90% of the system memory by default, like `clickhouse-server` does. [#65697](https://github.com/ClickHouse/ClickHouse/pull/65697) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
 * Add a script to backup your files to ClickHouse. [#65699](https://github.com/ClickHouse/ClickHouse/pull/65699) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
-* PostgreSQL source support cancel. [#65722](https://github.com/ClickHouse/ClickHouse/pull/65722) ([Maksim Kita](https://github.com/kitaisreal)).
-* Make allow_experimental_analyzer be controlled by the initiator for distributed queries. This ensures compatibility and correctness during operations in mixed version clusters. [#65777](https://github.com/ClickHouse/ClickHouse/pull/65777) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* PostgreSQL source to support query cancellations. [#65722](https://github.com/ClickHouse/ClickHouse/pull/65722) ([Maksim Kita](https://github.com/kitaisreal)).
+* Make `allow_experimental_analyzer` be controlled by the initiator for distributed queries. This ensures compatibility and correctness during operations in mixed version clusters. [#65777](https://github.com/ClickHouse/ClickHouse/pull/65777) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
 * Respect cgroup CPU limit in Keeper. [#65819](https://github.com/ClickHouse/ClickHouse/pull/65819) ([Antonio Andelic](https://github.com/antonio2368)).
-* Allow to use `concat` function with empty arguments ``` sql :) select concat();. [#65887](https://github.com/ClickHouse/ClickHouse/pull/65887) ([李扬](https://github.com/taiyang-li)).
-* Allow controlling named collections in clickhouse-local. [#65973](https://github.com/ClickHouse/ClickHouse/pull/65973) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
-* Improve Azure profile events. [#65999](https://github.com/ClickHouse/ClickHouse/pull/65999) ([alesapin](https://github.com/alesapin)).
-* Support ORC file read by writer time zone. [#66025](https://github.com/ClickHouse/ClickHouse/pull/66025) ([kevinyhzou](https://github.com/KevinyhZou)).
-* Add settings to control connection to the PostgreSQL. * Setting `postgresql_connection_attempt_timeout` specifies the value passed to `connect_timeout` parameter of connection URL. * Setting `postgresql_connection_pool_retries` specifies the number of retries to establish a connection to the PostgreSQL end-point. [#66232](https://github.com/ClickHouse/ClickHouse/pull/66232) ([Dmitry Novik](https://github.com/novikd)).
-* Reduce inaccuracy of input_wait_elapsed_us/input_wait_elapsed_us/elapsed_us. [#66239](https://github.com/ClickHouse/ClickHouse/pull/66239) ([Azat Khuzhin](https://github.com/azat)).
-* Improve FilesystemCache ProfileEvents. [#66249](https://github.com/ClickHouse/ClickHouse/pull/66249) ([zhukai](https://github.com/nauu)).
-* Add settings to ignore ON CLUSTER clause in queries for named collection management with replicated storage. [#66288](https://github.com/ClickHouse/ClickHouse/pull/66288) ([MikhailBurdukov](https://github.com/MikhailBurdukov)).
+* Allow to use `concat` function with empty arguments `:) select concat();`. [#65887](https://github.com/ClickHouse/ClickHouse/pull/65887) ([李扬](https://github.com/taiyang-li)).
+* Allow controlling named collections in `clickhouse-local`. [#65973](https://github.com/ClickHouse/ClickHouse/pull/65973) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Improve Azure-related profile events. [#65999](https://github.com/ClickHouse/ClickHouse/pull/65999) ([alesapin](https://github.com/alesapin)).
+* Support ORC file read by writer's time zone. [#66025](https://github.com/ClickHouse/ClickHouse/pull/66025) ([kevinyhzou](https://github.com/KevinyhZou)).
+* Add settings to control connections to PostgreSQL. The setting `postgresql_connection_attempt_timeout` specifies the value passed to `connect_timeout` parameter of connection URL. The setting `postgresql_connection_pool_retries` specifies the number of retries to establish a connection to the PostgreSQL end-point. [#66232](https://github.com/ClickHouse/ClickHouse/pull/66232) ([Dmitry Novik](https://github.com/novikd)).
+* Reduce inaccuracy of `input_wait_elapsed_us`/`elapsed_us` in the `system.processors_profile_log`. [#66239](https://github.com/ClickHouse/ClickHouse/pull/66239) ([Azat Khuzhin](https://github.com/azat)).
+* Improve ProfileEvents for the filesystem cache. [#66249](https://github.com/ClickHouse/ClickHouse/pull/66249) ([zhukai](https://github.com/nauu)).
+* Add settings to ignore the `ON CLUSTER` clause in queries for named collection management with the replicated storage. [#66288](https://github.com/ClickHouse/ClickHouse/pull/66288) ([MikhailBurdukov](https://github.com/MikhailBurdukov)).
 * Function `generateSnowflakeID` now allows to specify a machine ID as a parameter to prevent collisions in large clusters. [#66374](https://github.com/ClickHouse/ClickHouse/pull/66374) ([ZAWA_ll](https://github.com/Zawa-ll)).
-* Disable suspending on Ctrl+Z in interactive mode. This is a common trap and is not expected behavior for almost all users. I imagine only a few extreme power users could appreciate suspending terminal applications to the background, but I don't know any. [#66511](https://github.com/ClickHouse/ClickHouse/pull/66511) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
-* Add option for validating the Primary key type in Dictionaries. Without this option for simple layouts any column type will be implicitly converted to UInt64. ### Documentation entry for user-facing changes. [#66595](https://github.com/ClickHouse/ClickHouse/pull/66595) ([MikhailBurdukov](https://github.com/MikhailBurdukov)).
+* Disable suspending on `Ctrl+Z` in interactive mode. This is a common trap and is not expected behavior for almost all users. I imagine only a few extreme power users could appreciate suspending terminal applications to the background, but I don't know any. [#66511](https://github.com/ClickHouse/ClickHouse/pull/66511) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Add option for validating the primary key type in Dictionaries. Without this option for simple layouts any column type will be implicitly converted to UInt64. [#66595](https://github.com/ClickHouse/ClickHouse/pull/66595) ([MikhailBurdukov](https://github.com/MikhailBurdukov)).
 
 #### Bug Fix (user-visible misbehavior in an official stable release)
-* Fix unexpected size of low cardinality column in function calls. [#65298](https://github.com/ClickHouse/ClickHouse/pull/65298) ([Raúl Marín](https://github.com/Algunenano)).
 * Check cyclic dependencies on CREATE/REPLACE/RENAME/EXCHANGE queries and throw an exception if there is a cyclic dependency. Previously such cyclic dependencies could lead to a deadlock during server startup. Also fix some bugs in dependencies creation. [#65405](https://github.com/ClickHouse/ClickHouse/pull/65405) ([Kruglov Pavel](https://github.com/Avogar)).
+* Fix unexpected sizes of `LowCardinality` columns in function calls. [#65298](https://github.com/ClickHouse/ClickHouse/pull/65298) ([Raúl Marín](https://github.com/Algunenano)).
 * Fix crash in maxIntersections. [#65689](https://github.com/ClickHouse/ClickHouse/pull/65689) ([Raúl Marín](https://github.com/Algunenano)).
-* Fix the VALID UNTIL clause in the user definition resetting after a restart. [#66409](https://github.com/ClickHouse/ClickHouse/pull/66409) ([Nikolay Degterinsky](https://github.com/evillique)).
-* Fix SHOW MERGES remaining time. [#66735](https://github.com/ClickHouse/ClickHouse/pull/66735) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix the `VALID UNTIL` clause in the user definition resetting after a restart. [#66409](https://github.com/ClickHouse/ClickHouse/pull/66409) ([Nikolay Degterinsky](https://github.com/evillique)).
+* Fix the remaining time column in `SHOW MERGES`. [#66735](https://github.com/ClickHouse/ClickHouse/pull/66735) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
 * `Query was cancelled` might have been printed twice in clickhouse-client. This behaviour is fixed. [#66005](https://github.com/ClickHouse/ClickHouse/pull/66005) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
-* Fixed crash while using MaterializedMySQL with TABLE OVERRIDE that maps MySQL NULL field into ClickHouse not NULL field. [#54649](https://github.com/ClickHouse/ClickHouse/pull/54649) ([Filipp Ozinov](https://github.com/bakwc)).
-* Fix logical error when PREWHERE expression read no columns and table has no adaptive index granularity (very old table). [#59173](https://github.com/ClickHouse/ClickHouse/pull/59173) ([Alexander Gololobov](https://github.com/davenger)).
-* Fix bug with cancellation buffer when canceling a query. [#64478](https://github.com/ClickHouse/ClickHouse/pull/64478) ([Sema Checherinda](https://github.com/CheSema)).
+* Fixed crash while using `MaterializedMySQL` (which is an unsupported, experimental feature) with TABLE OVERRIDE that maps MySQL NULL field into ClickHouse not NULL field. [#54649](https://github.com/ClickHouse/ClickHouse/pull/54649) ([Filipp Ozinov](https://github.com/bakwc)).
+* Fix logical error when `PREWHERE` expression read no columns and table has no adaptive index granularity (very old table). [#59173](https://github.com/ClickHouse/ClickHouse/pull/59173) ([Alexander Gololobov](https://github.com/davenger)).
+* Fix bug with the cancellation buffer when canceling a query. [#64478](https://github.com/ClickHouse/ClickHouse/pull/64478) ([Sema Checherinda](https://github.com/CheSema)).
 * Fix filling parts columns from metadata (when columns.txt does not exists). [#64757](https://github.com/ClickHouse/ClickHouse/pull/64757) ([Azat Khuzhin](https://github.com/azat)).
 * Fix crash for `ALTER TABLE ... ON CLUSTER ... MODIFY SQL SECURITY`. [#64957](https://github.com/ClickHouse/ClickHouse/pull/64957) ([pufit](https://github.com/pufit)).
 * Fix crash on destroying AccessControl: add explicit shutdown. [#64993](https://github.com/ClickHouse/ClickHouse/pull/64993) ([Vitaly Baranov](https://github.com/vitlibar)).
@@ -178,9 +178,6 @@
 * Fix `indexHint` function case found by fuzzer. [#66286](https://github.com/ClickHouse/ClickHouse/pull/66286) ([Anton Popov](https://github.com/CurtizJ)).
 * Fix AST formatting of 'create table b empty as a'. [#64951](https://github.com/ClickHouse/ClickHouse/pull/64951) ([Michael Kolupaev](https://github.com/al13n321)).
 
-#### Build/Testing/Packaging Improvement
-* Instantiate template methods ahead in different .cpp files, avoid too large translation units during compiling. [#64818](https://github.com/ClickHouse/ClickHouse/pull/64818) ([lgbo](https://github.com/lgbo-ustc)).
-
 ### <a id="246"></a> ClickHouse release 24.6, 2024-07-01
 
 #### Backward Incompatible Change

From 3a7ffb3284003d853974baf12cf442bdc1105143 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Tue, 30 Jul 2024 05:19:10 +0200
Subject: [PATCH 1043/1488] Changelog sanity

---
 CHANGELOG.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 06f7bcdd84e..9d1a63cb3a9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -156,7 +156,7 @@
 * Fixed how the ClickHouse server detects the maximum number of usable CPU cores as specified by cgroups v2 if the server runs in a container such as Docker. In more detail, containers often run their process in the root cgroup which has an empty name. In that case, ClickHouse ignored the CPU limits set by cgroups v2. [#66237](https://github.com/ClickHouse/ClickHouse/pull/66237) ([filimonov](https://github.com/filimonov)).
 * Fix the `Not-ready set` error when a subquery with `IN` is used in the constraint. [#66261](https://github.com/ClickHouse/ClickHouse/pull/66261) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
 * Fix error reporting while copying to S3 or AzureBlobStorage. [#66295](https://github.com/ClickHouse/ClickHouse/pull/66295) ([Vitaly Baranov](https://github.com/vitlibar)).
-* Prevent watchdog from keeping descriptors of unlinked(rotated) log files. [#66334](https://github.com/ClickHouse/ClickHouse/pull/66334) ([Aleksei Filatov](https://github.com/aalexfvk)).
+* Prevent watchdog from keeping descriptors of unlinked (rotated) log files. [#66334](https://github.com/ClickHouse/ClickHouse/pull/66334) ([Aleksei Filatov](https://github.com/aalexfvk)).
 * Fix the bug that logicalexpressionoptimizerpass lost logical type of constant. [#66344](https://github.com/ClickHouse/ClickHouse/pull/66344) ([pn](https://github.com/chloro-pn)).
 * Fix `Column identifier is already registered` error with `group_by_use_nulls=true` and new analyzer. [#66400](https://github.com/ClickHouse/ClickHouse/pull/66400) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
 * Fix possible incorrect result for queries joining and filtering table external engine (like PostgreSQL), due to too aggressive filter pushdown. Since now, conditions from where section won't be send to external database in case of outer join with external table. [#66402](https://github.com/ClickHouse/ClickHouse/pull/66402) ([vdimir](https://github.com/vdimir)).

From 368b9a058379c2e7902fd9ee7a21b664f0500df9 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Tue, 30 Jul 2024 05:29:36 +0200
Subject: [PATCH 1044/1488] Changelog sanity

---
 CHANGELOG.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9d1a63cb3a9..722ae4f8268 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -39,11 +39,11 @@
 * Add system tables with main information about all detached tables. [#65400](https://github.com/ClickHouse/ClickHouse/pull/65400) ([Konstantin Morozov](https://github.com/k-morozov)).
 
 #### Experimental Feature
-* Change binary serialization of Variant data type: add `compact` mode to avoid writing the same discriminator multiple times for granules with single variant or with only NULL values. Add MergeTree setting `use_compact_variant_discriminators_serialization` that is enabled by default. Note that Variant type is still experimental and backward-incompatible change in serialization is ok. [#62774](https://github.com/ClickHouse/ClickHouse/pull/62774) ([Kruglov Pavel](https://github.com/Avogar)).
-* Support rocksdb as backend storage of clickhouse-keeper. [#56626](https://github.com/ClickHouse/ClickHouse/pull/56626) ([Han Fei](https://github.com/hanfei1991)).
+* Change binary serialization of the `Variant` data type: add `compact` mode to avoid writing the same discriminator multiple times for granules with single variant or with only NULL values. Add MergeTree setting `use_compact_variant_discriminators_serialization` that is enabled by default. Note that Variant type is still experimental and backward-incompatible change in serialization is ok. [#62774](https://github.com/ClickHouse/ClickHouse/pull/62774) ([Kruglov Pavel](https://github.com/Avogar)).
+* Support on-disk backend storage for clickhouse-keeper. [#56626](https://github.com/ClickHouse/ClickHouse/pull/56626) ([Han Fei](https://github.com/hanfei1991)).
 * Refactor JSONExtract functions, support more types including experimental Dynamic type. [#66046](https://github.com/ClickHouse/ClickHouse/pull/66046) ([Kruglov Pavel](https://github.com/Avogar)).
-* Support null map subcolumn for Variant and Dynamic subcolumns. [#66178](https://github.com/ClickHouse/ClickHouse/pull/66178) ([Kruglov Pavel](https://github.com/Avogar)).
-* Fix reading dynamic subcolumns from altered Memory table. Previously if `max_types` parameter of a Dynamic type was changed in Memory table via alter, further subcolumns reading can return wrong result. [#66066](https://github.com/ClickHouse/ClickHouse/pull/66066) ([Kruglov Pavel](https://github.com/Avogar)).
+* Support null map subcolumn for `Variant` and `Dynamic` subcolumns. [#66178](https://github.com/ClickHouse/ClickHouse/pull/66178) ([Kruglov Pavel](https://github.com/Avogar)).
+* Fix reading `Dynamic` subcolumns from altered `Memory` table. Previously if `max_types` parameter of a Dynamic type was changed in Memory table via alter, further subcolumns reading can return wrong result. [#66066](https://github.com/ClickHouse/ClickHouse/pull/66066) ([Kruglov Pavel](https://github.com/Avogar)).
 * Add support for `cluster_for_parallel_replicas` when using custom key parallel replicas. It allows you to use parallel replicas with custom key with MergeTree tables. [#65453](https://github.com/ClickHouse/ClickHouse/pull/65453) ([Antonio Andelic](https://github.com/antonio2368)).
 
 #### Performance Improvement

From 4df37538820f4874f54852e2418cf560c9da9ecc Mon Sep 17 00:00:00 2001
From: Michael Kolupaev <michael.kolupaev@clickhouse.com>
Date: Tue, 30 Jul 2024 03:50:28 +0000
Subject: [PATCH 1045/1488] Fix bloom filter index breaking some queries

---
 .../MergeTree/MergeTreeIndexBloomFilter.cpp   | 115 ++++++++++--------
 .../00908_bloom_filter_index.reference        |   2 +
 .../0_stateless/00908_bloom_filter_index.sh   |   4 +
 .../00945_bloom_filter_index.reference        |   5 +
 .../0_stateless/00945_bloom_filter_index.sql  |   9 ++
 5 files changed, 83 insertions(+), 52 deletions(-)

diff --git a/src/Storages/MergeTree/MergeTreeIndexBloomFilter.cpp b/src/Storages/MergeTree/MergeTreeIndexBloomFilter.cpp
index c6a00751f25..dc314ce53d4 100644
--- a/src/Storages/MergeTree/MergeTreeIndexBloomFilter.cpp
+++ b/src/Storages/MergeTree/MergeTreeIndexBloomFilter.cpp
@@ -371,67 +371,78 @@ bool MergeTreeIndexConditionBloomFilter::extractAtomFromTree(const RPNBuilderTre
 
 bool MergeTreeIndexConditionBloomFilter::traverseFunction(const RPNBuilderTreeNode & node, RPNElement & out, const RPNBuilderTreeNode * parent)
 {
-    bool maybe_useful = false;
+    if (!node.isFunction())
+        return false;
 
-    if (node.isFunction())
+    const auto function = node.toFunctionNode();
+    auto arguments_size = function.getArgumentsSize();
+    auto function_name = function.getFunctionName();
+
+    if (parent == nullptr)
     {
-        const auto function = node.toFunctionNode();
-        auto arguments_size = function.getArgumentsSize();
-        auto function_name = function.getFunctionName();
-
+        /// Recurse a little bit for indexOf().
         for (size_t i = 0; i < arguments_size; ++i)
         {
             auto argument = function.getArgumentAt(i);
             if (traverseFunction(argument, out, &node))
-                maybe_useful = true;
-        }
-
-        if (arguments_size != 2)
-            return false;
-
-        auto lhs_argument = function.getArgumentAt(0);
-        auto rhs_argument = function.getArgumentAt(1);
-
-        if (functionIsInOrGlobalInOperator(function_name))
-        {
-            if (auto future_set = rhs_argument.tryGetPreparedSet(); future_set)
-            {
-                if (auto prepared_set = future_set->buildOrderedSetInplace(rhs_argument.getTreeContext().getQueryContext()); prepared_set)
-                {
-                    if (prepared_set->hasExplicitSetElements())
-                    {
-                        const auto prepared_info = getPreparedSetInfo(prepared_set);
-                        if (traverseTreeIn(function_name, lhs_argument, prepared_set, prepared_info.type, prepared_info.column, out))
-                            maybe_useful = true;
-                    }
-                }
-            }
-        }
-        else if (function_name == "equals" ||
-                 function_name == "notEquals" ||
-                 function_name == "has" ||
-                 function_name == "mapContains" ||
-                 function_name == "indexOf" ||
-                 function_name == "hasAny" ||
-                 function_name == "hasAll")
-        {
-            Field const_value;
-            DataTypePtr const_type;
-
-            if (rhs_argument.tryGetConstant(const_value, const_type))
-            {
-                if (traverseTreeEquals(function_name, lhs_argument, const_type, const_value, out, parent))
-                    maybe_useful = true;
-            }
-            else if (lhs_argument.tryGetConstant(const_value, const_type))
-            {
-                if (traverseTreeEquals(function_name, rhs_argument, const_type, const_value, out, parent))
-                    maybe_useful = true;
-            }
+                return true;
         }
     }
 
-    return maybe_useful;
+    if (arguments_size != 2)
+        return false;
+
+    /// indexOf() should be inside comparison function, e.g. greater(indexOf(key, 42), 0).
+    /// Other conditions should be at top level, e.g. equals(key, 42), not equals(equals(key, 42), 1).
+    if ((function_name == "indexOf") != (parent != nullptr))
+        return false;
+
+    auto lhs_argument = function.getArgumentAt(0);
+    auto rhs_argument = function.getArgumentAt(1);
+
+    if (functionIsInOrGlobalInOperator(function_name))
+    {
+        if (auto future_set = rhs_argument.tryGetPreparedSet(); future_set)
+        {
+            if (auto prepared_set = future_set->buildOrderedSetInplace(rhs_argument.getTreeContext().getQueryContext()); prepared_set)
+            {
+                if (prepared_set->hasExplicitSetElements())
+                {
+                    const auto prepared_info = getPreparedSetInfo(prepared_set);
+                    if (traverseTreeIn(function_name, lhs_argument, prepared_set, prepared_info.type, prepared_info.column, out))
+                        return true;
+                }
+            }
+        }
+        return false;
+    }
+
+    if (function_name == "equals" ||
+        function_name == "notEquals" ||
+        function_name == "has" ||
+        function_name == "mapContains" ||
+        function_name == "indexOf" ||
+        function_name == "hasAny" ||
+        function_name == "hasAll")
+    {
+        Field const_value;
+        DataTypePtr const_type;
+
+        if (rhs_argument.tryGetConstant(const_value, const_type))
+        {
+            if (traverseTreeEquals(function_name, lhs_argument, const_type, const_value, out, parent))
+                return true;
+        }
+        else if (lhs_argument.tryGetConstant(const_value, const_type) && (function_name == "equals" || function_name == "notEquals"))
+        {
+            if (traverseTreeEquals(function_name, rhs_argument, const_type, const_value, out, parent))
+                return true;
+        }
+
+        return false;
+    }
+
+    return false;
 }
 
 bool MergeTreeIndexConditionBloomFilter::traverseTreeIn(
diff --git a/tests/queries/0_stateless/00908_bloom_filter_index.reference b/tests/queries/0_stateless/00908_bloom_filter_index.reference
index c0cbd6c0335..e2e13a9ed12 100644
--- a/tests/queries/0_stateless/00908_bloom_filter_index.reference
+++ b/tests/queries/0_stateless/00908_bloom_filter_index.reference
@@ -28,6 +28,8 @@
 		"rows_read": 3,
 8	aбвгдеёж
 		"rows_read": 2,
+13
+1
 1	column-oriented
 2	column-oriented
 		"rows_read": 4,
diff --git a/tests/queries/0_stateless/00908_bloom_filter_index.sh b/tests/queries/0_stateless/00908_bloom_filter_index.sh
index 88fc7944236..25a6567b894 100755
--- a/tests/queries/0_stateless/00908_bloom_filter_index.sh
+++ b/tests/queries/0_stateless/00908_bloom_filter_index.sh
@@ -103,6 +103,10 @@ $CLICKHOUSE_CLIENT --optimize_or_like_chain 0 --query="SELECT * FROM bloom_filte
 $CLICKHOUSE_CLIENT --optimize_or_like_chain 0 --query="SELECT * FROM bloom_filter_idx WHERE (s, lower(s)) IN (('aбвгдеёж', 'aбвгдеёж'), ('abc', 'cba')) ORDER BY k"
 $CLICKHOUSE_CLIENT --optimize_or_like_chain 0 --query="SELECT * FROM bloom_filter_idx WHERE (s, lower(s)) IN (('aбвгдеёж', 'aбвгдеёж'), ('abc', 'cba')) ORDER BY k FORMAT JSON" | grep "rows_read"
 
+# Weird conditions not supported by the index.
+$CLICKHOUSE_CLIENT --optimize_or_like_chain 0 --query="SELECT count() FROM bloom_filter_idx WHERE (s = 'asd') = (s = 'asd')"
+$CLICKHOUSE_CLIENT --optimize_or_like_chain 0 --query="SELECT count() FROM bloom_filter_idx WHERE has(['asd', 'some string'], s)"
+
 
 # TOKEN BF
 $CLICKHOUSE_CLIENT -n --query="
diff --git a/tests/queries/0_stateless/00945_bloom_filter_index.reference b/tests/queries/0_stateless/00945_bloom_filter_index.reference
index c0c2254648e..e6751fe4762 100644
--- a/tests/queries/0_stateless/00945_bloom_filter_index.reference
+++ b/tests/queries/0_stateless/00945_bloom_filter_index.reference
@@ -14,6 +14,11 @@
 0
 2
 2
+18
+100
+100
+3
+100
 1
 1
 1
diff --git a/tests/queries/0_stateless/00945_bloom_filter_index.sql b/tests/queries/0_stateless/00945_bloom_filter_index.sql
index 4c26988574a..2b7feacbd98 100644
--- a/tests/queries/0_stateless/00945_bloom_filter_index.sql
+++ b/tests/queries/0_stateless/00945_bloom_filter_index.sql
@@ -25,6 +25,15 @@ WITH ((1, 2), (2, 3)) AS liter_prepared_set SELECT COUNT() FROM single_column_bl
 WITH ((1, 1), (2, 2)) AS liter_prepared_set SELECT COUNT() FROM single_column_bloom_filter WHERE (i32, i64) IN liter_prepared_set SETTINGS max_rows_to_read = 6;
 WITH ((1, (1, 1)), (2, (2, 2))) AS liter_prepared_set SELECT COUNT() FROM single_column_bloom_filter WHERE (i64, (i64, i32)) IN liter_prepared_set SETTINGS max_rows_to_read = 6;
 
+-- Check that indexHint() works (but it doesn't work with COUNT()).
+SELECT SUM(ignore(*) + 1) FROM single_column_bloom_filter WHERE indexHint(i32 in (3, 15, 50));
+
+-- The index doesn't understand expressions like these, but it shouldn't break the query.
+SELECT COUNT() FROM single_column_bloom_filter WHERE (i32 = 200) = (i32 = 200);
+SELECT SUM(ignore(*) + 1) FROM single_column_bloom_filter WHERE indexHint((i32 = 200) != (i32 = 200));
+SELECT COUNT() FROM single_column_bloom_filter WHERE indexOf([10, 20, 30], i32) != 0;
+SELECT COUNT() FROM single_column_bloom_filter WHERE has([100, 200, 300], 200);
+
 DROP TABLE IF EXISTS single_column_bloom_filter;
 
 

From f90b88c978d87225a5cf5f66136714ff5535d69c Mon Sep 17 00:00:00 2001
From: Alexey Gerasimchuck <a.gerasimchuck@arenadata.io>
Date: Tue, 30 Jul 2024 05:57:07 +0000
Subject: [PATCH 1046/1488] Reduced comprexity of the test

---
 ...2832_alter_max_sessions_for_user.reference |  7 +--
 .../02832_alter_max_sessions_for_user.sh      | 62 ++++++++-----------
 2 files changed, 30 insertions(+), 39 deletions(-)

diff --git a/tests/queries/0_stateless/02832_alter_max_sessions_for_user.reference b/tests/queries/0_stateless/02832_alter_max_sessions_for_user.reference
index f80f8738ff8..c2e103d61cb 100644
--- a/tests/queries/0_stateless/02832_alter_max_sessions_for_user.reference
+++ b/tests/queries/0_stateless/02832_alter_max_sessions_for_user.reference
@@ -1,8 +1,7 @@
-test_alter_profile case: max_session_count 1 alter_sessions_count 1
-test_alter_profile case: max_session_count 2 alter_sessions_count 1
+test_alter_profile case: max_sessions_for_user 1
+USER_SESSION_LIMIT_EXCEEDED
+test_alter_profile case: max_sessions_for_user 2
 USER_SESSION_LIMIT_EXCEEDED
-test_alter_profile case: max_session_count 1 alter_sessions_count 2
-test_alter_profile case: max_session_count 2 alter_sessions_count 2
 READONLY
 READONLY
 READONLY
diff --git a/tests/queries/0_stateless/02832_alter_max_sessions_for_user.sh b/tests/queries/0_stateless/02832_alter_max_sessions_for_user.sh
index 87fbffdb1e6..55f9e3e97a4 100755
--- a/tests/queries/0_stateless/02832_alter_max_sessions_for_user.sh
+++ b/tests/queries/0_stateless/02832_alter_max_sessions_for_user.sh
@@ -5,7 +5,6 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
-SESSION_ID_PREFIX="02832_alter_max_sessions_session_$$"
 QUERY_ID_PREFIX="02832_alter_max_sessions_query_$$"
 PROFILE="02832_alter_max_sessions_profile_$$"
 USER="02832_alter_max_sessions_user_$$"
@@ -17,48 +16,41 @@ ${CLICKHOUSE_CLIENT} -q $"DROP PROFILE IF EXISTS ${PROFILE}"
 ${CLICKHOUSE_CLIENT} -q $"CREATE SETTINGS PROFILE ${PROFILE}"
 ${CLICKHOUSE_CLIENT} -q $"CREATE USER '${USER}' SETTINGS PROFILE '${PROFILE}'"
 
-function run_sessions_set()
+function wait_for_query_to_start()
 {
-    local sessions_count="$1"
-    local session_check="$2"
-    for ((i = 1 ; i <= ${sessions_count} ; i++)); do
-        local session_id="${SESSION_ID_PREFIX}_${i}"
-        local query_id="${QUERY_ID_PREFIX}_${i}"
-        # Write only expected error text
-        # More than alter_sessions_count queries will not start.
-        ${CLICKHOUSE_CURL} -sS -X POST "${CLICKHOUSE_URL}&user=${USER}&query_id=${query_id}&session_id=${session_id}&session_check=${session_check}&session_timeout=600&function_sleep_max_microseconds_per_block=120000000" --data-binary "SELECT sleep(120)" | grep -o  -m 1 'USER_SESSION_LIMIT_EXCEEDED' &
+    while [[ $($CLICKHOUSE_CLIENT -q "SELECT count() FROM system.processes WHERE query_id = '$1'") == 0 ]]; do sleep 0.2; done
+}
+
+function test_alter_max_sessions_for_user()
+{
+    local max_sessions_for_user="$1"
+    echo $"test_alter_profile case: max_sessions_for_user ${max_sessions_for_user}"
+
+    # Step 0: Set max_sessions_for_user.
+    ${CLICKHOUSE_CLIENT} -q $"ALTER SETTINGS PROFILE ${PROFILE} SETTINGS max_sessions_for_user = ${max_sessions_for_user}"
+
+    # Step 1: Simulaneously run `max_sessions_for_user` queries. These queries should run without any problems.
+    for ((i = 1 ; i <= max_sessions_for_user ; i++)); do
+        local query_id="${QUERY_ID_PREFIX}_${i}_${max_sessions_for_user}"
+        ${CLICKHOUSE_CLIENT} --max_block_size 1 --query_id $query_id --user $USER --function_sleep_max_microseconds_per_block=120000000 -q "SELECT sleepEachRow(0.1) FROM numbers(1200)" &>/dev/null &
+        wait_for_query_to_start $query_id
     done
 
-    for ((i = 1 ; i <= ${sessions_count} ; i++)); do
-        local query_id="${QUERY_ID_PREFIX}_${i}"
-        $CLICKHOUSE_CLIENT --query "KILL QUERY WHERE query_id='$query_id' SYNC" >/dev/null
+    # Step 2: Run another `max_sessions_for_user` + 1 query. That query should fail.
+    local query_id="${QUERY_ID_PREFIX}_should_fail"
+    ${CLICKHOUSE_CLIENT} --query_id $query_id --user $USER -q "SELECT 1" 2>&1 | grep -o -m 1 'USER_SESSION_LIMIT_EXCEEDED'
+    
+    # Step 3: Stop running queries launched at step 1.
+    for ((i = 1 ; i <= max_sessions_for_user ; i++)); do
+        local query_id="${QUERY_ID_PREFIX}_${i}_${max_sessions_for_user}"
+        $CLICKHOUSE_CLIENT --query "KILL QUERY WHERE query_id='$query_id' ASYNC" >/dev/null
     done
 
     wait
 }
 
-function test_alter_profile()
-{
-    local max_session_count="$1"
-    local alter_sessions_count="$2"
-    echo $"test_alter_profile case: max_session_count ${max_session_count} alter_sessions_count ${alter_sessions_count}"
-
-    ${CLICKHOUSE_CLIENT} -q $"ALTER SETTINGS PROFILE ${PROFILE} SETTINGS max_sessions_for_user = ${max_session_count}"
-
-    # Create sessions with $max_session_count restriction
-    run_sessions_set $max_session_count 0
-
-    # Update restriction to $alter_sessions_count
-    ${CLICKHOUSE_CLIENT} -q $"ALTER SETTINGS PROFILE ${PROFILE} SETTINGS max_sessions_for_user = ${alter_sessions_count}"
-
-    # Simultaneous sessions should use max settings from profile ($alter_sessions_count)
-    run_sessions_set $max_session_count 1
-}
-
-test_alter_profile 1 1
-test_alter_profile 2 1
-test_alter_profile 1 2
-test_alter_profile 2 2
+test_alter_max_sessions_for_user 1
+test_alter_max_sessions_for_user 2
 
 ${CLICKHOUSE_CLIENT} -q "SELECT 1 SETTINGS max_sessions_for_user = 1" 2>&1 | grep -m 1 -o 'READONLY' | head -1
 ${CLICKHOUSE_CLIENT} -q $"SET max_sessions_for_user = 1 " 2>&1 | grep -o  -m 1 'READONLY' | head -1

From 7a53a14940ae1be299305548f0d024de7f279fe3 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Tue, 30 Jul 2024 09:29:16 +0200
Subject: [PATCH 1047/1488] Update 03213_deep_json.sql

---
 tests/queries/0_stateless/03213_deep_json.sql | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/queries/0_stateless/03213_deep_json.sql b/tests/queries/0_stateless/03213_deep_json.sql
index 4f79d99eb26..2a9476381ff 100644
--- a/tests/queries/0_stateless/03213_deep_json.sql
+++ b/tests/queries/0_stateless/03213_deep_json.sql
@@ -1,5 +1,5 @@
 -- The default limit works.
-SELECT * FROM format("JSONCompactEachRow", 'x UInt32, y UInt32', REPEAT('[1,1,', 100000)) SETTINGS input_format_json_compact_allow_variable_number_of_columns = 1; -- { serverError TOO_DEEP_RECURSION }
+SELECT * FROM format("JSONCompactEachRow", 'x UInt32, y UInt32', REPEAT('[1,1,', 100000)) SETTINGS input_format_json_compact_allow_variable_number_of_columns = 1; -- { serverError TOO_DEEP_RECURSION, INCORRECT_DATA }
 -- Even if we relax the limit, it is also safe.
 SET input_format_json_max_depth = 100000;
-SELECT * FROM format("JSONCompactEachRow", 'x UInt32, y UInt32', REPEAT('[1,1,', 100000)) SETTINGS input_format_json_compact_allow_variable_number_of_columns = 1; -- { serverError TOO_DEEP_RECURSION }
+SELECT * FROM format("JSONCompactEachRow", 'x UInt32, y UInt32', REPEAT('[1,1,', 100000)) SETTINGS input_format_json_compact_allow_variable_number_of_columns = 1; -- { serverError TOO_DEEP_RECURSION, INCORRECT_DATA }

From 6ab67323862391e520ac571f609f52b582248da5 Mon Sep 17 00:00:00 2001
From: Michael Kolupaev <michael.kolupaev@clickhouse.com>
Date: Tue, 30 Jul 2024 07:36:38 +0000
Subject: [PATCH 1048/1488] Delete .reference

---
 ...2_refreshable_materialized_views.reference | 44 -------------------
 1 file changed, 44 deletions(-)
 delete mode 100644 tests/queries/0_stateless/02932_refreshable_materialized_views.reference

diff --git a/tests/queries/0_stateless/02932_refreshable_materialized_views.reference b/tests/queries/0_stateless/02932_refreshable_materialized_views.reference
deleted file mode 100644
index 2eb41590af1..00000000000
--- a/tests/queries/0_stateless/02932_refreshable_materialized_views.reference
+++ /dev/null
@@ -1,44 +0,0 @@
-<1: created view>	a	[]		1
-CREATE MATERIALIZED VIEW default.a\nREFRESH AFTER 2 SECOND\n(\n    `x` UInt64\n)\nENGINE = Memory\nAS SELECT number AS x\nFROM numbers(2)\nUNION ALL\nSELECT rand64() AS x
-<2: refreshed>	3	1	1
-<3: time difference at least>	1000
-<4: next refresh in>	2
-<4.5: altered>	Scheduled	Finished	2052-01-01 00:00:00
-CREATE MATERIALIZED VIEW default.a\nREFRESH EVERY 2 YEAR\n(\n    `x` UInt64\n)\nENGINE = Memory\nAS SELECT x * 2 AS x\nFROM default.src
-<5: no refresh>	3
-<6: refreshed>	2
-<7: refreshed>	Scheduled	Finished	2054-01-01 00:00:00
-CREATE MATERIALIZED VIEW default.b\nREFRESH EVERY 2 YEAR DEPENDS ON default.a\n(\n    `y` Int32\n)\nENGINE = MergeTree\nORDER BY y\nSETTINGS index_granularity = 8192\nAS SELECT x * 10 AS y\nFROM default.a
-<8: refreshed>	20
-<9: refreshed>	a	Scheduled	Finished	2054-01-01 00:00:00
-<9: refreshed>	b	Scheduled	Finished	2054-01-01 00:00:00
-<10: waiting>	a	Scheduled	[]	2054-01-01 00:00:00
-<10: waiting>	b	WaitingForDependencies	['default.a']	2054-01-01 00:00:00
-<11: chain-refreshed a>	4
-<12: chain-refreshed b>	40
-<13: chain-refreshed>	a	Scheduled	[]	Finished	2054-01-01 00:00:01	2056-01-01 00:00:00	
-<13: chain-refreshed>	b	Scheduled	['default.a']	Finished	2054-01-24 23:22:21	2056-01-01 00:00:00	
-<14: waiting for next cycle>	a	Scheduled	[]	2058-01-01 00:00:00
-<14: waiting for next cycle>	b	WaitingForDependencies	['default.a']	2060-01-01 00:00:00
-<15: chain-refreshed a>	6
-<16: chain-refreshed b>	60
-<17: chain-refreshed>	a	Scheduled	2062-01-01 00:00:00
-<17: chain-refreshed>	b	Scheduled	2062-01-01 00:00:00
-<18: removed dependency>	b	Scheduled	[]	2062-03-03 03:03:03	2064-01-01 00:00:00	5
-CREATE MATERIALIZED VIEW default.b\nREFRESH EVERY 2 YEAR\n(\n    `y` Int32\n)\nENGINE = MergeTree\nORDER BY y\nSETTINGS index_granularity = 8192\nAS SELECT x * 10 AS y\nFROM default.a
-<19: exception>	1
-<20: unexception>	1
-<21: rename>	1
-<22: rename>	d	Finished
-<23: simple refresh>	1
-<24: rename during refresh>	1
-<25: rename during refresh>	f	Running
-<27: cancelled>	f	Scheduled
-CREATE MATERIALIZED VIEW default.g\nREFRESH EVERY 1 WEEK OFFSET 3 DAY 4 HOUR RANDOMIZE FOR 4 DAY 1 HOUR\n(\n    `x` Int64\n)\nENGINE = Memory\nAS SELECT 42
-<29: randomize>	1	1
-CREATE MATERIALIZED VIEW default.h\nREFRESH EVERY 1 SECOND TO default.dest\n(\n    `x` Int64\n)\nAS SELECT x * 10 AS x\nFROM default.src
-<30: to existing table>	10
-<31: to existing table>	10
-<31: to existing table>	20
-<32: empty>	i	Scheduled	Unknown
-<32: empty>	j	Scheduled	Finished

From 861bdb51f8e098c4b06cb14988e00febbb5a0ac7 Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Tue, 30 Jul 2024 09:42:38 +0200
Subject: [PATCH 1049/1488] Fix test

---
 .../test_replicated_table_attach/test.py           | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/tests/integration/test_replicated_table_attach/test.py b/tests/integration/test_replicated_table_attach/test.py
index 02ef1ad6353..499220def2c 100644
--- a/tests/integration/test_replicated_table_attach/test.py
+++ b/tests/integration/test_replicated_table_attach/test.py
@@ -27,7 +27,20 @@ def started_cluster():
         cluster.shutdown()
 
 
+def start_clean_clickhouse():
+    # remove fault injection if present
+    if "fault_injection.xml" in node.exec_in_container(
+        ["bash", "-c", "ls /etc/clickhouse-server/config.d"]
+    ):
+        print("Removing fault injection")
+        node.exec_in_container(
+            ["bash", "-c", "rm /etc/clickhouse-server/config.d/fault_injection.xml"]
+        )
+        node.restart_clickhouse()
+
+
 def test_startup_with_small_bg_pool(started_cluster):
+    start_clean_clickhouse()
     node.query(
         "CREATE TABLE replicated_table (k UInt64, i32 Int32) ENGINE=ReplicatedMergeTree('/clickhouse/replicated_table', 'r1') ORDER BY k"
     )
@@ -45,6 +58,7 @@ def test_startup_with_small_bg_pool(started_cluster):
 
 
 def test_startup_with_small_bg_pool_partitioned(started_cluster):
+    start_clean_clickhouse()
     node.query(
         "CREATE TABLE replicated_table_partitioned (k UInt64, i32 Int32) ENGINE=ReplicatedMergeTree('/clickhouse/replicated_table_partitioned', 'r1') ORDER BY k"
     )

From cd036886ec9ba030da7b6b6151bb81e5cc3f7636 Mon Sep 17 00:00:00 2001
From: heguangnan <heguangnan@bytedance.com>
Date: Tue, 30 Jul 2024 15:53:48 +0800
Subject: [PATCH 1050/1488] add test

---
 ...unt_distinct_null_key_memory_leak.reference |  0
 ...214_count_distinct_null_key_memory_leak.sql | 18 ++++++++++++++++++
 2 files changed, 18 insertions(+)
 create mode 100644 tests/queries/0_stateless/03214_count_distinct_null_key_memory_leak.reference
 create mode 100644 tests/queries/0_stateless/03214_count_distinct_null_key_memory_leak.sql

diff --git a/tests/queries/0_stateless/03214_count_distinct_null_key_memory_leak.reference b/tests/queries/0_stateless/03214_count_distinct_null_key_memory_leak.reference
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/queries/0_stateless/03214_count_distinct_null_key_memory_leak.sql b/tests/queries/0_stateless/03214_count_distinct_null_key_memory_leak.sql
new file mode 100644
index 00000000000..847d3742dc3
--- /dev/null
+++ b/tests/queries/0_stateless/03214_count_distinct_null_key_memory_leak.sql
@@ -0,0 +1,18 @@
+-- Tags: no-fasttest
+
+DROP TABLE IF EXISTS testnull;
+CREATE TABLE testnull
+(
+    `a` Nullable(String),
+    `b` Nullable(String),
+    `c` Nullable(String)
+)
+ENGINE = MergeTree
+PARTITION BY tuple()
+ORDER BY c
+SETTINGS index_granularity = 8192, allow_nullable_key=1;
+
+INSERT INTO testnull(b,c) SELECT toString(rand64()) AS b, toString(rand64()) AS c FROM numbers(1000000)
+SELECT count(distinct b) FROM testnull GROUP BY a  SETTINGS max_memory_usage = 54748364; -- {serverError MEMORY_LIMIT_EXCEEDED}
+
+DROP TABLE testnull;
\ No newline at end of file

From a70571762f7d73a7ecc94981e8086418ecfdeb3b Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Tue, 30 Jul 2024 10:08:35 +0200
Subject: [PATCH 1051/1488] Enable text_log by default

---
 programs/server/config.xml | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/programs/server/config.xml b/programs/server/config.xml
index 94825a55f67..844aff8f668 100644
--- a/programs/server/config.xml
+++ b/programs/server/config.xml
@@ -1130,8 +1130,7 @@
         <flush_interval_milliseconds>7500</flush_interval_milliseconds>
     </query_views_log>
 
-    <!-- Uncomment if use part log.
-         Part log contains information about all actions with parts in MergeTree tables (creation, deletion, merges, downloads).-->
+    <!-- Part log contains information about all actions with parts in MergeTree tables (creation, deletion, merges, downloads). -->
     <part_log>
         <database>system</database>
         <table>part_log</table>
@@ -1143,9 +1142,9 @@
         <flush_on_crash>false</flush_on_crash>
     </part_log>
 
-    <!-- Uncomment to write text log into table.
-         Text log contains all information from usual server log but stores it in structured and efficient way.
+    <!-- Text log contains all information from usual server log but stores it in structured and efficient way.
          The level of the messages that goes to the table can be limited (<level>), if not specified all messages will go to the table.
+    -->
     <text_log>
         <database>system</database>
         <table>text_log</table>
@@ -1154,9 +1153,8 @@
         <reserved_size_rows>8192</reserved_size_rows>
         <buffer_size_rows_flush_threshold>524288</buffer_size_rows_flush_threshold>
         <flush_on_crash>false</flush_on_crash>
-        <level></level>
+        <level>trace</level>
     </text_log>
-    -->
 
     <!-- Metric log contains rows with current values of ProfileEvents, CurrentMetrics collected with "collect_interval_milliseconds" interval. -->
     <metric_log>

From c427c4e2bba852f6f8f9b9346a9a2d0a09f0e4be Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <mikhaylovnikitka@gmail.com>
Date: Tue, 30 Jul 2024 10:34:47 +0200
Subject: [PATCH 1052/1488] Typo

---
 src/Interpreters/DatabaseCatalog.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp
index 98526e5c1cd..a8e5fd7e6aa 100644
--- a/src/Interpreters/DatabaseCatalog.cpp
+++ b/src/Interpreters/DatabaseCatalog.cpp
@@ -1355,7 +1355,7 @@ void DatabaseCatalog::dropTableDataTask()
         }
         catch (...)
         {
-            /// We don't re-throw expection, because we are in a background pool.
+            /// We don't re-throw exception, because we are in a background pool.
             tryLogCurrentException(log, "Cannot drop tables. Will retry later.");
         }
     }

From 3e6a1b99e023eb3d592c72c17ae4913a9074b5af Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Tue, 30 Jul 2024 11:05:45 +0200
Subject: [PATCH 1053/1488] Fix file/URI parsing with archive syntax

---
 src/IO/Archives/ArchiveUtils.cpp              | 50 +++++++++++++++++++
 src/IO/Archives/ArchiveUtils.h                | 14 ++++++
 src/IO/Archives/createArchiveReader.cpp       | 13 ++---
 src/IO/Archives/createArchiveWriter.cpp       |  9 ++--
 src/IO/S3/URI.cpp                             | 39 +++++++--------
 src/IO/S3/URI.h                               |  3 +-
 src/Storages/StorageFile.cpp                  |  8 ++-
 src/TableFunctions/TableFunctionFile.cpp      |  9 ++--
 .../03214_parsing_archive_name_file.reference | 12 +++++
 .../03214_parsing_archive_name_file.sh        | 21 ++++++++
 .../03215_parsing_archive_name_s3.reference   |  2 +
 .../03215_parsing_archive_name_s3.sql         |  6 +++
 .../data_minio/::03215_archive.csv            |  1 +
 .../data_minio/test :: 03215_archive.csv      |  1 +
 .../data_minio/test::03215_archive.csv        |  1 +
 15 files changed, 145 insertions(+), 44 deletions(-)
 create mode 100644 src/IO/Archives/ArchiveUtils.cpp
 create mode 100644 tests/queries/0_stateless/03214_parsing_archive_name_file.reference
 create mode 100755 tests/queries/0_stateless/03214_parsing_archive_name_file.sh
 create mode 100644 tests/queries/0_stateless/03215_parsing_archive_name_s3.reference
 create mode 100644 tests/queries/0_stateless/03215_parsing_archive_name_s3.sql
 create mode 100644 tests/queries/0_stateless/data_minio/::03215_archive.csv
 create mode 100644 tests/queries/0_stateless/data_minio/test :: 03215_archive.csv
 create mode 100644 tests/queries/0_stateless/data_minio/test::03215_archive.csv

diff --git a/src/IO/Archives/ArchiveUtils.cpp b/src/IO/Archives/ArchiveUtils.cpp
new file mode 100644
index 00000000000..50009087de3
--- /dev/null
+++ b/src/IO/Archives/ArchiveUtils.cpp
@@ -0,0 +1,50 @@
+#include <IO/Archives/ArchiveUtils.h>
+
+#include <string_view>
+#include <array>
+
+namespace DB
+{
+
+namespace
+{
+
+using namespace std::literals;
+constexpr std::array tar_extensions{".tar"sv, ".tar.gz"sv, ".tgz"sv, ".tar.zst"sv, ".tzst"sv, ".tar.xz"sv, ".tar.bz2"sv, ".tar.lzma"sv};
+constexpr std::array zip_extensions{".zip"sv, ".zipx"sv};
+constexpr std::array sevenz_extensiosns{".7z"sv};
+
+bool hasSupportedExtension(std::string_view path, const auto & supported_extensions)
+{
+    for (auto supported_extension : supported_extensions)
+    {
+        if (path.ends_with(supported_extension))
+            return true;
+    }
+
+    return false;
+}
+
+}
+
+bool hasSupportedTarExtension(std::string_view path)
+{
+    return hasSupportedExtension(path, tar_extensions);
+}
+
+bool hasSupportedZipExtension(std::string_view path)
+{
+    return hasSupportedExtension(path, zip_extensions);
+}
+
+bool hasSupported7zExtension(std::string_view path)
+{
+    return hasSupportedExtension(path, sevenz_extensiosns);
+}
+
+bool hasSupportedArchiveExtension(std::string_view path)
+{
+    return hasSupportedTarExtension(path) || hasSupportedZipExtension(path) || hasSupported7zExtension(path);
+}
+
+}
diff --git a/src/IO/Archives/ArchiveUtils.h b/src/IO/Archives/ArchiveUtils.h
index 1b66be005a2..cdb731d1d57 100644
--- a/src/IO/Archives/ArchiveUtils.h
+++ b/src/IO/Archives/ArchiveUtils.h
@@ -10,3 +10,17 @@
 #include <archive.h>
 #include <archive_entry.h>
 #endif
+
+#include <string_view>
+
+namespace DB
+{
+
+bool hasSupportedTarExtension(std::string_view path);
+bool hasSupportedZipExtension(std::string_view path);
+bool hasSupported7zExtension(std::string_view path);
+
+bool hasSupportedArchiveExtension(std::string_view path);
+
+
+}
diff --git a/src/IO/Archives/createArchiveReader.cpp b/src/IO/Archives/createArchiveReader.cpp
index 782602091ac..dfa098eede0 100644
--- a/src/IO/Archives/createArchiveReader.cpp
+++ b/src/IO/Archives/createArchiveReader.cpp
@@ -1,6 +1,7 @@
 #include <IO/Archives/LibArchiveReader.h>
 #include <IO/Archives/ZipArchiveReader.h>
 #include <IO/Archives/createArchiveReader.h>
+#include <IO/Archives/ArchiveUtils.h>
 #include <Common/Exception.h>
 
 
@@ -12,7 +13,6 @@ extern const int CANNOT_UNPACK_ARCHIVE;
 extern const int SUPPORT_IS_DISABLED;
 }
 
-
 std::shared_ptr<IArchiveReader> createArchiveReader(const String & path_to_archive)
 {
     return createArchiveReader(path_to_archive, {}, 0);
@@ -24,11 +24,7 @@ std::shared_ptr<IArchiveReader> createArchiveReader(
     [[maybe_unused]] const std::function<std::unique_ptr<SeekableReadBuffer>()> & archive_read_function,
     [[maybe_unused]] size_t archive_size)
 {
-    using namespace std::literals;
-    static constexpr std::array tar_extensions{
-        ".tar"sv, ".tar.gz"sv, ".tgz"sv, ".tar.zst"sv, ".tzst"sv, ".tar.xz"sv, ".tar.bz2"sv, ".tar.lzma"sv};
-
-    if (path_to_archive.ends_with(".zip") || path_to_archive.ends_with(".zipx"))
+    if (hasSupportedZipExtension(path_to_archive))
     {
 #if USE_MINIZIP
         return std::make_shared<ZipArchiveReader>(path_to_archive, archive_read_function, archive_size);
@@ -36,8 +32,7 @@ std::shared_ptr<IArchiveReader> createArchiveReader(
         throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "minizip library is disabled");
 #endif
     }
-    else if (std::any_of(
-                 tar_extensions.begin(), tar_extensions.end(), [&](const auto extension) { return path_to_archive.ends_with(extension); }))
+    else if (hasSupportedTarExtension(path_to_archive))
     {
 #if USE_LIBARCHIVE
         return std::make_shared<TarArchiveReader>(path_to_archive, archive_read_function);
@@ -45,7 +40,7 @@ std::shared_ptr<IArchiveReader> createArchiveReader(
         throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "libarchive library is disabled");
 #endif
     }
-    else if (path_to_archive.ends_with(".7z"))
+    else if (hasSupported7zExtension(path_to_archive))
     {
 #if USE_LIBARCHIVE
         return std::make_shared<SevenZipArchiveReader>(path_to_archive);
diff --git a/src/IO/Archives/createArchiveWriter.cpp b/src/IO/Archives/createArchiveWriter.cpp
index 9a169587088..53be0a85a10 100644
--- a/src/IO/Archives/createArchiveWriter.cpp
+++ b/src/IO/Archives/createArchiveWriter.cpp
@@ -1,3 +1,4 @@
+#include <IO/Archives/ArchiveUtils.h>
 #include <IO/Archives/LibArchiveWriter.h>
 #include <IO/Archives/TarArchiveWriter.h>
 #include <IO/Archives/ZipArchiveWriter.h>
@@ -24,10 +25,7 @@ std::shared_ptr<IArchiveWriter> createArchiveWriter(const String & path_to_archi
 std::shared_ptr<IArchiveWriter>
 createArchiveWriter(const String & path_to_archive, [[maybe_unused]] std::unique_ptr<WriteBuffer> archive_write_buffer)
 {
-    using namespace std::literals;
-    static constexpr std::array tar_extensions{
-        ".tar"sv, ".tar.gz"sv, ".tgz"sv, ".tar.bz2"sv, ".tar.lzma"sv, ".tar.zst"sv, ".tzst"sv, ".tar.xz"sv};
-    if (path_to_archive.ends_with(".zip") || path_to_archive.ends_with(".zipx"))
+    if (hasSupportedZipExtension(path_to_archive))
     {
 #if USE_MINIZIP
         return std::make_shared<ZipArchiveWriter>(path_to_archive, std::move(archive_write_buffer));
@@ -35,8 +33,7 @@ createArchiveWriter(const String & path_to_archive, [[maybe_unused]] std::unique
         throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "minizip library is disabled");
 #endif
     }
-    else if (std::any_of(
-                 tar_extensions.begin(), tar_extensions.end(), [&](const auto extension) { return path_to_archive.ends_with(extension); }))
+    else if (hasSupportedTarExtension(path_to_archive))
     {
 #if USE_LIBARCHIVE
         return std::make_shared<TarArchiveWriter>(path_to_archive, std::move(archive_write_buffer));
diff --git a/src/IO/S3/URI.cpp b/src/IO/S3/URI.cpp
index 4bf7a3ddf86..b9c400d2b98 100644
--- a/src/IO/S3/URI.cpp
+++ b/src/IO/S3/URI.cpp
@@ -6,6 +6,7 @@
 #include <Common/Exception.h>
 #include <Common/quoteString.h>
 #include <Common/re2.h>
+#include <IO/Archives/ArchiveUtils.h>
 
 #include <boost/algorithm/string/case_conv.hpp>
 
@@ -54,10 +55,7 @@ URI::URI(const std::string & uri_)
     static constexpr auto OSS = "OSS";
     static constexpr auto EOS = "EOS";
 
-    if (containsArchive(uri_))
-        std::tie(uri_str, archive_pattern) = getPathToArchiveAndArchivePattern(uri_);
-    else
-        uri_str = uri_;
+    std::tie(uri_str, archive_pattern) = getURIAndArchivePattern(uri_);
     uri = Poco::URI(uri_str);
 
     std::unordered_map<std::string, std::string> mapper;
@@ -167,32 +165,29 @@ void URI::validateBucket(const String & bucket, const Poco::URI & uri)
             !uri.empty() ? " (" + uri.toString() + ")" : "");
 }
 
-bool URI::containsArchive(const std::string & source)
+std::pair<std::string, std::optional<std::string>> URI::getURIAndArchivePattern(const std::string & source)
 {
     size_t pos = source.find("::");
-    return (pos != std::string::npos);
-}
+    if (pos == String::npos)
+        return {source, std::nullopt};
 
-std::pair<std::string, std::string> URI::getPathToArchiveAndArchivePattern(const std::string & source)
-{
-    size_t pos = source.find("::");
-    assert(pos != std::string::npos);
+    std::string_view path_to_archive_view = std::string_view{source}.substr(0, pos);
+    while (path_to_archive_view.ends_with(' '))
+        path_to_archive_view.remove_suffix(1);
 
-    std::string path_to_archive = source.substr(0, pos);
-    while ((!path_to_archive.empty()) && path_to_archive.ends_with(' '))
-        path_to_archive.pop_back();
+    if (path_to_archive_view.empty() || !hasSupportedArchiveExtension(path_to_archive_view))
+        return {source, std::nullopt};
 
-    if (path_to_archive.empty())
-        throw Exception(ErrorCodes::BAD_ARGUMENTS, "Path to archive is empty");
+    auto archive_uri = path_to_archive_view;
 
-    std::string_view path_in_archive_view = std::string_view{source}.substr(pos + 2);
-    while (path_in_archive_view.front() == ' ')
-        path_in_archive_view.remove_prefix(1);
+    std::string_view archive_pattern_view = std::string_view{source}.substr(pos + 2);
+    while (archive_pattern_view.front() == ' ')
+        archive_pattern_view.remove_prefix(1);
 
-    if (path_in_archive_view.empty())
-        throw Exception(ErrorCodes::BAD_ARGUMENTS, "Filename is empty");
+    if (archive_pattern_view.empty())
+        throw Exception(ErrorCodes::BAD_ARGUMENTS, "Archive pattern is empty");
 
-    return {path_to_archive, std::string{path_in_archive_view}};
+    return std::pair{std::string{archive_uri}, std::string{archive_pattern_view}};
 }
 }
 
diff --git a/src/IO/S3/URI.h b/src/IO/S3/URI.h
index 363f98c46f5..e4bb0d9eae1 100644
--- a/src/IO/S3/URI.h
+++ b/src/IO/S3/URI.h
@@ -42,8 +42,7 @@ struct URI
     static void validateBucket(const std::string & bucket, const Poco::URI & uri);
 
 private:
-    bool containsArchive(const std::string & source);
-    std::pair<std::string, std::string> getPathToArchiveAndArchivePattern(const std::string & source);
+    std::pair<std::string, std::optional<std::string>> getURIAndArchivePattern(const std::string & source);
 };
 
 }
diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp
index 98cd5c4dfa9..de56fcf66a0 100644
--- a/src/Storages/StorageFile.cpp
+++ b/src/Storages/StorageFile.cpp
@@ -25,6 +25,7 @@
 #include <IO/WriteHelpers.h>
 #include <IO/Archives/createArchiveReader.h>
 #include <IO/Archives/IArchiveReader.h>
+#include <IO/Archives/ArchiveUtils.h>
 #include <IO/PeekableReadBuffer.h>
 #include <IO/AsynchronousReadBufferFromFile.h>
 #include <Disks/IO/IOUringReader.h>
@@ -2247,8 +2248,11 @@ void StorageFile::parseFileSource(String source, String & filename, String & pat
     while (path_to_archive_view.ends_with(' '))
         path_to_archive_view.remove_suffix(1);
 
-    if (path_to_archive_view.empty())
-        throw Exception(ErrorCodes::BAD_ARGUMENTS, "Path to archive is empty");
+    if (path_to_archive_view.empty() || !hasSupportedArchiveExtension(path_to_archive_view))
+    {
+        filename = std::move(source);
+        return;
+    }
 
     path_to_archive = path_to_archive_view;
 
diff --git a/src/TableFunctions/TableFunctionFile.cpp b/src/TableFunctions/TableFunctionFile.cpp
index 1b6d86f8fa5..12b88ae2b14 100644
--- a/src/TableFunctions/TableFunctionFile.cpp
+++ b/src/TableFunctions/TableFunctionFile.cpp
@@ -63,9 +63,12 @@ std::optional<String> TableFunctionFile::tryGetFormatFromFirstArgument()
         return FormatFactory::instance().tryGetFormatFromFileName(filename);
 }
 
-StoragePtr TableFunctionFile::getStorage(const String & source,
-    const String & format_, const ColumnsDescription & columns,
-    ContextPtr global_context, const std::string & table_name,
+StoragePtr TableFunctionFile::getStorage(
+    const String & source,
+    const String & format_,
+    const ColumnsDescription & columns,
+    ContextPtr global_context,
+    const std::string & table_name,
     const std::string & compression_method_) const
 {
     // For `file` table function, we are going to use format settings from the
diff --git a/tests/queries/0_stateless/03214_parsing_archive_name_file.reference b/tests/queries/0_stateless/03214_parsing_archive_name_file.reference
new file mode 100644
index 00000000000..243a7c8fd02
--- /dev/null
+++ b/tests/queries/0_stateless/03214_parsing_archive_name_file.reference
@@ -0,0 +1,12 @@
+::nonexistentfile.csv
+1
+nonexistent::nonexistentfile.csv
+1
+nonexistent :: nonexistentfile.csv
+1
+nonexistent ::nonexistentfile.csv
+1
+nonexistent.tar.gz :: nonexistentfile.csv
+1
+nonexistent.zip:: nonexistentfile.csv
+1
diff --git a/tests/queries/0_stateless/03214_parsing_archive_name_file.sh b/tests/queries/0_stateless/03214_parsing_archive_name_file.sh
new file mode 100755
index 00000000000..32bf3246c84
--- /dev/null
+++ b/tests/queries/0_stateless/03214_parsing_archive_name_file.sh
@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+function try_to_read_file()
+{
+    file_to_read=$1
+    file_argument=$2
+
+    echo $file_argument
+    $CLICKHOUSE_LOCAL -q "SELECT * FROM file('$file_argument')" 2>&1 | rg -c "Cannot stat file.*$file_to_read"
+}
+
+try_to_read_file "::nonexistentfile.csv" "::nonexistentfile.csv"
+try_to_read_file "nonexistent::nonexistentfile.csv" "nonexistent::nonexistentfile.csv"
+try_to_read_file "nonexistent :: nonexistentfile.csv" "nonexistent :: nonexistentfile.csv"
+try_to_read_file "nonexistent ::nonexistentfile.csv" "nonexistent ::nonexistentfile.csv"
+try_to_read_file "nonexistent.tar.gz" "nonexistent.tar.gz :: nonexistentfile.csv"
+try_to_read_file "nonexistent.zip" "nonexistent.zip:: nonexistentfile.csv"
diff --git a/tests/queries/0_stateless/03215_parsing_archive_name_s3.reference b/tests/queries/0_stateless/03215_parsing_archive_name_s3.reference
new file mode 100644
index 00000000000..9dd925a7480
--- /dev/null
+++ b/tests/queries/0_stateless/03215_parsing_archive_name_s3.reference
@@ -0,0 +1,2 @@
+::03215_archive.csv test/::03215_archive.csv
+test::03215_archive.csv test/test::03215_archive.csv
diff --git a/tests/queries/0_stateless/03215_parsing_archive_name_s3.sql b/tests/queries/0_stateless/03215_parsing_archive_name_s3.sql
new file mode 100644
index 00000000000..9d01f53c838
--- /dev/null
+++ b/tests/queries/0_stateless/03215_parsing_archive_name_s3.sql
@@ -0,0 +1,6 @@
+-- Tags: no-fasttest
+-- Tag no-fasttest: Depends on AWS
+
+SELECT _file, _path FROM s3(s3_conn, filename='::03215_archive.csv') ORDER BY (_file, _path);
+SELECT _file, _path FROM s3(s3_conn, filename='test :: 03215_archive.csv') ORDER BY (_file, _path); -- { serverError STD_EXCEPTION }
+SELECT _file, _path FROM s3(s3_conn, filename='test::03215_archive.csv') ORDER BY (_file, _path);
diff --git a/tests/queries/0_stateless/data_minio/::03215_archive.csv b/tests/queries/0_stateless/data_minio/::03215_archive.csv
new file mode 100644
index 00000000000..d00491fd7e5
--- /dev/null
+++ b/tests/queries/0_stateless/data_minio/::03215_archive.csv
@@ -0,0 +1 @@
+1
diff --git a/tests/queries/0_stateless/data_minio/test :: 03215_archive.csv b/tests/queries/0_stateless/data_minio/test :: 03215_archive.csv
new file mode 100644
index 00000000000..d00491fd7e5
--- /dev/null
+++ b/tests/queries/0_stateless/data_minio/test :: 03215_archive.csv	
@@ -0,0 +1 @@
+1
diff --git a/tests/queries/0_stateless/data_minio/test::03215_archive.csv b/tests/queries/0_stateless/data_minio/test::03215_archive.csv
new file mode 100644
index 00000000000..d00491fd7e5
--- /dev/null
+++ b/tests/queries/0_stateless/data_minio/test::03215_archive.csv
@@ -0,0 +1 @@
+1

From 5381619b2ab465386f11e86242883419e48e5f6b Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Tue, 30 Jul 2024 11:36:25 +0200
Subject: [PATCH 1054/1488] Remove bad feature.

---
 CHANGELOG.md | 1 -
 1 file changed, 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 722ae4f8268..a4c873ba3f9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -65,7 +65,6 @@
 * The setting `optimize_trivial_insert_select` is disabled by default. In most cases, it should be beneficial. Nevertheless, if you are seeing slower INSERT SELECT or increased memory usage, you can enable it back or `SET compatibility = '24.6'`. [#58970](https://github.com/ClickHouse/ClickHouse/pull/58970) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
 * Print stacktrace and diagnostic info if `clickhouse-client` or `clickhouse-local` crashes. [#61109](https://github.com/ClickHouse/ClickHouse/pull/61109) ([Alexander Tokmakov](https://github.com/tavplubix)).
 * The result of `SHOW INDEX | INDEXES | INDICES | KEYS` was previously sorted by the primary key column names. Since this was unintuitive, the result is now sorted by the position of the primary key columns within the primary key. [#61131](https://github.com/ClickHouse/ClickHouse/pull/61131) ([Robert Schulze](https://github.com/rschu1ze)).
-* Allow matching column names in a case-insensitive manner when reading json files (`input_format_json_case_insensitive_column_matching`). [#61750](https://github.com/ClickHouse/ClickHouse/pull/61750) ([kevinyhzou](https://github.com/KevinyhZou)).
 * Support reading partitioned data DeltaLake data. Infer DeltaLake schema by reading metadata instead of data. [#63201](https://github.com/ClickHouse/ClickHouse/pull/63201) ([Kseniia Sumarokova](https://github.com/kssenii)).
 * In composable protocols TLS layer accepted only `certificateFile` and `privateKeyFile` parameters. https://clickhouse.com/docs/en/operations/settings/composable-protocols. [#63985](https://github.com/ClickHouse/ClickHouse/pull/63985) ([Anton Ivashkin](https://github.com/ianton-ru)).
 * Added profile event `SelectQueriesWithPrimaryKeyUsage` which indicates how many SELECT queries use the primary key to evaluate the WHERE clause. [#64492](https://github.com/ClickHouse/ClickHouse/pull/64492) ([0x01f](https://github.com/0xfei)).

From 4d4fc8fd6f0123613305423d861429f54222d23f Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Tue, 30 Jul 2024 11:57:37 +0200
Subject: [PATCH 1055/1488] Add setting to disable archive path syntax

---
 src/Core/Settings.h                                |  1 +
 src/Core/SettingsChangesHistory.cpp                |  1 +
 src/IO/S3/URI.cpp                                  |  8 ++++++--
 src/IO/S3/URI.h                                    |  2 +-
 src/Storages/ObjectStorage/S3/Configuration.cpp    |  8 ++++----
 src/Storages/StorageFile.cpp                       | 14 ++++++++++++--
 src/Storages/StorageFile.h                         |  2 +-
 src/TableFunctions/TableFunctionFile.cpp           |  5 +++--
 .../03214_parsing_archive_name_file.reference      |  4 ++++
 .../0_stateless/03214_parsing_archive_name_file.sh | 10 ++++++++--
 .../03215_parsing_archive_name_s3.reference        |  1 +
 .../0_stateless/03215_parsing_archive_name_s3.sql  |  1 +
 .../data_minio/test.zip::03215_archive.csv         |  1 +
 13 files changed, 44 insertions(+), 14 deletions(-)
 create mode 100644 tests/queries/0_stateless/data_minio/test.zip::03215_archive.csv

diff --git a/src/Core/Settings.h b/src/Core/Settings.h
index 4fc2034b855..5114a8204cd 100644
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@@ -936,6 +936,7 @@ class IColumn;
     M(UInt64, parallel_replicas_min_number_of_rows_per_replica, 0, "Limit the number of replicas used in a query to (estimated rows to read / min_number_of_rows_per_replica). The max is still limited by 'max_parallel_replicas'", 0) \
     M(Bool, parallel_replicas_prefer_local_join, true, "If true, and JOIN can be executed with parallel replicas algorithm, and all storages of right JOIN part are *MergeTree, local JOIN will be used instead of GLOBAL JOIN.", 0) \
     M(UInt64, parallel_replicas_mark_segment_size, 128, "Parts virtually divided into segments to be distributed between replicas for parallel reading. This setting controls the size of these segments. Not recommended to change until you're absolutely sure in what you're doing", 0) \
+    M(Bool, allow_archive_path_syntax, true, "File/S3 engines/table function will parse paths with '::' as '<archive> :: <file>' if archive has correct extension", 0) \
     \
     M(Bool, allow_experimental_inverted_index, false, "If it is set to true, allow to use experimental inverted index.", 0) \
     M(Bool, allow_experimental_full_text_index, false, "If it is set to true, allow to use experimental full-text index.", 0) \
diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp
index 9faf77e9087..8483a267237 100644
--- a/src/Core/SettingsChangesHistory.cpp
+++ b/src/Core/SettingsChangesHistory.cpp
@@ -124,6 +124,7 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
               {"cast_string_to_dynamic_use_inference", false, false, "Add setting to allow converting String to Dynamic through parsing"},
               {"allow_experimental_dynamic_type", false, false, "Add new experimental Dynamic type"},
               {"azure_max_blocks_in_multipart_upload", 50000, 50000, "Maximum number of blocks in multipart upload for Azure."},
+              {"allow_archive_path_syntax", false, true, "Added new setting to allow disabling archive path syntax."},
               }},
     {"24.4", {{"input_format_json_throw_on_bad_escape_sequence", true, true, "Allow to save JSON strings with bad escape sequences"},
               {"max_parsing_threads", 0, 0, "Add a separate setting to control number of threads in parallel parsing from files"},
diff --git a/src/IO/S3/URI.cpp b/src/IO/S3/URI.cpp
index b9c400d2b98..33a4939c810 100644
--- a/src/IO/S3/URI.cpp
+++ b/src/IO/S3/URI.cpp
@@ -30,7 +30,7 @@ namespace ErrorCodes
 namespace S3
 {
 
-URI::URI(const std::string & uri_)
+URI::URI(const std::string & uri_, bool allow_archive_path_syntax)
 {
     /// Case when bucket name represented in domain name of S3 URL.
     /// E.g. (https://bucket-name.s3.region.amazonaws.com/key)
@@ -55,7 +55,11 @@ URI::URI(const std::string & uri_)
     static constexpr auto OSS = "OSS";
     static constexpr auto EOS = "EOS";
 
-    std::tie(uri_str, archive_pattern) = getURIAndArchivePattern(uri_);
+    if (!allow_archive_path_syntax)
+        uri_str = uri_;
+    else
+        std::tie(uri_str, archive_pattern) = getURIAndArchivePattern(uri_);
+
     uri = Poco::URI(uri_str);
 
     std::unordered_map<std::string, std::string> mapper;
diff --git a/src/IO/S3/URI.h b/src/IO/S3/URI.h
index e4bb0d9eae1..80e2da96cd4 100644
--- a/src/IO/S3/URI.h
+++ b/src/IO/S3/URI.h
@@ -36,7 +36,7 @@ struct URI
     bool is_virtual_hosted_style;
 
     URI() = default;
-    explicit URI(const std::string & uri_);
+    explicit URI(const std::string & uri_, bool allow_archive_path_syntax = false);
     void addRegionToURI(const std::string & region);
 
     static void validateBucket(const std::string & bucket, const Poco::URI & uri);
diff --git a/src/Storages/ObjectStorage/S3/Configuration.cpp b/src/Storages/ObjectStorage/S3/Configuration.cpp
index 094ca069e7a..7542f59dcc4 100644
--- a/src/Storages/ObjectStorage/S3/Configuration.cpp
+++ b/src/Storages/ObjectStorage/S3/Configuration.cpp
@@ -142,14 +142,14 @@ ObjectStoragePtr StorageS3Configuration::createObjectStorage(ContextPtr context,
 
 void StorageS3Configuration::fromNamedCollection(const NamedCollection & collection, ContextPtr context)
 {
-    const auto settings = context->getSettingsRef();
+    const auto & settings = context->getSettingsRef();
     validateNamedCollection(collection, required_configuration_keys, optional_configuration_keys);
 
     auto filename = collection.getOrDefault<String>("filename", "");
     if (!filename.empty())
-        url = S3::URI(std::filesystem::path(collection.get<String>("url")) / filename);
+        url = S3::URI(std::filesystem::path(collection.get<String>("url")) / filename, settings.allow_archive_path_syntax);
     else
-        url = S3::URI(collection.get<String>("url"));
+        url = S3::URI(collection.get<String>("url"), settings.allow_archive_path_syntax);
 
     auth_settings.access_key_id = collection.getOrDefault<String>("access_key_id", "");
     auth_settings.secret_access_key = collection.getOrDefault<String>("secret_access_key", "");
@@ -330,7 +330,7 @@ void StorageS3Configuration::fromAST(ASTs & args, ContextPtr context, bool with_
     }
 
     /// This argument is always the first
-    url = S3::URI(checkAndGetLiteralArgument<String>(args[0], "url"));
+    url = S3::URI(checkAndGetLiteralArgument<String>(args[0], "url"), context->getSettingsRef().allow_archive_path_syntax);
 
     if (engine_args_to_idx.contains("format"))
     {
diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp
index de56fcf66a0..efb39f90053 100644
--- a/src/Storages/StorageFile.cpp
+++ b/src/Storages/StorageFile.cpp
@@ -2208,7 +2208,11 @@ void registerStorageFile(StorageFactory & factory)
                 else if (type == Field::Types::UInt64)
                     source_fd = static_cast<int>(literal->value.get<UInt64>());
                 else if (type == Field::Types::String)
-                    StorageFile::parseFileSource(literal->value.get<String>(), source_path, storage_args.path_to_archive);
+                    StorageFile::parseFileSource(
+                        literal->value.get<String>(),
+                        source_path,
+                        storage_args.path_to_archive,
+                        factory_args.getLocalContext()->getSettingsRef().allow_archive_path_syntax);
                 else
                     throw Exception(ErrorCodes::BAD_ARGUMENTS, "Second argument must be path or file descriptor");
             }
@@ -2235,8 +2239,14 @@ SchemaCache & StorageFile::getSchemaCache(const ContextPtr & context)
     return schema_cache;
 }
 
-void StorageFile::parseFileSource(String source, String & filename, String & path_to_archive)
+void StorageFile::parseFileSource(String source, String & filename, String & path_to_archive, bool allow_archive_path_syntax)
 {
+    if (!allow_archive_path_syntax)
+    {
+        filename = std::move(source);
+        return;
+    }
+
     size_t pos = source.find("::");
     if (pos == String::npos)
     {
diff --git a/src/Storages/StorageFile.h b/src/Storages/StorageFile.h
index 895a8a663b8..bb969c1877c 100644
--- a/src/Storages/StorageFile.h
+++ b/src/Storages/StorageFile.h
@@ -128,7 +128,7 @@ public:
 
     static SchemaCache & getSchemaCache(const ContextPtr & context);
 
-    static void parseFileSource(String source, String & filename, String & path_to_archive);
+    static void parseFileSource(String source, String & filename, String & path_to_archive, bool allow_archive_path_syntax);
 
     static ArchiveInfo getArchiveInfo(
         const std::string & path_to_archive,
diff --git a/src/TableFunctions/TableFunctionFile.cpp b/src/TableFunctions/TableFunctionFile.cpp
index 12b88ae2b14..af327cfe54e 100644
--- a/src/TableFunctions/TableFunctionFile.cpp
+++ b/src/TableFunctions/TableFunctionFile.cpp
@@ -26,7 +26,7 @@ void TableFunctionFile::parseFirstArguments(const ASTPtr & arg, const ContextPtr
     if (context->getApplicationType() != Context::ApplicationType::LOCAL)
     {
         ITableFunctionFileLike::parseFirstArguments(arg, context);
-        StorageFile::parseFileSource(std::move(filename), filename, path_to_archive);
+        StorageFile::parseFileSource(std::move(filename), filename, path_to_archive, context->getSettingsRef().allow_archive_path_syntax);
         return;
     }
 
@@ -42,7 +42,8 @@ void TableFunctionFile::parseFirstArguments(const ASTPtr & arg, const ContextPtr
         else if (filename == "stderr")
             fd = STDERR_FILENO;
         else
-            StorageFile::parseFileSource(std::move(filename), filename, path_to_archive);
+            StorageFile::parseFileSource(
+                std::move(filename), filename, path_to_archive, context->getSettingsRef().allow_archive_path_syntax);
     }
     else if (type == Field::Types::Int64 || type == Field::Types::UInt64)
     {
diff --git a/tests/queries/0_stateless/03214_parsing_archive_name_file.reference b/tests/queries/0_stateless/03214_parsing_archive_name_file.reference
index 243a7c8fd02..d793d26dfc3 100644
--- a/tests/queries/0_stateless/03214_parsing_archive_name_file.reference
+++ b/tests/queries/0_stateless/03214_parsing_archive_name_file.reference
@@ -10,3 +10,7 @@ nonexistent.tar.gz :: nonexistentfile.csv
 1
 nonexistent.zip:: nonexistentfile.csv
 1
+nonexistent.tar.gz :: nonexistentfile.csv SETTINGS allow_archive_path_syntax=0
+1
+nonexistent.zip:: nonexistentfile.csv SETTINGS allow_archive_path_syntax=0
+1
diff --git a/tests/queries/0_stateless/03214_parsing_archive_name_file.sh b/tests/queries/0_stateless/03214_parsing_archive_name_file.sh
index 32bf3246c84..2f77627f6be 100755
--- a/tests/queries/0_stateless/03214_parsing_archive_name_file.sh
+++ b/tests/queries/0_stateless/03214_parsing_archive_name_file.sh
@@ -8,14 +8,20 @@ function try_to_read_file()
 {
     file_to_read=$1
     file_argument=$2
+    settings=$3
 
-    echo $file_argument
-    $CLICKHOUSE_LOCAL -q "SELECT * FROM file('$file_argument')" 2>&1 | rg -c "Cannot stat file.*$file_to_read"
+    echo $file_argument $settings
+    $CLICKHOUSE_LOCAL -q "SELECT * FROM file('$file_argument') $settings" 2>&1 | rg -c "Cannot stat file.*$file_to_read"
 }
 
+# if archive extension is not detected for part before '::', path is taken as is
 try_to_read_file "::nonexistentfile.csv" "::nonexistentfile.csv"
 try_to_read_file "nonexistent::nonexistentfile.csv" "nonexistent::nonexistentfile.csv"
 try_to_read_file "nonexistent :: nonexistentfile.csv" "nonexistent :: nonexistentfile.csv"
 try_to_read_file "nonexistent ::nonexistentfile.csv" "nonexistent ::nonexistentfile.csv"
+# if archive extension is detected for part before '::', path is split into archive and filename
 try_to_read_file "nonexistent.tar.gz" "nonexistent.tar.gz :: nonexistentfile.csv"
 try_to_read_file "nonexistent.zip" "nonexistent.zip:: nonexistentfile.csv"
+# disabling archive syntax will always parse path as is
+try_to_read_file "nonexistent.tar.gz :: nonexistentfile.csv" "nonexistent.tar.gz :: nonexistentfile.csv" "SETTINGS allow_archive_path_syntax=0"
+try_to_read_file "nonexistent.zip:: nonexistentfile.csv" "nonexistent.zip:: nonexistentfile.csv" "SETTINGS allow_archive_path_syntax=0"
diff --git a/tests/queries/0_stateless/03215_parsing_archive_name_s3.reference b/tests/queries/0_stateless/03215_parsing_archive_name_s3.reference
index 9dd925a7480..b4804c82dc2 100644
--- a/tests/queries/0_stateless/03215_parsing_archive_name_s3.reference
+++ b/tests/queries/0_stateless/03215_parsing_archive_name_s3.reference
@@ -1,2 +1,3 @@
 ::03215_archive.csv test/::03215_archive.csv
 test::03215_archive.csv test/test::03215_archive.csv
+test.zip::03215_archive.csv test/test.zip::03215_archive.csv
diff --git a/tests/queries/0_stateless/03215_parsing_archive_name_s3.sql b/tests/queries/0_stateless/03215_parsing_archive_name_s3.sql
index 9d01f53c838..3a7ed0b864c 100644
--- a/tests/queries/0_stateless/03215_parsing_archive_name_s3.sql
+++ b/tests/queries/0_stateless/03215_parsing_archive_name_s3.sql
@@ -4,3 +4,4 @@
 SELECT _file, _path FROM s3(s3_conn, filename='::03215_archive.csv') ORDER BY (_file, _path);
 SELECT _file, _path FROM s3(s3_conn, filename='test :: 03215_archive.csv') ORDER BY (_file, _path); -- { serverError STD_EXCEPTION }
 SELECT _file, _path FROM s3(s3_conn, filename='test::03215_archive.csv') ORDER BY (_file, _path);
+SELECT _file, _path FROM s3(s3_conn, filename='test.zip::03215_archive.csv') ORDER BY (_file, _path) SETTINGS allow_archive_path_syntax=0;
diff --git a/tests/queries/0_stateless/data_minio/test.zip::03215_archive.csv b/tests/queries/0_stateless/data_minio/test.zip::03215_archive.csv
new file mode 100644
index 00000000000..d00491fd7e5
--- /dev/null
+++ b/tests/queries/0_stateless/data_minio/test.zip::03215_archive.csv
@@ -0,0 +1 @@
+1

From b1e80883f0324995d84250d5edf37fd8ab475987 Mon Sep 17 00:00:00 2001
From: Nikita Fomichev <nikita.fomichev@clickhouse.com>
Date: Tue, 30 Jul 2024 11:39:22 +0200
Subject: [PATCH 1056/1488] `accept_invalid_certificate` in client config:
 additional testing #65238

---
 .../configs/ssl_config_strict.xml             | 17 +++++++++
 .../test_accept_invalid_certificate/test.py   | 35 +++++++++++++++++++
 2 files changed, 52 insertions(+)
 create mode 100644 tests/integration/test_accept_invalid_certificate/configs/ssl_config_strict.xml

diff --git a/tests/integration/test_accept_invalid_certificate/configs/ssl_config_strict.xml b/tests/integration/test_accept_invalid_certificate/configs/ssl_config_strict.xml
new file mode 100644
index 00000000000..a4383a77ac4
--- /dev/null
+++ b/tests/integration/test_accept_invalid_certificate/configs/ssl_config_strict.xml
@@ -0,0 +1,17 @@
+<clickhouse>
+    <!-- Native interface with TLS.
+         You have to configure certificate to enable this interface.
+         See the openSSL section below.
+    -->
+    <tcp_port_secure>9440</tcp_port_secure>
+
+    <!-- Used with https_port and tcp_port_secure. Full ssl options list: https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/SSLManager.h#L71 -->
+    <openSSL replace="replace">
+        <server> <!-- Used for https server AND secure tcp port -->
+            <certificateFile>/etc/clickhouse-server/config.d/self-cert.pem</certificateFile>
+            <privateKeyFile>/etc/clickhouse-server/config.d/self-key.pem</privateKeyFile>
+            <caConfig>/etc/clickhouse-server/config.d/ca-cert.pem</caConfig>
+            <verificationMode>strict</verificationMode>
+        </server>
+    </openSSL>
+</clickhouse>
diff --git a/tests/integration/test_accept_invalid_certificate/test.py b/tests/integration/test_accept_invalid_certificate/test.py
index 87229d75f90..f43e9e6140a 100644
--- a/tests/integration/test_accept_invalid_certificate/test.py
+++ b/tests/integration/test_accept_invalid_certificate/test.py
@@ -17,6 +17,19 @@ instance = cluster.add_instance(
         "certs/self-cert.pem",
         "certs/ca-cert.pem",
     ],
+    with_zookeeper=False,
+)
+
+
+node1 = cluster.add_instance(
+    "node1",
+    main_configs=[
+        "configs/ssl_config_strict.xml",
+        "certs/self-key.pem",
+        "certs/self-cert.pem",
+        "certs/ca-cert.pem",
+    ],
+    with_zookeeper=False,
 )
 
 
@@ -90,3 +103,25 @@ def test_connection_accept():
         )
         == "1\n"
     )
+
+
+def test_strict_reject():
+    with pytest.raises(Exception) as err:
+        execute_query_native(node1, "SELECT 1", "<clickhouse></clickhouse>")
+    assert "certificate verify failed" in str(err.value)
+
+
+def test_strict_reject_with_config():
+    with pytest.raises(Exception) as err:
+        execute_query_native(node1, "SELECT 1", config_accept)
+    assert "alert certificate required" in str(err.value)
+
+
+def test_strict_connection_reject():
+    with pytest.raises(Exception) as err:
+        execute_query_native(
+            node1,
+            "SELECT 1",
+            config_connection_accept.format(ip_address=f"{instance.ip_address}"),
+        )
+    assert "certificate verify failed" in str(err.value)

From bdf98cbcc0121ab94dd1db39fc5cf977a7ed42ad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Tue, 30 Jul 2024 10:06:01 +0000
Subject: [PATCH 1057/1488] Fix public backports

---
 tests/ci/cherry_pick.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/ci/cherry_pick.py b/tests/ci/cherry_pick.py
index 0b2aa9a2d35..b660ad2c040 100644
--- a/tests/ci/cherry_pick.py
+++ b/tests/ci/cherry_pick.py
@@ -505,7 +505,7 @@ class Backport:
                 ReleaseBranch(
                     (
                         br
-                        if self._repo_name == "ClickHouse/Clickhouse"
+                        if self._repo_name == "ClickHouse/ClickHouse"
                         else f"release/{br}"
                     ),
                     pr,

From 9a05a3ed9e7cccada42f49a0cd5c3896010f9edb Mon Sep 17 00:00:00 2001
From: Nikita Fomichev <nikita.fomichev@clickhouse.com>
Date: Tue, 30 Jul 2024 12:07:34 +0200
Subject: [PATCH 1058/1488] Add missing documentation for `groupConcat` after
 series of reverts and merges
 https://github.com/ClickHouse/ClickHouse/pull/65384

---
 .../reference/groupconcat.md                  | 90 +++++++++++++++++++
 .../aspell-ignore/en/aspell-dict.txt          |  2 +
 2 files changed, 92 insertions(+)
 create mode 100644 docs/en/sql-reference/aggregate-functions/reference/groupconcat.md

diff --git a/docs/en/sql-reference/aggregate-functions/reference/groupconcat.md b/docs/en/sql-reference/aggregate-functions/reference/groupconcat.md
new file mode 100644
index 00000000000..072252de8c9
--- /dev/null
+++ b/docs/en/sql-reference/aggregate-functions/reference/groupconcat.md
@@ -0,0 +1,90 @@
+---
+slug: /en/sql-reference/aggregate-functions/reference/groupconcat
+sidebar_position: 363
+sidebar_label: groupConcat
+title: groupConcat
+---
+
+Calculates a concatenated string from a group of strings, optionally separated by a delimiter, and optionally limited by a maximum number of elements.
+
+**Syntax**
+
+``` sql
+groupConcat(expression [, delimiter] [, limit]);
+```
+
+**Arguments**
+
+- `expression` — The expression or column name that outputs strings to be concatenated..
+- `delimiter` — A [string](../../../sql-reference/data-types/string.md) that will be used to separate concatenated values. This parameter is optional and defaults to an empty string if not specified.
+- `limit` — A positive [integer](../../../sql-reference/data-types/int-uint.md) specifying the maximum number of elements to concatenate. If more elements are present, excess elements are ignored. This parameter is optional.
+
+:::note
+If delimiter is specified without limit, it must be the first parameter following the expression. If both delimiter and limit are specified, delimiter must precede limit.
+:::
+
+**Returned value**
+
+- Returns a [string](../../../sql-reference/data-types/string.md) consisting of the concatenated values of the column or expression. If the group has no elements or only null elements, and the function does not specify a handling for only null values, the result is a nullable string with a null value.
+
+**Examples**
+
+Input table:
+
+``` text
+┌─id─┬─name─┐
+│ 1  │  John│
+│ 2  │  Jane│
+│ 3  │   Bob│
+└────┴──────┘
+```
+
+1.	Basic usage without a delimiter:
+
+Query:
+
+``` sql
+SELECT groupConcat(Name) FROM Employees;
+```
+
+Result:
+
+``` text
+JohnJaneBob
+```
+
+This concatenates all names into one continuous string without any separator.
+
+
+2. Using comma as a delimiter:
+
+Query:
+
+``` sql
+SELECT groupConcat(Name, ', ', 2) FROM Employees;
+```
+
+Result:
+
+``` text
+John, Jane, Bob
+```
+
+This output shows the names separated by a comma followed by a space.
+
+
+3. Limiting the number of concatenated elements
+
+Query:
+
+``` sql
+SELECT groupConcat(Name, ', ', 2) FROM Employees;
+```
+
+Result:
+
+``` text
+John, Jane
+```
+
+This query limits the output to the first two names, even though there are more names in the table.
diff --git a/utils/check-style/aspell-ignore/en/aspell-dict.txt b/utils/check-style/aspell-ignore/en/aspell-dict.txt
index b21ae0764c6..d82b70cfdb4 100644
--- a/utils/check-style/aspell-ignore/en/aspell-dict.txt
+++ b/utils/check-style/aspell-ignore/en/aspell-dict.txt
@@ -1733,6 +1733,7 @@ groupBitmap
 groupBitmapAnd
 groupBitmapOr
 groupBitmapXor
+groupConcat
 groupUniqArray
 grouparray
 grouparrayinsertat
@@ -1749,6 +1750,7 @@ groupbitmapor
 groupbitmapxor
 groupbitor
 groupbitxor
+groupconcat
 groupuniqarray
 grpc
 grpcio

From fb466287dac16801518547f34b42edbc16a57fae Mon Sep 17 00:00:00 2001
From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com>
Date: Tue, 30 Jul 2024 12:13:47 +0200
Subject: [PATCH 1059/1488] Update 02150_index_hypothesis_race_long.sh

---
 tests/queries/0_stateless/02150_index_hypothesis_race_long.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/02150_index_hypothesis_race_long.sh b/tests/queries/0_stateless/02150_index_hypothesis_race_long.sh
index be7cfa78492..c29b604d23d 100755
--- a/tests/queries/0_stateless/02150_index_hypothesis_race_long.sh
+++ b/tests/queries/0_stateless/02150_index_hypothesis_race_long.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Tags: no-distributed-cache
+# Tags: long, no-distributed-cache
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh

From d7534c5b40c315cdb3d52101f626dbee7d565cdb Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Tue, 30 Jul 2024 12:20:54 +0200
Subject: [PATCH 1060/1488] Update
 03210_optimize_rewrite_aggregate_function_with_if_return_type_bug.sql

---
 ...imize_rewrite_aggregate_function_with_if_return_type_bug.sql | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/queries/0_stateless/03210_optimize_rewrite_aggregate_function_with_if_return_type_bug.sql b/tests/queries/0_stateless/03210_optimize_rewrite_aggregate_function_with_if_return_type_bug.sql
index b620a6434bf..565a481940a 100644
--- a/tests/queries/0_stateless/03210_optimize_rewrite_aggregate_function_with_if_return_type_bug.sql
+++ b/tests/queries/0_stateless/03210_optimize_rewrite_aggregate_function_with_if_return_type_bug.sql
@@ -1,3 +1,5 @@
+SET allow_experimental_analyzer = 1;
+
 -- For function count, rewrite countState to countStateIf changes the type from AggregateFunction(count, Nullable(UInt64)) to AggregateFunction(count, UInt64)
 -- We can cast AggregateFunction(count, UInt64) back to AggregateFunction(count, Nullable(UInt64)) with additional _CAST
 select hex(countState(if(toNullable(number % 2 = 0), number, null))) from numbers(5) settings optimize_rewrite_aggregate_function_with_if=1;

From 6ef628a7c80ebd6ec727365ee69d8141a4f11400 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Tue, 30 Jul 2024 10:33:22 +0000
Subject: [PATCH 1061/1488] Fixing build.

---
 src/Processors/QueryPlan/DistinctStep.cpp | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/Processors/QueryPlan/DistinctStep.cpp b/src/Processors/QueryPlan/DistinctStep.cpp
index a481454139d..b1c24fc01ce 100644
--- a/src/Processors/QueryPlan/DistinctStep.cpp
+++ b/src/Processors/QueryPlan/DistinctStep.cpp
@@ -10,6 +10,11 @@
 namespace DB
 {
 
+namespace ErrorCodes
+{
+    extern const int LOGICAL_ERROR;
+}
+
 static ITransformingStep::Traits getTraits(bool pre_distinct)
 {
     const bool preserves_number_of_streams = pre_distinct;
@@ -90,7 +95,8 @@ void DistinctStep::transformPipeline(QueryPipelineBuilder & pipeline, const Buil
             /// final distinct for sorted stream (sorting inside and among chunks)
             if (input_stream.sort_scope == DataStream::SortScope::Global)
             {
-                assert(input_stream.has_single_port);
+                if (pipeline.getNumStreams() != 1)
+                    throw Exception(ErrorCodes::LOGICAL_ERROR, "DistinctStep with in-order expects single input");
 
                 if (distinct_sort_desc.size() < columns.size())
                 {

From 0f8feff4d3806fa6f81d24184ab68bcd6e727551 Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Tue, 30 Jul 2024 12:34:30 +0200
Subject: [PATCH 1062/1488] Add KeeperMap retries

---
 src/Storages/StorageKeeperMap.cpp | 132 +++++++++++++++++-------------
 1 file changed, 73 insertions(+), 59 deletions(-)

diff --git a/src/Storages/StorageKeeperMap.cpp b/src/Storages/StorageKeeperMap.cpp
index ef157239e26..5534bb7f346 100644
--- a/src/Storages/StorageKeeperMap.cpp
+++ b/src/Storages/StorageKeeperMap.cpp
@@ -35,6 +35,7 @@
 #include <Storages/StorageInMemoryMetadata.h>
 #include <Storages/checkAndGetLiteralArgument.h>
 
+#include "Common/ZooKeeper/ZooKeeperRetries.h"
 #include <Common/Base64.h>
 #include <Common/Exception.h>
 #include <Common/FailPoint.h>
@@ -120,7 +121,7 @@ public:
         : SinkToStorage(header), storage(storage_), context(std::move(context_))
     {
         auto primary_key = storage.getPrimaryKey();
-        assert(primary_key.size() == 1);
+        chassert(primary_key.size() == 1);
         primary_key_pos = getHeader().getPositionByName(primary_key[0]);
     }
 
@@ -171,76 +172,89 @@ public:
     template <bool for_update>
     void finalize(bool strict)
     {
-        auto zookeeper = storage.getClient();
+        const auto & settings = context->getSettingsRef();
 
-        auto keys_limit = storage.keysLimit();
+        ZooKeeperRetriesControl zk_retry{
+            getName(),
+            getLogger(getName()),
+            ZooKeeperRetriesInfo{
+                settings.insert_keeper_max_retries,
+                settings.insert_keeper_retry_initial_backoff_ms,
+                settings.insert_keeper_retry_max_backoff_ms},
+            context->getProcessListElement()};
 
-        size_t current_keys_num = 0;
-        size_t new_keys_num = 0;
-
-        // We use keys limit as a soft limit so we ignore some cases when it can be still exceeded
-        // (e.g if parallel insert queries are being run)
-        if (keys_limit != 0)
+        retries_ctl.retryLoop([&]()
         {
-            Coordination::Stat data_stat;
-            zookeeper->get(storage.dataPath(), &data_stat);
-            current_keys_num = data_stat.numChildren;
-        }
+            auto zookeeper = storage.getClient();
+            auto keys_limit = storage.keysLimit();
 
-        std::vector<std::string> key_paths;
-        key_paths.reserve(new_values.size());
-        for (const auto & [key, _] : new_values)
-            key_paths.push_back(storage.fullPathForKey(key));
+            size_t current_keys_num = 0;
+            size_t new_keys_num = 0;
 
-        zkutil::ZooKeeper::MultiExistsResponse results;
-
-        if constexpr (!for_update)
-        {
-            if (!strict)
-                results = zookeeper->exists(key_paths);
-        }
-
-        Coordination::Requests requests;
-        requests.reserve(key_paths.size());
-        for (size_t i = 0; i < key_paths.size(); ++i)
-        {
-            auto key = fs::path(key_paths[i]).filename();
-
-            if constexpr (for_update)
+            // We use keys limit as a soft limit so we ignore some cases when it can be still exceeded
+            // (e.g if parallel insert queries are being run)
+            if (keys_limit != 0)
             {
-                int32_t version = -1;
-                if (strict)
-                    version = versions.at(key);
-
-                requests.push_back(zkutil::makeSetRequest(key_paths[i], new_values[key], version));
+                Coordination::Stat data_stat;
+                zookeeper->get(storage.dataPath(), &data_stat);
+                current_keys_num = data_stat.numChildren;
             }
-            else
+
+            std::vector<std::string> key_paths;
+            key_paths.reserve(new_values.size());
+            for (const auto & [key, _] : new_values)
+                key_paths.push_back(storage.fullPathForKey(key));
+
+            zkutil::ZooKeeper::MultiExistsResponse results;
+
+            if constexpr (!for_update)
             {
-                if (!strict && results[i].error == Coordination::Error::ZOK)
+                if (!strict)
+                    results = zookeeper->exists(key_paths);
+            }
+
+            Coordination::Requests requests;
+            requests.reserve(key_paths.size());
+            for (size_t i = 0; i < key_paths.size(); ++i)
+            {
+                auto key = fs::path(key_paths[i]).filename();
+
+                if constexpr (for_update)
                 {
-                    requests.push_back(zkutil::makeSetRequest(key_paths[i], new_values[key], -1));
+                    int32_t version = -1;
+                    if (strict)
+                        version = versions.at(key);
+
+                    requests.push_back(zkutil::makeSetRequest(key_paths[i], new_values[key], version));
                 }
                 else
                 {
-                    requests.push_back(zkutil::makeCreateRequest(key_paths[i], new_values[key], zkutil::CreateMode::Persistent));
-                    ++new_keys_num;
+                    if (!strict && results[i].error == Coordination::Error::ZOK)
+                    {
+                        requests.push_back(zkutil::makeSetRequest(key_paths[i], new_values[key], -1));
+                    }
+                    else
+                    {
+                        requests.push_back(zkutil::makeCreateRequest(key_paths[i], new_values[key], zkutil::CreateMode::Persistent));
+                        ++new_keys_num;
+                    }
                 }
             }
-        }
 
-        if (new_keys_num != 0)
-        {
-            auto will_be = current_keys_num + new_keys_num;
-            if (keys_limit != 0 && will_be > keys_limit)
-                throw Exception(
-                    ErrorCodes::LIMIT_EXCEEDED,
-                    "Limit would be exceeded by inserting {} new key(s). Limit is {}, while the number of keys would be {}",
-                    new_keys_num,
-                    keys_limit,
-                    will_be);
-        }
+            if (new_keys_num != 0)
+            {
+                auto will_be = current_keys_num + new_keys_num;
+                if (keys_limit != 0 && will_be > keys_limit)
+                    throw Exception(
+                        ErrorCodes::LIMIT_EXCEEDED,
+                        "Limit would be exceeded by inserting {} new key(s). Limit is {}, while the number of keys would be {}",
+                        new_keys_num,
+                        keys_limit,
+                        will_be);
+            }
 
-        zookeeper->multi(requests, /* check_session_valid */ true);
+            zookeeper->multi(requests, /* check_session_valid */ true);
+        });
     }
 };
 
@@ -529,8 +543,8 @@ Pipe StorageKeeperMap::read(
         size_t num_keys = keys->size();
         size_t num_threads = std::min<size_t>(num_streams, keys->size());
 
-        assert(num_keys <= std::numeric_limits<uint32_t>::max());
-        assert(num_threads <= std::numeric_limits<uint32_t>::max());
+        chassert(num_keys <= std::numeric_limits<uint32_t>::max());
+        chassert(num_threads <= std::numeric_limits<uint32_t>::max());
 
         for (size_t thread_idx = 0; thread_idx < num_threads; ++thread_idx)
         {
@@ -1160,7 +1174,7 @@ void StorageKeeperMap::mutate(const MutationCommands & commands, ContextPtr loca
 
     bool strict = local_context->getSettingsRef().keeper_map_strict_mode;
 
-    assert(commands.size() == 1);
+    chassert(commands.size() == 1);
 
     auto metadata_snapshot = getInMemoryMetadataPtr();
     auto storage = getStorageID();
@@ -1236,7 +1250,7 @@ void StorageKeeperMap::mutate(const MutationCommands & commands, ContextPtr loca
         return;
     }
 
-    assert(commands.front().type == MutationCommand::Type::UPDATE);
+    chassert(commands.front().type == MutationCommand::Type::UPDATE);
     if (commands.front().column_to_update_expression.contains(primary_key))
         throw Exception(ErrorCodes::BAD_ARGUMENTS, "Primary key cannot be updated (cannot update column {})", primary_key);
 

From 2530c5eb41a759baded5380a4e697c2e884c0abd Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Tue, 30 Jul 2024 12:37:35 +0200
Subject: [PATCH 1063/1488] Fix tests

---
 tests/queries/0_stateless/02952_archive_parsing.reference    | 0
 tests/queries/0_stateless/02952_archive_parsing.sql          | 1 -
 tests/queries/0_stateless/03214_parsing_archive_name_file.sh | 2 +-
 3 files changed, 1 insertion(+), 2 deletions(-)
 delete mode 100644 tests/queries/0_stateless/02952_archive_parsing.reference
 delete mode 100644 tests/queries/0_stateless/02952_archive_parsing.sql

diff --git a/tests/queries/0_stateless/02952_archive_parsing.reference b/tests/queries/0_stateless/02952_archive_parsing.reference
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/tests/queries/0_stateless/02952_archive_parsing.sql b/tests/queries/0_stateless/02952_archive_parsing.sql
deleted file mode 100644
index 49b0223e6ec..00000000000
--- a/tests/queries/0_stateless/02952_archive_parsing.sql
+++ /dev/null
@@ -1 +0,0 @@
-SELECT * FROM file('::a'); -- { serverError BAD_ARGUMENTS }
diff --git a/tests/queries/0_stateless/03214_parsing_archive_name_file.sh b/tests/queries/0_stateless/03214_parsing_archive_name_file.sh
index 2f77627f6be..b54cbb10aa6 100755
--- a/tests/queries/0_stateless/03214_parsing_archive_name_file.sh
+++ b/tests/queries/0_stateless/03214_parsing_archive_name_file.sh
@@ -11,7 +11,7 @@ function try_to_read_file()
     settings=$3
 
     echo $file_argument $settings
-    $CLICKHOUSE_LOCAL -q "SELECT * FROM file('$file_argument') $settings" 2>&1 | rg -c "Cannot stat file.*$file_to_read"
+    $CLICKHOUSE_LOCAL -q "SELECT * FROM file('$file_argument') $settings" 2>&1 | grep -c "Cannot stat file.*$file_to_read"
 }
 
 # if archive extension is not detected for part before '::', path is taken as is

From 27a15bc5ace68acb0ba62791ca6e3d3f17ae569c Mon Sep 17 00:00:00 2001
From: avogar <avogar@clickhouse.com>
Date: Tue, 30 Jul 2024 10:39:35 +0000
Subject: [PATCH 1064/1488] Remove sh files

---
 .../03036_dynamic_read_subcolumns_1.sh        | 19 -------------------
 .../03036_dynamic_read_subcolumns_2.sh        | 19 -------------------
 .../03036_dynamic_read_subcolumns_3.sh        | 19 -------------------
 3 files changed, 57 deletions(-)
 delete mode 100755 tests/queries/0_stateless/03036_dynamic_read_subcolumns_1.sh
 delete mode 100755 tests/queries/0_stateless/03036_dynamic_read_subcolumns_2.sh
 delete mode 100755 tests/queries/0_stateless/03036_dynamic_read_subcolumns_3.sh

diff --git a/tests/queries/0_stateless/03036_dynamic_read_subcolumns_1.sh b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_1.sh
deleted file mode 100755
index ed548ae74e9..00000000000
--- a/tests/queries/0_stateless/03036_dynamic_read_subcolumns_1.sh
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/usr/bin/env bash
-# Tags: long
-
-CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# shellcheck source=../shell_config.sh
-. "$CUR_DIR"/../shell_config.sh
-
-
-# shellcheck source=./03036_dynamic_read_subcolumns.lib
-. "$CUR_DIR"/03036_dynamic_read_subcolumns.lib
-
-CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --use_variant_as_common_type=1 --allow_experimental_dynamic_type=1"
-
-$CH_CLIENT -q "drop table if exists test;"
-
-echo "Memory"
-$CH_CLIENT -q "create table test (id UInt64, d Dynamic) engine=Memory"
-test
-$CH_CLIENT -q "drop table test;"
diff --git a/tests/queries/0_stateless/03036_dynamic_read_subcolumns_2.sh b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_2.sh
deleted file mode 100755
index 95dafcf5832..00000000000
--- a/tests/queries/0_stateless/03036_dynamic_read_subcolumns_2.sh
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/usr/bin/env bash
-# Tags: long
-
-CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# shellcheck source=../shell_config.sh
-. "$CUR_DIR"/../shell_config.sh
-
-
-# shellcheck source=./03036_dynamic_read_subcolumns.lib
-. "$CUR_DIR"/03036_dynamic_read_subcolumns.lib
-
-CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --use_variant_as_common_type=1 --allow_experimental_dynamic_type=1"
-
-$CH_CLIENT -q "drop table if exists test;"
-
-echo "MergeTree compact"
-$CH_CLIENT -q "create table test (id UInt64, d Dynamic) engine=MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000;"
-test
-$CH_CLIENT -q "drop table test;"
diff --git a/tests/queries/0_stateless/03036_dynamic_read_subcolumns_3.sh b/tests/queries/0_stateless/03036_dynamic_read_subcolumns_3.sh
deleted file mode 100755
index a3c2d93e568..00000000000
--- a/tests/queries/0_stateless/03036_dynamic_read_subcolumns_3.sh
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/usr/bin/env bash
-# Tags: long
-
-CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# shellcheck source=../shell_config.sh
-. "$CUR_DIR"/../shell_config.sh
-
-
-# shellcheck source=./03036_dynamic_read_subcolumns.lib
-. "$CUR_DIR"/03036_dynamic_read_subcolumns.lib
-
-CH_CLIENT="$CLICKHOUSE_CLIENT --allow_experimental_variant_type=1 --use_variant_as_common_type=1 --allow_experimental_dynamic_type=1"
-
-$CH_CLIENT -q "drop table if exists test;"
-
-echo "MergeTree wide"
-$CH_CLIENT -q "create table test (id UInt64, d Dynamic) engine=MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1;"
-test
-$CH_CLIENT -q "drop table test;"

From 72d6467fd2c34a82e1ef8ac73a451240843279a6 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Tue, 30 Jul 2024 10:43:21 +0000
Subject: [PATCH 1065/1488] Bump Azure to 1.13

---
 contrib/azure | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contrib/azure b/contrib/azure
index ea3e19a7be0..67272b7ee0a 160000
--- a/contrib/azure
+++ b/contrib/azure
@@ -1 +1 @@
-Subproject commit ea3e19a7be08519134c643177d56c7484dfec884
+Subproject commit 67272b7ee0adff6b69921b26eb071ba1a353062c

From d4537d91875d1ecec832af94fde15073c45a63d7 Mon Sep 17 00:00:00 2001
From: Nikita Fomichev <nikita.fomichev@clickhouse.com>
Date: Tue, 30 Jul 2024 12:07:47 +0200
Subject: [PATCH 1066/1488] Add `groupConcat` to fuzzer

https://github.com/ClickHouse/ClickHouse/pull/65384
---
 tests/fuzz/dictionaries/functions.dict | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/fuzz/dictionaries/functions.dict b/tests/fuzz/dictionaries/functions.dict
index ec7f8017fb2..6f2a88c22fa 100644
--- a/tests/fuzz/dictionaries/functions.dict
+++ b/tests/fuzz/dictionaries/functions.dict
@@ -1588,6 +1588,7 @@
 "groupBitmapXorResample"
 "groupBitmapXorSimpleState"
 "groupBitmapXorState"
+"groupConcat"
 "groupUniqArray"
 "groupUniqArrayArgMax"
 "groupUniqArrayArgMin"

From d4d3d590e38436da44b13dbf11a92cc6d00863e7 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Mon, 29 Jul 2024 20:06:55 +0000
Subject: [PATCH 1067/1488] Remove obsolete --multiquery parameter from tests

---
 .../integration/test_distributed_format/test.py  |  8 ++++----
 ...80_client_break_at_exception_in_batch_mode.sh |  2 +-
 .../00463_long_sessions_in_http_interface.sh     | 10 +++++-----
 .../0_stateless/00474_readonly_settings.sh       |  4 ++--
 .../0_stateless/00612_pk_in_tuple_perf.sh        |  4 ++--
 .../0_stateless/00630_arbitrary_csv_delimiter.sh |  6 +++---
 .../00650_csv_with_specified_quote_rule.sh       |  4 ++--
 ...00651_default_database_on_client_reconnect.sh |  2 +-
 .../00652_mutations_default_database.sh          |  2 +-
 ...cated_mutations_default_database_zookeeper.sh |  2 +-
 .../00699_materialized_view_mutations.sh         |  8 ++++----
 .../00704_drop_truncate_memory_table.sh          |  6 +++---
 .../0_stateless/00705_drop_create_merge_tree.sh  |  4 ++--
 .../0_stateless/00763_lock_buffer_long.sh        |  4 ++--
 ...3_long_lock_buffer_alter_destination_table.sh |  4 ++--
 .../00825_protobuf_format_array_3dim.sh          |  2 +-
 .../00825_protobuf_format_array_of_arrays.sh     |  2 +-
 .../00825_protobuf_format_enum_mapping.sh        |  2 +-
 .../0_stateless/00825_protobuf_format_map.sh     |  2 +-
 .../00825_protobuf_format_nested_in_nested.sh    |  2 +-
 .../00825_protobuf_format_nested_optional.sh     |  2 +-
 .../00825_protobuf_format_no_length_delimiter.sh |  6 +++---
 .../0_stateless/00825_protobuf_format_persons.sh |  4 ++--
 ...5_protobuf_format_skipped_column_in_nested.sh |  2 +-
 .../00825_protobuf_format_splitted_nested.sh     |  2 +-
 .../0_stateless/00825_protobuf_format_squares.sh |  2 +-
 .../00825_protobuf_format_table_default.sh       |  2 +-
 .../0_stateless/00900_long_parquet_load.sh       |  2 +-
 .../0_stateless/00900_orc_arrow_parquet_maps.sh  |  2 +-
 .../00937_format_schema_rows_template.sh         |  4 ++--
 .../0_stateless/00956_sensitive_data_masking.sh  | 12 ++++++------
 .../01019_alter_materialized_view_atomic.sh      |  2 +-
 .../01019_alter_materialized_view_consistent.sh  |  2 +-
 .../0_stateless/01035_lc_empty_part_bug.sh       |  4 ++--
 ...ystem_reload_dictionary_reloads_completely.sh |  2 +-
 .../01052_window_view_proc_tumble_to_now.sh      |  2 +-
 .../01053_window_view_proc_hop_to_now.sh         |  2 +-
 .../01054_window_view_proc_tumble_to.sh          |  2 +-
 .../0_stateless/01055_window_view_proc_hop_to.sh |  2 +-
 ...057_window_view_event_tumble_to_strict_asc.sh |  2 +-
 .../01058_window_view_event_hop_to_strict_asc.sh |  2 +-
 .../01060_window_view_event_tumble_to_asc.sh     |  2 +-
 .../01061_window_view_event_hop_to_asc.sh        |  2 +-
 .../01063_window_view_event_tumble_to_bounded.sh |  2 +-
 .../01064_window_view_event_hop_to_bounded.sh    |  2 +-
 ...w_view_event_tumble_to_strict_asc_lateness.sh |  2 +-
 ...7_window_view_event_tumble_to_asc_lateness.sh |  2 +-
 ...ndow_view_event_tumble_to_bounded_lateness.sh |  2 +-
 .../01071_window_view_event_tumble_asc_join.sh   |  2 +-
 ...01072_window_view_multiple_columns_groupby.sh |  2 +-
 ...3_window_view_event_tumble_to_asc_populate.sh |  4 ++--
 ...window_view_event_tumble_asc_join_populate.sh |  2 +-
 ...75_window_view_proc_tumble_to_now_populate.sh |  2 +-
 .../01076_window_view_alter_query_to.sh          |  4 ++--
 ...7_window_view_alter_query_to_modify_source.sh |  4 ++--
 ...1079_window_view_inner_table_memory_tumble.sh |  2 +-
 .../01080_window_view_inner_table_memory_hop.sh  |  2 +-
 .../01081_window_view_target_table_engine.sh     |  2 +-
 .../0_stateless/01083_window_view_select.sh      |  2 +-
 .../01084_window_view_with_table_identifier.sh   |  2 +-
 .../0_stateless/01086_window_view_cleanup.sh     |  2 +-
 .../0_stateless/01087_window_view_alter_query.sh |  4 ++--
 .../01088_window_view_default_column.sh          |  2 +-
 .../0_stateless/01133_begin_commit_race.sh       |  6 +++---
 .../01169_alter_partition_isolation_stress.sh    |  4 ++--
 ...01169_old_alter_partition_isolation_stress.sh | 10 +++++-----
 .../01171_mv_select_insert_isolation_long.sh     | 14 +++++++-------
 .../0_stateless/01174_select_insert_isolation.sh |  8 ++++----
 .../0_stateless/01198_client_quota_key.sh        |  2 +-
 .../0_stateless/01285_engine_join_donmikel.sh    |  4 ++--
 .../0_stateless/01293_optimize_final_force.sh    |  2 +-
 .../queries/0_stateless/01304_direct_io_long.sh  |  4 ++--
 ...orage_file_tsv_csv_with_names_write_prefix.sh |  4 ++--
 .../0_stateless/01443_merge_truncate_long.sh     |  2 +-
 .../01527_clickhouse_local_optimize.sh           |  2 +-
 .../01543_avro_deserialization_with_lc.sh        |  2 +-
 .../0_stateless/01544_file_engine_settings.sh    |  4 ++--
 .../0_stateless/01600_detach_permanently.sh      |  2 +-
 .../01600_parts_states_metrics_long.sh           |  2 +-
 tests/queries/0_stateless/01606_git_import.sh    |  4 ++--
 .../0_stateless/01607_arrays_as_nested_csv.sh    |  4 ++--
 .../0_stateless/01632_tinylog_read_write.sh      |  4 ++--
 .../01658_read_file_to_stringcolumn.sh           |  4 ++--
 .../01666_merge_tree_max_query_limit.sh          |  2 +-
 .../0_stateless/01747_system_session_log_long.sh |  6 +++---
 .../01801_nullable_low_cardinality_tsv.sh        |  2 +-
 .../01834_alias_columns_laziness_filimonov.sh    |  2 +-
 .../01923_network_receive_time_metric_insert.sh  |  4 ++--
 .../01939_network_receive_bytes_metrics.sh       |  4 ++--
 .../01946_test_wrong_host_name_access.sh         |  2 +-
 ..._with_escape_sequence_at_the_end_of_buffer.sh |  2 +-
 tests/queries/0_stateless/02009_from_infile.sh   |  2 +-
 .../0_stateless/02024_compression_in_query.sh    |  6 +++---
 .../02048_parallel_reading_from_infile.sh        |  6 +++---
 ...02104_clickhouse_local_columns_description.sh |  2 +-
 ...insert_deduplication_token_multiple_blocks.sh | 10 +++++-----
 ...eduplication_token_multiple_blocks_replica.sh | 10 +++++-----
 .../queries/0_stateless/02125_many_mutations.sh  |  6 +++---
 .../0_stateless/02125_many_mutations_2.sh        |  6 +++---
 .../queries/0_stateless/02135_local_create_db.sh |  2 +-
 .../0_stateless/02151_client_option_echo.sh      |  4 ++--
 .../0_stateless/02151_hash_table_sizes_stats.sh  |  2 +-
 .../02151_hash_table_sizes_stats_distributed.sh  |  2 +-
 .../02158_explain_ast_alter_commands.sh          |  2 +-
 .../02206_clickhouse_local_use_database.sh       |  2 +-
 .../02226_filesystem_cache_profile_events.sh     | 14 +++++++-------
 .../02227_test_create_empty_sqlite_db.sh         |  4 ++--
 .../0_stateless/02235_remote_fs_cache_stress.sh  |  6 +++---
 .../02240_protobuflist_format_persons.sh         |  4 ++--
 .../02246_clickhouse_local_drop_database.sh      |  4 ++--
 .../0_stateless/02286_drop_filesystem_cache.sh   |  2 +-
 .../02337_drop_filesystem_cache_access.sh        |  8 ++++----
 .../0_stateless/02364_window_view_segfault.sh    |  2 +-
 .../0_stateless/02373_datetime64_monotonicity.sh |  2 +-
 .../0_stateless/02416_rename_database_rbac.sh    |  4 ++--
 .../02435_rollback_cancelled_queries.sh          |  2 +-
 ...60_projections_and_aggregate_null_if_empty.sh |  2 +-
 .../02494_query_cache_user_isolation.sh          |  8 ++++----
 ...503_cache_on_write_with_small_segment_size.sh |  4 ++--
 .../queries/0_stateless/02521_merge_over_gap.sh  |  2 +-
 .../02530_dictionaries_update_field.sh           |  2 +-
 .../0_stateless/02702_allow_skip_errors_enum.sh  |  2 +-
 .../0_stateless/02704_keeper_map_zk_nodes.sh     |  2 +-
 ...ckhouse_local_implicit_file_table_function.sh |  4 ++--
 .../02712_bool_better_exception_message.sh       |  8 ++++----
 .../0_stateless/02722_database_filesystem.sh     |  8 ++++----
 tests/queries/0_stateless/02724_database_s3.sh   | 14 +++++++-------
 tests/queries/0_stateless/02725_database_hdfs.sh | 10 +++++-----
 .../0_stateless/02725_local_query_parameters.sh  |  2 +-
 .../02751_multiquery_with_argument.reference     |  2 --
 .../02751_multiquery_with_argument.sh            | 16 ++++++----------
 .../02815_no_throw_in_simple_queries.sh          |  2 +-
 .../02843_insertion_table_schema_infer.sh        |  2 +-
 .../02864_restore_table_with_broken_part.sh      |  2 +-
 ...ultiple_batches_array_inconsistent_offsets.sh |  2 +-
 .../0_stateless/02875_merge_engine_set_index.sh  |  2 +-
 .../02877_optimize_read_in_order_from_view.sh    |  2 +-
 ...02884_create_view_with_sql_security_option.sh | 16 ++++++++--------
 ...885_async_insert_access_check_for_defaults.sh |  4 ++--
 ...900_clickhouse_local_drop_current_database.sh |  2 +-
 .../02956_clickhouse_local_system_parts.sh       |  2 +-
 .../02973_backup_of_in_memory_compressed.sh      | 10 +++++-----
 .../02973_parse_crlf_with_tsv_files.sh           |  2 +-
 .../0_stateless/02995_forget_partition.sh        |  4 ++--
 tests/queries/0_stateless/02995_index_1.sh       |  6 +++---
 tests/queries/0_stateless/02995_index_10.sh      |  6 +++---
 tests/queries/0_stateless/02995_index_2.sh       |  6 +++---
 tests/queries/0_stateless/02995_index_3.sh       |  6 +++---
 tests/queries/0_stateless/02995_index_4.sh       |  6 +++---
 tests/queries/0_stateless/02995_index_5.sh       |  6 +++---
 tests/queries/0_stateless/02995_index_6.sh       |  6 +++---
 tests/queries/0_stateless/02995_index_7.sh       |  6 +++---
 tests/queries/0_stateless/02995_index_8.sh       |  6 +++---
 tests/queries/0_stateless/02995_index_9.sh       |  6 +++---
 .../0_stateless/02998_native_parquet_reader.sh   |  2 +-
 .../03001_backup_matview_after_modify_query.sh   |  2 +-
 .../03001_matview_columns_after_modify_query.sh  |  2 +-
 .../03006_correct_revoke_for_partial_rights.sh   |  2 +-
 .../03147_system_columns_access_checks.sh        |  4 ++--
 .../0_stateless/03201_local_named_collections.sh |  6 +++---
 .../0_stateless/03212_thousand_exceptions.sh     |  2 +-
 161 files changed, 316 insertions(+), 322 deletions(-)

diff --git a/tests/integration/test_distributed_format/test.py b/tests/integration/test_distributed_format/test.py
index 91afb8f7b34..5611f465e8b 100644
--- a/tests/integration/test_distributed_format/test.py
+++ b/tests/integration/test_distributed_format/test.py
@@ -55,7 +55,7 @@ def test_single_file(started_cluster, cluster):
     path = get_dist_path(cluster, "distr_1", 1)
     query = f"select * from file('{path}/1.bin', 'Distributed')"
     out = node.exec_in_container(
-        ["/usr/bin/clickhouse", "local", "--multiquery", "--stacktrace", "-q", query]
+        ["/usr/bin/clickhouse", "local", "--stacktrace", "-q", query]
     )
 
     assert out == "1\ta\n2\tbb\n3\tccc\n"
@@ -65,7 +65,7 @@ def test_single_file(started_cluster, cluster):
     select * from t;
     """
     out = node.exec_in_container(
-        ["/usr/bin/clickhouse", "local", "--multiquery", "--stacktrace", "-q", query]
+        ["/usr/bin/clickhouse", "local", "--stacktrace", "-q", query]
     )
 
     assert out == "1\ta\n2\tbb\n3\tccc\n"
@@ -106,7 +106,7 @@ def test_two_files(started_cluster, cluster):
     select * from t order by x;
     """
     out = node.exec_in_container(
-        ["/usr/bin/clickhouse", "local", "--multiquery", "--stacktrace", "-q", query]
+        ["/usr/bin/clickhouse", "local", "--stacktrace", "-q", query]
     )
 
     assert out == "0\t_\n1\ta\n2\tbb\n3\tccc\n"
@@ -141,7 +141,7 @@ def test_single_file_old(started_cluster, cluster):
     select * from t;
     """
     out = node.exec_in_container(
-        ["/usr/bin/clickhouse", "local", "--multiquery", "--stacktrace", "-q", query]
+        ["/usr/bin/clickhouse", "local", "--stacktrace", "-q", query]
     )
 
     assert out == "1\ta\n2\tbb\n3\tccc\n"
diff --git a/tests/queries/0_stateless/00380_client_break_at_exception_in_batch_mode.sh b/tests/queries/0_stateless/00380_client_break_at_exception_in_batch_mode.sh
index 62f891db33c..0aab52d15c2 100755
--- a/tests/queries/0_stateless/00380_client_break_at_exception_in_batch_mode.sh
+++ b/tests/queries/0_stateless/00380_client_break_at_exception_in_batch_mode.sh
@@ -5,4 +5,4 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
-$CLICKHOUSE_CLIENT --multiquery --query="SELECT 1; SELECT xyz; SELECT 2;" 2> /dev/null || true;
+$CLICKHOUSE_CLIENT --query="SELECT 1; SELECT xyz; SELECT 2;" 2> /dev/null || true;
diff --git a/tests/queries/0_stateless/00463_long_sessions_in_http_interface.sh b/tests/queries/0_stateless/00463_long_sessions_in_http_interface.sh
index d41d6409315..6ee1649c9ed 100755
--- a/tests/queries/0_stateless/00463_long_sessions_in_http_interface.sh
+++ b/tests/queries/0_stateless/00463_long_sessions_in_http_interface.sh
@@ -25,7 +25,7 @@ ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&session_id=${CLICKHOUSE_DATABASE}_4&se
 ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&session_id=${CLICKHOUSE_DATABASE}_5&session_timeout=60" --data-binary "SELECT 1"
 
 echo "Sessions are local per user:"
-${CLICKHOUSE_CLIENT} --multiquery --query "DROP USER IF EXISTS test_00463; CREATE USER test_00463; GRANT ALL ON *.* TO test_00463;"
+${CLICKHOUSE_CLIENT} --query "DROP USER IF EXISTS test_00463; CREATE USER test_00463; GRANT ALL ON *.* TO test_00463;"
 
 ${CLICKHOUSE_CURL} -sS -X POST "${CLICKHOUSE_URL}&session_id=${CLICKHOUSE_DATABASE}_6&session_timeout=600" --data-binary "CREATE TEMPORARY TABLE t (s String)"
 ${CLICKHOUSE_CURL} -sS -X POST "${CLICKHOUSE_URL}&session_id=${CLICKHOUSE_DATABASE}_6" --data-binary "INSERT INTO t VALUES ('Hello')"
@@ -37,7 +37,7 @@ ${CLICKHOUSE_CURL} -sS -X POST "${CLICKHOUSE_URL}&user=test_00463&session_id=${C
 ${CLICKHOUSE_CURL} -sS -X POST "${CLICKHOUSE_URL}&session_id=${CLICKHOUSE_DATABASE}_6" --data-binary "SELECT * FROM t"
 ${CLICKHOUSE_CURL} -sS -X POST "${CLICKHOUSE_URL}&user=test_00463&session_id=${CLICKHOUSE_DATABASE}_6" --data-binary "SELECT * FROM t"
 
-${CLICKHOUSE_CLIENT} --multiquery --query "DROP USER test_00463";
+${CLICKHOUSE_CLIENT} --query "DROP USER test_00463";
 
 echo "And cannot be accessed for a non-existent user:"
 ${CLICKHOUSE_CURL} -sS -X POST "${CLICKHOUSE_URL}&user=test_00463&session_id=${CLICKHOUSE_DATABASE}_6" --data-binary "SELECT * FROM t" | grep -c -F 'Exception'
@@ -59,7 +59,7 @@ done
 
 echo "A session successfully expire after a timeout and the session's temporary table shadows the permanent table:"
 # An infinite loop is required to make the test reliable. We will check that the timeout corresponds to the observed time at least once
-${CLICKHOUSE_CLIENT} --multiquery --query "DROP TABLE IF EXISTS t; CREATE TABLE t (s String) ENGINE = Memory; INSERT INTO t VALUES ('World');"
+${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS t; CREATE TABLE t (s String) ENGINE = Memory; INSERT INTO t VALUES ('World');"
 while true
 do
     (
@@ -70,7 +70,7 @@ do
         ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&session_id=${CLICKHOUSE_DATABASE}_8" --data-binary "SELECT * FROM t"
     ) | tr -d '\n' | grep -F 'HelloWorld' && break || sleep 1
 done
-${CLICKHOUSE_CLIENT} --multiquery --query "DROP TABLE t"
+${CLICKHOUSE_CLIENT} --query "DROP TABLE t"
 
 echo "A session cannot be used by concurrent connections:"
 
@@ -83,5 +83,5 @@ do
 done
 
 ${CLICKHOUSE_CURL} -sS -X POST "${CLICKHOUSE_URL}&session_id=${CLICKHOUSE_DATABASE}_9" --data-binary "SELECT 1" | grep -c -F 'SESSION_IS_LOCKED'
-${CLICKHOUSE_CLIENT} --multiquery --query "KILL QUERY WHERE query_id = '${CLICKHOUSE_DATABASE}_9' SYNC FORMAT Null";
+${CLICKHOUSE_CLIENT} --query "KILL QUERY WHERE query_id = '${CLICKHOUSE_DATABASE}_9' SYNC FORMAT Null";
 wait
diff --git a/tests/queries/0_stateless/00474_readonly_settings.sh b/tests/queries/0_stateless/00474_readonly_settings.sh
index 3a857d81a74..ed3558c6d7a 100755
--- a/tests/queries/0_stateless/00474_readonly_settings.sh
+++ b/tests/queries/0_stateless/00474_readonly_settings.sh
@@ -8,8 +8,8 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 $CLICKHOUSE_CLIENT --query="select toUInt64(pow(2, 62)) as value format JSON" --output_format_json_quote_64bit_integers=0 | grep value
 $CLICKHOUSE_CLIENT --query="select toUInt64(pow(2, 62)) as value format JSON" --output_format_json_quote_64bit_integers=1 | grep value
 
-$CLICKHOUSE_CLIENT --readonly=1 --multiquery --query="set output_format_json_quote_64bit_integers=1 ; select toUInt64(pow(2, 63)) as value format JSON" --server_logs_file=/dev/null 2>&1 | grep -o -q 'value\|Cannot modify .* setting in readonly mode' && echo "OK" || echo "FAIL"
-$CLICKHOUSE_CLIENT --readonly=1 --multiquery --query="set output_format_json_quote_64bit_integers=0 ; select toUInt64(pow(2, 63)) as value format JSON" --server_logs_file=/dev/null 2>&1 | grep -o -q 'value\|Cannot modify .* setting in readonly mode' && echo "OK" || echo "FAIL"
+$CLICKHOUSE_CLIENT --readonly=1 --query="set output_format_json_quote_64bit_integers=1 ; select toUInt64(pow(2, 63)) as value format JSON" --server_logs_file=/dev/null 2>&1 | grep -o -q 'value\|Cannot modify .* setting in readonly mode' && echo "OK" || echo "FAIL"
+$CLICKHOUSE_CLIENT --readonly=1 --query="set output_format_json_quote_64bit_integers=0 ; select toUInt64(pow(2, 63)) as value format JSON" --server_logs_file=/dev/null 2>&1 | grep -o -q 'value\|Cannot modify .* setting in readonly mode' && echo "OK" || echo "FAIL"
 
 ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=SELECT+toUInt64(pow(2,+63))+as+value+format+JSON&output_format_json_quote_64bit_integers=1" | grep value
 ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=SELECT+toUInt64(pow(2,+63))+as+value+format+JSON&output_format_json_quote_64bit_integers=0" | grep value
diff --git a/tests/queries/0_stateless/00612_pk_in_tuple_perf.sh b/tests/queries/0_stateless/00612_pk_in_tuple_perf.sh
index c8297635c43..7b2973669de 100755
--- a/tests/queries/0_stateless/00612_pk_in_tuple_perf.sh
+++ b/tests/queries/0_stateless/00612_pk_in_tuple_perf.sh
@@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
 
 
-$CLICKHOUSE_CLIENT --multiquery <<EOF
+$CLICKHOUSE_CLIENT <<EOF
 DROP TABLE IF EXISTS pk_in_tuple_perf;
 CREATE TABLE pk_in_tuple_perf
 (
@@ -27,7 +27,7 @@ $CLICKHOUSE_CLIENT --query "$query FORMAT JSON" | grep "rows_read"
 
 ## Test with non-const args in tuple
 
-$CLICKHOUSE_CLIENT --multiquery <<EOF
+$CLICKHOUSE_CLIENT <<EOF
 DROP TABLE IF EXISTS pk_in_tuple_perf_non_const;
 CREATE TABLE pk_in_tuple_perf_non_const
 (
diff --git a/tests/queries/0_stateless/00630_arbitrary_csv_delimiter.sh b/tests/queries/0_stateless/00630_arbitrary_csv_delimiter.sh
index 978b7bdb671..921cab8d865 100755
--- a/tests/queries/0_stateless/00630_arbitrary_csv_delimiter.sh
+++ b/tests/queries/0_stateless/00630_arbitrary_csv_delimiter.sh
@@ -22,7 +22,7 @@ echo '"Hello, world"; 123; "2016-01-01"
 "Hello, ""world"""; "456"; 2016-01-02;
 Hello "world"; 789 ;2016-01-03
 "Hello
- world"; 100; 2016-01-04;' | $CLICKHOUSE_CLIENT --multiquery --query="SET format_csv_delimiter=';'; INSERT INTO csv FORMAT CSV";
+ world"; 100; 2016-01-04;' | $CLICKHOUSE_CLIENT --query="SET format_csv_delimiter=';'; INSERT INTO csv FORMAT CSV";
 
 $CLICKHOUSE_CLIENT --query="SELECT * FROM csv ORDER BY d";
 $CLICKHOUSE_CLIENT --format_csv_delimiter=";" --query="SELECT * FROM csv ORDER BY d FORMAT CSV";
@@ -33,7 +33,7 @@ $CLICKHOUSE_CLIENT --query="CREATE TABLE csv (s1 String, s2 String) ENGINE = Mem
 
 echo 'abc,def;hello;
 hello; world;
-"hello ""world""";abc,def;' | $CLICKHOUSE_CLIENT --multiquery --query="SET format_csv_delimiter=';'; INSERT INTO csv FORMAT CSV";
+"hello ""world""";abc,def;' | $CLICKHOUSE_CLIENT --query="SET format_csv_delimiter=';'; INSERT INTO csv FORMAT CSV";
 
 
 $CLICKHOUSE_CLIENT --query="SELECT * FROM csv";
@@ -44,7 +44,7 @@ $CLICKHOUSE_CLIENT --query="CREATE TABLE csv (s1 String, s2 String) ENGINE = Mem
 echo '"s1";"s2"
 abc,def;hello;
 hello; world;
-"hello ""world""";abc,def;' | $CLICKHOUSE_CLIENT --multiquery --query="SET format_csv_delimiter=';'; INSERT INTO csv FORMAT CSVWithNames";
+"hello ""world""";abc,def;' | $CLICKHOUSE_CLIENT --query="SET format_csv_delimiter=';'; INSERT INTO csv FORMAT CSVWithNames";
 
 $CLICKHOUSE_CLIENT --format_csv_delimiter=";" --query="SELECT * FROM csv FORMAT CSV";
 $CLICKHOUSE_CLIENT --format_csv_delimiter="," --query="SELECT * FROM csv FORMAT CSV";
diff --git a/tests/queries/0_stateless/00650_csv_with_specified_quote_rule.sh b/tests/queries/0_stateless/00650_csv_with_specified_quote_rule.sh
index 3971510f7f6..2f76b3e63ac 100755
--- a/tests/queries/0_stateless/00650_csv_with_specified_quote_rule.sh
+++ b/tests/queries/0_stateless/00650_csv_with_specified_quote_rule.sh
@@ -18,7 +18,7 @@ $CLICKHOUSE_CLIENT --query="DROP TABLE csv";
 $CLICKHOUSE_CLIENT --query="CREATE TABLE csv (s String, n UInt64, d Date) ENGINE = Memory";
 
 echo "'single quote' not end, 123, 2016-01-01
-'em good, 456, 2016-01-02" | $CLICKHOUSE_CLIENT --multiquery --query="SET format_csv_allow_single_quotes=0; INSERT INTO csv FORMAT CSV";
+'em good, 456, 2016-01-02" | $CLICKHOUSE_CLIENT --query="SET format_csv_allow_single_quotes=0; INSERT INTO csv FORMAT CSV";
 
 $CLICKHOUSE_CLIENT --query="SELECT * FROM csv ORDER BY d";
 
@@ -38,7 +38,7 @@ $CLICKHOUSE_CLIENT --query="DROP TABLE csv";
 $CLICKHOUSE_CLIENT --query="CREATE TABLE csv (s String, n UInt64, d Date) ENGINE = Memory";
 
 echo '"double quote" not end, 123, 2016-01-01
-"em good, 456, 2016-01-02' | $CLICKHOUSE_CLIENT --multiquery --query="SET format_csv_allow_double_quotes=0; INSERT INTO csv FORMAT CSV";
+"em good, 456, 2016-01-02' | $CLICKHOUSE_CLIENT --query="SET format_csv_allow_double_quotes=0; INSERT INTO csv FORMAT CSV";
 
 $CLICKHOUSE_CLIENT --query="SELECT * FROM csv ORDER BY d";
 
diff --git a/tests/queries/0_stateless/00651_default_database_on_client_reconnect.sh b/tests/queries/0_stateless/00651_default_database_on_client_reconnect.sh
index 994114c9c50..df0afeb707c 100755
--- a/tests/queries/0_stateless/00651_default_database_on_client_reconnect.sh
+++ b/tests/queries/0_stateless/00651_default_database_on_client_reconnect.sh
@@ -5,4 +5,4 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
-${CLICKHOUSE_CLIENT} --ignore-error --multiquery --query "DROP TABLE IF EXISTS tab_00651; CREATE TABLE tab_00651 (val UInt64) engine = Memory; SHOW CREATE TABLE tab_00651 format abcd; DESC tab_00651; DROP TABLE tab_00651;" 2>/dev/null ||:
+${CLICKHOUSE_CLIENT} --ignore-error --query "DROP TABLE IF EXISTS tab_00651; CREATE TABLE tab_00651 (val UInt64) engine = Memory; SHOW CREATE TABLE tab_00651 format abcd; DESC tab_00651; DROP TABLE tab_00651;" 2>/dev/null ||:
diff --git a/tests/queries/0_stateless/00652_mutations_default_database.sh b/tests/queries/0_stateless/00652_mutations_default_database.sh
index eed45540f9b..577943bc3fd 100755
--- a/tests/queries/0_stateless/00652_mutations_default_database.sh
+++ b/tests/queries/0_stateless/00652_mutations_default_database.sh
@@ -4,7 +4,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
-${CLICKHOUSE_CLIENT} --multiquery --mutations_sync=1 << EOF
+${CLICKHOUSE_CLIENT} --mutations_sync=1 << EOF
 DROP TABLE IF EXISTS mutations;
 DROP TABLE IF EXISTS for_subquery;
 
diff --git a/tests/queries/0_stateless/00652_replicated_mutations_default_database_zookeeper.sh b/tests/queries/0_stateless/00652_replicated_mutations_default_database_zookeeper.sh
index 0ac5a2f748a..d4f6d3b290c 100755
--- a/tests/queries/0_stateless/00652_replicated_mutations_default_database_zookeeper.sh
+++ b/tests/queries/0_stateless/00652_replicated_mutations_default_database_zookeeper.sh
@@ -9,7 +9,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=./mergetree_mutations.lib
 . "$CURDIR"/mergetree_mutations.lib
 
-${CLICKHOUSE_CLIENT} --allow_nondeterministic_mutations=1 --multiquery << EOF
+${CLICKHOUSE_CLIENT} --allow_nondeterministic_mutations=1 << EOF
 DROP TABLE IF EXISTS mutations_r1;
 DROP TABLE IF EXISTS for_subquery;
 
diff --git a/tests/queries/0_stateless/00699_materialized_view_mutations.sh b/tests/queries/0_stateless/00699_materialized_view_mutations.sh
index a0f7db536dc..07ca9bc0f67 100755
--- a/tests/queries/0_stateless/00699_materialized_view_mutations.sh
+++ b/tests/queries/0_stateless/00699_materialized_view_mutations.sh
@@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CURDIR"/../shell_config.sh
 
 
-${CLICKHOUSE_CLIENT} --multiquery --query="
+${CLICKHOUSE_CLIENT} --query="
 DROP TABLE IF EXISTS view_00699;
 DROP TABLE IF EXISTS null_00699;
 
@@ -20,14 +20,14 @@ SELECT count(), min(x), max(x) FROM view_00699;
 
 ALTER TABLE null_00699 DELETE WHERE x % 2 = 0;"  --mutations_sync=1
 
-${CLICKHOUSE_CLIENT} --multiquery --query="
+${CLICKHOUSE_CLIENT} --query="
 SELECT count(), min(x), max(x) FROM null_00699;
 SELECT count(), min(x), max(x) FROM view_00699;
 
 ALTER TABLE view_00699 DELETE WHERE x % 2 = 0;
 " --mutations_sync=1
 
-${CLICKHOUSE_CLIENT} --multiquery --query="
+${CLICKHOUSE_CLIENT} --query="
 SELECT count(), min(x), max(x) FROM null_00699;
 SELECT count(), min(x), max(x) FROM view_00699;
 
@@ -35,7 +35,7 @@ ALTER TABLE null_00699 DELETE WHERE x % 2 = 1;
 ALTER TABLE view_00699 DELETE WHERE x % 2 = 1;
 " --mutations_sync=1
 
-${CLICKHOUSE_CLIENT} --multiquery --query="
+${CLICKHOUSE_CLIENT} --query="
 SELECT count(), min(x), max(x) FROM null_00699;
 SELECT count(), min(x), max(x) FROM view_00699;
 
diff --git a/tests/queries/0_stateless/00704_drop_truncate_memory_table.sh b/tests/queries/0_stateless/00704_drop_truncate_memory_table.sh
index e1540d1a25e..e40da11b893 100755
--- a/tests/queries/0_stateless/00704_drop_truncate_memory_table.sh
+++ b/tests/queries/0_stateless/00704_drop_truncate_memory_table.sh
@@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
-${CLICKHOUSE_CLIENT} --multiquery --query="
+${CLICKHOUSE_CLIENT} --query="
 DROP TABLE IF EXISTS memory;
 CREATE TABLE memory (x UInt64) ENGINE = Memory;
 
@@ -21,13 +21,13 @@ INSERT INTO memory SELECT * FROM numbers(1000);"
 # But if the table will be dropped before query - just pass.
 # It's Ok, because otherwise the test will depend on the race condition in the test itself.
 
-${CLICKHOUSE_CLIENT} --multiquery --query="
+${CLICKHOUSE_CLIENT} --query="
 SET max_threads = 1;
 SELECT count() FROM memory WHERE NOT ignore(sleep(0.0001));" 2>&1 | grep -c -P '^1000$|^0$|Exception' &
 
 sleep 0.05;
 
-${CLICKHOUSE_CLIENT} --multiquery --query="
+${CLICKHOUSE_CLIENT} --query="
 TRUNCATE TABLE memory;
 DROP TABLE memory;
 "
diff --git a/tests/queries/0_stateless/00705_drop_create_merge_tree.sh b/tests/queries/0_stateless/00705_drop_create_merge_tree.sh
index ea8b9d02e49..fd002668696 100755
--- a/tests/queries/0_stateless/00705_drop_create_merge_tree.sh
+++ b/tests/queries/0_stateless/00705_drop_create_merge_tree.sh
@@ -5,8 +5,8 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
-yes 'CREATE TABLE IF NOT EXISTS table (x UInt8) ENGINE = MergeTree ORDER BY tuple();' | head -n 1000 | $CLICKHOUSE_CLIENT --multiquery &
-yes 'DROP TABLE IF EXISTS table;' | head -n 1000 | $CLICKHOUSE_CLIENT --multiquery &
+yes 'CREATE TABLE IF NOT EXISTS table (x UInt8) ENGINE = MergeTree ORDER BY tuple();' | head -n 1000 | $CLICKHOUSE_CLIENT &
+yes 'DROP TABLE IF EXISTS table;' | head -n 1000 | $CLICKHOUSE_CLIENT &
 wait
 
 ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS table"
diff --git a/tests/queries/0_stateless/00763_lock_buffer_long.sh b/tests/queries/0_stateless/00763_lock_buffer_long.sh
index 444a66767aa..92f917aa287 100755
--- a/tests/queries/0_stateless/00763_lock_buffer_long.sh
+++ b/tests/queries/0_stateless/00763_lock_buffer_long.sh
@@ -16,12 +16,12 @@ ${CLICKHOUSE_CLIENT} --query="CREATE TABLE buffer_00763_2 (s String) ENGINE = Bu
 
 function thread1()
 {
-    seq 1 500 | sed -r -e 's/.+/DROP TABLE IF EXISTS mt_00763_2; CREATE TABLE mt_00763_2 (s String) ENGINE = MergeTree ORDER BY s; INSERT INTO mt_00763_2 SELECT toString(number) FROM numbers(10);/' | ${CLICKHOUSE_CLIENT} --fsync-metadata 0 --multiquery --ignore-error ||:
+    seq 1 500 | sed -r -e 's/.+/DROP TABLE IF EXISTS mt_00763_2; CREATE TABLE mt_00763_2 (s String) ENGINE = MergeTree ORDER BY s; INSERT INTO mt_00763_2 SELECT toString(number) FROM numbers(10);/' | ${CLICKHOUSE_CLIENT} --fsync-metadata 0 --ignore-error ||:
 }
 
 function thread2()
 {
-    seq 1 500 | sed -r -e 's/.+/SELECT count() FROM buffer_00763_2;/' | ${CLICKHOUSE_CLIENT} --multiquery --server_logs_file='/dev/null' --ignore-error 2>&1 | grep -vP '^0$|^10$|^Received exception|^Code: 60|^Code: 218|^Code: 473' | grep -v '(query: '
+    seq 1 500 | sed -r -e 's/.+/SELECT count() FROM buffer_00763_2;/' | ${CLICKHOUSE_CLIENT} --server_logs_file='/dev/null' --ignore-error 2>&1 | grep -vP '^0$|^10$|^Received exception|^Code: 60|^Code: 218|^Code: 473' | grep -v '(query: '
 }
 
 thread1 &
diff --git a/tests/queries/0_stateless/00763_long_lock_buffer_alter_destination_table.sh b/tests/queries/0_stateless/00763_long_lock_buffer_alter_destination_table.sh
index 7e2384cfc52..79df667d45f 100755
--- a/tests/queries/0_stateless/00763_long_lock_buffer_alter_destination_table.sh
+++ b/tests/queries/0_stateless/00763_long_lock_buffer_alter_destination_table.sh
@@ -18,12 +18,12 @@ ${CLICKHOUSE_CLIENT} --query="INSERT INTO mt_00763_1 VALUES (1, '1'), (2, '2'),
 
 function thread1()
 {
-    seq 1 300 | sed -r -e 's/.+/ALTER TABLE mt_00763_1 MODIFY column s UInt32; ALTER TABLE mt_00763_1 MODIFY column s String;/' | ${CLICKHOUSE_CLIENT} --multiquery --ignore-error ||:
+    seq 1 300 | sed -r -e 's/.+/ALTER TABLE mt_00763_1 MODIFY column s UInt32; ALTER TABLE mt_00763_1 MODIFY column s String;/' | ${CLICKHOUSE_CLIENT} --ignore-error ||:
 }
 
 function thread2()
 {
-    seq 1 2000 | sed -r -e 's/.+/SELECT sum(length(s)) FROM buffer_00763_1;/' | ${CLICKHOUSE_CLIENT} --multiquery --ignore-error 2>&1 | grep -vP '(^3$|^Received exception from server|^Code: 473)'
+    seq 1 2000 | sed -r -e 's/.+/SELECT sum(length(s)) FROM buffer_00763_1;/' | ${CLICKHOUSE_CLIENT} --ignore-error 2>&1 | grep -vP '(^3$|^Received exception from server|^Code: 473)'
 }
 
 thread1 &
diff --git a/tests/queries/0_stateless/00825_protobuf_format_array_3dim.sh b/tests/queries/0_stateless/00825_protobuf_format_array_3dim.sh
index 3cd842a10ba..468ced802cd 100755
--- a/tests/queries/0_stateless/00825_protobuf_format_array_3dim.sh
+++ b/tests/queries/0_stateless/00825_protobuf_format_array_3dim.sh
@@ -9,7 +9,7 @@ SCHEMADIR=$CURDIR/format_schemas
 set -eo pipefail
 
 # Run the client.
-$CLICKHOUSE_CLIENT --multiquery <<EOF
+$CLICKHOUSE_CLIENT <<EOF
 DROP TABLE IF EXISTS array_3dim_protobuf_00825;
 
 CREATE TABLE array_3dim_protobuf_00825
diff --git a/tests/queries/0_stateless/00825_protobuf_format_array_of_arrays.sh b/tests/queries/0_stateless/00825_protobuf_format_array_of_arrays.sh
index 76c5a63c4f2..47f0e2a31e7 100755
--- a/tests/queries/0_stateless/00825_protobuf_format_array_of_arrays.sh
+++ b/tests/queries/0_stateless/00825_protobuf_format_array_of_arrays.sh
@@ -11,7 +11,7 @@ SCHEMADIR=$CURDIR/format_schemas
 set -eo pipefail
 
 # Run the client.
-$CLICKHOUSE_CLIENT --multiquery <<EOF
+$CLICKHOUSE_CLIENT <<EOF
 DROP TABLE IF EXISTS array_of_arrays_protobuf_00825;
 
 CREATE TABLE array_of_arrays_protobuf_00825
diff --git a/tests/queries/0_stateless/00825_protobuf_format_enum_mapping.sh b/tests/queries/0_stateless/00825_protobuf_format_enum_mapping.sh
index 1258230610d..da359f2ab5c 100755
--- a/tests/queries/0_stateless/00825_protobuf_format_enum_mapping.sh
+++ b/tests/queries/0_stateless/00825_protobuf_format_enum_mapping.sh
@@ -11,7 +11,7 @@ SCHEMADIR=$CURDIR/format_schemas
 set -eo pipefail
 
 # Run the client.
-$CLICKHOUSE_CLIENT --multiquery <<EOF
+$CLICKHOUSE_CLIENT <<EOF
 DROP TABLE IF EXISTS enum_mapping_protobuf_00825;
 
 CREATE TABLE enum_mapping_protobuf_00825
diff --git a/tests/queries/0_stateless/00825_protobuf_format_map.sh b/tests/queries/0_stateless/00825_protobuf_format_map.sh
index 82ebcc6959a..155754a2f00 100755
--- a/tests/queries/0_stateless/00825_protobuf_format_map.sh
+++ b/tests/queries/0_stateless/00825_protobuf_format_map.sh
@@ -9,7 +9,7 @@ SCHEMADIR=$CURDIR/format_schemas
 set -eo pipefail
 
 # Run the client.
-$CLICKHOUSE_CLIENT --multiquery <<EOF
+$CLICKHOUSE_CLIENT <<EOF
 DROP TABLE IF EXISTS map_protobuf_00825;
 
 CREATE TABLE map_protobuf_00825
diff --git a/tests/queries/0_stateless/00825_protobuf_format_nested_in_nested.sh b/tests/queries/0_stateless/00825_protobuf_format_nested_in_nested.sh
index b0a16c2fbba..9b4a12a3ae4 100755
--- a/tests/queries/0_stateless/00825_protobuf_format_nested_in_nested.sh
+++ b/tests/queries/0_stateless/00825_protobuf_format_nested_in_nested.sh
@@ -11,7 +11,7 @@ SCHEMADIR=$CURDIR/format_schemas
 set -eo pipefail
 
 # Run the client.
-$CLICKHOUSE_CLIENT --multiquery <<EOF
+$CLICKHOUSE_CLIENT <<EOF
 DROP TABLE IF EXISTS nested_in_nested_protobuf_00825;
 
 CREATE TABLE nested_in_nested_protobuf_00825 (x Nested (y Nested (z Int64))) ENGINE = MergeTree ORDER BY tuple();
diff --git a/tests/queries/0_stateless/00825_protobuf_format_nested_optional.sh b/tests/queries/0_stateless/00825_protobuf_format_nested_optional.sh
index cf9c47f5ea9..acec56b1790 100755
--- a/tests/queries/0_stateless/00825_protobuf_format_nested_optional.sh
+++ b/tests/queries/0_stateless/00825_protobuf_format_nested_optional.sh
@@ -11,7 +11,7 @@ SCHEMADIR=$CURDIR/format_schemas
 set -eo pipefail
 
 # Run the client.
-$CLICKHOUSE_CLIENT --multiquery <<EOF
+$CLICKHOUSE_CLIENT <<EOF
 DROP TABLE IF EXISTS nested_optional_protobuf_00825;
 
 CREATE TABLE nested_optional_protobuf_00825
diff --git a/tests/queries/0_stateless/00825_protobuf_format_no_length_delimiter.sh b/tests/queries/0_stateless/00825_protobuf_format_no_length_delimiter.sh
index 0f168c38395..ce6f6275028 100755
--- a/tests/queries/0_stateless/00825_protobuf_format_no_length_delimiter.sh
+++ b/tests/queries/0_stateless/00825_protobuf_format_no_length_delimiter.sh
@@ -9,7 +9,7 @@ SCHEMADIR=$CURDIR/format_schemas
 set -eo pipefail
 
 # Run the client.
-$CLICKHOUSE_CLIENT --multiquery <<EOF
+$CLICKHOUSE_CLIENT <<EOF
 DROP TABLE IF EXISTS no_length_delimiter_protobuf_00825;
 DROP TABLE IF EXISTS roundtrip_no_length_delimiter_protobuf_00825;
 
@@ -43,11 +43,11 @@ $CLICKHOUSE_CLIENT --query "SELECT * FROM roundtrip_no_length_delimiter_protobuf
 rm "$BINARY_FILE_PATH"
 
 # The ProtobufSingle format can't be used to write multiple rows because this format doesn't have any row delimiter.
-$CLICKHOUSE_CLIENT --multiquery > /dev/null <<EOF
+$CLICKHOUSE_CLIENT > /dev/null <<EOF
 SELECT * FROM no_length_delimiter_protobuf_00825 FORMAT ProtobufSingle SETTINGS format_schema = '$SCHEMADIR/00825_protobuf_format_no_length_delimiter:Message'; -- { clientError 546 }
 EOF
 
-$CLICKHOUSE_CLIENT --multiquery <<EOF
+$CLICKHOUSE_CLIENT <<EOF
 DROP TABLE no_length_delimiter_protobuf_00825;
 DROP TABLE roundtrip_no_length_delimiter_protobuf_00825;
 EOF
diff --git a/tests/queries/0_stateless/00825_protobuf_format_persons.sh b/tests/queries/0_stateless/00825_protobuf_format_persons.sh
index a9ac49a1e30..1f199dc4691 100755
--- a/tests/queries/0_stateless/00825_protobuf_format_persons.sh
+++ b/tests/queries/0_stateless/00825_protobuf_format_persons.sh
@@ -17,7 +17,7 @@ SCHEMADIR=$CURDIR/format_schemas
 set -eo pipefail
 
 # Run the client.
-$CLICKHOUSE_CLIENT --multiquery <<EOF
+$CLICKHOUSE_CLIENT <<EOF
 DROP TABLE IF EXISTS persons_00825;
 DROP TABLE IF EXISTS roundtrip_persons_00825;
 DROP TABLE IF EXISTS alt_persons_00825;
@@ -129,7 +129,7 @@ $CLICKHOUSE_CLIENT --query "INSERT INTO edition2023_persons_00825 SETTINGS forma
 $CLICKHOUSE_CLIENT --query "SELECT * FROM edition2023_persons_00825 ORDER BY name"
 rm "$BINARY_FILE_PATH"
 
-$CLICKHOUSE_CLIENT --multiquery <<EOF
+$CLICKHOUSE_CLIENT <<EOF
 DROP TABLE persons_00825;
 DROP TABLE roundtrip_persons_00825;
 DROP TABLE alt_persons_00825;
diff --git a/tests/queries/0_stateless/00825_protobuf_format_skipped_column_in_nested.sh b/tests/queries/0_stateless/00825_protobuf_format_skipped_column_in_nested.sh
index 1c1dde82b4a..c98205b5126 100755
--- a/tests/queries/0_stateless/00825_protobuf_format_skipped_column_in_nested.sh
+++ b/tests/queries/0_stateless/00825_protobuf_format_skipped_column_in_nested.sh
@@ -11,7 +11,7 @@ SCHEMADIR=$CURDIR/format_schemas
 set -eo pipefail
 
 # Run the client.
-$CLICKHOUSE_CLIENT --multiquery <<EOF
+$CLICKHOUSE_CLIENT <<EOF
 DROP TABLE IF EXISTS table_skipped_column_in_nested_00825;
 
 CREATE TABLE table_skipped_column_in_nested_00825 (
diff --git a/tests/queries/0_stateless/00825_protobuf_format_splitted_nested.sh b/tests/queries/0_stateless/00825_protobuf_format_splitted_nested.sh
index f4525ba609c..eafe1919606 100755
--- a/tests/queries/0_stateless/00825_protobuf_format_splitted_nested.sh
+++ b/tests/queries/0_stateless/00825_protobuf_format_splitted_nested.sh
@@ -9,7 +9,7 @@ SCHEMADIR=$CURDIR/format_schemas
 set -eo pipefail
 
 # Run the client.
-$CLICKHOUSE_CLIENT --multiquery <<EOF
+$CLICKHOUSE_CLIENT <<EOF
 DROP TABLE IF EXISTS splitted_nested_protobuf_00825;
 
 CREATE TABLE splitted_nested_protobuf_00825 (
diff --git a/tests/queries/0_stateless/00825_protobuf_format_squares.sh b/tests/queries/0_stateless/00825_protobuf_format_squares.sh
index f04cf60d78b..9c6778d519d 100755
--- a/tests/queries/0_stateless/00825_protobuf_format_squares.sh
+++ b/tests/queries/0_stateless/00825_protobuf_format_squares.sh
@@ -9,7 +9,7 @@ SCHEMADIR=$CURDIR/format_schemas
 set -eo pipefail
 
 # Run the client.
-$CLICKHOUSE_CLIENT --multiquery <<EOF
+$CLICKHOUSE_CLIENT <<EOF
 DROP TABLE IF EXISTS squares_protobuf_00825;
 
 CREATE TABLE squares_protobuf_00825 (number UInt32, square UInt64) ENGINE = MergeTree ORDER BY tuple();
diff --git a/tests/queries/0_stateless/00825_protobuf_format_table_default.sh b/tests/queries/0_stateless/00825_protobuf_format_table_default.sh
index 5701ee2cb4d..8ccb022b669 100755
--- a/tests/queries/0_stateless/00825_protobuf_format_table_default.sh
+++ b/tests/queries/0_stateless/00825_protobuf_format_table_default.sh
@@ -9,7 +9,7 @@ SCHEMADIR=$CURDIR/format_schemas
 set -eo pipefail
 
 # Run the client.
-$CLICKHOUSE_CLIENT --multiquery <<EOF
+$CLICKHOUSE_CLIENT <<EOF
 DROP TABLE IF EXISTS table_default_protobuf_00825;
 
 CREATE TABLE table_default_protobuf_00825
diff --git a/tests/queries/0_stateless/00900_long_parquet_load.sh b/tests/queries/0_stateless/00900_long_parquet_load.sh
index 97fee5e97ec..1bafb033f56 100755
--- a/tests/queries/0_stateless/00900_long_parquet_load.sh
+++ b/tests/queries/0_stateless/00900_long_parquet_load.sh
@@ -58,7 +58,7 @@ for NAME in $(find "$DATA_DIR"/*.parquet -print0 | xargs -0 -n 1 basename | LC_A
     COLUMNS=$(cat "$COLUMNS_FILE") || continue
 
     ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS parquet_load"
-    $CLICKHOUSE_CLIENT --multiquery <<EOF
+    $CLICKHOUSE_CLIENT <<EOF
 CREATE TABLE parquet_load ($COLUMNS) ENGINE = Memory;
 EOF
 
diff --git a/tests/queries/0_stateless/00900_orc_arrow_parquet_maps.sh b/tests/queries/0_stateless/00900_orc_arrow_parquet_maps.sh
index 4e258911078..db14a3b1032 100755
--- a/tests/queries/0_stateless/00900_orc_arrow_parquet_maps.sh
+++ b/tests/queries/0_stateless/00900_orc_arrow_parquet_maps.sh
@@ -6,7 +6,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CUR_DIR"/../shell_config.sh
 
 ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS maps"
-${CLICKHOUSE_CLIENT} --multiquery <<EOF
+${CLICKHOUSE_CLIENT} <<EOF
 CREATE TABLE maps (m1 Map(UInt32, UInt32), m2 Map(String, String), m3 Map(UInt32, Tuple(UInt32, UInt32)), m4 Map(UInt32, Array(UInt32)), m5 Array(Map(UInt32, UInt32)), m6 Tuple(Map(UInt32, UInt32), Map(String, String)), m7 Array(Map(UInt32, Array(Tuple(Map(UInt32, UInt32), Tuple(UInt32)))))) ENGINE=Memory();
 EOF
 
diff --git a/tests/queries/0_stateless/00937_format_schema_rows_template.sh b/tests/queries/0_stateless/00937_format_schema_rows_template.sh
index 0221527f9c9..f020e15b2d7 100755
--- a/tests/queries/0_stateless/00937_format_schema_rows_template.sh
+++ b/tests/queries/0_stateless/00937_format_schema_rows_template.sh
@@ -23,7 +23,7 @@ echo -e "\n"
 # Test that if both format_template_row_format setting and format_template_row are provided, error is thrown
 row_format_file="$CURDIR"/"${CLICKHOUSE_TEST_UNIQUE_NAME}"_template_output_format_row.tmp
 echo -ne 'Question: ${question:Quoted}, Answer: ${answer:Quoted}, Number of Likes: ${likes:Raw}, Date: ${date:Raw}' > $row_format_file
-$CLICKHOUSE_CLIENT --multiline --multiquery --query "SELECT * FROM template GROUP BY question, answer, likes, date WITH TOTALS ORDER BY date LIMIT 3 FORMAT Template SETTINGS \
+$CLICKHOUSE_CLIENT --multiline --query "SELECT * FROM template GROUP BY question, answer, likes, date WITH TOTALS ORDER BY date LIMIT 3 FORMAT Template SETTINGS \
 format_template_row = '$row_format_file', \
 format_template_row_format = 'Question: \${question:Quoted}, Answer: \${answer:Quoted}, Number of Likes: \${likes:Raw}, Date: \${date:Raw}', \
 format_template_rows_between_delimiter = ';\n'; --{clientError 474}"
@@ -38,7 +38,7 @@ format_template_rows_between_delimiter = ';\n'";
 # Test that if both format_template_result_format setting and format_template_resultset are provided, error is thrown
 resultset_output_file="$CURDIR"/"$CLICKHOUSE_TEST_UNIQUE_NAME"_template_output_format_resultset.tmp
 echo -ne '===== Resultset ===== \n \${data} \n ===============' > $resultset_output_file
-$CLICKHOUSE_CLIENT --multiline --multiquery --query "SELECT * FROM template GROUP BY question, answer, likes, date WITH TOTALS ORDER BY date LIMIT 3 FORMAT Template SETTINGS \
+$CLICKHOUSE_CLIENT --multiline --query "SELECT * FROM template GROUP BY question, answer, likes, date WITH TOTALS ORDER BY date LIMIT 3 FORMAT Template SETTINGS \
 format_template_resultset = '$resultset_output_file', \
 format_template_resultset_format = '===== Resultset ===== \n \${data} \n ===============', \
 format_template_row_format = 'Question: \${question:Quoted}, Answer: \${answer:Quoted}, Number of Likes: \${likes:Raw}, Date: \${date:Raw}', \
diff --git a/tests/queries/0_stateless/00956_sensitive_data_masking.sh b/tests/queries/0_stateless/00956_sensitive_data_masking.sh
index 926557e4ba6..bd65b937648 100755
--- a/tests/queries/0_stateless/00956_sensitive_data_masking.sh
+++ b/tests/queries/0_stateless/00956_sensitive_data_masking.sh
@@ -17,7 +17,7 @@ echo 1
 # normal execution
 $CLICKHOUSE_CLIENT \
   --query="SELECT 'find_me_TOPSECRET=TOPSECRET' FROM numbers(1) FORMAT Null" \
-  --log_queries=1 --ignore-error --multiquery >"$tmp_file" 2>&1
+  --log_queries=1 --ignore-error >"$tmp_file" 2>&1
 
 grep -F 'find_me_[hidden]' "$tmp_file" >/dev/null || echo 'fail 1a'
 grep -F 'TOPSECRET' "$tmp_file" && echo 'fail 1b'
@@ -38,7 +38,7 @@ echo 3
 # failure at before query start
 $CLICKHOUSE_CLIENT \
   --query="SELECT 1 FROM system.numbers WHERE credit_card_number='find_me_TOPSECRET=TOPSECRET' FORMAT Null" \
-  --log_queries=1 --ignore-error --multiquery |& grep -v '^(query: ' > "$tmp_file"
+  --log_queries=1 --ignore-error |& grep -v '^(query: ' > "$tmp_file"
 
 grep -F 'find_me_[hidden]' "$tmp_file" >/dev/null || echo 'fail 3a'
 grep -F 'TOPSECRET' "$tmp_file" && echo 'fail 3b'
@@ -56,7 +56,7 @@ echo 4
 # failure at the end of query
 $CLICKHOUSE_CLIENT \
   --query="SELECT 'find_me_TOPSECRET=TOPSECRET', intDiv( 100, number - 10) FROM numbers(11) FORMAT Null" \
-  --log_queries=1 --ignore-error --max_block_size=2 --multiquery |& grep -v '^(query: ' > "$tmp_file"
+  --log_queries=1 --ignore-error --max_block_size=2 |& grep -v '^(query: ' > "$tmp_file"
 
 grep -F 'find_me_[hidden]' "$tmp_file" >/dev/null || echo 'fail 4a'
 grep -F 'TOPSECRET' "$tmp_file" && echo 'fail 4b'
@@ -67,7 +67,7 @@ rm -f "$tmp_file2" >/dev/null 2>&1
 bash -c "$CLICKHOUSE_CLIENT \
   --function_sleep_max_microseconds_per_block 60000000 \
   --query=\"select sleepEachRow(1) from numbers(10) where ignore('find_me_TOPSECRET=TOPSECRET')=0 and ignore('fwerkh_that_magic_string_make_me_unique') = 0 FORMAT Null\" \
-  --log_queries=1 --ignore-error --multiquery |& grep -v '^(query: ' > $tmp_file2" &
+  --log_queries=1 --ignore-error |& grep -v '^(query: ' > $tmp_file2" &
 
 rm -f "$tmp_file" >/dev/null 2>&1
 # check that executing query doesn't expose secrets in processlist
@@ -133,7 +133,7 @@ insert into sensitive select number as id, toDate('2019-01-01') as date, 'abcd'
 insert into sensitive select number as id, toDate('2019-01-01') as date, 'find_me_TOPSECRET=TOPSECRET' as value1, rand() as valuer from numbers(10);
 insert into sensitive select number as id, toDate('2019-01-01') as date, 'abcd' as value1, rand() as valuer from numbers(10000);
 select * from sensitive WHERE value1 = 'find_me_TOPSECRET=TOPSECRET' FORMAT Null;
-drop table sensitive;" --log_queries=1 --ignore-error --multiquery >"$tmp_file" 2>&1
+drop table sensitive;" --log_queries=1 --ignore-error >"$tmp_file" 2>&1
 
 grep -F 'find_me_[hidden]' "$tmp_file" >/dev/null || echo 'fail 8a'
 grep -F 'TOPSECRET' "$tmp_file" && echo 'fail 8b'
@@ -144,7 +144,7 @@ echo 9
 $CLICKHOUSE_CLIENT \
    --server_logs_file=/dev/null \
    --query="SELECT if( count() > 0, 'text_log non empty', 'text_log empty') FROM system.text_log WHERE event_date >= yesterday() and message like '%find_me%';
-   select * from system.text_log where event_date >= yesterday() and message like '%TOPSECRET=TOPSECRET%';"  --ignore-error --multiquery
+   select * from system.text_log where event_date >= yesterday() and message like '%TOPSECRET=TOPSECRET%';"  --ignore-error 
 
 echo 'finish'
 rm -f "$tmp_file" >/dev/null 2>&1
diff --git a/tests/queries/0_stateless/01019_alter_materialized_view_atomic.sh b/tests/queries/0_stateless/01019_alter_materialized_view_atomic.sh
index 4bd21fcee02..eb12a76eb62 100755
--- a/tests/queries/0_stateless/01019_alter_materialized_view_atomic.sh
+++ b/tests/queries/0_stateless/01019_alter_materialized_view_atomic.sh
@@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
-$CLICKHOUSE_CLIENT --multiquery <<EOF
+$CLICKHOUSE_CLIENT <<EOF
 DROP TABLE IF EXISTS src;
 DROP TABLE IF EXISTS mv;
 
diff --git a/tests/queries/0_stateless/01019_alter_materialized_view_consistent.sh b/tests/queries/0_stateless/01019_alter_materialized_view_consistent.sh
index 26c2e54e637..eccf456d330 100755
--- a/tests/queries/0_stateless/01019_alter_materialized_view_consistent.sh
+++ b/tests/queries/0_stateless/01019_alter_materialized_view_consistent.sh
@@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
-$CLICKHOUSE_CLIENT --multiquery <<EOF
+$CLICKHOUSE_CLIENT <<EOF
 DROP TABLE IF EXISTS src_a;
 DROP TABLE IF EXISTS src_b;
 
diff --git a/tests/queries/0_stateless/01035_lc_empty_part_bug.sh b/tests/queries/0_stateless/01035_lc_empty_part_bug.sh
index 185c4ef4a4e..a509fcf629c 100755
--- a/tests/queries/0_stateless/01035_lc_empty_part_bug.sh
+++ b/tests/queries/0_stateless/01035_lc_empty_part_bug.sh
@@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
 # that test is failing on versions <= 19.11.12
 
-${CLICKHOUSE_CLIENT} --multiquery --query="
+${CLICKHOUSE_CLIENT} --query="
     DROP TABLE IF EXISTS lc_empty_part_bug;
     create table lc_empty_part_bug (id  UInt64, s String) Engine=MergeTree ORDER BY id SETTINGS number_of_free_entries_in_pool_to_execute_mutation=0;
     insert into lc_empty_part_bug select number as id, toString(rand()) from numbers(100);
@@ -15,7 +15,7 @@ ${CLICKHOUSE_CLIENT} --multiquery --query="
 
 echo 'Waited for mutation to finish'
 
-${CLICKHOUSE_CLIENT} --multiquery --query="
+${CLICKHOUSE_CLIENT} --query="
     alter table lc_empty_part_bug modify column s LowCardinality(String);
     SELECT 'still alive';
     insert into lc_empty_part_bug select number+100 as id, toString(rand()) from numbers(100);
diff --git a/tests/queries/0_stateless/01042_system_reload_dictionary_reloads_completely.sh b/tests/queries/0_stateless/01042_system_reload_dictionary_reloads_completely.sh
index 42488ca946c..03dd376f802 100755
--- a/tests/queries/0_stateless/01042_system_reload_dictionary_reloads_completely.sh
+++ b/tests/queries/0_stateless/01042_system_reload_dictionary_reloads_completely.sh
@@ -8,7 +8,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
 set -e -o pipefail
 
-$CLICKHOUSE_CLIENT --multiquery <<EOF
+$CLICKHOUSE_CLIENT <<EOF
 CREATE TABLE ${CLICKHOUSE_DATABASE}.table(x Int64, y Int64, insert_time DateTime) ENGINE = MergeTree ORDER BY tuple();
 INSERT INTO ${CLICKHOUSE_DATABASE}.table VALUES (12, 102, now());
 
diff --git a/tests/queries/0_stateless/01052_window_view_proc_tumble_to_now.sh b/tests/queries/0_stateless/01052_window_view_proc_tumble_to_now.sh
index 5c70806ea7b..4f8482558c8 100755
--- a/tests/queries/0_stateless/01052_window_view_proc_tumble_to_now.sh
+++ b/tests/queries/0_stateless/01052_window_view_proc_tumble_to_now.sh
@@ -9,7 +9,7 @@ opts=(
     "--allow_experimental_analyzer=0"
 )
 
-$CLICKHOUSE_CLIENT "${opts[@]}" --multiquery <<EOF
+$CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
 SET allow_experimental_window_view = 1;
 DROP TABLE IF EXISTS mt;
 DROP TABLE IF EXISTS dst;
diff --git a/tests/queries/0_stateless/01053_window_view_proc_hop_to_now.sh b/tests/queries/0_stateless/01053_window_view_proc_hop_to_now.sh
index 32c9c52ab09..efd1721e594 100755
--- a/tests/queries/0_stateless/01053_window_view_proc_hop_to_now.sh
+++ b/tests/queries/0_stateless/01053_window_view_proc_hop_to_now.sh
@@ -9,7 +9,7 @@ opts=(
     "--allow_experimental_analyzer=0"
 )
 
-$CLICKHOUSE_CLIENT "${opts[@]}" --multiquery <<EOF
+$CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
 SET allow_experimental_window_view = 1;
 DROP TABLE IF EXISTS mt;
 DROP TABLE IF EXISTS dst;
diff --git a/tests/queries/0_stateless/01054_window_view_proc_tumble_to.sh b/tests/queries/0_stateless/01054_window_view_proc_tumble_to.sh
index ba566bb4ae6..5e603382e5b 100755
--- a/tests/queries/0_stateless/01054_window_view_proc_tumble_to.sh
+++ b/tests/queries/0_stateless/01054_window_view_proc_tumble_to.sh
@@ -9,7 +9,7 @@ opts=(
     "--allow_experimental_analyzer=0"
 )
 
-$CLICKHOUSE_CLIENT "${opts[@]}" --multiquery <<EOF
+$CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
 SET allow_experimental_window_view = 1;
 DROP TABLE IF EXISTS mt;
 DROP TABLE IF EXISTS dst;
diff --git a/tests/queries/0_stateless/01055_window_view_proc_hop_to.sh b/tests/queries/0_stateless/01055_window_view_proc_hop_to.sh
index 0db4173b3dc..00f8f239336 100755
--- a/tests/queries/0_stateless/01055_window_view_proc_hop_to.sh
+++ b/tests/queries/0_stateless/01055_window_view_proc_hop_to.sh
@@ -9,7 +9,7 @@ opts=(
     "--allow_experimental_analyzer=0"
 )
 
-$CLICKHOUSE_CLIENT "${opts[@]}" --multiquery <<EOF
+$CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
 SET allow_experimental_window_view = 1;
 DROP TABLE IF EXISTS mt;
 DROP TABLE IF EXISTS dst;
diff --git a/tests/queries/0_stateless/01057_window_view_event_tumble_to_strict_asc.sh b/tests/queries/0_stateless/01057_window_view_event_tumble_to_strict_asc.sh
index 087e2577e5f..841d5e701a4 100755
--- a/tests/queries/0_stateless/01057_window_view_event_tumble_to_strict_asc.sh
+++ b/tests/queries/0_stateless/01057_window_view_event_tumble_to_strict_asc.sh
@@ -8,7 +8,7 @@ opts=(
     "--allow_experimental_analyzer=0"
 )
 
-$CLICKHOUSE_CLIENT "${opts[@]}" --multiquery <<EOF
+$CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
 SET allow_experimental_window_view = 1;
 DROP TABLE IF EXISTS mt;
 DROP TABLE IF EXISTS dst;
diff --git a/tests/queries/0_stateless/01058_window_view_event_hop_to_strict_asc.sh b/tests/queries/0_stateless/01058_window_view_event_hop_to_strict_asc.sh
index b74e8cc6d25..315959bca9f 100755
--- a/tests/queries/0_stateless/01058_window_view_event_hop_to_strict_asc.sh
+++ b/tests/queries/0_stateless/01058_window_view_event_hop_to_strict_asc.sh
@@ -8,7 +8,7 @@ opts=(
     "--allow_experimental_analyzer=0"
 )
 
-$CLICKHOUSE_CLIENT "${opts[@]}" --multiquery <<EOF
+$CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
 SET allow_experimental_window_view = 1;
 DROP TABLE IF EXISTS mt;
 DROP TABLE IF EXISTS dst;
diff --git a/tests/queries/0_stateless/01060_window_view_event_tumble_to_asc.sh b/tests/queries/0_stateless/01060_window_view_event_tumble_to_asc.sh
index 8161e54923d..592ee93a3af 100755
--- a/tests/queries/0_stateless/01060_window_view_event_tumble_to_asc.sh
+++ b/tests/queries/0_stateless/01060_window_view_event_tumble_to_asc.sh
@@ -8,7 +8,7 @@ opts=(
     "--allow_experimental_analyzer=0"
 )
 
-$CLICKHOUSE_CLIENT "${opts[@]}" --multiquery <<EOF
+$CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
 SET allow_experimental_window_view = 1;
 DROP TABLE IF EXISTS mt;
 DROP TABLE IF EXISTS dst;
diff --git a/tests/queries/0_stateless/01061_window_view_event_hop_to_asc.sh b/tests/queries/0_stateless/01061_window_view_event_hop_to_asc.sh
index 2230fc28fca..6089a56ecc7 100755
--- a/tests/queries/0_stateless/01061_window_view_event_hop_to_asc.sh
+++ b/tests/queries/0_stateless/01061_window_view_event_hop_to_asc.sh
@@ -8,7 +8,7 @@ opts=(
     "--allow_experimental_analyzer=0"
 )
 
-$CLICKHOUSE_CLIENT "${opts[@]}" --multiquery <<EOF
+$CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
 SET allow_experimental_window_view = 1;
 DROP TABLE IF EXISTS mt;
 DROP TABLE IF EXISTS dst;
diff --git a/tests/queries/0_stateless/01063_window_view_event_tumble_to_bounded.sh b/tests/queries/0_stateless/01063_window_view_event_tumble_to_bounded.sh
index 913f2b69b9c..349f73ec637 100755
--- a/tests/queries/0_stateless/01063_window_view_event_tumble_to_bounded.sh
+++ b/tests/queries/0_stateless/01063_window_view_event_tumble_to_bounded.sh
@@ -8,7 +8,7 @@ opts=(
     "--allow_experimental_analyzer=0"
 )
 
-$CLICKHOUSE_CLIENT "${opts[@]}" --multiquery <<EOF
+$CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
 SET allow_experimental_window_view = 1;
 DROP TABLE IF EXISTS mt;
 DROP TABLE IF EXISTS dst;
diff --git a/tests/queries/0_stateless/01064_window_view_event_hop_to_bounded.sh b/tests/queries/0_stateless/01064_window_view_event_hop_to_bounded.sh
index 20e6a72f7d9..8db211edc5d 100755
--- a/tests/queries/0_stateless/01064_window_view_event_hop_to_bounded.sh
+++ b/tests/queries/0_stateless/01064_window_view_event_hop_to_bounded.sh
@@ -8,7 +8,7 @@ opts=(
     "--allow_experimental_analyzer=0"
 )
 
-$CLICKHOUSE_CLIENT "${opts[@]}" --multiquery <<EOF
+$CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
 SET allow_experimental_window_view = 1;
 DROP TABLE IF EXISTS mt;
 DROP TABLE IF EXISTS dst;
diff --git a/tests/queries/0_stateless/01066_window_view_event_tumble_to_strict_asc_lateness.sh b/tests/queries/0_stateless/01066_window_view_event_tumble_to_strict_asc_lateness.sh
index 756c74d4fa9..72988bb5496 100755
--- a/tests/queries/0_stateless/01066_window_view_event_tumble_to_strict_asc_lateness.sh
+++ b/tests/queries/0_stateless/01066_window_view_event_tumble_to_strict_asc_lateness.sh
@@ -8,7 +8,7 @@ opts=(
     "--allow_experimental_analyzer=0"
 )
 
-$CLICKHOUSE_CLIENT "${opts[@]}" --multiquery <<EOF
+$CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
 SET allow_experimental_window_view = 1;
 DROP TABLE IF EXISTS mt;
 DROP TABLE IF EXISTS dst;
diff --git a/tests/queries/0_stateless/01067_window_view_event_tumble_to_asc_lateness.sh b/tests/queries/0_stateless/01067_window_view_event_tumble_to_asc_lateness.sh
index 8d2c58a7332..1c0ef8183ac 100755
--- a/tests/queries/0_stateless/01067_window_view_event_tumble_to_asc_lateness.sh
+++ b/tests/queries/0_stateless/01067_window_view_event_tumble_to_asc_lateness.sh
@@ -8,7 +8,7 @@ opts=(
     "--allow_experimental_analyzer=0"
 )
 
-$CLICKHOUSE_CLIENT "${opts[@]}" --multiquery <<EOF
+$CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
 SET allow_experimental_window_view = 1;
 DROP TABLE IF EXISTS mt;
 DROP TABLE IF EXISTS dst;
diff --git a/tests/queries/0_stateless/01068_window_view_event_tumble_to_bounded_lateness.sh b/tests/queries/0_stateless/01068_window_view_event_tumble_to_bounded_lateness.sh
index 3367bba1e03..af8f70e7ab8 100755
--- a/tests/queries/0_stateless/01068_window_view_event_tumble_to_bounded_lateness.sh
+++ b/tests/queries/0_stateless/01068_window_view_event_tumble_to_bounded_lateness.sh
@@ -8,7 +8,7 @@ opts=(
     "--allow_experimental_analyzer=0"
 )
 
-$CLICKHOUSE_CLIENT "${opts[@]}" --multiquery <<EOF
+$CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
 SET allow_experimental_window_view = 1;
 DROP TABLE IF EXISTS mt;
 DROP TABLE IF EXISTS dst;
diff --git a/tests/queries/0_stateless/01071_window_view_event_tumble_asc_join.sh b/tests/queries/0_stateless/01071_window_view_event_tumble_asc_join.sh
index f1d426228cf..b71ef68a9c8 100755
--- a/tests/queries/0_stateless/01071_window_view_event_tumble_asc_join.sh
+++ b/tests/queries/0_stateless/01071_window_view_event_tumble_asc_join.sh
@@ -8,7 +8,7 @@ opts=(
     "--allow_experimental_analyzer=0"
 )
 
-$CLICKHOUSE_CLIENT "${opts[@]}" --multiquery <<EOF
+$CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
 SET allow_experimental_window_view = 1;
 DROP TABLE IF EXISTS mt;
 DROP TABLE IF EXISTS info;
diff --git a/tests/queries/0_stateless/01072_window_view_multiple_columns_groupby.sh b/tests/queries/0_stateless/01072_window_view_multiple_columns_groupby.sh
index 062ecf648a7..7117e3207fe 100755
--- a/tests/queries/0_stateless/01072_window_view_multiple_columns_groupby.sh
+++ b/tests/queries/0_stateless/01072_window_view_multiple_columns_groupby.sh
@@ -10,7 +10,7 @@ opts=(
     "--allow_experimental_analyzer=0"
 )
 
-$CLICKHOUSE_CLIENT "${opts[@]}" --multiquery <<EOF
+$CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
 SET allow_experimental_window_view = 1;
 DROP TABLE IF EXISTS mt;
 DROP TABLE IF EXISTS dst;
diff --git a/tests/queries/0_stateless/01073_window_view_event_tumble_to_asc_populate.sh b/tests/queries/0_stateless/01073_window_view_event_tumble_to_asc_populate.sh
index b2a98897aff..27527df05f8 100755
--- a/tests/queries/0_stateless/01073_window_view_event_tumble_to_asc_populate.sh
+++ b/tests/queries/0_stateless/01073_window_view_event_tumble_to_asc_populate.sh
@@ -8,7 +8,7 @@ opts=(
     "--allow_experimental_analyzer=0"
 )
 
-$CLICKHOUSE_CLIENT "${opts[@]}" --multiquery <<EOF
+$CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
 SET allow_experimental_window_view = 1;
 DROP TABLE IF EXISTS mt;
 DROP TABLE IF EXISTS dst;
@@ -33,7 +33,7 @@ while true; do
 	$CLICKHOUSE_CLIENT "${opts[@]}" --query="SELECT count(*) FROM dst" | grep -q "7" && break || sleep .5 ||:
 done
 
-$CLICKHOUSE_CLIENT "${opts[@]}" --multiquery <<EOF
+$CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
 SELECT * FROM dst ORDER BY market, w_end;
 INSERT INTO mt VALUES (1, 8, '1990/01/01 12:00:35');
 INSERT INTO mt VALUES (1, 8, '1990/01/01 12:00:37');
diff --git a/tests/queries/0_stateless/01074_window_view_event_tumble_asc_join_populate.sh b/tests/queries/0_stateless/01074_window_view_event_tumble_asc_join_populate.sh
index 5f169920dee..3cd9f750f7c 100755
--- a/tests/queries/0_stateless/01074_window_view_event_tumble_asc_join_populate.sh
+++ b/tests/queries/0_stateless/01074_window_view_event_tumble_asc_join_populate.sh
@@ -8,7 +8,7 @@ opts=(
     "--allow_experimental_analyzer=0"
 )
 
-$CLICKHOUSE_CLIENT "${opts[@]}" --multiquery <<EOF
+$CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
 SET allow_experimental_window_view = 1;
 DROP TABLE IF EXISTS mt;
 DROP TABLE IF EXISTS info;
diff --git a/tests/queries/0_stateless/01075_window_view_proc_tumble_to_now_populate.sh b/tests/queries/0_stateless/01075_window_view_proc_tumble_to_now_populate.sh
index 67c249a9d0e..8bb604997af 100755
--- a/tests/queries/0_stateless/01075_window_view_proc_tumble_to_now_populate.sh
+++ b/tests/queries/0_stateless/01075_window_view_proc_tumble_to_now_populate.sh
@@ -9,7 +9,7 @@ opts=(
     "--allow_experimental_analyzer=0"
 )
 
-$CLICKHOUSE_CLIENT "${opts[@]}" --multiquery <<EOF
+$CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
 SET allow_experimental_window_view = 1;
 DROP TABLE IF EXISTS mt;
 DROP TABLE IF EXISTS dst;
diff --git a/tests/queries/0_stateless/01076_window_view_alter_query_to.sh b/tests/queries/0_stateless/01076_window_view_alter_query_to.sh
index 11c126ea6de..043a2108f0c 100755
--- a/tests/queries/0_stateless/01076_window_view_alter_query_to.sh
+++ b/tests/queries/0_stateless/01076_window_view_alter_query_to.sh
@@ -8,7 +8,7 @@ opts=(
     "--allow_experimental_analyzer=0"
 )
 
-$CLICKHOUSE_CLIENT "${opts[@]}" --multiquery <<EOF
+$CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
 SET allow_experimental_window_view = 1;
 DROP TABLE IF EXISTS mt;
 DROP TABLE IF EXISTS dst;
@@ -33,7 +33,7 @@ done
 $CLICKHOUSE_CLIENT "${opts[@]}" --query="SELECT * FROM dst ORDER BY market, w_end;"
 $CLICKHOUSE_CLIENT "${opts[@]}" --query="SELECT '----ALTER TABLE...MODIFY QUERY----';"
 
-$CLICKHOUSE_CLIENT "${opts[@]}" --multiquery <<EOF
+$CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
 ALTER TABLE wv MODIFY QUERY SELECT count(a) AS count, mt.market * 2 as market, tumbleEnd(wid) AS w_end FROM mt GROUP BY tumble(timestamp, INTERVAL '5' SECOND, 'US/Samoa') AS wid, mt.market;
 
 INSERT INTO mt VALUES (1, 6, '1990/01/01 12:00:10');
diff --git a/tests/queries/0_stateless/01077_window_view_alter_query_to_modify_source.sh b/tests/queries/0_stateless/01077_window_view_alter_query_to_modify_source.sh
index 17c28e2daf5..4d10d7fd2f3 100755
--- a/tests/queries/0_stateless/01077_window_view_alter_query_to_modify_source.sh
+++ b/tests/queries/0_stateless/01077_window_view_alter_query_to_modify_source.sh
@@ -8,7 +8,7 @@ opts=(
     "--allow_experimental_analyzer=0"
 )
 
-$CLICKHOUSE_CLIENT "${opts[@]}" --multiquery <<EOF
+$CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
 SET allow_experimental_window_view = 1;
 DROP TABLE IF EXISTS mt;
 DROP TABLE IF EXISTS mt2;
@@ -35,7 +35,7 @@ done
 $CLICKHOUSE_CLIENT "${opts[@]}" --query="SELECT * FROM dst ORDER BY market, w_end;"
 $CLICKHOUSE_CLIENT "${opts[@]}" --query="SELECT '----ALTER TABLE...MODIFY QUERY----';"
 
-$CLICKHOUSE_CLIENT "${opts[@]}" --multiquery <<EOF
+$CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
 ALTER TABLE wv MODIFY QUERY SELECT count(a) AS count, mt2.market * 2 as market, tumbleEnd(wid) AS w_end FROM mt2 GROUP BY tumble(timestamp, INTERVAL '5' SECOND, 'US/Samoa') AS wid, mt2.market;
 
 INSERT INTO mt2 VALUES (1, 6, '1990/01/01 12:00:10');
diff --git a/tests/queries/0_stateless/01079_window_view_inner_table_memory_tumble.sh b/tests/queries/0_stateless/01079_window_view_inner_table_memory_tumble.sh
index 7747e343bfb..299c0086874 100755
--- a/tests/queries/0_stateless/01079_window_view_inner_table_memory_tumble.sh
+++ b/tests/queries/0_stateless/01079_window_view_inner_table_memory_tumble.sh
@@ -8,7 +8,7 @@ opts=(
     "--allow_experimental_analyzer=0"
 )
 
-$CLICKHOUSE_CLIENT "${opts[@]}" --multiquery <<EOF
+$CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
 SET allow_experimental_window_view = 1;
 DROP TABLE IF EXISTS mt;
 DROP TABLE IF EXISTS dst;
diff --git a/tests/queries/0_stateless/01080_window_view_inner_table_memory_hop.sh b/tests/queries/0_stateless/01080_window_view_inner_table_memory_hop.sh
index 25576ff3c06..34a96d90e80 100755
--- a/tests/queries/0_stateless/01080_window_view_inner_table_memory_hop.sh
+++ b/tests/queries/0_stateless/01080_window_view_inner_table_memory_hop.sh
@@ -8,7 +8,7 @@ opts=(
     "--allow_experimental_analyzer=0"
 )
 
-$CLICKHOUSE_CLIENT "${opts[@]}" --multiquery <<EOF
+$CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
 SET allow_experimental_window_view = 1;
 DROP TABLE IF EXISTS mt;
 DROP TABLE IF EXISTS dst;
diff --git a/tests/queries/0_stateless/01081_window_view_target_table_engine.sh b/tests/queries/0_stateless/01081_window_view_target_table_engine.sh
index dea0e5a7ba0..180653d4931 100755
--- a/tests/queries/0_stateless/01081_window_view_target_table_engine.sh
+++ b/tests/queries/0_stateless/01081_window_view_target_table_engine.sh
@@ -8,7 +8,7 @@ opts=(
     "--allow_experimental_analyzer=0"
 )
 
-$CLICKHOUSE_CLIENT "${opts[@]}" --multiquery <<EOF
+$CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
 SET allow_experimental_window_view = 1;
 DROP TABLE IF EXISTS mt;
 DROP TABLE IF EXISTS wv;
diff --git a/tests/queries/0_stateless/01083_window_view_select.sh b/tests/queries/0_stateless/01083_window_view_select.sh
index ca9195aa00d..4d7c3cf5ace 100755
--- a/tests/queries/0_stateless/01083_window_view_select.sh
+++ b/tests/queries/0_stateless/01083_window_view_select.sh
@@ -8,7 +8,7 @@ opts=(
     "--allow_experimental_analyzer=0"
 )
 
-$CLICKHOUSE_CLIENT "${opts[@]}" --multiquery <<EOF
+$CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
 SET allow_experimental_window_view = 1;
 DROP TABLE IF EXISTS mt;
 DROP TABLE IF EXISTS dst;
diff --git a/tests/queries/0_stateless/01084_window_view_with_table_identifier.sh b/tests/queries/0_stateless/01084_window_view_with_table_identifier.sh
index d8a29be781d..64210de7ed0 100755
--- a/tests/queries/0_stateless/01084_window_view_with_table_identifier.sh
+++ b/tests/queries/0_stateless/01084_window_view_with_table_identifier.sh
@@ -8,7 +8,7 @@ opts=(
     "--allow_experimental_analyzer=0"
 )
 
-$CLICKHOUSE_CLIENT "${opts[@]}" --multiquery <<EOF
+$CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
 SET allow_experimental_window_view = 1;
 DROP TABLE IF EXISTS mt;
 DROP TABLE IF EXISTS wv;
diff --git a/tests/queries/0_stateless/01086_window_view_cleanup.sh b/tests/queries/0_stateless/01086_window_view_cleanup.sh
index 01bf50bc14d..113bcffb2af 100755
--- a/tests/queries/0_stateless/01086_window_view_cleanup.sh
+++ b/tests/queries/0_stateless/01086_window_view_cleanup.sh
@@ -12,7 +12,7 @@ opts=(
     "--allow_experimental_analyzer=0"
 )
 
-$CLICKHOUSE_CLIENT "${opts[@]}" --allow_deprecated_database_ordinary=1 --multiquery <<EOF
+$CLICKHOUSE_CLIENT "${opts[@]}" --allow_deprecated_database_ordinary=1 <<EOF
 SET allow_experimental_window_view = 1;
 SET window_view_clean_interval = 1;
 
diff --git a/tests/queries/0_stateless/01087_window_view_alter_query.sh b/tests/queries/0_stateless/01087_window_view_alter_query.sh
index a28ee4bc425..b9dd0d19234 100755
--- a/tests/queries/0_stateless/01087_window_view_alter_query.sh
+++ b/tests/queries/0_stateless/01087_window_view_alter_query.sh
@@ -8,7 +8,7 @@ opts=(
     "--allow_experimental_analyzer=0"
 )
 
-$CLICKHOUSE_CLIENT "${opts[@]}" --multiquery <<EOF
+$CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
 SET allow_experimental_window_view = 1;
 DROP TABLE IF EXISTS mt;
 DROP TABLE IF EXISTS wv;
@@ -31,7 +31,7 @@ done
 $CLICKHOUSE_CLIENT "${opts[@]}" --query="SELECT * FROM wv ORDER BY market, w_end;"
 $CLICKHOUSE_CLIENT "${opts[@]}" --query="SELECT '----ALTER TABLE...MODIFY QUERY----';"
 
-$CLICKHOUSE_CLIENT "${opts[@]}" --multiquery <<EOF
+$CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
 ALTER TABLE wv MODIFY QUERY SELECT count(a) AS count, mt.market * 2 as market, tumbleEnd(wid) AS w_end FROM mt GROUP BY tumble(timestamp, INTERVAL '5' SECOND, 'US/Samoa') AS wid, mt.market;
 
 INSERT INTO mt VALUES (1, 6, toDateTime('1990/01/01 12:00:10', 'US/Samoa'));
diff --git a/tests/queries/0_stateless/01088_window_view_default_column.sh b/tests/queries/0_stateless/01088_window_view_default_column.sh
index 1cdd9e3c9b9..03be3459e54 100755
--- a/tests/queries/0_stateless/01088_window_view_default_column.sh
+++ b/tests/queries/0_stateless/01088_window_view_default_column.sh
@@ -8,7 +8,7 @@ opts=(
     "--allow_experimental_analyzer=0"
 )
 
-$CLICKHOUSE_CLIENT "${opts[@]}" --multiquery <<EOF
+$CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
 SET allow_experimental_window_view = 1;
 DROP TABLE IF EXISTS mt;
 DROP TABLE IF EXISTS wv;
diff --git a/tests/queries/0_stateless/01133_begin_commit_race.sh b/tests/queries/0_stateless/01133_begin_commit_race.sh
index 2b266527541..81e88e387ee 100755
--- a/tests/queries/0_stateless/01133_begin_commit_race.sh
+++ b/tests/queries/0_stateless/01133_begin_commit_race.sh
@@ -14,7 +14,7 @@ $CLICKHOUSE_CLIENT --query "CREATE TABLE mt (n Int64) ENGINE=MergeTree ORDER BY
 function begin_commit_readonly()
 {
   while true; do
-    $CLICKHOUSE_CLIENT --multiquery --query "
+    $CLICKHOUSE_CLIENT --query "
             SET wait_changes_become_visible_after_commit_mode='wait';
             BEGIN TRANSACTION;
             COMMIT;" 2>&1| grep -Fa "Exception: " | grep -Fv UNKNOWN_STATUS_OF_TRANSACTION
@@ -24,7 +24,7 @@ function begin_commit_readonly()
 function begin_rollback_readonly()
 {
   while true; do
-    $CLICKHOUSE_CLIENT --wait_changes_become_visible_after_commit_mode=wait_unknown --multiquery --query "
+    $CLICKHOUSE_CLIENT --wait_changes_become_visible_after_commit_mode=wait_unknown --query "
             BEGIN TRANSACTION;
             SET TRANSACTION SNAPSHOT 42;
             ROLLBACK;"
@@ -34,7 +34,7 @@ function begin_rollback_readonly()
 function begin_insert_commit()
 {
   while true; do
-    $CLICKHOUSE_CLIENT --wait_changes_become_visible_after_commit_mode=async --multiquery --query "
+    $CLICKHOUSE_CLIENT --wait_changes_become_visible_after_commit_mode=async --query "
             BEGIN TRANSACTION;
             INSERT INTO mt VALUES ($RANDOM);
             COMMIT;" 2>&1| grep -Fa "Exception: " | grep -Fv UNKNOWN_STATUS_OF_TRANSACTION
diff --git a/tests/queries/0_stateless/01169_alter_partition_isolation_stress.sh b/tests/queries/0_stateless/01169_alter_partition_isolation_stress.sh
index d4884cbf457..8873fd88f0e 100755
--- a/tests/queries/0_stateless/01169_alter_partition_isolation_stress.sh
+++ b/tests/queries/0_stateless/01169_alter_partition_isolation_stress.sh
@@ -22,7 +22,7 @@ function thread_insert()
     set -eu
     val=1
     while true; do
-        $CLICKHOUSE_CLIENT --multiquery --query "
+        $CLICKHOUSE_CLIENT --query "
         BEGIN TRANSACTION;
         INSERT INTO src VALUES /* ($val, 1) */ ($val, 1);
         INSERT INTO src VALUES /* ($val, 2) */ ($val, 2);
@@ -210,7 +210,7 @@ function thread_select()
     set -eu
     while true; do
         output=$(
-        $CLICKHOUSE_CLIENT --multiquery --query "
+        $CLICKHOUSE_CLIENT --query "
         BEGIN TRANSACTION;
         -- no duplicates
         SELECT type, throwIf(count(n) != countDistinct(n)) FROM src GROUP BY type FORMAT Null;
diff --git a/tests/queries/0_stateless/01169_old_alter_partition_isolation_stress.sh b/tests/queries/0_stateless/01169_old_alter_partition_isolation_stress.sh
index 0d2016952d4..404042ab64e 100755
--- a/tests/queries/0_stateless/01169_old_alter_partition_isolation_stress.sh
+++ b/tests/queries/0_stateless/01169_old_alter_partition_isolation_stress.sh
@@ -19,7 +19,7 @@ function thread_insert()
     set -e
     val=1
     while true; do
-        $CLICKHOUSE_CLIENT --multiquery --query "
+        $CLICKHOUSE_CLIENT --query "
         BEGIN TRANSACTION;
         INSERT INTO src VALUES /* ($val, 1) */ ($val, 1);
         INSERT INTO src VALUES /* ($val, 2) */ ($val, 2);
@@ -40,7 +40,7 @@ function thread_partition_src_to_dst()
     sum=0
     for i in {1..20}; do
         out=$(
-        $CLICKHOUSE_CLIENT --multiquery --query "
+        $CLICKHOUSE_CLIENT --query "
         BEGIN TRANSACTION;
         INSERT INTO src VALUES /* ($i, 3) */ ($i, 3);
         INSERT INTO dst SELECT * FROM src;
@@ -49,7 +49,7 @@ function thread_partition_src_to_dst()
         SELECT throwIf((SELECT (count(), sum(n)) FROM merge(currentDatabase(), '') WHERE type=3) != ($count + 1, $sum + $i)) FORMAT Null;
         COMMIT;" 2>&1) ||:
 
-        echo "$out" | grep -Fv "SERIALIZATION_ERROR" | grep -F "Received from " && $CLICKHOUSE_CLIENT --multiquery --query "
+        echo "$out" | grep -Fv "SERIALIZATION_ERROR" | grep -F "Received from " && $CLICKHOUSE_CLIENT --query "
                                                                                    begin transaction;
                                                                                    set transaction snapshot 3;
                                                                                    select $i, 'src', type, n, _part from src order by type, n;
@@ -68,7 +68,7 @@ function thread_partition_dst_to_src()
         if (( i % 2 )); then
             action="COMMIT"
         fi
-        $CLICKHOUSE_CLIENT --multiquery --query "
+        $CLICKHOUSE_CLIENT --query "
         SYSTEM STOP MERGES dst;
         ALTER TABLE dst DROP PARTITION ID 'nonexistent';  -- STOP MERGES doesn't wait for started merges to finish, so we use this trick
         SYSTEM SYNC TRANSACTION LOG;
@@ -87,7 +87,7 @@ function thread_select()
 {
     set -e
     while true; do
-        $CLICKHOUSE_CLIENT --multiquery --query "
+        $CLICKHOUSE_CLIENT --query "
         BEGIN TRANSACTION;
         -- no duplicates
         SELECT type, throwIf(count(n) != countDistinct(n)) FROM src GROUP BY type FORMAT Null;
diff --git a/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh b/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh
index 3b9bb50517d..2fb58e4cc57 100755
--- a/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh
+++ b/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh
@@ -65,7 +65,7 @@ function insert_commit_action()
     local tag=$1; shift
 
     # some transactions will fail due to constraint
-    $CLICKHOUSE_CLIENT --multiquery --query "
+    $CLICKHOUSE_CLIENT --query "
         BEGIN TRANSACTION;
         INSERT INTO src VALUES /* ($i, $tag) */ ($i, $tag);
         SELECT throwIf((SELECT sum(nm) FROM mv) != $(($i * $tag))) /* ($i, $tag) */ FORMAT Null;
@@ -83,7 +83,7 @@ function insert_rollback_action()
     local i=$1; shift
     local tag=$1; shift
 
-    $CLICKHOUSE_CLIENT --multiquery --query "
+    $CLICKHOUSE_CLIENT --query "
         BEGIN TRANSACTION;
         INSERT INTO src VALUES /* (42, $tag) */ (42, $tag);
         SELECT throwIf((SELECT count() FROM src WHERE n=42 AND m=$tag) != 1) FORMAT Null;
@@ -112,7 +112,7 @@ function optimize_action()
         action="ROLLBACK"
     fi
 
-    $CLICKHOUSE_CLIENT --multiquery --query "
+    $CLICKHOUSE_CLIENT --query "
         BEGIN TRANSACTION;
             $optimize_query;
         $action;
@@ -126,7 +126,7 @@ function select_action()
 {
     set -e
 
-    $CLICKHOUSE_CLIENT --multiquery --query "
+    $CLICKHOUSE_CLIENT --query "
         BEGIN TRANSACTION;
         SELECT throwIf((SELECT (sum(n), count() % 2) FROM src) != (0, 1)) FORMAT Null;
         SELECT throwIf((SELECT (sum(nm), count() % 2) FROM mv) != (0, 1)) FORMAT Null;
@@ -140,7 +140,7 @@ function select_insert_action()
 {
     set -e
 
-    $CLICKHOUSE_CLIENT --multiquery --query "
+    $CLICKHOUSE_CLIENT --query "
         BEGIN TRANSACTION;
         SELECT throwIf((SELECT count() FROM tmp) != 0) FORMAT Null;
         INSERT INTO tmp SELECT 1, n*m FROM src;
@@ -199,7 +199,7 @@ wait $PID_8 || echo "second select_insert_action has failed with status $?" 2>&1
 
 wait_for_queries_to_finish $WAIT_FINISH
 
-$CLICKHOUSE_CLIENT --multiquery --query "
+$CLICKHOUSE_CLIENT --query "
     BEGIN TRANSACTION;
         SELECT throwIf((SELECT (sum(n), count() % 2) FROM src) != (0, 1)) FORMAT Null;
         SELECT throwIf((SELECT (sum(nm), count() % 2) FROM mv) != (0, 1)) FORMAT Null;
@@ -209,7 +209,7 @@ $CLICKHOUSE_CLIENT --multiquery --query "
     COMMIT;
 "
 
-$CLICKHOUSE_CLIENT --multiquery --query  "
+$CLICKHOUSE_CLIENT --query  "
     SELECT throwIf((SELECT (sum(n), count() % 2) FROM src) != (0, 1)) FORMAT Null;
     SELECT throwIf((SELECT (sum(nm), count() % 2) FROM mv) != (0, 1)) FORMAT Null;
     SELECT throwIf((SELECT (sum(nm), count() % 2) FROM dst) != (0, 1)) FORMAT Null;
diff --git a/tests/queries/0_stateless/01174_select_insert_isolation.sh b/tests/queries/0_stateless/01174_select_insert_isolation.sh
index 6321f6ff01b..235d98fb5de 100755
--- a/tests/queries/0_stateless/01174_select_insert_isolation.sh
+++ b/tests/queries/0_stateless/01174_select_insert_isolation.sh
@@ -16,7 +16,7 @@ $CLICKHOUSE_CLIENT --query "CREATE TABLE mt (n Int8, m Int8) ENGINE=MergeTree OR
 function thread_insert_commit()
 {
     for i in {1..50}; do
-        $CLICKHOUSE_CLIENT --multiquery --query "
+        $CLICKHOUSE_CLIENT --query "
         BEGIN TRANSACTION;
         INSERT INTO mt VALUES /* ($i, $1) */ ($i, $1);
         INSERT INTO mt VALUES /* (-$i, $1) */ (-$i, $1);
@@ -27,7 +27,7 @@ function thread_insert_commit()
 function thread_insert_rollback()
 {
     for _ in {1..50}; do
-        $CLICKHOUSE_CLIENT --multiquery --query "
+        $CLICKHOUSE_CLIENT --query "
         BEGIN TRANSACTION;
         INSERT INTO mt VALUES /* (42, $1) */ (42, $1);
         ROLLBACK;";
@@ -38,7 +38,7 @@ function thread_select()
 {
     while true; do
         # The first and the last queries must get the same result
-        $CLICKHOUSE_CLIENT --multiquery --query "
+        $CLICKHOUSE_CLIENT --query "
         BEGIN TRANSACTION;
         SET throw_on_unsupported_query_inside_transaction=0;
         CREATE TEMPORARY TABLE tmp AS SELECT arraySort(groupArray(n)), arraySort(groupArray(m)), arraySort(groupArray(_part)) FROM mt FORMAT Null;
@@ -58,7 +58,7 @@ kill -TERM $PID_4
 wait
 wait_for_queries_to_finish 40
 
-$CLICKHOUSE_CLIENT --multiquery --query "
+$CLICKHOUSE_CLIENT --query "
 BEGIN TRANSACTION;
 SELECT count(), sum(n), sum(m=1), sum(m=2), sum(m=3) FROM mt;";
 
diff --git a/tests/queries/0_stateless/01198_client_quota_key.sh b/tests/queries/0_stateless/01198_client_quota_key.sh
index 3f5f5df5071..d08aa2e364f 100755
--- a/tests/queries/0_stateless/01198_client_quota_key.sh
+++ b/tests/queries/0_stateless/01198_client_quota_key.sh
@@ -5,4 +5,4 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
-$CLICKHOUSE_CLIENT --quota_key Hello --query_id test_quota_key --log_queries 1 --multiquery --query "SELECT 1; SYSTEM FLUSH LOGS; SELECT DISTINCT quota_key FROM system.query_log WHERE current_database = currentDatabase() AND event_date >= yesterday() AND event_time >= now() - 300 AND query_id = 'test_quota_key'"
+$CLICKHOUSE_CLIENT --quota_key Hello --query_id test_quota_key --log_queries 1 --query "SELECT 1; SYSTEM FLUSH LOGS; SELECT DISTINCT quota_key FROM system.query_log WHERE current_database = currentDatabase() AND event_date >= yesterday() AND event_time >= now() - 300 AND query_id = 'test_quota_key'"
diff --git a/tests/queries/0_stateless/01285_engine_join_donmikel.sh b/tests/queries/0_stateless/01285_engine_join_donmikel.sh
index 7522ed9924b..ce273ab8e0c 100755
--- a/tests/queries/0_stateless/01285_engine_join_donmikel.sh
+++ b/tests/queries/0_stateless/01285_engine_join_donmikel.sh
@@ -4,7 +4,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
-$CLICKHOUSE_CLIENT --multiquery --query "
+$CLICKHOUSE_CLIENT --query "
 DROP TABLE IF EXISTS NmSubj;
 DROP TABLE IF EXISTS events;
 
@@ -60,7 +60,7 @@ FROM events as e INNER JOIN NmSubj as ns
 ON ns.NmId = toUInt32(e.Param1)
 WHERE e.EventDate = today() - 7 AND e.EventId = 'GCO' AND ns.SubjectId = 2073"
 
-$CLICKHOUSE_CLIENT --multiquery --query "
+$CLICKHOUSE_CLIENT --query "
 DROP TABLE NmSubj;
 DROP TABLE events;
 "
diff --git a/tests/queries/0_stateless/01293_optimize_final_force.sh b/tests/queries/0_stateless/01293_optimize_final_force.sh
index e838af8af9b..9c135d272e4 100755
--- a/tests/queries/0_stateless/01293_optimize_final_force.sh
+++ b/tests/queries/0_stateless/01293_optimize_final_force.sh
@@ -11,7 +11,7 @@ TIMELIMIT=31
 while [ $SECONDS -lt "$TIMELIMIT" ] && [ $it -lt 100 ];
 do
     it=$((it+1))
-    $CLICKHOUSE_CLIENT --multiquery --query "
+    $CLICKHOUSE_CLIENT --query "
         DROP TABLE IF EXISTS mt;
         CREATE TABLE mt (x UInt8, k UInt8 DEFAULT 0) ENGINE = SummingMergeTree ORDER BY k;
 
diff --git a/tests/queries/0_stateless/01304_direct_io_long.sh b/tests/queries/0_stateless/01304_direct_io_long.sh
index 2e27c2f7728..1241f299d94 100755
--- a/tests/queries/0_stateless/01304_direct_io_long.sh
+++ b/tests/queries/0_stateless/01304_direct_io_long.sh
@@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
-$CLICKHOUSE_CLIENT --multiquery --query "
+$CLICKHOUSE_CLIENT --query "
     DROP TABLE IF EXISTS bug;
     CREATE TABLE bug (UserID UInt64, Date Date) ENGINE = MergeTree ORDER BY Date
         SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi', merge_max_block_size = 8192;
@@ -18,5 +18,5 @@ cat "$LOG" | grep Loaded
 
 rm "$LOG"
 
-$CLICKHOUSE_CLIENT --multiquery --query "
+$CLICKHOUSE_CLIENT --query "
     DROP TABLE bug;"
diff --git a/tests/queries/0_stateless/01375_storage_file_tsv_csv_with_names_write_prefix.sh b/tests/queries/0_stateless/01375_storage_file_tsv_csv_with_names_write_prefix.sh
index af8d3f4e69b..a634f689dca 100755
--- a/tests/queries/0_stateless/01375_storage_file_tsv_csv_with_names_write_prefix.sh
+++ b/tests/queries/0_stateless/01375_storage_file_tsv_csv_with_names_write_prefix.sh
@@ -9,7 +9,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 echo 'zero rows'
 for format in TSVWithNames TSVWithNamesAndTypes CSVWithNames CSVWithNamesAndTypes JSONCompactEachRowWithNames JSONCompactEachRowWithNamesAndTypes JSONCompactStringsEachRow JSONCompactStringsEachRowWithNamesAndTypes; do
     echo $format
-    ${CLICKHOUSE_LOCAL} --multiquery --query="
+    ${CLICKHOUSE_LOCAL} --query="
         CREATE TABLE ${format}_01375 ENGINE File($format, '01375_$format') AS SELECT * FROM numbers(1) WHERE number < 0;
         SELECT * FROM ${format}_01375;
         DROP TABLE ${format}_01375;
@@ -22,7 +22,7 @@ echo 'multi clickhouse-local one file'
 for format in TSVWithNames TSVWithNamesAndTypes CSVWithNames CSVWithNamesAndTypes JSONCompactEachRowWithNames JSONCompactEachRowWithNamesAndTypes JSONCompactStringsEachRow JSONCompactStringsEachRowWithNamesAndTypes; do
     echo $format
     for _ in {1..2}; do
-        ${CLICKHOUSE_LOCAL} --multiquery --query="
+        ${CLICKHOUSE_LOCAL} --query="
             CREATE TABLE ${format}_01375 ENGINE File($format, '01375_$format') AS SELECT * FROM numbers(1);
             SELECT * FROM ${format}_01375;
             DROP TABLE ${format}_01375;
diff --git a/tests/queries/0_stateless/01443_merge_truncate_long.sh b/tests/queries/0_stateless/01443_merge_truncate_long.sh
index 65b9bcd366e..51654b2e4e1 100755
--- a/tests/queries/0_stateless/01443_merge_truncate_long.sh
+++ b/tests/queries/0_stateless/01443_merge_truncate_long.sh
@@ -34,7 +34,7 @@ do
         SELECT count() FROM t HAVING count() > 0;
         SELECT ${i};
         "
-done | ${CLICKHOUSE_CLIENT} --multiquery
+done | ${CLICKHOUSE_CLIENT}
 
 wait
 
diff --git a/tests/queries/0_stateless/01527_clickhouse_local_optimize.sh b/tests/queries/0_stateless/01527_clickhouse_local_optimize.sh
index d61d34244d9..c1d5c357308 100755
--- a/tests/queries/0_stateless/01527_clickhouse_local_optimize.sh
+++ b/tests/queries/0_stateless/01527_clickhouse_local_optimize.sh
@@ -10,6 +10,6 @@ rm -rf "${WORKING_FOLDER_01527}"
 mkdir -p "${WORKING_FOLDER_01527}"
 
 # OPTIMIZE was crashing due to lack of temporary volume in local
-${CLICKHOUSE_LOCAL} --multiquery --query "drop database if exists d; create database d; create table d.t engine MergeTree order by a as select 1 a; optimize table d.t final" --path="${WORKING_FOLDER_01527}"
+${CLICKHOUSE_LOCAL} --query "drop database if exists d; create database d; create table d.t engine MergeTree order by a as select 1 a; optimize table d.t final" --path="${WORKING_FOLDER_01527}"
 
 rm -rf "${WORKING_FOLDER_01527}"
diff --git a/tests/queries/0_stateless/01543_avro_deserialization_with_lc.sh b/tests/queries/0_stateless/01543_avro_deserialization_with_lc.sh
index a5697a62dc2..bc9efaedd5d 100755
--- a/tests/queries/0_stateless/01543_avro_deserialization_with_lc.sh
+++ b/tests/queries/0_stateless/01543_avro_deserialization_with_lc.sh
@@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
-$CLICKHOUSE_CLIENT --multiquery --query "
+$CLICKHOUSE_CLIENT --query "
 SET allow_suspicious_low_cardinality_types=1;
 CREATE TABLE IF NOT EXISTS test_01543 (value LowCardinality(String), value2 LowCardinality(UInt64)) ENGINE=Memory();
 "
diff --git a/tests/queries/0_stateless/01544_file_engine_settings.sh b/tests/queries/0_stateless/01544_file_engine_settings.sh
index b31754f9531..eb0a8a964d0 100755
--- a/tests/queries/0_stateless/01544_file_engine_settings.sh
+++ b/tests/queries/0_stateless/01544_file_engine_settings.sh
@@ -10,7 +10,7 @@ rm -f -- "$the_file"
 
 # We are going to check that format settings work for File engine,
 # by creating a table with a non-default delimiter, and reading from it.
-${CLICKHOUSE_LOCAL} --multiquery --query "
+${CLICKHOUSE_LOCAL} --query "
     create table t(a int, b int) engine File(CSV, '$the_file') settings format_csv_delimiter = '|';
     insert into t select 1 a, 1 b;
 "
@@ -18,7 +18,7 @@ ${CLICKHOUSE_LOCAL} --multiquery --query "
 # See what's in the file
 cat "$the_file"
 
-${CLICKHOUSE_LOCAL} --multiquery --query "
+${CLICKHOUSE_LOCAL} --query "
     create table t(a int, b int) engine File(CSV, '$the_file') settings format_csv_delimiter = '|';
     select * from t;
 " 
diff --git a/tests/queries/0_stateless/01600_detach_permanently.sh b/tests/queries/0_stateless/01600_detach_permanently.sh
index 6721dbf3015..679e9a749ee 100755
--- a/tests/queries/0_stateless/01600_detach_permanently.sh
+++ b/tests/queries/0_stateless/01600_detach_permanently.sh
@@ -18,7 +18,7 @@ mkdir -p "${WORKING_FOLDER_01600}"
 clickhouse_local() {
     local query="$1"
     shift
-    ${CLICKHOUSE_LOCAL} --allow_deprecated_database_ordinary=1 --multiquery --query "$query" "$@" --path="${WORKING_FOLDER_01600}"
+    ${CLICKHOUSE_LOCAL} --allow_deprecated_database_ordinary=1 --query "$query" "$@" --path="${WORKING_FOLDER_01600}"
 }
 
 test_detach_attach_sequence() {
diff --git a/tests/queries/0_stateless/01600_parts_states_metrics_long.sh b/tests/queries/0_stateless/01600_parts_states_metrics_long.sh
index 7215f270a4c..47b5a4dea13 100755
--- a/tests/queries/0_stateless/01600_parts_states_metrics_long.sh
+++ b/tests/queries/0_stateless/01600_parts_states_metrics_long.sh
@@ -24,7 +24,7 @@ verify()
 
         if [[ $i -eq 5000 ]]
         then
-            $CLICKHOUSE_CLIENT --multiquery "
+            $CLICKHOUSE_CLIENT "
               SELECT sumIf(value, metric = 'PartsActive'), sumIf(value, metric = 'PartsOutdated') FROM system.metrics;
               SELECT sum(active), sum(NOT active) FROM system.parts;
               SELECT sum(active), sum(NOT active) FROM system.projection_parts;
diff --git a/tests/queries/0_stateless/01606_git_import.sh b/tests/queries/0_stateless/01606_git_import.sh
index 48558d79f93..6986d6b14cf 100755
--- a/tests/queries/0_stateless/01606_git_import.sh
+++ b/tests/queries/0_stateless/01606_git_import.sh
@@ -19,7 +19,7 @@ done
 
 ${CLICKHOUSE_GIT_IMPORT} 2>&1 | wc -l
 
-${CLICKHOUSE_CLIENT} --multiline --multiquery --query "
+${CLICKHOUSE_CLIENT} --multiline --query "
 
 DROP TABLE IF EXISTS commits;
 DROP TABLE IF EXISTS file_changes;
@@ -122,7 +122,7 @@ ${CLICKHOUSE_CLIENT} --query "SELECT count() FROM commits"
 ${CLICKHOUSE_CLIENT} --query "SELECT count() FROM file_changes"
 ${CLICKHOUSE_CLIENT} --query "SELECT count(), round(avg(indent), 1) FROM line_changes"
 
-${CLICKHOUSE_CLIENT} --multiline --multiquery --query "
+${CLICKHOUSE_CLIENT} --multiline --query "
 DROP TABLE commits;
 DROP TABLE file_changes;
 DROP TABLE line_changes;
diff --git a/tests/queries/0_stateless/01607_arrays_as_nested_csv.sh b/tests/queries/0_stateless/01607_arrays_as_nested_csv.sh
index 946be7fb4af..2a1182c14c1 100755
--- a/tests/queries/0_stateless/01607_arrays_as_nested_csv.sh
+++ b/tests/queries/0_stateless/01607_arrays_as_nested_csv.sh
@@ -4,7 +4,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
-${CLICKHOUSE_CLIENT} --multiquery --query "
+${CLICKHOUSE_CLIENT} --query "
 DROP TABLE IF EXISTS test;
 CREATE TABLE test (a Array(String)) ENGINE = Memory;
 "
@@ -22,7 +22,7 @@ ${CLICKHOUSE_CLIENT} --input_format_csv_arrays_as_nested_csv 1 --query "INSERT I
 """Hello"", ""world"", ""42"""" TV"""
 END
 
-${CLICKHOUSE_CLIENT} --multiquery --query "
+${CLICKHOUSE_CLIENT} --query "
 SELECT * FROM test;
 DROP TABLE IF EXISTS test;
 "
diff --git a/tests/queries/0_stateless/01632_tinylog_read_write.sh b/tests/queries/0_stateless/01632_tinylog_read_write.sh
index 10625ec5d27..68d28b080e9 100755
--- a/tests/queries/0_stateless/01632_tinylog_read_write.sh
+++ b/tests/queries/0_stateless/01632_tinylog_read_write.sh
@@ -8,7 +8,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CURDIR"/../shell_config.sh
 
 
-$CLICKHOUSE_CLIENT --multiquery --query "DROP TABLE IF EXISTS test; CREATE TABLE IF NOT EXISTS test (x UInt64, s Array(Nullable(String))) ENGINE = TinyLog;"
+$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test; CREATE TABLE IF NOT EXISTS test (x UInt64, s Array(Nullable(String))) ENGINE = TinyLog;"
 
 function thread_select {
     local TIMELIMIT=$((SECONDS+$1))
@@ -47,4 +47,4 @@ thread_insert $TIMEOUT &
 wait
 echo "Done"
 
-$CLICKHOUSE_CLIENT --multiquery --query "DROP TABLE IF EXISTS test;"
+$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test;"
diff --git a/tests/queries/0_stateless/01658_read_file_to_stringcolumn.sh b/tests/queries/0_stateless/01658_read_file_to_stringcolumn.sh
index 685fe69642a..ceb6aa060ea 100755
--- a/tests/queries/0_stateless/01658_read_file_to_stringcolumn.sh
+++ b/tests/queries/0_stateless/01658_read_file_to_stringcolumn.sh
@@ -24,7 +24,7 @@ ${CLICKHOUSE_CLIENT} --query "select file('a.txt'), file('b.txt');";echo ":"$?
 ${CLICKHOUSE_CLIENT} --query "insert into data select file('a.txt'), file('b.txt');";echo ":"$?
 ${CLICKHOUSE_CLIENT} --query "insert into data select file('a.txt'), file('b.txt');";echo ":"$?
 ${CLICKHOUSE_CLIENT} --query "select file('c.txt'), * from data";echo ":"$?
-${CLICKHOUSE_CLIENT} --multiquery --query "
+${CLICKHOUSE_CLIENT} --query "
     create table filenames(name String) engine=MergeTree() order by tuple();
     insert into filenames values ('a.txt'), ('b.txt'), ('c.txt');
     select file(name) from filenames format TSV;
@@ -56,7 +56,7 @@ echo $c_count
 
 # Valid cases:
 # The default dir is the CWD path in LOCAL mode
-${CLICKHOUSE_LOCAL} --multiquery --query "
+${CLICKHOUSE_LOCAL} --query "
     drop table if exists data;
     create table data (A String, B String) engine=MergeTree() order by A;
     select file('a.txt'), file('b.txt');
diff --git a/tests/queries/0_stateless/01666_merge_tree_max_query_limit.sh b/tests/queries/0_stateless/01666_merge_tree_max_query_limit.sh
index e04c9515009..ec318db98bf 100755
--- a/tests/queries/0_stateless/01666_merge_tree_max_query_limit.sh
+++ b/tests/queries/0_stateless/01666_merge_tree_max_query_limit.sh
@@ -8,7 +8,7 @@ function wait_for_query_to_start() {
     while [[ $($CLICKHOUSE_CURL -sS "$CLICKHOUSE_URL" -d "SELECT sum(read_rows) FROM system.processes WHERE query_id = '$1'") == 0 ]]; do sleep 0.1; done
 }
 
-${CLICKHOUSE_CLIENT} --multiline --multiquery --query "
+${CLICKHOUSE_CLIENT} --multiline --query "
 drop table if exists simple;
 
 create table simple (i int, j int) engine = MergeTree order by i
diff --git a/tests/queries/0_stateless/01747_system_session_log_long.sh b/tests/queries/0_stateless/01747_system_session_log_long.sh
index 022bf488886..07055f96782 100755
--- a/tests/queries/0_stateless/01747_system_session_log_long.sh
+++ b/tests/queries/0_stateless/01747_system_session_log_long.sh
@@ -82,7 +82,7 @@ trap "cleanup" EXIT
 function executeQueryExpectError()
 {
     cat - > "${TMP_QUERY_FILE}"
-    ! ${CLICKHOUSE_CLIENT} --multiquery --queries-file "${TMP_QUERY_FILE}" "${@}"  2>&1 | tee -a "${TMP_QUERY_FILE}"
+    ! ${CLICKHOUSE_CLIENT} --queries-file "${TMP_QUERY_FILE}" "${@}"  2>&1 | tee -a "${TMP_QUERY_FILE}"
 }
 
 function createUser()
@@ -303,7 +303,7 @@ function runEndpointTests()
     if [[ -n "${setup_queries}" ]]
     then
         # echo "Executing setup queries: ${setup_queries}"
-        echo "${setup_queries}" | executeQuery --multiquery
+        echo "${setup_queries}" | executeQuery
     fi
 
     testTCP "${auth_type}" "${username}" "${password}"
@@ -357,7 +357,7 @@ testAsUserIdentifiedBy "plaintext_password"
 testAsUserIdentifiedBy "sha256_password"
 testAsUserIdentifiedBy "double_sha1_password"
 
-executeQuery --multiquery <<EOF
+executeQuery <<EOF
 SYSTEM FLUSH LOGS;
 
 WITH
diff --git a/tests/queries/0_stateless/01801_nullable_low_cardinality_tsv.sh b/tests/queries/0_stateless/01801_nullable_low_cardinality_tsv.sh
index 935bfe18cfc..43cd0014864 100755
--- a/tests/queries/0_stateless/01801_nullable_low_cardinality_tsv.sh
+++ b/tests/queries/0_stateless/01801_nullable_low_cardinality_tsv.sh
@@ -5,7 +5,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CUR_DIR"/../shell_config.sh
 
 $CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS nullable_low_cardinality_tsv_test;";
-$CLICKHOUSE_CLIENT --multiquery --query="CREATE TABLE nullable_low_cardinality_tsv_test
+$CLICKHOUSE_CLIENT --query="CREATE TABLE nullable_low_cardinality_tsv_test
 (
     A Date,
     S LowCardinality(Nullable(String)),
diff --git a/tests/queries/0_stateless/01834_alias_columns_laziness_filimonov.sh b/tests/queries/0_stateless/01834_alias_columns_laziness_filimonov.sh
index bf8bb716a9f..7b8d3ce1504 100755
--- a/tests/queries/0_stateless/01834_alias_columns_laziness_filimonov.sh
+++ b/tests/queries/0_stateless/01834_alias_columns_laziness_filimonov.sh
@@ -4,7 +4,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CUR_DIR"/../shell_config.sh
 
-${CLICKHOUSE_CLIENT} --multiquery --query "
+${CLICKHOUSE_CLIENT} --query "
 drop table if exists aliases_lazyness;
 create table aliases_lazyness (x UInt32, y ALIAS sleepEachRow(0.1)) Engine=MergeTree ORDER BY x;
 insert into aliases_lazyness(x) select * from numbers(100);
diff --git a/tests/queries/0_stateless/01923_network_receive_time_metric_insert.sh b/tests/queries/0_stateless/01923_network_receive_time_metric_insert.sh
index a6b83eba27d..97835d97965 100755
--- a/tests/queries/0_stateless/01923_network_receive_time_metric_insert.sh
+++ b/tests/queries/0_stateless/01923_network_receive_time_metric_insert.sh
@@ -6,13 +6,13 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
-${CLICKHOUSE_CLIENT} --multiquery --query "DROP TABLE IF EXISTS t; CREATE TABLE t (x UInt64) ENGINE = Memory;"
+${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS t; CREATE TABLE t (x UInt64) ENGINE = Memory;"
 
 # Rate limit is chosen for operation to spent more than one second.
 seq 1 1000 | pv --quiet --rate-limit 400 | ${CLICKHOUSE_CLIENT} --query "INSERT INTO t FORMAT TSV"
 
 # We check that the value of NetworkReceiveElapsedMicroseconds correctly includes the time spent waiting data from the client.
-${CLICKHOUSE_CLIENT} --multiquery --query "SYSTEM FLUSH LOGS;
+${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS;
     WITH ProfileEvents['NetworkReceiveElapsedMicroseconds'] AS time
     SELECT time >= 1000000 ? 1 : time FROM system.query_log
         WHERE current_database = currentDatabase() AND query_kind = 'Insert' AND event_date >= yesterday() AND type = 2 ORDER BY event_time DESC LIMIT 1;"
diff --git a/tests/queries/0_stateless/01939_network_receive_bytes_metrics.sh b/tests/queries/0_stateless/01939_network_receive_bytes_metrics.sh
index 03babad40f3..b2335a0365b 100755
--- a/tests/queries/0_stateless/01939_network_receive_bytes_metrics.sh
+++ b/tests/queries/0_stateless/01939_network_receive_bytes_metrics.sh
@@ -4,11 +4,11 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
-${CLICKHOUSE_CLIENT} --multiquery --query "DROP TABLE IF EXISTS t; CREATE TABLE t (x UInt64) ENGINE = Memory;"
+${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS t; CREATE TABLE t (x UInt64) ENGINE = Memory;"
 
 seq 1 1000 | ${CLICKHOUSE_CLIENT} --query "INSERT INTO t FORMAT TSV"
 
-${CLICKHOUSE_CLIENT} --multiquery --query "SYSTEM FLUSH LOGS;
+${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS;
     WITH ProfileEvents['NetworkReceiveBytes'] AS bytes
     SELECT bytes >= 8000 AND bytes < 9000 ? 1 : bytes FROM system.query_log
         WHERE current_database = currentDatabase() AND query_kind = 'Insert' AND event_date >= yesterday() AND type = 2 ORDER BY event_time DESC LIMIT 1;"
diff --git a/tests/queries/0_stateless/01946_test_wrong_host_name_access.sh b/tests/queries/0_stateless/01946_test_wrong_host_name_access.sh
index a00f307673e..ed2828c3f54 100755
--- a/tests/queries/0_stateless/01946_test_wrong_host_name_access.sh
+++ b/tests/queries/0_stateless/01946_test_wrong_host_name_access.sh
@@ -7,7 +7,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CUR_DIR"/../shell_config.sh
 
-${CLICKHOUSE_CLIENT} --multiquery --query "
+${CLICKHOUSE_CLIENT} --query "
     DROP USER IF EXISTS dns_fail_1, dns_fail_2;
     CREATE USER dns_fail_1 HOST NAME 'non.existing.host.name', '${MYHOSTNAME}';
     CREATE USER dns_fail_2 HOST NAME '${MYHOSTNAME}', 'non.existing.host.name';"
diff --git a/tests/queries/0_stateless/01946_test_zstd_decompression_with_escape_sequence_at_the_end_of_buffer.sh b/tests/queries/0_stateless/01946_test_zstd_decompression_with_escape_sequence_at_the_end_of_buffer.sh
index 0aedef028a2..b3748581f4f 100755
--- a/tests/queries/0_stateless/01946_test_zstd_decompression_with_escape_sequence_at_the_end_of_buffer.sh
+++ b/tests/queries/0_stateless/01946_test_zstd_decompression_with_escape_sequence_at_the_end_of_buffer.sh
@@ -9,7 +9,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 mkdir -p ${USER_FILES_PATH}/
 cp $CUR_DIR/data_zstd/test_01946.zstd ${USER_FILES_PATH}/
 
-${CLICKHOUSE_CLIENT} --multiline --multiquery --query "
+${CLICKHOUSE_CLIENT} --multiline --query "
 set min_chunk_bytes_for_parallel_parsing=10485760;
 set max_read_buffer_size = 65536;
 set input_format_parallel_parsing = 0;
diff --git a/tests/queries/0_stateless/02009_from_infile.sh b/tests/queries/0_stateless/02009_from_infile.sh
index 6a31aa4ac55..578ac14f558 100755
--- a/tests/queries/0_stateless/02009_from_infile.sh
+++ b/tests/queries/0_stateless/02009_from_infile.sh
@@ -19,7 +19,7 @@ ${CLICKHOUSE_CLIENT} --query "INSERT INTO test_infile FROM INFILE '${CLICKHOUSE_
 ${CLICKHOUSE_CLIENT} --query "SELECT * FROM test_infile;"
 
 # if it not fails, select will print information
-${CLICKHOUSE_LOCAL} --multiquery --query "CREATE TABLE test_infile (word String) ENGINE=Memory(); INSERT INTO test_infile FROM INFILE '${CLICKHOUSE_TMP}/test_infile.gz' FORMAT CSV; SELECT * from test_infile;"
+${CLICKHOUSE_LOCAL} --query "CREATE TABLE test_infile (word String) ENGINE=Memory(); INSERT INTO test_infile FROM INFILE '${CLICKHOUSE_TMP}/test_infile.gz' FORMAT CSV; SELECT * from test_infile;"
 
 ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=DROP+TABLE" -d 'IF EXISTS test_infile_url'
 ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=CREATE" -d 'TABLE test_infile_url (x String) ENGINE = Memory'
diff --git a/tests/queries/0_stateless/02024_compression_in_query.sh b/tests/queries/0_stateless/02024_compression_in_query.sh
index caa74523bd7..2936568c991 100755
--- a/tests/queries/0_stateless/02024_compression_in_query.sh
+++ b/tests/queries/0_stateless/02024_compression_in_query.sh
@@ -55,8 +55,8 @@ ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS test_compression_keyword;"
 [ -e "${CLICKHOUSE_TMP}"/test_comp_for_input_and_output_without_gz.gz ] && rm "${CLICKHOUSE_TMP}"/test_comp_for_input_and_output_without_gz.gz
 
 # create files using compression method and without it to check that both queries work correct
-${CLICKHOUSE_LOCAL} --multiquery --query "SELECT * FROM (SELECT 'Hello, World! From local.') INTO OUTFILE '${CLICKHOUSE_TMP}/test_comp_for_input_and_output.gz' FORMAT TabSeparated;"
-${CLICKHOUSE_LOCAL} --multiquery --query "SELECT * FROM (SELECT 'Hello, World! From local.') INTO OUTFILE '${CLICKHOUSE_TMP}/test_comp_for_input_and_output_without_gz' COMPRESSION 'GZ' FORMAT TabSeparated;"
+${CLICKHOUSE_LOCAL} --query "SELECT * FROM (SELECT 'Hello, World! From local.') INTO OUTFILE '${CLICKHOUSE_TMP}/test_comp_for_input_and_output.gz' FORMAT TabSeparated;"
+${CLICKHOUSE_LOCAL} --query "SELECT * FROM (SELECT 'Hello, World! From local.') INTO OUTFILE '${CLICKHOUSE_TMP}/test_comp_for_input_and_output_without_gz' COMPRESSION 'GZ' FORMAT TabSeparated;"
 
 # check content of files
 cp ${CLICKHOUSE_TMP}/test_comp_for_input_and_output.gz ${CLICKHOUSE_TMP}/test_comp_for_input_and_output_to_decomp.gz
@@ -68,7 +68,7 @@ gunzip ${CLICKHOUSE_TMP}/test_comp_for_input_and_output_without_gz_to_decomp.gz
 cat ${CLICKHOUSE_TMP}/test_comp_for_input_and_output_without_gz_to_decomp
 
 # create table to check inserts
-${CLICKHOUSE_LOCAL} --multiquery --query "
+${CLICKHOUSE_LOCAL} --query "
 DROP TABLE IF EXISTS test_compression_keyword;
 CREATE TABLE test_compression_keyword (text String) Engine=Memory;
 INSERT INTO TABLE test_compression_keyword FROM INFILE '${CLICKHOUSE_TMP}/test_comp_for_input_and_output.gz' FORMAT TabSeparated;
diff --git a/tests/queries/0_stateless/02048_parallel_reading_from_infile.sh b/tests/queries/0_stateless/02048_parallel_reading_from_infile.sh
index f055ea304b2..efc19cad054 100755
--- a/tests/queries/0_stateless/02048_parallel_reading_from_infile.sh
+++ b/tests/queries/0_stateless/02048_parallel_reading_from_infile.sh
@@ -17,7 +17,7 @@ echo -e "103" > "${CLICKHOUSE_TMP}"/test_infile_parallel_3
 
 gzip "${CLICKHOUSE_TMP}"/test_infile_parallel
 
-${CLICKHOUSE_CLIENT} --multiquery  <<EOF
+${CLICKHOUSE_CLIENT} <<EOF
 DROP TABLE IF EXISTS test_infile_parallel;
 CREATE TABLE test_infile_parallel (Id Int32,Value Enum('first' = 1, 'second' = 2)) ENGINE=Memory();
 SET input_format_allow_errors_num=1;
@@ -27,13 +27,13 @@ SELECT count() FROM test_infile_parallel WHERE Value='second';
 EOF
 
 # Error code is 27 (DB::ParsingException). It is not ignored.
-${CLICKHOUSE_CLIENT}  -m --multiquery --query "DROP TABLE IF EXISTS test_infile_parallel;
+${CLICKHOUSE_CLIENT}  -m --query "DROP TABLE IF EXISTS test_infile_parallel;
 CREATE TABLE test_infile_parallel (Id Int32,Value Enum('first' = 1, 'second' = 2)) ENGINE=Memory();
 SET input_format_allow_errors_num=0;
 INSERT INTO test_infile_parallel FROM INFILE '${CLICKHOUSE_TMP}/test_infile_parallel*' FORMAT TSV;
 " 2>&1 | grep -q "27" && echo "Correct" || echo 'Fail'
 
-${CLICKHOUSE_LOCAL} --multiquery <<EOF
+${CLICKHOUSE_LOCAL} <<EOF
 DROP TABLE IF EXISTS test_infile_parallel; 
 SET input_format_allow_errors_num=1;
 CREATE TABLE test_infile_parallel (Id Int32,Value Enum('first' = 1, 'second' = 2)) ENGINE=Memory(); 
diff --git a/tests/queries/0_stateless/02104_clickhouse_local_columns_description.sh b/tests/queries/0_stateless/02104_clickhouse_local_columns_description.sh
index f0ea5b4415c..f88a8de80f5 100755
--- a/tests/queries/0_stateless/02104_clickhouse_local_columns_description.sh
+++ b/tests/queries/0_stateless/02104_clickhouse_local_columns_description.sh
@@ -4,4 +4,4 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CUR_DIR"/../shell_config.sh
 
-${CLICKHOUSE_LOCAL} --multiquery --query "create table t (n int, m int default 42) engine=Memory;insert into t values (1, NULL);select * from t"
+${CLICKHOUSE_LOCAL} --query "create table t (n int, m int default 42) engine=Memory;insert into t values (1, NULL);select * from t"
diff --git a/tests/queries/0_stateless/02124_insert_deduplication_token_multiple_blocks.sh b/tests/queries/0_stateless/02124_insert_deduplication_token_multiple_blocks.sh
index 9a7ac2007f1..ca7ac0d51fe 100755
--- a/tests/queries/0_stateless/02124_insert_deduplication_token_multiple_blocks.sh
+++ b/tests/queries/0_stateless/02124_insert_deduplication_token_multiple_blocks.sh
@@ -14,24 +14,24 @@ $CLICKHOUSE_CLIENT --query="SYSTEM STOP MERGES block_dedup_token;"
 $CLICKHOUSE_CLIENT --query="SELECT 'insert 2 blocks with dedup token, 1 row per block'"
 DEDUP_TOKEN='dedup1'
 echo 'INSERT INTO block_dedup_token VALUES (1), (2)' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&$INSERT_BLOCK_SETTINGS&insert_deduplication_token='$DEDUP_TOKEN'&query=" --data-binary @-
-$CLICKHOUSE_CLIENT --multiquery --query "$QUERY_COUNT_ORIGIN_BLOCKS;$QUERY_SELECT_FROM_TABLE_ORDERED"
+$CLICKHOUSE_CLIENT --query "$QUERY_COUNT_ORIGIN_BLOCKS;$QUERY_SELECT_FROM_TABLE_ORDERED"
 
 $CLICKHOUSE_CLIENT --query="SELECT 'insert deduplicated by token'"
 echo 'INSERT INTO block_dedup_token VALUES (1), (2)' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&$INSERT_BLOCK_SETTINGS&insert_deduplication_token='$DEDUP_TOKEN'&query=" --data-binary @-
-$CLICKHOUSE_CLIENT --multiquery --query "$QUERY_COUNT_ORIGIN_BLOCKS;$QUERY_SELECT_FROM_TABLE_ORDERED"
+$CLICKHOUSE_CLIENT --query "$QUERY_COUNT_ORIGIN_BLOCKS;$QUERY_SELECT_FROM_TABLE_ORDERED"
 
 $CLICKHOUSE_CLIENT --query="SELECT 'insert the same data by providing different dedup token'"
 DEDUP_TOKEN='dedup2'
 echo 'INSERT INTO block_dedup_token VALUES (1), (2)' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&$INSERT_BLOCK_SETTINGS&insert_deduplication_token='$DEDUP_TOKEN'&query=" --data-binary @-
-$CLICKHOUSE_CLIENT --multiquery --query "$QUERY_COUNT_ORIGIN_BLOCKS;$QUERY_SELECT_FROM_TABLE_ORDERED"
+$CLICKHOUSE_CLIENT --query "$QUERY_COUNT_ORIGIN_BLOCKS;$QUERY_SELECT_FROM_TABLE_ORDERED"
 
 $CLICKHOUSE_CLIENT --query="SELECT 'insert 4 blocks, 2 deduplicated, 2 inserted'"
 echo 'INSERT INTO block_dedup_token VALUES (1), (2), (3), (4)' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&$INSERT_BLOCK_SETTINGS&insert_deduplication_token='$DEDUP_TOKEN'&query=" --data-binary @-
-$CLICKHOUSE_CLIENT --multiquery --query "$QUERY_COUNT_ORIGIN_BLOCKS;$QUERY_SELECT_FROM_TABLE_ORDERED"
+$CLICKHOUSE_CLIENT --query "$QUERY_COUNT_ORIGIN_BLOCKS;$QUERY_SELECT_FROM_TABLE_ORDERED"
 
 $CLICKHOUSE_CLIENT --query="SELECT 'disable token based deduplication, insert the same data as with token'"
 DEDUP_TOKEN=''
 echo 'INSERT INTO block_dedup_token VALUES (1), (2), (3), (4)' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&$INSERT_BLOCK_SETTINGS&insert_deduplication_token='$DEDUP_TOKEN'&query=" --data-binary @-
-$CLICKHOUSE_CLIENT --multiquery --query "$QUERY_COUNT_ORIGIN_BLOCKS;$QUERY_SELECT_FROM_TABLE_ORDERED"
+$CLICKHOUSE_CLIENT --query "$QUERY_COUNT_ORIGIN_BLOCKS;$QUERY_SELECT_FROM_TABLE_ORDERED"
 
 $CLICKHOUSE_CLIENT --query="DROP TABLE block_dedup_token SYNC"
diff --git a/tests/queries/0_stateless/02124_insert_deduplication_token_multiple_blocks_replica.sh b/tests/queries/0_stateless/02124_insert_deduplication_token_multiple_blocks_replica.sh
index 0c95abb9867..c4ce97aad40 100755
--- a/tests/queries/0_stateless/02124_insert_deduplication_token_multiple_blocks_replica.sh
+++ b/tests/queries/0_stateless/02124_insert_deduplication_token_multiple_blocks_replica.sh
@@ -15,24 +15,24 @@ $CLICKHOUSE_CLIENT --query="SYSTEM STOP MERGES block_dedup_token_replica"
 $CLICKHOUSE_CLIENT --query="SELECT 'insert 2 blocks with dedup token, 1 row per block'"
 DEDUP_TOKEN='dedup1'
 echo 'INSERT INTO block_dedup_token_replica VALUES (1), (2)' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&$INSERT_BLOCK_SETTINGS&insert_deduplication_token='$DEDUP_TOKEN'&query=" --data-binary @-
-$CLICKHOUSE_CLIENT --multiquery --query "$QUERY_COUNT_ORIGIN_BLOCKS;$QUERY_SELECT_FROM_TABLE_ORDERED"
+$CLICKHOUSE_CLIENT --query "$QUERY_COUNT_ORIGIN_BLOCKS;$QUERY_SELECT_FROM_TABLE_ORDERED"
 
 $CLICKHOUSE_CLIENT --query="SELECT 'insert deduplicated by token'"
 echo 'INSERT INTO block_dedup_token_replica VALUES (1), (2)' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&$INSERT_BLOCK_SETTINGS&insert_deduplication_token='$DEDUP_TOKEN'&query=" --data-binary @-
-$CLICKHOUSE_CLIENT --multiquery --query "$QUERY_COUNT_ORIGIN_BLOCKS;$QUERY_SELECT_FROM_TABLE_ORDERED"
+$CLICKHOUSE_CLIENT --query "$QUERY_COUNT_ORIGIN_BLOCKS;$QUERY_SELECT_FROM_TABLE_ORDERED"
 
 $CLICKHOUSE_CLIENT --query="SELECT 'insert the same data by providing different dedup token'"
 DEDUP_TOKEN='dedup2'
 echo 'INSERT INTO block_dedup_token_replica VALUES (1), (2)' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&$INSERT_BLOCK_SETTINGS&insert_deduplication_token='$DEDUP_TOKEN'&query=" --data-binary @-
-$CLICKHOUSE_CLIENT --multiquery --query "$QUERY_COUNT_ORIGIN_BLOCKS;$QUERY_SELECT_FROM_TABLE_ORDERED"
+$CLICKHOUSE_CLIENT --query "$QUERY_COUNT_ORIGIN_BLOCKS;$QUERY_SELECT_FROM_TABLE_ORDERED"
 
 $CLICKHOUSE_CLIENT --query="SELECT 'insert 4 blocks, 2 deduplicated, 2 inserted'"
 echo 'INSERT INTO block_dedup_token_replica VALUES (1), (2), (3), (4)' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&$INSERT_BLOCK_SETTINGS&insert_deduplication_token='$DEDUP_TOKEN'&query=" --data-binary @-
-$CLICKHOUSE_CLIENT --multiquery --query "$QUERY_COUNT_ORIGIN_BLOCKS;$QUERY_SELECT_FROM_TABLE_ORDERED"
+$CLICKHOUSE_CLIENT --query "$QUERY_COUNT_ORIGIN_BLOCKS;$QUERY_SELECT_FROM_TABLE_ORDERED"
 
 $CLICKHOUSE_CLIENT --query="SELECT 'disable token based deduplication, insert the same data as with token'"
 DEDUP_TOKEN=''
 echo 'INSERT INTO block_dedup_token_replica VALUES (1), (2), (3), (4)' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&$INSERT_BLOCK_SETTINGS&insert_deduplication_token='$DEDUP_TOKEN'&query=" --data-binary @-
-$CLICKHOUSE_CLIENT --multiquery --query "$QUERY_COUNT_ORIGIN_BLOCKS;$QUERY_SELECT_FROM_TABLE_ORDERED"
+$CLICKHOUSE_CLIENT --query "$QUERY_COUNT_ORIGIN_BLOCKS;$QUERY_SELECT_FROM_TABLE_ORDERED"
 
 $CLICKHOUSE_CLIENT --query="DROP TABLE block_dedup_token_replica SYNC"
diff --git a/tests/queries/0_stateless/02125_many_mutations.sh b/tests/queries/0_stateless/02125_many_mutations.sh
index 239f30811d4..9b0afaca585 100755
--- a/tests/queries/0_stateless/02125_many_mutations.sh
+++ b/tests/queries/0_stateless/02125_many_mutations.sh
@@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
 # "max_parts_to_merge_at_once = 1" prevents merges to start in background before our own OPTIMIZE FINAL
 
-$CLICKHOUSE_CLIENT --multiquery -q "
+$CLICKHOUSE_CLIENT -q "
 drop table if exists many_mutations;
 create table many_mutations (x UInt32, y UInt32) engine = MergeTree order by x settings number_of_mutations_to_delay = 0, number_of_mutations_to_throw = 0, max_parts_to_merge_at_once = 1;
 insert into many_mutations values (0, 0), (1, 1);
@@ -17,7 +17,7 @@ select x, y from many_mutations order by x;
 
 job()
 {
-   yes "alter table many_mutations update y = y + 1 where 1;" | head -n 1000 | $CLICKHOUSE_CLIENT --multiquery
+   yes "alter table many_mutations update y = y + 1 where 1;" | head -n 1000 | $CLICKHOUSE_CLIENT
 }
 
 job &
@@ -44,7 +44,7 @@ job &
 wait
 
 # truncate before drop, avoid removing all the mutations (it's slow) in DatabaseCatalog's thread (may affect other tests)
-$CLICKHOUSE_CLIENT --multiquery -q "
+$CLICKHOUSE_CLIENT -q "
 select count() from system.mutations where database = currentDatabase() and table = 'many_mutations' and not is_done;
 system start merges many_mutations;
 optimize table many_mutations final SETTINGS optimize_throw_if_noop = 1;
diff --git a/tests/queries/0_stateless/02125_many_mutations_2.sh b/tests/queries/0_stateless/02125_many_mutations_2.sh
index 958f50287b2..d0025e6d8cc 100755
--- a/tests/queries/0_stateless/02125_many_mutations_2.sh
+++ b/tests/queries/0_stateless/02125_many_mutations_2.sh
@@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
 # "max_parts_to_merge_at_once = 1" prevents merges to start in background before our own OPTIMIZE FINAL
 
-$CLICKHOUSE_CLIENT --multiquery -q "
+$CLICKHOUSE_CLIENT -q "
 drop table if exists many_mutations;
 create table many_mutations (x UInt32, y UInt32) engine = MergeTree order by x settings number_of_mutations_to_delay = 0, number_of_mutations_to_throw = 0, max_parts_to_merge_at_once = 1;
 insert into many_mutations select number, number + 1 from numbers(2000);
@@ -21,7 +21,7 @@ job()
    for i in {1..1000}
    do
       echo "alter table many_mutations delete where y = ${i} * 2 settings mutations_sync = 0;"
-   done | $CLICKHOUSE_CLIENT --multiquery
+   done | $CLICKHOUSE_CLIENT
 }
 
 job &
@@ -48,7 +48,7 @@ job &
 wait
 
 # truncate before drop, avoid removing all the mutations (it's slow) in DatabaseCatalog's thread (may affect other tests)
-$CLICKHOUSE_CLIENT --multiquery -q "
+$CLICKHOUSE_CLIENT -q "
 select count() from system.mutations where database = currentDatabase() and table = 'many_mutations' and not is_done;
 system start merges many_mutations;
 optimize table many_mutations final SETTINGS optimize_throw_if_noop = 1;
diff --git a/tests/queries/0_stateless/02135_local_create_db.sh b/tests/queries/0_stateless/02135_local_create_db.sh
index 55c5c85468f..5004cd35f99 100755
--- a/tests/queries/0_stateless/02135_local_create_db.sh
+++ b/tests/queries/0_stateless/02135_local_create_db.sh
@@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CURDIR"/../shell_config.sh
 
 for Engine in Atomic Ordinary; do
-    $CLICKHOUSE_LOCAL --allow_deprecated_database_ordinary=1 --multiquery --query """
+    $CLICKHOUSE_LOCAL --allow_deprecated_database_ordinary=1 --query """
     CREATE DATABASE foo_$Engine Engine=$Engine;
     DROP DATABASE foo_$Engine;
     """
diff --git a/tests/queries/0_stateless/02151_client_option_echo.sh b/tests/queries/0_stateless/02151_client_option_echo.sh
index 8056b3b5ed1..565a3a16fe9 100755
--- a/tests/queries/0_stateless/02151_client_option_echo.sh
+++ b/tests/queries/0_stateless/02151_client_option_echo.sh
@@ -11,6 +11,6 @@ ${CLICKHOUSE_CLIENT} --echo --query="DROP TABLE IF EXISTS echo_test_0"
 # single query echo off
 ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS echo_test_1"
 # multi query echo on
-${CLICKHOUSE_CLIENT} --echo --multiquery --query="DROP TABLE IF EXISTS echo_test_2;DROP TABLE IF EXISTS echo_test_3"
+${CLICKHOUSE_CLIENT} --echo --query="DROP TABLE IF EXISTS echo_test_2;DROP TABLE IF EXISTS echo_test_3"
 # multi query echo off
-${CLICKHOUSE_CLIENT} --multiquery --query="DROP TABLE IF EXISTS echo_test_4;DROP TABLE IF EXISTS echo_test_5"
+${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS echo_test_4;DROP TABLE IF EXISTS echo_test_5"
diff --git a/tests/queries/0_stateless/02151_hash_table_sizes_stats.sh b/tests/queries/0_stateless/02151_hash_table_sizes_stats.sh
index f99dbdacec2..b679c962232 100755
--- a/tests/queries/0_stateless/02151_hash_table_sizes_stats.sh
+++ b/tests/queries/0_stateless/02151_hash_table_sizes_stats.sh
@@ -35,7 +35,7 @@ prepare_table_with_sorting_key() {
 
 run_query() {
   query_id="${CLICKHOUSE_DATABASE}_hash_table_sizes_stats_$RANDOM$RANDOM"
-  $CLICKHOUSE_CLIENT --query_id="$query_id" --multiquery -q "
+  $CLICKHOUSE_CLIENT --query_id="$query_id" -q "
     SET max_block_size = $((table_size / 10));
     SET merge_tree_min_rows_for_concurrent_read = 1;
     SET max_untracked_memory = 0;
diff --git a/tests/queries/0_stateless/02151_hash_table_sizes_stats_distributed.sh b/tests/queries/0_stateless/02151_hash_table_sizes_stats_distributed.sh
index 056c176c1ff..67afa26d02a 100755
--- a/tests/queries/0_stateless/02151_hash_table_sizes_stats_distributed.sh
+++ b/tests/queries/0_stateless/02151_hash_table_sizes_stats_distributed.sh
@@ -40,7 +40,7 @@ prepare_table_with_sorting_key() {
 
 run_query() {
   query_id="${CLICKHOUSE_DATABASE}_hash_table_sizes_stats_$RANDOM$RANDOM"
-  $CLICKHOUSE_CLIENT --query_id="$query_id" --multiquery -q "
+  $CLICKHOUSE_CLIENT --query_id="$query_id" -q "
     SET max_block_size = $((table_size / 10));
     SET merge_tree_min_rows_for_concurrent_read = 1;
     SET max_untracked_memory = 0;
diff --git a/tests/queries/0_stateless/02158_explain_ast_alter_commands.sh b/tests/queries/0_stateless/02158_explain_ast_alter_commands.sh
index 8dfb61eedfb..7a504ab1d31 100755
--- a/tests/queries/0_stateless/02158_explain_ast_alter_commands.sh
+++ b/tests/queries/0_stateless/02158_explain_ast_alter_commands.sh
@@ -47,4 +47,4 @@ EXPLAIN AST ALTER TABLE t UPDATE c = 1 WHERE 1;
 EXPLAIN AST ALTER TABLE t UPDATE c = 1 IN PARTITION 'p' WHERE 1;
 EXPLAIN AST ALTER TABLE t DELETE WHERE c = 1;
 EXPLAIN AST ALTER TABLE t DELETE IN PARTITION 'p' WHERE c = 1;" | \
-    $CLICKHOUSE_CLIENT --readonly 1 --multiquery 2>&1 | grep 'AlterCommand'
+    $CLICKHOUSE_CLIENT --readonly 1 2>&1 | grep 'AlterCommand'
diff --git a/tests/queries/0_stateless/02206_clickhouse_local_use_database.sh b/tests/queries/0_stateless/02206_clickhouse_local_use_database.sh
index 59ede739e4a..3b71c8754c9 100755
--- a/tests/queries/0_stateless/02206_clickhouse_local_use_database.sh
+++ b/tests/queries/0_stateless/02206_clickhouse_local_use_database.sh
@@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CURDIR"/../shell_config.sh
 
 
-$CLICKHOUSE_LOCAL --echo --multiline --multiquery -q """
+$CLICKHOUSE_LOCAL --echo --multiline -q """
 SHOW TABLES;
 CREATE DATABASE test1;
 CREATE TABLE test1.table1 (a Int32) ENGINE=Memory;
diff --git a/tests/queries/0_stateless/02226_filesystem_cache_profile_events.sh b/tests/queries/0_stateless/02226_filesystem_cache_profile_events.sh
index d0e61541b15..18ae2d7b4b3 100755
--- a/tests/queries/0_stateless/02226_filesystem_cache_profile_events.sh
+++ b/tests/queries/0_stateless/02226_filesystem_cache_profile_events.sh
@@ -10,7 +10,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do
     echo "Using storage policy: $STORAGE_POLICY"
 
-    $CLICKHOUSE_CLIENT --multiquery --multiline  --query """
+    $CLICKHOUSE_CLIENT --multiline  --query """
     SET max_memory_usage='20G';
     SET enable_filesystem_cache_on_write_operations = 0;
 
@@ -25,7 +25,7 @@ for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do
 
     query_id=$($CLICKHOUSE_CLIENT --query "select queryID() from ($query) limit 1" 2>&1)
 
-    $CLICKHOUSE_CLIENT --multiquery --multiline  --query """
+    $CLICKHOUSE_CLIENT --multiline  --query """
     SYSTEM FLUSH LOGS;
     SELECT ProfileEvents['CachedReadBufferReadFromCacheHits'] > 0 as remote_fs_cache_hit,
            ProfileEvents['CachedReadBufferReadFromCacheMisses'] > 0 as remote_fs_cache_miss,
@@ -40,14 +40,14 @@ for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do
     LIMIT 1;
     """
 
-    $CLICKHOUSE_CLIENT --multiquery --multiline --query """
+    $CLICKHOUSE_CLIENT --multiline --query """
     set remote_filesystem_read_method = 'read';
     set local_filesystem_read_method = 'pread';
     """
 
     query_id=$($CLICKHOUSE_CLIENT --query "select queryID() from ($query) limit 1" 2>&1)
 
-    $CLICKHOUSE_CLIENT --multiquery --multiline  --query """
+    $CLICKHOUSE_CLIENT --multiline  --query """
     SYSTEM FLUSH LOGS;
     SELECT ProfileEvents['CachedReadBufferReadFromCacheHits'] > 0 as remote_fs_cache_hit,
            ProfileEvents['CachedReadBufferReadFromCacheMisses'] > 0 as remote_fs_cache_miss,
@@ -63,13 +63,13 @@ for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do
     """
 
 
-    $CLICKHOUSE_CLIENT --multiquery --multiline --query """
+    $CLICKHOUSE_CLIENT --multiline --query """
     set remote_filesystem_read_method='threadpool';
     """
 
     query_id=$($CLICKHOUSE_CLIENT --query "select queryID() from ($query) limit 1")
 
-    $CLICKHOUSE_CLIENT --multiquery --multiline  --query """
+    $CLICKHOUSE_CLIENT --multiline  --query """
     SYSTEM FLUSH LOGS;
     SELECT ProfileEvents['CachedReadBufferReadFromCacheHits'] > 0 as remote_fs_cache_hit,
            ProfileEvents['CachedReadBufferReadFromCacheMisses'] > 0 as remote_fs_cache_miss,
@@ -84,7 +84,7 @@ for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do
     LIMIT 1;
     """
 
-    $CLICKHOUSE_CLIENT --multiquery --multiline  --query """
+    $CLICKHOUSE_CLIENT --multiline  --query """
     SELECT * FROM test_02226 WHERE value LIKE '%abc%' ORDER BY value LIMIT 10 FORMAT Null;
 
     SET enable_filesystem_cache_on_write_operations = 1;
diff --git a/tests/queries/0_stateless/02227_test_create_empty_sqlite_db.sh b/tests/queries/0_stateless/02227_test_create_empty_sqlite_db.sh
index 344452767cc..a3fe5f19de0 100755
--- a/tests/queries/0_stateless/02227_test_create_empty_sqlite_db.sh
+++ b/tests/queries/0_stateless/02227_test_create_empty_sqlite_db.sh
@@ -17,7 +17,7 @@ export CURR_DATABASE="test_01889_sqllite_${CLICKHOUSE_DATABASE}"
 
 DB_PATH=${USER_FILES_PATH}/${CURR_DATABASE}_db1
 
-${CLICKHOUSE_CLIENT} --multiquery --multiline --query="""
+${CLICKHOUSE_CLIENT} --multiline --query="""
 DROP DATABASE IF EXISTS ${CURR_DATABASE};
 CREATE DATABASE ${CURR_DATABASE} ENGINE = SQLite('${DB_PATH}');
 SHOW TABLES FROM ${CURR_DATABASE};
@@ -25,6 +25,6 @@ SHOW TABLES FROM ${CURR_DATABASE};
 
 sqlite3 "${DB_PATH}" 'CREATE TABLE table1 (col1 text, col2 smallint);'
 
-${CLICKHOUSE_CLIENT} --multiquery --multiline --query="""
+${CLICKHOUSE_CLIENT} --multiline --query="""
 SHOW TABLES FROM ${CURR_DATABASE};
 """
diff --git a/tests/queries/0_stateless/02235_remote_fs_cache_stress.sh b/tests/queries/0_stateless/02235_remote_fs_cache_stress.sh
index ffc38c0c1bd..aa5db33417c 100755
--- a/tests/queries/0_stateless/02235_remote_fs_cache_stress.sh
+++ b/tests/queries/0_stateless/02235_remote_fs_cache_stress.sh
@@ -6,7 +6,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CUR_DIR"/../shell_config.sh
 
 
-${CLICKHOUSE_CLIENT} --allow_suspicious_low_cardinality_types=1 --multiquery --multiline --query="""
+${CLICKHOUSE_CLIENT} --allow_suspicious_low_cardinality_types=1 --multiline --query="""
 
 DROP TABLE IF EXISTS t_01411;
 DROP TABLE IF EXISTS t_01411_num;
@@ -35,7 +35,7 @@ insert into lc_dict_reading select number, if(number < 8192 * 4, number % 100, n
 function go()
 {
 
-${CLICKHOUSE_CLIENT} --multiquery --multiline --query="""
+${CLICKHOUSE_CLIENT} --multiline --query="""
 
 select sum(toUInt64(str)), sum(toUInt64(pat)) from lc_dict_reading where val < 8129 or val > 8192 * 4;
 
@@ -67,7 +67,7 @@ for _ in `seq 1 32`; do go | grep -q "Exception" && echo 'FAIL' || echo 'OK' ||:
 
 wait
 
-${CLICKHOUSE_CLIENT} --multiquery --multiline --query="""
+${CLICKHOUSE_CLIENT} --multiline --query="""
 DROP TABLE IF EXISTS t_01411;
 DROP TABLE IF EXISTS t_01411_num;
 """
diff --git a/tests/queries/0_stateless/02240_protobuflist_format_persons.sh b/tests/queries/0_stateless/02240_protobuflist_format_persons.sh
index 637e01b9e63..e5e717d00a8 100755
--- a/tests/queries/0_stateless/02240_protobuflist_format_persons.sh
+++ b/tests/queries/0_stateless/02240_protobuflist_format_persons.sh
@@ -15,7 +15,7 @@ SCHEMADIR=$CURDIR/format_schemas
 set -eo pipefail
 
 # Run the client.
-$CLICKHOUSE_CLIENT --multiquery <<EOF
+$CLICKHOUSE_CLIENT <<EOF
 DROP TABLE IF EXISTS persons_02240;
 DROP TABLE IF EXISTS roundtrip_persons_02240;
 DROP TABLE IF EXISTS alt_persons_02240;
@@ -118,7 +118,7 @@ $CLICKHOUSE_CLIENT --query "INSERT INTO syntax2_persons_02240 SETTINGS format_sc
 $CLICKHOUSE_CLIENT --query "SELECT * FROM syntax2_persons_02240 ORDER BY name"
 rm "$BINARY_FILE_PATH"
 
-$CLICKHOUSE_CLIENT --multiquery <<EOF
+$CLICKHOUSE_CLIENT <<EOF
 DROP TABLE persons_02240;
 DROP TABLE roundtrip_persons_02240;
 DROP TABLE alt_persons_02240;
diff --git a/tests/queries/0_stateless/02246_clickhouse_local_drop_database.sh b/tests/queries/0_stateless/02246_clickhouse_local_drop_database.sh
index 7e25e099626..b653c9f3e94 100755
--- a/tests/queries/0_stateless/02246_clickhouse_local_drop_database.sh
+++ b/tests/queries/0_stateless/02246_clickhouse_local_drop_database.sh
@@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 dir=${CLICKHOUSE_TEST_UNIQUE_NAME}
 [[ -d $dir ]] && rm -r $dir
 mkdir $dir
-$CLICKHOUSE_LOCAL --multiline --multiquery --path $dir -q """
+$CLICKHOUSE_LOCAL --multiline --path $dir -q """
 DROP DATABASE IF EXISTS test;
 CREATE DATABASE IF NOT EXISTS test;
 USE test;
@@ -15,7 +15,7 @@ CREATE TABLE test (id Int32) ENGINE=MergeTree() ORDER BY id;
 DROP DATABASE test;
 """
 
-$CLICKHOUSE_LOCAL --multiline --multiquery -q """
+$CLICKHOUSE_LOCAL --multiline -q """
 DROP DATABASE IF EXISTS test;
 CREATE DATABASE IF NOT EXISTS test;
 USE test;
diff --git a/tests/queries/0_stateless/02286_drop_filesystem_cache.sh b/tests/queries/0_stateless/02286_drop_filesystem_cache.sh
index 32c9e9cb060..dab777fcc31 100755
--- a/tests/queries/0_stateless/02286_drop_filesystem_cache.sh
+++ b/tests/queries/0_stateless/02286_drop_filesystem_cache.sh
@@ -31,7 +31,7 @@ for STORAGE_POLICY in 's3_cache' 'local_cache' 'azure_cache'; do
     $CLICKHOUSE_CLIENT --query "SELECT * FROM test_02286 FORMAT Null"
     $CLICKHOUSE_CLIENT --query "SELECT count() FROM system.filesystem_cache"
 
-    $CLICKHOUSE_CLIENT --multiline --multiquery --query "SYSTEM DROP FILESYSTEM CACHE 'ff'; --{serverError 36}"
+    $CLICKHOUSE_CLIENT --multiline --query "SYSTEM DROP FILESYSTEM CACHE 'ff'; --{serverError 36}"
 
     $CLICKHOUSE_CLIENT --query "SELECT count() FROM system.filesystem_cache"
 
diff --git a/tests/queries/0_stateless/02337_drop_filesystem_cache_access.sh b/tests/queries/0_stateless/02337_drop_filesystem_cache_access.sh
index 7f2103e6ddb..07d5f641571 100755
--- a/tests/queries/0_stateless/02337_drop_filesystem_cache_access.sh
+++ b/tests/queries/0_stateless/02337_drop_filesystem_cache_access.sh
@@ -7,17 +7,17 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
-${CLICKHOUSE_CLIENT} --multiline --multiquery -q """
+${CLICKHOUSE_CLIENT} --multiline -q """
 DROP USER IF EXISTS user_test_02337;
 CREATE USER user_test_02337 IDENTIFIED WITH plaintext_password BY 'user_test_02337';
 REVOKE ALL ON *.* FROM user_test_02337;
 """
-${CLICKHOUSE_CLIENT} --multiline --multiquery --user user_test_02337 --password user_test_02337 -q """
+${CLICKHOUSE_CLIENT} --multiline --user user_test_02337 --password user_test_02337 -q """
 SYSTEM DROP FILESYSTEM CACHE; -- { serverError 497 }
 """
-${CLICKHOUSE_CLIENT} --multiline --multiquery -q """
+${CLICKHOUSE_CLIENT} --multiline -q """
 GRANT SYSTEM DROP FILESYSTEM CACHE ON *.* TO user_test_02337 WITH GRANT OPTION;
 """
-${CLICKHOUSE_CLIENT} --multiline --multiquery --user user_test_02337 --password user_test_02337 -q """
+${CLICKHOUSE_CLIENT} --multiline --user user_test_02337 --password user_test_02337 -q """
 SYSTEM DROP FILESYSTEM CACHE;
 """
diff --git a/tests/queries/0_stateless/02364_window_view_segfault.sh b/tests/queries/0_stateless/02364_window_view_segfault.sh
index 9accf7f30c3..4173b6c6b8d 100755
--- a/tests/queries/0_stateless/02364_window_view_segfault.sh
+++ b/tests/queries/0_stateless/02364_window_view_segfault.sh
@@ -9,7 +9,7 @@ opts=(
     "--allow_experimental_analyzer=0"
 )
 
-${CLICKHOUSE_CLIENT} "${opts[@]}" --multiquery --multiline --query """
+${CLICKHOUSE_CLIENT} "${opts[@]}" --multiline --query """
 DROP TABLE IF EXISTS mt ON CLUSTER test_shard_localhost;
 DROP TABLE IF EXISTS wv ON CLUSTER test_shard_localhost;
 CREATE TABLE mt ON CLUSTER test_shard_localhost (a Int32, timestamp DateTime) ENGINE=MergeTree ORDER BY tuple();
diff --git a/tests/queries/0_stateless/02373_datetime64_monotonicity.sh b/tests/queries/0_stateless/02373_datetime64_monotonicity.sh
index 7e852aa24b0..73a32e28be9 100755
--- a/tests/queries/0_stateless/02373_datetime64_monotonicity.sh
+++ b/tests/queries/0_stateless/02373_datetime64_monotonicity.sh
@@ -7,7 +7,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 for tz in Asia/Tehran UTC Canada/Atlantic Europe/Berlin
 do
     echo "$tz"
-    TZ=$tz $CLICKHOUSE_LOCAL --multiline --multiquery "
+    TZ=$tz $CLICKHOUSE_LOCAL --multiline "
       drop table if exists dt64_monotonicity_test;
       drop table if exists dt64_monotonicity_test_string;
       CREATE TABLE dt64_monotonicity_test (date_time DateTime64(3, 'Europe/Berlin'), id String) ENGINE = MergeTree PARTITION BY toDate(date_time, 'Europe/Berlin') ORDER BY date_time;
diff --git a/tests/queries/0_stateless/02416_rename_database_rbac.sh b/tests/queries/0_stateless/02416_rename_database_rbac.sh
index c319136d29c..4d4b54e9384 100755
--- a/tests/queries/0_stateless/02416_rename_database_rbac.sh
+++ b/tests/queries/0_stateless/02416_rename_database_rbac.sh
@@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
-${CLICKHOUSE_CLIENT} --multiline --multiquery -q """
+${CLICKHOUSE_CLIENT} --multiline -q """
 DROP DATABASE IF EXISTS database_02416;
 CREATE DATABASE database_02416;
 
@@ -18,6 +18,6 @@ REVOKE DROP DATABASE ON database_02416.* FROM 'user_test_02416';
 GRANT CREATE TABLE ON *.* TO 'user_test_02416' WITH GRANT OPTION;
 GRANT DROP TABLE ON *.* TO 'user_test_02416' WITH GRANT OPTION;
 """
-${CLICKHOUSE_CLIENT} --multiline --multiquery --user user_test_02416 --password user_test_02416 -q """
+${CLICKHOUSE_CLIENT} --multiline --user user_test_02416 --password user_test_02416 -q """
 RENAME DATABASE user_test_02416 to aaaaaaaaa; -- { serverError 497 }
 """
diff --git a/tests/queries/0_stateless/02435_rollback_cancelled_queries.sh b/tests/queries/0_stateless/02435_rollback_cancelled_queries.sh
index a0312b7a40c..ba652013a57 100755
--- a/tests/queries/0_stateless/02435_rollback_cancelled_queries.sh
+++ b/tests/queries/0_stateless/02435_rollback_cancelled_queries.sh
@@ -41,7 +41,7 @@ function insert_data
     else
         # client will send 1000-rows blocks, server will squash them into 110000-rows blocks (more chances to catch a bug on query cancellation)
         $CLICKHOUSE_CLIENT --stacktrace --query_id="$ID" --throw_on_unsupported_query_inside_transaction=0 --implicit_transaction="$IMPLICIT" \
-            --max_block_size=1000 --max_insert_block_size=1000 --multiquery -q \
+            --max_block_size=1000 --max_insert_block_size=1000 -q \
             "${BEGIN}insert into dedup_test settings max_insert_block_size=110000, min_insert_block_size_rows=110000 format TSV$COMMIT" < $DATA_FILE \
             | grep -Fv "Transaction is not in RUNNING state"
     fi
diff --git a/tests/queries/0_stateless/02460_projections_and_aggregate_null_if_empty.sh b/tests/queries/0_stateless/02460_projections_and_aggregate_null_if_empty.sh
index 6e96b9b8afc..748e134ea99 100755
--- a/tests/queries/0_stateless/02460_projections_and_aggregate_null_if_empty.sh
+++ b/tests/queries/0_stateless/02460_projections_and_aggregate_null_if_empty.sh
@@ -6,4 +6,4 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
-${CLICKHOUSE_LOCAL} --aggregate_functions_null_for_empty=1 --multiquery --query "create table test_date (date Int32) ENGINE = MergeTree ORDER BY (date) as select 20220920; SELECT max(date) FROM test_date";
+${CLICKHOUSE_LOCAL} --aggregate_functions_null_for_empty=1 --query "create table test_date (date Int32) ENGINE = MergeTree ORDER BY (date) as select 20220920; SELECT max(date) FROM test_date";
diff --git a/tests/queries/0_stateless/02494_query_cache_user_isolation.sh b/tests/queries/0_stateless/02494_query_cache_user_isolation.sh
index d55e2460619..dcad921e013 100755
--- a/tests/queries/0_stateless/02494_query_cache_user_isolation.sh
+++ b/tests/queries/0_stateless/02494_query_cache_user_isolation.sh
@@ -91,16 +91,16 @@ ${CLICKHOUSE_CLIENT} --user "admin" --query "INSERT INTO user_data (ID, userID)
 
 # Test...
 ${CLICKHOUSE_CLIENT} --user "admin" --query "SELECT '-- policy_1 test'"
-${CLICKHOUSE_CLIENT} --user "admin" --multiquery "SET ROLE user_role_1; SELECT * FROM user_data" # should only return rows with userID = 1
+${CLICKHOUSE_CLIENT} --user "admin" "SET ROLE user_role_1; SELECT * FROM user_data" # should only return rows with userID = 1
 
 ${CLICKHOUSE_CLIENT} --user "admin" --query "SELECT '-- policy_2 test'"
-${CLICKHOUSE_CLIENT} --user "admin" --multiquery "SET ROLE user_role_2; SELECT * FROM user_data" # should only return rows with userID = 2
+${CLICKHOUSE_CLIENT} --user "admin" "SET ROLE user_role_2; SELECT * FROM user_data" # should only return rows with userID = 2
 
 ${CLICKHOUSE_CLIENT} --user "admin" --query "SELECT '-- policy_1 with query cache test'"
-${CLICKHOUSE_CLIENT} --user "admin" --multiquery "SET ROLE user_role_1; SELECT * FROM user_data SETTINGS use_query_cache = 1" # should only return rows with userID = 1
+${CLICKHOUSE_CLIENT} --user "admin" "SET ROLE user_role_1; SELECT * FROM user_data SETTINGS use_query_cache = 1" # should only return rows with userID = 1
 
 ${CLICKHOUSE_CLIENT} --user "admin" --query "SELECT '-- policy_2 with query cache test'"
-${CLICKHOUSE_CLIENT} --user "admin" --multiquery "SET ROLE user_role_2; SELECT * FROM user_data SETTINGS use_query_cache = 1" # should only return rows with userID = 2 (not userID = 1!)
+${CLICKHOUSE_CLIENT} --user "admin" "SET ROLE user_role_2; SELECT * FROM user_data SETTINGS use_query_cache = 1" # should only return rows with userID = 2 (not userID = 1!)
 
 # Cleanup
 ${CLICKHOUSE_CLIENT} --user "admin" --query "DROP ROLE user_role_1"
diff --git a/tests/queries/0_stateless/02503_cache_on_write_with_small_segment_size.sh b/tests/queries/0_stateless/02503_cache_on_write_with_small_segment_size.sh
index 5aeab4c746e..13412502dd4 100755
--- a/tests/queries/0_stateless/02503_cache_on_write_with_small_segment_size.sh
+++ b/tests/queries/0_stateless/02503_cache_on_write_with_small_segment_size.sh
@@ -11,7 +11,7 @@ function random {
      cat /dev/urandom | LC_ALL=C tr -dc 'a-zA-Z' | fold -w ${1:-8} | head -n 1
 }
 
-${CLICKHOUSE_CLIENT} --multiline --multiquery -q "
+${CLICKHOUSE_CLIENT} --multiline -q "
 drop table if exists ttt;
 
 CREATE TABLE ttt (id Int32, value String)
@@ -46,7 +46,7 @@ ${CLICKHOUSE_CLIENT}  -q "
 select count() from system.filesystem_cache_log where query_id = '$query_id' AND read_type == 'READ_FROM_CACHE';
 "
 
-${CLICKHOUSE_CLIENT} --multiline --multiquery -q "
+${CLICKHOUSE_CLIENT} --multiline -q "
 select count() from ttt;
 drop table ttt sync;
 "
diff --git a/tests/queries/0_stateless/02521_merge_over_gap.sh b/tests/queries/0_stateless/02521_merge_over_gap.sh
index 4f73b5bceee..3c549a2a0dc 100755
--- a/tests/queries/0_stateless/02521_merge_over_gap.sh
+++ b/tests/queries/0_stateless/02521_merge_over_gap.sh
@@ -15,7 +15,7 @@ $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS table_with_gap;"
 $CLICKHOUSE_CLIENT --query "CREATE TABLE table_with_gap (v UInt8) ENGINE = MergeTree() ORDER BY tuple() settings old_parts_lifetime = 10000;"
 $CLICKHOUSE_CLIENT --query "SYSTEM STOP MERGES table_with_gap;"
 
-$CLICKHOUSE_CLIENT --multiquery --query "
+$CLICKHOUSE_CLIENT --query "
     INSERT INTO table_with_gap VALUES (1);
     INSERT INTO table_with_gap VALUES (2);
     INSERT INTO table_with_gap VALUES (3);
diff --git a/tests/queries/0_stateless/02530_dictionaries_update_field.sh b/tests/queries/0_stateless/02530_dictionaries_update_field.sh
index a8e5804fba4..968756368d1 100755
--- a/tests/queries/0_stateless/02530_dictionaries_update_field.sh
+++ b/tests/queries/0_stateless/02530_dictionaries_update_field.sh
@@ -35,7 +35,7 @@ for layout in "${layouts[@]}"; do
             echo "$layout"
         fi
 
-        $CLICKHOUSE_CLIENT --multiquery "
+        $CLICKHOUSE_CLIENT "
             TRUNCATE TABLE table_for_update_field_dictionary;
 
             CREATE DICTIONARY $dictionary_name
diff --git a/tests/queries/0_stateless/02702_allow_skip_errors_enum.sh b/tests/queries/0_stateless/02702_allow_skip_errors_enum.sh
index e68f5517d52..9c73ad3a809 100755
--- a/tests/queries/0_stateless/02702_allow_skip_errors_enum.sh
+++ b/tests/queries/0_stateless/02702_allow_skip_errors_enum.sh
@@ -4,7 +4,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
-$CLICKHOUSE_CLIENT --multiquery --query "DROP TABLE IF EXISTS t; CREATE TABLE t (x Enum('Hello' = 1, 'World' = 2)) ENGINE = Memory;"
+$CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS t; CREATE TABLE t (x Enum('Hello' = 1, 'World' = 2)) ENGINE = Memory;"
 $CLICKHOUSE_CLIENT --input_format_allow_errors_num 1 --query "INSERT INTO t FORMAT CSV" <<END
 Hello
 Goodbye
diff --git a/tests/queries/0_stateless/02704_keeper_map_zk_nodes.sh b/tests/queries/0_stateless/02704_keeper_map_zk_nodes.sh
index 9689d4f5a50..758313814be 100755
--- a/tests/queries/0_stateless/02704_keeper_map_zk_nodes.sh
+++ b/tests/queries/0_stateless/02704_keeper_map_zk_nodes.sh
@@ -8,7 +8,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 table_name="02704_keeper_map_zk_nodes"
 table_name_another="02704_keeper_map_zk_nodes_new_table"
 
-$CLICKHOUSE_CLIENT --multiquery --query="
+$CLICKHOUSE_CLIENT --query="
 DROP TABLE IF EXISTS $table_name;
 DROP TABLE IF EXISTS $table_name_another;
 CREATE TABLE $table_name (key UInt64, value UInt64)
diff --git a/tests/queries/0_stateless/02707_clickhouse_local_implicit_file_table_function.sh b/tests/queries/0_stateless/02707_clickhouse_local_implicit_file_table_function.sh
index dabea8a373a..644767de85f 100755
--- a/tests/queries/0_stateless/02707_clickhouse_local_implicit_file_table_function.sh
+++ b/tests/queries/0_stateless/02707_clickhouse_local_implicit_file_table_function.sh
@@ -42,7 +42,7 @@ $CLICKHOUSE_LOCAL -q "SELECT COUNT(*) FROM \"${dir}/tmp.csv\""
 
 #################
 echo "Test 2: check Filesystem database"
-$CLICKHOUSE_LOCAL --multiline --multiquery -q """
+$CLICKHOUSE_LOCAL --multiline -q """
 DROP DATABASE IF EXISTS test;
 CREATE DATABASE test ENGINE = Filesystem('${dir}');
 SELECT COUNT(*) FROM test.\`tmp.csv\`;
@@ -57,7 +57,7 @@ DROP DATABASE test;
 
 #################
 echo "Test 3: check show database with Filesystem"
-$CLICKHOUSE_LOCAL --multiline --multiquery -q """
+$CLICKHOUSE_LOCAL --multiline -q """
 DROP DATABASE IF EXISTS test02707;
 CREATE DATABASE test02707 ENGINE = Filesystem('${dir}');
 SHOW DATABASES;
diff --git a/tests/queries/0_stateless/02712_bool_better_exception_message.sh b/tests/queries/0_stateless/02712_bool_better_exception_message.sh
index ee006958d93..1da70d7025b 100755
--- a/tests/queries/0_stateless/02712_bool_better_exception_message.sh
+++ b/tests/queries/0_stateless/02712_bool_better_exception_message.sh
@@ -6,19 +6,19 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CURDIR"/../shell_config.sh
 
 
-$CLICKHOUSE_LOCAL --multiquery <<END
+$CLICKHOUSE_LOCAL <<END
     SELECT * FROM format(JSONEachRow, 'x Bool', '{"x": 1}');
     SELECT * FROM format(JSONEachRow, 'x Bool', '{"x": null}');
 END
 
-$CLICKHOUSE_LOCAL --multiquery <<END 2>&1 | rg -Fc "'w' character"
+$CLICKHOUSE_LOCAL <<END 2>&1 | rg -Fc "'w' character"
     SELECT * FROM format(JSONEachRow, 'x Bool', '{"x": wtf}');
 END
 
-$CLICKHOUSE_LOCAL --multiquery <<END 2>&1 | rg -Fc "expected 'false'"
+$CLICKHOUSE_LOCAL <<END 2>&1 | rg -Fc "expected 'false'"
     SELECT * FROM format(JSONEachRow, 'x Bool', '{"x": ftw}');
 END
 
-$CLICKHOUSE_LOCAL --multiquery <<END 2>&1 | rg -Fc "'{' character"
+$CLICKHOUSE_LOCAL <<END 2>&1 | rg -Fc "'{' character"
     SELECT * FROM format(JSONEachRow, 'x Bool', '{"x": {}}');
 END
diff --git a/tests/queries/0_stateless/02722_database_filesystem.sh b/tests/queries/0_stateless/02722_database_filesystem.sh
index 2d0ff256c95..fa23d847d90 100755
--- a/tests/queries/0_stateless/02722_database_filesystem.sh
+++ b/tests/queries/0_stateless/02722_database_filesystem.sh
@@ -30,7 +30,7 @@ cp ${user_files_tmp_dir}/tmp_numbers_1.csv ${user_files_tmp_dir}/tmp/tmp_numbers
 
 #################
 echo "Test 1: create filesystem database and check implicit calls"
-${CLICKHOUSE_CLIENT} --multiline --multiquery -q """
+${CLICKHOUSE_CLIENT} --multiline -q """
 DROP DATABASE IF EXISTS test1;
 CREATE DATABASE test1 ENGINE = Filesystem;
 """
@@ -57,20 +57,20 @@ ${CLICKHOUSE_CLIENT} --query "SELECT COUNT(*) FROM test1.\`/tmp/tmp.csv\`;" 2>&1
 ${CLICKHOUSE_CLIENT} --query "SELECT COUNT(*) FROM test1.\`../*/tmp_numbers_*.csv\`;" 2>&1 | tr '\n' ' ' | grep -oF "PATH_ACCESS_DENIED" > /dev/null && echo "OK" || echo 'FAIL' ||:
 ${CLICKHOUSE_CLIENT} --query "SELECT COUNT(*) FROM test1.\`../tmp_numbers_*.csv\`;" 2>&1 | tr '\n' ' ' | grep -oF "PATH_ACCESS_DENIED" > /dev/null && echo "OK" || echo 'FAIL' ||:
 ${CLICKHOUSE_CLIENT} --query "SELECT COUNT(*) FROM test1.\`../*.csv\`;" 2>&1 | tr '\n' ' ' | grep -oF "PATH_ACCESS_DENIED" > /dev/null && echo "OK" || echo 'FAIL' ||:
-${CLICKHOUSE_CLIENT} --multiline --multiquery --query """
+${CLICKHOUSE_CLIENT} --multiline --query """
 USE test1;
 SELECT COUNT(*) FROM \"../${tmp_dir}/tmp.csv\";
 """ 2>&1 | tr '\n' ' ' | grep -oF "PATH_ACCESS_DENIED" > /dev/null && echo "OK" || echo 'FAIL' ||:
 ${CLICKHOUSE_CLIENT} --query "SELECT COUNT(*) FROM test1.\`../../../../../../tmp.csv\`;" 2>&1 | tr '\n' ' ' | grep -oF "PATH_ACCESS_DENIED" > /dev/null && echo "OK" || echo 'FAIL' ||:
 
 # BAD_ARGUMENTS: path should be inside user_files
-${CLICKHOUSE_CLIENT} --multiline --multiquery -q """
+${CLICKHOUSE_CLIENT} --multiline -q """
 DROP DATABASE IF EXISTS test2;
 CREATE DATABASE test2 ENGINE = Filesystem('/tmp');
 """ 2>&1 | tr '\n' ' ' | grep -oF -e "UNKNOWN_TABLE" -e "BAD_ARGUMENTS" > /dev/null && echo "OK" || echo 'FAIL' ||:
 
 # BAD_ARGUMENTS: .../user_files/relative_unknown_dir does not exist
-${CLICKHOUSE_CLIENT} --multiline --multiquery -q """
+${CLICKHOUSE_CLIENT} --multiline -q """
 DROP DATABASE IF EXISTS test2;
 CREATE DATABASE test2 ENGINE = Filesystem('relative_unknown_dir');
 """ 2>&1 | tr '\n' ' ' | grep -oF -e "UNKNOWN_TABLE" -e "BAD_ARGUMENTS" > /dev/null && echo "OK" || echo 'FAIL' ||:
diff --git a/tests/queries/0_stateless/02724_database_s3.sh b/tests/queries/0_stateless/02724_database_s3.sh
index 80b47282146..cc7f012c8cf 100755
--- a/tests/queries/0_stateless/02724_database_s3.sh
+++ b/tests/queries/0_stateless/02724_database_s3.sh
@@ -8,7 +8,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
 #################
 echo "Test 1: select from s3"
-${CLICKHOUSE_CLIENT} --multiline --multiquery -q """
+${CLICKHOUSE_CLIENT} --multiline -q """
 DROP DATABASE IF EXISTS test1;
 CREATE DATABASE test1 ENGINE = S3;
 USE test1;
@@ -17,7 +17,7 @@ SELECT * FROM \"http://localhost:11111/test/a.tsv\"
 ${CLICKHOUSE_CLIENT} -q "SHOW DATABASES;" | grep test1
 
 # check credentials with absolute path
-${CLICKHOUSE_CLIENT} --multiline --multiquery -q """
+${CLICKHOUSE_CLIENT} --multiline -q """
 DROP DATABASE IF EXISTS test2;
 CREATE DATABASE test2 ENGINE = S3('', 'test', 'testtest');
 USE test2;
@@ -25,7 +25,7 @@ SELECT * FROM \"http://localhost:11111/test/b.tsv\"
 """
 
 # check credentials with relative path
-${CLICKHOUSE_CLIENT} --multiline --multiquery -q """
+${CLICKHOUSE_CLIENT} --multiline -q """
 DROP DATABASE IF EXISTS test4;
 CREATE DATABASE test4 ENGINE = S3('http://localhost:11111/test', 'test', 'testtest');
 USE test4;
@@ -33,7 +33,7 @@ SELECT * FROM \"b.tsv\"
 """
 
 # Check named collection loading
-${CLICKHOUSE_CLIENT} --multiline --multiquery -q """
+${CLICKHOUSE_CLIENT} --multiline -q """
 DROP DATABASE IF EXISTS test5;
 CREATE DATABASE test5 ENGINE = S3(s3_conn_db);
 SELECT * FROM test5.\`b.tsv\`
@@ -41,20 +41,20 @@ SELECT * FROM test5.\`b.tsv\`
 
 #################
 echo "Test 2: check exceptions"
-${CLICKHOUSE_CLIENT} --multiline --multiquery -q """
+${CLICKHOUSE_CLIENT} --multiline -q """
 DROP DATABASE IF EXISTS test3;
 CREATE DATABASE test3 ENGINE = S3;
 USE test3;
 SELECT * FROM \"http://localhost:11111/test/a.myext\"
 """ 2>&1 | tr '\n' ' ' | grep -oF -e "UNKNOWN_TABLE" -e "S3_ERROR" > /dev/null && echo "OK" || echo 'FAIL' ||:
 
-${CLICKHOUSE_CLIENT} --multiline --multiquery -q """
+${CLICKHOUSE_CLIENT} --multiline -q """
 USE test3;
 SELECT * FROM \"abacaba\"
 """ 2>&1 | tr '\n' ' ' | grep -oF -e "UNKNOWN_TABLE" -e "BAD_ARGUMENTS" > /dev/null && echo "OK" || echo 'FAIL' ||:
 
 # Cleanup
-${CLICKHOUSE_CLIENT} --multiline --multiquery -q """
+${CLICKHOUSE_CLIENT} --multiline -q """
 DROP DATABASE IF EXISTS test1;
 DROP DATABASE IF EXISTS test2;
 DROP DATABASE IF EXISTS test3;
diff --git a/tests/queries/0_stateless/02725_database_hdfs.sh b/tests/queries/0_stateless/02725_database_hdfs.sh
index 1eb22976b84..7fd35c72ef1 100755
--- a/tests/queries/0_stateless/02725_database_hdfs.sh
+++ b/tests/queries/0_stateless/02725_database_hdfs.sh
@@ -25,7 +25,7 @@ fi
 echo "Test 1: select from hdfs database"
 
 # Database without specific host
-${CLICKHOUSE_CLIENT} --multiline --multiquery -q """
+${CLICKHOUSE_CLIENT} --multiline -q """
 DROP DATABASE IF EXISTS test_hdfs_1;
 CREATE DATABASE test_hdfs_1 ENGINE = HDFS;
 USE test_hdfs_1;
@@ -34,7 +34,7 @@ SELECT * FROM \"hdfs://localhost:12222/test_02725_1.tsv\"
 ${CLICKHOUSE_CLIENT} -q "SHOW DATABASES;" | grep test_hdfs_1
 
 # Database with host
-${CLICKHOUSE_CLIENT} --multiline --multiquery -q """
+${CLICKHOUSE_CLIENT} --multiline -q """
 DROP DATABASE IF EXISTS test_hdfs_2;
 CREATE DATABASE test_hdfs_2 ENGINE = HDFS('hdfs://localhost:12222');
 USE test_hdfs_2;
@@ -45,12 +45,12 @@ ${CLICKHOUSE_CLIENT} -q "SHOW DATABASES;" | grep test_hdfs_2
 #################
 echo "Test 2: check exceptions"
 
-${CLICKHOUSE_CLIENT} --multiline --multiquery -q """
+${CLICKHOUSE_CLIENT} --multiline -q """
 DROP DATABASE IF EXISTS test_hdfs_3;
 CREATE DATABASE test_hdfs_3 ENGINE = HDFS('abacaba');
 """ 2>&1 | tr '\n' ' ' | grep -oF "BAD_ARGUMENTS"
 
-${CLICKHOUSE_CLIENT} --multiline --multiquery -q """
+${CLICKHOUSE_CLIENT} --multiline -q """
 DROP DATABASE IF EXISTS test_hdfs_4;
 CREATE DATABASE test_hdfs_4 ENGINE = HDFS;
 USE test_hdfs_4;
@@ -64,7 +64,7 @@ ${CLICKHOUSE_CLIENT} --query "SELECT * FROM test_hdfs_4.\`hdfs://localhost:12222
 
 
 # Cleanup
-${CLICKHOUSE_CLIENT} --multiline --multiquery -q """
+${CLICKHOUSE_CLIENT} --multiline -q """
 DROP DATABASE IF EXISTS test_hdfs_1;
 DROP DATABASE IF EXISTS test_hdfs_2;
 DROP DATABASE IF EXISTS test_hdfs_3;
diff --git a/tests/queries/0_stateless/02725_local_query_parameters.sh b/tests/queries/0_stateless/02725_local_query_parameters.sh
index 92d7f645454..151a854d5b9 100755
--- a/tests/queries/0_stateless/02725_local_query_parameters.sh
+++ b/tests/queries/0_stateless/02725_local_query_parameters.sh
@@ -5,6 +5,6 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
-$CLICKHOUSE_LOCAL --multiquery -q "
+$CLICKHOUSE_LOCAL -q "
     SET param_x=1;
     SELECT {x:UInt64}, {x:String};"
diff --git a/tests/queries/0_stateless/02751_multiquery_with_argument.reference b/tests/queries/0_stateless/02751_multiquery_with_argument.reference
index 2e55712e49c..f02e9bab2cd 100644
--- a/tests/queries/0_stateless/02751_multiquery_with_argument.reference
+++ b/tests/queries/0_stateless/02751_multiquery_with_argument.reference
@@ -5,8 +5,6 @@
 Syntax error
 Empty query
 Empty query
-BAD_ARGUMENTS
-BAD_ARGUMENTS
 301
 302
 304
diff --git a/tests/queries/0_stateless/02751_multiquery_with_argument.sh b/tests/queries/0_stateless/02751_multiquery_with_argument.sh
index 7b959a3c3dc..4021194656b 100755
--- a/tests/queries/0_stateless/02751_multiquery_with_argument.sh
+++ b/tests/queries/0_stateless/02751_multiquery_with_argument.sh
@@ -4,18 +4,14 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
-$CLICKHOUSE_LOCAL --multiquery "SELECT 100"
-$CLICKHOUSE_LOCAL --multiquery "SELECT 101;"
-$CLICKHOUSE_LOCAL --multiquery "SELECT 102;SELECT 103;"
+$CLICKHOUSE_LOCAL "SELECT 100"
+$CLICKHOUSE_LOCAL "SELECT 101;"
+$CLICKHOUSE_LOCAL "SELECT 102;SELECT 103;"
 
 # Invalid SQL.
-$CLICKHOUSE_LOCAL --multiquery "SELECT 200; S" 2>&1 | grep -o 'Syntax error'
-$CLICKHOUSE_LOCAL --multiquery "; SELECT 201;" 2>&1 | grep -o 'Empty query'
-$CLICKHOUSE_LOCAL --multiquery "; S; SELECT 202" 2>&1 | grep -o 'Empty query'
-
-# Simultaneously passing --queries-file + --query (multiquery) is prohibited.
-$CLICKHOUSE_LOCAL --queries-file "queries.csv" --multiquery "SELECT 250;" 2>&1 | grep -o 'BAD_ARGUMENTS'
-$CLICKHOUSE_CLIENT --queries-file "queries.csv" --multiquery "SELECT 251;" 2>&1 | grep -o 'BAD_ARGUMENTS'
+$CLICKHOUSE_LOCAL "SELECT 200; S" 2>&1 | grep -o 'Syntax error'
+$CLICKHOUSE_LOCAL "; SELECT 201;" 2>&1 | grep -o 'Empty query'
+$CLICKHOUSE_LOCAL "; S; SELECT 202" 2>&1 | grep -o 'Empty query'
 
 # Error expectation cases.
 # -n <SQL> is also interpreted as a query
diff --git a/tests/queries/0_stateless/02815_no_throw_in_simple_queries.sh b/tests/queries/0_stateless/02815_no_throw_in_simple_queries.sh
index 68c55f9b66a..18ffc9dfec3 100755
--- a/tests/queries/0_stateless/02815_no_throw_in_simple_queries.sh
+++ b/tests/queries/0_stateless/02815_no_throw_in_simple_queries.sh
@@ -19,7 +19,7 @@ $CLICKHOUSE_CLIENT --query "SHOW TABLES" || echo "Failed"
 $CLICKHOUSE_CLIENT --query "SELECT * FROM system.tables WHERE database = currentDatabase() FORMAT Null" || echo "Failed"
 
 # Multi queries are ok:
-$CLICKHOUSE_LOCAL --multiquery "SELECT 1; SELECT 2;" || echo "Failed"
+$CLICKHOUSE_LOCAL "SELECT 1; SELECT 2;" || echo "Failed"
 
 # It can run in interactive mode:
 function run()
diff --git a/tests/queries/0_stateless/02843_insertion_table_schema_infer.sh b/tests/queries/0_stateless/02843_insertion_table_schema_infer.sh
index d806b678456..9207e48092f 100755
--- a/tests/queries/0_stateless/02843_insertion_table_schema_infer.sh
+++ b/tests/queries/0_stateless/02843_insertion_table_schema_infer.sh
@@ -9,7 +9,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
 DATA_DIR=$CUR_DIR/data_tsv
 
-$CLICKHOUSE_LOCAL --multiquery \
+$CLICKHOUSE_LOCAL \
 "CREATE VIEW users AS SELECT * FROM file('$DATA_DIR/mock_data.tsv', TSVWithNamesAndTypes); 
  CREATE TABLE users_output (name String, tag UInt64)ENGINE = Memory;
  INSERT INTO users_output WITH (SELECT groupUniqArrayArray(mapKeys(Tags)) FROM users) AS unique_tags SELECT UserName AS name, length(unique_tags) AS tag FROM users;
diff --git a/tests/queries/0_stateless/02864_restore_table_with_broken_part.sh b/tests/queries/0_stateless/02864_restore_table_with_broken_part.sh
index 08313e2fd3b..229f832ba14 100755
--- a/tests/queries/0_stateless/02864_restore_table_with_broken_part.sh
+++ b/tests/queries/0_stateless/02864_restore_table_with_broken_part.sh
@@ -39,7 +39,7 @@ $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS tbl"
 # Then try to restore with the setting `restore_broken_parts_as_detached` set to true.
 $CLICKHOUSE_CLIENT --query "RESTORE TABLE default.tbl AS tbl FROM Disk('backups', '${backup_name}') SETTINGS restore_broken_parts_as_detached = true" 2>/dev/null | awk -F '\t' '{print $2}'
 
-$CLICKHOUSE_CLIENT --multiquery <<EOF
+$CLICKHOUSE_CLIENT <<EOF
 SELECT * FROM tbl ORDER BY x;
 SELECT name, reason FROM system.detached_parts WHERE database = currentDatabase() AND table = 'tbl';
 
diff --git a/tests/queries/0_stateless/02874_parquet_multiple_batches_array_inconsistent_offsets.sh b/tests/queries/0_stateless/02874_parquet_multiple_batches_array_inconsistent_offsets.sh
index c96531ffea9..2df3e90c98c 100755
--- a/tests/queries/0_stateless/02874_parquet_multiple_batches_array_inconsistent_offsets.sh
+++ b/tests/queries/0_stateless/02874_parquet_multiple_batches_array_inconsistent_offsets.sh
@@ -123,7 +123,7 @@ echo "Parquet"
 
 DATA_FILE=$CUR_DIR/data_parquet/string_int_list_inconsistent_offset_multiple_batches.parquet
 
-${CLICKHOUSE_LOCAL} --multiquery "
+${CLICKHOUSE_LOCAL} "
 DROP TABLE IF EXISTS parquet_load;
 CREATE TABLE parquet_load (ints Array(Int64), strings Nullable(String)) ENGINE = Memory;
 INSERT INTO parquet_load FROM INFILE '$DATA_FILE';
diff --git a/tests/queries/0_stateless/02875_merge_engine_set_index.sh b/tests/queries/0_stateless/02875_merge_engine_set_index.sh
index 57b5db374c1..355d83167a6 100755
--- a/tests/queries/0_stateless/02875_merge_engine_set_index.sh
+++ b/tests/queries/0_stateless/02875_merge_engine_set_index.sh
@@ -38,7 +38,7 @@ $CLICKHOUSE_CLIENT -nq "
     ENGINE = Merge(currentDatabase(), 't*');"
 
 query_id="${CLICKHOUSE_DATABASE}_merge_engine_set_index_$RANDOM$RANDOM"
-$CLICKHOUSE_CLIENT --query_id="$query_id" --multiquery -q "
+$CLICKHOUSE_CLIENT --query_id="$query_id" -q "
 SELECT
     a,
     b
diff --git a/tests/queries/0_stateless/02877_optimize_read_in_order_from_view.sh b/tests/queries/0_stateless/02877_optimize_read_in_order_from_view.sh
index a3a906a9891..dc9cc71757e 100755
--- a/tests/queries/0_stateless/02877_optimize_read_in_order_from_view.sh
+++ b/tests/queries/0_stateless/02877_optimize_read_in_order_from_view.sh
@@ -4,7 +4,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CURDIR"/../shell_config.sh
 set -eo pipefail
 
-$CLICKHOUSE_CLIENT --multiquery <<EOF
+$CLICKHOUSE_CLIENT <<EOF
 DROP TABLE IF EXISTS view1;
 DROP TABLE IF EXISTS table1;
 CREATE TABLE table1 (number UInt64) ENGINE=MergeTree ORDER BY number SETTINGS index_granularity=1;
diff --git a/tests/queries/0_stateless/02884_create_view_with_sql_security_option.sh b/tests/queries/0_stateless/02884_create_view_with_sql_security_option.sh
index dd869cd9988..cc4e76a9ed9 100755
--- a/tests/queries/0_stateless/02884_create_view_with_sql_security_option.sh
+++ b/tests/queries/0_stateless/02884_create_view_with_sql_security_option.sh
@@ -11,7 +11,7 @@ user2="user02884_2_${CLICKHOUSE_DATABASE}_$RANDOM"
 user3="user02884_3_${CLICKHOUSE_DATABASE}_$RANDOM"
 db=${CLICKHOUSE_DATABASE}
 
-${CLICKHOUSE_CLIENT} --multiquery <<EOF
+${CLICKHOUSE_CLIENT} <<EOF
 CREATE TABLE $db.test_table (s String) ENGINE = MergeTree ORDER BY s;
 
 DROP USER IF EXISTS $user1, $user2, $user3;
@@ -20,7 +20,7 @@ GRANT SELECT ON $db.* TO $user1;
 EOF
 
 echo "===== StorageView ====="
-${CLICKHOUSE_CLIENT} --multiquery <<EOF
+${CLICKHOUSE_CLIENT} <<EOF
 CREATE VIEW $db.test_view_1 (s String)
 AS SELECT * FROM $db.test_table;
 
@@ -63,7 +63,7 @@ EOF
 (( $(${CLICKHOUSE_CLIENT} --query "SHOW TABLE $db.test_view_5" 2>&1 | grep -c "INVOKER") >= 1 )) && echo "OK" || echo "UNEXPECTED"
 (( $(${CLICKHOUSE_CLIENT} --query "SHOW TABLE $db.test_view_2" 2>&1 | grep -c "DEFINER = $user1") >= 1 )) && echo "OK" || echo "UNEXPECTED"
 
-${CLICKHOUSE_CLIENT} --multiquery <<EOF
+${CLICKHOUSE_CLIENT} <<EOF
 GRANT SELECT ON $db.test_view_1 TO $user2;
 GRANT SELECT ON $db.test_view_2 TO $user2;
 GRANT SELECT ON $db.test_view_3 TO $user2;
@@ -159,7 +159,7 @@ ${CLICKHOUSE_CLIENT} --query "REVOKE SELECT ON $db.test_table FROM $user1"
 (( $(${CLICKHOUSE_CLIENT} --user $user2 --query "SELECT * FROM $db.test_mv_4" 2>&1 | grep -c "Not enough privileges") >= 1 )) && echo "OK" || echo "UNEXPECTED"
 (( $(${CLICKHOUSE_CLIENT} --query "INSERT INTO $db.test_table VALUES ('foo'), ('bar');" 2>&1 | grep -c "Not enough privileges") >= 1 )) && echo "OK" || echo "UNEXPECTED"
 
-${CLICKHOUSE_CLIENT} --multiquery <<EOF
+${CLICKHOUSE_CLIENT} <<EOF
 CREATE TABLE $db.source
 (
     a UInt64
@@ -232,7 +232,7 @@ ${CLICKHOUSE_CLIENT} --user $user1 --query "
 ${CLICKHOUSE_CLIENT} --query "GRANT SET DEFINER ON $user2 TO $user1"
 
 echo "===== TestRowPolicy ====="
-${CLICKHOUSE_CLIENT} --multiquery <<EOF
+${CLICKHOUSE_CLIENT} <<EOF
 CREATE TABLE $db.test_row_t (x Int32, y Int32) ENGINE = MergeTree ORDER BY x;
 
 CREATE VIEW $db.test_view_row_1 DEFINER = $user1 SQL SECURITY DEFINER AS SELECT x, y AS z FROM $db.test_row_t;
@@ -246,7 +246,7 @@ EOF
 
 ${CLICKHOUSE_CLIENT} --user $user2 --query "SELECT * FROM $db.test_view_row_1"
 
-${CLICKHOUSE_CLIENT} --multiquery <<EOF
+${CLICKHOUSE_CLIENT} <<EOF
 CREATE TABLE $db.test_row_t2 (x Int32, y Int32) ENGINE = MergeTree ORDER BY x;
 
 CREATE VIEW $db.test_mv_row_2 DEFINER = $user1 SQL SECURITY DEFINER AS SELECT x, y AS z FROM $db.test_row_t2;
@@ -262,7 +262,7 @@ ${CLICKHOUSE_CLIENT} --user $user2 --query "SELECT * FROM $db.test_mv_row_2"
 
 echo "===== TestInsertChain ====="
 
-${CLICKHOUSE_CLIENT} --multiquery <<EOF
+${CLICKHOUSE_CLIENT} <<EOF
 CREATE TABLE $db.session_events(
     clientId UUID,
     sessionId UUID,
@@ -302,7 +302,7 @@ ${CLICKHOUSE_CLIENT} --user $user3 --query "SELECT count(*) FROM session_events"
 ${CLICKHOUSE_CLIENT} --query "SELECT count(*) FROM materialized_events"
 
 echo "===== TestOnCluster ====="
-${CLICKHOUSE_CLIENT} --multiquery <<EOF
+${CLICKHOUSE_CLIENT} <<EOF
 
 CREATE TABLE $db.test_cluster ON CLUSTER test_shard_localhost (a String) Engine = MergeTree() ORDER BY a FORMAT Null;
 CREATE TABLE $db.test_cluster_2 ON CLUSTER test_shard_localhost (a String) Engine = MergeTree() ORDER BY a FORMAT Null;
diff --git a/tests/queries/0_stateless/02885_async_insert_access_check_for_defaults.sh b/tests/queries/0_stateless/02885_async_insert_access_check_for_defaults.sh
index b2525e67d4a..72346e0cb59 100755
--- a/tests/queries/0_stateless/02885_async_insert_access_check_for_defaults.sh
+++ b/tests/queries/0_stateless/02885_async_insert_access_check_for_defaults.sh
@@ -8,7 +8,7 @@ restricted_user=${CLICKHOUSE_DATABASE}_restricted_user_$RANDOM$RANDOM
 # The 'restricted_user' will not have access to the dictionary 'dict',
 # so they shouldn't be able to insert to a table with DEFAULT dictGet(dict, ...)
 
-$CLICKHOUSE_CLIENT --multiquery <<EOF
+$CLICKHOUSE_CLIENT <<EOF
 DROP USER IF EXISTS ${restricted_user};
 DROP TABLE IF EXISTS table_with_default;
 DROP DICTIONARY IF EXISTS dict;
@@ -50,7 +50,7 @@ $CLICKHOUSE_CLIENT --user "${restricted_user}" --query "INSERT INTO table_with_d
 $CLICKHOUSE_CLIENT --user "${restricted_user}" --async_insert=1 --query "INSERT INTO table_with_default (key) VALUES (6)"
 $CLICKHOUSE_CLIENT --query "SELECT * FROM table_with_default WHERE key IN [5, 6] ORDER BY key"
 
-$CLICKHOUSE_CLIENT --multiquery <<EOF
+$CLICKHOUSE_CLIENT <<EOF
 DROP USER ${restricted_user};
 DROP TABLE table_with_default;
 DROP DICTIONARY dict;
diff --git a/tests/queries/0_stateless/02900_clickhouse_local_drop_current_database.sh b/tests/queries/0_stateless/02900_clickhouse_local_drop_current_database.sh
index 3250c70a268..7a52ccc3fd5 100755
--- a/tests/queries/0_stateless/02900_clickhouse_local_drop_current_database.sh
+++ b/tests/queries/0_stateless/02900_clickhouse_local_drop_current_database.sh
@@ -4,7 +4,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CUR_DIR"/../shell_config.sh
 
-${CLICKHOUSE_LOCAL} --echo --multiquery "
+${CLICKHOUSE_LOCAL} --echo "
     CREATE DATABASE foo;
     USE foo;
     SELECT 1;
diff --git a/tests/queries/0_stateless/02956_clickhouse_local_system_parts.sh b/tests/queries/0_stateless/02956_clickhouse_local_system_parts.sh
index dac0cc2b865..4f49adb26d5 100755
--- a/tests/queries/0_stateless/02956_clickhouse_local_system_parts.sh
+++ b/tests/queries/0_stateless/02956_clickhouse_local_system_parts.sh
@@ -5,7 +5,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CUR_DIR"/../shell_config.sh
 
-$CLICKHOUSE_LOCAL --multiquery "
+$CLICKHOUSE_LOCAL "
     CREATE TABLE test (x UInt8) ENGINE = MergeTree ORDER BY ();
     INSERT INTO test SELECT 1;
 
diff --git a/tests/queries/0_stateless/02973_backup_of_in_memory_compressed.sh b/tests/queries/0_stateless/02973_backup_of_in_memory_compressed.sh
index b212e42061f..2b92c2865ad 100755
--- a/tests/queries/0_stateless/02973_backup_of_in_memory_compressed.sh
+++ b/tests/queries/0_stateless/02973_backup_of_in_memory_compressed.sh
@@ -6,26 +6,26 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
-$CLICKHOUSE_CLIENT --multiquery "
+$CLICKHOUSE_CLIENT "
 DROP TABLE IF EXISTS test;
 CREATE TABLE test (x String) ENGINE = Memory SETTINGS compress = 1;
 INSERT INTO test SELECT 'Hello, world' FROM numbers(1000000);
 "
 
-$CLICKHOUSE_CLIENT --multiquery "
+$CLICKHOUSE_CLIENT "
 BACKUP TABLE test TO File('test.zip');
 " --format Null
 
-$CLICKHOUSE_CLIENT --multiquery "
+$CLICKHOUSE_CLIENT "
 TRUNCATE TABLE test;
 SELECT count() FROM test;
 "
 
-$CLICKHOUSE_CLIENT --multiquery "
+$CLICKHOUSE_CLIENT "
 RESTORE TABLE test FROM File('test.zip');
 " --format Null
 
-$CLICKHOUSE_CLIENT --multiquery "
+$CLICKHOUSE_CLIENT "
 SELECT count(), min(x), max(x) FROM test;
 DROP TABLE test;
 "
diff --git a/tests/queries/0_stateless/02973_parse_crlf_with_tsv_files.sh b/tests/queries/0_stateless/02973_parse_crlf_with_tsv_files.sh
index 6edac86be5b..9608925d087 100755
--- a/tests/queries/0_stateless/02973_parse_crlf_with_tsv_files.sh
+++ b/tests/queries/0_stateless/02973_parse_crlf_with_tsv_files.sh
@@ -17,7 +17,7 @@ echo -ne "Akiba_Hebrew_Academy\t2017-08-01\t241\r\nAegithina_tiphia\t2018-02-01\
 
 echo -e "<-- Read UNIX endings -->\n"
 $CLICKHOUSE_CLIENT --query "SELECT * FROM file(${UNIX_ENDINGS}, 'TabSeparated', 'SearchTerm String, Date Date, Hits UInt32');" 
-$CLICKHOUSE_CLIENT --multiquery --query "SELECT * FROM file(${DOS_ENDINGS}, 'TabSeparated', 'SearchTerm String, Date Date, Hits UInt32'); --{serverError 117}" 
+$CLICKHOUSE_CLIENT --query "SELECT * FROM file(${DOS_ENDINGS}, 'TabSeparated', 'SearchTerm String, Date Date, Hits UInt32'); --{serverError 117}" 
 
 echo -e "\n<-- Read DOS endings with setting input_format_tsv_crlf_end_of_line=1 -->\n"
 $CLICKHOUSE_CLIENT --query "SELECT * FROM file(${DOS_ENDINGS}, 'TabSeparated', 'SearchTerm String, Date Date, Hits UInt32') SETTINGS input_format_tsv_crlf_end_of_line = 1;"
diff --git a/tests/queries/0_stateless/02995_forget_partition.sh b/tests/queries/0_stateless/02995_forget_partition.sh
index c22f5829130..6fa0b96e90d 100755
--- a/tests/queries/0_stateless/02995_forget_partition.sh
+++ b/tests/queries/0_stateless/02995_forget_partition.sh
@@ -6,7 +6,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CUR_DIR"/../shell_config.sh
 
 
-${CLICKHOUSE_CLIENT} --multiline --multiquery -q """
+${CLICKHOUSE_CLIENT} --multiline -q """
 drop table if exists forget_partition;
 
 create table forget_partition
@@ -31,7 +31,7 @@ alter table forget_partition drop partition '20240102';
 # DROP PARTITION do not wait for a part to be removed from memory due to possible concurrent SELECTs, so we have to do wait manually here
 while [[ $(${CLICKHOUSE_CLIENT} -q "select count() from system.parts where database=currentDatabase() and table='forget_partition' and partition IN ('20240101', '20240102')") != 0 ]]; do sleep 1; done
 
-${CLICKHOUSE_CLIENT} --multiline --multiquery -q """
+${CLICKHOUSE_CLIENT} --multiline -q """
 set allow_unrestricted_reads_from_keeper=1;
 
 select '---before---';
diff --git a/tests/queries/0_stateless/02995_index_1.sh b/tests/queries/0_stateless/02995_index_1.sh
index a5f1b30c2e8..128697fd0fe 100755
--- a/tests/queries/0_stateless/02995_index_1.sh
+++ b/tests/queries/0_stateless/02995_index_1.sh
@@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
-${CLICKHOUSE_CLIENT} --multiquery "
+${CLICKHOUSE_CLIENT} "
 
 DROP TABLE IF EXISTS test;
 CREATE TABLE test (a String, b String, c String) ENGINE = MergeTree ORDER BY (a, b, c) SETTINGS index_granularity = 11;
@@ -39,6 +39,6 @@ WHERE a >= (round(pow(sipHash64(1, try), 1 / (3 + sipHash64(2, try) % 8))) AS a1
   AND c <= (c1 + round(pow(sipHash64(11, try), 1 / (3 + sipHash64(12, try) % 8))))::String
 HAVING count() > 0;
 "
-done | ${CLICKHOUSE_CLIENT} --multiquery
+done | ${CLICKHOUSE_CLIENT} 
 
-${CLICKHOUSE_CLIENT} --multiquery "DROP TABLE test"
+${CLICKHOUSE_CLIENT} "DROP TABLE test"
diff --git a/tests/queries/0_stateless/02995_index_10.sh b/tests/queries/0_stateless/02995_index_10.sh
index d72c7c72705..c15ba00fd05 100755
--- a/tests/queries/0_stateless/02995_index_10.sh
+++ b/tests/queries/0_stateless/02995_index_10.sh
@@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
-${CLICKHOUSE_CLIENT} --multiquery "
+${CLICKHOUSE_CLIENT} "
 
 DROP TABLE IF EXISTS test;
 CREATE TABLE test (a String, b String, c String) ENGINE = MergeTree ORDER BY (a, b, c) SETTINGS index_granularity = 11;
@@ -39,6 +39,6 @@ WHERE a >= (round(pow(sipHash64(1, try), 1 / (3 + sipHash64(2, try) % 8))) AS a1
   AND c <= (c1 + round(pow(sipHash64(11, try), 1 / (3 + sipHash64(12, try) % 8))))::String
 HAVING count() > 0;
 "
-done | ${CLICKHOUSE_CLIENT} --multiquery
+done | ${CLICKHOUSE_CLIENT} 
 
-${CLICKHOUSE_CLIENT} --multiquery "DROP TABLE test"
+${CLICKHOUSE_CLIENT} "DROP TABLE test"
diff --git a/tests/queries/0_stateless/02995_index_2.sh b/tests/queries/0_stateless/02995_index_2.sh
index e7451c7ee4b..a32f5c511f8 100755
--- a/tests/queries/0_stateless/02995_index_2.sh
+++ b/tests/queries/0_stateless/02995_index_2.sh
@@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
-${CLICKHOUSE_CLIENT} --multiquery "
+${CLICKHOUSE_CLIENT} "
 
 DROP TABLE IF EXISTS test;
 CREATE TABLE test (a String, b String, c String) ENGINE = MergeTree ORDER BY (a, b, c) SETTINGS index_granularity = 11;
@@ -39,6 +39,6 @@ WHERE a >= (round(pow(sipHash64(1, try), 1 / (3 + sipHash64(2, try) % 8))) AS a1
   AND c <= (c1 + round(pow(sipHash64(11, try), 1 / (3 + sipHash64(12, try) % 8))))::String
 HAVING count() > 0;
 "
-done | ${CLICKHOUSE_CLIENT} --multiquery
+done | ${CLICKHOUSE_CLIENT}
 
-${CLICKHOUSE_CLIENT} --multiquery "DROP TABLE test"
+${CLICKHOUSE_CLIENT} "DROP TABLE test"
diff --git a/tests/queries/0_stateless/02995_index_3.sh b/tests/queries/0_stateless/02995_index_3.sh
index 506429e2696..9cc937391fc 100755
--- a/tests/queries/0_stateless/02995_index_3.sh
+++ b/tests/queries/0_stateless/02995_index_3.sh
@@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
-${CLICKHOUSE_CLIENT} --multiquery "
+${CLICKHOUSE_CLIENT} "
 
 DROP TABLE IF EXISTS test;
 CREATE TABLE test (a String, b String, c String) ENGINE = MergeTree ORDER BY (a, b, c) SETTINGS index_granularity = 11;
@@ -39,6 +39,6 @@ WHERE a >= (round(pow(sipHash64(1, try), 1 / (3 + sipHash64(2, try) % 8))) AS a1
   AND c <= (c1 + round(pow(sipHash64(11, try), 1 / (3 + sipHash64(12, try) % 8))))::String
 HAVING count() > 0;
 "
-done | ${CLICKHOUSE_CLIENT} --multiquery
+done | ${CLICKHOUSE_CLIENT} 
 
-${CLICKHOUSE_CLIENT} --multiquery "DROP TABLE test"
+${CLICKHOUSE_CLIENT} "DROP TABLE test"
diff --git a/tests/queries/0_stateless/02995_index_4.sh b/tests/queries/0_stateless/02995_index_4.sh
index 1a0458728f9..e450997e48b 100755
--- a/tests/queries/0_stateless/02995_index_4.sh
+++ b/tests/queries/0_stateless/02995_index_4.sh
@@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
-${CLICKHOUSE_CLIENT} --multiquery "
+${CLICKHOUSE_CLIENT} "
 
 DROP TABLE IF EXISTS test;
 CREATE TABLE test (a String, b String, c String) ENGINE = MergeTree ORDER BY (a, b, c) SETTINGS index_granularity = 11;
@@ -39,6 +39,6 @@ WHERE a >= (round(pow(sipHash64(1, try), 1 / (3 + sipHash64(2, try) % 8))) AS a1
   AND c <= (c1 + round(pow(sipHash64(11, try), 1 / (3 + sipHash64(12, try) % 8))))::String
 HAVING count() > 0;
 "
-done | ${CLICKHOUSE_CLIENT} --multiquery
+done | ${CLICKHOUSE_CLIENT} 
 
-${CLICKHOUSE_CLIENT} --multiquery "DROP TABLE test"
+${CLICKHOUSE_CLIENT} "DROP TABLE test"
diff --git a/tests/queries/0_stateless/02995_index_5.sh b/tests/queries/0_stateless/02995_index_5.sh
index 60c12a8146d..80f75a532e3 100755
--- a/tests/queries/0_stateless/02995_index_5.sh
+++ b/tests/queries/0_stateless/02995_index_5.sh
@@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
-${CLICKHOUSE_CLIENT} --multiquery "
+${CLICKHOUSE_CLIENT} "
 
 DROP TABLE IF EXISTS test;
 CREATE TABLE test (a String, b String, c String) ENGINE = MergeTree ORDER BY (a, b, c) SETTINGS index_granularity = 11;
@@ -39,6 +39,6 @@ WHERE a >= (round(pow(sipHash64(1, try), 1 / (3 + sipHash64(2, try) % 8))) AS a1
   AND c <= (c1 + round(pow(sipHash64(11, try), 1 / (3 + sipHash64(12, try) % 8))))::String
 HAVING count() > 0;
 "
-done | ${CLICKHOUSE_CLIENT} --multiquery
+done | ${CLICKHOUSE_CLIENT} 
 
-${CLICKHOUSE_CLIENT} --multiquery "DROP TABLE test"
+${CLICKHOUSE_CLIENT} "DROP TABLE test"
diff --git a/tests/queries/0_stateless/02995_index_6.sh b/tests/queries/0_stateless/02995_index_6.sh
index 4936f73f36b..e90387c7c0c 100755
--- a/tests/queries/0_stateless/02995_index_6.sh
+++ b/tests/queries/0_stateless/02995_index_6.sh
@@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
-${CLICKHOUSE_CLIENT} --multiquery "
+${CLICKHOUSE_CLIENT} "
 
 DROP TABLE IF EXISTS test;
 CREATE TABLE test (a String, b String, c String) ENGINE = MergeTree ORDER BY (a, b, c) SETTINGS index_granularity = 11;
@@ -39,6 +39,6 @@ WHERE a >= (round(pow(sipHash64(1, try), 1 / (3 + sipHash64(2, try) % 8))) AS a1
   AND c <= (c1 + round(pow(sipHash64(11, try), 1 / (3 + sipHash64(12, try) % 8))))::String
 HAVING count() > 0;
 "
-done | ${CLICKHOUSE_CLIENT} --multiquery
+done | ${CLICKHOUSE_CLIENT}
 
-${CLICKHOUSE_CLIENT} --multiquery "DROP TABLE test"
+${CLICKHOUSE_CLIENT} "DROP TABLE test"
diff --git a/tests/queries/0_stateless/02995_index_7.sh b/tests/queries/0_stateless/02995_index_7.sh
index 26be310abce..a5fdd98b2f8 100755
--- a/tests/queries/0_stateless/02995_index_7.sh
+++ b/tests/queries/0_stateless/02995_index_7.sh
@@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
-${CLICKHOUSE_CLIENT} --multiquery "
+${CLICKHOUSE_CLIENT} "
 
 DROP TABLE IF EXISTS test;
 CREATE TABLE test (a String, b String, c String) ENGINE = MergeTree ORDER BY (a, b, c) SETTINGS index_granularity = 11;
@@ -39,6 +39,6 @@ WHERE a >= (round(pow(sipHash64(1, try), 1 / (3 + sipHash64(2, try) % 8))) AS a1
   AND c <= (c1 + round(pow(sipHash64(11, try), 1 / (3 + sipHash64(12, try) % 8))))::String
 HAVING count() > 0;
 "
-done | ${CLICKHOUSE_CLIENT} --multiquery
+done | ${CLICKHOUSE_CLIENT} 
 
-${CLICKHOUSE_CLIENT} --multiquery "DROP TABLE test"
+${CLICKHOUSE_CLIENT} "DROP TABLE test"
diff --git a/tests/queries/0_stateless/02995_index_8.sh b/tests/queries/0_stateless/02995_index_8.sh
index 8c2620b59fd..adb835aedca 100755
--- a/tests/queries/0_stateless/02995_index_8.sh
+++ b/tests/queries/0_stateless/02995_index_8.sh
@@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
-${CLICKHOUSE_CLIENT} --multiquery "
+${CLICKHOUSE_CLIENT} "
 
 DROP TABLE IF EXISTS test;
 CREATE TABLE test (a String, b String, c String) ENGINE = MergeTree ORDER BY (a, b, c) SETTINGS index_granularity = 11;
@@ -39,6 +39,6 @@ WHERE a >= (round(pow(sipHash64(1, try), 1 / (3 + sipHash64(2, try) % 8))) AS a1
   AND c <= (c1 + round(pow(sipHash64(11, try), 1 / (3 + sipHash64(12, try) % 8))))::String
 HAVING count() > 0;
 "
-done | ${CLICKHOUSE_CLIENT} --multiquery
+done | ${CLICKHOUSE_CLIENT} 
 
-${CLICKHOUSE_CLIENT} --multiquery "DROP TABLE test"
+${CLICKHOUSE_CLIENT} "DROP TABLE test"
diff --git a/tests/queries/0_stateless/02995_index_9.sh b/tests/queries/0_stateless/02995_index_9.sh
index 76160c62aaa..4b78777cd2a 100755
--- a/tests/queries/0_stateless/02995_index_9.sh
+++ b/tests/queries/0_stateless/02995_index_9.sh
@@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
-${CLICKHOUSE_CLIENT} --multiquery "
+${CLICKHOUSE_CLIENT} "
 
 DROP TABLE IF EXISTS test;
 CREATE TABLE test (a String, b String, c String) ENGINE = MergeTree ORDER BY (a, b, c) SETTINGS index_granularity = 11;
@@ -39,6 +39,6 @@ WHERE a >= (round(pow(sipHash64(1, try), 1 / (3 + sipHash64(2, try) % 8))) AS a1
   AND c <= (c1 + round(pow(sipHash64(11, try), 1 / (3 + sipHash64(12, try) % 8))))::String
 HAVING count() > 0;
 "
-done | ${CLICKHOUSE_CLIENT} --multiquery
+done | ${CLICKHOUSE_CLIENT} 
 
-${CLICKHOUSE_CLIENT} --multiquery "DROP TABLE test"
+${CLICKHOUSE_CLIENT} "DROP TABLE test"
diff --git a/tests/queries/0_stateless/02998_native_parquet_reader.sh b/tests/queries/0_stateless/02998_native_parquet_reader.sh
index d6369c4921b..95b659815ed 100755
--- a/tests/queries/0_stateless/02998_native_parquet_reader.sh
+++ b/tests/queries/0_stateless/02998_native_parquet_reader.sh
@@ -208,4 +208,4 @@ CH_SCHEMA="\
 QUERY="SELECT * from file('$PAR_PATH', 'Parquet', '$CH_SCHEMA')"
 
 # there may be more than on group in parquet files, unstable results may generated by multithreads
-$CLICKHOUSE_LOCAL --multiquery --max_threads 1 --max_parsing_threads 1 --input_format_parquet_use_native_reader true --query "$QUERY"
+$CLICKHOUSE_LOCAL --max_threads 1 --max_parsing_threads 1 --input_format_parquet_use_native_reader true --query "$QUERY"
diff --git a/tests/queries/0_stateless/03001_backup_matview_after_modify_query.sh b/tests/queries/0_stateless/03001_backup_matview_after_modify_query.sh
index f857358a5ea..8c6aa70f14c 100755
--- a/tests/queries/0_stateless/03001_backup_matview_after_modify_query.sh
+++ b/tests/queries/0_stateless/03001_backup_matview_after_modify_query.sh
@@ -9,7 +9,7 @@ db="$CLICKHOUSE_DATABASE"
 db_2="${db}_2"
 backup_name="${db}_backup"
 
-${CLICKHOUSE_CLIENT} --multiquery "
+${CLICKHOUSE_CLIENT} "
 DROP TABLE IF EXISTS src;
 DROP TABLE IF EXISTS mv;
 CREATE TABLE src(Timestamp DateTime64(9), c1 String, c2 String) ENGINE=MergeTree ORDER BY Timestamp;
diff --git a/tests/queries/0_stateless/03001_matview_columns_after_modify_query.sh b/tests/queries/0_stateless/03001_matview_columns_after_modify_query.sh
index 2ec5832fac6..96cbd391a44 100755
--- a/tests/queries/0_stateless/03001_matview_columns_after_modify_query.sh
+++ b/tests/queries/0_stateless/03001_matview_columns_after_modify_query.sh
@@ -4,7 +4,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
-${CLICKHOUSE_CLIENT} --multiquery "
+${CLICKHOUSE_CLIENT} "
 DROP TABLE IF EXISTS src;
 DROP TABLE IF EXISTS mv;
 CREATE TABLE src(Timestamp DateTime64(9), c1 String, c2 String) ENGINE=MergeTree ORDER BY Timestamp;
diff --git a/tests/queries/0_stateless/03006_correct_revoke_for_partial_rights.sh b/tests/queries/0_stateless/03006_correct_revoke_for_partial_rights.sh
index 8c79dfdbafc..312fb03668c 100755
--- a/tests/queries/0_stateless/03006_correct_revoke_for_partial_rights.sh
+++ b/tests/queries/0_stateless/03006_correct_revoke_for_partial_rights.sh
@@ -8,7 +8,7 @@ db=${CLICKHOUSE_DATABASE}
 user1="user1_03006_${db}_$RANDOM"
 user2="user2_03006_${db}_$RANDOM"
 
-${CLICKHOUSE_CLIENT} --multiquery <<EOF
+${CLICKHOUSE_CLIENT} <<EOF
 DROP DATABASE IF EXISTS $db;
 CREATE DATABASE $db;
 CREATE USER $user1, $user2;
diff --git a/tests/queries/0_stateless/03147_system_columns_access_checks.sh b/tests/queries/0_stateless/03147_system_columns_access_checks.sh
index b027ea28504..9887c29cff7 100755
--- a/tests/queries/0_stateless/03147_system_columns_access_checks.sh
+++ b/tests/queries/0_stateless/03147_system_columns_access_checks.sh
@@ -18,7 +18,7 @@ seq 1 $NUM_TABLES | xargs -P "${THREADS}" -I{} bash -c "
 "
 echo
 
-$CLICKHOUSE_CLIENT --multiquery "
+$CLICKHOUSE_CLIENT "
 DROP USER IF EXISTS test_03147;
 CREATE USER test_03147;
 GRANT SELECT (end) ON ${CLICKHOUSE_DATABASE}.test1 TO test_03147;
@@ -31,6 +31,6 @@ GRANT SELECT (end) ON ${CLICKHOUSE_DATABASE}.test1 TO test_03147;
 # The test could succeed even on the previous version, but it will show up as being too slow.
 $CLICKHOUSE_CLIENT --user test_03147 --query "SELECT name FROM system.columns WHERE database = currentDatabase()"
 
-$CLICKHOUSE_CLIENT --multiquery "
+$CLICKHOUSE_CLIENT "
 DROP USER test_03147;
 "
diff --git a/tests/queries/0_stateless/03201_local_named_collections.sh b/tests/queries/0_stateless/03201_local_named_collections.sh
index 809b4b52f41..86135d027f6 100755
--- a/tests/queries/0_stateless/03201_local_named_collections.sh
+++ b/tests/queries/0_stateless/03201_local_named_collections.sh
@@ -4,17 +4,17 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
-${CLICKHOUSE_CLIENT} --multiquery "
+${CLICKHOUSE_CLIENT} "
 DROP TABLE IF EXISTS test;
 CREATE TABLE test (s String) ORDER BY ();
 INSERT INTO test VALUES ('Hello, world!');
 "
 
-${CLICKHOUSE_LOCAL} --multiquery "
+${CLICKHOUSE_LOCAL} "
 CREATE NAMED COLLECTION mydb AS host = '${CLICKHOUSE_HOST}', port = ${CLICKHOUSE_PORT_TCP}, user = 'default', password = '', db = '${CLICKHOUSE_DATABASE}';
 SELECT * FROM remote(mydb, table = 'test');
 " 2>&1 | grep --text -F -v "ASan doesn't fully support makecontext/swapcontext functions"
 
-${CLICKHOUSE_CLIENT} --multiquery "
+${CLICKHOUSE_CLIENT} "
 DROP TABLE test;
 "
diff --git a/tests/queries/0_stateless/03212_thousand_exceptions.sh b/tests/queries/0_stateless/03212_thousand_exceptions.sh
index 0a6abf35c10..1237cbf537f 100755
--- a/tests/queries/0_stateless/03212_thousand_exceptions.sh
+++ b/tests/queries/0_stateless/03212_thousand_exceptions.sh
@@ -6,4 +6,4 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CURDIR"/../shell_config.sh
 
 # This should not be too slow, even under sanitizers.
-yes "SELECT throwIf(1); SELECT '.' FORMAT Values;" | head -n 1000 | $CLICKHOUSE_CLIENT --multiquery --ignore-error 2>/dev/null
+yes "SELECT throwIf(1); SELECT '.' FORMAT Values;" | head -n 1000 | $CLICKHOUSE_CLIENT --ignore-error 2>/dev/null

From 6372fdee6d344bd87d58ce89fa069b55750c9aba Mon Sep 17 00:00:00 2001
From: kssenii <sumarokovakseniia@mail.ru>
Date: Tue, 30 Jul 2024 13:46:05 +0200
Subject: [PATCH 1068/1488] Update tests

---
 tests/queries/0_stateless/02995_index_3.sh             | 2 +-
 tests/queries/0_stateless/03143_asof_join_ddb_long.sql | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/queries/0_stateless/02995_index_3.sh b/tests/queries/0_stateless/02995_index_3.sh
index 506429e2696..219ae81154f 100755
--- a/tests/queries/0_stateless/02995_index_3.sh
+++ b/tests/queries/0_stateless/02995_index_3.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Tags: long, no-debug, no-asan, no-tsan, no-msan, no-ubsan, no-sanitize-coverage
+# Tags: long, no-debug, no-asan, no-tsan, no-msan, no-ubsan, no-sanitize-coverage, no-distributed-cache
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
diff --git a/tests/queries/0_stateless/03143_asof_join_ddb_long.sql b/tests/queries/0_stateless/03143_asof_join_ddb_long.sql
index 17a67511030..a927e4f1e1f 100644
--- a/tests/queries/0_stateless/03143_asof_join_ddb_long.sql
+++ b/tests/queries/0_stateless/03143_asof_join_ddb_long.sql
@@ -1,4 +1,4 @@
--- Tags: long
+-- Tags: long, no-distributed-cache
 
 DROP TABLE IF EXISTS build;
 DROP TABLE IF EXISTS skewed_probe;

From eb129b539fce2a407182d892ce3bd00f782a5833 Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Tue, 30 Jul 2024 13:46:27 +0200
Subject: [PATCH 1069/1488] Add tests

---
 src/Storages/StorageKeeperMap.cpp             | 135 +++++++++++++-----
 src/Storages/StorageKeeperMap.h               |   3 +-
 .../test_keeper_map_retries/__init__.py       |   0
 .../configs/enable_keeper_map.xml             |   3 +
 .../configs/fault_injection.xml               |   6 +
 .../test_keeper_map_retries/test.py           |  78 ++++++++++
 .../02911_backup_restore_keeper_map.sh        |  15 +-
 7 files changed, 194 insertions(+), 46 deletions(-)
 create mode 100644 tests/integration/test_keeper_map_retries/__init__.py
 create mode 100644 tests/integration/test_keeper_map_retries/configs/enable_keeper_map.xml
 create mode 100644 tests/integration/test_keeper_map_retries/configs/fault_injection.xml
 create mode 100644 tests/integration/test_keeper_map_retries/test.py

diff --git a/src/Storages/StorageKeeperMap.cpp b/src/Storages/StorageKeeperMap.cpp
index 5534bb7f346..09c21ae28f5 100644
--- a/src/Storages/StorageKeeperMap.cpp
+++ b/src/Storages/StorageKeeperMap.cpp
@@ -183,7 +183,7 @@ public:
                 settings.insert_keeper_retry_max_backoff_ms},
             context->getProcessListElement()};
 
-        retries_ctl.retryLoop([&]()
+        zk_retry.retryLoop([&]()
         {
             auto zookeeper = storage.getClient();
             auto keys_limit = storage.keysLimit();
@@ -205,12 +205,12 @@ public:
             for (const auto & [key, _] : new_values)
                 key_paths.push_back(storage.fullPathForKey(key));
 
-            zkutil::ZooKeeper::MultiExistsResponse results;
+            zkutil::ZooKeeper::MultiTryGetResponse results;
 
             if constexpr (!for_update)
             {
                 if (!strict)
-                    results = zookeeper->exists(key_paths);
+                    results = zookeeper->tryGet(key_paths);
             }
 
             Coordination::Requests requests;
@@ -231,7 +231,8 @@ public:
                 {
                     if (!strict && results[i].error == Coordination::Error::ZOK)
                     {
-                        requests.push_back(zkutil::makeSetRequest(key_paths[i], new_values[key], -1));
+                        if (results[i].data != new_values[key])
+                            requests.push_back(zkutil::makeSetRequest(key_paths[i], new_values[key], -1));
                     }
                     else
                     {
@@ -241,6 +242,9 @@ public:
                 }
             }
 
+            if (requests.empty())
+                return;
+
             if (new_keys_num != 0)
             {
                 auto will_be = current_keys_num + new_keys_num;
@@ -259,7 +263,7 @@ public:
 };
 
 template <typename KeyContainer>
-class StorageKeeperMapSource : public ISource
+class StorageKeeperMapSource : public ISource, WithContext
 {
     const StorageKeeperMap & storage;
     size_t max_block_size;
@@ -290,8 +294,15 @@ public:
         KeyContainerPtr container_,
         KeyContainerIter begin_,
         KeyContainerIter end_,
-        bool with_version_column_)
-        : ISource(getHeader(header, with_version_column_)), storage(storage_), max_block_size(max_block_size_), container(std::move(container_)), it(begin_), end(end_)
+        bool with_version_column_,
+        ContextPtr context_)
+        : ISource(getHeader(header, with_version_column_))
+        , WithContext(std::move(context_))
+        , storage(storage_)
+        , max_block_size(max_block_size_)
+        , container(std::move(container_))
+        , it(begin_)
+        , end(end_)
         , with_version_column(with_version_column_)
     {
     }
@@ -316,12 +327,12 @@ public:
             for (auto & raw_key : raw_keys)
                 raw_key = base64Encode(raw_key, /* url_encoding */ true);
 
-            return storage.getBySerializedKeys(raw_keys, nullptr, with_version_column);
+            return storage.getBySerializedKeys(raw_keys, nullptr, with_version_column, getContext());
         }
         else
         {
             size_t elem_num = std::min(max_block_size, static_cast<size_t>(end - it));
-            auto chunk = storage.getBySerializedKeys(std::span{it, it + elem_num}, nullptr, with_version_column);
+            auto chunk = storage.getBySerializedKeys(std::span{it, it + elem_num}, nullptr, with_version_column, getContext());
             it += elem_num;
             return chunk;
         }
@@ -553,14 +564,31 @@ Pipe StorageKeeperMap::read(
 
             using KeyContainer = typename KeyContainerPtr::element_type;
             pipes.emplace_back(std::make_shared<StorageKeeperMapSource<KeyContainer>>(
-                *this, sample_block, max_block_size, keys, keys->begin() + begin, keys->begin() + end, with_version_column));
+                *this, sample_block, max_block_size, keys, keys->begin() + begin, keys->begin() + end, with_version_column, context_));
         }
         return Pipe::unitePipes(std::move(pipes));
     };
 
-    auto client = getClient();
     if (all_scan)
-        return process_keys(std::make_shared<std::vector<std::string>>(client->getChildren(zk_data_path)));
+    {
+        const auto & settings = context_->getSettingsRef();
+        ZooKeeperRetriesControl zk_retry{
+            getName(),
+            getLogger(getName()),
+            ZooKeeperRetriesInfo{
+                settings.keeper_max_retries,
+                settings.keeper_retry_initial_backoff_ms,
+                settings.keeper_retry_max_backoff_ms},
+            context_->getProcessListElement()};
+
+        std::vector<std::string> children;
+        zk_retry.retryLoop([&]
+        {
+            auto client = getClient();
+            children = client->getChildren(zk_data_path);
+        });
+        return process_keys(std::make_shared<std::vector<std::string>>(std::move(children)));
+    }
 
     return process_keys(std::move(filtered_keys));
 }
@@ -571,11 +599,24 @@ SinkToStoragePtr StorageKeeperMap::write(const ASTPtr & /*query*/, const Storage
     return std::make_shared<StorageKeeperMapSink>(*this, metadata_snapshot->getSampleBlock(), local_context);
 }
 
-void StorageKeeperMap::truncate(const ASTPtr &, const StorageMetadataPtr &, ContextPtr, TableExclusiveLockHolder &)
+void StorageKeeperMap::truncate(const ASTPtr &, const StorageMetadataPtr &, ContextPtr local_context, TableExclusiveLockHolder &)
 {
     checkTable<true>();
-    auto client = getClient();
-    client->tryRemoveChildrenRecursive(zk_data_path, true);
+    const auto & settings = local_context->getSettingsRef();
+    ZooKeeperRetriesControl zk_retry{
+        getName(),
+        getLogger(getName()),
+        ZooKeeperRetriesInfo{
+            settings.keeper_max_retries,
+            settings.keeper_retry_initial_backoff_ms,
+            settings.keeper_retry_max_backoff_ms},
+        local_context->getProcessListElement()};
+
+    zk_retry.retryLoop([&]
+    {
+        auto client = getClient();
+        client->tryRemoveChildrenRecursive(zk_data_path, true);
+    });
 }
 
 bool StorageKeeperMap::dropTable(zkutil::ZooKeeperPtr zookeeper, const zkutil::EphemeralNodeHolder::Ptr & metadata_drop_lock)
@@ -1064,10 +1105,11 @@ Chunk StorageKeeperMap::getByKeys(const ColumnsWithTypeAndName & keys, PaddedPOD
     if (raw_keys.size() != keys[0].column->size())
         throw Exception(ErrorCodes::LOGICAL_ERROR, "Assertion failed: {} != {}", raw_keys.size(), keys[0].column->size());
 
-    return getBySerializedKeys(raw_keys, &null_map, /* version_column */ false);
+    return getBySerializedKeys(raw_keys, &null_map, /* version_column */ false, getContext());
 }
 
-Chunk StorageKeeperMap::getBySerializedKeys(const std::span<const std::string> keys, PaddedPODArray<UInt8> * null_map, bool with_version) const
+Chunk StorageKeeperMap::getBySerializedKeys(
+    const std::span<const std::string> keys, PaddedPODArray<UInt8> * null_map, bool with_version, const ContextPtr & local_context) const
 {
     Block sample_block = getInMemoryMetadataPtr()->getSampleBlock();
     MutableColumns columns = sample_block.cloneEmptyColumns();
@@ -1084,17 +1126,27 @@ Chunk StorageKeeperMap::getBySerializedKeys(const std::span<const std::string> k
         null_map->resize_fill(keys.size(), 1);
     }
 
-    auto client = getClient();
-
     Strings full_key_paths;
     full_key_paths.reserve(keys.size());
 
     for (const auto & key : keys)
-    {
         full_key_paths.emplace_back(fullPathForKey(key));
-    }
 
-    auto values = client->tryGet(full_key_paths);
+    const auto & settings = local_context->getSettingsRef();
+    ZooKeeperRetriesControl zk_retry{
+        getName(),
+        getLogger(getName()),
+        ZooKeeperRetriesInfo{
+            settings.keeper_max_retries,
+            settings.keeper_retry_initial_backoff_ms,
+            settings.keeper_retry_max_backoff_ms},
+        local_context->getProcessListElement()};
+
+    zkutil::ZooKeeper::MultiTryGetResponse values;
+    zk_retry.retryLoop([&]{
+        auto client = getClient();
+        values = client->tryGet(full_key_paths);
+    });
 
     for (size_t i = 0; i < keys.size(); ++i)
     {
@@ -1182,16 +1234,16 @@ void StorageKeeperMap::mutate(const MutationCommands & commands, ContextPtr loca
 
     if (commands.front().type == MutationCommand::Type::DELETE)
     {
-        MutationsInterpreter::Settings settings(true);
-        settings.return_all_columns = true;
-        settings.return_mutated_rows = true;
+        MutationsInterpreter::Settings mutation_settings(true);
+        mutation_settings.return_all_columns = true;
+        mutation_settings.return_mutated_rows = true;
 
         auto interpreter = std::make_unique<MutationsInterpreter>(
             storage_ptr,
             metadata_snapshot,
             commands,
             local_context,
-            settings);
+            mutation_settings);
 
         auto pipeline = QueryPipelineBuilder::getPipeline(interpreter->execute());
         PullingPipelineExecutor executor(pipeline);
@@ -1200,8 +1252,6 @@ void StorageKeeperMap::mutate(const MutationCommands & commands, ContextPtr loca
         auto primary_key_pos = header.getPositionByName(primary_key);
         auto version_position = header.getPositionByName(std::string{version_column_name});
 
-        auto client = getClient();
-
         Block block;
         while (executor.pull(block))
         {
@@ -1229,7 +1279,23 @@ void StorageKeeperMap::mutate(const MutationCommands & commands, ContextPtr loca
             }
 
             Coordination::Responses responses;
-            auto status = client->tryMulti(delete_requests, responses, /* check_session_valid */ true);
+
+            const auto & settings = local_context->getSettingsRef();
+            ZooKeeperRetriesControl zk_retry{
+                getName(),
+                getLogger(getName()),
+                ZooKeeperRetriesInfo{
+                    settings.keeper_max_retries,
+                    settings.keeper_retry_initial_backoff_ms,
+                    settings.keeper_retry_max_backoff_ms},
+                local_context->getProcessListElement()};
+
+            Coordination::Error status;
+            zk_retry.retryLoop([&]
+            {
+                auto client = getClient();
+                status = client->tryMulti(delete_requests, responses, /* check_session_valid */ true);
+            });
 
             if (status == Coordination::Error::ZOK)
                 return;
@@ -1241,9 +1307,14 @@ void StorageKeeperMap::mutate(const MutationCommands & commands, ContextPtr loca
 
             for (const auto & delete_request : delete_requests)
             {
-                auto code = client->tryRemove(delete_request->getPath());
-                if (code != Coordination::Error::ZOK && code != Coordination::Error::ZNONODE)
-                    throw zkutil::KeeperException::fromPath(code, delete_request->getPath());
+                zk_retry.retryLoop([&]
+                {
+                    auto client = getClient();
+                    status = client->tryRemove(delete_request->getPath());
+                });
+
+                if (status != Coordination::Error::ZOK && status != Coordination::Error::ZNONODE)
+                    throw zkutil::KeeperException::fromPath(status, delete_request->getPath());
             }
         }
 
diff --git a/src/Storages/StorageKeeperMap.h b/src/Storages/StorageKeeperMap.h
index d4556792c48..cfbb35ab2fe 100644
--- a/src/Storages/StorageKeeperMap.h
+++ b/src/Storages/StorageKeeperMap.h
@@ -54,7 +54,8 @@ public:
     Names getPrimaryKey() const override { return {primary_key}; }
 
     Chunk getByKeys(const ColumnsWithTypeAndName & keys, PaddedPODArray<UInt8> & null_map, const Names &) const override;
-    Chunk getBySerializedKeys(std::span<const std::string> keys, PaddedPODArray<UInt8> * null_map, bool with_version) const;
+    Chunk getBySerializedKeys(
+        std::span<const std::string> keys, PaddedPODArray<UInt8> * null_map, bool with_version, const ContextPtr & local_context) const;
 
     Block getSampleBlock(const Names &) const override;
 
diff --git a/tests/integration/test_keeper_map_retries/__init__.py b/tests/integration/test_keeper_map_retries/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/integration/test_keeper_map_retries/configs/enable_keeper_map.xml b/tests/integration/test_keeper_map_retries/configs/enable_keeper_map.xml
new file mode 100644
index 00000000000..b4cbb6a954b
--- /dev/null
+++ b/tests/integration/test_keeper_map_retries/configs/enable_keeper_map.xml
@@ -0,0 +1,3 @@
+<clickhouse>
+    <keeper_map_path_prefix>/test_keeper_map</keeper_map_path_prefix>
+</clickhouse>
diff --git a/tests/integration/test_keeper_map_retries/configs/fault_injection.xml b/tests/integration/test_keeper_map_retries/configs/fault_injection.xml
new file mode 100644
index 00000000000..145945c7c7c
--- /dev/null
+++ b/tests/integration/test_keeper_map_retries/configs/fault_injection.xml
@@ -0,0 +1,6 @@
+<clickhouse>
+    <zookeeper>
+        <send_fault_probability>0.05</send_fault_probability>
+        <recv_fault_probability>0.05</recv_fault_probability>
+    </zookeeper>
+</clickhouse>
diff --git a/tests/integration/test_keeper_map_retries/test.py b/tests/integration/test_keeper_map_retries/test.py
new file mode 100644
index 00000000000..352119147cd
--- /dev/null
+++ b/tests/integration/test_keeper_map_retries/test.py
@@ -0,0 +1,78 @@
+import pytest
+
+from helpers.cluster import ClickHouseCluster
+
+import os
+
+CONFIG_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "configs")
+
+cluster = ClickHouseCluster(__file__)
+
+node = cluster.add_instance(
+    "node",
+    main_configs=["configs/enable_keeper_map.xml"],
+    with_zookeeper=True,
+    stay_alive=True,
+)
+
+
+def start_clean_clickhouse():
+    # remove fault injection if present
+    if "fault_injection.xml" in node.exec_in_container(
+        ["bash", "-c", "ls /etc/clickhouse-server/config.d"]
+    ):
+        print("Removing fault injection")
+        node.exec_in_container(
+            ["bash", "-c", "rm /etc/clickhouse-server/config.d/fault_injection.xml"]
+        )
+        node.restart_clickhouse()
+
+
+@pytest.fixture(scope="module")
+def started_cluster():
+    try:
+        cluster.start()
+        yield cluster
+
+    finally:
+        cluster.shutdown()
+
+
+def repeat_query(query, repeat):
+    for _ in range(repeat):
+        node.query(
+            query,
+            settings={
+                "keeper_max_retries": 20,
+                "keeper_retry_max_backoff_ms": 10000,
+            },
+        )
+
+
+def test_queries(started_cluster):
+    start_clean_clickhouse()
+
+    node.query("DROP TABLE IF EXISTS keeper_map_retries SYNC")
+    node.query(
+        "CREATE TABLE keeper_map_retries (a UInt64, b UInt64) Engine=KeeperMap('/keeper_map_retries') PRIMARY KEY a"
+    )
+
+    node.stop_clickhouse()
+    node.copy_file_to_container(
+        os.path.join(CONFIG_DIR, "fault_injection.xml"),
+        "/etc/clickhouse-server/config.d/fault_injection.xml",
+    )
+    node.start_clickhouse()
+
+    repeat_count = 10
+
+    repeat_query(
+        "INSERT INTO keeper_map_retries SELECT number, number FROM numbers(500)",
+        repeat_count,
+    )
+    repeat_query("SELECT * FROM keeper_map_retries", repeat_count)
+    repeat_query(
+        "ALTER TABLE keeper_map_retries UPDATE b = 3 WHERE a > 2", repeat_count
+    )
+    repeat_query("ALTER TABLE keeper_map_retries DELETE WHERE a > 2", repeat_count)
+    repeat_query("TRUNCATE keeper_map_retries", repeat_count)
diff --git a/tests/queries/0_stateless/02911_backup_restore_keeper_map.sh b/tests/queries/0_stateless/02911_backup_restore_keeper_map.sh
index ee070b40f6f..c04667505c3 100755
--- a/tests/queries/0_stateless/02911_backup_restore_keeper_map.sh
+++ b/tests/queries/0_stateless/02911_backup_restore_keeper_map.sh
@@ -13,20 +13,9 @@ $CLICKHOUSE_CLIENT -nm -q "
     CREATE TABLE $database_name.02911_backup_restore_keeper_map3 (key UInt64, value String) Engine=KeeperMap('/' || currentDatabase() || '/test02911_different') PRIMARY KEY key;
 "
 
-# KeeperMap table engine doesn't have internal retries for interaction with Keeper. Do it on our own, otherwise tests with overloaded server can be flaky.
-while true
-do
-    $CLICKHOUSE_CLIENT -nm -q "INSERT INTO $database_name.02911_backup_restore_keeper_map2 SELECT number, 'test' || toString(number) FROM system.numbers LIMIT 5000;
-    " 2>&1 | grep -q "KEEPER_EXCEPTION" && sleep 1 && continue
-    break
-done
+$CLICKHOUSE_CLIENT -nm -q "INSERT INTO $database_name.02911_backup_restore_keeper_map2 SELECT number, 'test' || toString(number) FROM system.numbers LIMIT 5000;"
 
-while true
-do
-    $CLICKHOUSE_CLIENT -nm -q "INSERT INTO $database_name.02911_backup_restore_keeper_map3 SELECT number, 'test' || toString(number) FROM system.numbers LIMIT 3000;
-    "  2>&1 | grep -q "KEEPER_EXCEPTION" && sleep 1 && continue
-    break
-done
+$CLICKHOUSE_CLIENT -nm -q "INSERT INTO $database_name.02911_backup_restore_keeper_map3 SELECT number, 'test' || toString(number) FROM system.numbers LIMIT 3000;"
 
 backup_path="$database_name"
 for i in $(seq 1 3); do

From a1ececb24c0a6f21d3985f5a9f8a726befef78c3 Mon Sep 17 00:00:00 2001
From: avogar <avogar@clickhouse.com>
Date: Tue, 30 Jul 2024 11:51:48 +0000
Subject: [PATCH 1070/1488] Fix use-of-unitialized-value

---
 src/Columns/ColumnAggregateFunction.cpp | 33 ++++++++++++++++++++++---
 1 file changed, 30 insertions(+), 3 deletions(-)

diff --git a/src/Columns/ColumnAggregateFunction.cpp b/src/Columns/ColumnAggregateFunction.cpp
index 9934970c868..4bc48c62eb4 100644
--- a/src/Columns/ColumnAggregateFunction.cpp
+++ b/src/Columns/ColumnAggregateFunction.cpp
@@ -332,9 +332,36 @@ void ColumnAggregateFunction::expand(const Filter & mask, bool inverted)
 {
     ensureOwnership();
     Arena & arena = createOrGetArena();
-    char * default_ptr = arena.alignedAlloc(func->sizeOfData(), func->alignOfData());
-    func->create(default_ptr);
-    expandDataByMask<char *>(data, mask, inverted, default_ptr);
+
+    if (mask.size() < data.size())
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "Mask size should be no less than data size.");
+
+    ssize_t from = data.size() - 1;
+    ssize_t index = mask.size() - 1;
+    data.resize(mask.size());
+    while (index >= 0)
+    {
+        if (!!mask[index] ^ inverted)
+        {
+            if (from < 0)
+                throw Exception(ErrorCodes::LOGICAL_ERROR, "Too many bytes in mask");
+
+            /// Copy only if it makes sense.
+            if (index != from)
+                data[index] = data[from];
+            --from;
+        }
+        else
+        {
+            data[index] = arena.alignedAlloc(func->sizeOfData(), func->alignOfData());
+            func->create(data[index]);
+        }
+
+        --index;
+    }
+
+    if (from != -1)
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "Not enough bytes in mask");
 }
 
 ColumnPtr ColumnAggregateFunction::permute(const Permutation & perm, size_t limit) const

From fd075470d6e141d5aa4d01ccefcc3d3ee04130c7 Mon Sep 17 00:00:00 2001
From: divanik <dan.ivanik@clickhouse.com>
Date: Tue, 30 Jul 2024 11:54:00 +0000
Subject: [PATCH 1071/1488] Add docs

---
 tests/integration/README.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/tests/integration/README.md b/tests/integration/README.md
index cde4cb05aec..d5137a9c148 100644
--- a/tests/integration/README.md
+++ b/tests/integration/README.md
@@ -130,6 +130,14 @@ docker build -t clickhouse/integration-test .
 ```
 
 The helper container used by the `runner` script is in `docker/test/integration/runner/Dockerfile`.
+It can be rebuild with 
+
+```
+cd docker/test/integration/runner
+docker build -t clickhouse/integration-test-runner .
+```
+
+Also you need to add option --network=host if you rebuild image for a local integration testsing.
 
 ### Adding new tests
 

From fd26672864a7e1557908b878d7daa018de20c61a Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Tue, 30 Jul 2024 13:54:30 +0200
Subject: [PATCH 1072/1488] Revert some change

---
 src/Storages/StorageKeeperMap.cpp | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/src/Storages/StorageKeeperMap.cpp b/src/Storages/StorageKeeperMap.cpp
index 09c21ae28f5..1559b442e43 100644
--- a/src/Storages/StorageKeeperMap.cpp
+++ b/src/Storages/StorageKeeperMap.cpp
@@ -35,7 +35,6 @@
 #include <Storages/StorageInMemoryMetadata.h>
 #include <Storages/checkAndGetLiteralArgument.h>
 
-#include "Common/ZooKeeper/ZooKeeperRetries.h"
 #include <Common/Base64.h>
 #include <Common/Exception.h>
 #include <Common/FailPoint.h>
@@ -44,6 +43,7 @@
 #include <Common/ZooKeeper/Types.h>
 #include <Common/ZooKeeper/ZooKeeper.h>
 #include <Common/ZooKeeper/ZooKeeperConstants.h>
+#include <Common/ZooKeeper/ZooKeeperRetries.h>
 
 #include <Backups/BackupEntriesCollector.h>
 #include <Backups/IBackupCoordination.h>
@@ -205,12 +205,12 @@ public:
             for (const auto & [key, _] : new_values)
                 key_paths.push_back(storage.fullPathForKey(key));
 
-            zkutil::ZooKeeper::MultiTryGetResponse results;
+            zkutil::ZooKeeper::MultiExistsResponse results;
 
             if constexpr (!for_update)
             {
                 if (!strict)
-                    results = zookeeper->tryGet(key_paths);
+                    results = zookeeper->exists(key_paths);
             }
 
             Coordination::Requests requests;
@@ -231,8 +231,7 @@ public:
                 {
                     if (!strict && results[i].error == Coordination::Error::ZOK)
                     {
-                        if (results[i].data != new_values[key])
-                            requests.push_back(zkutil::makeSetRequest(key_paths[i], new_values[key], -1));
+                        requests.push_back(zkutil::makeSetRequest(key_paths[i], new_values[key], -1));
                     }
                     else
                     {
@@ -242,9 +241,6 @@ public:
                 }
             }
 
-            if (requests.empty())
-                return;
-
             if (new_keys_num != 0)
             {
                 auto will_be = current_keys_num + new_keys_num;

From 0124d211ec81a3779fe0e99c868fe85be8856629 Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Tue, 30 Jul 2024 14:00:52 +0200
Subject: [PATCH 1073/1488] Better

---
 tests/integration/test_replicated_table_attach/test.py | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/tests/integration/test_replicated_table_attach/test.py b/tests/integration/test_replicated_table_attach/test.py
index 499220def2c..de60b7ec291 100644
--- a/tests/integration/test_replicated_table_attach/test.py
+++ b/tests/integration/test_replicated_table_attach/test.py
@@ -41,6 +41,7 @@ def start_clean_clickhouse():
 
 def test_startup_with_small_bg_pool(started_cluster):
     start_clean_clickhouse()
+    node.query("DROP TABLE IF EXISTS replicated_table SYNC")
     node.query(
         "CREATE TABLE replicated_table (k UInt64, i32 Int32) ENGINE=ReplicatedMergeTree('/clickhouse/replicated_table', 'r1') ORDER BY k"
     )
@@ -54,11 +55,10 @@ def test_startup_with_small_bg_pool(started_cluster):
     node.restart_clickhouse(stop_start_wait_sec=10)
     assert_values()
 
-    node.query("DROP TABLE replicated_table SYNC")
-
 
 def test_startup_with_small_bg_pool_partitioned(started_cluster):
     start_clean_clickhouse()
+    node.query("DROP TABLE IF EXISTS replicated_table_partitioned SYNC")
     node.query(
         "CREATE TABLE replicated_table_partitioned (k UInt64, i32 Int32) ENGINE=ReplicatedMergeTree('/clickhouse/replicated_table_partitioned', 'r1') ORDER BY k"
     )
@@ -81,5 +81,3 @@ def test_startup_with_small_bg_pool_partitioned(started_cluster):
 
     # check that we activate it in the end
     node.query_with_retry("INSERT INTO replicated_table_partitioned VALUES(20, 30)")
-
-    node.query("DROP TABLE replicated_table_partitioned SYNC")

From de99ee1b05e68b964535664d4197afd0944d0261 Mon Sep 17 00:00:00 2001
From: divanik <dan.ivanik@clickhouse.com>
Date: Tue, 30 Jul 2024 12:07:31 +0000
Subject: [PATCH 1074/1488] Change docs

---
 tests/integration/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/README.md b/tests/integration/README.md
index d5137a9c148..c1eb511fa44 100644
--- a/tests/integration/README.md
+++ b/tests/integration/README.md
@@ -137,7 +137,7 @@ cd docker/test/integration/runner
 docker build -t clickhouse/integration-test-runner .
 ```
 
-Also you need to add option --network=host if you rebuild image for a local integration testsing.
+If your docker configuration doesn't allow access to public internet with docker build command you may also need to add option --network=host if you rebuild image for a local integration testsing.
 
 ### Adding new tests
 

From c3e8825c8f14bb82f60c41754a021813d3dbc8aa Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Tue, 30 Jul 2024 14:23:32 +0200
Subject: [PATCH 1075/1488] Use correct order of fields in StorageURLSource

---
 src/Storages/StorageURL.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Storages/StorageURL.h b/src/Storages/StorageURL.h
index 63d01a02417..6c95cad474c 100644
--- a/src/Storages/StorageURL.h
+++ b/src/Storages/StorageURL.h
@@ -228,12 +228,12 @@ private:
     bool need_only_count;
     size_t total_rows_in_file = 0;
 
+    Poco::Net::HTTPBasicCredentials credentials;
+
     std::unique_ptr<ReadBuffer> read_buf;
     std::shared_ptr<IInputFormat> input_format;
     std::unique_ptr<QueryPipeline> pipeline;
     std::unique_ptr<PullingPipelineExecutor> reader;
-
-    Poco::Net::HTTPBasicCredentials credentials;
 };
 
 class StorageURLSink : public SinkToStorage

From 8b52d7b711d54f1d4bb5b2f39bf4aea3966f64dc Mon Sep 17 00:00:00 2001
From: Michael Stetsyuk <mike.stetsyuk@gmail.com>
Date: Tue, 30 Jul 2024 13:35:19 +0100
Subject: [PATCH 1076/1488] fxs

---
 src/IO/S3/Client.cpp                          | 19 +++++++++----------
 src/IO/S3/Client.h                            |  2 +-
 .../test_checking_s3_blobs_paranoid/test.py   | 12 ++++++++++++
 tests/integration/test_storage_delta/test.py  | 12 ++++++++++++
 4 files changed, 34 insertions(+), 11 deletions(-)

diff --git a/src/IO/S3/Client.cpp b/src/IO/S3/Client.cpp
index 7196dfa9bdc..8f037ea71be 100644
--- a/src/IO/S3/Client.cpp
+++ b/src/IO/S3/Client.cpp
@@ -385,10 +385,9 @@ Model::HeadObjectOutcome Client::HeadObject(HeadObjectRequest & request) const
 
     request.overrideURI(std::move(*bucket_uri));
 
-    if (isClientForDisk() && error.GetErrorType() == Aws::S3::S3Errors::NO_SUCH_KEY)
-        CurrentMetrics::add(CurrentMetrics::S3DiskNoKeyErrors);
-
-    return enrichErrorMessage(
+    /// The next call is NOT a recurcive call
+    /// This is a virtuall call Aws::S3::S3Client::HeadObject(const Model::HeadObjectRequest&)
+    return processRequestResult(
         HeadObject(static_cast<const Model::HeadObjectRequest&>(request)));
 }
 
@@ -409,11 +408,8 @@ Model::ListObjectsOutcome Client::ListObjects(ListObjectsRequest & request) cons
 
 Model::GetObjectOutcome Client::GetObject(GetObjectRequest & request) const
 {
-    auto resp = doRequest(request, [this](const Model::GetObjectRequest & req) { return GetObject(req); });
-    if (!resp.IsSuccess() && isClientForDisk() && resp.GetError().GetErrorType() == Aws::S3::S3Errors::NO_SUCH_KEY)
-        CurrentMetrics::add(CurrentMetrics::S3DiskNoKeyErrors);
-
-    return enrichErrorMessage(std::move(resp));
+    return processRequestResult(
+        doRequest(request, [this](const Model::GetObjectRequest & req) { return GetObject(req); }));
 }
 
 Model::AbortMultipartUploadOutcome Client::AbortMultipartUpload(AbortMultipartUploadRequest & request) const
@@ -699,11 +695,14 @@ Client::doRequestWithRetryNetworkErrors(RequestType & request, RequestFn request
 }
 
 template <typename RequestResult>
-RequestResult Client::enrichErrorMessage(RequestResult && outcome) const
+RequestResult Client::processRequestResult(RequestResult && outcome) const
 {
     if (outcome.IsSuccess() || !isClientForDisk())
         return std::forward<RequestResult>(outcome);
 
+    if (outcome.GetError().GetErrorType() == Aws::S3::S3Errors::NO_SUCH_KEY)
+        CurrentMetrics::add(CurrentMetrics::S3DiskNoKeyErrors);
+
     String enriched_message = fmt::format(
         "{} {}",
         outcome.GetError().GetMessage(),
diff --git a/src/IO/S3/Client.h b/src/IO/S3/Client.h
index 11cace4e1fd..e54953419e1 100644
--- a/src/IO/S3/Client.h
+++ b/src/IO/S3/Client.h
@@ -274,7 +274,7 @@ private:
     void insertRegionOverride(const std::string & bucket, const std::string & region) const;
 
     template <typename RequestResult>
-    RequestResult enrichErrorMessage(RequestResult && outcome) const;
+    RequestResult processRequestResult(RequestResult && outcome) const;
 
     String initial_endpoint;
     std::shared_ptr<Aws::Auth::AWSCredentialsProvider> credentials_provider;
diff --git a/tests/integration/test_checking_s3_blobs_paranoid/test.py b/tests/integration/test_checking_s3_blobs_paranoid/test.py
index 1ed70e20b79..dde636b5d29 100644
--- a/tests/integration/test_checking_s3_blobs_paranoid/test.py
+++ b/tests/integration/test_checking_s3_blobs_paranoid/test.py
@@ -689,3 +689,15 @@ def test_no_key_found_disk(cluster, broken_s3):
         "DB::Exception: The specified key does not exist. This error happened for S3 disk."
         in error
     )
+
+    s3_disk_no_key_errors_metric_value = int(
+        node.query(
+            """
+            SELECT value
+            FROM system.metrics
+            WHERE metric = 'S3DiskNoKeyErrors'
+            """
+        ).strip()
+    )
+
+    assert s3_disk_no_key_errors_metric_value > 0
diff --git a/tests/integration/test_storage_delta/test.py b/tests/integration/test_storage_delta/test.py
index d3dd7cfe52a..67cc7cdd6da 100644
--- a/tests/integration/test_storage_delta/test.py
+++ b/tests/integration/test_storage_delta/test.py
@@ -452,6 +452,18 @@ def test_restart_broken(started_cluster):
         f"SELECT count() FROM {TABLE_NAME}"
     )
 
+    s3_disk_no_key_errors_metric_value = int(
+        instance.query(
+            """
+            SELECT value
+            FROM system.metrics
+            WHERE metric = 'S3DiskNoKeyErrors'
+            """
+        ).strip()
+    )
+
+    assert s3_disk_no_key_errors_metric_value == 0
+
     minio_client.make_bucket(bucket)
 
     upload_directory(minio_client, bucket, f"/{TABLE_NAME}", "")

From d69f6cccde7633214eba48c08d7647e4ea9a40da Mon Sep 17 00:00:00 2001
From: kssenii <sumarokovakseniia@mail.ru>
Date: Tue, 30 Jul 2024 15:08:26 +0200
Subject: [PATCH 1077/1488] Fix

---
 src/Common/ThreadPoolTaskTracker.cpp | 5 ++++-
 src/IO/WriteBufferFromS3.cpp         | 9 ++++-----
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/src/Common/ThreadPoolTaskTracker.cpp b/src/Common/ThreadPoolTaskTracker.cpp
index 61d34801f7a..1697a13f780 100644
--- a/src/Common/ThreadPoolTaskTracker.cpp
+++ b/src/Common/ThreadPoolTaskTracker.cpp
@@ -19,6 +19,10 @@ TaskTracker::TaskTracker(ThreadPoolCallbackRunnerUnsafe<void> scheduler_, size_t
 
 TaskTracker::~TaskTracker()
 {
+    /// Tasks should be waited outside of dtor.
+    /// Important for WriteBufferFromS3/AzureBlobStorage, where TaskTracker is currently used.
+    chassert(finished_futures.empty() && futures.empty());
+
     safeWaitAll();
 }
 
@@ -170,4 +174,3 @@ bool TaskTracker::isAsync() const
 }
 
 }
-
diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp
index 3682e49b018..e702b4d35ad 100644
--- a/src/IO/WriteBufferFromS3.cpp
+++ b/src/IO/WriteBufferFromS3.cpp
@@ -277,12 +277,10 @@ WriteBufferFromS3::~WriteBufferFromS3()
             "The file might not be written to S3. "
             "{}.",
             getVerboseLogDetails());
-        return;
     }
-
-    /// That destructor could be call with finalized=false in case of exceptions
-    if (!finalized && !canceled)
+    else if (!finalized)
     {
+        /// That destructor could be call with finalized=false in case of exceptions
         LOG_INFO(
             log,
             "WriteBufferFromS3 is not finalized in destructor. "
@@ -291,9 +289,10 @@ WriteBufferFromS3::~WriteBufferFromS3()
             getVerboseLogDetails());
     }
 
+    /// Wait for all tasks, because they contain reference to this write buffer.
     task_tracker->safeWaitAll();
 
-    if (!multipart_upload_id.empty() && !multipart_upload_finished)
+    if (!canceled && !multipart_upload_id.empty() && !multipart_upload_finished)
     {
         LOG_WARNING(log, "WriteBufferFromS3 was neither finished nor aborted, try to abort upload in destructor. {}.", getVerboseLogDetails());
         tryToAbortMultipartUpload();

From 20faed85ca30c6352fd091e8d4d763fb98fe1311 Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Tue, 30 Jul 2024 15:18:37 +0200
Subject: [PATCH 1078/1488] Remove useless file

---
 tests/queries/0_stateless/data_minio/test :: 03215_archive.csv | 1 -
 1 file changed, 1 deletion(-)
 delete mode 100644 tests/queries/0_stateless/data_minio/test :: 03215_archive.csv

diff --git a/tests/queries/0_stateless/data_minio/test :: 03215_archive.csv b/tests/queries/0_stateless/data_minio/test :: 03215_archive.csv
deleted file mode 100644
index d00491fd7e5..00000000000
--- a/tests/queries/0_stateless/data_minio/test :: 03215_archive.csv	
+++ /dev/null
@@ -1 +0,0 @@
-1

From 21aa514c80cb463f079f9877ae97048a8b13dfbe Mon Sep 17 00:00:00 2001
From: Nikita Taranov <nikita.taranov@clickhouse.com>
Date: Tue, 30 Jul 2024 15:03:09 +0100
Subject: [PATCH 1079/1488] don't run removed tests

---
 tests/performance/scripts/entrypoint.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/performance/scripts/entrypoint.sh b/tests/performance/scripts/entrypoint.sh
index 0c3bfa550f4..db7d96ad150 100755
--- a/tests/performance/scripts/entrypoint.sh
+++ b/tests/performance/scripts/entrypoint.sh
@@ -118,7 +118,7 @@ then
     # far in the future and have unrelated test changes.
     base=$(git -C right/ch merge-base pr origin/master)
     git -C right/ch diff --name-only "$base" pr -- . | tee all-changed-files.txt
-    git -C right/ch diff --name-only "$base" pr -- tests/performance/*.xml | tee changed-test-definitions.txt
+    git -C right/ch diff --name-only --diff-filter=d "$base" pr -- tests/performance/*.xml | tee changed-test-definitions.txt
     git -C right/ch diff --name-only "$base" pr -- :!tests/performance/*.xml :!docker/test/performance-comparison | tee other-changed-files.txt
 fi
 

From 7d5c30e76cf0fd17515803fec96899f4aad1294e Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Tue, 30 Jul 2024 16:26:19 +0200
Subject: [PATCH 1080/1488] No retries when partitioned

---
 tests/integration/test_keeper_map/test.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/tests/integration/test_keeper_map/test.py b/tests/integration/test_keeper_map/test.py
index 31316af7b1e..7aee5df5746 100644
--- a/tests/integration/test_keeper_map/test.py
+++ b/tests/integration/test_keeper_map/test.py
@@ -46,7 +46,11 @@ def assert_keeper_exception_after_partition(query):
     with PartitionManager() as pm:
         pm.drop_instance_zk_connections(node)
         try:
-            error = node.query_and_get_error_with_retry(query, sleep_time=1)
+            error = node.query_and_get_error_with_retry(
+                query,
+                sleep_time=1,
+                settings={"insert_keeper_max_retries": 1, "keeper_max_retries": 1},
+            )
             assert "Coordination::Exception" in error
         except:
             print_iptables_rules()
@@ -84,7 +88,9 @@ def test_keeper_map_without_zk(started_cluster):
         node.restart_clickhouse(60)
         try:
             error = node.query_and_get_error_with_retry(
-                "SELECT * FROM test_keeper_map_without_zk", sleep_time=1
+                "SELECT * FROM test_keeper_map_without_zk",
+                sleep_time=1,
+                settings={"keeper_max_retries": 1},
             )
             assert "Failed to activate table because of connection issues" in error
         except:

From aec431f68bf16b45a6b36deb61c146e08cb4f644 Mon Sep 17 00:00:00 2001
From: heguangnan <heguangnan@bytedance.com>
Date: Tue, 30 Jul 2024 22:54:05 +0800
Subject: [PATCH 1081/1488] fix test

---
 .../0_stateless/03214_count_distinct_null_key_memory_leak.sql   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/03214_count_distinct_null_key_memory_leak.sql b/tests/queries/0_stateless/03214_count_distinct_null_key_memory_leak.sql
index 847d3742dc3..d8428ec6b4a 100644
--- a/tests/queries/0_stateless/03214_count_distinct_null_key_memory_leak.sql
+++ b/tests/queries/0_stateless/03214_count_distinct_null_key_memory_leak.sql
@@ -12,7 +12,7 @@ PARTITION BY tuple()
 ORDER BY c
 SETTINGS index_granularity = 8192, allow_nullable_key=1;
 
-INSERT INTO testnull(b,c) SELECT toString(rand64()) AS b, toString(rand64()) AS c FROM numbers(1000000)
+INSERT INTO testnull(b,c) SELECT toString(rand64()) AS b, toString(rand64()) AS c FROM numbers(1000000);
 SELECT count(distinct b) FROM testnull GROUP BY a  SETTINGS max_memory_usage = 54748364; -- {serverError MEMORY_LIMIT_EXCEEDED}
 
 DROP TABLE testnull;
\ No newline at end of file

From 83c6d97cd2aae0a3f79c2776ffc3a5691f8fd4bb Mon Sep 17 00:00:00 2001
From: maxvostrikov <max.vostrikov@clickhouse.com>
Date: Tue, 30 Jul 2024 17:34:38 +0200
Subject: [PATCH 1082/1488] squash! added somme tests in relation with
 https://github.com/ClickHouse/ClickHouse/pull/54881 with new behaviour when
 enable_named_columns_in_function_tuple=1 (default value)

---
 .../0_stateless/00309_formats.reference       | Bin 20353 -> 18666 bytes
 tests/queries/0_stateless/00309_formats.sql   |   5 +++++
 2 files changed, 5 insertions(+)

diff --git a/tests/queries/0_stateless/00309_formats.reference b/tests/queries/0_stateless/00309_formats.reference
index cab311692be229716b58af39079275d3942b01cc..f3ea45520bb50fb936caf6724e9fedf3cdd00b75 100644
GIT binary patch
delta 3052
zcmeIy$4}c(00wZIgtksdZ-w^I3JD}eL2?j6{R4XG1$t-?y;Q0c4wb5?2YL>Bj|8*#
z-Xp>6y?59U_TGDMNaN%a?SDX3#j^bK`<|bk?ZfAlN^UDiN<bJ6jY24SuCXUb!%+R!
z(?lWq$P}8rBt@#-Sf0!i4|+jw=mUMBAM}R-Fc1d8U>E{JVHgaD5e8+19DxExLIy^`
zXcz-yVH}Ky2`~{R!DN^MQ}xOSA`L-0%z&9N3ueO{m<#h@J}iKRum~2z5?IQq+n-P(
zP{A_D!g5#vD`6F^hBdGj*1>w%02^VGPHumqnY-wY76U{pY=iBv19rkL*bRGNFYJT;
zPz?v*AoeGQ5Ddc+I10z$IGlhQI0>iVG@OC6a1PGH1;h6f#3F(vxC~d|DqMr>a070_
zEw~ML;4a*Q`|v>j|8;{y1V``~p1@Oh2G8LIyo6Wq8s5NLcn9yHg%&qjk~HD*O`kk{
zMhS#wBF4@26;n)ok$HM~`}q3#2LuKMhlGZON5~bCOjLACY+QUoVp4KSYFc_mW>$7i
zZeD&tVNr2OsZv$OmRD3(RoB$k)i*RYHMg|3wRd!Ob@%l4^{WR4hlWQ+$Hpf#lT*_(
zvvczci%ZKZt842Un_JsEyL<ZwheyXJr)TFEmsi&}w|Dmf?L&1j3gH`4#3+R{X>2uP
zs~KC(*lPbltFhlLRQ$(oSyIc$t;|{F>ph;XVgHfpC$+5H%8ajnp%+Y<&DY2OORU_k
z(V7&oZmv$Gj7LgIiK)l0pFiA9KYx&VyyL(B=HYJsi3@C{mioxSQew%acGgGsx>HI?
tDE$HfH|yd|adXt)D_oo(mI%Z&<wVo8BQ16miz!<%?O^j~Y-3_4_zmv{|2Y5v

literal 20353
zcmeHP3wIP%5{6aK9zDL_&p18;axq=E-!Ke<i0?P}N)Q7X6ip_W3@R%6%l7O~+P|=0
zRo6hOl6Z`+M|N!w(_i=PTYbO&s&ea2b&tJW?SKFN*FT@{>He<Wy7Ldkn-p(W+@rWx
z@fO8f6_Y9M{>M!>@7a6Ht;zJ#Jk0d+*gt^JbdYKDu4%*X9d$a^ZOY<y#cBH;dVHti
zU5a-r-lKT0;(dzuD?Xt3pyES{4=X;R_^9GzijONkq4=cYjN(&@eZ{90pHX~P@j1m=
z#eItV6`xl;pcoVnD!!mNr<f{|VpPl&bHzfjRIC(h#YXXv;$g*s;t|E6;!(vH6^|(%
zSA0qFgyPGJCly~&d{yx^#n%<zP<&JIEycGL-%)&5@jb=&6+cisrFdHLL&c92KUVxi
z@l(Zl#RbJPik~T-Rs39WQSqGOdBr8gk>Ul#FBHF2yr}q<;@67HiYtm^#Z|>iir*-%
zDXuF{6fY}&tN5Mb_lj2(e^C5U@h8Qria#s<qKG-5+v_yg)y2WCP6NyYf5SM~eSM*H
z+wHwO?!4>nd+xpO{s$g>=;22meeCfko}78A|MWA@J~z8>|MLgJ!58LIidkNiRoxsq
zJUB8u`r@(UFP(V#<SVbf_WB!dzV-Gy@4ol`2d7Sd_|eCod^*2y=CiY(FP=NUG`jG`
zmlwbKdU<8Mdg+_B^~vRLzx)2m4?q5N_2*w8dH3%xRA9RH1%(59c8_;2-L$y2wtIcg
z<npEE(caUqjVEP(^NGdDsCR5_ZSl&!wq^g`4<>6D$6wrgio0oJnBji=r;A-&v0K--
z%ZqMZ-|En<>suYVb$zQtx30$|*-G(uTzhnP-g&=^d;Asr39~@AuK#;~vDmn-A16x*
zR`tnxBCGmdyM|AAwVs$&ebTP#ldV?ulqNAv(}O7;BueJfKp}=K<e>;<s6ri@Gl_ne
zTsXHdN%m11X0Kb!?(fB-)Jrr6DckZA0Y*fDVxTNg9;gUZ2C4$pftnE05HV~FF$*yd
zu?VpYu?n#cu?blkG71@3Wg*K$R)nk!SrxK6WKGD^kW<KG$U!;}c@gq5<W<P)kT;=7
zLqVa4p~ymkdn!UvhN22Z9f~HDX(%a_F_c*-^HAcJ%TQLKtV7v^Dh(BdDuyZxRUWD$
zR8XV}RUN7()M=<G)G^dqsPj-4p)NxW!Rk;qp-DqSp^2f%LX(H42u&H9Dm2iuff|q_
zg&rvcNufvzNm6K%LX;G$q>v>=KamanSU6UWrDN?_JXVk8gMI`72?`P<#P@i7&<`pS
zWF+WF5R#xIK}v#_1ThI}667T4Nf4BvC_z$!rUX$5suE-+=t>Zlpe#XJg0=*43F;E$
zCFn~Kn4mC0VuHp5kqIgjWG3iL5SpMgL28251hENf6XYi7O%R-*I6-oP<^<6RsuN@<
z=uQxxpgci(g7yUQ3F;H%C+JUb0Ko$U7Z7|va00;#1UC@;KyU=X69iWfd_iyq!5aj3
z5d48~#s?7ye?<5r!XFX-i10^*KO+1Q;g1M^M8t7~KO+1Q;g1M^MEE1Z9})hD9M2rk
z64B_#G0j2EQO#k^am|4Ze?<5r!XFX-i0}tWF!-P!{1M@g2!BNQBf=jM{)q5Lgg+wu
z5#f&re?<5r!XFX-i10^*KO+1Q;g1M^#G){NzzgHs7A`ETZCepq_<bdzt|o+Je}n%^
z_F<}?PTo9MWAfg?^nYv8P5#~F-%bAiqmw^gndqJb+~70xpFwZySwHlzui^Wlzva3g
z?bn_n^|2r|HoWYgHgGQ4<|1s?t^B){f4B1gOIH3H|Ae=l{<|l^H~Px`kK~0uz5?2r
zy7yx*>933?i~U2rWWJXqI8KuOcyVRaKQz-H_k$cyu3VDiwye}g+f_bcJ1@D;TzuN5
zD@s+avnf(-W3}#V+EUgwouv-u47MnGiFGeI6M7J0X>pSftOBKaoh3fnm@GvczL`Ta
z^L_N{6Fw0i&Ve~T1wJJ{6+SgSO&=q%!s~pgh07?nk!+)EGuh^{Eo58Dw&Hr*^k(>Z
zybQ>^Hqtrc{32`%3@;;YAm1kG-$5QXD0j)XNqnD@?1}G`_*RMUmH1|f@0J>DqOcTU
z*dnqtku4%yM7D@*5!oWLMaC8xTV$doTV!mJu|>uf8Czs*k+DV27CBqwY>|unY>~4?
z&K5aa<ZO|%MZp#YTNG?jutmWZ1zWIa{emqDwkX)5WQ&q5O13E3qGXGbElRd1*`j2N
zk}WE>sMw-ni;68Owy4;mVvC9`Dz>QDqGpSlEo!!?*`j8Pnk{O!sM(@si<&JOwrJR*
zVT*<>8n$TIqG5}MEgH6H*a8(t*|H6)j<RJdR32r^cBnqemMu|%l&#pR`^B~mfHnf!
z5NKnd4T3fb+AxqfKxLAz6U8>lxDpGfP)fUuE8&1TrL@Sn5)r6Ya*M<SDweV@oFyz!
zx0Dv)ERligrL+iV2@cdSxkbVQbxhfpaV0`f&6E}ySAqmJO>U7eL0wb!Wn76AR5ztX
z#+6_}jZ<1=T!|M{I=Mw62Gvg4mvJR%Q1g@)8CT*4l}~AraV2z6|Kt`49@Ic(U&fXA
zK_ygLWLya$)I+&Nf(SKH*_Ux8j!+qu78zGU3H4EFk#QxOP$lISi6>M_WnadXkV3sw
zT4Y>_DpXCSg~!FbC|e#E)l=EBEh?z8Wm{BHWy|BDk}6xVZKpC4TZt>wRQasL73!+8
zFSZg_sIE$j*tQdl#8s5ILY<Y*wv$Xd%e2!>JI}Nejl>nIuG~}N3YAyc7h8!d)L*4V
z#+A526;@hgT!|~xV!1`)3iVjom;05tLRD5;<bEZtP@Clzi7V7+WnadXxI&dyT8clA
zCKkq$GLCa;SN2|(?MdEJDqBf8Bq@g^<&dNtl9WS|a;TD?wQL*N9+T~H*}f#(6S9@-
z)5)2q7R>5#VP`KA{>G~DrX+|J4NE4^tt78W7My1Ewbk!9(?&_}d~o*M=)&sS=<M>v
zm5Y<Ji<7g<qs8?Jcj;a27DK#%m`b;CF*J`X&<}R)vx61S&R06PcsnIrdmh<8BP*wQ
z{O!QuQ!4FB<~LKp+$Iwo|J`-#7c5hFy6M(0eLPL&i0IZYvfRaUT2`Z5zoe4SyY-8F
zcfylpM8$7kzwC6<|9@Y%AW3art6;IxmL7JtP#GNker0*)+tJ$k#nthVS$ZLzof(bK
zuP)(*{UfudPn|qi&(0hj_6BRC^~=kXVJ{hQkq4Lz&aW<CUKy|R{`5fhlL1$KhU3AU
zAfHYKT=*HbRiD9}9EjOmdy(y~v9$RbTnyU56;M;zd<|10+QG%vH(#nP9qr&6Qcl`@
zoh>R#+gx;@nz8VMIrYYX7g?<00RC^)91khQIPJmG@OrgrFt^k;LYv_Lo5d@`!DxB3
zGDLsD_K3_;G)I{^%FR(>j!JV>nWNeqHPfRx<;2)BT_0Y1Xf&A9h!Y`)0|nvDH@x8n
zHr{gEJg2=CS+6-2Y@=}a^V*~wFA#5d<+f6RtRKlbl5c8*WMo}Q)|X_RN!FWW-AUK{
zDc6tAH<nejtfFNVEvsl*MawE$R?)JGmQ`k1WtLTDS!I@0W?5yHRc2XbmQ`k1WtLTL
zS>={hZdv7)Rc=}3mQ`+9<(5@$S>={hVObTHRbg2bmQ`U{6_yqMxorr-!m=tXtHQD>
zEvwS9DlMzhvMMdB(y}TotJ1P6EvwS9sw}I@vZ^er%Cf2~tID#fEUU`0sw}I@vZ^hs
z+On!GtJ<=vEvwqHsx7P9vZ^hs+OldatH!cwEUU({YAmb9vT7`=#<FTGtH!cIb=w3b
z-mP`VmKff-b>}TLyo>A3TXJ|u*PXZY@b0cX?~)z)dEQU-ex~<Ry`Ss-WbbEtKi#|e
z9#wkFH#+^?l#h3eU4N&aoAvP?vg_~ka}z(_Rkr=z)Q@KZ?!5D_oBi>gv+M8t>k<Io
zg?9a&e_am1`_i_*%K~@;;m$k#Tq3}`)~>(P&*cKVkL~(9{aiZ0vklwdB?P?7?an*>
zTu#9I-mbsX&!q*_8f<@;81VeWop<`V+<^DVU4N&aOAmPG-1T?*xeS5#(rtg2Bk;7w
zop<`VG=X>8U4N&a%M^I;-Sv0+xnzOoK(@b27kFphop<`VjDh#+U4N&aOB#6h-u8EC
z1J9b=d8ePt9C&Zv^>_NY<biklU4N&a%OCjiVfy2@0PfiG!*2xKdCL#K9dPHZe)vs+
zJ8${nw+8mS%OCg+f;;c{x%`1AYOcTI=kf=B%i#JuelCCD_YStd%O7|`=gvF*{Fero
zKk&N=XYcsA{DI$A*#0hm;Mt!$@A$d=f!}4g{!TxaKk(ZP*Wc;q@&|t3Vf(xMfhUdb
zywlI+5BxU7^>_NY{DCT;>+ke)`2)`|ZGV?P@Y@r2-syKefBb6wVB0s=_<vopxN_<6
zmcLbbZh3O}zw>`E`L+6m)Ezc_5xV{`a(e6A7cfwpXiGp`$XFl3xq_$>F>d{|a@uQ~
Lvprlx%YOd>&)TR8

diff --git a/tests/queries/0_stateless/00309_formats.sql b/tests/queries/0_stateless/00309_formats.sql
index b0939c00a10..0366cdeea5c 100644
--- a/tests/queries/0_stateless/00309_formats.sql
+++ b/tests/queries/0_stateless/00309_formats.sql
@@ -9,3 +9,8 @@ SELECT number * 246 + 10 AS n, toDate('2000-01-01') + n AS d, range(n) AS arr, a
 SELECT number * 246 + 10 AS n, toDate('2000-01-01') + n AS d, range(n) AS arr, arrayStringConcat(arrayMap(x -> reinterpretAsString(x), arr)) AS s, (n, d) AS tuple FROM system.numbers LIMIT 2 FORMAT JSON;
 SELECT number * 246 + 10 AS n, toDate('2000-01-01') + n AS d, range(n) AS arr, arrayStringConcat(arrayMap(x -> reinterpretAsString(x), arr)) AS s, (n, d) AS tuple FROM system.numbers LIMIT 2 FORMAT JSONCompact;
 SELECT number * 246 + 10 AS n, toDate('2000-01-01') + n AS d, range(n) AS arr, arrayStringConcat(arrayMap(x -> reinterpretAsString(x), arr)) AS s, (n, d) AS tuple FROM system.numbers LIMIT 2 FORMAT XML;
+
+SET enable_named_columns_in_function_tuple = 1;
+
+SELECT 36 AS n, toDate('2000-01-01') + n AS d, (n, d) AS tuple FROM system.numbers LIMIT 1 FORMAT RowBinaryWithNamesAndTypes;
+SELECT number * 246 + 10 AS n, toDate('2000-01-01') + n AS d, (n, d) AS tuple FROM system.numbers LIMIT 1 FORMAT TabSeparatedWithNamesAndTypes;

From a70cdb8bba5503f3723a2e29957617ea06106c4d Mon Sep 17 00:00:00 2001
From: Igor Nikonov <igor@clickhouse.com>
Date: Tue, 30 Jul 2024 15:37:59 +0000
Subject: [PATCH 1083/1488] Add comment regarding default value for
 force_connected

---
 src/Client/ConnectionEstablisher.h | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/Client/ConnectionEstablisher.h b/src/Client/ConnectionEstablisher.h
index 304ec4d34b4..ff071e59aea 100644
--- a/src/Client/ConnectionEstablisher.h
+++ b/src/Client/ConnectionEstablisher.h
@@ -24,8 +24,12 @@ public:
                           const QualifiedTableName * table_to_check = nullptr);
 
     /// Establish connection and save it in result, write possible exception message in fail_message.
-    /// The connection is returned from the pool, it can be stale. Use force_connected flag
-    /// to ensure that connection is working one
+    /// The connection is returned from connection pool and it can be stale. Use force_connected flag to ensure that connection is working one.
+    /// NOTE: force_connected is false by default due to the following consideration ...
+    ///      When true, it implies sending a Ping packet to another peer and, if it fails - reestablishing the connection.
+    ///      Ping-Pong round trip can be unnecessary in case of connection is still alive.
+    ///      So, the optimistic approach is used by default. In this case, stale connections can be handled by retrying,
+    ///      - see ConnectionPoolWithFailover, as example
     void run(TryResult & result, std::string & fail_message, bool force_connected = false);
 
     /// Set async callback that will be called when reading from socket blocks.

From aa26291ff25b16539efa3d50a540fc32a05b702d Mon Sep 17 00:00:00 2001
From: Michael Stetsyuk <mike.stetsyuk@gmail.com>
Date: Tue, 30 Jul 2024 18:15:24 +0100
Subject: [PATCH 1084/1488] fxs

---
 src/Databases/DatabaseReplicated.cpp          | 57 +++++++++++++++----
 src/Databases/DatabaseReplicatedWorker.cpp    | 21 +++++--
 src/Databases/DatabaseReplicatedWorker.h      |  4 +-
 src/Storages/System/StorageSystemClusters.cpp |  6 +-
 .../test_recovery_time_metric/test.py         | 36 ++++++++++--
 5 files changed, 98 insertions(+), 26 deletions(-)

diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp
index b11b9382732..06cea65d62e 100644
--- a/src/Databases/DatabaseReplicated.cpp
+++ b/src/Databases/DatabaseReplicated.cpp
@@ -12,6 +12,7 @@
 #include <Common/ZooKeeper/KeeperException.h>
 #include <Common/ZooKeeper/Types.h>
 #include <Common/ZooKeeper/ZooKeeper.h>
+#include <Common/ZooKeeper/IKeeper.h>
 #include <Common/PoolId.h>
 #include <Core/ServerSettings.h>
 #include <Core/Settings.h>
@@ -340,31 +341,63 @@ ClusterPtr DatabaseReplicated::getClusterImpl(bool all_groups) const
 
 ReplicasInfo DatabaseReplicated::tryGetReplicasInfo(const ClusterPtr & cluster_) const
 {
-    ReplicasInfo res;
+    Strings paths_get, paths_exists;
+
+    paths_get.emplace_back(fs::path(zookeeper_path) / "max_log_ptr");
 
-    auto zookeeper = getZooKeeper();
     const auto & addresses_with_failover = cluster_->getShardsAddresses();
     const auto & shards_info = cluster_->getShardsInfo();
-
+    for (size_t shard_index = 0; shard_index < shards_info.size(); ++shard_index)
+    {
+        for (const auto & replica : addresses_with_failover[shard_index])
+        {
+            String full_name = getFullReplicaName(replica.database_shard_name, replica.database_replica_name);
+            paths_exists.emplace_back(fs::path(zookeeper_path) / "replicas" / full_name / "active");
+            paths_get.emplace_back(fs::path(zookeeper_path) / "replicas" / full_name / "log_ptr");
+        }
+    }
+    
     try
     {
-        UInt32 max_log_ptr = parse<UInt32>(zookeeper->get(zookeeper_path + "/max_log_ptr"));
+        auto current_zookeeper = getZooKeeper();
+        auto get_res = current_zookeeper->get(paths_get);
+        auto exist_res = current_zookeeper->exists(paths_exists);
+        chassert(get_res.size() == exist_res.size() + 1);
 
+        auto max_log_ptr_zk = get_res[0];
+        if (max_log_ptr_zk.error != Coordination::Error::ZOK)
+            throw Coordination::Exception(max_log_ptr_zk.error);
+
+        UInt32 max_log_ptr = parse<UInt32>(max_log_ptr_zk.data);
+
+        ReplicasInfo replicas_info;
+        replicas_info.resize(exist_res.size());
+
+        size_t global_replica_index = 0;
         for (size_t shard_index = 0; shard_index < shards_info.size(); ++shard_index)
         {
             for (const auto & replica : addresses_with_failover[shard_index])
             {
-                String full_name = getFullReplicaName(replica.database_shard_name, replica.database_replica_name);
-                UInt32 log_ptr = parse<UInt32>(zookeeper->get(fs::path(zookeeper_path) / "replicas" / full_name / "log_ptr"));
-                bool is_active = zookeeper->exists(fs::path(zookeeper_path) / "replicas" / full_name / "active");
-                res.push_back(ReplicaInfo{
-                    .is_active = is_active,
-                    .replication_lag = max_log_ptr - log_ptr,
+                auto replica_active = exist_res[global_replica_index];
+                auto replica_log_ptr = get_res[global_replica_index + 1];
+
+                if (replica_active.error != Coordination::Error::ZOK && replica_active.error != Coordination::Error::ZNONODE)
+                    throw Coordination::Exception(replica_active.error);
+
+                if (replica_log_ptr.error != Coordination::Error::ZOK)
+                    throw Coordination::Exception(replica_log_ptr.error);
+
+                replicas_info[global_replica_index] = ReplicaInfo{
+                    .is_active = replica_active.error == Coordination::Error::ZOK,
+                    .replication_lag = max_log_ptr - parse<UInt32>(replica_log_ptr.data),
                     .recovery_time = replica.is_local ? ddl_worker->getCurrentInitializationDurationMs() : 0,
-                });
+                };
+
+                ++global_replica_index;
             }
         }
-        return res;
+    
+        return replicas_info;
     } catch (...)
     {
         tryLogCurrentException(log);
diff --git a/src/Databases/DatabaseReplicatedWorker.cpp b/src/Databases/DatabaseReplicatedWorker.cpp
index a9a74c5f56a..4e7408aa96e 100644
--- a/src/Databases/DatabaseReplicatedWorker.cpp
+++ b/src/Databases/DatabaseReplicatedWorker.cpp
@@ -32,8 +32,11 @@ DatabaseReplicatedDDLWorker::DatabaseReplicatedDDLWorker(DatabaseReplicated * db
 
 bool DatabaseReplicatedDDLWorker::initializeMainThread()
 {
-    initialization_duration_timer.restart();
-    initializing.store(true, std::memory_order_release);
+    {
+        std::lock_guard lock(initialization_duration_timer_mutex);
+        initialization_duration_timer.emplace();
+        initialization_duration_timer->start();
+    }
 
     while (!stop_flag)
     {
@@ -72,7 +75,10 @@ bool DatabaseReplicatedDDLWorker::initializeMainThread()
 
             initializeReplication();
             initialized = true;
-            initializing.store(false, std::memory_order_relaxed);
+            {
+                std::lock_guard lock(initialization_duration_timer_mutex);
+                initialization_duration_timer.reset();
+            }
             return true;
         }
         catch (...)
@@ -82,7 +88,11 @@ bool DatabaseReplicatedDDLWorker::initializeMainThread()
         }
     }
 
-    initializing.store(false, std::memory_order_relaxed);
+    {
+        std::lock_guard lock(initialization_duration_timer_mutex);
+        initialization_duration_timer.reset();
+    }
+
     return false;
 }
 
@@ -466,7 +476,8 @@ UInt32 DatabaseReplicatedDDLWorker::getLogPointer() const
 
 UInt64 DatabaseReplicatedDDLWorker::getCurrentInitializationDurationMs() const
 {
-    return initializing.load(std::memory_order_acquire) ? initialization_duration_timer.elapsedMilliseconds() : 0;
+    std::lock_guard lock(initialization_duration_timer_mutex);
+    return initialization_duration_timer ? initialization_duration_timer->elapsedMilliseconds() : 0;
 }
 
 }
diff --git a/src/Databases/DatabaseReplicatedWorker.h b/src/Databases/DatabaseReplicatedWorker.h
index 3e5887be825..2309c831839 100644
--- a/src/Databases/DatabaseReplicatedWorker.h
+++ b/src/Databases/DatabaseReplicatedWorker.h
@@ -59,8 +59,8 @@ private:
     /// It will remove "active" node when database is detached
     zkutil::EphemeralNodeHolderPtr active_node_holder;
 
-    Stopwatch initialization_duration_timer;
-    std::atomic<bool> initializing = false;
+    std::optional<Stopwatch> initialization_duration_timer;
+    mutable std::mutex initialization_duration_timer_mutex;
 };
 
 }
diff --git a/src/Storages/System/StorageSystemClusters.cpp b/src/Storages/System/StorageSystemClusters.cpp
index 0da4bd70cbd..4b9802c9089 100644
--- a/src/Storages/System/StorageSystemClusters.cpp
+++ b/src/Storages/System/StorageSystemClusters.cpp
@@ -71,7 +71,7 @@ void StorageSystemClusters::writeCluster(MutableColumns & res_columns, const Nam
     const auto & shards_info = cluster->getShardsInfo();
     const auto & addresses_with_failover = cluster->getShardsAddresses();
 
-    size_t replica_idx = 0;
+    size_t global_replica_idx = 0;
     for (size_t shard_index = 0; shard_index < shards_info.size(); ++shard_index)
     {
         const auto & shard_info = shards_info[shard_index];
@@ -108,7 +108,7 @@ void StorageSystemClusters::writeCluster(MutableColumns & res_columns, const Nam
             }
             else
             {
-                const auto & replica_info = replicas_info[replica_idx++];
+                const auto & replica_info = replicas_info[global_replica_idx];
                 res_columns[i++]->insert(replica_info.is_active);
                 res_columns[i++]->insert(replica_info.replication_lag);
                 if (replica_info.recovery_time != 0)
@@ -116,6 +116,8 @@ void StorageSystemClusters::writeCluster(MutableColumns & res_columns, const Nam
                 else
                     res_columns[i++]->insertDefault();
             }
+            
+            ++global_replica_idx;
         }
     }
 }
diff --git a/tests/integration/test_recovery_time_metric/test.py b/tests/integration/test_recovery_time_metric/test.py
index 4dad844b950..8f369d7759c 100644
--- a/tests/integration/test_recovery_time_metric/test.py
+++ b/tests/integration/test_recovery_time_metric/test.py
@@ -5,7 +5,6 @@ cluster = ClickHouseCluster(__file__)
 node = cluster.add_instance(
     "node",
     main_configs=["configs/config.xml"],
-    with_zookeeper=True,
     stay_alive=True,
 )
 
@@ -21,9 +20,36 @@ def start_cluster():
 
 def test_recovery_time_metric(start_cluster):
     node.query(
-        "CREATE DATABASE rdb ENGINE = Replicated('/test/test_recovery_time_metric', 'shard1', 'replica1');"
+        """
+        CREATE DATABASE rdb
+        ENGINE = Replicated('/test/test_recovery_time_metric', 'shard1', 'replica1')
+        """
     )
-    node.query("CREATE TABLE rdb.t (x UInt32) ENGINE = MergeTree ORDER BY x;")
-    node.exec_in_container(["bash", "-c", "rm /var/lib/clickhouse/metadata/rdb/t.sql"])
+
+    node.query(
+        """
+        CREATE TABLE rdb.t
+        (
+            `x` UInt32
+        )
+        ENGINE = MergeTree
+        ORDER BY x
+        """
+    )
+
+    node.exec_in_container(
+        ["bash", "-c", "rm /var/lib/clickhouse/metadata/rdb/t.sql"]
+    )
+
     node.restart_clickhouse()
-    assert node.query("SELECT any(recovery_time) FROM system.clusters;") != "0\n"
+
+    ret = int(
+        node.query(
+            """
+            SELECT recovery_time
+            FROM system.clusters
+            WHERE cluster = 'rdb'
+            """
+        ).strip()
+    )
+    assert ret > 0

From 7e51e9962c34320a9c60ba6abcf8b38cf517e86c Mon Sep 17 00:00:00 2001
From: Alex Katsman <alex.katsman@clickhouse.com>
Date: Tue, 30 Jul 2024 17:20:07 +0000
Subject: [PATCH 1085/1488] Fix WriteBuffer destructor when finalize has failed
 for MergeTreeDeduplicationLog::shutdown

---
 src/Storages/MergeTree/MergeTreeDeduplicationLog.cpp | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/Storages/MergeTree/MergeTreeDeduplicationLog.cpp b/src/Storages/MergeTree/MergeTreeDeduplicationLog.cpp
index 22ff9b7194f..a8110500f13 100644
--- a/src/Storages/MergeTree/MergeTreeDeduplicationLog.cpp
+++ b/src/Storages/MergeTree/MergeTreeDeduplicationLog.cpp
@@ -341,15 +341,19 @@ void MergeTreeDeduplicationLog::shutdown()
     stopped = true;
     if (current_writer)
     {
+        /// If an error has occurred during finalize, we'd like to have the exception set for reset.
+        /// Otherwise, we'll be in a situation when a finalization didn't happen, and we didn't get
+        /// any error, causing logical error (see ~MemoryBuffer()).
         try
         {
             current_writer->finalize();
+            current_writer.reset();
         }
         catch (...)
         {
             tryLogCurrentException(__PRETTY_FUNCTION__);
+            current_writer.reset();
         }
-        current_writer.reset();
     }
 }
 

From 51af0d305c9959fb4870bd8a57035d48207648b2 Mon Sep 17 00:00:00 2001
From: Alexander Gololobov <davenger@clickhouse.com>
Date: Tue, 30 Jul 2024 20:00:23 +0200
Subject: [PATCH 1086/1488] Reduce number of tested combinations

---
 tests/queries/0_stateless/02473_multistep_prewhere.python | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/02473_multistep_prewhere.python b/tests/queries/0_stateless/02473_multistep_prewhere.python
index a942568233c..11095202039 100644
--- a/tests/queries/0_stateless/02473_multistep_prewhere.python
+++ b/tests/queries/0_stateless/02473_multistep_prewhere.python
@@ -193,7 +193,7 @@ def main():
     url = os.environ["CLICKHOUSE_URL"] + "&max_threads=1"
 
     default_index_granularity = 10
-    total_rows = 8 * default_index_granularity
+    total_rows = 7 * default_index_granularity
     step = default_index_granularity
     session = requests.Session()
     for index_granularity in [

From dacf044c3dee65d799242b7f4846f7d6d8b2bd34 Mon Sep 17 00:00:00 2001
From: robot-clickhouse <robot-clickhouse@users.noreply.github.com>
Date: Tue, 30 Jul 2024 19:20:52 +0000
Subject: [PATCH 1087/1488] Update version_date.tsv and changelogs after
 v24.7.1.2915-stable

---
 SECURITY.md                            |   3 +-
 docker/keeper/Dockerfile               |   2 +-
 docker/server/Dockerfile.alpine        |   2 +-
 docker/server/Dockerfile.ubuntu        |   2 +-
 docs/changelogs/v24.7.1.2915-stable.md | 524 +++++++++++++++++++++++++
 utils/list-versions/version_date.tsv   |   1 +
 6 files changed, 530 insertions(+), 4 deletions(-)
 create mode 100644 docs/changelogs/v24.7.1.2915-stable.md

diff --git a/SECURITY.md b/SECURITY.md
index 53328b6e16b..8930dc96f8a 100644
--- a/SECURITY.md
+++ b/SECURITY.md
@@ -14,9 +14,10 @@ The following versions of ClickHouse server are currently supported with securit
 
 | Version | Supported |
 |:-|:-|
+| 24.7 | ✔️ |
 | 24.6 | ✔️ |
 | 24.5 | ✔️ |
-| 24.4 | ✔️ |
+| 24.4 | ❌ |
 | 24.3 | ✔️ |
 | 24.2 | ❌ |
 | 24.1 | ❌ |
diff --git a/docker/keeper/Dockerfile b/docker/keeper/Dockerfile
index c59ef1b919a..e99c86267f9 100644
--- a/docker/keeper/Dockerfile
+++ b/docker/keeper/Dockerfile
@@ -34,7 +34,7 @@ RUN arch=${TARGETARCH:-amd64} \
 # lts / testing / prestable / etc
 ARG REPO_CHANNEL="stable"
 ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}"
-ARG VERSION="24.6.2.17"
+ARG VERSION="24.7.1.2915"
 ARG PACKAGES="clickhouse-keeper"
 ARG DIRECT_DOWNLOAD_URLS=""
 
diff --git a/docker/server/Dockerfile.alpine b/docker/server/Dockerfile.alpine
index 240df79aeb1..fb562b911a3 100644
--- a/docker/server/Dockerfile.alpine
+++ b/docker/server/Dockerfile.alpine
@@ -32,7 +32,7 @@ RUN arch=${TARGETARCH:-amd64} \
 # lts / testing / prestable / etc
 ARG REPO_CHANNEL="stable"
 ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}"
-ARG VERSION="24.6.2.17"
+ARG VERSION="24.7.1.2915"
 ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
 ARG DIRECT_DOWNLOAD_URLS=""
 
diff --git a/docker/server/Dockerfile.ubuntu b/docker/server/Dockerfile.ubuntu
index ac64655991a..51f4e6a0f40 100644
--- a/docker/server/Dockerfile.ubuntu
+++ b/docker/server/Dockerfile.ubuntu
@@ -28,7 +28,7 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list
 
 ARG REPO_CHANNEL="stable"
 ARG REPOSITORY="deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb ${REPO_CHANNEL} main"
-ARG VERSION="24.6.2.17"
+ARG VERSION="24.7.1.2915"
 ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
 
 #docker-official-library:off
diff --git a/docs/changelogs/v24.7.1.2915-stable.md b/docs/changelogs/v24.7.1.2915-stable.md
new file mode 100644
index 00000000000..abffbe58bfc
--- /dev/null
+++ b/docs/changelogs/v24.7.1.2915-stable.md
@@ -0,0 +1,524 @@
+---
+sidebar_position: 1
+sidebar_label: 2024
+---
+
+# 2024 Changelog
+
+### ClickHouse release v24.7.1.2915-stable (a37d2d43da7) FIXME as compared to v24.7.1.1-new (aa023477a92)
+
+#### Backward Incompatible Change
+* Change binary serialization of Variant data type: add `compact` mode to avoid writing the same discriminator multiple times for granules with single variant or with only NULL values. Add MergeTree setting `use_compact_variant_discriminators_serialization` that is enabled by default. Note that Variant type is still experimental and backward-incompatible change in serialization is ok. [#62774](https://github.com/ClickHouse/ClickHouse/pull/62774) ([Kruglov Pavel](https://github.com/Avogar)).
+* Forbid `CREATE MATERIALIZED VIEW ... ENGINE Replicated*MergeTree POPULATE AS SELECT ...` with Replicated databases. [#63963](https://github.com/ClickHouse/ClickHouse/pull/63963) ([vdimir](https://github.com/vdimir)).
+* `clickhouse-keeper-client` will only accept paths in string literals, such as `ls '/hello/world'`, not bare strings such as `ls /hello/world`. [#65494](https://github.com/ClickHouse/ClickHouse/pull/65494) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Metric `KeeperOutstandingRequets` was renamed to `KeeperOutstandingRequests`. This fixes a typo reported in [#66179](https://github.com/ClickHouse/ClickHouse/issues/66179). [#66206](https://github.com/ClickHouse/ClickHouse/pull/66206) ([Robert Schulze](https://github.com/rschu1ze)).
+* Remove `is_deterministic` field from the `system.functions` table. [#66630](https://github.com/ClickHouse/ClickHouse/pull/66630) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+
+#### New Feature
+* Extend function `tuple` to construct named tuples in query. Introduce function `tupleNames` to extract names from tuples. [#54881](https://github.com/ClickHouse/ClickHouse/pull/54881) ([Amos Bird](https://github.com/amosbird)).
+* `ASOF JOIN` support for `full_sorting_join` algorithm Close [#54493](https://github.com/ClickHouse/ClickHouse/issues/54493). [#55051](https://github.com/ClickHouse/ClickHouse/pull/55051) ([vdimir](https://github.com/vdimir)).
+* A new table function, `fuzzQuery,` was added. This function allows you to modify a given query string with random variations. Example: `SELECT query FROM fuzzQuery('SELECT 1');`. [#62103](https://github.com/ClickHouse/ClickHouse/pull/62103) ([pufit](https://github.com/pufit)).
+* Add new window function `percent_rank`. [#62747](https://github.com/ClickHouse/ClickHouse/pull/62747) ([lgbo](https://github.com/lgbo-ustc)).
+* Support JWT authentication in `clickhouse-client`. [#62829](https://github.com/ClickHouse/ClickHouse/pull/62829) ([Konstantin Bogdanov](https://github.com/thevar1able)).
+* Add SQL functions `changeYear`, `changeMonth`, `changeDay`, `changeHour`, `changeMinute`, `changeSecond`. For example, `SELECT changeMonth(toDate('2024-06-14'), 7)` returns date `2024-07-14`. [#63186](https://github.com/ClickHouse/ClickHouse/pull/63186) ([cucumber95](https://github.com/cucumber95)).
+* Introduce startup scripts, which allow the execution of preconfigured queries at the startup stage. [#64889](https://github.com/ClickHouse/ClickHouse/pull/64889) ([pufit](https://github.com/pufit)).
+* Support accept_invalid_certificate in client's config in order to allow for client to connect over secure TCP to a server running with self-signed certificate - can be used as a shorthand for corresponding `openSSL` client settings `verificationMode=none` + `invalidCertificateHandler.name=AcceptCertificateHandler`. [#65238](https://github.com/ClickHouse/ClickHouse/pull/65238) ([peacewalker122](https://github.com/peacewalker122)).
+* Add system.error_log which contains history of error values from table system.errors, periodically flushed to disk. [#65381](https://github.com/ClickHouse/ClickHouse/pull/65381) ([Pablo Marcos](https://github.com/pamarcos)).
+* Add aggregate function `groupConcat`. About the same as `arrayStringConcat( groupArray(column), ',')` Can receive 2 parameters: a string delimiter and the number of elements to be processed. [#65451](https://github.com/ClickHouse/ClickHouse/pull/65451) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
+* Add AzureQueue storage. [#65458](https://github.com/ClickHouse/ClickHouse/pull/65458) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Add a new setting to disable/enable writing page index into parquet files. [#65475](https://github.com/ClickHouse/ClickHouse/pull/65475) ([lgbo](https://github.com/lgbo-ustc)).
+* Allow system administrators to configure `logger.console_log_level`. [#65559](https://github.com/ClickHouse/ClickHouse/pull/65559) ([Azat Khuzhin](https://github.com/azat)).
+* Automatically append a wildcard `*` to the end of a directory path with table function `file`. [#66019](https://github.com/ClickHouse/ClickHouse/pull/66019) ([Zhidong (David) Guo](https://github.com/Gun9niR)).
+* Add `--memory-usage` option to client in non interactive mode. [#66393](https://github.com/ClickHouse/ClickHouse/pull/66393) ([vdimir](https://github.com/vdimir)).
+
+#### Performance Improvement
+* Enable `optimize_functions_to_subcolumns` by default. [#58661](https://github.com/ClickHouse/ClickHouse/pull/58661) ([Anton Popov](https://github.com/CurtizJ)).
+* Replace int to string algorithm with a faster one (from a modified amdn/itoa to a modified jeaiii/itoa). [#61661](https://github.com/ClickHouse/ClickHouse/pull/61661) ([Raúl Marín](https://github.com/Algunenano)).
+* Sizes of hash tables created by join (`parallel_hash` algorithm) is collected and cached now. This information will be used to preallocate space in hash tables for subsequent query executions and save time on hash table resizes. [#64553](https://github.com/ClickHouse/ClickHouse/pull/64553) ([Nikita Taranov](https://github.com/nickitat)).
+* Optimized queries with `ORDER BY` primary key and `WHERE` that have a condition with high selectivity by using of buffering. It is controlled by setting `read_in_order_use_buffering` (enabled by default) and can increase memory usage of query. [#64607](https://github.com/ClickHouse/ClickHouse/pull/64607) ([Anton Popov](https://github.com/CurtizJ)).
+* Improve performance of loading `plain_rewritable` metadata. [#65634](https://github.com/ClickHouse/ClickHouse/pull/65634) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Attaching tables on read-only disks will use fewer resources by not loading outdated parts. [#65635](https://github.com/ClickHouse/ClickHouse/pull/65635) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Support minmax hyperrectangle for Set indices. [#65676](https://github.com/ClickHouse/ClickHouse/pull/65676) ([AntiTopQuark](https://github.com/AntiTopQuark)).
+* Unload primary index of outdated parts to reduce total memory usage. [#65852](https://github.com/ClickHouse/ClickHouse/pull/65852) ([Anton Popov](https://github.com/CurtizJ)).
+* Functions `replaceRegexpAll` and `replaceRegexpOne` are now significantly faster if the pattern is trivial, i.e. contains no metacharacters, pattern classes, flags, grouping characters etc. (Thanks to Taiyang Li). [#66185](https://github.com/ClickHouse/ClickHouse/pull/66185) ([Robert Schulze](https://github.com/rschu1ze)).
+
+#### Improvement
+* Support rocksdb as backend storage of keeper. [#56626](https://github.com/ClickHouse/ClickHouse/pull/56626) ([Han Fei](https://github.com/hanfei1991)).
+* The setting `optimize_trivial_insert_select` is disabled by default. In most cases, it should be beneficial. Nevertheless, if you are seeing slower INSERT SELECT or increased memory usage, you can enable it back or `SET compatibility = '24.6'`. [#58970](https://github.com/ClickHouse/ClickHouse/pull/58970) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Print stacktrace and diagnostic info if `clickhouse-client` or `clickhouse-local` crashes. [#61109](https://github.com/ClickHouse/ClickHouse/pull/61109) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* The result of `SHOW INDEX | INDEXES | INDICES | KEYS` was previously sorted by the primary key column names. Since this was unintuitive, the result is now sorted by the position of the primary key columns within the primary key. [#61131](https://github.com/ClickHouse/ClickHouse/pull/61131) ([Robert Schulze](https://github.com/rschu1ze)).
+* This PR changes how deduplication for MV works. Fixed a lot of cases like: - on destination table: data is split for 2 or more blocks and that blocks is considered as duplicate when that block is inserted in parallel. - on MV destination table: the equal blocks are deduplicated, that happens when MV often produces equal data as a result for different input data due to performing aggregation. - on MV destination table: the equal blocks which comes from different MV are deduplicated. [#61601](https://github.com/ClickHouse/ClickHouse/pull/61601) ([Sema Checherinda](https://github.com/CheSema)).
+* Allow matching column names in a case insensitive manner when reading json files (`input_format_json_case_insensitive_column_matching`). [#61750](https://github.com/ClickHouse/ClickHouse/pull/61750) ([kevinyhzou](https://github.com/KevinyhZou)).
+* Support reading partitioned data DeltaLake data. Infer DeltaLake schema by reading metadata instead of data. [#63201](https://github.com/ClickHouse/ClickHouse/pull/63201) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* In composable protocols TLS layer accepted only `certificateFile` and `privateKeyFile` parameters. https://clickhouse.com/docs/en/operations/settings/composable-protocols. [#63985](https://github.com/ClickHouse/ClickHouse/pull/63985) ([Anton Ivashkin](https://github.com/ianton-ru)).
+* Make an interactive client for clickhouse-disks, add local disk from the local directory. Fixes [#56791](https://github.com/ClickHouse/ClickHouse/issues/56791). [#64446](https://github.com/ClickHouse/ClickHouse/pull/64446) ([Daniil Ivanik](https://github.com/divanik)).
+* Added profile event `SelectQueriesWithPrimaryKeyUsage` which indicates how many SELECT queries use the primary key to evaluate the WHERE clause. [#64492](https://github.com/ClickHouse/ClickHouse/pull/64492) ([0x01f](https://github.com/0xfei)).
+* `StorageS3Queue` related fixes and improvements. Deduce a default value of `s3queue_processing_threads_num` according to the number of physical cpu cores on the server (instead of the previous default value as 1). Set default value of `s3queue_loading_retries` to 10. Fix possible vague "Uncaught exception" in exception column of `system.s3queue`. Do not increment retry count on `MEMORY_LIMIT_EXCEEDED` exception. Move files commit to a stage after insertion into table fully finished to avoid files being commited while not inserted. Add settings `s3queue_max_processed_files_before_commit`, `s3queue_max_processed_rows_before_commit`, `s3queue_max_processed_bytes_before_commit`, `s3queue_max_processing_time_sec_before_commit`, to better control commit and flush time. [#65046](https://github.com/ClickHouse/ClickHouse/pull/65046) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fixed broken multiple columns aggregation on s390x. [#65062](https://github.com/ClickHouse/ClickHouse/pull/65062) ([Harry Lee](https://github.com/HarryLeeIBM)).
+* Support aliases in parametrized view function (only new analyzer). [#65190](https://github.com/ClickHouse/ClickHouse/pull/65190) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* S3. reduce retires time for queries, increase retries count for backups. 8.5 minutes and 100 retires for queries, 1.2 hours and 1000 retries for backup restore. [#65232](https://github.com/ClickHouse/ClickHouse/pull/65232) ([Sema Checherinda](https://github.com/CheSema)).
+* Updated to mask account key in logs in azureBlobStorage. [#65273](https://github.com/ClickHouse/ClickHouse/pull/65273) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)).
+* Partition pruning for `IN` predicates when filter expression is a part of `PARTITION BY` expression. [#65335](https://github.com/ClickHouse/ClickHouse/pull/65335) ([Eduard Karacharov](https://github.com/korowa)).
+* Add system tables with main information about all detached tables. [#65400](https://github.com/ClickHouse/ClickHouse/pull/65400) ([Konstantin Morozov](https://github.com/k-morozov)).
+* Add support for `cluster_for_parallel_replicas` when using custom key parallel replicas. It allows you to use parallel replicas with custom key with MergeTree tables. [#65453](https://github.com/ClickHouse/ClickHouse/pull/65453) ([Antonio Andelic](https://github.com/antonio2368)).
+* Support query plan LIMIT optimization. Support LIMIT pushdown for PostgreSQL storage and table function. [#65454](https://github.com/ClickHouse/ClickHouse/pull/65454) ([Maksim Kita](https://github.com/kitaisreal)).
+* Arraymin/max can be applicable to all data types that are comparable. [#65455](https://github.com/ClickHouse/ClickHouse/pull/65455) ([pn](https://github.com/chloro-pn)).
+* Improved memory accounting for cgroups v2 to exclude the amount occupied by the page cache. [#65470](https://github.com/ClickHouse/ClickHouse/pull/65470) ([Nikita Taranov](https://github.com/nickitat)).
+* Do not create format settings for each rows when serializing chunks to insert to EmbeddedRocksDB table. [#65474](https://github.com/ClickHouse/ClickHouse/pull/65474) ([Duc Canh Le](https://github.com/canhld94)).
+* Fixed out-of-range exception in parsing Dwarf5 on s390x. [#65501](https://github.com/ClickHouse/ClickHouse/pull/65501) ([Harry Lee](https://github.com/HarryLeeIBM)).
+* Reduce `clickhouse-local` prompt to just `:)`. `getFQDNOrHostName()` takes too long on macOS, and we don't want a hostname in the prompt for `clickhouse-local` anyway. [#65510](https://github.com/ClickHouse/ClickHouse/pull/65510) ([Konstantin Bogdanov](https://github.com/thevar1able)).
+* Avoid printing a message from jemalloc about per-CPU arenas on low-end virtual machines. [#65532](https://github.com/ClickHouse/ClickHouse/pull/65532) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Disable filesystem cache background download by default. It will be enabled back when we fix the issue with possible "Memory limit exceeded" because memory deallocation is done outside of query context (while buffer is allocated inside of query context) if we use background download threads. Plus we need to add a separate setting to define max size to download for background workers (currently it is limited by max_file_segment_size, which might be too big). [#65534](https://github.com/ClickHouse/ClickHouse/pull/65534) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Add new option to config `<config_reload_interval_ms>` which allow to specify how often clickhouse will reload config. [#65545](https://github.com/ClickHouse/ClickHouse/pull/65545) ([alesapin](https://github.com/alesapin)).
+* Implement binary encoding for ClickHouse data types and add its specification in docs. Use it in Dynamic binary serialization, allow to use it in RowBinaryWithNamesAndTypes and Native formats under settings. [#65546](https://github.com/ClickHouse/ClickHouse/pull/65546) ([Kruglov Pavel](https://github.com/Avogar)).
+* Improved ZooKeeper load balancing. The current session doesn't expire until the optimal nodes become available despite `fallback_session_lifetime`. Added support for AZ-aware balancing. [#65570](https://github.com/ClickHouse/ClickHouse/pull/65570) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Server settings `compiled_expression_cache_size` and `compiled_expression_cache_elements_size` are now shown in `system.server_settings`. [#65584](https://github.com/ClickHouse/ClickHouse/pull/65584) ([Robert Schulze](https://github.com/rschu1ze)).
+* When lightweight delete happens on a table with projection(s), users have choices either throw an exception (by default) or drop the projection lightweight delete would happen. [#65594](https://github.com/ClickHouse/ClickHouse/pull/65594) ([jsc0218](https://github.com/jsc0218)).
+* Add support for user identification based on x509 SubjectAltName extension. [#65626](https://github.com/ClickHouse/ClickHouse/pull/65626) ([Anton Kozlov](https://github.com/tonickkozlov)).
+* `clickhouse-local` will respect the `max_server_memory_usage` and `max_server_memory_usage_to_ram_ratio` from the configuration file. It will also set the max memory usage to 90% of the system memory by default, like `clickhouse-server` does. This closes [#65695](https://github.com/ClickHouse/ClickHouse/issues/65695). [#65697](https://github.com/ClickHouse/ClickHouse/pull/65697) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Add a script to backup your files to ClickHouse. This is strange, but works. [#65699](https://github.com/ClickHouse/ClickHouse/pull/65699) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* PostgreSQL source support cancel. [#65722](https://github.com/ClickHouse/ClickHouse/pull/65722) ([Maksim Kita](https://github.com/kitaisreal)).
+* Make allow_experimental_analyzer be controlled by the initiator for distributed queries. This ensures compatibility and correctness during operations in mixed version clusters. [#65777](https://github.com/ClickHouse/ClickHouse/pull/65777) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Respect cgroup CPU limit in Keeper. [#65819](https://github.com/ClickHouse/ClickHouse/pull/65819) ([Antonio Andelic](https://github.com/antonio2368)).
+* Allow to use `concat` function with empty arguments ``` sql :) select concat();. [#65887](https://github.com/ClickHouse/ClickHouse/pull/65887) ([李扬](https://github.com/taiyang-li)).
+* Allow controlling named collections in clickhouse-local. [#65973](https://github.com/ClickHouse/ClickHouse/pull/65973) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Improve Azure profile events. [#65999](https://github.com/ClickHouse/ClickHouse/pull/65999) ([alesapin](https://github.com/alesapin)).
+* `Query was cancelled` might have been printed twice in clickhouse-client. This behaviour is fixed. [#66005](https://github.com/ClickHouse/ClickHouse/pull/66005) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Support ORC file read by writer time zone. [#66025](https://github.com/ClickHouse/ClickHouse/pull/66025) ([kevinyhzou](https://github.com/KevinyhZou)).
+* Refactor JSONExtract functions, support more types including experimental Dynamic type. [#66046](https://github.com/ClickHouse/ClickHouse/pull/66046) ([Kruglov Pavel](https://github.com/Avogar)).
+* DatabaseCatalog drops tables faster by using up to database_catalog_drop_table_concurrency threads. [#66065](https://github.com/ClickHouse/ClickHouse/pull/66065) ([Sema Checherinda](https://github.com/CheSema)).
+* This PR changes how deduplication for MV works. Fixed a lot of cases like: - on destination table: data is split for 2 or more blocks and that blocks is considered as duplicate when that block is inserted in parallel. - on MV destination table: the equal blocks are deduplicated, that happens when MV often produces equal data as a result for different input data due to performing aggregation. - on MV destination table: the equal blocks which comes from different MV are deduplicated. [#66144](https://github.com/ClickHouse/ClickHouse/pull/66144) ([Sema Checherinda](https://github.com/CheSema)).
+* Support null map subcolumn for Variant and Dynamic subcolumns. [#66178](https://github.com/ClickHouse/ClickHouse/pull/66178) ([Kruglov Pavel](https://github.com/Avogar)).
+* Add settings to control connection to the PostgreSQL. * Setting `postgresql_connection_attempt_timeout` specifies the value passed to `connect_timeout` parameter of connection URL. * Setting `postgresql_connection_pool_retries` specifies the number of retries to establish a connection to the PostgreSQL end-point. [#66232](https://github.com/ClickHouse/ClickHouse/pull/66232) ([Dmitry Novik](https://github.com/novikd)).
+* Reduce inaccuracy of input_wait_elapsed_us/input_wait_elapsed_us/elapsed_us. [#66239](https://github.com/ClickHouse/ClickHouse/pull/66239) ([Azat Khuzhin](https://github.com/azat)).
+* Improve FilesystemCache ProfileEvents. [#66249](https://github.com/ClickHouse/ClickHouse/pull/66249) ([zhukai](https://github.com/nauu)).
+* Add settings to ignore ON CLUSTER clause in queries for named collection management with replicated storage. [#66288](https://github.com/ClickHouse/ClickHouse/pull/66288) ([MikhailBurdukov](https://github.com/MikhailBurdukov)).
+* Upgraded `pocketfft` dependency to the recent commit https://github.com/mreineck/pocketfft/commit/f4c1aa8aa9ce79ad39e80f2c9c41b92ead90fda3. [#66291](https://github.com/ClickHouse/ClickHouse/pull/66291) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Upgraded `azure-sdk-for-cpp` to the recent commit https://github.com/ClickHouse/azure-sdk-for-cpp/commit/ea3e19a7be08519134c643177d56c7484dfec884. [#66292](https://github.com/ClickHouse/ClickHouse/pull/66292) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Function `generateSnowflakeID` now allows to specify a machine ID as a parameter to prevent collisions in large clusters. [#66374](https://github.com/ClickHouse/ClickHouse/pull/66374) ([ZAWA_ll](https://github.com/Zawa-ll)).
+* Disable suspending on Ctrl+Z in interactive mode. This is a common trap and is not expected behavior for almost all users. I imagine only a few extreme power users could appreciate suspending terminal applications to the background, but I don't know any. [#66511](https://github.com/ClickHouse/ClickHouse/pull/66511) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Add option for validating the Primary key type in Dictionaries. Without this option for simple layouts any column type will be implicitly converted to UInt64. ### Documentation entry for user-facing changes. [#66595](https://github.com/ClickHouse/ClickHouse/pull/66595) ([MikhailBurdukov](https://github.com/MikhailBurdukov)).
+* Fix SHOW MERGES remaining time. [#66735](https://github.com/ClickHouse/ClickHouse/pull/66735) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+
+#### Critical Bug Fix (crash, LOGICAL_ERROR, data loss, RBAC)
+* Fix unexpeced size of low cardinality column in function calls. [#65298](https://github.com/ClickHouse/ClickHouse/pull/65298) ([Raúl Marín](https://github.com/Algunenano)).
+* Check cyclic dependencies on CREATE/REPLACE/RENAME/EXCHANGE queries and throw an exception if there is a cyclic dependency. Previously such cyclic dependencies could lead to a deadlock during server startup. Closes [#65355](https://github.com/ClickHouse/ClickHouse/issues/65355). Also fix some bugs in dependencies creation. [#65405](https://github.com/ClickHouse/ClickHouse/pull/65405) ([Kruglov Pavel](https://github.com/Avogar)).
+* Fix crash in maxIntersections. [#65689](https://github.com/ClickHouse/ClickHouse/pull/65689) ([Raúl Marín](https://github.com/Algunenano)).
+* Fix the VALID UNTIL clause in the user definition resetting after a restart. Closes [#66405](https://github.com/ClickHouse/ClickHouse/issues/66405). [#66409](https://github.com/ClickHouse/ClickHouse/pull/66409) ([Nikolay Degterinsky](https://github.com/evillique)).
+
+#### Bug Fix (user-visible misbehavior in an official stable release)
+* Fixed crash while using MaterializedMySQL with TABLE OVERRIDE that maps MySQL NULL field into ClickHouse not NULL field. [#54649](https://github.com/ClickHouse/ClickHouse/pull/54649) ([Filipp Ozinov](https://github.com/bakwc)).
+* Fix logical error when PREWHERE expression read no columns and table has no adaptive index granularity (very old table). Fix [#56640](https://github.com/ClickHouse/ClickHouse/issues/56640). [#59173](https://github.com/ClickHouse/ClickHouse/pull/59173) ([Alexander Gololobov](https://github.com/davenger)).
+* Fix bug with cancelation buffer when canceling a query. [#64478](https://github.com/ClickHouse/ClickHouse/pull/64478) ([Sema Checherinda](https://github.com/CheSema)).
+* Fix filling parts columns from metadata (when columns.txt does not exists). [#64757](https://github.com/ClickHouse/ClickHouse/pull/64757) ([Azat Khuzhin](https://github.com/azat)).
+* Fix AST formatting of 'create table b empty as a'. [#64951](https://github.com/ClickHouse/ClickHouse/pull/64951) ([Michael Kolupaev](https://github.com/al13n321)).
+* Fix crash for `ALTER TABLE ... ON CLUSTER ... MODIFY SQL SECURITY`. [#64957](https://github.com/ClickHouse/ClickHouse/pull/64957) ([pufit](https://github.com/pufit)).
+* Fix crash on destroying AccessControl: add explicit shutdown. [#64993](https://github.com/ClickHouse/ClickHouse/pull/64993) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Eliminate injective function in argument of functions `uniq*` recursively. This used to work correctly but was broken in the new analyzer. [#65140](https://github.com/ClickHouse/ClickHouse/pull/65140) ([Duc Canh Le](https://github.com/canhld94)).
+* Fix unexpected projection name when query with CTE. [#65267](https://github.com/ClickHouse/ClickHouse/pull/65267) ([wudidapaopao](https://github.com/wudidapaopao)).
+* Require `dictGet` privilege when accessing dictionaries via direct query or the `Dictionary` table engine. [#65359](https://github.com/ClickHouse/ClickHouse/pull/65359) ([Joe Lynch](https://github.com/joelynch)).
+* Fix user-specific S3 auth with incremental backups. [#65481](https://github.com/ClickHouse/ClickHouse/pull/65481) ([Antonio Andelic](https://github.com/antonio2368)).
+* Disable `non-intersecting-parts` optimization for queries with `FINAL` in case of `read-in-order` optimization was enabled. This could lead to an incorrect query result. As a workaround, disable `do_not_merge_across_partitions_select_final` and `split_parts_ranges_into_intersecting_and_non_intersecting_final` before this fix is merged. [#65505](https://github.com/ClickHouse/ClickHouse/pull/65505) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix getting exception `Index out of bound for blob metadata` in case all files from list batch were filtered out. [#65523](https://github.com/ClickHouse/ClickHouse/pull/65523) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fix NOT_FOUND_COLUMN_IN_BLOCK for deduplicate merge of projection. [#65573](https://github.com/ClickHouse/ClickHouse/pull/65573) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
+* Fixed bug in MergeJoin. Column in sparse serialisation might be treated as a column of its nested type though the required conversion wasn't performed. [#65632](https://github.com/ClickHouse/ClickHouse/pull/65632) ([Nikita Taranov](https://github.com/nickitat)).
+* Fixed a bug that compatibility level '23.4' was not properly applied. [#65737](https://github.com/ClickHouse/ClickHouse/pull/65737) ([cw5121](https://github.com/cw5121)).
+* Fix odbc table with nullable fields. [#65738](https://github.com/ClickHouse/ClickHouse/pull/65738) ([Rodolphe Dugé de Bernonville](https://github.com/RodolpheDuge)).
+* Fix data race in `TCPHandler`, which could happen on fatal error. [#65744](https://github.com/ClickHouse/ClickHouse/pull/65744) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fix invalid exceptions in function `parseDateTime` with `%F` and `%D` placeholders. [#65768](https://github.com/ClickHouse/ClickHouse/pull/65768) ([Antonio Andelic](https://github.com/antonio2368)).
+* For queries that read from `PostgreSQL`, cancel the internal `PostgreSQL` query if the ClickHouse query is finished. Otherwise, `ClickHouse` query cannot be canceled until the internal `PostgreSQL` query is finished. [#65771](https://github.com/ClickHouse/ClickHouse/pull/65771) ([Maksim Kita](https://github.com/kitaisreal)).
+* Fix a bug in short circuit logic when old analyzer and dictGetOrDefault is used. [#65802](https://github.com/ClickHouse/ClickHouse/pull/65802) ([jsc0218](https://github.com/jsc0218)).
+* Fix a bug leads to EmbeddedRocksDB with TTL write corrupted SST files. [#65816](https://github.com/ClickHouse/ClickHouse/pull/65816) ([Duc Canh Le](https://github.com/canhld94)).
+* Functions `bitTest`, `bitTestAll`, and `bitTestAny` now return an error if the specified bit index is out-of-bounds (issue [#65517](https://github.com/ClickHouse/ClickHouse/issues/65517)). [#65818](https://github.com/ClickHouse/ClickHouse/pull/65818) ([Pablo Marcos](https://github.com/pamarcos)).
+* Setting `join_any_take_last_row` is supported in any query with hash join. [#65820](https://github.com/ClickHouse/ClickHouse/pull/65820) ([vdimir](https://github.com/vdimir)).
+* Better handling of join conditions involving `IS NULL` checks (for example `ON (a = b AND (a IS NOT NULL) AND (b IS NOT NULL) ) OR ( (a IS NULL) AND (b IS NULL) )` is rewritten to `ON a <=> b`), fix incorrect optimization when condition other then `IS NULL` are present. [#65835](https://github.com/ClickHouse/ClickHouse/pull/65835) ([vdimir](https://github.com/vdimir)).
+* Functions `bitShiftLeft` and `bitShitfRight` return an error for out of bounds shift positions (issue [#65516](https://github.com/ClickHouse/ClickHouse/issues/65516)). [#65838](https://github.com/ClickHouse/ClickHouse/pull/65838) ([Pablo Marcos](https://github.com/pamarcos)).
+* Fix growing memory usage in S3Queue. [#65839](https://github.com/ClickHouse/ClickHouse/pull/65839) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fix tie handling in `arrayAUC` to match sklearn. [#65840](https://github.com/ClickHouse/ClickHouse/pull/65840) ([gabrielmcg44](https://github.com/gabrielmcg44)).
+* Fix possible issues with MySQL server protocol TLS connections. [#65917](https://github.com/ClickHouse/ClickHouse/pull/65917) ([Azat Khuzhin](https://github.com/azat)).
+* Fix possible issues with MySQL client protocol TLS connections. [#65938](https://github.com/ClickHouse/ClickHouse/pull/65938) ([Azat Khuzhin](https://github.com/azat)).
+* Fix handling of `SSL_ERROR_WANT_READ`/`SSL_ERROR_WANT_WRITE` with zero timeout. [#65941](https://github.com/ClickHouse/ClickHouse/pull/65941) ([Azat Khuzhin](https://github.com/azat)).
+* Add missing settings `input_format_csv_skip_first_lines/input_format_tsv_skip_first_lines/input_format_csv_try_infer_numbers_from_strings/input_format_csv_try_infer_strings_from_quoted_tuples` in schema inference cache because they can change the resulting schema. It prevents from incorrect result of schema inference with these settings changed. [#65980](https://github.com/ClickHouse/ClickHouse/pull/65980) ([Kruglov Pavel](https://github.com/Avogar)).
+* Column _size in s3 engine and s3 table function denotes the size of a file inside the archive, not a size of the archive itself. [#65993](https://github.com/ClickHouse/ClickHouse/pull/65993) ([Daniil Ivanik](https://github.com/divanik)).
+* Fix resolving dynamic subcolumns in analyzer, avoid reading the whole column on dynamic subcolumn reading. [#66004](https://github.com/ClickHouse/ClickHouse/pull/66004) ([Kruglov Pavel](https://github.com/Avogar)).
+* Fix config merging for from_env with replace overrides. [#66034](https://github.com/ClickHouse/ClickHouse/pull/66034) ([Azat Khuzhin](https://github.com/azat)).
+* Fix a possible hanging in `GRPCServer` during shutdown. This PR fixes [#65622](https://github.com/ClickHouse/ClickHouse/issues/65622). [#66061](https://github.com/ClickHouse/ClickHouse/pull/66061) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Fix reading dynamic subcolumns from altered Memory table. Previously if `max_types` peremeter of a Dynamic type was changed in Memory table via alter, further subcolumns reading can return wrong result. [#66066](https://github.com/ClickHouse/ClickHouse/pull/66066) ([Kruglov Pavel](https://github.com/Avogar)).
+* Fixed several cases in function `has` with non-constant `LowCardinality` arguments. [#66088](https://github.com/ClickHouse/ClickHouse/pull/66088) ([Anton Popov](https://github.com/CurtizJ)).
+* Fix for `groupArrayIntersect`. It had incorrect behavior in the `merge()` function. Also, fixed behavior in `deserialise()` for numeric and general data. [#66103](https://github.com/ClickHouse/ClickHouse/pull/66103) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
+* Fixed buffer overflow bug in `unbin`/`unhex` implementation. [#66106](https://github.com/ClickHouse/ClickHouse/pull/66106) ([Nikita Taranov](https://github.com/nickitat)).
+* Disable the `merge-filters` optimization introduced in [#64760](https://github.com/ClickHouse/ClickHouse/issues/64760). It may cause an exception if optimization merges two filter expressions and does not apply a short-circuit evaluation. [#66126](https://github.com/ClickHouse/ClickHouse/pull/66126) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fixed the issue when the server failed to parse Avro files with negative block size arrays encoded, which is now allowed by the Avro specification. [#66130](https://github.com/ClickHouse/ClickHouse/pull/66130) ([Serge Klochkov](https://github.com/slvrtrn)).
+* Fixed a bug in ZooKeeper client: a session could get stuck in unusable state after receiving a hardware error from ZooKeeper. For example, this might happen due to "soft memory limit" in ClickHouse Keeper. [#66140](https://github.com/ClickHouse/ClickHouse/pull/66140) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Fix issue in SumIfToCountIfVisitor and signed integers. [#66146](https://github.com/ClickHouse/ClickHouse/pull/66146) ([Raúl Marín](https://github.com/Algunenano)).
+* Fix rare case with missing data in the result of distributed query, close [#61432](https://github.com/ClickHouse/ClickHouse/issues/61432). [#66174](https://github.com/ClickHouse/ClickHouse/pull/66174) ([vdimir](https://github.com/vdimir)).
+* Fix order of parsing metadata fields in StorageDeltaLake. [#66211](https://github.com/ClickHouse/ClickHouse/pull/66211) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Don't throw `TIMEOUT_EXCEEDED` for `none_only_active` mode of `distributed_ddl_output_mode`. [#66218](https://github.com/ClickHouse/ClickHouse/pull/66218) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Fix handling limit for `system.numbers_mt` when no index can be used. [#66231](https://github.com/ClickHouse/ClickHouse/pull/66231) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
+* Fixed how the ClickHouse server detects the maximum number of usable CPU cores as specified by cgroups v2 if the server runs in a container such as Docker. In more detail, containers often run their process in the root cgroup which has an empty name. In that case, ClickHouse ignored the CPU limits set by cgroups v2. [#66237](https://github.com/ClickHouse/ClickHouse/pull/66237) ([filimonov](https://github.com/filimonov)).
+* Fix the `Not-ready set` error when a subquery with `IN` is used in the constraint. [#66261](https://github.com/ClickHouse/ClickHouse/pull/66261) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix indexHint function case found by fuzzer. [#66286](https://github.com/ClickHouse/ClickHouse/pull/66286) ([Anton Popov](https://github.com/CurtizJ)).
+* Fix error reporting while copying to S3 or AzureBlobStorage. [#66295](https://github.com/ClickHouse/ClickHouse/pull/66295) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Prevent watchdog from keeping descriptors of unlinked(rotated) log files. [#66334](https://github.com/ClickHouse/ClickHouse/pull/66334) ([Aleksei Filatov](https://github.com/aalexfvk)).
+* Fix the bug that logicalexpressionoptimizerpass lost logical type of constant. closes [#64487](https://github.com/ClickHouse/ClickHouse/issues/64487). [#66344](https://github.com/ClickHouse/ClickHouse/pull/66344) ([pn](https://github.com/chloro-pn)).
+* Fix `Column identifier is already registered` error with `group_by_use_nulls=true` and new analyzer. [#66400](https://github.com/ClickHouse/ClickHouse/pull/66400) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix possible incorrect result for queries joining and filtering table external engine (like PostgreSQL), due to too aggressive filter pushdown. Since now, conditions from where section won't be send to external database in case of outer join with external table. [#66402](https://github.com/ClickHouse/ClickHouse/pull/66402) ([vdimir](https://github.com/vdimir)).
+* Added missing column materialization for cross join. [#66413](https://github.com/ClickHouse/ClickHouse/pull/66413) ([lgbo](https://github.com/lgbo-ustc)).
+* Fix `Cannot find column` error for queries with constant expression in `GROUP BY` key and new analyzer enabled. [#66433](https://github.com/ClickHouse/ClickHouse/pull/66433) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Avoid possible logical error during import from Npy format in case of bad array nesting level, fix testing of other kinds of errors. [#66461](https://github.com/ClickHouse/ClickHouse/pull/66461) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
+* Fix wrong count() result when there is non-deterministic function in predicate. [#66510](https://github.com/ClickHouse/ClickHouse/pull/66510) ([Duc Canh Le](https://github.com/canhld94)).
+* Correctly track memory for `Allocator::realloc`. [#66548](https://github.com/ClickHouse/ClickHouse/pull/66548) ([Antonio Andelic](https://github.com/antonio2368)).
+* Fix reading of uninitialized memory when hashing empty tuples. This closes [#66559](https://github.com/ClickHouse/ClickHouse/issues/66559). [#66562](https://github.com/ClickHouse/ClickHouse/pull/66562) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix an invalid result for queries with `WINDOW`. This could happen when `PARTITION` columns have sparse serialization and window functions are executed in parallel. [#66579](https://github.com/ClickHouse/ClickHouse/pull/66579) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix removing named collections in local storage. [#66599](https://github.com/ClickHouse/ClickHouse/pull/66599) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
+* Fix logical error in `PrometheusRequestHandler`. [#66621](https://github.com/ClickHouse/ClickHouse/pull/66621) ([Vitaly Baranov](https://github.com/vitlibar)).
+* `column_length` is not updated in `ColumnTuple::insertManyFrom`. [#66626](https://github.com/ClickHouse/ClickHouse/pull/66626) ([lgbo](https://github.com/lgbo-ustc)).
+* Fix `Unknown identifier` and `Column is not under aggregate function` errors for queries with the expression `(column IS NULL).` The bug was triggered by [#65088](https://github.com/ClickHouse/ClickHouse/issues/65088), with the disabled analyzer only. [#66654](https://github.com/ClickHouse/ClickHouse/pull/66654) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix `Method getResultType is not supported for QUERY query node` error when scalar subquery was used as the first argument of IN (with new analyzer). [#66655](https://github.com/ClickHouse/ClickHouse/pull/66655) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix possible PARAMETER_OUT_OF_BOUND error during reading variant subcolumn. [#66659](https://github.com/ClickHouse/ClickHouse/pull/66659) ([Kruglov Pavel](https://github.com/Avogar)).
+* Fix rare case of stuck merge after drop column. [#66707](https://github.com/ClickHouse/ClickHouse/pull/66707) ([Raúl Marín](https://github.com/Algunenano)).
+* Fix assertion `isUniqTypes` when insert select from remote sources. [#66722](https://github.com/ClickHouse/ClickHouse/pull/66722) ([Sema Checherinda](https://github.com/CheSema)).
+* Backported in [#67026](https://github.com/ClickHouse/ClickHouse/issues/67026): In rare cases ClickHouse could consider parts as broken because of some unexpected projections on disk. Now it's fixed. [#66898](https://github.com/ClickHouse/ClickHouse/pull/66898) ([alesapin](https://github.com/alesapin)).
+* Backported in [#67443](https://github.com/ClickHouse/ClickHouse/issues/67443): Forbid create as select even when database_replicated_allow_heavy_create is set. It was unconditionally forbidden in 23.12 and accidentally allowed under the setting in unreleased 24.7. [#66980](https://github.com/ClickHouse/ClickHouse/pull/66980) ([vdimir](https://github.com/vdimir)).
+* Backported in [#67201](https://github.com/ClickHouse/ClickHouse/issues/67201): TRUNCATE DATABASE used to stop replication as if it was a DROP DATABASE query, it's fixed. [#67129](https://github.com/ClickHouse/ClickHouse/pull/67129) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Backported in [#67383](https://github.com/ClickHouse/ClickHouse/issues/67383): Fix error `Cannot convert column because it is non constant in source stream but must be constant in result.` for a query that reads from the `Merge` table over the `Distriburted` table with one shard. [#67146](https://github.com/ClickHouse/ClickHouse/pull/67146) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#67246](https://github.com/ClickHouse/ClickHouse/issues/67246): This closes [#67156](https://github.com/ClickHouse/ClickHouse/issues/67156). This closes [#66447](https://github.com/ClickHouse/ClickHouse/issues/66447). The bug was introduced in https://github.com/ClickHouse/ClickHouse/pull/62907. [#67178](https://github.com/ClickHouse/ClickHouse/pull/67178) ([Maksim Kita](https://github.com/kitaisreal)).
+
+#### Build/Testing/Packaging Improvement
+* Instantiate template methods ahead in different .cpp files, avoid too large translation units during compiling. [#64818](https://github.com/ClickHouse/ClickHouse/pull/64818) ([lgbo](https://github.com/lgbo-ustc)).
+* Stateless tests: Improve tests speed and decrease number of parallel jobs. [#65186](https://github.com/ClickHouse/ClickHouse/pull/65186) ([Nikita Fomichev](https://github.com/fm4v)).
+* Add tests for `base64URLEncode` and `base64URLDecode`. Add analyzer tests. [#65979](https://github.com/ClickHouse/ClickHouse/pull/65979) ([Nikita Fomichev](https://github.com/fm4v)).
+* Fix problem when github terminate instances by timeout and artifacts are not collected and full test report is not generated. [#66036](https://github.com/ClickHouse/ClickHouse/pull/66036) ([Nikita Fomichev](https://github.com/fm4v)).
+* Fix test [test_grpc_protocol/test.py::test_progress](https://s3.amazonaws.com/clickhouse-test-reports/57695/188f8a3df74caf830ad1ced3c4cf6dfb0aa90093/integration_tests__asan__old_analyzer__[4_6].html). [#66063](https://github.com/ClickHouse/ClickHouse/pull/66063) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Stateless tests: Improve tests speed and decrease number of parallel jobs. [#66305](https://github.com/ClickHouse/ClickHouse/pull/66305) ([Nikita Fomichev](https://github.com/fm4v)).
+* Stateless tests: Improve tests speed and decrease number of parallel jobs 3. [#66363](https://github.com/ClickHouse/ClickHouse/pull/66363) ([Nikita Fomichev](https://github.com/fm4v)).
+* Tests: fix tests hang up in cases when gdb catches error. [#66411](https://github.com/ClickHouse/ClickHouse/pull/66411) ([Nikita Fomichev](https://github.com/fm4v)).
+* ... since [Release v24.6.1.4423-stable](https://github.com/ClickHouse/ClickHouse/releases/tag/v24.6.1.4423-stable) when build in ppc64le with dynamic openssl build (`cmake -DENABLE_OPENSSL_DYNAMIC=1 -DCMAKE_TOOLCHAIN_FILE= cmake/linux/toolchain-ppc64le.cmake `) got error: ` ld.lld: error: duplicate symbol: OPENSSL_cleanse`. [#66733](https://github.com/ClickHouse/ClickHouse/pull/66733) ([Yong Wang](https://github.com/kashwy)).
+
+#### NO CL CATEGORY
+
+* Backported in [#67084](https://github.com/ClickHouse/ClickHouse/issues/67084):. [#67040](https://github.com/ClickHouse/ClickHouse/pull/67040) ([Raúl Marín](https://github.com/Algunenano)).
+* Backported in [#67452](https://github.com/ClickHouse/ClickHouse/issues/67452):. [#67392](https://github.com/ClickHouse/ClickHouse/pull/67392) ([alesapin](https://github.com/alesapin)).
+
+#### NO CL ENTRY
+
+* NO CL ENTRY:  'Revert "Revert "Small fix for 02340_parts_refcnt_mergetree""'. [#65155](https://github.com/ClickHouse/ClickHouse/pull/65155) ([Nikita Taranov](https://github.com/nickitat)).
+* NO CL ENTRY:  'Revert "Use 1MB HTTP buffers to avoid frequent send syscalls"'. [#65498](https://github.com/ClickHouse/ClickHouse/pull/65498) ([Sergei Trifonov](https://github.com/serxa)).
+* NO CL ENTRY:  'Revert "Resubmit http_external_tables_memory_tracking test"'. [#65500](https://github.com/ClickHouse/ClickHouse/pull/65500) ([Nikita Taranov](https://github.com/nickitat)).
+* NO CL ENTRY:  'Revert "Add an assertion in ReplicatedMergeTreeQueue"'. [#65686](https://github.com/ClickHouse/ClickHouse/pull/65686) ([Raúl Marín](https://github.com/Algunenano)).
+* NO CL ENTRY:  'Revert "insertion deduplication on retries for materialised views"'. [#66134](https://github.com/ClickHouse/ClickHouse/pull/66134) ([Sema Checherinda](https://github.com/CheSema)).
+
+#### NOT FOR CHANGELOG / INSIGNIFICANT
+
+* Return and fix 01600_parts_states_metrics_long test. [#58748](https://github.com/ClickHouse/ClickHouse/pull/58748) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
+* Add azure_cache as storage policy to tests. [#59943](https://github.com/ClickHouse/ClickHouse/pull/59943) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)).
+* Minor: replaced expression with LEGACY_MAX_LEVEL. [#61268](https://github.com/ClickHouse/ClickHouse/pull/61268) ([Vasily Nemkov](https://github.com/Enmk)).
+* Make write to temporary data in cache do all checks and assertions as during write to ordinary cache. [#63348](https://github.com/ClickHouse/ClickHouse/pull/63348) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Refactoring near azure blob storage. [#63636](https://github.com/ClickHouse/ClickHouse/pull/63636) ([Anton Popov](https://github.com/CurtizJ)).
+* Everything should work with Analyzer. [#63643](https://github.com/ClickHouse/ClickHouse/pull/63643) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* add some log for non using detached tables. [#64992](https://github.com/ClickHouse/ClickHouse/pull/64992) ([Konstantin Morozov](https://github.com/k-morozov)).
+* Remove dag flags. [#65234](https://github.com/ClickHouse/ClickHouse/pull/65234) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Fix flaky autocompletion test. [#65246](https://github.com/ClickHouse/ClickHouse/pull/65246) ([Konstantin Bogdanov](https://github.com/thevar1able)).
+* Disable userspace page cache by default. [#65305](https://github.com/ClickHouse/ClickHouse/pull/65305) ([Michael Kolupaev](https://github.com/al13n321)).
+* Update version_date.tsv and changelogs after v24.4.3.25-stable. [#65308](https://github.com/ClickHouse/ClickHouse/pull/65308) ([robot-clickhouse](https://github.com/robot-clickhouse)).
+* Do not raise a NOT_IMPLEMENTED error when getting s3 metrics with a multiple disk configuration. [#65403](https://github.com/ClickHouse/ClickHouse/pull/65403) ([Elena Torró](https://github.com/elenatorro)).
+* Dodging reading from wrong table with parallel replicas. [#65417](https://github.com/ClickHouse/ClickHouse/pull/65417) ([Nikita Taranov](https://github.com/nickitat)).
+* Fix: return error if can't connect to any replicas chosen for query execution. [#65467](https://github.com/ClickHouse/ClickHouse/pull/65467) ([Igor Nikonov](https://github.com/devcrafter)).
+* Fix `AzureObjectStorage::exists` method. [#65471](https://github.com/ClickHouse/ClickHouse/pull/65471) ([Anton Popov](https://github.com/CurtizJ)).
+* Update version after release. [#65483](https://github.com/ClickHouse/ClickHouse/pull/65483) ([Raúl Marín](https://github.com/Algunenano)).
+* Generate 24.6 changelog. [#65485](https://github.com/ClickHouse/ClickHouse/pull/65485) ([Raúl Marín](https://github.com/Algunenano)).
+* Fix of `PlanSquashingTransform`: pipeline stuck. [#65487](https://github.com/ClickHouse/ClickHouse/pull/65487) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
+* Fix bad test `02922_deduplication_with_zero_copy`. [#65492](https://github.com/ClickHouse/ClickHouse/pull/65492) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Disable clang-format in special areas. [#65495](https://github.com/ClickHouse/ClickHouse/pull/65495) ([Nikita Taranov](https://github.com/nickitat)).
+* Fix `test_keeper_snapshots`. [#65497](https://github.com/ClickHouse/ClickHouse/pull/65497) ([Antonio Andelic](https://github.com/antonio2368)).
+* Update to libunwind 8.1.7. [#65509](https://github.com/ClickHouse/ClickHouse/pull/65509) ([Michael Kolupaev](https://github.com/al13n321)).
+* Setting `uniform_snowflake_conversion_functions` (not in any release yet) was replaced by setting `allow_deprecated_snowflake_conversion_functions`. The latter controls if the legacy snowflake conversion functions are available (by default, they are not). [#65522](https://github.com/ClickHouse/ClickHouse/pull/65522) ([Robert Schulze](https://github.com/rschu1ze)).
+* Try CI without RerunCheck, jobs can be easily rerun manually though extra amount of work in CI will follow on workflow restart. [#65524](https://github.com/ClickHouse/ClickHouse/pull/65524) ([Max K.](https://github.com/maxknv)).
+* Bump re2 to latest HEAD. [#65526](https://github.com/ClickHouse/ClickHouse/pull/65526) ([Robert Schulze](https://github.com/rschu1ze)).
+* OpenSSL: Replace temporary fix for unsynchronized access by official fix. [#65529](https://github.com/ClickHouse/ClickHouse/pull/65529) ([Robert Schulze](https://github.com/rschu1ze)).
+* Update README.md. [#65531](https://github.com/ClickHouse/ClickHouse/pull/65531) ([Tyler Hannan](https://github.com/tylerhannan)).
+* CI: some time there are timeouts on DROP TABLES for random tests. [#65535](https://github.com/ClickHouse/ClickHouse/pull/65535) ([Sema Checherinda](https://github.com/CheSema)).
+* Synchronize `MARK_CACHE_SIZE` value in default settings and config. [#65547](https://github.com/ClickHouse/ClickHouse/pull/65547) ([Denny Crane](https://github.com/den-crane)).
+* CI: Skip removed test files in stateless flaky check job. [#65553](https://github.com/ClickHouse/ClickHouse/pull/65553) ([Max K.](https://github.com/maxknv)).
+* Renames Build report jobs. [#65554](https://github.com/ClickHouse/ClickHouse/pull/65554) ([Max K.](https://github.com/maxknv)).
+* Parse user from URL for dashboard.html (useful for sharing). [#65556](https://github.com/ClickHouse/ClickHouse/pull/65556) ([Azat Khuzhin](https://github.com/azat)).
+* Remove tech debt. [#65561](https://github.com/ClickHouse/ClickHouse/pull/65561) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Maybe fix test `00763_lock_buffer_long.sh`. [#65562](https://github.com/ClickHouse/ClickHouse/pull/65562) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix clickhouse-keeper with not system-wide directories and provide override for local development. [#65563](https://github.com/ClickHouse/ClickHouse/pull/65563) ([Azat Khuzhin](https://github.com/azat)).
+* Re-configure yamllint to allow document-start. [#65565](https://github.com/ClickHouse/ClickHouse/pull/65565) ([Azat Khuzhin](https://github.com/azat)).
+* Fix flaky test `01254_dict_load_after_detach_attach.sql`. [#65571](https://github.com/ClickHouse/ClickHouse/pull/65571) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Improve flaky test to provide more diagnostics. [#65586](https://github.com/ClickHouse/ClickHouse/pull/65586) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix test_parallel_replicas_distributed_skip_shards flakiness. [#65588](https://github.com/ClickHouse/ClickHouse/pull/65588) ([Igor Nikonov](https://github.com/devcrafter)).
+* Fix an error in the test about metadata_type. [#65592](https://github.com/ClickHouse/ClickHouse/pull/65592) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix slow test. [#65593](https://github.com/ClickHouse/ClickHouse/pull/65593) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* fix flaky 02864_statistics_uniq. [#65599](https://github.com/ClickHouse/ClickHouse/pull/65599) ([Han Fei](https://github.com/hanfei1991)).
+* Fix 03172_error_log_table_not_empty. [#65604](https://github.com/ClickHouse/ClickHouse/pull/65604) ([Pablo Marcos](https://github.com/pamarcos)).
+* Enable realtime digest for Jepsen tests. [#65608](https://github.com/ClickHouse/ClickHouse/pull/65608) ([Antonio Andelic](https://github.com/antonio2368)).
+* CI: Return Job Rerun check. [#65613](https://github.com/ClickHouse/ClickHouse/pull/65613) ([Max K.](https://github.com/maxknv)).
+* Update CHANGELOG.md. [#65624](https://github.com/ClickHouse/ClickHouse/pull/65624) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Module is required for authenticating in GH (in cloud). [#65628](https://github.com/ClickHouse/ClickHouse/pull/65628) ([Max K.](https://github.com/maxknv)).
+* Update IObjectStorage.h. [#65631](https://github.com/ClickHouse/ClickHouse/pull/65631) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fix 02834_apache_arrow_abort flakiness with MSAN. [#65640](https://github.com/ClickHouse/ClickHouse/pull/65640) ([Michael Kolupaev](https://github.com/al13n321)).
+* Fix overflow in StorageWindowView. [#65641](https://github.com/ClickHouse/ClickHouse/pull/65641) ([Michael Kolupaev](https://github.com/al13n321)).
+* Fix inconsistent AST formatting when a keyword is used as type name. [#65648](https://github.com/ClickHouse/ClickHouse/pull/65648) ([Michael Kolupaev](https://github.com/al13n321)).
+* CI: Single point of setting mergeable check status. [#65658](https://github.com/ClickHouse/ClickHouse/pull/65658) ([Max K.](https://github.com/maxknv)).
+* Miscellaneous and insignificant changes around Client/ClientBase. [#65669](https://github.com/ClickHouse/ClickHouse/pull/65669) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Add Replicated database names to ZooKeeper for introspection. [#65675](https://github.com/ClickHouse/ClickHouse/pull/65675) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Introduce type contract checks in `IColumn`. [#65687](https://github.com/ClickHouse/ClickHouse/pull/65687) ([Nikita Taranov](https://github.com/nickitat)).
+* Print slightly more information in 02982_aggregation_states_destruction. [#65688](https://github.com/ClickHouse/ClickHouse/pull/65688) ([Michael Kolupaev](https://github.com/al13n321)).
+* Disable stacktrace collection in GWPAsan by default. [#65701](https://github.com/ClickHouse/ClickHouse/pull/65701) ([Antonio Andelic](https://github.com/antonio2368)).
+* Build jemalloc with profiler. [#65702](https://github.com/ClickHouse/ClickHouse/pull/65702) ([Antonio Andelic](https://github.com/antonio2368)).
+* Fix duplicate symbol linkage error. [#65705](https://github.com/ClickHouse/ClickHouse/pull/65705) ([Nikita Taranov](https://github.com/nickitat)).
+* Fix server restarts in performance tests. [#65717](https://github.com/ClickHouse/ClickHouse/pull/65717) ([Antonio Andelic](https://github.com/antonio2368)).
+* Update 03002_part_log_rmt_fetch_mutate_error.sql. [#65720](https://github.com/ClickHouse/ClickHouse/pull/65720) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Fix bug with session closing in Keeper. [#65735](https://github.com/ClickHouse/ClickHouse/pull/65735) ([Antonio Andelic](https://github.com/antonio2368)).
+* Fix flaky `02265_column_ttl`. Closes [#65719](https://github.com/ClickHouse/ClickHouse/issues/65719). [#65742](https://github.com/ClickHouse/ClickHouse/pull/65742) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
+* See [#65745](https://github.com/ClickHouse/ClickHouse/issues/65745). It doesn't solve the issue, but helps a bit. [#65746](https://github.com/ClickHouse/ClickHouse/pull/65746) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Update CHANGELOG.md. [#65752](https://github.com/ClickHouse/ClickHouse/pull/65752) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* document declarative ssh-keys authentication. [#65756](https://github.com/ClickHouse/ClickHouse/pull/65756) ([Tobias Florek](https://github.com/ibotty)).
+* `base64En/Decode64Url` --> `base64En/Decode64URL`. [#65760](https://github.com/ClickHouse/ClickHouse/pull/65760) ([Robert Schulze](https://github.com/rschu1ze)).
+* Fix for issue [#65757](https://github.com/ClickHouse/ClickHouse/issues/65757). [#65763](https://github.com/ClickHouse/ClickHouse/pull/65763) ([Yarik Briukhovetskyi](https://github.com/yariks5s)).
+* Fix flaky `test_replicated_database::test_alter_attach`. [#65766](https://github.com/ClickHouse/ClickHouse/pull/65766) ([Antonio Andelic](https://github.com/antonio2368)).
+* Fix: progress bar for read in order queries. [#65769](https://github.com/ClickHouse/ClickHouse/pull/65769) ([Igor Nikonov](https://github.com/devcrafter)).
+* CI: Fix for Builds report job in backports and releases. [#65774](https://github.com/ClickHouse/ClickHouse/pull/65774) ([Max K.](https://github.com/maxknv)).
+* CI: New create release workflow. [#65775](https://github.com/ClickHouse/ClickHouse/pull/65775) ([Max K.](https://github.com/maxknv)).
+* fixed misspelled word. [#65778](https://github.com/ClickHouse/ClickHouse/pull/65778) ([Linh Giang](https://github.com/linhgiang24)).
+* Refactor statistics interface. [#65792](https://github.com/ClickHouse/ClickHouse/pull/65792) ([Robert Schulze](https://github.com/rschu1ze)).
+* Try to make `test_ldap_external_user_directory` less flaky. [#65794](https://github.com/ClickHouse/ClickHouse/pull/65794) ([Andrey Zvonov](https://github.com/zvonand)).
+* AMI image with gh and jwt. [#65795](https://github.com/ClickHouse/ClickHouse/pull/65795) ([Max K.](https://github.com/maxknv)).
+* Forbid join algorithm randomisation for 03094_one_thousand_joins. [#65798](https://github.com/ClickHouse/ClickHouse/pull/65798) ([Nikita Taranov](https://github.com/nickitat)).
+* Fix 02931_rewrite_sum_column_and_constant flakiness. [#65800](https://github.com/ClickHouse/ClickHouse/pull/65800) ([Michael Kolupaev](https://github.com/al13n321)).
+* Update StorageMaterializedView.cpp. [#65801](https://github.com/ClickHouse/ClickHouse/pull/65801) ([Nikolay Degterinsky](https://github.com/evillique)).
+* Fix slow `getFQDNOrHostNameImpl` on macOS. [#65803](https://github.com/ClickHouse/ClickHouse/pull/65803) ([Konstantin Bogdanov](https://github.com/thevar1able)).
+* No jemalloc profiler for non-Linux. [#65834](https://github.com/ClickHouse/ClickHouse/pull/65834) ([Antonio Andelic](https://github.com/antonio2368)).
+* Add missing workload identity changes. [#65848](https://github.com/ClickHouse/ClickHouse/pull/65848) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)).
+* Fix rocksdb. [#65858](https://github.com/ClickHouse/ClickHouse/pull/65858) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Update the list of easy tasks. [#65865](https://github.com/ClickHouse/ClickHouse/pull/65865) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Update CHANGELOG.md. [#65866](https://github.com/ClickHouse/ClickHouse/pull/65866) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* This closes [#43003](https://github.com/ClickHouse/ClickHouse/issues/43003). [#65870](https://github.com/ClickHouse/ClickHouse/pull/65870) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Uninteresting changes. [#65871](https://github.com/ClickHouse/ClickHouse/pull/65871) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Max sessions for user tests improvements. [#65888](https://github.com/ClickHouse/ClickHouse/pull/65888) ([Alexey Gerasimchuck](https://github.com/Demilivor)).
+* Update version_date.tsv and changelogs after v24.6.1.4423-stable. [#65909](https://github.com/ClickHouse/ClickHouse/pull/65909) ([robot-clickhouse](https://github.com/robot-clickhouse)).
+* Remove standalone Keeper build. [#65910](https://github.com/ClickHouse/ClickHouse/pull/65910) ([Antonio Andelic](https://github.com/antonio2368)).
+* Add extra profiling helpers for Keeper. [#65918](https://github.com/ClickHouse/ClickHouse/pull/65918) ([Antonio Andelic](https://github.com/antonio2368)).
+* PostgreSQL source cancel query comments. [#65919](https://github.com/ClickHouse/ClickHouse/pull/65919) ([Maksim Kita](https://github.com/kitaisreal)).
+* Remove mysqlxx::Pool::Entry assignment operator. [#65920](https://github.com/ClickHouse/ClickHouse/pull/65920) ([Azat Khuzhin](https://github.com/azat)).
+* No random settings for a test with `Object(JSON)`. [#65921](https://github.com/ClickHouse/ClickHouse/pull/65921) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Follow up to [#65046](https://github.com/ClickHouse/ClickHouse/issues/65046). [#65928](https://github.com/ClickHouse/ClickHouse/pull/65928) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* add restriction for storage join. [#65936](https://github.com/ClickHouse/ClickHouse/pull/65936) ([Han Fei](https://github.com/hanfei1991)).
+* Update version_date.tsv and changelogs after v24.5.4.49-stable. [#65937](https://github.com/ClickHouse/ClickHouse/pull/65937) ([robot-clickhouse](https://github.com/robot-clickhouse)).
+* Add table name to MergeTreeSource spans. [#65940](https://github.com/ClickHouse/ClickHouse/pull/65940) ([Nikita Taranov](https://github.com/nickitat)).
+* Fix SettingsChangesHistory 24.7. [#65945](https://github.com/ClickHouse/ClickHouse/pull/65945) ([Raúl Marín](https://github.com/Algunenano)).
+* Fix logical error "Expected ReadBufferFromFile, but got DB::EmptyReadBuffer". [#65949](https://github.com/ClickHouse/ClickHouse/pull/65949) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Use -Og instead of -O0 for debug builds. [#65953](https://github.com/ClickHouse/ClickHouse/pull/65953) ([Michael Kolupaev](https://github.com/al13n321)).
+* Fix data race for Keeper snapshot queue. [#65970](https://github.com/ClickHouse/ClickHouse/pull/65970) ([Antonio Andelic](https://github.com/antonio2368)).
+* Minor changes in CHANGELOG. [#65971](https://github.com/ClickHouse/ClickHouse/pull/65971) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Remove unnatural punctuation from Parquet. [#65972](https://github.com/ClickHouse/ClickHouse/pull/65972) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Try fix "Check timeout expired" without any server logs in report in stateless tests. [#65977](https://github.com/ClickHouse/ClickHouse/pull/65977) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fix support of non-const scale arguments in rounding functions. [#65983](https://github.com/ClickHouse/ClickHouse/pull/65983) ([Mikhail Gorshkov](https://github.com/mgorshkov)).
+* More aesthetic error messages. [#65985](https://github.com/ClickHouse/ClickHouse/pull/65985) ([Robert Schulze](https://github.com/rschu1ze)).
+* Fix race in s3queue. [#65986](https://github.com/ClickHouse/ClickHouse/pull/65986) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Now it's possible to specify `s3-storage`, `azure-object-storage` and in general `object-storage`. [#65988](https://github.com/ClickHouse/ClickHouse/pull/65988) ([alesapin](https://github.com/alesapin)).
+* Fix flaky test_storage_s3_queue tests. [#66009](https://github.com/ClickHouse/ClickHouse/pull/66009) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Relax the check in 02982_aggregation_states_destruction. [#66011](https://github.com/ClickHouse/ClickHouse/pull/66011) ([Nikita Taranov](https://github.com/nickitat)).
+* Fix `01158_zookeeper_log_long`. [#66012](https://github.com/ClickHouse/ClickHouse/pull/66012) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Remove scary jemalloc log. [#66028](https://github.com/ClickHouse/ClickHouse/pull/66028) ([Antonio Andelic](https://github.com/antonio2368)).
+* Move experimental settings to the experimental block. [#66030](https://github.com/ClickHouse/ClickHouse/pull/66030) ([Raúl Marín](https://github.com/Algunenano)).
+* Fix lock-order-inversion in DatabaseCatalog. [#66038](https://github.com/ClickHouse/ClickHouse/pull/66038) ([Nikolay Degterinsky](https://github.com/evillique)).
+* Try disabling jemalloc background threads. [#66041](https://github.com/ClickHouse/ClickHouse/pull/66041) ([Antonio Andelic](https://github.com/antonio2368)).
+* Try to avoid conflicts in `SettingsChangesHistory.cpp`. [#66042](https://github.com/ClickHouse/ClickHouse/pull/66042) ([Anton Popov](https://github.com/CurtizJ)).
+* Add profile events for regex cache. [#66050](https://github.com/ClickHouse/ClickHouse/pull/66050) ([Antonio Andelic](https://github.com/antonio2368)).
+* Bump vectorscan to 5.4.10.1. [#66056](https://github.com/ClickHouse/ClickHouse/pull/66056) ([Robert Schulze](https://github.com/rschu1ze)).
+* Remove obsolete comment. [#66059](https://github.com/ClickHouse/ClickHouse/pull/66059) ([Robert Schulze](https://github.com/rschu1ze)).
+* Maybe fix tsan assert in `test_mysql_killed_while_insert_8_0`. [#66064](https://github.com/ClickHouse/ClickHouse/pull/66064) ([Robert Schulze](https://github.com/rschu1ze)).
+* Move some of `HTTPHandler` stuff to separate source files in order to reuse it in `PrometheusRequestHandler`. This PR is required for https://github.com/ClickHouse/ClickHouse/pull/64183. [#66067](https://github.com/ClickHouse/ClickHouse/pull/66067) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Bump rocksdb to v6.23.3. [#66068](https://github.com/ClickHouse/ClickHouse/pull/66068) ([Robert Schulze](https://github.com/rschu1ze)).
+* Add protobufs for `Prometheus` `remote-write` / `remote-read` protocols to our repository. Fix cmake script for compiling protobufs. [#66069](https://github.com/ClickHouse/ClickHouse/pull/66069) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Use pinned versions of all python packages in CI docker images. Also makes clang-18.1.8 work with sanitizers and surprisingly fixes [#66049](https://github.com/ClickHouse/ClickHouse/issues/66049). [#66070](https://github.com/ClickHouse/ClickHouse/pull/66070) ([alesapin](https://github.com/alesapin)).
+* Clean-up custom LLVM 15 patches. [#66072](https://github.com/ClickHouse/ClickHouse/pull/66072) ([Robert Schulze](https://github.com/rschu1ze)).
+* Minor JWT client fixes. [#66073](https://github.com/ClickHouse/ClickHouse/pull/66073) ([Konstantin Bogdanov](https://github.com/thevar1able)).
+* Bump vectorscan to 5.4.11. [#66082](https://github.com/ClickHouse/ClickHouse/pull/66082) ([Robert Schulze](https://github.com/rschu1ze)).
+* Print stacktrace in case of abort after logical error. [#66091](https://github.com/ClickHouse/ClickHouse/pull/66091) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* fix flaky 03172_error_log_table_not_empty. [#66093](https://github.com/ClickHouse/ClickHouse/pull/66093) ([Sema Checherinda](https://github.com/CheSema)).
+* Bump s2geometry to latest master. [#66094](https://github.com/ClickHouse/ClickHouse/pull/66094) ([Robert Schulze](https://github.com/rschu1ze)).
+* update keeper bench example config file. [#66095](https://github.com/ClickHouse/ClickHouse/pull/66095) ([Han Fei](https://github.com/hanfei1991)).
+* Avoid using source directory for generated files. [#66097](https://github.com/ClickHouse/ClickHouse/pull/66097) ([Azat Khuzhin](https://github.com/azat)).
+* More precise warning message about sanitizers. [#66098](https://github.com/ClickHouse/ClickHouse/pull/66098) ([Anton Popov](https://github.com/CurtizJ)).
+* Slightly better calculation of primary index. [#66099](https://github.com/ClickHouse/ClickHouse/pull/66099) ([Anton Popov](https://github.com/CurtizJ)).
+* Bump Azure to 1.12. [#66100](https://github.com/ClickHouse/ClickHouse/pull/66100) ([Robert Schulze](https://github.com/rschu1ze)).
+* Add a test for [#58998](https://github.com/ClickHouse/ClickHouse/issues/58998). [#66101](https://github.com/ClickHouse/ClickHouse/pull/66101) ([Anton Popov](https://github.com/CurtizJ)).
+* CI: Fix sync pr merge. [#66105](https://github.com/ClickHouse/ClickHouse/pull/66105) ([Max K.](https://github.com/maxknv)).
+* Remove flaky case from 02956_rocksdb_bulk_sink. [#66107](https://github.com/ClickHouse/ClickHouse/pull/66107) ([vdimir](https://github.com/vdimir)).
+* Fix bugfix checker. [#66120](https://github.com/ClickHouse/ClickHouse/pull/66120) ([Raúl Marín](https://github.com/Algunenano)).
+* Correctly print long processing requests in Keeper. [#66124](https://github.com/ClickHouse/ClickHouse/pull/66124) ([Antonio Andelic](https://github.com/antonio2368)).
+* Update version_date.tsv and changelogs after v24.6.2.17-stable. [#66127](https://github.com/ClickHouse/ClickHouse/pull/66127) ([robot-clickhouse](https://github.com/robot-clickhouse)).
+* Bump s2geometry again. [#66136](https://github.com/ClickHouse/ClickHouse/pull/66136) ([Robert Schulze](https://github.com/rschu1ze)).
+* Switch submodule `contrib/orc` to a proper commit in the [main](https://github.com/ClickHouse/orc/tree/main) branch. Previously a commit from a removed branch was used ([see](https://github.com/ClickHouse/orc/pull/13)). [#66137](https://github.com/ClickHouse/ClickHouse/pull/66137) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Finalize MergedBlockOutputStream in dtor. [#66138](https://github.com/ClickHouse/ClickHouse/pull/66138) ([Nikita Taranov](https://github.com/nickitat)).
+* Proper destruction order of AsyncLoader::Pool fields. [#66145](https://github.com/ClickHouse/ClickHouse/pull/66145) ([Sergei Trifonov](https://github.com/serxa)).
+* Playing minesweeper with build system. [#66147](https://github.com/ClickHouse/ClickHouse/pull/66147) ([Nikita Taranov](https://github.com/nickitat)).
+* Fix clang-tidy error in BufferWithOwnMemory.h. [#66161](https://github.com/ClickHouse/ClickHouse/pull/66161) ([Nikita Taranov](https://github.com/nickitat)).
+* Use peak_threads_usage instead of arrayUniq(thread_ids) in tests. [#66162](https://github.com/ClickHouse/ClickHouse/pull/66162) ([Azat Khuzhin](https://github.com/azat)).
+* Fix crash when adding empty tuple to query cache. [#66168](https://github.com/ClickHouse/ClickHouse/pull/66168) ([Michael Kolupaev](https://github.com/al13n321)).
+* tests: fix 01563_distributed_query_finish flakiness (due to system.*_log_sender). [#66171](https://github.com/ClickHouse/ClickHouse/pull/66171) ([Azat Khuzhin](https://github.com/azat)).
+* Refactor `OptimizeIfWithConstantConditionVisitor` using `InDepthNodeVisitor`. [#66184](https://github.com/ClickHouse/ClickHouse/pull/66184) ([zhongyuankai](https://github.com/zhongyuankai)).
+* Update README.md. [#66186](https://github.com/ClickHouse/ClickHouse/pull/66186) ([Tyler Hannan](https://github.com/tylerhannan)).
+* Fix 01246_buffer_flush flakiness. [#66188](https://github.com/ClickHouse/ClickHouse/pull/66188) ([Azat Khuzhin](https://github.com/azat)).
+* Avoid using harmful function `rand()` in grpc. [#66191](https://github.com/ClickHouse/ClickHouse/pull/66191) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Bump RocksDB. [#66216](https://github.com/ClickHouse/ClickHouse/pull/66216) ([Robert Schulze](https://github.com/rschu1ze)).
+* Update README.md. [#66217](https://github.com/ClickHouse/ClickHouse/pull/66217) ([Tyler Hannan](https://github.com/tylerhannan)).
+* Fixes peak_threads_usage metric when materialised views are involved. [#66230](https://github.com/ClickHouse/ClickHouse/pull/66230) ([Sema Checherinda](https://github.com/CheSema)).
+* Remove test as requested in https://github.com/ClickHouse/ClickHouse/pull/65277#issuecomment-2211361465. [#66233](https://github.com/ClickHouse/ClickHouse/pull/66233) ([Arthur Passos](https://github.com/arthurpassos)).
+* Fix test `00504_mergetree_arrays_rw.sql`. [#66248](https://github.com/ClickHouse/ClickHouse/pull/66248) ([Anton Popov](https://github.com/CurtizJ)).
+* CI: Do not finalize CI running status unless all success. [#66276](https://github.com/ClickHouse/ClickHouse/pull/66276) ([Max K.](https://github.com/maxknv)).
+* Collect core dumps in more tests. [#66281](https://github.com/ClickHouse/ClickHouse/pull/66281) ([Antonio Andelic](https://github.com/antonio2368)).
+* Add a stateless test for gRPC protocol. [#66284](https://github.com/ClickHouse/ClickHouse/pull/66284) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Log message: Failed to connect to replica ... [#66289](https://github.com/ClickHouse/ClickHouse/pull/66289) ([Igor Nikonov](https://github.com/devcrafter)).
+* Update run.sh. [#66290](https://github.com/ClickHouse/ClickHouse/pull/66290) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Some changes in the codebase as a preparation for LLVM 18. [#66293](https://github.com/ClickHouse/ClickHouse/pull/66293) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* There's some problem with iptables in parallel tests. [#66304](https://github.com/ClickHouse/ClickHouse/pull/66304) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Disable broken cases from 02911_join_on_nullsafe_optimization. [#66310](https://github.com/ClickHouse/ClickHouse/pull/66310) ([vdimir](https://github.com/vdimir)).
+* Build failure if ENABLE_AWS_S3 is OFF fixed. [#66335](https://github.com/ClickHouse/ClickHouse/pull/66335) ([Ilya Golshtein](https://github.com/ilejn)).
+* Enable checks in assert_cast under sanitizers. [#66336](https://github.com/ClickHouse/ClickHouse/pull/66336) ([Nikita Taranov](https://github.com/nickitat)).
+* Create release workflow. [#66339](https://github.com/ClickHouse/ClickHouse/pull/66339) ([Max K.](https://github.com/maxknv)).
+* Fix invalid XML. [#66342](https://github.com/ClickHouse/ClickHouse/pull/66342) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
+* Fix possible deadlock for jemalloc with enabled profiler. [#66346](https://github.com/ClickHouse/ClickHouse/pull/66346) ([Azat Khuzhin](https://github.com/azat)).
+* Fix test_parallel_replicas_custom_key. [#66349](https://github.com/ClickHouse/ClickHouse/pull/66349) ([Antonio Andelic](https://github.com/antonio2368)).
+* Collect logs from `minio` in stateless and statefull tests. [#66353](https://github.com/ClickHouse/ClickHouse/pull/66353) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
+* Fix "Sending a batch of X files to Y (0.00 rows, 0.00 B bytes)." in case of batch restoring. [#66375](https://github.com/ClickHouse/ClickHouse/pull/66375) ([Azat Khuzhin](https://github.com/azat)).
+* Fix 03030_system_flush_distributed_settings flakiness. [#66376](https://github.com/ClickHouse/ClickHouse/pull/66376) ([Azat Khuzhin](https://github.com/azat)).
+* PR cleanup: remove redundant code. [#66380](https://github.com/ClickHouse/ClickHouse/pull/66380) ([Igor Nikonov](https://github.com/devcrafter)).
+* New slack bot to post messages about CI events - Post message if OOM. [#66392](https://github.com/ClickHouse/ClickHouse/pull/66392) ([Max K.](https://github.com/maxknv)).
+* Ignore subquery for IN in DDLLoadingDependencyVisitor. [#66395](https://github.com/ClickHouse/ClickHouse/pull/66395) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Update test_storage_rabbitmq/test.py. [#66396](https://github.com/ClickHouse/ClickHouse/pull/66396) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Add query elapsed time for non-default format in play UI. [#66398](https://github.com/ClickHouse/ClickHouse/pull/66398) ([Azat Khuzhin](https://github.com/azat)).
+* Untangle setting headers. [#66404](https://github.com/ClickHouse/ClickHouse/pull/66404) ([Raúl Marín](https://github.com/Algunenano)).
+* Remove noisy message. [#66406](https://github.com/ClickHouse/ClickHouse/pull/66406) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* If job detected as in-progress in PR workflow run - just remove it from todo list, as it not affected by the change. [#66407](https://github.com/ClickHouse/ClickHouse/pull/66407) ([Max K.](https://github.com/maxknv)).
+* CI: CIBuddy to post from master only. [#66417](https://github.com/ClickHouse/ClickHouse/pull/66417) ([Max K.](https://github.com/maxknv)).
+* Add a test for [#66333](https://github.com/ClickHouse/ClickHouse/issues/66333). [#66432](https://github.com/ClickHouse/ClickHouse/pull/66432) ([max-vostrikov](https://github.com/max-vostrikov)).
+* Limit number of linker jobs on arm to avoid OOM during build. [#66435](https://github.com/ClickHouse/ClickHouse/pull/66435) ([Nikita Taranov](https://github.com/nickitat)).
+* [RFC] Fix jemalloc assertion due to non-monotonic CLOCK_MONOTONIC_COARSE. [#66439](https://github.com/ClickHouse/ClickHouse/pull/66439) ([Azat Khuzhin](https://github.com/azat)).
+* CI: Do not block CI on few number of test failures. [#66440](https://github.com/ClickHouse/ClickHouse/pull/66440) ([Max K.](https://github.com/maxknv)).
+* Stateless tests: fix flaky tests 01037_polygon_dicts*. [#66445](https://github.com/ClickHouse/ClickHouse/pull/66445) ([Nikita Fomichev](https://github.com/fm4v)).
+* Related to https://github.com/ClickHouse/ClickHouse/pull/62067 https://s3.amazonaws.com/clickhouse-test-reports/66410/5557dce188cabc7477bb4e874d47e3b80278ee66/stateless_tests__release_.html ``` 2024-07-12 16:04:29 +Queries for alter_table did not finish automatically after 250+ seconds 2024-07-12 16:04:29 +==================== QUERIES ==================== 2024-07-12 16:04:29 +Row 1: 2024-07-12 16:04:29 +────── 2024-07-12 16:04:29 +is_initial_query: 1 2024-07-12 16:04:29 +user: default 2024-07-12 16:04:29 +query_id: b43ffd7d-aee6-4161-aa82-bf9fff9d78c0 2024-07-12 16:04:29 +address: ::1 2024-07-12 16:04:29 +port: 58360 2024-07-12 16:04:29 +initial_user: default 2024-07-12 16:04:29 +initial_query_id: b43ffd7d-aee6-4161-aa82-bf9fff9d78c0 ... 2024-07-12 16:04:29 +query: OPTIMIZE TABLE alter_table0 FINAL ```. [#66460](https://github.com/ClickHouse/ClickHouse/pull/66460) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* OOM error was not visible since process is killed and status is not set Change sets ERROR status if job was killed. [#66463](https://github.com/ClickHouse/ClickHouse/pull/66463) ([Max K.](https://github.com/maxknv)).
+* Add AST fuzzers jobs for CI caching so that they can be skipped in PRs not related to build or tests. [#66468](https://github.com/ClickHouse/ClickHouse/pull/66468) ([Max K.](https://github.com/maxknv)).
+* If job with the same digest has been seen in master's CI it should be skipped in PR run. [#66471](https://github.com/ClickHouse/ClickHouse/pull/66471) ([Max K.](https://github.com/maxknv)).
+* CI: Check job's exit status and report if killed. [#66477](https://github.com/ClickHouse/ClickHouse/pull/66477) ([Max K.](https://github.com/maxknv)).
+* This closes [#37557](https://github.com/ClickHouse/ClickHouse/issues/37557). [#66482](https://github.com/ClickHouse/ClickHouse/pull/66482) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* CI: Add retry for GH set_status_comment call. [#66488](https://github.com/ClickHouse/ClickHouse/pull/66488) ([Max K.](https://github.com/maxknv)).
+* OpenSSL: Minor follow-up to [#66064](https://github.com/ClickHouse/ClickHouse/issues/66064). [#66489](https://github.com/ClickHouse/ClickHouse/pull/66489) ([Robert Schulze](https://github.com/rschu1ze)).
+* CI: Fix for job filtering in PRs. [#66490](https://github.com/ClickHouse/ClickHouse/pull/66490) ([Max K.](https://github.com/maxknv)).
+* CI: Create release workflow updates. [#66498](https://github.com/ClickHouse/ClickHouse/pull/66498) ([Max K.](https://github.com/maxknv)).
+* Add one more revision to ignore. [#66499](https://github.com/ClickHouse/ClickHouse/pull/66499) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Unit tests vomit a ton of garbage, see https://s3.amazonaws.com/clickhouse-test-reports/66457/0c82dc91f07b29ba503d7579c7d3ebecba532b73/unit_tests__tsan_/run.log - remove it. [#66501](https://github.com/ClickHouse/ClickHouse/pull/66501) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix double whitespace in EXPLAIN AST CREATE. [#66505](https://github.com/ClickHouse/ClickHouse/pull/66505) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix bad test `02530_dictionaries_update_field`. [#66507](https://github.com/ClickHouse/ClickHouse/pull/66507) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Https://play.clickhouse.com/play?user=play#u0vmrunuignozwnrx3n0yxj0x3rpbwusignozwnrx25hbwusihrlc3rfbmftzswgcmvwb3j0x3vybapguk9nignozwnrcwpxsevsrsbjagvja19zdgfydf90aw1lid49ig5vdygpic0gsu5urvjwquwgmjqwiehpvvikicagieforcbwdwxsx3jlcxvlc3rfbnvtymvyid0gmaogicagqu5eihrlc3rfc3rhdhvzice9icdts0lquevejwogicagqu5eihrlc3rfc3rhdhvziexjs0ugj0yljwogicagqu5eignozwnrx3n0yxr1cyahpsanc3vjy2vzcyckicagieforcbwb3npdglvbih0zxn0x25hbwusicdhcgfjagvfyxjyb3cnksa+idakt1jervigqlkgy2hly2tfc3rhcnrfdgltzq==. [#66508](https://github.com/ClickHouse/ClickHouse/pull/66508) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix MSan report in GRPC. [#66509](https://github.com/ClickHouse/ClickHouse/pull/66509) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* CI: Fix for skipping Builds_2 in PRs' CI. [#66512](https://github.com/ClickHouse/ClickHouse/pull/66512) ([Max K.](https://github.com/maxknv)).
+* CI: Do not block Tests_3 unless MAX_FAILED_TESTS exceeded. [#66513](https://github.com/ClickHouse/ClickHouse/pull/66513) ([Max K.](https://github.com/maxknv)).
+* Fix `02918_parallel_replicas_custom_key_unavailable_replica`. [#66516](https://github.com/ClickHouse/ClickHouse/pull/66516) ([Antonio Andelic](https://github.com/antonio2368)).
+* Stateless tests: improvements related to OOM of test runs. [#66520](https://github.com/ClickHouse/ClickHouse/pull/66520) ([Nikita Fomichev](https://github.com/fm4v)).
+* Tests: rename bad log names. [#66522](https://github.com/ClickHouse/ClickHouse/pull/66522) ([Nikita Fomichev](https://github.com/fm4v)).
+* Add additional log masking in CI. [#66523](https://github.com/ClickHouse/ClickHouse/pull/66523) ([Raúl Marín](https://github.com/Algunenano)).
+* CI: Multiple fixes for handling killed jobs. [#66524](https://github.com/ClickHouse/ClickHouse/pull/66524) ([Max K.](https://github.com/maxknv)).
+* Allow GWP Asan allocations only when initialization is finished. [#66526](https://github.com/ClickHouse/ClickHouse/pull/66526) ([Alexey Katsman](https://github.com/alexkats)).
+* Update 02443_detach_attach_partition.sh. [#66529](https://github.com/ClickHouse/ClickHouse/pull/66529) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Follow up [#66435](https://github.com/ClickHouse/ClickHouse/issues/66435). [#66530](https://github.com/ClickHouse/ClickHouse/pull/66530) ([Nikita Taranov](https://github.com/nickitat)).
+* fix log in keeper tcp handler. [#66531](https://github.com/ClickHouse/ClickHouse/pull/66531) ([Han Fei](https://github.com/hanfei1991)).
+* CI: Report job start and finish to CI DB. [#66533](https://github.com/ClickHouse/ClickHouse/pull/66533) ([Max K.](https://github.com/maxknv)).
+* Update 01396_inactive_replica_cleanup_nodes_zookeeper.sh. [#66535](https://github.com/ClickHouse/ClickHouse/pull/66535) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Add dedicated runner to libfuzzer, update docker. [#66551](https://github.com/ClickHouse/ClickHouse/pull/66551) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
+* fix tidy build. [#66552](https://github.com/ClickHouse/ClickHouse/pull/66552) ([Sema Checherinda](https://github.com/CheSema)).
+* No-op env change. [#66553](https://github.com/ClickHouse/ClickHouse/pull/66553) ([Raúl Marín](https://github.com/Algunenano)).
+* Fix typo in new_delete.cpp. [#66554](https://github.com/ClickHouse/ClickHouse/pull/66554) ([alesapin](https://github.com/alesapin)).
+* Fix something in Fast Test. [#66558](https://github.com/ClickHouse/ClickHouse/pull/66558) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* update trusted contributors. [#66561](https://github.com/ClickHouse/ClickHouse/pull/66561) ([Xu Jia](https://github.com/XuJia0210)).
+* Delete bad test `02805_distributed_queries_timeouts`. [#66563](https://github.com/ClickHouse/ClickHouse/pull/66563) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* More clarity in the test `03001_consider_lwd_when_merge`. [#66564](https://github.com/ClickHouse/ClickHouse/pull/66564) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Stateless tests: temporary disable sequential tests in parallel. [#66585](https://github.com/ClickHouse/ClickHouse/pull/66585) ([Nikita Fomichev](https://github.com/fm4v)).
+* Move view targets to separate AST class `ASTViewTargets` in order to allow extending it to support more kinds of view targets. [#66590](https://github.com/ClickHouse/ClickHouse/pull/66590) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Fix bsdtar for @nikitamikhaylov. [#66592](https://github.com/ClickHouse/ClickHouse/pull/66592) ([alesapin](https://github.com/alesapin)).
+* CI: POC for Auto Releases. [#66593](https://github.com/ClickHouse/ClickHouse/pull/66593) ([Max K.](https://github.com/maxknv)).
+* Fix clang tidy after [#66402](https://github.com/ClickHouse/ClickHouse/issues/66402). [#66597](https://github.com/ClickHouse/ClickHouse/pull/66597) ([vdimir](https://github.com/vdimir)).
+* Adjust the runtime of some slow performance test. [#66619](https://github.com/ClickHouse/ClickHouse/pull/66619) ([Robert Schulze](https://github.com/rschu1ze)).
+* CI: Scale down AutoScaling Groups from runners. [#66622](https://github.com/ClickHouse/ClickHouse/pull/66622) ([Max K.](https://github.com/maxknv)).
+* Allow to run clang-tidy with clang-19. [#66625](https://github.com/ClickHouse/ClickHouse/pull/66625) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix terrible test @arthurpassos. [#66632](https://github.com/ClickHouse/ClickHouse/pull/66632) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix bad log message. [#66633](https://github.com/ClickHouse/ClickHouse/pull/66633) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Negative sign in prewhere optimization. [#66637](https://github.com/ClickHouse/ClickHouse/pull/66637) ([cangyin](https://github.com/cangyin)).
+* Closes [#66639](https://github.com/ClickHouse/ClickHouse/issues/66639#event-13533944949). [#66640](https://github.com/ClickHouse/ClickHouse/pull/66640) ([Kruglov Pavel](https://github.com/Avogar)).
+* Avoid generating named tuple for special keywords (null, true, false). [#66641](https://github.com/ClickHouse/ClickHouse/pull/66641) ([Amos Bird](https://github.com/amosbird)).
+* rearrange heavy tests 03008_deduplication. [#66642](https://github.com/ClickHouse/ClickHouse/pull/66642) ([Sema Checherinda](https://github.com/CheSema)).
+* Fix data race in S3::ClientCache. [#66644](https://github.com/ClickHouse/ClickHouse/pull/66644) ([Konstantin Morozov](https://github.com/k-morozov)).
+* CI: Remove aws lambda packages from oss. [#66651](https://github.com/ClickHouse/ClickHouse/pull/66651) ([Max K.](https://github.com/maxknv)).
+* Introduce separate DEBUG_OR_SANITIZER_BUILD macro. [#66652](https://github.com/ClickHouse/ClickHouse/pull/66652) ([Nikita Taranov](https://github.com/nickitat)).
+* Increase backoff because with slow builds sometimes 100ms is not enough to recover. [#66653](https://github.com/ClickHouse/ClickHouse/pull/66653) ([alesapin](https://github.com/alesapin)).
+* Fix wrong queries hung error because of 02044_url_glob_parallel_connection_refused. [#66657](https://github.com/ClickHouse/ClickHouse/pull/66657) ([Nikita Taranov](https://github.com/nickitat)).
+* add log for splitBlockIntoParts. [#66658](https://github.com/ClickHouse/ClickHouse/pull/66658) ([Han Fei](https://github.com/hanfei1991)).
+* Minor: Make `CaseSensitiveness` an enum class. [#66673](https://github.com/ClickHouse/ClickHouse/pull/66673) ([Robert Schulze](https://github.com/rschu1ze)).
+* Fix bad test `02210_processors_profile_log`. [#66684](https://github.com/ClickHouse/ClickHouse/pull/66684) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix something around clang-tidy. [#66694](https://github.com/ClickHouse/ClickHouse/pull/66694) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* ci: dump dmesg in case of OOM. [#66705](https://github.com/ClickHouse/ClickHouse/pull/66705) ([Azat Khuzhin](https://github.com/azat)).
+* fix clang tidy. [#66706](https://github.com/ClickHouse/ClickHouse/pull/66706) ([Han Fei](https://github.com/hanfei1991)).
+* Https://s3.amazonaws.com/clickhouse-test-reports/61109/5cf2b53f146c1a4f24d8212f9f810d587c46bfc0/stateless_tests__release_.html. [#66724](https://github.com/ClickHouse/ClickHouse/pull/66724) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Support one more case in JOIN ON ... IS NULL. [#66725](https://github.com/ClickHouse/ClickHouse/pull/66725) ([vdimir](https://github.com/vdimir)).
+* CI: Fix issue with a skipped Build report. [#66726](https://github.com/ClickHouse/ClickHouse/pull/66726) ([Max K.](https://github.com/maxknv)).
+* relax condition in test, remove unused counters. [#66730](https://github.com/ClickHouse/ClickHouse/pull/66730) ([Sema Checherinda](https://github.com/CheSema)).
+* Remove bad test `host_resolver_fail_count`. [#66731](https://github.com/ClickHouse/ClickHouse/pull/66731) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix bad test `03036_join_filter_push_down_equivalent_sets`. [#66736](https://github.com/ClickHouse/ClickHouse/pull/66736) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix bad tests `long_select_and_alter`. [#66737](https://github.com/ClickHouse/ClickHouse/pull/66737) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Add test test_storage_mysql/test.py::test_joins. [#66743](https://github.com/ClickHouse/ClickHouse/pull/66743) ([vdimir](https://github.com/vdimir)).
+* Disallow build exclusion only by CI settings (ci_include_, ci_exclude_) to avoid running builds in auto sync prs. [#66744](https://github.com/ClickHouse/ClickHouse/pull/66744) ([Max K.](https://github.com/maxknv)).
+* Use non-existent address to check connection error at table creation. [#66760](https://github.com/ClickHouse/ClickHouse/pull/66760) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
+* Backported in [#67063](https://github.com/ClickHouse/ClickHouse/issues/67063): Increase asio pool size in case the server is tiny. [#66761](https://github.com/ClickHouse/ClickHouse/pull/66761) ([alesapin](https://github.com/alesapin)).
+* Fix flakiness of async insert tests due to adaptive timeout. [#66771](https://github.com/ClickHouse/ClickHouse/pull/66771) ([Raúl Marín](https://github.com/Algunenano)).
+* Attempt to fix flakiness of 01194_http_query_id. [#66774](https://github.com/ClickHouse/ClickHouse/pull/66774) ([Raúl Marín](https://github.com/Algunenano)).
+* Turn off randomization of harmful setting. [#66776](https://github.com/ClickHouse/ClickHouse/pull/66776) ([alesapin](https://github.com/alesapin)).
+* The number of batches was reduced in https://github.com/ClickHouse/ClickHouse/pull/65186, but then the parallel execution was disabled in https://github.com/ClickHouse/ClickHouse/pull/66585. So now tasks fail with timeout sometimes: https://s3.amazonaws.com/clickhouse-test-reports/66724/36275fdacc34206931f69087fe77539e25bbbedd/stateless_tests__tsan__s3_storage__[2_3].html. [#66783](https://github.com/ClickHouse/ClickHouse/pull/66783) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Remove obsolete code from CMakeLists. [#66786](https://github.com/ClickHouse/ClickHouse/pull/66786) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Stateless tests: split parallel tests more evenly. [#66787](https://github.com/ClickHouse/ClickHouse/pull/66787) ([Nikita Fomichev](https://github.com/fm4v)).
+* Fix test `02724_limit_num_mutations.sh`. [#66788](https://github.com/ClickHouse/ClickHouse/pull/66788) ([Anton Popov](https://github.com/CurtizJ)).
+* Better diagnostics in `test_disk_configuration`. [#66802](https://github.com/ClickHouse/ClickHouse/pull/66802) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix bad test `02950_part_log_bytes_uncompressed`. [#66803](https://github.com/ClickHouse/ClickHouse/pull/66803) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Better diagnostics for test trace_events_stress. [#66804](https://github.com/ClickHouse/ClickHouse/pull/66804) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Make test `00997_set_index_array` lighter. [#66817](https://github.com/ClickHouse/ClickHouse/pull/66817) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Miscellaneous. [#66818](https://github.com/ClickHouse/ClickHouse/pull/66818) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix inconsistent formatting of lambda functions inside composite types. [#66819](https://github.com/ClickHouse/ClickHouse/pull/66819) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Small fix in realloc memory tracking. [#66820](https://github.com/ClickHouse/ClickHouse/pull/66820) ([Antonio Andelic](https://github.com/antonio2368)).
+* Never await in CI on BuildReport - just redo (await can be longer) - Remove BuildReport if no build jobs in workflow (for instance: Docs change) - Do not fail CheckReadyForMerge job if the only non-green status is Cloud Sync. [#66822](https://github.com/ClickHouse/ClickHouse/pull/66822) ([Max K.](https://github.com/maxknv)).
+* Remove bad tests @azat. [#66823](https://github.com/ClickHouse/ClickHouse/pull/66823) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* CI: New Release workflow updates and fixes. [#66830](https://github.com/ClickHouse/ClickHouse/pull/66830) ([Max K.](https://github.com/maxknv)).
+* Fix signed integer overflow in function `age`. [#66831](https://github.com/ClickHouse/ClickHouse/pull/66831) ([Robert Schulze](https://github.com/rschu1ze)).
+* Fix RocksDB bs. [#66838](https://github.com/ClickHouse/ClickHouse/pull/66838) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Split a test for index. [#66839](https://github.com/ClickHouse/ClickHouse/pull/66839) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix inconsistent formatting of `NOT ((SELECT ...))`. [#66840](https://github.com/ClickHouse/ClickHouse/pull/66840) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Make test `01592_long_window_functions1` lighter. [#66841](https://github.com/ClickHouse/ClickHouse/pull/66841) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* What if I will change the test for SSL authentication?. [#66844](https://github.com/ClickHouse/ClickHouse/pull/66844) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Functions `[a-g]*`: Iterate over `input_rows_count` where appropriate. [#66846](https://github.com/ClickHouse/ClickHouse/pull/66846) ([Robert Schulze](https://github.com/rschu1ze)).
+* Revert libunwind patch. [#66850](https://github.com/ClickHouse/ClickHouse/pull/66850) ([Antonio Andelic](https://github.com/antonio2368)).
+* Split test 03038_nested_dynamic_merges to avoid timeouts. [#66863](https://github.com/ClickHouse/ClickHouse/pull/66863) ([Kruglov Pavel](https://github.com/Avogar)).
+* CI: Print instance info in runner's init script. [#66868](https://github.com/ClickHouse/ClickHouse/pull/66868) ([Max K.](https://github.com/maxknv)).
+* Backported in [#67257](https://github.com/ClickHouse/ClickHouse/issues/67257): Followup [#66725](https://github.com/ClickHouse/ClickHouse/issues/66725). [#66869](https://github.com/ClickHouse/ClickHouse/pull/66869) ([vdimir](https://github.com/vdimir)).
+* CI: CI Buddy to notify about fatal workflow failures. [#66890](https://github.com/ClickHouse/ClickHouse/pull/66890) ([Max K.](https://github.com/maxknv)).
+* CI: Add ec2 instance lifecycle metadata to CIDB. [#66918](https://github.com/ClickHouse/ClickHouse/pull/66918) ([Max K.](https://github.com/maxknv)).
+* CI: Remove ci runners scripts from oss. [#66920](https://github.com/ClickHouse/ClickHouse/pull/66920) ([Max K.](https://github.com/maxknv)).
+* Backported in [#67209](https://github.com/ClickHouse/ClickHouse/issues/67209): Decrease rate limit in `01923_network_receive_time_metric_insert`. [#66924](https://github.com/ClickHouse/ClickHouse/pull/66924) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
+* Backported in [#67227](https://github.com/ClickHouse/ClickHouse/issues/67227): Grouparrayintersect: fix serialization bug. [#66928](https://github.com/ClickHouse/ClickHouse/pull/66928) ([Raúl Marín](https://github.com/Algunenano)).
+* Backported in [#67207](https://github.com/ClickHouse/ClickHouse/issues/67207): Un-flake test_runtime_configurable_cache_size. [#66934](https://github.com/ClickHouse/ClickHouse/pull/66934) ([Robert Schulze](https://github.com/rschu1ze)).
+* Backported in [#66975](https://github.com/ClickHouse/ClickHouse/issues/66975): CI: Fixes docker server build for release branches. [#66955](https://github.com/ClickHouse/ClickHouse/pull/66955) ([Max K.](https://github.com/maxknv)).
+* Backported in [#67213](https://github.com/ClickHouse/ClickHouse/issues/67213): [CI Fest] Split dynamic tests and rewrite them from sh to sql to avoid timeouts. [#66981](https://github.com/ClickHouse/ClickHouse/pull/66981) ([Kruglov Pavel](https://github.com/Avogar)).
+* Backported in [#67033](https://github.com/ClickHouse/ClickHouse/issues/67033): [CI Fest] Fix use-of-uninitialized-value in JSONExtract* numeric functions. [#66984](https://github.com/ClickHouse/ClickHouse/pull/66984) ([Kruglov Pavel](https://github.com/Avogar)).
+* Backported in [#67051](https://github.com/ClickHouse/ClickHouse/issues/67051): CI: Fix for workflow results parsing. [#67000](https://github.com/ClickHouse/ClickHouse/pull/67000) ([Max K.](https://github.com/maxknv)).
+* Backported in [#67116](https://github.com/ClickHouse/ClickHouse/issues/67116): Disable setting `optimize_functions_to_subcolumns`. [#67046](https://github.com/ClickHouse/ClickHouse/pull/67046) ([Anton Popov](https://github.com/CurtizJ)).
+* Backported in [#67205](https://github.com/ClickHouse/ClickHouse/issues/67205): Increase max allocation size for sanitizers. [#67049](https://github.com/ClickHouse/ClickHouse/pull/67049) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
+* Backported in [#67124](https://github.com/ClickHouse/ClickHouse/issues/67124): Very sad failure: ``` 2024.07.24 13:28:45.517777 [ 10 ] {08745bf9-4bc1-4946-b9a8-c03d82ec55dc} <Debug> executeQuery: (from 172.16.11.1:55890) OPTIMIZE TABLE replicated_mt FINAL (stage: Complete) 2024.07.24 13:28:45.525945 [ 10 ] {08745bf9-4bc1-4946-b9a8-c03d82ec55dc} <Trace> default.replicated_mt (ReplicatedMergeTreeQueue): Waiting for 4 entries to be processed: queue-0000000004, queue-0000000002, queue-0000000001, queue-0000000000 2024.07.24 13:29:15.528024 [ 10 ] {08745bf9-4bc1-4946-b9a8-c03d82ec55dc} <Debug> default.replicated_mt (6581a6fb-8458-466d-8350-89951eb1ac8e) (MergerMutator): Selected 3 parts from all_0_0_0 to all_2_2_0 2024.07.24 13:29:15.530736 [ 10 ] {08745bf9-4bc1-4946-b9a8-c03d82ec55dc} <Trace> default.replicated_mt (6581a6fb-8458-466d-8350-89951eb1ac8e): Created log entry /clickhouse/tables/replicated_mt/log/log-0000000004 for merge all_0_2_1 2024.07.24 13:29:15.530873 [ 10 ] {08745bf9-4bc1-4946-b9a8-c03d82ec55dc} <Debug> default.replicated_mt (6581a6fb-8458-466d-8350-89951eb1ac8e): Waiting for node1 to process log entry 2024.07.24 13:29:15.530919 [ 10 ] {08745bf9-4bc1-4946-b9a8-c03d82ec55dc} <Debug> default.replicated_mt (6581a6fb-8458-466d-8350-89951eb1ac8e): Waiting for node1 to pull log-0000000004 to queue 2024.07.24 13:29:15.534286 [ 10 ] {08745bf9-4bc1-4946-b9a8-c03d82ec55dc} <Debug> default.replicated_mt (6581a6fb-8458-466d-8350-89951eb1ac8e): Looking for node corresponding to log-0000000004 in node1 queue 2024.07.24 13:29:15.534793 [ 10 ] {08745bf9-4bc1-4946-b9a8-c03d82ec55dc} <Debug> default.replicated_mt (6581a6fb-8458-466d-8350-89951eb1ac8e): Waiting for queue-0000000005 to disappear from node1 queue 2024.07.24 13:29:15.585533 [ 10 ] {08745bf9-4bc1-4946-b9a8-c03d82ec55dc} <Debug> TCPHandler: Processed in 30.067804125 sec. ```. [#67067](https://github.com/ClickHouse/ClickHouse/pull/67067) ([alesapin](https://github.com/alesapin)).
+* Backported in [#67203](https://github.com/ClickHouse/ClickHouse/issues/67203): Fix flaky `test_seekable_formats_url` and `test_seekable_formats` S3 storage tests. [#67070](https://github.com/ClickHouse/ClickHouse/pull/67070) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
+* Backported in [#67222](https://github.com/ClickHouse/ClickHouse/issues/67222): Fix 2680 flasky. [#67078](https://github.com/ClickHouse/ClickHouse/pull/67078) ([jsc0218](https://github.com/jsc0218)).
+* Backported in [#67190](https://github.com/ClickHouse/ClickHouse/issues/67190): Attempt to fix flakiness of some window view tests. [#67130](https://github.com/ClickHouse/ClickHouse/pull/67130) ([Robert Schulze](https://github.com/rschu1ze)).
+* Backported in [#67272](https://github.com/ClickHouse/ClickHouse/issues/67272): Rename (unreleased) bad setting. [#67149](https://github.com/ClickHouse/ClickHouse/pull/67149) ([Raúl Marín](https://github.com/Algunenano)).
+* Backported in [#67441](https://github.com/ClickHouse/ClickHouse/issues/67441): Try to fix 2572. [#67158](https://github.com/ClickHouse/ClickHouse/pull/67158) ([jsc0218](https://github.com/jsc0218)).
+* Backported in [#67416](https://github.com/ClickHouse/ClickHouse/issues/67416): CI: Fix build results for release branches. [#67402](https://github.com/ClickHouse/ClickHouse/pull/67402) ([Max K.](https://github.com/maxknv)).
+
diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv
index 271065a78fb..027b207d3ad 100644
--- a/utils/list-versions/version_date.tsv
+++ b/utils/list-versions/version_date.tsv
@@ -1,3 +1,4 @@
+v24.7.1.2915-stable	2024-07-30
 v24.6.2.17-stable	2024-07-05
 v24.6.1.4423-stable	2024-07-01
 v24.5.4.49-stable	2024-07-01

From 8d2b804c670d0941acc4fff059859017c1bd93c2 Mon Sep 17 00:00:00 2001
From: Michael Stetsyuk <mike.stetsyuk@gmail.com>
Date: Tue, 30 Jul 2024 21:57:08 +0100
Subject: [PATCH 1088/1488] fxs

---
 tests/integration/test_recovery_time_metric/test.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/tests/integration/test_recovery_time_metric/test.py b/tests/integration/test_recovery_time_metric/test.py
index 8f369d7759c..628f2e744e0 100644
--- a/tests/integration/test_recovery_time_metric/test.py
+++ b/tests/integration/test_recovery_time_metric/test.py
@@ -37,9 +37,7 @@ def test_recovery_time_metric(start_cluster):
         """
     )
 
-    node.exec_in_container(
-        ["bash", "-c", "rm /var/lib/clickhouse/metadata/rdb/t.sql"]
-    )
+    node.exec_in_container(["bash", "-c", "rm /var/lib/clickhouse/metadata/rdb/t.sql"])
 
     node.restart_clickhouse()
 

From 4aedb9d40298c1a3204bb72a3288ea711eb5e2f6 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Tue, 30 Jul 2024 23:02:22 +0200
Subject: [PATCH 1089/1488] Update test

---
 tests/integration/test_system_flush_logs/test.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tests/integration/test_system_flush_logs/test.py b/tests/integration/test_system_flush_logs/test.py
index 2022f9d4a89..713b327eb76 100644
--- a/tests/integration/test_system_flush_logs/test.py
+++ b/tests/integration/test_system_flush_logs/test.py
@@ -13,9 +13,8 @@ node = cluster.add_instance(
 )
 
 system_logs = [
-    # disabled by default
-    ("system.text_log", 0),
     # enabled by default
+    ("system.text_log", 1),
     ("system.query_log", 1),
     ("system.query_thread_log", 1),
     ("system.part_log", 1),

From 51212a414fd57270694c2653f730890b77714949 Mon Sep 17 00:00:00 2001
From: Michael Kolupaev <michael.kolupaev@clickhouse.com>
Date: Tue, 30 Jul 2024 21:13:20 +0000
Subject: [PATCH 1090/1488] Revert "Rename bad setting"

This reverts commit eb4ec0912ad3a1e89ea7aec424366bc268262e11.
---
 CHANGELOG.md                                              | 1 +
 src/Core/Settings.h                                       | 2 +-
 src/Core/SettingsChangesHistory.cpp                       | 2 +-
 src/Formats/FormatFactory.cpp                             | 2 +-
 src/Formats/FormatSettings.h                              | 2 +-
 src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp | 4 ++--
 tests/queries/0_stateless/03013_json_key_ignore_case.sh   | 4 ++--
 7 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index a4c873ba3f9..730346c0a2f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -65,6 +65,7 @@
 * The setting `optimize_trivial_insert_select` is disabled by default. In most cases, it should be beneficial. Nevertheless, if you are seeing slower INSERT SELECT or increased memory usage, you can enable it back or `SET compatibility = '24.6'`. [#58970](https://github.com/ClickHouse/ClickHouse/pull/58970) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
 * Print stacktrace and diagnostic info if `clickhouse-client` or `clickhouse-local` crashes. [#61109](https://github.com/ClickHouse/ClickHouse/pull/61109) ([Alexander Tokmakov](https://github.com/tavplubix)).
 * The result of `SHOW INDEX | INDEXES | INDICES | KEYS` was previously sorted by the primary key column names. Since this was unintuitive, the result is now sorted by the position of the primary key columns within the primary key. [#61131](https://github.com/ClickHouse/ClickHouse/pull/61131) ([Robert Schulze](https://github.com/rschu1ze)).
+* Change how deduplication for Materialized Views works. Fixed a lot of cases like: - on destination table: data is split for 2 or more blocks and that blocks is considered as duplicate when that block is inserted in parallel. - on MV destination table: the equal blocks are deduplicated, that happens when MV often produces equal data as a result for different input data due to performing aggregation. - on MV destination table: the equal blocks which comes from different MV are deduplicated. [#61601](https://github.com/ClickHouse/ClickHouse/pull/61601) ([Sema Checherinda](https://github.com/CheSema)).
 * Support reading partitioned data DeltaLake data. Infer DeltaLake schema by reading metadata instead of data. [#63201](https://github.com/ClickHouse/ClickHouse/pull/63201) ([Kseniia Sumarokova](https://github.com/kssenii)).
 * In composable protocols TLS layer accepted only `certificateFile` and `privateKeyFile` parameters. https://clickhouse.com/docs/en/operations/settings/composable-protocols. [#63985](https://github.com/ClickHouse/ClickHouse/pull/63985) ([Anton Ivashkin](https://github.com/ianton-ru)).
 * Added profile event `SelectQueriesWithPrimaryKeyUsage` which indicates how many SELECT queries use the primary key to evaluate the WHERE clause. [#64492](https://github.com/ClickHouse/ClickHouse/pull/64492) ([0x01f](https://github.com/0xfei)).
diff --git a/src/Core/Settings.h b/src/Core/Settings.h
index 27b71558bd3..0aa879fd9ad 100644
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@@ -1127,7 +1127,7 @@ class IColumn;
     M(Bool, input_format_json_defaults_for_missing_elements_in_named_tuple, true, "Insert default value in named tuple element if it's missing in json object", 0) \
     M(Bool, input_format_json_throw_on_bad_escape_sequence, true, "Throw an exception if JSON string contains bad escape sequence in JSON input formats. If disabled, bad escape sequences will remain as is in the data", 0) \
     M(Bool, input_format_json_ignore_unnecessary_fields, true, "Ignore unnecessary fields and not parse them. Enabling this may not throw exceptions on json strings of invalid format or with duplicated fields", 0) \
-    M(Bool, input_format_json_case_insensitive_column_matching, false, "Ignore case when matching JSON keys with CH columns", 0) \
+    M(Bool, input_format_json_ignore_key_case, false, "Ignore json key case while read json field from string", 0) \
     M(Bool, input_format_try_infer_integers, true, "Try to infer integers instead of floats while schema inference in text formats", 0) \
     M(Bool, input_format_try_infer_dates, true, "Try to infer dates from string fields while schema inference in text formats", 0) \
     M(Bool, input_format_try_infer_datetimes, true, "Try to infer datetimes from string fields while schema inference in text formats", 0) \
diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp
index 8bea0b1eed3..d38c8025227 100644
--- a/src/Core/SettingsChangesHistory.cpp
+++ b/src/Core/SettingsChangesHistory.cpp
@@ -64,7 +64,7 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
               {"input_format_native_decode_types_in_binary_format", false, false, "Added new setting to allow to read type names in binary format in Native output format"},
               {"read_in_order_use_buffering", false, true, "Use buffering before merging while reading in order of primary key"},
               {"enable_named_columns_in_function_tuple", false, true, "Generate named tuples in function tuple() when all names are unique and can be treated as unquoted identifiers."},
-              {"input_format_json_case_insensitive_column_matching", false, false, "Ignore case when matching JSON keys with CH columns."},
+              {"input_format_json_ignore_key_case", false, false, "Ignore json key case while read json field from string."},
               {"optimize_trivial_insert_select", true, false, "The optimization does not make sense in many cases."},
               {"dictionary_validate_primary_key_type", false, false, "Validate primary key type for dictionaries. By default id type for simple layouts will be implicitly converted to UInt64."},
               {"collect_hash_table_stats_during_joins", false, true, "New setting."},
diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp
index a2890ab849d..99bfe761d7b 100644
--- a/src/Formats/FormatFactory.cpp
+++ b/src/Formats/FormatFactory.cpp
@@ -150,7 +150,7 @@ FormatSettings getFormatSettings(const ContextPtr & context, const Settings & se
     format_settings.json.try_infer_objects_as_tuples = settings.input_format_json_try_infer_named_tuples_from_objects;
     format_settings.json.throw_on_bad_escape_sequence = settings.input_format_json_throw_on_bad_escape_sequence;
     format_settings.json.ignore_unnecessary_fields = settings.input_format_json_ignore_unnecessary_fields;
-    format_settings.json.case_insensitive_column_matching = settings.input_format_json_case_insensitive_column_matching;
+    format_settings.json.ignore_key_case = settings.input_format_json_ignore_key_case;
     format_settings.null_as_default = settings.input_format_null_as_default;
     format_settings.force_null_for_omitted_fields = settings.input_format_force_null_for_omitted_fields;
     format_settings.decimal_trailing_zeros = settings.output_format_decimal_trailing_zeros;
diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h
index 72d1515b7e1..a736bb1a558 100644
--- a/src/Formats/FormatSettings.h
+++ b/src/Formats/FormatSettings.h
@@ -233,7 +233,7 @@ struct FormatSettings
         bool infer_incomplete_types_as_strings = true;
         bool throw_on_bad_escape_sequence = true;
         bool ignore_unnecessary_fields = true;
-        bool case_insensitive_column_matching = false;
+        bool ignore_key_case = false;
     } json{};
 
     struct
diff --git a/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp b/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp
index 0e3972e8c45..be4e9430c34 100644
--- a/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp
+++ b/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp
@@ -46,7 +46,7 @@ JSONEachRowRowInputFormat::JSONEachRowRowInputFormat(
 {
     const auto & header = getPort().getHeader();
     name_map = header.getNamesToIndexesMap();
-    if (format_settings_.json.case_insensitive_column_matching)
+    if (format_settings_.json.ignore_key_case)
     {
         for (auto & it : name_map)
         {
@@ -181,7 +181,7 @@ void JSONEachRowRowInputFormat::readJSONObject(MutableColumns & columns)
             continue;
         }
         size_t column_index = 0;
-        if (format_settings.json.case_insensitive_column_matching)
+        if (format_settings.json.ignore_key_case)
         {
             String lower_case_name = transformFieldNameToLowerCase(name_ref);
             StringRef field_name_ref = lower_case_name_map[lower_case_name];
diff --git a/tests/queries/0_stateless/03013_json_key_ignore_case.sh b/tests/queries/0_stateless/03013_json_key_ignore_case.sh
index 7882a29ae4f..807e743b22a 100755
--- a/tests/queries/0_stateless/03013_json_key_ignore_case.sh
+++ b/tests/queries/0_stateless/03013_json_key_ignore_case.sh
@@ -13,6 +13,6 @@ cp "$CURDIR"/data_json/key_ignore_case.json $USER_FILES_PATH/
 
 $CLICKHOUSE_CLIENT -q "drop table if exists test_tbl"
 $CLICKHOUSE_CLIENT -q "create table test_tbl (id UInt16, reqid UInt32, name String) engine=MergeTree order by id"
-$CLICKHOUSE_CLIENT -q "INSERT INTO test_tbl SELECT * FROM file('key_ignore_case.json', 'JSONEachRow') SETTINGS input_format_json_case_insensitive_column_matching=true"
+$CLICKHOUSE_CLIENT -q "INSERT INTO test_tbl SELECT * FROM file('key_ignore_case.json', 'JSONEachRow') SETTINGS input_format_json_ignore_key_case=true"
 $CLICKHOUSE_CLIENT -q "select * from test_tbl"
-$CLICKHOUSE_CLIENT -q "drop table test_tbl"
+$CLICKHOUSE_CLIENT -q "drop table test_tbl"
\ No newline at end of file

From 07291cb70828dba7a1fc331a2c0e6a58bb00d774 Mon Sep 17 00:00:00 2001
From: Michael Kolupaev <michael.kolupaev@clickhouse.com>
Date: Tue, 30 Jul 2024 21:14:22 +0000
Subject: [PATCH 1091/1488] Revert "Merge pull request #61750 from
 KevinyhZou/improve_json_each_row_ignore_key_case"

This reverts commit 3229cb5874cc9c8b04e16d2fec3231a25a3fc171, reversing
changes made to f838c25d20bc87ed9a788eff0659d26be42cadb6.
---
 src/Core/Settings.h                           |   1 -
 src/Core/SettingsChangesHistory.cpp           |   1 -
 src/Formats/FormatFactory.cpp                 |   1 -
 src/Formats/FormatSettings.h                  |   1 -
 .../Impl/JSONEachRowRowInputFormat.cpp        |  19 +-----------------
 .../Formats/Impl/JSONEachRowRowInputFormat.h  |  11 ++--------
 .../03013_json_key_ignore_case.reference      |   3 ---
 .../0_stateless/03013_json_key_ignore_case.sh |  18 -----------------
 .../data_json/key_ignore_case.json            | Bin 123 -> 0 bytes
 9 files changed, 3 insertions(+), 52 deletions(-)
 delete mode 100644 tests/queries/0_stateless/03013_json_key_ignore_case.reference
 delete mode 100755 tests/queries/0_stateless/03013_json_key_ignore_case.sh
 delete mode 100644 tests/queries/0_stateless/data_json/key_ignore_case.json

diff --git a/src/Core/Settings.h b/src/Core/Settings.h
index 0aa879fd9ad..8cc25f42cc6 100644
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@@ -1127,7 +1127,6 @@ class IColumn;
     M(Bool, input_format_json_defaults_for_missing_elements_in_named_tuple, true, "Insert default value in named tuple element if it's missing in json object", 0) \
     M(Bool, input_format_json_throw_on_bad_escape_sequence, true, "Throw an exception if JSON string contains bad escape sequence in JSON input formats. If disabled, bad escape sequences will remain as is in the data", 0) \
     M(Bool, input_format_json_ignore_unnecessary_fields, true, "Ignore unnecessary fields and not parse them. Enabling this may not throw exceptions on json strings of invalid format or with duplicated fields", 0) \
-    M(Bool, input_format_json_ignore_key_case, false, "Ignore json key case while read json field from string", 0) \
     M(Bool, input_format_try_infer_integers, true, "Try to infer integers instead of floats while schema inference in text formats", 0) \
     M(Bool, input_format_try_infer_dates, true, "Try to infer dates from string fields while schema inference in text formats", 0) \
     M(Bool, input_format_try_infer_datetimes, true, "Try to infer datetimes from string fields while schema inference in text formats", 0) \
diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp
index d38c8025227..873578013e1 100644
--- a/src/Core/SettingsChangesHistory.cpp
+++ b/src/Core/SettingsChangesHistory.cpp
@@ -64,7 +64,6 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
               {"input_format_native_decode_types_in_binary_format", false, false, "Added new setting to allow to read type names in binary format in Native output format"},
               {"read_in_order_use_buffering", false, true, "Use buffering before merging while reading in order of primary key"},
               {"enable_named_columns_in_function_tuple", false, true, "Generate named tuples in function tuple() when all names are unique and can be treated as unquoted identifiers."},
-              {"input_format_json_ignore_key_case", false, false, "Ignore json key case while read json field from string."},
               {"optimize_trivial_insert_select", true, false, "The optimization does not make sense in many cases."},
               {"dictionary_validate_primary_key_type", false, false, "Validate primary key type for dictionaries. By default id type for simple layouts will be implicitly converted to UInt64."},
               {"collect_hash_table_stats_during_joins", false, true, "New setting."},
diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp
index 99bfe761d7b..1fe3e0f25ff 100644
--- a/src/Formats/FormatFactory.cpp
+++ b/src/Formats/FormatFactory.cpp
@@ -150,7 +150,6 @@ FormatSettings getFormatSettings(const ContextPtr & context, const Settings & se
     format_settings.json.try_infer_objects_as_tuples = settings.input_format_json_try_infer_named_tuples_from_objects;
     format_settings.json.throw_on_bad_escape_sequence = settings.input_format_json_throw_on_bad_escape_sequence;
     format_settings.json.ignore_unnecessary_fields = settings.input_format_json_ignore_unnecessary_fields;
-    format_settings.json.ignore_key_case = settings.input_format_json_ignore_key_case;
     format_settings.null_as_default = settings.input_format_null_as_default;
     format_settings.force_null_for_omitted_fields = settings.input_format_force_null_for_omitted_fields;
     format_settings.decimal_trailing_zeros = settings.output_format_decimal_trailing_zeros;
diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h
index a736bb1a558..891b107bb2a 100644
--- a/src/Formats/FormatSettings.h
+++ b/src/Formats/FormatSettings.h
@@ -233,7 +233,6 @@ struct FormatSettings
         bool infer_incomplete_types_as_strings = true;
         bool throw_on_bad_escape_sequence = true;
         bool ignore_unnecessary_fields = true;
-        bool ignore_key_case = false;
     } json{};
 
     struct
diff --git a/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp b/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp
index be4e9430c34..8855a1bc28d 100644
--- a/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp
+++ b/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp
@@ -46,15 +46,6 @@ JSONEachRowRowInputFormat::JSONEachRowRowInputFormat(
 {
     const auto & header = getPort().getHeader();
     name_map = header.getNamesToIndexesMap();
-    if (format_settings_.json.ignore_key_case)
-    {
-        for (auto & it : name_map)
-        {
-            StringRef key = it.first;
-            String lower_case_key = transformFieldNameToLowerCase(key);
-            lower_case_name_map[lower_case_key] = key;
-        }
-    }
     if (format_settings_.import_nested_json)
     {
         for (size_t i = 0; i != header.columns(); ++i)
@@ -180,15 +171,7 @@ void JSONEachRowRowInputFormat::readJSONObject(MutableColumns & columns)
             skipUnknownField(name_ref);
             continue;
         }
-        size_t column_index = 0;
-        if (format_settings.json.ignore_key_case)
-        {
-            String lower_case_name = transformFieldNameToLowerCase(name_ref);
-            StringRef field_name_ref = lower_case_name_map[lower_case_name];
-            column_index = columnIndex(field_name_ref, key_index);
-        }
-        else
-            column_index = columnIndex(name_ref, key_index);
+        const size_t column_index = columnIndex(name_ref, key_index);
 
         if (unlikely(ssize_t(column_index) < 0))
         {
diff --git a/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.h b/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.h
index 8a1cef8fa9f..d97aa2dad8d 100644
--- a/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.h
+++ b/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.h
@@ -55,13 +55,7 @@ private:
 
     virtual void readRowStart(MutableColumns &) {}
     virtual void skipRowStart() {}
-    String transformFieldNameToLowerCase(const StringRef & field_name)
-    {
-        String field_name_str = field_name.toString();
-        std::transform(field_name_str.begin(), field_name_str.end(), field_name_str.begin(),
-            [](unsigned char c) { return std::tolower(c); });
-        return field_name_str;
-    }
+
     /// Buffer for the read from the stream field name. Used when you have to copy it.
     /// Also, if processing of Nested data is in progress, it holds the common prefix
     /// of the nested column names (so that appending the field name to it produces
@@ -80,8 +74,7 @@ private:
 
     /// Hash table match `field name -> position in the block`. NOTE You can use perfect hash map.
     Block::NameMap name_map;
-    /// Hash table match `lower_case field name -> field name in the block`.
-    std::unordered_map<String, StringRef> lower_case_name_map;
+
     /// Cached search results for previous row (keyed as index in JSON object) - used as a hint.
     std::vector<Block::NameMap::const_iterator> prev_positions;
 
diff --git a/tests/queries/0_stateless/03013_json_key_ignore_case.reference b/tests/queries/0_stateless/03013_json_key_ignore_case.reference
deleted file mode 100644
index 54683d8fbc5..00000000000
--- a/tests/queries/0_stateless/03013_json_key_ignore_case.reference
+++ /dev/null
@@ -1,3 +0,0 @@
-1	77328912	Ben
-2	77328913	Jim
-3	77328914	Bill
diff --git a/tests/queries/0_stateless/03013_json_key_ignore_case.sh b/tests/queries/0_stateless/03013_json_key_ignore_case.sh
deleted file mode 100755
index 807e743b22a..00000000000
--- a/tests/queries/0_stateless/03013_json_key_ignore_case.sh
+++ /dev/null
@@ -1,18 +0,0 @@
-#!/usr/bin/env bash
-
-# NOTE: this sh wrapper is required because of shell_config
-
-CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
-# shellcheck source=../shell_config.sh
-. "$CURDIR"/../shell_config.sh
-
-
-USER_FILES_PATH=$($CLICKHOUSE_CLIENT --query "select _path,_file from file('nonexist.txt', 'CSV', 'val1 char')" 2>&1 | grep -E '^Code: 107.*FILE_DOESNT_EXIST' | head -1 | awk '{gsub("/nonexist.txt","",$9); print $9}')
-
-cp "$CURDIR"/data_json/key_ignore_case.json $USER_FILES_PATH/
-
-$CLICKHOUSE_CLIENT -q "drop table if exists test_tbl"
-$CLICKHOUSE_CLIENT -q "create table test_tbl (id UInt16, reqid UInt32, name String) engine=MergeTree order by id"
-$CLICKHOUSE_CLIENT -q "INSERT INTO test_tbl SELECT * FROM file('key_ignore_case.json', 'JSONEachRow') SETTINGS input_format_json_ignore_key_case=true"
-$CLICKHOUSE_CLIENT -q "select * from test_tbl"
-$CLICKHOUSE_CLIENT -q "drop table test_tbl"
\ No newline at end of file
diff --git a/tests/queries/0_stateless/data_json/key_ignore_case.json b/tests/queries/0_stateless/data_json/key_ignore_case.json
deleted file mode 100644
index ad8f7cb450780891d64ac8cbbc19de17b92e7db5..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 123
zcmb<c%1lwRGSpE}DoQQ%1TxLdjg2fU4UK>zd5O8HN>)lvsd-AZT-9KOAYoL6#t4O8
VnYnO<AYoL6CJ2R2nK?O1wE)OmAZ`Ev


From 4a4bd97b4b63e495f76273ea8c12045dc129d81b Mon Sep 17 00:00:00 2001
From: Peter Nguyen <petern0408@gmail.com>
Date: Tue, 30 Jul 2024 16:50:37 -0600
Subject: [PATCH 1092/1488] Fix case sensitivity for percent_rank, dense_rank,
 and their aliases

---
 src/Processors/Transforms/WindowTransform.cpp | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/Processors/Transforms/WindowTransform.cpp b/src/Processors/Transforms/WindowTransform.cpp
index a1b46c8e36c..5fad68e4968 100644
--- a/src/Processors/Transforms/WindowTransform.cpp
+++ b/src/Processors/Transforms/WindowTransform.cpp
@@ -2726,18 +2726,18 @@ void registerWindowFunctions(AggregateFunctionFactory & factory)
         {
             return std::make_shared<WindowFunctionDenseRank>(name, argument_types,
                 parameters);
-        }, properties}, AggregateFunctionFactory::Case::Insensitive);
+        }, properties});
 
-    factory.registerAlias("dense_rank", "denseRank", AggregateFunctionFactory::Case::Sensitive);
+    factory.registerAlias("dense_rank", "denseRank", AggregateFunctionFactory::Case::Insensitive);
 
     factory.registerFunction("percentRank", {[](const std::string & name,
             const DataTypes & argument_types, const Array & parameters, const Settings *)
         {
             return std::make_shared<WindowFunctionPercentRank>(name, argument_types,
                 parameters);
-        }, properties}, AggregateFunctionFactory::Case::Insensitive);
+        }, properties});
 
-    factory.registerAlias("percent_rank", "percentRank", AggregateFunctionFactory::Case::Sensitive);
+    factory.registerAlias("percent_rank", "percentRank", AggregateFunctionFactory::Case::Insensitive);
 
     factory.registerFunction("row_number", {[](const std::string & name,
             const DataTypes & argument_types, const Array & parameters, const Settings *)

From 3b12fec141fda8cb2a3ef68ac96e6e58f1fd69e3 Mon Sep 17 00:00:00 2001
From: Peter Nguyen <petern0408@gmail.com>
Date: Tue, 30 Jul 2024 17:05:48 -0600
Subject: [PATCH 1093/1488] Update dense_rank doc to mention the denseRank
 alias

---
 docs/en/sql-reference/window-functions/dense_rank.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/en/sql-reference/window-functions/dense_rank.md b/docs/en/sql-reference/window-functions/dense_rank.md
index d6445b68c55..2c8617fb668 100644
--- a/docs/en/sql-reference/window-functions/dense_rank.md
+++ b/docs/en/sql-reference/window-functions/dense_rank.md
@@ -12,6 +12,8 @@ The [rank](./rank.md) function provides the same behaviour, but with gaps in ran
 
 **Syntax**
 
+Alias: `denseRank` (case-sensitive)
+
 ```sql
 dense_rank (column_name)
   OVER ([[PARTITION BY grouping_column] [ORDER BY sorting_column] 

From 419a5e7f730dabe514becabc6c24ec5b87325e28 Mon Sep 17 00:00:00 2001
From: Peter Nguyen <petern0408@gmail.com>
Date: Tue, 30 Jul 2024 17:17:01 -0600
Subject: [PATCH 1094/1488] Update window-functions doc with denseRank and
 percentRank aliases

---
 docs/en/sql-reference/window-functions/index.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/en/sql-reference/window-functions/index.md b/docs/en/sql-reference/window-functions/index.md
index 0c3e2ea1cb6..27d4bd763c7 100644
--- a/docs/en/sql-reference/window-functions/index.md
+++ b/docs/en/sql-reference/window-functions/index.md
@@ -23,8 +23,8 @@ ClickHouse supports the standard grammar for defining windows and window functio
 | `INTERVAL` syntax for `DateTime` `RANGE OFFSET` frame                              | ❌ (specify the number of seconds instead (`RANGE` works with any numeric type).)                                                                                                                                 |
 | `GROUPS` frame                                                                     | ❌                                                                                                                                                                               |
 | Calculating aggregate functions over a frame (`sum(value) over (order by time)`)   | ✅ (All aggregate functions are supported)                                                                                                                                                       |
-| `rank()`, `dense_rank()`, `row_number()`                                           | ✅                                                                                                                                                                                   |
-| `percent_rank()` | ✅  Efficiently computes the relative standing of a value within a partition in a dataset. This function effectively replaces the more verbose and computationally intensive manual SQL calculation expressed as `ifNull((rank() OVER(PARTITION BY x ORDER BY y) - 1) / nullif(count(1) OVER(PARTITION BY x) - 1, 0), 0)`| 
+| `rank()`, `dense_rank()`, `row_number()`                                           | ✅ <br/>Alias: `denseRank()`                                                                                                                                                                                   |
+| `percent_rank()` | ✅  Efficiently computes the relative standing of a value within a partition in a dataset. This function effectively replaces the more verbose and computationally intensive manual SQL calculation expressed as `ifNull((rank() OVER(PARTITION BY x ORDER BY y) - 1) / nullif(count(1) OVER(PARTITION BY x) - 1, 0), 0)` <br/>Alias: `percentRank()`| 
 | `lag/lead(value, offset)`                                                          | ❌ <br/> You can use one of the following workarounds:<br/> 1) `any(value) over (.... rows between <offset> preceding and <offset> preceding)`, or `following` for `lead` <br/> 2) `lagInFrame/leadInFrame`, which are analogous, but respect the window frame. To get behavior identical to `lag/lead`, use `rows between unbounded preceding and unbounded following`                                                                 |
 | ntile(buckets) | ✅ <br/> Specify window like, (partition by x order by y rows between unbounded preceding and unrounded following). |
 

From bc312eb046db07d901e208cdc1bb0abb1df3eabd Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Tue, 30 Jul 2024 21:27:50 +0200
Subject: [PATCH 1095/1488] Improve check

---
 src/IO/S3/URI.cpp                             | 32 ++++++++++++-------
 src/Storages/StorageFile.cpp                  | 18 +++++------
 .../03215_parsing_archive_name_s3.sql         |  2 +-
 3 files changed, 30 insertions(+), 22 deletions(-)

diff --git a/src/IO/S3/URI.cpp b/src/IO/S3/URI.cpp
index 33a4939c810..fead18315d8 100644
--- a/src/IO/S3/URI.cpp
+++ b/src/IO/S3/URI.cpp
@@ -55,10 +55,10 @@ URI::URI(const std::string & uri_, bool allow_archive_path_syntax)
     static constexpr auto OSS = "OSS";
     static constexpr auto EOS = "EOS";
 
-    if (!allow_archive_path_syntax)
-        uri_str = uri_;
-    else
+    if (allow_archive_path_syntax)
         std::tie(uri_str, archive_pattern) = getURIAndArchivePattern(uri_);
+    else
+        uri_str = uri_;
 
     uri = Poco::URI(uri_str);
 
@@ -176,22 +176,30 @@ std::pair<std::string, std::optional<std::string>> URI::getURIAndArchivePattern(
         return {source, std::nullopt};
 
     std::string_view path_to_archive_view = std::string_view{source}.substr(0, pos);
+    bool contains_spaces_around_operator = false;
     while (path_to_archive_view.ends_with(' '))
+    {
+        contains_spaces_around_operator = true;
         path_to_archive_view.remove_suffix(1);
-
-    if (path_to_archive_view.empty() || !hasSupportedArchiveExtension(path_to_archive_view))
-        return {source, std::nullopt};
-
-    auto archive_uri = path_to_archive_view;
+    }
 
     std::string_view archive_pattern_view = std::string_view{source}.substr(pos + 2);
-    while (archive_pattern_view.front() == ' ')
+    while (archive_pattern_view.starts_with(' '))
+    {
+        contains_spaces_around_operator = true;
         archive_pattern_view.remove_prefix(1);
+    }
 
-    if (archive_pattern_view.empty())
-        throw Exception(ErrorCodes::BAD_ARGUMENTS, "Archive pattern is empty");
+    /// possible situations when the first part can be archive is only if one of the following is true:
+    /// - it contains supported extension
+    /// - it contains spaces after or before :: (URI cannot contain spaces)
+    /// - it contains characters that could mean glob expression
+    if (archive_pattern_view.empty() || path_to_archive_view.empty()
+        || (!contains_spaces_around_operator && !hasSupportedArchiveExtension(path_to_archive_view)
+            && path_to_archive_view.find_first_of("*?{") == std::string_view::npos))
+        return {source, std::nullopt};
 
-    return std::pair{std::string{archive_uri}, std::string{archive_pattern_view}};
+    return std::pair{std::string{path_to_archive_view}, std::string{archive_pattern_view}};
 }
 }
 
diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp
index efb39f90053..8c079aa4600 100644
--- a/src/Storages/StorageFile.cpp
+++ b/src/Storages/StorageFile.cpp
@@ -2258,21 +2258,21 @@ void StorageFile::parseFileSource(String source, String & filename, String & pat
     while (path_to_archive_view.ends_with(' '))
         path_to_archive_view.remove_suffix(1);
 
-    if (path_to_archive_view.empty() || !hasSupportedArchiveExtension(path_to_archive_view))
+    std::string_view filename_view = std::string_view{source}.substr(pos + 2);
+    while (filename_view.starts_with(' '))
+        filename_view.remove_prefix(1);
+
+    /// possible situations when the first part can be archive is only if one of the following is true:
+    /// - it contains supported extension
+    /// - it contains characters that could mean glob expression
+    if (filename_view.empty() || path_to_archive_view.empty()
+        || (!hasSupportedArchiveExtension(path_to_archive_view) && path_to_archive_view.find_first_of("*?{") == std::string_view::npos))
     {
         filename = std::move(source);
         return;
     }
 
     path_to_archive = path_to_archive_view;
-
-    std::string_view filename_view = std::string_view{source}.substr(pos + 2);
-    while (filename_view.front() == ' ')
-        filename_view.remove_prefix(1);
-
-    if (filename_view.empty())
-        throw Exception(ErrorCodes::BAD_ARGUMENTS, "Filename is empty");
-
     filename = filename_view;
 }
 
diff --git a/tests/queries/0_stateless/03215_parsing_archive_name_s3.sql b/tests/queries/0_stateless/03215_parsing_archive_name_s3.sql
index 3a7ed0b864c..e34be475c5a 100644
--- a/tests/queries/0_stateless/03215_parsing_archive_name_s3.sql
+++ b/tests/queries/0_stateless/03215_parsing_archive_name_s3.sql
@@ -2,6 +2,6 @@
 -- Tag no-fasttest: Depends on AWS
 
 SELECT _file, _path FROM s3(s3_conn, filename='::03215_archive.csv') ORDER BY (_file, _path);
-SELECT _file, _path FROM s3(s3_conn, filename='test :: 03215_archive.csv') ORDER BY (_file, _path); -- { serverError STD_EXCEPTION }
+SELECT _file, _path FROM s3(s3_conn, filename='test :: 03215_archive.csv') ORDER BY (_file, _path); -- { serverError S3_ERROR }
 SELECT _file, _path FROM s3(s3_conn, filename='test::03215_archive.csv') ORDER BY (_file, _path);
 SELECT _file, _path FROM s3(s3_conn, filename='test.zip::03215_archive.csv') ORDER BY (_file, _path) SETTINGS allow_archive_path_syntax=0;

From e664a144788b48c029f56548242baaeed82a80ff Mon Sep 17 00:00:00 2001
From: Michael Stetsyuk <mike.stetsyuk@gmail.com>
Date: Wed, 31 Jul 2024 08:49:14 +0100
Subject: [PATCH 1096/1488] fix style

---
 src/Databases/DatabaseReplicated.cpp          | 4 ++--
 src/Storages/System/StorageSystemClusters.cpp | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp
index 06cea65d62e..b2be593d326 100644
--- a/src/Databases/DatabaseReplicated.cpp
+++ b/src/Databases/DatabaseReplicated.cpp
@@ -356,7 +356,7 @@ ReplicasInfo DatabaseReplicated::tryGetReplicasInfo(const ClusterPtr & cluster_)
             paths_get.emplace_back(fs::path(zookeeper_path) / "replicas" / full_name / "log_ptr");
         }
     }
-    
+
     try
     {
         auto current_zookeeper = getZooKeeper();
@@ -396,7 +396,7 @@ ReplicasInfo DatabaseReplicated::tryGetReplicasInfo(const ClusterPtr & cluster_)
                 ++global_replica_index;
             }
         }
-    
+
         return replicas_info;
     } catch (...)
     {
diff --git a/src/Storages/System/StorageSystemClusters.cpp b/src/Storages/System/StorageSystemClusters.cpp
index 4b9802c9089..d03b600b6ef 100644
--- a/src/Storages/System/StorageSystemClusters.cpp
+++ b/src/Storages/System/StorageSystemClusters.cpp
@@ -116,7 +116,7 @@ void StorageSystemClusters::writeCluster(MutableColumns & res_columns, const Nam
                 else
                     res_columns[i++]->insertDefault();
             }
-            
+
             ++global_replica_idx;
         }
     }

From 5152248d438ef9162845507b68e18f1d8541a250 Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitlibar@clickhouse.com>
Date: Wed, 31 Jul 2024 09:25:59 +0200
Subject: [PATCH 1097/1488] Add test.

---
 .../02864_restore_table_with_broken_part.sh   |  25 ++--------------
 ...ackup_with_matview_inner_table_metadata.sh |  25 ++--------------
 ..._clear_old_temporary_directories.reference |   2 ++
 ...kup_and_clear_old_temporary_directories.sh |  22 ++++++++++++++
 .../0_stateless/backups/mt_250_parts.zip      | Bin 0 -> 265998 bytes
 .../helpers/install_predefined_backup.sh      |  27 ++++++++++++++++++
 6 files changed, 55 insertions(+), 46 deletions(-)
 create mode 100644 tests/queries/0_stateless/03214_backup_and_clear_old_temporary_directories.reference
 create mode 100755 tests/queries/0_stateless/03214_backup_and_clear_old_temporary_directories.sh
 create mode 100644 tests/queries/0_stateless/backups/mt_250_parts.zip
 create mode 100755 tests/queries/0_stateless/helpers/install_predefined_backup.sh

diff --git a/tests/queries/0_stateless/02864_restore_table_with_broken_part.sh b/tests/queries/0_stateless/02864_restore_table_with_broken_part.sh
index 08313e2fd3b..bf76727f76f 100755
--- a/tests/queries/0_stateless/02864_restore_table_with_broken_part.sh
+++ b/tests/queries/0_stateless/02864_restore_table_with_broken_part.sh
@@ -5,29 +5,8 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
-# Copies a test predefined backup from "/tests/queries/0_stateless/backups/" folder to the "backups" disk,
-# returns the path to the backup relative to that disk.
-function install_test_backup()
-{
-    local test_backup_filename="$1"
-    local test_backup_path="$CURDIR/backups/${test_backup_filename}"
-
-    local backups_disk_root
-    backups_disk_root=$($CLICKHOUSE_CLIENT --query "SELECT path FROM system.disks WHERE name='backups'")
-
-    if [ -z "${backups_disk_root}" ]; then
-        echo "Disk '${backups_disk_root}' not found"
-        exit 1
-    fi
-
-    local install_path=${backups_disk_root}/${CLICKHOUSE_DATABASE}/${test_backup_filename}
-    mkdir -p "$(dirname "${install_path}")"
-    ln -s "${test_backup_path}" "${install_path}"
-
-    echo "${CLICKHOUSE_DATABASE}/${test_backup_filename}"
-}
-
-backup_name="$(install_test_backup with_broken_part.zip)"
+# In this test we restore from "/tests/queries/0_stateless/backups/with_broken_part.zip"
+backup_name="$($CURDIR/helpers/install_predefined_backup.sh with_broken_part.zip)"
 
 $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS tbl"
 
diff --git a/tests/queries/0_stateless/03001_restore_from_old_backup_with_matview_inner_table_metadata.sh b/tests/queries/0_stateless/03001_restore_from_old_backup_with_matview_inner_table_metadata.sh
index 8d987dbf1df..2c70cb1e3be 100755
--- a/tests/queries/0_stateless/03001_restore_from_old_backup_with_matview_inner_table_metadata.sh
+++ b/tests/queries/0_stateless/03001_restore_from_old_backup_with_matview_inner_table_metadata.sh
@@ -5,29 +5,8 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
-# Copies a test predefined backup from "/tests/queries/0_stateless/backups/" folder to the "backups" disk,
-# returns the path to the backup relative to that disk.
-function install_test_backup()
-{
-    local test_backup_filename="$1"
-    local test_backup_path="$CURDIR/backups/${test_backup_filename}"
-
-    local backups_disk_root
-    backups_disk_root=$($CLICKHOUSE_CLIENT --query "SELECT path FROM system.disks WHERE name='backups'")
-
-    if [ -z "${backups_disk_root}" ]; then
-        echo "Disk '${backups_disk_root}' not found"
-        exit 1
-    fi
-
-    local install_path=${backups_disk_root}/${CLICKHOUSE_DATABASE}/${test_backup_filename}
-    mkdir -p "$(dirname "${install_path}")"
-    ln -s "${test_backup_path}" "${install_path}"
-
-    echo "${CLICKHOUSE_DATABASE}/${test_backup_filename}"
-}
-
-backup_name="$(install_test_backup old_backup_with_matview_inner_table_metadata.zip)"
+# In this test we restore from "/tests/queries/0_stateless/backups/old_backup_with_matview_inner_table_metadata.zip"
+backup_name="$($CURDIR/helpers/install_predefined_backup.sh old_backup_with_matview_inner_table_metadata.zip)"
 
 $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS mv"
 $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS src"
diff --git a/tests/queries/0_stateless/03214_backup_and_clear_old_temporary_directories.reference b/tests/queries/0_stateless/03214_backup_and_clear_old_temporary_directories.reference
new file mode 100644
index 00000000000..3f3fbd9ab58
--- /dev/null
+++ b/tests/queries/0_stateless/03214_backup_and_clear_old_temporary_directories.reference
@@ -0,0 +1,2 @@
+RESTORED
+250	31375
diff --git a/tests/queries/0_stateless/03214_backup_and_clear_old_temporary_directories.sh b/tests/queries/0_stateless/03214_backup_and_clear_old_temporary_directories.sh
new file mode 100755
index 00000000000..e0c8f08e695
--- /dev/null
+++ b/tests/queries/0_stateless/03214_backup_and_clear_old_temporary_directories.sh
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+# Tags: no-fasttest
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+# In this test we restore from "/tests/queries/0_stateless/backups/mt_250_parts.zip"
+backup_name="$($CURDIR/helpers/install_predefined_backup.sh mt_250_parts.zip)"
+
+${CLICKHOUSE_CLIENT} -nm --query "
+DROP TABLE IF EXISTS manyparts;
+CREATE TABLE manyparts (x Int64) ENGINE=MergeTree ORDER BY tuple() SETTINGS merge_tree_clear_old_temporary_directories_interval_seconds=1, temporary_directories_lifetime=1;
+"
+
+# RESTORE must protect its temporary directories from removing.
+${CLICKHOUSE_CLIENT} --query "RESTORE TABLE default.mt_250_parts AS manyparts FROM Disk('backups', '${backup_name}') SETTINGS allow_different_table_def=true" | grep -o "RESTORED"
+
+${CLICKHOUSE_CLIENT} -nm --query "
+SELECT count(), sum(x) FROM manyparts;
+DROP TABLE manyparts;
+"
diff --git a/tests/queries/0_stateless/backups/mt_250_parts.zip b/tests/queries/0_stateless/backups/mt_250_parts.zip
new file mode 100644
index 0000000000000000000000000000000000000000..15310b79054e09c75876efd20845bbdabdfec2f5
GIT binary patch
literal 265998
zcmZ^LbyU>d_qBycBi$g~(%s$NAT23KcS(1Hv~;I1q$7e1-7V7HNF(_UqxgN-`pxq%
zi?u%M-gC}A`|PvtD9Jv3hVkghix-bx^WJF%st<_?m_2%wHSzor?4w7I7>$igtex#v
zfqS3Ho1zSV(_TH{Cl!!FHF@ii#D+%V3LtZC0OSLHxL*cIVv#w>nQ$yQ9NY-Mexva=
zsnm9PanzoGNa2X{s=+tY_fqig;^!Uf?MjvJWrObx+4Wh-Y2Sj^#lqdqkniWShC`Fv
zt)Z2wuDh$4yI&2hH(Rg4|7q>}`&`gi<n|-^oyad=^50b{cXR~{x4(Y3`d(cv+^zb)
zd&(QO*^0<?h23d<mz!~D@B*;9m6tpG-q-D>tX1627ys_E;X*!oVJD{b#`>!7BCnwp
z5NcB2diVRi??vB&kD-F#E%2sm;r24*Y(wzQFHxWDrtP-M=x(pd_pZ<2u5aP?YQg8~
z`%dc(_%D~lD=|{P8w9`joc_H0cq=t<jKjUb(=??b+h;tZ`kgOr;`Zm&)y-jp;C1Bs
zr?9Ps+w1ez>m5Ofjw6)EO=o5_KTWB*C!Q_VN}HsCysUt9*{ydw3%9QbE<!BU<#62`
zDpOh6qh@34bLJd1=v)-pTJPutcNcDt4(^z5J^H_208p;FUW^F_rqq(te%DRwijmI`
z1C+d};A#2jDRSL+q04(aN8D;aWw46OXWf}HEtO|UbJ#)>_VLxz9JxlSYTvsU!JC2`
z@GTqMuHN1FMi*U+3KeMYb9n#?$IN4@M!K6x9wYzqd~&&S5hDl?yxa1vYbCV2Y`wdD
ze|ISeQa`>UtsIf~pxxGH{UOCPg}U%=IGH5MbNTY%t^oYc$UzsxB3q#WZwIY7$}x*8
z><cW#Nql@>xCR;f{*Ji|zH3p5wOmh>kr;4%G9~9dp16!nfsKEx!}JTUXy@%xiBN8V
z@O%3<cTwxuSN?bsH-#y;j@xTHPke%o6*2dRe0|{dZhNnW?0|WVLECo?@IFMRzUVUy
z7k51_TqrlK$eUP`=<RVG8P(JS;|_p0t8chG(mR_;2Q*lBP3_#DuI|+N8cc8L0^-z9
zm^%G3;%e1a;$+kLtoC4jWo6t^;p}?lE(q)OGJfsUM!6)JRKqC3CIp)HnQ0;9h|Y@Y
zIUVyB-tb)r|Lh5f*v=u+yHNbKXw1LV7N<*u24m7OzGj(WIJiXOt+TOujuElP-ze02
zL=u2?z8w{`V&$JdY<yBop`VcN>F_lUc8gu%JRn7{d{`_=AeLkwC54`Gi>FR6bLJrB
z?06B@h6;tTG{nPhNw6&-;xrJJoAH~7rHb0yWYb|svt1P#wW!g(EaMs=EcT`*QR4jb
z33_?etns<ffjs>a_>lnFR;mqr_gI6Ka@eBv(GYQ2t+^52C~kX_T-~!NUptBc=wE5O
zX;fD5-bOq=9S6~H4uz?xZ8<5{OwR`(oY0{tR0^n(3~_i3W{|BI?sApK*UlvzNAaZ?
ztkl6u_HTxdnhiVxM5h*}+0|IIRJ^&SKPMdXA-c}qTWu3RRYGCBrgF2Z$@*mox)@nE
z)o~*KCek!zl*c<RoLA==4mu;!%U<vnz;Pe!sGPAHU1GGy#dLlT;@a?19OOJQbdMVQ
zgrZ}^O_h;sZXUhGCyTMo7c)h8x}6v8DtG1nE1*6B7JEXJzwy_n#Ws*fdw}wmM+viy
z<A{cY1(j{8WQZ`<ZH>p@v$PDsA~?*Nm^z8Kwb8%{<KxH}SN%B*b(Q^O02Ze!Oxm#=
zhdEL{9zR#=Bo=1>>#xx#(UyWJKEp0Yi@6@fu-Lcge2rR)Zm}I=>~Zc+ek^5f_3%-;
z7&|<l7gXu%&ljKE*n&hW`KqIEkru~k)bP?w#$QH>I5-1Gs1j}XwC*Dl;jeTO=-6q>
zlAWXGomf0ILXOomjq=rGW+;HTE|agZm}t8#=<XQ`<0LxKu9hRqxJ2Z)469&n_4UwJ
z43RHITJH|pU@ntj@e|o{5KcMLb#qqLd3iZVD%?~*%y$P|;+gZO7#nj+YXoZHId7wn
zktkG9khQwC96~wCkXv=Ca@u53{kzf`O0kk*=~|QuPRzeKY-rV^Jx(^ky}>dN%_+fL
zpsgBb5);jPpT;p&wODG~0Esr^_+_BW@bg;E@I+1pFZ-eJ1+tl;4+!t5p5B^z6yN!@
z3%2<PN)MGMS%@on4GOdrv)NR<#d^nKO$3}*CavdOQ0zYtKq;4&dd|jO@3E!D^ihp=
z@<%pW6z>r*<`g;Cx7{<SB~**@c!m#j_jxpwrUQ;u6PUQFDEBk<0J|<~_Po%e@Miu3
z<<~;`X@Aj&?QQSCOG<Hu`mG8SMVKb#<y0^VwDA5yx=ex@fNiqIfX`f%cG#L6p5qBC
zoZavFEkP7G#Raqya0ufv8FZ@FDY3RU!BtDdTLPp<DMC##g&!*E;>9}W^C}o1Y7&*t
z4e>_ve`8;X-)|$tZi~avU{-N@L87F(lT6ty592|GvTU8<gN%1-X+hXj_@p$zTz1vc
z48xUi3aj167+T;nMw<<B4{H-y$}SU^0a1|}G@M`mMg9%i6VU87g*!&fbJE1<xhx=^
zVGE@2^;b@;+LEII0v35{^m2TlD+~36!N#SQGxtR*s1T;{tUOk6V;{XZ#74rE0&kM7
zjUQy}iNZL1aM{*<%w%ED4?=nYP=PJhwCw8Alt1R@Ug!?E24`3|=I0V}1$h>@E<RXC
zS*%;(eQf><XbHAUq}my)b3{O^9B;nEUa7KS#gSv)^7km2u*yO(bg2!SYR|{9oU6^~
zgG%+iv2Nhr;-jpCqaq_x;`DB|8CFg%pCF|)54CLR3m0I4d2gc)Wk!KWPYvaCvEA1n
z7JDaEHJML&JCtH})XAz;MTIB;SU>apAt?Jz7MXb*9hAfGI_5eFaDaWW<8ScVASYR4
zAM`*C6Q*+rOGloHHO!wiFqb_snIdXRUWW!R>xmRyS=zi-P9Kop4fOVlmqro@PN)9?
zF$4E%zlMw${!gkqzy=k3ipX1CTF$SRUpMdT9*i#@*ns^_7~N2+M`Q0Q3iYk~KZQ|a
zyH4D8)O~^#wBU8OQ|2EJK)8b=(km}=L7y1kt_eY;6^R#lRB@V!$uuwyPUD}Rb{~A!
zggYXY;d_K4<l3X<qs*Lw)xGs*#rHlc)gKyq-=n3~Q=$MF6;Mu7M?P}4uxhhiOT94E
z=mR_vvHJiqj9s^Q1N0x)0h%#CErD)g=e*f}T39wMN)pCf=3c|B$l4C_r}7D&?TqKE
zUSufS)){ApB5Txrg=xm5sn&2WmB-C!3L+zRHlfC5?76ub%n<uhhlON0WoZRjNs->=
z#78&{A;@{$w=1i&HJM`D-}cV}-ZDM`oV#e=!<gW0nEDBpZ(Y1~aj+pJi_K(rXDnRa
z6$nOH(B+Njoym916Tz%6KDS*a8bfznEiU@nc@*5m4&hg;A11BeV}>z+hMMLe2or<8
zv+lW5OF-8w%A!|O<Hk7RsCb}7F&B5>T^<7N$5m;H-ZHGg8rq%ZfrwbY316kI+u0;_
zP!;a=pCimj?0c&6kNLd6hP^l#@Gv7Dk&&3lGhOV?(slAPQ8}T|w(AD5x-2~2+yBP7
zsfktIOHqg}8`jZ<ISx-zlVXXgI<NH)Wg^lZvAPp##NZB~^F@wPFwWWX8~iN2Mu*Zb
zJ?JRaV>LW*Hn>cNg=1@EokBrYfQNor5&um`60bY8&6AK}&7`Zj@taz(S>rtdpx!qa
zigkrD8kyR&Rnl+@G?bADPC}+ipw~&*|7~PLyQOnS{ftc7IZDAgrDwK=Bl#A3+5bq)
zV%aT#Jnl%!am3=gc+p2Sr&8_a2b|eLORk`jU{WUg9vJ7w2=Lg)-2q7wLgyDNu)M4%
zje=$|pZPwkNR-YyoyDa+RHtoHhiMVKnS0?Q0^>)kYRgvJWGi3(vY3;7W)M6vEGOP1
zDSd_b9vXEa9ME7Ebs5@){ot1<#FBZ+M#~h%BJYse{#{yjJo@3nyalGi`%^Rn<Gk?n
z8S+4&)$Y>y4G04K1|vYVd}_R>rG3oEoOA9}hlq21R)N907HGK)hk_OJTlP&6LW6}Q
zYEzV4f0oL}2)e2H$2RvZd5#vYVFY#E{>#L-c=pF-sPzP4#7*<}WJyaO9F|v}{flZk
zA34xn6rVarv*7}s=v{AKK_g4r>{aA!%x@>+d#Sgyq}L8@(oC`_9{5FXK;T>C=e5EG
z`4^AJ3pr$8k2D}MaAIyoK~RQMd1ZXMprJ7`X*v~O%w#jsv&;U_IUgFDnN9EwScoFl
z5M5~n-^gU*26`FF!vB9<DK*=u#?}gl5W(sosg`Oz#kS|*zFn4A+28Wx(v3SZvWD0Q
zDO=In_c6D953ys0N-oD~WDTRGzr?ja+K`q^U+yfx!fN^p{&V28QkL1FL4t?nVIziX
zQozp4GuUer%z3vW3`Vk@&931s4nGw_vh@agl4f;+)X3L9h*wa0jG`OUOnf{*A$>VD
zCh2%JU;A5y-FKe{(l=-NqR4KKEAKqoVKYw(Z!qtS<qGs>+~>Z@5E+;=XuW)=Vddi%
z@^<tcmL`*$e0=R2J@GqFu`6!74|}HHKFOh+E+}45O&?Hgr4BuF7y8Z<>MmAcEt(Qa
zVxRbxXMCUf<<Im3s&&nIYw-}L5z+tvJj-GK57@IO%)n8?-}7A_iQu+OG@}|`kP2wh
zMPuet7$UdTfIWAxIUj~jV}9mz?T72RxggKZ=P|~-)TAUXdk(t=?#!;RUxizFb(sh^
zoVbwXx48OyMFK<^MA3t1wz4wbuL`?Gjop0O*s}V8P9RH`!Gtj$jWmwtIc8l3BM7Z)
zsvk?@cgve;6-FsPE|zzaV{I1Jd|(-KMIR;{rj)DCrkVsT@&Fpvsc#Sryu9!38x6_V
z&Mn2K1ErH}GI>kLe*G2pqQ?>x^iB*=Q$S9zJIp9vxHHRRaJrpa-5iG*v(c@x<8Pbc
z)vb1|n(#n6CU_sBfs=eXeclv;@M__{YNYO2xdIz?7r(?dE2Ad!(e64>nuc&4#MTOh
zEVH&)p=+cd0h91FSH5lib~7TQFAk2~|Jybjbd(UzW;{$j0i=5Ks>uok*e;LC&WS$V
zpQg2QYtAHk>Pv48%Ba#<PoC&exMhbPUjCH+#U+&XZ0@2=rf%~*%t5tsraAv2PKA(0
zrz7oOe{RciCZUs-?NTs!`^<<?k8BQf1=`E<{mW^5dq<0Z!0HyT)F#h6;9};k4HD)W
z8^}VPFac#QuR+;tHh2{|HO?go(T>PCIY$rtY(9Y|_Gdmun&m!U^trfsW~K#~76>w8
z_E~3~8<r)p5TS_3uW(cu?@vgez;4yqLyt~>NlV;XnpKL~`ty07UnHaP(cXnJegA6#
z8MW%)w?*qjyi~(j6ln9<w|Q)P!6=Y8#dV{n7N%Zpo+3F%iVy$2Fkdc}{f*b^9h3qc
z;=>-OFd=->g%Gj*);Nn$`N!&X$n-;K*4iu-@~aDs)(=0y=<?K8q*K75pabt)Ks7ke
z&c&TMI<cAnKJOwrN<#DG>6_Gptntabp|EDwmaa%1!Vjq>xJ8M1??{-F8wDarXa7?3
z!GMD1!*^n&Gv@wNORmpH@>cb}K(M(O%*_aaY~9&;iD@gPQIoL|VzS)@Y(Ztj-=WQ9
ztV7~-BZ<TMYOq;6boG=be)Zh4Uz?}hVCwFNa@T_%lE^9U<9D?#6ftGp(Lk^0=8+so
zHoJLc0<zn7F;|*l9w#bPtYs|gzgEqk?E?+K`@~oX!ZKE8B3>kH<=kGu-95XyA?IKE
zYBn7&`gy&pZxtv@gyC}7$d+!^k|*}={nt+^3wybgJr`UXhHl&9+}u|1J8UwR$Z4*t
zp2R!$9#OUPJ0Hdgj^uLED{E2qOPq*RLM2+ya(19{n!>9wkBsb&VX9i`QY|kR3%;>K
zO)x><y(cuT??F;>x$)Kk+F#DHtffgY(>e&dzqmzDyuL?ix2Rg8M4l8QF+gt=m+juD
zABXglz6{1Jh6`w1BZGmu%QMG07AM3Y-pqzSHEt|XWA&ey;WX*5##)~oGr(|M(*kF)
zpB?m*9>O-klhZyd86f!yOtZ3$48JW$=_JvV?Y7t6<2z_GO|56Yl471BD0jXl)lRxZ
z-LW%e(C(oZO!(J6CX5kB7_?2w$<0o5$Php=ga{2Jz)p7*<Z#jnb6cd}(-IXpAbBF9
zTAH6A-7nBu=lb4o^k$bd+LoK<Q$4BAotmIZ@`8s3y+0-#6x`WVw+{H~qAa_eK}gBj
zdf7H{KL>P!bB%L^1~_PfY4)N^!b}<|egw5#4tpWuH#DH(>$Wu8;?o(TVU#T8BMA_9
zj5kvH*FAAts10v6uU`<xTc?iTD-gcbA;gYpyP^Pp2CU7T(1z(~1s4+da=S#B^EQhP
zW!E|a8()E0Ei@&|W0qO8#N8=1SvIF!z|UhpNT>6GI%R%l)9PVW&yS~74~j1Q{!L1a
z(f0gWminh#^%rn;no^=5x7HE45*B;#oxG;dIoze$J^^jyLL-9XmH~@%1oFiq?^&he
zuriktd5#(z>n;d~|B67NUIcF^^8HUrLFoSZGtuA*@=^thK-piwvYbd+$xb2>CpM<z
z9#X0o+wh0XE5HDp0F<r<hoWGLbR1Re8|C;C8HEKPkcmGEst8&5!_kspKky<&pS5TN
zFWHjhcSpK+j&FNFq$cmE<DGSo>g+gDk%)6=gCtGUz(~l@HuihYd8l8BwhgAC2|>X{
zqPNWiHPs<AmCmXCQGoWFs1={Bns`$(^$RmMkW-wGc#@n$p!t+tq4l*#1_dFjIgi<}
zF{|XVqu$pnS`dXdbZ`!YjY0Z?V-;oUs8!t-8z1gGaXAxq@vmme^?xSnEN|0TBBFQF
z5SlN)%epQE(}1>>opP8wz&avKnYlIjI6=9q9o3n}-pmg)y4)Lr#&0exi;9IImj_hW
z=)30$Cy`f54E{o+&nBe9t*}N%=-4t%T@yq12F}rsTgKKYhN$6FOe0@bbbliP8Y~re
z(#SYAku6+ly}sSYF$ZeO<yZ)BG3tHyHH9hB?cw&Ra^D;K2A$T>!Ue9#s^{J@l`GYJ
zEFQGwK|8r;X>%Xm4_DE1(S~|l$0eCXc%l+3t+;#_pRkLD+}RSCq1(ki>B^_1jS}_2
zvQ)4aHYYlE`|@A?%x7l0mdC|HEbC>_AWKQ^s;PZW^50}Oa7Wd(>7r|7`K;KNq4U~y
z)+w|I`4qyOyrRb5hsdNehp0Ygi1*l^F0nexSABFb;|Qi`|5s;R52F@i#es>9%@uI^
z_=H_k<7p_o$Uo}bASxYRLK+ZJ?OqZfi@8&4$0Y|D0rRK0cG?IEd&WEySGa3B2_+I=
z%O--EMr$MUDAc+XK0=i><gr8+NwVaRVQ$<O`_ecu{%<@d;!W-XB$}8}P}?{#Sen8F
zA_sK+(`U3wT$kZ6Q*Fkx`Q{UZOA^8ZE&$MJQyICA<!qQx93I7Y%GyIN7o~`W&E@z~
z*FWR=xr@akwvV_s1_|?&fxcdiolZNZ<Pcm23p6)b{ai%9&8y~LanefS&wGT!v4=l4
z_zw<{@mX!$I!jvT<bx4W?WS5hJb)Yz^YCvjyAFv{QPgo<CbF1`5PpB25D`fNywp2E
z@8uw(zh{Bdm@682=Z&FpMMNmP;kuKQBNY0x{OiyLL2hU4JTlzVY3EN;<9Xy=k<qeA
zx-a`y{uPZJ;tY}*YfL1S3w}jU=sRAQFO9Jf`2>tJTVoV2%7p9Y0{sD}d;nCf2s#g(
zMSytfCn(~4zoW8fiZQ^Lj>f2uR&+W+w*Hy=|45*o&L`H5sayS69sJ^fTq~5+jzGw*
zO-^y`2_MT)a=%-_EL|If_wHZuDjPpE{&@q1Hz`(RD+;`D2*?PRH21f^>(_Eh%T|97
zfZqs8eZT4_{He_P$j(fdGj6lD?RAzD1euk<_}?5R@FFR#qoWd#v?5m)zg{CuqS<C$
zL=TlX716t&cuO}Xg$4CDO7L@JSDS=5+4q4Sa^QEiYCAQ0yXfd45t-|^@<rgp_-f%3
z;JpkyRz|t3!%q~Uf`<g;S+?)6Uz(@vV*4}|e+_klf;)E>(2t;@)l-TnBTQ$&D=}-I
zf1TxX>&XR~p<H*?X@=?H?Engvol%n31vt*42X^)wN3J9-T2<ik@vb*ydB#L48<3)U
zYyp#y=QzZQw<jwN7SS?)`w0K+_I_V#K64Ot_vj9&JyecW$AealQ<DZL#$@6F45~dL
z{sqp0mZuBZ2HL~SQS>^~9`ZVG2S->qgEt?)+82G$X}ps=s#B&a8YU~^?ao$$$vDpf
zB~E8be?vop&VYf`(vw1D?qZJ(jkW0@`$3K7xra@_1mACxw!%1D^LLvSn!=Hlv9OL1
zoffRpTo-*vp<4T)7D1$5hEB68Fl?9TVQ&R}t_jmHlFA=@S<o77@HAU&(ke6T`d>Eh
z&eVW1gLxf@fepB=rq81XgPgY^AaH_Lq^K_Q^jn&mNFc2Mf-63tHuMM16o&-Vk!C*p
zvD1sH$P$`w)r547<}#`i$5O`+wRy9+YxIq+{Nv`A5{*9i(Ts5O*Vsd8clS&KOlU5T
z+p-El&a_SatC%x~BXk3ka;oE15e;t8C_o@ALRnsioB&a*L&Z3^WKb4UZ=L_zztgZO
zl(T!Z>F`M-6FoVcUr8o+e+_}>wrji%&4G`l_ARfnk$EJqo4D8t#&MIXDzL7h@g|Ld
zkZ0+J7iL6VI})m!@ri|LC}Gmx4<wM))ib@Kkg-DdeOf^Lab4<)q}^=7>_3@?TbEOf
ziH0H!D?U}Hy<McBsLbXZ+Wv10CGA6pyaP&|z<2+szD3cXiNZelzv8*VTxOhO6z<zD
z6%Jm32<K;h1gZ#EI1m8nwa(Cg%|$X{QTzP^77SfX9JOZ|8QuZQ%zu!kKv;QR#0tAz
zE=+zBHHAahS~Zk!h!3r%Pc_*r$u_#Fy?S|nl|=}_gc6@~4Xn_DQO379Q1U#4;pZ#D
zz9x?S?WMNH6!X?b+hu5a(+~BZv*uYC2^U-;`V9MNylc$-5vm{mKL%K;_{&!Mh~7ei
zl=~$w4qYB(paBm~GXeGn!s_il_O+60LIhkT8~P0Ezd`5X(6zHcK0RGB)F<zi>?Gh!
zC2d`Ht2q7#m=?084to_tOTZ7O0x|NtM+Bj)e>f8uV-&^nfRoIs)<*3aTn&6H`cKPH
zKSpDx??c@+DK@@Ni%We=$-w{Af6X#ps3Pn{?Iank2Kd{@*k=tmT9`zQA>K^&Hq#;n
z4Us;~y?ORt2&UR%AFRSkG!nOR@&TxyGwTspi-ZJn8}K%RH)t;FS1>kY=fILI%T~_Y
zr`P9Z{9`wuhOuIQLdwLH9?Cbbl-a0KiVSA$?g)qXk-Y+Z8bUavib{Lvm`$)!yM9#J
zrkRxh{;V9pD{oh2Itf5*E$Jhysd7J{`ilbsMefRi-_RnXO%W&aiBzDbw~iXu=}V<@
zu7=%4;_-t9kmbJ~7gI6DLWNl)<=9V4Sb<2u+y@!YJHXA^G8-TEB{K0L(htOC57`0F
zVD8u|VBi(B(TwlqTGnzQH;OT-Cy+HZ2zbhvtNQz2Rc6c^Zr7AZZvm2cnpQ2onOh^}
zk|aa88CX3(AY#o4b(T%OutSQv&}Ho(@$5qC0-!WQ6`YQf0&yG(Y1VCLIYh7dx(<1S
z(YC~*9+IFT-D0T6arMnuf2`+hm|TGXoZmMP1e907%t_crjn!}~Lq<ND1C6jsz+|0L
z47=zIT4e&P1ZIR@D?gD$N9?yi90}n<%2QeTcbe5e?U!$HidN_8;*160CDqT}?}rRS
zpZncOCm&H*m|?yMP)NZWqeHG6_ZZ~!3tC)$8L9yKt)9k6N1$R`F>@zZ|IY8(dceyf
zq-F_mdX^>((@)Q7X(V?>r)67bP<%vwbOx-kD|l9GPv9M2xjmh6Dg`~0!8>=)GYbAM
zg>KGdh9s2%KHzND;eGZ@_~*$C-d7KbMsGlBbWCM1ugZ87pNP0puAZk0579DYIZq9S
z<tA$R(zp3)P5oG!w$hU-gv>S1Xx{KDT|(nb?gn**qctAR8|^H9^ra-7a)YT7YY+8*
z8bIJm$leuXV#aTV7Qs7Rr)>B90McV1<4murYQTt$d{VwbhITYMe?B(se=X_8Hn?4%
zXK$y<rz5Sk709+<jBv<32u)rF;~scjMH)=G_<WTNLy}{;fk^A}_uIz%eKuF|S|f1N
zxt{walMv920)1qqI?USAr|bXj&deuAtfLT^^U-~`S(#mNkkjBjv2!2H3@vkFx=YFk
z@E#(zC9E_}%rZ|^_aLBruLGw@^bGLeA}joEG;DHt|Gm5}(&?~d^Wf#)#y6PygvY)t
z{NuiYOP>r|KjoWG%49DEa;F^AQ{*Jku%as&Hn`Yx_APw>G4CbF+<KgaC7Pb)skRvB
zF)ro@A0QDQm%875E-+aJu9r_WKbLh2R}beLW_6_o?H!e)MGID*KV>9Z0*fZR$2b1<
zaaZ$dLv;)IcaLv9YqOXQ7vVGyqJqZqhY<=zx^9z4;^e>-@Q*r!bwcEff_FLGm{l6n
zi<jA3ekf@RX#nlu6<xNbpbV>gIcufCaGOq^9do^@mrX(zG!nBeS1`_I&n4lxCep?!
zL(5;O&MNQyu%Ow0CoEQ`MGybViZrbP$@pnE-W1jer1}rR(<~-ByVkl>2f;aZ1<LCz
z)W@<tql+`LjnD{lBrGU*KkSTeIli2w*^zcMy#th5HP2S%2eD1`2E;$k#fknQj)Xs9
z`ka;LSE_E^n$P`Y4%TM0QB+F!b2J%p^`hs~8E_4W?S*w)>eDx&O-;ux)E|(Z2xi5k
zA$GIwf6O7_93c3(_W+rLTE-bA+M%y?Y&jFsJUQvKT9S%MAUSYko#FesQ{)3kC(0j@
z``edj4xq!fm`W9NVE$!T<{r)BoIKk1C?Ib6{kAEdGWP!tGZ<jsI-*>-{33pYiafm_
zAoua&AN$X-b;g2t#p%3Cw;eUsO4hRL;tKaxjrW^Yw}^9S1ArSUT5@tCDr@4iVsaog
zmE3&EWctItnMp1{IogJUvu4Gw46&<7OPa@_$Kk;>QwD?1&JwP}Ctk`T<+E2d36E4_
zXXEG(HTsDKn{tunxO0*#e4%)Saj3tzvO2H+URV`hXuftG9;X6~P;Y)%*1&Vq4}1_v
z1`EJD!hJnDM8L-^Is?5h7hK$k&Pre18V{ibr8Xi>YMg&tUZpOetMe+SO@RMWgyD2~
zF!|t5^li^^a-}ssL<7*~!Lxsn9hnT4E7>sQL*RYWWB@L8j1fbj;!tKUB6_%=+$b)0
z>qYSjG-Td_EmJMkHlCRb$3+9k5wSCczd~64cNnhlBmp?duufFN`jN89cR=^v-4)3E
z?+unRS9z!jJ&(1_&7M5(dIoeYsrU^FP=zKj;i9N)`~ZCHMHb3Z;h$e(${b#AY)-cM
zD*+FQ+xRD~i{VDFQbm#CsM=tKv}-QNUj}}Tx}TTKWfvFqdDK*U02>(fG{YVok=H*#
zAevUmx=&BBrxl)G_nau*-30pj-}uC5*?sb0J?05ORej4XkVJZ*49i-J_s=$?@Z)Vd
z4;2|!Wy+Vyj7LT%J)`BbD)HwvS%u)x0JoYbY)JA<ZIlz6xa;}WQxv4Dyz!tzj5dQ?
z9li>TG+hnG=TYmd0ru3%2;$T)PiOupGdS3Tei9aT??}F{Ye>*hE5CYmndlBFOz^<`
zW;~j|E`pMKf1^<Z{V+`uul=cR>lwxgv@Y|@(230bWH%7muZcufD^>!2u~F%vFS`<*
z%roqBJn!E`U<kUAVo{E<mVUVd=`t5!#uU?&pVgP0SX3&M29wiZRw+-V@42)C^&{Ee
zs57{HVxPZp?r5Oj1db=iYu7)tvdkJ@PmHe~yM$!y0i(>I+$+Y>VztG;R0g~VP9;f@
zkd*2Q_LEK|aU{ppk<T+YS%gM0YGt0pNaiqIVH|B2XhdsQFjs(R4EWBk|7w};Ohb0^
zWlTB(a+}JzMdF-UVF}HUWdnF#nh7wZo@Z#O6JUbJ(jXKM-Q}`7pP}T3hM4_7bAqk{
zMZz42SwFJL$Sd;?voxYK{<8rkoIU1`C!90*5MM1qmCmKRBlix1WME}xN7zqUw$vSC
z2cyDedZ<XZCyJ`hX%Dmkt^Uoh+0Ax_y=4Sll6&pmb)b+@@miUjJzQBuS`|AXx~!_o
zjz{9U#aC79&^FgYu4#k6Y^Dl}4RNWPPeZEsh0Z7Yz7g`vgT~mt|D0tK`fzYXcuIR5
zwv<TNttLr&7s2MzzZftYP~jTfX3DW*Z5Mr_%jyz7y7VwH3B2*I@k3f|?}|^f($-Zg
zkK^qS{BqfS2MqvzW!D_?ojLA{+xSf`u5mk-Pzv!sNb{os!1g-4fgW-YrMHCY{xp-K
zb1~!_;DO1zSxh0?J#xQt#LGmEQc;FPV3c}o*>MD#{@)oeO7jiRLZCiM8s=vP_4gFV
zXx{C=H)i0@w9nT}K}<`CS*#wc)0-c;&o>5fG~CNFwz>NNX&wQDx~)`|)aOW<5)<)N
z0zS}~3>9Cf<c8~Ry<m$Nv9e0EKB@^(OH%N^tyx2Qab1TfVj)4g0H@|DGav1V{h&!Z
zB$e;Yd19qW6pRLKVd~@{@A+TL?C|w0Z{9WgLR~b~nlVl6u46jF_SnDIOVOFkW7=r*
zdys(X4%Ca^)H&i0KN?7gM$?KN=*ivrdawT;bp6gWDL!kI$|2@0Ny;e!o>^6>lyb!y
z>|FcoHd*(_?72k#KAfF1_Te)vN+Gs8v)sr4bVgOz$#{!o$-xyRPa-DHoVHiDZ1{Kq
zdF8fdn#Epa1*#imTV;w{IpDlXb>N@+$%VYaYsAy4x>-}1L^dQ-^=32pP--mi=dno~
zo!(6F^~V$!$GIwttBZZnQ>hK_U+3rZA*X>?xh<Q+uZoI&&}H?aOR8N6cFpWPy}<MT
ztS|BK!MvL^5kpAhlqD&<R{)tfe@5|cUJ~TumcVA9XBi`c*u_uHIH;LhZ3b5&JFz&O
z>FuD)^ZjFkui&`hvpp)jEy6w5D8HBS<~NExj5AXRW~`>%`p)(zaFZTtNaH05hKZ0>
z!~Oa^$*<I;L)|)YQ)|&=Fj4kpUw7{m$K4eLl=aMt@LDPAlrgQEXBZX}3?b*UMKr;s
zf61oQ5vzm{?8Vk#GK3~?=XD{on75GlXL){^i47xsoH2|fS-GkaR~}PNhO74l<Oz*6
zYjk}4V3aYgqP7xER@ozyCK-`xleF<bniQg``KVBY-*I!r$pEChchHW@EWZBr=r{>-
z5rj=Rd}hwSbYcun#&|BBzNeE0T>v=ZA|F%10}fsQFdAGkaxvz@w<b1n>Hhf5JQ-;T
z1b<7Ij=wObjZ5gQ5l%zPxBMTTm0@t=0;AQ0fP=J|%kxjU*IlYA{$p?M7{6Fu?7rt_
zJ?nse`TMA|e@2zD;M+eAP)wi2a2{9vQUW`gB@gz!a-&VM800(2U=aBHg*)93vocUs
zZdh4Y9Q}A^b9t{5S_fEayMQ~RPjj>_P7Z7jL??HKzY`*Qm}ZVhC@|q+@sf=zC`zC1
zQ1?)*Ykh?58Sgdb(+?o(2@9AB*fsKaQ@7-laoQ1Q124iYXkd0^_>vy|F@-2MWu-%6
zn21;`|ARpjOw7CpKB_rps+jTWPo1s@JI7OWA4AqZkcr8PuCOb-Uya-KS@-e?A!V73
zq4d<~HB^pihvxYb^H<($OLbV{_n5Kg`)WrIBJ<p(a55h}DUOlL*pmuj27*&}7EOr9
z{2t3ZSxD1ldg4?@wB3o!cXR|?HeQ+mQ=mF0+XHj}Xlm$$Jq*(Eq-G2RYEdGB|1Qgr
z7$;M@J%<$396e)gdqYiBi?I(02$4aN=19U*Co)`@^2vU+5aORbku+P=Df>F{|AAz_
zcv7d)_5lPra+$$hWq9<NLP=G&T&JuVf7V%qpUK8R(q_^O&2f)A+{YW2vh(d*lp(WO
zzzLo?uxM<I8qdb%WIAu)*%OmADoZ8QN`_vr#t21T&Ojdt4gWY=d?~b+N}u|8ro2?^
zCpgFcSae>~q{UK?Y*U<DAu$EP66%IL!*bmJYtaGk%zOLU`-=((;KPJB!qC(aOxjnV
zFK1MFEzK=b_zL){?E#gk)hAi3+MoUzn{1grZU>D;w&PgZlH|FjJVMXB1gWF7-t$bz
zWcC!SZIk*<m{~QQVL3`G6-YjAIn(He#?F_751xsYjxp#Z@VkwuE*31&=RJrFwiA5J
zoU2CV$|&EtH`&ESsgWy&G9W&%dpSOukAr4U>mVR<REHb6Vsv>V+Jwjsx{QrZM2}mZ
z%!B{q*9SA!aESm7-~^n&g9T&?rM}TL)UmJjX+b!^*!ED9TVc}qkEI!$XI5?V0nZaX
z<KW%^@eeTwgI>s20xzzi=~!O~dtp=+!{U+s6)MP!VwJj?zx%oT)qUwu%%p4-kegS$
zZ;DB!#p(COpE+i!K}L*zh|$Prcc_<#Ce8upg`cpNyD`%Ge<%!>!mRWO>O4r%pDdDA
z0RBvdAA6lJ4tf8+WvECyU~#}VHLR_jf*&wC@<S!pZh$lwur0^Oz^P+xr|Zy2y13WW
zrH5qvbDat7+<$yV;iI9sT!kAbSzG5M5YTKnE{dzO`A@|{#97IQS)SRVxH_qP%9PeV
z^BEapnNb3J%{Bta$Wvaxx*tioR!*RfDK}(qPyCF6#$yVJ>62mzvv9|~gWa1Idui;f
zv&|k121&5iw9?|Z=Ka-;u+rr00O~MNNj*b|&#dhdZ$nXA#$P)nY9};WS0Ynf*zDAD
zm=XGGi6J+;V%7kHQ)A-|J&%dh@Xc(-!H3%aDQ<JZ2Sm$Ovt@4jBGEJXYOF1-5OQ5$
zZRykJ#G1t7K86iE@xBbHZ0Xk^w9|;9eQ1Vx)uJKMlq$<IW%$R{Rtz+_9htR&k^jxG
zDO{C!maDUQFPbeYN6k;D8LQ+KA(7|%s^A)+rf<Co93TQTp1WVi&^bxG))XYX=7+W_
z@1OZi4f~@Qb?i{OC-RfE#vc7DS<QmV-**fO*m4~eTEUhA!EWN}wgEbFfcT_?d)<fk
zEa=MI1!ZktqevW-hT%(IVXB3Z>;LxC6xJi~K5o#fHQQ$OE9V63-g&35-`#w<IU<g{
zeicd5tF(Ci+;dhme6?tsqAy?;>*BhIf6}u5+G+vgKxvj=g}>mt+Q`Z#aRmIsM5YZE
zF>z5HDLOX1MRwkjOT*wxH>6hDL_)X!60N9GZ`=3Jz^BN>#4pj3mJstQ4g912@hNaT
z3Zf}vbbgm{?yqU!O=siAA;CEW@8p@tuo;2}WS%}D&=|w9RI0bN(UXH}V>=bAJEKNk
z-1cAf&Tsd?*i2Lj(tjxYhKRHpKxD1L#Kjn@J=a+j%~JlRsx9p>_z@F0k@uOO5bA(L
zdd&KF^NuUixK>%IssBTd;eL~AtVxU5Wt)tVyrsf5B;fI>;_bsTgCtm7l}}{KPeHp#
z%|&3@c~wCs9S9Lh!SmQ;fLpn+@Zz%%lRsmnOO<5aS5EHy$I3Sl_OWbY*l1A>JMS5h
zMs<9iK}pHf=)+mogyGNE!oq>ZLH6%-UA;Wi!z1-ZL7DInrzv<Q+iPpfJF{p)$0#vb
zXStLUEi)Bww?;66CX|CcEDayD=ZyM&*!<PV8T%e7W|b8^gvFhSPArd<Z`zGA>pKFM
zGH5ZXZ1=vkv>ky5Wpw_ToyN?vLc}P#=*FpGA}gozZEWJ-&=!lTEmn@10`adVbn=Le
zY{x`GwIqwpA8>!)DU3QWuSI2RJQEfypX;zK8|81JLV*9?`j-_cHrB<Qhaj>yn<qGK
zhF2aYJqDOFfhLx~p*Yl0+7P_%?%4b6F1bID(?Q-Ttz}F5=Q)GxC0{lth#qxHg;<z9
zLXi#+C>urU4N<Yc9^fHyaqClMSXISiE>@6k-7Ircm;=UV=zvMZ$Ax3E4e!cxMzEd*
zua^6qHBNL^`C;<2)!b{NXv#|}3FA(2z<=|mS@Ov7fyk7@%4#t5n9+U9lR@|$7CG+#
zK+gH}UmuwcDO?ewSb0}rt~vSgG!>st;TiDXL=JpuKQ3sImiI<lS+E*|)$vqj0s=GF
zhePE=8F|h!(<%ct&-JZK4H+kNdwwq?&Bwf%LcX20TU(5ws=~iMOi>RI9IV78C{Q?O
z%N5cZ8Hse_FqXuS)!B`PY_Pmq^-(J}Itwm$#VOPy`Zh0=l$i76`yjqQWe>221=Qat
z6@zU{BU)CWt4Oeq|K^dI<&?e`)D5{J4pWf?@(hh_48X;yuU52C(H>*zezkb7`y99^
z_sMYc2(EU!koxoG$-mt8XL7)U(6W1Rc`;O$y#~Q5O@KOM)7fr2#J9V$BK`u}GIlUN
zl(f4^F1>(`Zb_NfsxCOS;2S8D;jbTSjy7z?w52i7DSz^fb=nBgy5E}LYBc0CVB?>F
zs<i4$6-_g^a|}Kg7z}xVV6=}Bs!%=izetc|3k>DWx_Vq@@jf_Ul>^l^9_oJ%w?#S3
zbJhWe0}+b~gll1cJOWerXXFE~{XCQ;HlBBNmBo7wq?p?e5y-nmwX~KEL0w{|Dn4h8
zDQ;)biQh+)UnDsagX=pa(|kyPxuSq3QT%e20#z@?lsw@JDl0&q`NNaU6G!Fa6?b0@
z+Pxp!hD^_x*!?@dS3=*%DORMNH!`ML%dc^50jR&|BfhPG8>@b(|B5t40p>^6#)XT|
z7)Rhgui-7^H`qfe#s@57mIrESv6g(s8L;O&{5F|hkEZeEKc4=132atr<SQyL5Wepj
zAMxP#`Id)S?f$SOS3ed_tRR?^sSDu#;y4>wr$&ifnFvW{up!4|!b|w{hqnSk@wjyK
z2GNLp*RI*=f3E!V_9t|>18UT=&{?rclpn>+W#RYy8!ffB$rL{cmAID71hN_@Ib}4O
z7R;$-LtKAbs;>OB8a>Zgdoh}2BMdOmge;h&{)1#Fx>W;4nHh(Y^LRK{R=W2`OL;<7
zW;q_pfB#t6NqCp*5;ek>LRuy>k4(&Hivw~Ru)KWh6vmN*`YLXPUN0$RtcE*YZqwmE
z-b}1}E2mgWF_-c*Ye&`RYpPqma9{kNxp1yIYz9@9mVf5k(ukWV{^)nfE{>mhQxANm
zl}a_$YF4|u#-vH;dO{r-D#!ZS$>uKQKSa6X>cvn}3(|4z(4^o_u}-7%|9O~p!>pEO
zj}pC{XrfGow%f%ie>Bp7{nEd>1ntkRi!07}0x88jURui$@FR~Wdh8Oo9omc#r)Tlq
zJ^$I`CrISAeSPptv3en<mTJ(oEn8MYZ`w(LQ_b**0G-EY`t&c<_|AAm^&$`qxG>w%
zIjV()nPLbA&*u6=*5zIIo8O#)Ia$Fonxf@64dLZ25(*t5@fH`S38*GxGi<=JS`2nq
zk9wX%Vr8z0DQPDM(*vR5MOfb+C9$M+j*ev8sUfG3m7f)ndVreK>@$^Mc&i|PGEIU&
zy&O^2_mlqtHB@{ry4A!mjTzGmXO-71)P>exnEoqGZ8UO`0_|BZ0@>y=8H_EH<4HDr
z(?8-L7-&vOFiJZ#;LPRii@FF~q!}2F#&X5KKz{|;8GtNS@hcQJ!;qHaeBP0b%6oPH
z!I077!^HDmj)B!|rvfFNjhi-CiHHUl!Zf7rnTA(+cuGmWzTL9llNm2l8l+{1tSJE$
zXdad#mzW;p5sc2$67KyhW|@$PGqEmPO8p0IX{~L`bUY8cJfQ_~>)3ggQT$fdN&wQG
z!K?Bcz462nPbc5?fZ>kml|d6Baq*Fe8>l+=lCbqK)vB%4fVCRpfq;q~M{P;)12;d)
zj?|`7z%ttLu7^ut+gNd=JD~aFVff9iAP9E!H_!l*dn~l_O*fy`C=p7Z7(<nPQa5sA
zSB`eM_mfMs;13t&mLxRtS8X1)i@gENB|J!q7F|1+GH)sKWwedI2KkLbbbs)!yefhE
z2Zn^?iHccHGJ+#U4Ic$<vOez{s1Ly08S$8wb>^1Mv};@sg_W2K>k!WVq1m~q&P72!
zw?}2gmHl*Hf8|J!rdkUfxMc3<rF~`+ezY>4=z87{NEq~by3#DW%Ox$)1o=SNOgA$p
z(m;B3{-}1bF{ZTez=?;DfSC;Vy(BvAY{*E3;i!&C`1i-tT@c+D{ER^xC7zXpPSe$*
zE4YHWO*)sI05|8&DfE!EjX(DPyp@Qixj9Y^LgkmKNyW>1u#{C26X;1|`{QC`qC92d
z=zK{#&H)Dr`#9L~bLq98CBr|d_-K7T7*IT{T%9d)93g80U5g{lW@EFX=l5u{khg%Z
z3+#<O+?GFvx43`%TgCT6Va_wrn9<oEH3a7In9vaFT{a|%!EYD@JoU@7<w;g6OV!JM
z5Co`TY{oAB=l+V@{%cc%cJ7m8<R&|<*H%o%@5AJNJUp0kJpkE#;{G@)r?X5w0D$L7
z_`I6tG=Fc#!S;2H9Gf$+xifG#ZwP3K#-HS*6_d#=PXF2q8<*?<NZgOq-8XSY@(g)K
zZ@T6^ElOen(A*D~(Hrff<qqs;tRf2-ej0>JCHN+5q3UZp=gozbde#MB5wJm!YJzjm
zb=miuFV#-w8#H0k%a|V(BvOpZebn*BRx}^O4>L`{4nf6XYgYsvSz(G7xjaZ8XRH#W
ze%yUXYUlL^KYcfE0h(t;&2o}yQ}?FyOD52ef7c0Gu6?Q-6ME*j_!w*zxHGyezPtT!
z=YMni3b-V|lHr;`btDr5d{U=o*d1GWI3%q4KL6fejj|4CS@VAM;n+!?FiOH&)WoSn
z=VkL6=Oy?R$H=<|*Y1rra#K{R*P^0B-$Py|(el*seaTu0wcy>!G{o0_&+I+sfpGhD
zQa9h#Ho-NlcyJ}~jc-km_U#9bBVzWN$zKxT({mmO$1}W&w(ZLR=gDA7oXb#stu8!#
zORKf0TkzAgxAj+7#Kk?-X$;CI3hY(CCBSlK`ug=Q;OOOu%Or0b$LD$D%feHvYuNGz
zsrOn#z9LH2)1{X&y?pCKv;v|xMaVy9@M(-)cNqGu>G+j-*Jc|hALHcf!ilu8Atf^H
zLO#Oxb5GTfK17)Q)gT$6m!CP6td`e1*lm>eaD1dQgC11V5|g@e3w87UcYdc>TbWg!
z0%~Mk?qx3i;r#vbZ8IZ&uMqt>RXX-3byKEnyOQ)VYm0<nWZ9|2?pmU?WTbL7@o&Jt
z6|nuzwPFe{P(fcAo${2sPZS;YgYYp^UC2iP;+Hn&K=gNHI}C;Ib>6r6e)EVp865n&
z9Uc*^KFmsFom#~gW`x0*i(&v<Vdv5?-~PpA+k7jVZ#h=`7id>q-w$y=b2Oa<IC8)B
zaJ?hSPfH6Fh}C(aL|rdIQ6Eod&5-yBC~~{!4m<k3<toN!0eH1}9_Jb3!+F5RlB3M>
zd*yv9{*RxDZ;+DW${s0uW#PD-&mJ7Tk$fYuKP@u=V<P^9SR?-Bz?w00)P^@OYQ;EZ
z@63mPkHi}h`S$LIxdl#gZJ?I8_Y*-6ZF`Bj^o?8pJ(K2ut%OS6#q)+aVGr{vj|2(K
zgbT6A@TU%2vzJm6lJ8&NL}Ao49U}KzpW}Vd4jZL<xf)xCoui@>qZfbo{QU`7mI(M^
z+d6c@!|h?5J4kcog(R#tK}~~m={TFmC;%leggZf@`WAH~D%OGJ6ISPH8F+{QdT=7o
zRWvYF%~|oTi(u|yl%|93Vrwl&nPOmtMl|medFfq)=DX5!JB`=rPuUfFoX;H1hWY&C
z1ZLnJB<eCx6{@4k8&bYn0%7hR?=Xz=USV{gxkwOy&oTeiRFcGx9y48$q_rURvPJj3
zk^X1>NDW#5WyUNQPwgVfJ{LBw$ncqj?|})B>*t=%YpRnXj=LB!%3cdbp-ugLVcfMJ
z&%0T3_sU)UZA^in6Zsv63ly#Qaq~YC6#8peT;jj^YBXtwjMG*N62+K@a1t)<+5&^z
zZT4gqjeF2VrIkfA`zew0MRD@aPat1m`?;5(l}jomm(%fbBQv4rosc|JB*VI7U%+5!
zeeFD4ecUP57-vPpqM#vA+`}AXwh2--AdxlvUOZ@z$o-_A7Gv~pwa88$+cJ=0D&oF*
z#!^2BQ0c(|uB%c*!qQheS5K5I#exl|uKY@g(lV6&lA3rtYr;cP?5Fm@v!C4_5nJh#
zy1$64=ShJ^z*z^Us*CadFmc(SZB^D_6~5QUwLg_({v@&~5JO(KPaq_IBXX3b|CC<(
zsaS!+_5E4+)Ar8r6)?uN>{ii)1Mtny7%o@e1`7>Y0m-0TShYU<ES#rpFPf4ggF+hr
zm9lJ}fUcx(U~S7O);i$0>p4a)Mnq7`dhPS%rW0cyJG|TCLm3$e!2CQPWU&jM`1B)p
zU`3O*n*CS)#15a$<&OXmRER;zUi;l#A%XZj*N>)t(Kq>tJ)%3AI3zAT9=QD1?z2ef
z95Ro~^LSyK0^c1e+#e-t?-oy?Mn^*3^k7!OpTgc^R;ByLA(8VT)Gt&;ps1hP2FFFH
zq;AMs6XD4glF!cnO|V*h9B!oWTQA`re<O4}{S$ZbCx|Fh{aGJ&je$Hr-|Q%p{-T06
zj4WBp@NTvWdB%>R*uC__73!67JC8{l*V4-{$BC!opE>NII*1KnK9ozvl3eLAfuE3t
zEq_&Ux`%oNSH`gN!X~))ok!e7C1^h>oHg!ND{&4M&eI_u(oT8&i2?g^ugQ9;VJ<6Z
z!PS-iTHv$6lB;9G6V~jZW?TJ%S5HB&db-{rX+Oc(8AAWQpLO+Sz6AAo`m+)snn~=3
zczW#cA8cwK7ZS$}69PdPW-Z?DJ*0tf&#Eay^(THOM&*>6`Dw56Z!NOs&hedNzat(a
zl>6i~_qern4|x>-t2JMs%=?On@K~jCqhj;OIHssGnegdZH`w(FZ+QK|l2o0?5syuu
zRqYf9r?5zQ<+YyI5cRuU(dpyAaIZuO&k~^KC=H^CQ(SA3z8P_jHHVVPwN$_<JqD@h
zAH=Z;<^-Zgd@f@aN|6s2mMPddgIkVO#a~^@t{9j{ZeBL*^kozb$SCG5*1%ZFi5Z5(
zVt(neTRmaXvZB{P*Y6ZBasO`%J1BpH`&{5P@K?&lh$<1Kmi6aVnx>^%&(Xj2a)von
z)~qB^qSbh|I2R4)P3lZ+_A~_&7;N(v?-(US03xRW0bbd&kMn%nb*sMz>I7SO4<i3>
zOXe)+bjW>~Fj%{}A-;tXHpkj=+gPr*1_otF$>5vVG|qX^PsHJhKB1HOkY6~$s_Jup
zZTe|@E)_*NCpZ$76$wInBfpD3)M(u8)WSwUt4r+~OB*Ri9vu0!t14ZilSnxl(lTHi
z>7OQsCOe^*KxMYH+J1~k{kUZ7%Zs9jB}lJ40{6-dMNAy>$A&u`AFIgYZJ!_t1R!N;
zA3z}&4Ypc1Hn#|M&W0~>d;5XX629~r9{)<RGixUea|nrGy>HD_NIl3qJLuw9)Ddns
z<7D2CeI3POM~Z7lm-1&hN|EDbRu+{6h2|&@(17eX*iou0MGKVeUXM_6%A)o`my&bw
zA6TZRi@cyM-Dp|f$0wb*;bdbD9@+X!U|J^ea|WYc8O|XHL^Vxuo%5GcnY_MR)#mmV
z`@ii4e-gbkm7RjzJU>yIk5eppR7?01VZY~J981vJre5z+D8P3w>f_rtE2dDyLNoS+
zB=eBV)%bu>(r0p5I+j`1NwpT8aS`^Q|2T_dl~SAvt$B4?@h`nMuSU9#aG2u+cyU)>
zm_vxt)hXYGk@7M}hz2G-aC~<Nr$8wn4U%J;;AG~X>VF~pG>Hojl<hq3?%~mQD@iO6
z(|iFXGMwe+SQ}pp!&T5FjdSCf{3$+fbJu)%2tNyg?D-L%F4iZu9>>7u!&tx>ANa@T
zASCAFE&h2S9m=;6vgnbwPGta=c-_~=C_~Y2F7}#yjvMB-2NqnXY;OfVI~=<@&Y2-*
zUK^zZSW*!;HW`&2SAp!UKW~H?xenbhR{bc~2=x+jwte!}h3N%$Oz*QeJpO|n8-fkr
zp7SAMSpb!?Q&WfSECo*@CE>8liO#`@NeAwTZPpb+easeep<?sC-@<|m>{h03VB=F$
zoB|UOP&XD$lFaOjJb~l6mxJq_yi8T<E}QWP;W;sUnIGld6bh8q=L!##?&&Mj&ppia
z(UuQUsNU;oZS+~5wm&{CX?c2?K}F<heVQhr$Jz5Rh)M(HvTL&@Xo)8e6j_?yrfvpx
zLuEiB$9%DLqzJ>YQvcJJ`^y3Y%nfFh+HN;mJX;Mjd?9jV#^o3D9RB8KzoC@9>tNbs
z^zakr%du*#QM50=j-B*m-v8RIE+osi=Si=%QH)n3@lr#LHTt&|3BoHV(cdOR_&(@|
z?UYAWn$(lzrso<*o9uzYfKF!3o?vqnEa>G@yXJ1GpVXmZ(;pNl&35r@05xj$FS=0d
zupgqOeeTTVuQ?rVT9bik4ocK<|BEhXNC=`1Yc#m@8g1~V$UP3_!7(k+lFQYK|4AtP
zW4jAg`(Pt>|GVNDy7-vIzdZb7c@d2VlK=cH+pJFC+q$ONc3PQ22;E!zVOwQJ6SQ$k
zvVN9Me3Y1j%ETmUVfCN9g5_u-#(K-<1y<PHX^2#Rg&5(*Lrko&WNL&B4?Xg$?Lylp
zmcJkrX<PpK%Uvo@Nt`-HN`uil7fnY^)f5MGddj|<i(W#h64!l9D;Uw&FJ`;=GQxBs
zZIlQB(HIWC{}7Dd)o9hd#s4zKAjL$HBFep`<m1fmf8aug8282wXhOs@mRR=svmt`>
z^!K+t&?2I_1B%zlPg4~d?|kzjUj6msws-$Z@m5P{vOOGNs#-ihk&~b1P|N1@XJ~P&
z|G99Q?Xe*vSJ^adg6->(m&kf`vggd_8OsZWf@k7mc{Q4eBs@EIusmVg^%Wgcp5vkZ
z*CW~ahgb*F_z2CT$ev4;0Gi%S`ro}cg&f8C{zKhwLuhbfbkZb{Fmz8yc^8?I&96?Q
zp)l9)9!c^2y0+DtRe~uLuCdc9A;OM@U;hG&DUge6Vo?0s<EZeucp=9ljFf^P2&Dur
z+yLmc<$#HWI1KFZ8Dtl+OTxEblmkB@Z7yWdY5V-FF!}B6irIHSX07GJsabtdHI0g=
zO@twzFeF>MgaRUr8m+}YxrH->Ad9~Ab1s~MwLqIk*pUNYu772`eD(iGyAx=ty1oJ6
zt_&9`V?>6?lzAqUhz5ns5t1o$GLsCYOhrU8B}3*YnL<$%DKexGC4`7FPw^eM=Un%5
zd);$S-@0e5_gm|E_V(=aJOBOP`|Pu~`=wrBN!X3&_N9W-@w3A{yOcu%>;@v$F4OgR
zy`7m$zuX+`KK048(D%59h;OSK>GDd6+>wdI+CZ)|cKP3yhrWI6GndUE5X?TpTaX_7
z_PWOzlci8=dG=c$)pV2#GWkQj`Dl(g>ZJX7?yV|*?{^^*WQv|28{FruE=ZB1npc-w
zdAMTgZ^`}(?>H@bCAN}crB`RkzrVjm#7}ncB{($ZdDxysb~H?q#IKB3AzmZ>>&W>%
z6TeW;Jup0KW=s1-TN>$|c9w6iCa~Q~`kYtdSepLJzhz0<qcit|-=(Dw6emqvDb)7b
zhVPEGw$s1yU%IbQ601x$uxj0-C{lN&E9C5td(NJy=RWbr(;210T`t$eG<Pb>)yAiE
z#W5)FH)M7F+Ze23D$ttrZs4wKqs@M1?}=WTKMk1*9EkZZb(wk<o__tDdSB-qZ8Iy%
ztFbS#x=coZ%=NYv(0N5XjeT&0q;4p8?tuy;)84}Z+ZpPT|2F>K7yF$iB;RsObUrSK
z{e@rqC!_HJ<t20gJ2iKc?krL?vwVwKG}*n|US}Gtwf^KsK(b$@6ry~LjPtiwAysT2
z?M}grlVu7^I)Cr)wFwCw?mTf7ZQ;#-C_=LOnK%3gn1e|tMg}gVscfx{56Vq%uaXk|
zWu~g|Y%S(T0QoVET6(<F)Lspe9q;x@q{if(=4-b;{5|O}|1IkI`e#<$4Gw1#3dR>R
zR(LH85#QbN?7yc)Z<Mb?v8v(IUXJ&#gB>n?NXT@Ywg2<Dh;=)R?)144S{?#)g)R@`
zKXjakOyJ40Hoo|m%tdaFJew$NrA>9(Ho}vYQYky`HVbuskv98(nQI9Ti-ghfPAU}x
zfjb}AH1~DHk;=<W{BQ!koQ+sZk-xB0bF?5WYQ-XCDYKokV!~>QNX!40Qp$#8km&7T
zIPPS_ZgkyoM|SzV#M^`5_~?<z`*5y^a`))N%6Sq@MmD!pgBYjQr(d*$I6p`Q9%G}X
zQi_c$o>L5J6(42!ON%u%1sS|gmnxWiU8%rALSjR6mRwfB_(l2i&b9ffuap!WWZ|l0
zWYJ3ZhW*UgPD@4i^RG?uT$+3Ky;w`jzUJC?o9wjqu8TUk+tXBO2JTE<^10Nu<S?f)
zG1oKU^(^Az+ohi6ScVuQ5<%*ds^2;255KunadIfWd#ZMAz3J^|{XzDvYSesbhQDg7
zy*&AP%F}KM40@S5^_S5Aip!}T7zYNt_&n>jz7<(uPu7>r_&AkT;AwrR!fRak<No_{
zar;M}Y`Nsu&64hMkzuRP!PXsDit2V)8xL)-F&5ia4fK+z#HFg);!As&Cw)GPmb5zE
z6DFpP^28NXJO(lsW^+Mi&+XD?Cv?;-G9NuS#g!rd_3tw;l%qid9WUti+`B$-Ds{_I
zJ^KgMJ7~bpMQ@~dpFpAYR~;e>>e!TRc5!mW)^<TFB#**o&X;t143X*ap3vDN#U{3e
z$b65U6W5LZ(M#G^={+enTRA=!bC~{orw&^!75hC7a4a!P?-Z;xEB542C;@r?_Krx2
zSp$+u)P4VIKj$%Ylr?*I_TtB@B1(MqX+?z(yMLZ!`0K3OskW^LtJ>`{&i<&1v2R$G
zE$4H2%YvTWNj{?yQa^g`=spqg_pHYzPUc&Ax?gVltmwWrDw<s|7sxqASD`!eCY(&w
zy%zDsMMmD|h1cJn6`qIwnP={IC|47fYijZHiJDPSY`3eTZ(5r|yq%}7_2FjpFUG2H
z!U3-`v&AAaCNh(w4Zw~TK$4$lL&qj_v-7bETfO8pgp8|6iAd4Z|GF!b8Z}YJFIZYx
zi`yl~?GAdrKQ##bp!5!3e{7poZh0kME5!GB3tfBDLd>noh?#RG-E#C-uaX?Cp^<EU
zE|b@;b4pV!ZDp+H>t9kA>Ab_=uN>E@e8YJ+N%U>;6@{&(VRhD{!{F?W0?p4uW=!8M
ztB}W^J743qVq`~1{bwMt7K^Xz_!no@k$lp6psCfEO}f*p-MZtiS%t!go&@E!cJgY0
zV`s{>aw4x3&w}%<Pe<#<kCxr?Hx8@G;@u+q<lL2DW$*t+L_R}GAJ{|oc(Kl2D_51w
z-lM(k?S)M10^z@96xNj1+v8-+Y$8P%OZyHK+2+J^#nys}i|>c96BqJ#Td3VUI;8jg
zt8#-ju<he8n7354hrP+&9TV`_A_w7n(M~ap+OWu4j^S^$yzoBNIpxI92z+&wagU2E
zgn|8pnyBx8ncgp{sy)22K}|Ogx1KIT)P3v<mUTvTDZLXbOK%O7qcr82#g`IQ_BLd>
zPd3%1PTDK3&R(oo=pOI+Hc{au<;Uh#u)y!pe0#S1-0A$Hmj?0NwEQf5B*~<X`!%*!
z_>6ljT(?TTW>%-D5?2+jO040<@-j>6SnP6TL8<euYd~IX{Lt%C_9|!X=^*wU$q8$(
z1W3Ky!$<x)9*Q=QvEAt}lR8USt24<S#`SY}hv`x31m=nA%7IF{_^k*!|1ZTw*G*`5
zO!=LYuta6|z?2IxhPa~KO#acC$gO`XL$YU})F@}h;luTR>w~Y(L>2d{W%-U%rID9A
zZ}0ecFM9iD{XY}!xARETy`&t=cNQYkZ^g*Fc{;ysej2|*X~6(Y=z8Sy#MRs#R4puL
z&1%+Xr0IXRb?Ud3-?Zw~-$q2X3XBcWggNbcGVwe$irII_>Jp8>DLn@dlQdqw>*^1s
z_bNZybE0$g1QX+4R2MAV<X+LyA4_xgWz&9(lv!#?ZuO3-Ba>r$7ymk;n{-%9IbUkR
zks@DC?=}gG%0b!l;2g!n4m)nDFwF!t%Y^Ozx5$gVj0^Uk=AV|1G+s+td^Nm(?iFS%
z7Ru1oE|}#wQgF{_fpl^B1#nmX_5y@Lo}bvCpA*S^KaRuIA?jX240=SwK(R!3_7-7G
zzE^1Q`$w5nH6~v=b)EdSbk=!$m2~Hxa-C=F2q)Y^XrLQzkge*#c=_Z1iYqf-&i1p3
z)h}`>?}nELWCb*6Hf}HaN&aW{pmgLQ<qo?(D@T?$w%jagIce8x=uyx7;i8_WW&zr>
zhTC1#4M%?>i1M2$1kO(0Ssn}g>m+y~>p-Zcs$!#wORz?L`{Nt$_y=RJfl2<nae>+J
z*9=#hmaVcunrW+q0=r8WkFCwLi*^5yPOK93ks4Pa<%oLRx_igh_MR)VV;R5yI@{Wu
zr@_kjbzioH<WM+A6zBV`I$((Gj^3Z;8j#2L<U+OWQ5~ksCo^j$de^2vU#(4mOW9NH
zJNBBmXzS6M_`Uv8>C<^(5q}w}SW8mm5)U4Fb48@!rlcSj@z~|c`6FOl$+r?s9zsO&
zS;ahHDwcYwbH=mgcH@89N@;VvV{-Tf%KXEmBgRqjdd2Dwxd*Kioc|V9hL%MK+N;J2
z`^cIf+XP<WB&1_&+ByTKk%k3_zH_-wSWDcM&rrM^zmMaH{Y}(=$6ST!$f;}OyRPP(
zj7lZ8{b{&e^|;b|R+i+aUpt#TpZ&<G{On;@`T6kPO196wxeZj&0kMZTzXz9R9{i}u
z)EPV}FFH0}{&}!#TQvWkZB5A`lCg<~R*5{ydL_q$o0q(qU%eOUHg3!qj@Gr+{I$T>
zLCGvJB6{<HQTwlRb>IFPR9@dW&T`@rgPW?ZmDx4Pfv$^JFOUI^g&GMNJsg=yh<$U0
z{^D1QojYC9THe;EW`pmEKDej5>3FcjY<;Mmwd44qs`zcijcF?Ne;c8gDX14e%m}RV
z**ktxt)7hk%uZwR=iopn#^grTEeUzbOg;I-ilX$ixh?_3bKp40i8^Q3363-R!Nu+M
zJB|1hzoflZn9us_Z66w!pR6&oH!<(py-hKgMzF+SudEoDa=V)gXtq&{sC+r7WS|yN
zcUUiLYE}sx29?anI18G^@#F@|zDmCw!xVJ(#N_{&T<wf@WE_em8r0;})1YnV<Yr`7
zGMNK2A`69r2}Aa<-nOr+$EqazM7*!x=Tkw=6|Utfr`pEvoF1YKIu>>3W7*I^zubJ9
zs$;WB*WZ$&j-vu}<~y=7+_w;lIbK)GjZ87h0K4B=KHkIkZxShQVU|)=x_clo_dN};
zu62PKm|g{56BV1x_2sI!8Y#{0HqLSu5B6epkM1ur{M#h1bRN&vgQtDDs2;cT(zplg
zk4PD>((?JQXZ#^m6?!L3xo_TO?)nEG0a414FU>55U{EQT*{8N6`V-f@a?p00*Rm-G
zI*Iva$7cTa`eq`pZ<l2p$Kv@Dt)Y9EcY4>m6iXLdd(^Z3_hL<wLaFxnThiFGsjLq1
z_lfekY35P$c$Os<9hZf^MbN%AJNM<TUHikUUmwSQ+TYxJp7n2|hob`2batbn-}YTS
zLUS(T4gZNW%F$oo<ESmYbXP8j+)XDbM20FmlU%syRZ1$#aet@d=R$(2x7t#<=Y(vw
z7R>KGtf8s!yz=$ezh|pgXZA|&h@bPYYoNYkac;-z9HD{F6Bn}})Jrd}(O=a>_jq<4
zy{W0ps9UrkBce9bw^A<-^eU)X+)$km#PLb_t^R?EW+9q~e~vAmKk~P`-{jz(FG_L6
z(03b6aL~KJ^FqB-Q)6$z`|qBupQoc0URT9KpuE>8|190(s4J+)<@dN;!B*SqD}ovC
z?!9Gx6go6?pF3xyPcQK^@a?w4P{n5thq<NT(M!@NhW3UWj~OCP2ztbwL_r`jo07c9
zdZ}^Ax5nk!Xr{fDW|j?I9<juBjT!;3Tdm4dJYlny1*-=xd=i~rZCri1`fFl!$?vym
zcJTb_Zy`f6m%YhBj7ARE368O>3|U=1?Yyeu=gs}QbhXfLDZ+1cD5BqVb<l5xO%rv*
z3;~h=b;JyCsto%7h=KoQVdiQk{J;MZO7I`Cu(UFBvv=L(A7$G3k2<)Tib;x?o;SPT
z>axkd$Gz#lXJ&72DlWeE6Zx+U@UR@e=)o_nz`x4@T>OyB@Dw_0<EVWsOWriS+G{ST
z`Wd@Wvf5ny6-Guz8UM0_HxYycYpp>JoXPX2!NyV<ACCgXuU+Dht1&-kX>RA@=HMda
z>gkI9f7q>@E;yLEvb$SeaItZ6+{Z3Ko~+x<wxfPh<lH39#L>L!eSyz_IvxYp&kSNR
zb0;@P*R>K5_KCVxBk|D}v$b^$rs;@R2`J8AuyHWE;3Z^Z;YoEfqK+|}?#J1%u)>Z&
z*<dRI8W+n8HfHuV-e#^qJwmoFPL8_U5b-8EwnSYt``8k|_U*eZ0WqpAINGZMmd2<?
zxjq#tzdAK=?K}d>P3eE(_3W|Bds0H>23{OlA@vk8cer3DCOlzp8ARm&RQt#;Q$9<A
z8Eu5f<#gg_LUHd9JoC!?``c2M{zw9Ek@E|YGe9#EPyv@HaG|h)lKvkL%-f9dU#4y0
zPoTm}iMl0p1b;Kki?di!NpaJ)Yonj-UcP>yA3;1yLA9Ei7hfCGM|Z{a1Nl5J&Xu<Y
zaPKA|A=$?67}r36*dQpnA%zzdB#B2wXg0VEMVc-SGO4Mo%6Urey1aG78I~b)j=M+3
zO&a|=tzz0@GDZ~eGNjJROmk}6)oEJ<he6k+14#&EKl8A1ukMEv4#J-UT(W-(!Zsu)
z`@RA+g!}nhzu6dy5`cRmeotZ(MSF1G7BN#XEJ6L%g%O}2K8Pit^n_6q@yU27PqW<~
z^RW}Yb7$p*($=k8cPW0VuEo+*H@&|E)PwsO`txT(mUAp(>X-WN^G#M2-*mlBTF}6g
zvm=2wF7g#}G289o88@d<yyR%~2Mur(2eNX@UWmzTo#{p(bqfx)ZT<S?Iq!mPOl;_E
z7;HIeHv9QNIl>Tc4~81^v;G9!o5mj8J(sAMDe&7Ed3M(LRDy2!asFXo87Mh+z9PmE
z&aEd~QsUzYIy)Q5!JaGN8dlG>U%-SClmp?(`R`AKY)4jdT6aEFGd}X(do^7uy(_Tn
zB;K%8p%Pafdab7}aU-?wF<y4QIGFMG>S`!oh#Ro>@=3mrK$_MrvphHKy_z-Pd^yvD
zf)Tcz*6UKcKs&gfI`9@@=;=4}Ct%%F4}_je7U;QvYoi|`@}1||05@xy4}_izUCJLp
z^KKJ^$4fb8wFRRVm><cSNRx?*h@2;IesPcl_%Htd$xqBUO*cpgLf3_^2{Izu8R?jn
zIl66i!_O)5FF8Mig{Vc+SK}tJMsUYRmuGA*&A;M>sFO@WmuA`PT@YyoYu516djvAt
zIcBs{!I4lpfMY**XAT2wMCMmdUj-3CcmUC@hrvj$o&Z6*Y1f522uVwr0>6!j-0m0i
zLJvY<87M+aK0lwfL=)+YpQIML+~d*qBH_RExh4!N2$h~#hy(@UZUe9y$bOzod->$D
z={HRz#YflPOf!Pzhx-e~*(0t^7W&SF?`(&yYVh(y%N_dYSn3%YXTRSO>NB5Q5y;@y
zZF6y*x`_!#x63M}pB;ql=YA*u4bTtnAOyA@hMp;)3lsjwAS5OtYr1xA^z&k$MjG@W
zv|b8Ikle!$S_*0}2fMV?>K=8m4DAF3^>J(=+8BhYKQYIGgm4F;^_n0f%Gc}csz833
zwwNsTRhB}0g@uSgD*Ri{8*5V1PhY&kMdRY|LZoI@AX{!&He7D+F+%9e^%Q~pyj7G#
zZ%DS7aVM|n?sH7Gun~2HRHJ(`+z;S->tQge0D=(1KL#OD5mVr|5z(7qwnrc$e%zVR
ze`TNu@k>-*^f)*xRxEs=?DPj>dDHiMYOAZO5w(=3w*uqM|KE>oxu%u|62jdG;NJ=v
z(3K<-65#6}J8#{q*TyIEU%&!XW-_05K_@5P+Y2~am&f`TUV!#rz84WlqIdR@gL7t|
zH0copGUqavYtJDP<zqMF4W4W-l!FcE;g9+d=urr4I}ASkKok<%Gz#JF!=$85f!{_x
zru$trq4!~687MyHF5jnyFC_~<t>EoXbHCHy3l8AVTlsI)bN@^dF?2m(HIVg)_yoF8
z*1e5?64D<y>P`qRk*4R=cDQ6gCvvVz&_%bua2H;BCVols`QALE{<Mt$t4e>JC<57T
zr+nn?M?O8dDCq_BHXsLqI|ZcuhV3|XJz(2m=y3t=hxMj<cyR{~F==2NhV5ssxYBJf
z{6I*g(WRgS`52#->sx->9qqgF%*w9g=VN=CvuDnHA^3XL8#|G9Y#<N-3E}R;*8bK%
ze=lT2&wWehG?N1jDt9;7hb(bEfE9vvlI`tvIT_JmYnaaaTz8QcFGQ3=Dn!OtD)<U{
z5-aUfMfV_(E;n>2#V&=E^%|2hO_eN*!A5j!*Wr2)5rpuKZaoY}xz_&v-v7!-Le^9i
zxHg6%hIDfWFbqLpgf8U|qs2q1jU{2y&D^fVZ?ehLK7xTL?lsn2klMpTd+<<%t_ZRs
z((eg74^ose7)K>|35?%9087zyd%&GW5_i_uHMPCL2g4KbQq=zLG?B26u<O{im*rRI
zLa!r`GRG%^8w3V>Qo|pejWar?23t|(s(~6FiqNfxp(p{kC))p-2}uBmungU8cU1=s
z@#79?|CNCf)aczjQ}(#7apIW9=apIYrBU>3;wvmcr^(BXfPx@AN&i&?*-usI&(A%E
zKILpS^;@pJrILo_NB%BTS+Yxvjr4VdY-)aD6kdLIR1kz)^j3Wp8y+Wp{?b$$f!uYz
z-&AvCCZCO@#eiglq6xO2Hw8tVpdZ}BG2kAfZZn3Sesh0<{r?z)#DKY`wQFMx;s|5R
zg5H&_m+~hWYLsv`-kY-g(yO?}eIeU)KYC7x9=j{6`9iA$62je;t=9w@(cJr?n|U=a
zmW0fC2qm^;alu0L?eLE?yWf4)GQL>z`;NG1H(rQzpE3Ei5`9!6(2T8BPd#LgKpr?-
zJAXt+hFB!{sA<&l!%EnQ(ha?YKt#A7!1dO{U{vv(0HL3<+3O74`7|)s1pGE4QWj(J
z0TJO2LSPvvLi|flE%mdE$LN3aOcaa=Cy%#+!?B0h!?D(HT-6{U+>HR%0~rua&Lso;
zgIhe)Or;6N&r05f1&G+X;@3ejhYQJKWc4vpoV|Df+J9&)`nP{>*W(JG<bIMTnFyq!
z{)@Un|4%I{BDV&^4=6Um2IR*R)(-;09fiQQ!{8HU>F;mwuTe+>h(f@%5m3Q0Q3wbK
zcP6w}!k<!5gp4{Kny=E{{JfV<*zR*OYgA(!Na!vj`^GWZkE55KgM=VNA#_cU5nU4Y
zGL<^mO7m;V(erL;ZvretokmafH7k1dE+DiAEcVf|;DzV~fr(3|#)0OiPfjaku*ZfV
zkXboxjwWpbOu0Kb$If4NI0YNg7-y3%hzR!sxPG577%c&>D(`L@g?MqNF~FRqC~$3L
zgw%h%9Xg}668@BeVno~N*RR(ms&LrtJq24+ksN~r_<~~}c4tOUdes~{qxG5~Ga5ON
zy?0NkCp*h^iq}Kc-}7NH3XmFW8xoXA{(j;6FQ>QS_INS+)f1BNfJ^prH>KhjPZXCT
z0_l)8X2E12_UzZkQttt3@;TUy8hYhrpfg%;Jq$)hK#n^4kBr15M1eRWipA(iJAW8-
zM(d@Z7>z#rKy&wko5t@G(X;*UCtneRZ(92}v8Sj>WmsyUGg_|+GNa2%H|SsamOVWw
zPg%)(FjfQ>Bb(S2hU0y=9}zbYv)!>)rooF5BmLs}ZyW}<<6cY0=Sg|AA&@K?5~tcm
z?U%j{#hsbH=&}zsqu-Gslh7Hhw;l$gC18r0ZPS4n?r0<?Ap<ay!D6)MwJILbXuT8^
zql7(Wu1!WePBkdkeWloM#4RfYMx*W6(dg4#t<(500{>pfjE+9Y|CIB+kap+A#S((J
zYAaZb1k!)nw=>h-Ts`f6*YFfI1zwEA3Qmm*)z(svNt|bM89maAKn}({f8|G6q3w4z
zI*!)N2>xm_WKfj|K8(QD!(>!LfH<&eG{T))OGuakzl~j5E;mmi^zl7d#-9!?-;Bnq
zde6$t58Vep&h8HQ4vz0rUSN&yk5(!Yf`V{&>|iyJ{nURdxhc+1mi{)hEZX%7e=jUQ
zO}xriX99;#Og&k=xBp2&6JCD4b}#J9<1YMMGjxSw>#e$@2;^CD>nq<)p3(%|bQICn
zYBqu$h8Ax5p~DdF%o=Pv3_WeYnh%yuGi%&>WW-E?-$p$%N1S}1>jBF^>5(%@?Mbv{
zu==of>7r<Vj;;pys&#iT*2{W<F%wScdcbNR>uI<6;vqEoqJ|=7>$|{<^o+3dWa^ra
zKCoc7-0vCRC^kBcz)MdYyOW$=?c-v<`@rUWkIfem$i+jSTyvK6h1Gr@xcWqiQWv(K
z5v_xd!RUiq57>4XdMM8N`?GJV2X_}HDh{mT0<MjIS{C!_pl3qsrJw})j50iH^;CU!
zd3l%GT_;-3Sa5bonF2c#Vwy2?2MOUGm#x<X8PPpmmvHOH_vzdYv5p4+BASPV=!qHC
z@Lf-{y}A_q^zv*(7I-0|cPi7^b!sf2kY_chF}NxLm|~ng`ngUg)am3Ab%kBl+1{`b
zk(iMbfrxO2@AcNhU}R_J@4sVHM!16zFk1f0;NIowk3bL+KhCioSO$tvMe%&`m#*Zj
z!p4&ya+s`q7v6zGdqeEmp;siGvLGScjR4jI8PE+|THVHJg)IG^iRxQLzF%MgicC#Z
zQor+P#gduQ`Dth0XS@If{}OtCaqk`~U<=czInqdX1oDCzlWjuE)A9<($01A#Gt;mE
z#ZL!VgMe^HA+YT*__Uq%ClLHcKC)7#z;9y=dZ6;~2lPG+ECa=-EzD*tKH<*f$%ERL
z9{6;vWP%w`WHnX>#C)c@4Z0q%8pwJ=>FDxPgI(`v72GruKhe<zOOLEA(_Uhh8Fim0
z?Fk%Xex`Wo2`L<HI8QLV-(gSUds{AA4g`{LnX$V`DAfIGmn>q?-6$8f9vdraV(5Cn
zw!_e)cE;bI=pXfn0eLz0AS~T)tN?oO0n0$?2{Rq{UTNuNTb(jBib;-7t?K}Tj~I6F
zk+>$R4xJBJ4`e<!FY%<kc_&WWd;MYX!>dOQ!s0`{J)o~O?+A(DPKhfDG7lQ?;&V~)
zyma$(KBbs}nuDg#J1Y=KK~qgHv)D^ce)h-NkLSF+2b+&|V9r(Oe89HD;1dlb&^tHf
z!;7=u5fhaHUXWpjpM8iWT`>GWn3_hHf+D27(pBK_BcC?9`RuB1Pwj&#FaVvujFm(i
z`WE<uf^cU*>whj}Ll4JY^%H(fEFO}hPd@y-Y8I9u^AYKwhU;ab-a30f-e+$o!pqRe
zsZbBTzVhG3CGCu{if<SY$cm{u#v&m%+0uzMn-1KWhQHK<GjX~XG=#hJUvE7OMHRpx
zUV76(JOo7|fFj`9*ooC#e>Mz;Aqa}lrJxigOtmaErrix{%c=5M?sRSLEj=0>5D>5>
zn7K(*apMb)J<p#P;lT)96J$m&ZzpN*<bH2)N-KVk^#IK@EJh|REk);l^@x)urs~G%
zH<shY$gSJz!<idjWl~;SJ>M&GTLgh5Yun%8^mrk>G{IuDS;GhZTe6v{*cLn(p<54w
zksL5HJ+vt!+=;X(u#O9R*_qqPRcSC3@#73=U>PVvK9(Kr>9h}T=~l~jrPSnI)QhIt
zvSkY?l{8WeOHj*B?!%xU+>a7i4P-w#ip^INH3h~mrqVePo7HZE<;N&TNK$lW?j@m>
z>7kWh&mZCCM|qHRKP%(IP70w6qu&DdzzZ^=A)RD`SXOPP<84g_!$&+}cV!dr2kAgR
zxbtYR?J)GvTKE$j{MW8bT-sC&xHbl%c@0Hz&<}(lgf8WeARb=r?o?S{w?{iiiAv`W
zl!IN_&seOk>=c*gMNklgt_)ofWJ4@2KW>L=aA|(^=@>F<qnLzc=%}<;ONcFNO6_;M
zj68n39J~xQ`M9PwKPDhuZXOza#PI>hrhN;0%&xaaM6Jq%r@zluXohVF*ep5@G=%&4
zTf5PJZZ?J@pGE?N)4#sf5RnABGD$2&Dn~2J@KA&<<&PrQQ>jWz`e!crHv7Hm%?`*0
z6`jXcq!GR#frlb=MUWK*2AWEpk+~#VZ%8h7nu@F%mLfAN;)I!35j`%3Z^*NcRNuf$
z(Uw{HdP7-14rZ~vUyM%s-a;TFUQ<Nwx)yrT)%*j6ib3iN*op=eTio$bgl;_yMRI^5
z>rEBm?#aYuO@ZIWNF=pqiwu|!;qJ-6GEjo}trIgnKVO>s@s8w&2APjl8Tgw1@;>ZY
z#c7cqc90P6M+vM4GN8)zjPE{q;srHr68qkSY}bYbh)z(ODk&y;tY6i<a&X_*S9k%s
zN*N_A@0{mMp)N!c>iY)xifmzO=1Kd3FwO11EKGKfP|U*yM3dGu0|LU`m4R)C!AINL
zpTKR?t_(LHNm*0iw~>!B%j$FJe84hLe8Nt>xKuUT*;MI9sDJsnzOYFzIH>Qy9@O79
z7&V5@2doD&ABkA8v<GMDWPmMfg7%FW!CW<cdbTCKsHHfVFHul;NG%pGKEzMzW_5PC
zKCyOu@bFsFl>-Q*afzf+J$pRku*2#|?(@atu=%jte!dHx57>4Xe2Rb%IP5m%!;3oy
zmJu}-1+I;N3Ze?!p!Z>GCHyG`MabodW=_-R5`EnizH<$OMBB^3mG>X9SKePLrey#H
z;m(BCD}rn&V5uZ?ywAQnoW!=AfXyQiRtS0)`DVJ3<eRjZY8hD+QMMvphL}?$>1!k&
zEQ&|cAnXGLqJVitMor|BuJa9dSCJd@KkEu$8@m0CIs-I>JAALV9)_ZB;GP(5+J`|<
z1gsGf{mVKb9p`UBcql@bf>P8!r6Bmdsw{upH9z0da4?ev%!J+nz3zt35IY?h`|(hO
zt_ZRsEn}qpO{Q~CA7*_Z`<i;A9hM?%?*j^N3gcr45Vg`bX3QAyQZ#!k_F;_KFQekh
zjJ-jevF!+?#>t2+hiR-_6R0dp#}0P=fURiCRm>R=Md;SUP&5jxeKpxM7UAy7fUk>y
z-^NIk&q8_>dRGRPffA&aJSFVbA~fQ6?T}EiCujV7bXV4j-Id)Fl`sMc;eM3BdLRRu
z+#fx9#lJo4f_!W*34a9vEI{9hrKqNTyyt?+8?#fJ?p?$S&`RNC?Y5Hhy%aQeLkun7
zwjz+n;;FeVDt{{~iwQe*j*%<_HXsU@STrEqnGo1^7<{6EuFPgrJ`jSCDDV*(aBTz>
zkbM6I2nhF}eyxN*rJx8U%-P=UZSF3r$Y;&IcD&+gcqTZr_<AS%#+k)g(JdcALJ;~g
zbWM;EQE7c^4Lr-B6k+mNLUr{5ymcPScEm}XxibEVWs`&k4!~!c`0Qym@+8bzHKTmS
z(Sl*u#|y4L2xNWexZbl1&mL#JN7`Hde#r$JQDNM}b9h7{bn9U-;s?G(yzq~V#3X=G
zJ#cMgbk`E`9Xg}+Qc#SvEA7e+ZyJqErN0P!6IGt!i(d1}k3A-{OCgGY&S<?R$c)?<
z3hh3PHFZAzY?IU0ZAt|z78Ou(9%UvMo=4cfcvYNWOpO;Kf1*>qk}`x&(G(*B@AmV@
zAdq+N^E&Oz%j>u)e(88vq*M!RM*gjnsn8j%w;l$gB_Jg--83b_-I+;<n*zU$h<b)B
z?LkERIFoa*3>2ZIQtg8KQ>FQO(UU{^c6(Wu9sc_&n#2Knz3cm}wZ}n1xH~hj9>{=J
zg9mCp-13R1I8ga4n35abhxG~d4Ix$^&zWZxPO2QAQk}sIkfp)zTXttIvJwT(*ku1&
z0rvGsTKKqiF;|G1;%<Yve8<Eu*nnazmGO9254If!pEjT~bJ#Qr;pQU-><xq6l@09E
zP6zX8+<d??P<-U3G{2{1D1}@b+gs{2`{O=(tt&frHvN&I3mt%P>jA5QtjF1*N3JbW
zJ!wn#(Rrql(h*qU$F%a=TmT8eGn{}o;Vb7`GQ9L~=lZVF(^qek%+g(&s&DZ?AWM5+
zE$f_idi~K~KjyHkoH1-Y=V;FjfZ+#1D)irN#?TXO=}&O#AN2rh?f<gezR=I59gpDi
zUm1Ub&js`6GjZ8Gi+M!{SZR4@EWs@qNVZ@HpZn1dYVim@|J4JTPb}m1#xISYr%eXG
z7WM2(Iu47^ReOn50|_72E7rGIgS<*f@!}J6&;6Qr%m<!?nD?PCc|Pw(AeEjZSY|gh
zduCXe^poAZLI;~qqp>6U)dp@p;C;s6BYwu8V9Ta_AY{_Az<xl$wXyfI=(~oVl!lN=
zqf0>%QcKp6I+!Y5_^xZY>q=E_`)P2wv>6+A3Z(A)>I_H-LMDx_2{Ix%;gG=sy5}{&
zCokW;_lVj87NSpaTRRtn?Bv}g;yAAj9h1Tfk!jek(>(VbKOLv8t`TEOE<hlE*cvQ;
ze*7bgEzKe0E62xb*oZE+q~MVPt=;KAcN>FIH}H;rY14P~5EzL|0c)nQmwKEex>5$l
zAqb4nrTpo~Ix;@+j!XQWTk(JEB9*^7p9>Di_F;d^s-BY9f(IjXO^_Mw2;~<JAT)b1
z>CeRfmaRqu7Nd+ELKLy$>JPi_&6oxIkDb7a(O`Mn_uJZ>nG*qCy}aRzs|X~ApXQIe
zxfI{Xf{zWuvwX*3GYZHIQwF;-+-bnto&IyTF&K>k%NDn78jWzLLcmVvz;9zLDhb*m
z4?PtE%RmvLaMGn~<$t4XkofdV$}uxrMR2JHqp0G>A^o&FzcMHYcN>7!K=xxW7nR_C
zZik*{{h@aOCuG@S`B9fY8yb69>ISF49$z;47Dl}M?60mS_2wxvxx_F-`MmHZAp-f+
z?%`hGjc<U*wM@Bs9qS<2e%||y8iRgt_hn$)VdxP*>rcS>k9vTY|G;mfo|DIv<iHTb
zk8>ObmVwgaV$$zo^(}R#GdwqbQD-z_7X0q;@?PwIO!81zD0DtxJ&^erYurySF!vp@
zcb7GHQGJjHi_g*l=TY_)Wtz%9_GlL34EzT2OQXcsXjWTU74M%7w)jG&1q|d>ZY~k|
zK02o5!ttakt;`WNpRECE=&ufO^8woqgO4_lN$YOP2X`MPCIT#)2Cj{Oj6JR-fq-!L
zVe6%!2$_eQoT`{<lpMUyCM=fFR8{*Ke8tLu-G`;Q1`30OaHm4+H9<xs{cRg_os;0R
z;@6?o#U8Duun?I_Rdu_JA5YGbNnyEdePRl)7<BzmLA?cIj#-1T*e;H5Zqo?lmg^a5
zk;e*E&Q)Iiw(Z3GIM|3f83`|eF$j13UT-}NMis!P5Dc59Lb&@dX<1X?w-M19UXMK>
zBHU33ECWS||A`hazxd#Z#Tw%UJB_&2CG?gaE7-3#%$EqCfrM~30$2}ZKpzRNZ6WO4
zu_91l!1?3J(+OCB#ys*SD@t6KEfBn%nFB$m@d6a2GnFI0BUJx*#AQkyezhM6<hZ5>
z*+nJ09a7#eZJyFa9)b<1ojj@%1cW;Zfo+GuX9`&4;rFjmNCJ2@23#8f-Lo%v00M##
zh0vv-2#tO{cITF@dw}fW@qzXO`j*=z!4K#?u*dX+VofhWLJ*=5x+chojA#qRNX27r
zYu)b==qpVQfQ9Hfp$f;cxW%?4PkOO7EsZw35KWw*{!#20$1cbbX%~Bbgb0D$#%KLJ
zX^5Ww8*NBiPrpbHY(&XFJ<uPI;eG(u?)0C#jlpQCi2(6=(>ZAfotc=psTgo=WK==F
zstV@P5E!9LK{2u;|8hN7I$6?1S6{{Z>J1fBFq{4az~Atiz9_${4i81>iXbaGb;80S
z)$zN;%nN@m0(FK(Sc*>PJR4n7aZfODX{ED>=I6yr(USuA++QoiRride?rmA2IEX+p
z4-4|9FmK68s!^&bh`K5QTakuCJ^I)U2#V0HhoPvVnE)~OkBUTrPpsFjjf#j~e{Y2z
zjMhs*DKe5?>a=ZfvM#Z>G7&$RU0z=f4(m%qu?A%tlMC|D8Lig@nNil|!#&Q5InN1n
zS_Hb)y5LPQs?^P9`jE7bUi1E5>^0nY7%xV?>4wddY;XM2o_j5+H&eSIkdlYT$u8zk
zE~Y$<^dhvU&4kV9s;=JJ5nQOVqi{1?Z#@h~-7g6c-J7OFxPy_XB(P=(xHdAH_>^Ca
z2P1STC`RVT79LJCC(++XYB|7g+JbiA9JqDx2zFZ3|AEUC4@T&kAT#pfP4Zl!oTR$1
zvfsaOUl;;bG}?jK>J~UPyTI{YoQXfFI2A8Oj1<1f$3B&i>ns~z()149i$LCZbiU|~
z{=59cA{w$Y4TR@mGfM60L7$-jff2g(Fc>8OeOlYUMk8QdttfD9WR$Y_`Z)AHZLNeq
zrJxvT&-N6SbLHffXwGyDhU<~u0~at31F!Zr^l1dbvWa*oLRSP?(ca5F<5PQtK1T9a
z64aIE!~3!|%5pWn?@~_ST@5Q&?m6)zcqw8|&^;m1RNomL)?Ax-m-P{_FGHHf)B)XX
zY(|T|=387v9>P`>N%sc*x(tFMbn9U#8nyEGcl^f$TXbzxS=6qw|NoplBkCt3cmyMK
zDJVtV<qciZgBiozv8D}~F7ug%;7pM+HTFzV53xS_Oex$6_WIuonNixKqq5JvE63T2
z3GW0Quy_e87@cNr4sYLaCAVPe_KKzF_*J|Zv4xm(2Yt?CbPVj0=e_N$gg`!)qwYOL
zUoV?W@$;!S*~v-RjGn6@4e{vH(Dw<05kIg?49CAR0>*!$z_l?Lg*m$=;K2x83W|}7
z_VxvzXFoFI{U*y#P<-C+iQWJ<0ecBrN-13v9?=M06J$ma)m>NbTykpQws0A-(Yx^(
z79$atZi~nkRyCq$`_dmMhc4j7D52pVPpyLN<CHz`E}R{<2t^>Na)!HlhIx1+TN`)}
z-J3Rn&1m}S1bWI5LNr3R9tNZCHw1`;f9=y`foKF=8yQtwCoAF6r=d&v)2E3S$3Cf<
z5$g|M-4arN)NwbsoccWWc2!>u7%lNogsup(BI<CSApYpPdNQhHr6PX*>97>_^_&#w
zCOx-JZ^CGc`7gB&yc8W*&Rv|8XXZU)cFuf;`6(F!S?#>CuOLe}PoS}rMKgl=8f--u
zB$;>P5sc8ShoMLg*aP<3rU^FgIY(fdec-pzkSF)d888&_1EYY=PmTr4KnarT%A1KP
z<y{)M^V{s#v+*CF(95Z>Vkg(qi==%ZA>5A=SPx`C&fm6G3}kpIQ{P{luISU;3JVaa
z=7=gQr>oVu%eVL&Ttvq40+jOmeFN{c*SaT8RAp${R3Q<_GUk~$>H6hTH$sgGT04IM
z*nn!^<Sl}LaL+k{ZHK`}+`^wg`Ct1o;EN+s;Mxc%-2Qtg2nhFN(OL<AN<k4)d)!kz
zJ6c?pTe0K#JF?Dc@3Y`+(Hrd9qH}j%<FOO<dQFfK`4k^p)-w4jzvX5@Zrf=$GgyfB
zQ;ky6UVL@rjrrt#SAx^m@j~QAp4EMm>miXQa>zA7NQ4D=`+8T1J@}Q^vnT!Jrymqt
z-UAzv!6|+`_QPLqJq$((z--afzxHLKz)_pPwULqf>pO~gFhZAtVq|9|J>vV~{#f+q
z=lw%Yr4nClz+t`;_SCg&pqmgFiJ)ZG;NJ_Gk@|}rSGk%Ux_-IF(q!>pu!hA5kv%5f
zIlAQ$?Ps;$3NM}S<HaZ@et-70q`q_eCORCM#%MKwRr!sB-KT}PgHy=2%Ds=thre@G
z+q}kaFc#sC<7;>N&)vpg#Bbs6ue512!rhrkNtgn^jfgruS8YH<xH~hj3>2ZILYF?{
z8i54u$%MvBdJfdo=;hR;*o1Uv5wsv7+>HR%0~wI@)*HuT^C~5Fq*(^l%Mez;0`$|#
zxGSdM8C4Zon^sOg6@DAW)Rhn1dv|tJFELhUi%80T;04+;?f29loiSFT%!YxfEem9@
z<B*b{!6*m_cV`B+9R?p4U_~S8rhIsD<}spTKt>H*8v(UWub|K6fY6ztOF<E;@S0HV
z9CO;qxAU1(D9>+$sRD4$+7|mYewNTRCXf(>D1@#FGNQ%oulk`5_s!?+rp+Q1?2f`h
z<ZU7Q!_~=c`5Wip&TZxk9(YBeXGup}6PmTce?<F<4NqDF1GCdwXIM|nSWmJ)$#%Wt
zVhew-2BQK3ClC?t2XOsvV=&4Ew(n%xlo9SIBq3!A{5B%GP(*kVdK3c7KoK(9pR^h=
zSkq`H<8k4H_Po}wLXgljHX+UC@7W+B+>HR%0~wH#|EnL}3eNer^!BzlmomaT@weV7
zgYk`DK9S!2zQ!Y*RjYUb>ewDwyFIRaN!KH;Im&DaI34JX+GrkgJH0rauHiOU-ZU!M
zQD~@SQ5pn<I|_kqhry=}IATiSANj~gn*zU$d^E1#Fow<tECa=-+fyyqqF38(#JY|m
z*|Bsm4t-Q>ANGhW{hC@nbUt7`konjw*VpZ6K2{*r{(|O;nHx8(_+x$2M!SMvH^y;i
z%2Hp)2mD54PDeVv1$e98W+jlUrSG<$0KSkW&`|lr<-Je;NNcH3PU;!hd=diYuR`Yo
zwjBl^3TuD=^PBR)-HC|-OC;B>jexR=M7=>kxI3}+Qc#4#BKf~`ezJdP9s7#>ZA`a$
z0eTgPFm@hvo_M+fB!s&YTdxT+qKgPpf_sJ|=_Df_wf9GQNMRx3^inyd?Bge|&wQcx
z;@+ZaykgK9z9K`Z+-xsoyuhQvt?7;k<kBSdSB0PX%rUnI^bKgUp1?-5BU1*wUkvW}
zz215ljF##M5YIN97sAa*R2<0Bfomfp!5``9ZG>?%S}z5~$Xvl~rul}2+fDbYw}YB?
zFC7OzaJ?6UwfWhlREIL?iq<QFtjPP}^&}rVx<|i<-BP4kufY4hmFl?-)uSWtg?sJK
zb)~ZLzQ9Y-lgZmp*;_44&R_kUo?CWS7J=MjG({+u^-`^)<%o!V6G;qgMPxr~@z{WI
zz4b5@*#Vn1=>20b5)}iM`vccTMd6f&^U#CQdMPMH{LQ{)(>?uBE_2p%0_9Kn?-zmT
z(0A-~h$EW_y{Zv+Fj}t(G9#0Y4tfbgsnPoaCtNwT3g9h_wKQ$Dr8!Zx?Hft#DfR=>
z_zlbKN8}vrnzKEl&qy`x8gLRrAQ2Cn?0fj03sj3g&)iq%)(M-D%JrZeut&okjMiHZ
zgHf)zzyH2X8R5>TfsKRyvilHWi$8h|2tUrj9asj6P$PX`nXZIib9%oMePG<SN^3`O
zYDf{gKbuY+j|3wT?nVIXfegrpzI|fcZ$}f^+fS|AL?hs(M6XX>5z5+j-SB3~u~xDV
z`RaJZp(CD8RV+^*dKlJc7^-Jqw*!I9>$rb8E%TR}^yPgqF_F!LumR2Yk<fyGaA(wD
z+hOpb0H%c$H{}DNFO!i3)}mo=*1&qFzy*&egf0a|s6wh!pLaz6_4jTC6C0n3F)s9%
ziaglw?K8y~?t+9ML?LudkP(rIsx^ALk8^+Uj-2N$IN}Q{B?_16W>>WFiKaPok|R2W
zn+h*P`l^pexmr*1e!lo=QQ-GiIUovkJ^vcs9%<J2^A+VGMLH(fh^W{T@i;DV?N0x>
z+Zc?x8we0`|2Ro4CI;;J0bCn9v&!x9tk7S$u9t#h<Rg2X$N1#~qh=r9%lh@TITPRu
z*PoGCDG@=7*k0(0)+>UnX#4cOU+<JD%$g~jT2j17zrs@V&VVweZq~ZU^zlSX5?9l2
zyc9WJW(yQ&=~yAsIIhk>cvBLAbklo3y4cyKpd}>TGjm8&6}F-ycj@uiIcB}}FckR!
z$J}lIXGOre?Z50`C110B26`}BE8$NmC`F@Br#t19{6;D-9X7kpzY<^#Cf0qT*uf|$
z;8rU>jKG>8Gn%Rk__m!O;-M~8q;(r18@#n>mSVyQjxA62bXN*JRd_&k6fZ_ZLCCn5
zpJ#XU=N3A1Wee5-TbqkZC(|TE?02^$57;tw2L2BDdR3A6V2_478(6<jn2dnc9})k^
zNK^{=9viqe1|!mfJ=xG1t(SsgMEh;Wg3QFs@+ygkY+%*J83}ZshQMaD=X>%rbVlnn
zL1x6ibXxqzwh!kk(?b33im>Uzibgc=&6IPCSUYpx9DQ*ZaYG(2MzJ(p)ag3kSFe$k
z9go%D25g3wNY41vjfJYd>4D&*tZ?B9*wILHp9CdzM(eGI!N>^s<c#7UhxuaSz}c<9
zwUH5`ncECHqxDiyjO0E(eDu9#Pu<)nH<SAZcpUGdmw+^5uSHX<?{I<6XuT%LjQn0K
zl}H%uC+ppJPlK1{9wjVBnr4Rw-yt0Zqi@+YJE;)3;l=3Do`z{N-Nx;4RpiOu`b_e`
zCJynXsm~b;$VWT#29XQd6R;Waua4sJ;&r|CFc|p&>#LhLjYhZ!Xws6Vz;7d>I^o+~
zU}D6NbASexfg+T!RC3DR=U|D}lh}d4`;t>{zz(e@0V^%yQ#tS&6ofk>gVjLx<F&7h
zyxKQJa@C8ImpS+iya9fR!4*=v6Pe@Efx)LnAM4=v#^cmS0qdY?g~PXYXT(%D-2aV0
zo?B9s@}<~gz}_m&-2Q3V54NA|BO_yA7{Wb11KSQm&lIrM-RU34XJQh-lr?Z+`7xh}
z!y^cxOF;=*s{F8$6#HhYNl(evyEYj-$3Q`STd@1G2Dj%0AR*l2v-Q6hGNL2OHp3zk
z+2pgMPE7u{68&HWA+85X;)BVz?bKpB)3U?b^zaHod>KTZ#Du{Iv{ueVQh8qwLLghI
z@}Dg7&WxyAeJ6O|l6C|(qDh2WB!~$21GwIL7>t&HzKrc3`!Z2!8DL#C_E-1<2ODld
zXS80*pFmW2*32QXontqt)JR+Qo4GcSkr^#Eqg@|uc0p&fUK3<STQyWYy#?KdUq1Qq
z{ubj0cpp)-hb9u1s;ZCVHRxQYJ!m9>7o(r|#1>QyDD`zsSV>s7H3G3H{EpSR=bzrG
zoY@(<`&?G#JJ^ht7zDMTGg@yw3`X6+ir9pIWh4fCat2%*qtVubV*X%n2I0*Ix)c<n
zqR|M0x6V2;yN8Ej9rPClXTZ$*UKCb;Hg7a|4-ZA?iXbZ*wkTFV-~NR^rFtk*wRI*R
zRxk>$kG)l@bfIO-oq%We8IVnQDLOfFTZ+okFy%(~L6xsxf`P=Ecubj!{hE%u!Xldh
z!%5D|uodxLEJyDrgF6_l-RM6z8$(epFz0yeU;8vMF<^j>{r#)dp)ZEegV9<Ee@gk&
zr`gByxY`&Vr@lLAEOc~mAr}lreXQ8Q=p&h_19V2~H9=<dIAdhgY3KZsjV2LQXD&50
zEJh^(1V7oLEWUJee&|Guhc)BH=;O}=mMKFm9xL;AB7`%J+ar+YU-Yw8PBzZ$*vZ*>
znn)P_n-9fhay96|Xub6?7*znfqs?s^jBqE`z^-e+Z(}G*|Jg<Wh9Z8Pi8WZppZ?4*
zq0K<iEdCcC{f+X@2V4r^d?OX`(s;v1ixe|5=uKsCKT2RVkp0Xs2#^U3XLZq=g!(Di
z4@bcAb3Rxzo|sSRV8XY&(+S_l@XLw3*iw3(47HvGCEjE;c(o5WN;-La=1&5}z*Cmn
z+pNUy%D_L2leVK2j}y$mw!_d927GD8`;S3LR2EoQ4_q7ld}Vjeg&u^~OF;>utvbE)
zr{6t?C8=lgMncwaHNcaDUN~dFJJV-TN1x}4JF#A`2{IyswvScyJXK8ZLb^sr$c`4m
zLbNyaWm5Fn8DjUHrKbH2=eY0+LWB`_+gityI=fz9V!NoGG6an7>jP^)D^|GOvnDVK
z2#k`1-Io!3OGUpu!~FoRw;l!~Ibd}EYSYm@FU|lW27ElSc5P(zLD#tnI-~VcP>i~N
zeh?q%oeQ0Gs922utz>!#{O<LI1~#J{&*l|8`ZDz23z^Y5C3U?-lk#a91Lh$fH+Fb$
zK3-`Z<V$frKz4SUU29D$&u+XJF-4pd+(uJ-$mQ3$cBXDQ;3JQ-n(C2l1W}Gx8%HWT
z2wWb(W<*Noheu+)exEQHEdlS(4s6N@cV7m4+z<RVA{xl-vVh)~fn}fw`50duiSi*Z
z&&iy<?eX5EYde@!*Ti8Z)%t$E7eGO{A0@CF$bOWUEt!X`wg|t7`moIP@$P<DVd(KA
z3FEl#@g^G9ZU<@xef-u{=S2n0r@RO`NqN7*lp}u88yMe9i%~~i*SHiY&8BqjL?`@D
z6}!gy4uH83?nxrB?J)F+1CvDN|Js*{02|5x*G4}s$xIT^gV0(De@a0KGAdeCU2GT6
z9IvXMIU{%Sl|dJ{$DR%L(%4%qeMBH32z?p4Cdh~?1Abl=t~YGg->sB=;v`WFEJT!#
zmwj4R*c6x3Ws@a_G6wMqLg)Q|(=n4Vk=-kFO~2j~3+zioEWTj#V3w_tgirFuPyJCZ
z*g=S#WCD*#qV@ZP!AK2QN+j{GjAVh;0l>A9k@}-_T09t`OF=QRYxkU4-u-#;k=B;&
z`Ax)3ndotu2==(l_49j8JQ$&Cg3O5gC(D&p`7;L%KSn>U*ZQ>pi;=Mw$=1OZ)t&SK
zrE^sekJI7BXw`%}mAp;=Wl|$qq)?3iJOar*Zu`idkp9qzore~Te%Ot{X5^dRiO+V!
z=+?tvBnP~Hy|!szhC3w^1rGXLyEZb)phlSB!3bRnic!MrmG@&OKc)t{4owNFNk83&
zUX^wnJ0%+9ndirY5xOSGjQD1bYL0(vHgZBVcD$3KE{Da)cK?l*(Xlc~ZbWkIRyW;1
zycp@K*moseP5khKEbGn*ZEjHnGVsi*Lo4adopR!%yDJMha$qw`bm4jhc4@fNf%UtM
z!Ke|KB|84E(Fpj4L=?C-GP+sP>Wc>>bSWrC{2mfdqv97ue|0ga{TlXos|Kb;W;EE*
z=<v9r1|E#iH9=-HZ5eySNngvpJ=Qa5rK|WAEJmRwH|9t(tVs))WnQ&bvKZpU$k1fz
zb?Eus?_$dsJVO~&&mxd=x}LQseDf2D4wYTWpMC@X7@>FDEYSOYLSTe$Jq$((z?60L
zzs?d#n*OCr+o^De6GX(1bC3rVfGPt;$fsjfV(D2@SNd@2S33pS*a2{GrYw(5X!$#j
z8Au3sBY^cl22@qoM^U(|(!@w_I)ga%CA?#VY&(<eIvxs2-A^|=|FVjz3@<<pNeopQ
zMP(H~<>v!gu5lejAWya5$q%|JY^$-iONO}rB_-@QWJbJ=8;nD^Q){s8F!+R7`}=$T
zYZQ_KUYr5f#wavBmvaXM1mTSbx)c<lw!7Z4Gb>NrT#QVgyeSn9)pY`wSkq&t)`!#{
zm4k#Jq}J$~AR|)olI0sG9zM9gz2;5QJ$HIoQK+;{|F=oem!><#fhtooJ;(7vBzUW~
zMMBf#g}|&^b@#99HxNkCT@mJt?aORslP4NOx2uH0MwIjPIUY;W*6#G5yN$ufsD=Q+
z|Bq2fTG|x&ZAA2<tLrO>2zL|$%RmvDO0qARcB?IYLE1SOmuG!<e-Ai1dx}lyyYXpr
zkPz-D1l9u?P*Q-`_<W858=3XXN^uX0Ojv-D!qskCoML?$#&}a8M&sfoyZ{lME3uU1
zeHO)b=@X#`LuU~JIdnZhL)oLF?5TGr2Sb$UC)j{q^D=lrk3wMEVepv(HqkfTGzvkO
z9uhZ|#QxrlZq->6Iv=0_R2e8fVIlNZ3k!nhdnDdkes2(d#tlwXU*^N^#Pm+S%!1Ab
ztOqh5A(k*!`q(t7mL(Tiopin`SbXgEpR`0&>g3NShds8D&||`jPv)U7EY_z&DSsS!
zD}9<<Ef;~T5v@A<Wv9^Ww*ybFNTeP(3!6{*jSuLrgm8CaVB2Bv5eN8a|05qMQB&Zz
zG5V;q3CTd8T?ETO@hK7z{vrEe=1Y~&`v(_)3-^2i-;|xfJ_}FqfFK?z5LgXlJtQpG
z2AYiZ;&XPmDo=1cVuGdT*>2Zuau;@u@vz*vy|?7(9=!DIKC~U_;l1CNi7M-oy^{P+
z1k!4V<<U0dT&?-l%g;j6%}&79qfEY=9D48p+YUod5unFtQ$2fdX1~BH04zHvo_wqZ
zLk}Os68`jC$UCw&tv~Xj(n@cA?r`{7V)tL_L!yd3j<=St)CU6(?uX~k-w7E`j%xtB
z>$erH$d_+_Tpj70f(2*a&OF-%LS@rma#=jW!K<(Ef^+pLE2Zab93N|l*y_{Lcsm4g
zo;BjI2Mv#5h_=b%g<l`sVZ$-(@<Q(|EDZ7fV6ZW_@F!5+lnpQLLM4G6V}NU8*tzql
zjRK?tA#+BTg5oo39`Aa1^}W4HXV9~!`@?lYz+)7zp2ALh#^<l24<v_>IiqWWOz6rx
zChM75q>$s2nfZ+CZ+^of)HR{NI+*UacWWBwPH{Ec?|2cCmQQ%!*-OV(;A8vDFQ3XE
zfjs*B(`q97;n(-=?P}-_gx`QoXm4oLE-?1sP8!$l^q;$p!6*S(H&L`{<_v+63@{A@
zTpJlt4@=>(XTw?ve@a0yk~9AK>y=CCx7v7$na_>|iJ!W`?70H_GwC6p-~ccRL12Wg
z2{NO)<Gakl=`W`TIZVn9kLX9jV${90l7?_ws<=sd@91ppqZYgvnK;uoofbUc#9*d)
zEZi%G6M?i&INEosghEYMK~(YO@NW2#2oc>qi3cNe>tQhJ1{hUr$_RG`EiG;e{5B%e
z`BfAP#v*>45e+N@MW{_z@a?4+9MN-L4tWDPKf<<4|9AMy<-dI6%B@rt1`@*kEP?ev
z1{Ba!M<&Oy(9Y6Jzd$+H{RI}F)|lYD%I^biT}e@y6tz_N9oq5o<cNF^X%euD-{TZV
z)plTm^RU>m2-QwM(=^-mD9xys3b5l)sjWTw&<@;50N8dIe5QcTOk>l|3^yOtT0<;8
z3CE65LFWUOf#PH2^?SfJ%xUoWC*uI4Idye0@SFt_QS3ZumLv89bUt7`koo9rjj`Vo
z$Zc6b{k(;hG*<)`AA7Qk94FH5zW8pF7qqP>;u~J^r><EmYOLS;74mk;=X2@B0tn;^
zr%_MF+$o2?v}K3EghcqqNnhU6*$>7a+<d^c!{9>;>{})JuY7=GgRrAd!;HWyJox-q
z28xgQZ>#S=Dr|MBUtM#t$@aVJgN{Bj*yDHaGv^-S!RNnvAoB^^D!p)WmEt!i=kJHR
za^x?;;xjCzVyZyL_i44k)nO|KZz*1Ubh3gjyl`?JcKTFv^$p$AVBnCBaP3@^prG7^
zS@Ri|)H{b^N1yOw<^VkS{MUAve1L6voj2tJAqNrx#x}sUvHNN@zRn8*f{;q1OF<E8
z^qq|L9olmLZl1$ZSCXd|8%Rj`FUN2mTY0Dl5`vHep=*MS$l;QHIoa-!Gx8UDHH3$f
zJzycaBA6>JC~yhcWI%qvhVamNyb#fp1gjV5sfX+D(d~<A4FtYKESTM6cc+g+D}!ih
z+qon5jj$0h7iXdO&cogPuifcCcN>F|8nEM#&c8Ae1quePjf|RS4&26r5xNu<qos_=
z+0NTf=k5r(91xcj_;{xUOr;mFM`X8mXV~Du2wf9oMx6OC?a6mkGwEbh+&0jPCWgg`
z&9cjqRF;7FNmaPZEct2t4#B(ht2#IGUTg3vzQd`tj1N>0$QbW#&m$d(u>jXq#m|vn
z^<gtwrR~OJt>W69{&Tl67&QW~#4c{i2zM?7jP!utMnu(Pw$mUYew?W^SO$uaiyTu?
z=x6J;inrplQerHp2GNA9un9SZ+2b=A{omgT84!nik&|8-OXkf*7HKvG(mq&#bRDAJ
zCa^`XocujQLnTj*-&WL?#ZgTJS$-KgjH^*xr=)9vb!H*=4c~Fe`RX~!9~;lSl>{45
zmO>=@!+PAg5O|L;_=p3!kocy2aCc^65)#0QX>35+e)>!xAl#kVdMPME35E|C*uQ)%
z`5M_A@taMEO$7Z$RT!I4YR9`i=uv3BCdh~gGkhj;N;CUy3PuN*3+H%XA$q>>X5ZwO
zs-9gRUpE;2Vt$QRXND;3$vYV{^=q->xX1M5{3QgkLyaepN?KGbsW<Z4NSTiWY(x>t
zBGq6<gu63aZ#@h~OV0@qrkf7vAutjF_Fe(5jd4h(jvx#VM(9%hbY^x3M$}W58w9!J
zZ2cTf7CcnJ(-ji2Pgf8`cCg^V2wf9oM$ZJ9t&GGDQ7vXLAL>u}ISh*t!dO(C<o2yo
z?{~P-OmbV}w|(9(`syj(PhA>>i|!FY!p(aSNZadWq;nnu-@~QJ&a3k}!9Q1e@vKe}
z9*oefhrvh=I9*}rUl%J%Nds^Eu^H8*wdsH_$8blZwG#f6f?^crowstY!%vCv+H*nW
zdx88-;Onu%M6AX4NBv9ALRYk25oAT*chj5yii*+g_})k_a|6i;OVLhxr|Yu;WLH`e
zFTV{KdDwtgFdC8Pve<b^p~qn?fa|T#d=dirg;w_Ehtp=9+-WWEXqW9HVFx4mMkM;6
z7u>;Uz4b5@83A7bHU6U_G2m0#wQFNAk}zp0gsy146qKTV$}shZeKtOpABJQsyQj&1
zfQruF#!|FRm@WjmqV<X(E7~JmY+^D$&-#fjdtvBc_dZyPoK{QEdvrdlBB3c8>5-c2
z!Ansv!@=zxBs59XA6IuTKSBs2kh%P-qzRc{#B9GxA2~?n7Y<v|NsSZeoj!0YT5mlJ
zMHRq{Z;yXeBq|2%TLoMj6(M}oPeJd})=NPtqWIYE`*q^muJ@LWj)L>j3oTLLQJh)8
zu@D=+YTX-CLj|4DdQFfSrOj2{<QQnQeeWS%Byh<l6&54gp$qi8rjpAHjfI}Qlaa^o
zD9*!gdi%64irlX!yY6lgs~Unp8gWq9yBiBEk`Q>G2yE4ZKg{R;;e-Ab2sfkk*27?w
zd&b|N{9hAn;BZE<zZ}ImVW9C7I-|7`{*;1ZWOTl4%6y*n#^LVm!|fXO`zz2J3qKH5
z+{nl=UyvI*qxG5~Gm78x>FMj~CklJL@)ioW@9TiYh>6v*{(%(b{UXt15dw;{xA2Na
zQl?y)5f^V0th&;$sjiZrL?BD_`sC|x{cKVfbt$*0Eu(-Pjh=WeqtAQ6&1k*#Fc?(;
z$4h_MG{MF_%$Jcc1%4Z2kw=Yq1lXhT;~eILWuOSjHKu-fnKI;e+(cuHLVxUX64;+n
z0mFd}fyl{#`4K1xcN>7!K=u=L^yQOJCch`Wnr#aZKkMLq89mf{GWV0H#GV_>fg#i$
zH!|_^W8f@k^m9=5RK|^+rGY=LN+Xcloe2iVOhRXc?IdOPt#Vnw_7f`h41Ibc?omG2
zb{KjnfP-GB{xJxN0_PB|T^s!zVAwGSJqWFrf)ZqY=F=%g3E#nrZjH~5UbCwNYT)op
zzznM|>wRB@$59LGH9<xcBCI~-tV<-yA^wW*7@>*}tRTew(uuWsn~GfUq5LhYJV_RK
zAv)Lnj;6v_6k$l}P_`$I2$(8T%Ty=TCM@$kz*!$}E#U?qk)WprK3m10?=}V_BjB-C
z`^O+8DQyb;HX>ROGDU9`gF6U;WuOR!b^ZML`h!o&;YNW=S2IY}d2PY<XKL8*UC&TT
z;qzhie}5}vK!~wbNp<@Z9ZauQ>#5&3w88>3!I~JWZq}V8K&WGr=4y`L?&izO1BSO;
zztHU*bK?tt#j6P{U6k@U&Z^7gE%S+9O83Dc{G59K5kvHW(zv4#c#kmn6j}HasBIdB
zaQ9(AGX0l)I+~(K4~#*$`G94h`1rhBoLy|O<d{=8Ugr30=xt>JMxbC3tTf1QIQ9kf
zP7JIEG9QBct-H7;9Bcg|_>Ff_&<Mlg!+L9H(P5rP&+iuGFYoF0tidb(ygWzd;y$cG
z%gRJPEA6m6fIt>n$;y9uaq!y6j4jKA?f;Q>-tkoZe;jvHx-AuLqex|B?_Eifgvczi
zcVy3!B85~WrAXPUkj#>Z5E7!utb~y4mEUo7j{Cd!`*4OH_w#u4*ZX~Z&hxz9@6URF
z#OW~yv4>QrQ5^mO`;G`7S|B29Tp~U~{CdETRDCWb1-wR{0)aLN_*AS)|N83rZv~}^
zD2f}2`z~z+Q=qfJR2E4Jv>~4+3OPNX8^rX;vy8HOGrdh>*z5K@W?S_dO!Rb@2I&Sd
z7O&NKGkIBN$C(eP=t+36{(Hx3KbI?K*Wsj|TBhRgLF@fYpBD#SrCHCNE>d1%hMAr-
ztkKor<RZNIfPF_qPX;jE>%J&G2xh4Hg@9cj<U1E{zr$Y!<p+Lhan8cL6#_vD1$LQ7
z`M*tEND=xv=Kpkhya8NpwIScR$bI?fN)RFVRr>``h#@*MF{m^0_6)`P!LzpNyNj{i
zyD9vot=-CH@1EH!a`du(B0i`fN-p_3Zp(Vo;l4{A&*02=U=Q_#<XbuV<H>6VuMar6
z3aa2RL-hKcHj25$h2A5=$i0XHXR~Muga9Kxpu!N)NM&flu=95m7(rGD7}+0dI&ZFb
zwQTVHgb7o?&QNXe{7;c8c^WjmH$xT$Mvy1O80lwA^B4=waLhgvQT1s{5yYfL+yC5-
z9xjOpIg@m{*oE%qD^wT-Tz**j-sGfa+s`{YqUHF%;PB0XF5&hKRioAKW{ynGyq>{~
zk>J;JBA^z*4`t`x^!(c<#;BYEr?kY$MLu9Vz+5B6NbYmSEfg3*RtOmJ|EWE#Ry#W}
zTUVM@XA{AC6spQlLB8xNzalsh1xAo3#2D?69?IVI{daS`ltmfMafb((Fq)}Q&9vmZ
zQ*o4Cmtx&x6?7*$D8)H<4yN^P6+0C^bKsNNcN{)uF!pS#o>!Q2<Rp6sReUjKjNT>5
zenx>2)O$o2<pIS#?k*W4VA++Id`Pz8-ETWEDMAp`01NY02pIXNR5;!EYjJ_~t<W3&
zOA&g<!KAtw7z2=`MHi&*pePYD=VaayVv3d}of<e^%l*_Udep1_D3v!RiZnbghprZE
zrTfwP@Sx7=D<-JK==Y<D3zMr|C}WiwxI^xq1rBwX>b-TM*B*H6%lfVH_o|*Bm?_%7
zP6$O&hK1fErU)o^CcG#`@Z&NOQ9a;CDn+x!6+@sDaZpfD80kCca~m0(=sQ|Da9cYN
zuI{wew{x)P*0-{nGeEFGASk?MvOp$8u48#u+?st`oPS;X0H#H^$b~4nQMCgUB6vjx
z{U8SD9rrGgdykby2d9K-U7Ia5F#)vwsLxhj?oGwz)LsM1wO3A}0!T=Vj#+X04(IRz
zn&Nd0LO|KY6~a8YfE;&T0k(a?-ulARm;qX5{2N-fhtI6Rz9YiNA1DoTbcvZYFJO1B
zkpjeb=;1>UAb1g4uriNOZjwS?Ptm@l%>}V$)qgg8g{}g2C%>5^@@y*)h!A{cz2FHk
zL{1h?->-R3+<L<&H1+7pTSH6`(f!>K>5+bkLUN;`S@_g*bQ>8{jcP{k_)M%iY<-eH
zrDPD;lb^gS{zTo$Wz=3hnl~J3%Ed54WNpy30(mT3=shBgnt=6I_eE8R6;?rbd4WPz
zKqJLylSV@$@>sTDWj>buz9BR{c_AqHZQUx*1oql+u-;h|dErGGE}Bjh6hV)Rn4+ke
zSAjGdhWk&m+5WEV@W7T>cU-@&*uTOFxQm5apLq?XCn_o8b$e;6d#OW@u4+dd^Yy=t
zIDGG8wT-qq3>uAHRHCax_=7Q1RQaVGMPj}1nusXkF!Ay-SgIHa0Y?jgMoN+EVFMJ2
z^_+!yD+G${O|uG~omU7m99mlx`Q0w^6<B1bh>v`Hmb=a$y5t3c7(t#8V?=Fq=5Wo$
zD=&O&zH(j`QDnp<My@jf!M5Lye4#4XU663T&<Yhs{T%1cY-9>7FY4j3N^mU%ZYHms
zx}Di{S(Wocv>^Q(rybbGXZ+gbVJO51>OCTibb#g7iX{&7d4W66=Nc(S%Xt6#powXq
z6#_;X9)HRvor^EY+OyllT+ddYfwskdB~Od)Z0U_ap+?a2BF1Rymhj~-K9nboTJoFy
zEg1G=!idVuHF&0X7mF&6Do*^TdlxE<cC=LRJaK5Pd^S@dc1c2d2q=o5ViOj9p(C+F
z!-(CXO%waw3e?NCqp9Hu^&SyMO_jho*QLfZL7<2$&`2@*;c})91xAn+0!HFzJP%81
z*u0st44T#0>7LbY1rE^OlkXv_WUrz^ff3{hF-AJLiO_<gE%Z|vI5CH$R&2!=MX5Fx
z%4=?OdKt;XHyBBC3l&CNHyagrn0+&=bZ9tuyUd9ehu2+}v{CrIPS{Y1cWk1{z!2u3
zCJ?_B#V~*FP0zn=B8<X;dR6?3su6+#nxNi)TnhZ+^x=abL>#c`6<~m1gMg5GhWk{-
zQ{nys4z2Y3xRLJZHgFpn9w<>lGRE(6j)wNcA{e0k^Me>51(*CC;ihWpyRQdx#Ppfn
zz@$T(-=}-q^P^<l{tSnO?y+x11<>vBr;!dHd#m<Hf4_dW^X4sJx>4>=R25E-VWohF
zdy@WN9?SswJn@8zz#<r+{p&j-eAWOb&6Jm#6Nvyj8G%NsLWyD?oghF+256ua0zys0
z?X6P<juEt}V`nB_%)Y#J3{)X2^4%XPB|A}UL7RKl`R7Fp(YGsN`4-D0%x_)1)_HZr
z&jAxee<aKPh*EWI7&;)A8|dJTZqE9#kKcI3{VOfm>$^F3wB`rm@MS(+rVoO@#Tp-r
zUOh@}jy<&wbFM{G0v75$B8-}V-5)wjR3TvL>p#}d!q?YFfDpl}5NLyd5N-3&*?}g8
zp}2=xj&_X`=5f&Zj(GCr`^ePy55PbMuR@?7!~m^{(Kn}-h<5*U1Rv_n|GXO$KuQzQ
zR6k>+?!H~-pImPccmS0OIjtNRD}8(6WVj;b1sRjAkvKeK0K?T9SrHMo9k_#6JFF}*
zs}SXeYtT_ecohQsjtC!B;FI3GR22fM@$mwURE0KDAG!nrgg}KLEAzRK2h))ceQ#EG
z#c@qs%W5dl2R9J;29l@L$AkncL4**D%pgyQA)1&<Rdflxy6n*XBdgxLZRWrP(G{sv
z!TTxR(OxjB3k*!B`iTmnnn8oA>Kf1CEfwz-&xGx%!{KYk@2ao8d;L4JQqmW_Zc1s)
z5cNb`jG!2qLA^(WQ4?@0Io(oIYGGh#4m47XR^UD9&|m~wAz(C=n0;;s-{8kqN}5;v
z{loXpfpyS)*O6m%eK!M|W7E*{BE~4H<WzQZ`3BQU)qDoL`c`bGUYRmZg@wf4i_*~!
z-;;Q;6WxJDs^_)!h312*x~c*n`?g$E#o^0$GYCBCXXj=zq|u9e7Kwe$<GWz|T{IYh
z?}->A;0jK<MS~f989f0#^4+k1V|SW@2*H=po3}8Z3t8$mS`X$~R$U9IjPv<=?ie^f
zM7f5%{zu1$k`z#d;Kg{}3u1JP`{j4OH+c4z*7`W^?+1T+ObT@3UQd_T19ojO<4Wpx
z6rXrdDNyX*bm|RP=@Kq%O=_&ycZ$H_5040?N#vP?c5-i7PMNy546_2Uu6hkEm%*3O
zo9{UyYz%=Qwq>b74A`K}3p7&oVK{VD3PcA%5QD4`@Tq-vaptb%uw!5ghpOSA`_3My
z4E{=TeA2&jGK2Ua2x5>Y#0d4Z22GFfT3o(m@hI6ZbRq^5LaDRgWr8BtGt!^49G&=~
z7mo^|A)3P5X`^cgd#T%RUrcW|#o@>D+aE9<));2wU7orO*R=~XLOauR(bWEcdXEUB
zCSX7>zNq>jSnJ^72a1Z2Z)SXT^so{Nj36rnjA%EB%nm=RJ99^{a($$dR^#&&dB01S
zE>Zf?eI#Epd)Dz0)ukNZ^CHIR+pplv`q+A=J32!b)6Ha`V8SSW%LbFIL#OE0%0)}A
zH=IyJg^@caUA9&8@uHVQD-twBQr&TQflWN-59ujgjrU#>ru}kH95Y5M?sB3jqX)ew
zB8(J(+8+*!suBFW5HIjn$RV;<xV;TrFynxo7Xob%5K_HR#+Wy7b#vtGU|Y_<p*M-3
z5?vxc<XZUh^&3!#;0I)&8^rv0t(jD+uYURF)0snsZq2gtnE1J$F!>cHrkh3ATUWke
z9eXk=ex(0WJegg=wl3tw2i;7WwQ)E+bM&OuNozkB^Mki`J1u`QfSI4K^urgxBnUnS
z0Q-)Jo=#wzPIsyF%vGczi)1%ad{saRLO{<y8}syN%nD}w3}Wdl`f%B`QfGezSi_1^
zm^_XJhL265z~`SI#Q10)QVnurslz*Wey}U0<HU9xD0ls90q@-G{Z-@2!hb^Up*xdk
z%i0v=ukIP`{ap2X*l3ah4lftse*s_ns<Bhi?W)ZI7e>tZyz*ELok>K1&%eGS#s|3O
z?%bmIAc$eYz$_imNK>KhKW)oFfDoiYkQD+#ohO^}KlGYN_#2m13>6f__zV84<5|Q`
zzS3HLWjhy$5Q0<)@`M<o@^g&Kqasx@G)<K|ikUO@F=^1AnLS6PH{_IR8V#nM3>}z3
zr9n}Cqc@wIM=xp|5@@_B;=U1wf22merKdjaXSmVTX6rk)xtJk({y@$Q)F1@<4fP%o
zMmj(#*rcVVLO`8#UZ9a;WcWy&5(P$(6#_=$<#+h|s=gf3kNln+v$DZL!W_KP#fN-6
zpKEExjshdd6Jm^L=z7-mpRV3;m?=8HR5Cst6Gr*5v+0|z9?RQuvL?Y@q)rSKM#sKI
zzh=Ig#K~S~d5c}RB^ZZS+8d&OWT!4mA=Sg1ryg_*W5!4?Y19S<Mo{k&V+2eKtyy$h
z2tE}8W?BAYQmEqA(GKLP5NLydP}9b!{F)37A<GvPb`86DKdk@{2ekpG2uN1Q2AwXV
zNQFQ*i213`dNQP^cPikG(by*CkG3P2_{r+TO@1g~6L@Xm<Y=O)g>J(KUs__Mf!+9#
zwGIbxzkD_W=Q`52&kQW@PY}wJs)!%FLDP$wpSL;nbRa+Qu?*}xB6@~^Vg1fUV;KT^
z`1FLySLt`eAB1Y-!>2-k0fG$zJsdi0)>-4JDK9ULPBZ8<H3)z+LYMz*NU!r~8WbRS
zB?kQ<#%KGsvO-F^JNqi8_%^$Y^#)_YXYc+^8}8oMXAD<7Z~nK(6P^69UaT!z9?-gj
zo>yq>q;Y;L4zI%bd<AaieT1*95&dTdM^nuBWKX3kpup$fJ0ivh*fX(yiTH>L>j6K~
z2-d|c-2+B2czi$`1bmX7=VbgUe){A%t+CW$C*B9In!yd!*U00SQR{;`<oJMo5aToc
zi!<)L=igNquRNUPJ9!ve`Mlb{QaWnRI~4L}wfIxm=nYXtFON~1WAzm#Z6ihdt>az@
z&*1Rf*AkZbZ#QxAGe|iS%ir@7Gd?C`wWi4N0sD>!9|d3o_2DJr!wXz<Mt-LqZ^apD
z<gp91LBJ>PgOQ$oiDQ|XY~~-kKbb?V;I3C3aD^jD?4s|sfU+NW@d4c+rf25C%9svm
zSvFtZ-*)~DPflVIpEvSqvze0TH_PGqn1d>lAEKhiI$5rOkw)2XowpJ5aAfsiplTjn
z(eDh}hjE5Xk?|r&pYUR)=da0Q6x;8?z9XWC*1(H`X^Hgk^XLITQt{cPaKs6@_<%MD
z^bB!V>y17aaQZxyrCKbZ#Bc_>%+8Knd^CQ#^dT1?&<|pKybDVB+T{va4>dVQovnS_
zgb5$dho|IQuX}FhmT=2uG0$a3g^$~@tF;cM?v=|o-}ZQE-#>`Mf4DF9I^c}>?(MYc
z$;^dQI+*cseZ-Aoa}d~fMEC>)1z<%N#fKGk{*Q0&v_8;C0rEaCfTGOVf|dF7r@nnE
zXLfy2>SL!>BMAymrJ!YbadLzjGL2|Kgy4tq3!V@|^ojdr(}V7<yUy&4eq3{2+8+}{
zO&*3qRl;k1UmV|XWovdhx|)m1C9-(eiL|lK=C50t#-dl^@O8VjzbtR?r@vYGa`#7N
zc2&#}>9zzxcT2(R_d@RxVU!1qXq*=v#=~dOLOgoFj})S14OdnnS0T^_0ioJ)d!>=L
zv8%#m8>$1eMvqLN0r!_Rl5Y-buvlFNA_U(FpdZ8l^=tRhY&|A<hbEx9hS{#94--Ik
z&nlvEhc}HRCTibFWVf|Lr9$H7+A9OzbKK+XkZ7>q5TuL4=XNWtS$pMhV1!_0qT5f?
zSj+$&5ATHLf#3&VVBZnplK~9C=oVEWczpQz^nf2JJ`D}^50R%ppbY{(8Ezl{%1@j%
zYwOA5O>`}f;09MR4qhTF*O^+!+Ju}Q&<$dGJa!Ii+DISCGoRs@`qtRnfr%c`xRxPZ
zpR!^4`VA|*IluX%qNjYrsPad%onEi04rC{WcJTxEl6h#Ros!5Mi#bzYHm>RAftj8Q
zC;u!%o<M_rM?{Y*F#cMxC_M;b7!My%S%{n;XPd{8p!~p3MbBB7w?ZH&c=oMhmsVhd
z<JJqYNpxB1eb8xoRq_}%{IK^Ih!BDp26;jZ(I<(seAD|FM)vz=I;HKuiLJKBgvBPN
zV;w6~to#jmqHi5_MFmk#3vcBa+;)k?J8XMG`QA9=@H8fwC)Y6EU`Rgr*7IDF=VQze
z`K9$i*Sf&V_riN3!blvLijG@i3<GXE1Ae3;WU*J$5V;6}HV6m}Q3r82L<(z_q<t`J
z89b~K2ko*45;u}@*uh!l%OFAUeE_;a%+EW8ksayjrzt$NiicUlD!yY9q3Iao&%eu#
zO}_lD#r1pZ?aiq8d7%Ald}}a;no7PxX70MjlQ{f2#me)S6H4&a!tt@j0hZXW1q#_9
z_6@lRfqh3r&l*E7FP^1}5Rjnr0gaTOiw`XrP>2v@g+P$Kg-B`3d#!>2<!-&rY5b~5
zU>IxPPQIi{Eo_0N>@?&FF+}yJOq5>Rmt}u?Vc~PeNFo&zL`O}z%vYu9HK&K&=L@fS
zqKpb6>hDZh_REGg^D8&A+|dqRfy0Zw-=}c*W6D*V2fM|r?uOmNEJ7^j*djrQ;6K2*
zH$DHhi7;{pP99b-DnjtFj8_EM)B!Y7jCg(Mp&MP`W7&e0`E2OEa{8;>ERCQi#U3%^
zUBLt3gpj^CIYwu{G~ED|2tJlActVVk!|>OaQn8N*Pu`XLk+<Q}049u*7494}(5KOB
zFZF)K&`Hma3Zwgm`K%@RIxg?mmF=%y&25CkziW<3OEKMS@TEC;U1e6YF=mXe6^EQf
zj?qHz5n<E>)WC0A>UtQD-hZskJ8ZkU9MmEX*r@@)0Ko<Up<0RWqa#;77+vn<<mG5(
z{VWUCsJcu?j!==l`6mz|_-_gHgBT$F_a6%V+S@2rf1zNhH=~Zn1P~2(=F8Y;xlcNM
z)|BsU=s>s2m7#!*t7_empNci7z1<bJh2ij)+*{Xdn;m72S&`ZIZ7jbQGeAr`V^Qo4
z0{e~#AA2CV5ME+1;}r&qF#wHJg=TvvH-P{lP$9?)0imWkv+;qqxi)*O+#`!4I{&0X
z=buN&GwK=vp`#!|2vi92gcu?xeg-;gquN}qxN6Q7O&k7Vg6M#B>wAMzs$?du`i=)`
z4_2X4ArDJMc9j)(3tL%ChD~$teZ}Eh{SLGG-8=sK&Yzfmo^VI(cT1Hh{e|ki!he7Z
zZ<`1s9iRwi@1iP%Aeiw3n>&C;iqW5SatFbj8UaR-6#_<<e9q1<U7v~z@6|VXntQ0>
zA(&O00lS(=vTDZ%%e_%h1UW)XQHlJD@YlBsy4UM#NnObQCXR`s5XBAaKmHNh_%oIA
zPuw?FZB!J|8|B#eCHCwOS@xXuN6uRp9NtE#-X?kd<-TO9HpV0L@l2Sd$l|8hVHB|p
z>OCTgEP<LHCQB6~VBL)uXrvUm6dZVmLX03Q1d4*iZ#r32j_yc#QL?pO=tr&|m=d+K
zlE*W~RC#ET41pLyo)BYHo=L0Ep4(nsHy+<lWwjDpm8wkIQnL}IpWfT6%=1=`6c?ky
zXxl@*#~&+(-PuK-vcB7yJ&wck)m>RV%V)>ZtlMN>{9_v{W{en%_kBWv5!8D`7>NTl
z7v+{JMxuZi0gV)+b4Jv$XfOh;5HPZLy8r0Av+;eJCU40}U0vZJaDb*SO^%WJc$5PQ
zHG-ZOF-DEC`?y@h@dY~hqgvb6f5Ud6%SK%2YN<&_7Im5$zB9cmyisAqdb=*0-|_O`
z>+0clpNhN`;N)0Q^E>4dKGpYOn!73b^$uai$o_x<imUGC-t_$2Cc>x*h-p(x#fTqJ
zBcPFD<o@7VKMIT>EAy$5hxYgM;^!P+Cc}3#XNmi71NRHb{Kx(EcTf08puh<7gczfe
z(5SMjCuu_Y?6*D=6}^&)DX6V|X?L70UGLAQYw<g}Sd$!4VdVPD;p~9qN{{ojmA_sa
zX_VpcuWQR5dT^_}e|_wkYRjvt1DG+oAtVo7$bld=f_je#qh4Us$B88dH9i5Lk~Gjr
zF=~*o(L|2Xf)xTr9O91Mna-U9*Gm;Ay0%`tlK`!|+K^}1x=Eaj$T3>*gcu`5*OlMv
z4BhCjUNrB~%1#Nwgi(Q=qK`LkOBP3tN^_X!g$Ps_6+H0%b8~7;y)H?!cl~G;P)Imb
zt%j|@@H2nLbikUH0lV{<F}kIniK1%WLhlh_GzCl-wJthc1ix@5ETRYeNFmzQ@~aSp
zhy#9{|IY>iA=ULglSSt<W=npxt+1DpwQ>RxqLL=xfp2)DR~SSH{#yc{6)`}Ul0#NZ
z^$KqZ3}n3DZ<Brr6F^6WbsiLZoO=3m*Cw|^N0&*W0%*igLf+=Q?LI4!Z!2zUCE4Tf
zAyj2ysRLJLt@rIzQx0}IgBhTvaOD;-D}rBl1^bQ&pCMqJFTF$+;+@-+1T<2BJj;L0
zfB+#Fpg~p$2qmfZ7>4nAmIgn$AeuL7;V%jv;*j}|NoxaJmP`;K1gSOT2{A<F&y+tX
z+}<6=;JzX4;OzixN1_KlR|NW}sK4KF|9r3i+atNCAj(>%uztJuIqqS{$?$b|Pu#=d
zGlEKJlva7TTBMI(uRs398#6=&tL#^U5W#<d3vZhUBMxBggKtrc;7`*F>5&Jrmn$7^
zfn$95)AaKe2>6Kac<1OXIuc!dN+a%1%%OZvaOeGNKrczg_xlwnp{g+OAJM!M#N?du
z2oABZ;O0<hc_?UiKu{Z#{AgdZZqXFv&S@Iv;44+RE`g35M~=XUNqU;~T@zXE4ZXl}
z$HunU#}8vSwtP(fHj?@{-U2f@LL9QLAUW`d>F0Znh#Esce1sOIh7~sM@CpLO9e_qE
zKHjpFPf&;tWQ9Oa?`-Dpp!7nEOHU2NvqX~>?n2A<qU0%%5tE?}3h{wFAx20d)%ug?
zE`d*bH_x0Fk#-5hgpjUB?BVL+;{J!i{A(|zx*tM?(ChS%y1(LjnwLLpNZy&heK~L?
z2fZef{OB&`9uebX5>c(kFe4=2V|5Kg2!Z%Oy+?#mII#81e$f;N9wR;;;BHKykz!Qc
zNmUC95PS?<utLDdU5z{6+R0r`_R_1+pq~0ZJ5YpdeaZF@UA`S#h@7GYM~EpJ90@G4
zHHrH8T=Qr8Bem1m&LPT_?HP59Zg%qhn@#;_t1rr{$u<NtC*G|!Wa1T*c=F{KP?AC8
zv;GaKYMX?M(}6bU%znMXOi^zQ?Iz?DE%Y7{MdAQOrc0!VmmkQ4fJRDD!Z}G96k-Hf
zAy7nnK5tz1SIzBpcketI`SE~vD^ypAi#(YQP7Vw}j?u#NBF5<AcP5P*UyiE8Ym(aI
zjKkO}zA~9dn;zP2uS`)ac6igrATCs5)TqAVW?@GY&F^q${>kFN5*+^2gGv!0(~`Hi
zGu#}9&i%!HMOmZ|wJ&mv7J82eqh13qFRrD=G=5;=3}~bnt*U2XM}rY)g@930b9%FZ
zckk`nK2MW2?Ng8Jfnu5yIYv}tXQ3VW2oh@Oc@bkoIXND(Ta3-(0CON!M#mFdOfl^W
zQ%IyxP}cDPzJ!D=RxeJW!pNJxkAG@h#_q(|9xI-wn@w<dw-}QuX_NIFF-N}oRQj)X
z$E-$MKORJ(Mssg^{%sRsWNGT<<+N0c1c1Z!KqJMdGRfKlc}$zLFmHu`k?N4A>en)k
zm<%>~i?P}`W=SxneIP&5dWX*w+EI;wBFGV9ikLV3X|7RYX3Dag_PB6JuMHDL^i%7U
zc1&4&+a4dLtFyl(kBXx01$1$joX=|JHI7DUns|=k@IBV%sTW3yX^hps8NUc@YQs#?
z^!MMiD8vZrJtB&Ffop#mmr4;YkYEFilp?AExds$s1X&?aR1uc$F&lYr;Ax+GMvunZ
zTTQn?jFic5j#idvL=)2>Plz$<-n>icOYD{GZ#$|>pG7fX+e_4-`2DKsNJ;souUpu{
z)j0Z4VWb^>Woz}aCuX^Mt2Yds`l64+JH9C$o$4Nv_T04JJ=pDvI%bTbI7Xq16A>iX
zQ1208!~tMLvqX&efJv9RMv76*OD7cTuM1WP81eUrMmbDO^tqnQi2JN3nb=(e9&ybg
zzdF(39W$zHroiV#jL{Lt!}^Y|qIPV^VsUVeWjKgQjq0*GU8y3o)t~=4tg-d*!*i%G
zx>;UxF=FM7zTL-MXfDrW{>9;cPfnZ5p1#7--t+Zyhb2olW{f0XorjLt!(+7Yo`^8Y
z1GZOZEj7ay02a=GMv75*Vq7%}HG-@VFw*fTjj=A+eD&M=zEX$G>>bac^;bUf_194e
z3L_L5HslF0Miu&d)(`nLH(C^O6uh~5xfc^gRlASPB*~b{naYa!q?*eAL4{Gyg}-|a
z-?lIO%6R47#cvs#aQJqsJR9Nq_$SQM)w6r=pP0prQEOx@G+_;o(cGJ!f7?VDg#&lq
z1uwe(3cun3l(+tmTIOM1ZBZaZ9I!J_pbY{-3fesergmMgtn3}9oQ7J()uFXl3UY+<
zFGQo+rvL9*5d##}&wA$LiFbFTZ1sXYdk*?y0!Xwa$hLFT(RWLO1UJn^2|Ox*{C^c4
z>fRSJb0K*{8qHMF0USOqp)@gk{ql34lVk_3xIW*H8K5!S;`3lu1V32|z9S-h)|~b7
zqF8b;12%5+l4sU_VfVy9fDp_U0T$-15D>~^cyCa#CED6`vf`e>pr!XyP=(rQ$Pr>b
zUHT2V3e9<$_k<XtWg2_9$FpiJ&MFF=w`BS89uq{@Z(pmXdH?!%NJrok6Kj5Sho-eE
zacS&|w7a4LbUt-|sH?`|4Og-JjKGDdec@!*`}BBP3Nu8dVpsD(h~QOdq4$U|$^)t(
z++Aug;{m3NfJPe3ydLDC+1UtLnO7pC$`5N#r=71^*}gm{WcCX`h!N9OvPs8WPPH)P
z6wN*A{No~~$aOh&QEj~;x90)B6NO#Y(wHb>Hk@t0II+{`z@|e_*w*@_qmrWiVYKXu
z;f*{cDNn_-2G2&}@Hs2G8qTlm%h8wUJN;?!#&gUR^*?%mrkFX@dqfmf0GT!Y62*v@
z4|pv=Bc<rDg_t0UVLoJqK#{wCVM0yFbU@Ph!7rVO(OQ|{`0N8vH=AVr>>kA<Arurr
zju2Dy`Ay{Cdkq;rj6-s<mj*bSF;V1fWL9k536!E>6#cUDBfT#wii~?B8-iHcQfk<E
zsGnKn0_PTGJ@a-2dCU75%c^cx-%+v|Gez#ZJv)(W(Zb6nqR0{mX;e$52$+T9BcF40
zwH-xM5+ATIZ-qe7RJ!d8o5tu_`HJV7)Jmrx@nnJ3ge}Oc3HN_7V?~1z=m{}Kd9I?p
ze%HIHzkPfu{6@_R+fr;+7KNg~_hZbjk{xDpREMomVWfFNHHPo*rklncPhanRT;++w
zd-`zhIAB<@ilXMv72D|**vrVMNxfNz1|zWdh%ib5t_jj!6eIXWe11LhQ)iJ*{cpjn
z2!0WN-U0!ip)t)={^0Jcw{$@%bgd_hb-#ea`}gF_u-#i4G(ddd6T^8gh|x*<v9DcP
z==6K<ZLQW?6f0*iDNx_mXDhT`K9}CYs_^5h6^$P%bf!XI>l$jD;3(K-_BrR-jrG6<
zWgU6i`=yjSingf*@;x&6idlgUk55D8&)`?_=X;I_8$;m4#FBUC^8wfY0*y3=(dOSs
zLleV5D+GLWY~6Hcd?;mkm1yefJRD?Yz)UFW7Fj05v*$=VNDqRkYUpth6BHE9_gJWn
z*YD2|Wz>pY%VjVT#Br>IK|@s1qD#%QI!LQzFDin%r(XQ5qc&~9X$EooJ8;Y3@cDx`
zGt+E-9I0sZKa%q{2m7&Chx7szXYc1;^!&>vqR0~14tr!#@nMA>k@E@wcjp0(l%hxN
zEe}8ef*+PGSRqgp?zA`m>?fbAym$8`o#=iqr2)+`dXkIKL5~}@C@_Ma7coZZQd{lL
z^L)|iw^dp%p3Z$A6GpschbrSKj+C-Dncps;eprPHqvDNK^d|fEZM%}stu8JQWrM@N
z%rfZ^Ih~_-?><#IeO=dD%tF*r>WJoy2h@8+81(`h(WI7&5ic<71T<2N=q)2(pb#U-
z3IQWSmET)8-K}!EPkYXx!lYuj6ild<fm10Y2{q-fnd2xZf*c{HXk9MPAC8~5i`d=$
zUtQFFg>7&Cn)NhV&k{Epu4H2KNsqsYj-s%ZYjl5$64LoDh87wgJjo2yBHARe^Y*v(
znxVXhR3xPC`eCLhHCG@7MJ5FG9uY+xKqho@sT2tTXYheWO3^85hj<iX1X&?al%y`z
z8D)1&<<a-AxF;(#rYpdZrq4@0M`YL7$$|nS$P;3WIzELke|J+9lF;Jtq>2!t!X!rP
zbstVp4E##9J9l${dL!p0RAQ9#sa_~$t5$%>CesRPmY2YNcdmAmPZAy$O8(6$chh5t
zOvH@Q!E5%3C@_M0j|d|jV5ZuC(V1%an8wE|3QXORXGY;4MW&Hsv|xpRQ67)1vG?1C
zCTfqEvRUcrV9zEnvCbkNmsv>+`y$6^!4qPPwyaIi3A@7ZYl5$Z`jh;dhnO(ROMNg}
zNa1^phRHZSA$tqDvP9=`7a67tuHI>u$*p_R@DAADDEn8ytwG;`J!-Exev++H7BfbC
z+!@doStL`9;M*p`C=a+qI%`pk;KyYmdgK#CHs&RdL4@GPW%CvY_%u21;cV!8TjS4v
z?yXeP=gesru&kaJego<0X<lcOdJrG@aoM~V#OV0Scqr$X?TFaXy~i=)hYLF<bi@Z@
zIoL<*1gnRRl(q|8>PMwO%<4DfZ|>GS=2+yVGHz(h0#t@wyY}JXj25}qQ@d%lr?q3h
zfBb2+;&D)c;Kyb2Jx7F1=UFcb>P3SX{5lyho%0`aLOJTUL%>9c12!uMZ4l7ueMO&F
z5&fJyUTQR_-uQ;-B$x*6DkDpS_J3o31d@Z`)A{$Ph}qc_eXeNyCu?Kcj-4Dn`%A4c
zu~W(+-aIvR{+ngcadWkS-RPF<zcbw4?#n-&Xfye~*<<5wV1;AfFUytIMH7E=cbPuU
zUsg(uS#~z7&^`y*f&c2jS42clu#p#q{1WMz+e<>eaY$*ydo@sY;OPNv5a<ccldfj=
zDjLa)zBW_(m9I+=JQQ^KKUP`of6$2`#|QL-7#}vp!Ox$<LW2d@nD(A$@n^<_&pj&l
zxCgjWJ$jnV(BB$6TTw|*gVOT7j1+JF$gg^7Dr-D#fy2L9?a5_Yy2{3AzwEK#V^Y{l
zf3zLm=Y$*|u<wZQp#{VTw<tbHQXhUG^&!uoFR=gi0s(>_&d*txw?aTj=izSJgwrh^
zGt<rGeFK}Uc0eT?*vT_!l}h@25Fz+kp#@KfAv*f>zVy0nYh{XbtsY;K2r0t^QOjU_
z;0|_{>isQMPl9~ujZs0={HSXmUfY`$PdgPOTdUZE!|T2FIJv#CmcOZKI~T{7Br41r
zlvI%i#V-W<4ZSBKjPiiNvBsjQ4-$-k$iPd!vhgKbv=fYCNH7Ae5HK=y%lT3#pr9gh
zQ!-j<ysYX9SU)BksJlfnoG<InI*5iM&=F#aVtnzN{)(qee8FdDFLzCRfr%mk85cUn
zva+ia$E4ilZQh`}+a+IO!{!q^Q!X_=uP*G=zQBRQEBmeuN_8Dv9mbvO_ps*-8)k|!
zyQNSZ5Sn|@^Dmp2BH*1gERiB!;Be4fBOT7)J^Jh(axq%4LZGPgsbnm7s#DsxnZ7X&
zdhR1*U_CS$DsnNpWEFT1IYtYf5M$&nW7}V6eoU{&Z`Z1WDNkH5Vf1?ik8+Bj=Nh^V
zkp&bRYZ*|9(Q(OlT#EF9XGU66_0D$(4&(5VDy#9@5})@3`S-s#)G^GA8KdBit{5<)
z!HdyC?-5~C0W7nwT{Na4NT`8=4df}2PeWrZ2oVQ-LjBJM0iiVoLp<VjTKlvdnMzxG
zHN>jGjc1oPkt1~4g4G>F2)+}52PHf!Vt}|gLQk>T4X@L-dni11Y0?1`KsN)fJcy-_
z<9o#4L3i~11}{_q#TCZ}^WPp?m(Epwiq1a?D5>XNm3f$vOV>HH>$?38++OV4@t1d~
zLEG^WWYquqjtHM%11}1;Me%`;WxT+}O>>P@g+>iFLOU1XW7&cg0z&Rns@~P(x+b4`
zizehR1mc3ANyT;K2)Vf&5&;o{Pl*;hA%<wX!17!-IevD{Rdn+B4{g{~C?wK3?~6lb
z=8%L;r=ah%I#dviRz@WT4I2-eUAMGxoQMLh3M%_tx&QHUPhR=6+AFjBLNhR{P{VZ2
z3<?#3-Zl|NI>0#_%SAVy!K;vvpdRoeh3Lq0R_H_ryb6If2ndDG*gpAv{ZZ&us(=j5
zj1=8S@UBEEIr770x+8ZgL4@E{2=s#(prlxzWyQ}F6(y%n)!gtF!*=&LyUQfCw|}B|
z{;QYPG4xwXPytl$%hh}L)8X*!w`M$R+7Dd9;itr{PpBlUwyBy`nz^aG8T$!E#s2$H
zHUzIiVBZnpV-Jk$<rY;Tczl2weZY?tp8~Omv0y$8j}K^rfKO6Vgpp@dY~g8xYJrBK
zNmCY3feuEHt<={{yt{*(9?%V9dX%edD8F*2Ts)lUy*0AyQ9q^_rc<@^SVMDn8EfpZ
zN%oU`yHL?n!LjVv>tfyd_f?^1D}S90!r=`*a_Aao%V%Xz9-h&03&4IB$ceWPDq96l
z57>7^^aPuFQMfKj4}uuR2gER-k@6!caxDf-f)K<o$jW>SGZE7((3{cT<^1Mx#LA$E
zMDW^Zna$)eY_!~XABYfw7zTMl4AGiehQU=j+#`(*0xh|XvP_sD8VxoqFU-5<t73R-
z`1~0?bhjnGeh^9ZDKf6!JeFR5WnwFE=kK=K=TUE4?(DRpqj6dBmR1mR3=^sTJBcEO
zLA^(WQ4(-Sntf4>;6(^1&G#R57+Is;;*pCGXoG-|y~~K?GhyjnrorYaSB`aMae!0P
z8-Vo}k|DiW*a%cT4?cu}ZV>a+$b85z?RMG=d!;`spP7CW!o&~L+nIg04y`Dyy;RRB
zcjNvQR3a4EbD+Do;)2S{xaYk}&XRj@_~-J+znVufoA174UY*gnuNgBx^m2|U<`%)e
zBcjJ2C<kk{C_V7&VZwUk`#Tb>q@n8g@atjo76|0%G%Jhr&Du-XdhOdT-mgVp0UoWV
z`j4aaR)z+Yp!mRR@Vpnq;IN!f;I&$QDsH(<)~WdDiF261p-jRXe(dTkcH<A|yH=HZ
z4;46>jsp`rXJiUbj`+D#sa{jV;RTg9s2tFuJ1dycyhWiR_atU;M02k~M-buH!{&RA
z2pdBnb}fFS9$s_=`SgGvY2-@njfWx^eC7k%AfS`Seep!@kITpKMM7!9Nds}L(2B+?
z^0>8TCCvzk4!r7seh|a6SMkL~p&xsA^^<#NTv*1i)enw{@@zOehAVYCt?f;<<6|-^
zc>Le>hOk?0JpD${#JA|A2rUk8wK~=!JK5zFL#%pz!qtaknBjS`bMreeYQcYYVBZnp
z!w=k0M!Q7S5!EB#$dMc!@e)jX;8kbd!aO>kb@sEXUaZl!?M;h!UvK#oM2GS}%DwJ8
z#{tbj!~gm7UJ!$G!cnCs|LsGmv+Ir@>}Pwt4U_7Ks=oWMyYc1m(@u;V!hC7bO&s=k
zuJ#<u)XdVOziX$oo&Gxxe{Cn#T0T3q_yd-D+`>P8J7Wgtk!}zd2oAjJ%=a7-Hl4tD
zYw<F#@Tvn82myYis-v*}Lm={02ed&zr&D;dG3`57vv9**pW^#7{-}ZbH(2=(knZ0Q
zX|owXE;^ta#Oy5FM>}xs5I^^sgk$$y8F;)fu~X2f;>?4;bF0)}E!?_M6<zInGiwRl
zara6Ux8cL<wG)2<`=Wa^k~_61Qk6GbK73Xi^7|-ec5K=+p-E@>uMX@xB6?JTz4gY6
ziVpmEix03n*GT#4a1?@WriUMIEm)aPdBQCdKWVh`2u>d|Y`tF>R|_qp3X;dI_;lK8
z5Fz;S)`BO*5J{MH)>WTjNQ}+6sK|G22e!ki)%rbd`?7m9o0qK~&Qp!@L>0Msj&j*N
zs#g4|Gr%8iecZJQhmXEtWI5s;<mJwuC1vFiZ-W`4WKk0oCwmrpj|d}oU<C7Y(FHJg
z5fT*C1Ae3s^-ZcqgV75?>hsSA0U?FYCADose`|NVWJo(PaOiJ}HI2d9vuzX|RQBX^
zhF1Fbq4<R$_4(%qF+fEcGzSi53$%Iq@~?S)st+4Le<b4k_MYUHbWeZkdF`F~2~+?@
zb=}wFiO8nRcG1eiN6Ap(@SBEhSIN#E=_<7PAZj+-kH-v9T=Gd9FoMCW5cr;n@ZkqC
zXZt0p5U&7mN)>3N0A1b~a03JgUWFE{%*V0XhQ=JG-$yEwV|%xJy(-cIra(oT$PqI5
z%FcmYg%&&^hRE;rudNTJSlKjh(Wi<xMPI-a%F2IkOgiG}U0QfZ{?Mvj?|V@}G^L-{
zvd-2-^zXrqQ;IZy9Du?d6JjzMo2@3sDO_5klRsX<3{l3p`)nXY@E_np?-5}X4wQCM
zU3BFFK``S3>U08)6r+^5PwvPunzJx(g@DmiQ-8!WA6|0{^UR6T%``h(z%|q&N^*<_
z89MeL$7sP5VvMFj1DmEkoAs+sEpt!a$SH{lBa3XWnA0V<V%?JWPg~y6r$L30;o0w|
zCQ16WZI`?`*z}?#ad^DT0dHPi9lKkMx|d3*kN9K8NRGMzMXl+D-Xp@u9XNr$eu)_I
ziUMi`G*XNbgIvRqW3*s}fDwPrImugH>50FLKi^)JwK{ebQX^h+jQsLf7b3@K!4qPP
zxO85rIBnItRZ#ob{m@v`A50h>QkE^g@aL()E;`NaJ=Phes4!Zu^!4PB$+jzEx6<#n
zyq+os>b)lZ+;VQM?wK6^^7le}<r*+!wBGqFR0Rfs8bNQH2qSTz5P0cQF#;~|<OLcj
zMjx5D)`3Ga1VIh5Lcplv$NicX<&sD3hAyA=x)s}Rfm_SW47^AOXTBw?qES!;IYLYk
zLzkof+tsX_1I)9MrJB^)Fi|ARB`7cbE#g~V{lO`tVJ=Hl6lH94*b!qdP<)b=qWk0L
zinBQU$n;~rJTc*(J=I=<?|bvtV5Z0zH@+7|OoMulh$2g1C1dZRsS$i;EecE;kk1QM
ztBL;vrHF%qg2G7OL7&^m*hJsa%7NS30oa$%qi3sc=U~sRZ)G)SfMA0_Q18}bTkf5D
zIr~0WQqY3iN-PUn4Ev9IW@AF-CqRVYKPAu)Vt|~&6}KO>;w@RP+~oh#Zw0m;uAB#c
zdHSqb-A4Dx>fQ$b_(W6yJqum?BfL9oYNG<Tu#71iaC5@d&)+1P10U_4yj_wuxyj-J
zW`JDk)gOZZ!DrTB-x1-X04!w~Es76(O2i9*FxN;$$l&<COCUf9L<q7%Kq#F4Vci7J
z*?xf`g<AiJb6;O2f|WaBdC8`Tq5?aAf(XHn?-!mIF+@$Sau;zyEJrJ^pF4ZWgtHiv
z3gzJ<n)^*Twz2Mazstr%!;A`|zi;j}3|(t~czZIq{_Df7KXLfYi5sF^wDJ?Y%5#R<
znXX|!ubOgv=PwW<_z!TQ_lPhG2Vz<65@Q)3KOZnnM83G;H~jMn@>sTDg@94h&%M))
zsc*6q#Am;RG#=*|0%Ms=A=#Shefq>5$SGQIgqWi0Yw?$hZcxQP9#M-6iF>;S6GacM
z<g#jn@Fo`5RIg<<c`b#CqLQ6i)YiP*qU&&v-|*%12Lh||N^LKcLuTBzA39FowXeYz
zvlP9&&V=H)>O$`kQREK16YfPRf{$fD>H7bufA95C?JCF+2W%_@Z4d}j=-i+FLu*r{
z#L%byb49WZn$XsuK(ehtpPX3DL4x4FB+w0Fe$*V^h4ysv7Q1Y?t($(zo(B^@*A>)-
zr{CTTF!dNIY<ZB{go+>0OB-E355HiiKCva=P-#RNhi}tcF|_=p&*_TPLgr`rA##}c
zVc=VP7gQnmPzLrL5k0EFWiEn?hBA11fO4O}k2H{#{=9Mv3}o>1fHny9(B5mjv!}g&
z)ZQt8v8&FxS_s@=e_4urn)-&DzW{Q4KtG7_sr|VuO4V`O#4;IrmXPN=OEBSMLLcDf
zPg`OsE5y9_aKN1xsPGYcrX^eTH2b-MK}*F%-<TE-KW==d<5ApTp3;wv%TfcP`7q=2
zbDzj5<oJMnM}$u;FvsY*C_V@>Az|RUPN0zj^quY54^Vy(WI~V?0z#@cT+WwY$@I7U
zoBOW*Wx3==a9hx``(y$%MiV{?5`-WTf*c`c=s|hKBl+)r{gqaH0aW_aH!(4E>VSf@
z{>S{?^3zxUc#c`Np<<}sgzH?-XW<pa&a2eEJpLYo!#|0+^Ct9epX5x$_T$zv64<Xz
z$Xn-wVs2>eMbE!%B8qf?g^r{}7djA7#0%_C1R5ztZaS7iC@6xg5Gd-Mtlk*+yK7*i
zZR_7#uCu-M;Cfk<7a2u;o;T%DPy{(bOp$U(8`tZcDSWob=@a`Obz{3zGg|Q7$<#)t
z)@wFd6X)!b#!ykjeyhnxg~vI{r~a&t7VWYI9Da?VqqQn^9eYfkvfK;BC>6{~)NCFY
z2`UkM5-|6o=U+AvMI69EpJj_`5qu~U0nX!-&o2hQ<8c5Pf)8b&4FW-yp(#d<QKui9
z=VzAqTF&h81ygB1;HWLh_QVtHzvw`M;QIh{gP5O04=dl}KYJ6@r8mZOzZk2>q(adv
zMmY~S|N6X%$=B@4+A1?t{AgSpJkJteR@7Xv=a+BQbw(UMM(f<xB#E07z7_dJJ&&(f
zVCHAtFkcPG4}2m7_8k#D{s2ArMd^XpAU=NJ&@#|SHOTy?5Cg~$d?Z`2LLg|W@KkyA
z@2ovJ_1s@Z4^dB*fCRaa*Iq~s5s?B3f{$biju106YT0aBG4VBTZEtcB3#V!TCWh|W
zp4!Xvf&YS9L*BF)_lfVQ7&^nr=_9_mXXm7p`$hTGG+>MCcz?7r|MEZ&zk2oJH5aED
zF*B4-*9mPlga7^(dXI=Aci^3HFB-|<H3+yY5BQNXv{7eeD)Mw1v_T+9XZlXo^yl)G
zymUdWftvZXt_I-7Ax$2#S)t_Vydn@G_)Y-*AO@&a(^05v|F-E*1sVbScAg%=1kjlS
zkrU=m`7P4fTesP*v-^Sypw0E_c}jAUesL*ly?wl!>VYNuzX7`y@+`zEaVDHLPMK4f
z0pk4}CW2gqz`i5GCj$ryYZetD1o(*P36h5}1Ciw4DDe4bgMd$ln5)O&RjLWflP9#C
zBGrU_!FlP+|8Yp8vrTU`3Vi<gL5vU6K)l6~Uv0{w*Ze7Ew_b6@gb&Bg&}uHOBezbL
z?R?IqUUn4~KJvY7PCFA1<A#a^*A@G?S>o`Q2PV^5*`$5yeXpMElZe57uQ>M<19Z#~
zK867=ned*8@iFkC;8-F)z=c%6k5qjcE!hqr#|N}Qz=!`Y|KM1{bX>JtPo&t?)A--e
zt>Wv**H;4y^A01&2lRs&AK%dAm!<-*_B+d46funWpT>kwiqfG0-1JRD%e?%QFQ!xR
zsPM^95xBWm;x`=+W7n^$tefmOeBhe*UIK6K{}_Lo9$}I*gT3U!l~2o2ENg&$M}$u;
zpg!vs#RonG;uQeuCjyNWAh|&tiuvdTD+Gi(*PKvw-~Bl4g-V2HfttI%1{lAF`N`r}
z_$qcZgL&w25i`VR8u9q}HM2KAZsk4ItO>z(%EwYT$6M>Jqdfy-`YyrG%p9l~%1jxq
zk$rep*VZq^LU#X-1fZ5{XmNd3dg5d2<iwCDSLTnHBUqn$7F2)=KKd`bCL)S@O})G>
zER`Z2U~vp+q!iIVKG}H`n2A0GG%=tpzPdVRVcv?pv7NcTmARX~gSpK)E(?2`b4lvQ
zs;b#4D>>BDucxY)SG`wINk3^Puc)xUl1=5KO1Z*&#TBd7yuxB~2H7^OdSpQ9cR=NW
z=uJaSx}+y+q1tK>?nSN)NnAlPf@5lJ8pZ+7HP<9iJfF3(gZ|v#qUU66XFvCd4lWKn
z$_kXrH&HCZ<0-hsXEj-UT#G>kh95&L{ON=b$X@DOy2Ioa=RGOkI4&wBuqc`R+qCKX
zA3N?cGm9Bk0}nsfWKsMP|5my63~j7!?Tqc`{tn>(M#hG7e~paNVObg0<JtmjbAQ%q
z;LoBU{8=RvjoP5#!;e@N{w&0wt3&9RHj~6=1{HM;hT=PGuVIpW1}<K^iYKpHr)ClY
zdeh%pp!!^yTDF)@Uyi=6UtSe3E60@%6#SC-c74TBlUVvKF~&zEM7Lu8TsN^;K=Zc<
zK3C{_Ao^StfV)gKL7yu~+Yx6z-8stUj!wWIpICS;E?kDzc9wkN0|WW_0dfN_*$1Pv
zOdsF)yWEwEii&dAr^dutjlx}9-|HK-ojpsfZ_sA4oa7VA8fd-@enRj`(1K?}8^?1F
zbENV9|96AmVbux#C1|?H(8kKq`keioMTLJ#akPTFTyeh&s{$u`)%)s<lRV$})Q<YT
z(>lLLjfK_a$DOcyw(}=WH&Iah!yD*6=pTaMDGHR{0~)F9y~*!aM)B=KRtVp|`+KHy
zJEx~day;4OawV$8bZx+Ie>Zt0bj6*8?I=D+$P?nv@knH0W_(1CvrOSK7rOWOI!vD<
zv(@HuvDNx%>TB<%GJe0X4%O#)roOhgyj%NQ_4Iog#^*`nIDGx;j=eo4<q})>cqzI`
z_5Q{DIc_}l9ZmFvdXMOH6bE+O$}PH@3y%>Wa97V<BgJT?3M+JG4L*b{SRr7f+9h2V
zmwVJF$u>{!%cozQ9N<)V`+ppHG{4LYt;xZw`hq9K7`gZqX4wAidiSI7R%fs4p>|9d
z*>9Ho9mT(mO5u%PhJ)CiZ>TWRxm@G^uvc66$nuGjkL!*AbD5h5e%o8L6jVR^fQ$H)
zl2(HmBeCPj+JJsg07ZEz;3aFJ_lPiZ2QX4vB1XKTz$81+NHOZ7^Uy=1MxYe}Mj7k$
zEdLD4KJ1uf3*DZgC#V5VCKTbwM}uy;E<R|~2=s&)qixbFbh&4@=cVP>g-Jc#{23EQ
zr{7atf5Az$CnBftfx>|w=TTv_$?S<~*nV%R?LI5Z0xDm$;_%WgTC<{BTv58u_@#aK
zQDQ#=X|*-t8*+>m-Zl|N;=rcum5X8oKOZG3pa=X&A=-u$rvj(MIRMTV&*VWH1caJu
z{(ji~PO&m!x<JS6Sats}I3ex_>}4RyCdP3VX&^!HpAzT>F+Y7=PExI%4qa4&hqKZJ
zwXr3ap)Ch*xy4^Ur{DhYZknKBGb(<V?Yp%a-yNF$%eXpy*!MC$4j-}Q2d7v%&$j^6
zK)Jz5XY5OI=R)2e2Kj-{#K68IqQ@V&)k6eIE&nSeLO>6n-hW7tfcNP?padbH=bsG%
zJrz>78RKHI!mXdY=zghi(Nq-NTOzZTd~eB+G~NpZKL7k6#wTl>S@zed`Uo}lJEMla
z>#<#J7S<CgEG8w+G)U`tt1nJe2o*j|o9=%w5%y$zF}h7=S+zwP4&U`tET^oAUHSaA
zBkh-+W*IO?G20jN(CtD9@cGww#P|RslWmLQ179!%I8jI@J^ZKX1VMP<kM7S~AfS^M
zmi%?Z@>SoxXS6l;vV7`sVC<sgAV-H+<`EN!4*Z`#?*%b9793TaQQOU^752RA9-*P%
zjwx=H7;#k=4d}iumKyyY%ar&G6*!Hws+zAm#q72{Jru(+PICe%mbPtUU}!t8uBs{f
zbxtup4l_9W!j?goO~a2p=X;I_8$)9+3iU;?fzP0M1%Qk8fJPd%Xs1K(gXq9#&<j=w
z_!t_@_VU#CwP!7hl~2oCJ(>xfS369e`n*ZVi2?C}&!87PAx3DKpHpb8*JC5Y9<4xH
zN0&R85IQOLB9pd7%TRqfOV{ky5W2hZ!tV;x|0+wrC4v|7R+W^$fWuRGANMtoj`FUj
zV5DAIN0EdXp~9EDp_()BVSl0bh%gF&M}ebVbR`!7MnXVF3N%uT4)WVV3WNY7$O-`?
z9Ze7S8Xr0O61n8nc2ieOFM{hSzOv-wc^lERwJ0!xJR!zN`~021=hbM9Y!3O*ww1U4
z#)OgNZ@+R%+ChgAcjg(<4G!p5R3_w;jvcJye46))<!gF$dJYcnqNp7{VZtdjA;Pbt
zI(b<aGe#%2rJ>jmF!!eC-!>6OI>3d~{!7fDf#TTzQ3&0D<y-}*MI5jhG-!i>kT`Ge
zvg#rJF3!8(qUC)1o_(zcFA|z2M<{yBjm;oJ2)?C%&x#nJpsS|#DG4GyM~-$K5+69D
zfeE0@Jia@k$@WE`Lk%3vlUF`R70SxfIP;545?_z-RE*~)r|!n#m%mr@KQ+erbt3py
zM|(}icFX`3Nu7soEQe15z;{H1PbaXycFBt=e87?%&`4FNJ2n+fFoUcR5DFJLe|=cW
zsjE9E;$p(s11;-15TSZ<gvuw9ltF|L1T)AJVu-w6?_K#~75%j}WwWCDOe2*sL6p!o
zl@PdFP^;>eBI7HSi>FXQ#QnV@M@Vv`2jlqfxAY>4+`yG)@xOVWeHk*Bw(4(wv}LRh
zGep9=5@Fyt48aEo^&SyMNx*?Mi>2m7Jiz^mKqJLy#7iR*)FA}H46;JNXsBoM`Pmk2
zmG9qkY3}waJ>v&cY8fH&lzMN&F-{a1L7os}w2|Vps<rqstEAttzjpd0VcVyD=7vFZ
z^rh3D<5z1cDesjEqrxam_Kqe?<zu0-iP)Lcjw_i!L7I%8V&mMop?X7qQxYhqv6s^K
zR(UkH=!S4saqdmezilFnDgcbE7R3mDK+neu-1I;OQTQoFEf6C3U<TSCAmskwnd7NX
z_4F^k+0X=yy@+gul;|<J5^cS~7yu#!|1E)j5Ce2yy8NDugm_EwD{7@gN+m-~I`n0E
zi~aL^j*la^i3$jvK3a_mpb+EwsLo^eygmCXcfO4kxQWB7lsru`=3rGWWNG4()bUTn
ztV6f&MxdAv0{e~#A6j5!rm{qQfa3$ek5q%U<IX@eK;iKLZ4mI$vEdUx^~^ew^2x}S
zR!>@W2{4_eawZ>@t*|hIE>(ob2lRs&pKQ7@&SKZwrN>`xt9n;l+l2|AgO7CM-(+sF
zE*|E3e&A!ra#Z+iwm8#~T4ihClrHCYxh--v4lgPm+fo-(%XLV4=Bbi39rh~LTsMs9
zLH&Wxg}}Ze!p9!Sg`5|~2Yy7x#}AZ{0U9Yl^5#_gK>dLaVhdIX2sNE4a2&fu8TzXr
zbkK5W=APpV@YJ;~?*Y;o=~I`Rj(`dTp9?K`LJX1RO_ANJayGt7{Db!oDk;HXg6P*S
zFL`eH9p(jBsr(KPH;15tC{<y%=JDRMn|j0!d!_DeZ3BwQ##pChm>!T1q-WkeN;{6d
ziuIHDXs9R_{0F$udqfzyzXhtQEjl7YfDsQ6#ehbNQEMEfB?^onEAtrTCJkk_)qmf=
z{<4<tt7jeUU}ZJmVDdnw6`7EW0wc&1VvN52wv=@~?0HJ|;Lm5POT9i|!boOyO<-SK
z?aFTdM)^S&+Y6{LGGiQ?VC&8;ALLYww`X~8jKiOhKD+s*AoW_G=W8?$742PxIh5U7
zmbe$xA_N#gy+?$RIIz=y-xBAC_<-GqbBz?EEW?pU;NlnpHG-@VFtQA08@ONlESc+D
z$`6OJ;AI!UlaIgek)2r!<u39=K@sE#F-0Ll>Tf!OIhF<bPq=BQo$JO#QKRN@%J7Cr
zj^nCYuUR&pK_^D7tB<J$+w3ps${KzrEG_|D3-gyzLx{n{Zu#afiRt^Pw+mpVs55=7
zFbayG-Xo%D3b<=Ja)}i20;ieg8Yx9A<A;Kfi_wA=0!4YQlVjF8PpbKu8@@SDyA2h_
zfiZ2A{LILf`nW{o7%g~0jFH5i-0C~=XA6A%j!rFKVd94gBW5`-U3JMAN)_+0R1JJ>
zBPua!QTvd4IxIVNr;6wgzJih;z=gpxr`w;4HH`?I(un`Era%)jMg~KBOp#->(0fD}
zH37E^xi1>i;0I<R{CdET6r!mHhb^EKalp<pf;I>U*>i>WIh(xVjx0-Y=)W4ey&V*y
z4FL2c>-2Z{+MGdx-~}0UgP5Op*QRZ@Mr4?K2$WGY2feDn#LpEk`uvYkO-wDTdqX2u
z<(Q!2=U3^UXbZXMl<O>yUI?e9Zo%QVFnF3uRyCx{cKtP1RJd~rGe19yE`I{~flsW#
zz9XV%4KS=fyVS&*2MA?ABju-qbIW>=9|WNcvNE3$jop@1d8=@&Fu*cO>A>%jU*Jv;
z5%Rk|POLs=3lfANuZA2UX6UDQki*oZ%=dj7UhAGHux`P`kkY$NF}k8=(p+>BkJ@@V
zs8BI<N9a#hPkMk*heZr?*RWD74lgY1npQ#0uQr&?B~!ZQ^)zONuI}RQ1Q~+={^nlv
z{L3byNE~=4aZ3$lz|f5MKZa-iI35w?6wO(fw?d%EpZZ;2hwJ(Jubi)s#s5vE)&WnC
zJm)1Fodwc}G$E&G!4YDLd<2z$o(aPxxe4W#M7Xu3V4}$9B<_Yxpl#B%5N*oB`}gco
zQFOgYo%QIA#`e*%)o#Ho5x^AV=ho(Nxw}JamUnZlO3&*w!A#LMnaj`}G4K>E^d1pK
zmcX7Np`~gOQ0ImZXrx-i)8>5^IYo08=B*GYnqq#DZLF9t`<m%x<-P5$KhnTx_WK4|
zTCJtbuZ5hV1xJV}8Zf#VqOoi1#?IM@hk_R=^)XSjqGR*pyVurSeU~30V%7DV7ZpX>
zQkJUH?x{G&KITlSy^lq3c=MSo#aV+AmhIZLY2Be?t(Yll>1mZlp+(SZBBIE>lma)f
z#C%9dR1f%(GW438%M?tiIbic4&<24Zchd`9*+Mc|JFosOm3c5=ss!CPq(MH@nB13%
zVx|%FgBYOoAH>XmS}Tbvt?7Dc@g=+!6F}W^D+He9?%JvCEo^qb_W?Q?T9w=zdfi6K
zsgPy)*bn_a;21Xjstvy-DO?$DPT0I1)Vqj%)s-i?{WAy<d_n~F9T7hEz(lpi5=Dqt
z5LiF~8Yw`%xjomBi_n4<0zyM-Rem}XA`J#3Q9A?+Y^k4s2~k)DSxU{qXrhE%gcckj
zW=L`B<Ov1}N;XfU{Z+drAN<6`&^6rYh7S%GwAd+Pz2A*7{X!)|d;%A5@FlaQuGX*B
zJvE!8j>D%i9DK7g;K+?@eya9A8MMVPi_j7G)LxJw1fdLi*+dkD1MkFs(S!(oP$nd#
z2mDAGN?Coa3uFj^2>r7`Ac*5$mR!1ZuSUTM#Z!@vToYa3F^<A(WMS;n$B)J!LGXP5
zJ}P2<LMpcAHCTyXnYyKMAw}iWGEDqft51maOFrp;a{g1K!PiY$sQCF=e_!v@ezA)Y
zbdPhtP_g~O;X@^VOBew&m0xAPT=PwBq`=J2=`&xU`-c!{(7#tiM323J7sb9sH3*&_
zUQyrx2N^y7&6?kkhcM6vfgW-8?YSSitLvJlWxIAZx6xWyftP>ilZP-3-F>Xc@d5oH
z#^-X2_<7|MRC|Mam{m^P`#FUPADV9?g?D3Q)I%N9_B1y8*rLKm`EBj#cH<wbDrt74
zK6mv8E(|vLR&r2-UYMJA3*%%reLD^_K7(UtQIs<R`;G`7THp#I{UzcvcijT{QrX}I
zBSqx+fHnyD*iTFwOXsCOExNM%sa8e#-Bf6;brm^2kv{thkmCdTL5z<!bCvImY&73d
zzZ`A4TupjR_#7{z%w^`}dStZOQrh_G*B(^(M0z=f$Z%d28C>p`Y;vj~1lV_$OQYdZ
z(A6BtQ{I|)o_-KBK0995pqLT@`;G{oV4xJlszvc(g-w2Wg@Ju%KqHM_@*Pb{AVBa_
zLJL+12!->U(ly8mu3pE`zmtx!U&J0-YyHGWHkvoq8=e3Wf}ayw@Prtm09z+l%hLwC
zR>ra&y<$+c91}#*{O&6wj9T8QbOf)xu}1)12=ic|<t+#fdZweYo=u=$IS(j@RcC2G
zqN&Gm-stNx>62Psm?3Jkc1Kf^4|>}~80i2hkmaH&5IjbF{Gz}(oIH&2Eb~7CCPDBR
zEm$F7L|f^ue7o@bwzRWZzA4jNWV^vc+NGK-k-q1Y(vO^?1xJV}I`(e-P$}!4&wbWX
z@~S7-U|X?|DCQEqCnazH+`d$=+o~ZSl@#?$Ts)Gs;lh)UqmFvoMk3E}_)mA9Ppm#K
z+sCEaFTf)E_zq@@YTOylAWx(hdXI=Aci^22ERiB!Az))N&`8B-El1KW<P<GfAy8yE
z^p&M+I^0I}9cM<aa)DY9NReha8AS;>H;*HyXu%O;iWpjk#ce!R@onY&>O|>F`x28D
zi8;1BkY3e5RW>wni97CM2P%qY_eJVe>s{G-CpX`1P5e`F9A0evgw$ATQ}>QEilJO8
zqh**WV(tz^F(<UpdqfoJ0PjR$sT2Vfw|M`fx?0gA(GnCP4YD$y6}iqh4<+e+%)Nf-
zjJR-_krlM>!<BsB$6hz<$0#s@JR!zNz3s`+6Z>r1iJ?xpdKN}AOc;6ZFOA$QFxAm)
ztsJ$vbKPxJ81XW!JzvVH!M)KVfnBGdj}nLHZGHOdMJrz)bDf$jr^)77%ou6B>pqTL
zjOO0-{M#nNC>)qt99nd05k92x@&bj4fJQ1tX1M0tC@_Mo5HRYjSe^MLyYojCzhkx3
zsiXVuLtCjC$+uDu6oo_A!y?FxAWw)fvNPl?J9@^6Q{GSU^X+d!6__v@S^k8DsxN6~
zt@=<xSJP)vR2XUB6t4LCIA47wLu7cw*Um~D-t+b88^3t$SGZo<mLXwtH5M~QzrDBf
zpuh;~JtB-MfOQUuMH6iJ;XQCa)_+`vCXXv}10xy-Y?2PzARv_V;nvX^50`$~h9Aan
zKL+t%0dH+!BA*<puwm>3g$RCf2y}y(ADVsV4krkS<u=^eZKUA7Dhv}pOfihl;&rAi
zSyf99r@e>|Ma7S;eE*l8w-Xfm_a5i2+rScp!ymZk<6>WO)8$~9QH{XQCvPzG<9s;e
zHOLSA<Pg|*MD!>C_gAqjkscutJ>W+wLHPQ#+sNqwZ4l^DHJfl~FQ~k5@mjQY{r$6{
ztH2eqHlP>@$$pRh7pl3C(*wFeOpjW?gNQ9=WnG_n#M8oQ@GCIUvqrXBBj>(UH>=W@
zao`-_Jyi4v(oPt;-Ixqzo)lr3ay<C}hj*3{D6=z}-qfz~^X;`uSFqoICM~%e#i25=
z?}+HJ*Y~2>x+p#HA&iet2&gecJ~eH7JS+uy2wSj1AV|T{q@|skpF4WCU;1vq!`WHr
zD#qR9^F!5geg{B=;6vDgC&Uo3xZRmn?WWP%qR=EMB(fFTA$zW}8-HY{j~RXU6M54%
za}?d)#Y@YS=@U$fq)xod5c=GEZ~%vI^shY-tsyOy$1jp0Bcwi#S%S3opR5KY2wuJy
zdXESrci?Di&{8uY0br{J&`4!y^ktPDa*XCI%v&K~B(B4_KRZXwX0~cF@9Mpx8xr6w
zwePC`m3G!)RV`f@2fMqwK|$dhNxM6++as7Lhzg2gfvqTZ2dLO$cVc&8fO%Eyz9uSH
zzd3qmAJ^UM%&fgXpEJ+>{`h>?@^{{|X7=m}^?>^RjWQZ;Ik!Y2+o(hi7stGhiUbYY
z;~G=C(u;|@YLQ!!6W)b#H}klVX6K6??Xs5AREv5<BtQAu_WQidSB%HApBq@qHr>^&
z`67Olh^f`^#h^~UHd`Zg+vuX>gJE<VS?K*?qs8oup`1b+$wxwL!4I-(XvFei*|~}2
zBO$Kw+d@maS1EqkH{|`Q`%Rx7$XDnyKP6^t4)tK#XTpV3yb{TOOFRzQK=*9xHQ4eZ
z@6@CU--~}P>ldfXK&f`E%h9Su`#DeE*6iQSCf-Jqf%5tlEjf7J<d~LQZ?_omaXve=
zc<I=ue({IPUs~Yf-n!%DqSJI6sM~<xQ`|uEp%7>HhkaVH_hhAootfM|E(R0({I>>e
zZ#_4WZXd4k+dht$6RJFT{p4rio<9zy9b0sOVxKZ<`=qY9M>(-rZXX_pY@f7IiOVOq
zC|R}XOXD;Dt8-rHvQNd&Pd>)%{MmBohAyo#E}QGG$vz)8&dgK$OS)AT%bp+hJnX_G
z+jOJyoG8@UxZNk^=Teiq6<@4>BS(1slNv_SoZTPxx!NPtrm4_Ags~7iSJlYce+?A3
zD&jOZ5MhQ|QTb(|@9X#YoP6r*V>)oRar?9PGIUWov7G8oY{WI&QM>{X#zIOYvWfQl
zY^>n8+a~Gp+g*>x4r;3J(Ce#b?>ySpFwIy0_s=rsE_l1MCKIi`H|KqH+6rwa1Scnq
z9GURWHr)lcxRyK5#q4xhU&!g<X(tceHE7=KKFY>c`47<iP5=J3|FBVIb_t`MuoIKp
z$l%OQe`W2zHoExy!4tZTEGoZjlylggfaiC{JNk?$xUzYH?`b0Y@zY-&)kEr}&?KcJ
zliSD=iEN|6kCHv+KkgiIHpg5i+sGFB&fh<sa3pc-s3nD;t@d5z+2{L3O*Z<L@Uhjy
z0)aM<2F)tpvs$Tow&~vPZg4Lu@ZI4<oxa2mJ~Jq@ZW~RD_36gDGP#W`^!~8XIrd!d
z_<vR-gRA)n7WJEskfi%7HP}c|`DLSwovtV38~W&C+?Nest~c7hb)oWP=BKt%|KzUa
z=+(%QFWE*bGiCm~H6i;1$N7)<=3lcqSXY<!Y;oF3^DmD&|D@x&kay#k=Fw!Myxz}(
zt3G<QEXCU)hc`Jo7qv|{AjECkgJBcepIdk=OS5t{3hK5|>4ILPG}uV_p8l}WVzv~2
zyYN+qd`!d+Gh(0r)}k|G`nhVD9{N|~mxaO`B@ZpVaP)+Ujw^SrY=5-y75=hpi~88k
z!(NrQaR<qNN<3Ave`<E`*Lc*3Lm?%XMy@)Od4#@8h#cIVXWvTs`quC*bt)Gwv2mm(
z|Ab~(+Wm&3OR0kQvZPIuc2$IJx+Pa1S3Vd$wEwIUf$xehdyq!Ae_HPC6~p}_A5n95
zf4FBm`#viFzeUKIEj?rHzy4Wzti@vPANi2TtngdqmxCN`J__Da)%102vnt(p4xYQK
z72nA5NWJvqTh6~aa|cnnG9HNRq4*qK+`T?G%6#ld!4_vfO^wu5gC;%y{vo}~--{M@
zUUWFw`TRaj9%APqWG&X@*!h4qxi>GZpP8N2aN*3-K4ooQY#6@bK$p-<k@}b5KQMGv
zt`(O5{w!b9A1*rAGc>gG|LY>QHrv43e_eF&dHx+5x-v!Omy0@8*_-^T)9kcwyC!!}
zbllz46wgnta>!rozo#|Wc~u{z!A43XvW>O|xm@=4i=2OI*1cDIqdc4IvQgH~7aGNl
zOXZa3mLc!TMXx_;vQa0eho&+ChtDm){bNJ;$KBEF>1^7T>_sBaU8}t%qV$#t^Xus@
zMXQ`_G)yv@zv<uK_8&It#J1Tx{GVMJJIE6`%hN5gffp}D4$}8zT;rF8axPsrVpET;
zqug74{*vunV%F4r5p~V^szubJ>&)oJ9VGuL@i1in<ZZg6+}3g_w?E8x>b`T1+5x)!
zb8<oPowp{PKD<9)OriDbzlCe^&y19F$Di|kFsE>q1#O<*@_S~R?sSeKIZqxLGveI4
z_t(PRhK|$ipU<fe&Y*W?oZTPpxym+jbP*Pz^z57Gx9>RpzBvRts5<+f|F?fHy7~A2
z&aX@<GC$Kp#}qa;j;U;HievwG{~(jm)!8(_dr(kdg}{M+<6E8Uf7>}*a_)PxY=+dB
zeC^o8BD3nH%e14y43CM+@^$RkVO-f3QTEl}f9ldGBIl{{A-Q|^>fO8kUy<AP_pCQB
z_)B$T%Nk{ezwY19?&zc`Z9B%#oIn3-%DuB%H2F~Hec5TAOW&tRaB14gvuUwRy<L{p
zb!fEt^rx3U=AJK@_d~&1<%>KGpE{ys^&^|SeQi%~UNiC4<U2phYzlDg{5Ifp;I7UW
z^G?51$2Ik<6SvN;j*lxBT%$y|ebX}0ZKt&_TqtOTOWRkiZO2wjbvgakxUwail#RGo
z-)mxMmO5$MHjKGfW%b`lw~P5Wu5yVOYv0TFXufhaHuTI=YjLmgjaJok*_P(P)U`d@
z6+d^g$mQ{OMu%L#ym9I1B^M4Aa2y_!cI=A4lV@G0)(dR4+Zfz^?a>X>axK~Z;#>~L
zqaJH&7Ao!dqu_}1PcoLwl#u$ysgc#9c6yCDx8cj3UJV-z%eMSO){Smo>jk~Ia65B{
z0?T`@3}3mr?df)|nTjOdPM<&5!1$Vjy~;*ZUO8sK-0_=g?W^t+d)59?iyX~P=h@h5
z@3s2VTW-$0Y~T9v%`Y#y+bxfKi%!pzH{P#4=gzZ3{VTMe6XBk}?3!F5esz{Rc58Vy
z)6MbAH|MLJRQI2Hi`peRHYu?zX~E(HyYnnE_<ApQ>lE~P*zNqwn)US`^6-vF+hdD5
z9x!FNyCme!(sJ|89XJ}Wx^C7iDZ3Ot8Cu<@!Kn_1vJSc0WmNx|qw~img#H}aXx;wh
zdnV<!JAZTeutL#cho4OTlC@Z4&x}9V+f&;Vxl&qggBsW7Qk`nOY;2ZYPh*ponT`L6
z3N!z&3O?T5`wbqz{}N{C)ckL-^WdJ<?9QZqoxm^pE1AJxnZW;w<=5~#MJ!qY@K?9~
z_pko>%V57M>pEqC&U2xTx1XPh-NpOwb-WPY=-H3&|5^LFY*bkJ-m<$0xaO~(;CHf?
z>#Sa_$hYIqYHtyqLly?LpVQz~X5cYrWs<=>y61nQ!Y=orX8uQk<nq{;X3w>MS;Wp)
zCU0<M>%CbskJcdWtiwi*=D5wHe*Pya%wK~l3VimNC}oq0d7RJxM1|eZAT2~*+4H*c
zzTMn|^FL8xx%$zW5TToowEz9b{JZp*1~8g_Kcqw8iLqOzbKqsUAp37r*k+<43j4&z
zYP-0*q4{$8$w>T8R0z7UZ?&?#yEh;F&i_P(HS?#*6CuMgvQjE;zHJ`Eod3x1101sh
z&zAi+D(op)69rw(v1?`y8d|Pde;Em$tPpkFG&gV<A5T6iY)Sw%01-AUBM;>Ao~(?p
zrY<9?GUOld=VPBGX}bZ^q!m>Z_}n)+z_||=>6sd&g~+>4=sbbD8-Ayk2GX-9LWd5w
zA0!Ll?qXseHS)o$ciQ6)_akefz;`$OIs<q3RkE-U_v)Jt5ga$HRat|mG>EXF!+S~P
zjci)W+|;;_Rca3A=q)zfs6iD49($+YJPsUolz9frcvun^BEK;?Rw*jb;jM?zLlB`u
zhsQ_*m>1#xenktr%JY+oZsSD-*6_c{nkev;6C1DOz#)we!7>^=St0JNuTM|L9X@v`
zHGnSIucY#B?k4t8``c742+X-y{OvqUmNM5wVXyACG&K%;K3P}@yVd!k%FY2;POgSf
zGbh4^<)qwjnY_WsPMTm%jk{dMTJyQL;kbK%22~Vz|1&m@95^f|w=_r#kvA$-AKboS
zF-yLYbSAo>50nRRVUv}tsiEhM?(WaKObC6q22~V#<E?c{;n0hXk`hr87D7MpvwdS;
zRAANWI+`AW2p!h&-Gl&)If6X$T2M2N+=1JK{WmJ?J6RJ2-Z{gb1{^q~;S(yS@jF=|
z?(>_sgQIGA@=;;S*i4T2QbdO07Y!n8SXMqs<&Eq?!rasjFA<(^EnZeCkC7#9QAL3t
zxu3H&2M){1Dh<*?<k206DJOHl?=*2NJ$oW_=<q7zWC7Sy3i{|%+gNxwM-CkxO4dYy
z$2}<>%Yj20&&a|;+@tNnm6HWw$60Cul?D+ubog+oywQa@oHez>Hz%i>%yGjSenf*R
z3jB4$QAas&=x~RLG9H$Mg~(r(ya;ZEw-goMN%Ro9pdXb6FtF!Lb5lc4*Y0dq9D2IR
zva~I#DD>QECv?Z557Hnlgns5;6_0d`bT-!D4{6I3diJ`Yrw^9}FtU}e=B9?;;F?1?
z4!yYsRTO$!kHSZA=zBFt3!(orWAb|(`j@G6Cc2<|O_K*;2S~d7-eL!zIZK_bXT|CB
zG^nD`?_6jJ*1cGM`!*5ugoV)4jXwx>zgj|6n@$fwgbq8<`O*MvC(ys!i6HU~W$c#V
zL&V!;O%!;;12s}`m&4zEtr;>JJXs;`uX9E#9)}(1sF~CNx?o?I%6k}1?4vF!pBe-V
z$6@E6B}<uWqOcubbuEd*4kZf<VLxwKa|bUd5cYxD)Xa&n?fLKK*Xy;Mb7b;{itH@}
zYiitAzHK~#*YFM+R8imsK6ww}z~N}+m<DMf@-*IQwsYk02V^^s&V&dZmXnUs0Bm%K
z4iEzmy&l7nJ1AeI`H6EGSrY|*D$Pw#4jj@*Ghar7Co9Cg=wq02+W>TUtp(Hox?nGq
z%6qz**hlT}O?9`v#>>fFvL*_<V7J(-IP5wLWh{8ILfF2&vZv<WhOk2yQ3L3L9WRwP
zvcU;9Hi;hnYBTQaf{SG-b4?Vs%S1yn9QIhUun>0dG3%8VY!LS0CDhD`uwm`avs5Z?
zV7uj6Q+vCT?e{j^+we)Vvj$ZZc*l{OLpg9*yH9A479uaX<fC#`JFHeIBk4?ZLGLUN
z!1huZ|J_do2ZKkZ9e&0=4xz8mpo&5-b@b?V9D1fGDG?=MA@oll<|$``Lg<Z_(?bxU
z!wz$$G=PD<i@;t*3~uq$o%`GYbeK=cnkeuq`ODPjz#)xhD`YfyvO?T%uRqR(*YGJT
zsR4+vVOe<~l{Z#&XGbGqxRaXKsK{|chucNVlD4R#z&n)o8o`0XvNA)1v=I5F4hz7I
z$J{SbVHa1?v)2XPZnZRkvnwkrzxUU{vQoF<fju~Me+{ZA^yX8)y5P`nXpj~{e^lcN
zxV*yxJ=a<~6C!k2R{W&_*!iFT9svrGyOm1m!7Di|D_hB$DDaQYcZ+f0u&m@?C!@iW
z72?jZ^5Um-jInvX5RO*btfvOh1$(1Z-qq8@KI*Yb?ZQ8g;;`S5HBs2}+rR9B!*0Jp
z#)2m+gk7xfU&>Lo5cZsn)Br@-_WXDA>#_c&RNmlWV7K>SQ?r|AZ61u56GMzFX^Scf
zeALdIc{y-c!xw3g79yYjB>O7vZ}<aB*hJ4>7j(mBX#h4CZb0Y5TMv9P9)}*RK^295
z(?6ebVjhHkUxTy|`iCInKpc9Jt#l?t=&*(dO9QYaD(EVen8Ilr;$>w&SrY}m+~CrQ
zyBu1u<Te=%o~#ge#=v<BZdg`K+o=JFuwhx*C6zar?*w2?UB$zetv<$a!=Yia22~XJ
zXG6*a4jekX*A5vEOTt3rTNZ5XgF8HWCq0BN=>JFq7}*vyb5lc)>}mS~hhBS^ENzP_
z3cX)cs_8iN7!A@w=%I1%SK!cJ?xtr?gbvF}ojtMu>{?y+Q6s<mw5K0O?x1wa<|*1q
zWK9(K^-RZ=le}SBc}Er&;-1^Hwv|~!$9+^9x?qo$${X3rWvr+?s?;i)cbgFQDY7OC
zJ0b5a#oZ8gh5a%XJXs;^J~{h5;@*Zeyzc>O09~+;NafkPWAyd$meGONaA&7JC`*}Z
zqOe!b-gKOo5(wL$EG&e5u1kOA!5+e1cZixf5jN~FZ4S%i4W4X)IBV)c(&NEI<rx!p
znDsQMqQIMtJf&<7haKh?4bnp7Bd$bQsoig4=}dG%uYW`updvf+iZwO#7OBSV;U0(3
zr)f~dp+{}J!d(ubf6*W<gdVmgyf!Z?@Q37jjLw7z9d?)z(g17^I{T=R?;4-73SPtG
z$eJkdk3$@4bKsDM=W!Vgo~#h}_0cbtJ4Rqx8GM2oKo{)uQh7Ezi%!tCXjH2*_c!b?
zvz?Ts%r#Niu64qv;IN01g@v#)g*8@||3KI~PEj)_!iF_G<7t_^(V4x}WKHeuF2UoL
za@??-w9uf60$&+4#)$)m<z&AGX(95P`BH!*!T4x4D(veSIujyvSWa4=l?8D3G_j8w
zc{lsuQXDyS_&l;E3cUOG6yRbw3ymLSVIl5o8D<?w$5ho*vz5+MX%J!C^W=Zc7S5E)
z8(r8W3TtZI%l$TQ;<%y1<29(Fz#khgp2Qtq^MZ_rC1D}*W$%KORVlEG7=4i*LKpM|
zc>p83?w&O@^!>Sif;-6gw;2^y<dQUPMHPj<J?7vE9Qs5J(n9FX{ZA|V?IH9Nm+9H-
zf?ni`EP#P61T;4_^i*eG#Nkfwp+Oae-u(U7Dme6u8l;8LC$wpsf)^JEJ=0Y>6J5}I
z$^*Evd2!a%(BIyP*@;8npg|Rdo)}-+ghS7LO-e*bSO|T^-Mh+Wa0tECb$SS0&^O8h
zu=j-q^zCNUx3f+7h#c1R_Zn1D=r&HCOK|9&Zb*qJ2@9dm$aP+sS%J_Ovh(L9d)t%{
zx}d+82QcrRLpKw4%D(S8?sSg?Y1)b^3jJ>CcFM6HaHzOKgR~I((Z=aI;^pQ3O?vja
zpnKku24Dk413Fj?A2@h14t=x+RTO&E?pzmfr$5&qErkAY!dvB$2s+*F4xNcE=%Mle
zp3Ld2scU*{?`5^{@^VsxDhmB!`+UmP@zCi`ccny>goV%txfU#rJKg^tJp>Utd?-38
z4Zt>AqnpErmig3;``m$#7TJHJ!ZQ3NOPXt<z}>&4eZXA~EjWlQEW~~M{Kjt_H+<~h
zbf20z5w^Vo#(V}=>IX7;16u=PZfb{5^?Nae<Axn*V-2b(@IDLATG_|BQ-ib+dHW6y
zR(2|Xct~fW3wo18X#fK|rinE*^k!!l*1)09(x8e$4;nLAxi$`#m1GUlLg-hMn*YOn
z4u43M9@Ci+p~JE=TN=R4#l${p<Tr=@^ySE5S&1iWqQKjHw#|u`mFiDqG<dQ?+_grx
zf5~yfvNHTBH2@JdEGuzRd4rL4oUEyF$9?>v?8b*>CEqhy(iT+|_}U5+3vl4j;bSyN
z3z2u)Y!kqdLx;ycr)N)u4jrEVg)D%Jn~8nY$Ommt(E@jP7qTV_-0RYdbsRXPae^!?
z#Qk*FH+PO3I^6aZm4+_Z?WOV_?k4t8!)`t;q6-dt30V_`9X+ve3=TWhYZ(iktPu9%
zup}#cfNH#<1|Y(=2cwkvQhD>)&*rAay)Km@gyV+g<gNx)6!?JDANO<Mu$<I?E8}5F
zScp8&8V_Y5GyDOKe@73Y3;I2200TR%*WA?5%fF6RHWfnXrT&(sZBa#`k6Pis2ZugQ
zgR~HO`gg~z>;XFWo}Rrf=%qeL1F(Bx4gcN^3zzFp&D=2wcY0q9swngl>ryFig`m@~
zX^<8|uRCr0dS1z)({p^HGa*8U9cMpj0Jel2T}7VaN~kh%>!6e@^AqQ0vL*`r<BB&{
zPVmh8Sw@2=E5x1jAWcq=8+M$nzEA@YVZ&O!UMg?!V0Ub>rY<W*7H$s29sW*(Dhhm!
zF>fSZRyrohcvun^BF|QAiItOs7JQ|L&;|W(X#k^<t*&KF4gEmYMMdz5Ecb7+v@NPA
zbek(iaF-eX1x1BLX^<8|ZxET=%97@L-|5-wg6^Iy55QhHqs#WI9Lf3#clt;TswnjF
zVRPber$5yoErj0o;0)!KELh7+|DZF`1$~q>fWeKeMPyCw^t78xt;3-o*Px0*FZE*C
zM;v;EpE4qrgoV((D!F9l6&*tNv$2)E5mK1wf__38z*v!;k;R%Cy6vSh-Z=D(wz9M>
zswnivr*8z}&<AUf7D9LU`O?aS*7g+i?1|9fs3lWMSpaue6Z@!<pKzLAnpboOUUS)h
zqr$w%nkewHb*rD{z@Y_qlZA!2*EM+lihCT6T0W$r(jdZyqn1XgW%5QB_5{irg1h_M
z1-Qd!Xi!CgpBa2~GY1aG<=-?&3z5IKZLC}q1;0~|v~(uApwE;BFt~fLcb?dqM8}$E
z=HSrdHK?M{3nlm#!lBnpCnI7>SP1=l-tsNE&mr{D>FFVKK~IneFtT+ltf`%zsirZM
z7Z&)~UnGMxZABG@p492_avb_Z4bnpBWk>8)7CJ-dCo<Br*9E;uCRqTZksYVXni{%W
zo%<ber}xmHib79FZ}P{XU(_HigkB>#gOww(Gi9MOAwq{8XisSXwqpg|v=aU9stxzK
z1L#24ku_1^89#X_%blSGvt^ah;K>Sc=e)7o%6ThIvQYyNVZ#n|rBvSF$zC+Grgr$l
zVgone4u7se6$Rd|a5LrHEZBjz$}Zz!Nmz*7=E(|WrUibdGjq^G=z{)28o<cbJ(!ys
zdVHIK7jfv$Ib~^ER8i>bhhF-Dmz6~tq=nGa<{E9K15L<9&z=Y!mK8&8Spc@I5?u!R
zXw9u2cv%@t)<l7~8$D9FXct=W4p~@;`_`MX)%kb@j?44qrP3h6hGiu{DsOOROPyI$
zJ3Me%0T<liJ2j}Hz~@|fkP3Hr(R?x<mV|}KGo*QIWnA7RKRtvl=)2?rJlPa3Yij7t
zwngW}p(krlMWF}%b=?bx-m8F=h?1}n`pD>;R<<5S7o>;K1^pj+0CsMc0X;iwRd(C1
zIP}_uq-iUvDD+o$Ne(#l7!A@w=t}}ipW?*@R<D<Z>DlXoUZ;pO09#ODKo?ca+!Z$x
zhdxDvDhj>+n;Ok==pQvm3!z`m(nHy%1)&>@(U}mT!wxiD8i4&+(c+T7>cWy7xdRxN
zUm|Owz>{Ctd2`^9hI?@t4W6tJw|D9+Ww^&-2O3y{8h{8JcA#ga@&-3H$HJPrmVf-R
zej&#VANRABlqGFZMS<68ylpQB4#(vo8l;8D*B?4+<t4?QQuOSJ(4oV#mzD)^cQ>(*
z8u_tXw;poj(BW;!nkeuWT{<h5ctD3ABnu01pS90B40rg~GE^G6V0+og<k_{#=*`Nx
zeh&1)Vb39JqOjYxIJuU)8^TT|3kzZIjfhtkQ^0cKQI<+W7wqX$c{hWJebmmbd$ocG
z{_t~^tck)dU%kAQ+Fhxfj0I0t2)lNM;u&yf4{@LdAi^%CbZX|s`xm70Y@!xH-E?0<
zEcbO;MbZ5JC#$0@;opiV=-yQhwBxRZrW;I@6=LpqYnZal&<SL{xjZdvB4pS>rm7$n
zcV$bNSyOv@;kK(8IcV5HHqxMq!hVx9O*wuKc91(XNDG0Nn=;GFjkE8a=uC7$Z|p1$
z;K9b-tf`@Af97Uo!RJg3swniQ!zYDthePP!HAoAg+s?kNTu1}E22UfM2@yIRmCTX`
zaIa`$A2ssUe{Y}1eGW$@H^`bO@G`XyhI8PMMzxAE8a!Dc?%jR;lwJC;W`(#=1L%T%
zMJoU6g%1LIaia#=aM-zBWhrw_6!zy5^Wt&XBgw)-*keoGv~pYFUN>syx?pE>m&&t4
zC@^QA`Q&&Chuxa2iNX%))@LgY`v6&32z%<zf?@cmB*}wHg9saTn9V(9@&;!%ieOD$
zNaDIov*);BhdEb+Dhj;OkF9Mva9F#4YLFHpFSv89vKa+dtE!dhOmsn?Cl6p?&{<PM
zzx1iPm5Z$JXi!C=&o92p7l&T2ij;_wun_vsoVTsK=^0m*9)buRc9?gi0obSz9T>*G
z>id!V+yRVbi&v8+%{5Wr0pWgDo-8Mmg@w4+R-K`&GK3xGiR#qMiLhZ=DO5uyZ)7+A
znVZ_-twy&iz;VN};-f(o1-|0O<p;RK&ufqtBA;H{IhI#)SXMIBqBGG2y}LYsGh4#U
zni{&t2^%YypRUuOib5}+?nYr8dd}KXB1*zS=-+%=Te&!|MICwwUC`G{0~p-cML?{n
zp&wa!wKbnbflhy?K^2AGFm#s>4!uKN84*juLg-b_rBjYUfKH!Zj~+r7^moz#Mz%oB
z+|<xrx?b6WLwBn$OWUG~LeJuqYcdW!QiHS*dW+^wl{K{x`rQWf>~%qRYbXm~FtWw^
ztf`^f4sX^0ujwN+sG`uz%{`J0hyFx^v=Dlk*Ta<s6A*f-#&jmSppTRWFtUS|%uNmb
zZTsf$aOlT0sG`vO$CNLFLoeS%M#Pe^5c<BM$;xpE5PH9+^bkboaHx1(8o<NF#6D`|
z!AB}i=f%YV3>DKglO@eHQQ!gX3vT1UA&mgCun_kY<6-3tAvjdr(43k%5jGqurtp%<
z8`$v~=BCD7J#U^bxWgN0P(^{?yT0AZcJXZ*q=m@K4{dH`3G=%abS6aT(BTbR$^y8$
zo7hK<{A&f1vVa*nd>UC31>P;Qo0Vm6pUA>O+|?R%RL+Bg4mY%>($EEal2o4cnP|5;
zJLBE9+}{xPd9o%7yW_KY$MACE(niLDCo6=#G)-qKOC$!gr3TOi`;=7Pt)hv2)Xomu
zFf|sh-5J}-Qs$Z{>}&^eEXQFFA`1&)cd?JY#Gg5#vp2P;W=@0+Yj^4nGI=Ar(!ku*
zxM!VCH<o)FKKwM+po#)_8rvW(2M%WscWRIpB9A@Y(aOWmhmLe6x}Z1dBn@D`?ghP^
zCfmp0$2jy^8dOo}$z`_J#i1u_kQPFBz1!b|`y38}Ds`bV(FJ|BJOG;kHKJ3t^>1#U
zi9^4sK^2AGF!k*9IP}_Gr9_m3h0srLsCyHK9%iD4&;|V#Apo1uLT_2lcKY2QUd>^7
zDb`J%HdjTVkE%A;%F^Y@8l;8L9W%RGS<rdPo1Q%pIvg4n^N|Jc@HDZHx}yI)e(nSJ
zxdV9k=|R>+fk!8_ddOW4fA{Cf!b03BdcITU4`BzIt_PI{5jN~VO;ULSLv3zq+{Ri<
ztz6`>N`op2JhIzg&Kx-GK(qFg@vtN;MDFkNRJqv=ey2@)(L?BhzFHo@z!vATriPxY
zU3>ywR$ggPMWN@Yki*KQsBL;ni6{vRp<lRl)=CFDyAM5tF6gi10o>RkA=cE;ht=`1
z(t#R%rD-dwDD-`q!<9o2pwpLVkQPF(*zv4#(;Ia9t-kc^bwRJ#Pa1%oC1^lTK1^Zw
zF`id**ntkypo&7Do9E>+yu2i8kQPGEk|MvA4Ohke=}dG%50M9OWxL#1Q#<|1&@R_-
z=&>49QRo-ST}*>RFV|m6L`hf(-DPcFD?`OT0rU_==y23>L>hpdQHCB<wySfUnjE<U
z7`3DxAWNESqQIX9M=P)3;P2j_EG)!5e&tAIe;XXNtQ$zpoCq6^T5JZ%<P9F|O%Q8p
zhp(?RB9Y^UWu=}5RTOy4$)Z+XN^H>}EkwQ`_^OrNZEphUOo-5-!|Mmh0<dWobf#r-
z*6YhSa_I1IvL*^Va?^R`<`n4g4`g8>?v&XkE6atT!<~juX%JyUhfk2o8`zE?b5lD!
zdzM;Z95;0Mc@3&4@J4Q~7w{VH7A)goNmz*dMRZ>)XV?Y}rH9Z3{Q@BX8zQ=6L&VhS
zMi#`O=Nu+ao2#PGr<5#gWm()v4bnpBk6qW!$LCn~htRVpLWkQ@at)USa4})8-hQku
z+M5H1+fv#R6;ap^FP~J7#fHE514LONX#4cB%Df8PmhyQ7jRO%fEGLae%ES%q?XS71
z%Squ{4XSa_u$;`)po+rY_PBfryqtX3AT0!5bZb{Dt86_-)0yakK1&|JnH`DEni~3{
z26dX_&=WMMqR_WIjI*)_zgDP}h?1}ndg}S*t!zsP4Wox3LWiT2oALnWW5^NYfsJS7
z<1;I8lu~$%H0ggeQQ$`o#vbJ^hoh8nWMLuhyM;a~chbU|6+4!ixh~jw$4TYQ%g7Pf
z5it|2+#%74tck*&w)o3;9QHA?un_j45gC+aK@j%O@l+Z_*szATogkHWXAeEBsmn?J
zX9LdS<z$HlRTTKyarv#>1C(YWJz*j89ZTw4IhDE2Bzg#4(3i>suwB0fbdPfPsLCmE
z=!qIsQRw^n^|5jkcGJmHB1*zS=%x*ytz1nLK7}5F2p!h&N74Xn2#$`z&&_P`5U=6(
z;j*N;CJOxTA)~F-@abe>A?_1<uNUMaa9CE(O{He83wDWVQhB4v9h)mGu(qbMUm13o
zy~vs<>}{Q*lJOdTkt{5Pz32NT<rql_JHvD;4PCIkrShKcCiYQR@TVoJzu?6L!d^wz
zL}B;%YsobncIFu}7Cc!Y>|S54D))&)*bQe=1L%SsC6za@vud!TY7FPPRL9H6BeEt6
zyU_E#O5KJ-!N#*>EO@d)*gtx$u<{~#;%sUFU9j&-<tsXy*hlT`VvXw!!JS=Vjx1%a
ziNbF8Z0Kv;*;B~ELfFHioUF`$pPWn0oCq5Z1q;uU$s5g!B+(_EV+Oid*$L5IgDMI<
zerFHmhC=u*^MVFxA@Z*GrYpyKz#mY?1#~7v=x|ihLmGfxi-6t~ks^Lvdfee_$(lIu
zz8TKo4$rnwMuR6S#NA|ZqB8P^wY$k8Y5-lZS4!n88cpn@E+_Hv*O%h8`zcuyg}v#T
z{cRj}v&AwNJXs;^Q7zx6z=wcSmQVu_VZ(CrKq_x!N1~dW8h3EOimKe(aCXpcsVr%W
zDhmAL(wDC|a9B=eYLFHppS1P0a_j*tCzqDdvnN7_<-|Tx7J!{sgC1Gaw@lUUxWoO(
znkev^KfT6s;E={uvak^MoNSdI;SSHboJxZT8#=s~RNlyj66U6M_~@m53*!#os6iD4
zp2fYdmHnD|R>*i*5*8va)xT^cM-IQ!)+^~DbU}}i1~7QAc~jQZ&<*1YDMuc_vhqQL
zDhj>Zt=kdY<*=-DiIx$uBrJqJ`S_e}IP}G<=pl&EVOjYo4ZtP{(K*7Pd=pb~pF4nc
zC?2b2NpnpU_%Uza8yq;aU=&$ch`V^v)mG-e@2sI_PJ|81O2xG@c>}vr$=uWq|M~J;
zE{+>Ie3%AR6!;Yvw+0+Iba<i$X(95H&3`C++2D6td_A3sF6bfh0PN~81A2Yf`^@!g
z@~#s?kJX@xLLa;&)fgOl*$q-6O2R_ueP>jvgO`=w8|fi*L64OOV3R=xbn0s8;TorL
zr>BXLrmd)=(2HzpsLZE8rw`B|Ered=xZMpL`le0v>~%p;yICH9Io%U`!PdV=men})
zrW#aH=xvjh2jNcNtwCA{J+Ow4l|yVkZJ{&K1-;o;X@H8%>8z=1`tT)_-{8>aXi!C=
z&u=!q3J(2;25BMm2)BmH4t!YCt8Aw;(FJ|3G=Pzvns07u=<!to+T+k~Yfwd@cim9F
z2oAmO4jB<k!b0dXy?0vKhB9_1Jp>Ut94g+C24HvPq1WXVyZ7n{NA3WIip6%xlIEHy
z@a_>W-8pb*!AWFcA@19Yz3%a94kxsZ@1|x>gbjy^1^3A04dw}c^mW9@v#<W)xZ$Y9
zTZ1YJ+-E?kog6sqK+kEA79u~KK2SOE8h)qg_tBXUp~JO>KGFbgCak0!OdB+l4-?_q
z!f2u*3fr-9N@ck+G++AtG7`TN6@qqisH>cr57!pfK0uQvLWa&>AQd;bvh}U3scZL`
zx+z!U&c3HX6@}fSc=rI@*$ocLXjl>!0&n=jZ!#|?(Ag6X(L?Bh{+Bd>(U~2~#F`rV
zrNlH>xw~OGDScR$wnY_%K6cjXBRKR34bnpBFR$cO?j(fJ&&SfU*9E=w5m^93MOMRE
zQ$t_ZziMsV>HRdQqR?k=oHiAQeqDpK5PIbJEE902=R8Jdq6@koApo0Wsff+7q`zLO
z91eY(22~V#nlIgym&>r@EOcB>geNS7o;R+h6{mMRK@Xt|`gVB$c6qe{y}<gy2LBOw
zdHJeA6@@<6I8u4v38DL(loC-A7D7*c{Lae#n^C9eA#_3iDi2^_9Vu(-@{;Co{dnBz
z)lW;)R#Z{wUR4sTl$UiHq=nFzJA78o4}vxQ=^1+Vx}euMD+^#`V{&s-L%(uxh?R-0
z2^v&U=s~ATSUKY3od#(k^s0vjDk}~l^a|(cOmsn?C=Fn6VRJ03si7}kzvdF|^a~nP
zQRttm*z~|VQuhloB9?@O(Eo~Dtvr81=z}lPLlB|E$Nh`a0BpZ3x@C4j=>6p!xdV9I
z&v8kXG}lCd9~c#F<%M$ySy+hs%Yfqp`M3o>aqYTH&724uJ``oXB9k{7*b8UY)DEAt
zU|}VW8;;LgX;4Lhm*{-z5$^DV8l;8DPgR<@pCgA3{~kwYq6>QKtFizFXEr`(O$|MC
z$$l$uh8Ag1MWOF2-+BQKJ>@le!b0e8qLx_MG+XmJJ%ldki{t@}3_5FS=ym#5a>a*=
z_cf@Z&==Z7x#7?o-jEVe5*9)qUUH;zK@D{J#CUoLUC{3n0<dQcBlb`<Y`|wLJC(~M
z$kXPkDD*eC`q<;pXK0WXLT_E|sB)tOgnscRJ$oW_IGbp9OBSG_30un&Ipoa=UeDod
zqAyVqh5bBFyp;>|E)ivgpz9<hD|;m1Y-0L5G!8__u;c716*sVhCCyD;PI4ZI*ov2v
zXbq|;Z0FMTtX$lY`L2wHC1D}(plye(tVL{mj~+r7^k{hi7j|M0Yij6G5AH4D?uMh4
z7aCMi=ru<~48Y4t>%XK#l!S%QAD$X)<?a5g`}7dHpudy{U<XSY(8DDQ^^4qvJKgX=
znzo{fLQnbXpC&l;MH-}q&^PUKv2vGk!b5uYx}X~pr2*Jc$_Dg&<~CV^*5XbN)}V?)
zAC!<-3WxqcgR~HO&B=kvCBLxaEc%$vL>Kg-(f~#SJ203vb$MAF=aGy<Kcqnwh3;47
z;{+VK{Sz4xOTt3vMyKt{D>(?g*Hd~3B6K)fIV=s}VZz4b0lp_!aNuyXV*5;%@^3{H
zwjsFJbq*UoiT5VT3PEqa)U*kI>V%_}<<DtZ6CuMnmCsUfHUvjU;d|T8n!_u&1DI2(
z^g@;}*F-@(J}ME#K|{+eCkqQv=Wda62S*LR$-6J9nd^e>@=7Lep2)^#$D3z6-<o?G
z!X8T2L}3R$DOd@IeV;5Wgnhq`-4Gmhp*K_-MA&c~9w?PJvQt#dO<l7`m3tS;al;|u
zUJa@!@D>*e9pJ#>8=#VJWjrhi3z2U)w_hnG@CW4mjvhi6^nHW?te&6;1K--?naB$Y
zgl_Y<JZ-LuLO=6-cTF6+uLfx$^qtSESy@W4<~==oUC?bl$O0JHG7583JAHnGEemm{
z*VCYiLJ#wswh)KDMT4{u`s{~KSMlNko&M${oe2>->^SRxk_9lb^`2NQA0MA^l_Pfm
z9cMUM69wM6ul;Zi9Mbqe78c@8^{}<_Y8iH%PG6`rh_GQTpCFYtume8KP3`c{KG~Ib
zKd`Kv*Px054;fm^g9C@P+$~AQ!;-KNd3^HTe{hEfeWiyWLWd5&APwN=YGNNX@_IY$
zm3R5j;n}~*lIEHy@UEZcDyMfshYuqQ3voAnvq-7p(BV73Q#02EJ5#bup6#$fcLW`-
zv~)G^HX&>;vL*`q%Gc9#xw|3k-DF`Q>>-BN%5r@O`@=s}8bsKzoHY6&lQ$SyIbluh
z?W!J=SMll%%gGE4swnW!XM5Ul;IN#0(;zKG?s4gdl}ndAY*N_t@DB2J>d~{8_h7#u
zC;uS!+~;f>;5{fP(8=4+?-!5XOo-57IhiR9z_tOQJAv$@L)PLBzfRUff%jSPN!gMD
z9bUy&MuR6S#9gTVU@H$lLsL)#5Mjf)lS@*0cN4ZYWu0GjW!EyCJIS6>mhf*y6trJ=
zpOoC$@RuG;loew3+&58K2n6R&wx*(GO@s^`ohG$RoE`d)UQ)2><yxhPz?$8Ntcilo
zm^Q~J4jMXo8(CP0dRNihr}=mUev@y~P-zfhLr2$5E0gy$VbiZ0qn91z^%^>QB2f_q
zZP(vHd7^|IUK3@7n4`K*ufQ=wN86>RaUep5jvgTuXRDOah02YyUY>^6=vcBQ3fkj-
zz+2qWb{S+Oc(Ov&0S|U6&yet&?4FStfCw9oLiS1JjqD#_Zt79U!@+LK$%SwfVv|Xh
zv_%yKzII#AOB^^Hh4j@REkwT2vzL`65Nk8jvnN7_4!6xB3*hQ%Vjnf~*WX%2^Fjh^
zc3rY23f#|oky1)v&5j`p3vsX6*itzN6*~NRRw@l5Z0PVB*<|tt7q+sBH8t+B*{4Qu
z+|c0@G^nD$eeRaCa(?(b4bnp7+xssn%j-A%PAlY~GtmWoqBMZf$Tn87riLD~=EiUw
z`UMTDDD=MzKP-)x75AJnB9?@O(CcT3x3W)ja4vcXUC=K|0~kD+(^*qPUq5}=0vvj-
z+_JPSswnhH2QnPNp^wraErgymxSO&-89Mzy9(wk=py$pj4ZzOBGN5N-#TBoRnU7as
zdFiA<6@@<GWcp|v`f&}?Lg+nhla#x`A@mga=}d^wVF%ht8o-kc#Ice1rPh_7ai7Bu
zG?J`|0{`N6wiyQwX{0S6qrsCE;vRav?FwGaVFy~fAT<CHHtawbN#zY}ipJd3WyPn7
zos~<J?rTs*fiIty_>2RGL&QdfWIQYh3z5e(Df)^dhu`U>!t@Zjpg)iXFtXDy%}ot`
z(#K~j@v>r9M3%Ni6@|X2(E0*6^cfnYh0qU9DQ;zj)y1Op?1|7}S+Of73&1udqx+IW
z<62MT$YELON7h7v-xy-A+*$=K7)KTs;x25=q}*8r%Sx6KR2sTq_ms-Bjce%MwNuZ=
zDfh@i*z3reDC~%ka2M`w2s?X884I4Q5cV@Wk2buZK-f)7Q3DWR!x|nfl{dJvGq6}w
zmy^e3w>;tAhNF}h8dOo>Uvez|%z?ui-m0{Whb3Vl^8J~|TN#hfEJF{W3;GLr00Vo&
z!<rg;NT+^oICN(_Y1)b^3jOuS84Yph3pGd!q2KYHX63llczb&GMCh=FJC~IO@L=mJ
zu}7fUr=5d1a#&V^$eJkdX60TL<iKHBxlI-p;?Dfhcm*#jc^#-Uh_GQ<=`WQxR%FZD
zSW`RPcZOSGjvJPh9U4?o;G5=nRO7&5St;x&<6%iyh<sbYMCBC({7yTSr-vXyhYsH%
z4ZscwLeB}x-YM-S{NX2wtce2On7scO2M%d;t01GnlNI7N++3jCcn>?yB~H`;x?q2h
z%X^qyv2x<$vGE-a+ud23^1qrW?8N(~3^?pavak@g)Az#4avuo$wt<?tF4#t+OrD)S
zf}WO=(64L_9QF{hCJKA%k^OaW*!Rf7LfGSOq_Xl@U%-V*g9sbe?txNygA3ck!<xEw
z*Z<qr$`j^p4XP;c#p%mg84?zAmGQ77EJXgg?QUi1J*-w;-RL27LEkM8;LPSySW`nU
zw4!BV1GZGN4QrLalFffKs0Q{N<m>I{8{!@08_>T(-@t(WUX5*2<Vu;r#(ZmrQ{|uS
zi~n_-ty{IRF}Erp&Qf<N9wmGKo<ZK`r97q~JqHDvLj?^D`W-13i<ItHB>VBhj`LqF
z90DY1h=kf`=g#AWGqB{%tuFXS_;}F&5&tejGIbB|A26tAp!wgy{&$a_-7Wtb3%8SB
z6FTxz0Y@E|G=vm7H&>~?%JU<1Zstn#e-+|f{*BZNHg+|C6Xw_b@Rffr4FQH$=<YwL
zpYgxO+Wq#<SZ*xXuiV!l-aTM&{~!y0i<{CF@Jr;mLpaWr>2SIS_zm{&ANXegn}2^8
zf7r+d*3wP#0G@194QuM3_0a=wyzn2vM-8gzkDzJMBM)##cC8{Mq9iPIWcD_Xt<=&b
zRp}uRN6Mjpk_Irc6DZ704LxlOdu2fmtUQ&g$<nr{qR@TEmQW^<U>R7YL0SmCQj?8V
zt_?`6PS0K!^eQ!E0SumOPJ=Zy^rr<!m~f|$(V&V#57{?#Jnr;Y8l;8L{qinTZry?9
zrCcpK6C!l@#5YzNz|+OVK5FFoY%8bX57`cX)UT+pQ)EpPc#>`3tsFR{QK7bs22WOq
zyZ@Z?$}J7Bv*=rg8bBB9BXW783H7(jn9lFHzai|@b)_l)tBJz?P(58J4%?qBEQCE^
zS3~8{3<!H&J!<Ae*l<CmO?{a>TL^{tx?<;ot{gR7P+5zph=MlMjw;SULk??+vO>&u
z3sNY@*1^%$!v-`CM96S-;@(guZg61-60xQ(AUTugE#jbIM>$f1Dhm7Py1lD7Y}ipg
z)gUbde&k?DWlc7$Ql%TyndpK(N*=(Cb(E~Bq1UW9Kyf#Oeq4hp3O&GcYbG3eg(gxW
zO2R_u&GK!vGBxVglpcZz9d?u_<N=t`F`xUC>VJ;=+yQiy>6=ND{#O$P9-Sv5HwO+a
zIFKwX#C@#VK;@)BShHf9Q!^*RhGiv{msH-!CXiTDJN)_TO;*ZEBMqu3@UvexSUK}`
zhX!dOa`&!P7O-bRn{2GX@AQ2OIul*c8@H4OaAwCHv8IOpDYRWA_c$yoGc~B9(96U(
zD$iXGYx#E#(n9ECT0|-Lx<crlt?5jN&|z7bB@JLSu=4~E<cr%BQLc%GW#tB069t|(
zqKI;wG^A0jjf@6QR)~An{AE_g-yv<O0d&E>B9&*W6wrkV`E9!>`^F&b-0fs3b4?U>
z>b`xHBaR^Kkz`>Z><SS*3h{yhYxv&w)Xa&nVGYmLK_+jk$ku7IrY<K9ebd<C<)p0!
zRTOxevA65t<s?>vv=I5qwb9Cc1o)l)=tyUx3wpaw(g4n^p0K8d{@0q6{cz|@HK?M{
zJ4CMAheJ=>nVzr^`oa|Jtt=U?+l3xN7xZQF0PIzl5q;gYyZfgoK8pf7&PN(lQRwrW
zj!nX$H|;7Vq9iPYp13WOJ1;KK>ER}N2wl)0Ndp+zQXF$rJH1p=t?9Va%XO2bZBa#`
z=efRSJq~@225BL5pAV&b<Iv;0>Dd#Z!_kU^k1PPYD;m8n`g?p(9`17o@DLP0)<l8N
zHQo>Az@Y_ikcEY~pBx*j96$;?&>TIeG>EWa2kI-8H#)Pj!kXIQ56kDt%5lT8vRQ*F
z3cOi{Mpn*z&DT@L!;-KN`NU#QR?h8i+lwAT7xc~200uT<F*h~za9d{&9Qr2>swnhn
zIXrH2m&1pkuDxYMEC~yt=c_$dIm-e5ke2kJhtLK6lQe*lEsZxfHFW3uW4htcEBngQ
zwy2`eJ9t&>fJ2YgAT5M`@5Nu$@ml_{FFkuAbXdzP_mc&1^E9!K8hNZ!h2FgTgtdG$
zSrY}m@%2tC4?|DM!b0568z0={xM3|X=})CWgbizXh*aL-%H~#BQ{#TQ@5L0{;jtQ2
zQQ)PD^lFYfylj6N4@<&A<n@McddZQ)veG+%9)buRIy_byz>Qtni7lLn$uRjTM-CmH
zYJe<hu89J_5f|5!1BW#Fk%fi0J9oLO9IOEyzGfgbb6v21Nafk-(dYrv;dk0U!pljm
zL9&#&CJMV<S3Bjh3kZ8XSy%}B?=Rh~ERKH?NX=Xq?5aUBdGi){Y>e{vs2)vm*kNQ%
z6n5lzyTiD%pOb}!unWXqu`&)XJ%mbw2piV!5mI@BJ9`<yn!0v3e0X~U#|_8fM>VLT
zz<ZiLe&xX7kkBz$#>0}Z5P5?Ca4UOl`VOUs&;|XNG=Q-pJCU9>HFW0`;q`d+hS1Xw
zlcjA@MWHVo67vR!K1hSK5c-wjg}dM#=9Un8_PU^_A1(`EV8>UQn;Lp1hYiJW=*=~#
zqR`jvonhs|i#-~oh0seppJrto{&@tQi7x2PM@j>*O&|tzBgnBR&y#q0nX5q+g<dAa
ze+BOJpBkiv(A%AUZsi>Bs-x*lbU~jd55R8zGoZKs^|8G(3x|G3gDMLB?XktmVPLSP
z*9(;rQ4$tHe>Wz-GhSZCh0#M0p~La`U1<PzehhkqOhemaA-uRafZ4<1V`NElO%(XK
zfFg<9<#32Ni7YI{9b9KlEXNJU<HyHRGuH*X;5eDQn}>;g)RjEl^{umT*e0?j3fpni
zr3yIglVo8b?CMoZT3N4>Vgi*05jGqmc96;&&8MZK$E6QwvLFU8Cy^RdQQ(*APpZLz
z!*Y^-qKt<nVIlGkjxDWR3|)T`J%ldkQPKbg7dBPPn%d)^4_+LELw}+{6@`BPMj0#T
z3wlkK5wRpJgnrTSft5>(BBszo5TU~w{!|)(O^u_o<0F<2%g&L*8txD-OPXt<z<nlM
zIK^EKYxr!kun>2*0l}y68h&XiHFI6C%S@BWyBJLDqxSdq)^iha*nP>GDD2cV_b7WP
zA?!G^un=~5{gYNY%q-KXG>EWa4eu$HH?kdw=BCCSw<^NQJ@XqhsG`6}T(cd;-3@Da
z?in&3mV|}K=gsb;tjB`iX{(v^5W1jmlm;+3v)yg1si9wd{@@AsIDE{1uR#@so)ns-
zloSZP^DG$=OTt3v(_($P@uC8uFPcpcp$qy4c>s1otN}eGHYDl(N*uc99BJB$Dhj=S
za@TUW(^qPc7D6vw&}$S9{lQ#%_C)Bg<E%7K7Jw~*N0;oM8PiX>gBy07qsW>l@WWn}
zw{hUmf{)3<LfkF)_EN49g%3Z)7f@*sVZ&NJR4Q+9W^W@{Q<s&c$674rk3X=i9MYhQ
z0{@otug)AeEGu>kWjrhi3z0uCbXCq=gWqY-Mf4E5pdXS4V6T%6=qu%e&-3rV%Sx)n
z(zF#-6uO;P^+7oF{u-o(&{sLxSa}HAxP+cP5jrd@sh7$Eu)SC4_A9r@2439fu&gvB
zYofqM*O}3c1BVveLKYU{{+#;Y3|_@yS$VaLN`nYHy)xO9&E{Fn+L0-2_~8>y{KN@t
zH+gpO*ZThX*ZB1_{IB!l5sJ*u^pKq(W#gF2#-_OP*KKT~!WLqG{m<KLDg+Mn^J<(r
WjrnhVO8F_}M?YJeR4bU4&Hn(aX&2Q1

literal 0
HcmV?d00001

diff --git a/tests/queries/0_stateless/helpers/install_predefined_backup.sh b/tests/queries/0_stateless/helpers/install_predefined_backup.sh
new file mode 100755
index 00000000000..4445168bc33
--- /dev/null
+++ b/tests/queries/0_stateless/helpers/install_predefined_backup.sh
@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+
+# Copies a test predefined backup from "/tests/queries/0_stateless/backups/" folder to the "backups" disk,
+# returns the path to the backup relative to that disk.
+#
+# Usage:
+#     install_predefined_backup.sh <filename_in_backups_folder>
+
+HELPERS_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+
+src_backup_filename="$1"
+src_backup_path="$HELPERS_DIR/../backups/${src_backup_filename}"
+
+backups_disk_root=$($CLICKHOUSE_CLIENT --query "SELECT path FROM system.disks WHERE name='backups'")
+
+if [ -z "${backups_disk_root}" ]; then
+    echo "Disk 'backups' not found"
+    exit 1
+fi
+
+dest_relative_path=${CLICKHOUSE_DATABASE}/${src_backup_filename}
+dest_path=${backups_disk_root}/${dest_relative_path}
+
+mkdir -p "$(dirname "${dest_path}")"
+ln -s "${src_backup_path}" "${dest_path}"
+
+echo "${dest_relative_path}"

From d421636a5fdc3b73fa5cb05e83529483d69e75e2 Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitlibar@clickhouse.com>
Date: Wed, 31 Jul 2024 09:26:09 +0200
Subject: [PATCH 1098/1488] Protect temporary part directories from removing
 during RESTORE.

---
 src/Storages/MergeTree/MergeTreeData.cpp | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index 2e10f5a0227..ce27ad24e10 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -5557,12 +5557,16 @@ public:
         auto it = temp_part_dirs.find(part_name);
         if (it == temp_part_dirs.end())
         {
-            auto temp_part_dir = std::make_shared<TemporaryFileOnDisk>(disk, fs::path{storage->getRelativeDataPath()} / ("tmp_restore_" + part_name + "-"));
+            auto temp_dir_deleter = std::make_unique<TemporaryFileOnDisk>(disk, fs::path{storage->getRelativeDataPath()} / ("tmp_restore_" + part_name + "-"));
+            auto temp_part_dir = fs::path{temp_dir_deleter->getRelativePath()}.filename();
             /// Attaching parts will rename them so it's expected for a temporary part directory not to exist anymore in the end.
-            temp_part_dir->setShowWarningIfRemoved(false);
-            it = temp_part_dirs.emplace(part_name, temp_part_dir).first;
+            temp_dir_deleter->setShowWarningIfRemoved(false);
+            /// The following holder is needed to prevent clearOldTemporaryDirectories() from clearing `temp_part_dir` before we attach the part.
+            auto temp_dir_holder = storage->getTemporaryPartDirectoryHolder(temp_part_dir);
+            it = temp_part_dirs.emplace(part_name,
+                                        std::make_pair(std::move(temp_dir_deleter), std::move(temp_dir_holder))).first;
         }
-        return it->second->getRelativePath();
+        return it->second.first->getRelativePath();
     }
 
 private:
@@ -5588,7 +5592,7 @@ private:
     size_t num_parts = 0;
     size_t num_broken_parts = 0;
     MutableDataPartsVector parts;
-    std::map<String /* part_name*/, std::shared_ptr<TemporaryFileOnDisk>> temp_part_dirs;
+    std::map<String /* part_name*/, std::pair<std::unique_ptr<TemporaryFileOnDisk>, scope_guard>> temp_part_dirs;
     mutable std::mutex mutex;
 };
 

From 4e2f8576e5a6e8e39a16334d0c697d5cb09e0469 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Wed, 31 Jul 2024 11:50:58 +0200
Subject: [PATCH 1099/1488] Revert "Add settings to replace external engines to
 Null during create"

---
 docs/en/operations/settings/settings.md       |  12 -
 src/Core/Settings.h                           |   2 -
 src/Core/SettingsChangesHistory.cpp           |   4 +-
 src/Interpreters/InterpreterCreateQuery.cpp   |  35 ---
 .../test_restore_external_engines/__init__.py |   0
 .../configs/backups_disk.xml                  |  14 --
 .../configs/remote_servers.xml                |  21 --
 .../test_restore_external_engines/test.py     | 218 ------------------
 8 files changed, 1 insertion(+), 305 deletions(-)
 delete mode 100644 tests/integration/test_restore_external_engines/__init__.py
 delete mode 100644 tests/integration/test_restore_external_engines/configs/backups_disk.xml
 delete mode 100644 tests/integration/test_restore_external_engines/configs/remote_servers.xml
 delete mode 100644 tests/integration/test_restore_external_engines/test.py

diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md
index 8739414464e..c3f697c3bdc 100644
--- a/docs/en/operations/settings/settings.md
+++ b/docs/en/operations/settings/settings.md
@@ -5608,15 +5608,3 @@ Default value: `10000000`.
 Minimal size of block to compress in CROSS JOIN. Zero value means - disable this threshold. This block is compressed when any of the two thresholds (by rows or by bytes) are reached.
 
 Default value: `1GiB`.
-
-## restore_replace_external_engines_to_null
-
-For testing purposes. Replaces all external engines to Null to not initiate external connections.
-
-Default value: `False`
-
-## restore_replace_external_table_functions_to_null
-
-For testing purposes. Replaces all external table functions to Null to not initiate external connections.
-
-Default value: `False`
\ No newline at end of file
diff --git a/src/Core/Settings.h b/src/Core/Settings.h
index 27b71558bd3..4fc2034b855 100644
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@@ -893,8 +893,6 @@ class IColumn;
     M(Bool, optimize_distinct_in_order, true, "Enable DISTINCT optimization if some columns in DISTINCT form a prefix of sorting. For example, prefix of sorting key in merge tree or ORDER BY statement", 0) \
     M(Bool, keeper_map_strict_mode, false, "Enforce additional checks during operations on KeeperMap. E.g. throw an exception on an insert for already existing key", 0) \
     M(UInt64, extract_key_value_pairs_max_pairs_per_row, 1000, "Max number of pairs that can be produced by the `extractKeyValuePairs` function. Used as a safeguard against consuming too much memory.", 0) ALIAS(extract_kvp_max_pairs_per_row) \
-    M(Bool, restore_replace_external_engines_to_null, false, "Replace all the external table engines to Null on restore. Useful for testing purposes", 0) \
-    M(Bool, restore_replace_external_table_functions_to_null, false, "Replace all table functions to Null on restore. Useful for testing purposes", 0) \
     \
     \
     /* ###################################### */ \
diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp
index 8bea0b1eed3..9faf77e9087 100644
--- a/src/Core/SettingsChangesHistory.cpp
+++ b/src/Core/SettingsChangesHistory.cpp
@@ -80,9 +80,7 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
               {"ignore_on_cluster_for_replicated_named_collections_queries", false, false, "Ignore ON CLUSTER clause for replicated named collections management queries."},
               {"backup_restore_s3_retry_attempts", 1000,1000, "Setting for Aws::Client::RetryStrategy, Aws::Client does retries itself, 0 means no retries. It takes place only for backup/restore."},
               {"postgresql_connection_attempt_timeout", 2, 2, "Allow to control 'connect_timeout' parameter of PostgreSQL connection."},
-              {"postgresql_connection_pool_retries", 2, 2, "Allow to control the number of retries in PostgreSQL connection pool."},
-              {"restore_replace_external_table_functions_to_null", false, false, "New setting."},
-              {"restore_replace_external_engines_to_null", false, false, "New setting."}
+              {"postgresql_connection_pool_retries", 2, 2, "Allow to control the number of retries in PostgreSQL connection pool."}
               }},
     {"24.6", {{"materialize_skip_indexes_on_insert", true, true, "Added new setting to allow to disable materialization of skip indexes on insert"},
               {"materialize_statistics_on_insert", true, true, "Added new setting to allow to disable materialization of statistics on insert"},
diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp
index b8646e1a971..427378decd5 100644
--- a/src/Interpreters/InterpreterCreateQuery.cpp
+++ b/src/Interpreters/InterpreterCreateQuery.cpp
@@ -959,40 +959,12 @@ namespace
         engine_ast->no_empty_args = true;
         storage.set(storage.engine, engine_ast);
     }
-
-    void setNullTableEngine(ASTStorage & storage)
-    {
-        auto engine_ast = std::make_shared<ASTFunction>();
-        engine_ast->name = "Null";
-        engine_ast->no_empty_args = true;
-        storage.set(storage.engine, engine_ast);
-    }
-
 }
 
 void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const
 {
     if (create.as_table_function)
-    {
-        if (getContext()->getSettingsRef().restore_replace_external_table_functions_to_null)
-        {
-            const auto & factory = TableFunctionFactory::instance();
-
-            auto properties = factory.tryGetProperties(create.as_table_function->as<ASTFunction>()->name);
-            if (properties && properties->allow_readonly)
-                return;
-            if (!create.storage)
-            {
-                auto storage_ast = std::make_shared<ASTStorage>();
-                create.set(create.storage, storage_ast);
-            }
-            else
-                throw Exception(ErrorCodes::LOGICAL_ERROR, "Storage should not be created yet, it's a bug.");
-            create.as_table_function = nullptr;
-            setNullTableEngine(*create.storage);
-        }
         return;
-    }
 
     if (create.is_dictionary || create.is_ordinary_view || create.is_live_view || create.is_window_view)
         return;
@@ -1043,13 +1015,6 @@ void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const
             /// Some part of storage definition (such as PARTITION BY) is specified, but ENGINE is not: just set default one.
             setDefaultTableEngine(*create.storage, getContext()->getSettingsRef().default_table_engine.value);
         }
-        /// For external tables with restore_replace_external_engine_to_null setting we replace external engines to
-        /// Null table engine.
-        else if (getContext()->getSettingsRef().restore_replace_external_engines_to_null)
-        {
-            if (StorageFactory::instance().getStorageFeatures(create.storage->engine->name).source_access_type != AccessType::NONE)
-                setNullTableEngine(*create.storage);
-        }
         return;
     }
 
diff --git a/tests/integration/test_restore_external_engines/__init__.py b/tests/integration/test_restore_external_engines/__init__.py
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/tests/integration/test_restore_external_engines/configs/backups_disk.xml b/tests/integration/test_restore_external_engines/configs/backups_disk.xml
deleted file mode 100644
index f7d666c6542..00000000000
--- a/tests/integration/test_restore_external_engines/configs/backups_disk.xml
+++ /dev/null
@@ -1,14 +0,0 @@
-<clickhouse>
-    <storage_configuration>
-        <disks>
-            <backups>
-                <type>local</type>
-                <path>/backups/</path>
-            </backups>
-        </disks>
-    </storage_configuration>
-    <backups>
-        <allowed_disk>backups</allowed_disk>
-        <allowed_path>/backups/</allowed_path>
-    </backups>
-</clickhouse>
diff --git a/tests/integration/test_restore_external_engines/configs/remote_servers.xml b/tests/integration/test_restore_external_engines/configs/remote_servers.xml
deleted file mode 100644
index 76ad3618339..00000000000
--- a/tests/integration/test_restore_external_engines/configs/remote_servers.xml
+++ /dev/null
@@ -1,21 +0,0 @@
-<clickhouse>
-    <remote_servers>
-        <test_cluster>
-            <shard>
-                <internal_replication>true</internal_replication>
-                <replica>
-                    <host>replica1</host>
-                    <port>9000</port>
-                </replica>
-                <replica>
-                    <host>replica2</host>
-                    <port>9000</port>
-                </replica>
-                 <replica>
-                    <host>replica3</host>
-                    <port>9000</port>
-                </replica>
-            </shard>
-        </test_cluster>
-    </remote_servers>
-</clickhouse>
diff --git a/tests/integration/test_restore_external_engines/test.py b/tests/integration/test_restore_external_engines/test.py
deleted file mode 100644
index cf189f2a6ed..00000000000
--- a/tests/integration/test_restore_external_engines/test.py
+++ /dev/null
@@ -1,218 +0,0 @@
-import pytest
-
-import pymysql.cursors
-import pytest
-from helpers.cluster import ClickHouseCluster
-
-cluster = ClickHouseCluster(__file__)
-configs = ["configs/remote_servers.xml", "configs/backups_disk.xml"]
-
-node1 = cluster.add_instance(
-    "replica1",
-    with_zookeeper=True,
-    with_mysql8=True,
-    main_configs=configs,
-    external_dirs=["/backups/"],
-)
-node2 = cluster.add_instance(
-    "replica2",
-    with_zookeeper=True,
-    with_mysql8=True,
-    main_configs=configs,
-    external_dirs=["/backups/"],
-)
-node3 = cluster.add_instance(
-    "replica3",
-    with_zookeeper=True,
-    with_mysql8=True,
-    main_configs=configs,
-    external_dirs=["/backups/"],
-)
-nodes = [node1, node2, node3]
-
-backup_id_counter = 0
-
-
-def new_backup_name():
-    global backup_id_counter
-    backup_id_counter += 1
-    return f"Disk('backups', '{backup_id_counter}/')"
-
-
-def cleanup_nodes(nodes, dbname):
-    for node in nodes:
-        node.query(f"DROP DATABASE IF EXISTS {dbname} SYNC")
-
-
-def fill_nodes(nodes, dbname):
-    cleanup_nodes(nodes, dbname)
-    for node in nodes:
-        node.query(
-            f"CREATE DATABASE {dbname} ENGINE = Replicated('/clickhouse/databases/{dbname}', 'default', '{node.name}')"
-        )
-
-
-def drop_mysql_table(conn, tableName):
-    with conn.cursor() as cursor:
-        cursor.execute(f"DROP TABLE IF EXISTS `clickhouse`.`{tableName}`")
-
-
-def get_mysql_conn(cluster):
-    conn = pymysql.connect(
-        user="root",
-        password="clickhouse",
-        host=cluster.mysql8_ip,
-        port=cluster.mysql8_port,
-    )
-    return conn
-
-
-def fill_tables(cluster, dbname):
-    fill_nodes(nodes, dbname)
-
-    conn = get_mysql_conn(cluster)
-
-    with conn.cursor() as cursor:
-        cursor.execute("DROP DATABASE IF EXISTS clickhouse")
-        cursor.execute("CREATE DATABASE clickhouse")
-        cursor.execute("DROP TABLE IF EXISTS clickhouse.inference_table")
-        cursor.execute(
-            "CREATE TABLE clickhouse.inference_table (id INT PRIMARY KEY, data BINARY(16) NOT NULL)"
-        )
-        cursor.execute(
-            "INSERT INTO clickhouse.inference_table VALUES (100, X'9fad5e9eefdfb449')"
-        )
-        conn.commit()
-
-    parameters = "'mysql80:3306', 'clickhouse', 'inference_table', 'root', 'clickhouse'"
-
-    node1.query(
-        f"CREATE TABLE {dbname}.mysql_schema_inference_engine ENGINE=MySQL({parameters})"
-    )
-    node1.query(
-        f"CREATE TABLE {dbname}.mysql_schema_inference_function AS mysql({parameters})"
-    )
-
-    node1.query(f"CREATE TABLE {dbname}.merge_tree (id UInt64, b String) ORDER BY id")
-    node1.query(f"INSERT INTO {dbname}.merge_tree VALUES (100, 'abc')")
-
-    expected = "id\tInt32\t\t\t\t\t\ndata\tFixedString(16)\t\t\t\t\t\n"
-    assert (
-        node1.query(f"DESCRIBE TABLE {dbname}.mysql_schema_inference_engine")
-        == expected
-    )
-    assert (
-        node1.query(f"DESCRIBE TABLE {dbname}.mysql_schema_inference_function")
-        == expected
-    )
-    assert node1.query(f"SELECT id FROM mysql({parameters})") == "100\n"
-    assert (
-        node1.query(f"SELECT id FROM {dbname}.mysql_schema_inference_engine") == "100\n"
-    )
-    assert (
-        node1.query(f"SELECT id FROM {dbname}.mysql_schema_inference_function")
-        == "100\n"
-    )
-    assert node1.query(f"SELECT id FROM {dbname}.merge_tree") == "100\n"
-
-
-@pytest.fixture(scope="module")
-def start_cluster():
-    try:
-        cluster.start()
-        yield cluster
-
-    except Exception as ex:
-        print(ex)
-
-    finally:
-        cluster.shutdown()
-
-
-def test_restore_table(start_cluster):
-    fill_tables(cluster, "replicated")
-    backup_name = new_backup_name()
-    node2.query(f"SYSTEM SYNC DATABASE REPLICA replicated")
-
-    node2.query(f"BACKUP DATABASE replicated TO {backup_name}")
-
-    node2.query("DROP TABLE replicated.mysql_schema_inference_engine")
-    node2.query("DROP TABLE replicated.mysql_schema_inference_function")
-
-    node3.query(f"SYSTEM SYNC DATABASE REPLICA replicated")
-
-    assert node3.query("EXISTS replicated.mysql_schema_inference_engine") == "0\n"
-    assert node3.query("EXISTS replicated.mysql_schema_inference_function") == "0\n"
-
-    node3.query(
-        f"RESTORE DATABASE replicated FROM {backup_name} SETTINGS allow_different_database_def=true"
-    )
-    node1.query(f"SYSTEM SYNC DATABASE REPLICA replicated")
-
-    assert (
-        node1.query(
-            "SELECT count(), sum(id) FROM replicated.mysql_schema_inference_engine"
-        )
-        == "1\t100\n"
-    )
-    assert (
-        node1.query(
-            "SELECT count(), sum(id) FROM replicated.mysql_schema_inference_function"
-        )
-        == "1\t100\n"
-    )
-    assert (
-        node1.query("SELECT count(), sum(id) FROM replicated.merge_tree") == "1\t100\n"
-    )
-    cleanup_nodes(nodes, "replicated")
-
-
-def test_restore_table_null(start_cluster):
-    fill_tables(cluster, "replicated2")
-
-    backup_name = new_backup_name()
-    node2.query(f"SYSTEM SYNC DATABASE REPLICA replicated2")
-
-    node2.query(f"BACKUP DATABASE replicated2 TO {backup_name}")
-
-    node2.query("DROP TABLE replicated2.mysql_schema_inference_engine")
-    node2.query("DROP TABLE replicated2.mysql_schema_inference_function")
-
-    node3.query(f"SYSTEM SYNC DATABASE REPLICA replicated2")
-
-    assert node3.query("EXISTS replicated2.mysql_schema_inference_engine") == "0\n"
-    assert node3.query("EXISTS replicated2.mysql_schema_inference_function") == "0\n"
-
-    node3.query(
-        f"RESTORE DATABASE replicated2 FROM {backup_name} SETTINGS allow_different_database_def=1, allow_different_table_def=1 SETTINGS restore_replace_external_engines_to_null=1, restore_replace_external_table_functions_to_null=1"
-    )
-    node1.query(f"SYSTEM SYNC DATABASE REPLICA replicated2")
-
-    assert (
-        node1.query(
-            "SELECT count(), sum(id) FROM replicated2.mysql_schema_inference_engine"
-        )
-        == "0\t0\n"
-    )
-    assert (
-        node1.query(
-            "SELECT count(), sum(id) FROM replicated2.mysql_schema_inference_function"
-        )
-        == "0\t0\n"
-    )
-    assert (
-        node1.query("SELECT count(), sum(id) FROM replicated2.merge_tree") == "1\t100\n"
-    )
-    assert (
-        node1.query(
-            "SELECT engine FROM system.tables where database = 'replicated2' and name like '%mysql%'"
-        )
-        == "Null\nNull\n"
-    )
-    assert (
-        node1.query(
-            "SELECT engine FROM system.tables where database = 'replicated2' and name like '%merge_tree%'"
-        )
-        == "MergeTree\n"
-    )
-    cleanup_nodes(nodes, "replicated2")

From 06863cf4157765c04759109afa756022dc5e9c55 Mon Sep 17 00:00:00 2001
From: maxvostrikov <max.vostrikov@clickhouse.com>
Date: Wed, 31 Jul 2024 12:12:30 +0200
Subject: [PATCH 1100/1488] fix for allow_experimental_analyzer

---
 tests/queries/0_stateless/00309_formats.sql | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/queries/0_stateless/00309_formats.sql b/tests/queries/0_stateless/00309_formats.sql
index 0366cdeea5c..b784907be08 100644
--- a/tests/queries/0_stateless/00309_formats.sql
+++ b/tests/queries/0_stateless/00309_formats.sql
@@ -12,5 +12,5 @@ SELECT number * 246 + 10 AS n, toDate('2000-01-01') + n AS d, range(n) AS arr, a
 
 SET enable_named_columns_in_function_tuple = 1;
 
-SELECT 36 AS n, toDate('2000-01-01') + n AS d, (n, d) AS tuple FROM system.numbers LIMIT 1 FORMAT RowBinaryWithNamesAndTypes;
-SELECT number * 246 + 10 AS n, toDate('2000-01-01') + n AS d, (n, d) AS tuple FROM system.numbers LIMIT 1 FORMAT TabSeparatedWithNamesAndTypes;
+SELECT 36 AS n, toDate('2000-01-01') + n AS d, (n, d) AS tuple FROM system.numbers LIMIT 1 FORMAT RowBinaryWithNamesAndTypes SETTINGS allow_experimental_analyzer=1;
+SELECT number * 246 + 10 AS n, toDate('2000-01-01') + n AS d, (n, d) AS tuple FROM system.numbers LIMIT 1 FORMAT TabSeparatedWithNamesAndTypes SETTINGS allow_experimental_analyzer=1;

From debcc2e61053f763cb84e34e48275dbebd5bd544 Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitlibar@clickhouse.com>
Date: Wed, 31 Jul 2024 10:46:19 +0200
Subject: [PATCH 1101/1488] Fix test test_mutation

---
 tests/integration/test_backup_restore_on_cluster/test.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tests/integration/test_backup_restore_on_cluster/test.py b/tests/integration/test_backup_restore_on_cluster/test.py
index 1b7f4aaa97d..d20e10e8a04 100644
--- a/tests/integration/test_backup_restore_on_cluster/test.py
+++ b/tests/integration/test_backup_restore_on_cluster/test.py
@@ -1054,9 +1054,12 @@ def test_mutation():
     backup_name = new_backup_name()
     node1.query(f"BACKUP TABLE tbl ON CLUSTER 'cluster' TO {backup_name}")
 
-    assert not has_mutation_in_backup("0000000000", backup_name, "default", "tbl")
+    # mutation #0000000000: "UPDATE x=x+1 WHERE 1" could already finish before starting the backup
+    # mutation #0000000001: "UPDATE x=x+1+sleep(3) WHERE 1"
     assert has_mutation_in_backup("0000000001", backup_name, "default", "tbl")
+    # mutation #0000000002: "UPDATE x=x+1+sleep(3) WHERE 1"
     assert has_mutation_in_backup("0000000002", backup_name, "default", "tbl")
+    # mutation #0000000003: not expected
     assert not has_mutation_in_backup("0000000003", backup_name, "default", "tbl")
 
     node1.query("DROP TABLE tbl ON CLUSTER 'cluster' SYNC")

From f9f17fb61e2ab27f90434b5e3fc9081c061eaae4 Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Wed, 31 Jul 2024 12:32:17 +0200
Subject: [PATCH 1102/1488] Fix reference

---
 .../0_stateless/03215_parsing_archive_name_s3.reference     | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/queries/0_stateless/03215_parsing_archive_name_s3.reference b/tests/queries/0_stateless/03215_parsing_archive_name_s3.reference
index b4804c82dc2..b27524812c7 100644
--- a/tests/queries/0_stateless/03215_parsing_archive_name_s3.reference
+++ b/tests/queries/0_stateless/03215_parsing_archive_name_s3.reference
@@ -1,3 +1,3 @@
-::03215_archive.csv test/::03215_archive.csv
-test::03215_archive.csv test/test::03215_archive.csv
-test.zip::03215_archive.csv test/test.zip::03215_archive.csv
+::03215_archive.csv	test/::03215_archive.csv
+test::03215_archive.csv	test/test::03215_archive.csv
+test.zip::03215_archive.csv	test/test.zip::03215_archive.csv

From 4bf7aa1950f65aa82e85962ab3643f7df0e8bf2a Mon Sep 17 00:00:00 2001
From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com>
Date: Wed, 31 Jul 2024 13:30:14 +0200
Subject: [PATCH 1103/1488] Revert "Revert "Add settings to replace external
 engines to Null during create""

---
 docs/en/operations/settings/settings.md       |  12 +
 src/Core/Settings.h                           |   2 +
 src/Core/SettingsChangesHistory.cpp           |   4 +-
 src/Interpreters/InterpreterCreateQuery.cpp   |  35 +++
 .../test_restore_external_engines/__init__.py |   0
 .../configs/backups_disk.xml                  |  14 ++
 .../configs/remote_servers.xml                |  21 ++
 .../test_restore_external_engines/test.py     | 218 ++++++++++++++++++
 8 files changed, 305 insertions(+), 1 deletion(-)
 create mode 100644 tests/integration/test_restore_external_engines/__init__.py
 create mode 100644 tests/integration/test_restore_external_engines/configs/backups_disk.xml
 create mode 100644 tests/integration/test_restore_external_engines/configs/remote_servers.xml
 create mode 100644 tests/integration/test_restore_external_engines/test.py

diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md
index c3f697c3bdc..8739414464e 100644
--- a/docs/en/operations/settings/settings.md
+++ b/docs/en/operations/settings/settings.md
@@ -5608,3 +5608,15 @@ Default value: `10000000`.
 Minimal size of block to compress in CROSS JOIN. Zero value means - disable this threshold. This block is compressed when any of the two thresholds (by rows or by bytes) are reached.
 
 Default value: `1GiB`.
+
+## restore_replace_external_engines_to_null
+
+For testing purposes. Replaces all external engines to Null to not initiate external connections.
+
+Default value: `False`
+
+## restore_replace_external_table_functions_to_null
+
+For testing purposes. Replaces all external table functions to Null to not initiate external connections.
+
+Default value: `False`
\ No newline at end of file
diff --git a/src/Core/Settings.h b/src/Core/Settings.h
index 4fc2034b855..27b71558bd3 100644
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@@ -893,6 +893,8 @@ class IColumn;
     M(Bool, optimize_distinct_in_order, true, "Enable DISTINCT optimization if some columns in DISTINCT form a prefix of sorting. For example, prefix of sorting key in merge tree or ORDER BY statement", 0) \
     M(Bool, keeper_map_strict_mode, false, "Enforce additional checks during operations on KeeperMap. E.g. throw an exception on an insert for already existing key", 0) \
     M(UInt64, extract_key_value_pairs_max_pairs_per_row, 1000, "Max number of pairs that can be produced by the `extractKeyValuePairs` function. Used as a safeguard against consuming too much memory.", 0) ALIAS(extract_kvp_max_pairs_per_row) \
+    M(Bool, restore_replace_external_engines_to_null, false, "Replace all the external table engines to Null on restore. Useful for testing purposes", 0) \
+    M(Bool, restore_replace_external_table_functions_to_null, false, "Replace all table functions to Null on restore. Useful for testing purposes", 0) \
     \
     \
     /* ###################################### */ \
diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp
index 9faf77e9087..8bea0b1eed3 100644
--- a/src/Core/SettingsChangesHistory.cpp
+++ b/src/Core/SettingsChangesHistory.cpp
@@ -80,7 +80,9 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
               {"ignore_on_cluster_for_replicated_named_collections_queries", false, false, "Ignore ON CLUSTER clause for replicated named collections management queries."},
               {"backup_restore_s3_retry_attempts", 1000,1000, "Setting for Aws::Client::RetryStrategy, Aws::Client does retries itself, 0 means no retries. It takes place only for backup/restore."},
               {"postgresql_connection_attempt_timeout", 2, 2, "Allow to control 'connect_timeout' parameter of PostgreSQL connection."},
-              {"postgresql_connection_pool_retries", 2, 2, "Allow to control the number of retries in PostgreSQL connection pool."}
+              {"postgresql_connection_pool_retries", 2, 2, "Allow to control the number of retries in PostgreSQL connection pool."},
+              {"restore_replace_external_table_functions_to_null", false, false, "New setting."},
+              {"restore_replace_external_engines_to_null", false, false, "New setting."}
               }},
     {"24.6", {{"materialize_skip_indexes_on_insert", true, true, "Added new setting to allow to disable materialization of skip indexes on insert"},
               {"materialize_statistics_on_insert", true, true, "Added new setting to allow to disable materialization of statistics on insert"},
diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp
index 427378decd5..b8646e1a971 100644
--- a/src/Interpreters/InterpreterCreateQuery.cpp
+++ b/src/Interpreters/InterpreterCreateQuery.cpp
@@ -959,12 +959,40 @@ namespace
         engine_ast->no_empty_args = true;
         storage.set(storage.engine, engine_ast);
     }
+
+    void setNullTableEngine(ASTStorage & storage)
+    {
+        auto engine_ast = std::make_shared<ASTFunction>();
+        engine_ast->name = "Null";
+        engine_ast->no_empty_args = true;
+        storage.set(storage.engine, engine_ast);
+    }
+
 }
 
 void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const
 {
     if (create.as_table_function)
+    {
+        if (getContext()->getSettingsRef().restore_replace_external_table_functions_to_null)
+        {
+            const auto & factory = TableFunctionFactory::instance();
+
+            auto properties = factory.tryGetProperties(create.as_table_function->as<ASTFunction>()->name);
+            if (properties && properties->allow_readonly)
+                return;
+            if (!create.storage)
+            {
+                auto storage_ast = std::make_shared<ASTStorage>();
+                create.set(create.storage, storage_ast);
+            }
+            else
+                throw Exception(ErrorCodes::LOGICAL_ERROR, "Storage should not be created yet, it's a bug.");
+            create.as_table_function = nullptr;
+            setNullTableEngine(*create.storage);
+        }
         return;
+    }
 
     if (create.is_dictionary || create.is_ordinary_view || create.is_live_view || create.is_window_view)
         return;
@@ -1015,6 +1043,13 @@ void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const
             /// Some part of storage definition (such as PARTITION BY) is specified, but ENGINE is not: just set default one.
             setDefaultTableEngine(*create.storage, getContext()->getSettingsRef().default_table_engine.value);
         }
+        /// For external tables with restore_replace_external_engine_to_null setting we replace external engines to
+        /// Null table engine.
+        else if (getContext()->getSettingsRef().restore_replace_external_engines_to_null)
+        {
+            if (StorageFactory::instance().getStorageFeatures(create.storage->engine->name).source_access_type != AccessType::NONE)
+                setNullTableEngine(*create.storage);
+        }
         return;
     }
 
diff --git a/tests/integration/test_restore_external_engines/__init__.py b/tests/integration/test_restore_external_engines/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/integration/test_restore_external_engines/configs/backups_disk.xml b/tests/integration/test_restore_external_engines/configs/backups_disk.xml
new file mode 100644
index 00000000000..f7d666c6542
--- /dev/null
+++ b/tests/integration/test_restore_external_engines/configs/backups_disk.xml
@@ -0,0 +1,14 @@
+<clickhouse>
+    <storage_configuration>
+        <disks>
+            <backups>
+                <type>local</type>
+                <path>/backups/</path>
+            </backups>
+        </disks>
+    </storage_configuration>
+    <backups>
+        <allowed_disk>backups</allowed_disk>
+        <allowed_path>/backups/</allowed_path>
+    </backups>
+</clickhouse>
diff --git a/tests/integration/test_restore_external_engines/configs/remote_servers.xml b/tests/integration/test_restore_external_engines/configs/remote_servers.xml
new file mode 100644
index 00000000000..76ad3618339
--- /dev/null
+++ b/tests/integration/test_restore_external_engines/configs/remote_servers.xml
@@ -0,0 +1,21 @@
+<clickhouse>
+    <remote_servers>
+        <test_cluster>
+            <shard>
+                <internal_replication>true</internal_replication>
+                <replica>
+                    <host>replica1</host>
+                    <port>9000</port>
+                </replica>
+                <replica>
+                    <host>replica2</host>
+                    <port>9000</port>
+                </replica>
+                 <replica>
+                    <host>replica3</host>
+                    <port>9000</port>
+                </replica>
+            </shard>
+        </test_cluster>
+    </remote_servers>
+</clickhouse>
diff --git a/tests/integration/test_restore_external_engines/test.py b/tests/integration/test_restore_external_engines/test.py
new file mode 100644
index 00000000000..cf189f2a6ed
--- /dev/null
+++ b/tests/integration/test_restore_external_engines/test.py
@@ -0,0 +1,218 @@
+import pytest
+
+import pymysql.cursors
+import pytest
+from helpers.cluster import ClickHouseCluster
+
+cluster = ClickHouseCluster(__file__)
+configs = ["configs/remote_servers.xml", "configs/backups_disk.xml"]
+
+node1 = cluster.add_instance(
+    "replica1",
+    with_zookeeper=True,
+    with_mysql8=True,
+    main_configs=configs,
+    external_dirs=["/backups/"],
+)
+node2 = cluster.add_instance(
+    "replica2",
+    with_zookeeper=True,
+    with_mysql8=True,
+    main_configs=configs,
+    external_dirs=["/backups/"],
+)
+node3 = cluster.add_instance(
+    "replica3",
+    with_zookeeper=True,
+    with_mysql8=True,
+    main_configs=configs,
+    external_dirs=["/backups/"],
+)
+nodes = [node1, node2, node3]
+
+backup_id_counter = 0
+
+
+def new_backup_name():
+    global backup_id_counter
+    backup_id_counter += 1
+    return f"Disk('backups', '{backup_id_counter}/')"
+
+
+def cleanup_nodes(nodes, dbname):
+    for node in nodes:
+        node.query(f"DROP DATABASE IF EXISTS {dbname} SYNC")
+
+
+def fill_nodes(nodes, dbname):
+    cleanup_nodes(nodes, dbname)
+    for node in nodes:
+        node.query(
+            f"CREATE DATABASE {dbname} ENGINE = Replicated('/clickhouse/databases/{dbname}', 'default', '{node.name}')"
+        )
+
+
+def drop_mysql_table(conn, tableName):
+    with conn.cursor() as cursor:
+        cursor.execute(f"DROP TABLE IF EXISTS `clickhouse`.`{tableName}`")
+
+
+def get_mysql_conn(cluster):
+    conn = pymysql.connect(
+        user="root",
+        password="clickhouse",
+        host=cluster.mysql8_ip,
+        port=cluster.mysql8_port,
+    )
+    return conn
+
+
+def fill_tables(cluster, dbname):
+    fill_nodes(nodes, dbname)
+
+    conn = get_mysql_conn(cluster)
+
+    with conn.cursor() as cursor:
+        cursor.execute("DROP DATABASE IF EXISTS clickhouse")
+        cursor.execute("CREATE DATABASE clickhouse")
+        cursor.execute("DROP TABLE IF EXISTS clickhouse.inference_table")
+        cursor.execute(
+            "CREATE TABLE clickhouse.inference_table (id INT PRIMARY KEY, data BINARY(16) NOT NULL)"
+        )
+        cursor.execute(
+            "INSERT INTO clickhouse.inference_table VALUES (100, X'9fad5e9eefdfb449')"
+        )
+        conn.commit()
+
+    parameters = "'mysql80:3306', 'clickhouse', 'inference_table', 'root', 'clickhouse'"
+
+    node1.query(
+        f"CREATE TABLE {dbname}.mysql_schema_inference_engine ENGINE=MySQL({parameters})"
+    )
+    node1.query(
+        f"CREATE TABLE {dbname}.mysql_schema_inference_function AS mysql({parameters})"
+    )
+
+    node1.query(f"CREATE TABLE {dbname}.merge_tree (id UInt64, b String) ORDER BY id")
+    node1.query(f"INSERT INTO {dbname}.merge_tree VALUES (100, 'abc')")
+
+    expected = "id\tInt32\t\t\t\t\t\ndata\tFixedString(16)\t\t\t\t\t\n"
+    assert (
+        node1.query(f"DESCRIBE TABLE {dbname}.mysql_schema_inference_engine")
+        == expected
+    )
+    assert (
+        node1.query(f"DESCRIBE TABLE {dbname}.mysql_schema_inference_function")
+        == expected
+    )
+    assert node1.query(f"SELECT id FROM mysql({parameters})") == "100\n"
+    assert (
+        node1.query(f"SELECT id FROM {dbname}.mysql_schema_inference_engine") == "100\n"
+    )
+    assert (
+        node1.query(f"SELECT id FROM {dbname}.mysql_schema_inference_function")
+        == "100\n"
+    )
+    assert node1.query(f"SELECT id FROM {dbname}.merge_tree") == "100\n"
+
+
+@pytest.fixture(scope="module")
+def start_cluster():
+    try:
+        cluster.start()
+        yield cluster
+
+    except Exception as ex:
+        print(ex)
+
+    finally:
+        cluster.shutdown()
+
+
+def test_restore_table(start_cluster):
+    fill_tables(cluster, "replicated")
+    backup_name = new_backup_name()
+    node2.query(f"SYSTEM SYNC DATABASE REPLICA replicated")
+
+    node2.query(f"BACKUP DATABASE replicated TO {backup_name}")
+
+    node2.query("DROP TABLE replicated.mysql_schema_inference_engine")
+    node2.query("DROP TABLE replicated.mysql_schema_inference_function")
+
+    node3.query(f"SYSTEM SYNC DATABASE REPLICA replicated")
+
+    assert node3.query("EXISTS replicated.mysql_schema_inference_engine") == "0\n"
+    assert node3.query("EXISTS replicated.mysql_schema_inference_function") == "0\n"
+
+    node3.query(
+        f"RESTORE DATABASE replicated FROM {backup_name} SETTINGS allow_different_database_def=true"
+    )
+    node1.query(f"SYSTEM SYNC DATABASE REPLICA replicated")
+
+    assert (
+        node1.query(
+            "SELECT count(), sum(id) FROM replicated.mysql_schema_inference_engine"
+        )
+        == "1\t100\n"
+    )
+    assert (
+        node1.query(
+            "SELECT count(), sum(id) FROM replicated.mysql_schema_inference_function"
+        )
+        == "1\t100\n"
+    )
+    assert (
+        node1.query("SELECT count(), sum(id) FROM replicated.merge_tree") == "1\t100\n"
+    )
+    cleanup_nodes(nodes, "replicated")
+
+
+def test_restore_table_null(start_cluster):
+    fill_tables(cluster, "replicated2")
+
+    backup_name = new_backup_name()
+    node2.query(f"SYSTEM SYNC DATABASE REPLICA replicated2")
+
+    node2.query(f"BACKUP DATABASE replicated2 TO {backup_name}")
+
+    node2.query("DROP TABLE replicated2.mysql_schema_inference_engine")
+    node2.query("DROP TABLE replicated2.mysql_schema_inference_function")
+
+    node3.query(f"SYSTEM SYNC DATABASE REPLICA replicated2")
+
+    assert node3.query("EXISTS replicated2.mysql_schema_inference_engine") == "0\n"
+    assert node3.query("EXISTS replicated2.mysql_schema_inference_function") == "0\n"
+
+    node3.query(
+        f"RESTORE DATABASE replicated2 FROM {backup_name} SETTINGS allow_different_database_def=1, allow_different_table_def=1 SETTINGS restore_replace_external_engines_to_null=1, restore_replace_external_table_functions_to_null=1"
+    )
+    node1.query(f"SYSTEM SYNC DATABASE REPLICA replicated2")
+
+    assert (
+        node1.query(
+            "SELECT count(), sum(id) FROM replicated2.mysql_schema_inference_engine"
+        )
+        == "0\t0\n"
+    )
+    assert (
+        node1.query(
+            "SELECT count(), sum(id) FROM replicated2.mysql_schema_inference_function"
+        )
+        == "0\t0\n"
+    )
+    assert (
+        node1.query("SELECT count(), sum(id) FROM replicated2.merge_tree") == "1\t100\n"
+    )
+    assert (
+        node1.query(
+            "SELECT engine FROM system.tables where database = 'replicated2' and name like '%mysql%'"
+        )
+        == "Null\nNull\n"
+    )
+    assert (
+        node1.query(
+            "SELECT engine FROM system.tables where database = 'replicated2' and name like '%merge_tree%'"
+        )
+        == "MergeTree\n"
+    )
+    cleanup_nodes(nodes, "replicated2")

From c81d3322b18b0eb4b45b91ac019a8c4f42d7518d Mon Sep 17 00:00:00 2001
From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com>
Date: Wed, 31 Jul 2024 13:39:30 +0200
Subject: [PATCH 1104/1488] Update 02150_index_hypothesis_race_long.sh

---
 tests/queries/0_stateless/02150_index_hypothesis_race_long.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/02150_index_hypothesis_race_long.sh b/tests/queries/0_stateless/02150_index_hypothesis_race_long.sh
index c29b604d23d..5c432350768 100755
--- a/tests/queries/0_stateless/02150_index_hypothesis_race_long.sh
+++ b/tests/queries/0_stateless/02150_index_hypothesis_race_long.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Tags: long, no-distributed-cache
+# Tags: long, no-random-settings, no-distributed-cache
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh

From 64a7a413619cda2edb336079740cca2a3d6503f7 Mon Sep 17 00:00:00 2001
From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com>
Date: Wed, 31 Jul 2024 13:42:38 +0200
Subject: [PATCH 1105/1488] Update SettingsChangesHistory.cpp

---
 src/Core/SettingsChangesHistory.cpp | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp
index 8bea0b1eed3..5e846868478 100644
--- a/src/Core/SettingsChangesHistory.cpp
+++ b/src/Core/SettingsChangesHistory.cpp
@@ -57,6 +57,9 @@ String ClickHouseVersion::toString() const
 /// Note: please check if the key already exists to prevent duplicate entries.
 static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory::SettingsChanges>> settings_changes_history_initializer =
 {
+    {"24.8", {{"restore_replace_external_table_functions_to_null", false, false, "New setting."},
+              {"restore_replace_external_engines_to_null", false, false, "New setting."}
+              }},
     {"24.7", {{"output_format_parquet_write_page_index", false, true, "Add a possibility to write page index into parquet files."},
               {"output_format_binary_encode_types_in_binary_format", false, false, "Added new setting to allow to write type names in binary format in RowBinaryWithNamesAndTypes output format"},
               {"input_format_binary_decode_types_in_binary_format", false, false, "Added new setting to allow to read type names in binary format in RowBinaryWithNamesAndTypes input format"},
@@ -80,9 +83,7 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
               {"ignore_on_cluster_for_replicated_named_collections_queries", false, false, "Ignore ON CLUSTER clause for replicated named collections management queries."},
               {"backup_restore_s3_retry_attempts", 1000,1000, "Setting for Aws::Client::RetryStrategy, Aws::Client does retries itself, 0 means no retries. It takes place only for backup/restore."},
               {"postgresql_connection_attempt_timeout", 2, 2, "Allow to control 'connect_timeout' parameter of PostgreSQL connection."},
-              {"postgresql_connection_pool_retries", 2, 2, "Allow to control the number of retries in PostgreSQL connection pool."},
-              {"restore_replace_external_table_functions_to_null", false, false, "New setting."},
-              {"restore_replace_external_engines_to_null", false, false, "New setting."}
+              {"postgresql_connection_pool_retries", 2, 2, "Allow to control the number of retries in PostgreSQL connection pool."}
               }},
     {"24.6", {{"materialize_skip_indexes_on_insert", true, true, "Added new setting to allow to disable materialization of skip indexes on insert"},
               {"materialize_statistics_on_insert", true, true, "Added new setting to allow to disable materialization of statistics on insert"},

From d5f8540fbbcc7e5f8014212d67804412c7466d7d Mon Sep 17 00:00:00 2001
From: divanik <dan.ivanik@clickhouse.com>
Date: Wed, 31 Jul 2024 12:00:09 +0000
Subject: [PATCH 1106/1488] Fix

---
 tests/integration/helpers/cluster.py         | 25 +++++++++++++-------
 tests/integration/helpers/retry_decorator.py |  7 ++++--
 2 files changed, 21 insertions(+), 11 deletions(-)

diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py
index 7d80fbe90f8..7f0a9154be9 100644
--- a/tests/integration/helpers/cluster.py
+++ b/tests/integration/helpers/cluster.py
@@ -2692,11 +2692,13 @@ class ClickHouseCluster:
             images_pull_cmd = self.base_cmd + ["pull"]
             # sometimes dockerhub/proxy can be flaky
 
-            retry(
-                log_function=lambda exception: logging.info(
-                    "Got exception pulling images: %s", exception
-                ),
-            )(run_and_check)(images_pull_cmd)
+            def logging_pulling_images(**kwargs):
+                if "exception" in kwargs:
+                    logging.info(
+                        "Got exception pulling images: %s", kwargs["exception"]
+                    )
+
+            retry(log_function=logging_pulling_images)(run_and_check)(images_pull_cmd)
 
             if self.with_zookeeper_secure and self.base_zookeeper_cmd:
                 logging.debug("Setup ZooKeeper Secure")
@@ -2969,11 +2971,16 @@ class ClickHouseCluster:
                     "Trying to create Azurite instance by command %s",
                     " ".join(map(str, azurite_start_cmd)),
                 )
-                retry(
-                    log_function=lambda exception: logging.info(
+                def logging_azurite_initialization(exception, retry_number, sleep_time):
+                    logging.info(
                         f"Azurite initialization failed with error: {exception}"
-                    ),
-                )(run_and_check)(azurite_start_cmd)
+                    )
+
+                retry(
+                    log_function=logging_azurite_initialization,
+                )(
+                    run_and_check
+                )(azurite_start_cmd)
                 self.up_called = True
                 logging.info("Trying to connect to Azurite")
                 self.wait_azurite_to_start()
diff --git a/tests/integration/helpers/retry_decorator.py b/tests/integration/helpers/retry_decorator.py
index aaa040464c2..e7bafbe29c1 100644
--- a/tests/integration/helpers/retry_decorator.py
+++ b/tests/integration/helpers/retry_decorator.py
@@ -8,7 +8,7 @@ def retry(
     delay: float = 1,
     backoff: float = 1.5,
     jitter: float = 2,
-    log_function=lambda *args, **kwargs: None,
+    log_function=None,  # should take **kwargs or arguments: `retry_number`, `exception` and `sleep_time`
     retriable_expections_list: List[Type[BaseException]] = [Exception],
 ):
     def inner(func):
@@ -26,8 +26,11 @@ def retry(
                             break
                     if not should_retry or (retry == retries - 1):
                         raise e
-                    log_function(retry=retry, exception=e)
                     sleep_time = current_delay + random.uniform(0, jitter)
+                    if log_function is not None:
+                        log_function(
+                            retry_number=retry, exception=e, sleep_time=sleep_time
+                        )
                     time.sleep(sleep_time)
                     current_delay *= backoff
 

From 15e0033016eb0e23a7e6f512d5096e50863e3187 Mon Sep 17 00:00:00 2001
From: Pablo Marcos <pablo.marcos.oltra@clickhouse.com>
Date: Wed, 31 Jul 2024 12:09:49 +0000
Subject: [PATCH 1107/1488] Bring back the strict check

Also update the doc.
---
 tests/integration/README.md                     | 9 +++++----
 tests/integration/test_storage_s3_queue/test.py | 4 ++--
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/tests/integration/README.md b/tests/integration/README.md
index ab984b7bd04..a8deb97b526 100644
--- a/tests/integration/README.md
+++ b/tests/integration/README.md
@@ -124,10 +124,11 @@ You can just open shell inside a container by overwritting the command:
 ### Parallel test execution
 
 On the CI, we run a number of parallel runners (5 at the time of this writing), each on its own
-Docker container. These runner containers spawn more containers for the services needed such as
-ZooKeeper, MySQL, PostgreSQL and minio, among others. Within each runner, tests are parallelized
-using [pytest-xdist](https://pytest-xdist.readthedocs.io/en/stable/). We're using `--dist=loadfile`
-to [distribute the load](https://pytest-xdist.readthedocs.io/en/stable/distribution.html). In the
+Docker container. These runner containers spawn more containers for each test for the services
+needed such as ZooKeeper, MySQL, PostgreSQL and minio, among others. This means that tests do not
+share any services among them. Within each runner, tests are parallelized using
+[pytest-xdist](https://pytest-xdist.readthedocs.io/en/stable/). We're using `--dist=loadfile` to
+[distribute the load](https://pytest-xdist.readthedocs.io/en/stable/distribution.html). In the
 documentation words: this guarantees that all tests in a file run in the same worker. This means
 that any test within the same file will never execute their tests in parallel. They'll be executed
 on the same worker one after the other.
diff --git a/tests/integration/test_storage_s3_queue/test.py b/tests/integration/test_storage_s3_queue/test.py
index e3445d14cdb..9a97e8c23d1 100644
--- a/tests/integration/test_storage_s3_queue/test.py
+++ b/tests/integration/test_storage_s3_queue/test.py
@@ -907,7 +907,7 @@ def test_max_set_age(started_cluster):
     file_with_error = f"max_set_age_fail_{uuid4().hex[:8]}.csv"
     put_s3_file_content(started_cluster, f"{files_path}/{file_with_error}", values_csv)
 
-    wait_for_condition(lambda: failed_count + 1 <= get_object_storage_failures())
+    wait_for_condition(lambda: failed_count + 1 == get_object_storage_failures())
 
     node.query("SYSTEM FLUSH LOGS")
     assert "Cannot parse input" in node.query(
@@ -920,7 +920,7 @@ def test_max_set_age(started_cluster):
         )
     )
 
-    wait_for_condition(lambda: failed_count + 2 <= get_object_storage_failures())
+    wait_for_condition(lambda: failed_count + 2 == get_object_storage_failures())
 
     node.query("SYSTEM FLUSH LOGS")
     assert "Cannot parse input" in node.query(

From 9d14053cfe7867fd688c08b493c264ee679a4a61 Mon Sep 17 00:00:00 2001
From: avogar <avogar@clickhouse.com>
Date: Wed, 31 Jul 2024 12:17:56 +0000
Subject: [PATCH 1108/1488] Proper fix for short circuit execution with nested
 dictGetOrDefaultt

---
 src/Columns/ColumnFunction.cpp                | 26 ++++++++++++++-----
 src/Interpreters/ExpressionActions.cpp        |  4 ---
 ...sted_short_circuit_functions_bug.reference |  2 ++
 ...210_nested_short_circuit_functions_bug.sql |  3 +++
 4 files changed, 24 insertions(+), 11 deletions(-)
 create mode 100644 tests/queries/0_stateless/03210_nested_short_circuit_functions_bug.reference
 create mode 100644 tests/queries/0_stateless/03210_nested_short_circuit_functions_bug.sql

diff --git a/src/Columns/ColumnFunction.cpp b/src/Columns/ColumnFunction.cpp
index fc81efaac0c..18c343c6ca6 100644
--- a/src/Columns/ColumnFunction.cpp
+++ b/src/Columns/ColumnFunction.cpp
@@ -296,16 +296,28 @@ ColumnWithTypeAndName ColumnFunction::reduce() const
                         function->getName(), toString(args), toString(captured));
 
     ColumnsWithTypeAndName columns = captured_columns;
-    IFunction::ShortCircuitSettings settings;
     /// Arguments of lazy executed function can also be lazy executed.
-    /// But we shouldn't execute arguments if this function is short circuit,
-    /// because it will handle lazy executed arguments by itself.
-    if (is_short_circuit_argument && !function->isShortCircuit(settings, args))
+    if (is_short_circuit_argument)
     {
-        for (auto & col : columns)
+        IFunction::ShortCircuitSettings settings;
+        /// We shouldn't execute all arguments if this function is short circuit,
+        /// because it will handle lazy executed arguments by itself.
+        /// Execute only arguments with disabled lazy execution.
+        if (function->isShortCircuit(settings, args))
         {
-            if (const ColumnFunction * arg = checkAndGetShortCircuitArgument(col.column))
-                col = arg->reduce();
+            for (size_t i : settings.arguments_with_disabled_lazy_execution)
+            {
+                if (const ColumnFunction * arg = checkAndGetShortCircuitArgument(columns[i].column))
+                    columns[i] = arg->reduce();
+            }
+        }
+        else
+        {
+            for (auto & col : columns)
+            {
+                if (const ColumnFunction * arg = checkAndGetShortCircuitArgument(col.column))
+                    col = arg->reduce();
+            }
         }
     }
 
diff --git a/src/Interpreters/ExpressionActions.cpp b/src/Interpreters/ExpressionActions.cpp
index d832f568cb8..8993830af14 100644
--- a/src/Interpreters/ExpressionActions.cpp
+++ b/src/Interpreters/ExpressionActions.cpp
@@ -195,10 +195,6 @@ static void setLazyExecutionInfo(
             }
 
             lazy_execution_info.short_circuit_ancestors_info[parent].insert(indexes.begin(), indexes.end());
-            /// After checking arguments_with_disabled_lazy_execution, if there is no relation with parent,
-            /// disable the current node.
-            if (indexes.empty())
-                lazy_execution_info.can_be_lazy_executed = false;
         }
         else
             /// If lazy execution is disabled for one of parents, we should disable it for current node.
diff --git a/tests/queries/0_stateless/03210_nested_short_circuit_functions_bug.reference b/tests/queries/0_stateless/03210_nested_short_circuit_functions_bug.reference
new file mode 100644
index 00000000000..aa47d0d46d4
--- /dev/null
+++ b/tests/queries/0_stateless/03210_nested_short_circuit_functions_bug.reference
@@ -0,0 +1,2 @@
+0
+0
diff --git a/tests/queries/0_stateless/03210_nested_short_circuit_functions_bug.sql b/tests/queries/0_stateless/03210_nested_short_circuit_functions_bug.sql
new file mode 100644
index 00000000000..923f1e3be1f
--- /dev/null
+++ b/tests/queries/0_stateless/03210_nested_short_circuit_functions_bug.sql
@@ -0,0 +1,3 @@
+select if(equals(materialize('abc'), 'aws.lambda.duration'), if(toFloat64(materialize('x86_74')) < 50.0000, 0, 1), 0) settings short_circuit_function_evaluation='enable';
+select if(equals(materialize('abc'), 'aws.lambda.duration'), if(toFloat64(materialize('x86_74')) < 50.0000, 0, 1), 0) settings short_circuit_function_evaluation='force_enable';
+

From 6b7c5eb5da1be1fc31d4ebfd4f0dfa0c6a6e728c Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Wed, 31 Jul 2024 14:09:07 +0200
Subject: [PATCH 1109/1488] Fix drop

---
 src/Storages/StorageKeeperMap.cpp         | 34 +++++++++++++++--------
 src/Storages/StorageKeeperMap.h           | 21 ++++++++++----
 tests/integration/test_keeper_map/test.py |  5 ++--
 3 files changed, 41 insertions(+), 19 deletions(-)

diff --git a/src/Storages/StorageKeeperMap.cpp b/src/Storages/StorageKeeperMap.cpp
index 1559b442e43..0634c7be6ee 100644
--- a/src/Storages/StorageKeeperMap.cpp
+++ b/src/Storages/StorageKeeperMap.cpp
@@ -79,6 +79,7 @@ namespace ErrorCodes
     extern const int LOGICAL_ERROR;
     extern const int LIMIT_EXCEEDED;
     extern const int CANNOT_RESTORE_TABLE;
+    extern const int INVALID_STATE;
 }
 
 namespace
@@ -497,7 +498,7 @@ StorageKeeperMap::StorageKeeperMap(
         }
 
 
-        table_is_valid = true;
+        table_status = TableStatus::VALID;
         /// we are the first table created for the specified Keeper path, i.e. we are the first replica
         return;
     }
@@ -656,7 +657,18 @@ bool StorageKeeperMap::dropTable(zkutil::ZooKeeperPtr zookeeper, const zkutil::E
 
 void StorageKeeperMap::drop()
 {
-    checkTable<true>();
+    auto current_table_status = getTableStatus();
+    if (current_table_status == TableStatus::UNKNOWN)
+    {
+            static constexpr auto error_msg = "Failed to activate table because of connection issues. It will be activated "
+                                              "once a connection is established and metadata is verified";
+            throw Exception(ErrorCodes::INVALID_STATE, error_msg);
+    }
+
+    /// if only column metadata is wrong we can still drop the table correctly
+    if (current_table_status == TableStatus::INVALID_KEEPER_STRUCTURE)
+        return;
+
     auto client = getClient();
 
     // we allow ZNONODE in case we got hardware error on previous drop
@@ -1017,11 +1029,11 @@ UInt64 StorageKeeperMap::keysLimit() const
     return keys_limit;
 }
 
-std::optional<bool> StorageKeeperMap::isTableValid() const
+StorageKeeperMap::TableStatus StorageKeeperMap::getTableStatus() const
 {
     std::lock_guard lock{init_mutex};
-    if (table_is_valid.has_value())
-        return table_is_valid;
+    if (table_status != TableStatus::UNKNOWN)
+        return table_status;
 
     [&]
     {
@@ -1034,7 +1046,7 @@ std::optional<bool> StorageKeeperMap::isTableValid() const
 
             if (metadata_stat.numChildren == 0)
             {
-                table_is_valid = false;
+                table_status = TableStatus::INVALID_KEEPER_STRUCTURE;
                 return;
             }
 
@@ -1045,7 +1057,7 @@ std::optional<bool> StorageKeeperMap::isTableValid() const
                     "Table definition does not match to the one stored in the path {}. Stored definition: {}",
                     zk_root_path,
                     stored_metadata_string);
-                table_is_valid = false;
+                table_status = TableStatus::INVALID_METADATA;
                 return;
             }
 
@@ -1058,7 +1070,7 @@ std::optional<bool> StorageKeeperMap::isTableValid() const
             Coordination::Responses responses;
             client->tryMulti(requests, responses);
 
-            table_is_valid = false;
+            table_status = TableStatus::INVALID_KEEPER_STRUCTURE;
             if (responses[0]->error != Coordination::Error::ZOK)
             {
                 LOG_ERROR(log, "Table node ({}) is missing", zk_table_path);
@@ -1077,18 +1089,18 @@ std::optional<bool> StorageKeeperMap::isTableValid() const
                 return;
             }
 
-            table_is_valid = true;
+            table_status = TableStatus::VALID;
         }
         catch (const Coordination::Exception & e)
         {
             tryLogCurrentException(log);
 
             if (!Coordination::isHardwareError(e.code))
-                table_is_valid = false;
+                table_status = TableStatus::INVALID_KEEPER_STRUCTURE;
         }
     }();
 
-    return table_is_valid;
+    return table_status;
 }
 
 Chunk StorageKeeperMap::getByKeys(const ColumnsWithTypeAndName & keys, PaddedPODArray<UInt8> & null_map, const Names &) const
diff --git a/src/Storages/StorageKeeperMap.h b/src/Storages/StorageKeeperMap.h
index cfbb35ab2fe..8ed348a4f6f 100644
--- a/src/Storages/StorageKeeperMap.h
+++ b/src/Storages/StorageKeeperMap.h
@@ -80,8 +80,8 @@ public:
     template <bool throw_on_error>
     void checkTable() const
     {
-        auto is_table_valid = isTableValid();
-        if (!is_table_valid.has_value())
+        auto current_table_status = getTableStatus();
+        if (table_status == TableStatus::UNKNOWN)
         {
             static constexpr auto error_msg = "Failed to activate table because of connection issues. It will be activated "
                                                           "once a connection is established and metadata is verified";
@@ -94,10 +94,10 @@ public:
             }
         }
 
-        if (!*is_table_valid)
+        if (current_table_status != TableStatus::VALID)
         {
             static constexpr auto error_msg
-                = "Failed to activate table because of invalid metadata in ZooKeeper. Please DETACH table";
+                = "Failed to activate table because of invalid metadata in ZooKeeper. Please DROP/DETACH table";
             if constexpr (throw_on_error)
                 throw Exception(ErrorCodes::INVALID_STATE, error_msg);
             else
@@ -111,7 +111,15 @@ public:
 private:
     bool dropTable(zkutil::ZooKeeperPtr zookeeper, const zkutil::EphemeralNodeHolder::Ptr & metadata_drop_lock);
 
-    std::optional<bool> isTableValid() const;
+    enum class TableStatus : uint8_t
+    {
+        UNKNOWN,
+        INVALID_METADATA,
+        INVALID_KEEPER_STRUCTURE,
+        VALID
+    };
+
+    TableStatus getTableStatus() const;
 
     void restoreDataImpl(
         const BackupPtr & backup,
@@ -143,7 +151,8 @@ private:
     mutable zkutil::ZooKeeperPtr zookeeper_client{nullptr};
 
     mutable std::mutex init_mutex;
-    mutable std::optional<bool> table_is_valid;
+
+    mutable TableStatus table_status{TableStatus::UNKNOWN};
 
     LoggerPtr log;
 };
diff --git a/tests/integration/test_keeper_map/test.py b/tests/integration/test_keeper_map/test.py
index 7aee5df5746..4b1bcd11cfe 100644
--- a/tests/integration/test_keeper_map/test.py
+++ b/tests/integration/test_keeper_map/test.py
@@ -67,6 +67,7 @@ def run_query(query):
 
 
 def test_keeper_map_without_zk(started_cluster):
+    run_query("DROP TABLE IF EXISTS test_keeper_map_without_zk SYNC")
     assert_keeper_exception_after_partition(
         "CREATE TABLE test_keeper_map_without_zk (key UInt64, value UInt64) ENGINE = KeeperMap('/test_keeper_map_without_zk') PRIMARY KEY(key);"
     )
@@ -107,12 +108,12 @@ def test_keeper_map_without_zk(started_cluster):
     )
     assert "Failed to activate table because of invalid metadata in ZooKeeper" in error
 
-    node.query("DETACH TABLE test_keeper_map_without_zk")
-
     client.stop()
 
 
 def test_keeper_map_with_failed_drop(started_cluster):
+    run_query("DROP TABLE IF EXISTS test_keeper_map_with_failed_drop SYNC")
+    run_query("DROP TABLE IF EXISTS test_keeper_map_with_failed_drop_another SYNC")
     run_query(
         "CREATE TABLE test_keeper_map_with_failed_drop (key UInt64, value UInt64) ENGINE = KeeperMap('/test_keeper_map_with_failed_drop') PRIMARY KEY(key);"
     )

From 406ac2279ecbfc24913548dfcf459c55dd450723 Mon Sep 17 00:00:00 2001
From: Dmitry Novik <mrnovikd@gmail.com>
Date: Wed, 31 Jul 2024 14:48:33 +0200
Subject: [PATCH 1110/1488] Analyzer: Do not traverse unresolved subtrees

---
 src/Planner/findParallelReplicasQuery.cpp         | 12 +++++-------
 src/Planner/findQueryForParallelReplicas.h        |  2 +-
 ...5_analyzer_replace_with_dummy_tables.reference |  0
 .../03215_analyzer_replace_with_dummy_tables.sql  | 15 +++++++++++++++
 4 files changed, 21 insertions(+), 8 deletions(-)
 create mode 100644 tests/queries/0_stateless/03215_analyzer_replace_with_dummy_tables.reference
 create mode 100644 tests/queries/0_stateless/03215_analyzer_replace_with_dummy_tables.sql

diff --git a/src/Planner/findParallelReplicasQuery.cpp b/src/Planner/findParallelReplicasQuery.cpp
index c89a70be541..1140f30ad9c 100644
--- a/src/Planner/findParallelReplicasQuery.cpp
+++ b/src/Planner/findParallelReplicasQuery.cpp
@@ -113,13 +113,13 @@ std::stack<const QueryNode *> getSupportingParallelReplicasQuery(const IQueryTre
     return res;
 }
 
-class ReplaceTableNodeToDummyVisitor : public InDepthQueryTreeVisitor<ReplaceTableNodeToDummyVisitor, true>
+class ReplaceTableNodeToDummyVisitor : public InDepthQueryTreeVisitorWithContext<ReplaceTableNodeToDummyVisitor>
 {
 public:
-    using Base = InDepthQueryTreeVisitor<ReplaceTableNodeToDummyVisitor, true>;
+    using Base = InDepthQueryTreeVisitorWithContext<ReplaceTableNodeToDummyVisitor>;
     using Base::Base;
 
-    void visitImpl(const QueryTreeNodePtr & node)
+    void enterImpl(QueryTreeNodePtr & node)
     {
         auto * table_node = node->as<TableNode>();
         auto * table_function_node = node->as<TableFunctionNode>();
@@ -134,21 +134,19 @@ public:
                 ColumnsDescription(storage_snapshot->getColumns(get_column_options)),
                 storage_snapshot);
 
-            auto dummy_table_node = std::make_shared<TableNode>(std::move(storage_dummy), context);
+            auto dummy_table_node = std::make_shared<TableNode>(std::move(storage_dummy), getContext());
 
             dummy_table_node->setAlias(node->getAlias());
             replacement_map.emplace(node.get(), std::move(dummy_table_node));
         }
     }
 
-    ContextPtr context;
     std::unordered_map<const IQueryTreeNode *, QueryTreeNodePtr> replacement_map;
 };
 
 QueryTreeNodePtr replaceTablesWithDummyTables(const QueryTreeNodePtr & query, const ContextPtr & context)
 {
-    ReplaceTableNodeToDummyVisitor visitor;
-    visitor.context = context;
+    ReplaceTableNodeToDummyVisitor visitor(context);
     visitor.visit(query);
 
     return query->cloneAndReplace(visitor.replacement_map);
diff --git a/src/Planner/findQueryForParallelReplicas.h b/src/Planner/findQueryForParallelReplicas.h
index f5dc69dfa0e..cdce4ad0b47 100644
--- a/src/Planner/findQueryForParallelReplicas.h
+++ b/src/Planner/findQueryForParallelReplicas.h
@@ -13,7 +13,7 @@ using QueryTreeNodePtr = std::shared_ptr<IQueryTreeNode>;
 
 struct SelectQueryOptions;
 
-/// Find a qury which can be executed with parallel replicas up to WithMergableStage.
+/// Find a query which can be executed with parallel replicas up to WithMergableStage.
 /// Returned query will always contain some (>1) subqueries, possibly with joins.
 const QueryNode * findQueryForParallelReplicas(const QueryTreeNodePtr & query_tree_node, SelectQueryOptions & select_query_options);
 
diff --git a/tests/queries/0_stateless/03215_analyzer_replace_with_dummy_tables.reference b/tests/queries/0_stateless/03215_analyzer_replace_with_dummy_tables.reference
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/queries/0_stateless/03215_analyzer_replace_with_dummy_tables.sql b/tests/queries/0_stateless/03215_analyzer_replace_with_dummy_tables.sql
new file mode 100644
index 00000000000..12d2bd627a7
--- /dev/null
+++ b/tests/queries/0_stateless/03215_analyzer_replace_with_dummy_tables.sql
@@ -0,0 +1,15 @@
+create table t (number UInt64) engine MergeTree order by number;
+
+SELECT 1
+FROM
+(
+    SELECT number IN (
+            SELECT number
+            FROM view(
+                SELECT number
+                FROM numbers(1)
+            )
+        )
+    FROM t
+)
+SETTINGS allow_experimental_parallel_reading_from_replicas = 1, max_parallel_replicas = 2, allow_experimental_analyzer = 1;

From 7160e954c16100e963371e416878837437569d74 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Wed, 31 Jul 2024 13:32:24 +0200
Subject: [PATCH 1111/1488] 02995_new_settings_history: Update ref

---
 ..._23_12_1.tsv => 02995_baseline_24_7_1.tsv} | 182 +++++++++++++++---
 .../0_stateless/02995_new_settings_history.sh |  14 +-
 2 files changed, 167 insertions(+), 29 deletions(-)
 rename tests/queries/0_stateless/{02995_baseline_23_12_1.tsv => 02995_baseline_24_7_1.tsv} (82%)

diff --git a/tests/queries/0_stateless/02995_baseline_23_12_1.tsv b/tests/queries/0_stateless/02995_baseline_24_7_1.tsv
similarity index 82%
rename from tests/queries/0_stateless/02995_baseline_23_12_1.tsv
rename to tests/queries/0_stateless/02995_baseline_24_7_1.tsv
index a391473e7c9..6c830da8646 100644
--- a/tests/queries/0_stateless/02995_baseline_23_12_1.tsv
+++ b/tests/queries/0_stateless/02995_baseline_24_7_1.tsv
@@ -11,23 +11,28 @@ allow_create_index_without_type	0
 allow_custom_error_code_in_throwif	0
 allow_ddl	1
 allow_deprecated_database_ordinary	0
+allow_deprecated_error_prone_window_functions	0
+allow_deprecated_snowflake_conversion_functions	0
 allow_deprecated_syntax_for_merge_tree	0
 allow_distributed_ddl	1
 allow_drop_detached	0
 allow_execute_multiif_columnar	1
 allow_experimental_alter_materialized_view_structure	1
-allow_experimental_analyzer	0
+allow_experimental_analyzer	1
 allow_experimental_annoy_index	0
 allow_experimental_bigint_types	1
 allow_experimental_codecs	0
 allow_experimental_database_atomic	1
 allow_experimental_database_materialized_mysql	0
 allow_experimental_database_materialized_postgresql	0
-allow_experimental_database_replicated	0
+allow_experimental_database_replicated	1
+allow_experimental_dynamic_type	0
+allow_experimental_full_text_index	0
 allow_experimental_funnel_functions	0
 allow_experimental_geo_types	1
 allow_experimental_hash_functions	0
 allow_experimental_inverted_index	0
+allow_experimental_join_condition	0
 allow_experimental_lightweight_delete	1
 allow_experimental_live_view	0
 allow_experimental_map_type	1
@@ -40,12 +45,15 @@ allow_experimental_query_cache	1
 allow_experimental_query_deduplication	0
 allow_experimental_refreshable_materialized_view	0
 allow_experimental_s3queue	1
-allow_experimental_shared_merge_tree	0
+allow_experimental_shared_merge_tree	1
+allow_experimental_statistic	0
 allow_experimental_statistics	0
 allow_experimental_undrop_table_query	1
 allow_experimental_usearch_index	0
+allow_experimental_variant_type	0
 allow_experimental_window_functions	1
 allow_experimental_window_view	0
+allow_get_client_http_header	0
 allow_hyperscan	1
 allow_introspection_functions	0
 allow_named_collection_override_by_default	1
@@ -58,17 +66,21 @@ allow_prefetched_read_pool_for_remote_filesystem	1
 allow_push_predicate_when_subquery_contains_with	1
 allow_settings_after_format_in_insert	0
 allow_simdjson	1
+allow_statistic_optimize	0
 allow_statistics_optimize	0
 allow_suspicious_codecs	0
 allow_suspicious_fixed_string_types	0
 allow_suspicious_indices	0
 allow_suspicious_low_cardinality_types	0
+allow_suspicious_primary_key	0
 allow_suspicious_ttl_expressions	0
+allow_suspicious_variant_types	0
 allow_unrestricted_reads_from_keeper	0
 alter_move_to_space_execute_async	0
 alter_partition_verbose_result	0
 alter_sync	1
 analyze_index_with_space_filling_curves	1
+analyzer_compatibility_join_using_top_level_identifier	0
 annoy_index_search_k_nodes	-1
 any_join_distinct_right_table_keys	0
 apply_deleted_mask	1
@@ -76,20 +88,42 @@ apply_mutations_on_fly	0
 asterisk_include_alias_columns	0
 asterisk_include_materialized_columns	0
 async_insert	0
+async_insert_busy_timeout_decrease_rate	0.2
+async_insert_busy_timeout_increase_rate	0.2
+async_insert_busy_timeout_max_ms	200
+async_insert_busy_timeout_min_ms	50
 async_insert_busy_timeout_ms	200
 async_insert_cleanup_timeout_ms	1000
 async_insert_deduplicate	0
-async_insert_max_data_size	1000000
+async_insert_max_data_size	10485760
 async_insert_max_query_number	450
+async_insert_poll_timeout_ms	10
 async_insert_stale_timeout_ms	0
 async_insert_threads	16
+async_insert_use_adaptive_busy_timeout	1
 async_query_sending_for_remote	1
 async_socket_for_remote	1
+azure_allow_parallel_part_upload	1
 azure_create_new_file_on_insert	0
+azure_ignore_file_doesnt_exist	0
 azure_list_object_keys_size	1000
+azure_max_blocks_in_multipart_upload	50000
+azure_max_inflight_parts_for_one_file	20
+azure_max_single_part_copy_size	268435456
 azure_max_single_part_upload_size	104857600
 azure_max_single_read_retries	4
+azure_max_unexpected_write_error_retries	4
+azure_max_upload_part_size	5368709120
+azure_min_upload_part_size	16777216
+azure_sdk_max_retries	10
+azure_sdk_retry_initial_backoff_ms	10
+azure_sdk_retry_max_backoff_ms	1000
+azure_skip_empty_files	0
+azure_strict_upload_part_size	0
+azure_throw_on_zero_files_match	0
 azure_truncate_on_insert	0
+azure_upload_part_size_multiply_factor	2
+azure_upload_part_size_multiply_parts_count_threshold	500
 background_buffer_flush_schedule_pool_size	16
 background_common_pool_size	8
 background_distributed_schedule_pool_size	16
@@ -107,6 +141,7 @@ backup_restore_keeper_max_retries	20
 backup_restore_keeper_retry_initial_backoff_ms	100
 backup_restore_keeper_retry_max_backoff_ms	5000
 backup_restore_keeper_value_max_size	1048576
+backup_restore_s3_retry_attempts	1000
 backup_threads	16
 bool_false_representation	false
 bool_true_representation	true
@@ -115,6 +150,7 @@ calculate_text_stack_trace	1
 cancel_http_readonly_queries_on_client_close	0
 cast_ipv4_ipv6_default_on_conversion_error	0
 cast_keep_nullable	0
+cast_string_to_dynamic_use_inference	0
 check_query_single_value_result	1
 check_referential_table_dependencies	0
 check_table_dependencies	1
@@ -123,6 +159,7 @@ cloud_mode	0
 cloud_mode_engine	1
 cluster_for_parallel_replicas	
 collect_hash_table_stats_during_aggregation	1
+collect_hash_table_stats_during_joins	1
 column_names_for_schema_inference	
 compatibility	
 compatibility_ignore_auto_increment_in_create_table	0
@@ -141,9 +178,12 @@ count_distinct_optimization	0
 create_index_ignore_unique	0
 create_replicated_merge_tree_fault_injection_probability	0
 create_table_empty_primary_key_by_default	0
+cross_join_min_bytes_to_compress	1073741824
+cross_join_min_rows_to_compress	10000000
 cross_to_inner_join_rewrite	1
 data_type_default_nullable	0
 database_atomic_wait_for_drop_and_detach_synchronously	0
+database_replicated_allow_heavy_create	0
 database_replicated_allow_only_replicated_engine	0
 database_replicated_allow_replicated_engine_arguments	1
 database_replicated_always_detach_permanently	0
@@ -156,15 +196,19 @@ date_time_overflow_behavior	ignore
 decimal_check_overflow	1
 deduplicate_blocks_in_dependent_materialized_views	0
 default_database_engine	Atomic
+default_materialized_view_sql_security	DEFINER
 default_max_bytes_in_join	1000000000
-default_table_engine	None
+default_normal_view_sql_security	INVOKER
+default_table_engine	MergeTree
 default_temporary_table_engine	Memory
+default_view_definer	CURRENT_USER
 describe_compact_output	0
 describe_extend_object_types	0
 describe_include_subcolumns	0
 describe_include_virtual_columns	0
 dialect	clickhouse
 dictionary_use_async_executor	0
+dictionary_validate_primary_key_type	0
 distinct_overflow_mode	throw
 distributed_aggregation_memory_efficient	1
 distributed_background_insert_batch	0
@@ -182,6 +226,7 @@ distributed_directory_monitor_sleep_time_ms	100
 distributed_directory_monitor_split_batch_on_failure	0
 distributed_foreground_insert	0
 distributed_group_by_no_merge	0
+distributed_insert_skip_read_only_replicas	0
 distributed_product_mode	deny
 distributed_push_down_limit	1
 distributed_replica_error_cap	1000
@@ -191,6 +236,7 @@ do_not_merge_across_partitions_select_final	0
 drain_timeout	3
 empty_result_for_aggregation_by_constant_keys_on_empty_set	1
 empty_result_for_aggregation_by_empty_set	0
+enable_blob_storage_log	1
 enable_debug_queries	0
 enable_deflate_qpl_codec	0
 enable_early_constant_folding	1
@@ -205,6 +251,7 @@ enable_job_stack_trace	0
 enable_lightweight_delete	1
 enable_memory_bound_merging_of_aggregation_results	1
 enable_multiple_prewhere_read_steps	1
+enable_named_columns_in_function_tuple	1
 enable_optimize_predicate_expression	1
 enable_optimize_predicate_expression_to_final_subquery	1
 enable_order_by_all	1
@@ -216,7 +263,9 @@ enable_sharing_sets_for_mutations	1
 enable_software_prefetch_in_aggregation	1
 enable_unaligned_array_join	0
 enable_url_encoding	1
+enable_vertical_final	1
 enable_writes_to_query_cache	1
+enable_zstd_qat_codec	0
 engine_file_allow_create_multiple_files	0
 engine_file_empty_if_not_exists	0
 engine_file_skip_empty_files	0
@@ -231,10 +280,12 @@ external_storage_max_read_rows	0
 external_storage_rw_timeout_sec	300
 external_table_functions_use_nulls	1
 external_table_strict_query	0
+extract_key_value_pairs_max_pairs_per_row	1000
 extract_kvp_max_pairs_per_row	1000
 extremes	0
 fallback_to_stale_replicas_for_distributed_queries	1
 filesystem_cache_max_download_size	137438953472
+filesystem_cache_reserve_space_wait_lock_timeout_milliseconds	1000
 filesystem_cache_segments_batch_size	20
 filesystem_prefetch_max_memory_usage	1073741824
 filesystem_prefetch_min_bytes_for_single_read_task	2097152
@@ -278,7 +329,9 @@ format_regexp_escaping_rule	Raw
 format_regexp_skip_unmatched	0
 format_schema	
 format_template_resultset	
+format_template_resultset_format	
 format_template_row	
+format_template_row_format	
 format_template_rows_between_delimiter	\n
 format_tsv_null_representation	\\N
 formatdatetime_f_prints_single_zero	0
@@ -288,8 +341,11 @@ fsync_metadata	1
 function_implementation	
 function_json_value_return_type_allow_complex	0
 function_json_value_return_type_allow_nullable	0
+function_locate_has_mysql_compatible_argument_order	1
 function_range_max_elements_in_block	500000000
 function_sleep_max_microseconds_per_block	3000000
+function_visible_width_behavior	1
+geo_distance_returns_float64_on_float64_arguments	1
 glob_expansion_max_elements	1000
 grace_hash_join_initial_buckets	1
 grace_hash_join_max_buckets	1024
@@ -300,8 +356,10 @@ group_by_use_nulls	0
 handle_kafka_error_mode	default
 handshake_timeout_ms	10000
 hdfs_create_new_file_on_insert	0
+hdfs_ignore_file_doesnt_exist	0
 hdfs_replication	0
 hdfs_skip_empty_files	0
+hdfs_throw_on_zero_files_match	0
 hdfs_truncate_on_insert	0
 hedged_connection_timeout_ms	50
 hsts_max_age	0
@@ -326,10 +384,14 @@ http_skip_not_found_url_for_globs	1
 http_wait_end_of_query	0
 http_write_exception_in_output_format	1
 http_zlib_compression_level	3
+iceberg_engine_ignore_schema_evolution	0
 idle_connection_timeout	3600
 ignore_cold_parts_seconds	0
 ignore_data_skipping_indices	
+ignore_drop_queries_probability	0
+ignore_materialized_views_with_dropped_target_table	0
 ignore_on_cluster_for_replicated_access_entities_queries	0
+ignore_on_cluster_for_replicated_named_collections_queries	0
 ignore_on_cluster_for_replicated_udf_queries	0
 implicit_transaction	0
 input_format_allow_errors_num	0
@@ -341,12 +403,14 @@ input_format_arrow_import_nested	0
 input_format_arrow_skip_columns_with_unsupported_types_in_schema_inference	0
 input_format_avro_allow_missing_fields	0
 input_format_avro_null_as_default	0
+input_format_binary_decode_types_in_binary_format	0
 input_format_bson_skip_fields_with_unsupported_types_in_schema_inference	0
 input_format_capn_proto_skip_fields_with_unsupported_types_in_schema_inference	0
 input_format_csv_allow_cr_end_of_line	0
 input_format_csv_allow_variable_number_of_columns	0
 input_format_csv_allow_whitespace_or_tab_as_delimiter	0
 input_format_csv_arrays_as_nested_csv	0
+input_format_csv_deserialize_separate_columns_into_tuple	1
 input_format_csv_detect_header	1
 input_format_csv_empty_as_default	1
 input_format_csv_enum_as_number	0
@@ -354,29 +418,37 @@ input_format_csv_skip_first_lines	0
 input_format_csv_skip_trailing_empty_lines	0
 input_format_csv_trim_whitespaces	1
 input_format_csv_try_infer_numbers_from_strings	0
+input_format_csv_try_infer_strings_from_quoted_tuples	1
 input_format_csv_use_best_effort_in_schema_inference	1
 input_format_csv_use_default_on_bad_values	0
 input_format_custom_allow_variable_number_of_columns	0
 input_format_custom_detect_header	1
 input_format_custom_skip_trailing_empty_lines	0
 input_format_defaults_for_omitted_fields	1
+input_format_force_null_for_omitted_fields	0
+input_format_hive_text_allow_variable_number_of_columns	1
 input_format_hive_text_collection_items_delimiter	
 input_format_hive_text_fields_delimiter	

 input_format_hive_text_map_keys_delimiter	
 input_format_import_nested_json	0
 input_format_ipv4_default_on_conversion_error	0
 input_format_ipv6_default_on_conversion_error	0
+input_format_json_case_insensitive_column_matching	0
 input_format_json_compact_allow_variable_number_of_columns	0
 input_format_json_defaults_for_missing_elements_in_named_tuple	1
 input_format_json_ignore_unknown_keys_in_named_tuple	1
+input_format_json_ignore_unnecessary_fields	1
 input_format_json_infer_incomplete_types_as_strings	1
 input_format_json_named_tuples_as_objects	1
 input_format_json_read_arrays_as_strings	1
 input_format_json_read_bools_as_numbers	1
+input_format_json_read_bools_as_strings	1
 input_format_json_read_numbers_as_strings	1
 input_format_json_read_objects_as_strings	1
+input_format_json_throw_on_bad_escape_sequence	1
 input_format_json_try_infer_named_tuples_from_objects	1
 input_format_json_try_infer_numbers_from_strings	0
+input_format_json_use_string_type_for_ambiguous_paths_in_named_tuples_inference_from_objects	0
 input_format_json_validate_types_from_metadata	1
 input_format_max_bytes_to_read_for_schema_inference	33554432
 input_format_max_rows_to_read_for_schema_inference	25000
@@ -384,11 +456,13 @@ input_format_msgpack_number_of_columns	0
 input_format_mysql_dump_map_column_names	1
 input_format_mysql_dump_table_name	
 input_format_native_allow_types_conversion	1
+input_format_native_decode_types_in_binary_format	0
 input_format_null_as_default	1
 input_format_orc_allow_missing_columns	1
 input_format_orc_case_insensitive_column_matching	0
 input_format_orc_filter_push_down	1
 input_format_orc_import_nested	0
+input_format_orc_read_use_writer_time_zone	0
 input_format_orc_row_batch_size	100000
 input_format_orc_skip_columns_with_unsupported_types_in_schema_inference	0
 input_format_orc_use_fast_decoder	1
@@ -398,17 +472,21 @@ input_format_parquet_case_insensitive_column_matching	0
 input_format_parquet_filter_push_down	1
 input_format_parquet_import_nested	0
 input_format_parquet_local_file_min_bytes_for_seek	8192
-input_format_parquet_max_block_size	8192
+input_format_parquet_max_block_size	65409
+input_format_parquet_prefer_block_bytes	16744704
 input_format_parquet_preserve_order	0
 input_format_parquet_skip_columns_with_unsupported_types_in_schema_inference	0
+input_format_parquet_use_native_reader	0
 input_format_protobuf_flatten_google_wrappers	0
 input_format_protobuf_skip_fields_with_unsupported_types_in_schema_inference	0
 input_format_record_errors_file_path	
 input_format_skip_unknown_fields	1
 input_format_try_infer_dates	1
 input_format_try_infer_datetimes	1
+input_format_try_infer_exponent_floats	0
 input_format_try_infer_integers	1
 input_format_tsv_allow_variable_number_of_columns	0
+input_format_tsv_crlf_end_of_line	0
 input_format_tsv_detect_header	1
 input_format_tsv_empty_as_default	0
 input_format_tsv_enum_as_number	0
@@ -450,7 +528,12 @@ joined_subquery_requires_alias	1
 kafka_disable_num_consumers_limit	0
 kafka_max_wait_ms	5000
 keeper_map_strict_mode	0
+keeper_max_retries	10
+keeper_retry_initial_backoff_ms	100
+keeper_retry_max_backoff_ms	5000
 legacy_column_name_of_tuple_literal	0
+lightweight_deletes_sync	2
+lightweight_mutation_projection_mode	throw
 limit	0
 live_view_heartbeat_interval	15
 load_balancing	random
@@ -461,7 +544,7 @@ local_filesystem_read_prefetch	0
 lock_acquire_timeout	120
 log_comment	
 log_formatted_queries	0
-log_processors_profiles	0
+log_processors_profiles	1
 log_profile_events	1
 log_queries	1
 log_queries_cut_to_length	100000
@@ -474,6 +557,8 @@ log_query_views	1
 low_cardinality_allow_in_native_format	1
 low_cardinality_max_dictionary_size	8192
 low_cardinality_use_single_dictionary_for_part	0
+materialize_skip_indexes_on_insert	1
+materialize_statistics_on_insert	1
 materialize_ttl_after_modify	1
 materialized_views_ignore_errors	0
 max_alter_threads	\'auto(16)\'
@@ -501,6 +586,7 @@ max_distributed_depth	5
 max_download_buffer_size	10485760
 max_download_threads	4
 max_entries_for_hash_table_stats	10000
+max_estimated_execution_time	0
 max_execution_speed	0
 max_execution_speed_bytes	0
 max_execution_time	0
@@ -528,7 +614,9 @@ max_network_bandwidth_for_user	0
 max_network_bytes	0
 max_number_of_partitions_for_independent_aggregation	128
 max_parallel_replicas	1
+max_parser_backtracks	1000000
 max_parser_depth	1000
+max_parsing_threads	\'auto(16)\'
 max_partition_size_to_drop	50000000000
 max_partitions_per_insert_block	100
 max_partitions_to_read	-1
@@ -537,6 +625,7 @@ max_query_size	262144
 max_read_buffer_size	1048576
 max_read_buffer_size_local_fs	131072
 max_read_buffer_size_remote_fs	0
+max_recursive_cte_evaluation_depth	1000
 max_remote_read_network_bandwidth	0
 max_remote_read_network_bandwidth_for_server	0
 max_remote_write_network_bandwidth	0
@@ -549,7 +638,7 @@ max_result_rows	0
 max_rows_in_distinct	0
 max_rows_in_join	0
 max_rows_in_set	0
-max_rows_in_set_to_optimize_join	100000
+max_rows_in_set_to_optimize_join	0
 max_rows_to_group_by	0
 max_rows_to_read	0
 max_rows_to_read_leaf	0
@@ -557,6 +646,7 @@ max_rows_to_sort	0
 max_rows_to_transfer	0
 max_sessions_for_user	0
 max_size_to_preallocate_for_aggregation	100000000
+max_size_to_preallocate_for_joins	100000000
 max_streams_for_merge_tree_reading	0
 max_streams_multiplier_for_merge_tables	5
 max_streams_to_max_threads_ratio	1
@@ -592,6 +682,7 @@ merge_tree_min_bytes_per_task_for_remote_reading	4194304
 merge_tree_min_rows_for_concurrent_read	163840
 merge_tree_min_rows_for_concurrent_read_for_remote_filesystem	163840
 merge_tree_min_rows_for_seek	0
+merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability	0
 merge_tree_use_const_size_tasks_for_remote_reading	1
 metrics_perf_events_enabled	0
 metrics_perf_events_list	
@@ -604,6 +695,8 @@ min_count_to_compile_expression	3
 min_count_to_compile_sort_description	3
 min_execution_speed	0
 min_execution_speed_bytes	0
+min_external_table_block_size_bytes	268402944
+min_external_table_block_size_rows	1048449
 min_free_disk_space_for_temporary_data	0
 min_hit_rate_to_use_consecutive_keys_optimization	0.5
 min_insert_block_size_bytes	268402944
@@ -619,8 +712,8 @@ mutations_execute_subqueries_on_initiator	0
 mutations_max_literal_size_to_replace	16384
 mutations_sync	0
 mysql_datatypes_support_level	
-mysql_map_fixed_string_to_text_in_show_columns	0
-mysql_map_string_to_text_in_show_columns	0
+mysql_map_fixed_string_to_text_in_show_columns	1
+mysql_map_string_to_text_in_show_columns	1
 mysql_max_rows_to_insert	65536
 network_compression_method	LZ4
 network_zstd_compression_level	1
@@ -647,6 +740,7 @@ optimize_group_by_constant_keys	1
 optimize_group_by_function_keys	1
 optimize_if_chain_to_multiif	0
 optimize_if_transform_strings_to_enum	0
+optimize_injective_functions_in_group_by	1
 optimize_injective_functions_inside_uniq	1
 optimize_min_equality_disjunction_chain_length	3
 optimize_min_inequality_conjunction_chain_length	3
@@ -664,7 +758,7 @@ optimize_redundant_functions_in_order_by	1
 optimize_respect_aliases	1
 optimize_rewrite_aggregate_function_with_if	1
 optimize_rewrite_array_exists_to_has	0
-optimize_rewrite_sum_if_to_count_if	0
+optimize_rewrite_sum_if_to_count_if	1
 optimize_skip_merged_partitions	0
 optimize_skip_unused_shards	0
 optimize_skip_unused_shards_limit	1000
@@ -674,9 +768,10 @@ optimize_sorting_by_input_stream_properties	1
 optimize_substitute_columns	0
 optimize_syntax_fuse_functions	0
 optimize_throw_if_noop	0
+optimize_time_filter_with_preimage	1
 optimize_trivial_approximate_count_query	0
 optimize_trivial_count_query	1
-optimize_trivial_insert_select	1
+optimize_trivial_insert_select	0
 optimize_uniq_to_count	1
 optimize_use_implicit_projections	1
 optimize_use_projections	1
@@ -685,13 +780,19 @@ os_thread_priority	0
 output_format_arrow_compression_method	lz4_frame
 output_format_arrow_fixed_string_as_fixed_byte_array	1
 output_format_arrow_low_cardinality_as_dictionary	0
-output_format_arrow_string_as_string	0
+output_format_arrow_string_as_string	1
+output_format_arrow_use_64_bit_indexes_for_dictionary	0
+output_format_arrow_use_signed_indexes_for_dictionary	1
 output_format_avro_codec	
 output_format_avro_rows_in_file	1
 output_format_avro_string_column_pattern	
 output_format_avro_sync_interval	16384
+output_format_binary_encode_types_in_binary_format	0
 output_format_bson_string_as_string	0
+output_format_compression_level	3
+output_format_compression_zstd_window_log	0
 output_format_csv_crlf_end_of_line	0
+output_format_csv_serialize_tuple_into_separate_columns	1
 output_format_decimal_trailing_zeros	0
 output_format_enable_streaming	0
 output_format_json_array_of_rows	0
@@ -705,27 +806,34 @@ output_format_json_skip_null_value_in_named_tuples	0
 output_format_json_validate_utf8	0
 output_format_markdown_escape_special_characters	0
 output_format_msgpack_uuid_representation	ext
-output_format_orc_compression_method	lz4
+output_format_native_encode_types_in_binary_format	0
+output_format_orc_compression_method	zstd
 output_format_orc_row_index_stride	10000
-output_format_orc_string_as_string	0
+output_format_orc_string_as_string	1
 output_format_parallel_formatting	1
 output_format_parquet_batch_size	1024
 output_format_parquet_compliant_nested_types	1
-output_format_parquet_compression_method	lz4
+output_format_parquet_compression_method	zstd
 output_format_parquet_data_page_size	1048576
 output_format_parquet_fixed_string_as_fixed_byte_array	1
 output_format_parquet_parallel_encoding	1
 output_format_parquet_row_group_size	1000000
 output_format_parquet_row_group_size_bytes	536870912
-output_format_parquet_string_as_string	0
-output_format_parquet_use_custom_encoder	0
+output_format_parquet_string_as_string	1
+output_format_parquet_use_custom_encoder	1
 output_format_parquet_version	2.latest
-output_format_pretty_color	1
+output_format_parquet_write_page_index	1
+output_format_pretty_color	auto
+output_format_pretty_display_footer_column_names	1
+output_format_pretty_display_footer_column_names_min_rows	50
 output_format_pretty_grid_charset	UTF-8
+output_format_pretty_highlight_digit_groups	1
 output_format_pretty_max_column_pad_width	250
 output_format_pretty_max_rows	10000
 output_format_pretty_max_value_width	10000
-output_format_pretty_row_numbers	0
+output_format_pretty_max_value_width_apply_for_single_value	0
+output_format_pretty_row_numbers	1
+output_format_pretty_single_large_number_tip_threshold	1000000
 output_format_protobuf_nullables_with_google_wrappers	0
 output_format_schema	
 output_format_sql_insert_include_column_names	1
@@ -734,15 +842,22 @@ output_format_sql_insert_quote_names	1
 output_format_sql_insert_table_name	table
 output_format_sql_insert_use_replace	0
 output_format_tsv_crlf_end_of_line	0
+output_format_values_escape_quote_with_quote	0
 output_format_write_statistics	1
+page_cache_inject_eviction	0
 parallel_distributed_insert_select	0
 parallel_replica_offset	0
+parallel_replicas_allow_in_with_subquery	1
 parallel_replicas_count	0
 parallel_replicas_custom_key	
 parallel_replicas_custom_key_filter_type	default
+parallel_replicas_custom_key_range_lower	0
+parallel_replicas_custom_key_range_upper	0
 parallel_replicas_for_non_replicated_merge_tree	0
+parallel_replicas_mark_segment_size	128
 parallel_replicas_min_number_of_granules_to_enable	0
 parallel_replicas_min_number_of_rows_per_replica	0
+parallel_replicas_prefer_local_join	1
 parallel_replicas_single_task_marks_count_multiplier	2
 parallel_view_processing	0
 parallelize_output_from_storages	1
@@ -755,11 +870,14 @@ parts_to_delay_insert	0
 parts_to_throw_insert	0
 periodic_live_view_refresh	60
 poll_interval	10
+postgresql_connection_attempt_timeout	2
 postgresql_connection_pool_auto_close_connection	0
+postgresql_connection_pool_retries	2
 postgresql_connection_pool_size	16
 postgresql_connection_pool_wait_timeout	5000
 precise_float_parsing	0
 prefer_column_name_to_alias	0
+prefer_external_sort_block_bytes	16744704
 prefer_global_in_and_join	0
 prefer_localhost_replica	1
 prefer_warmed_unmerged_parts_seconds	0
@@ -767,7 +885,7 @@ preferred_block_size_bytes	1000000
 preferred_max_column_in_block_size_bytes	0
 preferred_optimize_projection_name	
 prefetch_buffer_size	1048576
-print_pretty_type_names	0
+print_pretty_type_names	1
 priority	0
 query_cache_compress_entries	1
 query_cache_max_entries	0
@@ -778,8 +896,10 @@ query_cache_nondeterministic_function_handling	throw
 query_cache_share_between_users	0
 query_cache_squash_partial_results	1
 query_cache_store_results_of_queries_with_nondeterministic_functions	0
+query_cache_system_table_handling	throw
 query_cache_ttl	60
 query_plan_aggregation_in_order	1
+query_plan_convert_outer_join_to_inner_join	1
 query_plan_enable_multithreading_after_window_functions	1
 query_plan_enable_optimizations	1
 query_plan_execute_functions_after_sorting	1
@@ -788,6 +908,8 @@ query_plan_lift_up_array_join	1
 query_plan_lift_up_union	1
 query_plan_max_optimizations_to_apply	10000
 query_plan_merge_expressions	1
+query_plan_merge_filters	0
+query_plan_optimize_prewhere	1
 query_plan_optimize_primary_key	1
 query_plan_optimize_projection	1
 query_plan_push_down_limit	1
@@ -806,7 +928,9 @@ read_backoff_min_events	2
 read_backoff_min_interval_between_events_ms	1000
 read_backoff_min_latency_ms	1000
 read_from_filesystem_cache_if_exists_otherwise_bypass_cache	0
+read_from_page_cache_if_exists_otherwise_bypass_cache	0
 read_in_order_two_level_merge_threshold	100
+read_in_order_use_buffering	1
 read_overflow_mode	throw
 read_overflow_mode_leaf	throw
 read_priority	0
@@ -835,17 +959,20 @@ result_overflow_mode	throw
 rewrite_count_distinct_if_with_count_distinct_implementation	0
 s3_allow_parallel_part_upload	1
 s3_check_objects_after_upload	0
+s3_connect_timeout_ms	1000
 s3_create_new_file_on_insert	0
 s3_disable_checksum	0
-s3_http_connection_pool_size	1000
+s3_ignore_file_doesnt_exist	0
 s3_list_object_keys_size	1000
 s3_max_connections	1024
 s3_max_get_burst	0
 s3_max_get_rps	0
 s3_max_inflight_parts_for_one_file	20
+s3_max_part_number	10000
 s3_max_put_burst	0
 s3_max_put_rps	0
 s3_max_redirects	10
+s3_max_single_operation_copy_size	33554432
 s3_max_single_part_upload_size	33554432
 s3_max_single_read_retries	4
 s3_max_unexpected_write_error_retries	4
@@ -860,6 +987,8 @@ s3_truncate_on_insert	0
 s3_upload_part_size_multiply_factor	2
 s3_upload_part_size_multiply_parts_count_threshold	500
 s3_use_adaptive_timeouts	1
+s3_validate_request_settings	1
+s3queue_allow_experimental_sharded_mode	0
 s3queue_default_zookeeper_path	/clickhouse/s3queue/
 s3queue_enable_logging_to_s3queue_log	0
 schema_inference_cache_require_modification_time_for_url	1
@@ -887,6 +1016,8 @@ sleep_after_receiving_query_ms	0
 sleep_in_send_data_ms	0
 sleep_in_send_tables_status_ms	0
 sort_overflow_mode	throw
+split_intersecting_parts_ranges_into_layers_final	1
+split_parts_ranges_into_intersecting_and_non_intersecting_final	1
 splitby_max_substrings_includes_remaining_string	0
 stop_refreshable_materialized_views_on_startup	0
 storage_file_read_method	pread
@@ -898,8 +1029,10 @@ stream_poll_timeout_ms	500
 system_events_show_zero_values	0
 table_function_remote_max_addresses	1000
 tcp_keep_alive_timeout	290
+temporary_data_in_cache_reserve_space_wait_lock_timeout_milliseconds	600000
 temporary_files_codec	LZ4
 temporary_live_view_timeout	1
+throw_if_deduplication_in_dependent_materialized_views_enabled_with_async_insert	1
 throw_if_no_data_to_insert	1
 throw_on_error_from_cache_on_write_operations	0
 throw_on_max_partitions_per_insert_block	1
@@ -912,8 +1045,10 @@ totals_mode	after_having_exclusive
 trace_profile_events	0
 transfer_overflow_mode	throw
 transform_null_in	0
+traverse_shadow_remote_data_paths	0
 union_default_mode	
 unknown_packet_in_send_data	0
+update_insert_deduplication_token_in_dependent_materialized_views	0
 use_cache_for_count_from_files	1
 use_client_time_zone	0
 use_compact_format_in_distributed_parts_names	1
@@ -923,12 +1058,15 @@ use_index_for_in_with_subqueries	1
 use_index_for_in_with_subqueries_max_values	0
 use_local_cache_for_remote_storage	1
 use_mysql_types_in_show_columns	0
+use_page_cache_for_disks_without_file_cache	0
 use_query_cache	0
 use_skip_indexes	1
 use_skip_indexes_if_final	0
 use_structure_from_insertion_table_in_table_functions	2
 use_uncompressed_cache	0
+use_variant_as_common_type	0
 use_with_fill_by_sorting_prefix	1
+validate_experimental_and_suspicious_types_inside_nested_types	1
 validate_polygons	1
 wait_changes_become_visible_after_commit_mode	wait_unknown
 wait_for_async_insert	1
diff --git a/tests/queries/0_stateless/02995_new_settings_history.sh b/tests/queries/0_stateless/02995_new_settings_history.sh
index 8de98c55b6a..917dacc04b0 100755
--- a/tests/queries/0_stateless/02995_new_settings_history.sh
+++ b/tests/queries/0_stateless/02995_new_settings_history.sh
@@ -7,12 +7,12 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CUR_DIR"/../shell_config.sh
 
 # Note that this is a broad check. A per version check is done in the upgrade test
-# Baseline generated with 23.12.1
-# clickhouse local --query "select name, default from system.settings order by name format TSV" > 02995_baseline_23_12_1.tsv
+# Baseline generated with 24.7.1
+# clickhouse local --query "select name, default from system.settings order by name format TSV" > 02995_baseline_24_7_1.tsv
 $CLICKHOUSE_LOCAL --query "
     WITH old_settings AS
     (
-        SELECT * FROM file('${CUR_DIR}/02995_baseline_23_12_1.tsv', 'TSV', 'name String, default String')
+        SELECT * FROM file('${CUR_DIR}/02995_baseline_24_7_1.tsv', 'TSV', 'name String, default String')
     ),
     new_settings AS
     (
@@ -21,7 +21,7 @@ $CLICKHOUSE_LOCAL --query "
     )
     SELECT * FROM
     (
-        SELECT 'PLEASE ADD THE NEW SETTING TO SettingsChangesHistory.h: ' || name || ' WAS ADDED',
+        SELECT 'PLEASE ADD THE NEW SETTING TO SettingsChangesHistory.cpp: ' || name || ' WAS ADDED',
         FROM new_settings
         WHERE (name NOT IN (
             SELECT name
@@ -29,17 +29,17 @@ $CLICKHOUSE_LOCAL --query "
         )) AND (name NOT IN (
             SELECT arrayJoin(tupleElement(changes, 'name'))
             FROM system.settings_changes
-            WHERE splitByChar('.', version())[1] >= '24'
+            WHERE splitByChar('.', version)[1]::UInt64 >= 24 AND splitByChar('.', version)[2]::UInt64 > 7
         ))
         UNION ALL
         (
-            SELECT 'PLEASE ADD THE SETTING VALUE CHANGE TO SettingsChangesHistory.h: ' || name || ' WAS CHANGED FROM ' || old_settings.default || ' TO ' || new_settings.default,
+            SELECT 'PLEASE ADD THE SETTING VALUE CHANGE TO SettingsChangesHistory.cpp: ' || name || ' WAS CHANGED FROM ' || old_settings.default || ' TO ' || new_settings.default,
             FROM new_settings
             LEFT JOIN old_settings ON new_settings.name = old_settings.name
             WHERE (new_settings.default != old_settings.default) AND (name NOT IN (
                 SELECT arrayJoin(tupleElement(changes, 'name'))
                 FROM system.settings_changes
-                WHERE splitByChar('.', version())[1] >= '24'
+                WHERE splitByChar('.', version)[1]::UInt64 >= 24 AND splitByChar('.', version)[2]::UInt64 > 7
             ))
         )
     )

From b178eea09ec80fed40b5043ccf1635d95b9cf19b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Wed, 31 Jul 2024 14:59:17 +0200
Subject: [PATCH 1112/1488] Fix broken settings

---
 src/Core/SettingsChangesHistory.cpp | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp
index 9faf77e9087..ecc558e64d7 100644
--- a/src/Core/SettingsChangesHistory.cpp
+++ b/src/Core/SettingsChangesHistory.cpp
@@ -57,6 +57,16 @@ String ClickHouseVersion::toString() const
 /// Note: please check if the key already exists to prevent duplicate entries.
 static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory::SettingsChanges>> settings_changes_history_initializer =
 {
+    {"24.12",{
+             }},
+    {"24.11",{
+             }},
+    {"24.10",{
+             }},
+    {"24.9", {
+             }},
+    {"24.8", {{"merge_tree_min_bytes_per_task_for_remote_reading", 4194304, 2097152, "Value is unified with `filesystem_prefetch_min_bytes_for_single_read_task`"},
+              }},
     {"24.7", {{"output_format_parquet_write_page_index", false, true, "Add a possibility to write page index into parquet files."},
               {"output_format_binary_encode_types_in_binary_format", false, false, "Added new setting to allow to write type names in binary format in RowBinaryWithNamesAndTypes output format"},
               {"input_format_binary_decode_types_in_binary_format", false, false, "Added new setting to allow to read type names in binary format in RowBinaryWithNamesAndTypes input format"},
@@ -76,7 +86,6 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
               {"azure_sdk_max_retries", 10, 10, "Maximum number of retries in azure sdk"},
               {"azure_sdk_retry_initial_backoff_ms", 10, 10, "Minimal backoff between retries in azure sdk"},
               {"azure_sdk_retry_max_backoff_ms", 1000, 1000, "Maximal backoff between retries in azure sdk"},
-              {"merge_tree_min_bytes_per_task_for_remote_reading", 4194304, 2097152, "Value is unified with `filesystem_prefetch_min_bytes_for_single_read_task`"},
               {"ignore_on_cluster_for_replicated_named_collections_queries", false, false, "Ignore ON CLUSTER clause for replicated named collections management queries."},
               {"backup_restore_s3_retry_attempts", 1000,1000, "Setting for Aws::Client::RetryStrategy, Aws::Client does retries itself, 0 means no retries. It takes place only for backup/restore."},
               {"postgresql_connection_attempt_timeout", 2, 2, "Allow to control 'connect_timeout' parameter of PostgreSQL connection."},

From baae45889bad6bb6827278ca677025b01dab1da7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Wed, 31 Jul 2024 15:11:55 +0200
Subject: [PATCH 1113/1488] Try a less conflict prone format

---
 src/Core/SettingsChangesHistory.cpp | 709 +++++++++++++++++-----------
 1 file changed, 441 insertions(+), 268 deletions(-)

diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp
index ecc558e64d7..21c89b3c5c5 100644
--- a/src/Core/SettingsChangesHistory.cpp
+++ b/src/Core/SettingsChangesHistory.cpp
@@ -57,274 +57,447 @@ String ClickHouseVersion::toString() const
 /// Note: please check if the key already exists to prevent duplicate entries.
 static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory::SettingsChanges>> settings_changes_history_initializer =
 {
-    {"24.12",{
-             }},
-    {"24.11",{
-             }},
-    {"24.10",{
-             }},
-    {"24.9", {
-             }},
-    {"24.8", {{"merge_tree_min_bytes_per_task_for_remote_reading", 4194304, 2097152, "Value is unified with `filesystem_prefetch_min_bytes_for_single_read_task`"},
-              }},
-    {"24.7", {{"output_format_parquet_write_page_index", false, true, "Add a possibility to write page index into parquet files."},
-              {"output_format_binary_encode_types_in_binary_format", false, false, "Added new setting to allow to write type names in binary format in RowBinaryWithNamesAndTypes output format"},
-              {"input_format_binary_decode_types_in_binary_format", false, false, "Added new setting to allow to read type names in binary format in RowBinaryWithNamesAndTypes input format"},
-              {"output_format_native_encode_types_in_binary_format", false, false, "Added new setting to allow to write type names in binary format in Native output format"},
-              {"input_format_native_decode_types_in_binary_format", false, false, "Added new setting to allow to read type names in binary format in Native output format"},
-              {"read_in_order_use_buffering", false, true, "Use buffering before merging while reading in order of primary key"},
-              {"enable_named_columns_in_function_tuple", false, true, "Generate named tuples in function tuple() when all names are unique and can be treated as unquoted identifiers."},
-              {"input_format_json_case_insensitive_column_matching", false, false, "Ignore case when matching JSON keys with CH columns."},
-              {"optimize_trivial_insert_select", true, false, "The optimization does not make sense in many cases."},
-              {"dictionary_validate_primary_key_type", false, false, "Validate primary key type for dictionaries. By default id type for simple layouts will be implicitly converted to UInt64."},
-              {"collect_hash_table_stats_during_joins", false, true, "New setting."},
-              {"max_size_to_preallocate_for_joins", 0, 100'000'000, "New setting."},
-              {"input_format_orc_read_use_writer_time_zone", false, false, "Whether use the writer's time zone in ORC stripe for ORC row reader, the default ORC row reader's time zone is GMT."},
-              {"lightweight_mutation_projection_mode", "throw", "throw", "When lightweight delete happens on a table with projection(s), the possible operations include throw the exception as projection exists, or drop all projection related to this table then do lightweight delete."},
-              {"database_replicated_allow_heavy_create", true, false, "Long-running DDL queries (CREATE AS SELECT and POPULATE) for Replicated database engine was forbidden"},
-              {"query_plan_merge_filters", false, false, "Allow to merge filters in the query plan"},
-              {"azure_sdk_max_retries", 10, 10, "Maximum number of retries in azure sdk"},
-              {"azure_sdk_retry_initial_backoff_ms", 10, 10, "Minimal backoff between retries in azure sdk"},
-              {"azure_sdk_retry_max_backoff_ms", 1000, 1000, "Maximal backoff between retries in azure sdk"},
-              {"ignore_on_cluster_for_replicated_named_collections_queries", false, false, "Ignore ON CLUSTER clause for replicated named collections management queries."},
-              {"backup_restore_s3_retry_attempts", 1000,1000, "Setting for Aws::Client::RetryStrategy, Aws::Client does retries itself, 0 means no retries. It takes place only for backup/restore."},
-              {"postgresql_connection_attempt_timeout", 2, 2, "Allow to control 'connect_timeout' parameter of PostgreSQL connection."},
-              {"postgresql_connection_pool_retries", 2, 2, "Allow to control the number of retries in PostgreSQL connection pool."}
-              }},
-    {"24.6", {{"materialize_skip_indexes_on_insert", true, true, "Added new setting to allow to disable materialization of skip indexes on insert"},
-              {"materialize_statistics_on_insert", true, true, "Added new setting to allow to disable materialization of statistics on insert"},
-              {"input_format_parquet_use_native_reader", false, false, "When reading Parquet files, to use native reader instead of arrow reader."},
-              {"hdfs_throw_on_zero_files_match", false, false, "Allow to throw an error when ListObjects request cannot match any files in HDFS engine instead of empty query result"},
-              {"azure_throw_on_zero_files_match", false, false, "Allow to throw an error when ListObjects request cannot match any files in AzureBlobStorage engine instead of empty query result"},
-              {"s3_validate_request_settings", true, true, "Allow to disable S3 request settings validation"},
-              {"allow_experimental_full_text_index", false, false, "Enable experimental full-text index"},
-              {"azure_skip_empty_files", false, false, "Allow to skip empty files in azure table engine"},
-              {"hdfs_ignore_file_doesnt_exist", false, false, "Allow to return 0 rows when the requested files don't exist instead of throwing an exception in HDFS table engine"},
-              {"azure_ignore_file_doesnt_exist", false, false, "Allow to return 0 rows when the requested files don't exist instead of throwing an exception in AzureBlobStorage table engine"},
-              {"s3_ignore_file_doesnt_exist", false, false, "Allow to return 0 rows when the requested files don't exist instead of throwing an exception in S3 table engine"},
-              {"s3_max_part_number", 10000, 10000, "Maximum part number number for s3 upload part"},
-              {"s3_max_single_operation_copy_size", 32 * 1024 * 1024, 32 * 1024 * 1024, "Maximum size for a single copy operation in s3"},
-              {"input_format_parquet_max_block_size", 8192, DEFAULT_BLOCK_SIZE, "Increase block size for parquet reader."},
-              {"input_format_parquet_prefer_block_bytes", 0, DEFAULT_BLOCK_SIZE * 256, "Average block bytes output by parquet reader."},
-              {"enable_blob_storage_log", true, true, "Write information about blob storage operations to system.blob_storage_log table"},
-              {"allow_deprecated_snowflake_conversion_functions", true, false, "Disabled deprecated functions snowflakeToDateTime[64] and dateTime[64]ToSnowflake."},
-              {"allow_statistic_optimize", false, false, "Old setting which popped up here being renamed."},
-              {"allow_experimental_statistic", false, false, "Old setting which popped up here being renamed."},
-              {"allow_statistics_optimize", false, false, "The setting was renamed. The previous name is `allow_statistic_optimize`."},
-              {"allow_experimental_statistics", false, false, "The setting was renamed. The previous name is `allow_experimental_statistic`."},
-              {"enable_vertical_final", false, true, "Enable vertical final by default again after fixing bug"},
-              {"parallel_replicas_custom_key_range_lower", 0, 0, "Add settings to control the range filter when using parallel replicas with dynamic shards"},
-              {"parallel_replicas_custom_key_range_upper", 0, 0, "Add settings to control the range filter when using parallel replicas with dynamic shards. A value of 0 disables the upper limit"},
-              {"output_format_pretty_display_footer_column_names", 0, 1, "Add a setting to display column names in the footer if there are many rows. Threshold value is controlled by output_format_pretty_display_footer_column_names_min_rows."},
-              {"output_format_pretty_display_footer_column_names_min_rows", 0, 50, "Add a setting to control the threshold value for setting output_format_pretty_display_footer_column_names_min_rows. Default 50."},
-              {"output_format_csv_serialize_tuple_into_separate_columns", true, true, "A new way of how interpret tuples in CSV format was added."},
-              {"input_format_csv_deserialize_separate_columns_into_tuple", true, true, "A new way of how interpret tuples in CSV format was added."},
-              {"input_format_csv_try_infer_strings_from_quoted_tuples", true, true, "A new way of how interpret tuples in CSV format was added."},
-              }},
-    {"24.5", {{"allow_deprecated_error_prone_window_functions", true, false, "Allow usage of deprecated error prone window functions (neighbor, runningAccumulate, runningDifferenceStartingWithFirstValue, runningDifference)"},
-              {"allow_experimental_join_condition", false, false, "Support join with inequal conditions which involve columns from both left and right table. e.g. t1.y < t2.y."},
-              {"input_format_tsv_crlf_end_of_line", false, false, "Enables reading of CRLF line endings with TSV formats"},
-              {"output_format_parquet_use_custom_encoder", false, true, "Enable custom Parquet encoder."},
-              {"cross_join_min_rows_to_compress", 0, 10000000, "Minimal count of rows to compress block in CROSS JOIN. Zero value means - disable this threshold. This block is compressed when any of the two thresholds (by rows or by bytes) are reached."},
-              {"cross_join_min_bytes_to_compress", 0, 1_GiB, "Minimal size of block to compress in CROSS JOIN. Zero value means - disable this threshold. This block is compressed when any of the two thresholds (by rows or by bytes) are reached."},
-              {"http_max_chunk_size", 0, 0, "Internal limitation"},
-              {"prefer_external_sort_block_bytes", 0, DEFAULT_BLOCK_SIZE * 256, "Prefer maximum block bytes for external sort, reduce the memory usage during merging."},
-              {"input_format_force_null_for_omitted_fields", false, false, "Disable type-defaults for omitted fields when needed"},
-              {"cast_string_to_dynamic_use_inference", false, false, "Add setting to allow converting String to Dynamic through parsing"},
-              {"allow_experimental_dynamic_type", false, false, "Add new experimental Dynamic type"},
-              {"azure_max_blocks_in_multipart_upload", 50000, 50000, "Maximum number of blocks in multipart upload for Azure."},
-              }},
-    {"24.4", {{"input_format_json_throw_on_bad_escape_sequence", true, true, "Allow to save JSON strings with bad escape sequences"},
-              {"max_parsing_threads", 0, 0, "Add a separate setting to control number of threads in parallel parsing from files"},
-              {"ignore_drop_queries_probability", 0, 0, "Allow to ignore drop queries in server with specified probability for testing purposes"},
-              {"lightweight_deletes_sync", 2, 2, "The same as 'mutation_sync', but controls only execution of lightweight deletes"},
-              {"query_cache_system_table_handling", "save", "throw", "The query cache no longer caches results of queries against system tables"},
-              {"input_format_json_ignore_unnecessary_fields", false, true, "Ignore unnecessary fields and not parse them. Enabling this may not throw exceptions on json strings of invalid format or with duplicated fields"},
-              {"input_format_hive_text_allow_variable_number_of_columns", false, true, "Ignore extra columns in Hive Text input (if file has more columns than expected) and treat missing fields in Hive Text input as default values."},
-              {"allow_experimental_database_replicated", false, true, "Database engine Replicated is now in Beta stage"},
-              {"temporary_data_in_cache_reserve_space_wait_lock_timeout_milliseconds", (10 * 60 * 1000), (10 * 60 * 1000), "Wait time to lock cache for sapce reservation in temporary data in filesystem cache"},
-              {"optimize_rewrite_sum_if_to_count_if", false, true, "Only available for the analyzer, where it works correctly"},
-              {"azure_allow_parallel_part_upload", "true", "true", "Use multiple threads for azure multipart upload."},
-              {"max_recursive_cte_evaluation_depth", DBMS_RECURSIVE_CTE_MAX_EVALUATION_DEPTH, DBMS_RECURSIVE_CTE_MAX_EVALUATION_DEPTH, "Maximum limit on recursive CTE evaluation depth"},
-              {"query_plan_convert_outer_join_to_inner_join", false, true, "Allow to convert OUTER JOIN to INNER JOIN if filter after JOIN always filters default values"},
-              }},
-    {"24.3", {{"s3_connect_timeout_ms", 1000, 1000, "Introduce new dedicated setting for s3 connection timeout"},
-              {"allow_experimental_shared_merge_tree", false, true, "The setting is obsolete"},
-              {"use_page_cache_for_disks_without_file_cache", false, false, "Added userspace page cache"},
-              {"read_from_page_cache_if_exists_otherwise_bypass_cache", false, false, "Added userspace page cache"},
-              {"page_cache_inject_eviction", false, false, "Added userspace page cache"},
-              {"default_table_engine", "None", "MergeTree", "Set default table engine to MergeTree for better usability"},
-              {"input_format_json_use_string_type_for_ambiguous_paths_in_named_tuples_inference_from_objects", false, false, "Allow to use String type for ambiguous paths during named tuple inference from JSON objects"},
-              {"traverse_shadow_remote_data_paths", false, false, "Traverse shadow directory when query system.remote_data_paths."},
-              {"throw_if_deduplication_in_dependent_materialized_views_enabled_with_async_insert", false, true, "Deduplication in dependent materialized view cannot work together with async inserts."},
-              {"parallel_replicas_allow_in_with_subquery", false, true, "If true, subquery for IN will be executed on every follower replica"},
-              {"log_processors_profiles", false, true, "Enable by default"},
-              {"function_locate_has_mysql_compatible_argument_order", false, true, "Increase compatibility with MySQL's locate function."},
-              {"allow_suspicious_primary_key", true, false, "Forbid suspicious PRIMARY KEY/ORDER BY for MergeTree (i.e. SimpleAggregateFunction)"},
-              {"filesystem_cache_reserve_space_wait_lock_timeout_milliseconds", 1000, 1000, "Wait time to lock cache for sapce reservation in filesystem cache"},
-              {"max_parser_backtracks", 0, 1000000, "Limiting the complexity of parsing"},
-              {"analyzer_compatibility_join_using_top_level_identifier", false, false, "Force to resolve identifier in JOIN USING from projection"},
-              {"distributed_insert_skip_read_only_replicas", false, false, "If true, INSERT into Distributed will skip read-only replicas"},
-              {"keeper_max_retries", 10, 10, "Max retries for general keeper operations"},
-              {"keeper_retry_initial_backoff_ms", 100, 100, "Initial backoff timeout for general keeper operations"},
-              {"keeper_retry_max_backoff_ms", 5000, 5000, "Max backoff timeout for general keeper operations"},
-              {"s3queue_allow_experimental_sharded_mode", false, false, "Enable experimental sharded mode of S3Queue table engine. It is experimental because it will be rewritten"},
-              {"allow_experimental_analyzer", false, true, "Enable analyzer and planner by default."},
-              {"merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability", 0.0, 0.0, "For testing of `PartsSplitter` - split read ranges into intersecting and non intersecting every time you read from MergeTree with the specified probability."},
-              {"allow_get_client_http_header", false, false, "Introduced a new function."},
-              {"output_format_pretty_row_numbers", false, true, "It is better for usability."},
-              {"output_format_pretty_max_value_width_apply_for_single_value", true, false, "Single values in Pretty formats won't be cut."},
-              {"output_format_parquet_string_as_string", false, true, "ClickHouse allows arbitrary binary data in the String data type, which is typically UTF-8. Parquet/ORC/Arrow Strings only support UTF-8. That's why you can choose which Arrow's data type to use for the ClickHouse String data type - String or Binary. While Binary would be more correct and compatible, using String by default will correspond to user expectations in most cases."},
-              {"output_format_orc_string_as_string", false, true, "ClickHouse allows arbitrary binary data in the String data type, which is typically UTF-8. Parquet/ORC/Arrow Strings only support UTF-8. That's why you can choose which Arrow's data type to use for the ClickHouse String data type - String or Binary. While Binary would be more correct and compatible, using String by default will correspond to user expectations in most cases."},
-              {"output_format_arrow_string_as_string", false, true, "ClickHouse allows arbitrary binary data in the String data type, which is typically UTF-8. Parquet/ORC/Arrow Strings only support UTF-8. That's why you can choose which Arrow's data type to use for the ClickHouse String data type - String or Binary. While Binary would be more correct and compatible, using String by default will correspond to user expectations in most cases."},
-              {"output_format_parquet_compression_method", "lz4", "zstd", "Parquet/ORC/Arrow support many compression methods, including lz4 and zstd. ClickHouse supports each and every compression method. Some inferior tools, such as 'duckdb', lack support for the faster `lz4` compression method, that's why we set zstd by default."},
-              {"output_format_orc_compression_method", "lz4", "zstd", "Parquet/ORC/Arrow support many compression methods, including lz4 and zstd. ClickHouse supports each and every compression method. Some inferior tools, such as 'duckdb', lack support for the faster `lz4` compression method, that's why we set zstd by default."},
-              {"output_format_pretty_highlight_digit_groups", false, true, "If enabled and if output is a terminal, highlight every digit corresponding to the number of thousands, millions, etc. with underline."},
-              {"geo_distance_returns_float64_on_float64_arguments", false, true, "Increase the default precision."},
-              {"azure_max_inflight_parts_for_one_file", 20, 20, "The maximum number of a concurrent loaded parts in multipart upload request. 0 means unlimited."},
-              {"azure_strict_upload_part_size", 0, 0, "The exact size of part to upload during multipart upload to Azure blob storage."},
-              {"azure_min_upload_part_size", 16*1024*1024, 16*1024*1024, "The minimum size of part to upload during multipart upload to Azure blob storage."},
-              {"azure_max_upload_part_size", 5ull*1024*1024*1024, 5ull*1024*1024*1024, "The maximum size of part to upload during multipart upload to Azure blob storage."},
-              {"azure_upload_part_size_multiply_factor", 2, 2, "Multiply azure_min_upload_part_size by this factor each time azure_multiply_parts_count_threshold parts were uploaded from a single write to Azure blob storage."},
-              {"azure_upload_part_size_multiply_parts_count_threshold", 500, 500, "Each time this number of parts was uploaded to Azure blob storage, azure_min_upload_part_size is multiplied by azure_upload_part_size_multiply_factor."},
-              {"output_format_csv_serialize_tuple_into_separate_columns", true, true, "A new way of how interpret tuples in CSV format was added."},
-              {"input_format_csv_deserialize_separate_columns_into_tuple", true, true, "A new way of how interpret tuples in CSV format was added."},
-              {"input_format_csv_try_infer_strings_from_quoted_tuples", true, true, "A new way of how interpret tuples in CSV format was added."},
-              }},
-    {"24.2", {{"allow_suspicious_variant_types", true, false, "Don't allow creating Variant type with suspicious variants by default"},
-              {"validate_experimental_and_suspicious_types_inside_nested_types", false, true, "Validate usage of experimental and suspicious types inside nested types"},
-              {"output_format_values_escape_quote_with_quote", false, false, "If true escape ' with '', otherwise quoted with \\'"},
-              {"output_format_pretty_single_large_number_tip_threshold", 0, 1'000'000, "Print a readable number tip on the right side of the table if the block consists of a single number which exceeds this value (except 0)"},
-              {"input_format_try_infer_exponent_floats", true, false, "Don't infer floats in exponential notation by default"},
-              {"query_plan_optimize_prewhere", true, true, "Allow to push down filter to PREWHERE expression for supported storages"},
-              {"async_insert_max_data_size", 1000000, 10485760, "The previous value appeared to be too small."},
-              {"async_insert_poll_timeout_ms", 10, 10, "Timeout in milliseconds for polling data from asynchronous insert queue"},
-              {"async_insert_use_adaptive_busy_timeout", false, true, "Use adaptive asynchronous insert timeout"},
-              {"async_insert_busy_timeout_min_ms", 50, 50, "The minimum value of the asynchronous insert timeout in milliseconds; it also serves as the initial value, which may be increased later by the adaptive algorithm"},
-              {"async_insert_busy_timeout_max_ms", 200, 200, "The minimum value of the asynchronous insert timeout in milliseconds; async_insert_busy_timeout_ms is aliased to async_insert_busy_timeout_max_ms"},
-              {"async_insert_busy_timeout_increase_rate", 0.2, 0.2, "The exponential growth rate at which the adaptive asynchronous insert timeout increases"},
-              {"async_insert_busy_timeout_decrease_rate", 0.2, 0.2, "The exponential growth rate at which the adaptive asynchronous insert timeout decreases"},
-              {"format_template_row_format", "", "", "Template row format string can be set directly in query"},
-              {"format_template_resultset_format", "", "", "Template result set format string can be set in query"},
-              {"split_parts_ranges_into_intersecting_and_non_intersecting_final", true, true, "Allow to split parts ranges into intersecting and non intersecting during FINAL optimization"},
-              {"split_intersecting_parts_ranges_into_layers_final", true, true, "Allow to split intersecting parts ranges into layers during FINAL optimization"},
-              {"azure_max_single_part_copy_size", 256*1024*1024, 256*1024*1024, "The maximum size of object to copy using single part copy to Azure blob storage."},
-              {"min_external_table_block_size_rows", DEFAULT_INSERT_BLOCK_SIZE, DEFAULT_INSERT_BLOCK_SIZE, "Squash blocks passed to external table to specified size in rows, if blocks are not big enough"},
-              {"min_external_table_block_size_bytes", DEFAULT_INSERT_BLOCK_SIZE * 256, DEFAULT_INSERT_BLOCK_SIZE * 256, "Squash blocks passed to external table to specified size in bytes, if blocks are not big enough."},
-              {"parallel_replicas_prefer_local_join", true, true, "If true, and JOIN can be executed with parallel replicas algorithm, and all storages of right JOIN part are *MergeTree, local JOIN will be used instead of GLOBAL JOIN."},
-              {"optimize_time_filter_with_preimage", true, true, "Optimize Date and DateTime predicates by converting functions into equivalent comparisons without conversions (e.g. toYear(col) = 2023 -> col >= '2023-01-01' AND col <= '2023-12-31')"},
-              {"extract_key_value_pairs_max_pairs_per_row", 0, 0, "Max number of pairs that can be produced by the `extractKeyValuePairs` function. Used as a safeguard against consuming too much memory."},
-              {"default_view_definer", "CURRENT_USER", "CURRENT_USER", "Allows to set default `DEFINER` option while creating a view"},
-              {"default_materialized_view_sql_security", "DEFINER", "DEFINER", "Allows to set a default value for SQL SECURITY option when creating a materialized view"},
-              {"default_normal_view_sql_security", "INVOKER", "INVOKER", "Allows to set default `SQL SECURITY` option while creating a normal view"},
-              {"mysql_map_string_to_text_in_show_columns", false, true, "Reduce the configuration effort to connect ClickHouse with BI tools."},
-              {"mysql_map_fixed_string_to_text_in_show_columns", false, true, "Reduce the configuration effort to connect ClickHouse with BI tools."},
-              }},
-    {"24.1", {{"print_pretty_type_names", false, true, "Better user experience."},
-              {"input_format_json_read_bools_as_strings", false, true, "Allow to read bools as strings in JSON formats by default"},
-              {"output_format_arrow_use_signed_indexes_for_dictionary", false, true, "Use signed indexes type for Arrow dictionaries by default as it's recommended"},
-              {"allow_experimental_variant_type", false, false, "Add new experimental Variant type"},
-              {"use_variant_as_common_type", false, false, "Allow to use Variant in if/multiIf if there is no common type"},
-              {"output_format_arrow_use_64_bit_indexes_for_dictionary", false, false, "Allow to use 64 bit indexes type in Arrow dictionaries"},
-              {"parallel_replicas_mark_segment_size", 128, 128, "Add new setting to control segment size in new parallel replicas coordinator implementation"},
-              {"ignore_materialized_views_with_dropped_target_table", false, false, "Add new setting to allow to ignore materialized views with dropped target table"},
-              {"output_format_compression_level", 3, 3, "Allow to change compression level in the query output"},
-              {"output_format_compression_zstd_window_log", 0, 0, "Allow to change zstd window log in the query output when zstd compression is used"},
-              {"enable_zstd_qat_codec", false, false, "Add new ZSTD_QAT codec"},
-              {"enable_vertical_final", false, true, "Use vertical final by default"},
-              {"output_format_arrow_use_64_bit_indexes_for_dictionary", false, false, "Allow to use 64 bit indexes type in Arrow dictionaries"},
-              {"max_rows_in_set_to_optimize_join", 100000, 0, "Disable join optimization as it prevents from read in order optimization"},
-              {"output_format_pretty_color", true, "auto", "Setting is changed to allow also for auto value, disabling ANSI escapes if output is not a tty"},
-              {"function_visible_width_behavior", 0, 1, "We changed the default behavior of `visibleWidth` to be more precise"},
-              {"max_estimated_execution_time", 0, 0, "Separate max_execution_time and max_estimated_execution_time"},
-              {"iceberg_engine_ignore_schema_evolution", false, false, "Allow to ignore schema evolution in Iceberg table engine"},
-              {"optimize_injective_functions_in_group_by", false, true, "Replace injective functions by it's arguments in GROUP BY section in analyzer"},
-              {"update_insert_deduplication_token_in_dependent_materialized_views", false, false, "Allow to update insert deduplication token with table identifier during insert in dependent materialized views"},
-              {"azure_max_unexpected_write_error_retries", 4, 4, "The maximum number of retries in case of unexpected errors during Azure blob storage write"},
-              {"split_parts_ranges_into_intersecting_and_non_intersecting_final", false, true, "Allow to split parts ranges into intersecting and non intersecting during FINAL optimization"},
-              {"split_intersecting_parts_ranges_into_layers_final", true, true, "Allow to split intersecting parts ranges into layers during FINAL optimization"}}},
-    {"23.12", {{"allow_suspicious_ttl_expressions", true, false, "It is a new setting, and in previous versions the behavior was equivalent to allowing."},
-              {"input_format_parquet_allow_missing_columns", false, true, "Allow missing columns in Parquet files by default"},
-              {"input_format_orc_allow_missing_columns", false, true, "Allow missing columns in ORC files by default"},
-              {"input_format_arrow_allow_missing_columns", false, true, "Allow missing columns in Arrow files by default"}}},
-    {"23.11", {{"parsedatetime_parse_without_leading_zeros", false, true, "Improved compatibility with MySQL DATE_FORMAT/STR_TO_DATE"}}},
-    {"23.9", {{"optimize_group_by_constant_keys", false, true, "Optimize group by constant keys by default"},
-              {"input_format_json_try_infer_named_tuples_from_objects", false, true, "Try to infer named Tuples from JSON objects by default"},
-              {"input_format_json_read_numbers_as_strings", false, true, "Allow to read numbers as strings in JSON formats by default"},
-              {"input_format_json_read_arrays_as_strings", false, true, "Allow to read arrays as strings in JSON formats by default"},
-              {"input_format_json_infer_incomplete_types_as_strings", false, true, "Allow to infer incomplete types as Strings in JSON formats by default"},
-              {"input_format_json_try_infer_numbers_from_strings", true, false, "Don't infer numbers from strings in JSON formats by default to prevent possible parsing errors"},
-              {"http_write_exception_in_output_format", false, true, "Output valid JSON/XML on exception in HTTP streaming."}}},
-    {"23.8", {{"rewrite_count_distinct_if_with_count_distinct_implementation", false, true, "Rewrite countDistinctIf with count_distinct_implementation configuration"}}},
-    {"23.7", {{"function_sleep_max_microseconds_per_block", 0, 3000000, "In previous versions, the maximum sleep time of 3 seconds was applied only for `sleep`, but not for `sleepEachRow` function. In the new version, we introduce this setting. If you set compatibility with the previous versions, we will disable the limit altogether."}}},
-    {"23.6", {{"http_send_timeout", 180, 30, "3 minutes seems crazy long. Note that this is timeout for a single network write call, not for the whole upload operation."},
-              {"http_receive_timeout", 180, 30, "See http_send_timeout."}}},
-    {"23.5", {{"input_format_parquet_preserve_order", true, false, "Allow Parquet reader to reorder rows for better parallelism."},
-              {"parallelize_output_from_storages", false, true, "Allow parallelism when executing queries that read from file/url/s3/etc. This may reorder rows."},
-              {"use_with_fill_by_sorting_prefix", false, true, "Columns preceding WITH FILL columns in ORDER BY clause form sorting prefix. Rows with different values in sorting prefix are filled independently"},
-              {"output_format_parquet_compliant_nested_types", false, true, "Change an internal field name in output Parquet file schema."}}},
-    {"23.4", {{"allow_suspicious_indices", true, false, "If true, index can defined with identical expressions"},
-              {"allow_nonconst_timezone_arguments", true, false, "Allow non-const timezone arguments in certain time-related functions like toTimeZone(), fromUnixTimestamp*(), snowflakeToDateTime*()."},
-              {"connect_timeout_with_failover_ms", 50, 1000, "Increase default connect timeout because of async connect"},
-              {"connect_timeout_with_failover_secure_ms", 100, 1000, "Increase default secure connect timeout because of async connect"},
-              {"hedged_connection_timeout_ms", 100, 50, "Start new connection in hedged requests after 50 ms instead of 100 to correspond with previous connect timeout"},
-              {"formatdatetime_f_prints_single_zero", true, false, "Improved compatibility with MySQL DATE_FORMAT()/STR_TO_DATE()"},
-              {"formatdatetime_parsedatetime_m_is_month_name", false, true, "Improved compatibility with MySQL DATE_FORMAT/STR_TO_DATE"}}},
-    {"23.3", {{"output_format_parquet_version", "1.0", "2.latest", "Use latest Parquet format version for output format"},
-              {"input_format_json_ignore_unknown_keys_in_named_tuple", false, true, "Improve parsing JSON objects as named tuples"},
-              {"input_format_native_allow_types_conversion", false, true, "Allow types conversion in Native input forma"},
-              {"output_format_arrow_compression_method", "none", "lz4_frame", "Use lz4 compression in Arrow output format by default"},
-              {"output_format_parquet_compression_method", "snappy", "lz4", "Use lz4 compression in Parquet output format by default"},
-              {"output_format_orc_compression_method", "none", "lz4_frame", "Use lz4 compression in ORC output format by default"},
-              {"async_query_sending_for_remote", false, true, "Create connections and send query async across shards"}}},
-    {"23.2", {{"output_format_parquet_fixed_string_as_fixed_byte_array", false, true, "Use Parquet FIXED_LENGTH_BYTE_ARRAY type for FixedString by default"},
-              {"output_format_arrow_fixed_string_as_fixed_byte_array", false, true, "Use Arrow FIXED_SIZE_BINARY type for FixedString by default"},
-              {"query_plan_remove_redundant_distinct", false, true, "Remove redundant Distinct step in query plan"},
-              {"optimize_duplicate_order_by_and_distinct", true, false, "Remove duplicate ORDER BY and DISTINCT if it's possible"},
-              {"insert_keeper_max_retries", 0, 20, "Enable reconnections to Keeper on INSERT, improve reliability"}}},
-    {"23.1", {{"input_format_json_read_objects_as_strings", 0, 1, "Enable reading nested json objects as strings while object type is experimental"},
-              {"input_format_json_defaults_for_missing_elements_in_named_tuple", false, true, "Allow missing elements in JSON objects while reading named tuples by default"},
-              {"input_format_csv_detect_header", false, true, "Detect header in CSV format by default"},
-              {"input_format_tsv_detect_header", false, true, "Detect header in TSV format by default"},
-              {"input_format_custom_detect_header", false, true, "Detect header in CustomSeparated format by default"},
-              {"query_plan_remove_redundant_sorting", false, true, "Remove redundant sorting in query plan. For example, sorting steps related to ORDER BY clauses in subqueries"}}},
-    {"22.12", {{"max_size_to_preallocate_for_aggregation", 10'000'000, 100'000'000, "This optimizes performance"},
-               {"query_plan_aggregation_in_order", 0, 1, "Enable some refactoring around query plan"},
-               {"format_binary_max_string_size", 0, 1_GiB, "Prevent allocating large amount of memory"}}},
-    {"22.11", {{"use_structure_from_insertion_table_in_table_functions", 0, 2, "Improve using structure from insertion table in table functions"}}},
-    {"22.9", {{"force_grouping_standard_compatibility", false, true, "Make GROUPING function output the same as in SQL standard and other DBMS"}}},
-    {"22.7", {{"cross_to_inner_join_rewrite", 1, 2, "Force rewrite comma join to inner"},
-              {"enable_positional_arguments", false, true, "Enable positional arguments feature by default"},
-              {"format_csv_allow_single_quotes", true, false, "Most tools don't treat single quote in CSV specially, don't do it by default too"}}},
-    {"22.6", {{"output_format_json_named_tuples_as_objects", false, true, "Allow to serialize named tuples as JSON objects in JSON formats by default"},
-              {"input_format_skip_unknown_fields", false, true, "Optimize reading subset of columns for some input formats"}}},
-    {"22.5", {{"memory_overcommit_ratio_denominator", 0, 1073741824, "Enable memory overcommit feature by default"},
-              {"memory_overcommit_ratio_denominator_for_user", 0, 1073741824, "Enable memory overcommit feature by default"}}},
-    {"22.4", {{"allow_settings_after_format_in_insert", true, false, "Do not allow SETTINGS after FORMAT for INSERT queries because ClickHouse interpret SETTINGS as some values, which is misleading"}}},
-    {"22.3", {{"cast_ipv4_ipv6_default_on_conversion_error", true, false, "Make functions cast(value, 'IPv4') and cast(value, 'IPv6') behave same as toIPv4 and toIPv6 functions"}}},
-    {"21.12", {{"stream_like_engine_allow_direct_select", true, false, "Do not allow direct select for Kafka/RabbitMQ/FileLog by default"}}},
-    {"21.9", {{"output_format_decimal_trailing_zeros", true, false, "Do not output trailing zeros in text representation of Decimal types by default for better looking output"},
-              {"use_hedged_requests", false, true, "Enable Hedged Requests feature by default"}}},
-    {"21.7", {{"legacy_column_name_of_tuple_literal", true, false, "Add this setting only for compatibility reasons. It makes sense to set to 'true', while doing rolling update of cluster from version lower than 21.7 to higher"}}},
-    {"21.5", {{"async_socket_for_remote", false, true, "Fix all problems and turn on asynchronous reads from socket for remote queries by default again"}}},
-    {"21.3", {{"async_socket_for_remote", true, false, "Turn off asynchronous reads from socket for remote queries because of some problems"},
-              {"optimize_normalize_count_variants", false, true, "Rewrite aggregate functions that semantically equals to count() as count() by default"},
-              {"normalize_function_names", false, true, "Normalize function names to their canonical names, this was needed for projection query routing"}}},
-    {"21.2", {{"enable_global_with_statement", false, true, "Propagate WITH statements to UNION queries and all subqueries by default"}}},
-    {"21.1", {{"insert_quorum_parallel", false, true, "Use parallel quorum inserts by default. It is significantly more convenient to use than sequential quorum inserts"},
-              {"input_format_null_as_default", false, true, "Allow to insert NULL as default for input formats by default"},
-              {"optimize_on_insert", false, true, "Enable data optimization on INSERT by default for better user experience"},
-              {"use_compact_format_in_distributed_parts_names", false, true, "Use compact format for async INSERT into Distributed tables by default"}}},
-    {"20.10", {{"format_regexp_escaping_rule", "Escaped", "Raw", "Use Raw as default escaping rule for Regexp format to male the behaviour more like to what users expect"}}},
-    {"20.7", {{"show_table_uuid_in_table_create_query_if_not_nil", true, false, "Stop showing  UID of the table in its CREATE query for Engine=Atomic"}}},
-    {"20.5", {{"input_format_with_names_use_header", false, true, "Enable using header with names for formats with WithNames/WithNamesAndTypes suffixes"},
-              {"allow_suspicious_codecs", true, false, "Don't allow to specify meaningless compression codecs"}}},
-    {"20.4", {{"validate_polygons", false, true, "Throw exception if polygon is invalid in function pointInPolygon by default instead of returning possibly wrong results"}}},
-    {"19.18", {{"enable_scalar_subquery_optimization", false, true, "Prevent scalar subqueries from (de)serializing large scalar values and possibly avoid running the same subquery more than once"}}},
-    {"19.14", {{"any_join_distinct_right_table_keys", true, false, "Disable ANY RIGHT and ANY FULL JOINs by default to avoid inconsistency"}}},
-    {"19.12", {{"input_format_defaults_for_omitted_fields", false, true, "Enable calculation of complex default expressions for omitted fields for some input formats, because it should be the expected behaviour"}}},
-    {"19.5", {{"max_partitions_per_insert_block", 0, 100, "Add a limit for the number of partitions in one block"}}},
-    {"18.12.17", {{"enable_optimize_predicate_expression", 0, 1, "Optimize predicates to subqueries by default"}}},
+    {"24.12",
+        {
+        }
+    },
+    {"24.11",
+        {
+        }
+    },
+    {"24.10",
+        {
+        }
+    },
+    {"24.9",
+        {
+        }
+    },
+    {"24.8",
+        {
+            {"merge_tree_min_bytes_per_task_for_remote_reading", 4194304, 2097152, "Value is unified with `filesystem_prefetch_min_bytes_for_single_read_task`"},
+        }
+    },
+    {"24.7",
+        {
+            {"output_format_parquet_write_page_index", false, true, "Add a possibility to write page index into parquet files."},
+            {"output_format_binary_encode_types_in_binary_format", false, false, "Added new setting to allow to write type names in binary format in RowBinaryWithNamesAndTypes output format"},
+            {"input_format_binary_decode_types_in_binary_format", false, false, "Added new setting to allow to read type names in binary format in RowBinaryWithNamesAndTypes input format"},
+            {"output_format_native_encode_types_in_binary_format", false, false, "Added new setting to allow to write type names in binary format in Native output format"},
+            {"input_format_native_decode_types_in_binary_format", false, false, "Added new setting to allow to read type names in binary format in Native output format"},
+            {"read_in_order_use_buffering", false, true, "Use buffering before merging while reading in order of primary key"},
+            {"enable_named_columns_in_function_tuple", false, true, "Generate named tuples in function tuple() when all names are unique and can be treated as unquoted identifiers."},
+            {"input_format_json_case_insensitive_column_matching", false, false, "Ignore case when matching JSON keys with CH columns."},
+            {"optimize_trivial_insert_select", true, false, "The optimization does not make sense in many cases."},
+            {"dictionary_validate_primary_key_type", false, false, "Validate primary key type for dictionaries. By default id type for simple layouts will be implicitly converted to UInt64."},
+            {"collect_hash_table_stats_during_joins", false, true, "New setting."},
+            {"max_size_to_preallocate_for_joins", 0, 100'000'000, "New setting."},
+            {"input_format_orc_read_use_writer_time_zone", false, false, "Whether use the writer's time zone in ORC stripe for ORC row reader, the default ORC row reader's time zone is GMT."},
+            {"lightweight_mutation_projection_mode", "throw", "throw", "When lightweight delete happens on a table with projection(s), the possible operations include throw the exception as projection exists, or drop all projection related to this table then do lightweight delete."},
+            {"database_replicated_allow_heavy_create", true, false, "Long-running DDL queries (CREATE AS SELECT and POPULATE) for Replicated database engine was forbidden"},
+            {"query_plan_merge_filters", false, false, "Allow to merge filters in the query plan"},
+            {"azure_sdk_max_retries", 10, 10, "Maximum number of retries in azure sdk"},
+            {"azure_sdk_retry_initial_backoff_ms", 10, 10, "Minimal backoff between retries in azure sdk"},
+            {"azure_sdk_retry_max_backoff_ms", 1000, 1000, "Maximal backoff between retries in azure sdk"},
+            {"ignore_on_cluster_for_replicated_named_collections_queries", false, false, "Ignore ON CLUSTER clause for replicated named collections management queries."},
+            {"backup_restore_s3_retry_attempts", 1000,1000, "Setting for Aws::Client::RetryStrategy, Aws::Client does retries itself, 0 means no retries. It takes place only for backup/restore."},
+            {"postgresql_connection_attempt_timeout", 2, 2, "Allow to control 'connect_timeout' parameter of PostgreSQL connection."},
+            {"postgresql_connection_pool_retries", 2, 2, "Allow to control the number of retries in PostgreSQL connection pool."}
+        }
+    },
+    {"24.6",
+        {
+            {"materialize_skip_indexes_on_insert", true, true, "Added new setting to allow to disable materialization of skip indexes on insert"},
+            {"materialize_statistics_on_insert", true, true, "Added new setting to allow to disable materialization of statistics on insert"},
+            {"input_format_parquet_use_native_reader", false, false, "When reading Parquet files, to use native reader instead of arrow reader."},
+            {"hdfs_throw_on_zero_files_match", false, false, "Allow to throw an error when ListObjects request cannot match any files in HDFS engine instead of empty query result"},
+            {"azure_throw_on_zero_files_match", false, false, "Allow to throw an error when ListObjects request cannot match any files in AzureBlobStorage engine instead of empty query result"},
+            {"s3_validate_request_settings", true, true, "Allow to disable S3 request settings validation"},
+            {"allow_experimental_full_text_index", false, false, "Enable experimental full-text index"},
+            {"azure_skip_empty_files", false, false, "Allow to skip empty files in azure table engine"},
+            {"hdfs_ignore_file_doesnt_exist", false, false, "Allow to return 0 rows when the requested files don't exist instead of throwing an exception in HDFS table engine"},
+            {"azure_ignore_file_doesnt_exist", false, false, "Allow to return 0 rows when the requested files don't exist instead of throwing an exception in AzureBlobStorage table engine"},
+            {"s3_ignore_file_doesnt_exist", false, false, "Allow to return 0 rows when the requested files don't exist instead of throwing an exception in S3 table engine"},
+            {"s3_max_part_number", 10000, 10000, "Maximum part number number for s3 upload part"},
+            {"s3_max_single_operation_copy_size", 32 * 1024 * 1024, 32 * 1024 * 1024, "Maximum size for a single copy operation in s3"},
+            {"input_format_parquet_max_block_size", 8192, DEFAULT_BLOCK_SIZE, "Increase block size for parquet reader."},
+            {"input_format_parquet_prefer_block_bytes", 0, DEFAULT_BLOCK_SIZE * 256, "Average block bytes output by parquet reader."},
+            {"enable_blob_storage_log", true, true, "Write information about blob storage operations to system.blob_storage_log table"},
+            {"allow_deprecated_snowflake_conversion_functions", true, false, "Disabled deprecated functions snowflakeToDateTime[64] and dateTime[64]ToSnowflake."},
+            {"allow_statistic_optimize", false, false, "Old setting which popped up here being renamed."},
+            {"allow_experimental_statistic", false, false, "Old setting which popped up here being renamed."},
+            {"allow_statistics_optimize", false, false, "The setting was renamed. The previous name is `allow_statistic_optimize`."},
+            {"allow_experimental_statistics", false, false, "The setting was renamed. The previous name is `allow_experimental_statistic`."},
+            {"enable_vertical_final", false, true, "Enable vertical final by default again after fixing bug"},
+            {"parallel_replicas_custom_key_range_lower", 0, 0, "Add settings to control the range filter when using parallel replicas with dynamic shards"},
+            {"parallel_replicas_custom_key_range_upper", 0, 0, "Add settings to control the range filter when using parallel replicas with dynamic shards. A value of 0 disables the upper limit"},
+            {"output_format_pretty_display_footer_column_names", 0, 1, "Add a setting to display column names in the footer if there are many rows. Threshold value is controlled by output_format_pretty_display_footer_column_names_min_rows."},
+            {"output_format_pretty_display_footer_column_names_min_rows", 0, 50, "Add a setting to control the threshold value for setting output_format_pretty_display_footer_column_names_min_rows. Default 50."},
+            {"output_format_csv_serialize_tuple_into_separate_columns", true, true, "A new way of how interpret tuples in CSV format was added."},
+            {"input_format_csv_deserialize_separate_columns_into_tuple", true, true, "A new way of how interpret tuples in CSV format was added."},
+            {"input_format_csv_try_infer_strings_from_quoted_tuples", true, true, "A new way of how interpret tuples in CSV format was added."},
+        }
+    },
+    {"24.5",
+        {
+            {"allow_deprecated_error_prone_window_functions", true, false, "Allow usage of deprecated error prone window functions (neighbor, runningAccumulate, runningDifferenceStartingWithFirstValue, runningDifference)"},
+            {"allow_experimental_join_condition", false, false, "Support join with inequal conditions which involve columns from both left and right table. e.g. t1.y < t2.y."},
+            {"input_format_tsv_crlf_end_of_line", false, false, "Enables reading of CRLF line endings with TSV formats"},
+            {"output_format_parquet_use_custom_encoder", false, true, "Enable custom Parquet encoder."},
+            {"cross_join_min_rows_to_compress", 0, 10000000, "Minimal count of rows to compress block in CROSS JOIN. Zero value means - disable this threshold. This block is compressed when any of the two thresholds (by rows or by bytes) are reached."},
+            {"cross_join_min_bytes_to_compress", 0, 1_GiB, "Minimal size of block to compress in CROSS JOIN. Zero value means - disable this threshold. This block is compressed when any of the two thresholds (by rows or by bytes) are reached."},
+            {"http_max_chunk_size", 0, 0, "Internal limitation"},
+            {"prefer_external_sort_block_bytes", 0, DEFAULT_BLOCK_SIZE * 256, "Prefer maximum block bytes for external sort, reduce the memory usage during merging."},
+            {"input_format_force_null_for_omitted_fields", false, false, "Disable type-defaults for omitted fields when needed"},
+            {"cast_string_to_dynamic_use_inference", false, false, "Add setting to allow converting String to Dynamic through parsing"},
+            {"allow_experimental_dynamic_type", false, false, "Add new experimental Dynamic type"},
+            {"azure_max_blocks_in_multipart_upload", 50000, 50000, "Maximum number of blocks in multipart upload for Azure."},
+        }
+    },
+    {"24.4",
+        {
+            {"input_format_json_throw_on_bad_escape_sequence", true, true, "Allow to save JSON strings with bad escape sequences"},
+            {"max_parsing_threads", 0, 0, "Add a separate setting to control number of threads in parallel parsing from files"},
+            {"ignore_drop_queries_probability", 0, 0, "Allow to ignore drop queries in server with specified probability for testing purposes"},
+            {"lightweight_deletes_sync", 2, 2, "The same as 'mutation_sync', but controls only execution of lightweight deletes"},
+            {"query_cache_system_table_handling", "save", "throw", "The query cache no longer caches results of queries against system tables"},
+            {"input_format_json_ignore_unnecessary_fields", false, true, "Ignore unnecessary fields and not parse them. Enabling this may not throw exceptions on json strings of invalid format or with duplicated fields"},
+            {"input_format_hive_text_allow_variable_number_of_columns", false, true, "Ignore extra columns in Hive Text input (if file has more columns than expected) and treat missing fields in Hive Text input as default values."},
+            {"allow_experimental_database_replicated", false, true, "Database engine Replicated is now in Beta stage"},
+            {"temporary_data_in_cache_reserve_space_wait_lock_timeout_milliseconds", (10 * 60 * 1000), (10 * 60 * 1000), "Wait time to lock cache for sapce reservation in temporary data in filesystem cache"},
+            {"optimize_rewrite_sum_if_to_count_if", false, true, "Only available for the analyzer, where it works correctly"},
+            {"azure_allow_parallel_part_upload", "true", "true", "Use multiple threads for azure multipart upload."},
+            {"max_recursive_cte_evaluation_depth", DBMS_RECURSIVE_CTE_MAX_EVALUATION_DEPTH, DBMS_RECURSIVE_CTE_MAX_EVALUATION_DEPTH, "Maximum limit on recursive CTE evaluation depth"},
+            {"query_plan_convert_outer_join_to_inner_join", false, true, "Allow to convert OUTER JOIN to INNER JOIN if filter after JOIN always filters default values"},
+        }
+    },
+    {"24.3",
+        {
+            {"s3_connect_timeout_ms", 1000, 1000, "Introduce new dedicated setting for s3 connection timeout"},
+            {"allow_experimental_shared_merge_tree", false, true, "The setting is obsolete"},
+            {"use_page_cache_for_disks_without_file_cache", false, false, "Added userspace page cache"},
+            {"read_from_page_cache_if_exists_otherwise_bypass_cache", false, false, "Added userspace page cache"},
+            {"page_cache_inject_eviction", false, false, "Added userspace page cache"},
+            {"default_table_engine", "None", "MergeTree", "Set default table engine to MergeTree for better usability"},
+            {"input_format_json_use_string_type_for_ambiguous_paths_in_named_tuples_inference_from_objects", false, false, "Allow to use String type for ambiguous paths during named tuple inference from JSON objects"},
+            {"traverse_shadow_remote_data_paths", false, false, "Traverse shadow directory when query system.remote_data_paths."},
+            {"throw_if_deduplication_in_dependent_materialized_views_enabled_with_async_insert", false, true, "Deduplication in dependent materialized view cannot work together with async inserts."},
+            {"parallel_replicas_allow_in_with_subquery", false, true, "If true, subquery for IN will be executed on every follower replica"},
+            {"log_processors_profiles", false, true, "Enable by default"},
+            {"function_locate_has_mysql_compatible_argument_order", false, true, "Increase compatibility with MySQL's locate function."},
+            {"allow_suspicious_primary_key", true, false, "Forbid suspicious PRIMARY KEY/ORDER BY for MergeTree (i.e. SimpleAggregateFunction)"},
+            {"filesystem_cache_reserve_space_wait_lock_timeout_milliseconds", 1000, 1000, "Wait time to lock cache for sapce reservation in filesystem cache"},
+            {"max_parser_backtracks", 0, 1000000, "Limiting the complexity of parsing"},
+            {"analyzer_compatibility_join_using_top_level_identifier", false, false, "Force to resolve identifier in JOIN USING from projection"},
+            {"distributed_insert_skip_read_only_replicas", false, false, "If true, INSERT into Distributed will skip read-only replicas"},
+            {"keeper_max_retries", 10, 10, "Max retries for general keeper operations"},
+            {"keeper_retry_initial_backoff_ms", 100, 100, "Initial backoff timeout for general keeper operations"},
+            {"keeper_retry_max_backoff_ms", 5000, 5000, "Max backoff timeout for general keeper operations"},
+            {"s3queue_allow_experimental_sharded_mode", false, false, "Enable experimental sharded mode of S3Queue table engine. It is experimental because it will be rewritten"},
+            {"allow_experimental_analyzer", false, true, "Enable analyzer and planner by default."},
+            {"merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability", 0.0, 0.0, "For testing of `PartsSplitter` - split read ranges into intersecting and non intersecting every time you read from MergeTree with the specified probability."},
+            {"allow_get_client_http_header", false, false, "Introduced a new function."},
+            {"output_format_pretty_row_numbers", false, true, "It is better for usability."},
+            {"output_format_pretty_max_value_width_apply_for_single_value", true, false, "Single values in Pretty formats won't be cut."},
+            {"output_format_parquet_string_as_string", false, true, "ClickHouse allows arbitrary binary data in the String data type, which is typically UTF-8. Parquet/ORC/Arrow Strings only support UTF-8. That's why you can choose which Arrow's data type to use for the ClickHouse String data type - String or Binary. While Binary would be more correct and compatible, using String by default will correspond to user expectations in most cases."},
+            {"output_format_orc_string_as_string", false, true, "ClickHouse allows arbitrary binary data in the String data type, which is typically UTF-8. Parquet/ORC/Arrow Strings only support UTF-8. That's why you can choose which Arrow's data type to use for the ClickHouse String data type - String or Binary. While Binary would be more correct and compatible, using String by default will correspond to user expectations in most cases."},
+            {"output_format_arrow_string_as_string", false, true, "ClickHouse allows arbitrary binary data in the String data type, which is typically UTF-8. Parquet/ORC/Arrow Strings only support UTF-8. That's why you can choose which Arrow's data type to use for the ClickHouse String data type - String or Binary. While Binary would be more correct and compatible, using String by default will correspond to user expectations in most cases."},
+            {"output_format_parquet_compression_method", "lz4", "zstd", "Parquet/ORC/Arrow support many compression methods, including lz4 and zstd. ClickHouse supports each and every compression method. Some inferior tools, such as 'duckdb', lack support for the faster `lz4` compression method, that's why we set zstd by default."},
+            {"output_format_orc_compression_method", "lz4", "zstd", "Parquet/ORC/Arrow support many compression methods, including lz4 and zstd. ClickHouse supports each and every compression method. Some inferior tools, such as 'duckdb', lack support for the faster `lz4` compression method, that's why we set zstd by default."},
+            {"output_format_pretty_highlight_digit_groups", false, true, "If enabled and if output is a terminal, highlight every digit corresponding to the number of thousands, millions, etc. with underline."},
+            {"geo_distance_returns_float64_on_float64_arguments", false, true, "Increase the default precision."},
+            {"azure_max_inflight_parts_for_one_file", 20, 20, "The maximum number of a concurrent loaded parts in multipart upload request. 0 means unlimited."},
+            {"azure_strict_upload_part_size", 0, 0, "The exact size of part to upload during multipart upload to Azure blob storage."},
+            {"azure_min_upload_part_size", 16*1024*1024, 16*1024*1024, "The minimum size of part to upload during multipart upload to Azure blob storage."},
+            {"azure_max_upload_part_size", 5ull*1024*1024*1024, 5ull*1024*1024*1024, "The maximum size of part to upload during multipart upload to Azure blob storage."},
+            {"azure_upload_part_size_multiply_factor", 2, 2, "Multiply azure_min_upload_part_size by this factor each time azure_multiply_parts_count_threshold parts were uploaded from a single write to Azure blob storage."},
+            {"azure_upload_part_size_multiply_parts_count_threshold", 500, 500, "Each time this number of parts was uploaded to Azure blob storage, azure_min_upload_part_size is multiplied by azure_upload_part_size_multiply_factor."},
+            {"output_format_csv_serialize_tuple_into_separate_columns", true, true, "A new way of how interpret tuples in CSV format was added."},
+            {"input_format_csv_deserialize_separate_columns_into_tuple", true, true, "A new way of how interpret tuples in CSV format was added."},
+            {"input_format_csv_try_infer_strings_from_quoted_tuples", true, true, "A new way of how interpret tuples in CSV format was added."},
+        }
+    },
+    {"24.2",
+        {
+            {"allow_suspicious_variant_types", true, false, "Don't allow creating Variant type with suspicious variants by default"},
+            {"validate_experimental_and_suspicious_types_inside_nested_types", false, true, "Validate usage of experimental and suspicious types inside nested types"},
+            {"output_format_values_escape_quote_with_quote", false, false, "If true escape ' with '', otherwise quoted with \\'"},
+            {"output_format_pretty_single_large_number_tip_threshold", 0, 1'000'000, "Print a readable number tip on the right side of the table if the block consists of a single number which exceeds this value (except 0)"},
+            {"input_format_try_infer_exponent_floats", true, false, "Don't infer floats in exponential notation by default"},
+            {"query_plan_optimize_prewhere", true, true, "Allow to push down filter to PREWHERE expression for supported storages"},
+            {"async_insert_max_data_size", 1000000, 10485760, "The previous value appeared to be too small."},
+            {"async_insert_poll_timeout_ms", 10, 10, "Timeout in milliseconds for polling data from asynchronous insert queue"},
+            {"async_insert_use_adaptive_busy_timeout", false, true, "Use adaptive asynchronous insert timeout"},
+            {"async_insert_busy_timeout_min_ms", 50, 50, "The minimum value of the asynchronous insert timeout in milliseconds; it also serves as the initial value, which may be increased later by the adaptive algorithm"},
+            {"async_insert_busy_timeout_max_ms", 200, 200, "The minimum value of the asynchronous insert timeout in milliseconds; async_insert_busy_timeout_ms is aliased to async_insert_busy_timeout_max_ms"},
+            {"async_insert_busy_timeout_increase_rate", 0.2, 0.2, "The exponential growth rate at which the adaptive asynchronous insert timeout increases"},
+            {"async_insert_busy_timeout_decrease_rate", 0.2, 0.2, "The exponential growth rate at which the adaptive asynchronous insert timeout decreases"},
+            {"format_template_row_format", "", "", "Template row format string can be set directly in query"},
+            {"format_template_resultset_format", "", "", "Template result set format string can be set in query"},
+            {"split_parts_ranges_into_intersecting_and_non_intersecting_final", true, true, "Allow to split parts ranges into intersecting and non intersecting during FINAL optimization"},
+            {"split_intersecting_parts_ranges_into_layers_final", true, true, "Allow to split intersecting parts ranges into layers during FINAL optimization"},
+            {"azure_max_single_part_copy_size", 256*1024*1024, 256*1024*1024, "The maximum size of object to copy using single part copy to Azure blob storage."},
+            {"min_external_table_block_size_rows", DEFAULT_INSERT_BLOCK_SIZE, DEFAULT_INSERT_BLOCK_SIZE, "Squash blocks passed to external table to specified size in rows, if blocks are not big enough"},
+            {"min_external_table_block_size_bytes", DEFAULT_INSERT_BLOCK_SIZE * 256, DEFAULT_INSERT_BLOCK_SIZE * 256, "Squash blocks passed to external table to specified size in bytes, if blocks are not big enough."},
+            {"parallel_replicas_prefer_local_join", true, true, "If true, and JOIN can be executed with parallel replicas algorithm, and all storages of right JOIN part are *MergeTree, local JOIN will be used instead of GLOBAL JOIN."},
+            {"optimize_time_filter_with_preimage", true, true, "Optimize Date and DateTime predicates by converting functions into equivalent comparisons without conversions (e.g. toYear(col) = 2023 -> col >= '2023-01-01' AND col <= '2023-12-31')"},
+            {"extract_key_value_pairs_max_pairs_per_row", 0, 0, "Max number of pairs that can be produced by the `extractKeyValuePairs` function. Used as a safeguard against consuming too much memory."},
+            {"default_view_definer", "CURRENT_USER", "CURRENT_USER", "Allows to set default `DEFINER` option while creating a view"},
+            {"default_materialized_view_sql_security", "DEFINER", "DEFINER", "Allows to set a default value for SQL SECURITY option when creating a materialized view"},
+            {"default_normal_view_sql_security", "INVOKER", "INVOKER", "Allows to set default `SQL SECURITY` option while creating a normal view"},
+            {"mysql_map_string_to_text_in_show_columns", false, true, "Reduce the configuration effort to connect ClickHouse with BI tools."},
+            {"mysql_map_fixed_string_to_text_in_show_columns", false, true, "Reduce the configuration effort to connect ClickHouse with BI tools."},
+        }
+    },
+    {"24.1",
+        {
+            {"print_pretty_type_names", false, true, "Better user experience."},
+            {"input_format_json_read_bools_as_strings", false, true, "Allow to read bools as strings in JSON formats by default"},
+            {"output_format_arrow_use_signed_indexes_for_dictionary", false, true, "Use signed indexes type for Arrow dictionaries by default as it's recommended"},
+            {"allow_experimental_variant_type", false, false, "Add new experimental Variant type"},
+            {"use_variant_as_common_type", false, false, "Allow to use Variant in if/multiIf if there is no common type"},
+            {"output_format_arrow_use_64_bit_indexes_for_dictionary", false, false, "Allow to use 64 bit indexes type in Arrow dictionaries"},
+            {"parallel_replicas_mark_segment_size", 128, 128, "Add new setting to control segment size in new parallel replicas coordinator implementation"},
+            {"ignore_materialized_views_with_dropped_target_table", false, false, "Add new setting to allow to ignore materialized views with dropped target table"},
+            {"output_format_compression_level", 3, 3, "Allow to change compression level in the query output"},
+            {"output_format_compression_zstd_window_log", 0, 0, "Allow to change zstd window log in the query output when zstd compression is used"},
+            {"enable_zstd_qat_codec", false, false, "Add new ZSTD_QAT codec"},
+            {"enable_vertical_final", false, true, "Use vertical final by default"},
+            {"output_format_arrow_use_64_bit_indexes_for_dictionary", false, false, "Allow to use 64 bit indexes type in Arrow dictionaries"},
+            {"max_rows_in_set_to_optimize_join", 100000, 0, "Disable join optimization as it prevents from read in order optimization"},
+            {"output_format_pretty_color", true, "auto", "Setting is changed to allow also for auto value, disabling ANSI escapes if output is not a tty"},
+            {"function_visible_width_behavior", 0, 1, "We changed the default behavior of `visibleWidth` to be more precise"},
+            {"max_estimated_execution_time", 0, 0, "Separate max_execution_time and max_estimated_execution_time"},
+            {"iceberg_engine_ignore_schema_evolution", false, false, "Allow to ignore schema evolution in Iceberg table engine"},
+            {"optimize_injective_functions_in_group_by", false, true, "Replace injective functions by it's arguments in GROUP BY section in analyzer"},
+            {"update_insert_deduplication_token_in_dependent_materialized_views", false, false, "Allow to update insert deduplication token with table identifier during insert in dependent materialized views"},
+            {"azure_max_unexpected_write_error_retries", 4, 4, "The maximum number of retries in case of unexpected errors during Azure blob storage write"},
+            {"split_parts_ranges_into_intersecting_and_non_intersecting_final", false, true, "Allow to split parts ranges into intersecting and non intersecting during FINAL optimization"},
+            {"split_intersecting_parts_ranges_into_layers_final", true, true, "Allow to split intersecting parts ranges into layers during FINAL optimization"}
+        }
+    },
+    {"23.12",
+        {
+            {"allow_suspicious_ttl_expressions", true, false, "It is a new setting, and in previous versions the behavior was equivalent to allowing."},
+            {"input_format_parquet_allow_missing_columns", false, true, "Allow missing columns in Parquet files by default"},
+            {"input_format_orc_allow_missing_columns", false, true, "Allow missing columns in ORC files by default"},
+            {"input_format_arrow_allow_missing_columns", false, true, "Allow missing columns in Arrow files by default"}
+        }
+    },
+    {"23.11",
+        {
+            {"parsedatetime_parse_without_leading_zeros", false, true, "Improved compatibility with MySQL DATE_FORMAT/STR_TO_DATE"}
+        }
+    },
+    {"23.9",
+        {
+            {"optimize_group_by_constant_keys", false, true, "Optimize group by constant keys by default"},
+            {"input_format_json_try_infer_named_tuples_from_objects", false, true, "Try to infer named Tuples from JSON objects by default"},
+            {"input_format_json_read_numbers_as_strings", false, true, "Allow to read numbers as strings in JSON formats by default"},
+            {"input_format_json_read_arrays_as_strings", false, true, "Allow to read arrays as strings in JSON formats by default"},
+            {"input_format_json_infer_incomplete_types_as_strings", false, true, "Allow to infer incomplete types as Strings in JSON formats by default"},
+            {"input_format_json_try_infer_numbers_from_strings", true, false, "Don't infer numbers from strings in JSON formats by default to prevent possible parsing errors"},
+            {"http_write_exception_in_output_format", false, true, "Output valid JSON/XML on exception in HTTP streaming."}
+        }
+    },
+    {"23.8",
+        {
+            {"rewrite_count_distinct_if_with_count_distinct_implementation", false, true, "Rewrite countDistinctIf with count_distinct_implementation configuration"}
+        }
+    },
+    {"23.7",
+        {
+            {"function_sleep_max_microseconds_per_block", 0, 3000000, "In previous versions, the maximum sleep time of 3 seconds was applied only for `sleep`, but not for `sleepEachRow` function. In the new version, we introduce this setting. If you set compatibility with the previous versions, we will disable the limit altogether."}
+        }
+    },
+    {"23.6",
+        {
+            {"http_send_timeout", 180, 30, "3 minutes seems crazy long. Note that this is timeout for a single network write call, not for the whole upload operation."},
+            {"http_receive_timeout", 180, 30, "See http_send_timeout."}
+        }
+    },
+    {"23.5",
+        {
+            {"input_format_parquet_preserve_order", true, false, "Allow Parquet reader to reorder rows for better parallelism."},
+            {"parallelize_output_from_storages", false, true, "Allow parallelism when executing queries that read from file/url/s3/etc. This may reorder rows."},
+            {"use_with_fill_by_sorting_prefix", false, true, "Columns preceding WITH FILL columns in ORDER BY clause form sorting prefix. Rows with different values in sorting prefix are filled independently"},
+            {"output_format_parquet_compliant_nested_types", false, true, "Change an internal field name in output Parquet file schema."}
+        }
+    },
+    {"23.4",
+        {
+            {"allow_suspicious_indices", true, false, "If true, index can defined with identical expressions"},
+            {"allow_nonconst_timezone_arguments", true, false, "Allow non-const timezone arguments in certain time-related functions like toTimeZone(), fromUnixTimestamp*(), snowflakeToDateTime*()."},
+            {"connect_timeout_with_failover_ms", 50, 1000, "Increase default connect timeout because of async connect"},
+            {"connect_timeout_with_failover_secure_ms", 100, 1000, "Increase default secure connect timeout because of async connect"},
+            {"hedged_connection_timeout_ms", 100, 50, "Start new connection in hedged requests after 50 ms instead of 100 to correspond with previous connect timeout"},
+            {"formatdatetime_f_prints_single_zero", true, false, "Improved compatibility with MySQL DATE_FORMAT()/STR_TO_DATE()"},
+            {"formatdatetime_parsedatetime_m_is_month_name", false, true, "Improved compatibility with MySQL DATE_FORMAT/STR_TO_DATE"}
+        }
+    },
+    {"23.3",
+        {
+            {"output_format_parquet_version", "1.0", "2.latest", "Use latest Parquet format version for output format"},
+            {"input_format_json_ignore_unknown_keys_in_named_tuple", false, true, "Improve parsing JSON objects as named tuples"},
+            {"input_format_native_allow_types_conversion", false, true, "Allow types conversion in Native input forma"},
+            {"output_format_arrow_compression_method", "none", "lz4_frame", "Use lz4 compression in Arrow output format by default"},
+            {"output_format_parquet_compression_method", "snappy", "lz4", "Use lz4 compression in Parquet output format by default"},
+            {"output_format_orc_compression_method", "none", "lz4_frame", "Use lz4 compression in ORC output format by default"},
+            {"async_query_sending_for_remote", false, true, "Create connections and send query async across shards"}
+        }
+    },
+    {"23.2",
+        {
+            {"output_format_parquet_fixed_string_as_fixed_byte_array", false, true, "Use Parquet FIXED_LENGTH_BYTE_ARRAY type for FixedString by default"},
+            {"output_format_arrow_fixed_string_as_fixed_byte_array", false, true, "Use Arrow FIXED_SIZE_BINARY type for FixedString by default"},
+            {"query_plan_remove_redundant_distinct", false, true, "Remove redundant Distinct step in query plan"},
+            {"optimize_duplicate_order_by_and_distinct", true, false, "Remove duplicate ORDER BY and DISTINCT if it's possible"},
+            {"insert_keeper_max_retries", 0, 20, "Enable reconnections to Keeper on INSERT, improve reliability"}
+        }
+    },
+    {"23.1",
+        {
+            {"input_format_json_read_objects_as_strings", 0, 1, "Enable reading nested json objects as strings while object type is experimental"},
+            {"input_format_json_defaults_for_missing_elements_in_named_tuple", false, true, "Allow missing elements in JSON objects while reading named tuples by default"},
+            {"input_format_csv_detect_header", false, true, "Detect header in CSV format by default"},
+            {"input_format_tsv_detect_header", false, true, "Detect header in TSV format by default"},
+            {"input_format_custom_detect_header", false, true, "Detect header in CustomSeparated format by default"},
+            {"query_plan_remove_redundant_sorting", false, true, "Remove redundant sorting in query plan. For example, sorting steps related to ORDER BY clauses in subqueries"}
+        }
+    },
+    {"22.12",
+        {
+            {"max_size_to_preallocate_for_aggregation", 10'000'000, 100'000'000, "This optimizes performance"},
+            {"query_plan_aggregation_in_order", 0, 1, "Enable some refactoring around query plan"},
+            {"format_binary_max_string_size", 0, 1_GiB, "Prevent allocating large amount of memory"}
+        }
+    },
+    {"22.11",
+        {
+            {"use_structure_from_insertion_table_in_table_functions", 0, 2, "Improve using structure from insertion table in table functions"}
+        }
+    },
+    {"22.9",
+        {
+            {"force_grouping_standard_compatibility", false, true, "Make GROUPING function output the same as in SQL standard and other DBMS"}
+        }
+    },
+    {"22.7",
+        {
+            {"cross_to_inner_join_rewrite", 1, 2, "Force rewrite comma join to inner"},
+            {"enable_positional_arguments", false, true, "Enable positional arguments feature by default"},
+            {"format_csv_allow_single_quotes", true, false, "Most tools don't treat single quote in CSV specially, don't do it by default too"}
+        }
+    },
+    {"22.6",
+        {
+            {"output_format_json_named_tuples_as_objects", false, true, "Allow to serialize named tuples as JSON objects in JSON formats by default"},
+            {"input_format_skip_unknown_fields", false, true, "Optimize reading subset of columns for some input formats"}
+        }
+    },
+    {"22.5",
+        {
+            {"memory_overcommit_ratio_denominator", 0, 1073741824, "Enable memory overcommit feature by default"},
+            {"memory_overcommit_ratio_denominator_for_user", 0, 1073741824, "Enable memory overcommit feature by default"}
+        }
+    },
+    {"22.4",
+        {
+            {"allow_settings_after_format_in_insert", true, false, "Do not allow SETTINGS after FORMAT for INSERT queries because ClickHouse interpret SETTINGS as some values, which is misleading"}
+        }
+    },
+    {"22.3",
+        {
+            {"cast_ipv4_ipv6_default_on_conversion_error", true, false, "Make functions cast(value, 'IPv4') and cast(value, 'IPv6') behave same as toIPv4 and toIPv6 functions"}
+        }
+    },
+    {"21.12",
+        {
+            {"stream_like_engine_allow_direct_select", true, false, "Do not allow direct select for Kafka/RabbitMQ/FileLog by default"}
+        }
+    },
+    {"21.9",
+        {
+            {"output_format_decimal_trailing_zeros", true, false, "Do not output trailing zeros in text representation of Decimal types by default for better looking output"},
+            {"use_hedged_requests", false, true, "Enable Hedged Requests feature by default"}
+        }
+    },
+    {"21.7",
+        {
+            {"legacy_column_name_of_tuple_literal", true, false, "Add this setting only for compatibility reasons. It makes sense to set to 'true', while doing rolling update of cluster from version lower than 21.7 to higher"}
+        }
+    },
+    {"21.5",
+        {
+            {"async_socket_for_remote", false, true, "Fix all problems and turn on asynchronous reads from socket for remote queries by default again"}
+        }
+    },
+    {"21.3",
+        {
+            {"async_socket_for_remote", true, false, "Turn off asynchronous reads from socket for remote queries because of some problems"},
+            {"optimize_normalize_count_variants", false, true, "Rewrite aggregate functions that semantically equals to count() as count() by default"},
+            {"normalize_function_names", false, true, "Normalize function names to their canonical names, this was needed for projection query routing"}
+        }
+    },
+    {"21.2",
+        {
+            {"enable_global_with_statement", false, true, "Propagate WITH statements to UNION queries and all subqueries by default"}
+        }
+    },
+    {"21.1",
+        {
+            {"insert_quorum_parallel", false, true, "Use parallel quorum inserts by default. It is significantly more convenient to use than sequential quorum inserts"},
+            {"input_format_null_as_default", false, true, "Allow to insert NULL as default for input formats by default"},
+            {"optimize_on_insert", false, true, "Enable data optimization on INSERT by default for better user experience"},
+            {"use_compact_format_in_distributed_parts_names", false, true, "Use compact format for async INSERT into Distributed tables by default"}
+        }
+    },
+    {"20.10",
+        {
+            {"format_regexp_escaping_rule", "Escaped", "Raw", "Use Raw as default escaping rule for Regexp format to male the behaviour more like to what users expect"}
+        }
+    },
+    {"20.7",
+        {
+            {"show_table_uuid_in_table_create_query_if_not_nil", true, false, "Stop showing  UID of the table in its CREATE query for Engine=Atomic"}
+        }
+    },
+    {"20.5",
+        {
+            {"input_format_with_names_use_header", false, true, "Enable using header with names for formats with WithNames/WithNamesAndTypes suffixes"},
+            {"allow_suspicious_codecs", true, false, "Don't allow to specify meaningless compression codecs"}
+        }
+    },
+    {"20.4",
+        {
+            {"validate_polygons", false, true, "Throw exception if polygon is invalid in function pointInPolygon by default instead of returning possibly wrong results"}
+        }
+    },
+    {"19.18",
+        {
+            {"enable_scalar_subquery_optimization", false, true, "Prevent scalar subqueries from (de)serializing large scalar values and possibly avoid running the same subquery more than once"}
+        }
+    },
+    {"19.14",
+        {
+            {"any_join_distinct_right_table_keys", true, false, "Disable ANY RIGHT and ANY FULL JOINs by default to avoid inconsistency"}
+        }
+    },
+    {"19.12",
+        {
+            {"input_format_defaults_for_omitted_fields", false, true, "Enable calculation of complex default expressions for omitted fields for some input formats, because it should be the expected behaviour"}
+        }
+    },
+    {"19.5",
+        {
+            {"max_partitions_per_insert_block", 0, 100, "Add a limit for the number of partitions in one block"}
+        }
+    },
+    {"18.12.17",
+        {
+            {"enable_optimize_predicate_expression", 0, 1, "Optimize predicates to subqueries by default"}
+        }
+    },
 };
 
 

From bca21bc6c735b710ed15061e41790066b5ffd843 Mon Sep 17 00:00:00 2001
From: sakulali <sakulali.dev@gmail.com>
Date: Wed, 31 Jul 2024 21:33:14 +0800
Subject: [PATCH 1114/1488] add test cases

---
 src/Common/Config/ConfigProcessor.cpp           | 11 ++++-------
 ...89_clickhouse_client_config_format.reference |  4 ++++
 .../01889_clickhouse_client_config_format.sh    | 17 +++++++++++++++++
 3 files changed, 25 insertions(+), 7 deletions(-)

diff --git a/src/Common/Config/ConfigProcessor.cpp b/src/Common/Config/ConfigProcessor.cpp
index f0b83c035c8..c4b4a1d5e7e 100644
--- a/src/Common/Config/ConfigProcessor.cpp
+++ b/src/Common/Config/ConfigProcessor.cpp
@@ -654,22 +654,19 @@ XMLDocumentPtr ConfigProcessor::parseConfig(const std::string & config_path)
             while (std::getline(file, line))
             {
                 const size_t pos = firstNonWhitespacePos(line);
-                if (pos == std::string::npos)
-                    continue;
 
                 if (pos < line.size() && '<' == line[pos])
                 {
                     maybe_xml = true;
+                    break;
                 }
-
-                break;
+                else if (pos != std::string::npos)
+                    break;
             }
         }
-
         if (maybe_xml)
             return dom_parser.parse(config_path);
-        else
-            return YAMLParser::parse(config_path);
+        return YAMLParser::parse(config_path);
     }
 }
 
diff --git a/tests/queries/0_stateless/01889_clickhouse_client_config_format.reference b/tests/queries/0_stateless/01889_clickhouse_client_config_format.reference
index afe27ddb063..ef0d9ffc538 100644
--- a/tests/queries/0_stateless/01889_clickhouse_client_config_format.reference
+++ b/tests/queries/0_stateless/01889_clickhouse_client_config_format.reference
@@ -18,3 +18,7 @@ autodetect xml (non leading whitespaces)
 2
 autodetect yaml
 2
+autodetect invalid xml
+Code: 1000, e.code() = 0, SAXParseException: Invalid token in '/config_test.badxml', line 2 column 12, Stack trace (when copying this message, always include the lines below):
+autodetect invalid yaml
+Code: 585. Unable to parse YAML configuration file /config_test.badyaml, yaml-cpp: error at line 2, column 12: illegal map value. (CANNOT_PARSE_YAML)
diff --git a/tests/queries/0_stateless/01889_clickhouse_client_config_format.sh b/tests/queries/0_stateless/01889_clickhouse_client_config_format.sh
index 14d67f943f1..9a44ec0d5f5 100755
--- a/tests/queries/0_stateless/01889_clickhouse_client_config_format.sh
+++ b/tests/queries/0_stateless/01889_clickhouse_client_config_format.sh
@@ -15,6 +15,8 @@ yaml_config=$CLICKHOUSE_TMP/config_$CLICKHOUSE_DATABASE.yaml
 autodetect_xml_with_leading_whitespace_config=$CLICKHOUSE_TMP/config_$CLICKHOUSE_DATABASE.config
 autodetect_xml_non_leading_whitespace_config=$CLICKHOUSE_TMP/config_$CLICKHOUSE_DATABASE.cfg
 autodetect_yaml_config=$CLICKHOUSE_TMP/config_$CLICKHOUSE_DATABASE.properties
+autodetect_invalid_xml_config=$CLICKHOUSE_TMP/config_$CLICKHOUSE_DATABASE.badxml
+autodetect_invalid_yaml_config=$CLICKHOUSE_TMP/config_$CLICKHOUSE_DATABASE.badyaml
 
 function cleanup()
 {
@@ -27,6 +29,8 @@ function cleanup()
     rm "${autodetect_xml_with_leading_whitespace_config:?}"
     rm "${autodetect_xml_non_leading_whitespace_config:?}"
     rm "${autodetect_yaml_config:?}"
+    rm "${autodetect_invalid_xml_config:?}"
+    rm "${autodetect_invalid_yaml_config:?}"
 }
 trap cleanup EXIT
 
@@ -70,6 +74,15 @@ EOL
 cat > "$autodetect_yaml_config" <<EOL
 max_threads: 2
 EOL
+cat > "$autodetect_invalid_xml_config" <<EOL
+<!-- This is a XML file comment -->
+<invalid tag><invalid tag>
+EOL
+cat > "$autodetect_invalid_yaml_config" <<EOL
+; This is a INI file comment
+max_threads: 2
+EOL
+
 
 echo 'default'
 $CLICKHOUSE_CLIENT --config "$config" -q "select getSetting('max_threads')"
@@ -95,3 +108,7 @@ echo 'autodetect xml (non leading whitespaces)'
 $CLICKHOUSE_CLIENT --config "$autodetect_xml_non_leading_whitespace_config" -q "select getSetting('max_threads')"
 echo 'autodetect yaml'
 $CLICKHOUSE_CLIENT --config "$autodetect_yaml_config" -q "select getSetting('max_threads')"
+echo 'autodetect invalid xml'
+$CLICKHOUSE_CLIENT --config "$autodetect_invalid_xml_config" -q "select getSetting('max_threads')" 2>&1 |& sed -n '1p' | sed -e "s#$CLICKHOUSE_TMP##" -e "s#Poco::Exception. ##"
+echo 'autodetect invalid yaml'
+$CLICKHOUSE_CLIENT --config "$autodetect_invalid_yaml_config" -q "select getSetting('max_threads')" 2>&1 |& sed -n '1p' | sed -e "s#$CLICKHOUSE_TMP##" -e "s#DB::Exception: ##"
\ No newline at end of file

From 7bd8061979204973d4a1c1ac956ab80fc1c6be38 Mon Sep 17 00:00:00 2001
From: heguangnan <heguangnan@bytedance.com>
Date: Wed, 31 Jul 2024 21:35:55 +0800
Subject: [PATCH 1115/1488] fix test

---
 .../0_stateless/03214_count_distinct_null_key_memory_leak.sql   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/03214_count_distinct_null_key_memory_leak.sql b/tests/queries/0_stateless/03214_count_distinct_null_key_memory_leak.sql
index d8428ec6b4a..84804e4e016 100644
--- a/tests/queries/0_stateless/03214_count_distinct_null_key_memory_leak.sql
+++ b/tests/queries/0_stateless/03214_count_distinct_null_key_memory_leak.sql
@@ -13,6 +13,6 @@ ORDER BY c
 SETTINGS index_granularity = 8192, allow_nullable_key=1;
 
 INSERT INTO testnull(b,c) SELECT toString(rand64()) AS b, toString(rand64()) AS c FROM numbers(1000000);
-SELECT count(distinct b) FROM testnull GROUP BY a  SETTINGS max_memory_usage = 54748364; -- {serverError MEMORY_LIMIT_EXCEEDED}
+SELECT count(distinct b) FROM testnull GROUP BY a  SETTINGS max_memory_usage = 10000000; -- {serverError MEMORY_LIMIT_EXCEEDED}
 
 DROP TABLE testnull;
\ No newline at end of file

From 31c142a96d49fbe1b46b21e4cdad366546dc7864 Mon Sep 17 00:00:00 2001
From: Michael Stetsyuk <mike.stetsyuk@gmail.com>
Date: Wed, 31 Jul 2024 14:44:54 +0100
Subject: [PATCH 1116/1488] make it possible to rerun test_storage_delta and
 test_checking_s3_blobs_paranoid

---
 .../test_checking_s3_blobs_paranoid/test.py   |  2 ++
 tests/integration/test_storage_delta/test.py  | 29 +++++++++++++++++++
 2 files changed, 31 insertions(+)

diff --git a/tests/integration/test_checking_s3_blobs_paranoid/test.py b/tests/integration/test_checking_s3_blobs_paranoid/test.py
index dde636b5d29..afe8449b44a 100644
--- a/tests/integration/test_checking_s3_blobs_paranoid/test.py
+++ b/tests/integration/test_checking_s3_blobs_paranoid/test.py
@@ -61,6 +61,7 @@ def test_upload_after_check_works(cluster, broken_s3):
 
     node.query(
         """
+        DROP TABLE IF EXISTS s3_upload_after_check_works;
         CREATE TABLE s3_upload_after_check_works (
             id Int64,
             data String
@@ -631,6 +632,7 @@ def test_no_key_found_disk(cluster, broken_s3):
 
     node.query(
         """
+        DROP TABLE IF EXISTS no_key_found_disk;
         CREATE TABLE no_key_found_disk (
             id Int64
         ) ENGINE=MergeTree()
diff --git a/tests/integration/test_storage_delta/test.py b/tests/integration/test_storage_delta/test.py
index 67cc7cdd6da..698becc18c4 100644
--- a/tests/integration/test_storage_delta/test.py
+++ b/tests/integration/test_storage_delta/test.py
@@ -52,6 +52,11 @@ def get_spark():
     return builder.master("local").getOrCreate()
 
 
+def remove_local_directory_contents(local_path):
+    for local_file in glob.glob(local_path + "/**"):
+        os.unlink(local_file)
+
+
 @pytest.fixture(scope="module")
 def started_cluster():
     try:
@@ -169,6 +174,9 @@ def test_single_log_file(started_cluster):
         inserted_data
     )
 
+    os.unlink(parquet_data_path)
+    remove_local_directory_contents(f"/{TABLE_NAME}")
+
 
 def test_partition_by(started_cluster):
     instance = started_cluster.instances["node1"]
@@ -191,6 +199,7 @@ def test_partition_by(started_cluster):
     create_delta_table(instance, TABLE_NAME)
     assert int(instance.query(f"SELECT count() FROM {TABLE_NAME}")) == 10
 
+    remove_local_directory_contents(f"/{TABLE_NAME}")
 
 def test_checkpoint(started_cluster):
     instance = started_cluster.instances["node1"]
@@ -266,6 +275,9 @@ def test_checkpoint(started_cluster):
         ).strip()
     )
 
+    remove_local_directory_contents(f"/{TABLE_NAME}")
+    spark.sql(f"DROP TABLE {TABLE_NAME}")
+
 
 def test_multiple_log_files(started_cluster):
     instance = started_cluster.instances["node1"]
@@ -304,6 +316,8 @@ def test_multiple_log_files(started_cluster):
         "SELECT number, toString(number + 1) FROM numbers(200)"
     )
 
+    remove_local_directory_contents(f"/{TABLE_NAME}")
+
 
 def test_metadata(started_cluster):
     instance = started_cluster.instances["node1"]
@@ -337,6 +351,9 @@ def test_metadata(started_cluster):
     create_delta_table(instance, TABLE_NAME)
     assert int(instance.query(f"SELECT count() FROM {TABLE_NAME}")) == 100
 
+    os.unlink(parquet_data_path)
+    remove_local_directory_contents(f"/{TABLE_NAME}")
+
 
 def test_types(started_cluster):
     TABLE_NAME = "test_types"
@@ -409,6 +426,9 @@ def test_types(started_cluster):
         ]
     )
 
+    remove_local_directory_contents(f"/{result_file}")
+    spark.sql(f"DROP TABLE {TABLE_NAME}")
+
 
 def test_restart_broken(started_cluster):
     instance = started_cluster.instances["node1"]
@@ -470,6 +490,9 @@ def test_restart_broken(started_cluster):
 
     assert int(instance.query(f"SELECT count() FROM {TABLE_NAME}")) == 100
 
+    os.unlink(parquet_data_path)
+    remove_local_directory_contents(f"/{TABLE_NAME}")
+
 
 def test_restart_broken_table_function(started_cluster):
     instance = started_cluster.instances["node1"]
@@ -524,6 +547,9 @@ def test_restart_broken_table_function(started_cluster):
 
     assert int(instance.query(f"SELECT count() FROM {TABLE_NAME}")) == 100
 
+    os.unlink(parquet_data_path)
+    remove_local_directory_contents(f"/{TABLE_NAME}")
+
 
 def test_partition_columns(started_cluster):
     instance = started_cluster.instances["node1"]
@@ -721,3 +747,6 @@ SELECT * FROM deltaLake('http://{started_cluster.minio_ip}:{started_cluster.mini
         )
         == 1
     )
+
+    remove_local_directory_contents(f"/{TABLE_NAME}")
+    spark.sql(f"DROP TABLE {TABLE_NAME}")

From 2a2dba63cc0182247754a5a4819cb89f21825bfd Mon Sep 17 00:00:00 2001
From: robot-clickhouse <robot-clickhouse@users.noreply.github.com>
Date: Wed, 31 Jul 2024 13:48:04 +0000
Subject: [PATCH 1117/1488] Automatic style fix

---
 tests/integration/helpers/cluster.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py
index 7f0a9154be9..2e38aec3512 100644
--- a/tests/integration/helpers/cluster.py
+++ b/tests/integration/helpers/cluster.py
@@ -2971,6 +2971,7 @@ class ClickHouseCluster:
                     "Trying to create Azurite instance by command %s",
                     " ".join(map(str, azurite_start_cmd)),
                 )
+
                 def logging_azurite_initialization(exception, retry_number, sleep_time):
                     logging.info(
                         f"Azurite initialization failed with error: {exception}"

From 7dbd3d75340522195e7d08a725cf5ae116288c8e Mon Sep 17 00:00:00 2001
From: robot-clickhouse <robot-clickhouse@users.noreply.github.com>
Date: Wed, 31 Jul 2024 13:51:38 +0000
Subject: [PATCH 1118/1488] Automatic style fix

---
 tests/integration/test_storage_delta/test.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/integration/test_storage_delta/test.py b/tests/integration/test_storage_delta/test.py
index 698becc18c4..e485bc90ee0 100644
--- a/tests/integration/test_storage_delta/test.py
+++ b/tests/integration/test_storage_delta/test.py
@@ -201,6 +201,7 @@ def test_partition_by(started_cluster):
 
     remove_local_directory_contents(f"/{TABLE_NAME}")
 
+
 def test_checkpoint(started_cluster):
     instance = started_cluster.instances["node1"]
     spark = started_cluster.spark_session

From 27f4f468b976e445e8b0dbc198ea9f0a9c62855b Mon Sep 17 00:00:00 2001
From: Michael Stetsyuk <mike.stetsyuk@gmail.com>
Date: Wed, 31 Jul 2024 14:55:00 +0100
Subject: [PATCH 1119/1488] make it possible to rerun test_recovery_time_metric
 multiple times

---
 tests/integration/test_recovery_time_metric/test.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/tests/integration/test_recovery_time_metric/test.py b/tests/integration/test_recovery_time_metric/test.py
index 628f2e744e0..6fcf2fad423 100644
--- a/tests/integration/test_recovery_time_metric/test.py
+++ b/tests/integration/test_recovery_time_metric/test.py
@@ -21,6 +21,7 @@ def start_cluster():
 def test_recovery_time_metric(start_cluster):
     node.query(
         """
+        DROP DATABASE IF EXISTS rdb;
         CREATE DATABASE rdb
         ENGINE = Replicated('/test/test_recovery_time_metric', 'shard1', 'replica1')
         """
@@ -28,6 +29,7 @@ def test_recovery_time_metric(start_cluster):
 
     node.query(
         """
+        DROP TABLE IF EXISTS rdb.t;
         CREATE TABLE rdb.t
         (
             `x` UInt32
@@ -51,3 +53,9 @@ def test_recovery_time_metric(start_cluster):
         ).strip()
     )
     assert ret > 0
+
+    node.query(
+        """
+        DROP DATABASE rdb
+        """
+    )

From d6de2be4395e1bcc62ab32ad1d5b02e9db080303 Mon Sep 17 00:00:00 2001
From: Dmitry Novik <mrnovikd@gmail.com>
Date: Wed, 31 Jul 2024 16:08:18 +0200
Subject: [PATCH 1120/1488] Fix build

---
 src/Planner/findParallelReplicasQuery.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Planner/findParallelReplicasQuery.cpp b/src/Planner/findParallelReplicasQuery.cpp
index 1140f30ad9c..39edb1e6516 100644
--- a/src/Planner/findParallelReplicasQuery.cpp
+++ b/src/Planner/findParallelReplicasQuery.cpp
@@ -144,7 +144,7 @@ public:
     std::unordered_map<const IQueryTreeNode *, QueryTreeNodePtr> replacement_map;
 };
 
-QueryTreeNodePtr replaceTablesWithDummyTables(const QueryTreeNodePtr & query, const ContextPtr & context)
+QueryTreeNodePtr replaceTablesWithDummyTables(QueryTreeNodePtr query, const ContextPtr & context)
 {
     ReplaceTableNodeToDummyVisitor visitor(context);
     visitor.visit(query);

From 9ffbd8f5073e180592a494742d1dc3af4427b55f Mon Sep 17 00:00:00 2001
From: divanik <dan.ivanik@clickhouse.com>
Date: Wed, 31 Jul 2024 14:13:43 +0000
Subject: [PATCH 1121/1488] Possible fix

---
 .../03164_s3_settings_for_queries_and_merges.sql          | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/queries/0_stateless/03164_s3_settings_for_queries_and_merges.sql b/tests/queries/0_stateless/03164_s3_settings_for_queries_and_merges.sql
index ac2070fbd76..e43c9ae7717 100644
--- a/tests/queries/0_stateless/03164_s3_settings_for_queries_and_merges.sql
+++ b/tests/queries/0_stateless/03164_s3_settings_for_queries_and_merges.sql
@@ -22,18 +22,18 @@ SELECT count() FROM t_compact_bytes_s3 WHERE NOT ignore(c2, c4);
 SYSTEM FLUSH LOGS;
 
 SELECT
-    ProfileEvents['S3ReadRequestsCount'],
+    ProfileEvents['S3ReadRequestsCount'] - ProfileEvents['S3ReadRequestsError'],
     ProfileEvents['ReadBufferFromS3Bytes'] < ProfileEvents['ReadCompressedBytes'] * 1.1
 FROM system.query_log
-WHERE event_date >= yesterday() AND type = 'QueryFinish'
+WHERE type = 'QueryFinish'
     AND current_database = currentDatabase()
     AND query ilike '%INSERT INTO t_compact_bytes_s3 SELECT number, number, number%';
 
 SELECT
-    ProfileEvents['S3ReadRequestsCount'],
+    ProfileEvents['S3ReadRequestsCount'] - ProfileEvents['S3ReadRequestsError'],
     ProfileEvents['ReadBufferFromS3Bytes'] < ProfileEvents['ReadCompressedBytes'] * 1.1
 FROM system.query_log
-WHERE event_date >= yesterday() AND type = 'QueryFinish'
+WHERE type = 'QueryFinish'
     AND current_database = currentDatabase()
     AND query ilike '%OPTIMIZE TABLE t_compact_bytes_s3 FINAL%';
 

From 67f4792b77f2a2cf0de21ead6e95c3635d26aa88 Mon Sep 17 00:00:00 2001
From: divanik <dan.ivanik@clickhouse.com>
Date: Wed, 31 Jul 2024 14:16:40 +0000
Subject: [PATCH 1122/1488] Style check

---
 tests/integration/helpers/cluster.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py
index 7f0a9154be9..2e38aec3512 100644
--- a/tests/integration/helpers/cluster.py
+++ b/tests/integration/helpers/cluster.py
@@ -2971,6 +2971,7 @@ class ClickHouseCluster:
                     "Trying to create Azurite instance by command %s",
                     " ".join(map(str, azurite_start_cmd)),
                 )
+
                 def logging_azurite_initialization(exception, retry_number, sleep_time):
                     logging.info(
                         f"Azurite initialization failed with error: {exception}"

From 1f1f0528ce3a1fb20ceee5513523787a14718b80 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Wed, 31 Jul 2024 14:32:07 +0000
Subject: [PATCH 1123/1488] Prefer constant to INPUT in PlannerActionsVisitor.

---
 src/Planner/PlannerActionsVisitor.cpp         | 11 +++++++-
 ...lyzer_materialized_constants_bug.reference |  3 +++
 ...15_analyzer_materialized_constants_bug.sql | 26 +++++++++++++++++++
 3 files changed, 39 insertions(+), 1 deletion(-)
 create mode 100644 tests/queries/0_stateless/03215_analyzer_materialized_constants_bug.reference
 create mode 100644 tests/queries/0_stateless/03215_analyzer_materialized_constants_bug.sql

diff --git a/src/Planner/PlannerActionsVisitor.cpp b/src/Planner/PlannerActionsVisitor.cpp
index 1960855792c..57457493844 100644
--- a/src/Planner/PlannerActionsVisitor.cpp
+++ b/src/Planner/PlannerActionsVisitor.cpp
@@ -491,7 +491,16 @@ public:
     {
         auto it = node_name_to_node.find(node_name);
         if (it != node_name_to_node.end())
-            return it->second;
+        {
+            /// It is possible that ActionsDAG already has an input with the same name as constant.
+            /// In this case, prefer constant to input.
+            /// Constatns affect function return type, which should be consistent with QueryTree.
+            /// Query example:
+            /// SELECT materialize(toLowCardinality('b')) || 'a' FROM remote('127.0.0.{1,2}', system, one) GROUP BY 'a'
+            bool materialized_input = it->second->type == ActionsDAG::ActionType::INPUT && !it->second->column;
+            if (!materialized_input)
+                return it->second;
+        }
 
         const auto * node = &actions_dag.addColumn(column);
         node_name_to_node[node->result_name] = node;
diff --git a/tests/queries/0_stateless/03215_analyzer_materialized_constants_bug.reference b/tests/queries/0_stateless/03215_analyzer_materialized_constants_bug.reference
new file mode 100644
index 00000000000..584e34c0cde
--- /dev/null
+++ b/tests/queries/0_stateless/03215_analyzer_materialized_constants_bug.reference
@@ -0,0 +1,3 @@
+ba
+\N
+1	111111111111111111111111111111111111111
diff --git a/tests/queries/0_stateless/03215_analyzer_materialized_constants_bug.sql b/tests/queries/0_stateless/03215_analyzer_materialized_constants_bug.sql
new file mode 100644
index 00000000000..f9ec28d09d8
--- /dev/null
+++ b/tests/queries/0_stateless/03215_analyzer_materialized_constants_bug.sql
@@ -0,0 +1,26 @@
+SET allow_experimental_analyzer = 1;
+
+SELECT concat(materialize(toLowCardinality('b')), 'a') FROM remote('127.0.0.{1,2}', system, one) GROUP BY 'a';
+
+SELECT concat(NULLIF(1, materialize(toLowCardinality(1))), concat(NULLIF(1, 1))) FROM remote('127.0.0.{1,2}', system, one) GROUP BY concat(NULLIF(1, 1));
+
+DROP TABLE IF EXISTS test__fuzz_21;
+CREATE TABLE test__fuzz_21
+(
+    `x` Decimal(18, 10)
+)
+ENGINE = MergeTree
+ORDER BY x;
+
+INSERT INTO test__fuzz_21 VALUES (1), (2), (3);
+
+WITH (
+        SELECT CAST(toFixedString(toFixedString(materialize(toFixedString('111111111111111111111111111111111111111', 39)), 39), 39), 'UInt128')
+    ) AS v
+SELECT
+    coalesce(materialize(toLowCardinality(toNullable(1))), 10, NULL),
+    max(v)
+FROM remote('127.0.0.{1,2}', default, test__fuzz_21)
+GROUP BY
+    coalesce(NULL),
+    coalesce(1, 10, 10, materialize(NULL));

From e31569a065d4c81cdea671727c39983d7f3a84e5 Mon Sep 17 00:00:00 2001
From: Dmitry Novik <mrnovikd@gmail.com>
Date: Wed, 31 Jul 2024 16:32:37 +0200
Subject: [PATCH 1124/1488] Expect an unknown cluster

---
 .../0_stateless/03215_analyzer_replace_with_dummy_tables.sql    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/03215_analyzer_replace_with_dummy_tables.sql b/tests/queries/0_stateless/03215_analyzer_replace_with_dummy_tables.sql
index 12d2bd627a7..6d084c2ac50 100644
--- a/tests/queries/0_stateless/03215_analyzer_replace_with_dummy_tables.sql
+++ b/tests/queries/0_stateless/03215_analyzer_replace_with_dummy_tables.sql
@@ -12,4 +12,4 @@ FROM
         )
     FROM t
 )
-SETTINGS allow_experimental_parallel_reading_from_replicas = 1, max_parallel_replicas = 2, allow_experimental_analyzer = 1;
+SETTINGS allow_experimental_parallel_reading_from_replicas = 1, max_parallel_replicas = 2, allow_experimental_analyzer = 1; -- { serverError CLUSTER_DOESNT_EXIST }

From 8c36fbf4eddeba9282b53f726976b55f62d3ee19 Mon Sep 17 00:00:00 2001
From: divanik <dan.ivanik@clickhouse.com>
Date: Wed, 31 Jul 2024 14:42:38 +0000
Subject: [PATCH 1125/1488] Remove unnecessary change

---
 access/quotas.list                                  | Bin 0 -> 1 bytes
 access/roles.list                                   | Bin 0 -> 1 bytes
 access/row_policies.list                            | Bin 0 -> 1 bytes
 access/settings_profiles.list                       | Bin 0 -> 1 bytes
 access/users.list                                   | Bin 0 -> 1 bytes
 .../03164_s3_settings_for_queries_and_merges.sql    |   4 ++--
 6 files changed, 2 insertions(+), 2 deletions(-)
 create mode 100644 access/quotas.list
 create mode 100644 access/roles.list
 create mode 100644 access/row_policies.list
 create mode 100644 access/settings_profiles.list
 create mode 100644 access/users.list

diff --git a/access/quotas.list b/access/quotas.list
new file mode 100644
index 0000000000000000000000000000000000000000..f76dd238ade08917e6712764a16a22005a50573d
GIT binary patch
literal 1
IcmZPo000310RR91

literal 0
HcmV?d00001

diff --git a/access/roles.list b/access/roles.list
new file mode 100644
index 0000000000000000000000000000000000000000..f76dd238ade08917e6712764a16a22005a50573d
GIT binary patch
literal 1
IcmZPo000310RR91

literal 0
HcmV?d00001

diff --git a/access/row_policies.list b/access/row_policies.list
new file mode 100644
index 0000000000000000000000000000000000000000..f76dd238ade08917e6712764a16a22005a50573d
GIT binary patch
literal 1
IcmZPo000310RR91

literal 0
HcmV?d00001

diff --git a/access/settings_profiles.list b/access/settings_profiles.list
new file mode 100644
index 0000000000000000000000000000000000000000..f76dd238ade08917e6712764a16a22005a50573d
GIT binary patch
literal 1
IcmZPo000310RR91

literal 0
HcmV?d00001

diff --git a/access/users.list b/access/users.list
new file mode 100644
index 0000000000000000000000000000000000000000..f76dd238ade08917e6712764a16a22005a50573d
GIT binary patch
literal 1
IcmZPo000310RR91

literal 0
HcmV?d00001

diff --git a/tests/queries/0_stateless/03164_s3_settings_for_queries_and_merges.sql b/tests/queries/0_stateless/03164_s3_settings_for_queries_and_merges.sql
index e43c9ae7717..94e390537df 100644
--- a/tests/queries/0_stateless/03164_s3_settings_for_queries_and_merges.sql
+++ b/tests/queries/0_stateless/03164_s3_settings_for_queries_and_merges.sql
@@ -25,7 +25,7 @@ SELECT
     ProfileEvents['S3ReadRequestsCount'] - ProfileEvents['S3ReadRequestsError'],
     ProfileEvents['ReadBufferFromS3Bytes'] < ProfileEvents['ReadCompressedBytes'] * 1.1
 FROM system.query_log
-WHERE type = 'QueryFinish'
+WHERE event_date >= yesterday() AND type = 'QueryFinish'
     AND current_database = currentDatabase()
     AND query ilike '%INSERT INTO t_compact_bytes_s3 SELECT number, number, number%';
 
@@ -33,7 +33,7 @@ SELECT
     ProfileEvents['S3ReadRequestsCount'] - ProfileEvents['S3ReadRequestsError'],
     ProfileEvents['ReadBufferFromS3Bytes'] < ProfileEvents['ReadCompressedBytes'] * 1.1
 FROM system.query_log
-WHERE type = 'QueryFinish'
+WHERE event_date >= yesterday() AND type = 'QueryFinish'
     AND current_database = currentDatabase()
     AND query ilike '%OPTIMIZE TABLE t_compact_bytes_s3 FINAL%';
 

From 20ec27f9dc79d7ee81cd06f1587de83c8ce81441 Mon Sep 17 00:00:00 2001
From: divanik <dan.ivanik@clickhouse.com>
Date: Wed, 31 Jul 2024 14:46:53 +0000
Subject: [PATCH 1126/1488] Remove trach dir

---
 access/quotas.list            | Bin 1 -> 0 bytes
 access/roles.list             | Bin 1 -> 0 bytes
 access/row_policies.list      | Bin 1 -> 0 bytes
 access/settings_profiles.list | Bin 1 -> 0 bytes
 access/users.list             | Bin 1 -> 0 bytes
 5 files changed, 0 insertions(+), 0 deletions(-)
 delete mode 100644 access/quotas.list
 delete mode 100644 access/roles.list
 delete mode 100644 access/row_policies.list
 delete mode 100644 access/settings_profiles.list
 delete mode 100644 access/users.list

diff --git a/access/quotas.list b/access/quotas.list
deleted file mode 100644
index f76dd238ade08917e6712764a16a22005a50573d..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 1
IcmZPo000310RR91

diff --git a/access/roles.list b/access/roles.list
deleted file mode 100644
index f76dd238ade08917e6712764a16a22005a50573d..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 1
IcmZPo000310RR91

diff --git a/access/row_policies.list b/access/row_policies.list
deleted file mode 100644
index f76dd238ade08917e6712764a16a22005a50573d..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 1
IcmZPo000310RR91

diff --git a/access/settings_profiles.list b/access/settings_profiles.list
deleted file mode 100644
index f76dd238ade08917e6712764a16a22005a50573d..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 1
IcmZPo000310RR91

diff --git a/access/users.list b/access/users.list
deleted file mode 100644
index f76dd238ade08917e6712764a16a22005a50573d..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 1
IcmZPo000310RR91


From 650737890299f8cad2c77ad46022ee0a37b284eb Mon Sep 17 00:00:00 2001
From: divanik <dan.ivanik@clickhouse.com>
Date: Wed, 31 Jul 2024 14:49:30 +0000
Subject: [PATCH 1127/1488] Fix erroe with profile event name

---
 .../0_stateless/03164_s3_settings_for_queries_and_merges.sql  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/queries/0_stateless/03164_s3_settings_for_queries_and_merges.sql b/tests/queries/0_stateless/03164_s3_settings_for_queries_and_merges.sql
index 94e390537df..001ef382850 100644
--- a/tests/queries/0_stateless/03164_s3_settings_for_queries_and_merges.sql
+++ b/tests/queries/0_stateless/03164_s3_settings_for_queries_and_merges.sql
@@ -22,7 +22,7 @@ SELECT count() FROM t_compact_bytes_s3 WHERE NOT ignore(c2, c4);
 SYSTEM FLUSH LOGS;
 
 SELECT
-    ProfileEvents['S3ReadRequestsCount'] - ProfileEvents['S3ReadRequestsError'],
+    ProfileEvents['S3ReadRequestsCount'] - ProfileEvents['S3ReadRequestsErrors'],
     ProfileEvents['ReadBufferFromS3Bytes'] < ProfileEvents['ReadCompressedBytes'] * 1.1
 FROM system.query_log
 WHERE event_date >= yesterday() AND type = 'QueryFinish'
@@ -30,7 +30,7 @@ WHERE event_date >= yesterday() AND type = 'QueryFinish'
     AND query ilike '%INSERT INTO t_compact_bytes_s3 SELECT number, number, number%';
 
 SELECT
-    ProfileEvents['S3ReadRequestsCount'] - ProfileEvents['S3ReadRequestsError'],
+    ProfileEvents['S3ReadRequestsCount'] - ProfileEvents['S3ReadRequestsErrors'],
     ProfileEvents['ReadBufferFromS3Bytes'] < ProfileEvents['ReadCompressedBytes'] * 1.1
 FROM system.query_log
 WHERE event_date >= yesterday() AND type = 'QueryFinish'

From c3c653e7692a755c3467b77e866555734d50ef50 Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <nikitamikhaylov@clickhouse.com>
Date: Wed, 31 Jul 2024 14:58:52 +0000
Subject: [PATCH 1128/1488] Better

---
 src/Interpreters/DatabaseCatalog.cpp                           | 2 ++
 .../config.d/database_catalog_drop_table_concurrency.xml       | 3 +++
 tests/config/install.sh                                        | 1 +
 3 files changed, 6 insertions(+)
 create mode 100644 tests/config/config.d/database_catalog_drop_table_concurrency.xml

diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp
index a8e5fd7e6aa..48b01a9df43 100644
--- a/src/Interpreters/DatabaseCatalog.cpp
+++ b/src/Interpreters/DatabaseCatalog.cpp
@@ -1406,6 +1406,8 @@ void DatabaseCatalog::waitTableFinallyDropped(const UUID & uuid)
         return !tables_marked_dropped_ids.contains(uuid) || is_shutting_down;
     });
 
+    LOG_DEBUG(log, "Done waiting for the table {} to be dropped. The outcome: {}", toString(uuid), tables_marked_dropped_ids).contains(uuid) ? "table still exists" : "table dropped successfully");
+
     /// TSA doesn't support unique_lock
     if (TSA_SUPPRESS_WARNING_FOR_READ(tables_marked_dropped_ids).contains(uuid))
         throw Exception(ErrorCodes::UNFINISHED, "Did not finish dropping the table with UUID {} because the server is shutting down, "
diff --git a/tests/config/config.d/database_catalog_drop_table_concurrency.xml b/tests/config/config.d/database_catalog_drop_table_concurrency.xml
new file mode 100644
index 00000000000..ac118625f4e
--- /dev/null
+++ b/tests/config/config.d/database_catalog_drop_table_concurrency.xml
@@ -0,0 +1,3 @@
+<clickhouse>
+    <database_catalog_drop_table_concurrency>256</database_catalog_drop_table_concurrency>
+</clickhouse>
diff --git a/tests/config/install.sh b/tests/config/install.sh
index 1b0edc5fc16..7c4b36dc4bd 100755
--- a/tests/config/install.sh
+++ b/tests/config/install.sh
@@ -21,6 +21,7 @@ ln -sf $SRC_PATH/config.d/listen.xml $DEST_SERVER_PATH/config.d/
 ln -sf $SRC_PATH/config.d/text_log.xml $DEST_SERVER_PATH/config.d/
 ln -sf $SRC_PATH/config.d/blob_storage_log.xml $DEST_SERVER_PATH/config.d/
 ln -sf $SRC_PATH/config.d/custom_settings_prefixes.xml $DEST_SERVER_PATH/config.d/
+ln -sf $SRC_PATH/config.d/database_catalog_drop_table_concurrency.xml $DEST_SERVER_PATH/config.d/
 ln -sf $SRC_PATH/config.d/enable_access_control_improvements.xml $DEST_SERVER_PATH/config.d/
 ln -sf $SRC_PATH/config.d/macros.xml $DEST_SERVER_PATH/config.d/
 ln -sf $SRC_PATH/config.d/secure_ports.xml $DEST_SERVER_PATH/config.d/

From 5ab8c0357a84d0265bdb43f43795b06317854772 Mon Sep 17 00:00:00 2001
From: kssenii <sumarokovakseniia@mail.ru>
Date: Wed, 31 Jul 2024 16:30:14 +0200
Subject: [PATCH 1129/1488] Fix potential busy loop in keepFreeSpaceRatioFunc

---
 src/Interpreters/Cache/FileCache.cpp          | 30 +++++++++++++------
 src/Interpreters/Cache/IFileCachePriority.h   |  8 ++++-
 .../Cache/LRUFileCachePriority.cpp            | 21 +++++++++++--
 src/Interpreters/Cache/LRUFileCachePriority.h |  2 +-
 .../Cache/SLRUFileCachePriority.cpp           | 15 ++++++----
 .../Cache/SLRUFileCachePriority.h             |  2 +-
 6 files changed, 57 insertions(+), 21 deletions(-)

diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp
index a88c0de2cfe..bf8dd24a1db 100644
--- a/src/Interpreters/Cache/FileCache.cpp
+++ b/src/Interpreters/Cache/FileCache.cpp
@@ -998,18 +998,19 @@ void FileCache::freeSpaceRatioKeepingThreadFunc()
     FileCacheReserveStat stat;
     EvictionCandidates eviction_candidates;
 
-    bool limits_satisfied = true;
+    IFileCachePriority::DesiredSizeStatus desired_size_status;
     try
     {
         /// Collect at most `keep_up_free_space_remove_batch` elements to evict,
         /// (we use batches to make sure we do not block cache for too long,
         /// by default the batch size is quite small).
-        limits_satisfied = main_priority->collectCandidatesForEviction(
+        desired_size_status = main_priority->collectCandidatesForEviction(
             desired_size, desired_elements_num, keep_up_free_space_remove_batch, stat, eviction_candidates, lock);
 
 #ifdef DEBUG_OR_SANITIZER_BUILD
         /// Let's make sure that we correctly processed the limits.
-        if (limits_satisfied && eviction_candidates.size() < keep_up_free_space_remove_batch)
+        if (desired_size_status == IFileCachePriority::DesiredSizeStatus::SUCCESS
+            && eviction_candidates.size() < keep_up_free_space_remove_batch)
         {
             const auto current_size = main_priority->getSize(lock);
             chassert(current_size >= stat.total_stat.releasable_size);
@@ -1063,13 +1064,24 @@ void FileCache::freeSpaceRatioKeepingThreadFunc()
     watch.stop();
     ProfileEvents::increment(ProfileEvents::FilesystemCacheFreeSpaceKeepingThreadWorkMilliseconds, watch.elapsedMilliseconds());
 
-    LOG_TRACE(log, "Free space ratio keeping thread finished in {} ms", watch.elapsedMilliseconds());
+    LOG_TRACE(log, "Free space ratio keeping thread finished in {} ms (status: {})",
+              watch.elapsedMilliseconds(), desired_size_status);
 
     [[maybe_unused]] bool scheduled = false;
-    if (limits_satisfied)
-        scheduled = keep_up_free_space_ratio_task->scheduleAfter(general_reschedule_ms);
-    else
-        scheduled = keep_up_free_space_ratio_task->schedule();
+    switch (desired_size_status)
+    {
+        case IFileCachePriority::DesiredSizeStatus::SUCCESS: [[fallthrough]];
+        case IFileCachePriority::DesiredSizeStatus::CANNOT_EVICT:
+        {
+            scheduled = keep_up_free_space_ratio_task->scheduleAfter(general_reschedule_ms);
+            break;
+        }
+        case IFileCachePriority::DesiredSizeStatus::REACHED_MAX_CANDIDATES_LIMIT:
+        {
+            scheduled = keep_up_free_space_ratio_task->schedule();
+            break;
+        }
+    }
     chassert(scheduled);
 }
 
@@ -1546,7 +1558,7 @@ void FileCache::applySettingsIfPossible(const FileCacheSettings & new_settings,
             FileCacheReserveStat stat;
             if (main_priority->collectCandidatesForEviction(
                     new_settings.max_size, new_settings.max_elements, 0/* max_candidates_to_evict */,
-                    stat, eviction_candidates, cache_lock))
+                    stat, eviction_candidates, cache_lock) == IFileCachePriority::DesiredSizeStatus::SUCCESS)
             {
                 if (eviction_candidates.size() == 0)
                 {
diff --git a/src/Interpreters/Cache/IFileCachePriority.h b/src/Interpreters/Cache/IFileCachePriority.h
index 5d8eb9dd54a..9885ab00f78 100644
--- a/src/Interpreters/Cache/IFileCachePriority.h
+++ b/src/Interpreters/Cache/IFileCachePriority.h
@@ -151,7 +151,13 @@ public:
     /// and `desired_elements_num` as current cache state.
     /// Collect no more than `max_candidates_to_evict` elements.
     /// Return `true` if the first condition is satisfied.
-    virtual bool collectCandidatesForEviction(
+    enum class DesiredSizeStatus
+    {
+        SUCCESS,
+        CANNOT_EVICT,
+        REACHED_MAX_CANDIDATES_LIMIT,
+    };
+    virtual DesiredSizeStatus collectCandidatesForEviction(
         size_t desired_size,
         size_t desired_elements_count,
         size_t max_candidates_to_evict,
diff --git a/src/Interpreters/Cache/LRUFileCachePriority.cpp b/src/Interpreters/Cache/LRUFileCachePriority.cpp
index ec96eb14a8a..7970eaa3e13 100644
--- a/src/Interpreters/Cache/LRUFileCachePriority.cpp
+++ b/src/Interpreters/Cache/LRUFileCachePriority.cpp
@@ -323,7 +323,7 @@ bool LRUFileCachePriority::collectCandidatesForEviction(
     }
 }
 
-bool LRUFileCachePriority::collectCandidatesForEviction(
+IFileCachePriority::DesiredSizeStatus LRUFileCachePriority::collectCandidatesForEviction(
     size_t desired_size,
     size_t desired_elements_count,
     size_t max_candidates_to_evict,
@@ -336,12 +336,24 @@ bool LRUFileCachePriority::collectCandidatesForEviction(
         return canFit(0, 0, stat.total_stat.releasable_size, stat.total_stat.releasable_count,
                       lock, &desired_size, &desired_elements_count);
     };
+    auto status = DesiredSizeStatus::CANNOT_EVICT;
     auto stop_condition = [&]()
     {
-        return desired_limits_satisfied() || (max_candidates_to_evict && res.size() >= max_candidates_to_evict);
+        if (desired_limits_satisfied())
+        {
+            status = DesiredSizeStatus::SUCCESS;
+            return true;
+        }
+        if (max_candidates_to_evict && res.size() >= max_candidates_to_evict)
+        {
+            status = DesiredSizeStatus::REACHED_MAX_CANDIDATES_LIMIT;
+            return true;
+        }
+        return false;
     };
     iterateForEviction(res, stat, stop_condition, lock);
-    return desired_limits_satisfied();
+    chassert(status != DesiredSizeStatus::SUCCESS || stop_condition());
+    return status;
 }
 
 void LRUFileCachePriority::iterateForEviction(
@@ -350,6 +362,9 @@ void LRUFileCachePriority::iterateForEviction(
     StopConditionFunc stop_condition,
     const CachePriorityGuard::Lock & lock)
 {
+    if (stop_condition())
+        return;
+
     ProfileEvents::increment(ProfileEvents::FilesystemCacheEvictionTries);
 
     IterateFunc iterate_func = [&](LockedKey & locked_key, const FileSegmentMetadataPtr & segment_metadata)
diff --git a/src/Interpreters/Cache/LRUFileCachePriority.h b/src/Interpreters/Cache/LRUFileCachePriority.h
index e0691cade43..9bced106727 100644
--- a/src/Interpreters/Cache/LRUFileCachePriority.h
+++ b/src/Interpreters/Cache/LRUFileCachePriority.h
@@ -63,7 +63,7 @@ public:
         const UserID & user_id,
         const CachePriorityGuard::Lock &) override;
 
-    bool collectCandidatesForEviction(
+    DesiredSizeStatus collectCandidatesForEviction(
         size_t desired_size,
         size_t desired_elements_count,
         size_t max_candidates_to_evict,
diff --git a/src/Interpreters/Cache/SLRUFileCachePriority.cpp b/src/Interpreters/Cache/SLRUFileCachePriority.cpp
index 7a3fdf5160e..dc0df223cb0 100644
--- a/src/Interpreters/Cache/SLRUFileCachePriority.cpp
+++ b/src/Interpreters/Cache/SLRUFileCachePriority.cpp
@@ -256,7 +256,7 @@ bool SLRUFileCachePriority::collectCandidatesForEvictionInProtected(
     return true;
 }
 
-bool SLRUFileCachePriority::collectCandidatesForEviction(
+IFileCachePriority::DesiredSizeStatus SLRUFileCachePriority::collectCandidatesForEviction(
     size_t desired_size,
     size_t desired_elements_count,
     size_t max_candidates_to_evict,
@@ -268,7 +268,7 @@ bool SLRUFileCachePriority::collectCandidatesForEviction(
     const auto desired_probationary_elements_num = getRatio(desired_elements_count, 1 - size_ratio);
 
     FileCacheReserveStat probationary_stat;
-    const bool probationary_limit_satisfied = probationary_queue.collectCandidatesForEviction(
+    const auto probationary_desired_size_status = probationary_queue.collectCandidatesForEviction(
         desired_probationary_size, desired_probationary_elements_num,
         max_candidates_to_evict, probationary_stat, res, lock);
 
@@ -285,14 +285,14 @@ bool SLRUFileCachePriority::collectCandidatesForEviction(
     chassert(!max_candidates_to_evict || res.size() <= max_candidates_to_evict);
     chassert(res.size() == stat.total_stat.releasable_count);
 
-    if (max_candidates_to_evict && res.size() >= max_candidates_to_evict)
-        return probationary_limit_satisfied;
+    if (probationary_desired_size_status == DesiredSizeStatus::REACHED_MAX_CANDIDATES_LIMIT)
+        return probationary_desired_size_status;
 
     const auto desired_protected_size = getRatio(desired_size, size_ratio);
     const auto desired_protected_elements_num = getRatio(desired_elements_count, size_ratio);
 
     FileCacheReserveStat protected_stat;
-    const bool protected_limit_satisfied = protected_queue.collectCandidatesForEviction(
+    const auto protected_desired_size_status = protected_queue.collectCandidatesForEviction(
         desired_protected_size, desired_protected_elements_num,
         max_candidates_to_evict - res.size(), protected_stat, res, lock);
 
@@ -306,7 +306,10 @@ bool SLRUFileCachePriority::collectCandidatesForEviction(
              desired_protected_size, desired_protected_elements_num,
              protected_queue.getStateInfoForLog(lock));
 
-    return probationary_limit_satisfied && protected_limit_satisfied;
+    if (probationary_desired_size_status == DesiredSizeStatus::SUCCESS)
+        return protected_desired_size_status;
+    else
+        return probationary_desired_size_status;
 }
 
 void SLRUFileCachePriority::downgrade(IteratorPtr iterator, const CachePriorityGuard::Lock & lock)
diff --git a/src/Interpreters/Cache/SLRUFileCachePriority.h b/src/Interpreters/Cache/SLRUFileCachePriority.h
index 2102a0ec558..e6d20e0d0ee 100644
--- a/src/Interpreters/Cache/SLRUFileCachePriority.h
+++ b/src/Interpreters/Cache/SLRUFileCachePriority.h
@@ -58,7 +58,7 @@ public:
         const UserID & user_id,
         const CachePriorityGuard::Lock &) override;
 
-    bool collectCandidatesForEviction(
+    DesiredSizeStatus collectCandidatesForEviction(
         size_t desired_size,
         size_t desired_elements_count,
         size_t max_candidates_to_evict,

From 19cd00000373a5707178214744444b4d8c4034a5 Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@clickhouse.com>
Date: Wed, 31 Jul 2024 17:18:55 +0200
Subject: [PATCH 1130/1488] Update src/Interpreters/DatabaseCatalog.cpp

---
 src/Interpreters/DatabaseCatalog.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp
index 48b01a9df43..56d9c323d39 100644
--- a/src/Interpreters/DatabaseCatalog.cpp
+++ b/src/Interpreters/DatabaseCatalog.cpp
@@ -1406,7 +1406,7 @@ void DatabaseCatalog::waitTableFinallyDropped(const UUID & uuid)
         return !tables_marked_dropped_ids.contains(uuid) || is_shutting_down;
     });
 
-    LOG_DEBUG(log, "Done waiting for the table {} to be dropped. The outcome: {}", toString(uuid), tables_marked_dropped_ids).contains(uuid) ? "table still exists" : "table dropped successfully");
+    LOG_DEBUG(log, "Done waiting for the table {} to be dropped. The outcome: {}", toString(uuid), tables_marked_dropped_ids.contains(uuid) ? "table still exists" : "table dropped successfully");
 
     /// TSA doesn't support unique_lock
     if (TSA_SUPPRESS_WARNING_FOR_READ(tables_marked_dropped_ids).contains(uuid))

From f032c015ca9ec10b7938bbf3d67bb6181776d24a Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Wed, 31 Jul 2024 15:40:37 +0000
Subject: [PATCH 1131/1488] Ignore some tests

---
 tests/queries/0_stateless/00705_drop_create_merge_tree.sh     | 4 ++--
 .../0_stateless/01019_alter_materialized_view_atomic.sh       | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/queries/0_stateless/00705_drop_create_merge_tree.sh b/tests/queries/0_stateless/00705_drop_create_merge_tree.sh
index fd002668696..ea8b9d02e49 100755
--- a/tests/queries/0_stateless/00705_drop_create_merge_tree.sh
+++ b/tests/queries/0_stateless/00705_drop_create_merge_tree.sh
@@ -5,8 +5,8 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
-yes 'CREATE TABLE IF NOT EXISTS table (x UInt8) ENGINE = MergeTree ORDER BY tuple();' | head -n 1000 | $CLICKHOUSE_CLIENT &
-yes 'DROP TABLE IF EXISTS table;' | head -n 1000 | $CLICKHOUSE_CLIENT &
+yes 'CREATE TABLE IF NOT EXISTS table (x UInt8) ENGINE = MergeTree ORDER BY tuple();' | head -n 1000 | $CLICKHOUSE_CLIENT --multiquery &
+yes 'DROP TABLE IF EXISTS table;' | head -n 1000 | $CLICKHOUSE_CLIENT --multiquery &
 wait
 
 ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS table"
diff --git a/tests/queries/0_stateless/01019_alter_materialized_view_atomic.sh b/tests/queries/0_stateless/01019_alter_materialized_view_atomic.sh
index eb12a76eb62..4bd21fcee02 100755
--- a/tests/queries/0_stateless/01019_alter_materialized_view_atomic.sh
+++ b/tests/queries/0_stateless/01019_alter_materialized_view_atomic.sh
@@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
-$CLICKHOUSE_CLIENT <<EOF
+$CLICKHOUSE_CLIENT --multiquery <<EOF
 DROP TABLE IF EXISTS src;
 DROP TABLE IF EXISTS mv;
 

From 1e257bd5be3a94e3c33de9f902d6de7958e588d5 Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <nikitamikhaylov@clickhouse.com>
Date: Wed, 31 Jul 2024 15:41:37 +0000
Subject: [PATCH 1132/1488] Fix build

---
 src/Interpreters/DatabaseCatalog.cpp | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp
index 56d9c323d39..273e5720679 100644
--- a/src/Interpreters/DatabaseCatalog.cpp
+++ b/src/Interpreters/DatabaseCatalog.cpp
@@ -1406,10 +1406,11 @@ void DatabaseCatalog::waitTableFinallyDropped(const UUID & uuid)
         return !tables_marked_dropped_ids.contains(uuid) || is_shutting_down;
     });
 
-    LOG_DEBUG(log, "Done waiting for the table {} to be dropped. The outcome: {}", toString(uuid), tables_marked_dropped_ids.contains(uuid) ? "table still exists" : "table dropped successfully");
-
     /// TSA doesn't support unique_lock
-    if (TSA_SUPPRESS_WARNING_FOR_READ(tables_marked_dropped_ids).contains(uuid))
+    const bool has_table = TSA_SUPPRESS_WARNING_FOR_READ(tables_marked_dropped_ids).contains(uuid);
+    LOG_DEBUG(log, "Done waiting for the table {} to be dropped. The outcome: {}", toString(uuid), has_table ? "table still exists" : "table dropped successfully");
+
+    if has_table)
         throw Exception(ErrorCodes::UNFINISHED, "Did not finish dropping the table with UUID {} because the server is shutting down, "
                                                 "will finish after restart", uuid);
 }

From 8afe61e04581d0b95ac2d6e927bb9d2427247c7a Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <mikhaylovnikitka@gmail.com>
Date: Wed, 31 Jul 2024 17:58:41 +0200
Subject: [PATCH 1133/1488] Better

---
 src/Interpreters/DatabaseCatalog.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp
index 273e5720679..fb4fad85f66 100644
--- a/src/Interpreters/DatabaseCatalog.cpp
+++ b/src/Interpreters/DatabaseCatalog.cpp
@@ -1410,7 +1410,7 @@ void DatabaseCatalog::waitTableFinallyDropped(const UUID & uuid)
     const bool has_table = TSA_SUPPRESS_WARNING_FOR_READ(tables_marked_dropped_ids).contains(uuid);
     LOG_DEBUG(log, "Done waiting for the table {} to be dropped. The outcome: {}", toString(uuid), has_table ? "table still exists" : "table dropped successfully");
 
-    if has_table)
+    if (has_table)
         throw Exception(ErrorCodes::UNFINISHED, "Did not finish dropping the table with UUID {} because the server is shutting down, "
                                                 "will finish after restart", uuid);
 }

From 743d63767a74f41b3628c52ccf166be773baecf2 Mon Sep 17 00:00:00 2001
From: serxa <sergei@clickhouse.com>
Date: Wed, 31 Jul 2024 17:06:49 +0000
Subject: [PATCH 1134/1488] fix AsyncLoader destruction race

---
 src/Common/AsyncLoader.cpp | 33 ++++++++++++++++++++-------------
 1 file changed, 20 insertions(+), 13 deletions(-)

diff --git a/src/Common/AsyncLoader.cpp b/src/Common/AsyncLoader.cpp
index 6264eb03106..d40e320e741 100644
--- a/src/Common/AsyncLoader.cpp
+++ b/src/Common/AsyncLoader.cpp
@@ -218,20 +218,27 @@ AsyncLoader::~AsyncLoader()
 {
     // All `LoadTask` objects should be destructed before AsyncLoader destruction because they hold a reference.
     // To make sure we check for all pending jobs to be finished.
-    std::unique_lock lock{mutex};
-    if (scheduled_jobs.empty() && finished_jobs.empty())
-        return;
+    {
+        std::unique_lock lock{mutex};
+        if (!scheduled_jobs.empty() || !finished_jobs.empty())
+        {
+            std::vector<String> scheduled;
+            std::vector<String> finished;
+            scheduled.reserve(scheduled_jobs.size());
+            finished.reserve(finished_jobs.size());
+            for (const auto & [job, _] : scheduled_jobs)
+                scheduled.push_back(job->name);
+            for (const auto & job : finished_jobs)
+                finished.push_back(job->name);
+            LOG_ERROR(log, "Bug. Destruction with pending ({}) and finished ({}) load jobs.", fmt::join(scheduled, ", "), fmt::join(finished, ", "));
+            abort();
+        }
+    }
 
-    std::vector<String> scheduled;
-    std::vector<String> finished;
-    scheduled.reserve(scheduled_jobs.size());
-    finished.reserve(finished_jobs.size());
-    for (const auto & [job, _] : scheduled_jobs)
-        scheduled.push_back(job->name);
-    for (const auto & job : finished_jobs)
-        finished.push_back(job->name);
-    LOG_ERROR(log, "Bug. Destruction with pending ({}) and finished ({}) load jobs.", fmt::join(scheduled, ", "), fmt::join(finished, ", "));
-    abort();
+    // When all jobs are done we could still have finalizing workers.
+    // These workers could call updateCurrentPriorityAndSpawn() that scans all pools.
+    // We need to stop all of them before destructing any of them.
+    stop();
 }
 
 void AsyncLoader::start()

From 2b79da36c0701bb9ca392fddd9129a7e0e04ef3f Mon Sep 17 00:00:00 2001
From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com>
Date: Wed, 31 Jul 2024 19:26:45 +0200
Subject: [PATCH 1135/1488] Update 01605_adaptive_granularity_block_borders.sql

---
 .../0_stateless/01605_adaptive_granularity_block_borders.sql    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/01605_adaptive_granularity_block_borders.sql b/tests/queries/0_stateless/01605_adaptive_granularity_block_borders.sql
index 5f09dc423b2..f9b8bb1c1c6 100644
--- a/tests/queries/0_stateless/01605_adaptive_granularity_block_borders.sql
+++ b/tests/queries/0_stateless/01605_adaptive_granularity_block_borders.sql
@@ -1,4 +1,4 @@
--- Tags: no-random-merge-tree-settings, no-tsan, no-debug, no-object-storage, no-distributed-cache
+-- Tags: long, no-random-merge-tree-settings, no-tsan, no-debug, no-object-storage, no-distributed-cache
 -- no-tsan: too slow
 -- no-object-storage: for remote tables we use thread pool even when reading with one stream, so memory consumption is higher
 

From bfb0133f26f85e64f2d608595b653c20a594abca Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 31 Jul 2024 17:38:20 +0200
Subject: [PATCH 1136/1488] Minor change

---
 src/Databases/DatabaseOnDisk.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp
index f419f5811a1..734f354d9a5 100644
--- a/src/Databases/DatabaseOnDisk.cpp
+++ b/src/Databases/DatabaseOnDisk.cpp
@@ -313,7 +313,7 @@ void DatabaseOnDisk::detachTablePermanently(ContextPtr query_context, const Stri
         std::lock_guard lock(mutex);
         if (const auto it = snapshot_detached_tables.find(table_name); it == snapshot_detached_tables.end())
         {
-            throw Exception(ErrorCodes::LOGICAL_ERROR, "Snapshot doesn't contain info about detached table={}", table_name);
+            throw Exception(ErrorCodes::LOGICAL_ERROR, "Snapshot doesn't contain info about detached table `{}`", table_name);
         }
         else
         {

From 2c6c5c5c94a2f4131d898b172d6af285eec07c2f Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 31 Jul 2024 19:31:22 +0200
Subject: [PATCH 1137/1488] Add an assertion

---
 src/Common/TimerDescriptor.cpp | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/Common/TimerDescriptor.cpp b/src/Common/TimerDescriptor.cpp
index 9a171ae9487..716dcee2747 100644
--- a/src/Common/TimerDescriptor.cpp
+++ b/src/Common/TimerDescriptor.cpp
@@ -110,6 +110,9 @@ void TimerDescriptor::drain() const
                     throw ErrnoException(ErrorCodes::CANNOT_READ_FROM_SOCKET, "Cannot readlink for a timer_fd {}", timer_fd);
 
                 LOG_TRACE(log, "Received EINTR while trying to drain a TimerDescriptor, fd {}: {}", timer_fd, std::string_view(link_path, link_path_length));
+
+                /// Check that it's actually a timerfd.
+                chassert(std::string_view(link_path, link_path_length).contains("timerfd"));
                 continue;
             }
 

From 26a81e366d722314bbda12d5232fa49117ef4498 Mon Sep 17 00:00:00 2001
From: shiyer7474 <shiyer@altinity.com>
Date: Wed, 31 Jul 2024 18:05:53 +0000
Subject: [PATCH 1138/1488] Fix the serialization of parameters for
 parameterized views

Removed the call to convertFieldToString() and added datatype
specific serialization code. Parameterized view substitution
was broken for multiple datatypes when parameter value was a
function or expression returning datatype instance. Testcase
added to cover Date/Date32/UUID/IP datatypes.
---
 src/Analyzer/Resolve/QueryAnalyzer.cpp        | 10 +-
 ...zed_view_with_non_literal_params.reference | 31 ++++++
 ...meterized_view_with_non_literal_params.sql | 97 +++++++++++++++++++
 3 files changed, 137 insertions(+), 1 deletion(-)
 create mode 100644 tests/queries/0_stateless/03209_parameterized_view_with_non_literal_params.reference
 create mode 100644 tests/queries/0_stateless/03209_parameterized_view_with_non_literal_params.sql

diff --git a/src/Analyzer/Resolve/QueryAnalyzer.cpp b/src/Analyzer/Resolve/QueryAnalyzer.cpp
index 767d5c11075..6113a38d463 100644
--- a/src/Analyzer/Resolve/QueryAnalyzer.cpp
+++ b/src/Analyzer/Resolve/QueryAnalyzer.cpp
@@ -4546,7 +4546,15 @@ void QueryAnalyzer::resolveTableFunction(QueryTreeNodePtr & table_function_node,
                     resolveExpressionNode(nodes[1], scope, /* allow_lambda_expression */false, /* allow_table_function */false);
                     if (auto * constant = nodes[1]->as<ConstantNode>())
                     {
-                        view_params[identifier_node->getIdentifier().getFullName()] = convertFieldToString(constant->getValue());
+                        /// Serialize the constant value using datatype specific
+                        /// interfaces to match the deserialization in ReplaceQueryParametersVistor.
+                        WriteBufferFromOwnString buf;
+                        auto constval = constant->getValue();
+                        auto realtype = constant->getResultType();
+                        auto tempcol  = realtype->createColumn();
+                        tempcol->insert(constval);
+                        realtype->getDefaultSerialization()->serializeTextEscaped(*tempcol, 0, buf, {});
+                        view_params[identifier_node->getIdentifier().getFullName()] = buf.str();
                     }
                 }
             }
diff --git a/tests/queries/0_stateless/03209_parameterized_view_with_non_literal_params.reference b/tests/queries/0_stateless/03209_parameterized_view_with_non_literal_params.reference
new file mode 100644
index 00000000000..e4e6c313b85
--- /dev/null
+++ b/tests/queries/0_stateless/03209_parameterized_view_with_non_literal_params.reference
@@ -0,0 +1,31 @@
+Test with Date parameter
+1
+2
+1
+3
+3
+3
+2
+Test with Date32 parameter
+1
+2
+1
+3
+5
+3
+4
+Test with UUID parameter
+4
+3
+3
+1
+2
+Test with 2 parameters
+1
+1
+3
+3
+Test with IPv4
+1
+2
+3
diff --git a/tests/queries/0_stateless/03209_parameterized_view_with_non_literal_params.sql b/tests/queries/0_stateless/03209_parameterized_view_with_non_literal_params.sql
new file mode 100644
index 00000000000..55795c7a785
--- /dev/null
+++ b/tests/queries/0_stateless/03209_parameterized_view_with_non_literal_params.sql
@@ -0,0 +1,97 @@
+
+select 'Test with Date parameter';
+
+drop table if exists date_table_pv;
+create table date_table_pv (id Int32, dt Date) engine = Memory();
+
+insert into date_table_pv values(1, today());
+insert into date_table_pv values(2, yesterday());
+insert into date_table_pv values(3, toDate('1974-04-07'));
+
+drop view if exists date_pv;
+create view date_pv as select * from date_table_pv where dt =  {dtparam:Date};
+
+select id from date_pv(dtparam=today());
+select id from date_pv(dtparam=yesterday());
+select id from date_pv(dtparam=yesterday()+1);
+select id from date_pv(dtparam='1974-04-07');
+select id from date_pv(dtparam=toDate('1974-04-07'));
+select id from date_pv(dtparam=toString(toDate('1974-04-07')));
+select id from date_pv(dtparam=toDate('1975-04-07'));
+select id from date_pv(dtparam=(select dt from date_table_pv where id = 2));
+
+select 'Test with Date32 parameter';
+
+drop table if exists date32_table_pv;
+create table date32_table_pv (id Int32, dt Date32) engine = Memory();
+
+insert into date32_table_pv values(1, today());
+insert into date32_table_pv values(2, yesterday());
+insert into date32_table_pv values(3, toDate32('2199-12-31'));
+insert into date32_table_pv values(4, toDate32('1950-12-25'));
+insert into date32_table_pv values(5, toDate32('1900-01-01'));
+
+drop view if exists date32_pv;
+create view date32_pv as select * from date32_table_pv where dt =  {dtparam:Date32};
+
+select id from date32_pv(dtparam=today());
+select id from date32_pv(dtparam=yesterday());
+select id from date32_pv(dtparam=yesterday()+1);
+select id from date32_pv(dtparam='2199-12-31');
+select id from date32_pv(dtparam=toDate32('1900-01-01'));
+select id from date32_pv(dtparam=(select dt from date32_table_pv where id = 3));
+select id from date32_pv(dtparam=(select dt from date32_table_pv where id = 4));
+
+
+select 'Test with UUID parameter';
+drop table if exists uuid_table_pv;
+create table uuid_table_pv (id Int32, uu UUID) engine = Memory();
+
+insert into uuid_table_pv values(1, generateUUIDv4());
+insert into uuid_table_pv values(2, generateUUIDv7());
+insert into uuid_table_pv values(3, toUUID('11111111-2222-3333-4444-555555555555'));
+insert into uuid_table_pv select 4, serverUUID();
+
+
+drop view if exists uuid_pv;
+create view uuid_pv as select * from uuid_table_pv where uu =  {uuidparam:UUID};
+select id from uuid_pv(uuidparam=serverUUID());
+select id from uuid_pv(uuidparam=toUUID('11111111-2222-3333-4444-555555555555'));
+select id from uuid_pv(uuidparam='11111111-2222-3333-4444-555555555555');
+select id from uuid_pv(uuidparam=(select uu from uuid_table_pv where id = 1));
+select id from uuid_pv(uuidparam=(select uu from uuid_table_pv where id = 2));
+-- generateUUIDv4() is not constant foldable, hence cannot be used as parameter value
+select id from uuid_pv(uuidparam=generateUUIDv4()); -- { serverError UNKNOWN_QUERY_PARAMETER }
+-- But nested "select generateUUIDv4()"  works!
+select id from uuid_pv(uuidparam=(select generateUUIDv4()));
+
+select 'Test with 2 parameters';
+
+drop view if exists date_pv2;
+create view date_pv2 as select * from date_table_pv where dt = {dtparam:Date} and id = {intparam:Int32};
+select id from date_pv2(dtparam=today(),intparam=1);
+select id from date_pv2(dtparam=today(),intparam=length('A'));
+select id from date_pv2(dtparam='1974-04-07',intparam=length('AAA'));
+select id from date_pv2(dtparam=toDate('1974-04-07'),intparam=length('BBB'));
+
+select 'Test with IPv4';
+
+drop table if exists ipv4_table_pv;
+create table ipv4_table_pv (id Int32, ipaddr IPv4) ENGINE = Memory();
+insert into ipv4_table_pv values (1, '116.106.34.242');
+insert into ipv4_table_pv values (2, '116.106.34.243');
+insert into ipv4_table_pv values (3, '116.106.34.244');
+
+drop view if exists ipv4_pv;
+create view ipv4_pv as select * from ipv4_table_pv where ipaddr = {ipv4param:IPv4};
+select id from ipv4_pv(ipv4param='116.106.34.242');
+select id from ipv4_pv(ipv4param=toIPv4('116.106.34.243'));
+select id from ipv4_pv(ipv4param=(select ipaddr from ipv4_table_pv where id=3));
+
+drop view date_pv;
+drop view date_pv2;
+drop view uuid_pv;
+drop view ipv4_pv;
+drop table date_table_pv;
+drop table uuid_table_pv;
+drop table ipv4_table_pv;

From dde274f6fad979aa94ea31395b0434c81f72328a Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Wed, 31 Jul 2024 18:08:14 +0000
Subject: [PATCH 1139/1488] Re-enable ICU on s390/x

---
 contrib/icu-cmake/CMakeLists.txt | 4 +---
 contrib/icudata                  | 2 +-
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/contrib/icu-cmake/CMakeLists.txt b/contrib/icu-cmake/CMakeLists.txt
index f9d05f7fe97..adeaa7dcf33 100644
--- a/contrib/icu-cmake/CMakeLists.txt
+++ b/contrib/icu-cmake/CMakeLists.txt
@@ -4,9 +4,7 @@ else ()
     option(ENABLE_ICU "Enable ICU" 0)
 endif ()
 
-# Temporarily disabled s390x because the ICU build links a blob (icudt71b_dat.S) and our friends from IBM did not explain how they generated
-# the blob on s390x: https://github.com/ClickHouse/icudata/pull/2#issuecomment-2226957255
-if (NOT ENABLE_ICU OR ARCH_S390X)
+if (NOT ENABLE_ICU)
     message(STATUS "Not using ICU")
     return()
 endif()
diff --git a/contrib/icudata b/contrib/icudata
index d345d6ac22f..4904951339a 160000
--- a/contrib/icudata
+++ b/contrib/icudata
@@ -1 +1 @@
-Subproject commit d345d6ac22f381c882420de9053d30ae1ff38d75
+Subproject commit 4904951339a70b4814d2d3723436b20d079cb01b

From e2af1766eb1ea0ee2f6b862f53a0d3c13f53365b Mon Sep 17 00:00:00 2001
From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com>
Date: Wed, 31 Jul 2024 20:25:28 +0200
Subject: [PATCH 1140/1488] init

---
 src/Functions/DateTimeTransforms.h | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/Functions/DateTimeTransforms.h b/src/Functions/DateTimeTransforms.h
index a7bd398cdaa..fe26c5cf353 100644
--- a/src/Functions/DateTimeTransforms.h
+++ b/src/Functions/DateTimeTransforms.h
@@ -381,11 +381,13 @@ struct ToStartOfWeekImpl
 
     static UInt16 execute(Int64 t, UInt8 week_mode, const DateLUTImpl & time_zone)
     {
-        return time_zone.toFirstDayNumOfWeek(time_zone.toDayNum(t), week_mode);
+        const auto & res = time_zone.toFirstDayNumOfWeek(time_zone.toDayNum(t), week_mode);
+        return res >= 0 ? res : 0;
     }
     static UInt16 execute(UInt32 t, UInt8 week_mode, const DateLUTImpl & time_zone)
     {
-        return time_zone.toFirstDayNumOfWeek(time_zone.toDayNum(t), week_mode);
+        const auto & res = time_zone.toFirstDayNumOfWeek(time_zone.toDayNum(t), week_mode);
+        return res >= 0 ? res : 0;
     }
     static UInt16 execute(Int32 d, UInt8 week_mode, const DateLUTImpl & time_zone)
     {

From d0c643180f408d84a5f10a917f413248b9267202 Mon Sep 17 00:00:00 2001
From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com>
Date: Wed, 31 Jul 2024 20:29:36 +0200
Subject: [PATCH 1141/1488] add tests

---
 .../03215_toStartOfWeek_with_dateTime64_fix.reference           | 2 ++
 .../0_stateless/03215_toStartOfWeek_with_dateTime64_fix.sql     | 2 ++
 2 files changed, 4 insertions(+)
 create mode 100644 tests/queries/0_stateless/03215_toStartOfWeek_with_dateTime64_fix.reference
 create mode 100644 tests/queries/0_stateless/03215_toStartOfWeek_with_dateTime64_fix.sql

diff --git a/tests/queries/0_stateless/03215_toStartOfWeek_with_dateTime64_fix.reference b/tests/queries/0_stateless/03215_toStartOfWeek_with_dateTime64_fix.reference
new file mode 100644
index 00000000000..fd698107f22
--- /dev/null
+++ b/tests/queries/0_stateless/03215_toStartOfWeek_with_dateTime64_fix.reference
@@ -0,0 +1,2 @@
+1970-01-01
+1970-01-01
diff --git a/tests/queries/0_stateless/03215_toStartOfWeek_with_dateTime64_fix.sql b/tests/queries/0_stateless/03215_toStartOfWeek_with_dateTime64_fix.sql
new file mode 100644
index 00000000000..0f00a52cb86
--- /dev/null
+++ b/tests/queries/0_stateless/03215_toStartOfWeek_with_dateTime64_fix.sql
@@ -0,0 +1,2 @@
+SELECT toStartOfWeek(toDateTime64('1970-02-01', 6));
+SELECT toStartOfWeek(toDateTime('1970-01-01'));

From 636c3f642340de6e5ca4892481ca156cb236a4cd Mon Sep 17 00:00:00 2001
From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com>
Date: Wed, 31 Jul 2024 20:31:22 +0200
Subject: [PATCH 1142/1488] Update DateTimeTransforms.h

---
 src/Functions/DateTimeTransforms.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Functions/DateTimeTransforms.h b/src/Functions/DateTimeTransforms.h
index fe26c5cf353..1970ec3bdb0 100644
--- a/src/Functions/DateTimeTransforms.h
+++ b/src/Functions/DateTimeTransforms.h
@@ -382,12 +382,12 @@ struct ToStartOfWeekImpl
     static UInt16 execute(Int64 t, UInt8 week_mode, const DateLUTImpl & time_zone)
     {
         const auto & res = time_zone.toFirstDayNumOfWeek(time_zone.toDayNum(t), week_mode);
-        return res >= 0 ? res : 0;
+        return std::max(res, 0);
     }
     static UInt16 execute(UInt32 t, UInt8 week_mode, const DateLUTImpl & time_zone)
     {
         const auto & res = time_zone.toFirstDayNumOfWeek(time_zone.toDayNum(t), week_mode);
-        return res >= 0 ? res : 0;
+        return std::max(res, 0);
     }
     static UInt16 execute(Int32 d, UInt8 week_mode, const DateLUTImpl & time_zone)
     {

From 34ca3128ed0de60e03a105c43dc0924541f4a2c1 Mon Sep 17 00:00:00 2001
From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com>
Date: Wed, 31 Jul 2024 20:36:13 +0200
Subject: [PATCH 1143/1488] Update DateTimeTransforms.h

---
 src/Functions/DateTimeTransforms.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Functions/DateTimeTransforms.h b/src/Functions/DateTimeTransforms.h
index 1970ec3bdb0..46fb3bb9f57 100644
--- a/src/Functions/DateTimeTransforms.h
+++ b/src/Functions/DateTimeTransforms.h
@@ -381,12 +381,12 @@ struct ToStartOfWeekImpl
 
     static UInt16 execute(Int64 t, UInt8 week_mode, const DateLUTImpl & time_zone)
     {
-        const auto & res = time_zone.toFirstDayNumOfWeek(time_zone.toDayNum(t), week_mode);
+        const auto res = time_zone.toFirstDayNumOfWeek(time_zone.toDayNum(t), week_mode);
         return std::max(res, 0);
     }
     static UInt16 execute(UInt32 t, UInt8 week_mode, const DateLUTImpl & time_zone)
     {
-        const auto & res = time_zone.toFirstDayNumOfWeek(time_zone.toDayNum(t), week_mode);
+        const auto res = time_zone.toFirstDayNumOfWeek(time_zone.toDayNum(t), week_mode);
         return std::max(res, 0);
     }
     static UInt16 execute(Int32 d, UInt8 week_mode, const DateLUTImpl & time_zone)

From 9def8cea8121ae8001649629a96e73eb1e10159b Mon Sep 17 00:00:00 2001
From: robot-clickhouse <robot-clickhouse@users.noreply.github.com>
Date: Wed, 31 Jul 2024 18:57:08 +0000
Subject: [PATCH 1144/1488] Update version_date.tsv and changelogs after
 v24.4.4.107-stable

---
 docs/changelogs/v24.4.4.107-stable.md | 70 +++++++++++++++++++++++++++
 utils/list-versions/version_date.tsv  |  1 +
 2 files changed, 71 insertions(+)
 create mode 100644 docs/changelogs/v24.4.4.107-stable.md

diff --git a/docs/changelogs/v24.4.4.107-stable.md b/docs/changelogs/v24.4.4.107-stable.md
new file mode 100644
index 00000000000..ba7c576715e
--- /dev/null
+++ b/docs/changelogs/v24.4.4.107-stable.md
@@ -0,0 +1,70 @@
+---
+sidebar_position: 1
+sidebar_label: 2024
+---
+
+# 2024 Changelog
+
+### ClickHouse release v24.4.4.107-stable (af0ed6b197e) FIXME as compared to v24.4.3.25-stable (a915dd4eda4)
+
+#### Improvement
+* Backported in [#65884](https://github.com/ClickHouse/ClickHouse/issues/65884): Always start Keeper with sufficient amount of threads in global thread pool. [#64444](https://github.com/ClickHouse/ClickHouse/pull/64444) ([Duc Canh Le](https://github.com/canhld94)).
+* Backported in [#65303](https://github.com/ClickHouse/ClickHouse/issues/65303): Returned back the behaviour of how ClickHouse works and interprets Tuples in CSV format. This change effectively reverts https://github.com/ClickHouse/ClickHouse/pull/60994 and makes it available only under a few settings: `output_format_csv_serialize_tuple_into_separate_columns`, `input_format_csv_deserialize_separate_columns_into_tuple` and `input_format_csv_try_infer_strings_from_quoted_tuples`. [#65170](https://github.com/ClickHouse/ClickHouse/pull/65170) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Backported in [#65894](https://github.com/ClickHouse/ClickHouse/issues/65894): Respect cgroup CPU limit in Keeper. [#65819](https://github.com/ClickHouse/ClickHouse/pull/65819) ([Antonio Andelic](https://github.com/antonio2368)).
+
+#### Critical Bug Fix (crash, LOGICAL_ERROR, data loss, RBAC)
+* Backported in [#65372](https://github.com/ClickHouse/ClickHouse/issues/65372): Fix a bug in ClickHouse Keeper that causes digest mismatch during closing session. [#65198](https://github.com/ClickHouse/ClickHouse/pull/65198) ([Aleksei Filatov](https://github.com/aalexfvk)).
+* Backported in [#66883](https://github.com/ClickHouse/ClickHouse/issues/66883): Fix unexpeced size of low cardinality column in function calls. [#65298](https://github.com/ClickHouse/ClickHouse/pull/65298) ([Raúl Marín](https://github.com/Algunenano)).
+* Backported in [#65435](https://github.com/ClickHouse/ClickHouse/issues/65435): Forbid `QUALIFY` clause in the old analyzer. The old analyzer ignored `QUALIFY`, so it could lead to unexpected data removal in mutations. [#65356](https://github.com/ClickHouse/ClickHouse/pull/65356) ([Dmitry Novik](https://github.com/novikd)).
+* Backported in [#65448](https://github.com/ClickHouse/ClickHouse/issues/65448): Use correct memory alignment for Distinct combinator. Previously, crash could happen because of invalid memory allocation when the combinator was used. [#65379](https://github.com/ClickHouse/ClickHouse/pull/65379) ([Antonio Andelic](https://github.com/antonio2368)).
+* Backported in [#65710](https://github.com/ClickHouse/ClickHouse/issues/65710): Fix crash in maxIntersections. [#65689](https://github.com/ClickHouse/ClickHouse/pull/65689) ([Raúl Marín](https://github.com/Algunenano)).
+* Backported in [#66689](https://github.com/ClickHouse/ClickHouse/issues/66689): Fix the VALID UNTIL clause in the user definition resetting after a restart. Closes [#66405](https://github.com/ClickHouse/ClickHouse/issues/66405). [#66409](https://github.com/ClickHouse/ClickHouse/pull/66409) ([Nikolay Degterinsky](https://github.com/evillique)).
+
+#### Bug Fix (user-visible misbehavior in an official stable release)
+* Backported in [#65353](https://github.com/ClickHouse/ClickHouse/issues/65353): Fix possible abort on uncaught exception in ~WriteBufferFromFileDescriptor in StatusFile. [#64206](https://github.com/ClickHouse/ClickHouse/pull/64206) ([Kruglov Pavel](https://github.com/Avogar)).
+* Backported in [#65060](https://github.com/ClickHouse/ClickHouse/issues/65060): Fix the `Expression nodes list expected 1 projection names` and `Unknown expression or identifier` errors for queries with aliases to `GLOBAL IN.`. [#64517](https://github.com/ClickHouse/ClickHouse/pull/64517) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#65329](https://github.com/ClickHouse/ClickHouse/issues/65329): Fix the crash loop when restoring from backup is blocked by creating an MV with a definer that hasn't been restored yet. [#64595](https://github.com/ClickHouse/ClickHouse/pull/64595) ([pufit](https://github.com/pufit)).
+* Backported in [#64833](https://github.com/ClickHouse/ClickHouse/issues/64833): Fix bug which could lead to non-working TTLs with expressions. [#64694](https://github.com/ClickHouse/ClickHouse/pull/64694) ([alesapin](https://github.com/alesapin)).
+* Backported in [#65086](https://github.com/ClickHouse/ClickHouse/issues/65086): Fix removing the `WHERE` and `PREWHERE` expressions, which are always true (for the new analyzer). [#64695](https://github.com/ClickHouse/ClickHouse/pull/64695) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#65540](https://github.com/ClickHouse/ClickHouse/issues/65540): Fix crash for `ALTER TABLE ... ON CLUSTER ... MODIFY SQL SECURITY`. [#64957](https://github.com/ClickHouse/ClickHouse/pull/64957) ([pufit](https://github.com/pufit)).
+* Backported in [#65578](https://github.com/ClickHouse/ClickHouse/issues/65578): Fix crash on destroying AccessControl: add explicit shutdown. [#64993](https://github.com/ClickHouse/ClickHouse/pull/64993) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Backported in [#65161](https://github.com/ClickHouse/ClickHouse/issues/65161): Fix pushing arithmetic operations out of aggregation. In the new analyzer, optimization was applied only once. [#65104](https://github.com/ClickHouse/ClickHouse/pull/65104) ([Dmitry Novik](https://github.com/novikd)).
+* Backported in [#65616](https://github.com/ClickHouse/ClickHouse/issues/65616): Fix aggregate function name rewriting in the new analyzer. [#65110](https://github.com/ClickHouse/ClickHouse/pull/65110) ([Dmitry Novik](https://github.com/novikd)).
+* Backported in [#65730](https://github.com/ClickHouse/ClickHouse/issues/65730): Eliminate injective function in argument of functions `uniq*` recursively. This used to work correctly but was broken in the new analyzer. [#65140](https://github.com/ClickHouse/ClickHouse/pull/65140) ([Duc Canh Le](https://github.com/canhld94)).
+* Backported in [#65668](https://github.com/ClickHouse/ClickHouse/issues/65668): Disable `non-intersecting-parts` optimization for queries with `FINAL` in case of `read-in-order` optimization was enabled. This could lead to an incorrect query result. As a workaround, disable `do_not_merge_across_partitions_select_final` and `split_parts_ranges_into_intersecting_and_non_intersecting_final` before this fix is merged. [#65505](https://github.com/ClickHouse/ClickHouse/pull/65505) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#65786](https://github.com/ClickHouse/ClickHouse/issues/65786): Fixed bug in MergeJoin. Column in sparse serialisation might be treated as a column of its nested type though the required conversion wasn't performed. [#65632](https://github.com/ClickHouse/ClickHouse/pull/65632) ([Nikita Taranov](https://github.com/nickitat)).
+* Backported in [#65810](https://github.com/ClickHouse/ClickHouse/issues/65810): Fix invalid exceptions in function `parseDateTime` with `%F` and `%D` placeholders. [#65768](https://github.com/ClickHouse/ClickHouse/pull/65768) ([Antonio Andelic](https://github.com/antonio2368)).
+* Backported in [#65931](https://github.com/ClickHouse/ClickHouse/issues/65931): For queries that read from `PostgreSQL`, cancel the internal `PostgreSQL` query if the ClickHouse query is finished. Otherwise, `ClickHouse` query cannot be canceled until the internal `PostgreSQL` query is finished. [#65771](https://github.com/ClickHouse/ClickHouse/pull/65771) ([Maksim Kita](https://github.com/kitaisreal)).
+* Backported in [#65826](https://github.com/ClickHouse/ClickHouse/issues/65826): Fix a bug in short circuit logic when old analyzer and dictGetOrDefault is used. [#65802](https://github.com/ClickHouse/ClickHouse/pull/65802) ([jsc0218](https://github.com/jsc0218)).
+* Backported in [#66299](https://github.com/ClickHouse/ClickHouse/issues/66299): Better handling of join conditions involving `IS NULL` checks (for example `ON (a = b AND (a IS NOT NULL) AND (b IS NOT NULL) ) OR ( (a IS NULL) AND (b IS NULL) )` is rewritten to `ON a <=> b`), fix incorrect optimization when condition other then `IS NULL` are present. [#65835](https://github.com/ClickHouse/ClickHouse/pull/65835) ([vdimir](https://github.com/vdimir)).
+* Backported in [#66326](https://github.com/ClickHouse/ClickHouse/issues/66326): Add missing settings `input_format_csv_skip_first_lines/input_format_tsv_skip_first_lines/input_format_csv_try_infer_numbers_from_strings/input_format_csv_try_infer_strings_from_quoted_tuples` in schema inference cache because they can change the resulting schema. It prevents from incorrect result of schema inference with these settings changed. [#65980](https://github.com/ClickHouse/ClickHouse/pull/65980) ([Kruglov Pavel](https://github.com/Avogar)).
+* Backported in [#66153](https://github.com/ClickHouse/ClickHouse/issues/66153): Fixed buffer overflow bug in `unbin`/`unhex` implementation. [#66106](https://github.com/ClickHouse/ClickHouse/pull/66106) ([Nikita Taranov](https://github.com/nickitat)).
+* Backported in [#66459](https://github.com/ClickHouse/ClickHouse/issues/66459): Fixed a bug in ZooKeeper client: a session could get stuck in unusable state after receiving a hardware error from ZooKeeper. For example, this might happen due to "soft memory limit" in ClickHouse Keeper. [#66140](https://github.com/ClickHouse/ClickHouse/pull/66140) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Backported in [#66224](https://github.com/ClickHouse/ClickHouse/issues/66224): Fix issue in SumIfToCountIfVisitor and signed integers. [#66146](https://github.com/ClickHouse/ClickHouse/pull/66146) ([Raúl Marín](https://github.com/Algunenano)).
+* Backported in [#66267](https://github.com/ClickHouse/ClickHouse/issues/66267): Don't throw `TIMEOUT_EXCEEDED` for `none_only_active` mode of `distributed_ddl_output_mode`. [#66218](https://github.com/ClickHouse/ClickHouse/pull/66218) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Backported in [#66678](https://github.com/ClickHouse/ClickHouse/issues/66678): Fix handling limit for `system.numbers_mt` when no index can be used. [#66231](https://github.com/ClickHouse/ClickHouse/pull/66231) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
+* Backported in [#66603](https://github.com/ClickHouse/ClickHouse/issues/66603): Fixed how the ClickHouse server detects the maximum number of usable CPU cores as specified by cgroups v2 if the server runs in a container such as Docker. In more detail, containers often run their process in the root cgroup which has an empty name. In that case, ClickHouse ignored the CPU limits set by cgroups v2. [#66237](https://github.com/ClickHouse/ClickHouse/pull/66237) ([filimonov](https://github.com/filimonov)).
+* Backported in [#66358](https://github.com/ClickHouse/ClickHouse/issues/66358): Fix the `Not-ready set` error when a subquery with `IN` is used in the constraint. [#66261](https://github.com/ClickHouse/ClickHouse/pull/66261) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#66971](https://github.com/ClickHouse/ClickHouse/issues/66971): Fix `Column identifier is already registered` error with `group_by_use_nulls=true` and new analyzer. [#66400](https://github.com/ClickHouse/ClickHouse/pull/66400) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#66968](https://github.com/ClickHouse/ClickHouse/issues/66968): Fix `Cannot find column` error for queries with constant expression in `GROUP BY` key and new analyzer enabled. [#66433](https://github.com/ClickHouse/ClickHouse/pull/66433) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#66719](https://github.com/ClickHouse/ClickHouse/issues/66719): Correctly track memory for `Allocator::realloc`. [#66548](https://github.com/ClickHouse/ClickHouse/pull/66548) ([Antonio Andelic](https://github.com/antonio2368)).
+* Backported in [#66950](https://github.com/ClickHouse/ClickHouse/issues/66950): Fix an invalid result for queries with `WINDOW`. This could happen when `PARTITION` columns have sparse serialization and window functions are executed in parallel. [#66579](https://github.com/ClickHouse/ClickHouse/pull/66579) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#66947](https://github.com/ClickHouse/ClickHouse/issues/66947): Fix `Method getResultType is not supported for QUERY query node` error when scalar subquery was used as the first argument of IN (with new analyzer). [#66655](https://github.com/ClickHouse/ClickHouse/pull/66655) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#67195](https://github.com/ClickHouse/ClickHouse/issues/67195): TRUNCATE DATABASE used to stop replication as if it was a DROP DATABASE query, it's fixed. [#67129](https://github.com/ClickHouse/ClickHouse/pull/67129) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Backported in [#67377](https://github.com/ClickHouse/ClickHouse/issues/67377): Fix error `Cannot convert column because it is non constant in source stream but must be constant in result.` for a query that reads from the `Merge` table over the `Distriburted` table with one shard. [#67146](https://github.com/ClickHouse/ClickHouse/pull/67146) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#67240](https://github.com/ClickHouse/ClickHouse/issues/67240): This closes [#67156](https://github.com/ClickHouse/ClickHouse/issues/67156). This closes [#66447](https://github.com/ClickHouse/ClickHouse/issues/66447). The bug was introduced in https://github.com/ClickHouse/ClickHouse/pull/62907. [#67178](https://github.com/ClickHouse/ClickHouse/pull/67178) ([Maksim Kita](https://github.com/kitaisreal)).
+
+#### NOT FOR CHANGELOG / INSIGNIFICANT
+
+* Backported in [#65410](https://github.com/ClickHouse/ClickHouse/issues/65410): Re-enable OpenSSL session caching. [#65111](https://github.com/ClickHouse/ClickHouse/pull/65111) ([Robert Schulze](https://github.com/rschu1ze)).
+* Backported in [#65903](https://github.com/ClickHouse/ClickHouse/issues/65903): Fix bug with session closing in Keeper. [#65735](https://github.com/ClickHouse/ClickHouse/pull/65735) ([Antonio Andelic](https://github.com/antonio2368)).
+* Backported in [#66385](https://github.com/ClickHouse/ClickHouse/issues/66385): Disable broken cases from 02911_join_on_nullsafe_optimization. [#66310](https://github.com/ClickHouse/ClickHouse/pull/66310) ([vdimir](https://github.com/vdimir)).
+* Backported in [#66424](https://github.com/ClickHouse/ClickHouse/issues/66424): Ignore subquery for IN in DDLLoadingDependencyVisitor. [#66395](https://github.com/ClickHouse/ClickHouse/pull/66395) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#66542](https://github.com/ClickHouse/ClickHouse/issues/66542): Add additional log masking in CI. [#66523](https://github.com/ClickHouse/ClickHouse/pull/66523) ([Raúl Marín](https://github.com/Algunenano)).
+* Backported in [#66857](https://github.com/ClickHouse/ClickHouse/issues/66857): Fix data race in S3::ClientCache. [#66644](https://github.com/ClickHouse/ClickHouse/pull/66644) ([Konstantin Morozov](https://github.com/k-morozov)).
+* Backported in [#66873](https://github.com/ClickHouse/ClickHouse/issues/66873): Support one more case in JOIN ON ... IS NULL. [#66725](https://github.com/ClickHouse/ClickHouse/pull/66725) ([vdimir](https://github.com/vdimir)).
+* Backported in [#67057](https://github.com/ClickHouse/ClickHouse/issues/67057): Increase asio pool size in case the server is tiny. [#66761](https://github.com/ClickHouse/ClickHouse/pull/66761) ([alesapin](https://github.com/alesapin)).
+* Backported in [#66944](https://github.com/ClickHouse/ClickHouse/issues/66944): Small fix in realloc memory tracking. [#66820](https://github.com/ClickHouse/ClickHouse/pull/66820) ([Antonio Andelic](https://github.com/antonio2368)).
+* Backported in [#67250](https://github.com/ClickHouse/ClickHouse/issues/67250): Followup [#66725](https://github.com/ClickHouse/ClickHouse/issues/66725). [#66869](https://github.com/ClickHouse/ClickHouse/pull/66869) ([vdimir](https://github.com/vdimir)).
+* Backported in [#67410](https://github.com/ClickHouse/ClickHouse/issues/67410): CI: Fix build results for release branches. [#67402](https://github.com/ClickHouse/ClickHouse/pull/67402) ([Max K.](https://github.com/maxknv)).
+
diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv
index 027b207d3ad..abd8f84ec74 100644
--- a/utils/list-versions/version_date.tsv
+++ b/utils/list-versions/version_date.tsv
@@ -5,6 +5,7 @@ v24.5.4.49-stable	2024-07-01
 v24.5.3.5-stable	2024-06-13
 v24.5.2.34-stable	2024-06-13
 v24.5.1.1763-stable	2024-06-01
+v24.4.4.107-stable	2024-07-31
 v24.4.3.25-stable	2024-06-14
 v24.4.2.141-stable	2024-06-07
 v24.4.1.2088-stable	2024-05-01

From cd3c6c3ae807a321d94079087eb1fb29f4764549 Mon Sep 17 00:00:00 2001
From: sakulali <sakulali.dev@gmail.com>
Date: Thu, 1 Aug 2024 03:37:45 +0800
Subject: [PATCH 1145/1488] try to fix flaky test
 01889_clickhouse_client_config_format

---
 .../01889_clickhouse_client_config_format.reference         | 2 +-
 .../0_stateless/01889_clickhouse_client_config_format.sh    | 6 ++++--
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/tests/queries/0_stateless/01889_clickhouse_client_config_format.reference b/tests/queries/0_stateless/01889_clickhouse_client_config_format.reference
index ef0d9ffc538..2575200e6fa 100644
--- a/tests/queries/0_stateless/01889_clickhouse_client_config_format.reference
+++ b/tests/queries/0_stateless/01889_clickhouse_client_config_format.reference
@@ -19,6 +19,6 @@ autodetect xml (non leading whitespaces)
 autodetect yaml
 2
 autodetect invalid xml
-Code: 1000, e.code() = 0, SAXParseException: Invalid token in '/config_test.badxml', line 2 column 12, Stack trace (when copying this message, always include the lines below):
+Correct: invalid xml parsed with exception
 autodetect invalid yaml
 Code: 585. Unable to parse YAML configuration file /config_test.badyaml, yaml-cpp: error at line 2, column 12: illegal map value. (CANNOT_PARSE_YAML)
diff --git a/tests/queries/0_stateless/01889_clickhouse_client_config_format.sh b/tests/queries/0_stateless/01889_clickhouse_client_config_format.sh
index 9a44ec0d5f5..58fd6852116 100755
--- a/tests/queries/0_stateless/01889_clickhouse_client_config_format.sh
+++ b/tests/queries/0_stateless/01889_clickhouse_client_config_format.sh
@@ -108,7 +108,9 @@ echo 'autodetect xml (non leading whitespaces)'
 $CLICKHOUSE_CLIENT --config "$autodetect_xml_non_leading_whitespace_config" -q "select getSetting('max_threads')"
 echo 'autodetect yaml'
 $CLICKHOUSE_CLIENT --config "$autodetect_yaml_config" -q "select getSetting('max_threads')"
+
+# Error code is 1000 (Poco::Exception). It is not ignored.
 echo 'autodetect invalid xml'
-$CLICKHOUSE_CLIENT --config "$autodetect_invalid_xml_config" -q "select getSetting('max_threads')" 2>&1 |& sed -n '1p' | sed -e "s#$CLICKHOUSE_TMP##" -e "s#Poco::Exception. ##"
+$CLICKHOUSE_CLIENT --config "$autodetect_invalid_xml_config" -q "select getSetting('max_threads')" 2>&1 |& grep -q "Code: 1000" && echo "Correct: invalid xml parsed with exception" || echo 'Fail: expected error code 1000 but got other'
 echo 'autodetect invalid yaml'
-$CLICKHOUSE_CLIENT --config "$autodetect_invalid_yaml_config" -q "select getSetting('max_threads')" 2>&1 |& sed -n '1p' | sed -e "s#$CLICKHOUSE_TMP##" -e "s#DB::Exception: ##"
\ No newline at end of file
+$CLICKHOUSE_CLIENT --config "$autodetect_invalid_yaml_config" -q "select getSetting('max_threads')" 2>&1 |& sed -e "s#$CLICKHOUSE_TMP##" -e "s#DB::Exception: ##"
\ No newline at end of file

From 89ca6aee4ea23ed9cb9b36bb4fa9a6490efe576a Mon Sep 17 00:00:00 2001
From: Nikolay Degterinsky <evillique@gmail.com>
Date: Wed, 31 Jul 2024 20:00:37 +0000
Subject: [PATCH 1146/1488] Check for timeout when we stop logs replication

---
 docker/test/base/setup_export_logs.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docker/test/base/setup_export_logs.sh b/docker/test/base/setup_export_logs.sh
index 0c869a95db2..db141bcc55e 100755
--- a/docker/test/base/setup_export_logs.sh
+++ b/docker/test/base/setup_export_logs.sh
@@ -215,7 +215,8 @@ function setup_logs_replication
 function stop_logs_replication
 {
     echo "Detach all logs replication"
-    clickhouse-client --query "select database||'.'||table from system.tables where database = 'system' and (table like '%_sender' or table like '%_watcher')" | {
+    timeout --preserve-status --signal TERM --kill-after 10m 20m \
+        clickhouse-client --query "select database||'.'||table from system.tables where database = 'system' and (table like '%_sender' or table like '%_watcher')" | {
         tee /dev/stderr
     } | {
         xargs -n1 -r -i clickhouse-client --query "drop table {}"

From cc27c254abd4b6fd8f64b47e0bdf6195041bd5ef Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 31 Jul 2024 17:38:20 +0200
Subject: [PATCH 1147/1488] Minor change

---
 src/Databases/DatabaseOnDisk.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp
index f419f5811a1..734f354d9a5 100644
--- a/src/Databases/DatabaseOnDisk.cpp
+++ b/src/Databases/DatabaseOnDisk.cpp
@@ -313,7 +313,7 @@ void DatabaseOnDisk::detachTablePermanently(ContextPtr query_context, const Stri
         std::lock_guard lock(mutex);
         if (const auto it = snapshot_detached_tables.find(table_name); it == snapshot_detached_tables.end())
         {
-            throw Exception(ErrorCodes::LOGICAL_ERROR, "Snapshot doesn't contain info about detached table={}", table_name);
+            throw Exception(ErrorCodes::LOGICAL_ERROR, "Snapshot doesn't contain info about detached table `{}`", table_name);
         }
         else
         {

From 867784d55c989943d0c79eb9179b01e878fabcbe Mon Sep 17 00:00:00 2001
From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com>
Date: Wed, 31 Jul 2024 22:48:16 +0200
Subject: [PATCH 1148/1488] Update DateTimeTransforms.h

---
 src/Functions/DateTimeTransforms.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/Functions/DateTimeTransforms.h b/src/Functions/DateTimeTransforms.h
index 46fb3bb9f57..ce7da406e9a 100644
--- a/src/Functions/DateTimeTransforms.h
+++ b/src/Functions/DateTimeTransforms.h
@@ -1,5 +1,6 @@
 #pragma once
 
+#include <algorithm>
 #include <base/types.h>
 #include <Core/DecimalFunctions.h>
 #include <Common/Exception.h>

From 9cb52bd1381ad3e0929062801df7c4b542cf1117 Mon Sep 17 00:00:00 2001
From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com>
Date: Wed, 31 Jul 2024 23:11:35 +0200
Subject: [PATCH 1149/1488] fix build

---
 src/Functions/DateTimeTransforms.h | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/src/Functions/DateTimeTransforms.h b/src/Functions/DateTimeTransforms.h
index ce7da406e9a..15f1b9580f3 100644
--- a/src/Functions/DateTimeTransforms.h
+++ b/src/Functions/DateTimeTransforms.h
@@ -1,6 +1,5 @@
 #pragma once
 
-#include <algorithm>
 #include <base/types.h>
 #include <Core/DecimalFunctions.h>
 #include <Common/Exception.h>
@@ -25,7 +24,7 @@ namespace DB
 
 static constexpr auto millisecond_multiplier = 1'000;
 static constexpr auto microsecond_multiplier = 1'000'000;
-static constexpr auto nanosecond_multiplier  = 1'000'000'000;
+static constexpr auto nanosecond_multiplier = 1'000'000'000;
 
 static constexpr FormatSettings::DateTimeOverflowBehavior default_date_time_overflow_behavior = FormatSettings::DateTimeOverflowBehavior::Ignore;
 
@@ -382,12 +381,12 @@ struct ToStartOfWeekImpl
 
     static UInt16 execute(Int64 t, UInt8 week_mode, const DateLUTImpl & time_zone)
     {
-        const auto res = time_zone.toFirstDayNumOfWeek(time_zone.toDayNum(t), week_mode);
+        const int res = time_zone.toFirstDayNumOfWeek(time_zone.toDayNum(t), week_mode);
         return std::max(res, 0);
     }
     static UInt16 execute(UInt32 t, UInt8 week_mode, const DateLUTImpl & time_zone)
     {
-        const auto res = time_zone.toFirstDayNumOfWeek(time_zone.toDayNum(t), week_mode);
+        const int res = time_zone.toFirstDayNumOfWeek(time_zone.toDayNum(t), week_mode);
         return std::max(res, 0);
     }
     static UInt16 execute(Int32 d, UInt8 week_mode, const DateLUTImpl & time_zone)

From 6403f3f545bee153ffaf4ce5bda6fcde33ef88d2 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 31 Jul 2024 23:29:54 +0200
Subject: [PATCH 1150/1488] Miscellaneous

---
 src/Common/Epoll.cpp | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/Common/Epoll.cpp b/src/Common/Epoll.cpp
index 49c86222cf0..ef7c6e143a0 100644
--- a/src/Common/Epoll.cpp
+++ b/src/Common/Epoll.cpp
@@ -19,7 +19,7 @@ Epoll::Epoll() : events_count(0)
 {
     epoll_fd = epoll_create1(0);
     if (epoll_fd == -1)
-        throw DB::ErrnoException(DB::ErrorCodes::EPOLL_ERROR, "Cannot open epoll descriptor");
+        throw ErrnoException(ErrorCodes::EPOLL_ERROR, "Cannot open epoll descriptor");
 }
 
 Epoll::Epoll(Epoll && other) noexcept : epoll_fd(other.epoll_fd), events_count(other.events_count.load())
@@ -47,7 +47,7 @@ void Epoll::add(int fd, void * ptr, uint32_t events)
     ++events_count;
 
     if (epoll_ctl(epoll_fd, EPOLL_CTL_ADD, fd, &event) == -1)
-        throw DB::ErrnoException(DB::ErrorCodes::EPOLL_ERROR, "Cannot add new descriptor to epoll");
+        throw ErrnoException(ErrorCodes::EPOLL_ERROR, "Cannot add new descriptor to epoll");
 }
 
 void Epoll::remove(int fd)
@@ -55,7 +55,7 @@ void Epoll::remove(int fd)
     --events_count;
 
     if (epoll_ctl(epoll_fd, EPOLL_CTL_DEL, fd, nullptr) == -1)
-        throw DB::ErrnoException(DB::ErrorCodes::EPOLL_ERROR, "Cannot remove descriptor from epoll");
+        throw ErrnoException(ErrorCodes::EPOLL_ERROR, "Cannot remove descriptor from epoll");
 }
 
 size_t Epoll::getManyReady(int max_events, epoll_event * events_out, int timeout) const
@@ -82,7 +82,7 @@ size_t Epoll::getManyReady(int max_events, epoll_event * events_out, int timeout
                 continue;
             }
             else
-                throw DB::ErrnoException(DB::ErrorCodes::EPOLL_ERROR, "Error in epoll_wait");
+                throw ErrnoException(ErrorCodes::EPOLL_ERROR, "Error in epoll_wait");
         }
         else
             break;

From 6e914ff6da67be1c1381ffed2d04b5758704baf3 Mon Sep 17 00:00:00 2001
From: Thom O'Connor <thom@clickhouse.com>
Date: Wed, 31 Jul 2024 21:59:37 +0000
Subject: [PATCH 1151/1488] Update settings.md

Removing duplicate header "## background_merges_mutations_scheduling_policy"
---
 docs/en/operations/server-configuration-parameters/settings.md | 2 --
 1 file changed, 2 deletions(-)

diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md
index 8278f8c8699..a1e3c292b04 100644
--- a/docs/en/operations/server-configuration-parameters/settings.md
+++ b/docs/en/operations/server-configuration-parameters/settings.md
@@ -103,8 +103,6 @@ Default: 2
 
 The policy on how to perform a scheduling for background merges and mutations. Possible values are: `round_robin` and `shortest_task_first`.
 
-## background_merges_mutations_scheduling_policy
-
 Algorithm used to select next merge or mutation to be executed by background thread pool. Policy may be changed at runtime without server restart.
 Could be applied from the `default` profile for backward compatibility.
 

From 0e36db543762cb146aa6c233d4536fd62c6101b2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?=
 <benjamin.antal@clickhouse.com>
Date: Wed, 31 Jul 2024 22:09:04 +0000
Subject: [PATCH 1152/1488] Do not check the value of `num_messages_read`
 because it is not reliable

librdkafka usually polls both messages on the first SELECT, but sometimes only one. If it polls only one message first, then it will read two messages before rebalancing at the second SELECT from the table. This means it usually reads a single message twice (thus num_messages_read = 4 is usually fine as 1 discarded message + 3 actually consumed message). But when only one message is read in the first SELECT, then 2 messages are discarded, thus num_messages_read will be 5 as 2 discarded message + 3 actually consumed messages.
---
 tests/integration/test_storage_kafka/test.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/tests/integration/test_storage_kafka/test.py b/tests/integration/test_storage_kafka/test.py
index 8393e88db88..37457e00701 100644
--- a/tests/integration/test_storage_kafka/test.py
+++ b/tests/integration/test_storage_kafka/test.py
@@ -4771,7 +4771,7 @@ def test_system_kafka_consumers_rebalance(kafka_cluster, max_retries=15):
           assignments.current_offset,
           if(length(exceptions.time)>0, exceptions.time[1]::String, 'never') as last_exception_time_,
           if(length(exceptions.text)>0, exceptions.text[1], 'no exception') as last_exception_,
-          stable_timestamp(last_poll_time) as last_poll_time_, num_messages_read, stable_timestamp(last_commit_time) as last_commit_time_,
+          stable_timestamp(last_poll_time) as last_poll_time_, stable_timestamp(last_commit_time) as last_commit_time_,
           num_commits, stable_timestamp(last_rebalance_time) as last_rebalance_time_,
           num_rebalance_revocations, num_rebalance_assignments, is_currently_used
           FROM system.kafka_consumers WHERE database='test' and table IN ('kafka', 'kafka2') format Vertical;
@@ -4791,7 +4791,6 @@ assignments.current_offset: [2]
 last_exception_time_:       never
 last_exception_:            no exception
 last_poll_time_:            now
-num_messages_read:          4
 last_commit_time_:          now
 num_commits:                2
 last_rebalance_time_:       now
@@ -4810,7 +4809,6 @@ assignments.current_offset: [2]
 last_exception_time_:       never
 last_exception_:            no exception
 last_poll_time_:            now
-num_messages_read:          1
 last_commit_time_:          now
 num_commits:                1
 last_rebalance_time_:       never

From a592ad3624dedd3080b95422151b17bb92f3c10a Mon Sep 17 00:00:00 2001
From: jsc0218 <jsc0218@gmail.com>
Date: Wed, 31 Jul 2024 23:06:53 +0000
Subject: [PATCH 1153/1488] change error code

---
 src/Storages/MergeTree/MergeTreeData.cpp           |  5 +++--
 src/Storages/StorageFactory.cpp                    |  7 ++++---
 src/Storages/StorageMergeTree.cpp                  |  2 +-
 src/Storages/StorageReplicatedMergeTree.cpp        |  2 +-
 .../0_stateless/03174_projection_deduplicate.sql   |  2 +-
 .../03206_projection_merge_special_mergetree.sql   | 14 +++++++-------
 6 files changed, 17 insertions(+), 15 deletions(-)

diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index 677c4a92cda..7a0980a0e3b 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -3216,9 +3216,10 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, Context
     {
         if (auto storage_name = getName(); storage_name != "MergeTree" && storage_name != "ReplicatedMergeTree"
             && settings_from_storage->deduplicate_merge_projection_mode == DeduplicateMergeProjectionMode::THROW)
-            throw Exception(ErrorCodes::NOT_IMPLEMENTED,
+            throw Exception(ErrorCodes::SUPPORT_IS_DISABLED,
                 "Projection is fully supported in (Replictaed)MergeTree, but also allowed in non-throw mode with other"
-                " mergetree family members. Consider drop or rebuild option of deduplicate_merge_projection_mode.");
+                " mergetree family members. Consider drop or rebuild option of deduplicate_merge_projection_mode."
+                " Current storage name is {}.", storage_name);
     }
 
     commands.apply(new_metadata, local_context);
diff --git a/src/Storages/StorageFactory.cpp b/src/Storages/StorageFactory.cpp
index a059d624cd8..7360d351e8a 100644
--- a/src/Storages/StorageFactory.cpp
+++ b/src/Storages/StorageFactory.cpp
@@ -20,7 +20,7 @@ namespace ErrorCodes
     extern const int FUNCTION_CANNOT_HAVE_PARAMETERS;
     extern const int BAD_ARGUMENTS;
     extern const int DATA_TYPE_CANNOT_BE_USED_IN_TABLES;
-    extern const int NOT_IMPLEMENTED;
+    extern const int SUPPORT_IS_DISABLED;
 }
 
 
@@ -223,9 +223,10 @@ StoragePtr StorageFactory::get(
                         }
                     }
                     if (!projection_allowed)
-                        throw Exception(ErrorCodes::NOT_IMPLEMENTED,
+                        throw Exception(ErrorCodes::SUPPORT_IS_DISABLED,
                             "Projection is fully supported in (Replictaed)MergeTree, but also allowed in non-throw mode with other"
-                            " mergetree family members. Consider drop or rebuild option of deduplicate_merge_projection_mode.");
+                            " mergetree family members. Consider drop or rebuild option of deduplicate_merge_projection_mode."
+                            " Current storage name is {}.", engine_name);
                 }
             }
         }
diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp
index a5d434796ba..84393a3f1b0 100644
--- a/src/Storages/StorageMergeTree.cpp
+++ b/src/Storages/StorageMergeTree.cpp
@@ -1582,7 +1582,7 @@ bool StorageMergeTree::optimize(
 
     if (deduplicate && getInMemoryMetadataPtr()->hasProjections()
         && getSettings()->deduplicate_merge_projection_mode == DeduplicateMergeProjectionMode::THROW)
-        throw Exception(ErrorCodes::NOT_IMPLEMENTED,
+        throw Exception(ErrorCodes::SUPPORT_IS_DISABLED,
                     "OPTIMIZE DEDUPLICATE query is not supported for table {} as it has projections. "
                     "User should drop all the projections manually before running the query, "
                     "or consider drop or rebuild option of deduplicate_merge_projection_mode",
diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp
index 3751883df24..ad578242010 100644
--- a/src/Storages/StorageReplicatedMergeTree.cpp
+++ b/src/Storages/StorageReplicatedMergeTree.cpp
@@ -5796,7 +5796,7 @@ bool StorageReplicatedMergeTree::optimize(
 
     if (deduplicate && getInMemoryMetadataPtr()->hasProjections()
         && getSettings()->deduplicate_merge_projection_mode == DeduplicateMergeProjectionMode::THROW)
-        throw Exception(ErrorCodes::NOT_IMPLEMENTED,
+        throw Exception(ErrorCodes::SUPPORT_IS_DISABLED,
                     "OPTIMIZE DEDUPLICATE query is not supported for table {} as it has projections. "
                     "User should drop all the projections manually before running the query, "
                     "or consider drop or rebuild option of deduplicate_merge_projection_mode",
diff --git a/tests/queries/0_stateless/03174_projection_deduplicate.sql b/tests/queries/0_stateless/03174_projection_deduplicate.sql
index 46222b69dc7..f43f0a1f236 100644
--- a/tests/queries/0_stateless/03174_projection_deduplicate.sql
+++ b/tests/queries/0_stateless/03174_projection_deduplicate.sql
@@ -17,7 +17,7 @@ PRIMARY KEY id;
 INSERT INTO test_projection_deduplicate VALUES (1, 'one');
 INSERT INTO test_projection_deduplicate VALUES (1, 'one');
 
-OPTIMIZE TABLE test_projection_deduplicate DEDUPLICATE; -- { serverError NOT_IMPLEMENTED }
+OPTIMIZE TABLE test_projection_deduplicate DEDUPLICATE; -- { serverError SUPPORT_IS_DISABLED }
 
 SELECT * FROM test_projection_deduplicate;
 
diff --git a/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql b/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql
index e0a4f4f8cec..d3448138396 100644
--- a/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql
+++ b/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql
@@ -9,7 +9,7 @@ CREATE TABLE tp (
 
 INSERT INTO tp SELECT number%3, 1 FROM numbers(3);
 
-OPTIMIZE TABLE tp DEDUPLICATE;  -- { serverError NOT_IMPLEMENTED }
+OPTIMIZE TABLE tp DEDUPLICATE;  -- { serverError SUPPORT_IS_DISABLED }
 
 DROP TABLE tp;
 
@@ -26,7 +26,7 @@ OPTIMIZE TABLE tp DEDUPLICATE;
 
 ALTER TABLE tp MODIFY SETTING deduplicate_merge_projection_mode = 'throw';
 
-OPTIMIZE TABLE tp DEDUPLICATE;  -- { serverError NOT_IMPLEMENTED }
+OPTIMIZE TABLE tp DEDUPLICATE;  -- { serverError SUPPORT_IS_DISABLED }
 
 DROP TABLE tp;
 
@@ -36,14 +36,14 @@ CREATE TABLE tp (
     type Int32,
     eventcnt UInt64,
     PROJECTION p (select sum(eventcnt), type group by type)
-) engine = ReplacingMergeTree order by type;  -- { serverError NOT_IMPLEMENTED }
+) engine = ReplacingMergeTree order by type;  -- { serverError SUPPORT_IS_DISABLED }
 
 CREATE TABLE tp (
     type Int32,
     eventcnt UInt64,
     PROJECTION p (select sum(eventcnt), type group by type)
 ) engine = ReplacingMergeTree order by type
-SETTINGS deduplicate_merge_projection_mode = 'throw';  -- { serverError NOT_IMPLEMENTED }
+SETTINGS deduplicate_merge_projection_mode = 'throw';  -- { serverError SUPPORT_IS_DISABLED }
 
 CREATE TABLE tp (
     type Int32,
@@ -65,7 +65,7 @@ WHERE (database = currentDatabase()) AND (`table` = 'tp') AND (active = 1);
 
 ALTER TABLE tp MODIFY SETTING deduplicate_merge_projection_mode = 'throw';
 
-OPTIMIZE TABLE tp DEDUPLICATE;  -- { serverError NOT_IMPLEMENTED }
+OPTIMIZE TABLE tp DEDUPLICATE;  -- { serverError SUPPORT_IS_DISABLED }
 
 DROP TABLE tp;
 
@@ -78,7 +78,7 @@ SETTINGS deduplicate_merge_projection_mode = 'rebuild';
 
 ALTER TABLE tp MODIFY SETTING deduplicate_merge_projection_mode = 'throw';
 
-OPTIMIZE TABLE tp DEDUPLICATE;  -- { serverError NOT_IMPLEMENTED }
+OPTIMIZE TABLE tp DEDUPLICATE;  -- { serverError SUPPORT_IS_DISABLED }
 
 DROP TABLE tp;
 
@@ -88,7 +88,7 @@ CREATE TABLE tp (
     eventcnt UInt64
 ) engine = ReplacingMergeTree order by type;
 
-ALTER TABLE tp ADD PROJECTION p (SELECT sum(eventcnt), type GROUP BY type);  -- { serverError NOT_IMPLEMENTED }
+ALTER TABLE tp ADD PROJECTION p (SELECT sum(eventcnt), type GROUP BY type);  -- { serverError SUPPORT_IS_DISABLED }
 
 ALTER TABLE tp MODIFY SETTING deduplicate_merge_projection_mode = 'drop';
 

From 587d50380ad3d1cadc56bda0fa700e3441c16ab4 Mon Sep 17 00:00:00 2001
From: sakulali <sakulali.dev@gmail.com>
Date: Thu, 1 Aug 2024 09:12:00 +0800
Subject: [PATCH 1154/1488] fix clickhouse-test reference

---
 .../0_stateless/01889_clickhouse_client_config_format.reference | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/01889_clickhouse_client_config_format.reference b/tests/queries/0_stateless/01889_clickhouse_client_config_format.reference
index 2575200e6fa..149315ad9d5 100644
--- a/tests/queries/0_stateless/01889_clickhouse_client_config_format.reference
+++ b/tests/queries/0_stateless/01889_clickhouse_client_config_format.reference
@@ -21,4 +21,4 @@ autodetect yaml
 autodetect invalid xml
 Correct: invalid xml parsed with exception
 autodetect invalid yaml
-Code: 585. Unable to parse YAML configuration file /config_test.badyaml, yaml-cpp: error at line 2, column 12: illegal map value. (CANNOT_PARSE_YAML)
+Code: 585. Unable to parse YAML configuration file /config_default.badyaml, yaml-cpp: error at line 2, column 12: illegal map value. (CANNOT_PARSE_YAML)

From f162d6bd5e03c6f717b4f45cf4c7ba6491aaa5fa Mon Sep 17 00:00:00 2001
From: robot-clickhouse <robot-clickhouse@users.noreply.github.com>
Date: Thu, 1 Aug 2024 06:35:22 +0000
Subject: [PATCH 1155/1488] Update version_date.tsv and changelogs after
 v24.7.2.13-stable

---
 docker/keeper/Dockerfile             |  2 +-
 docker/server/Dockerfile.alpine      |  2 +-
 docker/server/Dockerfile.ubuntu      |  2 +-
 docs/changelogs/v24.7.2.13-stable.md | 24 ++++++++++++++++++++++++
 utils/list-versions/version_date.tsv |  2 +-
 5 files changed, 28 insertions(+), 4 deletions(-)
 create mode 100644 docs/changelogs/v24.7.2.13-stable.md

diff --git a/docker/keeper/Dockerfile b/docker/keeper/Dockerfile
index e99c86267f9..94603763572 100644
--- a/docker/keeper/Dockerfile
+++ b/docker/keeper/Dockerfile
@@ -34,7 +34,7 @@ RUN arch=${TARGETARCH:-amd64} \
 # lts / testing / prestable / etc
 ARG REPO_CHANNEL="stable"
 ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}"
-ARG VERSION="24.7.1.2915"
+ARG VERSION="24.7.2.13"
 ARG PACKAGES="clickhouse-keeper"
 ARG DIRECT_DOWNLOAD_URLS=""
 
diff --git a/docker/server/Dockerfile.alpine b/docker/server/Dockerfile.alpine
index fb562b911a3..f40118c7b06 100644
--- a/docker/server/Dockerfile.alpine
+++ b/docker/server/Dockerfile.alpine
@@ -32,7 +32,7 @@ RUN arch=${TARGETARCH:-amd64} \
 # lts / testing / prestable / etc
 ARG REPO_CHANNEL="stable"
 ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}"
-ARG VERSION="24.7.1.2915"
+ARG VERSION="24.7.2.13"
 ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
 ARG DIRECT_DOWNLOAD_URLS=""
 
diff --git a/docker/server/Dockerfile.ubuntu b/docker/server/Dockerfile.ubuntu
index 51f4e6a0f40..032aa862e4a 100644
--- a/docker/server/Dockerfile.ubuntu
+++ b/docker/server/Dockerfile.ubuntu
@@ -28,7 +28,7 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list
 
 ARG REPO_CHANNEL="stable"
 ARG REPOSITORY="deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb ${REPO_CHANNEL} main"
-ARG VERSION="24.7.1.2915"
+ARG VERSION="24.7.2.13"
 ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
 
 #docker-official-library:off
diff --git a/docs/changelogs/v24.7.2.13-stable.md b/docs/changelogs/v24.7.2.13-stable.md
new file mode 100644
index 00000000000..4a2fb665116
--- /dev/null
+++ b/docs/changelogs/v24.7.2.13-stable.md
@@ -0,0 +1,24 @@
+---
+sidebar_position: 1
+sidebar_label: 2024
+---
+
+# 2024 Changelog
+
+### ClickHouse release v24.7.2.13-stable (6e41f601b2f) FIXME as compared to v24.7.1.2915-stable (a37d2d43da7)
+
+#### Improvement
+* Backported in [#67531](https://github.com/ClickHouse/ClickHouse/issues/67531): In pr : https://github.com/ClickHouse/ClickHouse/pull/66025, we introduce a settings `input_format_orc_read_use_writer_time_zone` to fix when read orc file, make the reader use writer timezone, not always use `GMT`. [#67175](https://github.com/ClickHouse/ClickHouse/pull/67175) ([kevinyhzou](https://github.com/KevinyhZou)).
+
+#### Critical Bug Fix (crash, LOGICAL_ERROR, data loss, RBAC)
+* Backported in [#67505](https://github.com/ClickHouse/ClickHouse/issues/67505): Fix crash in DistributedAsyncInsert when connection is empty. [#67219](https://github.com/ClickHouse/ClickHouse/pull/67219) ([Pablo Marcos](https://github.com/pamarcos)).
+
+#### Bug Fix (user-visible misbehavior in an official stable release)
+* Backported in [#67580](https://github.com/ClickHouse/ClickHouse/issues/67580): Fix execution of nested short-circuit functions. [#67520](https://github.com/ClickHouse/ClickHouse/pull/67520) ([Kruglov Pavel](https://github.com/Avogar)).
+
+#### NOT FOR CHANGELOG / INSIGNIFICANT
+
+* Backported in [#67551](https://github.com/ClickHouse/ClickHouse/issues/67551): [Green CI] Fix test test_storage_s3_queue/test.py::test_max_set_age. [#67035](https://github.com/ClickHouse/ClickHouse/pull/67035) ([Pablo Marcos](https://github.com/pamarcos)).
+* Backported in [#67514](https://github.com/ClickHouse/ClickHouse/issues/67514): Split test 02967_parallel_replicas_join_algo_and_analyzer. [#67211](https://github.com/ClickHouse/ClickHouse/pull/67211) ([Nikita Taranov](https://github.com/nickitat)).
+* Backported in [#67545](https://github.com/ClickHouse/ClickHouse/issues/67545): [Green CI] Fix WriteBuffer destructor when finalize has failed for MergeTreeDeduplicationLog::shutdown. [#67474](https://github.com/ClickHouse/ClickHouse/pull/67474) ([Alexey Katsman](https://github.com/alexkats)).
+
diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv
index abd8f84ec74..b1391c2d781 100644
--- a/utils/list-versions/version_date.tsv
+++ b/utils/list-versions/version_date.tsv
@@ -1,3 +1,4 @@
+v24.7.2.13-stable	2024-08-01
 v24.7.1.2915-stable	2024-07-30
 v24.6.2.17-stable	2024-07-05
 v24.6.1.4423-stable	2024-07-01
@@ -5,7 +6,6 @@ v24.5.4.49-stable	2024-07-01
 v24.5.3.5-stable	2024-06-13
 v24.5.2.34-stable	2024-06-13
 v24.5.1.1763-stable	2024-06-01
-v24.4.4.107-stable	2024-07-31
 v24.4.3.25-stable	2024-06-14
 v24.4.2.141-stable	2024-06-07
 v24.4.1.2088-stable	2024-05-01

From c2df527a32d640f52296ea7aefae177e22504082 Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Thu, 1 Aug 2024 08:42:54 +0200
Subject: [PATCH 1156/1488] Reduce fault rate

---
 .../test_keeper_map_retries/configs/fault_injection.xml       | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/integration/test_keeper_map_retries/configs/fault_injection.xml b/tests/integration/test_keeper_map_retries/configs/fault_injection.xml
index 145945c7c7c..0933b6b3031 100644
--- a/tests/integration/test_keeper_map_retries/configs/fault_injection.xml
+++ b/tests/integration/test_keeper_map_retries/configs/fault_injection.xml
@@ -1,6 +1,6 @@
 <clickhouse>
     <zookeeper>
-        <send_fault_probability>0.05</send_fault_probability>
-        <recv_fault_probability>0.05</recv_fault_probability>
+        <send_fault_probability>0.005</send_fault_probability>
+        <recv_fault_probability>0.005</recv_fault_probability>
     </zookeeper>
 </clickhouse>

From 5564489cca1c14e95e7c543e03c508849abaf079 Mon Sep 17 00:00:00 2001
From: taiyang-li <654010905@qq.com>
Date: Thu, 1 Aug 2024 15:31:54 +0800
Subject: [PATCH 1157/1488] change as request

---
 .../functions/tuple-map-functions.md          | 18 +++++++++++---
 src/Functions/map.cpp                         | 24 ++++++++++---------
 .../0_stateless/01651_map_functions.reference |  4 ++++
 .../0_stateless/01651_map_functions.sql       |  7 +++++-
 4 files changed, 38 insertions(+), 15 deletions(-)

diff --git a/docs/en/sql-reference/functions/tuple-map-functions.md b/docs/en/sql-reference/functions/tuple-map-functions.md
index db66188b1f5..d670ed42a2a 100644
--- a/docs/en/sql-reference/functions/tuple-map-functions.md
+++ b/docs/en/sql-reference/functions/tuple-map-functions.md
@@ -43,7 +43,7 @@ Result:
 
 ## mapFromArrays
 
-Creates a map from an array of keys and an array of values.
+Creates a map from an array or map of keys and an array or map of values.
 
 The function is a convenient alternative to syntax `CAST([...], 'Map(key_type, value_type)')`.
 For example, instead of writing
@@ -62,8 +62,8 @@ Alias: `MAP_FROM_ARRAYS(keys, values)`
 
 **Arguments**
 
-- `keys` —  Array or map of keys to create the map from. [Array(T)](../data-types/array.md) where `T` can be any type supported by [Map](../data-types/map.md) as key type, or [Map](../data-types/map.md).
-- `values`  - Array or map of values to create the map from. [Array](../data-types/array.md) or [Map](../data-types/map.md).
+- `keys` —  Array or map of keys to create the map from [Array](../data-types/array.md) or [Map](../data-types/map.md). If `keys` is an array, we accept `Array(Nullable(T))` or `Array(LowCardinality(Nullable(T)))` as its type as long as it doesn't contain NULL value.
+- `values`  - Array or map of values to create the map from [Array](../data-types/array.md) or [Map](../data-types/map.md).
 
 **Returned value**
 
@@ -99,6 +99,18 @@ Result:
 └───────────────────────────────────────────────────────┘
 ```
 
+```sql
+SELECT mapFromArrays(map('a', 1, 'b', 2, 'c', 3), [1, 2, 3])
+```
+
+Result:
+
+```
+┌─mapFromArrays(map('a', 1, 'b', 2, 'c', 3), [1, 2, 3])─┐
+│ {('a',1):1,('b',2):2,('c',3):3}                       │
+└───────────────────────────────────────────────────────┘
+```
+
 ## extractKeyValuePairs
 
 Converts a string of key-value pairs to a [Map(String, String)](../data-types/map.md).
diff --git a/src/Functions/map.cpp b/src/Functions/map.cpp
index a8e5f7ad90e..738c61164a3 100644
--- a/src/Functions/map.cpp
+++ b/src/Functions/map.cpp
@@ -23,6 +23,7 @@ namespace ErrorCodes
     extern const int ILLEGAL_TYPE_OF_ARGUMENT;
     extern const int SIZES_OF_ARRAYS_DONT_MATCH;
     extern const int ILLEGAL_COLUMN;
+    extern const int BAD_ARGUMENTS;
 }
 
 namespace
@@ -157,7 +158,7 @@ private:
     bool use_variant_as_common_type = false;
 };
 
-/// mapFromArrays(keys, values) is a function that allows you to make key-value pair from a pair of arrays
+/// mapFromArrays(keys, values) is a function that allows you to make key-value pair from a pair of arrays or maps
 class FunctionMapFromArrays : public IFunction
 {
 public:
@@ -181,13 +182,13 @@ public:
                 getName(),
                 arguments.size());
 
-        auto get_nested_type = [this](const DataTypePtr & type) -> DataTypePtr
+        auto get_nested_type = [&](const DataTypePtr & type)
         {
             DataTypePtr nested;
-            if (const auto * array_type = checkAndGetDataType<DataTypeArray>(type.get()))
-                nested = array_type->getNestedType();
-            else if (const auto * map_type = checkAndGetDataType<DataTypeMap>(type.get()))
-                nested = std::make_shared<DataTypeTuple>(map_type->getKeyValueTypes());
+            if (const auto * type_as_array = checkAndGetDataType<DataTypeArray>(type.get()))
+                nested = type_as_array->getNestedType();
+            else if (const auto * type_as_map = checkAndGetDataType<DataTypeMap>(type.get()))
+                nested = std::make_shared<DataTypeTuple>(type_as_map->getKeyValueTypes());
             else
                 throw Exception(
                     ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
@@ -201,8 +202,9 @@ public:
         auto key_type = get_nested_type(arguments[0]);
         auto value_type = get_nested_type(arguments[1]);
 
-        /// Remove Nullable from key_type if needed for map key must not be Nullable
+        /// We accept Array(Nullable(T)) or Array(LowCardinality(Nullable(T))) as key types as long as the actual array doesn't contain NULL value(this is checked in executeImpl).
         key_type = removeNullableOrLowCardinalityNullable(key_type);
+
         DataTypes key_value_types{key_type, value_type};
         return std::make_shared<DataTypeMap>(key_value_types);
     }
@@ -210,7 +212,7 @@ public:
     ColumnPtr executeImpl(
         const ColumnsWithTypeAndName & arguments, const DataTypePtr & /* result_type */, size_t /* input_rows_count */) const override
     {
-        auto get_array_column = [this](const ColumnPtr & column) -> std::pair<const ColumnArray *, ColumnPtr>
+        auto get_array_column = [&](const ColumnPtr & column) -> std::pair<const ColumnArray *, ColumnPtr>
         {
             bool is_const = isColumnConst(*column);
             ColumnPtr holder = is_const ? column->convertToFullColumnIfConst() : column;
@@ -231,8 +233,9 @@ public:
         };
 
         auto [col_keys, key_holder] = get_array_column(arguments[0].column);
+        auto [col_values, values_holder] = get_array_column(arguments[1].column);
 
-        /// Check if nested column of first argument contains NULL value in case its nested type is Nullable(T) type.
+        /// Nullable(T) or LowCardinality(Nullable(T)) are okay as nested key types but actual NULL values are not okay.
         ColumnPtr data_keys = col_keys->getDataPtr();
         if (isColumnNullableOrLowCardinalityNullable(*data_keys))
         {
@@ -253,10 +256,9 @@ public:
 
             if (null_map && !memoryIsZero(null_map->data(), 0, null_map->size()))
                 throw Exception(
-                    ErrorCodes::ILLEGAL_COLUMN, "The nested column of first argument in function {} must not contain NULLs", getName());
+                    ErrorCodes::BAD_ARGUMENTS, "The nested column of first argument in function {} must not contain NULLs", getName());
         }
 
-        auto [col_values, values_holder] = get_array_column(arguments[1].column);
         if (!col_keys->hasEqualOffsets(*col_values))
             throw Exception(ErrorCodes::SIZES_OF_ARRAYS_DONT_MATCH, "Two arguments of function {} must have equal sizes", getName());
 
diff --git a/tests/queries/0_stateless/01651_map_functions.reference b/tests/queries/0_stateless/01651_map_functions.reference
index 9114aa419b1..e336a02318d 100644
--- a/tests/queries/0_stateless/01651_map_functions.reference
+++ b/tests/queries/0_stateless/01651_map_functions.reference
@@ -55,3 +55,7 @@
 {1:3,2:4}
 {1:3,2:4}
 {1:3,2:4}	{(1,3):'a',(2,4):'b'}
+{(1,'a'):'c',(2,'b'):'d'}
+{(1,'a'):'c',(2,'b'):'d'}
+{(1,'a'):'c',(2,'b'):'d'}
+{(1,'a'):'c',(2,'b'):'d'}
diff --git a/tests/queries/0_stateless/01651_map_functions.sql b/tests/queries/0_stateless/01651_map_functions.sql
index 4604ddd6db1..dc93a38b265 100644
--- a/tests/queries/0_stateless/01651_map_functions.sql
+++ b/tests/queries/0_stateless/01651_map_functions.sql
@@ -68,7 +68,7 @@ select mapFromArrays([[1,2], [3,4]], [4, 5, 6]); -- { serverError SIZES_OF_ARRAY
 select mapFromArrays(['a', 2], [4, 5]); -- { serverError NO_COMMON_TYPE}
 select mapFromArrays([1, 2], [4, 'a']); -- { serverError NO_COMMON_TYPE}
 select mapFromArrays(['aa', 'bb'], map('a', 4)); -- { serverError SIZES_OF_ARRAYS_DONT_MATCH }
-select mapFromArrays([1,null]::Array(Nullable(UInt8)), [3,4]); -- { serverError ILLEGAL_COLUMN }
+select mapFromArrays([1,null]::Array(Nullable(UInt8)), [3,4]); -- { serverError BAD_ARGUMENTS }
 
 select mapFromArrays(['aa', 'bb'], map('a', 4, 'b', 5));
 select mapFromArrays(['aa', 'bb'], materialize(map('a', 4, 'b', 5))) from numbers(2);
@@ -79,3 +79,8 @@ select mapFromArrays([toLowCardinality(1), toLowCardinality(2)], materialize([4,
 select mapFromArrays([1,2], [3,4]);
 select mapFromArrays([1,2]::Array(Nullable(UInt8)), [3,4]);
 select mapFromArrays([1,2], [3,4]) as x, mapFromArrays(x, ['a', 'b']);
+
+select mapFromArrays(map(1, 'a', 2, 'b'), array('c', 'd'));
+select mapFromArrays(materialize(map(1, 'a', 2, 'b')), array('c', 'd'));
+select mapFromArrays(map(1, 'a', 2, 'b'), materialize(array('c', 'd')));
+select mapFromArrays(materialize(map(1, 'a', 2, 'b')), materialize(array('c', 'd')));

From 681441e170202bc3963fb3fa1d7b7785192dbd2e Mon Sep 17 00:00:00 2001
From: taiyang-li <654010905@qq.com>
Date: Thu, 1 Aug 2024 16:01:39 +0800
Subject: [PATCH 1158/1488] fix style

---
 src/Functions/map.cpp | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/src/Functions/map.cpp b/src/Functions/map.cpp
index 738c61164a3..6e389f39dec 100644
--- a/src/Functions/map.cpp
+++ b/src/Functions/map.cpp
@@ -192,7 +192,7 @@ public:
             else
                 throw Exception(
                     ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
-                    "Argument types of function {} must be Array or Map, but {} is given",
+                    "Arguments of function {} must be Array or Map, but {} is given",
                     getName(),
                     type->getName());
 
@@ -275,10 +275,7 @@ public:
     static constexpr auto name = "mapUpdate";
     static FunctionPtr create(ContextPtr) { return std::make_shared<FunctionMapUpdate>(); }
 
-    String getName() const override
-    {
-        return name;
-    }
+    String getName() const override { return name; }
 
     size_t getNumberOfArguments() const override { return 2; }
 
@@ -287,9 +284,11 @@ public:
     DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
     {
         if (arguments.size() != 2)
-            throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
+            throw Exception(
+                ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
                 "Number of arguments for function {} doesn't match: passed {}, should be 2",
-                getName(), arguments.size());
+                getName(),
+                arguments.size());
 
         const auto * left = checkAndGetDataType<DataTypeMap>(arguments[0].type.get());
         const auto * right = checkAndGetDataType<DataTypeMap>(arguments[1].type.get());
@@ -405,7 +404,6 @@ public:
         return ColumnMap::create(nested_column);
     }
 };
-
 }
 
 REGISTER_FUNCTION(Map)

From 7db4065898633ace1f909711d4caeda8d135cace Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Thu, 1 Aug 2024 10:30:50 +0200
Subject: [PATCH 1159/1488] Add retries to create

---
 src/Storages/StorageKeeperMap.cpp             | 373 +++++++++++-------
 src/Storages/StorageKeeperMap.h               |   6 +-
 .../configs/keeper_retries.xml                |  14 +
 tests/integration/test_keeper_map/test.py     |   3 +-
 .../configs/fault_injection.xml               |   1 +
 .../configs/keeper_retries.xml                |  14 +
 .../test_keeper_map_retries/test.py           |  13 +-
 7 files changed, 275 insertions(+), 149 deletions(-)
 create mode 100644 tests/integration/test_keeper_map/configs/keeper_retries.xml
 create mode 100644 tests/integration/test_keeper_map_retries/configs/keeper_retries.xml

diff --git a/src/Storages/StorageKeeperMap.cpp b/src/Storages/StorageKeeperMap.cpp
index 0634c7be6ee..a6be9f8da04 100644
--- a/src/Storages/StorageKeeperMap.cpp
+++ b/src/Storages/StorageKeeperMap.cpp
@@ -408,104 +408,192 @@ StorageKeeperMap::StorageKeeperMap(
 
     if (attach)
     {
-        checkTable<false>();
+        checkTable<false>(context_);
         return;
     }
 
-    auto client = getClient();
+    const auto & settings = context_->getSettingsRef();
+    ZooKeeperRetriesControl zk_retry{
+        getName(),
+        getLogger(getName()),
+        ZooKeeperRetriesInfo{settings.keeper_max_retries, settings.keeper_retry_initial_backoff_ms, settings.keeper_retry_max_backoff_ms},
+        context_->getProcessListElement()};
 
-    if (zk_root_path != "/" && !client->exists(zk_root_path))
-    {
-        LOG_TRACE(log, "Creating root path {}", zk_root_path);
-        client->createAncestors(zk_root_path);
-        client->createIfNotExists(zk_root_path, "");
-    }
+    zk_retry.retryLoop(
+        [&]
+        {
+            auto client = getClient();
 
+            if (zk_root_path != "/" && !client->exists(zk_root_path))
+            {
+                LOG_TRACE(log, "Creating root path {}", zk_root_path);
+                client->createAncestors(zk_root_path);
+                client->createIfNotExists(zk_root_path, "");
+            }
+        });
+
+    std::shared_ptr<zkutil::EphemeralNodeHolder> metadata_drop_lock;
+    int32_t drop_lock_version = -1;
     for (size_t i = 0; i < 1000; ++i)
     {
-        std::string stored_metadata_string;
-        auto exists = client->tryGet(zk_metadata_path, stored_metadata_string);
-
-        if (exists)
-        {
-            // this requires same name for columns
-            // maybe we can do a smarter comparison for columns and primary key expression
-            if (stored_metadata_string != metadata_string)
-                throw Exception(
-                    ErrorCodes::BAD_ARGUMENTS,
-                    "Path {} is already used but the stored table definition doesn't match. Stored metadata: {}",
-                    zk_root_path,
-                    stored_metadata_string);
-
-            auto code = client->tryCreate(zk_table_path, "", zkutil::CreateMode::Persistent);
-
-            /// A table on the same Keeper path already exists, we just appended our table id to subscribe as a new replica
-            /// We still don't know if the table matches the expected metadata so table_is_valid is not changed
-            /// It will be checked lazily on the first operation
-            if (code == Coordination::Error::ZOK)
-                return;
-
-            if (code != Coordination::Error::ZNONODE)
-                throw zkutil::KeeperException(code, "Failed to create table on path {} because a table with same UUID already exists", zk_root_path);
-
-            /// ZNONODE means we dropped zk_tables_path but didn't finish drop completely
-        }
-
-        if (client->exists(zk_dropped_path))
-        {
-            LOG_INFO(log, "Removing leftover nodes");
-            auto code = client->tryCreate(zk_dropped_lock_path, "", zkutil::CreateMode::Ephemeral);
-
-            if (code == Coordination::Error::ZNONODE)
+        bool success = false;
+        zk_retry.retryLoop(
+            [&]
             {
-                LOG_INFO(log, "Someone else removed leftover nodes");
-            }
-            else if (code == Coordination::Error::ZNODEEXISTS)
-            {
-                LOG_INFO(log, "Someone else is removing leftover nodes");
-                continue;
-            }
-            else if (code != Coordination::Error::ZOK)
-            {
-                throw Coordination::Exception::fromPath(code, zk_dropped_lock_path);
-            }
-            else
-            {
-                auto metadata_drop_lock = zkutil::EphemeralNodeHolder::existing(zk_dropped_lock_path, *client);
-                if (!dropTable(client, metadata_drop_lock))
-                    continue;
-            }
-        }
+                auto client = getClient();
+                std::string stored_metadata_string;
+                auto exists = client->tryGet(zk_metadata_path, stored_metadata_string);
 
-        Coordination::Requests create_requests
-        {
-            zkutil::makeCreateRequest(zk_metadata_path, metadata_string, zkutil::CreateMode::Persistent),
-            zkutil::makeCreateRequest(zk_data_path, metadata_string, zkutil::CreateMode::Persistent),
-            zkutil::makeCreateRequest(zk_tables_path, "", zkutil::CreateMode::Persistent),
-            zkutil::makeCreateRequest(zk_table_path, "", zkutil::CreateMode::Persistent),
-        };
+                if (exists)
+                {
+                    // this requires same name for columns
+                    // maybe we can do a smarter comparison for columns and primary key expression
+                    if (stored_metadata_string != metadata_string)
+                        throw Exception(
+                            ErrorCodes::BAD_ARGUMENTS,
+                            "Path {} is already used but the stored table definition doesn't match. Stored metadata: {}",
+                            zk_root_path,
+                            stored_metadata_string);
 
-        Coordination::Responses create_responses;
-        auto code = client->tryMulti(create_requests, create_responses);
-        if (code == Coordination::Error::ZNODEEXISTS)
-        {
-            LOG_INFO(log, "It looks like a table on path {} was created by another server at the same moment, will retry", zk_root_path);
-            continue;
-        }
-        else if (code != Coordination::Error::ZOK)
-        {
-            zkutil::KeeperMultiException::check(code, create_requests, create_responses);
-        }
+                    auto code = client->tryCreate(zk_table_path, "", zkutil::CreateMode::Persistent);
 
+                    /// A table on the same Keeper path already exists, we just appended our table id to subscribe as a new replica
+                    /// We still don't know if the table matches the expected metadata so table_is_valid is not changed
+                    /// It will be checked lazily on the first operation
+                    if (code == Coordination::Error::ZOK)
+                    {
+                        success = true;
+                        return;
+                    }
 
-        table_status = TableStatus::VALID;
-        /// we are the first table created for the specified Keeper path, i.e. we are the first replica
-        return;
+                    /// We most likely created the path but got a timeout or disconnect
+                    if (code == Coordination::Error::ZNODEEXISTS && zk_retry.isRetry())
+                    {
+                        success = true;
+                        return;
+                    }
+
+                    if (code != Coordination::Error::ZNONODE)
+                        throw zkutil::KeeperException(
+                            code, "Failed to create table on path {} because a table with same UUID already exists", zk_root_path);
+
+                    /// ZNONODE means we dropped zk_tables_path but didn't finish drop completely
+                }
+
+                if (client->exists(zk_dropped_path))
+                {
+                    LOG_INFO(log, "Removing leftover nodes");
+
+                    bool drop_finished = false;
+                    if (zk_retry.isRetry() && metadata_drop_lock != nullptr && drop_lock_version != -1)
+                    {
+                        /// if we have leftover lock from previous try, we need to recreate the ephemeral with our session
+                        Coordination::Requests drop_lock_requests{
+                            zkutil::makeRemoveRequest(zk_dropped_lock_path, drop_lock_version),
+                            zkutil::makeCreateRequest(zk_dropped_lock_path, "", zkutil::CreateMode::Ephemeral),
+                        };
+
+                        Coordination::Responses drop_lock_responses;
+                        auto lock_code = client->tryMulti(drop_lock_requests, drop_lock_responses);
+                        if (lock_code == Coordination::Error::ZBADVERSION)
+                        {
+                            LOG_INFO(log, "Someone else is removing leftover nodes");
+                            metadata_drop_lock->setAlreadyRemoved();
+                            metadata_drop_lock.reset();
+                            return;
+                        }
+
+                        if (drop_lock_responses[0]->error == Coordination::Error::ZNONODE)
+                        {
+                            /// someone else removed metadata nodes or the previous ephemeral node expired
+                            /// we will try creating dropped lock again to make sure
+                            metadata_drop_lock->setAlreadyRemoved();
+                            metadata_drop_lock.reset();
+                        }
+                        else if (lock_code == Coordination::Error::ZOK)
+                        {
+                            metadata_drop_lock->setAlreadyRemoved();
+                            metadata_drop_lock = zkutil::EphemeralNodeHolder::existing(zk_dropped_lock_path, *client);
+                            drop_lock_version = -1;
+                            Coordination::Stat lock_stat;
+                            client->get(zk_dropped_lock_path, &lock_stat);
+                            drop_lock_version = lock_stat.version;
+                            if (!dropTable(client, metadata_drop_lock))
+                            {
+                                metadata_drop_lock.reset();
+                                return;
+                            }
+                            drop_finished = true;
+                        }
+                    }
+
+                    if (!drop_finished)
+                    {
+                        auto code = client->tryCreate(zk_dropped_lock_path, "", zkutil::CreateMode::Ephemeral);
+
+                        if (code == Coordination::Error::ZNONODE)
+                        {
+                            LOG_INFO(log, "Someone else removed leftover nodes");
+                        }
+                        else if (code == Coordination::Error::ZNODEEXISTS)
+                        {
+                            LOG_INFO(log, "Someone else is removing leftover nodes");
+                            return;
+                        }
+                        else if (code != Coordination::Error::ZOK)
+                        {
+                            throw Coordination::Exception::fromPath(code, zk_dropped_lock_path);
+                        }
+                        else
+                        {
+                            metadata_drop_lock = zkutil::EphemeralNodeHolder::existing(zk_dropped_lock_path, *client);
+                            drop_lock_version = -1;
+                            Coordination::Stat lock_stat;
+                            client->get(zk_dropped_lock_path, &lock_stat);
+                            drop_lock_version = lock_stat.version;
+                            if (!dropTable(client, metadata_drop_lock))
+                            {
+                                metadata_drop_lock.reset();
+                                return;
+                            }
+                        }
+                    }
+                }
+
+                Coordination::Requests create_requests{
+                    zkutil::makeCreateRequest(zk_metadata_path, metadata_string, zkutil::CreateMode::Persistent),
+                    zkutil::makeCreateRequest(zk_data_path, metadata_string, zkutil::CreateMode::Persistent),
+                    zkutil::makeCreateRequest(zk_tables_path, "", zkutil::CreateMode::Persistent),
+                    zkutil::makeCreateRequest(zk_table_path, "", zkutil::CreateMode::Persistent),
+                };
+
+                Coordination::Responses create_responses;
+                auto code = client->tryMulti(create_requests, create_responses);
+                if (code == Coordination::Error::ZNODEEXISTS)
+                {
+                    LOG_INFO(
+                        log, "It looks like a table on path {} was created by another server at the same moment, will retry", zk_root_path);
+                    return;
+                }
+                else if (code != Coordination::Error::ZOK)
+                {
+                    zkutil::KeeperMultiException::check(code, create_requests, create_responses);
+                }
+
+                table_status = TableStatus::VALID;
+                /// we are the first table created for the specified Keeper path, i.e. we are the first replica
+                success = true;
+            });
+
+        if (success)
+            return;
     }
 
-    throw Exception(ErrorCodes::BAD_ARGUMENTS,
-                    "Cannot create metadata for table, because it is removed concurrently or because "
-                    "of wrong zk_root_path ({})", zk_root_path);
+    throw Exception(
+        ErrorCodes::BAD_ARGUMENTS,
+        "Cannot create metadata for table, because it is removed concurrently or because "
+        "of wrong zk_root_path ({})",
+        zk_root_path);
 }
 
 
@@ -518,7 +606,7 @@ Pipe StorageKeeperMap::read(
     size_t max_block_size,
     size_t num_streams)
 {
-    checkTable<true>();
+    checkTable<true>(context_);
     storage_snapshot->check(column_names);
 
     FieldVectorPtr filtered_keys;
@@ -592,13 +680,13 @@ Pipe StorageKeeperMap::read(
 
 SinkToStoragePtr StorageKeeperMap::write(const ASTPtr & /*query*/, const StorageMetadataPtr & metadata_snapshot, ContextPtr local_context, bool /*async_insert*/)
 {
-    checkTable<true>();
+    checkTable<true>(local_context);
     return std::make_shared<StorageKeeperMapSink>(*this, metadata_snapshot->getSampleBlock(), local_context);
 }
 
 void StorageKeeperMap::truncate(const ASTPtr &, const StorageMetadataPtr &, ContextPtr local_context, TableExclusiveLockHolder &)
 {
-    checkTable<true>();
+    checkTable<true>(local_context);
     const auto & settings = local_context->getSettingsRef();
     ZooKeeperRetriesControl zk_retry{
         getName(),
@@ -657,7 +745,7 @@ bool StorageKeeperMap::dropTable(zkutil::ZooKeeperPtr zookeeper, const zkutil::E
 
 void StorageKeeperMap::drop()
 {
-    auto current_table_status = getTableStatus();
+    auto current_table_status = getTableStatus(getContext());
     if (current_table_status == TableStatus::UNKNOWN)
     {
             static constexpr auto error_msg = "Failed to activate table because of connection issues. It will be activated "
@@ -666,7 +754,7 @@ void StorageKeeperMap::drop()
     }
 
     /// if only column metadata is wrong we can still drop the table correctly
-    if (current_table_status == TableStatus::INVALID_KEEPER_STRUCTURE)
+    if (current_table_status == TableStatus::INVALID_METADATA)
         return;
 
     auto client = getClient();
@@ -1029,7 +1117,7 @@ UInt64 StorageKeeperMap::keysLimit() const
     return keys_limit;
 }
 
-StorageKeeperMap::TableStatus StorageKeeperMap::getTableStatus() const
+StorageKeeperMap::TableStatus StorageKeeperMap::getTableStatus(const ContextPtr & local_context) const
 {
     std::lock_guard lock{init_mutex};
     if (table_status != TableStatus::UNKNOWN)
@@ -1039,57 +1127,70 @@ StorageKeeperMap::TableStatus StorageKeeperMap::getTableStatus() const
     {
         try
         {
-            auto client = getClient();
+            const auto & settings = local_context->getSettingsRef();
+            ZooKeeperRetriesControl zk_retry{
+                getName(),
+                getLogger(getName()),
+                ZooKeeperRetriesInfo{
+                    settings.keeper_max_retries,
+                    settings.keeper_retry_initial_backoff_ms,
+                    settings.keeper_retry_max_backoff_ms},
+                local_context->getProcessListElement()};
 
-            Coordination::Stat metadata_stat;
-            auto stored_metadata_string = client->get(zk_metadata_path, &metadata_stat);
-
-            if (metadata_stat.numChildren == 0)
+            zk_retry.retryLoop([&]
             {
+                auto client = getClient();
+
+                Coordination::Stat metadata_stat;
+                auto stored_metadata_string = client->get(zk_metadata_path, &metadata_stat);
+
+                if (metadata_stat.numChildren == 0)
+                {
+                    table_status = TableStatus::INVALID_KEEPER_STRUCTURE;
+                    return;
+                }
+
+                if (metadata_string != stored_metadata_string)
+                {
+                    LOG_ERROR(
+                        log,
+                        "Table definition does not match to the one stored in the path {}. Stored definition: {}",
+                        zk_root_path,
+                        stored_metadata_string);
+                    table_status = TableStatus::INVALID_METADATA;
+                    return;
+                }
+
+                // validate all metadata and data nodes are present
+                Coordination::Requests requests;
+                requests.push_back(zkutil::makeCheckRequest(zk_table_path, -1));
+                requests.push_back(zkutil::makeCheckRequest(zk_data_path, -1));
+                requests.push_back(zkutil::makeCheckRequest(zk_dropped_path, -1));
+
+                Coordination::Responses responses;
+                client->tryMulti(requests, responses);
+
                 table_status = TableStatus::INVALID_KEEPER_STRUCTURE;
-                return;
-            }
+                if (responses[0]->error != Coordination::Error::ZOK)
+                {
+                    LOG_ERROR(log, "Table node ({}) is missing", zk_table_path);
+                    return;
+                }
 
-            if (metadata_string != stored_metadata_string)
-            {
-                LOG_ERROR(
-                    log,
-                    "Table definition does not match to the one stored in the path {}. Stored definition: {}",
-                    zk_root_path,
-                    stored_metadata_string);
-                table_status = TableStatus::INVALID_METADATA;
-                return;
-            }
+                if (responses[1]->error != Coordination::Error::ZOK)
+                {
+                    LOG_ERROR(log, "Data node ({}) is missing", zk_data_path);
+                    return;
+                }
 
-            // validate all metadata and data nodes are present
-            Coordination::Requests requests;
-            requests.push_back(zkutil::makeCheckRequest(zk_table_path, -1));
-            requests.push_back(zkutil::makeCheckRequest(zk_data_path, -1));
-            requests.push_back(zkutil::makeCheckRequest(zk_dropped_path, -1));
+                if (responses[2]->error == Coordination::Error::ZOK)
+                {
+                    LOG_ERROR(log, "Tables with root node {} are being dropped", zk_root_path);
+                    return;
+                }
 
-            Coordination::Responses responses;
-            client->tryMulti(requests, responses);
-
-            table_status = TableStatus::INVALID_KEEPER_STRUCTURE;
-            if (responses[0]->error != Coordination::Error::ZOK)
-            {
-                LOG_ERROR(log, "Table node ({}) is missing", zk_table_path);
-                return;
-            }
-
-            if (responses[1]->error != Coordination::Error::ZOK)
-            {
-                LOG_ERROR(log, "Data node ({}) is missing", zk_data_path);
-                return;
-            }
-
-            if (responses[2]->error == Coordination::Error::ZOK)
-            {
-                LOG_ERROR(log, "Tables with root node {} are being dropped", zk_root_path);
-                return;
-            }
-
-            table_status = TableStatus::VALID;
+                table_status = TableStatus::VALID;
+            });
         }
         catch (const Coordination::Exception & e)
         {
@@ -1227,7 +1328,7 @@ void StorageKeeperMap::checkMutationIsPossible(const MutationCommands & commands
 
 void StorageKeeperMap::mutate(const MutationCommands & commands, ContextPtr local_context)
 {
-    checkTable<true>();
+    checkTable<true>(local_context);
 
     if (commands.empty())
         return;
diff --git a/src/Storages/StorageKeeperMap.h b/src/Storages/StorageKeeperMap.h
index 8ed348a4f6f..1464eeaabad 100644
--- a/src/Storages/StorageKeeperMap.h
+++ b/src/Storages/StorageKeeperMap.h
@@ -78,9 +78,9 @@ public:
     UInt64 keysLimit() const;
 
     template <bool throw_on_error>
-    void checkTable() const
+    void checkTable(const ContextPtr & local_context) const
     {
-        auto current_table_status = getTableStatus();
+        auto current_table_status = getTableStatus(local_context);
         if (table_status == TableStatus::UNKNOWN)
         {
             static constexpr auto error_msg = "Failed to activate table because of connection issues. It will be activated "
@@ -119,7 +119,7 @@ private:
         VALID
     };
 
-    TableStatus getTableStatus() const;
+    TableStatus getTableStatus(const ContextPtr & context) const;
 
     void restoreDataImpl(
         const BackupPtr & backup,
diff --git a/tests/integration/test_keeper_map/configs/keeper_retries.xml b/tests/integration/test_keeper_map/configs/keeper_retries.xml
new file mode 100644
index 00000000000..43e5b9a09e8
--- /dev/null
+++ b/tests/integration/test_keeper_map/configs/keeper_retries.xml
@@ -0,0 +1,14 @@
+<clickhouse>
+    <profiles>
+        <default>
+            <insert_keeper_max_retries>0</insert_keeper_max_retries>
+            <keeper_max_retries>0</keeper_max_retries>
+        </default>
+    </profiles>
+    <users>
+        <default>
+            <password></password>
+            <profile>default</profile>
+        </default>
+    </users>
+</clickhouse>
diff --git a/tests/integration/test_keeper_map/test.py b/tests/integration/test_keeper_map/test.py
index 4b1bcd11cfe..861a7c47687 100644
--- a/tests/integration/test_keeper_map/test.py
+++ b/tests/integration/test_keeper_map/test.py
@@ -10,6 +10,7 @@ cluster = ClickHouseCluster(__file__)
 node = cluster.add_instance(
     "node",
     main_configs=["configs/enable_keeper_map.xml"],
+    user_configs=["configs/keeper_retries.xml"],
     with_zookeeper=True,
     stay_alive=True,
 )
@@ -49,7 +50,6 @@ def assert_keeper_exception_after_partition(query):
             error = node.query_and_get_error_with_retry(
                 query,
                 sleep_time=1,
-                settings={"insert_keeper_max_retries": 1, "keeper_max_retries": 1},
             )
             assert "Coordination::Exception" in error
         except:
@@ -91,7 +91,6 @@ def test_keeper_map_without_zk(started_cluster):
             error = node.query_and_get_error_with_retry(
                 "SELECT * FROM test_keeper_map_without_zk",
                 sleep_time=1,
-                settings={"keeper_max_retries": 1},
             )
             assert "Failed to activate table because of connection issues" in error
         except:
diff --git a/tests/integration/test_keeper_map_retries/configs/fault_injection.xml b/tests/integration/test_keeper_map_retries/configs/fault_injection.xml
index 0933b6b3031..8406b7db785 100644
--- a/tests/integration/test_keeper_map_retries/configs/fault_injection.xml
+++ b/tests/integration/test_keeper_map_retries/configs/fault_injection.xml
@@ -1,5 +1,6 @@
 <clickhouse>
     <zookeeper>
+        <enable_fault_injections_during_startup>1</enable_fault_injections_during_startup>
         <send_fault_probability>0.005</send_fault_probability>
         <recv_fault_probability>0.005</recv_fault_probability>
     </zookeeper>
diff --git a/tests/integration/test_keeper_map_retries/configs/keeper_retries.xml b/tests/integration/test_keeper_map_retries/configs/keeper_retries.xml
new file mode 100644
index 00000000000..208dd6e47fa
--- /dev/null
+++ b/tests/integration/test_keeper_map_retries/configs/keeper_retries.xml
@@ -0,0 +1,14 @@
+<clickhouse>
+    <profiles>
+        <default>
+            <keeper_max_retries>20</keeper_max_retries>
+            <keeper_retry_max_backoff_ms>10000</keeper_retry_max_backoff_ms>
+        </default>
+    </profiles>
+    <users>
+        <default>
+            <password></password>
+            <profile>default</profile>
+        </default>
+    </users>
+</clickhouse>
diff --git a/tests/integration/test_keeper_map_retries/test.py b/tests/integration/test_keeper_map_retries/test.py
index 352119147cd..c6760e5d1a2 100644
--- a/tests/integration/test_keeper_map_retries/test.py
+++ b/tests/integration/test_keeper_map_retries/test.py
@@ -11,6 +11,7 @@ cluster = ClickHouseCluster(__file__)
 node = cluster.add_instance(
     "node",
     main_configs=["configs/enable_keeper_map.xml"],
+    user_configs=["configs/keeper_retries.xml"],
     with_zookeeper=True,
     stay_alive=True,
 )
@@ -42,10 +43,6 @@ def repeat_query(query, repeat):
     for _ in range(repeat):
         node.query(
             query,
-            settings={
-                "keeper_max_retries": 20,
-                "keeper_retry_max_backoff_ms": 10000,
-            },
         )
 
 
@@ -53,10 +50,6 @@ def test_queries(started_cluster):
     start_clean_clickhouse()
 
     node.query("DROP TABLE IF EXISTS keeper_map_retries SYNC")
-    node.query(
-        "CREATE TABLE keeper_map_retries (a UInt64, b UInt64) Engine=KeeperMap('/keeper_map_retries') PRIMARY KEY a"
-    )
-
     node.stop_clickhouse()
     node.copy_file_to_container(
         os.path.join(CONFIG_DIR, "fault_injection.xml"),
@@ -66,6 +59,10 @@ def test_queries(started_cluster):
 
     repeat_count = 10
 
+    node.query(
+        "CREATE TABLE keeper_map_retries (a UInt64, b UInt64) Engine=KeeperMap('/keeper_map_retries') PRIMARY KEY a",
+    )
+
     repeat_query(
         "INSERT INTO keeper_map_retries SELECT number, number FROM numbers(500)",
         repeat_count,

From bababe2dd2fe490e877f9c4ac03d979bd739376a Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Mon, 29 Jul 2024 15:07:07 +0000
Subject: [PATCH 1160/1488] Fix remainders I

---
 src/Functions/FunctionsMultiStringSearch.h    |  8 +++---
 src/Functions/MultiMatchAnyImpl.h             | 27 ++++++++++---------
 src/Functions/MultiSearchFirstIndexImpl.h     | 16 +++++------
 src/Functions/MultiSearchFirstPositionImpl.h  | 18 ++++++-------
 src/Functions/MultiSearchImpl.h               | 18 ++++++-------
 .../0_stateless/00927_disable_hyperscan.sql   |  2 +-
 6 files changed, 45 insertions(+), 44 deletions(-)

diff --git a/src/Functions/FunctionsMultiStringSearch.h b/src/Functions/FunctionsMultiStringSearch.h
index 03db2651fd0..6bcc8581a38 100644
--- a/src/Functions/FunctionsMultiStringSearch.h
+++ b/src/Functions/FunctionsMultiStringSearch.h
@@ -81,7 +81,7 @@ public:
         return Impl::getReturnType();
     }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
         const ColumnPtr & haystack_ptr = arguments[0].column;
         const ColumnPtr & needles_ptr = arguments[1].column;
@@ -110,13 +110,15 @@ public:
                 col_haystack_vector->getChars(), col_haystack_vector->getOffsets(),
                 col_needles_const->getValue<Array>(),
                 vec_res, offsets_res,
-                allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps);
+                allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps,
+                input_rows_count);
         else
             Impl::vectorVector(
                 col_haystack_vector->getChars(), col_haystack_vector->getOffsets(),
                 col_needles_vector->getData(), col_needles_vector->getOffsets(),
                 vec_res, offsets_res,
-                allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps);
+                allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps,
+                input_rows_count);
 
         // the combination of const haystack + const needle is not implemented because
         // useDefaultImplementationForConstants() == true makes upper layers convert both to
diff --git a/src/Functions/MultiMatchAnyImpl.h b/src/Functions/MultiMatchAnyImpl.h
index 20b2150048b..06d9eded9c9 100644
--- a/src/Functions/MultiMatchAnyImpl.h
+++ b/src/Functions/MultiMatchAnyImpl.h
@@ -66,9 +66,10 @@ struct MultiMatchAnyImpl
         bool allow_hyperscan,
         size_t max_hyperscan_regexp_length,
         size_t max_hyperscan_regexp_total_length,
-        bool reject_expensive_hyperscan_regexps)
+        bool reject_expensive_hyperscan_regexps,
+        size_t input_rows_count)
     {
-        vectorConstant(haystack_data, haystack_offsets, needles_arr, res, offsets, std::nullopt, allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps);
+        vectorConstant(haystack_data, haystack_offsets, needles_arr, res, offsets, std::nullopt, allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps, input_rows_count);
     }
 
     static void vectorConstant(
@@ -81,7 +82,8 @@ struct MultiMatchAnyImpl
         bool allow_hyperscan,
         size_t max_hyperscan_regexp_length,
         size_t max_hyperscan_regexp_total_length,
-        bool reject_expensive_hyperscan_regexps)
+        bool reject_expensive_hyperscan_regexps,
+        size_t input_rows_count)
     {
         if (!allow_hyperscan)
             throw Exception(ErrorCodes::FUNCTION_NOT_ALLOWED, "Hyperscan functions are disabled, because setting 'allow_hyperscan' is set to 0");
@@ -101,7 +103,7 @@ struct MultiMatchAnyImpl
                     throw Exception(ErrorCodes::HYPERSCAN_CANNOT_SCAN_TEXT, "Regular expression evaluation in vectorscan will be too slow. To ignore this error, disable setting 'reject_expensive_hyperscan_regexps'.");
         }
 
-        res.resize(haystack_offsets.size());
+        res.resize(input_rows_count);
 
         if (needles_arr.empty())
         {
@@ -133,9 +135,8 @@ struct MultiMatchAnyImpl
             /// Once we hit the callback, there is no need to search for others.
             return 1;
         };
-        const size_t haystack_offsets_size = haystack_offsets.size();
         UInt64 offset = 0;
-        for (size_t i = 0; i < haystack_offsets_size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             UInt64 length = haystack_offsets[i] - offset - 1;
             /// vectorscan restriction.
@@ -186,9 +187,10 @@ struct MultiMatchAnyImpl
         bool allow_hyperscan,
         size_t max_hyperscan_regexp_length,
         size_t max_hyperscan_regexp_total_length,
-        bool reject_expensive_hyperscan_regexps)
+        bool reject_expensive_hyperscan_regexps,
+        size_t input_rows_count)
     {
-        vectorVector(haystack_data, haystack_offsets, needles_data, needles_offsets, res, offsets, std::nullopt, allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps);
+        vectorVector(haystack_data, haystack_offsets, needles_data, needles_offsets, res, offsets, std::nullopt, allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps, input_rows_count);
     }
 
     static void vectorVector(
@@ -202,12 +204,13 @@ struct MultiMatchAnyImpl
         bool allow_hyperscan,
         size_t max_hyperscan_regexp_length,
         size_t max_hyperscan_regexp_total_length,
-        bool reject_expensive_hyperscan_regexps)
+        bool reject_expensive_hyperscan_regexps,
+        size_t input_rows_count)
     {
         if (!allow_hyperscan)
             throw Exception(ErrorCodes::FUNCTION_NOT_ALLOWED, "Hyperscan functions are disabled, because setting 'allow_hyperscan' is set to 0");
 
-        res.resize(haystack_offsets.size());
+        res.resize(input_rows_count);
 #if USE_VECTORSCAN
         size_t prev_haystack_offset = 0;
         size_t prev_needles_offset = 0;
@@ -216,7 +219,7 @@ struct MultiMatchAnyImpl
 
         std::vector<std::string_view> needles;
 
-        for (size_t i = 0; i < haystack_offsets.size(); ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             needles.reserve(needles_offsets[i] - prev_needles_offset);
 
@@ -306,7 +309,7 @@ struct MultiMatchAnyImpl
 
         std::vector<std::string_view> needles;
 
-        for (size_t i = 0; i < haystack_offsets.size(); ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             const auto * const cur_haystack_data = &haystack_data[prev_haystack_offset];
             const size_t cur_haystack_length = haystack_offsets[i] - prev_haystack_offset - 1;
diff --git a/src/Functions/MultiSearchFirstIndexImpl.h b/src/Functions/MultiSearchFirstIndexImpl.h
index 36a5fd514d9..5b34dbfe601 100644
--- a/src/Functions/MultiSearchFirstIndexImpl.h
+++ b/src/Functions/MultiSearchFirstIndexImpl.h
@@ -33,7 +33,8 @@ struct MultiSearchFirstIndexImpl
         bool /*allow_hyperscan*/,
         size_t /*max_hyperscan_regexp_length*/,
         size_t /*max_hyperscan_regexp_total_length*/,
-        bool /*reject_expensive_hyperscan_regexps*/)
+        bool /*reject_expensive_hyperscan_regexps*/,
+        size_t input_rows_count)
     {
         // For performance of Volnitsky search, it is crucial to save only one byte for pattern number.
         if (needles_arr.size() > std::numeric_limits<UInt8>::max())
@@ -48,14 +49,13 @@ struct MultiSearchFirstIndexImpl
 
         auto searcher = Impl::createMultiSearcherInBigHaystack(needles);
 
-        const size_t haystack_size = haystack_offsets.size();
-        res.resize(haystack_size);
+        res.resize(input_rows_count);
 
         size_t iteration = 0;
         while (searcher.hasMoreToSearch())
         {
             size_t prev_haystack_offset = 0;
-            for (size_t j = 0; j < haystack_size; ++j)
+            for (size_t j = 0; j < input_rows_count; ++j)
             {
                 const auto * haystack = &haystack_data[prev_haystack_offset];
                 const auto * haystack_end = haystack + haystack_offsets[j] - prev_haystack_offset - 1;
@@ -80,10 +80,10 @@ struct MultiSearchFirstIndexImpl
         bool /*allow_hyperscan*/,
         size_t /*max_hyperscan_regexp_length*/,
         size_t /*max_hyperscan_regexp_total_length*/,
-        bool /*reject_expensive_hyperscan_regexps*/)
+        bool /*reject_expensive_hyperscan_regexps*/,
+        size_t input_rows_count)
     {
-        const size_t haystack_size = haystack_offsets.size();
-        res.resize(haystack_size);
+        res.resize(input_rows_count);
 
         size_t prev_haystack_offset = 0;
         size_t prev_needles_offset = 0;
@@ -92,7 +92,7 @@ struct MultiSearchFirstIndexImpl
 
         std::vector<std::string_view> needles;
 
-        for (size_t i = 0; i < haystack_size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             needles.reserve(needles_offsets[i] - prev_needles_offset);
 
diff --git a/src/Functions/MultiSearchFirstPositionImpl.h b/src/Functions/MultiSearchFirstPositionImpl.h
index ccdd82a0ee5..06bf7aa94d8 100644
--- a/src/Functions/MultiSearchFirstPositionImpl.h
+++ b/src/Functions/MultiSearchFirstPositionImpl.h
@@ -33,7 +33,8 @@ struct MultiSearchFirstPositionImpl
         bool /*allow_hyperscan*/,
         size_t /*max_hyperscan_regexp_length*/,
         size_t /*max_hyperscan_regexp_total_length*/,
-        bool /*reject_expensive_hyperscan_regexps*/)
+        bool /*reject_expensive_hyperscan_regexps*/,
+        size_t input_rows_count)
     {
         // For performance of Volnitsky search, it is crucial to save only one byte for pattern number.
         if (needles_arr.size() > std::numeric_limits<UInt8>::max())
@@ -52,14 +53,13 @@ struct MultiSearchFirstPositionImpl
         };
         auto searcher = Impl::createMultiSearcherInBigHaystack(needles);
 
-        const size_t haystack_size = haystack_offsets.size();
-        res.resize(haystack_size);
+        res.resize(input_rows_count);
 
         size_t iteration = 0;
         while (searcher.hasMoreToSearch())
         {
             size_t prev_haystack_offset = 0;
-            for (size_t j = 0; j < haystack_size; ++j)
+            for (size_t j = 0; j < input_rows_count; ++j)
             {
                 const auto * haystack = &haystack_data[prev_haystack_offset];
                 const auto * haystack_end = haystack + haystack_offsets[j] - prev_haystack_offset - 1;
@@ -89,10 +89,10 @@ struct MultiSearchFirstPositionImpl
         bool /*allow_hyperscan*/,
         size_t /*max_hyperscan_regexp_length*/,
         size_t /*max_hyperscan_regexp_total_length*/,
-        bool /*reject_expensive_hyperscan_regexps*/)
+        bool /*reject_expensive_hyperscan_regexps*/,
+        size_t input_rows_count)
     {
-        const size_t haystack_size = haystack_offsets.size();
-        res.resize(haystack_size);
+        res.resize(input_rows_count);
 
         size_t prev_haystack_offset = 0;
         size_t prev_needles_offset = 0;
@@ -106,14 +106,12 @@ struct MultiSearchFirstPositionImpl
             return 1 + Impl::countChars(reinterpret_cast<const char *>(start), reinterpret_cast<const char *>(end));
         };
 
-        for (size_t i = 0; i < haystack_size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             needles.reserve(needles_offsets[i] - prev_needles_offset);
 
             for (size_t j = prev_needles_offset; j < needles_offsets[i]; ++j)
-            {
                 needles.emplace_back(needles_data_string.getDataAt(j).toView());
-            }
 
             auto searcher = Impl::createMultiSearcherInBigHaystack(needles); // sub-optimal
 
diff --git a/src/Functions/MultiSearchImpl.h b/src/Functions/MultiSearchImpl.h
index 467cc96a95f..909425f5a93 100644
--- a/src/Functions/MultiSearchImpl.h
+++ b/src/Functions/MultiSearchImpl.h
@@ -33,7 +33,8 @@ struct MultiSearchImpl
         bool /*allow_hyperscan*/,
         size_t /*max_hyperscan_regexp_length*/,
         size_t /*max_hyperscan_regexp_total_length*/,
-        bool /*reject_expensive_hyperscan_regexps*/)
+        bool /*reject_expensive_hyperscan_regexps*/,
+        size_t input_rows_count)
     {
         // For performance of Volnitsky search, it is crucial to save only one byte for pattern number.
         if (needles_arr.size() > std::numeric_limits<UInt8>::max())
@@ -48,14 +49,13 @@ struct MultiSearchImpl
 
         auto searcher = Impl::createMultiSearcherInBigHaystack(needles);
 
-        const size_t haystack_size = haystack_offsets.size();
-        res.resize(haystack_size);
+        res.resize(input_rows_count);
 
         size_t iteration = 0;
         while (searcher.hasMoreToSearch())
         {
             size_t prev_haystack_offset = 0;
-            for (size_t j = 0; j < haystack_size; ++j)
+            for (size_t j = 0; j < input_rows_count; ++j)
             {
                 const auto * haystack = &haystack_data[prev_haystack_offset];
                 const auto * haystack_end = haystack + haystack_offsets[j] - prev_haystack_offset - 1;
@@ -79,10 +79,10 @@ struct MultiSearchImpl
         bool /*allow_hyperscan*/,
         size_t /*max_hyperscan_regexp_length*/,
         size_t /*max_hyperscan_regexp_total_length*/,
-        bool /*reject_expensive_hyperscan_regexps*/)
+        bool /*reject_expensive_hyperscan_regexps*/,
+        size_t input_rows_count)
     {
-        const size_t haystack_size = haystack_offsets.size();
-        res.resize(haystack_size);
+        res.resize(input_rows_count);
 
         size_t prev_haystack_offset = 0;
         size_t prev_needles_offset = 0;
@@ -91,14 +91,12 @@ struct MultiSearchImpl
 
         std::vector<std::string_view> needles;
 
-        for (size_t i = 0; i < haystack_size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             needles.reserve(needles_offsets[i] - prev_needles_offset);
 
             for (size_t j = prev_needles_offset; j < needles_offsets[i]; ++j)
-            {
                 needles.emplace_back(needles_data_string.getDataAt(j).toView());
-            }
 
             const auto * const haystack = &haystack_data[prev_haystack_offset];
             const size_t haystack_length = haystack_offsets[i] - prev_haystack_offset - 1;
diff --git a/tests/queries/0_stateless/00927_disable_hyperscan.sql b/tests/queries/0_stateless/00927_disable_hyperscan.sql
index c07848a4fcc..24ec7a35adb 100644
--- a/tests/queries/0_stateless/00927_disable_hyperscan.sql
+++ b/tests/queries/0_stateless/00927_disable_hyperscan.sql
@@ -1,4 +1,4 @@
--- Tags: no-debug
+-- Tags: no-fasttest
 
 SET allow_hyperscan = 1;
 

From 0f850952fa304bcc63f7008d4bae54c47ea0564d Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Tue, 30 Jul 2024 09:52:25 +0000
Subject: [PATCH 1161/1488] Fix remainders II

---
 .../FunctionsMultiStringFuzzySearch.h         |  8 +++---
 src/Functions/MultiMatchAllIndicesImpl.h      | 27 +++++++++++--------
 2 files changed, 21 insertions(+), 14 deletions(-)

diff --git a/src/Functions/FunctionsMultiStringFuzzySearch.h b/src/Functions/FunctionsMultiStringFuzzySearch.h
index a92a6570279..8346380c35d 100644
--- a/src/Functions/FunctionsMultiStringFuzzySearch.h
+++ b/src/Functions/FunctionsMultiStringFuzzySearch.h
@@ -71,7 +71,7 @@ public:
         return Impl::getReturnType();
     }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
         const ColumnPtr & haystack_ptr = arguments[0].column;
         const ColumnPtr & edit_distance_ptr = arguments[1].column;
@@ -114,14 +114,16 @@ public:
                 col_needles_const->getValue<Array>(),
                 vec_res, offsets_res,
                 edit_distance,
-                allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps);
+                allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps,
+                input_rows_count);
         else
             Impl::vectorVector(
                 col_haystack_vector->getChars(), col_haystack_vector->getOffsets(),
                 col_needles_vector->getData(), col_needles_vector->getOffsets(),
                 vec_res, offsets_res,
                 edit_distance,
-                allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps);
+                allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps,
+                input_rows_count);
 
         // the combination of const haystack + const needle is not implemented because
         // useDefaultImplementationForConstants() == true makes upper layers convert both to
diff --git a/src/Functions/MultiMatchAllIndicesImpl.h b/src/Functions/MultiMatchAllIndicesImpl.h
index 3e9c8fba215..3aeac808880 100644
--- a/src/Functions/MultiMatchAllIndicesImpl.h
+++ b/src/Functions/MultiMatchAllIndicesImpl.h
@@ -52,9 +52,10 @@ struct MultiMatchAllIndicesImpl
         bool allow_hyperscan,
         size_t max_hyperscan_regexp_length,
         size_t max_hyperscan_regexp_total_length,
-        bool reject_expensive_hyperscan_regexps)
+        bool reject_expensive_hyperscan_regexps,
+        size_t input_rows_count)
     {
-        vectorConstant(haystack_data, haystack_offsets, needles_arr, res, offsets, std::nullopt, allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps);
+        vectorConstant(haystack_data, haystack_offsets, needles_arr, res, offsets, std::nullopt, allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps, input_rows_count);
     }
 
     static void vectorConstant(
@@ -67,7 +68,8 @@ struct MultiMatchAllIndicesImpl
         bool allow_hyperscan,
         size_t max_hyperscan_regexp_length,
         size_t max_hyperscan_regexp_total_length,
-        bool reject_expensive_hyperscan_regexps)
+        bool reject_expensive_hyperscan_regexps,
+        size_t input_rows_count)
     {
         if (!allow_hyperscan)
             throw Exception(ErrorCodes::FUNCTION_NOT_ALLOWED, "Hyperscan functions are disabled, because setting 'allow_hyperscan' is set to 0");
@@ -87,7 +89,7 @@ struct MultiMatchAllIndicesImpl
                     throw Exception(ErrorCodes::HYPERSCAN_CANNOT_SCAN_TEXT, "Regular expression evaluation in vectorscan will be too slow. To ignore this error, disable setting 'reject_expensive_hyperscan_regexps'.");
         }
 
-        offsets.resize(haystack_offsets.size());
+        offsets.resize(input_rows_count);
 
         if (needles_arr.empty())
         {
@@ -114,9 +116,8 @@ struct MultiMatchAllIndicesImpl
             static_cast<PaddedPODArray<ResultType>*>(context)->push_back(id);
             return 0;
         };
-        const size_t haystack_offsets_size = haystack_offsets.size();
         UInt64 offset = 0;
-        for (size_t i = 0; i < haystack_offsets_size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             UInt64 length = haystack_offsets[i] - offset - 1;
             /// vectorscan restriction.
@@ -146,6 +147,7 @@ struct MultiMatchAllIndicesImpl
         (void)max_hyperscan_regexp_length;
         (void)max_hyperscan_regexp_total_length;
         (void)reject_expensive_hyperscan_regexps;
+        (void)input_rows_count;
         throw Exception(ErrorCodes::NOT_IMPLEMENTED, "multi-search all indices is not implemented when vectorscan is off");
 #endif // USE_VECTORSCAN
     }
@@ -160,9 +162,10 @@ struct MultiMatchAllIndicesImpl
         bool allow_hyperscan,
         size_t max_hyperscan_regexp_length,
         size_t max_hyperscan_regexp_total_length,
-        bool reject_expensive_hyperscan_regexps)
+        bool reject_expensive_hyperscan_regexps,
+        size_t input_rows_count)
     {
-        vectorVector(haystack_data, haystack_offsets, needles_data, needles_offsets, res, offsets, std::nullopt, allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps);
+        vectorVector(haystack_data, haystack_offsets, needles_data, needles_offsets, res, offsets, std::nullopt, allow_hyperscan, max_hyperscan_regexp_length, max_hyperscan_regexp_total_length, reject_expensive_hyperscan_regexps, input_rows_count);
     }
 
     static void vectorVector(
@@ -176,12 +179,13 @@ struct MultiMatchAllIndicesImpl
         bool allow_hyperscan,
         size_t max_hyperscan_regexp_length,
         size_t max_hyperscan_regexp_total_length,
-        bool reject_expensive_hyperscan_regexps)
+        bool reject_expensive_hyperscan_regexps,
+        size_t input_rows_count)
     {
         if (!allow_hyperscan)
             throw Exception(ErrorCodes::FUNCTION_NOT_ALLOWED, "Hyperscan functions are disabled, because setting 'allow_hyperscan' is set to 0");
 #if USE_VECTORSCAN
-        offsets.resize(haystack_offsets.size());
+        offsets.resize(input_rows_count);
         size_t prev_haystack_offset = 0;
         size_t prev_needles_offset = 0;
 
@@ -189,7 +193,7 @@ struct MultiMatchAllIndicesImpl
 
         std::vector<std::string_view> needles;
 
-        for (size_t i = 0; i < haystack_offsets.size(); ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             needles.reserve(needles_offsets[i] - prev_needles_offset);
 
@@ -271,6 +275,7 @@ struct MultiMatchAllIndicesImpl
         (void)max_hyperscan_regexp_length;
         (void)max_hyperscan_regexp_total_length;
         (void)reject_expensive_hyperscan_regexps;
+        (void)input_rows_count;
         throw Exception(ErrorCodes::NOT_IMPLEMENTED, "multi-search all indices is not implemented when vectorscan is off");
 #endif // USE_VECTORSCAN
     }

From 99760ad7e4b1b7d4ee86dc2f5aa4df98ead9e260 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Tue, 30 Jul 2024 17:17:40 +0000
Subject: [PATCH 1162/1488] Fix remainders, pt. III

---
 src/Functions/FunctionsStringSearchToString.h | 4 ++--
 src/Functions/FunctionsVisitParam.h           | 5 +++--
 src/Functions/URL/extractURLParameter.cpp     | 7 ++++---
 src/Functions/extract.cpp                     | 7 ++++---
 4 files changed, 13 insertions(+), 10 deletions(-)

diff --git a/src/Functions/FunctionsStringSearchToString.h b/src/Functions/FunctionsStringSearchToString.h
index 978a84de472..c889cf062a3 100644
--- a/src/Functions/FunctionsStringSearchToString.h
+++ b/src/Functions/FunctionsStringSearchToString.h
@@ -60,7 +60,7 @@ public:
         return std::make_shared<DataTypeString>();
     }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t /*input_rows_count*/) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
     {
         const ColumnPtr column = arguments[0].column;
         const ColumnPtr column_needle = arguments[1].column;
@@ -75,7 +75,7 @@ public:
 
             ColumnString::Chars & vec_res = col_res->getChars();
             ColumnString::Offsets & offsets_res = col_res->getOffsets();
-            Impl::vector(col->getChars(), col->getOffsets(), col_needle->getValue<String>(), vec_res, offsets_res);
+            Impl::vector(col->getChars(), col->getOffsets(), col_needle->getValue<String>(), vec_res, offsets_res, input_rows_count);
 
             return col_res;
         }
diff --git a/src/Functions/FunctionsVisitParam.h b/src/Functions/FunctionsVisitParam.h
index 5e13fbbad5c..bcaaf0a1c20 100644
--- a/src/Functions/FunctionsVisitParam.h
+++ b/src/Functions/FunctionsVisitParam.h
@@ -168,11 +168,12 @@ struct ExtractParamToStringImpl
 {
     static void vector(const ColumnString::Chars & haystack_data, const ColumnString::Offsets & haystack_offsets,
                        std::string needle,
-                       ColumnString::Chars & res_data, ColumnString::Offsets & res_offsets)
+                       ColumnString::Chars & res_data, ColumnString::Offsets & res_offsets,
+                       size_t input_rows_count)
     {
         /// Constant 5 is taken from a function that performs a similar task FunctionsStringSearch.h::ExtractImpl
         res_data.reserve(haystack_data.size() / 5);
-        res_offsets.resize(haystack_offsets.size());
+        res_offsets.resize(input_rows_count);
 
         /// We are looking for a parameter simply as a substring of the form "name"
         needle = "\"" + needle + "\":";
diff --git a/src/Functions/URL/extractURLParameter.cpp b/src/Functions/URL/extractURLParameter.cpp
index f75875e0200..590c2779d9c 100644
--- a/src/Functions/URL/extractURLParameter.cpp
+++ b/src/Functions/URL/extractURLParameter.cpp
@@ -10,10 +10,11 @@ struct ExtractURLParameterImpl
     static void vector(const ColumnString::Chars & data,
         const ColumnString::Offsets & offsets,
         std::string pattern,
-        ColumnString::Chars & res_data, ColumnString::Offsets & res_offsets)
+        ColumnString::Chars & res_data, ColumnString::Offsets & res_offsets,
+        size_t input_rows_count)
     {
         res_data.reserve(data.size() / 5);
-        res_offsets.resize(offsets.size());
+        res_offsets.resize(input_rows_count);
 
         pattern += '=';
         const char * param_str = pattern.c_str();
@@ -22,7 +23,7 @@ struct ExtractURLParameterImpl
         ColumnString::Offset prev_offset = 0;
         ColumnString::Offset res_offset = 0;
 
-        for (size_t i = 0; i < offsets.size(); ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             ColumnString::Offset cur_offset = offsets[i];
 
diff --git a/src/Functions/extract.cpp b/src/Functions/extract.cpp
index 6bbdaff0e3f..c78ee9898b7 100644
--- a/src/Functions/extract.cpp
+++ b/src/Functions/extract.cpp
@@ -16,10 +16,11 @@ struct ExtractImpl
         const ColumnString::Offsets & offsets,
         const std::string & pattern,
         ColumnString::Chars & res_data,
-        ColumnString::Offsets & res_offsets)
+        ColumnString::Offsets & res_offsets,
+        size_t input_rows_count)
     {
         res_data.reserve(data.size() / 5);
-        res_offsets.resize(offsets.size());
+        res_offsets.resize(input_rows_count);
 
         const OptimizedRegularExpression regexp = Regexps::createRegexp<false, false, false>(pattern);
 
@@ -29,7 +30,7 @@ struct ExtractImpl
         size_t prev_offset = 0;
         size_t res_offset = 0;
 
-        for (size_t i = 0; i < offsets.size(); ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             size_t cur_offset = offsets[i];
 

From 0452768983f379d201e1871ffce63452430bd0c6 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Wed, 31 Jul 2024 08:10:01 +0000
Subject: [PATCH 1163/1488] Fix remainders, pt. IV

---
 src/Functions/CountSubstringsImpl.h   | 17 ++++-----
 src/Functions/FunctionsStringSearch.h | 17 ++++++---
 src/Functions/FunctionsVisitParam.h   |  3 +-
 src/Functions/HasTokenImpl.h          |  5 ++-
 src/Functions/MatchImpl.h             | 54 +++++++++++++--------------
 src/Functions/MultiMatchAnyImpl.h     |  2 +-
 src/Functions/PositionImpl.h          | 22 +++++------
 7 files changed, 62 insertions(+), 58 deletions(-)

diff --git a/src/Functions/CountSubstringsImpl.h b/src/Functions/CountSubstringsImpl.h
index 9ff3e4e1f2a..8e91bc3aeb4 100644
--- a/src/Functions/CountSubstringsImpl.h
+++ b/src/Functions/CountSubstringsImpl.h
@@ -37,7 +37,8 @@ struct CountSubstringsImpl
         const std::string & needle,
         const ColumnPtr & start_pos,
         PaddedPODArray<UInt64> & res,
-        [[maybe_unused]] ColumnUInt8 * res_null)
+        [[maybe_unused]] ColumnUInt8 * res_null,
+        size_t /*input_rows_count*/)
     {
         /// `res_null` serves as an output parameter for implementing an XYZOrNull variant.
         assert(!res_null);
@@ -150,7 +151,8 @@ struct CountSubstringsImpl
         const ColumnString::Offsets & needle_offsets,
         const ColumnPtr & start_pos,
         PaddedPODArray<UInt64> & res,
-        [[maybe_unused]] ColumnUInt8 * res_null)
+        [[maybe_unused]] ColumnUInt8 * res_null,
+        size_t input_rows_count)
     {
         /// `res_null` serves as an output parameter for implementing an XYZOrNull variant.
         assert(!res_null);
@@ -158,9 +160,7 @@ struct CountSubstringsImpl
         ColumnString::Offset prev_haystack_offset = 0;
         ColumnString::Offset prev_needle_offset = 0;
 
-        size_t size = haystack_offsets.size();
-
-        for (size_t i = 0; i < size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             size_t needle_size = needle_offsets[i] - prev_needle_offset - 1;
             size_t haystack_size = haystack_offsets[i] - prev_haystack_offset - 1;
@@ -207,7 +207,8 @@ struct CountSubstringsImpl
         const ColumnString::Offsets & needle_offsets,
         const ColumnPtr & start_pos,
         PaddedPODArray<UInt64> & res,
-        [[maybe_unused]] ColumnUInt8 * res_null)
+        [[maybe_unused]] ColumnUInt8 * res_null,
+        size_t input_rows_count)
     {
         /// `res_null` serves as an output parameter for implementing an XYZOrNull variant.
         assert(!res_null);
@@ -215,9 +216,7 @@ struct CountSubstringsImpl
         /// NOTE You could use haystack indexing. But this is a rare case.
         ColumnString::Offset prev_needle_offset = 0;
 
-        size_t size = needle_offsets.size();
-
-        for (size_t i = 0; i < size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             res[i] = 0;
             auto start = start_pos != nullptr ? std::max(start_pos->getUInt(i), UInt64(1)) : UInt64(1);
diff --git a/src/Functions/FunctionsStringSearch.h b/src/Functions/FunctionsStringSearch.h
index fba6336ebff..7ec0076e395 100644
--- a/src/Functions/FunctionsStringSearch.h
+++ b/src/Functions/FunctionsStringSearch.h
@@ -163,7 +163,7 @@ public:
         return return_type;
     }
 
-    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t /*input_rows_count*/) const override
+    ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override
     {
         const ColumnPtr & column_haystack = (argument_order == ArgumentOrder::HaystackNeedle) ? arguments[0].column : arguments[1].column;
         const ColumnPtr & column_needle = (argument_order == ArgumentOrder::HaystackNeedle) ? arguments[1].column : arguments[0].column;
@@ -236,7 +236,8 @@ public:
                 col_needle_vector->getOffsets(),
                 column_start_pos,
                 vec_res,
-                null_map.get());
+                null_map.get(),
+                input_rows_count);
         else if (col_haystack_vector && col_needle_const)
             Impl::vectorConstant(
                 col_haystack_vector->getChars(),
@@ -244,7 +245,8 @@ public:
                 col_needle_const->getValue<String>(),
                 column_start_pos,
                 vec_res,
-                null_map.get());
+                null_map.get(),
+                input_rows_count);
         else if (col_haystack_vector_fixed && col_needle_vector)
             Impl::vectorFixedVector(
                 col_haystack_vector_fixed->getChars(),
@@ -253,14 +255,16 @@ public:
                 col_needle_vector->getOffsets(),
                 column_start_pos,
                 vec_res,
-                null_map.get());
+                null_map.get(),
+                input_rows_count);
         else if (col_haystack_vector_fixed && col_needle_const)
             Impl::vectorFixedConstant(
                 col_haystack_vector_fixed->getChars(),
                 col_haystack_vector_fixed->getN(),
                 col_needle_const->getValue<String>(),
                 vec_res,
-                null_map.get());
+                null_map.get(),
+                input_rows_count);
         else if (col_haystack_const && col_needle_vector)
             Impl::constantVector(
                 col_haystack_const->getValue<String>(),
@@ -268,7 +272,8 @@ public:
                 col_needle_vector->getOffsets(),
                 column_start_pos,
                 vec_res,
-                null_map.get());
+                null_map.get(),
+                input_rows_count);
         else
             throw Exception(
                 ErrorCodes::ILLEGAL_COLUMN,
diff --git a/src/Functions/FunctionsVisitParam.h b/src/Functions/FunctionsVisitParam.h
index bcaaf0a1c20..a77fa740f9c 100644
--- a/src/Functions/FunctionsVisitParam.h
+++ b/src/Functions/FunctionsVisitParam.h
@@ -93,7 +93,8 @@ struct ExtractParamImpl
         std::string needle,
         const ColumnPtr & start_pos,
         PaddedPODArray<ResultType> & res,
-        [[maybe_unused]] ColumnUInt8 * res_null)
+        [[maybe_unused]] ColumnUInt8 * res_null,
+        size_t /*input_rows_count*/)
     {
         /// `res_null` serves as an output parameter for implementing an XYZOrNull variant.
         assert(!res_null);
diff --git a/src/Functions/HasTokenImpl.h b/src/Functions/HasTokenImpl.h
index a4ff49859cc..4943bf708c5 100644
--- a/src/Functions/HasTokenImpl.h
+++ b/src/Functions/HasTokenImpl.h
@@ -35,12 +35,13 @@ struct HasTokenImpl
         const std::string & pattern,
         const ColumnPtr & start_pos,
         PaddedPODArray<UInt8> & res,
-        ColumnUInt8 * res_null)
+        ColumnUInt8 * res_null,
+        size_t input_rows_count)
     {
         if (start_pos != nullptr)
             throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Function '{}' does not support start_pos argument", name);
 
-        if (haystack_offsets.empty())
+        if (input_rows_count == 0)
             return;
 
         const UInt8 * const begin = haystack_data.data();
diff --git a/src/Functions/MatchImpl.h b/src/Functions/MatchImpl.h
index 55b2fee5400..ceac753fe79 100644
--- a/src/Functions/MatchImpl.h
+++ b/src/Functions/MatchImpl.h
@@ -127,17 +127,17 @@ struct MatchImpl
         const String & needle,
         [[maybe_unused]] const ColumnPtr & start_pos_,
         PaddedPODArray<UInt8> & res,
-        [[maybe_unused]] ColumnUInt8 * res_null)
+        [[maybe_unused]] ColumnUInt8 * res_null,
+        size_t input_rows_count)
     {
         /// `res_null` serves as an output parameter for implementing an XYZOrNull variant.
         assert(!res_null);
 
-        const size_t haystack_size = haystack_offsets.size();
-
-        assert(haystack_size == res.size());
+        assert(res.size() == haystack_offsets.size());
+        assert(res.size() == input_rows_count);
         assert(start_pos_ == nullptr);
 
-        if (haystack_offsets.empty())
+        if (input_rows_count == 0)
             return;
 
         /// Shortcut for the silly but practical case that the pattern matches everything/nothing independently of the haystack:
@@ -202,11 +202,11 @@ struct MatchImpl
         if (required_substring.empty())
         {
             if (!regexp.getRE2()) /// An empty regexp. Always matches.
-                memset(res.data(), !negate, haystack_size * sizeof(res[0]));
+                memset(res.data(), !negate, input_rows_count * sizeof(res[0]));
             else
             {
                 size_t prev_offset = 0;
-                for (size_t i = 0; i < haystack_size; ++i)
+                for (size_t i = 0; i < input_rows_count; ++i)
                 {
                     const bool match = regexp.getRE2()->Match(
                             {reinterpret_cast<const char *>(&haystack_data[prev_offset]), haystack_offsets[i] - prev_offset - 1},
@@ -291,16 +291,16 @@ struct MatchImpl
         size_t N,
         const String & needle,
         PaddedPODArray<UInt8> & res,
-        [[maybe_unused]] ColumnUInt8 * res_null)
+        [[maybe_unused]] ColumnUInt8 * res_null,
+        size_t input_rows_count)
     {
         /// `res_null` serves as an output parameter for implementing an XYZOrNull variant.
         assert(!res_null);
 
-        const size_t haystack_size = haystack.size() / N;
+        assert(res.size() == haystack.size() / N);
+        assert(res.size() == input_rows_count);
 
-        assert(haystack_size == res.size());
-
-        if (haystack.empty())
+        if (input_rows_count == 0)
             return;
 
         /// Shortcut for the silly but practical case that the pattern matches everything/nothing independently of the haystack:
@@ -370,11 +370,11 @@ struct MatchImpl
         if (required_substring.empty())
         {
             if (!regexp.getRE2()) /// An empty regexp. Always matches.
-                memset(res.data(), !negate, haystack_size * sizeof(res[0]));
+                memset(res.data(), !negate, input_rows_count * sizeof(res[0]));
             else
             {
                 size_t offset = 0;
-                for (size_t i = 0; i < haystack_size; ++i)
+                for (size_t i = 0; i < input_rows_count; ++i)
                 {
                     const bool match = regexp.getRE2()->Match(
                             {reinterpret_cast<const char *>(&haystack[offset]), N},
@@ -464,18 +464,18 @@ struct MatchImpl
         const ColumnString::Offsets & needle_offset,
         [[maybe_unused]] const ColumnPtr & start_pos_,
         PaddedPODArray<UInt8> & res,
-        [[maybe_unused]] ColumnUInt8 * res_null)
+        [[maybe_unused]] ColumnUInt8 * res_null,
+        size_t input_rows_count)
     {
         /// `res_null` serves as an output parameter for implementing an XYZOrNull variant.
         assert(!res_null);
 
-        const size_t haystack_size = haystack_offsets.size();
-
-        assert(haystack_size == needle_offset.size());
-        assert(haystack_size == res.size());
+        assert(haystack_offsets.size() == needle_offset.size());
+        assert(res.size() == haystack_offsets.size());
+        assert(res.size() == input_rows_count);
         assert(start_pos_ == nullptr);
 
-        if (haystack_offsets.empty())
+        if (input_rows_count == 0)
             return;
 
         String required_substr;
@@ -488,7 +488,7 @@ struct MatchImpl
         Regexps::LocalCacheTable cache;
         Regexps::RegexpPtr regexp;
 
-        for (size_t i = 0; i < haystack_size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             const auto * const cur_haystack_data = &haystack_data[prev_haystack_offset];
             const size_t cur_haystack_length = haystack_offsets[i] - prev_haystack_offset - 1;
@@ -573,15 +573,15 @@ struct MatchImpl
         const ColumnString::Offsets & needle_offset,
         [[maybe_unused]] const ColumnPtr & start_pos_,
         PaddedPODArray<UInt8> & res,
-        [[maybe_unused]] ColumnUInt8 * res_null)
+        [[maybe_unused]] ColumnUInt8 * res_null,
+        size_t input_rows_count)
     {
         /// `res_null` serves as an output parameter for implementing an XYZOrNull variant.
         assert(!res_null);
 
-        const size_t haystack_size = haystack.size()/N;
-
-        assert(haystack_size == needle_offset.size());
-        assert(haystack_size == res.size());
+        assert(res.size() == input_rows_count);
+        assert(res.size() == haystack.size() / N);
+        assert(res.size() == needle_offset.size());
         assert(start_pos_ == nullptr);
 
         if (haystack.empty())
@@ -597,7 +597,7 @@ struct MatchImpl
         Regexps::LocalCacheTable cache;
         Regexps::RegexpPtr regexp;
 
-        for (size_t i = 0; i < haystack_size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             const auto * const cur_haystack_data = &haystack[prev_haystack_offset];
             const size_t cur_haystack_length = N;
diff --git a/src/Functions/MultiMatchAnyImpl.h b/src/Functions/MultiMatchAnyImpl.h
index 06d9eded9c9..fda752cbacc 100644
--- a/src/Functions/MultiMatchAnyImpl.h
+++ b/src/Functions/MultiMatchAnyImpl.h
@@ -165,7 +165,7 @@ struct MultiMatchAnyImpl
         memset(accum.data(), 0, accum.size());
         for (size_t j = 0; j < needles.size(); ++j)
         {
-            MatchImpl<Name, MatchTraits::Syntax::Re2, MatchTraits::Case::Sensitive, MatchTraits::Result::DontNegate>::vectorConstant(haystack_data, haystack_offsets, String(needles[j].data(), needles[j].size()), nullptr, accum, nullptr);
+            MatchImpl<Name, MatchTraits::Syntax::Re2, MatchTraits::Case::Sensitive, MatchTraits::Result::DontNegate>::vectorConstant(haystack_data, haystack_offsets, String(needles[j].data(), needles[j].size()), nullptr, accum, nullptr, input_rows_count);
             for (size_t i = 0; i < res.size(); ++i)
             {
                 if constexpr (FindAny)
diff --git a/src/Functions/PositionImpl.h b/src/Functions/PositionImpl.h
index eeb9d8b6a59..e525b5fab57 100644
--- a/src/Functions/PositionImpl.h
+++ b/src/Functions/PositionImpl.h
@@ -193,7 +193,8 @@ struct PositionImpl
         const std::string & needle,
         const ColumnPtr & start_pos,
         PaddedPODArray<UInt64> & res,
-        [[maybe_unused]] ColumnUInt8 * res_null)
+        [[maybe_unused]] ColumnUInt8 * res_null,
+        size_t input_rows_count)
     {
         /// `res_null` serves as an output parameter for implementing an XYZOrNull variant.
         assert(!res_null);
@@ -214,13 +215,12 @@ struct PositionImpl
             }
 
             ColumnString::Offset prev_offset = 0;
-            size_t rows = haystack_offsets.size();
 
             if (const ColumnConst * start_pos_const = typeid_cast<const ColumnConst *>(&*start_pos))
             {
                 /// Needle is empty and start_pos is constant
                 UInt64 start = std::max(start_pos_const->getUInt(0), static_cast<UInt64>(1));
-                for (size_t i = 0; i < rows; ++i)
+                for (size_t i = 0; i < input_rows_count; ++i)
                 {
                     size_t haystack_size = Impl::countChars(
                         reinterpret_cast<const char *>(pos), reinterpret_cast<const char *>(pos + haystack_offsets[i] - prev_offset - 1));
@@ -234,7 +234,7 @@ struct PositionImpl
             else
             {
                 /// Needle is empty and start_pos is not constant
-                for (size_t i = 0; i < rows; ++i)
+                for (size_t i = 0; i < input_rows_count; ++i)
                 {
                     size_t haystack_size = Impl::countChars(
                         reinterpret_cast<const char *>(pos), reinterpret_cast<const char *>(pos + haystack_offsets[i] - prev_offset - 1));
@@ -359,7 +359,8 @@ struct PositionImpl
         const ColumnString::Offsets & needle_offsets,
         const ColumnPtr & start_pos,
         PaddedPODArray<UInt64> & res,
-        [[maybe_unused]] ColumnUInt8 * res_null)
+        [[maybe_unused]] ColumnUInt8 * res_null,
+        size_t input_rows_count)
     {
         /// `res_null` serves as an output parameter for implementing an XYZOrNull variant.
         assert(!res_null);
@@ -367,9 +368,7 @@ struct PositionImpl
         ColumnString::Offset prev_haystack_offset = 0;
         ColumnString::Offset prev_needle_offset = 0;
 
-        size_t size = haystack_offsets.size();
-
-        for (size_t i = 0; i < size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             size_t needle_size = needle_offsets[i] - prev_needle_offset - 1;
             size_t haystack_size = haystack_offsets[i] - prev_haystack_offset - 1;
@@ -423,7 +422,8 @@ struct PositionImpl
         const ColumnString::Offsets & needle_offsets,
         const ColumnPtr & start_pos,
         PaddedPODArray<UInt64> & res,
-        [[maybe_unused]] ColumnUInt8 * res_null)
+        [[maybe_unused]] ColumnUInt8 * res_null,
+        size_t input_rows_count)
     {
         /// `res_null` serves as an output parameter for implementing an XYZOrNull variant.
         assert(!res_null);
@@ -431,9 +431,7 @@ struct PositionImpl
         /// NOTE You could use haystack indexing. But this is a rare case.
         ColumnString::Offset prev_needle_offset = 0;
 
-        size_t size = needle_offsets.size();
-
-        for (size_t i = 0; i < size; ++i)
+        for (size_t i = 0; i < input_rows_count; ++i)
         {
             size_t needle_size = needle_offsets[i] - prev_needle_offset - 1;
 

From 59e51de6f8df35a0df0e08e8b3aacd3ba40c4cbd Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Wed, 31 Jul 2024 14:58:01 +0000
Subject: [PATCH 1164/1488] Fix join on tuple with NULLs

---
 .../Passes/ComparisonTupleEliminationPass.cpp    | 10 +++++++++-
 ...on_tuple_comparison_elimination_bug.reference |  8 ++++++++
 ..._join_on_tuple_comparison_elimination_bug.sql | 16 ++++++++++++++++
 3 files changed, 33 insertions(+), 1 deletion(-)
 create mode 100644 tests/queries/0_stateless/03214_join_on_tuple_comparison_elimination_bug.reference
 create mode 100644 tests/queries/0_stateless/03214_join_on_tuple_comparison_elimination_bug.sql

diff --git a/src/Analyzer/Passes/ComparisonTupleEliminationPass.cpp b/src/Analyzer/Passes/ComparisonTupleEliminationPass.cpp
index 180470952cd..76dc8ab94b4 100644
--- a/src/Analyzer/Passes/ComparisonTupleEliminationPass.cpp
+++ b/src/Analyzer/Passes/ComparisonTupleEliminationPass.cpp
@@ -11,6 +11,7 @@
 #include <Analyzer/InDepthQueryTreeVisitor.h>
 #include <Analyzer/ConstantNode.h>
 #include <Analyzer/FunctionNode.h>
+#include <Analyzer/JoinNode.h>
 #include <Analyzer/Utils.h>
 
 namespace DB
@@ -25,8 +26,15 @@ public:
     using Base = InDepthQueryTreeVisitorWithContext<ComparisonTupleEliminationPassVisitor>;
     using Base::Base;
 
-    static bool needChildVisit(QueryTreeNodePtr &, QueryTreeNodePtr & child)
+    static bool needChildVisit(QueryTreeNodePtr & parent, QueryTreeNodePtr & child)
     {
+        if (parent->getNodeType() == QueryTreeNodeType::JOIN)
+        {
+            /// In JOIN ON section comparison of tuples works a bit differently.
+            /// For example we can join on tuple(NULL) = tuple(NULL), join algorithms consider only NULLs on the top level.
+            if (parent->as<const JoinNode &>().getJoinExpression().get() == child.get())
+                return false;
+        }
         return child->getNodeType() != QueryTreeNodeType::TABLE_FUNCTION;
     }
 
diff --git a/tests/queries/0_stateless/03214_join_on_tuple_comparison_elimination_bug.reference b/tests/queries/0_stateless/03214_join_on_tuple_comparison_elimination_bug.reference
new file mode 100644
index 00000000000..c90a64e4a47
--- /dev/null
+++ b/tests/queries/0_stateless/03214_join_on_tuple_comparison_elimination_bug.reference
@@ -0,0 +1,8 @@
+1
+\N
+1
+\N
+1
+\N
+1
+\N
diff --git a/tests/queries/0_stateless/03214_join_on_tuple_comparison_elimination_bug.sql b/tests/queries/0_stateless/03214_join_on_tuple_comparison_elimination_bug.sql
new file mode 100644
index 00000000000..7ef98f88cc7
--- /dev/null
+++ b/tests/queries/0_stateless/03214_join_on_tuple_comparison_elimination_bug.sql
@@ -0,0 +1,16 @@
+DROP TABLE IF EXISTS a;
+DROP TABLE IF EXISTS b;
+
+CREATE TABLE a (key Nullable(String)) ENGINE = MergeTree ORDER BY tuple();
+INSERT INTO a VALUES (NULL), ('1');
+
+CREATE TABLE b (key Nullable(String)) ENGINE = MergeTree ORDER BY tuple();
+INSERT INTO b VALUES (NULL), ('1');
+
+SELECT a.key FROM a LEFT SEMI JOIN b ON tuple(a.key) = tuple(b.key) ORDER BY a.key;
+SELECT a.key FROM a LEFT SEMI JOIN b ON a.key IS NOT DISTINCT FROM b.key ORDER BY a.key;
+SELECT a.key FROM a LEFT SEMI JOIN b ON tuple(a.key) = tuple(b.key) ORDER BY a.key;
+SELECT a.key FROM a LEFT ANY JOIN b ON tuple(a.key) = tuple(b.key) ORDER BY a.key;
+
+DROP TABLE IF EXISTS a;
+DROP TABLE IF EXISTS b;

From ca01c1c5691e4562ae6fc71af7b1867cf39f7ad1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Thu, 1 Aug 2024 11:33:22 +0200
Subject: [PATCH 1165/1488] Fix bad merge

---
 src/Core/SettingsChangesHistory.cpp | 259 ----------------------------
 1 file changed, 259 deletions(-)

diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp
index c3b9805700f..8f73e10c44f 100644
--- a/src/Core/SettingsChangesHistory.cpp
+++ b/src/Core/SettingsChangesHistory.cpp
@@ -497,265 +497,6 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
             {"enable_optimize_predicate_expression", 0, 1, "Optimize predicates to subqueries by default"}
         }
     },
-    {"24.7", {{"output_format_parquet_write_page_index", false, true, "Add a possibility to write page index into parquet files."},
-              {"output_format_binary_encode_types_in_binary_format", false, false, "Added new setting to allow to write type names in binary format in RowBinaryWithNamesAndTypes output format"},
-              {"input_format_binary_decode_types_in_binary_format", false, false, "Added new setting to allow to read type names in binary format in RowBinaryWithNamesAndTypes input format"},
-              {"output_format_native_encode_types_in_binary_format", false, false, "Added new setting to allow to write type names in binary format in Native output format"},
-              {"input_format_native_decode_types_in_binary_format", false, false, "Added new setting to allow to read type names in binary format in Native output format"},
-              {"read_in_order_use_buffering", false, true, "Use buffering before merging while reading in order of primary key"},
-              {"enable_named_columns_in_function_tuple", false, true, "Generate named tuples in function tuple() when all names are unique and can be treated as unquoted identifiers."},
-              {"input_format_json_case_insensitive_column_matching", false, false, "Ignore case when matching JSON keys with CH columns."},
-              {"optimize_trivial_insert_select", true, false, "The optimization does not make sense in many cases."},
-              {"dictionary_validate_primary_key_type", false, false, "Validate primary key type for dictionaries. By default id type for simple layouts will be implicitly converted to UInt64."},
-              {"collect_hash_table_stats_during_joins", false, true, "New setting."},
-              {"max_size_to_preallocate_for_joins", 0, 100'000'000, "New setting."},
-              {"input_format_orc_reader_time_zone_name", "GMT", "GMT", "The time zone name for ORC row reader, the default ORC row reader's time zone is GMT."},
-              {"lightweight_mutation_projection_mode", "throw", "throw", "When lightweight delete happens on a table with projection(s), the possible operations include throw the exception as projection exists, or drop all projection related to this table then do lightweight delete."},
-              {"database_replicated_allow_heavy_create", true, false, "Long-running DDL queries (CREATE AS SELECT and POPULATE) for Replicated database engine was forbidden"},
-              {"query_plan_merge_filters", false, false, "Allow to merge filters in the query plan"},
-              {"azure_sdk_max_retries", 10, 10, "Maximum number of retries in azure sdk"},
-              {"azure_sdk_retry_initial_backoff_ms", 10, 10, "Minimal backoff between retries in azure sdk"},
-              {"azure_sdk_retry_max_backoff_ms", 1000, 1000, "Maximal backoff between retries in azure sdk"},
-              {"merge_tree_min_bytes_per_task_for_remote_reading", 4194304, 2097152, "Value is unified with `filesystem_prefetch_min_bytes_for_single_read_task`"},
-              {"ignore_on_cluster_for_replicated_named_collections_queries", false, false, "Ignore ON CLUSTER clause for replicated named collections management queries."},
-              {"backup_restore_s3_retry_attempts", 1000,1000, "Setting for Aws::Client::RetryStrategy, Aws::Client does retries itself, 0 means no retries. It takes place only for backup/restore."},
-              {"postgresql_connection_attempt_timeout", 2, 2, "Allow to control 'connect_timeout' parameter of PostgreSQL connection."},
-              {"postgresql_connection_pool_retries", 2, 2, "Allow to control the number of retries in PostgreSQL connection pool."}
-              }},
-    {"24.6", {{"materialize_skip_indexes_on_insert", true, true, "Added new setting to allow to disable materialization of skip indexes on insert"},
-              {"materialize_statistics_on_insert", true, true, "Added new setting to allow to disable materialization of statistics on insert"},
-              {"input_format_parquet_use_native_reader", false, false, "When reading Parquet files, to use native reader instead of arrow reader."},
-              {"hdfs_throw_on_zero_files_match", false, false, "Allow to throw an error when ListObjects request cannot match any files in HDFS engine instead of empty query result"},
-              {"azure_throw_on_zero_files_match", false, false, "Allow to throw an error when ListObjects request cannot match any files in AzureBlobStorage engine instead of empty query result"},
-              {"s3_validate_request_settings", true, true, "Allow to disable S3 request settings validation"},
-              {"allow_experimental_full_text_index", false, false, "Enable experimental full-text index"},
-              {"azure_skip_empty_files", false, false, "Allow to skip empty files in azure table engine"},
-              {"hdfs_ignore_file_doesnt_exist", false, false, "Allow to return 0 rows when the requested files don't exist instead of throwing an exception in HDFS table engine"},
-              {"azure_ignore_file_doesnt_exist", false, false, "Allow to return 0 rows when the requested files don't exist instead of throwing an exception in AzureBlobStorage table engine"},
-              {"s3_ignore_file_doesnt_exist", false, false, "Allow to return 0 rows when the requested files don't exist instead of throwing an exception in S3 table engine"},
-              {"s3_max_part_number", 10000, 10000, "Maximum part number number for s3 upload part"},
-              {"s3_max_single_operation_copy_size", 32 * 1024 * 1024, 32 * 1024 * 1024, "Maximum size for a single copy operation in s3"},
-              {"input_format_parquet_max_block_size", 8192, DEFAULT_BLOCK_SIZE, "Increase block size for parquet reader."},
-              {"input_format_parquet_prefer_block_bytes", 0, DEFAULT_BLOCK_SIZE * 256, "Average block bytes output by parquet reader."},
-              {"enable_blob_storage_log", true, true, "Write information about blob storage operations to system.blob_storage_log table"},
-              {"allow_deprecated_snowflake_conversion_functions", true, false, "Disabled deprecated functions snowflakeToDateTime[64] and dateTime[64]ToSnowflake."},
-              {"allow_statistic_optimize", false, false, "Old setting which popped up here being renamed."},
-              {"allow_experimental_statistic", false, false, "Old setting which popped up here being renamed."},
-              {"allow_statistics_optimize", false, false, "The setting was renamed. The previous name is `allow_statistic_optimize`."},
-              {"allow_experimental_statistics", false, false, "The setting was renamed. The previous name is `allow_experimental_statistic`."},
-              {"enable_vertical_final", false, true, "Enable vertical final by default again after fixing bug"},
-              {"parallel_replicas_custom_key_range_lower", 0, 0, "Add settings to control the range filter when using parallel replicas with dynamic shards"},
-              {"parallel_replicas_custom_key_range_upper", 0, 0, "Add settings to control the range filter when using parallel replicas with dynamic shards. A value of 0 disables the upper limit"},
-              {"output_format_pretty_display_footer_column_names", 0, 1, "Add a setting to display column names in the footer if there are many rows. Threshold value is controlled by output_format_pretty_display_footer_column_names_min_rows."},
-              {"output_format_pretty_display_footer_column_names_min_rows", 0, 50, "Add a setting to control the threshold value for setting output_format_pretty_display_footer_column_names_min_rows. Default 50."},
-              {"output_format_csv_serialize_tuple_into_separate_columns", true, true, "A new way of how interpret tuples in CSV format was added."},
-              {"input_format_csv_deserialize_separate_columns_into_tuple", true, true, "A new way of how interpret tuples in CSV format was added."},
-              {"input_format_csv_try_infer_strings_from_quoted_tuples", true, true, "A new way of how interpret tuples in CSV format was added."},
-              }},
-    {"24.5", {{"allow_deprecated_error_prone_window_functions", true, false, "Allow usage of deprecated error prone window functions (neighbor, runningAccumulate, runningDifferenceStartingWithFirstValue, runningDifference)"},
-              {"allow_experimental_join_condition", false, false, "Support join with inequal conditions which involve columns from both left and right table. e.g. t1.y < t2.y."},
-              {"input_format_tsv_crlf_end_of_line", false, false, "Enables reading of CRLF line endings with TSV formats"},
-              {"output_format_parquet_use_custom_encoder", false, true, "Enable custom Parquet encoder."},
-              {"cross_join_min_rows_to_compress", 0, 10000000, "Minimal count of rows to compress block in CROSS JOIN. Zero value means - disable this threshold. This block is compressed when any of the two thresholds (by rows or by bytes) are reached."},
-              {"cross_join_min_bytes_to_compress", 0, 1_GiB, "Minimal size of block to compress in CROSS JOIN. Zero value means - disable this threshold. This block is compressed when any of the two thresholds (by rows or by bytes) are reached."},
-              {"http_max_chunk_size", 0, 0, "Internal limitation"},
-              {"prefer_external_sort_block_bytes", 0, DEFAULT_BLOCK_SIZE * 256, "Prefer maximum block bytes for external sort, reduce the memory usage during merging."},
-              {"input_format_force_null_for_omitted_fields", false, false, "Disable type-defaults for omitted fields when needed"},
-              {"cast_string_to_dynamic_use_inference", false, false, "Add setting to allow converting String to Dynamic through parsing"},
-              {"allow_experimental_dynamic_type", false, false, "Add new experimental Dynamic type"},
-              {"azure_max_blocks_in_multipart_upload", 50000, 50000, "Maximum number of blocks in multipart upload for Azure."},
-              }},
-    {"24.4", {{"input_format_json_throw_on_bad_escape_sequence", true, true, "Allow to save JSON strings with bad escape sequences"},
-              {"max_parsing_threads", 0, 0, "Add a separate setting to control number of threads in parallel parsing from files"},
-              {"ignore_drop_queries_probability", 0, 0, "Allow to ignore drop queries in server with specified probability for testing purposes"},
-              {"lightweight_deletes_sync", 2, 2, "The same as 'mutation_sync', but controls only execution of lightweight deletes"},
-              {"query_cache_system_table_handling", "save", "throw", "The query cache no longer caches results of queries against system tables"},
-              {"input_format_json_ignore_unnecessary_fields", false, true, "Ignore unnecessary fields and not parse them. Enabling this may not throw exceptions on json strings of invalid format or with duplicated fields"},
-              {"input_format_hive_text_allow_variable_number_of_columns", false, true, "Ignore extra columns in Hive Text input (if file has more columns than expected) and treat missing fields in Hive Text input as default values."},
-              {"allow_experimental_database_replicated", false, true, "Database engine Replicated is now in Beta stage"},
-              {"temporary_data_in_cache_reserve_space_wait_lock_timeout_milliseconds", (10 * 60 * 1000), (10 * 60 * 1000), "Wait time to lock cache for sapce reservation in temporary data in filesystem cache"},
-              {"optimize_rewrite_sum_if_to_count_if", false, true, "Only available for the analyzer, where it works correctly"},
-              {"azure_allow_parallel_part_upload", "true", "true", "Use multiple threads for azure multipart upload."},
-              {"max_recursive_cte_evaluation_depth", DBMS_RECURSIVE_CTE_MAX_EVALUATION_DEPTH, DBMS_RECURSIVE_CTE_MAX_EVALUATION_DEPTH, "Maximum limit on recursive CTE evaluation depth"},
-              {"query_plan_convert_outer_join_to_inner_join", false, true, "Allow to convert OUTER JOIN to INNER JOIN if filter after JOIN always filters default values"},
-              }},
-    {"24.3", {{"s3_connect_timeout_ms", 1000, 1000, "Introduce new dedicated setting for s3 connection timeout"},
-              {"allow_experimental_shared_merge_tree", false, true, "The setting is obsolete"},
-              {"use_page_cache_for_disks_without_file_cache", false, false, "Added userspace page cache"},
-              {"read_from_page_cache_if_exists_otherwise_bypass_cache", false, false, "Added userspace page cache"},
-              {"page_cache_inject_eviction", false, false, "Added userspace page cache"},
-              {"default_table_engine", "None", "MergeTree", "Set default table engine to MergeTree for better usability"},
-              {"input_format_json_use_string_type_for_ambiguous_paths_in_named_tuples_inference_from_objects", false, false, "Allow to use String type for ambiguous paths during named tuple inference from JSON objects"},
-              {"traverse_shadow_remote_data_paths", false, false, "Traverse shadow directory when query system.remote_data_paths."},
-              {"throw_if_deduplication_in_dependent_materialized_views_enabled_with_async_insert", false, true, "Deduplication in dependent materialized view cannot work together with async inserts."},
-              {"parallel_replicas_allow_in_with_subquery", false, true, "If true, subquery for IN will be executed on every follower replica"},
-              {"log_processors_profiles", false, true, "Enable by default"},
-              {"function_locate_has_mysql_compatible_argument_order", false, true, "Increase compatibility with MySQL's locate function."},
-              {"allow_suspicious_primary_key", true, false, "Forbid suspicious PRIMARY KEY/ORDER BY for MergeTree (i.e. SimpleAggregateFunction)"},
-              {"filesystem_cache_reserve_space_wait_lock_timeout_milliseconds", 1000, 1000, "Wait time to lock cache for sapce reservation in filesystem cache"},
-              {"max_parser_backtracks", 0, 1000000, "Limiting the complexity of parsing"},
-              {"analyzer_compatibility_join_using_top_level_identifier", false, false, "Force to resolve identifier in JOIN USING from projection"},
-              {"distributed_insert_skip_read_only_replicas", false, false, "If true, INSERT into Distributed will skip read-only replicas"},
-              {"keeper_max_retries", 10, 10, "Max retries for general keeper operations"},
-              {"keeper_retry_initial_backoff_ms", 100, 100, "Initial backoff timeout for general keeper operations"},
-              {"keeper_retry_max_backoff_ms", 5000, 5000, "Max backoff timeout for general keeper operations"},
-              {"s3queue_allow_experimental_sharded_mode", false, false, "Enable experimental sharded mode of S3Queue table engine. It is experimental because it will be rewritten"},
-              {"allow_experimental_analyzer", false, true, "Enable analyzer and planner by default."},
-              {"merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability", 0.0, 0.0, "For testing of `PartsSplitter` - split read ranges into intersecting and non intersecting every time you read from MergeTree with the specified probability."},
-              {"allow_get_client_http_header", false, false, "Introduced a new function."},
-              {"output_format_pretty_row_numbers", false, true, "It is better for usability."},
-              {"output_format_pretty_max_value_width_apply_for_single_value", true, false, "Single values in Pretty formats won't be cut."},
-              {"output_format_parquet_string_as_string", false, true, "ClickHouse allows arbitrary binary data in the String data type, which is typically UTF-8. Parquet/ORC/Arrow Strings only support UTF-8. That's why you can choose which Arrow's data type to use for the ClickHouse String data type - String or Binary. While Binary would be more correct and compatible, using String by default will correspond to user expectations in most cases."},
-              {"output_format_orc_string_as_string", false, true, "ClickHouse allows arbitrary binary data in the String data type, which is typically UTF-8. Parquet/ORC/Arrow Strings only support UTF-8. That's why you can choose which Arrow's data type to use for the ClickHouse String data type - String or Binary. While Binary would be more correct and compatible, using String by default will correspond to user expectations in most cases."},
-              {"output_format_arrow_string_as_string", false, true, "ClickHouse allows arbitrary binary data in the String data type, which is typically UTF-8. Parquet/ORC/Arrow Strings only support UTF-8. That's why you can choose which Arrow's data type to use for the ClickHouse String data type - String or Binary. While Binary would be more correct and compatible, using String by default will correspond to user expectations in most cases."},
-              {"output_format_parquet_compression_method", "lz4", "zstd", "Parquet/ORC/Arrow support many compression methods, including lz4 and zstd. ClickHouse supports each and every compression method. Some inferior tools, such as 'duckdb', lack support for the faster `lz4` compression method, that's why we set zstd by default."},
-              {"output_format_orc_compression_method", "lz4", "zstd", "Parquet/ORC/Arrow support many compression methods, including lz4 and zstd. ClickHouse supports each and every compression method. Some inferior tools, such as 'duckdb', lack support for the faster `lz4` compression method, that's why we set zstd by default."},
-              {"output_format_pretty_highlight_digit_groups", false, true, "If enabled and if output is a terminal, highlight every digit corresponding to the number of thousands, millions, etc. with underline."},
-              {"geo_distance_returns_float64_on_float64_arguments", false, true, "Increase the default precision."},
-              {"azure_max_inflight_parts_for_one_file", 20, 20, "The maximum number of a concurrent loaded parts in multipart upload request. 0 means unlimited."},
-              {"azure_strict_upload_part_size", 0, 0, "The exact size of part to upload during multipart upload to Azure blob storage."},
-              {"azure_min_upload_part_size", 16*1024*1024, 16*1024*1024, "The minimum size of part to upload during multipart upload to Azure blob storage."},
-              {"azure_max_upload_part_size", 5ull*1024*1024*1024, 5ull*1024*1024*1024, "The maximum size of part to upload during multipart upload to Azure blob storage."},
-              {"azure_upload_part_size_multiply_factor", 2, 2, "Multiply azure_min_upload_part_size by this factor each time azure_multiply_parts_count_threshold parts were uploaded from a single write to Azure blob storage."},
-              {"azure_upload_part_size_multiply_parts_count_threshold", 500, 500, "Each time this number of parts was uploaded to Azure blob storage, azure_min_upload_part_size is multiplied by azure_upload_part_size_multiply_factor."},
-              {"output_format_csv_serialize_tuple_into_separate_columns", true, true, "A new way of how interpret tuples in CSV format was added."},
-              {"input_format_csv_deserialize_separate_columns_into_tuple", true, true, "A new way of how interpret tuples in CSV format was added."},
-              {"input_format_csv_try_infer_strings_from_quoted_tuples", true, true, "A new way of how interpret tuples in CSV format was added."},
-              }},
-    {"24.2", {{"allow_suspicious_variant_types", true, false, "Don't allow creating Variant type with suspicious variants by default"},
-              {"validate_experimental_and_suspicious_types_inside_nested_types", false, true, "Validate usage of experimental and suspicious types inside nested types"},
-              {"output_format_values_escape_quote_with_quote", false, false, "If true escape ' with '', otherwise quoted with \\'"},
-              {"output_format_pretty_single_large_number_tip_threshold", 0, 1'000'000, "Print a readable number tip on the right side of the table if the block consists of a single number which exceeds this value (except 0)"},
-              {"input_format_try_infer_exponent_floats", true, false, "Don't infer floats in exponential notation by default"},
-              {"query_plan_optimize_prewhere", true, true, "Allow to push down filter to PREWHERE expression for supported storages"},
-              {"async_insert_max_data_size", 1000000, 10485760, "The previous value appeared to be too small."},
-              {"async_insert_poll_timeout_ms", 10, 10, "Timeout in milliseconds for polling data from asynchronous insert queue"},
-              {"async_insert_use_adaptive_busy_timeout", false, true, "Use adaptive asynchronous insert timeout"},
-              {"async_insert_busy_timeout_min_ms", 50, 50, "The minimum value of the asynchronous insert timeout in milliseconds; it also serves as the initial value, which may be increased later by the adaptive algorithm"},
-              {"async_insert_busy_timeout_max_ms", 200, 200, "The minimum value of the asynchronous insert timeout in milliseconds; async_insert_busy_timeout_ms is aliased to async_insert_busy_timeout_max_ms"},
-              {"async_insert_busy_timeout_increase_rate", 0.2, 0.2, "The exponential growth rate at which the adaptive asynchronous insert timeout increases"},
-              {"async_insert_busy_timeout_decrease_rate", 0.2, 0.2, "The exponential growth rate at which the adaptive asynchronous insert timeout decreases"},
-              {"format_template_row_format", "", "", "Template row format string can be set directly in query"},
-              {"format_template_resultset_format", "", "", "Template result set format string can be set in query"},
-              {"split_parts_ranges_into_intersecting_and_non_intersecting_final", true, true, "Allow to split parts ranges into intersecting and non intersecting during FINAL optimization"},
-              {"split_intersecting_parts_ranges_into_layers_final", true, true, "Allow to split intersecting parts ranges into layers during FINAL optimization"},
-              {"azure_max_single_part_copy_size", 256*1024*1024, 256*1024*1024, "The maximum size of object to copy using single part copy to Azure blob storage."},
-              {"min_external_table_block_size_rows", DEFAULT_INSERT_BLOCK_SIZE, DEFAULT_INSERT_BLOCK_SIZE, "Squash blocks passed to external table to specified size in rows, if blocks are not big enough"},
-              {"min_external_table_block_size_bytes", DEFAULT_INSERT_BLOCK_SIZE * 256, DEFAULT_INSERT_BLOCK_SIZE * 256, "Squash blocks passed to external table to specified size in bytes, if blocks are not big enough."},
-              {"parallel_replicas_prefer_local_join", true, true, "If true, and JOIN can be executed with parallel replicas algorithm, and all storages of right JOIN part are *MergeTree, local JOIN will be used instead of GLOBAL JOIN."},
-              {"optimize_time_filter_with_preimage", true, true, "Optimize Date and DateTime predicates by converting functions into equivalent comparisons without conversions (e.g. toYear(col) = 2023 -> col >= '2023-01-01' AND col <= '2023-12-31')"},
-              {"extract_key_value_pairs_max_pairs_per_row", 0, 0, "Max number of pairs that can be produced by the `extractKeyValuePairs` function. Used as a safeguard against consuming too much memory."},
-              {"default_view_definer", "CURRENT_USER", "CURRENT_USER", "Allows to set default `DEFINER` option while creating a view"},
-              {"default_materialized_view_sql_security", "DEFINER", "DEFINER", "Allows to set a default value for SQL SECURITY option when creating a materialized view"},
-              {"default_normal_view_sql_security", "INVOKER", "INVOKER", "Allows to set default `SQL SECURITY` option while creating a normal view"},
-              {"mysql_map_string_to_text_in_show_columns", false, true, "Reduce the configuration effort to connect ClickHouse with BI tools."},
-              {"mysql_map_fixed_string_to_text_in_show_columns", false, true, "Reduce the configuration effort to connect ClickHouse with BI tools."},
-              }},
-    {"24.1", {{"print_pretty_type_names", false, true, "Better user experience."},
-              {"input_format_json_read_bools_as_strings", false, true, "Allow to read bools as strings in JSON formats by default"},
-              {"output_format_arrow_use_signed_indexes_for_dictionary", false, true, "Use signed indexes type for Arrow dictionaries by default as it's recommended"},
-              {"allow_experimental_variant_type", false, false, "Add new experimental Variant type"},
-              {"use_variant_as_common_type", false, false, "Allow to use Variant in if/multiIf if there is no common type"},
-              {"output_format_arrow_use_64_bit_indexes_for_dictionary", false, false, "Allow to use 64 bit indexes type in Arrow dictionaries"},
-              {"parallel_replicas_mark_segment_size", 128, 128, "Add new setting to control segment size in new parallel replicas coordinator implementation"},
-              {"ignore_materialized_views_with_dropped_target_table", false, false, "Add new setting to allow to ignore materialized views with dropped target table"},
-              {"output_format_compression_level", 3, 3, "Allow to change compression level in the query output"},
-              {"output_format_compression_zstd_window_log", 0, 0, "Allow to change zstd window log in the query output when zstd compression is used"},
-              {"enable_zstd_qat_codec", false, false, "Add new ZSTD_QAT codec"},
-              {"enable_vertical_final", false, true, "Use vertical final by default"},
-              {"output_format_arrow_use_64_bit_indexes_for_dictionary", false, false, "Allow to use 64 bit indexes type in Arrow dictionaries"},
-              {"max_rows_in_set_to_optimize_join", 100000, 0, "Disable join optimization as it prevents from read in order optimization"},
-              {"output_format_pretty_color", true, "auto", "Setting is changed to allow also for auto value, disabling ANSI escapes if output is not a tty"},
-              {"function_visible_width_behavior", 0, 1, "We changed the default behavior of `visibleWidth` to be more precise"},
-              {"max_estimated_execution_time", 0, 0, "Separate max_execution_time and max_estimated_execution_time"},
-              {"iceberg_engine_ignore_schema_evolution", false, false, "Allow to ignore schema evolution in Iceberg table engine"},
-              {"optimize_injective_functions_in_group_by", false, true, "Replace injective functions by it's arguments in GROUP BY section in analyzer"},
-              {"update_insert_deduplication_token_in_dependent_materialized_views", false, false, "Allow to update insert deduplication token with table identifier during insert in dependent materialized views"},
-              {"azure_max_unexpected_write_error_retries", 4, 4, "The maximum number of retries in case of unexpected errors during Azure blob storage write"},
-              {"split_parts_ranges_into_intersecting_and_non_intersecting_final", false, true, "Allow to split parts ranges into intersecting and non intersecting during FINAL optimization"},
-              {"split_intersecting_parts_ranges_into_layers_final", true, true, "Allow to split intersecting parts ranges into layers during FINAL optimization"}}},
-    {"23.12", {{"allow_suspicious_ttl_expressions", true, false, "It is a new setting, and in previous versions the behavior was equivalent to allowing."},
-              {"input_format_parquet_allow_missing_columns", false, true, "Allow missing columns in Parquet files by default"},
-              {"input_format_orc_allow_missing_columns", false, true, "Allow missing columns in ORC files by default"},
-              {"input_format_arrow_allow_missing_columns", false, true, "Allow missing columns in Arrow files by default"}}},
-    {"23.11", {{"parsedatetime_parse_without_leading_zeros", false, true, "Improved compatibility with MySQL DATE_FORMAT/STR_TO_DATE"}}},
-    {"23.9", {{"optimize_group_by_constant_keys", false, true, "Optimize group by constant keys by default"},
-              {"input_format_json_try_infer_named_tuples_from_objects", false, true, "Try to infer named Tuples from JSON objects by default"},
-              {"input_format_json_read_numbers_as_strings", false, true, "Allow to read numbers as strings in JSON formats by default"},
-              {"input_format_json_read_arrays_as_strings", false, true, "Allow to read arrays as strings in JSON formats by default"},
-              {"input_format_json_infer_incomplete_types_as_strings", false, true, "Allow to infer incomplete types as Strings in JSON formats by default"},
-              {"input_format_json_try_infer_numbers_from_strings", true, false, "Don't infer numbers from strings in JSON formats by default to prevent possible parsing errors"},
-              {"http_write_exception_in_output_format", false, true, "Output valid JSON/XML on exception in HTTP streaming."}}},
-    {"23.8", {{"rewrite_count_distinct_if_with_count_distinct_implementation", false, true, "Rewrite countDistinctIf with count_distinct_implementation configuration"}}},
-    {"23.7", {{"function_sleep_max_microseconds_per_block", 0, 3000000, "In previous versions, the maximum sleep time of 3 seconds was applied only for `sleep`, but not for `sleepEachRow` function. In the new version, we introduce this setting. If you set compatibility with the previous versions, we will disable the limit altogether."}}},
-    {"23.6", {{"http_send_timeout", 180, 30, "3 minutes seems crazy long. Note that this is timeout for a single network write call, not for the whole upload operation."},
-              {"http_receive_timeout", 180, 30, "See http_send_timeout."}}},
-    {"23.5", {{"input_format_parquet_preserve_order", true, false, "Allow Parquet reader to reorder rows for better parallelism."},
-              {"parallelize_output_from_storages", false, true, "Allow parallelism when executing queries that read from file/url/s3/etc. This may reorder rows."},
-              {"use_with_fill_by_sorting_prefix", false, true, "Columns preceding WITH FILL columns in ORDER BY clause form sorting prefix. Rows with different values in sorting prefix are filled independently"},
-              {"output_format_parquet_compliant_nested_types", false, true, "Change an internal field name in output Parquet file schema."}}},
-    {"23.4", {{"allow_suspicious_indices", true, false, "If true, index can defined with identical expressions"},
-              {"allow_nonconst_timezone_arguments", true, false, "Allow non-const timezone arguments in certain time-related functions like toTimeZone(), fromUnixTimestamp*(), snowflakeToDateTime*()."},
-              {"connect_timeout_with_failover_ms", 50, 1000, "Increase default connect timeout because of async connect"},
-              {"connect_timeout_with_failover_secure_ms", 100, 1000, "Increase default secure connect timeout because of async connect"},
-              {"hedged_connection_timeout_ms", 100, 50, "Start new connection in hedged requests after 50 ms instead of 100 to correspond with previous connect timeout"},
-              {"formatdatetime_f_prints_single_zero", true, false, "Improved compatibility with MySQL DATE_FORMAT()/STR_TO_DATE()"},
-              {"formatdatetime_parsedatetime_m_is_month_name", false, true, "Improved compatibility with MySQL DATE_FORMAT/STR_TO_DATE"}}},
-    {"23.3", {{"output_format_parquet_version", "1.0", "2.latest", "Use latest Parquet format version for output format"},
-              {"input_format_json_ignore_unknown_keys_in_named_tuple", false, true, "Improve parsing JSON objects as named tuples"},
-              {"input_format_native_allow_types_conversion", false, true, "Allow types conversion in Native input forma"},
-              {"output_format_arrow_compression_method", "none", "lz4_frame", "Use lz4 compression in Arrow output format by default"},
-              {"output_format_parquet_compression_method", "snappy", "lz4", "Use lz4 compression in Parquet output format by default"},
-              {"output_format_orc_compression_method", "none", "lz4_frame", "Use lz4 compression in ORC output format by default"},
-              {"async_query_sending_for_remote", false, true, "Create connections and send query async across shards"}}},
-    {"23.2", {{"output_format_parquet_fixed_string_as_fixed_byte_array", false, true, "Use Parquet FIXED_LENGTH_BYTE_ARRAY type for FixedString by default"},
-              {"output_format_arrow_fixed_string_as_fixed_byte_array", false, true, "Use Arrow FIXED_SIZE_BINARY type for FixedString by default"},
-              {"query_plan_remove_redundant_distinct", false, true, "Remove redundant Distinct step in query plan"},
-              {"optimize_duplicate_order_by_and_distinct", true, false, "Remove duplicate ORDER BY and DISTINCT if it's possible"},
-              {"insert_keeper_max_retries", 0, 20, "Enable reconnections to Keeper on INSERT, improve reliability"}}},
-    {"23.1", {{"input_format_json_read_objects_as_strings", 0, 1, "Enable reading nested json objects as strings while object type is experimental"},
-              {"input_format_json_defaults_for_missing_elements_in_named_tuple", false, true, "Allow missing elements in JSON objects while reading named tuples by default"},
-              {"input_format_csv_detect_header", false, true, "Detect header in CSV format by default"},
-              {"input_format_tsv_detect_header", false, true, "Detect header in TSV format by default"},
-              {"input_format_custom_detect_header", false, true, "Detect header in CustomSeparated format by default"},
-              {"query_plan_remove_redundant_sorting", false, true, "Remove redundant sorting in query plan. For example, sorting steps related to ORDER BY clauses in subqueries"}}},
-    {"22.12", {{"max_size_to_preallocate_for_aggregation", 10'000'000, 100'000'000, "This optimizes performance"},
-               {"query_plan_aggregation_in_order", 0, 1, "Enable some refactoring around query plan"},
-               {"format_binary_max_string_size", 0, 1_GiB, "Prevent allocating large amount of memory"}}},
-    {"22.11", {{"use_structure_from_insertion_table_in_table_functions", 0, 2, "Improve using structure from insertion table in table functions"}}},
-    {"22.9", {{"force_grouping_standard_compatibility", false, true, "Make GROUPING function output the same as in SQL standard and other DBMS"}}},
-    {"22.7", {{"cross_to_inner_join_rewrite", 1, 2, "Force rewrite comma join to inner"},
-              {"enable_positional_arguments", false, true, "Enable positional arguments feature by default"},
-              {"format_csv_allow_single_quotes", true, false, "Most tools don't treat single quote in CSV specially, don't do it by default too"}}},
-    {"22.6", {{"output_format_json_named_tuples_as_objects", false, true, "Allow to serialize named tuples as JSON objects in JSON formats by default"},
-              {"input_format_skip_unknown_fields", false, true, "Optimize reading subset of columns for some input formats"}}},
-    {"22.5", {{"memory_overcommit_ratio_denominator", 0, 1073741824, "Enable memory overcommit feature by default"},
-              {"memory_overcommit_ratio_denominator_for_user", 0, 1073741824, "Enable memory overcommit feature by default"}}},
-    {"22.4", {{"allow_settings_after_format_in_insert", true, false, "Do not allow SETTINGS after FORMAT for INSERT queries because ClickHouse interpret SETTINGS as some values, which is misleading"}}},
-    {"22.3", {{"cast_ipv4_ipv6_default_on_conversion_error", true, false, "Make functions cast(value, 'IPv4') and cast(value, 'IPv6') behave same as toIPv4 and toIPv6 functions"}}},
-    {"21.12", {{"stream_like_engine_allow_direct_select", true, false, "Do not allow direct select for Kafka/RabbitMQ/FileLog by default"}}},
-    {"21.9", {{"output_format_decimal_trailing_zeros", true, false, "Do not output trailing zeros in text representation of Decimal types by default for better looking output"},
-              {"use_hedged_requests", false, true, "Enable Hedged Requests feature by default"}}},
-    {"21.7", {{"legacy_column_name_of_tuple_literal", true, false, "Add this setting only for compatibility reasons. It makes sense to set to 'true', while doing rolling update of cluster from version lower than 21.7 to higher"}}},
-    {"21.5", {{"async_socket_for_remote", false, true, "Fix all problems and turn on asynchronous reads from socket for remote queries by default again"}}},
-    {"21.3", {{"async_socket_for_remote", true, false, "Turn off asynchronous reads from socket for remote queries because of some problems"},
-              {"optimize_normalize_count_variants", false, true, "Rewrite aggregate functions that semantically equals to count() as count() by default"},
-              {"normalize_function_names", false, true, "Normalize function names to their canonical names, this was needed for projection query routing"}}},
-    {"21.2", {{"enable_global_with_statement", false, true, "Propagate WITH statements to UNION queries and all subqueries by default"}}},
-    {"21.1", {{"insert_quorum_parallel", false, true, "Use parallel quorum inserts by default. It is significantly more convenient to use than sequential quorum inserts"},
-              {"input_format_null_as_default", false, true, "Allow to insert NULL as default for input formats by default"},
-              {"optimize_on_insert", false, true, "Enable data optimization on INSERT by default for better user experience"},
-              {"use_compact_format_in_distributed_parts_names", false, true, "Use compact format for async INSERT into Distributed tables by default"}}},
-    {"20.10", {{"format_regexp_escaping_rule", "Escaped", "Raw", "Use Raw as default escaping rule for Regexp format to male the behaviour more like to what users expect"}}},
-    {"20.7", {{"show_table_uuid_in_table_create_query_if_not_nil", true, false, "Stop showing  UID of the table in its CREATE query for Engine=Atomic"}}},
-    {"20.5", {{"input_format_with_names_use_header", false, true, "Enable using header with names for formats with WithNames/WithNamesAndTypes suffixes"},
-              {"allow_suspicious_codecs", true, false, "Don't allow to specify meaningless compression codecs"}}},
-    {"20.4", {{"validate_polygons", false, true, "Throw exception if polygon is invalid in function pointInPolygon by default instead of returning possibly wrong results"}}},
-    {"19.18", {{"enable_scalar_subquery_optimization", false, true, "Prevent scalar subqueries from (de)serializing large scalar values and possibly avoid running the same subquery more than once"}}},
-    {"19.14", {{"any_join_distinct_right_table_keys", true, false, "Disable ANY RIGHT and ANY FULL JOINs by default to avoid inconsistency"}}},
-    {"19.12", {{"input_format_defaults_for_omitted_fields", false, true, "Enable calculation of complex default expressions for omitted fields for some input formats, because it should be the expected behaviour"}}},
-    {"19.5", {{"max_partitions_per_insert_block", 0, 100, "Add a limit for the number of partitions in one block"}}},
-    {"18.12.17", {{"enable_optimize_predicate_expression", 0, 1, "Optimize predicates to subqueries by default"}}},
 };
 
 

From 4768e3878552ae0ce9007c1e4f400943a5712825 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Thu, 1 Aug 2024 11:37:42 +0200
Subject: [PATCH 1166/1488] Update ref to 24.7.2

---
 ...{02995_baseline_24_7_1.tsv => 02995_baseline_24_7_2.tsv} | 2 +-
 tests/queries/0_stateless/02995_new_settings_history.sh     | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)
 rename tests/queries/0_stateless/{02995_baseline_24_7_1.tsv => 02995_baseline_24_7_2.tsv} (99%)

diff --git a/tests/queries/0_stateless/02995_baseline_24_7_1.tsv b/tests/queries/0_stateless/02995_baseline_24_7_2.tsv
similarity index 99%
rename from tests/queries/0_stateless/02995_baseline_24_7_1.tsv
rename to tests/queries/0_stateless/02995_baseline_24_7_2.tsv
index 6c830da8646..10b392f3e04 100644
--- a/tests/queries/0_stateless/02995_baseline_24_7_1.tsv
+++ b/tests/queries/0_stateless/02995_baseline_24_7_2.tsv
@@ -462,7 +462,7 @@ input_format_orc_allow_missing_columns	1
 input_format_orc_case_insensitive_column_matching	0
 input_format_orc_filter_push_down	1
 input_format_orc_import_nested	0
-input_format_orc_read_use_writer_time_zone	0
+input_format_orc_reader_time_zone_name	GMT
 input_format_orc_row_batch_size	100000
 input_format_orc_skip_columns_with_unsupported_types_in_schema_inference	0
 input_format_orc_use_fast_decoder	1
diff --git a/tests/queries/0_stateless/02995_new_settings_history.sh b/tests/queries/0_stateless/02995_new_settings_history.sh
index 917dacc04b0..7fb21f88fae 100755
--- a/tests/queries/0_stateless/02995_new_settings_history.sh
+++ b/tests/queries/0_stateless/02995_new_settings_history.sh
@@ -7,12 +7,12 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CUR_DIR"/../shell_config.sh
 
 # Note that this is a broad check. A per version check is done in the upgrade test
-# Baseline generated with 24.7.1
-# clickhouse local --query "select name, default from system.settings order by name format TSV" > 02995_baseline_24_7_1.tsv
+# Baseline generated with 24.7.2
+# clickhouse local --query "select name, default from system.settings order by name format TSV" > 02995_baseline_24_7_2.tsv
 $CLICKHOUSE_LOCAL --query "
     WITH old_settings AS
     (
-        SELECT * FROM file('${CUR_DIR}/02995_baseline_24_7_1.tsv', 'TSV', 'name String, default String')
+        SELECT * FROM file('${CUR_DIR}/02995_baseline_24_7_2.tsv', 'TSV', 'name String, default String')
     ),
     new_settings AS
     (

From da3a37c561679daaecbcdece74f92ce98380b2b5 Mon Sep 17 00:00:00 2001
From: Pablo Marcos <pablo.marcos.oltra@clickhouse.com>
Date: Thu, 1 Aug 2024 09:38:27 +0000
Subject: [PATCH 1167/1488] Improve regex to take into account the xdist name
 in the instance

---
 .../test.py                                   | 24 +++++++++----------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/tests/integration/test_zookeeper_config_load_balancing/test.py b/tests/integration/test_zookeeper_config_load_balancing/test.py
index 9cdf7db2b08..cc0a9022674 100644
--- a/tests/integration/test_zookeeper_config_load_balancing/test.py
+++ b/tests/integration/test_zookeeper_config_load_balancing/test.py
@@ -71,7 +71,7 @@ def test_first_or_random(started_cluster):
                     [
                         "bash",
                         "-c",
-                        "lsof -a -i4 -i6 -itcp -w | grep 'testzookeeperconfigloadbalancing_zoo1_1.*testzookeeperconfigloadbalancing_default:2181' | grep ESTABLISHED | wc -l",
+                        "lsof -a -i4 -i6 -itcp -w | grep -P 'testzookeeperconfigloadbalancing_(gw\\d+_)?zoo1_1.*testzookeeperconfigloadbalancing_(gw\\d+_)?default:2181' | grep ESTABLISHED | wc -l",
                     ],
                     privileged=True,
                     user="root",
@@ -99,7 +99,7 @@ def test_first_or_random(started_cluster):
                     [
                         "bash",
                         "-c",
-                        "lsof -a -i4 -i6 -itcp -w | grep 'testzookeeperconfigloadbalancing_zoo1_1.*testzookeeperconfigloadbalancing_default:2181' | grep ESTABLISHED | wc -l",
+                        "lsof -a -i4 -i6 -itcp -w | grep -P 'testzookeeperconfigloadbalancing_(gw\\d+_)?zoo1_1.*testzookeeperconfigloadbalancing_(gw\\d+_)?default:2181' | grep ESTABLISHED | wc -l",
                     ],
                     privileged=True,
                     user="root",
@@ -127,7 +127,7 @@ def test_first_or_random(started_cluster):
                     [
                         "bash",
                         "-c",
-                        "lsof -a -i4 -i6 -itcp -w | grep 'testzookeeperconfigloadbalancing_zoo1_1.*testzookeeperconfigloadbalancing_default:2181' | grep ESTABLISHED | wc -l",
+                        "lsof -a -i4 -i6 -itcp -w | grep -P 'testzookeeperconfigloadbalancing_(gw\\d+_)?zoo1_1.*testzookeeperconfigloadbalancing_(gw\\d+_)?default:2181' | grep ESTABLISHED | wc -l",
                     ],
                     privileged=True,
                     user="root",
@@ -161,7 +161,7 @@ def test_in_order(started_cluster):
                     [
                         "bash",
                         "-c",
-                        "lsof -a -i4 -i6 -itcp -w | grep 'testzookeeperconfigloadbalancing_zoo1_1.*testzookeeperconfigloadbalancing_default:2181' | grep ESTABLISHED | wc -l",
+                        "lsof -a -i4 -i6 -itcp -w | grep -P 'testzookeeperconfigloadbalancing_(gw\\d+_)?zoo1_1.*testzookeeperconfigloadbalancing_(gw\\d+_)?default:2181' | grep ESTABLISHED | wc -l",
                     ],
                     privileged=True,
                     user="root",
@@ -189,7 +189,7 @@ def test_in_order(started_cluster):
                     [
                         "bash",
                         "-c",
-                        "lsof -a -i4 -i6 -itcp -w | grep 'testzookeeperconfigloadbalancing_zoo1_1.*testzookeeperconfigloadbalancing_default:2181' | grep ESTABLISHED | wc -l",
+                        "lsof -a -i4 -i6 -itcp -w | grep -P 'testzookeeperconfigloadbalancing_(gw\\d+_)?zoo1_1.*testzookeeperconfigloadbalancing_(gw\\d+_)?default:2181' | grep ESTABLISHED | wc -l",
                     ],
                     privileged=True,
                     user="root",
@@ -217,7 +217,7 @@ def test_in_order(started_cluster):
                     [
                         "bash",
                         "-c",
-                        "lsof -a -i4 -i6 -itcp -w | grep 'testzookeeperconfigloadbalancing_zoo1_1.*testzookeeperconfigloadbalancing_default:2181' | grep ESTABLISHED | wc -l",
+                        "lsof -a -i4 -i6 -itcp -w | grep -P 'testzookeeperconfigloadbalancing_(gw\\d+_)?zoo1_1.*testzookeeperconfigloadbalancing_(gw\\d+_)?default:2181' | grep ESTABLISHED | wc -l",
                     ],
                     privileged=True,
                     user="root",
@@ -251,7 +251,7 @@ def test_nearest_hostname(started_cluster):
                     [
                         "bash",
                         "-c",
-                        "lsof -a -i4 -i6 -itcp -w | grep 'testzookeeperconfigloadbalancing_zoo1_1.*testzookeeperconfigloadbalancing_default:2181' | grep ESTABLISHED | wc -l",
+                        "lsof -a -i4 -i6 -itcp -w | grep -P 'testzookeeperconfigloadbalancing_(gw\\d+_)?zoo1_1.*testzookeeperconfigloadbalancing_(gw\\d+_)?default:2181' | grep ESTABLISHED | wc -l",
                     ],
                     privileged=True,
                     user="root",
@@ -279,7 +279,7 @@ def test_nearest_hostname(started_cluster):
                     [
                         "bash",
                         "-c",
-                        "lsof -a -i4 -i6 -itcp -w | grep 'testzookeeperconfigloadbalancing_zoo2_1.*testzookeeperconfigloadbalancing_default:2181' | grep ESTABLISHED | wc -l",
+                        "lsof -a -i4 -i6 -itcp -w | grep -P 'testzookeeperconfigloadbalancing_(gw\\d+_)?zoo2_1.*testzookeeperconfigloadbalancing_(gw\\d+_)?default:2181' | grep ESTABLISHED | wc -l",
                     ],
                     privileged=True,
                     user="root",
@@ -307,7 +307,7 @@ def test_nearest_hostname(started_cluster):
                     [
                         "bash",
                         "-c",
-                        "lsof -a -i4 -i6 -itcp -w | grep 'testzookeeperconfigloadbalancing_zoo3_1.*testzookeeperconfigloadbalancing_default:2181' | grep ESTABLISHED | wc -l",
+                        "lsof -a -i4 -i6 -itcp -w | grep -P 'testzookeeperconfigloadbalancing_(gw\\d+_)?zoo3_1.*testzookeeperconfigloadbalancing_(gw\\d+_)?default:2181' | grep ESTABLISHED | wc -l",
                     ],
                     privileged=True,
                     user="root",
@@ -341,7 +341,7 @@ def test_hostname_levenshtein_distance(started_cluster):
                     [
                         "bash",
                         "-c",
-                        "lsof -a -i4 -i6 -itcp -w | grep 'testzookeeperconfigloadbalancing_zoo1_1.*testzookeeperconfigloadbalancing_default:2181' | grep ESTABLISHED | wc -l",
+                        "lsof -a -i4 -i6 -itcp -w | grep -P 'testzookeeperconfigloadbalancing_(gw\\d+_)?zoo1_1.*testzookeeperconfigloadbalancing_(gw\\d+_)?default:2181' | grep ESTABLISHED | wc -l",
                     ],
                     privileged=True,
                     user="root",
@@ -369,7 +369,7 @@ def test_hostname_levenshtein_distance(started_cluster):
                     [
                         "bash",
                         "-c",
-                        "lsof -a -i4 -i6 -itcp -w | grep 'testzookeeperconfigloadbalancing_zoo2_1.*testzookeeperconfigloadbalancing_default:2181' | grep ESTABLISHED | wc -l",
+                        "lsof -a -i4 -i6 -itcp -w | grep -P 'testzookeeperconfigloadbalancing_(gw\\d+_)?zoo2_1.*testzookeeperconfigloadbalancing_(gw\\d+_)?default:2181' | grep ESTABLISHED | wc -l",
                     ],
                     privileged=True,
                     user="root",
@@ -397,7 +397,7 @@ def test_hostname_levenshtein_distance(started_cluster):
                     [
                         "bash",
                         "-c",
-                        "lsof -a -i4 -i6 -itcp -w | grep 'testzookeeperconfigloadbalancing_zoo3_1.*testzookeeperconfigloadbalancing_default:2181' | grep ESTABLISHED | wc -l",
+                        "lsof -a -i4 -i6 -itcp -w | grep -P 'testzookeeperconfigloadbalancing_(gw\\d+_)?zoo3_1.*testzookeeperconfigloadbalancing_(gw\\d+_)?default:2181' | grep ESTABLISHED | wc -l",
                     ],
                     privileged=True,
                     user="root",

From c77f6d78d976430faf4353e350d0205bbecf2837 Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Thu, 1 Aug 2024 12:09:58 +0200
Subject: [PATCH 1168/1488] Update minio

---
 tests/integration/compose/docker_compose_minio.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/compose/docker_compose_minio.yml b/tests/integration/compose/docker_compose_minio.yml
index 4255a529f6d..40098d05b04 100644
--- a/tests/integration/compose/docker_compose_minio.yml
+++ b/tests/integration/compose/docker_compose_minio.yml
@@ -2,7 +2,7 @@ version: '2.3'
 
 services:
   minio1:
-    image: minio/minio:RELEASE.2023-09-30T07-02-29Z
+    image: minio/minio:RELEASE.2024-07-31T05-46-26Z
     volumes:
       - data1-1:/data1
       - ${MINIO_CERTS_DIR:-}:/certs

From 928d5df3f02559fab4fd9bbd258f6e9c6ac4c18a Mon Sep 17 00:00:00 2001
From: maxvostrikov <max.vostrikov@clickhouse.com>
Date: Thu, 1 Aug 2024 12:32:43 +0200
Subject: [PATCH 1169/1488] added tests for
 https://github.com/ClickHouse/ClickHouse/pull/65475 Support writting page
 index into parquet file

---
 .../test/integration/runner/requirements.txt  |  1 +
 .../test_parquet_page_index/__init__.py       |  0
 .../test_parquet_page_index/test.py           | 77 +++++++++++++++++++
 3 files changed, 78 insertions(+)
 create mode 100644 tests/integration/test_parquet_page_index/__init__.py
 create mode 100644 tests/integration/test_parquet_page_index/test.py

diff --git a/docker/test/integration/runner/requirements.txt b/docker/test/integration/runner/requirements.txt
index 8a77d8abf77..428986b5562 100644
--- a/docker/test/integration/runner/requirements.txt
+++ b/docker/test/integration/runner/requirements.txt
@@ -74,6 +74,7 @@ protobuf==4.25.2
 psycopg2-binary==2.9.6
 py4j==0.10.9.5
 py==1.11.0
+pyarrow==17.0.0
 pycparser==2.22
 pycryptodome==3.20.0
 pymongo==3.11.0
diff --git a/tests/integration/test_parquet_page_index/__init__.py b/tests/integration/test_parquet_page_index/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/integration/test_parquet_page_index/test.py b/tests/integration/test_parquet_page_index/test.py
new file mode 100644
index 00000000000..366216ea2c6
--- /dev/null
+++ b/tests/integration/test_parquet_page_index/test.py
@@ -0,0 +1,77 @@
+import pytest
+from helpers.cluster import ClickHouseCluster
+import pyarrow.parquet as pq
+import os
+import time
+
+cluster = ClickHouseCluster(__file__)
+path_to_userfiles = "/var/lib/clickhouse/user_files/"
+path_to_external_dirs = "/ClickHouse/tests/integration/test_parquet_page_index/_instances"
+node = cluster.add_instance("node", external_dirs=[path_to_userfiles])
+
+
+@pytest.fixture(scope="module")
+def start_cluster():
+    try:
+        cluster.start()
+        yield cluster
+    finally:
+        cluster.shutdown()
+
+
+def check_page_index(file_path):
+    metadata = pq.read_metadata(file_path)
+    assert metadata, "pyarrow.parquet library can't read parquet file written by Clickhouse"
+    return metadata.row_group(0).column(0).has_offset_index
+
+
+def delete_if_exists(file_path):
+    if os.path.exists(file_path):
+        os.remove(file_path)
+
+
+@pytest.mark.parametrize("query, expected_result", {
+    ("SElECT number, number+1 FROM system.numbers LIMIT 100 "
+     "INTO OUTFILE '{file_name}' FORMAT Parquet "
+     "SETTINGS output_format_parquet_use_custom_encoder = false, "
+     "output_format_parquet_write_page_index = true;", True),
+    ("SElECT number, number+1 FROM system.numbers LIMIT 100 "
+     "INTO OUTFILE '{file_name}' FORMAT Parquet "
+     "SETTINGS output_format_parquet_use_custom_encoder = false, "
+     "output_format_parquet_write_page_index = false;", False),
+    # # default settings:
+    # # output_format_parquet_use_custom_encoder = true
+    ("SElECT number, number+1 FROM system.numbers LIMIT 100 "
+     "INTO OUTFILE '{file_name}' FORMAT Parquet;", False),
+})
+def test_parquet_page_index_select_into_outfile(query, expected_result, start_cluster):
+    file_name = 'export.parquet'
+    query = query.format(file_name=file_name)
+    delete_if_exists(file_name)
+    assert node.query(query) == ''
+    assert check_page_index(file_name) == expected_result, "Page offset index have wrong value"
+    delete_if_exists(file_name)
+
+
+@pytest.mark.parametrize("query, expected_result", {
+    ("INSERT INTO TABLE FUNCTION file('{file_name}') "
+     "SElECT number, number+1 FROM system.numbers LIMIT 100 "
+     "SETTINGS output_format_parquet_use_custom_encoder=false, "
+     "output_format_parquet_write_page_index=true FORMAT Parquet", True),
+    ("INSERT INTO TABLE FUNCTION file('{file_name}') "
+     "SElECT number, number+1 FROM system.numbers LIMIT 100 "
+     "SETTINGS output_format_parquet_use_custom_encoder=false, "
+     "output_format_parquet_write_page_index=false FORMAT Parquet", False),
+    # # default settings:
+    # # output_format_parquet_use_custom_encoder = true
+    ("INSERT INTO TABLE FUNCTION file('{file_name}') "
+     "SElECT number, number+1 FROM system.numbers LIMIT 100 FORMAT Parquet", False),
+})
+def test_parquet_page_index_insert_into_table_function_file(query, expected_result, start_cluster):
+    file_name = 'export.parquet'
+    query = query.format(file_name=file_name)
+    file_path = f"{path_to_external_dirs}{path_to_userfiles}{file_name}"
+    delete_if_exists(file_path)
+    assert node.query(query) == ''
+    assert check_page_index(file_path) == expected_result, "Page offset index have wrong value"
+    delete_if_exists(file_path)

From 572ad2f6fe193d69e9c2c1c64125b3ffdff1de0d Mon Sep 17 00:00:00 2001
From: robot-clickhouse <robot-clickhouse@users.noreply.github.com>
Date: Thu, 1 Aug 2024 11:11:52 +0000
Subject: [PATCH 1170/1488] Automatic style fix

---
 .../test_parquet_page_index/test.py           | 108 ++++++++++++------
 1 file changed, 71 insertions(+), 37 deletions(-)

diff --git a/tests/integration/test_parquet_page_index/test.py b/tests/integration/test_parquet_page_index/test.py
index 366216ea2c6..a820d1b5bc5 100644
--- a/tests/integration/test_parquet_page_index/test.py
+++ b/tests/integration/test_parquet_page_index/test.py
@@ -6,7 +6,9 @@ import time
 
 cluster = ClickHouseCluster(__file__)
 path_to_userfiles = "/var/lib/clickhouse/user_files/"
-path_to_external_dirs = "/ClickHouse/tests/integration/test_parquet_page_index/_instances"
+path_to_external_dirs = (
+    "/ClickHouse/tests/integration/test_parquet_page_index/_instances"
+)
 node = cluster.add_instance("node", external_dirs=[path_to_userfiles])
 
 
@@ -21,7 +23,9 @@ def start_cluster():
 
 def check_page_index(file_path):
     metadata = pq.read_metadata(file_path)
-    assert metadata, "pyarrow.parquet library can't read parquet file written by Clickhouse"
+    assert (
+        metadata
+    ), "pyarrow.parquet library can't read parquet file written by Clickhouse"
     return metadata.row_group(0).column(0).has_offset_index
 
 
@@ -30,48 +34,78 @@ def delete_if_exists(file_path):
         os.remove(file_path)
 
 
-@pytest.mark.parametrize("query, expected_result", {
-    ("SElECT number, number+1 FROM system.numbers LIMIT 100 "
-     "INTO OUTFILE '{file_name}' FORMAT Parquet "
-     "SETTINGS output_format_parquet_use_custom_encoder = false, "
-     "output_format_parquet_write_page_index = true;", True),
-    ("SElECT number, number+1 FROM system.numbers LIMIT 100 "
-     "INTO OUTFILE '{file_name}' FORMAT Parquet "
-     "SETTINGS output_format_parquet_use_custom_encoder = false, "
-     "output_format_parquet_write_page_index = false;", False),
-    # # default settings:
-    # # output_format_parquet_use_custom_encoder = true
-    ("SElECT number, number+1 FROM system.numbers LIMIT 100 "
-     "INTO OUTFILE '{file_name}' FORMAT Parquet;", False),
-})
+@pytest.mark.parametrize(
+    "query, expected_result",
+    {
+        (
+            "SElECT number, number+1 FROM system.numbers LIMIT 100 "
+            "INTO OUTFILE '{file_name}' FORMAT Parquet "
+            "SETTINGS output_format_parquet_use_custom_encoder = false, "
+            "output_format_parquet_write_page_index = true;",
+            True,
+        ),
+        (
+            "SElECT number, number+1 FROM system.numbers LIMIT 100 "
+            "INTO OUTFILE '{file_name}' FORMAT Parquet "
+            "SETTINGS output_format_parquet_use_custom_encoder = false, "
+            "output_format_parquet_write_page_index = false;",
+            False,
+        ),
+        # # default settings:
+        # # output_format_parquet_use_custom_encoder = true
+        (
+            "SElECT number, number+1 FROM system.numbers LIMIT 100 "
+            "INTO OUTFILE '{file_name}' FORMAT Parquet;",
+            False,
+        ),
+    },
+)
 def test_parquet_page_index_select_into_outfile(query, expected_result, start_cluster):
-    file_name = 'export.parquet'
+    file_name = "export.parquet"
     query = query.format(file_name=file_name)
     delete_if_exists(file_name)
-    assert node.query(query) == ''
-    assert check_page_index(file_name) == expected_result, "Page offset index have wrong value"
+    assert node.query(query) == ""
+    assert (
+        check_page_index(file_name) == expected_result
+    ), "Page offset index have wrong value"
     delete_if_exists(file_name)
 
 
-@pytest.mark.parametrize("query, expected_result", {
-    ("INSERT INTO TABLE FUNCTION file('{file_name}') "
-     "SElECT number, number+1 FROM system.numbers LIMIT 100 "
-     "SETTINGS output_format_parquet_use_custom_encoder=false, "
-     "output_format_parquet_write_page_index=true FORMAT Parquet", True),
-    ("INSERT INTO TABLE FUNCTION file('{file_name}') "
-     "SElECT number, number+1 FROM system.numbers LIMIT 100 "
-     "SETTINGS output_format_parquet_use_custom_encoder=false, "
-     "output_format_parquet_write_page_index=false FORMAT Parquet", False),
-    # # default settings:
-    # # output_format_parquet_use_custom_encoder = true
-    ("INSERT INTO TABLE FUNCTION file('{file_name}') "
-     "SElECT number, number+1 FROM system.numbers LIMIT 100 FORMAT Parquet", False),
-})
-def test_parquet_page_index_insert_into_table_function_file(query, expected_result, start_cluster):
-    file_name = 'export.parquet'
+@pytest.mark.parametrize(
+    "query, expected_result",
+    {
+        (
+            "INSERT INTO TABLE FUNCTION file('{file_name}') "
+            "SElECT number, number+1 FROM system.numbers LIMIT 100 "
+            "SETTINGS output_format_parquet_use_custom_encoder=false, "
+            "output_format_parquet_write_page_index=true FORMAT Parquet",
+            True,
+        ),
+        (
+            "INSERT INTO TABLE FUNCTION file('{file_name}') "
+            "SElECT number, number+1 FROM system.numbers LIMIT 100 "
+            "SETTINGS output_format_parquet_use_custom_encoder=false, "
+            "output_format_parquet_write_page_index=false FORMAT Parquet",
+            False,
+        ),
+        # # default settings:
+        # # output_format_parquet_use_custom_encoder = true
+        (
+            "INSERT INTO TABLE FUNCTION file('{file_name}') "
+            "SElECT number, number+1 FROM system.numbers LIMIT 100 FORMAT Parquet",
+            False,
+        ),
+    },
+)
+def test_parquet_page_index_insert_into_table_function_file(
+    query, expected_result, start_cluster
+):
+    file_name = "export.parquet"
     query = query.format(file_name=file_name)
     file_path = f"{path_to_external_dirs}{path_to_userfiles}{file_name}"
     delete_if_exists(file_path)
-    assert node.query(query) == ''
-    assert check_page_index(file_path) == expected_result, "Page offset index have wrong value"
+    assert node.query(query) == ""
+    assert (
+        check_page_index(file_path) == expected_result
+    ), "Page offset index have wrong value"
     delete_if_exists(file_path)

From 8fc77bec6639c8f0361858d9e031f3cb1175ae30 Mon Sep 17 00:00:00 2001
From: Yarik Briukhovetskyi <114298166+yariks5s@users.noreply.github.com>
Date: Thu, 1 Aug 2024 13:14:14 +0200
Subject: [PATCH 1171/1488] fix tests

---
 ...403_enable_extended_results_for_datetime_functions.reference | 2 +-
 .../0_stateless/03215_toStartOfWeek_with_dateTime64_fix.sql     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/queries/0_stateless/02403_enable_extended_results_for_datetime_functions.reference b/tests/queries/0_stateless/02403_enable_extended_results_for_datetime_functions.reference
index c830d790000..147e368b5c9 100644
--- a/tests/queries/0_stateless/02403_enable_extended_results_for_datetime_functions.reference
+++ b/tests/queries/0_stateless/02403_enable_extended_results_for_datetime_functions.reference
@@ -64,7 +64,7 @@ toStartOfMonth;toDateTime64;false	2099-07-07
 type;toStartOfMonth;toDateTime64;false	Date
 toStartOfWeek;toDate32;false	2099-07-07
 type;toStartOfWeek;toDate32;false	Date
-toStartOfWeek;toDateTime64;false	2099-07-07
+toStartOfWeek;toDateTime64;false	1970-01-01
 type;toStartOfWeek;toDateTime64;false	Date
 toMonday;toDate32;false	2099-07-08
 type;toMonday;toDate32;false	Date
diff --git a/tests/queries/0_stateless/03215_toStartOfWeek_with_dateTime64_fix.sql b/tests/queries/0_stateless/03215_toStartOfWeek_with_dateTime64_fix.sql
index 0f00a52cb86..1769d96aa8d 100644
--- a/tests/queries/0_stateless/03215_toStartOfWeek_with_dateTime64_fix.sql
+++ b/tests/queries/0_stateless/03215_toStartOfWeek_with_dateTime64_fix.sql
@@ -1,2 +1,2 @@
-SELECT toStartOfWeek(toDateTime64('1970-02-01', 6));
+SELECT toStartOfWeek(toDateTime64('1970-01-01', 6));
 SELECT toStartOfWeek(toDateTime('1970-01-01'));

From d0b514f12843fd09f18888666aa98170a52fb9d5 Mon Sep 17 00:00:00 2001
From: kssenii <sumarokovakseniia@mail.ru>
Date: Thu, 1 Aug 2024 13:27:09 +0200
Subject: [PATCH 1172/1488] Review fixes

---
 src/Interpreters/Cache/FileCache.cpp             | 12 ++++++------
 src/Interpreters/Cache/IFileCachePriority.h      |  6 +++---
 src/Interpreters/Cache/LRUFileCachePriority.cpp  | 10 +++++-----
 src/Interpreters/Cache/LRUFileCachePriority.h    |  2 +-
 src/Interpreters/Cache/SLRUFileCachePriority.cpp |  6 +++---
 src/Interpreters/Cache/SLRUFileCachePriority.h   |  2 +-
 6 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp
index bf8dd24a1db..1f79c7d9032 100644
--- a/src/Interpreters/Cache/FileCache.cpp
+++ b/src/Interpreters/Cache/FileCache.cpp
@@ -998,7 +998,7 @@ void FileCache::freeSpaceRatioKeepingThreadFunc()
     FileCacheReserveStat stat;
     EvictionCandidates eviction_candidates;
 
-    IFileCachePriority::DesiredSizeStatus desired_size_status;
+    IFileCachePriority::CollectStatus desired_size_status;
     try
     {
         /// Collect at most `keep_up_free_space_remove_batch` elements to evict,
@@ -1009,7 +1009,7 @@ void FileCache::freeSpaceRatioKeepingThreadFunc()
 
 #ifdef DEBUG_OR_SANITIZER_BUILD
         /// Let's make sure that we correctly processed the limits.
-        if (desired_size_status == IFileCachePriority::DesiredSizeStatus::SUCCESS
+        if (desired_size_status == IFileCachePriority::CollectStatus::SUCCESS
             && eviction_candidates.size() < keep_up_free_space_remove_batch)
         {
             const auto current_size = main_priority->getSize(lock);
@@ -1070,13 +1070,13 @@ void FileCache::freeSpaceRatioKeepingThreadFunc()
     [[maybe_unused]] bool scheduled = false;
     switch (desired_size_status)
     {
-        case IFileCachePriority::DesiredSizeStatus::SUCCESS: [[fallthrough]];
-        case IFileCachePriority::DesiredSizeStatus::CANNOT_EVICT:
+        case IFileCachePriority::CollectStatus::SUCCESS: [[fallthrough]];
+        case IFileCachePriority::CollectStatus::CANNOT_EVICT:
         {
             scheduled = keep_up_free_space_ratio_task->scheduleAfter(general_reschedule_ms);
             break;
         }
-        case IFileCachePriority::DesiredSizeStatus::REACHED_MAX_CANDIDATES_LIMIT:
+        case IFileCachePriority::CollectStatus::REACHED_MAX_CANDIDATES_LIMIT:
         {
             scheduled = keep_up_free_space_ratio_task->schedule();
             break;
@@ -1558,7 +1558,7 @@ void FileCache::applySettingsIfPossible(const FileCacheSettings & new_settings,
             FileCacheReserveStat stat;
             if (main_priority->collectCandidatesForEviction(
                     new_settings.max_size, new_settings.max_elements, 0/* max_candidates_to_evict */,
-                    stat, eviction_candidates, cache_lock) == IFileCachePriority::DesiredSizeStatus::SUCCESS)
+                    stat, eviction_candidates, cache_lock) == IFileCachePriority::CollectStatus::SUCCESS)
             {
                 if (eviction_candidates.size() == 0)
                 {
diff --git a/src/Interpreters/Cache/IFileCachePriority.h b/src/Interpreters/Cache/IFileCachePriority.h
index 9885ab00f78..6970d02473a 100644
--- a/src/Interpreters/Cache/IFileCachePriority.h
+++ b/src/Interpreters/Cache/IFileCachePriority.h
@@ -150,14 +150,14 @@ public:
     /// Collect eviction candidates sufficient to have `desired_size`
     /// and `desired_elements_num` as current cache state.
     /// Collect no more than `max_candidates_to_evict` elements.
-    /// Return `true` if the first condition is satisfied.
-    enum class DesiredSizeStatus
+    /// Return SUCCESS status if the first condition is satisfied.
+    enum class CollectStatus
     {
         SUCCESS,
         CANNOT_EVICT,
         REACHED_MAX_CANDIDATES_LIMIT,
     };
-    virtual DesiredSizeStatus collectCandidatesForEviction(
+    virtual CollectStatus collectCandidatesForEviction(
         size_t desired_size,
         size_t desired_elements_count,
         size_t max_candidates_to_evict,
diff --git a/src/Interpreters/Cache/LRUFileCachePriority.cpp b/src/Interpreters/Cache/LRUFileCachePriority.cpp
index 7970eaa3e13..0e0170c76e3 100644
--- a/src/Interpreters/Cache/LRUFileCachePriority.cpp
+++ b/src/Interpreters/Cache/LRUFileCachePriority.cpp
@@ -323,7 +323,7 @@ bool LRUFileCachePriority::collectCandidatesForEviction(
     }
 }
 
-IFileCachePriority::DesiredSizeStatus LRUFileCachePriority::collectCandidatesForEviction(
+IFileCachePriority::CollectStatus LRUFileCachePriority::collectCandidatesForEviction(
     size_t desired_size,
     size_t desired_elements_count,
     size_t max_candidates_to_evict,
@@ -336,23 +336,23 @@ IFileCachePriority::DesiredSizeStatus LRUFileCachePriority::collectCandidatesFor
         return canFit(0, 0, stat.total_stat.releasable_size, stat.total_stat.releasable_count,
                       lock, &desired_size, &desired_elements_count);
     };
-    auto status = DesiredSizeStatus::CANNOT_EVICT;
+    auto status = CollectStatus::CANNOT_EVICT;
     auto stop_condition = [&]()
     {
         if (desired_limits_satisfied())
         {
-            status = DesiredSizeStatus::SUCCESS;
+            status = CollectStatus::SUCCESS;
             return true;
         }
         if (max_candidates_to_evict && res.size() >= max_candidates_to_evict)
         {
-            status = DesiredSizeStatus::REACHED_MAX_CANDIDATES_LIMIT;
+            status = CollectStatus::REACHED_MAX_CANDIDATES_LIMIT;
             return true;
         }
         return false;
     };
     iterateForEviction(res, stat, stop_condition, lock);
-    chassert(status != DesiredSizeStatus::SUCCESS || stop_condition());
+    chassert(status != CollectStatus::SUCCESS || stop_condition());
     return status;
 }
 
diff --git a/src/Interpreters/Cache/LRUFileCachePriority.h b/src/Interpreters/Cache/LRUFileCachePriority.h
index 9bced106727..0ca62b19d37 100644
--- a/src/Interpreters/Cache/LRUFileCachePriority.h
+++ b/src/Interpreters/Cache/LRUFileCachePriority.h
@@ -63,7 +63,7 @@ public:
         const UserID & user_id,
         const CachePriorityGuard::Lock &) override;
 
-    DesiredSizeStatus collectCandidatesForEviction(
+    CollectStatus collectCandidatesForEviction(
         size_t desired_size,
         size_t desired_elements_count,
         size_t max_candidates_to_evict,
diff --git a/src/Interpreters/Cache/SLRUFileCachePriority.cpp b/src/Interpreters/Cache/SLRUFileCachePriority.cpp
index dc0df223cb0..f5ea519d7d4 100644
--- a/src/Interpreters/Cache/SLRUFileCachePriority.cpp
+++ b/src/Interpreters/Cache/SLRUFileCachePriority.cpp
@@ -256,7 +256,7 @@ bool SLRUFileCachePriority::collectCandidatesForEvictionInProtected(
     return true;
 }
 
-IFileCachePriority::DesiredSizeStatus SLRUFileCachePriority::collectCandidatesForEviction(
+IFileCachePriority::CollectStatus SLRUFileCachePriority::collectCandidatesForEviction(
     size_t desired_size,
     size_t desired_elements_count,
     size_t max_candidates_to_evict,
@@ -285,7 +285,7 @@ IFileCachePriority::DesiredSizeStatus SLRUFileCachePriority::collectCandidatesFo
     chassert(!max_candidates_to_evict || res.size() <= max_candidates_to_evict);
     chassert(res.size() == stat.total_stat.releasable_count);
 
-    if (probationary_desired_size_status == DesiredSizeStatus::REACHED_MAX_CANDIDATES_LIMIT)
+    if (probationary_desired_size_status == CollectStatus::REACHED_MAX_CANDIDATES_LIMIT)
         return probationary_desired_size_status;
 
     const auto desired_protected_size = getRatio(desired_size, size_ratio);
@@ -306,7 +306,7 @@ IFileCachePriority::DesiredSizeStatus SLRUFileCachePriority::collectCandidatesFo
              desired_protected_size, desired_protected_elements_num,
              protected_queue.getStateInfoForLog(lock));
 
-    if (probationary_desired_size_status == DesiredSizeStatus::SUCCESS)
+    if (probationary_desired_size_status == CollectStatus::SUCCESS)
         return protected_desired_size_status;
     else
         return probationary_desired_size_status;
diff --git a/src/Interpreters/Cache/SLRUFileCachePriority.h b/src/Interpreters/Cache/SLRUFileCachePriority.h
index e6d20e0d0ee..23bc8c0908b 100644
--- a/src/Interpreters/Cache/SLRUFileCachePriority.h
+++ b/src/Interpreters/Cache/SLRUFileCachePriority.h
@@ -58,7 +58,7 @@ public:
         const UserID & user_id,
         const CachePriorityGuard::Lock &) override;
 
-    DesiredSizeStatus collectCandidatesForEviction(
+    CollectStatus collectCandidatesForEviction(
         size_t desired_size,
         size_t desired_elements_count,
         size_t max_candidates_to_evict,

From 2c018d6f4d4d6fa1c04e91306b60b6e85d8e468f Mon Sep 17 00:00:00 2001
From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com>
Date: Thu, 1 Aug 2024 13:38:41 +0200
Subject: [PATCH 1173/1488] Update cluster.py

---
 tests/integration/helpers/cluster.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py
index 90b28a4cda3..acf033de46d 100644
--- a/tests/integration/helpers/cluster.py
+++ b/tests/integration/helpers/cluster.py
@@ -248,7 +248,7 @@ def check_rabbitmq_is_available(rabbitmq_id, cookie):
         ),
         stdout=subprocess.PIPE,
     )
-    p.communicate()
+    p.wait(timeout=60)
     return p.returncode == 0
 
 

From 53d93b177e371b8dcc13e404f48209e3df456fe4 Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Thu, 1 Aug 2024 11:56:07 +0000
Subject: [PATCH 1174/1488] Fix 02434_cancel_insert_when_client_dies

---
 .../02434_cancel_insert_when_client_dies.sh          | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/tests/queries/0_stateless/02434_cancel_insert_when_client_dies.sh b/tests/queries/0_stateless/02434_cancel_insert_when_client_dies.sh
index f85aaed7716..45f4194104e 100755
--- a/tests/queries/0_stateless/02434_cancel_insert_when_client_dies.sh
+++ b/tests/queries/0_stateless/02434_cancel_insert_when_client_dies.sh
@@ -10,22 +10,26 @@ export DATA_FILE="$CLICKHOUSE_TMP/deduptest.tsv"
 export TEST_MARK="02434_insert_${CLICKHOUSE_DATABASE}_"
 
 $CLICKHOUSE_CLIENT -q 'select * from numbers(5000000) format TSV' > $DATA_FILE
-$CLICKHOUSE_CLIENT -q 'create table dedup_test(A Int64) Engine = MergeTree order by A settings non_replicated_deduplication_window=1000;'
+$CLICKHOUSE_CLIENT -q "create table dedup_test(A Int64) Engine = MergeTree order by A
+    settings non_replicated_deduplication_window=1000
+    , merge_tree_clear_old_temporary_directories_interval_seconds = 1
+    ;"
 $CLICKHOUSE_CLIENT -q "create table dedup_dist(A Int64) Engine = Distributed('test_cluster_one_shard_two_replicas', currentDatabase(), dedup_test)"
 
 function insert_data
 {
-    SETTINGS="query_id=$ID&max_insert_block_size=110000&min_insert_block_size_rows=110000"
+    # send_logs_level: https://github.com/ClickHouse/ClickHouse/issues/67599
+    SETTINGS="query_id=$ID&max_insert_block_size=110000&min_insert_block_size_rows=110000&send_logs_level=fatal"
     # max_block_size=10000, so external table will contain smaller blocks that will be squashed on insert-select (more chances to catch a bug on query cancellation)
     TRASH_SETTINGS="query_id=$ID&input_format_parallel_parsing=0&max_threads=1&max_insert_threads=1&max_insert_block_size=110000&max_block_size=10000&min_insert_block_size_bytes=0&min_insert_block_size_rows=110000&max_insert_block_size=110000"
     TYPE=$(( RANDOM % 5 ))
 
     if [[ "$TYPE" -eq 0 ]]; then
         # client will send 10000-rows blocks, server will squash them into 110000-rows blocks (more chances to catch a bug on query cancellation)
-        $CLICKHOUSE_CLIENT --max_block_size=10000 --max_insert_block_size=10000 --query_id="$ID" \
+        $CLICKHOUSE_CLIENT --allow_repeated_settings --send_logs_level=fatal --max_block_size=10000 --max_insert_block_size=10000 --query_id="$ID" \
             -q 'insert into dedup_test settings max_insert_block_size=110000, min_insert_block_size_rows=110000 format TSV' < $DATA_FILE
     elif [[ "$TYPE" -eq 1 ]]; then
-        $CLICKHOUSE_CLIENT --max_block_size=10000 --max_insert_block_size=10000 --query_id="$ID" --prefer_localhost_replica="$(( RANDOM % 2))" \
+        $CLICKHOUSE_CLIENT --allow_repeated_settings --send_logs_level=fatal --max_block_size=10000 --max_insert_block_size=10000 --query_id="$ID" --prefer_localhost_replica="$(( RANDOM % 2))" \
             -q 'insert into dedup_dist settings max_insert_block_size=110000, min_insert_block_size_rows=110000 format TSV' < $DATA_FILE
     elif [[ "$TYPE" -eq 2 ]]; then
         $CLICKHOUSE_CURL -sS -X POST --data-binary @- "$CLICKHOUSE_URL&$SETTINGS&query=insert+into+dedup_test+format+TSV" < $DATA_FILE

From 029deaeee8431d0ef6f2a460c2bd8631c8025254 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Thu, 1 Aug 2024 14:13:11 +0200
Subject: [PATCH 1175/1488] Fix 02910_bad_logs_level_in_local in fast tests!

---
 tests/queries/0_stateless/02910_bad_logs_level_in_local.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/queries/0_stateless/02910_bad_logs_level_in_local.sh b/tests/queries/0_stateless/02910_bad_logs_level_in_local.sh
index badf7232a95..b5de10bf191 100755
--- a/tests/queries/0_stateless/02910_bad_logs_level_in_local.sh
+++ b/tests/queries/0_stateless/02910_bad_logs_level_in_local.sh
@@ -1,14 +1,14 @@
 #!/usr/bin/expect -f
 
 log_user 0
-set timeout 60
+set timeout 30
 match_max 100000
 
 spawn bash -c "clickhouse-local"
 
 expect ":) "
 send -- "SET send_logs_level = 't'\r"
-expect "Exception on client:"
+expect "Unexpected value of LogsLevel:" {} timeout {exit 1}
 expect ":) "
 send -- "exit\r"
 expect eof

From bcc75d3681d45b6637211aca0367703b3e957c05 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Thu, 1 Aug 2024 14:38:41 +0200
Subject: [PATCH 1176/1488] Make 02477_analyzer_function_hints.sh
 parallelizable

---
 .../0_stateless/02477_analyzer_function_hints.sh     | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/tests/queries/0_stateless/02477_analyzer_function_hints.sh b/tests/queries/0_stateless/02477_analyzer_function_hints.sh
index d49c20cab75..f83935e47fb 100755
--- a/tests/queries/0_stateless/02477_analyzer_function_hints.sh
+++ b/tests/queries/0_stateless/02477_analyzer_function_hints.sh
@@ -1,7 +1,5 @@
 #!/usr/bin/env bash
 
-# Tags: no-parallel
-
 set -e
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
@@ -14,11 +12,11 @@ $CLICKHOUSE_CLIENT -q "SELECT plu(1, 1) SETTINGS allow_experimental_analyzer = 1
 $CLICKHOUSE_CLIENT -q "SELECT uniqExac(1, 1) SETTINGS allow_experimental_analyzer = 1;" 2>&1 \
     | grep "Maybe you meant: \['uniqExact'\]" &>/dev/null;
 
-$CLICKHOUSE_CLIENT -q "DROP FUNCTION IF EXISTS test_user_defined_function;"
-$CLICKHOUSE_CLIENT -q "CREATE FUNCTION test_user_defined_function AS x -> x + 1;"
-$CLICKHOUSE_CLIENT -q "SELECT test_user_defined_functio(1) SETTINGS allow_experimental_analyzer = 1;" 2>&1 \
-    | grep "Maybe you meant: \['test_user_defined_function'\]" &>/dev/null;
-$CLICKHOUSE_CLIENT -q "DROP FUNCTION test_user_defined_function";
+$CLICKHOUSE_CLIENT -q "DROP FUNCTION IF EXISTS test_user_defined_function_$CLICKHOUSE_DATABASE;"
+$CLICKHOUSE_CLIENT -q "CREATE FUNCTION test_user_defined_function_$CLICKHOUSE_DATABASE AS x -> x + 1;"
+$CLICKHOUSE_CLIENT -q "SELECT test_user_defined_function_${CLICKHOUSE_DATABASE}A(1) SETTINGS allow_experimental_analyzer = 1;" 2>&1 \
+   | grep -E "Maybe you meant: \[.*'test_user_defined_function_$CLICKHOUSE_DATABASE'.*\]" &>/dev/null;
+$CLICKHOUSE_CLIENT -q "DROP FUNCTION test_user_defined_function_$CLICKHOUSE_DATABASE";
 
 $CLICKHOUSE_CLIENT -q "WITH (x -> x + 1) AS lambda_function SELECT lambda_functio(1) SETTINGS allow_experimental_analyzer = 1;" 2>&1 \
     | grep "Maybe you meant: \['lambda_function'\]" &>/dev/null;

From 4adc9523e403ab103ed3dec537b02566287b76ee Mon Sep 17 00:00:00 2001
From: maxvostrikov <max.vostrikov@clickhouse.com>
Date: Thu, 1 Aug 2024 14:38:07 +0200
Subject: [PATCH 1177/1488] added a stateless test for ENGINE=File(Parquet)

---
 .../0_stateless/03215_parquet_index.reference | 300 ++++++++++++++++++
 .../0_stateless/03215_parquet_index.sql       |  17 +
 2 files changed, 317 insertions(+)
 create mode 100644 tests/queries/0_stateless/03215_parquet_index.reference
 create mode 100644 tests/queries/0_stateless/03215_parquet_index.sql

diff --git a/tests/queries/0_stateless/03215_parquet_index.reference b/tests/queries/0_stateless/03215_parquet_index.reference
new file mode 100644
index 00000000000..334f2f3824e
--- /dev/null
+++ b/tests/queries/0_stateless/03215_parquet_index.reference
@@ -0,0 +1,300 @@
+0	0
+1	1
+2	2
+3	3
+4	4
+5	5
+6	6
+7	7
+8	8
+9	9
+10	10
+11	11
+12	12
+13	13
+14	14
+15	15
+16	16
+17	17
+18	18
+19	19
+20	20
+21	21
+22	22
+23	23
+24	24
+25	25
+26	26
+27	27
+28	28
+29	29
+30	30
+31	31
+32	32
+33	33
+34	34
+35	35
+36	36
+37	37
+38	38
+39	39
+40	40
+41	41
+42	42
+43	43
+44	44
+45	45
+46	46
+47	47
+48	48
+49	49
+50	50
+51	51
+52	52
+53	53
+54	54
+55	55
+56	56
+57	57
+58	58
+59	59
+60	60
+61	61
+62	62
+63	63
+64	64
+65	65
+66	66
+67	67
+68	68
+69	69
+70	70
+71	71
+72	72
+73	73
+74	74
+75	75
+76	76
+77	77
+78	78
+79	79
+80	80
+81	81
+82	82
+83	83
+84	84
+85	85
+86	86
+87	87
+88	88
+89	89
+90	90
+91	91
+92	92
+93	93
+94	94
+95	95
+96	96
+97	97
+98	98
+99	99
+0	0
+1	1
+2	2
+3	3
+4	4
+5	5
+6	6
+7	7
+8	8
+9	9
+10	10
+11	11
+12	12
+13	13
+14	14
+15	15
+16	16
+17	17
+18	18
+19	19
+20	20
+21	21
+22	22
+23	23
+24	24
+25	25
+26	26
+27	27
+28	28
+29	29
+30	30
+31	31
+32	32
+33	33
+34	34
+35	35
+36	36
+37	37
+38	38
+39	39
+40	40
+41	41
+42	42
+43	43
+44	44
+45	45
+46	46
+47	47
+48	48
+49	49
+50	50
+51	51
+52	52
+53	53
+54	54
+55	55
+56	56
+57	57
+58	58
+59	59
+60	60
+61	61
+62	62
+63	63
+64	64
+65	65
+66	66
+67	67
+68	68
+69	69
+70	70
+71	71
+72	72
+73	73
+74	74
+75	75
+76	76
+77	77
+78	78
+79	79
+80	80
+81	81
+82	82
+83	83
+84	84
+85	85
+86	86
+87	87
+88	88
+89	89
+90	90
+91	91
+92	92
+93	93
+94	94
+95	95
+96	96
+97	97
+98	98
+99	99
+0	0
+1	1
+2	2
+3	3
+4	4
+5	5
+6	6
+7	7
+8	8
+9	9
+10	10
+11	11
+12	12
+13	13
+14	14
+15	15
+16	16
+17	17
+18	18
+19	19
+20	20
+21	21
+22	22
+23	23
+24	24
+25	25
+26	26
+27	27
+28	28
+29	29
+30	30
+31	31
+32	32
+33	33
+34	34
+35	35
+36	36
+37	37
+38	38
+39	39
+40	40
+41	41
+42	42
+43	43
+44	44
+45	45
+46	46
+47	47
+48	48
+49	49
+50	50
+51	51
+52	52
+53	53
+54	54
+55	55
+56	56
+57	57
+58	58
+59	59
+60	60
+61	61
+62	62
+63	63
+64	64
+65	65
+66	66
+67	67
+68	68
+69	69
+70	70
+71	71
+72	72
+73	73
+74	74
+75	75
+76	76
+77	77
+78	78
+79	79
+80	80
+81	81
+82	82
+83	83
+84	84
+85	85
+86	86
+87	87
+88	88
+89	89
+90	90
+91	91
+92	92
+93	93
+94	94
+95	95
+96	96
+97	97
+98	98
+99	99
diff --git a/tests/queries/0_stateless/03215_parquet_index.sql b/tests/queries/0_stateless/03215_parquet_index.sql
new file mode 100644
index 00000000000..5b176ff70ba
--- /dev/null
+++ b/tests/queries/0_stateless/03215_parquet_index.sql
@@ -0,0 +1,17 @@
+-- default settings.
+DROP TABLE IF EXISTS test_parquet;
+CREATE TABLE test_parquet (col1 int, col2 String) ENGINE=File(Parquet);
+INSERT INTO test_parquet SELECT number, toString(number) FROM numbers(100);
+SELECT col1, col2 FROM test_parquet;
+
+-- Parquet will have indexes in columns. We are not checking that indexes exist here, there is an integration test test_parquet_page_index for that. We just check that a setting doesn't break the SELECT
+DROP TABLE IF EXISTS test_parquet;
+CREATE TABLE test_parquet (col1 int, col2 String) ENGINE=File(Parquet)  SETTINGS output_format_parquet_use_custom_encoder=false, output_format_parquet_write_page_index=true;
+INSERT INTO test_parquet SELECT number, toString(number) FROM numbers(100);
+SELECT col1, col2 FROM test_parquet;
+
+-- Parquet will not have indexes in columns.
+DROP TABLE IF EXISTS test_parquet;
+CREATE TABLE test_parquet (col1 int, col2 String) ENGINE=File(Parquet)  SETTINGS output_format_parquet_use_custom_encoder=false, output_format_parquet_write_page_index=false;
+INSERT INTO test_parquet SELECT number, toString(number) FROM numbers(100);
+SELECT col1, col2 FROM test_parquet;

From a8e7c8ae9a5e345fa1725d664fd490891e4fa9a5 Mon Sep 17 00:00:00 2001
From: maxvostrikov <max.vostrikov@clickhouse.com>
Date: Thu, 1 Aug 2024 14:45:46 +0200
Subject: [PATCH 1178/1488] fix for parallel execution

---
 tests/integration/test_parquet_page_index/test.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/tests/integration/test_parquet_page_index/test.py b/tests/integration/test_parquet_page_index/test.py
index a820d1b5bc5..db291e20b74 100644
--- a/tests/integration/test_parquet_page_index/test.py
+++ b/tests/integration/test_parquet_page_index/test.py
@@ -61,7 +61,7 @@ def delete_if_exists(file_path):
     },
 )
 def test_parquet_page_index_select_into_outfile(query, expected_result, start_cluster):
-    file_name = "export.parquet"
+    file_name = f"export{time.time()}.parquet"
     query = query.format(file_name=file_name)
     delete_if_exists(file_name)
     assert node.query(query) == ""
@@ -76,14 +76,14 @@ def test_parquet_page_index_select_into_outfile(query, expected_result, start_cl
     {
         (
             "INSERT INTO TABLE FUNCTION file('{file_name}') "
-            "SElECT number, number+1 FROM system.numbers LIMIT 100 "
+            "SELECT number, number+1 FROM system.numbers LIMIT 100 "
             "SETTINGS output_format_parquet_use_custom_encoder=false, "
             "output_format_parquet_write_page_index=true FORMAT Parquet",
             True,
         ),
         (
             "INSERT INTO TABLE FUNCTION file('{file_name}') "
-            "SElECT number, number+1 FROM system.numbers LIMIT 100 "
+            "SELECT number, number+1 FROM system.numbers LIMIT 100 "
             "SETTINGS output_format_parquet_use_custom_encoder=false, "
             "output_format_parquet_write_page_index=false FORMAT Parquet",
             False,
@@ -92,7 +92,7 @@ def test_parquet_page_index_select_into_outfile(query, expected_result, start_cl
         # # output_format_parquet_use_custom_encoder = true
         (
             "INSERT INTO TABLE FUNCTION file('{file_name}') "
-            "SElECT number, number+1 FROM system.numbers LIMIT 100 FORMAT Parquet",
+            "SELECT number, number+1 FROM system.numbers LIMIT 100 FORMAT Parquet",
             False,
         ),
     },
@@ -100,7 +100,7 @@ def test_parquet_page_index_select_into_outfile(query, expected_result, start_cl
 def test_parquet_page_index_insert_into_table_function_file(
     query, expected_result, start_cluster
 ):
-    file_name = "export.parquet"
+    file_name = f"export{time.time()}.parquet"
     query = query.format(file_name=file_name)
     file_path = f"{path_to_external_dirs}{path_to_userfiles}{file_name}"
     delete_if_exists(file_path)

From 008408c81f23ed615cb899048418dc46aa3c2a9f Mon Sep 17 00:00:00 2001
From: Nikita Taranov <nikita.taranov@clickhouse.com>
Date: Thu, 1 Aug 2024 13:47:34 +0100
Subject: [PATCH 1179/1488] impl

---
 .../0_stateless/01605_adaptive_granularity_block_borders.sql  | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/queries/0_stateless/01605_adaptive_granularity_block_borders.sql b/tests/queries/0_stateless/01605_adaptive_granularity_block_borders.sql
index 9b96ce3e586..aaeee466794 100644
--- a/tests/queries/0_stateless/01605_adaptive_granularity_block_borders.sql
+++ b/tests/queries/0_stateless/01605_adaptive_granularity_block_borders.sql
@@ -1,4 +1,4 @@
--- Tags: no-random-merge-tree-settings, no-tsan, no-debug, no-object-storage
+-- Tags: no-random-merge-tree-settings, no-random-settings, no-tsan, no-debug, no-object-storage, long
 -- no-tsan: too slow
 -- no-object-storage: for remote tables we use thread pool even when reading with one stream, so memory consumption is higher
 
@@ -16,7 +16,7 @@ CREATE TABLE adaptive_table(
     value String
 ) ENGINE MergeTree()
 ORDER BY key
-SETTINGS index_granularity_bytes=1048576,
+SETTINGS index_granularity_bytes = 1048576,
 min_bytes_for_wide_part = 0,
 min_rows_for_wide_part = 0,
 enable_vertical_merge_algorithm = 0;

From 70228acd7e809230582883a0b6b70c4cd9c04daa Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Thu, 1 Aug 2024 15:02:29 +0200
Subject: [PATCH 1180/1488] Update CHANGELOG.md

---
 CHANGELOG.md | 1 -
 1 file changed, 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index a4c873ba3f9..5cd4200d9ad 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -22,7 +22,6 @@
 
 #### New Feature
 * Add `ASOF JOIN` support for `full_sorting_join` algorithm. [#55051](https://github.com/ClickHouse/ClickHouse/pull/55051) ([vdimir](https://github.com/vdimir)).
-* Add new window function `percent_rank`. [#62747](https://github.com/ClickHouse/ClickHouse/pull/62747) ([lgbo](https://github.com/lgbo-ustc)).
 * Support JWT authentication in `clickhouse-client` (will be available only in ClickHouse Cloud). [#62829](https://github.com/ClickHouse/ClickHouse/pull/62829) ([Konstantin Bogdanov](https://github.com/thevar1able)).
 * Add SQL functions `changeYear`, `changeMonth`, `changeDay`, `changeHour`, `changeMinute`, `changeSecond`. For example, `SELECT changeMonth(toDate('2024-06-14'), 7)` returns date `2024-07-14`. [#63186](https://github.com/ClickHouse/ClickHouse/pull/63186) ([cucumber95](https://github.com/cucumber95)).
 * Introduce startup scripts, which allow the execution of preconfigured queries at the startup stage. [#64889](https://github.com/ClickHouse/ClickHouse/pull/64889) ([pufit](https://github.com/pufit)).

From 48c6e36dfd23f297907575ce4696f761aec49e11 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Thu, 1 Aug 2024 15:03:41 +0200
Subject: [PATCH 1181/1488] Make 01062_window_view_event_hop_watch_asc
 parallelizable

---
 .../01062_window_view_event_hop_watch_asc.py  | 28 ++++++++-----------
 1 file changed, 12 insertions(+), 16 deletions(-)

diff --git a/tests/queries/0_stateless/01062_window_view_event_hop_watch_asc.py b/tests/queries/0_stateless/01062_window_view_event_hop_watch_asc.py
index d6cc3ee1a88..3c85ff30ba8 100755
--- a/tests/queries/0_stateless/01062_window_view_event_hop_watch_asc.py
+++ b/tests/queries/0_stateless/01062_window_view_event_hop_watch_asc.py
@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# Tags: no-parallel
 
 import os
 import sys
@@ -17,6 +16,7 @@ log = None
 with client(name="client1>", log=log) as client1, client(
     name="client2>", log=log
 ) as client2:
+    database_name = os.environ["CLICKHOUSE_DATABASE"]
     client1.expect(prompt)
     client2.expect(prompt)
 
@@ -31,40 +31,38 @@ with client(name="client1>", log=log) as client1, client(
     client2.send("SET allow_experimental_analyzer = 0")
     client2.expect(prompt)
 
-    client1.send("CREATE DATABASE IF NOT EXISTS 01062_window_view_event_hop_watch_asc")
+    client1.send(f"DROP TABLE IF EXISTS {database_name}.mt")
     client1.expect(prompt)
-    client1.send("DROP TABLE IF EXISTS 01062_window_view_event_hop_watch_asc.mt")
-    client1.expect(prompt)
-    client1.send("DROP TABLE IF EXISTS 01062_window_view_event_hop_watch_asc.wv SYNC")
+    client1.send(f"DROP TABLE IF EXISTS {database_name}.wv SYNC")
     client1.expect(prompt)
 
     client1.send(
-        "CREATE TABLE 01062_window_view_event_hop_watch_asc.mt(a Int32, timestamp DateTime) ENGINE=MergeTree ORDER BY tuple()"
+        f"CREATE TABLE {database_name}.mt(a Int32, timestamp DateTime) ENGINE=MergeTree ORDER BY tuple()"
     )
     client1.expect(prompt)
     client1.send(
-        "CREATE WINDOW VIEW 01062_window_view_event_hop_watch_asc.wv ENGINE Memory WATERMARK=ASCENDING AS SELECT count(a) AS count, hopEnd(wid) AS w_end FROM 01062_window_view_event_hop_watch_asc.mt GROUP BY hop(timestamp, INTERVAL '2' SECOND, INTERVAL '3' SECOND, 'US/Samoa') AS wid"
+        f"CREATE WINDOW VIEW {database_name}.wv ENGINE Memory WATERMARK=ASCENDING AS SELECT count(a) AS count, hopEnd(wid) AS w_end FROM {database_name}.mt GROUP BY hop(timestamp, INTERVAL '2' SECOND, INTERVAL '3' SECOND, 'US/Samoa') AS wid"
     )
     client1.expect(prompt)
 
-    client1.send("WATCH 01062_window_view_event_hop_watch_asc.wv")
+    client1.send(f"WATCH {database_name}.wv")
     client1.expect("Query id" + end_of_block)
     client1.expect("Progress: 0.00 rows.*\\)")
     client2.send(
-        "INSERT INTO 01062_window_view_event_hop_watch_asc.mt VALUES (1, toDateTime('1990/01/01 12:00:00', 'US/Samoa'));"
+        f"INSERT INTO {database_name}.mt VALUES (1, toDateTime('1990/01/01 12:00:00', 'US/Samoa'));"
     )
     client2.expect(prompt)
     client2.send(
-        "INSERT INTO 01062_window_view_event_hop_watch_asc.mt VALUES (1, toDateTime('1990/01/01 12:00:05', 'US/Samoa'));"
+        f"INSERT INTO {database_name}.mt VALUES (1, toDateTime('1990/01/01 12:00:05', 'US/Samoa'));"
     )
     client2.expect(prompt)
     client1.expect("1*" + end_of_block)
     client2.send(
-        "INSERT INTO 01062_window_view_event_hop_watch_asc.mt VALUES (1, toDateTime('1990/01/01 12:00:06', 'US/Samoa'));"
+        f"INSERT INTO {database_name}.mt VALUES (1, toDateTime('1990/01/01 12:00:06', 'US/Samoa'));"
     )
     client2.expect(prompt)
     client2.send(
-        "INSERT INTO 01062_window_view_event_hop_watch_asc.mt VALUES (1, toDateTime('1990/01/01 12:00:10', 'US/Samoa'));"
+        f"INSERT INTO {database_name}.mt VALUES (1, toDateTime('1990/01/01 12:00:10', 'US/Samoa'));"
     )
     client2.expect(prompt)
     client1.expect("1" + end_of_block)
@@ -77,9 +75,7 @@ with client(name="client1>", log=log) as client1, client(
     if match.groups()[1]:
         client1.send(client1.command)
         client1.expect(prompt)
-    client1.send("DROP TABLE 01062_window_view_event_hop_watch_asc.wv SYNC")
+    client1.send(f"DROP TABLE {database_name}.wv SYNC")
     client1.expect(prompt)
-    client1.send("DROP TABLE 01062_window_view_event_hop_watch_asc.mt")
-    client1.expect(prompt)
-    client1.send("DROP DATABASE IF EXISTS 01062_window_view_event_hop_watch_asc")
+    client1.send(f"DROP TABLE {database_name}.mt")
     client1.expect(prompt)

From 7b72362e99a093a4f880d333c1b50cd114b590c0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Thu, 1 Aug 2024 15:06:27 +0200
Subject: [PATCH 1182/1488] 01493_alter_remove_properties_zookeeper is already
 parallelizable

---
 .../0_stateless/01493_alter_remove_properties_zookeeper.sql     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/01493_alter_remove_properties_zookeeper.sql b/tests/queries/0_stateless/01493_alter_remove_properties_zookeeper.sql
index 92e6fce2c93..362da3ac364 100644
--- a/tests/queries/0_stateless/01493_alter_remove_properties_zookeeper.sql
+++ b/tests/queries/0_stateless/01493_alter_remove_properties_zookeeper.sql
@@ -1,4 +1,4 @@
--- Tags: zookeeper, no-parallel
+-- Tags: zookeeper
 
 DROP TABLE IF EXISTS r_prop_table1;
 DROP TABLE IF EXISTS r_prop_table2;

From 56e48cf43b4d13810e2bb4b4e941954b654a1cb3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Thu, 1 Aug 2024 15:09:56 +0200
Subject: [PATCH 1183/1488] Make 01676_range_hashed_dictionary parallelizable

---
 .../01676_range_hashed_dictionary.sql         | 92 +++++++++----------
 1 file changed, 42 insertions(+), 50 deletions(-)

diff --git a/tests/queries/0_stateless/01676_range_hashed_dictionary.sql b/tests/queries/0_stateless/01676_range_hashed_dictionary.sql
index 430f3a86dc1..ba2a9eba87f 100644
--- a/tests/queries/0_stateless/01676_range_hashed_dictionary.sql
+++ b/tests/queries/0_stateless/01676_range_hashed_dictionary.sql
@@ -1,10 +1,4 @@
--- Tags: no-parallel
-
-DROP DATABASE IF EXISTS database_for_range_dict;
-
-CREATE DATABASE database_for_range_dict;
-
-CREATE TABLE database_for_range_dict.date_table
+CREATE TABLE date_table
 (
   CountryID UInt64,
   StartDate Date,
@@ -14,11 +8,11 @@ CREATE TABLE database_for_range_dict.date_table
 ENGINE = MergeTree()
 ORDER BY CountryID;
 
-INSERT INTO database_for_range_dict.date_table VALUES(1, toDate('2019-05-05'), toDate('2019-05-20'), 0.33);
-INSERT INTO database_for_range_dict.date_table VALUES(1, toDate('2019-05-21'), toDate('2019-05-30'), 0.42);
-INSERT INTO database_for_range_dict.date_table VALUES(2, toDate('2019-05-21'), toDate('2019-05-30'), 0.46);
+INSERT INTO date_table VALUES(1, toDate('2019-05-05'), toDate('2019-05-20'), 0.33);
+INSERT INTO date_table VALUES(1, toDate('2019-05-21'), toDate('2019-05-30'), 0.42);
+INSERT INTO date_table VALUES(2, toDate('2019-05-21'), toDate('2019-05-30'), 0.46);
 
-CREATE DICTIONARY database_for_range_dict.range_dictionary
+CREATE DICTIONARY range_dictionary
 (
   CountryID UInt64,
   StartDate Date,
@@ -26,7 +20,7 @@ CREATE DICTIONARY database_for_range_dict.range_dictionary
   Tax Float64 DEFAULT 0.2
 )
 PRIMARY KEY CountryID
-SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'date_table' DB 'database_for_range_dict'))
+SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'date_table' DB currentDatabase()))
 LIFETIME(MIN 1 MAX 1000)
 LAYOUT(RANGE_HASHED())
 RANGE(MIN StartDate MAX EndDate)
@@ -35,30 +29,30 @@ SETTINGS(dictionary_use_async_executor=1, max_threads=8)
 
 SELECT 'Dictionary not nullable';
 SELECT 'dictGet';
-SELECT dictGet('database_for_range_dict.range_dictionary', 'Tax', toUInt64(1), toDate('2019-05-15'));
-SELECT dictGet('database_for_range_dict.range_dictionary', 'Tax', toUInt64(1), toDate('2019-05-29'));
-SELECT dictGet('database_for_range_dict.range_dictionary', 'Tax', toUInt64(2), toDate('2019-05-29'));
-SELECT dictGet('database_for_range_dict.range_dictionary', 'Tax', toUInt64(2), toDate('2019-05-31'));
-SELECT dictGetOrDefault('database_for_range_dict.range_dictionary', 'Tax', toUInt64(2), toDate('2019-05-31'), 0.4);
+SELECT dictGet('range_dictionary', 'Tax', toUInt64(1), toDate('2019-05-15'));
+SELECT dictGet('range_dictionary', 'Tax', toUInt64(1), toDate('2019-05-29'));
+SELECT dictGet('range_dictionary', 'Tax', toUInt64(2), toDate('2019-05-29'));
+SELECT dictGet('range_dictionary', 'Tax', toUInt64(2), toDate('2019-05-31'));
+SELECT dictGetOrDefault('range_dictionary', 'Tax', toUInt64(2), toDate('2019-05-31'), 0.4);
 SELECT 'dictHas';
-SELECT dictHas('database_for_range_dict.range_dictionary', toUInt64(1), toDate('2019-05-15'));
-SELECT dictHas('database_for_range_dict.range_dictionary', toUInt64(1), toDate('2019-05-29'));
-SELECT dictHas('database_for_range_dict.range_dictionary', toUInt64(2), toDate('2019-05-29'));
-SELECT dictHas('database_for_range_dict.range_dictionary', toUInt64(2), toDate('2019-05-31'));
+SELECT dictHas('range_dictionary', toUInt64(1), toDate('2019-05-15'));
+SELECT dictHas('range_dictionary', toUInt64(1), toDate('2019-05-29'));
+SELECT dictHas('range_dictionary', toUInt64(2), toDate('2019-05-29'));
+SELECT dictHas('range_dictionary', toUInt64(2), toDate('2019-05-31'));
 SELECT 'select columns from dictionary';
 SELECT 'allColumns';
-SELECT * FROM database_for_range_dict.range_dictionary ORDER BY CountryID, StartDate, EndDate;
+SELECT * FROM range_dictionary ORDER BY CountryID, StartDate, EndDate;
 SELECT 'noColumns';
-SELECT 1 FROM database_for_range_dict.range_dictionary ORDER BY CountryID, StartDate, EndDate;
+SELECT 1 FROM range_dictionary ORDER BY CountryID, StartDate, EndDate;
 SELECT 'onlySpecificColumns';
-SELECT CountryID, StartDate, Tax FROM database_for_range_dict.range_dictionary ORDER BY CountryID, StartDate, EndDate;
+SELECT CountryID, StartDate, Tax FROM range_dictionary ORDER BY CountryID, StartDate, EndDate;
 SELECT 'onlySpecificColumn';
-SELECT Tax FROM database_for_range_dict.range_dictionary ORDER BY CountryID, StartDate, EndDate;
+SELECT Tax FROM range_dictionary ORDER BY CountryID, StartDate, EndDate;
 
-DROP DICTIONARY database_for_range_dict.range_dictionary;
-DROP TABLE database_for_range_dict.date_table;
+DROP DICTIONARY range_dictionary;
+DROP TABLE date_table;
 
-CREATE TABLE database_for_range_dict.date_table
+CREATE TABLE date_table
 (
   CountryID UInt64,
   StartDate Date,
@@ -68,11 +62,11 @@ CREATE TABLE database_for_range_dict.date_table
 ENGINE = MergeTree()
 ORDER BY CountryID;
 
-INSERT INTO database_for_range_dict.date_table VALUES(1, toDate('2019-05-05'), toDate('2019-05-20'), 0.33);
-INSERT INTO database_for_range_dict.date_table VALUES(1, toDate('2019-05-21'), toDate('2019-05-30'), 0.42);
-INSERT INTO database_for_range_dict.date_table VALUES(2, toDate('2019-05-21'), toDate('2019-05-30'), NULL);
+INSERT INTO date_table VALUES(1, toDate('2019-05-05'), toDate('2019-05-20'), 0.33);
+INSERT INTO date_table VALUES(1, toDate('2019-05-21'), toDate('2019-05-30'), 0.42);
+INSERT INTO date_table VALUES(2, toDate('2019-05-21'), toDate('2019-05-30'), NULL);
 
-CREATE DICTIONARY database_for_range_dict.range_dictionary_nullable
+CREATE DICTIONARY range_dictionary_nullable
 (
   CountryID UInt64,
   StartDate Date,
@@ -80,35 +74,33 @@ CREATE DICTIONARY database_for_range_dict.range_dictionary_nullable
   Tax Nullable(Float64) DEFAULT 0.2
 )
 PRIMARY KEY CountryID
-SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'date_table' DB 'database_for_range_dict'))
+SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'date_table' DB currentDatabase()))
 LIFETIME(MIN 1 MAX 1000)
 LAYOUT(RANGE_HASHED())
 RANGE(MIN StartDate MAX EndDate);
 
 SELECT 'Dictionary nullable';
 SELECT 'dictGet';
-SELECT dictGet('database_for_range_dict.range_dictionary_nullable', 'Tax', toUInt64(1), toDate('2019-05-15'));
-SELECT dictGet('database_for_range_dict.range_dictionary_nullable', 'Tax', toUInt64(1), toDate('2019-05-29'));
-SELECT dictGet('database_for_range_dict.range_dictionary_nullable', 'Tax', toUInt64(2), toDate('2019-05-29'));
-SELECT dictGet('database_for_range_dict.range_dictionary_nullable', 'Tax', toUInt64(2), toDate('2019-05-31'));
-SELECT dictGetOrDefault('database_for_range_dict.range_dictionary_nullable', 'Tax', toUInt64(2), toDate('2019-05-31'), 0.4);
+SELECT dictGet('range_dictionary_nullable', 'Tax', toUInt64(1), toDate('2019-05-15'));
+SELECT dictGet('range_dictionary_nullable', 'Tax', toUInt64(1), toDate('2019-05-29'));
+SELECT dictGet('range_dictionary_nullable', 'Tax', toUInt64(2), toDate('2019-05-29'));
+SELECT dictGet('range_dictionary_nullable', 'Tax', toUInt64(2), toDate('2019-05-31'));
+SELECT dictGetOrDefault('range_dictionary_nullable', 'Tax', toUInt64(2), toDate('2019-05-31'), 0.4);
 SELECT 'dictHas';
-SELECT dictHas('database_for_range_dict.range_dictionary_nullable', toUInt64(1), toDate('2019-05-15'));
-SELECT dictHas('database_for_range_dict.range_dictionary_nullable', toUInt64(1), toDate('2019-05-29'));
-SELECT dictHas('database_for_range_dict.range_dictionary_nullable', toUInt64(2), toDate('2019-05-29'));
-SELECT dictHas('database_for_range_dict.range_dictionary_nullable', toUInt64(2), toDate('2019-05-31'));
+SELECT dictHas('range_dictionary_nullable', toUInt64(1), toDate('2019-05-15'));
+SELECT dictHas('range_dictionary_nullable', toUInt64(1), toDate('2019-05-29'));
+SELECT dictHas('range_dictionary_nullable', toUInt64(2), toDate('2019-05-29'));
+SELECT dictHas('range_dictionary_nullable', toUInt64(2), toDate('2019-05-31'));
 SELECT 'select columns from dictionary';
 SELECT 'allColumns';
-SELECT * FROM database_for_range_dict.range_dictionary_nullable ORDER BY CountryID, StartDate, EndDate;
+SELECT * FROM range_dictionary_nullable ORDER BY CountryID, StartDate, EndDate;
 SELECT 'noColumns';
-SELECT 1 FROM database_for_range_dict.range_dictionary_nullable ORDER BY CountryID, StartDate, EndDate;
+SELECT 1 FROM range_dictionary_nullable ORDER BY CountryID, StartDate, EndDate;
 SELECT 'onlySpecificColumns';
-SELECT CountryID, StartDate, Tax FROM database_for_range_dict.range_dictionary_nullable ORDER BY CountryID, StartDate, EndDate;
+SELECT CountryID, StartDate, Tax FROM range_dictionary_nullable ORDER BY CountryID, StartDate, EndDate;
 SELECT 'onlySpecificColumn';
-SELECT Tax FROM database_for_range_dict.range_dictionary_nullable ORDER BY CountryID, StartDate, EndDate;
+SELECT Tax FROM range_dictionary_nullable ORDER BY CountryID, StartDate, EndDate;
 
-DROP DICTIONARY database_for_range_dict.range_dictionary_nullable;
-DROP TABLE database_for_range_dict.date_table;
-
-DROP DATABASE database_for_range_dict;
+DROP DICTIONARY range_dictionary_nullable;
+DROP TABLE date_table;
 

From bb7039eeec01ce59008103727f8c03ddd26a3d29 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Thu, 1 Aug 2024 15:17:00 +0200
Subject: [PATCH 1184/1488] Make 01107_atomic_db_detach_attach parallelizable

---
 .../01107_atomic_db_detach_attach.sh          | 31 ++++++++++---------
 1 file changed, 16 insertions(+), 15 deletions(-)

diff --git a/tests/queries/0_stateless/01107_atomic_db_detach_attach.sh b/tests/queries/0_stateless/01107_atomic_db_detach_attach.sh
index bcaa70abbb5..e9879344259 100755
--- a/tests/queries/0_stateless/01107_atomic_db_detach_attach.sh
+++ b/tests/queries/0_stateless/01107_atomic_db_detach_attach.sh
@@ -1,29 +1,30 @@
 #!/usr/bin/env bash
-# Tags: no-parallel, no-fasttest
+# Tags: no-fasttest
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
-$CLICKHOUSE_CLIENT -q "DROP DATABASE IF EXISTS test_01107"
-$CLICKHOUSE_CLIENT -q "CREATE DATABASE test_01107 ENGINE=Atomic"
-$CLICKHOUSE_CLIENT -q "CREATE TABLE test_01107.mt (n UInt64) ENGINE=MergeTree() ORDER BY tuple()"
+NEW_DATABASE=test_01107_${CLICKHOUSE_DATABASE}
+$CLICKHOUSE_CLIENT -q "DROP DATABASE IF EXISTS ${NEW_DATABASE}"
+$CLICKHOUSE_CLIENT -q "CREATE DATABASE ${NEW_DATABASE} ENGINE=Atomic"
+$CLICKHOUSE_CLIENT -q "CREATE TABLE ${NEW_DATABASE}.mt (n UInt64) ENGINE=MergeTree() ORDER BY tuple()"
 
-$CLICKHOUSE_CLIENT --function_sleep_max_microseconds_per_block 60000000 -q "INSERT INTO test_01107.mt SELECT number + sleepEachRow(3) FROM numbers(5)" &
+$CLICKHOUSE_CLIENT --function_sleep_max_microseconds_per_block 60000000 -q "INSERT INTO ${NEW_DATABASE}.mt SELECT number + sleepEachRow(3) FROM numbers(5)" &
 sleep 1
 
-$CLICKHOUSE_CLIENT -q "DETACH TABLE test_01107.mt" --database_atomic_wait_for_drop_and_detach_synchronously=0
-$CLICKHOUSE_CLIENT -q "ATTACH TABLE test_01107.mt" --database_atomic_wait_for_drop_and_detach_synchronously=0 2>&1 | grep -F "Code: 57" > /dev/null && echo "OK"
-$CLICKHOUSE_CLIENT -q "DETACH DATABASE test_01107" --database_atomic_wait_for_drop_and_detach_synchronously=0 2>&1 | grep -F "Code: 219" > /dev/null && echo "OK"
+$CLICKHOUSE_CLIENT -q "DETACH TABLE ${NEW_DATABASE}.mt" --database_atomic_wait_for_drop_and_detach_synchronously=0
+$CLICKHOUSE_CLIENT -q "ATTACH TABLE ${NEW_DATABASE}.mt" --database_atomic_wait_for_drop_and_detach_synchronously=0 2>&1 | grep -F "Code: 57" > /dev/null && echo "OK"
+$CLICKHOUSE_CLIENT -q "DETACH DATABASE ${NEW_DATABASE}" --database_atomic_wait_for_drop_and_detach_synchronously=0 2>&1 | grep -F "Code: 219" > /dev/null && echo "OK"
 
 wait
-$CLICKHOUSE_CLIENT -q "ATTACH TABLE test_01107.mt"
-$CLICKHOUSE_CLIENT -q "SELECT count(n), sum(n) FROM test_01107.mt"
-$CLICKHOUSE_CLIENT -q "DETACH DATABASE test_01107" --database_atomic_wait_for_drop_and_detach_synchronously=0
-$CLICKHOUSE_CLIENT -q "ATTACH DATABASE test_01107"
-$CLICKHOUSE_CLIENT -q "SELECT count(n), sum(n) FROM test_01107.mt"
+$CLICKHOUSE_CLIENT -q "ATTACH TABLE ${NEW_DATABASE}.mt"
+$CLICKHOUSE_CLIENT -q "SELECT count(n), sum(n) FROM ${NEW_DATABASE}.mt"
+$CLICKHOUSE_CLIENT -q "DETACH DATABASE ${NEW_DATABASE}" --database_atomic_wait_for_drop_and_detach_synchronously=0
+$CLICKHOUSE_CLIENT -q "ATTACH DATABASE ${NEW_DATABASE}"
+$CLICKHOUSE_CLIENT -q "SELECT count(n), sum(n) FROM ${NEW_DATABASE}.mt"
 
-$CLICKHOUSE_CLIENT --function_sleep_max_microseconds_per_block 60000000 -q "INSERT INTO test_01107.mt SELECT number + sleepEachRow(1) FROM numbers(5)" && echo "end" &
+$CLICKHOUSE_CLIENT --function_sleep_max_microseconds_per_block 60000000 -q "INSERT INTO ${NEW_DATABASE}.mt SELECT number + sleepEachRow(1) FROM numbers(5)" && echo "end" &
 sleep 1
-$CLICKHOUSE_CLIENT -q "DROP DATABASE test_01107" --database_atomic_wait_for_drop_and_detach_synchronously=0 && sleep 1 && echo "dropped"
+$CLICKHOUSE_CLIENT -q "DROP DATABASE ${NEW_DATABASE}" --database_atomic_wait_for_drop_and_detach_synchronously=0 && sleep 1 && echo "dropped"
 wait

From 1d85f9b1cba3c8fe168286a660d3c0a4fd471a95 Mon Sep 17 00:00:00 2001
From: Michael Stetsyuk <mike.stetsyuk@gmail.com>
Date: Thu, 1 Aug 2024 14:42:58 +0100
Subject: [PATCH 1185/1488] fix remove_local_directory_contents

---
 tests/integration/test_storage_delta/test.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/tests/integration/test_storage_delta/test.py b/tests/integration/test_storage_delta/test.py
index e485bc90ee0..384b8296f66 100644
--- a/tests/integration/test_storage_delta/test.py
+++ b/tests/integration/test_storage_delta/test.py
@@ -52,9 +52,13 @@ def get_spark():
     return builder.master("local").getOrCreate()
 
 
-def remove_local_directory_contents(local_path):
-    for local_file in glob.glob(local_path + "/**"):
-        os.unlink(local_file)
+def remove_local_directory_contents(full_path):
+    for path in glob.glob(f"{full_path}/**"):
+        if os.path.isfile(path):
+            os.unlink(path)
+        else:
+            remove_local_directory_contents(path)
+            os.rmdir(path)
 
 
 @pytest.fixture(scope="module")

From 0913f0189ba350236d32d774265770b654860a80 Mon Sep 17 00:00:00 2001
From: Alex Katsman <alex.katsman@clickhouse.com>
Date: Wed, 31 Jul 2024 09:06:30 +0000
Subject: [PATCH 1186/1488] Don't count a search query as a search pattern
 match

---
 tests/integration/helpers/cluster.py              | 10 +++++++---
 .../integration/test_mask_sensitive_info/test.py  | 15 +++++++--------
 2 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py
index 90b28a4cda3..6bc0ece63ca 100644
--- a/tests/integration/helpers/cluster.py
+++ b/tests/integration/helpers/cluster.py
@@ -3922,7 +3922,11 @@ class ClickHouseInstance:
         )
 
     def contains_in_log(
-        self, substring, from_host=False, filename="clickhouse-server.log"
+        self,
+        substring,
+        from_host=False,
+        filename="clickhouse-server.log",
+        exclusion_substring="",
     ):
         if from_host:
             # We check fist file exists but want to look for all rotated logs as well
@@ -3930,7 +3934,7 @@ class ClickHouseInstance:
                 [
                     "bash",
                     "-c",
-                    f'[ -f {self.logs_dir}/{filename} ] && zgrep -aH "{substring}" {self.logs_dir}/{filename}* || true',
+                    f'[ -f {self.logs_dir}/{filename} ] && zgrep -aH "{substring}" {self.logs_dir}/{filename}* | ( [ -z "{exclusion_substring}" ] && cat || grep -v "${exclusion_substring}" ) || true',
                 ]
             )
         else:
@@ -3938,7 +3942,7 @@ class ClickHouseInstance:
                 [
                     "bash",
                     "-c",
-                    f'[ -f /var/log/clickhouse-server/{filename} ] && zgrep -aH "{substring}" /var/log/clickhouse-server/{filename} || true',
+                    f'[ -f /var/log/clickhouse-server/{filename} ] && zgrep -aH "{substring}" /var/log/clickhouse-server/{filename} | ( [ -z "{exclusion_substring}" ] && cat || grep -v "${exclusion_substring}" ) || true',
                 ]
             )
         return len(result) > 0
diff --git a/tests/integration/test_mask_sensitive_info/test.py b/tests/integration/test_mask_sensitive_info/test.py
index 902d3800324..6f6dc4d287f 100644
--- a/tests/integration/test_mask_sensitive_info/test.py
+++ b/tests/integration/test_mask_sensitive_info/test.py
@@ -13,6 +13,7 @@ node = cluster.add_instance(
     with_zookeeper=True,
     with_azurite=True,
 )
+base_search_query = "SELECT COUNT() FROM system.query_log WHERE query LIKE "
 
 
 @pytest.fixture(scope="module", autouse=True)
@@ -35,7 +36,7 @@ def check_logs(must_contain=[], must_not_contain=[]):
             .replace("]", "\\]")
             .replace("*", "\\*")
         )
-        assert node.contains_in_log(escaped_str)
+        assert node.contains_in_log(escaped_str, exclusion_substring=base_search_query)
 
     for str in must_not_contain:
         escaped_str = (
@@ -44,7 +45,9 @@ def check_logs(must_contain=[], must_not_contain=[]):
             .replace("]", "\\]")
             .replace("*", "\\*")
         )
-        assert not node.contains_in_log(escaped_str)
+        assert not node.contains_in_log(
+            escaped_str, exclusion_substring=base_search_query
+        )
 
     for str in must_contain:
         escaped_str = str.replace("'", "\\'")
@@ -60,7 +63,7 @@ def system_query_log_contains_search_pattern(search_pattern):
     return (
         int(
             node.query(
-                f"SELECT COUNT() FROM system.query_log WHERE query LIKE '%{search_pattern}%'"
+                f"{base_search_query}'%{search_pattern}%' AND query NOT LIKE '{base_search_query}%'"
             ).strip()
         )
         >= 1
@@ -105,7 +108,6 @@ def test_create_alter_user():
         must_not_contain=[
             password,
             "IDENTIFIED BY",
-            "IDENTIFIED BY",
             "IDENTIFIED WITH plaintext_password BY",
         ],
     )
@@ -366,10 +368,7 @@ def test_table_functions():
         f"remoteSecure(named_collection_6, addresses_expr = '127.{{2..11}}', database = 'default', table = 'remote_table', user = 'remote_user', password = '{password}')",
         f"s3('http://minio1:9001/root/data/test9.csv.gz', 'NOSIGN', 'CSV')",
         f"s3('http://minio1:9001/root/data/test10.csv.gz', 'minio', '{password}')",
-        (
-            f"deltaLake('http://minio1:9001/root/data/test11.csv.gz', 'minio', '{password}')",
-            "DNS_ERROR",
-        ),
+        f"deltaLake('http://minio1:9001/root/data/test11.csv.gz', 'minio', '{password}')",
         f"azureBlobStorage('{azure_conn_string}', 'cont', 'test_simple.csv', 'CSV')",
         f"azureBlobStorage('{azure_conn_string}', 'cont', 'test_simple_1.csv', 'CSV', 'none')",
         f"azureBlobStorage('{azure_conn_string}', 'cont', 'test_simple_2.csv', 'CSV', 'none', 'auto')",

From 048fbacc40062b05510916134c9d9525e7fab63a Mon Sep 17 00:00:00 2001
From: Tyler Hannan <tyler@clickhouse.com>
Date: Thu, 1 Aug 2024 16:48:19 +0200
Subject: [PATCH 1187/1488] Update README.md

---
 README.md | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/README.md b/README.md
index 98f9108f14c..2120a4d1211 100644
--- a/README.md
+++ b/README.md
@@ -34,17 +34,13 @@ curl https://clickhouse.com/ | sh
 
 Every month we get together with the community (users, contributors, customers, those interested in learning more about ClickHouse) to discuss what is coming in the latest release. If you are interested in sharing what you've built on ClickHouse, let us know.
 
-* [v24.7 Community Call](https://clickhouse.com/company/events/v24-7-community-release-call) - Jul 30
+* [v24.8 Community Call](https://clickhouse.com/company/events/v24-8-community-release-call) - August 29
 
 ## Upcoming Events
 
 Keep an eye out for upcoming meetups and events around the world. Somewhere else you want us to be? Please feel free to reach out to tyler `<at>` clickhouse `<dot>` com. You can also peruse [ClickHouse Events](https://clickhouse.com/company/news-events) for a list of all upcoming trainings, meetups, speaking engagements, etc.
 
-* [ClickHouse Meetup in Paris](https://www.meetup.com/clickhouse-france-user-group/events/300783448/) - Jul 9
-* [ClickHouse Cloud - Live Update Call](https://clickhouse.com/company/events/202407-cloud-update-live) - Jul 9
-* [ClickHouse Meetup @ Ramp - New York City](https://www.meetup.com/clickhouse-new-york-user-group/events/300595845/) - Jul 9
-* [AWS Summit in New York](https://clickhouse.com/company/events/2024-07-awssummit-nyc) - Jul 10
-* [ClickHouse Meetup @ Klaviyo - Boston](https://www.meetup.com/clickhouse-boston-user-group/events/300907870) - Jul 11
+* MORE COMING SOON!
 
 ## Recent Recordings
 * **Recent Meetup Videos**: [Meetup Playlist](https://www.youtube.com/playlist?list=PL0Z2YDlm0b3iNDUzpY1S3L_iV4nARda_U) Whenever possible recordings of the ClickHouse Community Meetups are edited and presented as individual talks. Current featuring "Modern SQL in 2023", "Fast, Concurrent, and Consistent Asynchronous INSERTS in ClickHouse", and "Full-Text Indices: Design and Experiments"

From 582bcfdc03e1124500325c3104497719473657cc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Thu, 1 Aug 2024 16:57:41 +0200
Subject: [PATCH 1188/1488] Add no-parallel back to
 01107_atomic_db_detach_attach

---
 tests/queries/0_stateless/01107_atomic_db_detach_attach.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/01107_atomic_db_detach_attach.sh b/tests/queries/0_stateless/01107_atomic_db_detach_attach.sh
index e9879344259..a6a99aadac2 100755
--- a/tests/queries/0_stateless/01107_atomic_db_detach_attach.sh
+++ b/tests/queries/0_stateless/01107_atomic_db_detach_attach.sh
@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
-# Tags: no-fasttest
+# Tags: no-fasttest, no-parallel
+# no-parallel: FIXME: Timing issues with INSERT + DETACH (https://github.com/ClickHouse/ClickHouse/pull/67610/files#r1700345054)
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh

From 0978441a0261c6003c7a9f4661ac87138e909622 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 31 Jul 2024 17:38:20 +0200
Subject: [PATCH 1189/1488] Minor change

---
 src/Databases/DatabaseOnDisk.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp
index f419f5811a1..734f354d9a5 100644
--- a/src/Databases/DatabaseOnDisk.cpp
+++ b/src/Databases/DatabaseOnDisk.cpp
@@ -313,7 +313,7 @@ void DatabaseOnDisk::detachTablePermanently(ContextPtr query_context, const Stri
         std::lock_guard lock(mutex);
         if (const auto it = snapshot_detached_tables.find(table_name); it == snapshot_detached_tables.end())
         {
-            throw Exception(ErrorCodes::LOGICAL_ERROR, "Snapshot doesn't contain info about detached table={}", table_name);
+            throw Exception(ErrorCodes::LOGICAL_ERROR, "Snapshot doesn't contain info about detached table `{}`", table_name);
         }
         else
         {

From 53cbb4811047cad2bdf2b882bc89ff9a83ac4577 Mon Sep 17 00:00:00 2001
From: Nikita Taranov <nikita.taranov@clickhouse.com>
Date: Thu, 1 Aug 2024 18:06:05 +0200
Subject: [PATCH 1190/1488] Try fix 03143_asof_join_ddb_long

---
 tests/queries/0_stateless/03143_asof_join_ddb_long.sql | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/03143_asof_join_ddb_long.sql b/tests/queries/0_stateless/03143_asof_join_ddb_long.sql
index 17a67511030..0b17ade5d1c 100644
--- a/tests/queries/0_stateless/03143_asof_join_ddb_long.sql
+++ b/tests/queries/0_stateless/03143_asof_join_ddb_long.sql
@@ -1,4 +1,5 @@
--- Tags: long
+-- Tags: long, no-random-merge-tree-settings
+-- no-random-merge-tree-settings - times out in private
 
 DROP TABLE IF EXISTS build;
 DROP TABLE IF EXISTS skewed_probe;

From 9362d1a5668bcd6e4e629ab26ec44d4bc8cb6513 Mon Sep 17 00:00:00 2001
From: Max K <maxkaynov@gmail.com>
Date: Wed, 31 Jul 2024 15:04:13 +0200
Subject: [PATCH 1191/1488] CI: Create release workflow dry run fix

fix
---
 .github/actions/release/action.yml   |  30 +++---
 .github/workflows/create_release.yml | 144 +++++++++++++++++++++++++--
 tests/ci/create_release.py           |   2 +-
 3 files changed, 153 insertions(+), 23 deletions(-)

diff --git a/.github/actions/release/action.yml b/.github/actions/release/action.yml
index c3897682a33..a287aa8b41d 100644
--- a/.github/actions/release/action.yml
+++ b/.github/actions/release/action.yml
@@ -16,8 +16,7 @@ inputs:
       - new
   dry-run:
     description: 'Dry run'
-    required: false
-    default: true
+    required: true
     type: boolean
   token:
     required: true
@@ -30,8 +29,7 @@ runs:
       shell: bash
       run: |
         python3 ./tests/ci/create_release.py --prepare-release-info \
-          --ref ${{ inputs.ref }} --release-type ${{ inputs.type }} \
-          ${{ inputs.dry-run && '--dry-run' || '' }}
+          --ref ${{ inputs.ref }} --release-type ${{ inputs.type }} ${{ inputs.dry-run == true && '--dry-run' || '' }}
         echo "::group::Release Info"
         python3 -m json.tool /tmp/release_info.json
         echo "::endgroup::"
@@ -44,20 +42,20 @@ runs:
       if: ${{ inputs.type == 'patch' }}
       shell: bash
       run: |
-        python3 ./tests/ci/create_release.py --download-packages ${{ inputs.dry-run && '--dry-run' || '' }}
+        python3 ./tests/ci/create_release.py --download-packages ${{ inputs.dry-run == true && '--dry-run' || '' }}
     - name: Push Git Tag for the Release
       shell: bash
       run: |
-        python3 ./tests/ci/create_release.py --push-release-tag ${{ inputs.dry-run && '--dry-run' || '' }}
+        python3 ./tests/ci/create_release.py --push-release-tag ${{ inputs.dry-run == true && '--dry-run' || '' }}
     - name: Push New Release Branch
       if: ${{ inputs.type == 'new' }}
       shell: bash
       run: |
-        python3 ./tests/ci/create_release.py --push-new-release-branch ${{ inputs.dry-run && '--dry-run' || '' }}
+        python3 ./tests/ci/create_release.py --push-new-release-branch ${{ inputs.dry-run == true && '--dry-run' || '' }}
     - name: Bump CH Version and Update Contributors' List
       shell: bash
       run: |
-        python3 ./tests/ci/create_release.py --create-bump-version-pr ${{ inputs.dry-run && '--dry-run' || '' }}
+        python3 ./tests/ci/create_release.py --create-bump-version-pr ${{ inputs.dry-run == true && '--dry-run' || '' }}
     - name: Bump Docker versions, Changelog, Security
       if: ${{ inputs.type == 'patch' }}
       shell: bash
@@ -107,37 +105,37 @@ runs:
       shell: bash
       if: ${{ inputs.type == 'patch' }}
       run: |
-        python3 ./tests/ci/create_release.py --create-gh-release ${{ inputs.dry-run && '--dry-run' || '' }}
+        python3 ./tests/ci/create_release.py --create-gh-release ${{ inputs.dry-run == true && '--dry-run' || '' }}
     - name: Export TGZ Packages
       if: ${{ inputs.type == 'patch' }}
       shell: bash
       run: |
-        python3 ./tests/ci/artifactory.py --export-tgz ${{ inputs.dry-run && '--dry-run' || '' }}
+        python3 ./tests/ci/artifactory.py --export-tgz ${{ inputs.dry-run == true && '--dry-run' || '' }}
     - name: Test TGZ Packages
       if: ${{ inputs.type == 'patch' }}
       shell: bash
       run: |
-        python3 ./tests/ci/artifactory.py --test-tgz ${{ inputs.dry-run && '--dry-run' || '' }}
+        python3 ./tests/ci/artifactory.py --test-tgz ${{ inputs.dry-run == true && '--dry-run' || '' }}
     - name: Export RPM Packages
       if: ${{ inputs.type == 'patch' }}
       shell: bash
       run: |
-        python3 ./tests/ci/artifactory.py --export-rpm ${{ inputs.dry-run && '--dry-run' || '' }}
+        python3 ./tests/ci/artifactory.py --export-rpm ${{ inputs.dry-run == true && '--dry-run' || '' }}
     - name: Test RPM Packages
       if: ${{ inputs.type == 'patch' }}
       shell: bash
       run: |
-        python3 ./tests/ci/artifactory.py --test-rpm ${{ inputs.dry-run && '--dry-run' || '' }}
+        python3 ./tests/ci/artifactory.py --test-rpm ${{ inputs.dry-run == true && '--dry-run' || '' }}
     - name: Export Debian Packages
       if: ${{ inputs.type == 'patch' }}
       shell: bash
       run: |
-        python3 ./tests/ci/artifactory.py --export-debian ${{ inputs.dry-run && '--dry-run' || '' }}
+        python3 ./tests/ci/artifactory.py --export-debian ${{ inputs.dry-run == true && '--dry-run' || '' }}
     - name: Test Debian Packages
       if: ${{ inputs.type == 'patch' }}
       shell: bash
       run: |
-        python3 ./tests/ci/artifactory.py --test-debian ${{ inputs.dry-run && '--dry-run' || '' }}
+        python3 ./tests/ci/artifactory.py --test-debian ${{ inputs.dry-run == true && '--dry-run' || '' }}
     - name: Docker clickhouse/clickhouse-server building
       if: ${{ inputs.type == 'patch' }}
       shell: bash
@@ -165,4 +163,4 @@ runs:
       if: ${{ !cancelled() }}
       shell: bash
       run: |
-        python3 ./tests/ci/create_release.py --post-status ${{ inputs.dry-run && '--dry-run' || '' }}
+        python3 ./tests/ci/create_release.py --post-status ${{ inputs.dry-run == true && '--dry-run' || '' }}
diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml
index 55644bdd503..217f27086c5 100644
--- a/.github/workflows/create_release.yml
+++ b/.github/workflows/create_release.yml
@@ -35,10 +35,142 @@ jobs:
         with:
           token: ${{secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN}}
           fetch-depth: 0
-      - name: Call Release Action
-        uses: ./.github/actions/release
+      - name: Prepare Release Info
+        shell: bash
+        run: |
+          python3 ./tests/ci/create_release.py --prepare-release-info \
+            --ref ${{ inputs.ref }} --release-type ${{ inputs.type }} ${{ inputs.dry-run == true && '--dry-run' || '' }}
+          echo "::group::Release Info"
+          python3 -m json.tool /tmp/release_info.json
+          echo "::endgroup::"
+          release_tag=$(jq -r '.release_tag' /tmp/release_info.json)
+          commit_sha=$(jq -r '.commit_sha' /tmp/release_info.json)
+          echo "Release Tag: $release_tag"
+          echo "RELEASE_TAG=$release_tag" >> "$GITHUB_ENV"
+          echo "COMMIT_SHA=$commit_sha" >> "$GITHUB_ENV"
+      - name: Download All Release Artifacts
+        if: ${{ inputs.type == 'patch' }}
+        shell: bash
+        run: |
+          python3 ./tests/ci/create_release.py --download-packages ${{ inputs.dry-run == true && '--dry-run' || '' }}
+      - name: Push Git Tag for the Release
+        shell: bash
+        run: |
+          python3 ./tests/ci/create_release.py --push-release-tag ${{ inputs.dry-run == true && '--dry-run' || '' }}
+      - name: Push New Release Branch
+        if: ${{ inputs.type == 'new' }}
+        shell: bash
+        run: |
+          python3 ./tests/ci/create_release.py --push-new-release-branch ${{ inputs.dry-run == true && '--dry-run' || '' }}
+      - name: Bump CH Version and Update Contributors' List
+        shell: bash
+        run: |
+          python3 ./tests/ci/create_release.py --create-bump-version-pr ${{ inputs.dry-run == true && '--dry-run' || '' }}
+      - name: Bump Docker versions, Changelog, Security
+        if: ${{ inputs.type == 'patch' }}
+        shell: bash
+        run: |
+          git checkout master
+          python3 ./tests/ci/create_release.py --set-progress-started --progress "update changelog, docker version, security"
+          echo "List versions"
+          ./utils/list-versions/list-versions.sh > ./utils/list-versions/version_date.tsv
+          echo "Update docker version"
+          ./utils/list-versions/update-docker-version.sh
+          echo "Generate ChangeLog"
+          export CI=1
+          docker run -u "${UID}:${GID}" -e PYTHONUNBUFFERED=1 -e CI=1 --network=host \
+            --volume=".:/ClickHouse" clickhouse/style-test \
+            /ClickHouse/tests/ci/changelog.py -v --debug-helpers \
+            --gh-user-or-token=${{secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN}} --jobs=5 \
+            --output="/ClickHouse/docs/changelogs/${{ env.RELEASE_TAG }}.md" ${{ env.RELEASE_TAG }}
+          git add ./docs/changelogs/${{ env.RELEASE_TAG }}.md
+          echo "Generate Security"
+          python3 ./utils/security-generator/generate_security.py > SECURITY.md
+          git diff HEAD
+      - name: Create ChangeLog PR
+        if: ${{ inputs.type == 'patch' && ! inputs.dry-run }}
+        uses: peter-evans/create-pull-request@v6
         with:
-          ref: ${{ inputs.ref }}
-          type: ${{ inputs.type }}
-          dry-run: ${{ inputs.dry-run }}
-          token: ${{secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN}}
+          author: "robot-clickhouse <robot-clickhouse@users.noreply.github.com>"
+          token: ${{ secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN }}
+          committer: "robot-clickhouse <robot-clickhouse@users.noreply.github.com>"
+          commit-message: Update version_date.tsv and changelogs after ${{ env.RELEASE_TAG }}
+          branch: auto/${{ env.RELEASE_TAG }}
+          assignees: ${{ github.event.sender.login }}  # assign the PR to the tag pusher
+          delete-branch: true
+          title: Update version_date.tsv and changelog after ${{ env.RELEASE_TAG }}
+          labels: do not test
+          body: |
+            Update version_date.tsv and changelogs after ${{ env.RELEASE_TAG }}
+            ### Changelog category (leave one):
+            - Not for changelog (changelog entry is not required)
+      - name: Complete previous steps and Restore git state
+        if: ${{ inputs.type == 'patch' }}
+        shell: bash
+        run: |
+          python3 ./tests/ci/create_release.py --set-progress-completed
+          git reset --hard HEAD
+          git checkout "$GITHUB_REF_NAME"
+      - name: Create GH Release
+        shell: bash
+        if: ${{ inputs.type == 'patch' }}
+        run: |
+          python3 ./tests/ci/create_release.py --create-gh-release ${{ inputs.dry-run == true && '--dry-run' || '' }}
+      - name: Export TGZ Packages
+        if: ${{ inputs.type == 'patch' }}
+        shell: bash
+        run: |
+          python3 ./tests/ci/artifactory.py --export-tgz ${{ inputs.dry-run == true && '--dry-run' || '' }}
+      - name: Test TGZ Packages
+        if: ${{ inputs.type == 'patch' }}
+        shell: bash
+        run: |
+          python3 ./tests/ci/artifactory.py --test-tgz ${{ inputs.dry-run == true && '--dry-run' || '' }}
+      - name: Export RPM Packages
+        if: ${{ inputs.type == 'patch' }}
+        shell: bash
+        run: |
+          python3 ./tests/ci/artifactory.py --export-rpm ${{ inputs.dry-run == true && '--dry-run' || '' }}
+      - name: Test RPM Packages
+        if: ${{ inputs.type == 'patch' }}
+        shell: bash
+        run: |
+          python3 ./tests/ci/artifactory.py --test-rpm ${{ inputs.dry-run == true && '--dry-run' || '' }}
+      - name: Export Debian Packages
+        if: ${{ inputs.type == 'patch' }}
+        shell: bash
+        run: |
+          python3 ./tests/ci/artifactory.py --export-debian ${{ inputs.dry-run == true && '--dry-run' || '' }}
+      - name: Test Debian Packages
+        if: ${{ inputs.type == 'patch' }}
+        shell: bash
+        run: |
+          python3 ./tests/ci/artifactory.py --test-debian ${{ inputs.dry-run == true && '--dry-run' || '' }}
+      - name: Docker clickhouse/clickhouse-server building
+        if: ${{ inputs.type == 'patch' }}
+        shell: bash
+        run: |
+          cd "./tests/ci"
+          python3 ./create_release.py --set-progress-started --progress "docker server release"
+          export CHECK_NAME="Docker server image"
+          python3 docker_server.py --release-type auto --version ${{ env.RELEASE_TAG }} --check-name "$CHECK_NAME" --sha ${{ env.COMMIT_SHA }} ${{ ! inputs.dry-run && '--push' || '' }}
+          python3 ./create_release.py --set-progress-completed
+      - name: Docker clickhouse/clickhouse-keeper building
+        if: ${{ inputs.type == 'patch' }}
+        shell: bash
+        run: |
+          cd "./tests/ci"
+          python3 ./create_release.py --set-progress-started --progress "docker keeper release"
+          export CHECK_NAME="Docker keeper image"
+          python3 docker_server.py --release-type auto --version ${{ env.RELEASE_TAG }} --check-name "$CHECK_NAME" --sha ${{ env.COMMIT_SHA }} ${{ ! inputs.dry-run && '--push'  || '' }}
+          python3 ./create_release.py --set-progress-completed
+      - name: Set current Release progress to Completed with OK
+        shell: bash
+        run: |
+          python3 ./tests/ci/create_release.py --set-progress-started --progress "completed"
+          python3 ./tests/ci/create_release.py --set-progress-completed
+      - name: Post Slack Message
+        if: ${{ !cancelled() }}
+        shell: bash
+        run: |
+          python3 ./tests/ci/create_release.py --post-status ${{ inputs.dry-run == true && '--dry-run' || '' }}
diff --git a/tests/ci/create_release.py b/tests/ci/create_release.py
index a0b4083b673..b02a0bb8ed5 100755
--- a/tests/ci/create_release.py
+++ b/tests/ci/create_release.py
@@ -315,7 +315,7 @@ class ReleaseInfo:
                 cmd_push_branch = f"{GIT_PREFIX} push --set-upstream origin {branch_upd_version_contributors}"
                 body_file = get_abs_path(".github/PULL_REQUEST_TEMPLATE.md")
                 actor = os.getenv("GITHUB_ACTOR", "") or "me"
-                cmd_create_pr = f"gh pr create --repo {GITHUB_REPOSITORY} --title 'Update version after release' --head {branch_upd_version_contributors} --base {self.release_branch} --body-file '{body_file} --label 'do not test' --assignee @{actor}"
+                cmd_create_pr = f"gh pr create --repo {GITHUB_REPOSITORY} --title 'Update version after release' --head {branch_upd_version_contributors} --base {self.release_branch} --body-file {body_file} --label 'do not test' --assignee {actor}"
                 Shell.run(cmd_commit_version_upd, check=True, dry_run=dry_run)
                 Shell.run(cmd_push_branch, check=True, dry_run=dry_run)
                 Shell.run(cmd_create_pr, check=True, dry_run=dry_run)

From c534cd5bc21b59788750bdfcfb4177ebba0afc85 Mon Sep 17 00:00:00 2001
From: Max K <maxkaynov@gmail.com>
Date: Wed, 31 Jul 2024 16:22:43 +0200
Subject: [PATCH 1192/1488] changelog.py to retrieve best token

s3fs fix

changelog.py to use base branch to filter prs
---
 .github/workflows/create_release.yml | 12 ++++++------
 tests/ci/artifactory.py              |  4 ++++
 tests/ci/changelog.py                | 23 +++++++++++++++++++++++
 3 files changed, 33 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml
index 217f27086c5..5e34f50fab5 100644
--- a/.github/workflows/create_release.yml
+++ b/.github/workflows/create_release.yml
@@ -81,7 +81,7 @@ jobs:
           docker run -u "${UID}:${GID}" -e PYTHONUNBUFFERED=1 -e CI=1 --network=host \
             --volume=".:/ClickHouse" clickhouse/style-test \
             /ClickHouse/tests/ci/changelog.py -v --debug-helpers \
-            --gh-user-or-token=${{secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN}} --jobs=5 \
+            --jobs=5 \
             --output="/ClickHouse/docs/changelogs/${{ env.RELEASE_TAG }}.md" ${{ env.RELEASE_TAG }}
           git add ./docs/changelogs/${{ env.RELEASE_TAG }}.md
           echo "Generate Security"
@@ -111,11 +111,11 @@ jobs:
           python3 ./tests/ci/create_release.py --set-progress-completed
           git reset --hard HEAD
           git checkout "$GITHUB_REF_NAME"
-      - name: Create GH Release
-        shell: bash
-        if: ${{ inputs.type == 'patch' }}
-        run: |
-          python3 ./tests/ci/create_release.py --create-gh-release ${{ inputs.dry-run == true && '--dry-run' || '' }}
+#      - name: Create GH Release
+#        shell: bash
+#        if: ${{ inputs.type == 'patch' }}
+#        run: |
+#          python3 ./tests/ci/create_release.py --create-gh-release ${{ inputs.dry-run == true && '--dry-run' || '' }}
       - name: Export TGZ Packages
         if: ${{ inputs.type == 'patch' }}
         shell: bash
diff --git a/tests/ci/artifactory.py b/tests/ci/artifactory.py
index 86dcaf79854..4ee326593e6 100644
--- a/tests/ci/artifactory.py
+++ b/tests/ci/artifactory.py
@@ -52,6 +52,10 @@ class R2MountPoint:
                 if self.CACHE_ENABLED
                 else ""
             )
+            if not dry_run:
+                self.aux_mount_options += (
+                    "-o passwd_file /home/ubuntu/.passwd-s3fs_packages "
+                )
             # without -o nomultipart there are errors like "Error 5 writing to /home/ubuntu/***.deb: Input/output error"
             self.mount_cmd = f"s3fs {self.bucket_name} {self.MOUNT_POINT} -o url={self.API_ENDPOINT} -o use_path_request_style -o umask=0000 -o nomultipart -o logfile={self.LOG_FILE} {self.aux_mount_options}"
         elif self.app == MountPointApp.RCLONE:
diff --git a/tests/ci/changelog.py b/tests/ci/changelog.py
index 3ba618f3ae5..e23dd8e4c67 100755
--- a/tests/ci/changelog.py
+++ b/tests/ci/changelog.py
@@ -19,6 +19,8 @@ from env_helper import TEMP_PATH
 from git_helper import git_runner, is_shallow
 from github_helper import GitHub, PullRequest, PullRequests, Repository
 from s3_helper import S3Helper
+from get_robot_token import get_best_robot_token
+from ci_utils import Shell
 from version_helper import (
     FILE_WITH_VERSION_PATH,
     get_abs_path,
@@ -171,6 +173,7 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument(
         "--gh-user-or-token",
         help="user name or GH token to authenticate",
+        default=get_best_robot_token(),
     )
     parser.add_argument(
         "--gh-password",
@@ -397,6 +400,15 @@ def get_year(prs: PullRequests) -> int:
     return max(pr.created_at.year for pr in prs)
 
 
+def get_branch_by_tag(tag: str) -> Optional[str]:
+    tag.removeprefix("v")
+    versions = tag.split(".")
+    if len(versions) < 3:
+        print("ERROR: Can't get branch by tag")
+        return None
+    return f"{versions[0]}.{versions[1]}"
+
+
 def main():
     log_levels = [logging.WARN, logging.INFO, logging.DEBUG]
     args = parse_args()
@@ -446,6 +458,17 @@ def main():
     gh_cache = GitHubCache(gh.cache_path, temp_path, S3Helper())
     gh_cache.download()
     query = f"type:pr repo:{args.repo} is:merged"
+    branch = get_branch_by_tag(TO_REF)
+    if branch and Shell.check(f"git show-ref --quiet {branch}"):
+        try:
+            if int(branch.split(".")[-1]) > 1:
+                query += f" base:{branch}"
+            print(f"NOTE: will use base branch to filter PRs {branch}")
+        except ValueError:
+            print(f"ERROR: cannot get minor version from branch {branch} - pass")
+            pass
+    else:
+        print(f"ERROR: invalid branch {branch} - pass")
     prs = gh.get_pulls_from_search(
         query=query, merged=merged, sort="created", progress_func=tqdm.tqdm
     )

From 8214910cc7bd84f613631c2fada9682820df8003 Mon Sep 17 00:00:00 2001
From: Max K <maxkaynov@gmail.com>
Date: Wed, 31 Jul 2024 20:14:22 +0200
Subject: [PATCH 1193/1488] add geesfs

---
 .github/workflows/create_release.yml | 10 +++----
 .github/workflows/release.yml        |  4 +--
 tests/ci/artifactory.py              | 35 +++++++++++++++++++++---
 tests/ci/changelog.py                | 41 ++++++++++++++++++----------
 tests/ci/ci_utils.py                 | 11 ++++----
 5 files changed, 69 insertions(+), 32 deletions(-)

diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml
index 5e34f50fab5..c3126abe461 100644
--- a/.github/workflows/create_release.yml
+++ b/.github/workflows/create_release.yml
@@ -62,15 +62,14 @@ jobs:
         shell: bash
         run: |
           python3 ./tests/ci/create_release.py --push-new-release-branch ${{ inputs.dry-run == true && '--dry-run' || '' }}
-      - name: Bump CH Version and Update Contributors' List
-        shell: bash
-        run: |
-          python3 ./tests/ci/create_release.py --create-bump-version-pr ${{ inputs.dry-run == true && '--dry-run' || '' }}
+#      - name: Bump CH Version and Update Contributors' List
+#        shell: bash
+#        run: |
+#          python3 ./tests/ci/create_release.py --create-bump-version-pr ${{ inputs.dry-run == true && '--dry-run' || '' }}
       - name: Bump Docker versions, Changelog, Security
         if: ${{ inputs.type == 'patch' }}
         shell: bash
         run: |
-          git checkout master
           python3 ./tests/ci/create_release.py --set-progress-started --progress "update changelog, docker version, security"
           echo "List versions"
           ./utils/list-versions/list-versions.sh > ./utils/list-versions/version_date.tsv
@@ -96,6 +95,7 @@ jobs:
           committer: "robot-clickhouse <robot-clickhouse@users.noreply.github.com>"
           commit-message: Update version_date.tsv and changelogs after ${{ env.RELEASE_TAG }}
           branch: auto/${{ env.RELEASE_TAG }}
+          base: master
           assignees: ${{ github.event.sender.login }}  # assign the PR to the tag pusher
           delete-branch: true
           title: Update version_date.tsv and changelog after ${{ env.RELEASE_TAG }}
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 3bd6dfae6ca..8620d15ec19 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -55,12 +55,12 @@ jobs:
       run: |
         cd "$GITHUB_WORKSPACE/tests/ci"
         export CHECK_NAME="Docker server image"
-        python3 docker_server.py --release-type auto --version "$GITHUB_TAG" --check-name "$CHECK_NAME" --push
+        python3 docker_server.py --release-type auto --version "$GITHUB_TAG" --sha "$(git rev-list -n 1 $GITHUB_TAG)" --check-name "$CHECK_NAME" --push
     - name: Check docker clickhouse/clickhouse-keeper building
       run: |
         cd "$GITHUB_WORKSPACE/tests/ci"
         export CHECK_NAME="Docker keeper image"
-        python3 docker_server.py --release-type auto --version "$GITHUB_TAG" --check-name "$CHECK_NAME" --push
+        python3 docker_server.py --release-type auto --version "$GITHUB_TAG" --sha "$(git rev-list -n 1 $GITHUB_TAG)" --check-name "$CHECK_NAME" --push
     - name: Cleanup
       if: always()
       run: |
diff --git a/tests/ci/artifactory.py b/tests/ci/artifactory.py
index 4ee326593e6..a508374f856 100644
--- a/tests/ci/artifactory.py
+++ b/tests/ci/artifactory.py
@@ -15,6 +15,7 @@ from ci_utils import WithIter, Shell
 class MountPointApp(metaclass=WithIter):
     RCLONE = "rclone"
     S3FS = "s3fs"
+    GEESEFS = "geesefs"
 
 
 class R2MountPoint:
@@ -70,6 +71,20 @@ class R2MountPoint:
             )
             # Use --no-modtime to try to avoid: ERROR : rpm/lts/clickhouse-client-24.3.6.5.x86_64.rpm: Failed to apply pending mod time
             self.mount_cmd = f"rclone mount remote:{self.bucket_name} {self.MOUNT_POINT} --daemon --cache-dir {self.cache_dir} --umask 0000 --log-file {self.LOG_FILE} {self.aux_mount_options}"
+        elif self.app == MountPointApp.GEESEFS:
+            self.cache_dir = "/home/ubuntu/geesefs_cache"
+            self.aux_mount_options += (
+                f" --cache={self.cache_dir} " if self.CACHE_ENABLED else ""
+            )
+            if not dry_run:
+                self.aux_mount_options += f" --shared-config=/home/ubuntu/.r2_auth "
+            else:
+                self.aux_mount_options += (
+                    f" --shared-config=/home/ubuntu/.r2_auth_test "
+                )
+            if self.DEBUG:
+                self.aux_mount_options += " --debug_s3 --debug_fuse "
+            self.mount_cmd = f"geesefs --endpoint={self.API_ENDPOINT} --cheap --memory-limit=2050 --gc-interval=100 --max-flushers=5 --max-parallel-parts=1 --max-parallel-copy=2 --log-file={self.LOG_FILE} {self.aux_mount_options} {self.bucket_name} {self.MOUNT_POINT}"
         else:
             assert False
 
@@ -87,7 +102,7 @@ class R2MountPoint:
         Shell.run(_UNMOUNT_CMD)
         Shell.run(_MKDIR_CMD)
         Shell.run(_MKDIR_FOR_CACHE)
-        if self.app == MountPointApp.S3FS:
+        if self.app != MountPointApp.RCLONE:
             Shell.run(self.mount_cmd, check=True)
         else:
             # didn't manage to use simple run() and without blocking or failure
@@ -158,7 +173,13 @@ class DebianArtifactory:
         cmd = f'docker run --rm ubuntu:latest bash -c "apt update -y; apt install -y sudo gnupg ca-certificates; apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 8919F6BD2B48D754; {debian_command}"'
         print("Running test command:")
         print(f"  {cmd}")
-        Shell.run(cmd, check=True)
+        assert Shell.check(cmd)
+        print(f"Test packages installation, version [latest]")
+        debian_command_2 = f"echo 'deb {self.repo_url} stable main' | tee /etc/apt/sources.list.d/clickhouse.list; apt update -y; apt-get install -y clickhouse-common-static clickhouse-client"
+        cmd = f'docker run --rm ubuntu:latest bash -c "apt update -y; apt install -y sudo gnupg ca-certificates; apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 8919F6BD2B48D754; {debian_command_2}"'
+        print("Running test command:")
+        print(f"  {cmd}")
+        assert Shell.check(cmd)
         self.release_info.debian_command = debian_command
         self.release_info.dump()
 
@@ -234,7 +255,13 @@ class RpmArtifactory:
         cmd = f'docker run --rm fedora:latest /bin/bash -c "dnf -y install dnf-plugins-core && dnf config-manager --add-repo={self.repo_url} && {rpm_command}"'
         print("Running test command:")
         print(f"  {cmd}")
-        Shell.run(cmd, check=True)
+        assert Shell.check(cmd)
+        print(f"Test package installation, version [latest]")
+        rpm_command_2 = f"dnf config-manager --add-repo={self.repo_url} && dnf makecache && dnf -y install clickhouse-client"
+        cmd = f'docker run --rm fedora:latest /bin/bash -c "dnf -y install dnf-plugins-core && dnf config-manager --add-repo={self.repo_url} && {rpm_command_2}"'
+        print("Running test command:")
+        print(f"  {cmd}")
+        assert Shell.check(cmd)
         self.release_info.rpm_command = rpm_command
         self.release_info.dump()
 
@@ -350,7 +377,7 @@ if __name__ == "__main__":
            ERROR : IO error: NotImplemented: versionId not implemented
            Failed to copy: NotImplemented: versionId not implemented
     """
-    mp = R2MountPoint(MountPointApp.S3FS, dry_run=args.dry_run)
+    mp = R2MountPoint(MountPointApp.GEESEFS, dry_run=args.dry_run)
     if args.export_debian:
         with ReleaseContextManager(
             release_progress=ReleaseProgress.EXPORT_DEB
diff --git a/tests/ci/changelog.py b/tests/ci/changelog.py
index e23dd8e4c67..929f0f3523a 100755
--- a/tests/ci/changelog.py
+++ b/tests/ci/changelog.py
@@ -7,7 +7,7 @@ import re
 from datetime import date, timedelta
 from pathlib import Path
 from subprocess import DEVNULL
-from typing import Any, Dict, List, Optional, TextIO
+from typing import Any, Dict, List, Optional, TextIO, Tuple
 
 import tqdm  # type: ignore
 from github.GithubException import RateLimitExceededException, UnknownObjectException
@@ -400,13 +400,19 @@ def get_year(prs: PullRequests) -> int:
     return max(pr.created_at.year for pr in prs)
 
 
-def get_branch_by_tag(tag: str) -> Optional[str]:
-    tag.removeprefix("v")
+def get_branch_and_patch_by_tag(tag: str) -> Tuple[Optional[str], Optional[int]]:
+    tag = tag.removeprefix("v")
     versions = tag.split(".")
-    if len(versions) < 3:
+    if len(versions) < 4:
         print("ERROR: Can't get branch by tag")
-        return None
-    return f"{versions[0]}.{versions[1]}"
+        return None, None
+    try:
+        patch_version = int(versions[2])
+        branch = f"{int(versions[0])}.{int(versions[1])}"
+        print(f"Branch [{branch}], patch version [{patch_version}]")
+    except ValueError:
+        return None, None
+    return branch, patch_version
 
 
 def main():
@@ -458,17 +464,22 @@ def main():
     gh_cache = GitHubCache(gh.cache_path, temp_path, S3Helper())
     gh_cache.download()
     query = f"type:pr repo:{args.repo} is:merged"
-    branch = get_branch_by_tag(TO_REF)
-    if branch and Shell.check(f"git show-ref --quiet {branch}"):
-        try:
-            if int(branch.split(".")[-1]) > 1:
-                query += f" base:{branch}"
-            print(f"NOTE: will use base branch to filter PRs {branch}")
-        except ValueError:
-            print(f"ERROR: cannot get minor version from branch {branch} - pass")
-            pass
+
+    branch, patch = get_branch_and_patch_by_tag(TO_REF)
+    if branch and patch and Shell.check(f"git show-ref --quiet {branch}"):
+        if patch > 1:
+            query += f" base:{branch}"
+            print(
+                f"NOTE: It's a patch [{patch}]. will use base branch to filter PRs [{branch}]"
+            )
+        else:
+            print(
+                f"NOTE: It's a first patch version. should count PRs merged on master - won't filter PRs by branch"
+            )
     else:
         print(f"ERROR: invalid branch {branch} - pass")
+
+    print(f"Fetch PRs with query {query}")
     prs = gh.get_pulls_from_search(
         query=query, merged=merged, sort="created", progress_func=tqdm.tqdm
     )
diff --git a/tests/ci/ci_utils.py b/tests/ci/ci_utils.py
index 447aac74c7f..3182c0bc5d8 100644
--- a/tests/ci/ci_utils.py
+++ b/tests/ci/ci_utils.py
@@ -246,15 +246,14 @@ class Shell:
 
     @classmethod
     def check(cls, command):
-        result = subprocess.run(
+        proc = subprocess.Popen(
             command,
             shell=True,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
-            text=True,
-            check=False,
+            stdout=subprocess.STDOUT,
+            stderr=subprocess.STDOUT,
         )
-        return result.returncode == 0
+        proc.wait()
+        return proc.returncode == 0
 
 
 class Utils:

From 4802ea540a4691c13386b74416352155b93f713d Mon Sep 17 00:00:00 2001
From: Max K <maxkaynov@gmail.com>
Date: Thu, 1 Aug 2024 11:57:54 +0200
Subject: [PATCH 1194/1488] improve ci_utils' Shell

---
 .github/workflows/pull_request.yml |  9 +++-
 .github/workflows/release.yml      |  6 ++-
 pyproject.toml                     |  1 +
 tests/ci/artifactory.py            | 84 +++++++++++------------------
 tests/ci/auto_release.py           |  5 +-
 tests/ci/ci_buddy.py               |  6 ++-
 tests/ci/ci_definitions.py         |  3 +-
 tests/ci/ci_utils.py               | 72 ++++++++++++-------------
 tests/ci/create_release.py         | 86 +++++++++++++++++-------------
 tests/ci/docker_images_helper.py   |  8 +--
 10 files changed, 141 insertions(+), 139 deletions(-)

diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
index 854dff530e7..04bef1460a6 100644
--- a/.github/workflows/pull_request.yml
+++ b/.github/workflows/pull_request.yml
@@ -142,8 +142,13 @@ jobs:
   # Reports should run even if Builds_1/2 fail - run them separately (not in Tests_1/2/3)
   Builds_Report:
     # run report check for failed builds to indicate the CI error
-    if: ${{ !cancelled() && needs.RunConfig.result == 'success' && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'Builds') }}
-    needs: [RunConfig, StyleCheck, Builds_1, Builds_2]
+    if: ${{ !cancelled()
+      && needs.RunConfig.result == 'success'
+      && needs.StyleCheck.result != 'failure'
+      && needs.FastTest.result != 'failure'
+      && needs.BuildDockers.result != 'failure'
+      && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'Builds') }}
+    needs: [RunConfig, BuildDockers, StyleCheck, FastTest, Builds_1, Builds_2]
     uses: ./.github/workflows/reusable_test.yml
     with:
       test_name: Builds
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 8620d15ec19..7dc4e3298a6 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -55,12 +55,14 @@ jobs:
       run: |
         cd "$GITHUB_WORKSPACE/tests/ci"
         export CHECK_NAME="Docker server image"
-        python3 docker_server.py --release-type auto --version "$GITHUB_TAG" --sha "$(git rev-list -n 1 $GITHUB_TAG)" --check-name "$CHECK_NAME" --push
+        SHA=$(git rev-list -n 1 "$GITHUB_TAG")
+        python3 docker_server.py --release-type auto --version "$GITHUB_TAG" --sha "$SHA"  --check-name "$CHECK_NAME" --push
     - name: Check docker clickhouse/clickhouse-keeper building
       run: |
         cd "$GITHUB_WORKSPACE/tests/ci"
         export CHECK_NAME="Docker keeper image"
-        python3 docker_server.py --release-type auto --version "$GITHUB_TAG" --sha "$(git rev-list -n 1 $GITHUB_TAG)" --check-name "$CHECK_NAME" --push
+        SHA=$(git rev-list -n 1 "$GITHUB_TAG")
+        python3 docker_server.py --release-type auto --version "$GITHUB_TAG" --sha "$SHA" --check-name "$CHECK_NAME" --push
     - name: Cleanup
       if: always()
       run: |
diff --git a/pyproject.toml b/pyproject.toml
index c89d46c0929..9bbeac3ddae 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -39,6 +39,7 @@ disable = '''
           no-else-return,
           global-statement,
           f-string-without-interpolation,
+          consider-using-with,
           '''
 
 [tool.pylint.SIMILARITIES]
diff --git a/tests/ci/artifactory.py b/tests/ci/artifactory.py
index a508374f856..71deaccf917 100644
--- a/tests/ci/artifactory.py
+++ b/tests/ci/artifactory.py
@@ -13,7 +13,6 @@ from ci_utils import WithIter, Shell
 
 
 class MountPointApp(metaclass=WithIter):
-    RCLONE = "rclone"
     S3FS = "s3fs"
     GEESEFS = "geesefs"
 
@@ -31,9 +30,6 @@ class R2MountPoint:
     DEBUG = True
     # enable cache for mountpoint
     CACHE_ENABLED = False
-    # TODO: which mode is better: minimal/writes/full/off
-    _RCLONE_CACHE_MODE = "minimal"
-    UMASK = "0000"
 
     def __init__(self, app: str, dry_run: bool) -> None:
         assert app in MountPointApp
@@ -59,18 +55,6 @@ class R2MountPoint:
                 )
             # without -o nomultipart there are errors like "Error 5 writing to /home/ubuntu/***.deb: Input/output error"
             self.mount_cmd = f"s3fs {self.bucket_name} {self.MOUNT_POINT} -o url={self.API_ENDPOINT} -o use_path_request_style -o umask=0000 -o nomultipart -o logfile={self.LOG_FILE} {self.aux_mount_options}"
-        elif self.app == MountPointApp.RCLONE:
-            # run rclone mount process asynchronously, otherwise subprocess.run(daemonized command) will not return
-            self.cache_dir = "/home/ubuntu/rclone_cache"
-            self.aux_mount_options += "--no-modtime " if self.NOMODTIME else ""
-            self.aux_mount_options += "-v " if self.DEBUG else ""  # -vv too verbose
-            self.aux_mount_options += (
-                f"--vfs-cache-mode {self._RCLONE_CACHE_MODE} --vfs-cache-max-size {self._CACHE_MAX_SIZE_GB}G"
-                if self.CACHE_ENABLED
-                else "--vfs-cache-mode off"
-            )
-            # Use --no-modtime to try to avoid: ERROR : rpm/lts/clickhouse-client-24.3.6.5.x86_64.rpm: Failed to apply pending mod time
-            self.mount_cmd = f"rclone mount remote:{self.bucket_name} {self.MOUNT_POINT} --daemon --cache-dir {self.cache_dir} --umask 0000 --log-file {self.LOG_FILE} {self.aux_mount_options}"
         elif self.app == MountPointApp.GEESEFS:
             self.cache_dir = "/home/ubuntu/geesefs_cache"
             self.aux_mount_options += (
@@ -98,22 +82,17 @@ class R2MountPoint:
         )
 
         _TEST_MOUNT_CMD = f"mount | grep -q {self.MOUNT_POINT}"
-        Shell.run(_CLEAN_LOG_FILE_CMD)
-        Shell.run(_UNMOUNT_CMD)
-        Shell.run(_MKDIR_CMD)
-        Shell.run(_MKDIR_FOR_CACHE)
-        if self.app != MountPointApp.RCLONE:
-            Shell.run(self.mount_cmd, check=True)
-        else:
-            # didn't manage to use simple run() and without blocking or failure
-            Shell.run_as_daemon(self.mount_cmd)
+        Shell.check(_CLEAN_LOG_FILE_CMD, verbose=True)
+        Shell.check(_UNMOUNT_CMD, verbose=True)
+        Shell.check(_MKDIR_CMD, verbose=True)
+        Shell.check(_MKDIR_FOR_CACHE, verbose=True)
+        Shell.check(self.mount_cmd, strict=True, verbose=True)
         time.sleep(3)
-        Shell.run(_TEST_MOUNT_CMD, check=True)
+        Shell.check(_TEST_MOUNT_CMD, strict=True, verbose=True)
 
     @classmethod
     def teardown(cls):
-        print(f"Unmount [{cls.MOUNT_POINT}]")
-        Shell.run(f"umount {cls.MOUNT_POINT}")
+        Shell.check(f"umount {cls.MOUNT_POINT}", verbose=True)
 
 
 class RepoCodenames(metaclass=WithIter):
@@ -148,10 +127,9 @@ class DebianArtifactory:
         ]
         REPREPRO_CMD_PREFIX = f"reprepro --basedir {R2MountPoint.MOUNT_POINT}/configs/deb --outdir {R2MountPoint.MOUNT_POINT}/deb --verbose"
         cmd = f"{REPREPRO_CMD_PREFIX} includedeb {self.codename} {' '.join(paths)}"
-        print("Running export command:")
-        print(f"  {cmd}")
-        Shell.run(cmd, check=True)
-        Shell.run("sync")
+        print("Running export commands:")
+        Shell.check(cmd, strict=True, verbose=True)
+        Shell.check("sync")
 
         if self.codename == RepoCodenames.LTS:
             packages_with_version = [
@@ -163,11 +141,11 @@ class DebianArtifactory:
             cmd = f"{REPREPRO_CMD_PREFIX} copy {RepoCodenames.STABLE} {RepoCodenames.LTS} {' '.join(packages_with_version)}"
             print("Running copy command:")
             print(f"  {cmd}")
-            Shell.run(cmd, check=True)
-            Shell.run("sync")
+            Shell.check(cmd, strict=True)
+            Shell.check("sync")
 
     def test_packages(self):
-        Shell.run("docker pull ubuntu:latest")
+        Shell.check("docker pull ubuntu:latest", strict=True)
         print(f"Test packages installation, version [{self.version}]")
         debian_command = f"echo 'deb {self.repo_url} stable main' | tee /etc/apt/sources.list.d/clickhouse.list; apt update -y; apt-get install -y clickhouse-common-static={self.version} clickhouse-client={self.version}"
         cmd = f'docker run --rm ubuntu:latest bash -c "apt update -y; apt install -y sudo gnupg ca-certificates; apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 8919F6BD2B48D754; {debian_command}"'
@@ -236,20 +214,18 @@ class RpmArtifactory:
         print(f"Exporting RPM packages into [{codename}]")
 
         for command in commands:
-            print("Running command:")
-            print(f"    {command}")
-            Shell.run(command, check=True)
+            Shell.check(command, strict=True, verbose=True)
 
         update_public_key = f"gpg --armor --export {self._SIGN_KEY}"
         pub_key_path = dest_dir / "repodata" / "repomd.xml.key"
         print("Updating repomd.xml.key")
-        pub_key_path.write_text(Shell.run(update_public_key, check=True))
+        pub_key_path.write_text(Shell.get_output_or_raise(update_public_key))
         if codename == RepoCodenames.LTS:
             self.export_packages(RepoCodenames.STABLE)
-        Shell.run("sync")
+        Shell.check("sync")
 
     def test_packages(self):
-        Shell.run("docker pull fedora:latest")
+        Shell.check("docker pull fedora:latest", strict=True)
         print(f"Test package installation, version [{self.version}]")
         rpm_command = f"dnf config-manager --add-repo={self.repo_url} && dnf makecache && dnf -y install clickhouse-client-{self.version}-1"
         cmd = f'docker run --rm fedora:latest /bin/bash -c "dnf -y install dnf-plugins-core && dnf config-manager --add-repo={self.repo_url} && {rpm_command}"'
@@ -302,26 +278,30 @@ class TgzArtifactory:
 
         if codename == RepoCodenames.LTS:
             self.export_packages(RepoCodenames.STABLE)
-        Shell.run("sync")
+        Shell.check("sync")
 
     def test_packages(self):
         tgz_file = "/tmp/tmp.tgz"
         tgz_sha_file = "/tmp/tmp.tgz.sha512"
         cmd = f"curl -o {tgz_file} -f0 {self.repo_url}/stable/clickhouse-client-{self.version}-arm64.tgz"
-        Shell.run(
+        Shell.check(
             cmd,
-            check=True,
+            strict=True,
+            verbose=True,
         )
-        Shell.run(
+        Shell.check(
             f"curl -o {tgz_sha_file} -f0 {self.repo_url}/stable/clickhouse-client-{self.version}-arm64.tgz.sha512",
-            check=True,
+            strict=True,
+            verbose=True,
+        )
+        expected_checksum = Shell.get_output_or_raise(f"cut -d ' ' -f 1 {tgz_sha_file}")
+        actual_checksum = Shell.get_output_or_raise(
+            f"sha512sum {tgz_file} | cut -d ' ' -f 1"
         )
-        expected_checksum = Shell.run(f"cut -d ' ' -f 1 {tgz_sha_file}", check=True)
-        actual_checksum = Shell.run(f"sha512sum {tgz_file} | cut -d ' ' -f 1")
         assert (
             expected_checksum == actual_checksum
         ), f"[{actual_checksum} != {expected_checksum}]"
-        Shell.run("rm /tmp/tmp.tgz*")
+        Shell.check("rm /tmp/tmp.tgz*", verbose=True)
         self.release_info.tgz_command = cmd
         self.release_info.dump()
 
@@ -373,9 +353,9 @@ if __name__ == "__main__":
     args = parse_args()
 
     """
-    Use S3FS. RCLONE has some errors with r2 remote which I didn't figure out how to resolve:
-           ERROR : IO error: NotImplemented: versionId not implemented
-           Failed to copy: NotImplemented: versionId not implemented
+    S3FS - very slow with a big repo
+    RCLONE - fuse had many different errors with r2 remote and completely removed
+    GEESEFS ? 
     """
     mp = R2MountPoint(MountPointApp.GEESEFS, dry_run=args.dry_run)
     if args.export_debian:
diff --git a/tests/ci/auto_release.py b/tests/ci/auto_release.py
index f2386fe207f..6c17b4c74ad 100644
--- a/tests/ci/auto_release.py
+++ b/tests/ci/auto_release.py
@@ -85,7 +85,7 @@ class AutoReleaseInfo:
 def _prepare(token):
     assert len(token) > 10
     os.environ["GH_TOKEN"] = token
-    Shell.run("gh auth status", check=True)
+    Shell.check("gh auth status")
 
     gh = GitHub(token)
     prs = gh.get_release_pulls(GITHUB_REPOSITORY)
@@ -106,9 +106,8 @@ def _prepare(token):
         latest_release_tag_ref = refs[-1]
         latest_release_tag = repo.get_git_tag(latest_release_tag_ref.object.sha)
 
-        commits = Shell.run(
+        commits = Shell.get_output_or_raise(
             f"git rev-list --first-parent {latest_release_tag.tag}..origin/{pr.head.ref}",
-            check=True,
         ).split("\n")
         commit_num = len(commits)
         print(
diff --git a/tests/ci/ci_buddy.py b/tests/ci/ci_buddy.py
index dfb5885270a..138909c1db0 100644
--- a/tests/ci/ci_buddy.py
+++ b/tests/ci/ci_buddy.py
@@ -120,8 +120,10 @@ class CIBuddy:
     ) -> None:
         instance_id, instance_type = "unknown", "unknown"
         if with_instance_info:
-            instance_id = Shell.run("ec2metadata --instance-id") or instance_id
-            instance_type = Shell.run("ec2metadata --instance-type") or instance_type
+            instance_id = Shell.get_output("ec2metadata --instance-id") or instance_id
+            instance_type = (
+                Shell.get_output("ec2metadata --instance-type") or instance_type
+            )
         if not job_name:
             job_name = os.getenv("CHECK_NAME", "unknown")
         sign = ":red_circle:" if not critical else ":black_circle:"
diff --git a/tests/ci/ci_definitions.py b/tests/ci/ci_definitions.py
index 054b554b8fa..51de8c63509 100644
--- a/tests/ci/ci_definitions.py
+++ b/tests/ci/ci_definitions.py
@@ -554,7 +554,7 @@ class CommonJobConfigs:
         run_command="sqllogic_test.py",
         timeout=10800,
         release_only=True,
-        runner_type=Runners.STYLE_CHECKER,
+        runner_type=Runners.FUNC_TESTER,
     )
     SQL_TEST = JobConfig(
         job_name_keyword="sqltest",
@@ -582,6 +582,7 @@ class CommonJobConfigs:
         digest=DigestConfig(
             include_paths=[
                 "tests/ci/docker_server.py",
+                "tests/ci/docker_images_helper.py",
                 "./docker/server",
             ]
         ),
diff --git a/tests/ci/ci_utils.py b/tests/ci/ci_utils.py
index 3182c0bc5d8..cd21554788c 100644
--- a/tests/ci/ci_utils.py
+++ b/tests/ci/ci_utils.py
@@ -2,6 +2,7 @@ import json
 import os
 import re
 import subprocess
+import sys
 import time
 from contextlib import contextmanager
 from pathlib import Path
@@ -192,7 +193,7 @@ class GHActions:
         get_url_cmd = (
             f"gh pr list --repo {repo} --head {branch} --json url --jq '.[0].url'"
         )
-        url = Shell.run(get_url_cmd)
+        url = Shell.get_output(get_url_cmd)
         if not url:
             print(f"ERROR: PR nor found, branch [{branch}]")
         return url
@@ -200,59 +201,56 @@ class GHActions:
 
 class Shell:
     @classmethod
-    def run_strict(cls, command):
+    def get_output_or_raise(cls, command):
+        return cls.get_output(command, strict=True)
+
+    @classmethod
+    def get_output(cls, command, strict=False):
         res = subprocess.run(
             command,
             shell=True,
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
             text=True,
-            check=True,
+            check=strict,
         )
         return res.stdout.strip()
 
     @classmethod
-    def run(cls, command, check=False, dry_run=False, **kwargs):
+    def check(
+        cls,
+        command,
+        strict=False,
+        verbose=False,
+        dry_run=False,
+        stdin_str=None,
+        **kwargs,
+    ):
         if dry_run:
             print(f"Dry-ryn. Would run command [{command}]")
-            return ""
-        print(f"Run command [{command}]")
-        res = ""
-        result = subprocess.run(
-            command,
-            shell=True,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
-            text=True,
-            check=False,
-            **kwargs,
-        )
-        if result.returncode == 0:
-            print(f"stdout: {result.stdout.strip()}")
-            res = result.stdout
-        else:
-            print(
-                f"ERROR: stdout: {result.stdout.strip()}, stderr: {result.stderr.strip()}"
-            )
-            if check:
-                assert result.returncode == 0
-        return res.strip()
-
-    @classmethod
-    def run_as_daemon(cls, command):
-        print(f"Run daemon command [{command}]")
-        subprocess.Popen(command.split(" "))  # pylint:disable=consider-using-with
-        return 0, ""
-
-    @classmethod
-    def check(cls, command):
+            return 0
+        if verbose:
+            print(f"Run command [{command}]")
         proc = subprocess.Popen(
             command,
             shell=True,
-            stdout=subprocess.STDOUT,
             stderr=subprocess.STDOUT,
+            stdout=subprocess.PIPE,
+            stdin=subprocess.PIPE if stdin_str else None,
+            universal_newlines=True,
+            start_new_session=True,
+            bufsize=1,
+            errors="backslashreplace",
+            **kwargs,
         )
+        if stdin_str:
+            proc.communicate(input=stdin_str)
+        elif proc.stdout:
+            for line in proc.stdout:
+                sys.stdout.write(line)
         proc.wait()
+        if strict:
+            assert proc.returncode == 0
         return proc.returncode == 0
 
 
@@ -277,7 +275,7 @@ class Utils:
 
     @staticmethod
     def clear_dmesg():
-        Shell.run("sudo dmesg --clear ||:")
+        Shell.check("sudo dmesg --clear", verbose=True)
 
     @staticmethod
     def check_pr_description(pr_body: str, repo_name: str) -> Tuple[str, str]:
diff --git a/tests/ci/create_release.py b/tests/ci/create_release.py
index b02a0bb8ed5..0d505d6ccc7 100755
--- a/tests/ci/create_release.py
+++ b/tests/ci/create_release.py
@@ -137,12 +137,13 @@ class ReleaseInfo:
         assert release_type in ("patch", "new")
         if release_type == "new":
             # check commit_ref is right and on a right branch
-            Shell.run(
+            Shell.check(
                 f"git merge-base --is-ancestor {commit_ref} origin/master",
-                check=True,
+                strict=True,
+                verbose=True,
             )
             with checkout(commit_ref):
-                commit_sha = Shell.run(f"git rev-parse {commit_ref}", check=True)
+                commit_sha = Shell.get_output_or_raise(f"git rev-parse {commit_ref}")
                 # Git() must be inside "with checkout" contextmanager
                 git = Git()
                 version = get_version_from_repo(git=git)
@@ -154,13 +155,13 @@ class ReleaseInfo:
                 ), f"BUG: latest tag [{git.latest_tag}], expected [{expected_prev_tag}]"
                 release_tag = version.describe
                 previous_release_tag = expected_prev_tag
-                previous_release_sha = Shell.run_strict(
+                previous_release_sha = Shell.get_output_or_raise(
                     f"git rev-parse {previous_release_tag}"
                 )
                 assert previous_release_sha
         if release_type == "patch":
             with checkout(commit_ref):
-                commit_sha = Shell.run(f"git rev-parse {commit_ref}", check=True)
+                commit_sha = Shell.get_output_or_raise(f"git rev-parse {commit_ref}")
                 # Git() must be inside "with checkout" contextmanager
                 git = Git()
                 version = get_version_from_repo(git=git)
@@ -168,11 +169,16 @@ class ReleaseInfo:
                 version.with_description(codename)
                 release_branch = f"{version.major}.{version.minor}"
                 release_tag = version.describe
-            Shell.run(f"{GIT_PREFIX} fetch origin {release_branch} --tags", check=True)
+            Shell.check(
+                f"{GIT_PREFIX} fetch origin {release_branch} --tags",
+                strict=True,
+                verbose=True,
+            )
             # check commit is right and on a right branch
-            Shell.run(
+            Shell.check(
                 f"git merge-base --is-ancestor {commit_ref} origin/{release_branch}",
-                check=True,
+                strict=True,
+                verbose=True,
             )
             if version.patch == 1:
                 expected_version = copy(version)
@@ -197,7 +203,7 @@ class ReleaseInfo:
                     False
                 ), f"BUG: Unexpected latest tag [{git.latest_tag}] expected [{expected_tag_prefix}*{expected_tag_suffix}]"
 
-            previous_release_sha = Shell.run_strict(
+            previous_release_sha = Shell.get_output_or_raise(
                 f"git rev-parse {previous_release_tag}"
             )
             assert previous_release_sha
@@ -226,25 +232,26 @@ class ReleaseInfo:
     def push_release_tag(self, dry_run: bool) -> None:
         if dry_run:
             # remove locally created tag from prev run
-            Shell.run(
-                f"{GIT_PREFIX} tag -l | grep -q {self.release_tag} && git tag -d {self.release_tag} ||:"
+            Shell.check(
+                f"{GIT_PREFIX} tag -l | grep -q {self.release_tag} && git tag -d {self.release_tag}"
             )
         # Create release tag
         print(
             f"Create and push release tag [{self.release_tag}], commit [{self.commit_sha}]"
         )
         tag_message = f"Release {self.release_tag}"
-        Shell.run(
+        Shell.check(
             f"{GIT_PREFIX} tag -a -m '{tag_message}' {self.release_tag} {self.commit_sha}",
-            check=True,
+            strict=True,
+            verbose=True,
         )
         cmd_push_tag = f"{GIT_PREFIX} push origin {self.release_tag}:{self.release_tag}"
-        Shell.run(cmd_push_tag, dry_run=dry_run, check=True)
+        Shell.check(cmd_push_tag, dry_run=dry_run, strict=True, verbose=True)
 
     @staticmethod
     def _create_gh_label(label: str, color_hex: str, dry_run: bool) -> None:
         cmd = f"gh api repos/{GITHUB_REPOSITORY}/labels -f name={label} -f color={color_hex}"
-        Shell.run(cmd, dry_run=dry_run, check=True)
+        Shell.check(cmd, dry_run=dry_run, strict=True)
 
     def push_new_release_branch(self, dry_run: bool) -> None:
         assert (
@@ -261,7 +268,7 @@ class ReleaseInfo:
         ), f"Unexpected current version in git, must precede [{self.version}] by one step, actual [{version.string}]"
         if dry_run:
             # remove locally created branch from prev run
-            Shell.run(
+            Shell.check(
                 f"{GIT_PREFIX} branch -l | grep -q {new_release_branch} && git branch -d {new_release_branch}"
             )
         print(
@@ -275,7 +282,7 @@ class ReleaseInfo:
                 cmd_push_branch = (
                     f"{GIT_PREFIX} push --set-upstream origin {new_release_branch}"
                 )
-                Shell.run(cmd_push_branch, dry_run=dry_run, check=True)
+                Shell.check(cmd_push_branch, dry_run=dry_run, strict=True, verbose=True)
 
         print("Create and push backport tags for new release branch")
         ReleaseInfo._create_gh_label(
@@ -284,13 +291,14 @@ class ReleaseInfo:
         ReleaseInfo._create_gh_label(
             f"v{new_release_branch}-affected", "c2bfff", dry_run=dry_run
         )
-        Shell.run(
+        Shell.check(
             f"""gh pr create --repo {GITHUB_REPOSITORY} --title 'Release pull request for branch {new_release_branch}'
             --head {new_release_branch} {pr_labels}
             --body 'This PullRequest is a part of ClickHouse release cycle. It is used by CI system only. Do not perform any changes with it.'
             """,
             dry_run=dry_run,
-            check=True,
+            strict=True,
+            verbose=True,
         )
 
     def update_version_and_contributors_list(self, dry_run: bool) -> None:
@@ -316,13 +324,19 @@ class ReleaseInfo:
                 body_file = get_abs_path(".github/PULL_REQUEST_TEMPLATE.md")
                 actor = os.getenv("GITHUB_ACTOR", "") or "me"
                 cmd_create_pr = f"gh pr create --repo {GITHUB_REPOSITORY} --title 'Update version after release' --head {branch_upd_version_contributors} --base {self.release_branch} --body-file {body_file} --label 'do not test' --assignee {actor}"
-                Shell.run(cmd_commit_version_upd, check=True, dry_run=dry_run)
-                Shell.run(cmd_push_branch, check=True, dry_run=dry_run)
-                Shell.run(cmd_create_pr, check=True, dry_run=dry_run)
+                Shell.check(
+                    cmd_commit_version_upd, strict=True, dry_run=dry_run, verbose=True
+                )
+                Shell.check(cmd_push_branch, strict=True, dry_run=dry_run, verbose=True)
+                Shell.check(cmd_create_pr, strict=True, dry_run=dry_run, verbose=True)
                 if dry_run:
-                    Shell.run(f"{GIT_PREFIX} diff '{CMAKE_PATH}' '{CONTRIBUTORS_PATH}'")
-                    Shell.run(
-                        f"{GIT_PREFIX} checkout '{CMAKE_PATH}' '{CONTRIBUTORS_PATH}'"
+                    Shell.check(
+                        f"{GIT_PREFIX} diff '{CMAKE_PATH}' '{CONTRIBUTORS_PATH}'",
+                        verbose=True,
+                    )
+                    Shell.check(
+                        f"{GIT_PREFIX} checkout '{CMAKE_PATH}' '{CONTRIBUTORS_PATH}'",
+                        verbose=True,
                     )
                     self.version_bump_pr = "dry-run"
                 else:
@@ -358,7 +372,7 @@ class ReleaseInfo:
             cmds.append(f"gh release upload {self.release_tag} {file}")
         if not dry_run:
             for cmd in cmds:
-                Shell.run(cmd, check=True)
+                Shell.check(cmd, strict=True, verbose=True)
             self.release_url = f"https://github.com/{GITHUB_REPOSITORY}/releases/tag/{self.release_tag}"
         else:
             print("Dry-run, would run commands:")
@@ -424,7 +438,7 @@ class PackageDownloader:
         self.macos_package_files = ["clickhouse-macos", "clickhouse-macos-aarch64"]
         self.file_to_type = {}
 
-        Shell.run(f"mkdir -p {self.LOCAL_DIR}")
+        Shell.check(f"mkdir -p {self.LOCAL_DIR}")
 
         for package_type in self.PACKAGE_TYPES:
             for package in self.package_names:
@@ -474,7 +488,7 @@ class PackageDownloader:
         return res
 
     def run(self):
-        Shell.run(f"rm -rf {self.LOCAL_DIR}/*")
+        Shell.check(f"rm -rf {self.LOCAL_DIR}/*")
         for package_file in (
             self.deb_package_files + self.rpm_package_files + self.tgz_package_files
         ):
@@ -549,33 +563,33 @@ class PackageDownloader:
 
 @contextmanager
 def checkout(ref: str) -> Iterator[None]:
-    orig_ref = Shell.run(f"{GIT_PREFIX} symbolic-ref --short HEAD", check=True)
+    orig_ref = Shell.get_output_or_raise(f"{GIT_PREFIX} symbolic-ref --short HEAD")
     rollback_cmd = f"{GIT_PREFIX} checkout {orig_ref}"
     assert orig_ref
     if ref not in (orig_ref,):
-        Shell.run(f"{GIT_PREFIX} checkout {ref}")
+        Shell.check(f"{GIT_PREFIX} checkout {ref}", strict=True, verbose=True)
     try:
         yield
     except (Exception, KeyboardInterrupt) as e:
         print(f"ERROR: Exception [{e}]")
-        Shell.run(rollback_cmd)
+        Shell.check(rollback_cmd, verbose=True)
         raise
-    Shell.run(rollback_cmd)
+    Shell.check(rollback_cmd, verbose=True)
 
 
 @contextmanager
 def checkout_new(ref: str) -> Iterator[None]:
-    orig_ref = Shell.run(f"{GIT_PREFIX} symbolic-ref --short HEAD", check=True)
+    orig_ref = Shell.get_output_or_raise(f"{GIT_PREFIX} symbolic-ref --short HEAD")
     rollback_cmd = f"{GIT_PREFIX} checkout {orig_ref}"
     assert orig_ref
-    Shell.run(f"{GIT_PREFIX} checkout -b {ref}", check=True)
+    Shell.check(f"{GIT_PREFIX} checkout -b {ref}", strict=True, verbose=True)
     try:
         yield
     except (Exception, KeyboardInterrupt) as e:
         print(f"ERROR: Exception [{e}]")
-        Shell.run(rollback_cmd)
+        Shell.check(rollback_cmd, verbose=True)
         raise
-    Shell.run(rollback_cmd)
+    Shell.check(rollback_cmd, verbose=True)
 
 
 def parse_args() -> argparse.Namespace:
diff --git a/tests/ci/docker_images_helper.py b/tests/ci/docker_images_helper.py
index e6869852c4e..f0323145cfa 100644
--- a/tests/ci/docker_images_helper.py
+++ b/tests/ci/docker_images_helper.py
@@ -19,11 +19,11 @@ def docker_login(relogin: bool = True) -> None:
     if relogin or not Shell.check(
         "docker system info | grep --quiet -E 'Username|Registry'"
     ):
-        Shell.run(  # pylint: disable=unexpected-keyword-arg
+        Shell.check(  # pylint: disable=unexpected-keyword-arg
             "docker login --username 'robotclickhouse' --password-stdin",
-            input=get_parameter_from_ssm("dockerhub_robot_password"),
+            strict=True,
+            stdin_str=get_parameter_from_ssm("dockerhub_robot_password"),
             encoding="utf-8",
-            check=True,
         )
 
 
@@ -42,7 +42,7 @@ class DockerImage:
 def pull_image(image: DockerImage) -> DockerImage:
     try:
         logging.info("Pulling image %s - start", image)
-        Shell.run(f"docker pull {image}", check=True)
+        Shell.check(f"docker pull {image}", strict=True)
         logging.info("Pulling image %s - done", image)
     except Exception as ex:
         logging.info("Got exception pulling docker %s", ex)

From a6d0b7afbb8299eb8cf056368e93267ef51359ba Mon Sep 17 00:00:00 2001
From: Max K <maxkaynov@gmail.com>
Date: Thu, 1 Aug 2024 18:05:19 +0200
Subject: [PATCH 1195/1488] recovery option

---
 .github/workflows/create_release.yml | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml
index c3126abe461..424dfe60be4 100644
--- a/.github/workflows/create_release.yml
+++ b/.github/workflows/create_release.yml
@@ -16,6 +16,11 @@ concurrency:
         options:
           - patch
           - new
+      only-repo:
+        description: 'Run only repos updates including docker (recovery)'
+        required: false
+        default: false
+        type: boolean
       dry-run:
         description: 'Dry run'
         required: false
@@ -54,11 +59,12 @@ jobs:
         run: |
           python3 ./tests/ci/create_release.py --download-packages ${{ inputs.dry-run == true && '--dry-run' || '' }}
       - name: Push Git Tag for the Release
+        if: ${{ ! inputs.only-repo }}
         shell: bash
         run: |
           python3 ./tests/ci/create_release.py --push-release-tag ${{ inputs.dry-run == true && '--dry-run' || '' }}
       - name: Push New Release Branch
-        if: ${{ inputs.type == 'new' }}
+        if: ${{ inputs.type == 'new' && ! inputs.only-repo }}
         shell: bash
         run: |
           python3 ./tests/ci/create_release.py --push-new-release-branch ${{ inputs.dry-run == true && '--dry-run' || '' }}
@@ -67,7 +73,7 @@ jobs:
 #        run: |
 #          python3 ./tests/ci/create_release.py --create-bump-version-pr ${{ inputs.dry-run == true && '--dry-run' || '' }}
       - name: Bump Docker versions, Changelog, Security
-        if: ${{ inputs.type == 'patch' }}
+        if: ${{ inputs.type == 'patch' && ! inputs.only-repo }}
         shell: bash
         run: |
           python3 ./tests/ci/create_release.py --set-progress-started --progress "update changelog, docker version, security"
@@ -87,7 +93,7 @@ jobs:
           python3 ./utils/security-generator/generate_security.py > SECURITY.md
           git diff HEAD
       - name: Create ChangeLog PR
-        if: ${{ inputs.type == 'patch' && ! inputs.dry-run }}
+        if: ${{ inputs.type == 'patch' && ! inputs.dry-run && ! inputs.only-repo }}
         uses: peter-evans/create-pull-request@v6
         with:
           author: "robot-clickhouse <robot-clickhouse@users.noreply.github.com>"
@@ -105,7 +111,7 @@ jobs:
             ### Changelog category (leave one):
             - Not for changelog (changelog entry is not required)
       - name: Complete previous steps and Restore git state
-        if: ${{ inputs.type == 'patch' }}
+        if: ${{ inputs.type == 'patch' && ! inputs.only-repo }}
         shell: bash
         run: |
           python3 ./tests/ci/create_release.py --set-progress-completed

From dab5eb9c24cc2f43a0ad8ee65ecac613896cff10 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Thu, 1 Aug 2024 16:16:34 +0000
Subject: [PATCH 1196/1488] Fix INTERPOLATE by constant. Fix other tests.

---
 src/Analyzer/InterpolateNode.cpp              |  2 +-
 src/Analyzer/InterpolateNode.h                |  2 ++
 src/Analyzer/Resolve/QueryAnalyzer.cpp        | 13 ++++-------
 src/Planner/CollectTableExpressionData.cpp    |  2 +-
 src/Planner/Planner.cpp                       | 22 +++++++++++++++++++
 src/Planner/PlannerExpressionAnalysis.cpp     |  3 +++
 src/Processors/QueryPlan/FillingStep.cpp      | 14 +++++++++++-
 ..._no_aggregates_and_constant_keys.reference |  4 ++--
 ...15_analyzer_materialized_constants_bug.sql |  2 +-
 9 files changed, 49 insertions(+), 15 deletions(-)

diff --git a/src/Analyzer/InterpolateNode.cpp b/src/Analyzer/InterpolateNode.cpp
index 97dc79f565b..17c734cf386 100644
--- a/src/Analyzer/InterpolateNode.cpp
+++ b/src/Analyzer/InterpolateNode.cpp
@@ -24,7 +24,7 @@ void InterpolateNode::dumpTreeImpl(WriteBuffer & buffer, FormatState & format_st
 {
     buffer << std::string(indent, ' ') << "INTERPOLATE id: " << format_state.getNodeId(this);
 
-    buffer << '\n' << std::string(indent + 2, ' ') << "EXPRESSION\n";
+    buffer << '\n' << std::string(indent + 2, ' ') << "EXPRESSION " << expression_name << " \n";
     getExpression()->dumpTreeImpl(buffer, format_state, indent + 4);
 
     buffer << '\n' << std::string(indent + 2, ' ') << "INTERPOLATE_EXPRESSION\n";
diff --git a/src/Analyzer/InterpolateNode.h b/src/Analyzer/InterpolateNode.h
index ec493ed8bdd..eb3d64d7170 100644
--- a/src/Analyzer/InterpolateNode.h
+++ b/src/Analyzer/InterpolateNode.h
@@ -50,6 +50,8 @@ public:
         return QueryTreeNodeType::INTERPOLATE;
     }
 
+    const std::string & getExpressionName() const { return expression_name; }
+
     void dumpTreeImpl(WriteBuffer & buffer, FormatState & format_state, size_t indent) const override;
 
 protected:
diff --git a/src/Analyzer/Resolve/QueryAnalyzer.cpp b/src/Analyzer/Resolve/QueryAnalyzer.cpp
index 767d5c11075..e973bd8fb34 100644
--- a/src/Analyzer/Resolve/QueryAnalyzer.cpp
+++ b/src/Analyzer/Resolve/QueryAnalyzer.cpp
@@ -64,6 +64,8 @@
 #include <Analyzer/Resolve/TableExpressionsAliasVisitor.h>
 #include <Analyzer/Resolve/ReplaceColumnsVisitor.h>
 
+#include <Planner/PlannerActionsVisitor.h>
+
 #include <Core/Settings.h>
 
 namespace ProfileEvents
@@ -4122,11 +4124,7 @@ void QueryAnalyzer::resolveInterpolateColumnsNodeList(QueryTreeNodePtr & interpo
     {
         auto & interpolate_node_typed = interpolate_node->as<InterpolateNode &>();
 
-        auto * column_to_interpolate = interpolate_node_typed.getExpression()->as<IdentifierNode>();
-        if (!column_to_interpolate)
-            throw Exception(ErrorCodes::LOGICAL_ERROR, "INTERPOLATE can work only for indentifiers, but {} is found",
-                interpolate_node_typed.getExpression()->formatASTForErrorMessage());
-        auto column_to_interpolate_name = column_to_interpolate->getIdentifier().getFullName();
+        auto column_to_interpolate_name = interpolate_node_typed.getExpressionName();
 
         resolveExpressionNode(interpolate_node_typed.getExpression(), scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/);
 
@@ -4135,14 +4133,11 @@ void QueryAnalyzer::resolveInterpolateColumnsNodeList(QueryTreeNodePtr & interpo
         auto & interpolation_to_resolve = interpolate_node_typed.getInterpolateExpression();
         IdentifierResolveScope interpolate_scope(interpolation_to_resolve, &scope /*parent_scope*/);
 
-        auto fake_column_node = std::make_shared<ColumnNode>(NameAndTypePair(column_to_interpolate_name, interpolate_node_typed.getExpression()->getResultType()), interpolate_node_typed.getExpression());
+        auto fake_column_node = std::make_shared<ColumnNode>(NameAndTypePair(column_to_interpolate_name, interpolate_node_typed.getExpression()->getResultType()), interpolate_node);
         if (is_column_constant)
             interpolate_scope.expression_argument_name_to_node.emplace(column_to_interpolate_name, fake_column_node);
 
         resolveExpressionNode(interpolation_to_resolve, interpolate_scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/);
-
-        if (is_column_constant)
-            interpolation_to_resolve = interpolation_to_resolve->cloneAndReplace(fake_column_node, interpolate_node_typed.getExpression());
     }
 }
 
diff --git a/src/Planner/CollectTableExpressionData.cpp b/src/Planner/CollectTableExpressionData.cpp
index 2fe62aa9be0..c48813a4ed4 100644
--- a/src/Planner/CollectTableExpressionData.cpp
+++ b/src/Planner/CollectTableExpressionData.cpp
@@ -46,7 +46,7 @@ public:
         auto column_source_node = column_node->getColumnSource();
         auto column_source_node_type = column_source_node->getNodeType();
 
-        if (column_source_node_type == QueryTreeNodeType::LAMBDA)
+        if (column_source_node_type == QueryTreeNodeType::LAMBDA || column_source_node_type == QueryTreeNodeType::INTERPOLATE)
             return;
 
         /// JOIN using expression
diff --git a/src/Planner/Planner.cpp b/src/Planner/Planner.cpp
index 968642dc9de..b837d9428a1 100644
--- a/src/Planner/Planner.cpp
+++ b/src/Planner/Planner.cpp
@@ -744,6 +744,8 @@ void addWithFillStepIfNeeded(QueryPlan & query_plan,
         }
         else
         {
+            ActionsDAG rename_dag;
+
             for (auto & interpolate_node : interpolate_list_nodes)
             {
                 auto & interpolate_node_typed = interpolate_node->as<InterpolateNode &>();
@@ -772,8 +774,28 @@ void addWithFillStepIfNeeded(QueryPlan & query_plan,
 
                 const auto * alias_node = &interpolate_actions_dag.addAlias(*interpolate_expression, expression_to_interpolate_name);
                 interpolate_actions_dag.getOutputs().push_back(alias_node);
+
+                /// Here we fix INTERPOLATE by constant expression.
+                /// Example from 02336_sort_optimization_with_fill:
+                ///
+                /// SELECT 5 AS x, 'Hello' AS s ORDER BY x WITH FILL FROM 1 TO 10 INTERPOLATE (s AS s||'A')
+                ///
+                /// For this query, INTERPOLATE_EXPRESSION would be : s AS concat(s, 'A'),
+                /// so that interpolate_actions_dag would have INPUT `s`.
+                ///
+                /// However, INPUT `s` does not exist. Instead, we have a constant with execution name 'Hello'_String.
+                /// To fix this, we prepend a rename : 'Hello'_String -> s
+                if (const auto * constant_node = interpolate_node_typed.getExpression()->as<const ConstantNode>())
+                {
+                    const auto * node = &rename_dag.addInput(alias_node->result_name, alias_node->result_type);
+                    node = &rename_dag.addAlias(*node, interpolate_node_typed.getExpressionName());
+                    rename_dag.getOutputs().push_back(node);
+                }
             }
 
+            if (!rename_dag.getOutputs().empty())
+                interpolate_actions_dag = ActionsDAG::merge(std::move(rename_dag), std::move(interpolate_actions_dag));
+
             interpolate_actions_dag.removeUnusedActions();
         }
 
diff --git a/src/Planner/PlannerExpressionAnalysis.cpp b/src/Planner/PlannerExpressionAnalysis.cpp
index 2b67c96d843..ed3f78193ee 100644
--- a/src/Planner/PlannerExpressionAnalysis.cpp
+++ b/src/Planner/PlannerExpressionAnalysis.cpp
@@ -462,6 +462,9 @@ SortAnalysisResult analyzeSort(const QueryNode & query_node,
         for (auto & interpolate_node : interpolate_list_node.getNodes())
         {
             auto & interpolate_node_typed = interpolate_node->as<InterpolateNode &>();
+            if (interpolate_node_typed.getExpression()->getNodeType() == QueryTreeNodeType::CONSTANT)
+               continue;
+
             interpolate_actions_visitor.visit(interpolate_actions_dag, interpolate_node_typed.getInterpolateExpression());
         }
 
diff --git a/src/Processors/QueryPlan/FillingStep.cpp b/src/Processors/QueryPlan/FillingStep.cpp
index 81622389ada..8687886447a 100644
--- a/src/Processors/QueryPlan/FillingStep.cpp
+++ b/src/Processors/QueryPlan/FillingStep.cpp
@@ -2,6 +2,7 @@
 #include <Processors/Transforms/FillingTransform.h>
 #include <QueryPipeline/QueryPipelineBuilder.h>
 #include <IO/Operators.h>
+#include <Interpreters/ExpressionActions.h>
 #include <Common/JSONBuilder.h>
 
 namespace DB
@@ -58,14 +59,25 @@ void FillingStep::transformPipeline(QueryPipelineBuilder & pipeline, const Build
 
 void FillingStep::describeActions(FormatSettings & settings) const
 {
-    settings.out << String(settings.offset, ' ');
+    String prefix(settings.offset, settings.indent_char);
+    settings.out << prefix;
     dumpSortDescription(sort_description, settings.out);
     settings.out << '\n';
+    if (interpolate_description)
+    {
+        auto expression = std::make_shared<ExpressionActions>(interpolate_description->actions.clone());
+        expression->describeActions(settings.out, prefix);
+    }
 }
 
 void FillingStep::describeActions(JSONBuilder::JSONMap & map) const
 {
     map.add("Sort Description", explainSortDescription(sort_description));
+    if (interpolate_description)
+    {
+        auto expression = std::make_shared<ExpressionActions>(interpolate_description->actions.clone());
+        map.add("Expression", expression->toTree());
+    }
 }
 
 void FillingStep::updateOutputStream()
diff --git a/tests/queries/0_stateless/00257_shard_no_aggregates_and_constant_keys.reference b/tests/queries/0_stateless/00257_shard_no_aggregates_and_constant_keys.reference
index 63b8a9d14fc..fc77ed8a241 100644
--- a/tests/queries/0_stateless/00257_shard_no_aggregates_and_constant_keys.reference
+++ b/tests/queries/0_stateless/00257_shard_no_aggregates_and_constant_keys.reference
@@ -8,13 +8,13 @@
 40
 41
 
-0
+41
 2	42
 
 2	42
 43
 
-0
+43
 11
 
 11
diff --git a/tests/queries/0_stateless/03215_analyzer_materialized_constants_bug.sql b/tests/queries/0_stateless/03215_analyzer_materialized_constants_bug.sql
index f9ec28d09d8..b2fd69d75d0 100644
--- a/tests/queries/0_stateless/03215_analyzer_materialized_constants_bug.sql
+++ b/tests/queries/0_stateless/03215_analyzer_materialized_constants_bug.sql
@@ -20,7 +20,7 @@ WITH (
 SELECT
     coalesce(materialize(toLowCardinality(toNullable(1))), 10, NULL),
     max(v)
-FROM remote('127.0.0.{1,2}', default, test__fuzz_21)
+FROM remote('127.0.0.{1,2}', currentDatabase(), test__fuzz_21)
 GROUP BY
     coalesce(NULL),
     coalesce(1, 10, 10, materialize(NULL));

From 4e9761acf93a58a93186f59d3ad083fd438329dc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Thu, 1 Aug 2024 18:59:17 +0200
Subject: [PATCH 1197/1488] Don't run ASAN unit tests under gdb

---
 docker/test/unit/Dockerfile  |  2 +-
 docker/test/unit/run.sh      | 21 ++++++++++++++++++++-
 tests/ci/unit_tests_check.py |  5 ++++-
 3 files changed, 25 insertions(+), 3 deletions(-)

diff --git a/docker/test/unit/Dockerfile b/docker/test/unit/Dockerfile
index af44dc930b2..5f907d94d39 100644
--- a/docker/test/unit/Dockerfile
+++ b/docker/test/unit/Dockerfile
@@ -4,4 +4,4 @@ ARG FROM_TAG=latest
 FROM clickhouse/test-base:$FROM_TAG
 
 COPY run.sh /
-CMD ["/bin/bash", "/run.sh"]
+ENTRYPOINT ["/run.sh"]
diff --git a/docker/test/unit/run.sh b/docker/test/unit/run.sh
index 7323c384d9c..ba11f568218 100644
--- a/docker/test/unit/run.sh
+++ b/docker/test/unit/run.sh
@@ -2,4 +2,23 @@
 
 set -x
 
-timeout 40m gdb -q  -ex 'set print inferior-events off' -ex 'set confirm off' -ex 'set print thread-events off' -ex run -ex bt -ex quit --args ./unit_tests_dbms --gtest_output='json:test_output/test_result.json' | tee test_output/test_result.txt
+if [ "$#" -ne 1 ]; then
+    echo "Expected exactly one argument"
+    exit 1
+fi
+
+if [ "$1" = "GDB" ];
+then
+  timeout 40m \
+    gdb -q  -ex "set print inferior-events off" -ex "set confirm off" -ex "set print thread-events off" -ex run -ex bt -ex quit --args \
+    ./unit_tests_dbms --gtest_output='json:test_output/test_result.json' \
+    | tee test_output/test_result.txt
+elif [ "$1" = "NO_GDB" ];
+then
+  timeout 40m \
+    ./unit_tests_dbms --gtest_output='json:test_output/test_result.json' \
+    | tee test_output/test_result.txt
+else
+    echo "Unknown argument: $1"
+    exit 1
+fi
diff --git a/tests/ci/unit_tests_check.py b/tests/ci/unit_tests_check.py
index b66a4312657..716625d7077 100644
--- a/tests/ci/unit_tests_check.py
+++ b/tests/ci/unit_tests_check.py
@@ -174,10 +174,13 @@ def main():
     test_output = temp_path / "test_output"
     test_output.mkdir(parents=True, exist_ok=True)
 
+    # Don't run ASAN under gdb since that breaks leak detection
+    gdb_enabled = "NO_GDB" if "asan" in check_name else "GDB"
+
     run_command = (
         f"docker run --cap-add=SYS_PTRACE --volume={tests_binary}:/unit_tests_dbms "
         "--security-opt seccomp=unconfined "  # required to issue io_uring sys-calls
-        f"--volume={test_output}:/test_output {docker_image}"
+        f"--volume={test_output}:/test_output {docker_image} ${gdb_enabled}"
     )
 
     run_log_path = test_output / "run.log"

From d683fb05a009ed3f58c0e11fc329c3783f934369 Mon Sep 17 00:00:00 2001
From: kssenii <sumarokovakseniia@mail.ru>
Date: Thu, 1 Aug 2024 19:05:30 +0200
Subject: [PATCH 1198/1488] Fix

---
 .../IO/CachedOnDiskReadBufferFromFile.cpp     | 17 ++++++---
 src/Interpreters/Cache/FileCache.cpp          | 35 ++++++++++++-------
 src/Interpreters/Cache/FileCache.h            |  4 ++-
 src/Interpreters/Cache/FileSegment.cpp        |  9 ++++-
 src/Interpreters/Cache/FileSegment.h          |  4 ++-
 tests/config/config.d/storage_conf.xml        |  3 +-
 6 files changed, 51 insertions(+), 21 deletions(-)

diff --git a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp
index c928d25c7b8..b471f3fc58f 100644
--- a/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp
+++ b/src/Disks/IO/CachedOnDiskReadBufferFromFile.cpp
@@ -135,8 +135,11 @@ bool CachedOnDiskReadBufferFromFile::nextFileSegmentsBatch()
     else
     {
         CreateFileSegmentSettings create_settings(FileSegmentKind::Regular);
-        file_segments = cache->getOrSet(cache_key, file_offset_of_buffer_end, size, file_size.value(), create_settings, settings.filesystem_cache_segments_batch_size, user);
+        file_segments = cache->getOrSet(
+            cache_key, file_offset_of_buffer_end, size, file_size.value(),
+            create_settings, settings.filesystem_cache_segments_batch_size, user);
     }
+
     return !file_segments->empty();
 }
 
@@ -158,8 +161,8 @@ void CachedOnDiskReadBufferFromFile::initialize()
 
     LOG_TEST(
         log,
-        "Having {} file segments to read: {}, current offset: {}",
-        file_segments->size(), file_segments->toString(), file_offset_of_buffer_end);
+        "Having {} file segments to read: {}, current read range: [{}, {})",
+        file_segments->size(), file_segments->toString(), file_offset_of_buffer_end, read_until_position);
 
     initialized = true;
 }
@@ -1043,6 +1046,10 @@ bool CachedOnDiskReadBufferFromFile::nextImplStep()
         if (file_segments->size() == 1)
         {
             size_t remaining_size_to_read = std::min(current_read_range.right, read_until_position - 1) - file_offset_of_buffer_end + 1;
+
+            LOG_TEST(log, "Remaining size to read: {}, read: {}. Resizing buffer to {}",
+                     remaining_size_to_read, size, nextimpl_working_buffer_offset + std::min(size, remaining_size_to_read));
+
             size = std::min(size, remaining_size_to_read);
             chassert(implementation_buffer->buffer().size() >= nextimpl_working_buffer_offset + size);
             implementation_buffer->buffer().resize(nextimpl_working_buffer_offset + size);
@@ -1055,8 +1062,8 @@ bool CachedOnDiskReadBufferFromFile::nextImplStep()
 
         chassert(
             file_offset_of_buffer_end <= read_until_position,
-            fmt::format("Expected {} <= {} (size: {}, read range: {})",
-                        file_offset_of_buffer_end, read_until_position, size, current_read_range.toString()));
+            fmt::format("Expected {} <= {} (size: {}, read range: {}, hold file segments: {} ({}))",
+                        file_offset_of_buffer_end, read_until_position, size, current_read_range.toString(), file_segments->size(), file_segments->toString(true)));
     }
 
     swap(*implementation_buffer);
diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp
index a88c0de2cfe..0a03f5dcc7d 100644
--- a/src/Interpreters/Cache/FileCache.cpp
+++ b/src/Interpreters/Cache/FileCache.cpp
@@ -316,14 +316,14 @@ FileSegments FileCache::getImpl(const LockedKey & locked_key, const FileSegment:
     return result;
 }
 
-std::vector<FileSegment::Range> FileCache::splitRange(size_t offset, size_t size)
+std::vector<FileSegment::Range> FileCache::splitRange(size_t offset, size_t size, size_t aligned_size)
 {
     assert(size > 0);
     std::vector<FileSegment::Range> ranges;
 
     size_t current_pos = offset;
     size_t end_pos_non_included = offset + size;
-    size_t remaining_size = size;
+    size_t remaining_size = aligned_size;
 
     FileSegments file_segments;
     const size_t max_size = max_file_segment_size.load();
@@ -343,17 +343,20 @@ FileSegments FileCache::splitRangeIntoFileSegments(
     LockedKey & locked_key,
     size_t offset,
     size_t size,
+    size_t aligned_size,
     FileSegment::State state,
     size_t file_segments_limit,
     const CreateFileSegmentSettings & create_settings)
 {
-    assert(size > 0);
+    chassert(size > 0);
+    chassert(size <= aligned_size);
+    /// We take `size` as a soft limit and `aligned_size` as a hard limit.
 
     auto current_pos = offset;
     auto end_pos_non_included = offset + size;
 
     size_t current_file_segment_size;
-    size_t remaining_size = size;
+    size_t remaining_size = aligned_size;
 
     FileSegments file_segments;
     const size_t max_size = max_file_segment_size.load();
@@ -369,6 +372,8 @@ FileSegments FileCache::splitRangeIntoFileSegments(
         current_pos += current_file_segment_size;
     }
 
+    chassert(file_segments.size() == file_segments_limit || file_segments.back()->range().contains(offset + size - 1),
+             fmt::format("Offset: {}, size: {}, file segments: {}", offset, size, toString(file_segments)));
     return file_segments;
 }
 
@@ -376,6 +381,7 @@ void FileCache::fillHolesWithEmptyFileSegments(
     LockedKey & locked_key,
     FileSegments & file_segments,
     const FileSegment::Range & range,
+    size_t non_aligned_right_offset,
     size_t file_segments_limit,
     bool fill_with_detached_file_segments,
     const CreateFileSegmentSettings & create_settings)
@@ -442,7 +448,7 @@ void FileCache::fillHolesWithEmptyFileSegments(
         }
         else
         {
-            auto ranges = splitRange(current_pos, hole_size);
+            auto ranges = splitRange(current_pos, hole_size, hole_size);
             FileSegments hole;
             for (const auto & r : ranges)
             {
@@ -479,7 +485,7 @@ void FileCache::fillHolesWithEmptyFileSegments(
 
     chassert(!file_segments_limit || file_segments.size() < file_segments_limit);
 
-    if (current_pos <= range.right)
+    if (current_pos <= non_aligned_right_offset)
     {
         ///   ________]     -- requested range
         ///   _____]
@@ -487,6 +493,7 @@ void FileCache::fillHolesWithEmptyFileSegments(
         /// segmentN
 
         auto hole_size = range.right - current_pos + 1;
+        auto non_aligned_size = non_aligned_right_offset - current_pos + 1;
 
         if (fill_with_detached_file_segments)
         {
@@ -497,7 +504,7 @@ void FileCache::fillHolesWithEmptyFileSegments(
         }
         else
         {
-            auto ranges = splitRange(current_pos, hole_size);
+            auto ranges = splitRange(current_pos, non_aligned_size, hole_size);
             FileSegments hole;
             for (const auto & r : ranges)
             {
@@ -542,7 +549,7 @@ FileSegmentsHolderPtr FileCache::set(
     else
     {
         file_segments = splitRangeIntoFileSegments(
-            *locked_key, offset, size, FileSegment::State::EMPTY, /* file_segments_limit */0, create_settings);
+            *locked_key, offset, size, size, FileSegment::State::EMPTY, /* file_segments_limit */0, create_settings);
     }
 
     return std::make_unique<FileSegmentsHolder>(std::move(file_segments));
@@ -659,9 +666,13 @@ FileCache::getOrSet(
         }
     }
 
+    chassert(range.left >= aligned_offset);
+
     if (file_segments.empty())
     {
-        file_segments = splitRangeIntoFileSegments(*locked_key, range.left, range.size(), FileSegment::State::EMPTY, file_segments_limit, create_settings);
+        file_segments = splitRangeIntoFileSegments(
+            *locked_key, range.left, /* size */offset + size - range.left, /* aligned_size */range.size(),
+            FileSegment::State::EMPTY, file_segments_limit, create_settings);
     }
     else
     {
@@ -669,9 +680,9 @@ FileCache::getOrSet(
         chassert(file_segments.back()->range().left <= range.right);
 
         fillHolesWithEmptyFileSegments(
-            *locked_key, file_segments, range, file_segments_limit, /* fill_with_detached */false, create_settings);
+            *locked_key, file_segments, range, offset + size - 1, file_segments_limit, /* fill_with_detached */false, create_settings);
 
-        if (!file_segments.front()->range().contains(offset))
+        if (!file_segments.front()->range().contains(range.left))
         {
             throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected {} to include {} "
                             "(end offset: {}, aligned offset: {}, aligned end offset: {})",
@@ -713,7 +724,7 @@ FileSegmentsHolderPtr FileCache::get(
             }
 
             fillHolesWithEmptyFileSegments(
-                *locked_key, file_segments, range, file_segments_limit, /* fill_with_detached */true, CreateFileSegmentSettings{});
+                *locked_key, file_segments, range, offset + size - 1, file_segments_limit, /* fill_with_detached */true, CreateFileSegmentSettings{});
 
             chassert(!file_segments_limit || file_segments.size() <= file_segments_limit);
             return std::make_unique<FileSegmentsHolder>(std::move(file_segments));
diff --git a/src/Interpreters/Cache/FileCache.h b/src/Interpreters/Cache/FileCache.h
index 527fd9d5edf..3f7eec73b56 100644
--- a/src/Interpreters/Cache/FileCache.h
+++ b/src/Interpreters/Cache/FileCache.h
@@ -263,7 +263,7 @@ private:
 
     /// Split range into subranges by max_file_segment_size,
     /// each subrange size must be less or equal to max_file_segment_size.
-    std::vector<FileSegment::Range> splitRange(size_t offset, size_t size);
+    std::vector<FileSegment::Range> splitRange(size_t offset, size_t size, size_t aligned_size);
 
     /// Split range into subranges by max_file_segment_size (same as in splitRange())
     /// and create a new file segment for each subrange.
@@ -273,6 +273,7 @@ private:
         LockedKey & locked_key,
         size_t offset,
         size_t size,
+        size_t aligned_size,
         FileSegment::State state,
         size_t file_segments_limit,
         const CreateFileSegmentSettings & create_settings);
@@ -281,6 +282,7 @@ private:
         LockedKey & locked_key,
         FileSegments & file_segments,
         const FileSegment::Range & range,
+        size_t non_aligned_right_offset,
         size_t file_segments_limit,
         bool fill_with_detached_file_segments,
         const CreateFileSegmentSettings & settings);
diff --git a/src/Interpreters/Cache/FileSegment.cpp b/src/Interpreters/Cache/FileSegment.cpp
index 1664a91b694..c46fb978ae4 100644
--- a/src/Interpreters/Cache/FileSegment.cpp
+++ b/src/Interpreters/Cache/FileSegment.cpp
@@ -1008,7 +1008,12 @@ FileSegment & FileSegmentsHolder::add(FileSegmentPtr && file_segment)
     return *file_segments.back();
 }
 
-String FileSegmentsHolder::toString()
+String FileSegmentsHolder::toString(bool with_state)
+{
+    return DB::toString(file_segments, with_state);
+}
+
+String toString(const FileSegments & file_segments, bool with_state)
 {
     String ranges;
     for (const auto & file_segment : file_segments)
@@ -1018,6 +1023,8 @@ String FileSegmentsHolder::toString()
         ranges += file_segment->range().toString();
         if (file_segment->isUnbound())
             ranges += "(unbound)";
+        if (with_state)
+            ranges += "(" + FileSegment::stateToString(file_segment->state()) + ")";
     }
     return ranges;
 }
diff --git a/src/Interpreters/Cache/FileSegment.h b/src/Interpreters/Cache/FileSegment.h
index d6b37b60dc1..25ffb880b45 100644
--- a/src/Interpreters/Cache/FileSegment.h
+++ b/src/Interpreters/Cache/FileSegment.h
@@ -291,7 +291,7 @@ struct FileSegmentsHolder : private boost::noncopyable
 
     size_t size() const { return file_segments.size(); }
 
-    String toString();
+    String toString(bool with_state = false);
 
     void popFront() { completeAndPopFrontImpl(); }
 
@@ -317,4 +317,6 @@ private:
 
 using FileSegmentsHolderPtr = std::unique_ptr<FileSegmentsHolder>;
 
+String toString(const FileSegments & file_segments, bool with_state = false);
+
 }
diff --git a/tests/config/config.d/storage_conf.xml b/tests/config/config.d/storage_conf.xml
index 7a9b579c00a..4daa64b520d 100644
--- a/tests/config/config.d/storage_conf.xml
+++ b/tests/config/config.d/storage_conf.xml
@@ -19,7 +19,8 @@
                 <type>cache</type>
                 <disk>s3_disk</disk>
                 <path>s3_cache/</path>
-                <max_size>104857600</max_size>
+                <max_size>100Mi</max_size>
+                <max_file_segment_size>5Mi</max_file_segment_size>
                 <cache_on_write_operations>1</cache_on_write_operations>
                 <delayed_cleanup_interval_ms>100</delayed_cleanup_interval_ms>
                 <cache_policy>LRU</cache_policy>

From 1e8d0d4a5e8d83a1d123a4b5b6c5a91b41caac1c Mon Sep 17 00:00:00 2001
From: Julia Kartseva <yulia.kartseva@gmail.com>
Date: Wed, 31 Jul 2024 06:09:14 +0000
Subject: [PATCH 1199/1488] disable parallel run for
 network_receive_time_metric_insert

If run in parallel, several tests may affect the
value of the `NetworkReceiveElapsedMicroseconds` profile event.
This may contribute to test flakiness.
---
 .../0_stateless/01923_network_receive_time_metric_insert.sh    | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/01923_network_receive_time_metric_insert.sh b/tests/queries/0_stateless/01923_network_receive_time_metric_insert.sh
index 97835d97965..77b909ed89e 100755
--- a/tests/queries/0_stateless/01923_network_receive_time_metric_insert.sh
+++ b/tests/queries/0_stateless/01923_network_receive_time_metric_insert.sh
@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
-# Tags: no-fasttest
+# Tags: no-fasttest, no-parallel
 # Tag no-fasttest: needs pv
+# Tag no-parallel: reads from a system table
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh

From c5a8653daf7dc9cb1031c2ac4b2be3623117848d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Thu, 1 Aug 2024 19:27:12 +0200
Subject: [PATCH 1200/1488] Playing with Docker and the CI

---
 docker/test/unit/Dockerfile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/docker/test/unit/Dockerfile b/docker/test/unit/Dockerfile
index 5f907d94d39..9f4b86aa0ca 100644
--- a/docker/test/unit/Dockerfile
+++ b/docker/test/unit/Dockerfile
@@ -4,4 +4,5 @@ ARG FROM_TAG=latest
 FROM clickhouse/test-base:$FROM_TAG
 
 COPY run.sh /
+RUN chmod +x run.sh
 ENTRYPOINT ["/run.sh"]

From 4709222dd1f3a37c5f97e638526c21ade6b5218f Mon Sep 17 00:00:00 2001
From: Julia Kartseva <yulia.kartseva@gmail.com>
Date: Wed, 31 Jul 2024 23:16:43 +0000
Subject: [PATCH 1201/1488] print debug info if the test fails

---
 ...1923_network_receive_time_metric_insert.sh | 22 +++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)

diff --git a/tests/queries/0_stateless/01923_network_receive_time_metric_insert.sh b/tests/queries/0_stateless/01923_network_receive_time_metric_insert.sh
index 77b909ed89e..adf4fd96a00 100755
--- a/tests/queries/0_stateless/01923_network_receive_time_metric_insert.sh
+++ b/tests/queries/0_stateless/01923_network_receive_time_metric_insert.sh
@@ -13,9 +13,23 @@ ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS t; CREATE TABLE t (x UInt64)
 seq 1 1000 | pv --quiet --rate-limit 400 | ${CLICKHOUSE_CLIENT} --query "INSERT INTO t FORMAT TSV"
 
 # We check that the value of NetworkReceiveElapsedMicroseconds correctly includes the time spent waiting data from the client.
-${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS;
-    WITH ProfileEvents['NetworkReceiveElapsedMicroseconds'] AS time
-    SELECT time >= 1000000 ? 1 : time FROM system.query_log
-        WHERE current_database = currentDatabase() AND query_kind = 'Insert' AND event_date >= yesterday() AND type = 2 ORDER BY event_time DESC LIMIT 1;"
+result=$(${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS;
+    WITH ProfileEvents['NetworkReceiveElapsedMicroseconds'] AS elapsed_us
+    SELECT elapsed_us FROM system.query_log
+    WHERE current_database = currentDatabase() AND query_kind = 'Insert' AND event_date >= yesterday() AND type = 'QueryFinish'
+    ORDER BY event_time DESC LIMIT 1;")
+
+elapsed_us=$(echo $result | sed 's/ .*//')
+
+min_elapsed_us=1000000
+if [[ "$elapsed_us" -ge "$min_elapsed_us" ]]; then
+    echo 1
+else
+    # Print debug info
+    ${CLICKHOUSE_CLIENT} --query "
+    WITH ProfileEvents['NetworkReceiveElapsedMicroseconds'] AS elapsed_us
+    SELECT query_start_time_microseconds, event_time_microseconds, query_duration_ms, elapsed_us, query FROM system.query_log
+    WHERE current_database = currentDatabase() and event_date >= yesterday() AND type = 'QueryFinish' ORDER BY query_start_time;"
+fi
 
 ${CLICKHOUSE_CLIENT} --query "DROP TABLE t"

From 35b6112b7bb06f18cd5b07860ec1b9c6ce38014f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Thu, 1 Aug 2024 19:50:13 +0200
Subject: [PATCH 1202/1488] Try to stop on leaks

---
 docker/test/base/Dockerfile   | 2 ++
 docker/test/stateless/run.sh  | 2 +-
 tests/queries/shell_config.sh | 3 ---
 3 files changed, 3 insertions(+), 4 deletions(-)

diff --git a/docker/test/base/Dockerfile b/docker/test/base/Dockerfile
index a81826ed6b5..38c4df459ae 100644
--- a/docker/test/base/Dockerfile
+++ b/docker/test/base/Dockerfile
@@ -28,12 +28,14 @@ RUN echo "TSAN_OPTIONS='verbosity=1000 halt_on_error=1 abort_on_error=1 history_
 RUN echo "UBSAN_OPTIONS='print_stacktrace=1 max_allocation_size_mb=32768'" >> /etc/environment
 RUN echo "MSAN_OPTIONS='abort_on_error=1 poison_in_dtor=1 max_allocation_size_mb=32768'" >> /etc/environment
 RUN echo "LSAN_OPTIONS='suppressions=/usr/share/clickhouse-test/config/lsan_suppressions.txt max_allocation_size_mb=32768'" >> /etc/environment
+RUN echo "ASAN_OPTIONS='halt_on_error=1 abort_on_error=1'" >> /etc/environment
 # Sanitizer options for current shell (not current, but the one that will be spawned on "docker run")
 # (but w/o verbosity for TSAN, otherwise test.reference will not match)
 ENV TSAN_OPTIONS='halt_on_error=1 abort_on_error=1 history_size=7 memory_limit_mb=46080 second_deadlock_stack=1 max_allocation_size_mb=32768'
 ENV UBSAN_OPTIONS='print_stacktrace=1 max_allocation_size_mb=32768'
 ENV MSAN_OPTIONS='abort_on_error=1 poison_in_dtor=1 max_allocation_size_mb=32768'
 ENV LSAN_OPTIONS='max_allocation_size_mb=32768'
+ENV ASAN_OPTIONS='halt_on_error=1 abort_on_error=1'
 
 # for external_symbolizer_path
 RUN ln -s /usr/bin/llvm-symbolizer-${LLVM_VERSION} /usr/bin/llvm-symbolizer
diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh
index f9f96c76d59..c359d8a1847 100755
--- a/docker/test/stateless/run.sh
+++ b/docker/test/stateless/run.sh
@@ -164,7 +164,7 @@ do
 done
 
 setup_logs_replication
-attach_gdb_to_clickhouse || true  # FIXME: to not break old builds, clean on 2023-09-01
+attach_gdb_to_clickhouse
 
 function fn_exists() {
     declare -F "$1" > /dev/null;
diff --git a/tests/queries/shell_config.sh b/tests/queries/shell_config.sh
index f7017958635..9e5bf75d335 100644
--- a/tests/queries/shell_config.sh
+++ b/tests/queries/shell_config.sh
@@ -1,9 +1,6 @@
 #!/usr/bin/env bash
 # shellcheck disable=SC2120
 
-# Don't check for ODR violation, since we may test shared build with ASAN
-export ASAN_OPTIONS=detect_odr_violation=0
-
 # If ClickHouse was built with coverage - dump the coverage information at exit
 # (in other cases this environment variable has no effect)
 export CLICKHOUSE_WRITE_COVERAGE="coverage"

From fc72742e37ae78f47b3e55a969e088f5c372ee36 Mon Sep 17 00:00:00 2001
From: sakulali <sakulali.dev@gmail.com>
Date: Fri, 2 Aug 2024 02:00:20 +0800
Subject: [PATCH 1203/1488] ping CI


From 30e0c1a1b8479e9b6be0701ba21e6050906a7e43 Mon Sep 17 00:00:00 2001
From: Max K <maxkaynov@gmail.com>
Date: Thu, 1 Aug 2024 18:46:09 +0200
Subject: [PATCH 1204/1488] try less mem for geesefs

---
 .github/workflows/create_release.yml | 21 +++++++++++----------
 tests/ci/artifactory.py              |  8 +++++---
 2 files changed, 16 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml
index 424dfe60be4..3c61fa4cfe1 100644
--- a/.github/workflows/create_release.yml
+++ b/.github/workflows/create_release.yml
@@ -17,7 +17,7 @@ concurrency:
           - patch
           - new
       only-repo:
-        description: 'Run only repos updates including docker (recovery)'
+        description: 'Run only repos updates including docker (repo-recovery, tests)'
         required: false
         default: false
         type: boolean
@@ -68,10 +68,11 @@ jobs:
         shell: bash
         run: |
           python3 ./tests/ci/create_release.py --push-new-release-branch ${{ inputs.dry-run == true && '--dry-run' || '' }}
-#      - name: Bump CH Version and Update Contributors' List
-#        shell: bash
-#        run: |
-#          python3 ./tests/ci/create_release.py --create-bump-version-pr ${{ inputs.dry-run == true && '--dry-run' || '' }}
+      - name: Bump CH Version and Update Contributors' List
+        if: ${{ ! inputs.only-repo }}
+        shell: bash
+        run: |
+          python3 ./tests/ci/create_release.py --create-bump-version-pr ${{ inputs.dry-run == true && '--dry-run' || '' }}
       - name: Bump Docker versions, Changelog, Security
         if: ${{ inputs.type == 'patch' && ! inputs.only-repo }}
         shell: bash
@@ -117,11 +118,11 @@ jobs:
           python3 ./tests/ci/create_release.py --set-progress-completed
           git reset --hard HEAD
           git checkout "$GITHUB_REF_NAME"
-#      - name: Create GH Release
-#        shell: bash
-#        if: ${{ inputs.type == 'patch' }}
-#        run: |
-#          python3 ./tests/ci/create_release.py --create-gh-release ${{ inputs.dry-run == true && '--dry-run' || '' }}
+      - name: Create GH Release
+        if: ${{ inputs.type == 'patch' && ! inputs.only-repo }}
+        shell: bash
+        run: |
+          python3 ./tests/ci/create_release.py --create-gh-release ${{ inputs.dry-run == true && '--dry-run' || '' }}
       - name: Export TGZ Packages
         if: ${{ inputs.type == 'patch' }}
         shell: bash
diff --git a/tests/ci/artifactory.py b/tests/ci/artifactory.py
index 71deaccf917..8bba7bca30e 100644
--- a/tests/ci/artifactory.py
+++ b/tests/ci/artifactory.py
@@ -67,8 +67,8 @@ class R2MountPoint:
                     f" --shared-config=/home/ubuntu/.r2_auth_test "
                 )
             if self.DEBUG:
-                self.aux_mount_options += " --debug_s3 --debug_fuse "
-            self.mount_cmd = f"geesefs --endpoint={self.API_ENDPOINT} --cheap --memory-limit=2050 --gc-interval=100 --max-flushers=5 --max-parallel-parts=1 --max-parallel-copy=2 --log-file={self.LOG_FILE} {self.aux_mount_options} {self.bucket_name} {self.MOUNT_POINT}"
+                self.aux_mount_options += " --debug_s3 "
+            self.mount_cmd = f"geesefs --endpoint={self.API_ENDPOINT} --cheap --memory-limit=1000 --gc-interval=100 --max-flushers=10 --max-parallel-parts=1 --max-parallel-copy=10 --log-file={self.LOG_FILE} {self.aux_mount_options} {self.bucket_name} {self.MOUNT_POINT}"
         else:
             assert False
 
@@ -207,8 +207,10 @@ class RpmArtifactory:
         for package in paths:
             _copy_if_not_exists(Path(package), dest_dir)
 
+        # switching between different fuse providers invalidates --update option (apparently some fuse(s) can mess around with mtime)
+        #   add --skip-stat to skip mtime check
         commands = (
-            f"createrepo_c --local-sqlite --workers=2 --update --verbose {dest_dir}",
+            f"createrepo_c --local-sqlite --workers=2 --update --skip-stat --verbose {dest_dir}",
             f"gpg --sign-with {self._SIGN_KEY} --detach-sign --batch --yes --armor {dest_dir / 'repodata' / 'repomd.xml'}",
         )
         print(f"Exporting RPM packages into [{codename}]")

From e034558f74a1cd46bb8fbdfac3b7dc6d25165f4e Mon Sep 17 00:00:00 2001
From: Max K <maxkaynov@gmail.com>
Date: Thu, 1 Aug 2024 20:51:36 +0200
Subject: [PATCH 1205/1488] add automerge prs step

---
 .github/workflows/create_release.yml |  4 +++
 pyproject.toml                       |  1 +
 tests/ci/ci_utils.py                 |  2 +-
 tests/ci/create_release.py           | 47 +++++++++++++++++++++++++++-
 tests/ci/release.py                  |  1 +
 5 files changed, 53 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml
index 3c61fa4cfe1..e27db1b09a4 100644
--- a/.github/workflows/create_release.yml
+++ b/.github/workflows/create_release.yml
@@ -171,6 +171,10 @@ jobs:
           export CHECK_NAME="Docker keeper image"
           python3 docker_server.py --release-type auto --version ${{ env.RELEASE_TAG }} --check-name "$CHECK_NAME" --sha ${{ env.COMMIT_SHA }} ${{ ! inputs.dry-run && '--push'  || '' }}
           python3 ./create_release.py --set-progress-completed
+      - name: Update release info. Merge created PRs
+        shell: bash
+        run: |
+          python3 ./tests/ci/create_release.py --merge-prs ${{ inputs.dry-run == true && '--dry-run' || '' }}
       - name: Set current Release progress to Completed with OK
         shell: bash
         run: |
diff --git a/pyproject.toml b/pyproject.toml
index 9bbeac3ddae..4268901e7f9 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -40,6 +40,7 @@ disable = '''
           global-statement,
           f-string-without-interpolation,
           consider-using-with,
+          use-maxsplit-arg,
           '''
 
 [tool.pylint.SIMILARITIES]
diff --git a/tests/ci/ci_utils.py b/tests/ci/ci_utils.py
index cd21554788c..4f696a2c55a 100644
--- a/tests/ci/ci_utils.py
+++ b/tests/ci/ci_utils.py
@@ -228,7 +228,7 @@ class Shell:
     ):
         if dry_run:
             print(f"Dry-ryn. Would run command [{command}]")
-            return 0
+            return True
         if verbose:
             print(f"Run command [{command}]")
         proc = subprocess.Popen(
diff --git a/tests/ci/create_release.py b/tests/ci/create_release.py
index 0d505d6ccc7..c407a74fbf0 100755
--- a/tests/ci/create_release.py
+++ b/tests/ci/create_release.py
@@ -43,6 +43,7 @@ class ReleaseProgress:
     TEST_TGZ = "test TGZ packages"
     TEST_RPM = "test RPM packages"
     TEST_DEB = "test DEB packages"
+    MERGE_CREATED_PRS = "merge created PRs"
     COMPLETED = "completed"
 
 
@@ -101,6 +102,7 @@ class ReleaseInfo:
     previous_release_sha: str
     changelog_pr: str = ""
     version_bump_pr: str = ""
+    prs_merged: bool = False
     release_url: str = ""
     debian_command: str = ""
     rpm_command: str = ""
@@ -380,6 +382,38 @@ class ReleaseInfo:
             self.release_url = f"dry-run"
         self.dump()
 
+    def merge_prs(self, dry_run: bool) -> None:
+        repo = CI.Envs.GITHUB_REPOSITORY
+        assert self.version_bump_pr
+        if dry_run:
+            version_bump_pr_num = 12345
+        else:
+            version_bump_pr_num = int(self.version_bump_pr.split("/")[-1])
+        print("Merging Version bump PR")
+        res_1 = Shell.check(
+            f"gh pr merge {version_bump_pr_num} --repo {repo} --merge --auto",
+            verbose=True,
+            dry_run=dry_run,
+        )
+
+        res_2 = True
+        if not self.release_tag.endswith("-new"):
+            assert self.changelog_pr
+            print("Merging ChangeLog PR")
+            if dry_run:
+                changelog_pr_num = 23456
+            else:
+                changelog_pr_num = int(self.changelog_pr.split("/")[-1])
+            res_2 = Shell.check(
+                f"gh pr merge {changelog_pr_num} --repo {repo} --merge --auto",
+                verbose=True,
+                dry_run=dry_run,
+            )
+        else:
+            assert not self.changelog_pr
+
+        self.prs_merged = res_1 and res_2
+
 
 class RepoTypes:
     RPM = "rpm"
@@ -627,6 +661,11 @@ def parse_args() -> argparse.Namespace:
         action="store_true",
         help="Create GH Release object and attach all packages",
     )
+    parser.add_argument(
+        "--merge-prs",
+        action="store_true",
+        help="Merge PRs with version, changelog updates",
+    )
     parser.add_argument(
         "--post-status",
         action="store_true",
@@ -732,7 +771,6 @@ if __name__ == "__main__":
 
     if args.post_status:
         release_info = ReleaseInfo.from_file()
-        release_info.update_release_info(dry_run=args.dry_run)
         if release_info.is_new_release_branch():
             title = "New release branch"
         else:
@@ -766,6 +804,13 @@ if __name__ == "__main__":
         ri.progress_description = ReleaseProgressDescription.OK
         ri.dump()
 
+    if args.merge_prs:
+        with ReleaseContextManager(
+            release_progress=ReleaseProgress.MERGE_CREATED_PRS
+        ) as release_info:
+            release_info.update_release_info(dry_run=args.dry_run)
+            release_info.merge_prs(dry_run=args.dry_run)
+
     # tear down ssh
     if _ssh_agent and _key_pub:
         _ssh_agent.remove(_key_pub)
diff --git a/tests/ci/release.py b/tests/ci/release.py
index 2de20d00a00..b26d6205f3b 100755
--- a/tests/ci/release.py
+++ b/tests/ci/release.py
@@ -689,4 +689,5 @@ def main():
 
 
 if __name__ == "__main__":
+    assert False, "Script Deprecated, ask ci team for help"
     main()

From 67b11300e45f6e24c3515a978d23a9bc998a666e Mon Sep 17 00:00:00 2001
From: Michael Kolupaev <michael.kolupaev@clickhouse.com>
Date: Thu, 1 Aug 2024 21:05:49 +0000
Subject: [PATCH 1206/1488] Fix crash in
 KeyCondition::cloneASTWithInversionPushDown() caused by type change

---
 src/Storages/MergeTree/KeyCondition.cpp       | 82 +++++++++----------
 .../03215_key_condition_bug.reference         |  1 +
 .../0_stateless/03215_key_condition_bug.sql   |  3 +
 3 files changed, 44 insertions(+), 42 deletions(-)
 create mode 100644 tests/queries/0_stateless/03215_key_condition_bug.reference
 create mode 100644 tests/queries/0_stateless/03215_key_condition_bug.sql

diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp
index 69bffac9160..eaf9f0af623 100644
--- a/src/Storages/MergeTree/KeyCondition.cpp
+++ b/src/Storages/MergeTree/KeyCondition.cpp
@@ -566,6 +566,7 @@ static const ActionsDAG::Node & cloneASTWithInversionPushDown(
     }
 
     const ActionsDAG::Node * res = nullptr;
+    bool handled_inversion = false;
 
     switch (node.type)
     {
@@ -582,7 +583,7 @@ static const ActionsDAG::Node & cloneASTWithInversionPushDown(
                 /// Re-generate column name for constant.
                 /// DAG form query (with enabled analyzer) uses suffixes for constants, like 1_UInt8.
                 /// DAG from PK does not use it. This breaks matching by column name sometimes.
-                /// Ideally, we should not compare manes, but DAG subtrees instead.
+                /// Ideally, we should not compare names, but DAG subtrees instead.
                 name = ASTLiteral(column_const->getDataColumn()[0]).getColumnName();
             else
                 name = node.result_name;
@@ -593,9 +594,9 @@ static const ActionsDAG::Node & cloneASTWithInversionPushDown(
         case (ActionsDAG::ActionType::ALIAS):
         {
             /// Ignore aliases
-            const auto & alias = cloneASTWithInversionPushDown(*node.children.front(), inverted_dag, to_inverted, context, need_inversion);
-            to_inverted[&node] = &alias;
-            return alias;
+            res = &cloneASTWithInversionPushDown(*node.children.front(), inverted_dag, to_inverted, context, need_inversion);
+            handled_inversion = true;
+            break;
         }
         case (ActionsDAG::ActionType::ARRAY_JOIN):
         {
@@ -608,20 +609,10 @@ static const ActionsDAG::Node & cloneASTWithInversionPushDown(
             auto name = node.function_base->getName();
             if (name == "not")
             {
-                const auto & arg = cloneASTWithInversionPushDown(*node.children.front(), inverted_dag, to_inverted, context, !need_inversion);
-                to_inverted[&node] = &arg;
-                return arg;
+                res = &cloneASTWithInversionPushDown(*node.children.front(), inverted_dag, to_inverted, context, !need_inversion);
+                handled_inversion = true;
             }
-
-            if (name == "materialize")
-            {
-                /// Ignore materialize
-                const auto & arg = cloneASTWithInversionPushDown(*node.children.front(), inverted_dag, to_inverted, context, need_inversion);
-                to_inverted[&node] = &arg;
-                return arg;
-            }
-
-            if (name == "indexHint")
+            else if (name == "indexHint")
             {
                 ActionsDAG::NodeRawConstPtrs children;
                 if (const auto * adaptor = typeid_cast<const FunctionToFunctionBaseAdaptor *>(node.function_base.get()))
@@ -636,12 +627,10 @@ static const ActionsDAG::Node & cloneASTWithInversionPushDown(
                     }
                 }
 
-                const auto & func = inverted_dag.addFunction(node.function_base, children, "");
-                to_inverted[&node] = &func;
-                return func;
+                res = &inverted_dag.addFunction(node.function_base, children, "");
+                handled_inversion = true;
             }
-
-            if (need_inversion && (name == "and" || name == "or"))
+            else if (need_inversion && (name == "and" || name == "or"))
             {
                 ActionsDAG::NodeRawConstPtrs children(node.children);
 
@@ -659,34 +648,43 @@ static const ActionsDAG::Node & cloneASTWithInversionPushDown(
 
                 /// We match columns by name, so it is important to fill name correctly.
                 /// So, use empty string to make it automatically.
-                const auto & func = inverted_dag.addFunction(function_builder, children, "");
-                to_inverted[&node] = &func;
-                return func;
+                res = &inverted_dag.addFunction(function_builder, children, "");
+                handled_inversion = true;
             }
-
-            ActionsDAG::NodeRawConstPtrs children(node.children);
-
-            for (auto & arg : children)
-                arg = &cloneASTWithInversionPushDown(*arg, inverted_dag, to_inverted, context, false);
-
-            auto it = inverse_relations.find(name);
-            if (it != inverse_relations.end())
+            else
             {
-                const auto & func_name = need_inversion ? it->second : it->first;
-                auto function_builder = FunctionFactory::instance().get(func_name, context);
-                const auto & func = inverted_dag.addFunction(function_builder, children, "");
-                to_inverted[&node] = &func;
-                return func;
-            }
+                ActionsDAG::NodeRawConstPtrs children(node.children);
 
-            res = &inverted_dag.addFunction(node.function_base, children, "");
-            chassert(res->result_type == node.result_type);
+                for (auto & arg : children)
+                    arg = &cloneASTWithInversionPushDown(*arg, inverted_dag, to_inverted, context, false);
+
+                auto it = inverse_relations.find(name);
+                if (it != inverse_relations.end())
+                {
+                    const auto & func_name = need_inversion ? it->second : it->first;
+                    auto function_builder = FunctionFactory::instance().get(func_name, context);
+                    res = &inverted_dag.addFunction(function_builder, children, "");
+                    handled_inversion = true;
+                }
+                else
+                {
+                    res = &inverted_dag.addFunction(node.function_base, children, "");
+                    chassert(res->result_type == node.result_type);
+                }
+            }
         }
     }
 
-    if (need_inversion)
+    if (!handled_inversion && need_inversion)
         res = &inverted_dag.addFunction(FunctionFactory::instance().get("not", context), {res}, "");
 
+    /// Make sure we don't change any data types (e.g. remove LowCardinality).
+    /// If it turns out that we actually want to change data types sometimes, it's ok to remove this
+    /// check *and* replace all `addFunction(node.function_base, ...)` calls above with
+    /// `addFunction(FunctionFactory::instance().get(name, context), ...)` to re-resolve overloads.
+    if (!node.result_type->equals(*res->result_type))
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "KeyCondition inadvertently changed subexpression data type: '{}' -> '{}', column `{}`", node.result_type->getName(), res->result_type->getName(), node.result_name);
+
     to_inverted[&node] = res;
     return *res;
 }
diff --git a/tests/queries/0_stateless/03215_key_condition_bug.reference b/tests/queries/0_stateless/03215_key_condition_bug.reference
new file mode 100644
index 00000000000..84ab67a85e0
--- /dev/null
+++ b/tests/queries/0_stateless/03215_key_condition_bug.reference
@@ -0,0 +1 @@
+(0)
diff --git a/tests/queries/0_stateless/03215_key_condition_bug.sql b/tests/queries/0_stateless/03215_key_condition_bug.sql
new file mode 100644
index 00000000000..ef2113e81f8
--- /dev/null
+++ b/tests/queries/0_stateless/03215_key_condition_bug.sql
@@ -0,0 +1,3 @@
+CREATE TABLE t (x Int8) ENGINE MergeTree ORDER BY x;
+INSERT INTO t VALUES (1);
+SELECT arrayJoin([tuple((toNullable(10) * toLowCardinality(20)) < materialize(30))]) AS row FROM t WHERE row.1 = 0;
\ No newline at end of file

From 2e7a15df89a7a3d28445095cb4392b056c3f19a9 Mon Sep 17 00:00:00 2001
From: Nikolay Degterinsky <43110995+evillique@users.noreply.github.com>
Date: Thu, 1 Aug 2024 23:23:06 +0200
Subject: [PATCH 1207/1488] Update setup_export_logs.sh

---
 docker/test/base/setup_export_logs.sh | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/docker/test/base/setup_export_logs.sh b/docker/test/base/setup_export_logs.sh
index db141bcc55e..e544397dd0c 100755
--- a/docker/test/base/setup_export_logs.sh
+++ b/docker/test/base/setup_export_logs.sh
@@ -215,10 +215,9 @@ function setup_logs_replication
 function stop_logs_replication
 {
     echo "Detach all logs replication"
-    timeout --preserve-status --signal TERM --kill-after 10m 20m \
-        clickhouse-client --query "select database||'.'||table from system.tables where database = 'system' and (table like '%_sender' or table like '%_watcher')" | {
+    clickhouse-client --query "select database||'.'||table from system.tables where database = 'system' and (table like '%_sender' or table like '%_watcher')" | {
         tee /dev/stderr
     } | {
-        xargs -n1 -r -i clickhouse-client --query "drop table {}"
+        timeout --preserve-status --signal TERM --kill-after 5m 15m xargs -n1 -r -i clickhouse-client --query "drop table {}"
     }
 }

From eac2c9fc3d8a88c1033e0f23e048421ecf4db850 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 31 Jul 2024 17:38:20 +0200
Subject: [PATCH 1208/1488] Minor change

---
 src/Databases/DatabaseOnDisk.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp
index f419f5811a1..734f354d9a5 100644
--- a/src/Databases/DatabaseOnDisk.cpp
+++ b/src/Databases/DatabaseOnDisk.cpp
@@ -313,7 +313,7 @@ void DatabaseOnDisk::detachTablePermanently(ContextPtr query_context, const Stri
         std::lock_guard lock(mutex);
         if (const auto it = snapshot_detached_tables.find(table_name); it == snapshot_detached_tables.end())
         {
-            throw Exception(ErrorCodes::LOGICAL_ERROR, "Snapshot doesn't contain info about detached table={}", table_name);
+            throw Exception(ErrorCodes::LOGICAL_ERROR, "Snapshot doesn't contain info about detached table `{}`", table_name);
         }
         else
         {

From 69bd306a445a6bc8a55be14bb0080864921f8b69 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 2 Aug 2024 00:48:25 +0200
Subject: [PATCH 1209/1488] Fix race condition in system.processes and Settings

---
 src/Backups/RestoreCoordinationRemote.cpp | 2 +-
 src/Backups/RestoreCoordinationRemote.h   | 2 --
 src/Databases/DatabaseLazy.cpp            | 2 +-
 src/Databases/DatabaseLazy.h              | 2 +-
 src/Interpreters/ProcessList.cpp          | 2 +-
 5 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/src/Backups/RestoreCoordinationRemote.cpp b/src/Backups/RestoreCoordinationRemote.cpp
index 44214d00be5..0a69bc0eafb 100644
--- a/src/Backups/RestoreCoordinationRemote.cpp
+++ b/src/Backups/RestoreCoordinationRemote.cpp
@@ -323,7 +323,7 @@ bool RestoreCoordinationRemote::hasConcurrentRestores(const std::atomic<size_t>
         return false;
 
     bool result = false;
-    std::string path = zookeeper_path +"/stage";
+    std::string path = zookeeper_path + "/stage";
 
     auto holder = with_retries.createRetriesControlHolder("createRootNodes");
     holder.retries_ctl.retryLoop(
diff --git a/src/Backups/RestoreCoordinationRemote.h b/src/Backups/RestoreCoordinationRemote.h
index 9c299865cfa..a3d57e9a4d0 100644
--- a/src/Backups/RestoreCoordinationRemote.h
+++ b/src/Backups/RestoreCoordinationRemote.h
@@ -61,8 +61,6 @@ private:
     void createRootNodes();
     void removeAllNodes();
 
-    class ReplicatedDatabasesMetadataSync;
-
     /// get_zookeeper will provide a zookeeper client without any fault injection
     const zkutil::GetZooKeeper get_zookeeper;
     const String root_zookeeper_path;
diff --git a/src/Databases/DatabaseLazy.cpp b/src/Databases/DatabaseLazy.cpp
index ca30ee6db15..3fb6d30fcb8 100644
--- a/src/Databases/DatabaseLazy.cpp
+++ b/src/Databases/DatabaseLazy.cpp
@@ -44,7 +44,7 @@ namespace ErrorCodes
 
 
 DatabaseLazy::DatabaseLazy(const String & name_, const String & metadata_path_, time_t expiration_time_, ContextPtr context_)
-    : DatabaseOnDisk(name_, metadata_path_, "data/" + escapeForFileName(name_) + "/", "DatabaseLazy (" + name_ + ")", context_)
+    : DatabaseOnDisk(name_, metadata_path_, std::filesystem::path("data") / escapeForFileName(name_) / "", "DatabaseLazy (" + name_ + ")", context_)
     , expiration_time(expiration_time_)
 {
 }
diff --git a/src/Databases/DatabaseLazy.h b/src/Databases/DatabaseLazy.h
index 4347649117d..41cfb751141 100644
--- a/src/Databases/DatabaseLazy.h
+++ b/src/Databases/DatabaseLazy.h
@@ -12,7 +12,7 @@ class DatabaseLazyIterator;
 class Context;
 
 /** Lazy engine of databases.
-  * Works like DatabaseOrdinary, but stores in memory only cache.
+  * Works like DatabaseOrdinary, but stores in memory only the cache.
   * Can be used only with *Log engines.
   */
 class DatabaseLazy final : public DatabaseOnDisk
diff --git a/src/Interpreters/ProcessList.cpp b/src/Interpreters/ProcessList.cpp
index 271e23a7288..6cb50b310ad 100644
--- a/src/Interpreters/ProcessList.cpp
+++ b/src/Interpreters/ProcessList.cpp
@@ -657,7 +657,7 @@ QueryStatusInfo QueryStatus::getInfo(bool get_thread_list, bool get_profile_even
     {
         if (auto ctx = context.lock())
         {
-            res.query_settings = std::make_shared<Settings>(ctx->getSettingsRef());
+            res.query_settings = std::make_shared<Settings>(ctx->getSettingsCopy());
             res.current_database = ctx->getCurrentDatabase();
         }
     }

From 572831f865d66e046d3e507d214e0f5aeae49ad4 Mon Sep 17 00:00:00 2001
From: Julia Kartseva <yulia.kartseva@gmail.com>
Date: Fri, 2 Aug 2024 00:09:26 +0000
Subject: [PATCH 1210/1488] async_insert_race_long flakiness fixes

1. Make the test truly asynchronous.
The setting `--async_insert_max_data_size 1` leads to data
being flushed synchronously for all inserts in this test.
This triggers part creation and extra resource consumption.

2. Do not run the `--wait_for_async_insert` query as a background process
with a fixed (50ms) sleep time.
If the actual execution time is longer than the anticipated delay
time, it may lead to excessive process creation.
---
 tests/queries/0_stateless/02481_async_insert_race_long.sh | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/tests/queries/0_stateless/02481_async_insert_race_long.sh b/tests/queries/0_stateless/02481_async_insert_race_long.sh
index b0088017d32..91e6c4960e0 100755
--- a/tests/queries/0_stateless/02481_async_insert_race_long.sh
+++ b/tests/queries/0_stateless/02481_async_insert_race_long.sh
@@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
-export MY_CLICKHOUSE_CLIENT="$CLICKHOUSE_CLIENT --async_insert_busy_timeout_ms 10 --async_insert_max_data_size 1 --async_insert 1"
+export MY_CLICKHOUSE_CLIENT="$CLICKHOUSE_CLIENT --async_insert_busy_timeout_min_ms 50 --async_insert_busy_timeout_max_ms 50 --async_insert 1"
 
 function insert1()
 {
@@ -29,11 +29,8 @@ function insert3()
 {
     local TIMELIMIT=$((SECONDS+$1))
     while [ $SECONDS -lt "$TIMELIMIT" ]; do
-        ${MY_CLICKHOUSE_CLIENT} --insert_keeper_fault_injection_probability=0 --wait_for_async_insert 1 -q "INSERT INTO async_inserts_race VALUES (7, 'g') (8, 'h')" &
-        sleep 0.05
+        ${MY_CLICKHOUSE_CLIENT} --insert_keeper_fault_injection_probability=0 --wait_for_async_insert 1 -q "INSERT INTO async_inserts_race VALUES (7, 'g') (8, 'h')"
     done
-
-    wait
 }
 
 function select1()

From 0772ed7f6ac92bb2e016e5db00e85deafeecb127 Mon Sep 17 00:00:00 2001
From: shiyer7474 <shiyer@altinity.com>
Date: Fri, 2 Aug 2024 04:02:43 +0000
Subject: [PATCH 1211/1488] Code style feedback incorporated

---
 src/Analyzer/Resolve/QueryAnalyzer.cpp                 | 10 +++++-----
 ...3209_parameterized_view_with_non_literal_params.sql |  2 ++
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/src/Analyzer/Resolve/QueryAnalyzer.cpp b/src/Analyzer/Resolve/QueryAnalyzer.cpp
index 6113a38d463..bffdba2f58a 100644
--- a/src/Analyzer/Resolve/QueryAnalyzer.cpp
+++ b/src/Analyzer/Resolve/QueryAnalyzer.cpp
@@ -4549,11 +4549,11 @@ void QueryAnalyzer::resolveTableFunction(QueryTreeNodePtr & table_function_node,
                         /// Serialize the constant value using datatype specific
                         /// interfaces to match the deserialization in ReplaceQueryParametersVistor.
                         WriteBufferFromOwnString buf;
-                        auto constval = constant->getValue();
-                        auto realtype = constant->getResultType();
-                        auto tempcol  = realtype->createColumn();
-                        tempcol->insert(constval);
-                        realtype->getDefaultSerialization()->serializeTextEscaped(*tempcol, 0, buf, {});
+                        const auto & value = constant->getValue();
+                        auto real_type = constant->getResultType();
+                        auto temporary_column  = real_type->createColumn();
+                        temporary_column->insert(value);
+                        real_type->getDefaultSerialization()->serializeTextEscaped(*temporary_column, 0, buf, {});
                         view_params[identifier_node->getIdentifier().getFullName()] = buf.str();
                     }
                 }
diff --git a/tests/queries/0_stateless/03209_parameterized_view_with_non_literal_params.sql b/tests/queries/0_stateless/03209_parameterized_view_with_non_literal_params.sql
index 55795c7a785..f2c61e5cb1d 100644
--- a/tests/queries/0_stateless/03209_parameterized_view_with_non_literal_params.sql
+++ b/tests/queries/0_stateless/03209_parameterized_view_with_non_literal_params.sql
@@ -90,8 +90,10 @@ select id from ipv4_pv(ipv4param=(select ipaddr from ipv4_table_pv where id=3));
 
 drop view date_pv;
 drop view date_pv2;
+drop view date32_pv;
 drop view uuid_pv;
 drop view ipv4_pv;
 drop table date_table_pv;
+drop table date32_table_pv;
 drop table uuid_table_pv;
 drop table ipv4_table_pv;

From d6da86dad282e6ad176b115d4344944daa8b9756 Mon Sep 17 00:00:00 2001
From: Julia Kartseva <yulia.kartseva@gmail.com>
Date: Wed, 26 Jun 2024 01:27:47 +0000
Subject: [PATCH 1212/1488] Store plain_rewritable metadata in a separate
 layout

---
 .../CommonPathPrefixKeyGenerator.cpp          |   6 +-
 .../MetadataStorageFromPlainObjectStorage.cpp |  36 ++++--
 .../MetadataStorageFromPlainObjectStorage.h   |  16 ++-
 ...torageFromPlainObjectStorageOperations.cpp |  71 +++++++----
 ...aStorageFromPlainObjectStorageOperations.h |  14 ++-
 ...torageFromPlainRewritableObjectStorage.cpp | 119 +++++++++++++++---
 ...aStorageFromPlainRewritableObjectStorage.h |  15 ++-
 .../test_s3_plain_rewritable/test.py          |  13 ++
 8 files changed, 225 insertions(+), 65 deletions(-)

diff --git a/src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.cpp b/src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.cpp
index e321c8a3c5a..2a06d56e5c7 100644
--- a/src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.cpp
+++ b/src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.cpp
@@ -19,11 +19,11 @@ ObjectStorageKey CommonPathPrefixKeyGenerator::generate(const String & path, boo
 {
     const auto & [object_key_prefix, suffix_parts] = getLongestObjectKeyPrefix(path);
 
-    auto key = std::filesystem::path(object_key_prefix.empty() ? storage_key_prefix : object_key_prefix);
+    auto key = std::filesystem::path(object_key_prefix.empty() ? std::string() : object_key_prefix);
 
     /// The longest prefix is the same as path, meaning that the  path is already mapped.
     if (suffix_parts.empty())
-        return ObjectStorageKey::createAsRelative(std::move(key));
+        return ObjectStorageKey::createAsRelative(storage_key_prefix, std::move(key));
 
     /// File and top-level directory paths are mapped as is.
     if (!is_directory || object_key_prefix.empty())
@@ -39,7 +39,7 @@ ObjectStorageKey CommonPathPrefixKeyGenerator::generate(const String & path, boo
         key /= getRandomASCIIString(part_size);
     }
 
-    return ObjectStorageKey::createAsRelative(key);
+    return ObjectStorageKey::createAsRelative(storage_key_prefix, key);
 }
 
 std::tuple<std::string, std::vector<std::string>> CommonPathPrefixKeyGenerator::getLongestObjectKeyPrefix(const std::string & path) const
diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.cpp b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.cpp
index 30111d04d20..3da190c7256 100644
--- a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.cpp
+++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.cpp
@@ -7,6 +7,7 @@
 
 #include <filesystem>
 #include <tuple>
+#include <unordered_set>
 
 namespace DB
 {
@@ -79,14 +80,16 @@ std::vector<std::string> MetadataStorageFromPlainObjectStorage::listDirectory(co
 
     object_storage->listObjects(abs_key, files, 0);
 
-    return getDirectChildrenOnDisk(abs_key, files, path);
+    std::unordered_set<std::string> directories;
+    getDirectChildrenOnDisk(abs_key, object_storage->getCommonKeyPrefix(), files, path, directories);
+    return std::vector<std::string>(std::make_move_iterator(directories.begin()), std::make_move_iterator(directories.end()));
 }
 
 DirectoryIteratorPtr MetadataStorageFromPlainObjectStorage::iterateDirectory(const std::string & path) const
 {
     /// Required for MergeTree
     auto paths = listDirectory(path);
-    // Prepend path, since iterateDirectory() includes path, unlike listDirectory()
+    /// Prepend path, since iterateDirectory() includes path, unlike listDirectory()
     std::for_each(paths.begin(), paths.end(), [&](auto & child) { child = fs::path(path) / child; });
     std::vector<std::filesystem::path> fs_paths(paths.begin(), paths.end());
     return std::make_unique<StaticDirectoryIterator>(std::move(fs_paths));
@@ -99,10 +102,13 @@ StoredObjects MetadataStorageFromPlainObjectStorage::getStorageObjects(const std
     return {StoredObject(object_key.serialize(), path, object_size)};
 }
 
-std::vector<std::string> MetadataStorageFromPlainObjectStorage::getDirectChildrenOnDisk(
-    const std::string & storage_key, const RelativePathsWithMetadata & remote_paths, const std::string & /* local_path */) const
+void MetadataStorageFromPlainObjectStorage::getDirectChildrenOnDisk(
+    const std::string & storage_key,
+    const std::string & /* storage_key_perfix */,
+    const RelativePathsWithMetadata & remote_paths,
+    const std::string & /* local_path */,
+    std::unordered_set<std::string> & result) const
 {
-    std::unordered_set<std::string> duplicates_filter;
     for (const auto & elem : remote_paths)
     {
         const auto & path = elem->relative_path;
@@ -111,11 +117,10 @@ std::vector<std::string> MetadataStorageFromPlainObjectStorage::getDirectChildre
         /// string::npos is ok.
         const auto slash_pos = path.find('/', child_pos);
         if (slash_pos == std::string::npos)
-            duplicates_filter.emplace(path.substr(child_pos));
+            result.emplace(path.substr(child_pos));
         else
-            duplicates_filter.emplace(path.substr(child_pos, slash_pos - child_pos));
+            result.emplace(path.substr(child_pos, slash_pos - child_pos));
     }
-    return std::vector<std::string>(std::make_move_iterator(duplicates_filter.begin()), std::make_move_iterator(duplicates_filter.end()));
 }
 
 const IMetadataStorage & MetadataStorageFromPlainObjectStorageTransaction::getStorageForNonTransactionalReads() const
@@ -140,7 +145,7 @@ void MetadataStorageFromPlainObjectStorageTransaction::removeDirectory(const std
     else
     {
         addOperation(std::make_unique<MetadataStorageFromPlainObjectStorageRemoveDirectoryOperation>(
-            normalizeDirectoryPath(path), *metadata_storage.getPathMap(), object_storage));
+            normalizeDirectoryPath(path), *metadata_storage.getPathMap(), object_storage, metadata_storage.getMetadataKeyPrefix()));
     }
 }
 
@@ -151,8 +156,13 @@ void MetadataStorageFromPlainObjectStorageTransaction::createDirectory(const std
 
     auto normalized_path = normalizeDirectoryPath(path);
     auto key_prefix = object_storage->generateObjectKeyPrefixForDirectoryPath(normalized_path).serialize();
+    chassert(key_prefix.starts_with(object_storage->getCommonKeyPrefix()));
     auto op = std::make_unique<MetadataStorageFromPlainObjectStorageCreateDirectoryOperation>(
-        std::move(normalized_path), std::move(key_prefix), *metadata_storage.getPathMap(), object_storage);
+        std::move(normalized_path),
+        key_prefix.substr(object_storage->getCommonKeyPrefix().size()),
+        *metadata_storage.getPathMap(),
+        object_storage,
+        metadata_storage.getMetadataKeyPrefix());
     addOperation(std::move(op));
 }
 
@@ -167,7 +177,11 @@ void MetadataStorageFromPlainObjectStorageTransaction::moveDirectory(const std::
         throwNotImplemented();
 
     addOperation(std::make_unique<MetadataStorageFromPlainObjectStorageMoveDirectoryOperation>(
-        normalizeDirectoryPath(path_from), normalizeDirectoryPath(path_to), *metadata_storage.getPathMap(), object_storage));
+        normalizeDirectoryPath(path_from),
+        normalizeDirectoryPath(path_to),
+        *metadata_storage.getPathMap(),
+        object_storage,
+        metadata_storage.getMetadataKeyPrefix()));
 }
 
 void MetadataStorageFromPlainObjectStorageTransaction::addBlobToMetadata(
diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.h b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.h
index 66da0f2431e..97c5715a937 100644
--- a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.h
+++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.h
@@ -6,6 +6,8 @@
 #include <Disks/ObjectStorages/MetadataStorageTransactionState.h>
 
 #include <map>
+#include <string>
+#include <unordered_set>
 
 namespace DB
 {
@@ -78,10 +80,20 @@ public:
     bool supportsStat() const override { return false; }
 
 protected:
+    /// Get the object storage prefix for storing metadata files. If stored behind a separate endpoint,
+    /// the metadata keys reflect the layout of the regular files.
+    virtual std::string getMetadataKeyPrefix() const { return object_storage->getCommonKeyPrefix(); }
+
+    /// Returns a map of local paths to paths in object storage.
     virtual std::shared_ptr<PathMap> getPathMap() const { throwNotImplemented(); }
 
-    virtual std::vector<std::string> getDirectChildrenOnDisk(
-        const std::string & storage_key, const RelativePathsWithMetadata & remote_paths, const std::string & local_path) const;
+    /// Retrieves the immediate files and directories within a given directory on a disk.
+    virtual void getDirectChildrenOnDisk(
+        const std::string & storage_key,
+        const std::string & storage_key_perfix,
+        const RelativePathsWithMetadata & remote_paths,
+        const std::string & local_path,
+        std::unordered_set<std::string> & result) const;
 };
 
 class MetadataStorageFromPlainObjectStorageTransaction final : public IMetadataTransaction, private MetadataOperationsHolder
diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp
index 7e4b1f69962..0a6086bd39d 100644
--- a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp
+++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp
@@ -20,14 +20,24 @@ namespace
 
 constexpr auto PREFIX_PATH_FILE_NAME = "prefix.path";
 
+ObjectStorageKey createMetadataObjectKey(const std::string & key_prefix, const std::string & metadata_key_prefix)
+{
+    auto prefix = std::filesystem::path(metadata_key_prefix) / key_prefix;
+    return ObjectStorageKey::createAsRelative(prefix.string(), PREFIX_PATH_FILE_NAME);
+}
 }
 
 MetadataStorageFromPlainObjectStorageCreateDirectoryOperation::MetadataStorageFromPlainObjectStorageCreateDirectoryOperation(
     std::filesystem::path && path_,
     std::string && key_prefix_,
     MetadataStorageFromPlainObjectStorage::PathMap & path_map_,
-    ObjectStoragePtr object_storage_)
-    : path(std::move(path_)), key_prefix(key_prefix_), path_map(path_map_), object_storage(object_storage_)
+    ObjectStoragePtr object_storage_,
+    const std::string & metadata_key_prefix_)
+    : path(std::move(path_))
+    , key_prefix(key_prefix_)
+    , path_map(path_map_)
+    , object_storage(object_storage_)
+    , metadata_key_prefix(metadata_key_prefix_)
 {
 }
 
@@ -36,13 +46,17 @@ void MetadataStorageFromPlainObjectStorageCreateDirectoryOperation::execute(std:
     if (path_map.contains(path))
         return;
 
-    LOG_TRACE(getLogger("MetadataStorageFromPlainObjectStorageCreateDirectoryOperation"), "Creating metadata for directory '{}'", path);
+    auto metadata_object_key = createMetadataObjectKey(key_prefix, metadata_key_prefix);
 
-    auto object_key = ObjectStorageKey::createAsRelative(key_prefix, PREFIX_PATH_FILE_NAME);
+    LOG_TRACE(
+        getLogger("MetadataStorageFromPlainObjectStorageCreateDirectoryOperation"),
+        "Creating metadata for directory '{}' with remote path='{}'",
+        path,
+        metadata_object_key.serialize());
 
-    auto object = StoredObject(object_key.serialize(), path / PREFIX_PATH_FILE_NAME);
+    auto metadata_object = StoredObject(metadata_object_key.serialize(), path / PREFIX_PATH_FILE_NAME);
     auto buf = object_storage->writeObject(
-        object,
+        metadata_object,
         WriteMode::Rewrite,
         /* object_attributes */ std::nullopt,
         /* buf_size */ DBMS_DEFAULT_BUFFER_SIZE,
@@ -66,25 +80,31 @@ void MetadataStorageFromPlainObjectStorageCreateDirectoryOperation::execute(std:
 
 void MetadataStorageFromPlainObjectStorageCreateDirectoryOperation::undo(std::unique_lock<SharedMutex> &)
 {
-    auto object_key = ObjectStorageKey::createAsRelative(key_prefix, PREFIX_PATH_FILE_NAME);
+    auto metadata_object_key = createMetadataObjectKey(key_prefix, metadata_key_prefix);
+
     if (write_finalized)
     {
         path_map.erase(path);
         auto metric = object_storage->getMetadataStorageMetrics().directory_map_size;
         CurrentMetrics::sub(metric, 1);
 
-        object_storage->removeObject(StoredObject(object_key.serialize(), path / PREFIX_PATH_FILE_NAME));
+        object_storage->removeObject(StoredObject(metadata_object_key.serialize(), path / PREFIX_PATH_FILE_NAME));
     }
     else if (write_created)
-        object_storage->removeObjectIfExists(StoredObject(object_key.serialize(), path / PREFIX_PATH_FILE_NAME));
+        object_storage->removeObjectIfExists(StoredObject(metadata_object_key.serialize(), path / PREFIX_PATH_FILE_NAME));
 }
 
 MetadataStorageFromPlainObjectStorageMoveDirectoryOperation::MetadataStorageFromPlainObjectStorageMoveDirectoryOperation(
     std::filesystem::path && path_from_,
     std::filesystem::path && path_to_,
     MetadataStorageFromPlainObjectStorage::PathMap & path_map_,
-    ObjectStoragePtr object_storage_)
-    : path_from(std::move(path_from_)), path_to(std::move(path_to_)), path_map(path_map_), object_storage(object_storage_)
+    ObjectStoragePtr object_storage_,
+    const std::string & metadata_key_prefix_)
+    : path_from(std::move(path_from_))
+    , path_to(std::move(path_to_))
+    , path_map(path_map_)
+    , object_storage(object_storage_)
+    , metadata_key_prefix(metadata_key_prefix_)
 {
 }
 
@@ -98,26 +118,26 @@ std::unique_ptr<WriteBufferFromFileBase> MetadataStorageFromPlainObjectStorageMo
     if (path_map.contains(new_path))
         throw Exception(ErrorCodes::FILE_ALREADY_EXISTS, "Metadata object for the new (destination) path '{}' already exists", new_path);
 
-    auto object_key = ObjectStorageKey::createAsRelative(expected_it->second, PREFIX_PATH_FILE_NAME);
+    auto metadata_object_key = createMetadataObjectKey(expected_it->second, metadata_key_prefix);
 
-    auto object = StoredObject(object_key.serialize(), expected_path / PREFIX_PATH_FILE_NAME);
+    auto metadata_object = StoredObject(metadata_object_key.serialize(), expected_path / PREFIX_PATH_FILE_NAME);
 
     if (validate_content)
     {
         std::string data;
-        auto read_buf = object_storage->readObject(object);
+        auto read_buf = object_storage->readObject(metadata_object);
         readStringUntilEOF(data, *read_buf);
         if (data != path_from)
             throw Exception(
                 ErrorCodes::INCORRECT_DATA,
                 "Incorrect data for object key {}, expected {}, got {}",
-                object_key.serialize(),
+                metadata_object_key.serialize(),
                 expected_path,
                 data);
     }
 
     auto write_buf = object_storage->writeObject(
-        object,
+        metadata_object,
         WriteMode::Rewrite,
         /* object_attributes */ std::nullopt,
         /*buf_size*/ DBMS_DEFAULT_BUFFER_SIZE,
@@ -156,8 +176,11 @@ void MetadataStorageFromPlainObjectStorageMoveDirectoryOperation::undo(std::uniq
 }
 
 MetadataStorageFromPlainObjectStorageRemoveDirectoryOperation::MetadataStorageFromPlainObjectStorageRemoveDirectoryOperation(
-    std::filesystem::path && path_, MetadataStorageFromPlainObjectStorage::PathMap & path_map_, ObjectStoragePtr object_storage_)
-    : path(std::move(path_)), path_map(path_map_), object_storage(object_storage_)
+    std::filesystem::path && path_,
+    MetadataStorageFromPlainObjectStorage::PathMap & path_map_,
+    ObjectStoragePtr object_storage_,
+    const std::string & metadata_key_prefix_)
+    : path(std::move(path_)), path_map(path_map_), object_storage(object_storage_), metadata_key_prefix(metadata_key_prefix_)
 {
 }
 
@@ -170,9 +193,9 @@ void MetadataStorageFromPlainObjectStorageRemoveDirectoryOperation::execute(std:
     LOG_TRACE(getLogger("MetadataStorageFromPlainObjectStorageRemoveDirectoryOperation"), "Removing directory '{}'", path);
 
     key_prefix = path_it->second;
-    auto object_key = ObjectStorageKey::createAsRelative(key_prefix, PREFIX_PATH_FILE_NAME);
-    auto object = StoredObject(object_key.serialize(), path / PREFIX_PATH_FILE_NAME);
-    object_storage->removeObject(object);
+    auto metadata_object_key = createMetadataObjectKey(key_prefix, metadata_key_prefix);
+    auto metadata_object = StoredObject(metadata_object_key.serialize(), path / PREFIX_PATH_FILE_NAME);
+    object_storage->removeObject(metadata_object);
 
     path_map.erase(path_it);
     auto metric = object_storage->getMetadataStorageMetrics().directory_map_size;
@@ -189,10 +212,10 @@ void MetadataStorageFromPlainObjectStorageRemoveDirectoryOperation::undo(std::un
     if (!removed)
         return;
 
-    auto object_key = ObjectStorageKey::createAsRelative(key_prefix, PREFIX_PATH_FILE_NAME);
-    auto object = StoredObject(object_key.serialize(), path / PREFIX_PATH_FILE_NAME);
+    auto metadata_object_key = createMetadataObjectKey(key_prefix, metadata_key_prefix);
+    auto metadata_object = StoredObject(metadata_object_key.serialize(), path / PREFIX_PATH_FILE_NAME);
     auto buf = object_storage->writeObject(
-        object,
+        metadata_object,
         WriteMode::Rewrite,
         /* object_attributes */ std::nullopt,
         /* buf_size */ DBMS_DEFAULT_BUFFER_SIZE,
diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.h b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.h
index 4b196f787fd..e31e3cbb262 100644
--- a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.h
+++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.h
@@ -16,6 +16,7 @@ private:
     std::string key_prefix;
     MetadataStorageFromPlainObjectStorage::PathMap & path_map;
     ObjectStoragePtr object_storage;
+    const std::string metadata_key_prefix;
 
     bool write_created = false;
     bool write_finalized = false;
@@ -26,7 +27,8 @@ public:
         std::filesystem::path && path_,
         std::string && key_prefix_,
         MetadataStorageFromPlainObjectStorage::PathMap & path_map_,
-        ObjectStoragePtr object_storage_);
+        ObjectStoragePtr object_storage_,
+        const std::string & metadata_key_prefix_);
 
     void execute(std::unique_lock<SharedMutex> & metadata_lock) override;
     void undo(std::unique_lock<SharedMutex> & metadata_lock) override;
@@ -39,6 +41,7 @@ private:
     std::filesystem::path path_to;
     MetadataStorageFromPlainObjectStorage::PathMap & path_map;
     ObjectStoragePtr object_storage;
+    const std::string metadata_key_prefix;
 
     bool write_created = false;
     bool write_finalized = false;
@@ -51,7 +54,8 @@ public:
         std::filesystem::path && path_from_,
         std::filesystem::path && path_to_,
         MetadataStorageFromPlainObjectStorage::PathMap & path_map_,
-        ObjectStoragePtr object_storage_);
+        ObjectStoragePtr object_storage_,
+        const std::string & metadata_key_prefix_);
 
     void execute(std::unique_lock<SharedMutex> & metadata_lock) override;
 
@@ -65,13 +69,17 @@ private:
 
     MetadataStorageFromPlainObjectStorage::PathMap & path_map;
     ObjectStoragePtr object_storage;
+    const std::string metadata_key_prefix;
 
     std::string key_prefix;
     bool removed = false;
 
 public:
     MetadataStorageFromPlainObjectStorageRemoveDirectoryOperation(
-        std::filesystem::path && path_, MetadataStorageFromPlainObjectStorage::PathMap & path_map_, ObjectStoragePtr object_storage_);
+        std::filesystem::path && path_,
+        MetadataStorageFromPlainObjectStorage::PathMap & path_map_,
+        ObjectStoragePtr object_storage_,
+        const std::string & metadata_key_prefix_);
 
     void execute(std::unique_lock<SharedMutex> & metadata_lock) override;
     void undo(std::unique_lock<SharedMutex> & metadata_lock) override;
diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp b/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp
index 7718fba9c28..f3d00a928e3 100644
--- a/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp
+++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp
@@ -1,6 +1,7 @@
 #include <Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.h>
 #include <Disks/ObjectStorages/ObjectStorageIterator.h>
 
+#include <unordered_set>
 #include <IO/ReadHelpers.h>
 #include <IO/SharedThreadPools.h>
 #include <IO/S3Common.h>
@@ -21,8 +22,22 @@ namespace
 {
 
 constexpr auto PREFIX_PATH_FILE_NAME = "prefix.path";
+constexpr auto METADATA_PATH_TOKEN = "__meta/";
 
-MetadataStorageFromPlainObjectStorage::PathMap loadPathPrefixMap(const std::string & root, ObjectStoragePtr object_storage)
+/// Use a separate layout for metadata iff:
+/// 1. The disk endpoint does not contain objects, OR
+/// 2. The metadata is already stored behind a separate endpoint.
+/// Otherwise, store metadata along with regular data for backward compatibility.
+std::string getMetadataKeyPrefix(ObjectStoragePtr object_storage)
+{
+    const auto common_key_prefix = std::filesystem::path(object_storage->getCommonKeyPrefix());
+    const auto metadata_key_prefix = std::filesystem::path(common_key_prefix) / METADATA_PATH_TOKEN;
+    return !object_storage->existsOrHasAnyChild(metadata_key_prefix / "") && object_storage->existsOrHasAnyChild(common_key_prefix / "")
+        ? common_key_prefix
+        : metadata_key_prefix;
+}
+
+MetadataStorageFromPlainObjectStorage::PathMap loadPathPrefixMap(const std::string & metadata_key_prefix, ObjectStoragePtr object_storage)
 {
     MetadataStorageFromPlainObjectStorage::PathMap result;
 
@@ -39,16 +54,16 @@ MetadataStorageFromPlainObjectStorage::PathMap loadPathPrefixMap(const std::stri
 
     LOG_DEBUG(log, "Loading metadata");
     size_t num_files = 0;
-    for (auto iterator = object_storage->iterate(root, 0); iterator->isValid(); iterator->next())
+    for (auto iterator = object_storage->iterate(metadata_key_prefix, 0); iterator->isValid(); iterator->next())
     {
         ++num_files;
         auto file = iterator->current();
         String path = file->getPath();
-        auto remote_path = std::filesystem::path(path);
-        if (remote_path.filename() != PREFIX_PATH_FILE_NAME)
+        auto remote_metadata_path = std::filesystem::path(path);
+        if (remote_metadata_path.filename() != PREFIX_PATH_FILE_NAME)
             continue;
 
-        runner([remote_path, path, &object_storage, &result, &mutex, &log, &settings]
+        runner([remote_metadata_path, path, &object_storage, &result, &mutex, &log, &settings, &metadata_key_prefix]
         {
             setThreadName("PlainRWMetaLoad");
 
@@ -75,7 +90,10 @@ MetadataStorageFromPlainObjectStorage::PathMap loadPathPrefixMap(const std::stri
                 throw;
             }
 
-            chassert(remote_path.has_parent_path());
+            chassert(remote_metadata_path.has_parent_path());
+            chassert(remote_metadata_path.string().starts_with(metadata_key_prefix));
+            auto suffix = remote_metadata_path.string().substr(metadata_key_prefix.size());
+            auto remote_path = std::filesystem::path(std::move(suffix));
             std::pair<MetadataStorageFromPlainObjectStorage::PathMap::iterator, bool> res;
             {
                 std::lock_guard lock(mutex);
@@ -103,17 +121,17 @@ MetadataStorageFromPlainObjectStorage::PathMap loadPathPrefixMap(const std::stri
     return result;
 }
 
-std::vector<std::string> getDirectChildrenOnRewritableDisk(
+void getDirectChildrenOnRewritableDisk(
     const std::string & storage_key,
+    const std::string & storage_key_perfix,
     const RelativePathsWithMetadata & remote_paths,
     const std::string & local_path,
     const MetadataStorageFromPlainObjectStorage::PathMap & local_path_prefixes,
-    SharedMutex & shared_mutex)
+    SharedMutex & shared_mutex,
+    std::unordered_set<std::string> & result)
 {
     using PathMap = MetadataStorageFromPlainObjectStorage::PathMap;
 
-    std::unordered_set<std::string> duplicates_filter;
-
     /// Map remote paths into local subdirectories.
     std::unordered_map<PathMap::mapped_type, PathMap::key_type> remote_to_local_subdir;
 
@@ -149,22 +167,21 @@ std::vector<std::string> getDirectChildrenOnRewritableDisk(
             /// File names.
             auto filename = path.substr(child_pos);
             if (!skip_list.contains(filename))
-                duplicates_filter.emplace(std::move(filename));
+                result.emplace(std::move(filename));
         }
         else
         {
             /// Subdirectories.
-            auto it = remote_to_local_subdir.find(path.substr(0, slash_pos));
+            chassert(path.find(storage_key_perfix) == 0);
+            auto it = remote_to_local_subdir.find(path.substr(storage_key_perfix.size(), slash_pos - storage_key_perfix.size()));
             /// Mapped subdirectories.
             if (it != remote_to_local_subdir.end())
-                duplicates_filter.emplace(it->second);
+                result.emplace(it->second);
             /// The remote subdirectory name is the same as the local subdirectory.
             else
-                duplicates_filter.emplace(path.substr(child_pos, slash_pos - child_pos));
+                result.emplace(path.substr(child_pos, slash_pos - child_pos));
         }
     }
-
-    return std::vector<std::string>(std::make_move_iterator(duplicates_filter.begin()), std::make_move_iterator(duplicates_filter.end()));
 }
 
 }
@@ -172,7 +189,8 @@ std::vector<std::string> getDirectChildrenOnRewritableDisk(
 MetadataStorageFromPlainRewritableObjectStorage::MetadataStorageFromPlainRewritableObjectStorage(
     ObjectStoragePtr object_storage_, String storage_path_prefix_)
     : MetadataStorageFromPlainObjectStorage(object_storage_, storage_path_prefix_)
-    , path_map(std::make_shared<PathMap>(loadPathPrefixMap(object_storage->getCommonKeyPrefix(), object_storage)))
+    , metadata_key_prefix(DB::getMetadataKeyPrefix(object_storage))
+    , path_map(std::make_shared<PathMap>(loadPathPrefixMap(metadata_key_prefix, object_storage)))
 {
     if (object_storage->isWriteOnce())
         throw Exception(
@@ -190,10 +208,71 @@ MetadataStorageFromPlainRewritableObjectStorage::~MetadataStorageFromPlainRewrit
     CurrentMetrics::sub(metric, path_map->size());
 }
 
-std::vector<std::string> MetadataStorageFromPlainRewritableObjectStorage::getDirectChildrenOnDisk(
-    const std::string & storage_key, const RelativePathsWithMetadata & remote_paths, const std::string & local_path) const
+bool MetadataStorageFromPlainRewritableObjectStorage::exists(const std::string & path) const
 {
-    return getDirectChildrenOnRewritableDisk(storage_key, remote_paths, local_path, *getPathMap(), metadata_mutex);
+    if (MetadataStorageFromPlainObjectStorage::exists(path))
+        return true;
+
+    if (getMetadataKeyPrefix() != object_storage->getCommonKeyPrefix())
+    {
+        auto key_prefix = object_storage->generateObjectKeyForPath(path).serialize();
+        chassert(key_prefix.starts_with(object_storage->getCommonKeyPrefix()));
+        auto metadata_key = std::filesystem::path(getMetadataKeyPrefix()) / key_prefix.substr(object_storage->getCommonKeyPrefix().size());
+        return object_storage->existsOrHasAnyChild(metadata_key);
+    }
+
+    return false;
+}
+
+bool MetadataStorageFromPlainRewritableObjectStorage::isDirectory(const std::string & path) const
+{
+    if (getMetadataKeyPrefix() != object_storage->getCommonKeyPrefix())
+    {
+        auto directory = std::filesystem::path(object_storage->generateObjectKeyForPath(path).serialize()) / "";
+        chassert(directory.string().starts_with(object_storage->getCommonKeyPrefix()));
+        auto metadata_key
+            = std::filesystem::path(getMetadataKeyPrefix()) / directory.string().substr(object_storage->getCommonKeyPrefix().size());
+        return object_storage->existsOrHasAnyChild(metadata_key);
+    }
+    else
+        return MetadataStorageFromPlainObjectStorage::isDirectory(path);
+}
+
+std::vector<std::string> MetadataStorageFromPlainRewritableObjectStorage::listDirectory(const std::string & path) const
+{
+    auto key_prefix = object_storage->generateObjectKeyForPath(path).serialize();
+
+    RelativePathsWithMetadata files;
+    std::string abs_key = key_prefix;
+    if (!abs_key.ends_with('/'))
+        abs_key += '/';
+
+    object_storage->listObjects(abs_key, files, 0);
+
+    std::unordered_set<std::string> directories;
+    getDirectChildrenOnDisk(abs_key, object_storage->getCommonKeyPrefix(), files, path, directories);
+    /// List empty directories that are identified by the `prefix.path` metadata files. This is required to, e.g., remove
+    /// metadata along with regular files.
+    if (object_storage->getCommonKeyPrefix() != getMetadataKeyPrefix())
+    {
+        chassert(abs_key.starts_with(object_storage->getCommonKeyPrefix()));
+        auto metadata_key = std::filesystem::path(getMetadataKeyPrefix()) / abs_key.substr(object_storage->getCommonKeyPrefix().size());
+        RelativePathsWithMetadata metadata_files;
+        object_storage->listObjects(metadata_key, metadata_files, 0);
+        getDirectChildrenOnDisk(metadata_key, getMetadataKeyPrefix(), metadata_files, path, directories);
+    }
+
+    return std::vector<std::string>(std::make_move_iterator(directories.begin()), std::make_move_iterator(directories.end()));
+}
+
+void MetadataStorageFromPlainRewritableObjectStorage::getDirectChildrenOnDisk(
+    const std::string & storage_key,
+    const std::string & storage_key_perfix,
+    const RelativePathsWithMetadata & remote_paths,
+    const std::string & local_path,
+    std::unordered_set<std::string> & result) const
+{
+    getDirectChildrenOnRewritableDisk(storage_key, storage_key_perfix, remote_paths, local_path, *getPathMap(), metadata_mutex, result);
 }
 
 }
diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.h b/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.h
index a5394b9428d..71153cbdc25 100644
--- a/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.h
+++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.h
@@ -3,6 +3,7 @@
 #include <Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.h>
 
 #include <memory>
+#include <unordered_set>
 
 
 namespace DB
@@ -11,6 +12,7 @@ namespace DB
 class MetadataStorageFromPlainRewritableObjectStorage final : public MetadataStorageFromPlainObjectStorage
 {
 private:
+    const std::string metadata_key_prefix;
     std::shared_ptr<PathMap> path_map;
 
 public:
@@ -18,11 +20,20 @@ public:
     ~MetadataStorageFromPlainRewritableObjectStorage() override;
 
     MetadataStorageType getType() const override { return MetadataStorageType::PlainRewritable; }
+    bool exists(const std::string & path) const override;
+    bool isDirectory(const std::string & path) const override;
+    std::vector<std::string> listDirectory(const std::string & path) const override;
+
 
 protected:
+    std::string getMetadataKeyPrefix() const override { return metadata_key_prefix; }
     std::shared_ptr<PathMap> getPathMap() const override { return path_map; }
-    std::vector<std::string> getDirectChildrenOnDisk(
-        const std::string & storage_key, const RelativePathsWithMetadata & remote_paths, const std::string & local_path) const override;
+    void getDirectChildrenOnDisk(
+        const std::string & storage_key,
+        const std::string & storage_key_perfix,
+        const RelativePathsWithMetadata & remote_paths,
+        const std::string & local_path,
+        std::unordered_set<std::string> & result) const override;
 };
 
 }
diff --git a/tests/integration/test_s3_plain_rewritable/test.py b/tests/integration/test_s3_plain_rewritable/test.py
index 4b1aaafc814..020e170eb48 100644
--- a/tests/integration/test_s3_plain_rewritable/test.py
+++ b/tests/integration/test_s3_plain_rewritable/test.py
@@ -139,6 +139,19 @@ def test(storage_policy):
             == insert_values_arr[i]
         )
 
+    metadata_it = cluster.minio_client.list_objects(
+        cluster.minio_bucket, "data/", recursive=True
+    )
+    metadata_count = 0
+    for obj in list(metadata_it):
+        if "/__meta/" in obj.object_name:
+            assert obj.object_name.endswith("/prefix.path")
+            metadata_count += 1
+        else:
+            assert not obj.object_name.endswith("/prefix.path")
+
+    assert metadata_count > 0
+
     for i in range(NUM_WORKERS):
         node = cluster.instances[f"node{i + 1}"]
         node.query("DROP TABLE IF EXISTS test SYNC")

From 98ad45ba960de4cc29ee794e2eeccf9fe6f8e0a8 Mon Sep 17 00:00:00 2001
From: Julia Kartseva <yulia.kartseva@gmail.com>
Date: Fri, 28 Jun 2024 00:58:20 +0000
Subject: [PATCH 1213/1488] Add prefix_path parameter to object key generator

---
 src/Common/ObjectStorageKeyGenerator.cpp        | 11 ++++++++---
 src/Common/ObjectStorageKeyGenerator.h          |  3 ++-
 .../AzureBlobStorage/AzureObjectStorage.cpp     |  4 +++-
 .../AzureBlobStorage/AzureObjectStorage.h       |  2 +-
 .../Cached/CachedObjectStorage.cpp              | 10 ++++++----
 .../ObjectStorages/Cached/CachedObjectStorage.h |  5 +++--
 .../CommonPathPrefixKeyGenerator.cpp            |  3 ++-
 .../CommonPathPrefixKeyGenerator.h              |  3 ++-
 .../DiskObjectStorageTransaction.cpp            |  6 +++---
 .../ObjectStorages/HDFS/HDFSObjectStorage.cpp   |  6 ++++--
 .../ObjectStorages/HDFS/HDFSObjectStorage.h     |  2 +-
 src/Disks/ObjectStorages/IObjectStorage.h       |  5 +++--
 .../ObjectStorages/Local/LocalObjectStorage.cpp | 14 ++++++++------
 .../ObjectStorages/Local/LocalObjectStorage.h   |  2 +-
 .../MetadataStorageFromPlainObjectStorage.cpp   | 14 +++++++-------
 ...aStorageFromPlainRewritableObjectStorage.cpp |  6 +++---
 src/Disks/ObjectStorages/PlainObjectStorage.h   |  2 +-
 .../PlainRewritableObjectStorage.h              | 17 +++++++++++------
 src/Disks/ObjectStorages/S3/DiskS3Utils.cpp     |  2 +-
 src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp |  4 ++--
 src/Disks/ObjectStorages/S3/S3ObjectStorage.h   |  2 +-
 src/Disks/ObjectStorages/Web/WebObjectStorage.h |  2 +-
 22 files changed, 74 insertions(+), 51 deletions(-)

diff --git a/src/Common/ObjectStorageKeyGenerator.cpp b/src/Common/ObjectStorageKeyGenerator.cpp
index e9212c3f04d..3e7bf3116bd 100644
--- a/src/Common/ObjectStorageKeyGenerator.cpp
+++ b/src/Common/ObjectStorageKeyGenerator.cpp
@@ -3,6 +3,7 @@
 #include <Common/getRandomASCIIString.h>
 #include <Common/MatchGenerator.h>
 
+#include <optional>
 #include <fmt/format.h>
 
 
@@ -14,7 +15,10 @@ public:
     , re_gen(key_template)
     {
     }
-    DB::ObjectStorageKey generate(const String &, bool) const override { return DB::ObjectStorageKey::createAsAbsolute(re_gen.generate()); }
+    DB::ObjectStorageKey generate(const String &, bool /* is_directory */, const std::optional<String> & /* key_prefix */) const override
+    {
+        return DB::ObjectStorageKey::createAsAbsolute(re_gen.generate());
+    }
 
 private:
     String key_template;
@@ -29,7 +33,7 @@ public:
         : key_prefix(std::move(key_prefix_))
     {}
 
-    DB::ObjectStorageKey generate(const String &, bool) const override
+    DB::ObjectStorageKey generate(const String &, bool /* is_directory */, const std::optional<String> & /* key_prefix */) const override
     {
         /// Path to store the new S3 object.
 
@@ -60,7 +64,8 @@ public:
         : key_prefix(std::move(key_prefix_))
     {}
 
-    DB::ObjectStorageKey generate(const String & path, bool) const override
+    DB::ObjectStorageKey
+    generate(const String & path, bool /* is_directory */, const std::optional<String> & /* key_prefix */) const override
     {
         return DB::ObjectStorageKey::createAsRelative(key_prefix, path);
     }
diff --git a/src/Common/ObjectStorageKeyGenerator.h b/src/Common/ObjectStorageKeyGenerator.h
index 11da039b33b..12aeec1714d 100644
--- a/src/Common/ObjectStorageKeyGenerator.h
+++ b/src/Common/ObjectStorageKeyGenerator.h
@@ -11,7 +11,8 @@ class IObjectStorageKeysGenerator
 public:
     virtual ~IObjectStorageKeysGenerator() = default;
 
-    virtual ObjectStorageKey generate(const String & path, bool is_directory) const = 0;
+    /// Generates an object storage key based on a path in the virtual filesystem.
+    virtual ObjectStorageKey generate(const String & path, bool is_directory, const std::optional<String> & key_prefix) const = 0;
 };
 
 using ObjectStorageKeysGeneratorPtr = std::shared_ptr<IObjectStorageKeysGenerator>;
diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp
index bc16955143b..0d92561d142 100644
--- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp
+++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp
@@ -1,3 +1,4 @@
+#include <optional>
 #include <Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h>
 #include "Common/Exception.h"
 
@@ -117,7 +118,8 @@ AzureObjectStorage::AzureObjectStorage(
 {
 }
 
-ObjectStorageKey AzureObjectStorage::generateObjectKeyForPath(const std::string & /* path */) const
+ObjectStorageKey
+AzureObjectStorage::generateObjectKeyForPath(const std::string & /* path */, const std::optional<std::string> & /* key_prefix */) const
 {
     return ObjectStorageKey::createAsRelative(getRandomASCIIString(32));
 }
diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h
index 2c7ce5e18dc..bc90b05e64d 100644
--- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h
+++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureObjectStorage.h
@@ -101,7 +101,7 @@ public:
         const std::string & config_prefix,
         ContextPtr context) override;
 
-    ObjectStorageKey generateObjectKeyForPath(const std::string & path) const override;
+    ObjectStorageKey generateObjectKeyForPath(const std::string & path, const std::optional<std::string> & key_prefix) const override;
 
     bool isRemote() const override { return true; }
 
diff --git a/src/Disks/ObjectStorages/Cached/CachedObjectStorage.cpp b/src/Disks/ObjectStorages/Cached/CachedObjectStorage.cpp
index a3b6e25e8ea..fb817005399 100644
--- a/src/Disks/ObjectStorages/Cached/CachedObjectStorage.cpp
+++ b/src/Disks/ObjectStorages/Cached/CachedObjectStorage.cpp
@@ -34,14 +34,16 @@ FileCache::Key CachedObjectStorage::getCacheKey(const std::string & path) const
     return cache->createKeyForPath(path);
 }
 
-ObjectStorageKey CachedObjectStorage::generateObjectKeyForPath(const std::string & path) const
+ObjectStorageKey
+CachedObjectStorage::generateObjectKeyForPath(const std::string & path, const std::optional<std::string> & key_prefix) const
 {
-    return object_storage->generateObjectKeyForPath(path);
+    return object_storage->generateObjectKeyForPath(path, key_prefix);
 }
 
-ObjectStorageKey CachedObjectStorage::generateObjectKeyPrefixForDirectoryPath(const std::string & path) const
+ObjectStorageKey
+CachedObjectStorage::generateObjectKeyPrefixForDirectoryPath(const std::string & path, const std::optional<std::string> & key_prefix) const
 {
-    return object_storage->generateObjectKeyPrefixForDirectoryPath(path);
+    return object_storage->generateObjectKeyPrefixForDirectoryPath(path, key_prefix);
 }
 
 ReadSettings CachedObjectStorage::patchSettings(const ReadSettings & read_settings) const
diff --git a/src/Disks/ObjectStorages/Cached/CachedObjectStorage.h b/src/Disks/ObjectStorages/Cached/CachedObjectStorage.h
index 93ef2659cbb..efcdbfebabf 100644
--- a/src/Disks/ObjectStorages/Cached/CachedObjectStorage.h
+++ b/src/Disks/ObjectStorages/Cached/CachedObjectStorage.h
@@ -98,9 +98,10 @@ public:
 
     const std::string & getCacheName() const override { return cache_config_name; }
 
-    ObjectStorageKey generateObjectKeyForPath(const std::string & path) const override;
+    ObjectStorageKey generateObjectKeyForPath(const std::string & path, const std::optional<std::string> & key_prefix) const override;
 
-    ObjectStorageKey generateObjectKeyPrefixForDirectoryPath(const std::string & path) const override;
+    ObjectStorageKey
+    generateObjectKeyPrefixForDirectoryPath(const std::string & path, const std::optional<std::string> & key_prefix) const override;
 
     void setKeysGenerator(ObjectStorageKeysGeneratorPtr gen) override { object_storage->setKeysGenerator(gen); }
 
diff --git a/src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.cpp b/src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.cpp
index 2a06d56e5c7..0a4426e8e66 100644
--- a/src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.cpp
+++ b/src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.cpp
@@ -15,7 +15,8 @@ CommonPathPrefixKeyGenerator::CommonPathPrefixKeyGenerator(
 {
 }
 
-ObjectStorageKey CommonPathPrefixKeyGenerator::generate(const String & path, bool is_directory) const
+ObjectStorageKey
+CommonPathPrefixKeyGenerator::generate(const String & path, bool is_directory, const std::optional<String> & /* key_prefix */) const
 {
     const auto & [object_key_prefix, suffix_parts] = getLongestObjectKeyPrefix(path);
 
diff --git a/src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.h b/src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.h
index fb1140de908..08495738505 100644
--- a/src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.h
+++ b/src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.h
@@ -5,6 +5,7 @@
 
 #include <filesystem>
 #include <map>
+#include <optional>
 
 namespace DB
 {
@@ -26,7 +27,7 @@ public:
 
     explicit CommonPathPrefixKeyGenerator(String key_prefix_, SharedMutex & shared_mutex_, std::weak_ptr<PathMap> path_map_);
 
-    ObjectStorageKey generate(const String & path, bool is_directory) const override;
+    ObjectStorageKey generate(const String & path, bool is_directory, const std::optional<String> & key_prefix) const override;
 
 private:
     /// Longest key prefix and unresolved parts of the source path.
diff --git a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp
index b5805f6d23a..880911b9958 100644
--- a/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp
+++ b/src/Disks/ObjectStorages/DiskObjectStorageTransaction.cpp
@@ -537,7 +537,7 @@ struct CopyFileObjectStorageOperation final : public IDiskObjectStorageOperation
 
         for (const auto & object_from : source_blobs)
         {
-            auto object_key = destination_object_storage.generateObjectKeyForPath(to_path);
+            auto object_key = destination_object_storage.generateObjectKeyForPath(to_path, std::nullopt /* key_prefix */);
             auto object_to = StoredObject(object_key.serialize());
 
             object_storage.copyObjectToAnotherObjectStorage(object_from, object_to,read_settings,write_settings, destination_object_storage);
@@ -738,7 +738,7 @@ std::unique_ptr<WriteBufferFromFileBase> DiskObjectStorageTransaction::writeFile
     const WriteSettings & settings,
     bool autocommit)
 {
-    auto object_key = object_storage.generateObjectKeyForPath(path);
+    auto object_key = object_storage.generateObjectKeyForPath(path, std::nullopt /* key_prefix */);
     std::optional<ObjectAttributes> object_attributes;
 
     if (metadata_helper)
@@ -835,7 +835,7 @@ void DiskObjectStorageTransaction::writeFileUsingBlobWritingFunction(
     const String & path, WriteMode mode, WriteBlobFunction && write_blob_function)
 {
     /// This function is a simplified and adapted version of DiskObjectStorageTransaction::writeFile().
-    auto object_key = object_storage.generateObjectKeyForPath(path);
+    auto object_key = object_storage.generateObjectKeyForPath(path, std::nullopt /* key_prefix */);
     std::optional<ObjectAttributes> object_attributes;
 
     if (metadata_helper)
diff --git a/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.cpp b/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.cpp
index dcb2af9d4d3..3ce2a0f4903 100644
--- a/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.cpp
+++ b/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.cpp
@@ -4,8 +4,9 @@
 #include <Storages/ObjectStorage/HDFS/WriteBufferFromHDFS.h>
 #include <Storages/ObjectStorage/HDFS/HDFSCommon.h>
 
-#include <Storages/ObjectStorage/HDFS/ReadBufferFromHDFS.h>
+#include <optional>
 #include <Disks/IO/ReadBufferFromRemoteFSGather.h>
+#include <Storages/ObjectStorage/HDFS/ReadBufferFromHDFS.h>
 #include <Common/getRandomASCIIString.h>
 #include <Common/logger_useful.h>
 
@@ -53,7 +54,8 @@ std::string HDFSObjectStorage::extractObjectKeyFromURL(const StoredObject & obje
     return path;
 }
 
-ObjectStorageKey HDFSObjectStorage::generateObjectKeyForPath(const std::string & /* path */) const
+ObjectStorageKey
+HDFSObjectStorage::generateObjectKeyForPath(const std::string & /* path */, const std::optional<std::string> & /* key_prefix */) const
 {
     initializeHDFSFS();
     /// what ever data_source_description.description value is, consider that key as relative key
diff --git a/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.h b/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.h
index 8aae90d0721..0cb31eb8b8b 100644
--- a/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.h
+++ b/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.h
@@ -111,7 +111,7 @@ public:
         const std::string & config_prefix,
         ContextPtr context) override;
 
-    ObjectStorageKey generateObjectKeyForPath(const std::string & path) const override;
+    ObjectStorageKey generateObjectKeyForPath(const std::string & path, const std::optional<std::string> & key_prefix) const override;
 
     bool isRemote() const override { return true; }
 
diff --git a/src/Disks/ObjectStorages/IObjectStorage.h b/src/Disks/ObjectStorages/IObjectStorage.h
index ceea4d5a2bb..529c79790fd 100644
--- a/src/Disks/ObjectStorages/IObjectStorage.h
+++ b/src/Disks/ObjectStorages/IObjectStorage.h
@@ -232,10 +232,11 @@ public:
 
     /// Generate blob name for passed absolute local path.
     /// Path can be generated either independently or based on `path`.
-    virtual ObjectStorageKey generateObjectKeyForPath(const std::string & path) const = 0;
+    virtual ObjectStorageKey generateObjectKeyForPath(const std::string & path, const std::optional<std::string> & key_prefix) const = 0;
 
     /// Object key prefix for local paths in the directory 'path'.
-    virtual ObjectStorageKey generateObjectKeyPrefixForDirectoryPath(const std::string & /* path */) const
+    virtual ObjectStorageKey
+    generateObjectKeyPrefixForDirectoryPath(const std::string & /* path */, const std::optional<std::string> & /* key_prefix */) const
     {
         throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method 'generateObjectKeyPrefixForDirectoryPath' is not implemented");
     }
diff --git a/src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp b/src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp
index a247d86ddce..20ef135cdf7 100644
--- a/src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp
+++ b/src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp
@@ -1,15 +1,16 @@
 #include <Disks/ObjectStorages/Local/LocalObjectStorage.h>
 
-#include <Interpreters/Context.h>
-#include <Common/filesystemHelpers.h>
-#include <Common/logger_useful.h>
+#include <filesystem>
+#include <optional>
+#include <Disks/IO/AsynchronousBoundedReadBuffer.h>
 #include <Disks/IO/ReadBufferFromRemoteFSGather.h>
 #include <Disks/IO/createReadBufferFromFileBase.h>
-#include <Disks/IO/AsynchronousBoundedReadBuffer.h>
 #include <IO/WriteBufferFromFile.h>
 #include <IO/copyData.h>
+#include <Interpreters/Context.h>
+#include <Common/filesystemHelpers.h>
 #include <Common/getRandomASCIIString.h>
-#include <filesystem>
+#include <Common/logger_useful.h>
 
 namespace fs = std::filesystem;
 
@@ -222,7 +223,8 @@ std::unique_ptr<IObjectStorage> LocalObjectStorage::cloneObjectStorage(
     throw Exception(ErrorCodes::NOT_IMPLEMENTED, "cloneObjectStorage() is not implemented for LocalObjectStorage");
 }
 
-ObjectStorageKey LocalObjectStorage::generateObjectKeyForPath(const std::string & /* path */) const
+ObjectStorageKey
+LocalObjectStorage::generateObjectKeyForPath(const std::string & /* path */, const std::optional<std::string> & /* key_prefix */) const
 {
     constexpr size_t key_name_total_size = 32;
     return ObjectStorageKey::createAsRelative(key_prefix, getRandomASCIIString(key_name_total_size));
diff --git a/src/Disks/ObjectStorages/Local/LocalObjectStorage.h b/src/Disks/ObjectStorages/Local/LocalObjectStorage.h
index 371cd37f8b2..564d49bf876 100644
--- a/src/Disks/ObjectStorages/Local/LocalObjectStorage.h
+++ b/src/Disks/ObjectStorages/Local/LocalObjectStorage.h
@@ -81,7 +81,7 @@ public:
         const std::string & config_prefix,
         ContextPtr context) override;
 
-    ObjectStorageKey generateObjectKeyForPath(const std::string & path) const override;
+    ObjectStorageKey generateObjectKeyForPath(const std::string & path, const std::optional<std::string> & key_prefix) const override;
 
     bool isRemote() const override { return false; }
 
diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.cpp b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.cpp
index 3da190c7256..589b18abca8 100644
--- a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.cpp
+++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.cpp
@@ -42,7 +42,7 @@ bool MetadataStorageFromPlainObjectStorage::exists(const std::string & path) con
 {
     /// NOTE: exists() cannot be used here since it works only for existing
     /// key, and does not work for some intermediate path.
-    auto object_key = object_storage->generateObjectKeyForPath(path);
+    auto object_key = object_storage->generateObjectKeyForPath(path, std::nullopt /* key_prefix */);
     return object_storage->existsOrHasAnyChild(object_key.serialize());
 }
 
@@ -54,7 +54,7 @@ bool MetadataStorageFromPlainObjectStorage::isFile(const std::string & path) con
 
 bool MetadataStorageFromPlainObjectStorage::isDirectory(const std::string & path) const
 {
-    auto key_prefix = object_storage->generateObjectKeyForPath(path).serialize();
+    auto key_prefix = object_storage->generateObjectKeyForPath(path, std::nullopt /* key_prefix */).serialize();
     auto directory = std::filesystem::path(std::move(key_prefix)) / "";
 
     return object_storage->existsOrHasAnyChild(directory);
@@ -62,7 +62,7 @@ bool MetadataStorageFromPlainObjectStorage::isDirectory(const std::string & path
 
 uint64_t MetadataStorageFromPlainObjectStorage::getFileSize(const String & path) const
 {
-    auto object_key = object_storage->generateObjectKeyForPath(path);
+    auto object_key = object_storage->generateObjectKeyForPath(path, std::nullopt /* key_prefix */);
     auto metadata = object_storage->tryGetObjectMetadata(object_key.serialize());
     if (metadata)
         return metadata->size_bytes;
@@ -71,7 +71,7 @@ uint64_t MetadataStorageFromPlainObjectStorage::getFileSize(const String & path)
 
 std::vector<std::string> MetadataStorageFromPlainObjectStorage::listDirectory(const std::string & path) const
 {
-    auto key_prefix = object_storage->generateObjectKeyForPath(path).serialize();
+    auto key_prefix = object_storage->generateObjectKeyForPath(path, std::nullopt /* key_prefix */).serialize();
 
     RelativePathsWithMetadata files;
     std::string abs_key = key_prefix;
@@ -98,7 +98,7 @@ DirectoryIteratorPtr MetadataStorageFromPlainObjectStorage::iterateDirectory(con
 StoredObjects MetadataStorageFromPlainObjectStorage::getStorageObjects(const std::string & path) const
 {
     size_t object_size = getFileSize(path);
-    auto object_key = object_storage->generateObjectKeyForPath(path);
+    auto object_key = object_storage->generateObjectKeyForPath(path, std::nullopt /* key_prefix */);
     return {StoredObject(object_key.serialize(), path, object_size)};
 }
 
@@ -130,7 +130,7 @@ const IMetadataStorage & MetadataStorageFromPlainObjectStorageTransaction::getSt
 
 void MetadataStorageFromPlainObjectStorageTransaction::unlinkFile(const std::string & path)
 {
-    auto object_key = metadata_storage.object_storage->generateObjectKeyForPath(path);
+    auto object_key = metadata_storage.object_storage->generateObjectKeyForPath(path, std::nullopt /* key_prefix */);
     auto object = StoredObject(object_key.serialize());
     metadata_storage.object_storage->removeObject(object);
 }
@@ -155,7 +155,7 @@ void MetadataStorageFromPlainObjectStorageTransaction::createDirectory(const std
         return;
 
     auto normalized_path = normalizeDirectoryPath(path);
-    auto key_prefix = object_storage->generateObjectKeyPrefixForDirectoryPath(normalized_path).serialize();
+    auto key_prefix = object_storage->generateObjectKeyPrefixForDirectoryPath(normalized_path, std::nullopt /* key_prefix */).serialize();
     chassert(key_prefix.starts_with(object_storage->getCommonKeyPrefix()));
     auto op = std::make_unique<MetadataStorageFromPlainObjectStorageCreateDirectoryOperation>(
         std::move(normalized_path),
diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp b/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp
index f3d00a928e3..de65cd5c233 100644
--- a/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp
+++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp
@@ -215,7 +215,7 @@ bool MetadataStorageFromPlainRewritableObjectStorage::exists(const std::string &
 
     if (getMetadataKeyPrefix() != object_storage->getCommonKeyPrefix())
     {
-        auto key_prefix = object_storage->generateObjectKeyForPath(path).serialize();
+        auto key_prefix = object_storage->generateObjectKeyForPath(path, std::nullopt /* key_prefix */).serialize();
         chassert(key_prefix.starts_with(object_storage->getCommonKeyPrefix()));
         auto metadata_key = std::filesystem::path(getMetadataKeyPrefix()) / key_prefix.substr(object_storage->getCommonKeyPrefix().size());
         return object_storage->existsOrHasAnyChild(metadata_key);
@@ -228,7 +228,7 @@ bool MetadataStorageFromPlainRewritableObjectStorage::isDirectory(const std::str
 {
     if (getMetadataKeyPrefix() != object_storage->getCommonKeyPrefix())
     {
-        auto directory = std::filesystem::path(object_storage->generateObjectKeyForPath(path).serialize()) / "";
+        auto directory = std::filesystem::path(object_storage->generateObjectKeyForPath(path, std::nullopt /* key_prefix */).serialize()) / "";
         chassert(directory.string().starts_with(object_storage->getCommonKeyPrefix()));
         auto metadata_key
             = std::filesystem::path(getMetadataKeyPrefix()) / directory.string().substr(object_storage->getCommonKeyPrefix().size());
@@ -240,7 +240,7 @@ bool MetadataStorageFromPlainRewritableObjectStorage::isDirectory(const std::str
 
 std::vector<std::string> MetadataStorageFromPlainRewritableObjectStorage::listDirectory(const std::string & path) const
 {
-    auto key_prefix = object_storage->generateObjectKeyForPath(path).serialize();
+    auto key_prefix = object_storage->generateObjectKeyForPath(path, std::nullopt /* key_prefix */).serialize();
 
     RelativePathsWithMetadata files;
     std::string abs_key = key_prefix;
diff --git a/src/Disks/ObjectStorages/PlainObjectStorage.h b/src/Disks/ObjectStorages/PlainObjectStorage.h
index e0907d0b4d8..805b3436fce 100644
--- a/src/Disks/ObjectStorages/PlainObjectStorage.h
+++ b/src/Disks/ObjectStorages/PlainObjectStorage.h
@@ -26,7 +26,7 @@ public:
 
     bool isPlain() const override { return true; }
 
-    ObjectStorageKey generateObjectKeyForPath(const std::string & path) const override
+    ObjectStorageKey generateObjectKeyForPath(const std::string & path, const std::optional<std::string> & /* key_prefix */) const override
     {
         return ObjectStorageKey::createAsRelative(BaseObjectStorage::getCommonKeyPrefix(), path);
     }
diff --git a/src/Disks/ObjectStorages/PlainRewritableObjectStorage.h b/src/Disks/ObjectStorages/PlainRewritableObjectStorage.h
index 5f000afe625..dcea5964fc5 100644
--- a/src/Disks/ObjectStorages/PlainRewritableObjectStorage.h
+++ b/src/Disks/ObjectStorages/PlainRewritableObjectStorage.h
@@ -1,5 +1,7 @@
 #pragma once
 
+#include <optional>
+#include <string>
 #include <Disks/ObjectStorages/IObjectStorage.h>
 #include <Common/ObjectStorageKeyGenerator.h>
 #include "CommonPathPrefixKeyGenerator.h"
@@ -33,9 +35,10 @@ public:
 
     bool isPlain() const override { return true; }
 
-    ObjectStorageKey generateObjectKeyForPath(const std::string & path) const override;
+    ObjectStorageKey generateObjectKeyForPath(const std::string & path, const std::optional<std::string> & key_prefix) const override;
 
-    ObjectStorageKey generateObjectKeyPrefixForDirectoryPath(const std::string & path) const override;
+    ObjectStorageKey
+    generateObjectKeyPrefixForDirectoryPath(const std::string & path, const std::optional<std::string> & key_prefix) const override;
 
     void setKeysGenerator(ObjectStorageKeysGeneratorPtr gen) override { key_generator = gen; }
 
@@ -46,20 +49,22 @@ private:
 
 
 template <typename BaseObjectStorage>
-ObjectStorageKey PlainRewritableObjectStorage<BaseObjectStorage>::generateObjectKeyForPath(const std::string & path) const
+ObjectStorageKey PlainRewritableObjectStorage<BaseObjectStorage>::generateObjectKeyForPath(
+    const std::string & path, const std::optional<std::string> & key_prefix) const
 {
     if (!key_generator)
         throw Exception(ErrorCodes::LOGICAL_ERROR, "Key generator is not set");
 
-    return key_generator->generate(path, /* is_directory */ false);
+    return key_generator->generate(path, /* is_directory */ false, key_prefix);
 }
 
 template <typename BaseObjectStorage>
-ObjectStorageKey PlainRewritableObjectStorage<BaseObjectStorage>::generateObjectKeyPrefixForDirectoryPath(const std::string & path) const
+ObjectStorageKey PlainRewritableObjectStorage<BaseObjectStorage>::generateObjectKeyPrefixForDirectoryPath(
+    const std::string & path, const std::optional<std::string> & key_prefix) const
 {
     if (!key_generator)
         throw Exception(ErrorCodes::LOGICAL_ERROR, "Key generator is not set");
 
-    return key_generator->generate(path, /* is_directory */ true);
+    return key_generator->generate(path, /* is_directory */ true, key_prefix);
 }
 }
diff --git a/src/Disks/ObjectStorages/S3/DiskS3Utils.cpp b/src/Disks/ObjectStorages/S3/DiskS3Utils.cpp
index 63e7ebb00c5..b20a2940e47 100644
--- a/src/Disks/ObjectStorages/S3/DiskS3Utils.cpp
+++ b/src/Disks/ObjectStorages/S3/DiskS3Utils.cpp
@@ -79,7 +79,7 @@ bool checkBatchRemove(S3ObjectStorage & storage)
     /// We are using generateObjectKeyForPath() which returns random object key.
     /// That generated key is placed in a right directory where we should have write access.
     const String path = fmt::format("clickhouse_remove_objects_capability_{}", getServerUUID());
-    const auto key = storage.generateObjectKeyForPath(path);
+    const auto key = storage.generateObjectKeyForPath(path, {} /* key_prefix */);
     StoredObject object(key.serialize(), path);
     try
     {
diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp
index a6672e14e10..3c4b4d76bf5 100644
--- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp
+++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.cpp
@@ -624,12 +624,12 @@ std::unique_ptr<IObjectStorage> S3ObjectStorage::cloneObjectStorage(
         std::move(new_client), std::move(new_s3_settings), new_uri, s3_capabilities, key_generator, disk_name);
 }
 
-ObjectStorageKey S3ObjectStorage::generateObjectKeyForPath(const std::string & path) const
+ObjectStorageKey S3ObjectStorage::generateObjectKeyForPath(const std::string & path, const std::optional<std::string> & key_prefix) const
 {
     if (!key_generator)
         throw Exception(ErrorCodes::LOGICAL_ERROR, "Key generator is not set");
 
-    return key_generator->generate(path, /* is_directory */ false);
+    return key_generator->generate(path, /* is_directory */ false, key_prefix);
 }
 
 std::shared_ptr<const S3::Client> S3ObjectStorage::getS3StorageClient()
diff --git a/src/Disks/ObjectStorages/S3/S3ObjectStorage.h b/src/Disks/ObjectStorages/S3/S3ObjectStorage.h
index cbe004bc298..d786a6b37f3 100644
--- a/src/Disks/ObjectStorages/S3/S3ObjectStorage.h
+++ b/src/Disks/ObjectStorages/S3/S3ObjectStorage.h
@@ -164,7 +164,7 @@ public:
 
     bool supportParallelWrite() const override { return true; }
 
-    ObjectStorageKey generateObjectKeyForPath(const std::string & path) const override;
+    ObjectStorageKey generateObjectKeyForPath(const std::string & path, const std::optional<std::string> & key_prefix) const override;
 
     bool isReadOnly() const override { return s3_settings.get()->read_only; }
 
diff --git a/src/Disks/ObjectStorages/Web/WebObjectStorage.h b/src/Disks/ObjectStorages/Web/WebObjectStorage.h
index 9ca2950dae0..ab357d6f50d 100644
--- a/src/Disks/ObjectStorages/Web/WebObjectStorage.h
+++ b/src/Disks/ObjectStorages/Web/WebObjectStorage.h
@@ -82,7 +82,7 @@ public:
         const std::string & config_prefix,
         ContextPtr context) override;
 
-    ObjectStorageKey generateObjectKeyForPath(const std::string & path) const override
+    ObjectStorageKey generateObjectKeyForPath(const std::string & path, const std::optional<std::string> & /* key_prefix */) const override
     {
         return ObjectStorageKey::createAsRelative(path);
     }

From 27392fee6eec22c7f2dac3d17f73ab9a528f1fc8 Mon Sep 17 00:00:00 2001
From: Julia Kartseva <yulia.kartseva@gmail.com>
Date: Fri, 28 Jun 2024 01:30:49 +0000
Subject: [PATCH 1214/1488] Minor refactor

---
 .../CommonPathPrefixKeyGenerator.cpp          |  8 ++--
 .../MetadataStorageFromPlainObjectStorage.cpp | 38 +++++++------------
 .../MetadataStorageFromPlainObjectStorage.h   |  8 ----
 ...torageFromPlainRewritableObjectStorage.cpp | 26 +++++--------
 ...aStorageFromPlainRewritableObjectStorage.h |  2 +-
 5 files changed, 28 insertions(+), 54 deletions(-)

diff --git a/src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.cpp b/src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.cpp
index 0a4426e8e66..ef599a2f366 100644
--- a/src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.cpp
+++ b/src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.cpp
@@ -16,15 +16,15 @@ CommonPathPrefixKeyGenerator::CommonPathPrefixKeyGenerator(
 }
 
 ObjectStorageKey
-CommonPathPrefixKeyGenerator::generate(const String & path, bool is_directory, const std::optional<String> & /* key_prefix */) const
+CommonPathPrefixKeyGenerator::generate(const String & path, bool is_directory, const std::optional<String> & key_prefix) const
 {
     const auto & [object_key_prefix, suffix_parts] = getLongestObjectKeyPrefix(path);
 
-    auto key = std::filesystem::path(object_key_prefix.empty() ? std::string() : object_key_prefix);
+    auto key = std::filesystem::path(object_key_prefix);
 
     /// The longest prefix is the same as path, meaning that the  path is already mapped.
     if (suffix_parts.empty())
-        return ObjectStorageKey::createAsRelative(storage_key_prefix, std::move(key));
+        return ObjectStorageKey::createAsRelative(key_prefix.has_value() ? *key_prefix : storage_key_prefix, std::move(key));
 
     /// File and top-level directory paths are mapped as is.
     if (!is_directory || object_key_prefix.empty())
@@ -40,7 +40,7 @@ CommonPathPrefixKeyGenerator::generate(const String & path, bool is_directory, c
         key /= getRandomASCIIString(part_size);
     }
 
-    return ObjectStorageKey::createAsRelative(storage_key_prefix, key);
+    return ObjectStorageKey::createAsRelative(key_prefix.has_value() ? *key_prefix : storage_key_prefix, key);
 }
 
 std::tuple<std::string, std::vector<std::string>> CommonPathPrefixKeyGenerator::getLongestObjectKeyPrefix(const std::string & path) const
diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.cpp b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.cpp
index 589b18abca8..02048c07a57 100644
--- a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.cpp
+++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.cpp
@@ -80,9 +80,20 @@ std::vector<std::string> MetadataStorageFromPlainObjectStorage::listDirectory(co
 
     object_storage->listObjects(abs_key, files, 0);
 
-    std::unordered_set<std::string> directories;
-    getDirectChildrenOnDisk(abs_key, object_storage->getCommonKeyPrefix(), files, path, directories);
-    return std::vector<std::string>(std::make_move_iterator(directories.begin()), std::make_move_iterator(directories.end()));
+    std::unordered_set<std::string> result;
+    for (const auto & elem : files)
+    {
+        const auto & p = elem->relative_path;
+        chassert(p.find(abs_key) == 0);
+        const auto child_pos = abs_key.size();
+        /// string::npos is ok.
+        const auto slash_pos = p.find('/', child_pos);
+        if (slash_pos == std::string::npos)
+            result.emplace(p.substr(child_pos));
+        else
+            result.emplace(p.substr(child_pos, slash_pos - child_pos));
+    }
+    return std::vector<std::string>(std::make_move_iterator(result.begin()), std::make_move_iterator(result.end()));
 }
 
 DirectoryIteratorPtr MetadataStorageFromPlainObjectStorage::iterateDirectory(const std::string & path) const
@@ -102,27 +113,6 @@ StoredObjects MetadataStorageFromPlainObjectStorage::getStorageObjects(const std
     return {StoredObject(object_key.serialize(), path, object_size)};
 }
 
-void MetadataStorageFromPlainObjectStorage::getDirectChildrenOnDisk(
-    const std::string & storage_key,
-    const std::string & /* storage_key_perfix */,
-    const RelativePathsWithMetadata & remote_paths,
-    const std::string & /* local_path */,
-    std::unordered_set<std::string> & result) const
-{
-    for (const auto & elem : remote_paths)
-    {
-        const auto & path = elem->relative_path;
-        chassert(path.find(storage_key) == 0);
-        const auto child_pos = storage_key.size();
-        /// string::npos is ok.
-        const auto slash_pos = path.find('/', child_pos);
-        if (slash_pos == std::string::npos)
-            result.emplace(path.substr(child_pos));
-        else
-            result.emplace(path.substr(child_pos, slash_pos - child_pos));
-    }
-}
-
 const IMetadataStorage & MetadataStorageFromPlainObjectStorageTransaction::getStorageForNonTransactionalReads() const
 {
     return metadata_storage;
diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.h b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.h
index 97c5715a937..237327cd1f4 100644
--- a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.h
+++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.h
@@ -86,14 +86,6 @@ protected:
 
     /// Returns a map of local paths to paths in object storage.
     virtual std::shared_ptr<PathMap> getPathMap() const { throwNotImplemented(); }
-
-    /// Retrieves the immediate files and directories within a given directory on a disk.
-    virtual void getDirectChildrenOnDisk(
-        const std::string & storage_key,
-        const std::string & storage_key_perfix,
-        const RelativePathsWithMetadata & remote_paths,
-        const std::string & local_path,
-        std::unordered_set<std::string> & result) const;
 };
 
 class MetadataStorageFromPlainObjectStorageTransaction final : public IMetadataTransaction, private MetadataOperationsHolder
diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp b/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp
index de65cd5c233..b904c0d92b9 100644
--- a/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp
+++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp
@@ -121,7 +121,7 @@ MetadataStorageFromPlainObjectStorage::PathMap loadPathPrefixMap(const std::stri
     return result;
 }
 
-void getDirectChildrenOnRewritableDisk(
+void getDirectChildrenOnDiskImpl(
     const std::string & storage_key,
     const std::string & storage_key_perfix,
     const RelativePathsWithMetadata & remote_paths,
@@ -215,10 +215,8 @@ bool MetadataStorageFromPlainRewritableObjectStorage::exists(const std::string &
 
     if (getMetadataKeyPrefix() != object_storage->getCommonKeyPrefix())
     {
-        auto key_prefix = object_storage->generateObjectKeyForPath(path, std::nullopt /* key_prefix */).serialize();
-        chassert(key_prefix.starts_with(object_storage->getCommonKeyPrefix()));
-        auto metadata_key = std::filesystem::path(getMetadataKeyPrefix()) / key_prefix.substr(object_storage->getCommonKeyPrefix().size());
-        return object_storage->existsOrHasAnyChild(metadata_key);
+        auto key_prefix = object_storage->generateObjectKeyForPath(path, getMetadataKeyPrefix()).serialize();
+        return object_storage->existsOrHasAnyChild(key_prefix);
     }
 
     return false;
@@ -228,11 +226,8 @@ bool MetadataStorageFromPlainRewritableObjectStorage::isDirectory(const std::str
 {
     if (getMetadataKeyPrefix() != object_storage->getCommonKeyPrefix())
     {
-        auto directory = std::filesystem::path(object_storage->generateObjectKeyForPath(path, std::nullopt /* key_prefix */).serialize()) / "";
-        chassert(directory.string().starts_with(object_storage->getCommonKeyPrefix()));
-        auto metadata_key
-            = std::filesystem::path(getMetadataKeyPrefix()) / directory.string().substr(object_storage->getCommonKeyPrefix().size());
-        return object_storage->existsOrHasAnyChild(metadata_key);
+        auto directory = std::filesystem::path(object_storage->generateObjectKeyForPath(path, getMetadataKeyPrefix()).serialize()) / "";
+        return object_storage->existsOrHasAnyChild(directory);
     }
     else
         return MetadataStorageFromPlainObjectStorage::isDirectory(path);
@@ -240,12 +235,10 @@ bool MetadataStorageFromPlainRewritableObjectStorage::isDirectory(const std::str
 
 std::vector<std::string> MetadataStorageFromPlainRewritableObjectStorage::listDirectory(const std::string & path) const
 {
-    auto key_prefix = object_storage->generateObjectKeyForPath(path, std::nullopt /* key_prefix */).serialize();
+    auto key_prefix = object_storage->generateObjectKeyForPath(path, "" /* key_prefix */).serialize();
 
     RelativePathsWithMetadata files;
-    std::string abs_key = key_prefix;
-    if (!abs_key.ends_with('/'))
-        abs_key += '/';
+    auto abs_key = std::filesystem::path(object_storage->getCommonKeyPrefix()) / key_prefix / "";
 
     object_storage->listObjects(abs_key, files, 0);
 
@@ -255,8 +248,7 @@ std::vector<std::string> MetadataStorageFromPlainRewritableObjectStorage::listDi
     /// metadata along with regular files.
     if (object_storage->getCommonKeyPrefix() != getMetadataKeyPrefix())
     {
-        chassert(abs_key.starts_with(object_storage->getCommonKeyPrefix()));
-        auto metadata_key = std::filesystem::path(getMetadataKeyPrefix()) / abs_key.substr(object_storage->getCommonKeyPrefix().size());
+        auto metadata_key = std::filesystem::path(getMetadataKeyPrefix()) / key_prefix / "";
         RelativePathsWithMetadata metadata_files;
         object_storage->listObjects(metadata_key, metadata_files, 0);
         getDirectChildrenOnDisk(metadata_key, getMetadataKeyPrefix(), metadata_files, path, directories);
@@ -272,7 +264,7 @@ void MetadataStorageFromPlainRewritableObjectStorage::getDirectChildrenOnDisk(
     const std::string & local_path,
     std::unordered_set<std::string> & result) const
 {
-    getDirectChildrenOnRewritableDisk(storage_key, storage_key_perfix, remote_paths, local_path, *getPathMap(), metadata_mutex, result);
+    getDirectChildrenOnDiskImpl(storage_key, storage_key_perfix, remote_paths, local_path, *getPathMap(), metadata_mutex, result);
 }
 
 }
diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.h b/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.h
index 71153cbdc25..b067b391878 100644
--- a/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.h
+++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.h
@@ -33,7 +33,7 @@ protected:
         const std::string & storage_key_perfix,
         const RelativePathsWithMetadata & remote_paths,
         const std::string & local_path,
-        std::unordered_set<std::string> & result) const override;
+        std::unordered_set<std::string> & result) const;
 };
 
 }

From ecca720f9e076e49f280e48c6ff4046a19894b2a Mon Sep 17 00:00:00 2001
From: Julia Kartseva <yulia.kartseva@gmail.com>
Date: Fri, 28 Jun 2024 06:42:38 +0000
Subject: [PATCH 1215/1488] minor

---
 .../ObjectStorages/MetadataStorageFromPlainObjectStorage.cpp | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.cpp b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.cpp
index 02048c07a57..7553c7733b5 100644
--- a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.cpp
+++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.cpp
@@ -145,11 +145,10 @@ void MetadataStorageFromPlainObjectStorageTransaction::createDirectory(const std
         return;
 
     auto normalized_path = normalizeDirectoryPath(path);
-    auto key_prefix = object_storage->generateObjectKeyPrefixForDirectoryPath(normalized_path, std::nullopt /* key_prefix */).serialize();
-    chassert(key_prefix.starts_with(object_storage->getCommonKeyPrefix()));
+    auto key_prefix = object_storage->generateObjectKeyPrefixForDirectoryPath(normalized_path, "" /* key_prefix */).serialize();
     auto op = std::make_unique<MetadataStorageFromPlainObjectStorageCreateDirectoryOperation>(
         std::move(normalized_path),
-        key_prefix.substr(object_storage->getCommonKeyPrefix().size()),
+        std::move(key_prefix),
         *metadata_storage.getPathMap(),
         object_storage,
         metadata_storage.getMetadataKeyPrefix());

From 97519ae800b9a26942973d888354a2b013d53cc6 Mon Sep 17 00:00:00 2001
From: Julia Kartseva <yulia.kartseva@gmail.com>
Date: Thu, 4 Jul 2024 07:02:13 +0000
Subject: [PATCH 1216/1488] in-memory map path comparator

---
 .../CommonPathPrefixKeyGenerator.cpp          |  5 +++--
 .../CommonPathPrefixKeyGenerator.h            |  4 +++-
 .../MetadataStorageFromPlainObjectStorage.h   |  3 ++-
 ...torageFromPlainObjectStorageOperations.cpp | 18 ++++++++---------
 ...torageFromPlainRewritableObjectStorage.cpp |  9 ++++-----
 src/Disks/ObjectStorages/PathComparator.h     | 20 +++++++++++++++++++
 6 files changed, 41 insertions(+), 18 deletions(-)
 create mode 100644 src/Disks/ObjectStorages/PathComparator.h

diff --git a/src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.cpp b/src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.cpp
index ef599a2f366..062a2542654 100644
--- a/src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.cpp
+++ b/src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.cpp
@@ -18,7 +18,8 @@ CommonPathPrefixKeyGenerator::CommonPathPrefixKeyGenerator(
 ObjectStorageKey
 CommonPathPrefixKeyGenerator::generate(const String & path, bool is_directory, const std::optional<String> & key_prefix) const
 {
-    const auto & [object_key_prefix, suffix_parts] = getLongestObjectKeyPrefix(path);
+    const auto & [object_key_prefix, suffix_parts]
+        = getLongestObjectKeyPrefix(is_directory ? std::filesystem::path(path).parent_path().string() : path);
 
     auto key = std::filesystem::path(object_key_prefix);
 
@@ -54,7 +55,7 @@ std::tuple<std::string, std::vector<std::string>> CommonPathPrefixKeyGenerator::
 
     while (p != p.root_path())
     {
-        auto it = ptr->find(p / "");
+        auto it = ptr->find(p);
         if (it != ptr->end())
         {
             std::vector<std::string> vec(std::make_move_iterator(dq.begin()), std::make_move_iterator(dq.end()));
diff --git a/src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.h b/src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.h
index 08495738505..bca4f7060c4 100644
--- a/src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.h
+++ b/src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.h
@@ -3,6 +3,8 @@
 #include <Common/ObjectStorageKeyGenerator.h>
 #include <Common/SharedMutex.h>
 
+#include <Disks/ObjectStorages/PathComparator.h>
+
 #include <filesystem>
 #include <map>
 #include <optional>
@@ -23,7 +25,7 @@ class CommonPathPrefixKeyGenerator : public IObjectStorageKeysGenerator
 {
 public:
     /// Local to remote path map. Leverages filesystem::path comparator for paths.
-    using PathMap = std::map<std::filesystem::path, std::string>;
+    using PathMap = std::map<std::filesystem::path, std::string, PathComparator>;
 
     explicit CommonPathPrefixKeyGenerator(String key_prefix_, SharedMutex & shared_mutex_, std::weak_ptr<PathMap> path_map_);
 
diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.h b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.h
index 237327cd1f4..9ea1c475821 100644
--- a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.h
+++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.h
@@ -4,6 +4,7 @@
 #include <Disks/ObjectStorages/IMetadataStorage.h>
 #include <Disks/ObjectStorages/MetadataOperationsHolder.h>
 #include <Disks/ObjectStorages/MetadataStorageTransactionState.h>
+#include <Disks/ObjectStorages/PathComparator.h>
 
 #include <map>
 #include <string>
@@ -29,7 +30,7 @@ class MetadataStorageFromPlainObjectStorage : public IMetadataStorage
 {
 public:
     /// Local path prefixes mapped to storage key prefixes.
-    using PathMap = std::map<std::filesystem::path, std::string>;
+    using PathMap = std::map<std::filesystem::path, std::string, PathComparator>;
 
 private:
     friend class MetadataStorageFromPlainObjectStorageTransaction;
diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp
index 0a6086bd39d..b0b384f62c7 100644
--- a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp
+++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp
@@ -43,7 +43,7 @@ MetadataStorageFromPlainObjectStorageCreateDirectoryOperation::MetadataStorageFr
 
 void MetadataStorageFromPlainObjectStorageCreateDirectoryOperation::execute(std::unique_lock<SharedMutex> &)
 {
-    if (path_map.contains(path))
+    if (path_map.contains(path.parent_path()))
         return;
 
     auto metadata_object_key = createMetadataObjectKey(key_prefix, metadata_key_prefix);
@@ -64,7 +64,7 @@ void MetadataStorageFromPlainObjectStorageCreateDirectoryOperation::execute(std:
 
     write_created = true;
 
-    [[maybe_unused]] auto result = path_map.emplace(path, std::move(key_prefix));
+    [[maybe_unused]] auto result = path_map.emplace(path.parent_path(), std::move(key_prefix));
     chassert(result.second);
     auto metric = object_storage->getMetadataStorageMetrics().directory_map_size;
     CurrentMetrics::add(metric, 1);
@@ -84,7 +84,7 @@ void MetadataStorageFromPlainObjectStorageCreateDirectoryOperation::undo(std::un
 
     if (write_finalized)
     {
-        path_map.erase(path);
+        path_map.erase(path.parent_path());
         auto metric = object_storage->getMetadataStorageMetrics().directory_map_size;
         CurrentMetrics::sub(metric, 1);
 
@@ -111,11 +111,11 @@ MetadataStorageFromPlainObjectStorageMoveDirectoryOperation::MetadataStorageFrom
 std::unique_ptr<WriteBufferFromFileBase> MetadataStorageFromPlainObjectStorageMoveDirectoryOperation::createWriteBuf(
     const std::filesystem::path & expected_path, const std::filesystem::path & new_path, bool validate_content)
 {
-    auto expected_it = path_map.find(expected_path);
+    auto expected_it = path_map.find(expected_path.parent_path());
     if (expected_it == path_map.end())
         throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "Metadata object for the expected (source) path '{}' does not exist", expected_path);
 
-    if (path_map.contains(new_path))
+    if (path_map.contains(new_path.parent_path()))
         throw Exception(ErrorCodes::FILE_ALREADY_EXISTS, "Metadata object for the new (destination) path '{}' already exists", new_path);
 
     auto metadata_object_key = createMetadataObjectKey(expected_it->second, metadata_key_prefix);
@@ -156,7 +156,7 @@ void MetadataStorageFromPlainObjectStorageMoveDirectoryOperation::execute(std::u
     writeString(path_to.string(), *write_buf);
     write_buf->finalize();
 
-    [[maybe_unused]] auto result = path_map.emplace(path_to, path_map.extract(path_from).mapped());
+    [[maybe_unused]] auto result = path_map.emplace(path_to.parent_path(), path_map.extract(path_from.parent_path()).mapped());
     chassert(result.second);
 
     write_finalized = true;
@@ -165,7 +165,7 @@ void MetadataStorageFromPlainObjectStorageMoveDirectoryOperation::execute(std::u
 void MetadataStorageFromPlainObjectStorageMoveDirectoryOperation::undo(std::unique_lock<SharedMutex> &)
 {
     if (write_finalized)
-        path_map.emplace(path_from, path_map.extract(path_to).mapped());
+        path_map.emplace(path_from.parent_path(), path_map.extract(path_to.parent_path()).mapped());
 
     if (write_created)
     {
@@ -186,7 +186,7 @@ MetadataStorageFromPlainObjectStorageRemoveDirectoryOperation::MetadataStorageFr
 
 void MetadataStorageFromPlainObjectStorageRemoveDirectoryOperation::execute(std::unique_lock<SharedMutex> & /* metadata_lock */)
 {
-    auto path_it = path_map.find(path);
+    auto path_it = path_map.find(path.parent_path());
     if (path_it == path_map.end())
         return;
 
@@ -223,7 +223,7 @@ void MetadataStorageFromPlainObjectStorageRemoveDirectoryOperation::undo(std::un
     writeString(path.string(), *buf);
     buf->finalize();
 
-    path_map.emplace(path, std::move(key_prefix));
+    path_map.emplace(path.parent_path(), std::move(key_prefix));
     auto metric = object_storage->getMetadataStorageMetrics().directory_map_size;
     CurrentMetrics::add(metric, 1);
 }
diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp b/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp
index b904c0d92b9..ba8dfc891dd 100644
--- a/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp
+++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp
@@ -97,7 +97,7 @@ MetadataStorageFromPlainObjectStorage::PathMap loadPathPrefixMap(const std::stri
             std::pair<MetadataStorageFromPlainObjectStorage::PathMap::iterator, bool> res;
             {
                 std::lock_guard lock(mutex);
-                res = result.emplace(local_path, remote_path.parent_path());
+                res = result.emplace(std::filesystem::path(local_path).parent_path(), remote_path.parent_path());
             }
 
             /// This can happen if table replication is enabled, then the same local path is written
@@ -145,11 +145,10 @@ void getDirectChildrenOnDiskImpl(
                 break;
 
             auto slash_num = count(k.begin() + local_path.size(), k.end(), '/');
-            if (slash_num != 1)
+            if (slash_num != 0)
                 continue;
 
-            chassert(k.back() == '/');
-            remote_to_local_subdir.emplace(v, std::string(k.begin() + local_path.size(), k.end() - 1));
+            remote_to_local_subdir.emplace(v, std::string(k.begin() + local_path.size(), k.end()) + "/");
         }
     }
 
@@ -243,7 +242,7 @@ std::vector<std::string> MetadataStorageFromPlainRewritableObjectStorage::listDi
     object_storage->listObjects(abs_key, files, 0);
 
     std::unordered_set<std::string> directories;
-    getDirectChildrenOnDisk(abs_key, object_storage->getCommonKeyPrefix(), files, path, directories);
+    getDirectChildrenOnDisk(abs_key, object_storage->getCommonKeyPrefix(), files, std::filesystem::path(path) / "", directories);
     /// List empty directories that are identified by the `prefix.path` metadata files. This is required to, e.g., remove
     /// metadata along with regular files.
     if (object_storage->getCommonKeyPrefix() != getMetadataKeyPrefix())
diff --git a/src/Disks/ObjectStorages/PathComparator.h b/src/Disks/ObjectStorages/PathComparator.h
new file mode 100644
index 00000000000..fe97a465937
--- /dev/null
+++ b/src/Disks/ObjectStorages/PathComparator.h
@@ -0,0 +1,20 @@
+#pragma once
+
+#include <filesystem>
+
+namespace DB
+{
+// TODO: rename
+struct PathComparator
+{
+    bool operator()(const std::filesystem::path & path1, const std::filesystem::path & path2) const
+    {
+        auto d1 = std::distance(path1.begin(), path1.end());
+        auto d2 = std::distance(path2.begin(), path2.end());
+        if (d1 != d2)
+            return d1 < d2;
+        return path1 < path2;
+    }
+};
+
+}

From aa290b6398a5affa8405d3584795bce6bf7450d4 Mon Sep 17 00:00:00 2001
From: Julia Kartseva <yulia.kartseva@gmail.com>
Date: Wed, 10 Jul 2024 01:15:57 +0000
Subject: [PATCH 1217/1488] use a designated mutex for path_map

---
 .../CommonPathPrefixKeyGenerator.cpp          | 13 ++-
 .../CommonPathPrefixKeyGenerator.h            |  6 +-
 .../MetadataStorageFromPlainObjectStorage.h   |  6 +-
 ...torageFromPlainObjectStorageOperations.cpp | 99 ++++++++++++++-----
 ...aStorageFromPlainObjectStorageOperations.h | 13 +--
 ...torageFromPlainRewritableObjectStorage.cpp | 25 ++---
 ...aStorageFromPlainRewritableObjectStorage.h |  4 +-
 src/Disks/ObjectStorages/PathComparator.h     | 27 +++--
 8 files changed, 122 insertions(+), 71 deletions(-)

diff --git a/src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.cpp b/src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.cpp
index 062a2542654..19dd819fc17 100644
--- a/src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.cpp
+++ b/src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.cpp
@@ -1,4 +1,5 @@
 #include "CommonPathPrefixKeyGenerator.h"
+#include "Disks/ObjectStorages/PathComparator.h"
 
 #include <Common/getRandomASCIIString.h>
 
@@ -9,9 +10,8 @@
 namespace DB
 {
 
-CommonPathPrefixKeyGenerator::CommonPathPrefixKeyGenerator(
-    String key_prefix_, SharedMutex & shared_mutex_, std::weak_ptr<PathMap> path_map_)
-    : storage_key_prefix(key_prefix_), shared_mutex(shared_mutex_), path_map(std::move(path_map_))
+CommonPathPrefixKeyGenerator::CommonPathPrefixKeyGenerator(String key_prefix_, std::weak_ptr<InMemoryPathMap> path_map_)
+    : storage_key_prefix(key_prefix_), path_map(std::move(path_map_))
 {
 }
 
@@ -49,14 +49,13 @@ std::tuple<std::string, std::vector<std::string>> CommonPathPrefixKeyGenerator::
     std::filesystem::path p(path);
     std::deque<std::string> dq;
 
-    std::shared_lock lock(shared_mutex);
-
     auto ptr = path_map.lock();
+    std::shared_lock lock(ptr->mutex);
 
     while (p != p.root_path())
     {
-        auto it = ptr->find(p);
-        if (it != ptr->end())
+        auto it = ptr->map.find(p);
+        if (it != ptr->map.end())
         {
             std::vector<std::string> vec(std::make_move_iterator(dq.begin()), std::make_move_iterator(dq.end()));
             return std::make_tuple(it->second, std::move(vec));
diff --git a/src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.h b/src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.h
index bca4f7060c4..e337745b627 100644
--- a/src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.h
+++ b/src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.h
@@ -25,9 +25,8 @@ class CommonPathPrefixKeyGenerator : public IObjectStorageKeysGenerator
 {
 public:
     /// Local to remote path map. Leverages filesystem::path comparator for paths.
-    using PathMap = std::map<std::filesystem::path, std::string, PathComparator>;
 
-    explicit CommonPathPrefixKeyGenerator(String key_prefix_, SharedMutex & shared_mutex_, std::weak_ptr<PathMap> path_map_);
+    explicit CommonPathPrefixKeyGenerator(String key_prefix_, std::weak_ptr<InMemoryPathMap> path_map_);
 
     ObjectStorageKey generate(const String & path, bool is_directory, const std::optional<String> & key_prefix) const override;
 
@@ -37,8 +36,7 @@ private:
 
     const String storage_key_prefix;
 
-    SharedMutex & shared_mutex;
-    std::weak_ptr<PathMap> path_map;
+    std::weak_ptr<InMemoryPathMap> path_map;
 };
 
 }
diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.h b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.h
index 9ea1c475821..dfb9632666c 100644
--- a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.h
+++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.h
@@ -28,10 +28,6 @@ using UnlinkMetadataFileOperationOutcomePtr = std::shared_ptr<UnlinkMetadataFile
 /// structure as on disk MergeTree, and does not require metadata from a local disk to restore.
 class MetadataStorageFromPlainObjectStorage : public IMetadataStorage
 {
-public:
-    /// Local path prefixes mapped to storage key prefixes.
-    using PathMap = std::map<std::filesystem::path, std::string, PathComparator>;
-
 private:
     friend class MetadataStorageFromPlainObjectStorageTransaction;
 
@@ -86,7 +82,7 @@ protected:
     virtual std::string getMetadataKeyPrefix() const { return object_storage->getCommonKeyPrefix(); }
 
     /// Returns a map of local paths to paths in object storage.
-    virtual std::shared_ptr<PathMap> getPathMap() const { throwNotImplemented(); }
+    virtual std::shared_ptr<InMemoryPathMap> getPathMap() const { throwNotImplemented(); }
 };
 
 class MetadataStorageFromPlainObjectStorageTransaction final : public IMetadataTransaction, private MetadataOperationsHolder
diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp
index b0b384f62c7..b4a85efbaab 100644
--- a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp
+++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp
@@ -1,4 +1,5 @@
 #include "MetadataStorageFromPlainObjectStorageOperations.h"
+#include "Disks/ObjectStorages/PathComparator.h"
 
 #include <IO/ReadHelpers.h>
 #include <IO/WriteHelpers.h>
@@ -30,7 +31,7 @@ ObjectStorageKey createMetadataObjectKey(const std::string & key_prefix, const s
 MetadataStorageFromPlainObjectStorageCreateDirectoryOperation::MetadataStorageFromPlainObjectStorageCreateDirectoryOperation(
     std::filesystem::path && path_,
     std::string && key_prefix_,
-    MetadataStorageFromPlainObjectStorage::PathMap & path_map_,
+    InMemoryPathMap & path_map_,
     ObjectStoragePtr object_storage_,
     const std::string & metadata_key_prefix_)
     : path(std::move(path_))
@@ -43,8 +44,13 @@ MetadataStorageFromPlainObjectStorageCreateDirectoryOperation::MetadataStorageFr
 
 void MetadataStorageFromPlainObjectStorageCreateDirectoryOperation::execute(std::unique_lock<SharedMutex> &)
 {
-    if (path_map.contains(path.parent_path()))
-        return;
+    auto & map = path_map.map;
+    auto & mutex = path_map.mutex;
+    {
+        std::shared_lock lock(mutex);
+        if (map.contains(path.parent_path()))
+            return;
+    }
 
     auto metadata_object_key = createMetadataObjectKey(key_prefix, metadata_key_prefix);
 
@@ -64,8 +70,11 @@ void MetadataStorageFromPlainObjectStorageCreateDirectoryOperation::execute(std:
 
     write_created = true;
 
-    [[maybe_unused]] auto result = path_map.emplace(path.parent_path(), std::move(key_prefix));
-    chassert(result.second);
+    {
+        std::unique_lock lock(mutex);
+        [[maybe_unused]] auto result = map.emplace(path.parent_path(), std::move(key_prefix));
+        chassert(result.second);
+    }
     auto metric = object_storage->getMetadataStorageMetrics().directory_map_size;
     CurrentMetrics::add(metric, 1);
 
@@ -80,11 +89,17 @@ void MetadataStorageFromPlainObjectStorageCreateDirectoryOperation::execute(std:
 
 void MetadataStorageFromPlainObjectStorageCreateDirectoryOperation::undo(std::unique_lock<SharedMutex> &)
 {
+    auto & map = path_map.map;
+    auto & mutex = path_map.mutex;
+
     auto metadata_object_key = createMetadataObjectKey(key_prefix, metadata_key_prefix);
 
     if (write_finalized)
     {
-        path_map.erase(path.parent_path());
+        {
+            std::unique_lock lock(mutex);
+            map.erase(path.parent_path());
+        }
         auto metric = object_storage->getMetadataStorageMetrics().directory_map_size;
         CurrentMetrics::sub(metric, 1);
 
@@ -97,7 +112,7 @@ void MetadataStorageFromPlainObjectStorageCreateDirectoryOperation::undo(std::un
 MetadataStorageFromPlainObjectStorageMoveDirectoryOperation::MetadataStorageFromPlainObjectStorageMoveDirectoryOperation(
     std::filesystem::path && path_from_,
     std::filesystem::path && path_to_,
-    MetadataStorageFromPlainObjectStorage::PathMap & path_map_,
+    InMemoryPathMap & path_map_,
     ObjectStoragePtr object_storage_,
     const std::string & metadata_key_prefix_)
     : path_from(std::move(path_from_))
@@ -111,14 +126,25 @@ MetadataStorageFromPlainObjectStorageMoveDirectoryOperation::MetadataStorageFrom
 std::unique_ptr<WriteBufferFromFileBase> MetadataStorageFromPlainObjectStorageMoveDirectoryOperation::createWriteBuf(
     const std::filesystem::path & expected_path, const std::filesystem::path & new_path, bool validate_content)
 {
-    auto expected_it = path_map.find(expected_path.parent_path());
-    if (expected_it == path_map.end())
-        throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "Metadata object for the expected (source) path '{}' does not exist", expected_path);
+    auto & map = path_map.map;
+    auto & mutex = path_map.mutex;
 
-    if (path_map.contains(new_path.parent_path()))
-        throw Exception(ErrorCodes::FILE_ALREADY_EXISTS, "Metadata object for the new (destination) path '{}' already exists", new_path);
+    std::filesystem::path remote_path;
+    {
+        std::shared_lock lock(mutex);
+        auto expected_it = map.find(expected_path.parent_path());
+        if (expected_it == map.end())
+            throw Exception(
+                ErrorCodes::FILE_DOESNT_EXIST, "Metadata object for the expected (source) path '{}' does not exist", expected_path);
 
-    auto metadata_object_key = createMetadataObjectKey(expected_it->second, metadata_key_prefix);
+        if (map.contains(new_path.parent_path()))
+            throw Exception(
+                ErrorCodes::FILE_ALREADY_EXISTS, "Metadata object for the new (destination) path '{}' already exists", new_path);
+
+        remote_path = expected_it->second;
+    }
+
+    auto metadata_object_key = createMetadataObjectKey(remote_path, metadata_key_prefix);
 
     auto metadata_object = StoredObject(metadata_object_key.serialize(), expected_path / PREFIX_PATH_FILE_NAME);
 
@@ -156,8 +182,13 @@ void MetadataStorageFromPlainObjectStorageMoveDirectoryOperation::execute(std::u
     writeString(path_to.string(), *write_buf);
     write_buf->finalize();
 
-    [[maybe_unused]] auto result = path_map.emplace(path_to.parent_path(), path_map.extract(path_from.parent_path()).mapped());
-    chassert(result.second);
+    auto & map = path_map.map;
+    auto & mutex = path_map.mutex;
+    {
+        std::unique_lock lock(mutex);
+        [[maybe_unused]] auto result = map.emplace(path_to.parent_path(), map.extract(path_from.parent_path()).mapped());
+        chassert(result.second);
+    }
 
     write_finalized = true;
 }
@@ -165,7 +196,12 @@ void MetadataStorageFromPlainObjectStorageMoveDirectoryOperation::execute(std::u
 void MetadataStorageFromPlainObjectStorageMoveDirectoryOperation::undo(std::unique_lock<SharedMutex> &)
 {
     if (write_finalized)
-        path_map.emplace(path_from.parent_path(), path_map.extract(path_to.parent_path()).mapped());
+    {
+        auto & map = path_map.map;
+        auto & mutex = path_map.mutex;
+        std::unique_lock lock(mutex);
+        map.emplace(path_from.parent_path(), map.extract(path_to.parent_path()).mapped());
+    }
 
     if (write_created)
     {
@@ -176,28 +212,34 @@ void MetadataStorageFromPlainObjectStorageMoveDirectoryOperation::undo(std::uniq
 }
 
 MetadataStorageFromPlainObjectStorageRemoveDirectoryOperation::MetadataStorageFromPlainObjectStorageRemoveDirectoryOperation(
-    std::filesystem::path && path_,
-    MetadataStorageFromPlainObjectStorage::PathMap & path_map_,
-    ObjectStoragePtr object_storage_,
-    const std::string & metadata_key_prefix_)
+    std::filesystem::path && path_, InMemoryPathMap & path_map_, ObjectStoragePtr object_storage_, const std::string & metadata_key_prefix_)
     : path(std::move(path_)), path_map(path_map_), object_storage(object_storage_), metadata_key_prefix(metadata_key_prefix_)
 {
 }
 
 void MetadataStorageFromPlainObjectStorageRemoveDirectoryOperation::execute(std::unique_lock<SharedMutex> & /* metadata_lock */)
 {
-    auto path_it = path_map.find(path.parent_path());
-    if (path_it == path_map.end())
-        return;
+    auto & map = path_map.map;
+    auto & mutex = path_map.mutex;
+    {
+        std::shared_lock lock(mutex);
+        auto path_it = map.find(path.parent_path());
+        if (path_it == map.end())
+            return;
+        key_prefix = path_it->second;
+    }
 
     LOG_TRACE(getLogger("MetadataStorageFromPlainObjectStorageRemoveDirectoryOperation"), "Removing directory '{}'", path);
 
-    key_prefix = path_it->second;
     auto metadata_object_key = createMetadataObjectKey(key_prefix, metadata_key_prefix);
     auto metadata_object = StoredObject(metadata_object_key.serialize(), path / PREFIX_PATH_FILE_NAME);
     object_storage->removeObject(metadata_object);
 
-    path_map.erase(path_it);
+    {
+        std::unique_lock lock(mutex);
+        map.erase(path.parent_path());
+    }
+
     auto metric = object_storage->getMetadataStorageMetrics().directory_map_size;
     CurrentMetrics::sub(metric, 1);
 
@@ -223,7 +265,12 @@ void MetadataStorageFromPlainObjectStorageRemoveDirectoryOperation::undo(std::un
     writeString(path.string(), *buf);
     buf->finalize();
 
-    path_map.emplace(path.parent_path(), std::move(key_prefix));
+    auto & map = path_map.map;
+    auto & mutex = path_map.mutex;
+    {
+        std::unique_lock lock(mutex);
+        map.emplace(path.parent_path(), std::move(key_prefix));
+    }
     auto metric = object_storage->getMetadataStorageMetrics().directory_map_size;
     CurrentMetrics::add(metric, 1);
 }
diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.h b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.h
index e31e3cbb262..1b2471dd316 100644
--- a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.h
+++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.h
@@ -2,6 +2,7 @@
 
 #include <Disks/ObjectStorages/IMetadataOperation.h>
 #include <Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.h>
+#include "Disks/ObjectStorages/PathComparator.h"
 
 #include <filesystem>
 #include <map>
@@ -14,7 +15,7 @@ class MetadataStorageFromPlainObjectStorageCreateDirectoryOperation final : publ
 private:
     std::filesystem::path path;
     std::string key_prefix;
-    MetadataStorageFromPlainObjectStorage::PathMap & path_map;
+    InMemoryPathMap & path_map;
     ObjectStoragePtr object_storage;
     const std::string metadata_key_prefix;
 
@@ -26,7 +27,7 @@ public:
     MetadataStorageFromPlainObjectStorageCreateDirectoryOperation(
         std::filesystem::path && path_,
         std::string && key_prefix_,
-        MetadataStorageFromPlainObjectStorage::PathMap & path_map_,
+        InMemoryPathMap & path_map_,
         ObjectStoragePtr object_storage_,
         const std::string & metadata_key_prefix_);
 
@@ -39,7 +40,7 @@ class MetadataStorageFromPlainObjectStorageMoveDirectoryOperation final : public
 private:
     std::filesystem::path path_from;
     std::filesystem::path path_to;
-    MetadataStorageFromPlainObjectStorage::PathMap & path_map;
+    InMemoryPathMap & path_map;
     ObjectStoragePtr object_storage;
     const std::string metadata_key_prefix;
 
@@ -53,7 +54,7 @@ public:
     MetadataStorageFromPlainObjectStorageMoveDirectoryOperation(
         std::filesystem::path && path_from_,
         std::filesystem::path && path_to_,
-        MetadataStorageFromPlainObjectStorage::PathMap & path_map_,
+        InMemoryPathMap & path_map_,
         ObjectStoragePtr object_storage_,
         const std::string & metadata_key_prefix_);
 
@@ -67,7 +68,7 @@ class MetadataStorageFromPlainObjectStorageRemoveDirectoryOperation final : publ
 private:
     std::filesystem::path path;
 
-    MetadataStorageFromPlainObjectStorage::PathMap & path_map;
+    InMemoryPathMap & path_map;
     ObjectStoragePtr object_storage;
     const std::string metadata_key_prefix;
 
@@ -77,7 +78,7 @@ private:
 public:
     MetadataStorageFromPlainObjectStorageRemoveDirectoryOperation(
         std::filesystem::path && path_,
-        MetadataStorageFromPlainObjectStorage::PathMap & path_map_,
+        InMemoryPathMap & path_map_,
         ObjectStoragePtr object_storage_,
         const std::string & metadata_key_prefix_);
 
diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp b/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp
index ba8dfc891dd..cf51a6a5314 100644
--- a/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp
+++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp
@@ -3,11 +3,12 @@
 
 #include <unordered_set>
 #include <IO/ReadHelpers.h>
-#include <IO/SharedThreadPools.h>
 #include <IO/S3Common.h>
+#include <IO/SharedThreadPools.h>
 #include <Common/ErrorCodes.h>
 #include <Common/logger_useful.h>
 #include "CommonPathPrefixKeyGenerator.h"
+#include "Disks/ObjectStorages/PathComparator.h"
 
 
 namespace DB
@@ -37,9 +38,10 @@ std::string getMetadataKeyPrefix(ObjectStoragePtr object_storage)
         : metadata_key_prefix;
 }
 
-MetadataStorageFromPlainObjectStorage::PathMap loadPathPrefixMap(const std::string & metadata_key_prefix, ObjectStoragePtr object_storage)
+InMemoryPathMap::Map loadPathPrefixMap(const std::string & metadata_key_prefix, ObjectStoragePtr object_storage)
 {
-    MetadataStorageFromPlainObjectStorage::PathMap result;
+    using Map = InMemoryPathMap::Map;
+    Map result;
 
     ThreadPool & pool = getIOThreadPool().get();
     ThreadPoolCallbackRunnerLocal<void> runner(pool, "PlainRWMetaLoad");
@@ -94,7 +96,7 @@ MetadataStorageFromPlainObjectStorage::PathMap loadPathPrefixMap(const std::stri
             chassert(remote_metadata_path.string().starts_with(metadata_key_prefix));
             auto suffix = remote_metadata_path.string().substr(metadata_key_prefix.size());
             auto remote_path = std::filesystem::path(std::move(suffix));
-            std::pair<MetadataStorageFromPlainObjectStorage::PathMap::iterator, bool> res;
+            std::pair<Map::iterator, bool> res;
             {
                 std::lock_guard lock(mutex);
                 res = result.emplace(std::filesystem::path(local_path).parent_path(), remote_path.parent_path());
@@ -126,14 +128,13 @@ void getDirectChildrenOnDiskImpl(
     const std::string & storage_key_perfix,
     const RelativePathsWithMetadata & remote_paths,
     const std::string & local_path,
-    const MetadataStorageFromPlainObjectStorage::PathMap & local_path_prefixes,
+    const InMemoryPathMap::Map & local_path_prefixes,
     SharedMutex & shared_mutex,
     std::unordered_set<std::string> & result)
 {
-    using PathMap = MetadataStorageFromPlainObjectStorage::PathMap;
-
     /// Map remote paths into local subdirectories.
-    std::unordered_map<PathMap::mapped_type, PathMap::key_type> remote_to_local_subdir;
+    using Map = InMemoryPathMap::Map;
+    std::unordered_map<Map::mapped_type, Map::key_type> remote_to_local_subdir;
 
     {
         std::shared_lock lock(shared_mutex);
@@ -189,7 +190,7 @@ MetadataStorageFromPlainRewritableObjectStorage::MetadataStorageFromPlainRewrita
     ObjectStoragePtr object_storage_, String storage_path_prefix_)
     : MetadataStorageFromPlainObjectStorage(object_storage_, storage_path_prefix_)
     , metadata_key_prefix(DB::getMetadataKeyPrefix(object_storage))
-    , path_map(std::make_shared<PathMap>(loadPathPrefixMap(metadata_key_prefix, object_storage)))
+    , path_map(std::make_shared<InMemoryPathMap>(loadPathPrefixMap(metadata_key_prefix, object_storage)))
 {
     if (object_storage->isWriteOnce())
         throw Exception(
@@ -197,14 +198,14 @@ MetadataStorageFromPlainRewritableObjectStorage::MetadataStorageFromPlainRewrita
             "MetadataStorageFromPlainRewritableObjectStorage is not compatible with write-once storage '{}'",
             object_storage->getName());
 
-    auto keys_gen = std::make_shared<CommonPathPrefixKeyGenerator>(object_storage->getCommonKeyPrefix(), metadata_mutex, path_map);
+    auto keys_gen = std::make_shared<CommonPathPrefixKeyGenerator>(object_storage->getCommonKeyPrefix(), path_map);
     object_storage->setKeysGenerator(keys_gen);
 }
 
 MetadataStorageFromPlainRewritableObjectStorage::~MetadataStorageFromPlainRewritableObjectStorage()
 {
     auto metric = object_storage->getMetadataStorageMetrics().directory_map_size;
-    CurrentMetrics::sub(metric, path_map->size());
+    CurrentMetrics::sub(metric, path_map->map.size());
 }
 
 bool MetadataStorageFromPlainRewritableObjectStorage::exists(const std::string & path) const
@@ -263,7 +264,7 @@ void MetadataStorageFromPlainRewritableObjectStorage::getDirectChildrenOnDisk(
     const std::string & local_path,
     std::unordered_set<std::string> & result) const
 {
-    getDirectChildrenOnDiskImpl(storage_key, storage_key_perfix, remote_paths, local_path, *getPathMap(), metadata_mutex, result);
+    getDirectChildrenOnDiskImpl(storage_key, storage_key_perfix, remote_paths, local_path, getPathMap()->map, getPathMap()->mutex, result);
 }
 
 }
diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.h b/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.h
index b067b391878..fea461abab8 100644
--- a/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.h
+++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.h
@@ -13,7 +13,7 @@ class MetadataStorageFromPlainRewritableObjectStorage final : public MetadataSto
 {
 private:
     const std::string metadata_key_prefix;
-    std::shared_ptr<PathMap> path_map;
+    std::shared_ptr<InMemoryPathMap> path_map;
 
 public:
     MetadataStorageFromPlainRewritableObjectStorage(ObjectStoragePtr object_storage_, String storage_path_prefix_);
@@ -27,7 +27,7 @@ public:
 
 protected:
     std::string getMetadataKeyPrefix() const override { return metadata_key_prefix; }
-    std::shared_ptr<PathMap> getPathMap() const override { return path_map; }
+    std::shared_ptr<InMemoryPathMap> getPathMap() const override { return path_map; }
     void getDirectChildrenOnDisk(
         const std::string & storage_key,
         const std::string & storage_key_perfix,
diff --git a/src/Disks/ObjectStorages/PathComparator.h b/src/Disks/ObjectStorages/PathComparator.h
index fe97a465937..fae82108789 100644
--- a/src/Disks/ObjectStorages/PathComparator.h
+++ b/src/Disks/ObjectStorages/PathComparator.h
@@ -1,20 +1,29 @@
 #pragma once
 
 #include <filesystem>
+#include <map>
+#include "Common/SharedMutex.h"
 
 namespace DB
 {
-// TODO: rename
-struct PathComparator
+
+
+struct InMemoryPathMap
 {
-    bool operator()(const std::filesystem::path & path1, const std::filesystem::path & path2) const
+    struct PathComparator
     {
-        auto d1 = std::distance(path1.begin(), path1.end());
-        auto d2 = std::distance(path2.begin(), path2.end());
-        if (d1 != d2)
-            return d1 < d2;
-        return path1 < path2;
-    }
+        bool operator()(const std::filesystem::path & path1, const std::filesystem::path & path2) const
+        {
+            auto d1 = std::distance(path1.begin(), path1.end());
+            auto d2 = std::distance(path2.begin(), path2.end());
+            if (d1 != d2)
+                return d1 < d2;
+            return path1 < path2;
+        }
+    };
+    using Map = std::map<std::filesystem::path, std::string, PathComparator>;
+    Map map;
+    SharedMutex mutex;
 };
 
 }

From 0e78ed6b580646cc08721eef415ffb3fe2f697cb Mon Sep 17 00:00:00 2001
From: Julia Kartseva <yulia.kartseva@gmail.com>
Date: Wed, 10 Jul 2024 23:04:21 +0000
Subject: [PATCH 1218/1488] simplify listDirectory

---
 ...torageFromPlainRewritableObjectStorage.cpp | 28 ++++---------------
 ...aStorageFromPlainRewritableObjectStorage.h |  1 -
 2 files changed, 5 insertions(+), 24 deletions(-)

diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp b/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp
index cf51a6a5314..6a0eff0a136 100644
--- a/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp
+++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp
@@ -125,17 +125,12 @@ InMemoryPathMap::Map loadPathPrefixMap(const std::string & metadata_key_prefix,
 
 void getDirectChildrenOnDiskImpl(
     const std::string & storage_key,
-    const std::string & storage_key_perfix,
     const RelativePathsWithMetadata & remote_paths,
     const std::string & local_path,
     const InMemoryPathMap::Map & local_path_prefixes,
     SharedMutex & shared_mutex,
     std::unordered_set<std::string> & result)
 {
-    /// Map remote paths into local subdirectories.
-    using Map = InMemoryPathMap::Map;
-    std::unordered_map<Map::mapped_type, Map::key_type> remote_to_local_subdir;
-
     {
         std::shared_lock lock(shared_mutex);
         auto end_it = local_path_prefixes.end();
@@ -147,9 +142,9 @@ void getDirectChildrenOnDiskImpl(
 
             auto slash_num = count(k.begin() + local_path.size(), k.end(), '/');
             if (slash_num != 0)
-                continue;
+                break;
 
-            remote_to_local_subdir.emplace(v, std::string(k.begin() + local_path.size(), k.end()) + "/");
+            result.emplace(std::string(k.begin() + local_path.size(), k.end()) + "/");
         }
     }
 
@@ -169,18 +164,6 @@ void getDirectChildrenOnDiskImpl(
             if (!skip_list.contains(filename))
                 result.emplace(std::move(filename));
         }
-        else
-        {
-            /// Subdirectories.
-            chassert(path.find(storage_key_perfix) == 0);
-            auto it = remote_to_local_subdir.find(path.substr(storage_key_perfix.size(), slash_pos - storage_key_perfix.size()));
-            /// Mapped subdirectories.
-            if (it != remote_to_local_subdir.end())
-                result.emplace(it->second);
-            /// The remote subdirectory name is the same as the local subdirectory.
-            else
-                result.emplace(path.substr(child_pos, slash_pos - child_pos));
-        }
     }
 }
 
@@ -243,7 +226,7 @@ std::vector<std::string> MetadataStorageFromPlainRewritableObjectStorage::listDi
     object_storage->listObjects(abs_key, files, 0);
 
     std::unordered_set<std::string> directories;
-    getDirectChildrenOnDisk(abs_key, object_storage->getCommonKeyPrefix(), files, std::filesystem::path(path) / "", directories);
+    getDirectChildrenOnDisk(abs_key, files, std::filesystem::path(path) / "", directories);
     /// List empty directories that are identified by the `prefix.path` metadata files. This is required to, e.g., remove
     /// metadata along with regular files.
     if (object_storage->getCommonKeyPrefix() != getMetadataKeyPrefix())
@@ -251,7 +234,7 @@ std::vector<std::string> MetadataStorageFromPlainRewritableObjectStorage::listDi
         auto metadata_key = std::filesystem::path(getMetadataKeyPrefix()) / key_prefix / "";
         RelativePathsWithMetadata metadata_files;
         object_storage->listObjects(metadata_key, metadata_files, 0);
-        getDirectChildrenOnDisk(metadata_key, getMetadataKeyPrefix(), metadata_files, path, directories);
+        getDirectChildrenOnDisk(metadata_key, metadata_files, std::filesystem::path(path) / "", directories);
     }
 
     return std::vector<std::string>(std::make_move_iterator(directories.begin()), std::make_move_iterator(directories.end()));
@@ -259,12 +242,11 @@ std::vector<std::string> MetadataStorageFromPlainRewritableObjectStorage::listDi
 
 void MetadataStorageFromPlainRewritableObjectStorage::getDirectChildrenOnDisk(
     const std::string & storage_key,
-    const std::string & storage_key_perfix,
     const RelativePathsWithMetadata & remote_paths,
     const std::string & local_path,
     std::unordered_set<std::string> & result) const
 {
-    getDirectChildrenOnDiskImpl(storage_key, storage_key_perfix, remote_paths, local_path, getPathMap()->map, getPathMap()->mutex, result);
+    getDirectChildrenOnDiskImpl(storage_key, remote_paths, local_path, getPathMap()->map, getPathMap()->mutex, result);
 }
 
 }
diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.h b/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.h
index fea461abab8..8fd147e15b9 100644
--- a/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.h
+++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.h
@@ -30,7 +30,6 @@ protected:
     std::shared_ptr<InMemoryPathMap> getPathMap() const override { return path_map; }
     void getDirectChildrenOnDisk(
         const std::string & storage_key,
-        const std::string & storage_key_perfix,
         const RelativePathsWithMetadata & remote_paths,
         const std::string & local_path,
         std::unordered_set<std::string> & result) const;

From 82f5aceb484a322960065f973bec2b61c31219aa Mon Sep 17 00:00:00 2001
From: Julia Kartseva <yulia.kartseva@gmail.com>
Date: Thu, 11 Jul 2024 06:22:26 +0000
Subject: [PATCH 1219/1488] introduce flat structure

---
 .../FlatStructureKeyGenerator.cpp             | 51 +++++++++++++++++++
 .../FlatStructureKeyGenerator.h               | 23 +++++++++
 ...torageFromPlainRewritableObjectStorage.cpp | 15 ++++--
 3 files changed, 86 insertions(+), 3 deletions(-)
 create mode 100644 src/Disks/ObjectStorages/FlatStructureKeyGenerator.cpp
 create mode 100644 src/Disks/ObjectStorages/FlatStructureKeyGenerator.h

diff --git a/src/Disks/ObjectStorages/FlatStructureKeyGenerator.cpp b/src/Disks/ObjectStorages/FlatStructureKeyGenerator.cpp
new file mode 100644
index 00000000000..d6fb32b65d4
--- /dev/null
+++ b/src/Disks/ObjectStorages/FlatStructureKeyGenerator.cpp
@@ -0,0 +1,51 @@
+#include "FlatStructureKeyGenerator.h"
+#include "Common/ObjectStorageKey.h"
+#include "Common/SharedMutex.h"
+#include "Disks/ObjectStorages/PathComparator.h"
+#include <Common/getRandomASCIIString.h>
+
+#include <optional>
+#include <shared_mutex>
+#include <string>
+
+namespace DB
+{
+
+FlatStructureKeyGenerator::FlatStructureKeyGenerator(String storage_key_prefix_, std::weak_ptr<InMemoryPathMap> path_map_)
+    : storage_key_prefix(storage_key_prefix_), path_map(std::move(path_map_))
+{
+}
+
+ObjectStorageKey FlatStructureKeyGenerator::generate(const String & path, bool is_directory, const std::optional<String> & key_prefix) const
+{
+    if (is_directory)
+        chassert(path.ends_with('/'));
+
+    const auto p = std::filesystem::path(path);
+    auto directory = p.parent_path();
+
+    constexpr size_t part_size = 32;
+
+    std::optional<std::filesystem::path> remote_path;
+    {
+        auto ptr = path_map.lock();
+        std::shared_lock<SharedMutex> lock(ptr->mutex);
+        auto it = ptr->map.find(p);
+        if (it != ptr->map.end())
+            return ObjectStorageKey::createAsRelative(key_prefix.has_value() ? *key_prefix : storage_key_prefix, it->second);
+
+        it = ptr->map.find(directory);
+        if (it != ptr->map.end())
+            remote_path = it->second;
+    }
+    std::filesystem::path key = remote_path.has_value() ? *remote_path
+        : is_directory                                  ? std::filesystem::path(getRandomASCIIString(part_size))
+                                                        : directory;
+
+    if (!is_directory)
+        key /= p.filename();
+
+    return ObjectStorageKey::createAsRelative(key_prefix.has_value() ? *key_prefix : storage_key_prefix, key);
+}
+
+}
diff --git a/src/Disks/ObjectStorages/FlatStructureKeyGenerator.h b/src/Disks/ObjectStorages/FlatStructureKeyGenerator.h
new file mode 100644
index 00000000000..2c585dffb81
--- /dev/null
+++ b/src/Disks/ObjectStorages/FlatStructureKeyGenerator.h
@@ -0,0 +1,23 @@
+#pragma once
+
+#include <Disks/ObjectStorages/PathComparator.h>
+#include <Common/ObjectStorageKeyGenerator.h>
+
+#include <memory>
+namespace DB
+{
+
+class FlatStructureKeyGenerator : public IObjectStorageKeysGenerator
+{
+public:
+    explicit FlatStructureKeyGenerator(String storage_key_prefix_, std::weak_ptr<InMemoryPathMap> path_map_);
+
+    ObjectStorageKey generate(const String & path, bool is_directory, const std::optional<String> & key_prefix) const override;
+
+private:
+    const String storage_key_prefix;
+
+    std::weak_ptr<InMemoryPathMap> path_map;
+};
+
+}
diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp b/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp
index 6a0eff0a136..afaa7bf06ff 100644
--- a/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp
+++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp
@@ -1,5 +1,7 @@
 #include <Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.h>
+#include <Disks/ObjectStorages/FlatStructureKeyGenerator.h>
 #include <Disks/ObjectStorages/ObjectStorageIterator.h>
+#include <Disks/ObjectStorages/PathComparator.h>
 
 #include <unordered_set>
 #include <IO/ReadHelpers.h>
@@ -8,7 +10,6 @@
 #include <Common/ErrorCodes.h>
 #include <Common/logger_useful.h>
 #include "CommonPathPrefixKeyGenerator.h"
-#include "Disks/ObjectStorages/PathComparator.h"
 
 
 namespace DB
@@ -181,8 +182,16 @@ MetadataStorageFromPlainRewritableObjectStorage::MetadataStorageFromPlainRewrita
             "MetadataStorageFromPlainRewritableObjectStorage is not compatible with write-once storage '{}'",
             object_storage->getName());
 
-    auto keys_gen = std::make_shared<CommonPathPrefixKeyGenerator>(object_storage->getCommonKeyPrefix(), path_map);
-    object_storage->setKeysGenerator(keys_gen);
+    if (getMetadataKeyPrefix() == object_storage->getCommonKeyPrefix())
+    {
+        auto keys_gen = std::make_shared<CommonPathPrefixKeyGenerator>(object_storage->getCommonKeyPrefix(), path_map);
+        object_storage->setKeysGenerator(keys_gen);
+    }
+    else
+    {
+        auto keys_gen = std::make_shared<FlatStructureKeyGenerator>(object_storage->getCommonKeyPrefix(), path_map);
+        object_storage->setKeysGenerator(keys_gen);
+    }
 }
 
 MetadataStorageFromPlainRewritableObjectStorage::~MetadataStorageFromPlainRewritableObjectStorage()

From c0e6780dfe5977316e50e587877f3fe6ef11d048 Mon Sep 17 00:00:00 2001
From: Julia Kartseva <yulia.kartseva@gmail.com>
Date: Thu, 11 Jul 2024 22:48:07 +0000
Subject: [PATCH 1220/1488] rename PathComparator.h -> InMemoryPathMap.h

---
 src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.cpp    | 4 ++--
 src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.h      | 5 ++---
 src/Disks/ObjectStorages/FlatStructureKeyGenerator.cpp       | 2 +-
 src/Disks/ObjectStorages/FlatStructureKeyGenerator.h         | 2 +-
 .../ObjectStorages/{PathComparator.h => InMemoryPathMap.h}   | 0
 .../ObjectStorages/MetadataStorageFromPlainObjectStorage.cpp | 1 +
 .../ObjectStorages/MetadataStorageFromPlainObjectStorage.h   | 3 ++-
 .../MetadataStorageFromPlainObjectStorageOperations.cpp      | 2 +-
 .../MetadataStorageFromPlainObjectStorageOperations.h        | 2 +-
 .../MetadataStorageFromPlainRewritableObjectStorage.cpp      | 4 ++--
 10 files changed, 13 insertions(+), 12 deletions(-)
 rename src/Disks/ObjectStorages/{PathComparator.h => InMemoryPathMap.h} (100%)

diff --git a/src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.cpp b/src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.cpp
index 19dd819fc17..1fa06823bae 100644
--- a/src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.cpp
+++ b/src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.cpp
@@ -1,5 +1,5 @@
-#include "CommonPathPrefixKeyGenerator.h"
-#include "Disks/ObjectStorages/PathComparator.h"
+#include <Disks/ObjectStorages/CommonPathPrefixKeyGenerator.h>
+#include <Disks/ObjectStorages/InMemoryPathMap.h>
 
 #include <Common/getRandomASCIIString.h>
 
diff --git a/src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.h b/src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.h
index e337745b627..8b5037e3804 100644
--- a/src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.h
+++ b/src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.h
@@ -1,9 +1,6 @@
 #pragma once
 
 #include <Common/ObjectStorageKeyGenerator.h>
-#include <Common/SharedMutex.h>
-
-#include <Disks/ObjectStorages/PathComparator.h>
 
 #include <filesystem>
 #include <map>
@@ -21,6 +18,8 @@ namespace DB
 ///
 /// The key generator ensures that the original directory hierarchy is
 /// preserved, which is required for the MergeTree family.
+
+struct InMemoryPathMap;
 class CommonPathPrefixKeyGenerator : public IObjectStorageKeysGenerator
 {
 public:
diff --git a/src/Disks/ObjectStorages/FlatStructureKeyGenerator.cpp b/src/Disks/ObjectStorages/FlatStructureKeyGenerator.cpp
index d6fb32b65d4..414aea2b08b 100644
--- a/src/Disks/ObjectStorages/FlatStructureKeyGenerator.cpp
+++ b/src/Disks/ObjectStorages/FlatStructureKeyGenerator.cpp
@@ -1,7 +1,7 @@
 #include "FlatStructureKeyGenerator.h"
+#include <Disks/ObjectStorages/InMemoryPathMap.h>
 #include "Common/ObjectStorageKey.h"
 #include "Common/SharedMutex.h"
-#include "Disks/ObjectStorages/PathComparator.h"
 #include <Common/getRandomASCIIString.h>
 
 #include <optional>
diff --git a/src/Disks/ObjectStorages/FlatStructureKeyGenerator.h b/src/Disks/ObjectStorages/FlatStructureKeyGenerator.h
index 2c585dffb81..6b5b2203bed 100644
--- a/src/Disks/ObjectStorages/FlatStructureKeyGenerator.h
+++ b/src/Disks/ObjectStorages/FlatStructureKeyGenerator.h
@@ -1,12 +1,12 @@
 #pragma once
 
-#include <Disks/ObjectStorages/PathComparator.h>
 #include <Common/ObjectStorageKeyGenerator.h>
 
 #include <memory>
 namespace DB
 {
 
+struct InMemoryPathMap;
 class FlatStructureKeyGenerator : public IObjectStorageKeysGenerator
 {
 public:
diff --git a/src/Disks/ObjectStorages/PathComparator.h b/src/Disks/ObjectStorages/InMemoryPathMap.h
similarity index 100%
rename from src/Disks/ObjectStorages/PathComparator.h
rename to src/Disks/ObjectStorages/InMemoryPathMap.h
diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.cpp b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.cpp
index 7553c7733b5..364d04e2b52 100644
--- a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.cpp
+++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.cpp
@@ -1,5 +1,6 @@
 #include "MetadataStorageFromPlainObjectStorage.h"
 #include <Disks/IDisk.h>
+#include <Disks/ObjectStorages/InMemoryPathMap.h>
 #include <Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.h>
 #include <Disks/ObjectStorages/StaticDirectoryIterator.h>
 
diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.h b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.h
index dfb9632666c..a9a1a648f96 100644
--- a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.h
+++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.h
@@ -2,9 +2,9 @@
 
 #include <Disks/IDisk.h>
 #include <Disks/ObjectStorages/IMetadataStorage.h>
+#include <Disks/ObjectStorages/InMemoryPathMap.h>
 #include <Disks/ObjectStorages/MetadataOperationsHolder.h>
 #include <Disks/ObjectStorages/MetadataStorageTransactionState.h>
-#include <Disks/ObjectStorages/PathComparator.h>
 
 #include <map>
 #include <string>
@@ -13,6 +13,7 @@
 namespace DB
 {
 
+struct InMemoryPathMap;
 struct UnlinkMetadataFileOperationOutcome;
 using UnlinkMetadataFileOperationOutcomePtr = std::shared_ptr<UnlinkMetadataFileOperationOutcome>;
 
diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp
index b4a85efbaab..6f5109faec4 100644
--- a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp
+++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp
@@ -1,5 +1,5 @@
 #include "MetadataStorageFromPlainObjectStorageOperations.h"
-#include "Disks/ObjectStorages/PathComparator.h"
+#include <Disks/ObjectStorages/InMemoryPathMap.h>
 
 #include <IO/ReadHelpers.h>
 #include <IO/WriteHelpers.h>
diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.h b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.h
index 1b2471dd316..778585fa758 100644
--- a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.h
+++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.h
@@ -1,8 +1,8 @@
 #pragma once
 
 #include <Disks/ObjectStorages/IMetadataOperation.h>
+#include <Disks/ObjectStorages/InMemoryPathMap.h>
 #include <Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.h>
-#include "Disks/ObjectStorages/PathComparator.h"
 
 #include <filesystem>
 #include <map>
diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp b/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp
index afaa7bf06ff..c312eae4077 100644
--- a/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp
+++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp
@@ -1,7 +1,7 @@
-#include <Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.h>
 #include <Disks/ObjectStorages/FlatStructureKeyGenerator.h>
+#include <Disks/ObjectStorages/InMemoryPathMap.h>
+#include <Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.h>
 #include <Disks/ObjectStorages/ObjectStorageIterator.h>
-#include <Disks/ObjectStorages/PathComparator.h>
 
 #include <unordered_set>
 #include <IO/ReadHelpers.h>

From 3f066018fb0c74783ee486f54fc472ffd9cd7cc1 Mon Sep 17 00:00:00 2001
From: Julia Kartseva <yulia.kartseva@gmail.com>
Date: Fri, 12 Jul 2024 02:27:45 +0000
Subject: [PATCH 1221/1488] style and doc

---
 src/Common/ObjectStorageKeyGenerator.cpp      |  1 -
 src/Common/ObjectStorageKeyGenerator.h        |  4 ++++
 .../CommonPathPrefixKeyGenerator.h            |  1 +
 ...=> FlatDirectoryStructureKeyGenerator.cpp} |  9 ++++---
 ...h => FlatDirectoryStructureKeyGenerator.h} |  4 ++--
 .../ObjectStorages/HDFS/HDFSObjectStorage.cpp |  1 -
 src/Disks/ObjectStorages/InMemoryPathMap.h    |  2 +-
 .../Local/LocalObjectStorage.cpp              |  1 -
 .../MetadataStorageFromPlainObjectStorage.cpp |  2 --
 .../MetadataStorageFromPlainObjectStorage.h   |  5 ++--
 ...torageFromPlainObjectStorageOperations.cpp | 24 +++++++++----------
 ...aStorageFromPlainObjectStorageOperations.h |  3 +--
 ...torageFromPlainRewritableObjectStorage.cpp | 12 +++++++---
 13 files changed, 35 insertions(+), 34 deletions(-)
 rename src/Disks/ObjectStorages/{FlatStructureKeyGenerator.cpp => FlatDirectoryStructureKeyGenerator.cpp} (80%)
 rename src/Disks/ObjectStorages/{FlatStructureKeyGenerator.h => FlatDirectoryStructureKeyGenerator.h} (64%)

diff --git a/src/Common/ObjectStorageKeyGenerator.cpp b/src/Common/ObjectStorageKeyGenerator.cpp
index 3e7bf3116bd..3bdc0004198 100644
--- a/src/Common/ObjectStorageKeyGenerator.cpp
+++ b/src/Common/ObjectStorageKeyGenerator.cpp
@@ -3,7 +3,6 @@
 #include <Common/getRandomASCIIString.h>
 #include <Common/MatchGenerator.h>
 
-#include <optional>
 #include <fmt/format.h>
 
 
diff --git a/src/Common/ObjectStorageKeyGenerator.h b/src/Common/ObjectStorageKeyGenerator.h
index 12aeec1714d..008e3c88fac 100644
--- a/src/Common/ObjectStorageKeyGenerator.h
+++ b/src/Common/ObjectStorageKeyGenerator.h
@@ -1,6 +1,7 @@
 #pragma once
 
 #include <memory>
+#include <optional>
 #include "ObjectStorageKey.h"
 
 namespace DB
@@ -12,6 +13,9 @@ public:
     virtual ~IObjectStorageKeysGenerator() = default;
 
     /// Generates an object storage key based on a path in the virtual filesystem.
+    /// @param path         - Path in the virtual filesystem.
+    /// @param is_directory - If the path in the virtual filesystem corresponds to a directory.
+    /// @param key_prefix   - Optional key prefix for the generated object storage key. If provided, this prefix will be added to the beginning of the generated key.
     virtual ObjectStorageKey generate(const String & path, bool is_directory, const std::optional<String> & key_prefix) const = 0;
 };
 
diff --git a/src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.h b/src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.h
index 8b5037e3804..ea91d78600d 100644
--- a/src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.h
+++ b/src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.h
@@ -9,6 +9,7 @@
 namespace DB
 {
 
+/// Deprecated. Used for backward compatibility with plain rewritable disks without a separate metadata layout.
 /// Object storage key generator used specifically with the
 /// MetadataStorageFromPlainObjectStorage if multiple writes are allowed.
 
diff --git a/src/Disks/ObjectStorages/FlatStructureKeyGenerator.cpp b/src/Disks/ObjectStorages/FlatDirectoryStructureKeyGenerator.cpp
similarity index 80%
rename from src/Disks/ObjectStorages/FlatStructureKeyGenerator.cpp
rename to src/Disks/ObjectStorages/FlatDirectoryStructureKeyGenerator.cpp
index 414aea2b08b..64959b729b6 100644
--- a/src/Disks/ObjectStorages/FlatStructureKeyGenerator.cpp
+++ b/src/Disks/ObjectStorages/FlatDirectoryStructureKeyGenerator.cpp
@@ -1,4 +1,4 @@
-#include "FlatStructureKeyGenerator.h"
+#include "FlatDirectoryStructureKeyGenerator.h"
 #include <Disks/ObjectStorages/InMemoryPathMap.h>
 #include "Common/ObjectStorageKey.h"
 #include "Common/SharedMutex.h"
@@ -11,12 +11,12 @@
 namespace DB
 {
 
-FlatStructureKeyGenerator::FlatStructureKeyGenerator(String storage_key_prefix_, std::weak_ptr<InMemoryPathMap> path_map_)
+FlatDirectoryStructureKeyGenerator::FlatDirectoryStructureKeyGenerator(String storage_key_prefix_, std::weak_ptr<InMemoryPathMap> path_map_)
     : storage_key_prefix(storage_key_prefix_), path_map(std::move(path_map_))
 {
 }
 
-ObjectStorageKey FlatStructureKeyGenerator::generate(const String & path, bool is_directory, const std::optional<String> & key_prefix) const
+ObjectStorageKey FlatDirectoryStructureKeyGenerator::generate(const String & path, bool is_directory, const std::optional<String> & key_prefix) const
 {
     if (is_directory)
         chassert(path.ends_with('/'));
@@ -24,8 +24,6 @@ ObjectStorageKey FlatStructureKeyGenerator::generate(const String & path, bool i
     const auto p = std::filesystem::path(path);
     auto directory = p.parent_path();
 
-    constexpr size_t part_size = 32;
-
     std::optional<std::filesystem::path> remote_path;
     {
         auto ptr = path_map.lock();
@@ -38,6 +36,7 @@ ObjectStorageKey FlatStructureKeyGenerator::generate(const String & path, bool i
         if (it != ptr->map.end())
             remote_path = it->second;
     }
+    constexpr size_t part_size = 32;
     std::filesystem::path key = remote_path.has_value() ? *remote_path
         : is_directory                                  ? std::filesystem::path(getRandomASCIIString(part_size))
                                                         : directory;
diff --git a/src/Disks/ObjectStorages/FlatStructureKeyGenerator.h b/src/Disks/ObjectStorages/FlatDirectoryStructureKeyGenerator.h
similarity index 64%
rename from src/Disks/ObjectStorages/FlatStructureKeyGenerator.h
rename to src/Disks/ObjectStorages/FlatDirectoryStructureKeyGenerator.h
index 6b5b2203bed..4dbac5d3003 100644
--- a/src/Disks/ObjectStorages/FlatStructureKeyGenerator.h
+++ b/src/Disks/ObjectStorages/FlatDirectoryStructureKeyGenerator.h
@@ -7,10 +7,10 @@ namespace DB
 {
 
 struct InMemoryPathMap;
-class FlatStructureKeyGenerator : public IObjectStorageKeysGenerator
+class FlatDirectoryStructureKeyGenerator : public IObjectStorageKeysGenerator
 {
 public:
-    explicit FlatStructureKeyGenerator(String storage_key_prefix_, std::weak_ptr<InMemoryPathMap> path_map_);
+    explicit FlatDirectoryStructureKeyGenerator(String storage_key_prefix_, std::weak_ptr<InMemoryPathMap> path_map_);
 
     ObjectStorageKey generate(const String & path, bool is_directory, const std::optional<String> & key_prefix) const override;
 
diff --git a/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.cpp b/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.cpp
index 3ce2a0f4903..00ef4b63e6f 100644
--- a/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.cpp
+++ b/src/Disks/ObjectStorages/HDFS/HDFSObjectStorage.cpp
@@ -4,7 +4,6 @@
 #include <Storages/ObjectStorage/HDFS/WriteBufferFromHDFS.h>
 #include <Storages/ObjectStorage/HDFS/HDFSCommon.h>
 
-#include <optional>
 #include <Disks/IO/ReadBufferFromRemoteFSGather.h>
 #include <Storages/ObjectStorage/HDFS/ReadBufferFromHDFS.h>
 #include <Common/getRandomASCIIString.h>
diff --git a/src/Disks/ObjectStorages/InMemoryPathMap.h b/src/Disks/ObjectStorages/InMemoryPathMap.h
index fae82108789..ea08784719e 100644
--- a/src/Disks/ObjectStorages/InMemoryPathMap.h
+++ b/src/Disks/ObjectStorages/InMemoryPathMap.h
@@ -2,7 +2,7 @@
 
 #include <filesystem>
 #include <map>
-#include "Common/SharedMutex.h"
+#include <Common/SharedMutex.h>
 
 namespace DB
 {
diff --git a/src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp b/src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp
index 20ef135cdf7..5b61c57ca21 100644
--- a/src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp
+++ b/src/Disks/ObjectStorages/Local/LocalObjectStorage.cpp
@@ -1,7 +1,6 @@
 #include <Disks/ObjectStorages/Local/LocalObjectStorage.h>
 
 #include <filesystem>
-#include <optional>
 #include <Disks/IO/AsynchronousBoundedReadBuffer.h>
 #include <Disks/IO/ReadBufferFromRemoteFSGather.h>
 #include <Disks/IO/createReadBufferFromFileBase.h>
diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.cpp b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.cpp
index 364d04e2b52..2036208c389 100644
--- a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.cpp
+++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.cpp
@@ -146,10 +146,8 @@ void MetadataStorageFromPlainObjectStorageTransaction::createDirectory(const std
         return;
 
     auto normalized_path = normalizeDirectoryPath(path);
-    auto key_prefix = object_storage->generateObjectKeyPrefixForDirectoryPath(normalized_path, "" /* key_prefix */).serialize();
     auto op = std::make_unique<MetadataStorageFromPlainObjectStorageCreateDirectoryOperation>(
         std::move(normalized_path),
-        std::move(key_prefix),
         *metadata_storage.getPathMap(),
         object_storage,
         metadata_storage.getMetadataKeyPrefix());
diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.h b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.h
index a9a1a648f96..2aac7158bd5 100644
--- a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.h
+++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorage.h
@@ -78,11 +78,10 @@ public:
     bool supportsStat() const override { return false; }
 
 protected:
-    /// Get the object storage prefix for storing metadata files. If stored behind a separate endpoint,
-    /// the metadata keys reflect the layout of the regular files.
+    /// Get the object storage prefix for storing metadata files.
     virtual std::string getMetadataKeyPrefix() const { return object_storage->getCommonKeyPrefix(); }
 
-    /// Returns a map of local paths to paths in object storage.
+    /// Returns a map of virtual filesystem paths to paths in the object storage.
     virtual std::shared_ptr<InMemoryPathMap> getPathMap() const { throwNotImplemented(); }
 };
 
diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp
index 6f5109faec4..9e18f6cdb08 100644
--- a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp
+++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp
@@ -29,25 +29,21 @@ ObjectStorageKey createMetadataObjectKey(const std::string & key_prefix, const s
 }
 
 MetadataStorageFromPlainObjectStorageCreateDirectoryOperation::MetadataStorageFromPlainObjectStorageCreateDirectoryOperation(
-    std::filesystem::path && path_,
-    std::string && key_prefix_,
-    InMemoryPathMap & path_map_,
-    ObjectStoragePtr object_storage_,
-    const std::string & metadata_key_prefix_)
+    std::filesystem::path && path_, InMemoryPathMap & path_map_, ObjectStoragePtr object_storage_, const std::string & metadata_key_prefix_)
     : path(std::move(path_))
-    , key_prefix(key_prefix_)
     , path_map(path_map_)
     , object_storage(object_storage_)
     , metadata_key_prefix(metadata_key_prefix_)
+    , key_prefix(object_storage->generateObjectKeyPrefixForDirectoryPath(path, "" /* key_prefix */).serialize())
 {
 }
 
 void MetadataStorageFromPlainObjectStorageCreateDirectoryOperation::execute(std::unique_lock<SharedMutex> &)
 {
-    auto & map = path_map.map;
     auto & mutex = path_map.mutex;
     {
         std::shared_lock lock(mutex);
+        auto & map = path_map.map;
         if (map.contains(path.parent_path()))
             return;
     }
@@ -72,6 +68,7 @@ void MetadataStorageFromPlainObjectStorageCreateDirectoryOperation::execute(std:
 
     {
         std::unique_lock lock(mutex);
+        auto & map = path_map.map;
         [[maybe_unused]] auto result = map.emplace(path.parent_path(), std::move(key_prefix));
         chassert(result.second);
     }
@@ -89,7 +86,6 @@ void MetadataStorageFromPlainObjectStorageCreateDirectoryOperation::execute(std:
 
 void MetadataStorageFromPlainObjectStorageCreateDirectoryOperation::undo(std::unique_lock<SharedMutex> &)
 {
-    auto & map = path_map.map;
     auto & mutex = path_map.mutex;
 
     auto metadata_object_key = createMetadataObjectKey(key_prefix, metadata_key_prefix);
@@ -98,6 +94,7 @@ void MetadataStorageFromPlainObjectStorageCreateDirectoryOperation::undo(std::un
     {
         {
             std::unique_lock lock(mutex);
+            auto & map = path_map.map;
             map.erase(path.parent_path());
         }
         auto metric = object_storage->getMetadataStorageMetrics().directory_map_size;
@@ -126,12 +123,12 @@ MetadataStorageFromPlainObjectStorageMoveDirectoryOperation::MetadataStorageFrom
 std::unique_ptr<WriteBufferFromFileBase> MetadataStorageFromPlainObjectStorageMoveDirectoryOperation::createWriteBuf(
     const std::filesystem::path & expected_path, const std::filesystem::path & new_path, bool validate_content)
 {
-    auto & map = path_map.map;
     auto & mutex = path_map.mutex;
 
     std::filesystem::path remote_path;
     {
         std::shared_lock lock(mutex);
+        auto & map = path_map.map;
         auto expected_it = map.find(expected_path.parent_path());
         if (expected_it == map.end())
             throw Exception(
@@ -182,10 +179,10 @@ void MetadataStorageFromPlainObjectStorageMoveDirectoryOperation::execute(std::u
     writeString(path_to.string(), *write_buf);
     write_buf->finalize();
 
-    auto & map = path_map.map;
     auto & mutex = path_map.mutex;
     {
         std::unique_lock lock(mutex);
+        auto & map = path_map.map;
         [[maybe_unused]] auto result = map.emplace(path_to.parent_path(), map.extract(path_from.parent_path()).mapped());
         chassert(result.second);
     }
@@ -197,9 +194,9 @@ void MetadataStorageFromPlainObjectStorageMoveDirectoryOperation::undo(std::uniq
 {
     if (write_finalized)
     {
-        auto & map = path_map.map;
         auto & mutex = path_map.mutex;
         std::unique_lock lock(mutex);
+        auto & map = path_map.map;
         map.emplace(path_from.parent_path(), map.extract(path_to.parent_path()).mapped());
     }
 
@@ -219,10 +216,10 @@ MetadataStorageFromPlainObjectStorageRemoveDirectoryOperation::MetadataStorageFr
 
 void MetadataStorageFromPlainObjectStorageRemoveDirectoryOperation::execute(std::unique_lock<SharedMutex> & /* metadata_lock */)
 {
-    auto & map = path_map.map;
     auto & mutex = path_map.mutex;
     {
         std::shared_lock lock(mutex);
+        auto & map = path_map.map;
         auto path_it = map.find(path.parent_path());
         if (path_it == map.end())
             return;
@@ -237,6 +234,7 @@ void MetadataStorageFromPlainObjectStorageRemoveDirectoryOperation::execute(std:
 
     {
         std::unique_lock lock(mutex);
+        auto & map = path_map.map;
         map.erase(path.parent_path());
     }
 
@@ -265,10 +263,10 @@ void MetadataStorageFromPlainObjectStorageRemoveDirectoryOperation::undo(std::un
     writeString(path.string(), *buf);
     buf->finalize();
 
-    auto & map = path_map.map;
     auto & mutex = path_map.mutex;
     {
         std::unique_lock lock(mutex);
+        auto & map = path_map.map;
         map.emplace(path.parent_path(), std::move(key_prefix));
     }
     auto metric = object_storage->getMetadataStorageMetrics().directory_map_size;
diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.h b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.h
index 778585fa758..3ac0ffef8d2 100644
--- a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.h
+++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.h
@@ -14,10 +14,10 @@ class MetadataStorageFromPlainObjectStorageCreateDirectoryOperation final : publ
 {
 private:
     std::filesystem::path path;
-    std::string key_prefix;
     InMemoryPathMap & path_map;
     ObjectStoragePtr object_storage;
     const std::string metadata_key_prefix;
+    const std::string key_prefix;
 
     bool write_created = false;
     bool write_finalized = false;
@@ -26,7 +26,6 @@ public:
     // Assuming that paths are normalized.
     MetadataStorageFromPlainObjectStorageCreateDirectoryOperation(
         std::filesystem::path && path_,
-        std::string && key_prefix_,
         InMemoryPathMap & path_map_,
         ObjectStoragePtr object_storage_,
         const std::string & metadata_key_prefix_);
diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp b/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp
index c312eae4077..fd3b9523df6 100644
--- a/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp
+++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp
@@ -1,4 +1,4 @@
-#include <Disks/ObjectStorages/FlatStructureKeyGenerator.h>
+#include <Disks/ObjectStorages/FlatDirectoryStructureKeyGenerator.h>
 #include <Disks/ObjectStorages/InMemoryPathMap.h>
 #include <Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.h>
 #include <Disks/ObjectStorages/ObjectStorageIterator.h>
@@ -132,16 +132,20 @@ void getDirectChildrenOnDiskImpl(
     SharedMutex & shared_mutex,
     std::unordered_set<std::string> & result)
 {
+    /// Directories are retrieved from the in-memory path map.
     {
         std::shared_lock lock(shared_mutex);
         auto end_it = local_path_prefixes.end();
         for (auto it = local_path_prefixes.lower_bound(local_path); it != end_it; ++it)
         {
-            const auto & [k, v] = std::make_tuple(it->first.string(), it->second);
+            const auto & [k, _] = std::make_tuple(it->first.string(), it->second);
             if (!k.starts_with(local_path))
                 break;
 
             auto slash_num = count(k.begin() + local_path.size(), k.end(), '/');
+            /// The local_path_prefixes comparator ensures that the paths with the smallest number of
+            /// hops from the local_path are iterated first. The paths do not end with '/', hence
+            /// break the loop if the number of slashes is greater than 0.
             if (slash_num != 0)
                 break;
 
@@ -149,6 +153,7 @@ void getDirectChildrenOnDiskImpl(
         }
     }
 
+    /// Files.
     auto skip_list = std::set<std::string>{PREFIX_PATH_FILE_NAME};
     for (const auto & elem : remote_paths)
     {
@@ -189,7 +194,8 @@ MetadataStorageFromPlainRewritableObjectStorage::MetadataStorageFromPlainRewrita
     }
     else
     {
-        auto keys_gen = std::make_shared<FlatStructureKeyGenerator>(object_storage->getCommonKeyPrefix(), path_map);
+        /// Use flat directory structure if the metadata is stored separately from the table data.
+        auto keys_gen = std::make_shared<FlatDirectoryStructureKeyGenerator>(object_storage->getCommonKeyPrefix(), path_map);
         object_storage->setKeysGenerator(keys_gen);
     }
 }

From d4c13714abb6b307c4344c74bd4b7973c03e68df Mon Sep 17 00:00:00 2001
From: Julia Kartseva <yulia.kartseva@gmail.com>
Date: Sat, 13 Jul 2024 06:51:02 +0000
Subject: [PATCH 1222/1488] address feedback: TSA_GUARDED_BY

---
 .../CommonPathPrefixKeyGenerator.cpp          |  5 +--
 .../FlatDirectoryStructureKeyGenerator.cpp    |  7 ++--
 src/Disks/ObjectStorages/InMemoryPathMap.h    |  4 +--
 ...torageFromPlainObjectStorageOperations.cpp | 34 +++++++------------
 ...torageFromPlainRewritableObjectStorage.cpp | 34 +++++++++++--------
 5 files changed, 40 insertions(+), 44 deletions(-)

diff --git a/src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.cpp b/src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.cpp
index 1fa06823bae..1d041626a7e 100644
--- a/src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.cpp
+++ b/src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.cpp
@@ -2,6 +2,7 @@
 #include <Disks/ObjectStorages/InMemoryPathMap.h>
 
 #include <Common/getRandomASCIIString.h>
+#include <Common/SharedLockGuard.h>
 
 #include <deque>
 #include <filesystem>
@@ -49,8 +50,8 @@ std::tuple<std::string, std::vector<std::string>> CommonPathPrefixKeyGenerator::
     std::filesystem::path p(path);
     std::deque<std::string> dq;
 
-    auto ptr = path_map.lock();
-    std::shared_lock lock(ptr->mutex);
+    const auto ptr = path_map.lock();
+    SharedLockGuard lock(ptr->mutex);
 
     while (p != p.root_path())
     {
diff --git a/src/Disks/ObjectStorages/FlatDirectoryStructureKeyGenerator.cpp b/src/Disks/ObjectStorages/FlatDirectoryStructureKeyGenerator.cpp
index 64959b729b6..0f35bfd2427 100644
--- a/src/Disks/ObjectStorages/FlatDirectoryStructureKeyGenerator.cpp
+++ b/src/Disks/ObjectStorages/FlatDirectoryStructureKeyGenerator.cpp
@@ -1,7 +1,8 @@
 #include "FlatDirectoryStructureKeyGenerator.h"
 #include <Disks/ObjectStorages/InMemoryPathMap.h>
 #include "Common/ObjectStorageKey.h"
-#include "Common/SharedMutex.h"
+#include <Common/SharedLockGuard.h>
+#include <Common/SharedMutex.h>
 #include <Common/getRandomASCIIString.h>
 
 #include <optional>
@@ -26,8 +27,8 @@ ObjectStorageKey FlatDirectoryStructureKeyGenerator::generate(const String & pat
 
     std::optional<std::filesystem::path> remote_path;
     {
-        auto ptr = path_map.lock();
-        std::shared_lock<SharedMutex> lock(ptr->mutex);
+        const auto ptr = path_map.lock();
+        SharedLockGuard lock(ptr->mutex);
         auto it = ptr->map.find(p);
         if (it != ptr->map.end())
             return ObjectStorageKey::createAsRelative(key_prefix.has_value() ? *key_prefix : storage_key_prefix, it->second);
diff --git a/src/Disks/ObjectStorages/InMemoryPathMap.h b/src/Disks/ObjectStorages/InMemoryPathMap.h
index ea08784719e..dcd28dfaf6c 100644
--- a/src/Disks/ObjectStorages/InMemoryPathMap.h
+++ b/src/Disks/ObjectStorages/InMemoryPathMap.h
@@ -22,8 +22,8 @@ struct InMemoryPathMap
         }
     };
     using Map = std::map<std::filesystem::path, std::string, PathComparator>;
-    Map map;
-    SharedMutex mutex;
+    mutable SharedMutex mutex;
+    Map map TSA_GUARDED_BY(mutex);
 };
 
 }
diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp
index 9e18f6cdb08..8a06b204cfc 100644
--- a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp
+++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp
@@ -1,4 +1,5 @@
 #include "MetadataStorageFromPlainObjectStorageOperations.h"
+#include "Common/SharedLockGuard.h"
 #include <Disks/ObjectStorages/InMemoryPathMap.h>
 
 #include <IO/ReadHelpers.h>
@@ -40,11 +41,9 @@ MetadataStorageFromPlainObjectStorageCreateDirectoryOperation::MetadataStorageFr
 
 void MetadataStorageFromPlainObjectStorageCreateDirectoryOperation::execute(std::unique_lock<SharedMutex> &)
 {
-    auto & mutex = path_map.mutex;
     {
-        std::shared_lock lock(mutex);
-        auto & map = path_map.map;
-        if (map.contains(path.parent_path()))
+        SharedLockGuard lock(path_map.mutex);
+        if (path_map.map.contains(path.parent_path()))
             return;
     }
 
@@ -67,7 +66,7 @@ void MetadataStorageFromPlainObjectStorageCreateDirectoryOperation::execute(std:
     write_created = true;
 
     {
-        std::unique_lock lock(mutex);
+        std::lock_guard lock(path_map.mutex);
         auto & map = path_map.map;
         [[maybe_unused]] auto result = map.emplace(path.parent_path(), std::move(key_prefix));
         chassert(result.second);
@@ -86,16 +85,13 @@ void MetadataStorageFromPlainObjectStorageCreateDirectoryOperation::execute(std:
 
 void MetadataStorageFromPlainObjectStorageCreateDirectoryOperation::undo(std::unique_lock<SharedMutex> &)
 {
-    auto & mutex = path_map.mutex;
-
     auto metadata_object_key = createMetadataObjectKey(key_prefix, metadata_key_prefix);
 
     if (write_finalized)
     {
         {
-            std::unique_lock lock(mutex);
-            auto & map = path_map.map;
-            map.erase(path.parent_path());
+            std::lock_guard lock(path_map.mutex);
+            path_map.map.erase(path.parent_path());
         }
         auto metric = object_storage->getMetadataStorageMetrics().directory_map_size;
         CurrentMetrics::sub(metric, 1);
@@ -123,11 +119,9 @@ MetadataStorageFromPlainObjectStorageMoveDirectoryOperation::MetadataStorageFrom
 std::unique_ptr<WriteBufferFromFileBase> MetadataStorageFromPlainObjectStorageMoveDirectoryOperation::createWriteBuf(
     const std::filesystem::path & expected_path, const std::filesystem::path & new_path, bool validate_content)
 {
-    auto & mutex = path_map.mutex;
-
     std::filesystem::path remote_path;
     {
-        std::shared_lock lock(mutex);
+        SharedLockGuard lock(path_map.mutex);
         auto & map = path_map.map;
         auto expected_it = map.find(expected_path.parent_path());
         if (expected_it == map.end())
@@ -179,9 +173,8 @@ void MetadataStorageFromPlainObjectStorageMoveDirectoryOperation::execute(std::u
     writeString(path_to.string(), *write_buf);
     write_buf->finalize();
 
-    auto & mutex = path_map.mutex;
     {
-        std::unique_lock lock(mutex);
+        std::lock_guard lock(path_map.mutex);
         auto & map = path_map.map;
         [[maybe_unused]] auto result = map.emplace(path_to.parent_path(), map.extract(path_from.parent_path()).mapped());
         chassert(result.second);
@@ -194,8 +187,7 @@ void MetadataStorageFromPlainObjectStorageMoveDirectoryOperation::undo(std::uniq
 {
     if (write_finalized)
     {
-        auto & mutex = path_map.mutex;
-        std::unique_lock lock(mutex);
+        std::lock_guard lock(path_map.mutex);
         auto & map = path_map.map;
         map.emplace(path_from.parent_path(), map.extract(path_to.parent_path()).mapped());
     }
@@ -216,9 +208,8 @@ MetadataStorageFromPlainObjectStorageRemoveDirectoryOperation::MetadataStorageFr
 
 void MetadataStorageFromPlainObjectStorageRemoveDirectoryOperation::execute(std::unique_lock<SharedMutex> & /* metadata_lock */)
 {
-    auto & mutex = path_map.mutex;
     {
-        std::shared_lock lock(mutex);
+        SharedLockGuard lock(path_map.mutex);
         auto & map = path_map.map;
         auto path_it = map.find(path.parent_path());
         if (path_it == map.end())
@@ -233,7 +224,7 @@ void MetadataStorageFromPlainObjectStorageRemoveDirectoryOperation::execute(std:
     object_storage->removeObject(metadata_object);
 
     {
-        std::unique_lock lock(mutex);
+        std::lock_guard lock(path_map.mutex);
         auto & map = path_map.map;
         map.erase(path.parent_path());
     }
@@ -263,9 +254,8 @@ void MetadataStorageFromPlainObjectStorageRemoveDirectoryOperation::undo(std::un
     writeString(path.string(), *buf);
     buf->finalize();
 
-    auto & mutex = path_map.mutex;
     {
-        std::unique_lock lock(mutex);
+        std::lock_guard lock(path_map.mutex);
         auto & map = path_map.map;
         map.emplace(path.parent_path(), std::move(key_prefix));
     }
diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp b/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp
index fd3b9523df6..22e73e36372 100644
--- a/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp
+++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp
@@ -9,6 +9,8 @@
 #include <IO/SharedThreadPools.h>
 #include <Common/ErrorCodes.h>
 #include <Common/logger_useful.h>
+#include "Common/SharedLockGuard.h"
+#include "Common/SharedMutex.h"
 #include "CommonPathPrefixKeyGenerator.h"
 
 
@@ -39,14 +41,13 @@ std::string getMetadataKeyPrefix(ObjectStoragePtr object_storage)
         : metadata_key_prefix;
 }
 
-InMemoryPathMap::Map loadPathPrefixMap(const std::string & metadata_key_prefix, ObjectStoragePtr object_storage)
+std::shared_ptr<InMemoryPathMap> loadPathPrefixMap(const std::string & metadata_key_prefix, ObjectStoragePtr object_storage)
 {
+    auto result = std::make_shared<InMemoryPathMap>();
     using Map = InMemoryPathMap::Map;
-    Map result;
 
     ThreadPool & pool = getIOThreadPool().get();
     ThreadPoolCallbackRunnerLocal<void> runner(pool, "PlainRWMetaLoad");
-    std::mutex mutex;
 
     LoggerPtr log = getLogger("MetadataStorageFromPlainObjectStorage");
 
@@ -66,7 +67,7 @@ InMemoryPathMap::Map loadPathPrefixMap(const std::string & metadata_key_prefix,
         if (remote_metadata_path.filename() != PREFIX_PATH_FILE_NAME)
             continue;
 
-        runner([remote_metadata_path, path, &object_storage, &result, &mutex, &log, &settings, &metadata_key_prefix]
+        runner([remote_metadata_path, path, &object_storage, &result, &log, &settings, &metadata_key_prefix]
         {
             setThreadName("PlainRWMetaLoad");
 
@@ -99,8 +100,8 @@ InMemoryPathMap::Map loadPathPrefixMap(const std::string & metadata_key_prefix,
             auto remote_path = std::filesystem::path(std::move(suffix));
             std::pair<Map::iterator, bool> res;
             {
-                std::lock_guard lock(mutex);
-                res = result.emplace(std::filesystem::path(local_path).parent_path(), remote_path.parent_path());
+                std::lock_guard lock(result->mutex);
+                res = result->map.emplace(std::filesystem::path(local_path).parent_path(), remote_path.parent_path());
             }
 
             /// This can happen if table replication is enabled, then the same local path is written
@@ -117,10 +118,13 @@ InMemoryPathMap::Map loadPathPrefixMap(const std::string & metadata_key_prefix,
     }
 
     runner.waitForAllToFinishAndRethrowFirstError();
-    LOG_DEBUG(log, "Loaded metadata for {} files, found {} directories", num_files, result.size());
+    {
+        SharedLockGuard lock(result->mutex);
+        LOG_DEBUG(log, "Loaded metadata for {} files, found {} directories", num_files, result->map.size());
 
-    auto metric = object_storage->getMetadataStorageMetrics().directory_map_size;
-    CurrentMetrics::add(metric, result.size());
+        auto metric = object_storage->getMetadataStorageMetrics().directory_map_size;
+        CurrentMetrics::add(metric, result->map.size());
+    }
     return result;
 }
 
@@ -128,14 +132,14 @@ void getDirectChildrenOnDiskImpl(
     const std::string & storage_key,
     const RelativePathsWithMetadata & remote_paths,
     const std::string & local_path,
-    const InMemoryPathMap::Map & local_path_prefixes,
-    SharedMutex & shared_mutex,
+    const InMemoryPathMap & path_map,
     std::unordered_set<std::string> & result)
 {
     /// Directories are retrieved from the in-memory path map.
     {
-        std::shared_lock lock(shared_mutex);
-        auto end_it = local_path_prefixes.end();
+        SharedLockGuard lock(path_map.mutex);
+        const auto & local_path_prefixes = path_map.map;
+        const auto end_it = local_path_prefixes.end();
         for (auto it = local_path_prefixes.lower_bound(local_path); it != end_it; ++it)
         {
             const auto & [k, _] = std::make_tuple(it->first.string(), it->second);
@@ -179,7 +183,7 @@ MetadataStorageFromPlainRewritableObjectStorage::MetadataStorageFromPlainRewrita
     ObjectStoragePtr object_storage_, String storage_path_prefix_)
     : MetadataStorageFromPlainObjectStorage(object_storage_, storage_path_prefix_)
     , metadata_key_prefix(DB::getMetadataKeyPrefix(object_storage))
-    , path_map(std::make_shared<InMemoryPathMap>(loadPathPrefixMap(metadata_key_prefix, object_storage)))
+    , path_map(loadPathPrefixMap(metadata_key_prefix, object_storage))
 {
     if (object_storage->isWriteOnce())
         throw Exception(
@@ -261,7 +265,7 @@ void MetadataStorageFromPlainRewritableObjectStorage::getDirectChildrenOnDisk(
     const std::string & local_path,
     std::unordered_set<std::string> & result) const
 {
-    getDirectChildrenOnDiskImpl(storage_key, remote_paths, local_path, getPathMap()->map, getPathMap()->mutex, result);
+    getDirectChildrenOnDiskImpl(storage_key, remote_paths, local_path, *getPathMap(), result);
 }
 
 }

From db13ba2c488303e90717fbcc5adf5304241ac474 Mon Sep 17 00:00:00 2001
From: Julia Kartseva <yulia.kartseva@gmail.com>
Date: Sat, 13 Jul 2024 06:51:40 +0000
Subject: [PATCH 1223/1488] style fix

---
 .../CommonPathPrefixKeyGenerator.cpp          |  2 +-
 ...torageFromPlainObjectStorageOperations.cpp |  2 +-
 ...torageFromPlainRewritableObjectStorage.cpp | 91 ++++++++++---------
 3 files changed, 48 insertions(+), 47 deletions(-)

diff --git a/src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.cpp b/src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.cpp
index 1d041626a7e..521d5c037ab 100644
--- a/src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.cpp
+++ b/src/Disks/ObjectStorages/CommonPathPrefixKeyGenerator.cpp
@@ -1,8 +1,8 @@
 #include <Disks/ObjectStorages/CommonPathPrefixKeyGenerator.h>
 #include <Disks/ObjectStorages/InMemoryPathMap.h>
 
-#include <Common/getRandomASCIIString.h>
 #include <Common/SharedLockGuard.h>
+#include <Common/getRandomASCIIString.h>
 
 #include <deque>
 #include <filesystem>
diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp
index 8a06b204cfc..76090411bb9 100644
--- a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp
+++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp
@@ -1,6 +1,6 @@
 #include "MetadataStorageFromPlainObjectStorageOperations.h"
-#include "Common/SharedLockGuard.h"
 #include <Disks/ObjectStorages/InMemoryPathMap.h>
+#include "Common/SharedLockGuard.h"
 
 #include <IO/ReadHelpers.h>
 #include <IO/WriteHelpers.h>
diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp b/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp
index 22e73e36372..dba63bba321 100644
--- a/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp
+++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp
@@ -7,10 +7,10 @@
 #include <IO/ReadHelpers.h>
 #include <IO/S3Common.h>
 #include <IO/SharedThreadPools.h>
-#include <Common/ErrorCodes.h>
-#include <Common/logger_useful.h>
 #include "Common/SharedLockGuard.h"
 #include "Common/SharedMutex.h"
+#include <Common/ErrorCodes.h>
+#include <Common/logger_useful.h>
 #include "CommonPathPrefixKeyGenerator.h"
 
 
@@ -67,54 +67,55 @@ std::shared_ptr<InMemoryPathMap> loadPathPrefixMap(const std::string & metadata_
         if (remote_metadata_path.filename() != PREFIX_PATH_FILE_NAME)
             continue;
 
-        runner([remote_metadata_path, path, &object_storage, &result, &log, &settings, &metadata_key_prefix]
-        {
-            setThreadName("PlainRWMetaLoad");
-
-            StoredObject object{path};
-            String local_path;
-
-            try
+        runner(
+            [remote_metadata_path, path, &object_storage, &result, &log, &settings, &metadata_key_prefix]
             {
-                auto read_buf = object_storage->readObject(object, settings);
-                readStringUntilEOF(local_path, *read_buf);
-            }
+                setThreadName("PlainRWMetaLoad");
+
+                StoredObject object{path};
+                String local_path;
+
+                try
+                {
+                    auto read_buf = object_storage->readObject(object, settings);
+                    readStringUntilEOF(local_path, *read_buf);
+                }
 #if USE_AWS_S3
-            catch (const S3Exception & e)
-            {
-                /// It is ok if a directory was removed just now.
-                /// We support attaching a filesystem that is concurrently modified by someone else.
-                if (e.getS3ErrorCode() == Aws::S3::S3Errors::NO_SUCH_KEY)
-                    return;
-                throw;
-            }
+                catch (const S3Exception & e)
+                {
+                    /// It is ok if a directory was removed just now.
+                    /// We support attaching a filesystem that is concurrently modified by someone else.
+                    if (e.getS3ErrorCode() == Aws::S3::S3Errors::NO_SUCH_KEY)
+                        return;
+                    throw;
+                }
 #endif
-            catch (...)
-            {
-                throw;
-            }
+                catch (...)
+                {
+                    throw;
+                }
 
-            chassert(remote_metadata_path.has_parent_path());
-            chassert(remote_metadata_path.string().starts_with(metadata_key_prefix));
-            auto suffix = remote_metadata_path.string().substr(metadata_key_prefix.size());
-            auto remote_path = std::filesystem::path(std::move(suffix));
-            std::pair<Map::iterator, bool> res;
-            {
-                std::lock_guard lock(result->mutex);
-                res = result->map.emplace(std::filesystem::path(local_path).parent_path(), remote_path.parent_path());
-            }
+                chassert(remote_metadata_path.has_parent_path());
+                chassert(remote_metadata_path.string().starts_with(metadata_key_prefix));
+                auto suffix = remote_metadata_path.string().substr(metadata_key_prefix.size());
+                auto remote_path = std::filesystem::path(std::move(suffix));
+                std::pair<Map::iterator, bool> res;
+                {
+                    std::lock_guard lock(result->mutex);
+                    res = result->map.emplace(std::filesystem::path(local_path).parent_path(), remote_path.parent_path());
+                }
 
-            /// This can happen if table replication is enabled, then the same local path is written
-            /// in `prefix.path` of each replica.
-            /// TODO: should replicated tables (e.g., RMT) be explicitly disallowed?
-            if (!res.second)
-                LOG_WARNING(
-                    log,
-                    "The local path '{}' is already mapped to a remote path '{}', ignoring: '{}'",
-                    local_path,
-                    res.first->second,
-                    remote_path.parent_path().string());
-        });
+                /// This can happen if table replication is enabled, then the same local path is written
+                /// in `prefix.path` of each replica.
+                /// TODO: should replicated tables (e.g., RMT) be explicitly disallowed?
+                if (!res.second)
+                    LOG_WARNING(
+                        log,
+                        "The local path '{}' is already mapped to a remote path '{}', ignoring: '{}'",
+                        local_path,
+                        res.first->second,
+                        remote_path.parent_path().string());
+            });
     }
 
     runner.waitForAllToFinishAndRethrowFirstError();

From 912bddf86f53f207b76ba453e43b6724b24ef6df Mon Sep 17 00:00:00 2001
From: Julia Kartseva <yulia.kartseva@gmail.com>
Date: Sun, 14 Jul 2024 20:49:37 -0700
Subject: [PATCH 1224/1488] Update src/Disks/ObjectStorages/InMemoryPathMap.h

Co-authored-by: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com>
---
 src/Disks/ObjectStorages/InMemoryPathMap.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/Disks/ObjectStorages/InMemoryPathMap.h b/src/Disks/ObjectStorages/InMemoryPathMap.h
index dcd28dfaf6c..2ac291dfaf0 100644
--- a/src/Disks/ObjectStorages/InMemoryPathMap.h
+++ b/src/Disks/ObjectStorages/InMemoryPathMap.h
@@ -21,6 +21,7 @@ struct InMemoryPathMap
             return path1 < path2;
         }
     };
+    /// Local -> Remote path.
     using Map = std::map<std::filesystem::path, std::string, PathComparator>;
     mutable SharedMutex mutex;
     Map map TSA_GUARDED_BY(mutex);

From 727f5ed108e3b92c81d4ed295e0de438de8bae2b Mon Sep 17 00:00:00 2001
From: Julia Kartseva <yulia.kartseva@gmail.com>
Date: Sun, 14 Jul 2024 20:49:57 -0700
Subject: [PATCH 1225/1488] Update
 src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp

Co-authored-by: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com>
---
 .../MetadataStorageFromPlainRewritableObjectStorage.cpp         | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp b/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp
index dba63bba321..3380dec60ca 100644
--- a/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp
+++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp
@@ -29,7 +29,7 @@ constexpr auto PREFIX_PATH_FILE_NAME = "prefix.path";
 constexpr auto METADATA_PATH_TOKEN = "__meta/";
 
 /// Use a separate layout for metadata iff:
-/// 1. The disk endpoint does not contain objects, OR
+/// 1. The disk endpoint does not contain any objects yet (empty), OR
 /// 2. The metadata is already stored behind a separate endpoint.
 /// Otherwise, store metadata along with regular data for backward compatibility.
 std::string getMetadataKeyPrefix(ObjectStoragePtr object_storage)

From 4c78531c9c0681a84309e02ecfde17a36f1c1ad5 Mon Sep 17 00:00:00 2001
From: Julia Kartseva <yulia.kartseva@gmail.com>
Date: Sun, 14 Jul 2024 20:50:20 -0700
Subject: [PATCH 1226/1488] Update
 src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp

Co-authored-by: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com>
---
 .../MetadataStorageFromPlainRewritableObjectStorage.cpp         | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp b/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp
index 3380dec60ca..40aed32c047 100644
--- a/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp
+++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp
@@ -28,7 +28,7 @@ namespace
 constexpr auto PREFIX_PATH_FILE_NAME = "prefix.path";
 constexpr auto METADATA_PATH_TOKEN = "__meta/";
 
-/// Use a separate layout for metadata iff:
+/// Use a separate layout for metadata if:
 /// 1. The disk endpoint does not contain any objects yet (empty), OR
 /// 2. The metadata is already stored behind a separate endpoint.
 /// Otherwise, store metadata along with regular data for backward compatibility.

From 3b986ef3400021cf18797a7872b926dcc191b547 Mon Sep 17 00:00:00 2001
From: Julia Kartseva <yulia.kartseva@gmail.com>
Date: Mon, 15 Jul 2024 05:50:42 +0000
Subject: [PATCH 1227/1488] address feedback: useSeparateLayoutForMetadata

---
 ...StorageFromPlainRewritableObjectStorage.cpp | 18 +++++++++++-------
 ...taStorageFromPlainRewritableObjectStorage.h |  4 +++-
 2 files changed, 14 insertions(+), 8 deletions(-)

diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp b/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp
index 40aed32c047..39b11d9a3e3 100644
--- a/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp
+++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.cpp
@@ -192,15 +192,15 @@ MetadataStorageFromPlainRewritableObjectStorage::MetadataStorageFromPlainRewrita
             "MetadataStorageFromPlainRewritableObjectStorage is not compatible with write-once storage '{}'",
             object_storage->getName());
 
-    if (getMetadataKeyPrefix() == object_storage->getCommonKeyPrefix())
+    if (useSeparateLayoutForMetadata())
     {
-        auto keys_gen = std::make_shared<CommonPathPrefixKeyGenerator>(object_storage->getCommonKeyPrefix(), path_map);
+        /// Use flat directory structure if the metadata is stored separately from the table data.
+        auto keys_gen = std::make_shared<FlatDirectoryStructureKeyGenerator>(object_storage->getCommonKeyPrefix(), path_map);
         object_storage->setKeysGenerator(keys_gen);
     }
     else
     {
-        /// Use flat directory structure if the metadata is stored separately from the table data.
-        auto keys_gen = std::make_shared<FlatDirectoryStructureKeyGenerator>(object_storage->getCommonKeyPrefix(), path_map);
+        auto keys_gen = std::make_shared<CommonPathPrefixKeyGenerator>(object_storage->getCommonKeyPrefix(), path_map);
         object_storage->setKeysGenerator(keys_gen);
     }
 }
@@ -216,7 +216,7 @@ bool MetadataStorageFromPlainRewritableObjectStorage::exists(const std::string &
     if (MetadataStorageFromPlainObjectStorage::exists(path))
         return true;
 
-    if (getMetadataKeyPrefix() != object_storage->getCommonKeyPrefix())
+    if (useSeparateLayoutForMetadata())
     {
         auto key_prefix = object_storage->generateObjectKeyForPath(path, getMetadataKeyPrefix()).serialize();
         return object_storage->existsOrHasAnyChild(key_prefix);
@@ -227,7 +227,7 @@ bool MetadataStorageFromPlainRewritableObjectStorage::exists(const std::string &
 
 bool MetadataStorageFromPlainRewritableObjectStorage::isDirectory(const std::string & path) const
 {
-    if (getMetadataKeyPrefix() != object_storage->getCommonKeyPrefix())
+    if (useSeparateLayoutForMetadata())
     {
         auto directory = std::filesystem::path(object_storage->generateObjectKeyForPath(path, getMetadataKeyPrefix()).serialize()) / "";
         return object_storage->existsOrHasAnyChild(directory);
@@ -249,7 +249,7 @@ std::vector<std::string> MetadataStorageFromPlainRewritableObjectStorage::listDi
     getDirectChildrenOnDisk(abs_key, files, std::filesystem::path(path) / "", directories);
     /// List empty directories that are identified by the `prefix.path` metadata files. This is required to, e.g., remove
     /// metadata along with regular files.
-    if (object_storage->getCommonKeyPrefix() != getMetadataKeyPrefix())
+    if (useSeparateLayoutForMetadata())
     {
         auto metadata_key = std::filesystem::path(getMetadataKeyPrefix()) / key_prefix / "";
         RelativePathsWithMetadata metadata_files;
@@ -269,4 +269,8 @@ void MetadataStorageFromPlainRewritableObjectStorage::getDirectChildrenOnDisk(
     getDirectChildrenOnDiskImpl(storage_key, remote_paths, local_path, *getPathMap(), result);
 }
 
+bool MetadataStorageFromPlainRewritableObjectStorage::useSeparateLayoutForMetadata() const
+{
+    return getMetadataKeyPrefix() != object_storage->getCommonKeyPrefix();
+}
 }
diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.h b/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.h
index 8fd147e15b9..82d93e3e7ae 100644
--- a/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.h
+++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainRewritableObjectStorage.h
@@ -24,7 +24,6 @@ public:
     bool isDirectory(const std::string & path) const override;
     std::vector<std::string> listDirectory(const std::string & path) const override;
 
-
 protected:
     std::string getMetadataKeyPrefix() const override { return metadata_key_prefix; }
     std::shared_ptr<InMemoryPathMap> getPathMap() const override { return path_map; }
@@ -33,6 +32,9 @@ protected:
         const RelativePathsWithMetadata & remote_paths,
         const std::string & local_path,
         std::unordered_set<std::string> & result) const;
+
+private:
+    bool useSeparateLayoutForMetadata() const;
 };
 
 }

From 359b42738a25aa02436c1bebc49d0b751e456ccb Mon Sep 17 00:00:00 2001
From: Julia Kartseva <yulia.kartseva@gmail.com>
Date: Mon, 15 Jul 2024 06:11:51 +0000
Subject: [PATCH 1228/1488] address feedback: key_prefix -> object_key_prefix

---
 ...tadataStorageFromPlainObjectStorageOperations.cpp | 12 ++++++------
 ...MetadataStorageFromPlainObjectStorageOperations.h |  2 +-
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp
index 76090411bb9..31fb8c7ef97 100644
--- a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp
+++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp
@@ -22,9 +22,9 @@ namespace
 
 constexpr auto PREFIX_PATH_FILE_NAME = "prefix.path";
 
-ObjectStorageKey createMetadataObjectKey(const std::string & key_prefix, const std::string & metadata_key_prefix)
+ObjectStorageKey createMetadataObjectKey(const std::string & object_key_prefix, const std::string & metadata_key_prefix)
 {
-    auto prefix = std::filesystem::path(metadata_key_prefix) / key_prefix;
+    auto prefix = std::filesystem::path(metadata_key_prefix) / object_key_prefix;
     return ObjectStorageKey::createAsRelative(prefix.string(), PREFIX_PATH_FILE_NAME);
 }
 }
@@ -35,7 +35,7 @@ MetadataStorageFromPlainObjectStorageCreateDirectoryOperation::MetadataStorageFr
     , path_map(path_map_)
     , object_storage(object_storage_)
     , metadata_key_prefix(metadata_key_prefix_)
-    , key_prefix(object_storage->generateObjectKeyPrefixForDirectoryPath(path, "" /* key_prefix */).serialize())
+    , object_key_prefix(object_storage->generateObjectKeyPrefixForDirectoryPath(path, "" /* object_key_prefix */).serialize())
 {
 }
 
@@ -47,7 +47,7 @@ void MetadataStorageFromPlainObjectStorageCreateDirectoryOperation::execute(std:
             return;
     }
 
-    auto metadata_object_key = createMetadataObjectKey(key_prefix, metadata_key_prefix);
+    auto metadata_object_key = createMetadataObjectKey(object_key_prefix, metadata_key_prefix);
 
     LOG_TRACE(
         getLogger("MetadataStorageFromPlainObjectStorageCreateDirectoryOperation"),
@@ -68,7 +68,7 @@ void MetadataStorageFromPlainObjectStorageCreateDirectoryOperation::execute(std:
     {
         std::lock_guard lock(path_map.mutex);
         auto & map = path_map.map;
-        [[maybe_unused]] auto result = map.emplace(path.parent_path(), std::move(key_prefix));
+        [[maybe_unused]] auto result = map.emplace(path.parent_path(), std::move(object_key_prefix));
         chassert(result.second);
     }
     auto metric = object_storage->getMetadataStorageMetrics().directory_map_size;
@@ -85,7 +85,7 @@ void MetadataStorageFromPlainObjectStorageCreateDirectoryOperation::execute(std:
 
 void MetadataStorageFromPlainObjectStorageCreateDirectoryOperation::undo(std::unique_lock<SharedMutex> &)
 {
-    auto metadata_object_key = createMetadataObjectKey(key_prefix, metadata_key_prefix);
+    auto metadata_object_key = createMetadataObjectKey(object_key_prefix, metadata_key_prefix);
 
     if (write_finalized)
     {
diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.h b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.h
index 3ac0ffef8d2..02305767faf 100644
--- a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.h
+++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.h
@@ -17,7 +17,7 @@ private:
     InMemoryPathMap & path_map;
     ObjectStoragePtr object_storage;
     const std::string metadata_key_prefix;
-    const std::string key_prefix;
+    const std::string object_key_prefix;
 
     bool write_created = false;
     bool write_finalized = false;

From 79a8cbe0c595e877a750c26cc27a8c68202279c7 Mon Sep 17 00:00:00 2001
From: Julia Kartseva <yulia.kartseva@gmail.com>
Date: Mon, 15 Jul 2024 06:16:00 +0000
Subject: [PATCH 1229/1488] address feedback: documentation

---
 .../MetadataStorageFromPlainObjectStorageOperations.cpp    | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp
index 31fb8c7ef97..be5168c5385 100644
--- a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp
+++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp
@@ -55,7 +55,7 @@ void MetadataStorageFromPlainObjectStorageCreateDirectoryOperation::execute(std:
         path,
         metadata_object_key.serialize());
 
-    auto metadata_object = StoredObject(metadata_object_key.serialize(), path / PREFIX_PATH_FILE_NAME);
+    auto metadata_object = StoredObject(/*remote_path*/ metadata_object_key.serialize(), /*local_path*/ path / PREFIX_PATH_FILE_NAME);
     auto buf = object_storage->writeObject(
         metadata_object,
         WriteMode::Rewrite,
@@ -137,7 +137,8 @@ std::unique_ptr<WriteBufferFromFileBase> MetadataStorageFromPlainObjectStorageMo
 
     auto metadata_object_key = createMetadataObjectKey(remote_path, metadata_key_prefix);
 
-    auto metadata_object = StoredObject(metadata_object_key.serialize(), expected_path / PREFIX_PATH_FILE_NAME);
+    auto metadata_object
+        = StoredObject(/*remote_path*/ metadata_object_key.serialize(), /*local_path*/ expected_path / PREFIX_PATH_FILE_NAME);
 
     if (validate_content)
     {
@@ -220,7 +221,7 @@ void MetadataStorageFromPlainObjectStorageRemoveDirectoryOperation::execute(std:
     LOG_TRACE(getLogger("MetadataStorageFromPlainObjectStorageRemoveDirectoryOperation"), "Removing directory '{}'", path);
 
     auto metadata_object_key = createMetadataObjectKey(key_prefix, metadata_key_prefix);
-    auto metadata_object = StoredObject(metadata_object_key.serialize(), path / PREFIX_PATH_FILE_NAME);
+    auto metadata_object = StoredObject(/*remote_path*/ metadata_object_key.serialize(), /*local_path*/ path / PREFIX_PATH_FILE_NAME);
     object_storage->removeObject(metadata_object);
 
     {

From 42bd49dae6244590ba406ad502083bf610276eb9 Mon Sep 17 00:00:00 2001
From: Julia Kartseva <yulia.kartseva@gmail.com>
Date: Mon, 15 Jul 2024 07:05:51 +0000
Subject: [PATCH 1230/1488] address feedback: parent_path() for directories

---
 ...torageFromPlainObjectStorageOperations.cpp | 30 ++++++++++++-------
 ...aStorageFromPlainObjectStorageOperations.h |  4 ++-
 2 files changed, 23 insertions(+), 11 deletions(-)

diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp
index be5168c5385..c0e3f8e1fc9 100644
--- a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp
+++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp
@@ -31,7 +31,7 @@ ObjectStorageKey createMetadataObjectKey(const std::string & object_key_prefix,
 
 MetadataStorageFromPlainObjectStorageCreateDirectoryOperation::MetadataStorageFromPlainObjectStorageCreateDirectoryOperation(
     std::filesystem::path && path_, InMemoryPathMap & path_map_, ObjectStoragePtr object_storage_, const std::string & metadata_key_prefix_)
-    : path(std::move(path_))
+    : path((chassert(path_.string().ends_with('/')), std::move(path_)))
     , path_map(path_map_)
     , object_storage(object_storage_)
     , metadata_key_prefix(metadata_key_prefix_)
@@ -41,9 +41,11 @@ MetadataStorageFromPlainObjectStorageCreateDirectoryOperation::MetadataStorageFr
 
 void MetadataStorageFromPlainObjectStorageCreateDirectoryOperation::execute(std::unique_lock<SharedMutex> &)
 {
+    /// parent_path() removes the trailing '/'
+    const auto base_path = path.parent_path();
     {
         SharedLockGuard lock(path_map.mutex);
-        if (path_map.map.contains(path.parent_path()))
+        if (path_map.map.contains(base_path))
             return;
     }
 
@@ -68,7 +70,7 @@ void MetadataStorageFromPlainObjectStorageCreateDirectoryOperation::execute(std:
     {
         std::lock_guard lock(path_map.mutex);
         auto & map = path_map.map;
-        [[maybe_unused]] auto result = map.emplace(path.parent_path(), std::move(object_key_prefix));
+        [[maybe_unused]] auto result = map.emplace(base_path, std::move(object_key_prefix));
         chassert(result.second);
     }
     auto metric = object_storage->getMetadataStorageMetrics().directory_map_size;
@@ -89,9 +91,10 @@ void MetadataStorageFromPlainObjectStorageCreateDirectoryOperation::undo(std::un
 
     if (write_finalized)
     {
+        const auto base_path = path.parent_path();
         {
             std::lock_guard lock(path_map.mutex);
-            path_map.map.erase(path.parent_path());
+            path_map.map.erase(base_path);
         }
         auto metric = object_storage->getMetadataStorageMetrics().directory_map_size;
         CurrentMetrics::sub(metric, 1);
@@ -108,8 +111,8 @@ MetadataStorageFromPlainObjectStorageMoveDirectoryOperation::MetadataStorageFrom
     InMemoryPathMap & path_map_,
     ObjectStoragePtr object_storage_,
     const std::string & metadata_key_prefix_)
-    : path_from(std::move(path_from_))
-    , path_to(std::move(path_to_))
+    : path_from((chassert(path_from_.string().ends_with('/')), std::move(path_from_)))
+    , path_to((chassert(path_to_.string().ends_with('/')), std::move(path_to_)))
     , path_map(path_map_)
     , object_storage(object_storage_)
     , metadata_key_prefix(metadata_key_prefix_)
@@ -123,6 +126,7 @@ std::unique_ptr<WriteBufferFromFileBase> MetadataStorageFromPlainObjectStorageMo
     {
         SharedLockGuard lock(path_map.mutex);
         auto & map = path_map.map;
+        /// parent_path() removes the trailing '/'.
         auto expected_it = map.find(expected_path.parent_path());
         if (expected_it == map.end())
             throw Exception(
@@ -174,10 +178,14 @@ void MetadataStorageFromPlainObjectStorageMoveDirectoryOperation::execute(std::u
     writeString(path_to.string(), *write_buf);
     write_buf->finalize();
 
+    /// parent_path() removes the trailing '/'.
+    auto base_path_to = path_to.parent_path();
+    auto base_path_from = path_from.parent_path();
+
     {
         std::lock_guard lock(path_map.mutex);
         auto & map = path_map.map;
-        [[maybe_unused]] auto result = map.emplace(path_to.parent_path(), map.extract(path_from.parent_path()).mapped());
+        [[maybe_unused]] auto result = map.emplace(base_path_to, map.extract(base_path_from).mapped());
         chassert(result.second);
     }
 
@@ -203,16 +211,18 @@ void MetadataStorageFromPlainObjectStorageMoveDirectoryOperation::undo(std::uniq
 
 MetadataStorageFromPlainObjectStorageRemoveDirectoryOperation::MetadataStorageFromPlainObjectStorageRemoveDirectoryOperation(
     std::filesystem::path && path_, InMemoryPathMap & path_map_, ObjectStoragePtr object_storage_, const std::string & metadata_key_prefix_)
-    : path(std::move(path_)), path_map(path_map_), object_storage(object_storage_), metadata_key_prefix(metadata_key_prefix_)
+    : path((chassert(path_.string().ends_with('/')), std::move(path_))), path_map(path_map_), object_storage(object_storage_), metadata_key_prefix(metadata_key_prefix_)
 {
 }
 
 void MetadataStorageFromPlainObjectStorageRemoveDirectoryOperation::execute(std::unique_lock<SharedMutex> & /* metadata_lock */)
 {
+    /// parent_path() removes the trailing '/'
+    const auto base_path = path.parent_path();
     {
         SharedLockGuard lock(path_map.mutex);
         auto & map = path_map.map;
-        auto path_it = map.find(path.parent_path());
+        auto path_it = map.find(base_path);
         if (path_it == map.end())
             return;
         key_prefix = path_it->second;
@@ -227,7 +237,7 @@ void MetadataStorageFromPlainObjectStorageRemoveDirectoryOperation::execute(std:
     {
         std::lock_guard lock(path_map.mutex);
         auto & map = path_map.map;
-        map.erase(path.parent_path());
+        map.erase(base_path);
     }
 
     auto metric = object_storage->getMetadataStorageMetrics().directory_map_size;
diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.h b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.h
index 02305767faf..93ebe668d56 100644
--- a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.h
+++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.h
@@ -23,8 +23,8 @@ private:
     bool write_finalized = false;
 
 public:
-    // Assuming that paths are normalized.
     MetadataStorageFromPlainObjectStorageCreateDirectoryOperation(
+        /// path_ must end with a trailing '/'.
         std::filesystem::path && path_,
         InMemoryPathMap & path_map_,
         ObjectStoragePtr object_storage_,
@@ -51,6 +51,7 @@ private:
 
 public:
     MetadataStorageFromPlainObjectStorageMoveDirectoryOperation(
+        /// Both path_from_ and path_to_ must end with a trailing '/'.
         std::filesystem::path && path_from_,
         std::filesystem::path && path_to_,
         InMemoryPathMap & path_map_,
@@ -76,6 +77,7 @@ private:
 
 public:
     MetadataStorageFromPlainObjectStorageRemoveDirectoryOperation(
+        /// path_ must end with a trailing '/'.
         std::filesystem::path && path_,
         InMemoryPathMap & path_map_,
         ObjectStoragePtr object_storage_,

From 41fc84bb2df441d117681c589dbea5c516ed4748 Mon Sep 17 00:00:00 2001
From: Julia Kartseva <yulia.kartseva@gmail.com>
Date: Mon, 15 Jul 2024 21:16:27 +0000
Subject: [PATCH 1231/1488] fix build

---
 src/Disks/ObjectStorages/InMemoryPathMap.h       |  1 +
 ...taStorageFromPlainObjectStorageOperations.cpp | 16 ++++++++++------
 2 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/src/Disks/ObjectStorages/InMemoryPathMap.h b/src/Disks/ObjectStorages/InMemoryPathMap.h
index 2ac291dfaf0..e319c187ca7 100644
--- a/src/Disks/ObjectStorages/InMemoryPathMap.h
+++ b/src/Disks/ObjectStorages/InMemoryPathMap.h
@@ -2,6 +2,7 @@
 
 #include <filesystem>
 #include <map>
+#include <base/defines.h>
 #include <Common/SharedMutex.h>
 
 namespace DB
diff --git a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp
index c0e3f8e1fc9..bfd203ef2e0 100644
--- a/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp
+++ b/src/Disks/ObjectStorages/MetadataStorageFromPlainObjectStorageOperations.cpp
@@ -1,10 +1,10 @@
 #include "MetadataStorageFromPlainObjectStorageOperations.h"
 #include <Disks/ObjectStorages/InMemoryPathMap.h>
-#include "Common/SharedLockGuard.h"
 
 #include <IO/ReadHelpers.h>
 #include <IO/WriteHelpers.h>
 #include <Common/Exception.h>
+#include <Common/SharedLockGuard.h>
 #include <Common/logger_useful.h>
 
 namespace DB
@@ -31,12 +31,13 @@ ObjectStorageKey createMetadataObjectKey(const std::string & object_key_prefix,
 
 MetadataStorageFromPlainObjectStorageCreateDirectoryOperation::MetadataStorageFromPlainObjectStorageCreateDirectoryOperation(
     std::filesystem::path && path_, InMemoryPathMap & path_map_, ObjectStoragePtr object_storage_, const std::string & metadata_key_prefix_)
-    : path((chassert(path_.string().ends_with('/')), std::move(path_)))
+    : path(std::move(path_))
     , path_map(path_map_)
     , object_storage(object_storage_)
     , metadata_key_prefix(metadata_key_prefix_)
     , object_key_prefix(object_storage->generateObjectKeyPrefixForDirectoryPath(path, "" /* object_key_prefix */).serialize())
 {
+    chassert(path.string().ends_with('/'));
 }
 
 void MetadataStorageFromPlainObjectStorageCreateDirectoryOperation::execute(std::unique_lock<SharedMutex> &)
@@ -70,7 +71,7 @@ void MetadataStorageFromPlainObjectStorageCreateDirectoryOperation::execute(std:
     {
         std::lock_guard lock(path_map.mutex);
         auto & map = path_map.map;
-        [[maybe_unused]] auto result = map.emplace(base_path, std::move(object_key_prefix));
+        [[maybe_unused]] auto result = map.emplace(base_path, object_key_prefix);
         chassert(result.second);
     }
     auto metric = object_storage->getMetadataStorageMetrics().directory_map_size;
@@ -111,12 +112,14 @@ MetadataStorageFromPlainObjectStorageMoveDirectoryOperation::MetadataStorageFrom
     InMemoryPathMap & path_map_,
     ObjectStoragePtr object_storage_,
     const std::string & metadata_key_prefix_)
-    : path_from((chassert(path_from_.string().ends_with('/')), std::move(path_from_)))
-    , path_to((chassert(path_to_.string().ends_with('/')), std::move(path_to_)))
+    : path_from(std::move(path_from_))
+    , path_to(std::move(path_to_))
     , path_map(path_map_)
     , object_storage(object_storage_)
     , metadata_key_prefix(metadata_key_prefix_)
 {
+    chassert(path_from.string().ends_with('/'));
+    chassert(path_to.string().ends_with('/'));
 }
 
 std::unique_ptr<WriteBufferFromFileBase> MetadataStorageFromPlainObjectStorageMoveDirectoryOperation::createWriteBuf(
@@ -211,8 +214,9 @@ void MetadataStorageFromPlainObjectStorageMoveDirectoryOperation::undo(std::uniq
 
 MetadataStorageFromPlainObjectStorageRemoveDirectoryOperation::MetadataStorageFromPlainObjectStorageRemoveDirectoryOperation(
     std::filesystem::path && path_, InMemoryPathMap & path_map_, ObjectStoragePtr object_storage_, const std::string & metadata_key_prefix_)
-    : path((chassert(path_.string().ends_with('/')), std::move(path_))), path_map(path_map_), object_storage(object_storage_), metadata_key_prefix(metadata_key_prefix_)
+    : path(std::move(path_)), path_map(path_map_), object_storage(object_storage_), metadata_key_prefix(metadata_key_prefix_)
 {
+    chassert(path.string().ends_with('/'));
 }
 
 void MetadataStorageFromPlainObjectStorageRemoveDirectoryOperation::execute(std::unique_lock<SharedMutex> & /* metadata_lock */)

From 9b4e02e8dabb649076389ee96a271da025913ddf Mon Sep 17 00:00:00 2001
From: Julia Kartseva <yulia.kartseva@gmail.com>
Date: Tue, 16 Jul 2024 22:48:37 +0000
Subject: [PATCH 1232/1488] fix macOs build

---
 src/Disks/ObjectStorages/InMemoryPathMap.h | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/src/Disks/ObjectStorages/InMemoryPathMap.h b/src/Disks/ObjectStorages/InMemoryPathMap.h
index e319c187ca7..a9859d5e2b8 100644
--- a/src/Disks/ObjectStorages/InMemoryPathMap.h
+++ b/src/Disks/ObjectStorages/InMemoryPathMap.h
@@ -25,7 +25,13 @@ struct InMemoryPathMap
     /// Local -> Remote path.
     using Map = std::map<std::filesystem::path, std::string, PathComparator>;
     mutable SharedMutex mutex;
-    Map map TSA_GUARDED_BY(mutex);
+
+#ifdef OS_LINUX
+    Map TSA_GUARDED_BY(mutex) map;
+/// std::shared_mutex may not be annotated with the 'capability' attribute in libcxx.
+#else
+    Map map;
+#endif
 };
 
 }

From 774cba09dfd4ab347d05caf45f8135a3a51771c3 Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Fri, 2 Aug 2024 08:48:41 +0200
Subject: [PATCH 1233/1488] Fix flaky test_replicated_table_attach

---
 .../test_replicated_table_attach/configs/config.xml         | 2 +-
 tests/integration/test_replicated_table_attach/test.py      | 6 +++++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/tests/integration/test_replicated_table_attach/configs/config.xml b/tests/integration/test_replicated_table_attach/configs/config.xml
index fea3eab4126..3f72f638776 100644
--- a/tests/integration/test_replicated_table_attach/configs/config.xml
+++ b/tests/integration/test_replicated_table_attach/configs/config.xml
@@ -1,6 +1,6 @@
 <clickhouse>
     <background_schedule_pool_size>1</background_schedule_pool_size>
     <merge_tree>
-        <initialization_retry_period>5</initialization_retry_period>
+        <initialization_retry_period>3</initialization_retry_period>
     </merge_tree>
 </clickhouse>
diff --git a/tests/integration/test_replicated_table_attach/test.py b/tests/integration/test_replicated_table_attach/test.py
index de60b7ec291..4fe8064b26a 100644
--- a/tests/integration/test_replicated_table_attach/test.py
+++ b/tests/integration/test_replicated_table_attach/test.py
@@ -80,4 +80,8 @@ def test_startup_with_small_bg_pool_partitioned(started_cluster):
         assert_values()
 
     # check that we activate it in the end
-    node.query_with_retry("INSERT INTO replicated_table_partitioned VALUES(20, 30)")
+    node.query_with_retry(
+        "INSERT INTO replicated_table_partitioned VALUES(20, 30)",
+        retry_count=20,
+        sleep_time=3,
+    )

From 797144270b3e20e9e4306949bde95c9a9a32c5e0 Mon Sep 17 00:00:00 2001
From: robot-clickhouse <robot-clickhouse@users.noreply.github.com>
Date: Fri, 2 Aug 2024 07:09:39 +0000
Subject: [PATCH 1234/1488] Update version_date.tsv and changelogs after
 v24.4.4.113-stable

---
 docs/changelogs/v24.4.4.113-stable.md | 73 +++++++++++++++++++++++++++
 utils/list-versions/version_date.tsv  |  1 +
 2 files changed, 74 insertions(+)
 create mode 100644 docs/changelogs/v24.4.4.113-stable.md

diff --git a/docs/changelogs/v24.4.4.113-stable.md b/docs/changelogs/v24.4.4.113-stable.md
new file mode 100644
index 00000000000..1f8a221a0a2
--- /dev/null
+++ b/docs/changelogs/v24.4.4.113-stable.md
@@ -0,0 +1,73 @@
+---
+sidebar_position: 1
+sidebar_label: 2024
+---
+
+# 2024 Changelog
+
+### ClickHouse release v24.4.4.113-stable (d63a54957bd) FIXME as compared to v24.4.3.25-stable (a915dd4eda4)
+
+#### Improvement
+* Backported in [#65884](https://github.com/ClickHouse/ClickHouse/issues/65884): Always start Keeper with sufficient amount of threads in global thread pool. [#64444](https://github.com/ClickHouse/ClickHouse/pull/64444) ([Duc Canh Le](https://github.com/canhld94)).
+* Backported in [#65303](https://github.com/ClickHouse/ClickHouse/issues/65303): Returned back the behaviour of how ClickHouse works and interprets Tuples in CSV format. This change effectively reverts https://github.com/ClickHouse/ClickHouse/pull/60994 and makes it available only under a few settings: `output_format_csv_serialize_tuple_into_separate_columns`, `input_format_csv_deserialize_separate_columns_into_tuple` and `input_format_csv_try_infer_strings_from_quoted_tuples`. [#65170](https://github.com/ClickHouse/ClickHouse/pull/65170) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+* Backported in [#65894](https://github.com/ClickHouse/ClickHouse/issues/65894): Respect cgroup CPU limit in Keeper. [#65819](https://github.com/ClickHouse/ClickHouse/pull/65819) ([Antonio Andelic](https://github.com/antonio2368)).
+
+#### Critical Bug Fix (crash, LOGICAL_ERROR, data loss, RBAC)
+* Backported in [#65372](https://github.com/ClickHouse/ClickHouse/issues/65372): Fix a bug in ClickHouse Keeper that causes digest mismatch during closing session. [#65198](https://github.com/ClickHouse/ClickHouse/pull/65198) ([Aleksei Filatov](https://github.com/aalexfvk)).
+* Backported in [#66883](https://github.com/ClickHouse/ClickHouse/issues/66883): Fix unexpeced size of low cardinality column in function calls. [#65298](https://github.com/ClickHouse/ClickHouse/pull/65298) ([Raúl Marín](https://github.com/Algunenano)).
+* Backported in [#65435](https://github.com/ClickHouse/ClickHouse/issues/65435): Forbid `QUALIFY` clause in the old analyzer. The old analyzer ignored `QUALIFY`, so it could lead to unexpected data removal in mutations. [#65356](https://github.com/ClickHouse/ClickHouse/pull/65356) ([Dmitry Novik](https://github.com/novikd)).
+* Backported in [#65448](https://github.com/ClickHouse/ClickHouse/issues/65448): Use correct memory alignment for Distinct combinator. Previously, crash could happen because of invalid memory allocation when the combinator was used. [#65379](https://github.com/ClickHouse/ClickHouse/pull/65379) ([Antonio Andelic](https://github.com/antonio2368)).
+* Backported in [#65710](https://github.com/ClickHouse/ClickHouse/issues/65710): Fix crash in maxIntersections. [#65689](https://github.com/ClickHouse/ClickHouse/pull/65689) ([Raúl Marín](https://github.com/Algunenano)).
+* Backported in [#66689](https://github.com/ClickHouse/ClickHouse/issues/66689): Fix the VALID UNTIL clause in the user definition resetting after a restart. Closes [#66405](https://github.com/ClickHouse/ClickHouse/issues/66405). [#66409](https://github.com/ClickHouse/ClickHouse/pull/66409) ([Nikolay Degterinsky](https://github.com/evillique)).
+* Backported in [#67499](https://github.com/ClickHouse/ClickHouse/issues/67499): Fix crash in DistributedAsyncInsert when connection is empty. [#67219](https://github.com/ClickHouse/ClickHouse/pull/67219) ([Pablo Marcos](https://github.com/pamarcos)).
+
+#### Bug Fix (user-visible misbehavior in an official stable release)
+* Backported in [#65353](https://github.com/ClickHouse/ClickHouse/issues/65353): Fix possible abort on uncaught exception in ~WriteBufferFromFileDescriptor in StatusFile. [#64206](https://github.com/ClickHouse/ClickHouse/pull/64206) ([Kruglov Pavel](https://github.com/Avogar)).
+* Backported in [#65060](https://github.com/ClickHouse/ClickHouse/issues/65060): Fix the `Expression nodes list expected 1 projection names` and `Unknown expression or identifier` errors for queries with aliases to `GLOBAL IN.`. [#64517](https://github.com/ClickHouse/ClickHouse/pull/64517) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#65329](https://github.com/ClickHouse/ClickHouse/issues/65329): Fix the crash loop when restoring from backup is blocked by creating an MV with a definer that hasn't been restored yet. [#64595](https://github.com/ClickHouse/ClickHouse/pull/64595) ([pufit](https://github.com/pufit)).
+* Backported in [#64833](https://github.com/ClickHouse/ClickHouse/issues/64833): Fix bug which could lead to non-working TTLs with expressions. [#64694](https://github.com/ClickHouse/ClickHouse/pull/64694) ([alesapin](https://github.com/alesapin)).
+* Backported in [#65086](https://github.com/ClickHouse/ClickHouse/issues/65086): Fix removing the `WHERE` and `PREWHERE` expressions, which are always true (for the new analyzer). [#64695](https://github.com/ClickHouse/ClickHouse/pull/64695) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#65540](https://github.com/ClickHouse/ClickHouse/issues/65540): Fix crash for `ALTER TABLE ... ON CLUSTER ... MODIFY SQL SECURITY`. [#64957](https://github.com/ClickHouse/ClickHouse/pull/64957) ([pufit](https://github.com/pufit)).
+* Backported in [#65578](https://github.com/ClickHouse/ClickHouse/issues/65578): Fix crash on destroying AccessControl: add explicit shutdown. [#64993](https://github.com/ClickHouse/ClickHouse/pull/64993) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Backported in [#65161](https://github.com/ClickHouse/ClickHouse/issues/65161): Fix pushing arithmetic operations out of aggregation. In the new analyzer, optimization was applied only once. [#65104](https://github.com/ClickHouse/ClickHouse/pull/65104) ([Dmitry Novik](https://github.com/novikd)).
+* Backported in [#65616](https://github.com/ClickHouse/ClickHouse/issues/65616): Fix aggregate function name rewriting in the new analyzer. [#65110](https://github.com/ClickHouse/ClickHouse/pull/65110) ([Dmitry Novik](https://github.com/novikd)).
+* Backported in [#65730](https://github.com/ClickHouse/ClickHouse/issues/65730): Eliminate injective function in argument of functions `uniq*` recursively. This used to work correctly but was broken in the new analyzer. [#65140](https://github.com/ClickHouse/ClickHouse/pull/65140) ([Duc Canh Le](https://github.com/canhld94)).
+* Backported in [#65668](https://github.com/ClickHouse/ClickHouse/issues/65668): Disable `non-intersecting-parts` optimization for queries with `FINAL` in case of `read-in-order` optimization was enabled. This could lead to an incorrect query result. As a workaround, disable `do_not_merge_across_partitions_select_final` and `split_parts_ranges_into_intersecting_and_non_intersecting_final` before this fix is merged. [#65505](https://github.com/ClickHouse/ClickHouse/pull/65505) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#65786](https://github.com/ClickHouse/ClickHouse/issues/65786): Fixed bug in MergeJoin. Column in sparse serialisation might be treated as a column of its nested type though the required conversion wasn't performed. [#65632](https://github.com/ClickHouse/ClickHouse/pull/65632) ([Nikita Taranov](https://github.com/nickitat)).
+* Backported in [#65810](https://github.com/ClickHouse/ClickHouse/issues/65810): Fix invalid exceptions in function `parseDateTime` with `%F` and `%D` placeholders. [#65768](https://github.com/ClickHouse/ClickHouse/pull/65768) ([Antonio Andelic](https://github.com/antonio2368)).
+* Backported in [#65931](https://github.com/ClickHouse/ClickHouse/issues/65931): For queries that read from `PostgreSQL`, cancel the internal `PostgreSQL` query if the ClickHouse query is finished. Otherwise, `ClickHouse` query cannot be canceled until the internal `PostgreSQL` query is finished. [#65771](https://github.com/ClickHouse/ClickHouse/pull/65771) ([Maksim Kita](https://github.com/kitaisreal)).
+* Backported in [#65826](https://github.com/ClickHouse/ClickHouse/issues/65826): Fix a bug in short circuit logic when old analyzer and dictGetOrDefault is used. [#65802](https://github.com/ClickHouse/ClickHouse/pull/65802) ([jsc0218](https://github.com/jsc0218)).
+* Backported in [#66299](https://github.com/ClickHouse/ClickHouse/issues/66299): Better handling of join conditions involving `IS NULL` checks (for example `ON (a = b AND (a IS NOT NULL) AND (b IS NOT NULL) ) OR ( (a IS NULL) AND (b IS NULL) )` is rewritten to `ON a <=> b`), fix incorrect optimization when condition other then `IS NULL` are present. [#65835](https://github.com/ClickHouse/ClickHouse/pull/65835) ([vdimir](https://github.com/vdimir)).
+* Backported in [#66326](https://github.com/ClickHouse/ClickHouse/issues/66326): Add missing settings `input_format_csv_skip_first_lines/input_format_tsv_skip_first_lines/input_format_csv_try_infer_numbers_from_strings/input_format_csv_try_infer_strings_from_quoted_tuples` in schema inference cache because they can change the resulting schema. It prevents from incorrect result of schema inference with these settings changed. [#65980](https://github.com/ClickHouse/ClickHouse/pull/65980) ([Kruglov Pavel](https://github.com/Avogar)).
+* Backported in [#66153](https://github.com/ClickHouse/ClickHouse/issues/66153): Fixed buffer overflow bug in `unbin`/`unhex` implementation. [#66106](https://github.com/ClickHouse/ClickHouse/pull/66106) ([Nikita Taranov](https://github.com/nickitat)).
+* Backported in [#66459](https://github.com/ClickHouse/ClickHouse/issues/66459): Fixed a bug in ZooKeeper client: a session could get stuck in unusable state after receiving a hardware error from ZooKeeper. For example, this might happen due to "soft memory limit" in ClickHouse Keeper. [#66140](https://github.com/ClickHouse/ClickHouse/pull/66140) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Backported in [#66224](https://github.com/ClickHouse/ClickHouse/issues/66224): Fix issue in SumIfToCountIfVisitor and signed integers. [#66146](https://github.com/ClickHouse/ClickHouse/pull/66146) ([Raúl Marín](https://github.com/Algunenano)).
+* Backported in [#66267](https://github.com/ClickHouse/ClickHouse/issues/66267): Don't throw `TIMEOUT_EXCEEDED` for `none_only_active` mode of `distributed_ddl_output_mode`. [#66218](https://github.com/ClickHouse/ClickHouse/pull/66218) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Backported in [#66678](https://github.com/ClickHouse/ClickHouse/issues/66678): Fix handling limit for `system.numbers_mt` when no index can be used. [#66231](https://github.com/ClickHouse/ClickHouse/pull/66231) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
+* Backported in [#66603](https://github.com/ClickHouse/ClickHouse/issues/66603): Fixed how the ClickHouse server detects the maximum number of usable CPU cores as specified by cgroups v2 if the server runs in a container such as Docker. In more detail, containers often run their process in the root cgroup which has an empty name. In that case, ClickHouse ignored the CPU limits set by cgroups v2. [#66237](https://github.com/ClickHouse/ClickHouse/pull/66237) ([filimonov](https://github.com/filimonov)).
+* Backported in [#66358](https://github.com/ClickHouse/ClickHouse/issues/66358): Fix the `Not-ready set` error when a subquery with `IN` is used in the constraint. [#66261](https://github.com/ClickHouse/ClickHouse/pull/66261) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#66971](https://github.com/ClickHouse/ClickHouse/issues/66971): Fix `Column identifier is already registered` error with `group_by_use_nulls=true` and new analyzer. [#66400](https://github.com/ClickHouse/ClickHouse/pull/66400) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#66968](https://github.com/ClickHouse/ClickHouse/issues/66968): Fix `Cannot find column` error for queries with constant expression in `GROUP BY` key and new analyzer enabled. [#66433](https://github.com/ClickHouse/ClickHouse/pull/66433) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#66719](https://github.com/ClickHouse/ClickHouse/issues/66719): Correctly track memory for `Allocator::realloc`. [#66548](https://github.com/ClickHouse/ClickHouse/pull/66548) ([Antonio Andelic](https://github.com/antonio2368)).
+* Backported in [#66950](https://github.com/ClickHouse/ClickHouse/issues/66950): Fix an invalid result for queries with `WINDOW`. This could happen when `PARTITION` columns have sparse serialization and window functions are executed in parallel. [#66579](https://github.com/ClickHouse/ClickHouse/pull/66579) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#66947](https://github.com/ClickHouse/ClickHouse/issues/66947): Fix `Method getResultType is not supported for QUERY query node` error when scalar subquery was used as the first argument of IN (with new analyzer). [#66655](https://github.com/ClickHouse/ClickHouse/pull/66655) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#67631](https://github.com/ClickHouse/ClickHouse/issues/67631): Fix for occasional deadlock in Context::getDDLWorker. [#66843](https://github.com/ClickHouse/ClickHouse/pull/66843) ([Alexander Gololobov](https://github.com/davenger)).
+* Backported in [#67195](https://github.com/ClickHouse/ClickHouse/issues/67195): TRUNCATE DATABASE used to stop replication as if it was a DROP DATABASE query, it's fixed. [#67129](https://github.com/ClickHouse/ClickHouse/pull/67129) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Backported in [#67377](https://github.com/ClickHouse/ClickHouse/issues/67377): Fix error `Cannot convert column because it is non constant in source stream but must be constant in result.` for a query that reads from the `Merge` table over the `Distriburted` table with one shard. [#67146](https://github.com/ClickHouse/ClickHouse/pull/67146) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#67240](https://github.com/ClickHouse/ClickHouse/issues/67240): This closes [#67156](https://github.com/ClickHouse/ClickHouse/issues/67156). This closes [#66447](https://github.com/ClickHouse/ClickHouse/issues/66447). The bug was introduced in https://github.com/ClickHouse/ClickHouse/pull/62907. [#67178](https://github.com/ClickHouse/ClickHouse/pull/67178) ([Maksim Kita](https://github.com/kitaisreal)).
+* Backported in [#67574](https://github.com/ClickHouse/ClickHouse/issues/67574): Fix execution of nested short-circuit functions. [#67520](https://github.com/ClickHouse/ClickHouse/pull/67520) ([Kruglov Pavel](https://github.com/Avogar)).
+
+#### NOT FOR CHANGELOG / INSIGNIFICANT
+
+* Backported in [#65410](https://github.com/ClickHouse/ClickHouse/issues/65410): Re-enable OpenSSL session caching. [#65111](https://github.com/ClickHouse/ClickHouse/pull/65111) ([Robert Schulze](https://github.com/rschu1ze)).
+* Backported in [#65903](https://github.com/ClickHouse/ClickHouse/issues/65903): Fix bug with session closing in Keeper. [#65735](https://github.com/ClickHouse/ClickHouse/pull/65735) ([Antonio Andelic](https://github.com/antonio2368)).
+* Backported in [#66385](https://github.com/ClickHouse/ClickHouse/issues/66385): Disable broken cases from 02911_join_on_nullsafe_optimization. [#66310](https://github.com/ClickHouse/ClickHouse/pull/66310) ([vdimir](https://github.com/vdimir)).
+* Backported in [#66424](https://github.com/ClickHouse/ClickHouse/issues/66424): Ignore subquery for IN in DDLLoadingDependencyVisitor. [#66395](https://github.com/ClickHouse/ClickHouse/pull/66395) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#66542](https://github.com/ClickHouse/ClickHouse/issues/66542): Add additional log masking in CI. [#66523](https://github.com/ClickHouse/ClickHouse/pull/66523) ([Raúl Marín](https://github.com/Algunenano)).
+* Backported in [#66857](https://github.com/ClickHouse/ClickHouse/issues/66857): Fix data race in S3::ClientCache. [#66644](https://github.com/ClickHouse/ClickHouse/pull/66644) ([Konstantin Morozov](https://github.com/k-morozov)).
+* Backported in [#66873](https://github.com/ClickHouse/ClickHouse/issues/66873): Support one more case in JOIN ON ... IS NULL. [#66725](https://github.com/ClickHouse/ClickHouse/pull/66725) ([vdimir](https://github.com/vdimir)).
+* Backported in [#67057](https://github.com/ClickHouse/ClickHouse/issues/67057): Increase asio pool size in case the server is tiny. [#66761](https://github.com/ClickHouse/ClickHouse/pull/66761) ([alesapin](https://github.com/alesapin)).
+* Backported in [#66944](https://github.com/ClickHouse/ClickHouse/issues/66944): Small fix in realloc memory tracking. [#66820](https://github.com/ClickHouse/ClickHouse/pull/66820) ([Antonio Andelic](https://github.com/antonio2368)).
+* Backported in [#67250](https://github.com/ClickHouse/ClickHouse/issues/67250): Followup [#66725](https://github.com/ClickHouse/ClickHouse/issues/66725). [#66869](https://github.com/ClickHouse/ClickHouse/pull/66869) ([vdimir](https://github.com/vdimir)).
+* Backported in [#67410](https://github.com/ClickHouse/ClickHouse/issues/67410): CI: Fix build results for release branches. [#67402](https://github.com/ClickHouse/ClickHouse/pull/67402) ([Max K.](https://github.com/maxknv)).
+
diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv
index b1391c2d781..7b5dcda82e3 100644
--- a/utils/list-versions/version_date.tsv
+++ b/utils/list-versions/version_date.tsv
@@ -6,6 +6,7 @@ v24.5.4.49-stable	2024-07-01
 v24.5.3.5-stable	2024-06-13
 v24.5.2.34-stable	2024-06-13
 v24.5.1.1763-stable	2024-06-01
+v24.4.4.113-stable	2024-08-02
 v24.4.3.25-stable	2024-06-14
 v24.4.2.141-stable	2024-06-07
 v24.4.1.2088-stable	2024-05-01

From dc65c0aa078cf06357291c0fe68f6c035698320f Mon Sep 17 00:00:00 2001
From: Pablo Marcos <pablo.marcos.oltra@clickhouse.com>
Date: Fri, 2 Aug 2024 07:15:40 +0000
Subject: [PATCH 1235/1488] Fix doc for parallel test execution

Copy-pasterino strikes again. I forgot to remove the single quote.
With it, pytest thinks the whole argument is a file:

(no name '/ClickHouse/tests/integration/test_storage_s3_queue/test.py::test_max_set_age -- --count 10 -n 5' in any of [<Module test.py>])
---
 tests/integration/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/README.md b/tests/integration/README.md
index a8deb97b526..85146c79b1e 100644
--- a/tests/integration/README.md
+++ b/tests/integration/README.md
@@ -142,7 +142,7 @@ of parallel workers for `pytest-xdist`.
 $ export CLICKHOUSE_TESTS_BASE_CONFIG_DIR=$HOME/ClickHouse/programs/server/
 $ export CLICKHOUSE_TESTS_SERVER_BIN_PATH=$HOME/ClickHouse/programs/clickhouse
 $ export CLICKHOUSE_TESTS_ODBC_BRIDGE_BIN_PATH=$HOME/ClickHouse/programs/clickhouse-odbc-bridge
-$ ./runner 'test_storage_s3_queue/test.py::test_max_set_age -- --count 10 -n 5'
+$ ./runner test_storage_s3_queue/test.py::test_max_set_age --count 10 -n 5
 Start tests
 =============================================================================== test session starts ================================================================================
 platform linux -- Python 3.10.12, pytest-7.4.4, pluggy-1.5.0 -- /usr/bin/python3

From 9c05a0ad5a0269af02ae2234e1d01dc3ce64bce2 Mon Sep 17 00:00:00 2001
From: Michael Stetsyuk <mike.stetsyuk@gmail.com>
Date: Fri, 2 Aug 2024 09:34:32 +0100
Subject: [PATCH 1236/1488] rm dirs in test_storage_delta

---
 tests/integration/test_storage_delta/test.py | 28 +++++++-------------
 1 file changed, 10 insertions(+), 18 deletions(-)

diff --git a/tests/integration/test_storage_delta/test.py b/tests/integration/test_storage_delta/test.py
index 384b8296f66..92a870ab360 100644
--- a/tests/integration/test_storage_delta/test.py
+++ b/tests/integration/test_storage_delta/test.py
@@ -8,6 +8,7 @@ import os
 import json
 import time
 import glob
+import shutil
 
 import pyspark
 import delta
@@ -52,15 +53,6 @@ def get_spark():
     return builder.master("local").getOrCreate()
 
 
-def remove_local_directory_contents(full_path):
-    for path in glob.glob(f"{full_path}/**"):
-        if os.path.isfile(path):
-            os.unlink(path)
-        else:
-            remove_local_directory_contents(path)
-            os.rmdir(path)
-
-
 @pytest.fixture(scope="module")
 def started_cluster():
     try:
@@ -179,7 +171,7 @@ def test_single_log_file(started_cluster):
     )
 
     os.unlink(parquet_data_path)
-    remove_local_directory_contents(f"/{TABLE_NAME}")
+    shutil.rmtree(f"/{TABLE_NAME}")
 
 
 def test_partition_by(started_cluster):
@@ -203,7 +195,7 @@ def test_partition_by(started_cluster):
     create_delta_table(instance, TABLE_NAME)
     assert int(instance.query(f"SELECT count() FROM {TABLE_NAME}")) == 10
 
-    remove_local_directory_contents(f"/{TABLE_NAME}")
+    shutil.rmtree(f"/{TABLE_NAME}")
 
 
 def test_checkpoint(started_cluster):
@@ -280,7 +272,7 @@ def test_checkpoint(started_cluster):
         ).strip()
     )
 
-    remove_local_directory_contents(f"/{TABLE_NAME}")
+    shutil.rmtree(f"/{TABLE_NAME}")
     spark.sql(f"DROP TABLE {TABLE_NAME}")
 
 
@@ -321,7 +313,7 @@ def test_multiple_log_files(started_cluster):
         "SELECT number, toString(number + 1) FROM numbers(200)"
     )
 
-    remove_local_directory_contents(f"/{TABLE_NAME}")
+    shutil.rmtree(f"/{TABLE_NAME}")
 
 
 def test_metadata(started_cluster):
@@ -357,7 +349,7 @@ def test_metadata(started_cluster):
     assert int(instance.query(f"SELECT count() FROM {TABLE_NAME}")) == 100
 
     os.unlink(parquet_data_path)
-    remove_local_directory_contents(f"/{TABLE_NAME}")
+    shutil.rmtree(f"/{TABLE_NAME}")
 
 
 def test_types(started_cluster):
@@ -431,7 +423,7 @@ def test_types(started_cluster):
         ]
     )
 
-    remove_local_directory_contents(f"/{result_file}")
+    shutil.rmtree(f"/{result_file}")
     spark.sql(f"DROP TABLE {TABLE_NAME}")
 
 
@@ -496,7 +488,7 @@ def test_restart_broken(started_cluster):
     assert int(instance.query(f"SELECT count() FROM {TABLE_NAME}")) == 100
 
     os.unlink(parquet_data_path)
-    remove_local_directory_contents(f"/{TABLE_NAME}")
+    shutil.rmtree(f"/{TABLE_NAME}")
 
 
 def test_restart_broken_table_function(started_cluster):
@@ -553,7 +545,7 @@ def test_restart_broken_table_function(started_cluster):
     assert int(instance.query(f"SELECT count() FROM {TABLE_NAME}")) == 100
 
     os.unlink(parquet_data_path)
-    remove_local_directory_contents(f"/{TABLE_NAME}")
+    shutil.rmtree(f"/{TABLE_NAME}")
 
 
 def test_partition_columns(started_cluster):
@@ -753,5 +745,5 @@ SELECT * FROM deltaLake('http://{started_cluster.minio_ip}:{started_cluster.mini
         == 1
     )
 
-    remove_local_directory_contents(f"/{TABLE_NAME}")
+    shutil.rmtree(f"/{TABLE_NAME}")
     spark.sql(f"DROP TABLE {TABLE_NAME}")

From 01ca36cb5a157ab961dbd4460acc7e2ebb37e72a Mon Sep 17 00:00:00 2001
From: Michael Stetsyuk <mike.stetsyuk@gmail.com>
Date: Fri, 2 Aug 2024 09:37:47 +0100
Subject: [PATCH 1237/1488] empty


From 27f4e1808e4cf299cd8eaf4a19c3bb979aa4e5bd Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Fri, 2 Aug 2024 10:43:02 +0200
Subject: [PATCH 1238/1488] Update
 tests/queries/0_stateless/02434_cancel_insert_when_client_dies.sh

---
 .../0_stateless/02434_cancel_insert_when_client_dies.sh      | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/tests/queries/0_stateless/02434_cancel_insert_when_client_dies.sh b/tests/queries/0_stateless/02434_cancel_insert_when_client_dies.sh
index 45f4194104e..dca8dae22c3 100755
--- a/tests/queries/0_stateless/02434_cancel_insert_when_client_dies.sh
+++ b/tests/queries/0_stateless/02434_cancel_insert_when_client_dies.sh
@@ -10,10 +10,7 @@ export DATA_FILE="$CLICKHOUSE_TMP/deduptest.tsv"
 export TEST_MARK="02434_insert_${CLICKHOUSE_DATABASE}_"
 
 $CLICKHOUSE_CLIENT -q 'select * from numbers(5000000) format TSV' > $DATA_FILE
-$CLICKHOUSE_CLIENT -q "create table dedup_test(A Int64) Engine = MergeTree order by A
-    settings non_replicated_deduplication_window=1000
-    , merge_tree_clear_old_temporary_directories_interval_seconds = 1
-    ;"
+$CLICKHOUSE_CLIENT -q "create table dedup_test(A Int64) Engine = MergeTree order by A settings non_replicated_deduplication_window=1000, merge_tree_clear_old_temporary_directories_interval_seconds = 1;"
 $CLICKHOUSE_CLIENT -q "create table dedup_dist(A Int64) Engine = Distributed('test_cluster_one_shard_two_replicas', currentDatabase(), dedup_test)"
 
 function insert_data

From 6c8f458b0bf9981068c7fecfdd9cef627406419b Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Fri, 2 Aug 2024 11:13:41 +0200
Subject: [PATCH 1239/1488] Fix reloading SQL UDFs with UNION

---
 .../UserDefinedSQLFunctionFactory.cpp         |  8 +++++--
 .../UserDefinedSQLObjectsDiskStorage.cpp      |  6 ++---
 .../UserDefinedSQLObjectsDiskStorage.h        |  1 -
 .../UserDefinedSQLObjectsStorageBase.cpp      | 15 +++++++++---
 .../UserDefinedSQLObjectsStorageBase.h        |  4 ++++
 .../UserDefinedSQLObjectsZooKeeperStorage.cpp |  2 +-
 .../UserDefinedSQLObjectsZooKeeperStorage.h   |  2 --
 .../NormalizeSelectWithUnionQueryVisitor.h    |  2 --
 .../test.py                                   | 23 +++++++++++++++++--
 .../test.py                                   | 12 ++++++++++
 .../03215_udf_with_union.reference            |  1 +
 .../0_stateless/03215_udf_with_union.sql      | 14 +++++++++++
 12 files changed, 74 insertions(+), 16 deletions(-)
 create mode 100644 tests/queries/0_stateless/03215_udf_with_union.reference
 create mode 100644 tests/queries/0_stateless/03215_udf_with_union.sql

diff --git a/src/Functions/UserDefined/UserDefinedSQLFunctionFactory.cpp b/src/Functions/UserDefined/UserDefinedSQLFunctionFactory.cpp
index e6796874e50..d0bc812f91d 100644
--- a/src/Functions/UserDefined/UserDefinedSQLFunctionFactory.cpp
+++ b/src/Functions/UserDefined/UserDefinedSQLFunctionFactory.cpp
@@ -2,6 +2,7 @@
 
 #include <AggregateFunctions/AggregateFunctionFactory.h>
 #include <Backups/RestorerFromBackup.h>
+#include <Core/Settings.h>
 #include <Functions/FunctionFactory.h>
 #include <Functions/UserDefined/IUserDefinedSQLObjectsStorage.h>
 #include <Functions/UserDefined/UserDefinedExecutableFunctionFactory.h>
@@ -9,6 +10,7 @@
 #include <Functions/UserDefined/UserDefinedSQLObjectsBackup.h>
 #include <Interpreters/Context.h>
 #include <Interpreters/FunctionNameNormalizer.h>
+#include <Interpreters/NormalizeSelectWithUnionQueryVisitor.h>
 #include <Parsers/ASTCreateFunctionQuery.h>
 #include <Parsers/ASTFunction.h>
 #include <Parsers/ASTIdentifier.h>
@@ -80,13 +82,15 @@ namespace
         validateFunctionRecursiveness(*function_body, name);
     }
 
-    ASTPtr normalizeCreateFunctionQuery(const IAST & create_function_query)
+    ASTPtr normalizeCreateFunctionQuery(const IAST & create_function_query, const ContextPtr & context)
     {
         auto ptr = create_function_query.clone();
         auto & res = typeid_cast<ASTCreateFunctionQuery &>(*ptr);
         res.if_not_exists = false;
         res.or_replace = false;
         FunctionNameNormalizer::visit(res.function_core.get());
+        NormalizeSelectWithUnionQueryVisitor::Data data{context->getSettingsRef().union_default_mode};
+        NormalizeSelectWithUnionQueryVisitor{data}.visit(res.function_core);
         return ptr;
     }
 }
@@ -125,7 +129,7 @@ void UserDefinedSQLFunctionFactory::checkCanBeUnregistered(const ContextPtr & co
 bool UserDefinedSQLFunctionFactory::registerFunction(const ContextMutablePtr & context, const String & function_name, ASTPtr create_function_query, bool throw_if_exists, bool replace_if_exists)
 {
     checkCanBeRegistered(context, function_name, *create_function_query);
-    create_function_query = normalizeCreateFunctionQuery(*create_function_query);
+    create_function_query = normalizeCreateFunctionQuery(*create_function_query, context);
 
     try
     {
diff --git a/src/Functions/UserDefined/UserDefinedSQLObjectsDiskStorage.cpp b/src/Functions/UserDefined/UserDefinedSQLObjectsDiskStorage.cpp
index 4c004d2537c..8910b45e79d 100644
--- a/src/Functions/UserDefined/UserDefinedSQLObjectsDiskStorage.cpp
+++ b/src/Functions/UserDefined/UserDefinedSQLObjectsDiskStorage.cpp
@@ -1,7 +1,7 @@
 #include "Functions/UserDefined/UserDefinedSQLObjectsDiskStorage.h"
 
-#include "Functions/UserDefined/UserDefinedSQLFunctionFactory.h"
-#include "Functions/UserDefined/UserDefinedSQLObjectType.h"
+#include <Functions/UserDefined/UserDefinedSQLObjectType.h>
+#include <Functions/UserDefined/UserDefinedSQLObjectsStorageBase.h>
 
 #include <Common/StringUtils.h>
 #include <Common/atomicRename.h>
@@ -54,7 +54,7 @@ namespace
 }
 
 UserDefinedSQLObjectsDiskStorage::UserDefinedSQLObjectsDiskStorage(const ContextPtr & global_context_, const String & dir_path_)
-    : global_context(global_context_)
+    : UserDefinedSQLObjectsStorageBase(global_context_)
     , dir_path{makeDirectoryPathCanonical(dir_path_)}
     , log{getLogger("UserDefinedSQLObjectsLoaderFromDisk")}
 {
diff --git a/src/Functions/UserDefined/UserDefinedSQLObjectsDiskStorage.h b/src/Functions/UserDefined/UserDefinedSQLObjectsDiskStorage.h
index ae0cbd0c589..cafbd140598 100644
--- a/src/Functions/UserDefined/UserDefinedSQLObjectsDiskStorage.h
+++ b/src/Functions/UserDefined/UserDefinedSQLObjectsDiskStorage.h
@@ -42,7 +42,6 @@ private:
     ASTPtr tryLoadObject(UserDefinedSQLObjectType object_type, const String & object_name, const String & file_path, bool check_file_exists);
     String getFilePath(UserDefinedSQLObjectType object_type, const String & object_name) const;
 
-    ContextPtr global_context;
     String dir_path;
     LoggerPtr log;
     std::atomic<bool> objects_loaded = false;
diff --git a/src/Functions/UserDefined/UserDefinedSQLObjectsStorageBase.cpp b/src/Functions/UserDefined/UserDefinedSQLObjectsStorageBase.cpp
index f251d11789f..225e919301d 100644
--- a/src/Functions/UserDefined/UserDefinedSQLObjectsStorageBase.cpp
+++ b/src/Functions/UserDefined/UserDefinedSQLObjectsStorageBase.cpp
@@ -2,7 +2,10 @@
 
 #include <boost/container/flat_set.hpp>
 
+#include <Core/Settings.h>
+#include <Interpreters/Context.h>
 #include <Interpreters/FunctionNameNormalizer.h>
+#include <Interpreters/NormalizeSelectWithUnionQueryVisitor.h>
 #include <Parsers/ASTCreateFunctionQuery.h>
 
 namespace DB
@@ -17,18 +20,24 @@ namespace ErrorCodes
 namespace
 {
 
-ASTPtr normalizeCreateFunctionQuery(const IAST & create_function_query)
+ASTPtr normalizeCreateFunctionQuery(const IAST & create_function_query, const ContextPtr & context)
 {
     auto ptr = create_function_query.clone();
     auto & res = typeid_cast<ASTCreateFunctionQuery &>(*ptr);
     res.if_not_exists = false;
     res.or_replace = false;
     FunctionNameNormalizer::visit(res.function_core.get());
+    NormalizeSelectWithUnionQueryVisitor::Data data{context->getSettingsRef().union_default_mode};
+    NormalizeSelectWithUnionQueryVisitor{data}.visit(res.function_core);
     return ptr;
 }
 
 }
 
+UserDefinedSQLObjectsStorageBase::UserDefinedSQLObjectsStorageBase(ContextPtr global_context_)
+    : global_context(std::move(global_context_))
+{}
+
 ASTPtr UserDefinedSQLObjectsStorageBase::get(const String & object_name) const
 {
     std::lock_guard lock(mutex);
@@ -148,7 +157,7 @@ void UserDefinedSQLObjectsStorageBase::setAllObjects(const std::vector<std::pair
 {
     std::unordered_map<String, ASTPtr> normalized_functions;
     for (const auto & [function_name, create_query] : new_objects)
-        normalized_functions[function_name] = normalizeCreateFunctionQuery(*create_query);
+        normalized_functions[function_name] = normalizeCreateFunctionQuery(*create_query, global_context);
 
     std::lock_guard lock(mutex);
     object_name_to_create_object_map = std::move(normalized_functions);
@@ -166,7 +175,7 @@ std::vector<std::pair<String, ASTPtr>> UserDefinedSQLObjectsStorageBase::getAllO
 void UserDefinedSQLObjectsStorageBase::setObject(const String & object_name, const IAST & create_object_query)
 {
     std::lock_guard lock(mutex);
-    object_name_to_create_object_map[object_name] = normalizeCreateFunctionQuery(create_object_query);
+    object_name_to_create_object_map[object_name] = normalizeCreateFunctionQuery(create_object_query, global_context);
 }
 
 void UserDefinedSQLObjectsStorageBase::removeObject(const String & object_name)
diff --git a/src/Functions/UserDefined/UserDefinedSQLObjectsStorageBase.h b/src/Functions/UserDefined/UserDefinedSQLObjectsStorageBase.h
index cab63a3bfcf..0dbc5586f08 100644
--- a/src/Functions/UserDefined/UserDefinedSQLObjectsStorageBase.h
+++ b/src/Functions/UserDefined/UserDefinedSQLObjectsStorageBase.h
@@ -4,6 +4,7 @@
 #include <mutex>
 
 #include <Functions/UserDefined/IUserDefinedSQLObjectsStorage.h>
+#include <Interpreters/Context_fwd.h>
 
 #include <Parsers/IAST.h>
 
@@ -13,6 +14,7 @@ namespace DB
 class UserDefinedSQLObjectsStorageBase : public IUserDefinedSQLObjectsStorage
 {
 public:
+    explicit UserDefinedSQLObjectsStorageBase(ContextPtr global_context_);
     ASTPtr get(const String & object_name) const override;
 
     ASTPtr tryGet(const String & object_name) const override;
@@ -64,6 +66,8 @@ protected:
 
     std::unordered_map<String, ASTPtr> object_name_to_create_object_map;
     mutable std::recursive_mutex mutex;
+
+    ContextPtr global_context;
 };
 
 }
diff --git a/src/Functions/UserDefined/UserDefinedSQLObjectsZooKeeperStorage.cpp b/src/Functions/UserDefined/UserDefinedSQLObjectsZooKeeperStorage.cpp
index 01e7e3995fa..12c1302a3fe 100644
--- a/src/Functions/UserDefined/UserDefinedSQLObjectsZooKeeperStorage.cpp
+++ b/src/Functions/UserDefined/UserDefinedSQLObjectsZooKeeperStorage.cpp
@@ -48,7 +48,7 @@ namespace
 
 UserDefinedSQLObjectsZooKeeperStorage::UserDefinedSQLObjectsZooKeeperStorage(
     const ContextPtr & global_context_, const String & zookeeper_path_)
-    : global_context{global_context_}
+    : UserDefinedSQLObjectsStorageBase(global_context_)
     , zookeeper_getter{[global_context_]() { return global_context_->getZooKeeper(); }}
     , zookeeper_path{zookeeper_path_}
     , watch_queue{std::make_shared<ConcurrentBoundedQueue<std::pair<UserDefinedSQLObjectType, String>>>(std::numeric_limits<size_t>::max())}
diff --git a/src/Functions/UserDefined/UserDefinedSQLObjectsZooKeeperStorage.h b/src/Functions/UserDefined/UserDefinedSQLObjectsZooKeeperStorage.h
index 61002be2bfd..0aa9b198398 100644
--- a/src/Functions/UserDefined/UserDefinedSQLObjectsZooKeeperStorage.h
+++ b/src/Functions/UserDefined/UserDefinedSQLObjectsZooKeeperStorage.h
@@ -68,8 +68,6 @@ private:
     void refreshObjects(const zkutil::ZooKeeperPtr & zookeeper, UserDefinedSQLObjectType object_type);
     void syncObjects(const zkutil::ZooKeeperPtr & zookeeper, UserDefinedSQLObjectType object_type);
 
-    ContextPtr global_context;
-
     zkutil::ZooKeeperCachingGetter zookeeper_getter;
     String zookeeper_path;
     std::atomic<bool> objects_loaded = false;
diff --git a/src/Interpreters/NormalizeSelectWithUnionQueryVisitor.h b/src/Interpreters/NormalizeSelectWithUnionQueryVisitor.h
index b2f55003da5..b642b5def91 100644
--- a/src/Interpreters/NormalizeSelectWithUnionQueryVisitor.h
+++ b/src/Interpreters/NormalizeSelectWithUnionQueryVisitor.h
@@ -4,8 +4,6 @@
 #include <Interpreters/InDepthNodeVisitor.h>
 #include <Parsers/IAST_fwd.h>
 
-#include <unordered_set>
-
 namespace DB
 {
 
diff --git a/tests/integration/test_replicated_user_defined_functions/test.py b/tests/integration/test_replicated_user_defined_functions/test.py
index e5f6683b90b..92d86a8fd2c 100644
--- a/tests/integration/test_replicated_user_defined_functions/test.py
+++ b/tests/integration/test_replicated_user_defined_functions/test.py
@@ -141,6 +141,9 @@ def test_drop_if_exists():
 
 def test_replication():
     node1.query("CREATE FUNCTION f2 AS (x, y) -> x - y")
+    node1.query(
+        "CREATE FUNCTION f3 AS () -> (SELECT sum(s) FROM (SELECT 1 as s UNION ALL SELECT 1 as s))"
+    )
 
     assert (
         node1.query("SELECT create_query FROM system.functions WHERE name='f2'")
@@ -154,7 +157,11 @@ def test_replication():
     assert node1.query("SELECT f2(12,3)") == "9\n"
     assert node2.query("SELECT f2(12,3)") == "9\n"
 
+    assert node1.query("SELECT f3()") == "2\n"
+    assert node2.query("SELECT f3()") == "2\n"
+
     node1.query("DROP FUNCTION f2")
+    node1.query("DROP FUNCTION f3")
     assert (
         node1.query("SELECT create_query FROM system.functions WHERE name='f2'") == ""
     )
@@ -214,7 +221,9 @@ def test_reload_zookeeper():
     )
 
     # config reloads, but can still work
-    node1.query("CREATE FUNCTION f2 AS (x, y) -> x - y")
+    node1.query(
+        "CREATE FUNCTION f2 AS () -> (SELECT sum(s) FROM (SELECT 1 as s UNION ALL SELECT 1 as s))"
+    )
     assert_eq_with_retry(
         node2,
         "SELECT name FROM system.functions WHERE name IN ['f1', 'f2'] ORDER BY name",
@@ -269,7 +278,7 @@ def test_reload_zookeeper():
         TSV(["f1", "f2", "f3"]),
     )
 
-    assert node2.query("SELECT f1(12, 3), f2(12, 3), f3(12, 3)") == TSV([[15, 9, 4]])
+    assert node2.query("SELECT f1(12, 3), f2(), f3(12, 3)") == TSV([[15, 2, 4]])
 
     active_zk_connections = get_active_zk_connections()
     assert (
@@ -307,3 +316,13 @@ def test_start_without_zookeeper():
         "CREATE FUNCTION f1 AS (x, y) -> (x + y)\n",
     )
     node1.query("DROP FUNCTION f1")
+
+
+def test_server_restart():
+    node1.query(
+        "CREATE FUNCTION f1 AS () -> (SELECT sum(s) FROM (SELECT 1 as s UNION ALL SELECT 1 as s))"
+    )
+    assert node1.query("SELECT f1()") == "2\n"
+    node1.restart_clickhouse()
+    assert node1.query("SELECT f1()") == "2\n"
+    node1.query("DROP FUNCTION f1")
diff --git a/tests/integration/test_user_defined_object_persistence/test.py b/tests/integration/test_user_defined_object_persistence/test.py
index 986438a4eed..bd491dfa195 100644
--- a/tests/integration/test_user_defined_object_persistence/test.py
+++ b/tests/integration/test_user_defined_object_persistence/test.py
@@ -18,20 +18,25 @@ def started_cluster():
 def test_persistence():
     create_function_query1 = "CREATE FUNCTION MySum1 AS (a, b) -> a + b"
     create_function_query2 = "CREATE FUNCTION MySum2 AS (a, b) -> MySum1(a, b) + b"
+    create_function_query3 = "CREATE FUNCTION MyUnion AS () -> (SELECT sum(s) FROM (SELECT 1 as s UNION ALL SELECT 1 as s))"
 
     instance.query(create_function_query1)
     instance.query(create_function_query2)
+    instance.query(create_function_query3)
 
     assert instance.query("SELECT MySum1(1,2)") == "3\n"
     assert instance.query("SELECT MySum2(1,2)") == "5\n"
+    assert instance.query("SELECT MyUnion()") == "2\n"
 
     instance.restart_clickhouse()
 
     assert instance.query("SELECT MySum1(1,2)") == "3\n"
     assert instance.query("SELECT MySum2(1,2)") == "5\n"
+    assert instance.query("SELECT MyUnion()") == "2\n"
 
     instance.query("DROP FUNCTION MySum2")
     instance.query("DROP FUNCTION MySum1")
+    instance.query("DROP FUNCTION MyUnion")
 
     instance.restart_clickhouse()
 
@@ -48,3 +53,10 @@ def test_persistence():
         or "Function with name 'MySum2' does not exist. In scope SELECT MySum2(1, 2)"
         in error_message
     )
+
+    error_message = instance.query_and_get_error("SELECT MyUnion()")
+    assert (
+        "Unknown function MyUnion" in error_message
+        or "Function with name 'MyUnion' does not exist. In scope SELECT MyUnion"
+        in error_message
+    )
diff --git a/tests/queries/0_stateless/03215_udf_with_union.reference b/tests/queries/0_stateless/03215_udf_with_union.reference
new file mode 100644
index 00000000000..0cfbf08886f
--- /dev/null
+++ b/tests/queries/0_stateless/03215_udf_with_union.reference
@@ -0,0 +1 @@
+2
diff --git a/tests/queries/0_stateless/03215_udf_with_union.sql b/tests/queries/0_stateless/03215_udf_with_union.sql
new file mode 100644
index 00000000000..00390c5d930
--- /dev/null
+++ b/tests/queries/0_stateless/03215_udf_with_union.sql
@@ -0,0 +1,14 @@
+DROP FUNCTION IF EXISTS 03215_udf_with_union;
+CREATE FUNCTION 03215_udf_with_union AS () -> (
+    SELECT sum(s)
+    FROM
+    (
+        SELECT 1 AS s
+        UNION ALL
+        SELECT 1 AS s
+    )
+);
+
+SELECT 03215_udf_with_union();
+
+DROP FUNCTION 03215_udf_with_union;

From 900e08d6e7c433758d13f5a669c2112bb3856007 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Fri, 2 Aug 2024 11:32:09 +0200
Subject: [PATCH 1240/1488] Try fix

---
 tests/ci/unit_tests_check.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/ci/unit_tests_check.py b/tests/ci/unit_tests_check.py
index 716625d7077..6430fa78801 100644
--- a/tests/ci/unit_tests_check.py
+++ b/tests/ci/unit_tests_check.py
@@ -180,7 +180,7 @@ def main():
     run_command = (
         f"docker run --cap-add=SYS_PTRACE --volume={tests_binary}:/unit_tests_dbms "
         "--security-opt seccomp=unconfined "  # required to issue io_uring sys-calls
-        f"--volume={test_output}:/test_output {docker_image} ${gdb_enabled}"
+        f"--volume={test_output}:/test_output {docker_image} {gdb_enabled}"
     )
 
     run_log_path = test_output / "run.log"

From af53ed4c02ba52b3f57e97941b37a5931620d447 Mon Sep 17 00:00:00 2001
From: kssenii <sumarokovakseniia@mail.ru>
Date: Fri, 2 Aug 2024 12:08:49 +0200
Subject: [PATCH 1241/1488] Ping CI


From 62f0e09ecbb226ea72b5ee8d812436ef75038e33 Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Fri, 2 Aug 2024 12:17:08 +0200
Subject: [PATCH 1242/1488] Fix setting changes

---
 src/Core/SettingsChangesHistory.cpp | 264 +---------------------------
 1 file changed, 2 insertions(+), 262 deletions(-)

diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp
index 2438202f6a3..b6ef654438e 100644
--- a/src/Core/SettingsChangesHistory.cpp
+++ b/src/Core/SettingsChangesHistory.cpp
@@ -57,268 +57,6 @@ String ClickHouseVersion::toString() const
 /// Note: please check if the key already exists to prevent duplicate entries.
 static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory::SettingsChanges>> settings_changes_history_initializer =
 {
-    {"24.7", {{"output_format_parquet_write_page_index", false, true, "Add a possibility to write page index into parquet files."},
-              {"output_format_binary_encode_types_in_binary_format", false, false, "Added new setting to allow to write type names in binary format in RowBinaryWithNamesAndTypes output format"},
-              {"input_format_binary_decode_types_in_binary_format", false, false, "Added new setting to allow to read type names in binary format in RowBinaryWithNamesAndTypes input format"},
-              {"output_format_native_encode_types_in_binary_format", false, false, "Added new setting to allow to write type names in binary format in Native output format"},
-              {"input_format_native_decode_types_in_binary_format", false, false, "Added new setting to allow to read type names in binary format in Native output format"},
-              {"read_in_order_use_buffering", false, true, "Use buffering before merging while reading in order of primary key"},
-              {"enable_named_columns_in_function_tuple", false, true, "Generate named tuples in function tuple() when all names are unique and can be treated as unquoted identifiers."},
-              {"input_format_json_case_insensitive_column_matching", false, false, "Ignore case when matching JSON keys with CH columns."},
-              {"optimize_trivial_insert_select", true, false, "The optimization does not make sense in many cases."},
-              {"dictionary_validate_primary_key_type", false, false, "Validate primary key type for dictionaries. By default id type for simple layouts will be implicitly converted to UInt64."},
-              {"collect_hash_table_stats_during_joins", false, true, "New setting."},
-              {"max_size_to_preallocate_for_joins", 0, 100'000'000, "New setting."},
-              {"input_format_orc_read_use_writer_time_zone", false, false, "Whether use the writer's time zone in ORC stripe for ORC row reader, the default ORC row reader's time zone is GMT."},
-              {"lightweight_mutation_projection_mode", "throw", "throw", "When lightweight delete happens on a table with projection(s), the possible operations include throw the exception as projection exists, or drop all projection related to this table then do lightweight delete."},
-              {"database_replicated_allow_heavy_create", true, false, "Long-running DDL queries (CREATE AS SELECT and POPULATE) for Replicated database engine was forbidden"},
-              {"query_plan_merge_filters", false, false, "Allow to merge filters in the query plan"},
-              {"azure_sdk_max_retries", 10, 10, "Maximum number of retries in azure sdk"},
-              {"azure_sdk_retry_initial_backoff_ms", 10, 10, "Minimal backoff between retries in azure sdk"},
-              {"azure_sdk_retry_max_backoff_ms", 1000, 1000, "Maximal backoff between retries in azure sdk"},
-              {"merge_tree_min_bytes_per_task_for_remote_reading", 4194304, 2097152, "Value is unified with `filesystem_prefetch_min_bytes_for_single_read_task`"},
-              {"ignore_on_cluster_for_replicated_named_collections_queries", false, false, "Ignore ON CLUSTER clause for replicated named collections management queries."},
-              {"backup_restore_s3_retry_attempts", 1000,1000, "Setting for Aws::Client::RetryStrategy, Aws::Client does retries itself, 0 means no retries. It takes place only for backup/restore."},
-              {"postgresql_connection_attempt_timeout", 2, 2, "Allow to control 'connect_timeout' parameter of PostgreSQL connection."},
-              {"postgresql_connection_pool_retries", 2, 2, "Allow to control the number of retries in PostgreSQL connection pool."},
-              {"restore_replace_external_table_functions_to_null", false, false, "New setting."},
-              {"restore_replace_external_engines_to_null", false, false, "New setting."}
-              }},
-    {"24.6", {{"materialize_skip_indexes_on_insert", true, true, "Added new setting to allow to disable materialization of skip indexes on insert"},
-              {"materialize_statistics_on_insert", true, true, "Added new setting to allow to disable materialization of statistics on insert"},
-              {"input_format_parquet_use_native_reader", false, false, "When reading Parquet files, to use native reader instead of arrow reader."},
-              {"hdfs_throw_on_zero_files_match", false, false, "Allow to throw an error when ListObjects request cannot match any files in HDFS engine instead of empty query result"},
-              {"azure_throw_on_zero_files_match", false, false, "Allow to throw an error when ListObjects request cannot match any files in AzureBlobStorage engine instead of empty query result"},
-              {"s3_validate_request_settings", true, true, "Allow to disable S3 request settings validation"},
-              {"allow_experimental_full_text_index", false, false, "Enable experimental full-text index"},
-              {"azure_skip_empty_files", false, false, "Allow to skip empty files in azure table engine"},
-              {"hdfs_ignore_file_doesnt_exist", false, false, "Allow to return 0 rows when the requested files don't exist instead of throwing an exception in HDFS table engine"},
-              {"azure_ignore_file_doesnt_exist", false, false, "Allow to return 0 rows when the requested files don't exist instead of throwing an exception in AzureBlobStorage table engine"},
-              {"s3_ignore_file_doesnt_exist", false, false, "Allow to return 0 rows when the requested files don't exist instead of throwing an exception in S3 table engine"},
-              {"s3_max_part_number", 10000, 10000, "Maximum part number number for s3 upload part"},
-              {"s3_max_single_operation_copy_size", 32 * 1024 * 1024, 32 * 1024 * 1024, "Maximum size for a single copy operation in s3"},
-              {"input_format_parquet_max_block_size", 8192, DEFAULT_BLOCK_SIZE, "Increase block size for parquet reader."},
-              {"input_format_parquet_prefer_block_bytes", 0, DEFAULT_BLOCK_SIZE * 256, "Average block bytes output by parquet reader."},
-              {"enable_blob_storage_log", true, true, "Write information about blob storage operations to system.blob_storage_log table"},
-              {"allow_deprecated_snowflake_conversion_functions", true, false, "Disabled deprecated functions snowflakeToDateTime[64] and dateTime[64]ToSnowflake."},
-              {"allow_statistic_optimize", false, false, "Old setting which popped up here being renamed."},
-              {"allow_experimental_statistic", false, false, "Old setting which popped up here being renamed."},
-              {"allow_statistics_optimize", false, false, "The setting was renamed. The previous name is `allow_statistic_optimize`."},
-              {"allow_experimental_statistics", false, false, "The setting was renamed. The previous name is `allow_experimental_statistic`."},
-              {"enable_vertical_final", false, true, "Enable vertical final by default again after fixing bug"},
-              {"parallel_replicas_custom_key_range_lower", 0, 0, "Add settings to control the range filter when using parallel replicas with dynamic shards"},
-              {"parallel_replicas_custom_key_range_upper", 0, 0, "Add settings to control the range filter when using parallel replicas with dynamic shards. A value of 0 disables the upper limit"},
-              {"output_format_pretty_display_footer_column_names", 0, 1, "Add a setting to display column names in the footer if there are many rows. Threshold value is controlled by output_format_pretty_display_footer_column_names_min_rows."},
-              {"output_format_pretty_display_footer_column_names_min_rows", 0, 50, "Add a setting to control the threshold value for setting output_format_pretty_display_footer_column_names_min_rows. Default 50."},
-              {"output_format_csv_serialize_tuple_into_separate_columns", true, true, "A new way of how interpret tuples in CSV format was added."},
-              {"input_format_csv_deserialize_separate_columns_into_tuple", true, true, "A new way of how interpret tuples in CSV format was added."},
-              {"input_format_csv_try_infer_strings_from_quoted_tuples", true, true, "A new way of how interpret tuples in CSV format was added."},
-              }},
-    {"24.5", {{"allow_deprecated_error_prone_window_functions", true, false, "Allow usage of deprecated error prone window functions (neighbor, runningAccumulate, runningDifferenceStartingWithFirstValue, runningDifference)"},
-              {"allow_experimental_join_condition", false, false, "Support join with inequal conditions which involve columns from both left and right table. e.g. t1.y < t2.y."},
-              {"input_format_tsv_crlf_end_of_line", false, false, "Enables reading of CRLF line endings with TSV formats"},
-              {"output_format_parquet_use_custom_encoder", false, true, "Enable custom Parquet encoder."},
-              {"cross_join_min_rows_to_compress", 0, 10000000, "Minimal count of rows to compress block in CROSS JOIN. Zero value means - disable this threshold. This block is compressed when any of the two thresholds (by rows or by bytes) are reached."},
-              {"cross_join_min_bytes_to_compress", 0, 1_GiB, "Minimal size of block to compress in CROSS JOIN. Zero value means - disable this threshold. This block is compressed when any of the two thresholds (by rows or by bytes) are reached."},
-              {"http_max_chunk_size", 0, 0, "Internal limitation"},
-              {"prefer_external_sort_block_bytes", 0, DEFAULT_BLOCK_SIZE * 256, "Prefer maximum block bytes for external sort, reduce the memory usage during merging."},
-              {"input_format_force_null_for_omitted_fields", false, false, "Disable type-defaults for omitted fields when needed"},
-              {"cast_string_to_dynamic_use_inference", false, false, "Add setting to allow converting String to Dynamic through parsing"},
-              {"allow_experimental_dynamic_type", false, false, "Add new experimental Dynamic type"},
-              {"azure_max_blocks_in_multipart_upload", 50000, 50000, "Maximum number of blocks in multipart upload for Azure."},
-              {"allow_archive_path_syntax", false, true, "Added new setting to allow disabling archive path syntax."},
-              }},
-    {"24.4", {{"input_format_json_throw_on_bad_escape_sequence", true, true, "Allow to save JSON strings with bad escape sequences"},
-              {"max_parsing_threads", 0, 0, "Add a separate setting to control number of threads in parallel parsing from files"},
-              {"ignore_drop_queries_probability", 0, 0, "Allow to ignore drop queries in server with specified probability for testing purposes"},
-              {"lightweight_deletes_sync", 2, 2, "The same as 'mutation_sync', but controls only execution of lightweight deletes"},
-              {"query_cache_system_table_handling", "save", "throw", "The query cache no longer caches results of queries against system tables"},
-              {"input_format_json_ignore_unnecessary_fields", false, true, "Ignore unnecessary fields and not parse them. Enabling this may not throw exceptions on json strings of invalid format or with duplicated fields"},
-              {"input_format_hive_text_allow_variable_number_of_columns", false, true, "Ignore extra columns in Hive Text input (if file has more columns than expected) and treat missing fields in Hive Text input as default values."},
-              {"allow_experimental_database_replicated", false, true, "Database engine Replicated is now in Beta stage"},
-              {"temporary_data_in_cache_reserve_space_wait_lock_timeout_milliseconds", (10 * 60 * 1000), (10 * 60 * 1000), "Wait time to lock cache for sapce reservation in temporary data in filesystem cache"},
-              {"optimize_rewrite_sum_if_to_count_if", false, true, "Only available for the analyzer, where it works correctly"},
-              {"azure_allow_parallel_part_upload", "true", "true", "Use multiple threads for azure multipart upload."},
-              {"max_recursive_cte_evaluation_depth", DBMS_RECURSIVE_CTE_MAX_EVALUATION_DEPTH, DBMS_RECURSIVE_CTE_MAX_EVALUATION_DEPTH, "Maximum limit on recursive CTE evaluation depth"},
-              {"query_plan_convert_outer_join_to_inner_join", false, true, "Allow to convert OUTER JOIN to INNER JOIN if filter after JOIN always filters default values"},
-              }},
-    {"24.3", {{"s3_connect_timeout_ms", 1000, 1000, "Introduce new dedicated setting for s3 connection timeout"},
-              {"allow_experimental_shared_merge_tree", false, true, "The setting is obsolete"},
-              {"use_page_cache_for_disks_without_file_cache", false, false, "Added userspace page cache"},
-              {"read_from_page_cache_if_exists_otherwise_bypass_cache", false, false, "Added userspace page cache"},
-              {"page_cache_inject_eviction", false, false, "Added userspace page cache"},
-              {"default_table_engine", "None", "MergeTree", "Set default table engine to MergeTree for better usability"},
-              {"input_format_json_use_string_type_for_ambiguous_paths_in_named_tuples_inference_from_objects", false, false, "Allow to use String type for ambiguous paths during named tuple inference from JSON objects"},
-              {"traverse_shadow_remote_data_paths", false, false, "Traverse shadow directory when query system.remote_data_paths."},
-              {"throw_if_deduplication_in_dependent_materialized_views_enabled_with_async_insert", false, true, "Deduplication in dependent materialized view cannot work together with async inserts."},
-              {"parallel_replicas_allow_in_with_subquery", false, true, "If true, subquery for IN will be executed on every follower replica"},
-              {"log_processors_profiles", false, true, "Enable by default"},
-              {"function_locate_has_mysql_compatible_argument_order", false, true, "Increase compatibility with MySQL's locate function."},
-              {"allow_suspicious_primary_key", true, false, "Forbid suspicious PRIMARY KEY/ORDER BY for MergeTree (i.e. SimpleAggregateFunction)"},
-              {"filesystem_cache_reserve_space_wait_lock_timeout_milliseconds", 1000, 1000, "Wait time to lock cache for sapce reservation in filesystem cache"},
-              {"max_parser_backtracks", 0, 1000000, "Limiting the complexity of parsing"},
-              {"analyzer_compatibility_join_using_top_level_identifier", false, false, "Force to resolve identifier in JOIN USING from projection"},
-              {"distributed_insert_skip_read_only_replicas", false, false, "If true, INSERT into Distributed will skip read-only replicas"},
-              {"keeper_max_retries", 10, 10, "Max retries for general keeper operations"},
-              {"keeper_retry_initial_backoff_ms", 100, 100, "Initial backoff timeout for general keeper operations"},
-              {"keeper_retry_max_backoff_ms", 5000, 5000, "Max backoff timeout for general keeper operations"},
-              {"s3queue_allow_experimental_sharded_mode", false, false, "Enable experimental sharded mode of S3Queue table engine. It is experimental because it will be rewritten"},
-              {"allow_experimental_analyzer", false, true, "Enable analyzer and planner by default."},
-              {"merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability", 0.0, 0.0, "For testing of `PartsSplitter` - split read ranges into intersecting and non intersecting every time you read from MergeTree with the specified probability."},
-              {"allow_get_client_http_header", false, false, "Introduced a new function."},
-              {"output_format_pretty_row_numbers", false, true, "It is better for usability."},
-              {"output_format_pretty_max_value_width_apply_for_single_value", true, false, "Single values in Pretty formats won't be cut."},
-              {"output_format_parquet_string_as_string", false, true, "ClickHouse allows arbitrary binary data in the String data type, which is typically UTF-8. Parquet/ORC/Arrow Strings only support UTF-8. That's why you can choose which Arrow's data type to use for the ClickHouse String data type - String or Binary. While Binary would be more correct and compatible, using String by default will correspond to user expectations in most cases."},
-              {"output_format_orc_string_as_string", false, true, "ClickHouse allows arbitrary binary data in the String data type, which is typically UTF-8. Parquet/ORC/Arrow Strings only support UTF-8. That's why you can choose which Arrow's data type to use for the ClickHouse String data type - String or Binary. While Binary would be more correct and compatible, using String by default will correspond to user expectations in most cases."},
-              {"output_format_arrow_string_as_string", false, true, "ClickHouse allows arbitrary binary data in the String data type, which is typically UTF-8. Parquet/ORC/Arrow Strings only support UTF-8. That's why you can choose which Arrow's data type to use for the ClickHouse String data type - String or Binary. While Binary would be more correct and compatible, using String by default will correspond to user expectations in most cases."},
-              {"output_format_parquet_compression_method", "lz4", "zstd", "Parquet/ORC/Arrow support many compression methods, including lz4 and zstd. ClickHouse supports each and every compression method. Some inferior tools, such as 'duckdb', lack support for the faster `lz4` compression method, that's why we set zstd by default."},
-              {"output_format_orc_compression_method", "lz4", "zstd", "Parquet/ORC/Arrow support many compression methods, including lz4 and zstd. ClickHouse supports each and every compression method. Some inferior tools, such as 'duckdb', lack support for the faster `lz4` compression method, that's why we set zstd by default."},
-              {"output_format_pretty_highlight_digit_groups", false, true, "If enabled and if output is a terminal, highlight every digit corresponding to the number of thousands, millions, etc. with underline."},
-              {"geo_distance_returns_float64_on_float64_arguments", false, true, "Increase the default precision."},
-              {"azure_max_inflight_parts_for_one_file", 20, 20, "The maximum number of a concurrent loaded parts in multipart upload request. 0 means unlimited."},
-              {"azure_strict_upload_part_size", 0, 0, "The exact size of part to upload during multipart upload to Azure blob storage."},
-              {"azure_min_upload_part_size", 16*1024*1024, 16*1024*1024, "The minimum size of part to upload during multipart upload to Azure blob storage."},
-              {"azure_max_upload_part_size", 5ull*1024*1024*1024, 5ull*1024*1024*1024, "The maximum size of part to upload during multipart upload to Azure blob storage."},
-              {"azure_upload_part_size_multiply_factor", 2, 2, "Multiply azure_min_upload_part_size by this factor each time azure_multiply_parts_count_threshold parts were uploaded from a single write to Azure blob storage."},
-              {"azure_upload_part_size_multiply_parts_count_threshold", 500, 500, "Each time this number of parts was uploaded to Azure blob storage, azure_min_upload_part_size is multiplied by azure_upload_part_size_multiply_factor."},
-              {"output_format_csv_serialize_tuple_into_separate_columns", true, true, "A new way of how interpret tuples in CSV format was added."},
-              {"input_format_csv_deserialize_separate_columns_into_tuple", true, true, "A new way of how interpret tuples in CSV format was added."},
-              {"input_format_csv_try_infer_strings_from_quoted_tuples", true, true, "A new way of how interpret tuples in CSV format was added."},
-              }},
-    {"24.2", {{"allow_suspicious_variant_types", true, false, "Don't allow creating Variant type with suspicious variants by default"},
-              {"validate_experimental_and_suspicious_types_inside_nested_types", false, true, "Validate usage of experimental and suspicious types inside nested types"},
-              {"output_format_values_escape_quote_with_quote", false, false, "If true escape ' with '', otherwise quoted with \\'"},
-              {"output_format_pretty_single_large_number_tip_threshold", 0, 1'000'000, "Print a readable number tip on the right side of the table if the block consists of a single number which exceeds this value (except 0)"},
-              {"input_format_try_infer_exponent_floats", true, false, "Don't infer floats in exponential notation by default"},
-              {"query_plan_optimize_prewhere", true, true, "Allow to push down filter to PREWHERE expression for supported storages"},
-              {"async_insert_max_data_size", 1000000, 10485760, "The previous value appeared to be too small."},
-              {"async_insert_poll_timeout_ms", 10, 10, "Timeout in milliseconds for polling data from asynchronous insert queue"},
-              {"async_insert_use_adaptive_busy_timeout", false, true, "Use adaptive asynchronous insert timeout"},
-              {"async_insert_busy_timeout_min_ms", 50, 50, "The minimum value of the asynchronous insert timeout in milliseconds; it also serves as the initial value, which may be increased later by the adaptive algorithm"},
-              {"async_insert_busy_timeout_max_ms", 200, 200, "The minimum value of the asynchronous insert timeout in milliseconds; async_insert_busy_timeout_ms is aliased to async_insert_busy_timeout_max_ms"},
-              {"async_insert_busy_timeout_increase_rate", 0.2, 0.2, "The exponential growth rate at which the adaptive asynchronous insert timeout increases"},
-              {"async_insert_busy_timeout_decrease_rate", 0.2, 0.2, "The exponential growth rate at which the adaptive asynchronous insert timeout decreases"},
-              {"format_template_row_format", "", "", "Template row format string can be set directly in query"},
-              {"format_template_resultset_format", "", "", "Template result set format string can be set in query"},
-              {"split_parts_ranges_into_intersecting_and_non_intersecting_final", true, true, "Allow to split parts ranges into intersecting and non intersecting during FINAL optimization"},
-              {"split_intersecting_parts_ranges_into_layers_final", true, true, "Allow to split intersecting parts ranges into layers during FINAL optimization"},
-              {"azure_max_single_part_copy_size", 256*1024*1024, 256*1024*1024, "The maximum size of object to copy using single part copy to Azure blob storage."},
-              {"min_external_table_block_size_rows", DEFAULT_INSERT_BLOCK_SIZE, DEFAULT_INSERT_BLOCK_SIZE, "Squash blocks passed to external table to specified size in rows, if blocks are not big enough"},
-              {"min_external_table_block_size_bytes", DEFAULT_INSERT_BLOCK_SIZE * 256, DEFAULT_INSERT_BLOCK_SIZE * 256, "Squash blocks passed to external table to specified size in bytes, if blocks are not big enough."},
-              {"parallel_replicas_prefer_local_join", true, true, "If true, and JOIN can be executed with parallel replicas algorithm, and all storages of right JOIN part are *MergeTree, local JOIN will be used instead of GLOBAL JOIN."},
-              {"optimize_time_filter_with_preimage", true, true, "Optimize Date and DateTime predicates by converting functions into equivalent comparisons without conversions (e.g. toYear(col) = 2023 -> col >= '2023-01-01' AND col <= '2023-12-31')"},
-              {"extract_key_value_pairs_max_pairs_per_row", 0, 0, "Max number of pairs that can be produced by the `extractKeyValuePairs` function. Used as a safeguard against consuming too much memory."},
-              {"default_view_definer", "CURRENT_USER", "CURRENT_USER", "Allows to set default `DEFINER` option while creating a view"},
-              {"default_materialized_view_sql_security", "DEFINER", "DEFINER", "Allows to set a default value for SQL SECURITY option when creating a materialized view"},
-              {"default_normal_view_sql_security", "INVOKER", "INVOKER", "Allows to set default `SQL SECURITY` option while creating a normal view"},
-              {"mysql_map_string_to_text_in_show_columns", false, true, "Reduce the configuration effort to connect ClickHouse with BI tools."},
-              {"mysql_map_fixed_string_to_text_in_show_columns", false, true, "Reduce the configuration effort to connect ClickHouse with BI tools."},
-              }},
-    {"24.1", {{"print_pretty_type_names", false, true, "Better user experience."},
-              {"input_format_json_read_bools_as_strings", false, true, "Allow to read bools as strings in JSON formats by default"},
-              {"output_format_arrow_use_signed_indexes_for_dictionary", false, true, "Use signed indexes type for Arrow dictionaries by default as it's recommended"},
-              {"allow_experimental_variant_type", false, false, "Add new experimental Variant type"},
-              {"use_variant_as_common_type", false, false, "Allow to use Variant in if/multiIf if there is no common type"},
-              {"output_format_arrow_use_64_bit_indexes_for_dictionary", false, false, "Allow to use 64 bit indexes type in Arrow dictionaries"},
-              {"parallel_replicas_mark_segment_size", 128, 128, "Add new setting to control segment size in new parallel replicas coordinator implementation"},
-              {"ignore_materialized_views_with_dropped_target_table", false, false, "Add new setting to allow to ignore materialized views with dropped target table"},
-              {"output_format_compression_level", 3, 3, "Allow to change compression level in the query output"},
-              {"output_format_compression_zstd_window_log", 0, 0, "Allow to change zstd window log in the query output when zstd compression is used"},
-              {"enable_zstd_qat_codec", false, false, "Add new ZSTD_QAT codec"},
-              {"enable_vertical_final", false, true, "Use vertical final by default"},
-              {"output_format_arrow_use_64_bit_indexes_for_dictionary", false, false, "Allow to use 64 bit indexes type in Arrow dictionaries"},
-              {"max_rows_in_set_to_optimize_join", 100000, 0, "Disable join optimization as it prevents from read in order optimization"},
-              {"output_format_pretty_color", true, "auto", "Setting is changed to allow also for auto value, disabling ANSI escapes if output is not a tty"},
-              {"function_visible_width_behavior", 0, 1, "We changed the default behavior of `visibleWidth` to be more precise"},
-              {"max_estimated_execution_time", 0, 0, "Separate max_execution_time and max_estimated_execution_time"},
-              {"iceberg_engine_ignore_schema_evolution", false, false, "Allow to ignore schema evolution in Iceberg table engine"},
-              {"optimize_injective_functions_in_group_by", false, true, "Replace injective functions by it's arguments in GROUP BY section in analyzer"},
-              {"update_insert_deduplication_token_in_dependent_materialized_views", false, false, "Allow to update insert deduplication token with table identifier during insert in dependent materialized views"},
-              {"azure_max_unexpected_write_error_retries", 4, 4, "The maximum number of retries in case of unexpected errors during Azure blob storage write"},
-              {"split_parts_ranges_into_intersecting_and_non_intersecting_final", false, true, "Allow to split parts ranges into intersecting and non intersecting during FINAL optimization"},
-              {"split_intersecting_parts_ranges_into_layers_final", true, true, "Allow to split intersecting parts ranges into layers during FINAL optimization"}}},
-    {"23.12", {{"allow_suspicious_ttl_expressions", true, false, "It is a new setting, and in previous versions the behavior was equivalent to allowing."},
-              {"input_format_parquet_allow_missing_columns", false, true, "Allow missing columns in Parquet files by default"},
-              {"input_format_orc_allow_missing_columns", false, true, "Allow missing columns in ORC files by default"},
-              {"input_format_arrow_allow_missing_columns", false, true, "Allow missing columns in Arrow files by default"}}},
-    {"23.11", {{"parsedatetime_parse_without_leading_zeros", false, true, "Improved compatibility with MySQL DATE_FORMAT/STR_TO_DATE"}}},
-    {"23.9", {{"optimize_group_by_constant_keys", false, true, "Optimize group by constant keys by default"},
-              {"input_format_json_try_infer_named_tuples_from_objects", false, true, "Try to infer named Tuples from JSON objects by default"},
-              {"input_format_json_read_numbers_as_strings", false, true, "Allow to read numbers as strings in JSON formats by default"},
-              {"input_format_json_read_arrays_as_strings", false, true, "Allow to read arrays as strings in JSON formats by default"},
-              {"input_format_json_infer_incomplete_types_as_strings", false, true, "Allow to infer incomplete types as Strings in JSON formats by default"},
-              {"input_format_json_try_infer_numbers_from_strings", true, false, "Don't infer numbers from strings in JSON formats by default to prevent possible parsing errors"},
-              {"http_write_exception_in_output_format", false, true, "Output valid JSON/XML on exception in HTTP streaming."}}},
-    {"23.8", {{"rewrite_count_distinct_if_with_count_distinct_implementation", false, true, "Rewrite countDistinctIf with count_distinct_implementation configuration"}}},
-    {"23.7", {{"function_sleep_max_microseconds_per_block", 0, 3000000, "In previous versions, the maximum sleep time of 3 seconds was applied only for `sleep`, but not for `sleepEachRow` function. In the new version, we introduce this setting. If you set compatibility with the previous versions, we will disable the limit altogether."}}},
-    {"23.6", {{"http_send_timeout", 180, 30, "3 minutes seems crazy long. Note that this is timeout for a single network write call, not for the whole upload operation."},
-              {"http_receive_timeout", 180, 30, "See http_send_timeout."}}},
-    {"23.5", {{"input_format_parquet_preserve_order", true, false, "Allow Parquet reader to reorder rows for better parallelism."},
-              {"parallelize_output_from_storages", false, true, "Allow parallelism when executing queries that read from file/url/s3/etc. This may reorder rows."},
-              {"use_with_fill_by_sorting_prefix", false, true, "Columns preceding WITH FILL columns in ORDER BY clause form sorting prefix. Rows with different values in sorting prefix are filled independently"},
-              {"output_format_parquet_compliant_nested_types", false, true, "Change an internal field name in output Parquet file schema."}}},
-    {"23.4", {{"allow_suspicious_indices", true, false, "If true, index can defined with identical expressions"},
-              {"allow_nonconst_timezone_arguments", true, false, "Allow non-const timezone arguments in certain time-related functions like toTimeZone(), fromUnixTimestamp*(), snowflakeToDateTime*()."},
-              {"connect_timeout_with_failover_ms", 50, 1000, "Increase default connect timeout because of async connect"},
-              {"connect_timeout_with_failover_secure_ms", 100, 1000, "Increase default secure connect timeout because of async connect"},
-              {"hedged_connection_timeout_ms", 100, 50, "Start new connection in hedged requests after 50 ms instead of 100 to correspond with previous connect timeout"},
-              {"formatdatetime_f_prints_single_zero", true, false, "Improved compatibility with MySQL DATE_FORMAT()/STR_TO_DATE()"},
-              {"formatdatetime_parsedatetime_m_is_month_name", false, true, "Improved compatibility with MySQL DATE_FORMAT/STR_TO_DATE"}}},
-    {"23.3", {{"output_format_parquet_version", "1.0", "2.latest", "Use latest Parquet format version for output format"},
-              {"input_format_json_ignore_unknown_keys_in_named_tuple", false, true, "Improve parsing JSON objects as named tuples"},
-              {"input_format_native_allow_types_conversion", false, true, "Allow types conversion in Native input forma"},
-              {"output_format_arrow_compression_method", "none", "lz4_frame", "Use lz4 compression in Arrow output format by default"},
-              {"output_format_parquet_compression_method", "snappy", "lz4", "Use lz4 compression in Parquet output format by default"},
-              {"output_format_orc_compression_method", "none", "lz4_frame", "Use lz4 compression in ORC output format by default"},
-              {"async_query_sending_for_remote", false, true, "Create connections and send query async across shards"}}},
-    {"23.2", {{"output_format_parquet_fixed_string_as_fixed_byte_array", false, true, "Use Parquet FIXED_LENGTH_BYTE_ARRAY type for FixedString by default"},
-              {"output_format_arrow_fixed_string_as_fixed_byte_array", false, true, "Use Arrow FIXED_SIZE_BINARY type for FixedString by default"},
-              {"query_plan_remove_redundant_distinct", false, true, "Remove redundant Distinct step in query plan"},
-              {"optimize_duplicate_order_by_and_distinct", true, false, "Remove duplicate ORDER BY and DISTINCT if it's possible"},
-              {"insert_keeper_max_retries", 0, 20, "Enable reconnections to Keeper on INSERT, improve reliability"}}},
-    {"23.1", {{"input_format_json_read_objects_as_strings", 0, 1, "Enable reading nested json objects as strings while object type is experimental"},
-              {"input_format_json_defaults_for_missing_elements_in_named_tuple", false, true, "Allow missing elements in JSON objects while reading named tuples by default"},
-              {"input_format_csv_detect_header", false, true, "Detect header in CSV format by default"},
-              {"input_format_tsv_detect_header", false, true, "Detect header in TSV format by default"},
-              {"input_format_custom_detect_header", false, true, "Detect header in CustomSeparated format by default"},
-              {"query_plan_remove_redundant_sorting", false, true, "Remove redundant sorting in query plan. For example, sorting steps related to ORDER BY clauses in subqueries"}}},
-    {"22.12", {{"max_size_to_preallocate_for_aggregation", 10'000'000, 100'000'000, "This optimizes performance"},
-               {"query_plan_aggregation_in_order", 0, 1, "Enable some refactoring around query plan"},
-               {"format_binary_max_string_size", 0, 1_GiB, "Prevent allocating large amount of memory"}}},
-    {"22.11", {{"use_structure_from_insertion_table_in_table_functions", 0, 2, "Improve using structure from insertion table in table functions"}}},
-    {"22.9", {{"force_grouping_standard_compatibility", false, true, "Make GROUPING function output the same as in SQL standard and other DBMS"}}},
-    {"22.7", {{"cross_to_inner_join_rewrite", 1, 2, "Force rewrite comma join to inner"},
-              {"enable_positional_arguments", false, true, "Enable positional arguments feature by default"},
-              {"format_csv_allow_single_quotes", true, false, "Most tools don't treat single quote in CSV specially, don't do it by default too"}}},
-    {"22.6", {{"output_format_json_named_tuples_as_objects", false, true, "Allow to serialize named tuples as JSON objects in JSON formats by default"},
-              {"input_format_skip_unknown_fields", false, true, "Optimize reading subset of columns for some input formats"}}},
-    {"22.5", {{"memory_overcommit_ratio_denominator", 0, 1073741824, "Enable memory overcommit feature by default"},
-              {"memory_overcommit_ratio_denominator_for_user", 0, 1073741824, "Enable memory overcommit feature by default"}}},
-    {"22.4", {{"allow_settings_after_format_in_insert", true, false, "Do not allow SETTINGS after FORMAT for INSERT queries because ClickHouse interpret SETTINGS as some values, which is misleading"}}},
-    {"22.3", {{"cast_ipv4_ipv6_default_on_conversion_error", true, false, "Make functions cast(value, 'IPv4') and cast(value, 'IPv6') behave same as toIPv4 and toIPv6 functions"}}},
-    {"21.12", {{"stream_like_engine_allow_direct_select", true, false, "Do not allow direct select for Kafka/RabbitMQ/FileLog by default"}}},
-    {"21.9", {{"output_format_decimal_trailing_zeros", true, false, "Do not output trailing zeros in text representation of Decimal types by default for better looking output"},
-              {"use_hedged_requests", false, true, "Enable Hedged Requests feature by default"}}},
-    {"21.7", {{"legacy_column_name_of_tuple_literal", true, false, "Add this setting only for compatibility reasons. It makes sense to set to 'true', while doing rolling update of cluster from version lower than 21.7 to higher"}}},
-    {"21.5", {{"async_socket_for_remote", false, true, "Fix all problems and turn on asynchronous reads from socket for remote queries by default again"}}},
-    {"21.3", {{"async_socket_for_remote", true, false, "Turn off asynchronous reads from socket for remote queries because of some problems"},
-              {"optimize_normalize_count_variants", false, true, "Rewrite aggregate functions that semantically equals to count() as count() by default"},
-              {"normalize_function_names", false, true, "Normalize function names to their canonical names, this was needed for projection query routing"}}},
-    {"21.2", {{"enable_global_with_statement", false, true, "Propagate WITH statements to UNION queries and all subqueries by default"}}},
-    {"21.1", {{"insert_quorum_parallel", false, true, "Use parallel quorum inserts by default. It is significantly more convenient to use than sequential quorum inserts"},
-              {"input_format_null_as_default", false, true, "Allow to insert NULL as default for input formats by default"},
-              {"optimize_on_insert", false, true, "Enable data optimization on INSERT by default for better user experience"},
-              {"use_compact_format_in_distributed_parts_names", false, true, "Use compact format for async INSERT into Distributed tables by default"}}},
-    {"20.10", {{"format_regexp_escaping_rule", "Escaped", "Raw", "Use Raw as default escaping rule for Regexp format to male the behaviour more like to what users expect"}}},
-    {"20.7", {{"show_table_uuid_in_table_create_query_if_not_nil", true, false, "Stop showing  UID of the table in its CREATE query for Engine=Atomic"}}},
-    {"20.5", {{"input_format_with_names_use_header", false, true, "Enable using header with names for formats with WithNames/WithNamesAndTypes suffixes"},
-              {"allow_suspicious_codecs", true, false, "Don't allow to specify meaningless compression codecs"}}},
-    {"20.4", {{"validate_polygons", false, true, "Throw exception if polygon is invalid in function pointInPolygon by default instead of returning possibly wrong results"}}},
-    {"19.18", {{"enable_scalar_subquery_optimization", false, true, "Prevent scalar subqueries from (de)serializing large scalar values and possibly avoid running the same subquery more than once"}}},
-    {"19.14", {{"any_join_distinct_right_table_keys", true, false, "Disable ANY RIGHT and ANY FULL JOINs by default to avoid inconsistency"}}},
-    {"19.12", {{"input_format_defaults_for_omitted_fields", false, true, "Enable calculation of complex default expressions for omitted fields for some input formats, because it should be the expected behaviour"}}},
-    {"19.5", {{"max_partitions_per_insert_block", 0, 100, "Add a limit for the number of partitions in one block"}}},
-    {"18.12.17", {{"enable_optimize_predicate_expression", 0, 1, "Optimize predicates to subqueries by default"}}},
     {"24.12",
         {
         }
@@ -338,6 +76,7 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
     {"24.8",
         {
             {"merge_tree_min_bytes_per_task_for_remote_reading", 4194304, 2097152, "Value is unified with `filesystem_prefetch_min_bytes_for_single_read_task`"},
+            {"allow_archive_path_syntax", true, true, "Added new setting to allow disabling archive path syntax."},
         }
     },
     {"24.7",
@@ -413,6 +152,7 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
             {"cast_string_to_dynamic_use_inference", false, false, "Add setting to allow converting String to Dynamic through parsing"},
             {"allow_experimental_dynamic_type", false, false, "Add new experimental Dynamic type"},
             {"azure_max_blocks_in_multipart_upload", 50000, 50000, "Maximum number of blocks in multipart upload for Azure."},
+            {"allow_archive_path_syntax", false, true, "Added new setting to allow disabling archive path syntax."},
         }
     },
     {"24.4",

From eb3fd60f3ec7a1212a58003fb1e21e9bf77f9eab Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Mon, 29 Jul 2024 12:54:36 +0000
Subject: [PATCH 1243/1488] Trying to fix test_cache_evicted_by_temporary_data
 and print debug info

---
 .../config.d/storage_configuration.xml        |  6 +--
 .../test_temporary_data_in_cache/test.py      | 44 ++++++++++++-------
 2 files changed, 31 insertions(+), 19 deletions(-)

diff --git a/tests/integration/test_temporary_data_in_cache/configs/config.d/storage_configuration.xml b/tests/integration/test_temporary_data_in_cache/configs/config.d/storage_configuration.xml
index 5a087d03266..107864fde0c 100644
--- a/tests/integration/test_temporary_data_in_cache/configs/config.d/storage_configuration.xml
+++ b/tests/integration/test_temporary_data_in_cache/configs/config.d/storage_configuration.xml
@@ -10,9 +10,9 @@
                 <type>cache</type>
                 <disk>local_disk</disk>
                 <path>/tiny_local_cache/</path>
-                <max_size>10M</max_size>
-                <max_file_segment_size>1M</max_file_segment_size>
-                <boundary_alignment>1M</boundary_alignment>
+                <max_size>12M</max_size>
+                <max_file_segment_size>100K</max_file_segment_size>
+                <boundary_alignment>100K</boundary_alignment>
                 <cache_on_write_operations>1</cache_on_write_operations>
             </tiny_local_cache>
 
diff --git a/tests/integration/test_temporary_data_in_cache/test.py b/tests/integration/test_temporary_data_in_cache/test.py
index cab134dcce2..abdfb5f4064 100644
--- a/tests/integration/test_temporary_data_in_cache/test.py
+++ b/tests/integration/test_temporary_data_in_cache/test.py
@@ -7,6 +7,9 @@ import fnmatch
 from helpers.cluster import ClickHouseCluster
 from helpers.client import QueryRuntimeException
 
+
+MB = 1024 * 1024
+
 cluster = ClickHouseCluster(__file__)
 
 node = cluster.add_instance(
@@ -36,15 +39,28 @@ def test_cache_evicted_by_temporary_data(start_cluster):
         q("SELECT sum(size) FROM system.filesystem_cache").strip()
     )
 
-    assert get_cache_size() == 0
+    def dump_debug_info():
+        return "\n".join(
+            [
+                ">>> filesystem_cache <<<",
+                q("SELECT * FROM system.filesystem_cache FORMAT Vertical"),
+                ">>> remote_data_paths <<<",
+                q("SELECT * FROM system.remote_data_paths FORMAT Vertical"),
+                ">>> tiny_local_cache_local_disk <<<",
+                q(
+                    "SELECT * FROM system.disks WHERE name = 'tiny_local_cache_local_disk' FORMAT Vertical"
+                ),
+            ]
+        )
 
-    assert get_free_space() > 8 * 1024 * 1024
+    assert get_cache_size() == 0, dump_debug_info()
+    assert get_free_space() > 8 * MB, dump_debug_info()
 
     # Codec is NONE to make cache size predictable
     q(
-        "CREATE TABLE t1 (x UInt64 CODEC(NONE), y UInt64 CODEC(NONE)) ENGINE = MergeTree ORDER BY x SETTINGS storage_policy = 'tiny_local_cache'"
+        "CREATE TABLE t1 (x UInt64 CODEC(NONE)) ENGINE = MergeTree ORDER BY x SETTINGS storage_policy = 'tiny_local_cache'"
     )
-    q("INSERT INTO t1 SELECT number, number FROM numbers(1024 * 1024)")
+    q("INSERT INTO t1 SELECT number FROM numbers(1024 * 1024)")
 
     # To be sure that nothing is reading the cache and entries for t1 can be evited
     q("OPTIMIZE TABLE t1 FINAL")
@@ -54,11 +70,11 @@ def test_cache_evicted_by_temporary_data(start_cluster):
     q("SELECT sum(x) FROM t1")
 
     cache_size_with_t1 = get_cache_size()
-    assert cache_size_with_t1 > 8 * 1024 * 1024
+    assert cache_size_with_t1 > 8 * MB, dump_debug_info()
 
     # Almost all disk space is occupied by t1 cache
     free_space_with_t1 = get_free_space()
-    assert free_space_with_t1 < 4 * 1024 * 1024
+    assert free_space_with_t1 < 4 * MB, dump_debug_info()
 
     # Try to sort the table, but fail because of lack of disk space
     with pytest.raises(QueryRuntimeException) as exc:
@@ -76,31 +92,27 @@ def test_cache_evicted_by_temporary_data(start_cluster):
 
     # Some data evicted from cache by temporary data
     cache_size_after_eviction = get_cache_size()
-    assert cache_size_after_eviction < cache_size_with_t1
+    assert cache_size_after_eviction < cache_size_with_t1, dump_debug_info()
 
     # Disk space freed, at least 3 MB, because temporary data tried to write 4 MB
-    assert get_free_space() > free_space_with_t1 + 3 * 1024 * 1024
+    assert get_free_space() > free_space_with_t1 + 3 * MB, dump_debug_info()
 
     # Read some data to fill the cache again
-    q("SELECT avg(y) FROM t1")
+    q("SELECT avg(x) FROM t1")
 
     cache_size_with_t1 = get_cache_size()
-    assert cache_size_with_t1 > 8 * 1024 * 1024, q(
-        "SELECT * FROM system.filesystem_cache FORMAT Vertical"
-    )
+    assert cache_size_with_t1 > 8 * MB, dump_debug_info()
 
     # Almost all disk space is occupied by t1 cache
     free_space_with_t1 = get_free_space()
-    assert free_space_with_t1 < 4 * 1024 * 1024, q(
-        "SELECT * FROM system.disks WHERE name = 'tiny_local_cache_local_disk' FORMAT Vertical"
-    )
+    assert free_space_with_t1 < 4 * MB, dump_debug_info()
 
     node.http_query(
         "SELECT randomPrintableASCII(1024) FROM numbers(8 * 1024) FORMAT TSV",
         params={"buffer_size": 0, "wait_end_of_query": 1},
     )
 
-    assert get_free_space() > free_space_with_t1 + 3 * 1024 * 1024
+    assert get_free_space() > free_space_with_t1 + 3 * MB, dump_debug_info()
 
     # not enough space for buffering 32 MB
     with pytest.raises(Exception) as exc:

From e2b686efea175e4ddc6472849934aa953f13138d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Fri, 2 Aug 2024 12:26:38 +0200
Subject: [PATCH 1244/1488] Fix test

---
 src/Common/tests/gtest_lsan.cpp | 23 ++++++++++++-----------
 1 file changed, 12 insertions(+), 11 deletions(-)

diff --git a/src/Common/tests/gtest_lsan.cpp b/src/Common/tests/gtest_lsan.cpp
index f6e1984ec58..7fc4ad2749e 100644
--- a/src/Common/tests/gtest_lsan.cpp
+++ b/src/Common/tests/gtest_lsan.cpp
@@ -14,20 +14,21 @@
 /// because of broken getauxval() [1].
 ///
 ///   [1]: https://github.com/ClickHouse/ClickHouse/pull/33957
-TEST(Common, LSan)
+TEST(SanitizerDeathTest, LSan)
 {
-    int sanitizers_exit_code = 1;
-
-    ASSERT_EXIT({
-        std::thread leak_in_thread([]()
+    EXPECT_DEATH(
         {
-            void * leak = malloc(4096);
-            ASSERT_NE(leak, nullptr);
-        });
-        leak_in_thread.join();
+            std::thread leak_in_thread(
+                []()
+                {
+                    void * leak = malloc(4096);
+                    ASSERT_NE(leak, nullptr);
+                });
+            leak_in_thread.join();
 
-        __lsan_do_leak_check();
-    }, ::testing::ExitedWithCode(sanitizers_exit_code), ".*LeakSanitizer: detected memory leaks.*");
+            __lsan_do_leak_check();
+        },
+        ".*LeakSanitizer: detected memory leaks.*");
 }
 
 #endif

From 064c0eb9587d9dbd1fa81cdbae8554c22dd11734 Mon Sep 17 00:00:00 2001
From: Andrey Zvonov <azvonov@altinity.com>
Date: Fri, 2 Aug 2024 10:35:32 +0000
Subject: [PATCH 1245/1488] even better healthcheck for ldap

---
 tests/integration/compose/docker_compose_ldap.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/integration/compose/docker_compose_ldap.yml b/tests/integration/compose/docker_compose_ldap.yml
index 1f50b34735d..f49e00400a2 100644
--- a/tests/integration/compose/docker_compose_ldap.yml
+++ b/tests/integration/compose/docker_compose_ldap.yml
@@ -19,6 +19,7 @@ services:
                 ldapsearch -x -H ldap://localhost:$$LDAP_PORT_NUMBER -D $$LDAP_ADMIN_DN -w $$LDAP_ADMIN_PASSWORD -b $$LDAP_ROOT
                 | grep -c -E "member: cn=j(ohn|ane)doe"
                 | grep 2 >> /dev/null
+                && cat /run/slapd/slapd.pid
             interval: 10s
             retries: 10
             timeout: 2s

From 02e48436057e45a884a1381e1c9cda9e1fe7de17 Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Tue, 30 Jul 2024 09:30:39 +0000
Subject: [PATCH 1246/1488] test_cache_evicted_by_temporary_data drop cache

---
 .../test_temporary_data_in_cache/test.py      | 30 ++++++++++---------
 1 file changed, 16 insertions(+), 14 deletions(-)

diff --git a/tests/integration/test_temporary_data_in_cache/test.py b/tests/integration/test_temporary_data_in_cache/test.py
index abdfb5f4064..87192a20975 100644
--- a/tests/integration/test_temporary_data_in_cache/test.py
+++ b/tests/integration/test_temporary_data_in_cache/test.py
@@ -39,19 +39,21 @@ def test_cache_evicted_by_temporary_data(start_cluster):
         q("SELECT sum(size) FROM system.filesystem_cache").strip()
     )
 
-    def dump_debug_info():
-        return "\n".join(
-            [
-                ">>> filesystem_cache <<<",
-                q("SELECT * FROM system.filesystem_cache FORMAT Vertical"),
-                ">>> remote_data_paths <<<",
-                q("SELECT * FROM system.remote_data_paths FORMAT Vertical"),
-                ">>> tiny_local_cache_local_disk <<<",
-                q(
-                    "SELECT * FROM system.disks WHERE name = 'tiny_local_cache_local_disk' FORMAT Vertical"
-                ),
-            ]
-        )
+    dump_debug_info = lambda: "\n".join(
+        [
+            ">>> filesystem_cache <<<",
+            q("SELECT * FROM system.filesystem_cache FORMAT Vertical"),
+            ">>> remote_data_paths <<<",
+            q("SELECT * FROM system.remote_data_paths FORMAT Vertical"),
+            ">>> tiny_local_cache_local_disk <<<",
+            q(
+                "SELECT * FROM system.disks WHERE name = 'tiny_local_cache_local_disk' FORMAT Vertical"
+            ),
+        ]
+    )
+
+    q("SYSTEM DROP FILESYSTEM CACHE")
+    q("DROP TABLE IF EXISTS t1 SYNC")
 
     assert get_cache_size() == 0, dump_debug_info()
     assert get_free_space() > 8 * MB, dump_debug_info()
@@ -124,4 +126,4 @@ def test_cache_evicted_by_temporary_data(start_cluster):
         str(exc.value), "*Failed to reserve * for temporary file*"
     ), exc.value
 
-    q("DROP TABLE IF EXISTS t1")
+    q("DROP TABLE IF EXISTS t1 SYNC")

From 97f1f6e22174916cc3b401fcebdf01dcd0fb0107 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Fri, 2 Aug 2024 12:46:37 +0200
Subject: [PATCH 1247/1488] Don't hide errors on clickhouse local runs

---
 docker/test/stateless/stress_tests.lib | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/docker/test/stateless/stress_tests.lib b/docker/test/stateless/stress_tests.lib
index 682da1df837..2a833b17f14 100644
--- a/docker/test/stateless/stress_tests.lib
+++ b/docker/test/stateless/stress_tests.lib
@@ -308,7 +308,8 @@ function collect_query_and_trace_logs()
 {
     for table in query_log trace_log metric_log
     do
-        clickhouse-local --config-file=/etc/clickhouse-server/config.xml --only-system-tables -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.tsv.zst ||:
+        # Don't ignore errors here, it leads to ignore sanitizer reports when running clickhouse-local
+        clickhouse-local --config-file=/etc/clickhouse-server/config.xml --only-system-tables -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.tsv.zst
     done
 }
 

From e5cf376c4ff3742fb0d3127ef6b50e08180eb153 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Fri, 2 Aug 2024 12:56:49 +0200
Subject: [PATCH 1248/1488] Don't attach gdb in ASAN runs of fuzzer, stateless
 or stress checks

---
 docker/test/fuzzer/run-fuzzer.sh     | 106 +++++++++++++++------------
 docker/test/stateless/attach_gdb.lib |  78 +++++++++++---------
 2 files changed, 101 insertions(+), 83 deletions(-)

diff --git a/docker/test/fuzzer/run-fuzzer.sh b/docker/test/fuzzer/run-fuzzer.sh
index b8f967ed9c2..ae1b9e94bed 100755
--- a/docker/test/fuzzer/run-fuzzer.sh
+++ b/docker/test/fuzzer/run-fuzzer.sh
@@ -193,53 +193,60 @@ function fuzz
 
     kill -0 $server_pid
 
-    # Set follow-fork-mode to parent, because we attach to clickhouse-server, not to watchdog
-    # and clickhouse-server can do fork-exec, for example, to run some bridge.
-    # Do not set nostop noprint for all signals, because some it may cause gdb to hang,
-    # explicitly ignore non-fatal signals that are used by server.
-    # Number of SIGRTMIN can be determined only in runtime.
-    RTMIN=$(kill -l SIGRTMIN)
-    echo "
-set follow-fork-mode parent
-handle SIGHUP nostop noprint pass
-handle SIGINT nostop noprint pass
-handle SIGQUIT nostop noprint pass
-handle SIGPIPE nostop noprint pass
-handle SIGTERM nostop noprint pass
-handle SIGUSR1 nostop noprint pass
-handle SIGUSR2 nostop noprint pass
-handle SIG$RTMIN nostop noprint pass
-info signals
-continue
-backtrace full
-thread apply all backtrace full
-info registers
-disassemble /s
-up
-disassemble /s
-up
-disassemble /s
-p \"done\"
-detach
-quit
-" > script.gdb
+    IS_ASAN=$(clickhouse-client --query "SELECT count() FROM system.build_options WHERE name = 'CXX_FLAGS' AND position('sanitize=address' IN value)")
+    if [[ "$IS_ASAN" = "1" ]];
+    then
+        echo "ASAN build detected. Not using gdb since it disables LeakSanitizer detections"
+    else
+        # Set follow-fork-mode to parent, because we attach to clickhouse-server, not to watchdog
+        # and clickhouse-server can do fork-exec, for example, to run some bridge.
+        # Do not set nostop noprint for all signals, because some it may cause gdb to hang,
+        # explicitly ignore non-fatal signals that are used by server.
+        # Number of SIGRTMIN can be determined only in runtime.
+        RTMIN=$(kill -l SIGRTMIN)
+        echo "
+    set follow-fork-mode parent
+    handle SIGHUP nostop noprint pass
+    handle SIGINT nostop noprint pass
+    handle SIGQUIT nostop noprint pass
+    handle SIGPIPE nostop noprint pass
+    handle SIGTERM nostop noprint pass
+    handle SIGUSR1 nostop noprint pass
+    handle SIGUSR2 nostop noprint pass
+    handle SIG$RTMIN nostop noprint pass
+    info signals
+    continue
+    backtrace full
+    thread apply all backtrace full
+    info registers
+    disassemble /s
+    up
+    disassemble /s
+    up
+    disassemble /s
+    p \"done\"
+    detach
+    quit
+    " > script.gdb
 
-    gdb -batch -command script.gdb -p $server_pid &
-    sleep 5
-    # gdb will send SIGSTOP, spend some time loading debug info, and then send SIGCONT, wait for it (up to send_timeout, 300s)
-    time clickhouse-client --query "SELECT 'Connected to clickhouse-server after attaching gdb'" ||:
+        gdb -batch -command script.gdb -p $server_pid &
+        sleep 5
+        # gdb will send SIGSTOP, spend some time loading debug info, and then send SIGCONT, wait for it (up to send_timeout, 300s)
+        time clickhouse-client --query "SELECT 'Connected to clickhouse-server after attaching gdb'" ||:
+
+        # Check connectivity after we attach gdb, because it might cause the server
+        # to freeze, and the fuzzer will fail. In debug build, it can take a lot of time.
+        for _ in {1..180}
+        do
+            if clickhouse-client --query "select 1"
+            then
+                break
+            fi
+            sleep 1
+        done
+        kill -0 $server_pid # This checks that it is our server that is started and not some other one
+    fi
 
-    # Check connectivity after we attach gdb, because it might cause the server
-    # to freeze, and the fuzzer will fail. In debug build, it can take a lot of time.
-    for _ in {1..180}
-    do
-        if clickhouse-client --query "select 1"
-        then
-            break
-        fi
-        sleep 1
-    done
-    kill -0 $server_pid # This checks that it is our server that is started and not some other one
     echo 'Server started and responded.'
 
     setup_logs_replication
@@ -264,8 +271,13 @@ quit
     # The fuzzer_pid belongs to the timeout process.
     actual_fuzzer_pid=$(ps -o pid= --ppid "$fuzzer_pid")
 
-    echo "Attaching gdb to the fuzzer itself"
-    gdb -batch -command script.gdb -p $actual_fuzzer_pid &
+    if [[ "$IS_ASAN" = "1" ]];
+    then
+        echo "ASAN build detected. Not using gdb since it disables LeakSanitizer detections"
+    else
+        echo "Attaching gdb to the fuzzer itself"
+        gdb -batch -command script.gdb -p $actual_fuzzer_pid &
+    fi
 
     # Wait for the fuzzer to complete.
     # Note that the 'wait || ...' thing is required so that the script doesn't
diff --git a/docker/test/stateless/attach_gdb.lib b/docker/test/stateless/attach_gdb.lib
index d288288bb17..d0cac24481f 100644
--- a/docker/test/stateless/attach_gdb.lib
+++ b/docker/test/stateless/attach_gdb.lib
@@ -5,43 +5,49 @@ source /utils.lib
 
 function attach_gdb_to_clickhouse()
 {
-    # Set follow-fork-mode to parent, because we attach to clickhouse-server, not to watchdog
-    # and clickhouse-server can do fork-exec, for example, to run some bridge.
-    # Do not set nostop noprint for all signals, because some it may cause gdb to hang,
-    # explicitly ignore non-fatal signals that are used by server.
-    # Number of SIGRTMIN can be determined only in runtime.
-    RTMIN=$(kill -l SIGRTMIN)
-  echo "
-set follow-fork-mode parent
-handle SIGHUP nostop noprint pass
-handle SIGINT nostop noprint pass
-handle SIGQUIT nostop noprint pass
-handle SIGPIPE nostop noprint pass
-handle SIGTERM nostop noprint pass
-handle SIGUSR1 nostop noprint pass
-handle SIGUSR2 nostop noprint pass
-handle SIG$RTMIN nostop noprint pass
-info signals
-continue
-backtrace full
-thread apply all backtrace full
-info registers
-disassemble /s
-up
-disassemble /s
-up
-disassemble /s
-p \"done\"
-detach
-quit
-" > script.gdb
+    IS_ASAN=$(clickhouse-client --query "SELECT count() FROM system.build_options WHERE name = 'CXX_FLAGS' AND position('sanitize=address' IN value)")
+    if [[ "$IS_ASAN" = "1" ]];
+    then
+        echo "ASAN build detected. Not using gdb since it disables LeakSanitizer detections"
+    else
+            # Set follow-fork-mode to parent, because we attach to clickhouse-server, not to watchdog
+            # and clickhouse-server can do fork-exec, for example, to run some bridge.
+            # Do not set nostop noprint for all signals, because some it may cause gdb to hang,
+            # explicitly ignore non-fatal signals that are used by server.
+            # Number of SIGRTMIN can be determined only in runtime.
+            RTMIN=$(kill -l SIGRTMIN)
+          echo "
+        set follow-fork-mode parent
+        handle SIGHUP nostop noprint pass
+        handle SIGINT nostop noprint pass
+        handle SIGQUIT nostop noprint pass
+        handle SIGPIPE nostop noprint pass
+        handle SIGTERM nostop noprint pass
+        handle SIGUSR1 nostop noprint pass
+        handle SIGUSR2 nostop noprint pass
+        handle SIG$RTMIN nostop noprint pass
+        info signals
+        continue
+        backtrace full
+        thread apply all backtrace full
+        info registers
+        disassemble /s
+        up
+        disassemble /s
+        up
+        disassemble /s
+        p \"done\"
+        detach
+        quit
+        " > script.gdb
 
-    # FIXME Hung check may work incorrectly because of attached gdb
-    # We cannot attach another gdb to get stacktraces if some queries hung
-    gdb -batch -command script.gdb -p "$(cat /var/run/clickhouse-server/clickhouse-server.pid)" | ts '%Y-%m-%d %H:%M:%S' >> /test_output/gdb.log &
-    sleep 5
-    # gdb will send SIGSTOP, spend some time loading debug info and then send SIGCONT, wait for it (up to send_timeout, 300s)
-    run_with_retry 60 clickhouse-client --query "SELECT 'Connected to clickhouse-server after attaching gdb'"
+            # FIXME Hung check may work incorrectly because of attached gdb
+            # We cannot attach another gdb to get stacktraces if some queries hung
+            gdb -batch -command script.gdb -p "$(cat /var/run/clickhouse-server/clickhouse-server.pid)" | ts '%Y-%m-%d %H:%M:%S' >> /test_output/gdb.log &
+            sleep 5
+            # gdb will send SIGSTOP, spend some time loading debug info and then send SIGCONT, wait for it (up to send_timeout, 300s)
+            run_with_retry 60 clickhouse-client --query "SELECT 'Connected to clickhouse-server after attaching gdb'"
+    fi
 }
 
 # vi: ft=bash

From 092c837119a9be11cfcc85b4696e9a9c74d9bbc8 Mon Sep 17 00:00:00 2001
From: Michael Stetsyuk <mike.stetsyuk@gmail.com>
Date: Fri, 2 Aug 2024 12:13:26 +0100
Subject: [PATCH 1249/1488] randomize table name in test_storage_delta

---
 tests/integration/test_storage_delta/test.py | 53 +++++++-------------
 1 file changed, 17 insertions(+), 36 deletions(-)

diff --git a/tests/integration/test_storage_delta/test.py b/tests/integration/test_storage_delta/test.py
index 92a870ab360..054b79ff6fe 100644
--- a/tests/integration/test_storage_delta/test.py
+++ b/tests/integration/test_storage_delta/test.py
@@ -8,7 +8,8 @@ import os
 import json
 import time
 import glob
-import shutil
+import random
+import string
 
 import pyspark
 import delta
@@ -53,6 +54,11 @@ def get_spark():
     return builder.master("local").getOrCreate()
 
 
+def randomize_table_name(table_name, random_suffix_length=10):
+    letters = string.ascii_letters + string.digits
+    return f"{table_name}{''.join(random.choice(letters) for _ in range(random_suffix_length))}"
+
+
 @pytest.fixture(scope="module")
 def started_cluster():
     try:
@@ -152,7 +158,7 @@ def test_single_log_file(started_cluster):
     spark = started_cluster.spark_session
     minio_client = started_cluster.minio_client
     bucket = started_cluster.minio_bucket
-    TABLE_NAME = "test_single_log_file"
+    TABLE_NAME = randomize_table_name("test_single_log_file")
 
     inserted_data = "SELECT number as a, toString(number + 1) as b FROM numbers(100)"
     parquet_data_path = create_initial_data_file(
@@ -170,16 +176,13 @@ def test_single_log_file(started_cluster):
         inserted_data
     )
 
-    os.unlink(parquet_data_path)
-    shutil.rmtree(f"/{TABLE_NAME}")
-
 
 def test_partition_by(started_cluster):
     instance = started_cluster.instances["node1"]
     spark = started_cluster.spark_session
     minio_client = started_cluster.minio_client
     bucket = started_cluster.minio_bucket
-    TABLE_NAME = "test_partition_by"
+    TABLE_NAME = randomize_table_name("test_partition_by")
 
     write_delta_from_df(
         spark,
@@ -195,15 +198,13 @@ def test_partition_by(started_cluster):
     create_delta_table(instance, TABLE_NAME)
     assert int(instance.query(f"SELECT count() FROM {TABLE_NAME}")) == 10
 
-    shutil.rmtree(f"/{TABLE_NAME}")
-
 
 def test_checkpoint(started_cluster):
     instance = started_cluster.instances["node1"]
     spark = started_cluster.spark_session
     minio_client = started_cluster.minio_client
     bucket = started_cluster.minio_bucket
-    TABLE_NAME = "test_checkpoint"
+    TABLE_NAME = randomize_table_name("test_checkpoint")
 
     write_delta_from_df(
         spark,
@@ -272,16 +273,13 @@ def test_checkpoint(started_cluster):
         ).strip()
     )
 
-    shutil.rmtree(f"/{TABLE_NAME}")
-    spark.sql(f"DROP TABLE {TABLE_NAME}")
-
 
 def test_multiple_log_files(started_cluster):
     instance = started_cluster.instances["node1"]
     spark = started_cluster.spark_session
     minio_client = started_cluster.minio_client
     bucket = started_cluster.minio_bucket
-    TABLE_NAME = "test_multiple_log_files"
+    TABLE_NAME = randomize_table_name("test_multiple_log_files")
 
     write_delta_from_df(
         spark, generate_data(spark, 0, 100), f"/{TABLE_NAME}", mode="overwrite"
@@ -313,15 +311,13 @@ def test_multiple_log_files(started_cluster):
         "SELECT number, toString(number + 1) FROM numbers(200)"
     )
 
-    shutil.rmtree(f"/{TABLE_NAME}")
-
 
 def test_metadata(started_cluster):
     instance = started_cluster.instances["node1"]
     spark = started_cluster.spark_session
     minio_client = started_cluster.minio_client
     bucket = started_cluster.minio_bucket
-    TABLE_NAME = "test_metadata"
+    TABLE_NAME = randomize_table_name("test_metadata")
 
     parquet_data_path = create_initial_data_file(
         started_cluster,
@@ -348,14 +344,11 @@ def test_metadata(started_cluster):
     create_delta_table(instance, TABLE_NAME)
     assert int(instance.query(f"SELECT count() FROM {TABLE_NAME}")) == 100
 
-    os.unlink(parquet_data_path)
-    shutil.rmtree(f"/{TABLE_NAME}")
-
 
 def test_types(started_cluster):
-    TABLE_NAME = "test_types"
+    TABLE_NAME = randomize_table_name("test_types")
     spark = started_cluster.spark_session
-    result_file = f"{TABLE_NAME}_result_2"
+    result_file = randomize_table_name(f"{TABLE_NAME}_result_2")
 
     delta_table = (
         DeltaTable.create(spark)
@@ -423,16 +416,13 @@ def test_types(started_cluster):
         ]
     )
 
-    shutil.rmtree(f"/{result_file}")
-    spark.sql(f"DROP TABLE {TABLE_NAME}")
-
 
 def test_restart_broken(started_cluster):
     instance = started_cluster.instances["node1"]
     spark = started_cluster.spark_session
     minio_client = started_cluster.minio_client
     bucket = "broken"
-    TABLE_NAME = "test_restart_broken"
+    TABLE_NAME = randomize_table_name("test_restart_broken")
 
     if not minio_client.bucket_exists(bucket):
         minio_client.make_bucket(bucket)
@@ -487,16 +477,13 @@ def test_restart_broken(started_cluster):
 
     assert int(instance.query(f"SELECT count() FROM {TABLE_NAME}")) == 100
 
-    os.unlink(parquet_data_path)
-    shutil.rmtree(f"/{TABLE_NAME}")
-
 
 def test_restart_broken_table_function(started_cluster):
     instance = started_cluster.instances["node1"]
     spark = started_cluster.spark_session
     minio_client = started_cluster.minio_client
     bucket = "broken2"
-    TABLE_NAME = "test_restart_broken_table_function"
+    TABLE_NAME = randomize_table_name("test_restart_broken_table_function")
 
     if not minio_client.bucket_exists(bucket):
         minio_client.make_bucket(bucket)
@@ -544,16 +531,13 @@ def test_restart_broken_table_function(started_cluster):
 
     assert int(instance.query(f"SELECT count() FROM {TABLE_NAME}")) == 100
 
-    os.unlink(parquet_data_path)
-    shutil.rmtree(f"/{TABLE_NAME}")
-
 
 def test_partition_columns(started_cluster):
     instance = started_cluster.instances["node1"]
     spark = started_cluster.spark_session
     minio_client = started_cluster.minio_client
     bucket = started_cluster.minio_bucket
-    TABLE_NAME = "test_partition_columns"
+    TABLE_NAME = randomize_table_name("test_partition_columns")
     result_file = f"{TABLE_NAME}"
     partition_columns = ["b", "c", "d", "e"]
 
@@ -744,6 +728,3 @@ SELECT * FROM deltaLake('http://{started_cluster.minio_ip}:{started_cluster.mini
         )
         == 1
     )
-
-    shutil.rmtree(f"/{TABLE_NAME}")
-    spark.sql(f"DROP TABLE {TABLE_NAME}")

From 7d1e958097e716f3ea1e0b7e51d6dfa575229c4c Mon Sep 17 00:00:00 2001
From: Nikita Fomichev <nikita.fomichev@clickhouse.com>
Date: Fri, 2 Aug 2024 13:32:59 +0200
Subject: [PATCH 1250/1488] Integration tests: fix ports clashing problem

---
 tests/integration/conftest.py        | 44 ++++++++++++++++++
 tests/integration/helpers/cluster.py | 67 ++++++++++++++++++++++++----
 2 files changed, 103 insertions(+), 8 deletions(-)

diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py
index f4be31cc532..0a47840ede3 100644
--- a/tests/integration/conftest.py
+++ b/tests/integration/conftest.py
@@ -2,6 +2,8 @@
 
 import logging
 import os
+import socket
+import multiprocessing
 
 import pytest  # pylint:disable=import-error; for style check
 from helpers.cluster import run_and_check
@@ -11,6 +13,7 @@ from helpers.network import _NetworkManager
 #
 #   [1]: https://github.com/pytest-dev/pytest/issues/5502
 logging.raiseExceptions = False
+PORTS_PER_WORKER = 50
 
 
 @pytest.fixture(scope="session", autouse=True)
@@ -111,5 +114,46 @@ def pytest_addoption(parser):
     )
 
 
+def get_n_free_ports(total):
+    ports = []
+
+    while len(ports) < total:
+        with socket.socket() as s:
+            s.bind(("", 0))
+            ports.append(s.getsockname()[1])
+
+    return ports
+
+
 def pytest_configure(config):
     os.environ["INTEGRATION_TESTS_RUN_ID"] = config.option.run_id
+
+    # When running tests without pytest-xdist,
+    # the `pytest_xdist_setupnodes` hook is not executed
+    worker_ports = os.getenv("WORKER_FREE_PORTS", None)
+    if worker_ports is None:
+        os.environ["WORKER_FREE_PORTS"] = " ".join(
+            ([str(p) for p in get_n_free_ports(PORTS_PER_WORKER)])
+        )
+
+
+def pytest_xdist_setupnodes(config, specs):
+    # Find {PORTS_PER_WORKER} * {number of xdist workers} ports and
+    # allocate pool of {PORTS_PER_WORKER} ports to each worker
+
+    # Get number of xdist workers
+    num_workers = 1
+    if os.environ.get("PYTEST_XDIST_WORKER", "master") == "master":
+        num_workers = config.getoption("numprocesses", 1)
+        if num_workers == "auto":
+            num_workers = multiprocessing.cpu_count()
+
+    # Get free ports which will be distributed across workers
+    ports = get_n_free_ports(num_workers * PORTS_PER_WORKER)
+
+    # Iterate over specs of workers and add allocated ports to env variable
+    for i, spec in enumerate(specs):
+        start_range = i * PORTS_PER_WORKER
+        spec.env["WORKER_FREE_PORTS"] = " ".join(
+            ([str(p) for p in ports[start_range : start_range + PORTS_PER_WORKER]])
+        )
diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py
index 6bc0ece63ca..3480a3089fe 100644
--- a/tests/integration/helpers/cluster.py
+++ b/tests/integration/helpers/cluster.py
@@ -135,6 +135,52 @@ def get_free_port():
         return s.getsockname()[1]
 
 
+def is_port_free(port: int) -> bool:
+    try:
+        with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
+            s.bind(("", port))
+            return True
+    except socket.error:
+        return False
+
+
+class PortPoolManager:
+    """
+    This class is used for distribution of ports allocated to single pytest-xdist worker
+    It can be used by multiple ClickHouseCluster instances
+    """
+
+    # Shared between instances
+    all_ports = None
+    free_ports = None
+
+    def __init__(self):
+        self.used_ports = []
+
+        if self.all_ports is None:
+            worker_ports = os.getenv("WORKER_FREE_PORTS")
+            ports = [int(p) for p in worker_ports.split(" ")]
+
+            # Static vars
+            PortPoolManager.all_ports = ports
+            PortPoolManager.free_ports = ports
+
+    def get_port(self):
+        for port in self.free_ports:
+            if is_port_free(port):
+                self.free_ports.remove(port)
+                self.used_ports.append(port)
+                return port
+
+        raise Exception(
+            f"No free ports: {self.all_ports}",
+        )
+
+    def return_used_ports(self):
+        self.free_ports.extend(self.used_ports)
+        self.used_ports.clear()
+
+
 def retry_exception(num, delay, func, exception=Exception, *args, **kwargs):
     """
     Retry if `func()` throws, `num` times.
@@ -716,62 +762,67 @@ class ClickHouseCluster:
                 .stop()
             )
 
+        self.port_pool = PortPoolManager()
+
     @property
     def kafka_port(self):
         if self._kafka_port:
             return self._kafka_port
-        self._kafka_port = get_free_port()
+        self._kafka_port = self.port_pool.get_port()
         return self._kafka_port
 
     @property
     def schema_registry_port(self):
         if self._schema_registry_port:
             return self._schema_registry_port
-        self._schema_registry_port = get_free_port()
+        self._schema_registry_port = self.port_pool.get_port()
         return self._schema_registry_port
 
     @property
     def schema_registry_auth_port(self):
         if self._schema_registry_auth_port:
             return self._schema_registry_auth_port
-        self._schema_registry_auth_port = get_free_port()
+        self._schema_registry_auth_port = self.port_pool.get_port()
         return self._schema_registry_auth_port
 
     @property
     def kerberized_kafka_port(self):
         if self._kerberized_kafka_port:
             return self._kerberized_kafka_port
-        self._kerberized_kafka_port = get_free_port()
+        self._kerberized_kafka_port = self.port_pool.get_port()
         return self._kerberized_kafka_port
 
     @property
     def azurite_port(self):
         if self._azurite_port:
             return self._azurite_port
-        self._azurite_port = get_free_port()
+        self._azurite_port = self.port_pool.get_port()
         return self._azurite_port
 
     @property
     def mongo_port(self):
         if self._mongo_port:
             return self._mongo_port
-        self._mongo_port = get_free_port()
+        self._mongo_port = self.port_pool.get_port()
         return self._mongo_port
 
     @property
     def mongo_no_cred_port(self):
         if self._mongo_no_cred_port:
             return self._mongo_no_cred_port
-        self._mongo_no_cred_port = get_free_port()
+        self._mongo_no_cred_port = self.port_pool.get_port()
         return self._mongo_no_cred_port
 
     @property
     def redis_port(self):
         if self._redis_port:
             return self._redis_port
-        self._redis_port = get_free_port()
+        self._redis_port = self.port_pool.get_port()
         return self._redis_port
 
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.port_pool.return_used_ports()
+
     def print_all_docker_pieces(self):
         res_networks = subprocess.check_output(
             f"docker network ls --filter name='{self.project_name}*'",

From 7d45529fe8d28a6b39deb32d060343bb5d03b64f Mon Sep 17 00:00:00 2001
From: Michael Stetsyuk <mike.stetsyuk@gmail.com>
Date: Fri, 2 Aug 2024 12:35:40 +0100
Subject: [PATCH 1251/1488] randomize query id in
 test_checking_s3_blobs_paranoid

---
 .../test_checking_s3_blobs_paranoid/test.py   | 31 ++++++++++++-------
 1 file changed, 19 insertions(+), 12 deletions(-)

diff --git a/tests/integration/test_checking_s3_blobs_paranoid/test.py b/tests/integration/test_checking_s3_blobs_paranoid/test.py
index afe8449b44a..c22142046c7 100644
--- a/tests/integration/test_checking_s3_blobs_paranoid/test.py
+++ b/tests/integration/test_checking_s3_blobs_paranoid/test.py
@@ -4,6 +4,8 @@ import logging
 import pytest
 import os
 import minio
+import random
+import string
 
 from helpers.cluster import ClickHouseCluster
 from helpers.mock_servers import start_s3_mock
@@ -45,6 +47,11 @@ def cluster():
         cluster.shutdown()
 
 
+def randomize_query_id(query_id, random_suffix_length=10):
+    letters = string.ascii_letters + string.digits
+    return f"{query_id}_{''.join(random.choice(letters) for _ in range(random_suffix_length))}"
+
+
 @pytest.fixture(scope="module")
 def init_broken_s3(cluster):
     yield start_s3_mock(cluster, "broken_s3", "8083")
@@ -128,7 +135,7 @@ def test_upload_s3_fail_create_multi_part_upload(cluster, broken_s3, compression
 
     broken_s3.setup_at_create_multi_part_upload()
 
-    insert_query_id = f"INSERT_INTO_TABLE_FUNCTION_FAIL_CREATE_MPU_{compression}"
+    insert_query_id = randomize_query_id(f"INSERT_INTO_TABLE_FUNCTION_FAIL_CREATE_MPU_{compression}")
     error = node.query_and_get_error(
         f"""
         INSERT INTO
@@ -170,7 +177,7 @@ def test_upload_s3_fail_upload_part_when_multi_part_upload(
     broken_s3.setup_fake_multpartuploads()
     broken_s3.setup_at_part_upload(count=1, after=2)
 
-    insert_query_id = f"INSERT_INTO_TABLE_FUNCTION_FAIL_UPLOAD_PART_{compression}"
+    insert_query_id = randomize_query_id(f"INSERT_INTO_TABLE_FUNCTION_FAIL_UPLOAD_PART_{compression}")
     error = node.query_and_get_error(
         f"""
         INSERT INTO
@@ -222,7 +229,7 @@ def test_when_error_is_retried(cluster, broken_s3, action_and_message):
     broken_s3.setup_fake_multpartuploads()
     broken_s3.setup_at_part_upload(count=3, after=2, action=action)
 
-    insert_query_id = f"INSERT_INTO_TABLE_{action}_RETRIED"
+    insert_query_id = randomize_query_id(f"INSERT_INTO_TABLE_{action}_RETRIED")
     node.query(
         f"""
         INSERT INTO
@@ -251,7 +258,7 @@ def test_when_error_is_retried(cluster, broken_s3, action_and_message):
     assert s3_errors == 3
 
     broken_s3.setup_at_part_upload(count=1000, after=2, action=action)
-    insert_query_id = f"INSERT_INTO_TABLE_{action}_RETRIED_1"
+    insert_query_id = randomize_query_id(f"INSERT_INTO_TABLE_{action}_RETRIED_1")
     error = node.query_and_get_error(
         f"""
             INSERT INTO
@@ -286,7 +293,7 @@ def test_when_s3_broken_pipe_at_upload_is_retried(cluster, broken_s3):
         action="broken_pipe",
     )
 
-    insert_query_id = f"TEST_WHEN_S3_BROKEN_PIPE_AT_UPLOAD"
+    insert_query_id = randomize_query_id(f"TEST_WHEN_S3_BROKEN_PIPE_AT_UPLOAD")
     node.query(
         f"""
         INSERT INTO
@@ -320,7 +327,7 @@ def test_when_s3_broken_pipe_at_upload_is_retried(cluster, broken_s3):
         after=2,
         action="broken_pipe",
     )
-    insert_query_id = f"TEST_WHEN_S3_BROKEN_PIPE_AT_UPLOAD_1"
+    insert_query_id = randomize_query_id(f"TEST_WHEN_S3_BROKEN_PIPE_AT_UPLOAD_1")
     error = node.query_and_get_error(
         f"""
                INSERT INTO
@@ -362,7 +369,7 @@ def test_when_s3_connection_reset_by_peer_at_upload_is_retried(
         action_args=["1"] if send_something else ["0"],
     )
 
-    insert_query_id = (
+    insert_query_id = randomize_query_id(
         f"TEST_WHEN_S3_CONNECTION_RESET_BY_PEER_AT_UPLOAD_{send_something}"
     )
     node.query(
@@ -399,7 +406,7 @@ def test_when_s3_connection_reset_by_peer_at_upload_is_retried(
         action="connection_reset_by_peer",
         action_args=["1"] if send_something else ["0"],
     )
-    insert_query_id = (
+    insert_query_id = randomize_query_id(
         f"TEST_WHEN_S3_CONNECTION_RESET_BY_PEER_AT_UPLOAD_{send_something}_1"
     )
     error = node.query_and_get_error(
@@ -444,7 +451,7 @@ def test_when_s3_connection_reset_by_peer_at_create_mpu_retried(
         action_args=["1"] if send_something else ["0"],
     )
 
-    insert_query_id = (
+    insert_query_id = randomize_query_id(
         f"TEST_WHEN_S3_CONNECTION_RESET_BY_PEER_AT_MULTIPARTUPLOAD_{send_something}"
     )
     node.query(
@@ -482,7 +489,7 @@ def test_when_s3_connection_reset_by_peer_at_create_mpu_retried(
         action_args=["1"] if send_something else ["0"],
     )
 
-    insert_query_id = (
+    insert_query_id = randomize_query_id(
         f"TEST_WHEN_S3_CONNECTION_RESET_BY_PEER_AT_MULTIPARTUPLOAD_{send_something}_1"
     )
     error = node.query_and_get_error(
@@ -522,7 +529,7 @@ def test_query_is_canceled_with_inf_retries(cluster, broken_s3):
         action="connection_refused",
     )
 
-    insert_query_id = f"TEST_QUERY_IS_CANCELED_WITH_INF_RETRIES"
+    insert_query_id = randomize_query_id(f"TEST_QUERY_IS_CANCELED_WITH_INF_RETRIES")
     request = node.get_query_request(
         f"""
         INSERT INTO
@@ -580,7 +587,7 @@ def test_adaptive_timeouts(cluster, broken_s3, node_name):
         count=1000000,
     )
 
-    insert_query_id = f"TEST_ADAPTIVE_TIMEOUTS_{node_name}"
+    insert_query_id = randomize_query_id(f"TEST_ADAPTIVE_TIMEOUTS_{node_name}")
     node.query(
         f"""
             INSERT INTO

From 34cba1bdda55cdd2409c535be56e4fe6165c894a Mon Sep 17 00:00:00 2001
From: robot-clickhouse <robot-clickhouse@users.noreply.github.com>
Date: Fri, 2 Aug 2024 11:46:03 +0000
Subject: [PATCH 1252/1488] Automatic style fix

---
 tests/integration/test_checking_s3_blobs_paranoid/test.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/tests/integration/test_checking_s3_blobs_paranoid/test.py b/tests/integration/test_checking_s3_blobs_paranoid/test.py
index c22142046c7..b995b4d6461 100644
--- a/tests/integration/test_checking_s3_blobs_paranoid/test.py
+++ b/tests/integration/test_checking_s3_blobs_paranoid/test.py
@@ -135,7 +135,9 @@ def test_upload_s3_fail_create_multi_part_upload(cluster, broken_s3, compression
 
     broken_s3.setup_at_create_multi_part_upload()
 
-    insert_query_id = randomize_query_id(f"INSERT_INTO_TABLE_FUNCTION_FAIL_CREATE_MPU_{compression}")
+    insert_query_id = randomize_query_id(
+        f"INSERT_INTO_TABLE_FUNCTION_FAIL_CREATE_MPU_{compression}"
+    )
     error = node.query_and_get_error(
         f"""
         INSERT INTO
@@ -177,7 +179,9 @@ def test_upload_s3_fail_upload_part_when_multi_part_upload(
     broken_s3.setup_fake_multpartuploads()
     broken_s3.setup_at_part_upload(count=1, after=2)
 
-    insert_query_id = randomize_query_id(f"INSERT_INTO_TABLE_FUNCTION_FAIL_UPLOAD_PART_{compression}")
+    insert_query_id = randomize_query_id(
+        f"INSERT_INTO_TABLE_FUNCTION_FAIL_UPLOAD_PART_{compression}"
+    )
     error = node.query_and_get_error(
         f"""
         INSERT INTO

From d2d8a16ca6c5c3df31a62894fe2bcfb26d570061 Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Fri, 2 Aug 2024 13:51:03 +0200
Subject: [PATCH 1253/1488] Fix refreshable MVs

---
 src/Databases/DatabaseOrdinary.cpp | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp
index 3ab5d3fa697..8808261654f 100644
--- a/src/Databases/DatabaseOrdinary.cpp
+++ b/src/Databases/DatabaseOrdinary.cpp
@@ -16,6 +16,7 @@
 #include <Interpreters/Context.h>
 #include <Interpreters/InterpreterCreateQuery.h>
 #include <Interpreters/FunctionNameNormalizer.h>
+#include <Interpreters/NormalizeSelectWithUnionQueryVisitor.h>
 #include <Parsers/ASTCreateQuery.h>
 #include <Parsers/ASTSetQuery.h>
 #include <Parsers/ParserCreateQuery.h>
@@ -250,6 +251,8 @@ void DatabaseOrdinary::loadTablesMetadata(ContextPtr local_context, ParsedTables
 
                 convertMergeTreeToReplicatedIfNeeded(ast, qualified_name, file_name);
 
+                NormalizeSelectWithUnionQueryVisitor::Data data{local_context->getSettingsRef().union_default_mode};
+                NormalizeSelectWithUnionQueryVisitor{data}.visit(ast);
                 std::lock_guard lock{metadata.mutex};
                 metadata.parsed_tables[qualified_name] = ParsedTableMetadata{full_path.string(), ast};
                 metadata.total_dictionaries += create_query->is_dictionary;

From c9b29ad11351b774d834ada642951a81d33b14e7 Mon Sep 17 00:00:00 2001
From: maxvostrikov <max.vostrikov@clickhouse.com>
Date: Fri, 2 Aug 2024 13:53:48 +0200
Subject: [PATCH 1254/1488] squash! fix for parallel execution

---
 tests/integration/test_parquet_page_index/test.py | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/tests/integration/test_parquet_page_index/test.py b/tests/integration/test_parquet_page_index/test.py
index db291e20b74..59dbab09be5 100644
--- a/tests/integration/test_parquet_page_index/test.py
+++ b/tests/integration/test_parquet_page_index/test.py
@@ -6,9 +6,6 @@ import time
 
 cluster = ClickHouseCluster(__file__)
 path_to_userfiles = "/var/lib/clickhouse/user_files/"
-path_to_external_dirs = (
-    "/ClickHouse/tests/integration/test_parquet_page_index/_instances"
-)
 node = cluster.add_instance("node", external_dirs=[path_to_userfiles])
 
 
@@ -45,7 +42,7 @@ def delete_if_exists(file_path):
             True,
         ),
         (
-            "SElECT number, number+1 FROM system.numbers LIMIT 100 "
+            "SELECT number, number+1 FROM system.numbers LIMIT 100 "
             "INTO OUTFILE '{file_name}' FORMAT Parquet "
             "SETTINGS output_format_parquet_use_custom_encoder = false, "
             "output_format_parquet_write_page_index = false;",
@@ -54,7 +51,7 @@ def delete_if_exists(file_path):
         # # default settings:
         # # output_format_parquet_use_custom_encoder = true
         (
-            "SElECT number, number+1 FROM system.numbers LIMIT 100 "
+            "SELECT number, number+1 FROM system.numbers LIMIT 100 "
             "INTO OUTFILE '{file_name}' FORMAT Parquet;",
             False,
         ),
@@ -102,7 +99,7 @@ def test_parquet_page_index_insert_into_table_function_file(
 ):
     file_name = f"export{time.time()}.parquet"
     query = query.format(file_name=file_name)
-    file_path = f"{path_to_external_dirs}{path_to_userfiles}{file_name}"
+    file_path = f"{cluster.instances_dir}{path_to_userfiles}{file_name}"
     delete_if_exists(file_path)
     assert node.query(query) == ""
     assert (

From cae76458504f46fefe52c7d93594870d0bac4479 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Fri, 2 Aug 2024 14:00:50 +0200
Subject: [PATCH 1255/1488] Make 02514_null_dictionary_source parallelizable

---
 tests/queries/0_stateless/02514_null_dictionary_source.sql | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tests/queries/0_stateless/02514_null_dictionary_source.sql b/tests/queries/0_stateless/02514_null_dictionary_source.sql
index 74fb57707ff..bfd36042f57 100644
--- a/tests/queries/0_stateless/02514_null_dictionary_source.sql
+++ b/tests/queries/0_stateless/02514_null_dictionary_source.sql
@@ -1,5 +1,3 @@
--- Tags: no-parallel
-
 DROP DICTIONARY IF EXISTS null_dict;
 CREATE DICTIONARY null_dict (
     id              UInt64,

From 45f54c3633a388075898e3dcabc850dd1347587d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Fri, 2 Aug 2024 14:01:01 +0200
Subject: [PATCH 1256/1488] Make 01086_window_view_cleanup parallelizable

---
 .../0_stateless/01086_window_view_cleanup.sh  | 47 ++++++++++---------
 1 file changed, 24 insertions(+), 23 deletions(-)

diff --git a/tests/queries/0_stateless/01086_window_view_cleanup.sh b/tests/queries/0_stateless/01086_window_view_cleanup.sh
index 113bcffb2af..0bce08523e2 100755
--- a/tests/queries/0_stateless/01086_window_view_cleanup.sh
+++ b/tests/queries/0_stateless/01086_window_view_cleanup.sh
@@ -1,5 +1,4 @@
 #!/usr/bin/env bash
-# Tags: no-parallel
 
 # Creation of a database with Ordinary engine emits a warning.
 CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=fatal
@@ -12,38 +11,40 @@ opts=(
     "--allow_experimental_analyzer=0"
 )
 
-$CLICKHOUSE_CLIENT "${opts[@]}" --allow_deprecated_database_ordinary=1 <<EOF
-SET allow_experimental_window_view = 1;
-SET window_view_clean_interval = 1;
+DATABASE_ORDINARY="${CLICKHOUSE_DATABASE}_ordinary"
 
-DROP DATABASE IF EXISTS test_01086;
-CREATE DATABASE test_01086 ENGINE=Ordinary;
+$CLICKHOUSE_CLIENT "${opts[@]}" --allow_deprecated_database_ordinary=1 --multiquery "
+    SET allow_experimental_window_view = 1;
+    SET window_view_clean_interval = 1;
 
-CREATE TABLE test_01086.dst(count UInt64, market Int32, w_end DateTime) Engine=MergeTree ORDER BY tuple();
-CREATE TABLE test_01086.mt(a Int32, market Int32, timestamp DateTime) ENGINE=MergeTree ORDER BY tuple();
-CREATE WINDOW VIEW test_01086.wv TO test_01086.dst WATERMARK=ASCENDING AS SELECT count(a) AS count, market, tumbleEnd(wid) AS w_end FROM test_01086.mt GROUP BY tumble(timestamp, INTERVAL '5' SECOND, 'US/Samoa') AS wid, market;
+    DROP DATABASE IF EXISTS ${DATABASE_ORDINARY};
+    CREATE DATABASE ${DATABASE_ORDINARY} ENGINE=Ordinary;
 
-INSERT INTO test_01086.mt VALUES (1, 1, toDateTime('1990/01/01 12:00:00', 'US/Samoa'));
-INSERT INTO test_01086.mt VALUES (1, 2, toDateTime('1990/01/01 12:00:01', 'US/Samoa'));
-INSERT INTO test_01086.mt VALUES (1, 3, toDateTime('1990/01/01 12:00:02', 'US/Samoa'));
-INSERT INTO test_01086.mt VALUES (1, 4, toDateTime('1990/01/01 12:00:05', 'US/Samoa'));
-INSERT INTO test_01086.mt VALUES (1, 5, toDateTime('1990/01/01 12:00:06', 'US/Samoa'));
-EOF
+    CREATE TABLE ${DATABASE_ORDINARY}.dst(count UInt64, market Int32, w_end DateTime) Engine=MergeTree ORDER BY tuple();
+    CREATE TABLE ${DATABASE_ORDINARY}.mt(a Int32, market Int32, timestamp DateTime) ENGINE=MergeTree ORDER BY tuple();
+    CREATE WINDOW VIEW ${DATABASE_ORDINARY}.wv TO ${DATABASE_ORDINARY}.dst WATERMARK=ASCENDING AS SELECT count(a) AS count, market, tumbleEnd(wid) AS w_end FROM ${DATABASE_ORDINARY}.mt GROUP BY tumble(timestamp, INTERVAL '5' SECOND, 'US/Samoa') AS wid, market;
+
+    INSERT INTO ${DATABASE_ORDINARY}.mt VALUES (1, 1, toDateTime('1990/01/01 12:00:00', 'US/Samoa'));
+    INSERT INTO ${DATABASE_ORDINARY}.mt VALUES (1, 2, toDateTime('1990/01/01 12:00:01', 'US/Samoa'));
+    INSERT INTO ${DATABASE_ORDINARY}.mt VALUES (1, 3, toDateTime('1990/01/01 12:00:02', 'US/Samoa'));
+    INSERT INTO ${DATABASE_ORDINARY}.mt VALUES (1, 4, toDateTime('1990/01/01 12:00:05', 'US/Samoa'));
+    INSERT INTO ${DATABASE_ORDINARY}.mt VALUES (1, 5, toDateTime('1990/01/01 12:00:06', 'US/Samoa'));
+"
 
 while true; do
-	$CLICKHOUSE_CLIENT "${opts[@]}" --query="SELECT count(*) FROM test_01086.\`.inner.wv\`" | grep -q "5" && break || sleep .5 ||:
+	$CLICKHOUSE_CLIENT "${opts[@]}" --query="SELECT count(*) FROM ${DATABASE_ORDINARY}.\`.inner.wv\`" | grep -q "5" && break || sleep .5 ||:
 done
 
 $CLICKHOUSE_CLIENT "${opts[@]}" --query="SELECT sleep(2);"
 
-$CLICKHOUSE_CLIENT "${opts[@]}" --query="INSERT INTO test_01086.mt VALUES (1, 6, toDateTime('1990/01/01 12:00:11', 'US/Samoa'));"
+$CLICKHOUSE_CLIENT "${opts[@]}" --query="INSERT INTO ${DATABASE_ORDINARY}.mt VALUES (1, 6, toDateTime('1990/01/01 12:00:11', 'US/Samoa'));"
 
 while true; do
-	$CLICKHOUSE_CLIENT "${opts[@]}" --query="SELECT count(*) FROM test_01086.\`.inner.wv\`" | grep -q "3" && break || sleep .5 ||:
+	$CLICKHOUSE_CLIENT "${opts[@]}" --query="SELECT count(*) FROM ${DATABASE_ORDINARY}.\`.inner.wv\`" | grep -q "3" && break || sleep .5 ||:
 done
 
-$CLICKHOUSE_CLIENT "${opts[@]}" --query="SELECT market, wid FROM test_01086.\`.inner.wv\` ORDER BY market, \`windowID(timestamp, toIntervalSecond('5'), 'US/Samoa')\` as wid";
-$CLICKHOUSE_CLIENT "${opts[@]}" --query="DROP TABLE test_01086.wv SYNC;"
-$CLICKHOUSE_CLIENT "${opts[@]}" --query="DROP TABLE test_01086.mt SYNC;"
-$CLICKHOUSE_CLIENT "${opts[@]}" --query="DROP TABLE test_01086.dst SYNC;"
-$CLICKHOUSE_CLIENT "${opts[@]}" --query="DROP DATABASE test_01086 SYNC;"
+$CLICKHOUSE_CLIENT "${opts[@]}" --query="SELECT market, wid FROM ${DATABASE_ORDINARY}.\`.inner.wv\` ORDER BY market, \`windowID(timestamp, toIntervalSecond('5'), 'US/Samoa')\` as wid";
+$CLICKHOUSE_CLIENT "${opts[@]}" --query="DROP TABLE ${DATABASE_ORDINARY}.wv SYNC;"
+$CLICKHOUSE_CLIENT "${opts[@]}" --query="DROP TABLE ${DATABASE_ORDINARY}.mt SYNC;"
+$CLICKHOUSE_CLIENT "${opts[@]}" --query="DROP TABLE ${DATABASE_ORDINARY}.dst SYNC;"
+$CLICKHOUSE_CLIENT "${opts[@]}" --query="DROP DATABASE ${DATABASE_ORDINARY} SYNC;"

From a64e625f0ba20751f929222bd49aaf1295ff15ed Mon Sep 17 00:00:00 2001
From: Nikita Fomichev <nikita.fomichev@clickhouse.com>
Date: Fri, 2 Aug 2024 14:03:27 +0200
Subject: [PATCH 1257/1488] Integration tests: fix ports clashing problem 2

---
 tests/integration/conftest.py | 36 ++++++++++++++---------------------
 1 file changed, 14 insertions(+), 22 deletions(-)

diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py
index 0a47840ede3..a386ed53009 100644
--- a/tests/integration/conftest.py
+++ b/tests/integration/conftest.py
@@ -2,11 +2,9 @@
 
 import logging
 import os
-import socket
-import multiprocessing
 
 import pytest  # pylint:disable=import-error; for style check
-from helpers.cluster import run_and_check
+from helpers.cluster import run_and_check, is_port_free
 from helpers.network import _NetworkManager
 
 # This is a workaround for a problem with logging in pytest [1].
@@ -114,15 +112,16 @@ def pytest_addoption(parser):
     )
 
 
-def get_n_free_ports(total):
+def get_unique_free_ports(total):
     ports = []
+    for port in range(30000, 55000):
+        if is_port_free(port) and port not in ports:
+            ports.append(port)
 
-    while len(ports) < total:
-        with socket.socket() as s:
-            s.bind(("", 0))
-            ports.append(s.getsockname()[1])
+        if len(ports) == total:
+            return ports
 
-    return ports
+    raise Exception(f"Can't collect {total} ports. Collected: {len(ports)}")
 
 
 def pytest_configure(config):
@@ -132,9 +131,8 @@ def pytest_configure(config):
     # the `pytest_xdist_setupnodes` hook is not executed
     worker_ports = os.getenv("WORKER_FREE_PORTS", None)
     if worker_ports is None:
-        os.environ["WORKER_FREE_PORTS"] = " ".join(
-            ([str(p) for p in get_n_free_ports(PORTS_PER_WORKER)])
-        )
+        master_ports = get_unique_free_ports(PORTS_PER_WORKER)
+        os.environ["WORKER_FREE_PORTS"] = " ".join(([str(p) for p in master_ports]))
 
 
 def pytest_xdist_setupnodes(config, specs):
@@ -142,18 +140,12 @@ def pytest_xdist_setupnodes(config, specs):
     # allocate pool of {PORTS_PER_WORKER} ports to each worker
 
     # Get number of xdist workers
-    num_workers = 1
-    if os.environ.get("PYTEST_XDIST_WORKER", "master") == "master":
-        num_workers = config.getoption("numprocesses", 1)
-        if num_workers == "auto":
-            num_workers = multiprocessing.cpu_count()
-
+    num_workers = len(specs)
     # Get free ports which will be distributed across workers
-    ports = get_n_free_ports(num_workers * PORTS_PER_WORKER)
+    ports = get_unique_free_ports(num_workers * PORTS_PER_WORKER)
 
     # Iterate over specs of workers and add allocated ports to env variable
     for i, spec in enumerate(specs):
         start_range = i * PORTS_PER_WORKER
-        spec.env["WORKER_FREE_PORTS"] = " ".join(
-            ([str(p) for p in ports[start_range : start_range + PORTS_PER_WORKER]])
-        )
+        per_workrer_ports = ports[start_range : start_range + PORTS_PER_WORKER]
+        spec.env["WORKER_FREE_PORTS"] = " ".join(([str(p) for p in per_workrer_ports]))

From 85cecf990ddefc25ff0903b6f97f84cd46dc471a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Fri, 2 Aug 2024 14:06:51 +0200
Subject: [PATCH 1258/1488] Parallelize
 02703_row_policies_for_database_combination

---
 ...3_row_policies_for_database_combination.sh | 92 +++++++++++++++++++
 ..._row_policies_for_database_combination.sql | 88 ------------------
 2 files changed, 92 insertions(+), 88 deletions(-)
 create mode 100755 tests/queries/0_stateless/02703_row_policies_for_database_combination.sh
 delete mode 100644 tests/queries/0_stateless/02703_row_policies_for_database_combination.sql

diff --git a/tests/queries/0_stateless/02703_row_policies_for_database_combination.sh b/tests/queries/0_stateless/02703_row_policies_for_database_combination.sh
new file mode 100755
index 00000000000..f7b7c814d29
--- /dev/null
+++ b/tests/queries/0_stateless/02703_row_policies_for_database_combination.sh
@@ -0,0 +1,92 @@
+#!/usr/bin/env bash
+CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CUR_DIR"/../shell_config.sh
+
+$CLICKHOUSE_CLIENT --multiquery "
+
+DROP TABLE IF EXISTS 02703_rptable;
+DROP TABLE IF EXISTS 02703_rptable_another;
+CREATE TABLE 02703_rptable (x UInt8, y UInt8) ENGINE = MergeTree ORDER BY x;
+
+INSERT INTO 02703_rptable VALUES (1, 10), (2, 20), (3, 30), (4, 40);
+
+CREATE TABLE 02703_rptable_another ENGINE = MergeTree ORDER BY x AS SELECT * FROM 02703_rptable;
+
+
+DROP ROW POLICY IF EXISTS 02703_filter_1 ON ${CLICKHOUSE_DATABASE}.02703_rptable;
+DROP ROW POLICY IF EXISTS 02703_filter_2 ON ${CLICKHOUSE_DATABASE}.*;
+DROP ROW POLICY IF EXISTS 02703_filter_3 ON ${CLICKHOUSE_DATABASE}.02703_rptable;
+DROP ROW POLICY IF EXISTS 02703_filter_4 ON ${CLICKHOUSE_DATABASE}.02703_rptable;
+DROP ROW POLICY IF EXISTS 02703_filter_5 ON ${CLICKHOUSE_DATABASE}.*;
+
+-- the test assumes users_without_row_policies_can_read_rows is true
+
+SELECT 'None';
+SELECT * FROM 02703_rptable;
+
+CREATE ROW POLICY 02703_filter_1 ON ${CLICKHOUSE_DATABASE}.02703_rptable USING x=1 AS permissive TO ALL;
+SELECT 'R1: x == 1';
+SELECT * FROM 02703_rptable;
+
+CREATE ROW POLICY 02703_filter_2 ON ${CLICKHOUSE_DATABASE}.* USING x=2 AS permissive TO ALL;
+SELECT 'R1, R2: (x == 1) OR (x == 2)';
+SELECT * FROM 02703_rptable;
+
+SELECT 'R1, R2: (x == 2) FROM ANOTHER';
+SELECT * FROM 02703_rptable_another;
+
+CREATE ROW POLICY 02703_filter_3 ON ${CLICKHOUSE_DATABASE}.02703_rptable USING x=3 AS permissive TO ALL;
+SELECT 'R1, R2, R3: (x == 1) OR (x == 2) OR (x == 3)';
+SELECT * FROM 02703_rptable;
+
+CREATE ROW POLICY 02703_filter_4 ON ${CLICKHOUSE_DATABASE}.02703_rptable USING x<=2 AS restrictive TO ALL;
+SELECT 'R1, R2, R3, R4: ((x == 1) OR (x == 2) OR (x == 3)) AND (x <= 2)';
+SELECT * FROM 02703_rptable;
+
+CREATE ROW POLICY 02703_filter_5 ON ${CLICKHOUSE_DATABASE}.* USING y>=20 AS restrictive TO ALL;
+SELECT 'R1, R2, R3, R4, R5: ((x == 1) OR (x == 2) OR (x == 3)) AND (x <= 2) AND (y >= 20)';
+SELECT * FROM 02703_rptable;
+
+CREATE TABLE 02703_after_rp ENGINE = MergeTree ORDER BY x AS SELECT * FROM 02703_rptable;
+SELECT * FROM 02703_after_rp;
+
+-- does not matter if policies or table are created first
+SELECT 'R1, R2, R3, R4, R5: (x == 2) AND (y >= 20) FROM AFTER_RP';
+SELECT * FROM 02703_after_rp;
+
+SELECT 'R1, R2, R3, R4, R5: (x == 2) AND (y >= 20) FROM ANOTHER';
+SELECT * FROM 02703_rptable_another;
+
+DROP ROW POLICY 02703_filter_1 ON ${CLICKHOUSE_DATABASE}.02703_rptable;
+SELECT 'R2, R3, R4, R5: ((x == 2) OR (x == 3)) AND (x <= 2) AND (y >= 20)';
+SELECT * FROM 02703_rptable;
+
+DROP ROW POLICY 02703_filter_2 ON ${CLICKHOUSE_DATABASE}.*;
+SELECT 'R3, R4, R5: (x == 3) AND (x <= 2) AND (y >= 20)';
+SELECT * FROM 02703_rptable;
+
+DROP ROW POLICY 02703_filter_3 ON ${CLICKHOUSE_DATABASE}.02703_rptable;
+SELECT 'R4, R5: (x <= 2) AND (y >= 20)';
+SELECT * FROM 02703_rptable;
+
+DROP ROW POLICY 02703_filter_4 ON ${CLICKHOUSE_DATABASE}.02703_rptable;
+SELECT 'R5: (x >= 2)';
+SELECT * FROM 02703_rptable;
+
+CREATE TABLE 02703_unexpected_columns (xx UInt8, yy UInt8) ENGINE = MergeTree ORDER BY xx;
+SELECT 'Policy not applicable';
+SELECT * FROM 02703_unexpected_columns; -- { serverError UNKNOWN_IDENTIFIER } -- Missing columns: 'x' while processing query
+
+DROP ROW POLICY 02703_filter_5 ON ${CLICKHOUSE_DATABASE}.*;
+SELECT 'None';
+SELECT * FROM 02703_rptable;
+
+SELECT 'No problematic policy, select works';
+SELECT 'Ok' FROM ${CLICKHOUSE_DATABASE}.02703_unexpected_columns;
+
+DROP TABLE 02703_rptable;
+DROP TABLE 02703_rptable_another;
+DROP TABLE 02703_unexpected_columns;
+
+"
diff --git a/tests/queries/0_stateless/02703_row_policies_for_database_combination.sql b/tests/queries/0_stateless/02703_row_policies_for_database_combination.sql
deleted file mode 100644
index 8c93fc595ba..00000000000
--- a/tests/queries/0_stateless/02703_row_policies_for_database_combination.sql
+++ /dev/null
@@ -1,88 +0,0 @@
--- Tags: no-parallel
-
-DROP DATABASE IF EXISTS 02703_db;
-CREATE DATABASE 02703_db;
-DROP TABLE IF EXISTS 02703_db.02703_rptable;
-DROP TABLE IF EXISTS 02703_db.02703_rptable_another;
-CREATE TABLE 02703_db.02703_rptable (x UInt8, y UInt8) ENGINE = MergeTree ORDER BY x;
-
-INSERT INTO 02703_db.02703_rptable VALUES (1, 10), (2, 20), (3, 30), (4, 40);
-
-CREATE TABLE 02703_db.02703_rptable_another ENGINE = MergeTree ORDER BY x AS SELECT * FROM 02703_db.02703_rptable;
-
-
-DROP ROW POLICY IF EXISTS 02703_filter_1 ON 02703_db.02703_rptable;
-DROP ROW POLICY IF EXISTS 02703_filter_2 ON 02703_db.*;
-DROP ROW POLICY IF EXISTS 02703_filter_3 ON 02703_db.02703_rptable;
-DROP ROW POLICY IF EXISTS 02703_filter_4 ON 02703_db.02703_rptable;
-DROP ROW POLICY IF EXISTS 02703_filter_5 ON 02703_db.*;
-
--- the test assumes users_without_row_policies_can_read_rows is true
-
-SELECT 'None';
-SELECT * FROM 02703_db.02703_rptable;
-
-CREATE ROW POLICY 02703_filter_1 ON 02703_db.02703_rptable USING x=1 AS permissive TO ALL;
-SELECT 'R1: x == 1';
-SELECT * FROM 02703_db.02703_rptable;
-
-CREATE ROW POLICY 02703_filter_2 ON 02703_db.* USING x=2 AS permissive TO ALL;
-SELECT 'R1, R2: (x == 1) OR (x == 2)';
-SELECT * FROM 02703_db.02703_rptable;
-
-SELECT 'R1, R2: (x == 2) FROM ANOTHER';
-SELECT * FROM 02703_db.02703_rptable_another;
-
-CREATE ROW POLICY 02703_filter_3 ON 02703_db.02703_rptable USING x=3 AS permissive TO ALL;
-SELECT 'R1, R2, R3: (x == 1) OR (x == 2) OR (x == 3)';
-SELECT * FROM 02703_db.02703_rptable;
-
-CREATE ROW POLICY 02703_filter_4 ON 02703_db.02703_rptable USING x<=2 AS restrictive TO ALL;
-SELECT 'R1, R2, R3, R4: ((x == 1) OR (x == 2) OR (x == 3)) AND (x <= 2)';
-SELECT * FROM 02703_db.02703_rptable;
-
-CREATE ROW POLICY 02703_filter_5 ON 02703_db.* USING y>=20 AS restrictive TO ALL;
-SELECT 'R1, R2, R3, R4, R5: ((x == 1) OR (x == 2) OR (x == 3)) AND (x <= 2) AND (y >= 20)';
-SELECT * FROM 02703_db.02703_rptable;
-
-CREATE TABLE 02703_db.02703_after_rp ENGINE = MergeTree ORDER BY x AS SELECT * FROM 02703_db.02703_rptable;
-SELECT * FROM 02703_db.02703_after_rp;
-
--- does not matter if policies or table are created first
-SELECT 'R1, R2, R3, R4, R5: (x == 2) AND (y >= 20) FROM AFTER_RP';
-SELECT * FROM 02703_db.02703_after_rp;
-
-SELECT 'R1, R2, R3, R4, R5: (x == 2) AND (y >= 20) FROM ANOTHER';
-SELECT * FROM 02703_db.02703_rptable_another;
-
-DROP ROW POLICY 02703_filter_1 ON 02703_db.02703_rptable;
-SELECT 'R2, R3, R4, R5: ((x == 2) OR (x == 3)) AND (x <= 2) AND (y >= 20)';
-SELECT * FROM 02703_db.02703_rptable;
-
-DROP ROW POLICY 02703_filter_2 ON 02703_db.*;
-SELECT 'R3, R4, R5: (x == 3) AND (x <= 2) AND (y >= 20)';
-SELECT * FROM 02703_db.02703_rptable;
-
-DROP ROW POLICY 02703_filter_3 ON 02703_db.02703_rptable;
-SELECT 'R4, R5: (x <= 2) AND (y >= 20)';
-SELECT * FROM 02703_db.02703_rptable;
-
-DROP ROW POLICY 02703_filter_4 ON 02703_db.02703_rptable;
-SELECT 'R5: (x >= 2)';
-SELECT * FROM 02703_db.02703_rptable;
-
-CREATE TABLE 02703_db.02703_unexpected_columns (xx UInt8, yy UInt8) ENGINE = MergeTree ORDER BY xx;
-SELECT 'Policy not applicable';
-SELECT * FROM 02703_db.02703_unexpected_columns; -- { serverError UNKNOWN_IDENTIFIER } -- Missing columns: 'x' while processing query
-
-DROP ROW POLICY 02703_filter_5 ON 02703_db.*;
-SELECT 'None';
-SELECT * FROM 02703_db.02703_rptable;
-
-SELECT 'No problematic policy, select works';
-SELECT 'Ok' FROM 02703_db.02703_unexpected_columns;
-
-DROP TABLE 02703_db.02703_rptable;
-DROP TABLE 02703_db.02703_rptable_another;
-DROP TABLE 02703_db.02703_unexpected_columns;
-DROP DATABASE 02703_db;

From f55784c636c1dcb503dadb2f75bd6b586271bf0d Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Fri, 2 Aug 2024 12:07:05 +0000
Subject: [PATCH 1259/1488] Fix 03203_client_benchmark_options

---
 tests/queries/0_stateless/03203_client_benchmark_options.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/queries/0_stateless/03203_client_benchmark_options.sh b/tests/queries/0_stateless/03203_client_benchmark_options.sh
index 37a1f2cd3ac..967db056c0b 100755
--- a/tests/queries/0_stateless/03203_client_benchmark_options.sh
+++ b/tests/queries/0_stateless/03203_client_benchmark_options.sh
@@ -5,10 +5,10 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CUR_DIR"/../shell_config.sh
 
 output=$(${CLICKHOUSE_CLIENT} -t -q "SELECT sleepEachRow(2) FORMAT Null" 2>&1)
-{ echo "$output" | grep -q "^2\." && echo "Ok"; } || { echo "Fail"; echo "'$output'"; }
+{ number=$(echo "$output" | grep -o "^[0-9]"); [[ -n "$number" && "$number" -ge 2 ]] && echo "Ok"; } || { echo "Fail"; echo "'$output'"; }
 
 output=$(${CLICKHOUSE_CLIENT} --time -q "SELECT sleepEachRow(2) FORMAT Null" 2>&1)
-{ echo "$output" | grep -q "^2\." && echo "Ok"; } || { echo "Fail"; echo "'$output'"; }
+{ number=$(echo "$output" | grep -o "^[0-9]"); [[ -n "$number" && "$number" -ge 2 ]] && echo "Ok"; } || { echo "Fail"; echo "'$output'"; }
 
 output=$(${CLICKHOUSE_CLIENT} --memory-usage -q "SELECT sum(number) FROM numbers(10_000) FORMAT Null" 2>&1)
 { echo "$output" | grep -q "^[0-9]\+$" && echo "Ok"; } || { echo "Fail"; echo "'$output'"; }

From cb83274d37f8e90ef9c79451ce01a2ba95363158 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Fri, 2 Aug 2024 14:18:31 +0200
Subject: [PATCH 1260/1488] Make 02703_row_policy_for_database parallelizable

---
 .../02703_row_policy_for_database.reference   | 22 +++----
 .../02703_row_policy_for_database.sh          | 59 +++++++++++++++++++
 .../02703_row_policy_for_database.sql         | 53 -----------------
 3 files changed, 70 insertions(+), 64 deletions(-)
 create mode 100755 tests/queries/0_stateless/02703_row_policy_for_database.sh
 delete mode 100644 tests/queries/0_stateless/02703_row_policy_for_database.sql

diff --git a/tests/queries/0_stateless/02703_row_policy_for_database.reference b/tests/queries/0_stateless/02703_row_policy_for_database.reference
index b67ea69ae72..56b0d8c6f20 100644
--- a/tests/queries/0_stateless/02703_row_policy_for_database.reference
+++ b/tests/queries/0_stateless/02703_row_policy_for_database.reference
@@ -1,20 +1,20 @@
 -- row policies for database
-  -- SHOW CREATE POLICY db1_02703 ON db1_02703.*
-CREATE ROW POLICY db1_02703 ON db1_02703.* FOR SELECT USING 1 TO ALL
-  -- SHOW CREATE POLICY ON db1_02703.*
-CREATE ROW POLICY db1_02703 ON db1_02703.* FOR SELECT USING 1 TO ALL
-CREATE ROW POLICY tbl1_02703 ON db1_02703.`table` FOR SELECT USING 1 TO ALL
-  -- SHOW CREATE POLICY ON db1_02703.`*`
+  -- SHOW CREATE POLICY default ON default.*
+CREATE ROW POLICY default_db_policy ON default.* FOR SELECT USING 1 TO ALL
+  -- SHOW CREATE POLICY ON default.*
+CREATE ROW POLICY default_db_policy ON default.* FOR SELECT USING 1 TO ALL
+CREATE ROW POLICY default_tb_policy ON default.`table` FOR SELECT USING 1 TO ALL
+  -- SHOW CREATE POLICY ON default.`*`
 R1, R2: (x == 1) OR (x == 2)
 1
 2
 Check system.query_log
 SELECT \'-- row policies for database\';	[]
-SELECT \'  -- SHOW CREATE POLICY db1_02703 ON db1_02703.*\';	[]
-SELECT \'  -- SHOW CREATE POLICY ON db1_02703.*\';	[]
-SELECT \'  -- SHOW CREATE POLICY ON db1_02703.`*`\';	[]
+SELECT \'  -- SHOW CREATE POLICY default ON default.*\';	[]
+SELECT \'  -- SHOW CREATE POLICY ON default.*\';	[]
+SELECT \'  -- SHOW CREATE POLICY ON default.`*`\';	[]
 SELECT \'R1, R2: (x == 1) OR (x == 2)\';	[]
-SELECT * FROM 02703_rqtable_default;	['`02703_filter_11_db` ON default.*','`02703_filter_11` ON default.`02703_rqtable_default`']
+SELECT * FROM 02703_rqtable_default;	['default_filter_11_db_policy ON default.*','default_filter_11_policy ON default.`02703_rqtable_default`']
 SELECT \'Check system.query_log\';	[]
   -- CREATE DATABASE-LEVEL POLICY IN CURRENT DATABASE
-CREATE ROW POLICY db2_02703 ON db1_02703.* TO u1_02703
+CREATE ROW POLICY db2_02703 ON default.* TO user_default
diff --git a/tests/queries/0_stateless/02703_row_policy_for_database.sh b/tests/queries/0_stateless/02703_row_policy_for_database.sh
new file mode 100755
index 00000000000..e94bc7acd5e
--- /dev/null
+++ b/tests/queries/0_stateless/02703_row_policy_for_database.sh
@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CUR_DIR"/../shell_config.sh
+
+CLICKHOUSE_USER="user_$CLICKHOUSE_DATABASE"
+
+$CLICKHOUSE_CLIENT --multiquery "
+
+DROP USER IF EXISTS ${CLICKHOUSE_USER};
+CREATE USER ${CLICKHOUSE_USER};
+
+CREATE TABLE ${CLICKHOUSE_DATABASE}.02703_rqtable (x UInt8) ENGINE = MergeTree ORDER BY x;
+INSERT INTO ${CLICKHOUSE_DATABASE}.02703_rqtable VALUES (1), (2), (3), (4);
+
+SELECT '-- row policies for database';
+CREATE ROW POLICY ${CLICKHOUSE_DATABASE}_db_policy ON ${CLICKHOUSE_DATABASE}.* USING 1 AS PERMISSIVE TO ALL;
+CREATE ROW POLICY ${CLICKHOUSE_DATABASE}_tb_policy ON ${CLICKHOUSE_DATABASE}.table USING 1 AS PERMISSIVE TO ALL;
+SELECT '  -- SHOW CREATE POLICY ${CLICKHOUSE_DATABASE} ON ${CLICKHOUSE_DATABASE}.*';
+SHOW CREATE POLICY ${CLICKHOUSE_DATABASE}_db_policy ON ${CLICKHOUSE_DATABASE}.*;
+SELECT '  -- SHOW CREATE POLICY ON ${CLICKHOUSE_DATABASE}.*';
+SHOW CREATE POLICY ON ${CLICKHOUSE_DATABASE}.*;
+SELECT '  -- SHOW CREATE POLICY ON ${CLICKHOUSE_DATABASE}.\`*\`';
+SHOW CREATE POLICY ON ${CLICKHOUSE_DATABASE}.\`*\`;
+DROP POLICY ${CLICKHOUSE_DATABASE}_db_policy ON ${CLICKHOUSE_DATABASE}.*;
+DROP POLICY ${CLICKHOUSE_DATABASE}_tb_policy ON ${CLICKHOUSE_DATABASE}.table;
+"
+
+$CLICKHOUSE_CLIENT --query "CREATE ROW POLICY any_02703 ON *.some_table USING 1 AS PERMISSIVE TO ALL;" 2>&1 | grep -q "SYNTAX_ERROR"
+
+$CLICKHOUSE_CLIENT --multiquery "
+CREATE TABLE 02703_rqtable_default (x UInt8) ENGINE = MergeTree ORDER BY x;
+
+CREATE ROW POLICY ${CLICKHOUSE_DATABASE}_filter_11_db_policy ON * USING x=1 AS permissive TO ALL;
+CREATE ROW POLICY ${CLICKHOUSE_DATABASE}_filter_11_policy ON 02703_rqtable_default USING x=2 AS permissive TO ALL;
+
+INSERT INTO 02703_rqtable_default VALUES (1), (2), (3), (4);
+
+SELECT 'R1, R2: (x == 1) OR (x == 2)';
+SELECT * FROM 02703_rqtable_default;
+
+DROP TABLE 02703_rqtable_default;
+
+SELECT 'Check system.query_log';
+SYSTEM FLUSH LOGS;
+SELECT query, used_row_policies FROM system.query_log WHERE current_database == currentDatabase() AND type == 'QueryStart' AND query_kind == 'Select' ORDER BY event_time_microseconds;
+
+DROP ROW POLICY ${CLICKHOUSE_DATABASE}_filter_11_db_policy ON *;
+DROP ROW POLICY ${CLICKHOUSE_DATABASE}_filter_11_policy ON 02703_rqtable_default;
+
+USE ${CLICKHOUSE_DATABASE};
+SELECT '  -- CREATE DATABASE-LEVEL POLICY IN CURRENT DATABASE';
+CREATE ROW POLICY db2_02703 ON * TO ${CLICKHOUSE_USER};
+SHOW CREATE POLICY db2_02703 ON *;
+
+DROP ROW POLICY db2_02703 ON *;
+
+DROP USER ${CLICKHOUSE_USER};
+"
diff --git a/tests/queries/0_stateless/02703_row_policy_for_database.sql b/tests/queries/0_stateless/02703_row_policy_for_database.sql
deleted file mode 100644
index 51ce5f4f870..00000000000
--- a/tests/queries/0_stateless/02703_row_policy_for_database.sql
+++ /dev/null
@@ -1,53 +0,0 @@
--- Tags: no-parallel
-
-DROP DATABASE IF EXISTS db1_02703;
-DROP USER IF EXISTS u1_02703;
-CREATE USER u1_02703;
-
-CREATE DATABASE db1_02703;
-
-CREATE TABLE db1_02703.02703_rqtable (x UInt8) ENGINE = MergeTree ORDER BY x;
-INSERT INTO db1_02703.02703_rqtable VALUES (1), (2), (3), (4);
-
-
-SELECT '-- row policies for database';
-CREATE ROW POLICY db1_02703 ON db1_02703.* USING 1 AS PERMISSIVE TO ALL;
-CREATE ROW POLICY tbl1_02703 ON db1_02703.table USING 1 AS PERMISSIVE TO ALL;
-SELECT '  -- SHOW CREATE POLICY db1_02703 ON db1_02703.*';
-SHOW CREATE POLICY db1_02703 ON db1_02703.*;
-SELECT '  -- SHOW CREATE POLICY ON db1_02703.*';
-SHOW CREATE POLICY ON db1_02703.*;
-SELECT '  -- SHOW CREATE POLICY ON db1_02703.`*`';
-SHOW CREATE POLICY ON db1_02703.`*`;
-DROP POLICY db1_02703 ON db1_02703.*;
-DROP POLICY tbl1_02703 ON db1_02703.table;
-
-CREATE ROW POLICY any_02703 ON *.some_table USING 1 AS PERMISSIVE TO ALL; -- { clientError SYNTAX_ERROR }
-
-CREATE TABLE 02703_rqtable_default (x UInt8) ENGINE = MergeTree ORDER BY x;
-
-CREATE ROW POLICY 02703_filter_11_db ON * USING x=1 AS permissive TO ALL;
-CREATE ROW POLICY 02703_filter_11 ON 02703_rqtable_default USING x=2 AS permissive TO ALL;
-
-INSERT INTO 02703_rqtable_default VALUES (1), (2), (3), (4);
-
-SELECT 'R1, R2: (x == 1) OR (x == 2)';
-SELECT * FROM 02703_rqtable_default;
-
-DROP TABLE 02703_rqtable_default;
-
-SELECT 'Check system.query_log';
-SYSTEM FLUSH LOGS;
-SELECT query, used_row_policies FROM system.query_log WHERE current_database == currentDatabase() AND type == 'QueryStart' AND query_kind == 'Select' ORDER BY event_time_microseconds;
-
-DROP ROW POLICY 02703_filter_11_db ON *;
-DROP ROW POLICY 02703_filter_11 ON 02703_rqtable_default;
-
-USE db1_02703;
-SELECT '  -- CREATE DATABASE-LEVEL POLICY IN CURRENT DATABASE';
-CREATE ROW POLICY db2_02703 ON * TO u1_02703;
-SHOW CREATE POLICY db2_02703 ON *;
-
-DROP ROW POLICY db2_02703 ON *;
-
-DROP USER u1_02703;

From 829d07c3a54716bb62cd36d4f2fa24da64ea6a0a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Fri, 2 Aug 2024 14:22:37 +0200
Subject: [PATCH 1261/1488] Make 02703_row_policies_for_asterisk parallelizable

---
 .../02703_row_policies_for_asterisk.sh        | 14 +++++++++
 .../02703_row_policies_for_asterisk.sql       | 11 -------
 ...3_row_policies_for_database_combination.sh | 30 +++++++++----------
 3 files changed, 29 insertions(+), 26 deletions(-)
 create mode 100755 tests/queries/0_stateless/02703_row_policies_for_asterisk.sh
 delete mode 100644 tests/queries/0_stateless/02703_row_policies_for_asterisk.sql

diff --git a/tests/queries/0_stateless/02703_row_policies_for_asterisk.sh b/tests/queries/0_stateless/02703_row_policies_for_asterisk.sh
new file mode 100755
index 00000000000..f9670e5f6f8
--- /dev/null
+++ b/tests/queries/0_stateless/02703_row_policies_for_asterisk.sh
@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CUR_DIR"/../shell_config.sh
+
+$CLICKHOUSE_CLIENT --multiquery "
+    SELECT 'Policy for table \`*\` does not affect other tables in the database';
+    CREATE ROW POLICY 02703_asterisk_${CLICKHOUSE_DATABASE}_policy ON ${CLICKHOUSE_DATABASE}.\`*\` USING x=1 AS permissive TO ALL;
+    CREATE TABLE ${CLICKHOUSE_DATABASE}.\`*\` (x UInt8, y UInt8) ENGINE = MergeTree ORDER BY x AS SELECT 100, 20;
+    CREATE TABLE ${CLICKHOUSE_DATABASE}.\`other\` (x UInt8, y UInt8) ENGINE = MergeTree ORDER BY x AS SELECT 100, 20;
+    SELECT 'star', * FROM ${CLICKHOUSE_DATABASE}.\`*\`;
+    SELECT 'other', * FROM ${CLICKHOUSE_DATABASE}.other;
+    DROP ROW POLICY 02703_asterisk_${CLICKHOUSE_DATABASE}_policy ON ${CLICKHOUSE_DATABASE}.\`*\`;
+"
diff --git a/tests/queries/0_stateless/02703_row_policies_for_asterisk.sql b/tests/queries/0_stateless/02703_row_policies_for_asterisk.sql
deleted file mode 100644
index 96b1c01a6d6..00000000000
--- a/tests/queries/0_stateless/02703_row_policies_for_asterisk.sql
+++ /dev/null
@@ -1,11 +0,0 @@
--- Tags: no-parallel
-
-SELECT 'Policy for table `*` does not affect other tables in the database';
-CREATE DATABASE 02703_db_asterisk;
-CREATE ROW POLICY 02703_asterisk ON 02703_db_asterisk.`*` USING x=1 AS permissive TO ALL;
-CREATE TABLE 02703_db_asterisk.`*` (x UInt8, y UInt8) ENGINE = MergeTree ORDER BY x AS SELECT 100, 20;
-CREATE TABLE 02703_db_asterisk.`other` (x UInt8, y UInt8) ENGINE = MergeTree ORDER BY x AS SELECT 100, 20;
-SELECT 'star', * FROM 02703_db_asterisk.`*`;
-SELECT 'other', * FROM 02703_db_asterisk.other;
-DROP ROW POLICY 02703_asterisk ON 02703_db_asterisk.`*`;
-DROP DATABASE 02703_db_asterisk;
diff --git a/tests/queries/0_stateless/02703_row_policies_for_database_combination.sh b/tests/queries/0_stateless/02703_row_policies_for_database_combination.sh
index f7b7c814d29..35151eed220 100755
--- a/tests/queries/0_stateless/02703_row_policies_for_database_combination.sh
+++ b/tests/queries/0_stateless/02703_row_policies_for_database_combination.sh
@@ -14,37 +14,37 @@ INSERT INTO 02703_rptable VALUES (1, 10), (2, 20), (3, 30), (4, 40);
 CREATE TABLE 02703_rptable_another ENGINE = MergeTree ORDER BY x AS SELECT * FROM 02703_rptable;
 
 
-DROP ROW POLICY IF EXISTS 02703_filter_1 ON ${CLICKHOUSE_DATABASE}.02703_rptable;
-DROP ROW POLICY IF EXISTS 02703_filter_2 ON ${CLICKHOUSE_DATABASE}.*;
-DROP ROW POLICY IF EXISTS 02703_filter_3 ON ${CLICKHOUSE_DATABASE}.02703_rptable;
-DROP ROW POLICY IF EXISTS 02703_filter_4 ON ${CLICKHOUSE_DATABASE}.02703_rptable;
-DROP ROW POLICY IF EXISTS 02703_filter_5 ON ${CLICKHOUSE_DATABASE}.*;
+DROP ROW POLICY IF EXISTS 02703_filter_policy_${CLICKHOUSE_DATABASE}_1 ON ${CLICKHOUSE_DATABASE}.02703_rptable;
+DROP ROW POLICY IF EXISTS 02703_filter_policy_${CLICKHOUSE_DATABASE}_2 ON ${CLICKHOUSE_DATABASE}.*;
+DROP ROW POLICY IF EXISTS 02703_filter_policy_${CLICKHOUSE_DATABASE}_3 ON ${CLICKHOUSE_DATABASE}.02703_rptable;
+DROP ROW POLICY IF EXISTS 02703_filter_policy_${CLICKHOUSE_DATABASE}_4 ON ${CLICKHOUSE_DATABASE}.02703_rptable;
+DROP ROW POLICY IF EXISTS 02703_filter_policy_${CLICKHOUSE_DATABASE}_5 ON ${CLICKHOUSE_DATABASE}.*;
 
 -- the test assumes users_without_row_policies_can_read_rows is true
 
 SELECT 'None';
 SELECT * FROM 02703_rptable;
 
-CREATE ROW POLICY 02703_filter_1 ON ${CLICKHOUSE_DATABASE}.02703_rptable USING x=1 AS permissive TO ALL;
+CREATE ROW POLICY 02703_filter_policy_${CLICKHOUSE_DATABASE}_1 ON ${CLICKHOUSE_DATABASE}.02703_rptable USING x=1 AS permissive TO ALL;
 SELECT 'R1: x == 1';
 SELECT * FROM 02703_rptable;
 
-CREATE ROW POLICY 02703_filter_2 ON ${CLICKHOUSE_DATABASE}.* USING x=2 AS permissive TO ALL;
+CREATE ROW POLICY 02703_filter_policy_${CLICKHOUSE_DATABASE}_2 ON ${CLICKHOUSE_DATABASE}.* USING x=2 AS permissive TO ALL;
 SELECT 'R1, R2: (x == 1) OR (x == 2)';
 SELECT * FROM 02703_rptable;
 
 SELECT 'R1, R2: (x == 2) FROM ANOTHER';
 SELECT * FROM 02703_rptable_another;
 
-CREATE ROW POLICY 02703_filter_3 ON ${CLICKHOUSE_DATABASE}.02703_rptable USING x=3 AS permissive TO ALL;
+CREATE ROW POLICY 02703_filter_policy_${CLICKHOUSE_DATABASE}_3 ON ${CLICKHOUSE_DATABASE}.02703_rptable USING x=3 AS permissive TO ALL;
 SELECT 'R1, R2, R3: (x == 1) OR (x == 2) OR (x == 3)';
 SELECT * FROM 02703_rptable;
 
-CREATE ROW POLICY 02703_filter_4 ON ${CLICKHOUSE_DATABASE}.02703_rptable USING x<=2 AS restrictive TO ALL;
+CREATE ROW POLICY 02703_filter_policy_${CLICKHOUSE_DATABASE}_4 ON ${CLICKHOUSE_DATABASE}.02703_rptable USING x<=2 AS restrictive TO ALL;
 SELECT 'R1, R2, R3, R4: ((x == 1) OR (x == 2) OR (x == 3)) AND (x <= 2)';
 SELECT * FROM 02703_rptable;
 
-CREATE ROW POLICY 02703_filter_5 ON ${CLICKHOUSE_DATABASE}.* USING y>=20 AS restrictive TO ALL;
+CREATE ROW POLICY 02703_filter_policy_${CLICKHOUSE_DATABASE}_5 ON ${CLICKHOUSE_DATABASE}.* USING y>=20 AS restrictive TO ALL;
 SELECT 'R1, R2, R3, R4, R5: ((x == 1) OR (x == 2) OR (x == 3)) AND (x <= 2) AND (y >= 20)';
 SELECT * FROM 02703_rptable;
 
@@ -58,19 +58,19 @@ SELECT * FROM 02703_after_rp;
 SELECT 'R1, R2, R3, R4, R5: (x == 2) AND (y >= 20) FROM ANOTHER';
 SELECT * FROM 02703_rptable_another;
 
-DROP ROW POLICY 02703_filter_1 ON ${CLICKHOUSE_DATABASE}.02703_rptable;
+DROP ROW POLICY 02703_filter_policy_${CLICKHOUSE_DATABASE}_1 ON ${CLICKHOUSE_DATABASE}.02703_rptable;
 SELECT 'R2, R3, R4, R5: ((x == 2) OR (x == 3)) AND (x <= 2) AND (y >= 20)';
 SELECT * FROM 02703_rptable;
 
-DROP ROW POLICY 02703_filter_2 ON ${CLICKHOUSE_DATABASE}.*;
+DROP ROW POLICY 02703_filter_policy_${CLICKHOUSE_DATABASE}_2 ON ${CLICKHOUSE_DATABASE}.*;
 SELECT 'R3, R4, R5: (x == 3) AND (x <= 2) AND (y >= 20)';
 SELECT * FROM 02703_rptable;
 
-DROP ROW POLICY 02703_filter_3 ON ${CLICKHOUSE_DATABASE}.02703_rptable;
+DROP ROW POLICY 02703_filter_policy_${CLICKHOUSE_DATABASE}_3 ON ${CLICKHOUSE_DATABASE}.02703_rptable;
 SELECT 'R4, R5: (x <= 2) AND (y >= 20)';
 SELECT * FROM 02703_rptable;
 
-DROP ROW POLICY 02703_filter_4 ON ${CLICKHOUSE_DATABASE}.02703_rptable;
+DROP ROW POLICY 02703_filter_policy_${CLICKHOUSE_DATABASE}_4 ON ${CLICKHOUSE_DATABASE}.02703_rptable;
 SELECT 'R5: (x >= 2)';
 SELECT * FROM 02703_rptable;
 
@@ -78,7 +78,7 @@ CREATE TABLE 02703_unexpected_columns (xx UInt8, yy UInt8) ENGINE = MergeTree OR
 SELECT 'Policy not applicable';
 SELECT * FROM 02703_unexpected_columns; -- { serverError UNKNOWN_IDENTIFIER } -- Missing columns: 'x' while processing query
 
-DROP ROW POLICY 02703_filter_5 ON ${CLICKHOUSE_DATABASE}.*;
+DROP ROW POLICY 02703_filter_policy_${CLICKHOUSE_DATABASE}_5 ON ${CLICKHOUSE_DATABASE}.*;
 SELECT 'None';
 SELECT * FROM 02703_rptable;
 

From 5256e8e6d08b5076be49b56fdebadb668892771c Mon Sep 17 00:00:00 2001
From: Nikita Fomichev <nikita.fomichev@clickhouse.com>
Date: Fri, 2 Aug 2024 14:36:33 +0200
Subject: [PATCH 1262/1488] Integration tests: fix ports clashing problem 3

---
 tests/integration/conftest.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py
index a386ed53009..b4c86a1cd2f 100644
--- a/tests/integration/conftest.py
+++ b/tests/integration/conftest.py
@@ -132,7 +132,7 @@ def pytest_configure(config):
     worker_ports = os.getenv("WORKER_FREE_PORTS", None)
     if worker_ports is None:
         master_ports = get_unique_free_ports(PORTS_PER_WORKER)
-        os.environ["WORKER_FREE_PORTS"] = " ".join(([str(p) for p in master_ports]))
+        os.environ["WORKER_FREE_PORTS"] = " ".join([str(p) for p in master_ports])
 
 
 def pytest_xdist_setupnodes(config, specs):
@@ -148,4 +148,4 @@ def pytest_xdist_setupnodes(config, specs):
     for i, spec in enumerate(specs):
         start_range = i * PORTS_PER_WORKER
         per_workrer_ports = ports[start_range : start_range + PORTS_PER_WORKER]
-        spec.env["WORKER_FREE_PORTS"] = " ".join(([str(p) for p in per_workrer_ports]))
+        spec.env["WORKER_FREE_PORTS"] = " ".join([str(p) for p in per_workrer_ports])

From db0bce33526abf16e705b9e56d178d6e2c45a36b Mon Sep 17 00:00:00 2001
From: kssenii <sumarokovakseniia@mail.ru>
Date: Fri, 2 Aug 2024 15:01:15 +0200
Subject: [PATCH 1263/1488] Try make the code more understandable

---
 src/Interpreters/Cache/FileCache.cpp   | 164 ++++++++++++-------------
 src/Interpreters/Cache/FileCache.h     |  12 +-
 tests/config/config.d/storage_conf.xml |   2 +-
 3 files changed, 80 insertions(+), 98 deletions(-)

diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp
index 0a03f5dcc7d..4c17afb79be 100644
--- a/src/Interpreters/Cache/FileCache.cpp
+++ b/src/Interpreters/Cache/FileCache.cpp
@@ -318,7 +318,29 @@ FileSegments FileCache::getImpl(const LockedKey & locked_key, const FileSegment:
 
 std::vector<FileSegment::Range> FileCache::splitRange(size_t offset, size_t size, size_t aligned_size)
 {
-    assert(size > 0);
+    chassert(size > 0);
+    chassert(size <= aligned_size);
+
+    /// Consider this example to understand why we need to account here for both `size` and `aligned_size`.
+    /// [________________]__________________] <-- requested range
+    ///                  ^                  ^
+    ///                right offset         aligned_right_offset
+    /// [_________]                           <-- last cached file segment, e.g. we have uncovered suffix of the requested range
+    /// [________________]
+    ///        size
+    /// [____________________________________]
+    ///        aligned_size
+    ///
+    /// So it is possible that we split this hole range into sub-segments by `max_file_segment_size`
+    /// and get something like this:
+    ///
+    /// [________________________]
+    ///                  ^             ^
+    ///              right_offset   right_offset + max_file_segment_size
+    /// e.g. there is no need to create sub-segment for range (right_offset + max_file_segment_size, aligned_right_offset].
+    /// Because its left offset would be bigger than right_offset.
+    /// Therefore, we set end_pos_non_included as offset+size, but remaining_size as aligned_size.
+
     std::vector<FileSegment::Range> ranges;
 
     size_t current_pos = offset;
@@ -339,42 +361,23 @@ std::vector<FileSegment::Range> FileCache::splitRange(size_t offset, size_t size
     return ranges;
 }
 
-FileSegments FileCache::splitRangeIntoFileSegments(
+FileSegments FileCache::createFileSegmentsFromRanges(
     LockedKey & locked_key,
-    size_t offset,
-    size_t size,
-    size_t aligned_size,
-    FileSegment::State state,
+    const std::vector<FileSegment::Range> & ranges,
+    size_t & file_segments_count,
     size_t file_segments_limit,
     const CreateFileSegmentSettings & create_settings)
 {
-    chassert(size > 0);
-    chassert(size <= aligned_size);
-    /// We take `size` as a soft limit and `aligned_size` as a hard limit.
-
-    auto current_pos = offset;
-    auto end_pos_non_included = offset + size;
-
-    size_t current_file_segment_size;
-    size_t remaining_size = aligned_size;
-
-    FileSegments file_segments;
-    const size_t max_size = max_file_segment_size.load();
-    while (current_pos < end_pos_non_included && (!file_segments_limit || file_segments.size() < file_segments_limit))
+    FileSegments result;
+    for (const auto & r : ranges)
     {
-        current_file_segment_size = std::min(remaining_size, max_size);
-        remaining_size -= current_file_segment_size;
-
-        auto file_segment_metadata_it = addFileSegment(
-            locked_key, current_pos, current_file_segment_size, state, create_settings, nullptr);
-        file_segments.push_back(file_segment_metadata_it->second->file_segment);
-
-        current_pos += current_file_segment_size;
+        if (file_segments_limit && file_segments_count >= file_segments_limit)
+            break;
+        auto metadata_it = addFileSegment(locked_key, r.left, r.size(), FileSegment::State::EMPTY, create_settings, nullptr);
+        result.push_back(metadata_it->second->file_segment);
+        ++file_segments_count;
     }
-
-    chassert(file_segments.size() == file_segments_limit || file_segments.back()->range().contains(offset + size - 1),
-             fmt::format("Offset: {}, size: {}, file segments: {}", offset, size, toString(file_segments)));
-    return file_segments;
+    return result;
 }
 
 void FileCache::fillHolesWithEmptyFileSegments(
@@ -448,18 +451,9 @@ void FileCache::fillHolesWithEmptyFileSegments(
         }
         else
         {
-            auto ranges = splitRange(current_pos, hole_size, hole_size);
-            FileSegments hole;
-            for (const auto & r : ranges)
-            {
-                auto metadata_it = addFileSegment(locked_key, r.left, r.size(), FileSegment::State::EMPTY, create_settings, nullptr);
-                hole.push_back(metadata_it->second->file_segment);
-                ++processed_count;
-
-                if (is_limit_reached())
-                    break;
-            }
-            file_segments.splice(it, std::move(hole));
+            const auto ranges = splitRange(current_pos, hole_size, hole_size);
+            auto hole_segments = createFileSegmentsFromRanges(locked_key, ranges, processed_count, file_segments_limit, create_settings);
+            file_segments.splice(it, std::move(hole_segments));
         }
 
         if (is_limit_reached())
@@ -493,29 +487,20 @@ void FileCache::fillHolesWithEmptyFileSegments(
         /// segmentN
 
         auto hole_size = range.right - current_pos + 1;
-        auto non_aligned_size = non_aligned_right_offset - current_pos + 1;
+        auto non_aligned_hole_size = non_aligned_right_offset - current_pos + 1;
 
         if (fill_with_detached_file_segments)
         {
             auto file_segment = std::make_shared<FileSegment>(
-                locked_key.getKey(), current_pos, hole_size, FileSegment::State::DETACHED, create_settings);
+                locked_key.getKey(), current_pos, non_aligned_hole_size, FileSegment::State::DETACHED, create_settings);
 
             file_segments.insert(file_segments.end(), file_segment);
         }
         else
         {
-            auto ranges = splitRange(current_pos, non_aligned_size, hole_size);
-            FileSegments hole;
-            for (const auto & r : ranges)
-            {
-                auto metadata_it = addFileSegment(locked_key, r.left, r.size(), FileSegment::State::EMPTY, create_settings, nullptr);
-                hole.push_back(metadata_it->second->file_segment);
-                ++processed_count;
-
-                if (is_limit_reached())
-                    break;
-            }
-            file_segments.splice(it, std::move(hole));
+            const auto ranges = splitRange(current_pos, non_aligned_hole_size, hole_size);
+            auto hole_segments = createFileSegmentsFromRanges(locked_key, ranges, processed_count, file_segments_limit, create_settings);
+            file_segments.splice(it, std::move(hole_segments));
 
             if (is_limit_reached())
                 erase_unprocessed();
@@ -548,8 +533,9 @@ FileSegmentsHolderPtr FileCache::set(
     }
     else
     {
-        file_segments = splitRangeIntoFileSegments(
-            *locked_key, offset, size, size, FileSegment::State::EMPTY, /* file_segments_limit */0, create_settings);
+        const auto ranges = splitRange(offset, size, size);
+        size_t file_segments_count = 0;
+        file_segments = createFileSegmentsFromRanges(*locked_key, ranges, file_segments_count, /* file_segments_limit */0, create_settings);
     }
 
     return std::make_unique<FileSegmentsHolder>(std::move(file_segments));
@@ -569,23 +555,27 @@ FileCache::getOrSet(
 
     assertInitialized();
 
-    FileSegment::Range range(offset, offset + size - 1);
+    FileSegment::Range initial_range(offset, offset + size - 1);
+    /// result_range is initial range, which will be adjusted according to
+    /// 1. aligned offset, alighed_end_offset
+    /// 2. max_file_segments_limit
+    FileSegment::Range result_range = initial_range;
 
-    const auto aligned_offset = roundDownToMultiple(range.left, boundary_alignment);
-    auto aligned_end_offset = std::min(roundUpToMultiple(offset + size, boundary_alignment), file_size) - 1;
+    const auto aligned_offset = roundDownToMultiple(initial_range.left, boundary_alignment);
+    auto aligned_end_offset = std::min(roundUpToMultiple(initial_range.right + 1, boundary_alignment), file_size) - 1;
 
-    chassert(aligned_offset <= range.left);
-    chassert(aligned_end_offset >= range.right);
+    chassert(aligned_offset <= initial_range.left);
+    chassert(aligned_end_offset >= initial_range.right);
 
     auto locked_key = metadata.lockKeyMetadata(key, CacheMetadata::KeyNotFoundPolicy::CREATE_EMPTY, user);
     /// Get all segments which intersect with the given range.
-    auto file_segments = getImpl(*locked_key, range, file_segments_limit);
+    auto file_segments = getImpl(*locked_key, initial_range, file_segments_limit);
 
     if (file_segments_limit)
     {
         chassert(file_segments.size() <= file_segments_limit);
         if (file_segments.size() == file_segments_limit)
-            range.right = aligned_end_offset = file_segments.back()->range().right;
+            result_range.right = aligned_end_offset = file_segments.back()->range().right;
     }
 
     /// Check case if we have uncovered prefix, e.g.
@@ -597,11 +587,11 @@ FileCache::getOrSet(
     ///   [    ]
     ///   ^----^
     ///   uncovered prefix.
-    const bool has_uncovered_prefix = file_segments.empty() || range.left < file_segments.front()->range().left;
+    const bool has_uncovered_prefix = file_segments.empty() || result_range.left < file_segments.front()->range().left;
 
-    if (aligned_offset < range.left && has_uncovered_prefix)
+    if (aligned_offset < result_range.left && has_uncovered_prefix)
     {
-        auto prefix_range = FileSegment::Range(aligned_offset, file_segments.empty() ? range.left - 1 : file_segments.front()->range().left - 1);
+        auto prefix_range = FileSegment::Range(aligned_offset, file_segments.empty() ? result_range.left - 1 : file_segments.front()->range().left - 1);
         auto prefix_file_segments = getImpl(*locked_key, prefix_range, /* file_segments_limit */0);
 
         if (prefix_file_segments.empty())
@@ -610,7 +600,7 @@ FileCache::getOrSet(
             ///   ^                     ^               ^
             ///   aligned_offset        range.left      range.right
             ///                             [___] [__________]         <-- current cache (example)
-            range.left = aligned_offset;
+            result_range.left = aligned_offset;
         }
         else
         {
@@ -621,10 +611,10 @@ FileCache::getOrSet(
             ///                  ^
             ///                  prefix_file_segments.back().right
 
-            chassert(prefix_file_segments.back()->range().right < range.left);
+            chassert(prefix_file_segments.back()->range().right < result_range.left);
             chassert(prefix_file_segments.back()->range().right >= aligned_offset);
 
-            range.left = prefix_file_segments.back()->range().right + 1;
+            result_range.left = prefix_file_segments.back()->range().right + 1;
         }
     }
 
@@ -637,11 +627,11 @@ FileCache::getOrSet(
     ///                   [___]
     ///                   ^---^
     ///                    uncovered_suffix
-    const bool has_uncovered_suffix = file_segments.empty() || file_segments.back()->range().right < range.right;
+    const bool has_uncovered_suffix = file_segments.empty() || file_segments.back()->range().right < result_range.right;
 
-    if (range.right < aligned_end_offset && has_uncovered_suffix)
+    if (result_range.right < aligned_end_offset && has_uncovered_suffix)
     {
-        auto suffix_range = FileSegment::Range(range.right, aligned_end_offset);
+        auto suffix_range = FileSegment::Range(result_range.right, aligned_end_offset);
         /// We need to get 1 file segment, so file_segments_limit = 1 here.
         auto suffix_file_segments = getImpl(*locked_key, suffix_range, /* file_segments_limit */1);
 
@@ -652,7 +642,7 @@ FileCache::getOrSet(
             ///   range.left         range.right              aligned_end_offset
             ///      [___]   [___]                                    <-- current cache (example)
 
-            range.right = aligned_end_offset;
+            result_range.right = aligned_end_offset;
         }
         else
         {
@@ -662,35 +652,33 @@ FileCache::getOrSet(
             ///      [___]   [___]          [_________]               <-- current cache (example)
             ///                             ^
             ///                             suffix_file_segments.front().left
-            range.right = suffix_file_segments.front()->range().left - 1;
+            result_range.right = suffix_file_segments.front()->range().left - 1;
         }
     }
 
-    chassert(range.left >= aligned_offset);
-
     if (file_segments.empty())
     {
-        file_segments = splitRangeIntoFileSegments(
-            *locked_key, range.left, /* size */offset + size - range.left, /* aligned_size */range.size(),
-            FileSegment::State::EMPTY, file_segments_limit, create_settings);
+        auto ranges = splitRange(result_range.left, initial_range.size() + (initial_range.left - result_range.left), result_range.size());
+        size_t file_segments_count = file_segments.size();
+        file_segments.splice(file_segments.end(), createFileSegmentsFromRanges(*locked_key, ranges, file_segments_count, file_segments_limit, create_settings));
     }
     else
     {
-        chassert(file_segments.front()->range().right >= range.left);
-        chassert(file_segments.back()->range().left <= range.right);
+        chassert(file_segments.front()->range().right >= result_range.left);
+        chassert(file_segments.back()->range().left <= result_range.right);
 
         fillHolesWithEmptyFileSegments(
-            *locked_key, file_segments, range, offset + size - 1, file_segments_limit, /* fill_with_detached */false, create_settings);
+            *locked_key, file_segments, result_range, offset + size - 1, file_segments_limit, /* fill_with_detached */false, create_settings);
 
-        if (!file_segments.front()->range().contains(range.left))
+        if (!file_segments.front()->range().contains(result_range.left))
         {
             throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected {} to include {} "
                             "(end offset: {}, aligned offset: {}, aligned end offset: {})",
-                            file_segments.front()->range().toString(), offset, range.right, aligned_offset, aligned_end_offset);
+                            file_segments.front()->range().toString(), offset, result_range.right, aligned_offset, aligned_end_offset);
         }
     }
 
-    chassert(file_segments_limit ? file_segments.back()->range().left <= range.right : file_segments.back()->range().contains(range.right));
+    chassert(file_segments_limit ? file_segments.back()->range().left <= result_range.right : file_segments.back()->range().contains(result_range.right));
     chassert(!file_segments_limit || file_segments.size() <= file_segments_limit);
 
     return std::make_unique<FileSegmentsHolder>(std::move(file_segments));
diff --git a/src/Interpreters/Cache/FileCache.h b/src/Interpreters/Cache/FileCache.h
index 3f7eec73b56..07be802a940 100644
--- a/src/Interpreters/Cache/FileCache.h
+++ b/src/Interpreters/Cache/FileCache.h
@@ -265,16 +265,10 @@ private:
     /// each subrange size must be less or equal to max_file_segment_size.
     std::vector<FileSegment::Range> splitRange(size_t offset, size_t size, size_t aligned_size);
 
-    /// Split range into subranges by max_file_segment_size (same as in splitRange())
-    /// and create a new file segment for each subrange.
-    /// If `file_segments_limit` > 0, create no more than first file_segments_limit
-    /// file segments.
-    FileSegments splitRangeIntoFileSegments(
+    FileSegments createFileSegmentsFromRanges(
         LockedKey & locked_key,
-        size_t offset,
-        size_t size,
-        size_t aligned_size,
-        FileSegment::State state,
+        const std::vector<FileSegment::Range> & ranges,
+        size_t & file_segments_count,
         size_t file_segments_limit,
         const CreateFileSegmentSettings & create_settings);
 
diff --git a/tests/config/config.d/storage_conf.xml b/tests/config/config.d/storage_conf.xml
index 4daa64b520d..e106e3a0e6b 100644
--- a/tests/config/config.d/storage_conf.xml
+++ b/tests/config/config.d/storage_conf.xml
@@ -19,7 +19,7 @@
                 <type>cache</type>
                 <disk>s3_disk</disk>
                 <path>s3_cache/</path>
-                <max_size>100Mi</max_size>
+                <max_size>104857600</max_size>
                 <max_file_segment_size>5Mi</max_file_segment_size>
                 <cache_on_write_operations>1</cache_on_write_operations>
                 <delayed_cleanup_interval_ms>100</delayed_cleanup_interval_ms>

From a94df1fb9af032c8ccccb704186513561e28e11c Mon Sep 17 00:00:00 2001
From: Sema Checherinda <Sema.Checherinda@clickhouse.com>
Date: Wed, 31 Jul 2024 23:50:48 +0200
Subject: [PATCH 1264/1488] enable parallel_view_processing in perf tests

---
 tests/performance/materialized_view_parallel_insert.xml | 4 ++++
 tests/performance/views_max_insert_threads.xml          | 4 ++++
 2 files changed, 8 insertions(+)

diff --git a/tests/performance/materialized_view_parallel_insert.xml b/tests/performance/materialized_view_parallel_insert.xml
index df0a23182c3..c3e7cdc50c0 100644
--- a/tests/performance/materialized_view_parallel_insert.xml
+++ b/tests/performance/materialized_view_parallel_insert.xml
@@ -1,4 +1,8 @@
 <test>
+    <settings>
+        <parallel_view_processing>1</parallel_view_processing>
+    </settings>
+
     <create_query>
         CREATE MATERIALIZED VIEW hits_mv ENGINE MergeTree
         PARTITION BY toYYYYMM(EventDate)
diff --git a/tests/performance/views_max_insert_threads.xml b/tests/performance/views_max_insert_threads.xml
index 2988984f5d8..c16fb330b35 100644
--- a/tests/performance/views_max_insert_threads.xml
+++ b/tests/performance/views_max_insert_threads.xml
@@ -1,5 +1,9 @@
 <!-- https://github.com/ClickHouse/ClickHouse/issues/37900 -->
 <test>
+    <settings>
+        <parallel_view_processing>1</parallel_view_processing>
+    </settings>
+
     <create_query>create table views_max_insert_threads_null (a UInt64) Engine = Null</create_query>
     <create_query>create materialized view views_max_insert_threads_mv Engine = Null AS select now() as ts, max(a) from views_max_insert_threads_null group by ts</create_query>
 

From c3a1381d70e6d5da59e4b29e7a147e24e7fcd90b Mon Sep 17 00:00:00 2001
From: Sema Checherinda <Sema.Checherinda@clickhouse.com>
Date: Thu, 1 Aug 2024 00:10:59 +0200
Subject: [PATCH 1265/1488] some corner case optimize

---
 src/Interpreters/Squashing.cpp | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/src/Interpreters/Squashing.cpp b/src/Interpreters/Squashing.cpp
index 488177c3b4f..483112df6cb 100644
--- a/src/Interpreters/Squashing.cpp
+++ b/src/Interpreters/Squashing.cpp
@@ -106,6 +106,17 @@ Chunk Squashing::convertToChunk(CurrentData && data) const
 
 Chunk Squashing::squash(std::vector<Chunk> && input_chunks, Chunk::ChunkInfoCollection && infos)
 {
+    if (input_chunks.size() == 1)
+    {
+        /// this is just optimization, no logic changes
+        Chunk result = std::move(input_chunks.front());
+        infos.appendIfUniq(std::move(result.getChunkInfos()));
+        result.setChunkInfos(infos);
+
+        chassert(result);
+        return result;
+    }
+
     std::vector<IColumn::MutablePtr> mutable_columns = {};
     size_t rows = 0;
     for (const Chunk & chunk : input_chunks)

From 8d979680060f10e6bcec3fc83fb3bdbaa7bb3deb Mon Sep 17 00:00:00 2001
From: Lennard Eijsackers <lennardeijsackers92@gmail.com>
Date: Fri, 2 Aug 2024 15:13:37 +0200
Subject: [PATCH 1266/1488] Use FunctionArgumentDescriptors to check bitSlice
 function + add test case

Signed-off-by: Lennard Eijsackers <lennardeijsackers92@gmail.com>
---
 src/Functions/bitSlice.cpp                    | 30 ++++++++-----------
 ...214_bitslice_argument_evaluation.reference |  0
 .../03214_bitslice_argument_evaluation.sql    | 10 +++++++
 3 files changed, 22 insertions(+), 18 deletions(-)
 create mode 100644 tests/queries/0_stateless/03214_bitslice_argument_evaluation.reference
 create mode 100644 tests/queries/0_stateless/03214_bitslice_argument_evaluation.sql

diff --git a/src/Functions/bitSlice.cpp b/src/Functions/bitSlice.cpp
index e2b455846d8..f1d3bb57221 100644
--- a/src/Functions/bitSlice.cpp
+++ b/src/Functions/bitSlice.cpp
@@ -40,28 +40,22 @@ public:
 
     bool useDefaultImplementationForConstants() const override { return true; }
 
-    DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
+    DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
     {
-        const size_t number_of_arguments = arguments.size();
+        FunctionArgumentDescriptors mandatory_args{
+            {"s", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), nullptr, "String"},
+            {"offset", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNativeNumber), nullptr, "(U)Int8, (U)Int16, (U)Int32, (U)Int64 or Float"},
+        };
 
-        if (number_of_arguments < 2 || number_of_arguments > 3)
-            throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
-                "Number of arguments for function {} doesn't match: passed {}, should be 2 or 3",
-                getName(), number_of_arguments);
+        FunctionArgumentDescriptors optional_args{
+            {"length", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNativeNumber), nullptr, "(U)Int8, (U)Int16, (U)Int32, (U)Int64 or Float"},
+        };
 
-        if (!isString(arguments[0]) && !isStringOrFixedString(arguments[0]))
-            throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of argument of function {}",
-                arguments[0]->getName(), getName());
-        if (arguments[0]->onlyNull())
-            return arguments[0];
+        validateFunctionArguments(*this, arguments, mandatory_args, optional_args);
 
-        if (!isNativeNumber(arguments[1]))
-            throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of second argument of function {}",
-                arguments[1]->getName(), getName());
-
-        if (number_of_arguments == 3 && !isNativeNumber(arguments[2]))
-            throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of second argument of function {}",
-                arguments[2]->getName(), getName());
+        const auto & type = arguments[0].type;
+        if (type->onlyNull())
+            return type;
 
         return std::make_shared<DataTypeString>();
     }
diff --git a/tests/queries/0_stateless/03214_bitslice_argument_evaluation.reference b/tests/queries/0_stateless/03214_bitslice_argument_evaluation.reference
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/queries/0_stateless/03214_bitslice_argument_evaluation.sql b/tests/queries/0_stateless/03214_bitslice_argument_evaluation.sql
new file mode 100644
index 00000000000..b8488600fcb
--- /dev/null
+++ b/tests/queries/0_stateless/03214_bitslice_argument_evaluation.sql
@@ -0,0 +1,10 @@
+-- No arguments passed
+SELECT bitSlice(); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH }
+-- Invalid 1st argument passed
+SELECT bitSlice(1, 1); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT }
+-- Valid 1st argument, invalid 2nd argument passed
+SELECT bitSlice('Hello', 'World'); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT }
+-- Valid 1st argument & 2nd argument, invalid 3rd argument passed
+SELECT bitSlice('Hello', 1, 'World'); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT }
+-- More arguments then expected
+SELECT bitSlice('Hello', 1, 1, 'World'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH }

From 83096249a7480b0bfa1d9246c17136727bba904c Mon Sep 17 00:00:00 2001
From: robot-clickhouse <robot-clickhouse@users.noreply.github.com>
Date: Fri, 2 Aug 2024 13:54:32 +0000
Subject: [PATCH 1267/1488] Update version_date.tsv and changelogs after
 v24.3.6.48-lts

---
 docs/changelogs/v24.3.6.48-lts.md    | 39 ++++++++++++++++++++++++++++
 utils/list-versions/version_date.tsv |  1 +
 2 files changed, 40 insertions(+)
 create mode 100644 docs/changelogs/v24.3.6.48-lts.md

diff --git a/docs/changelogs/v24.3.6.48-lts.md b/docs/changelogs/v24.3.6.48-lts.md
new file mode 100644
index 00000000000..f045afc619b
--- /dev/null
+++ b/docs/changelogs/v24.3.6.48-lts.md
@@ -0,0 +1,39 @@
+---
+sidebar_position: 1
+sidebar_label: 2024
+---
+
+# 2024 Changelog
+
+### ClickHouse release v24.3.6.48-lts (b2d33c3c45d) FIXME as compared to v24.3.5.46-lts (fe54cead6b6)
+
+#### Critical Bug Fix (crash, LOGICAL_ERROR, data loss, RBAC)
+* Backported in [#66889](https://github.com/ClickHouse/ClickHouse/issues/66889): Fix unexpeced size of low cardinality column in function calls. [#65298](https://github.com/ClickHouse/ClickHouse/pull/65298) ([Raúl Marín](https://github.com/Algunenano)).
+* Backported in [#66687](https://github.com/ClickHouse/ClickHouse/issues/66687): Fix the VALID UNTIL clause in the user definition resetting after a restart. Closes [#66405](https://github.com/ClickHouse/ClickHouse/issues/66405). [#66409](https://github.com/ClickHouse/ClickHouse/pull/66409) ([Nikolay Degterinsky](https://github.com/evillique)).
+* Backported in [#67497](https://github.com/ClickHouse/ClickHouse/issues/67497): Fix crash in DistributedAsyncInsert when connection is empty. [#67219](https://github.com/ClickHouse/ClickHouse/pull/67219) ([Pablo Marcos](https://github.com/pamarcos)).
+
+#### Bug Fix (user-visible misbehavior in an official stable release)
+* Backported in [#66324](https://github.com/ClickHouse/ClickHouse/issues/66324): Add missing settings `input_format_csv_skip_first_lines/input_format_tsv_skip_first_lines/input_format_csv_try_infer_numbers_from_strings/input_format_csv_try_infer_strings_from_quoted_tuples` in schema inference cache because they can change the resulting schema. It prevents from incorrect result of schema inference with these settings changed. [#65980](https://github.com/ClickHouse/ClickHouse/pull/65980) ([Kruglov Pavel](https://github.com/Avogar)).
+* Backported in [#66151](https://github.com/ClickHouse/ClickHouse/issues/66151): Fixed buffer overflow bug in `unbin`/`unhex` implementation. [#66106](https://github.com/ClickHouse/ClickHouse/pull/66106) ([Nikita Taranov](https://github.com/nickitat)).
+* Backported in [#66451](https://github.com/ClickHouse/ClickHouse/issues/66451): Fixed a bug in ZooKeeper client: a session could get stuck in unusable state after receiving a hardware error from ZooKeeper. For example, this might happen due to "soft memory limit" in ClickHouse Keeper. [#66140](https://github.com/ClickHouse/ClickHouse/pull/66140) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Backported in [#66222](https://github.com/ClickHouse/ClickHouse/issues/66222): Fix issue in SumIfToCountIfVisitor and signed integers. [#66146](https://github.com/ClickHouse/ClickHouse/pull/66146) ([Raúl Marín](https://github.com/Algunenano)).
+* Backported in [#66676](https://github.com/ClickHouse/ClickHouse/issues/66676): Fix handling limit for `system.numbers_mt` when no index can be used. [#66231](https://github.com/ClickHouse/ClickHouse/pull/66231) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
+* Backported in [#66602](https://github.com/ClickHouse/ClickHouse/issues/66602): Fixed how the ClickHouse server detects the maximum number of usable CPU cores as specified by cgroups v2 if the server runs in a container such as Docker. In more detail, containers often run their process in the root cgroup which has an empty name. In that case, ClickHouse ignored the CPU limits set by cgroups v2. [#66237](https://github.com/ClickHouse/ClickHouse/pull/66237) ([filimonov](https://github.com/filimonov)).
+* Backported in [#66356](https://github.com/ClickHouse/ClickHouse/issues/66356): Fix the `Not-ready set` error when a subquery with `IN` is used in the constraint. [#66261](https://github.com/ClickHouse/ClickHouse/pull/66261) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#66970](https://github.com/ClickHouse/ClickHouse/issues/66970): Fix `Column identifier is already registered` error with `group_by_use_nulls=true` and new analyzer. [#66400](https://github.com/ClickHouse/ClickHouse/pull/66400) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#66967](https://github.com/ClickHouse/ClickHouse/issues/66967): Fix `Cannot find column` error for queries with constant expression in `GROUP BY` key and new analyzer enabled. [#66433](https://github.com/ClickHouse/ClickHouse/pull/66433) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#66718](https://github.com/ClickHouse/ClickHouse/issues/66718): Correctly track memory for `Allocator::realloc`. [#66548](https://github.com/ClickHouse/ClickHouse/pull/66548) ([Antonio Andelic](https://github.com/antonio2368)).
+* Backported in [#66949](https://github.com/ClickHouse/ClickHouse/issues/66949): Fix an invalid result for queries with `WINDOW`. This could happen when `PARTITION` columns have sparse serialization and window functions are executed in parallel. [#66579](https://github.com/ClickHouse/ClickHouse/pull/66579) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#66946](https://github.com/ClickHouse/ClickHouse/issues/66946): Fix `Method getResultType is not supported for QUERY query node` error when scalar subquery was used as the first argument of IN (with new analyzer). [#66655](https://github.com/ClickHouse/ClickHouse/pull/66655) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#67629](https://github.com/ClickHouse/ClickHouse/issues/67629): Fix for occasional deadlock in Context::getDDLWorker. [#66843](https://github.com/ClickHouse/ClickHouse/pull/66843) ([Alexander Gololobov](https://github.com/davenger)).
+* Backported in [#67193](https://github.com/ClickHouse/ClickHouse/issues/67193): TRUNCATE DATABASE used to stop replication as if it was a DROP DATABASE query, it's fixed. [#67129](https://github.com/ClickHouse/ClickHouse/pull/67129) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Backported in [#67375](https://github.com/ClickHouse/ClickHouse/issues/67375): Fix error `Cannot convert column because it is non constant in source stream but must be constant in result.` for a query that reads from the `Merge` table over the `Distriburted` table with one shard. [#67146](https://github.com/ClickHouse/ClickHouse/pull/67146) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#67572](https://github.com/ClickHouse/ClickHouse/issues/67572): Fix execution of nested short-circuit functions. [#67520](https://github.com/ClickHouse/ClickHouse/pull/67520) ([Kruglov Pavel](https://github.com/Avogar)).
+
+#### NOT FOR CHANGELOG / INSIGNIFICANT
+
+* Backported in [#66422](https://github.com/ClickHouse/ClickHouse/issues/66422): Ignore subquery for IN in DDLLoadingDependencyVisitor. [#66395](https://github.com/ClickHouse/ClickHouse/pull/66395) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#66855](https://github.com/ClickHouse/ClickHouse/issues/66855): Fix data race in S3::ClientCache. [#66644](https://github.com/ClickHouse/ClickHouse/pull/66644) ([Konstantin Morozov](https://github.com/k-morozov)).
+* Backported in [#67055](https://github.com/ClickHouse/ClickHouse/issues/67055): Increase asio pool size in case the server is tiny. [#66761](https://github.com/ClickHouse/ClickHouse/pull/66761) ([alesapin](https://github.com/alesapin)).
+* Backported in [#66943](https://github.com/ClickHouse/ClickHouse/issues/66943): Small fix in realloc memory tracking. [#66820](https://github.com/ClickHouse/ClickHouse/pull/66820) ([Antonio Andelic](https://github.com/antonio2368)).
+
diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv
index 7b5dcda82e3..24488066190 100644
--- a/utils/list-versions/version_date.tsv
+++ b/utils/list-versions/version_date.tsv
@@ -10,6 +10,7 @@ v24.4.4.113-stable	2024-08-02
 v24.4.3.25-stable	2024-06-14
 v24.4.2.141-stable	2024-06-07
 v24.4.1.2088-stable	2024-05-01
+v24.3.6.48-lts	2024-08-02
 v24.3.5.46-lts	2024-07-03
 v24.3.4.147-lts	2024-06-13
 v24.3.3.102-lts	2024-05-01

From 9c7464e0653782af385dbc884dd3acecfc69c6cc Mon Sep 17 00:00:00 2001
From: Nikita Fomichev <nikita.fomichev@clickhouse.com>
Date: Fri, 2 Aug 2024 16:04:11 +0200
Subject: [PATCH 1268/1488] Stateless tests: reduce pure_http_client timeout to
 get reasons of timed out tests

---
 tests/queries/0_stateless/helpers/pure_http_client.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/queries/0_stateless/helpers/pure_http_client.py b/tests/queries/0_stateless/helpers/pure_http_client.py
index 0e7a4d27f4f..7a8efec36bb 100644
--- a/tests/queries/0_stateless/helpers/pure_http_client.py
+++ b/tests/queries/0_stateless/helpers/pure_http_client.py
@@ -18,7 +18,7 @@ class ClickHouseClient:
         self.host = host
 
     def query(
-        self, query, connection_timeout=1500, settings=dict(), binary_result=False
+        self, query, connection_timeout=500, settings=dict(), binary_result=False
     ):
         NUMBER_OF_TRIES = 30
         DELAY = 10
@@ -47,12 +47,12 @@ class ClickHouseClient:
                 else:
                     raise ValueError(r.text)
 
-    def query_return_df(self, query, connection_timeout=1500):
+    def query_return_df(self, query, connection_timeout=500):
         data = self.query(query, connection_timeout)
         df = pd.read_csv(io.StringIO(data), sep="\t")
         return df
 
-    def query_with_data(self, query, data, connection_timeout=1500, settings=dict()):
+    def query_with_data(self, query, data, connection_timeout=500, settings=dict()):
         params = {
             "query": query,
             "timeout_before_checking_execution_speed": 120,

From 2c9cef38e56c65ec9bbe7f3af21d4865662f6e9a Mon Sep 17 00:00:00 2001
From: Nikita Fomichev <nikita.fomichev@clickhouse.com>
Date: Fri, 2 Aug 2024 16:05:39 +0200
Subject: [PATCH 1269/1488] Stateless tests: fix hanging tests
 `02473_multistep_prewhere*` `00411_long_accurate_number_comparison*`

---
 ...411_long_accurate_number_comparison.python | 19 ++++++-------------
 .../02473_multistep_prewhere.python           |  4 ++--
 .../02473_multistep_split_prewhere.python     |  4 ++--
 .../0_stateless/helpers/pure_http_client.py   | 17 ++++++++++++++++-
 4 files changed, 26 insertions(+), 18 deletions(-)

diff --git a/tests/queries/0_stateless/00411_long_accurate_number_comparison.python b/tests/queries/0_stateless/00411_long_accurate_number_comparison.python
index 045de9ee7ee..38b108a696f 100644
--- a/tests/queries/0_stateless/00411_long_accurate_number_comparison.python
+++ b/tests/queries/0_stateless/00411_long_accurate_number_comparison.python
@@ -2,23 +2,16 @@
 
 import os, itertools, urllib.request, urllib.parse, urllib.error, urllib.request, urllib.error, urllib.parse, sys
 
+CURDIR = os.path.dirname(os.path.realpath(__file__))
+sys.path.insert(0, os.path.join(CURDIR, "helpers"))
 
-def get_ch_answer(query):
-    return (
-        urllib.request.urlopen(
-            os.environ.get(
-                "CLICKHOUSE_URL",
-                "http://localhost:" + os.environ.get("CLICKHOUSE_PORT_HTTP", "8123"),
-            ),
-            data=query.encode(),
-        )
-        .read()
-        .decode()
-    )
+from pure_http_client import ClickHouseClient
+
+client = ClickHouseClient()
 
 
 def check_answers(query, answer):
-    ch_answer = get_ch_answer(query)
+    ch_answer = client.query(query)
     if ch_answer.strip() != answer.strip():
         print("FAIL on query:", query)
         print("Expected answer:", answer)
diff --git a/tests/queries/0_stateless/02473_multistep_prewhere.python b/tests/queries/0_stateless/02473_multistep_prewhere.python
index 11095202039..09326b6365d 100644
--- a/tests/queries/0_stateless/02473_multistep_prewhere.python
+++ b/tests/queries/0_stateless/02473_multistep_prewhere.python
@@ -6,7 +6,7 @@ import sys
 CURDIR = os.path.dirname(os.path.realpath(__file__))
 sys.path.insert(0, os.path.join(CURDIR, "helpers"))
 
-from pure_http_client import ClickHouseClient
+from pure_http_client import ClickHouseClient, requests_session_with_retries
 
 
 class Tester:
@@ -195,7 +195,7 @@ def main():
     default_index_granularity = 10
     total_rows = 7 * default_index_granularity
     step = default_index_granularity
-    session = requests.Session()
+    session = requests_session_with_retries()
     for index_granularity in [
         default_index_granularity - 1,
         default_index_granularity,
diff --git a/tests/queries/0_stateless/02473_multistep_split_prewhere.python b/tests/queries/0_stateless/02473_multistep_split_prewhere.python
index 19444994fd2..10e94059171 100644
--- a/tests/queries/0_stateless/02473_multistep_split_prewhere.python
+++ b/tests/queries/0_stateless/02473_multistep_split_prewhere.python
@@ -6,7 +6,7 @@ import sys
 CURDIR = os.path.dirname(os.path.realpath(__file__))
 sys.path.insert(0, os.path.join(CURDIR, "helpers"))
 
-from pure_http_client import ClickHouseClient
+from pure_http_client import ClickHouseClient, requests_session_with_retries
 
 
 class Tester:
@@ -161,7 +161,7 @@ def main():
     default_index_granularity = 10
     total_rows = 8 * default_index_granularity
     step = default_index_granularity
-    session = requests.Session()
+    session = requests_session_with_retries()
     for index_granularity in [default_index_granularity - 1, default_index_granularity]:
         tester = Tester(session, url, index_granularity, total_rows)
         # Test combinations of ranges of columns c and d
diff --git a/tests/queries/0_stateless/helpers/pure_http_client.py b/tests/queries/0_stateless/helpers/pure_http_client.py
index 7a8efec36bb..a31a91e0550 100644
--- a/tests/queries/0_stateless/helpers/pure_http_client.py
+++ b/tests/queries/0_stateless/helpers/pure_http_client.py
@@ -1,7 +1,8 @@
 import os
 import io
-import sys
 import requests
+from requests.adapters import HTTPAdapter
+from requests.packages.urllib3.util.retry import Retry
 import time
 import pandas as pd
 
@@ -77,3 +78,17 @@ class ClickHouseClient:
             return result
         else:
             raise ValueError(r.text)
+
+
+def requests_session_with_retries(retries=3, timeout=180):
+    session = requests.Session()
+    retry = Retry(
+        total=retries,
+        read=retries,
+        connect=retries,
+    )
+    adapter = HTTPAdapter(max_retries=retry)
+    session.mount("http://", adapter)
+    session.mount("https://", adapter)
+    session.timeout = timeout
+    return session

From aefed7cdd62e874f7507afe69d803c9164a283ea Mon Sep 17 00:00:00 2001
From: alesapin <alesapin@clickhouse.com>
Date: Fri, 2 Aug 2024 16:06:53 +0200
Subject: [PATCH 1270/1488] Update
 tests/queries/0_stateless/03164_s3_settings_for_queries_and_merges.sql

---
 .../0_stateless/03164_s3_settings_for_queries_and_merges.sql     | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/queries/0_stateless/03164_s3_settings_for_queries_and_merges.sql b/tests/queries/0_stateless/03164_s3_settings_for_queries_and_merges.sql
index 001ef382850..a6932e0536c 100644
--- a/tests/queries/0_stateless/03164_s3_settings_for_queries_and_merges.sql
+++ b/tests/queries/0_stateless/03164_s3_settings_for_queries_and_merges.sql
@@ -21,6 +21,7 @@ SYSTEM DROP MARK CACHE;
 SELECT count() FROM t_compact_bytes_s3 WHERE NOT ignore(c2, c4);
 SYSTEM FLUSH LOGS;
 
+-- Errors in S3 requests will be automatically retried, however ProfileEvents can be wrong. That is why we subtract errors.
 SELECT
     ProfileEvents['S3ReadRequestsCount'] - ProfileEvents['S3ReadRequestsErrors'],
     ProfileEvents['ReadBufferFromS3Bytes'] < ProfileEvents['ReadCompressedBytes'] * 1.1

From 86bab5a78a9593862815869c5964f557159352ef Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Fri, 2 Aug 2024 16:09:43 +0200
Subject: [PATCH 1271/1488] Unit test: Don't return ok if leaks are detected

---
 docker/test/unit/run.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docker/test/unit/run.sh b/docker/test/unit/run.sh
index ba11f568218..210f31170a5 100644
--- a/docker/test/unit/run.sh
+++ b/docker/test/unit/run.sh
@@ -1,6 +1,9 @@
 #!/bin/bash
 
 set -x
+# Need to keep error from tests after `tee`. Otherwise we don't alert on asan errors
+set -o pipefail
+set -e
 
 if [ "$#" -ne 1 ]; then
     echo "Expected exactly one argument"

From 664e131f4f2e46fc216305c440e840a5a5784328 Mon Sep 17 00:00:00 2001
From: Nikita Fomichev <nikita.fomichev@clickhouse.com>
Date: Fri, 2 Aug 2024 16:09:48 +0200
Subject: [PATCH 1272/1488] Integration tests: fix ports clashing problem 4

---
 tests/integration/conftest.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py
index b4c86a1cd2f..aa235118aed 100644
--- a/tests/integration/conftest.py
+++ b/tests/integration/conftest.py
@@ -1,4 +1,6 @@
 #!/usr/bin/env python3
+# pylint: disable=unused-argument
+# pylint: disable=broad-exception-raised
 
 import logging
 import os

From b3e2ce695514d4d314ed8ac1ecdb111c5f94ac7d Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 31 Jul 2024 17:38:20 +0200
Subject: [PATCH 1273/1488] Minor change

---
 src/Databases/DatabaseOnDisk.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp
index f419f5811a1..734f354d9a5 100644
--- a/src/Databases/DatabaseOnDisk.cpp
+++ b/src/Databases/DatabaseOnDisk.cpp
@@ -313,7 +313,7 @@ void DatabaseOnDisk::detachTablePermanently(ContextPtr query_context, const Stri
         std::lock_guard lock(mutex);
         if (const auto it = snapshot_detached_tables.find(table_name); it == snapshot_detached_tables.end())
         {
-            throw Exception(ErrorCodes::LOGICAL_ERROR, "Snapshot doesn't contain info about detached table={}", table_name);
+            throw Exception(ErrorCodes::LOGICAL_ERROR, "Snapshot doesn't contain info about detached table `{}`", table_name);
         }
         else
         {

From 2110b765d19ce4e68d0c23eab219e199aec0ea96 Mon Sep 17 00:00:00 2001
From: Lennard Eijsackers <lennardeijsackers92@gmail.com>
Date: Fri, 2 Aug 2024 17:53:12 +0200
Subject: [PATCH 1274/1488] Style check fix + adding debug info to query output

Signed-off-by: Lennard Eijsackers <lennardeijsackers92@gmail.com>
---
 src/Functions/bitSlice.cpp                            |  2 --
 .../03214_bitslice_argument_evaluation.reference      | 11 +++++++++++
 .../03214_bitslice_argument_evaluation.sql            |  1 +
 3 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/src/Functions/bitSlice.cpp b/src/Functions/bitSlice.cpp
index f1d3bb57221..f24473351ae 100644
--- a/src/Functions/bitSlice.cpp
+++ b/src/Functions/bitSlice.cpp
@@ -18,9 +18,7 @@ using namespace GatherUtils;
 namespace ErrorCodes
 {
     extern const int ILLEGAL_COLUMN;
-    extern const int ILLEGAL_TYPE_OF_ARGUMENT;
     extern const int ZERO_ARRAY_OR_TUPLE_INDEX;
-    extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
 }
 
 class FunctionBitSlice : public IFunction
diff --git a/tests/queries/0_stateless/03214_bitslice_argument_evaluation.reference b/tests/queries/0_stateless/03214_bitslice_argument_evaluation.reference
index e69de29bb2d..1731dfa0d79 100644
--- a/tests/queries/0_stateless/03214_bitslice_argument_evaluation.reference
+++ b/tests/queries/0_stateless/03214_bitslice_argument_evaluation.reference
@@ -0,0 +1,11 @@
+-- { echo }
+-- No arguments passed
+SELECT bitSlice(); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH }
+-- Invalid 1st argument passed
+SELECT bitSlice(1, 1); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT }
+-- Valid 1st argument, invalid 2nd argument passed
+SELECT bitSlice('Hello', 'World'); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT }
+-- Valid 1st argument & 2nd argument, invalid 3rd argument passed
+SELECT bitSlice('Hello', 1, 'World'); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT }
+-- More arguments then expected
+SELECT bitSlice('Hello', 1, 1, 'World'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH }
diff --git a/tests/queries/0_stateless/03214_bitslice_argument_evaluation.sql b/tests/queries/0_stateless/03214_bitslice_argument_evaluation.sql
index b8488600fcb..1731dfa0d79 100644
--- a/tests/queries/0_stateless/03214_bitslice_argument_evaluation.sql
+++ b/tests/queries/0_stateless/03214_bitslice_argument_evaluation.sql
@@ -1,3 +1,4 @@
+-- { echo }
 -- No arguments passed
 SELECT bitSlice(); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH }
 -- Invalid 1st argument passed

From 364622f567028ffc70785b681fc246d7151eef04 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 31 Jul 2024 17:38:20 +0200
Subject: [PATCH 1275/1488] Minor change

---
 src/Databases/DatabaseOnDisk.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp
index f419f5811a1..734f354d9a5 100644
--- a/src/Databases/DatabaseOnDisk.cpp
+++ b/src/Databases/DatabaseOnDisk.cpp
@@ -313,7 +313,7 @@ void DatabaseOnDisk::detachTablePermanently(ContextPtr query_context, const Stri
         std::lock_guard lock(mutex);
         if (const auto it = snapshot_detached_tables.find(table_name); it == snapshot_detached_tables.end())
         {
-            throw Exception(ErrorCodes::LOGICAL_ERROR, "Snapshot doesn't contain info about detached table={}", table_name);
+            throw Exception(ErrorCodes::LOGICAL_ERROR, "Snapshot doesn't contain info about detached table `{}`", table_name);
         }
         else
         {

From fcb0ce7361f74dd8d97a3007f77248f293b2ce5f Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 2 Aug 2024 18:17:32 +0200
Subject: [PATCH 1276/1488] Fix docs build

---
 docker/docs/builder/run.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/docker/docs/builder/run.sh b/docker/docs/builder/run.sh
index 01c15cb4b0f..d73adb5d279 100755
--- a/docker/docs/builder/run.sh
+++ b/docker/docs/builder/run.sh
@@ -26,7 +26,6 @@ sed -i '/onBrokenMarkdownLinks:/ s/ignore/error/g' docusaurus.config.js
 
 if [[ $# -lt 1 ]] || [[ "$1" == "--"* ]]; then
   export CI=true
-  yarn install
   exec yarn build "$@"
 fi
 

From cebb3668380f65187b201e638013df40f8ac8ada Mon Sep 17 00:00:00 2001
From: Max K <maxkaynov@gmail.com>
Date: Fri, 2 Aug 2024 09:23:40 +0200
Subject: [PATCH 1277/1488] more fixes

---
 .github/actions/check_workflow/action.yml |  21 ++++
 .github/workflows/create_release.yml      |  27 +++--
 .github/workflows/pull_request.yml        |   9 +-
 tests/ci/artifactory.py                   |   6 +-
 tests/ci/auto_release.py                  |   6 +-
 tests/ci/ci.py                            |   4 +-
 tests/ci/ci_buddy.py                      |  30 +++--
 tests/ci/ci_cache.py                      |  14 +--
 tests/ci/ci_config.py                     |   3 +-
 tests/ci/ci_definitions.py                |   3 +-
 tests/ci/ci_metadata.py                   |   4 +-
 tests/ci/ci_utils.py                      |  30 ++++-
 tests/ci/create_release.py                | 131 ++++++++++++++--------
 tests/ci/docker_server.py                 |  73 +++---------
 tests/ci/test_docker.py                   |  46 +-------
 15 files changed, 213 insertions(+), 194 deletions(-)
 create mode 100644 .github/actions/check_workflow/action.yml

diff --git a/.github/actions/check_workflow/action.yml b/.github/actions/check_workflow/action.yml
new file mode 100644
index 00000000000..19a3cec76f5
--- /dev/null
+++ b/.github/actions/check_workflow/action.yml
@@ -0,0 +1,21 @@
+name: CheckWorkflowResults
+
+description: Check overall workflow status and post error to slack if any
+
+inputs:
+  needs:
+    description: github needs context as a json string
+    required: true
+    type: string
+
+runs:
+  using: "composite"
+  steps:
+    - name: Check Workflow
+      shell: bash
+      run: |
+        export WORKFLOW_RESULT_FILE="/tmp/workflow_results.json"
+        cat > "$WORKFLOW_RESULT_FILE" << 'EOF'
+        ${{ inputs.needs }}
+        EOF
+        python3 ./tests/ci/ci_buddy.py --check-wf-status
diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml
index e27db1b09a4..29094cc51a6 100644
--- a/.github/workflows/create_release.yml
+++ b/.github/workflows/create_release.yml
@@ -24,7 +24,7 @@ concurrency:
       dry-run:
         description: 'Dry run'
         required: false
-        default: true
+        default: false
         type: boolean
 
 jobs:
@@ -43,16 +43,27 @@ jobs:
       - name: Prepare Release Info
         shell: bash
         run: |
+          if [ ${{ inputs.only-repo }} == "true" ]; then
+              git tag -l ${{ inputs.ref }} || { echo "With only-repo option ref must be a valid release tag"; exit 1; }
+          fi
           python3 ./tests/ci/create_release.py --prepare-release-info \
-            --ref ${{ inputs.ref }} --release-type ${{ inputs.type }} ${{ inputs.dry-run == true && '--dry-run' || '' }}
+            --ref ${{ inputs.ref }} --release-type ${{ inputs.type }} \
+            ${{ inputs.dry-run == true && '--dry-run' || '' }} \
+            ${{ inputs.only-repo == true && '--skip-tag-check' || '' }}
           echo "::group::Release Info"
           python3 -m json.tool /tmp/release_info.json
           echo "::endgroup::"
           release_tag=$(jq -r '.release_tag' /tmp/release_info.json)
           commit_sha=$(jq -r '.commit_sha' /tmp/release_info.json)
+          is_latest=$(jq -r '.latest' /tmp/release_info.json)
           echo "Release Tag: $release_tag"
           echo "RELEASE_TAG=$release_tag" >> "$GITHUB_ENV"
           echo "COMMIT_SHA=$commit_sha" >> "$GITHUB_ENV"
+          if [ "$is_latest" == "true" ]; then
+            echo "DOCKER_TAG_TYPE=release-latest" >> "$GITHUB_ENV"
+          else
+            echo "DOCKER_TAG_TYPE=release" >> "$GITHUB_ENV"
+          fi
       - name: Download All Release Artifacts
         if: ${{ inputs.type == 'patch' }}
         shell: bash
@@ -85,10 +96,11 @@ jobs:
           echo "Generate ChangeLog"
           export CI=1
           docker run -u "${UID}:${GID}" -e PYTHONUNBUFFERED=1 -e CI=1 --network=host \
-            --volume=".:/ClickHouse" clickhouse/style-test \
-            /ClickHouse/tests/ci/changelog.py -v --debug-helpers \
+            --volume=".:/wd" --workdir="/wd" \
+            clickhouse/style-test \
+            ./tests/ci/changelog.py -v --debug-helpers \
             --jobs=5 \
-            --output="/ClickHouse/docs/changelogs/${{ env.RELEASE_TAG }}.md" ${{ env.RELEASE_TAG }}
+            --output="./docs/changelogs/${{ env.RELEASE_TAG }}.md" ${{ env.RELEASE_TAG }}
           git add ./docs/changelogs/${{ env.RELEASE_TAG }}.md
           echo "Generate Security"
           python3 ./utils/security-generator/generate_security.py > SECURITY.md
@@ -160,7 +172,7 @@ jobs:
           cd "./tests/ci"
           python3 ./create_release.py --set-progress-started --progress "docker server release"
           export CHECK_NAME="Docker server image"
-          python3 docker_server.py --release-type auto --version ${{ env.RELEASE_TAG }} --check-name "$CHECK_NAME" --sha ${{ env.COMMIT_SHA }} ${{ ! inputs.dry-run && '--push' || '' }}
+          python3 docker_server.py --tag-type ${{ env.DOCKER_TAG_TYPE }} --version ${{ env.RELEASE_TAG }} --check-name "$CHECK_NAME" --sha ${{ env.COMMIT_SHA }} ${{ ! inputs.dry-run && '--push' || '' }}
           python3 ./create_release.py --set-progress-completed
       - name: Docker clickhouse/clickhouse-keeper building
         if: ${{ inputs.type == 'patch' }}
@@ -169,7 +181,7 @@ jobs:
           cd "./tests/ci"
           python3 ./create_release.py --set-progress-started --progress "docker keeper release"
           export CHECK_NAME="Docker keeper image"
-          python3 docker_server.py --release-type auto --version ${{ env.RELEASE_TAG }} --check-name "$CHECK_NAME" --sha ${{ env.COMMIT_SHA }} ${{ ! inputs.dry-run && '--push'  || '' }}
+          python3 docker_server.py --tag-type ${{ env.DOCKER_TAG_TYPE }} --version ${{ env.RELEASE_TAG }} --check-name "$CHECK_NAME" --sha ${{ env.COMMIT_SHA }} ${{ ! inputs.dry-run && '--push'  || '' }}
           python3 ./create_release.py --set-progress-completed
       - name: Update release info. Merge created PRs
         shell: bash
@@ -178,6 +190,7 @@ jobs:
       - name: Set current Release progress to Completed with OK
         shell: bash
         run: |
+          # dummy stage to finalize release info with "progress: completed; status: OK"
           python3 ./tests/ci/create_release.py --set-progress-started --progress "completed"
           python3 ./tests/ci/create_release.py --set-progress-completed
       - name: Post Slack Message
diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
index 04bef1460a6..071f0f1e20a 100644
--- a/.github/workflows/pull_request.yml
+++ b/.github/workflows/pull_request.yml
@@ -172,12 +172,9 @@ jobs:
           cd "$GITHUB_WORKSPACE/tests/ci"
           python3 merge_pr.py --set-ci-status --wf-status ${{ contains(needs.*.result, 'failure') && 'failure' || 'success' }}
       - name: Check Workflow results
-        run: |
-          export WORKFLOW_RESULT_FILE="/tmp/workflow_results.json"
-          cat > "$WORKFLOW_RESULT_FILE" << 'EOF'
-          ${{ toJson(needs) }}
-          EOF
-          python3 ./tests/ci/ci_buddy.py --check-wf-status
+        uses: ./.github/actions/check_workflow
+        with:
+          needs: ${{ toJson(needs) }}
 
   ################################# Stage Final #################################
   #
diff --git a/tests/ci/artifactory.py b/tests/ci/artifactory.py
index 8bba7bca30e..f3d7d24f717 100644
--- a/tests/ci/artifactory.py
+++ b/tests/ci/artifactory.py
@@ -158,7 +158,7 @@ class DebianArtifactory:
         print("Running test command:")
         print(f"  {cmd}")
         assert Shell.check(cmd)
-        self.release_info.debian_command = debian_command
+        self.release_info.debian = debian_command
         self.release_info.dump()
 
 
@@ -240,7 +240,7 @@ class RpmArtifactory:
         print("Running test command:")
         print(f"  {cmd}")
         assert Shell.check(cmd)
-        self.release_info.rpm_command = rpm_command
+        self.release_info.rpm = rpm_command
         self.release_info.dump()
 
 
@@ -304,7 +304,7 @@ class TgzArtifactory:
             expected_checksum == actual_checksum
         ), f"[{actual_checksum} != {expected_checksum}]"
         Shell.check("rm /tmp/tmp.tgz*", verbose=True)
-        self.release_info.tgz_command = cmd
+        self.release_info.tgz = cmd
         self.release_info.dump()
 
 
diff --git a/tests/ci/auto_release.py b/tests/ci/auto_release.py
index 6c17b4c74ad..3cc88634004 100644
--- a/tests/ci/auto_release.py
+++ b/tests/ci/auto_release.py
@@ -127,15 +127,13 @@ def _prepare(token):
             )
             commit_num -= 1
 
-            is_completed = CI.GHActions.check_wf_completed(
-                token=token, commit_sha=commit
-            )
+            is_completed = CI.GH.check_wf_completed(token=token, commit_sha=commit)
             if not is_completed:
                 print(f"CI is in progress for [{commit}] - check previous commit")
                 commits_to_branch_head += 1
                 continue
 
-            commit_ci_status = CI.GHActions.get_commit_status_by_name(
+            commit_ci_status = CI.GH.get_commit_status_by_name(
                 token=token,
                 commit_sha=commit,
                 status_name=(CI.JobNames.BUILD_CHECK, "ClickHouse build check"),
diff --git a/tests/ci/ci.py b/tests/ci/ci.py
index 935fe472e50..2565c8944e4 100644
--- a/tests/ci/ci.py
+++ b/tests/ci/ci.py
@@ -16,7 +16,7 @@ import upload_result_helper
 from build_check import get_release_or_pr
 from ci_config import CI
 from ci_metadata import CiMetadata
-from ci_utils import GHActions, normalize_string, Utils
+from ci_utils import GH, normalize_string, Utils
 from clickhouse_helper import (
     CiLogsCredentials,
     ClickHouseHelper,
@@ -368,7 +368,7 @@ def _pre_action(s3, job_name, batch, indata, pr_info):
                 )
                 to_be_skipped = True
                 # skip_status = SUCCESS already there
-                GHActions.print_in_group("Commit Status Data", job_status)
+                GH.print_in_group("Commit Status Data", job_status)
 
     # create pre report
     jr = JobReport.create_pre_report(status=skip_status, job_skipped=to_be_skipped)
diff --git a/tests/ci/ci_buddy.py b/tests/ci/ci_buddy.py
index 138909c1db0..f0e73e925fe 100644
--- a/tests/ci/ci_buddy.py
+++ b/tests/ci/ci_buddy.py
@@ -8,7 +8,7 @@ import requests
 from botocore.exceptions import ClientError
 
 from pr_info import PRInfo
-from ci_utils import Shell, GHActions
+from ci_config import CI
 
 
 class CIBuddy:
@@ -31,10 +31,19 @@ class CIBuddy:
         self.sha = pr_info.sha[:10]
 
     def check_workflow(self):
-        GHActions.print_workflow_results()
-        res = GHActions.get_workflow_job_result(GHActions.ActionsNames.RunConfig)
-        if res != GHActions.ActionStatuses.SUCCESS:
-            self.post_job_error("Workflow Configuration Failed", critical=True)
+        CI.GH.print_workflow_results()
+        if CI.Envs.GITHUB_WORKFLOW == CI.WorkFlowNames.CreateRelease:
+            if not CI.GH.is_workflow_ok():
+                self.post_job_error(
+                    f"{CI.Envs.GITHUB_WORKFLOW} Workflow Failed", critical=True
+                )
+        else:
+            res = CI.GH.get_workflow_job_result(CI.GH.ActionsNames.RunConfig)
+            if res != CI.GH.ActionStatuses.SUCCESS:
+                print(f"ERROR: RunConfig status is [{res}] - post report to slack")
+                self.post_job_error(
+                    f"{CI.Envs.GITHUB_WORKFLOW} Workflow Failed", critical=True
+                )
 
     @staticmethod
     def _get_webhooks():
@@ -74,10 +83,13 @@ class CIBuddy:
         message = title
         if isinstance(body, dict):
             for name, value in body.items():
-                if "commit_sha" in name:
+                if "sha" in name and value and len(value) == 40:
                     value = (
                         f"<https://github.com/{self.repo}/commit/{value}|{value[:8]}>"
                     )
+                elif isinstance(value, str) and value.startswith("https://github.com/"):
+                    value_shorten = value.split("/")[-1]
+                    value = f"<{value}|{value_shorten}>"
                 message += f"      *{name}*:    {value}\n"
         else:
             message += body + "\n"
@@ -120,9 +132,11 @@ class CIBuddy:
     ) -> None:
         instance_id, instance_type = "unknown", "unknown"
         if with_instance_info:
-            instance_id = Shell.get_output("ec2metadata --instance-id") or instance_id
+            instance_id = (
+                CI.Shell.get_output("ec2metadata --instance-id") or instance_id
+            )
             instance_type = (
-                Shell.get_output("ec2metadata --instance-type") or instance_type
+                CI.Shell.get_output("ec2metadata --instance-type") or instance_type
             )
         if not job_name:
             job_name = os.getenv("CHECK_NAME", "unknown")
diff --git a/tests/ci/ci_cache.py b/tests/ci/ci_cache.py
index 85eabb84f9f..4846233ab03 100644
--- a/tests/ci/ci_cache.py
+++ b/tests/ci/ci_cache.py
@@ -7,7 +7,7 @@ from typing import Dict, Optional, Any, Union, Sequence, List, Set
 
 from ci_config import CI
 
-from ci_utils import is_hex, GHActions
+from ci_utils import is_hex, GH
 from commit_status_helper import CommitStatusData
 from env_helper import (
     TEMP_PATH,
@@ -258,15 +258,15 @@ class CiCache:
     def print_status(self):
         print(f"Cache enabled: [{self.enabled}]")
         for record_type in self.RecordType:
-            GHActions.print_in_group(
+            GH.print_in_group(
                 f"Cache records: [{record_type}]", list(self.records[record_type])
             )
-        GHActions.print_in_group(
+        GH.print_in_group(
             "Jobs to do:",
             list(self.jobs_to_do.items()),
         )
-        GHActions.print_in_group("Jobs to skip:", self.jobs_to_skip)
-        GHActions.print_in_group(
+        GH.print_in_group("Jobs to skip:", self.jobs_to_skip)
+        GH.print_in_group(
             "Jobs to wait:",
             list(self.jobs_to_wait.items()),
         )
@@ -788,7 +788,7 @@ class CiCache:
 
         while round_cnt < MAX_ROUNDS_TO_WAIT:
             round_cnt += 1
-            GHActions.print_in_group(
+            GH.print_in_group(
                 f"Wait pending jobs, round [{round_cnt}/{MAX_ROUNDS_TO_WAIT}]:",
                 list(self.jobs_to_wait),
             )
@@ -853,7 +853,7 @@ class CiCache:
                     # make up for 2 iterations in dry_run
                     expired_sec += int(TIMEOUT / 2) + 1
 
-        GHActions.print_in_group(
+        GH.print_in_group(
             "Remaining jobs:",
             [list(self.jobs_to_wait)],
         )
diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py
index a7df884a091..c031ca9b805 100644
--- a/tests/ci/ci_config.py
+++ b/tests/ci/ci_config.py
@@ -34,7 +34,8 @@ class CI:
     from ci_definitions import Runners as Runners
     from ci_utils import Envs as Envs
     from ci_utils import Utils as Utils
-    from ci_utils import GHActions as GHActions
+    from ci_utils import GH as GH
+    from ci_utils import Shell as Shell
     from ci_definitions import Labels as Labels
     from ci_definitions import TRUSTED_CONTRIBUTORS as TRUSTED_CONTRIBUTORS
     from ci_definitions import WorkFlowNames as WorkFlowNames
diff --git a/tests/ci/ci_definitions.py b/tests/ci/ci_definitions.py
index 51de8c63509..de6791acda8 100644
--- a/tests/ci/ci_definitions.py
+++ b/tests/ci/ci_definitions.py
@@ -112,6 +112,7 @@ class WorkFlowNames(metaclass=WithIter):
     """
 
     JEPSEN = "JepsenWorkflow"
+    CreateRelease = "CreateRelease"
 
 
 class BuildNames(metaclass=WithIter):
@@ -578,7 +579,7 @@ class CommonJobConfigs:
     DOCKER_SERVER = JobConfig(
         job_name_keyword="docker",
         required_on_release_branch=True,
-        run_command='docker_server.py --check-name "$CHECK_NAME" --release-type head --allow-build-reuse',
+        run_command='docker_server.py --check-name "$CHECK_NAME" --tag-type head --allow-build-reuse',
         digest=DigestConfig(
             include_paths=[
                 "tests/ci/docker_server.py",
diff --git a/tests/ci/ci_metadata.py b/tests/ci/ci_metadata.py
index a767d102811..67106262634 100644
--- a/tests/ci/ci_metadata.py
+++ b/tests/ci/ci_metadata.py
@@ -9,7 +9,7 @@ from env_helper import (
     S3_BUILDS_BUCKET_PUBLIC,
 )
 from s3_helper import S3Helper
-from ci_utils import GHActions
+from ci_utils import GH
 from synchronizer_utils import SYNC_BRANCH_PREFIX
 
 
@@ -111,7 +111,7 @@ class CiMetadata:
         else:
             log_title = f"Storing workflow metadata: PR [{self.pr_number}], upstream PR [{self.upstream_pr_number}]"
 
-        GHActions.print_in_group(
+        GH.print_in_group(
             log_title,
             [f"run_id: {self.run_id}"],
         )
diff --git a/tests/ci/ci_utils.py b/tests/ci/ci_utils.py
index 4f696a2c55a..dae1520afb6 100644
--- a/tests/ci/ci_utils.py
+++ b/tests/ci/ci_utils.py
@@ -16,6 +16,8 @@ class Envs:
     WORKFLOW_RESULT_FILE = os.getenv(
         "WORKFLOW_RESULT_FILE", "/tmp/workflow_results.json"
     )
+    S3_BUILDS_BUCKET = os.getenv("S3_BUILDS_BUCKET", "clickhouse-builds")
+    GITHUB_WORKFLOW = os.getenv("GITHUB_WORKFLOW", "")
 
 
 LABEL_CATEGORIES = {
@@ -83,7 +85,7 @@ def normalize_string(string: str) -> str:
     return res
 
 
-class GHActions:
+class GH:
     class ActionsNames:
         RunConfig = "RunConfig"
 
@@ -117,6 +119,14 @@ class GHActions:
         results = [f"{job}: {data['result']}" for job, data in res.items()]
         cls.print_in_group("Workflow results", results)
 
+    @classmethod
+    def is_workflow_ok(cls) -> bool:
+        res = cls._get_workflow_results()
+        for _job, data in res.items():
+            if data["result"] == "failure":
+                return False
+        return bool(res)
+
     @classmethod
     def get_workflow_job_result(cls, wf_job_name: str) -> Optional[str]:
         res = cls._get_workflow_results()
@@ -189,15 +199,25 @@ class GHActions:
         return False
 
     @staticmethod
-    def get_pr_url_by_branch(repo, branch):
-        get_url_cmd = (
-            f"gh pr list --repo {repo} --head {branch} --json url --jq '.[0].url'"
-        )
+    def get_pr_url_by_branch(branch, repo=None):
+        repo = repo or Envs.GITHUB_REPOSITORY
+        get_url_cmd = f"gh pr list --repo {repo} --head {branch} --json url --jq '.[0].url' --state open"
         url = Shell.get_output(get_url_cmd)
+        if not url:
+            print(f"WARNING: No open PR found, branch [{branch}] - search for merged")
+            get_url_cmd = f"gh pr list --repo {repo} --head {branch} --json url --jq '.[0].url' --state merged"
+            url = Shell.get_output(get_url_cmd)
         if not url:
             print(f"ERROR: PR nor found, branch [{branch}]")
         return url
 
+    @staticmethod
+    def is_latest_release_branch(branch):
+        latest_branch = Shell.get_output(
+            'gh pr list --label release --repo ClickHouse/ClickHouse --search "sort:created" -L1 --json headRefName'
+        )
+        return latest_branch == branch
+
 
 class Shell:
     @classmethod
diff --git a/tests/ci/create_release.py b/tests/ci/create_release.py
index c407a74fbf0..b4e08f29dbe 100755
--- a/tests/ci/create_release.py
+++ b/tests/ci/create_release.py
@@ -10,9 +10,8 @@ from typing import Iterator, List
 
 from git_helper import Git, GIT_PREFIX
 from ssh import SSHAgent
-from env_helper import GITHUB_REPOSITORY, S3_BUILDS_BUCKET
 from s3_helper import S3Helper
-from ci_utils import Shell, GHActions
+from ci_utils import Shell, GH
 from ci_buddy import CIBuddy
 from version_helper import (
     FILE_WITH_VERSION_PATH,
@@ -69,13 +68,14 @@ class ReleaseContextManager:
                 previous_release_tag="NA",
                 previous_release_sha="NA",
                 release_progress=ReleaseProgress.STARTED,
+                latest=False,
             ).dump()
         else:
             # fetch release info from fs and update
             self.release_info = ReleaseInfo.from_file()
             assert self.release_info
             assert (
-                self.release_info.progress_description == ReleaseProgressDescription.OK
+                self.release_info.progress_status == ReleaseProgressDescription.OK
             ), "Must be OK on the start of new context"
             self.release_info.release_progress = self.release_progress
             self.release_info.dump()
@@ -84,9 +84,9 @@ class ReleaseContextManager:
     def __exit__(self, exc_type, exc_value, traceback):
         assert self.release_info
         if exc_type is not None:
-            self.release_info.progress_description = ReleaseProgressDescription.FAILED
+            self.release_info.progress_status = ReleaseProgressDescription.FAILED
         else:
-            self.release_info.progress_description = ReleaseProgressDescription.OK
+            self.release_info.progress_status = ReleaseProgressDescription.OK
         self.release_info.dump()
 
 
@@ -96,6 +96,7 @@ class ReleaseInfo:
     release_tag: str
     release_branch: str
     commit_sha: str
+    latest: bool
     # lts or stable
     codename: str
     previous_release_tag: str
@@ -104,12 +105,12 @@ class ReleaseInfo:
     version_bump_pr: str = ""
     prs_merged: bool = False
     release_url: str = ""
-    debian_command: str = ""
-    rpm_command: str = ""
-    tgz_command: str = ""
-    docker_command: str = ""
+    debian: str = ""
+    rpm: str = ""
+    tgz: str = ""
+    docker: str = ""
     release_progress: str = ""
-    progress_description: str = ""
+    progress_status: str = ""
 
     def is_patch(self):
         return self.release_branch != "master"
@@ -129,12 +130,15 @@ class ReleaseInfo:
             print(json.dumps(dataclasses.asdict(self), indent=2), file=f)
         return self
 
-    def prepare(self, commit_ref: str, release_type: str) -> "ReleaseInfo":
+    def prepare(
+        self, commit_ref: str, release_type: str, skip_tag_check: bool
+    ) -> "ReleaseInfo":
         version = None
         release_branch = None
         release_tag = None
         previous_release_tag = None
         previous_release_sha = None
+        latest_release = False
         codename = ""
         assert release_type in ("patch", "new")
         if release_type == "new":
@@ -145,7 +149,7 @@ class ReleaseInfo:
                 verbose=True,
             )
             with checkout(commit_ref):
-                commit_sha = Shell.get_output_or_raise(f"git rev-parse {commit_ref}")
+                commit_sha = Shell.get_output_or_raise(f"git rev-list -n1 {commit_ref}")
                 # Git() must be inside "with checkout" contextmanager
                 git = Git()
                 version = get_version_from_repo(git=git)
@@ -158,12 +162,12 @@ class ReleaseInfo:
                 release_tag = version.describe
                 previous_release_tag = expected_prev_tag
                 previous_release_sha = Shell.get_output_or_raise(
-                    f"git rev-parse {previous_release_tag}"
+                    f"git rev-list -n1 {previous_release_tag}"
                 )
                 assert previous_release_sha
         if release_type == "patch":
             with checkout(commit_ref):
-                commit_sha = Shell.get_output_or_raise(f"git rev-parse {commit_ref}")
+                commit_sha = Shell.get_output_or_raise(f"git rev-list -n1 {commit_ref}")
                 # Git() must be inside "with checkout" contextmanager
                 git = Git()
                 version = get_version_from_repo(git=git)
@@ -200,16 +204,20 @@ class ReleaseInfo:
                 expected_tag_prefix
             ) and git.latest_tag.endswith(expected_tag_suffix):
                 pass
-            else:
+            elif not skip_tag_check:
                 assert (
                     False
-                ), f"BUG: Unexpected latest tag [{git.latest_tag}] expected [{expected_tag_prefix}*{expected_tag_suffix}]"
+                ), f"BUG: Unexpected latest tag [{git.latest_tag}] expected [{expected_tag_prefix}*{expected_tag_suffix}]. Already Released?"
 
             previous_release_sha = Shell.get_output_or_raise(
-                f"git rev-parse {previous_release_tag}"
+                f"git rev-list -n1 {previous_release_tag}"
             )
             assert previous_release_sha
 
+            if CI.GH.is_latest_release_branch(release_branch):
+                print("This is going to be the latest release!")
+                latest_release = True
+
         assert (
             release_branch
             and previous_release_tag
@@ -218,7 +226,7 @@ class ReleaseInfo:
             and release_tag
             and version
             and (codename in ("lts", "stable") or release_type == "new")
-        )
+        ), f"Check: {release_branch}, {previous_release_tag}, {previous_release_sha}, {commit_sha}, {release_tag}, {version}"
 
         self.release_branch = release_branch
         self.commit_sha = commit_sha
@@ -228,7 +236,8 @@ class ReleaseInfo:
         self.previous_release_tag = previous_release_tag
         self.previous_release_sha = previous_release_sha
         self.release_progress = ReleaseProgress.STARTED
-        self.progress_description = ReleaseProgressDescription.OK
+        self.progress_status = ReleaseProgressDescription.OK
+        self.latest = latest_release
         return self
 
     def push_release_tag(self, dry_run: bool) -> None:
@@ -252,7 +261,7 @@ class ReleaseInfo:
 
     @staticmethod
     def _create_gh_label(label: str, color_hex: str, dry_run: bool) -> None:
-        cmd = f"gh api repos/{GITHUB_REPOSITORY}/labels -f name={label} -f color={color_hex}"
+        cmd = f"gh api repos/{CI.Envs.GITHUB_REPOSITORY}/labels -f name={label} -f color={color_hex}"
         Shell.check(cmd, dry_run=dry_run, strict=True)
 
     def push_new_release_branch(self, dry_run: bool) -> None:
@@ -294,7 +303,7 @@ class ReleaseInfo:
             f"v{new_release_branch}-affected", "c2bfff", dry_run=dry_run
         )
         Shell.check(
-            f"""gh pr create --repo {GITHUB_REPOSITORY} --title 'Release pull request for branch {new_release_branch}'
+            f"""gh pr create --repo {CI.Envs.GITHUB_REPOSITORY} --title 'Release pull request for branch {new_release_branch}'
             --head {new_release_branch} {pr_labels}
             --body 'This PullRequest is a part of ClickHouse release cycle. It is used by CI system only. Do not perform any changes with it.'
             """,
@@ -303,9 +312,12 @@ class ReleaseInfo:
             verbose=True,
         )
 
+    def get_version_bump_branch(self):
+        return f"bump_version_{self.version}"
+
     def update_version_and_contributors_list(self, dry_run: bool) -> None:
         # Bump version, update contributors list, create PR
-        branch_upd_version_contributors = f"bump_version_{self.version}"
+        branch_upd_version_contributors = self.get_version_bump_branch()
         with checkout(self.commit_sha):
             git = Git()
             version = get_version_from_repo(git=git)
@@ -323,9 +335,9 @@ class ReleaseInfo:
                 update_contributors(raise_error=True)
                 cmd_commit_version_upd = f"{GIT_PREFIX} commit '{CMAKE_PATH}' '{CONTRIBUTORS_PATH}' -m 'Update autogenerated version to {self.version} and contributors'"
                 cmd_push_branch = f"{GIT_PREFIX} push --set-upstream origin {branch_upd_version_contributors}"
-                body_file = get_abs_path(".github/PULL_REQUEST_TEMPLATE.md")
                 actor = os.getenv("GITHUB_ACTOR", "") or "me"
-                cmd_create_pr = f"gh pr create --repo {GITHUB_REPOSITORY} --title 'Update version after release' --head {branch_upd_version_contributors} --base {self.release_branch} --body-file {body_file} --label 'do not test' --assignee {actor}"
+                body = f"Automatic version bump after release {self.release_tag}\n### Changelog category (leave one):\n- Not for changelog (changelog entry is not required)\n"
+                cmd_create_pr = f"gh pr create --repo {CI.Envs.GITHUB_REPOSITORY} --title 'Update version after release' --head {branch_upd_version_contributors} --base {self.release_branch} --body \"{body}\" --assignee {actor}"
                 Shell.check(
                     cmd_commit_version_upd, strict=True, dry_run=dry_run, verbose=True
                 )
@@ -342,30 +354,42 @@ class ReleaseInfo:
                     )
                     self.version_bump_pr = "dry-run"
                 else:
-                    self.version_bump_pr = GHActions.get_pr_url_by_branch(
-                        repo=GITHUB_REPOSITORY, branch=branch_upd_version_contributors
+                    self.version_bump_pr = GH.get_pr_url_by_branch(
+                        branch=branch_upd_version_contributors
                     )
 
+    def get_change_log_branch(self):
+        return f"auto/{self.release_tag}"
+
     def update_release_info(self, dry_run: bool) -> "ReleaseInfo":
         if self.release_branch != "master":
-            branch = f"auto/{release_info.release_tag}"
-            if not dry_run:
-                url = GHActions.get_pr_url_by_branch(
-                    repo=GITHUB_REPOSITORY, branch=branch
-                )
-            else:
-                url = "dry-run"
-            print(f"ChangeLog PR url [{url}]")
-            self.changelog_pr = url
-            print(f"Release url [{url}]")
-            self.release_url = f"https://github.com/{GITHUB_REPOSITORY}/releases/tag/{self.release_tag}"
-            if self.release_progress == ReleaseProgress.COMPLETED:
-                self.docker_command = f"docker run --rm clickhouse/clickhouse:{self.version} clickhouse --version"
+            if not self.changelog_pr:
+                branch = self.get_change_log_branch()
+                if not dry_run:
+                    url = GH.get_pr_url_by_branch(branch=branch)
+                else:
+                    url = "dry-run"
+                print(f"ChangeLog PR url [{url}]")
+                self.changelog_pr = url
+
+            if not self.version_bump_pr:
+                branch = self.get_version_bump_branch()
+                if not dry_run:
+                    url = GH.get_pr_url_by_branch(branch=branch)
+                else:
+                    url = "dry-run"
+                print(f"Version bump PR url [{url}]")
+                self.version_bump_pr = url
+
+            self.release_url = f"https://github.com/{CI.Envs.GITHUB_REPOSITORY}/releases/tag/{self.release_tag}"
+            print(f"Release url [{self.release_url}]")
+
+            self.docker = f"docker run --rm clickhouse/clickhouse:{self.version} clickhouse --version"
         self.dump()
         return self
 
     def create_gh_release(self, packages_files: List[str], dry_run: bool) -> None:
-        repo = os.getenv("GITHUB_REPOSITORY")
+        repo = CI.Envs.GITHUB_REPOSITORY
         assert repo
         cmds = [
             f"gh release create --repo {repo} --title 'Release {self.release_tag}' {self.release_tag}"
@@ -375,7 +399,9 @@ class ReleaseInfo:
         if not dry_run:
             for cmd in cmds:
                 Shell.check(cmd, strict=True, verbose=True)
-            self.release_url = f"https://github.com/{GITHUB_REPOSITORY}/releases/tag/{self.release_tag}"
+            self.release_url = (
+                f"https://github.com/{repo}/releases/tag/{self.release_tag}"
+            )
         else:
             print("Dry-run, would run commands:")
             print("\n  * ".join(cmds))
@@ -536,7 +562,7 @@ class PackageDownloader:
                 ]
             )
             self.s3.download_file(
-                bucket=S3_BUILDS_BUCKET,
+                bucket=CI.Envs.S3_BUILDS_BUCKET,
                 s3_path=s3_path,
                 local_file_path="/".join([self.LOCAL_DIR, package_file]),
             )
@@ -557,7 +583,7 @@ class PackageDownloader:
                 ]
             )
             self.s3.download_file(
-                bucket=S3_BUILDS_BUCKET,
+                bucket=CI.Envs.S3_BUILDS_BUCKET,
                 s3_path=s3_path,
                 local_file_path="/".join([self.LOCAL_DIR, destination_binary_name]),
             )
@@ -636,6 +662,11 @@ def parse_args() -> argparse.Namespace:
         action="store_true",
         help="Initial step to prepare info like release branch, release tag, etc.",
     )
+    parser.add_argument(
+        "--skip-tag-check",
+        action="store_true",
+        help="To skip check against latest git tag on a release branch",
+    )
     parser.add_argument(
         "--push-release-tag",
         action="store_true",
@@ -725,7 +756,11 @@ if __name__ == "__main__":
             assert (
                 args.ref and args.release_type
             ), "--ref and --release-type must be provided with --prepare-release-info"
-            release_info.prepare(commit_ref=args.ref, release_type=args.release_type)
+            release_info.prepare(
+                commit_ref=args.ref,
+                release_type=args.release_type,
+                skip_tag_check=args.skip_tag_check,
+            )
 
     if args.download_packages:
         with ReleaseContextManager(
@@ -776,7 +811,7 @@ if __name__ == "__main__":
         else:
             title = "New release"
         if (
-            release_info.progress_description == ReleaseProgressDescription.OK
+            release_info.progress_status == ReleaseProgressDescription.OK
             and release_info.release_progress == ReleaseProgress.COMPLETED
         ):
             title = "Completed: " + title
@@ -792,16 +827,16 @@ if __name__ == "__main__":
     if args.set_progress_started:
         ri = ReleaseInfo.from_file()
         ri.release_progress = args.progress
-        ri.progress_description = ReleaseProgressDescription.FAILED
+        ri.progress_status = ReleaseProgressDescription.FAILED
         ri.dump()
         assert args.progress, "Progress step name must be provided"
 
     if args.set_progress_completed:
         ri = ReleaseInfo.from_file()
         assert (
-            ri.progress_description == ReleaseProgressDescription.FAILED
+            ri.progress_status == ReleaseProgressDescription.FAILED
         ), "Must be FAILED before set to OK"
-        ri.progress_description = ReleaseProgressDescription.OK
+        ri.progress_status = ReleaseProgressDescription.OK
         ri.dump()
 
     if args.merge_prs:
diff --git a/tests/ci/docker_server.py b/tests/ci/docker_server.py
index 3e782c079c6..8f0474d5053 100644
--- a/tests/ci/docker_server.py
+++ b/tests/ci/docker_server.py
@@ -69,13 +69,14 @@ def parse_args() -> argparse.Namespace:
         help="sha of the commit to use packages from",
     )
     parser.add_argument(
-        "--release-type",
+        "--tag-type",
         type=str,
-        choices=("auto", "latest", "major", "minor", "patch", "head"),
+        choices=("head", "release", "latest-release"),
         default="head",
-        help="version part that will be updated when '--version' is set; "
-        "'auto' is a special case, it will get versions from github and detect the "
-        "release type (latest, major, minor or patch) automatically",
+        help="defines required tags for resulting docker image. "
+        "head - for master image (tag: head) "
+        "release - for release image (tags: XX, XX.XX, XX.XX.XX, XX.XX.XX.XX) "
+        "release-latest - for latest release image (tags: XX, XX.XX, XX.XX.XX, XX.XX.XX.XX, latest) ",
     )
     parser.add_argument(
         "--image-path",
@@ -149,74 +150,35 @@ def retry_popen(cmd: str, log_file: Path) -> int:
     return retcode
 
 
-def auto_release_type(version: ClickHouseVersion, release_type: str) -> str:
-    if release_type != "auto":
-        return release_type
-
-    git_versions = get_tagged_versions()
-    reference_version = git_versions[0]
-    for i in reversed(range(len(git_versions))):
-        if git_versions[i] <= version:
-            if i == len(git_versions) - 1:
-                return "latest"
-            reference_version = git_versions[i + 1]
-            break
-
-    if version.major < reference_version.major:
-        return "major"
-    if version.minor < reference_version.minor:
-        return "minor"
-    if version.patch < reference_version.patch:
-        return "patch"
-
-    raise ValueError(
-        "Release type 'tweak' is not supported for "
-        f"{version.string} < {reference_version.string}"
-    )
-
-
-def gen_tags(version: ClickHouseVersion, release_type: str) -> List[str]:
+def gen_tags(version: ClickHouseVersion, tag_type: str) -> List[str]:
     """
-    22.2.2.2 + latest:
+    @tag_type release-latest, @version 22.2.2.2:
     - latest
     - 22
     - 22.2
     - 22.2.2
     - 22.2.2.2
-    22.2.2.2 + major:
+    @tag_type release, @version 22.2.2.2:
     - 22
     - 22.2
     - 22.2.2
     - 22.2.2.2
-    22.2.2.2 + minor:
-    - 22.2
-    - 22.2.2
-    - 22.2.2.2
-    22.2.2.2 + patch:
-    - 22.2.2
-    - 22.2.2.2
-    22.2.2.2 + head:
+    @tag_type head:
     - head
     """
     parts = version.string.split(".")
     tags = []
-    if release_type == "latest":
-        tags.append(release_type)
+    if tag_type == "release-latest":
+        tags.append("latest")
         for i in range(len(parts)):
             tags.append(".".join(parts[: i + 1]))
-    elif release_type == "major":
+    elif tag_type == "head":
+        tags.append(tag_type)
+    elif tag_type == "release":
         for i in range(len(parts)):
             tags.append(".".join(parts[: i + 1]))
-    elif release_type == "minor":
-        for i in range(1, len(parts)):
-            tags.append(".".join(parts[: i + 1]))
-    elif release_type == "patch":
-        for i in range(2, len(parts)):
-            tags.append(".".join(parts[: i + 1]))
-    elif release_type == "head":
-        tags.append(release_type)
     else:
-        raise ValueError(f"{release_type} is not valid release part")
+        assert False, f"Invalid release type [{tag_type}]"
     return tags
 
 
@@ -370,8 +332,7 @@ def main():
         push = True
 
     image = DockerImageData(image_path, image_repo, False)
-    args.release_type = auto_release_type(args.version, args.release_type)
-    tags = gen_tags(args.version, args.release_type)
+    tags = gen_tags(args.version, args.tag_type)
     repo_urls = {}
     direct_urls: Dict[str, List[str]] = {}
 
diff --git a/tests/ci/test_docker.py b/tests/ci/test_docker.py
index 662143bfd9b..58ebe4ecbb1 100644
--- a/tests/ci/test_docker.py
+++ b/tests/ci/test_docker.py
@@ -1,61 +1,19 @@
 #!/usr/bin/env python
 
 import unittest
-from unittest.mock import patch, MagicMock
 
 from version_helper import get_version_from_string
 import docker_server as ds
 
-# di.logging.basicConfig(level=di.logging.INFO)
-
 
 class TestDockerServer(unittest.TestCase):
     def test_gen_tags(self):
         version = get_version_from_string("22.2.2.2")
         cases = (
-            ("latest", ["latest", "22", "22.2", "22.2.2", "22.2.2.2"]),
-            ("major", ["22", "22.2", "22.2.2", "22.2.2.2"]),
-            ("minor", ["22.2", "22.2.2", "22.2.2.2"]),
-            ("patch", ["22.2.2", "22.2.2.2"]),
+            ("release-latest", ["latest", "22", "22.2", "22.2.2", "22.2.2.2"]),
+            ("release", ["22", "22.2", "22.2.2", "22.2.2.2"]),
             ("head", ["head"]),
         )
         for case in cases:
             release_type = case[0]
             self.assertEqual(case[1], ds.gen_tags(version, release_type))
-
-        with self.assertRaises(ValueError):
-            ds.gen_tags(version, "auto")
-
-    @patch("docker_server.get_tagged_versions")
-    def test_auto_release_type(self, mock_tagged_versions: MagicMock) -> None:
-        mock_tagged_versions.return_value = [
-            get_version_from_string("1.1.1.1"),
-            get_version_from_string("1.2.1.1"),
-            get_version_from_string("2.1.1.1"),
-            get_version_from_string("2.2.1.1"),
-            get_version_from_string("2.2.2.1"),
-        ]
-
-        cases_less = (
-            (get_version_from_string("1.0.1.1"), "minor"),
-            (get_version_from_string("1.1.2.1"), "minor"),
-            (get_version_from_string("1.3.1.1"), "major"),
-            (get_version_from_string("2.1.2.1"), "minor"),
-            (get_version_from_string("2.2.1.3"), "patch"),
-            (get_version_from_string("2.2.3.1"), "latest"),
-            (get_version_from_string("2.3.1.1"), "latest"),
-        )
-        for case in cases_less:
-            release = ds.auto_release_type(case[0], "auto")
-            self.assertEqual(case[1], release)
-
-        cases_equal = (
-            (get_version_from_string("1.1.1.1"), "minor"),
-            (get_version_from_string("1.2.1.1"), "major"),
-            (get_version_from_string("2.1.1.1"), "minor"),
-            (get_version_from_string("2.2.1.1"), "patch"),
-            (get_version_from_string("2.2.2.1"), "latest"),
-        )
-        for case in cases_equal:
-            release = ds.auto_release_type(case[0], "auto")
-            self.assertEqual(case[1], release)

From ae6d4bdd8aa2b010f27dd8ebbb6f816362eb2a9e Mon Sep 17 00:00:00 2001
From: Sasha Sheikin <myminitrue@gmail.com>
Date: Mon, 29 Jul 2024 10:31:35 +0200
Subject: [PATCH 1278/1488] Fix positionCaseInsensitive example

---
 .../en/sql-reference/functions/string-search-functions.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/en/sql-reference/functions/string-search-functions.md b/docs/en/sql-reference/functions/string-search-functions.md
index b7ba1d4feb7..e9ff7ebf33b 100644
--- a/docs/en/sql-reference/functions/string-search-functions.md
+++ b/docs/en/sql-reference/functions/string-search-functions.md
@@ -150,15 +150,15 @@ A case insensitive invariant of [position](#position).
 Query:
 
 ``` sql
-SELECT position('Hello, world!', 'hello');
+SELECT positionCaseInsensitive('Hello, world!', 'hello');
 ```
 
 Result:
 
 ``` text
-┌─position('Hello, world!', 'hello')─┐
-│                                  0 │
-└────────────────────────────────────┘
+┌─positionCaseInsensitive('Hello, world!', 'hello')─┐
+│                                                 1 │
+└───────────────────────────────────────────────────┘
 ```
 
 ## positionUTF8

From 89c47df559ba23d988f8af3c342e0c8d5531f4b8 Mon Sep 17 00:00:00 2001
From: avogar <avogar@clickhouse.com>
Date: Fri, 2 Aug 2024 16:52:46 +0000
Subject: [PATCH 1279/1488] Fix variant as common type in if function with
 Tuples and Maps

---
 src/Functions/if.cpp                                   | 10 ++++++++++
 .../03215_varian_as_common_type_tuple_map.reference    | 10 ++++++++++
 .../03215_varian_as_common_type_tuple_map.sql          |  7 +++++++
 3 files changed, 27 insertions(+)
 create mode 100644 tests/queries/0_stateless/03215_varian_as_common_type_tuple_map.reference
 create mode 100644 tests/queries/0_stateless/03215_varian_as_common_type_tuple_map.sql

diff --git a/src/Functions/if.cpp b/src/Functions/if.cpp
index 07dbee27a9d..64da6e95a43 100644
--- a/src/Functions/if.cpp
+++ b/src/Functions/if.cpp
@@ -632,6 +632,11 @@ private:
 
     ColumnPtr executeTuple(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const
     {
+        /// For different Tuples the result type can be Variant with this Tuples if use_variant_as_common_type=1.
+        /// In this case we should use generic implementation.
+        if (!isTuple(result_type))
+            return nullptr;
+
         /// Calculate function for each corresponding elements of tuples.
 
         const ColumnWithTypeAndName & arg1 = arguments[1];
@@ -677,6 +682,11 @@ private:
 
     ColumnPtr executeMap(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const
     {
+        /// For different Maps the result type can be Variant with this Maps if use_variant_as_common_type=1.
+        /// In this case we should use generic implementation.
+        if (!isMap(result_type))
+            return nullptr;
+
         auto extract_kv_from_map = [](const ColumnMap * map)
         {
             const ColumnTuple & tuple = map->getNestedData();
diff --git a/tests/queries/0_stateless/03215_varian_as_common_type_tuple_map.reference b/tests/queries/0_stateless/03215_varian_as_common_type_tuple_map.reference
new file mode 100644
index 00000000000..7dabd5388f4
--- /dev/null
+++ b/tests/queries/0_stateless/03215_varian_as_common_type_tuple_map.reference
@@ -0,0 +1,10 @@
+('0')	Variant(Tuple(String), Tuple(\n    number UInt64))
+(1)	Variant(Tuple(String), Tuple(\n    number UInt64))
+('2')	Variant(Tuple(String), Tuple(\n    number UInt64))
+(3)	Variant(Tuple(String), Tuple(\n    number UInt64))
+('4')	Variant(Tuple(String), Tuple(\n    number UInt64))
+{'0':'0'}	Variant(Map(String, String), Map(UInt64, UInt64))
+{1:1}	Variant(Map(String, String), Map(UInt64, UInt64))
+{'2':'2'}	Variant(Map(String, String), Map(UInt64, UInt64))
+{3:3}	Variant(Map(String, String), Map(UInt64, UInt64))
+{'4':'4'}	Variant(Map(String, String), Map(UInt64, UInt64))
diff --git a/tests/queries/0_stateless/03215_varian_as_common_type_tuple_map.sql b/tests/queries/0_stateless/03215_varian_as_common_type_tuple_map.sql
new file mode 100644
index 00000000000..4a9a788ab18
--- /dev/null
+++ b/tests/queries/0_stateless/03215_varian_as_common_type_tuple_map.sql
@@ -0,0 +1,7 @@
+set use_variant_as_common_type = 1;
+set allow_experimental_variant_type = 1;
+
+SELECT if(number % 2, tuple(number), tuple(toString(number))) as res, toTypeName(res) FROM numbers(5);
+SELECT if(number % 2, map(number, number), map(toString(number), toString(number))) as res, toTypeName(res) FROM numbers(5);
+
+

From b38c46a87d6eacbc7805562deb07ce586fd7e0fb Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 31 Jul 2024 17:38:20 +0200
Subject: [PATCH 1280/1488] Minor change

---
 src/Databases/DatabaseOnDisk.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp
index f419f5811a1..734f354d9a5 100644
--- a/src/Databases/DatabaseOnDisk.cpp
+++ b/src/Databases/DatabaseOnDisk.cpp
@@ -313,7 +313,7 @@ void DatabaseOnDisk::detachTablePermanently(ContextPtr query_context, const Stri
         std::lock_guard lock(mutex);
         if (const auto it = snapshot_detached_tables.find(table_name); it == snapshot_detached_tables.end())
         {
-            throw Exception(ErrorCodes::LOGICAL_ERROR, "Snapshot doesn't contain info about detached table={}", table_name);
+            throw Exception(ErrorCodes::LOGICAL_ERROR, "Snapshot doesn't contain info about detached table `{}`", table_name);
         }
         else
         {

From e3239c6ee11eb5bf0466fb750c58125868885ec8 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 2 Aug 2024 19:09:16 +0200
Subject: [PATCH 1281/1488] Fix bad log message in JIT for sorting

---
 src/Core/SortDescription.cpp | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/src/Core/SortDescription.cpp b/src/Core/SortDescription.cpp
index 9edc79a1ff1..1b3f81f8547 100644
--- a/src/Core/SortDescription.cpp
+++ b/src/Core/SortDescription.cpp
@@ -103,7 +103,15 @@ static std::string getSortDescriptionDump(const SortDescription & description, c
     WriteBufferFromOwnString buffer;
 
     for (size_t i = 0; i < description.size(); ++i)
-        buffer << header_types[i]->getName() << ' ' << description[i].direction << ' ' << description[i].nulls_direction;
+    {
+        if (i != 0)
+            buffer << ", ";
+
+        buffer << "(type: " << header_types[i]->getName()
+            << ", direction: " << description[i].direction
+            << ", nulls_direction: " << description[i].nulls_direction
+            << ")";
+    }
 
     return buffer.str();
 }

From 3c4389ec4d78db55ce742e5d5a3b0ed050c9c9e6 Mon Sep 17 00:00:00 2001
From: Denny Crane <denis.zhuravlov@gmail.com>
Date: Fri, 2 Aug 2024 14:57:19 -0300
Subject: [PATCH 1282/1488] doc/fix max_partitions_to_read description

---
 .../operations/settings/merge-tree-settings.md |  7 ++-----
 .../en/operations/settings/query-complexity.md | 18 ++++++++++++++++--
 2 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/docs/en/operations/settings/merge-tree-settings.md b/docs/en/operations/settings/merge-tree-settings.md
index 7278b91f90d..67fa45c20cd 100644
--- a/docs/en/operations/settings/merge-tree-settings.md
+++ b/docs/en/operations/settings/merge-tree-settings.md
@@ -119,11 +119,6 @@ Minimum size of blocks of uncompressed data required for compression when writin
 You can also specify this setting in the global settings (see [min_compress_block_size](/docs/en/operations/settings/settings.md/#min-compress-block-size) setting).
 The value specified when table is created overrides the global value for this setting.
 
-## max_partitions_to_read
-
-Limits the maximum number of partitions that can be accessed in one query.
-You can also specify setting [max_partitions_to_read](/docs/en/operations/settings/merge-tree-settings.md/#max-partitions-to-read) in the global setting.
-
 ## max_suspicious_broken_parts
 
 If the number of broken parts in a single partition exceeds the `max_suspicious_broken_parts` value, automatic deletion is denied.
@@ -691,6 +686,8 @@ Possible values:
 
 Default value: -1 (unlimited).
 
+You can also specify a query complexity setting [max_partitions_to_read](query-complexity#max-partitions-to-read) at a query / session / profile level.
+
 ## min_age_to_force_merge_seconds {#min_age_to_force_merge_seconds}
 
 Merge parts if every part in the range is older than the value of `min_age_to_force_merge_seconds`.
diff --git a/docs/en/operations/settings/query-complexity.md b/docs/en/operations/settings/query-complexity.md
index 2a20e74e20f..14ccb1167f9 100644
--- a/docs/en/operations/settings/query-complexity.md
+++ b/docs/en/operations/settings/query-complexity.md
@@ -188,7 +188,7 @@ If you set `timeout_before_checking_execution_speed `to 0, ClickHouse will use c
 
 What to do if the query is run longer than `max_execution_time` or the estimated running time is longer than `max_estimated_execution_time`: `throw` or `break`. By default, `throw`.
 
-# max_execution_time_leaf
+## max_execution_time_leaf
 
 Similar semantic to `max_execution_time` but only apply on leaf node for distributed or remote queries.
 
@@ -204,7 +204,7 @@ We can use `max_execution_time_leaf` as the query settings:
 SELECT count() FROM cluster(cluster, view(SELECT * FROM t)) SETTINGS max_execution_time_leaf = 10;
 ```
 
-# timeout_overflow_mode_leaf
+## timeout_overflow_mode_leaf
 
 What to do when the query in leaf node run longer than `max_execution_time_leaf`: `throw` or `break`. By default, `throw`.
 
@@ -426,3 +426,17 @@ Example:
 ```
 
 Default value: 0 (Infinite count of simultaneous sessions).
+
+## max_partitions_to_read {#max-partitions-to-read}
+
+Limits the maximum number of partitions that can be accessed in one query.
+
+The setting value specified when the table is created can be overridden via query-level setting.
+
+Possible values:
+
+- Any positive integer.
+
+Default value: -1 (unlimited).
+
+You can also specify a MergeTree setting [max_partitions_to_read](merge-tree-settings#max-partitions-to-read) in tables' setting.

From ce39957983af8bdd7d97e4a3729e2f97d3e0cb85 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 2 Aug 2024 20:11:25 +0200
Subject: [PATCH 1283/1488] Remove capitalization in test reports

---
 tests/ci/report.py | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/tests/ci/report.py b/tests/ci/report.py
index f50ed4c1f85..3f0fc596824 100644
--- a/tests/ci/report.py
+++ b/tests/ci/report.py
@@ -293,9 +293,9 @@ class JobReport:
     start_time: str
     duration: float
     additional_files: Union[Sequence[str], Sequence[Path]]
-    # clickhouse version, build job only
+    # ClickHouse version, build job only
     version: str = ""
-    # checkname to set in commit status, set if differs from jjob name
+    # check_name to be set in commit status, set it if it differs from the job name
     check_name: str = ""
     # directory with artifacts to upload on s3
     build_dir_for_upload: Union[Path, str] = ""
@@ -667,11 +667,7 @@ ColorTheme = Tuple[str, str, str]
 def _format_header(
     header: str, branch_name: str, branch_url: Optional[str] = None
 ) -> str:
-    # Following line does not lower CI->Ci and SQLancer->Sqlancer. It only
-    # capitalizes the first letter and doesn't touch the rest of the word
-    result = " ".join([w[0].upper() + w[1:] for w in header.split(" ") if w])
-    result = result.replace("Clickhouse", "ClickHouse")
-    result = result.replace("clickhouse", "ClickHouse")
+    result = header
     if "ClickHouse" not in result:
         result = f"ClickHouse {result}"
     if branch_url:

From 2c9b61d047c1afe22b0fa0a967a87db8bd4cf62f Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 2 Aug 2024 20:16:44 +0200
Subject: [PATCH 1284/1488] Miscellaneous

---
 tests/ci/ci.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/ci/ci.py b/tests/ci/ci.py
index 935fe472e50..6ca84a346e2 100644
--- a/tests/ci/ci.py
+++ b/tests/ci/ci.py
@@ -1019,7 +1019,7 @@ def _get_ext_check_name(check_name: str) -> str:
     return check_name_with_group
 
 
-def _cancel_pr_wf(s3: S3Helper, pr_number: int, cancel_sync: bool = False) -> None:
+def _cancel_pr_workflow(s3: S3Helper, pr_number: int, cancel_sync: bool = False) -> None:
     wf_data = CiMetadata(s3, pr_number).fetch_meta()
     if not cancel_sync:
         if not wf_data.run_id:
@@ -1368,12 +1368,12 @@ def main() -> int:
         assert indata, "Run config must be provided via --infile"
         _update_gh_statuses_action(indata=indata, s3=s3)
 
-    ### CANCEL PREVIOUS WORKFLOW RUN
+    ### CANCEL THE PREVIOUS WORKFLOW RUN
     elif args.cancel_previous_run:
         if pr_info.is_merge_queue:
-            _cancel_pr_wf(s3, pr_info.merged_pr)
+            _cancel_pr_workflow(s3, pr_info.merged_pr)
         elif pr_info.is_pr:
-            _cancel_pr_wf(s3, pr_info.number, cancel_sync=True)
+            _cancel_pr_workflow(s3, pr_info.number, cancel_sync=True)
         else:
             assert False, "BUG! Not supported scenario"
 

From bd3606dac4954c673ec6c38dd6fbdb70bc7b53cc Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 2 Aug 2024 20:17:07 +0200
Subject: [PATCH 1285/1488] Fix typos

---
 tests/ci/commit_status_helper.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/ci/commit_status_helper.py b/tests/ci/commit_status_helper.py
index fdc9c002b66..908ac4a7dca 100644
--- a/tests/ci/commit_status_helper.py
+++ b/tests/ci/commit_status_helper.py
@@ -301,7 +301,7 @@ def get_worst_state(statuses: CommitStatuses) -> StatusType:
 
 
 def create_ci_report(pr_info: PRInfo, statuses: CommitStatuses) -> str:
-    """The function converst the statuses to TestResults and uploads the report
+    """The function converts the statuses to TestResults and uploads the report
     to S3 tests bucket. Then it returns the URL"""
     test_results = []  # type: TestResults
     for status in statuses:

From 675afda17210ca7e8e71e0899a5ed14d7227fb55 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Fri, 2 Aug 2024 20:22:08 +0200
Subject: [PATCH 1286/1488] Fix check names in the CI Logs database

---
 tests/ci/clickhouse_helper.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tests/ci/clickhouse_helper.py b/tests/ci/clickhouse_helper.py
index 287970cce9a..0725f7100d1 100644
--- a/tests/ci/clickhouse_helper.py
+++ b/tests/ci/clickhouse_helper.py
@@ -3,6 +3,7 @@ import fileinput
 import json
 import logging
 import time
+import os
 from pathlib import Path
 from typing import Any, Dict, List, Optional
 
@@ -298,6 +299,11 @@ class CiLogsCredentials:
     def get_docker_arguments(
         self, pr_info: PRInfo, check_start_time: str, check_name: str
     ) -> str:
+        run_by_hash_total = int(os.getenv("RUN_BY_HASH_TOTAL", "0"))
+        if run_by_hash_total > 1:
+            run_by_hash_num = int(os.getenv("RUN_BY_HASH_NUM", "0"))
+            check_name = f"{check_name} [{run_by_hash_num + 1}/{run_by_hash_total}]"
+
         self.create_ci_logs_credentials()
         if not self.config_path.exists():
             logging.info("Do not use external logs pushing")

From b282be83c500bac5544424378b9505fc8c28e432 Mon Sep 17 00:00:00 2001
From: Max K <maxkaynov@gmail.com>
Date: Fri, 2 Aug 2024 20:03:43 +0200
Subject: [PATCH 1287/1488] remove old workflows

---
 .github/actions/release/action.yml   | 166 ---------------------------
 .github/workflows/auto_release.yml   | 111 ------------------
 .github/workflows/create_release.yml |   2 +
 .github/workflows/release.yml        |  71 ------------
 .github/workflows/tags_stable.yml    |  74 ------------
 tests/ci/docker_server.py            |   1 -
 6 files changed, 2 insertions(+), 423 deletions(-)
 delete mode 100644 .github/actions/release/action.yml
 delete mode 100644 .github/workflows/auto_release.yml
 delete mode 100644 .github/workflows/release.yml
 delete mode 100644 .github/workflows/tags_stable.yml

diff --git a/.github/actions/release/action.yml b/.github/actions/release/action.yml
deleted file mode 100644
index a287aa8b41d..00000000000
--- a/.github/actions/release/action.yml
+++ /dev/null
@@ -1,166 +0,0 @@
-name: Release
-
-description: Makes patch releases and creates new release branch
-
-inputs:
-  ref:
-    description: 'Git reference (branch or commit sha) from which to create the release'
-    required: true
-    type: string
-  type:
-    description: 'The type of release: "new" for a new release or "patch" for a patch release'
-    required: true
-    type: choice
-    options:
-      - patch
-      - new
-  dry-run:
-    description: 'Dry run'
-    required: true
-    type: boolean
-  token:
-    required: true
-    type: string
-
-runs:
-  using: "composite"
-  steps:
-    - name: Prepare Release Info
-      shell: bash
-      run: |
-        python3 ./tests/ci/create_release.py --prepare-release-info \
-          --ref ${{ inputs.ref }} --release-type ${{ inputs.type }} ${{ inputs.dry-run == true && '--dry-run' || '' }}
-        echo "::group::Release Info"
-        python3 -m json.tool /tmp/release_info.json
-        echo "::endgroup::"
-        release_tag=$(jq -r '.release_tag' /tmp/release_info.json)
-        commit_sha=$(jq -r '.commit_sha' /tmp/release_info.json)
-        echo "Release Tag: $release_tag"
-        echo "RELEASE_TAG=$release_tag" >> "$GITHUB_ENV"
-        echo "COMMIT_SHA=$commit_sha" >> "$GITHUB_ENV"
-    - name: Download All Release Artifacts
-      if: ${{ inputs.type == 'patch' }}
-      shell: bash
-      run: |
-        python3 ./tests/ci/create_release.py --download-packages ${{ inputs.dry-run == true && '--dry-run' || '' }}
-    - name: Push Git Tag for the Release
-      shell: bash
-      run: |
-        python3 ./tests/ci/create_release.py --push-release-tag ${{ inputs.dry-run == true && '--dry-run' || '' }}
-    - name: Push New Release Branch
-      if: ${{ inputs.type == 'new' }}
-      shell: bash
-      run: |
-        python3 ./tests/ci/create_release.py --push-new-release-branch ${{ inputs.dry-run == true && '--dry-run' || '' }}
-    - name: Bump CH Version and Update Contributors' List
-      shell: bash
-      run: |
-        python3 ./tests/ci/create_release.py --create-bump-version-pr ${{ inputs.dry-run == true && '--dry-run' || '' }}
-    - name: Bump Docker versions, Changelog, Security
-      if: ${{ inputs.type == 'patch' }}
-      shell: bash
-      run: |
-        git checkout master
-        python3 ./tests/ci/create_release.py --set-progress-started --progress "update changelog, docker version, security"
-        echo "List versions"
-        ./utils/list-versions/list-versions.sh > ./utils/list-versions/version_date.tsv
-        echo "Update docker version"
-        ./utils/list-versions/update-docker-version.sh
-        echo "Generate ChangeLog"
-        export CI=1
-        docker run -u "${UID}:${GID}" -e PYTHONUNBUFFERED=1 -e CI=1 --network=host \
-          --volume=".:/ClickHouse" clickhouse/style-test \
-          /ClickHouse/tests/ci/changelog.py -v --debug-helpers \
-          --gh-user-or-token=${{ inputs.token }} --jobs=5 \
-          --output="/ClickHouse/docs/changelogs/${{ env.RELEASE_TAG }}.md" ${{ env.RELEASE_TAG }}
-        git add ./docs/changelogs/${{ env.RELEASE_TAG }}.md
-        echo "Generate Security"
-        python3 ./utils/security-generator/generate_security.py > SECURITY.md
-        git diff HEAD
-    - name: Create ChangeLog PR
-      if: ${{ inputs.type == 'patch' && ! inputs.dry-run }}
-      uses: peter-evans/create-pull-request@v6
-      with:
-        author: "robot-clickhouse <robot-clickhouse@users.noreply.github.com>"
-        token: ${{ inputs.token }}
-        committer: "robot-clickhouse <robot-clickhouse@users.noreply.github.com>"
-        commit-message: Update version_date.tsv and changelogs after ${{ env.RELEASE_TAG }}
-        branch: auto/${{ env.RELEASE_TAG }}
-        assignees: ${{ github.event.sender.login }}  # assign the PR to the tag pusher
-        delete-branch: true
-        title: Update version_date.tsv and changelog after ${{ env.RELEASE_TAG }}
-        labels: do not test
-        body: |
-          Update version_date.tsv and changelogs after ${{ env.RELEASE_TAG }}
-          ### Changelog category (leave one):
-          - Not for changelog (changelog entry is not required)
-    - name: Complete previous steps and Restore git state
-      if: ${{ inputs.type == 'patch' }}
-      shell: bash
-      run: |
-        python3 ./tests/ci/create_release.py --set-progress-completed
-        git reset --hard HEAD
-        git checkout "$GITHUB_REF_NAME"
-    - name: Create GH Release
-      shell: bash
-      if: ${{ inputs.type == 'patch' }}
-      run: |
-        python3 ./tests/ci/create_release.py --create-gh-release ${{ inputs.dry-run == true && '--dry-run' || '' }}
-    - name: Export TGZ Packages
-      if: ${{ inputs.type == 'patch' }}
-      shell: bash
-      run: |
-        python3 ./tests/ci/artifactory.py --export-tgz ${{ inputs.dry-run == true && '--dry-run' || '' }}
-    - name: Test TGZ Packages
-      if: ${{ inputs.type == 'patch' }}
-      shell: bash
-      run: |
-        python3 ./tests/ci/artifactory.py --test-tgz ${{ inputs.dry-run == true && '--dry-run' || '' }}
-    - name: Export RPM Packages
-      if: ${{ inputs.type == 'patch' }}
-      shell: bash
-      run: |
-        python3 ./tests/ci/artifactory.py --export-rpm ${{ inputs.dry-run == true && '--dry-run' || '' }}
-    - name: Test RPM Packages
-      if: ${{ inputs.type == 'patch' }}
-      shell: bash
-      run: |
-        python3 ./tests/ci/artifactory.py --test-rpm ${{ inputs.dry-run == true && '--dry-run' || '' }}
-    - name: Export Debian Packages
-      if: ${{ inputs.type == 'patch' }}
-      shell: bash
-      run: |
-        python3 ./tests/ci/artifactory.py --export-debian ${{ inputs.dry-run == true && '--dry-run' || '' }}
-    - name: Test Debian Packages
-      if: ${{ inputs.type == 'patch' }}
-      shell: bash
-      run: |
-        python3 ./tests/ci/artifactory.py --test-debian ${{ inputs.dry-run == true && '--dry-run' || '' }}
-    - name: Docker clickhouse/clickhouse-server building
-      if: ${{ inputs.type == 'patch' }}
-      shell: bash
-      run: |
-        cd "./tests/ci"
-        python3 ./create_release.py --set-progress-started --progress "docker server release"
-        export CHECK_NAME="Docker server image"
-        python3 docker_server.py --release-type auto --version ${{ env.RELEASE_TAG }} --check-name "$CHECK_NAME" --sha ${{ env.COMMIT_SHA }} ${{ ! inputs.dry-run && '--push' || '' }}
-        python3 ./create_release.py --set-progress-completed
-    - name: Docker clickhouse/clickhouse-keeper building
-      if: ${{ inputs.type == 'patch' }}
-      shell: bash
-      run: |
-        cd "./tests/ci"
-        python3 ./create_release.py --set-progress-started --progress "docker keeper release"
-        export CHECK_NAME="Docker keeper image"
-        python3 docker_server.py --release-type auto --version ${{ env.RELEASE_TAG }} --check-name "$CHECK_NAME" --sha ${{ env.COMMIT_SHA }} ${{ ! inputs.dry-run && '--push'  || '' }}
-        python3 ./create_release.py --set-progress-completed
-    - name: Set current Release progress to Completed with OK
-      shell: bash
-      run: |
-        python3 ./tests/ci/create_release.py --set-progress-started --progress "completed"
-        python3 ./tests/ci/create_release.py --set-progress-completed
-    - name: Post Slack Message
-      if: ${{ !cancelled() }}
-      shell: bash
-      run: |
-        python3 ./tests/ci/create_release.py --post-status ${{ inputs.dry-run == true && '--dry-run' || '' }}
diff --git a/.github/workflows/auto_release.yml b/.github/workflows/auto_release.yml
deleted file mode 100644
index 457ffacc7a8..00000000000
--- a/.github/workflows/auto_release.yml
+++ /dev/null
@@ -1,111 +0,0 @@
-name: AutoRelease
-
-env:
-  PYTHONUNBUFFERED: 1
-  DRY_RUN: true
-
-concurrency:
-  group: release
-on: # yamllint disable-line rule:truthy
-  # Workflow uses a test bucket for packages and dry run mode (no real releases)
-  schedule:
-    - cron: '0 9 * * *'
-    - cron: '0 15 * * *'
-  workflow_dispatch:
-    inputs:
-      dry-run:
-        description: 'Dry run'
-        required: false
-        default: true
-        type: boolean
-
-jobs:
-  AutoRelease:
-    runs-on: [self-hosted, release-maker]
-    steps:
-      - name: DebugInfo
-        uses: hmarr/debug-action@f7318c783045ac39ed9bb497e22ce835fdafbfe6
-      - name: Set envs
-        run: |
-          cat >> "$GITHUB_ENV" << 'EOF'
-          ROBOT_CLICKHOUSE_SSH_KEY<<RCSK
-          ${{secrets.ROBOT_CLICKHOUSE_SSH_KEY}}
-          RCSK
-          EOF
-      - name: Set DRY_RUN for schedule
-        if: ${{ github.event_name == 'schedule' }}
-        run: echo "DRY_RUN=true" >> "$GITHUB_ENV"
-      - name: Set DRY_RUN for dispatch
-        if: ${{ github.event_name == 'workflow_dispatch' }}
-        run: echo "DRY_RUN=${{ github.event.inputs.dry-run }}" >> "$GITHUB_ENV"
-      - name: Check out repository code
-        uses: ClickHouse/checkout@v1
-        with:
-          token: ${{secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN}}
-          fetch-depth: 0
-      - name: Auto Release Prepare
-        run: |
-          cd "$GITHUB_WORKSPACE/tests/ci"
-          python3 auto_release.py --prepare
-          echo "::group::Auto Release Info"
-          python3 -m json.tool /tmp/autorelease_info.json
-          echo "::endgroup::"
-          {
-              echo 'AUTO_RELEASE_PARAMS<<EOF'
-              cat  /tmp/autorelease_info.json
-              echo 'EOF'
-          } >> "$GITHUB_ENV"
-      - name: Post Release Branch statuses
-        run: |
-          cd "$GITHUB_WORKSPACE/tests/ci"
-          python3 auto_release.py --post-status
-      - name: Release ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[0].release_branch }}
-        if: ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[0] && fromJson(env.AUTO_RELEASE_PARAMS).releases[0].ready }}
-        uses: ./.github/actions/release
-        with:
-          ref: ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[0].commit_sha }}
-          type: patch
-          dry-run: ${{ env.DRY_RUN }}
-          token: ${{secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN}}
-      - name: Release ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[1].release_branch }}
-        if: ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[0] && fromJson(env.AUTO_RELEASE_PARAMS).releases[1].ready }}
-        uses: ./.github/actions/release
-        with:
-          ref: ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[1].commit_sha }}
-          type: patch
-          dry-run: ${{ env.DRY_RUN }}
-          token: ${{secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN}}
-      - name: Release ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[2].release_branch }}
-        if: ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[2] && fromJson(env.AUTO_RELEASE_PARAMS).releases[2].ready }}
-        uses: ./.github/actions/release
-        with:
-          ref: ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[2].commit_sha }}
-          type: patch
-          dry-run: ${{ env.DRY_RUN }}
-          token: ${{secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN}}
-      - name: Release ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[3].release_branch }}
-        if: ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[3] && fromJson(env.AUTO_RELEASE_PARAMS).releases[3].ready }}
-        uses: ./.github/actions/release
-        with:
-          ref: ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[3].commit_sha }}
-          type: patch
-          dry-run: ${{ env.DRY_RUN }}
-          token: ${{secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN}}
-      - name: Release ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[4].release_branch }}
-        if: ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[4] && fromJson(env.AUTO_RELEASE_PARAMS).releases[4].ready }}
-        uses: ./.github/actions/release
-        with:
-          ref: ${{ fromJson(env.AUTO_RELEASE_PARAMS).releases[4].commit_sha }}
-          type: patch
-          dry-run: ${{ env.DRY_RUN }}
-          token: ${{secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN}}
-      - name: Post Slack Message
-        if: ${{ !cancelled() }}
-        run: |
-          cd "$GITHUB_WORKSPACE/tests/ci"
-          python3 auto_release.py --post-auto-release-complete --wf-status ${{ job.status }}
-      - name: Clean up
-        run: |
-          docker ps --quiet | xargs --no-run-if-empty docker kill ||:
-          docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
-          sudo rm -fr "$TEMP_PATH"
diff --git a/.github/workflows/create_release.yml b/.github/workflows/create_release.yml
index 29094cc51a6..d4993b373df 100644
--- a/.github/workflows/create_release.yml
+++ b/.github/workflows/create_release.yml
@@ -89,6 +89,8 @@ jobs:
         shell: bash
         run: |
           python3 ./tests/ci/create_release.py --set-progress-started --progress "update changelog, docker version, security"
+
+          git checkout master  # in case WF started from feature branch
           echo "List versions"
           ./utils/list-versions/list-versions.sh > ./utils/list-versions/version_date.tsv
           echo "Update docker version"
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
deleted file mode 100644
index 7dc4e3298a6..00000000000
--- a/.github/workflows/release.yml
+++ /dev/null
@@ -1,71 +0,0 @@
-name: PublishedReleaseCI
-# - Gets artifacts from S3
-# - Sends it to JFROG Artifactory
-# - Adds them to the release assets
-
-on: # yamllint disable-line rule:truthy
-  release:
-    types:
-    - published
-  workflow_dispatch:
-    inputs:
-      tag:
-        description: 'Release tag'
-        required: true
-        type: string
-
-jobs:
-  ReleasePublish:
-    runs-on: [self-hosted, style-checker]
-    steps:
-    - name: Set tag from input
-      if: github.event_name == 'workflow_dispatch'
-      run: |
-        echo "GITHUB_TAG=${{ github.event.inputs.tag }}" >> "$GITHUB_ENV"
-    - name: Set tag from REF
-      if: github.event_name == 'release'
-      run: |
-        echo "GITHUB_TAG=${GITHUB_REF#refs/tags/}" >> "$GITHUB_ENV"
-    - name: Deploy packages and assets
-      run: |
-        curl --silent --data '' --no-buffer \
-          '${{ secrets.PACKAGES_RELEASE_URL }}/release/'"${GITHUB_TAG}"'?binary=binary_darwin&binary=binary_darwin_aarch64&sync=true'
-  ############################################################################################
-  ##################################### Docker images  #######################################
-  ############################################################################################
-  DockerServerImages:
-    runs-on: [self-hosted, style-checker]
-    steps:
-    - name: Set tag from input
-      if: github.event_name == 'workflow_dispatch'
-      run: |
-        echo "GITHUB_TAG=${{ github.event.inputs.tag }}" >> "$GITHUB_ENV"
-    - name: Set tag from REF
-      if: github.event_name == 'release'
-      run: |
-        echo "GITHUB_TAG=${GITHUB_REF#refs/tags/}" >> "$GITHUB_ENV"
-    - name: Check out repository code
-      uses: ClickHouse/checkout@v1
-      with:
-        clear-repository: true
-        fetch-depth: 0  # otherwise we will have no version info
-        filter: tree:0
-        ref: ${{ env.GITHUB_TAG }}
-    - name: Check docker clickhouse/clickhouse-server building
-      run: |
-        cd "$GITHUB_WORKSPACE/tests/ci"
-        export CHECK_NAME="Docker server image"
-        SHA=$(git rev-list -n 1 "$GITHUB_TAG")
-        python3 docker_server.py --release-type auto --version "$GITHUB_TAG" --sha "$SHA"  --check-name "$CHECK_NAME" --push
-    - name: Check docker clickhouse/clickhouse-keeper building
-      run: |
-        cd "$GITHUB_WORKSPACE/tests/ci"
-        export CHECK_NAME="Docker keeper image"
-        SHA=$(git rev-list -n 1 "$GITHUB_TAG")
-        python3 docker_server.py --release-type auto --version "$GITHUB_TAG" --sha "$SHA" --check-name "$CHECK_NAME" --push
-    - name: Cleanup
-      if: always()
-      run: |
-        docker ps --quiet | xargs --no-run-if-empty docker kill ||:
-        docker ps --all --quiet | xargs --no-run-if-empty docker rm -f ||:
-        sudo rm -fr "$TEMP_PATH"
diff --git a/.github/workflows/tags_stable.yml b/.github/workflows/tags_stable.yml
deleted file mode 100644
index 2aa7694bc41..00000000000
--- a/.github/workflows/tags_stable.yml
+++ /dev/null
@@ -1,74 +0,0 @@
-name: TagsStableWorkflow
-# - Gets artifacts from S3
-# - Sends it to JFROG Artifactory
-# - Adds them to the release assets
-
-env:
-  # Force the stdout and stderr streams to be unbuffered
-  PYTHONUNBUFFERED: 1
-
-on: # yamllint disable-line rule:truthy
-  push:
-    tags:
-    - 'v*-prestable'
-    - 'v*-stable'
-    - 'v*-lts'
-  workflow_dispatch:
-    inputs:
-      tag:
-        description: 'Test tag'
-        required: true
-        type: string
-
-
-jobs:
-  UpdateVersions:
-    runs-on: [self-hosted, style-checker]
-    steps:
-    - name: Set test tag
-      if: github.event_name == 'workflow_dispatch'
-      run: |
-        echo "GITHUB_TAG=${{ github.event.inputs.tag }}" >> "$GITHUB_ENV"
-    - name: Get tag name
-      if: github.event_name != 'workflow_dispatch'
-      run: |
-        echo "GITHUB_TAG=${GITHUB_REF#refs/tags/}" >> "$GITHUB_ENV"
-    - name: Check out repository code
-      uses: ClickHouse/checkout@v1
-      with:
-        ref: master
-        fetch-depth: 0
-        filter: tree:0
-    - name: Update versions, docker version, changelog, security
-      env:
-        GITHUB_TOKEN: ${{ secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN }}
-      run: |
-        ./utils/list-versions/list-versions.sh > ./utils/list-versions/version_date.tsv
-        ./utils/list-versions/update-docker-version.sh
-        GID=$(id -g "${UID}")
-        # --network=host and CI=1 are required for the S3 access from a container
-        docker run -u "${UID}:${GID}" -e PYTHONUNBUFFERED=1 -e CI=1 --network=host \
-            --volume="${GITHUB_WORKSPACE}:/ClickHouse" clickhouse/style-test \
-                /ClickHouse/tests/ci/changelog.py -v --debug-helpers \
-                --gh-user-or-token="$GITHUB_TOKEN" --jobs=5 \
-                --output="/ClickHouse/docs/changelogs/${GITHUB_TAG}.md" "${GITHUB_TAG}"
-        git add "./docs/changelogs/${GITHUB_TAG}.md"
-        python3 ./utils/security-generator/generate_security.py > SECURITY.md
-        git diff HEAD
-    - name: Create Pull Request
-      uses: peter-evans/create-pull-request@v6
-      with:
-        author: "robot-clickhouse <robot-clickhouse@users.noreply.github.com>"
-        token: ${{ secrets.ROBOT_CLICKHOUSE_COMMIT_TOKEN }}
-        committer: "robot-clickhouse <robot-clickhouse@users.noreply.github.com>"
-        commit-message: Update version_date.tsv and changelogs after ${{ env.GITHUB_TAG }}
-        branch: auto/${{ env.GITHUB_TAG }}
-        assignees: ${{ github.event.sender.login }}  # assign the PR to the tag pusher
-        delete-branch: true
-        title: Update version_date.tsv and changelogs after ${{ env.GITHUB_TAG }}
-        labels: do not test
-        body: |
-          Update version_date.tsv and changelogs after ${{ env.GITHUB_TAG }}
-
-          ### Changelog category (leave one):
-          - Not for changelog (changelog entry is not required)
diff --git a/tests/ci/docker_server.py b/tests/ci/docker_server.py
index 8f0474d5053..3251ec5644e 100644
--- a/tests/ci/docker_server.py
+++ b/tests/ci/docker_server.py
@@ -27,7 +27,6 @@ from stopwatch import Stopwatch
 from tee_popen import TeePopen
 from version_helper import (
     ClickHouseVersion,
-    get_tagged_versions,
     get_version_from_repo,
     version_arg,
 )

From aa38e78d7238d843737d1d268de6ee189c19edc3 Mon Sep 17 00:00:00 2001
From: Max K <maxkaynov@gmail.com>
Date: Fri, 2 Aug 2024 20:27:59 +0200
Subject: [PATCH 1288/1488] update version_date.tsv

---
 docs/changelogs/v23.8.16.40-lts.md   | 35 ++++++++++++++++++++++++++++
 utils/list-versions/version_date.tsv |  1 +
 2 files changed, 36 insertions(+)
 create mode 100644 docs/changelogs/v23.8.16.40-lts.md

diff --git a/docs/changelogs/v23.8.16.40-lts.md b/docs/changelogs/v23.8.16.40-lts.md
new file mode 100644
index 00000000000..75caf1ea277
--- /dev/null
+++ b/docs/changelogs/v23.8.16.40-lts.md
@@ -0,0 +1,35 @@
+---
+sidebar_position: 1
+sidebar_label: 2024
+---
+
+# 2024 Changelog
+
+### ClickHouse release v23.8.16.40-lts (e143a9039ba) FIXME as compared to v23.8.15.35-lts (060ff8e813a)
+
+#### Improvement
+* Backported in [#66962](https://github.com/ClickHouse/ClickHouse/issues/66962): Added support for parameterized view with analyzer to not analyze create parameterized view. Refactor existing parameterized view logic to not analyze create parameterized view. [#54211](https://github.com/ClickHouse/ClickHouse/pull/54211) ([SmitaRKulkarni](https://github.com/SmitaRKulkarni)).
+* Backported in [#65461](https://github.com/ClickHouse/ClickHouse/issues/65461): Reload certificate chain during certificate reload. [#61671](https://github.com/ClickHouse/ClickHouse/pull/61671) ([Pervakov Grigorii](https://github.com/GrigoryPervakov)).
+* Backported in [#65880](https://github.com/ClickHouse/ClickHouse/issues/65880): Always start Keeper with sufficient amount of threads in global thread pool. [#64444](https://github.com/ClickHouse/ClickHouse/pull/64444) ([Duc Canh Le](https://github.com/canhld94)).
+* Backported in [#65912](https://github.com/ClickHouse/ClickHouse/issues/65912): Respect cgroup CPU limit in Keeper. [#65819](https://github.com/ClickHouse/ClickHouse/pull/65819) ([Antonio Andelic](https://github.com/antonio2368)).
+
+#### Critical Bug Fix (crash, LOGICAL_ERROR, data loss, RBAC)
+* Backported in [#65281](https://github.com/ClickHouse/ClickHouse/issues/65281): Fix crash with UniqInjectiveFunctionsEliminationPass and uniqCombined. [#65188](https://github.com/ClickHouse/ClickHouse/pull/65188) ([Raúl Marín](https://github.com/Algunenano)).
+* Backported in [#65368](https://github.com/ClickHouse/ClickHouse/issues/65368): Fix a bug in ClickHouse Keeper that causes digest mismatch during closing session. [#65198](https://github.com/ClickHouse/ClickHouse/pull/65198) ([Aleksei Filatov](https://github.com/aalexfvk)).
+* Backported in [#65743](https://github.com/ClickHouse/ClickHouse/issues/65743): Fix crash in maxIntersections. [#65689](https://github.com/ClickHouse/ClickHouse/pull/65689) ([Raúl Marín](https://github.com/Algunenano)).
+
+#### Bug Fix (user-visible misbehavior in an official stable release)
+* Backported in [#65351](https://github.com/ClickHouse/ClickHouse/issues/65351): Fix possible abort on uncaught exception in ~WriteBufferFromFileDescriptor in StatusFile. [#64206](https://github.com/ClickHouse/ClickHouse/pull/64206) ([Kruglov Pavel](https://github.com/Avogar)).
+* Backported in [#66037](https://github.com/ClickHouse/ClickHouse/issues/66037): Fix crash on destroying AccessControl: add explicit shutdown. [#64993](https://github.com/ClickHouse/ClickHouse/pull/64993) ([Vitaly Baranov](https://github.com/vitlibar)).
+* Backported in [#65782](https://github.com/ClickHouse/ClickHouse/issues/65782): Fixed bug in MergeJoin. Column in sparse serialisation might be treated as a column of its nested type though the required conversion wasn't performed. [#65632](https://github.com/ClickHouse/ClickHouse/pull/65632) ([Nikita Taranov](https://github.com/nickitat)).
+* Backported in [#65926](https://github.com/ClickHouse/ClickHouse/issues/65926): For queries that read from `PostgreSQL`, cancel the internal `PostgreSQL` query if the ClickHouse query is finished. Otherwise, `ClickHouse` query cannot be canceled until the internal `PostgreSQL` query is finished. [#65771](https://github.com/ClickHouse/ClickHouse/pull/65771) ([Maksim Kita](https://github.com/kitaisreal)).
+* Backported in [#65822](https://github.com/ClickHouse/ClickHouse/issues/65822): Fix a bug in short circuit logic when old analyzer and dictGetOrDefault is used. [#65802](https://github.com/ClickHouse/ClickHouse/pull/65802) ([jsc0218](https://github.com/jsc0218)).
+* Backported in [#66449](https://github.com/ClickHouse/ClickHouse/issues/66449): Fixed a bug in ZooKeeper client: a session could get stuck in unusable state after receiving a hardware error from ZooKeeper. For example, this might happen due to "soft memory limit" in ClickHouse Keeper. [#66140](https://github.com/ClickHouse/ClickHouse/pull/66140) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Backported in [#66717](https://github.com/ClickHouse/ClickHouse/issues/66717): Correctly track memory for `Allocator::realloc`. [#66548](https://github.com/ClickHouse/ClickHouse/pull/66548) ([Antonio Andelic](https://github.com/antonio2368)).
+
+#### NOT FOR CHANGELOG / INSIGNIFICANT
+
+* Backported in [#65080](https://github.com/ClickHouse/ClickHouse/issues/65080): Follow up to [#56541](https://github.com/ClickHouse/ClickHouse/issues/56541). [#57141](https://github.com/ClickHouse/ClickHouse/pull/57141) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Backported in [#65913](https://github.com/ClickHouse/ClickHouse/issues/65913): Fix bug with session closing in Keeper. [#65735](https://github.com/ClickHouse/ClickHouse/pull/65735) ([Antonio Andelic](https://github.com/antonio2368)).
+* Backported in [#66853](https://github.com/ClickHouse/ClickHouse/issues/66853): Fix data race in S3::ClientCache. [#66644](https://github.com/ClickHouse/ClickHouse/pull/66644) ([Konstantin Morozov](https://github.com/k-morozov)).
+
diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv
index 24488066190..cb6b8f588da 100644
--- a/utils/list-versions/version_date.tsv
+++ b/utils/list-versions/version_date.tsv
@@ -50,6 +50,7 @@ v23.9.4.11-stable	2023-11-08
 v23.9.3.12-stable	2023-10-31
 v23.9.2.56-stable	2023-10-19
 v23.9.1.1854-stable	2023-09-29
+v23.8.16.40-lts	2024-08-02
 v23.8.15.35-lts	2024-06-14
 v23.8.14.6-lts	2024-05-02
 v23.8.13.25-lts	2024-04-26

From a45ba44dbaa2ed43eb63e49fe609a01be978eac9 Mon Sep 17 00:00:00 2001
From: robot-clickhouse <robot-clickhouse@users.noreply.github.com>
Date: Fri, 2 Aug 2024 18:28:38 +0000
Subject: [PATCH 1289/1488] Automatic style fix

---
 tests/ci/ci.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/ci/ci.py b/tests/ci/ci.py
index 6ca84a346e2..805296d2bb2 100644
--- a/tests/ci/ci.py
+++ b/tests/ci/ci.py
@@ -1019,7 +1019,9 @@ def _get_ext_check_name(check_name: str) -> str:
     return check_name_with_group
 
 
-def _cancel_pr_workflow(s3: S3Helper, pr_number: int, cancel_sync: bool = False) -> None:
+def _cancel_pr_workflow(
+    s3: S3Helper, pr_number: int, cancel_sync: bool = False
+) -> None:
     wf_data = CiMetadata(s3, pr_number).fetch_meta()
     if not cancel_sync:
         if not wf_data.run_id:

From a37eeb0f211c0c7b6251a9108d6b939e73c9a66e Mon Sep 17 00:00:00 2001
From: Michael Kolupaev <michael.kolupaev@clickhouse.com>
Date: Fri, 2 Aug 2024 20:13:27 +0000
Subject: [PATCH 1290/1488] Allow types to change, re-resolve overloads

---
 src/Storages/MergeTree/KeyCondition.cpp | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)

diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp
index eaf9f0af623..91f054c3a71 100644
--- a/src/Storages/MergeTree/KeyCondition.cpp
+++ b/src/Storages/MergeTree/KeyCondition.cpp
@@ -627,7 +627,8 @@ static const ActionsDAG::Node & cloneASTWithInversionPushDown(
                     }
                 }
 
-                res = &inverted_dag.addFunction(node.function_base, children, "");
+                auto function_builder = FunctionFactory::instance().get(name, context);
+                res = &inverted_dag.addFunction(function_builder, children, "");
                 handled_inversion = true;
             }
             else if (need_inversion && (name == "and" || name == "or"))
@@ -668,8 +669,13 @@ static const ActionsDAG::Node & cloneASTWithInversionPushDown(
                 }
                 else
                 {
-                    res = &inverted_dag.addFunction(node.function_base, children, "");
-                    chassert(res->result_type == node.result_type);
+                    /// Can't just addFunction(node.function_base) because argument types may have
+                    /// changed slightly because of our transformations, e.g. maybe some subexpression
+                    /// changed constness, which caused some function return value to change LowCardinality-ness.
+                    /// (I don't have a specific counterexample, but it seems likely that it exists.
+                    ///  One was fixed in the past: https://github.com/ClickHouse/ClickHouse/issues/65143 )
+                    auto function_builder = FunctionFactory::instance().get(name, context);
+                    res = &inverted_dag.addFunction(function_builder, children, "");
                 }
             }
         }
@@ -678,13 +684,6 @@ static const ActionsDAG::Node & cloneASTWithInversionPushDown(
     if (!handled_inversion && need_inversion)
         res = &inverted_dag.addFunction(FunctionFactory::instance().get("not", context), {res}, "");
 
-    /// Make sure we don't change any data types (e.g. remove LowCardinality).
-    /// If it turns out that we actually want to change data types sometimes, it's ok to remove this
-    /// check *and* replace all `addFunction(node.function_base, ...)` calls above with
-    /// `addFunction(FunctionFactory::instance().get(name, context), ...)` to re-resolve overloads.
-    if (!node.result_type->equals(*res->result_type))
-        throw Exception(ErrorCodes::LOGICAL_ERROR, "KeyCondition inadvertently changed subexpression data type: '{}' -> '{}', column `{}`", node.result_type->getName(), res->result_type->getName(), node.result_name);
-
     to_inverted[&node] = res;
     return *res;
 }

From eb0e12099a8fa45d79b2cff96e02ee273e879efa Mon Sep 17 00:00:00 2001
From: Michael Kolupaev <michael.kolupaev@clickhouse.com>
Date: Fri, 2 Aug 2024 21:18:35 +0000
Subject: [PATCH 1291/1488] Another attempt

---
 src/Storages/MergeTree/KeyCondition.cpp | 27 ++++++++++++++++---------
 1 file changed, 18 insertions(+), 9 deletions(-)

diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp
index 91f054c3a71..9115cb5608e 100644
--- a/src/Storages/MergeTree/KeyCondition.cpp
+++ b/src/Storages/MergeTree/KeyCondition.cpp
@@ -627,8 +627,7 @@ static const ActionsDAG::Node & cloneASTWithInversionPushDown(
                     }
                 }
 
-                auto function_builder = FunctionFactory::instance().get(name, context);
-                res = &inverted_dag.addFunction(function_builder, children, "");
+                res = &inverted_dag.addFunction(node.function_base, children, "");
                 handled_inversion = true;
             }
             else if (need_inversion && (name == "and" || name == "or"))
@@ -669,13 +668,23 @@ static const ActionsDAG::Node & cloneASTWithInversionPushDown(
                 }
                 else
                 {
-                    /// Can't just addFunction(node.function_base) because argument types may have
-                    /// changed slightly because of our transformations, e.g. maybe some subexpression
-                    /// changed constness, which caused some function return value to change LowCardinality-ness.
-                    /// (I don't have a specific counterexample, but it seems likely that it exists.
-                    ///  One was fixed in the past: https://github.com/ClickHouse/ClickHouse/issues/65143 )
-                    auto function_builder = FunctionFactory::instance().get(name, context);
-                    res = &inverted_dag.addFunction(function_builder, children, "");
+                    /// Make sure we don't change types of function arguments (e.g. remove LowCardinality).
+                    /// Otherwise the function may crash when passed columns of unexpected types.
+                    ///  * Why not check this for all subexperessions rather than function arguments?
+                    ///    Because types may change, e.g. in `NOT (u64 AND u64)` -> `(NOT u64 OR NOT u64)`
+                    ///    the AND's args were UInt64, but OR's args are UInt8.
+                    ///  * Why not re-resolve function overload, using FunctionFactory::instance().get(name, context)?
+                    ///    Because some functions can't be found through FunctionFactory, e.g. FunctionCapture.
+                    ///    (But maybe we could re-resolve only if argument types changed.)
+                    for (size_t i = 0; i < children.size(); ++i)
+                    {
+                        if (!node.children[i]->result_type->equals(*children[i]->result_type))
+                            throw Exception(
+                                ErrorCodes::LOGICAL_ERROR, "KeyCondition inadvertently changed subexpression data type: '{}' -> '{}', column `{}`",
+                                node.children[i]->result_type->getName(), children[i]->result_type->getName(), node.children[i]->result_name);
+                    }
+
+                    res = &inverted_dag.addFunction(node.function_base, children, "");
                 }
             }
         }

From dd0ae04f90314ce6d5dbe748605e66f1a6d9024f Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 31 Jul 2024 17:38:20 +0200
Subject: [PATCH 1292/1488] Minor change

---
 src/Databases/DatabaseOnDisk.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp
index f419f5811a1..734f354d9a5 100644
--- a/src/Databases/DatabaseOnDisk.cpp
+++ b/src/Databases/DatabaseOnDisk.cpp
@@ -313,7 +313,7 @@ void DatabaseOnDisk::detachTablePermanently(ContextPtr query_context, const Stri
         std::lock_guard lock(mutex);
         if (const auto it = snapshot_detached_tables.find(table_name); it == snapshot_detached_tables.end())
         {
-            throw Exception(ErrorCodes::LOGICAL_ERROR, "Snapshot doesn't contain info about detached table={}", table_name);
+            throw Exception(ErrorCodes::LOGICAL_ERROR, "Snapshot doesn't contain info about detached table `{}`", table_name);
         }
         else
         {

From a431ab3e4b6f925924a81d99997e6c028ae7950f Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 3 Aug 2024 01:31:02 +0200
Subject: [PATCH 1293/1488] Improve dashboard

---
 programs/server/dashboard.html | 66 +++++++++++++++++++++++++++-------
 1 file changed, 54 insertions(+), 12 deletions(-)

diff --git a/programs/server/dashboard.html b/programs/server/dashboard.html
index 45f988f7b1e..71880b9e228 100644
--- a/programs/server/dashboard.html
+++ b/programs/server/dashboard.html
@@ -17,7 +17,7 @@
             --input-shadow-color: rgba(0, 255, 0, 1);
             --error-color: red;
             --global-error-color: white;
-            --legend-background: rgba(255, 255, 255, 0.75);
+            --legend-background: rgba(255, 255, 0, 0.75);
             --title-color: #666;
             --text-color: black;
             --edit-title-background: #FEE;
@@ -41,7 +41,7 @@
             --moving-shadow-color: rgba(255, 255, 255, 0.25);
             --input-shadow-color: rgba(255, 128, 0, 0.25);
             --error-color: #F66;
-            --legend-background: rgba(255, 255, 255, 0.25);
+            --legend-background: rgba(0, 96, 128, 0.75);
             --title-color: white;
             --text-color: white;
             --edit-title-background: #364f69;
@@ -1004,14 +1004,14 @@ function legendAsTooltipPlugin({ className, style = { background: "var(--legend-
         className && legendEl.classList.add(className);
 
         uPlot.assign(legendEl.style, {
-            textAlign: "left",
+            textAlign: "right",
             pointerEvents: "none",
             display: "none",
             position: "absolute",
             left: 0,
             top: 0,
-            zIndex: 100,
-            boxShadow: "2px 2px 10px rgba(0,0,0,0.1)",
+            zIndex: 200,
+            boxShadow: "2px 2px 10px rgba(0, 0, 0, 0.1)",
             ...style
         });
 
@@ -1051,8 +1051,10 @@ function legendAsTooltipPlugin({ className, style = { background: "var(--legend-
 
     function update(u) {
         let { left, top } = u.cursor;
-        left -= legendEl.clientWidth / 2;
-        top -= legendEl.clientHeight / 2;
+        /// This will make the balloon to the right of the cursor when the cursor is on the left side, and vise-versa,
+        /// avoiding the borders of the chart.
+        left -= legendEl.clientWidth * (left / u.width);
+        top -= legendEl.clientHeight;
         legendEl.style.transform = "translate(" + left + "px, " + top + "px)";
 
         if (multiline) {
@@ -1229,14 +1231,53 @@ async function draw(idx, chart, url_params, query) {
 
     let sync = uPlot.sync("sync");
 
-    let axis = {
+    function formatDateTime(t) {
+        return (new Date(t * 1000)).toISOString().replace('T', '\n').replace('.000Z', '');
+    }
+
+    function formatDateTimes(self, ticks) {
+        return ticks.map((t, idx) => {
+            let res = formatDateTime(t);
+            if (idx == 0 || res.substring(0, 10) != formatDateTime(ticks[idx - 1]).substring(0, 10)) {
+                return res;
+            } else {
+                return res.substring(11);
+            }
+        });
+    }
+
+    function formatValue(v) {
+        const a = Math.abs(v);
+        if (a >= 1000000000000000) { return (v / 1000000000000000) + 'P'; }
+        if (a >= 1000000000000) { return (v / 1000000000000) + 'T'; }
+        if (a >= 1000000000) { return (v / 1000000000) + 'G'; }
+        if (a >= 1000000) { return (v / 1000000) + 'M'; }
+        if (a >= 1000) { return (v / 1000) + 'K'; }
+        if (a > 0 && a < 0.001) { return (v * 1000000) + "μ"; }
+        return v;
+    }
+
+    let axis_x = {
         stroke: axes_color,
         grid: { width: 1 / devicePixelRatio, stroke: grid_color },
-        ticks: { width: 1 / devicePixelRatio, stroke: grid_color }
+        ticks: { width: 1 / devicePixelRatio, stroke: grid_color },
+        values: formatDateTimes,
+        space: 80,
+        incrs: [1, 5, 10, 15, 30,
+            60, 60 * 5, 60 * 10, 60 * 15, 60 * 30,
+            3600, 3600 * 2, 3600 * 3, 3600 * 4, 3600 * 6, 3600 * 12,
+            3600 * 24],
     };
 
-    let axes = [axis, axis];
-    let series = [{ label: "x" }];
+    let axis_y = {
+        stroke: axes_color,
+        grid: { width: 1 / devicePixelRatio, stroke: grid_color },
+        ticks: { width: 1 / devicePixelRatio, stroke: grid_color },
+        values: (self, ticks) => ticks.map(formatValue)
+    };
+
+    let axes = [axis_x, axis_y];
+    let series = [{ label: "time", value: (self, t) => formatDateTime(t) }];
     let data = [reply.data[reply.meta[0].name]];
 
     // Treat every column as series
@@ -1254,9 +1295,10 @@ async function draw(idx, chart, url_params, query) {
     const opts = {
         width: chart.clientWidth,
         height: chart.clientHeight,
+        scales: { x: { time: false } }, /// Because we want to split and format time on our own.
         axes,
         series,
-        padding: [ null, null, null, (Math.round(max_value * 100) / 100).toString().length * 6 - 10 ],
+        padding: [ null, null, null, 3 ],
         plugins: [ legendAsTooltipPlugin() ],
         cursor: {
             sync: {

From a6f9dd4447cbb475cbf77b07de35b40fbcad50b1 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 3 Aug 2024 01:35:10 +0200
Subject: [PATCH 1294/1488] Improve dashboard

---
 programs/server/dashboard.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/programs/server/dashboard.html b/programs/server/dashboard.html
index 71880b9e228..c69acec7858 100644
--- a/programs/server/dashboard.html
+++ b/programs/server/dashboard.html
@@ -1010,7 +1010,7 @@ function legendAsTooltipPlugin({ className, style = { background: "var(--legend-
             position: "absolute",
             left: 0,
             top: 0,
-            zIndex: 200,
+            zIndex: 100,
             boxShadow: "2px 2px 10px rgba(0, 0, 0, 0.1)",
             ...style
         });

From 95659de26573bdb17ab2b5649e6dad96fb75c479 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 3 Aug 2024 01:45:41 +0200
Subject: [PATCH 1295/1488] Fix invalid detection of an empty result

---
 programs/server/dashboard.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/programs/server/dashboard.html b/programs/server/dashboard.html
index c69acec7858..238254f4ef8 100644
--- a/programs/server/dashboard.html
+++ b/programs/server/dashboard.html
@@ -1141,7 +1141,7 @@ async function draw(idx, chart, url_params, query) {
 
     let {reply, error} = await doFetch(query, url_params);
     if (!error) {
-        if (reply.rows.length == 0) {
+        if (reply.rows == 0) {
             error = "Query returned empty result.";
         } else if (reply.meta.length < 2) {
             error = "Query should return at least two columns: unix timestamp and value.";

From a99f9bb603f78437fba8d3ebb031c2f41d00cd58 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 3 Aug 2024 01:50:27 +0200
Subject: [PATCH 1296/1488] Focus on the mass editor

---
 programs/server/dashboard.html | 1 +
 1 file changed, 1 insertion(+)

diff --git a/programs/server/dashboard.html b/programs/server/dashboard.html
index 238254f4ef8..8fb07d5da3b 100644
--- a/programs/server/dashboard.html
+++ b/programs/server/dashboard.html
@@ -945,6 +945,7 @@ function showMassEditor() {
     let editor = document.getElementById('mass-editor-textarea');
     editor.value = JSON.stringify({params: params, queries: queries}, null, 2);
 
+    editor.focus();
     mass_editor_active = true;
 }
 

From eeb8c1caac9e8e2ba2f3a1a86f5603281e161610 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 3 Aug 2024 02:06:53 +0200
Subject: [PATCH 1297/1488] Improve margins when there are many parameters

---
 programs/server/dashboard.html | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/programs/server/dashboard.html b/programs/server/dashboard.html
index 8fb07d5da3b..344de779065 100644
--- a/programs/server/dashboard.html
+++ b/programs/server/dashboard.html
@@ -256,6 +256,7 @@
             font-weight: bold;
             user-select: none;
             cursor: pointer;
+            margin-bottom: 1rem;
         }
 
         #run:hover {
@@ -309,7 +310,7 @@
             color: var(--param-text-color);
             display: inline-block;
             box-shadow: 1px 1px 0 var(--shadow-color);
-            margin-bottom: 1rem;
+            margin-bottom: 0.5rem;
         }
 
         input:focus {

From 090fb59194462324507d75f032aa803303c3e041 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 3 Aug 2024 02:20:58 +0200
Subject: [PATCH 1298/1488] Automatic field width of chart parameters

---
 programs/server/dashboard.html | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/programs/server/dashboard.html b/programs/server/dashboard.html
index 344de779065..0b099b15536 100644
--- a/programs/server/dashboard.html
+++ b/programs/server/dashboard.html
@@ -218,6 +218,7 @@
 
         #chart-params .param {
             width: 6%;
+            font-family: monospace;
         }
 
         input {
@@ -658,6 +659,10 @@ function insertParam(name, value) {
     param_value.value = value;
     param_value.spellcheck = false;
 
+    let setWidth = e => { e.style.width = (e.value.length + 1) + 'ch' };
+    if (value) { setWidth(param_value); }
+    param_value.addEventListener('input', e => setWidth(e.target));
+
     param_wrapper.appendChild(param_name);
     param_wrapper.appendChild(param_value);
     document.getElementById('chart-params').appendChild(param_wrapper);

From 9a017528a4685fc4ed7eec7ba37f9e9804972c3b Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Wed, 31 Jul 2024 17:38:20 +0200
Subject: [PATCH 1299/1488] Minor change

---
 src/Databases/DatabaseOnDisk.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp
index f419f5811a1..734f354d9a5 100644
--- a/src/Databases/DatabaseOnDisk.cpp
+++ b/src/Databases/DatabaseOnDisk.cpp
@@ -313,7 +313,7 @@ void DatabaseOnDisk::detachTablePermanently(ContextPtr query_context, const Stri
         std::lock_guard lock(mutex);
         if (const auto it = snapshot_detached_tables.find(table_name); it == snapshot_detached_tables.end())
         {
-            throw Exception(ErrorCodes::LOGICAL_ERROR, "Snapshot doesn't contain info about detached table={}", table_name);
+            throw Exception(ErrorCodes::LOGICAL_ERROR, "Snapshot doesn't contain info about detached table `{}`", table_name);
         }
         else
         {

From dfeb1991164bd6c8b0efc8bdcfe9dcd5b8906928 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 3 Aug 2024 03:06:37 +0200
Subject: [PATCH 1300/1488] Fix locking inside TimerDescriptor

---
 src/Common/TimerDescriptor.cpp | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/src/Common/TimerDescriptor.cpp b/src/Common/TimerDescriptor.cpp
index 9a171ae9487..ce290a1cb31 100644
--- a/src/Common/TimerDescriptor.cpp
+++ b/src/Common/TimerDescriptor.cpp
@@ -2,6 +2,7 @@
 
 #include <Common/TimerDescriptor.h>
 #include <Common/Exception.h>
+#include <Common/Epoll.h>
 #include <Common/logger_useful.h>
 
 #include <sys/timerfd.h>
@@ -75,10 +76,22 @@ void TimerDescriptor::drain() const
     /// or since the last successful read(2), then the buffer given to read(2) returns an unsigned 8-byte integer (uint64_t)
     /// containing the number of expirations that have occurred.
     /// (The returned value is in host byte order—that is, the native byte order for integers on the host machine.)
+
+    /// Due to a bug in Linux Kernel, reading from timerfd in non-blocking mode can be still blocking.
+    /// Avoid it with polling.
+    Epoll epoll;
+    epoll.add(timer_fd);
+    epoll_event event;
+    event.data.fd = -1;
+    size_t ready_count = epoll.getManyReady(1, &event, 0);
+    if (!ready_count)
+        return;
+
     uint64_t buf;
     while (true)
     {
         ssize_t res = ::read(timer_fd, &buf, sizeof(buf));
+
         if (res < 0)
         {
             /// man timerfd_create:

From e491c51cdc9a94018fc6918f74232cfff747436a Mon Sep 17 00:00:00 2001
From: Michael Kolupaev <michael.kolupaev@clickhouse.com>
Date: Sat, 3 Aug 2024 01:49:53 +0000
Subject: [PATCH 1301/1488] e

---
 src/Storages/MergeTree/KeyCondition.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp
index 9115cb5608e..2b89344d3d9 100644
--- a/src/Storages/MergeTree/KeyCondition.cpp
+++ b/src/Storages/MergeTree/KeyCondition.cpp
@@ -670,7 +670,7 @@ static const ActionsDAG::Node & cloneASTWithInversionPushDown(
                 {
                     /// Make sure we don't change types of function arguments (e.g. remove LowCardinality).
                     /// Otherwise the function may crash when passed columns of unexpected types.
-                    ///  * Why not check this for all subexperessions rather than function arguments?
+                    ///  * Why not check this for all subexpressions rather than function arguments?
                     ///    Because types may change, e.g. in `NOT (u64 AND u64)` -> `(NOT u64 OR NOT u64)`
                     ///    the AND's args were UInt64, but OR's args are UInt8.
                     ///  * Why not re-resolve function overload, using FunctionFactory::instance().get(name, context)?

From 8cb2e308f41638ebb6ba7fddbd4f0bf89d4d612e Mon Sep 17 00:00:00 2001
From: shiyer7474 <shiyer@altinity.com>
Date: Sat, 3 Aug 2024 01:55:24 +0000
Subject: [PATCH 1302/1488] Only new analyzer

---
 .../03209_parameterized_view_with_non_literal_params.sql        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/03209_parameterized_view_with_non_literal_params.sql b/tests/queries/0_stateless/03209_parameterized_view_with_non_literal_params.sql
index f2c61e5cb1d..7fe84929910 100644
--- a/tests/queries/0_stateless/03209_parameterized_view_with_non_literal_params.sql
+++ b/tests/queries/0_stateless/03209_parameterized_view_with_non_literal_params.sql
@@ -1,4 +1,4 @@
-
+SET allow_experimental_analyzer = 1;
 select 'Test with Date parameter';
 
 drop table if exists date_table_pv;

From 28ec383739d9a4974549e8c6491797f0eaafaffb Mon Sep 17 00:00:00 2001
From: jsc0218 <jsc0218@gmail.com>
Date: Sat, 3 Aug 2024 02:07:24 +0000
Subject: [PATCH 1303/1488] add sharedmergetree

---
 src/Storages/MergeTree/MergeTreeData.cpp | 5 +++--
 src/Storages/StorageFactory.cpp          | 8 ++++----
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index a158e375ae9..e849c4b794f 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -3216,10 +3216,11 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, Context
     /// Block the case of alter table add projection for special merge trees.
     if (std::any_of(commands.begin(), commands.end(), [](const AlterCommand & c) { return c.type == AlterCommand::ADD_PROJECTION; }))
     {
-        if (auto storage_name = getName(); storage_name != "MergeTree" && storage_name != "ReplicatedMergeTree"
+        const std::unordered_set<String> allowed_storages{"MergeTree", "ReplicatedMergeTree", "SharedMergeTree"};
+        if (auto storage_name = getName(); !allowed_storages.contains(storage_name)
             && settings_from_storage->deduplicate_merge_projection_mode == DeduplicateMergeProjectionMode::THROW)
             throw Exception(ErrorCodes::SUPPORT_IS_DISABLED,
-                "Projection is fully supported in (Replictaed)MergeTree, but also allowed in non-throw mode with other"
+                "Projection is fully supported in (Replictaed, Shared)MergeTree, but also allowed in non-throw mode with other"
                 " mergetree family members. Consider drop or rebuild option of deduplicate_merge_projection_mode."
                 " Current storage name is {}.", storage_name);
     }
diff --git a/src/Storages/StorageFactory.cpp b/src/Storages/StorageFactory.cpp
index 7360d351e8a..557f53a9ada 100644
--- a/src/Storages/StorageFactory.cpp
+++ b/src/Storages/StorageFactory.cpp
@@ -204,10 +204,10 @@ StoragePtr StorageFactory::get(
 
                 /// Now let's handle the merge tree family. Note we only handle in the mode of CREATE due to backward compatibility.
                 /// Otherwise, it would fail to start in the case of existing projections with special mergetree.
-                /// Projection is fully supported in (Replictaed)MergeTree, but also allowed in non-throw mode with other mergetree family members.
                 chassert(query.storage->engine);
-                if (std::string_view engine_name(query.storage->engine->name); mode == LoadingStrictnessLevel::CREATE
-                    && engine_name != "MergeTree" && engine_name != "ReplicatedMergeTree")
+                const std::unordered_set<String> allowed_engines{"MergeTree", "ReplicatedMergeTree", "SharedMergeTree"};
+                if (auto engine_name(query.storage->engine->name); mode == LoadingStrictnessLevel::CREATE
+                    && !allowed_engines.contains(engine_name))
                 {
                     /// default throw mode in deduplicate_merge_projection_mode
                     bool projection_allowed = false;
@@ -224,7 +224,7 @@ StoragePtr StorageFactory::get(
                     }
                     if (!projection_allowed)
                         throw Exception(ErrorCodes::SUPPORT_IS_DISABLED,
-                            "Projection is fully supported in (Replictaed)MergeTree, but also allowed in non-throw mode with other"
+                            "Projection is fully supported in (Replictaed, Shared)MergeTree, but also allowed in non-throw mode with other"
                             " mergetree family members. Consider drop or rebuild option of deduplicate_merge_projection_mode."
                             " Current storage name is {}.", engine_name);
                 }

From 96e826d154e9b55cf035a3ce025ac81194455ebc Mon Sep 17 00:00:00 2001
From: Michael Kolupaev <michael.kolupaev@clickhouse.com>
Date: Sat, 3 Aug 2024 02:57:33 +0000
Subject: [PATCH 1304/1488] Change tactics again

---
 src/Storages/MergeTree/KeyCondition.cpp | 31 +++++++++++++++----------
 1 file changed, 19 insertions(+), 12 deletions(-)

diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp
index 2b89344d3d9..dfb43c4e75d 100644
--- a/src/Storages/MergeTree/KeyCondition.cpp
+++ b/src/Storages/MergeTree/KeyCondition.cpp
@@ -668,23 +668,30 @@ static const ActionsDAG::Node & cloneASTWithInversionPushDown(
                 }
                 else
                 {
-                    /// Make sure we don't change types of function arguments (e.g. remove LowCardinality).
-                    /// Otherwise the function may crash when passed columns of unexpected types.
-                    ///  * Why not check this for all subexpressions rather than function arguments?
-                    ///    Because types may change, e.g. in `NOT (u64 AND u64)` -> `(NOT u64 OR NOT u64)`
-                    ///    the AND's args were UInt64, but OR's args are UInt8.
-                    ///  * Why not re-resolve function overload, using FunctionFactory::instance().get(name, context)?
-                    ///    Because some functions can't be found through FunctionFactory, e.g. FunctionCapture.
-                    ///    (But maybe we could re-resolve only if argument types changed.)
+                    /// Argument types could change slightly because of our transformations, e.g.
+                    /// LowCardinality can be added because some subexpressions became constant
+                    /// (in particular, sets). If that happens, re-run function overload resolver.
+                    /// Otherwise don't re-run it because some functions may not be available
+                    /// through FunctionFactory::get(), e.g. FunctionCapture.
+                    bool types_changed = false;
                     for (size_t i = 0; i < children.size(); ++i)
                     {
                         if (!node.children[i]->result_type->equals(*children[i]->result_type))
-                            throw Exception(
-                                ErrorCodes::LOGICAL_ERROR, "KeyCondition inadvertently changed subexpression data type: '{}' -> '{}', column `{}`",
-                                node.children[i]->result_type->getName(), children[i]->result_type->getName(), node.children[i]->result_name);
+                        {
+                            types_changed = true;
+                            break;
+                        }
                     }
 
-                    res = &inverted_dag.addFunction(node.function_base, children, "");
+                    if (types_changed)
+                    {
+                        auto function_builder = FunctionFactory::instance().get(name, context);
+                        res = &inverted_dag.addFunction(function_builder, children, "");
+                    }
+                    else
+                    {
+                        res = &inverted_dag.addFunction(node.function_base, children, "");
+                    }
                 }
             }
         }

From f97abf69949f8822d70f4b1251e1945f279dd0ec Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Sat, 3 Aug 2024 09:35:37 +0200
Subject: [PATCH 1305/1488] tests: avoid endless wait in
 01042_system_reload_dictionary_reloads_completely

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 ...em_reload_dictionary_reloads_completely.sh | 21 ++++++++++++++-----
 1 file changed, 16 insertions(+), 5 deletions(-)

diff --git a/tests/queries/0_stateless/01042_system_reload_dictionary_reloads_completely.sh b/tests/queries/0_stateless/01042_system_reload_dictionary_reloads_completely.sh
index 03dd376f802..ebc4110332f 100755
--- a/tests/queries/0_stateless/01042_system_reload_dictionary_reloads_completely.sh
+++ b/tests/queries/0_stateless/01042_system_reload_dictionary_reloads_completely.sh
@@ -8,6 +8,18 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
 set -e -o pipefail
 
+# Wait when the dictionary will update the value for 13 on its own:
+function wait_for_dict_upate()
+{
+    for ((i = 0; i < 100; ++i)); do
+        if [ "$(${CLICKHOUSE_CLIENT} --query "SELECT dictGetInt64('${CLICKHOUSE_DATABASE}.dict', 'y', toUInt64(13))")" != -1 ]; then
+            return 0
+        fi
+        sleep 0.5
+    done
+    return 1
+}
+
 $CLICKHOUSE_CLIENT <<EOF
 CREATE TABLE ${CLICKHOUSE_DATABASE}.table(x Int64, y Int64, insert_time DateTime) ENGINE = MergeTree ORDER BY tuple();
 INSERT INTO ${CLICKHOUSE_DATABASE}.table VALUES (12, 102, now());
@@ -29,11 +41,10 @@ $CLICKHOUSE_CLIENT --query "SELECT '12 -> ', dictGetInt64('${CLICKHOUSE_DATABASE
 $CLICKHOUSE_CLIENT --query "INSERT INTO ${CLICKHOUSE_DATABASE}.table VALUES (13, 103, now())"
 $CLICKHOUSE_CLIENT --query "INSERT INTO ${CLICKHOUSE_DATABASE}.table VALUES (14, 104, now() - INTERVAL 1 DAY)"
 
-# Wait when the dictionary will update the value for 13 on its own:
-while [ "$(${CLICKHOUSE_CLIENT} --query "SELECT dictGetInt64('${CLICKHOUSE_DATABASE}.dict', 'y', toUInt64(13))")" = -1 ]
-do
-    sleep 0.5
-done
+if ! wait_for_dict_upate; then
+    echo "Dictionary had not been reloaded" >&2
+    exit 1
+fi
 
 $CLICKHOUSE_CLIENT --query "SELECT '13 -> ', dictGetInt64('${CLICKHOUSE_DATABASE}.dict', 'y', toUInt64(13))"
 

From 40cd5467c18d65a6624d273ac1a8fd9cc9257d8c Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Sat, 3 Aug 2024 09:39:33 +0200
Subject: [PATCH 1306/1488] tests: fix
 01042_system_reload_dictionary_reloads_completely flakiness (increase lag)

The test fails in case of INSERT takes > 1 sec:

    2024.08.02 13:06:07.746869 [ 45445 ] {c9b55378-6bc5-46d5-80c1-5385a880f88b} <Debug> executeQuery: (from [::1]:37208) (comment: 01042_system_reload_dictionary_reloads_completely.sh) CREATE DICTIONARY test_m4lx2bit.dict ( x Int64 DEFAULT -1, y Int64 DEFAULT -1, insert_time DateTime ) PRIMARY KEY x SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'table' DB 'test_m4lx2bit' UPDATE_FIELD 'insert_time')) LAYOUT(FLAT()) LIFETIME(1); (stage: Complete)
    ...
    2024.08.02 13:06:08.263832 [ 59987 ] {744da223-67b9-4e32-b12a-eb2550a92fdb} <Trace> DictionaryFactory: Created dictionary source 'ClickHouse: test_m4lx2bit.table' for dictionary '5b2b98a9-9372-47c9-bda3-830794cb96e7'
    2024.08.02 13:06:08.268118 [ 59987 ] {744da223-67b9-4e32-b12a-eb2550a92fdb} <Debug> executeQuery: (internal) SELECT `x`, `y`, `insert_time` FROM `test_m4lx2bit`.`table`; (stage: Complete)
    ...
    2024.08.02 13:06:09.193190 [ 45445 ] {b6033498-4666-452f-bcf9-02ecf257ba7f} <Debug> executeQuery: (from [::1]:37262) (comment: 01042_system_reload_dictionary_reloads_completely.sh) INSERT INTO test_m4lx2bit.table VALUES  (stage: Complete)
    ...
    2024.08.02 13:06:11.342119 [ 50962 ] {} <Debug> executeQuery: (internal) SELECT `x`, `y`, `insert_time` FROM `test_m4lx2bit`.`table` WHERE insert_time >= '2024-08-02 13:06:07'; (stage: Complete)
    ...
    2024.08.02 13:06:11.832158 [ 45445 ] {b6033498-4666-452f-bcf9-02ecf257ba7f} <Debug> TCPHandler: Processed in 2.642106236 sec.
    ...
    2024.08.02 13:06:16.357448 [ 41632 ] {} <Debug> executeQuery: (internal) SELECT `x`, `y`, `insert_time` FROM `test_m4lx2bit`.`table` WHERE insert_time >= '2024-08-02 13:06:10'; (stage: Complete)

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 .../01042_system_reload_dictionary_reloads_completely.sh        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/01042_system_reload_dictionary_reloads_completely.sh b/tests/queries/0_stateless/01042_system_reload_dictionary_reloads_completely.sh
index ebc4110332f..453e1bb8f0a 100755
--- a/tests/queries/0_stateless/01042_system_reload_dictionary_reloads_completely.sh
+++ b/tests/queries/0_stateless/01042_system_reload_dictionary_reloads_completely.sh
@@ -31,7 +31,7 @@ CREATE DICTIONARY ${CLICKHOUSE_DATABASE}.dict
   insert_time DateTime
 )
 PRIMARY KEY x
-SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'table' DB '${CLICKHOUSE_DATABASE}' UPDATE_FIELD 'insert_time'))
+SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'table' DB '${CLICKHOUSE_DATABASE}' UPDATE_FIELD 'insert_time' UPDATE_LAG 60))
 LAYOUT(FLAT())
 LIFETIME(1);
 EOF

From 6ce6af0647590f4b58a6ab87ee5f29b8487e8c2f Mon Sep 17 00:00:00 2001
From: Vitaly Baranov <vitlibar@clickhouse.com>
Date: Sat, 3 Aug 2024 14:16:24 +0200
Subject: [PATCH 1307/1488] Fix completion RESTORE ON CLUSTER.

---
 src/Backups/RestorerFromBackup.cpp | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)

diff --git a/src/Backups/RestorerFromBackup.cpp b/src/Backups/RestorerFromBackup.cpp
index 3056f9fe421..278af9d4eb3 100644
--- a/src/Backups/RestorerFromBackup.cpp
+++ b/src/Backups/RestorerFromBackup.cpp
@@ -222,10 +222,19 @@ void RestorerFromBackup::setStage(const String & new_stage, const String & messa
     if (restore_coordination)
     {
         restore_coordination->setStage(new_stage, message);
-        if (new_stage == Stage::FINDING_TABLES_IN_BACKUP)
-            restore_coordination->waitForStage(new_stage, on_cluster_first_sync_timeout);
-        else
-            restore_coordination->waitForStage(new_stage);
+
+        /// The initiator of a RESTORE ON CLUSTER query waits for other hosts to complete their work (see waitForStage(Stage::COMPLETED) in BackupsWorker::doRestore),
+        /// but other hosts shouldn't wait for each others' completion. (That's simply unnecessary and also
+        /// the initiator may start cleaning up (e.g. removing restore-coordination ZooKeeper nodes) once all other hosts are in Stage::COMPLETED.)
+        bool need_wait = (new_stage != Stage::COMPLETED);
+
+        if (need_wait)
+        {
+            if (new_stage == Stage::FINDING_TABLES_IN_BACKUP)
+                restore_coordination->waitForStage(new_stage, on_cluster_first_sync_timeout);
+            else
+                restore_coordination->waitForStage(new_stage);
+        }
     }
 }
 

From 73080d25a93aa715c4126ed82b89e3ada89d7a3f Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 3 Aug 2024 15:10:05 +0200
Subject: [PATCH 1308/1488] Fix test
 `00002_log_and_exception_messages_formatting`

---
 src/Databases/DatabaseHDFS.cpp                              | 4 ++--
 src/Storages/ObjectStorage/HDFS/Configuration.cpp           | 6 +++---
 .../00002_log_and_exception_messages_formatting.sql         | 5 ++++-
 3 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/src/Databases/DatabaseHDFS.cpp b/src/Databases/DatabaseHDFS.cpp
index eccaae5f22e..f58f1b76e71 100644
--- a/src/Databases/DatabaseHDFS.cpp
+++ b/src/Databases/DatabaseHDFS.cpp
@@ -75,8 +75,8 @@ std::string DatabaseHDFS::getTablePath(const std::string & table_name) const
         return table_name;
 
     if (source.empty())
-        throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad hdfs url: {}. "
-                        "It should have structure 'hdfs://<host_name>:<port>/path'", table_name);
+        throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad HDFS URL: {}. "
+            "It should have the following structure 'hdfs://<host_name>:<port>/path'", table_name);
 
     return fs::path(source) / table_name;
 }
diff --git a/src/Storages/ObjectStorage/HDFS/Configuration.cpp b/src/Storages/ObjectStorage/HDFS/Configuration.cpp
index e8071be6f02..85eb29a3868 100644
--- a/src/Storages/ObjectStorage/HDFS/Configuration.cpp
+++ b/src/Storages/ObjectStorage/HDFS/Configuration.cpp
@@ -142,11 +142,11 @@ void StorageHDFSConfiguration::setURL(const std::string & url_)
 {
     auto pos = url_.find("//");
     if (pos == std::string::npos)
-        throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad hdfs url: {}", url_);
+        throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad HDFS URL: {}. It should have the following structure 'hdfs://<host_name>:<port>/path'", url_);
 
     pos = url_.find('/', pos + 2);
     if (pos == std::string::npos)
-        throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad hdfs url: {}", url_);
+        throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad HDFS URL: {}. It should have the following structure 'hdfs://<host_name>:<port>/path'", url_);
 
     path = url_.substr(pos + 1);
     if (!path.starts_with('/'))
@@ -155,7 +155,7 @@ void StorageHDFSConfiguration::setURL(const std::string & url_)
     url = url_.substr(0, pos);
     paths = {path};
 
-    LOG_TRACE(getLogger("StorageHDFSConfiguration"), "Using url: {}, path: {}", url, path);
+    LOG_TRACE(getLogger("StorageHDFSConfiguration"), "Using URL: {}, path: {}", url, path);
 }
 
 void StorageHDFSConfiguration::addStructureAndFormatToArgs(
diff --git a/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql b/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql
index 07c42d6d039..e916fdfc1ff 100644
--- a/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql
+++ b/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql
@@ -163,7 +163,10 @@ create temporary table known_short_messages (s String) as select * from (select
     '{} -> {}',
     '{} {}',
     '{}%',
-    '{}: {}'
+    '{}: {}',
+    'Unknown data type family: {}',
+    'Cannot load time zone {}',
+    'Unknown table engine {}'
     ] as arr) array join arr;
 
 -- Check that we don't have too many short meaningless message patterns.

From a749223251b7e580f5d7bbcb4fc59aa6b5fffbe2 Mon Sep 17 00:00:00 2001
From: taiyang-li <654010905@qq.com>
Date: Sat, 3 Aug 2024 22:43:12 +0800
Subject: [PATCH 1309/1488] change as request

---
 src/Functions/printf.cpp                      | 39 ++++++++++++++-----
 .../0_stateless/03203_function_printf.sql     |  7 +++-
 2 files changed, 36 insertions(+), 10 deletions(-)

diff --git a/src/Functions/printf.cpp b/src/Functions/printf.cpp
index 3efe854a53b..3cf3efaf534 100644
--- a/src/Functions/printf.cpp
+++ b/src/Functions/printf.cpp
@@ -6,11 +6,10 @@
 #include <Functions/FunctionHelpers.h>
 #include <Functions/IFunction.h>
 #include <Functions/formatString.h>
+#include <IO/Operators.h>
 #include <IO/WriteHelpers.h>
 
 #include <memory>
-#include <sstream>
-#include <string>
 #include <vector>
 #include <fmt/format.h>
 #include <fmt/printf.h>
@@ -22,6 +21,7 @@ namespace ErrorCodes
 extern const int ILLEGAL_COLUMN;
 extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
 extern const int ILLEGAL_TYPE_OF_ARGUMENT;
+extern const int BAD_ARGUMENTS;
 }
 
 namespace
@@ -52,10 +52,9 @@ private:
 
         [[maybe_unused]] String toString() const
         {
-            std::ostringstream oss; // STYLE_CHECK_ALLOW_STD_STRING_STREAM
-            oss << "format:" << format << ", rows:" << rows << ", is_literal:" << is_literal << ", input:" << input.dumpStructure()
-                << std::endl;
-            return oss.str();
+            WriteBufferFromOwnString buf;
+            buf << "format:" << format << ", rows:" << rows << ", is_literal:" << is_literal << ", input:" << input.dumpStructure() << "\n";
+            return buf.str();
         }
 
     private:
@@ -229,9 +228,31 @@ public:
         ColumnsWithTypeAndName concat_args(instructions.size());
         for (size_t i = 0; i < instructions.size(); ++i)
         {
-            // std::cout << "instruction[" << i << "]:" << instructions[i].toString() << std::endl;
-            concat_args[i] = instructions[i].execute();
-            // std::cout << "concat_args[" << i << "]:" << concat_args[i].dumpStructure() << std::endl;
+            const auto & instruction = instructions[i];
+            try
+            {
+                // std::cout << "instruction[" << i << "]:" << instructions[i].toString() << std::endl;
+                concat_args[i] = instruction.execute();
+                // std::cout << "concat_args[" << i << "]:" << concat_args[i].dumpStructure() << std::endl;
+            }
+            catch (const fmt::v9::format_error & e)
+            {
+                if (instruction.is_literal)
+                    throw Exception(
+                        ErrorCodes::BAD_ARGUMENTS,
+                        "Bad format {} in function {} without input argument, reason: {}",
+                        instruction.format,
+                        getName(),
+                        e.what());
+                else
+                    throw Exception(
+                        ErrorCodes::BAD_ARGUMENTS,
+                        "Bad format {} in function {} with {} as input argument, reason: {}",
+                        instructions[i].format,
+                        getName(),
+                        instruction.input.dumpStructure(),
+                        e.what());
+            }
         }
 
         auto res = function_concat->build(concat_args)->execute(concat_args, std::make_shared<DataTypeString>(), input_rows_count);
diff --git a/tests/queries/0_stateless/03203_function_printf.sql b/tests/queries/0_stateless/03203_function_printf.sql
index c41cbf0b5e9..6ff4699c8a7 100644
--- a/tests/queries/0_stateless/03203_function_printf.sql
+++ b/tests/queries/0_stateless/03203_function_printf.sql
@@ -31,4 +31,9 @@ select printf('%%.2e: %.2e', 123.456) = '%.2e: 1.23e+02';
 select printf('%%.2g: %.2g', 123.456) = '%.2g: 1.2e+02';
 
 -- Testing character formats with precision
-select printf('%%.2s: %.2s', 'abc') = '%.2s: ab';
\ No newline at end of file
+select printf('%%.2s: %.2s', 'abc') = '%.2s: ab';
+
+select printf('%%X: %X', 123.123); -- { serverError BAD_ARGUMENTS }
+select printf('%%A: %A', 'abc'); -- { serverError BAD_ARGUMENTS }
+select printf('%%s: %s', 100); -- { serverError BAD_ARGUMENTS }
+select printf('%%n: %n', 100); -- { serverError BAD_ARGUMENTS }

From abb747498d04fd6d6cac71d6097741b41bf572e3 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 3 Aug 2024 16:54:44 +0200
Subject: [PATCH 1310/1488] Fix test
 `02789_reading_from_s3_with_connection_pool`

---
 ...89_reading_from_s3_with_connection_pool.sh | 84 +++++++++++--------
 1 file changed, 50 insertions(+), 34 deletions(-)

diff --git a/tests/queries/0_stateless/02789_reading_from_s3_with_connection_pool.sh b/tests/queries/0_stateless/02789_reading_from_s3_with_connection_pool.sh
index 751b2798243..39399842db1 100755
--- a/tests/queries/0_stateless/02789_reading_from_s3_with_connection_pool.sh
+++ b/tests/queries/0_stateless/02789_reading_from_s3_with_connection_pool.sh
@@ -14,43 +14,59 @@ SETTINGS disk = 's3_disk', min_bytes_for_wide_part = 0;
 
 INSERT INTO test_s3 SELECT number, number FROM numbers_mt(1e7);
 "
-query="SELECT a, b FROM test_s3"
-query_id=$(${CLICKHOUSE_CLIENT} --query "select queryID() from ($query) limit 1" 2>&1)
-${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS"
-${CLICKHOUSE_CLIENT} -nm --query "
-SELECT ProfileEvents['DiskConnectionsPreserved'] > 0
-FROM system.query_log
-WHERE type = 'QueryFinish'
-    AND current_database = currentDatabase()
-    AND query_id='$query_id';
-"
+
+# This (reusing connections from the pool) is not guaranteed to always happen,
+# (due to random time difference between the queries and random activity in parallel)
+# but should happen most of the time.
+
+while true
+do
+    query="SELECT a, b FROM test_s3"
+    query_id=$(${CLICKHOUSE_CLIENT} --query "select queryID() from ($query) limit 1" 2>&1)
+    ${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS"
+
+    RES=$(${CLICKHOUSE_CLIENT} -nm --query "
+    SELECT ProfileEvents['DiskConnectionsPreserved'] > 0
+    FROM system.query_log
+    WHERE type = 'QueryFinish'
+        AND current_database = currentDatabase()
+        AND query_id='$query_id';
+    ")
+
+    [[ $RES -eq 1 ]] && echo "$RES" && break;
+done
 
 
 # Test connection pool in ReadWriteBufferFromHTTP
 
-query_id=$(${CLICKHOUSE_CLIENT} -nq "
-create table mut (n int, m int, k int) engine=ReplicatedMergeTree('/test/02441/{database}/mut', '1') order by n;
-set insert_keeper_fault_injection_probability=0;
-insert into mut values (1, 2, 3), (10, 20, 30);
+while true
+do
+    query_id=$(${CLICKHOUSE_CLIENT} -nq "
+    create table mut (n int, m int, k int) engine=ReplicatedMergeTree('/test/02441/{database}/mut', '1') order by n;
+    set insert_keeper_fault_injection_probability=0;
+    insert into mut values (1, 2, 3), (10, 20, 30);
 
-system stop merges mut;
-alter table mut delete where n = 10;
+    system stop merges mut;
+    alter table mut delete where n = 10;
 
-select queryID() from(
-    -- a funny way to wait for a MUTATE_PART to be assigned
-    select sleepEachRow(2) from url('http://localhost:8123/?param_tries={1..10}&query=' || encodeURLComponent(
-        'select 1 where ''MUTATE_PART'' not in (select type from system.replication_queue where database=''' || currentDatabase() || ''' and table=''mut'')'
-        ), 'LineAsString', 's String')
-    -- queryID() will be returned for each row, since the query above doesn't return anything we need to return a fake row
-    union all
-    select 1
-) limit 1 settings max_threads=1;
-" 2>&1)
-${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS"
-${CLICKHOUSE_CLIENT} -nm --query "
-SELECT ProfileEvents['StorageConnectionsPreserved'] > 0
-FROM system.query_log
-WHERE type = 'QueryFinish'
-    AND current_database = currentDatabase()
-    AND query_id='$query_id';
-"
+    select queryID() from(
+        -- a funny way to wait for a MUTATE_PART to be assigned
+        select sleepEachRow(2) from url('http://localhost:8123/?param_tries={1..10}&query=' || encodeURLComponent(
+            'select 1 where ''MUTATE_PART'' not in (select type from system.replication_queue where database=''' || currentDatabase() || ''' and table=''mut'')'
+            ), 'LineAsString', 's String')
+        -- queryID() will be returned for each row, since the query above doesn't return anything we need to return a fake row
+        union all
+        select 1
+    ) limit 1 settings max_threads=1;
+    " 2>&1)
+    ${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS"
+    RES=$(${CLICKHOUSE_CLIENT} -nm --query "
+    SELECT ProfileEvents['StorageConnectionsPreserved'] > 0
+    FROM system.query_log
+    WHERE type = 'QueryFinish'
+        AND current_database = currentDatabase()
+        AND query_id='$query_id';
+    ")
+
+    [[ $RES -eq 1 ]] && echo "$RES" && break;
+done

From fc651cc0c61feb37e9cf104612cc0ac0cd7448e9 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 3 Aug 2024 17:57:48 +0200
Subject: [PATCH 1311/1488] Fix strange code in HostResolvePool

---
 src/Common/HostResolvePool.cpp | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/src/Common/HostResolvePool.cpp b/src/Common/HostResolvePool.cpp
index cad64ee7204..e8a05a269bc 100644
--- a/src/Common/HostResolvePool.cpp
+++ b/src/Common/HostResolvePool.cpp
@@ -253,18 +253,18 @@ void HostResolver::updateImpl(Poco::Timestamp now, std::vector<Poco::Net::IPAddr
         }
     }
 
-    for (auto & rec : merged)
+    for (auto & record : merged)
     {
-            if (!rec.failed)
-                continue;
+        if (!record.failed || !record.consecutive_fail_count)
+            continue;
 
-            /// Exponential increased time for each consecutive fail
-            auto banned_until = now - Poco::Timespan(history.totalMicroseconds() * (1ull << (rec.consecutive_fail_count - 1)));
-            if (rec.fail_time < banned_until)
-            {
-                rec.failed = false;
-                CurrentMetrics::sub(metrics.banned_count);
-            }
+        /// Exponential increased time for each consecutive fail
+        auto banned_until = now - Poco::Timespan(history.totalMicroseconds() * (1ull << (record.consecutive_fail_count - 1)));
+        if (record.fail_time < banned_until)
+        {
+            record.failed = false;
+            CurrentMetrics::sub(metrics.banned_count);
+        }
     }
 
     chassert(std::is_sorted(merged.begin(), merged.end()));

From 65bba77771248cacd9925fddd3c12e2ed11d205f Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 3 Aug 2024 18:30:33 +0200
Subject: [PATCH 1312/1488] Fix typo

---
 programs/server/Server.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp
index 20db4c2773c..7800ee9ff00 100644
--- a/programs/server/Server.cpp
+++ b/programs/server/Server.cpp
@@ -849,7 +849,7 @@ try
 #endif
 
 #if defined(SANITIZER)
-    LOG_INFO(log, "Query Profiler disabled because they cannot work under sanitizers"
+    LOG_INFO(log, "Query Profiler is disabled because it cannot work under sanitizers"
         " when two different stack unwinding methods will interfere with each other.");
 #endif
 

From 60648e5240fecb92344ff029d2b280f542c3a86e Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 3 Aug 2024 18:41:17 +0200
Subject: [PATCH 1313/1488] Revert "Add replication lag and recovery time
 metrics"

---
 src/Databases/DatabaseReplicated.cpp          | 57 ++++-------------
 src/Databases/DatabaseReplicated.h            | 10 +--
 src/Databases/DatabaseReplicatedWorker.cpp    | 21 -------
 src/Databases/DatabaseReplicatedWorker.h      |  5 --
 src/Storages/System/StorageSystemClusters.cpp | 37 ++++-------
 src/Storages/System/StorageSystemClusters.h   |  4 +-
 .../test_recovery_time_metric/__init__.py     |  0
 .../configs/config.xml                        | 41 -------------
 .../test_recovery_time_metric/test.py         | 61 -------------------
 .../02117_show_create_table_system.reference  |  2 -
 .../03206_replication_lag_metric.reference    |  4 --
 .../03206_replication_lag_metric.sql          | 11 ----
 12 files changed, 27 insertions(+), 226 deletions(-)
 delete mode 100644 tests/integration/test_recovery_time_metric/__init__.py
 delete mode 100644 tests/integration/test_recovery_time_metric/configs/config.xml
 delete mode 100644 tests/integration/test_recovery_time_metric/test.py
 delete mode 100644 tests/queries/0_stateless/03206_replication_lag_metric.reference
 delete mode 100644 tests/queries/0_stateless/03206_replication_lag_metric.sql

diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp
index d2dee9b5994..f127ccbc224 100644
--- a/src/Databases/DatabaseReplicated.cpp
+++ b/src/Databases/DatabaseReplicated.cpp
@@ -12,7 +12,6 @@
 #include <Common/ZooKeeper/KeeperException.h>
 #include <Common/ZooKeeper/Types.h>
 #include <Common/ZooKeeper/ZooKeeper.h>
-#include <Common/ZooKeeper/IKeeper.h>
 #include <Common/PoolId.h>
 #include <Core/ServerSettings.h>
 #include <Core/Settings.h>
@@ -339,12 +338,9 @@ ClusterPtr DatabaseReplicated::getClusterImpl(bool all_groups) const
     return std::make_shared<Cluster>(getContext()->getSettingsRef(), shards, params);
 }
 
-ReplicasInfo DatabaseReplicated::tryGetReplicasInfo(const ClusterPtr & cluster_) const
+std::vector<UInt8> DatabaseReplicated::tryGetAreReplicasActive(const ClusterPtr & cluster_) const
 {
-    Strings paths_get, paths_exists;
-
-    paths_get.emplace_back(fs::path(zookeeper_path) / "max_log_ptr");
-
+    Strings paths;
     const auto & addresses_with_failover = cluster_->getShardsAddresses();
     const auto & shards_info = cluster_->getShardsInfo();
     for (size_t shard_index = 0; shard_index < shards_info.size(); ++shard_index)
@@ -352,59 +348,32 @@ ReplicasInfo DatabaseReplicated::tryGetReplicasInfo(const ClusterPtr & cluster_)
         for (const auto & replica : addresses_with_failover[shard_index])
         {
             String full_name = getFullReplicaName(replica.database_shard_name, replica.database_replica_name);
-            paths_exists.emplace_back(fs::path(zookeeper_path) / "replicas" / full_name / "active");
-            paths_get.emplace_back(fs::path(zookeeper_path) / "replicas" / full_name / "log_ptr");
+            paths.emplace_back(fs::path(zookeeper_path) / "replicas" / full_name / "active");
         }
     }
 
     try
     {
         auto current_zookeeper = getZooKeeper();
-        auto get_res = current_zookeeper->get(paths_get);
-        auto exist_res = current_zookeeper->exists(paths_exists);
-        chassert(get_res.size() == exist_res.size() + 1);
+        auto res = current_zookeeper->exists(paths);
 
-        auto max_log_ptr_zk = get_res[0];
-        if (max_log_ptr_zk.error != Coordination::Error::ZOK)
-            throw Coordination::Exception(max_log_ptr_zk.error);
+        std::vector<UInt8> statuses;
+        statuses.resize(paths.size());
 
-        UInt32 max_log_ptr = parse<UInt32>(max_log_ptr_zk.data);
+        for (size_t i = 0; i < res.size(); ++i)
+            if (res[i].error == Coordination::Error::ZOK)
+                statuses[i] = 1;
 
-        ReplicasInfo replicas_info;
-        replicas_info.resize(exist_res.size());
-
-        size_t global_replica_index = 0;
-        for (size_t shard_index = 0; shard_index < shards_info.size(); ++shard_index)
-        {
-            for (const auto & replica : addresses_with_failover[shard_index])
-            {
-                auto replica_active = exist_res[global_replica_index];
-                auto replica_log_ptr = get_res[global_replica_index + 1];
-
-                if (replica_active.error != Coordination::Error::ZOK && replica_active.error != Coordination::Error::ZNONODE)
-                    throw Coordination::Exception(replica_active.error);
-
-                if (replica_log_ptr.error != Coordination::Error::ZOK)
-                    throw Coordination::Exception(replica_log_ptr.error);
-
-                replicas_info[global_replica_index] = ReplicaInfo{
-                    .is_active = replica_active.error == Coordination::Error::ZOK,
-                    .replication_lag = max_log_ptr - parse<UInt32>(replica_log_ptr.data),
-                    .recovery_time = replica.is_local ? ddl_worker->getCurrentInitializationDurationMs() : 0,
-                };
-
-                ++global_replica_index;
-            }
-        }
-
-        return replicas_info;
-    } catch (...)
+        return statuses;
+    }
+    catch (...)
     {
         tryLogCurrentException(log);
         return {};
     }
 }
 
+
 void DatabaseReplicated::fillClusterAuthInfo(String collection_name, const Poco::Util::AbstractConfiguration & config_ref)
 {
     const auto & config_prefix = fmt::format("named_collections.{}", collection_name);
diff --git a/src/Databases/DatabaseReplicated.h b/src/Databases/DatabaseReplicated.h
index 5a1570ae2e2..27ab262d1f1 100644
--- a/src/Databases/DatabaseReplicated.h
+++ b/src/Databases/DatabaseReplicated.h
@@ -17,14 +17,6 @@ using ZooKeeperPtr = std::shared_ptr<zkutil::ZooKeeper>;
 class Cluster;
 using ClusterPtr = std::shared_ptr<Cluster>;
 
-struct ReplicaInfo
-{
-    bool is_active;
-    UInt32 replication_lag;
-    UInt64 recovery_time;
-};
-using ReplicasInfo = std::vector<ReplicaInfo>;
-
 class DatabaseReplicated : public DatabaseAtomic
 {
 public:
@@ -92,7 +84,7 @@ public:
 
     static void dropReplica(DatabaseReplicated * database, const String & database_zookeeper_path, const String & shard, const String & replica, bool throw_if_noop);
 
-    ReplicasInfo tryGetReplicasInfo(const ClusterPtr & cluster_) const;
+    std::vector<UInt8> tryGetAreReplicasActive(const ClusterPtr & cluster_) const;
 
     void renameDatabase(ContextPtr query_context, const String & new_name) override;
 
diff --git a/src/Databases/DatabaseReplicatedWorker.cpp b/src/Databases/DatabaseReplicatedWorker.cpp
index 4e7408aa96e..1ef88dc03bc 100644
--- a/src/Databases/DatabaseReplicatedWorker.cpp
+++ b/src/Databases/DatabaseReplicatedWorker.cpp
@@ -32,12 +32,6 @@ DatabaseReplicatedDDLWorker::DatabaseReplicatedDDLWorker(DatabaseReplicated * db
 
 bool DatabaseReplicatedDDLWorker::initializeMainThread()
 {
-    {
-        std::lock_guard lock(initialization_duration_timer_mutex);
-        initialization_duration_timer.emplace();
-        initialization_duration_timer->start();
-    }
-
     while (!stop_flag)
     {
         try
@@ -75,10 +69,6 @@ bool DatabaseReplicatedDDLWorker::initializeMainThread()
 
             initializeReplication();
             initialized = true;
-            {
-                std::lock_guard lock(initialization_duration_timer_mutex);
-                initialization_duration_timer.reset();
-            }
             return true;
         }
         catch (...)
@@ -88,11 +78,6 @@ bool DatabaseReplicatedDDLWorker::initializeMainThread()
         }
     }
 
-    {
-        std::lock_guard lock(initialization_duration_timer_mutex);
-        initialization_duration_timer.reset();
-    }
-
     return false;
 }
 
@@ -474,10 +459,4 @@ UInt32 DatabaseReplicatedDDLWorker::getLogPointer() const
     return max_id.load();
 }
 
-UInt64 DatabaseReplicatedDDLWorker::getCurrentInitializationDurationMs() const
-{
-    std::lock_guard lock(initialization_duration_timer_mutex);
-    return initialization_duration_timer ? initialization_duration_timer->elapsedMilliseconds() : 0;
-}
-
 }
diff --git a/src/Databases/DatabaseReplicatedWorker.h b/src/Databases/DatabaseReplicatedWorker.h
index 2309c831839..41edf2221b8 100644
--- a/src/Databases/DatabaseReplicatedWorker.h
+++ b/src/Databases/DatabaseReplicatedWorker.h
@@ -36,8 +36,6 @@ public:
                                    DatabaseReplicated * const database, bool committed = false); /// NOLINT
 
     UInt32 getLogPointer() const;
-
-    UInt64 getCurrentInitializationDurationMs() const;
 private:
     bool initializeMainThread() override;
     void initializeReplication();
@@ -58,9 +56,6 @@ private:
     ZooKeeperPtr active_node_holder_zookeeper;
     /// It will remove "active" node when database is detached
     zkutil::EphemeralNodeHolderPtr active_node_holder;
-
-    std::optional<Stopwatch> initialization_duration_timer;
-    mutable std::mutex initialization_duration_timer_mutex;
 };
 
 }
diff --git a/src/Storages/System/StorageSystemClusters.cpp b/src/Storages/System/StorageSystemClusters.cpp
index d03b600b6ef..160c8d6270e 100644
--- a/src/Storages/System/StorageSystemClusters.cpp
+++ b/src/Storages/System/StorageSystemClusters.cpp
@@ -31,8 +31,6 @@ ColumnsDescription StorageSystemClusters::getColumnsDescription()
         {"database_shard_name", std::make_shared<DataTypeString>(), "The name of the `Replicated` database shard (for clusters that belong to a `Replicated` database)."},
         {"database_replica_name", std::make_shared<DataTypeString>(), "The name of the `Replicated` database replica (for clusters that belong to a `Replicated` database)."},
         {"is_active", std::make_shared<DataTypeNullable>(std::make_shared<DataTypeUInt8>()), "The status of the Replicated database replica (for clusters that belong to a Replicated database): 1 means 'replica is online', 0 means 'replica is offline', NULL means 'unknown'."},
-        {"replication_lag", std::make_shared<DataTypeNullable>(std::make_shared<DataTypeUInt32>()), "The replication lag of the `Replicated` database replica (for clusters that belong to a Replicated database)."},
-        {"recovery_time", std::make_shared<DataTypeNullable>(std::make_shared<DataTypeUInt64>()), "The recovery time of the `Replicated` database replica (for clusters that belong to a Replicated database), in milliseconds."},
     };
 
     description.setAliases({
@@ -48,30 +46,31 @@ void StorageSystemClusters::fillData(MutableColumns & res_columns, ContextPtr co
         writeCluster(res_columns, name_and_cluster, {});
 
     const auto databases = DatabaseCatalog::instance().getDatabases();
-    for (const auto & [database_name, database] : databases)
+    for (const auto & name_and_database : databases)
     {
-        if (const auto * replicated = typeid_cast<const DatabaseReplicated *>(database.get()))
+        if (const auto * replicated = typeid_cast<const DatabaseReplicated *>(name_and_database.second.get()))
         {
+
             if (auto database_cluster = replicated->tryGetCluster())
-                writeCluster(res_columns, {database_name, database_cluster},
-                             replicated->tryGetReplicasInfo(database_cluster));
+                writeCluster(res_columns, {name_and_database.first, database_cluster},
+                             replicated->tryGetAreReplicasActive(database_cluster));
 
             if (auto database_cluster = replicated->tryGetAllGroupsCluster())
-                writeCluster(res_columns, {DatabaseReplicated::ALL_GROUPS_CLUSTER_PREFIX + database_name, database_cluster},
-                             replicated->tryGetReplicasInfo(database_cluster));
+                writeCluster(res_columns, {DatabaseReplicated::ALL_GROUPS_CLUSTER_PREFIX + name_and_database.first, database_cluster},
+                             replicated->tryGetAreReplicasActive(database_cluster));
         }
     }
 }
 
 void StorageSystemClusters::writeCluster(MutableColumns & res_columns, const NameAndCluster & name_and_cluster,
-                                         const ReplicasInfo & replicas_info)
+                                         const std::vector<UInt8> & is_active)
 {
     const String & cluster_name = name_and_cluster.first;
     const ClusterPtr & cluster = name_and_cluster.second;
     const auto & shards_info = cluster->getShardsInfo();
     const auto & addresses_with_failover = cluster->getShardsAddresses();
 
-    size_t global_replica_idx = 0;
+    size_t replica_idx = 0;
     for (size_t shard_index = 0; shard_index < shards_info.size(); ++shard_index)
     {
         const auto & shard_info = shards_info[shard_index];
@@ -100,24 +99,10 @@ void StorageSystemClusters::writeCluster(MutableColumns & res_columns, const Nam
             res_columns[i++]->insert(pool_status[replica_index].estimated_recovery_time.count());
             res_columns[i++]->insert(address.database_shard_name);
             res_columns[i++]->insert(address.database_replica_name);
-            if (replicas_info.empty())
-            {
+            if (is_active.empty())
                 res_columns[i++]->insertDefault();
-                res_columns[i++]->insertDefault();
-                res_columns[i++]->insertDefault();
-            }
             else
-            {
-                const auto & replica_info = replicas_info[global_replica_idx];
-                res_columns[i++]->insert(replica_info.is_active);
-                res_columns[i++]->insert(replica_info.replication_lag);
-                if (replica_info.recovery_time != 0)
-                    res_columns[i++]->insert(replica_info.recovery_time);
-                else
-                    res_columns[i++]->insertDefault();
-            }
-
-            ++global_replica_idx;
+                res_columns[i++]->insert(is_active[replica_idx++]);
         }
     }
 }
diff --git a/src/Storages/System/StorageSystemClusters.h b/src/Storages/System/StorageSystemClusters.h
index f6e08734896..0f7c792261d 100644
--- a/src/Storages/System/StorageSystemClusters.h
+++ b/src/Storages/System/StorageSystemClusters.h
@@ -1,10 +1,10 @@
 #pragma once
 
-#include <Databases/DatabaseReplicated.h>
 #include <DataTypes/DataTypeString.h>
 #include <DataTypes/DataTypesNumber.h>
 #include <Storages/System/IStorageSystemOneBlock.h>
 
+
 namespace DB
 {
 
@@ -27,7 +27,7 @@ protected:
     using NameAndCluster = std::pair<String, std::shared_ptr<Cluster>>;
 
     void fillData(MutableColumns & res_columns, ContextPtr context, const ActionsDAG::Node *, std::vector<UInt8>) const override;
-    static void writeCluster(MutableColumns & res_columns, const NameAndCluster & name_and_cluster, const ReplicasInfo & replicas_info);
+    static void writeCluster(MutableColumns & res_columns, const NameAndCluster & name_and_cluster, const std::vector<UInt8> & is_active);
 };
 
 }
diff --git a/tests/integration/test_recovery_time_metric/__init__.py b/tests/integration/test_recovery_time_metric/__init__.py
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/tests/integration/test_recovery_time_metric/configs/config.xml b/tests/integration/test_recovery_time_metric/configs/config.xml
deleted file mode 100644
index bad9b1fa9ea..00000000000
--- a/tests/integration/test_recovery_time_metric/configs/config.xml
+++ /dev/null
@@ -1,41 +0,0 @@
-<clickhouse>
-    <tcp_port>9000</tcp_port>
-
-    <profiles>
-        <default>
-        </default>
-    </profiles>
-
-    <users>
-        <default>
-            <profile>default</profile>
-            <no_password></no_password>
-        </default>
-    </users>
-
-    <keeper_server>
-        <tcp_port>2181</tcp_port>
-        <server_id>1</server_id>
-        <log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
-        <snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>
-        <coordination_settings>
-            <session_timeout_ms>20000</session_timeout_ms>
-        </coordination_settings>
-        <raft_configuration>
-            <server>
-                <id>1</id>
-                <hostname>localhost</hostname>
-                <port>9444</port>
-            </server>
-        </raft_configuration>
-    </keeper_server>
-
-    <zookeeper>
-        <node index="1">
-            <host>localhost</host>
-            <port>2181</port>
-        </node>
-        <session_timeout_ms>20000</session_timeout_ms>
-    </zookeeper>
-
-</clickhouse>
diff --git a/tests/integration/test_recovery_time_metric/test.py b/tests/integration/test_recovery_time_metric/test.py
deleted file mode 100644
index 6fcf2fad423..00000000000
--- a/tests/integration/test_recovery_time_metric/test.py
+++ /dev/null
@@ -1,61 +0,0 @@
-import pytest
-from helpers.cluster import ClickHouseCluster
-
-cluster = ClickHouseCluster(__file__)
-node = cluster.add_instance(
-    "node",
-    main_configs=["configs/config.xml"],
-    stay_alive=True,
-)
-
-
-@pytest.fixture(scope="module")
-def start_cluster():
-    try:
-        cluster.start()
-        yield cluster
-    finally:
-        cluster.shutdown()
-
-
-def test_recovery_time_metric(start_cluster):
-    node.query(
-        """
-        DROP DATABASE IF EXISTS rdb;
-        CREATE DATABASE rdb
-        ENGINE = Replicated('/test/test_recovery_time_metric', 'shard1', 'replica1')
-        """
-    )
-
-    node.query(
-        """
-        DROP TABLE IF EXISTS rdb.t;
-        CREATE TABLE rdb.t
-        (
-            `x` UInt32
-        )
-        ENGINE = MergeTree
-        ORDER BY x
-        """
-    )
-
-    node.exec_in_container(["bash", "-c", "rm /var/lib/clickhouse/metadata/rdb/t.sql"])
-
-    node.restart_clickhouse()
-
-    ret = int(
-        node.query(
-            """
-            SELECT recovery_time
-            FROM system.clusters
-            WHERE cluster = 'rdb'
-            """
-        ).strip()
-    )
-    assert ret > 0
-
-    node.query(
-        """
-        DROP DATABASE rdb
-        """
-    )
diff --git a/tests/queries/0_stateless/02117_show_create_table_system.reference b/tests/queries/0_stateless/02117_show_create_table_system.reference
index 32e8b2f4312..cfae4fee6c2 100644
--- a/tests/queries/0_stateless/02117_show_create_table_system.reference
+++ b/tests/queries/0_stateless/02117_show_create_table_system.reference
@@ -52,8 +52,6 @@ CREATE TABLE system.clusters
     `database_shard_name` String,
     `database_replica_name` String,
     `is_active` Nullable(UInt8),
-    `replication_lag` Nullable(UInt32),
-    `recovery_time` Nullable(UInt64),
     `name` String ALIAS cluster
 )
 ENGINE = SystemClusters
diff --git a/tests/queries/0_stateless/03206_replication_lag_metric.reference b/tests/queries/0_stateless/03206_replication_lag_metric.reference
deleted file mode 100644
index 02f4a7264b1..00000000000
--- a/tests/queries/0_stateless/03206_replication_lag_metric.reference
+++ /dev/null
@@ -1,4 +0,0 @@
-0
-2
-0
-2
diff --git a/tests/queries/0_stateless/03206_replication_lag_metric.sql b/tests/queries/0_stateless/03206_replication_lag_metric.sql
deleted file mode 100644
index 998c332a11c..00000000000
--- a/tests/queries/0_stateless/03206_replication_lag_metric.sql
+++ /dev/null
@@ -1,11 +0,0 @@
--- Tags: no-parallel
-
-CREATE DATABASE rdb1 ENGINE = Replicated('/test/test_replication_lag_metric', 'shard1', 'replica1');
-CREATE DATABASE rdb2 ENGINE = Replicated('/test/test_replication_lag_metric', 'shard1', 'replica2');
-
-SET distributed_ddl_task_timeout = 0;
-CREATE TABLE rdb1.t (id UInt32) ENGINE = ReplicatedMergeTree ORDER BY id;
-SELECT replication_lag FROM system.clusters WHERE cluster IN ('rdb1', 'rdb2') ORDER BY cluster ASC, replica_num ASC;
-
-DROP DATABASE rdb1;
-DROP DATABASE rdb2;

From 3dedd8d76b1c3fdb533e9f47d537e4cdf369af5e Mon Sep 17 00:00:00 2001
From: Max K <maxkaynov@gmail.com>
Date: Sat, 3 Aug 2024 10:40:12 +0200
Subject: [PATCH 1314/1488] CI: Minor refactoring in ci_utils

---
 tests/ci/bugfix_validate_check.py |   5 +-
 tests/ci/ci.py                    |   8 +-
 tests/ci/ci_cache.py              |   4 +-
 tests/ci/ci_config.py             |   7 +-
 tests/ci/ci_definitions.py        |  22 -----
 tests/ci/ci_settings.py           |   9 +-
 tests/ci/ci_utils.py              | 150 +++++------------------------
 tests/ci/report.py                |   3 +-
 tests/ci/run_check.py             | 152 +++++++++++++++++++++++++++---
 tests/ci/test_ci_config.py        |  12 ++-
 10 files changed, 186 insertions(+), 186 deletions(-)

diff --git a/tests/ci/bugfix_validate_check.py b/tests/ci/bugfix_validate_check.py
index 71b18572938..932d709a7b8 100644
--- a/tests/ci/bugfix_validate_check.py
+++ b/tests/ci/bugfix_validate_check.py
@@ -8,7 +8,6 @@ from pathlib import Path
 from typing import List, Sequence, Tuple
 
 from ci_config import CI
-from ci_utils import normalize_string
 from env_helper import TEMP_PATH
 from functional_test_check import NO_CHANGES_MSG
 from report import (
@@ -142,7 +141,9 @@ def main():
         for file in set(jr.additional_files):
             file_ = Path(file)
             file_name = file_.name
-            file_name = file_name.replace(".", "__" + normalize_string(job_id) + ".", 1)
+            file_name = file_name.replace(
+                ".", "__" + CI.Utils.normalize_string(job_id) + ".", 1
+            )
             file_ = file_.rename(file_.parent / file_name)
             additional_files.append(file_)
 
diff --git a/tests/ci/ci.py b/tests/ci/ci.py
index 2565c8944e4..e36f2904182 100644
--- a/tests/ci/ci.py
+++ b/tests/ci/ci.py
@@ -16,7 +16,7 @@ import upload_result_helper
 from build_check import get_release_or_pr
 from ci_config import CI
 from ci_metadata import CiMetadata
-from ci_utils import GH, normalize_string, Utils
+from ci_utils import GH, Utils
 from clickhouse_helper import (
     CiLogsCredentials,
     ClickHouseHelper,
@@ -296,7 +296,7 @@ def _pre_action(s3, job_name, batch, indata, pr_info):
         # do not set report prefix for scheduled or dispatched wf (in case it started from feature branch while
         #   testing), otherwise reports won't be found
         if not (pr_info.is_scheduled or pr_info.is_dispatched):
-            report_prefix = normalize_string(pr_info.head_ref)
+            report_prefix = Utils.normalize_string(pr_info.head_ref)
     print(
         f"Use report prefix [{report_prefix}], pr_num [{pr_info.number}], head_ref [{pr_info.head_ref}]"
     )
@@ -718,7 +718,7 @@ def _upload_build_artifacts(
         (
             get_release_or_pr(pr_info, get_version_from_repo())[1],
             pr_info.sha,
-            normalize_string(build_name),
+            Utils.normalize_string(build_name),
             "performance.tar.zst",
         )
     )
@@ -1248,7 +1248,7 @@ def main() -> int:
                     (
                         get_release_or_pr(pr_info, get_version_from_repo())[0],
                         pr_info.sha,
-                        normalize_string(
+                        Utils.normalize_string(
                             job_report.check_name or _get_ext_check_name(args.job_name)
                         ),
                     )
diff --git a/tests/ci/ci_cache.py b/tests/ci/ci_cache.py
index 4846233ab03..a59fd3e5a29 100644
--- a/tests/ci/ci_cache.py
+++ b/tests/ci/ci_cache.py
@@ -7,7 +7,7 @@ from typing import Dict, Optional, Any, Union, Sequence, List, Set
 
 from ci_config import CI
 
-from ci_utils import is_hex, GH
+from ci_utils import Utils, GH
 from commit_status_helper import CommitStatusData
 from env_helper import (
     TEMP_PATH,
@@ -240,7 +240,7 @@ class CiCache:
             int(job_properties[-1]),
         )
 
-        if not is_hex(job_digest):
+        if not Utils.is_hex(job_digest):
             print("ERROR: wrong record job digest")
             return None
 
diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py
index c031ca9b805..ef48466e451 100644
--- a/tests/ci/ci_config.py
+++ b/tests/ci/ci_config.py
@@ -3,7 +3,7 @@ import re
 from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser
 from typing import Dict, Optional, List
 
-from ci_utils import normalize_string
+from ci_utils import Utils
 from ci_definitions import *
 
 
@@ -13,7 +13,6 @@ class CI:
     each config item in the below dicts should be an instance of JobConfig class or inherited from it
     """
 
-    MAX_TOTAL_FAILURES_BEFORE_BLOCKING_CI = 5
     MAX_TOTAL_FAILURES_PER_JOB_BEFORE_BLOCKING_CI = 2
 
     # reimport types to CI class so that they visible as CI.* and mypy is happy
@@ -37,9 +36,7 @@ class CI:
     from ci_utils import GH as GH
     from ci_utils import Shell as Shell
     from ci_definitions import Labels as Labels
-    from ci_definitions import TRUSTED_CONTRIBUTORS as TRUSTED_CONTRIBUTORS
     from ci_definitions import WorkFlowNames as WorkFlowNames
-    from ci_utils import CATEGORY_TO_LABEL as CATEGORY_TO_LABEL
 
     # Jobs that run for doc related updates
     _DOCS_CHECK_JOBS = [JobNames.DOCS_CHECK, JobNames.STYLE_CHECK]
@@ -558,7 +555,7 @@ class CI:
     @classmethod
     def get_tag_config(cls, label_name: str) -> Optional[LabelConfig]:
         for label, config in cls.TAG_CONFIGS.items():
-            if normalize_string(label_name) == normalize_string(label):
+            if Utils.normalize_string(label_name) == Utils.normalize_string(label):
                 return config
         return None
 
diff --git a/tests/ci/ci_definitions.py b/tests/ci/ci_definitions.py
index de6791acda8..795bda3d4b0 100644
--- a/tests/ci/ci_definitions.py
+++ b/tests/ci/ci_definitions.py
@@ -32,28 +32,6 @@ class Labels:
     AUTO_BACKPORT = {"pr-critical-bugfix"}
 
 
-TRUSTED_CONTRIBUTORS = {
-    e.lower()
-    for e in [
-        "amosbird",
-        "azat",  # SEMRush
-        "bharatnc",  # Many contributions.
-        "cwurm",  # ClickHouse, Inc
-        "den-crane",  # Documentation contributor
-        "ildus",  # adjust, ex-pgpro
-        "nvartolomei",  # Seasoned contributor, CloudFlare
-        "taiyang-li",
-        "ucasFL",  # Amos Bird's friend
-        "thomoco",  # ClickHouse, Inc
-        "tonickkozlov",  # Cloudflare
-        "tylerhannan",  # ClickHouse, Inc
-        "tsolodov",  # ClickHouse, Inc
-        "justindeguzman",  # ClickHouse, Inc
-        "XuJia0210",  # ClickHouse, Inc
-    ]
-}
-
-
 class WorkflowStages(metaclass=WithIter):
     """
     Stages of GitHUb actions workflow
diff --git a/tests/ci/ci_settings.py b/tests/ci/ci_settings.py
index d6e9765ceb7..05929179e06 100644
--- a/tests/ci/ci_settings.py
+++ b/tests/ci/ci_settings.py
@@ -2,7 +2,6 @@ import re
 from dataclasses import dataclass, asdict
 from typing import Optional, List, Dict, Any, Iterable
 
-from ci_utils import normalize_string
 from ci_config import CI
 from git_helper import Runner as GitRunner, GIT_PREFIX
 from pr_info import PRInfo
@@ -89,14 +88,14 @@ class CiSettings:
                 if not res.include_keywords:
                     res.include_keywords = []
                 res.include_keywords.append(
-                    normalize_string(match.removeprefix("ci_include_"))
+                    CI.Utils.normalize_string(match.removeprefix("ci_include_"))
                 )
             elif match.startswith("ci_exclude_"):
                 if not res.exclude_keywords:
                     res.exclude_keywords = []
                 keywords = match.removeprefix("ci_exclude_").split("|")
                 res.exclude_keywords += [
-                    normalize_string(keyword) for keyword in keywords
+                    CI.Utils.normalize_string(keyword) for keyword in keywords
                 ]
             elif match == CI.Tags.NO_CI_CACHE:
                 res.no_ci_cache = True
@@ -163,7 +162,7 @@ class CiSettings:
         # do not exclude builds
         if self.exclude_keywords and not CI.is_build_job(job):
             for keyword in self.exclude_keywords:
-                if keyword in normalize_string(job):
+                if keyword in CI.Utils.normalize_string(job):
                     print(f"Job [{job}] matches Exclude keyword [{keyword}] - deny")
                     return False
 
@@ -174,7 +173,7 @@ class CiSettings:
                 # never exclude Style Check by include keywords
                 return True
             for keyword in self.include_keywords:
-                if keyword in normalize_string(job):
+                if keyword in CI.Utils.normalize_string(job):
                     print(f"Job [{job}] matches Include keyword [{keyword}] - pass")
                     return True
             to_deny = True
diff --git a/tests/ci/ci_utils.py b/tests/ci/ci_utils.py
index dae1520afb6..067bedb19c3 100644
--- a/tests/ci/ci_utils.py
+++ b/tests/ci/ci_utils.py
@@ -6,7 +6,7 @@ import sys
 import time
 from contextlib import contextmanager
 from pathlib import Path
-from typing import Any, Iterator, List, Union, Optional, Sequence, Tuple
+from typing import Any, Iterator, List, Union, Optional, Sequence
 
 import requests
 
@@ -20,41 +20,6 @@ class Envs:
     GITHUB_WORKFLOW = os.getenv("GITHUB_WORKFLOW", "")
 
 
-LABEL_CATEGORIES = {
-    "pr-backward-incompatible": ["Backward Incompatible Change"],
-    "pr-bugfix": [
-        "Bug Fix",
-        "Bug Fix (user-visible misbehavior in an official stable release)",
-        "Bug Fix (user-visible misbehaviour in official stable or prestable release)",
-        "Bug Fix (user-visible misbehavior in official stable or prestable release)",
-    ],
-    "pr-critical-bugfix": ["Critical Bug Fix (crash, LOGICAL_ERROR, data loss, RBAC)"],
-    "pr-build": [
-        "Build/Testing/Packaging Improvement",
-        "Build Improvement",
-        "Build/Testing Improvement",
-        "Build",
-        "Packaging Improvement",
-    ],
-    "pr-documentation": [
-        "Documentation (changelog entry is not required)",
-        "Documentation",
-    ],
-    "pr-feature": ["New Feature"],
-    "pr-improvement": ["Improvement"],
-    "pr-not-for-changelog": [
-        "Not for changelog (changelog entry is not required)",
-        "Not for changelog",
-    ],
-    "pr-performance": ["Performance Improvement"],
-    "pr-ci": ["CI Fix or Improvement (changelog entry is not required)"],
-}
-
-CATEGORY_TO_LABEL = {
-    c: lb for lb, categories in LABEL_CATEGORIES.items() for c in categories
-}
-
-
 class WithIter(type):
     def __iter__(cls):
         return (v for k, v in cls.__dict__.items() if not k.startswith("_"))
@@ -70,21 +35,6 @@ def cd(path: Union[Path, str]) -> Iterator[None]:
         os.chdir(oldpwd)
 
 
-def is_hex(s):
-    try:
-        int(s, 16)
-        return True
-    except ValueError:
-        return False
-
-
-def normalize_string(string: str) -> str:
-    res = string.lower()
-    for r in ((" ", "_"), ("(", "_"), (")", "_"), (",", "_"), ("/", "_"), ("-", "_")):
-        res = res.replace(*r)
-    return res
-
-
 class GH:
     class ActionsNames:
         RunConfig = "RunConfig"
@@ -149,8 +99,8 @@ class GH:
     ) -> str:
         assert len(token) == 40
         assert len(commit_sha) == 40
-        assert is_hex(commit_sha)
-        assert not is_hex(token)
+        assert Utils.is_hex(commit_sha)
+        assert not Utils.is_hex(token)
         url = f"https://api.github.com/repos/{Envs.GITHUB_REPOSITORY}/commits/{commit_sha}/statuses?per_page={200}"
         headers = {
             "Authorization": f"token {token}",
@@ -298,79 +248,23 @@ class Utils:
         Shell.check("sudo dmesg --clear", verbose=True)
 
     @staticmethod
-    def check_pr_description(pr_body: str, repo_name: str) -> Tuple[str, str]:
-        """The function checks the body to being properly formatted according to
-        .github/PULL_REQUEST_TEMPLATE.md, if the first returned string is not empty,
-        then there is an error."""
-        lines = list(map(lambda x: x.strip(), pr_body.split("\n") if pr_body else []))
-        lines = [re.sub(r"\s+", " ", line) for line in lines]
+    def is_hex(s):
+        try:
+            int(s, 16)
+            return True
+        except ValueError:
+            return False
 
-        # Check if body contains "Reverts ClickHouse/ClickHouse#36337"
-        if [
-            True for line in lines if re.match(rf"\AReverts {repo_name}#[\d]+\Z", line)
-        ]:
-            return "", LABEL_CATEGORIES["pr-not-for-changelog"][0]
-
-        category = ""
-        entry = ""
-        description_error = ""
-
-        i = 0
-        while i < len(lines):
-            if re.match(r"(?i)^[#>*_ ]*change\s*log\s*category", lines[i]):
-                i += 1
-                if i >= len(lines):
-                    break
-                # Can have one empty line between header and the category
-                # itself. Filter it out.
-                if not lines[i]:
-                    i += 1
-                    if i >= len(lines):
-                        break
-                category = re.sub(r"^[-*\s]*", "", lines[i])
-                i += 1
-
-                # Should not have more than one category. Require empty line
-                # after the first found category.
-                if i >= len(lines):
-                    break
-                if lines[i]:
-                    second_category = re.sub(r"^[-*\s]*", "", lines[i])
-                    description_error = (
-                        "More than one changelog category specified: "
-                        f"'{category}', '{second_category}'"
-                    )
-                    return description_error, category
-
-            elif re.match(
-                r"(?i)^[#>*_ ]*(short\s*description|change\s*log\s*entry)", lines[i]
-            ):
-                i += 1
-                # Can have one empty line between header and the entry itself.
-                # Filter it out.
-                if i < len(lines) and not lines[i]:
-                    i += 1
-                # All following lines until empty one are the changelog entry.
-                entry_lines = []
-                while i < len(lines) and lines[i]:
-                    entry_lines.append(lines[i])
-                    i += 1
-                entry = " ".join(entry_lines)
-                # Don't accept changelog entries like '...'.
-                entry = re.sub(r"[#>*_.\- ]", "", entry)
-                # Don't accept changelog entries like 'Close #12345'.
-                entry = re.sub(r"^[\w\-\s]{0,10}#?\d{5,6}\.?$", "", entry)
-            else:
-                i += 1
-
-        if not category:
-            description_error = "Changelog category is empty"
-        # Filter out the PR categories that are not for changelog.
-        elif "(changelog entry is not required)" in category:
-            pass  # to not check the rest of the conditions
-        elif category not in CATEGORY_TO_LABEL:
-            description_error, category = f"Category '{category}' is not valid", ""
-        elif not entry:
-            description_error = f"Changelog entry required for category '{category}'"
-
-        return description_error, category
+    @staticmethod
+    def normalize_string(string: str) -> str:
+        res = string.lower()
+        for r in (
+            (" ", "_"),
+            ("(", "_"),
+            (")", "_"),
+            (",", "_"),
+            ("/", "_"),
+            ("-", "_"),
+        ):
+            res = res.replace(*r)
+        return res
diff --git a/tests/ci/report.py b/tests/ci/report.py
index f50ed4c1f85..f5571939d0b 100644
--- a/tests/ci/report.py
+++ b/tests/ci/report.py
@@ -22,7 +22,6 @@ from typing import (
 
 from build_download_helper import get_gh_api
 from ci_config import CI
-from ci_utils import normalize_string
 from env_helper import REPORT_PATH, GITHUB_WORKSPACE
 
 logger = logging.getLogger(__name__)
@@ -622,7 +621,7 @@ class BuildResult:
 
     def write_json(self, directory: Union[Path, str] = REPORT_PATH) -> Path:
         path = Path(directory) / self.get_report_name(
-            self.build_name, self.pr_number or normalize_string(self.head_ref)
+            self.build_name, self.pr_number or CI.Utils.normalize_string(self.head_ref)
         )
         path.write_text(
             json.dumps(
diff --git a/tests/ci/run_check.py b/tests/ci/run_check.py
index 0ad01e3accd..55a0c383812 100644
--- a/tests/ci/run_check.py
+++ b/tests/ci/run_check.py
@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 import logging
+import re
 import sys
 from typing import Tuple
 
@@ -16,7 +17,6 @@ from commit_status_helper import (
 from env_helper import GITHUB_REPOSITORY, GITHUB_SERVER_URL
 from get_robot_token import get_best_robot_token
 from ci_config import CI
-from ci_utils import Utils
 from pr_info import PRInfo
 from report import FAILURE, PENDING, SUCCESS, StatusType
 
@@ -25,12 +25,144 @@ TRUSTED_ORG_IDS = {
     54801242,  # clickhouse
 }
 
+TRUSTED_CONTRIBUTORS = {
+    e.lower()
+    for e in [
+        "amosbird",
+        "azat",  # SEMRush
+        "bharatnc",  # Many contributions.
+        "cwurm",  # ClickHouse, Inc
+        "den-crane",  # Documentation contributor
+        "ildus",  # adjust, ex-pgpro
+        "nvartolomei",  # Seasoned contributor, CloudFlare
+        "taiyang-li",
+        "ucasFL",  # Amos Bird's friend
+        "thomoco",  # ClickHouse, Inc
+        "tonickkozlov",  # Cloudflare
+        "tylerhannan",  # ClickHouse, Inc
+        "tsolodov",  # ClickHouse, Inc
+        "justindeguzman",  # ClickHouse, Inc
+        "XuJia0210",  # ClickHouse, Inc
+    ]
+}
+
 OK_SKIP_LABELS = {CI.Labels.RELEASE, CI.Labels.PR_BACKPORT, CI.Labels.PR_CHERRYPICK}
 PR_CHECK = "PR Check"
 
 
+LABEL_CATEGORIES = {
+    "pr-backward-incompatible": ["Backward Incompatible Change"],
+    "pr-bugfix": [
+        "Bug Fix",
+        "Bug Fix (user-visible misbehavior in an official stable release)",
+        "Bug Fix (user-visible misbehaviour in official stable or prestable release)",
+        "Bug Fix (user-visible misbehavior in official stable or prestable release)",
+    ],
+    "pr-critical-bugfix": ["Critical Bug Fix (crash, LOGICAL_ERROR, data loss, RBAC)"],
+    "pr-build": [
+        "Build/Testing/Packaging Improvement",
+        "Build Improvement",
+        "Build/Testing Improvement",
+        "Build",
+        "Packaging Improvement",
+    ],
+    "pr-documentation": [
+        "Documentation (changelog entry is not required)",
+        "Documentation",
+    ],
+    "pr-feature": ["New Feature"],
+    "pr-improvement": ["Improvement"],
+    "pr-not-for-changelog": [
+        "Not for changelog (changelog entry is not required)",
+        "Not for changelog",
+    ],
+    "pr-performance": ["Performance Improvement"],
+    "pr-ci": ["CI Fix or Improvement (changelog entry is not required)"],
+}
+
+CATEGORY_TO_LABEL = {
+    c: lb for lb, categories in LABEL_CATEGORIES.items() for c in categories
+}
+
+
+def check_pr_description(pr_body: str, repo_name: str) -> Tuple[str, str]:
+    """The function checks the body to being properly formatted according to
+    .github/PULL_REQUEST_TEMPLATE.md, if the first returned string is not empty,
+    then there is an error."""
+    lines = list(map(lambda x: x.strip(), pr_body.split("\n") if pr_body else []))
+    lines = [re.sub(r"\s+", " ", line) for line in lines]
+
+    # Check if body contains "Reverts ClickHouse/ClickHouse#36337"
+    if [True for line in lines if re.match(rf"\AReverts {repo_name}#[\d]+\Z", line)]:
+        return "", LABEL_CATEGORIES["pr-not-for-changelog"][0]
+
+    category = ""
+    entry = ""
+    description_error = ""
+
+    i = 0
+    while i < len(lines):
+        if re.match(r"(?i)^[#>*_ ]*change\s*log\s*category", lines[i]):
+            i += 1
+            if i >= len(lines):
+                break
+            # Can have one empty line between header and the category
+            # itself. Filter it out.
+            if not lines[i]:
+                i += 1
+                if i >= len(lines):
+                    break
+            category = re.sub(r"^[-*\s]*", "", lines[i])
+            i += 1
+
+            # Should not have more than one category. Require empty line
+            # after the first found category.
+            if i >= len(lines):
+                break
+            if lines[i]:
+                second_category = re.sub(r"^[-*\s]*", "", lines[i])
+                description_error = (
+                    "More than one changelog category specified: "
+                    f"'{category}', '{second_category}'"
+                )
+                return description_error, category
+
+        elif re.match(
+            r"(?i)^[#>*_ ]*(short\s*description|change\s*log\s*entry)", lines[i]
+        ):
+            i += 1
+            # Can have one empty line between header and the entry itself.
+            # Filter it out.
+            if i < len(lines) and not lines[i]:
+                i += 1
+            # All following lines until empty one are the changelog entry.
+            entry_lines = []
+            while i < len(lines) and lines[i]:
+                entry_lines.append(lines[i])
+                i += 1
+            entry = " ".join(entry_lines)
+            # Don't accept changelog entries like '...'.
+            entry = re.sub(r"[#>*_.\- ]", "", entry)
+            # Don't accept changelog entries like 'Close #12345'.
+            entry = re.sub(r"^[\w\-\s]{0,10}#?\d{5,6}\.?$", "", entry)
+        else:
+            i += 1
+
+    if not category:
+        description_error = "Changelog category is empty"
+    # Filter out the PR categories that are not for changelog.
+    elif "(changelog entry is not required)" in category:
+        pass  # to not check the rest of the conditions
+    elif category not in CATEGORY_TO_LABEL:
+        description_error, category = f"Category '{category}' is not valid", ""
+    elif not entry:
+        description_error = f"Changelog entry required for category '{category}'"
+
+    return description_error, category
+
+
 def pr_is_by_trusted_user(pr_user_login, pr_user_orgs):
-    if pr_user_login.lower() in CI.TRUSTED_CONTRIBUTORS:
+    if pr_user_login.lower() in TRUSTED_CONTRIBUTORS:
         logging.info("User '%s' is trusted", pr_user_login)
         return True
 
@@ -92,22 +224,20 @@ def main():
     commit = get_commit(gh, pr_info.sha)
     status = SUCCESS  # type: StatusType
 
-    description_error, category = Utils.check_pr_description(
-        pr_info.body, GITHUB_REPOSITORY
-    )
+    description_error, category = check_pr_description(pr_info.body, GITHUB_REPOSITORY)
     pr_labels_to_add = []
     pr_labels_to_remove = []
     if (
-        category in CI.CATEGORY_TO_LABEL
-        and CI.CATEGORY_TO_LABEL[category] not in pr_info.labels
+        category in CATEGORY_TO_LABEL
+        and CATEGORY_TO_LABEL[category] not in pr_info.labels
     ):
-        pr_labels_to_add.append(CI.CATEGORY_TO_LABEL[category])
+        pr_labels_to_add.append(CATEGORY_TO_LABEL[category])
 
     for label in pr_info.labels:
         if (
-            label in CI.CATEGORY_TO_LABEL.values()
-            and category in CI.CATEGORY_TO_LABEL
-            and label != CI.CATEGORY_TO_LABEL[category]
+            label in CATEGORY_TO_LABEL.values()
+            and category in CATEGORY_TO_LABEL
+            and label != CATEGORY_TO_LABEL[category]
         ):
             pr_labels_to_remove.append(label)
 
diff --git a/tests/ci/test_ci_config.py b/tests/ci/test_ci_config.py
index f376a129e6f..6ffedfdecd4 100644
--- a/tests/ci/test_ci_config.py
+++ b/tests/ci/test_ci_config.py
@@ -9,7 +9,7 @@ from ci_settings import CiSettings
 from pr_info import PRInfo, EventType
 from s3_helper import S3Helper
 from ci_cache import CiCache
-from ci_utils import normalize_string
+from ci_utils import Utils
 
 
 _TEST_EVENT_JSON = {"dummy": "dummy"}
@@ -55,7 +55,7 @@ class TestCIConfig(unittest.TestCase):
             if CI.JOB_CONFIGS[job].job_name_keyword:
                 self.assertTrue(
                     CI.JOB_CONFIGS[job].job_name_keyword.lower()
-                    in normalize_string(job),
+                    in Utils.normalize_string(job),
                     f"Job [{job}] apparently uses wrong common config with job keyword [{CI.JOB_CONFIGS[job].job_name_keyword}]",
                 )
 
@@ -291,7 +291,9 @@ class TestCIConfig(unittest.TestCase):
             assert tag_config
             set_jobs = tag_config.run_jobs
             for job in set_jobs:
-                if any(k in normalize_string(job) for k in settings.exclude_keywords):
+                if any(
+                    k in Utils.normalize_string(job) for k in settings.exclude_keywords
+                ):
                     continue
                 expected_jobs_to_do.append(job)
         for job, config in CI.JOB_CONFIGS.items():
@@ -303,12 +305,12 @@ class TestCIConfig(unittest.TestCase):
                 # expected to run all builds jobs
                 expected_jobs_to_do.append(job)
             if not any(
-                keyword in normalize_string(job)
+                keyword in Utils.normalize_string(job)
                 for keyword in settings.include_keywords
             ):
                 continue
             if any(
-                keyword in normalize_string(job)
+                keyword in Utils.normalize_string(job)
                 for keyword in settings.exclude_keywords
             ):
                 continue

From a19750234153e760907f3c7bc040f949100534df Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Sat, 3 Aug 2024 23:11:20 +0200
Subject: [PATCH 1315/1488] Fix test retries

Should fix issues like:
- 02494_zero_copy_projection_cancel_fetch - https://s3.amazonaws.com/clickhouse-test-reports/67719/40cd5467c18d65a6624d273ac1a8fd9cc9257d8c/stateless_tests__tsan__s3_storage__[4_4].html

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 tests/clickhouse-test | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/clickhouse-test b/tests/clickhouse-test
index a29c786e998..877548e577e 100755
--- a/tests/clickhouse-test
+++ b/tests/clickhouse-test
@@ -2218,7 +2218,6 @@ def run_tests_array(all_tests_with_params: Tuple[List[str], int, TestSuite, bool
                         args, test_suite, client_options, server_logs_level
                     )
                     test_result = test_case.process_result(test_result, MESSAGES)
-                    break
                 except TimeoutError:
                     break
                 finally:

From 087aff87dd23b4821965904e0a760b3ec7c4f8f4 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 3 Aug 2024 23:12:07 +0200
Subject: [PATCH 1316/1488] Something

---
 src/Databases/DatabaseLazy.cpp     | 2 +-
 src/Databases/DatabaseOnDisk.cpp   | 8 ++++----
 src/Databases/DatabaseOnDisk.h     | 2 +-
 src/Databases/DatabaseOrdinary.cpp | 2 +-
 4 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/Databases/DatabaseLazy.cpp b/src/Databases/DatabaseLazy.cpp
index ca30ee6db15..baa6910f6a8 100644
--- a/src/Databases/DatabaseLazy.cpp
+++ b/src/Databases/DatabaseLazy.cpp
@@ -52,7 +52,7 @@ DatabaseLazy::DatabaseLazy(const String & name_, const String & metadata_path_,
 
 void DatabaseLazy::loadStoredObjects(ContextMutablePtr local_context, LoadingStrictnessLevel /*mode*/)
 {
-    iterateMetadataFiles(local_context, [this, &local_context](const String & file_name)
+    iterateMetadataFiles([this, &local_context](const String & file_name)
     {
         const std::string table_name = unescapeForFileName(file_name.substr(0, file_name.size() - 4));
 
diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp
index 01d8867661b..82a81b0b32d 100644
--- a/src/Databases/DatabaseOnDisk.cpp
+++ b/src/Databases/DatabaseOnDisk.cpp
@@ -579,14 +579,14 @@ void DatabaseOnDisk::drop(ContextPtr local_context)
     assert(TSA_SUPPRESS_WARNING_FOR_READ(tables).empty());
     if (local_context->getSettingsRef().force_remove_data_recursively_on_drop)
     {
-        (void)fs::remove_all(local_context->getPath() + getDataPath());
+        (void)fs::remove_all(std::filesystem::path(getContext()->getPath()) / data_path);
         (void)fs::remove_all(getMetadataPath());
     }
     else
     {
         try
         {
-            (void)fs::remove(local_context->getPath() + getDataPath());
+            (void)fs::remove(std::filesystem::path(getContext()->getPath()) / data_path);
             (void)fs::remove(getMetadataPath());
         }
         catch (const fs::filesystem_error & e)
@@ -624,7 +624,7 @@ time_t DatabaseOnDisk::getObjectMetadataModificationTime(const String & object_n
     }
 }
 
-void DatabaseOnDisk::iterateMetadataFiles(ContextPtr local_context, const IteratingFunction & process_metadata_file) const
+void DatabaseOnDisk::iterateMetadataFiles(const IteratingFunction & process_metadata_file) const
 {
     if (!fs::exists(metadata_path))
         return;
@@ -635,7 +635,7 @@ void DatabaseOnDisk::iterateMetadataFiles(ContextPtr local_context, const Iterat
         static const char * tmp_drop_ext = ".sql.tmp_drop";
         const std::string object_name = file_name.substr(0, file_name.size() - strlen(tmp_drop_ext));
 
-        if (fs::exists(local_context->getPath() + getDataPath() + '/' + object_name))
+        if (fs::exists(std::filesystem::path(getContext()->getPath()) / data_path / object_name))
         {
             fs::rename(getMetadataPath() + file_name, getMetadataPath() + object_name + ".sql");
             LOG_WARNING(log, "Object {} was not dropped previously and will be restored", backQuote(object_name));
diff --git a/src/Databases/DatabaseOnDisk.h b/src/Databases/DatabaseOnDisk.h
index a8be674a4e2..0c0ecf76a26 100644
--- a/src/Databases/DatabaseOnDisk.h
+++ b/src/Databases/DatabaseOnDisk.h
@@ -83,7 +83,7 @@ protected:
 
     using IteratingFunction = std::function<void(const String &)>;
 
-    void iterateMetadataFiles(ContextPtr context, const IteratingFunction & process_metadata_file) const;
+    void iterateMetadataFiles(const IteratingFunction & process_metadata_file) const;
 
     ASTPtr getCreateTableQueryImpl(
         const String & table_name,
diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp
index 49719c25319..dd8a3f42ea8 100644
--- a/src/Databases/DatabaseOrdinary.cpp
+++ b/src/Databases/DatabaseOrdinary.cpp
@@ -265,7 +265,7 @@ void DatabaseOrdinary::loadTablesMetadata(ContextPtr local_context, ParsedTables
         }
     };
 
-    iterateMetadataFiles(local_context, process_metadata);
+    iterateMetadataFiles(process_metadata);
 
     size_t objects_in_database = metadata.parsed_tables.size() - prev_tables_count;
     size_t dictionaries_in_database = metadata.total_dictionaries - prev_total_dictionaries;

From ced8c3445c929efdb62ec707a7a8af7ef1a9f541 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 3 Aug 2024 23:14:40 +0200
Subject: [PATCH 1317/1488] Something

---
 src/Databases/DatabaseLazy.cpp | 1 +
 src/Databases/DatabaseLazy.h   | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/Databases/DatabaseLazy.cpp b/src/Databases/DatabaseLazy.cpp
index baa6910f6a8..e80ee930d79 100644
--- a/src/Databases/DatabaseLazy.cpp
+++ b/src/Databases/DatabaseLazy.cpp
@@ -47,6 +47,7 @@ DatabaseLazy::DatabaseLazy(const String & name_, const String & metadata_path_,
     : DatabaseOnDisk(name_, metadata_path_, "data/" + escapeForFileName(name_) + "/", "DatabaseLazy (" + name_ + ")", context_)
     , expiration_time(expiration_time_)
 {
+    createDirectories();
 }
 
 
diff --git a/src/Databases/DatabaseLazy.h b/src/Databases/DatabaseLazy.h
index 4347649117d..aeac130594f 100644
--- a/src/Databases/DatabaseLazy.h
+++ b/src/Databases/DatabaseLazy.h
@@ -12,7 +12,7 @@ class DatabaseLazyIterator;
 class Context;
 
 /** Lazy engine of databases.
-  * Works like DatabaseOrdinary, but stores in memory only cache.
+  * Works like DatabaseOrdinary, but stores only recently accessed tables in memory.
   * Can be used only with *Log engines.
   */
 class DatabaseLazy final : public DatabaseOnDisk

From f06ae2f5518ff8cb610b337d4900fd6f0088190f Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Sat, 3 Aug 2024 23:27:19 +0200
Subject: [PATCH 1318/1488] Fill only selected columns from system.clusters

Some of them pretty heavy, i.e. is_active for ReplicatedDatabase

This should fix 02903_rmt_retriable_merge_exception flakiness [1].

 [1]: https://s3.amazonaws.com/clickhouse-test-reports/67687/89c47df559ba23d988f8af3c342e0c8d5531f4b8/fast_test.html

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 src/Storages/System/StorageSystemClusters.cpp | 82 ++++++++++++-------
 src/Storages/System/StorageSystemClusters.h   |  6 +-
 2 files changed, 56 insertions(+), 32 deletions(-)

diff --git a/src/Storages/System/StorageSystemClusters.cpp b/src/Storages/System/StorageSystemClusters.cpp
index 160c8d6270e..9c5c07ae49f 100644
--- a/src/Storages/System/StorageSystemClusters.cpp
+++ b/src/Storages/System/StorageSystemClusters.cpp
@@ -40,10 +40,10 @@ ColumnsDescription StorageSystemClusters::getColumnsDescription()
     return description;
 }
 
-void StorageSystemClusters::fillData(MutableColumns & res_columns, ContextPtr context, const ActionsDAG::Node *, std::vector<UInt8>) const
+void StorageSystemClusters::fillData(MutableColumns & res_columns, ContextPtr context, const ActionsDAG::Node *, std::vector<UInt8> columns_mask) const
 {
     for (const auto & name_and_cluster : context->getClusters())
-        writeCluster(res_columns, name_and_cluster, {});
+        writeCluster(res_columns, columns_mask, name_and_cluster, /* replicated= */ nullptr);
 
     const auto databases = DatabaseCatalog::instance().getDatabases();
     for (const auto & name_and_database : databases)
@@ -52,18 +52,15 @@ void StorageSystemClusters::fillData(MutableColumns & res_columns, ContextPtr co
         {
 
             if (auto database_cluster = replicated->tryGetCluster())
-                writeCluster(res_columns, {name_and_database.first, database_cluster},
-                             replicated->tryGetAreReplicasActive(database_cluster));
+                writeCluster(res_columns, columns_mask, {name_and_database.first, database_cluster}, replicated);
 
             if (auto database_cluster = replicated->tryGetAllGroupsCluster())
-                writeCluster(res_columns, {DatabaseReplicated::ALL_GROUPS_CLUSTER_PREFIX + name_and_database.first, database_cluster},
-                             replicated->tryGetAreReplicasActive(database_cluster));
+                writeCluster(res_columns, columns_mask, {DatabaseReplicated::ALL_GROUPS_CLUSTER_PREFIX + name_and_database.first, database_cluster}, replicated);
         }
     }
 }
 
-void StorageSystemClusters::writeCluster(MutableColumns & res_columns, const NameAndCluster & name_and_cluster,
-                                         const std::vector<UInt8> & is_active)
+void StorageSystemClusters::writeCluster(MutableColumns & res_columns, const std::vector<UInt8> & columns_mask, const NameAndCluster & name_and_cluster, const DatabaseReplicated * replicated)
 {
     const String & cluster_name = name_and_cluster.first;
     const ClusterPtr & cluster = name_and_cluster.second;
@@ -79,30 +76,55 @@ void StorageSystemClusters::writeCluster(MutableColumns & res_columns, const Nam
 
         for (size_t replica_index = 0; replica_index < shard_addresses.size(); ++replica_index)
         {
-            size_t i = 0;
+            size_t src_index = 0, res_index = 0;
             const auto & address = shard_addresses[replica_index];
 
-            res_columns[i++]->insert(cluster_name);
-            res_columns[i++]->insert(shard_info.shard_num);
-            res_columns[i++]->insert(shard_info.weight);
-            res_columns[i++]->insert(shard_info.has_internal_replication);
-            res_columns[i++]->insert(replica_index + 1);
-            res_columns[i++]->insert(address.host_name);
-            auto resolved = address.getResolvedAddress();
-            res_columns[i++]->insert(resolved ? resolved->host().toString() : String());
-            res_columns[i++]->insert(address.port);
-            res_columns[i++]->insert(address.is_local);
-            res_columns[i++]->insert(address.user);
-            res_columns[i++]->insert(address.default_database);
-            res_columns[i++]->insert(pool_status[replica_index].error_count);
-            res_columns[i++]->insert(pool_status[replica_index].slowdown_count);
-            res_columns[i++]->insert(pool_status[replica_index].estimated_recovery_time.count());
-            res_columns[i++]->insert(address.database_shard_name);
-            res_columns[i++]->insert(address.database_replica_name);
-            if (is_active.empty())
-                res_columns[i++]->insertDefault();
-            else
-                res_columns[i++]->insert(is_active[replica_idx++]);
+            if (columns_mask[src_index++])
+                res_columns[res_index++]->insert(cluster_name);
+            if (columns_mask[src_index++])
+                res_columns[res_index++]->insert(shard_info.shard_num);
+            if (columns_mask[src_index++])
+                res_columns[res_index++]->insert(shard_info.weight);
+            if (columns_mask[src_index++])
+                res_columns[res_index++]->insert(shard_info.has_internal_replication);
+            if (columns_mask[src_index++])
+                res_columns[res_index++]->insert(replica_index + 1);
+            if (columns_mask[src_index++])
+                res_columns[res_index++]->insert(address.host_name);
+            if (columns_mask[src_index++])
+            {
+                auto resolved = address.getResolvedAddress();
+                res_columns[res_index++]->insert(resolved ? resolved->host().toString() : String());
+            }
+            if (columns_mask[src_index++])
+                res_columns[res_index++]->insert(address.port);
+            if (columns_mask[src_index++])
+                res_columns[res_index++]->insert(address.is_local);
+            if (columns_mask[src_index++])
+                res_columns[res_index++]->insert(address.user);
+            if (columns_mask[src_index++])
+                res_columns[res_index++]->insert(address.default_database);
+            if (columns_mask[src_index++])
+                res_columns[res_index++]->insert(pool_status[replica_index].error_count);
+            if (columns_mask[src_index++])
+                res_columns[res_index++]->insert(pool_status[replica_index].slowdown_count);
+            if (columns_mask[src_index++])
+                res_columns[res_index++]->insert(pool_status[replica_index].estimated_recovery_time.count());
+            if (columns_mask[src_index++])
+                res_columns[res_index++]->insert(address.database_shard_name);
+            if (columns_mask[src_index++])
+                res_columns[res_index++]->insert(address.database_replica_name);
+            if (columns_mask[src_index++])
+            {
+                std::vector<UInt8> is_active;
+                if (replicated)
+                    is_active = replicated->tryGetAreReplicasActive(name_and_cluster.second);
+
+                if (is_active.empty())
+                    res_columns[res_index++]->insertDefault();
+                else
+                    res_columns[res_index++]->insert(is_active[replica_idx++]);
+            }
         }
     }
 }
diff --git a/src/Storages/System/StorageSystemClusters.h b/src/Storages/System/StorageSystemClusters.h
index 0f7c792261d..f6adb902f43 100644
--- a/src/Storages/System/StorageSystemClusters.h
+++ b/src/Storages/System/StorageSystemClusters.h
@@ -10,6 +10,7 @@ namespace DB
 
 class Context;
 class Cluster;
+class DatabaseReplicated;
 
 /** Implements system table 'clusters'
   *  that allows to obtain information about available clusters
@@ -26,8 +27,9 @@ protected:
     using IStorageSystemOneBlock::IStorageSystemOneBlock;
     using NameAndCluster = std::pair<String, std::shared_ptr<Cluster>>;
 
-    void fillData(MutableColumns & res_columns, ContextPtr context, const ActionsDAG::Node *, std::vector<UInt8>) const override;
-    static void writeCluster(MutableColumns & res_columns, const NameAndCluster & name_and_cluster, const std::vector<UInt8> & is_active);
+    void fillData(MutableColumns & res_columns, ContextPtr context, const ActionsDAG::Node *, std::vector<UInt8> columns_mask) const override;
+    static void writeCluster(MutableColumns & res_columns, const std::vector<UInt8> & columns_mask, const NameAndCluster & name_and_cluster, const DatabaseReplicated * replicated);
+    bool supportsColumnsMask() const override { return true; }
 };
 
 }

From 9d0e066cda8d0ccb6bd4f9e07fee36a2bfae707a Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Sat, 3 Aug 2024 23:41:10 +0200
Subject: [PATCH 1319/1488] Bump NuRaft (to properly catch thread exceptions)

Refs: https://github.com/ClickHouse/NuRaft/pull/75
Refs: https://github.com/eBay/NuRaft/pull/525
Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 contrib/NuRaft | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contrib/NuRaft b/contrib/NuRaft
index cb5dc3c906e..c2b0811f164 160000
--- a/contrib/NuRaft
+++ b/contrib/NuRaft
@@ -1 +1 @@
-Subproject commit cb5dc3c906e80f253e9ce9535807caef827cc2e0
+Subproject commit c2b0811f164a7948208489562dab4f186eb305ce

From 8562a6106c286882f26383086e52e399106893be Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 3 Aug 2024 23:45:18 +0200
Subject: [PATCH 1320/1488] Better safety thresholds in `arrayWithConstant`

---
 src/Functions/array/arrayWithConstant.cpp | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/src/Functions/array/arrayWithConstant.cpp b/src/Functions/array/arrayWithConstant.cpp
index 48262870553..4cbc6404b9b 100644
--- a/src/Functions/array/arrayWithConstant.cpp
+++ b/src/Functions/array/arrayWithConstant.cpp
@@ -1,9 +1,9 @@
 #include <Functions/IFunction.h>
-#include <Functions/FunctionHelpers.h>
 #include <Functions/FunctionFactory.h>
 #include <DataTypes/DataTypeArray.h>
 #include <DataTypes/DataTypesNumber.h>
 #include <Columns/ColumnArray.h>
+#include <base/arithmeticOverflow.h>
 
 
 namespace DB
@@ -15,7 +15,8 @@ namespace ErrorCodes
     extern const int TOO_LARGE_ARRAY_SIZE;
 }
 
-/// Reasonable threshold.
+/// Reasonable thresholds.
+static constexpr Int64 max_array_size_in_columns_bytes = 1000000000;
 static constexpr size_t max_arrays_size_in_columns = 1000000000;
 
 
@@ -63,12 +64,19 @@ public:
             auto array_size = col_num->getInt(i);
 
             if (unlikely(array_size < 0))
-                throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Array size cannot be negative: while executing function {}", getName());
+                throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Array size {} cannot be negative: while executing function {}", array_size, getName());
+
+            Int64 estimated_size = 0;
+            if (unlikely(common::mulOverflow(array_size, col_value->byteSize(), estimated_size)))
+                throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Array size {} with element size {} bytes is too large: while executing function {}", array_size, col_value->byteSize(), getName());
+
+            if (unlikely(estimated_size > max_array_size_in_columns_bytes))
+                throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Array size {} with element size {} bytes is too large: while executing function {}", array_size, col_value->byteSize(), getName());
 
             offset += array_size;
 
             if (unlikely(offset > max_arrays_size_in_columns))
-                throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Too large array size while executing function {}", getName());
+                throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Too large array size {} (will generate at least {} elements) while executing function {}", array_size, offset, getName());
 
             offsets.push_back(offset);
         }

From 7a066a6505108b14bd49da8766c1bc473a978b1f Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 3 Aug 2024 23:48:51 +0200
Subject: [PATCH 1321/1488] Add a test

---
 .../0_stateless/03216_arrayWithConstant_limits.reference       | 1 +
 tests/queries/0_stateless/03216_arrayWithConstant_limits.sql   | 3 +++
 2 files changed, 4 insertions(+)
 create mode 100644 tests/queries/0_stateless/03216_arrayWithConstant_limits.reference
 create mode 100644 tests/queries/0_stateless/03216_arrayWithConstant_limits.sql

diff --git a/tests/queries/0_stateless/03216_arrayWithConstant_limits.reference b/tests/queries/0_stateless/03216_arrayWithConstant_limits.reference
new file mode 100644
index 00000000000..825319e1c5b
--- /dev/null
+++ b/tests/queries/0_stateless/03216_arrayWithConstant_limits.reference
@@ -0,0 +1 @@
+10000000
diff --git a/tests/queries/0_stateless/03216_arrayWithConstant_limits.sql b/tests/queries/0_stateless/03216_arrayWithConstant_limits.sql
new file mode 100644
index 00000000000..c46524c50e6
--- /dev/null
+++ b/tests/queries/0_stateless/03216_arrayWithConstant_limits.sql
@@ -0,0 +1,3 @@
+SELECT arrayWithConstant(96142475, ['qMUF']); -- { serverError TOO_LARGE_ARRAY_SIZE }
+SELECT arrayWithConstant(100000000, materialize([[[[[[[[[['Hello, world!']]]]]]]]]])); -- { serverError TOO_LARGE_ARRAY_SIZE }
+SELECT length(arrayWithConstant(10000000, materialize([[[[[[[[[['Hello world']]]]]]]]]])));

From 6013e4b81d29f2b46bec64859132a4e2ff1bbdc9 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Wed, 17 Jul 2024 20:52:48 +0200
Subject: [PATCH 1322/1488] poco/MongoDB: Support broader UUID types (generated
 by python uuid.UUID)

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 base/poco/MongoDB/src/Binary.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/base/poco/MongoDB/src/Binary.cpp b/base/poco/MongoDB/src/Binary.cpp
index ea814d6969f..47f3453630f 100644
--- a/base/poco/MongoDB/src/Binary.cpp
+++ b/base/poco/MongoDB/src/Binary.cpp
@@ -76,7 +76,7 @@ std::string Binary::toString(int indent) const
 
 UUID Binary::uuid() const
 {
-	if (_subtype == 0x04 && _buffer.size() == 16)
+	if ((_subtype == 0x04 || _subtype == 0x03) && _buffer.size() == 16)
 	{
 		UUID uuid;
 		uuid.copyFrom((const char*) _buffer.begin());

From 2f00c962711e13ca00af324366421fe4593b4ce6 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Wed, 17 Jul 2024 20:34:15 +0200
Subject: [PATCH 1323/1488] Support true UUID type for MongoDB engine

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 base/poco/Foundation/include/Poco/UUID.h      |  7 +++
 base/poco/MongoDB/src/Binary.cpp              |  2 +-
 src/Processors/Sources/MongoDBSource.cpp      | 46 +++++++++++++++++--
 .../integration/test_storage_mongodb/test.py  | 33 +++++++++++++
 4 files changed, 82 insertions(+), 6 deletions(-)

diff --git a/base/poco/Foundation/include/Poco/UUID.h b/base/poco/Foundation/include/Poco/UUID.h
index df67ef73e4b..6466d226b2e 100644
--- a/base/poco/Foundation/include/Poco/UUID.h
+++ b/base/poco/Foundation/include/Poco/UUID.h
@@ -19,6 +19,7 @@
 
 
 #include "Poco/Foundation.h"
+#include <Poco/Types.h>
 
 
 namespace Poco
@@ -135,6 +136,12 @@ public:
     static const UUID & x500();
     /// Returns the namespace identifier for the X500 namespace.
 
+    UInt32 getTimeLow() const { return _timeLow; }
+    UInt16 getTimeMid() const { return _timeMid; }
+    UInt16 getTimeHiAndVersion() const { return _timeHiAndVersion; }
+    UInt16 getClockSeq() const { return _clockSeq; }
+    std::array<UInt8, 6> getNode() const { return std::array<UInt8, 6>{_node[0], _node[1], _node[2], _node[3], _node[4], _node[5]}; }
+
 protected:
     UUID(UInt32 timeLow, UInt32 timeMid, UInt32 timeHiAndVersion, UInt16 clockSeq, UInt8 node[]);
     UUID(const char * bytes, Version version);
diff --git a/base/poco/MongoDB/src/Binary.cpp b/base/poco/MongoDB/src/Binary.cpp
index 47f3453630f..8b0e6baeccb 100644
--- a/base/poco/MongoDB/src/Binary.cpp
+++ b/base/poco/MongoDB/src/Binary.cpp
@@ -82,7 +82,7 @@ UUID Binary::uuid() const
 		uuid.copyFrom((const char*) _buffer.begin());
 		return uuid;
 	}
-	throw BadCastException("Invalid subtype");
+	throw BadCastException("Invalid subtype: " + std::to_string(_subtype) + ", size: " + std::to_string(_buffer.size()));
 }
 
 
diff --git a/src/Processors/Sources/MongoDBSource.cpp b/src/Processors/Sources/MongoDBSource.cpp
index 0d583cf6be5..e00a541b300 100644
--- a/src/Processors/Sources/MongoDBSource.cpp
+++ b/src/Processors/Sources/MongoDBSource.cpp
@@ -4,6 +4,7 @@
 #include <vector>
 
 #include <Poco/MongoDB/Array.h>
+#include <Poco/MongoDB/Binary.h>
 #include <Poco/MongoDB/Database.h>
 #include <Poco/MongoDB/Connection.h>
 #include <Poco/MongoDB/Cursor.h>
@@ -17,6 +18,7 @@
 #include <IO/ReadHelpers.h>
 #include <Common/assert_cast.h>
 #include <Common/quoteString.h>
+#include "base/types.h"
 #include <base/range.h>
 #include <Poco/URI.h>
 
@@ -45,8 +47,28 @@ namespace
     using ValueType = ExternalResultDescription::ValueType;
     using ObjectId = Poco::MongoDB::ObjectId;
     using MongoArray = Poco::MongoDB::Array;
+    using MongoUUID = Poco::MongoDB::Binary::Ptr;
 
 
+    UUID parsePocoUUID(const Poco::UUID & src)
+    {
+        UUID uuid;
+
+        std::array<Poco::UInt8, 6> src_node = src.getNode();
+        UInt64 node = 0;
+        node |= UInt64(src_node[0]) << 40;
+        node |= UInt64(src_node[1]) << 32;
+        node |= UInt64(src_node[2]) << 24;
+        node |= UInt64(src_node[3]) << 16;
+        node |= UInt64(src_node[4]) << 8;
+        node |= src_node[5];
+
+        UUIDHelpers::getHighBytes(uuid) = UInt64(src.getTimeLow()) << 32 | UInt32(src.getTimeMid() << 16 | src.getTimeHiAndVersion());
+        UUIDHelpers::getLowBytes(uuid) = UInt64(src.getClockSeq()) << 48 | node;
+
+        return uuid;
+    }
+
     template <typename T>
     Field getNumber(const Poco::MongoDB::Element & value, const std::string & name)
     {
@@ -149,12 +171,20 @@ namespace
         else if (which.isUUID())
             parser = [](const Poco::MongoDB::Element & value, const std::string & name) -> Field
             {
-                if (value.type() != Poco::MongoDB::ElementTraits<String>::TypeId)
-                    throw Exception(ErrorCodes::TYPE_MISMATCH, "Type mismatch, expected String (UUID), got type id = {} for column {}",
+                if (value.type() == Poco::MongoDB::ElementTraits<String>::TypeId)
+                {
+                    String string = static_cast<const Poco::MongoDB::ConcreteElement<String> &>(value).value();
+                    return parse<UUID>(string);
+                }
+                else if (value.type() == Poco::MongoDB::ElementTraits<MongoUUID>::TypeId)
+                {
+                    const Poco::UUID & poco_uuid = static_cast<const Poco::MongoDB::ConcreteElement<MongoUUID> &>(value).value()->uuid();
+                    return parsePocoUUID(poco_uuid);
+                }
+                else
+                    throw Exception(ErrorCodes::TYPE_MISMATCH, "Type mismatch, expected String/UUID, got type id = {} for column {}",
                                         toString(value.type()), name);
 
-                String string = static_cast<const Poco::MongoDB::ConcreteElement<String> &>(value).value();
-                return parse<UUID>(string);
             };
         else
             throw Exception(ErrorCodes::BAD_ARGUMENTS, "Type conversion to {} is not supported", nested->getName());
@@ -286,8 +316,14 @@ namespace
                     String string = static_cast<const Poco::MongoDB::ConcreteElement<String> &>(value).value();
                     assert_cast<ColumnUUID &>(column).getData().push_back(parse<UUID>(string));
                 }
+                else if (value.type() == Poco::MongoDB::ElementTraits<MongoUUID>::TypeId)
+                {
+                    const Poco::UUID & poco_uuid = static_cast<const Poco::MongoDB::ConcreteElement<MongoUUID> &>(value).value()->uuid();
+                    UUID uuid = parsePocoUUID(poco_uuid);
+                    assert_cast<ColumnUUID &>(column).getData().push_back(uuid);
+                }
                 else
-                    throw Exception(ErrorCodes::TYPE_MISMATCH, "Type mismatch, expected String (UUID), got type id = {} for column {}",
+                    throw Exception(ErrorCodes::TYPE_MISMATCH, "Type mismatch, expected String/UUID, got type id = {} for column {}",
                                         toString(value.type()), name);
                 break;
             }
diff --git a/tests/integration/test_storage_mongodb/test.py b/tests/integration/test_storage_mongodb/test.py
index 3957afe8b29..1a1a790e8e8 100644
--- a/tests/integration/test_storage_mongodb/test.py
+++ b/tests/integration/test_storage_mongodb/test.py
@@ -1,4 +1,5 @@
 import pymongo
+from uuid import UUID
 
 import pytest
 from helpers.client import QueryRuntimeException
@@ -72,6 +73,28 @@ def test_simple_select(started_cluster):
     simple_mongo_table.drop()
 
 
+@pytest.mark.parametrize("started_cluster", [False], indirect=["started_cluster"])
+def test_uuid(started_cluster):
+    mongo_connection = get_mongo_connection(started_cluster)
+    db = mongo_connection["test"]
+    db.add_user("root", "clickhouse")
+    mongo_table = db["uuid_table"]
+    mongo_table.insert({"key": 0, "data": UUID("f0e77736-91d1-48ce-8f01-15123ca1c7ed")})
+
+    node = started_cluster.instances["node"]
+    node.query(
+        "CREATE TABLE uuid_mongo_table(key UInt64, data UUID) ENGINE = MongoDB('mongo1:27017', 'test', 'uuid_table', 'root', 'clickhouse')"
+    )
+
+    assert node.query("SELECT COUNT() FROM uuid_mongo_table") == "1\n"
+    assert (
+        node.query("SELECT data from uuid_mongo_table where key = 0")
+        == "f0e77736-91d1-48ce-8f01-15123ca1c7ed\n"
+    )
+    node.query("DROP TABLE uuid_mongo_table")
+    mongo_table.drop()
+
+
 @pytest.mark.parametrize("started_cluster", [False], indirect=["started_cluster"])
 def test_simple_select_from_view(started_cluster):
     mongo_connection = get_mongo_connection(started_cluster)
@@ -140,6 +163,10 @@ def test_arrays(started_cluster):
                     "f0e77736-91d1-48ce-8f01-15123ca1c7ed",
                     "93376a07-c044-4281-a76e-ad27cf6973c5",
                 ],
+                "arr_mongo_uuid": [
+                    UUID("f0e77736-91d1-48ce-8f01-15123ca1c7ed"),
+                    UUID("93376a07-c044-4281-a76e-ad27cf6973c5"),
+                ],
                 "arr_arr_bool": [
                     [True, False, True],
                     [True],
@@ -174,6 +201,7 @@ def test_arrays(started_cluster):
         "arr_datetime Array(DateTime),"
         "arr_string Array(String),"
         "arr_uuid Array(UUID),"
+        "arr_mongo_uuid Array(UUID),"
         "arr_arr_bool Array(Array(Bool)),"
         "arr_empty Array(UInt64),"
         "arr_null Array(UInt64),"
@@ -222,6 +250,11 @@ def test_arrays(started_cluster):
         == "['f0e77736-91d1-48ce-8f01-15123ca1c7ed','93376a07-c044-4281-a76e-ad27cf6973c5']\n"
     )
 
+    assert (
+        node.query(f"SELECT arr_mongo_uuid FROM arrays_mongo_table WHERE key = 42")
+        == "['f0e77736-91d1-48ce-8f01-15123ca1c7ed','93376a07-c044-4281-a76e-ad27cf6973c5']\n"
+    )
+
     assert (
         node.query(f"SELECT arr_arr_bool FROM arrays_mongo_table WHERE key = 42")
         == "[[true,false,true],[true],[],[],[false],[false]]\n"

From a051eb2a5d1c29a61d3c2d9e2f3f0841bbb2817c Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 4 Aug 2024 00:24:17 +0200
Subject: [PATCH 1324/1488] Fix tests

---
 src/Databases/DatabaseHDFS.cpp                 | 2 +-
 src/Storages/ObjectStorage/HDFS/HDFSCommon.cpp | 2 +-
 tests/integration/test_storage_hdfs/test.py    | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/Databases/DatabaseHDFS.cpp b/src/Databases/DatabaseHDFS.cpp
index f58f1b76e71..7fa67a5678e 100644
--- a/src/Databases/DatabaseHDFS.cpp
+++ b/src/Databases/DatabaseHDFS.cpp
@@ -51,7 +51,7 @@ DatabaseHDFS::DatabaseHDFS(const String & name_, const String & source_url, Cont
     if (!source.empty())
     {
         if (!re2::RE2::FullMatch(source, std::string(HDFS_HOST_REGEXP)))
-            throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad hdfs host: {}. "
+            throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad HDFS host: {}. "
                             "It should have structure 'hdfs://<host_name>:<port>'", source);
 
         context_->getGlobalContext()->getRemoteHostFilter().checkURL(Poco::URI(source));
diff --git a/src/Storages/ObjectStorage/HDFS/HDFSCommon.cpp b/src/Storages/ObjectStorage/HDFS/HDFSCommon.cpp
index 365828bcc40..7f8727eea1c 100644
--- a/src/Storages/ObjectStorage/HDFS/HDFSCommon.cpp
+++ b/src/Storages/ObjectStorage/HDFS/HDFSCommon.cpp
@@ -192,7 +192,7 @@ String getNameNodeCluster(const String &hdfs_url)
 void checkHDFSURL(const String & url)
 {
     if (!re2::RE2::FullMatch(url, std::string(HDFS_URL_REGEXP)))
-        throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad hdfs url: {}. It should have structure 'hdfs://<host_name>:<port>/<path>'", url);
+        throw Exception(ErrorCodes::BAD_ARGUMENTS, "Bad HDFS URL: {}. It should have structure 'hdfs://<host_name>:<port>/<path>'", url);
 }
 
 }
diff --git a/tests/integration/test_storage_hdfs/test.py b/tests/integration/test_storage_hdfs/test.py
index 47d8f44c0b7..ccd2c7eaf11 100644
--- a/tests/integration/test_storage_hdfs/test.py
+++ b/tests/integration/test_storage_hdfs/test.py
@@ -157,7 +157,7 @@ def test_bad_hdfs_uri(started_cluster):
         )
     except Exception as ex:
         print(ex)
-        assert "Bad hdfs url" in str(ex)
+        assert "Bad HDFS URL" in str(ex)
     try:
         node1.query(
             "create table BadStorage2 (id UInt32, name String, weight Float64) ENGINE = HDFS('hdfs://hdfs100500:9000/other_storage', 'TSV')"

From 185b6a54da8f24a97f130bdebe7bb1ec2bd266c0 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 4 Aug 2024 00:34:58 +0200
Subject: [PATCH 1325/1488] Merge with master

---
 src/Core/SettingsChangesHistory.cpp | 261 +---------------------------
 1 file changed, 1 insertion(+), 260 deletions(-)

diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp
index 3f07bfdb933..5b94391bade 100644
--- a/src/Core/SettingsChangesHistory.cpp
+++ b/src/Core/SettingsChangesHistory.cpp
@@ -57,266 +57,6 @@ String ClickHouseVersion::toString() const
 /// Note: please check if the key already exists to prevent duplicate entries.
 static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory::SettingsChanges>> settings_changes_history_initializer =
 {
-    {"24.8", {{"input_format_json_max_depth", 1000000, 1000, "It was unlimited in previous versions, but that was unsafe."}}},
-    {"24.7", {{"output_format_parquet_write_page_index", false, true, "Add a possibility to write page index into parquet files."},
-              {"output_format_binary_encode_types_in_binary_format", false, false, "Added new setting to allow to write type names in binary format in RowBinaryWithNamesAndTypes output format"},
-              {"input_format_binary_decode_types_in_binary_format", false, false, "Added new setting to allow to read type names in binary format in RowBinaryWithNamesAndTypes input format"},
-              {"output_format_native_encode_types_in_binary_format", false, false, "Added new setting to allow to write type names in binary format in Native output format"},
-              {"input_format_native_decode_types_in_binary_format", false, false, "Added new setting to allow to read type names in binary format in Native output format"},
-              {"read_in_order_use_buffering", false, true, "Use buffering before merging while reading in order of primary key"},
-              {"enable_named_columns_in_function_tuple", false, true, "Generate named tuples in function tuple() when all names are unique and can be treated as unquoted identifiers."},
-              {"input_format_json_case_insensitive_column_matching", false, false, "Ignore case when matching JSON keys with CH columns."},
-              {"optimize_trivial_insert_select", true, false, "The optimization does not make sense in many cases."},
-              {"dictionary_validate_primary_key_type", false, false, "Validate primary key type for dictionaries. By default id type for simple layouts will be implicitly converted to UInt64."},
-              {"collect_hash_table_stats_during_joins", false, true, "New setting."},
-              {"max_size_to_preallocate_for_joins", 0, 100'000'000, "New setting."},
-              {"input_format_orc_reader_time_zone_name", "GMT", "GMT", "The time zone name for ORC row reader, the default ORC row reader's time zone is GMT."},
-              {"lightweight_mutation_projection_mode", "throw", "throw", "When lightweight delete happens on a table with projection(s), the possible operations include throw the exception as projection exists, or drop all projection related to this table then do lightweight delete."},
-              {"database_replicated_allow_heavy_create", true, false, "Long-running DDL queries (CREATE AS SELECT and POPULATE) for Replicated database engine was forbidden"},
-              {"query_plan_merge_filters", false, false, "Allow to merge filters in the query plan"},
-              {"azure_sdk_max_retries", 10, 10, "Maximum number of retries in azure sdk"},
-              {"azure_sdk_retry_initial_backoff_ms", 10, 10, "Minimal backoff between retries in azure sdk"},
-              {"azure_sdk_retry_max_backoff_ms", 1000, 1000, "Maximal backoff between retries in azure sdk"},
-              {"merge_tree_min_bytes_per_task_for_remote_reading", 4194304, 2097152, "Value is unified with `filesystem_prefetch_min_bytes_for_single_read_task`"},
-              {"ignore_on_cluster_for_replicated_named_collections_queries", false, false, "Ignore ON CLUSTER clause for replicated named collections management queries."},
-              {"backup_restore_s3_retry_attempts", 1000,1000, "Setting for Aws::Client::RetryStrategy, Aws::Client does retries itself, 0 means no retries. It takes place only for backup/restore."},
-              {"postgresql_connection_attempt_timeout", 2, 2, "Allow to control 'connect_timeout' parameter of PostgreSQL connection."},
-              {"postgresql_connection_pool_retries", 2, 2, "Allow to control the number of retries in PostgreSQL connection pool."}
-              }},
-    {"24.6", {{"materialize_skip_indexes_on_insert", true, true, "Added new setting to allow to disable materialization of skip indexes on insert"},
-              {"materialize_statistics_on_insert", true, true, "Added new setting to allow to disable materialization of statistics on insert"},
-              {"input_format_parquet_use_native_reader", false, false, "When reading Parquet files, to use native reader instead of arrow reader."},
-              {"hdfs_throw_on_zero_files_match", false, false, "Allow to throw an error when ListObjects request cannot match any files in HDFS engine instead of empty query result"},
-              {"azure_throw_on_zero_files_match", false, false, "Allow to throw an error when ListObjects request cannot match any files in AzureBlobStorage engine instead of empty query result"},
-              {"s3_validate_request_settings", true, true, "Allow to disable S3 request settings validation"},
-              {"allow_experimental_full_text_index", false, false, "Enable experimental full-text index"},
-              {"azure_skip_empty_files", false, false, "Allow to skip empty files in azure table engine"},
-              {"hdfs_ignore_file_doesnt_exist", false, false, "Allow to return 0 rows when the requested files don't exist instead of throwing an exception in HDFS table engine"},
-              {"azure_ignore_file_doesnt_exist", false, false, "Allow to return 0 rows when the requested files don't exist instead of throwing an exception in AzureBlobStorage table engine"},
-              {"s3_ignore_file_doesnt_exist", false, false, "Allow to return 0 rows when the requested files don't exist instead of throwing an exception in S3 table engine"},
-              {"s3_max_part_number", 10000, 10000, "Maximum part number number for s3 upload part"},
-              {"s3_max_single_operation_copy_size", 32 * 1024 * 1024, 32 * 1024 * 1024, "Maximum size for a single copy operation in s3"},
-              {"input_format_parquet_max_block_size", 8192, DEFAULT_BLOCK_SIZE, "Increase block size for parquet reader."},
-              {"input_format_parquet_prefer_block_bytes", 0, DEFAULT_BLOCK_SIZE * 256, "Average block bytes output by parquet reader."},
-              {"enable_blob_storage_log", true, true, "Write information about blob storage operations to system.blob_storage_log table"},
-              {"allow_deprecated_snowflake_conversion_functions", true, false, "Disabled deprecated functions snowflakeToDateTime[64] and dateTime[64]ToSnowflake."},
-              {"allow_statistic_optimize", false, false, "Old setting which popped up here being renamed."},
-              {"allow_experimental_statistic", false, false, "Old setting which popped up here being renamed."},
-              {"allow_statistics_optimize", false, false, "The setting was renamed. The previous name is `allow_statistic_optimize`."},
-              {"allow_experimental_statistics", false, false, "The setting was renamed. The previous name is `allow_experimental_statistic`."},
-              {"enable_vertical_final", false, true, "Enable vertical final by default again after fixing bug"},
-              {"parallel_replicas_custom_key_range_lower", 0, 0, "Add settings to control the range filter when using parallel replicas with dynamic shards"},
-              {"parallel_replicas_custom_key_range_upper", 0, 0, "Add settings to control the range filter when using parallel replicas with dynamic shards. A value of 0 disables the upper limit"},
-              {"output_format_pretty_display_footer_column_names", 0, 1, "Add a setting to display column names in the footer if there are many rows. Threshold value is controlled by output_format_pretty_display_footer_column_names_min_rows."},
-              {"output_format_pretty_display_footer_column_names_min_rows", 0, 50, "Add a setting to control the threshold value for setting output_format_pretty_display_footer_column_names_min_rows. Default 50."},
-              {"output_format_csv_serialize_tuple_into_separate_columns", true, true, "A new way of how interpret tuples in CSV format was added."},
-              {"input_format_csv_deserialize_separate_columns_into_tuple", true, true, "A new way of how interpret tuples in CSV format was added."},
-              {"input_format_csv_try_infer_strings_from_quoted_tuples", true, true, "A new way of how interpret tuples in CSV format was added."},
-              }},
-    {"24.5", {{"allow_deprecated_error_prone_window_functions", true, false, "Allow usage of deprecated error prone window functions (neighbor, runningAccumulate, runningDifferenceStartingWithFirstValue, runningDifference)"},
-              {"allow_experimental_join_condition", false, false, "Support join with inequal conditions which involve columns from both left and right table. e.g. t1.y < t2.y."},
-              {"input_format_tsv_crlf_end_of_line", false, false, "Enables reading of CRLF line endings with TSV formats"},
-              {"output_format_parquet_use_custom_encoder", false, true, "Enable custom Parquet encoder."},
-              {"cross_join_min_rows_to_compress", 0, 10000000, "Minimal count of rows to compress block in CROSS JOIN. Zero value means - disable this threshold. This block is compressed when any of the two thresholds (by rows or by bytes) are reached."},
-              {"cross_join_min_bytes_to_compress", 0, 1_GiB, "Minimal size of block to compress in CROSS JOIN. Zero value means - disable this threshold. This block is compressed when any of the two thresholds (by rows or by bytes) are reached."},
-              {"http_max_chunk_size", 0, 0, "Internal limitation"},
-              {"prefer_external_sort_block_bytes", 0, DEFAULT_BLOCK_SIZE * 256, "Prefer maximum block bytes for external sort, reduce the memory usage during merging."},
-              {"input_format_force_null_for_omitted_fields", false, false, "Disable type-defaults for omitted fields when needed"},
-              {"cast_string_to_dynamic_use_inference", false, false, "Add setting to allow converting String to Dynamic through parsing"},
-              {"allow_experimental_dynamic_type", false, false, "Add new experimental Dynamic type"},
-              {"azure_max_blocks_in_multipart_upload", 50000, 50000, "Maximum number of blocks in multipart upload for Azure."},
-              }},
-    {"24.4", {{"input_format_json_throw_on_bad_escape_sequence", true, true, "Allow to save JSON strings with bad escape sequences"},
-              {"max_parsing_threads", 0, 0, "Add a separate setting to control number of threads in parallel parsing from files"},
-              {"ignore_drop_queries_probability", 0, 0, "Allow to ignore drop queries in server with specified probability for testing purposes"},
-              {"lightweight_deletes_sync", 2, 2, "The same as 'mutation_sync', but controls only execution of lightweight deletes"},
-              {"query_cache_system_table_handling", "save", "throw", "The query cache no longer caches results of queries against system tables"},
-              {"input_format_json_ignore_unnecessary_fields", false, true, "Ignore unnecessary fields and not parse them. Enabling this may not throw exceptions on json strings of invalid format or with duplicated fields"},
-              {"input_format_hive_text_allow_variable_number_of_columns", false, true, "Ignore extra columns in Hive Text input (if file has more columns than expected) and treat missing fields in Hive Text input as default values."},
-              {"allow_experimental_database_replicated", false, true, "Database engine Replicated is now in Beta stage"},
-              {"temporary_data_in_cache_reserve_space_wait_lock_timeout_milliseconds", (10 * 60 * 1000), (10 * 60 * 1000), "Wait time to lock cache for sapce reservation in temporary data in filesystem cache"},
-              {"optimize_rewrite_sum_if_to_count_if", false, true, "Only available for the analyzer, where it works correctly"},
-              {"azure_allow_parallel_part_upload", "true", "true", "Use multiple threads for azure multipart upload."},
-              {"max_recursive_cte_evaluation_depth", DBMS_RECURSIVE_CTE_MAX_EVALUATION_DEPTH, DBMS_RECURSIVE_CTE_MAX_EVALUATION_DEPTH, "Maximum limit on recursive CTE evaluation depth"},
-              {"query_plan_convert_outer_join_to_inner_join", false, true, "Allow to convert OUTER JOIN to INNER JOIN if filter after JOIN always filters default values"},
-              }},
-    {"24.3", {{"s3_connect_timeout_ms", 1000, 1000, "Introduce new dedicated setting for s3 connection timeout"},
-              {"allow_experimental_shared_merge_tree", false, true, "The setting is obsolete"},
-              {"use_page_cache_for_disks_without_file_cache", false, false, "Added userspace page cache"},
-              {"read_from_page_cache_if_exists_otherwise_bypass_cache", false, false, "Added userspace page cache"},
-              {"page_cache_inject_eviction", false, false, "Added userspace page cache"},
-              {"default_table_engine", "None", "MergeTree", "Set default table engine to MergeTree for better usability"},
-              {"input_format_json_use_string_type_for_ambiguous_paths_in_named_tuples_inference_from_objects", false, false, "Allow to use String type for ambiguous paths during named tuple inference from JSON objects"},
-              {"traverse_shadow_remote_data_paths", false, false, "Traverse shadow directory when query system.remote_data_paths."},
-              {"throw_if_deduplication_in_dependent_materialized_views_enabled_with_async_insert", false, true, "Deduplication in dependent materialized view cannot work together with async inserts."},
-              {"parallel_replicas_allow_in_with_subquery", false, true, "If true, subquery for IN will be executed on every follower replica"},
-              {"log_processors_profiles", false, true, "Enable by default"},
-              {"function_locate_has_mysql_compatible_argument_order", false, true, "Increase compatibility with MySQL's locate function."},
-              {"allow_suspicious_primary_key", true, false, "Forbid suspicious PRIMARY KEY/ORDER BY for MergeTree (i.e. SimpleAggregateFunction)"},
-              {"filesystem_cache_reserve_space_wait_lock_timeout_milliseconds", 1000, 1000, "Wait time to lock cache for sapce reservation in filesystem cache"},
-              {"max_parser_backtracks", 0, 1000000, "Limiting the complexity of parsing"},
-              {"analyzer_compatibility_join_using_top_level_identifier", false, false, "Force to resolve identifier in JOIN USING from projection"},
-              {"distributed_insert_skip_read_only_replicas", false, false, "If true, INSERT into Distributed will skip read-only replicas"},
-              {"keeper_max_retries", 10, 10, "Max retries for general keeper operations"},
-              {"keeper_retry_initial_backoff_ms", 100, 100, "Initial backoff timeout for general keeper operations"},
-              {"keeper_retry_max_backoff_ms", 5000, 5000, "Max backoff timeout for general keeper operations"},
-              {"s3queue_allow_experimental_sharded_mode", false, false, "Enable experimental sharded mode of S3Queue table engine. It is experimental because it will be rewritten"},
-              {"allow_experimental_analyzer", false, true, "Enable analyzer and planner by default."},
-              {"merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability", 0.0, 0.0, "For testing of `PartsSplitter` - split read ranges into intersecting and non intersecting every time you read from MergeTree with the specified probability."},
-              {"allow_get_client_http_header", false, false, "Introduced a new function."},
-              {"output_format_pretty_row_numbers", false, true, "It is better for usability."},
-              {"output_format_pretty_max_value_width_apply_for_single_value", true, false, "Single values in Pretty formats won't be cut."},
-              {"output_format_parquet_string_as_string", false, true, "ClickHouse allows arbitrary binary data in the String data type, which is typically UTF-8. Parquet/ORC/Arrow Strings only support UTF-8. That's why you can choose which Arrow's data type to use for the ClickHouse String data type - String or Binary. While Binary would be more correct and compatible, using String by default will correspond to user expectations in most cases."},
-              {"output_format_orc_string_as_string", false, true, "ClickHouse allows arbitrary binary data in the String data type, which is typically UTF-8. Parquet/ORC/Arrow Strings only support UTF-8. That's why you can choose which Arrow's data type to use for the ClickHouse String data type - String or Binary. While Binary would be more correct and compatible, using String by default will correspond to user expectations in most cases."},
-              {"output_format_arrow_string_as_string", false, true, "ClickHouse allows arbitrary binary data in the String data type, which is typically UTF-8. Parquet/ORC/Arrow Strings only support UTF-8. That's why you can choose which Arrow's data type to use for the ClickHouse String data type - String or Binary. While Binary would be more correct and compatible, using String by default will correspond to user expectations in most cases."},
-              {"output_format_parquet_compression_method", "lz4", "zstd", "Parquet/ORC/Arrow support many compression methods, including lz4 and zstd. ClickHouse supports each and every compression method. Some inferior tools, such as 'duckdb', lack support for the faster `lz4` compression method, that's why we set zstd by default."},
-              {"output_format_orc_compression_method", "lz4", "zstd", "Parquet/ORC/Arrow support many compression methods, including lz4 and zstd. ClickHouse supports each and every compression method. Some inferior tools, such as 'duckdb', lack support for the faster `lz4` compression method, that's why we set zstd by default."},
-              {"output_format_pretty_highlight_digit_groups", false, true, "If enabled and if output is a terminal, highlight every digit corresponding to the number of thousands, millions, etc. with underline."},
-              {"geo_distance_returns_float64_on_float64_arguments", false, true, "Increase the default precision."},
-              {"azure_max_inflight_parts_for_one_file", 20, 20, "The maximum number of a concurrent loaded parts in multipart upload request. 0 means unlimited."},
-              {"azure_strict_upload_part_size", 0, 0, "The exact size of part to upload during multipart upload to Azure blob storage."},
-              {"azure_min_upload_part_size", 16*1024*1024, 16*1024*1024, "The minimum size of part to upload during multipart upload to Azure blob storage."},
-              {"azure_max_upload_part_size", 5ull*1024*1024*1024, 5ull*1024*1024*1024, "The maximum size of part to upload during multipart upload to Azure blob storage."},
-              {"azure_upload_part_size_multiply_factor", 2, 2, "Multiply azure_min_upload_part_size by this factor each time azure_multiply_parts_count_threshold parts were uploaded from a single write to Azure blob storage."},
-              {"azure_upload_part_size_multiply_parts_count_threshold", 500, 500, "Each time this number of parts was uploaded to Azure blob storage, azure_min_upload_part_size is multiplied by azure_upload_part_size_multiply_factor."},
-              {"output_format_csv_serialize_tuple_into_separate_columns", true, true, "A new way of how interpret tuples in CSV format was added."},
-              {"input_format_csv_deserialize_separate_columns_into_tuple", true, true, "A new way of how interpret tuples in CSV format was added."},
-              {"input_format_csv_try_infer_strings_from_quoted_tuples", true, true, "A new way of how interpret tuples in CSV format was added."},
-              }},
-    {"24.2", {{"allow_suspicious_variant_types", true, false, "Don't allow creating Variant type with suspicious variants by default"},
-              {"validate_experimental_and_suspicious_types_inside_nested_types", false, true, "Validate usage of experimental and suspicious types inside nested types"},
-              {"output_format_values_escape_quote_with_quote", false, false, "If true escape ' with '', otherwise quoted with \\'"},
-              {"output_format_pretty_single_large_number_tip_threshold", 0, 1'000'000, "Print a readable number tip on the right side of the table if the block consists of a single number which exceeds this value (except 0)"},
-              {"input_format_try_infer_exponent_floats", true, false, "Don't infer floats in exponential notation by default"},
-              {"query_plan_optimize_prewhere", true, true, "Allow to push down filter to PREWHERE expression for supported storages"},
-              {"async_insert_max_data_size", 1000000, 10485760, "The previous value appeared to be too small."},
-              {"async_insert_poll_timeout_ms", 10, 10, "Timeout in milliseconds for polling data from asynchronous insert queue"},
-              {"async_insert_use_adaptive_busy_timeout", false, true, "Use adaptive asynchronous insert timeout"},
-              {"async_insert_busy_timeout_min_ms", 50, 50, "The minimum value of the asynchronous insert timeout in milliseconds; it also serves as the initial value, which may be increased later by the adaptive algorithm"},
-              {"async_insert_busy_timeout_max_ms", 200, 200, "The minimum value of the asynchronous insert timeout in milliseconds; async_insert_busy_timeout_ms is aliased to async_insert_busy_timeout_max_ms"},
-              {"async_insert_busy_timeout_increase_rate", 0.2, 0.2, "The exponential growth rate at which the adaptive asynchronous insert timeout increases"},
-              {"async_insert_busy_timeout_decrease_rate", 0.2, 0.2, "The exponential growth rate at which the adaptive asynchronous insert timeout decreases"},
-              {"format_template_row_format", "", "", "Template row format string can be set directly in query"},
-              {"format_template_resultset_format", "", "", "Template result set format string can be set in query"},
-              {"split_parts_ranges_into_intersecting_and_non_intersecting_final", true, true, "Allow to split parts ranges into intersecting and non intersecting during FINAL optimization"},
-              {"split_intersecting_parts_ranges_into_layers_final", true, true, "Allow to split intersecting parts ranges into layers during FINAL optimization"},
-              {"azure_max_single_part_copy_size", 256*1024*1024, 256*1024*1024, "The maximum size of object to copy using single part copy to Azure blob storage."},
-              {"min_external_table_block_size_rows", DEFAULT_INSERT_BLOCK_SIZE, DEFAULT_INSERT_BLOCK_SIZE, "Squash blocks passed to external table to specified size in rows, if blocks are not big enough"},
-              {"min_external_table_block_size_bytes", DEFAULT_INSERT_BLOCK_SIZE * 256, DEFAULT_INSERT_BLOCK_SIZE * 256, "Squash blocks passed to external table to specified size in bytes, if blocks are not big enough."},
-              {"parallel_replicas_prefer_local_join", true, true, "If true, and JOIN can be executed with parallel replicas algorithm, and all storages of right JOIN part are *MergeTree, local JOIN will be used instead of GLOBAL JOIN."},
-              {"optimize_time_filter_with_preimage", true, true, "Optimize Date and DateTime predicates by converting functions into equivalent comparisons without conversions (e.g. toYear(col) = 2023 -> col >= '2023-01-01' AND col <= '2023-12-31')"},
-              {"extract_key_value_pairs_max_pairs_per_row", 0, 0, "Max number of pairs that can be produced by the `extractKeyValuePairs` function. Used as a safeguard against consuming too much memory."},
-              {"default_view_definer", "CURRENT_USER", "CURRENT_USER", "Allows to set default `DEFINER` option while creating a view"},
-              {"default_materialized_view_sql_security", "DEFINER", "DEFINER", "Allows to set a default value for SQL SECURITY option when creating a materialized view"},
-              {"default_normal_view_sql_security", "INVOKER", "INVOKER", "Allows to set default `SQL SECURITY` option while creating a normal view"},
-              {"mysql_map_string_to_text_in_show_columns", false, true, "Reduce the configuration effort to connect ClickHouse with BI tools."},
-              {"mysql_map_fixed_string_to_text_in_show_columns", false, true, "Reduce the configuration effort to connect ClickHouse with BI tools."},
-              }},
-    {"24.1", {{"print_pretty_type_names", false, true, "Better user experience."},
-              {"input_format_json_read_bools_as_strings", false, true, "Allow to read bools as strings in JSON formats by default"},
-              {"output_format_arrow_use_signed_indexes_for_dictionary", false, true, "Use signed indexes type for Arrow dictionaries by default as it's recommended"},
-              {"allow_experimental_variant_type", false, false, "Add new experimental Variant type"},
-              {"use_variant_as_common_type", false, false, "Allow to use Variant in if/multiIf if there is no common type"},
-              {"output_format_arrow_use_64_bit_indexes_for_dictionary", false, false, "Allow to use 64 bit indexes type in Arrow dictionaries"},
-              {"parallel_replicas_mark_segment_size", 128, 128, "Add new setting to control segment size in new parallel replicas coordinator implementation"},
-              {"ignore_materialized_views_with_dropped_target_table", false, false, "Add new setting to allow to ignore materialized views with dropped target table"},
-              {"output_format_compression_level", 3, 3, "Allow to change compression level in the query output"},
-              {"output_format_compression_zstd_window_log", 0, 0, "Allow to change zstd window log in the query output when zstd compression is used"},
-              {"enable_zstd_qat_codec", false, false, "Add new ZSTD_QAT codec"},
-              {"enable_vertical_final", false, true, "Use vertical final by default"},
-              {"output_format_arrow_use_64_bit_indexes_for_dictionary", false, false, "Allow to use 64 bit indexes type in Arrow dictionaries"},
-              {"max_rows_in_set_to_optimize_join", 100000, 0, "Disable join optimization as it prevents from read in order optimization"},
-              {"output_format_pretty_color", true, "auto", "Setting is changed to allow also for auto value, disabling ANSI escapes if output is not a tty"},
-              {"function_visible_width_behavior", 0, 1, "We changed the default behavior of `visibleWidth` to be more precise"},
-              {"max_estimated_execution_time", 0, 0, "Separate max_execution_time and max_estimated_execution_time"},
-              {"iceberg_engine_ignore_schema_evolution", false, false, "Allow to ignore schema evolution in Iceberg table engine"},
-              {"optimize_injective_functions_in_group_by", false, true, "Replace injective functions by it's arguments in GROUP BY section in analyzer"},
-              {"update_insert_deduplication_token_in_dependent_materialized_views", false, false, "Allow to update insert deduplication token with table identifier during insert in dependent materialized views"},
-              {"azure_max_unexpected_write_error_retries", 4, 4, "The maximum number of retries in case of unexpected errors during Azure blob storage write"},
-              {"split_parts_ranges_into_intersecting_and_non_intersecting_final", false, true, "Allow to split parts ranges into intersecting and non intersecting during FINAL optimization"},
-              {"split_intersecting_parts_ranges_into_layers_final", true, true, "Allow to split intersecting parts ranges into layers during FINAL optimization"}}},
-    {"23.12", {{"allow_suspicious_ttl_expressions", true, false, "It is a new setting, and in previous versions the behavior was equivalent to allowing."},
-              {"input_format_parquet_allow_missing_columns", false, true, "Allow missing columns in Parquet files by default"},
-              {"input_format_orc_allow_missing_columns", false, true, "Allow missing columns in ORC files by default"},
-              {"input_format_arrow_allow_missing_columns", false, true, "Allow missing columns in Arrow files by default"}}},
-    {"23.11", {{"parsedatetime_parse_without_leading_zeros", false, true, "Improved compatibility with MySQL DATE_FORMAT/STR_TO_DATE"}}},
-    {"23.9", {{"optimize_group_by_constant_keys", false, true, "Optimize group by constant keys by default"},
-              {"input_format_json_try_infer_named_tuples_from_objects", false, true, "Try to infer named Tuples from JSON objects by default"},
-              {"input_format_json_read_numbers_as_strings", false, true, "Allow to read numbers as strings in JSON formats by default"},
-              {"input_format_json_read_arrays_as_strings", false, true, "Allow to read arrays as strings in JSON formats by default"},
-              {"input_format_json_infer_incomplete_types_as_strings", false, true, "Allow to infer incomplete types as Strings in JSON formats by default"},
-              {"input_format_json_try_infer_numbers_from_strings", true, false, "Don't infer numbers from strings in JSON formats by default to prevent possible parsing errors"},
-              {"http_write_exception_in_output_format", false, true, "Output valid JSON/XML on exception in HTTP streaming."}}},
-    {"23.8", {{"rewrite_count_distinct_if_with_count_distinct_implementation", false, true, "Rewrite countDistinctIf with count_distinct_implementation configuration"}}},
-    {"23.7", {{"function_sleep_max_microseconds_per_block", 0, 3000000, "In previous versions, the maximum sleep time of 3 seconds was applied only for `sleep`, but not for `sleepEachRow` function. In the new version, we introduce this setting. If you set compatibility with the previous versions, we will disable the limit altogether."}}},
-    {"23.6", {{"http_send_timeout", 180, 30, "3 minutes seems crazy long. Note that this is timeout for a single network write call, not for the whole upload operation."},
-              {"http_receive_timeout", 180, 30, "See http_send_timeout."}}},
-    {"23.5", {{"input_format_parquet_preserve_order", true, false, "Allow Parquet reader to reorder rows for better parallelism."},
-              {"parallelize_output_from_storages", false, true, "Allow parallelism when executing queries that read from file/url/s3/etc. This may reorder rows."},
-              {"use_with_fill_by_sorting_prefix", false, true, "Columns preceding WITH FILL columns in ORDER BY clause form sorting prefix. Rows with different values in sorting prefix are filled independently"},
-              {"output_format_parquet_compliant_nested_types", false, true, "Change an internal field name in output Parquet file schema."}}},
-    {"23.4", {{"allow_suspicious_indices", true, false, "If true, index can defined with identical expressions"},
-              {"allow_nonconst_timezone_arguments", true, false, "Allow non-const timezone arguments in certain time-related functions like toTimeZone(), fromUnixTimestamp*(), snowflakeToDateTime*()."},
-              {"connect_timeout_with_failover_ms", 50, 1000, "Increase default connect timeout because of async connect"},
-              {"connect_timeout_with_failover_secure_ms", 100, 1000, "Increase default secure connect timeout because of async connect"},
-              {"hedged_connection_timeout_ms", 100, 50, "Start new connection in hedged requests after 50 ms instead of 100 to correspond with previous connect timeout"},
-              {"formatdatetime_f_prints_single_zero", true, false, "Improved compatibility with MySQL DATE_FORMAT()/STR_TO_DATE()"},
-              {"formatdatetime_parsedatetime_m_is_month_name", false, true, "Improved compatibility with MySQL DATE_FORMAT/STR_TO_DATE"}}},
-    {"23.3", {{"output_format_parquet_version", "1.0", "2.latest", "Use latest Parquet format version for output format"},
-              {"input_format_json_ignore_unknown_keys_in_named_tuple", false, true, "Improve parsing JSON objects as named tuples"},
-              {"input_format_native_allow_types_conversion", false, true, "Allow types conversion in Native input forma"},
-              {"output_format_arrow_compression_method", "none", "lz4_frame", "Use lz4 compression in Arrow output format by default"},
-              {"output_format_parquet_compression_method", "snappy", "lz4", "Use lz4 compression in Parquet output format by default"},
-              {"output_format_orc_compression_method", "none", "lz4_frame", "Use lz4 compression in ORC output format by default"},
-              {"async_query_sending_for_remote", false, true, "Create connections and send query async across shards"}}},
-    {"23.2", {{"output_format_parquet_fixed_string_as_fixed_byte_array", false, true, "Use Parquet FIXED_LENGTH_BYTE_ARRAY type for FixedString by default"},
-              {"output_format_arrow_fixed_string_as_fixed_byte_array", false, true, "Use Arrow FIXED_SIZE_BINARY type for FixedString by default"},
-              {"query_plan_remove_redundant_distinct", false, true, "Remove redundant Distinct step in query plan"},
-              {"optimize_duplicate_order_by_and_distinct", true, false, "Remove duplicate ORDER BY and DISTINCT if it's possible"},
-              {"insert_keeper_max_retries", 0, 20, "Enable reconnections to Keeper on INSERT, improve reliability"}}},
-    {"23.1", {{"input_format_json_read_objects_as_strings", 0, 1, "Enable reading nested json objects as strings while object type is experimental"},
-              {"input_format_json_defaults_for_missing_elements_in_named_tuple", false, true, "Allow missing elements in JSON objects while reading named tuples by default"},
-              {"input_format_csv_detect_header", false, true, "Detect header in CSV format by default"},
-              {"input_format_tsv_detect_header", false, true, "Detect header in TSV format by default"},
-              {"input_format_custom_detect_header", false, true, "Detect header in CustomSeparated format by default"},
-              {"query_plan_remove_redundant_sorting", false, true, "Remove redundant sorting in query plan. For example, sorting steps related to ORDER BY clauses in subqueries"}}},
-    {"22.12", {{"max_size_to_preallocate_for_aggregation", 10'000'000, 100'000'000, "This optimizes performance"},
-               {"query_plan_aggregation_in_order", 0, 1, "Enable some refactoring around query plan"},
-               {"format_binary_max_string_size", 0, 1_GiB, "Prevent allocating large amount of memory"}}},
-    {"22.11", {{"use_structure_from_insertion_table_in_table_functions", 0, 2, "Improve using structure from insertion table in table functions"}}},
-    {"22.9", {{"force_grouping_standard_compatibility", false, true, "Make GROUPING function output the same as in SQL standard and other DBMS"}}},
-    {"22.7", {{"cross_to_inner_join_rewrite", 1, 2, "Force rewrite comma join to inner"},
-              {"enable_positional_arguments", false, true, "Enable positional arguments feature by default"},
-              {"format_csv_allow_single_quotes", true, false, "Most tools don't treat single quote in CSV specially, don't do it by default too"}}},
-    {"22.6", {{"output_format_json_named_tuples_as_objects", false, true, "Allow to serialize named tuples as JSON objects in JSON formats by default"},
-              {"input_format_skip_unknown_fields", false, true, "Optimize reading subset of columns for some input formats"}}},
-    {"22.5", {{"memory_overcommit_ratio_denominator", 0, 1073741824, "Enable memory overcommit feature by default"},
-              {"memory_overcommit_ratio_denominator_for_user", 0, 1073741824, "Enable memory overcommit feature by default"}}},
-    {"22.4", {{"allow_settings_after_format_in_insert", true, false, "Do not allow SETTINGS after FORMAT for INSERT queries because ClickHouse interpret SETTINGS as some values, which is misleading"}}},
-    {"22.3", {{"cast_ipv4_ipv6_default_on_conversion_error", true, false, "Make functions cast(value, 'IPv4') and cast(value, 'IPv6') behave same as toIPv4 and toIPv6 functions"}}},
-    {"21.12", {{"stream_like_engine_allow_direct_select", true, false, "Do not allow direct select for Kafka/RabbitMQ/FileLog by default"}}},
-    {"21.9", {{"output_format_decimal_trailing_zeros", true, false, "Do not output trailing zeros in text representation of Decimal types by default for better looking output"},
-              {"use_hedged_requests", false, true, "Enable Hedged Requests feature by default"}}},
-    {"21.7", {{"legacy_column_name_of_tuple_literal", true, false, "Add this setting only for compatibility reasons. It makes sense to set to 'true', while doing rolling update of cluster from version lower than 21.7 to higher"}}},
-    {"21.5", {{"async_socket_for_remote", false, true, "Fix all problems and turn on asynchronous reads from socket for remote queries by default again"}}},
-    {"21.3", {{"async_socket_for_remote", true, false, "Turn off asynchronous reads from socket for remote queries because of some problems"},
-              {"optimize_normalize_count_variants", false, true, "Rewrite aggregate functions that semantically equals to count() as count() by default"},
-              {"normalize_function_names", false, true, "Normalize function names to their canonical names, this was needed for projection query routing"}}},
-    {"21.2", {{"enable_global_with_statement", false, true, "Propagate WITH statements to UNION queries and all subqueries by default"}}},
-    {"21.1", {{"insert_quorum_parallel", false, true, "Use parallel quorum inserts by default. It is significantly more convenient to use than sequential quorum inserts"},
-              {"input_format_null_as_default", false, true, "Allow to insert NULL as default for input formats by default"},
-              {"optimize_on_insert", false, true, "Enable data optimization on INSERT by default for better user experience"},
-              {"use_compact_format_in_distributed_parts_names", false, true, "Use compact format for async INSERT into Distributed tables by default"}}},
-    {"20.10", {{"format_regexp_escaping_rule", "Escaped", "Raw", "Use Raw as default escaping rule for Regexp format to male the behaviour more like to what users expect"}}},
-    {"20.7", {{"show_table_uuid_in_table_create_query_if_not_nil", true, false, "Stop showing  UID of the table in its CREATE query for Engine=Atomic"}}},
-    {"20.5", {{"input_format_with_names_use_header", false, true, "Enable using header with names for formats with WithNames/WithNamesAndTypes suffixes"},
-              {"allow_suspicious_codecs", true, false, "Don't allow to specify meaningless compression codecs"}}},
-    {"20.4", {{"validate_polygons", false, true, "Throw exception if polygon is invalid in function pointInPolygon by default instead of returning possibly wrong results"}}},
-    {"19.18", {{"enable_scalar_subquery_optimization", false, true, "Prevent scalar subqueries from (de)serializing large scalar values and possibly avoid running the same subquery more than once"}}},
-    {"19.14", {{"any_join_distinct_right_table_keys", true, false, "Disable ANY RIGHT and ANY FULL JOINs by default to avoid inconsistency"}}},
-    {"19.12", {{"input_format_defaults_for_omitted_fields", false, true, "Enable calculation of complex default expressions for omitted fields for some input formats, because it should be the expected behaviour"}}},
-    {"19.5", {{"max_partitions_per_insert_block", 0, 100, "Add a limit for the number of partitions in one block"}}},
-    {"18.12.17", {{"enable_optimize_predicate_expression", 0, 1, "Optimize predicates to subqueries by default"}}},
     {"24.12",
         {
         }
@@ -335,6 +75,7 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
     },
     {"24.8",
         {
+            {"input_format_json_max_depth", 1000000, 1000, "It was unlimited in previous versions, but that was unsafe."},
             {"merge_tree_min_bytes_per_task_for_remote_reading", 4194304, 2097152, "Value is unified with `filesystem_prefetch_min_bytes_for_single_read_task`"},
             {"allow_archive_path_syntax", true, true, "Added new setting to allow disabling archive path syntax."},
         }

From 59e5a09b491d78eff188ecd685bb6ca3769831ce Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Sun, 4 Aug 2024 09:02:19 +0200
Subject: [PATCH 1326/1488] Use RabbitMQ without management in tests (attempt
 to improve startup)

Sometimes startup can take ~90 seconds [1]:

    2024-08-03 23:11:38.756067+00:00 [info] <0.9.0> Time to start RabbitMQ: 94651980 us

Unlike normally ~10 [2]:

    2024-08-04 00:33:07.016137+00:00 [info] <0.9.0> Time to start RabbitMQ: 10082489 us

  [1]: https://s3.amazonaws.com/clickhouse-test-reports/67737/b4e3bbcb82158bea4f5db1d9f5c28cfb741d1d51/integration_tests__asan__old_analyzer__[4_6].html
  [2]: https://s3.amazonaws.com/clickhouse-test-reports/66671/2f00c962711e13ca00af324366421fe4593b4ce6/integration_tests__asan__old_analyzer__[4_6].html

I've tried locally, the difference is very small, 3135665 us
(+management) vs (2740747 us), but still something, and who knows how it
works under pressure.

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 tests/integration/compose/docker_compose_rabbitmq.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/compose/docker_compose_rabbitmq.yml b/tests/integration/compose/docker_compose_rabbitmq.yml
index 61b21e0e3d9..94c7f0111c4 100644
--- a/tests/integration/compose/docker_compose_rabbitmq.yml
+++ b/tests/integration/compose/docker_compose_rabbitmq.yml
@@ -2,7 +2,7 @@ version: '2.3'
 
 services:
     rabbitmq1:
-        image: rabbitmq:3.12.6-management-alpine
+        image: rabbitmq:3.12.6-alpine
         hostname: rabbitmq1
         expose:
             - ${RABBITMQ_PORT:-5672}

From 264be9c598b42d91ee0a19f718c4d9a4291c7bc4 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Sun, 4 Aug 2024 09:06:50 +0200
Subject: [PATCH 1327/1488] tests: increase timeout for RabbitMQ startup

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 tests/integration/helpers/cluster.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py
index 9259c720ff0..a1bdee33d57 100644
--- a/tests/integration/helpers/cluster.py
+++ b/tests/integration/helpers/cluster.py
@@ -2371,7 +2371,7 @@ class ClickHouseCluster:
                 time.sleep(0.5)
         raise Exception("Cannot wait PostgreSQL Java Client container")
 
-    def wait_rabbitmq_to_start(self, timeout=30):
+    def wait_rabbitmq_to_start(self, timeout=60):
         self.print_all_docker_pieces()
         self.rabbitmq_ip = self.get_instance_ip(self.rabbitmq_host)
 

From dc527b6fd1dfdffb1d177237bffc69cd110cd2a7 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Sun, 4 Aug 2024 09:07:05 +0200
Subject: [PATCH 1328/1488] tests: detailed errors for RabbitMQ startup

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 tests/integration/helpers/cluster.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py
index a1bdee33d57..56d111629c9 100644
--- a/tests/integration/helpers/cluster.py
+++ b/tests/integration/helpers/cluster.py
@@ -2399,7 +2399,7 @@ class ClickHouseCluster:
                 )
             rabbitmq_debuginfo(self.rabbitmq_docker_id, self.rabbitmq_cookie)
         except Exception as e:
-            logging.debug("Unable to get logs from docker.")
+            logging.debug(f"Unable to get logs from docker: {e}.")
         raise Exception("Cannot wait RabbitMQ container")
 
     def wait_nats_is_available(self, max_retries=5):

From 27db715761ec66b0dfc915269270697e6cf67909 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Sun, 4 Aug 2024 09:52:36 +0000
Subject: [PATCH 1329/1488] Incorporate review feedback

---
 src/Functions/CountSubstringsImpl.h | 16 +++++++++-----
 src/Functions/MatchImpl.h           | 34 ++++++++++++++---------------
 2 files changed, 28 insertions(+), 22 deletions(-)

diff --git a/src/Functions/CountSubstringsImpl.h b/src/Functions/CountSubstringsImpl.h
index 8e91bc3aeb4..b1cefae6f1d 100644
--- a/src/Functions/CountSubstringsImpl.h
+++ b/src/Functions/CountSubstringsImpl.h
@@ -38,10 +38,10 @@ struct CountSubstringsImpl
         const ColumnPtr & start_pos,
         PaddedPODArray<UInt64> & res,
         [[maybe_unused]] ColumnUInt8 * res_null,
-        size_t /*input_rows_count*/)
+        size_t input_rows_count)
     {
         /// `res_null` serves as an output parameter for implementing an XYZOrNull variant.
-        assert(!res_null);
+        chassert(!res_null);
 
         const UInt8 * const begin = haystack_data.data();
         const UInt8 * const end = haystack_data.data() + haystack_data.size();
@@ -81,6 +81,8 @@ struct CountSubstringsImpl
             }
             pos = begin + haystack_offsets[i];
             ++i;
+
+            chassert(i < input_rows_count);
         }
     }
 
@@ -116,7 +118,7 @@ struct CountSubstringsImpl
         [[maybe_unused]] ColumnUInt8 * res_null)
     {
         /// `res_null` serves as an output parameter for implementing an XYZOrNull variant.
-        assert(!res_null);
+        chassert(!res_null);
 
         Impl::toLowerIfNeed(haystack);
         Impl::toLowerIfNeed(needle);
@@ -154,8 +156,10 @@ struct CountSubstringsImpl
         [[maybe_unused]] ColumnUInt8 * res_null,
         size_t input_rows_count)
     {
+        chassert(input_rows_count == haystack_offsets.size());
+
         /// `res_null` serves as an output parameter for implementing an XYZOrNull variant.
-        assert(!res_null);
+        chassert(!res_null);
 
         ColumnString::Offset prev_haystack_offset = 0;
         ColumnString::Offset prev_needle_offset = 0;
@@ -210,8 +214,10 @@ struct CountSubstringsImpl
         [[maybe_unused]] ColumnUInt8 * res_null,
         size_t input_rows_count)
     {
+        chassert(input_rows_count == needle_offsets.size());
+
         /// `res_null` serves as an output parameter for implementing an XYZOrNull variant.
-        assert(!res_null);
+        chassert(!res_null);
 
         /// NOTE You could use haystack indexing. But this is a rare case.
         ColumnString::Offset prev_needle_offset = 0;
diff --git a/src/Functions/MatchImpl.h b/src/Functions/MatchImpl.h
index ceac753fe79..7dc93ba79e0 100644
--- a/src/Functions/MatchImpl.h
+++ b/src/Functions/MatchImpl.h
@@ -131,11 +131,11 @@ struct MatchImpl
         size_t input_rows_count)
     {
         /// `res_null` serves as an output parameter for implementing an XYZOrNull variant.
-        assert(!res_null);
+        chassert(!res_null);
 
-        assert(res.size() == haystack_offsets.size());
-        assert(res.size() == input_rows_count);
-        assert(start_pos_ == nullptr);
+        chassert(res.size() == haystack_offsets.size());
+        chassert(res.size() == input_rows_count);
+        chassert(start_pos_ == nullptr);
 
         if (input_rows_count == 0)
             return;
@@ -295,10 +295,10 @@ struct MatchImpl
         size_t input_rows_count)
     {
         /// `res_null` serves as an output parameter for implementing an XYZOrNull variant.
-        assert(!res_null);
+        chassert(!res_null);
 
-        assert(res.size() == haystack.size() / N);
-        assert(res.size() == input_rows_count);
+        chassert(res.size() == haystack.size() / N);
+        chassert(res.size() == input_rows_count);
 
         if (input_rows_count == 0)
             return;
@@ -468,12 +468,12 @@ struct MatchImpl
         size_t input_rows_count)
     {
         /// `res_null` serves as an output parameter for implementing an XYZOrNull variant.
-        assert(!res_null);
+        chassert(!res_null);
 
-        assert(haystack_offsets.size() == needle_offset.size());
-        assert(res.size() == haystack_offsets.size());
-        assert(res.size() == input_rows_count);
-        assert(start_pos_ == nullptr);
+        chassert(haystack_offsets.size() == needle_offset.size());
+        chassert(res.size() == haystack_offsets.size());
+        chassert(res.size() == input_rows_count);
+        chassert(start_pos_ == nullptr);
 
         if (input_rows_count == 0)
             return;
@@ -577,12 +577,12 @@ struct MatchImpl
         size_t input_rows_count)
     {
         /// `res_null` serves as an output parameter for implementing an XYZOrNull variant.
-        assert(!res_null);
+        chassert(!res_null);
 
-        assert(res.size() == input_rows_count);
-        assert(res.size() == haystack.size() / N);
-        assert(res.size() == needle_offset.size());
-        assert(start_pos_ == nullptr);
+        chassert(res.size() == input_rows_count);
+        chassert(res.size() == haystack.size() / N);
+        chassert(res.size() == needle_offset.size());
+        chassert(start_pos_ == nullptr);
 
         if (haystack.empty())
             return;

From f0aaac3bd19c21a796f54ba080fd67f92959131a Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Sun, 4 Aug 2024 14:14:28 +0200
Subject: [PATCH 1330/1488] tests: remove useless retries from
 test_ttl_move::test_alter_with_merge_work

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 tests/integration/test_ttl_move/test.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/integration/test_ttl_move/test.py b/tests/integration/test_ttl_move/test.py
index 3b79ea7916d..48a6224347d 100644
--- a/tests/integration/test_ttl_move/test.py
+++ b/tests/integration/test_ttl_move/test.py
@@ -76,7 +76,7 @@ def get_used_disks_for_table(node, table_name, partition=None):
     )
 
 
-def check_used_disks_with_retry(node, table_name, expected_disks, retries):
+def check_used_disks_with_retry(node, table_name, expected_disks, retries=1):
     for _ in range(retries):
         used_disks = get_used_disks_for_table(node, table_name)
         if set(used_disks).issubset(expected_disks):
@@ -1635,9 +1635,9 @@ def test_alter_with_merge_work(started_cluster, name, engine, positive):
         optimize_table(20)
 
         if positive:
-            assert check_used_disks_with_retry(node1, name, set(["external"]), 100)
+            assert check_used_disks_with_retry(node1, name, set(["external"]))
         else:
-            assert check_used_disks_with_retry(node1, name, set(["jbod1", "jbod2"]), 50)
+            assert check_used_disks_with_retry(node1, name, set(["jbod1", "jbod2"]))
 
         time.sleep(5)
 

From 47dbc5e05b12213a08c25ade9536603a3fd2b175 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Sun, 4 Aug 2024 14:16:17 +0200
Subject: [PATCH 1331/1488] tests: add debug info into
 test_ttl_move::test_alter_with_merge_work

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 tests/integration/test_ttl_move/test.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/tests/integration/test_ttl_move/test.py b/tests/integration/test_ttl_move/test.py
index 48a6224347d..4ebe9a30699 100644
--- a/tests/integration/test_ttl_move/test.py
+++ b/tests/integration/test_ttl_move/test.py
@@ -1635,9 +1635,17 @@ def test_alter_with_merge_work(started_cluster, name, engine, positive):
         optimize_table(20)
 
         if positive:
-            assert check_used_disks_with_retry(node1, name, set(["external"]))
+            assert check_used_disks_with_retry(
+                node1, name, set(["external"])
+            ), "Parts: " + node1.query(
+                f"SELECT disk_name, name FROM system.parts WHERE table = '{name}' AND active = 1"
+            )
         else:
-            assert check_used_disks_with_retry(node1, name, set(["jbod1", "jbod2"]))
+            assert check_used_disks_with_retry(
+                node1, name, set(["jbod1", "jbod2"])
+            ), "Parts: " + node1.query(
+                f"SELECT disk_name, name FROM system.parts WHERE table = '{name}' AND active = 1"
+            )
 
         time.sleep(5)
 

From 062490e1b40a8df8d63fca567b11e7dd26cf52ee Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Sun, 4 Aug 2024 14:22:24 +0200
Subject: [PATCH 1332/1488] tests: fix
 test_ttl_move::test_alter_with_merge_work flakiness
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Increase timeout for TTL DELETE, since otherwise if other routines will
take too long, the part will be removed when it should be still be on
"external" disk:

    2024.08.04 03:48:53.803032 [ 622 ] {} <Information> default.mt_test_alter_with_merge_work_1722743323 (9dc6904a-f082-4f06-be7a-efe4733e811c): Will drop empty part all_1_3_4_4

And this is how part_log looks like:

    SELECT
        event_time,
        event_type,
        rows,
        part_name,
        error,
        database,
        disk_name
    FROM system.part_log
    WHERE `table` = 'mt_test_alter_with_merge_work_1722743323'
    ORDER BY event_time ASC

    Query id: a118b3cd-e4fe-45a5-b675-d73bdd887d79

        ┌──────────event_time─┬─event_type─┬─rows─┬─part_name───┬─error─┬─database─┬─disk_name─┐
     1. │ 2024-08-04 03:48:44 │ NewPart    │    2 │ all_1_1_0   │     0 │ default  │ jbod1     │
     2. │ 2024-08-04 03:48:44 │ NewPart    │    2 │ all_2_2_0   │     0 │ default  │ jbod2     │
     3. │ 2024-08-04 03:48:45 │ NewPart    │    2 │ all_3_3_0   │     0 │ default  │ jbod1     │
     4. │ 2024-08-04 03:48:46 │ MutatePart │    2 │ all_1_1_0_4 │     0 │ default  │ jbod1     │
     5. │ 2024-08-04 03:48:46 │ MutatePart │    2 │ all_2_2_0_4 │     0 │ default  │ jbod2     │
     6. │ 2024-08-04 03:48:46 │ MutatePart │    2 │ all_3_3_0_4 │     0 │ default  │ jbod1     │
     7. │ 2024-08-04 03:48:47 │ MovePart   │    2 │ all_1_1_0_4 │     0 │ default  │ external  │
     8. │ 2024-08-04 03:48:47 │ MovePart   │    2 │ all_3_3_0_4 │     0 │ default  │ jbod2     │
     9. │ 2024-08-04 03:48:47 │ MergeParts │    6 │ all_1_3_1_4 │     0 │ default  │ jbod2     │
    10. │ 2024-08-04 03:48:48 │ MovePart   │    6 │ all_1_3_1_4 │     0 │ default  │ external  │
    11. │ 2024-08-04 03:48:52 │ MergeParts │    4 │ all_1_3_2_4 │     0 │ default  │ external  │
    12. │ 2024-08-04 03:48:53 │ MergeParts │    0 │ all_1_3_3_4 │     0 │ default  │ external  │ # rows==0
    13. │ 2024-08-04 03:48:53 │ MergeParts │    0 │ all_1_3_4_4 │     0 │ default  │ external  │
        └─────────────────────┴────────────┴──────┴─────────────┴───────┴──────────┴───────────┘

CI: https://s3.amazonaws.com/clickhouse-test-reports/66671/2f00c962711e13ca00af324366421fe4593b4ce6/integration_tests__tsan__[5_6].html
Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 tests/integration/test_ttl_move/test.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/integration/test_ttl_move/test.py b/tests/integration/test_ttl_move/test.py
index 4ebe9a30699..925bdf9baaa 100644
--- a/tests/integration/test_ttl_move/test.py
+++ b/tests/integration/test_ttl_move/test.py
@@ -1613,7 +1613,7 @@ def test_alter_with_merge_work(started_cluster, name, engine, positive):
             ALTER TABLE {name} MODIFY
             TTL d1 + INTERVAL 0 SECOND TO DISK 'jbod2',
                 d1 + INTERVAL 5 SECOND TO VOLUME 'external',
-                d1 + INTERVAL 10 SECOND DELETE
+                d1 + INTERVAL 30 SECOND DELETE
         """.format(
                 name=name
             )
@@ -1647,7 +1647,7 @@ def test_alter_with_merge_work(started_cluster, name, engine, positive):
                 f"SELECT disk_name, name FROM system.parts WHERE table = '{name}' AND active = 1"
             )
 
-        time.sleep(5)
+        time.sleep(25)
 
         optimize_table(20)
 

From 9f31488e502c2b2c02e3058f9794829aac14f8b9 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Sun, 4 Aug 2024 15:43:18 +0200
Subject: [PATCH 1333/1488] Fix dictionary hang in case of CANNOT_SCHEDULE_TASK
 while loading

On CI you can find that 01747_executable_pool_dictionary_implicit_key
can hang [1], it is possible due to after CANNOT_SCHEDULE_TASK the async
loading will hang:

    2024.07.18 03:56:32.365226 [ 6138 ] {6206a18f-668c-4a5c-a5ad-07f577220762} <Trace> ExternalDictionariesLoader: Will load the object 'executable_pool_simple_implicit_key' in background, force = false, loading_id = 2
    2024.07.18 03:56:32.368005 [ 6138 ] {6206a18f-668c-4a5c-a5ad-07f577220762} <Error> executeQuery: Code: 439. DB::Exception: Cannot schedule a task: fault injected (threads=766, jobs=746): In scope SELECT dictGet('executable_pool_simple_implicit_key', 'a', toUInt64(1)). (CANNOT_SCHEDULE_TASK) (version 24.7.1.2241) (from [::1]:56446) (comment: 01747_executable_pool_dictionary_implicit_key.sql) (in query: SELECT dictGet('executable_pool_simple_implicit_key', 'a', toUInt64(1));), Stack trace (when copying this message, always include the lines below):
    0. /build/contrib/llvm-project/libcxx/include/exception:141: Poco::Exception::Exception(String const&, int) @ 0x0000000015f8a292
    1. /build/src/Common/Exception.cpp:110: DB::Exception::Exception(DB::Exception::MessageMasked&&, int, bool) @ 0x000000000c3df6b9
    2. /build/contrib/llvm-project/libcxx/include/string:1499: DB::Exception::Exception(PreformattedMessage&&, int) @ 0x0000000006de714c
    3. /build/contrib/llvm-project/libcxx/include/vector:438: DB::Exception::Exception<String const&, unsigned long, unsigned long&>(int, FormatStringHelperImpl<std::type_identity<String const&>::type, std::type_identity<unsigned long>::type, std::type_identity<unsigned long&>::type>, String const&, unsigned long&&, unsigned long&) @ 0x000000000c4838eb
    4. /build/src/Common/ThreadPool.cpp:0: void ThreadPoolImpl<std::thread>::scheduleImpl<void>(std::function<void ()>, Priority, std::optional<unsigned long>, bool)::'lambda'(String const&)::operator()(String const&) const @ 0x000000000c4832d3
    5. /build/src/Common/ThreadPool.cpp:186: void ThreadPoolImpl<std::thread>::scheduleImpl<void>(std::function<void ()>, Priority, std::optional<unsigned long>, bool) @ 0x000000000c47e7db
    6. /build/contrib/llvm-project/libcxx/include/__functional/function.h:818: ? @ 0x000000000c47ec8d
    7. /build/contrib/llvm-project/libcxx/include/__functional/function.h:818: ? @ 0x000000001114b16e
    8. /build/contrib/llvm-project/libcxx/include/__memory/shared_ptr.h:701: DB::ExternalLoader::LoadingDispatcher::startLoading(DB::ExternalLoader::LoadingDispatcher::Info&, bool, unsigned long) @ 0x0000000011147733
    9. /build/src/Interpreters/ExternalLoader.cpp:837: DB::ExternalLoader::LoadingDispatcher::loadImpl(String const&, std::chrono::duration<long long, std::ratio<1l, 1000l>>, bool, std::unique_lock<std::mutex>&)::'lambda'()::operator()() const @ 0x0000000011158bf9
    10. /build/contrib/llvm-project/libcxx/include/__mutex_base:397: DB::ExternalLoader::LoadingDispatcher::loadImpl(String const&, std::chrono::duration<long long, std::ratio<1l, 1000l>>, bool, std::unique_lock<std::mutex>&) @ 0x00000000111588bc
    11. /build/src/Interpreters/ExternalLoader.cpp:604: DB::ExternalLoader::LoadResult DB::ExternalLoader::LoadingDispatcher::tryLoad<DB::ExternalLoader::LoadResult>(String const&, std::chrono::duration<long long, std::ratio<1l, 1000l>>) @ 0x00000000111440bf
    12. /build/src/Interpreters/ExternalLoader.cpp:1381: std::shared_ptr<DB::IExternalLoadable const> DB::ExternalLoader::load<std::shared_ptr<DB::IExternalLoadable const>, void>(String const&) const @ 0x00000000111442f5
    13. /build/contrib/llvm-project/libcxx/include/__memory/shared_ptr.h:587: DB::ExternalDictionariesLoader::getDictionary(String const&, std::shared_ptr<DB::Context const>) const @ 0x0000000011141028
    14. /build/src/Functions/FunctionsExternalDictionaries.h:76: DB::FunctionDictHelper::getDictionary(String const&) @ 0x00000000071d28ec
    ...
    2024.07.18 03:58:29.000900 [ 48468 ] {8cf63d7e-dcbf-4af6-bd7c-0e1789ddce3b} <Debug> executeQuery: (from [::1]:40410) (comment: 01747_executable_pool_dictionary_implicit_key.sql) SELECT dictGet('executable_pool_simple_implicit_key', 'a', toUInt64(1)); (stage: Complete)
    # and no more rows for 8cf63d7e-dcbf-4af6-bd7c-0e1789ddce3b

  [1]: https://s3.amazonaws.com/clickhouse-test-reports/66495/bc029ed8207ac75e96e9cb48cb79d27a9ffa4e2f/stress_test__debug_.html

The problem that it should be properly cancelled, otherwise it will not
be loaded in loadImpl(), but will be waited.

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 src/Interpreters/ExternalLoader.cpp | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/src/Interpreters/ExternalLoader.cpp b/src/Interpreters/ExternalLoader.cpp
index 96405f35f3f..511300be2e0 100644
--- a/src/Interpreters/ExternalLoader.cpp
+++ b/src/Interpreters/ExternalLoader.cpp
@@ -922,7 +922,16 @@ private:
         if (enable_async_loading)
         {
             /// Put a job to the thread pool for the loading.
-            auto thread = ThreadFromGlobalPool{&LoadingDispatcher::doLoading, this, info.name, loading_id, forced_to_reload, min_id_to_finish_loading_dependencies_, true, CurrentThread::getGroup()};
+            ThreadFromGlobalPool thread;
+            try
+            {
+                thread = ThreadFromGlobalPool{&LoadingDispatcher::doLoading, this, info.name, loading_id, forced_to_reload, min_id_to_finish_loading_dependencies_, true, CurrentThread::getGroup()};
+            }
+            catch (...)
+            {
+                cancelLoading(info);
+                throw;
+            }
             loading_threads.try_emplace(loading_id, std::move(thread));
         }
         else

From dea95e6c332392a87ec3175bb32a50770a57d65b Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Sun, 14 Jul 2024 16:06:07 +0200
Subject: [PATCH 1334/1488] Ensure that llvm-symbolizer is used for symbolizing
 sanitizer reports

Since you don't want to fallback to addr2line:

    # addr2line
    $ time /bin/test
    set_flag_impl: Success
    set_flag_if: Success
    /usr/bin/addr2line: DWARF error: invalid or unhandled FORM value: 0x23
    ==================
    WARNING: ThreadSanitizer: data race (pid=18)
    ...
    real    3m8.580s
    user    0m21.967s
    sys     0m40.628s

    # llvm-symbolizer
    $ time ./test
    set_flag_impl: Success
    set_flag_if: Success
    ==================
    WARNING: ThreadSanitizer: data race (pid=24884)
    real    0m0.028s
    user    0m0.003s
    sys     0m0.006s

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 docker/test/base/Dockerfile | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/docker/test/base/Dockerfile b/docker/test/base/Dockerfile
index a81826ed6b5..e7ac62604dc 100644
--- a/docker/test/base/Dockerfile
+++ b/docker/test/base/Dockerfile
@@ -35,7 +35,9 @@ ENV UBSAN_OPTIONS='print_stacktrace=1 max_allocation_size_mb=32768'
 ENV MSAN_OPTIONS='abort_on_error=1 poison_in_dtor=1 max_allocation_size_mb=32768'
 ENV LSAN_OPTIONS='max_allocation_size_mb=32768'
 
-# for external_symbolizer_path
+# for external_symbolizer_path, and also ensure that llvm-symbolizer really
+# exists (since you don't want to fallback to addr2line, it is very slow)
+RUN test -f /usr/bin/llvm-symbolizer-${LLVM_VERSION}
 RUN ln -s /usr/bin/llvm-symbolizer-${LLVM_VERSION} /usr/bin/llvm-symbolizer
 
 RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen en_US.UTF-8

From 1f2bb6676f39259ec20dd0b077381d251b591d3a Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Sun, 14 Jul 2024 16:08:09 +0200
Subject: [PATCH 1335/1488] Remove outdated comments about workaround for
 #64086

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 docker/test/util/Dockerfile | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/docker/test/util/Dockerfile b/docker/test/util/Dockerfile
index babddbd228c..dc928ba7195 100644
--- a/docker/test/util/Dockerfile
+++ b/docker/test/util/Dockerfile
@@ -5,14 +5,6 @@ FROM ubuntu:22.04
 ARG apt_archive="http://archive.ubuntu.com"
 RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
 
-# FIXME: rebuild for clang 18.1.3, that contains a workaround [1] for
-# sanitizers issue [2]:
-#
-# $ git tag --contains  c2a57034eff048cd36c563c8e0051db3a70991b3 | tail -1
-# llvmorg-18.1.3
-#
-#  [1]: https://github.com/llvm/llvm-project/commit/c2a57034eff048cd36c563c8e0051db3a70991b3
-#  [2]: https://github.com/ClickHouse/ClickHouse/issues/64086
 ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=18
 
 RUN apt-get update \

From eed2edd7db5f2424d85a66bfee63281cba5a5c94 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Tue, 2 Jul 2024 12:23:10 +0200
Subject: [PATCH 1336/1488] Comment certificateFile/privateKeyFile/dhParamsFile
 in keeper config

Otherwise you will get annoying messages at startup:

    2024.07.02 10:03:38.331593 [ 1 ] {} <Error> CertificateReloader: Cannot obtain modification time for certificate file /etc/clickhouse-keeper/server.crt, skipping update. errno: 2, strerror: 0
    2024.07.02 10:03:38.331658 [ 1 ] {} <Error> CertificateReloader: Cannot obtain modification time for key file /etc/clickhouse-keeper/server.key, skipping update. errno: 2, strerror: 0
    2024.07.02 10:03:38.341085 [ 1 ] {} <Error> CertificateReloader: Poco::Exception. Code: 1000, e.code() = 0, SSL context exception: Error loading private key from file /etc/clickhouse-keeper/server.key: error:80000002:system library::No such file or directory

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 programs/keeper/keeper_config.xml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/programs/keeper/keeper_config.xml b/programs/keeper/keeper_config.xml
index 4cf84cffc86..efd0010d184 100644
--- a/programs/keeper/keeper_config.xml
+++ b/programs/keeper/keeper_config.xml
@@ -66,14 +66,14 @@
       <server>
             <!-- Used for secure tcp port -->
             <!-- openssl req -subj "/CN=localhost" -new -newkey rsa:2048 -days 365 -nodes -x509 -keyout /etc/clickhouse-server/server.key -out /etc/clickhouse-server/server.crt -->
-            <certificateFile>/etc/clickhouse-keeper/server.crt</certificateFile>
-            <privateKeyFile>/etc/clickhouse-keeper/server.key</privateKeyFile>
+            <!-- <certificateFile>/etc/clickhouse-keeper/server.crt</certificateFile> -->
+            <!-- <privateKeyFile>/etc/clickhouse-keeper/server.key</privateKeyFile> -->
             <!-- dhparams are optional. You can delete the <dhParamsFile> element.
                  To generate dhparams, use the following command:
                   openssl dhparam -out /etc/clickhouse-keeper/dhparam.pem 4096
                  Only file format with BEGIN DH PARAMETERS is supported.
               -->
-            <dhParamsFile>/etc/clickhouse-keeper/dhparam.pem</dhParamsFile>
+            <!-- <dhParamsFile>/etc/clickhouse-keeper/dhparam.pem</dhParamsFile> -->
             <verificationMode>none</verificationMode>
             <loadDefaultCAFile>true</loadDefaultCAFile>
             <cacheSessions>true</cacheSessions>

From a75c4b9f9fe519cb74a47d88efb55216da22a64b Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 4 Aug 2024 18:07:40 +0200
Subject: [PATCH 1337/1488] Introduce `no-flaky-check` tag

---
 tests/clickhouse-test                                          | 3 +++
 .../00002_log_and_exception_messages_formatting.sql            | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/tests/clickhouse-test b/tests/clickhouse-test
index 893fded8d23..b70dd61a25a 100755
--- a/tests/clickhouse-test
+++ b/tests/clickhouse-test
@@ -1232,6 +1232,9 @@ class TestCase:
         ):
             return FailureReason.SKIP
 
+        elif "no-flaky-check" in tags and (args.test_runs > 1):
+            return FailureReason.SKIP
+
         elif tags:
             for build_flag in args.build_flags:
                 if "no-" + build_flag in tags:
diff --git a/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql b/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql
index 07c42d6d039..32db77cd8dd 100644
--- a/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql
+++ b/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql
@@ -1,4 +1,4 @@
--- Tags: no-parallel, no-fasttest, no-ubsan, no-batch
+-- Tags: no-parallel, no-fasttest, no-ubsan, no-batch, no-flaky-check
 -- no-parallel because we want to run this test when most of the other tests already passed
 
 -- If this test fails, see the "Top patterns of log messages" diagnostics in the end of run.log

From 87e0cf6b7635b7249d72c6ac0e7aad75c863d01d Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 4 Aug 2024 18:25:36 +0200
Subject: [PATCH 1338/1488] Fix stack overflow in JSONMergePatch

---
 src/Common/JSONParsers/RapidJSONParser.h | 12 ++++++---
 src/Functions/jsonMergePatch.cpp         | 34 +++++++++++++++---------
 2 files changed, 29 insertions(+), 17 deletions(-)

diff --git a/src/Common/JSONParsers/RapidJSONParser.h b/src/Common/JSONParsers/RapidJSONParser.h
index 6c5ea938bfe..ad7a4cbf53a 100644
--- a/src/Common/JSONParsers/RapidJSONParser.h
+++ b/src/Common/JSONParsers/RapidJSONParser.h
@@ -3,10 +3,14 @@
 #include "config.h"
 
 #if USE_RAPIDJSON
-#    include <base/types.h>
-#    include <base/defines.h>
-#    include <rapidjson/document.h>
-#    include "ElementTypes.h"
+
+/// Prevent stack overflow:
+#define RAPIDJSON_PARSE_DEFAULT_FLAGS (kParseIterativeFlag)
+
+#include <base/types.h>
+#include <base/defines.h>
+#include <rapidjson/document.h>
+#include "ElementTypes.h"
 
 namespace DB
 {
diff --git a/src/Functions/jsonMergePatch.cpp b/src/Functions/jsonMergePatch.cpp
index a83daacdbf6..3bde415aabf 100644
--- a/src/Functions/jsonMergePatch.cpp
+++ b/src/Functions/jsonMergePatch.cpp
@@ -10,12 +10,14 @@
 
 #if USE_RAPIDJSON
 
-#include "rapidjson/document.h"
-#include "rapidjson/writer.h"
-#include "rapidjson/stringbuffer.h"
-#include "rapidjson/filewritestream.h"
-#include "rapidjson/prettywriter.h"
-#include "rapidjson/filereadstream.h"
+/// Prevent stack overflow:
+#define RAPIDJSON_PARSE_DEFAULT_FLAGS (kParseIterativeFlag)
+
+#include <rapidjson/document.h>
+#include <rapidjson/writer.h>
+#include <rapidjson/stringbuffer.h>
+#include <rapidjson/filereadstream.h>
+#include <rapidjson/error/en.h>
 
 
 namespace DB
@@ -31,17 +33,17 @@ namespace ErrorCodes
 
 namespace
 {
-    // select jsonMergePatch('{"a":1}','{"name": "joey"}','{"name": "tom"}','{"name": "zoey"}');
+    // select JSONMergePatch('{"a":1}','{"name": "joey"}','{"name": "tom"}','{"name": "zoey"}');
     //           ||
     //           \/
     // ┌───────────────────────┐
     // │ {"a":1,"name":"zoey"} │
     // └───────────────────────┘
-    class FunctionjsonMergePatch : public IFunction
+    class FunctionJSONMergePatch : public IFunction
     {
     public:
-        static constexpr auto name = "jsonMergePatch";
-        static FunctionPtr create(ContextPtr) { return std::make_shared<FunctionjsonMergePatch>(); }
+        static constexpr auto name = "JSONMergePatch";
+        static FunctionPtr create(ContextPtr) { return std::make_shared<FunctionJSONMergePatch>(); }
 
         String getName() const override { return name; }
         bool isVariadic() const override { return true; }
@@ -98,7 +100,11 @@ namespace
                 const char * json = str_ref.data;
 
                 document.Parse(json);
-                if (document.HasParseError() || !document.IsObject())
+
+                if (document.HasParseError())
+                    throw Exception(ErrorCodes::BAD_ARGUMENTS, "Wrong JSON string to merge: {}", rapidjson::GetParseError_En(document.GetParseError()));
+
+                if (!document.IsObject())
                     throw Exception(ErrorCodes::BAD_ARGUMENTS, "Wrong JSON string to merge. Expected JSON object");
             };
 
@@ -162,10 +168,12 @@ namespace
 
 }
 
-REGISTER_FUNCTION(jsonMergePatch)
+REGISTER_FUNCTION(JSONMergePatch)
 {
-    factory.registerFunction<FunctionjsonMergePatch>(FunctionDocumentation{
+    factory.registerFunction<FunctionJSONMergePatch>(FunctionDocumentation{
         .description="Returns the merged JSON object string, which is formed by merging multiple JSON objects."});
+
+    factory.registerAlias("jsonMergePatch", "JSONMergePatch");
 }
 
 }

From 37da0b3c307d249a078b94cb7bc53d8acdd22a98 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 4 Aug 2024 18:27:37 +0200
Subject: [PATCH 1339/1488] Add a test

---
 .../03217_json_merge_patch_stack_overflow.reference      | 0
 .../03217_json_merge_patch_stack_overflow.sql            | 9 +++++++++
 2 files changed, 9 insertions(+)
 create mode 100644 tests/queries/0_stateless/03217_json_merge_patch_stack_overflow.reference
 create mode 100644 tests/queries/0_stateless/03217_json_merge_patch_stack_overflow.sql

diff --git a/tests/queries/0_stateless/03217_json_merge_patch_stack_overflow.reference b/tests/queries/0_stateless/03217_json_merge_patch_stack_overflow.reference
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/queries/0_stateless/03217_json_merge_patch_stack_overflow.sql b/tests/queries/0_stateless/03217_json_merge_patch_stack_overflow.sql
new file mode 100644
index 00000000000..4b366b08c6b
--- /dev/null
+++ b/tests/queries/0_stateless/03217_json_merge_patch_stack_overflow.sql
@@ -0,0 +1,9 @@
+-- Tags: no-fasttest
+-- Needs rapidjson library
+SELECT JSONMergePatch(REPEAT('{"c":', 1000000)); -- { serverError BAD_ARGUMENTS }
+SELECT JSONMergePatch(REPEAT('{"c":', 100000)); -- { serverError BAD_ARGUMENTS }
+SELECT JSONMergePatch(REPEAT('{"c":', 10000)); -- { serverError BAD_ARGUMENTS }
+SELECT JSONMergePatch(REPEAT('{"c":', 1000)); -- { serverError BAD_ARGUMENTS }
+SELECT JSONMergePatch(REPEAT('{"c":', 100)); -- { serverError BAD_ARGUMENTS }
+SELECT JSONMergePatch(REPEAT('{"c":', 10)); -- { serverError BAD_ARGUMENTS }
+SELECT JSONMergePatch(REPEAT('{"c":', 1)); -- { serverError BAD_ARGUMENTS }

From a433115434ebe4c1d69f2ed200005fa93c7adcb7 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 4 Aug 2024 19:18:00 +0200
Subject: [PATCH 1340/1488] Fix typo

---
 docker/test/util/process_functional_tests_result.py |  6 +++---
 docs/changelogs/v20.5.1.3833-prestable.md           |  2 +-
 docs/changelogs/v21.11.1.8636-prestable.md          |  6 +++---
 docs/changelogs/v21.12.1.9017-prestable.md          | 10 +++++-----
 docs/changelogs/v21.5.1.6601-prestable.md           |  2 +-
 docs/changelogs/v21.6.9.7-stable.md                 |  2 +-
 docs/changelogs/v21.7.9.7-stable.md                 |  2 +-
 docs/changelogs/v21.8.1.7409-prestable.md           |  2 +-
 docs/changelogs/v21.8.5.7-lts.md                    |  2 +-
 docs/changelogs/v21.9.1.8000-prestable.md           |  2 +-
 docs/changelogs/v22.1.1.2542-prestable.md           |  4 ++--
 docs/changelogs/v22.4.1.2305-prestable.md           |  2 +-
 docs/changelogs/v23.4.1.1943-stable.md              |  3 +--
 docs/changelogs/v23.6.1.1524-stable.md              |  2 +-
 docs/zh/changelog/index.md                          |  2 +-
 src/Common/SystemLogBase.cpp                        |  2 +-
 16 files changed, 25 insertions(+), 26 deletions(-)

diff --git a/docker/test/util/process_functional_tests_result.py b/docker/test/util/process_functional_tests_result.py
index 3da1a8f3674..aa2ea686c46 100755
--- a/docker/test/util/process_functional_tests_result.py
+++ b/docker/test/util/process_functional_tests_result.py
@@ -161,11 +161,11 @@ def process_result(result_path, broken_tests):
             retries,
             test_results,
         ) = process_test_log(result_path, broken_tests)
-        is_flacky_check = 1 < int(os.environ.get("NUM_TRIES", 1))
-        logging.info("Is flaky check: %s", is_flacky_check)
+        is_flaky_check = 1 < int(os.environ.get("NUM_TRIES", 1))
+        logging.info("Is flaky check: %s", is_flaky_check)
         # If no tests were run (success == 0) it indicates an error (e.g. server did not start or crashed immediately)
         # But it's Ok for "flaky checks" - they can contain just one test for check which is marked as skipped.
-        if failed != 0 or unknown != 0 or (success == 0 and (not is_flacky_check)):
+        if failed != 0 or unknown != 0 or (success == 0 and (not is_flaky_check)):
             state = "failure"
 
         if hung:
diff --git a/docs/changelogs/v20.5.1.3833-prestable.md b/docs/changelogs/v20.5.1.3833-prestable.md
index 79d61fb12f9..3de263f5e37 100644
--- a/docs/changelogs/v20.5.1.3833-prestable.md
+++ b/docs/changelogs/v20.5.1.3833-prestable.md
@@ -331,7 +331,7 @@
 * Fix several non significant errors in unit tests. [#11262](https://github.com/ClickHouse/ClickHouse/pull/11262) ([alesapin](https://github.com/alesapin)).
 * Add a test for Join table engine from @donmikel. This closes [#9158](https://github.com/ClickHouse/ClickHouse/issues/9158). [#11265](https://github.com/ClickHouse/ClickHouse/pull/11265) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
 * Repeat test in CI if `curl` invocation was timed out. It is possible due to system hangups for 10+ seconds that are typical in our CI infrastructure. This fixes [#11267](https://github.com/ClickHouse/ClickHouse/issues/11267). [#11268](https://github.com/ClickHouse/ClickHouse/pull/11268) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
-* Fix potentially flacky test `00731_long_merge_tree_select_opened_files.sh`. It does not fail frequently but we have discovered potential race condition in this test while experimenting with ThreadFuzzer: [#9814](https://github.com/ClickHouse/ClickHouse/issues/9814) See [link](https://clickhouse-test-reports.s3.yandex.net/9814/40e3023e215df22985d275bf85f4d2290897b76b/functional_stateless_tests_(unbundled).html#fail1) for the example. [#11270](https://github.com/ClickHouse/ClickHouse/pull/11270) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Fix potentially flaky test `00731_long_merge_tree_select_opened_files.sh`. It does not fail frequently but we have discovered potential race condition in this test while experimenting with ThreadFuzzer: [#9814](https://github.com/ClickHouse/ClickHouse/issues/9814) See [link](https://clickhouse-test-reports.s3.yandex.net/9814/40e3023e215df22985d275bf85f4d2290897b76b/functional_stateless_tests_(unbundled).html#fail1) for the example. [#11270](https://github.com/ClickHouse/ClickHouse/pull/11270) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
 * Now clickhouse-test check the server aliveness before tests run. [#11285](https://github.com/ClickHouse/ClickHouse/pull/11285) ([alesapin](https://github.com/alesapin)).
 * Emit a warning if server was build in debug or with sanitizers. [#11304](https://github.com/ClickHouse/ClickHouse/pull/11304) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
 * Better check for hung queries in clickhouse-test. [#11321](https://github.com/ClickHouse/ClickHouse/pull/11321) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
diff --git a/docs/changelogs/v21.11.1.8636-prestable.md b/docs/changelogs/v21.11.1.8636-prestable.md
index d6a435dd3ce..95c8580d591 100644
--- a/docs/changelogs/v21.11.1.8636-prestable.md
+++ b/docs/changelogs/v21.11.1.8636-prestable.md
@@ -280,7 +280,7 @@ sidebar_label: 2022
 * Cleanup unbundled image [#29689](https://github.com/ClickHouse/ClickHouse/pull/29689) ([Azat Khuzhin](https://github.com/azat)).
 * Fix memory tracking for merges and mutations [#29691](https://github.com/ClickHouse/ClickHouse/pull/29691) ([Azat Khuzhin](https://github.com/azat)).
 * Fix data-race in WriteIndirectBuffer (used in DiskMemory) [#29692](https://github.com/ClickHouse/ClickHouse/pull/29692) ([Azat Khuzhin](https://github.com/azat)).
-* Fix flacky test [#29706](https://github.com/ClickHouse/ClickHouse/pull/29706) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fix flaky test [#29706](https://github.com/ClickHouse/ClickHouse/pull/29706) ([Kseniia Sumarokova](https://github.com/kssenii)).
 * BorrowedObjectPool condition variable notify fix [#29722](https://github.com/ClickHouse/ClickHouse/pull/29722) ([Maksim Kita](https://github.com/kitaisreal)).
 * Better exception message for local interactive [#29737](https://github.com/ClickHouse/ClickHouse/pull/29737) ([Kseniia Sumarokova](https://github.com/kssenii)).
 * Fix --stage for clickhouse-local [#29745](https://github.com/ClickHouse/ClickHouse/pull/29745) ([Azat Khuzhin](https://github.com/azat)).
@@ -308,7 +308,7 @@ sidebar_label: 2022
 * Fix client [#29864](https://github.com/ClickHouse/ClickHouse/pull/29864) ([Kseniia Sumarokova](https://github.com/kssenii)).
 * Remove some more streams. [#29898](https://github.com/ClickHouse/ClickHouse/pull/29898) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
 * Add logging in ZooKeeper client [#29901](https://github.com/ClickHouse/ClickHouse/pull/29901) ([Alexander Tokmakov](https://github.com/tavplubix)).
-* Fix some flacky tests [#29902](https://github.com/ClickHouse/ClickHouse/pull/29902) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fix some flaky tests [#29902](https://github.com/ClickHouse/ClickHouse/pull/29902) ([Kseniia Sumarokova](https://github.com/kssenii)).
 * Grep server log even if it contains binary data [#29903](https://github.com/ClickHouse/ClickHouse/pull/29903) ([Alexander Tokmakov](https://github.com/tavplubix)).
 * Cosmetic refactoring of server constants. [#29913](https://github.com/ClickHouse/ClickHouse/pull/29913) ([Amos Bird](https://github.com/amosbird)).
 * Format improvement of AlterQuery [#29916](https://github.com/ClickHouse/ClickHouse/pull/29916) ([flynn](https://github.com/ucasfl)).
@@ -465,7 +465,7 @@ sidebar_label: 2022
 * Fix docs release [#30933](https://github.com/ClickHouse/ClickHouse/pull/30933) ([alesapin](https://github.com/alesapin)).
 * Fix style check [#30937](https://github.com/ClickHouse/ClickHouse/pull/30937) ([alesapin](https://github.com/alesapin)).
 * Fix file progress for clickhouse-local [#30938](https://github.com/ClickHouse/ClickHouse/pull/30938) ([Kseniia Sumarokova](https://github.com/kssenii)).
-* Fix flacky test [#30940](https://github.com/ClickHouse/ClickHouse/pull/30940) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fix flaky test [#30940](https://github.com/ClickHouse/ClickHouse/pull/30940) ([Kseniia Sumarokova](https://github.com/kssenii)).
 * Fix reading from TinyLog [#30941](https://github.com/ClickHouse/ClickHouse/pull/30941) ([Vitaly Baranov](https://github.com/vitlibar)).
 * Add github to known hosts in docs release [#30947](https://github.com/ClickHouse/ClickHouse/pull/30947) ([alesapin](https://github.com/alesapin)).
 * Parse json from response in ci checks [#30948](https://github.com/ClickHouse/ClickHouse/pull/30948) ([alesapin](https://github.com/alesapin)).
diff --git a/docs/changelogs/v21.12.1.9017-prestable.md b/docs/changelogs/v21.12.1.9017-prestable.md
index bd84873e67a..f5416664d35 100644
--- a/docs/changelogs/v21.12.1.9017-prestable.md
+++ b/docs/changelogs/v21.12.1.9017-prestable.md
@@ -220,7 +220,7 @@ sidebar_label: 2022
 * Fix test_backward_compatibility [#30950](https://github.com/ClickHouse/ClickHouse/pull/30950) ([Ilya Yatsishin](https://github.com/qoega)).
 * Add stress test to github actions [#30952](https://github.com/ClickHouse/ClickHouse/pull/30952) ([alesapin](https://github.com/alesapin)).
 * Try smaller blacklist of non parallel integration tests  [#30963](https://github.com/ClickHouse/ClickHouse/pull/30963) ([Ilya Yatsishin](https://github.com/qoega)).
-* Fix flacky test [#30967](https://github.com/ClickHouse/ClickHouse/pull/30967) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fix flaky test [#30967](https://github.com/ClickHouse/ClickHouse/pull/30967) ([Kseniia Sumarokova](https://github.com/kssenii)).
 * Move access-rights source code [#30973](https://github.com/ClickHouse/ClickHouse/pull/30973) ([Vitaly Baranov](https://github.com/vitlibar)).
 * Set output_format_avro_rows_in_file default to 1 [#30990](https://github.com/ClickHouse/ClickHouse/pull/30990) ([Kruglov Pavel](https://github.com/Avogar)).
 * Remove remaining usages of Y_IGNORE [#30993](https://github.com/ClickHouse/ClickHouse/pull/30993) ([Yuriy Chernyshov](https://github.com/georgthegreat)).
@@ -353,7 +353,7 @@ sidebar_label: 2022
 * Support toUInt8/toInt8 for if constant condition optimization. [#31866](https://github.com/ClickHouse/ClickHouse/pull/31866) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
 * Added -no-sanitize=unsigned-integer-overflow build flag [#31881](https://github.com/ClickHouse/ClickHouse/pull/31881) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
 * Fix typos [#31886](https://github.com/ClickHouse/ClickHouse/pull/31886) ([Anton Popov](https://github.com/CurtizJ)).
-* Try to fix flacky test. [#31889](https://github.com/ClickHouse/ClickHouse/pull/31889) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Try to fix flaky test. [#31889](https://github.com/ClickHouse/ClickHouse/pull/31889) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
 * Reduce the files that depend on parser headers [#31896](https://github.com/ClickHouse/ClickHouse/pull/31896) ([Raúl Marín](https://github.com/Algunenano)).
 * Fix magic_enum for debug helpers (fixes build w/ USE_DEBUG_HELPERS) [#31922](https://github.com/ClickHouse/ClickHouse/pull/31922) ([Azat Khuzhin](https://github.com/azat)).
 * Remove some trash from build [#31923](https://github.com/ClickHouse/ClickHouse/pull/31923) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
@@ -387,7 +387,7 @@ sidebar_label: 2022
 * make looping in H3 funcs uniform [#32110](https://github.com/ClickHouse/ClickHouse/pull/32110) ([Bharat Nallan](https://github.com/bharatnc)).
 * Remove PVS check from master [#32114](https://github.com/ClickHouse/ClickHouse/pull/32114) ([alesapin](https://github.com/alesapin)).
 * Fix flaky keeper whitelist test [#32115](https://github.com/ClickHouse/ClickHouse/pull/32115) ([alesapin](https://github.com/alesapin)).
-* Fix flacky test test_executable_storage_input [#32118](https://github.com/ClickHouse/ClickHouse/pull/32118) ([Maksim Kita](https://github.com/kitaisreal)).
+* Fix flaky test test_executable_storage_input [#32118](https://github.com/ClickHouse/ClickHouse/pull/32118) ([Maksim Kita](https://github.com/kitaisreal)).
 * Fix data race in `removePartAndEnqueueFetch(...)` [#32119](https://github.com/ClickHouse/ClickHouse/pull/32119) ([Alexander Tokmakov](https://github.com/tavplubix)).
 * Move fuzzers and unit tests to another group [#32120](https://github.com/ClickHouse/ClickHouse/pull/32120) ([alesapin](https://github.com/alesapin)).
 * Add a test with 20000 mutations in one query [#32122](https://github.com/ClickHouse/ClickHouse/pull/32122) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
@@ -411,11 +411,11 @@ sidebar_label: 2022
 * Add test for [#32186](https://github.com/ClickHouse/ClickHouse/issues/32186) [#32203](https://github.com/ClickHouse/ClickHouse/pull/32203) ([Raúl Marín](https://github.com/Algunenano)).
 * Fix uncaught exception in DatabaseLazy [#32206](https://github.com/ClickHouse/ClickHouse/pull/32206) ([Alexander Tokmakov](https://github.com/tavplubix)).
 * Update ASTCreateQuery.cpp [#32208](https://github.com/ClickHouse/ClickHouse/pull/32208) ([Kseniia Sumarokova](https://github.com/kssenii)).
-* Fix flacky fileLog test (probably) [#32209](https://github.com/ClickHouse/ClickHouse/pull/32209) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fix flaky fileLog test (probably) [#32209](https://github.com/ClickHouse/ClickHouse/pull/32209) ([Kseniia Sumarokova](https://github.com/kssenii)).
 * Fix jemalloc under osx [#32219](https://github.com/ClickHouse/ClickHouse/pull/32219) ([Azat Khuzhin](https://github.com/azat)).
 * Add missing timezones to some tests [#32222](https://github.com/ClickHouse/ClickHouse/pull/32222) ([Kseniia Sumarokova](https://github.com/kssenii)).
 * Fix versioning of aggregate functions (fixes performance tests) [#32236](https://github.com/ClickHouse/ClickHouse/pull/32236) ([Azat Khuzhin](https://github.com/azat)).
-* Disable window view tests temporarily because still flacky [#32257](https://github.com/ClickHouse/ClickHouse/pull/32257) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Disable window view tests temporarily because still flaky [#32257](https://github.com/ClickHouse/ClickHouse/pull/32257) ([Kseniia Sumarokova](https://github.com/kssenii)).
 * Fix typo in tupleToNameValuePairs doc [#32262](https://github.com/ClickHouse/ClickHouse/pull/32262) ([Vladimir C](https://github.com/vdimir)).
 * Fix possible Pipeline stuck in case of StrictResize processor. [#32270](https://github.com/ClickHouse/ClickHouse/pull/32270) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
 * Fix possible crash in DataTypeAggregateFunction [#32287](https://github.com/ClickHouse/ClickHouse/pull/32287) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
diff --git a/docs/changelogs/v21.5.1.6601-prestable.md b/docs/changelogs/v21.5.1.6601-prestable.md
index b7dd8ae87c0..69ea9cb8d0e 100644
--- a/docs/changelogs/v21.5.1.6601-prestable.md
+++ b/docs/changelogs/v21.5.1.6601-prestable.md
@@ -158,7 +158,7 @@ sidebar_label: 2022
 * MemoryStorage sync comments and code [#22721](https://github.com/ClickHouse/ClickHouse/pull/22721) ([Maksim Kita](https://github.com/kitaisreal)).
 * Fix potential segfault on Keeper startup [#22743](https://github.com/ClickHouse/ClickHouse/pull/22743) ([alesapin](https://github.com/alesapin)).
 * Avoid using harmful function rand() [#22744](https://github.com/ClickHouse/ClickHouse/pull/22744) ([Amos Bird](https://github.com/amosbird)).
-* Fix flacky hedged tests [#22746](https://github.com/ClickHouse/ClickHouse/pull/22746) ([Kruglov Pavel](https://github.com/Avogar)).
+* Fix flaky hedged tests [#22746](https://github.com/ClickHouse/ClickHouse/pull/22746) ([Kruglov Pavel](https://github.com/Avogar)).
 * add more messages when flushing the logs [#22761](https://github.com/ClickHouse/ClickHouse/pull/22761) ([Alexander Kuzmenkov](https://github.com/akuzm)).
 * Moved BorrowedObjectPool to common [#22764](https://github.com/ClickHouse/ClickHouse/pull/22764) ([Maksim Kita](https://github.com/kitaisreal)).
 * Functions ExternalDictionaries standardize exception throw [#22821](https://github.com/ClickHouse/ClickHouse/pull/22821) ([Maksim Kita](https://github.com/kitaisreal)).
diff --git a/docs/changelogs/v21.6.9.7-stable.md b/docs/changelogs/v21.6.9.7-stable.md
index 0a989e4d6b7..533c58badac 100644
--- a/docs/changelogs/v21.6.9.7-stable.md
+++ b/docs/changelogs/v21.6.9.7-stable.md
@@ -55,7 +55,7 @@ sidebar_label: 2022
 * Try fix rabbitmq tests [#26826](https://github.com/ClickHouse/ClickHouse/pull/26826) ([Kseniia Sumarokova](https://github.com/kssenii)).
 * One more library bridge fix [#26873](https://github.com/ClickHouse/ClickHouse/pull/26873) ([Kseniia Sumarokova](https://github.com/kssenii)).
 * Update PVS checksum [#27317](https://github.com/ClickHouse/ClickHouse/pull/27317) ([Alexander Tokmakov](https://github.com/tavplubix)).
-* Fix flacky test [#27383](https://github.com/ClickHouse/ClickHouse/pull/27383) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fix flaky test [#27383](https://github.com/ClickHouse/ClickHouse/pull/27383) ([Kseniia Sumarokova](https://github.com/kssenii)).
 * Fix throw without exception in MySQL source. [#28027](https://github.com/ClickHouse/ClickHouse/pull/28027) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
 * Fix race between REPLACE PARTITION and MOVE PARTITION [#28035](https://github.com/ClickHouse/ClickHouse/pull/28035) ([Alexander Tokmakov](https://github.com/tavplubix)).
 * Follow-up to [#28016](https://github.com/ClickHouse/ClickHouse/issues/28016) [#28036](https://github.com/ClickHouse/ClickHouse/pull/28036) ([Alexander Tokmakov](https://github.com/tavplubix)).
diff --git a/docs/changelogs/v21.7.9.7-stable.md b/docs/changelogs/v21.7.9.7-stable.md
index 7aaab54af6b..684d0e8995e 100644
--- a/docs/changelogs/v21.7.9.7-stable.md
+++ b/docs/changelogs/v21.7.9.7-stable.md
@@ -35,7 +35,7 @@ sidebar_label: 2022
 #### NOT FOR CHANGELOG / INSIGNIFICANT
 
 * Fix prometheus metric name [#26140](https://github.com/ClickHouse/ClickHouse/pull/26140) ([Vladimir C](https://github.com/vdimir)).
-* Fix flacky test [#27383](https://github.com/ClickHouse/ClickHouse/pull/27383) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fix flaky test [#27383](https://github.com/ClickHouse/ClickHouse/pull/27383) ([Kseniia Sumarokova](https://github.com/kssenii)).
 * Fix throw without exception in MySQL source. [#28027](https://github.com/ClickHouse/ClickHouse/pull/28027) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
 * Fix race between REPLACE PARTITION and MOVE PARTITION [#28035](https://github.com/ClickHouse/ClickHouse/pull/28035) ([Alexander Tokmakov](https://github.com/tavplubix)).
 * Follow-up to [#28016](https://github.com/ClickHouse/ClickHouse/issues/28016) [#28036](https://github.com/ClickHouse/ClickHouse/pull/28036) ([Alexander Tokmakov](https://github.com/tavplubix)).
diff --git a/docs/changelogs/v21.8.1.7409-prestable.md b/docs/changelogs/v21.8.1.7409-prestable.md
index cb6ab82b30f..6ef2f1b50d2 100644
--- a/docs/changelogs/v21.8.1.7409-prestable.md
+++ b/docs/changelogs/v21.8.1.7409-prestable.md
@@ -101,7 +101,7 @@ sidebar_label: 2022
 * Separate log files for separate runs in stress test [#25741](https://github.com/ClickHouse/ClickHouse/pull/25741) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
 * Fix slow performance test [#25742](https://github.com/ClickHouse/ClickHouse/pull/25742) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
 * DatabaseAtomic EXCHANGE DICTIONARIES fix test [#25753](https://github.com/ClickHouse/ClickHouse/pull/25753) ([Maksim Kita](https://github.com/kitaisreal)).
-* Try fix flacky rabbitmq test [#25756](https://github.com/ClickHouse/ClickHouse/pull/25756) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Try fix flaky rabbitmq test [#25756](https://github.com/ClickHouse/ClickHouse/pull/25756) ([Kseniia Sumarokova](https://github.com/kssenii)).
 * Add a test for [#13993](https://github.com/ClickHouse/ClickHouse/issues/13993) [#25758](https://github.com/ClickHouse/ClickHouse/pull/25758) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
 * Set follow-fork-mode child for gdb in stress/fasttest/fuzzer [#25769](https://github.com/ClickHouse/ClickHouse/pull/25769) ([Azat Khuzhin](https://github.com/azat)).
 * Ignore TOO_DEEP_RECURSION server exception during fuzzing [#25770](https://github.com/ClickHouse/ClickHouse/pull/25770) ([Azat Khuzhin](https://github.com/azat)).
diff --git a/docs/changelogs/v21.8.5.7-lts.md b/docs/changelogs/v21.8.5.7-lts.md
index fa459e093f7..4d0727e362c 100644
--- a/docs/changelogs/v21.8.5.7-lts.md
+++ b/docs/changelogs/v21.8.5.7-lts.md
@@ -40,7 +40,7 @@ sidebar_label: 2022
 
 * Fix several bugs in ZooKeeper snapshots deserialization [#26127](https://github.com/ClickHouse/ClickHouse/pull/26127) ([alesapin](https://github.com/alesapin)).
 * Fix prometheus metric name [#26140](https://github.com/ClickHouse/ClickHouse/pull/26140) ([Vladimir C](https://github.com/vdimir)).
-* Fix flacky test [#27383](https://github.com/ClickHouse/ClickHouse/pull/27383) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fix flaky test [#27383](https://github.com/ClickHouse/ClickHouse/pull/27383) ([Kseniia Sumarokova](https://github.com/kssenii)).
 * Fix throw without exception in MySQL source. [#28027](https://github.com/ClickHouse/ClickHouse/pull/28027) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
 * Fix race between REPLACE PARTITION and MOVE PARTITION [#28035](https://github.com/ClickHouse/ClickHouse/pull/28035) ([Alexander Tokmakov](https://github.com/tavplubix)).
 * Follow-up to [#28016](https://github.com/ClickHouse/ClickHouse/issues/28016) [#28036](https://github.com/ClickHouse/ClickHouse/pull/28036) ([Alexander Tokmakov](https://github.com/tavplubix)).
diff --git a/docs/changelogs/v21.9.1.8000-prestable.md b/docs/changelogs/v21.9.1.8000-prestable.md
index bc921a68693..70ae3697e92 100644
--- a/docs/changelogs/v21.9.1.8000-prestable.md
+++ b/docs/changelogs/v21.9.1.8000-prestable.md
@@ -346,7 +346,7 @@ sidebar_label: 2022
 * Update PVS checksum [#27317](https://github.com/ClickHouse/ClickHouse/pull/27317) ([Alexander Tokmakov](https://github.com/tavplubix)).
 * Fix 01300_client_save_history_when_terminated_long [#27324](https://github.com/ClickHouse/ClickHouse/pull/27324) ([Raúl Marín](https://github.com/Algunenano)).
 * Try update contrib/zlib-ng [#27327](https://github.com/ClickHouse/ClickHouse/pull/27327) ([Ilya Yatsishin](https://github.com/qoega)).
-* Fix flacky test [#27383](https://github.com/ClickHouse/ClickHouse/pull/27383) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fix flaky test [#27383](https://github.com/ClickHouse/ClickHouse/pull/27383) ([Kseniia Sumarokova](https://github.com/kssenii)).
 * Add and check system.mutations for database filter [#27384](https://github.com/ClickHouse/ClickHouse/pull/27384) ([Azat Khuzhin](https://github.com/azat)).
 * Correct the key data type used in mapContains [#27423](https://github.com/ClickHouse/ClickHouse/pull/27423) ([Fuwang Hu](https://github.com/fuwhu)).
 * Fix tests for WithMergeableStateAfterAggregationAndLimit [#27424](https://github.com/ClickHouse/ClickHouse/pull/27424) ([Azat Khuzhin](https://github.com/azat)).
diff --git a/docs/changelogs/v22.1.1.2542-prestable.md b/docs/changelogs/v22.1.1.2542-prestable.md
index cacd13c1e12..3b0422abb11 100644
--- a/docs/changelogs/v22.1.1.2542-prestable.md
+++ b/docs/changelogs/v22.1.1.2542-prestable.md
@@ -398,7 +398,7 @@ sidebar_label: 2022
 * test for [#24410](https://github.com/ClickHouse/ClickHouse/issues/24410) [#33265](https://github.com/ClickHouse/ClickHouse/pull/33265) ([Denny Crane](https://github.com/den-crane)).
 * Wait for RabbitMQ container to actually start when it was restarted in test on purpose [#33266](https://github.com/ClickHouse/ClickHouse/pull/33266) ([Kseniia Sumarokova](https://github.com/kssenii)).
 * Mark max_alter_threads as obsolete [#33268](https://github.com/ClickHouse/ClickHouse/pull/33268) ([Denny Crane](https://github.com/den-crane)).
-* Fix azure tests flackyness because of azure server closing connection [#33269](https://github.com/ClickHouse/ClickHouse/pull/33269) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fix azure tests flakyness because of azure server closing connection [#33269](https://github.com/ClickHouse/ClickHouse/pull/33269) ([Kseniia Sumarokova](https://github.com/kssenii)).
 * Test for [#26920](https://github.com/ClickHouse/ClickHouse/issues/26920) [#33272](https://github.com/ClickHouse/ClickHouse/pull/33272) ([Denny Crane](https://github.com/den-crane)).
 * Fix test_storage_kafka failures by adjusting retention.ms [#33278](https://github.com/ClickHouse/ClickHouse/pull/33278) ([Azat Khuzhin](https://github.com/azat)).
 * Disable FunctionConvertFromString::canBeExecutedOnDefaultArguments [#33286](https://github.com/ClickHouse/ClickHouse/pull/33286) ([Vladimir C](https://github.com/vdimir)).
@@ -447,7 +447,7 @@ sidebar_label: 2022
 * Update mongodb.md [#33585](https://github.com/ClickHouse/ClickHouse/pull/33585) ([Kseniia Sumarokova](https://github.com/kssenii)).
 * Restore existing static builds links [#33597](https://github.com/ClickHouse/ClickHouse/pull/33597) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
 * Fix pylint for run_check.py [#33600](https://github.com/ClickHouse/ClickHouse/pull/33600) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
-* Fix flacky test_dictionaries_postgresql/ [#33601](https://github.com/ClickHouse/ClickHouse/pull/33601) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fix flaky test_dictionaries_postgresql/ [#33601](https://github.com/ClickHouse/ClickHouse/pull/33601) ([Kseniia Sumarokova](https://github.com/kssenii)).
 * Make ZooKeeper client better interpret keeper server connection reject [#33602](https://github.com/ClickHouse/ClickHouse/pull/33602) ([alesapin](https://github.com/alesapin)).
 * Fix broken workflow dependencies [#33608](https://github.com/ClickHouse/ClickHouse/pull/33608) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
 * Force rebuild images in CI [#33609](https://github.com/ClickHouse/ClickHouse/pull/33609) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
diff --git a/docs/changelogs/v22.4.1.2305-prestable.md b/docs/changelogs/v22.4.1.2305-prestable.md
index b277137ca7e..e8304e6505f 100644
--- a/docs/changelogs/v22.4.1.2305-prestable.md
+++ b/docs/changelogs/v22.4.1.2305-prestable.md
@@ -410,7 +410,7 @@ sidebar_label: 2022
 * Fix mongodb test with new cert [#36161](https://github.com/ClickHouse/ClickHouse/pull/36161) ([alesapin](https://github.com/alesapin)).
 * Some fixes for ReplicatedMergeTree [#36163](https://github.com/ClickHouse/ClickHouse/pull/36163) ([Alexander Tokmakov](https://github.com/tavplubix)).
 * clickhouse-client: properly cancel query in case of error during formatting data [#36164](https://github.com/ClickHouse/ClickHouse/pull/36164) ([Azat Khuzhin](https://github.com/azat)).
-* Fix flacky test 01161_all_system_tables under s3 storage [#36175](https://github.com/ClickHouse/ClickHouse/pull/36175) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Fix flaky test 01161_all_system_tables under s3 storage [#36175](https://github.com/ClickHouse/ClickHouse/pull/36175) ([Kseniia Sumarokova](https://github.com/kssenii)).
 * Revert "Fix possible mutation stuck due to race with DROP_RANGE" [#36190](https://github.com/ClickHouse/ClickHouse/pull/36190) ([Azat Khuzhin](https://github.com/azat)).
 * Use atomic instead of mutex + condvar in ParallelReadBuffer [#36192](https://github.com/ClickHouse/ClickHouse/pull/36192) ([Kruglov Pavel](https://github.com/Avogar)).
 * Follow-up to [#36138](https://github.com/ClickHouse/ClickHouse/issues/36138) [#36194](https://github.com/ClickHouse/ClickHouse/pull/36194) ([Alexander Tokmakov](https://github.com/tavplubix)).
diff --git a/docs/changelogs/v23.4.1.1943-stable.md b/docs/changelogs/v23.4.1.1943-stable.md
index ea16f5856be..34590ba9d37 100644
--- a/docs/changelogs/v23.4.1.1943-stable.md
+++ b/docs/changelogs/v23.4.1.1943-stable.md
@@ -321,7 +321,7 @@ sidebar_label: 2023
 * Add a test for [#38128](https://github.com/ClickHouse/ClickHouse/issues/38128) [#48817](https://github.com/ClickHouse/ClickHouse/pull/48817) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
 * Remove excessive logging [#48826](https://github.com/ClickHouse/ClickHouse/pull/48826) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
 * remove duplicate indentwith in clang-format [#48834](https://github.com/ClickHouse/ClickHouse/pull/48834) ([cluster](https://github.com/infdahai)).
-* Try fix flacky test_concurrent_alter_move_and_drop [#48843](https://github.com/ClickHouse/ClickHouse/pull/48843) ([Sergei Trifonov](https://github.com/serxa)).
+* Try fix flaky test_concurrent_alter_move_and_drop [#48843](https://github.com/ClickHouse/ClickHouse/pull/48843) ([Sergei Trifonov](https://github.com/serxa)).
 * fix the race wait loading parts [#48844](https://github.com/ClickHouse/ClickHouse/pull/48844) ([Sema Checherinda](https://github.com/CheSema)).
 * suppress assert of progress for test_system_replicated_fetches [#48856](https://github.com/ClickHouse/ClickHouse/pull/48856) ([Han Fei](https://github.com/hanfei1991)).
 * Fix: do not run test_store_cleanup_disk_s3 in parallel [#48863](https://github.com/ClickHouse/ClickHouse/pull/48863) ([Igor Nikonov](https://github.com/devcrafter)).
@@ -372,4 +372,3 @@ sidebar_label: 2023
 * suppress two timeout tests [#49175](https://github.com/ClickHouse/ClickHouse/pull/49175) ([Han Fei](https://github.com/hanfei1991)).
 * Document makeDateTime() and its variants [#49183](https://github.com/ClickHouse/ClickHouse/pull/49183) ([Robert Schulze](https://github.com/rschu1ze)).
 * Fix after [#49110](https://github.com/ClickHouse/ClickHouse/issues/49110) [#49206](https://github.com/ClickHouse/ClickHouse/pull/49206) ([Kseniia Sumarokova](https://github.com/kssenii)).
-
diff --git a/docs/changelogs/v23.6.1.1524-stable.md b/docs/changelogs/v23.6.1.1524-stable.md
index b91c5340789..0de9ab37653 100644
--- a/docs/changelogs/v23.6.1.1524-stable.md
+++ b/docs/changelogs/v23.6.1.1524-stable.md
@@ -263,7 +263,7 @@ sidebar_label: 2023
 * Fix broken labeling for `manual approve` [#51405](https://github.com/ClickHouse/ClickHouse/pull/51405) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
 * Fix parts lifetime in `MergeTreeTransaction` [#51407](https://github.com/ClickHouse/ClickHouse/pull/51407) ([Alexander Tokmakov](https://github.com/tavplubix)).
 * Fix flaky test test_skip_empty_files [#51409](https://github.com/ClickHouse/ClickHouse/pull/51409) ([Kruglov Pavel](https://github.com/Avogar)).
-* fix flacky test test_profile_events_s3 [#51412](https://github.com/ClickHouse/ClickHouse/pull/51412) ([Sema Checherinda](https://github.com/CheSema)).
+* fix flaky test test_profile_events_s3 [#51412](https://github.com/ClickHouse/ClickHouse/pull/51412) ([Sema Checherinda](https://github.com/CheSema)).
 * Update README.md [#51413](https://github.com/ClickHouse/ClickHouse/pull/51413) ([Tyler Hannan](https://github.com/tylerhannan)).
 * Replace try/catch logic in hasTokenOrNull() by something more lightweight [#51425](https://github.com/ClickHouse/ClickHouse/pull/51425) ([Robert Schulze](https://github.com/rschu1ze)).
 * Add retries to `tlsv1_3` tests [#51434](https://github.com/ClickHouse/ClickHouse/pull/51434) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
diff --git a/docs/zh/changelog/index.md b/docs/zh/changelog/index.md
index cd77a8c03cf..fb50dfcee85 100644
--- a/docs/zh/changelog/index.md
+++ b/docs/zh/changelog/index.md
@@ -252,7 +252,7 @@ sidebar_label: "\u53D8\u66F4\u65E5\u5FD7"
 -   抑制MSan下的一些测试失败。 [#8780](https://github.com/ClickHouse/ClickHouse/pull/8780) ([Alexander Kuzmenkov](https://github.com/akuzm))
 -   加速 “exception while insert” 测试 此测试通常在具有复盖率的调试版本中超时。 [#8711](https://github.com/ClickHouse/ClickHouse/pull/8711) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov))
 -   更新 `libcxx` 和 `libcxxabi` 为了主人 在准备 [#9304](https://github.com/ClickHouse/ClickHouse/issues/9304) [#9308](https://github.com/ClickHouse/ClickHouse/pull/9308) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov))
--   修复flacky测试 `00910_zookeeper_test_alter_compression_codecs`. [#9525](https://github.com/ClickHouse/ClickHouse/pull/9525) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov))
+-   修复flaky测试 `00910_zookeeper_test_alter_compression_codecs`. [#9525](https://github.com/ClickHouse/ClickHouse/pull/9525) ([阿列克谢-米洛维多夫](https://github.com/alexey-milovidov))
 -   清理重复的链接器标志。 确保链接器不会查找意想不到的符号。 [#9433](https://github.com/ClickHouse/ClickHouse/pull/9433) ([阿莫斯鸟](https://github.com/amosbird))
 -   添加 `clickhouse-odbc` 驱动程序进入测试图像。 这允许通过自己的ODBC驱动程序测试ClickHouse与ClickHouse的交互。 [#9348](https://github.com/ClickHouse/ClickHouse/pull/9348) ([filimonov](https://github.com/filimonov))
 -   修复单元测试中的几个错误。 [#9047](https://github.com/ClickHouse/ClickHouse/pull/9047) ([阿利沙平](https://github.com/alesapin))
diff --git a/src/Common/SystemLogBase.cpp b/src/Common/SystemLogBase.cpp
index a9307c3be99..7d2c15714e2 100644
--- a/src/Common/SystemLogBase.cpp
+++ b/src/Common/SystemLogBase.cpp
@@ -65,7 +65,7 @@ void SystemLogQueue<LogElement>::push(LogElement&& element)
     /// Memory can be allocated while resizing on queue.push_back.
     /// The size of allocation can be in order of a few megabytes.
     /// But this should not be accounted for query memory usage.
-    /// Otherwise the tests like 01017_uniqCombined_memory_usage.sql will be flacky.
+    /// Otherwise the tests like 01017_uniqCombined_memory_usage.sql will be flaky.
     MemoryTrackerBlockerInThread temporarily_disable_memory_tracker;
 
     /// Should not log messages under mutex.

From bd5022432909bcb633f31617a764ca223ad585d5 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 4 Aug 2024 20:57:22 +0200
Subject: [PATCH 1341/1488] Whitespaces

---
 docs/changelogs/v23.4.1.1943-stable.md            | 1 -
 src/Processors/Chunk.h                            | 2 +-
 tests/queries/0_stateless/02099_tsv_raw_format.sh | 7 +++----
 3 files changed, 4 insertions(+), 6 deletions(-)

diff --git a/docs/changelogs/v23.4.1.1943-stable.md b/docs/changelogs/v23.4.1.1943-stable.md
index ea16f5856be..6ab6d8b457a 100644
--- a/docs/changelogs/v23.4.1.1943-stable.md
+++ b/docs/changelogs/v23.4.1.1943-stable.md
@@ -372,4 +372,3 @@ sidebar_label: 2023
 * suppress two timeout tests [#49175](https://github.com/ClickHouse/ClickHouse/pull/49175) ([Han Fei](https://github.com/hanfei1991)).
 * Document makeDateTime() and its variants [#49183](https://github.com/ClickHouse/ClickHouse/pull/49183) ([Robert Schulze](https://github.com/rschu1ze)).
 * Fix after [#49110](https://github.com/ClickHouse/ClickHouse/issues/49110) [#49206](https://github.com/ClickHouse/ClickHouse/pull/49206) ([Kseniia Sumarokova](https://github.com/kssenii)).
-
diff --git a/src/Processors/Chunk.h b/src/Processors/Chunk.h
index 1348966c0d3..f45e2c4619e 100644
--- a/src/Processors/Chunk.h
+++ b/src/Processors/Chunk.h
@@ -22,7 +22,7 @@ public:
 };
 
 
-template<class Derived>
+template <typename Derived>
 class ChunkInfoCloneable : public ChunkInfo
 {
 public:
diff --git a/tests/queries/0_stateless/02099_tsv_raw_format.sh b/tests/queries/0_stateless/02099_tsv_raw_format.sh
index 16b695e4037..026607ac6d5 100755
--- a/tests/queries/0_stateless/02099_tsv_raw_format.sh
+++ b/tests/queries/0_stateless/02099_tsv_raw_format.sh
@@ -14,7 +14,7 @@ do
     echo $format
     $CLICKHOUSE_CLIENT -q "INSERT INTO test_02099 SELECT number, toString(number), toDate(number) FROM numbers(3)"
     $CLICKHOUSE_CLIENT -q "SELECT * FROM test_02099 FORMAT $format"
-        
+
     $CLICKHOUSE_CLIENT -q "SELECT * FROM test_02099 FORMAT $format" | $CLICKHOUSE_CLIENT -q "INSERT INTO test_02099 FORMAT $format"
     $CLICKHOUSE_CLIENT -q "SELECT * FROM test_02099"
 
@@ -49,13 +49,12 @@ $CLICKHOUSE_CLIENT -q "DROP TABLE test_nullable_string_02099"
 
 $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS test_parallel_parsing_02099"
 $CLICKHOUSE_CLIENT -q "CREATE TABLE test_parallel_parsing_02099 (x UInt64, a Array(UInt64), s String) ENGINE=Memory()";
-$CLICKHOUSE_CLIENT -q "SELECT number AS x, range(number % 50) AS a, toString(a) AS s FROM numbers(1000000) FORMAT TSVRaw" |  $CLICKHOUSE_CLIENT --input_format_parallel_parsing=0 -q "INSERT INTO test_parallel_parsing_02099 FORMAT TSVRaw"
+$CLICKHOUSE_CLIENT -q "SELECT number AS x, range(number % 50) AS a, toString(a) AS s FROM numbers(1000000) FORMAT TSVRaw" | $CLICKHOUSE_CLIENT --input_format_parallel_parsing=0 -q "INSERT INTO test_parallel_parsing_02099 FORMAT TSVRaw"
 $CLICKHOUSE_CLIENT -q "SELECT * FROM test_parallel_parsing_02099 ORDER BY x" | md5sum
 
 $CLICKHOUSE_CLIENT -q "TRUNCATE TABLE test_parallel_parsing_02099"
 
-$CLICKHOUSE_CLIENT -q "SELECT number AS x, range(number % 50) AS a, toString(a) AS s FROM numbers(1000000) FORMAT TSVRaw" |  $CLICKHOUSE_CLIENT --input_format_parallel_parsing=1 -q "INSERT INTO test_parallel_parsing_02099 FORMAT TSVRaw"
+$CLICKHOUSE_CLIENT -q "SELECT number AS x, range(number % 50) AS a, toString(a) AS s FROM numbers(1000000) FORMAT TSVRaw" | $CLICKHOUSE_CLIENT --input_format_parallel_parsing=1 -q "INSERT INTO test_parallel_parsing_02099 FORMAT TSVRaw"
 $CLICKHOUSE_CLIENT -q "SELECT * FROM test_parallel_parsing_02099 ORDER BY x" | md5sum
 
 $CLICKHOUSE_CLIENT -q "DROP TABLE test_parallel_parsing_02099"
-

From a49924b3834d577827279f6278e5df21e3065035 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Sun, 4 Aug 2024 13:27:06 +0200
Subject: [PATCH 1342/1488] tests: avoid leaving leftovers after
 test_storage_mongodb (fixes flaky chec)

CI: https://s3.amazonaws.com/clickhouse-test-reports/66671/2f00c962711e13ca00af324366421fe4593b4ce6/integration_tests_flaky_check__asan_.html
Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 tests/integration/test_storage_mongodb/test.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tests/integration/test_storage_mongodb/test.py b/tests/integration/test_storage_mongodb/test.py
index 1a1a790e8e8..6e2d15b03be 100644
--- a/tests/integration/test_storage_mongodb/test.py
+++ b/tests/integration/test_storage_mongodb/test.py
@@ -410,6 +410,7 @@ def test_no_credentials(started_cluster):
     simple_mongo_table.insert_many(data)
 
     node = started_cluster.instances["node"]
+    node.query("drop table if exists simple_mongo_table_2")
     node.query(
         "create table simple_mongo_table_2(key UInt64, data String) engine = MongoDB('mongo2:27017', 'test', 'simple_table', '', '')"
     )
@@ -439,10 +440,13 @@ def test_auth_source(started_cluster):
     simple_mongo_table.insert_many(data)
 
     node = started_cluster.instances["node"]
+    node.query("drop table if exists simple_mongo_table_fail")
     node.query(
         "create table simple_mongo_table_fail(key UInt64, data String) engine = MongoDB('mongo2:27017', 'test', 'simple_table', 'root', 'clickhouse')"
     )
     node.query_and_get_error("SELECT count() FROM simple_mongo_table_fail")
+
+    node.query("drop table if exists simple_mongo_table_ok")
     node.query(
         "create table simple_mongo_table_ok(key UInt64, data String) engine = MongoDB('mongo2:27017', 'test', 'simple_table', 'root', 'clickhouse', 'authSource=admin')"
     )

From c200f437746ddbd03e5ce7c7ad9d9613015ce929 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Sun, 4 Aug 2024 16:00:46 +0200
Subject: [PATCH 1343/1488] ci: fix stateless runner to correctly gather
 artifacts when server failed

Right now it simply does not work due to "set -e", with it you cannot
use "foo=$(false)" since bash will break execution after, rewrite it to
a plain shell, with "if".

Also use ZSTD everywhere (ugh)

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 docker/test/stateless/run.sh | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh
index da17b82d91b..5c15c05652b 100755
--- a/docker/test/stateless/run.sh
+++ b/docker/test/stateless/run.sh
@@ -299,22 +299,22 @@ stop_logs_replication
 failed_to_save_logs=0
 for table in query_log zookeeper_log trace_log transactions_info_log metric_log blob_storage_log error_log
 do
-    err=$(clickhouse-client -q "select * from system.$table into outfile '/test_output/$table.tsv.gz' format TSVWithNamesAndTypes")
-    echo "$err"
-    [[ "0" != "${#err}"  ]] && failed_to_save_logs=1
+    if ! clickhouse-client -q "select * from system.$table into outfile '/test_output/$table.tsv.zst' format TSVWithNamesAndTypes"; then
+        failed_to_save_logs=1
+    fi
     if [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
-        err=$( { clickhouse-client --port 19000 -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.1.tsv.zst; } 2>&1 )
-        echo "$err"
-        [[ "0" != "${#err}"  ]] && failed_to_save_logs=1
-        err=$( { clickhouse-client --port 29000 -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.2.tsv.zst; } 2>&1 )
-        echo "$err"
-        [[ "0" != "${#err}"  ]] && failed_to_save_logs=1
+        if ! clickhouse-client --port 19000 -q "select * from system.$table into outfile '/test_output/$table.1.tsv.zst' format TSVWithNamesAndTypes"; then
+            failed_to_save_logs=1
+        fi
+        if ! clickhouse-client --port 29000 -q "select * from system.$table into outfile '/test_output/$table.2.tsv.zst' format TSVWithNamesAndTypes"; then
+            failed_to_save_logs=1
+        fi
     fi
 
     if [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then
-        err=$( { clickhouse-client --port 19000 -q "select * from system.$table format TSVWithNamesAndTypes" | zstd --threads=0 > /test_output/$table.1.tsv.zst; } 2>&1 )
-        echo "$err"
-        [[ "0" != "${#err}"  ]] && failed_to_save_logs=1
+        if ! clickhouse-client --port 29000 -q "select * from system.$table into outfile '/test_output/$table.2.tsv.zst' format TSVWithNamesAndTypes"; then
+            failed_to_save_logs=1
+        fi
     fi
 done
 

From 43cf85ef2841190a5d12a592a65a5181bb110661 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Sun, 4 Aug 2024 16:09:48 +0200
Subject: [PATCH 1344/1488] ci: collect basic issues in stateless tests (dmesg,
 fatal and similar)

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 docker/test/stateless/run.sh           | 9 +++++++++
 docker/test/stateless/stress_tests.lib | 2 +-
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh
index 5c15c05652b..b352539cc1a 100755
--- a/docker/test/stateless/run.sh
+++ b/docker/test/stateless/run.sh
@@ -3,6 +3,12 @@
 # shellcheck disable=SC1091
 source /setup_export_logs.sh
 
+# shellcheck source=../stateless/stress_tests.lib
+source /stress_tests.lib
+
+# Avoid overlaps with previous runs
+dmesg --clear
+
 # fail on errors, verbose and export all env variables
 set -e -x -a
 
@@ -420,4 +426,7 @@ if [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then
     tar -chf /test_output/coordination1.tar /var/lib/clickhouse1/coordination ||:
 fi
 
+# Grep logs for sanitizer asserts, crashes and other critical errors
+check_logs_for_critical_errors
+
 collect_core_dumps
diff --git a/docker/test/stateless/stress_tests.lib b/docker/test/stateless/stress_tests.lib
index 682da1df837..36782101fa7 100644
--- a/docker/test/stateless/stress_tests.lib
+++ b/docker/test/stateless/stress_tests.lib
@@ -242,7 +242,7 @@ function check_server_start()
 function check_logs_for_critical_errors()
 {
     # Sanitizer asserts
-    sed -n '/WARNING:.*anitizer/,/^$/p' /var/log/clickhouse-server/stderr.log >> /test_output/tmp
+    sed -n '/WARNING:.*anitizer/,/^$/p' /var/log/clickhouse-server/stderr*.log >> /test_output/tmp
     rg -Fav -e "ASan doesn't fully support makecontext/swapcontext functions" -e "DB::Exception" /test_output/tmp > /dev/null \
         && echo -e "Sanitizer assert (in stderr.log)$FAIL$(head_escaped /test_output/tmp)" >> /test_output/test_results.tsv \
         || echo -e "No sanitizer asserts$OK" >> /test_output/test_results.tsv

From 83be27cdf59ec5886abc23441f5cf92740b04c57 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 4 Aug 2024 23:34:01 +0200
Subject: [PATCH 1345/1488] Fix test `02833_concurrent_sessions`, Fix test
 `02835_drop_user_during_session`

---
 .../0_stateless/02833_concurrent_sessions.sh  | 18 ++++++++++++-
 .../02835_drop_user_during_session.sh         | 26 +++++++++++++++----
 2 files changed, 38 insertions(+), 6 deletions(-)

diff --git a/tests/queries/0_stateless/02833_concurrent_sessions.sh b/tests/queries/0_stateless/02833_concurrent_sessions.sh
index 846661cfeed..feaff3a38a3 100755
--- a/tests/queries/0_stateless/02833_concurrent_sessions.sh
+++ b/tests/queries/0_stateless/02833_concurrent_sessions.sh
@@ -137,7 +137,23 @@ ${CLICKHOUSE_CLIENT} -q "SELECT count(*) FROM system.session_log WHERE user IN (
 for user in "${ALL_USERS[@]}"; do
     ${CLICKHOUSE_CLIENT} -q "DROP USER ${user}"
     echo "Corresponding LoginSuccess/Logout"
-    ${CLICKHOUSE_CLIENT} -q "SELECT COUNT(*) FROM (SELECT ${SESSION_LOG_MATCHING_FIELDS} FROM system.session_log WHERE user = '${user}' AND type = 'LoginSuccess' INTERSECT SELECT ${SESSION_LOG_MATCHING_FIELDS} FROM system.session_log WHERE user = '${user}' AND type = 'Logout')"
+
+    # The client can exit sooner than the server records its disconnection and closes the session.
+    # When the client disconnects, two processes happen at the same time and are in the race condition:
+    # - the client application exits and returns control to the shell;
+    # - the server closes the session and records the logout event to the session log.
+    # We cannot expect that after the control is returned to the shell, the server records the logout event.
+    while true
+    do
+        [[ 3 -eq $(${CLICKHOUSE_CLIENT} -q "
+            SELECT COUNT(*) FROM (
+                SELECT ${SESSION_LOG_MATCHING_FIELDS} FROM system.session_log WHERE user = '${user}' AND type = 'LoginSuccess'
+                INTERSECT
+                SELECT ${SESSION_LOG_MATCHING_FIELDS} FROM system.session_log WHERE user = '${user}' AND type = 'Logout'
+            )") ]] && echo 3 && break;
+        sleep 0.1
+    done
+
     echo "LoginFailure"
     ${CLICKHOUSE_CLIENT} -q "SELECT COUNT(*) FROM system.session_log WHERE user = '${user}' AND type = 'LoginFailure'"
  done
diff --git a/tests/queries/0_stateless/02835_drop_user_during_session.sh b/tests/queries/0_stateless/02835_drop_user_during_session.sh
index 347ebd22f96..c32003a2a11 100755
--- a/tests/queries/0_stateless/02835_drop_user_during_session.sh
+++ b/tests/queries/0_stateless/02835_drop_user_during_session.sh
@@ -24,7 +24,7 @@ function http_session()
     ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&user=${user}&password=pass" -d "SELECT COUNT(*) FROM system.numbers"
 }
 
-function http_with_session_id_session() 
+function http_with_session_id_session()
 {
     local user=$1
     ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&user=${user}&password=pass" -d "SELECT COUNT(*) FROM system.numbers"
@@ -104,11 +104,27 @@ wait
 
 ${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH LOGS"
 
-echo "port_0_sessions:" 
+echo "port_0_sessions:"
 ${CLICKHOUSE_CLIENT} -q "SELECT count(*) FROM system.session_log WHERE user = '${TEST_USER}' AND client_port = 0"
 echo "address_0_sessions:"
 ${CLICKHOUSE_CLIENT} -q "SELECT count(*) FROM system.session_log WHERE user = '${TEST_USER}' AND client_address = toIPv6('::')"
-echo "Corresponding LoginSuccess/Logout" 
-${CLICKHOUSE_CLIENT} -q "SELECT COUNT(*) FROM (SELECT ${SESSION_LOG_MATCHING_FIELDS} FROM system.session_log WHERE user = '${TEST_USER}' AND type = 'LoginSuccess' INTERSECT SELECT ${SESSION_LOG_MATCHING_FIELDS}, FROM system.session_log WHERE user = '${TEST_USER}' AND type = 'Logout')"
+echo "Corresponding LoginSuccess/Logout"
+
+# The client can exit sooner than the server records its disconnection and closes the session.
+# When the client disconnects, two processes happen at the same time and are in the race condition:
+# - the client application exits and returns control to the shell;
+# - the server closes the session and records the logout event to the session log.
+# We cannot expect that after the control is returned to the shell, the server records the logout event.
+while true
+do
+    [[ 9 -eq $(${CLICKHOUSE_CLIENT} -q "
+        SELECT COUNT(*) FROM (
+            SELECT ${SESSION_LOG_MATCHING_FIELDS} FROM system.session_log WHERE user = '${TEST_USER}' AND type = 'LoginSuccess'
+            INTERSECT
+            SELECT ${SESSION_LOG_MATCHING_FIELDS}, FROM system.session_log WHERE user = '${TEST_USER}' AND type = 'Logout'
+        )") ]] && echo 9 && break;
+    sleep 0.1
+done
+
 echo "LoginFailure"
-${CLICKHOUSE_CLIENT} -q "SELECT COUNT(*) FROM system.session_log WHERE user = '${TEST_USER}' AND type = 'LoginFailure'" 
+${CLICKHOUSE_CLIENT} -q "SELECT COUNT(*) FROM system.session_log WHERE user = '${TEST_USER}' AND type = 'LoginFailure'"

From 0bdaa57023ef69e49ab5cf9d54ed2e52c1fd2dae Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 4 Aug 2024 23:53:54 +0200
Subject: [PATCH 1346/1488] Fix diagnostics in the test script

---
 docker/test/stateless/run.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh
index da17b82d91b..bec8b9cc4d1 100755
--- a/docker/test/stateless/run.sh
+++ b/docker/test/stateless/run.sh
@@ -260,7 +260,7 @@ function run_tests()
     | ts '%Y-%m-%d %H:%M:%S' \
     | tee -a test_output/test_result.txt
     set -e
-    DURATION=$((START_TIME - SECONDS))
+    DURATION=$((SECONDS - START_TIME))
 
     echo "Elapsed ${DURATION} seconds."
     if [[ $DURATION -ge $TIMEOUT ]]

From a573b2926e13ea31c7947b0429b1b5723c7fb938 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sun, 4 Aug 2024 23:59:01 +0200
Subject: [PATCH 1347/1488] Fixes for the script

---
 docker/test/stateless/run.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh
index da17b82d91b..8b9e729970c 100755
--- a/docker/test/stateless/run.sh
+++ b/docker/test/stateless/run.sh
@@ -72,8 +72,12 @@ if [[ -n "$BUGFIX_VALIDATE_CHECK" ]] && [[ "$BUGFIX_VALIDATE_CHECK" -eq 1 ]]; th
     remove_keeper_config "latest_logs_cache_size_threshold" "[[:digit:]]\+"
 fi
 
+export IS_FLAKY_CHECK=0
+
 # For flaky check we also enable thread fuzzer
 if [ "$NUM_TRIES" -gt "1" ]; then
+    export IS_FLAKY_CHECK=1
+
     export THREAD_FUZZER_CPU_TIME_PERIOD_US=1000
     export THREAD_FUZZER_SLEEP_PROBABILITY=0.1
     export THREAD_FUZZER_SLEEP_TIME_US_MAX=100000

From 538761b43dbf704d0700548e61b4034ef66c5766 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 5 Aug 2024 00:02:32 +0200
Subject: [PATCH 1348/1488] Fix flaky check

---
 tests/clickhouse-test | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/clickhouse-test b/tests/clickhouse-test
index b70dd61a25a..38b0e99760e 100755
--- a/tests/clickhouse-test
+++ b/tests/clickhouse-test
@@ -1232,7 +1232,7 @@ class TestCase:
         ):
             return FailureReason.SKIP
 
-        elif "no-flaky-check" in tags and (args.test_runs > 1):
+        elif "no-flaky-check" in tags and (1 == int(os.environ.get("IS_FLAKY_CHECK", 0))):
             return FailureReason.SKIP
 
         elif tags:

From 270dddc00fd5533ba24914e7d9ae0aebc50a3fd0 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 5 Aug 2024 00:22:48 +0200
Subject: [PATCH 1349/1488] Fix test `02231_bloom_filter_sizing`

---
 tests/queries/0_stateless/02231_bloom_filter_sizing.reference | 4 ++--
 tests/queries/0_stateless/02231_bloom_filter_sizing.sql       | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/queries/0_stateless/02231_bloom_filter_sizing.reference b/tests/queries/0_stateless/02231_bloom_filter_sizing.reference
index bdba311c092..aabadfc92fd 100644
--- a/tests/queries/0_stateless/02231_bloom_filter_sizing.reference
+++ b/tests/queries/0_stateless/02231_bloom_filter_sizing.reference
@@ -1,6 +1,6 @@
 Bloom filter on sort key
-10000
+1000
 0
 Bloom filter on non-sort key
-10000
+1000
 0
diff --git a/tests/queries/0_stateless/02231_bloom_filter_sizing.sql b/tests/queries/0_stateless/02231_bloom_filter_sizing.sql
index 233e3111067..ee896675d64 100644
--- a/tests/queries/0_stateless/02231_bloom_filter_sizing.sql
+++ b/tests/queries/0_stateless/02231_bloom_filter_sizing.sql
@@ -12,7 +12,7 @@ INSERT INTO bloom_filter_sizing_pk
 SELECT
 number % 100 as key, -- 100 unique keys
 number as value -- whatever
-FROM numbers(1000 * 1000);
+FROM numbers(100_000);
 
 --
 -- Merge everything into a single part
@@ -40,7 +40,7 @@ SELECT
 number % 100 as key1, -- 100 unique keys
 rand() % 100 as key2, -- 100 unique keys
 number as value -- whatever
-FROM numbers(1000 * 1000);
+FROM numbers(100_000);
 
 --
 -- Merge everything into a single part

From 5a860fcc3aaf4e77025cd9286da10d0587134c3f Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 5 Aug 2024 00:25:56 +0200
Subject: [PATCH 1350/1488] Update 02099_tsv_raw_format.sh

---
 tests/queries/0_stateless/02099_tsv_raw_format.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/queries/0_stateless/02099_tsv_raw_format.sh b/tests/queries/0_stateless/02099_tsv_raw_format.sh
index 026607ac6d5..a69c96ab613 100755
--- a/tests/queries/0_stateless/02099_tsv_raw_format.sh
+++ b/tests/queries/0_stateless/02099_tsv_raw_format.sh
@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
+# Tags: long
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh

From 7adeaf9c28016b29453d6be2bca1aa1ef866ecdf Mon Sep 17 00:00:00 2001
From: robot-clickhouse <robot-clickhouse@users.noreply.github.com>
Date: Sun, 4 Aug 2024 22:35:07 +0000
Subject: [PATCH 1351/1488] Automatic style fix

---
 tests/clickhouse-test | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tests/clickhouse-test b/tests/clickhouse-test
index 38b0e99760e..907d773337a 100755
--- a/tests/clickhouse-test
+++ b/tests/clickhouse-test
@@ -1232,7 +1232,9 @@ class TestCase:
         ):
             return FailureReason.SKIP
 
-        elif "no-flaky-check" in tags and (1 == int(os.environ.get("IS_FLAKY_CHECK", 0))):
+        elif "no-flaky-check" in tags and (
+            1 == int(os.environ.get("IS_FLAKY_CHECK", 0))
+        ):
             return FailureReason.SKIP
 
         elif tags:

From 6b1e184e12bed759487d89f54f5ac4f269dffda2 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Sun, 4 Aug 2024 22:50:46 +0200
Subject: [PATCH 1352/1488] Print original query for AST formatting check on CI

It may be tricky to understand the root cause of the AST formatting
issue in case of syntax error, so add one knob to control this -
debug_ast_formatting_print_original_query.

And CI contains core dumps anyway, so let's enable for CI.

P.S. There was concern from @al13n321 that printing original query even
in debug build is not a good idea [1], hence a knob for this.

  [1]: https://github.com/ClickHouse/ClickHouse/pull/63357/files#r1674809348

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 src/Interpreters/executeQuery.cpp | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp
index ba6fc0f14a0..ce58f7f922c 100644
--- a/src/Interpreters/executeQuery.cpp
+++ b/src/Interpreters/executeQuery.cpp
@@ -796,10 +796,9 @@ static std::tuple<ASTPtr, BlockIO> executeQueryImpl(
             catch (const Exception & e)
             {
                 if (e.code() == ErrorCodes::SYNTAX_ERROR)
-                    /// Don't print the original query text because it may contain sensitive data.
                     throw Exception(ErrorCodes::LOGICAL_ERROR,
-                        "Inconsistent AST formatting: the query:\n{}\ncannot parse.",
-                        formatted1);
+                        "Inconsistent AST formatting: the query:\n{}\ncannot parse query back from {}",
+                        formatted1, std::string_view(begin, end-begin));
                 else
                     throw;
             }

From 8bca80f4dd08d8ad05db0325b96365d82e6c4076 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Sun, 4 Aug 2024 22:35:21 +0200
Subject: [PATCH 1353/1488] Fix REPLACE modifier formatting (forbid omitting
 brackets)

It is too tricky to verify does brackets required or not, i.e. "SELECT *
REPLACE(1/3/3 AS dummy)" will be formatted to "SELECT * REPLACE (1/3)/3 AS dummy"
which is already invalid query.

So let's simply always print them.

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
v2: move the fix into correct place ASTColumnsReplaceTransformer::formatImpl() instead of ASTColumnsReplaceTransformer::Replacement::formatImpl()
---
 src/Parsers/ASTColumnsTransformers.cpp           |  8 ++------
 .../03220_replace_formatting.reference           | 16 ++++++++++++++++
 .../0_stateless/03220_replace_formatting.sh      | 14 ++++++++++++++
 3 files changed, 32 insertions(+), 6 deletions(-)
 create mode 100644 tests/queries/0_stateless/03220_replace_formatting.reference
 create mode 100755 tests/queries/0_stateless/03220_replace_formatting.sh

diff --git a/src/Parsers/ASTColumnsTransformers.cpp b/src/Parsers/ASTColumnsTransformers.cpp
index 2a61892f8cc..332ebca3bdb 100644
--- a/src/Parsers/ASTColumnsTransformers.cpp
+++ b/src/Parsers/ASTColumnsTransformers.cpp
@@ -323,9 +323,7 @@ void ASTColumnsReplaceTransformer::formatImpl(const FormatSettings & settings, F
 {
     settings.ostr << (settings.hilite ? hilite_keyword : "") << "REPLACE" << (is_strict ? " STRICT " : " ") << (settings.hilite ? hilite_none : "");
 
-    if (children.size() > 1)
-        settings.ostr << "(";
-
+    settings.ostr << "(";
     for (ASTs::const_iterator it = children.begin(); it != children.end(); ++it)
     {
         if (it != children.begin())
@@ -333,9 +331,7 @@ void ASTColumnsReplaceTransformer::formatImpl(const FormatSettings & settings, F
 
         (*it)->formatImpl(settings, state, frame);
     }
-
-    if (children.size() > 1)
-        settings.ostr << ")";
+    settings.ostr << ")";
 }
 
 void ASTColumnsReplaceTransformer::appendColumnName(WriteBuffer & ostr) const
diff --git a/tests/queries/0_stateless/03220_replace_formatting.reference b/tests/queries/0_stateless/03220_replace_formatting.reference
new file mode 100644
index 00000000000..cbcd63839b1
--- /dev/null
+++ b/tests/queries/0_stateless/03220_replace_formatting.reference
@@ -0,0 +1,16 @@
+SELECT * REPLACE ((1 / 3) / 3 AS dummy)
+SELECT * REPLACE ((1 / 3) / 3 AS dummy)
+SELECT * REPLACE STRICT (1 AS id, 2 AS value)
+FROM
+(
+    SELECT
+        0 AS id,
+        1 AS value
+)
+SELECT * REPLACE STRICT (1 AS id, 2 AS value)
+FROM
+(
+    SELECT
+        0 AS id,
+        1 AS value
+)
diff --git a/tests/queries/0_stateless/03220_replace_formatting.sh b/tests/queries/0_stateless/03220_replace_formatting.sh
new file mode 100755
index 00000000000..1c11ed6da8d
--- /dev/null
+++ b/tests/queries/0_stateless/03220_replace_formatting.sh
@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CUR_DIR"/../shell_config.sh
+
+q=$($CLICKHOUSE_FORMAT <<<"SELECT * REPLACE(1/3/3 AS dummy)")
+echo "$q"
+$CLICKHOUSE_FORMAT <<<"$q"
+
+# multiple columns
+q=$($CLICKHOUSE_FORMAT <<<"SELECT * REPLACE STRICT (1 AS id, 2 AS value) FROM (SELECT 0 id, 1 value)")
+echo "$q"
+$CLICKHOUSE_FORMAT <<<"$q"

From 2a7ad3a1f979708fd152e364296c505db8926aba Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Sun, 4 Aug 2024 23:21:19 +0200
Subject: [PATCH 1354/1488] Update test references for new REPLACE modifier
 syntax

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 .../01913_fix_column_transformer_replace_format.reference       | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/01913_fix_column_transformer_replace_format.reference b/tests/queries/0_stateless/01913_fix_column_transformer_replace_format.reference
index 33be11c07d5..6fabd33c804 100644
--- a/tests/queries/0_stateless/01913_fix_column_transformer_replace_format.reference
+++ b/tests/queries/0_stateless/01913_fix_column_transformer_replace_format.reference
@@ -1 +1 @@
-CREATE VIEW default.my_view\n(\n    `Id` UInt32,\n    `Object.Key` Array(UInt16),\n    `Object.Value` Array(String)\n)\nAS SELECT * REPLACE arrayMap(x -> (x + 1), `Object.Key`) AS `Object.Key`\nFROM default.my_table
+CREATE VIEW default.my_view\n(\n    `Id` UInt32,\n    `Object.Key` Array(UInt16),\n    `Object.Value` Array(String)\n)\nAS SELECT * REPLACE (arrayMap(x -> (x + 1), `Object.Key`) AS `Object.Key`)\nFROM default.my_table

From cebdc5ecf6b7e41cf3c75fd8ea9765972afa084c Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Wed, 31 Jul 2024 15:19:51 +0000
Subject: [PATCH 1355/1488] Bump rocksdb to v8.0.0

---
 contrib/rocksdb                                 |  2 +-
 contrib/rocksdb-cmake/CMakeLists.txt            |  1 +
 src/Coordination/KeeperContext.cpp              |  7 ++++---
 src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp | 17 ++++++++++-------
 tests/config/config.d/rocksdb.xml               |  3 ++-
 5 files changed, 18 insertions(+), 12 deletions(-)

diff --git a/contrib/rocksdb b/contrib/rocksdb
index 01e43568fa9..fdf403f5918 160000
--- a/contrib/rocksdb
+++ b/contrib/rocksdb
@@ -1 +1 @@
-Subproject commit 01e43568fa9f3f7bf107b2b66c00b286b456f33e
+Subproject commit fdf403f5918a2b4355cf75ebe5e21d0fc22db880
diff --git a/contrib/rocksdb-cmake/CMakeLists.txt b/contrib/rocksdb-cmake/CMakeLists.txt
index 98790158baa..8660bd0e7ba 100644
--- a/contrib/rocksdb-cmake/CMakeLists.txt
+++ b/contrib/rocksdb-cmake/CMakeLists.txt
@@ -309,6 +309,7 @@ set(SOURCES
     ${ROCKSDB_SOURCE_DIR}/util/compression_context_cache.cc
     ${ROCKSDB_SOURCE_DIR}/util/concurrent_task_limiter_impl.cc
     ${ROCKSDB_SOURCE_DIR}/util/crc32c.cc
+    ${ROCKSDB_SOURCE_DIR}/util/data_structure.cc
     ${ROCKSDB_SOURCE_DIR}/util/dynamic_bloom.cc
     ${ROCKSDB_SOURCE_DIR}/util/hash.cc
     ${ROCKSDB_SOURCE_DIR}/util/murmurhash.cc
diff --git a/src/Coordination/KeeperContext.cpp b/src/Coordination/KeeperContext.cpp
index 1f66882ecad..dd2c1d59d56 100644
--- a/src/Coordination/KeeperContext.cpp
+++ b/src/Coordination/KeeperContext.cpp
@@ -23,6 +23,7 @@
 #if USE_ROCKSDB
 #include <rocksdb/table.h>
 #include <rocksdb/convenience.h>
+#include <rocksdb/statistics.h>
 #include <rocksdb/utilities/db_ttl.h>
 #endif
 
@@ -88,7 +89,7 @@ static rocksdb::Options getRocksDBOptionsFromConfig(const Poco::Util::AbstractCo
     if (config.has("keeper_server.rocksdb.options"))
     {
         auto config_options = getOptionsFromConfig(config, "keeper_server.rocksdb.options");
-        status = rocksdb::GetDBOptionsFromMap(merged, config_options, &merged);
+        status = rocksdb::GetDBOptionsFromMap({}, merged, config_options, &merged);
         if (!status.ok())
         {
             throw Exception(ErrorCodes::ROCKSDB_ERROR, "Fail to merge rocksdb options from 'rocksdb.options' : {}",
@@ -98,7 +99,7 @@ static rocksdb::Options getRocksDBOptionsFromConfig(const Poco::Util::AbstractCo
     if (config.has("rocksdb.column_family_options"))
     {
         auto column_family_options = getOptionsFromConfig(config, "rocksdb.column_family_options");
-        status = rocksdb::GetColumnFamilyOptionsFromMap(merged, column_family_options, &merged);
+        status = rocksdb::GetColumnFamilyOptionsFromMap({}, merged, column_family_options, &merged);
         if (!status.ok())
         {
             throw Exception(ErrorCodes::ROCKSDB_ERROR, "Fail to merge rocksdb options from 'rocksdb.column_family_options' at: {}", status.ToString());
@@ -107,7 +108,7 @@ static rocksdb::Options getRocksDBOptionsFromConfig(const Poco::Util::AbstractCo
     if (config.has("rocksdb.block_based_table_options"))
     {
         auto block_based_table_options = getOptionsFromConfig(config, "rocksdb.block_based_table_options");
-        status = rocksdb::GetBlockBasedTableOptionsFromMap(table_options, block_based_table_options, &table_options);
+        status = rocksdb::GetBlockBasedTableOptionsFromMap({}, table_options, block_based_table_options, &table_options);
         if (!status.ok())
         {
             throw Exception(ErrorCodes::ROCKSDB_ERROR, "Fail to merge rocksdb options from 'rocksdb.block_based_table_options' at: {}", status.ToString());
diff --git a/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp b/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp
index fafc72da04e..50f6266cb2f 100644
--- a/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp
+++ b/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp
@@ -36,10 +36,12 @@
 #include <base/sort.h>
 
 #include <rocksdb/advanced_options.h>
+#include <rocksdb/compression_type.h>
+#include <rocksdb/convenience.h>
 #include <rocksdb/env.h>
 #include <rocksdb/options.h>
+#include <rocksdb/statistics.h>
 #include <rocksdb/table.h>
-#include <rocksdb/convenience.h>
 #include <rocksdb/utilities/db_ttl.h>
 
 #include <cstddef>
@@ -428,6 +430,7 @@ void StorageEmbeddedRocksDB::initDB()
     rocksdb::Options base;
 
     base.create_if_missing = true;
+    base.compression = rocksdb::CompressionType::kZSTD;
     base.statistics = rocksdb::CreateDBStatistics();
     /// It is too verbose by default, and in fact we don't care about rocksdb logs at all.
     base.info_log_level = rocksdb::ERROR_LEVEL;
@@ -439,7 +442,7 @@ void StorageEmbeddedRocksDB::initDB()
     if (config.has("rocksdb.options"))
     {
         auto config_options = getOptionsFromConfig(config, "rocksdb.options");
-        status = rocksdb::GetDBOptionsFromMap(merged, config_options, &merged);
+        status = rocksdb::GetDBOptionsFromMap({}, merged, config_options, &merged);
         if (!status.ok())
         {
             throw Exception(ErrorCodes::ROCKSDB_ERROR, "Fail to merge rocksdb options from 'rocksdb.options' at: {}: {}",
@@ -449,7 +452,7 @@ void StorageEmbeddedRocksDB::initDB()
     if (config.has("rocksdb.column_family_options"))
     {
         auto column_family_options = getOptionsFromConfig(config, "rocksdb.column_family_options");
-        status = rocksdb::GetColumnFamilyOptionsFromMap(merged, column_family_options, &merged);
+        status = rocksdb::GetColumnFamilyOptionsFromMap({}, merged, column_family_options, &merged);
         if (!status.ok())
         {
             throw Exception(ErrorCodes::ROCKSDB_ERROR, "Fail to merge rocksdb options from 'rocksdb.column_family_options' at: {}: {}",
@@ -459,7 +462,7 @@ void StorageEmbeddedRocksDB::initDB()
     if (config.has("rocksdb.block_based_table_options"))
     {
         auto block_based_table_options = getOptionsFromConfig(config, "rocksdb.block_based_table_options");
-        status = rocksdb::GetBlockBasedTableOptionsFromMap(table_options, block_based_table_options, &table_options);
+        status = rocksdb::GetBlockBasedTableOptionsFromMap({}, table_options, block_based_table_options, &table_options);
         if (!status.ok())
         {
             throw Exception(ErrorCodes::ROCKSDB_ERROR, "Fail to merge rocksdb options from 'rocksdb.block_based_table_options' at: {}: {}",
@@ -484,7 +487,7 @@ void StorageEmbeddedRocksDB::initDB()
             if (config.has(config_key))
             {
                 auto table_config_options = getOptionsFromConfig(config, config_key);
-                status = rocksdb::GetDBOptionsFromMap(merged, table_config_options, &merged);
+                status = rocksdb::GetDBOptionsFromMap({}, merged, table_config_options, &merged);
                 if (!status.ok())
                 {
                     throw Exception(ErrorCodes::ROCKSDB_ERROR, "Fail to merge rocksdb options from '{}' at: {}: {}",
@@ -496,7 +499,7 @@ void StorageEmbeddedRocksDB::initDB()
             if (config.has(config_key))
             {
                 auto table_column_family_options = getOptionsFromConfig(config, config_key);
-                status = rocksdb::GetColumnFamilyOptionsFromMap(merged, table_column_family_options, &merged);
+                status = rocksdb::GetColumnFamilyOptionsFromMap({}, merged, table_column_family_options, &merged);
                 if (!status.ok())
                 {
                     throw Exception(ErrorCodes::ROCKSDB_ERROR, "Fail to merge rocksdb options from '{}' at: {}: {}",
@@ -508,7 +511,7 @@ void StorageEmbeddedRocksDB::initDB()
             if (config.has(config_key))
             {
                 auto block_based_table_options = getOptionsFromConfig(config, config_key);
-                status = rocksdb::GetBlockBasedTableOptionsFromMap(table_options, block_based_table_options, &table_options);
+                status = rocksdb::GetBlockBasedTableOptionsFromMap({}, table_options, block_based_table_options, &table_options);
                 if (!status.ok())
                 {
                     throw Exception(ErrorCodes::ROCKSDB_ERROR, "Fail to merge rocksdb options from '{}' at: {}: {}",
diff --git a/tests/config/config.d/rocksdb.xml b/tests/config/config.d/rocksdb.xml
index a3790a3dc1d..3002e008a2d 100644
--- a/tests/config/config.d/rocksdb.xml
+++ b/tests/config/config.d/rocksdb.xml
@@ -1,7 +1,8 @@
 <clickhouse>
     <rocksdb>
         <options>
-            <info_log_level>DEBUG_LEVEL</info_log_level>
+            <!-- https://github.com/ClickHouse/ClickHouse/pull/67274#issuecomment-2255301116 -->
+            <info_log_level>ERROR_LEVEL</info_log_level>
         </options>
     </rocksdb>
 </clickhouse>

From d0bc728d52140b60a4c8b9d24d2c4dd4cb9582a8 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Sat, 13 Jul 2024 19:09:45 +0000
Subject: [PATCH 1356/1488] Bump rocksdb to v8.9.1

---
 contrib/rocksdb                               |  2 +-
 contrib/rocksdb-cmake/CMakeLists.txt          | 23 ++++++++-----------
 src/Coordination/tests/gtest_coordination.cpp | 11 +++++----
 3 files changed, 17 insertions(+), 19 deletions(-)

diff --git a/contrib/rocksdb b/contrib/rocksdb
index fdf403f5918..49ce8a1064d 160000
--- a/contrib/rocksdb
+++ b/contrib/rocksdb
@@ -1 +1 @@
-Subproject commit fdf403f5918a2b4355cf75ebe5e21d0fc22db880
+Subproject commit 49ce8a1064dd1ad89117899839bf136365e49e79
diff --git a/contrib/rocksdb-cmake/CMakeLists.txt b/contrib/rocksdb-cmake/CMakeLists.txt
index 8660bd0e7ba..57c056532c6 100644
--- a/contrib/rocksdb-cmake/CMakeLists.txt
+++ b/contrib/rocksdb-cmake/CMakeLists.txt
@@ -1,6 +1,6 @@
 option (ENABLE_ROCKSDB "Enable RocksDB" ${ENABLE_LIBRARIES})
 
-if (NOT ENABLE_ROCKSDB)
+if (NOT ENABLE_ROCKSDB OR NO_SSE3_OR_HIGHER) # assumes SSE4.2 and PCLMUL
   message (STATUS "Not using RocksDB")
   return()
 endif()
@@ -39,13 +39,6 @@ if(WITH_ZSTD)
   list(APPEND THIRDPARTY_LIBS ch_contrib::zstd)
 endif()
 
-add_definitions(-DROCKSDB_PORTABLE)
-
-if(ENABLE_SSE42 AND ENABLE_PCLMULQDQ)
-  add_definitions(-DHAVE_SSE42)
-  add_definitions(-DHAVE_PCLMUL)
-endif()
-
 if(CMAKE_SYSTEM_PROCESSOR MATCHES "arm64|aarch64|AARCH64")
   set (HAS_ARMV8_CRC 1)
   # the original build descriptions set specific flags for ARM. These flags are already subsumed by ClickHouse's general
@@ -91,7 +84,9 @@ set(SOURCES
     ${ROCKSDB_SOURCE_DIR}/cache/compressed_secondary_cache.cc
     ${ROCKSDB_SOURCE_DIR}/cache/lru_cache.cc
     ${ROCKSDB_SOURCE_DIR}/cache/secondary_cache.cc
+    ${ROCKSDB_SOURCE_DIR}/cache/secondary_cache_adapter.cc
     ${ROCKSDB_SOURCE_DIR}/cache/sharded_cache.cc
+    ${ROCKSDB_SOURCE_DIR}/cache/tiered_secondary_cache.cc
     ${ROCKSDB_SOURCE_DIR}/db/arena_wrapped_db_iter.cc
     ${ROCKSDB_SOURCE_DIR}/db/blob/blob_contents.cc
     ${ROCKSDB_SOURCE_DIR}/db/blob/blob_fetcher.cc
@@ -174,9 +169,11 @@ set(SOURCES
     ${ROCKSDB_SOURCE_DIR}/db/wal_manager.cc
     ${ROCKSDB_SOURCE_DIR}/db/wide/wide_column_serialization.cc
     ${ROCKSDB_SOURCE_DIR}/db/wide/wide_columns.cc
+    ${ROCKSDB_SOURCE_DIR}/db/wide/wide_columns_helper.cc
     ${ROCKSDB_SOURCE_DIR}/db/write_batch.cc
     ${ROCKSDB_SOURCE_DIR}/db/write_batch_base.cc
     ${ROCKSDB_SOURCE_DIR}/db/write_controller.cc
+    ${ROCKSDB_SOURCE_DIR}/db/write_stall_stats.cc
     ${ROCKSDB_SOURCE_DIR}/db/write_thread.cc
     ${ROCKSDB_SOURCE_DIR}/env/composite_env.cc
     ${ROCKSDB_SOURCE_DIR}/env/env.cc
@@ -229,6 +226,7 @@ set(SOURCES
     ${ROCKSDB_SOURCE_DIR}/options/configurable.cc
     ${ROCKSDB_SOURCE_DIR}/options/customizable.cc
     ${ROCKSDB_SOURCE_DIR}/options/db_options.cc
+    ${ROCKSDB_SOURCE_DIR}/options/offpeak_time_info.cc
     ${ROCKSDB_SOURCE_DIR}/options/options.cc
     ${ROCKSDB_SOURCE_DIR}/options/options_helper.cc
     ${ROCKSDB_SOURCE_DIR}/options/options_parser.cc
@@ -268,6 +266,7 @@ set(SOURCES
     ${ROCKSDB_SOURCE_DIR}/table/get_context.cc
     ${ROCKSDB_SOURCE_DIR}/table/iterator.cc
     ${ROCKSDB_SOURCE_DIR}/table/merging_iterator.cc
+    ${ROCKSDB_SOURCE_DIR}/table/compaction_merging_iterator.cc
     ${ROCKSDB_SOURCE_DIR}/table/meta_blocks.cc
     ${ROCKSDB_SOURCE_DIR}/table/persistent_cache_helper.cc
     ${ROCKSDB_SOURCE_DIR}/table/plain/plain_table_bloom.cc
@@ -323,6 +322,8 @@ set(SOURCES
     ${ROCKSDB_SOURCE_DIR}/util/string_util.cc
     ${ROCKSDB_SOURCE_DIR}/util/thread_local.cc
     ${ROCKSDB_SOURCE_DIR}/util/threadpool_imp.cc
+    ${ROCKSDB_SOURCE_DIR}/util/udt_util.cc
+    ${ROCKSDB_SOURCE_DIR}/util/write_batch_util.cc
     ${ROCKSDB_SOURCE_DIR}/util/xxhash.cc
     ${ROCKSDB_SOURCE_DIR}/utilities/agg_merge/agg_merge.cc
     ${ROCKSDB_SOURCE_DIR}/utilities/backup/backup_engine.cc
@@ -405,12 +406,6 @@ set(SOURCES
     ${ROCKSDB_SOURCE_DIR}/utilities/transactions/lock/range/range_tree/lib/util/memarena.cc
     build_version.cc) # generated by hand
 
-if(ENABLE_SSE42 AND ENABLE_PCLMULQDQ)
-  set_source_files_properties(
-    "${ROCKSDB_SOURCE_DIR}/util/crc32c.cc"
-    PROPERTIES COMPILE_FLAGS "-msse4.2 -mpclmul")
-endif()
-
 if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(powerpc|ppc)64")
   list(APPEND SOURCES
     "${ROCKSDB_SOURCE_DIR}/util/crc32c_ppc.c"
diff --git a/src/Coordination/tests/gtest_coordination.cpp b/src/Coordination/tests/gtest_coordination.cpp
index beae6254562..d39031773cd 100644
--- a/src/Coordination/tests/gtest_coordination.cpp
+++ b/src/Coordination/tests/gtest_coordination.cpp
@@ -111,10 +111,13 @@ public:
     }
 };
 
-using Implementation = testing::Types<TestParam<DB::KeeperMemoryStorage, true>,
-                                      TestParam<DB::KeeperMemoryStorage, false>,
-                                      TestParam<DB::KeeperRocksStorage, true>,
-                                      TestParam<DB::KeeperRocksStorage, false>>;
+using Implementation = testing::Types<TestParam<DB::KeeperMemoryStorage, true>
+                                      ,TestParam<DB::KeeperMemoryStorage, false>
+#if USE_ROCKSDB
+                                      ,TestParam<DB::KeeperRocksStorage, true>
+                                      ,TestParam<DB::KeeperRocksStorage, false>
+#endif
+                                      >;
 TYPED_TEST_SUITE(CoordinationTest, Implementation);
 
 TYPED_TEST(CoordinationTest, RaftServerConfigParse)

From b47f8a733f926c0f52a5837414bcd6ecfece9089 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Mon, 5 Aug 2024 08:40:35 +0200
Subject: [PATCH 1357/1488] ci: fix basic errors collecting after stateless
 tests

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 docker/test/stateless/run.sh | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh
index b352539cc1a..c582d3a982b 100755
--- a/docker/test/stateless/run.sh
+++ b/docker/test/stateless/run.sh
@@ -393,6 +393,8 @@ do
         | zstd --threads=0 > "/test_output/trace-log-$trace_type-flamegraph.tsv.zst" ||:
 done
 
+# Grep logs for sanitizer asserts, crashes and other critical errors
+check_logs_for_critical_errors
 
 # Compressed (FIXME: remove once only github actions will be left)
 rm /var/log/clickhouse-server/clickhouse-server.log
@@ -426,7 +428,4 @@ if [[ "$USE_SHARED_CATALOG" -eq 1 ]]; then
     tar -chf /test_output/coordination1.tar /var/lib/clickhouse1/coordination ||:
 fi
 
-# Grep logs for sanitizer asserts, crashes and other critical errors
-check_logs_for_critical_errors
-
 collect_core_dumps

From cdbc4f357324ad0b41d46b6e54475ac2cebdc630 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Mon, 5 Aug 2024 08:45:20 +0200
Subject: [PATCH 1358/1488] ci: fail the test if the entrypoint script failed

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 tests/ci/functional_test_check.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tests/ci/functional_test_check.py b/tests/ci/functional_test_check.py
index 66db082677f..52970404d2d 100644
--- a/tests/ci/functional_test_check.py
+++ b/tests/ci/functional_test_check.py
@@ -166,6 +166,7 @@ def _get_statless_tests_to_run(pr_info: PRInfo) -> List[str]:
 
 
 def process_results(
+    ret_code: int,
     result_directory: Path,
     server_log_path: Path,
 ) -> Tuple[StatusType, str, TestResults, List[Path]]:
@@ -192,6 +193,9 @@ def process_results(
         logging.info("Files in result folder %s", os.listdir(result_directory))
         return ERROR, "Invalid check_status.tsv", test_results, additional_files
     state, description = status[0][0], status[0][1]
+    if ret_code != 0:
+        state = ERROR
+        description += " (but script exited with an error)"
 
     try:
         results_path = result_directory / "test_results.tsv"
@@ -339,7 +343,7 @@ def main():
         ci_logs_credentials.clean_ci_logs_from_credentials(run_log_path)
 
         state, description, test_results, additional_logs = process_results(
-            result_path, server_log_path
+            retcode, result_path, server_log_path
         )
     else:
         print(

From 9ce55b69b49ac8426ed0b3db16b95964e3c4db4d Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Mon, 5 Aug 2024 11:22:55 +0200
Subject: [PATCH 1359/1488] Fix possible CANNOT_READ_ALL_DATA during server
 startup in performance tests

CI [1]:

    2024.08.04 22:09:11.646800 [ 1052 ] {} <Error> Application: Code: 33. DB::Exception: Cannot read all data. Bytes read: 0. Bytes expected: 4.: While checking access for disk backups. (CANNOT_READ_ALL_DATA), Stack trace (when copying this message, always include the lines below):

  [1]: https://s3.amazonaws.com/clickhouse-test-reports/64955/6702acf6f2e4a0ee9697066e38006631fc7f69df/performance_comparison__aarch64__[2_4].html

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 tests/performance/scripts/compare.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/performance/scripts/compare.sh b/tests/performance/scripts/compare.sh
index cb56ab6c5bf..da7bbf77a28 100755
--- a/tests/performance/scripts/compare.sh
+++ b/tests/performance/scripts/compare.sh
@@ -71,6 +71,8 @@ function configure
 {
     # Use the new config for both servers, so that we can change it in a PR.
     rm right/config/config.d/text_log.xml ||:
+    # backups disk uses absolute path, and this overlaps between servers, that could lead to errors
+    rm right/config/config.d/backups.xml ||:
     cp -rv right/config left ||:
 
     # Start a temporary server to rename the tables

From a499cd25c7e12c05f2f8fa3fe546715c751ad88d Mon Sep 17 00:00:00 2001
From: avogar <avogar@clickhouse.com>
Date: Mon, 5 Aug 2024 09:31:41 +0000
Subject: [PATCH 1360/1488] Fix for integers

---
 src/Functions/if.cpp                             | 16 ++++++----------
 ...3215_varian_as_common_type_integers.reference |  8 ++++++++
 .../03215_varian_as_common_type_integers.sql     |  8 ++++++++
 3 files changed, 22 insertions(+), 10 deletions(-)
 create mode 100644 tests/queries/0_stateless/03215_varian_as_common_type_integers.reference
 create mode 100644 tests/queries/0_stateless/03215_varian_as_common_type_integers.sql

diff --git a/src/Functions/if.cpp b/src/Functions/if.cpp
index 64da6e95a43..8829b3c4ff1 100644
--- a/src/Functions/if.cpp
+++ b/src/Functions/if.cpp
@@ -632,11 +632,6 @@ private:
 
     ColumnPtr executeTuple(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const
     {
-        /// For different Tuples the result type can be Variant with this Tuples if use_variant_as_common_type=1.
-        /// In this case we should use generic implementation.
-        if (!isTuple(result_type))
-            return nullptr;
-
         /// Calculate function for each corresponding elements of tuples.
 
         const ColumnWithTypeAndName & arg1 = arguments[1];
@@ -682,11 +677,6 @@ private:
 
     ColumnPtr executeMap(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const
     {
-        /// For different Maps the result type can be Variant with this Maps if use_variant_as_common_type=1.
-        /// In this case we should use generic implementation.
-        if (!isMap(result_type))
-            return nullptr;
-
         auto extract_kv_from_map = [](const ColumnMap * map)
         {
             const ColumnTuple & tuple = map->getNestedData();
@@ -1243,6 +1233,12 @@ public:
             throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of first argument of function {}. "
                 "Must be ColumnUInt8 or ColumnConstUInt8.", arg_cond.column->getName(), getName());
 
+        /// If result is Variant, always use generic implementation.
+        /// Using typed implementations may lead to incorrect result column type when
+        /// resulting Variant is created by use_variant_when_no_common_type.
+        if (isVariant(result_type))
+            return executeGeneric(cond_col, arguments, input_rows_count, use_variant_when_no_common_type);
+
         auto call = [&](const auto & types) -> bool
         {
             using Types = std::decay_t<decltype(types)>;
diff --git a/tests/queries/0_stateless/03215_varian_as_common_type_integers.reference b/tests/queries/0_stateless/03215_varian_as_common_type_integers.reference
new file mode 100644
index 00000000000..c5edc9e9963
--- /dev/null
+++ b/tests/queries/0_stateless/03215_varian_as_common_type_integers.reference
@@ -0,0 +1,8 @@
+0	Variant(Int64, UInt64)
+1	Variant(Int64, UInt64)
+0	Variant(Int32, UInt64)
+1	Variant(Int32, UInt64)
+0	Variant(Int16, UInt64)
+1	Variant(Int16, UInt64)
+0	Variant(Int8, UInt64)
+1	Variant(Int8, UInt64)
diff --git a/tests/queries/0_stateless/03215_varian_as_common_type_integers.sql b/tests/queries/0_stateless/03215_varian_as_common_type_integers.sql
new file mode 100644
index 00000000000..dcc69735534
--- /dev/null
+++ b/tests/queries/0_stateless/03215_varian_as_common_type_integers.sql
@@ -0,0 +1,8 @@
+set use_variant_as_common_type = 1;
+set allow_experimental_variant_type = 1;
+
+SELECT if(number % 2, number::Int64, number::UInt64) as res, toTypeName(res) FROM numbers(2);
+SELECT if(number % 2, number::Int32, number::UInt64) as res, toTypeName(res) FROM numbers(2);
+SELECT if(number % 2, number::Int16, number::UInt64) as res, toTypeName(res) FROM numbers(2);
+SELECT if(number % 2, number::Int8, number::UInt64) as res, toTypeName(res) FROM numbers(2);
+

From 2b369cccdd58902e9da3fd3947e5cb5759ba2881 Mon Sep 17 00:00:00 2001
From: avogar <avogar@clickhouse.com>
Date: Mon, 5 Aug 2024 09:35:56 +0000
Subject: [PATCH 1361/1488] Reduce table size in
 03037_dynamic_merges_2_vertical_wide_merge_tree test

---
 ...3037_dynamic_merges_2_vertical_wide_merge_tree.reference | 6 +++---
 .../03037_dynamic_merges_2_vertical_wide_merge_tree.sql     | 6 +++---
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/tests/queries/0_stateless/03037_dynamic_merges_2_vertical_wide_merge_tree.reference b/tests/queries/0_stateless/03037_dynamic_merges_2_vertical_wide_merge_tree.reference
index afd392002e5..253d87de5f0 100644
--- a/tests/queries/0_stateless/03037_dynamic_merges_2_vertical_wide_merge_tree.reference
+++ b/tests/queries/0_stateless/03037_dynamic_merges_2_vertical_wide_merge_tree.reference
@@ -1,3 +1,3 @@
-1000000	Array(UInt16)
-1000000	String
-1000000	UInt64
+200000	Array(UInt16)
+200000	String
+200000	UInt64
diff --git a/tests/queries/0_stateless/03037_dynamic_merges_2_vertical_wide_merge_tree.sql b/tests/queries/0_stateless/03037_dynamic_merges_2_vertical_wide_merge_tree.sql
index 104d6018e41..dd643f8dffd 100644
--- a/tests/queries/0_stateless/03037_dynamic_merges_2_vertical_wide_merge_tree.sql
+++ b/tests/queries/0_stateless/03037_dynamic_merges_2_vertical_wide_merge_tree.sql
@@ -5,9 +5,9 @@ set allow_experimental_dynamic_type = 1;
 drop table if exists test;
 create table test (id UInt64, d Dynamic) engine=MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1, lock_acquire_timeout_for_background_operations=600;
 system stop merges test;
-insert into test select number, number from numbers(1000000);
-insert into test select number, 'str_' || toString(number) from numbers(1000000, 1000000);
-insert into test select number, range(number % 10 + 1) from numbers(2000000, 1000000);
+insert into test select number, number from numbers(200000);
+insert into test select number, 'str_' || toString(number) from numbers(200000, 200000);
+insert into test select number, range(number % 10 + 1) from numbers(400000, 200000);
 system start merges test;
 optimize table test final;
 select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);

From cec8a5d52b83f0c1cdcaed833aec9bf79941b2a8 Mon Sep 17 00:00:00 2001
From: avogar <avogar@clickhouse.com>
Date: Mon, 5 Aug 2024 09:42:22 +0000
Subject: [PATCH 1362/1488] Reduce table size in similar tests

---
 ...dynamic_merges_2_horizontal_compact_merge_tree.reference | 6 +++---
 ...03037_dynamic_merges_2_horizontal_compact_merge_tree.sql | 6 +++---
 ...37_dynamic_merges_2_horizontal_wide_merge_tree.reference | 6 +++---
 .../03037_dynamic_merges_2_horizontal_wide_merge_tree.sql   | 6 +++---
 ...7_dynamic_merges_2_vertical_compact_merge_tree.reference | 6 +++---
 .../03037_dynamic_merges_2_vertical_compact_merge_tree.sql  | 6 +++---
 6 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/tests/queries/0_stateless/03037_dynamic_merges_2_horizontal_compact_merge_tree.reference b/tests/queries/0_stateless/03037_dynamic_merges_2_horizontal_compact_merge_tree.reference
index afd392002e5..253d87de5f0 100644
--- a/tests/queries/0_stateless/03037_dynamic_merges_2_horizontal_compact_merge_tree.reference
+++ b/tests/queries/0_stateless/03037_dynamic_merges_2_horizontal_compact_merge_tree.reference
@@ -1,3 +1,3 @@
-1000000	Array(UInt16)
-1000000	String
-1000000	UInt64
+200000	Array(UInt16)
+200000	String
+200000	UInt64
diff --git a/tests/queries/0_stateless/03037_dynamic_merges_2_horizontal_compact_merge_tree.sql b/tests/queries/0_stateless/03037_dynamic_merges_2_horizontal_compact_merge_tree.sql
index e133ac3001f..fa64ed2f8fd 100644
--- a/tests/queries/0_stateless/03037_dynamic_merges_2_horizontal_compact_merge_tree.sql
+++ b/tests/queries/0_stateless/03037_dynamic_merges_2_horizontal_compact_merge_tree.sql
@@ -5,9 +5,9 @@ set allow_experimental_dynamic_type = 1;
 drop table if exists test;
 create table test (id UInt64, d Dynamic) engine=MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000, lock_acquire_timeout_for_background_operations=600;
 system stop merges test;
-insert into test select number, number from numbers(1000000);
-insert into test select number, 'str_' || toString(number) from numbers(1000000, 1000000);
-insert into test select number, range(number % 10 + 1) from numbers(2000000, 1000000);
+insert into test select number, number from numbers(200000);
+insert into test select number, 'str_' || toString(number) from numbers(200000, 200000);
+insert into test select number, range(number % 10 + 1) from numbers(400000, 200000);
 system start merges test;
 optimize table test final;
 select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
diff --git a/tests/queries/0_stateless/03037_dynamic_merges_2_horizontal_wide_merge_tree.reference b/tests/queries/0_stateless/03037_dynamic_merges_2_horizontal_wide_merge_tree.reference
index afd392002e5..253d87de5f0 100644
--- a/tests/queries/0_stateless/03037_dynamic_merges_2_horizontal_wide_merge_tree.reference
+++ b/tests/queries/0_stateless/03037_dynamic_merges_2_horizontal_wide_merge_tree.reference
@@ -1,3 +1,3 @@
-1000000	Array(UInt16)
-1000000	String
-1000000	UInt64
+200000	Array(UInt16)
+200000	String
+200000	UInt64
diff --git a/tests/queries/0_stateless/03037_dynamic_merges_2_horizontal_wide_merge_tree.sql b/tests/queries/0_stateless/03037_dynamic_merges_2_horizontal_wide_merge_tree.sql
index d527081b763..4b8a036f166 100644
--- a/tests/queries/0_stateless/03037_dynamic_merges_2_horizontal_wide_merge_tree.sql
+++ b/tests/queries/0_stateless/03037_dynamic_merges_2_horizontal_wide_merge_tree.sql
@@ -5,9 +5,9 @@ set allow_experimental_dynamic_type = 1;
 drop table if exists test;
 create table test (id UInt64, d Dynamic) engine=MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1, lock_acquire_timeout_for_background_operations=600;
 system stop merges test;
-insert into test select number, number from numbers(1000000);
-insert into test select number, 'str_' || toString(number) from numbers(1000000, 1000000);
-insert into test select number, range(number % 10 + 1) from numbers(2000000, 1000000);
+insert into test select number, number from numbers(200000);
+insert into test select number, 'str_' || toString(number) from numbers(200000, 200000);
+insert into test select number, range(number % 10 + 1) from numbers(400000, 200000);
 system start merges test;
 optimize table test final;
 select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);
diff --git a/tests/queries/0_stateless/03037_dynamic_merges_2_vertical_compact_merge_tree.reference b/tests/queries/0_stateless/03037_dynamic_merges_2_vertical_compact_merge_tree.reference
index afd392002e5..253d87de5f0 100644
--- a/tests/queries/0_stateless/03037_dynamic_merges_2_vertical_compact_merge_tree.reference
+++ b/tests/queries/0_stateless/03037_dynamic_merges_2_vertical_compact_merge_tree.reference
@@ -1,3 +1,3 @@
-1000000	Array(UInt16)
-1000000	String
-1000000	UInt64
+200000	Array(UInt16)
+200000	String
+200000	UInt64
diff --git a/tests/queries/0_stateless/03037_dynamic_merges_2_vertical_compact_merge_tree.sql b/tests/queries/0_stateless/03037_dynamic_merges_2_vertical_compact_merge_tree.sql
index ebccfb77922..a4e67de76db 100644
--- a/tests/queries/0_stateless/03037_dynamic_merges_2_vertical_compact_merge_tree.sql
+++ b/tests/queries/0_stateless/03037_dynamic_merges_2_vertical_compact_merge_tree.sql
@@ -5,9 +5,9 @@ set allow_experimental_dynamic_type = 1;
 drop table if exists test;
 create table test (id UInt64, d Dynamic) engine=MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1, lock_acquire_timeout_for_background_operations=600;
 system stop merges test;
-insert into test select number, number from numbers(1000000);
-insert into test select number, 'str_' || toString(number) from numbers(1000000, 1000000);
-insert into test select number, range(number % 10 + 1) from numbers(2000000, 1000000);
+insert into test select number, number from numbers(200000);
+insert into test select number, 'str_' || toString(number) from numbers(200000, 200000);
+insert into test select number, range(number % 10 + 1) from numbers(400000, 200000);
 system start merges test;
 optimize table test final;
 select count(), dynamicType(d) from test group by dynamicType(d) order by count(), dynamicType(d);

From e0362b00f5a24ea19e16b27b71963efe64a174c0 Mon Sep 17 00:00:00 2001
From: maxvostrikov <max.vostrikov@clickhouse.com>
Date: Mon, 5 Aug 2024 11:52:07 +0200
Subject: [PATCH 1363/1488] squash! fix for parallel execution

---
 tests/integration/test_parquet_page_index/test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/integration/test_parquet_page_index/test.py b/tests/integration/test_parquet_page_index/test.py
index 59dbab09be5..778b6618a61 100644
--- a/tests/integration/test_parquet_page_index/test.py
+++ b/tests/integration/test_parquet_page_index/test.py
@@ -35,7 +35,7 @@ def delete_if_exists(file_path):
     "query, expected_result",
     {
         (
-            "SElECT number, number+1 FROM system.numbers LIMIT 100 "
+            "SELECT number, number+1 FROM system.numbers LIMIT 100 "
             "INTO OUTFILE '{file_name}' FORMAT Parquet "
             "SETTINGS output_format_parquet_use_custom_encoder = false, "
             "output_format_parquet_write_page_index = true;",

From 0a7a67b8e0e05bc9476d4f9dd38747bf61b6bb8e Mon Sep 17 00:00:00 2001
From: avogar <avogar@clickhouse.com>
Date: Mon, 5 Aug 2024 09:57:13 +0000
Subject: [PATCH 1364/1488] Disable 03038_nested_dynamic_merges* under
 sanitizers because it's too slow

---
 ...sted_dynamic_merges_compact_horizontal.sql |  2 +-
 ...nested_dynamic_merges_compact_vertical.sql |  2 +-
 ...8_nested_dynamic_merges_small.reference.j2 | 84 +++++++++++++++++++
 .../03038_nested_dynamic_merges_small.sql.j2  | 35 ++++++++
 ..._nested_dynamic_merges_wide_horizontal.sql |  2 +-
 ...38_nested_dynamic_merges_wide_vertical.sql |  2 +-
 6 files changed, 123 insertions(+), 4 deletions(-)
 create mode 100644 tests/queries/0_stateless/03038_nested_dynamic_merges_small.reference.j2
 create mode 100644 tests/queries/0_stateless/03038_nested_dynamic_merges_small.sql.j2

diff --git a/tests/queries/0_stateless/03038_nested_dynamic_merges_compact_horizontal.sql b/tests/queries/0_stateless/03038_nested_dynamic_merges_compact_horizontal.sql
index 1d5c63dcdf1..81888946681 100644
--- a/tests/queries/0_stateless/03038_nested_dynamic_merges_compact_horizontal.sql
+++ b/tests/queries/0_stateless/03038_nested_dynamic_merges_compact_horizontal.sql
@@ -1,4 +1,4 @@
--- Tags: long
+-- Tags: long, no-tsan, no-msan, no-ubsan, no-asan
 
 set allow_experimental_variant_type = 1;
 set use_variant_as_common_type = 1;
diff --git a/tests/queries/0_stateless/03038_nested_dynamic_merges_compact_vertical.sql b/tests/queries/0_stateless/03038_nested_dynamic_merges_compact_vertical.sql
index 2bffe35c577..ba58ca471a2 100644
--- a/tests/queries/0_stateless/03038_nested_dynamic_merges_compact_vertical.sql
+++ b/tests/queries/0_stateless/03038_nested_dynamic_merges_compact_vertical.sql
@@ -1,4 +1,4 @@
--- Tags: long
+-- Tags: long, no-tsan, no-msan, no-ubsan, no-asan
 
 set allow_experimental_variant_type = 1;
 set use_variant_as_common_type = 1;
diff --git a/tests/queries/0_stateless/03038_nested_dynamic_merges_small.reference.j2 b/tests/queries/0_stateless/03038_nested_dynamic_merges_small.reference.j2
new file mode 100644
index 00000000000..ae07c164074
--- /dev/null
+++ b/tests/queries/0_stateless/03038_nested_dynamic_merges_small.reference.j2
@@ -0,0 +1,84 @@
+2	Tuple(a Dynamic(max_types=3)):Date
+3	Tuple(a Dynamic(max_types=3)):Array(UInt8)
+5	Tuple(a Dynamic(max_types=3)):String
+5	Tuple(a Dynamic(max_types=3)):UInt64
+10	UInt64:None
+3	Tuple(a Dynamic(max_types=3)):Array(UInt8)
+5	Tuple(a Dynamic(max_types=3)):UInt64
+7	Tuple(a Dynamic(max_types=3)):String
+10	UInt64:None
+2	Tuple(a Dynamic(max_types=3)):DateTime
+3	Tuple(a Dynamic(max_types=3)):Array(UInt8)
+5	Tuple(a Dynamic(max_types=3)):UInt64
+7	Tuple(a Dynamic(max_types=3)):String
+10	Tuple(a Dynamic(max_types=3)):Tuple(UInt64)
+10	UInt64:None
+13	Tuple(a Dynamic(max_types=3)):None
+5	Tuple(a Dynamic(max_types=3)):UInt64
+10	Tuple(a Dynamic(max_types=3)):Tuple(UInt64)
+10	UInt64:None
+12	Tuple(a Dynamic(max_types=3)):String
+13	Tuple(a Dynamic(max_types=3)):None
+2	Tuple(a Dynamic(max_types=3)):Date
+3	Tuple(a Dynamic(max_types=3)):Array(UInt8)
+5	Tuple(a Dynamic(max_types=3)):String
+5	Tuple(a Dynamic(max_types=3)):UInt64
+10	UInt64:None
+3	Tuple(a Dynamic(max_types=3)):Array(UInt8)
+5	Tuple(a Dynamic(max_types=3)):UInt64
+7	Tuple(a Dynamic(max_types=3)):String
+10	UInt64:None
+2	Tuple(a Dynamic(max_types=3)):DateTime
+3	Tuple(a Dynamic(max_types=3)):Array(UInt8)
+5	Tuple(a Dynamic(max_types=3)):UInt64
+7	Tuple(a Dynamic(max_types=3)):String
+10	Tuple(a Dynamic(max_types=3)):Tuple(UInt64)
+10	UInt64:None
+13	Tuple(a Dynamic(max_types=3)):None
+5	Tuple(a Dynamic(max_types=3)):UInt64
+10	Tuple(a Dynamic(max_types=3)):Tuple(UInt64)
+10	UInt64:None
+12	Tuple(a Dynamic(max_types=3)):String
+13	Tuple(a Dynamic(max_types=3)):None
+2	Tuple(a Dynamic(max_types=3)):Date
+3	Tuple(a Dynamic(max_types=3)):Array(UInt8)
+5	Tuple(a Dynamic(max_types=3)):String
+5	Tuple(a Dynamic(max_types=3)):UInt64
+10	UInt64:None
+3	Tuple(a Dynamic(max_types=3)):Array(UInt8)
+5	Tuple(a Dynamic(max_types=3)):UInt64
+7	Tuple(a Dynamic(max_types=3)):String
+10	UInt64:None
+2	Tuple(a Dynamic(max_types=3)):DateTime
+3	Tuple(a Dynamic(max_types=3)):Array(UInt8)
+5	Tuple(a Dynamic(max_types=3)):UInt64
+7	Tuple(a Dynamic(max_types=3)):String
+10	Tuple(a Dynamic(max_types=3)):Tuple(UInt64)
+10	UInt64:None
+13	Tuple(a Dynamic(max_types=3)):None
+5	Tuple(a Dynamic(max_types=3)):UInt64
+10	Tuple(a Dynamic(max_types=3)):Tuple(UInt64)
+10	UInt64:None
+12	Tuple(a Dynamic(max_types=3)):String
+13	Tuple(a Dynamic(max_types=3)):None
+2	Tuple(a Dynamic(max_types=3)):Date
+3	Tuple(a Dynamic(max_types=3)):Array(UInt8)
+5	Tuple(a Dynamic(max_types=3)):String
+5	Tuple(a Dynamic(max_types=3)):UInt64
+10	UInt64:None
+3	Tuple(a Dynamic(max_types=3)):Array(UInt8)
+5	Tuple(a Dynamic(max_types=3)):UInt64
+7	Tuple(a Dynamic(max_types=3)):String
+10	UInt64:None
+2	Tuple(a Dynamic(max_types=3)):DateTime
+3	Tuple(a Dynamic(max_types=3)):Array(UInt8)
+5	Tuple(a Dynamic(max_types=3)):UInt64
+7	Tuple(a Dynamic(max_types=3)):String
+10	Tuple(a Dynamic(max_types=3)):Tuple(UInt64)
+10	UInt64:None
+13	Tuple(a Dynamic(max_types=3)):None
+5	Tuple(a Dynamic(max_types=3)):UInt64
+10	Tuple(a Dynamic(max_types=3)):Tuple(UInt64)
+10	UInt64:None
+12	Tuple(a Dynamic(max_types=3)):String
+13	Tuple(a Dynamic(max_types=3)):None
diff --git a/tests/queries/0_stateless/03038_nested_dynamic_merges_small.sql.j2 b/tests/queries/0_stateless/03038_nested_dynamic_merges_small.sql.j2
new file mode 100644
index 00000000000..7828c2af49c
--- /dev/null
+++ b/tests/queries/0_stateless/03038_nested_dynamic_merges_small.sql.j2
@@ -0,0 +1,35 @@
+set allow_experimental_variant_type = 1;
+set use_variant_as_common_type = 1;
+set allow_experimental_dynamic_type = 1;
+set enable_named_columns_in_function_tuple = 0;
+
+drop table if exists test;
+
+{% for engine in ['MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000',
+                  'MergeTree order by id settings min_rows_for_wide_part=1000000000, min_bytes_for_wide_part=10000000000, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1',
+                  'MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1',
+                  'MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1'] -%}
+
+create table test (id UInt64, d Dynamic(max_types=3)) engine=MergeTree order by id settings min_rows_for_wide_part=1, min_bytes_for_wide_part=1, vertical_merge_algorithm_min_rows_to_activate=1, vertical_merge_algorithm_min_columns_to_activate=1, lock_acquire_timeout_for_background_operations=600;
+
+system stop merges test;
+insert into test select number, number from numbers(10);
+insert into test select number, tuple(if(number % 2 == 0, number, 'str_' || toString(number)))::Tuple(a Dynamic(max_types=3)) from numbers(10);
+insert into test select number, tuple(if(number % 3 == 0, toDate(number), range(number % 10)))::Tuple(a Dynamic(max_types=3)) from numbers(5);
+
+select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=3))`.a) as type from test group by type order by count(), type;
+system start merges test;
+optimize table test final;
+select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=3))`.a) as type from test group by type order by count(), type;
+
+insert into test select number, tuple(if(number % 3 == 0, toDateTime(number), NULL))::Tuple(a Dynamic(max_types=3)) from numbers(5);
+insert into test select number, tuple(if(number % 2 == 0, tuple(number), NULL))::Tuple(a Dynamic(max_types=3)) from numbers(20);
+
+select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=3))`.a) as type from test group by type order by count(), type;
+system start merges test;
+optimize table test final;
+select count(), dynamicType(d) || ':' || dynamicType(d.`Tuple(a Dynamic(max_types=3))`.a) as type from test group by type order by count(), type;
+
+drop table test;
+
+{% endfor -%}
diff --git a/tests/queries/0_stateless/03038_nested_dynamic_merges_wide_horizontal.sql b/tests/queries/0_stateless/03038_nested_dynamic_merges_wide_horizontal.sql
index fb686091ebb..a53c5b0b2a5 100644
--- a/tests/queries/0_stateless/03038_nested_dynamic_merges_wide_horizontal.sql
+++ b/tests/queries/0_stateless/03038_nested_dynamic_merges_wide_horizontal.sql
@@ -1,4 +1,4 @@
--- Tags: long
+-- Tags: long, no-tsan, no-msan, no-ubsan, no-asan
 
 set allow_experimental_variant_type = 1;
 set use_variant_as_common_type = 1;
diff --git a/tests/queries/0_stateless/03038_nested_dynamic_merges_wide_vertical.sql b/tests/queries/0_stateless/03038_nested_dynamic_merges_wide_vertical.sql
index ed195452d56..4256b010ec0 100644
--- a/tests/queries/0_stateless/03038_nested_dynamic_merges_wide_vertical.sql
+++ b/tests/queries/0_stateless/03038_nested_dynamic_merges_wide_vertical.sql
@@ -1,4 +1,4 @@
--- Tags: long
+-- Tags: long,  no-tsan, no-msan, no-ubsan, no-asan
 
 set allow_experimental_variant_type = 1;
 set use_variant_as_common_type = 1;

From b53e757656e298bb308862a8294cde5718e37580 Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Wed, 24 Jul 2024 10:01:50 +0000
Subject: [PATCH 1365/1488] Check argument types in DataTypeAggregateFunction
 ctor

---
 src/DataTypes/DataTypeAggregateFunction.cpp | 27 +++++++++++++++++++++
 src/DataTypes/DataTypeAggregateFunction.h   |  8 +-----
 2 files changed, 28 insertions(+), 7 deletions(-)

diff --git a/src/DataTypes/DataTypeAggregateFunction.cpp b/src/DataTypes/DataTypeAggregateFunction.cpp
index 09175617bf1..ee42e4fea11 100644
--- a/src/DataTypes/DataTypeAggregateFunction.cpp
+++ b/src/DataTypes/DataTypeAggregateFunction.cpp
@@ -33,6 +33,33 @@ namespace ErrorCodes
     extern const int LOGICAL_ERROR;
 }
 
+
+DataTypeAggregateFunction::DataTypeAggregateFunction(AggregateFunctionPtr function_, const DataTypes & argument_types_,
+                            const Array & parameters_, std::optional<size_t> version_)
+    : function(std::move(function_))
+    , argument_types(argument_types_)
+    , parameters(parameters_)
+    , version(version_)
+{
+    Strings argument_type_names;
+    for (const auto & argument_type : argument_types)
+        argument_type_names.push_back(argument_type->getName());
+
+    Strings function_argument_type_names;
+    const auto & function_argument_types = function->getArgumentTypes();
+    for (const auto & argument_type : function_argument_types)
+        function_argument_type_names.push_back(argument_type->getName());
+
+    size_t argument_types_size = std::max(argument_types.size(), function_argument_types.size());
+    for (size_t i = 0; i < argument_types_size; ++i)
+    {
+        if (argument_types.size() != function_argument_types.size() || !argument_types[i]->equals(*function_argument_types[i]))
+            throw Exception(ErrorCodes::LOGICAL_ERROR,
+                            "Data type AggregateFunction {} got argument types different from function argument types: [{}] != [{}]",
+                            function->getName(), fmt::join(argument_type_names, ", "), fmt::join(function_argument_type_names, ", "));
+    }
+}
+
 String DataTypeAggregateFunction::getFunctionName() const
 {
     return function->getName();
diff --git a/src/DataTypes/DataTypeAggregateFunction.h b/src/DataTypes/DataTypeAggregateFunction.h
index 52ed151107e..e3a4f9726d9 100644
--- a/src/DataTypes/DataTypeAggregateFunction.h
+++ b/src/DataTypes/DataTypeAggregateFunction.h
@@ -30,13 +30,7 @@ public:
     static constexpr bool is_parametric = true;
 
     DataTypeAggregateFunction(AggregateFunctionPtr function_, const DataTypes & argument_types_,
-                              const Array & parameters_, std::optional<size_t> version_ = std::nullopt)
-        : function(std::move(function_))
-        , argument_types(argument_types_)
-        , parameters(parameters_)
-        , version(version_)
-    {
-    }
+                              const Array & parameters_, std::optional<size_t> version_ = std::nullopt);
 
     size_t getVersion() const;
 

From 55fd2e04e331a58b83516fbefa6bad921fa842a3 Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Wed, 24 Jul 2024 17:00:59 +0000
Subject: [PATCH 1366/1488] wip

---
 src/Storages/StorageBuffer.cpp | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp
index 04e6d6676d1..4ae9e029e1b 100644
--- a/src/Storages/StorageBuffer.cpp
+++ b/src/Storages/StorageBuffer.cpp
@@ -271,6 +271,8 @@ void StorageBuffer::read(
         }
         else
         {
+            if (processed_stage > QueryProcessingStage::FetchColumns)
+                throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot read from Buffer table with different structure in non-initial stage of query execution");
             /// There is a struct mismatch and we need to convert read blocks from the destination table.
             const Block header = metadata_snapshot->getSampleBlock();
             Names columns_intersection = column_names;

From 12d917b74268ecb6a86b032d5c00c418c8a48f4e Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Wed, 24 Jul 2024 17:01:26 +0000
Subject: [PATCH 1367/1488] wip

---
 src/DataTypes/DataTypeAggregateFunction.cpp | 17 -----------------
 1 file changed, 17 deletions(-)

diff --git a/src/DataTypes/DataTypeAggregateFunction.cpp b/src/DataTypes/DataTypeAggregateFunction.cpp
index ee42e4fea11..a4cd3b9e511 100644
--- a/src/DataTypes/DataTypeAggregateFunction.cpp
+++ b/src/DataTypes/DataTypeAggregateFunction.cpp
@@ -41,23 +41,6 @@ DataTypeAggregateFunction::DataTypeAggregateFunction(AggregateFunctionPtr functi
     , parameters(parameters_)
     , version(version_)
 {
-    Strings argument_type_names;
-    for (const auto & argument_type : argument_types)
-        argument_type_names.push_back(argument_type->getName());
-
-    Strings function_argument_type_names;
-    const auto & function_argument_types = function->getArgumentTypes();
-    for (const auto & argument_type : function_argument_types)
-        function_argument_type_names.push_back(argument_type->getName());
-
-    size_t argument_types_size = std::max(argument_types.size(), function_argument_types.size());
-    for (size_t i = 0; i < argument_types_size; ++i)
-    {
-        if (argument_types.size() != function_argument_types.size() || !argument_types[i]->equals(*function_argument_types[i]))
-            throw Exception(ErrorCodes::LOGICAL_ERROR,
-                            "Data type AggregateFunction {} got argument types different from function argument types: [{}] != [{}]",
-                            function->getName(), fmt::join(argument_type_names, ", "), fmt::join(function_argument_type_names, ", "));
-    }
 }
 
 String DataTypeAggregateFunction::getFunctionName() const

From 1329b5eb0b17d5499639bbb973aab7b17c95b644 Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Fri, 26 Jul 2024 16:22:54 +0000
Subject: [PATCH 1368/1488] Fix Buffer over Distributed

---
 src/Storages/StorageBuffer.cpp                | 46 ++++++++++++--
 src/Storages/StorageBuffer.h                  |  1 +
 ...r_over_distributed_type_mismatch.reference | 18 ++++++
 ..._buffer_over_distributed_type_mismatch.sql | 60 +++++++++++++++++++
 4 files changed, 119 insertions(+), 6 deletions(-)
 create mode 100644 tests/queries/0_stateless/03208_buffer_over_distributed_type_mismatch.reference
 create mode 100644 tests/queries/0_stateless/03208_buffer_over_distributed_type_mismatch.sql

diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp
index 4ae9e029e1b..f753d369d2d 100644
--- a/src/Storages/StorageBuffer.cpp
+++ b/src/Storages/StorageBuffer.cpp
@@ -1,3 +1,7 @@
+#include <Storages/StorageBuffer.h>
+
+#include <Analyzer/TableNode.h>
+#include <Analyzer/Utils.h>
 #include <Interpreters/InterpreterInsertQuery.h>
 #include <Interpreters/InterpreterSelectQuery.h>
 #include <Interpreters/InterpreterSelectQueryAnalyzer.h>
@@ -23,7 +27,6 @@
 #include <Processors/Transforms/PartialSortingTransform.h>
 #include <Processors/Transforms/ReverseTransform.h>
 #include <Storages/AlterCommands.h>
-#include <Storages/StorageBuffer.h>
 #include <Storages/StorageFactory.h>
 #include <Storages/StorageValues.h>
 #include <Storages/checkAndGetLiteralArgument.h>
@@ -232,6 +235,12 @@ QueryProcessingStage::Enum StorageBuffer::getQueryProcessingStage(
     return QueryProcessingStage::FetchColumns;
 }
 
+bool StorageBuffer::isRemote() const
+{
+    auto destination = getDestinationTable();
+    return destination && destination->isRemote();
+}
+
 void StorageBuffer::read(
     QueryPlan & query_plan,
     const Names & column_names,
@@ -242,6 +251,29 @@ void StorageBuffer::read(
     size_t max_block_size,
     size_t num_streams)
 {
+    bool allow_experimental_analyzer = local_context->getSettingsRef().allow_experimental_analyzer;
+
+    if (allow_experimental_analyzer && processed_stage > QueryProcessingStage::FetchColumns)
+    {
+        /** For query processing stages after FetchColumns, we do not allow using the same table more than once in the query.
+          * For example: SELECT * FROM buffer t1 JOIN buffer t2 USING (column)
+          * In that case, we will execute this query separately for the destination table and for the buffer, resulting in incorrect results.
+          */
+        const auto & current_storage_id = getStorageID();
+        auto table_nodes = extractAllTableReferences(query_info.query_tree);
+        size_t count_of_current_storage = 0;
+        for (const auto & node : table_nodes)
+        {
+            const auto & table_node = node->as<TableNode &>();
+            if (table_node.getStorageID().getFullNameNotQuoted() == current_storage_id.getFullNameNotQuoted())
+            {
+                count_of_current_storage++;
+                if (count_of_current_storage > 1)
+                    throw Exception(ErrorCodes::NOT_IMPLEMENTED, "StorageBuffer over Distributed does not support using the same table more than once in the query");
+            }
+        }
+    }
+
     const auto & metadata_snapshot = storage_snapshot->metadata;
 
     if (auto destination = getDestinationTable())
@@ -271,8 +303,6 @@ void StorageBuffer::read(
         }
         else
         {
-            if (processed_stage > QueryProcessingStage::FetchColumns)
-                throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Cannot read from Buffer table with different structure in non-initial stage of query execution");
             /// There is a struct mismatch and we need to convert read blocks from the destination table.
             const Block header = metadata_snapshot->getSampleBlock();
             Names columns_intersection = column_names;
@@ -330,13 +360,17 @@ void StorageBuffer::read(
                     }
                 }
 
+                src_table_query_info.merge_storage_snapshot = storage_snapshot;
                 destination->read(
                         query_plan, columns_intersection, destination_snapshot, src_table_query_info,
                         local_context, processed_stage, max_block_size, num_streams);
 
-                if (query_plan.isInitialized())
+                if (query_plan.isInitialized() && processed_stage <= QueryProcessingStage::FetchColumns)
                 {
-
+                    /** The code below converts columns from metadata_snapshot to columns from destination_metadata_snapshot.
+                      * This conversion is not applicable for processed_stage > FetchColumns.
+                      * Instead, we rely on the converting actions at the end of this function.
+                      */
                     auto actions = addMissingDefaults(
                             query_plan.getCurrentDataStream().header,
                             header_after_adding_defaults.getNamesAndTypesList(),
@@ -399,7 +433,7 @@ void StorageBuffer::read(
     /// TODO: Find a way to support projections for StorageBuffer
     if (processed_stage > QueryProcessingStage::FetchColumns)
     {
-        if (local_context->getSettingsRef().allow_experimental_analyzer)
+        if (allow_experimental_analyzer)
         {
             auto storage = std::make_shared<StorageValues>(
                     getStorageID(),
diff --git a/src/Storages/StorageBuffer.h b/src/Storages/StorageBuffer.h
index cd6dd7b933f..02376f286b1 100644
--- a/src/Storages/StorageBuffer.h
+++ b/src/Storages/StorageBuffer.h
@@ -84,6 +84,7 @@ public:
         QueryProcessingStage::Enum processed_stage,
         size_t max_block_size,
         size_t num_streams) override;
+    bool isRemote() const override;
 
     bool supportsParallelInsert() const override { return true; }
 
diff --git a/tests/queries/0_stateless/03208_buffer_over_distributed_type_mismatch.reference b/tests/queries/0_stateless/03208_buffer_over_distributed_type_mismatch.reference
new file mode 100644
index 00000000000..1dc3acfeccb
--- /dev/null
+++ b/tests/queries/0_stateless/03208_buffer_over_distributed_type_mismatch.reference
@@ -0,0 +1,18 @@
+100
+100
+101
+101
+101
+102
+101
+101
+102
+100
+100
+101
+101
+101
+102
+101
+101
+102
diff --git a/tests/queries/0_stateless/03208_buffer_over_distributed_type_mismatch.sql b/tests/queries/0_stateless/03208_buffer_over_distributed_type_mismatch.sql
new file mode 100644
index 00000000000..5a7c89074cf
--- /dev/null
+++ b/tests/queries/0_stateless/03208_buffer_over_distributed_type_mismatch.sql
@@ -0,0 +1,60 @@
+
+DROP TABLE IF EXISTS realtimedrep;
+CREATE TABLE realtimedrep (`amount` Int32) ENGINE = MergeTree() ORDER BY tuple();
+INSERT INTO realtimedrep FORMAT Values (100);
+
+DROP TABLE IF EXISTS realtimedistributed;
+CREATE TABLE realtimedistributed (`amount` Int32) ENGINE = Distributed(test_cluster_two_shards, currentDatabase(), realtimedrep, rand());
+
+DROP TABLE IF EXISTS realtimebuff__fuzz_19;
+CREATE TABLE realtimebuff__fuzz_19 (`amount` UInt32) ENGINE = Buffer(currentDatabase(), 'realtimedistributed', 16, 3600, 36000, 10000, 1000000, 10000000, 100000000);
+INSERT INTO realtimebuff__fuzz_19 FORMAT Values (101);
+
+DROP TABLE IF EXISTS realtimebuff__fuzz_20;
+CREATE TABLE realtimebuff__fuzz_20 (`amount` Nullable(Int32)) ENGINE = Buffer(currentDatabase(), 'realtimedistributed', 16, 3600, 36000, 10000, 1000000, 10000000, 100000000);
+INSERT INTO realtimebuff__fuzz_20 FORMAT Values (101);
+
+SELECT amount FROM realtimebuff__fuzz_19 t1 ORDER BY ALL;
+SELECT amount + 1 FROM realtimebuff__fuzz_19 t1 ORDER BY ALL;
+SELECT amount + 1 FROM realtimebuff__fuzz_20 t1 ORDER BY ALL;
+SELECT sum(amount) = 100 FROM realtimebuff__fuzz_19 ORDER BY ALL; -- { serverError CANNOT_CONVERT_TYPE }
+SELECT sum(amount) = 100 FROM realtimebuff__fuzz_20 ORDER BY ALL; -- { serverError CANNOT_CONVERT_TYPE }
+
+SELECT amount FROM realtimebuff__fuzz_19 t1
+JOIN (SELECT number :: UInt32 AS amount FROM numbers(3) ) t2 ON t1.amount = t2.amount
+ORDER BY ALL
+SETTINGS allow_experimental_analyzer = 0; -- { serverError UNKNOWN_IDENTIFIER }
+
+SELECT amount FROM realtimebuff__fuzz_19 t1
+JOIN (SELECT number :: UInt32 AS amount FROM numbers(3) ) t2 ON t1.amount = t2.amount
+ORDER BY ALL
+SETTINGS allow_experimental_analyzer = 1;
+
+SELECT amount FROM realtimebuff__fuzz_19 t1
+JOIN (SELECT number :: UInt32 AS amount FROM numbers(300) ) t2 ON t1.amount = t2.amount
+ORDER BY ALL
+SETTINGS allow_experimental_analyzer = 0; -- { serverError UNKNOWN_IDENTIFIER }
+
+SELECT amount FROM realtimebuff__fuzz_19 t1
+JOIN (SELECT number :: UInt32 AS amount FROM numbers(300) ) t2 ON t1.amount = t2.amount
+ORDER BY ALL
+SETTINGS allow_experimental_analyzer = 1;
+
+SELECT t2.amount + 1 FROM (SELECT number :: UInt32 AS amount FROM numbers(300) ) t1
+JOIN realtimebuff__fuzz_19 t2 USING (amount)
+ORDER BY ALL
+;
+
+SELECT t2.amount + 1 FROM (SELECT number :: UInt32 AS amount FROM numbers(300) ) t1
+JOIN realtimebuff__fuzz_19 t2 ON t1.amount = t2.amount
+ORDER BY ALL
+;
+
+SELECT amount FROM realtimebuff__fuzz_19 t1
+JOIN realtimebuff__fuzz_19 t2 ON t1.amount = t2.amount
+; -- { serverError NOT_IMPLEMENTED,UNKNOWN_IDENTIFIER }
+
+SELECT amount FROM realtimebuff__fuzz_19 t1
+JOIN realtimebuff__fuzz_19 t2 ON t1.amount = t2.amount
+JOIN realtimebuff__fuzz_19 t3 ON t1.amount = t3.amount
+; -- { serverError NOT_IMPLEMENTED,AMBIGUOUS_COLUMN_NAME }

From 65c0efb2d50dc4ac37750505cde1b8d26729b871 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?=
 <benjamin.antal@clickhouse.com>
Date: Mon, 5 Aug 2024 10:29:49 +0000
Subject: [PATCH 1369/1488] Revert "Merge pull request #66510 from
 canhld94/fix_trivial_count_non_deterministic_func"

This reverts commit bf595ca374af503c087e2eb0f80f79490e5b8faa, reversing
changes made to b6b1a7a7790fcce40d2de67c62998a228246e729.
---
 src/Storages/MergeTree/MergeTreeData.cpp      |  2 +-
 src/Storages/VirtualColumnUtils.cpp           | 21 +++++++------------
 src/Storages/VirtualColumnUtils.h             | 10 +--------
 ..._with_non_deterministic_function.reference |  2 --
 ..._count_with_non_deterministic_function.sql |  4 ----
 5 files changed, 9 insertions(+), 30 deletions(-)
 delete mode 100644 tests/queries/0_stateless/03203_count_with_non_deterministic_function.reference
 delete mode 100644 tests/queries/0_stateless/03203_count_with_non_deterministic_function.sql

diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index ce27ad24e10..2286530aa83 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -1146,7 +1146,7 @@ std::optional<UInt64> MergeTreeData::totalRowsByPartitionPredicateImpl(
     auto metadata_snapshot = getInMemoryMetadataPtr();
     auto virtual_columns_block = getBlockWithVirtualsForFilter(metadata_snapshot, {parts[0]});
 
-    auto filter_dag = VirtualColumnUtils::splitFilterDagForAllowedInputs(filter_actions_dag.getOutputs().at(0), nullptr, /*allow_non_deterministic_functions=*/ false);
+    auto filter_dag = VirtualColumnUtils::splitFilterDagForAllowedInputs(filter_actions_dag->getOutputs().at(0), nullptr);
     if (!filter_dag)
         return {};
 
diff --git a/src/Storages/VirtualColumnUtils.cpp b/src/Storages/VirtualColumnUtils.cpp
index 90c2c7f93c1..ba1f4488005 100644
--- a/src/Storages/VirtualColumnUtils.cpp
+++ b/src/Storages/VirtualColumnUtils.cpp
@@ -275,8 +275,7 @@ bool isDeterministicInScopeOfQuery(const ActionsDAG::Node * node)
 static const ActionsDAG::Node * splitFilterNodeForAllowedInputs(
     const ActionsDAG::Node * node,
     const Block * allowed_inputs,
-    ActionsDAG::Nodes & additional_nodes,
-    bool allow_non_deterministic_functions)
+    ActionsDAG::Nodes & additional_nodes)
 {
     if (node->type == ActionsDAG::ActionType::FUNCTION)
     {
@@ -285,14 +284,8 @@ static const ActionsDAG::Node * splitFilterNodeForAllowedInputs(
             auto & node_copy = additional_nodes.emplace_back(*node);
             node_copy.children.clear();
             for (const auto * child : node->children)
-                if (const auto * child_copy = splitFilterNodeForAllowedInputs(child, allowed_inputs, additional_nodes, allow_non_deterministic_functions))
+                if (const auto * child_copy = splitFilterNodeForAllowedInputs(child, allowed_inputs, additional_nodes))
                     node_copy.children.push_back(child_copy);
-                /// Expression like (now_allowed AND allowed) is not allowed if allow_non_deterministic_functions = true. This is important for
-                /// trivial count optimization, otherwise we can get incorrect results. For example, if the query is
-                /// SELECT count() FROM table WHERE _partition_id = '0' AND rowNumberInBlock() = 1, we cannot apply
-                /// trivial count.
-                else if (!allow_non_deterministic_functions)
-                    return nullptr;
 
             if (node_copy.children.empty())
                 return nullptr;
@@ -318,7 +311,7 @@ static const ActionsDAG::Node * splitFilterNodeForAllowedInputs(
         {
             auto & node_copy = additional_nodes.emplace_back(*node);
             for (auto & child : node_copy.children)
-                if (child = splitFilterNodeForAllowedInputs(child, allowed_inputs, additional_nodes, allow_non_deterministic_functions); !child)
+                if (child = splitFilterNodeForAllowedInputs(child, allowed_inputs, additional_nodes); !child)
                     return nullptr;
 
             return &node_copy;
@@ -332,7 +325,7 @@ static const ActionsDAG::Node * splitFilterNodeForAllowedInputs(
                     auto index_hint_dag = index_hint->getActions().clone();
                     ActionsDAG::NodeRawConstPtrs atoms;
                     for (const auto & output : index_hint_dag.getOutputs())
-                        if (const auto * child_copy = splitFilterNodeForAllowedInputs(output, allowed_inputs, additional_nodes, allow_non_deterministic_functions))
+                        if (const auto * child_copy = splitFilterNodeForAllowedInputs(output, allowed_inputs, additional_nodes))
                             atoms.push_back(child_copy);
 
                     if (!atoms.empty())
@@ -366,13 +359,13 @@ static const ActionsDAG::Node * splitFilterNodeForAllowedInputs(
     return node;
 }
 
-std::optional<ActionsDAG> splitFilterDagForAllowedInputs(const ActionsDAG::Node * predicate, const Block * allowed_inputs, bool allow_non_deterministic_functions)
+std::optional<ActionsDAG> splitFilterDagForAllowedInputs(const ActionsDAG::Node * predicate, const Block * allowed_inputs)
 {
     if (!predicate)
         return {};
 
     ActionsDAG::Nodes additional_nodes;
-    const auto * res = splitFilterNodeForAllowedInputs(predicate, allowed_inputs, additional_nodes, allow_non_deterministic_functions);
+    const auto * res = splitFilterNodeForAllowedInputs(predicate, allowed_inputs, additional_nodes);
     if (!res)
         return {};
 
@@ -381,7 +374,7 @@ std::optional<ActionsDAG> splitFilterDagForAllowedInputs(const ActionsDAG::Node
 
 void filterBlockWithPredicate(const ActionsDAG::Node * predicate, Block & block, ContextPtr context)
 {
-    auto dag = splitFilterDagForAllowedInputs(predicate, &block,  /*allow_non_deterministic_functions=*/ false);
+    auto dag = splitFilterDagForAllowedInputs(predicate, &block);
     if (dag)
         filterBlockWithExpression(buildFilterExpression(std::move(*dag), context), block);
 }
diff --git a/src/Storages/VirtualColumnUtils.h b/src/Storages/VirtualColumnUtils.h
index 73b7908b75c..919513b3b38 100644
--- a/src/Storages/VirtualColumnUtils.h
+++ b/src/Storages/VirtualColumnUtils.h
@@ -33,15 +33,7 @@ void buildSetsForDAG(const ActionsDAG & dag, const ContextPtr & context);
 bool isDeterministicInScopeOfQuery(const ActionsDAG::Node * node);
 
 /// Extract a part of predicate that can be evaluated using only columns from input_names.
-/// When allow_non_deterministic_functions is true then even if the predicate contains non-deterministic
-/// functions, we still allow to extract a part of the predicate, otherwise we return nullptr.
-/// allow_non_deterministic_functions must be false when we are going to use the result to filter parts in
-/// MergeTreeData::totalRowsByPartitionPredicateImp. For example, if the query is
-/// `SELECT count() FROM table  WHERE _partition_id = '0' AND rowNumberInBlock() = 1`
-/// The predicate will be `_partition_id = '0' AND rowNumberInBlock() = 1`, and `rowNumberInBlock()` is
-/// non-deterministic. If we still extract the part `_partition_id = '0'` for filtering parts, then trivial
-/// count optimization will be mistakenly applied to the query.
-std::optional<ActionsDAG> splitFilterDagForAllowedInputs(const ActionsDAG::Node * predicate, const Block * allowed_inputs, bool allow_non_deterministic_functions = true);
+std::optional<ActionsDAG> splitFilterDagForAllowedInputs(const ActionsDAG::Node * predicate, const Block * allowed_inputs);
 
 /// Extract from the input stream a set of `name` column values
 template <typename T>
diff --git a/tests/queries/0_stateless/03203_count_with_non_deterministic_function.reference b/tests/queries/0_stateless/03203_count_with_non_deterministic_function.reference
deleted file mode 100644
index 6ed281c757a..00000000000
--- a/tests/queries/0_stateless/03203_count_with_non_deterministic_function.reference
+++ /dev/null
@@ -1,2 +0,0 @@
-1
-1
diff --git a/tests/queries/0_stateless/03203_count_with_non_deterministic_function.sql b/tests/queries/0_stateless/03203_count_with_non_deterministic_function.sql
deleted file mode 100644
index bb3269da597..00000000000
--- a/tests/queries/0_stateless/03203_count_with_non_deterministic_function.sql
+++ /dev/null
@@ -1,4 +0,0 @@
-CREATE TABLE t (p UInt8, x UInt64) Engine = MergeTree PARTITION BY p ORDER BY x;
-INSERT INTO t SELECT 0, number FROM numbers(10) SETTINGS max_block_size = 100;
-SELECT count() FROM t WHERE p = 0 AND rowNumberInAllBlocks() = 1 SETTINGS allow_experimental_analyzer = 0;
-SELECT count() FROM t WHERE p = 0 AND rowNumberInAllBlocks() = 1 SETTINGS allow_experimental_analyzer = 1;

From 3802b1ed6c7174b0d95bf1c89d339187fe6dc69d Mon Sep 17 00:00:00 2001
From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com>
Date: Mon, 5 Aug 2024 12:42:57 +0200
Subject: [PATCH 1370/1488] Update comment

---
 src/Interpreters/Cache/FileCache.cpp | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp
index 4c17afb79be..aff4e48d01d 100644
--- a/src/Interpreters/Cache/FileCache.cpp
+++ b/src/Interpreters/Cache/FileCache.cpp
@@ -326,6 +326,8 @@ std::vector<FileSegment::Range> FileCache::splitRange(size_t offset, size_t size
     ///                  ^                  ^
     ///                right offset         aligned_right_offset
     /// [_________]                           <-- last cached file segment, e.g. we have uncovered suffix of the requested range
+    ///           ^
+    ///           last_file_segment_right_offset
     /// [________________]
     ///        size
     /// [____________________________________]
@@ -335,8 +337,9 @@ std::vector<FileSegment::Range> FileCache::splitRange(size_t offset, size_t size
     /// and get something like this:
     ///
     /// [________________________]
-    ///                  ^             ^
-    ///              right_offset   right_offset + max_file_segment_size
+    ///          ^               ^ 
+    ///          |               last_file_segment_right_offset + max_file_segment_size
+    ///          last_file_segment_right_offset
     /// e.g. there is no need to create sub-segment for range (right_offset + max_file_segment_size, aligned_right_offset].
     /// Because its left offset would be bigger than right_offset.
     /// Therefore, we set end_pos_non_included as offset+size, but remaining_size as aligned_size.

From 6a0c0e7b1d4922b616d706820d9000ffe8040d63 Mon Sep 17 00:00:00 2001
From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com>
Date: Mon, 5 Aug 2024 12:48:21 +0200
Subject: [PATCH 1371/1488] Update FileCache.cpp

---
 src/Interpreters/Cache/FileCache.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp
index aff4e48d01d..ed91e41db17 100644
--- a/src/Interpreters/Cache/FileCache.cpp
+++ b/src/Interpreters/Cache/FileCache.cpp
@@ -340,7 +340,7 @@ std::vector<FileSegment::Range> FileCache::splitRange(size_t offset, size_t size
     ///          ^               ^ 
     ///          |               last_file_segment_right_offset + max_file_segment_size
     ///          last_file_segment_right_offset
-    /// e.g. there is no need to create sub-segment for range (right_offset + max_file_segment_size, aligned_right_offset].
+    /// e.g. there is no need to create sub-segment for range (last_file_segment_right_offset + max_file_segment_size, aligned_right_offset].
     /// Because its left offset would be bigger than right_offset.
     /// Therefore, we set end_pos_non_included as offset+size, but remaining_size as aligned_size.
 

From 67fe443133c277f78ecf1a11d542c61b1d805a59 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?J=C3=A1nos=20Benjamin=20Antal?=
 <benjamin.antal@clickhouse.com>
Date: Mon, 5 Aug 2024 10:48:47 +0000
Subject: [PATCH 1372/1488] Fix build

---
 src/Storages/MergeTree/MergeTreeData.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index 2286530aa83..49888596fbb 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -1146,7 +1146,7 @@ std::optional<UInt64> MergeTreeData::totalRowsByPartitionPredicateImpl(
     auto metadata_snapshot = getInMemoryMetadataPtr();
     auto virtual_columns_block = getBlockWithVirtualsForFilter(metadata_snapshot, {parts[0]});
 
-    auto filter_dag = VirtualColumnUtils::splitFilterDagForAllowedInputs(filter_actions_dag->getOutputs().at(0), nullptr);
+    auto filter_dag = VirtualColumnUtils::splitFilterDagForAllowedInputs(filter_actions_dag.getOutputs().at(0), nullptr);
     if (!filter_dag)
         return {};
 

From 67bddde6287f21a702d2e134921a9f00073959f6 Mon Sep 17 00:00:00 2001
From: Max K <maxkaynov@gmail.com>
Date: Sat, 3 Aug 2024 21:26:05 +0200
Subject: [PATCH 1373/1488] move Check Descriptions to commit_status_helper

---
 .yamllint                        |   6 -
 tests/ci/ci_config.py            |   2 -
 tests/ci/ci_definitions.py       | 187 +---------------------------
 tests/ci/commit_status_helper.py | 203 +++++++++++++++++++++++++++++--
 4 files changed, 195 insertions(+), 203 deletions(-)

diff --git a/.yamllint b/.yamllint
index 7fb741ec9f4..b8f7c93e246 100644
--- a/.yamllint
+++ b/.yamllint
@@ -5,12 +5,6 @@ rules:
     indentation:
         level: warning
         indent-sequences: consistent
-    line-length:
-        # there are:
-        # - bash -c "", so this is OK
-        # - yaml in tests
-        max: 1000
-        level: warning
     comments:
         min-spaces-from-content: 1
     document-start: disable
diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py
index ef48466e451..8cb587a1062 100644
--- a/tests/ci/ci_config.py
+++ b/tests/ci/ci_config.py
@@ -20,12 +20,10 @@ class CI:
     from ci_definitions import BuildConfig as BuildConfig
     from ci_definitions import DigestConfig as DigestConfig
     from ci_definitions import JobConfig as JobConfig
-    from ci_definitions import CheckDescription as CheckDescription
     from ci_definitions import Tags as Tags
     from ci_definitions import JobNames as JobNames
     from ci_definitions import BuildNames as BuildNames
     from ci_definitions import StatusNames as StatusNames
-    from ci_definitions import CHECK_DESCRIPTIONS as CHECK_DESCRIPTIONS
     from ci_definitions import REQUIRED_CHECKS as REQUIRED_CHECKS
     from ci_definitions import SyncState as SyncState
     from ci_definitions import MQ_JOBS as MQ_JOBS
diff --git a/tests/ci/ci_definitions.py b/tests/ci/ci_definitions.py
index 795bda3d4b0..48847b0d7a6 100644
--- a/tests/ci/ci_definitions.py
+++ b/tests/ci/ci_definitions.py
@@ -1,7 +1,7 @@
 import copy
 from dataclasses import dataclass, field
 from pathlib import Path
-from typing import Callable, List, Union, Iterable, Optional, Literal, Any
+from typing import List, Union, Iterable, Optional, Literal, Any
 
 from ci_utils import WithIter
 from integration_test_images import IMAGES
@@ -646,188 +646,3 @@ MQ_JOBS = [
     BuildNames.BINARY_RELEASE,
     JobNames.UNIT_TEST,
 ]
-
-
-@dataclass
-class CheckDescription:
-    name: str
-    description: str  # the check descriptions, will be put into the status table
-    match_func: Callable[[str], bool]  # the function to check vs the commit status
-
-    def __hash__(self) -> int:
-        return hash(self.name + self.description)
-
-
-CHECK_DESCRIPTIONS = [
-    CheckDescription(
-        StatusNames.PR_CHECK,
-        "Checks correctness of the PR's body",
-        lambda x: x == "PR Check",
-    ),
-    CheckDescription(
-        StatusNames.SYNC,
-        "If it fails, ask a maintainer for help",
-        lambda x: x == StatusNames.SYNC,
-    ),
-    CheckDescription(
-        "AST fuzzer",
-        "Runs randomly generated queries to catch program errors. "
-        "The build type is optionally given in parenthesis. "
-        "If it fails, ask a maintainer for help",
-        lambda x: x.startswith("AST fuzzer"),
-    ),
-    CheckDescription(
-        JobNames.BUGFIX_VALIDATE,
-        "Checks that either a new test (functional or integration) or there "
-        "some changed tests that fail with the binary built on master branch",
-        lambda x: x == JobNames.BUGFIX_VALIDATE,
-    ),
-    CheckDescription(
-        StatusNames.CI,
-        "A meta-check that indicates the running CI. Normally, it's in <b>success</b> or "
-        "<b>pending</b> state. The failed status indicates some problems with the PR",
-        lambda x: x == "CI running",
-    ),
-    CheckDescription(
-        "Builds",
-        "Builds ClickHouse in various configurations for use in further steps. "
-        "You have to fix the builds that fail. Build logs often has enough "
-        "information to fix the error, but you might have to reproduce the failure "
-        "locally. The <b>cmake</b> options can be found in the build log, grepping for "
-        '<b>cmake</b>. Use these options and follow the <a href="'
-        'https://clickhouse.com/docs/en/development/build">general build process</a>',
-        lambda x: x.startswith("ClickHouse") and x.endswith("build check"),
-    ),
-    CheckDescription(
-        "Compatibility check",
-        "Checks that <b>clickhouse</b> binary runs on distributions with old libc "
-        "versions. If it fails, ask a maintainer for help",
-        lambda x: x.startswith("Compatibility check"),
-    ),
-    CheckDescription(
-        JobNames.DOCKER_SERVER,
-        "The check to build and optionally push the mentioned image to docker hub",
-        lambda x: x.startswith("Docker server"),
-    ),
-    CheckDescription(
-        JobNames.DOCKER_KEEPER,
-        "The check to build and optionally push the mentioned image to docker hub",
-        lambda x: x.startswith("Docker keeper"),
-    ),
-    CheckDescription(
-        JobNames.DOCS_CHECK,
-        "Builds and tests the documentation",
-        lambda x: x == JobNames.DOCS_CHECK,
-    ),
-    CheckDescription(
-        JobNames.FAST_TEST,
-        "Normally this is the first check that is ran for a PR. It builds ClickHouse "
-        'and runs most of <a href="https://clickhouse.com/docs/en/development/tests'
-        '#functional-tests">stateless functional tests</a>, '
-        "omitting some. If it fails, further checks are not started until it is fixed. "
-        "Look at the report to see which tests fail, then reproduce the failure "
-        'locally as described <a href="https://clickhouse.com/docs/en/development/'
-        'tests#functional-test-locally">here</a>',
-        lambda x: x == JobNames.FAST_TEST,
-    ),
-    CheckDescription(
-        "Flaky tests",
-        "Checks if new added or modified tests are flaky by running them repeatedly, "
-        "in parallel, with more randomization. Functional tests are run 100 times "
-        "with address sanitizer, and additional randomization of thread scheduling. "
-        "Integration tests are run up to 10 times. If at least once a new test has "
-        "failed, or was too long, this check will be red. We don't allow flaky tests, "
-        'read <a href="https://clickhouse.com/blog/decorating-a-christmas-tree-with-'
-        'the-help-of-flaky-tests/">the doc</a>',
-        lambda x: "tests flaky check" in x,
-    ),
-    CheckDescription(
-        "Install packages",
-        "Checks that the built packages are installable in a clear environment",
-        lambda x: x.startswith("Install packages ("),
-    ),
-    CheckDescription(
-        "Integration tests",
-        "The integration tests report. In parenthesis the package type is given, "
-        "and in square brackets are the optional part/total tests",
-        lambda x: x.startswith("Integration tests ("),
-    ),
-    CheckDescription(
-        StatusNames.MERGEABLE,
-        "Checks if all other necessary checks are successful",
-        lambda x: x == StatusNames.MERGEABLE,
-    ),
-    CheckDescription(
-        "Performance Comparison",
-        "Measure changes in query performance. The performance test report is "
-        'described in detail <a href="https://github.com/ClickHouse/ClickHouse/tree'
-        '/master/docker/test/performance-comparison#how-to-read-the-report">here</a>. '
-        "In square brackets are the optional part/total tests",
-        lambda x: x.startswith("Performance Comparison"),
-    ),
-    CheckDescription(
-        "Push to Dockerhub",
-        "The check for building and pushing the CI related docker images to docker hub",
-        lambda x: x.startswith("Push") and "to Dockerhub" in x,
-    ),
-    CheckDescription(
-        "Sqllogic",
-        "Run clickhouse on the "
-        '<a href="https://www.sqlite.org/sqllogictest">sqllogic</a> '
-        "test set against sqlite and checks that all statements are passed",
-        lambda x: x.startswith("Sqllogic test"),
-    ),
-    CheckDescription(
-        "SQLancer",
-        "Fuzzing tests that detect logical bugs with "
-        '<a href="https://github.com/sqlancer/sqlancer">SQLancer</a> tool',
-        lambda x: x.startswith("SQLancer"),
-    ),
-    CheckDescription(
-        "Stateful tests",
-        "Runs stateful functional tests for ClickHouse binaries built in various "
-        "configurations -- release, debug, with sanitizers, etc",
-        lambda x: x.startswith("Stateful tests ("),
-    ),
-    CheckDescription(
-        "Stateless tests",
-        "Runs stateless functional tests for ClickHouse binaries built in various "
-        "configurations -- release, debug, with sanitizers, etc",
-        lambda x: x.startswith("Stateless tests ("),
-    ),
-    CheckDescription(
-        "Stress test",
-        "Runs stateless functional tests concurrently from several clients to detect "
-        "concurrency-related errors",
-        lambda x: x.startswith("Stress test ("),
-    ),
-    CheckDescription(
-        JobNames.STYLE_CHECK,
-        "Runs a set of checks to keep the code style clean. If some of tests failed, "
-        "see the related log from the report",
-        lambda x: x == JobNames.STYLE_CHECK,
-    ),
-    CheckDescription(
-        "Unit tests",
-        "Runs the unit tests for different release types",
-        lambda x: x.startswith("Unit tests ("),
-    ),
-    CheckDescription(
-        "Upgrade check",
-        "Runs stress tests on server version from last release and then tries to "
-        "upgrade it to the version from the PR. It checks if the new server can "
-        "successfully startup without any errors, crashes or sanitizer asserts",
-        lambda x: x.startswith("Upgrade check ("),
-    ),
-    CheckDescription(
-        "ClickBench",
-        "Runs [ClickBench](https://github.com/ClickHouse/ClickBench/) with instant-attach table",
-        lambda x: x.startswith("ClickBench"),
-    ),
-    CheckDescription(
-        "Fallback for unknown",
-        "There's no description for the check yet, please add it to "
-        "tests/ci/ci_config.py:CHECK_DESCRIPTIONS",
-        lambda x: True,
-    ),
-]
diff --git a/tests/ci/commit_status_helper.py b/tests/ci/commit_status_helper.py
index fdc9c002b66..8967d453622 100644
--- a/tests/ci/commit_status_helper.py
+++ b/tests/ci/commit_status_helper.py
@@ -7,7 +7,7 @@ import time
 from collections import defaultdict
 from dataclasses import asdict, dataclass
 from pathlib import Path
-from typing import Dict, List, Optional, Union
+from typing import Dict, List, Optional, Union, Callable
 
 from github import Github
 from github.Commit import Commit
@@ -176,7 +176,7 @@ def set_status_comment(commit: Commit, pr_info: PRInfo) -> None:
 
     if not [status for status in statuses if status.context == CI.StatusNames.CI]:
         # This is the case, when some statuses already exist for the check,
-        # but not the StatusNames.CI. We should create it as pending.
+        # but not the CI.StatusNames.CI. We should create it as pending.
         # W/o pr_info to avoid recursion, and yes, one extra create_ci_report
         post_commit_status(
             commit,
@@ -226,20 +226,20 @@ def generate_status_comment(pr_info: PRInfo, statuses: CommitStatuses) -> str:
         f"\n"
     )
     # group checks by the name to get the worst one per each
-    grouped_statuses = {}  # type: Dict[CI.CheckDescription, CommitStatuses]
+    grouped_statuses = {}  # type: Dict[CheckDescription, CommitStatuses]
     for status in statuses:
         cd = None
-        for c in CI.CHECK_DESCRIPTIONS:
+        for c in CHECK_DESCRIPTIONS:
             if c.match_func(status.context):
                 cd = c
                 break
 
-        if cd is None or cd == CI.CHECK_DESCRIPTIONS[-1]:
+        if cd is None or cd == CHECK_DESCRIPTIONS[-1]:
             # This is the case for either non-found description or a fallback
-            cd = CI.CheckDescription(
+            cd = CheckDescription(
                 status.context,
-                CI.CHECK_DESCRIPTIONS[-1].description,
-                CI.CHECK_DESCRIPTIONS[-1].match_func,
+                CHECK_DESCRIPTIONS[-1].description,
+                CHECK_DESCRIPTIONS[-1].match_func,
             )
 
         if cd in grouped_statuses:
@@ -459,7 +459,7 @@ def trigger_mergeable_check(
     set_from_sync: bool = False,
     workflow_failed: bool = False,
 ) -> StatusType:
-    """calculate and update StatusNames.MERGEABLE"""
+    """calculate and update CI.StatusNames.MERGEABLE"""
     required_checks = [status for status in statuses if CI.is_required(status.context)]
 
     mergeable_status = None
@@ -536,3 +536,188 @@ def update_upstream_sync_status(
         get_commit_filtered_statuses(last_synced_upstream_commit),
         set_from_sync=True,
     )
+
+
+@dataclass
+class CheckDescription:
+    name: str
+    description: str  # the check descriptions, will be put into the status table
+    match_func: Callable[[str], bool]  # the function to check vs the commit status
+
+    def __hash__(self) -> int:
+        return hash(self.name + self.description)
+
+
+CHECK_DESCRIPTIONS = [
+    CheckDescription(
+        CI.StatusNames.PR_CHECK,
+        "Checks correctness of the PR's body",
+        lambda x: x == "PR Check",
+    ),
+    CheckDescription(
+        CI.StatusNames.SYNC,
+        "If it fails, ask a maintainer for help",
+        lambda x: x == CI.StatusNames.SYNC,
+    ),
+    CheckDescription(
+        "AST fuzzer",
+        "Runs randomly generated queries to catch program errors. "
+        "The build type is optionally given in parenthesis. "
+        "If it fails, ask a maintainer for help",
+        lambda x: x.startswith("AST fuzzer"),
+    ),
+    CheckDescription(
+        CI.JobNames.BUGFIX_VALIDATE,
+        "Checks that either a new test (functional or integration) or there "
+        "some changed tests that fail with the binary built on master branch",
+        lambda x: x == CI.JobNames.BUGFIX_VALIDATE,
+    ),
+    CheckDescription(
+        CI.StatusNames.CI,
+        "A meta-check that indicates the running CI. Normally, it's in <b>success</b> or "
+        "<b>pending</b> state. The failed status indicates some problems with the PR",
+        lambda x: x == "CI running",
+    ),
+    CheckDescription(
+        "Builds",
+        "Builds ClickHouse in various configurations for use in further steps. "
+        "You have to fix the builds that fail. Build logs often has enough "
+        "information to fix the error, but you might have to reproduce the failure "
+        "locally. The <b>cmake</b> options can be found in the build log, grepping for "
+        '<b>cmake</b>. Use these options and follow the <a href="'
+        'https://clickhouse.com/docs/en/development/build">general build process</a>',
+        lambda x: x.startswith("ClickHouse") and x.endswith("build check"),
+    ),
+    CheckDescription(
+        "Compatibility check",
+        "Checks that <b>clickhouse</b> binary runs on distributions with old libc "
+        "versions. If it fails, ask a maintainer for help",
+        lambda x: x.startswith("Compatibility check"),
+    ),
+    CheckDescription(
+        CI.JobNames.DOCKER_SERVER,
+        "The check to build and optionally push the mentioned image to docker hub",
+        lambda x: x.startswith("Docker server"),
+    ),
+    CheckDescription(
+        CI.JobNames.DOCKER_KEEPER,
+        "The check to build and optionally push the mentioned image to docker hub",
+        lambda x: x.startswith("Docker keeper"),
+    ),
+    CheckDescription(
+        CI.JobNames.DOCS_CHECK,
+        "Builds and tests the documentation",
+        lambda x: x == CI.JobNames.DOCS_CHECK,
+    ),
+    CheckDescription(
+        CI.JobNames.FAST_TEST,
+        "Normally this is the first check that is ran for a PR. It builds ClickHouse "
+        'and runs most of <a href="https://clickhouse.com/docs/en/development/tests'
+        '#functional-tests">stateless functional tests</a>, '
+        "omitting some. If it fails, further checks are not started until it is fixed. "
+        "Look at the report to see which tests fail, then reproduce the failure "
+        'locally as described <a href="https://clickhouse.com/docs/en/development/'
+        'tests#functional-test-locally">here</a>',
+        lambda x: x == CI.JobNames.FAST_TEST,
+    ),
+    CheckDescription(
+        "Flaky tests",
+        "Checks if new added or modified tests are flaky by running them repeatedly, "
+        "in parallel, with more randomization. Functional tests are run 100 times "
+        "with address sanitizer, and additional randomization of thread scheduling. "
+        "Integration tests are run up to 10 times. If at least once a new test has "
+        "failed, or was too long, this check will be red. We don't allow flaky tests, "
+        'read <a href="https://clickhouse.com/blog/decorating-a-christmas-tree-with-'
+        'the-help-of-flaky-tests/">the doc</a>',
+        lambda x: "tests flaky check" in x,
+    ),
+    CheckDescription(
+        "Install packages",
+        "Checks that the built packages are installable in a clear environment",
+        lambda x: x.startswith("Install packages ("),
+    ),
+    CheckDescription(
+        "Integration tests",
+        "The integration tests report. In parenthesis the package type is given, "
+        "and in square brackets are the optional part/total tests",
+        lambda x: x.startswith("Integration tests ("),
+    ),
+    CheckDescription(
+        CI.StatusNames.MERGEABLE,
+        "Checks if all other necessary checks are successful",
+        lambda x: x == CI.StatusNames.MERGEABLE,
+    ),
+    CheckDescription(
+        "Performance Comparison",
+        "Measure changes in query performance. The performance test report is "
+        'described in detail <a href="https://github.com/ClickHouse/ClickHouse/tree'
+        '/master/docker/test/performance-comparison#how-to-read-the-report">here</a>. '
+        "In square brackets are the optional part/total tests",
+        lambda x: x.startswith("Performance Comparison"),
+    ),
+    CheckDescription(
+        "Push to Dockerhub",
+        "The check for building and pushing the CI related docker images to docker hub",
+        lambda x: x.startswith("Push") and "to Dockerhub" in x,
+    ),
+    CheckDescription(
+        "Sqllogic",
+        "Run clickhouse on the "
+        '<a href="https://www.sqlite.org/sqllogictest">sqllogic</a> '
+        "test set against sqlite and checks that all statements are passed",
+        lambda x: x.startswith("Sqllogic test"),
+    ),
+    CheckDescription(
+        "SQLancer",
+        "Fuzzing tests that detect logical bugs with "
+        '<a href="https://github.com/sqlancer/sqlancer">SQLancer</a> tool',
+        lambda x: x.startswith("SQLancer"),
+    ),
+    CheckDescription(
+        "Stateful tests",
+        "Runs stateful functional tests for ClickHouse binaries built in various "
+        "configurations -- release, debug, with sanitizers, etc",
+        lambda x: x.startswith("Stateful tests ("),
+    ),
+    CheckDescription(
+        "Stateless tests",
+        "Runs stateless functional tests for ClickHouse binaries built in various "
+        "configurations -- release, debug, with sanitizers, etc",
+        lambda x: x.startswith("Stateless tests ("),
+    ),
+    CheckDescription(
+        "Stress test",
+        "Runs stateless functional tests concurrently from several clients to detect "
+        "concurrency-related errors",
+        lambda x: x.startswith("Stress test ("),
+    ),
+    CheckDescription(
+        CI.JobNames.STYLE_CHECK,
+        "Runs a set of checks to keep the code style clean. If some of tests failed, "
+        "see the related log from the report",
+        lambda x: x == CI.JobNames.STYLE_CHECK,
+    ),
+    CheckDescription(
+        "Unit tests",
+        "Runs the unit tests for different release types",
+        lambda x: x.startswith("Unit tests ("),
+    ),
+    CheckDescription(
+        "Upgrade check",
+        "Runs stress tests on server version from last release and then tries to "
+        "upgrade it to the version from the PR. It checks if the new server can "
+        "successfully startup without any errors, crashes or sanitizer asserts",
+        lambda x: x.startswith("Upgrade check ("),
+    ),
+    CheckDescription(
+        "ClickBench",
+        "Runs [ClickBench](https://github.com/ClickHouse/ClickBench/) with instant-attach table",
+        lambda x: x.startswith("ClickBench"),
+    ),
+    CheckDescription(
+        "Fallback for unknown",
+        "There's no description for the check yet, please add it to "
+        "tests/ci/ci_config.py:CHECK_DESCRIPTIONS",
+        lambda x: True,
+    ),
+]

From 2d92cd71a83ed07e6f22c2ba37d39926837a9df0 Mon Sep 17 00:00:00 2001
From: Anton Popov <anton@clickhouse.com>
Date: Mon, 5 Aug 2024 09:36:09 +0000
Subject: [PATCH 1374/1488] add some comments

---
 src/Interpreters/inplaceBlockConversions.cpp | 9 +++++++--
 src/Storages/MergeTree/IMergeTreeReader.cpp  | 1 +
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/src/Interpreters/inplaceBlockConversions.cpp b/src/Interpreters/inplaceBlockConversions.cpp
index 429d467ffbf..68254768a7d 100644
--- a/src/Interpreters/inplaceBlockConversions.cpp
+++ b/src/Interpreters/inplaceBlockConversions.cpp
@@ -309,6 +309,7 @@ static bool hasDefault(const StorageMetadataPtr & metadata_snapshot, const NameA
 
 static String removeTupleElementsFromSubcolumn(String subcolumn_name, const Names & tuple_elements)
 {
+    /// Add a dot to the end of name for convenience.
     subcolumn_name += ".";
     for (const auto & elem : tuple_elements)
     {
@@ -396,19 +397,23 @@ void fillMissingColumns(
             Names tuple_elements;
             auto serialization = IDataType::getSerialization(*requested_column);
 
+            /// For Nested columns collect names of tuple elements and skip them while getting the base type of array.
             IDataType::forEachSubcolumn([&](const auto & path, const auto &, const auto &)
             {
                 if (path.back().type == ISerialization::Substream::TupleElement)
                     tuple_elements.push_back(path.back().name_of_substream);
             }, ISerialization::SubstreamData(serialization));
 
+            /// The number of dimensions that belongs to the array itself but not shared in Nested column.
+            /// For example for column "n Nested(a UInt64, b Array(UInt64))" this value is 0 for `n.a` and 1 for `n.b`.
             size_t num_empty_dimensions = num_dimensions - current_offsets.size();
+
             auto base_type = getBaseTypeOfArray(requested_column->getTypeInStorage(), tuple_elements);
             auto scalar_type = createArrayOfType(base_type, num_empty_dimensions);
-
             size_t data_size = assert_cast<const ColumnUInt64 &>(*current_offsets.back()).getData().back();
-            auto subcolumn_name = removeTupleElementsFromSubcolumn(requested_column->getSubcolumnName(), tuple_elements);
 
+            /// Remove names of tuple elements because they are already processed by 'getBaseTypeOfArray'.
+            auto subcolumn_name = removeTupleElementsFromSubcolumn(requested_column->getSubcolumnName(), tuple_elements);
             res_columns[i] = createColumnWithDefaultValue(*scalar_type, subcolumn_name, data_size);
 
             for (auto it = current_offsets.rbegin(); it != current_offsets.rend(); ++it)
diff --git a/src/Storages/MergeTree/IMergeTreeReader.cpp b/src/Storages/MergeTree/IMergeTreeReader.cpp
index 05a0b5a7dbc..e0b2710c61f 100644
--- a/src/Storages/MergeTree/IMergeTreeReader.cpp
+++ b/src/Storages/MergeTree/IMergeTreeReader.cpp
@@ -150,6 +150,7 @@ void IMergeTreeReader::evaluateMissingDefaults(Block additional_columns, Columns
         NamesAndTypesList full_requested_columns;
 
         /// Convert columns list to block. And convert subcolumns to full columns.
+        /// Defaults should be executed on full columns to get correct values for subcolumns.
         /// TODO: rewrite with columns interface. It will be possible after changes in ExpressionActions.
 
         auto it = original_requested_columns.begin();

From 3244002cae58da99c8b088888376c0fda7f3f1f4 Mon Sep 17 00:00:00 2001
From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com>
Date: Mon, 5 Aug 2024 13:22:10 +0200
Subject: [PATCH 1375/1488] Update FileCache.cpp

---
 src/Interpreters/Cache/FileCache.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp
index ed91e41db17..1a15efa7cf8 100644
--- a/src/Interpreters/Cache/FileCache.cpp
+++ b/src/Interpreters/Cache/FileCache.cpp
@@ -560,7 +560,7 @@ FileCache::getOrSet(
 
     FileSegment::Range initial_range(offset, offset + size - 1);
     /// result_range is initial range, which will be adjusted according to
-    /// 1. aligned offset, alighed_end_offset
+    /// 1. aligned_offset, aligned_end_offset
     /// 2. max_file_segments_limit
     FileSegment::Range result_range = initial_range;
 

From a296beb39084492a6825879ff5258f4377ead75b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Mon, 5 Aug 2024 13:31:53 +0200
Subject: [PATCH 1376/1488] Unit test: Mark as FAILURE if retcode != 0

---
 tests/ci/unit_tests_check.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tests/ci/unit_tests_check.py b/tests/ci/unit_tests_check.py
index 6430fa78801..9cc8ec379bf 100644
--- a/tests/ci/unit_tests_check.py
+++ b/tests/ci/unit_tests_check.py
@@ -197,6 +197,11 @@ def main():
     subprocess.check_call(f"sudo chown -R ubuntu:ubuntu {TEMP_PATH}", shell=True)
 
     state, description, test_results = process_results(test_output)
+    if retcode != 0 and state == SUCCESS:
+        # The process might have failed without reporting it in the test_output (e.g. LeakSanitizer)
+        state = FAILURE
+        description = "Invalid return code. Check run.log"
+
     additional_files = [run_log_path] + [
         p for p in test_output.iterdir() if not p.is_dir()
     ]

From 07cfcdeaaec0289d78b033260dd657bebad70674 Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Mon, 5 Aug 2024 11:31:46 +0000
Subject: [PATCH 1377/1488] Fix filter pushdown for aggregation without keys

---
 .../Optimizations/filterPushDown.cpp          |  9 +++++++
 .../03217_fliter_pushdown_no_keys.reference   |  6 +++++
 .../03217_fliter_pushdown_no_keys.sql         | 26 +++++++++++++++++++
 3 files changed, 41 insertions(+)
 create mode 100644 tests/queries/0_stateless/03217_fliter_pushdown_no_keys.reference
 create mode 100644 tests/queries/0_stateless/03217_fliter_pushdown_no_keys.sql

diff --git a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp
index 73314f005b6..b71326ff75b 100644
--- a/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp
+++ b/src/Processors/QueryPlan/Optimizations/filterPushDown.cpp
@@ -442,6 +442,15 @@ size_t tryPushDownFilter(QueryPlan::Node * parent_node, QueryPlan::Nodes & nodes
 
         const auto & params = aggregating->getParams();
         const auto & keys = params.keys;
+        /** The filter is applied either to aggregation keys or aggregation result
+          * (columns under aggregation is not available in outer scope, so we can't have a filter for them).
+          * The filter for the aggregation result is not pushed down, so the only valid case is filtering aggregation keys.
+          * In case keys are empty, do not push down the filter.
+          * Also with empty keys we can have an issue with `empty_result_for_aggregation_by_empty_set`,
+          * since we can gen a result row when everything is filtered.
+          */
+        if (keys.empty())
+            return 0;
 
         const bool filter_column_is_not_among_aggregation_keys
             = std::find(keys.begin(), keys.end(), filter->getFilterColumnName()) == keys.end();
diff --git a/tests/queries/0_stateless/03217_fliter_pushdown_no_keys.reference b/tests/queries/0_stateless/03217_fliter_pushdown_no_keys.reference
new file mode 100644
index 00000000000..9838dd1b936
--- /dev/null
+++ b/tests/queries/0_stateless/03217_fliter_pushdown_no_keys.reference
@@ -0,0 +1,6 @@
+---
+1	1
+---
+3	3
+---
+---
diff --git a/tests/queries/0_stateless/03217_fliter_pushdown_no_keys.sql b/tests/queries/0_stateless/03217_fliter_pushdown_no_keys.sql
new file mode 100644
index 00000000000..cb8bf59e790
--- /dev/null
+++ b/tests/queries/0_stateless/03217_fliter_pushdown_no_keys.sql
@@ -0,0 +1,26 @@
+
+
+
+select * from ( select sum(last_seen) as dates_seen, materialize(1) as last_seen ) where last_seen > 2;
+select * from ( select sum(last_seen) as dates_seen, materialize(2) as last_seen ) where last_seen < 2;
+select * from ( select sum(last_seen) as dates_seen, materialize(2) as last_seen GROUP BY 'a' ) where last_seen < 2;
+
+select '---';
+select * from ( select sum(last_seen) as dates_seen, 1 as last_seen UNION ALL select sum(last_seen) as dates_seen, 3 as last_seen ) where last_seen < 2;
+
+select '---';
+select * from ( select sum(last_seen) as dates_seen, 1 as last_seen UNION ALL select sum(last_seen) as dates_seen, 3 as last_seen ) where last_seen > 2;
+
+select '---';
+with activity as (
+  select
+    groupUniqArrayState(toDate('2025-01-01 01:00:00')) as dates_seen,
+    toDateTime('2025-01-01 01:00:00') as last_seen
+  union all
+  select
+    groupUniqArrayState(toDate('2023-11-11 11:11:11')) as dates_seen,
+    toDateTime('2023-11-11 11:11:11') as last_seen
+)
+select last_seen from activity
+where last_seen < toDateTime('2020-01-01 00:00:00');
+select '---';

From 23190c30cf696075017f09a4997a7969c1d2f651 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 5 Aug 2024 14:32:08 +0200
Subject: [PATCH 1378/1488] Fix bad test `03032_redundant_equals`

---
 tests/queries/0_stateless/03032_redundant_equals.sql | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/queries/0_stateless/03032_redundant_equals.sql b/tests/queries/0_stateless/03032_redundant_equals.sql
index de85ec5cf00..eee2be4ebf0 100644
--- a/tests/queries/0_stateless/03032_redundant_equals.sql
+++ b/tests/queries/0_stateless/03032_redundant_equals.sql
@@ -5,9 +5,9 @@ CREATE TABLE test_table
     k UInt64,
 )
 ENGINE = MergeTree
-ORDER BY k;
+ORDER BY k SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';
 
-INSERT INTO test_table SELECT number FROM numbers(10000000);
+INSERT INTO test_table SELECT number FROM numbers(100000);
 
 SET allow_experimental_analyzer = 1;
 
@@ -25,7 +25,7 @@ SELECT * FROM test_table WHERE (NOT ((k not in (100) = 0) OR (k in (100) = 1)))
 SELECT * FROM test_table WHERE (NOT ((k in (101) = 0) OR (k in (100) = 1))) = 1;
 SELECT * FROM test_table WHERE ((k not in (101) = 0) OR (k in (100) = 1)) = 1;
 SELECT * FROM test_table WHERE ((k not in (99) = 1) AND (k in (100) = 1)) = 1;
--- we skip optimizing queries with toNullable(0 or 1) but lets make sure they still work 
+-- we skip optimizing queries with toNullable(0 or 1) but lets make sure they still work
 SELECT * FROM test_table WHERE (k = 101) = toLowCardinality(toNullable(1));
 SELECT * FROM test_table WHERE (k = 101) = toNullable(1);
 SELECT * FROM test_table WHERE (k = 101) = toLowCardinality(1);

From ff0b8889ab8bf347343e18b4e9a407b3cab8264b Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 5 Aug 2024 14:36:24 +0200
Subject: [PATCH 1379/1488] Update
 02789_reading_from_s3_with_connection_pool.sh

---
 .../0_stateless/02789_reading_from_s3_with_connection_pool.sh   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/02789_reading_from_s3_with_connection_pool.sh b/tests/queries/0_stateless/02789_reading_from_s3_with_connection_pool.sh
index 39399842db1..5a37d51233d 100755
--- a/tests/queries/0_stateless/02789_reading_from_s3_with_connection_pool.sh
+++ b/tests/queries/0_stateless/02789_reading_from_s3_with_connection_pool.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Tags: no-fasttest, no-random-settings, no-replicated-database
+# Tags: no-fasttest, no-random-settings, no-replicated-database, no-distributed-cache
 
 CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh

From c21b97672c86a18b693c8c60335271248c279fdd Mon Sep 17 00:00:00 2001
From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com>
Date: Mon, 5 Aug 2024 14:36:54 +0200
Subject: [PATCH 1380/1488] Fix trailing whitespace

---
 src/Interpreters/Cache/FileCache.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Interpreters/Cache/FileCache.cpp b/src/Interpreters/Cache/FileCache.cpp
index 1a15efa7cf8..217ae614c22 100644
--- a/src/Interpreters/Cache/FileCache.cpp
+++ b/src/Interpreters/Cache/FileCache.cpp
@@ -337,7 +337,7 @@ std::vector<FileSegment::Range> FileCache::splitRange(size_t offset, size_t size
     /// and get something like this:
     ///
     /// [________________________]
-    ///          ^               ^ 
+    ///          ^               ^
     ///          |               last_file_segment_right_offset + max_file_segment_size
     ///          last_file_segment_right_offset
     /// e.g. there is no need to create sub-segment for range (last_file_segment_right_offset + max_file_segment_size, aligned_right_offset].

From 8393f7f54737c0ff5c0845e9481784b5d3764bfc Mon Sep 17 00:00:00 2001
From: Nikita Taranov <nikita.taranov@clickhouse.com>
Date: Fri, 2 Aug 2024 16:26:14 +0100
Subject: [PATCH 1381/1488] impl

---
 .../0_stateless/02313_filesystem_cache_seeks.sh        | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/tests/queries/0_stateless/02313_filesystem_cache_seeks.sh b/tests/queries/0_stateless/02313_filesystem_cache_seeks.sh
index b54e3d7f805..fc91f3f1448 100755
--- a/tests/queries/0_stateless/02313_filesystem_cache_seeks.sh
+++ b/tests/queries/0_stateless/02313_filesystem_cache_seeks.sh
@@ -8,13 +8,17 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CUR_DIR"/../shell_config.sh
 
 
+client_opts=(
+  --distributed_ddl_output_mode  'null_status_on_timeout'
+)
+
 for STORAGE_POLICY in 's3_cache' 'local_cache' 's3_cache_multi' 'azure_cache'; do
     echo "Using storage policy: $STORAGE_POLICY"
     $CLICKHOUSE_CLIENT --query "SYSTEM DROP FILESYSTEM CACHE"
 
-    $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS test_02313"
+    $CLICKHOUSE_CLIENT "${client_opts[@]}" --query "DROP TABLE IF EXISTS test_02313"
 
-    $CLICKHOUSE_CLIENT --query "CREATE TABLE test_02313 (id Int32, val String)
+    $CLICKHOUSE_CLIENT "${client_opts[@]}" --query "CREATE TABLE test_02313 (id Int32, val String)
     ENGINE = MergeTree()
     ORDER BY tuple()
     SETTINGS storage_policy = '$STORAGE_POLICY'"
@@ -32,6 +36,6 @@ for STORAGE_POLICY in 's3_cache' 'local_cache' 's3_cache_multi' 'azure_cache'; d
     $CLICKHOUSE_CLIENT --query "SELECT * FROM test_02313 WHERE val LIKE concat('%', randomPrintableASCII(3), '%') FORMAT Null"
     $CLICKHOUSE_CLIENT --query "SELECT * FROM test_02313 WHERE val LIKE concat('%', randomPrintableASCII(3), '%') FORMAT Null"
 
-    $CLICKHOUSE_CLIENT --query "DROP TABLE test_02313"
+    $CLICKHOUSE_CLIENT "${client_opts[@]}" --query "DROP TABLE test_02313"
 
 done

From 19820a65f461839803a91a081a7b7be7cab80c5e Mon Sep 17 00:00:00 2001
From: kssenii <sumarokovakseniia@mail.ru>
Date: Mon, 5 Aug 2024 15:38:15 +0200
Subject: [PATCH 1382/1488] Update tags

---
 tests/queries/0_stateless/02995_index_1.sh  | 4 ++--
 tests/queries/0_stateless/02995_index_10.sh | 4 ++--
 tests/queries/0_stateless/02995_index_5.sh  | 4 ++--
 tests/queries/0_stateless/02995_index_6.sh  | 2 +-
 4 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/tests/queries/0_stateless/02995_index_1.sh b/tests/queries/0_stateless/02995_index_1.sh
index 128697fd0fe..76be6341d53 100755
--- a/tests/queries/0_stateless/02995_index_1.sh
+++ b/tests/queries/0_stateless/02995_index_1.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Tags: long, no-debug, no-asan, no-tsan, no-msan, no-ubsan, no-sanitize-coverage
+# Tags: long, no-debug, no-asan, no-tsan, no-msan, no-ubsan, no-sanitize-coverage, no-distributed-cache
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
@@ -39,6 +39,6 @@ WHERE a >= (round(pow(sipHash64(1, try), 1 / (3 + sipHash64(2, try) % 8))) AS a1
   AND c <= (c1 + round(pow(sipHash64(11, try), 1 / (3 + sipHash64(12, try) % 8))))::String
 HAVING count() > 0;
 "
-done | ${CLICKHOUSE_CLIENT} 
+done | ${CLICKHOUSE_CLIENT}
 
 ${CLICKHOUSE_CLIENT} "DROP TABLE test"
diff --git a/tests/queries/0_stateless/02995_index_10.sh b/tests/queries/0_stateless/02995_index_10.sh
index c15ba00fd05..813cc49cbd8 100755
--- a/tests/queries/0_stateless/02995_index_10.sh
+++ b/tests/queries/0_stateless/02995_index_10.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Tags: long, no-debug, no-asan, no-tsan, no-msan, no-ubsan, no-sanitize-coverage
+# Tags: long, no-debug, no-asan, no-tsan, no-msan, no-ubsan, no-sanitize-coverage, no-distributed-cache
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
@@ -39,6 +39,6 @@ WHERE a >= (round(pow(sipHash64(1, try), 1 / (3 + sipHash64(2, try) % 8))) AS a1
   AND c <= (c1 + round(pow(sipHash64(11, try), 1 / (3 + sipHash64(12, try) % 8))))::String
 HAVING count() > 0;
 "
-done | ${CLICKHOUSE_CLIENT} 
+done | ${CLICKHOUSE_CLIENT}
 
 ${CLICKHOUSE_CLIENT} "DROP TABLE test"
diff --git a/tests/queries/0_stateless/02995_index_5.sh b/tests/queries/0_stateless/02995_index_5.sh
index 80f75a532e3..58e53a2c481 100755
--- a/tests/queries/0_stateless/02995_index_5.sh
+++ b/tests/queries/0_stateless/02995_index_5.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Tags: long, no-debug, no-asan, no-tsan, no-msan, no-ubsan, no-sanitize-coverage
+# Tags: long, no-debug, no-asan, no-tsan, no-msan, no-ubsan, no-sanitize-coverage, no-distributed-cache
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
@@ -39,6 +39,6 @@ WHERE a >= (round(pow(sipHash64(1, try), 1 / (3 + sipHash64(2, try) % 8))) AS a1
   AND c <= (c1 + round(pow(sipHash64(11, try), 1 / (3 + sipHash64(12, try) % 8))))::String
 HAVING count() > 0;
 "
-done | ${CLICKHOUSE_CLIENT} 
+done | ${CLICKHOUSE_CLIENT}
 
 ${CLICKHOUSE_CLIENT} "DROP TABLE test"
diff --git a/tests/queries/0_stateless/02995_index_6.sh b/tests/queries/0_stateless/02995_index_6.sh
index e90387c7c0c..ef35f0e1126 100755
--- a/tests/queries/0_stateless/02995_index_6.sh
+++ b/tests/queries/0_stateless/02995_index_6.sh
@@ -1,5 +1,5 @@
 #!/usr/bin/env bash
-# Tags: long, no-debug, no-asan, no-tsan, no-msan, no-ubsan, no-sanitize-coverage
+# Tags: long, no-debug, no-asan, no-tsan, no-msan, no-ubsan, no-sanitize-coverage, no-distributed-cache
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh

From 41dfec0fe76a14bde54711de1cd8781b5d00a83e Mon Sep 17 00:00:00 2001
From: Nikita Taranov <nikita.taranov@clickhouse.com>
Date: Mon, 5 Aug 2024 14:58:40 +0100
Subject: [PATCH 1383/1488] fix test

---
 tests/queries/0_stateless/02313_filesystem_cache_seeks.sh | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/queries/0_stateless/02313_filesystem_cache_seeks.sh b/tests/queries/0_stateless/02313_filesystem_cache_seeks.sh
index fc91f3f1448..b7adde6fcbb 100755
--- a/tests/queries/0_stateless/02313_filesystem_cache_seeks.sh
+++ b/tests/queries/0_stateless/02313_filesystem_cache_seeks.sh
@@ -16,12 +16,12 @@ for STORAGE_POLICY in 's3_cache' 'local_cache' 's3_cache_multi' 'azure_cache'; d
     echo "Using storage policy: $STORAGE_POLICY"
     $CLICKHOUSE_CLIENT --query "SYSTEM DROP FILESYSTEM CACHE"
 
-    $CLICKHOUSE_CLIENT "${client_opts[@]}" --query "DROP TABLE IF EXISTS test_02313"
+    $CLICKHOUSE_CLIENT "${client_opts[@]}" --query "DROP TABLE IF EXISTS test_02313" > /dev/null
 
     $CLICKHOUSE_CLIENT "${client_opts[@]}" --query "CREATE TABLE test_02313 (id Int32, val String)
     ENGINE = MergeTree()
     ORDER BY tuple()
-    SETTINGS storage_policy = '$STORAGE_POLICY'"
+    SETTINGS storage_policy = '$STORAGE_POLICY'" > /dev/null
 
     $CLICKHOUSE_CLIENT --enable_filesystem_cache_on_write_operations=0 -n --query "INSERT INTO test_02313
     SELECT * FROM
@@ -36,6 +36,6 @@ for STORAGE_POLICY in 's3_cache' 'local_cache' 's3_cache_multi' 'azure_cache'; d
     $CLICKHOUSE_CLIENT --query "SELECT * FROM test_02313 WHERE val LIKE concat('%', randomPrintableASCII(3), '%') FORMAT Null"
     $CLICKHOUSE_CLIENT --query "SELECT * FROM test_02313 WHERE val LIKE concat('%', randomPrintableASCII(3), '%') FORMAT Null"
 
-    $CLICKHOUSE_CLIENT "${client_opts[@]}" --query "DROP TABLE test_02313"
+    $CLICKHOUSE_CLIENT "${client_opts[@]}" --query "DROP TABLE test_02313" > /dev/null
 
 done

From e9506202d699c24e9b7a3e5b643bc9871680e5dc Mon Sep 17 00:00:00 2001
From: kssenii <sumarokovakseniia@mail.ru>
Date: Mon, 5 Aug 2024 16:40:54 +0200
Subject: [PATCH 1384/1488] Add debug logging

---
 src/Storages/WindowView/StorageWindowView.cpp | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/src/Storages/WindowView/StorageWindowView.cpp b/src/Storages/WindowView/StorageWindowView.cpp
index 65bf6768b1b..bf934ed00d9 100644
--- a/src/Storages/WindowView/StorageWindowView.cpp
+++ b/src/Storages/WindowView/StorageWindowView.cpp
@@ -1055,6 +1055,8 @@ void StorageWindowView::threadFuncFireProc()
     /// TODO: consider using time_t instead (for every timestamp in this class)
     UInt32 timestamp_now = now();
 
+    LOG_TRACE(log, "Now: {}, next fire signal: {}, max watermark: {}", timestamp_now, next_fire_signal, max_watermark);
+
     while (next_fire_signal <= timestamp_now)
     {
         try
@@ -1072,6 +1074,9 @@ void StorageWindowView::threadFuncFireProc()
         if (slide_kind > IntervalKind::Kind::Day)
             slide_interval *= 86400;
         next_fire_signal += slide_interval;
+
+        LOG_TRACE(log, "Now: {}, next fire signal: {}, max watermark: {}, max fired watermark: {}, slide interval: {}",
+                  timestamp_now, next_fire_signal, max_watermark, max_fired_watermark, slide_interval);
     }
 
     if (max_watermark >= timestamp_now)
@@ -1433,16 +1438,19 @@ void StorageWindowView::writeIntoWindowView(
     while (window_view.modifying_query)
         std::this_thread::sleep_for(std::chrono::milliseconds(100));
 
-    if (!window_view.is_proctime && window_view.max_watermark == 0 && block.rows() > 0)
+    const size_t block_rows = block.rows();
+    if (!window_view.is_proctime && window_view.max_watermark == 0 && block_rows > 0)
     {
         std::lock_guard lock(window_view.fire_signal_mutex);
         const auto & window_column = block.getByName(window_view.timestamp_column_name);
         const ColumnUInt32::Container & window_end_data = static_cast<const ColumnUInt32 &>(*window_column.column).getData();
         UInt32 first_record_timestamp = window_end_data[0];
         window_view.max_watermark = window_view.getWindowUpperBound(first_record_timestamp);
+
+        LOG_TRACE(window_view.log, "New max watermark: {}", window_view.max_watermark);
     }
 
-    Pipe pipe(std::make_shared<SourceFromSingleChunk>(block));
+    Pipe pipe(std::make_shared<SourceFromSingleChunk>(std::move(block)));
 
     UInt32 lateness_bound = 0;
     UInt32 t_max_watermark = 0;
@@ -1649,6 +1657,8 @@ void StorageWindowView::writeIntoWindowView(
 
     auto executor = builder.execute();
     executor->execute(builder.getNumThreads(), local_context->getSettingsRef().use_concurrency_control);
+
+    LOG_TRACE(window_view.log, "Wrote {} rows into inner table ({})", block_rows, inner_table->getStorageID().getFullTableName());
 }
 
 void StorageWindowView::startup()

From b531f6b78c080605f1ab91cb45c4a11b4b6aafb8 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Mon, 5 Aug 2024 15:00:38 +0000
Subject: [PATCH 1385/1488] Fixing tests.

---
 src/Interpreters/ActionsDAG.cpp | 91 ++++++++++++++++++++-------------
 1 file changed, 55 insertions(+), 36 deletions(-)

diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp
index 7bf65ee2416..df1c0aa1f2a 100644
--- a/src/Interpreters/ActionsDAG.cpp
+++ b/src/Interpreters/ActionsDAG.cpp
@@ -567,49 +567,68 @@ void ActionsDAG::removeUnusedActions(const std::unordered_set<const Node *> & us
 
     std::stack<Frame> stack;
 
-    for (const auto * root : roots)
+    enum class VisitStage { NonDeterministic, Required };
+
+    for (auto stage : {VisitStage::NonDeterministic, VisitStage::Required})
     {
-        if (!required_nodes.contains(root))
-        {
-            required_nodes.insert(root);
-            stack.push({.node = root});
-        }
+        required_nodes.clear();
 
-        while (!stack.empty())
+        for (const auto * root : roots)
         {
-            auto & frame = stack.top();
-            auto * node = const_cast<Node *>(frame.node);
-
-            while (frame.next_child_to_visit < node->children.size())
+            if (!required_nodes.contains(root))
             {
-                const auto * child = node->children[frame.next_child_to_visit];
-                ++frame.next_child_to_visit;
-
-                if (!required_nodes.contains(child))
-                {
-                    required_nodes.insert(child);
-                    stack.push({.node = child});
-                    break;
-                }
-
-                if (non_deterministic_nodes.contains(child))
-                    non_deterministic_nodes.insert(node);
+                required_nodes.insert(root);
+                stack.push({.node = root});
             }
 
-            if (stack.top().node != node)
-                continue;
-
-            if (!node->isDeterministic())
-                non_deterministic_nodes.insert(node);
-
-            stack.pop();
-
-            /// Constant folding.
-            if (allow_constant_folding && !node->children.empty()
-                && node->column && isColumnConst(*node->column) && !non_deterministic_nodes.contains(node))
+            while (!stack.empty())
             {
-                node->type = ActionsDAG::ActionType::COLUMN;
-                node->children.clear();
+                auto & frame = stack.top();
+                auto * node = const_cast<Node *>(frame.node);
+
+                while (frame.next_child_to_visit < node->children.size())
+                {
+                    const auto * child = node->children[frame.next_child_to_visit];
+                    ++frame.next_child_to_visit;
+
+                    if (!required_nodes.contains(child))
+                    {
+                        required_nodes.insert(child);
+                        stack.push({.node = child});
+                        break;
+                    }
+                }
+
+                if (stack.top().node != node)
+                    continue;
+
+                stack.pop();
+
+                if (stage == VisitStage::Required)
+                    continue;
+
+                if (!node->isDeterministic())
+                    non_deterministic_nodes.insert(node);
+                else
+                {
+                    for (const auto * child : node->children)
+                    {
+                        if (non_deterministic_nodes.contains(child))
+                        {
+                            non_deterministic_nodes.insert(node);
+                            break;
+                        }
+                    }
+                }
+
+                /// Constant folding.
+                if (allow_constant_folding && !node->children.empty()
+                    && node->column && isColumnConst(*node->column))
+                {
+                    node->type = ActionsDAG::ActionType::COLUMN;
+                    node->children.clear();
+                    node->is_deterministic_constant = !non_deterministic_nodes.contains(node);
+                }
             }
         }
     }

From 59bffda9328a9d78c6a4422431f1014a16dd9cc2 Mon Sep 17 00:00:00 2001
From: Anton Popov <anton@clickhouse.com>
Date: Mon, 5 Aug 2024 17:06:28 +0200
Subject: [PATCH 1386/1488] Revert "Revert "Slightly better calculation of
 primary index""

---
 .../MergeTree/IMergeTreeDataPartWriter.cpp    | 19 +++++-
 .../MergeTreeDataPartWriterOnDisk.cpp         | 65 ++++++++++---------
 .../MergeTree/MergeTreeDataPartWriterOnDisk.h |  9 +--
 .../02993_lazy_index_loading.reference        |  2 +-
 ..._system_unload_primary_key_table.reference |  8 +--
 .../03128_system_unload_primary_key.reference |  4 +-
 6 files changed, 62 insertions(+), 45 deletions(-)

diff --git a/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp b/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp
index 6152da78395..c87f66b64f3 100644
--- a/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp
+++ b/src/Storages/MergeTree/IMergeTreeDataPartWriter.cpp
@@ -1,4 +1,5 @@
 #include <Storages/MergeTree/IMergeTreeDataPartWriter.h>
+#include <Common/MemoryTrackerBlockerInThread.h>
 
 namespace DB
 {
@@ -71,9 +72,21 @@ IMergeTreeDataPartWriter::IMergeTreeDataPartWriter(
 
 Columns IMergeTreeDataPartWriter::releaseIndexColumns()
 {
-    return Columns(
-        std::make_move_iterator(index_columns.begin()),
-        std::make_move_iterator(index_columns.end()));
+    /// The memory for index was allocated without thread memory tracker.
+    /// We need to deallocate it in shrinkToFit without memory tracker as well.
+    MemoryTrackerBlockerInThread temporarily_disable_memory_tracker;
+
+    Columns result;
+    result.reserve(index_columns.size());
+
+    for (auto & column : index_columns)
+    {
+        column->shrinkToFit();
+        result.push_back(std::move(column));
+    }
+
+    index_columns.clear();
+    return result;
 }
 
 SerializationPtr IMergeTreeDataPartWriter::getSerialization(const String & column_name) const
diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp
index 46dd766139a..6dc7e649b06 100644
--- a/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp
+++ b/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp
@@ -255,6 +255,12 @@ void MergeTreeDataPartWriterOnDisk::initPrimaryIndex()
             index_compressor_stream = std::make_unique<CompressedWriteBuffer>(*index_file_hashing_stream, primary_key_compression_codec, settings.primary_key_compress_block_size);
             index_source_hashing_stream = std::make_unique<HashingWriteBuffer>(*index_compressor_stream);
         }
+
+        const auto & primary_key_types = metadata_snapshot->getPrimaryKey().data_types;
+        index_serializations.reserve(primary_key_types.size());
+
+        for (const auto & type : primary_key_types)
+            index_serializations.push_back(type->getDefaultSerialization());
     }
 }
 
@@ -300,22 +306,33 @@ void MergeTreeDataPartWriterOnDisk::initSkipIndices()
             store = std::make_shared<GinIndexStore>(stream_name, data_part_storage, data_part_storage, storage_settings->max_digestion_size_per_segment);
             gin_index_stores[stream_name] = store;
         }
+
         skip_indices_aggregators.push_back(skip_index->createIndexAggregatorForPart(store, settings));
         skip_index_accumulated_marks.push_back(0);
     }
 }
 
+void MergeTreeDataPartWriterOnDisk::calculateAndSerializePrimaryIndexRow(const Block & index_block, size_t row)
+{
+    chassert(index_block.columns() == index_serializations.size());
+    auto & index_stream = compress_primary_key ? *index_source_hashing_stream : *index_file_hashing_stream;
+
+    for (size_t i = 0; i < index_block.columns(); ++i)
+    {
+        const auto & column = index_block.getByPosition(i).column;
+
+        index_columns[i]->insertFrom(*column, row);
+        index_serializations[i]->serializeBinary(*column, row, index_stream, {});
+    }
+}
+
 void MergeTreeDataPartWriterOnDisk::calculateAndSerializePrimaryIndex(const Block & primary_index_block, const Granules & granules_to_write)
 {
-    size_t primary_columns_num = primary_index_block.columns();
+    if (!metadata_snapshot->hasPrimaryKey())
+        return;
+
     if (index_columns.empty())
-    {
-        index_types = primary_index_block.getDataTypes();
-        index_columns.resize(primary_columns_num);
-        last_block_index_columns.resize(primary_columns_num);
-        for (size_t i = 0; i < primary_columns_num; ++i)
-            index_columns[i] = primary_index_block.getByPosition(i).column->cloneEmpty();
-    }
+        index_columns = primary_index_block.cloneEmptyColumns();
 
     {
         /** While filling index (index_columns), disable memory tracker.
@@ -329,22 +346,14 @@ void MergeTreeDataPartWriterOnDisk::calculateAndSerializePrimaryIndex(const Bloc
         /// Write index. The index contains Primary Key value for each `index_granularity` row.
         for (const auto & granule : granules_to_write)
         {
-            if (metadata_snapshot->hasPrimaryKey() && granule.mark_on_start)
-            {
-                for (size_t j = 0; j < primary_columns_num; ++j)
-                {
-                    const auto & primary_column = primary_index_block.getByPosition(j);
-                    index_columns[j]->insertFrom(*primary_column.column, granule.start_row);
-                    primary_column.type->getDefaultSerialization()->serializeBinary(
-                        *primary_column.column, granule.start_row, compress_primary_key ? *index_source_hashing_stream : *index_file_hashing_stream, {});
-                }
-            }
+            if (granule.mark_on_start)
+                calculateAndSerializePrimaryIndexRow(primary_index_block, granule.start_row);
         }
     }
 
-    /// store last index row to write final mark at the end of column
-    for (size_t j = 0; j < primary_columns_num; ++j)
-        last_block_index_columns[j] = primary_index_block.getByPosition(j).column;
+    /// Store block with last index row to write final mark at the end of column
+    if (with_final_mark)
+        last_index_block = primary_index_block;
 }
 
 void MergeTreeDataPartWriterOnDisk::calculateAndSerializeStatistics(const Block & block)
@@ -421,17 +430,11 @@ void MergeTreeDataPartWriterOnDisk::fillPrimaryIndexChecksums(MergeTreeData::Dat
 
     if (index_file_hashing_stream)
     {
-        if (write_final_mark)
+        if (write_final_mark && last_index_block)
         {
-            for (size_t j = 0; j < index_columns.size(); ++j)
-            {
-                const auto & column = *last_block_index_columns[j];
-                size_t last_row_number = column.size() - 1;
-                index_columns[j]->insertFrom(column, last_row_number);
-                index_types[j]->getDefaultSerialization()->serializeBinary(
-                    column, last_row_number, compress_primary_key ? *index_source_hashing_stream : *index_file_hashing_stream, {});
-            }
-            last_block_index_columns.clear();
+            MemoryTrackerBlockerInThread temporarily_disable_memory_tracker;
+            calculateAndSerializePrimaryIndexRow(last_index_block, last_index_block.rows() - 1);
+            last_index_block.clear();
         }
 
         if (compress_primary_key)
diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.h b/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.h
index bdf0fdb7f32..8d84442981e 100644
--- a/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.h
+++ b/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.h
@@ -173,10 +173,10 @@ protected:
     std::unique_ptr<HashingWriteBuffer> index_source_hashing_stream;
     bool compress_primary_key;
 
-    DataTypes index_types;
-    /// Index columns from the last block
-    /// It's written to index file in the `writeSuffixAndFinalizePart` method
-    Columns last_block_index_columns;
+    /// Last block with index columns.
+    /// It's written to index file in the `writeSuffixAndFinalizePart` method.
+    Block last_index_block;
+    Serializations index_serializations;
 
     bool data_written = false;
 
@@ -193,6 +193,7 @@ private:
     void initStatistics();
 
     virtual void fillIndexGranularity(size_t index_granularity_for_block, size_t rows_in_block) = 0;
+    void calculateAndSerializePrimaryIndexRow(const Block & index_block, size_t row);
 
     struct ExecutionStatistics
     {
diff --git a/tests/queries/0_stateless/02993_lazy_index_loading.reference b/tests/queries/0_stateless/02993_lazy_index_loading.reference
index 5bc329ae4eb..08f07a92815 100644
--- a/tests/queries/0_stateless/02993_lazy_index_loading.reference
+++ b/tests/queries/0_stateless/02993_lazy_index_loading.reference
@@ -1,4 +1,4 @@
-100000000	140000000
+100000000	100000000
 0	0
 1
 100000000	100000000
diff --git a/tests/queries/0_stateless/03127_system_unload_primary_key_table.reference b/tests/queries/0_stateless/03127_system_unload_primary_key_table.reference
index 3ac6127fb21..2d33f7f6683 100644
--- a/tests/queries/0_stateless/03127_system_unload_primary_key_table.reference
+++ b/tests/queries/0_stateless/03127_system_unload_primary_key_table.reference
@@ -1,8 +1,8 @@
-100000000	140000000
-100000000	140000000
-100000000	140000000
+100000000	100000000
+100000000	100000000
+100000000	100000000
 0	0
-100000000	140000000
+100000000	100000000
 0	0
 0	0
 1
diff --git a/tests/queries/0_stateless/03128_system_unload_primary_key.reference b/tests/queries/0_stateless/03128_system_unload_primary_key.reference
index c7b40ae5b06..2646dc7247f 100644
--- a/tests/queries/0_stateless/03128_system_unload_primary_key.reference
+++ b/tests/queries/0_stateless/03128_system_unload_primary_key.reference
@@ -1,4 +1,4 @@
-100000000	140000000
-100000000	140000000
+100000000	100000000
+100000000	100000000
 0	0
 0	0

From 5448bf7b86a27f7061e8fe2a02ef5b64ea4b2cc2 Mon Sep 17 00:00:00 2001
From: Anton Popov <anton@clickhouse.com>
Date: Mon, 5 Aug 2024 15:07:55 +0000
Subject: [PATCH 1387/1488] fix memory leak during index calculation

---
 .../MergeTree/MergeTreeDataPartWriterOnDisk.cpp   |  9 +++++----
 .../03217_primary_index_memory_leak.reference     |  1 +
 .../03217_primary_index_memory_leak.sql           | 15 +++++++++++++++
 3 files changed, 21 insertions(+), 4 deletions(-)
 create mode 100644 tests/queries/0_stateless/03217_primary_index_memory_leak.reference
 create mode 100644 tests/queries/0_stateless/03217_primary_index_memory_leak.sql

diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp
index 6dc7e649b06..b0e70e94b73 100644
--- a/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp
+++ b/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp
@@ -331,9 +331,6 @@ void MergeTreeDataPartWriterOnDisk::calculateAndSerializePrimaryIndex(const Bloc
     if (!metadata_snapshot->hasPrimaryKey())
         return;
 
-    if (index_columns.empty())
-        index_columns = primary_index_block.cloneEmptyColumns();
-
     {
         /** While filling index (index_columns), disable memory tracker.
          * Because memory is allocated here (maybe in context of INSERT query),
@@ -343,6 +340,9 @@ void MergeTreeDataPartWriterOnDisk::calculateAndSerializePrimaryIndex(const Bloc
          */
         MemoryTrackerBlockerInThread temporarily_disable_memory_tracker;
 
+        if (index_columns.empty())
+            index_columns = primary_index_block.cloneEmptyColumns();
+
         /// Write index. The index contains Primary Key value for each `index_granularity` row.
         for (const auto & granule : granules_to_write)
         {
@@ -434,9 +434,10 @@ void MergeTreeDataPartWriterOnDisk::fillPrimaryIndexChecksums(MergeTreeData::Dat
         {
             MemoryTrackerBlockerInThread temporarily_disable_memory_tracker;
             calculateAndSerializePrimaryIndexRow(last_index_block, last_index_block.rows() - 1);
-            last_index_block.clear();
         }
 
+        last_index_block.clear();
+
         if (compress_primary_key)
         {
             index_source_hashing_stream->finalize();
diff --git a/tests/queries/0_stateless/03217_primary_index_memory_leak.reference b/tests/queries/0_stateless/03217_primary_index_memory_leak.reference
new file mode 100644
index 00000000000..4913dd5e690
--- /dev/null
+++ b/tests/queries/0_stateless/03217_primary_index_memory_leak.reference
@@ -0,0 +1 @@
+150000
diff --git a/tests/queries/0_stateless/03217_primary_index_memory_leak.sql b/tests/queries/0_stateless/03217_primary_index_memory_leak.sql
new file mode 100644
index 00000000000..d5a553c7d72
--- /dev/null
+++ b/tests/queries/0_stateless/03217_primary_index_memory_leak.sql
@@ -0,0 +1,15 @@
+-- Tags: no-debug, no-tsan, no-msan, no-asan, no-random-settings, no-random-merge-tree-settings
+
+DROP TABLE IF EXISTS t_primary_index_memory;
+CREATE TABLE t_primary_index_memory (s String) ENGINE = MergeTree
+ORDER BY s SETTINGS index_granularity = 1;
+
+INSERT INTO t_primary_index_memory SELECT repeat('a', 10000) FROM numbers(150000)
+SETTINGS
+    max_block_size = 32,
+    max_memory_usage = '100M',
+    max_insert_block_size = 1024,
+    min_insert_block_size_rows = 1024;
+
+SELECT count() FROM t_primary_index_memory;
+DROP TABLE t_primary_index_memory;

From 146b8afce61ca2db2cdfd0621b0378746e90de59 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Mon, 5 Aug 2024 17:20:41 +0200
Subject: [PATCH 1388/1488] Disable bad tests

---
 .../Nodes/tests/gtest_throttler_constraint.cpp     | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/src/Common/Scheduler/Nodes/tests/gtest_throttler_constraint.cpp b/src/Common/Scheduler/Nodes/tests/gtest_throttler_constraint.cpp
index 6cfccb252fa..363e286c91c 100644
--- a/src/Common/Scheduler/Nodes/tests/gtest_throttler_constraint.cpp
+++ b/src/Common/Scheduler/Nodes/tests/gtest_throttler_constraint.cpp
@@ -10,7 +10,9 @@ using namespace DB;
 
 using ResourceTest = ResourceTestClass;
 
-TEST(SchedulerThrottlerConstraint, LeakyBucketConstraint)
+/// Test disabled because of leaks in the test themselves: https://github.com/ClickHouse/ClickHouse/issues/67678
+
+TEST(DISABLED_SchedulerThrottlerConstraint, LeakyBucketConstraint)
 {
     ResourceTest t;
     EventQueue::TimePoint start = std::chrono::system_clock::now();
@@ -40,7 +42,7 @@ TEST(SchedulerThrottlerConstraint, LeakyBucketConstraint)
     t.consumed("A", 10);
 }
 
-TEST(SchedulerThrottlerConstraint, Unlimited)
+TEST(DISABLED_SchedulerThrottlerConstraint, Unlimited)
 {
     ResourceTest t;
     EventQueue::TimePoint start = std::chrono::system_clock::now();
@@ -57,7 +59,7 @@ TEST(SchedulerThrottlerConstraint, Unlimited)
     }
 }
 
-TEST(SchedulerThrottlerConstraint, Pacing)
+TEST(DISABLED_SchedulerThrottlerConstraint, Pacing)
 {
     ResourceTest t;
     EventQueue::TimePoint start = std::chrono::system_clock::now();
@@ -77,7 +79,7 @@ TEST(SchedulerThrottlerConstraint, Pacing)
     }
 }
 
-TEST(SchedulerThrottlerConstraint, BucketFilling)
+TEST(DISABLED_SchedulerThrottlerConstraint, BucketFilling)
 {
     ResourceTest t;
     EventQueue::TimePoint start = std::chrono::system_clock::now();
@@ -111,7 +113,7 @@ TEST(SchedulerThrottlerConstraint, BucketFilling)
     t.consumed("A", 3);
 }
 
-TEST(SchedulerThrottlerConstraint, PeekAndAvgLimits)
+TEST(DISABLED_SchedulerThrottlerConstraint, PeekAndAvgLimits)
 {
     ResourceTest t;
     EventQueue::TimePoint start = std::chrono::system_clock::now();
@@ -139,7 +141,7 @@ TEST(SchedulerThrottlerConstraint, PeekAndAvgLimits)
     }
 }
 
-TEST(SchedulerThrottlerConstraint, ThrottlerAndFairness)
+TEST(DISABLED_SchedulerThrottlerConstraint, ThrottlerAndFairness)
 {
     ResourceTest t;
     EventQueue::TimePoint start = std::chrono::system_clock::now();

From 4e9c3baa25cc7f02a8ba9ad5be6d68964b6a04f5 Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <nikitamikhaylov@clickhouse.com>
Date: Fri, 12 Jul 2024 12:49:26 +0000
Subject: [PATCH 1389/1488] Move analyzer to Beta stage

---
 .../test/fuzzer/query-fuzzer-tweaks-users.xml |   4 +-
 docker/test/stateless/stress_tests.lib        |   4 +-
 docs/en/development/architecture.md           |   2 +-
 docs/en/operations/analyzer.md                |   8 +-
 docs/en/operations/settings/settings.md       |   2 +-
 docs/ru/development/architecture.md           |   2 +-
 src/Client/HedgedConnections.cpp              |   4 +-
 src/Client/MultiplexedConnections.cpp         |   4 +-
 src/Core/Settings.h                           |   5 +-
 src/Core/SettingsChangesHistory.cpp           | 259 ++++++++++++++++++
 src/Interpreters/ActionsVisitor.cpp           |   2 +-
 .../ClusterProxy/SelectStreamFactory.cpp      |   4 +-
 .../ClusterProxy/executeQuery.cpp             |   4 +-
 src/Interpreters/InterpreterCreateQuery.cpp   |   4 +-
 src/Interpreters/InterpreterDescribeQuery.cpp |   2 +-
 src/Interpreters/InterpreterExplainQuery.cpp  |  10 +-
 src/Interpreters/InterpreterFactory.cpp       |   6 +-
 src/Interpreters/InterpreterInsertQuery.cpp   |   2 +-
 src/Interpreters/MutationsInterpreter.cpp     |   6 +-
 src/Interpreters/executeQuery.cpp             |   6 +-
 .../getHeaderForProcessingStage.cpp           |   2 +-
 .../QueryPlan/DistributedCreateLocalPlan.cpp  |   2 +-
 .../Transforms/buildPushingToViewsChain.cpp   |   4 +-
 src/Server/TCPHandler.cpp                     |   6 +-
 src/Storages/AlterCommands.cpp                |   2 +-
 src/Storages/IStorageCluster.cpp              |   2 +-
 src/Storages/LiveView/StorageLiveView.cpp     |  10 +-
 src/Storages/MergeTree/MergeTreeData.cpp      |   2 +-
 src/Storages/MergeTree/RPNBuilder.cpp         |  18 +-
 src/Storages/StorageBuffer.cpp                |   2 +-
 src/Storages/StorageDistributed.cpp           |   6 +-
 src/Storages/StorageExecutable.cpp            |   2 +-
 src/Storages/StorageMerge.cpp                 |  20 +-
 src/Storages/StorageMergeTree.cpp             |   6 +-
 src/Storages/StorageReplicatedMergeTree.cpp   |   6 +-
 src/Storages/StorageView.cpp                  |   2 +-
 src/Storages/TTLDescription.cpp               |   2 +-
 src/Storages/WindowView/StorageWindowView.cpp |   6 +-
 src/TableFunctions/TableFunctionView.cpp      |   2 +-
 .../TableFunctionViewIfPermitted.cpp          |   2 +-
 tests/config/users.d/analyzer.xml             |   2 +-
 .../helpers/0_common_enable_old_analyzer.xml  |   2 +-
 tests/integration/helpers/cluster.py          |   2 +-
 .../test_analyzer_compatibility/test.py       |   8 +-
 .../test_distributed_type_object/test.py      |   2 +-
 .../configs/enable_parallel_replicas.xml      |   2 +-
 .../integration/test_settings_profile/test.py |   6 +-
 .../performance/storage_join_direct_join.xml  |   4 +-
 tests/performance/uniq_to_count.xml           |   4 +-
 .../queries/0_stateless/00116_storage_set.sql |   4 +-
 .../queries/0_stateless/00202_cross_join.sql  |   3 +-
 ..._shard_no_aggregates_and_constant_keys.sql |   2 +-
 .../00313_const_totals_extremes.sh            |  12 +-
 ...inal_and_prewhere_condition_ver_column.sql |   4 +-
 .../00370_duplicate_columns_in_subqueries.sql |   2 +-
 .../00378_json_quote_64bit_integers.sql       |   2 +-
 .../0_stateless/00445_join_nullable_keys.sql  |   2 +-
 .../queries/0_stateless/00490_with_select.sql |   2 +-
 .../00597_push_down_predicate_long.reference  |  10 +-
 .../00597_push_down_predicate_long.sql        |  30 +-
 ...00621_regression_for_in_operator.reference |   4 +-
 .../00621_regression_for_in_operator.sql      |   8 +-
 .../0_stateless/00674_join_on_syntax.sql      |   2 +-
 .../0_stateless/00700_decimal_compare.sql     |   2 +-
 .../queries/0_stateless/00722_inner_join.sql  |   2 +-
 .../00736_disjunction_optimisation.reference  |  18 +-
 .../00736_disjunction_optimisation.sql        |  36 +--
 .../0_stateless/00757_enum_defaults_const.sql |   2 +-
 .../00757_enum_defaults_const_analyzer.sql    |   2 +-
 .../00800_low_cardinality_join.sql            |   2 +-
 .../00800_low_cardinality_merge_join.sql.j2   |   2 +-
 .../0_stateless/00818_alias_bug_4110.sql      |   2 +-
 .../0_stateless/00818_inner_join_bug_3567.sql |   2 +-
 ...00819_full_join_wrong_columns_in_block.sql |   2 +-
 .../0_stateless/00820_multiple_joins.sql      |   2 +-
 ...multiple_joins_subquery_requires_alias.sql |   2 +-
 .../0_stateless/00830_join_overwrite.sql      |   2 +-
 .../0_stateless/00835_if_generic_case.sql     |   2 +-
 .../00848_join_use_nulls_segfault.sql         |   2 +-
 .../00849_multiple_comma_join_2.sql           |  34 +--
 .../00855_join_with_array_join.sql            |   2 +-
 .../queries/0_stateless/00858_issue_4756.sql  |   2 +-
 tests/queries/0_stateless/00897_flatten.sql   |   2 +-
 ...0940_order_by_read_in_order_query_plan.sql |   2 +-
 .../01000_subquery_requires_alias.sql         |   2 +-
 .../01013_totals_without_aggregation.sql      |   2 +-
 .../0_stateless/01018_ambiguous_column.sql    |   2 +-
 .../01047_window_view_parser_inner_table.sql  |   2 +-
 .../0_stateless/01048_window_view_parser.sql  |   2 +-
 .../01049_join_low_card_bug_long.reference.j2 |  64 ++---
 .../01049_join_low_card_bug_long.sql.j2       |   4 +-
 .../01050_window_view_parser_tumble.sql       |   2 +-
 .../01051_window_view_parser_hop.sql          |   2 +-
 .../01052_window_view_proc_tumble_to_now.sh   |   2 +-
 .../01053_window_view_proc_hop_to_now.sh      |   2 +-
 .../01054_window_view_proc_tumble_to.sh       |   2 +-
 .../01055_window_view_proc_hop_to.sh          |   2 +-
 .../01056_window_view_proc_hop_watch.py       |   4 +-
 ..._window_view_event_tumble_to_strict_asc.sh |   2 +-
 ...058_window_view_event_hop_to_strict_asc.sh |   2 +-
 ..._window_view_event_hop_watch_strict_asc.py |   4 +-
 .../01060_window_view_event_tumble_to_asc.sh  |   2 +-
 .../01061_window_view_event_hop_to_asc.sh     |   2 +-
 .../01062_window_view_event_hop_watch_asc.py  |   4 +-
 ...063_window_view_event_tumble_to_bounded.sh |   2 +-
 .../01064_window_view_event_hop_to_bounded.sh |   2 +-
 ...065_window_view_event_hop_watch_bounded.py |   4 +-
 ...iew_event_tumble_to_strict_asc_lateness.sh |   2 +-
 ...indow_view_event_tumble_to_asc_lateness.sh |   2 +-
 ...w_view_event_tumble_to_bounded_lateness.sh |   2 +-
 .../01069_window_view_proc_tumble_watch.py    |   4 +-
 .../01070_window_view_watch_events.py         |   4 +-
 ...01071_window_view_event_tumble_asc_join.sh |   2 +-
 ...72_window_view_multiple_columns_groupby.sh |   2 +-
 ...indow_view_event_tumble_to_asc_populate.sh |   2 +-
 ...dow_view_event_tumble_asc_join_populate.sh |   2 +-
 ...window_view_proc_tumble_to_now_populate.sh |   2 +-
 .../01076_window_view_alter_query_to.sh       |   2 +-
 ...indow_view_alter_query_to_modify_source.sh |   2 +-
 .../01078_window_view_alter_query_watch.py    |   6 +-
 ...9_window_view_inner_table_memory_tumble.sh |   2 +-
 ...1080_window_view_inner_table_memory_hop.sh |   2 +-
 .../01081_window_view_target_table_engine.sh  |   2 +-
 .../01082_window_view_watch_limit.py          |   4 +-
 .../0_stateless/01083_window_view_select.sh   |   2 +-
 ...01084_window_view_with_table_identifier.sh |   2 +-
 .../0_stateless/01085_window_view_attach.sql  |   2 +-
 .../0_stateless/01086_window_view_cleanup.sh  |   2 +-
 .../01087_window_view_alter_query.sh          |   2 +-
 .../01088_window_view_default_column.sh       |   2 +-
 .../01142_join_lc_and_nullable_in_key.sql     |  28 +-
 ...1142_merge_join_lc_and_nullable_in_key.sql |  24 +-
 tests/queries/0_stateless/01232_untuple.sql   |   2 +-
 ...01300_group_by_other_keys_having.reference |   6 +-
 .../01300_group_by_other_keys_having.sql      |   8 +-
 ..._redundant_functions_in_order_by.reference |  16 +-
 .../01323_redundant_functions_in_order_by.sql |  28 +-
 .../01353_low_cardinality_join_types.sql      |   4 +-
 ...BY_injective_elimination_dictGet.reference |   2 +-
 ...GROUP_BY_injective_elimination_dictGet.sql |   2 +-
 .../0_stateless/01428_nullable_asof_join.sql  |  12 +-
 .../01455_opentelemetry_distributed.reference |   4 +-
 .../01455_opentelemetry_distributed.sh        |   6 +-
 .../01476_right_full_join_switch.sql          |   4 +-
 .../01477_lc_in_merge_join_left_key.sql.j2    |   6 +-
 .../0_stateless/01479_cross_join_9855.sql     |   4 +-
 .../0_stateless/01508_explain_header.sql      |   2 +-
 .../01556_explain_select_with_union_query.sql |   2 +-
 .../01561_clickhouse_client_stage.reference   |  10 +-
 .../01561_clickhouse_client_stage.sh          |   2 +-
 .../0_stateless/01591_window_functions.sql    |   2 +-
 .../0_stateless/01600_detach_permanently.sh   |   4 +-
 ..._constraints_simple_optimization.reference |   6 +-
 .../01622_constraints_simple_optimization.sql |   6 +-
 ...2_constraints_where_optimization.reference |  12 +-
 .../01622_constraints_where_optimization.sql  |  12 +-
 .../01623_constraints_column_swap.reference   |  26 +-
 .../01623_constraints_column_swap.sql         |  26 +-
 .../0_stateless/01646_rewrite_sum_if.sql      |   2 +-
 .../0_stateless/01651_bugs_from_15889.sql     |   2 +-
 .../0_stateless/01655_plan_optimizations.sh   |  48 ++--
 ...01655_plan_optimizations_merge_filters.sql |   4 +-
 ...ns_optimize_read_in_window_order.reference |   8 +-
 ...mizations_optimize_read_in_window_order.sh |  24 +-
 .../01671_merge_join_and_constants.sql        |   2 +-
 .../01721_join_implicit_cast_long.sql.j2      |  12 +-
 .../0_stateless/01739_index_hint.reference    |   4 +-
 .../queries/0_stateless/01739_index_hint.sql  |   4 +-
 ...ze_skip_unused_shards_rewrite_in.reference |   4 +-
 ...optimize_skip_unused_shards_rewrite_in.sql |   4 +-
 ...1757_optimize_skip_unused_shards_limit.sql |   6 +-
 .../01763_filter_push_down_bugs.sql           |   4 +-
 .../0_stateless/01786_explain_merge_tree.sh   |   2 +-
 .../queries/0_stateless/01823_explain_json.sh |   2 +-
 .../01852_multiple_joins_with_union_join.sql  |   2 +-
 ...01872_functions_to_subcolumns_analyzer.sql |   2 +-
 .../01890_cross_join_explain_crash.sql        |   2 +-
 .../01913_names_of_tuple_literal.sql          |   2 +-
 .../01925_join_materialized_columns.sql       |   8 +-
 ...25_test_storage_merge_aliases_analyzer.sql |   2 +-
 ...hree_parts_identifiers_in_wrong_places.sql |   2 +-
 .../01942_dateTimeToSnowflakeID.sql           |   2 +-
 .../01942_snowflakeIDToDateTime.sql           |   2 +-
 ...istributed_group_by_sharding_key.reference |   2 +-
 ...mize_distributed_group_by_sharding_key.sql |   4 +-
 .../0_stateless/02000_join_on_const.reference |  12 +-
 .../0_stateless/02000_join_on_const.sql       |  41 ++-
 .../0_stateless/02030_tuple_filter.sql        |   2 +-
 .../02048_clickhouse_local_stage.reference    |  10 +-
 .../02048_clickhouse_local_stage.sh           |   2 +-
 .../02115_map_contains_analyzer.sql           |   2 +-
 .../02116_tuple_element_analyzer.sql          |   2 +-
 .../02125_query_views_log_window_function.sql |   2 +-
 ...window_functions_disable_optimizations.sql |   2 +-
 .../02136_scalar_read_rows_json.sh            |   2 +-
 .../02149_read_in_order_fixed_prefix.sql      |  12 +-
 .../02151_hash_table_sizes_stats_joins.sh     |   4 +-
 .../02154_dictionary_get_http_json.sh         |   2 +-
 .../02156_storage_merge_prewhere.sql          |   4 +-
 .../0_stateless/02174_cte_scalar_cache.sql    |   4 +-
 .../0_stateless/02174_cte_scalar_cache_mv.sql |  12 +-
 .../02184_hash_functions_and_ip_types.sql     |   2 +-
 .../02226_analyzer_or_like_combine.reference  |   4 +-
 .../02226_analyzer_or_like_combine.sql        |  14 +-
 .../0_stateless/02227_union_match_by_name.sql |   2 +-
 .../02233_with_total_empty_chunk.sql          |   2 +-
 .../02234_clickhouse_local_test_mode.sh       |   5 +-
 .../02267_join_dup_columns_issue36199.sql     |   4 +-
 ..._column_matcher_and_column_transformer.sql |   2 +-
 .../0_stateless/02303_query_kind.reference    |   8 +-
 tests/queries/0_stateless/02303_query_kind.sh |   2 +-
 .../02315_grouping_constant_folding.reference |   2 +-
 .../02315_grouping_constant_folding.sql       |   2 +-
 ..._distinct_in_order_optimization_explain.sh |   4 +-
 .../02337_analyzer_columns_basic.sql          |   2 +-
 .../02337_multiple_joins_original_names.sql   |   2 +-
 .../02338_analyzer_constants_basic.sql        |   2 +-
 .../02339_analyzer_matcher_basic.sql          |   2 +-
 .../0_stateless/02340_analyzer_functions.sql  |   2 +-
 .../02341_analyzer_aliases_basics.sql         |   2 +-
 .../02341_global_join_cte.reference           |   4 +-
 .../0_stateless/02341_global_join_cte.sql     |   4 +-
 .../02342_analyzer_compound_types.sql         |   2 +-
 .../02342_window_view_different_struct.sql    |   2 +-
 ...43_analyzer_column_transformers_strict.sql |   2 +-
 .../0_stateless/02343_analyzer_lambdas.sql    |   2 +-
 .../02343_analyzer_lambdas_issue_28083.sql    |   6 +-
 .../02343_analyzer_lambdas_issue_36677.sql    |   2 +-
 ...alyzer_multiple_aliases_for_expression.sql |   2 +-
 .../0_stateless/02345_analyzer_subqueries.sql |   2 +-
 .../02346_fulltext_index_bug52019.sql         |   6 +-
 .../02346_fulltext_index_match_predicate.sql  |  12 +-
 .../0_stateless/02354_vector_search_bugs.sql  |   2 +-
 .../0_stateless/02364_window_view_segfault.sh |   2 +-
 .../0_stateless/02366_explain_query_tree.sql  |   2 +-
 .../02367_analyzer_table_alias_columns.sql    |   2 +-
 .../02368_analyzer_table_functions.sql        |   2 +-
 .../02369_analyzer_array_join_function.sql    |   2 +-
 .../02370_analyzer_in_function.sql            |   2 +-
 .../0_stateless/02371_analyzer_join_cross.sql |   2 +-
 .../0_stateless/02372_analyzer_join.sql.j2    |   2 +-
 .../02373_analyzer_join_use_nulls.sql         |   2 +-
 .../0_stateless/02374_analyzer_array_join.sql |   2 +-
 .../02374_analyzer_join_using.sql.j2          |   2 +-
 ...2374_combine_multi_if_and_count_if_opt.sql |   2 +-
 .../0_stateless/02375_analyzer_union.sql      |   2 +-
 .../02376_analyzer_in_function_subquery.sql   |   2 +-
 .../02377_analyzer_in_function_set.sql        |   2 +-
 ...ting_by_input_stream_properties_explain.sh |   4 +-
 .../02378_analyzer_projection_names.sql       |   2 +-
 .../02379_analyzer_subquery_depth.sql         |   2 +-
 .../02380_analyzer_join_sample.sql            |   2 +-
 .../0_stateless/02381_analyzer_join_final.sql |   2 +-
 .../02381_join_dup_columns_in_plan.sql.j2     |   2 +-
 .../02382_analyzer_matcher_join_using.sql     |   2 +-
 .../02383_analyzer_merge_tree_self_join.sql   |   2 +-
 .../02384_analyzer_dict_get_join_get.sql      |   2 +-
 ...5_analyzer_aliases_compound_expression.sql |   2 +-
 ...analyzer_in_function_nested_subqueries.sql |   2 +-
 .../0_stateless/02387_analyzer_cte.sql        |   2 +-
 .../02388_analyzer_recursive_lambda.sql       |   2 +-
 .../02389_analyzer_nested_lambda.sql          |   2 +-
 .../02420_final_setting_analyzer.reference    |   2 +-
 .../02420_final_setting_analyzer.sql          |   2 +-
 ...02421_decimal_in_precision_issue_41125.sql |   2 +-
 .../0_stateless/02421_explain_subquery.sql    |   4 +-
 ...2428_decimal_in_floating_point_literal.sql |   2 +-
 .../0_stateless/02428_parameterized_view.sh   |   4 +-
 .../0_stateless/02451_order_by_monotonic.sh   |   2 +-
 .../0_stateless/02459_group_by_all.sql        |   2 +-
 ..._subqueries_table_expression_modifiers.sql |   2 +-
 .../02475_analyzer_join_tree_subquery.sql     |   2 +-
 ..._analyzer_subquery_compound_expression.sql |   2 +-
 ...2475_or_function_alias_and_const_where.sql |   2 +-
 .../02476_analyzer_identifier_hints.sh        |  90 +++---
 ...2476_analyzer_join_with_unused_columns.sql |   2 +-
 .../0_stateless/02476_fuse_sum_count.sql      |   2 +-
 .../02477_analyzer_array_join_with_join.sql   |   2 +-
 ...02477_analyzer_ast_key_condition_crash.sql |   2 +-
 .../02477_analyzer_function_hints.sh          |  10 +-
 .../0_stateless/02477_exists_fuzz_43478.sql   |   2 +-
 .../0_stateless/02477_fuse_quantiles.sql      |   2 +-
 ...ssions_optimizer_low_cardinality.reference |  12 +-
 ..._expressions_optimizer_low_cardinality.sql |  12 +-
 ...2478_analyzer_table_expression_aliases.sql |   2 +-
 .../02478_window_frame_type_groups.sql        |   4 +-
 .../02479_analyzer_aggregation_crash.sql      |   2 +-
 ...er_aggregation_totals_rollup_crash_fix.sql |   2 +-
 .../02479_analyzer_join_with_constants.sql    |   2 +-
 .../02479_mysql_connect_to_self.sql           |   2 +-
 .../02480_analyzer_alias_nullptr.sql          |   2 +-
 .../0_stateless/02480_tlp_nan.reference       |  20 +-
 tests/queries/0_stateless/02480_tlp_nan.sql   |  20 +-
 .../02481_aggregation_in_order_plan.sql       |   4 +-
 ...er_join_alias_unknown_identifier_crash.sql |   2 +-
 ...lyzer_optimize_aggregation_arithmetics.sql |   2 +-
 ...1_analyzer_optimize_grouping_sets_keys.sql |   2 +-
 .../02483_cuturlparameter_with_arrays.sql     |   2 +-
 .../queries/0_stateless/02483_elapsed_time.sh |   2 +-
 .../0_stateless/02489_analyzer_indexes.sql    |   3 +-
 .../02493_analyzer_sum_if_to_count_if.sql     |   2 +-
 ...02493_analyzer_table_functions_untuple.sql |   2 +-
 ...r_uniq_injective_functions_elimination.sql |   2 +-
 ...analyzer_compound_expression_crash_fix.sql |   2 +-
 .../0_stateless/02494_query_cache_explain.sql |   2 +-
 .../02494_query_cache_nested_query_bug.sh     |   4 +-
 .../02495_analyzer_storage_join.sql           |   2 +-
 .../02495_sum_if_to_count_if_bug.sql          |   5 +-
 .../02496_remove_redundant_sorting.sh         |   4 +-
 ...nalyzer_sum_if_count_if_pass_crash_fix.sql |   2 +-
 ..._having_without_actual_aggregation_bug.sql |   2 +-
 .../02497_if_transform_strings_to_enum.sql    |   2 +-
 .../02497_storage_join_right_assert.sql       |   4 +-
 ...nctions_arithmetic_operations_pass_fix.sql |   2 +-
 .../02498_analyzer_settings_push_down.sql     |   2 +-
 .../02498_storage_join_key_positions.sql.j2   |   2 +-
 ...er_aggregate_function_lambda_crash_fix.sql |   2 +-
 .../0_stateless/02499_analyzer_set_index.sql  |   2 +-
 .../02500_analyzer_storage_view_crash_fix.sql |   2 +-
 .../02500_remove_redundant_distinct.sh        |   4 +-
 ...501_analyzer_expired_context_crash_fix.sql |   2 +-
 ...02502_analyzer_insert_select_crash_fix.sql |   2 +-
 .../02503_join_switch_alias_fuzz.sql          |   2 +-
 ...513_analyzer_duplicate_alias_crash_fix.sql |   2 +-
 .../0_stateless/02513_analyzer_sort_msan.sql  |   2 +-
 .../02514_analyzer_drop_join_on.sql           |   2 +-
 .../02515_analyzer_null_for_empty.sql         |   2 +-
 ...6_join_with_totals_and_subquery_bug.sql.j2 |   4 +-
 .../02516_projections_and_context.sql         |   4 +-
 ...518_rewrite_aggregate_function_with_if.sql |   2 +-
 ...21_analyzer_aggregation_without_column.sql |   2 +-
 .../02521_analyzer_array_join_crash.sql       |   2 +-
 .../02525_analyzer_function_in_crash_fix.sql  |   2 +-
 ...02532_analyzer_aggregation_with_rollup.sql |   2 +-
 .../02534_analyzer_grouping_function.sql      |   2 +-
 .../02535_analyzer_group_by_use_nulls.sql     |   2 +-
 .../02535_analyzer_limit_offset.sql           |   2 +-
 .../02538_analyzer_create_table_as_select.sql |   2 +-
 ...zer_matcher_alias_materialized_columns.sql |   2 +-
 ...json_ignore_unknown_keys_in_named_tuple.sh |   2 +-
 ...02541_analyzer_grouping_sets_crash_fix.sql |   2 +-
 ..._optimize_group_by_function_keys_crash.sql |   2 +-
 .../02553_type_object_analyzer.sql            |   2 +-
 ..._fix_grouping_sets_predicate_push_down.sql |   8 +-
 .../02560_analyzer_materialized_view.sql      |   2 +-
 .../0_stateless/02563_analyzer_merge.sql      |   2 +-
 .../02564_analyzer_cross_to_inner.sql         |   2 +-
 .../02565_analyzer_limit_settings.sql         |   2 +-
 ...66_analyzer_limit_settings_distributed.sql |   2 +-
 .../02567_and_consistency.reference           |   2 +-
 .../0_stateless/02567_and_consistency.sql     |   4 +-
 .../02576_predicate_push_down_sorting_fix.sql |   2 +-
 .../02576_rewrite_array_exists_to_has.sql     |   4 +-
 .../02577_analyzer_array_join_calc_twice.sql  |   2 +-
 .../0_stateless/02579_fill_empty_chunk.sql    |   2 +-
 .../02579_fill_empty_chunk_analyzer.sql       |   2 +-
 ...alyzer_join_subquery_empty_column_list.sql |   4 +-
 .../02661_quantile_approx.reference           |  12 +-
 .../0_stateless/02661_quantile_approx.sql     |  14 +-
 .../02662_first_last_value.reference          |   2 +-
 .../0_stateless/02662_first_last_value.sql    |   2 +-
 ...al_optimizer_removing_redundant_checks.sql |   2 +-
 .../02674_trivial_count_analyzer.reference    |   2 +-
 .../02674_trivial_count_analyzer.sql          |   2 +-
 ...75_predicate_push_down_filled_join_fix.sql |   2 +-
 .../02676_analyzer_limit_offset.sql           |   2 +-
 ...676_distinct_reading_in_order_analyzer.sql |   2 +-
 .../02677_analyzer_bitmap_has_any.sql         |   5 +-
 .../02677_analyzer_compound_expressions.sql   |   2 +-
 .../02677_get_subcolumn_array_of_tuples.sql   |   2 +-
 ...explain_merge_tree_prewhere_row_policy.sql |   4 +-
 .../02699_polygons_sym_difference_rollup.sql  |   4 +-
 .../02699_polygons_sym_difference_total.sql   |   2 +-
 ...polygons_sym_difference_total_analyzer.sql |   2 +-
 .../02701_invalid_having_NOT_AN_AGGREGATE.sql |   2 +-
 .../02702_logical_optimizer_with_nulls.sql    |   2 +-
 ...ry_tree_is_forbidden_with_old_analyzer.sql |   2 +-
 ...2704_storage_merge_explain_graph_crash.sql |   2 +-
 .../02707_analyzer_nested_lambdas_types.sql   |  10 +-
 .../02722_matcher_join_use_nulls.sql.j2       |   2 +-
 .../0_stateless/02725_cnf_large_check.sql     |   8 +-
 .../02731_analyzer_join_resolve_nested.sql.j2 |   2 +-
 ..._parallel_replicas_join_subquery.reference |   8 +-
 .../02731_parallel_replicas_join_subquery.sql |  12 +-
 .../0_stateless/02734_optimize_group_by.sql   |   4 +-
 ...s_with_subqueries_profile_events.reference |  24 +-
 ..._queries_with_subqueries_profile_events.sh |  26 +-
 .../02767_into_outfile_extensions_msan.sh     |   2 +-
 .../02771_ignore_data_skipping_indices.sql    |   4 +-
 ...02771_parallel_replicas_analyzer.reference |   2 +-
 .../02771_parallel_replicas_analyzer.sql      |   2 +-
 .../02771_semi_join_use_nulls.sql.j2          |   4 +-
 .../02783_date_predicate_optimizations.sql    | 100 +++----
 ...lel_replicas_trivial_count_optimization.sh |   8 +-
 ...l_conditions_to_prewhere_analyzer_asan.sql |   2 +-
 ...mizations_ast_query_tree_rewrite.reference |  56 ++--
 ...e_optimizations_ast_query_tree_rewrite.sql |  56 ++--
 .../02803_remote_cannot_clone_block.sql       |   2 +-
 .../02812_bug_with_unused_join_columns.sql    |   2 +-
 ...r_aggregate_functions_of_group_by_keys.sql |   2 +-
 .../02815_join_algorithm_setting.sql          |   4 +-
 ...834_analyzer_with_statement_references.sql |   2 +-
 .../0_stateless/02835_join_step_explain.sql   |   2 +-
 .../02840_merge__table_or_filter.reference    |  32 +--
 .../02840_merge__table_or_filter.sql.j2       |   8 +-
 ...41_valid_json_and_xml_on_http_exception.sh |   2 +-
 .../0_stateless/02841_with_clause_resolve.sql |  48 ++--
 .../02861_filter_pushdown_const_bug.sql       |   2 +-
 .../02864_statistics_materialize_in_merge.sql |   2 +-
 .../02866_size_of_marks_skip_idx_explain.sql  |   2 +-
 ...8_distinct_to_count_optimization.reference |  32 +--
 .../02868_distinct_to_count_optimization.sql  |  64 ++---
 ..._key_index_in_function_different_types.sql |   4 +-
 .../02890_named_tuple_functions.sql           |   2 +-
 .../02890_untuple_column_names.reference      |   2 +-
 .../02890_untuple_column_names.sql            |  44 +--
 .../02911_analyzer_explain_estimate.sql       |   2 +-
 ...yzer_order_by_read_in_order_query_plan.sql |   2 +-
 ...lyzer_remove_unused_projection_columns.sql |   2 +-
 .../02911_join_on_nullsafe_optimization.sql   |   2 +-
 .../02911_support_alias_column_in_indices.sql |   8 +-
 .../0_stateless/02915_analyzer_fuzz_1.sql     |   2 +-
 .../0_stateless/02915_analyzer_fuzz_2.sql     |   3 +-
 .../0_stateless/02915_analyzer_fuzz_5.sql     |   2 +-
 .../0_stateless/02915_analyzer_fuzz_6.sql     |   2 +-
 .../0_stateless/02918_join_pm_lc_crash.sql    |   9 +-
 .../02918_optimize_count_for_merge_tables.sql |   4 +-
 .../02922_respect_nulls_parser.sql            |   2 +-
 ...alyzer_rewrite_sum_column_and_constant.sql |   2 +-
 .../02932_parallel_replicas_fuzzer.sql        |   2 +-
 .../queries/0_stateless/02933_paste_join.sql  |   6 +-
 .../0_stateless/02943_order_by_all.sql        |  32 +--
 ...ngrambf_indexes_support_match_function.sql |  24 +-
 .../02944_variant_as_common_type_analyzer.sql |   3 +-
 .../02952_conjunction_optimization.sql        |   2 +-
 .../02954_analyzer_fuzz_i57086.sql            |   2 +-
 ...nalyzer_using_functional_args.reference.j2 |  20 +-
 ...2955_analyzer_using_functional_args.sql.j2 |  28 +-
 .../02955_sparkBar_alias_sparkbar.sql         |   3 +-
 .../02962_join_using_bug_57894.sql            |   4 +-
 .../0_stateless/02967_analyzer_fuzz.sql       |   4 +-
 ...allel_replicas_join_algo_and_analyzer_3.sh |  20 +-
 ...llel_replicas_joins_and_analyzer.reference |  72 ++---
 ...arallel_replicas_joins_and_analyzer.sql.j2 |  36 +--
 ...analyzer_eliminate_injective_functions.sql |   2 +-
 .../02969_functions_to_subcolumns_if_null.sql |   8 +-
 .../0_stateless/02971_analyzer_remote_id.sh   |   2 +-
 ...1_functions_to_subcolumns_column_names.sql |   2 +-
 .../02971_functions_to_subcolumns_map.sql     |   2 +-
 .../02971_functions_to_subcolumns_variant.sql |   2 +-
 .../02972_parallel_replicas_cte.sql           |   6 +-
 .../02974_analyzer_array_join_subcolumn.sql   |  10 +-
 ..._logical_optimizer_pass_lowcardinality.sql |   2 +-
 .../02989_join_using_parent_scope.reference   |  32 +--
 .../02989_join_using_parent_scope.sql         |  32 +--
 .../02991_count_rewrite_analyzer.sql          |   2 +-
 .../02992_analyzer_group_by_const.sql         |   2 +-
 .../02996_analyzer_prewhere_projection.sql    |   2 +-
 .../02998_analyzer_prewhere_report.sql        |   2 +-
 ...8_analyzer_secret_args_tree_node.reference |  10 +-
 .../02998_analyzer_secret_args_tree_node.sql  |   2 +-
 ...rojection_after_attach_partition.reference |   4 +-
 ...2998_projection_after_attach_partition.sql |   4 +-
 .../02999_analyzer_preimage_null.sql          |   2 +-
 .../03001_analyzer_nullable_nothing.sql       |   2 +-
 .../0_stateless/03002_analyzer_prewhere.sql   |   2 +-
 .../0_stateless/03003_analyzer_setting.sql    |   8 +-
 .../03003_functions_to_subcolumns_final.sql   |   2 +-
 .../03006_join_on_inequal_expression_2.sql.j2 |   2 +-
 .../03006_join_on_inequal_expression_3.sql.j2 |   2 +-
 .../03006_join_on_inequal_expression_4.sql.j2 |   2 +-
 ...006_join_on_inequal_expression_fast.sql.j2 |   2 +-
 ...llel_replicas_cte_explain_syntax_crash.sql |   2 +-
 ...007_column_nullable_uninitialzed_value.sql |   2 +-
 .../03010_sum_to_to_count_if_nullable.sql     |   4 +-
 ...se_nulls_with_materialize_and_analyzer.sql |   3 +-
 .../03014_analyzer_groupby_fuzz_60317.sql     |   4 +-
 ...nulls_injective_functions_and_analyzer.sql |   3 +-
 .../03015_analyzer_groupby_fuzz_60772.sql     |   4 +-
 .../03016_analyzer_groupby_fuzz_59796.sql     |   2 +-
 .../03017_analyzer_groupby_fuzz_61600.sql     |   4 +-
 ...mize_group_by_function_keys_with_nulls.sql |   3 +-
 ...23_group_by_use_nulls_analyzer_crashes.sql |   4 +-
 .../03023_remove_unused_column_distinct.sql   |   2 +-
 .../03031_filter_float64_logical_error.sql    |   4 +-
 .../03031_tuple_elimination_analyzer.sql      |   2 +-
 .../0_stateless/03032_redundant_equals.sql    |   4 +-
 ...analyzer_merge_engine_filter_push_down.sql |   3 +-
 .../03033_analyzer_query_parameters.sh        |   4 +-
 .../0_stateless/03033_cte_numbers_memory.sql  |   2 +-
 .../03033_recursive_cte_basic.reference       |   2 +-
 .../0_stateless/03033_recursive_cte_basic.sql |   2 +-
 .../03033_with_fill_interpolate.sql           |   2 +-
 .../0_stateless/03034_normalized_ast.sql      |   2 +-
 .../0_stateless/03034_recursive_cte_tree.sql  |   2 +-
 ...3034_recursive_cte_tree_fuzz_crash_fix.sql |   2 +-
 ...34_recursive_cte_tree_merge_tree.reference |   2 +-
 .../03034_recursive_cte_tree_merge_tree.sql   |   2 +-
 .../03035_alias_column_bug_distributed.sql    |   2 +-
 .../03035_internal_functions_direct_call.sql  |   2 +-
 .../03035_recursive_cte_postgres_1.reference  |   2 +-
 .../03035_recursive_cte_postgres_1.sql        |   2 +-
 ..._join_filter_push_down_equivalent_sets.sql |   2 +-
 .../03036_recursive_cte_postgres_2.reference  |   2 +-
 .../03036_recursive_cte_postgres_2.sql        |   2 +-
 .../0_stateless/03036_with_numbers.sql        |   2 +-
 .../03037_recursive_cte_postgres_3.reference  |   2 +-
 .../03037_recursive_cte_postgres_3.sql        |   2 +-
 .../queries/0_stateless/03037_union_view.sql  |   4 +-
 .../0_stateless/03038_ambiguous_column.sql    |   2 +-
 .../03038_recursive_cte_postgres_4.reference  |   2 +-
 .../03038_recursive_cte_postgres_4.sql        |   2 +-
 .../03039_recursive_cte_postgres_5.reference  |   2 +-
 .../03039_recursive_cte_postgres_5.sql        |   2 +-
 ...039_unknown_identifier_window_function.sql |   2 +-
 .../0_stateless/03040_alias_column_join.sql   |   2 +-
 .../0_stateless/03040_array_sum_and_join.sql  |   2 +-
 .../03040_recursive_cte_postgres_6.reference  |   2 +-
 .../03040_recursive_cte_postgres_6.sql        |   2 +-
 .../03041_analyzer_gigachad_join.sql          |   2 +-
 .../03041_recursive_cte_postgres_7.reference  |   2 +-
 .../03041_recursive_cte_postgres_7.sql        |   2 +-
 .../03041_select_with_query_result.sql        |   2 +-
 .../0_stateless/03042_analyzer_alias_join.sql |   2 +-
 .../0_stateless/03042_not_found_column_c1.sql |   2 +-
 .../03043_group_array_result_is_expected.sql  |   2 +-
 .../0_stateless/03044_analyzer_alias_join.sql |   2 +-
 ...044_array_join_columns_in_nested_table.sql |   2 +-
 .../03045_analyzer_alias_join_with_if.sql     |   2 +-
 ..._unknown_identifier_alias_substitution.sql |   2 +-
 .../03046_column_in_block_array_join.sql      |   2 +-
 .../0_stateless/03047_analyzer_alias_join.sql |   2 +-
 ..._group_by_field_identified_aggregation.sql |   2 +-
 .../03048_not_found_column_xxx_in_block.sql   |   2 +-
 .../03049_analyzer_group_by_alias.sql         |   2 +-
 ...unknown_identifier_materialized_column.sql |   2 +-
 .../0_stateless/03050_select_one_one_one.sql  |   2 +-
 tests/queries/0_stateless/03051_many_ctes.sql |   2 +-
 .../03052_query_hash_includes_aliases.sql     |   2 +-
 .../0_stateless/03053_analyzer_join_alias.sql |   2 +-
 .../0_stateless/03054_analyzer_join_alias.sql |   2 +-
 .../03055_analyzer_subquery_group_array.sql   |   2 +-
 .../03057_analyzer_subquery_alias_join.sql    |   2 +-
 .../03058_analyzer_ambiguous_columns.sql      |   3 +-
 ...59_analyzer_join_engine_missing_column.sql |   2 +-
 .../03060_analyzer_regular_view_alias.sql     |   2 +-
 ...61_analyzer_alias_as_right_key_in_join.sql |   2 +-
 ...62_analyzer_join_engine_missing_column.sql |   2 +-
 ...lyzer_multi_join_wrong_table_specifier.sql |   3 +-
 .../03064_analyzer_named_subqueries.sql       |   2 +-
 ...065_analyzer_cross_join_and_array_join.sql |   2 +-
 .../03066_analyzer_global_with_statement.sql  |   2 +-
 .../03067_analyzer_complex_alias_join.sql     |   2 +-
 .../03068_analyzer_distributed_join.sql       |   2 +-
 ...3069_analyzer_with_alias_in_array_join.sql |   2 +-
 .../03070_analyzer_CTE_scalar_as_numbers.sql  |   2 +-
 ...array_join_forbid_non_existing_columns.sql |   2 +-
 .../03071_fix_short_circuit_logic.sql         |   2 +-
 ...analyzer_missing_columns_from_subquery.sql |   2 +-
 .../03073_analyzer_alias_as_column_name.sql   |   2 +-
 .../03074_analyzer_alias_column_in_view.sql   |   2 +-
 .../03075_analyzer_subquery_alias.sql         |   2 +-
 .../03076_analyzer_multiple_joins_alias.sql   |   3 +-
 ...analyzer_multi_scalar_subquery_aliases.sql |   2 +-
 ...analyzer_multi_scalar_subquery_aliases.sql |   2 +-
 ...lyzer_numeric_literals_as_column_names.sql |   2 +-
 ..._column_name_to_alias__virtual_columns.sql |   2 +-
 .../03080_incorrect_join_with_merge.sql       |   2 +-
 .../03081_analyzer_agg_func_CTE.sql           |   2 +-
 ...3082_analyzer_left_join_correct_column.sql |   2 +-
 .../03084_analyzer_join_column_alias.sql      |   2 +-
 .../03085_analyzer_alias_column_group_by.sql  |   2 +-
 ..._analyzer_window_func_part_of_group_by.sql |   2 +-
 .../03087_analyzer_subquery_with_alias.sql    |   2 +-
 ...8_analyzer_ambiguous_column_multi_call.sql |   2 +-
 .../03089_analyzer_alias_replacement.sql      |   2 +-
 ...090_analyzer_multiple_using_statements.sql |   2 +-
 ...same_table_name_in_different_databases.sql |   2 +-
 ...same_table_name_in_different_databases.sql |   2 +-
 .../03093_analyzer_column_alias.sql           |   2 +-
 .../0_stateless/03093_analyzer_miel_test.sql  |   2 +-
 .../03093_bug37909_query_does_not_finish.sql  |   2 +-
 .../03094_analyzer_fiddle_multiif.sql         |   2 +-
 .../03094_named_tuple_bug24607.sql            |   2 +-
 .../0_stateless/03094_one_thousand_joins.sql  |   2 +-
 .../03095_window_functions_qualify.sql        |   2 +-
 ..._text_log_format_string_args_not_empty.sql |   2 +-
 .../03097_query_log_join_processes.sql        |   2 +-
 .../03098_prefer_column_to_alias_subquery.sql |   4 +-
 .../0_stateless/03099_analyzer_multi_join.sql |   2 +-
 .../03100_analyzer_constants_in_multiif.sql   |   2 +-
 .../03101_analyzer_identifiers_1.sql          |   2 +-
 .../03101_analyzer_identifiers_2.sql          |   2 +-
 .../03101_analyzer_identifiers_3.sql          |   2 +-
 .../03101_analyzer_identifiers_4.sql          |   2 +-
 .../03101_analyzer_invalid_join_on.sql        |  14 +-
 .../03102_prefer_column_name_to_alias.sql     |   2 +-
 .../03103_positional_arguments.sql            |   2 +-
 .../0_stateless/03104_create_view_join.sql    |   8 +-
 ...ill_formed_select_in_materialized_view.sql |   4 +-
 .../0_stateless/03108_describe_union_all.sql  |   4 +-
 .../queries/0_stateless/03109_ast_too_big.sql |   4 +-
 .../0_stateless/03110_unicode_alias.sql       |   2 +-
 .../0_stateless/03111_inner_join_group_by.sql |   4 +-
 ...112_analyzer_not_found_column_in_block.sql |   4 +-
 ...3_analyzer_not_found_column_in_block_2.sql |   6 +-
 .../03114_analyzer_cte_with_join.sql          |   2 +-
 .../0_stateless/03115_alias_exists_column.sql |   2 +-
 ...analyzer_explicit_alias_as_column_name.sql |   2 +-
 ...3117_analyzer_same_column_name_as_func.sql |   2 +-
 .../03118_analyzer_multi_join_prewhere.sql    |   2 +-
 ..._analyzer_window_function_in_CTE_alias.sql |   2 +-
 .../0_stateless/03120_analyzer_dist_join.sql  |   2 +-
 .../03120_analyzer_param_in_CTE_alias.sql     |   2 +-
 ...nalyzer_filed_redefenition_in_subquery.sql |   2 +-
 ...22_analyzer_collate_in_window_function.sql |   2 +-
 .../03123_analyzer_dist_join_CTE.sql          |   2 +-
 .../03124_analyzer_nested_CTE_dist_in.sql     |   2 +-
 .../03125_analyzer_CTE_two_joins.sql          |   2 +-
 .../03126_column_not_under_group_by.sql       |   3 +-
 .../0_stateless/03129_cte_with_final.sql      |   2 +-
 .../03130_analyzer_self_join_group_by.sql     |   2 +-
 ...03130_convert_outer_join_to_inner_join.sql |   2 +-
 ...gregate_function_with_if_implicit_cast.sql |   2 +-
 .../0_stateless/03132_sqlancer_union_all.sql  |   2 +-
 .../0_stateless/03142_untuple_crash.sql       |   2 +-
 .../03142_window_function_limit_by.sql        |   7 +-
 tests/queries/0_stateless/03143_cte_scope.sql |   2 +-
 .../03143_group_by_constant_secondary.sql     |   2 +-
 .../03143_parallel_replicas_mat_view_bug.sql  |   2 +-
 ..._aggregate_states_with_different_types.sql |   2 +-
 .../0_stateless/03144_invalid_filter.sql      |   2 +-
 tests/queries/0_stateless/03146_bug47862.sql  |   2 +-
 .../0_stateless/03146_tpc_ds_grouping.sql     |   2 +-
 .../03148_query_log_used_dictionaries.sql     |   8 +-
 ..._streams_to_max_threads_ratio_overflow.sql |   4 +-
 ...03150_grouping_sets_use_nulls_pushdown.sql |   6 +-
 ...lyzer_view_read_only_necessary_columns.sql |   2 +-
 ...in_filter_push_down_equivalent_columns.sql |   2 +-
 .../03154_recursive_cte_distributed.sql       |   2 +-
 .../03155_analyzer_interpolate.sql            |   2 +-
 .../03155_in_nested_subselects.sql            |   8 +-
 .../0_stateless/03161_cnf_reduction.reference |   4 +-
 .../0_stateless/03161_cnf_reduction.sql       |  12 +-
 .../03164_analyzer_global_in_alias.sql        |   2 +-
 .../03164_early_constant_folding_analyzer.sql |   2 +-
 .../03164_materialize_skip_index.sql          |   2 +-
 .../03165_order_by_duplicate.reference        |   2 +-
 .../0_stateless/03165_order_by_duplicate.sql  |   2 +-
 ...03166_mv_prewhere_duplicating_name_bug.sql |   4 +-
 .../03166_skip_indexes_vertical_merge_1.sql   |   2 +-
 .../03167_parametrized_view_with_cte.sql      |   2 +-
 ..._injective_functions_inside_uniq_crash.sql |   4 +-
 .../03170_part_offset_as_table_column.sql     |   4 +-
 .../0_stateless/03171_condition_pushdown.sql  |   2 +-
 .../03171_function_to_subcolumns_fuzzer.sql   |   2 +-
 .../0_stateless/03173_forbid_qualify.sql      |   4 +-
 .../03173_parallel_replicas_join_bug.sh       |   2 +-
 .../0_stateless/03174_merge_join_bug.sql      |   2 +-
 .../03199_join_with_materialized_column.sql   |   2 +-
 .../03199_queries_with_new_analyzer.sql       |   3 +-
 .../03200_memory_engine_alter_dynamic.sql     |   5 +-
 .../03200_subcolumns_join_use_nulls.sql       |   2 +-
 .../03201_sumIf_to_countIf_return_type.sql    |   2 +-
 .../00081_group_by_without_key_and_totals.sql |   3 +-
 .../00172_early_constant_folding.sql          |   4 +-
 .../00173_group_by_use_nulls.reference        |   2 +-
 .../1_stateful/00173_group_by_use_nulls.sql   |   2 +-
 668 files changed, 1903 insertions(+), 1667 deletions(-)

diff --git a/docker/test/fuzzer/query-fuzzer-tweaks-users.xml b/docker/test/fuzzer/query-fuzzer-tweaks-users.xml
index d5b876a4c85..476464e9cc2 100644
--- a/docker/test/fuzzer/query-fuzzer-tweaks-users.xml
+++ b/docker/test/fuzzer/query-fuzzer-tweaks-users.xml
@@ -28,9 +28,9 @@
                 </table_function_remote_max_addresses>
 
                 <!-- Don't waste cycles testing the old interpreter. Spend time in the new analyzer instead -->
-                <allow_experimental_analyzer>
+                <enable_analyzer>
                     <readonly/>
-                </allow_experimental_analyzer>
+                </enable_analyzer>
 
                 <!-- This feature is broken, deprecated and will be removed. We don't want more reports about it -->
                 <allow_experimental_object_type>
diff --git a/docker/test/stateless/stress_tests.lib b/docker/test/stateless/stress_tests.lib
index 36782101fa7..73e0376d95a 100644
--- a/docker/test/stateless/stress_tests.lib
+++ b/docker/test/stateless/stress_tests.lib
@@ -139,9 +139,9 @@ EOL
                 </table_function_remote_max_addresses>
 
                 <!-- Don't waste cycles testing the old interpreter. Spend time in the new analyzer instead -->
-                <allow_experimental_analyzer>
+                <enable_analyzer>
                     <readonly/>
-                </allow_experimental_analyzer>
+                </enable_analyzer>
 
                 <!-- This feature is broken, deprecated and will be removed. We don't want more reports about it -->
                 <allow_experimental_object_type>
diff --git a/docs/en/development/architecture.md b/docs/en/development/architecture.md
index 6428c0e90d5..c5d13ab63a5 100644
--- a/docs/en/development/architecture.md
+++ b/docs/en/development/architecture.md
@@ -118,7 +118,7 @@ And the result of interpreting the `INSERT SELECT` query is a "completed" `Query
 
 `InterpreterSelectQuery` uses `ExpressionAnalyzer` and `ExpressionActions` machinery for query analysis and transformations. This is where most rule-based query optimizations are performed. `ExpressionAnalyzer` is quite messy and should be rewritten: various query transformations and optimizations should be extracted into separate classes to allow for modular transformations of the query.
 
-To address current problems that exist in interpreters, a new `InterpreterSelectQueryAnalyzer` is being developed. It is a new version of `InterpreterSelectQuery` that does not use `ExpressionAnalyzer` and introduces an additional abstraction level between `AST` and `QueryPipeline` called `QueryTree`. It is not production-ready yet, but it can be tested with the `allow_experimental_analyzer` flag.
+To address current problems that exist in interpreters, a new `InterpreterSelectQueryAnalyzer` is being developed. It is a new version of `InterpreterSelectQuery` that does not use `ExpressionAnalyzer` and introduces an additional abstraction level between `AST` and `QueryPipeline` called `QueryTree`. It is not production-ready yet, but it can be tested with the `enable_analyzer` flag.
 
 ## Functions {#functions}
 
diff --git a/docs/en/operations/analyzer.md b/docs/en/operations/analyzer.md
index 298c6dacd06..c9b3c30d30d 100644
--- a/docs/en/operations/analyzer.md
+++ b/docs/en/operations/analyzer.md
@@ -123,7 +123,7 @@ To ensure consistent and expected results, especially when migrating old queries
 In the new version of the analyzer, the rules for determining the common supertype for columns specified in the `USING` clause have been standardized to produce more predictable outcomes, especially when dealing with type modifiers like `LowCardinality` and `Nullable`.
 
 - `LowCardinality(T)` and `T`: When a column of type `LowCardinality(T)` is joined with a column of type `T`, the resulting common supertype will be `T`, effectively discarding the `LowCardinality` modifier.
-  
+
 - `Nullable(T)` and `T`: When a column of type `Nullable(T)` is joined with a column of type `T`, the resulting common supertype will be `Nullable(T)`, ensuring that the nullable property is preserved.
 
 **Example:**
@@ -144,7 +144,7 @@ During projection names computation, aliases are not substituted.
 SELECT
     1 + 1 AS x,
     x + 1
-SETTINGS allow_experimental_analyzer = 0
+SETTINGS enable_analyzer = 0
 FORMAT PrettyCompact
 
    ┌─x─┬─plus(plus(1, 1), 1)─┐
@@ -154,7 +154,7 @@ FORMAT PrettyCompact
 SELECT
     1 + 1 AS x,
     x + 1
-SETTINGS allow_experimental_analyzer = 1
+SETTINGS enable_analyzer = 1
 FORMAT PrettyCompact
 
    ┌─x─┬─plus(x, 1)─┐
@@ -177,7 +177,7 @@ SELECT toTypeName(if(0, [2, 3, 4], 'String'))
 
 ### Heterogeneous clusters
 
-The new analyzer significantly changed the communication protocol between servers in the cluster. Thus, it's impossible to run distributed queries on servers with different `allow_experimental_analyzer` setting values.
+The new analyzer significantly changed the communication protocol between servers in the cluster. Thus, it's impossible to run distributed queries on servers with different `enable_analyzer` setting values.
 
 ### Mutations are interpreted by previous analyzer
 
diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md
index 143ce836beb..35547c3a9a6 100644
--- a/docs/en/operations/settings/settings.md
+++ b/docs/en/operations/settings/settings.md
@@ -4051,7 +4051,7 @@ Rewrite aggregate functions with if expression as argument when logically equiva
 For example, `avg(if(cond, col, null))` can be rewritten to `avgOrNullIf(cond, col)`. It may improve performance.
 
 :::note
-Supported only with experimental analyzer (`allow_experimental_analyzer = 1`).
+Supported only with experimental analyzer (`enable_analyzer = 1`).
 :::
 
 ## database_replicated_initial_query_timeout_sec {#database_replicated_initial_query_timeout_sec}
diff --git a/docs/ru/development/architecture.md b/docs/ru/development/architecture.md
index 575799cccc4..0701c8f4a51 100644
--- a/docs/ru/development/architecture.md
+++ b/docs/ru/development/architecture.md
@@ -115,7 +115,7 @@ ClickHouse — полноценная столбцовая СУБД. Данны
 
 `InterpreterSelectQuery` использует `ExpressionAnalyzer` и `ExpressionActions` механизмы для анализа запросов и преобразований. Именно здесь выполняется большинство оптимизаций запросов на основе правил. `ExpressionAnalyzer` написан довольно грязно и должен быть переписан: различные преобразования запросов и оптимизации должны быть извлечены в отдельные классы, чтобы позволить модульные преобразования или запросы.
 
-Для решения текущих проблем, существующих в интерпретаторах, разрабатывается новый `InterpreterSelectQueryAnalyzer`. Это новая версия `InterpreterSelectQuery`, которая не использует `ExpressionAnalyzer` и вводит дополнительный уровень абстракции между `AST` и `QueryPipeline`, называемый `QueryTree`. Он еще не готов к использованию в продакшене, но его можно протестировать с помощью флага `allow_experimental_analyzer`.
+Для решения текущих проблем, существующих в интерпретаторах, разрабатывается новый `InterpreterSelectQueryAnalyzer`. Это новая версия `InterpreterSelectQuery`, которая не использует `ExpressionAnalyzer` и вводит дополнительный уровень абстракции между `AST` и `QueryPipeline`, называемый `QueryTree`. Он еще не готов к использованию в продакшене, но его можно протестировать с помощью флага `enable_analyzer`.
 
 ## Функции {#functions}
 
diff --git a/src/Client/HedgedConnections.cpp b/src/Client/HedgedConnections.cpp
index dd8348ea04f..1c7f222aa78 100644
--- a/src/Client/HedgedConnections.cpp
+++ b/src/Client/HedgedConnections.cpp
@@ -196,11 +196,11 @@ void HedgedConnections::sendQuery(
             modified_settings.parallel_replica_offset = fd_to_replica_location[replica.packet_receiver->getFileDescriptor()].offset;
         }
 
-        /// FIXME: Remove once we will make `allow_experimental_analyzer` obsolete setting.
+        /// FIXME: Remove once we will make `enable_analyzer` obsolete setting.
         /// Make the analyzer being set, so it will be effectively applied on the remote server.
         /// In other words, the initiator always controls whether the analyzer enabled or not for
         /// all servers involved in the distributed query processing.
-        modified_settings.set("allow_experimental_analyzer", static_cast<bool>(modified_settings.allow_experimental_analyzer));
+        modified_settings.set("enable_analyzer", static_cast<bool>(modified_settings.enable_analyzer));
 
         replica.connection->sendQuery(
             timeouts, query, /* query_parameters */ {}, query_id, stage, &modified_settings, &client_info, with_pending_data, {});
diff --git a/src/Client/MultiplexedConnections.cpp b/src/Client/MultiplexedConnections.cpp
index 244eccf1ed9..7ca22ae4c81 100644
--- a/src/Client/MultiplexedConnections.cpp
+++ b/src/Client/MultiplexedConnections.cpp
@@ -150,11 +150,11 @@ void MultiplexedConnections::sendQuery(
         client_info.number_of_current_replica = replica_info->number_of_current_replica;
     }
 
-    /// FIXME: Remove once we will make `allow_experimental_analyzer` obsolete setting.
+    /// FIXME: Remove once we will make `enable_analyzer` obsolete setting.
     /// Make the analyzer being set, so it will be effectively applied on the remote server.
     /// In other words, the initiator always controls whether the analyzer enabled or not for
     /// all servers involved in the distributed query processing.
-    modified_settings.set("allow_experimental_analyzer", static_cast<bool>(modified_settings.allow_experimental_analyzer));
+    modified_settings.set("enable_analyzer", static_cast<bool>(modified_settings.enable_analyzer));
 
     const bool enable_offset_parallel_processing = context->canUseOffsetParallelReplicas();
 
diff --git a/src/Core/Settings.h b/src/Core/Settings.h
index 0d498ce7699..d6c0dc223b2 100644
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@@ -638,7 +638,7 @@ class IColumn;
     M(Bool, allow_non_metadata_alters, true, "Allow to execute alters which affects not only tables metadata, but also data on disk", 0) \
     M(Bool, enable_global_with_statement, true, "Propagate WITH statements to UNION queries and all subqueries", 0) \
     M(Bool, aggregate_functions_null_for_empty, false, "Rewrite all aggregate functions in a query, adding -OrNull suffix to them", 0) \
-    M(Bool, optimize_syntax_fuse_functions, false, "Allow apply fuse aggregating function. Available only with `allow_experimental_analyzer`", 0) \
+    M(Bool, optimize_syntax_fuse_functions, false, "Allow apply fuse aggregating function. Available only with `enable_analyzer`", 0) \
     M(Bool, flatten_nested, true, "If true, columns of type Nested will be flatten to separate array columns instead of one array of tuples", 0) \
     M(Bool, asterisk_include_materialized_columns, false, "Include MATERIALIZED columns for wildcard query", 0) \
     M(Bool, asterisk_include_alias_columns, false, "Include ALIAS columns for wildcard query", 0) \
@@ -943,8 +943,7 @@ class IColumn;
     \
     M(Bool, allow_experimental_join_condition, false, "Support join with inequal conditions which involve columns from both left and right table. e.g. t1.y < t2.y.", 0) \
     \
-    /* Analyzer: It's not experimental anymore (WIP) */ \
-    M(Bool, allow_experimental_analyzer, true, "Allow new query analyzer.", IMPORTANT) \
+    M(Bool, enable_analyzer, true, "Allow new query analyzer.", IMPORTANT) ALIAS(allow_experimental_analyzer) \
     M(Bool, analyzer_compatibility_join_using_top_level_identifier, false, "Force to resolve identifier in JOIN USING from projection (for example, in `SELECT a + 1 AS b FROM t1 JOIN t2 USING (b)` join will be performed by `t1.a + 1 = t2.b`, rather then `t1.b = t2.b`).", 0) \
     \
     M(Bool, allow_experimental_live_view, false, "Enable LIVE VIEW. Not mature enough.", 0) \
diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp
index 5b94391bade..08fb6dc3301 100644
--- a/src/Core/SettingsChangesHistory.cpp
+++ b/src/Core/SettingsChangesHistory.cpp
@@ -500,6 +500,265 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
             {"enable_optimize_predicate_expression", 0, 1, "Optimize predicates to subqueries by default"}
         }
     },
+    {"24.7", {{"output_format_parquet_write_page_index", false, true, "Add a possibility to write page index into parquet files."},
+              {"output_format_binary_encode_types_in_binary_format", false, false, "Added new setting to allow to write type names in binary format in RowBinaryWithNamesAndTypes output format"},
+              {"input_format_binary_decode_types_in_binary_format", false, false, "Added new setting to allow to read type names in binary format in RowBinaryWithNamesAndTypes input format"},
+              {"output_format_native_encode_types_in_binary_format", false, false, "Added new setting to allow to write type names in binary format in Native output format"},
+              {"input_format_native_decode_types_in_binary_format", false, false, "Added new setting to allow to read type names in binary format in Native output format"},
+              {"read_in_order_use_buffering", false, true, "Use buffering before merging while reading in order of primary key"},
+              {"enable_named_columns_in_function_tuple", false, true, "Generate named tuples in function tuple() when all names are unique and can be treated as unquoted identifiers."},
+              {"input_format_json_case_insensitive_column_matching", false, false, "Ignore case when matching JSON keys with CH columns."},
+              {"optimize_trivial_insert_select", true, false, "The optimization does not make sense in many cases."},
+              {"dictionary_validate_primary_key_type", false, false, "Validate primary key type for dictionaries. By default id type for simple layouts will be implicitly converted to UInt64."},
+              {"collect_hash_table_stats_during_joins", false, true, "New setting."},
+              {"max_size_to_preallocate_for_joins", 0, 100'000'000, "New setting."},
+              {"input_format_orc_reader_time_zone_name", "GMT", "GMT", "The time zone name for ORC row reader, the default ORC row reader's time zone is GMT."},
+              {"lightweight_mutation_projection_mode", "throw", "throw", "When lightweight delete happens on a table with projection(s), the possible operations include throw the exception as projection exists, or drop all projection related to this table then do lightweight delete."},
+              {"database_replicated_allow_heavy_create", true, false, "Long-running DDL queries (CREATE AS SELECT and POPULATE) for Replicated database engine was forbidden"},
+              {"query_plan_merge_filters", false, false, "Allow to merge filters in the query plan"},
+              {"azure_sdk_max_retries", 10, 10, "Maximum number of retries in azure sdk"},
+              {"azure_sdk_retry_initial_backoff_ms", 10, 10, "Minimal backoff between retries in azure sdk"},
+              {"azure_sdk_retry_max_backoff_ms", 1000, 1000, "Maximal backoff between retries in azure sdk"},
+              {"merge_tree_min_bytes_per_task_for_remote_reading", 4194304, 2097152, "Value is unified with `filesystem_prefetch_min_bytes_for_single_read_task`"},
+              {"ignore_on_cluster_for_replicated_named_collections_queries", false, false, "Ignore ON CLUSTER clause for replicated named collections management queries."},
+              {"backup_restore_s3_retry_attempts", 1000,1000, "Setting for Aws::Client::RetryStrategy, Aws::Client does retries itself, 0 means no retries. It takes place only for backup/restore."},
+              {"postgresql_connection_attempt_timeout", 2, 2, "Allow to control 'connect_timeout' parameter of PostgreSQL connection."},
+              {"postgresql_connection_pool_retries", 2, 2, "Allow to control the number of retries in PostgreSQL connection pool."}
+              }},
+    {"24.6", {{"materialize_skip_indexes_on_insert", true, true, "Added new setting to allow to disable materialization of skip indexes on insert"},
+              {"materialize_statistics_on_insert", true, true, "Added new setting to allow to disable materialization of statistics on insert"},
+              {"input_format_parquet_use_native_reader", false, false, "When reading Parquet files, to use native reader instead of arrow reader."},
+              {"hdfs_throw_on_zero_files_match", false, false, "Allow to throw an error when ListObjects request cannot match any files in HDFS engine instead of empty query result"},
+              {"azure_throw_on_zero_files_match", false, false, "Allow to throw an error when ListObjects request cannot match any files in AzureBlobStorage engine instead of empty query result"},
+              {"s3_validate_request_settings", true, true, "Allow to disable S3 request settings validation"},
+              {"allow_experimental_full_text_index", false, false, "Enable experimental full-text index"},
+              {"azure_skip_empty_files", false, false, "Allow to skip empty files in azure table engine"},
+              {"hdfs_ignore_file_doesnt_exist", false, false, "Allow to return 0 rows when the requested files don't exist instead of throwing an exception in HDFS table engine"},
+              {"azure_ignore_file_doesnt_exist", false, false, "Allow to return 0 rows when the requested files don't exist instead of throwing an exception in AzureBlobStorage table engine"},
+              {"s3_ignore_file_doesnt_exist", false, false, "Allow to return 0 rows when the requested files don't exist instead of throwing an exception in S3 table engine"},
+              {"s3_max_part_number", 10000, 10000, "Maximum part number number for s3 upload part"},
+              {"s3_max_single_operation_copy_size", 32 * 1024 * 1024, 32 * 1024 * 1024, "Maximum size for a single copy operation in s3"},
+              {"input_format_parquet_max_block_size", 8192, DEFAULT_BLOCK_SIZE, "Increase block size for parquet reader."},
+              {"input_format_parquet_prefer_block_bytes", 0, DEFAULT_BLOCK_SIZE * 256, "Average block bytes output by parquet reader."},
+              {"enable_blob_storage_log", true, true, "Write information about blob storage operations to system.blob_storage_log table"},
+              {"allow_deprecated_snowflake_conversion_functions", true, false, "Disabled deprecated functions snowflakeToDateTime[64] and dateTime[64]ToSnowflake."},
+              {"allow_statistic_optimize", false, false, "Old setting which popped up here being renamed."},
+              {"allow_experimental_statistic", false, false, "Old setting which popped up here being renamed."},
+              {"allow_statistics_optimize", false, false, "The setting was renamed. The previous name is `allow_statistic_optimize`."},
+              {"allow_experimental_statistics", false, false, "The setting was renamed. The previous name is `allow_experimental_statistic`."},
+              {"enable_vertical_final", false, true, "Enable vertical final by default again after fixing bug"},
+              {"parallel_replicas_custom_key_range_lower", 0, 0, "Add settings to control the range filter when using parallel replicas with dynamic shards"},
+              {"parallel_replicas_custom_key_range_upper", 0, 0, "Add settings to control the range filter when using parallel replicas with dynamic shards. A value of 0 disables the upper limit"},
+              {"output_format_pretty_display_footer_column_names", 0, 1, "Add a setting to display column names in the footer if there are many rows. Threshold value is controlled by output_format_pretty_display_footer_column_names_min_rows."},
+              {"output_format_pretty_display_footer_column_names_min_rows", 0, 50, "Add a setting to control the threshold value for setting output_format_pretty_display_footer_column_names_min_rows. Default 50."},
+              {"output_format_csv_serialize_tuple_into_separate_columns", true, true, "A new way of how interpret tuples in CSV format was added."},
+              {"input_format_csv_deserialize_separate_columns_into_tuple", true, true, "A new way of how interpret tuples in CSV format was added."},
+              {"input_format_csv_try_infer_strings_from_quoted_tuples", true, true, "A new way of how interpret tuples in CSV format was added."},
+              }},
+    {"24.5", {{"allow_deprecated_error_prone_window_functions", true, false, "Allow usage of deprecated error prone window functions (neighbor, runningAccumulate, runningDifferenceStartingWithFirstValue, runningDifference)"},
+              {"allow_experimental_join_condition", false, false, "Support join with inequal conditions which involve columns from both left and right table. e.g. t1.y < t2.y."},
+              {"input_format_tsv_crlf_end_of_line", false, false, "Enables reading of CRLF line endings with TSV formats"},
+              {"output_format_parquet_use_custom_encoder", false, true, "Enable custom Parquet encoder."},
+              {"cross_join_min_rows_to_compress", 0, 10000000, "Minimal count of rows to compress block in CROSS JOIN. Zero value means - disable this threshold. This block is compressed when any of the two thresholds (by rows or by bytes) are reached."},
+              {"cross_join_min_bytes_to_compress", 0, 1_GiB, "Minimal size of block to compress in CROSS JOIN. Zero value means - disable this threshold. This block is compressed when any of the two thresholds (by rows or by bytes) are reached."},
+              {"http_max_chunk_size", 0, 0, "Internal limitation"},
+              {"prefer_external_sort_block_bytes", 0, DEFAULT_BLOCK_SIZE * 256, "Prefer maximum block bytes for external sort, reduce the memory usage during merging."},
+              {"input_format_force_null_for_omitted_fields", false, false, "Disable type-defaults for omitted fields when needed"},
+              {"cast_string_to_dynamic_use_inference", false, false, "Add setting to allow converting String to Dynamic through parsing"},
+              {"allow_experimental_dynamic_type", false, false, "Add new experimental Dynamic type"},
+              {"azure_max_blocks_in_multipart_upload", 50000, 50000, "Maximum number of blocks in multipart upload for Azure."},
+              }},
+    {"24.4", {{"input_format_json_throw_on_bad_escape_sequence", true, true, "Allow to save JSON strings with bad escape sequences"},
+              {"max_parsing_threads", 0, 0, "Add a separate setting to control number of threads in parallel parsing from files"},
+              {"ignore_drop_queries_probability", 0, 0, "Allow to ignore drop queries in server with specified probability for testing purposes"},
+              {"lightweight_deletes_sync", 2, 2, "The same as 'mutation_sync', but controls only execution of lightweight deletes"},
+              {"query_cache_system_table_handling", "save", "throw", "The query cache no longer caches results of queries against system tables"},
+              {"input_format_json_ignore_unnecessary_fields", false, true, "Ignore unnecessary fields and not parse them. Enabling this may not throw exceptions on json strings of invalid format or with duplicated fields"},
+              {"input_format_hive_text_allow_variable_number_of_columns", false, true, "Ignore extra columns in Hive Text input (if file has more columns than expected) and treat missing fields in Hive Text input as default values."},
+              {"allow_experimental_database_replicated", false, true, "Database engine Replicated is now in Beta stage"},
+              {"temporary_data_in_cache_reserve_space_wait_lock_timeout_milliseconds", (10 * 60 * 1000), (10 * 60 * 1000), "Wait time to lock cache for sapce reservation in temporary data in filesystem cache"},
+              {"optimize_rewrite_sum_if_to_count_if", false, true, "Only available for the analyzer, where it works correctly"},
+              {"azure_allow_parallel_part_upload", "true", "true", "Use multiple threads for azure multipart upload."},
+              {"max_recursive_cte_evaluation_depth", DBMS_RECURSIVE_CTE_MAX_EVALUATION_DEPTH, DBMS_RECURSIVE_CTE_MAX_EVALUATION_DEPTH, "Maximum limit on recursive CTE evaluation depth"},
+              {"query_plan_convert_outer_join_to_inner_join", false, true, "Allow to convert OUTER JOIN to INNER JOIN if filter after JOIN always filters default values"},
+              }},
+    {"24.3", {{"s3_connect_timeout_ms", 1000, 1000, "Introduce new dedicated setting for s3 connection timeout"},
+              {"allow_experimental_shared_merge_tree", false, true, "The setting is obsolete"},
+              {"use_page_cache_for_disks_without_file_cache", false, false, "Added userspace page cache"},
+              {"read_from_page_cache_if_exists_otherwise_bypass_cache", false, false, "Added userspace page cache"},
+              {"page_cache_inject_eviction", false, false, "Added userspace page cache"},
+              {"default_table_engine", "None", "MergeTree", "Set default table engine to MergeTree for better usability"},
+              {"input_format_json_use_string_type_for_ambiguous_paths_in_named_tuples_inference_from_objects", false, false, "Allow to use String type for ambiguous paths during named tuple inference from JSON objects"},
+              {"traverse_shadow_remote_data_paths", false, false, "Traverse shadow directory when query system.remote_data_paths."},
+              {"throw_if_deduplication_in_dependent_materialized_views_enabled_with_async_insert", false, true, "Deduplication in dependent materialized view cannot work together with async inserts."},
+              {"parallel_replicas_allow_in_with_subquery", false, true, "If true, subquery for IN will be executed on every follower replica"},
+              {"log_processors_profiles", false, true, "Enable by default"},
+              {"function_locate_has_mysql_compatible_argument_order", false, true, "Increase compatibility with MySQL's locate function."},
+              {"allow_suspicious_primary_key", true, false, "Forbid suspicious PRIMARY KEY/ORDER BY for MergeTree (i.e. SimpleAggregateFunction)"},
+              {"filesystem_cache_reserve_space_wait_lock_timeout_milliseconds", 1000, 1000, "Wait time to lock cache for sapce reservation in filesystem cache"},
+              {"max_parser_backtracks", 0, 1000000, "Limiting the complexity of parsing"},
+              {"analyzer_compatibility_join_using_top_level_identifier", false, false, "Force to resolve identifier in JOIN USING from projection"},
+              {"distributed_insert_skip_read_only_replicas", false, false, "If true, INSERT into Distributed will skip read-only replicas"},
+              {"keeper_max_retries", 10, 10, "Max retries for general keeper operations"},
+              {"keeper_retry_initial_backoff_ms", 100, 100, "Initial backoff timeout for general keeper operations"},
+              {"keeper_retry_max_backoff_ms", 5000, 5000, "Max backoff timeout for general keeper operations"},
+              {"s3queue_allow_experimental_sharded_mode", false, false, "Enable experimental sharded mode of S3Queue table engine. It is experimental because it will be rewritten"},
+              {"enable_analyzer", false, true, "Enable analyzer and planner by default."},
+              {"merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability", 0.0, 0.0, "For testing of `PartsSplitter` - split read ranges into intersecting and non intersecting every time you read from MergeTree with the specified probability."},
+              {"allow_get_client_http_header", false, false, "Introduced a new function."},
+              {"output_format_pretty_row_numbers", false, true, "It is better for usability."},
+              {"output_format_pretty_max_value_width_apply_for_single_value", true, false, "Single values in Pretty formats won't be cut."},
+              {"output_format_parquet_string_as_string", false, true, "ClickHouse allows arbitrary binary data in the String data type, which is typically UTF-8. Parquet/ORC/Arrow Strings only support UTF-8. That's why you can choose which Arrow's data type to use for the ClickHouse String data type - String or Binary. While Binary would be more correct and compatible, using String by default will correspond to user expectations in most cases."},
+              {"output_format_orc_string_as_string", false, true, "ClickHouse allows arbitrary binary data in the String data type, which is typically UTF-8. Parquet/ORC/Arrow Strings only support UTF-8. That's why you can choose which Arrow's data type to use for the ClickHouse String data type - String or Binary. While Binary would be more correct and compatible, using String by default will correspond to user expectations in most cases."},
+              {"output_format_arrow_string_as_string", false, true, "ClickHouse allows arbitrary binary data in the String data type, which is typically UTF-8. Parquet/ORC/Arrow Strings only support UTF-8. That's why you can choose which Arrow's data type to use for the ClickHouse String data type - String or Binary. While Binary would be more correct and compatible, using String by default will correspond to user expectations in most cases."},
+              {"output_format_parquet_compression_method", "lz4", "zstd", "Parquet/ORC/Arrow support many compression methods, including lz4 and zstd. ClickHouse supports each and every compression method. Some inferior tools, such as 'duckdb', lack support for the faster `lz4` compression method, that's why we set zstd by default."},
+              {"output_format_orc_compression_method", "lz4", "zstd", "Parquet/ORC/Arrow support many compression methods, including lz4 and zstd. ClickHouse supports each and every compression method. Some inferior tools, such as 'duckdb', lack support for the faster `lz4` compression method, that's why we set zstd by default."},
+              {"output_format_pretty_highlight_digit_groups", false, true, "If enabled and if output is a terminal, highlight every digit corresponding to the number of thousands, millions, etc. with underline."},
+              {"geo_distance_returns_float64_on_float64_arguments", false, true, "Increase the default precision."},
+              {"azure_max_inflight_parts_for_one_file", 20, 20, "The maximum number of a concurrent loaded parts in multipart upload request. 0 means unlimited."},
+              {"azure_strict_upload_part_size", 0, 0, "The exact size of part to upload during multipart upload to Azure blob storage."},
+              {"azure_min_upload_part_size", 16*1024*1024, 16*1024*1024, "The minimum size of part to upload during multipart upload to Azure blob storage."},
+              {"azure_max_upload_part_size", 5ull*1024*1024*1024, 5ull*1024*1024*1024, "The maximum size of part to upload during multipart upload to Azure blob storage."},
+              {"azure_upload_part_size_multiply_factor", 2, 2, "Multiply azure_min_upload_part_size by this factor each time azure_multiply_parts_count_threshold parts were uploaded from a single write to Azure blob storage."},
+              {"azure_upload_part_size_multiply_parts_count_threshold", 500, 500, "Each time this number of parts was uploaded to Azure blob storage, azure_min_upload_part_size is multiplied by azure_upload_part_size_multiply_factor."},
+              {"output_format_csv_serialize_tuple_into_separate_columns", true, true, "A new way of how interpret tuples in CSV format was added."},
+              {"input_format_csv_deserialize_separate_columns_into_tuple", true, true, "A new way of how interpret tuples in CSV format was added."},
+              {"input_format_csv_try_infer_strings_from_quoted_tuples", true, true, "A new way of how interpret tuples in CSV format was added."},
+              }},
+    {"24.2", {{"allow_suspicious_variant_types", true, false, "Don't allow creating Variant type with suspicious variants by default"},
+              {"validate_experimental_and_suspicious_types_inside_nested_types", false, true, "Validate usage of experimental and suspicious types inside nested types"},
+              {"output_format_values_escape_quote_with_quote", false, false, "If true escape ' with '', otherwise quoted with \\'"},
+              {"output_format_pretty_single_large_number_tip_threshold", 0, 1'000'000, "Print a readable number tip on the right side of the table if the block consists of a single number which exceeds this value (except 0)"},
+              {"input_format_try_infer_exponent_floats", true, false, "Don't infer floats in exponential notation by default"},
+              {"query_plan_optimize_prewhere", true, true, "Allow to push down filter to PREWHERE expression for supported storages"},
+              {"async_insert_max_data_size", 1000000, 10485760, "The previous value appeared to be too small."},
+              {"async_insert_poll_timeout_ms", 10, 10, "Timeout in milliseconds for polling data from asynchronous insert queue"},
+              {"async_insert_use_adaptive_busy_timeout", false, true, "Use adaptive asynchronous insert timeout"},
+              {"async_insert_busy_timeout_min_ms", 50, 50, "The minimum value of the asynchronous insert timeout in milliseconds; it also serves as the initial value, which may be increased later by the adaptive algorithm"},
+              {"async_insert_busy_timeout_max_ms", 200, 200, "The minimum value of the asynchronous insert timeout in milliseconds; async_insert_busy_timeout_ms is aliased to async_insert_busy_timeout_max_ms"},
+              {"async_insert_busy_timeout_increase_rate", 0.2, 0.2, "The exponential growth rate at which the adaptive asynchronous insert timeout increases"},
+              {"async_insert_busy_timeout_decrease_rate", 0.2, 0.2, "The exponential growth rate at which the adaptive asynchronous insert timeout decreases"},
+              {"format_template_row_format", "", "", "Template row format string can be set directly in query"},
+              {"format_template_resultset_format", "", "", "Template result set format string can be set in query"},
+              {"split_parts_ranges_into_intersecting_and_non_intersecting_final", true, true, "Allow to split parts ranges into intersecting and non intersecting during FINAL optimization"},
+              {"split_intersecting_parts_ranges_into_layers_final", true, true, "Allow to split intersecting parts ranges into layers during FINAL optimization"},
+              {"azure_max_single_part_copy_size", 256*1024*1024, 256*1024*1024, "The maximum size of object to copy using single part copy to Azure blob storage."},
+              {"min_external_table_block_size_rows", DEFAULT_INSERT_BLOCK_SIZE, DEFAULT_INSERT_BLOCK_SIZE, "Squash blocks passed to external table to specified size in rows, if blocks are not big enough"},
+              {"min_external_table_block_size_bytes", DEFAULT_INSERT_BLOCK_SIZE * 256, DEFAULT_INSERT_BLOCK_SIZE * 256, "Squash blocks passed to external table to specified size in bytes, if blocks are not big enough."},
+              {"parallel_replicas_prefer_local_join", true, true, "If true, and JOIN can be executed with parallel replicas algorithm, and all storages of right JOIN part are *MergeTree, local JOIN will be used instead of GLOBAL JOIN."},
+              {"optimize_time_filter_with_preimage", true, true, "Optimize Date and DateTime predicates by converting functions into equivalent comparisons without conversions (e.g. toYear(col) = 2023 -> col >= '2023-01-01' AND col <= '2023-12-31')"},
+              {"extract_key_value_pairs_max_pairs_per_row", 0, 0, "Max number of pairs that can be produced by the `extractKeyValuePairs` function. Used as a safeguard against consuming too much memory."},
+              {"default_view_definer", "CURRENT_USER", "CURRENT_USER", "Allows to set default `DEFINER` option while creating a view"},
+              {"default_materialized_view_sql_security", "DEFINER", "DEFINER", "Allows to set a default value for SQL SECURITY option when creating a materialized view"},
+              {"default_normal_view_sql_security", "INVOKER", "INVOKER", "Allows to set default `SQL SECURITY` option while creating a normal view"},
+              {"mysql_map_string_to_text_in_show_columns", false, true, "Reduce the configuration effort to connect ClickHouse with BI tools."},
+              {"mysql_map_fixed_string_to_text_in_show_columns", false, true, "Reduce the configuration effort to connect ClickHouse with BI tools."},
+              }},
+    {"24.1", {{"print_pretty_type_names", false, true, "Better user experience."},
+              {"input_format_json_read_bools_as_strings", false, true, "Allow to read bools as strings in JSON formats by default"},
+              {"output_format_arrow_use_signed_indexes_for_dictionary", false, true, "Use signed indexes type for Arrow dictionaries by default as it's recommended"},
+              {"allow_experimental_variant_type", false, false, "Add new experimental Variant type"},
+              {"use_variant_as_common_type", false, false, "Allow to use Variant in if/multiIf if there is no common type"},
+              {"output_format_arrow_use_64_bit_indexes_for_dictionary", false, false, "Allow to use 64 bit indexes type in Arrow dictionaries"},
+              {"parallel_replicas_mark_segment_size", 128, 128, "Add new setting to control segment size in new parallel replicas coordinator implementation"},
+              {"ignore_materialized_views_with_dropped_target_table", false, false, "Add new setting to allow to ignore materialized views with dropped target table"},
+              {"output_format_compression_level", 3, 3, "Allow to change compression level in the query output"},
+              {"output_format_compression_zstd_window_log", 0, 0, "Allow to change zstd window log in the query output when zstd compression is used"},
+              {"enable_zstd_qat_codec", false, false, "Add new ZSTD_QAT codec"},
+              {"enable_vertical_final", false, true, "Use vertical final by default"},
+              {"output_format_arrow_use_64_bit_indexes_for_dictionary", false, false, "Allow to use 64 bit indexes type in Arrow dictionaries"},
+              {"max_rows_in_set_to_optimize_join", 100000, 0, "Disable join optimization as it prevents from read in order optimization"},
+              {"output_format_pretty_color", true, "auto", "Setting is changed to allow also for auto value, disabling ANSI escapes if output is not a tty"},
+              {"function_visible_width_behavior", 0, 1, "We changed the default behavior of `visibleWidth` to be more precise"},
+              {"max_estimated_execution_time", 0, 0, "Separate max_execution_time and max_estimated_execution_time"},
+              {"iceberg_engine_ignore_schema_evolution", false, false, "Allow to ignore schema evolution in Iceberg table engine"},
+              {"optimize_injective_functions_in_group_by", false, true, "Replace injective functions by it's arguments in GROUP BY section in analyzer"},
+              {"update_insert_deduplication_token_in_dependent_materialized_views", false, false, "Allow to update insert deduplication token with table identifier during insert in dependent materialized views"},
+              {"azure_max_unexpected_write_error_retries", 4, 4, "The maximum number of retries in case of unexpected errors during Azure blob storage write"},
+              {"split_parts_ranges_into_intersecting_and_non_intersecting_final", false, true, "Allow to split parts ranges into intersecting and non intersecting during FINAL optimization"},
+              {"split_intersecting_parts_ranges_into_layers_final", true, true, "Allow to split intersecting parts ranges into layers during FINAL optimization"}}},
+    {"23.12", {{"allow_suspicious_ttl_expressions", true, false, "It is a new setting, and in previous versions the behavior was equivalent to allowing."},
+              {"input_format_parquet_allow_missing_columns", false, true, "Allow missing columns in Parquet files by default"},
+              {"input_format_orc_allow_missing_columns", false, true, "Allow missing columns in ORC files by default"},
+              {"input_format_arrow_allow_missing_columns", false, true, "Allow missing columns in Arrow files by default"}}},
+    {"23.11", {{"parsedatetime_parse_without_leading_zeros", false, true, "Improved compatibility with MySQL DATE_FORMAT/STR_TO_DATE"}}},
+    {"23.9", {{"optimize_group_by_constant_keys", false, true, "Optimize group by constant keys by default"},
+              {"input_format_json_try_infer_named_tuples_from_objects", false, true, "Try to infer named Tuples from JSON objects by default"},
+              {"input_format_json_read_numbers_as_strings", false, true, "Allow to read numbers as strings in JSON formats by default"},
+              {"input_format_json_read_arrays_as_strings", false, true, "Allow to read arrays as strings in JSON formats by default"},
+              {"input_format_json_infer_incomplete_types_as_strings", false, true, "Allow to infer incomplete types as Strings in JSON formats by default"},
+              {"input_format_json_try_infer_numbers_from_strings", true, false, "Don't infer numbers from strings in JSON formats by default to prevent possible parsing errors"},
+              {"http_write_exception_in_output_format", false, true, "Output valid JSON/XML on exception in HTTP streaming."}}},
+    {"23.8", {{"rewrite_count_distinct_if_with_count_distinct_implementation", false, true, "Rewrite countDistinctIf with count_distinct_implementation configuration"}}},
+    {"23.7", {{"function_sleep_max_microseconds_per_block", 0, 3000000, "In previous versions, the maximum sleep time of 3 seconds was applied only for `sleep`, but not for `sleepEachRow` function. In the new version, we introduce this setting. If you set compatibility with the previous versions, we will disable the limit altogether."}}},
+    {"23.6", {{"http_send_timeout", 180, 30, "3 minutes seems crazy long. Note that this is timeout for a single network write call, not for the whole upload operation."},
+              {"http_receive_timeout", 180, 30, "See http_send_timeout."}}},
+    {"23.5", {{"input_format_parquet_preserve_order", true, false, "Allow Parquet reader to reorder rows for better parallelism."},
+              {"parallelize_output_from_storages", false, true, "Allow parallelism when executing queries that read from file/url/s3/etc. This may reorder rows."},
+              {"use_with_fill_by_sorting_prefix", false, true, "Columns preceding WITH FILL columns in ORDER BY clause form sorting prefix. Rows with different values in sorting prefix are filled independently"},
+              {"output_format_parquet_compliant_nested_types", false, true, "Change an internal field name in output Parquet file schema."}}},
+    {"23.4", {{"allow_suspicious_indices", true, false, "If true, index can defined with identical expressions"},
+              {"allow_nonconst_timezone_arguments", true, false, "Allow non-const timezone arguments in certain time-related functions like toTimeZone(), fromUnixTimestamp*(), snowflakeToDateTime*()."},
+              {"connect_timeout_with_failover_ms", 50, 1000, "Increase default connect timeout because of async connect"},
+              {"connect_timeout_with_failover_secure_ms", 100, 1000, "Increase default secure connect timeout because of async connect"},
+              {"hedged_connection_timeout_ms", 100, 50, "Start new connection in hedged requests after 50 ms instead of 100 to correspond with previous connect timeout"},
+              {"formatdatetime_f_prints_single_zero", true, false, "Improved compatibility with MySQL DATE_FORMAT()/STR_TO_DATE()"},
+              {"formatdatetime_parsedatetime_m_is_month_name", false, true, "Improved compatibility with MySQL DATE_FORMAT/STR_TO_DATE"}}},
+    {"23.3", {{"output_format_parquet_version", "1.0", "2.latest", "Use latest Parquet format version for output format"},
+              {"input_format_json_ignore_unknown_keys_in_named_tuple", false, true, "Improve parsing JSON objects as named tuples"},
+              {"input_format_native_allow_types_conversion", false, true, "Allow types conversion in Native input forma"},
+              {"output_format_arrow_compression_method", "none", "lz4_frame", "Use lz4 compression in Arrow output format by default"},
+              {"output_format_parquet_compression_method", "snappy", "lz4", "Use lz4 compression in Parquet output format by default"},
+              {"output_format_orc_compression_method", "none", "lz4_frame", "Use lz4 compression in ORC output format by default"},
+              {"async_query_sending_for_remote", false, true, "Create connections and send query async across shards"}}},
+    {"23.2", {{"output_format_parquet_fixed_string_as_fixed_byte_array", false, true, "Use Parquet FIXED_LENGTH_BYTE_ARRAY type for FixedString by default"},
+              {"output_format_arrow_fixed_string_as_fixed_byte_array", false, true, "Use Arrow FIXED_SIZE_BINARY type for FixedString by default"},
+              {"query_plan_remove_redundant_distinct", false, true, "Remove redundant Distinct step in query plan"},
+              {"optimize_duplicate_order_by_and_distinct", true, false, "Remove duplicate ORDER BY and DISTINCT if it's possible"},
+              {"insert_keeper_max_retries", 0, 20, "Enable reconnections to Keeper on INSERT, improve reliability"}}},
+    {"23.1", {{"input_format_json_read_objects_as_strings", 0, 1, "Enable reading nested json objects as strings while object type is experimental"},
+              {"input_format_json_defaults_for_missing_elements_in_named_tuple", false, true, "Allow missing elements in JSON objects while reading named tuples by default"},
+              {"input_format_csv_detect_header", false, true, "Detect header in CSV format by default"},
+              {"input_format_tsv_detect_header", false, true, "Detect header in TSV format by default"},
+              {"input_format_custom_detect_header", false, true, "Detect header in CustomSeparated format by default"},
+              {"query_plan_remove_redundant_sorting", false, true, "Remove redundant sorting in query plan. For example, sorting steps related to ORDER BY clauses in subqueries"}}},
+    {"22.12", {{"max_size_to_preallocate_for_aggregation", 10'000'000, 100'000'000, "This optimizes performance"},
+               {"query_plan_aggregation_in_order", 0, 1, "Enable some refactoring around query plan"},
+               {"format_binary_max_string_size", 0, 1_GiB, "Prevent allocating large amount of memory"}}},
+    {"22.11", {{"use_structure_from_insertion_table_in_table_functions", 0, 2, "Improve using structure from insertion table in table functions"}}},
+    {"22.9", {{"force_grouping_standard_compatibility", false, true, "Make GROUPING function output the same as in SQL standard and other DBMS"}}},
+    {"22.7", {{"cross_to_inner_join_rewrite", 1, 2, "Force rewrite comma join to inner"},
+              {"enable_positional_arguments", false, true, "Enable positional arguments feature by default"},
+              {"format_csv_allow_single_quotes", true, false, "Most tools don't treat single quote in CSV specially, don't do it by default too"}}},
+    {"22.6", {{"output_format_json_named_tuples_as_objects", false, true, "Allow to serialize named tuples as JSON objects in JSON formats by default"},
+              {"input_format_skip_unknown_fields", false, true, "Optimize reading subset of columns for some input formats"}}},
+    {"22.5", {{"memory_overcommit_ratio_denominator", 0, 1073741824, "Enable memory overcommit feature by default"},
+              {"memory_overcommit_ratio_denominator_for_user", 0, 1073741824, "Enable memory overcommit feature by default"}}},
+    {"22.4", {{"allow_settings_after_format_in_insert", true, false, "Do not allow SETTINGS after FORMAT for INSERT queries because ClickHouse interpret SETTINGS as some values, which is misleading"}}},
+    {"22.3", {{"cast_ipv4_ipv6_default_on_conversion_error", true, false, "Make functions cast(value, 'IPv4') and cast(value, 'IPv6') behave same as toIPv4 and toIPv6 functions"}}},
+    {"21.12", {{"stream_like_engine_allow_direct_select", true, false, "Do not allow direct select for Kafka/RabbitMQ/FileLog by default"}}},
+    {"21.9", {{"output_format_decimal_trailing_zeros", true, false, "Do not output trailing zeros in text representation of Decimal types by default for better looking output"},
+              {"use_hedged_requests", false, true, "Enable Hedged Requests feature by default"}}},
+    {"21.7", {{"legacy_column_name_of_tuple_literal", true, false, "Add this setting only for compatibility reasons. It makes sense to set to 'true', while doing rolling update of cluster from version lower than 21.7 to higher"}}},
+    {"21.5", {{"async_socket_for_remote", false, true, "Fix all problems and turn on asynchronous reads from socket for remote queries by default again"}}},
+    {"21.3", {{"async_socket_for_remote", true, false, "Turn off asynchronous reads from socket for remote queries because of some problems"},
+              {"optimize_normalize_count_variants", false, true, "Rewrite aggregate functions that semantically equals to count() as count() by default"},
+              {"normalize_function_names", false, true, "Normalize function names to their canonical names, this was needed for projection query routing"}}},
+    {"21.2", {{"enable_global_with_statement", false, true, "Propagate WITH statements to UNION queries and all subqueries by default"}}},
+    {"21.1", {{"insert_quorum_parallel", false, true, "Use parallel quorum inserts by default. It is significantly more convenient to use than sequential quorum inserts"},
+              {"input_format_null_as_default", false, true, "Allow to insert NULL as default for input formats by default"},
+              {"optimize_on_insert", false, true, "Enable data optimization on INSERT by default for better user experience"},
+              {"use_compact_format_in_distributed_parts_names", false, true, "Use compact format for async INSERT into Distributed tables by default"}}},
+    {"20.10", {{"format_regexp_escaping_rule", "Escaped", "Raw", "Use Raw as default escaping rule for Regexp format to male the behaviour more like to what users expect"}}},
+    {"20.7", {{"show_table_uuid_in_table_create_query_if_not_nil", true, false, "Stop showing  UID of the table in its CREATE query for Engine=Atomic"}}},
+    {"20.5", {{"input_format_with_names_use_header", false, true, "Enable using header with names for formats with WithNames/WithNamesAndTypes suffixes"},
+              {"allow_suspicious_codecs", true, false, "Don't allow to specify meaningless compression codecs"}}},
+    {"20.4", {{"validate_polygons", false, true, "Throw exception if polygon is invalid in function pointInPolygon by default instead of returning possibly wrong results"}}},
+    {"19.18", {{"enable_scalar_subquery_optimization", false, true, "Prevent scalar subqueries from (de)serializing large scalar values and possibly avoid running the same subquery more than once"}}},
+    {"19.14", {{"any_join_distinct_right_table_keys", true, false, "Disable ANY RIGHT and ANY FULL JOINs by default to avoid inconsistency"}}},
+    {"19.12", {{"input_format_defaults_for_omitted_fields", false, true, "Enable calculation of complex default expressions for omitted fields for some input formats, because it should be the expected behaviour"}}},
+    {"19.5", {{"max_partitions_per_insert_block", 0, 100, "Add a limit for the number of partitions in one block"}}},
+    {"18.12.17", {{"enable_optimize_predicate_expression", 0, 1, "Optimize predicates to subqueries by default"}}},
 };
 
 
diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp
index e1b7e92ee5d..b8d70e5d5dd 100644
--- a/src/Interpreters/ActionsVisitor.cpp
+++ b/src/Interpreters/ActionsVisitor.cpp
@@ -1411,7 +1411,7 @@ FutureSetPtr ActionsMatcher::makeSet(const ASTFunction & node, Data & data, bool
             return {};
 
         PreparedSets::Hash set_key;
-        if (data.getContext()->getSettingsRef().allow_experimental_analyzer && !identifier)
+        if (data.getContext()->getSettingsRef().enable_analyzer && !identifier)
         {
             /// Here we can be only from mutation interpreter. Normal selects with analyzed use other interpreter.
             /// This is a hacky way to allow reusing cache for prepared sets.
diff --git a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp
index e35d31d2350..0948f24eca0 100644
--- a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp
+++ b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp
@@ -68,7 +68,7 @@ ASTPtr rewriteSelectQuery(
     // are written into the query context and will be sent by the query pipeline.
     select_query.setExpression(ASTSelectQuery::Expression::SETTINGS, {});
 
-    if (!context->getSettingsRef().allow_experimental_analyzer)
+    if (!context->getSettingsRef().enable_analyzer)
     {
         if (table_function_ptr)
             select_query.addTableFunction(table_function_ptr);
@@ -165,7 +165,7 @@ void SelectStreamFactory::createForShardImpl(
     auto emplace_remote_stream = [&](bool lazy = false, time_t local_delay = 0)
     {
         Block shard_header;
-        if (context->getSettingsRef().allow_experimental_analyzer)
+        if (context->getSettingsRef().enable_analyzer)
             shard_header = InterpreterSelectQueryAnalyzer::getSampleBlock(query_tree, context, SelectQueryOptions(processed_stage).analyze());
         else
             shard_header = header;
diff --git a/src/Interpreters/ClusterProxy/executeQuery.cpp b/src/Interpreters/ClusterProxy/executeQuery.cpp
index d04a73e384e..6c8ab11bfc9 100644
--- a/src/Interpreters/ClusterProxy/executeQuery.cpp
+++ b/src/Interpreters/ClusterProxy/executeQuery.cpp
@@ -300,7 +300,7 @@ void executeQuery(
 
     const size_t shards = cluster->getShardCount();
 
-    if (context->getSettingsRef().allow_experimental_analyzer)
+    if (context->getSettingsRef().enable_analyzer)
     {
         for (size_t i = 0, s = cluster->getShardsInfo().size(); i < s; ++i)
         {
@@ -581,7 +581,7 @@ void executeQueryWithParallelReplicasCustomKey(
     /// Return directly (with correct header) if no shard to query.
     if (query_info.getCluster()->getShardsInfo().empty())
     {
-        if (context->getSettingsRef().allow_experimental_analyzer)
+        if (context->getSettingsRef().enable_analyzer)
             return;
 
         Pipe pipe(std::make_shared<NullSource>(header));
diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp
index 971f90bd3cd..ea631ef01d5 100644
--- a/src/Interpreters/InterpreterCreateQuery.cpp
+++ b/src/Interpreters/InterpreterCreateQuery.cpp
@@ -834,7 +834,7 @@ InterpreterCreateQuery::TableProperties InterpreterCreateQuery::getTableProperti
 
         Block as_select_sample;
 
-        if (getContext()->getSettingsRef().allow_experimental_analyzer)
+        if (getContext()->getSettingsRef().enable_analyzer)
         {
             as_select_sample = InterpreterSelectQueryAnalyzer::getSampleBlock(create.select->clone(), getContext());
         }
@@ -1327,7 +1327,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create)
         {
             Block input_block;
 
-            if (getContext()->getSettingsRef().allow_experimental_analyzer)
+            if (getContext()->getSettingsRef().enable_analyzer)
             {
                 input_block = InterpreterSelectQueryAnalyzer::getSampleBlock(create.select->clone(), getContext());
             }
diff --git a/src/Interpreters/InterpreterDescribeQuery.cpp b/src/Interpreters/InterpreterDescribeQuery.cpp
index 39fc85a5e23..4a061f02c2b 100644
--- a/src/Interpreters/InterpreterDescribeQuery.cpp
+++ b/src/Interpreters/InterpreterDescribeQuery.cpp
@@ -129,7 +129,7 @@ void InterpreterDescribeQuery::fillColumnsFromSubquery(const ASTTableExpression
     auto select_query = table_expression.subquery->children.at(0);
     auto current_context = getContext();
 
-    if (settings.allow_experimental_analyzer)
+    if (settings.enable_analyzer)
     {
         SelectQueryOptions select_query_options;
         sample_block = InterpreterSelectQueryAnalyzer(select_query, current_context, select_query_options).getSampleBlock();
diff --git a/src/Interpreters/InterpreterExplainQuery.cpp b/src/Interpreters/InterpreterExplainQuery.cpp
index bedd9cb4a80..2fbfbf3a809 100644
--- a/src/Interpreters/InterpreterExplainQuery.cpp
+++ b/src/Interpreters/InterpreterExplainQuery.cpp
@@ -394,9 +394,9 @@ QueryPipeline InterpreterExplainQuery::executeImpl()
         }
         case ASTExplainQuery::QueryTree:
         {
-            if (!getContext()->getSettingsRef().allow_experimental_analyzer)
+            if (!getContext()->getSettingsRef().enable_analyzer)
                 throw Exception(ErrorCodes::NOT_IMPLEMENTED,
-                    "EXPLAIN QUERY TREE is only supported with a new analyzer. Set allow_experimental_analyzer = 1.");
+                    "EXPLAIN QUERY TREE is only supported with a new analyzer. Set enable_analyzer = 1.");
 
             if (ast.getExplainedQuery()->as<ASTSelectWithUnionQuery>() == nullptr)
                 throw Exception(ErrorCodes::INCORRECT_QUERY, "Only SELECT is supported for EXPLAIN QUERY TREE query");
@@ -453,7 +453,7 @@ QueryPipeline InterpreterExplainQuery::executeImpl()
 
             ContextPtr context;
 
-            if (getContext()->getSettingsRef().allow_experimental_analyzer)
+            if (getContext()->getSettingsRef().enable_analyzer)
             {
                 InterpreterSelectQueryAnalyzer interpreter(ast.getExplainedQuery(), getContext(), options);
                 context = interpreter.getContext();
@@ -499,7 +499,7 @@ QueryPipeline InterpreterExplainQuery::executeImpl()
                 QueryPlan plan;
                 ContextPtr context;
 
-                if (getContext()->getSettingsRef().allow_experimental_analyzer)
+                if (getContext()->getSettingsRef().enable_analyzer)
                 {
                     InterpreterSelectQueryAnalyzer interpreter(ast.getExplainedQuery(), getContext(), options);
                     context = interpreter.getContext();
@@ -558,7 +558,7 @@ QueryPipeline InterpreterExplainQuery::executeImpl()
             QueryPlan plan;
             ContextPtr context = getContext();
 
-            if (context->getSettingsRef().allow_experimental_analyzer)
+            if (context->getSettingsRef().enable_analyzer)
             {
                 InterpreterSelectQueryAnalyzer interpreter(ast.getExplainedQuery(), getContext(), SelectQueryOptions());
                 context = interpreter.getContext();
diff --git a/src/Interpreters/InterpreterFactory.cpp b/src/Interpreters/InterpreterFactory.cpp
index 12b3b510098..a909c4e602d 100644
--- a/src/Interpreters/InterpreterFactory.cpp
+++ b/src/Interpreters/InterpreterFactory.cpp
@@ -118,7 +118,7 @@ InterpreterFactory::InterpreterPtr InterpreterFactory::get(ASTPtr & query, Conte
 
     if (query->as<ASTSelectQuery>())
     {
-        if (context->getSettingsRef().allow_experimental_analyzer)
+        if (context->getSettingsRef().enable_analyzer)
             interpreter_name = "InterpreterSelectQueryAnalyzer";
         /// This is internal part of ASTSelectWithUnionQuery.
         /// Even if there is SELECT without union, it is represented by ASTSelectWithUnionQuery with single ASTSelectQuery as a child.
@@ -129,7 +129,7 @@ InterpreterFactory::InterpreterPtr InterpreterFactory::get(ASTPtr & query, Conte
     {
         ProfileEvents::increment(ProfileEvents::SelectQuery);
 
-        if (context->getSettingsRef().allow_experimental_analyzer)
+        if (context->getSettingsRef().enable_analyzer)
             interpreter_name = "InterpreterSelectQueryAnalyzer";
         else
             interpreter_name = "InterpreterSelectWithUnionQuery";
@@ -222,7 +222,7 @@ InterpreterFactory::InterpreterPtr InterpreterFactory::get(ASTPtr & query, Conte
     {
         const auto kind = query->as<ASTExplainQuery>()->getKind();
         if (kind == ASTExplainQuery::ParsedAST || kind == ASTExplainQuery::AnalyzedSyntax)
-            context->setSetting("allow_experimental_analyzer", false);
+            context->setSetting("enable_analyzer", false);
 
         interpreter_name = "InterpreterExplainQuery";
     }
diff --git a/src/Interpreters/InterpreterInsertQuery.cpp b/src/Interpreters/InterpreterInsertQuery.cpp
index c97593a1781..0213e2a2c42 100644
--- a/src/Interpreters/InterpreterInsertQuery.cpp
+++ b/src/Interpreters/InterpreterInsertQuery.cpp
@@ -95,7 +95,7 @@ StoragePtr InterpreterInsertQuery::getTable(ASTInsertQuery & query)
             Block header_block;
             auto select_query_options = SelectQueryOptions(QueryProcessingStage::Complete, 1);
 
-            if (current_context->getSettingsRef().allow_experimental_analyzer)
+            if (current_context->getSettingsRef().enable_analyzer)
             {
                 InterpreterSelectQueryAnalyzer interpreter_select(query.select, current_context, select_query_options);
                 header_block = interpreter_select.getSampleBlock();
diff --git a/src/Interpreters/MutationsInterpreter.cpp b/src/Interpreters/MutationsInterpreter.cpp
index 57ad5caa4c7..c049dbc9cc1 100644
--- a/src/Interpreters/MutationsInterpreter.cpp
+++ b/src/Interpreters/MutationsInterpreter.cpp
@@ -189,7 +189,7 @@ bool isStorageTouchedByMutations(
     std::optional<InterpreterSelectQuery> interpreter_select_query;
     BlockIO io;
 
-    if (context->getSettingsRef().allow_experimental_analyzer)
+    if (context->getSettingsRef().enable_analyzer)
     {
         auto select_query_tree = prepareQueryAffectedQueryTree(commands, storage.shared_from_this(), context);
         InterpreterSelectQueryAnalyzer interpreter(select_query_tree, context, SelectQueryOptions().ignoreLimits());
@@ -415,9 +415,9 @@ MutationsInterpreter::MutationsInterpreter(
     , logger(getLogger("MutationsInterpreter(" + source.getStorage()->getStorageID().getFullTableName() + ")"))
 {
     auto new_context = Context::createCopy(context_);
-    if (new_context->getSettingsRef().allow_experimental_analyzer)
+    if (new_context->getSettingsRef().enable_analyzer)
     {
-        new_context->setSetting("allow_experimental_analyzer", false);
+        new_context->setSetting("enable_analyzer", false);
         LOG_DEBUG(logger, "Will use old analyzer to prepare mutation");
     }
     context = std::move(new_context);
diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp
index ce58f7f922c..7476915ab8a 100644
--- a/src/Interpreters/executeQuery.cpp
+++ b/src/Interpreters/executeQuery.cpp
@@ -685,10 +685,10 @@ void validateAnalyzerSettings(ASTPtr ast, bool context_value)
 
         if (auto * set_query = node->as<ASTSetQuery>())
         {
-            if (auto * value = set_query->changes.tryGet("allow_experimental_analyzer"))
+            if (auto * value = set_query->changes.tryGet("enable_analyzer"))
             {
                 if (top_level != value->safeGet<bool>())
-                    throw Exception(ErrorCodes::INCORRECT_QUERY, "Setting 'allow_experimental_analyzer' is changed in the subquery. Top level value: {}", top_level);
+                    throw Exception(ErrorCodes::INCORRECT_QUERY, "Setting 'enable_analyzer' is changed in the subquery. Top level value: {}", top_level);
             }
         }
 
@@ -912,7 +912,7 @@ static std::tuple<ASTPtr, BlockIO> executeQueryImpl(
         /// Interpret SETTINGS clauses as early as possible (before invoking the corresponding interpreter),
         /// to allow settings to take effect.
         InterpreterSetQuery::applySettingsFromQuery(ast, context);
-        validateAnalyzerSettings(ast, context->getSettingsRef().allow_experimental_analyzer);
+        validateAnalyzerSettings(ast, context->getSettingsRef().enable_analyzer);
 
         if (auto * insert_query = ast->as<ASTInsertQuery>())
             insert_query->tail = istr;
diff --git a/src/Interpreters/getHeaderForProcessingStage.cpp b/src/Interpreters/getHeaderForProcessingStage.cpp
index cf18cbbb54a..c4a791e85e1 100644
--- a/src/Interpreters/getHeaderForProcessingStage.cpp
+++ b/src/Interpreters/getHeaderForProcessingStage.cpp
@@ -141,7 +141,7 @@ Block getHeaderForProcessingStage(
 
             Block result;
 
-            if (context->getSettingsRef().allow_experimental_analyzer)
+            if (context->getSettingsRef().enable_analyzer)
             {
                 auto storage = std::make_shared<StorageDummy>(storage_snapshot->storage.getStorageID(),
                                                                                         storage_snapshot->getAllColumnsDescription(),
diff --git a/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp b/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp
index d8624a1c99b..dc4b7fd733b 100644
--- a/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp
+++ b/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp
@@ -65,7 +65,7 @@ std::unique_ptr<QueryPlan> createLocalPlan(
         .setShardInfo(static_cast<UInt32>(shard_num), static_cast<UInt32>(shard_count))
         .ignoreASTOptimizations();
 
-    if (context->getSettingsRef().allow_experimental_analyzer)
+    if (context->getSettingsRef().enable_analyzer)
     {
         /// For Analyzer, identifier in GROUP BY/ORDER BY/LIMIT BY lists has been resolved to
         /// ConstantNode in QueryTree if it is an alias of a constant, so we should not replace
diff --git a/src/Processors/Transforms/buildPushingToViewsChain.cpp b/src/Processors/Transforms/buildPushingToViewsChain.cpp
index 98d66ed77c3..a2d5ec5d1cb 100644
--- a/src/Processors/Transforms/buildPushingToViewsChain.cpp
+++ b/src/Processors/Transforms/buildPushingToViewsChain.cpp
@@ -319,7 +319,7 @@ std::optional<Chain> generateViewChain(
         Block header;
 
         /// Get list of columns we get from select query.
-        if (select_context->getSettingsRef().allow_experimental_analyzer)
+        if (select_context->getSettingsRef().enable_analyzer)
             header = InterpreterSelectQueryAnalyzer::getSampleBlock(query, select_context);
         else
             header = InterpreterSelectQuery(query, select_context, SelectQueryOptions()).getSampleBlock();
@@ -613,7 +613,7 @@ static QueryPipeline process(Block block, ViewRuntimeData & view, const ViewsDat
 
     QueryPipelineBuilder pipeline;
 
-    if (local_context->getSettingsRef().allow_experimental_analyzer)
+    if (local_context->getSettingsRef().enable_analyzer)
     {
         InterpreterSelectQueryAnalyzer interpreter(view.query, local_context, local_context->getViewSource(), SelectQueryOptions().ignoreAccessCheck());
         pipeline = interpreter.buildQueryPipeline();
diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp
index c5dfe3e6e5f..4262716b406 100644
--- a/src/Server/TCPHandler.cpp
+++ b/src/Server/TCPHandler.cpp
@@ -1904,14 +1904,14 @@ void TCPHandler::receiveQuery()
     /// Settings
     ///
 
-    /// FIXME: Remove when allow_experimental_analyzer will become obsolete.
+    /// FIXME: Remove when enable_analyzer will become obsolete.
     /// Analyzer became Beta in 24.3 and started to be enabled by default.
     /// We have to disable it for ourselves to make sure we don't have different settings on
     /// different servers.
     if (query_kind == ClientInfo::QueryKind::SECONDARY_QUERY
         && client_info.getVersionNumber() < VersionNumber(23, 3, 0)
-        && !passed_settings.allow_experimental_analyzer.changed)
-        passed_settings.set("allow_experimental_analyzer", false);
+        && !passed_settings.enable_analyzer.changed)
+        passed_settings.set("enable_analyzer", false);
 
     auto settings_changes = passed_settings.changes();
     query_kind = query_context->getClientInfo().query_kind;
diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp
index 7891042bb96..2843ff5a14e 100644
--- a/src/Storages/AlterCommands.cpp
+++ b/src/Storages/AlterCommands.cpp
@@ -806,7 +806,7 @@ void AlterCommand::apply(StorageInMemoryMetadata & metadata, ContextPtr context)
         metadata.select = SelectQueryDescription::getSelectQueryFromASTForMatView(select, metadata.refresh != nullptr, context);
         Block as_select_sample;
 
-        if (context->getSettingsRef().allow_experimental_analyzer)
+        if (context->getSettingsRef().enable_analyzer)
         {
             as_select_sample = InterpreterSelectQueryAnalyzer::getSampleBlock(select->clone(), context);
         }
diff --git a/src/Storages/IStorageCluster.cpp b/src/Storages/IStorageCluster.cpp
index 63467603d16..b485ab9cbb5 100644
--- a/src/Storages/IStorageCluster.cpp
+++ b/src/Storages/IStorageCluster.cpp
@@ -125,7 +125,7 @@ void IStorageCluster::read(
     Block sample_block;
     ASTPtr query_to_send = query_info.query;
 
-    if (context->getSettingsRef().allow_experimental_analyzer)
+    if (context->getSettingsRef().enable_analyzer)
     {
         sample_block = InterpreterSelectQueryAnalyzer::getSampleBlock(query_info.query, context, SelectQueryOptions(processed_stage));
     }
diff --git a/src/Storages/LiveView/StorageLiveView.cpp b/src/Storages/LiveView/StorageLiveView.cpp
index 71b1a0a73c9..c93da7ca512 100644
--- a/src/Storages/LiveView/StorageLiveView.cpp
+++ b/src/Storages/LiveView/StorageLiveView.cpp
@@ -378,7 +378,7 @@ void StorageLiveView::writeBlock(StorageLiveView & live_view, Block && block, Ch
 
         QueryPipelineBuilder builder;
 
-        if (local_context->getSettingsRef().allow_experimental_analyzer)
+        if (local_context->getSettingsRef().enable_analyzer)
         {
             auto select_description = buildSelectQueryTreeDescription(select_query_description.inner_query, local_context);
             if (select_description.dependent_table_node)
@@ -475,7 +475,7 @@ Block StorageLiveView::getHeader() const
 
     if (!sample_block)
     {
-        if (live_view_context->getSettingsRef().allow_experimental_analyzer)
+        if (live_view_context->getSettingsRef().enable_analyzer)
         {
             sample_block = InterpreterSelectQueryAnalyzer::getSampleBlock(select_query_description.select_query,
                 live_view_context,
@@ -519,7 +519,7 @@ ASTPtr StorageLiveView::getInnerBlocksQuery()
         auto & select_with_union_query = select_query_description.select_query->as<ASTSelectWithUnionQuery &>();
         auto blocks_query = select_with_union_query.list_of_selects->children.at(0)->clone();
 
-        if (!live_view_context->getSettingsRef().allow_experimental_analyzer)
+        if (!live_view_context->getSettingsRef().enable_analyzer)
         {
             /// Rewrite inner query with right aliases for JOIN.
             /// It cannot be done in constructor or startup() because InterpreterSelectQuery may access table,
@@ -543,7 +543,7 @@ MergeableBlocksPtr StorageLiveView::collectMergeableBlocks(ContextPtr local_cont
 
     QueryPipelineBuilder builder;
 
-    if (local_context->getSettingsRef().allow_experimental_analyzer)
+    if (local_context->getSettingsRef().enable_analyzer)
     {
         InterpreterSelectQueryAnalyzer interpreter(select_query_description.inner_query,
             local_context,
@@ -599,7 +599,7 @@ QueryPipelineBuilder StorageLiveView::completeQuery(Pipes pipes)
 
     QueryPipelineBuilder builder;
 
-    if (block_context->getSettingsRef().allow_experimental_analyzer)
+    if (block_context->getSettingsRef().enable_analyzer)
     {
         auto select_description = buildSelectQueryTreeDescription(select_query_description.select_query, block_context);
 
diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index ce27ad24e10..01ef0a409b0 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -7097,7 +7097,7 @@ QueryProcessingStage::Enum MergeTreeData::getQueryProcessingStage(
     SelectQueryInfo &) const
 {
     /// with new analyzer, Planner make decision regarding parallel replicas usage, and so about processing stage on reading
-    if (!query_context->getSettingsRef().allow_experimental_analyzer)
+    if (!query_context->getSettingsRef().enable_analyzer)
     {
         const auto & settings = query_context->getSettingsRef();
         if (query_context->canUseParallelReplicasCustomKey())
diff --git a/src/Storages/MergeTree/RPNBuilder.cpp b/src/Storages/MergeTree/RPNBuilder.cpp
index 6e963066f39..fccb20c2b0a 100644
--- a/src/Storages/MergeTree/RPNBuilder.cpp
+++ b/src/Storages/MergeTree/RPNBuilder.cpp
@@ -33,7 +33,7 @@ namespace ErrorCodes
 namespace
 {
 
-void appendColumnNameWithoutAlias(const ActionsDAG::Node & node, WriteBuffer & out, bool allow_experimental_analyzer, bool legacy = false)
+void appendColumnNameWithoutAlias(const ActionsDAG::Node & node, WriteBuffer & out, bool enable_analyzer, bool legacy = false)
 {
     switch (node.type)
     {
@@ -45,18 +45,18 @@ void appendColumnNameWithoutAlias(const ActionsDAG::Node & node, WriteBuffer & o
             /// If it was created from ASTLiteral, then result_name can be an alias.
             /// We need to convert value back to string here.
             const auto * column_const = typeid_cast<const ColumnConst *>(node.column.get());
-            if (column_const && !allow_experimental_analyzer)
+            if (column_const && !enable_analyzer)
                 writeString(applyVisitor(FieldVisitorToString(), column_const->getField()), out);
             else
                 writeString(node.result_name, out);
             break;
         }
         case ActionsDAG::ActionType::ALIAS:
-            appendColumnNameWithoutAlias(*node.children.front(), out, allow_experimental_analyzer, legacy);
+            appendColumnNameWithoutAlias(*node.children.front(), out, enable_analyzer, legacy);
             break;
         case ActionsDAG::ActionType::ARRAY_JOIN:
             writeCString("arrayJoin(", out);
-            appendColumnNameWithoutAlias(*node.children.front(), out, allow_experimental_analyzer, legacy);
+            appendColumnNameWithoutAlias(*node.children.front(), out, enable_analyzer, legacy);
             writeChar(')', out);
             break;
         case ActionsDAG::ActionType::FUNCTION:
@@ -75,17 +75,17 @@ void appendColumnNameWithoutAlias(const ActionsDAG::Node & node, WriteBuffer & o
                     writeCString(", ", out);
                 first = false;
 
-                appendColumnNameWithoutAlias(*arg, out, allow_experimental_analyzer, legacy);
+                appendColumnNameWithoutAlias(*arg, out, enable_analyzer, legacy);
             }
             writeChar(')', out);
         }
     }
 }
 
-String getColumnNameWithoutAlias(const ActionsDAG::Node & node, bool allow_experimental_analyzer, bool legacy = false)
+String getColumnNameWithoutAlias(const ActionsDAG::Node & node, bool enable_analyzer, bool legacy = false)
 {
     WriteBufferFromOwnString out;
-    appendColumnNameWithoutAlias(node, out, allow_experimental_analyzer, legacy);
+    appendColumnNameWithoutAlias(node, out, enable_analyzer, legacy);
 
     return std::move(out.str());
 }
@@ -131,7 +131,7 @@ std::string RPNBuilderTreeNode::getColumnName() const
     if (ast_node)
         return ast_node->getColumnNameWithoutAlias();
     else
-        return getColumnNameWithoutAlias(*dag_node, getTreeContext().getSettings().allow_experimental_analyzer);
+        return getColumnNameWithoutAlias(*dag_node, getTreeContext().getSettings().enable_analyzer);
 }
 
 std::string RPNBuilderTreeNode::getColumnNameWithModuloLegacy() const
@@ -144,7 +144,7 @@ std::string RPNBuilderTreeNode::getColumnNameWithModuloLegacy() const
     }
     else
     {
-        return getColumnNameWithoutAlias(*dag_node, getTreeContext().getSettings().allow_experimental_analyzer, true /*legacy*/);
+        return getColumnNameWithoutAlias(*dag_node, getTreeContext().getSettings().enable_analyzer, true /*legacy*/);
     }
 }
 
diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp
index 04e6d6676d1..da427ca4a6a 100644
--- a/src/Storages/StorageBuffer.cpp
+++ b/src/Storages/StorageBuffer.cpp
@@ -397,7 +397,7 @@ void StorageBuffer::read(
     /// TODO: Find a way to support projections for StorageBuffer
     if (processed_stage > QueryProcessingStage::FetchColumns)
     {
-        if (local_context->getSettingsRef().allow_experimental_analyzer)
+        if (local_context->getSettingsRef().enable_analyzer)
         {
             auto storage = std::make_shared<StorageValues>(
                     getStorageID(),
diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp
index 3e38ddf830a..f1fe70b4594 100644
--- a/src/Storages/StorageDistributed.cpp
+++ b/src/Storages/StorageDistributed.cpp
@@ -833,7 +833,7 @@ void StorageDistributed::read(
 
     const auto & settings = local_context->getSettingsRef();
 
-    if (settings.allow_experimental_analyzer)
+    if (settings.enable_analyzer)
     {
         StorageID remote_storage_id = StorageID::createEmpty();
         if (!remote_table_function_ptr)
@@ -1057,7 +1057,7 @@ static std::optional<ActionsDAG> getFilterFromQuery(const ASTPtr & ast, ContextP
     QueryPlan plan;
     SelectQueryOptions options;
     options.only_analyze = true;
-    if (context->getSettingsRef().allow_experimental_analyzer)
+    if (context->getSettingsRef().enable_analyzer)
     {
         InterpreterSelectQueryAnalyzer interpreter(ast, context, options);
         plan = std::move(interpreter).extractQueryPlan();
@@ -1611,7 +1611,7 @@ ClusterPtr StorageDistributed::skipUnusedShards(
     const StorageSnapshotPtr & storage_snapshot,
     ContextPtr local_context) const
 {
-    if (local_context->getSettingsRef().allow_experimental_analyzer)
+    if (local_context->getSettingsRef().enable_analyzer)
         return skipUnusedShardsWithAnalyzer(cluster, query_info, storage_snapshot, local_context);
 
     const auto & select = query_info.query->as<ASTSelectQuery &>();
diff --git a/src/Storages/StorageExecutable.cpp b/src/Storages/StorageExecutable.cpp
index 0094723e3fd..27bfa6f854c 100644
--- a/src/Storages/StorageExecutable.cpp
+++ b/src/Storages/StorageExecutable.cpp
@@ -150,7 +150,7 @@ void StorageExecutable::read(
     for (auto & input_query : input_queries)
     {
         QueryPipelineBuilder builder;
-        if (context->getSettingsRef().allow_experimental_analyzer)
+        if (context->getSettingsRef().enable_analyzer)
             builder = InterpreterSelectQueryAnalyzer(input_query, context, {}).buildQueryPipeline();
         else
             builder = InterpreterSelectWithUnionQuery(input_query, context, {}).buildQueryPipeline();
diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp
index 7c268d36a7b..613317b2564 100644
--- a/src/Storages/StorageMerge.cpp
+++ b/src/Storages/StorageMerge.cpp
@@ -590,7 +590,7 @@ std::vector<ReadFromMerge::ChildPlan> ReadFromMerge::createChildrenPlans(SelectQ
         auto modified_query_info
             = getModifiedQueryInfo(modified_context, table, nested_storage_snaphsot, real_column_names, column_names_as_aliases, aliases);
 
-        if (!context->getSettingsRef().allow_experimental_analyzer)
+        if (!context->getSettingsRef().enable_analyzer)
         {
             auto storage_columns = storage_metadata_snapshot->getColumns();
             auto syntax_result = TreeRewriter(context).analyzeSelect(
@@ -1047,13 +1047,13 @@ void ReadFromMerge::addVirtualColumns(
     const StorageWithLockAndName & storage_with_lock) const
 {
     const auto & [database_name, _, storage, table_name] = storage_with_lock;
-    bool allow_experimental_analyzer = context->getSettingsRef().allow_experimental_analyzer;
+    bool enable_analyzer = context->getSettingsRef().enable_analyzer;
 
     /// Add virtual columns if we don't already have them.
 
     Block plan_header = child.plan.getCurrentDataStream().header;
 
-    if (allow_experimental_analyzer)
+    if (enable_analyzer)
     {
         String table_alias = modified_query_info.query_tree->as<QueryNode>()->getJoinTree()->as<TableNode>()->getAlias();
 
@@ -1133,8 +1133,8 @@ QueryPipelineBuilderPtr ReadFromMerge::buildPipeline(
     if (!builder->initialized())
         return builder;
 
-    bool allow_experimental_analyzer = context->getSettingsRef().allow_experimental_analyzer;
-    if (processed_stage > child.stage || (allow_experimental_analyzer && processed_stage != QueryProcessingStage::FetchColumns))
+    bool enable_analyzer = context->getSettingsRef().enable_analyzer;
+    if (processed_stage > child.stage || (enable_analyzer && processed_stage != QueryProcessingStage::FetchColumns))
     {
         /** Materialization is needed, since from distributed storage the constants come materialized.
           * If you do not do this, different types (Const and non-Const) columns will be produced in different threads,
@@ -1168,7 +1168,7 @@ ReadFromMerge::ChildPlan ReadFromMerge::createPlanForTable(
         modified_select.setFinal();
     }
 
-    bool allow_experimental_analyzer = modified_context->getSettingsRef().allow_experimental_analyzer;
+    bool enable_analyzer = modified_context->getSettingsRef().enable_analyzer;
 
     auto storage_stage = storage->getQueryProcessingStage(modified_context,
         processed_stage,
@@ -1201,13 +1201,13 @@ ReadFromMerge::ChildPlan ReadFromMerge::createPlanForTable(
                 row_policy_data_opt->addStorageFilter(source_step_with_filter);
         }
     }
-    else if (processed_stage > storage_stage || allow_experimental_analyzer)
+    else if (processed_stage > storage_stage || enable_analyzer)
     {
         /// Maximum permissible parallelism is streams_num
         modified_context->setSetting("max_threads", streams_num);
         modified_context->setSetting("max_streams_to_max_threads_ratio", 1);
 
-        if (allow_experimental_analyzer)
+        if (enable_analyzer)
         {
             /// Converting query to AST because types might be different in the source table.
             /// Need to resolve types again.
@@ -1479,7 +1479,7 @@ void ReadFromMerge::convertAndFilterSourceStream(
     auto storage_sample_block = snapshot->metadata->getSampleBlock();
     auto pipe_columns = before_block_header.getNamesAndTypesList();
 
-    if (local_context->getSettingsRef().allow_experimental_analyzer)
+    if (local_context->getSettingsRef().enable_analyzer)
     {
         for (const auto & alias : aliases)
         {
@@ -1522,7 +1522,7 @@ void ReadFromMerge::convertAndFilterSourceStream(
 
     ActionsDAG::MatchColumnsMode convert_actions_match_columns_mode = ActionsDAG::MatchColumnsMode::Name;
 
-    if (local_context->getSettingsRef().allow_experimental_analyzer
+    if (local_context->getSettingsRef().enable_analyzer
         && (child.stage != QueryProcessingStage::FetchColumns || dynamic_cast<const StorageDistributed *>(&snapshot->storage) != nullptr))
         convert_actions_match_columns_mode = ActionsDAG::MatchColumnsMode::Position;
 
diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp
index f55f672fe5e..ebc88993ee4 100644
--- a/src/Storages/StorageMergeTree.cpp
+++ b/src/Storages/StorageMergeTree.cpp
@@ -208,7 +208,7 @@ void StorageMergeTree::read(
     const auto & settings = local_context->getSettingsRef();
     /// reading step for parallel replicas with new analyzer is built in Planner, so don't do it here
     if (local_context->canUseParallelReplicasOnInitiator() && settings.parallel_replicas_for_non_replicated_merge_tree
-        && !settings.allow_experimental_analyzer)
+        && !settings.enable_analyzer)
     {
         ClusterProxy::executeQueryWithParallelReplicas(
             query_plan, getStorageID(), processed_stage, query_info.query, local_context, query_info.storage_limits);
@@ -216,7 +216,7 @@ void StorageMergeTree::read(
     }
 
     if (local_context->canUseParallelReplicasCustomKey() && settings.parallel_replicas_for_non_replicated_merge_tree
-        && !settings.allow_experimental_analyzer && local_context->getClientInfo().distributed_depth == 0)
+        && !settings.enable_analyzer && local_context->getClientInfo().distributed_depth == 0)
     {
         if (auto cluster = local_context->getClusterForParallelReplicas();
             local_context->canUseParallelReplicasCustomKeyForCluster(*cluster))
@@ -244,7 +244,7 @@ void StorageMergeTree::read(
 
     const bool enable_parallel_reading = local_context->canUseParallelReplicasOnFollower()
         && local_context->getSettingsRef().parallel_replicas_for_non_replicated_merge_tree
-        && (!local_context->getSettingsRef().allow_experimental_analyzer || query_info.current_table_chosen_for_reading_with_parallel_replicas);
+        && (!local_context->getSettingsRef().enable_analyzer || query_info.current_table_chosen_for_reading_with_parallel_replicas);
 
     if (auto plan = reader.read(
             column_names,
diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp
index 2d826c6c2df..a3965e7a6d4 100644
--- a/src/Storages/StorageReplicatedMergeTree.cpp
+++ b/src/Storages/StorageReplicatedMergeTree.cpp
@@ -5480,13 +5480,13 @@ void StorageReplicatedMergeTree::read(
         return;
     }
     /// reading step for parallel replicas with new analyzer is built in Planner, so don't do it here
-    if (local_context->canUseParallelReplicasOnInitiator() && !settings.allow_experimental_analyzer)
+    if (local_context->canUseParallelReplicasOnInitiator() && !settings.enable_analyzer)
     {
         readParallelReplicasImpl(query_plan, column_names, query_info, local_context, processed_stage);
         return;
     }
 
-    if (local_context->canUseParallelReplicasCustomKey() && !settings.allow_experimental_analyzer
+    if (local_context->canUseParallelReplicasCustomKey() && !settings.enable_analyzer
         && local_context->getClientInfo().distributed_depth == 0)
     {
         if (auto cluster = local_context->getClusterForParallelReplicas();
@@ -5555,7 +5555,7 @@ void StorageReplicatedMergeTree::readLocalImpl(
     const size_t num_streams)
 {
     const bool enable_parallel_reading = local_context->canUseParallelReplicasOnFollower()
-        && (!local_context->getSettingsRef().allow_experimental_analyzer
+        && (!local_context->getSettingsRef().enable_analyzer
             || query_info.current_table_chosen_for_reading_with_parallel_replicas);
 
     auto plan = reader.read(
diff --git a/src/Storages/StorageView.cpp b/src/Storages/StorageView.cpp
index 878998ebf12..dcb5ef2ae77 100644
--- a/src/Storages/StorageView.cpp
+++ b/src/Storages/StorageView.cpp
@@ -164,7 +164,7 @@ void StorageView::read(
 
     auto options = SelectQueryOptions(QueryProcessingStage::Complete, 0, false, query_info.settings_limit_offset_done);
 
-    if (context->getSettingsRef().allow_experimental_analyzer)
+    if (context->getSettingsRef().enable_analyzer)
     {
         InterpreterSelectQueryAnalyzer interpreter(current_inner_query, getViewContext(context, storage_snapshot), options, column_names);
         interpreter.addStorageLimits(*query_info.storage_limits);
diff --git a/src/Storages/TTLDescription.cpp b/src/Storages/TTLDescription.cpp
index d674f054632..16eccfd7343 100644
--- a/src/Storages/TTLDescription.cpp
+++ b/src/Storages/TTLDescription.cpp
@@ -172,7 +172,7 @@ static ExpressionAndSets buildExpressionAndSets(ASTPtr & ast, const NamesAndType
     /// with subqueries it's possible that new analyzer will be enabled in ::read method
     /// of underlying storage when all other parts of infra are not ready for it
     /// (built with old analyzer).
-    context_copy->setSetting("allow_experimental_analyzer", false);
+    context_copy->setSetting("enable_analyzer", false);
     auto syntax_analyzer_result = TreeRewriter(context_copy).analyze(ast, columns);
     ExpressionAnalyzer analyzer(ast, syntax_analyzer_result, context_copy);
     auto dag = analyzer.getActionsDAG(false);
diff --git a/src/Storages/WindowView/StorageWindowView.cpp b/src/Storages/WindowView/StorageWindowView.cpp
index 65bf6768b1b..a2b1704f24b 100644
--- a/src/Storages/WindowView/StorageWindowView.cpp
+++ b/src/Storages/WindowView/StorageWindowView.cpp
@@ -1197,7 +1197,7 @@ StorageWindowView::StorageWindowView(
     , fire_signal_timeout_s(context_->getSettingsRef().wait_for_window_view_fire_signal_timeout.totalSeconds())
     , clean_interval_usec(context_->getSettingsRef().window_view_clean_interval.totalMicroseconds())
 {
-    if (context_->getSettingsRef().allow_experimental_analyzer)
+    if (context_->getSettingsRef().enable_analyzer)
         disabled_due_to_analyzer = true;
 
     if (mode <= LoadingStrictnessLevel::CREATE)
@@ -1753,9 +1753,9 @@ StoragePtr StorageWindowView::getTargetTable() const
 
 void StorageWindowView::throwIfWindowViewIsDisabled(ContextPtr local_context) const
 {
-    if (disabled_due_to_analyzer || (local_context && local_context->getSettingsRef().allow_experimental_analyzer))
+    if (disabled_due_to_analyzer || (local_context && local_context->getSettingsRef().enable_analyzer))
         throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Experimental WINDOW VIEW feature is not supported "
-                        "in the current infrastructure for query analysis (the setting 'allow_experimental_analyzer')");
+                        "in the current infrastructure for query analysis (the setting 'enable_analyzer')");
 }
 
 void registerStorageWindowView(StorageFactory & factory)
diff --git a/src/TableFunctions/TableFunctionView.cpp b/src/TableFunctions/TableFunctionView.cpp
index 57501df6d4d..02a278cf590 100644
--- a/src/TableFunctions/TableFunctionView.cpp
+++ b/src/TableFunctions/TableFunctionView.cpp
@@ -50,7 +50,7 @@ ColumnsDescription TableFunctionView::getActualTableStructure(ContextPtr context
 
     Block sample_block;
 
-    if (context->getSettingsRef().allow_experimental_analyzer)
+    if (context->getSettingsRef().enable_analyzer)
         sample_block = InterpreterSelectQueryAnalyzer::getSampleBlock(create.children[0], context);
     else
         sample_block = InterpreterSelectWithUnionQuery::getSampleBlock(create.children[0], context);
diff --git a/src/TableFunctions/TableFunctionViewIfPermitted.cpp b/src/TableFunctions/TableFunctionViewIfPermitted.cpp
index 935be6c1987..7bae2731525 100644
--- a/src/TableFunctions/TableFunctionViewIfPermitted.cpp
+++ b/src/TableFunctions/TableFunctionViewIfPermitted.cpp
@@ -114,7 +114,7 @@ bool TableFunctionViewIfPermitted::isPermitted(const ContextPtr & context, const
 
     try
     {
-        if (context->getSettingsRef().allow_experimental_analyzer)
+        if (context->getSettingsRef().enable_analyzer)
         {
             sample_block = InterpreterSelectQueryAnalyzer::getSampleBlock(create.children[0], context);
         }
diff --git a/tests/config/users.d/analyzer.xml b/tests/config/users.d/analyzer.xml
index 4b9764526fa..edba8b8578e 100644
--- a/tests/config/users.d/analyzer.xml
+++ b/tests/config/users.d/analyzer.xml
@@ -1,7 +1,7 @@
 <clickhouse>
     <profiles>
         <default>
-            <allow_experimental_analyzer>0</allow_experimental_analyzer>
+            <enable_analyzer>0</enable_analyzer>
         </default>
     </profiles>
 </clickhouse>
diff --git a/tests/integration/helpers/0_common_enable_old_analyzer.xml b/tests/integration/helpers/0_common_enable_old_analyzer.xml
index 4b9764526fa..edba8b8578e 100644
--- a/tests/integration/helpers/0_common_enable_old_analyzer.xml
+++ b/tests/integration/helpers/0_common_enable_old_analyzer.xml
@@ -1,7 +1,7 @@
 <clickhouse>
     <profiles>
         <default>
-            <allow_experimental_analyzer>0</allow_experimental_analyzer>
+            <enable_analyzer>0</enable_analyzer>
         </default>
     </profiles>
 </clickhouse>
diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py
index 673fc07fe94..5e0352df617 100644
--- a/tests/integration/helpers/cluster.py
+++ b/tests/integration/helpers/cluster.py
@@ -4484,7 +4484,7 @@ class ClickHouseInstance:
 
         use_old_analyzer = os.environ.get("CLICKHOUSE_USE_OLD_ANALYZER") is not None
         # If specific version was used there can be no
-        # allow_experimental_analyzer setting, so do this only if it was
+        # enable_analyzer setting, so do this only if it was
         # explicitly requested.
         if self.tag:
             use_old_analyzer = False
diff --git a/tests/integration/test_analyzer_compatibility/test.py b/tests/integration/test_analyzer_compatibility/test.py
index d4ded420c61..2c840154eb5 100644
--- a/tests/integration/test_analyzer_compatibility/test.py
+++ b/tests/integration/test_analyzer_compatibility/test.py
@@ -51,7 +51,7 @@ def test_two_new_versions(start_cluster):
     assert (
         current.query(
             """
-SELECT hostname() AS h, getSetting('allow_experimental_analyzer')
+SELECT hostname() AS h, getSetting('enable_analyzer')
 FROM clusterAllReplicas('test_cluster_mixed', system.one)
 ORDER BY h;"""
         )
@@ -62,7 +62,7 @@ ORDER BY h;"""
     analyzer_enabled = current.query(
         f"""
 SELECT
-DISTINCT Settings['allow_experimental_analyzer']
+DISTINCT Settings['enable_analyzer']
 FROM clusterAllReplicas('test_cluster_mixed', system.query_log)
 WHERE initial_query_id = '{query_id}';"""
     )
@@ -81,7 +81,7 @@ WHERE initial_query_id = '{query_id}';"""
     assert (
         backward.query(
             """
-SELECT hostname() AS h, getSetting('allow_experimental_analyzer')
+SELECT hostname() AS h, getSetting('enable_analyzer')
 FROM clusterAllReplicas('test_cluster_mixed', system.one)
 ORDER BY h;"""
         )
@@ -92,7 +92,7 @@ ORDER BY h;"""
     analyzer_enabled = backward.query(
         f"""
 SELECT
-DISTINCT Settings['allow_experimental_analyzer']
+DISTINCT Settings['enable_analyzer']
 FROM clusterAllReplicas('test_cluster_mixed', system.query_log)
 WHERE initial_query_id = '{query_id}';"""
     )
diff --git a/tests/integration/test_distributed_type_object/test.py b/tests/integration/test_distributed_type_object/test.py
index 360087c9dda..e774876bc8b 100644
--- a/tests/integration/test_distributed_type_object/test.py
+++ b/tests/integration/test_distributed_type_object/test.py
@@ -89,7 +89,7 @@ def test_distributed_type_object(started_cluster):
     assert (
         TSV(
             node1.query(
-                "SELECT id, data.k1, data.k2.k3, data.k2.k4, data.k5 FROM dist_table ORDER BY id SETTINGS allow_experimental_analyzer = 0"
+                "SELECT id, data.k1, data.k2.k3, data.k2.k4, data.k5 FROM dist_table ORDER BY id SETTINGS enable_analyzer = 0"
             )
         )
         == expected
diff --git a/tests/integration/test_replicated_merge_tree_replicated_db_ttl/configs/enable_parallel_replicas.xml b/tests/integration/test_replicated_merge_tree_replicated_db_ttl/configs/enable_parallel_replicas.xml
index c654074740a..30a0b6276b7 100644
--- a/tests/integration/test_replicated_merge_tree_replicated_db_ttl/configs/enable_parallel_replicas.xml
+++ b/tests/integration/test_replicated_merge_tree_replicated_db_ttl/configs/enable_parallel_replicas.xml
@@ -1,7 +1,7 @@
 <clickhouse>
     <profiles>
         <default>
-            <allow_experimental_analyzer>1</allow_experimental_analyzer>
+            <enable_analyzer>1</enable_analyzer>
             <allow_experimental_parallel_reading_from_replicas>1</allow_experimental_parallel_reading_from_replicas>
             <cluster_for_parallel_replicas>default</cluster_for_parallel_replicas>
             <max_parallel_replicas>100</max_parallel_replicas>
diff --git a/tests/integration/test_settings_profile/test.py b/tests/integration/test_settings_profile/test.py
index e5c0a072ff9..4800ab798bf 100644
--- a/tests/integration/test_settings_profile/test.py
+++ b/tests/integration/test_settings_profile/test.py
@@ -459,7 +459,7 @@ def test_show_profiles():
 
     query_possible_response = [
         "CREATE SETTINGS PROFILE `default`\n",
-        "CREATE SETTINGS PROFILE `default` SETTINGS allow_experimental_analyzer = true\n",
+        "CREATE SETTINGS PROFILE `default` SETTINGS enable_analyzer = true\n",
     ]
     assert (
         instance.query("SHOW CREATE SETTINGS PROFILE default")
@@ -470,7 +470,7 @@ def test_show_profiles():
         "CREATE SETTINGS PROFILE `default`\n"
         "CREATE SETTINGS PROFILE `readonly` SETTINGS readonly = 1\n"
         "CREATE SETTINGS PROFILE `xyz`\n",
-        "CREATE SETTINGS PROFILE `default` SETTINGS allow_experimental_analyzer = true\n"
+        "CREATE SETTINGS PROFILE `default` SETTINGS enable_analyzer = true\n"
         "CREATE SETTINGS PROFILE `readonly` SETTINGS readonly = 1\n"
         "CREATE SETTINGS PROFILE `xyz`\n",
     ]
@@ -482,7 +482,7 @@ def test_show_profiles():
         "CREATE SETTINGS PROFILE `xyz`\n"
     )
     expected_access_analyzer = (
-        "CREATE SETTINGS PROFILE `default` SETTINGS allow_experimental_analyzer = true\n"
+        "CREATE SETTINGS PROFILE `default` SETTINGS enable_analyzer = true\n"
         "CREATE SETTINGS PROFILE `readonly` SETTINGS readonly = 1\n"
         "CREATE SETTINGS PROFILE `xyz`\n"
     )
diff --git a/tests/performance/storage_join_direct_join.xml b/tests/performance/storage_join_direct_join.xml
index 2fc63c2c926..987500bb4f0 100644
--- a/tests/performance/storage_join_direct_join.xml
+++ b/tests/performance/storage_join_direct_join.xml
@@ -15,5 +15,5 @@
 
     <query>SELECT keys.key, value1 FROM keys ANY LEFT JOIN dict AS d ON (keys.key = d.key) FORMAT Null;</query>
     <query>SELECT keys.key, value1 FROM keys ANY LEFT JOIN dict AS d ON (keys.key = d.key) FORMAT Null SETTINGS
-        allow_experimental_analyzer=1</query>
-</test>
\ No newline at end of file
+        enable_analyzer=1</query>
+</test>
diff --git a/tests/performance/uniq_to_count.xml b/tests/performance/uniq_to_count.xml
index 64e4cf1cc0d..57b0085d8fa 100644
--- a/tests/performance/uniq_to_count.xml
+++ b/tests/performance/uniq_to_count.xml
@@ -3,6 +3,6 @@
     <query>select uniq(number) from (select number from numbers(1000000) group by number)</query>
 
     <!--For new analyzer-->
-    <query>select uniq(number) from (select DISTINCT number from numbers(1000000)) SETTINGS allow_experimental_analyzer=1</query>
-    <query>select uniq(number) from (select number from numbers(1000000) group by number) SETTINGS allow_experimental_analyzer=1</query>
+    <query>select uniq(number) from (select DISTINCT number from numbers(1000000)) SETTINGS enable_analyzer=1</query>
+    <query>select uniq(number) from (select number from numbers(1000000) group by number) SETTINGS enable_analyzer=1</query>
 </test>
diff --git a/tests/queries/0_stateless/00116_storage_set.sql b/tests/queries/0_stateless/00116_storage_set.sql
index c156b387c8f..36ad015c69e 100644
--- a/tests/queries/0_stateless/00116_storage_set.sql
+++ b/tests/queries/0_stateless/00116_storage_set.sql
@@ -28,8 +28,8 @@ RENAME TABLE set2 TO set;
 SELECT arrayJoin(['Hello', 'test', 'World', 'world', 'abc', 'xyz']) AS s WHERE s IN set;
 
 create table tab (x String) engine = MergeTree order by x as select 'Hello';
-SELECT * FROM tab PREWHERE x IN (set) WHERE x IN (set) LIMIT 1 settings allow_experimental_analyzer=0;
-SELECT * FROM tab PREWHERE x IN (set) WHERE x IN (set) LIMIT 1 settings allow_experimental_analyzer=1;
+SELECT * FROM tab PREWHERE x IN (set) WHERE x IN (set) LIMIT 1 settings enable_analyzer=0;
+SELECT * FROM tab PREWHERE x IN (set) WHERE x IN (set) LIMIT 1 settings enable_analyzer=1;
 DROP TABLE tab;
 
 DROP TABLE set;
diff --git a/tests/queries/0_stateless/00202_cross_join.sql b/tests/queries/0_stateless/00202_cross_join.sql
index 8d62c56b3f1..ea327817a58 100644
--- a/tests/queries/0_stateless/00202_cross_join.sql
+++ b/tests/queries/0_stateless/00202_cross_join.sql
@@ -3,5 +3,6 @@ SELECT x, y FROM (SELECT number AS x FROM system.numbers LIMIT 3) js1 CROSS JOIN
 SET join_algorithm = 'auto';
 SELECT x, y FROM (SELECT number AS x FROM system.numbers LIMIT 3) js1 CROSS JOIN (SELECT number AS y FROM system.numbers LIMIT 5) js2;
 
-SET allow_experimental_analyzer = 1;
+-- Just to test that we preserved old setting name this we use `enable_analyzer` instead of `enable_analyzer` here.
+SET enable_analyzer = 1;
 SELECT x, y FROM (SELECT number AS x FROM system.numbers LIMIT 3) js1 CROSS JOIN (SELECT number AS y FROM system.numbers LIMIT 5) js2;
diff --git a/tests/queries/0_stateless/00257_shard_no_aggregates_and_constant_keys.sql b/tests/queries/0_stateless/00257_shard_no_aggregates_and_constant_keys.sql
index e3634141613..b72162be49a 100644
--- a/tests/queries/0_stateless/00257_shard_no_aggregates_and_constant_keys.sql
+++ b/tests/queries/0_stateless/00257_shard_no_aggregates_and_constant_keys.sql
@@ -1,6 +1,6 @@
 -- Tags: shard
 
-set allow_experimental_analyzer = 1;
+set enable_analyzer = 1;
 set enable_positional_arguments = 0;
 
 select 40 as z from (select * from system.numbers limit 3) group by z;
diff --git a/tests/queries/0_stateless/00313_const_totals_extremes.sh b/tests/queries/0_stateless/00313_const_totals_extremes.sh
index 539a19817e2..6267d5066dd 100755
--- a/tests/queries/0_stateless/00313_const_totals_extremes.sh
+++ b/tests/queries/0_stateless/00313_const_totals_extremes.sh
@@ -4,10 +4,10 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
-${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&extremes=1&output_format_write_statistics=0" -d "SELECT 1 AS k, count() GROUP BY k WITH TOTALS SETTINGS allow_experimental_analyzer = 1";
-${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&extremes=1&output_format_write_statistics=0" -d "SELECT 1234567890123 AS k, count() GROUP BY k WITH TOTALS SETTINGS allow_experimental_analyzer = 1 FORMAT JSON";
-${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&extremes=1&output_format_write_statistics=0" -d "SELECT toFloat32(1.23) AS k, count() GROUP BY k WITH TOTALS SETTINGS allow_experimental_analyzer = 1 FORMAT JSONCompact";
+${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&extremes=1&output_format_write_statistics=0" -d "SELECT 1 AS k, count() GROUP BY k WITH TOTALS SETTINGS enable_analyzer = 1";
+${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&extremes=1&output_format_write_statistics=0" -d "SELECT 1234567890123 AS k, count() GROUP BY k WITH TOTALS SETTINGS enable_analyzer = 1 FORMAT JSON";
+${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&extremes=1&output_format_write_statistics=0" -d "SELECT toFloat32(1.23) AS k, count() GROUP BY k WITH TOTALS SETTINGS enable_analyzer = 1 FORMAT JSONCompact";
 
-${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&extremes=1&output_format_write_statistics=0" -d "SELECT toDate('2010-01-01') AS k, count() GROUP BY k WITH TOTALS SETTINGS allow_experimental_analyzer = 1";
-${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&extremes=1&output_format_write_statistics=0" -d "SELECT toDateTime('2010-01-01 01:02:03', 'UTC') AS k, count() GROUP BY k WITH TOTALS SETTINGS allow_experimental_analyzer = 1 FORMAT JSON";
-${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&extremes=1&output_format_write_statistics=0" -d "SELECT 1.1 AS k, count() GROUP BY k WITH TOTALS SETTINGS allow_experimental_analyzer = 1 FORMAT JSONCompact";
+${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&extremes=1&output_format_write_statistics=0" -d "SELECT toDate('2010-01-01') AS k, count() GROUP BY k WITH TOTALS SETTINGS enable_analyzer = 1";
+${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&extremes=1&output_format_write_statistics=0" -d "SELECT toDateTime('2010-01-01 01:02:03', 'UTC') AS k, count() GROUP BY k WITH TOTALS SETTINGS enable_analyzer = 1 FORMAT JSON";
+${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&extremes=1&output_format_write_statistics=0" -d "SELECT 1.1 AS k, count() GROUP BY k WITH TOTALS SETTINGS enable_analyzer = 1 FORMAT JSONCompact";
diff --git a/tests/queries/0_stateless/00331_final_and_prewhere_condition_ver_column.sql b/tests/queries/0_stateless/00331_final_and_prewhere_condition_ver_column.sql
index 78a58a979d1..a3c499f1688 100644
--- a/tests/queries/0_stateless/00331_final_and_prewhere_condition_ver_column.sql
+++ b/tests/queries/0_stateless/00331_final_and_prewhere_condition_ver_column.sql
@@ -1,8 +1,8 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 -- https://github.com/ClickHouse/ClickHouse/issues/45804
 
-CREATE TABLE myRMT(   
+CREATE TABLE myRMT(
   key Int64,
   someCol String,
   ver DateTime
diff --git a/tests/queries/0_stateless/00370_duplicate_columns_in_subqueries.sql b/tests/queries/0_stateless/00370_duplicate_columns_in_subqueries.sql
index 118e50c35e0..8a93c0a9d26 100644
--- a/tests/queries/0_stateless/00370_duplicate_columns_in_subqueries.sql
+++ b/tests/queries/0_stateless/00370_duplicate_columns_in_subqueries.sql
@@ -1,6 +1,6 @@
 SET any_join_distinct_right_table_keys = 1;
 SET joined_subquery_requires_alias = 0;
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 select x, y from (select 1 as x, 2 as y, x, y);
 select x, y from (select 1 as x, 1 as y, x, y);
diff --git a/tests/queries/0_stateless/00378_json_quote_64bit_integers.sql b/tests/queries/0_stateless/00378_json_quote_64bit_integers.sql
index e7b59bc3f7f..941d5ecd135 100644
--- a/tests/queries/0_stateless/00378_json_quote_64bit_integers.sql
+++ b/tests/queries/0_stateless/00378_json_quote_64bit_integers.sql
@@ -2,7 +2,7 @@
 
 SET output_format_write_statistics = 0;
 SET extremes = 1;
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 SET output_format_json_quote_64bit_integers = 1;
 SELECT toInt64(0) as i0, toUInt64(0) as u0, toInt64(9223372036854775807) as ip, toInt64(-9223372036854775808) as in, toUInt64(18446744073709551615) as up, [toInt64(0)] as arr, (toUInt64(0), toUInt64(0)) as tuple GROUP BY i0, u0, ip, in, up, arr, tuple WITH TOTALS FORMAT JSON;
diff --git a/tests/queries/0_stateless/00445_join_nullable_keys.sql b/tests/queries/0_stateless/00445_join_nullable_keys.sql
index 774594f90f3..bec0c76eb5f 100644
--- a/tests/queries/0_stateless/00445_join_nullable_keys.sql
+++ b/tests/queries/0_stateless/00445_join_nullable_keys.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 SET join_use_nulls = 0;
 SET any_join_distinct_right_table_keys = 1;
 
diff --git a/tests/queries/0_stateless/00490_with_select.sql b/tests/queries/0_stateless/00490_with_select.sql
index c803cf1d3ad..d63f0ca4f31 100644
--- a/tests/queries/0_stateless/00490_with_select.sql
+++ b/tests/queries/0_stateless/00490_with_select.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 with pow(2,2) as four select pow(four, 2), 2 as two, pow(two, 2);
 select `pow(four, 2)`, `pow(two, 2)` from (with pow(2,2) as four select pow(four, 2), 2 as two, pow(two, 2));
diff --git a/tests/queries/0_stateless/00597_push_down_predicate_long.reference b/tests/queries/0_stateless/00597_push_down_predicate_long.reference
index 2c46edc98bf..55b7cdd3c64 100644
--- a/tests/queries/0_stateless/00597_push_down_predicate_long.reference
+++ b/tests/queries/0_stateless/00597_push_down_predicate_long.reference
@@ -390,7 +390,7 @@ ANY LEFT JOIN
 ) USING (id)
 WHERE id = 1
 2000-01-01	1	test string 1	1	2000-01-01	test string 1	1
-------- allow_experimental_analyzer=1 -------
+------- enable_analyzer=1 -------
 1	2000-01-01	test string 1	1	2000-01-01	test string 1	1
 SELECT
     id,
@@ -454,7 +454,7 @@ FROM
 )
 WHERE id = 1
 2000-01-01	1	test string 1	1
-------- allow_experimental_analyzer=1 -------
+------- enable_analyzer=1 -------
 1	2000-01-01	test string 1	1	2000-01-01	test string 1	1
 SELECT
     date,
@@ -484,7 +484,7 @@ ANY LEFT JOIN
 ) AS b USING (id)
 WHERE b.id = 1
 2000-01-01	1	test string 1	1	2000-01-01	test string 1	1
-------- allow_experimental_analyzer=1 -------
+------- enable_analyzer=1 -------
 1	2000-01-01	test string 1	1	2000-01-01	test string 1	1
 SELECT
     id,
@@ -510,7 +510,7 @@ ANY LEFT JOIN
 ) AS b USING (date, id)
 WHERE b.date = toDate(\'2000-01-01\')
 1	2000-01-01	test string 1	1
-------- allow_experimental_analyzer=1 -------
+------- enable_analyzer=1 -------
 2000-01-01	1	test string 1	1
 SELECT
     date,
@@ -593,7 +593,7 @@ SEMI LEFT JOIN
 ) AS r USING (id)
 WHERE r.id = 1
 2000-01-01	1	test string 1	1	2000-01-01	test string 1	1
-------- allow_experimental_analyzer=1 -------
+------- enable_analyzer=1 -------
 1	2000-01-01	test string 1	1	2000-01-01	test string 1	1
 SELECT value + t1.value AS expr
 FROM
diff --git a/tests/queries/0_stateless/00597_push_down_predicate_long.sql b/tests/queries/0_stateless/00597_push_down_predicate_long.sql
index caf6edd7372..f79b24abe56 100644
--- a/tests/queries/0_stateless/00597_push_down_predicate_long.sql
+++ b/tests/queries/0_stateless/00597_push_down_predicate_long.sql
@@ -110,9 +110,9 @@ SELECT * FROM (SELECT * FROM test_00597 UNION ALL SELECT * FROM test_00597) WHER
 
 -- Optimize predicate expression with join query
 EXPLAIN SYNTAX SELECT * FROM (SELECT * FROM test_00597) ANY LEFT JOIN (SELECT * FROM test_00597) USING id WHERE id = 1;
-SELECT * FROM (SELECT * FROM test_00597) ANY LEFT JOIN (SELECT * FROM test_00597) USING id WHERE id = 1 SETTINGS allow_experimental_analyzer=0;
-SELECT '------- allow_experimental_analyzer=1 -------';
-SELECT * FROM (SELECT * FROM test_00597) ANY LEFT JOIN (SELECT * FROM test_00597) USING id WHERE id = 1 SETTINGS allow_experimental_analyzer=1;
+SELECT * FROM (SELECT * FROM test_00597) ANY LEFT JOIN (SELECT * FROM test_00597) USING id WHERE id = 1 SETTINGS enable_analyzer=0;
+SELECT '------- enable_analyzer=1 -------';
+SELECT * FROM (SELECT * FROM test_00597) ANY LEFT JOIN (SELECT * FROM test_00597) USING id WHERE id = 1 SETTINGS enable_analyzer=1;
 
 EXPLAIN SYNTAX SELECT * FROM (SELECT toInt8(1) AS id) ANY LEFT JOIN test_00597 USING id WHERE value = 1;
 SELECT * FROM (SELECT toInt8(1) AS id) ANY LEFT JOIN test_00597 USING id WHERE value = 1;
@@ -123,30 +123,30 @@ SELECT b.value FROM (SELECT toInt8(1) AS id) ANY LEFT JOIN test_00597 AS b USING
 
 -- Optimize predicate expression with join and nested subquery
 EXPLAIN SYNTAX SELECT * FROM (SELECT * FROM (SELECT * FROM test_00597) ANY LEFT JOIN (SELECT * FROM test_00597) USING id) WHERE id = 1;
-SELECT * FROM (SELECT * FROM (SELECT * FROM test_00597) ANY LEFT JOIN (SELECT * FROM test_00597) USING id) WHERE id = 1 SETTINGS allow_experimental_analyzer=0;
-SELECT '------- allow_experimental_analyzer=1 -------';
-SELECT * FROM (SELECT * FROM (SELECT * FROM test_00597) ANY LEFT JOIN (SELECT * FROM test_00597) USING id) WHERE id = 1 SETTINGS allow_experimental_analyzer=1;
+SELECT * FROM (SELECT * FROM (SELECT * FROM test_00597) ANY LEFT JOIN (SELECT * FROM test_00597) USING id) WHERE id = 1 SETTINGS enable_analyzer=0;
+SELECT '------- enable_analyzer=1 -------';
+SELECT * FROM (SELECT * FROM (SELECT * FROM test_00597) ANY LEFT JOIN (SELECT * FROM test_00597) USING id) WHERE id = 1 SETTINGS enable_analyzer=1;
 
 -- Optimize predicate expression with join query and qualified
 EXPLAIN SYNTAX SELECT * FROM (SELECT * FROM test_00597) ANY LEFT JOIN (SELECT * FROM test_00597) AS b USING id WHERE b.id = 1;
-SELECT * FROM (SELECT * FROM test_00597) ANY LEFT JOIN (SELECT * FROM test_00597) AS b USING id WHERE b.id = 1 SETTINGS allow_experimental_analyzer=0;
-SELECT '------- allow_experimental_analyzer=1 -------';
-SELECT * FROM (SELECT * FROM test_00597) ANY LEFT JOIN (SELECT * FROM test_00597) AS b USING id WHERE b.id = 1 SETTINGS allow_experimental_analyzer=1;
+SELECT * FROM (SELECT * FROM test_00597) ANY LEFT JOIN (SELECT * FROM test_00597) AS b USING id WHERE b.id = 1 SETTINGS enable_analyzer=0;
+SELECT '------- enable_analyzer=1 -------';
+SELECT * FROM (SELECT * FROM test_00597) ANY LEFT JOIN (SELECT * FROM test_00597) AS b USING id WHERE b.id = 1 SETTINGS enable_analyzer=1;
 
 -- Compatibility test
 EXPLAIN SYNTAX SELECT * FROM (SELECT toInt8(1) AS id, toDate('2000-01-01') AS date FROM system.numbers LIMIT 1) ANY LEFT JOIN (SELECT * FROM test_00597) AS b USING date, id WHERE b.date = toDate('2000-01-01');
-SELECT * FROM (SELECT toInt8(1) AS id, toDate('2000-01-01') AS date FROM system.numbers LIMIT 1) ANY LEFT JOIN (SELECT * FROM test_00597) AS b USING date, id WHERE b.date = toDate('2000-01-01') SETTINGS allow_experimental_analyzer=0;
-SELECT '------- allow_experimental_analyzer=1 -------';
-SELECT * FROM (SELECT toInt8(1) AS id, toDate('2000-01-01') AS date FROM system.numbers LIMIT 1) ANY LEFT JOIN (SELECT * FROM test_00597) AS b USING date, id WHERE b.date = toDate('2000-01-01') SETTINGS allow_experimental_analyzer=1;
+SELECT * FROM (SELECT toInt8(1) AS id, toDate('2000-01-01') AS date FROM system.numbers LIMIT 1) ANY LEFT JOIN (SELECT * FROM test_00597) AS b USING date, id WHERE b.date = toDate('2000-01-01') SETTINGS enable_analyzer=0;
+SELECT '------- enable_analyzer=1 -------';
+SELECT * FROM (SELECT toInt8(1) AS id, toDate('2000-01-01') AS date FROM system.numbers LIMIT 1) ANY LEFT JOIN (SELECT * FROM test_00597) AS b USING date, id WHERE b.date = toDate('2000-01-01') SETTINGS enable_analyzer=1;
 
 EXPLAIN SYNTAX SELECT * FROM (SELECT * FROM (SELECT * FROM test_00597) AS a ANY LEFT JOIN (SELECT * FROM test_00597) AS b  ON  a.id = b.id) WHERE id = 1;
 SELECT * FROM (SELECT * FROM (SELECT * FROM test_00597) AS a ANY LEFT JOIN (SELECT * FROM test_00597) AS b  ON  a.id = b.id) WHERE id = 1;
 
 -- Explain with join subquery
 EXPLAIN SYNTAX SELECT * FROM (SELECT * FROM test_00597) ANY INNER JOIN (SELECT * FROM (SELECT * FROM test_00597)) as r USING id WHERE r.id = 1;
-SELECT * FROM (SELECT * FROM test_00597) ANY INNER JOIN (SELECT * FROM (SELECT * FROM test_00597)) as r USING id WHERE r.id = 1 SETTINGS allow_experimental_analyzer=0;
-SELECT '------- allow_experimental_analyzer=1 -------';
-SELECT * FROM (SELECT * FROM test_00597) ANY INNER JOIN (SELECT * FROM (SELECT * FROM test_00597)) as r USING id WHERE r.id = 1 SETTINGS allow_experimental_analyzer=1;
+SELECT * FROM (SELECT * FROM test_00597) ANY INNER JOIN (SELECT * FROM (SELECT * FROM test_00597)) as r USING id WHERE r.id = 1 SETTINGS enable_analyzer=0;
+SELECT '------- enable_analyzer=1 -------';
+SELECT * FROM (SELECT * FROM test_00597) ANY INNER JOIN (SELECT * FROM (SELECT * FROM test_00597)) as r USING id WHERE r.id = 1 SETTINGS enable_analyzer=1;
 
 -- issue 20497
 EXPLAIN SYNTAX SELECT value + t1.value AS expr FROM (SELECT t0.value, t1.value FROM test_00597 AS t0 FULL JOIN test_00597 AS t1 USING date) WHERE expr < 3;
diff --git a/tests/queries/0_stateless/00621_regression_for_in_operator.reference b/tests/queries/0_stateless/00621_regression_for_in_operator.reference
index b68f550a742..ab8a1499f6d 100644
--- a/tests/queries/0_stateless/00621_regression_for_in_operator.reference
+++ b/tests/queries/0_stateless/00621_regression_for_in_operator.reference
@@ -17,7 +17,7 @@ QUERY id: 0
         LIST id: 5, nodes: 2
           COLUMN id: 6, column_name: g, result_type: String, source_id: 3
           CONSTANT id: 7, constant_value: Tuple_(\'5\', \'6\'), constant_value_type: Tuple(String, String)
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 2
 2
 QUERY id: 0
@@ -42,4 +42,4 @@ QUERY id: 0
               LIST id: 11, nodes: 2
                 COLUMN id: 8, column_name: g, result_type: String, source_id: 3
                 CONSTANT id: 12, constant_value: \'6\', constant_value_type: String
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
diff --git a/tests/queries/0_stateless/00621_regression_for_in_operator.sql b/tests/queries/0_stateless/00621_regression_for_in_operator.sql
index db1bcb4a39a..0d8c4933c65 100644
--- a/tests/queries/0_stateless/00621_regression_for_in_operator.sql
+++ b/tests/queries/0_stateless/00621_regression_for_in_operator.sql
@@ -12,13 +12,13 @@ SELECT count() FROM regression_for_in_operator_view WHERE g IN ('5','6');
 
 SET optimize_min_equality_disjunction_chain_length = 1;
 SELECT count() FROM regression_for_in_operator_view WHERE g = '5' OR g = '6';
-SELECT count() FROM regression_for_in_operator_view WHERE g = '5' OR g = '6' SETTINGS allow_experimental_analyzer = 1;
-EXPLAIN QUERY TREE SELECT count() FROM regression_for_in_operator_view WHERE g = '5' OR g = '6' SETTINGS allow_experimental_analyzer = 1;
+SELECT count() FROM regression_for_in_operator_view WHERE g = '5' OR g = '6' SETTINGS enable_analyzer = 1;
+EXPLAIN QUERY TREE SELECT count() FROM regression_for_in_operator_view WHERE g = '5' OR g = '6' SETTINGS enable_analyzer = 1;
 
 SET optimize_min_equality_disjunction_chain_length = 3;
 SELECT count() FROM regression_for_in_operator_view WHERE g = '5' OR g = '6';
-SELECT count() FROM regression_for_in_operator_view WHERE g = '5' OR g = '6' SETTINGS allow_experimental_analyzer = 1;
-EXPLAIN QUERY TREE SELECT count() FROM regression_for_in_operator_view WHERE g = '5' OR g = '6' SETTINGS allow_experimental_analyzer = 1;
+SELECT count() FROM regression_for_in_operator_view WHERE g = '5' OR g = '6' SETTINGS enable_analyzer = 1;
+EXPLAIN QUERY TREE SELECT count() FROM regression_for_in_operator_view WHERE g = '5' OR g = '6' SETTINGS enable_analyzer = 1;
 
 DROP TABLE regression_for_in_operator_view;
 DROP TABLE regression_for_in_operator;
diff --git a/tests/queries/0_stateless/00674_join_on_syntax.sql b/tests/queries/0_stateless/00674_join_on_syntax.sql
index 9ff26db1536..584e43b88bf 100644
--- a/tests/queries/0_stateless/00674_join_on_syntax.sql
+++ b/tests/queries/0_stateless/00674_join_on_syntax.sql
@@ -1,5 +1,5 @@
 SET joined_subquery_requires_alias = 0;
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 drop table if exists tab1;
 drop table if exists tab2;
diff --git a/tests/queries/0_stateless/00700_decimal_compare.sql b/tests/queries/0_stateless/00700_decimal_compare.sql
index beadbdade16..1b901e04c28 100644
--- a/tests/queries/0_stateless/00700_decimal_compare.sql
+++ b/tests/queries/0_stateless/00700_decimal_compare.sql
@@ -27,7 +27,7 @@ SELECT a > 0, b > 0, g > 0 FROM decimal ORDER BY a DESC;
 SELECT a, g > toInt8(0), g > toInt16(0), g > toInt32(0), g > toInt64(0) FROM decimal ORDER BY a;
 SELECT a, g > toUInt8(0), g > toUInt16(0), g > toUInt32(0), g > toUInt64(0) FROM decimal ORDER BY a;
 SELECT a, b, g FROM decimal WHERE a IN(42) AND b IN(42) AND g IN(42);
-SELECT a, b, g FROM decimal WHERE a IN(42) AND b IN(42) AND g IN(42) SETTINGS allow_experimental_analyzer = 1;
+SELECT a, b, g FROM decimal WHERE a IN(42) AND b IN(42) AND g IN(42) SETTINGS enable_analyzer = 1;
 SELECT a, b, g FROM decimal WHERE a > 0 AND a <= 42 AND b <= 42 AND g <= 42;
 
 SELECT d, e, f from decimal WHERE d > 0 AND d < 1 AND e > 0 AND e < 1 AND f > 0 AND f < 1;
diff --git a/tests/queries/0_stateless/00722_inner_join.sql b/tests/queries/0_stateless/00722_inner_join.sql
index 0d5a543b99d..aa590f470ae 100644
--- a/tests/queries/0_stateless/00722_inner_join.sql
+++ b/tests/queries/0_stateless/00722_inner_join.sql
@@ -1,6 +1,6 @@
 -- Tags: no-parallel
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS one;
 CREATE TABLE one(dummy UInt8) ENGINE = Memory;
diff --git a/tests/queries/0_stateless/00736_disjunction_optimisation.reference b/tests/queries/0_stateless/00736_disjunction_optimisation.reference
index f28dcacef0e..4e7f2e09dea 100644
--- a/tests/queries/0_stateless/00736_disjunction_optimisation.reference
+++ b/tests/queries/0_stateless/00736_disjunction_optimisation.reference
@@ -49,7 +49,7 @@ QUERY id: 0
               LIST id: 12, nodes: 2
                 COLUMN id: 13, column_name: s, result_type: UInt64, source_id: 3
                 CONSTANT id: 14, constant_value: Tuple_(UInt64_21, UInt64_22, UInt64_23), constant_value_type: Tuple(UInt8, UInt8, UInt8)
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 1	21
 1	22
 1	23
@@ -99,7 +99,7 @@ QUERY id: 0
         LIST id: 14, nodes: 2
           COLUMN id: 15, column_name: s, result_type: UInt64, source_id: 3
           CONSTANT id: 16, constant_value: Tuple_(UInt64_21, UInt64_22, UInt64_23), constant_value_type: Tuple(UInt8, UInt8, UInt8)
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 1	1	21	1	1	1
 1	1	22	0	1	1
 1	1	23	0	0	1
@@ -152,7 +152,7 @@ QUERY id: 0
             CONSTANT id: 16, constant_value: Tuple_(UInt64_21, UInt64_22, UInt64_23), constant_value_type: Tuple(UInt8, UInt8, UInt8)
   JOIN TREE
     TABLE id: 3, alias: __table1, table_name: default.bug
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 21	1
 22	1
 23	1
@@ -185,7 +185,7 @@ QUERY id: 0
             CONSTANT id: 6, constant_value: Tuple_(UInt64_21, UInt64_22, UInt64_23), constant_value_type: Tuple(UInt8, UInt8, UInt8)
   JOIN TREE
     TABLE id: 3, alias: __table1, table_name: default.bug
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 1	21
 1	22
 1	23
@@ -237,7 +237,7 @@ QUERY id: 0
               LIST id: 12, nodes: 2
                 COLUMN id: 13, column_name: s, result_type: UInt64, source_id: 3
                 CONSTANT id: 14, constant_value: Tuple_(UInt64_21, UInt64_22, UInt64_23), constant_value_type: Tuple(UInt8, UInt8, UInt8)
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 1	21
 1	22
 1	23
@@ -287,7 +287,7 @@ QUERY id: 0
         LIST id: 14, nodes: 2
           COLUMN id: 15, column_name: s, result_type: UInt64, source_id: 3
           CONSTANT id: 16, constant_value: Tuple_(UInt64_21, UInt64_22, UInt64_23), constant_value_type: Tuple(UInt8, UInt8, UInt8)
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 1	1	21	1	1	1
 1	1	22	0	1	1
 1	1	23	0	0	1
@@ -348,7 +348,7 @@ QUERY id: 0
             CONSTANT id: 21, constant_value: Tuple_(UInt64_21, UInt64_22, UInt64_23), constant_value_type: Tuple(UInt8, UInt8, UInt8)
   JOIN TREE
     TABLE id: 3, alias: __table1, table_name: default.bug
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 21	1
 22	1
 23	1
@@ -381,7 +381,7 @@ QUERY id: 0
             CONSTANT id: 6, constant_value: Tuple_(UInt64_21, UInt64_22, UInt64_23), constant_value_type: Tuple(UInt8, UInt8, UInt8)
   JOIN TREE
     TABLE id: 3, alias: __table1, table_name: default.bug
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 21	1
 22	1
 23	1
@@ -414,4 +414,4 @@ QUERY id: 0
             CONSTANT id: 6, constant_value: Tuple_(UInt64_21, UInt64_22, UInt64_23), constant_value_type: Tuple(UInt8, UInt8, UInt8)
   JOIN TREE
     TABLE id: 3, alias: __table1, table_name: default.bug
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
diff --git a/tests/queries/0_stateless/00736_disjunction_optimisation.sql b/tests/queries/0_stateless/00736_disjunction_optimisation.sql
index e5bfc81f7ae..38f77622bd4 100644
--- a/tests/queries/0_stateless/00736_disjunction_optimisation.sql
+++ b/tests/queries/0_stateless/00736_disjunction_optimisation.sql
@@ -7,43 +7,43 @@ set optimize_min_equality_disjunction_chain_length = 2;
 select * from bug;
 
 select * from bug where (k =1 or k=2 or k =3) and (s=21 or s=22 or s=23);
-select * from bug where (k =1 or k=2 or k =3) and (s=21 or s=22 or s=23) SETTINGS allow_experimental_analyzer = 1;
-explain query tree select * from bug where (k =1 or k=2 or k =3) and (s=21 or s=22 or s=23) SETTINGS allow_experimental_analyzer = 1;;
+select * from bug where (k =1 or k=2 or k =3) and (s=21 or s=22 or s=23) SETTINGS enable_analyzer = 1;
+explain query tree select * from bug where (k =1 or k=2 or k =3) and (s=21 or s=22 or s=23) SETTINGS enable_analyzer = 1;;
 
 select * from (select * from bug where k=1 or k=2 or k=3) where (s=21 or s=22 or s=23);
-select * from (select * from bug where k=1 or k=2 or k=3) where (s=21 or s=22 or s=23) SETTINGS allow_experimental_analyzer = 1;;
-explain query tree select * from (select * from bug where k=1 or k=2 or k=3) where (s=21 or s=22 or s=23) SETTINGS allow_experimental_analyzer = 1;;
+select * from (select * from bug where k=1 or k=2 or k=3) where (s=21 or s=22 or s=23) SETTINGS enable_analyzer = 1;;
+explain query tree select * from (select * from bug where k=1 or k=2 or k=3) where (s=21 or s=22 or s=23) SETTINGS enable_analyzer = 1;;
 
 select k, (k=1 or k=2 or k=3), s, (s=21), (s=21 or s=22), (s=21 or s=22 or s=23) from bug;
-select k, (k=1 or k=2 or k=3), s, (s=21), (s=21 or s=22), (s=21 or s=22 or s=23) from bug SETTINGS allow_experimental_analyzer = 1;;
-explain query tree select k, (k=1 or k=2 or k=3), s, (s=21), (s=21 or s=22), (s=21 or s=22 or s=23) from bug SETTINGS allow_experimental_analyzer = 1;;
+select k, (k=1 or k=2 or k=3), s, (s=21), (s=21 or s=22), (s=21 or s=22 or s=23) from bug SETTINGS enable_analyzer = 1;;
+explain query tree select k, (k=1 or k=2 or k=3), s, (s=21), (s=21 or s=22), (s=21 or s=22 or s=23) from bug SETTINGS enable_analyzer = 1;;
 
 select s, (s=21 or s=22 or s=23) from bug;
-select s, (s=21 or s=22 or s=23) from bug SETTINGS allow_experimental_analyzer = 1;;
-explain query tree select s, (s=21 or s=22 or s=23) from bug SETTINGS allow_experimental_analyzer = 1;;
+select s, (s=21 or s=22 or s=23) from bug SETTINGS enable_analyzer = 1;;
+explain query tree select s, (s=21 or s=22 or s=23) from bug SETTINGS enable_analyzer = 1;;
 
 set optimize_min_equality_disjunction_chain_length = 3;
 
 select * from bug;
 
 select * from bug where (k =1 or k=2 or k =3) and (s=21 or s=22 or s=23);
-select * from bug where (k =1 or k=2 or k =3) and (s=21 or s=22 or s=23) SETTINGS allow_experimental_analyzer = 1;
-explain query tree select * from bug where (k =1 or k=2 or k =3) and (s=21 or s=22 or s=23) SETTINGS allow_experimental_analyzer = 1;;
+select * from bug where (k =1 or k=2 or k =3) and (s=21 or s=22 or s=23) SETTINGS enable_analyzer = 1;
+explain query tree select * from bug where (k =1 or k=2 or k =3) and (s=21 or s=22 or s=23) SETTINGS enable_analyzer = 1;;
 
 select * from (select * from bug where k=1 or k=2 or k=3) where (s=21 or s=22 or s=23);
-select * from (select * from bug where k=1 or k=2 or k=3) where (s=21 or s=22 or s=23) SETTINGS allow_experimental_analyzer = 1;;
-explain query tree select * from (select * from bug where k=1 or k=2 or k=3) where (s=21 or s=22 or s=23) SETTINGS allow_experimental_analyzer = 1;;
+select * from (select * from bug where k=1 or k=2 or k=3) where (s=21 or s=22 or s=23) SETTINGS enable_analyzer = 1;;
+explain query tree select * from (select * from bug where k=1 or k=2 or k=3) where (s=21 or s=22 or s=23) SETTINGS enable_analyzer = 1;;
 
 select k, (k=1 or k=2 or k=3), s, (s=21), (s=21 or s=22), (s=21 or s=22 or s=23) from bug;
-select k, (k=1 or k=2 or k=3), s, (s=21), (s=21 or s=22), (s=21 or s=22 or s=23) from bug SETTINGS allow_experimental_analyzer = 1;;
-explain query tree select k, (k=1 or k=2 or k=3), s, (s=21), (s=21 or s=22), (s=21 or s=22 or s=23) from bug SETTINGS allow_experimental_analyzer = 1;;
+select k, (k=1 or k=2 or k=3), s, (s=21), (s=21 or s=22), (s=21 or s=22 or s=23) from bug SETTINGS enable_analyzer = 1;;
+explain query tree select k, (k=1 or k=2 or k=3), s, (s=21), (s=21 or s=22), (s=21 or s=22 or s=23) from bug SETTINGS enable_analyzer = 1;;
 
 select s, (s=21 or s=22 or s=23) from bug;
-select s, (s=21 or s=22 or s=23) from bug SETTINGS allow_experimental_analyzer = 1;;
-explain query tree select s, (s=21 or s=22 or s=23) from bug SETTINGS allow_experimental_analyzer = 1;;
+select s, (s=21 or s=22 or s=23) from bug SETTINGS enable_analyzer = 1;;
+explain query tree select s, (s=21 or s=22 or s=23) from bug SETTINGS enable_analyzer = 1;;
 
 select s, (s=21 or 22=s or 23=s) from bug;
-select s, (s=21 or 22=s or 23=s) from bug SETTINGS allow_experimental_analyzer = 1;;
-explain query tree select s, (s=21 or 22=s or 23=s) from bug SETTINGS allow_experimental_analyzer = 1;;
+select s, (s=21 or 22=s or 23=s) from bug SETTINGS enable_analyzer = 1;;
+explain query tree select s, (s=21 or 22=s or 23=s) from bug SETTINGS enable_analyzer = 1;;
 
 DROP TABLE bug;
diff --git a/tests/queries/0_stateless/00757_enum_defaults_const.sql b/tests/queries/0_stateless/00757_enum_defaults_const.sql
index 64271a37473..048c9dee88f 100644
--- a/tests/queries/0_stateless/00757_enum_defaults_const.sql
+++ b/tests/queries/0_stateless/00757_enum_defaults_const.sql
@@ -1,3 +1,3 @@
-SET allow_experimental_analyzer=0;
+SET enable_analyzer=0;
 select os_name, count() from (SELECT CAST('iphone' AS Enum8('iphone' = 1, 'android' = 2)) AS os_name) group by os_name WITH TOTALS;
 select toNullable(os_name) AS os_name, count() from (SELECT CAST('iphone' AS Enum8('iphone' = 1, 'android' = 2)) AS os_name) group by os_name WITH TOTALS;
diff --git a/tests/queries/0_stateless/00757_enum_defaults_const_analyzer.sql b/tests/queries/0_stateless/00757_enum_defaults_const_analyzer.sql
index bf079539019..c202ed630db 100644
--- a/tests/queries/0_stateless/00757_enum_defaults_const_analyzer.sql
+++ b/tests/queries/0_stateless/00757_enum_defaults_const_analyzer.sql
@@ -1,3 +1,3 @@
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 select os_name, count() from (SELECT CAST('iphone' AS Enum8('iphone' = 1, 'android' = 2)) AS os_name) group by os_name WITH TOTALS;
 select toNullable(os_name) AS os_name, count() from (SELECT CAST('iphone' AS Enum8('iphone' = 1, 'android' = 2)) AS os_name) group by os_name WITH TOTALS;
diff --git a/tests/queries/0_stateless/00800_low_cardinality_join.sql b/tests/queries/0_stateless/00800_low_cardinality_join.sql
index ecb5194253c..fc5f5d1860c 100644
--- a/tests/queries/0_stateless/00800_low_cardinality_join.sql
+++ b/tests/queries/0_stateless/00800_low_cardinality_join.sql
@@ -1,4 +1,4 @@
-set allow_experimental_analyzer = 1;
+set enable_analyzer = 1;
 set joined_subquery_requires_alias = 0;
 
 select * from (select dummy as val from system.one) any left join (select dummy as val from system.one) using val;
diff --git a/tests/queries/0_stateless/00800_low_cardinality_merge_join.sql.j2 b/tests/queries/0_stateless/00800_low_cardinality_merge_join.sql.j2
index 8e2037480c7..a5f5e9b33b8 100644
--- a/tests/queries/0_stateless/00800_low_cardinality_merge_join.sql.j2
+++ b/tests/queries/0_stateless/00800_low_cardinality_merge_join.sql.j2
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 {% for join_algorithm in ['partial_merge', 'full_sorting_merge', 'grace_hash'] -%}
 
diff --git a/tests/queries/0_stateless/00818_alias_bug_4110.sql b/tests/queries/0_stateless/00818_alias_bug_4110.sql
index d057bacc908..1242a3b605d 100644
--- a/tests/queries/0_stateless/00818_alias_bug_4110.sql
+++ b/tests/queries/0_stateless/00818_alias_bug_4110.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 select s.a as a, s.a + 1 as b from (select 10 as a) s;
 select s.a + 1 as a, s.a as b from (select 10 as a) s;
diff --git a/tests/queries/0_stateless/00818_inner_join_bug_3567.sql b/tests/queries/0_stateless/00818_inner_join_bug_3567.sql
index 2dec5ce3221..3b4b3cd77d3 100644
--- a/tests/queries/0_stateless/00818_inner_join_bug_3567.sql
+++ b/tests/queries/0_stateless/00818_inner_join_bug_3567.sql
@@ -1,5 +1,5 @@
 SET output_format_pretty_color = 1;
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS table1;
 DROP TABLE IF EXISTS table2;
diff --git a/tests/queries/0_stateless/00819_full_join_wrong_columns_in_block.sql b/tests/queries/0_stateless/00819_full_join_wrong_columns_in_block.sql
index 3c0246619da..8b3d3ec1bc1 100644
--- a/tests/queries/0_stateless/00819_full_join_wrong_columns_in_block.sql
+++ b/tests/queries/0_stateless/00819_full_join_wrong_columns_in_block.sql
@@ -1,6 +1,6 @@
 SET any_join_distinct_right_table_keys = 1;
 SET joined_subquery_requires_alias = 0;
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 SELECT * FROM (SELECT 1 AS a, 'x' AS b) join (SELECT 1 as a, 'y' as b) using a;
 SELECT * FROM (SELECT 1 AS a, 'x' AS b) left join (SELECT 1 as a, 'y' as b) using a;
diff --git a/tests/queries/0_stateless/00820_multiple_joins.sql b/tests/queries/0_stateless/00820_multiple_joins.sql
index 5c7a7bebb0b..b4197570cfa 100644
--- a/tests/queries/0_stateless/00820_multiple_joins.sql
+++ b/tests/queries/0_stateless/00820_multiple_joins.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS table1;
 DROP TABLE IF EXISTS table2;
diff --git a/tests/queries/0_stateless/00820_multiple_joins_subquery_requires_alias.sql b/tests/queries/0_stateless/00820_multiple_joins_subquery_requires_alias.sql
index 3da2cad4eff..538e6967ff3 100644
--- a/tests/queries/0_stateless/00820_multiple_joins_subquery_requires_alias.sql
+++ b/tests/queries/0_stateless/00820_multiple_joins_subquery_requires_alias.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS table1;
 DROP TABLE IF EXISTS table2;
diff --git a/tests/queries/0_stateless/00830_join_overwrite.sql b/tests/queries/0_stateless/00830_join_overwrite.sql
index bc3662528db..f51a152ea46 100644
--- a/tests/queries/0_stateless/00830_join_overwrite.sql
+++ b/tests/queries/0_stateless/00830_join_overwrite.sql
@@ -13,7 +13,7 @@ SELECT joinGet('kv_overwrite', 'v', toUInt32(1));
 CREATE TABLE t2 (k UInt32, v UInt32) ENGINE = Memory;
 INSERT INTO t2 VALUES (1, 2), (1, 3);
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 SELECT v FROM (SELECT 1 as k) t1 ANY INNER JOIN t2 USING (k) SETTINGS join_any_take_last_row = 0;
 SELECT v FROM (SELECT 1 as k) t1 ANY INNER JOIN t2 USING (k) SETTINGS join_any_take_last_row = 1;
diff --git a/tests/queries/0_stateless/00835_if_generic_case.sql b/tests/queries/0_stateless/00835_if_generic_case.sql
index 051fad14603..e06e610dbb5 100644
--- a/tests/queries/0_stateless/00835_if_generic_case.sql
+++ b/tests/queries/0_stateless/00835_if_generic_case.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 SELECT toDateTime('2000-01-01 00:00:00', 'Asia/Istanbul') AS x, toDate('2000-01-02') AS y, x > y ? x : y AS z;
 SELECT materialize(toDateTime('2000-01-01 00:00:00', 'Asia/Istanbul')) AS x, toDate('2000-01-02') AS y, x > y ? x : y AS z;
diff --git a/tests/queries/0_stateless/00848_join_use_nulls_segfault.sql b/tests/queries/0_stateless/00848_join_use_nulls_segfault.sql
index 2f6cca0284c..275968236ae 100644
--- a/tests/queries/0_stateless/00848_join_use_nulls_segfault.sql
+++ b/tests/queries/0_stateless/00848_join_use_nulls_segfault.sql
@@ -1,5 +1,5 @@
 SET any_join_distinct_right_table_keys = 1;
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS t1_00848;
 DROP TABLE IF EXISTS t2_00848;
diff --git a/tests/queries/0_stateless/00849_multiple_comma_join_2.sql b/tests/queries/0_stateless/00849_multiple_comma_join_2.sql
index 6530f691087..250dd4a47ab 100644
--- a/tests/queries/0_stateless/00849_multiple_comma_join_2.sql
+++ b/tests/queries/0_stateless/00849_multiple_comma_join_2.sql
@@ -12,7 +12,7 @@ CREATE TABLE t2 (a UInt32, b Nullable(Int32)) ENGINE = Memory;
 CREATE TABLE t3 (a UInt32, b Nullable(Int32)) ENGINE = Memory;
 CREATE TABLE t4 (a UInt32, b Nullable(Int32)) ENGINE = Memory;
 
-SET allow_experimental_analyzer = 0;
+SET enable_analyzer = 0;
 
 --- EXPLAIN SYNTAX (old AST based optimization)
 SELECT countIf(explain like '%COMMA%' OR explain like '%CROSS%'), countIf(explain like '%INNER%') FROM (
@@ -62,56 +62,56 @@ SELECT countIf(explain like '%COMMA%' OR explain like '%CROSS%'), countIf(explai
 
 --- EXPLAIN QUERY TREE
 SELECT countIf(explain like '%COMMA%' OR explain like '%CROSS%'), countIf(explain like '%INNER%') FROM (
-    EXPLAIN QUERY TREE SELECT t1.a FROM t1, t2 WHERE t1.a = t2.a) SETTINGS allow_experimental_analyzer = 1;
+    EXPLAIN QUERY TREE SELECT t1.a FROM t1, t2 WHERE t1.a = t2.a) SETTINGS enable_analyzer = 1;
 
 SELECT countIf(explain like '%COMMA%' OR explain like '%CROSS%'), countIf(explain like '%INNER%') FROM (
-    EXPLAIN QUERY TREE SELECT t1.a FROM t1, t2 WHERE t1.b = t2.b) SETTINGS allow_experimental_analyzer = 1;
+    EXPLAIN QUERY TREE SELECT t1.a FROM t1, t2 WHERE t1.b = t2.b) SETTINGS enable_analyzer = 1;
 
 SELECT countIf(explain like '%COMMA%' OR explain like '%CROSS%'), countIf(explain like '%INNER%') FROM (
-    EXPLAIN QUERY TREE SELECT t1.a FROM t1, t2, t3 WHERE t1.a = t2.a AND t1.a = t3.a) SETTINGS allow_experimental_analyzer = 1;
+    EXPLAIN QUERY TREE SELECT t1.a FROM t1, t2, t3 WHERE t1.a = t2.a AND t1.a = t3.a) SETTINGS enable_analyzer = 1;
 
 SELECT countIf(explain like '%COMMA%' OR explain like '%CROSS%'), countIf(explain like '%INNER%') FROM (
-    EXPLAIN QUERY TREE SELECT t1.a FROM t1, t2, t3 WHERE t1.b = t2.b AND t1.b = t3.b) SETTINGS allow_experimental_analyzer = 1;
+    EXPLAIN QUERY TREE SELECT t1.a FROM t1, t2, t3 WHERE t1.b = t2.b AND t1.b = t3.b) SETTINGS enable_analyzer = 1;
 
 SELECT countIf(explain like '%COMMA%' OR explain like '%CROSS%'), countIf(explain like '%INNER%') FROM (
-    EXPLAIN QUERY TREE SELECT t1.a FROM t1, t2, t3, t4 WHERE t1.a = t2.a AND t1.a = t3.a AND t1.a = t4.a) SETTINGS allow_experimental_analyzer = 1;
+    EXPLAIN QUERY TREE SELECT t1.a FROM t1, t2, t3, t4 WHERE t1.a = t2.a AND t1.a = t3.a AND t1.a = t4.a) SETTINGS enable_analyzer = 1;
 
 SELECT countIf(explain like '%COMMA%' OR explain like '%CROSS%'), countIf(explain like '%INNER%') FROM (
-    EXPLAIN QUERY TREE SELECT t1.a FROM t1, t2, t3, t4 WHERE t1.b = t2.b AND t1.b = t3.b AND t1.b = t4.b) SETTINGS allow_experimental_analyzer = 1;
+    EXPLAIN QUERY TREE SELECT t1.a FROM t1, t2, t3, t4 WHERE t1.b = t2.b AND t1.b = t3.b AND t1.b = t4.b) SETTINGS enable_analyzer = 1;
 
 SELECT countIf(explain like '%COMMA%' OR explain like '%CROSS%'), countIf(explain like '%INNER%') FROM (
-    EXPLAIN QUERY TREE SELECT t1.a FROM t1, t2, t3, t4 WHERE t2.a = t1.a AND t2.a = t3.a AND t2.a = t4.a) SETTINGS allow_experimental_analyzer = 1;
+    EXPLAIN QUERY TREE SELECT t1.a FROM t1, t2, t3, t4 WHERE t2.a = t1.a AND t2.a = t3.a AND t2.a = t4.a) SETTINGS enable_analyzer = 1;
 
 SELECT countIf(explain like '%COMMA%' OR explain like '%CROSS%'), countIf(explain like '%INNER%') FROM (
-    EXPLAIN QUERY TREE SELECT t1.a FROM t1, t2, t3, t4 WHERE t3.a = t1.a AND t3.a = t2.a AND t3.a = t4.a) SETTINGS allow_experimental_analyzer = 1;
+    EXPLAIN QUERY TREE SELECT t1.a FROM t1, t2, t3, t4 WHERE t3.a = t1.a AND t3.a = t2.a AND t3.a = t4.a) SETTINGS enable_analyzer = 1;
 
 SELECT countIf(explain like '%COMMA%' OR explain like '%CROSS%'), countIf(explain like '%INNER%') FROM (
-    EXPLAIN QUERY TREE SELECT t1.a FROM t1, t2, t3, t4 WHERE t4.a = t1.a AND t4.a = t2.a AND t4.a = t3.a) SETTINGS allow_experimental_analyzer = 1;
+    EXPLAIN QUERY TREE SELECT t1.a FROM t1, t2, t3, t4 WHERE t4.a = t1.a AND t4.a = t2.a AND t4.a = t3.a) SETTINGS enable_analyzer = 1;
 
 SELECT countIf(explain like '%COMMA%' OR explain like '%CROSS%'), countIf(explain like '%INNER%') FROM (
-    EXPLAIN QUERY TREE SELECT t1.a FROM t1, t2, t3, t4 WHERE t1.a = t2.a AND t2.a = t3.a AND t3.a = t4.a) SETTINGS allow_experimental_analyzer = 1;
+    EXPLAIN QUERY TREE SELECT t1.a FROM t1, t2, t3, t4 WHERE t1.a = t2.a AND t2.a = t3.a AND t3.a = t4.a) SETTINGS enable_analyzer = 1;
 
 SELECT countIf(explain like '%COMMA%' OR explain like '%CROSS%'), countIf(explain like '%INNER%') FROM (
-    EXPLAIN QUERY TREE SELECT t1.a FROM t1, t2, t3, t4) SETTINGS allow_experimental_analyzer = 1;
+    EXPLAIN QUERY TREE SELECT t1.a FROM t1, t2, t3, t4) SETTINGS enable_analyzer = 1;
 
 SELECT countIf(explain like '%COMMA%' OR explain like '%CROSS%'), countIf(explain like '%INNER%') FROM (
-    EXPLAIN QUERY TREE SELECT t1.a FROM t1 CROSS JOIN t2 CROSS JOIN t3 CROSS JOIN t4) SETTINGS allow_experimental_analyzer = 1;
+    EXPLAIN QUERY TREE SELECT t1.a FROM t1 CROSS JOIN t2 CROSS JOIN t3 CROSS JOIN t4) SETTINGS enable_analyzer = 1;
 
 SELECT countIf(explain like '%COMMA%' OR explain like '%CROSS%'), countIf(explain like '%INNER%') FROM (
-    EXPLAIN QUERY TREE SELECT t1.a FROM t1, t2 CROSS JOIN t3) SETTINGS allow_experimental_analyzer = 1;
+    EXPLAIN QUERY TREE SELECT t1.a FROM t1, t2 CROSS JOIN t3) SETTINGS enable_analyzer = 1;
 
 SELECT countIf(explain like '%COMMA%' OR explain like '%CROSS%'), countIf(explain like '%INNER%') FROM (
-    EXPLAIN QUERY TREE SELECT t1.a FROM t1 JOIN t2 USING a CROSS JOIN t3) SETTINGS allow_experimental_analyzer = 1;
+    EXPLAIN QUERY TREE SELECT t1.a FROM t1 JOIN t2 USING a CROSS JOIN t3) SETTINGS enable_analyzer = 1;
 
 SELECT countIf(explain like '%COMMA%' OR explain like '%CROSS%'), countIf(explain like '%INNER%') FROM (
-    EXPLAIN QUERY TREE SELECT t1.a FROM t1 JOIN t2 ON t1.a = t2.a CROSS JOIN t3) SETTINGS allow_experimental_analyzer = 1;
+    EXPLAIN QUERY TREE SELECT t1.a FROM t1 JOIN t2 ON t1.a = t2.a CROSS JOIN t3) SETTINGS enable_analyzer = 1;
 
 INSERT INTO t1 values (1,1), (2,2), (3,3), (4,4);
 INSERT INTO t2 values (1,1), (1, Null);
 INSERT INTO t3 values (1,1), (1, Null);
 INSERT INTO t4 values (1,1), (1, Null);
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 SELECT 'SELECT * FROM t1, t2';
 SELECT * FROM t1, t2
diff --git a/tests/queries/0_stateless/00855_join_with_array_join.sql b/tests/queries/0_stateless/00855_join_with_array_join.sql
index c278ff0738a..2d5bc6cb1f6 100644
--- a/tests/queries/0_stateless/00855_join_with_array_join.sql
+++ b/tests/queries/0_stateless/00855_join_with_array_join.sql
@@ -1,5 +1,5 @@
 SET joined_subquery_requires_alias = 0;
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 SELECT ax, c FROM (SELECT [1,2] ax, 0 c) ARRAY JOIN ax JOIN (SELECT 0 c) USING (c);
 SELECT ax, c FROM (SELECT [3,4] ax, 0 c) JOIN (SELECT 0 c) USING (c) ARRAY JOIN ax;
diff --git a/tests/queries/0_stateless/00858_issue_4756.sql b/tests/queries/0_stateless/00858_issue_4756.sql
index 9eacd5ef364..3f6ab037c02 100644
--- a/tests/queries/0_stateless/00858_issue_4756.sql
+++ b/tests/queries/0_stateless/00858_issue_4756.sql
@@ -1,4 +1,4 @@
-set allow_experimental_analyzer = 1;
+set enable_analyzer = 1;
 set distributed_product_mode = 'local';
 
 drop table if exists shard1;
diff --git a/tests/queries/0_stateless/00897_flatten.sql b/tests/queries/0_stateless/00897_flatten.sql
index 0d67a1708fd..45d1a225a49 100644
--- a/tests/queries/0_stateless/00897_flatten.sql
+++ b/tests/queries/0_stateless/00897_flatten.sql
@@ -2,7 +2,7 @@ SELECT flatten(arrayJoin([[[1, 2, 3], [4, 5]], [[6], [7, 8]]]));
 SELECT arrayFlatten(arrayJoin([[[[]], [[1], [], [2, 3]]], [[[4]]]]));
 SELECT flatten(arrayMap(x -> arrayMap(y -> arrayMap(z -> range(x), range(x)), range(x)), range(number))) FROM numbers(6);
 SELECT flatten(arrayMap(x -> arrayMap(y -> arrayMap(z -> range(z), range(y)), range(x)), range(number))) FROM numbers(6);
-SELECT flatten(arrayMap(x -> arrayMap(x -> arrayMap(x -> range(x), range(x)), range(x)), range(number))) FROM numbers(6) SETTINGS allow_experimental_analyzer=1;
+SELECT flatten(arrayMap(x -> arrayMap(x -> arrayMap(x -> range(x), range(x)), range(x)), range(number))) FROM numbers(6) SETTINGS enable_analyzer=1;
 SELECT arrayFlatten([[[1, 2, 3], [4, 5]], [[6], [7, 8]]]);
 SELECT flatten([[[]]]);
 SELECT arrayFlatten([]);
diff --git a/tests/queries/0_stateless/00940_order_by_read_in_order_query_plan.sql b/tests/queries/0_stateless/00940_order_by_read_in_order_query_plan.sql
index 532539206f7..0421cadf868 100644
--- a/tests/queries/0_stateless/00940_order_by_read_in_order_query_plan.sql
+++ b/tests/queries/0_stateless/00940_order_by_read_in_order_query_plan.sql
@@ -1,4 +1,4 @@
-SET optimize_read_in_order = 1, query_plan_read_in_order = 1, allow_experimental_analyzer = 0;
+SET optimize_read_in_order = 1, query_plan_read_in_order = 1, enable_analyzer = 0;
 
 drop table if exists tab;
 drop table if exists tab2;
diff --git a/tests/queries/0_stateless/01000_subquery_requires_alias.sql b/tests/queries/0_stateless/01000_subquery_requires_alias.sql
index 3cd522a8389..38ba1798dc1 100644
--- a/tests/queries/0_stateless/01000_subquery_requires_alias.sql
+++ b/tests/queries/0_stateless/01000_subquery_requires_alias.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 SET joined_subquery_requires_alias = 1;
 
 SELECT * FROM (SELECT 1 as A, 2 as B) X
diff --git a/tests/queries/0_stateless/01013_totals_without_aggregation.sql b/tests/queries/0_stateless/01013_totals_without_aggregation.sql
index ab656cd92b5..08be45754bb 100644
--- a/tests/queries/0_stateless/01013_totals_without_aggregation.sql
+++ b/tests/queries/0_stateless/01013_totals_without_aggregation.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 SELECT 11 AS n GROUP BY n WITH TOTALS;
 SELECT 12 AS n GROUP BY n WITH ROLLUP;
diff --git a/tests/queries/0_stateless/01018_ambiguous_column.sql b/tests/queries/0_stateless/01018_ambiguous_column.sql
index e9e754ed7a8..b2e0e8fc522 100644
--- a/tests/queries/0_stateless/01018_ambiguous_column.sql
+++ b/tests/queries/0_stateless/01018_ambiguous_column.sql
@@ -1,5 +1,5 @@
 SET output_format_pretty_color=1;
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 select * from system.one cross join system.one;
 select * from system.one cross join system.one r;
diff --git a/tests/queries/0_stateless/01047_window_view_parser_inner_table.sql b/tests/queries/0_stateless/01047_window_view_parser_inner_table.sql
index f17f3ac63b0..de436f1b28e 100644
--- a/tests/queries/0_stateless/01047_window_view_parser_inner_table.sql
+++ b/tests/queries/0_stateless/01047_window_view_parser_inner_table.sql
@@ -1,5 +1,5 @@
 SET send_logs_level = 'fatal';
-SET allow_experimental_analyzer = 0;
+SET enable_analyzer = 0;
 SET allow_experimental_window_view = 1;
 DROP DATABASE IF EXISTS {CLICKHOUSE_DATABASE:Identifier};
 set allow_deprecated_database_ordinary=1;
diff --git a/tests/queries/0_stateless/01048_window_view_parser.sql b/tests/queries/0_stateless/01048_window_view_parser.sql
index adcb4a6364d..bcd80e37c00 100644
--- a/tests/queries/0_stateless/01048_window_view_parser.sql
+++ b/tests/queries/0_stateless/01048_window_view_parser.sql
@@ -1,7 +1,7 @@
 -- Tags: no-parallel
 
 SET send_logs_level = 'fatal';
-SET allow_experimental_analyzer = 0;
+SET enable_analyzer = 0;
 SET allow_experimental_window_view = 1;
 DROP DATABASE IF EXISTS test_01048;
 set allow_deprecated_database_ordinary=1;
diff --git a/tests/queries/0_stateless/01049_join_low_card_bug_long.reference.j2 b/tests/queries/0_stateless/01049_join_low_card_bug_long.reference.j2
index 2ebe5c373b2..872bb448027 100644
--- a/tests/queries/0_stateless/01049_join_low_card_bug_long.reference.j2
+++ b/tests/queries/0_stateless/01049_join_low_card_bug_long.reference.j2
@@ -1,5 +1,5 @@
 -- { echoOn }
-SET allow_experimental_analyzer = 0;
+SET enable_analyzer = 0;
 {% for join_algorithm in ['default', 'partial_merge', 'parallel_hash'] -%}
 SET join_algorithm = '{{ join_algorithm }}';
 SET join_use_nulls = 0;
@@ -19,17 +19,17 @@ str_r	LowCardinality(String)
 str_l	LowCardinality(String)
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l RIGHT JOIN r_lc AS r USING (x) ORDER BY x, r.lc, l.lc;
 LowCardinality(String)	LowCardinality(String)	str	str	LowCardinality(String)	LowCardinality(String)	str	str
-LowCardinality(String)	LowCardinality(String)	str_r	str_r	LowCardinality(String)	LowCardinality(String)		
+LowCardinality(String)	LowCardinality(String)	str_r	str_r	LowCardinality(String)	LowCardinality(String)
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l RIGHT JOIN r_lc AS r USING (lc) ORDER BY x, r.lc, l.lc;
 LowCardinality(String)	LowCardinality(String)	str	str	LowCardinality(String)	LowCardinality(String)	str	str
-LowCardinality(String)	LowCardinality(String)	str_r	str_r	LowCardinality(String)	LowCardinality(String)		
+LowCardinality(String)	LowCardinality(String)	str_r	str_r	LowCardinality(String)	LowCardinality(String)
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l FULL JOIN r_lc AS r USING (x) ORDER BY x, r.lc, l.lc;
 LowCardinality(String)	LowCardinality(String)	str	str	LowCardinality(String)	LowCardinality(String)	str	str
-LowCardinality(String)	LowCardinality(String)	str_r	str_r	LowCardinality(String)	LowCardinality(String)		
+LowCardinality(String)	LowCardinality(String)	str_r	str_r	LowCardinality(String)	LowCardinality(String)
 LowCardinality(String)	LowCardinality(String)			LowCardinality(String)	LowCardinality(String)	str_l	str_l
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l FULL JOIN r_lc AS r USING (lc) ORDER BY x, r.lc, l.lc;
 LowCardinality(String)	LowCardinality(String)	str	str	LowCardinality(String)	LowCardinality(String)	str	str
-LowCardinality(String)	LowCardinality(String)	str_r	str_r	LowCardinality(String)	LowCardinality(String)		
+LowCardinality(String)	LowCardinality(String)	str_r	str_r	LowCardinality(String)	LowCardinality(String)
 LowCardinality(String)	LowCardinality(String)			LowCardinality(String)	LowCardinality(String)	str_l	str_l
 --
 
@@ -49,17 +49,17 @@ str_r	LowCardinality(String)
 str_l	LowCardinality(String)
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l RIGHT JOIN r USING (x) ORDER BY x, r.lc, l.lc;
 String	String	str	str	LowCardinality(String)	LowCardinality(String)	str	str
-String	String	str_r	str_r	LowCardinality(String)	LowCardinality(String)		
+String	String	str_r	str_r	LowCardinality(String)	LowCardinality(String)
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l RIGHT JOIN r USING (lc) ORDER BY x, r.lc, l.lc;
 String	String	str	str	LowCardinality(String)	LowCardinality(String)	str	str
-String	String	str_r	str_r	LowCardinality(String)	LowCardinality(String)		
+String	String	str_r	str_r	LowCardinality(String)	LowCardinality(String)
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l FULL JOIN r USING (x) ORDER BY x, r.lc, l.lc;
 String	String	str	str	LowCardinality(String)	LowCardinality(String)	str	str
-String	String	str_r	str_r	LowCardinality(String)	LowCardinality(String)		
+String	String	str_r	str_r	LowCardinality(String)	LowCardinality(String)
 String	String			LowCardinality(String)	LowCardinality(String)	str_l	str_l
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l FULL JOIN r USING (lc) ORDER BY x, r.lc, l.lc;
 String	String	str	str	LowCardinality(String)	LowCardinality(String)	str	str
-String	String	str_r	str_r	LowCardinality(String)	LowCardinality(String)		
+String	String	str_r	str_r	LowCardinality(String)	LowCardinality(String)
 String	String			LowCardinality(String)	LowCardinality(String)	str_l	str_l
 --
 
@@ -79,17 +79,17 @@ str_r	String
 str_l	String
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l RIGHT JOIN r_lc AS r USING (x) ORDER BY x, r.lc, l.lc;
 LowCardinality(String)	LowCardinality(String)	str	str	String	String	str	str
-LowCardinality(String)	LowCardinality(String)	str_r	str_r	String	String		
+LowCardinality(String)	LowCardinality(String)	str_r	str_r	String	String
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l RIGHT JOIN r_lc AS r USING (lc) ORDER BY x, r.lc, l.lc;
 LowCardinality(String)	LowCardinality(String)	str	str	String	String	str	str
-LowCardinality(String)	LowCardinality(String)	str_r	str_r	String	String		
+LowCardinality(String)	LowCardinality(String)	str_r	str_r	String	String
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l FULL JOIN r_lc AS r USING (x) ORDER BY x, r.lc, l.lc;
 LowCardinality(String)	LowCardinality(String)	str	str	String	String	str	str
-LowCardinality(String)	LowCardinality(String)	str_r	str_r	String	String		
+LowCardinality(String)	LowCardinality(String)	str_r	str_r	String	String
 LowCardinality(String)	LowCardinality(String)			String	String	str_l	str_l
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l FULL JOIN r_lc AS r USING (lc) ORDER BY x, r.lc, l.lc;
 LowCardinality(String)	LowCardinality(String)	str	str	String	String	str	str
-LowCardinality(String)	LowCardinality(String)	str_r	str_r	String	String		
+LowCardinality(String)	LowCardinality(String)	str_r	str_r	String	String
 LowCardinality(String)	LowCardinality(String)			String	String	str_l	str_l
 --
 
@@ -109,17 +109,17 @@ str_r	LowCardinality(String)
 str_l	LowCardinality(String)
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l RIGHT JOIN nr AS r USING (x) ORDER BY x, r.lc, l.lc;
 Nullable(String)	Nullable(String)	str	str	LowCardinality(String)	LowCardinality(String)	str	str
-Nullable(String)	Nullable(String)	str_r	str_r	LowCardinality(String)	LowCardinality(String)		
+Nullable(String)	Nullable(String)	str_r	str_r	LowCardinality(String)	LowCardinality(String)
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l RIGHT JOIN nr AS r USING (lc) ORDER BY x, r.lc, l.lc;
 Nullable(String)	Nullable(String)	str	str	LowCardinality(String)	LowCardinality(String)	str	str
-Nullable(String)	Nullable(String)	str_r	str_r	LowCardinality(String)	LowCardinality(String)		
+Nullable(String)	Nullable(String)	str_r	str_r	LowCardinality(String)	LowCardinality(String)
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l FULL JOIN nr AS r USING (x) ORDER BY x, r.lc, l.lc;
 Nullable(String)	Nullable(String)	str	str	LowCardinality(String)	LowCardinality(String)	str	str
-Nullable(String)	Nullable(String)	str_r	str_r	LowCardinality(String)	LowCardinality(String)		
+Nullable(String)	Nullable(String)	str_r	str_r	LowCardinality(String)	LowCardinality(String)
 Nullable(String)	Nullable(String)	\N	\N	LowCardinality(String)	LowCardinality(String)	str_l	str_l
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l FULL JOIN nr AS r USING (lc) ORDER BY x, r.lc, l.lc;
 Nullable(String)	Nullable(String)	str	str	LowCardinality(String)	LowCardinality(String)	str	str
-Nullable(String)	Nullable(String)	str_r	str_r	LowCardinality(String)	LowCardinality(String)		
+Nullable(String)	Nullable(String)	str_r	str_r	LowCardinality(String)	LowCardinality(String)
 Nullable(String)	Nullable(String)	\N	\N	LowCardinality(String)	LowCardinality(String)	str_l	str_l
 --
 
@@ -313,7 +313,7 @@ SELECT r.lc, materialize(r.lc), toTypeName(r.lc), toTypeName(materialize(r.lc))
 str	str	LowCardinality(Nullable(String))	LowCardinality(Nullable(String))
 str_r	str_r	LowCardinality(Nullable(String))	LowCardinality(Nullable(String))
 \N	\N	LowCardinality(Nullable(String))	LowCardinality(Nullable(String))
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 {% for join_algorithm in ['default', 'partial_merge', 'parallel_hash'] -%}
 SET join_algorithm = '{{ join_algorithm }}';
 SET join_use_nulls = 0;
@@ -333,17 +333,17 @@ str_r	LowCardinality(String)
 str_l	LowCardinality(String)
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l RIGHT JOIN r_lc AS r USING (x) ORDER BY x, r.lc, l.lc;
 LowCardinality(String)	LowCardinality(String)	str	str	LowCardinality(String)	LowCardinality(String)	str	str
-LowCardinality(String)	LowCardinality(String)	str_r	str_r	LowCardinality(String)	LowCardinality(String)		
+LowCardinality(String)	LowCardinality(String)	str_r	str_r	LowCardinality(String)	LowCardinality(String)
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l RIGHT JOIN r_lc AS r USING (lc) ORDER BY x, r.lc, l.lc;
 LowCardinality(String)	LowCardinality(String)	str	str	LowCardinality(String)	LowCardinality(String)	str	str
-LowCardinality(String)	LowCardinality(String)	str_r	str_r	LowCardinality(String)	LowCardinality(String)		
+LowCardinality(String)	LowCardinality(String)	str_r	str_r	LowCardinality(String)	LowCardinality(String)
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l FULL JOIN r_lc AS r USING (x) ORDER BY x, r.lc, l.lc;
 LowCardinality(String)	LowCardinality(String)	str	str	LowCardinality(String)	LowCardinality(String)	str	str
-LowCardinality(String)	LowCardinality(String)	str_r	str_r	LowCardinality(String)	LowCardinality(String)		
+LowCardinality(String)	LowCardinality(String)	str_r	str_r	LowCardinality(String)	LowCardinality(String)
 LowCardinality(String)	LowCardinality(String)			LowCardinality(String)	LowCardinality(String)	str_l	str_l
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l FULL JOIN r_lc AS r USING (lc) ORDER BY x, r.lc, l.lc;
 LowCardinality(String)	LowCardinality(String)	str	str	LowCardinality(String)	LowCardinality(String)	str	str
-LowCardinality(String)	LowCardinality(String)	str_r	str_r	LowCardinality(String)	LowCardinality(String)		
+LowCardinality(String)	LowCardinality(String)	str_r	str_r	LowCardinality(String)	LowCardinality(String)
 LowCardinality(String)	LowCardinality(String)			LowCardinality(String)	LowCardinality(String)	str_l	str_l
 --
 
@@ -363,17 +363,17 @@ str_r	String
 str_l	String
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l RIGHT JOIN r USING (x) ORDER BY x, r.lc, l.lc;
 String	String	str	str	LowCardinality(String)	LowCardinality(String)	str	str
-String	String	str_r	str_r	LowCardinality(String)	LowCardinality(String)		
+String	String	str_r	str_r	LowCardinality(String)	LowCardinality(String)
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l RIGHT JOIN r USING (lc) ORDER BY x, r.lc, l.lc;
 String	String	str	str	String	String	str	str
-String	String	str_r	str_r	String	String		
+String	String	str_r	str_r	String	String
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l FULL JOIN r USING (x) ORDER BY x, r.lc, l.lc;
 String	String	str	str	LowCardinality(String)	LowCardinality(String)	str	str
-String	String	str_r	str_r	LowCardinality(String)	LowCardinality(String)		
+String	String	str_r	str_r	LowCardinality(String)	LowCardinality(String)
 String	String			LowCardinality(String)	LowCardinality(String)	str_l	str_l
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l FULL JOIN r USING (lc) ORDER BY x, r.lc, l.lc;
 String	String	str	str	String	String	str	str
-String	String	str_r	str_r	String	String		
+String	String	str_r	str_r	String	String
 String	String			String	String	str_l	str_l
 --
 
@@ -393,17 +393,17 @@ str_r	String
 str_l	String
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l RIGHT JOIN r_lc AS r USING (x) ORDER BY x, r.lc, l.lc;
 LowCardinality(String)	LowCardinality(String)	str	str	String	String	str	str
-LowCardinality(String)	LowCardinality(String)	str_r	str_r	String	String		
+LowCardinality(String)	LowCardinality(String)	str_r	str_r	String	String
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l RIGHT JOIN r_lc AS r USING (lc) ORDER BY x, r.lc, l.lc;
 String	String	str	str	String	String	str	str
-String	String	str_r	str_r	String	String		
+String	String	str_r	str_r	String	String
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l FULL JOIN r_lc AS r USING (x) ORDER BY x, r.lc, l.lc;
 LowCardinality(String)	LowCardinality(String)	str	str	String	String	str	str
-LowCardinality(String)	LowCardinality(String)	str_r	str_r	String	String		
+LowCardinality(String)	LowCardinality(String)	str_r	str_r	String	String
 LowCardinality(String)	LowCardinality(String)			String	String	str_l	str_l
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l FULL JOIN r_lc AS r USING (lc) ORDER BY x, r.lc, l.lc;
 String	String	str	str	String	String	str	str
-String	String	str_r	str_r	String	String		
+String	String	str_r	str_r	String	String
 String	String			String	String	str_l	str_l
 --
 
@@ -423,13 +423,13 @@ str_r	Nullable(String)
 str_l	Nullable(String)
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l RIGHT JOIN nr AS r USING (x) ORDER BY x, r.lc, l.lc;
 Nullable(String)	Nullable(String)	str	str	LowCardinality(String)	LowCardinality(String)	str	str
-Nullable(String)	Nullable(String)	str_r	str_r	LowCardinality(String)	LowCardinality(String)		
+Nullable(String)	Nullable(String)	str_r	str_r	LowCardinality(String)	LowCardinality(String)
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l RIGHT JOIN nr AS r USING (lc) ORDER BY x, r.lc, l.lc;
 Nullable(String)	Nullable(String)	str	str	Nullable(String)	Nullable(String)	str	str
 Nullable(String)	Nullable(String)	str_r	str_r	Nullable(String)	Nullable(String)	\N	\N
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l FULL JOIN nr AS r USING (x) ORDER BY x, r.lc, l.lc;
 Nullable(String)	Nullable(String)	str	str	LowCardinality(String)	LowCardinality(String)	str	str
-Nullable(String)	Nullable(String)	str_r	str_r	LowCardinality(String)	LowCardinality(String)		
+Nullable(String)	Nullable(String)	str_r	str_r	LowCardinality(String)	LowCardinality(String)
 Nullable(String)	Nullable(String)	\N	\N	LowCardinality(String)	LowCardinality(String)	str_l	str_l
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l FULL JOIN nr AS r USING (lc) ORDER BY x, r.lc, l.lc;
 Nullable(String)	Nullable(String)	str	str	Nullable(String)	Nullable(String)	str	str
diff --git a/tests/queries/0_stateless/01049_join_low_card_bug_long.sql.j2 b/tests/queries/0_stateless/01049_join_low_card_bug_long.sql.j2
index 64ec34ef1bf..7e7b5cb1fed 100644
--- a/tests/queries/0_stateless/01049_join_low_card_bug_long.sql.j2
+++ b/tests/queries/0_stateless/01049_join_low_card_bug_long.sql.j2
@@ -23,9 +23,9 @@ INSERT INTO nl VALUES (0, 'str'), (2, 'str_l');
 INSERT INTO l_lc VALUES (0, 'str'), (2, 'str_l');
 
 -- { echoOn }
-{% for allow_experimental_analyzer in [0, 1] -%}
+{% for enable_analyzer in [0, 1] -%}
 
-SET allow_experimental_analyzer = {{ allow_experimental_analyzer }};
+SET enable_analyzer = {{ enable_analyzer }};
 
 {% for join_algorithm in ['default', 'partial_merge', 'parallel_hash'] -%}
 SET join_algorithm = '{{ join_algorithm }}';
diff --git a/tests/queries/0_stateless/01050_window_view_parser_tumble.sql b/tests/queries/0_stateless/01050_window_view_parser_tumble.sql
index c52a6fefacb..9c4d312bab6 100644
--- a/tests/queries/0_stateless/01050_window_view_parser_tumble.sql
+++ b/tests/queries/0_stateless/01050_window_view_parser_tumble.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 0;
+SET enable_analyzer = 0;
 SET allow_experimental_window_view = 1;
 
 DROP TABLE IF EXISTS mt;
diff --git a/tests/queries/0_stateless/01051_window_view_parser_hop.sql b/tests/queries/0_stateless/01051_window_view_parser_hop.sql
index b37e4ed3095..569e3b2a6bf 100644
--- a/tests/queries/0_stateless/01051_window_view_parser_hop.sql
+++ b/tests/queries/0_stateless/01051_window_view_parser_hop.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 0;
+SET enable_analyzer = 0;
 SET allow_experimental_window_view = 1;
 
 DROP TABLE IF EXISTS mt;
diff --git a/tests/queries/0_stateless/01052_window_view_proc_tumble_to_now.sh b/tests/queries/0_stateless/01052_window_view_proc_tumble_to_now.sh
index 4f8482558c8..c473bf766b0 100755
--- a/tests/queries/0_stateless/01052_window_view_proc_tumble_to_now.sh
+++ b/tests/queries/0_stateless/01052_window_view_proc_tumble_to_now.sh
@@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CURDIR"/../shell_config.sh
 
 opts=(
-    "--allow_experimental_analyzer=0"
+    "--enable_analyzer=0"
 )
 
 $CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
diff --git a/tests/queries/0_stateless/01053_window_view_proc_hop_to_now.sh b/tests/queries/0_stateless/01053_window_view_proc_hop_to_now.sh
index efd1721e594..ca89dd9daf1 100755
--- a/tests/queries/0_stateless/01053_window_view_proc_hop_to_now.sh
+++ b/tests/queries/0_stateless/01053_window_view_proc_hop_to_now.sh
@@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CURDIR"/../shell_config.sh
 
 opts=(
-    "--allow_experimental_analyzer=0"
+    "--enable_analyzer=0"
 )
 
 $CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
diff --git a/tests/queries/0_stateless/01054_window_view_proc_tumble_to.sh b/tests/queries/0_stateless/01054_window_view_proc_tumble_to.sh
index 5e603382e5b..67cac226de6 100755
--- a/tests/queries/0_stateless/01054_window_view_proc_tumble_to.sh
+++ b/tests/queries/0_stateless/01054_window_view_proc_tumble_to.sh
@@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CURDIR"/../shell_config.sh
 
 opts=(
-    "--allow_experimental_analyzer=0"
+    "--enable_analyzer=0"
 )
 
 $CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
diff --git a/tests/queries/0_stateless/01055_window_view_proc_hop_to.sh b/tests/queries/0_stateless/01055_window_view_proc_hop_to.sh
index 00f8f239336..e44fc2e363c 100755
--- a/tests/queries/0_stateless/01055_window_view_proc_hop_to.sh
+++ b/tests/queries/0_stateless/01055_window_view_proc_hop_to.sh
@@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CURDIR"/../shell_config.sh
 
 opts=(
-    "--allow_experimental_analyzer=0"
+    "--enable_analyzer=0"
 )
 
 $CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
diff --git a/tests/queries/0_stateless/01056_window_view_proc_hop_watch.py b/tests/queries/0_stateless/01056_window_view_proc_hop_watch.py
index e65650816ab..6bb8db2c38e 100755
--- a/tests/queries/0_stateless/01056_window_view_proc_hop_watch.py
+++ b/tests/queries/0_stateless/01056_window_view_proc_hop_watch.py
@@ -20,7 +20,7 @@ with client(name="client1>", log=log) as client1, client(
     client1.expect(prompt)
     client2.expect(prompt)
 
-    client1.send("SET allow_experimental_analyzer = 0")
+    client1.send("SET enable_analyzer = 0")
     client1.expect(prompt)
     client1.send("SET allow_experimental_window_view = 1")
     client1.expect(prompt)
@@ -28,7 +28,7 @@ with client(name="client1>", log=log) as client1, client(
     client1.expect(prompt)
     client2.send("SET allow_experimental_window_view = 1")
     client2.expect(prompt)
-    client2.send("SET allow_experimental_analyzer = 0")
+    client2.send("SET enable_analyzer = 0")
     client2.expect(prompt)
 
     client1.send("CREATE DATABASE IF NOT EXISTS 01056_window_view_proc_hop_watch")
diff --git a/tests/queries/0_stateless/01057_window_view_event_tumble_to_strict_asc.sh b/tests/queries/0_stateless/01057_window_view_event_tumble_to_strict_asc.sh
index 841d5e701a4..ab0a6c147a9 100755
--- a/tests/queries/0_stateless/01057_window_view_event_tumble_to_strict_asc.sh
+++ b/tests/queries/0_stateless/01057_window_view_event_tumble_to_strict_asc.sh
@@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CURDIR"/../shell_config.sh
 
 opts=(
-    "--allow_experimental_analyzer=0"
+    "--enable_analyzer=0"
 )
 
 $CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
diff --git a/tests/queries/0_stateless/01058_window_view_event_hop_to_strict_asc.sh b/tests/queries/0_stateless/01058_window_view_event_hop_to_strict_asc.sh
index 315959bca9f..529d7ba195f 100755
--- a/tests/queries/0_stateless/01058_window_view_event_hop_to_strict_asc.sh
+++ b/tests/queries/0_stateless/01058_window_view_event_hop_to_strict_asc.sh
@@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CURDIR"/../shell_config.sh
 
 opts=(
-    "--allow_experimental_analyzer=0"
+    "--enable_analyzer=0"
 )
 
 $CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
diff --git a/tests/queries/0_stateless/01059_window_view_event_hop_watch_strict_asc.py b/tests/queries/0_stateless/01059_window_view_event_hop_watch_strict_asc.py
index 3dbb176b0dc..33e63502ef2 100755
--- a/tests/queries/0_stateless/01059_window_view_event_hop_watch_strict_asc.py
+++ b/tests/queries/0_stateless/01059_window_view_event_hop_watch_strict_asc.py
@@ -20,13 +20,13 @@ with client(name="client1>", log=log) as client1, client(
     client1.expect(prompt)
     client2.expect(prompt)
 
-    client1.send("SET allow_experimental_analyzer = 0")
+    client1.send("SET enable_analyzer = 0")
     client1.expect(prompt)
     client1.send("SET allow_experimental_window_view = 1")
     client1.expect(prompt)
     client1.send("SET window_view_heartbeat_interval = 1")
     client1.expect(prompt)
-    client2.send("SET allow_experimental_analyzer = 0")
+    client2.send("SET enable_analyzer = 0")
     client2.expect(prompt)
 
     client1.send("CREATE DATABASE IF NOT EXISTS db_01059_event_hop_watch_strict_asc")
diff --git a/tests/queries/0_stateless/01060_window_view_event_tumble_to_asc.sh b/tests/queries/0_stateless/01060_window_view_event_tumble_to_asc.sh
index 592ee93a3af..1da599f4d49 100755
--- a/tests/queries/0_stateless/01060_window_view_event_tumble_to_asc.sh
+++ b/tests/queries/0_stateless/01060_window_view_event_tumble_to_asc.sh
@@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CURDIR"/../shell_config.sh
 
 opts=(
-    "--allow_experimental_analyzer=0"
+    "--enable_analyzer=0"
 )
 
 $CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
diff --git a/tests/queries/0_stateless/01061_window_view_event_hop_to_asc.sh b/tests/queries/0_stateless/01061_window_view_event_hop_to_asc.sh
index 6089a56ecc7..52a187f256e 100755
--- a/tests/queries/0_stateless/01061_window_view_event_hop_to_asc.sh
+++ b/tests/queries/0_stateless/01061_window_view_event_hop_to_asc.sh
@@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CURDIR"/../shell_config.sh
 
 opts=(
-    "--allow_experimental_analyzer=0"
+    "--enable_analyzer=0"
 )
 
 $CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
diff --git a/tests/queries/0_stateless/01062_window_view_event_hop_watch_asc.py b/tests/queries/0_stateless/01062_window_view_event_hop_watch_asc.py
index 3c85ff30ba8..22a171781d9 100755
--- a/tests/queries/0_stateless/01062_window_view_event_hop_watch_asc.py
+++ b/tests/queries/0_stateless/01062_window_view_event_hop_watch_asc.py
@@ -20,7 +20,7 @@ with client(name="client1>", log=log) as client1, client(
     client1.expect(prompt)
     client2.expect(prompt)
 
-    client1.send("SET allow_experimental_analyzer = 0")
+    client1.send("SET enable_analyzer = 0")
     client1.expect(prompt)
     client1.send("SET allow_experimental_window_view = 1")
     client1.expect(prompt)
@@ -28,7 +28,7 @@ with client(name="client1>", log=log) as client1, client(
     client1.expect(prompt)
     client2.send("SET allow_experimental_window_view = 1")
     client2.expect(prompt)
-    client2.send("SET allow_experimental_analyzer = 0")
+    client2.send("SET enable_analyzer = 0")
     client2.expect(prompt)
 
     client1.send(f"DROP TABLE IF EXISTS {database_name}.mt")
diff --git a/tests/queries/0_stateless/01063_window_view_event_tumble_to_bounded.sh b/tests/queries/0_stateless/01063_window_view_event_tumble_to_bounded.sh
index 349f73ec637..1b08841bcf6 100755
--- a/tests/queries/0_stateless/01063_window_view_event_tumble_to_bounded.sh
+++ b/tests/queries/0_stateless/01063_window_view_event_tumble_to_bounded.sh
@@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CURDIR"/../shell_config.sh
 
 opts=(
-    "--allow_experimental_analyzer=0"
+    "--enable_analyzer=0"
 )
 
 $CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
diff --git a/tests/queries/0_stateless/01064_window_view_event_hop_to_bounded.sh b/tests/queries/0_stateless/01064_window_view_event_hop_to_bounded.sh
index 8db211edc5d..48a01592121 100755
--- a/tests/queries/0_stateless/01064_window_view_event_hop_to_bounded.sh
+++ b/tests/queries/0_stateless/01064_window_view_event_hop_to_bounded.sh
@@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CURDIR"/../shell_config.sh
 
 opts=(
-    "--allow_experimental_analyzer=0"
+    "--enable_analyzer=0"
 )
 
 $CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
diff --git a/tests/queries/0_stateless/01065_window_view_event_hop_watch_bounded.py b/tests/queries/0_stateless/01065_window_view_event_hop_watch_bounded.py
index e5f9ab59f60..b9dacedd090 100755
--- a/tests/queries/0_stateless/01065_window_view_event_hop_watch_bounded.py
+++ b/tests/queries/0_stateless/01065_window_view_event_hop_watch_bounded.py
@@ -19,7 +19,7 @@ with client(name="client1>", log=log) as client1, client(
     client1.expect(prompt)
     client2.expect(prompt)
 
-    client1.send("SET allow_experimental_analyzer = 0")
+    client1.send("SET enable_analyzer = 0")
     client1.expect(prompt)
     client1.send("SET allow_experimental_window_view = 1")
     client1.expect(prompt)
@@ -27,7 +27,7 @@ with client(name="client1>", log=log) as client1, client(
     client1.expect(prompt)
     client2.send("SET allow_experimental_window_view = 1")
     client2.expect(prompt)
-    client2.send("SET allow_experimental_analyzer = 0")
+    client2.send("SET enable_analyzer = 0")
     client2.expect(prompt)
 
     client1.send(
diff --git a/tests/queries/0_stateless/01066_window_view_event_tumble_to_strict_asc_lateness.sh b/tests/queries/0_stateless/01066_window_view_event_tumble_to_strict_asc_lateness.sh
index 72988bb5496..d61502331bc 100755
--- a/tests/queries/0_stateless/01066_window_view_event_tumble_to_strict_asc_lateness.sh
+++ b/tests/queries/0_stateless/01066_window_view_event_tumble_to_strict_asc_lateness.sh
@@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CURDIR"/../shell_config.sh
 
 opts=(
-    "--allow_experimental_analyzer=0"
+    "--enable_analyzer=0"
 )
 
 $CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
diff --git a/tests/queries/0_stateless/01067_window_view_event_tumble_to_asc_lateness.sh b/tests/queries/0_stateless/01067_window_view_event_tumble_to_asc_lateness.sh
index 1c0ef8183ac..b272a7c5762 100755
--- a/tests/queries/0_stateless/01067_window_view_event_tumble_to_asc_lateness.sh
+++ b/tests/queries/0_stateless/01067_window_view_event_tumble_to_asc_lateness.sh
@@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CURDIR"/../shell_config.sh
 
 opts=(
-    "--allow_experimental_analyzer=0"
+    "--enable_analyzer=0"
 )
 
 $CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
diff --git a/tests/queries/0_stateless/01068_window_view_event_tumble_to_bounded_lateness.sh b/tests/queries/0_stateless/01068_window_view_event_tumble_to_bounded_lateness.sh
index af8f70e7ab8..9e184e5ba14 100755
--- a/tests/queries/0_stateless/01068_window_view_event_tumble_to_bounded_lateness.sh
+++ b/tests/queries/0_stateless/01068_window_view_event_tumble_to_bounded_lateness.sh
@@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CURDIR"/../shell_config.sh
 
 opts=(
-    "--allow_experimental_analyzer=0"
+    "--enable_analyzer=0"
 )
 
 $CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
diff --git a/tests/queries/0_stateless/01069_window_view_proc_tumble_watch.py b/tests/queries/0_stateless/01069_window_view_proc_tumble_watch.py
index 8c3a46992dc..904723f650f 100755
--- a/tests/queries/0_stateless/01069_window_view_proc_tumble_watch.py
+++ b/tests/queries/0_stateless/01069_window_view_proc_tumble_watch.py
@@ -20,7 +20,7 @@ with client(name="client1>", log=log) as client1, client(
     client1.expect(prompt)
     client2.expect(prompt)
 
-    client1.send("SET allow_experimental_analyzer = 0")
+    client1.send("SET enable_analyzer = 0")
     client1.expect(prompt)
     client1.send("SET allow_experimental_window_view = 1")
     client1.expect(prompt)
@@ -28,7 +28,7 @@ with client(name="client1>", log=log) as client1, client(
     client1.expect(prompt)
     client2.send("SET allow_experimental_window_view = 1")
     client2.expect(prompt)
-    client2.send("SET allow_experimental_analyzer = 0")
+    client2.send("SET enable_analyzer = 0")
     client2.expect(prompt)
 
     client1.send("CREATE DATABASE 01069_window_view_proc_tumble_watch")
diff --git a/tests/queries/0_stateless/01070_window_view_watch_events.py b/tests/queries/0_stateless/01070_window_view_watch_events.py
index 172a82a29da..35f0155c624 100755
--- a/tests/queries/0_stateless/01070_window_view_watch_events.py
+++ b/tests/queries/0_stateless/01070_window_view_watch_events.py
@@ -20,7 +20,7 @@ with client(name="client1>", log=log) as client1, client(
     client1.expect(prompt)
     client2.expect(prompt)
 
-    client1.send("SET allow_experimental_analyzer = 0")
+    client1.send("SET enable_analyzer = 0")
     client1.expect(prompt)
     client1.send("SET allow_experimental_window_view = 1")
     client1.expect(prompt)
@@ -28,7 +28,7 @@ with client(name="client1>", log=log) as client1, client(
     client1.expect(prompt)
     client2.send("SET allow_experimental_window_view = 1")
     client2.expect(prompt)
-    client2.send("SET allow_experimental_analyzer = 0")
+    client2.send("SET enable_analyzer = 0")
     client2.expect(prompt)
 
     client1.send("CREATE DATABASE IF NOT EXISTS 01070_window_view_watch_events")
diff --git a/tests/queries/0_stateless/01071_window_view_event_tumble_asc_join.sh b/tests/queries/0_stateless/01071_window_view_event_tumble_asc_join.sh
index b71ef68a9c8..6614c8a578a 100755
--- a/tests/queries/0_stateless/01071_window_view_event_tumble_asc_join.sh
+++ b/tests/queries/0_stateless/01071_window_view_event_tumble_asc_join.sh
@@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CURDIR"/../shell_config.sh
 
 opts=(
-    "--allow_experimental_analyzer=0"
+    "--enable_analyzer=0"
 )
 
 $CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
diff --git a/tests/queries/0_stateless/01072_window_view_multiple_columns_groupby.sh b/tests/queries/0_stateless/01072_window_view_multiple_columns_groupby.sh
index 7117e3207fe..998ee9855ae 100755
--- a/tests/queries/0_stateless/01072_window_view_multiple_columns_groupby.sh
+++ b/tests/queries/0_stateless/01072_window_view_multiple_columns_groupby.sh
@@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CURDIR"/../shell_config.sh
 
 opts=(
-    "--allow_experimental_analyzer=0"
+    "--enable_analyzer=0"
 )
 
 $CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
diff --git a/tests/queries/0_stateless/01073_window_view_event_tumble_to_asc_populate.sh b/tests/queries/0_stateless/01073_window_view_event_tumble_to_asc_populate.sh
index 27527df05f8..a26221d0328 100755
--- a/tests/queries/0_stateless/01073_window_view_event_tumble_to_asc_populate.sh
+++ b/tests/queries/0_stateless/01073_window_view_event_tumble_to_asc_populate.sh
@@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CURDIR"/../shell_config.sh
 
 opts=(
-    "--allow_experimental_analyzer=0"
+    "--enable_analyzer=0"
 )
 
 $CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
diff --git a/tests/queries/0_stateless/01074_window_view_event_tumble_asc_join_populate.sh b/tests/queries/0_stateless/01074_window_view_event_tumble_asc_join_populate.sh
index 3cd9f750f7c..97dfef61b1a 100755
--- a/tests/queries/0_stateless/01074_window_view_event_tumble_asc_join_populate.sh
+++ b/tests/queries/0_stateless/01074_window_view_event_tumble_asc_join_populate.sh
@@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CURDIR"/../shell_config.sh
 
 opts=(
-    "--allow_experimental_analyzer=0"
+    "--enable_analyzer=0"
 )
 
 $CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
diff --git a/tests/queries/0_stateless/01075_window_view_proc_tumble_to_now_populate.sh b/tests/queries/0_stateless/01075_window_view_proc_tumble_to_now_populate.sh
index 8bb604997af..859b9a86a2a 100755
--- a/tests/queries/0_stateless/01075_window_view_proc_tumble_to_now_populate.sh
+++ b/tests/queries/0_stateless/01075_window_view_proc_tumble_to_now_populate.sh
@@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CURDIR"/../shell_config.sh
 
 opts=(
-    "--allow_experimental_analyzer=0"
+    "--enable_analyzer=0"
 )
 
 $CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
diff --git a/tests/queries/0_stateless/01076_window_view_alter_query_to.sh b/tests/queries/0_stateless/01076_window_view_alter_query_to.sh
index 043a2108f0c..de782261a59 100755
--- a/tests/queries/0_stateless/01076_window_view_alter_query_to.sh
+++ b/tests/queries/0_stateless/01076_window_view_alter_query_to.sh
@@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CURDIR"/../shell_config.sh
 
 opts=(
-    "--allow_experimental_analyzer=0"
+    "--enable_analyzer=0"
 )
 
 $CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
diff --git a/tests/queries/0_stateless/01077_window_view_alter_query_to_modify_source.sh b/tests/queries/0_stateless/01077_window_view_alter_query_to_modify_source.sh
index 4d10d7fd2f3..f3c03b06b86 100755
--- a/tests/queries/0_stateless/01077_window_view_alter_query_to_modify_source.sh
+++ b/tests/queries/0_stateless/01077_window_view_alter_query_to_modify_source.sh
@@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CURDIR"/../shell_config.sh
 
 opts=(
-    "--allow_experimental_analyzer=0"
+    "--enable_analyzer=0"
 )
 
 $CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
diff --git a/tests/queries/0_stateless/01078_window_view_alter_query_watch.py b/tests/queries/0_stateless/01078_window_view_alter_query_watch.py
index 05aeb1b4ccb..275edb094c7 100755
--- a/tests/queries/0_stateless/01078_window_view_alter_query_watch.py
+++ b/tests/queries/0_stateless/01078_window_view_alter_query_watch.py
@@ -20,7 +20,7 @@ with client(name="client1>", log=log) as client1, client(
     client2.expect(prompt)
     client3.expect(prompt)
 
-    client1.send("SET allow_experimental_analyzer = 0")
+    client1.send("SET enable_analyzer = 0")
     client1.expect(prompt)
     client1.send("SET allow_experimental_window_view = 1")
     client1.expect(prompt)
@@ -28,13 +28,13 @@ with client(name="client1>", log=log) as client1, client(
     client1.expect(prompt)
     client2.send("SET allow_experimental_window_view = 1")
     client2.expect(prompt)
-    client2.send("SET allow_experimental_analyzer = 0")
+    client2.send("SET enable_analyzer = 0")
     client2.expect(prompt)
     client3.send("SET allow_experimental_window_view = 1")
     client3.expect(prompt)
     client3.send("SET window_view_heartbeat_interval = 1")
     client3.expect(prompt)
-    client3.send("SET allow_experimental_analyzer = 0")
+    client3.send("SET enable_analyzer = 0")
     client3.expect(prompt)
 
     client1.send("CREATE DATABASE IF NOT EXISTS 01078_window_view_alter_query_watch")
diff --git a/tests/queries/0_stateless/01079_window_view_inner_table_memory_tumble.sh b/tests/queries/0_stateless/01079_window_view_inner_table_memory_tumble.sh
index 299c0086874..476688cee30 100755
--- a/tests/queries/0_stateless/01079_window_view_inner_table_memory_tumble.sh
+++ b/tests/queries/0_stateless/01079_window_view_inner_table_memory_tumble.sh
@@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CURDIR"/../shell_config.sh
 
 opts=(
-    "--allow_experimental_analyzer=0"
+    "--enable_analyzer=0"
 )
 
 $CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
diff --git a/tests/queries/0_stateless/01080_window_view_inner_table_memory_hop.sh b/tests/queries/0_stateless/01080_window_view_inner_table_memory_hop.sh
index 34a96d90e80..10dea880b26 100755
--- a/tests/queries/0_stateless/01080_window_view_inner_table_memory_hop.sh
+++ b/tests/queries/0_stateless/01080_window_view_inner_table_memory_hop.sh
@@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CURDIR"/../shell_config.sh
 
 opts=(
-    "--allow_experimental_analyzer=0"
+    "--enable_analyzer=0"
 )
 
 $CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
diff --git a/tests/queries/0_stateless/01081_window_view_target_table_engine.sh b/tests/queries/0_stateless/01081_window_view_target_table_engine.sh
index 180653d4931..806db683251 100755
--- a/tests/queries/0_stateless/01081_window_view_target_table_engine.sh
+++ b/tests/queries/0_stateless/01081_window_view_target_table_engine.sh
@@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CURDIR"/../shell_config.sh
 
 opts=(
-    "--allow_experimental_analyzer=0"
+    "--enable_analyzer=0"
 )
 
 $CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
diff --git a/tests/queries/0_stateless/01082_window_view_watch_limit.py b/tests/queries/0_stateless/01082_window_view_watch_limit.py
index 5dcdfdb5020..778ac786e88 100755
--- a/tests/queries/0_stateless/01082_window_view_watch_limit.py
+++ b/tests/queries/0_stateless/01082_window_view_watch_limit.py
@@ -19,7 +19,7 @@ with client(name="client1>", log=log) as client1, client(
     client1.expect(prompt)
     client2.expect(prompt)
 
-    client1.send("SET allow_experimental_analyzer = 0")
+    client1.send("SET enable_analyzer = 0")
     client1.expect(prompt)
     client1.send("SET allow_experimental_window_view = 1")
     client1.expect(prompt)
@@ -27,7 +27,7 @@ with client(name="client1>", log=log) as client1, client(
     client1.expect(prompt)
     client2.send("SET allow_experimental_window_view = 1")
     client2.expect(prompt)
-    client2.send("SET allow_experimental_analyzer = 0")
+    client2.send("SET enable_analyzer = 0")
     client2.expect(prompt)
 
     client1.send("CREATE DATABASE IF NOT EXISTS 01082_window_view_watch_limit")
diff --git a/tests/queries/0_stateless/01083_window_view_select.sh b/tests/queries/0_stateless/01083_window_view_select.sh
index 4d7c3cf5ace..729b119a561 100755
--- a/tests/queries/0_stateless/01083_window_view_select.sh
+++ b/tests/queries/0_stateless/01083_window_view_select.sh
@@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CURDIR"/../shell_config.sh
 
 opts=(
-    "--allow_experimental_analyzer=0"
+    "--enable_analyzer=0"
 )
 
 $CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
diff --git a/tests/queries/0_stateless/01084_window_view_with_table_identifier.sh b/tests/queries/0_stateless/01084_window_view_with_table_identifier.sh
index 64210de7ed0..b2d66f5db4f 100755
--- a/tests/queries/0_stateless/01084_window_view_with_table_identifier.sh
+++ b/tests/queries/0_stateless/01084_window_view_with_table_identifier.sh
@@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CURDIR"/../shell_config.sh
 
 opts=(
-    "--allow_experimental_analyzer=0"
+    "--enable_analyzer=0"
 )
 
 $CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
diff --git a/tests/queries/0_stateless/01085_window_view_attach.sql b/tests/queries/0_stateless/01085_window_view_attach.sql
index a40c6f20a1a..8f81e443aca 100644
--- a/tests/queries/0_stateless/01085_window_view_attach.sql
+++ b/tests/queries/0_stateless/01085_window_view_attach.sql
@@ -1,6 +1,6 @@
 
 SET send_logs_level = 'fatal';
-SET allow_experimental_analyzer = 0;
+SET enable_analyzer = 0;
 SET allow_experimental_window_view = 1;
 
 DROP DATABASE IF EXISTS {CLICKHOUSE_DATABASE:Identifier};
diff --git a/tests/queries/0_stateless/01086_window_view_cleanup.sh b/tests/queries/0_stateless/01086_window_view_cleanup.sh
index 0bce08523e2..8b8e794c8ff 100755
--- a/tests/queries/0_stateless/01086_window_view_cleanup.sh
+++ b/tests/queries/0_stateless/01086_window_view_cleanup.sh
@@ -8,7 +8,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CURDIR"/../shell_config.sh
 
 opts=(
-    "--allow_experimental_analyzer=0"
+    "--enable_analyzer=0"
 )
 
 DATABASE_ORDINARY="${CLICKHOUSE_DATABASE}_ordinary"
diff --git a/tests/queries/0_stateless/01087_window_view_alter_query.sh b/tests/queries/0_stateless/01087_window_view_alter_query.sh
index b9dd0d19234..03a6e39adb1 100755
--- a/tests/queries/0_stateless/01087_window_view_alter_query.sh
+++ b/tests/queries/0_stateless/01087_window_view_alter_query.sh
@@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CURDIR"/../shell_config.sh
 
 opts=(
-    "--allow_experimental_analyzer=0"
+    "--enable_analyzer=0"
 )
 
 $CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
diff --git a/tests/queries/0_stateless/01088_window_view_default_column.sh b/tests/queries/0_stateless/01088_window_view_default_column.sh
index 03be3459e54..0e989444bd7 100755
--- a/tests/queries/0_stateless/01088_window_view_default_column.sh
+++ b/tests/queries/0_stateless/01088_window_view_default_column.sh
@@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CURDIR"/../shell_config.sh
 
 opts=(
-    "--allow_experimental_analyzer=0"
+    "--enable_analyzer=0"
 )
 
 $CLICKHOUSE_CLIENT "${opts[@]}" <<EOF
diff --git a/tests/queries/0_stateless/01142_join_lc_and_nullable_in_key.sql b/tests/queries/0_stateless/01142_join_lc_and_nullable_in_key.sql
index 2464b7a57cf..dda396f4b5d 100644
--- a/tests/queries/0_stateless/01142_join_lc_and_nullable_in_key.sql
+++ b/tests/queries/0_stateless/01142_join_lc_and_nullable_in_key.sql
@@ -16,36 +16,36 @@ SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l FULL JOIN nr AS r USING (x) ORD
 SELECT '-';
 
 -- lc should be supertype for l.lc and r.lc, so expect Nullable(String)
-SELECT x, lc, toTypeName(lc), r.lc, toTypeName(r.lc) FROM t AS l LEFT JOIN nr AS r USING (lc) ORDER BY x SETTINGS allow_experimental_analyzer = 1;
-SELECT x, lc, toTypeName(lc), r.lc, toTypeName(r.lc) FROM t AS l RIGHT JOIN nr AS r USING (lc) ORDER BY x SETTINGS allow_experimental_analyzer = 1;
-SELECT x, lc, toTypeName(lc), r.lc, toTypeName(r.lc) FROM t AS l FULL JOIN nr AS r USING (lc) ORDER BY x SETTINGS allow_experimental_analyzer = 1;
+SELECT x, lc, toTypeName(lc), r.lc, toTypeName(r.lc) FROM t AS l LEFT JOIN nr AS r USING (lc) ORDER BY x SETTINGS enable_analyzer = 1;
+SELECT x, lc, toTypeName(lc), r.lc, toTypeName(r.lc) FROM t AS l RIGHT JOIN nr AS r USING (lc) ORDER BY x SETTINGS enable_analyzer = 1;
+SELECT x, lc, toTypeName(lc), r.lc, toTypeName(r.lc) FROM t AS l FULL JOIN nr AS r USING (lc) ORDER BY x SETTINGS enable_analyzer = 1;
 
 SELECT '-';
 
 -- old behavior is different
-SELECT x, lc, toTypeName(lc), r.lc, toTypeName(r.lc) FROM t AS l LEFT JOIN nr AS r USING (lc) ORDER BY x SETTINGS allow_experimental_analyzer = 0;
-SELECT x, lc, toTypeName(lc), r.lc, toTypeName(r.lc) FROM t AS l RIGHT JOIN nr AS r USING (lc) ORDER BY x SETTINGS allow_experimental_analyzer = 0;
-SELECT x, lc, toTypeName(lc), r.lc, toTypeName(r.lc) FROM t AS l FULL JOIN nr AS r USING (lc) ORDER BY x SETTINGS allow_experimental_analyzer = 0;
+SELECT x, lc, toTypeName(lc), r.lc, toTypeName(r.lc) FROM t AS l LEFT JOIN nr AS r USING (lc) ORDER BY x SETTINGS enable_analyzer = 0;
+SELECT x, lc, toTypeName(lc), r.lc, toTypeName(r.lc) FROM t AS l RIGHT JOIN nr AS r USING (lc) ORDER BY x SETTINGS enable_analyzer = 0;
+SELECT x, lc, toTypeName(lc), r.lc, toTypeName(r.lc) FROM t AS l FULL JOIN nr AS r USING (lc) ORDER BY x SETTINGS enable_analyzer = 0;
 
 SELECT '-';
 
-SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l LEFT JOIN nr AS r USING (lc) ORDER BY x SETTINGS allow_experimental_analyzer = 1;
-SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l RIGHT JOIN nr AS r USING (lc) ORDER BY x SETTINGS allow_experimental_analyzer = 1;
-SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l FULL JOIN nr AS r USING (lc) ORDER BY x SETTINGS allow_experimental_analyzer = 1;
+SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l LEFT JOIN nr AS r USING (lc) ORDER BY x SETTINGS enable_analyzer = 1;
+SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l RIGHT JOIN nr AS r USING (lc) ORDER BY x SETTINGS enable_analyzer = 1;
+SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l FULL JOIN nr AS r USING (lc) ORDER BY x SETTINGS enable_analyzer = 1;
 
 SELECT '-';
 
-SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l LEFT JOIN nr AS r USING (lc) ORDER BY x SETTINGS allow_experimental_analyzer = 0;
-SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l RIGHT JOIN nr AS r USING (lc) ORDER BY x SETTINGS allow_experimental_analyzer = 0;
-SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l FULL JOIN nr AS r USING (lc) ORDER BY x SETTINGS allow_experimental_analyzer = 0;
+SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l LEFT JOIN nr AS r USING (lc) ORDER BY x SETTINGS enable_analyzer = 0;
+SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l RIGHT JOIN nr AS r USING (lc) ORDER BY x SETTINGS enable_analyzer = 0;
+SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l FULL JOIN nr AS r USING (lc) ORDER BY x SETTINGS enable_analyzer = 0;
 
 SELECT '-';
 
-SELECT x, lc FROM t AS l RIGHT JOIN nr AS r USING (lc) SETTINGS allow_experimental_analyzer = 1;
+SELECT x, lc FROM t AS l RIGHT JOIN nr AS r USING (lc) SETTINGS enable_analyzer = 1;
 
 SELECT '-';
 
-SELECT x, lc FROM t AS l RIGHT JOIN nr AS r USING (lc) SETTINGS allow_experimental_analyzer = 0;
+SELECT x, lc FROM t AS l RIGHT JOIN nr AS r USING (lc) SETTINGS enable_analyzer = 0;
 
 SELECT '-';
 
diff --git a/tests/queries/0_stateless/01142_merge_join_lc_and_nullable_in_key.sql b/tests/queries/0_stateless/01142_merge_join_lc_and_nullable_in_key.sql
index 718e8358c64..862c26c41ce 100644
--- a/tests/queries/0_stateless/01142_merge_join_lc_and_nullable_in_key.sql
+++ b/tests/queries/0_stateless/01142_merge_join_lc_and_nullable_in_key.sql
@@ -17,27 +17,27 @@ SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l FULL JOIN nr AS r USING (x) ORD
 
 SELECT '-';
 
-SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l LEFT JOIN nr AS r USING (lc) ORDER BY x SETTINGS allow_experimental_analyzer = 1;
-SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l RIGHT JOIN nr AS r USING (lc) ORDER BY x SETTINGS allow_experimental_analyzer = 1;
-SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l FULL JOIN nr AS r USING (lc) ORDER BY x SETTINGS allow_experimental_analyzer = 1;
+SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l LEFT JOIN nr AS r USING (lc) ORDER BY x SETTINGS enable_analyzer = 1;
+SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l RIGHT JOIN nr AS r USING (lc) ORDER BY x SETTINGS enable_analyzer = 1;
+SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l FULL JOIN nr AS r USING (lc) ORDER BY x SETTINGS enable_analyzer = 1;
 
 SELECT '-';
 
-SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l LEFT JOIN nr AS r USING (lc) ORDER BY x SETTINGS allow_experimental_analyzer = 1;
-SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l RIGHT JOIN nr AS r USING (lc) ORDER BY x SETTINGS allow_experimental_analyzer = 1;
-SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l FULL JOIN nr AS r USING (lc) ORDER BY x SETTINGS allow_experimental_analyzer = 1;
+SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l LEFT JOIN nr AS r USING (lc) ORDER BY x SETTINGS enable_analyzer = 1;
+SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l RIGHT JOIN nr AS r USING (lc) ORDER BY x SETTINGS enable_analyzer = 1;
+SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l FULL JOIN nr AS r USING (lc) ORDER BY x SETTINGS enable_analyzer = 1;
 
 SELECT '-';
 
-SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l LEFT JOIN nr AS r USING (lc) ORDER BY x SETTINGS allow_experimental_analyzer = 0;
-SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l RIGHT JOIN nr AS r USING (lc) ORDER BY x SETTINGS allow_experimental_analyzer = 0;
-SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l FULL JOIN nr AS r USING (lc) ORDER BY x SETTINGS allow_experimental_analyzer = 0;
+SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l LEFT JOIN nr AS r USING (lc) ORDER BY x SETTINGS enable_analyzer = 0;
+SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l RIGHT JOIN nr AS r USING (lc) ORDER BY x SETTINGS enable_analyzer = 0;
+SELECT x, lc, r.lc, toTypeName(r.lc) FROM t AS l FULL JOIN nr AS r USING (lc) ORDER BY x SETTINGS enable_analyzer = 0;
 
 SELECT '-';
 
-SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l LEFT JOIN nr AS r USING (lc) ORDER BY x SETTINGS allow_experimental_analyzer = 0;
-SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l RIGHT JOIN nr AS r USING (lc) ORDER BY x SETTINGS allow_experimental_analyzer = 0;
-SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l FULL JOIN nr AS r USING (lc) ORDER BY x SETTINGS allow_experimental_analyzer = 0;
+SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l LEFT JOIN nr AS r USING (lc) ORDER BY x SETTINGS enable_analyzer = 0;
+SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l RIGHT JOIN nr AS r USING (lc) ORDER BY x SETTINGS enable_analyzer = 0;
+SELECT x, lc, materialize(r.lc) y, toTypeName(y) FROM t AS l FULL JOIN nr AS r USING (lc) ORDER BY x SETTINGS enable_analyzer = 0;
 
 SELECT '-';
 
diff --git a/tests/queries/0_stateless/01232_untuple.sql b/tests/queries/0_stateless/01232_untuple.sql
index 391d08ab859..e2049e0a453 100644
--- a/tests/queries/0_stateless/01232_untuple.sql
+++ b/tests/queries/0_stateless/01232_untuple.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 SET enable_named_columns_in_function_tuple = 1;
 
 select untuple((* except (b),)) from (select 1 a, 2 b, 3 c);
diff --git a/tests/queries/0_stateless/01300_group_by_other_keys_having.reference b/tests/queries/0_stateless/01300_group_by_other_keys_having.reference
index f861da3da2b..23028e3fb6e 100644
--- a/tests/queries/0_stateless/01300_group_by_other_keys_having.reference
+++ b/tests/queries/0_stateless/01300_group_by_other_keys_having.reference
@@ -3,7 +3,7 @@ SELECT round(avg(log(2) * number), 6) AS k FROM numbers(10000000) GROUP BY (numb
 3465735.9028
 3465735.9028
 3465736.595947
-SELECT round(avg(log(2) * number), 6) AS k FROM numbers(10000000) GROUP BY (number % 2) * (number % 3), number % 3, number % 2 HAVING avg(log(2) * number) > 3465735.3 ORDER BY k SETTINGS allow_experimental_analyzer=1;
+SELECT round(avg(log(2) * number), 6) AS k FROM numbers(10000000) GROUP BY (number % 2) * (number % 3), number % 3, number % 2 HAVING avg(log(2) * number) > 3465735.3 ORDER BY k SETTINGS enable_analyzer=1;
 3465735.9028
 3465735.9028
 3465736.595947
@@ -11,7 +11,7 @@ SELECT round(avg(log(2) * number), 6) AS k FROM numbers(10000000) GROUP BY numbe
 3465734.169932
 3465734.863079
 3465735.556226
-SELECT round(avg(log(2) * number), 6) AS k FROM numbers(10000000) GROUP BY number % 5, ((number % 5) * (number % 5)) HAVING ((number % 5) * (number % 5)) < 5 ORDER BY k SETTINGS allow_experimental_analyzer=1;
+SELECT round(avg(log(2) * number), 6) AS k FROM numbers(10000000) GROUP BY number % 5, ((number % 5) * (number % 5)) HAVING ((number % 5) * (number % 5)) < 5 ORDER BY k SETTINGS enable_analyzer=1;
 3465734.169932
 3465734.863079
 3465735.556226
@@ -19,7 +19,7 @@ SELECT (number % 5) * (number % 5) AS k FROM numbers(10000000) GROUP BY number %
 0
 1
 4
-SELECT (number % 5) * (number % 5) AS k FROM numbers(10000000) GROUP BY number % 5, ((number % 5) * (number % 5)) HAVING ((number % 5) * (number % 5)) < 5 ORDER BY k SETTINGS allow_experimental_analyzer=1;
+SELECT (number % 5) * (number % 5) AS k FROM numbers(10000000) GROUP BY number % 5, ((number % 5) * (number % 5)) HAVING ((number % 5) * (number % 5)) < 5 ORDER BY k SETTINGS enable_analyzer=1;
 0
 1
 4
diff --git a/tests/queries/0_stateless/01300_group_by_other_keys_having.sql b/tests/queries/0_stateless/01300_group_by_other_keys_having.sql
index 203e8322ad9..a73b122a9f9 100644
--- a/tests/queries/0_stateless/01300_group_by_other_keys_having.sql
+++ b/tests/queries/0_stateless/01300_group_by_other_keys_having.sql
@@ -1,16 +1,16 @@
 set optimize_group_by_function_keys = 1;
 set optimize_syntax_fuse_functions = 0;
-set allow_experimental_analyzer = 1;
+set enable_analyzer = 1;
 
 -- { echoOn }
 SELECT round(avg(log(2) * number), 6) AS k FROM numbers(10000000) GROUP BY (number % 2) * (number % 3), number % 3, number % 2 HAVING avg(log(2) * number) > 3465735.3 ORDER BY k;
-SELECT round(avg(log(2) * number), 6) AS k FROM numbers(10000000) GROUP BY (number % 2) * (number % 3), number % 3, number % 2 HAVING avg(log(2) * number) > 3465735.3 ORDER BY k SETTINGS allow_experimental_analyzer=1;
+SELECT round(avg(log(2) * number), 6) AS k FROM numbers(10000000) GROUP BY (number % 2) * (number % 3), number % 3, number % 2 HAVING avg(log(2) * number) > 3465735.3 ORDER BY k SETTINGS enable_analyzer=1;
 
 SELECT round(avg(log(2) * number), 6) AS k FROM numbers(10000000) GROUP BY number % 5, ((number % 5) * (number % 5)) HAVING ((number % 5) * (number % 5)) < 5 ORDER BY k;
-SELECT round(avg(log(2) * number), 6) AS k FROM numbers(10000000) GROUP BY number % 5, ((number % 5) * (number % 5)) HAVING ((number % 5) * (number % 5)) < 5 ORDER BY k SETTINGS allow_experimental_analyzer=1;
+SELECT round(avg(log(2) * number), 6) AS k FROM numbers(10000000) GROUP BY number % 5, ((number % 5) * (number % 5)) HAVING ((number % 5) * (number % 5)) < 5 ORDER BY k SETTINGS enable_analyzer=1;
 
 SELECT (number % 5) * (number % 5) AS k FROM numbers(10000000) GROUP BY number % 5, ((number % 5) * (number % 5)) HAVING ((number % 5) * (number % 5)) < 5 ORDER BY k;
-SELECT (number % 5) * (number % 5) AS k FROM numbers(10000000) GROUP BY number % 5, ((number % 5) * (number % 5)) HAVING ((number % 5) * (number % 5)) < 5 ORDER BY k SETTINGS allow_experimental_analyzer=1;
+SELECT (number % 5) * (number % 5) AS k FROM numbers(10000000) GROUP BY number % 5, ((number % 5) * (number % 5)) HAVING ((number % 5) * (number % 5)) < 5 ORDER BY k SETTINGS enable_analyzer=1;
 
 -- { echoOff }
 
diff --git a/tests/queries/0_stateless/01323_redundant_functions_in_order_by.reference b/tests/queries/0_stateless/01323_redundant_functions_in_order_by.reference
index d47f12ff4d1..c2c37cc4de6 100644
--- a/tests/queries/0_stateless/01323_redundant_functions_in_order_by.reference
+++ b/tests/queries/0_stateless/01323_redundant_functions_in_order_by.reference
@@ -65,7 +65,7 @@ QUERY id: 0
           SORT id: 12, sort_direction: ASCENDING, with_fill: 0
             EXPRESSION
               COLUMN id: 7, column_name: number, result_type: UInt64, source_id: 8
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 SELECT groupArray(x)
 FROM
 (
@@ -99,7 +99,7 @@ QUERY id: 0
           SORT id: 12, sort_direction: ASCENDING, with_fill: 0
             EXPRESSION
               COLUMN id: 7, column_name: number, result_type: UInt64, source_id: 8
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 SELECT groupArray(x)
 FROM
 (
@@ -141,7 +141,7 @@ QUERY id: 0
           SORT id: 15, sort_direction: ASCENDING, with_fill: 0
             EXPRESSION
               COLUMN id: 7, column_name: number, result_type: UInt64, source_id: 8
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 SELECT
     key,
     a,
@@ -203,7 +203,7 @@ QUERY id: 0
       SORT id: 25, sort_direction: ASCENDING, with_fill: 0
         EXPRESSION
           COLUMN id: 26, column_name: key, result_type: UInt64, source_id: 5
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 SELECT
     key,
     a
@@ -229,7 +229,7 @@ QUERY id: 0
       SORT id: 7, sort_direction: ASCENDING, with_fill: 0
         EXPRESSION
           COLUMN id: 4, column_name: a, result_type: UInt8, source_id: 3
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 SELECT
     key,
     a
@@ -262,7 +262,7 @@ QUERY id: 0
                     LIST id: 11, nodes: 2
                       COLUMN id: 2, column_name: key, result_type: UInt64, source_id: 3
                       COLUMN id: 4, column_name: a, result_type: UInt8, source_id: 3
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 QUERY id: 0
   PROJECTION COLUMNS
     key UInt64
@@ -285,7 +285,7 @@ QUERY id: 0
       SORT id: 10, sort_direction: ASCENDING, with_fill: 0
         EXPRESSION
           COLUMN id: 2, column_name: key, result_type: UInt64, source_id: 3
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 QUERY id: 0
   PROJECTION COLUMNS
     t1.id UInt64
@@ -314,7 +314,7 @@ QUERY id: 0
       SORT id: 14, sort_direction: ASCENDING, with_fill: 0
         EXPRESSION
           COLUMN id: 15, column_name: id, result_type: UInt64, source_id: 5
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 [0,1,2]
 [0,1,2]
 [0,1,2]
diff --git a/tests/queries/0_stateless/01323_redundant_functions_in_order_by.sql b/tests/queries/0_stateless/01323_redundant_functions_in_order_by.sql
index 738ad581e3d..fb1eed1666e 100644
--- a/tests/queries/0_stateless/01323_redundant_functions_in_order_by.sql
+++ b/tests/queries/0_stateless/01323_redundant_functions_in_order_by.sql
@@ -8,37 +8,37 @@ INSERT INTO test SELECT number, number, toString(number), number from numbers(4)
 set optimize_redundant_functions_in_order_by = 1;
 
 SELECT groupArray(x) from (SELECT number as x FROM numbers(3) ORDER BY x, exp(x));
-SELECT groupArray(x) from (SELECT number as x FROM numbers(3) ORDER BY x, exp(x)) SETTINGS allow_experimental_analyzer=1;
+SELECT groupArray(x) from (SELECT number as x FROM numbers(3) ORDER BY x, exp(x)) SETTINGS enable_analyzer=1;
 SELECT groupArray(x) from (SELECT number as x FROM numbers(3) ORDER BY x, exp(exp(x)));
-SELECT groupArray(x) from (SELECT number as x FROM numbers(3) ORDER BY x, exp(exp(x))) SETTINGS allow_experimental_analyzer=1;
+SELECT groupArray(x) from (SELECT number as x FROM numbers(3) ORDER BY x, exp(exp(x))) SETTINGS enable_analyzer=1;
 SELECT groupArray(x) from (SELECT number as x FROM numbers(3) ORDER BY exp(x), x);
-SELECT groupArray(x) from (SELECT number as x FROM numbers(3) ORDER BY exp(x), x) SETTINGS allow_experimental_analyzer=1;
+SELECT groupArray(x) from (SELECT number as x FROM numbers(3) ORDER BY exp(x), x) SETTINGS enable_analyzer=1;
 SELECT * FROM (SELECT number + 2 AS key FROM numbers(4)) s FULL JOIN test t USING(key) ORDER BY s.key, t.key;
-SELECT * FROM (SELECT number + 2 AS key FROM numbers(4)) s FULL JOIN test t USING(key) ORDER BY s.key, t.key SETTINGS allow_experimental_analyzer=1;
+SELECT * FROM (SELECT number + 2 AS key FROM numbers(4)) s FULL JOIN test t USING(key) ORDER BY s.key, t.key SETTINGS enable_analyzer=1;
 SELECT key, a FROM test ORDER BY key, a, exp(key + a);
-SELECT key, a FROM test ORDER BY key, a, exp(key + a) SETTINGS allow_experimental_analyzer=1;
+SELECT key, a FROM test ORDER BY key, a, exp(key + a) SETTINGS enable_analyzer=1;
 SELECT key, a FROM test ORDER BY key, exp(key + a);
-SELECT key, a FROM test ORDER BY key, exp(key + a) SETTINGS allow_experimental_analyzer=1;
+SELECT key, a FROM test ORDER BY key, exp(key + a) SETTINGS enable_analyzer=1;
 EXPLAIN SYNTAX SELECT groupArray(x) from (SELECT number as x FROM numbers(3) ORDER BY x, exp(x));
-EXPLAIN QUERY TREE run_passes=1 SELECT groupArray(x) from (SELECT number as x FROM numbers(3) ORDER BY x, exp(x)) settings allow_experimental_analyzer=1;
+EXPLAIN QUERY TREE run_passes=1 SELECT groupArray(x) from (SELECT number as x FROM numbers(3) ORDER BY x, exp(x)) settings enable_analyzer=1;
 EXPLAIN SYNTAX SELECT groupArray(x) from (SELECT number as x FROM numbers(3) ORDER BY x, exp(exp(x)));
-EXPLAIN QUERY TREE run_passes=1 SELECT groupArray(x) from (SELECT number as x FROM numbers(3) ORDER BY x, exp(exp(x))) settings allow_experimental_analyzer=1;
+EXPLAIN QUERY TREE run_passes=1 SELECT groupArray(x) from (SELECT number as x FROM numbers(3) ORDER BY x, exp(exp(x))) settings enable_analyzer=1;
 EXPLAIN SYNTAX SELECT groupArray(x) from (SELECT number as x FROM numbers(3) ORDER BY exp(x), x);
-EXPLAIN QUERY TREE run_passes=1 SELECT groupArray(x) from (SELECT number as x FROM numbers(3) ORDER BY exp(x), x) settings allow_experimental_analyzer=1;
+EXPLAIN QUERY TREE run_passes=1 SELECT groupArray(x) from (SELECT number as x FROM numbers(3) ORDER BY exp(x), x) settings enable_analyzer=1;
 EXPLAIN SYNTAX SELECT * FROM (SELECT number + 2 AS key FROM numbers(4)) s FULL JOIN test t USING(key) ORDER BY s.key, t.key;
-EXPLAIN QUERY TREE run_passes=1 SELECT * FROM (SELECT number + 2 AS key FROM numbers(4)) s FULL JOIN test t USING(key) ORDER BY s.key, t.key settings allow_experimental_analyzer=1;
+EXPLAIN QUERY TREE run_passes=1 SELECT * FROM (SELECT number + 2 AS key FROM numbers(4)) s FULL JOIN test t USING(key) ORDER BY s.key, t.key settings enable_analyzer=1;
 EXPLAIN SYNTAX SELECT key, a FROM test ORDER BY key, a, exp(key + a);
-EXPLAIN QUERY TREE run_passes=1 SELECT key, a FROM test ORDER BY key, a, exp(key + a) settings allow_experimental_analyzer=1;
+EXPLAIN QUERY TREE run_passes=1 SELECT key, a FROM test ORDER BY key, a, exp(key + a) settings enable_analyzer=1;
 EXPLAIN SYNTAX SELECT key, a FROM test ORDER BY key, exp(key + a);
-EXPLAIN QUERY TREE run_passes=1 SELECT key, a FROM test ORDER BY key, exp(key + a) settings allow_experimental_analyzer=1;
-EXPLAIN QUERY TREE run_passes=1 SELECT key FROM test GROUP BY key ORDER BY avg(a), key settings allow_experimental_analyzer=1;
+EXPLAIN QUERY TREE run_passes=1 SELECT key, a FROM test ORDER BY key, exp(key + a) settings enable_analyzer=1;
+EXPLAIN QUERY TREE run_passes=1 SELECT key FROM test GROUP BY key ORDER BY avg(a), key settings enable_analyzer=1;
 
 DROP TABLE IF EXISTS t1;
 DROP TABLE IF EXISTS t2;
 CREATE TABLE t1 (id UInt64) ENGINE = MergeTree() ORDER BY id;
 CREATE TABLE t2 (id UInt64) ENGINE = MergeTree() ORDER BY id;
 
-EXPLAIN QUERY TREE run_passes=1 SELECT * FROM t1 INNER JOIN t2 ON t1.id = t2.id ORDER BY t1.id, t2.id settings allow_experimental_analyzer=1;
+EXPLAIN QUERY TREE run_passes=1 SELECT * FROM t1 INNER JOIN t2 ON t1.id = t2.id ORDER BY t1.id, t2.id settings enable_analyzer=1;
 
 set optimize_redundant_functions_in_order_by = 0;
 
diff --git a/tests/queries/0_stateless/01353_low_cardinality_join_types.sql b/tests/queries/0_stateless/01353_low_cardinality_join_types.sql
index 93953f1d74a..2aa42f33fd3 100644
--- a/tests/queries/0_stateless/01353_low_cardinality_join_types.sql
+++ b/tests/queries/0_stateless/01353_low_cardinality_join_types.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 0;
+SET enable_analyzer = 0;
 set join_algorithm = 'hash';
 
 select '-';
@@ -75,7 +75,7 @@ from (select toLowCardinality(number) k, toLowCardinality(toString(number)) s fr
 full join (select toLowCardinality(number+1) k, toLowCardinality(toString(number+1)) s from numbers(2)) as js2
 using k order by js1.k, js2.k;
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 set join_algorithm = 'hash';
 
 select '-';
diff --git a/tests/queries/0_stateless/01376_GROUP_BY_injective_elimination_dictGet.reference b/tests/queries/0_stateless/01376_GROUP_BY_injective_elimination_dictGet.reference
index 6de0a5be0a5..2d16e71f099 100644
--- a/tests/queries/0_stateless/01376_GROUP_BY_injective_elimination_dictGet.reference
+++ b/tests/queries/0_stateless/01376_GROUP_BY_injective_elimination_dictGet.reference
@@ -21,4 +21,4 @@ QUERY id: 0
   GROUP BY
     LIST id: 10, nodes: 1
       COLUMN id: 6, column_name: number, result_type: UInt64, source_id: 7
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
diff --git a/tests/queries/0_stateless/01376_GROUP_BY_injective_elimination_dictGet.sql b/tests/queries/0_stateless/01376_GROUP_BY_injective_elimination_dictGet.sql
index a868b38b4d7..08ca9ed3c2d 100644
--- a/tests/queries/0_stateless/01376_GROUP_BY_injective_elimination_dictGet.sql
+++ b/tests/queries/0_stateless/01376_GROUP_BY_injective_elimination_dictGet.sql
@@ -38,7 +38,7 @@ EXPLAIN QUERY TREE
 SELECT dictGet('dictdb_01376.dict_exists', 'value', number) as val
 FROM numbers(2)
 GROUP BY val
-SETTINGS allow_experimental_analyzer = 1;
+SETTINGS enable_analyzer = 1;
 
 DROP DICTIONARY dictdb_01376.dict_exists;
 DROP TABLE dictdb_01376.table_for_dict;
diff --git a/tests/queries/0_stateless/01428_nullable_asof_join.sql b/tests/queries/0_stateless/01428_nullable_asof_join.sql
index f07a26edd97..41f6ba8a03d 100644
--- a/tests/queries/0_stateless/01428_nullable_asof_join.sql
+++ b/tests/queries/0_stateless/01428_nullable_asof_join.sql
@@ -18,13 +18,13 @@ SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(ma
 FROM (SELECT toUInt8(number) > 0 as pk, toUInt8(number) as dt FROM numbers(3)) a
 ASOF LEFT JOIN (SELECT 1 as pk, toNullable(0) as dt) b
 USING(pk, dt)
-ORDER BY a.dt SETTINGS allow_experimental_analyzer = 0;
+ORDER BY a.dt SETTINGS enable_analyzer = 0;
 
 SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt))
 FROM (SELECT toUInt8(number) > 0 as pk, toUInt8(number) as dt FROM numbers(3)) a
 ASOF LEFT JOIN (SELECT 1 as pk, toNullable(0) as dt) b
 USING(pk, dt)
-ORDER BY a.dt SETTINGS allow_experimental_analyzer = 1;
+ORDER BY a.dt SETTINGS enable_analyzer = 1;
 
 SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt))
 FROM (SELECT toUInt8(number) > 0 as pk, toNullable(toUInt8(number)) as dt FROM numbers(3)) a
@@ -70,25 +70,25 @@ SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(ma
 FROM (SELECT toUInt8(number) > 0 as pk, toNullable(toUInt8(number)) as dt FROM numbers(3)) a
 ASOF JOIN (SELECT 1 as pk, 2 as dt) b
 USING(pk, dt)
-ORDER BY a.dt SETTINGS allow_experimental_analyzer = 0;
+ORDER BY a.dt SETTINGS enable_analyzer = 0;
 
 SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt))
 FROM (SELECT toUInt8(number) > 0 as pk, toNullable(toUInt8(number)) as dt FROM numbers(3)) a
 ASOF JOIN (SELECT 1 as pk, 2 as dt) b
 USING(pk, dt)
-ORDER BY a.dt SETTINGS allow_experimental_analyzer = 1;
+ORDER BY a.dt SETTINGS enable_analyzer = 1;
 
 SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt))
 FROM (SELECT toUInt8(number) > 0 as pk, toUInt8(number) as dt FROM numbers(3)) a
 ASOF JOIN (SELECT 1 as pk, toNullable(0) as dt) b
 USING(pk, dt)
-ORDER BY a.dt SETTINGS allow_experimental_analyzer = 0;
+ORDER BY a.dt SETTINGS enable_analyzer = 0;
 
 SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt))
 FROM (SELECT toUInt8(number) > 0 as pk, toUInt8(number) as dt FROM numbers(3)) a
 ASOF JOIN (SELECT 1 as pk, toNullable(0) as dt) b
 USING(pk, dt)
-ORDER BY a.dt SETTINGS allow_experimental_analyzer = 1;
+ORDER BY a.dt SETTINGS enable_analyzer = 1;
 
 SELECT a.pk, b.pk, a.dt, b.dt, toTypeName(a.pk), toTypeName(b.pk), toTypeName(materialize(a.dt)), toTypeName(materialize(b.dt))
 FROM (SELECT toUInt8(number) > 0 as pk, toNullable(toUInt8(number)) as dt FROM numbers(3)) a
diff --git a/tests/queries/0_stateless/01455_opentelemetry_distributed.reference b/tests/queries/0_stateless/01455_opentelemetry_distributed.reference
index 2920b387aa2..2b08db1f27f 100644
--- a/tests/queries/0_stateless/01455_opentelemetry_distributed.reference
+++ b/tests/queries/0_stateless/01455_opentelemetry_distributed.reference
@@ -1,10 +1,10 @@
 ===http===
-{"query":"select 1 from remote('127.0.0.2', system, one) settings allow_experimental_analyzer = 1 format Null\n","status":"QueryFinish","tracestate":"some custom state","sorted_by_start_time":1}
+{"query":"select 1 from remote('127.0.0.2', system, one) settings enable_analyzer = 1 format Null\n","status":"QueryFinish","tracestate":"some custom state","sorted_by_start_time":1}
 {"query":"DESC TABLE system.one","status":"QueryFinish","tracestate":"some custom state","sorted_by_start_time":1}
 {"query":"SELECT 1 AS `1` FROM `system`.`one` AS `__table1`","status":"QueryFinish","tracestate":"some custom state","sorted_by_start_time":1}
 {"query":"DESC TABLE system.one","query_status":"QueryFinish","tracestate":"some custom state","sorted_by_finish_time":1}
 {"query":"SELECT 1 AS `1` FROM `system`.`one` AS `__table1`","query_status":"QueryFinish","tracestate":"some custom state","sorted_by_finish_time":1}
-{"query":"select 1 from remote('127.0.0.2', system, one) settings allow_experimental_analyzer = 1 format Null\n","query_status":"QueryFinish","tracestate":"some custom state","sorted_by_finish_time":1}
+{"query":"select 1 from remote('127.0.0.2', system, one) settings enable_analyzer = 1 format Null\n","query_status":"QueryFinish","tracestate":"some custom state","sorted_by_finish_time":1}
 {"total spans":"3","unique spans":"3","unique non-zero parent spans":"3"}
 {"initial query spans with proper parent":"2"}
 {"unique non-empty tracestate values":"1"}
diff --git a/tests/queries/0_stateless/01455_opentelemetry_distributed.sh b/tests/queries/0_stateless/01455_opentelemetry_distributed.sh
index e19810dd7f1..2b6da6132ed 100755
--- a/tests/queries/0_stateless/01455_opentelemetry_distributed.sh
+++ b/tests/queries/0_stateless/01455_opentelemetry_distributed.sh
@@ -10,7 +10,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 function check_log
 {
 ${CLICKHOUSE_CLIENT} --format=JSONEachRow -nq "
-set allow_experimental_analyzer = 1;
+set enable_analyzer = 1;
 system flush logs;
 
 -- Show queries sorted by start time.
@@ -75,7 +75,7 @@ select uniqExact(value) "'"'"unique non-empty tracestate values"'"'"
 
 # Generate some random trace id so that the prevous runs of the test do not interfere.
 echo "===http==="
-trace_id=$(${CLICKHOUSE_CLIENT} -q "select lower(hex(reverse(reinterpretAsString(generateUUIDv4())))) settings allow_experimental_analyzer = 1")
+trace_id=$(${CLICKHOUSE_CLIENT} -q "select lower(hex(reverse(reinterpretAsString(generateUUIDv4())))) settings enable_analyzer = 1")
 
 # Check that the HTTP traceparent is read, and then passed through `remote`
 # table function. We expect 4 queries -- one initial, one SELECT and two
@@ -85,7 +85,7 @@ ${CLICKHOUSE_CURL} \
     --header "traceparent: 00-$trace_id-0000000000000073-01" \
     --header "tracestate: some custom state" "$CLICKHOUSE_URL" \
     --get \
-    --data-urlencode "query=select 1 from remote('127.0.0.2', system, one) settings allow_experimental_analyzer = 1 format Null"
+    --data-urlencode "query=select 1 from remote('127.0.0.2', system, one) settings enable_analyzer = 1 format Null"
 
 check_log
 
diff --git a/tests/queries/0_stateless/01476_right_full_join_switch.sql b/tests/queries/0_stateless/01476_right_full_join_switch.sql
index dfbdec47e1f..7c8c3157844 100644
--- a/tests/queries/0_stateless/01476_right_full_join_switch.sql
+++ b/tests/queries/0_stateless/01476_right_full_join_switch.sql
@@ -13,7 +13,7 @@ INSERT INTO nr VALUES (2, NULL);
 
 SET join_use_nulls = 0;
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 -- t.x is supertupe for `x` from left and right since `x` is inside `USING`.
 SELECT t.x, l.s, r.s, toTypeName(l.s), toTypeName(r.s) FROM t AS l LEFT JOIN nr AS r USING (x) ORDER BY t.x;
@@ -28,7 +28,7 @@ SELECT t.x, l.s, r.s, toTypeName(l.s), toTypeName(r.s) FROM nr AS l FULL JOIN t
 
 SELECT '-';
 
-SET allow_experimental_analyzer = 0;
+SET enable_analyzer = 0;
 
 -- t.x is supertupe for `x` from left and right since `x` is inside `USING`.
 SELECT t.x, l.s, r.s, toTypeName(l.s), toTypeName(r.s) FROM t AS l LEFT JOIN nr AS r USING (x) ORDER BY t.x;
diff --git a/tests/queries/0_stateless/01477_lc_in_merge_join_left_key.sql.j2 b/tests/queries/0_stateless/01477_lc_in_merge_join_left_key.sql.j2
index 6eafd41b411..b43db222e43 100644
--- a/tests/queries/0_stateless/01477_lc_in_merge_join_left_key.sql.j2
+++ b/tests/queries/0_stateless/01477_lc_in_merge_join_left_key.sql.j2
@@ -10,11 +10,11 @@ CREATE TABLE nr (`x` Nullable(UInt32), `s` Nullable(String)) ENGINE = Memory;
 INSERT INTO t VALUES (1, 'l');
 INSERT INTO nr VALUES (2, NULL);
 
-{% for allow_experimental_analyzer in [0, 1] -%}
+{% for enable_analyzer in [0, 1] -%}
 
-SET allow_experimental_analyzer = {{ allow_experimental_analyzer }};
+SET enable_analyzer = {{ enable_analyzer }};
 
-{% if allow_experimental_analyzer -%}
+{% if enable_analyzer -%}
 SELECT '- analyzer -';
 {% endif -%}
 
diff --git a/tests/queries/0_stateless/01479_cross_join_9855.sql b/tests/queries/0_stateless/01479_cross_join_9855.sql
index 9dcf209a1cd..19cd0ab18fd 100644
--- a/tests/queries/0_stateless/01479_cross_join_9855.sql
+++ b/tests/queries/0_stateless/01479_cross_join_9855.sql
@@ -2,8 +2,8 @@ SET cross_to_inner_join_rewrite = 1;
 
 SELECT count()
 FROM numbers(4) AS n1, numbers(3) AS n2
-WHERE n1.number > (select avg(n.number) from numbers(3) n) SETTINGS allow_experimental_analyzer=0;
+WHERE n1.number > (select avg(n.number) from numbers(3) n) SETTINGS enable_analyzer=0;
 
 SELECT count()
 FROM numbers(4) AS n1, numbers(3) AS n2, numbers(6) AS n3
-WHERE n1.number > (select avg(n.number) from numbers(3) n) SETTINGS allow_experimental_analyzer=0;
+WHERE n1.number > (select avg(n.number) from numbers(3) n) SETTINGS enable_analyzer=0;
diff --git a/tests/queries/0_stateless/01508_explain_header.sql b/tests/queries/0_stateless/01508_explain_header.sql
index a9f876068aa..03452e4bdac 100644
--- a/tests/queries/0_stateless/01508_explain_header.sql
+++ b/tests/queries/0_stateless/01508_explain_header.sql
@@ -1,3 +1,3 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 explain header = 1 select 1 as x;
diff --git a/tests/queries/0_stateless/01556_explain_select_with_union_query.sql b/tests/queries/0_stateless/01556_explain_select_with_union_query.sql
index bbd96ef5c69..d8278e1887a 100644
--- a/tests/queries/0_stateless/01556_explain_select_with_union_query.sql
+++ b/tests/queries/0_stateless/01556_explain_select_with_union_query.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 SET union_default_mode = 'DISTINCT';
 
 set enable_global_with_statement = 1;
diff --git a/tests/queries/0_stateless/01561_clickhouse_client_stage.reference b/tests/queries/0_stateless/01561_clickhouse_client_stage.reference
index 2631199cbab..b6200464293 100644
--- a/tests/queries/0_stateless/01561_clickhouse_client_stage.reference
+++ b/tests/queries/0_stateless/01561_clickhouse_client_stage.reference
@@ -1,15 +1,15 @@
-execute: --allow_experimental_analyzer=1
+execute: --enable_analyzer=1
 "foo"
 1
-execute: --allow_experimental_analyzer=1 --stage fetch_columns
+execute: --enable_analyzer=1 --stage fetch_columns
 "__table1.dummy"
 0
-execute: --allow_experimental_analyzer=1 --stage with_mergeable_state
+execute: --enable_analyzer=1 --stage with_mergeable_state
 "1_UInt8"
 1
-execute: --allow_experimental_analyzer=1 --stage with_mergeable_state_after_aggregation
+execute: --enable_analyzer=1 --stage with_mergeable_state_after_aggregation
 "1_UInt8"
 1
-execute: --allow_experimental_analyzer=1 --stage complete
+execute: --enable_analyzer=1 --stage complete
 "foo"
 1
diff --git a/tests/queries/0_stateless/01561_clickhouse_client_stage.sh b/tests/queries/0_stateless/01561_clickhouse_client_stage.sh
index 99267458421..79c9bb6ae10 100755
--- a/tests/queries/0_stateless/01561_clickhouse_client_stage.sh
+++ b/tests/queries/0_stateless/01561_clickhouse_client_stage.sh
@@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CURDIR"/../shell_config.sh
 
 opts=(
-    "--allow_experimental_analyzer=1"
+    "--enable_analyzer=1"
 )
 
 function execute_query()
diff --git a/tests/queries/0_stateless/01591_window_functions.sql b/tests/queries/0_stateless/01591_window_functions.sql
index b821ba13721..db727599d2c 100644
--- a/tests/queries/0_stateless/01591_window_functions.sql
+++ b/tests/queries/0_stateless/01591_window_functions.sql
@@ -1,6 +1,6 @@
 -- Tags: long
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 -- Too slow
 SET max_bytes_before_external_sort = 0;
diff --git a/tests/queries/0_stateless/01600_detach_permanently.sh b/tests/queries/0_stateless/01600_detach_permanently.sh
index 679e9a749ee..be405f8e7d9 100755
--- a/tests/queries/0_stateless/01600_detach_permanently.sh
+++ b/tests/queries/0_stateless/01600_detach_permanently.sh
@@ -111,8 +111,8 @@ clickhouse_local "INSERT INTO db_ordinary.src SELECT * FROM numbers(10)"
 clickhouse_local "SELECT if(count() = 10, 'MV is working', 'MV failed') FROM db_ordinary.src_mv_with_inner"
 
 clickhouse_local "DETACH VIEW db_ordinary.src_mv_with_inner PERMANENTLY; INSERT INTO db_ordinary.src SELECT * FROM numbers(10)" --stacktrace
-clickhouse_local "SELECT if(count() = 10, 'MV can be detached permanently', 'MV detach failed') FROM db_ordinary.src_mv_with_inner SETTINGS allow_experimental_analyzer = 0" 2>&1 | grep -c "db_ordinary.src_mv_with_inner does not exist"
-clickhouse_local "SELECT if(count() = 10, 'MV can be detached permanently', 'MV detach failed') FROM db_ordinary.src_mv_with_inner SETTINGS allow_experimental_analyzer = 1" 2>&1 | grep -c "Unknown table expression identifier 'db_ordinary.src_mv_with_inner'"
+clickhouse_local "SELECT if(count() = 10, 'MV can be detached permanently', 'MV detach failed') FROM db_ordinary.src_mv_with_inner SETTINGS enable_analyzer = 0" 2>&1 | grep -c "db_ordinary.src_mv_with_inner does not exist"
+clickhouse_local "SELECT if(count() = 10, 'MV can be detached permanently', 'MV detach failed') FROM db_ordinary.src_mv_with_inner SETTINGS enable_analyzer = 1" 2>&1 | grep -c "Unknown table expression identifier 'db_ordinary.src_mv_with_inner'"
 
 ## Quite silly: ATTACH MATERIALIZED VIEW don't work with short syntax (w/o select), but i can attach it using ATTACH TABLE ...
 clickhouse_local "ATTACH TABLE db_ordinary.src_mv_with_inner"
diff --git a/tests/queries/0_stateless/01622_constraints_simple_optimization.reference b/tests/queries/0_stateless/01622_constraints_simple_optimization.reference
index 84c872856ff..64a9db37a68 100644
--- a/tests/queries/0_stateless/01622_constraints_simple_optimization.reference
+++ b/tests/queries/0_stateless/01622_constraints_simple_optimization.reference
@@ -52,7 +52,7 @@ QUERY id: 0
         LIST id: 5, nodes: 2
           COLUMN id: 6, column_name: c, result_type: Int64, source_id: 3
           CONSTANT id: 7, constant_value: UInt64_100, constant_value_type: UInt8
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 SELECT count()
 FROM constraint_test_constants
 WHERE c > 100
@@ -70,7 +70,7 @@ QUERY id: 0
         LIST id: 5, nodes: 2
           COLUMN id: 6, column_name: c, result_type: Int64, source_id: 3
           CONSTANT id: 7, constant_value: UInt64_100, constant_value_type: UInt8
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 SELECT count()
 FROM constraint_test_constants
 QUERY id: 0
@@ -81,4 +81,4 @@ QUERY id: 0
       FUNCTION id: 2, function_name: count, function_type: aggregate, result_type: UInt64
   JOIN TREE
     TABLE id: 3, alias: __table1, table_name: default.constraint_test_constants
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
diff --git a/tests/queries/0_stateless/01622_constraints_simple_optimization.sql b/tests/queries/0_stateless/01622_constraints_simple_optimization.sql
index acde02e2c67..e549467de02 100644
--- a/tests/queries/0_stateless/01622_constraints_simple_optimization.sql
+++ b/tests/queries/0_stateless/01622_constraints_simple_optimization.sql
@@ -101,10 +101,10 @@ SELECT count() FROM constraint_test_constants WHERE 11 <= a; ---> assumption ->
 EXPLAIN SYNTAX SELECT count() FROM constraint_test_constants WHERE (a > 100 OR b > 100 OR c > 100) AND (a <= 100 OR b > 100 OR c > 100);
 -- EXPLAIN QUERY TREE SELECT count() FROM constraint_test_constants WHERE (a > 100 OR b > 100 OR c > 100) AND (a <= 100 OR b > 100 OR c > 100); ---> the order of the generated checks is not consistent
 EXPLAIN SYNTAX SELECT count() FROM constraint_test_constants WHERE (a > 100 OR b > 100 OR c > 100) AND (a <= 100 OR b > 100 OR c > 100) AND (NOT b > 100 OR c > 100);
-EXPLAIN QUERY TREE SELECT count() FROM constraint_test_constants WHERE (a > 100 OR b > 100 OR c > 100) AND (a <= 100 OR b > 100 OR c > 100) AND (NOT b > 100 OR c > 100) SETTINGS allow_experimental_analyzer = 1;
+EXPLAIN QUERY TREE SELECT count() FROM constraint_test_constants WHERE (a > 100 OR b > 100 OR c > 100) AND (a <= 100 OR b > 100 OR c > 100) AND (NOT b > 100 OR c > 100) SETTINGS enable_analyzer = 1;
 EXPLAIN SYNTAX SELECT count() FROM constraint_test_constants WHERE (a > 100 OR b > 100 OR c > 100) AND (a <= 100 OR b > 100 OR c > 100) AND (NOT b > 100 OR c > 100) AND (c > 100);
-EXPLAIN QUERY TREE SELECT count() FROM constraint_test_constants WHERE (a > 100 OR b > 100 OR c > 100) AND (a <= 100 OR b > 100 OR c > 100) AND (NOT b > 100 OR c > 100) AND (c > 100) SETTINGS allow_experimental_analyzer = 1;
+EXPLAIN QUERY TREE SELECT count() FROM constraint_test_constants WHERE (a > 100 OR b > 100 OR c > 100) AND (a <= 100 OR b > 100 OR c > 100) AND (NOT b > 100 OR c > 100) AND (c > 100) SETTINGS enable_analyzer = 1;
 EXPLAIN SYNTAX SELECT count() FROM constraint_test_constants WHERE (a > 100 OR b > 100 OR c > 100) AND (a <= 100 OR b > 100 OR c > 100) AND (NOT b > 100 OR c > 100) AND (c <= 100);
-EXPLAIN QUERY TREE SELECT count() FROM constraint_test_constants WHERE (a > 100 OR b > 100 OR c > 100) AND (a <= 100 OR b > 100 OR c > 100) AND (NOT b > 100 OR c > 100) AND (c <= 100) SETTINGS allow_experimental_analyzer = 1;
+EXPLAIN QUERY TREE SELECT count() FROM constraint_test_constants WHERE (a > 100 OR b > 100 OR c > 100) AND (a <= 100 OR b > 100 OR c > 100) AND (NOT b > 100 OR c > 100) AND (c <= 100) SETTINGS enable_analyzer = 1;
 
 DROP TABLE constraint_test_constants;
diff --git a/tests/queries/0_stateless/01622_constraints_where_optimization.reference b/tests/queries/0_stateless/01622_constraints_where_optimization.reference
index 3f6e8211f1a..09a6dd3d0e7 100644
--- a/tests/queries/0_stateless/01622_constraints_where_optimization.reference
+++ b/tests/queries/0_stateless/01622_constraints_where_optimization.reference
@@ -11,7 +11,7 @@ QUERY id: 0
     TABLE id: 3, alias: __table1, table_name: default.t_constraints_where
   WHERE
     CONSTANT id: 4, constant_value: UInt64_0, constant_value_type: UInt8
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 SELECT count()
 FROM t_constraints_where
 WHERE 0
@@ -25,7 +25,7 @@ QUERY id: 0
     TABLE id: 3, alias: __table1, table_name: default.t_constraints_where
   WHERE
     CONSTANT id: 4, constant_value: UInt64_0, constant_value_type: UInt8
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 SELECT count()
 FROM t_constraints_where
 WHERE 0
@@ -39,7 +39,7 @@ QUERY id: 0
     TABLE id: 3, alias: __table1, table_name: default.t_constraints_where
   WHERE
     CONSTANT id: 4, constant_value: UInt64_0, constant_value_type: UInt8
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 SELECT count()
 FROM t_constraints_where
 WHERE b < 8
@@ -57,7 +57,7 @@ QUERY id: 0
         LIST id: 5, nodes: 2
           COLUMN id: 6, column_name: b, result_type: UInt32, source_id: 3
           CONSTANT id: 7, constant_value: UInt64_8, constant_value_type: UInt8
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 SELECT count()
 FROM t_constraints_where
 PREWHERE (b > 20) OR (b < 8)
@@ -75,7 +75,7 @@ QUERY id: 0
         LIST id: 5, nodes: 2
           COLUMN id: 6, column_name: b, result_type: UInt32, source_id: 3
           CONSTANT id: 7, constant_value: UInt64_8, constant_value_type: UInt8
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 SELECT count()
 FROM t_constraints_where
 QUERY id: 0
@@ -86,4 +86,4 @@ QUERY id: 0
       FUNCTION id: 2, function_name: count, function_type: aggregate, result_type: UInt64
   JOIN TREE
     TABLE id: 3, alias: __table1, table_name: default.t_constraints_where
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
diff --git a/tests/queries/0_stateless/01622_constraints_where_optimization.sql b/tests/queries/0_stateless/01622_constraints_where_optimization.sql
index d41b1988bdd..63803ec8ce6 100644
--- a/tests/queries/0_stateless/01622_constraints_where_optimization.sql
+++ b/tests/queries/0_stateless/01622_constraints_where_optimization.sql
@@ -9,15 +9,15 @@ CREATE TABLE t_constraints_where(a UInt32, b UInt32, CONSTRAINT c1 ASSUME b >= 5
 INSERT INTO t_constraints_where VALUES (1, 7);
 
 EXPLAIN SYNTAX SELECT count() FROM t_constraints_where WHERE b > 15; -- assumption -> 0
-EXPLAIN QUERY TREE SELECT count() FROM t_constraints_where WHERE b > 15 SETTINGS allow_experimental_analyzer = 1; -- assumption -> 0
+EXPLAIN QUERY TREE SELECT count() FROM t_constraints_where WHERE b > 15 SETTINGS enable_analyzer = 1; -- assumption -> 0
 EXPLAIN SYNTAX SELECT count() FROM t_constraints_where WHERE b = 20; -- assumption -> 0
-EXPLAIN QUERY TREE SELECT count() FROM t_constraints_where WHERE b = 20 SETTINGS allow_experimental_analyzer = 1; -- assumption -> 0
+EXPLAIN QUERY TREE SELECT count() FROM t_constraints_where WHERE b = 20 SETTINGS enable_analyzer = 1; -- assumption -> 0
 EXPLAIN SYNTAX SELECT count() FROM t_constraints_where WHERE b < 2; -- assumption -> 0
-EXPLAIN QUERY TREE SELECT count() FROM t_constraints_where WHERE b < 2 SETTINGS allow_experimental_analyzer = 1; -- assumption -> 0
+EXPLAIN QUERY TREE SELECT count() FROM t_constraints_where WHERE b < 2 SETTINGS enable_analyzer = 1; -- assumption -> 0
 EXPLAIN SYNTAX SELECT count() FROM t_constraints_where WHERE b > 20 OR b < 8; -- assumption -> remove (b < 20)
-EXPLAIN QUERY TREE SELECT count() FROM t_constraints_where WHERE b > 20 OR b < 8 SETTINGS allow_experimental_analyzer = 1; -- assumption -> remove (b < 20)
+EXPLAIN QUERY TREE SELECT count() FROM t_constraints_where WHERE b > 20 OR b < 8 SETTINGS enable_analyzer = 1; -- assumption -> remove (b < 20)
 EXPLAIN SYNTAX SELECT count() FROM t_constraints_where PREWHERE b > 20 OR b < 8; -- assumption -> remove (b < 20)
-EXPLAIN QUERY TREE SELECT count() FROM t_constraints_where PREWHERE b > 20 OR b < 8 SETTINGS allow_experimental_analyzer = 1; -- assumption -> remove (b < 20)
+EXPLAIN QUERY TREE SELECT count() FROM t_constraints_where PREWHERE b > 20 OR b < 8 SETTINGS enable_analyzer = 1; -- assumption -> remove (b < 20)
 
 DROP TABLE t_constraints_where;
 
@@ -26,6 +26,6 @@ CREATE TABLE t_constraints_where(a UInt32, b UInt32, CONSTRAINT c1 ASSUME b < 10
 INSERT INTO t_constraints_where VALUES (1, 7);
 
 EXPLAIN SYNTAX SELECT count() FROM t_constraints_where WHERE b = 1 OR b < 18 OR b > 5; -- assumption -> (b < 20) -> 0;
-EXPLAIN QUERY TREE SELECT count() FROM t_constraints_where WHERE b = 1 OR b < 18 OR b > 5 SETTINGS allow_experimental_analyzer = 1; -- assumption -> (b < 20) -> 0;
+EXPLAIN QUERY TREE SELECT count() FROM t_constraints_where WHERE b = 1 OR b < 18 OR b > 5 SETTINGS enable_analyzer = 1; -- assumption -> (b < 20) -> 0;
 
 DROP TABLE t_constraints_where;
diff --git a/tests/queries/0_stateless/01623_constraints_column_swap.reference b/tests/queries/0_stateless/01623_constraints_column_swap.reference
index d504a86365b..b49884b4798 100644
--- a/tests/queries/0_stateless/01623_constraints_column_swap.reference
+++ b/tests/queries/0_stateless/01623_constraints_column_swap.reference
@@ -27,7 +27,7 @@ QUERY id: 0
         LIST id: 12, nodes: 2
           COLUMN id: 13, column_name: b, result_type: UInt64, source_id: 5
           CONSTANT id: 14, constant_value: UInt64_1, constant_value_type: UInt8
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 SELECT
     cityHash64(a) + 10,
     b + 3
@@ -57,7 +57,7 @@ QUERY id: 0
         LIST id: 12, nodes: 2
           COLUMN id: 13, column_name: b, result_type: UInt64, source_id: 5
           CONSTANT id: 14, constant_value: UInt64_1, constant_value_type: UInt8
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 SELECT
     (b AS `cityHash64(a)`) + 10,
     (b AS b) + 3
@@ -87,7 +87,7 @@ QUERY id: 0
         LIST id: 12, nodes: 2
           COLUMN id: 13, column_name: b, result_type: UInt64, source_id: 5
           CONSTANT id: 14, constant_value: UInt64_0, constant_value_type: UInt8
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 SELECT
     (b AS `cityHash64(a)`) + 10,
     (b AS b) + 3
@@ -117,7 +117,7 @@ QUERY id: 0
         LIST id: 12, nodes: 2
           COLUMN id: 13, column_name: b, result_type: UInt64, source_id: 5
           CONSTANT id: 14, constant_value: UInt64_0, constant_value_type: UInt8
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 SELECT
     (b AS `cityHash64(a)`) + 10,
     (b AS b) + 3
@@ -147,7 +147,7 @@ QUERY id: 0
         LIST id: 12, nodes: 2
           COLUMN id: 13, column_name: b, result_type: UInt64, source_id: 5
           CONSTANT id: 14, constant_value: UInt64_1, constant_value_type: UInt8
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 SELECT (b AS `cityHash64(a)`) + 10
 FROM column_swap_test_test
 WHERE b = 0
@@ -169,7 +169,7 @@ QUERY id: 0
         LIST id: 8, nodes: 2
           COLUMN id: 9, column_name: b, result_type: UInt64, source_id: 5
           CONSTANT id: 10, constant_value: UInt64_0, constant_value_type: UInt8
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 SELECT
     (cityHash64(a) AS `cityHash64(a)`) + 10,
     a
@@ -201,7 +201,7 @@ QUERY id: 0
               LIST id: 13, nodes: 1
                 COLUMN id: 14, column_name: a, result_type: String, source_id: 7
           CONSTANT id: 15, constant_value: UInt64_0, constant_value_type: UInt8
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 SELECT
     (cityHash64(a) AS b) + 10,
     a
@@ -233,7 +233,7 @@ QUERY id: 0
               LIST id: 13, nodes: 1
                 COLUMN id: 14, column_name: a, result_type: String, source_id: 7
           CONSTANT id: 15, constant_value: UInt64_0, constant_value_type: UInt8
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 SELECT
     a AS `substring(reverse(b), 1, 1)`,
     a AS a
@@ -255,7 +255,7 @@ QUERY id: 0
         LIST id: 6, nodes: 2
           COLUMN id: 7, column_name: a, result_type: String, source_id: 3
           CONSTANT id: 8, constant_value: \'c\', constant_value_type: String
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 SELECT
     a AS `substring(reverse(b), 1, 1)`,
     a AS a
@@ -277,7 +277,7 @@ QUERY id: 0
         LIST id: 6, nodes: 2
           COLUMN id: 7, column_name: a, result_type: String, source_id: 3
           CONSTANT id: 8, constant_value: \'c\', constant_value_type: String
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 SELECT
     a AS t1,
     a AS t2
@@ -299,7 +299,7 @@ QUERY id: 0
         LIST id: 6, nodes: 2
           COLUMN id: 7, column_name: a, result_type: String, source_id: 3
           CONSTANT id: 8, constant_value: \'c\', constant_value_type: String
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 SELECT a AS `substring(reverse(b), 1, 1)`
 FROM column_swap_test_test
 WHERE a = \'c\'
@@ -317,7 +317,7 @@ QUERY id: 0
         LIST id: 5, nodes: 2
           COLUMN id: 6, column_name: a, result_type: String, source_id: 3
           CONSTANT id: 7, constant_value: \'c\', constant_value_type: String
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 SELECT a
 FROM t_bad_constraint
 QUERY id: 0
@@ -328,4 +328,4 @@ QUERY id: 0
       COLUMN id: 2, column_name: a, result_type: UInt32, source_id: 3
   JOIN TREE
     TABLE id: 3, alias: __table1, table_name: default.t_bad_constraint
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
diff --git a/tests/queries/0_stateless/01623_constraints_column_swap.sql b/tests/queries/0_stateless/01623_constraints_column_swap.sql
index 242be87938d..ccd387c9f8d 100644
--- a/tests/queries/0_stateless/01623_constraints_column_swap.sql
+++ b/tests/queries/0_stateless/01623_constraints_column_swap.sql
@@ -14,22 +14,22 @@ INSERT INTO column_swap_test_test VALUES (1, 'cat', 1), (2, 'dog', 2);
 INSERT INTO column_swap_test_test SELECT number AS i, format('test {} kek {}', toString(number), toString(number + 10))  AS a, 1 AS b FROM system.numbers LIMIT 1000000;
 
 EXPLAIN SYNTAX SELECT cityHash64(a) + 10, b + 3 FROM column_swap_test_test WHERE cityHash64(a) = 1;
-EXPLAIN QUERY TREE SELECT cityHash64(a) + 10, b + 3 FROM column_swap_test_test WHERE cityHash64(a) = 1 SETTINGS allow_experimental_analyzer = 1;
+EXPLAIN QUERY TREE SELECT cityHash64(a) + 10, b + 3 FROM column_swap_test_test WHERE cityHash64(a) = 1 SETTINGS enable_analyzer = 1;
 EXPLAIN SYNTAX SELECT cityHash64(a) + 10, b + 3 FROM column_swap_test_test PREWHERE cityHash64(a) = 1;
-EXPLAIN QUERY TREE SELECT cityHash64(a) + 10, b + 3 FROM column_swap_test_test PREWHERE cityHash64(a) = 1 SETTINGS allow_experimental_analyzer = 1;
+EXPLAIN QUERY TREE SELECT cityHash64(a) + 10, b + 3 FROM column_swap_test_test PREWHERE cityHash64(a) = 1 SETTINGS enable_analyzer = 1;
 EXPLAIN SYNTAX SELECT cityHash64(a) + 10, b + 3 FROM column_swap_test_test WHERE cityHash64(a) = 0;
-EXPLAIN QUERY TREE SELECT cityHash64(a) + 10, b + 3 FROM column_swap_test_test WHERE cityHash64(a) = 0 SETTINGS allow_experimental_analyzer = 1;
+EXPLAIN QUERY TREE SELECT cityHash64(a) + 10, b + 3 FROM column_swap_test_test WHERE cityHash64(a) = 0 SETTINGS enable_analyzer = 1;
 EXPLAIN SYNTAX SELECT cityHash64(a) + 10, b + 3 FROM column_swap_test_test WHERE b = 0;
-EXPLAIN QUERY TREE SELECT cityHash64(a) + 10, b + 3 FROM column_swap_test_test WHERE b = 0 SETTINGS allow_experimental_analyzer = 1;
+EXPLAIN QUERY TREE SELECT cityHash64(a) + 10, b + 3 FROM column_swap_test_test WHERE b = 0 SETTINGS enable_analyzer = 1;
 EXPLAIN SYNTAX SELECT cityHash64(a) + 10, b + 3 FROM column_swap_test_test WHERE b = 1;
-EXPLAIN QUERY TREE SELECT cityHash64(a) + 10, b + 3 FROM column_swap_test_test WHERE b = 1 SETTINGS allow_experimental_analyzer = 1;
+EXPLAIN QUERY TREE SELECT cityHash64(a) + 10, b + 3 FROM column_swap_test_test WHERE b = 1 SETTINGS enable_analyzer = 1;
 
 EXPLAIN SYNTAX SELECT cityHash64(a) + 10 FROM column_swap_test_test WHERE cityHash64(a) = 0;
-EXPLAIN QUERY TREE SELECT cityHash64(a) + 10 FROM column_swap_test_test WHERE cityHash64(a) = 0 SETTINGS allow_experimental_analyzer = 1;
+EXPLAIN QUERY TREE SELECT cityHash64(a) + 10 FROM column_swap_test_test WHERE cityHash64(a) = 0 SETTINGS enable_analyzer = 1;
 EXPLAIN SYNTAX SELECT cityHash64(a) + 10, a FROM column_swap_test_test WHERE cityHash64(a) = 0;
-EXPLAIN QUERY TREE SELECT cityHash64(a) + 10, a FROM column_swap_test_test WHERE cityHash64(a) = 0 SETTINGS allow_experimental_analyzer = 1;
+EXPLAIN QUERY TREE SELECT cityHash64(a) + 10, a FROM column_swap_test_test WHERE cityHash64(a) = 0 SETTINGS enable_analyzer = 1;
 EXPLAIN SYNTAX SELECT b + 10, a FROM column_swap_test_test WHERE b = 0;
-EXPLAIN QUERY TREE SELECT b + 10, a FROM column_swap_test_test WHERE b = 0 SETTINGS allow_experimental_analyzer = 1;
+EXPLAIN QUERY TREE SELECT b + 10, a FROM column_swap_test_test WHERE b = 0 SETTINGS enable_analyzer = 1;
 
 DROP TABLE column_swap_test_test;
 
@@ -37,13 +37,13 @@ CREATE TABLE column_swap_test_test (i Int64, a String, b String, CONSTRAINT c1 A
 INSERT INTO column_swap_test_test SELECT number AS i, toString(number) AS a, format('test {} kek {}', toString(number), toString(number + 10)) b FROM system.numbers LIMIT 1000000;
 
 EXPLAIN SYNTAX SELECT substring(reverse(b), 1, 1), a FROM column_swap_test_test WHERE a = 'c';
-EXPLAIN QUERY TREE SELECT substring(reverse(b), 1, 1), a FROM column_swap_test_test WHERE a = 'c' SETTINGS allow_experimental_analyzer = 1;
+EXPLAIN QUERY TREE SELECT substring(reverse(b), 1, 1), a FROM column_swap_test_test WHERE a = 'c' SETTINGS enable_analyzer = 1;
 EXPLAIN SYNTAX SELECT substring(reverse(b), 1, 1), a FROM column_swap_test_test WHERE substring(reverse(b), 1, 1) = 'c';
-EXPLAIN QUERY TREE SELECT substring(reverse(b), 1, 1), a FROM column_swap_test_test WHERE substring(reverse(b), 1, 1) = 'c' SETTINGS allow_experimental_analyzer = 1;
+EXPLAIN QUERY TREE SELECT substring(reverse(b), 1, 1), a FROM column_swap_test_test WHERE substring(reverse(b), 1, 1) = 'c' SETTINGS enable_analyzer = 1;
 EXPLAIN SYNTAX SELECT substring(reverse(b), 1, 1) AS t1, a AS t2 FROM column_swap_test_test WHERE substring(reverse(b), 1, 1) = 'c';
-EXPLAIN QUERY TREE SELECT substring(reverse(b), 1, 1) AS t1, a AS t2 FROM column_swap_test_test WHERE substring(reverse(b), 1, 1) = 'c' SETTINGS allow_experimental_analyzer = 1;
+EXPLAIN QUERY TREE SELECT substring(reverse(b), 1, 1) AS t1, a AS t2 FROM column_swap_test_test WHERE substring(reverse(b), 1, 1) = 'c' SETTINGS enable_analyzer = 1;
 EXPLAIN SYNTAX SELECT substring(reverse(b), 1, 1) FROM column_swap_test_test WHERE substring(reverse(b), 1, 1) = 'c';
-EXPLAIN QUERY TREE SELECT substring(reverse(b), 1, 1) FROM column_swap_test_test WHERE substring(reverse(b), 1, 1) = 'c' SETTINGS allow_experimental_analyzer = 1;
+EXPLAIN QUERY TREE SELECT substring(reverse(b), 1, 1) FROM column_swap_test_test WHERE substring(reverse(b), 1, 1) = 'c' SETTINGS enable_analyzer = 1;
 
 DROP TABLE column_swap_test_test;
 
@@ -54,6 +54,6 @@ CREATE TABLE t_bad_constraint(a UInt32, s String, CONSTRAINT c1 ASSUME a = toUIn
 INSERT INTO t_bad_constraint SELECT number, randomPrintableASCII(100) FROM numbers(10000);
 
 EXPLAIN SYNTAX SELECT a FROM t_bad_constraint;
-EXPLAIN QUERY TREE SELECT a FROM t_bad_constraint SETTINGS allow_experimental_analyzer = 1;
+EXPLAIN QUERY TREE SELECT a FROM t_bad_constraint SETTINGS enable_analyzer = 1;
 
 DROP TABLE t_bad_constraint;
diff --git a/tests/queries/0_stateless/01646_rewrite_sum_if.sql b/tests/queries/0_stateless/01646_rewrite_sum_if.sql
index da341a3f7db..fd98a157ac6 100644
--- a/tests/queries/0_stateless/01646_rewrite_sum_if.sql
+++ b/tests/queries/0_stateless/01646_rewrite_sum_if.sql
@@ -34,7 +34,7 @@ SELECT sum(if(number % 2 == 0, 0, 1)) FROM numbers(100);
 SELECT sum(if(number % 2 == 0 as cond_expr, 0 as zero_expr, 1 as one_expr) as if_expr), sum(cond_expr), sum(if_expr), one_expr, zero_expr FROM numbers(100);
 SELECT countIf(number % 2 != 0) FROM numbers(100);
 
-set allow_experimental_analyzer = true;
+set enable_analyzer = true;
 
 EXPLAIN QUERY TREE run_passes=1 SELECT sumIf(123, number % 2 == 0) FROM numbers(100);
 EXPLAIN QUERY TREE run_passes=1 SELECT sum(if(number % 2 == 0, 123, 0)) FROM numbers(100);
diff --git a/tests/queries/0_stateless/01651_bugs_from_15889.sql b/tests/queries/0_stateless/01651_bugs_from_15889.sql
index dd31f2941ef..b98feaa1f65 100644
--- a/tests/queries/0_stateless/01651_bugs_from_15889.sql
+++ b/tests/queries/0_stateless/01651_bugs_from_15889.sql
@@ -111,5 +111,5 @@ WITH (
     ) AS t)
 SELECT if(dateDiff('second', toDateTime(time_with_microseconds), toDateTime(t)) = -9223372036854775808, 'ok', '');
 
-set joined_subquery_requires_alias=0, allow_experimental_analyzer=0; -- the query is invalid with a new analyzer
+set joined_subquery_requires_alias=0, enable_analyzer=0; -- the query is invalid with a new analyzer
 SELECT number, number / 2 AS n, j1, j2 FROM remote('127.0.0.{2,3}', system.numbers) GLOBAL ANY LEFT JOIN (SELECT number / 3 AS n, number AS j1, 'Hello' AS j2 FROM system.numbers LIMIT 1048577) USING (n) LIMIT 10 format Null;
diff --git a/tests/queries/0_stateless/01655_plan_optimizations.sh b/tests/queries/0_stateless/01655_plan_optimizations.sh
index 4bd0eb7d908..42cdac8c01f 100755
--- a/tests/queries/0_stateless/01655_plan_optimizations.sh
+++ b/tests/queries/0_stateless/01655_plan_optimizations.sh
@@ -26,13 +26,13 @@ $CLICKHOUSE_CLIENT -q "
     settings enable_optimize_predicate_expression=0"
 
 echo "> filter should be pushed down after aggregating, column after aggregation is const"
-$CLICKHOUSE_CLIENT --allow_experimental_analyzer=0 -q "
+$CLICKHOUSE_CLIENT --enable_analyzer=0 -q "
     explain actions = 1 select s, y, y != 0 from (select sum(x) as s, y from (
         select number as x, number + 1 as y from numbers(10)) group by y
     ) where y != 0
     settings enable_optimize_predicate_expression=0" | grep -o "Aggregating\|Filter\|COLUMN Const(UInt8) -> notEquals(y, 0)"
 echo "> (analyzer) filter should be pushed down after aggregating, column after aggregation is const"
-$CLICKHOUSE_CLIENT --allow_experimental_analyzer=1 -q "
+$CLICKHOUSE_CLIENT --enable_analyzer=1 -q "
     explain actions = 1 select s, y, y != 0 from (select sum(x) as s, y from (
         select number as x, number + 1 as y from numbers(10)) group by y
     ) where y != 0
@@ -44,14 +44,14 @@ $CLICKHOUSE_CLIENT -q "
     settings enable_optimize_predicate_expression=0"
 
 echo "> one condition of filter should be pushed down after aggregating, other condition is aliased"
-$CLICKHOUSE_CLIENT --allow_experimental_analyzer=0 -q "
+$CLICKHOUSE_CLIENT --enable_analyzer=0 -q "
     explain actions = 1 select s, y from (
         select sum(x) as s, y from (select number as x, number + 1 as y from numbers(10)) group by y
     ) where y != 0 and s != 4
     settings enable_optimize_predicate_expression=0" |
     grep -o "Aggregating\|Filter column\|Filter column: notEquals(y, 0)\|ALIAS notEquals(s, 4) :: 4 -> and(notEquals(y, 0), notEquals(s, 4)) UInt8 : 2"
 echo "> (analyzer) one condition of filter should be pushed down after aggregating, other condition is aliased"
-$CLICKHOUSE_CLIENT --allow_experimental_analyzer=1 -q "
+$CLICKHOUSE_CLIENT --enable_analyzer=1 -q "
     explain actions = 1 select s, y from (
         select sum(x) as s, y from (select number as x, number + 1 as y from numbers(10)) group by y
     ) where y != 0 and s != 4
@@ -64,14 +64,14 @@ $CLICKHOUSE_CLIENT -q "
     settings enable_optimize_predicate_expression=0"
 
 echo "> one condition of filter should be pushed down after aggregating, other condition is casted"
-$CLICKHOUSE_CLIENT --allow_experimental_analyzer=0 -q "
+$CLICKHOUSE_CLIENT --enable_analyzer=0 -q "
     explain actions = 1 select s, y from (
         select sum(x) as s, y from (select number as x, number + 1 as y from numbers(10)) group by y
     ) where y != 0 and s - 4
     settings enable_optimize_predicate_expression=0" |
     grep -o "Aggregating\|Filter column\|Filter column: notEquals(y, 0)\|FUNCTION and(minus(s, 4) :: 5, 1 :: 3) -> and(notEquals(y, 0), minus(s, 4))"
 echo "> (analyzer) one condition of filter should be pushed down after aggregating, other condition is casted"
-$CLICKHOUSE_CLIENT --allow_experimental_analyzer=1 -q "
+$CLICKHOUSE_CLIENT --enable_analyzer=1 -q "
     explain actions = 1 select s, y from (
         select sum(x) as s, y from (select number as x, number + 1 as y from numbers(10)) group by y
     ) where y != 0 and s - 4
@@ -84,14 +84,14 @@ $CLICKHOUSE_CLIENT -q "
     settings enable_optimize_predicate_expression=0"
 
 echo "> one condition of filter should be pushed down after aggregating, other two conditions are ANDed"
-$CLICKHOUSE_CLIENT --allow_experimental_analyzer=0 --convert_query_to_cnf=0 -q "
+$CLICKHOUSE_CLIENT --enable_analyzer=0 --convert_query_to_cnf=0 -q "
     explain actions = 1 select s, y from (
         select sum(x) as s, y from (select number as x, number + 1 as y from numbers(10)) group by y
     ) where y != 0 and s - 8 and s - 4
     settings enable_optimize_predicate_expression=0" |
     grep -o "Aggregating\|Filter column\|Filter column: notEquals(y, 0)\|FUNCTION and(minus(s, 8) :: 5, minus(s, 4) :: 2) -> and(notEquals(y, 0), minus(s, 8), minus(s, 4))"
 echo "> (analyzer) one condition of filter should be pushed down after aggregating, other two conditions are ANDed"
-$CLICKHOUSE_CLIENT --allow_experimental_analyzer=1 --convert_query_to_cnf=0 -q "
+$CLICKHOUSE_CLIENT --enable_analyzer=1 --convert_query_to_cnf=0 -q "
     explain actions = 1 select s, y from (
         select sum(x) as s, y from (select number as x, number + 1 as y from numbers(10)) group by y
     ) where y != 0 and s - 8 and s - 4
@@ -104,14 +104,14 @@ $CLICKHOUSE_CLIENT -q "
     settings enable_optimize_predicate_expression=0"
 
 echo "> two conditions of filter should be pushed down after aggregating and ANDed, one condition is aliased"
-$CLICKHOUSE_CLIENT --allow_experimental_analyzer=0 --convert_query_to_cnf=0 -q "
+$CLICKHOUSE_CLIENT --enable_analyzer=0 --convert_query_to_cnf=0 -q "
     explain actions = 1 select s, y from (
         select sum(x) as s, y from (select number as x, number + 1 as y from numbers(10)) group by y
     ) where y != 0 and s != 8 and y - 4
     settings enable_optimize_predicate_expression=0" |
     grep -o "Aggregating\|Filter column\|Filter column: and(notEquals(y, 0), minus(y, 4))\|ALIAS notEquals(s, 8) :: 4 -> and(notEquals(y, 0), notEquals(s, 8), minus(y, 4))"
 echo "> (analyzer) two conditions of filter should be pushed down after aggregating and ANDed, one condition is aliased"
-$CLICKHOUSE_CLIENT --allow_experimental_analyzer=1 --convert_query_to_cnf=0 -q "
+$CLICKHOUSE_CLIENT --enable_analyzer=1 --convert_query_to_cnf=0 -q "
     explain actions = 1 select s, y from (
         select sum(x) as s, y from (select number as x, number + 1 as y from numbers(10)) group by y
     ) where y != 0 and s != 8 and y - 4
@@ -124,13 +124,13 @@ $CLICKHOUSE_CLIENT -q "
     settings enable_optimize_predicate_expression=0"
 
 echo "> filter is split, one part is filtered before ARRAY JOIN"
-$CLICKHOUSE_CLIENT --allow_experimental_analyzer=0  -q "
+$CLICKHOUSE_CLIENT --enable_analyzer=0  -q "
     explain actions = 1 select x, y from (
         select range(number) as x, number + 1 as y from numbers(3)
     ) array join x where y != 2 and x != 0" |
     grep -o "Filter column: and(notEquals(y, 2), notEquals(x, 0))\|ARRAY JOIN x\|Filter column: notEquals(y, 2)"
 echo "> (analyzer) filter is split, one part is filtered before ARRAY JOIN"
-$CLICKHOUSE_CLIENT --allow_experimental_analyzer=1 -q "
+$CLICKHOUSE_CLIENT --enable_analyzer=1 -q "
     explain actions = 1 select x, y from (
         select range(number) as x, number + 1 as y from numbers(3)
     ) array join x where y != 2 and x != 0" |
@@ -154,14 +154,14 @@ $CLICKHOUSE_CLIENT -q "
 #     settings enable_optimize_predicate_expression=0"
 
 echo "> filter is pushed down before Distinct"
-$CLICKHOUSE_CLIENT --allow_experimental_analyzer=0 -q "
+$CLICKHOUSE_CLIENT --enable_analyzer=0 -q "
     explain actions = 1 select x, y from (
         select distinct x, y from (select number % 2 as x, number % 3 as y from numbers(10))
     ) where y != 2
     settings enable_optimize_predicate_expression=0" |
     grep -o "Distinct\|Filter column: notEquals(y, 2)"
 echo "> (analyzer) filter is pushed down before Distinct"
-$CLICKHOUSE_CLIENT --allow_experimental_analyzer=1 -q "
+$CLICKHOUSE_CLIENT --enable_analyzer=1 -q "
     explain actions = 1 select x, y from (
         select distinct x, y from (select number % 2 as x, number % 3 as y from numbers(10))
     ) where y != 2
@@ -174,14 +174,14 @@ $CLICKHOUSE_CLIENT -q "
     settings enable_optimize_predicate_expression=0"
 
 echo "> filter is pushed down before sorting steps"
-$CLICKHOUSE_CLIENT --allow_experimental_analyzer=0 --convert_query_to_cnf=0 -q "
+$CLICKHOUSE_CLIENT --enable_analyzer=0 --convert_query_to_cnf=0 -q "
     explain actions = 1 select x, y from (
         select number % 2 as x, number % 3 as y from numbers(6) order by y desc
     ) where x != 0 and y != 0
     settings enable_optimize_predicate_expression = 0" |
     grep -o "Sorting\|Filter column: and(notEquals(x, 0), notEquals(y, 0))"
 echo "> (analyzer) filter is pushed down before sorting steps"
-$CLICKHOUSE_CLIENT --allow_experimental_analyzer=1 --convert_query_to_cnf=0 -q "
+$CLICKHOUSE_CLIENT --enable_analyzer=1 --convert_query_to_cnf=0 -q "
     explain actions = 1 select x, y from (
         select number % 2 as x, number % 3 as y from numbers(6) order by y desc
     ) where x != 0 and y != 0
@@ -194,14 +194,14 @@ $CLICKHOUSE_CLIENT -q "
     settings enable_optimize_predicate_expression = 0"
 
 echo "> filter is pushed down before TOTALS HAVING and aggregating"
-$CLICKHOUSE_CLIENT --allow_experimental_analyzer=0 -q "
+$CLICKHOUSE_CLIENT --enable_analyzer=0 -q "
     explain actions = 1 select * from (
         select y, sum(x) from (select number as x, number % 4 as y from numbers(10)) group by y with totals
     ) where y != 2
     settings enable_optimize_predicate_expression=0" |
     grep -o "TotalsHaving\|Aggregating\|Filter column: notEquals(y, 2)"
 echo "> (analyzer) filter is pushed down before TOTALS HAVING and aggregating"
-$CLICKHOUSE_CLIENT --allow_experimental_analyzer=1 -q "
+$CLICKHOUSE_CLIENT --enable_analyzer=1 -q "
     explain actions = 1 select * from (
         select y, sum(x) from (select number as x, number % 4 as y from numbers(10)) group by y with totals
     ) where y != 2
@@ -224,14 +224,14 @@ $CLICKHOUSE_CLIENT -q "
     ) where number != 2 settings enable_optimize_predicate_expression=0"
 
 echo "> one condition of filter is pushed down before LEFT JOIN"
-$CLICKHOUSE_CLIENT --allow_experimental_analyzer=0 -q "
+$CLICKHOUSE_CLIENT --enable_analyzer=0 -q "
     explain actions = 1
     select number as a, r.b from numbers(4) as l any left join (
         select number + 2 as b from numbers(3)
     ) as r on a = r.b where a != 1 and b != 2 settings enable_optimize_predicate_expression = 0" |
     grep -o "Join\|Filter column: notEquals(number, 1)"
 echo "> (analyzer) one condition of filter is pushed down before LEFT JOIN"
-$CLICKHOUSE_CLIENT --allow_experimental_analyzer=1 -q "
+$CLICKHOUSE_CLIENT --enable_analyzer=1 -q "
     explain actions = 1
     select number as a, r.b from numbers(4) as l any left join (
         select number + 2 as b from numbers(3)
@@ -243,14 +243,14 @@ $CLICKHOUSE_CLIENT -q "
     ) as r on a = r.b where a != 1 and b != 2 settings enable_optimize_predicate_expression = 0" | sort
 
 echo "> one condition of filter is pushed down before INNER JOIN"
-$CLICKHOUSE_CLIENT --allow_experimental_analyzer=0 -q "
+$CLICKHOUSE_CLIENT --enable_analyzer=0 -q "
     explain actions = 1
     select number as a, r.b from numbers(4) as l any inner join (
         select number + 2 as b from numbers(3)
     ) as r on a = r.b where a != 1 and b != 2 settings enable_optimize_predicate_expression = 0" |
     grep -o "Join\|Filter column: and(notEquals(number, 1), notEquals(number, 2))\|Filter column: and(notEquals(b, 2), notEquals(b, 1))"
 echo "> (analyzer) one condition of filter is pushed down before INNER JOIN"
-$CLICKHOUSE_CLIENT --allow_experimental_analyzer=1 -q "
+$CLICKHOUSE_CLIENT --enable_analyzer=1 -q "
     explain actions = 1
     select number as a, r.b from numbers(4) as l any inner join (
         select number + 2 as b from numbers(3)
@@ -274,12 +274,12 @@ $CLICKHOUSE_CLIENT -q "
 
 echo "> function calculation should be done after sorting and limit (if possible)"
 echo "> Expression should be divided into two subexpressions and only one of them should be moved after Sorting"
-$CLICKHOUSE_CLIENT --allow_experimental_analyzer=0 -q "
+$CLICKHOUSE_CLIENT --enable_analyzer=0 -q "
     explain actions = 1 select number as n, sipHash64(n) from numbers(100) order by number + 1 limit 5" |
     sed 's/^ *//g' | grep -o "^ *\(Expression (.*Before ORDER BY.*)\|Sorting\|FUNCTION \w\+\)"
 echo "> (analyzer) function calculation should be done after sorting and limit (if possible)"
 echo "> Expression should be divided into two subexpressions and only one of them should be moved after Sorting"
-$CLICKHOUSE_CLIENT --allow_experimental_analyzer=1 -q "
+$CLICKHOUSE_CLIENT --enable_analyzer=1 -q "
     explain actions = 1 select number as n, sipHash64(n) from numbers(100) order by number + 1 limit 5" |
     sed 's/^ *//g' | grep -o "^ *\(Expression (.*Before ORDER BY.*)\|Sorting\|FUNCTION \w\+\)"
 echo "> this query should be executed without throwing an exception"
diff --git a/tests/queries/0_stateless/01655_plan_optimizations_merge_filters.sql b/tests/queries/0_stateless/01655_plan_optimizations_merge_filters.sql
index 2193fc7a8f4..c6620184a43 100644
--- a/tests/queries/0_stateless/01655_plan_optimizations_merge_filters.sql
+++ b/tests/queries/0_stateless/01655_plan_optimizations_merge_filters.sql
@@ -1,7 +1,7 @@
 set query_plan_merge_filters=1;
 
-set allow_experimental_analyzer=1;
+set enable_analyzer=1;
 select explain from (explain actions = 1 select * from (select sum(number) as v, bitAnd(number, 15) as key from numbers(1e8) group by key having v != 0) where key = 7) where explain like '%Filter%' or explain like '%Aggregating%';
 
-set allow_experimental_analyzer=0;
+set enable_analyzer=0;
 select explain from (explain actions = 1 select * from (select sum(number) as v, bitAnd(number, 15) as key from numbers(1e8) group by key having v != 0) where key = 7) where explain like '%Filter%' or explain like '%Aggregating%';
diff --git a/tests/queries/0_stateless/01655_plan_optimizations_optimize_read_in_window_order.reference b/tests/queries/0_stateless/01655_plan_optimizations_optimize_read_in_window_order.reference
index 7c2753124b3..c8b3c5ca954 100644
--- a/tests/queries/0_stateless/01655_plan_optimizations_optimize_read_in_window_order.reference
+++ b/tests/queries/0_stateless/01655_plan_optimizations_optimize_read_in_window_order.reference
@@ -1,23 +1,23 @@
 Partial sorting plan
   optimize_read_in_window_order=0
     Sort description: n ASC, x ASC
-  optimize_read_in_window_order=0, allow_experimental_analyzer=1
+  optimize_read_in_window_order=0, enable_analyzer=1
     Sort description: n ASC, x ASC
   optimize_read_in_window_order=1
     Prefix sort description: n ASC
     Result sort description: n ASC, x ASC
-  optimize_read_in_window_order=1, allow_experimental_analyzer=1
+  optimize_read_in_window_order=1, enable_analyzer=1
     Prefix sort description: __table1.n ASC
     Result sort description: __table1.n ASC, __table1.x ASC
 No sorting plan
   optimize_read_in_window_order=0
     Sort description: n ASC, x ASC
-  optimize_read_in_window_order=0, allow_experimental_analyzer=1
+  optimize_read_in_window_order=0, enable_analyzer=1
     Sort description: __table1.n ASC, __table1.x ASC
   optimize_read_in_window_order=1
     Prefix sort description: n ASC, x ASC
     Result sort description: n ASC, x ASC
-  optimize_read_in_window_order=1, allow_experimental_analyzer=1
+  optimize_read_in_window_order=1, enable_analyzer=1
     Prefix sort description: __table1.n ASC, __table1.x ASC
     Result sort description: __table1.n ASC, __table1.x ASC
 Complex ORDER BY
diff --git a/tests/queries/0_stateless/01655_plan_optimizations_optimize_read_in_window_order.sh b/tests/queries/0_stateless/01655_plan_optimizations_optimize_read_in_window_order.sh
index 24c8cf5052e..d74ea328d04 100755
--- a/tests/queries/0_stateless/01655_plan_optimizations_optimize_read_in_window_order.sh
+++ b/tests/queries/0_stateless/01655_plan_optimizations_optimize_read_in_window_order.sh
@@ -19,25 +19,25 @@ $CLICKHOUSE_CLIENT -q "optimize table ${name}_n_x final"
 
 echo 'Partial sorting plan'
 echo '  optimize_read_in_window_order=0'
-$CLICKHOUSE_CLIENT -q "explain plan actions=1, description=1 select n, sum(x) OVER (ORDER BY n, x ROWS BETWEEN 100 PRECEDING AND CURRENT ROW) from ${name}_n SETTINGS optimize_read_in_order=0,optimize_read_in_window_order=0,allow_experimental_analyzer=0" | grep -i "sort description"
-echo '  optimize_read_in_window_order=0, allow_experimental_analyzer=1'
-$CLICKHOUSE_CLIENT -q "explain plan actions=1, description=1 select n, sum(x) OVER (ORDER BY n, x ROWS BETWEEN 100 PRECEDING AND CURRENT ROW) from ${name}_n SETTINGS optimize_read_in_order=0,optimize_read_in_window_order=0,allow_experimental_analyzer=0" | grep -i "sort description"
+$CLICKHOUSE_CLIENT -q "explain plan actions=1, description=1 select n, sum(x) OVER (ORDER BY n, x ROWS BETWEEN 100 PRECEDING AND CURRENT ROW) from ${name}_n SETTINGS optimize_read_in_order=0,optimize_read_in_window_order=0,enable_analyzer=0" | grep -i "sort description"
+echo '  optimize_read_in_window_order=0, enable_analyzer=1'
+$CLICKHOUSE_CLIENT -q "explain plan actions=1, description=1 select n, sum(x) OVER (ORDER BY n, x ROWS BETWEEN 100 PRECEDING AND CURRENT ROW) from ${name}_n SETTINGS optimize_read_in_order=0,optimize_read_in_window_order=0,enable_analyzer=0" | grep -i "sort description"
 
 echo '  optimize_read_in_window_order=1'
-$CLICKHOUSE_CLIENT -q "explain plan actions=1, description=1 select n, sum(x) OVER (ORDER BY n, x ROWS BETWEEN 100 PRECEDING AND CURRENT ROW) from ${name}_n SETTINGS optimize_read_in_order=1,allow_experimental_analyzer=0" | grep -i "sort description"
-echo '  optimize_read_in_window_order=1, allow_experimental_analyzer=1'
-$CLICKHOUSE_CLIENT -q "explain plan actions=1, description=1 select n, sum(x) OVER (ORDER BY n, x ROWS BETWEEN 100 PRECEDING AND CURRENT ROW) from ${name}_n SETTINGS optimize_read_in_order=1,allow_experimental_analyzer=1" | grep -i "sort description"
+$CLICKHOUSE_CLIENT -q "explain plan actions=1, description=1 select n, sum(x) OVER (ORDER BY n, x ROWS BETWEEN 100 PRECEDING AND CURRENT ROW) from ${name}_n SETTINGS optimize_read_in_order=1,enable_analyzer=0" | grep -i "sort description"
+echo '  optimize_read_in_window_order=1, enable_analyzer=1'
+$CLICKHOUSE_CLIENT -q "explain plan actions=1, description=1 select n, sum(x) OVER (ORDER BY n, x ROWS BETWEEN 100 PRECEDING AND CURRENT ROW) from ${name}_n SETTINGS optimize_read_in_order=1,enable_analyzer=1" | grep -i "sort description"
 
 echo 'No sorting plan'
 echo '  optimize_read_in_window_order=0'
-$CLICKHOUSE_CLIENT -q "explain plan actions=1, description=1 select n, sum(x) OVER (ORDER BY n, x ROWS BETWEEN 100 PRECEDING AND CURRENT ROW) from ${name}_n_x SETTINGS optimize_read_in_order=0,optimize_read_in_window_order=0,allow_experimental_analyzer=0" | grep -i "sort description"
-echo '  optimize_read_in_window_order=0, allow_experimental_analyzer=1'
-$CLICKHOUSE_CLIENT -q "explain plan actions=1, description=1 select n, sum(x) OVER (ORDER BY n, x ROWS BETWEEN 100 PRECEDING AND CURRENT ROW) from ${name}_n_x SETTINGS optimize_read_in_order=0,optimize_read_in_window_order=0,allow_experimental_analyzer=1" | grep -i "sort description"
+$CLICKHOUSE_CLIENT -q "explain plan actions=1, description=1 select n, sum(x) OVER (ORDER BY n, x ROWS BETWEEN 100 PRECEDING AND CURRENT ROW) from ${name}_n_x SETTINGS optimize_read_in_order=0,optimize_read_in_window_order=0,enable_analyzer=0" | grep -i "sort description"
+echo '  optimize_read_in_window_order=0, enable_analyzer=1'
+$CLICKHOUSE_CLIENT -q "explain plan actions=1, description=1 select n, sum(x) OVER (ORDER BY n, x ROWS BETWEEN 100 PRECEDING AND CURRENT ROW) from ${name}_n_x SETTINGS optimize_read_in_order=0,optimize_read_in_window_order=0,enable_analyzer=1" | grep -i "sort description"
 
 echo '  optimize_read_in_window_order=1'
-$CLICKHOUSE_CLIENT -q "explain plan actions=1, description=1 select n, sum(x) OVER (ORDER BY n, x ROWS BETWEEN 100 PRECEDING AND CURRENT ROW) from ${name}_n_x SETTINGS optimize_read_in_order=1,allow_experimental_analyzer=0" | grep -i "sort description"
-echo '  optimize_read_in_window_order=1, allow_experimental_analyzer=1'
-$CLICKHOUSE_CLIENT -q "explain plan actions=1, description=1 select n, sum(x) OVER (ORDER BY n, x ROWS BETWEEN 100 PRECEDING AND CURRENT ROW) from ${name}_n_x SETTINGS optimize_read_in_order=1,allow_experimental_analyzer=1" | grep -i "sort description"
+$CLICKHOUSE_CLIENT -q "explain plan actions=1, description=1 select n, sum(x) OVER (ORDER BY n, x ROWS BETWEEN 100 PRECEDING AND CURRENT ROW) from ${name}_n_x SETTINGS optimize_read_in_order=1,enable_analyzer=0" | grep -i "sort description"
+echo '  optimize_read_in_window_order=1, enable_analyzer=1'
+$CLICKHOUSE_CLIENT -q "explain plan actions=1, description=1 select n, sum(x) OVER (ORDER BY n, x ROWS BETWEEN 100 PRECEDING AND CURRENT ROW) from ${name}_n_x SETTINGS optimize_read_in_order=1,enable_analyzer=1" | grep -i "sort description"
 
 echo 'Complex ORDER BY'
 $CLICKHOUSE_CLIENT -q "CREATE TABLE ${name}_complex (unique1 Int32, unique2 Int32, ten Int32) ENGINE=MergeTree ORDER BY tuple() SETTINGS index_granularity = 8192"
diff --git a/tests/queries/0_stateless/01671_merge_join_and_constants.sql b/tests/queries/0_stateless/01671_merge_join_and_constants.sql
index 7a84bd4e97a..a2153bf0093 100644
--- a/tests/queries/0_stateless/01671_merge_join_and_constants.sql
+++ b/tests/queries/0_stateless/01671_merge_join_and_constants.sql
@@ -1,5 +1,5 @@
 SET output_format_pretty_color=1;
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS table1;
 DROP TABLE IF EXISTS table2;
diff --git a/tests/queries/0_stateless/01721_join_implicit_cast_long.sql.j2 b/tests/queries/0_stateless/01721_join_implicit_cast_long.sql.j2
index db004c13d96..3c2fa9877db 100644
--- a/tests/queries/0_stateless/01721_join_implicit_cast_long.sql.j2
+++ b/tests/queries/0_stateless/01721_join_implicit_cast_long.sql.j2
@@ -37,8 +37,8 @@ SELECT a, t1.a, t2.a FROM t1 FULL JOIN t2 USING (a) ORDER BY (t1.a, t2.a); {{ is
 SELECT '= left =';
 SELECT a, t1.a, t2.a FROM t1 LEFT JOIN t2 USING (a) ORDER BY (t1.a, t2.a);
 SELECT '= right =';
-SELECT a, t1.a, t2.a FROM t1 RIGHT JOIN t2 USING (a) ORDER BY (t1.a, t2.a) SETTINGS allow_experimental_analyzer = 0; {{ is_implemented(join_algorithm) }}
-SELECT a, t1.a, t2.a FROM t1 RIGHT JOIN t2 USING (a) ORDER BY (t1.a, t2.a) SETTINGS allow_experimental_analyzer = 1; {{ is_implemented(join_algorithm) }}
+SELECT a, t1.a, t2.a FROM t1 RIGHT JOIN t2 USING (a) ORDER BY (t1.a, t2.a) SETTINGS enable_analyzer = 0; {{ is_implemented(join_algorithm) }}
+SELECT a, t1.a, t2.a FROM t1 RIGHT JOIN t2 USING (a) ORDER BY (t1.a, t2.a) SETTINGS enable_analyzer = 1; {{ is_implemented(join_algorithm) }}
 SELECT '= inner =';
 SELECT a, t1.a, t2.a FROM t1 INNER JOIN t2 USING (a) ORDER BY (t1.a, t2.a);
 
@@ -119,8 +119,8 @@ SELECT a, t1.a, t2.a FROM t1 FULL JOIN t2 USING (a) ORDER BY (t1.a, t2.a); {{ is
 SELECT '= left =';
 SELECT a, t1.a, t2.a FROM t1 LEFT JOIN t2 USING (a) ORDER BY (t1.a, t2.a);
 SELECT '= right =';
-SELECT a, t1.a, t2.a FROM t1 RIGHT JOIN t2 USING (a) ORDER BY (t1.a, t2.a) SETTINGS allow_experimental_analyzer = 0; {{ is_implemented(join_algorithm) }}
-SELECT a, t1.a, t2.a FROM t1 RIGHT JOIN t2 USING (a) ORDER BY (t1.a, t2.a) SETTINGS allow_experimental_analyzer = 1; {{ is_implemented(join_algorithm) }}
+SELECT a, t1.a, t2.a FROM t1 RIGHT JOIN t2 USING (a) ORDER BY (t1.a, t2.a) SETTINGS enable_analyzer = 0; {{ is_implemented(join_algorithm) }}
+SELECT a, t1.a, t2.a FROM t1 RIGHT JOIN t2 USING (a) ORDER BY (t1.a, t2.a) SETTINGS enable_analyzer = 1; {{ is_implemented(join_algorithm) }}
 SELECT '= inner =';
 SELECT a, t1.a, t2.a FROM t1 INNER JOIN t2 USING (a) ORDER BY (t1.a, t2.a);
 
@@ -166,8 +166,8 @@ SELECT '= types =';
 SELECT any(toTypeName(a)) == 'Nullable(Int32)' AND any(toTypeName(t2.a)) == 'Nullable(Int32)' FROM t1 FULL JOIN t2 USING (a); {{ is_implemented(join_algorithm) }}
 SELECT any(toTypeName(a)) == 'Int32' AND any(toTypeName(t2.a)) == 'Nullable(Int32)' FROM t1 LEFT JOIN t2 USING (a);
 
-SELECT any(toTypeName(a)) == 'Nullable(Int32)' AND any(toTypeName(t2.a)) == 'Int32' FROM t1 RIGHT JOIN t2 USING (a) SETTINGS allow_experimental_analyzer = 0; {{ is_implemented(join_algorithm) }}
-SELECT any(toTypeName(a)) == 'Int32' AND any(toTypeName(t2.a)) == 'Int32' FROM t1 RIGHT JOIN t2 USING (a) SETTINGS allow_experimental_analyzer = 1; {{ is_implemented(join_algorithm) }}
+SELECT any(toTypeName(a)) == 'Nullable(Int32)' AND any(toTypeName(t2.a)) == 'Int32' FROM t1 RIGHT JOIN t2 USING (a) SETTINGS enable_analyzer = 0; {{ is_implemented(join_algorithm) }}
+SELECT any(toTypeName(a)) == 'Int32' AND any(toTypeName(t2.a)) == 'Int32' FROM t1 RIGHT JOIN t2 USING (a) SETTINGS enable_analyzer = 1; {{ is_implemented(join_algorithm) }}
 
 SELECT any(toTypeName(a)) == 'Int32' AND any(toTypeName(t2.a)) == 'Int32' FROM t1 INNER JOIN t2 USING (a);
 
diff --git a/tests/queries/0_stateless/01739_index_hint.reference b/tests/queries/0_stateless/01739_index_hint.reference
index 21f4edc0049..b921dc6c1f9 100644
--- a/tests/queries/0_stateless/01739_index_hint.reference
+++ b/tests/queries/0_stateless/01739_index_hint.reference
@@ -35,9 +35,9 @@ SELECT count() FROM XXXX WHERE indexHint(t = toDateTime(0)) SETTINGS optimize_us
 drop table XXXX;
 CREATE TABLE XXXX (p Nullable(Int64), k Decimal(76, 39)) ENGINE = MergeTree PARTITION BY toDate(p) ORDER BY k SETTINGS index_granularity = 1, allow_nullable_key = 1;
 INSERT INTO XXXX FORMAT Values ('2020-09-01 00:01:02', 1), ('2020-09-01 20:01:03', 2), ('2020-09-02 00:01:03', 3);
-SELECT count() FROM XXXX WHERE indexHint(p = 1.) SETTINGS optimize_use_implicit_projections = 1, allow_experimental_analyzer=0;
+SELECT count() FROM XXXX WHERE indexHint(p = 1.) SETTINGS optimize_use_implicit_projections = 1, enable_analyzer=0;
 0
 -- TODO: optimize_use_implicit_projections ignores indexHint (with analyzer) because source columns might be aliased.
-SELECT count() FROM XXXX WHERE indexHint(p = 1.) SETTINGS optimize_use_implicit_projections = 1, allow_experimental_analyzer=1;
+SELECT count() FROM XXXX WHERE indexHint(p = 1.) SETTINGS optimize_use_implicit_projections = 1, enable_analyzer=1;
 3
 drop table XXXX;
diff --git a/tests/queries/0_stateless/01739_index_hint.sql b/tests/queries/0_stateless/01739_index_hint.sql
index 1eca65f0892..b208063e7c4 100644
--- a/tests/queries/0_stateless/01739_index_hint.sql
+++ b/tests/queries/0_stateless/01739_index_hint.sql
@@ -38,8 +38,8 @@ CREATE TABLE XXXX (p Nullable(Int64), k Decimal(76, 39)) ENGINE = MergeTree PART
 
 INSERT INTO XXXX FORMAT Values ('2020-09-01 00:01:02', 1), ('2020-09-01 20:01:03', 2), ('2020-09-02 00:01:03', 3);
 
-SELECT count() FROM XXXX WHERE indexHint(p = 1.) SETTINGS optimize_use_implicit_projections = 1, allow_experimental_analyzer=0;
+SELECT count() FROM XXXX WHERE indexHint(p = 1.) SETTINGS optimize_use_implicit_projections = 1, enable_analyzer=0;
 -- TODO: optimize_use_implicit_projections ignores indexHint (with analyzer) because source columns might be aliased.
-SELECT count() FROM XXXX WHERE indexHint(p = 1.) SETTINGS optimize_use_implicit_projections = 1, allow_experimental_analyzer=1;
+SELECT count() FROM XXXX WHERE indexHint(p = 1.) SETTINGS optimize_use_implicit_projections = 1, enable_analyzer=1;
 
 drop table XXXX;
diff --git a/tests/queries/0_stateless/01756_optimize_skip_unused_shards_rewrite_in.reference b/tests/queries/0_stateless/01756_optimize_skip_unused_shards_rewrite_in.reference
index 28dbb9215a8..74a0356b11e 100644
--- a/tests/queries/0_stateless/01756_optimize_skip_unused_shards_rewrite_in.reference
+++ b/tests/queries/0_stateless/01756_optimize_skip_unused_shards_rewrite_in.reference
@@ -103,7 +103,7 @@ errors
 -- optimize_skip_unused_shards does not support non-constants
 select * from dist_01756 where dummy in (select * from system.one); -- { serverError UNABLE_TO_SKIP_UNUSED_SHARDS }
 -- this is a constant for analyzer
-select * from dist_01756 where dummy in (toUInt8(0)) settings allow_experimental_analyzer=0; -- { serverError UNABLE_TO_SKIP_UNUSED_SHARDS }
+select * from dist_01756 where dummy in (toUInt8(0)) settings enable_analyzer=0; -- { serverError UNABLE_TO_SKIP_UNUSED_SHARDS }
 -- NOT IN does not supported
 select * from dist_01756 where dummy not in (0, 2); -- { serverError UNABLE_TO_SKIP_UNUSED_SHARDS }
 --
@@ -140,7 +140,7 @@ select * from dist_01756_str where key in ('0', '2');
 select * from dist_01756_str where key in (0, 2);
 0
 -- analyzer does support this
-select * from dist_01756_str where key in ('0', Null) settings allow_experimental_analyzer=0; -- { serverError UNABLE_TO_SKIP_UNUSED_SHARDS }
+select * from dist_01756_str where key in ('0', Null) settings enable_analyzer=0; -- { serverError UNABLE_TO_SKIP_UNUSED_SHARDS }
 -- select * from dist_01756_str where key in (0, 2); -- { serverError TYPE_MISMATCH }
 -- select * from dist_01756_str where key in (0, Null); -- { serverError TYPE_MISMATCH }
 
diff --git a/tests/queries/0_stateless/01756_optimize_skip_unused_shards_rewrite_in.sql b/tests/queries/0_stateless/01756_optimize_skip_unused_shards_rewrite_in.sql
index 9a1a00cc0a1..bcbedeb3ada 100644
--- a/tests/queries/0_stateless/01756_optimize_skip_unused_shards_rewrite_in.sql
+++ b/tests/queries/0_stateless/01756_optimize_skip_unused_shards_rewrite_in.sql
@@ -113,7 +113,7 @@ select 'errors';
 -- optimize_skip_unused_shards does not support non-constants
 select * from dist_01756 where dummy in (select * from system.one); -- { serverError UNABLE_TO_SKIP_UNUSED_SHARDS }
 -- this is a constant for analyzer
-select * from dist_01756 where dummy in (toUInt8(0)) settings allow_experimental_analyzer=0; -- { serverError UNABLE_TO_SKIP_UNUSED_SHARDS }
+select * from dist_01756 where dummy in (toUInt8(0)) settings enable_analyzer=0; -- { serverError UNABLE_TO_SKIP_UNUSED_SHARDS }
 -- NOT IN does not supported
 select * from dist_01756 where dummy not in (0, 2); -- { serverError UNABLE_TO_SKIP_UNUSED_SHARDS }
 
@@ -146,7 +146,7 @@ create table dist_01756_str as data_01756_str engine=Distributed(test_cluster_tw
 select * from dist_01756_str where key in ('0', '2');
 select * from dist_01756_str where key in (0, 2);
 -- analyzer does support this
-select * from dist_01756_str where key in ('0', Null) settings allow_experimental_analyzer=0; -- { serverError UNABLE_TO_SKIP_UNUSED_SHARDS }
+select * from dist_01756_str where key in ('0', Null) settings enable_analyzer=0; -- { serverError UNABLE_TO_SKIP_UNUSED_SHARDS }
 -- select * from dist_01756_str where key in (0, 2); -- { serverError TYPE_MISMATCH }
 -- select * from dist_01756_str where key in (0, Null); -- { serverError TYPE_MISMATCH }
 
diff --git a/tests/queries/0_stateless/01757_optimize_skip_unused_shards_limit.sql b/tests/queries/0_stateless/01757_optimize_skip_unused_shards_limit.sql
index 3853ccb4080..6fcf98d47cb 100644
--- a/tests/queries/0_stateless/01757_optimize_skip_unused_shards_limit.sql
+++ b/tests/queries/0_stateless/01757_optimize_skip_unused_shards_limit.sql
@@ -21,9 +21,9 @@ select * from dist_01757 where dummy = 0 or dummy = 1 format Null settings optim
 
 -- and negative
 -- disabled for analyzer cause new implementation consider `dummy = 0 and dummy = 1` as constant False.
-select * from dist_01757 where dummy = 0 and dummy = 1 settings optimize_skip_unused_shards_limit=1, allow_experimental_analyzer=0; -- { serverError UNABLE_TO_SKIP_UNUSED_SHARDS }
-select * from dist_01757 where dummy = 0 and dummy = 2 and dummy = 3 settings optimize_skip_unused_shards_limit=1, allow_experimental_analyzer=0; -- { serverError UNABLE_TO_SKIP_UNUSED_SHARDS }
-select * from dist_01757 where dummy = 0 and dummy = 2 and dummy = 3 settings optimize_skip_unused_shards_limit=2, allow_experimental_analyzer=0; -- { serverError UNABLE_TO_SKIP_UNUSED_SHARDS }
+select * from dist_01757 where dummy = 0 and dummy = 1 settings optimize_skip_unused_shards_limit=1, enable_analyzer=0; -- { serverError UNABLE_TO_SKIP_UNUSED_SHARDS }
+select * from dist_01757 where dummy = 0 and dummy = 2 and dummy = 3 settings optimize_skip_unused_shards_limit=1, enable_analyzer=0; -- { serverError UNABLE_TO_SKIP_UNUSED_SHARDS }
+select * from dist_01757 where dummy = 0 and dummy = 2 and dummy = 3 settings optimize_skip_unused_shards_limit=2, enable_analyzer=0; -- { serverError UNABLE_TO_SKIP_UNUSED_SHARDS }
 
 -- and
 select * from dist_01757 where dummy = 0 and dummy = 1 settings optimize_skip_unused_shards_limit=2;
diff --git a/tests/queries/0_stateless/01763_filter_push_down_bugs.sql b/tests/queries/0_stateless/01763_filter_push_down_bugs.sql
index 367baef142b..aa610e19c12 100644
--- a/tests/queries/0_stateless/01763_filter_push_down_bugs.sql
+++ b/tests/queries/0_stateless/01763_filter_push_down_bugs.sql
@@ -57,13 +57,13 @@ EXPLAIN indexes=1 SELECT id, delete_time FROM t1
  CROSS JOIN (
     SELECT delete_time
     FROM t2
-) AS d WHERE create_time < delete_time AND id = 101 SETTINGS allow_experimental_analyzer=0;
+) AS d WHERE create_time < delete_time AND id = 101 SETTINGS enable_analyzer=0;
 
 EXPLAIN indexes=1 SELECT id, delete_time FROM t1
  CROSS JOIN (
     SELECT delete_time
     FROM t2
-) AS d WHERE create_time < delete_time AND id = 101 SETTINGS allow_experimental_analyzer=1;
+) AS d WHERE create_time < delete_time AND id = 101 SETTINGS enable_analyzer=1;
 
 DROP TABLE IF EXISTS t1;
 DROP TABLE IF EXISTS t2;
diff --git a/tests/queries/0_stateless/01786_explain_merge_tree.sh b/tests/queries/0_stateless/01786_explain_merge_tree.sh
index e3b28acdc41..828012f56bc 100755
--- a/tests/queries/0_stateless/01786_explain_merge_tree.sh
+++ b/tests/queries/0_stateless/01786_explain_merge_tree.sh
@@ -7,7 +7,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
 for i in $(seq 0 1)
 do
-    CH_CLIENT="$CLICKHOUSE_CLIENT --optimize_move_to_prewhere=1 --convert_query_to_cnf=0 --optimize_read_in_order=1 --allow_experimental_analyzer=$i"
+    CH_CLIENT="$CLICKHOUSE_CLIENT --optimize_move_to_prewhere=1 --convert_query_to_cnf=0 --optimize_read_in_order=1 --enable_analyzer=$i"
 
     $CH_CLIENT -q "drop table if exists test_index"
     $CH_CLIENT -q "drop table if exists idx"
diff --git a/tests/queries/0_stateless/01823_explain_json.sh b/tests/queries/0_stateless/01823_explain_json.sh
index 39128773069..356a317ae57 100755
--- a/tests/queries/0_stateless/01823_explain_json.sh
+++ b/tests/queries/0_stateless/01823_explain_json.sh
@@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CURDIR"/../shell_config.sh
 
 opts=(
-    "--allow_experimental_analyzer=1"
+    "--enable_analyzer=1"
 )
 $CLICKHOUSE_CLIENT "${opts[@]}" -q "EXPLAIN json = 1, description = 0 SELECT 1 UNION ALL SELECT 2 FORMAT TSVRaw"
 echo "--------"
diff --git a/tests/queries/0_stateless/01852_multiple_joins_with_union_join.sql b/tests/queries/0_stateless/01852_multiple_joins_with_union_join.sql
index 8c6937eb581..4387a697a83 100644
--- a/tests/queries/0_stateless/01852_multiple_joins_with_union_join.sql
+++ b/tests/queries/0_stateless/01852_multiple_joins_with_union_join.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS v1;
 DROP TABLE IF EXISTS v2;
diff --git a/tests/queries/0_stateless/01872_functions_to_subcolumns_analyzer.sql b/tests/queries/0_stateless/01872_functions_to_subcolumns_analyzer.sql
index b544f6829cf..032d83890ec 100644
--- a/tests/queries/0_stateless/01872_functions_to_subcolumns_analyzer.sql
+++ b/tests/queries/0_stateless/01872_functions_to_subcolumns_analyzer.sql
@@ -1,6 +1,6 @@
 DROP TABLE IF EXISTS t_func_to_subcolumns;
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 SET optimize_functions_to_subcolumns = 1;
 
 CREATE TABLE t_func_to_subcolumns (id UInt64, arr Array(UInt64), n Nullable(String), m Map(String, UInt64))
diff --git a/tests/queries/0_stateless/01890_cross_join_explain_crash.sql b/tests/queries/0_stateless/01890_cross_join_explain_crash.sql
index bb2bc606870..79aea3884b7 100644
--- a/tests/queries/0_stateless/01890_cross_join_explain_crash.sql
+++ b/tests/queries/0_stateless/01890_cross_join_explain_crash.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 SET joined_subquery_requires_alias = 0;
 
 select * FROM (SELECT 1), (SELECT 1), (SELECT 1);
diff --git a/tests/queries/0_stateless/01913_names_of_tuple_literal.sql b/tests/queries/0_stateless/01913_names_of_tuple_literal.sql
index 879f4c91587..d6dda4fda9b 100644
--- a/tests/queries/0_stateless/01913_names_of_tuple_literal.sql
+++ b/tests/queries/0_stateless/01913_names_of_tuple_literal.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 0;
+SET enable_analyzer = 0;
 
 SELECT ((1, 2), (2, 3), (3, 4)) FORMAT TSVWithNames;
 SELECT ((1, 2), (2, 3), (3, 4)) FORMAT TSVWithNames SETTINGS legacy_column_name_of_tuple_literal = 1;
diff --git a/tests/queries/0_stateless/01925_join_materialized_columns.sql b/tests/queries/0_stateless/01925_join_materialized_columns.sql
index abb6fda3bfb..ce71cfed8a3 100644
--- a/tests/queries/0_stateless/01925_join_materialized_columns.sql
+++ b/tests/queries/0_stateless/01925_join_materialized_columns.sql
@@ -32,8 +32,8 @@ SELECT t1.dt, t2.dt FROM t1 JOIN t2 ON t1.foo = t2.bar ORDER BY t1.dt;
 SELECT '-';
 SELECT * FROM t1 ALL JOIN t2 ON t1.dt = t2.dt ORDER BY t1.time, t2.time;
 SELECT '-';
-SELECT * FROM t1 ALL JOIN t2 USING (dt) ORDER BY t1.time, t2.time settings allow_experimental_analyzer=0;
-SELECT * FROM t1 ALL JOIN t2 USING (dt) ORDER BY t1.time, t2.time settings allow_experimental_analyzer=1;
+SELECT * FROM t1 ALL JOIN t2 USING (dt) ORDER BY t1.time, t2.time settings enable_analyzer=0;
+SELECT * FROM t1 ALL JOIN t2 USING (dt) ORDER BY t1.time, t2.time settings enable_analyzer=1;
 SELECT '-';
 SELECT * FROM t1 JOIN t2 ON t1.dt1 = t2.dt2 ORDER BY t1.time, t1.dimension_1, t2.time, t2.dimension_2;
 SELECT '-';
@@ -53,5 +53,5 @@ SELECT t1.time as talias FROM t1 JOIN t2 ON talias = t2.time_alias;
 SELECT t2.time as talias FROM t1 JOIN t2 ON t1.time = talias;
 SELECT t2.time as talias FROM t1 JOIN t2 ON t1.time_alias = talias;
 SELECT time as talias FROM t1 JOIN t2 ON t1.time = talias; -- { serverError AMBIGUOUS_COLUMN_NAME, INVALID_JOIN_ON_EXPRESSION }
-SELECT time as talias FROM t1 JOIN t2 ON talias = t2.time settings allow_experimental_analyzer=0; -- { serverError AMBIGUOUS_COLUMN_NAME }
-SELECT time as talias FROM t1 JOIN t2 ON talias = t2.time settings allow_experimental_analyzer=1;
+SELECT time as talias FROM t1 JOIN t2 ON talias = t2.time settings enable_analyzer=0; -- { serverError AMBIGUOUS_COLUMN_NAME }
+SELECT time as talias FROM t1 JOIN t2 ON talias = t2.time settings enable_analyzer=1;
diff --git a/tests/queries/0_stateless/01925_test_storage_merge_aliases_analyzer.sql b/tests/queries/0_stateless/01925_test_storage_merge_aliases_analyzer.sql
index 31035aa80cd..28b9c8650af 100644
--- a/tests/queries/0_stateless/01925_test_storage_merge_aliases_analyzer.sql
+++ b/tests/queries/0_stateless/01925_test_storage_merge_aliases_analyzer.sql
@@ -1,7 +1,7 @@
 -- Tags: no-parallel
 
 drop table if exists merge;
-set allow_experimental_analyzer = 1;
+set enable_analyzer = 1;
 create table merge
 (
     dt Date,
diff --git a/tests/queries/0_stateless/01936_three_parts_identifiers_in_wrong_places.sql b/tests/queries/0_stateless/01936_three_parts_identifiers_in_wrong_places.sql
index f344b7007d0..4efbb461501 100644
--- a/tests/queries/0_stateless/01936_three_parts_identifiers_in_wrong_places.sql
+++ b/tests/queries/0_stateless/01936_three_parts_identifiers_in_wrong_places.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 SELECT dictGet(t.nest.a, concat(currentDatabase(), '.dict.dict'), 's', number) FROM numbers(5); -- { serverError INVALID_IDENTIFIER }
 
diff --git a/tests/queries/0_stateless/01942_dateTimeToSnowflakeID.sql b/tests/queries/0_stateless/01942_dateTimeToSnowflakeID.sql
index 945b399157f..0154265ef72 100644
--- a/tests/queries/0_stateless/01942_dateTimeToSnowflakeID.sql
+++ b/tests/queries/0_stateless/01942_dateTimeToSnowflakeID.sql
@@ -1,5 +1,5 @@
 SET session_timezone = 'UTC'; -- disable timezone randomization
-SET allow_experimental_analyzer = 1; -- The old path formats the result with different whitespaces
+SET enable_analyzer = 1; -- The old path formats the result with different whitespaces
 
 SELECT '-- Negative tests';
 SELECT dateTimeToSnowflakeID();  -- {serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH}
diff --git a/tests/queries/0_stateless/01942_snowflakeIDToDateTime.sql b/tests/queries/0_stateless/01942_snowflakeIDToDateTime.sql
index 48316691c71..41e5beb9c16 100644
--- a/tests/queries/0_stateless/01942_snowflakeIDToDateTime.sql
+++ b/tests/queries/0_stateless/01942_snowflakeIDToDateTime.sql
@@ -1,5 +1,5 @@
 SET session_timezone = 'UTC'; -- disable timezone randomization
-SET allow_experimental_analyzer = 1; -- The old path formats the result with different whitespaces
+SET enable_analyzer = 1; -- The old path formats the result with different whitespaces
 
 SELECT '-- Negative tests';
 SELECT snowflakeIDToDateTime();  -- {serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH}
diff --git a/tests/queries/0_stateless/01952_optimize_distributed_group_by_sharding_key.reference b/tests/queries/0_stateless/01952_optimize_distributed_group_by_sharding_key.reference
index 5acefdb365e..e786532f25a 100644
--- a/tests/queries/0_stateless/01952_optimize_distributed_group_by_sharding_key.reference
+++ b/tests/queries/0_stateless/01952_optimize_distributed_group_by_sharding_key.reference
@@ -71,7 +71,7 @@ Expression (Projection)
             Expression ((Before ORDER BY + (Convert VIEW subquery result to VIEW table structure + (Materialize constants after VIEW subquery + (Projection + Before ORDER BY)))))
               ReadFromSystemNumbers
       ReadFromRemote (Read from remote replica)
-set allow_experimental_analyzer = 1;
+set enable_analyzer = 1;
 explain select distinct k1 from remote('127.{1,2}', view(select 1 k1, 2 k2, 3 v from numbers(2)), cityHash64(k1, k2)); -- not optimized
 Expression (Project names)
   Distinct (DISTINCT)
diff --git a/tests/queries/0_stateless/01952_optimize_distributed_group_by_sharding_key.sql b/tests/queries/0_stateless/01952_optimize_distributed_group_by_sharding_key.sql
index adf55a9dd7f..960fd227a39 100644
--- a/tests/queries/0_stateless/01952_optimize_distributed_group_by_sharding_key.sql
+++ b/tests/queries/0_stateless/01952_optimize_distributed_group_by_sharding_key.sql
@@ -4,7 +4,7 @@ set optimize_skip_unused_shards=1;
 set optimize_distributed_group_by_sharding_key=1;
 set prefer_localhost_replica=1;
 
-set allow_experimental_analyzer = 0;
+set enable_analyzer = 0;
 
 -- { echo }
 explain select distinct k1 from remote('127.{1,2}', view(select 1 k1, 2 k2, 3 v from numbers(2)), cityHash64(k1, k2)); -- not optimized
@@ -17,7 +17,7 @@ explain select distinct k1, k2 from remote('127.{1,2}', view(select 1 k1, 2 k2,
 explain select distinct on (k1) k2 from remote('127.{1,2}', view(select 1 k1, 2 k2, 3 v from numbers(2)), cityHash64(k1, k2)) order by v; -- not optimized
 explain select distinct on (k1, k2) v from remote('127.{1,2}', view(select 1 k1, 2 k2, 3 v from numbers(2)), cityHash64(k1, k2)) order by v; -- optimized
 
-set allow_experimental_analyzer = 1;
+set enable_analyzer = 1;
 
 explain select distinct k1 from remote('127.{1,2}', view(select 1 k1, 2 k2, 3 v from numbers(2)), cityHash64(k1, k2)); -- not optimized
 explain select distinct k1, k2 from remote('127.{1,2}', view(select 1 k1, 2 k2, 3 v from numbers(2)), cityHash64(k1, k2)); -- optimized
diff --git a/tests/queries/0_stateless/02000_join_on_const.reference b/tests/queries/0_stateless/02000_join_on_const.reference
index 848ecedf9e3..3bd1633ce32 100644
--- a/tests/queries/0_stateless/02000_join_on_const.reference
+++ b/tests/queries/0_stateless/02000_join_on_const.reference
@@ -33,23 +33,23 @@
 2	2
 2	2
 -- { echoOn }
-SELECT * FROM t1 LEFT JOIN t2 ON t1.id = t2.id AND 1 = 1 SETTINGS allow_experimental_analyzer = 1;
+SELECT * FROM t1 LEFT JOIN t2 ON t1.id = t2.id AND 1 = 1 SETTINGS enable_analyzer = 1;
 1	0
 2	2
-SELECT * FROM t1 RIGHT JOIN t2 ON t1.id = t2.id AND 1 = 1 SETTINGS allow_experimental_analyzer = 1;
+SELECT * FROM t1 RIGHT JOIN t2 ON t1.id = t2.id AND 1 = 1 SETTINGS enable_analyzer = 1;
 2	2
 0	3
-SELECT * FROM t1 FULL JOIN t2 ON t1.id = t2.id AND 1 = 1 SETTINGS allow_experimental_analyzer = 1;
+SELECT * FROM t1 FULL JOIN t2 ON t1.id = t2.id AND 1 = 1 SETTINGS enable_analyzer = 1;
 1	0
 2	2
 0	3
-SELECT * FROM t1 LEFT JOIN t2 ON t1.id = t2.id AND 1 = 2 SETTINGS allow_experimental_analyzer = 1;
+SELECT * FROM t1 LEFT JOIN t2 ON t1.id = t2.id AND 1 = 2 SETTINGS enable_analyzer = 1;
 1	0
 2	0
-SELECT * FROM t1 RIGHT JOIN t2 ON t1.id = t2.id AND 1 = 2 SETTINGS allow_experimental_analyzer = 1;
+SELECT * FROM t1 RIGHT JOIN t2 ON t1.id = t2.id AND 1 = 2 SETTINGS enable_analyzer = 1;
 0	2
 0	3
-SELECT * FROM t1 FULL JOIN t2 ON t1.id = t2.id AND 1 = 2 SETTINGS allow_experimental_analyzer = 1;
+SELECT * FROM t1 FULL JOIN t2 ON t1.id = t2.id AND 1 = 2 SETTINGS enable_analyzer = 1;
 1	0
 2	0
 0	2
diff --git a/tests/queries/0_stateless/02000_join_on_const.sql b/tests/queries/0_stateless/02000_join_on_const.sql
index 2c1152e0ae6..da70973ed87 100644
--- a/tests/queries/0_stateless/02000_join_on_const.sql
+++ b/tests/queries/0_stateless/02000_join_on_const.sql
@@ -56,30 +56,30 @@ SELECT * FROM t1 RIGHT JOIN t2 ON NULL SETTINGS join_algorithm = 'auto'; -- { se
 SELECT * FROM t1 FULL JOIN t2 ON NULL SETTINGS join_algorithm = 'partial_merge'; -- { serverError INVALID_JOIN_ON_EXPRESSION,NOT_IMPLEMENTED }
 
 -- mixing of constant and non-constant expressions in ON is not allowed
-SELECT * FROM t1 JOIN t2 ON t1.id = t2.id AND 1 == 1 SETTINGS allow_experimental_analyzer = 0; -- { serverError AMBIGUOUS_COLUMN_NAME }
-SELECT * FROM t1 JOIN t2 ON t1.id = t2.id AND 1 == 1 SETTINGS allow_experimental_analyzer = 1;
-SELECT * FROM t1 JOIN t2 ON t1.id = t2.id AND 1 == 2 SETTINGS allow_experimental_analyzer = 0; -- { serverError AMBIGUOUS_COLUMN_NAME }
-SELECT * FROM t1 JOIN t2 ON t1.id = t2.id AND 1 == 2 SETTINGS allow_experimental_analyzer = 1;
+SELECT * FROM t1 JOIN t2 ON t1.id = t2.id AND 1 == 1 SETTINGS enable_analyzer = 0; -- { serverError AMBIGUOUS_COLUMN_NAME }
+SELECT * FROM t1 JOIN t2 ON t1.id = t2.id AND 1 == 1 SETTINGS enable_analyzer = 1;
+SELECT * FROM t1 JOIN t2 ON t1.id = t2.id AND 1 == 2 SETTINGS enable_analyzer = 0; -- { serverError AMBIGUOUS_COLUMN_NAME }
+SELECT * FROM t1 JOIN t2 ON t1.id = t2.id AND 1 == 2 SETTINGS enable_analyzer = 1;
 
-SELECT * FROM t1 JOIN t2 ON t1.id = t2.id AND 1 != 1 SETTINGS allow_experimental_analyzer = 0; -- { serverError INVALID_JOIN_ON_EXPRESSION }
-SELECT * FROM t1 JOIN t2 ON t1.id = t2.id AND 1 != 1 SETTINGS allow_experimental_analyzer = 1;
+SELECT * FROM t1 JOIN t2 ON t1.id = t2.id AND 1 != 1 SETTINGS enable_analyzer = 0; -- { serverError INVALID_JOIN_ON_EXPRESSION }
+SELECT * FROM t1 JOIN t2 ON t1.id = t2.id AND 1 != 1 SETTINGS enable_analyzer = 1;
 SELECT * FROM t1 JOIN t2 ON t1.id = t2.id AND NULL; -- { serverError INVALID_JOIN_ON_EXPRESSION }
 SELECT * FROM t1 JOIN t2 ON t1.id = t2.id AND 'aaa'; -- { serverError INVALID_JOIN_ON_EXPRESSION,ILLEGAL_TYPE_OF_ARGUMENT }
 SELECT * FROM t1 JOIN t2 ON 'aaa'; -- { serverError INVALID_JOIN_ON_EXPRESSION }
 
-SELECT * FROM t1 JOIN t2 ON t1.id = t2.id AND 0 SETTINGS allow_experimental_analyzer = 0; -- { serverError INVALID_JOIN_ON_EXPRESSION }
-SELECT * FROM t1 JOIN t2 ON t1.id = t2.id AND 0 SETTINGS allow_experimental_analyzer = 1;
-SELECT * FROM t1 JOIN t2 ON t1.id = t2.id AND 1 SETTINGS allow_experimental_analyzer = 0; -- { serverError INVALID_JOIN_ON_EXPRESSION }
-SELECT * FROM t1 JOIN t2 ON t1.id = t2.id AND 1 SETTINGS allow_experimental_analyzer = 1;
+SELECT * FROM t1 JOIN t2 ON t1.id = t2.id AND 0 SETTINGS enable_analyzer = 0; -- { serverError INVALID_JOIN_ON_EXPRESSION }
+SELECT * FROM t1 JOIN t2 ON t1.id = t2.id AND 0 SETTINGS enable_analyzer = 1;
+SELECT * FROM t1 JOIN t2 ON t1.id = t2.id AND 1 SETTINGS enable_analyzer = 0; -- { serverError INVALID_JOIN_ON_EXPRESSION }
+SELECT * FROM t1 JOIN t2 ON t1.id = t2.id AND 1 SETTINGS enable_analyzer = 1;
 
 -- { echoOn }
-SELECT * FROM t1 LEFT JOIN t2 ON t1.id = t2.id AND 1 = 1 SETTINGS allow_experimental_analyzer = 1;
-SELECT * FROM t1 RIGHT JOIN t2 ON t1.id = t2.id AND 1 = 1 SETTINGS allow_experimental_analyzer = 1;
-SELECT * FROM t1 FULL JOIN t2 ON t1.id = t2.id AND 1 = 1 SETTINGS allow_experimental_analyzer = 1;
+SELECT * FROM t1 LEFT JOIN t2 ON t1.id = t2.id AND 1 = 1 SETTINGS enable_analyzer = 1;
+SELECT * FROM t1 RIGHT JOIN t2 ON t1.id = t2.id AND 1 = 1 SETTINGS enable_analyzer = 1;
+SELECT * FROM t1 FULL JOIN t2 ON t1.id = t2.id AND 1 = 1 SETTINGS enable_analyzer = 1;
 
-SELECT * FROM t1 LEFT JOIN t2 ON t1.id = t2.id AND 1 = 2 SETTINGS allow_experimental_analyzer = 1;
-SELECT * FROM t1 RIGHT JOIN t2 ON t1.id = t2.id AND 1 = 2 SETTINGS allow_experimental_analyzer = 1;
-SELECT * FROM t1 FULL JOIN t2 ON t1.id = t2.id AND 1 = 2 SETTINGS allow_experimental_analyzer = 1;
+SELECT * FROM t1 LEFT JOIN t2 ON t1.id = t2.id AND 1 = 2 SETTINGS enable_analyzer = 1;
+SELECT * FROM t1 RIGHT JOIN t2 ON t1.id = t2.id AND 1 = 2 SETTINGS enable_analyzer = 1;
+SELECT * FROM t1 FULL JOIN t2 ON t1.id = t2.id AND 1 = 2 SETTINGS enable_analyzer = 1;
 
 SELECT * FROM (SELECT 1 as a) as t1 INNER JOIN  ( SELECT ('b', 256) as b ) AS t2 ON NULL;
 SELECT * FROM (SELECT 1 as a) as t1 LEFT JOIN  ( SELECT ('b', 256) as b ) AS t2 ON NULL;
@@ -94,27 +94,26 @@ SELECT a + 1
 FROM (SELECT 1 as x) as t1
 LEFT JOIN ( SELECT 1 AS a ) AS t2
 ON TRUE
-SETTINGS allow_experimental_analyzer=1, join_use_nulls=1;
+SETTINGS enable_analyzer=1, join_use_nulls=1;
 
 SELECT a + 1, x + 1, toTypeName(a), toTypeName(x)
 FROM (SELECT 1 as x) as t1
 LEFT JOIN ( SELECT sum(number) as a from numbers(3) GROUP BY NULL) AS t2
 ON TRUE
-SETTINGS allow_experimental_analyzer=1, join_use_nulls=1;
+SETTINGS enable_analyzer=1, join_use_nulls=1;
 
 SELECT a + 1, x + 1, toTypeName(a), toTypeName(x)
 FROM (SELECT 1 as x) as t1
 RIGHT JOIN ( SELECT sum(number) as a from numbers(3) GROUP BY NULL) AS t2
 ON TRUE
-SETTINGS allow_experimental_analyzer=1, join_use_nulls=1;
+SETTINGS enable_analyzer=1, join_use_nulls=1;
 
 SELECT a + 1, x + 1, toTypeName(a), toTypeName(x)
 FROM (SELECT 1 as x) as t1
 FULL JOIN ( SELECT sum(number) as a from numbers(3) GROUP BY NULL) AS t2
 ON TRUE
-SETTINGS allow_experimental_analyzer=1, join_use_nulls=1;
+SETTINGS enable_analyzer=1, join_use_nulls=1;
 
 
 DROP TABLE IF EXISTS t1;
 DROP TABLE IF EXISTS t2;
-
diff --git a/tests/queries/0_stateless/02030_tuple_filter.sql b/tests/queries/0_stateless/02030_tuple_filter.sql
index 42853dec681..c8f344f5076 100644
--- a/tests/queries/0_stateless/02030_tuple_filter.sql
+++ b/tests/queries/0_stateless/02030_tuple_filter.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS test_tuple_filter;
 
diff --git a/tests/queries/0_stateless/02048_clickhouse_local_stage.reference b/tests/queries/0_stateless/02048_clickhouse_local_stage.reference
index 2631199cbab..b6200464293 100644
--- a/tests/queries/0_stateless/02048_clickhouse_local_stage.reference
+++ b/tests/queries/0_stateless/02048_clickhouse_local_stage.reference
@@ -1,15 +1,15 @@
-execute: --allow_experimental_analyzer=1
+execute: --enable_analyzer=1
 "foo"
 1
-execute: --allow_experimental_analyzer=1 --stage fetch_columns
+execute: --enable_analyzer=1 --stage fetch_columns
 "__table1.dummy"
 0
-execute: --allow_experimental_analyzer=1 --stage with_mergeable_state
+execute: --enable_analyzer=1 --stage with_mergeable_state
 "1_UInt8"
 1
-execute: --allow_experimental_analyzer=1 --stage with_mergeable_state_after_aggregation
+execute: --enable_analyzer=1 --stage with_mergeable_state_after_aggregation
 "1_UInt8"
 1
-execute: --allow_experimental_analyzer=1 --stage complete
+execute: --enable_analyzer=1 --stage complete
 "foo"
 1
diff --git a/tests/queries/0_stateless/02048_clickhouse_local_stage.sh b/tests/queries/0_stateless/02048_clickhouse_local_stage.sh
index 182acc23a13..09a7e8efefc 100755
--- a/tests/queries/0_stateless/02048_clickhouse_local_stage.sh
+++ b/tests/queries/0_stateless/02048_clickhouse_local_stage.sh
@@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CURDIR"/../shell_config.sh
 
 opts=(
-    "--allow_experimental_analyzer=1"
+    "--enable_analyzer=1"
 )
 
 function execute_query()
diff --git a/tests/queries/0_stateless/02115_map_contains_analyzer.sql b/tests/queries/0_stateless/02115_map_contains_analyzer.sql
index 46e02eca4f0..00285404987 100644
--- a/tests/queries/0_stateless/02115_map_contains_analyzer.sql
+++ b/tests/queries/0_stateless/02115_map_contains_analyzer.sql
@@ -5,7 +5,7 @@ CREATE TABLE t_map_contains (m Map(String, UInt32)) ENGINE = Memory;
 INSERT INTO t_map_contains VALUES (map('a', 1, 'b', 2)), (map('c', 3, 'd', 4));
 
 SET optimize_functions_to_subcolumns = 1;
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 EXPLAIN QUERY TREE dump_tree = 0, dump_ast = 1 SELECT mapContains(m, 'a') FROM t_map_contains;
 SELECT mapContains(m, 'a') FROM t_map_contains;
diff --git a/tests/queries/0_stateless/02116_tuple_element_analyzer.sql b/tests/queries/0_stateless/02116_tuple_element_analyzer.sql
index 5aeb72c9ee4..ef3729bdc95 100644
--- a/tests/queries/0_stateless/02116_tuple_element_analyzer.sql
+++ b/tests/queries/0_stateless/02116_tuple_element_analyzer.sql
@@ -4,7 +4,7 @@ CREATE TABLE t_tuple_element(t1 Tuple(a UInt32, s String), t2 Tuple(UInt32, Stri
 INSERT INTO t_tuple_element VALUES ((1, 'a'), (2, 'b'));
 
 SET optimize_functions_to_subcolumns = 1;
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 SELECT t1.1 FROM t_tuple_element;
 EXPLAIN QUERY TREE dump_tree = 0, dump_ast = 1 SELECT t1.1 FROM t_tuple_element;
diff --git a/tests/queries/0_stateless/02125_query_views_log_window_function.sql b/tests/queries/0_stateless/02125_query_views_log_window_function.sql
index fff1e943c58..3f15cf5b998 100644
--- a/tests/queries/0_stateless/02125_query_views_log_window_function.sql
+++ b/tests/queries/0_stateless/02125_query_views_log_window_function.sql
@@ -1,4 +1,4 @@
-set allow_experimental_analyzer = 0;
+set enable_analyzer = 0;
 set allow_experimental_window_view = 1;
 
 CREATE TABLE data ( `id` UInt64, `timestamp` DateTime) ENGINE = Memory;
diff --git a/tests/queries/0_stateless/02129_window_functions_disable_optimizations.sql b/tests/queries/0_stateless/02129_window_functions_disable_optimizations.sql
index 0f12bc2eb9c..20a4f2bcf5f 100644
--- a/tests/queries/0_stateless/02129_window_functions_disable_optimizations.sql
+++ b/tests/queries/0_stateless/02129_window_functions_disable_optimizations.sql
@@ -29,4 +29,4 @@ HAVING sum(log(2) * number) > 346.57353 ORDER BY k;
 SELECT round(sum(log(2) * number), 6) AS k FROM numbers(10000)
 GROUP BY (number % 2) * (number % 3), number % 3, number % 2
 HAVING sum(log(2) * number) > 346.57353 ORDER BY k
-SETTINGS allow_experimental_analyzer=1;
+SETTINGS enable_analyzer=1;
diff --git a/tests/queries/0_stateless/02136_scalar_read_rows_json.sh b/tests/queries/0_stateless/02136_scalar_read_rows_json.sh
index 1fe345d266d..00f2c92161f 100755
--- a/tests/queries/0_stateless/02136_scalar_read_rows_json.sh
+++ b/tests/queries/0_stateless/02136_scalar_read_rows_json.sh
@@ -7,4 +7,4 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 echo "#1"
 ${CLICKHOUSE_CLIENT} --query='SELECT count() FROM numbers(100) FORMAT JSON;' | grep -a -v "elapsed"
 echo "#2"
-${CLICKHOUSE_CLIENT} --query='SELECT (SELECT max(number), count(number) FROM numbers(100000) as n) SETTINGS max_block_size = 65505, allow_experimental_analyzer = 1 FORMAT JSON;' | grep -a -v "elapsed" | grep -v "_subquery"
+${CLICKHOUSE_CLIENT} --query='SELECT (SELECT max(number), count(number) FROM numbers(100000) as n) SETTINGS max_block_size = 65505, enable_analyzer = 1 FORMAT JSON;' | grep -a -v "elapsed" | grep -v "_subquery"
diff --git a/tests/queries/0_stateless/02149_read_in_order_fixed_prefix.sql b/tests/queries/0_stateless/02149_read_in_order_fixed_prefix.sql
index ae8c39b49bc..7bbdecf5501 100644
--- a/tests/queries/0_stateless/02149_read_in_order_fixed_prefix.sql
+++ b/tests/queries/0_stateless/02149_read_in_order_fixed_prefix.sql
@@ -22,20 +22,20 @@ SELECT toStartOfMonth(date) as d, i FROM t_read_in_order ORDER BY d, -i LIMIT 5;
 EXPLAIN PIPELINE SELECT toStartOfMonth(date) as d, i FROM t_read_in_order ORDER BY d, -i LIMIT 5;
 
 SELECT date, i FROM t_read_in_order WHERE date = '2020-10-11' ORDER BY i LIMIT 5;
-EXPLAIN PIPELINE SELECT date, i FROM t_read_in_order WHERE date = '2020-10-11' ORDER BY i LIMIT 5 settings allow_experimental_analyzer=0;
-EXPLAIN PIPELINE SELECT date, i FROM t_read_in_order WHERE date = '2020-10-11' ORDER BY i LIMIT 5 settings allow_experimental_analyzer=1;
+EXPLAIN PIPELINE SELECT date, i FROM t_read_in_order WHERE date = '2020-10-11' ORDER BY i LIMIT 5 settings enable_analyzer=0;
+EXPLAIN PIPELINE SELECT date, i FROM t_read_in_order WHERE date = '2020-10-11' ORDER BY i LIMIT 5 settings enable_analyzer=1;
 
 SELECT * FROM t_read_in_order WHERE date = '2020-10-11' ORDER BY i, v LIMIT 5;
-EXPLAIN PIPELINE SELECT * FROM t_read_in_order WHERE date = '2020-10-11' ORDER BY i, v LIMIT 5 settings allow_experimental_analyzer=0;
-EXPLAIN PIPELINE SELECT * FROM t_read_in_order WHERE date = '2020-10-11' ORDER BY i, v LIMIT 5 settings allow_experimental_analyzer=1;
+EXPLAIN PIPELINE SELECT * FROM t_read_in_order WHERE date = '2020-10-11' ORDER BY i, v LIMIT 5 settings enable_analyzer=0;
+EXPLAIN PIPELINE SELECT * FROM t_read_in_order WHERE date = '2020-10-11' ORDER BY i, v LIMIT 5 settings enable_analyzer=1;
 
 INSERT INTO t_read_in_order SELECT '2020-10-12', number, number FROM numbers(100000);
 
 SELECT date, i FROM t_read_in_order WHERE date = '2020-10-12' ORDER BY i LIMIT 5;
 
 EXPLAIN SYNTAX SELECT date, i FROM t_read_in_order WHERE date = '2020-10-12' ORDER BY i DESC LIMIT 5;
-EXPLAIN PIPELINE SELECT date, i FROM t_read_in_order WHERE date = '2020-10-12' ORDER BY i DESC LIMIT 5 settings allow_experimental_analyzer=0;
-EXPLAIN PIPELINE SELECT date, i FROM t_read_in_order WHERE date = '2020-10-12' ORDER BY i DESC LIMIT 5 settings allow_experimental_analyzer=1;
+EXPLAIN PIPELINE SELECT date, i FROM t_read_in_order WHERE date = '2020-10-12' ORDER BY i DESC LIMIT 5 settings enable_analyzer=0;
+EXPLAIN PIPELINE SELECT date, i FROM t_read_in_order WHERE date = '2020-10-12' ORDER BY i DESC LIMIT 5 settings enable_analyzer=1;
 SELECT date, i FROM t_read_in_order WHERE date = '2020-10-12' ORDER BY i DESC LIMIT 5;
 
 DROP TABLE IF EXISTS t_read_in_order;
diff --git a/tests/queries/0_stateless/02151_hash_table_sizes_stats_joins.sh b/tests/queries/0_stateless/02151_hash_table_sizes_stats_joins.sh
index 6d715604d93..007dae6e427 100755
--- a/tests/queries/0_stateless/02151_hash_table_sizes_stats_joins.sh
+++ b/tests/queries/0_stateless/02151_hash_table_sizes_stats_joins.sh
@@ -58,7 +58,7 @@ $CLICKHOUSE_CLIENT -q "SYSTEM FLUSH LOGS"
 for i in "${!queries_without_preallocation[@]}"; do
   $CLICKHOUSE_CLIENT --param_query_id="${queries_without_preallocation[$i]}" -q "
     -- the old analyzer is not supported
-    SELECT sum(if(getSetting('allow_experimental_analyzer'), ProfileEvents['HashJoinPreallocatedElementsInHashTables'] = 0, 1))
+    SELECT sum(if(getSetting('enable_analyzer'), ProfileEvents['HashJoinPreallocatedElementsInHashTables'] = 0, 1))
       FROM system.query_log
      WHERE event_date >= yesterday() AND query_id = {query_id:String} AND current_database = currentDatabase() AND type = 'QueryFinish'
   "
@@ -67,7 +67,7 @@ done
 for i in "${!queries_with_preallocation[@]}"; do
   $CLICKHOUSE_CLIENT --param_query_id="${queries_with_preallocation[$i]}" -q "
     -- the old analyzer is not supported
-    SELECT sum(if(getSetting('allow_experimental_analyzer'), ProfileEvents['HashJoinPreallocatedElementsInHashTables'] > 0, 1))
+    SELECT sum(if(getSetting('enable_analyzer'), ProfileEvents['HashJoinPreallocatedElementsInHashTables'] > 0, 1))
       FROM system.query_log
      WHERE event_date >= yesterday() AND query_id = {query_id:String} AND current_database = currentDatabase() AND type = 'QueryFinish'
   "
diff --git a/tests/queries/0_stateless/02154_dictionary_get_http_json.sh b/tests/queries/0_stateless/02154_dictionary_get_http_json.sh
index fbaf67fff2f..bcd9f4f5c1a 100755
--- a/tests/queries/0_stateless/02154_dictionary_get_http_json.sh
+++ b/tests/queries/0_stateless/02154_dictionary_get_http_json.sh
@@ -32,7 +32,7 @@ $CLICKHOUSE_CLIENT -q """
 
 echo """
     SELECT dictGet(02154_test_dictionary, 'value', toUInt64(0)), dictGet(02154_test_dictionary, 'value', toUInt64(1))
-    SETTINGS allow_experimental_analyzer = 1
+    SETTINGS enable_analyzer = 1
     FORMAT JSON
 """ | ${CLICKHOUSE_CURL} -sSg "${CLICKHOUSE_URL}&wait_end_of_query=1&output_format_write_statistics=0" -d @-
 
diff --git a/tests/queries/0_stateless/02156_storage_merge_prewhere.sql b/tests/queries/0_stateless/02156_storage_merge_prewhere.sql
index 4f010ebadfd..2e5066fc488 100644
--- a/tests/queries/0_stateless/02156_storage_merge_prewhere.sql
+++ b/tests/queries/0_stateless/02156_storage_merge_prewhere.sql
@@ -24,8 +24,8 @@ INSERT INTO t_02156_mt1 SELECT number, toString(number) FROM numbers(10000);
 INSERT INTO t_02156_mt2 SELECT number, toString(number) FROM numbers(10000);
 INSERT INTO t_02156_log SELECT number, toString(number) FROM numbers(10000);
 
-SELECT replaceRegexpAll(explain, '__table1\.|_UInt8', '') FROM (EXPLAIN actions=1 SELECT count() FROM t_02156_merge1 WHERE k = 3 AND notEmpty(v)) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%' settings allow_experimental_analyzer=1;
-SELECT replaceRegexpAll(explain, '__table1\.|_UInt8', '') FROM (EXPLAIN actions=1 SELECT count() FROM t_02156_merge1 WHERE k = 3 AND notEmpty(v)) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%' settings allow_experimental_analyzer=0;
+SELECT replaceRegexpAll(explain, '__table1\.|_UInt8', '') FROM (EXPLAIN actions=1 SELECT count() FROM t_02156_merge1 WHERE k = 3 AND notEmpty(v)) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%' settings enable_analyzer=1;
+SELECT replaceRegexpAll(explain, '__table1\.|_UInt8', '') FROM (EXPLAIN actions=1 SELECT count() FROM t_02156_merge1 WHERE k = 3 AND notEmpty(v)) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%' settings enable_analyzer=0;
 SELECT count() FROM t_02156_merge1 WHERE k = 3 AND notEmpty(v);
 
 SELECT replaceRegexpAll(explain, '__table1\.|_UInt8', '') FROM (EXPLAIN actions=1 SELECT count() FROM t_02156_merge2 WHERE k = 3 AND notEmpty(v)) WHERE explain LIKE '%Prewhere%' OR explain LIKE '%Filter column%';
diff --git a/tests/queries/0_stateless/02174_cte_scalar_cache.sql b/tests/queries/0_stateless/02174_cte_scalar_cache.sql
index 86cfff21446..d14475c843a 100644
--- a/tests/queries/0_stateless/02174_cte_scalar_cache.sql
+++ b/tests/queries/0_stateless/02174_cte_scalar_cache.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 WITH
     ( SELECT sleep(0.0001) FROM system.one ) as a1,
@@ -28,7 +28,7 @@ WITH
     ( SELECT sleep(0.0001) FROM system.one ) as a5
 SELECT '02177_CTE_NEW_ANALYZER', a1, a2, a3, a4, a5 FROM system.numbers LIMIT 100
         FORMAT Null
-SETTINGS allow_experimental_analyzer = 1;
+SETTINGS enable_analyzer = 1;
 
 SYSTEM FLUSH LOGS;
 SELECT
diff --git a/tests/queries/0_stateless/02174_cte_scalar_cache_mv.sql b/tests/queries/0_stateless/02174_cte_scalar_cache_mv.sql
index ca54b9e1400..a2be0a11faf 100644
--- a/tests/queries/0_stateless/02174_cte_scalar_cache_mv.sql
+++ b/tests/queries/0_stateless/02174_cte_scalar_cache_mv.sql
@@ -14,7 +14,7 @@ CREATE MATERIALIZED VIEW mv1 TO t2 AS
     FROM t1
     LIMIT 5;
 
-set allow_experimental_analyzer = 0;
+set enable_analyzer = 0;
 
 -- FIRST INSERT
 INSERT INTO t1
@@ -61,7 +61,7 @@ WHERE
   AND event_date >= yesterday() AND event_time > now() - interval 10 minute;
 
 truncate table t2;
-set allow_experimental_analyzer = 1;
+set enable_analyzer = 1;
 
 -- FIRST INSERT ANALYZER
 INSERT INTO t1
@@ -100,7 +100,7 @@ WHERE
 
 DROP TABLE mv1;
 
-set allow_experimental_analyzer = 0;
+set enable_analyzer = 0;
 
 CREATE TABLE t3 (z Int64) ENGINE = Memory;
 CREATE MATERIALIZED VIEW mv2 TO t3 AS
@@ -134,7 +134,7 @@ WHERE
   AND event_date >= yesterday() AND event_time > now() - interval 10 minute;
 
 truncate table t3;
-set allow_experimental_analyzer = 1;
+set enable_analyzer = 1;
 
 -- SECOND INSERT ANALYZER
 INSERT INTO t1
@@ -162,7 +162,7 @@ WHERE
 
 DROP TABLE mv2;
 
-set allow_experimental_analyzer = 0;
+set enable_analyzer = 0;
 
 CREATE TABLE t4 (z Int64) ENGINE = Memory;
 CREATE MATERIALIZED VIEW mv3 TO t4 AS
@@ -197,7 +197,7 @@ WHERE
   AND event_date >= yesterday() AND event_time > now() - interval 10 minute;
 
 truncate table t4;
-set allow_experimental_analyzer = 1;
+set enable_analyzer = 1;
 
 -- THIRD INSERT ANALYZER
 INSERT INTO t1
diff --git a/tests/queries/0_stateless/02184_hash_functions_and_ip_types.sql b/tests/queries/0_stateless/02184_hash_functions_and_ip_types.sql
index c90c8b90c9e..e7d1909cae6 100644
--- a/tests/queries/0_stateless/02184_hash_functions_and_ip_types.sql
+++ b/tests/queries/0_stateless/02184_hash_functions_and_ip_types.sql
@@ -1,6 +1,6 @@
 -- Tags: no-fasttest
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 SELECT
     toIPv4('1.2.3.4') AS ipv4,
diff --git a/tests/queries/0_stateless/02226_analyzer_or_like_combine.reference b/tests/queries/0_stateless/02226_analyzer_or_like_combine.reference
index 0ff24b39709..61dfd1c6779 100644
--- a/tests/queries/0_stateless/02226_analyzer_or_like_combine.reference
+++ b/tests/queries/0_stateless/02226_analyzer_or_like_combine.reference
@@ -40,7 +40,7 @@ QUERY id: 0
                     LIST id: 3, nodes: 1
                       CONSTANT id: 4, constant_value: \'Привет, World\', constant_value_type: String
                 CONSTANT id: 16, constant_value: \'world%\', constant_value_type: String
-  SETTINGS optimize_or_like_chain=0 allow_experimental_analyzer=1
+  SETTINGS optimize_or_like_chain=0 enable_analyzer=1
 SELECT materialize(\'Привет, World\') AS s
 WHERE multiMatchAny(s, [\'^hell\', \'(?i)привет\', \'(?i)^world\']) OR false
 SETTINGS optimize_or_like_chain = 1
@@ -68,7 +68,7 @@ QUERY id: 0
                       CONSTANT id: 4, constant_value: \'Привет, World\', constant_value_type: String
                 CONSTANT id: 10, constant_value: Array_[\'^hell\', \'(?i)привет\', \'(?i)^world\'], constant_value_type: Array(String)
           CONSTANT id: 11, constant_value: UInt64_0, constant_value_type: UInt8
-  SETTINGS optimize_or_like_chain=1 allow_experimental_analyzer=1
+  SETTINGS optimize_or_like_chain=1 enable_analyzer=1
 SELECT
     materialize(\'Привет, World\') AS s1,
     materialize(\'Привет, World\') AS s2
diff --git a/tests/queries/0_stateless/02226_analyzer_or_like_combine.sql b/tests/queries/0_stateless/02226_analyzer_or_like_combine.sql
index fbebfc6d281..b23e5640b8f 100644
--- a/tests/queries/0_stateless/02226_analyzer_or_like_combine.sql
+++ b/tests/queries/0_stateless/02226_analyzer_or_like_combine.sql
@@ -1,7 +1,7 @@
 EXPLAIN SYNTAX SELECT materialize('Привет, World') AS s WHERE (s LIKE 'hell%') OR (s ILIKE '%привет%') OR (s ILIKE 'world%') SETTINGS optimize_or_like_chain = 0;
-EXPLAIN QUERY TREE run_passes=1 SELECT materialize('Привет, World') AS s WHERE (s LIKE 'hell%') OR (s ILIKE '%привет%') OR (s ILIKE 'world%') SETTINGS optimize_or_like_chain = 0, allow_experimental_analyzer = 1;
+EXPLAIN QUERY TREE run_passes=1 SELECT materialize('Привет, World') AS s WHERE (s LIKE 'hell%') OR (s ILIKE '%привет%') OR (s ILIKE 'world%') SETTINGS optimize_or_like_chain = 0, enable_analyzer = 1;
 EXPLAIN SYNTAX SELECT materialize('Привет, World') AS s WHERE (s LIKE 'hell%') OR (s ILIKE '%привет%') OR (s ILIKE 'world%') SETTINGS optimize_or_like_chain = 1;
-EXPLAIN QUERY TREE run_passes=1 SELECT materialize('Привет, World') AS s WHERE (s LIKE 'hell%') OR (s ILIKE '%привет%') OR (s ILIKE 'world%') SETTINGS optimize_or_like_chain = 1, allow_experimental_analyzer = 1;
+EXPLAIN QUERY TREE run_passes=1 SELECT materialize('Привет, World') AS s WHERE (s LIKE 'hell%') OR (s ILIKE '%привет%') OR (s ILIKE 'world%') SETTINGS optimize_or_like_chain = 1, enable_analyzer = 1;
 
 EXPLAIN SYNTAX SELECT materialize('Привет, World') AS s1, materialize('Привет, World') AS s2 WHERE (s1 LIKE 'hell%') OR (s2 ILIKE '%привет%') OR (s1 ILIKE 'world%') SETTINGS optimize_or_like_chain = 1;
 EXPLAIN SYNTAX SELECT materialize('Привет, World') AS s1, materialize('Привет, World') AS s2 WHERE (s1 LIKE 'hell%') OR (s2 ILIKE '%привет%') OR (s1 ILIKE 'world%') SETTINGS optimize_or_like_chain = 1, allow_hyperscan = 0;
@@ -11,18 +11,18 @@ EXPLAIN SYNTAX SELECT materialize('Привет, World') AS s1, materialize('П
 
 
 SELECT materialize('Привет, optimized World') AS s WHERE (s LIKE 'hell%') OR (s LIKE '%привет%') OR (s ILIKE '%world') SETTINGS optimize_or_like_chain = 1;
-SELECT materialize('Привет, optimized World') AS s WHERE (s LIKE 'hell%') OR (s LIKE '%привет%') OR (s ILIKE '%world') SETTINGS optimize_or_like_chain = 1, allow_experimental_analyzer = 1;
+SELECT materialize('Привет, optimized World') AS s WHERE (s LIKE 'hell%') OR (s LIKE '%привет%') OR (s ILIKE '%world') SETTINGS optimize_or_like_chain = 1, enable_analyzer = 1;
 
 SELECT materialize('Привет, World') AS s WHERE (s LIKE 'hell%') OR (s LIKE '%привет%') OR (s ILIKE '%world') SETTINGS optimize_or_like_chain = 0;
-SELECT materialize('Привет, World') AS s WHERE (s LIKE 'hell%') OR (s LIKE '%привет%') OR (s ILIKE '%world') SETTINGS optimize_or_like_chain = 0, allow_experimental_analyzer = 1;
+SELECT materialize('Привет, World') AS s WHERE (s LIKE 'hell%') OR (s LIKE '%привет%') OR (s ILIKE '%world') SETTINGS optimize_or_like_chain = 0, enable_analyzer = 1;
 
 SELECT materialize('Привет, optimized World') AS s WHERE (s LIKE 'hell%') OR (s ILIKE '%привет%') OR (s LIKE 'world%') SETTINGS optimize_or_like_chain = 1;
-SELECT materialize('Привет, optimized World') AS s WHERE (s LIKE 'hell%') OR (s ILIKE '%привет%') OR (s LIKE 'world%') SETTINGS optimize_or_like_chain = 1, allow_experimental_analyzer = 1;
+SELECT materialize('Привет, optimized World') AS s WHERE (s LIKE 'hell%') OR (s ILIKE '%привет%') OR (s LIKE 'world%') SETTINGS optimize_or_like_chain = 1, enable_analyzer = 1;
 
 SELECT materialize('Привет, World') AS s WHERE (s LIKE 'hell%') OR (s ILIKE '%привет%') OR (s LIKE 'world%') SETTINGS optimize_or_like_chain = 0;
-SELECT materialize('Привет, World') AS s WHERE (s LIKE 'hell%') OR (s ILIKE '%привет%') OR (s LIKE 'world%') SETTINGS optimize_or_like_chain = 0, allow_experimental_analyzer = 1;
+SELECT materialize('Привет, World') AS s WHERE (s LIKE 'hell%') OR (s ILIKE '%привет%') OR (s LIKE 'world%') SETTINGS optimize_or_like_chain = 0, enable_analyzer = 1;
 
-SELECT materialize('Привет, World') AS s WHERE (s LIKE 'hell%') OR (s ILIKE '%привет%') OR (s ILIKE 'world%') SETTINGS optimize_or_like_chain = 1, allow_experimental_analyzer = 1;
+SELECT materialize('Привет, World') AS s WHERE (s LIKE 'hell%') OR (s ILIKE '%привет%') OR (s ILIKE 'world%') SETTINGS optimize_or_like_chain = 1, enable_analyzer = 1;
 
 -- Aliases
 
diff --git a/tests/queries/0_stateless/02227_union_match_by_name.sql b/tests/queries/0_stateless/02227_union_match_by_name.sql
index 6a19add1d37..489c3d976ea 100644
--- a/tests/queries/0_stateless/02227_union_match_by_name.sql
+++ b/tests/queries/0_stateless/02227_union_match_by_name.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 -- { echoOn }
 
diff --git a/tests/queries/0_stateless/02233_with_total_empty_chunk.sql b/tests/queries/0_stateless/02233_with_total_empty_chunk.sql
index d59319ac75e..c70b35df459 100644
--- a/tests/queries/0_stateless/02233_with_total_empty_chunk.sql
+++ b/tests/queries/0_stateless/02233_with_total_empty_chunk.sql
@@ -1,3 +1,3 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 SELECT (NULL, NULL, NULL, NULL, NULL, NULL, NULL) FROM numbers(0) GROUP BY number WITH TOTALS HAVING sum(number) <= arrayJoin([]) -- { serverError ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER };
diff --git a/tests/queries/0_stateless/02234_clickhouse_local_test_mode.sh b/tests/queries/0_stateless/02234_clickhouse_local_test_mode.sh
index a6c47d80fa9..d387fda746c 100755
--- a/tests/queries/0_stateless/02234_clickhouse_local_test_mode.sh
+++ b/tests/queries/0_stateless/02234_clickhouse_local_test_mode.sh
@@ -4,6 +4,5 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
-
-$CLICKHOUSE_LOCAL --query="SELECT n SETTINGS allow_experimental_analyzer = 1" 2>&1 | grep -q "Code: 47. DB::Exception:" && echo 'OK' || echo 'FAIL'
-$CLICKHOUSE_LOCAL --query="SELECT n SETTINGS allow_experimental_analyzer = 0" 2>&1 | grep -q "Code: 47. DB::Exception:" && echo 'OK' || echo 'FAIL'
+$CLICKHOUSE_LOCAL --query="SELECT n SETTINGS enable_analyzer = 1" 2>&1 | grep -q "Code: 47. DB::Exception:" && echo 'OK' || echo 'FAIL'
+$CLICKHOUSE_LOCAL --query="SELECT n SETTINGS enable_analyzer = 0" 2>&1 | grep -q "Code: 47. DB::Exception:" && echo 'OK' || echo 'FAIL'
diff --git a/tests/queries/0_stateless/02267_join_dup_columns_issue36199.sql b/tests/queries/0_stateless/02267_join_dup_columns_issue36199.sql
index fbcc374ba10..ecb2198237f 100644
--- a/tests/queries/0_stateless/02267_join_dup_columns_issue36199.sql
+++ b/tests/queries/0_stateless/02267_join_dup_columns_issue36199.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 0;
+SET enable_analyzer = 0;
 
 SET join_algorithm = 'hash';
 
@@ -22,7 +22,7 @@ SELECT * FROM ( SELECT count('a'), count('b'), count('c'), 2 AS x ) as t1 RIGHT
 SELECT 'y', * FROM (SELECT count('y'), count('y'), 2 AS x) AS t1 RIGHT JOIN (SELECT count('x'), count('y'), 3 AS x) AS t2 ON t1.x = t2.x;
 SELECT * FROM (SELECT arrayJoin([NULL]), 9223372036854775806, arrayJoin([NULL]), NULL AS x) AS t1 RIGHT JOIN (SELECT arrayJoin([arrayJoin([10000000000.])]), NULL AS x) AS t2 ON t1.x = t2.x;
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 SET join_algorithm = 'hash';
 
 SELECT * FROM ( SELECT 2 AS x ) AS t1 RIGHT JOIN ( SELECT count('x'), count('y'), 2 AS x ) AS t2 ON t1.x = t2.x;
diff --git a/tests/queries/0_stateless/02271_fix_column_matcher_and_column_transformer.sql b/tests/queries/0_stateless/02271_fix_column_matcher_and_column_transformer.sql
index f8faa3e653b..ab89d98c608 100644
--- a/tests/queries/0_stateless/02271_fix_column_matcher_and_column_transformer.sql
+++ b/tests/queries/0_stateless/02271_fix_column_matcher_and_column_transformer.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS github_events;
 
diff --git a/tests/queries/0_stateless/02303_query_kind.reference b/tests/queries/0_stateless/02303_query_kind.reference
index 9f1c026f889..c26242098e3 100644
--- a/tests/queries/0_stateless/02303_query_kind.reference
+++ b/tests/queries/0_stateless/02303_query_kind.reference
@@ -1,4 +1,4 @@
-clickhouse-client --allow_experimental_analyzer=1 --query_kind secondary_query -q explain plan header=1 select toString(dummy) as dummy from system.one group by dummy
+clickhouse-client --enable_analyzer=1 --query_kind secondary_query -q explain plan header=1 select toString(dummy) as dummy from system.one group by dummy
 Expression ((Project names + Projection))
 Header: dummy String
   Aggregating
@@ -7,7 +7,7 @@ Header: dummy String
     Header: toString(__table1.dummy) String
       ReadFromStorage (SystemOne)
       Header: dummy UInt8
-clickhouse-local --allow_experimental_analyzer=1 --query_kind secondary_query -q explain plan header=1 select toString(dummy) as dummy from system.one group by dummy
+clickhouse-local --enable_analyzer=1 --query_kind secondary_query -q explain plan header=1 select toString(dummy) as dummy from system.one group by dummy
 Expression ((Project names + Projection))
 Header: dummy String
   Aggregating
@@ -16,7 +16,7 @@ Header: dummy String
     Header: toString(__table1.dummy) String
       ReadFromStorage (SystemOne)
       Header: dummy UInt8
-clickhouse-client --allow_experimental_analyzer=1 --query_kind initial_query -q explain plan header=1 select toString(dummy) as dummy from system.one group by dummy
+clickhouse-client --enable_analyzer=1 --query_kind initial_query -q explain plan header=1 select toString(dummy) as dummy from system.one group by dummy
 Expression ((Project names + Projection))
 Header: dummy String
   Aggregating
@@ -25,7 +25,7 @@ Header: dummy String
     Header: __table1.dummy UInt8
       ReadFromStorage (SystemOne)
       Header: dummy UInt8
-clickhouse-local --allow_experimental_analyzer=1 --query_kind initial_query -q explain plan header=1 select toString(dummy) as dummy from system.one group by dummy
+clickhouse-local --enable_analyzer=1 --query_kind initial_query -q explain plan header=1 select toString(dummy) as dummy from system.one group by dummy
 Expression ((Project names + Projection))
 Header: dummy String
   Aggregating
diff --git a/tests/queries/0_stateless/02303_query_kind.sh b/tests/queries/0_stateless/02303_query_kind.sh
index 1d883a2dcc7..7fe491b9deb 100755
--- a/tests/queries/0_stateless/02303_query_kind.sh
+++ b/tests/queries/0_stateless/02303_query_kind.sh
@@ -5,7 +5,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CUR_DIR"/../shell_config.sh
 
 opts=(
-    "--allow_experimental_analyzer=1"
+    "--enable_analyzer=1"
 )
 
 function run_query()
diff --git a/tests/queries/0_stateless/02315_grouping_constant_folding.reference b/tests/queries/0_stateless/02315_grouping_constant_folding.reference
index 31816318a42..7b8c75b2304 100644
--- a/tests/queries/0_stateless/02315_grouping_constant_folding.reference
+++ b/tests/queries/0_stateless/02315_grouping_constant_folding.reference
@@ -27,7 +27,7 @@ SELECT count() AS amount, a, b, GROUPING(a, b) FROM test02315 GROUP BY ROLLUP(a,
 5	0	0	2
 5	1	0	2
 10	0	0	0
-SELECT count() AS amount, a, b, GROUPING(a, b) FROM test02315 GROUP BY GROUPING SETS ((a, b), (a, a), ()) ORDER BY (amount, a, b) SETTINGS force_grouping_standard_compatibility=0, allow_experimental_analyzer=1;
+SELECT count() AS amount, a, b, GROUPING(a, b) FROM test02315 GROUP BY GROUPING SETS ((a, b), (a, a), ()) ORDER BY (amount, a, b) SETTINGS force_grouping_standard_compatibility=0, enable_analyzer=1;
 1	0	0	3
 1	0	2	3
 1	0	4	3
diff --git a/tests/queries/0_stateless/02315_grouping_constant_folding.sql b/tests/queries/0_stateless/02315_grouping_constant_folding.sql
index f992aa0da32..5e305d2e6c5 100644
--- a/tests/queries/0_stateless/02315_grouping_constant_folding.sql
+++ b/tests/queries/0_stateless/02315_grouping_constant_folding.sql
@@ -9,7 +9,7 @@ SELECT count() AS amount, a, b, GROUPING(a, b) FROM test02315 GROUP BY GROUPING
 
 SELECT count() AS amount, a, b, GROUPING(a, b) FROM test02315 GROUP BY ROLLUP(a, b) ORDER BY (amount, a, b) SETTINGS force_grouping_standard_compatibility=0;
 
-SELECT count() AS amount, a, b, GROUPING(a, b) FROM test02315 GROUP BY GROUPING SETS ((a, b), (a, a), ()) ORDER BY (amount, a, b) SETTINGS force_grouping_standard_compatibility=0, allow_experimental_analyzer=1;
+SELECT count() AS amount, a, b, GROUPING(a, b) FROM test02315 GROUP BY GROUPING SETS ((a, b), (a, a), ()) ORDER BY (amount, a, b) SETTINGS force_grouping_standard_compatibility=0, enable_analyzer=1;
 
 -- { echoOff }
 DROP TABLE test02315;
diff --git a/tests/queries/0_stateless/02317_distinct_in_order_optimization_explain.sh b/tests/queries/0_stateless/02317_distinct_in_order_optimization_explain.sh
index 07c7bc4af56..bd7e6be3987 100755
--- a/tests/queries/0_stateless/02317_distinct_in_order_optimization_explain.sh
+++ b/tests/queries/0_stateless/02317_distinct_in_order_optimization_explain.sh
@@ -77,7 +77,7 @@ echo "-- enabled, only part of distinct columns form prefix of sorting key"
 $CLICKHOUSE_CLIENT --max_threads=0 -nq "$ENABLE_OPTIMIZATION;explain pipeline select distinct a, c from distinct_in_order_explain" | eval $FIND_READING_DEFAULT
 
 echo "=== disable new analyzer ==="
-DISABLE_ANALYZER="set allow_experimental_analyzer=0"
+DISABLE_ANALYZER="set enable_analyzer=0"
 
 echo "-- enabled, check that sorting properties are propagated from ReadFromMergeTree till preliminary distinct"
 $CLICKHOUSE_CLIENT -nq "$DISABLE_ANALYZER;$ENABLE_OPTIMIZATION;explain plan sorting=1 select distinct b, a from distinct_in_order_explain where a > 0" | eval $FIND_SORTING_PROPERTIES
@@ -99,7 +99,7 @@ echo "-- enabled, check that disabling other 'read in order' optimizations do no
 $CLICKHOUSE_CLIENT -nq "$DISABLE_ANALYZER;$ENABLE_OPTIMIZATION;set optimize_read_in_order=0;set optimize_aggregation_in_order=0;set optimize_read_in_window_order=0;explain plan sorting=1 select distinct a,b from distinct_in_order_explain" | eval $FIND_SORTING_PROPERTIES
 
 echo "=== enable new analyzer ==="
-ENABLE_ANALYZER="set allow_experimental_analyzer=1"
+ENABLE_ANALYZER="set enable_analyzer=1"
 
 echo "-- enabled, check that sorting properties are propagated from ReadFromMergeTree till preliminary distinct"
 $CLICKHOUSE_CLIENT -nq "$ENABLE_ANALYZER;$ENABLE_OPTIMIZATION;explain plan sorting=1 select distinct b, a from distinct_in_order_explain where a > 0 settings optimize_move_to_prewhere=1" | eval $FIND_SORTING_PROPERTIES
diff --git a/tests/queries/0_stateless/02337_analyzer_columns_basic.sql b/tests/queries/0_stateless/02337_analyzer_columns_basic.sql
index 167eecc6fb8..c132a69ac21 100644
--- a/tests/queries/0_stateless/02337_analyzer_columns_basic.sql
+++ b/tests/queries/0_stateless/02337_analyzer_columns_basic.sql
@@ -1,6 +1,6 @@
 -- Tags: no-parallel
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 -- Empty from section
 
diff --git a/tests/queries/0_stateless/02337_multiple_joins_original_names.sql b/tests/queries/0_stateless/02337_multiple_joins_original_names.sql
index 63bbfe6873d..37c7077b56e 100644
--- a/tests/queries/0_stateless/02337_multiple_joins_original_names.sql
+++ b/tests/queries/0_stateless/02337_multiple_joins_original_names.sql
@@ -1,6 +1,6 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/34697
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 SELECT table1_id FROM (
     SELECT first.table1_id
diff --git a/tests/queries/0_stateless/02338_analyzer_constants_basic.sql b/tests/queries/0_stateless/02338_analyzer_constants_basic.sql
index 6d6249538a4..536202dc2cf 100644
--- a/tests/queries/0_stateless/02338_analyzer_constants_basic.sql
+++ b/tests/queries/0_stateless/02338_analyzer_constants_basic.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DESCRIBE (SELECT 1);
 SELECT 1;
diff --git a/tests/queries/0_stateless/02339_analyzer_matcher_basic.sql b/tests/queries/0_stateless/02339_analyzer_matcher_basic.sql
index a09325fa43b..57c6a9479ec 100644
--- a/tests/queries/0_stateless/02339_analyzer_matcher_basic.sql
+++ b/tests/queries/0_stateless/02339_analyzer_matcher_basic.sql
@@ -1,6 +1,6 @@
 -- Tags: no-parallel
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 SELECT 'Matchers without FROM section';
 
diff --git a/tests/queries/0_stateless/02340_analyzer_functions.sql b/tests/queries/0_stateless/02340_analyzer_functions.sql
index 101a5bfcc86..bd018302913 100644
--- a/tests/queries/0_stateless/02340_analyzer_functions.sql
+++ b/tests/queries/0_stateless/02340_analyzer_functions.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DESCRIBE (SELECT 1 + 1);
 SELECT 1 + 1;
diff --git a/tests/queries/0_stateless/02341_analyzer_aliases_basics.sql b/tests/queries/0_stateless/02341_analyzer_aliases_basics.sql
index 9f21db8e659..8bed0c8bc1e 100644
--- a/tests/queries/0_stateless/02341_analyzer_aliases_basics.sql
+++ b/tests/queries/0_stateless/02341_analyzer_aliases_basics.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 SELECT 'Aliases to constants';
 
diff --git a/tests/queries/0_stateless/02341_global_join_cte.reference b/tests/queries/0_stateless/02341_global_join_cte.reference
index f2cfe994ffa..4f854ca3bef 100644
--- a/tests/queries/0_stateless/02341_global_join_cte.reference
+++ b/tests/queries/0_stateless/02341_global_join_cte.reference
@@ -1,6 +1,6 @@
 -- { echo }
-with rhs as (select * from remote('127.{1,2}', view(select dummy d1, dummy d2 from system.one))) select lhs.d2 from remote('127.{1,2}', view(select dummy d1, dummy d2 from system.one)) lhs global join rhs using (d1) order by rhs.d2 settings allow_experimental_analyzer=0; -- { serverError ALIAS_REQUIRED }
-with rhs as (select * from remote('127.{1,2}', view(select dummy d1, dummy d2 from system.one))) select lhs.d2 from remote('127.{1,2}', view(select dummy d1, dummy d2 from system.one)) lhs global join rhs using (d1) order by rhs.d2 settings allow_experimental_analyzer=1; -- It works with analyzer; rhs is an alias itself.
+with rhs as (select * from remote('127.{1,2}', view(select dummy d1, dummy d2 from system.one))) select lhs.d2 from remote('127.{1,2}', view(select dummy d1, dummy d2 from system.one)) lhs global join rhs using (d1) order by rhs.d2 settings enable_analyzer=0; -- { serverError ALIAS_REQUIRED }
+with rhs as (select * from remote('127.{1,2}', view(select dummy d1, dummy d2 from system.one))) select lhs.d2 from remote('127.{1,2}', view(select dummy d1, dummy d2 from system.one)) lhs global join rhs using (d1) order by rhs.d2 settings enable_analyzer=1; -- It works with analyzer; rhs is an alias itself.
 0
 0
 0
diff --git a/tests/queries/0_stateless/02341_global_join_cte.sql b/tests/queries/0_stateless/02341_global_join_cte.sql
index b9b906afd70..f6acd822f14 100644
--- a/tests/queries/0_stateless/02341_global_join_cte.sql
+++ b/tests/queries/0_stateless/02341_global_join_cte.sql
@@ -1,5 +1,5 @@
 -- { echo }
-with rhs as (select * from remote('127.{1,2}', view(select dummy d1, dummy d2 from system.one))) select lhs.d2 from remote('127.{1,2}', view(select dummy d1, dummy d2 from system.one)) lhs global join rhs using (d1) order by rhs.d2 settings allow_experimental_analyzer=0; -- { serverError ALIAS_REQUIRED }
-with rhs as (select * from remote('127.{1,2}', view(select dummy d1, dummy d2 from system.one))) select lhs.d2 from remote('127.{1,2}', view(select dummy d1, dummy d2 from system.one)) lhs global join rhs using (d1) order by rhs.d2 settings allow_experimental_analyzer=1; -- It works with analyzer; rhs is an alias itself.
+with rhs as (select * from remote('127.{1,2}', view(select dummy d1, dummy d2 from system.one))) select lhs.d2 from remote('127.{1,2}', view(select dummy d1, dummy d2 from system.one)) lhs global join rhs using (d1) order by rhs.d2 settings enable_analyzer=0; -- { serverError ALIAS_REQUIRED }
+with rhs as (select * from remote('127.{1,2}', view(select dummy d1, dummy d2 from system.one))) select lhs.d2 from remote('127.{1,2}', view(select dummy d1, dummy d2 from system.one)) lhs global join rhs using (d1) order by rhs.d2 settings enable_analyzer=1; -- It works with analyzer; rhs is an alias itself.
 with rhs as (select * from remote('127.{1,2}', view(select dummy d1, dummy d2 from system.one))) select lhs.d2 from remote('127.{1,2}', view(select dummy d1, dummy d2 from system.one)) lhs global join rhs using (d1) order by rhs.d2 settings joined_subquery_requires_alias=0;
 with rhs_ as (select * from remote('127.{1,2}', view(select dummy d1, dummy d2 from system.one))) select lhs.d2 from remote('127.{1,2}', view(select dummy d1, dummy d2 from system.one)) lhs global join rhs_ rhs using (d1) order by rhs.d2 settings joined_subquery_requires_alias=0;
diff --git a/tests/queries/0_stateless/02342_analyzer_compound_types.sql b/tests/queries/0_stateless/02342_analyzer_compound_types.sql
index 0fd96928496..36617aab2f8 100644
--- a/tests/queries/0_stateless/02342_analyzer_compound_types.sql
+++ b/tests/queries/0_stateless/02342_analyzer_compound_types.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 SELECT 'Constant tuple';
 
diff --git a/tests/queries/0_stateless/02342_window_view_different_struct.sql b/tests/queries/0_stateless/02342_window_view_different_struct.sql
index a5b2b8daa5a..9c35459ecef 100644
--- a/tests/queries/0_stateless/02342_window_view_different_struct.sql
+++ b/tests/queries/0_stateless/02342_window_view_different_struct.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 0;
+SET enable_analyzer = 0;
 SET allow_experimental_window_view = 1;
 
 DROP TABLE IF EXISTS data_02342;
diff --git a/tests/queries/0_stateless/02343_analyzer_column_transformers_strict.sql b/tests/queries/0_stateless/02343_analyzer_column_transformers_strict.sql
index 7e323c570b8..b55cb85c0da 100644
--- a/tests/queries/0_stateless/02343_analyzer_column_transformers_strict.sql
+++ b/tests/queries/0_stateless/02343_analyzer_column_transformers_strict.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS test_table;
 CREATE TABLE test_table
diff --git a/tests/queries/0_stateless/02343_analyzer_lambdas.sql b/tests/queries/0_stateless/02343_analyzer_lambdas.sql
index 80fa47fc325..07f382700ae 100644
--- a/tests/queries/0_stateless/02343_analyzer_lambdas.sql
+++ b/tests/queries/0_stateless/02343_analyzer_lambdas.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS test_table;
 CREATE TABLE test_table
diff --git a/tests/queries/0_stateless/02343_analyzer_lambdas_issue_28083.sql b/tests/queries/0_stateless/02343_analyzer_lambdas_issue_28083.sql
index 3b780e1dec3..dff0d7f82ca 100644
--- a/tests/queries/0_stateless/02343_analyzer_lambdas_issue_28083.sql
+++ b/tests/queries/0_stateless/02343_analyzer_lambdas_issue_28083.sql
@@ -1,11 +1,11 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 select so,
        r
 from
-    (select  [('y',0),('n',1)] as cg, 
+    (select  [('y',0),('n',1)] as cg,
              if( arrayMap( x -> x.1, cg ) != ['y', 'n'], 'y', 'n')  as so,
-             arrayFilter( x -> x.1 = so , cg) as r  
+             arrayFilter( x -> x.1 = so , cg) as r
     );
 
 select
diff --git a/tests/queries/0_stateless/02343_analyzer_lambdas_issue_36677.sql b/tests/queries/0_stateless/02343_analyzer_lambdas_issue_36677.sql
index b07f3f33ac3..da0b4e8ef57 100644
--- a/tests/queries/0_stateless/02343_analyzer_lambdas_issue_36677.sql
+++ b/tests/queries/0_stateless/02343_analyzer_lambdas_issue_36677.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 SELECT
     arraySum(x -> ((x.1) / ((x.2) * (x.2))), arrayZip(mag, magerr)) / arraySum(x -> (1. / (x * x)), magerr) AS weightedmeanmag,
diff --git a/tests/queries/0_stateless/02344_analyzer_multiple_aliases_for_expression.sql b/tests/queries/0_stateless/02344_analyzer_multiple_aliases_for_expression.sql
index ee02b79cc32..3c7ea467734 100644
--- a/tests/queries/0_stateless/02344_analyzer_multiple_aliases_for_expression.sql
+++ b/tests/queries/0_stateless/02344_analyzer_multiple_aliases_for_expression.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS test_table;
 CREATE TABLE test_table
diff --git a/tests/queries/0_stateless/02345_analyzer_subqueries.sql b/tests/queries/0_stateless/02345_analyzer_subqueries.sql
index c0cc242b57b..d1ec9b58e27 100644
--- a/tests/queries/0_stateless/02345_analyzer_subqueries.sql
+++ b/tests/queries/0_stateless/02345_analyzer_subqueries.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS test_table;
 CREATE TABLE test_table
diff --git a/tests/queries/0_stateless/02346_fulltext_index_bug52019.sql b/tests/queries/0_stateless/02346_fulltext_index_bug52019.sql
index a643df65095..e29c3c51e5e 100644
--- a/tests/queries/0_stateless/02346_fulltext_index_bug52019.sql
+++ b/tests/queries/0_stateless/02346_fulltext_index_bug52019.sql
@@ -13,8 +13,8 @@ ORDER BY k
 SETTINGS index_granularity = 2, index_granularity_bytes = '10Mi';
 
 INSERT INTO tab (k) VALUES (0);
-SELECT * FROM tab PREWHERE (s[NULL]) = 'Click a03' SETTINGS allow_experimental_analyzer=1;
-SELECT * FROM tab PREWHERE (s[1]) = 'Click a03' SETTINGS allow_experimental_analyzer=1; -- { serverError ILLEGAL_TYPE_OF_ARGUMENT }
-SELECT * FROM tab PREWHERE (s['foo']) = 'Click a03' SETTINGS allow_experimental_analyzer=1;
+SELECT * FROM tab PREWHERE (s[NULL]) = 'Click a03' SETTINGS enable_analyzer=1;
+SELECT * FROM tab PREWHERE (s[1]) = 'Click a03' SETTINGS enable_analyzer=1; -- { serverError ILLEGAL_TYPE_OF_ARGUMENT }
+SELECT * FROM tab PREWHERE (s['foo']) = 'Click a03' SETTINGS enable_analyzer=1;
 
 DROP TABLE tab;
diff --git a/tests/queries/0_stateless/02346_fulltext_index_match_predicate.sql b/tests/queries/0_stateless/02346_fulltext_index_match_predicate.sql
index 7f36c423a41..a20c8dc9afd 100644
--- a/tests/queries/0_stateless/02346_fulltext_index_match_predicate.sql
+++ b/tests/queries/0_stateless/02346_fulltext_index_match_predicate.sql
@@ -31,7 +31,7 @@ FROM
 WHERE
     explain LIKE '%Granules: %'
 SETTINGS
-    allow_experimental_analyzer = 0;
+    enable_analyzer = 0;
 
 SELECT *
 FROM
@@ -42,7 +42,7 @@ FROM
 WHERE
     explain LIKE '%Granules: %'
 SETTINGS
-    allow_experimental_analyzer = 1;
+    enable_analyzer = 1;
 
 SELECT '---';
 
@@ -61,7 +61,7 @@ FROM
 WHERE
     explain LIKE '%Granules: %'
 SETTINGS
-    allow_experimental_analyzer = 0;
+    enable_analyzer = 0;
 
 SELECT *
 FROM
@@ -72,7 +72,7 @@ FROM
 WHERE
     explain LIKE '%Granules: %'
 SETTINGS
-    allow_experimental_analyzer = 1;
+    enable_analyzer = 1;
 
 SELECT '---';
 
@@ -91,7 +91,7 @@ FROM
 WHERE
     explain LIKE '%Granules: %'
 SETTINGS
-    allow_experimental_analyzer = 0;
+    enable_analyzer = 0;
 
 SELECT *
 FROM
@@ -102,6 +102,6 @@ FROM
 WHERE
     explain LIKE '%Granules: %'
 SETTINGS
-    allow_experimental_analyzer = 1;
+    enable_analyzer = 1;
 
 DROP TABLE tab;
diff --git a/tests/queries/0_stateless/02354_vector_search_bugs.sql b/tests/queries/0_stateless/02354_vector_search_bugs.sql
index db0c06c4e16..f03c36f6550 100644
--- a/tests/queries/0_stateless/02354_vector_search_bugs.sql
+++ b/tests/queries/0_stateless/02354_vector_search_bugs.sql
@@ -8,7 +8,7 @@
 SET allow_experimental_annoy_index = 1;
 SET allow_experimental_usearch_index = 1;
 
-SET allow_experimental_analyzer = 1; -- 0 vs. 1 produce slightly different error codes, make it future-proof
+SET enable_analyzer = 1; -- 0 vs. 1 produce slightly different error codes, make it future-proof
 
 DROP TABLE IF EXISTS tab;
 
diff --git a/tests/queries/0_stateless/02364_window_view_segfault.sh b/tests/queries/0_stateless/02364_window_view_segfault.sh
index 4173b6c6b8d..833d8967558 100755
--- a/tests/queries/0_stateless/02364_window_view_segfault.sh
+++ b/tests/queries/0_stateless/02364_window_view_segfault.sh
@@ -6,7 +6,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CURDIR"/../shell_config.sh
 
 opts=(
-    "--allow_experimental_analyzer=0"
+    "--enable_analyzer=0"
 )
 
 ${CLICKHOUSE_CLIENT} "${opts[@]}" --multiline --query """
diff --git a/tests/queries/0_stateless/02366_explain_query_tree.sql b/tests/queries/0_stateless/02366_explain_query_tree.sql
index c38b2d819d1..82621ec90f7 100644
--- a/tests/queries/0_stateless/02366_explain_query_tree.sql
+++ b/tests/queries/0_stateless/02366_explain_query_tree.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 EXPLAIN QUERY TREE run_passes = 0 SELECT 1;
 
diff --git a/tests/queries/0_stateless/02367_analyzer_table_alias_columns.sql b/tests/queries/0_stateless/02367_analyzer_table_alias_columns.sql
index f41680cd9f4..a6408abca9b 100644
--- a/tests/queries/0_stateless/02367_analyzer_table_alias_columns.sql
+++ b/tests/queries/0_stateless/02367_analyzer_table_alias_columns.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS test_table;
 CREATE TABLE test_table
diff --git a/tests/queries/0_stateless/02368_analyzer_table_functions.sql b/tests/queries/0_stateless/02368_analyzer_table_functions.sql
index 456e095c6c1..cc65848a5d8 100644
--- a/tests/queries/0_stateless/02368_analyzer_table_functions.sql
+++ b/tests/queries/0_stateless/02368_analyzer_table_functions.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 SELECT c1, c2, c3, c4 FROM format('CSV', '1,2,"[1,2,3]","[[\'abc\'], [], [\'d\', \'e\']]"');
 SELECT f.c1, f.c2, f.c3, f.c4 FROM format('CSV', '1,2,"[1,2,3]","[[\'abc\'], [], [\'d\', \'e\']]"') AS f;
diff --git a/tests/queries/0_stateless/02369_analyzer_array_join_function.sql b/tests/queries/0_stateless/02369_analyzer_array_join_function.sql
index e60ec7e71a9..f84c96ee6a6 100644
--- a/tests/queries/0_stateless/02369_analyzer_array_join_function.sql
+++ b/tests/queries/0_stateless/02369_analyzer_array_join_function.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 SELECT arrayJoin([1, 2, 3]);
 
diff --git a/tests/queries/0_stateless/02370_analyzer_in_function.sql b/tests/queries/0_stateless/02370_analyzer_in_function.sql
index a6e4400e101..9fb05ada829 100644
--- a/tests/queries/0_stateless/02370_analyzer_in_function.sql
+++ b/tests/queries/0_stateless/02370_analyzer_in_function.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 SELECT 1 IN 1;
 SELECT 1 IN (1);
diff --git a/tests/queries/0_stateless/02371_analyzer_join_cross.sql b/tests/queries/0_stateless/02371_analyzer_join_cross.sql
index 3624a1d2282..660dc19770a 100644
--- a/tests/queries/0_stateless/02371_analyzer_join_cross.sql
+++ b/tests/queries/0_stateless/02371_analyzer_join_cross.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 SET single_join_prefer_left_table = 0;
 
 DROP TABLE IF EXISTS test_table_join_1;
diff --git a/tests/queries/0_stateless/02372_analyzer_join.sql.j2 b/tests/queries/0_stateless/02372_analyzer_join.sql.j2
index 45ae63b9a49..fb75ddeac09 100644
--- a/tests/queries/0_stateless/02372_analyzer_join.sql.j2
+++ b/tests/queries/0_stateless/02372_analyzer_join.sql.j2
@@ -1,6 +1,6 @@
 -- Tags: long
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 SET single_join_prefer_left_table = 0;
 
 DROP TABLE IF EXISTS test_table_join_1;
diff --git a/tests/queries/0_stateless/02373_analyzer_join_use_nulls.sql b/tests/queries/0_stateless/02373_analyzer_join_use_nulls.sql
index bcec6d178a8..16a37ee7f50 100644
--- a/tests/queries/0_stateless/02373_analyzer_join_use_nulls.sql
+++ b/tests/queries/0_stateless/02373_analyzer_join_use_nulls.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 SET join_use_nulls = 1;
 
 DROP TABLE IF EXISTS test_table_join_1;
diff --git a/tests/queries/0_stateless/02374_analyzer_array_join.sql b/tests/queries/0_stateless/02374_analyzer_array_join.sql
index 8c26df1806e..fe1508f5f78 100644
--- a/tests/queries/0_stateless/02374_analyzer_array_join.sql
+++ b/tests/queries/0_stateless/02374_analyzer_array_join.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS test_table;
 CREATE TABLE test_table
diff --git a/tests/queries/0_stateless/02374_analyzer_join_using.sql.j2 b/tests/queries/0_stateless/02374_analyzer_join_using.sql.j2
index 26fb52716ff..2c64efc7fc8 100644
--- a/tests/queries/0_stateless/02374_analyzer_join_using.sql.j2
+++ b/tests/queries/0_stateless/02374_analyzer_join_using.sql.j2
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS test_table_join_1;
 CREATE TABLE test_table_join_1
diff --git a/tests/queries/0_stateless/02374_combine_multi_if_and_count_if_opt.sql b/tests/queries/0_stateless/02374_combine_multi_if_and_count_if_opt.sql
index 4371c2e5641..05472e5e35a 100644
--- a/tests/queries/0_stateless/02374_combine_multi_if_and_count_if_opt.sql
+++ b/tests/queries/0_stateless/02374_combine_multi_if_and_count_if_opt.sql
@@ -4,7 +4,7 @@ create table m (a int) engine Log;
 
 insert into m values (1);
 
-set allow_experimental_analyzer = true, optimize_rewrite_sum_if_to_count_if=1;
+set enable_analyzer = true, optimize_rewrite_sum_if_to_count_if=1;
 
 EXPLAIN QUERY TREE select sum(multiIf(a = 1, 1, 0)) from m;
 
diff --git a/tests/queries/0_stateless/02375_analyzer_union.sql b/tests/queries/0_stateless/02375_analyzer_union.sql
index 5e41f07d217..cf9a99ffbba 100644
--- a/tests/queries/0_stateless/02375_analyzer_union.sql
+++ b/tests/queries/0_stateless/02375_analyzer_union.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 0;
+SET enable_analyzer = 0;
 
 DROP TABLE IF EXISTS test_table;
 CREATE TABLE test_table
diff --git a/tests/queries/0_stateless/02376_analyzer_in_function_subquery.sql b/tests/queries/0_stateless/02376_analyzer_in_function_subquery.sql
index 72a4edb8567..295d8a8f97e 100644
--- a/tests/queries/0_stateless/02376_analyzer_in_function_subquery.sql
+++ b/tests/queries/0_stateless/02376_analyzer_in_function_subquery.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS test_table;
 CREATE TABLE test_table
diff --git a/tests/queries/0_stateless/02377_analyzer_in_function_set.sql b/tests/queries/0_stateless/02377_analyzer_in_function_set.sql
index e3cbcf75a9c..00aa40ccf17 100644
--- a/tests/queries/0_stateless/02377_analyzer_in_function_set.sql
+++ b/tests/queries/0_stateless/02377_analyzer_in_function_set.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS test_table;
 CREATE TABLE test_table
diff --git a/tests/queries/0_stateless/02377_optimize_sorting_by_input_stream_properties_explain.sh b/tests/queries/0_stateless/02377_optimize_sorting_by_input_stream_properties_explain.sh
index c223fcc86bc..4b9793da5bb 100755
--- a/tests/queries/0_stateless/02377_optimize_sorting_by_input_stream_properties_explain.sh
+++ b/tests/queries/0_stateless/02377_optimize_sorting_by_input_stream_properties_explain.sh
@@ -20,9 +20,9 @@ function explain_sorting {
 
 function explain_sortmode {
     echo "-- QUERY: "$1
-    $CLICKHOUSE_CLIENT --allow_experimental_analyzer=0 --merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability=0.0 -nq "$1" | eval $FIND_SORTMODE
+    $CLICKHOUSE_CLIENT --enable_analyzer=0 --merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability=0.0 -nq "$1" | eval $FIND_SORTMODE
     echo "-- QUERY (analyzer): "$1
-    $CLICKHOUSE_CLIENT --allow_experimental_analyzer=1 --merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability=0.0 -nq "$1" | eval $FIND_SORTMODE
+    $CLICKHOUSE_CLIENT --enable_analyzer=1 --merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability=0.0 -nq "$1" | eval $FIND_SORTMODE
 }
 
 $CLICKHOUSE_CLIENT -q "drop table if exists optimize_sorting sync"
diff --git a/tests/queries/0_stateless/02378_analyzer_projection_names.sql b/tests/queries/0_stateless/02378_analyzer_projection_names.sql
index f41afe6a950..7b3099c4f36 100644
--- a/tests/queries/0_stateless/02378_analyzer_projection_names.sql
+++ b/tests/queries/0_stateless/02378_analyzer_projection_names.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 SET single_join_prefer_left_table = 0;
 
 DROP TABLE IF EXISTS test_table;
diff --git a/tests/queries/0_stateless/02379_analyzer_subquery_depth.sql b/tests/queries/0_stateless/02379_analyzer_subquery_depth.sql
index 5699a15aead..40303e0f92a 100644
--- a/tests/queries/0_stateless/02379_analyzer_subquery_depth.sql
+++ b/tests/queries/0_stateless/02379_analyzer_subquery_depth.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 SELECT (SELECT a FROM (SELECT 1 AS a)) SETTINGS max_subquery_depth = 1; -- { serverError TOO_DEEP_SUBQUERIES }
 SELECT (SELECT a FROM (SELECT 1 AS a)) SETTINGS max_subquery_depth = 2;
diff --git a/tests/queries/0_stateless/02380_analyzer_join_sample.sql b/tests/queries/0_stateless/02380_analyzer_join_sample.sql
index e417f47d173..bc77f3623ae 100644
--- a/tests/queries/0_stateless/02380_analyzer_join_sample.sql
+++ b/tests/queries/0_stateless/02380_analyzer_join_sample.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS test_table_join_1;
 CREATE TABLE test_table_join_1
diff --git a/tests/queries/0_stateless/02381_analyzer_join_final.sql b/tests/queries/0_stateless/02381_analyzer_join_final.sql
index 57fc3aedd8f..0db81ac7728 100644
--- a/tests/queries/0_stateless/02381_analyzer_join_final.sql
+++ b/tests/queries/0_stateless/02381_analyzer_join_final.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS test_table_join_1;
 CREATE TABLE test_table_join_1
diff --git a/tests/queries/0_stateless/02381_join_dup_columns_in_plan.sql.j2 b/tests/queries/0_stateless/02381_join_dup_columns_in_plan.sql.j2
index ca4af4df6b6..4e88b67e3dd 100644
--- a/tests/queries/0_stateless/02381_join_dup_columns_in_plan.sql.j2
+++ b/tests/queries/0_stateless/02381_join_dup_columns_in_plan.sql.j2
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 -- { echoOn }
 
diff --git a/tests/queries/0_stateless/02382_analyzer_matcher_join_using.sql b/tests/queries/0_stateless/02382_analyzer_matcher_join_using.sql
index 7983b05a69e..6a0b58e7b28 100644
--- a/tests/queries/0_stateless/02382_analyzer_matcher_join_using.sql
+++ b/tests/queries/0_stateless/02382_analyzer_matcher_join_using.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS test_table_join_1;
 CREATE TABLE test_table_join_1
diff --git a/tests/queries/0_stateless/02383_analyzer_merge_tree_self_join.sql b/tests/queries/0_stateless/02383_analyzer_merge_tree_self_join.sql
index c22a0f4244b..fbd6fe4db4f 100644
--- a/tests/queries/0_stateless/02383_analyzer_merge_tree_self_join.sql
+++ b/tests/queries/0_stateless/02383_analyzer_merge_tree_self_join.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS test_table_join_1;
 CREATE TABLE test_table_join_1
diff --git a/tests/queries/0_stateless/02384_analyzer_dict_get_join_get.sql b/tests/queries/0_stateless/02384_analyzer_dict_get_join_get.sql
index f4619f20765..a8ad5c4d957 100644
--- a/tests/queries/0_stateless/02384_analyzer_dict_get_join_get.sql
+++ b/tests/queries/0_stateless/02384_analyzer_dict_get_join_get.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS test_table;
 CREATE TABLE test_table
diff --git a/tests/queries/0_stateless/02385_analyzer_aliases_compound_expression.sql b/tests/queries/0_stateless/02385_analyzer_aliases_compound_expression.sql
index 861ada9623a..025e064d23c 100644
--- a/tests/queries/0_stateless/02385_analyzer_aliases_compound_expression.sql
+++ b/tests/queries/0_stateless/02385_analyzer_aliases_compound_expression.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 SELECT cast(tuple(1, 'Value'), 'Tuple(first UInt64, second String)') AS value, value.first, value.second;
 
diff --git a/tests/queries/0_stateless/02386_analyzer_in_function_nested_subqueries.sql b/tests/queries/0_stateless/02386_analyzer_in_function_nested_subqueries.sql
index c8ca3ff21d4..64d24aae1b1 100644
--- a/tests/queries/0_stateless/02386_analyzer_in_function_nested_subqueries.sql
+++ b/tests/queries/0_stateless/02386_analyzer_in_function_nested_subqueries.sql
@@ -1,3 +1,3 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 SELECT (NULL IN (SELECT 9223372036854775806 IN (SELECT 65536), inf, NULL IN (NULL))) IN (SELECT NULL IN (NULL));
diff --git a/tests/queries/0_stateless/02387_analyzer_cte.sql b/tests/queries/0_stateless/02387_analyzer_cte.sql
index 1f10ac10438..149eab7f741 100644
--- a/tests/queries/0_stateless/02387_analyzer_cte.sql
+++ b/tests/queries/0_stateless/02387_analyzer_cte.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS test_table;
 CREATE TABLE test_table
diff --git a/tests/queries/0_stateless/02388_analyzer_recursive_lambda.sql b/tests/queries/0_stateless/02388_analyzer_recursive_lambda.sql
index 9fd2f73703d..31d6f91a39d 100644
--- a/tests/queries/0_stateless/02388_analyzer_recursive_lambda.sql
+++ b/tests/queries/0_stateless/02388_analyzer_recursive_lambda.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 WITH x -> plus(lambda(1), x) AS lambda SELECT lambda(1048576); -- { serverError UNSUPPORTED_METHOD };
 
diff --git a/tests/queries/0_stateless/02389_analyzer_nested_lambda.sql b/tests/queries/0_stateless/02389_analyzer_nested_lambda.sql
index 8e3777ebc15..51a10adae6a 100644
--- a/tests/queries/0_stateless/02389_analyzer_nested_lambda.sql
+++ b/tests/queries/0_stateless/02389_analyzer_nested_lambda.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 -- { echoOn }
 
diff --git a/tests/queries/0_stateless/02420_final_setting_analyzer.reference b/tests/queries/0_stateless/02420_final_setting_analyzer.reference
index 780a6e5de68..763ecad84ab 100644
--- a/tests/queries/0_stateless/02420_final_setting_analyzer.reference
+++ b/tests/queries/0_stateless/02420_final_setting_analyzer.reference
@@ -1,5 +1,5 @@
 -- { echoOn }
-set allow_experimental_analyzer=1;
+set enable_analyzer=1;
 SYSTEM STOP MERGES tbl;
 -- simple test case
 create table if not exists replacing_mt (x String) engine=ReplacingMergeTree() ORDER BY x;
diff --git a/tests/queries/0_stateless/02420_final_setting_analyzer.sql b/tests/queries/0_stateless/02420_final_setting_analyzer.sql
index cbdec017602..89fff094825 100644
--- a/tests/queries/0_stateless/02420_final_setting_analyzer.sql
+++ b/tests/queries/0_stateless/02420_final_setting_analyzer.sql
@@ -1,5 +1,5 @@
 -- { echoOn }
-set allow_experimental_analyzer=1;
+set enable_analyzer=1;
 SYSTEM STOP MERGES tbl;
 
 -- simple test case
diff --git a/tests/queries/0_stateless/02421_decimal_in_precision_issue_41125.sql b/tests/queries/0_stateless/02421_decimal_in_precision_issue_41125.sql
index fde893626c1..5a4e23c10bd 100644
--- a/tests/queries/0_stateless/02421_decimal_in_precision_issue_41125.sql
+++ b/tests/queries/0_stateless/02421_decimal_in_precision_issue_41125.sql
@@ -21,7 +21,7 @@ SELECT count() == 1 FROM dtest WHERE b IN toDecimal64('44.4000', 4);
 SELECT count() == 1 FROM dtest WHERE b IN toDecimal128('44.4000', 4);
 SELECT count() == 1 FROM dtest WHERE b IN toDecimal256('44.4000', 4);
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 SELECT count() == 0 FROM (SELECT '33.3' :: Decimal(9, 1) AS a WHERE a IN ('33.33' :: Decimal(9, 2)));
 
diff --git a/tests/queries/0_stateless/02421_explain_subquery.sql b/tests/queries/0_stateless/02421_explain_subquery.sql
index 2970003cb1c..02f45e8cc3b 100644
--- a/tests/queries/0_stateless/02421_explain_subquery.sql
+++ b/tests/queries/0_stateless/02421_explain_subquery.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 0;
+SET enable_analyzer = 0;
 
 SELECT count() > 3 FROM (EXPLAIN PIPELINE header = 1 SELECT * FROM system.numbers ORDER BY number DESC) WHERE explain LIKE '%Header: number UInt64%';
 SELECT count() > 0 FROM (EXPLAIN PLAN SELECT * FROM system.numbers ORDER BY number DESC) WHERE explain ILIKE '%Sort%';
@@ -32,7 +32,7 @@ SELECT count() == 1 FROM (EXPLAIN ESTIMATE SELECT sum(a) FROM t1);
 
 DROP TABLE t1;
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 SELECT count() > 3 FROM (EXPLAIN PIPELINE header = 1 SELECT * FROM system.numbers ORDER BY number DESC) WHERE explain LIKE '%Header: \_\_table1.number UInt64%';
 SELECT count() > 0 FROM (EXPLAIN PLAN SELECT * FROM system.numbers ORDER BY number DESC) WHERE explain ILIKE '%Sort%';
diff --git a/tests/queries/0_stateless/02428_decimal_in_floating_point_literal.sql b/tests/queries/0_stateless/02428_decimal_in_floating_point_literal.sql
index a84cb5572ba..a0d92115188 100644
--- a/tests/queries/0_stateless/02428_decimal_in_floating_point_literal.sql
+++ b/tests/queries/0_stateless/02428_decimal_in_floating_point_literal.sql
@@ -30,7 +30,7 @@ SELECT count() == 1 FROM decimal_in_float_test WHERE a NOT IN (33.333);
 SELECT count() == 1 FROM decimal_in_float_test WHERE b IN (44.44);
 SELECT count() == 1 FROM decimal_in_float_test WHERE b NOT IN (44.4,44.444);
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 
 SELECT toDecimal32(1.555,3) IN (1.5551);
diff --git a/tests/queries/0_stateless/02428_parameterized_view.sh b/tests/queries/0_stateless/02428_parameterized_view.sh
index c6f0927db36..95f82db5454 100755
--- a/tests/queries/0_stateless/02428_parameterized_view.sh
+++ b/tests/queries/0_stateless/02428_parameterized_view.sh
@@ -72,8 +72,8 @@ $CLICKHOUSE_CLIENT -q "INSERT INTO ${CLICKHOUSE_TEST_UNIQUE_NAME}.Catalog VALUES
 $CLICKHOUSE_CLIENT -q "INSERT INTO ${CLICKHOUSE_TEST_UNIQUE_NAME}.Catalog VALUES ('Paper', 20, 1)"
 $CLICKHOUSE_CLIENT -q "CREATE VIEW ${CLICKHOUSE_TEST_UNIQUE_NAME}.pv1 AS SELECT * FROM ${CLICKHOUSE_TEST_UNIQUE_NAME}.Catalog WHERE Price={price:UInt64}"
 $CLICKHOUSE_CLIENT -q "SELECT Price FROM ${CLICKHOUSE_TEST_UNIQUE_NAME}.pv1(price=20)"
-$CLICKHOUSE_CLIENT -q "SELECT Price FROM \`${CLICKHOUSE_TEST_UNIQUE_NAME}.pv1\`(price=20) SETTINGS allow_experimental_analyzer = 0"  2>&1 |  grep -Fq "UNKNOWN_FUNCTION" &&  echo 'ERROR' || echo 'OK'
-$CLICKHOUSE_CLIENT -q "SELECT Price FROM \`${CLICKHOUSE_TEST_UNIQUE_NAME}.pv1\`(price=20) SETTINGS allow_experimental_analyzer = 1"
+$CLICKHOUSE_CLIENT -q "SELECT Price FROM \`${CLICKHOUSE_TEST_UNIQUE_NAME}.pv1\`(price=20) SETTINGS enable_analyzer = 0"  2>&1 |  grep -Fq "UNKNOWN_FUNCTION" &&  echo 'ERROR' || echo 'OK'
+$CLICKHOUSE_CLIENT -q "SELECT Price FROM \`${CLICKHOUSE_TEST_UNIQUE_NAME}.pv1\`(price=20) SETTINGS enable_analyzer = 1"
 
 
 $CLICKHOUSE_CLIENT -q "INSERT INTO test_02428_Catalog VALUES ('Book2', 30, 8)"
diff --git a/tests/queries/0_stateless/02451_order_by_monotonic.sh b/tests/queries/0_stateless/02451_order_by_monotonic.sh
index 7d1356b4445..fa0a37678f0 100755
--- a/tests/queries/0_stateless/02451_order_by_monotonic.sh
+++ b/tests/queries/0_stateless/02451_order_by_monotonic.sh
@@ -5,7 +5,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CURDIR"/../shell_config.sh
 
 opts=(
-    "--allow_experimental_analyzer=1"
+    "--enable_analyzer=1"
 )
 
 function explain_sort_description()
diff --git a/tests/queries/0_stateless/02459_group_by_all.sql b/tests/queries/0_stateless/02459_group_by_all.sql
index 4f08ee331a4..8281c201edf 100644
--- a/tests/queries/0_stateless/02459_group_by_all.sql
+++ b/tests/queries/0_stateless/02459_group_by_all.sql
@@ -21,7 +21,7 @@ select substring(a, 1, 3), substring(substring(substring(a, c, count(b)), 1, cou
 select substring(a, 1, 3), substring(a, 1, count(b)) from group_by_all group by all;
 select count(b) AS len, substring(a, 1, 3), substring(a, 1, len) from group_by_all group by all;
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 select a, count(b) from group_by_all group by all order by a;
 select substring(a, 1, 3), count(b) from group_by_all group by all;
diff --git a/tests/queries/0_stateless/02474_analyzer_subqueries_table_expression_modifiers.sql b/tests/queries/0_stateless/02474_analyzer_subqueries_table_expression_modifiers.sql
index 5ac8c79d4ed..e86c867043d 100644
--- a/tests/queries/0_stateless/02474_analyzer_subqueries_table_expression_modifiers.sql
+++ b/tests/queries/0_stateless/02474_analyzer_subqueries_table_expression_modifiers.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 SELECT * FROM (SELECT 1) FINAL; -- { serverError UNSUPPORTED_METHOD }
 SELECT * FROM (SELECT 1) SAMPLE 1/2; -- { serverError UNSUPPORTED_METHOD }
diff --git a/tests/queries/0_stateless/02475_analyzer_join_tree_subquery.sql b/tests/queries/0_stateless/02475_analyzer_join_tree_subquery.sql
index eda90529166..c9e7ac19157 100644
--- a/tests/queries/0_stateless/02475_analyzer_join_tree_subquery.sql
+++ b/tests/queries/0_stateless/02475_analyzer_join_tree_subquery.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 WITH subquery AS (SELECT sum(number) FROM numbers(10)) SELECT * FROM subquery;
 
diff --git a/tests/queries/0_stateless/02475_analyzer_subquery_compound_expression.sql b/tests/queries/0_stateless/02475_analyzer_subquery_compound_expression.sql
index f96c834c057..fc9e9d44b42 100644
--- a/tests/queries/0_stateless/02475_analyzer_subquery_compound_expression.sql
+++ b/tests/queries/0_stateless/02475_analyzer_subquery_compound_expression.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 
 SELECT cast(tuple(1, 2), 'Tuple(value_1 UInt64, value_2 UInt64)') AS value, value.value_1, value.value_2;
 
diff --git a/tests/queries/0_stateless/02475_or_function_alias_and_const_where.sql b/tests/queries/0_stateless/02475_or_function_alias_and_const_where.sql
index ddb0f70c6de..53b97a1dd2a 100644
--- a/tests/queries/0_stateless/02475_or_function_alias_and_const_where.sql
+++ b/tests/queries/0_stateless/02475_or_function_alias_and_const_where.sql
@@ -1,2 +1,2 @@
 SELECT (number = 1) AND (number = 2) AS value, sum(value) OVER () FROM numbers(1) WHERE 1;
-SELECT (number = 1) AND (number = 2) AS value, sum(value) OVER () FROM numbers(1) WHERE 1 SETTINGS allow_experimental_analyzer=1;
\ No newline at end of file
+SELECT (number = 1) AND (number = 2) AS value, sum(value) OVER () FROM numbers(1) WHERE 1 SETTINGS enable_analyzer=1;
diff --git a/tests/queries/0_stateless/02476_analyzer_identifier_hints.sh b/tests/queries/0_stateless/02476_analyzer_identifier_hints.sh
index 0702c146426..4c850a6ec9e 100755
--- a/tests/queries/0_stateless/02476_analyzer_identifier_hints.sh
+++ b/tests/queries/0_stateless/02476_analyzer_identifier_hints.sh
@@ -17,61 +17,61 @@ $CLICKHOUSE_CLIENT -n -q "
     INSERT INTO test_table VALUES (0, 'Value');
 ";
 
-$CLICKHOUSE_CLIENT -q "SELECT value_ FROM test_table SETTINGS allow_experimental_analyzer = 1;" 2>&1 \
+$CLICKHOUSE_CLIENT -q "SELECT value_ FROM test_table SETTINGS enable_analyzer = 1;" 2>&1 \
     | grep "Maybe you meant: \['value'\]" &>/dev/null;
 
-$CLICKHOUSE_CLIENT -q "SELECT test_table.value_ FROM test_table SETTINGS allow_experimental_analyzer = 1;" 2>&1 \
+$CLICKHOUSE_CLIENT -q "SELECT test_table.value_ FROM test_table SETTINGS enable_analyzer = 1;" 2>&1 \
     | grep "Maybe you meant: \['test_table.value'\]" &>/dev/null;
 
-$CLICKHOUSE_CLIENT -q "SELECT test_tabl.value_ FROM test_table SETTINGS allow_experimental_analyzer = 1;" 2>&1 \
+$CLICKHOUSE_CLIENT -q "SELECT test_tabl.value_ FROM test_table SETTINGS enable_analyzer = 1;" 2>&1 \
     | grep "Maybe you meant: \['test_table.value'\]" &>/dev/null;
 
-$CLICKHOUSE_CLIENT -q "SELECT test_table.value_ FROM test_table AS test_table_alias SETTINGS allow_experimental_analyzer = 1;" 2>&1 \
+$CLICKHOUSE_CLIENT -q "SELECT test_table.value_ FROM test_table AS test_table_alias SETTINGS enable_analyzer = 1;" 2>&1 \
     | grep "Maybe you meant: \['test_table.value'\]" &>/dev/null;
 
-$CLICKHOUSE_CLIENT -q "SELECT test_tabl.value_ FROM test_table AS test_table_alias SETTINGS allow_experimental_analyzer = 1;" 2>&1 \
+$CLICKHOUSE_CLIENT -q "SELECT test_tabl.value_ FROM test_table AS test_table_alias SETTINGS enable_analyzer = 1;" 2>&1 \
     | grep "Maybe you meant: \['test_table.value'\]" &>/dev/null;
 
-$CLICKHOUSE_CLIENT -q "SELECT test_table_alias.value_ FROM test_table AS test_table_alias SETTINGS allow_experimental_analyzer = 1;" 2>&1 \
+$CLICKHOUSE_CLIENT -q "SELECT test_table_alias.value_ FROM test_table AS test_table_alias SETTINGS enable_analyzer = 1;" 2>&1 \
     | grep "Maybe you meant: \['test_table_alias.value'\]" &>/dev/null;
 
-$CLICKHOUSE_CLIENT -q "SELECT test_table_alia.value_ FROM test_table AS test_table_alias SETTINGS allow_experimental_analyzer = 1;" 2>&1 \
+$CLICKHOUSE_CLIENT -q "SELECT test_table_alia.value_ FROM test_table AS test_table_alias SETTINGS enable_analyzer = 1;" 2>&1 \
     | grep "Maybe you meant: \['test_table_alias.value'\]" &>/dev/null;
 
-$CLICKHOUSE_CLIENT -q "SELECT value_ FROM (SELECT 1 AS value) SETTINGS allow_experimental_analyzer = 1;" 2>&1 \
+$CLICKHOUSE_CLIENT -q "SELECT value_ FROM (SELECT 1 AS value) SETTINGS enable_analyzer = 1;" 2>&1 \
     | grep "Maybe you meant: \['value'\]" &>/dev/null;
 
-$CLICKHOUSE_CLIENT -q "SELECT subquery.value_ FROM (SELECT 1 AS value) AS subquery SETTINGS allow_experimental_analyzer = 1;" 2>&1 \
+$CLICKHOUSE_CLIENT -q "SELECT subquery.value_ FROM (SELECT 1 AS value) AS subquery SETTINGS enable_analyzer = 1;" 2>&1 \
     | grep "Maybe you meant: \['subquery.value'\]" &>/dev/null;
 
-$CLICKHOUSE_CLIENT -q "SELECT subquer.value_ FROM (SELECT 1 AS value) AS subquery SETTINGS allow_experimental_analyzer = 1;" 2>&1 \
+$CLICKHOUSE_CLIENT -q "SELECT subquer.value_ FROM (SELECT 1 AS value) AS subquery SETTINGS enable_analyzer = 1;" 2>&1 \
     | grep "Maybe you meant: \['subquery.value'\]" &>/dev/null;
 
-$CLICKHOUSE_CLIENT -q "WITH cte_subquery AS (SELECT 1 AS value) SELECT value_ FROM cte_subquery SETTINGS allow_experimental_analyzer = 1;" 2>&1 \
+$CLICKHOUSE_CLIENT -q "WITH cte_subquery AS (SELECT 1 AS value) SELECT value_ FROM cte_subquery SETTINGS enable_analyzer = 1;" 2>&1 \
     | grep "Maybe you meant: \['value'\]" &>/dev/null;
 
-$CLICKHOUSE_CLIENT -q "WITH cte_subquery AS (SELECT 1 AS value) SELECT cte_subquery.value_ FROM cte_subquery SETTINGS allow_experimental_analyzer = 1;" 2>&1 \
+$CLICKHOUSE_CLIENT -q "WITH cte_subquery AS (SELECT 1 AS value) SELECT cte_subquery.value_ FROM cte_subquery SETTINGS enable_analyzer = 1;" 2>&1 \
     | grep "Maybe you meant: \['cte_subquery.value'\]" &>/dev/null;
 
-$CLICKHOUSE_CLIENT -q "WITH cte_subquery AS (SELECT 1 AS value) SELECT cte_subquer.value_ FROM cte_subquery SETTINGS allow_experimental_analyzer = 1;" 2>&1 \
+$CLICKHOUSE_CLIENT -q "WITH cte_subquery AS (SELECT 1 AS value) SELECT cte_subquer.value_ FROM cte_subquery SETTINGS enable_analyzer = 1;" 2>&1 \
     | grep "Maybe you meant: \['cte_subquery.value'\]" &>/dev/null;
 
-$CLICKHOUSE_CLIENT -q "WITH cte_subquery AS (SELECT 1 AS value) SELECT cte_subquery_alias.value_ FROM cte_subquery AS cte_subquery_alias SETTINGS allow_experimental_analyzer = 1;" 2>&1 \
+$CLICKHOUSE_CLIENT -q "WITH cte_subquery AS (SELECT 1 AS value) SELECT cte_subquery_alias.value_ FROM cte_subquery AS cte_subquery_alias SETTINGS enable_analyzer = 1;" 2>&1 \
     | grep "Maybe you meant: \['cte_subquery_alias.value'\]" &>/dev/null;
 
-$CLICKHOUSE_CLIENT -q "WITH cte_subquery AS (SELECT 1 AS value) SELECT cte_subquery_alia.value_ FROM cte_subquery AS cte_subquery_alias SETTINGS allow_experimental_analyzer = 1;" 2>&1 \
+$CLICKHOUSE_CLIENT -q "WITH cte_subquery AS (SELECT 1 AS value) SELECT cte_subquery_alia.value_ FROM cte_subquery AS cte_subquery_alias SETTINGS enable_analyzer = 1;" 2>&1 \
     | grep "Maybe you meant: \['cte_subquery_alias.value'\]" &>/dev/null;
 
-$CLICKHOUSE_CLIENT -q "SELECT 1 AS constant_value, constant_valu SETTINGS allow_experimental_analyzer = 1;" 2>&1 \
+$CLICKHOUSE_CLIENT -q "SELECT 1 AS constant_value, constant_valu SETTINGS enable_analyzer = 1;" 2>&1 \
     | grep "Maybe you meant: \['constant_value'\]" &>/dev/null;
 
-$CLICKHOUSE_CLIENT -q "SELECT 1 AS constant_value, constant_valu SETTINGS allow_experimental_analyzer = 1;" 2>&1 \
+$CLICKHOUSE_CLIENT -q "SELECT 1 AS constant_value, constant_valu SETTINGS enable_analyzer = 1;" 2>&1 \
     | grep "Maybe you meant: \['constant_value'\]" &>/dev/null;
 
-$CLICKHOUSE_CLIENT -q "SELECT 1 AS constant_value, arrayMap(lambda_argument -> lambda_argument + constant_valu, [1, 2, 3]) SETTINGS allow_experimental_analyzer = 1;" 2>&1 \
+$CLICKHOUSE_CLIENT -q "SELECT 1 AS constant_value, arrayMap(lambda_argument -> lambda_argument + constant_valu, [1, 2, 3]) SETTINGS enable_analyzer = 1;" 2>&1 \
     | grep "Maybe you meant: \['constant_value'\]" &>/dev/null;
 
-$CLICKHOUSE_CLIENT -q "WITH 1 AS constant_value SELECT (SELECT constant_valu) SETTINGS allow_experimental_analyzer = 1;" 2>&1 \
+$CLICKHOUSE_CLIENT -q "WITH 1 AS constant_value SELECT (SELECT constant_valu) SETTINGS enable_analyzer = 1;" 2>&1 \
     | grep "Maybe you meant: \['constant_value'\]" &>/dev/null;
 
 $CLICKHOUSE_CLIENT -n -q "
@@ -85,61 +85,61 @@ $CLICKHOUSE_CLIENT -n -q "
     INSERT INTO test_table_compound VALUES (0, tuple('Value_1'));
 ";
 
-$CLICKHOUSE_CLIENT -q "SELECT value.value_ FROM test_table_compound SETTINGS allow_experimental_analyzer = 1;" 2>&1 \
+$CLICKHOUSE_CLIENT -q "SELECT value.value_ FROM test_table_compound SETTINGS enable_analyzer = 1;" 2>&1 \
     | grep "Maybe you meant: \['value.value_1'\]" &>/dev/null;
 
-$CLICKHOUSE_CLIENT -q "SELECT test_table_compound.value.value_ FROM test_table_compound SETTINGS allow_experimental_analyzer = 1;" 2>&1 \
+$CLICKHOUSE_CLIENT -q "SELECT test_table_compound.value.value_ FROM test_table_compound SETTINGS enable_analyzer = 1;" 2>&1 \
     | grep "Maybe you meant: \['test_table_compound.value.value_1'\]" &>/dev/null;
 
-$CLICKHOUSE_CLIENT -q "SELECT test_tabl_compound.value.value_ FROM test_table_compound SETTINGS allow_experimental_analyzer = 1;" 2>&1 \
+$CLICKHOUSE_CLIENT -q "SELECT test_tabl_compound.value.value_ FROM test_table_compound SETTINGS enable_analyzer = 1;" 2>&1 \
     | grep "Maybe you meant: \['test_table_compound.value.value_1'\]" &>/dev/null;
 
-$CLICKHOUSE_CLIENT -q "SELECT test_table_compound.value.value_ FROM test_table_compound AS test_table_compound_alias SETTINGS allow_experimental_analyzer = 1;" 2>&1 \
+$CLICKHOUSE_CLIENT -q "SELECT test_table_compound.value.value_ FROM test_table_compound AS test_table_compound_alias SETTINGS enable_analyzer = 1;" 2>&1 \
     | grep "Maybe you meant: \['test_table_compound.value.value_1'\]" &>/dev/null;
 
-$CLICKHOUSE_CLIENT -q "SELECT test_tabl_compound.value.value_ FROM test_table_compound AS test_table_compound_alias SETTINGS allow_experimental_analyzer = 1;" 2>&1 \
+$CLICKHOUSE_CLIENT -q "SELECT test_tabl_compound.value.value_ FROM test_table_compound AS test_table_compound_alias SETTINGS enable_analyzer = 1;" 2>&1 \
     | grep "Maybe you meant: \['test_table_compound.value.value_1'\]" &>/dev/null;
 
-$CLICKHOUSE_CLIENT -q "SELECT test_table_compound_alias.value.value_ FROM test_table_compound AS test_table_compound_alias SETTINGS allow_experimental_analyzer = 1;" 2>&1 \
+$CLICKHOUSE_CLIENT -q "SELECT test_table_compound_alias.value.value_ FROM test_table_compound AS test_table_compound_alias SETTINGS enable_analyzer = 1;" 2>&1 \
     | grep "Maybe you meant: \['test_table_compound_alias.value.value_1'\]" &>/dev/null;
 
-$CLICKHOUSE_CLIENT -q "SELECT test_table_compound_alia.value.value_ FROM test_table_compound AS test_table_compound_alias SETTINGS allow_experimental_analyzer = 1;" 2>&1 \
+$CLICKHOUSE_CLIENT -q "SELECT test_table_compound_alia.value.value_ FROM test_table_compound AS test_table_compound_alias SETTINGS enable_analyzer = 1;" 2>&1 \
     | grep "Maybe you meant: \['test_table_compound_alias.value.value_1'\]" &>/dev/null;
 
-$CLICKHOUSE_CLIENT -q "SELECT value.value_ FROM (SELECT cast(tuple(1), 'Tuple(value_1 String)') AS value) SETTINGS allow_experimental_analyzer = 1;" 2>&1 \
+$CLICKHOUSE_CLIENT -q "SELECT value.value_ FROM (SELECT cast(tuple(1), 'Tuple(value_1 String)') AS value) SETTINGS enable_analyzer = 1;" 2>&1 \
     | grep "Maybe you meant: \['value.value_1'\]" &>/dev/null;
 
-$CLICKHOUSE_CLIENT -q "SELECT subquery.value.value_ FROM (SELECT cast(tuple(1), 'Tuple(value_1 String)') AS value) AS subquery SETTINGS allow_experimental_analyzer = 1;" 2>&1 \
+$CLICKHOUSE_CLIENT -q "SELECT subquery.value.value_ FROM (SELECT cast(tuple(1), 'Tuple(value_1 String)') AS value) AS subquery SETTINGS enable_analyzer = 1;" 2>&1 \
     | grep "Maybe you meant: \['subquery.value.value_1'\]" &>/dev/null;
 
-$CLICKHOUSE_CLIENT -q "SELECT subquer.value.value_ FROM (SELECT cast(tuple(1), 'Tuple(value_1 String)') AS value) AS subquery SETTINGS allow_experimental_analyzer = 1;" 2>&1 \
+$CLICKHOUSE_CLIENT -q "SELECT subquer.value.value_ FROM (SELECT cast(tuple(1), 'Tuple(value_1 String)') AS value) AS subquery SETTINGS enable_analyzer = 1;" 2>&1 \
     | grep "Maybe you meant: \['subquery.value.value_1'\]" &>/dev/null;
 
-$CLICKHOUSE_CLIENT -q "WITH cte_subquery AS (SELECT cast(tuple(1), 'Tuple(value_1 String)') AS value) SELECT value.value_ FROM cte_subquery SETTINGS allow_experimental_analyzer = 1;" 2>&1 \
+$CLICKHOUSE_CLIENT -q "WITH cte_subquery AS (SELECT cast(tuple(1), 'Tuple(value_1 String)') AS value) SELECT value.value_ FROM cte_subquery SETTINGS enable_analyzer = 1;" 2>&1 \
     | grep "Maybe you meant: \['value.value_1'\]" &>/dev/null;
 
-$CLICKHOUSE_CLIENT -q "WITH cte_subquery AS (SELECT cast(tuple(1), 'Tuple(value_1 String)') AS value) SELECT cte_subquery.value.value_ FROM cte_subquery SETTINGS allow_experimental_analyzer = 1;" 2>&1 \
+$CLICKHOUSE_CLIENT -q "WITH cte_subquery AS (SELECT cast(tuple(1), 'Tuple(value_1 String)') AS value) SELECT cte_subquery.value.value_ FROM cte_subquery SETTINGS enable_analyzer = 1;" 2>&1 \
     | grep "Maybe you meant: \['cte_subquery.value.value_1'\]" &>/dev/null;
 
-$CLICKHOUSE_CLIENT -q "WITH cte_subquery AS (SELECT cast(tuple(1), 'Tuple(value_1 String)') AS value) SELECT cte_subquer.value.value_ FROM cte_subquery SETTINGS allow_experimental_analyzer = 1;" 2>&1 \
+$CLICKHOUSE_CLIENT -q "WITH cte_subquery AS (SELECT cast(tuple(1), 'Tuple(value_1 String)') AS value) SELECT cte_subquer.value.value_ FROM cte_subquery SETTINGS enable_analyzer = 1;" 2>&1 \
     | grep "Maybe you meant: \['cte_subquery.value.value_1'\]" &>/dev/null;
 
-$CLICKHOUSE_CLIENT -q "WITH cte_subquery AS (SELECT cast(tuple(1), 'Tuple(value_1 String)') AS value) SELECT cte_subquery_alias.value.value_ FROM cte_subquery AS cte_subquery_alias SETTINGS allow_experimental_analyzer = 1;" 2>&1 \
+$CLICKHOUSE_CLIENT -q "WITH cte_subquery AS (SELECT cast(tuple(1), 'Tuple(value_1 String)') AS value) SELECT cte_subquery_alias.value.value_ FROM cte_subquery AS cte_subquery_alias SETTINGS enable_analyzer = 1;" 2>&1 \
     | grep "Maybe you meant: \['cte_subquery_alias.value.value_1'\]" &>/dev/null;
 
-$CLICKHOUSE_CLIENT -q "WITH cte_subquery AS (SELECT cast(tuple(1), 'Tuple(value_1 String)') AS value) SELECT cte_subquery_alia.value.value_ FROM cte_subquery AS cte_subquery_alias SETTINGS allow_experimental_analyzer = 1;" 2>&1 \
+$CLICKHOUSE_CLIENT -q "WITH cte_subquery AS (SELECT cast(tuple(1), 'Tuple(value_1 String)') AS value) SELECT cte_subquery_alia.value.value_ FROM cte_subquery AS cte_subquery_alias SETTINGS enable_analyzer = 1;" 2>&1 \
     | grep "Maybe you meant: \['cte_subquery_alias.value.value_1'\]" &>/dev/null;
 
-$CLICKHOUSE_CLIENT -q "SELECT cast(tuple(1), 'Tuple(value_1 String)') AS constant_value, constant_value.value_ SETTINGS allow_experimental_analyzer = 1;" 2>&1 \
+$CLICKHOUSE_CLIENT -q "SELECT cast(tuple(1), 'Tuple(value_1 String)') AS constant_value, constant_value.value_ SETTINGS enable_analyzer = 1;" 2>&1 \
     | grep "Maybe you meant: \['constant_value.value_1'\]" &>/dev/null;
 
-$CLICKHOUSE_CLIENT -q "SELECT cast(tuple(1), 'Tuple(value_1 String)') AS constant_value, constant_valu.value_ SETTINGS allow_experimental_analyzer = 1;" 2>&1 \
+$CLICKHOUSE_CLIENT -q "SELECT cast(tuple(1), 'Tuple(value_1 String)') AS constant_value, constant_valu.value_ SETTINGS enable_analyzer = 1;" 2>&1 \
     | grep "Maybe you meant: \['constant_value.value_1'\]" &>/dev/null;
 
-$CLICKHOUSE_CLIENT -q "SELECT cast(tuple(1), 'Tuple(value_1 String)') AS constant_value, arrayMap(lambda_argument -> lambda_argument + constant_value.value_, [1, 2, 3]) SETTINGS allow_experimental_analyzer = 1;" 2>&1 \
+$CLICKHOUSE_CLIENT -q "SELECT cast(tuple(1), 'Tuple(value_1 String)') AS constant_value, arrayMap(lambda_argument -> lambda_argument + constant_value.value_, [1, 2, 3]) SETTINGS enable_analyzer = 1;" 2>&1 \
     | grep "Maybe you meant: \['constant_value.value_1'\]" &>/dev/null;
 
-$CLICKHOUSE_CLIENT -q "WITH cast(tuple(1), 'Tuple(value_1 String)') AS constant_value SELECT (SELECT constant_value.value_) SETTINGS allow_experimental_analyzer = 1;" 2>&1 \
+$CLICKHOUSE_CLIENT -q "WITH cast(tuple(1), 'Tuple(value_1 String)') AS constant_value SELECT (SELECT constant_value.value_) SETTINGS enable_analyzer = 1;" 2>&1 \
     | grep "Maybe you meant: \['constant_value.value_1'\]" &>/dev/null;
 
 $CLICKHOUSE_CLIENT -n -q "
@@ -162,25 +162,25 @@ $CLICKHOUSE_CLIENT -n -q "
     INSERT INTO test_table_2 VALUES (0, 'Value');
 ";
 
-$CLICKHOUSE_CLIENT -q "SELECT test_table_1.value_ FROM test_table_1 INNER JOIN test_table_2 ON test_table_1.id = test_table_2.id SETTINGS allow_experimental_analyzer = 1;" 2>&1 \
+$CLICKHOUSE_CLIENT -q "SELECT test_table_1.value_ FROM test_table_1 INNER JOIN test_table_2 ON test_table_1.id = test_table_2.id SETTINGS enable_analyzer = 1;" 2>&1 \
     | grep "Maybe you meant: \['test_table_1.value'\]" &>/dev/null;
 
-$CLICKHOUSE_CLIENT -q "SELECT test_table_2.value_ FROM test_table_1 INNER JOIN test_table_2 ON test_table_1.id = test_table_2.id SETTINGS allow_experimental_analyzer = 1;" 2>&1 \
+$CLICKHOUSE_CLIENT -q "SELECT test_table_2.value_ FROM test_table_1 INNER JOIN test_table_2 ON test_table_1.id = test_table_2.id SETTINGS enable_analyzer = 1;" 2>&1 \
     | grep "Maybe you meant: \['test_table_2.value'\]" &>/dev/null;
 
-$CLICKHOUSE_CLIENT -q "SELECT t1.value_ FROM test_table_1 AS t1 INNER JOIN test_table_2 AS t2 ON t1.id = t2.id SETTINGS allow_experimental_analyzer = 1;" 2>&1 \
+$CLICKHOUSE_CLIENT -q "SELECT t1.value_ FROM test_table_1 AS t1 INNER JOIN test_table_2 AS t2 ON t1.id = t2.id SETTINGS enable_analyzer = 1;" 2>&1 \
     | grep "Maybe you meant: \['t1.value'\]" &>/dev/null;
 
-$CLICKHOUSE_CLIENT -q "SELECT t2.value_ FROM test_table_1 AS t1 INNER JOIN test_table_2 AS t2 ON t1.id = t2.id SETTINGS allow_experimental_analyzer = 1;" 2>&1 \
+$CLICKHOUSE_CLIENT -q "SELECT t2.value_ FROM test_table_1 AS t1 INNER JOIN test_table_2 AS t2 ON t1.id = t2.id SETTINGS enable_analyzer = 1;" 2>&1 \
     | grep "Maybe you meant: \['t2.value'\]" &>/dev/null;
 
-$CLICKHOUSE_CLIENT -q "SELECT [1] AS a, a.size1 SETTINGS allow_experimental_analyzer = 1;" 2>&1 \
+$CLICKHOUSE_CLIENT -q "SELECT [1] AS a, a.size1 SETTINGS enable_analyzer = 1;" 2>&1 \
     | grep "Maybe you meant: \['a.size0'\]" &>/dev/null;
 
-$CLICKHOUSE_CLIENT -q "SELECT ((1))::Tuple(a Tuple(b UInt32)) AS t, t.c SETTINGS allow_experimental_analyzer = 1;" 2>&1 \
+$CLICKHOUSE_CLIENT -q "SELECT ((1))::Tuple(a Tuple(b UInt32)) AS t, t.c SETTINGS enable_analyzer = 1;" 2>&1 \
     | grep "Maybe you meant: \['t.a'\]" &>/dev/null;
 
-$CLICKHOUSE_CLIENT -q "SELECT ((1))::Tuple(a Tuple(b UInt32)) AS t, t.a.c SETTINGS allow_experimental_analyzer = 1;" 2>&1 \
+$CLICKHOUSE_CLIENT -q "SELECT ((1))::Tuple(a Tuple(b UInt32)) AS t, t.a.c SETTINGS enable_analyzer = 1;" 2>&1 \
     | grep "Maybe you meant: \['t.a.b'\]" &>/dev/null;
 
 $CLICKHOUSE_CLIENT -q "SELECT 1";
diff --git a/tests/queries/0_stateless/02476_analyzer_join_with_unused_columns.sql b/tests/queries/0_stateless/02476_analyzer_join_with_unused_columns.sql
index ca937e01238..feb6786ffb6 100644
--- a/tests/queries/0_stateless/02476_analyzer_join_with_unused_columns.sql
+++ b/tests/queries/0_stateless/02476_analyzer_join_with_unused_columns.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 SELECT subquery_1.id, subquery_2.id FROM (SELECT 1 AS id, 2 AS value) AS subquery_1, (SELECT 3 AS id, 4 AS value) AS subquery_2;
 
diff --git a/tests/queries/0_stateless/02476_fuse_sum_count.sql b/tests/queries/0_stateless/02476_fuse_sum_count.sql
index 315bbd10a65..2319e81440b 100644
--- a/tests/queries/0_stateless/02476_fuse_sum_count.sql
+++ b/tests/queries/0_stateless/02476_fuse_sum_count.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 SET optimize_syntax_fuse_functions = 1;
 
 DROP TABLE IF EXISTS fuse_tbl;
diff --git a/tests/queries/0_stateless/02477_analyzer_array_join_with_join.sql b/tests/queries/0_stateless/02477_analyzer_array_join_with_join.sql
index 7e4ecb98ae8..3d2cc1b1620 100644
--- a/tests/queries/0_stateless/02477_analyzer_array_join_with_join.sql
+++ b/tests/queries/0_stateless/02477_analyzer_array_join_with_join.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS test_table;
 CREATE TABLE test_table
diff --git a/tests/queries/0_stateless/02477_analyzer_ast_key_condition_crash.sql b/tests/queries/0_stateless/02477_analyzer_ast_key_condition_crash.sql
index 53f3a9b23ec..2fc1cc45ce0 100644
--- a/tests/queries/0_stateless/02477_analyzer_ast_key_condition_crash.sql
+++ b/tests/queries/0_stateless/02477_analyzer_ast_key_condition_crash.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS test_table;
 CREATE TABLE test_table
diff --git a/tests/queries/0_stateless/02477_analyzer_function_hints.sh b/tests/queries/0_stateless/02477_analyzer_function_hints.sh
index f83935e47fb..23cc5651257 100755
--- a/tests/queries/0_stateless/02477_analyzer_function_hints.sh
+++ b/tests/queries/0_stateless/02477_analyzer_function_hints.sh
@@ -6,22 +6,22 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
-$CLICKHOUSE_CLIENT -q "SELECT plu(1, 1) SETTINGS allow_experimental_analyzer = 1;" 2>&1 \
+$CLICKHOUSE_CLIENT -q "SELECT plu(1, 1) SETTINGS enable_analyzer = 1;" 2>&1 \
     | grep "Maybe you meant: \['plus'\]" &>/dev/null;
 
-$CLICKHOUSE_CLIENT -q "SELECT uniqExac(1, 1) SETTINGS allow_experimental_analyzer = 1;" 2>&1 \
+$CLICKHOUSE_CLIENT -q "SELECT uniqExac(1, 1) SETTINGS enable_analyzer = 1;" 2>&1 \
     | grep "Maybe you meant: \['uniqExact'\]" &>/dev/null;
 
 $CLICKHOUSE_CLIENT -q "DROP FUNCTION IF EXISTS test_user_defined_function_$CLICKHOUSE_DATABASE;"
 $CLICKHOUSE_CLIENT -q "CREATE FUNCTION test_user_defined_function_$CLICKHOUSE_DATABASE AS x -> x + 1;"
-$CLICKHOUSE_CLIENT -q "SELECT test_user_defined_function_${CLICKHOUSE_DATABASE}A(1) SETTINGS allow_experimental_analyzer = 1;" 2>&1 \
+$CLICKHOUSE_CLIENT -q "SELECT test_user_defined_function_${CLICKHOUSE_DATABASE}A(1) SETTINGS enable_analyzer = 1;" 2>&1 \
    | grep -E "Maybe you meant: \[.*'test_user_defined_function_$CLICKHOUSE_DATABASE'.*\]" &>/dev/null;
 $CLICKHOUSE_CLIENT -q "DROP FUNCTION test_user_defined_function_$CLICKHOUSE_DATABASE";
 
-$CLICKHOUSE_CLIENT -q "WITH (x -> x + 1) AS lambda_function SELECT lambda_functio(1) SETTINGS allow_experimental_analyzer = 1;" 2>&1 \
+$CLICKHOUSE_CLIENT -q "WITH (x -> x + 1) AS lambda_function SELECT lambda_functio(1) SETTINGS enable_analyzer = 1;" 2>&1 \
     | grep "Maybe you meant: \['lambda_function'\]" &>/dev/null;
 
-$CLICKHOUSE_CLIENT -q "SELECT row_numbe() OVER (PARTITION BY 1) SETTINGS allow_experimental_analyzer = 1;" 2>&1 \
+$CLICKHOUSE_CLIENT -q "SELECT row_numbe() OVER (PARTITION BY 1) SETTINGS enable_analyzer = 1;" 2>&1 \
     | grep "Maybe you meant: \['row_number'\]" &>/dev/null;
 
 $CLICKHOUSE_CLIENT -q "SELECT 1";
diff --git a/tests/queries/0_stateless/02477_exists_fuzz_43478.sql b/tests/queries/0_stateless/02477_exists_fuzz_43478.sql
index 8ec876eb252..c225befed0e 100644
--- a/tests/queries/0_stateless/02477_exists_fuzz_43478.sql
+++ b/tests/queries/0_stateless/02477_exists_fuzz_43478.sql
@@ -1,3 +1,3 @@
 create table test_rows_compact_part__fuzz_11 (x UInt32) engine = MergeTree order by x;
 insert into test_rows_compact_part__fuzz_11 select 1;
-select 1 from test_rows_compact_part__fuzz_11 where exists(select 1) settings allow_experimental_analyzer=1;
+select 1 from test_rows_compact_part__fuzz_11 where exists(select 1) settings enable_analyzer=1;
diff --git a/tests/queries/0_stateless/02477_fuse_quantiles.sql b/tests/queries/0_stateless/02477_fuse_quantiles.sql
index c0719d771d7..8ddc029f75f 100644
--- a/tests/queries/0_stateless/02477_fuse_quantiles.sql
+++ b/tests/queries/0_stateless/02477_fuse_quantiles.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 SET optimize_syntax_fuse_functions = 1;
 
 DROP TABLE IF EXISTS fuse_tbl;
diff --git a/tests/queries/0_stateless/02477_logical_expressions_optimizer_low_cardinality.reference b/tests/queries/0_stateless/02477_logical_expressions_optimizer_low_cardinality.reference
index 11c178ac0d0..0ce90cde108 100644
--- a/tests/queries/0_stateless/02477_logical_expressions_optimizer_low_cardinality.reference
+++ b/tests/queries/0_stateless/02477_logical_expressions_optimizer_low_cardinality.reference
@@ -19,7 +19,7 @@ QUERY id: 0
                 COLUMN id: 2, column_name: a, result_type: LowCardinality(String), source_id: 3
                 CONSTANT id: 8, constant_value: Tuple_(\'x\', \'y\'), constant_value_type: Tuple(String, String)
           CONSTANT id: 9, constant_value: UInt64_0, constant_value_type: UInt8
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 SELECT a
 FROM t_logical_expressions_optimizer_low_cardinality
 WHERE (a = \'x\') OR (\'y\' = a)
@@ -41,7 +41,7 @@ QUERY id: 0
                 COLUMN id: 2, column_name: a, result_type: LowCardinality(String), source_id: 3
                 CONSTANT id: 8, constant_value: Tuple_(\'x\', \'y\'), constant_value_type: Tuple(String, String)
           CONSTANT id: 9, constant_value: UInt64_0, constant_value_type: UInt8
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 SELECT a
 FROM t_logical_expressions_optimizer_low_cardinality
 WHERE (a != \'x\') AND (a != \'y\')
@@ -63,7 +63,7 @@ QUERY id: 0
                 COLUMN id: 2, column_name: a, result_type: LowCardinality(String), source_id: 3
                 CONSTANT id: 8, constant_value: Tuple_(\'x\', \'y\'), constant_value_type: Tuple(String, String)
           CONSTANT id: 9, constant_value: \'UInt8\', constant_value_type: String
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 SELECT a
 FROM t_logical_expressions_optimizer_low_cardinality
 WHERE (a != \'x\') AND (\'y\' != a)
@@ -85,7 +85,7 @@ QUERY id: 0
                 COLUMN id: 2, column_name: a, result_type: LowCardinality(String), source_id: 3
                 CONSTANT id: 8, constant_value: Tuple_(\'x\', \'y\'), constant_value_type: Tuple(String, String)
           CONSTANT id: 9, constant_value: \'UInt8\', constant_value_type: String
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 SELECT a
 FROM t_logical_expressions_optimizer_low_cardinality
 WHERE (b = 0) OR (b = 1)
@@ -111,7 +111,7 @@ QUERY id: 0
               LIST id: 11, nodes: 2
                 COLUMN id: 8, column_name: b, result_type: UInt32, source_id: 3
                 CONSTANT id: 12, constant_value: UInt64_1, constant_value_type: UInt8
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 SELECT a
 FROM t_logical_expressions_optimizer_low_cardinality
 WHERE (b != 0) AND (b != 1)
@@ -137,4 +137,4 @@ QUERY id: 0
               LIST id: 11, nodes: 2
                 COLUMN id: 8, column_name: b, result_type: UInt32, source_id: 3
                 CONSTANT id: 12, constant_value: UInt64_1, constant_value_type: UInt8
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
diff --git a/tests/queries/0_stateless/02477_logical_expressions_optimizer_low_cardinality.sql b/tests/queries/0_stateless/02477_logical_expressions_optimizer_low_cardinality.sql
index 976b21a7e29..b328e9658d0 100644
--- a/tests/queries/0_stateless/02477_logical_expressions_optimizer_low_cardinality.sql
+++ b/tests/queries/0_stateless/02477_logical_expressions_optimizer_low_cardinality.sql
@@ -5,21 +5,21 @@ CREATE TABLE t_logical_expressions_optimizer_low_cardinality (a LowCardinality(S
 -- LowCardinality case, ignore optimize_min_equality_disjunction_chain_length limit, optimizer applied
 -- Chain of OR equals
 EXPLAIN SYNTAX SELECT a FROM t_logical_expressions_optimizer_low_cardinality WHERE a = 'x' OR a = 'y';
-EXPLAIN QUERY TREE SELECT a FROM t_logical_expressions_optimizer_low_cardinality WHERE a = 'x' OR a = 'y' SETTINGS allow_experimental_analyzer = 1;
+EXPLAIN QUERY TREE SELECT a FROM t_logical_expressions_optimizer_low_cardinality WHERE a = 'x' OR a = 'y' SETTINGS enable_analyzer = 1;
 EXPLAIN SYNTAX SELECT a FROM t_logical_expressions_optimizer_low_cardinality WHERE a = 'x' OR 'y' = a;
-EXPLAIN QUERY TREE SELECT a FROM t_logical_expressions_optimizer_low_cardinality WHERE a = 'x' OR 'y' = a SETTINGS allow_experimental_analyzer = 1;
+EXPLAIN QUERY TREE SELECT a FROM t_logical_expressions_optimizer_low_cardinality WHERE a = 'x' OR 'y' = a SETTINGS enable_analyzer = 1;
 -- Chain of AND notEquals
 EXPLAIN SYNTAX SELECT a FROM t_logical_expressions_optimizer_low_cardinality WHERE a <> 'x' AND a <> 'y';
-EXPLAIN QUERY TREE SELECT a FROM t_logical_expressions_optimizer_low_cardinality WHERE a <> 'x' AND a <> 'y' SETTINGS allow_experimental_analyzer = 1;
+EXPLAIN QUERY TREE SELECT a FROM t_logical_expressions_optimizer_low_cardinality WHERE a <> 'x' AND a <> 'y' SETTINGS enable_analyzer = 1;
 EXPLAIN SYNTAX SELECT a FROM t_logical_expressions_optimizer_low_cardinality WHERE a <> 'x' AND 'y' <> a;
-EXPLAIN QUERY TREE SELECT a FROM t_logical_expressions_optimizer_low_cardinality WHERE a <> 'x' AND 'y' <> a SETTINGS allow_experimental_analyzer = 1;
+EXPLAIN QUERY TREE SELECT a FROM t_logical_expressions_optimizer_low_cardinality WHERE a <> 'x' AND 'y' <> a SETTINGS enable_analyzer = 1;
 
 -- Non-LowCardinality case, optimizer not applied for short chains
 -- Chain of OR equals
 EXPLAIN SYNTAX SELECT a FROM t_logical_expressions_optimizer_low_cardinality WHERE b = 0 OR b = 1;
-EXPLAIN QUERY TREE SELECT a FROM t_logical_expressions_optimizer_low_cardinality WHERE b = 0 OR b = 1 SETTINGS allow_experimental_analyzer = 1;
+EXPLAIN QUERY TREE SELECT a FROM t_logical_expressions_optimizer_low_cardinality WHERE b = 0 OR b = 1 SETTINGS enable_analyzer = 1;
 -- Chain of AND notEquals
 EXPLAIN SYNTAX SELECT a FROM t_logical_expressions_optimizer_low_cardinality WHERE b <> 0 AND b <> 1;
-EXPLAIN QUERY TREE SELECT a FROM t_logical_expressions_optimizer_low_cardinality WHERE b <> 0 AND b <> 1 SETTINGS allow_experimental_analyzer = 1;
+EXPLAIN QUERY TREE SELECT a FROM t_logical_expressions_optimizer_low_cardinality WHERE b <> 0 AND b <> 1 SETTINGS enable_analyzer = 1;
 
 DROP TABLE t_logical_expressions_optimizer_low_cardinality;
diff --git a/tests/queries/0_stateless/02478_analyzer_table_expression_aliases.sql b/tests/queries/0_stateless/02478_analyzer_table_expression_aliases.sql
index 66f50f7b26d..a1eb88c634d 100644
--- a/tests/queries/0_stateless/02478_analyzer_table_expression_aliases.sql
+++ b/tests/queries/0_stateless/02478_analyzer_table_expression_aliases.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS test_table;
 CREATE TABLE test_table
diff --git a/tests/queries/0_stateless/02478_window_frame_type_groups.sql b/tests/queries/0_stateless/02478_window_frame_type_groups.sql
index f762bcb61ee..a01e1813cc1 100644
--- a/tests/queries/0_stateless/02478_window_frame_type_groups.sql
+++ b/tests/queries/0_stateless/02478_window_frame_type_groups.sql
@@ -1,7 +1,7 @@
-SET allow_experimental_analyzer = 0;
+SET enable_analyzer = 0;
 
 SELECT toUInt64(dense_rank(1) OVER (ORDER BY 100 ASC GROUPS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)) FROM numbers(10); -- { serverError NOT_IMPLEMENTED }
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 SELECT toUInt64(dense_rank(1) OVER (ORDER BY 100 ASC GROUPS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)) FROM numbers(10); -- { serverError NOT_IMPLEMENTED }
diff --git a/tests/queries/0_stateless/02479_analyzer_aggregation_crash.sql b/tests/queries/0_stateless/02479_analyzer_aggregation_crash.sql
index c931a3ab634..1e890740665 100644
--- a/tests/queries/0_stateless/02479_analyzer_aggregation_crash.sql
+++ b/tests/queries/0_stateless/02479_analyzer_aggregation_crash.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 SET compile_aggregate_expressions = 1;
 SET min_count_to_compile_aggregate_expression = 0;
 
diff --git a/tests/queries/0_stateless/02479_analyzer_aggregation_totals_rollup_crash_fix.sql b/tests/queries/0_stateless/02479_analyzer_aggregation_totals_rollup_crash_fix.sql
index 6cd3e6a9385..004e61ee1a2 100644
--- a/tests/queries/0_stateless/02479_analyzer_aggregation_totals_rollup_crash_fix.sql
+++ b/tests/queries/0_stateless/02479_analyzer_aggregation_totals_rollup_crash_fix.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 SELECT anyLast(number) FROM numbers(1) GROUP BY number WITH ROLLUP WITH TOTALS;
 
diff --git a/tests/queries/0_stateless/02479_analyzer_join_with_constants.sql b/tests/queries/0_stateless/02479_analyzer_join_with_constants.sql
index 9f77cf39f47..503bbe63347 100644
--- a/tests/queries/0_stateless/02479_analyzer_join_with_constants.sql
+++ b/tests/queries/0_stateless/02479_analyzer_join_with_constants.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 SELECT * FROM (SELECT 1 AS id) AS t1 INNER JOIN (SELECT 1 AS id) AS t2 ON t1.id = t2.id AND 1;
 
diff --git a/tests/queries/0_stateless/02479_mysql_connect_to_self.sql b/tests/queries/0_stateless/02479_mysql_connect_to_self.sql
index cf2220073d3..b8ed7e0c03e 100644
--- a/tests/queries/0_stateless/02479_mysql_connect_to_self.sql
+++ b/tests/queries/0_stateless/02479_mysql_connect_to_self.sql
@@ -7,7 +7,7 @@ DROP TABLE IF EXISTS foo;
 CREATE TABLE foo (key UInt32, a String, b Int64, c String) ENGINE = TinyLog;
 INSERT INTO foo VALUES (1, 'one', -1, 'een'), (2, 'two', -2, 'twee'), (3, 'three', -3, 'drie'), (4, 'four', -4, 'vier'), (5, 'five', -5, 'vijf');
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 SELECT '---';
 SELECT * FROM mysql('127.0.0.1:9004', currentDatabase(), foo, 'default', '', SETTINGS connect_timeout = 100, connection_wait_timeout = 100) ORDER BY key;
diff --git a/tests/queries/0_stateless/02480_analyzer_alias_nullptr.sql b/tests/queries/0_stateless/02480_analyzer_alias_nullptr.sql
index f6b381e5c70..07503de1b10 100644
--- a/tests/queries/0_stateless/02480_analyzer_alias_nullptr.sql
+++ b/tests/queries/0_stateless/02480_analyzer_alias_nullptr.sql
@@ -1,3 +1,3 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 SELECT min(b), x AS b FROM (SELECT max(number) FROM numbers(1)); -- { serverError UNKNOWN_IDENTIFIER }
diff --git a/tests/queries/0_stateless/02480_tlp_nan.reference b/tests/queries/0_stateless/02480_tlp_nan.reference
index befd1f66564..29e5bffa7f0 100644
--- a/tests/queries/0_stateless/02480_tlp_nan.reference
+++ b/tests/queries/0_stateless/02480_tlp_nan.reference
@@ -1,21 +1,21 @@
 -- {echo}
-SELECT sqrt(-1) as x, not(x), not(not(x)), (not(x)) IS NULL SETTINGS allow_experimental_analyzer=1;
+SELECT sqrt(-1) as x, not(x), not(not(x)), (not(x)) IS NULL SETTINGS enable_analyzer=1;
 nan	0	1	0
-SELECT sqrt(-1) as x, not(x), not(not(x)), (not(x)) IS NULL SETTINGS allow_experimental_analyzer=0;
+SELECT sqrt(-1) as x, not(x), not(not(x)), (not(x)) IS NULL SETTINGS enable_analyzer=0;
 nan	0	1	0
-SELECT -inf as x, not(x), not(not(x)), (not(x)) IS NULL SETTINGS allow_experimental_analyzer=1;
+SELECT -inf as x, not(x), not(not(x)), (not(x)) IS NULL SETTINGS enable_analyzer=1;
 -inf	0	1	0
-SELECT -inf as x, not(x), not(not(x)), (not(x)) IS NULL SETTINGS allow_experimental_analyzer=0;
+SELECT -inf as x, not(x), not(not(x)), (not(x)) IS NULL SETTINGS enable_analyzer=0;
 -inf	0	1	0
-SELECT NULL as x, not(x), not(not(x)), (not(x)) IS NULL SETTINGS allow_experimental_analyzer=1;
+SELECT NULL as x, not(x), not(not(x)), (not(x)) IS NULL SETTINGS enable_analyzer=1;
 \N	\N	\N	1
-SELECT NULL as x, not(x), not(not(x)), (not(x)) IS NULL SETTINGS allow_experimental_analyzer=0;
+SELECT NULL as x, not(x), not(not(x)), (not(x)) IS NULL SETTINGS enable_analyzer=0;
 \N	\N	\N	1
-SELECT inf as x, not(x), not(not(x)), (not(x)) IS NULL SETTINGS allow_experimental_analyzer=1;
+SELECT inf as x, not(x), not(not(x)), (not(x)) IS NULL SETTINGS enable_analyzer=1;
 inf	0	1	0
-SELECT inf as x, not(x), not(not(x)), (not(x)) IS NULL SETTINGS allow_experimental_analyzer=0;
+SELECT inf as x, not(x), not(not(x)), (not(x)) IS NULL SETTINGS enable_analyzer=0;
 inf	0	1	0
-SELECT nan as x, not(x), not(not(x)), (not(x)) IS NULL SETTINGS allow_experimental_analyzer=1;
+SELECT nan as x, not(x), not(not(x)), (not(x)) IS NULL SETTINGS enable_analyzer=1;
 nan	0	1	0
-SELECT nan as x, not(x), not(not(x)), (not(x)) IS NULL SETTINGS allow_experimental_analyzer=0;
+SELECT nan as x, not(x), not(not(x)), (not(x)) IS NULL SETTINGS enable_analyzer=0;
 nan	0	1	0
diff --git a/tests/queries/0_stateless/02480_tlp_nan.sql b/tests/queries/0_stateless/02480_tlp_nan.sql
index e24bc9a9830..55318e0cb30 100644
--- a/tests/queries/0_stateless/02480_tlp_nan.sql
+++ b/tests/queries/0_stateless/02480_tlp_nan.sql
@@ -1,15 +1,15 @@
 -- {echo}
-SELECT sqrt(-1) as x, not(x), not(not(x)), (not(x)) IS NULL SETTINGS allow_experimental_analyzer=1;
-SELECT sqrt(-1) as x, not(x), not(not(x)), (not(x)) IS NULL SETTINGS allow_experimental_analyzer=0;
+SELECT sqrt(-1) as x, not(x), not(not(x)), (not(x)) IS NULL SETTINGS enable_analyzer=1;
+SELECT sqrt(-1) as x, not(x), not(not(x)), (not(x)) IS NULL SETTINGS enable_analyzer=0;
 
-SELECT -inf as x, not(x), not(not(x)), (not(x)) IS NULL SETTINGS allow_experimental_analyzer=1;
-SELECT -inf as x, not(x), not(not(x)), (not(x)) IS NULL SETTINGS allow_experimental_analyzer=0;
+SELECT -inf as x, not(x), not(not(x)), (not(x)) IS NULL SETTINGS enable_analyzer=1;
+SELECT -inf as x, not(x), not(not(x)), (not(x)) IS NULL SETTINGS enable_analyzer=0;
 
-SELECT NULL as x, not(x), not(not(x)), (not(x)) IS NULL SETTINGS allow_experimental_analyzer=1;
-SELECT NULL as x, not(x), not(not(x)), (not(x)) IS NULL SETTINGS allow_experimental_analyzer=0;
+SELECT NULL as x, not(x), not(not(x)), (not(x)) IS NULL SETTINGS enable_analyzer=1;
+SELECT NULL as x, not(x), not(not(x)), (not(x)) IS NULL SETTINGS enable_analyzer=0;
 
-SELECT inf as x, not(x), not(not(x)), (not(x)) IS NULL SETTINGS allow_experimental_analyzer=1;
-SELECT inf as x, not(x), not(not(x)), (not(x)) IS NULL SETTINGS allow_experimental_analyzer=0;
+SELECT inf as x, not(x), not(not(x)), (not(x)) IS NULL SETTINGS enable_analyzer=1;
+SELECT inf as x, not(x), not(not(x)), (not(x)) IS NULL SETTINGS enable_analyzer=0;
 
-SELECT nan as x, not(x), not(not(x)), (not(x)) IS NULL SETTINGS allow_experimental_analyzer=1;
-SELECT nan as x, not(x), not(not(x)), (not(x)) IS NULL SETTINGS allow_experimental_analyzer=0;
+SELECT nan as x, not(x), not(not(x)), (not(x)) IS NULL SETTINGS enable_analyzer=1;
+SELECT nan as x, not(x), not(not(x)), (not(x)) IS NULL SETTINGS enable_analyzer=0;
diff --git a/tests/queries/0_stateless/02481_aggregation_in_order_plan.sql b/tests/queries/0_stateless/02481_aggregation_in_order_plan.sql
index 490060dee6f..139e0ed4b77 100644
--- a/tests/queries/0_stateless/02481_aggregation_in_order_plan.sql
+++ b/tests/queries/0_stateless/02481_aggregation_in_order_plan.sql
@@ -5,5 +5,5 @@ insert into tab select 0, number % 3, 2 - intDiv(number, 3), (number % 3 + 1) *
 insert into tab select 0, number % 3, 2 - intDiv(number, 3), (number % 3 + 1) * 100 from numbers(6);
 
 select a, any(b), c, d from tab where b = 1 group by a, c, d order by c, d settings optimize_aggregation_in_order=1, query_plan_aggregation_in_order=1;
-select * from (explain actions = 1, sorting=1 select a, any(b), c, d from tab where b = 1 group by a, c, d settings optimize_aggregation_in_order=1, query_plan_aggregation_in_order=1) where explain like '%ReadFromMergeTree%' or explain like '%Aggregating%' or explain like '%Order:%' settings allow_experimental_analyzer=0;
-select * from (explain actions = 1, sorting=1 select a, any(b), c, d from tab where b = 1 group by a, c, d settings optimize_aggregation_in_order=1, query_plan_aggregation_in_order=1) where explain like '%ReadFromMergeTree%' or explain like '%Aggregating%' or explain like '%Order:%' settings allow_experimental_analyzer=1;
+select * from (explain actions = 1, sorting=1 select a, any(b), c, d from tab where b = 1 group by a, c, d settings optimize_aggregation_in_order=1, query_plan_aggregation_in_order=1) where explain like '%ReadFromMergeTree%' or explain like '%Aggregating%' or explain like '%Order:%' settings enable_analyzer=0;
+select * from (explain actions = 1, sorting=1 select a, any(b), c, d from tab where b = 1 group by a, c, d settings optimize_aggregation_in_order=1, query_plan_aggregation_in_order=1) where explain like '%ReadFromMergeTree%' or explain like '%Aggregating%' or explain like '%Order:%' settings enable_analyzer=1;
diff --git a/tests/queries/0_stateless/02481_analyzer_join_alias_unknown_identifier_crash.sql b/tests/queries/0_stateless/02481_analyzer_join_alias_unknown_identifier_crash.sql
index 0c5f0eba750..8b5b272f547 100644
--- a/tests/queries/0_stateless/02481_analyzer_join_alias_unknown_identifier_crash.sql
+++ b/tests/queries/0_stateless/02481_analyzer_join_alias_unknown_identifier_crash.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS test_table_join_1;
 CREATE TABLE test_table_join_1
diff --git a/tests/queries/0_stateless/02481_analyzer_optimize_aggregation_arithmetics.sql b/tests/queries/0_stateless/02481_analyzer_optimize_aggregation_arithmetics.sql
index ca91d137bf4..e68de0af522 100644
--- a/tests/queries/0_stateless/02481_analyzer_optimize_aggregation_arithmetics.sql
+++ b/tests/queries/0_stateless/02481_analyzer_optimize_aggregation_arithmetics.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 SET optimize_arithmetic_operations_in_aggregate_functions = 1;
 
 -- { echoOn }
diff --git a/tests/queries/0_stateless/02481_analyzer_optimize_grouping_sets_keys.sql b/tests/queries/0_stateless/02481_analyzer_optimize_grouping_sets_keys.sql
index fef71fdf94f..8e6b132f5b7 100644
--- a/tests/queries/0_stateless/02481_analyzer_optimize_grouping_sets_keys.sql
+++ b/tests/queries/0_stateless/02481_analyzer_optimize_grouping_sets_keys.sql
@@ -1,4 +1,4 @@
-set allow_experimental_analyzer = 1;
+set enable_analyzer = 1;
 set optimize_syntax_fuse_functions = 0;
 
 EXPLAIN QUERY TREE run_passes=1
diff --git a/tests/queries/0_stateless/02483_cuturlparameter_with_arrays.sql b/tests/queries/0_stateless/02483_cuturlparameter_with_arrays.sql
index 6d64d2685b7..0cf95d10a88 100644
--- a/tests/queries/0_stateless/02483_cuturlparameter_with_arrays.sql
+++ b/tests/queries/0_stateless/02483_cuturlparameter_with_arrays.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 -- { echoOn }
 
diff --git a/tests/queries/0_stateless/02483_elapsed_time.sh b/tests/queries/0_stateless/02483_elapsed_time.sh
index fdb23d6da01..274b30cefb9 100755
--- a/tests/queries/0_stateless/02483_elapsed_time.sh
+++ b/tests/queries/0_stateless/02483_elapsed_time.sh
@@ -17,7 +17,7 @@ EXCEPTION_BEFORE_START_QUERY="WITH
                                   FROM system.numbers
                                   WHERE number IN (sub)
                               )
-                              SETTINGS enable_global_with_statement = 0, allow_experimental_analyzer = 1"
+                              SETTINGS enable_global_with_statement = 0, enable_analyzer = 1"
 
 
 # For this query the system.query_log needs to show ExceptionBeforeStart and elapsed seconds <= 1.0
diff --git a/tests/queries/0_stateless/02489_analyzer_indexes.sql b/tests/queries/0_stateless/02489_analyzer_indexes.sql
index b5438ddd31a..dcf18016da6 100644
--- a/tests/queries/0_stateless/02489_analyzer_indexes.sql
+++ b/tests/queries/0_stateless/02489_analyzer_indexes.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS test_table;
 CREATE TABLE test_table
@@ -59,4 +59,3 @@ WHERE t1.id = 1 AND t1.value_1 = '1' AND t1.value_2 = '1' AND t1.value_3 = '1'
 SETTINGS force_primary_key = 1, force_data_skipping_indices = 'value_1_idx, value_2_idx, value_3_idx';
 
 DROP TABLE test_table;
-
diff --git a/tests/queries/0_stateless/02493_analyzer_sum_if_to_count_if.sql b/tests/queries/0_stateless/02493_analyzer_sum_if_to_count_if.sql
index f1dbfa1f32a..171e080961a 100644
--- a/tests/queries/0_stateless/02493_analyzer_sum_if_to_count_if.sql
+++ b/tests/queries/0_stateless/02493_analyzer_sum_if_to_count_if.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 SET optimize_rewrite_sum_if_to_count_if = 1;
 
 EXPLAIN QUERY TREE (SELECT sumIf(1, (number % 2) == 0) FROM numbers(10));
diff --git a/tests/queries/0_stateless/02493_analyzer_table_functions_untuple.sql b/tests/queries/0_stateless/02493_analyzer_table_functions_untuple.sql
index bdbe65c643b..c9687783dc9 100644
--- a/tests/queries/0_stateless/02493_analyzer_table_functions_untuple.sql
+++ b/tests/queries/0_stateless/02493_analyzer_table_functions_untuple.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 SELECT number FROM numbers(untuple(tuple(1)));
 
diff --git a/tests/queries/0_stateless/02493_analyzer_uniq_injective_functions_elimination.sql b/tests/queries/0_stateless/02493_analyzer_uniq_injective_functions_elimination.sql
index 5a3b2379fde..ca37c6f3833 100644
--- a/tests/queries/0_stateless/02493_analyzer_uniq_injective_functions_elimination.sql
+++ b/tests/queries/0_stateless/02493_analyzer_uniq_injective_functions_elimination.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1, optimize_injective_functions_inside_uniq = 1;
+SET enable_analyzer = 1, optimize_injective_functions_inside_uniq = 1;
 
 -- Simple test
 EXPLAIN QUERY TREE SELECT uniqCombined(tuple('')) FROM numbers(1);
diff --git a/tests/queries/0_stateless/02494_analyzer_compound_expression_crash_fix.sql b/tests/queries/0_stateless/02494_analyzer_compound_expression_crash_fix.sql
index 3e6f9f42724..20b0bdd46b0 100644
--- a/tests/queries/0_stateless/02494_analyzer_compound_expression_crash_fix.sql
+++ b/tests/queries/0_stateless/02494_analyzer_compound_expression_crash_fix.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS test_table;
 CREATE TABLE test_table (
diff --git a/tests/queries/0_stateless/02494_query_cache_explain.sql b/tests/queries/0_stateless/02494_query_cache_explain.sql
index bf376b47fdb..decdd92c477 100644
--- a/tests/queries/0_stateless/02494_query_cache_explain.sql
+++ b/tests/queries/0_stateless/02494_query_cache_explain.sql
@@ -1,7 +1,7 @@
 -- Tags: no-parallel
 -- Tag no-parallel: Messes with internal cache
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 SET query_cache_system_table_handling = 'save';
 
 SYSTEM DROP QUERY CACHE;
diff --git a/tests/queries/0_stateless/02494_query_cache_nested_query_bug.sh b/tests/queries/0_stateless/02494_query_cache_nested_query_bug.sh
index 24803ed7420..9a07ddf0386 100755
--- a/tests/queries/0_stateless/02494_query_cache_nested_query_bug.sh
+++ b/tests/queries/0_stateless/02494_query_cache_nested_query_bug.sh
@@ -15,8 +15,8 @@ ${CLICKHOUSE_CLIENT} --query "CREATE TABLE tab (a UInt64) ENGINE=MergeTree() ORD
 ${CLICKHOUSE_CLIENT} --query "INSERT INTO tab VALUES (1) (2) (3)"
 ${CLICKHOUSE_CLIENT} --query "INSERT INTO tab VALUES (3) (4) (5)"
 
-SETTINGS_NO_ANALYZER="SETTINGS use_query_cache=1, max_threads=1, allow_experimental_analyzer=0, merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability=0.0"
-SETTINGS_ANALYZER="SETTINGS use_query_cache=1, max_threads=1, allow_experimental_analyzer=1, merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability=0.0"
+SETTINGS_NO_ANALYZER="SETTINGS use_query_cache=1, max_threads=1, enable_analyzer=0, merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability=0.0"
+SETTINGS_ANALYZER="SETTINGS use_query_cache=1, max_threads=1, enable_analyzer=1, merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability=0.0"
 
 # Verify that the first query does two aggregations and the second query zero aggregations. Since query cache is currently not integrated
 # with EXPLAIN PLAN, we need to check the logs.
diff --git a/tests/queries/0_stateless/02495_analyzer_storage_join.sql b/tests/queries/0_stateless/02495_analyzer_storage_join.sql
index 7e6c03971f9..1148cd68338 100644
--- a/tests/queries/0_stateless/02495_analyzer_storage_join.sql
+++ b/tests/queries/0_stateless/02495_analyzer_storage_join.sql
@@ -2,7 +2,7 @@ DROP TABLE IF EXISTS t;
 DROP TABLE IF EXISTS t1;
 DROP TABLE IF EXISTS tj;
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 SET single_join_prefer_left_table = 0;
 
 CREATE TABLE tj (key2 UInt64, key1 Int64, a UInt64, b UInt64, x UInt64, y UInt64) ENGINE = Join(ALL, RIGHT, key1, key2);
diff --git a/tests/queries/0_stateless/02495_sum_if_to_count_if_bug.sql b/tests/queries/0_stateless/02495_sum_if_to_count_if_bug.sql
index 0791b374668..c00c0ba4e07 100644
--- a/tests/queries/0_stateless/02495_sum_if_to_count_if_bug.sql
+++ b/tests/queries/0_stateless/02495_sum_if_to_count_if_bug.sql
@@ -1,4 +1,3 @@
 select sum(if((number % NULL) = 2, 0, 1)) FROM numbers(1024) settings optimize_rewrite_sum_if_to_count_if=0;
-select sum(if((number % NULL) = 2, 0, 1)) FROM numbers(1024) settings optimize_rewrite_sum_if_to_count_if=1, allow_experimental_analyzer=0;
-select sum(if((number % NULL) = 2, 0, 1)) FROM numbers(1024) settings optimize_rewrite_sum_if_to_count_if=1, allow_experimental_analyzer=1;
-
+select sum(if((number % NULL) = 2, 0, 1)) FROM numbers(1024) settings optimize_rewrite_sum_if_to_count_if=1, enable_analyzer=0;
+select sum(if((number % NULL) = 2, 0, 1)) FROM numbers(1024) settings optimize_rewrite_sum_if_to_count_if=1, enable_analyzer=1;
diff --git a/tests/queries/0_stateless/02496_remove_redundant_sorting.sh b/tests/queries/0_stateless/02496_remove_redundant_sorting.sh
index 661b32fce72..646e2501a99 100755
--- a/tests/queries/0_stateless/02496_remove_redundant_sorting.sh
+++ b/tests/queries/0_stateless/02496_remove_redundant_sorting.sh
@@ -8,8 +8,8 @@ if [ -z ${ENABLE_ANALYZER+x} ]; then
     ENABLE_ANALYZER=0
 fi
 
-DISABLE_OPTIMIZATION="SET allow_experimental_analyzer=$ENABLE_ANALYZER;SET query_plan_remove_redundant_sorting=0;SET optimize_duplicate_order_by_and_distinct=0"
-ENABLE_OPTIMIZATION="SET allow_experimental_analyzer=$ENABLE_ANALYZER;SET query_plan_remove_redundant_sorting=1;SET optimize_duplicate_order_by_and_distinct=0"
+DISABLE_OPTIMIZATION="SET enable_analyzer=$ENABLE_ANALYZER;SET query_plan_remove_redundant_sorting=0;SET optimize_duplicate_order_by_and_distinct=0"
+ENABLE_OPTIMIZATION="SET enable_analyzer=$ENABLE_ANALYZER;SET query_plan_remove_redundant_sorting=1;SET optimize_duplicate_order_by_and_distinct=0"
 
 echo "-- Disabled query_plan_remove_redundant_sorting"
 echo "-- ORDER BY clauses in subqueries are untouched"
diff --git a/tests/queries/0_stateless/02497_analyzer_sum_if_count_if_pass_crash_fix.sql b/tests/queries/0_stateless/02497_analyzer_sum_if_count_if_pass_crash_fix.sql
index 51522565014..7533a333225 100644
--- a/tests/queries/0_stateless/02497_analyzer_sum_if_count_if_pass_crash_fix.sql
+++ b/tests/queries/0_stateless/02497_analyzer_sum_if_count_if_pass_crash_fix.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 SET optimize_rewrite_sum_if_to_count_if = 1;
 
 SELECT sum(if((number % 2) = 0 AS cond_expr, 1 AS one_expr, 0 AS zero_expr) AS if_expr), sum(cond_expr), sum(if_expr), one_expr, zero_expr FROM numbers(100);
diff --git a/tests/queries/0_stateless/02497_having_without_actual_aggregation_bug.sql b/tests/queries/0_stateless/02497_having_without_actual_aggregation_bug.sql
index b28cbd4861e..e5fd26e484e 100644
--- a/tests/queries/0_stateless/02497_having_without_actual_aggregation_bug.sql
+++ b/tests/queries/0_stateless/02497_having_without_actual_aggregation_bug.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 select number from numbers_mt(10) having number >= 9;
 
diff --git a/tests/queries/0_stateless/02497_if_transform_strings_to_enum.sql b/tests/queries/0_stateless/02497_if_transform_strings_to_enum.sql
index 131eac390f1..cc2d839fbc1 100644
--- a/tests/queries/0_stateless/02497_if_transform_strings_to_enum.sql
+++ b/tests/queries/0_stateless/02497_if_transform_strings_to_enum.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 SET optimize_if_transform_strings_to_enum = 1;
 
 SELECT transform(number, [2, 4, 6], ['google', 'censor.net', 'yahoo'], 'other') FROM system.numbers LIMIT 10;
diff --git a/tests/queries/0_stateless/02497_storage_join_right_assert.sql b/tests/queries/0_stateless/02497_storage_join_right_assert.sql
index 8f9134e9504..eabaa236379 100644
--- a/tests/queries/0_stateless/02497_storage_join_right_assert.sql
+++ b/tests/queries/0_stateless/02497_storage_join_right_assert.sql
@@ -7,8 +7,8 @@ CREATE TABLE t2 (key UInt64, a UInt64) ENGINE = Join(ALL, RIGHT, key);
 INSERT INTO t1 VALUES (1, 1), (2, 2);
 INSERT INTO t2 VALUES (2, 2), (3, 3);
 
-SET allow_experimental_analyzer = 0;
+SET enable_analyzer = 0;
 SELECT * FROM t1 ALL RIGHT JOIN t2 USING (key) ORDER BY key;
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 SELECT * FROM t1 ALL RIGHT JOIN t2 USING (key) ORDER BY key;
diff --git a/tests/queries/0_stateless/02498_analyzer_aggregate_functions_arithmetic_operations_pass_fix.sql b/tests/queries/0_stateless/02498_analyzer_aggregate_functions_arithmetic_operations_pass_fix.sql
index 8491018eb72..76c44f9e0cb 100644
--- a/tests/queries/0_stateless/02498_analyzer_aggregate_functions_arithmetic_operations_pass_fix.sql
+++ b/tests/queries/0_stateless/02498_analyzer_aggregate_functions_arithmetic_operations_pass_fix.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 SET optimize_arithmetic_operations_in_aggregate_functions = 1;
 
 DROP TABLE IF EXISTS test_table;
diff --git a/tests/queries/0_stateless/02498_analyzer_settings_push_down.sql b/tests/queries/0_stateless/02498_analyzer_settings_push_down.sql
index 67623869f0a..472ab358d97 100644
--- a/tests/queries/0_stateless/02498_analyzer_settings_push_down.sql
+++ b/tests/queries/0_stateless/02498_analyzer_settings_push_down.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 SET optimize_functions_to_subcolumns = 0;
 
 DROP TABLE IF EXISTS test_table;
diff --git a/tests/queries/0_stateless/02498_storage_join_key_positions.sql.j2 b/tests/queries/0_stateless/02498_storage_join_key_positions.sql.j2
index e2dad61a93e..e814b8eaf3f 100644
--- a/tests/queries/0_stateless/02498_storage_join_key_positions.sql.j2
+++ b/tests/queries/0_stateless/02498_storage_join_key_positions.sql.j2
@@ -13,7 +13,7 @@ INSERT INTO tjj VALUES (11, 11, 11, 1000), (21, 21, 21, 2000), (31, 31, 31, 3000
 
 
 {% for use_analyzer in [0, 1] -%}
-SET allow_experimental_analyzer = '{{ use_analyzer }}';
+SET enable_analyzer = '{{ use_analyzer }}';
 
 SELECT '--- using ---';
 SELECT * FROM t1 ALL INNER JOIN tj USING (key1, key2, key3) ORDER BY key1;
diff --git a/tests/queries/0_stateless/02499_analyzer_aggregate_function_lambda_crash_fix.sql b/tests/queries/0_stateless/02499_analyzer_aggregate_function_lambda_crash_fix.sql
index f2698512112..7ac817aecde 100644
--- a/tests/queries/0_stateless/02499_analyzer_aggregate_function_lambda_crash_fix.sql
+++ b/tests/queries/0_stateless/02499_analyzer_aggregate_function_lambda_crash_fix.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 SELECT count((t, x_0, x_1) -> ((key_2, x_0, x_1) IN (NULL, NULL, '0.3'))) FROM numbers(10); -- { serverError UNSUPPORTED_METHOD }
 SELECT count((t, x_0, x_1) -> ((key_2, x_0, x_1) IN (NULL, NULL, '0.3'))) OVER (PARTITION BY id) FROM numbers(10); -- { serverError UNSUPPORTED_METHOD }
diff --git a/tests/queries/0_stateless/02499_analyzer_set_index.sql b/tests/queries/0_stateless/02499_analyzer_set_index.sql
index f90ae61541f..52d96cfcabf 100644
--- a/tests/queries/0_stateless/02499_analyzer_set_index.sql
+++ b/tests/queries/0_stateless/02499_analyzer_set_index.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS test_table;
 CREATE TABLE test_table
diff --git a/tests/queries/0_stateless/02500_analyzer_storage_view_crash_fix.sql b/tests/queries/0_stateless/02500_analyzer_storage_view_crash_fix.sql
index 8f4d14b95cc..f0484a68566 100644
--- a/tests/queries/0_stateless/02500_analyzer_storage_view_crash_fix.sql
+++ b/tests/queries/0_stateless/02500_analyzer_storage_view_crash_fix.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS test_table;
 CREATE TABLE test_table
diff --git a/tests/queries/0_stateless/02500_remove_redundant_distinct.sh b/tests/queries/0_stateless/02500_remove_redundant_distinct.sh
index c4f0994cd13..3c06119e8d2 100755
--- a/tests/queries/0_stateless/02500_remove_redundant_distinct.sh
+++ b/tests/queries/0_stateless/02500_remove_redundant_distinct.sh
@@ -9,8 +9,8 @@ if [ -z ${ENABLE_ANALYZER+x} ]; then
 fi
 
 OPTIMIZATION_SETTING="query_plan_remove_redundant_distinct"
-DISABLE_OPTIMIZATION="set allow_experimental_analyzer=$ENABLE_ANALYZER;SET $OPTIMIZATION_SETTING=0;SET optimize_duplicate_order_by_and_distinct=0"
-ENABLE_OPTIMIZATION="set allow_experimental_analyzer=$ENABLE_ANALYZER;SET $OPTIMIZATION_SETTING=1;SET optimize_duplicate_order_by_and_distinct=0"
+DISABLE_OPTIMIZATION="set enable_analyzer=$ENABLE_ANALYZER;SET $OPTIMIZATION_SETTING=0;SET optimize_duplicate_order_by_and_distinct=0"
+ENABLE_OPTIMIZATION="set enable_analyzer=$ENABLE_ANALYZER;SET $OPTIMIZATION_SETTING=1;SET optimize_duplicate_order_by_and_distinct=0"
 
 echo "-- Disabled $OPTIMIZATION_SETTING"
 query="SELECT DISTINCT *
diff --git a/tests/queries/0_stateless/02501_analyzer_expired_context_crash_fix.sql b/tests/queries/0_stateless/02501_analyzer_expired_context_crash_fix.sql
index b9ec14501bd..e2c940c829a 100644
--- a/tests/queries/0_stateless/02501_analyzer_expired_context_crash_fix.sql
+++ b/tests/queries/0_stateless/02501_analyzer_expired_context_crash_fix.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS test_table;
 CREATE TABLE test_table
diff --git a/tests/queries/0_stateless/02502_analyzer_insert_select_crash_fix.sql b/tests/queries/0_stateless/02502_analyzer_insert_select_crash_fix.sql
index 4643f65988a..a438276bda5 100644
--- a/tests/queries/0_stateless/02502_analyzer_insert_select_crash_fix.sql
+++ b/tests/queries/0_stateless/02502_analyzer_insert_select_crash_fix.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS test_table;
 CREATE TABLE test_table
diff --git a/tests/queries/0_stateless/02503_join_switch_alias_fuzz.sql b/tests/queries/0_stateless/02503_join_switch_alias_fuzz.sql
index 28d64bf3881..113a8493de6 100644
--- a/tests/queries/0_stateless/02503_join_switch_alias_fuzz.sql
+++ b/tests/queries/0_stateless/02503_join_switch_alias_fuzz.sql
@@ -1,4 +1,4 @@
 SELECT * FROM (SELECT 1 AS id, '' AS test) AS a
 LEFT JOIN (SELECT test, 1 AS id, NULL AS test) AS b ON b.id = a.id
-SETTINGS join_algorithm = 'auto', max_rows_in_join = 1, allow_experimental_analyzer = 1
+SETTINGS join_algorithm = 'auto', max_rows_in_join = 1, enable_analyzer = 1
 ;
diff --git a/tests/queries/0_stateless/02513_analyzer_duplicate_alias_crash_fix.sql b/tests/queries/0_stateless/02513_analyzer_duplicate_alias_crash_fix.sql
index fb50ea2c4ca..e54252b5c5f 100644
--- a/tests/queries/0_stateless/02513_analyzer_duplicate_alias_crash_fix.sql
+++ b/tests/queries/0_stateless/02513_analyzer_duplicate_alias_crash_fix.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 SELECT toUInt64(NULL) AS x FROM (SELECT 1) HAVING x IN
     (SELECT NULL FROM (SELECT x IN (SELECT x IN (SELECT 1), x IN (SELECT 1) FROM (SELECT 1 WHERE x IN (SELECT NULL FROM (SELECT NULL))))));
diff --git a/tests/queries/0_stateless/02513_analyzer_sort_msan.sql b/tests/queries/0_stateless/02513_analyzer_sort_msan.sql
index e5beccaff2a..b86a15e9ef0 100644
--- a/tests/queries/0_stateless/02513_analyzer_sort_msan.sql
+++ b/tests/queries/0_stateless/02513_analyzer_sort_msan.sql
@@ -1,6 +1,6 @@
 DROP TABLE IF EXISTS products;
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 CREATE TABLE products (`price` UInt32) ENGINE = Memory;
 INSERT INTO products VALUES (1);
diff --git a/tests/queries/0_stateless/02514_analyzer_drop_join_on.sql b/tests/queries/0_stateless/02514_analyzer_drop_join_on.sql
index 2406be13aa8..df84e2f50b2 100644
--- a/tests/queries/0_stateless/02514_analyzer_drop_join_on.sql
+++ b/tests/queries/0_stateless/02514_analyzer_drop_join_on.sql
@@ -15,7 +15,7 @@ INSERT INTO c VALUES (1, 1, 'a'), (2, 2, 'b'), (3, 3, 'c');
 CREATE TABLE d (k UInt64, d1 UInt64, d2 String) ENGINE = Memory;
 INSERT INTO d VALUES (1, 1, 'a'), (2, 2, 'b'), (3, 3, 'c');
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 -- { echoOn }
 
diff --git a/tests/queries/0_stateless/02515_analyzer_null_for_empty.sql b/tests/queries/0_stateless/02515_analyzer_null_for_empty.sql
index de21e9b475e..e12f215743b 100644
--- a/tests/queries/0_stateless/02515_analyzer_null_for_empty.sql
+++ b/tests/queries/0_stateless/02515_analyzer_null_for_empty.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 SET aggregate_functions_null_for_empty = 1;
 
 SELECT max(aggr) FROM (SELECT max('92233720368547758.06') AS aggr FROM system.one);
diff --git a/tests/queries/0_stateless/02516_join_with_totals_and_subquery_bug.sql.j2 b/tests/queries/0_stateless/02516_join_with_totals_and_subquery_bug.sql.j2
index 09447dfce65..a199165a38f 100644
--- a/tests/queries/0_stateless/02516_join_with_totals_and_subquery_bug.sql.j2
+++ b/tests/queries/0_stateless/02516_join_with_totals_and_subquery_bug.sql.j2
@@ -13,7 +13,7 @@ INNER JOIN
     GROUP BY 1
         WITH TOTALS
 ) AS t2 USING (a)
-SETTINGS allow_experimental_analyzer=0;
+SETTINGS enable_analyzer=0;
 
 SELECT *
 FROM
@@ -30,7 +30,7 @@ INNER JOIN
     GROUP BY 1
         WITH TOTALS
 ) AS t2 USING (a)
-SETTINGS allow_experimental_analyzer=1;
+SETTINGS enable_analyzer=1;
 
 SELECT a
 FROM
diff --git a/tests/queries/0_stateless/02516_projections_and_context.sql b/tests/queries/0_stateless/02516_projections_and_context.sql
index 334544eb4fa..ec14fc0aa9e 100644
--- a/tests/queries/0_stateless/02516_projections_and_context.sql
+++ b/tests/queries/0_stateless/02516_projections_and_context.sql
@@ -1,10 +1,10 @@
 DROP TABLE IF EXISTS test1__fuzz_37;
 CREATE TABLE test1__fuzz_37 (`i` Date) ENGINE = MergeTree ORDER BY i;
 insert into test1__fuzz_37 values ('2020-10-10');
-set allow_experimental_analyzer = 0;
+set enable_analyzer = 0;
 SELECT count() FROM test1__fuzz_37 GROUP BY dictHas(NULL, (dictHas(NULL, (('', materialize(NULL)), materialize(NULL))), 'KeyKey')), dictHas('test_dictionary', tuple(materialize('Ke\0'))), tuple(dictHas(NULL, (tuple('Ke\0Ke\0Ke\0Ke\0Ke\0Ke\0\0\0\0Ke\0'), materialize(NULL)))), 'test_dicti\0nary', (('', materialize(NULL)), dictHas(NULL, (dictHas(NULL, tuple(materialize(NULL))), 'KeyKeyKeyKeyKeyKeyKeyKey')), materialize(NULL)); -- { serverError BAD_ARGUMENTS }
 SELECT count() FROM test1__fuzz_37 GROUP BY dictHas('non_existing_dictionary', materialize('a')); -- { serverError BAD_ARGUMENTS }
-set allow_experimental_analyzer = 1;
+set enable_analyzer = 1;
 SELECT count() FROM test1__fuzz_37 GROUP BY dictHas(NULL, (dictHas(NULL, (('', materialize(NULL)), materialize(NULL))), 'KeyKey')), dictHas('test_dictionary', tuple(materialize('Ke\0'))), tuple(dictHas(NULL, (tuple('Ke\0Ke\0Ke\0Ke\0Ke\0Ke\0\0\0\0Ke\0'), materialize(NULL)))), 'test_dicti\0nary', (('', materialize(NULL)), dictHas(NULL, (dictHas(NULL, tuple(materialize(NULL))), 'KeyKeyKeyKeyKeyKeyKeyKey')), materialize(NULL)); -- { serverError BAD_ARGUMENTS }
 SELECT count() FROM test1__fuzz_37 GROUP BY dictHas('non_existing_dictionary', materialize('a')); -- { serverError BAD_ARGUMENTS }
 DROP TABLE test1__fuzz_37;
diff --git a/tests/queries/0_stateless/02518_rewrite_aggregate_function_with_if.sql b/tests/queries/0_stateless/02518_rewrite_aggregate_function_with_if.sql
index fe882da67cb..4ed13307c29 100644
--- a/tests/queries/0_stateless/02518_rewrite_aggregate_function_with_if.sql
+++ b/tests/queries/0_stateless/02518_rewrite_aggregate_function_with_if.sql
@@ -1,4 +1,4 @@
-set allow_experimental_analyzer = true;
+set enable_analyzer = true;
 -- { echoOn }
 
 set optimize_rewrite_aggregate_function_with_if = false;
diff --git a/tests/queries/0_stateless/02521_analyzer_aggregation_without_column.sql b/tests/queries/0_stateless/02521_analyzer_aggregation_without_column.sql
index 105bce6711c..50bf3cd45e4 100644
--- a/tests/queries/0_stateless/02521_analyzer_aggregation_without_column.sql
+++ b/tests/queries/0_stateless/02521_analyzer_aggregation_without_column.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS test_table;
 CREATE TABLE test_table
diff --git a/tests/queries/0_stateless/02521_analyzer_array_join_crash.sql b/tests/queries/0_stateless/02521_analyzer_array_join_crash.sql
index 7842d47d757..f5d601303a2 100644
--- a/tests/queries/0_stateless/02521_analyzer_array_join_crash.sql
+++ b/tests/queries/0_stateless/02521_analyzer_array_join_crash.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS test_table;
 CREATE TABLE test_table
diff --git a/tests/queries/0_stateless/02525_analyzer_function_in_crash_fix.sql b/tests/queries/0_stateless/02525_analyzer_function_in_crash_fix.sql
index 95b896d38ab..dd1688ad400 100644
--- a/tests/queries/0_stateless/02525_analyzer_function_in_crash_fix.sql
+++ b/tests/queries/0_stateless/02525_analyzer_function_in_crash_fix.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS test_table;
 CREATE TABLE test_table
diff --git a/tests/queries/0_stateless/02532_analyzer_aggregation_with_rollup.sql b/tests/queries/0_stateless/02532_analyzer_aggregation_with_rollup.sql
index 09097eb029f..587ef71df8a 100644
--- a/tests/queries/0_stateless/02532_analyzer_aggregation_with_rollup.sql
+++ b/tests/queries/0_stateless/02532_analyzer_aggregation_with_rollup.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 SELECT
     sum(a.number) AS total,
diff --git a/tests/queries/0_stateless/02534_analyzer_grouping_function.sql b/tests/queries/0_stateless/02534_analyzer_grouping_function.sql
index 3163e03d579..ee1cc1d88d1 100644
--- a/tests/queries/0_stateless/02534_analyzer_grouping_function.sql
+++ b/tests/queries/0_stateless/02534_analyzer_grouping_function.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS test_table;
 CREATE TABLE test_table
diff --git a/tests/queries/0_stateless/02535_analyzer_group_by_use_nulls.sql b/tests/queries/0_stateless/02535_analyzer_group_by_use_nulls.sql
index 4ae5df9629a..59bbfc96289 100644
--- a/tests/queries/0_stateless/02535_analyzer_group_by_use_nulls.sql
+++ b/tests/queries/0_stateless/02535_analyzer_group_by_use_nulls.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 
 -- { echoOn }
 SELECT number, number % 2, sum(number) AS val
diff --git a/tests/queries/0_stateless/02535_analyzer_limit_offset.sql b/tests/queries/0_stateless/02535_analyzer_limit_offset.sql
index 8f98d823e5c..96aef9557c5 100644
--- a/tests/queries/0_stateless/02535_analyzer_limit_offset.sql
+++ b/tests/queries/0_stateless/02535_analyzer_limit_offset.sql
@@ -1,3 +1,3 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 SELECT number FROM numbers(100) LIMIT 10 OFFSET 10;
diff --git a/tests/queries/0_stateless/02538_analyzer_create_table_as_select.sql b/tests/queries/0_stateless/02538_analyzer_create_table_as_select.sql
index 168066ce2f9..16634e996fe 100644
--- a/tests/queries/0_stateless/02538_analyzer_create_table_as_select.sql
+++ b/tests/queries/0_stateless/02538_analyzer_create_table_as_select.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS test_table_data;
 CREATE TABLE test_table_data
diff --git a/tests/queries/0_stateless/02540_analyzer_matcher_alias_materialized_columns.sql b/tests/queries/0_stateless/02540_analyzer_matcher_alias_materialized_columns.sql
index cc622dde8fe..58840796c7e 100644
--- a/tests/queries/0_stateless/02540_analyzer_matcher_alias_materialized_columns.sql
+++ b/tests/queries/0_stateless/02540_analyzer_matcher_alias_materialized_columns.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS test_table;
 CREATE TABLE test_table
diff --git a/tests/queries/0_stateless/02540_input_format_json_ignore_unknown_keys_in_named_tuple.sh b/tests/queries/0_stateless/02540_input_format_json_ignore_unknown_keys_in_named_tuple.sh
index eccac543215..48b3b27680c 100755
--- a/tests/queries/0_stateless/02540_input_format_json_ignore_unknown_keys_in_named_tuple.sh
+++ b/tests/queries/0_stateless/02540_input_format_json_ignore_unknown_keys_in_named_tuple.sh
@@ -115,7 +115,7 @@ EOL
 # NOTE: due to [1] we cannot use dot.dot notation, only tupleElement()
 #
 #   [1]: https://github.com/ClickHouse/ClickHouse/issues/24607
-$CLICKHOUSE_LOCAL --allow_experimental_analyzer=1 "${gharchive_settings[@]}" --structure="${gharchive_structure[*]}" -q "
+$CLICKHOUSE_LOCAL --enable_analyzer=1 "${gharchive_settings[@]}" --structure="${gharchive_structure[*]}" -q "
     SELECT
         payload.issue.labels.name AS labels,
         payload.pull_request.merged_by.login AS merged_by
diff --git a/tests/queries/0_stateless/02541_analyzer_grouping_sets_crash_fix.sql b/tests/queries/0_stateless/02541_analyzer_grouping_sets_crash_fix.sql
index d7af475bbe7..b9aa251bc96 100644
--- a/tests/queries/0_stateless/02541_analyzer_grouping_sets_crash_fix.sql
+++ b/tests/queries/0_stateless/02541_analyzer_grouping_sets_crash_fix.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 WITH pow(NULL, 256) AS four SELECT NULL AS two GROUP BY GROUPING SETS ((pow(two, 65536)));
 
diff --git a/tests/queries/0_stateless/02552_analyzer_optimize_group_by_function_keys_crash.sql b/tests/queries/0_stateless/02552_analyzer_optimize_group_by_function_keys_crash.sql
index ee9032472a7..85740cd85a2 100644
--- a/tests/queries/0_stateless/02552_analyzer_optimize_group_by_function_keys_crash.sql
+++ b/tests/queries/0_stateless/02552_analyzer_optimize_group_by_function_keys_crash.sql
@@ -1,3 +1,3 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 SELECT NULL GROUP BY tuple('0.0000000007'), count(NULL) OVER (ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING) -- { serverError ILLEGAL_AGGREGATION };
diff --git a/tests/queries/0_stateless/02553_type_object_analyzer.sql b/tests/queries/0_stateless/02553_type_object_analyzer.sql
index 55482a02ed1..eb4e49757cf 100644
--- a/tests/queries/0_stateless/02553_type_object_analyzer.sql
+++ b/tests/queries/0_stateless/02553_type_object_analyzer.sql
@@ -1,6 +1,6 @@
 SET output_format_json_named_tuples_as_objects = 1;
 SET allow_experimental_object_type = 1;
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS t_json_analyzer;
 CREATE TABLE t_json_analyzer (a JSON) ENGINE = Memory;
diff --git a/tests/queries/0_stateless/02554_fix_grouping_sets_predicate_push_down.sql b/tests/queries/0_stateless/02554_fix_grouping_sets_predicate_push_down.sql
index f926b9037d2..0eed4a8c592 100644
--- a/tests/queries/0_stateless/02554_fix_grouping_sets_predicate_push_down.sql
+++ b/tests/queries/0_stateless/02554_fix_grouping_sets_predicate_push_down.sql
@@ -60,7 +60,7 @@ FROM
                 (day_))
     ) AS t
 )
-WHERE type_1 = 'all' settings allow_experimental_analyzer=0;
+WHERE type_1 = 'all' settings enable_analyzer=0;
 
 -- Query plan with analyzer has less Filter steps (which is more optimal)
 EXPLAIN PIPELINE
@@ -83,7 +83,7 @@ FROM
                 (day_))
     ) AS t
 )
-WHERE type_1 = 'all' settings allow_experimental_analyzer=1;
+WHERE type_1 = 'all' settings enable_analyzer=1;
 
 SELECT '';
 SELECT '---Result---';
@@ -129,7 +129,7 @@ FROM
                 (day_))
     ) AS t
 )
-WHERE day_ = '2023-01-05' settings allow_experimental_analyzer=0;
+WHERE day_ = '2023-01-05' settings enable_analyzer=0;
 
 -- Query plan with analyzer has less Filter steps (which is more optimal)
 EXPLAIN PIPELINE
@@ -151,6 +151,6 @@ FROM
                 (day_))
     ) AS t
 )
-WHERE day_ = '2023-01-05' settings allow_experimental_analyzer=1;
+WHERE day_ = '2023-01-05' settings enable_analyzer=1;
 
 DROP TABLE test_grouping_sets_predicate;
diff --git a/tests/queries/0_stateless/02560_analyzer_materialized_view.sql b/tests/queries/0_stateless/02560_analyzer_materialized_view.sql
index 1f268fe1e16..3fdef366dc9 100644
--- a/tests/queries/0_stateless/02560_analyzer_materialized_view.sql
+++ b/tests/queries/0_stateless/02560_analyzer_materialized_view.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS test_table;
 CREATE TABLE test_table
diff --git a/tests/queries/0_stateless/02563_analyzer_merge.sql b/tests/queries/0_stateless/02563_analyzer_merge.sql
index 217fb7019c4..6c252c22773 100644
--- a/tests/queries/0_stateless/02563_analyzer_merge.sql
+++ b/tests/queries/0_stateless/02563_analyzer_merge.sql
@@ -1,6 +1,6 @@
 -- Tags: no-parallel
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP DATABASE IF EXISTS 02563_db;
 CREATE DATABASE 02563_db;
diff --git a/tests/queries/0_stateless/02564_analyzer_cross_to_inner.sql b/tests/queries/0_stateless/02564_analyzer_cross_to_inner.sql
index a83cd238982..7032559e066 100644
--- a/tests/queries/0_stateless/02564_analyzer_cross_to_inner.sql
+++ b/tests/queries/0_stateless/02564_analyzer_cross_to_inner.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS t1;
 DROP TABLE IF EXISTS t2;
diff --git a/tests/queries/0_stateless/02565_analyzer_limit_settings.sql b/tests/queries/0_stateless/02565_analyzer_limit_settings.sql
index 7c02c2d0d20..1dd6735e64d 100644
--- a/tests/queries/0_stateless/02565_analyzer_limit_settings.sql
+++ b/tests/queries/0_stateless/02565_analyzer_limit_settings.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 -- { echoOn }
 SET limit = 0;
diff --git a/tests/queries/0_stateless/02566_analyzer_limit_settings_distributed.sql b/tests/queries/0_stateless/02566_analyzer_limit_settings_distributed.sql
index 1624344b5a9..a2620f436f4 100644
--- a/tests/queries/0_stateless/02566_analyzer_limit_settings_distributed.sql
+++ b/tests/queries/0_stateless/02566_analyzer_limit_settings_distributed.sql
@@ -1,6 +1,6 @@
 -- Tags: distributed
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 SELECT 'limit', * FROM remote('127.1', view(SELECT * FROM numbers(10))) SETTINGS limit=5;
 SELECT 'offset', * FROM remote('127.1', view(SELECT * FROM numbers(10))) SETTINGS offset=5;
diff --git a/tests/queries/0_stateless/02567_and_consistency.reference b/tests/queries/0_stateless/02567_and_consistency.reference
index e0014f187a8..7e6d1f24f43 100644
--- a/tests/queries/0_stateless/02567_and_consistency.reference
+++ b/tests/queries/0_stateless/02567_and_consistency.reference
@@ -8,7 +8,7 @@ true
 =====
 1
 =====
-allow_experimental_analyzer
+enable_analyzer
 true
 #45440
 2086579505	0	1	0	0
diff --git a/tests/queries/0_stateless/02567_and_consistency.sql b/tests/queries/0_stateless/02567_and_consistency.sql
index 0eeab99e539..0442a6dad7f 100644
--- a/tests/queries/0_stateless/02567_and_consistency.sql
+++ b/tests/queries/0_stateless/02567_and_consistency.sql
@@ -50,9 +50,9 @@ SELECT 1 and sin(1);
 
 SELECT '=====';
 
-SELECT 'allow_experimental_analyzer';
+SELECT 'enable_analyzer';
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 SELECT toBool(sin(SUM(number))) AS x
 FROM
diff --git a/tests/queries/0_stateless/02576_predicate_push_down_sorting_fix.sql b/tests/queries/0_stateless/02576_predicate_push_down_sorting_fix.sql
index 2dade7837b7..486a26613f6 100644
--- a/tests/queries/0_stateless/02576_predicate_push_down_sorting_fix.sql
+++ b/tests/queries/0_stateless/02576_predicate_push_down_sorting_fix.sql
@@ -1,3 +1,3 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 EXPLAIN header = 1, actions = 1 SELECT number FROM (SELECT number FROM numbers(2) ORDER BY ignore(2)) WHERE ignore(2);
diff --git a/tests/queries/0_stateless/02576_rewrite_array_exists_to_has.sql b/tests/queries/0_stateless/02576_rewrite_array_exists_to_has.sql
index 5233f2f7e3c..b5a123e3767 100644
--- a/tests/queries/0_stateless/02576_rewrite_array_exists_to_has.sql
+++ b/tests/queries/0_stateless/02576_rewrite_array_exists_to_has.sql
@@ -1,4 +1,4 @@
-set allow_experimental_analyzer = true;
+set enable_analyzer = true;
 
 set optimize_rewrite_array_exists_to_has = false;
 EXPLAIN QUERY TREE run_passes = 1  select arrayExists(x -> x = 5 , materialize(range(10))) from numbers(10);
@@ -8,7 +8,7 @@ set optimize_rewrite_array_exists_to_has = true;
 EXPLAIN QUERY TREE run_passes = 1  select arrayExists(x -> x = 5 , materialize(range(10))) from numbers(10);
 EXPLAIN QUERY TREE run_passes = 1  select arrayExists(x -> 5 = x , materialize(range(10))) from numbers(10);
 
-set allow_experimental_analyzer = false;
+set enable_analyzer = false;
 
 set optimize_rewrite_array_exists_to_has = false;
 EXPLAIN SYNTAX select arrayExists(x -> x = 5 , materialize(range(10))) from numbers(10);
diff --git a/tests/queries/0_stateless/02577_analyzer_array_join_calc_twice.sql b/tests/queries/0_stateless/02577_analyzer_array_join_calc_twice.sql
index b6bb258db28..0b281dd4f81 100644
--- a/tests/queries/0_stateless/02577_analyzer_array_join_calc_twice.sql
+++ b/tests/queries/0_stateless/02577_analyzer_array_join_calc_twice.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 SELECT 1 + arrayJoin(a) AS m FROM (SELECT [1, 2, 3] AS a) GROUP BY m;
 
diff --git a/tests/queries/0_stateless/02579_fill_empty_chunk.sql b/tests/queries/0_stateless/02579_fill_empty_chunk.sql
index 30942b154c9..aeae98df7a3 100644
--- a/tests/queries/0_stateless/02579_fill_empty_chunk.sql
+++ b/tests/queries/0_stateless/02579_fill_empty_chunk.sql
@@ -1,7 +1,7 @@
 -- this SELECT produces empty chunk in FillingTransform
 
 SET enable_positional_arguments = 0;
-SET allow_experimental_analyzer = 0;
+SET enable_analyzer = 0;
 
 SELECT
     2 AS x,
diff --git a/tests/queries/0_stateless/02579_fill_empty_chunk_analyzer.sql b/tests/queries/0_stateless/02579_fill_empty_chunk_analyzer.sql
index 8350173f443..144640149ea 100644
--- a/tests/queries/0_stateless/02579_fill_empty_chunk_analyzer.sql
+++ b/tests/queries/0_stateless/02579_fill_empty_chunk_analyzer.sql
@@ -1,7 +1,7 @@
 -- this SELECT produces empty chunk in FillingTransform
 
 SET enable_positional_arguments = 0;
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 -- With analyzer this special query has correct output
 SELECT
diff --git a/tests/queries/0_stateless/02582_analyzer_join_subquery_empty_column_list.sql b/tests/queries/0_stateless/02582_analyzer_join_subquery_empty_column_list.sql
index 10e5871cc44..33c9296a0d8 100644
--- a/tests/queries/0_stateless/02582_analyzer_join_subquery_empty_column_list.sql
+++ b/tests/queries/0_stateless/02582_analyzer_join_subquery_empty_column_list.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 -- { echoOn }
 
 SELECT a FROM ( select 1 AS a ) AS t1, ( select 2 AS b, 3 AS c) AS t2;
@@ -10,5 +10,3 @@ SELECT b FROM ( select 1 AS a UNION ALL select 1 as a ) AS t1, ( select 2 AS b,
 SELECT c FROM ( select 1 AS a UNION ALL select 1 as a ) AS t1, ( select 2 AS b, 3 AS c UNION ALL select 2 as b, 3 as c) AS t2;
 SELECT 42 FROM ( select 1 AS a UNION ALL select 1 as a ) AS t1, ( select 2 AS b, 3 AS c UNION ALL select 2 as b, 3 as c) AS t2;
 SELECT count() FROM ( select 1 AS a UNION ALL select 1 as a ) AS t1, ( select 2 AS b, 3 AS c UNION ALL select 2 as b, 3 as c) AS t2;
-
-
diff --git a/tests/queries/0_stateless/02661_quantile_approx.reference b/tests/queries/0_stateless/02661_quantile_approx.reference
index 0ee846a268b..239516f9ac4 100644
--- a/tests/queries/0_stateless/02661_quantile_approx.reference
+++ b/tests/queries/0_stateless/02661_quantile_approx.reference
@@ -33,10 +33,10 @@ FROM
     FROM numbers(49999)
 );
 [24902,44518,49999]
-select medianGK()(number) from numbers(10) SETTINGS allow_experimental_analyzer = 0; -- { serverError BAD_ARGUMENTS }
-select medianGK()(number) from numbers(10) SETTINGS allow_experimental_analyzer = 1; -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH }
-select quantileGK()(number) from numbers(10) SETTINGS allow_experimental_analyzer = 0; -- { serverError BAD_ARGUMENTS }
-select quantileGK()(number) from numbers(10) SETTINGS allow_experimental_analyzer = 1; -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH }
+select medianGK()(number) from numbers(10) SETTINGS enable_analyzer = 0; -- { serverError BAD_ARGUMENTS }
+select medianGK()(number) from numbers(10) SETTINGS enable_analyzer = 1; -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH }
+select quantileGK()(number) from numbers(10) SETTINGS enable_analyzer = 0; -- { serverError BAD_ARGUMENTS }
+select quantileGK()(number) from numbers(10) SETTINGS enable_analyzer = 1; -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH }
 select medianGK(100)(number) from numbers(10);
 4
 select quantileGK(100)(number) from numbers(10);
@@ -47,8 +47,8 @@ select quantileGK(100, 0.5, 0.75)(number) from numbers(10); -- { serverError NUM
 select quantileGK('abc', 0.5)(number) from numbers(10); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT }
 select quantileGK(1.23, 0.5)(number) from numbers(10); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT }
 select quantileGK(-100, 0.5)(number) from numbers(10); -- { serverError BAD_ARGUMENTS }
-select quantilesGK()(number) from numbers(10) SETTINGS allow_experimental_analyzer = 0; -- { serverError BAD_ARGUMENTS }
-select quantilesGK()(number) from numbers(10) SETTINGS allow_experimental_analyzer = 1; -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH }
+select quantilesGK()(number) from numbers(10) SETTINGS enable_analyzer = 0; -- { serverError BAD_ARGUMENTS }
+select quantilesGK()(number) from numbers(10) SETTINGS enable_analyzer = 1; -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH }
 select quantilesGK(100)(number) from numbers(10); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH }
 select quantilesGK(100, 0.5)(number) from numbers(10);
 [4]
diff --git a/tests/queries/0_stateless/02661_quantile_approx.sql b/tests/queries/0_stateless/02661_quantile_approx.sql
index c0004260fa1..732ce645c98 100644
--- a/tests/queries/0_stateless/02661_quantile_approx.sql
+++ b/tests/queries/0_stateless/02661_quantile_approx.sql
@@ -1,4 +1,4 @@
-set allow_experimental_analyzer = 1;
+set enable_analyzer = 1;
 
 -- { echoOn }
 with arrayJoin([0, 1, 2, 10]) as x select quantilesGK(100, 0.5, 0.4, 0.1)(x);
@@ -29,11 +29,11 @@ FROM
     FROM numbers(49999)
 );
 
-select medianGK()(number) from numbers(10) SETTINGS allow_experimental_analyzer = 0; -- { serverError BAD_ARGUMENTS }
-select medianGK()(number) from numbers(10) SETTINGS allow_experimental_analyzer = 1; -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH }
+select medianGK()(number) from numbers(10) SETTINGS enable_analyzer = 0; -- { serverError BAD_ARGUMENTS }
+select medianGK()(number) from numbers(10) SETTINGS enable_analyzer = 1; -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH }
 
-select quantileGK()(number) from numbers(10) SETTINGS allow_experimental_analyzer = 0; -- { serverError BAD_ARGUMENTS }
-select quantileGK()(number) from numbers(10) SETTINGS allow_experimental_analyzer = 1; -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH }
+select quantileGK()(number) from numbers(10) SETTINGS enable_analyzer = 0; -- { serverError BAD_ARGUMENTS }
+select quantileGK()(number) from numbers(10) SETTINGS enable_analyzer = 1; -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH }
 
 select medianGK(100)(number) from numbers(10);
 select quantileGK(100)(number) from numbers(10);
@@ -43,8 +43,8 @@ select quantileGK('abc', 0.5)(number) from numbers(10); -- { serverError ILLEGAL
 select quantileGK(1.23, 0.5)(number) from numbers(10); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT }
 select quantileGK(-100, 0.5)(number) from numbers(10); -- { serverError BAD_ARGUMENTS }
 
-select quantilesGK()(number) from numbers(10) SETTINGS allow_experimental_analyzer = 0; -- { serverError BAD_ARGUMENTS }
-select quantilesGK()(number) from numbers(10) SETTINGS allow_experimental_analyzer = 1; -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH }
+select quantilesGK()(number) from numbers(10) SETTINGS enable_analyzer = 0; -- { serverError BAD_ARGUMENTS }
+select quantilesGK()(number) from numbers(10) SETTINGS enable_analyzer = 1; -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH }
 
 select quantilesGK(100)(number) from numbers(10); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH }
 select quantilesGK(100, 0.5)(number) from numbers(10);
diff --git a/tests/queries/0_stateless/02662_first_last_value.reference b/tests/queries/0_stateless/02662_first_last_value.reference
index b0783399623..308cbf850b0 100644
--- a/tests/queries/0_stateless/02662_first_last_value.reference
+++ b/tests/queries/0_stateless/02662_first_last_value.reference
@@ -18,7 +18,7 @@ select last_value(b) ignore nulls from test;
 5
 select last_value(b) respect nulls from test;
 \N
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 -- first value
 select first_value(b) from test;
 3
diff --git a/tests/queries/0_stateless/02662_first_last_value.sql b/tests/queries/0_stateless/02662_first_last_value.sql
index 8e429e2e27d..16768bd6f1e 100644
--- a/tests/queries/0_stateless/02662_first_last_value.sql
+++ b/tests/queries/0_stateless/02662_first_last_value.sql
@@ -15,7 +15,7 @@ select last_value(b) from test;
 select last_value(b) ignore nulls from test;
 select last_value(b) respect nulls from test;
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 -- first value
 select first_value(b) from test;
diff --git a/tests/queries/0_stateless/02668_logical_optimizer_removing_redundant_checks.sql b/tests/queries/0_stateless/02668_logical_optimizer_removing_redundant_checks.sql
index 7d624195df9..dabdcfd5507 100644
--- a/tests/queries/0_stateless/02668_logical_optimizer_removing_redundant_checks.sql
+++ b/tests/queries/0_stateless/02668_logical_optimizer_removing_redundant_checks.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS 02668_logical_optimizer;
 
diff --git a/tests/queries/0_stateless/02674_trivial_count_analyzer.reference b/tests/queries/0_stateless/02674_trivial_count_analyzer.reference
index 05feadb58a0..9d7f393e71c 100644
--- a/tests/queries/0_stateless/02674_trivial_count_analyzer.reference
+++ b/tests/queries/0_stateless/02674_trivial_count_analyzer.reference
@@ -1,5 +1,5 @@
 -- { echoOn }
-set allow_experimental_analyzer=1;
+set enable_analyzer=1;
 set optimize_trivial_count_query=1;
 create table m3(a Int64, b UInt64) Engine=MergeTree order by tuple();
 select count() from m3;
diff --git a/tests/queries/0_stateless/02674_trivial_count_analyzer.sql b/tests/queries/0_stateless/02674_trivial_count_analyzer.sql
index 988d1b9ba92..c13a9dc68f2 100644
--- a/tests/queries/0_stateless/02674_trivial_count_analyzer.sql
+++ b/tests/queries/0_stateless/02674_trivial_count_analyzer.sql
@@ -2,7 +2,7 @@ drop table if exists m3;
 drop table if exists replacing_m3;
 
 -- { echoOn }
-set allow_experimental_analyzer=1;
+set enable_analyzer=1;
 set optimize_trivial_count_query=1;
 
 create table m3(a Int64, b UInt64) Engine=MergeTree order by tuple();
diff --git a/tests/queries/0_stateless/02675_predicate_push_down_filled_join_fix.sql b/tests/queries/0_stateless/02675_predicate_push_down_filled_join_fix.sql
index 930127497ae..6c8932b5d58 100644
--- a/tests/queries/0_stateless/02675_predicate_push_down_filled_join_fix.sql
+++ b/tests/queries/0_stateless/02675_predicate_push_down_filled_join_fix.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 SET single_join_prefer_left_table = 0;
 SET optimize_move_to_prewhere = 0;
 
diff --git a/tests/queries/0_stateless/02676_analyzer_limit_offset.sql b/tests/queries/0_stateless/02676_analyzer_limit_offset.sql
index 39c6b85f088..5dbe55e916c 100644
--- a/tests/queries/0_stateless/02676_analyzer_limit_offset.sql
+++ b/tests/queries/0_stateless/02676_analyzer_limit_offset.sql
@@ -1,4 +1,4 @@
-set allow_experimental_analyzer=1;
+set enable_analyzer=1;
 
 DROP TABLE IF EXISTS test;
 CREATE TABLE test (i UInt64) Engine = MergeTree() order by i;
diff --git a/tests/queries/0_stateless/02676_distinct_reading_in_order_analyzer.sql b/tests/queries/0_stateless/02676_distinct_reading_in_order_analyzer.sql
index f9ff1eed111..6a219cd3781 100644
--- a/tests/queries/0_stateless/02676_distinct_reading_in_order_analyzer.sql
+++ b/tests/queries/0_stateless/02676_distinct_reading_in_order_analyzer.sql
@@ -1,6 +1,6 @@
 drop table if exists t;
 
-set allow_experimental_analyzer=1;
+set enable_analyzer=1;
 
 create table t (a UInt64, b UInt64) engine=MergeTree() order by (a);
 insert into t select number % 2, number from numbers(10);
diff --git a/tests/queries/0_stateless/02677_analyzer_bitmap_has_any.sql b/tests/queries/0_stateless/02677_analyzer_bitmap_has_any.sql
index c06ea009c1d..dc906a92f1e 100644
--- a/tests/queries/0_stateless/02677_analyzer_bitmap_has_any.sql
+++ b/tests/queries/0_stateless/02677_analyzer_bitmap_has_any.sql
@@ -18,7 +18,7 @@ FROM
         bitmapHasAny(bitmapBuild([toUInt64(1)]), (
             SELECT groupBitmapState(toUInt64(2))
         )) has2
-) SETTINGS allow_experimental_analyzer = 0; -- { serverError ILLEGAL_TYPE_OF_ARGUMENT }
+) SETTINGS enable_analyzer = 0; -- { serverError ILLEGAL_TYPE_OF_ARGUMENT }
 
 SELECT '--------------';
 
@@ -32,5 +32,4 @@ FROM
         bitmapHasAny(bitmapBuild([toUInt64(1)]), (
             SELECT groupBitmapState(toUInt64(2))
         )) has2
-) SETTINGS allow_experimental_analyzer = 1;
-
+) SETTINGS enable_analyzer = 1;
diff --git a/tests/queries/0_stateless/02677_analyzer_compound_expressions.sql b/tests/queries/0_stateless/02677_analyzer_compound_expressions.sql
index 6b7fdab8993..90781f70158 100644
--- a/tests/queries/0_stateless/02677_analyzer_compound_expressions.sql
+++ b/tests/queries/0_stateless/02677_analyzer_compound_expressions.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 WITH ('a', 'b')::Tuple(c1 String, c2 String) AS t
 SELECT t.c1, t.c2;
diff --git a/tests/queries/0_stateless/02677_get_subcolumn_array_of_tuples.sql b/tests/queries/0_stateless/02677_get_subcolumn_array_of_tuples.sql
index 5779821afaa..95665979857 100644
--- a/tests/queries/0_stateless/02677_get_subcolumn_array_of_tuples.sql
+++ b/tests/queries/0_stateless/02677_get_subcolumn_array_of_tuples.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS t_get_subcolumn;
 
diff --git a/tests/queries/0_stateless/02679_explain_merge_tree_prewhere_row_policy.sql b/tests/queries/0_stateless/02679_explain_merge_tree_prewhere_row_policy.sql
index 4bc7be13490..4911979394a 100644
--- a/tests/queries/0_stateless/02679_explain_merge_tree_prewhere_row_policy.sql
+++ b/tests/queries/0_stateless/02679_explain_merge_tree_prewhere_row_policy.sql
@@ -10,8 +10,8 @@ INSERT INTO test_table VALUES (0, 'Value');
 DROP ROW POLICY IF EXISTS test_row_policy ON test_table;
 CREATE ROW POLICY test_row_policy ON test_table USING id >= 5 TO ALL;
 
-EXPLAIN header = 1, actions = 1 SELECT id, value FROM test_table PREWHERE id = 5 settings allow_experimental_analyzer=0;
-EXPLAIN header = 1, actions = 1 SELECT id, value FROM test_table PREWHERE id = 5 settings allow_experimental_analyzer=1;
+EXPLAIN header = 1, actions = 1 SELECT id, value FROM test_table PREWHERE id = 5 settings enable_analyzer=0;
+EXPLAIN header = 1, actions = 1 SELECT id, value FROM test_table PREWHERE id = 5 settings enable_analyzer=1;
 
 DROP ROW POLICY test_row_policy ON test_table;
 DROP TABLE test_table;
diff --git a/tests/queries/0_stateless/02699_polygons_sym_difference_rollup.sql b/tests/queries/0_stateless/02699_polygons_sym_difference_rollup.sql
index 502cca20ab2..680b98fb1bd 100644
--- a/tests/queries/0_stateless/02699_polygons_sym_difference_rollup.sql
+++ b/tests/queries/0_stateless/02699_polygons_sym_difference_rollup.sql
@@ -1,4 +1,4 @@
 SELECT polygonsSymDifferenceCartesian([[[(1., 1.)]] AS x], [x]) GROUP BY x WITH ROLLUP;
-SELECT [[(2147483647, 0.), (10.0001, 65535), (1, 255), (1023, 2147483646)]], polygonsSymDifferenceCartesian([[[(2147483647, 0.), (10.0001, 65535), (1023, 2147483646)]]], [[[(1000.0001, 10.0001)]]]) GROUP BY [[(2147483647, 0.), (10.0001, 65535), (1023, 2147483646)]] WITH ROLLUP SETTINGS allow_experimental_analyzer=0;
-SELECT [[(2147483647, 0.), (10.0001, 65535), (1, 255), (1023, 2147483646)]], polygonsSymDifferenceCartesian([[[(2147483647, 0.), (10.0001, 65535), (1023, 2147483646)]]], [[[(1000.0001, 10.0001)]]]) GROUP BY [[(2147483647, 0.), (10.0001, 65535), (1023, 2147483646)]] WITH ROLLUP SETTINGS allow_experimental_analyzer=1;
+SELECT [[(2147483647, 0.), (10.0001, 65535), (1, 255), (1023, 2147483646)]], polygonsSymDifferenceCartesian([[[(2147483647, 0.), (10.0001, 65535), (1023, 2147483646)]]], [[[(1000.0001, 10.0001)]]]) GROUP BY [[(2147483647, 0.), (10.0001, 65535), (1023, 2147483646)]] WITH ROLLUP SETTINGS enable_analyzer=0;
+SELECT [[(2147483647, 0.), (10.0001, 65535), (1, 255), (1023, 2147483646)]], polygonsSymDifferenceCartesian([[[(2147483647, 0.), (10.0001, 65535), (1023, 2147483646)]]], [[[(1000.0001, 10.0001)]]]) GROUP BY [[(2147483647, 0.), (10.0001, 65535), (1023, 2147483646)]] WITH ROLLUP SETTINGS enable_analyzer=1;
 SELECT polygonsSymDifferenceCartesian([[[(100.0001, 1000.0001), (-20., 20.), (10., 10.), (20., 20.), (20., -20.), (1000.0001, 1.1920928955078125e-7)]],[[(0.0001, 100000000000000000000.)]] AS x],[x]) GROUP BY x WITH ROLLUP;
diff --git a/tests/queries/0_stateless/02699_polygons_sym_difference_total.sql b/tests/queries/0_stateless/02699_polygons_sym_difference_total.sql
index 0fac4b11320..53d0a3bb543 100644
--- a/tests/queries/0_stateless/02699_polygons_sym_difference_total.sql
+++ b/tests/queries/0_stateless/02699_polygons_sym_difference_total.sql
@@ -1,2 +1,2 @@
-SET allow_experimental_analyzer=0;
+SET enable_analyzer=0;
 SELECT [(9223372036854775807, 1.1754943508222875e-38)], x, NULL, polygonsSymDifferenceCartesian([[[(1.1754943508222875e-38, 1.1920928955078125e-7), (0.5, 0.5)]], [[(1.1754943508222875e-38, 1.1920928955078125e-7), (1.1754943508222875e-38, 1.1920928955078125e-7)], [(0., 1.0001)]], [[(1., 1.0001)]] AS x], [[[(3.4028234663852886e38, 0.9999)]]]) GROUP BY GROUPING SETS ((x)) WITH TOTALS
diff --git a/tests/queries/0_stateless/02699_polygons_sym_difference_total_analyzer.sql b/tests/queries/0_stateless/02699_polygons_sym_difference_total_analyzer.sql
index 879e0e5297f..40f610ae5a6 100644
--- a/tests/queries/0_stateless/02699_polygons_sym_difference_total_analyzer.sql
+++ b/tests/queries/0_stateless/02699_polygons_sym_difference_total_analyzer.sql
@@ -1,2 +1,2 @@
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 SELECT [(9223372036854775807, 1.1754943508222875e-38)], x, NULL, polygonsSymDifferenceCartesian([[[(1.1754943508222875e-38, 1.1920928955078125e-7), (0.5, 0.5)]], [[(1.1754943508222875e-38, 1.1920928955078125e-7), (1.1754943508222875e-38, 1.1920928955078125e-7)], [(0., 1.0001)]], [[(1., 1.0001)]] AS x], [[[(3.4028234663852886e38, 0.9999)]]]) GROUP BY GROUPING SETS ((x)) WITH TOTALS
diff --git a/tests/queries/0_stateless/02701_invalid_having_NOT_AN_AGGREGATE.sql b/tests/queries/0_stateless/02701_invalid_having_NOT_AN_AGGREGATE.sql
index 092bda23164..9cfc4d83058 100644
--- a/tests/queries/0_stateless/02701_invalid_having_NOT_AN_AGGREGATE.sql
+++ b/tests/queries/0_stateless/02701_invalid_having_NOT_AN_AGGREGATE.sql
@@ -1 +1 @@
-SELECT a, sum(b) FROM (SELECT 1 AS a, 1 AS b, 0 AS c) GROUP BY a HAVING c SETTINGS allow_experimental_analyzer=1 -- { serverError NOT_AN_AGGREGATE }
+SELECT a, sum(b) FROM (SELECT 1 AS a, 1 AS b, 0 AS c) GROUP BY a HAVING c SETTINGS enable_analyzer=1 -- { serverError NOT_AN_AGGREGATE }
diff --git a/tests/queries/0_stateless/02702_logical_optimizer_with_nulls.sql b/tests/queries/0_stateless/02702_logical_optimizer_with_nulls.sql
index 72ab507f541..9e2927334e1 100644
--- a/tests/queries/0_stateless/02702_logical_optimizer_with_nulls.sql
+++ b/tests/queries/0_stateless/02702_logical_optimizer_with_nulls.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS 02702_logical_optimizer;
 
diff --git a/tests/queries/0_stateless/02703_explain_query_tree_is_forbidden_with_old_analyzer.sql b/tests/queries/0_stateless/02703_explain_query_tree_is_forbidden_with_old_analyzer.sql
index d351bfe402c..c028e74f1b5 100644
--- a/tests/queries/0_stateless/02703_explain_query_tree_is_forbidden_with_old_analyzer.sql
+++ b/tests/queries/0_stateless/02703_explain_query_tree_is_forbidden_with_old_analyzer.sql
@@ -1,2 +1,2 @@
-set allow_experimental_analyzer=0;
+set enable_analyzer=0;
 EXPLAIN QUERY TREE run_passes = true, dump_passes = true SELECT 1; -- { serverError NOT_IMPLEMENTED }
diff --git a/tests/queries/0_stateless/02704_storage_merge_explain_graph_crash.sql b/tests/queries/0_stateless/02704_storage_merge_explain_graph_crash.sql
index 44a8fe4f049..db5eddf2a90 100644
--- a/tests/queries/0_stateless/02704_storage_merge_explain_graph_crash.sql
+++ b/tests/queries/0_stateless/02704_storage_merge_explain_graph_crash.sql
@@ -13,4 +13,4 @@ CREATE TABLE foo2_dist (`Id` UInt32, `Val` String) ENGINE = Distributed(test_sha
 CREATE TABLE merge1 AS foo ENGINE = Merge(currentDatabase(), '^(foo|foo2_dist)$');
 
 EXPLAIN PIPELINE graph = 1, compact = 1 SELECT * FROM merge1 FORMAT Null;
-EXPLAIN PIPELINE graph = 1, compact = 1 SELECT * FROM merge1 FORMAT Null SETTINGS allow_experimental_analyzer=1;
+EXPLAIN PIPELINE graph = 1, compact = 1 SELECT * FROM merge1 FORMAT Null SETTINGS enable_analyzer=1;
diff --git a/tests/queries/0_stateless/02707_analyzer_nested_lambdas_types.sql b/tests/queries/0_stateless/02707_analyzer_nested_lambdas_types.sql
index f9258d61900..320e1111e65 100644
--- a/tests/queries/0_stateless/02707_analyzer_nested_lambdas_types.sql
+++ b/tests/queries/0_stateless/02707_analyzer_nested_lambdas_types.sql
@@ -1,24 +1,24 @@
 SELECT
     range(1),
     arrayMap(x -> arrayMap(x -> x, range(x)), [1])
-SETTINGS allow_experimental_analyzer = 0;
+SETTINGS enable_analyzer = 0;
 
 SELECT
     range(1),
     arrayMap(x -> arrayMap(x -> x, range(x)), [1])
-SETTINGS allow_experimental_analyzer = 1;
+SETTINGS enable_analyzer = 1;
 
 SELECT
     range(1),
     arrayMap(x -> arrayMap(x -> 1, range(x)), [1])
-SETTINGS allow_experimental_analyzer = 0;
+SETTINGS enable_analyzer = 0;
 
 SELECT
     range(1),
     arrayMap(x -> arrayMap(x -> 1, range(x)), [1])
-SETTINGS allow_experimental_analyzer = 1;
+SETTINGS enable_analyzer = 1;
 
 SELECT
     range(1),
     arrayMap(x -> arrayMap(y -> 1, range(x)), [1])
-SETTINGS allow_experimental_analyzer = 1;
+SETTINGS enable_analyzer = 1;
diff --git a/tests/queries/0_stateless/02722_matcher_join_use_nulls.sql.j2 b/tests/queries/0_stateless/02722_matcher_join_use_nulls.sql.j2
index 25451a34867..6a8472fecdf 100644
--- a/tests/queries/0_stateless/02722_matcher_join_use_nulls.sql.j2
+++ b/tests/queries/0_stateless/02722_matcher_join_use_nulls.sql.j2
@@ -8,7 +8,7 @@ INSERT INTO t1 VALUES (1);
 INSERT INTO t2 VALUES (2, 2);
 
 SET join_use_nulls = 1;
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 -- { echoOn }
 
diff --git a/tests/queries/0_stateless/02725_cnf_large_check.sql b/tests/queries/0_stateless/02725_cnf_large_check.sql
index 0780e6bcdd3..2567636c02c 100644
--- a/tests/queries/0_stateless/02725_cnf_large_check.sql
+++ b/tests/queries/0_stateless/02725_cnf_large_check.sql
@@ -7,21 +7,21 @@ INSERT INTO 02725_cnf VALUES (0, 0, 0, 0, 0, 0, 0, 0, 0, 0), (0, 0, 0, 0, 0, 0,
 SELECT count()
 FROM 02725_cnf
 WHERE (c5 AND (NOT c0)) OR ((NOT c3) AND (NOT c6) AND (NOT c1) AND (NOT c6)) OR (c7 AND (NOT c3) AND (NOT c5) AND (NOT c7)) OR ((NOT c8) AND c5) OR ((NOT c0)) OR ((NOT c8) AND (NOT c5) AND c1 AND c6 AND c3) OR (c7 AND (NOT c0) AND c6 AND c1 AND (NOT c2)) OR (c3 AND (NOT c9) AND c1)
-SETTINGS convert_query_to_cnf = 1, allow_experimental_analyzer = 1;
+SETTINGS convert_query_to_cnf = 1, enable_analyzer = 1;
 
 SELECT count()
 FROM 02725_cnf
 WHERE (c5 AND (NOT c0)) OR ((NOT c3) AND (NOT c6) AND (NOT c1) AND (NOT c6)) OR (c7 AND (NOT c3) AND (NOT c5) AND (NOT c7)) OR ((NOT c8) AND c5) OR ((NOT c0)) OR ((NOT c8) AND (NOT c5) AND c1 AND c6 AND c3) OR (c7 AND (NOT c0) AND c6 AND c1 AND (NOT c2)) OR (c3 AND (NOT c9) AND c1)
-SETTINGS convert_query_to_cnf = 1, allow_experimental_analyzer = 0;
+SETTINGS convert_query_to_cnf = 1, enable_analyzer = 0;
 
 SELECT count()
 FROM 02725_cnf
 WHERE ((NOT c2) AND c2 AND (NOT c1)) OR ((NOT c2) AND c3 AND (NOT c5)) OR ((NOT c7) AND (NOT c8)) OR (c9 AND c6 AND c8 AND (NOT c8) AND (NOT c7))
-SETTINGS convert_query_to_cnf = 1, allow_experimental_analyzer = 1;
+SETTINGS convert_query_to_cnf = 1, enable_analyzer = 1;
 
 SELECT count()
 FROM 02725_cnf
 WHERE ((NOT c2) AND c2 AND (NOT c1)) OR ((NOT c2) AND c3 AND (NOT c5)) OR ((NOT c7) AND (NOT c8)) OR (c9 AND c6 AND c8 AND (NOT c8) AND (NOT c7))
-SETTINGS convert_query_to_cnf = 1, allow_experimental_analyzer = 0;
+SETTINGS convert_query_to_cnf = 1, enable_analyzer = 0;
 
 DROP TABLE 02725_cnf;
diff --git a/tests/queries/0_stateless/02731_analyzer_join_resolve_nested.sql.j2 b/tests/queries/0_stateless/02731_analyzer_join_resolve_nested.sql.j2
index c2f3c51b17a..4ddf41c4d6d 100644
--- a/tests/queries/0_stateless/02731_analyzer_join_resolve_nested.sql.j2
+++ b/tests/queries/0_stateless/02731_analyzer_join_resolve_nested.sql.j2
@@ -31,7 +31,7 @@ INSERT INTO nnna VALUES (1, [[([([(1,'d')],'d')], 's')]], ['s']);
 CREATE TABLE nnnb ( x UInt64, t Nested(t Nested(t Nested(t Nested(t UInt32, s String), s String), s String), s String) ) ENGINE = MergeTree ORDER BY x;
 INSERT INTO nnnb VALUES (1, [[([([(1,'d')],'d')], 's')]], ['s']);
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 {% for join_use_nulls in [0, 1] -%}
 
diff --git a/tests/queries/0_stateless/02731_parallel_replicas_join_subquery.reference b/tests/queries/0_stateless/02731_parallel_replicas_join_subquery.reference
index 451f0d6d485..125fa524c4a 100644
--- a/tests/queries/0_stateless/02731_parallel_replicas_join_subquery.reference
+++ b/tests/queries/0_stateless/02731_parallel_replicas_join_subquery.reference
@@ -33,8 +33,8 @@
 =============== QUERIES EXECUTED BY PARALLEL INNER QUERY ALONE ===============
 0	3	SELECT `__table1`.`key` AS `key`, `__table1`.`value1` AS `value1`, `__table1`.`value2` AS `value2`, toUInt64(min(`__table1`.`time`)) AS `start_ts` FROM `default`.`join_inner_table` AS `__table1` PREWHERE (`__table1`.`id` = \'833c9e22-c245-4eb5-8745-117a9a1f26b1\') AND (`__table1`.`number` > _CAST(1610517366120, \'UInt64\')) GROUP BY `__table1`.`key`, `__table1`.`value1`, `__table1`.`value2` ORDER BY `__table1`.`key` ASC, `__table1`.`value1` ASC, `__table1`.`value2` ASC LIMIT _CAST(10, \'UInt64\')
 0	3	SELECT `key`, `value1`, `value2`, toUInt64(min(`time`)) AS `start_ts` FROM `default`.`join_inner_table` PREWHERE (`id` = \'833c9e22-c245-4eb5-8745-117a9a1f26b1\') AND (`number` > toUInt64(\'1610517366120\')) GROUP BY `key`, `value1`, `value2` ORDER BY `key` ASC, `value1` ASC, `value2` ASC LIMIT 10
-1	1	-- Parallel inner query alone\nSELECT\n    key,\n    value1,\n    value2,\n    toUInt64(min(time)) AS start_ts\nFROM join_inner_table\nPREWHERE (id = \'833c9e22-c245-4eb5-8745-117a9a1f26b1\') AND (number > toUInt64(\'1610517366120\'))\nGROUP BY key, value1, value2\nORDER BY key, value1, value2\nLIMIT 10\nSETTINGS allow_experimental_parallel_reading_from_replicas = 1, allow_experimental_analyzer=0;
-1	1	-- Parallel inner query alone\nSELECT\n    key,\n    value1,\n    value2,\n    toUInt64(min(time)) AS start_ts\nFROM join_inner_table\nPREWHERE (id = \'833c9e22-c245-4eb5-8745-117a9a1f26b1\') AND (number > toUInt64(\'1610517366120\'))\nGROUP BY key, value1, value2\nORDER BY key, value1, value2\nLIMIT 10\nSETTINGS allow_experimental_parallel_reading_from_replicas = 1, allow_experimental_analyzer=1;
+1	1	-- Parallel inner query alone\nSELECT\n    key,\n    value1,\n    value2,\n    toUInt64(min(time)) AS start_ts\nFROM join_inner_table\nPREWHERE (id = \'833c9e22-c245-4eb5-8745-117a9a1f26b1\') AND (number > toUInt64(\'1610517366120\'))\nGROUP BY key, value1, value2\nORDER BY key, value1, value2\nLIMIT 10\nSETTINGS allow_experimental_parallel_reading_from_replicas = 1, enable_analyzer=0;
+1	1	-- Parallel inner query alone\nSELECT\n    key,\n    value1,\n    value2,\n    toUInt64(min(time)) AS start_ts\nFROM join_inner_table\nPREWHERE (id = \'833c9e22-c245-4eb5-8745-117a9a1f26b1\') AND (number > toUInt64(\'1610517366120\'))\nGROUP BY key, value1, value2\nORDER BY key, value1, value2\nLIMIT 10\nSETTINGS allow_experimental_parallel_reading_from_replicas = 1, enable_analyzer=1;
 =============== OUTER QUERY (NO PARALLEL) ===============
 >T%O	,z<	10
 NQTpY#	W\\Xx4	10
@@ -61,5 +61,5 @@ t<iT	X48q:Z]t0	10
 0	3	SELECT `__table2`.`value1` AS `value1`, `__table2`.`value2` AS `value2`, count() AS `count` FROM `default`.`join_outer_table` AS `__table1` ALL INNER JOIN (SELECT `__table3`.`key` AS `key`, `__table3`.`value1` AS `value1`, `__table3`.`value2` AS `value2` FROM `default`.`join_inner_table` AS `__table3` PREWHERE (`__table3`.`id` = \'833c9e22-c245-4eb5-8745-117a9a1f26b1\') AND (`__table3`.`number` > _CAST(1610517366120, \'UInt64\')) GROUP BY `__table3`.`key`, `__table3`.`value1`, `__table3`.`value2`) AS `__table2` USING (`key`) GROUP BY `__table1`.`key`, `__table2`.`value1`, `__table2`.`value2`
 0	3	SELECT `key`, `value1`, `value2` FROM `default`.`join_inner_table` PREWHERE (`id` = \'833c9e22-c245-4eb5-8745-117a9a1f26b1\') AND (`number` > toUInt64(\'1610517366120\')) GROUP BY `key`, `value1`, `value2`
 0	3	SELECT `value1`, `value2`, count() AS `count` FROM `default`.`join_outer_table` ALL INNER JOIN `_data_` USING (`key`) GROUP BY `key`, `value1`, `value2`
-1	1	-- Parallel full query\nSELECT\n    value1,\n    value2,\n    avg(count) AS avg\nFROM\n    (\n        SELECT\n            key,\n            value1,\n            value2,\n            count() AS count\n        FROM join_outer_table\n        INNER JOIN\n        (\n            SELECT\n                key,\n                value1,\n                value2,\n                toUInt64(min(time)) AS start_ts\n            FROM join_inner_table\n            PREWHERE (id = \'833c9e22-c245-4eb5-8745-117a9a1f26b1\') AND (number > toUInt64(\'1610517366120\'))\n            GROUP BY key, value1, value2\n        ) USING (key)\n        GROUP BY key, value1, value2\n        )\nGROUP BY value1, value2\nORDER BY value1, value2\nSETTINGS allow_experimental_parallel_reading_from_replicas = 1, allow_experimental_analyzer=0;
-1	1	-- Parallel full query\nSELECT\n    value1,\n    value2,\n    avg(count) AS avg\nFROM\n    (\n        SELECT\n            key,\n            value1,\n            value2,\n            count() AS count\n        FROM join_outer_table\n        INNER JOIN\n        (\n            SELECT\n                key,\n                value1,\n                value2,\n                toUInt64(min(time)) AS start_ts\n            FROM join_inner_table\n            PREWHERE (id = \'833c9e22-c245-4eb5-8745-117a9a1f26b1\') AND (number > toUInt64(\'1610517366120\'))\n            GROUP BY key, value1, value2\n        ) USING (key)\n        GROUP BY key, value1, value2\n        )\nGROUP BY value1, value2\nORDER BY value1, value2\nSETTINGS allow_experimental_parallel_reading_from_replicas = 1, allow_experimental_analyzer=1;
+1	1	-- Parallel full query\nSELECT\n    value1,\n    value2,\n    avg(count) AS avg\nFROM\n    (\n        SELECT\n            key,\n            value1,\n            value2,\n            count() AS count\n        FROM join_outer_table\n        INNER JOIN\n        (\n            SELECT\n                key,\n                value1,\n                value2,\n                toUInt64(min(time)) AS start_ts\n            FROM join_inner_table\n            PREWHERE (id = \'833c9e22-c245-4eb5-8745-117a9a1f26b1\') AND (number > toUInt64(\'1610517366120\'))\n            GROUP BY key, value1, value2\n        ) USING (key)\n        GROUP BY key, value1, value2\n        )\nGROUP BY value1, value2\nORDER BY value1, value2\nSETTINGS allow_experimental_parallel_reading_from_replicas = 1, enable_analyzer=0;
+1	1	-- Parallel full query\nSELECT\n    value1,\n    value2,\n    avg(count) AS avg\nFROM\n    (\n        SELECT\n            key,\n            value1,\n            value2,\n            count() AS count\n        FROM join_outer_table\n        INNER JOIN\n        (\n            SELECT\n                key,\n                value1,\n                value2,\n                toUInt64(min(time)) AS start_ts\n            FROM join_inner_table\n            PREWHERE (id = \'833c9e22-c245-4eb5-8745-117a9a1f26b1\') AND (number > toUInt64(\'1610517366120\'))\n            GROUP BY key, value1, value2\n        ) USING (key)\n        GROUP BY key, value1, value2\n        )\nGROUP BY value1, value2\nORDER BY value1, value2\nSETTINGS allow_experimental_parallel_reading_from_replicas = 1, enable_analyzer=1;
diff --git a/tests/queries/0_stateless/02731_parallel_replicas_join_subquery.sql b/tests/queries/0_stateless/02731_parallel_replicas_join_subquery.sql
index 7693d0da295..8121d60a05b 100644
--- a/tests/queries/0_stateless/02731_parallel_replicas_join_subquery.sql
+++ b/tests/queries/0_stateless/02731_parallel_replicas_join_subquery.sql
@@ -38,7 +38,7 @@ FROM join_inner_table
 GROUP BY key, value1, value2
 ORDER BY key, value1, value2
 LIMIT 10;
--- settings allow_experimental_analyzer=0;
+-- settings enable_analyzer=0;
 
 -- SELECT
 --     key,
@@ -49,7 +49,7 @@ LIMIT 10;
 --     PREWHERE (id = '833c9e22-c245-4eb5-8745-117a9a1f26b1') AND (number > toUInt64('1610517366120'))
 -- GROUP BY key, value1, value2
 -- ORDER BY key, value1, value2
--- LIMIT 10 settings allow_experimental_analyzer=1;
+-- LIMIT 10 settings enable_analyzer=1;
 
 SELECT '=============== INNER QUERY (PARALLEL) ===============';
 
@@ -64,7 +64,7 @@ PREWHERE (id = '833c9e22-c245-4eb5-8745-117a9a1f26b1') AND (number > toUInt64('1
 GROUP BY key, value1, value2
 ORDER BY key, value1, value2
 LIMIT 10
-SETTINGS allow_experimental_parallel_reading_from_replicas = 1, allow_experimental_analyzer=0;
+SETTINGS allow_experimental_parallel_reading_from_replicas = 1, enable_analyzer=0;
 
 -- Parallel inner query alone
 SELECT
@@ -77,7 +77,7 @@ PREWHERE (id = '833c9e22-c245-4eb5-8745-117a9a1f26b1') AND (number > toUInt64('1
 GROUP BY key, value1, value2
 ORDER BY key, value1, value2
 LIMIT 10
-SETTINGS allow_experimental_parallel_reading_from_replicas = 1, allow_experimental_analyzer=1;
+SETTINGS allow_experimental_parallel_reading_from_replicas = 1, enable_analyzer=1;
 
 SELECT '=============== QUERIES EXECUTED BY PARALLEL INNER QUERY ALONE ===============';
 
@@ -184,7 +184,7 @@ FROM
         )
 GROUP BY value1, value2
 ORDER BY value1, value2
-SETTINGS allow_experimental_parallel_reading_from_replicas = 1, allow_experimental_analyzer=0;
+SETTINGS allow_experimental_parallel_reading_from_replicas = 1, enable_analyzer=0;
 
 -- Parallel full query
 SELECT
@@ -214,7 +214,7 @@ FROM
         )
 GROUP BY value1, value2
 ORDER BY value1, value2
-SETTINGS allow_experimental_parallel_reading_from_replicas = 1, allow_experimental_analyzer=1;
+SETTINGS allow_experimental_parallel_reading_from_replicas = 1, enable_analyzer=1;
 
 SYSTEM FLUSH LOGS;
 
diff --git a/tests/queries/0_stateless/02734_optimize_group_by.sql b/tests/queries/0_stateless/02734_optimize_group_by.sql
index 28e86c04b0f..626805d028d 100644
--- a/tests/queries/0_stateless/02734_optimize_group_by.sql
+++ b/tests/queries/0_stateless/02734_optimize_group_by.sql
@@ -1,5 +1,5 @@
-SELECT 'a' AS key, 'b' as value GROUP BY key WITH CUBE SETTINGS allow_experimental_analyzer = 0;
-SELECT 'a' AS key, 'b' as value GROUP BY key WITH CUBE SETTINGS allow_experimental_analyzer = 1;
+SELECT 'a' AS key, 'b' as value GROUP BY key WITH CUBE SETTINGS enable_analyzer = 0;
+SELECT 'a' AS key, 'b' as value GROUP BY key WITH CUBE SETTINGS enable_analyzer = 1;
 
 SELECT 'a' AS key, 'b' as value GROUP BY ignore(1) WITH CUBE;
 
diff --git a/tests/queries/0_stateless/02765_queries_with_subqueries_profile_events.reference b/tests/queries/0_stateless/02765_queries_with_subqueries_profile_events.reference
index d21f914f0dc..a6c8c0bbc3e 100644
--- a/tests/queries/0_stateless/02765_queries_with_subqueries_profile_events.reference
+++ b/tests/queries/0_stateless/02765_queries_with_subqueries_profile_events.reference
@@ -1,24 +1,24 @@
-view	allow_experimental_analyzer	InsertQuery	SelectQuery	InsertQueriesWithSubqueries	SelectQueriesWithSubqueries	QueriesWithSubqueries
+view	enable_analyzer	InsertQuery	SelectQuery	InsertQueriesWithSubqueries	SelectQueriesWithSubqueries	QueriesWithSubqueries
 1	0	1	0	1	2	3
-subquery	allow_experimental_analyzer	InsertQuery	SelectQuery	InsertQueriesWithSubqueries	SelectQueriesWithSubqueries	QueriesWithSubqueries
+subquery	enable_analyzer	InsertQuery	SelectQuery	InsertQueriesWithSubqueries	SelectQueriesWithSubqueries	QueriesWithSubqueries
 1	0	0	1	0	2	2
-CSE	allow_experimental_analyzer	InsertQuery	SelectQuery	InsertQueriesWithSubqueries	SelectQueriesWithSubqueries	QueriesWithSubqueries
+CSE	enable_analyzer	InsertQuery	SelectQuery	InsertQueriesWithSubqueries	SelectQueriesWithSubqueries	QueriesWithSubqueries
 1	0	0	1	0	2	2
-CSE_Multi	allow_experimental_analyzer	InsertQuery	SelectQuery	InsertQueriesWithSubqueries	SelectQueriesWithSubqueries	QueriesWithSubqueries
+CSE_Multi	enable_analyzer	InsertQuery	SelectQuery	InsertQueriesWithSubqueries	SelectQueriesWithSubqueries	QueriesWithSubqueries
 1	0	0	1	0	2	2
-CTE	allow_experimental_analyzer	InsertQuery	SelectQuery	InsertQueriesWithSubqueries	SelectQueriesWithSubqueries	QueriesWithSubqueries
+CTE	enable_analyzer	InsertQuery	SelectQuery	InsertQueriesWithSubqueries	SelectQueriesWithSubqueries	QueriesWithSubqueries
 1	0	0	1	0	2	2
-CTE_Multi	allow_experimental_analyzer	InsertQuery	SelectQuery	InsertQueriesWithSubqueries	SelectQueriesWithSubqueries	QueriesWithSubqueries
+CTE_Multi	enable_analyzer	InsertQuery	SelectQuery	InsertQueriesWithSubqueries	SelectQueriesWithSubqueries	QueriesWithSubqueries
 1	0	0	1	0	4	4
-view	allow_experimental_analyzer	InsertQuery	SelectQuery	InsertQueriesWithSubqueries	SelectQueriesWithSubqueries	QueriesWithSubqueries
+view	enable_analyzer	InsertQuery	SelectQuery	InsertQueriesWithSubqueries	SelectQueriesWithSubqueries	QueriesWithSubqueries
 1	1	1	0	1	3	4
-subquery	allow_experimental_analyzer	InsertQuery	SelectQuery	InsertQueriesWithSubqueries	SelectQueriesWithSubqueries	QueriesWithSubqueries
+subquery	enable_analyzer	InsertQuery	SelectQuery	InsertQueriesWithSubqueries	SelectQueriesWithSubqueries	QueriesWithSubqueries
 1	1	0	1	0	2	2
-CSE	allow_experimental_analyzer	InsertQuery	SelectQuery	InsertQueriesWithSubqueries	SelectQueriesWithSubqueries	QueriesWithSubqueries
+CSE	enable_analyzer	InsertQuery	SelectQuery	InsertQueriesWithSubqueries	SelectQueriesWithSubqueries	QueriesWithSubqueries
 1	1	0	1	0	2	2
-CSE_Multi	allow_experimental_analyzer	InsertQuery	SelectQuery	InsertQueriesWithSubqueries	SelectQueriesWithSubqueries	QueriesWithSubqueries
+CSE_Multi	enable_analyzer	InsertQuery	SelectQuery	InsertQueriesWithSubqueries	SelectQueriesWithSubqueries	QueriesWithSubqueries
 1	1	0	1	0	2	2
-CTE	allow_experimental_analyzer	InsertQuery	SelectQuery	InsertQueriesWithSubqueries	SelectQueriesWithSubqueries	QueriesWithSubqueries
+CTE	enable_analyzer	InsertQuery	SelectQuery	InsertQueriesWithSubqueries	SelectQueriesWithSubqueries	QueriesWithSubqueries
 1	1	0	1	0	2	2
-CTE_Multi	allow_experimental_analyzer	InsertQuery	SelectQuery	InsertQueriesWithSubqueries	SelectQueriesWithSubqueries	QueriesWithSubqueries
+CTE_Multi	enable_analyzer	InsertQuery	SelectQuery	InsertQueriesWithSubqueries	SelectQueriesWithSubqueries	QueriesWithSubqueries
 1	1	0	1	0	4	4
diff --git a/tests/queries/0_stateless/02765_queries_with_subqueries_profile_events.sh b/tests/queries/0_stateless/02765_queries_with_subqueries_profile_events.sh
index 84031ad9081..b7d93b5396c 100755
--- a/tests/queries/0_stateless/02765_queries_with_subqueries_profile_events.sh
+++ b/tests/queries/0_stateless/02765_queries_with_subqueries_profile_events.sh
@@ -14,14 +14,14 @@ $CLICKHOUSE_CLIENT -n -q "
     CREATE MATERIALIZED VIEW mv TO output SQL SECURITY NONE AS SELECT * FROM input;
 "
 
-for allow_experimental_analyzer in 0 1; do
+for enable_analyzer in 0 1; do
     query_id="$(random_str 10)"
-    $CLICKHOUSE_CLIENT --allow_experimental_analyzer "$allow_experimental_analyzer" --query_id "$query_id" -q "INSERT INTO input SELECT * FROM numbers(1)"
+    $CLICKHOUSE_CLIENT --enable_analyzer "$enable_analyzer" --query_id "$query_id" -q "INSERT INTO input SELECT * FROM numbers(1)"
     $CLICKHOUSE_CLIENT -mn -q "
         SYSTEM FLUSH LOGS;
         SELECT
             1 view,
-            $allow_experimental_analyzer allow_experimental_analyzer,
+            $enable_analyzer enable_analyzer,
             ProfileEvents['InsertQuery'] InsertQuery,
             ProfileEvents['SelectQuery'] SelectQuery,
             ProfileEvents['InsertQueriesWithSubqueries'] InsertQueriesWithSubqueries,
@@ -34,12 +34,12 @@ for allow_experimental_analyzer in 0 1; do
     "
 
     query_id="$(random_str 10)"
-    $CLICKHOUSE_CLIENT --allow_experimental_analyzer "$allow_experimental_analyzer" --query_id "$query_id" -q "SELECT * FROM system.one WHERE dummy IN (SELECT * FROM system.one) FORMAT Null"
+    $CLICKHOUSE_CLIENT --enable_analyzer "$enable_analyzer" --query_id "$query_id" -q "SELECT * FROM system.one WHERE dummy IN (SELECT * FROM system.one) FORMAT Null"
     $CLICKHOUSE_CLIENT -mn -q "
         SYSTEM FLUSH LOGS;
         SELECT
             1 subquery,
-            $allow_experimental_analyzer allow_experimental_analyzer,
+            $enable_analyzer enable_analyzer,
             ProfileEvents['InsertQuery'] InsertQuery,
             ProfileEvents['SelectQuery'] SelectQuery,
             ProfileEvents['InsertQueriesWithSubqueries'] InsertQueriesWithSubqueries,
@@ -51,12 +51,12 @@ for allow_experimental_analyzer in 0 1; do
     "
 
     query_id="$(random_str 10)"
-    $CLICKHOUSE_CLIENT --allow_experimental_analyzer "$allow_experimental_analyzer" --query_id "$query_id" -q "WITH (SELECT * FROM system.one) AS x SELECT x FORMAT Null"
+    $CLICKHOUSE_CLIENT --enable_analyzer "$enable_analyzer" --query_id "$query_id" -q "WITH (SELECT * FROM system.one) AS x SELECT x FORMAT Null"
     $CLICKHOUSE_CLIENT -mn -q "
         SYSTEM FLUSH LOGS;
         SELECT
             1 CSE,
-            $allow_experimental_analyzer allow_experimental_analyzer,
+            $enable_analyzer enable_analyzer,
             ProfileEvents['InsertQuery'] InsertQuery,
             ProfileEvents['SelectQuery'] SelectQuery,
             ProfileEvents['InsertQueriesWithSubqueries'] InsertQueriesWithSubqueries,
@@ -68,12 +68,12 @@ for allow_experimental_analyzer in 0 1; do
     "
 
     query_id="$(random_str 10)"
-    $CLICKHOUSE_CLIENT --allow_experimental_analyzer "$allow_experimental_analyzer" --query_id "$query_id" -q "WITH (SELECT * FROM system.one) AS x SELECT x, x FORMAT Null"
+    $CLICKHOUSE_CLIENT --enable_analyzer "$enable_analyzer" --query_id "$query_id" -q "WITH (SELECT * FROM system.one) AS x SELECT x, x FORMAT Null"
     $CLICKHOUSE_CLIENT -mn -q "
         SYSTEM FLUSH LOGS;
         SELECT
             1 CSE_Multi,
-            $allow_experimental_analyzer allow_experimental_analyzer,
+            $enable_analyzer enable_analyzer,
             ProfileEvents['InsertQuery'] InsertQuery,
             ProfileEvents['SelectQuery'] SelectQuery,
             ProfileEvents['InsertQueriesWithSubqueries'] InsertQueriesWithSubqueries,
@@ -85,12 +85,12 @@ for allow_experimental_analyzer in 0 1; do
     "
 
     query_id="$(random_str 10)"
-    $CLICKHOUSE_CLIENT --allow_experimental_analyzer "$allow_experimental_analyzer" --query_id "$query_id" -q "WITH x AS (SELECT * FROM system.one) SELECT * FROM x FORMAT Null"
+    $CLICKHOUSE_CLIENT --enable_analyzer "$enable_analyzer" --query_id "$query_id" -q "WITH x AS (SELECT * FROM system.one) SELECT * FROM x FORMAT Null"
     $CLICKHOUSE_CLIENT -mn -q "
         SYSTEM FLUSH LOGS;
         SELECT
             1 CTE,
-            $allow_experimental_analyzer allow_experimental_analyzer,
+            $enable_analyzer enable_analyzer,
             ProfileEvents['InsertQuery'] InsertQuery,
             ProfileEvents['SelectQuery'] SelectQuery,
             ProfileEvents['InsertQueriesWithSubqueries'] InsertQueriesWithSubqueries,
@@ -102,12 +102,12 @@ for allow_experimental_analyzer in 0 1; do
     "
 
     query_id="$(random_str 10)"
-    $CLICKHOUSE_CLIENT --allow_experimental_analyzer "$allow_experimental_analyzer" --query_id "$query_id" -q "WITH x AS (SELECT * FROM system.one) SELECT * FROM x UNION ALL SELECT * FROM x FORMAT Null"
+    $CLICKHOUSE_CLIENT --enable_analyzer "$enable_analyzer" --query_id "$query_id" -q "WITH x AS (SELECT * FROM system.one) SELECT * FROM x UNION ALL SELECT * FROM x FORMAT Null"
     $CLICKHOUSE_CLIENT -mn -q "
         SYSTEM FLUSH LOGS;
         SELECT
             1 CTE_Multi,
-            $allow_experimental_analyzer allow_experimental_analyzer,
+            $enable_analyzer enable_analyzer,
             ProfileEvents['InsertQuery'] InsertQuery,
             ProfileEvents['SelectQuery'] SelectQuery,
             ProfileEvents['InsertQueriesWithSubqueries'] InsertQueriesWithSubqueries,
diff --git a/tests/queries/0_stateless/02767_into_outfile_extensions_msan.sh b/tests/queries/0_stateless/02767_into_outfile_extensions_msan.sh
index 0c5767314d5..2d2ee328a29 100755
--- a/tests/queries/0_stateless/02767_into_outfile_extensions_msan.sh
+++ b/tests/queries/0_stateless/02767_into_outfile_extensions_msan.sh
@@ -6,6 +6,6 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 
 out="explain1.$CLICKHOUSE_TEST_UNIQUE_NAME.out"
 # only EXPLAIN triggers the problem under MSan
-$CLICKHOUSE_CLIENT --allow_experimental_analyzer=0 -q "explain select * from numbers(1) into outfile '$out'"
+$CLICKHOUSE_CLIENT --enable_analyzer=0 -q "explain select * from numbers(1) into outfile '$out'"
 cat "$out"
 rm -f "$out"
diff --git a/tests/queries/0_stateless/02771_ignore_data_skipping_indices.sql b/tests/queries/0_stateless/02771_ignore_data_skipping_indices.sql
index d86b65c3291..91ca5ef0340 100644
--- a/tests/queries/0_stateless/02771_ignore_data_skipping_indices.sql
+++ b/tests/queries/0_stateless/02771_ignore_data_skipping_indices.sql
@@ -23,12 +23,12 @@ SELECT * FROM data_02771 SETTINGS ignore_data_skipping_indices='na_idx';
 SELECT * FROM data_02771 WHERE x = 1 AND y = 1 SETTINGS ignore_data_skipping_indices='xy_idx',force_data_skipping_indices='xy_idx' ; -- { serverError INDEX_NOT_USED }
 SELECT * FROM data_02771 WHERE x = 1 AND y = 2 SETTINGS ignore_data_skipping_indices='xy_idx';
 
-SET allow_experimental_analyzer = 0;
+SET enable_analyzer = 0;
 
 SELECT * from ( EXPLAIN indexes = 1 SELECT * FROM data_02771 WHERE x = 1 AND y = 2 ) WHERE explain NOT LIKE '%Expression%' AND explain NOT LIKE '%Filter%';
 SELECT * from ( EXPLAIN indexes = 1 SELECT * FROM data_02771 WHERE x = 1 AND y = 2 SETTINGS ignore_data_skipping_indices='xy_idx' ) WHERE explain NOT LIKE '%Expression%' AND explain NOT LIKE '%Filter%';
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 SELECT * from ( EXPLAIN indexes = 1 SELECT * FROM data_02771 WHERE x = 1 AND y = 2 ) WHERE explain NOT LIKE '%Expression%' AND explain NOT LIKE '%Filter%';
 SELECT * from ( EXPLAIN indexes = 1 SELECT * FROM data_02771 WHERE x = 1 AND y = 2 SETTINGS ignore_data_skipping_indices='xy_idx' ) WHERE explain NOT LIKE '%Expression%' AND explain NOT LIKE '%Filter%';
diff --git a/tests/queries/0_stateless/02771_parallel_replicas_analyzer.reference b/tests/queries/0_stateless/02771_parallel_replicas_analyzer.reference
index 5bf3520ccdb..f60f1e0a376 100644
--- a/tests/queries/0_stateless/02771_parallel_replicas_analyzer.reference
+++ b/tests/queries/0_stateless/02771_parallel_replicas_analyzer.reference
@@ -8,5 +8,5 @@
 5935810273536892891
 7885388429666205427
 8124171311239967992
-1	1	-- Simple query with analyzer and pure parallel replicas\nSELECT number\nFROM join_inner_table__fuzz_146_replicated\n    SETTINGS\n    allow_experimental_analyzer = 1,\n    max_parallel_replicas = 2,\n    cluster_for_parallel_replicas = \'test_cluster_one_shard_three_replicas_localhost\',\n    allow_experimental_parallel_reading_from_replicas = 1;
+1	1	-- Simple query with analyzer and pure parallel replicas\nSELECT number\nFROM join_inner_table__fuzz_146_replicated\n    SETTINGS\n    enable_analyzer = 1,\n    max_parallel_replicas = 2,\n    cluster_for_parallel_replicas = \'test_cluster_one_shard_three_replicas_localhost\',\n    allow_experimental_parallel_reading_from_replicas = 1;
 0	2	SELECT `__table1`.`number` AS `number` FROM `default`.`join_inner_table__fuzz_146_replicated` AS `__table1`
diff --git a/tests/queries/0_stateless/02771_parallel_replicas_analyzer.sql b/tests/queries/0_stateless/02771_parallel_replicas_analyzer.sql
index 88a0d2163d6..e60049f2756 100644
--- a/tests/queries/0_stateless/02771_parallel_replicas_analyzer.sql
+++ b/tests/queries/0_stateless/02771_parallel_replicas_analyzer.sql
@@ -21,7 +21,7 @@ INSERT INTO join_inner_table__fuzz_146_replicated
 SELECT number
 FROM join_inner_table__fuzz_146_replicated
     SETTINGS
-    allow_experimental_analyzer = 1,
+    enable_analyzer = 1,
     max_parallel_replicas = 2,
     cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost',
     allow_experimental_parallel_reading_from_replicas = 1;
diff --git a/tests/queries/0_stateless/02771_semi_join_use_nulls.sql.j2 b/tests/queries/0_stateless/02771_semi_join_use_nulls.sql.j2
index 248461a98bb..74e252c785b 100644
--- a/tests/queries/0_stateless/02771_semi_join_use_nulls.sql.j2
+++ b/tests/queries/0_stateless/02771_semi_join_use_nulls.sql.j2
@@ -1,11 +1,11 @@
 
-{% for allow_experimental_analyzer in [0, 1] -%}
+{% for enable_analyzer in [0, 1] -%}
 {% for join_use_nulls in [0, 1] -%}
 {% for kind in ['LEFT', 'RIGHT'] -%}
 {% for strictness in ['SEMI', 'ANTI'] -%}
 {% for maybe_materialize in ['', 'materialize'] -%}
 
-SET allow_experimental_analyzer = {{ allow_experimental_analyzer }};
+SET enable_analyzer = {{ enable_analyzer }};
 
 SET join_use_nulls = {{ join_use_nulls }};
 
diff --git a/tests/queries/0_stateless/02783_date_predicate_optimizations.sql b/tests/queries/0_stateless/02783_date_predicate_optimizations.sql
index 4da8cebff1c..b127af677ee 100644
--- a/tests/queries/0_stateless/02783_date_predicate_optimizations.sql
+++ b/tests/queries/0_stateless/02783_date_predicate_optimizations.sql
@@ -11,8 +11,8 @@ INSERT INTO source values ('2021-12-31 23:00:00', 0);
 
 SELECT * FROM source WHERE toYYYYMM(ts) = 202112;
 SELECT * FROM source WHERE toYear(ts) = 2021;
-SELECT * FROM source WHERE toYYYYMM(ts) = 202112 SETTINGS allow_experimental_analyzer=1;
-SELECT * FROM source WHERE toYear(ts) = 2021 SETTINGS allow_experimental_analyzer=1;
+SELECT * FROM source WHERE toYYYYMM(ts) = 202112 SETTINGS enable_analyzer=1;
+SELECT * FROM source WHERE toYear(ts) = 2021 SETTINGS enable_analyzer=1;
 
 DROP TABLE IF EXISTS source;
 CREATE TABLE source
@@ -46,18 +46,18 @@ SELECT count(*) FROM source WHERE toYear(dt) < 2023;
 SELECT count(*) FROM source WHERE toYear(dt) <= 2023;
 SELECT count(*) FROM source WHERE toYear(dt) > 2023;
 SELECT count(*) FROM source WHERE toYear(dt) >= 2023;
-SELECT count(*) FROM source WHERE toYYYYMM(dt) = 202312 SETTINGS allow_experimental_analyzer=1;
-SELECT count(*) FROM source WHERE toYYYYMM(dt) <> 202312 SETTINGS allow_experimental_analyzer=1;
-SELECT count(*) FROM source WHERE toYYYYMM(dt) < 202312 SETTINGS allow_experimental_analyzer=1;
-SELECT count(*) FROM source WHERE toYYYYMM(dt) <= 202312 SETTINGS allow_experimental_analyzer=1;
-SELECT count(*) FROM source WHERE toYYYYMM(dt) > 202312 SETTINGS allow_experimental_analyzer=1;
-SELECT count(*) FROM source WHERE toYYYYMM(dt) >= 202312 SETTINGS allow_experimental_analyzer=1;
-SELECT count(*) FROM source WHERE toYear(dt) = 2023 SETTINGS allow_experimental_analyzer=1;
-SELECT count(*) FROM source WHERE toYear(dt) <> 2023 SETTINGS allow_experimental_analyzer=1;
-SELECT count(*) FROM source WHERE toYear(dt) < 2023 SETTINGS allow_experimental_analyzer=1;
-SELECT count(*) FROM source WHERE toYear(dt) <= 2023 SETTINGS allow_experimental_analyzer=1;
-SELECT count(*) FROM source WHERE toYear(dt) > 2023 SETTINGS allow_experimental_analyzer=1;
-SELECT count(*) FROM source WHERE toYear(dt) >= 2023 SETTINGS allow_experimental_analyzer=1;
+SELECT count(*) FROM source WHERE toYYYYMM(dt) = 202312 SETTINGS enable_analyzer=1;
+SELECT count(*) FROM source WHERE toYYYYMM(dt) <> 202312 SETTINGS enable_analyzer=1;
+SELECT count(*) FROM source WHERE toYYYYMM(dt) < 202312 SETTINGS enable_analyzer=1;
+SELECT count(*) FROM source WHERE toYYYYMM(dt) <= 202312 SETTINGS enable_analyzer=1;
+SELECT count(*) FROM source WHERE toYYYYMM(dt) > 202312 SETTINGS enable_analyzer=1;
+SELECT count(*) FROM source WHERE toYYYYMM(dt) >= 202312 SETTINGS enable_analyzer=1;
+SELECT count(*) FROM source WHERE toYear(dt) = 2023 SETTINGS enable_analyzer=1;
+SELECT count(*) FROM source WHERE toYear(dt) <> 2023 SETTINGS enable_analyzer=1;
+SELECT count(*) FROM source WHERE toYear(dt) < 2023 SETTINGS enable_analyzer=1;
+SELECT count(*) FROM source WHERE toYear(dt) <= 2023 SETTINGS enable_analyzer=1;
+SELECT count(*) FROM source WHERE toYear(dt) > 2023 SETTINGS enable_analyzer=1;
+SELECT count(*) FROM source WHERE toYear(dt) >= 2023 SETTINGS enable_analyzer=1;
 
 SELECT 'DateTime';
 SELECT count(*) FROM source WHERE toYYYYMM(ts) = 202312;
@@ -72,18 +72,18 @@ SELECT count(*) FROM source WHERE toYear(ts) < 2023;
 SELECT count(*) FROM source WHERE toYear(ts) <= 2023;
 SELECT count(*) FROM source WHERE toYear(ts) > 2023;
 SELECT count(*) FROM source WHERE toYear(ts) >= 2023;
-SELECT count(*) FROM source WHERE toYYYYMM(ts) = 202312 SETTINGS allow_experimental_analyzer=1;
-SELECT count(*) FROM source WHERE toYYYYMM(ts) <> 202312 SETTINGS allow_experimental_analyzer=1;
-SELECT count(*) FROM source WHERE toYYYYMM(ts) < 202312 SETTINGS allow_experimental_analyzer=1;
-SELECT count(*) FROM source WHERE toYYYYMM(ts) <= 202312 SETTINGS allow_experimental_analyzer=1;
-SELECT count(*) FROM source WHERE toYYYYMM(ts) > 202312 SETTINGS allow_experimental_analyzer=1;
-SELECT count(*) FROM source WHERE toYYYYMM(ts) >= 202312 SETTINGS allow_experimental_analyzer=1;
-SELECT count(*) FROM source WHERE toYear(ts) = 2023 SETTINGS allow_experimental_analyzer=1;
-SELECT count(*) FROM source WHERE toYear(ts) <> 2023 SETTINGS allow_experimental_analyzer=1;
-SELECT count(*) FROM source WHERE toYear(ts) < 2023 SETTINGS allow_experimental_analyzer=1;
-SELECT count(*) FROM source WHERE toYear(ts) <= 2023 SETTINGS allow_experimental_analyzer=1;
-SELECT count(*) FROM source WHERE toYear(ts) > 2023 SETTINGS allow_experimental_analyzer=1;
-SELECT count(*) FROM source WHERE toYear(ts) >= 2023 SETTINGS allow_experimental_analyzer=1;
+SELECT count(*) FROM source WHERE toYYYYMM(ts) = 202312 SETTINGS enable_analyzer=1;
+SELECT count(*) FROM source WHERE toYYYYMM(ts) <> 202312 SETTINGS enable_analyzer=1;
+SELECT count(*) FROM source WHERE toYYYYMM(ts) < 202312 SETTINGS enable_analyzer=1;
+SELECT count(*) FROM source WHERE toYYYYMM(ts) <= 202312 SETTINGS enable_analyzer=1;
+SELECT count(*) FROM source WHERE toYYYYMM(ts) > 202312 SETTINGS enable_analyzer=1;
+SELECT count(*) FROM source WHERE toYYYYMM(ts) >= 202312 SETTINGS enable_analyzer=1;
+SELECT count(*) FROM source WHERE toYear(ts) = 2023 SETTINGS enable_analyzer=1;
+SELECT count(*) FROM source WHERE toYear(ts) <> 2023 SETTINGS enable_analyzer=1;
+SELECT count(*) FROM source WHERE toYear(ts) < 2023 SETTINGS enable_analyzer=1;
+SELECT count(*) FROM source WHERE toYear(ts) <= 2023 SETTINGS enable_analyzer=1;
+SELECT count(*) FROM source WHERE toYear(ts) > 2023 SETTINGS enable_analyzer=1;
+SELECT count(*) FROM source WHERE toYear(ts) >= 2023 SETTINGS enable_analyzer=1;
 
 SELECT 'Date32';
 SELECT count(*) FROM source WHERE toYYYYMM(dt_32) = 202312;
@@ -98,18 +98,18 @@ SELECT count(*) FROM source WHERE toYear(dt_32) < 2023;
 SELECT count(*) FROM source WHERE toYear(dt_32) <= 2023;
 SELECT count(*) FROM source WHERE toYear(dt_32) > 2023;
 SELECT count(*) FROM source WHERE toYear(dt_32) >= 2023;
-SELECT count(*) FROM source WHERE toYYYYMM(dt_32) = 202312 SETTINGS allow_experimental_analyzer=1;
-SELECT count(*) FROM source WHERE toYYYYMM(dt_32) <> 202312 SETTINGS allow_experimental_analyzer=1;
-SELECT count(*) FROM source WHERE toYYYYMM(dt_32) < 202312 SETTINGS allow_experimental_analyzer=1;
-SELECT count(*) FROM source WHERE toYYYYMM(dt_32) <= 202312 SETTINGS allow_experimental_analyzer=1;
-SELECT count(*) FROM source WHERE toYYYYMM(dt_32) > 202312 SETTINGS allow_experimental_analyzer=1;
-SELECT count(*) FROM source WHERE toYYYYMM(dt_32) >= 202312 SETTINGS allow_experimental_analyzer=1;
-SELECT count(*) FROM source WHERE toYear(dt_32) = 2023 SETTINGS allow_experimental_analyzer=1;
-SELECT count(*) FROM source WHERE toYear(dt_32) <> 2023 SETTINGS allow_experimental_analyzer=1;
-SELECT count(*) FROM source WHERE toYear(dt_32) < 2023 SETTINGS allow_experimental_analyzer=1;
-SELECT count(*) FROM source WHERE toYear(dt_32) <= 2023 SETTINGS allow_experimental_analyzer=1;
-SELECT count(*) FROM source WHERE toYear(dt_32) > 2023 SETTINGS allow_experimental_analyzer=1;
-SELECT count(*) FROM source WHERE toYear(dt_32) >= 2023 SETTINGS allow_experimental_analyzer=1;
+SELECT count(*) FROM source WHERE toYYYYMM(dt_32) = 202312 SETTINGS enable_analyzer=1;
+SELECT count(*) FROM source WHERE toYYYYMM(dt_32) <> 202312 SETTINGS enable_analyzer=1;
+SELECT count(*) FROM source WHERE toYYYYMM(dt_32) < 202312 SETTINGS enable_analyzer=1;
+SELECT count(*) FROM source WHERE toYYYYMM(dt_32) <= 202312 SETTINGS enable_analyzer=1;
+SELECT count(*) FROM source WHERE toYYYYMM(dt_32) > 202312 SETTINGS enable_analyzer=1;
+SELECT count(*) FROM source WHERE toYYYYMM(dt_32) >= 202312 SETTINGS enable_analyzer=1;
+SELECT count(*) FROM source WHERE toYear(dt_32) = 2023 SETTINGS enable_analyzer=1;
+SELECT count(*) FROM source WHERE toYear(dt_32) <> 2023 SETTINGS enable_analyzer=1;
+SELECT count(*) FROM source WHERE toYear(dt_32) < 2023 SETTINGS enable_analyzer=1;
+SELECT count(*) FROM source WHERE toYear(dt_32) <= 2023 SETTINGS enable_analyzer=1;
+SELECT count(*) FROM source WHERE toYear(dt_32) > 2023 SETTINGS enable_analyzer=1;
+SELECT count(*) FROM source WHERE toYear(dt_32) >= 2023 SETTINGS enable_analyzer=1;
 
 SELECT 'DateTime64';
 SELECT count(*) FROM source WHERE toYYYYMM(ts_64) = 202312;
@@ -124,16 +124,16 @@ SELECT count(*) FROM source WHERE toYear(ts_64) < 2023;
 SELECT count(*) FROM source WHERE toYear(ts_64) <= 2023;
 SELECT count(*) FROM source WHERE toYear(ts_64) > 2023;
 SELECT count(*) FROM source WHERE toYear(ts_64) >= 2023;
-SELECT count(*) FROM source WHERE toYYYYMM(ts_64) = 202312 SETTINGS allow_experimental_analyzer=1;
-SELECT count(*) FROM source WHERE toYYYYMM(ts_64) <> 202312 SETTINGS allow_experimental_analyzer=1;
-SELECT count(*) FROM source WHERE toYYYYMM(ts_64) < 202312 SETTINGS allow_experimental_analyzer=1;
-SELECT count(*) FROM source WHERE toYYYYMM(ts_64) <= 202312 SETTINGS allow_experimental_analyzer=1;
-SELECT count(*) FROM source WHERE toYYYYMM(ts_64) > 202312 SETTINGS allow_experimental_analyzer=1;
-SELECT count(*) FROM source WHERE toYYYYMM(ts_64) >= 202312 SETTINGS allow_experimental_analyzer=1;
-SELECT count(*) FROM source WHERE toYear(ts_64) = 2023 SETTINGS allow_experimental_analyzer=1;
-SELECT count(*) FROM source WHERE toYear(ts_64) <> 2023 SETTINGS allow_experimental_analyzer=1;
-SELECT count(*) FROM source WHERE toYear(ts_64) < 2023 SETTINGS allow_experimental_analyzer=1;
-SELECT count(*) FROM source WHERE toYear(ts_64) <= 2023 SETTINGS allow_experimental_analyzer=1;
-SELECT count(*) FROM source WHERE toYear(ts_64) > 2023 SETTINGS allow_experimental_analyzer=1;
-SELECT count(*) FROM source WHERE toYear(ts_64) >= 2023 SETTINGS allow_experimental_analyzer=1;
+SELECT count(*) FROM source WHERE toYYYYMM(ts_64) = 202312 SETTINGS enable_analyzer=1;
+SELECT count(*) FROM source WHERE toYYYYMM(ts_64) <> 202312 SETTINGS enable_analyzer=1;
+SELECT count(*) FROM source WHERE toYYYYMM(ts_64) < 202312 SETTINGS enable_analyzer=1;
+SELECT count(*) FROM source WHERE toYYYYMM(ts_64) <= 202312 SETTINGS enable_analyzer=1;
+SELECT count(*) FROM source WHERE toYYYYMM(ts_64) > 202312 SETTINGS enable_analyzer=1;
+SELECT count(*) FROM source WHERE toYYYYMM(ts_64) >= 202312 SETTINGS enable_analyzer=1;
+SELECT count(*) FROM source WHERE toYear(ts_64) = 2023 SETTINGS enable_analyzer=1;
+SELECT count(*) FROM source WHERE toYear(ts_64) <> 2023 SETTINGS enable_analyzer=1;
+SELECT count(*) FROM source WHERE toYear(ts_64) < 2023 SETTINGS enable_analyzer=1;
+SELECT count(*) FROM source WHERE toYear(ts_64) <= 2023 SETTINGS enable_analyzer=1;
+SELECT count(*) FROM source WHERE toYear(ts_64) > 2023 SETTINGS enable_analyzer=1;
+SELECT count(*) FROM source WHERE toYear(ts_64) >= 2023 SETTINGS enable_analyzer=1;
 DROP TABLE source;
diff --git a/tests/queries/0_stateless/02783_parallel_replicas_trivial_count_optimization.sh b/tests/queries/0_stateless/02783_parallel_replicas_trivial_count_optimization.sh
index 20b3efedd49..bf7170fd7fb 100755
--- a/tests/queries/0_stateless/02783_parallel_replicas_trivial_count_optimization.sh
+++ b/tests/queries/0_stateless/02783_parallel_replicas_trivial_count_optimization.sh
@@ -34,7 +34,7 @@ function run_query_with_pure_parallel_replicas () {
         --prefer_localhost_replica 1 \
         --cluster_for_parallel_replicas 'test_cluster_one_shard_three_replicas_localhost' \
         --allow_experimental_parallel_reading_from_replicas 1 \
-        --allow_experimental_analyzer 0
+        --enable_analyzer 0
 
     $CLICKHOUSE_CLIENT \
         --query "$2" \
@@ -43,7 +43,7 @@ function run_query_with_pure_parallel_replicas () {
         --prefer_localhost_replica 1 \
         --cluster_for_parallel_replicas 'test_cluster_one_shard_three_replicas_localhost' \
         --allow_experimental_parallel_reading_from_replicas 1 \
-        --allow_experimental_analyzer 1
+        --enable_analyzer 1
 }
 
 function run_query_with_custom_key_parallel_replicas () {
@@ -58,7 +58,7 @@ function run_query_with_custom_key_parallel_replicas () {
         --max_parallel_replicas 3 \
         --parallel_replicas_custom_key_filter_type 'default' \
         --parallel_replicas_custom_key "$2" \
-        --allow_experimental_analyzer 0
+        --enable_analyzer 0
 
     $CLICKHOUSE_CLIENT \
         --query "$2" \
@@ -66,7 +66,7 @@ function run_query_with_custom_key_parallel_replicas () {
         --max_parallel_replicas 3 \
         --parallel_replicas_custom_key_filter_type 'default' \
         --parallel_replicas_custom_key "$2" \
-        --allow_experimental_analyzer 1
+        --enable_analyzer 1
 }
 
 $CLICKHOUSE_CLIENT --query "
diff --git a/tests/queries/0_stateless/02784_move_all_conditions_to_prewhere_analyzer_asan.sql b/tests/queries/0_stateless/02784_move_all_conditions_to_prewhere_analyzer_asan.sql
index 44b9ce4fdc1..3766e5b0c8f 100644
--- a/tests/queries/0_stateless/02784_move_all_conditions_to_prewhere_analyzer_asan.sql
+++ b/tests/queries/0_stateless/02784_move_all_conditions_to_prewhere_analyzer_asan.sql
@@ -4,7 +4,7 @@ CREATE TABLE t_02784 (c1 UInt64, c2 UInt64) ENGINE=MergeTree() ORDER BY c1 SETTI
 
 INSERT INTO t_02784 SELECT number, number FROM numbers(1);
 
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 SET move_all_conditions_to_prewhere=1;
 
 SELECT c1, c2 FROM t_02784 WHERE c1 = 0 AND c2 = 0;
diff --git a/tests/queries/0_stateless/02785_date_predicate_optimizations_ast_query_tree_rewrite.reference b/tests/queries/0_stateless/02785_date_predicate_optimizations_ast_query_tree_rewrite.reference
index fca48238778..c2c77a4aa31 100644
--- a/tests/queries/0_stateless/02785_date_predicate_optimizations_ast_query_tree_rewrite.reference
+++ b/tests/queries/0_stateless/02785_date_predicate_optimizations_ast_query_tree_rewrite.reference
@@ -39,7 +39,7 @@ QUERY id: 0
                     LIST id: 22, nodes: 2
                       COLUMN id: 19, column_name: id, result_type: UInt32, source_id: 3
                       CONSTANT id: 23, constant_value: UInt64_3, constant_value_type: UInt8
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 SELECT value1
 FROM date_t
 WHERE ((date1 < \'1993-01-01\') OR (date1 >= \'1994-01-01\')) AND ((id >= 1) AND (id <= 3))
@@ -81,7 +81,7 @@ QUERY id: 0
                     LIST id: 22, nodes: 2
                       COLUMN id: 19, column_name: id, result_type: UInt32, source_id: 3
                       CONSTANT id: 23, constant_value: UInt64_3, constant_value_type: UInt8
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 SELECT value1
 FROM date_t
 WHERE (date1 < \'1993-01-01\') AND ((id >= 1) AND (id <= 3))
@@ -115,7 +115,7 @@ QUERY id: 0
                     LIST id: 17, nodes: 2
                       COLUMN id: 14, column_name: id, result_type: UInt32, source_id: 3
                       CONSTANT id: 18, constant_value: UInt64_3, constant_value_type: UInt8
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 SELECT value1
 FROM date_t
 WHERE (date1 >= \'1994-01-01\') AND ((id >= 1) AND (id <= 3))
@@ -149,7 +149,7 @@ QUERY id: 0
                     LIST id: 17, nodes: 2
                       COLUMN id: 14, column_name: id, result_type: UInt32, source_id: 3
                       CONSTANT id: 18, constant_value: UInt64_3, constant_value_type: UInt8
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 SELECT value1
 FROM date_t
 WHERE (date1 < \'1994-01-01\') AND ((id >= 1) AND (id <= 3))
@@ -183,7 +183,7 @@ QUERY id: 0
                     LIST id: 17, nodes: 2
                       COLUMN id: 14, column_name: id, result_type: UInt32, source_id: 3
                       CONSTANT id: 18, constant_value: UInt64_3, constant_value_type: UInt8
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 SELECT value1
 FROM date_t
 WHERE (date1 >= \'1993-01-01\') AND ((id >= 1) AND (id <= 3))
@@ -217,7 +217,7 @@ QUERY id: 0
                     LIST id: 17, nodes: 2
                       COLUMN id: 14, column_name: id, result_type: UInt32, source_id: 3
                       CONSTANT id: 18, constant_value: UInt64_3, constant_value_type: UInt8
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 SELECT value1
 FROM date_t
 WHERE ((date1 >= \'1993-01-01\') AND (date1 < \'1998-01-01\')) AND ((id >= 1) AND (id <= 3))
@@ -259,7 +259,7 @@ QUERY id: 0
                     LIST id: 22, nodes: 2
                       COLUMN id: 19, column_name: id, result_type: UInt32, source_id: 3
                       CONSTANT id: 23, constant_value: UInt64_3, constant_value_type: UInt8
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 SELECT value1
 FROM date_t
 WHERE (((date1 >= \'1993-01-01\') AND (date1 < \'1994-01-01\')) OR ((date1 >= \'1994-01-01\') AND (date1 < \'1995-01-01\'))) AND ((id >= 1) AND (id <= 3))
@@ -317,7 +317,7 @@ QUERY id: 0
                     LIST id: 32, nodes: 2
                       COLUMN id: 29, column_name: id, result_type: UInt32, source_id: 3
                       CONSTANT id: 33, constant_value: UInt64_3, constant_value_type: UInt8
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 SELECT
     value1,
     toYear(date1) AS year1
@@ -366,7 +366,7 @@ QUERY id: 0
                     LIST id: 24, nodes: 2
                       COLUMN id: 21, column_name: id, result_type: UInt32, source_id: 3
                       CONSTANT id: 25, constant_value: UInt64_3, constant_value_type: UInt8
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 SELECT value1
 FROM date_t
 WHERE (date1 < \'1993-01-01\') AND ((id >= 1) AND (id <= 3))
@@ -400,7 +400,7 @@ QUERY id: 0
                     LIST id: 17, nodes: 2
                       COLUMN id: 14, column_name: id, result_type: UInt32, source_id: 3
                       CONSTANT id: 18, constant_value: UInt64_3, constant_value_type: UInt8
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 SELECT value1
 FROM date_t
 PREWHERE (date1 >= \'1993-01-01\') AND (date1 < \'1994-01-01\')
@@ -441,7 +441,7 @@ QUERY id: 0
               LIST id: 20, nodes: 2
                 COLUMN id: 17, column_name: id, result_type: UInt32, source_id: 3
                 CONSTANT id: 21, constant_value: UInt64_3, constant_value_type: UInt8
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 SELECT value1
 FROM date_t
 WHERE ((id >= 1) AND (id <= 3)) AND ((date1 >= \'1993-01-01\') AND (date1 < \'1994-01-01\'))
@@ -481,7 +481,7 @@ QUERY id: 0
               LIST id: 20, nodes: 2
                 COLUMN id: 17, column_name: date1, result_type: Date, source_id: 3
                 CONSTANT id: 21, constant_value: \'1994-01-01\', constant_value_type: String
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 SELECT value1
 FROM date_t
 WHERE (toYYYYMM(date1) = 199300) AND ((id >= 1) AND (id <= 3))
@@ -518,7 +518,7 @@ QUERY id: 0
                     LIST id: 19, nodes: 2
                       COLUMN id: 16, column_name: id, result_type: UInt32, source_id: 3
                       CONSTANT id: 20, constant_value: UInt64_3, constant_value_type: UInt8
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 SELECT value1
 FROM date_t
 WHERE (toYYYYMM(date1) = 199313) AND ((id >= 1) AND (id <= 3))
@@ -555,7 +555,7 @@ QUERY id: 0
                     LIST id: 19, nodes: 2
                       COLUMN id: 16, column_name: id, result_type: UInt32, source_id: 3
                       CONSTANT id: 20, constant_value: UInt64_3, constant_value_type: UInt8
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 SELECT value1
 FROM date_t
 WHERE ((date1 >= \'1993-12-01\') AND (date1 < \'1994-01-01\')) AND ((id >= 1) AND (id <= 3))
@@ -597,7 +597,7 @@ QUERY id: 0
                     LIST id: 22, nodes: 2
                       COLUMN id: 19, column_name: id, result_type: UInt32, source_id: 3
                       CONSTANT id: 23, constant_value: UInt64_3, constant_value_type: UInt8
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 SELECT value1
 FROM date_t
 WHERE ((date1 >= \'1992-03-01\') AND (date1 < \'1992-04-01\')) AND ((id >= 1) AND (id <= 3))
@@ -639,7 +639,7 @@ QUERY id: 0
                     LIST id: 22, nodes: 2
                       COLUMN id: 19, column_name: id, result_type: UInt32, source_id: 3
                       CONSTANT id: 23, constant_value: UInt64_3, constant_value_type: UInt8
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 SELECT value1
 FROM date_t
 WHERE ((date1 < \'1992-03-01\') OR (date1 >= \'1992-04-01\')) AND ((id >= 1) AND (id <= 3))
@@ -681,7 +681,7 @@ QUERY id: 0
                     LIST id: 22, nodes: 2
                       COLUMN id: 19, column_name: id, result_type: UInt32, source_id: 3
                       CONSTANT id: 23, constant_value: UInt64_3, constant_value_type: UInt8
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 SELECT value1
 FROM date_t
 WHERE (date1 < \'1992-03-01\') AND ((id >= 1) AND (id <= 3))
@@ -715,7 +715,7 @@ QUERY id: 0
                     LIST id: 17, nodes: 2
                       COLUMN id: 14, column_name: id, result_type: UInt32, source_id: 3
                       CONSTANT id: 18, constant_value: UInt64_3, constant_value_type: UInt8
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 SELECT value1
 FROM date_t
 WHERE (date1 >= \'1992-04-01\') AND ((id >= 1) AND (id <= 3))
@@ -749,7 +749,7 @@ QUERY id: 0
                     LIST id: 17, nodes: 2
                       COLUMN id: 14, column_name: id, result_type: UInt32, source_id: 3
                       CONSTANT id: 18, constant_value: UInt64_3, constant_value_type: UInt8
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 SELECT value1
 FROM date_t
 WHERE (date1 < \'1992-04-01\') AND ((id >= 1) AND (id <= 3))
@@ -783,7 +783,7 @@ QUERY id: 0
                     LIST id: 17, nodes: 2
                       COLUMN id: 14, column_name: id, result_type: UInt32, source_id: 3
                       CONSTANT id: 18, constant_value: UInt64_3, constant_value_type: UInt8
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 SELECT value1
 FROM date_t
 WHERE (date1 >= \'1992-03-01\') AND ((id >= 1) AND (id <= 3))
@@ -817,7 +817,7 @@ QUERY id: 0
                     LIST id: 17, nodes: 2
                       COLUMN id: 14, column_name: id, result_type: UInt32, source_id: 3
                       CONSTANT id: 18, constant_value: UInt64_3, constant_value_type: UInt8
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 SELECT value1
 FROM date_t
 WHERE ((date1 >= \'1992-03-01\') OR ((date1 >= \'1993-01-01\') AND (date1 < \'1994-01-01\'))) AND ((id >= 1) AND (id <= 3))
@@ -867,7 +867,7 @@ QUERY id: 0
                     LIST id: 27, nodes: 2
                       COLUMN id: 24, column_name: id, result_type: UInt32, source_id: 3
                       CONSTANT id: 28, constant_value: UInt64_3, constant_value_type: UInt8
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 SELECT value1
 FROM datetime_t
 WHERE ((date1 >= \'1993-01-01 00:00:00\') AND (date1 < \'1994-01-01 00:00:00\')) AND ((id >= 1) AND (id <= 3))
@@ -909,7 +909,7 @@ QUERY id: 0
                     LIST id: 22, nodes: 2
                       COLUMN id: 19, column_name: id, result_type: UInt32, source_id: 3
                       CONSTANT id: 23, constant_value: UInt64_3, constant_value_type: UInt8
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 SELECT value1
 FROM datetime_t
 WHERE ((date1 >= \'1993-12-01 00:00:00\') AND (date1 < \'1994-01-01 00:00:00\')) AND ((id >= 1) AND (id <= 3))
@@ -951,7 +951,7 @@ QUERY id: 0
                     LIST id: 22, nodes: 2
                       COLUMN id: 19, column_name: id, result_type: UInt32, source_id: 3
                       CONSTANT id: 23, constant_value: UInt64_3, constant_value_type: UInt8
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 SELECT value1
 FROM date32_t
 WHERE ((date1 >= \'1993-01-01\') AND (date1 < \'1994-01-01\')) AND ((id >= 1) AND (id <= 3))
@@ -993,7 +993,7 @@ QUERY id: 0
                     LIST id: 22, nodes: 2
                       COLUMN id: 19, column_name: id, result_type: UInt32, source_id: 3
                       CONSTANT id: 23, constant_value: UInt64_3, constant_value_type: UInt8
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 SELECT value1
 FROM date32_t
 WHERE ((date1 >= \'1993-12-01\') AND (date1 < \'1994-01-01\')) AND ((id >= 1) AND (id <= 3))
@@ -1035,7 +1035,7 @@ QUERY id: 0
                     LIST id: 22, nodes: 2
                       COLUMN id: 19, column_name: id, result_type: UInt32, source_id: 3
                       CONSTANT id: 23, constant_value: UInt64_3, constant_value_type: UInt8
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 SELECT value1
 FROM datetime64_t
 WHERE ((date1 >= \'1993-01-01 00:00:00\') AND (date1 < \'1994-01-01 00:00:00\')) AND ((id >= 1) AND (id <= 3))
@@ -1077,7 +1077,7 @@ QUERY id: 0
                     LIST id: 22, nodes: 2
                       COLUMN id: 19, column_name: id, result_type: UInt32, source_id: 3
                       CONSTANT id: 23, constant_value: UInt64_3, constant_value_type: UInt8
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 SELECT value1
 FROM datetime64_t
 WHERE ((date1 >= \'1993-12-01 00:00:00\') AND (date1 < \'1994-01-01 00:00:00\')) AND ((id >= 1) AND (id <= 3))
@@ -1119,4 +1119,4 @@ QUERY id: 0
                     LIST id: 22, nodes: 2
                       COLUMN id: 19, column_name: id, result_type: UInt32, source_id: 3
                       CONSTANT id: 23, constant_value: UInt64_3, constant_value_type: UInt8
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
diff --git a/tests/queries/0_stateless/02785_date_predicate_optimizations_ast_query_tree_rewrite.sql b/tests/queries/0_stateless/02785_date_predicate_optimizations_ast_query_tree_rewrite.sql
index 9cc8dd74e5d..5ff62cb4bb3 100644
--- a/tests/queries/0_stateless/02785_date_predicate_optimizations_ast_query_tree_rewrite.sql
+++ b/tests/queries/0_stateless/02785_date_predicate_optimizations_ast_query_tree_rewrite.sql
@@ -2,74 +2,74 @@ DROP TABLE IF EXISTS date_t;
 CREATE TABLE date_t (id UInt32, value1 String, date1 Date) ENGINE ReplacingMergeTree() ORDER BY id;
 
 EXPLAIN SYNTAX SELECT value1 FROM date_t WHERE toYear(date1) = 1993 AND id BETWEEN 1 AND 3;
-EXPLAIN QUERY TREE run_passes=1 SELECT value1 FROM date_t WHERE toYear(date1) = 1993 AND id BETWEEN 1 AND 3 SETTINGS allow_experimental_analyzer=1;
+EXPLAIN QUERY TREE run_passes=1 SELECT value1 FROM date_t WHERE toYear(date1) = 1993 AND id BETWEEN 1 AND 3 SETTINGS enable_analyzer=1;
 EXPLAIN SYNTAX SELECT value1 FROM date_t WHERE toYear(date1) <> 1993 AND id BETWEEN 1 AND 3;
-EXPLAIN QUERY TREE run_passes=1 SELECT value1 FROM date_t WHERE toYear(date1) <> 1993 AND id BETWEEN 1 AND 3 SETTINGS allow_experimental_analyzer=1;
+EXPLAIN QUERY TREE run_passes=1 SELECT value1 FROM date_t WHERE toYear(date1) <> 1993 AND id BETWEEN 1 AND 3 SETTINGS enable_analyzer=1;
 EXPLAIN SYNTAX SELECT value1 FROM date_t WHERE toYear(date1) < 1993 AND id BETWEEN 1 AND 3;
-EXPLAIN QUERY TREE run_passes=1 SELECT value1 FROM date_t WHERE toYear(date1) < 1993 AND id BETWEEN 1 AND 3 SETTINGS allow_experimental_analyzer=1;
+EXPLAIN QUERY TREE run_passes=1 SELECT value1 FROM date_t WHERE toYear(date1) < 1993 AND id BETWEEN 1 AND 3 SETTINGS enable_analyzer=1;
 EXPLAIN SYNTAX SELECT value1 FROM date_t WHERE toYear(date1) > 1993 AND id BETWEEN 1 AND 3;
-EXPLAIN QUERY TREE run_passes=1 SELECT value1 FROM date_t WHERE toYear(date1) > 1993 AND id BETWEEN 1 AND 3 SETTINGS allow_experimental_analyzer=1;
+EXPLAIN QUERY TREE run_passes=1 SELECT value1 FROM date_t WHERE toYear(date1) > 1993 AND id BETWEEN 1 AND 3 SETTINGS enable_analyzer=1;
 EXPLAIN SYNTAX SELECT value1 FROM date_t WHERE toYear(date1) <= 1993 AND id BETWEEN 1 AND 3;
-EXPLAIN QUERY TREE run_passes=1 SELECT value1 FROM date_t WHERE toYear(date1) <= 1993 AND id BETWEEN 1 AND 3 SETTINGS allow_experimental_analyzer=1;
+EXPLAIN QUERY TREE run_passes=1 SELECT value1 FROM date_t WHERE toYear(date1) <= 1993 AND id BETWEEN 1 AND 3 SETTINGS enable_analyzer=1;
 EXPLAIN SYNTAX SELECT value1 FROM date_t WHERE toYear(date1) >= 1993 AND id BETWEEN 1 AND 3;
-EXPLAIN QUERY TREE run_passes=1 SELECT value1 FROM date_t WHERE toYear(date1) >= 1993 AND id BETWEEN 1 AND 3 SETTINGS allow_experimental_analyzer=1;
+EXPLAIN QUERY TREE run_passes=1 SELECT value1 FROM date_t WHERE toYear(date1) >= 1993 AND id BETWEEN 1 AND 3 SETTINGS enable_analyzer=1;
 EXPLAIN SYNTAX SELECT value1 FROM date_t WHERE toYear(date1) BETWEEN 1993 AND 1997 AND id BETWEEN 1 AND 3;
-EXPLAIN QUERY TREE run_passes=1 SELECT value1 FROM date_t WHERE toYear(date1) BETWEEN 1993 AND 1997 AND id BETWEEN 1 AND 3 SETTINGS allow_experimental_analyzer=1;
+EXPLAIN QUERY TREE run_passes=1 SELECT value1 FROM date_t WHERE toYear(date1) BETWEEN 1993 AND 1997 AND id BETWEEN 1 AND 3 SETTINGS enable_analyzer=1;
 EXPLAIN SYNTAX SELECT value1 FROM date_t WHERE (toYear(date1) = 1993 OR toYear(date1) = 1994) AND id BETWEEN 1 AND 3;
-EXPLAIN QUERY TREE run_passes=1 SELECT value1 FROM date_t WHERE (toYear(date1) = 1993 OR toYear(date1) = 1994) AND id BETWEEN 1 AND 3 SETTINGS allow_experimental_analyzer=1;
+EXPLAIN QUERY TREE run_passes=1 SELECT value1 FROM date_t WHERE (toYear(date1) = 1993 OR toYear(date1) = 1994) AND id BETWEEN 1 AND 3 SETTINGS enable_analyzer=1;
 EXPLAIN SYNTAX SELECT value1, toYear(date1) as year1 FROM date_t WHERE year1 = 1993 AND id BETWEEN 1 AND 3;
-EXPLAIN QUERY TREE run_passes=1 SELECT value1, toYear(date1) as year1 FROM date_t WHERE year1 = 1993 AND id BETWEEN 1 AND 3 SETTINGS allow_experimental_analyzer=1;
+EXPLAIN QUERY TREE run_passes=1 SELECT value1, toYear(date1) as year1 FROM date_t WHERE year1 = 1993 AND id BETWEEN 1 AND 3 SETTINGS enable_analyzer=1;
 EXPLAIN SYNTAX SELECT value1 FROM date_t WHERE 1993 > toYear(date1) AND id BETWEEN 1 AND 3;
-EXPLAIN QUERY TREE run_passes=1 SELECT value1 FROM date_t WHERE 1993 > toYear(date1) AND id BETWEEN 1 AND 3 SETTINGS allow_experimental_analyzer=1;
+EXPLAIN QUERY TREE run_passes=1 SELECT value1 FROM date_t WHERE 1993 > toYear(date1) AND id BETWEEN 1 AND 3 SETTINGS enable_analyzer=1;
 EXPLAIN SYNTAX SELECT value1 FROM date_t PREWHERE toYear(date1) = 1993 WHERE id BETWEEN 1 AND 3;
-EXPLAIN QUERY TREE run_passes=1 SELECT value1 FROM date_t PREWHERE toYear(date1) = 1993 WHERE id BETWEEN 1 AND 3 SETTINGS allow_experimental_analyzer=1;
+EXPLAIN QUERY TREE run_passes=1 SELECT value1 FROM date_t PREWHERE toYear(date1) = 1993 WHERE id BETWEEN 1 AND 3 SETTINGS enable_analyzer=1;
 EXPLAIN SYNTAX SELECT value1 FROM date_t WHERE id BETWEEN 1 AND 3 HAVING toYear(date1) = 1993;
-EXPLAIN QUERY TREE run_passes=1 SELECT value1 FROM date_t WHERE id BETWEEN 1 AND 3 HAVING toYear(date1) = 1993 SETTINGS allow_experimental_analyzer=1;
+EXPLAIN QUERY TREE run_passes=1 SELECT value1 FROM date_t WHERE id BETWEEN 1 AND 3 HAVING toYear(date1) = 1993 SETTINGS enable_analyzer=1;
 EXPLAIN SYNTAX SELECT value1 FROM date_t WHERE toYYYYMM(date1) = 199300 AND id BETWEEN 1 AND 3;
-EXPLAIN QUERY TREE run_passes=1 SELECT value1 FROM date_t WHERE toYYYYMM(date1) = 199300 AND id BETWEEN 1 AND 3 SETTINGS allow_experimental_analyzer=1;
+EXPLAIN QUERY TREE run_passes=1 SELECT value1 FROM date_t WHERE toYYYYMM(date1) = 199300 AND id BETWEEN 1 AND 3 SETTINGS enable_analyzer=1;
 EXPLAIN SYNTAX SELECT value1 FROM date_t WHERE toYYYYMM(date1) = 199313 AND id BETWEEN 1 AND 3;
-EXPLAIN QUERY TREE run_passes=1 SELECT value1 FROM date_t WHERE toYYYYMM(date1) = 199313 AND id BETWEEN 1 AND 3 SETTINGS allow_experimental_analyzer=1;
+EXPLAIN QUERY TREE run_passes=1 SELECT value1 FROM date_t WHERE toYYYYMM(date1) = 199313 AND id BETWEEN 1 AND 3 SETTINGS enable_analyzer=1;
 EXPLAIN SYNTAX SELECT value1 FROM date_t WHERE toYYYYMM(date1) = 199312 AND id BETWEEN 1 AND 3;
-EXPLAIN QUERY TREE run_passes=1 SELECT value1 FROM date_t WHERE toYYYYMM(date1) = 199312 AND id BETWEEN 1 AND 3 SETTINGS allow_experimental_analyzer=1;
+EXPLAIN QUERY TREE run_passes=1 SELECT value1 FROM date_t WHERE toYYYYMM(date1) = 199312 AND id BETWEEN 1 AND 3 SETTINGS enable_analyzer=1;
 EXPLAIN SYNTAX SELECT value1 FROM date_t WHERE toYYYYMM(date1) = 199203 AND id BETWEEN 1 AND 3;
-EXPLAIN QUERY TREE run_passes=1 SELECT value1 FROM date_t WHERE toYYYYMM(date1) = 199203 AND id BETWEEN 1 AND 3 SETTINGS allow_experimental_analyzer=1;
+EXPLAIN QUERY TREE run_passes=1 SELECT value1 FROM date_t WHERE toYYYYMM(date1) = 199203 AND id BETWEEN 1 AND 3 SETTINGS enable_analyzer=1;
 EXPLAIN SYNTAX SELECT value1 FROM date_t WHERE toYYYYMM(date1) <> 199203 AND id BETWEEN 1 AND 3;
-EXPLAIN QUERY TREE run_passes=1 SELECT value1 FROM date_t WHERE toYYYYMM(date1) <> 199203 AND id BETWEEN 1 AND 3 SETTINGS allow_experimental_analyzer=1;
+EXPLAIN QUERY TREE run_passes=1 SELECT value1 FROM date_t WHERE toYYYYMM(date1) <> 199203 AND id BETWEEN 1 AND 3 SETTINGS enable_analyzer=1;
 EXPLAIN SYNTAX SELECT value1 FROM date_t WHERE toYYYYMM(date1) < 199203 AND id BETWEEN 1 AND 3;
-EXPLAIN QUERY TREE run_passes=1 SELECT value1 FROM date_t WHERE toYYYYMM(date1) < 199203 AND id BETWEEN 1 AND 3 SETTINGS allow_experimental_analyzer=1;
+EXPLAIN QUERY TREE run_passes=1 SELECT value1 FROM date_t WHERE toYYYYMM(date1) < 199203 AND id BETWEEN 1 AND 3 SETTINGS enable_analyzer=1;
 EXPLAIN SYNTAX SELECT value1 FROM date_t WHERE toYYYYMM(date1) > 199203 AND id BETWEEN 1 AND 3;
-EXPLAIN QUERY TREE run_passes=1 SELECT value1 FROM date_t WHERE toYYYYMM(date1) > 199203 AND id BETWEEN 1 AND 3 SETTINGS allow_experimental_analyzer=1;
+EXPLAIN QUERY TREE run_passes=1 SELECT value1 FROM date_t WHERE toYYYYMM(date1) > 199203 AND id BETWEEN 1 AND 3 SETTINGS enable_analyzer=1;
 EXPLAIN SYNTAX SELECT value1 FROM date_t WHERE toYYYYMM(date1) <= 199203 AND id BETWEEN 1 AND 3;
-EXPLAIN QUERY TREE run_passes=1 SELECT value1 FROM date_t WHERE toYYYYMM(date1) <= 199203 AND id BETWEEN 1 AND 3 SETTINGS allow_experimental_analyzer=1;
+EXPLAIN QUERY TREE run_passes=1 SELECT value1 FROM date_t WHERE toYYYYMM(date1) <= 199203 AND id BETWEEN 1 AND 3 SETTINGS enable_analyzer=1;
 EXPLAIN SYNTAX SELECT value1 FROM date_t WHERE toYYYYMM(date1) >= 199203 AND id BETWEEN 1 AND 3;
-EXPLAIN QUERY TREE run_passes=1 SELECT value1 FROM date_t WHERE toYYYYMM(date1) >= 199203 AND id BETWEEN 1 AND 3 SETTINGS allow_experimental_analyzer=1;
+EXPLAIN QUERY TREE run_passes=1 SELECT value1 FROM date_t WHERE toYYYYMM(date1) >= 199203 AND id BETWEEN 1 AND 3 SETTINGS enable_analyzer=1;
 EXPLAIN SYNTAX SELECT value1 FROM date_t WHERE (toYYYYMM(date1) >= 199203 OR toYear(date1) = 1993) AND id BETWEEN 1 AND 3;
-EXPLAIN QUERY TREE run_passes=1 SELECT value1 FROM date_t WHERE (toYYYYMM(date1) >= 199203 OR toYear(date1) = 1993) AND id BETWEEN 1 AND 3 SETTINGS allow_experimental_analyzer=1;
+EXPLAIN QUERY TREE run_passes=1 SELECT value1 FROM date_t WHERE (toYYYYMM(date1) >= 199203 OR toYear(date1) = 1993) AND id BETWEEN 1 AND 3 SETTINGS enable_analyzer=1;
 DROP TABLE date_t;
 
 DROP TABLE IF EXISTS datetime_t;
 CREATE TABLE datetime_t (id UInt32, value1 String, date1 Datetime) ENGINE ReplacingMergeTree() ORDER BY id;
 
 EXPLAIN SYNTAX SELECT value1 FROM datetime_t WHERE toYear(date1) = 1993 AND id BETWEEN 1 AND 3;
-EXPLAIN QUERY TREE run_passes=1 SELECT value1 FROM datetime_t WHERE toYear(date1) = 1993 AND id BETWEEN 1 AND 3 SETTINGS allow_experimental_analyzer=1;
+EXPLAIN QUERY TREE run_passes=1 SELECT value1 FROM datetime_t WHERE toYear(date1) = 1993 AND id BETWEEN 1 AND 3 SETTINGS enable_analyzer=1;
 EXPLAIN SYNTAX SELECT value1 FROM datetime_t WHERE toYYYYMM(date1) = 199312 AND id BETWEEN 1 AND 3;
-EXPLAIN QUERY TREE run_passes=1 SELECT value1 FROM datetime_t WHERE toYYYYMM(date1) = 199312 AND id BETWEEN 1 AND 3 SETTINGS allow_experimental_analyzer=1;
+EXPLAIN QUERY TREE run_passes=1 SELECT value1 FROM datetime_t WHERE toYYYYMM(date1) = 199312 AND id BETWEEN 1 AND 3 SETTINGS enable_analyzer=1;
 DROP TABLE datetime_t;
 
 DROP TABLE IF EXISTS date32_t;
 CREATE TABLE date32_t (id UInt32, value1 String, date1 Date32) ENGINE ReplacingMergeTree() ORDER BY id;
 
 EXPLAIN SYNTAX SELECT value1 FROM date32_t WHERE toYear(date1) = 1993 AND id BETWEEN 1 AND 3;
-EXPLAIN QUERY TREE run_passes=1 SELECT value1 FROM date32_t WHERE toYear(date1) = 1993 AND id BETWEEN 1 AND 3 SETTINGS allow_experimental_analyzer=1;
+EXPLAIN QUERY TREE run_passes=1 SELECT value1 FROM date32_t WHERE toYear(date1) = 1993 AND id BETWEEN 1 AND 3 SETTINGS enable_analyzer=1;
 EXPLAIN SYNTAX SELECT value1 FROM date32_t WHERE toYYYYMM(date1) = 199312 AND id BETWEEN 1 AND 3;
-EXPLAIN QUERY TREE run_passes=1 SELECT value1 FROM date32_t WHERE toYYYYMM(date1) = 199312 AND id BETWEEN 1 AND 3 SETTINGS allow_experimental_analyzer=1;
+EXPLAIN QUERY TREE run_passes=1 SELECT value1 FROM date32_t WHERE toYYYYMM(date1) = 199312 AND id BETWEEN 1 AND 3 SETTINGS enable_analyzer=1;
 DROP TABLE date32_t;
 
 DROP TABLE IF EXISTS datetime64_t;
 CREATE TABLE datetime64_t (id UInt32, value1 String, date1 Datetime64) ENGINE ReplacingMergeTree() ORDER BY id;
 
 EXPLAIN SYNTAX SELECT value1 FROM datetime64_t WHERE toYear(date1) = 1993 AND id BETWEEN 1 AND 3;
-EXPLAIN QUERY TREE run_passes=1 SELECT value1 FROM datetime64_t WHERE toYear(date1) = 1993 AND id BETWEEN 1 AND 3 SETTINGS allow_experimental_analyzer=1;
+EXPLAIN QUERY TREE run_passes=1 SELECT value1 FROM datetime64_t WHERE toYear(date1) = 1993 AND id BETWEEN 1 AND 3 SETTINGS enable_analyzer=1;
 EXPLAIN SYNTAX SELECT value1 FROM datetime64_t WHERE toYYYYMM(date1) = 199312 AND id BETWEEN 1 AND 3;
-EXPLAIN QUERY TREE run_passes=1 SELECT value1 FROM datetime64_t WHERE toYYYYMM(date1) = 199312 AND id BETWEEN 1 AND 3 SETTINGS allow_experimental_analyzer=1;
+EXPLAIN QUERY TREE run_passes=1 SELECT value1 FROM datetime64_t WHERE toYYYYMM(date1) = 199312 AND id BETWEEN 1 AND 3 SETTINGS enable_analyzer=1;
 DROP TABLE datetime64_t;
diff --git a/tests/queries/0_stateless/02803_remote_cannot_clone_block.sql b/tests/queries/0_stateless/02803_remote_cannot_clone_block.sql
index 6d79aa76d18..dd72b990445 100644
--- a/tests/queries/0_stateless/02803_remote_cannot_clone_block.sql
+++ b/tests/queries/0_stateless/02803_remote_cannot_clone_block.sql
@@ -6,7 +6,7 @@ SELECT *
 FROM system.numbers
 LIMIT 10000;
 
-SET allow_experimental_analyzer = 0;
+SET enable_analyzer = 0;
 
 SELECT *
 FROM
diff --git a/tests/queries/0_stateless/02812_bug_with_unused_join_columns.sql b/tests/queries/0_stateless/02812_bug_with_unused_join_columns.sql
index 6c801b5b73e..d791b8f3367 100644
--- a/tests/queries/0_stateless/02812_bug_with_unused_join_columns.sql
+++ b/tests/queries/0_stateless/02812_bug_with_unused_join_columns.sql
@@ -1 +1 @@
-SELECT concat(func.name, comb.name) AS x FROM system.functions AS func JOIN system.aggregate_function_combinators AS comb using name WHERE is_aggregate settings allow_experimental_analyzer=1;
+SELECT concat(func.name, comb.name) AS x FROM system.functions AS func JOIN system.aggregate_function_combinators AS comb using name WHERE is_aggregate settings enable_analyzer=1;
diff --git a/tests/queries/0_stateless/02815_analyzer_aggregate_functions_of_group_by_keys.sql b/tests/queries/0_stateless/02815_analyzer_aggregate_functions_of_group_by_keys.sql
index ca03cbb6f9f..dfb885f5a69 100644
--- a/tests/queries/0_stateless/02815_analyzer_aggregate_functions_of_group_by_keys.sql
+++ b/tests/queries/0_stateless/02815_analyzer_aggregate_functions_of_group_by_keys.sql
@@ -1,4 +1,4 @@
-set allow_experimental_analyzer = 1;
+set enable_analyzer = 1;
 set optimize_move_functions_out_of_any = 0;
 
 SELECT 'set optimize_aggregators_of_group_by_keys = 1';
diff --git a/tests/queries/0_stateless/02815_join_algorithm_setting.sql b/tests/queries/0_stateless/02815_join_algorithm_setting.sql
index a8733eebc91..4a5ae784b31 100644
--- a/tests/queries/0_stateless/02815_join_algorithm_setting.sql
+++ b/tests/queries/0_stateless/02815_join_algorithm_setting.sql
@@ -104,9 +104,9 @@ JOIN ( SELECT k AS key, k + 100 AS key2 FROM t2 ) AS t2 ON t1.key = t2.key OR t1
 -- But for CROSS choose `hash` algorithm even though it's not enabled
 SELECT * FROM ( SELECT number AS key, number * 10 AS key2 FROM numbers_mt(10) ) AS t1
 CROSS JOIN ( SELECT k AS key, k + 100 AS key2 FROM t2 ) AS t2 FORMAT Null
-SETTINGS allow_experimental_analyzer = 1;
+SETTINGS enable_analyzer = 1;
 
 -- ... (not for old analyzer)
 SELECT * FROM ( SELECT number AS key, number * 10 AS key2 FROM numbers_mt(10) ) AS t1
 CROSS JOIN ( SELECT k AS key, k + 100 AS key2 FROM t2 ) AS t2 FORMAT Null
-SETTINGS allow_experimental_analyzer = 0; -- { serverError NOT_IMPLEMENTED }
+SETTINGS enable_analyzer = 0; -- { serverError NOT_IMPLEMENTED }
diff --git a/tests/queries/0_stateless/02834_analyzer_with_statement_references.sql b/tests/queries/0_stateless/02834_analyzer_with_statement_references.sql
index 29ed6e3f0da..ce1eaa7ae77 100644
--- a/tests/queries/0_stateless/02834_analyzer_with_statement_references.sql
+++ b/tests/queries/0_stateless/02834_analyzer_with_statement_references.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 WITH test_aliases AS (SELECT number FROM numbers(20)), alias2 AS (SELECT number FROM test_aliases)
 SELECT number FROM alias2 SETTINGS enable_global_with_statement = 1;
diff --git a/tests/queries/0_stateless/02835_join_step_explain.sql b/tests/queries/0_stateless/02835_join_step_explain.sql
index d0475fa14b6..1cdd3684a0b 100644
--- a/tests/queries/0_stateless/02835_join_step_explain.sql
+++ b/tests/queries/0_stateless/02835_join_step_explain.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS test_table_1;
 CREATE TABLE test_table_1
diff --git a/tests/queries/0_stateless/02840_merge__table_or_filter.reference b/tests/queries/0_stateless/02840_merge__table_or_filter.reference
index ff5e0865a22..21b54a0121e 100644
--- a/tests/queries/0_stateless/02840_merge__table_or_filter.reference
+++ b/tests/queries/0_stateless/02840_merge__table_or_filter.reference
@@ -1,38 +1,38 @@
 -- { echoOn }
 
-select _table, key from m where (value = 10 and _table = 'v1') or (value = 20 and _table = 'v1') settings allow_experimental_analyzer=0, convert_query_to_cnf=0;
+select _table, key from m where (value = 10 and _table = 'v1') or (value = 20 and _table = 'v1') settings enable_analyzer=0, convert_query_to_cnf=0;
 v1	1
 v1	2
-select _table, key from m where (value = 10 and _table = 'v1') or (value = 20 and _table = 'v2') settings allow_experimental_analyzer=0, convert_query_to_cnf=0;
+select _table, key from m where (value = 10 and _table = 'v1') or (value = 20 and _table = 'v2') settings enable_analyzer=0, convert_query_to_cnf=0;
 v1	1
 v2	2
-select _table, key from m where (value = 10 and _table = 'v1') or (value = 20 and _table = 'v3') settings allow_experimental_analyzer=0, convert_query_to_cnf=0;
+select _table, key from m where (value = 10 and _table = 'v1') or (value = 20 and _table = 'v3') settings enable_analyzer=0, convert_query_to_cnf=0;
 v1	1
-select _table, key from m where (value = 10 and _table = 'v3') or (value = 20 and _table = 'v3') settings allow_experimental_analyzer=0, convert_query_to_cnf=0;
-select _table, key from m where (value = 10 and _table = 'v1') or (value = 20 and _table = 'v1') settings allow_experimental_analyzer=0, convert_query_to_cnf=1;
+select _table, key from m where (value = 10 and _table = 'v3') or (value = 20 and _table = 'v3') settings enable_analyzer=0, convert_query_to_cnf=0;
+select _table, key from m where (value = 10 and _table = 'v1') or (value = 20 and _table = 'v1') settings enable_analyzer=0, convert_query_to_cnf=1;
 v1	1
 v1	2
-select _table, key from m where (value = 10 and _table = 'v1') or (value = 20 and _table = 'v2') settings allow_experimental_analyzer=0, convert_query_to_cnf=1;
+select _table, key from m where (value = 10 and _table = 'v1') or (value = 20 and _table = 'v2') settings enable_analyzer=0, convert_query_to_cnf=1;
 v1	1
 v2	2
-select _table, key from m where (value = 10 and _table = 'v1') or (value = 20 and _table = 'v3') settings allow_experimental_analyzer=0, convert_query_to_cnf=1;
+select _table, key from m where (value = 10 and _table = 'v1') or (value = 20 and _table = 'v3') settings enable_analyzer=0, convert_query_to_cnf=1;
 v1	1
-select _table, key from m where (value = 10 and _table = 'v3') or (value = 20 and _table = 'v3') settings allow_experimental_analyzer=0, convert_query_to_cnf=1;
-select _table, key from m where (value = 10 and _table = 'v1') or (value = 20 and _table = 'v1') settings allow_experimental_analyzer=1, convert_query_to_cnf=0;
+select _table, key from m where (value = 10 and _table = 'v3') or (value = 20 and _table = 'v3') settings enable_analyzer=0, convert_query_to_cnf=1;
+select _table, key from m where (value = 10 and _table = 'v1') or (value = 20 and _table = 'v1') settings enable_analyzer=1, convert_query_to_cnf=0;
 v1	1
 v1	2
-select _table, key from m where (value = 10 and _table = 'v1') or (value = 20 and _table = 'v2') settings allow_experimental_analyzer=1, convert_query_to_cnf=0;
+select _table, key from m where (value = 10 and _table = 'v1') or (value = 20 and _table = 'v2') settings enable_analyzer=1, convert_query_to_cnf=0;
 v1	1
 v2	2
-select _table, key from m where (value = 10 and _table = 'v1') or (value = 20 and _table = 'v3') settings allow_experimental_analyzer=1, convert_query_to_cnf=0;
+select _table, key from m where (value = 10 and _table = 'v1') or (value = 20 and _table = 'v3') settings enable_analyzer=1, convert_query_to_cnf=0;
 v1	1
-select _table, key from m where (value = 10 and _table = 'v3') or (value = 20 and _table = 'v3') settings allow_experimental_analyzer=1, convert_query_to_cnf=0;
-select _table, key from m where (value = 10 and _table = 'v1') or (value = 20 and _table = 'v1') settings allow_experimental_analyzer=1, convert_query_to_cnf=1;
+select _table, key from m where (value = 10 and _table = 'v3') or (value = 20 and _table = 'v3') settings enable_analyzer=1, convert_query_to_cnf=0;
+select _table, key from m where (value = 10 and _table = 'v1') or (value = 20 and _table = 'v1') settings enable_analyzer=1, convert_query_to_cnf=1;
 v1	1
 v1	2
-select _table, key from m where (value = 10 and _table = 'v1') or (value = 20 and _table = 'v2') settings allow_experimental_analyzer=1, convert_query_to_cnf=1;
+select _table, key from m where (value = 10 and _table = 'v1') or (value = 20 and _table = 'v2') settings enable_analyzer=1, convert_query_to_cnf=1;
 v1	1
 v2	2
-select _table, key from m where (value = 10 and _table = 'v1') or (value = 20 and _table = 'v3') settings allow_experimental_analyzer=1, convert_query_to_cnf=1;
+select _table, key from m where (value = 10 and _table = 'v1') or (value = 20 and _table = 'v3') settings enable_analyzer=1, convert_query_to_cnf=1;
 v1	1
-select _table, key from m where (value = 10 and _table = 'v3') or (value = 20 and _table = 'v3') settings allow_experimental_analyzer=1, convert_query_to_cnf=1;
+select _table, key from m where (value = 10 and _table = 'v3') or (value = 20 and _table = 'v3') settings enable_analyzer=1, convert_query_to_cnf=1;
diff --git a/tests/queries/0_stateless/02840_merge__table_or_filter.sql.j2 b/tests/queries/0_stateless/02840_merge__table_or_filter.sql.j2
index 286e4545ef7..f7413dc3ee6 100644
--- a/tests/queries/0_stateless/02840_merge__table_or_filter.sql.j2
+++ b/tests/queries/0_stateless/02840_merge__table_or_filter.sql.j2
@@ -27,10 +27,10 @@ select _table, key from m where (value = 10 and _table = 'v3') or (value = 20 an
 set max_threads=1;
 -- { echoOn }
 {% for settings in [
-    'allow_experimental_analyzer=0, convert_query_to_cnf=0',
-    'allow_experimental_analyzer=0, convert_query_to_cnf=1',
-    'allow_experimental_analyzer=1, convert_query_to_cnf=0',
-    'allow_experimental_analyzer=1, convert_query_to_cnf=1'
+    'enable_analyzer=0, convert_query_to_cnf=0',
+    'enable_analyzer=0, convert_query_to_cnf=1',
+    'enable_analyzer=1, convert_query_to_cnf=0',
+    'enable_analyzer=1, convert_query_to_cnf=1'
 ] %}
 select _table, key from m where (value = 10 and _table = 'v1') or (value = 20 and _table = 'v1') settings {{ settings }};
 select _table, key from m where (value = 10 and _table = 'v1') or (value = 20 and _table = 'v2') settings {{ settings }};
diff --git a/tests/queries/0_stateless/02841_valid_json_and_xml_on_http_exception.sh b/tests/queries/0_stateless/02841_valid_json_and_xml_on_http_exception.sh
index c47fe5c7e94..3dda63e1e49 100755
--- a/tests/queries/0_stateless/02841_valid_json_and_xml_on_http_exception.sh
+++ b/tests/queries/0_stateless/02841_valid_json_and_xml_on_http_exception.sh
@@ -4,7 +4,7 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CUR_DIR"/../shell_config.sh
 
-CH_URL_BASE="$CLICKHOUSE_URL&http_write_exception_in_output_format=1&allow_experimental_analyzer=0"
+CH_URL_BASE="$CLICKHOUSE_URL&http_write_exception_in_output_format=1&enable_analyzer=0"
 
 for wait_end_of_query in 0 1
 do
diff --git a/tests/queries/0_stateless/02841_with_clause_resolve.sql b/tests/queries/0_stateless/02841_with_clause_resolve.sql
index b416446461b..fe94a26110d 100644
--- a/tests/queries/0_stateless/02841_with_clause_resolve.sql
+++ b/tests/queries/0_stateless/02841_with_clause_resolve.sql
@@ -1,16 +1,16 @@
-set allow_experimental_analyzer = 1;
+set enable_analyzer = 1;
 
 WITH
     -- Input
     44100 AS sample_frequency
     , number AS tick
     , tick / sample_frequency AS time
-    
+
     -- Delay
     , (time, wave, delay_, decay, count) -> arraySum(n1 -> wave(time - delay_ * n1), range(count)) AS delay
 
     , delay(time, (time -> 0.5), 0.2, 0.5, 5) AS kick
-    
+
 SELECT
 
     kick
@@ -23,29 +23,29 @@ WITH
     44100 AS sample_frequency
     , number AS tick
     , tick / sample_frequency AS time
-    
+
     -- Output control
     , 1 AS master_volume
     , level -> least(1.0, greatest(-1.0, level)) AS clamp
     , level -> (clamp(level) * 0x7FFF * master_volume)::Int16 AS output
     , x -> (x, x) AS mono
-    
+
     -- Basic waves
     , time -> sin(time * 2 * pi()) AS sine_wave
     , time -> time::UInt64 % 2 * 2 - 1 AS square_wave
     , time -> (time - floor(time)) * 2 - 1 AS sawtooth_wave
     , time -> abs(sawtooth_wave(time)) * 2 - 1 AS triangle_wave
-    
+
     -- Helpers
     , (from, to, wave, time) -> from + ((wave(time) + 1) / 2) * (to - from) AS lfo
     , (from, to, steps, time) -> from + floor((time - floor(time)) * steps) / steps * (to - from) AS step_lfo
     , (from, to, steps, time) -> exp(step_lfo(log(from), log(to), steps, time)) AS exp_step_lfo
-    
+
     -- Noise
     , time -> cityHash64(time) / 0xFFFFFFFFFFFFFFFF AS uniform_noise
     , time -> erf(uniform_noise(time)) AS white_noise
     , time -> cityHash64(time) % 2 ? 1 : -1 AS bernoulli_noise
-    
+
     -- Distortion
     , (x, amount) -> clamp(x * amount) AS clipping
     , (x, amount) -> clamp(x > 0 ? pow(x, amount) : -pow(-x, amount)) AS power_distortion
@@ -53,10 +53,10 @@ WITH
     , (time, sample_frequency) -> round(time * sample_frequency) / sample_frequency AS desample
     , (time, wave, amount) -> (time - floor(time) < (1 - amount)) ? wave(time * (1 - amount)) : 0 AS thin
     , (time, wave, amount) -> wave(floor(time) + pow(time - floor(time), amount)) AS skew
-    
+
     -- Combining
     , (a, b, weight) -> a * (1 - weight) + b * weight AS combine
-    
+
     -- Envelopes
     , (time, offset, attack, hold, release) ->
         time < offset ? 0
@@ -64,7 +64,7 @@ WITH
         : (time < offset + attack + hold           ? 1
         : (time < offset + attack + hold + release ? (offset + attack + hold + release - time) / release
         : 0))) AS envelope
-    
+
     , (bpm, time, offset, attack, hold, release) ->
         envelope(
             time * (bpm / 60) - floor(time * (bpm / 60)),
@@ -72,20 +72,20 @@ WITH
             attack,
             hold,
             release) AS running_envelope
-    
+
     -- Sequencers
     , (sequence, time) -> sequence[1 + time::UInt64 % length(sequence)] AS sequencer
-    
+
     -- Delay
     , (time, wave, delay, decay, count) -> arraySum(n -> wave(time - delay * n) * pow(decay, n), range(count)) AS delay
-    
-    
+
+
     , delay(time, (time -> power_distortion(sine_wave(time * 80 + sine_wave(time * 2)), lfo(0.5, 1, sine_wave, time / 16))
             * running_envelope(60, time, 0, 0.0, 0.01, 0.1)),
             0.2, 0.5, 5) AS kick
-    
+
 SELECT
-    
+
     (output(
         kick +
         delay(time, (time ->
@@ -95,7 +95,7 @@ SELECT
                 lfo(1, 0.75, triangle_wave, time / 8))),
             0.2, 0.5, 10)
         * lfo(0.5, 1, triangle_wave, time / 7)
-    
+
         + delay(time, (time ->
             power_distortion(
                 sine_wave(time * sequencer([50, 100, 200, 400], time / 2) + 1 * sine_wave(time * sequencer([50, 100, 200], time / 4) + 1/4))
@@ -103,16 +103,16 @@ SELECT
                 lfo(1, 0.75, triangle_wave, time / 8))),
             0.2, 0.5, 10)
         * lfo(0.5, 1, triangle_wave, 16 + time / 11)
-    
+
         + delay(time, (time ->
             white_noise(time) * running_envelope(60, time, 0.75, 0.01, 0.01, 0.1)),
             0.2, 0.5, 10)
         * lfo(0.5, 1, triangle_wave, 24 + time / 13)
-    
+
         + sine_wave(time * 100 + 1 * sine_wave(time * 10 + 1/4))
             * running_envelope(120, time, 0, 0.01, 0.01, 0.1)
     ),
-    
+
     output(
         kick +
         delay(time + 0.01, (time ->
@@ -122,7 +122,7 @@ SELECT
                 lfo(1, 0.75, triangle_wave, time / 8))),
             0.2, 0.5, 10)
         * lfo(0.5, 1, triangle_wave, time / 7)
-    
+
         + delay(time - 0.01, (time ->
             power_distortion(
                 sine_wave(time * sequencer([50, 100, 200, 400], time / 2) + 1 * sine_wave(time * sequencer([50, 100, 200], time / 4) + 1/4))
@@ -130,12 +130,12 @@ SELECT
                 lfo(1, 0.75, triangle_wave, time / 8))),
             0.2, 0.5, 10)
         * lfo(0.5, 1, triangle_wave, 16 + time / 11)
-    
+
         + delay(time + 0.005, (time ->
             white_noise(time) * running_envelope(60, time, 0.75, 0.01, 0.01, 0.1)),
             0.2, 0.5, 10)
         * lfo(0.5, 1, triangle_wave, 24 + time / 13)
     ))
-    
+
 FROM system.numbers
 LIMIT 10;
diff --git a/tests/queries/0_stateless/02861_filter_pushdown_const_bug.sql b/tests/queries/0_stateless/02861_filter_pushdown_const_bug.sql
index ea52df5d4b4..c9cdab0cea2 100644
--- a/tests/queries/0_stateless/02861_filter_pushdown_const_bug.sql
+++ b/tests/queries/0_stateless/02861_filter_pushdown_const_bug.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS t1;
 
diff --git a/tests/queries/0_stateless/02864_statistics_materialize_in_merge.sql b/tests/queries/0_stateless/02864_statistics_materialize_in_merge.sql
index 3e15ec1148e..6606cff263f 100644
--- a/tests/queries/0_stateless/02864_statistics_materialize_in_merge.sql
+++ b/tests/queries/0_stateless/02864_statistics_materialize_in_merge.sql
@@ -2,7 +2,7 @@
 
 DROP TABLE IF EXISTS tab;
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 SET allow_experimental_statistics = 1;
 SET allow_statistics_optimize = 1;
 
diff --git a/tests/queries/0_stateless/02866_size_of_marks_skip_idx_explain.sql b/tests/queries/0_stateless/02866_size_of_marks_skip_idx_explain.sql
index b916c5ca13d..b3adf38710a 100644
--- a/tests/queries/0_stateless/02866_size_of_marks_skip_idx_explain.sql
+++ b/tests/queries/0_stateless/02866_size_of_marks_skip_idx_explain.sql
@@ -4,7 +4,7 @@ SET optimize_move_to_prewhere = 1;
 SET convert_query_to_cnf = 0;
 SET optimize_read_in_order = 1;
 
-SET allow_experimental_analyzer = 1; -- slightly different operator names than w/o
+SET enable_analyzer = 1; -- slightly different operator names than w/o
 
 DROP TABLE IF EXISTS test_skip_idx;
 
diff --git a/tests/queries/0_stateless/02868_distinct_to_count_optimization.reference b/tests/queries/0_stateless/02868_distinct_to_count_optimization.reference
index c2075f72f33..3dedcedbcee 100644
--- a/tests/queries/0_stateless/02868_distinct_to_count_optimization.reference
+++ b/tests/queries/0_stateless/02868_distinct_to_count_optimization.reference
@@ -6,7 +6,7 @@ FROM
     SELECT DISTINCT a
     FROM test_rewrite_uniq_to_count
 )
-SETTINGS allow_experimental_analyzer = 0
+SETTINGS enable_analyzer = 0
 3
 QUERY id: 0
   PROJECTION COLUMNS
@@ -23,7 +23,7 @@ QUERY id: 0
           COLUMN id: 5, column_name: a, result_type: UInt8, source_id: 6
       JOIN TREE
         TABLE id: 6, alias: __table2, table_name: default.test_rewrite_uniq_to_count
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 2. test distinct with subquery alias
 3
 SELECT count()
@@ -32,7 +32,7 @@ FROM
     SELECT DISTINCT a
     FROM test_rewrite_uniq_to_count
 ) AS t
-SETTINGS allow_experimental_analyzer = 0
+SETTINGS enable_analyzer = 0
 3
 QUERY id: 0
   PROJECTION COLUMNS
@@ -49,7 +49,7 @@ QUERY id: 0
           COLUMN id: 5, column_name: a, result_type: UInt8, source_id: 6
       JOIN TREE
         TABLE id: 6, alias: __table2, table_name: default.test_rewrite_uniq_to_count
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 3. test distinct with compound column name
 3
 SELECT count()
@@ -58,7 +58,7 @@ FROM
     SELECT DISTINCT a
     FROM test_rewrite_uniq_to_count
 ) AS t
-SETTINGS allow_experimental_analyzer = 0
+SETTINGS enable_analyzer = 0
 3
 QUERY id: 0
   PROJECTION COLUMNS
@@ -75,7 +75,7 @@ QUERY id: 0
           COLUMN id: 5, column_name: a, result_type: UInt8, source_id: 6
       JOIN TREE
         TABLE id: 6, alias: __table2, table_name: default.test_rewrite_uniq_to_count
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 4. test distinct with select expression alias
 3
 SELECT count()
@@ -84,7 +84,7 @@ FROM
     SELECT DISTINCT a AS alias_of_a
     FROM test_rewrite_uniq_to_count
 ) AS t
-SETTINGS allow_experimental_analyzer = 0
+SETTINGS enable_analyzer = 0
 3
 QUERY id: 0
   PROJECTION COLUMNS
@@ -101,7 +101,7 @@ QUERY id: 0
           COLUMN id: 5, column_name: a, result_type: UInt8, source_id: 6
       JOIN TREE
         TABLE id: 6, alias: __table2, table_name: default.test_rewrite_uniq_to_count
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 5. test simple group by
 3
 SELECT count()
@@ -113,7 +113,7 @@ FROM
     FROM test_rewrite_uniq_to_count
     GROUP BY a
 )
-SETTINGS allow_experimental_analyzer = 0
+SETTINGS enable_analyzer = 0
 3
 QUERY id: 0
   PROJECTION COLUMNS
@@ -133,7 +133,7 @@ QUERY id: 0
       GROUP BY
         LIST id: 7, nodes: 1
           COLUMN id: 5, column_name: a, result_type: UInt8, source_id: 6
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 6. test group by with subquery alias
 3
 SELECT count()
@@ -145,7 +145,7 @@ FROM
     FROM test_rewrite_uniq_to_count
     GROUP BY a
 ) AS t
-SETTINGS allow_experimental_analyzer = 0
+SETTINGS enable_analyzer = 0
 3
 QUERY id: 0
   PROJECTION COLUMNS
@@ -165,7 +165,7 @@ QUERY id: 0
       GROUP BY
         LIST id: 7, nodes: 1
           COLUMN id: 5, column_name: a, result_type: UInt8, source_id: 6
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 7. test group by with compound column name
 3
 SELECT count()
@@ -177,7 +177,7 @@ FROM
     FROM test_rewrite_uniq_to_count
     GROUP BY a
 ) AS t
-SETTINGS allow_experimental_analyzer = 0
+SETTINGS enable_analyzer = 0
 3
 QUERY id: 0
   PROJECTION COLUMNS
@@ -197,7 +197,7 @@ QUERY id: 0
       GROUP BY
         LIST id: 7, nodes: 1
           COLUMN id: 5, column_name: a, result_type: UInt8, source_id: 6
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
 8. test group by with select expression alias
 3
 SELECT count()
@@ -209,7 +209,7 @@ FROM
     FROM test_rewrite_uniq_to_count
     GROUP BY alias_of_a
 ) AS t
-SETTINGS allow_experimental_analyzer = 0
+SETTINGS enable_analyzer = 0
 3
 QUERY id: 0
   PROJECTION COLUMNS
@@ -229,4 +229,4 @@ QUERY id: 0
       GROUP BY
         LIST id: 7, nodes: 1
           COLUMN id: 5, column_name: a, result_type: UInt8, source_id: 6
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
diff --git a/tests/queries/0_stateless/02868_distinct_to_count_optimization.sql b/tests/queries/0_stateless/02868_distinct_to_count_optimization.sql
index 66431b7c36b..d30bade4dd5 100644
--- a/tests/queries/0_stateless/02868_distinct_to_count_optimization.sql
+++ b/tests/queries/0_stateless/02868_distinct_to_count_optimization.sql
@@ -16,53 +16,53 @@ set optimize_uniq_to_count=true;
 
 
 SELECT '1. test simple distinct';
-SELECT uniq(a) FROM (SELECT DISTINCT a FROM test_rewrite_uniq_to_count) settings allow_experimental_analyzer=0;
-EXPLAIN SYNTAX SELECT uniq(a) FROM (SELECT DISTINCT a FROM test_rewrite_uniq_to_count) settings allow_experimental_analyzer=0;
-SELECT uniq(a) FROM (SELECT DISTINCT a FROM test_rewrite_uniq_to_count) settings allow_experimental_analyzer=1;
-EXPLAIN QUERY TREE SELECT uniq(a) FROM (SELECT DISTINCT a FROM test_rewrite_uniq_to_count) settings allow_experimental_analyzer=1;
+SELECT uniq(a) FROM (SELECT DISTINCT a FROM test_rewrite_uniq_to_count) settings enable_analyzer=0;
+EXPLAIN SYNTAX SELECT uniq(a) FROM (SELECT DISTINCT a FROM test_rewrite_uniq_to_count) settings enable_analyzer=0;
+SELECT uniq(a) FROM (SELECT DISTINCT a FROM test_rewrite_uniq_to_count) settings enable_analyzer=1;
+EXPLAIN QUERY TREE SELECT uniq(a) FROM (SELECT DISTINCT a FROM test_rewrite_uniq_to_count) settings enable_analyzer=1;
 
 
 SELECT '2. test distinct with subquery alias';
-SELECT uniq(t.a) FROM (SELECT DISTINCT a FROM test_rewrite_uniq_to_count) t settings allow_experimental_analyzer=0;
-EXPLAIN SYNTAX SELECT uniq(a) FROM (SELECT DISTINCT a FROM test_rewrite_uniq_to_count) t settings allow_experimental_analyzer=0;
-SELECT uniq(t.a) FROM (SELECT DISTINCT a FROM test_rewrite_uniq_to_count) t settings allow_experimental_analyzer=1;
-EXPLAIN QUERY TREE SELECT uniq(t.a) FROM (SELECT DISTINCT a FROM test_rewrite_uniq_to_count) t settings allow_experimental_analyzer=1;
+SELECT uniq(t.a) FROM (SELECT DISTINCT a FROM test_rewrite_uniq_to_count) t settings enable_analyzer=0;
+EXPLAIN SYNTAX SELECT uniq(a) FROM (SELECT DISTINCT a FROM test_rewrite_uniq_to_count) t settings enable_analyzer=0;
+SELECT uniq(t.a) FROM (SELECT DISTINCT a FROM test_rewrite_uniq_to_count) t settings enable_analyzer=1;
+EXPLAIN QUERY TREE SELECT uniq(t.a) FROM (SELECT DISTINCT a FROM test_rewrite_uniq_to_count) t settings enable_analyzer=1;
 
 SELECT '3. test distinct with compound column name';
-SELECT uniq(a) FROM (SELECT DISTINCT test_rewrite_uniq_to_count.a FROM test_rewrite_uniq_to_count) t settings allow_experimental_analyzer=0;
-EXPLAIN SYNTAX SELECT uniq(a) FROM (SELECT DISTINCT test_rewrite_uniq_to_count.a FROM test_rewrite_uniq_to_count) t settings allow_experimental_analyzer=0;
-SELECT uniq(a) FROM (SELECT DISTINCT test_rewrite_uniq_to_count.a FROM test_rewrite_uniq_to_count) t settings allow_experimental_analyzer=1;
-EXPLAIN QUERY TREE SELECT uniq(a) FROM (SELECT DISTINCT test_rewrite_uniq_to_count.a FROM test_rewrite_uniq_to_count) t settings allow_experimental_analyzer=1;
+SELECT uniq(a) FROM (SELECT DISTINCT test_rewrite_uniq_to_count.a FROM test_rewrite_uniq_to_count) t settings enable_analyzer=0;
+EXPLAIN SYNTAX SELECT uniq(a) FROM (SELECT DISTINCT test_rewrite_uniq_to_count.a FROM test_rewrite_uniq_to_count) t settings enable_analyzer=0;
+SELECT uniq(a) FROM (SELECT DISTINCT test_rewrite_uniq_to_count.a FROM test_rewrite_uniq_to_count) t settings enable_analyzer=1;
+EXPLAIN QUERY TREE SELECT uniq(a) FROM (SELECT DISTINCT test_rewrite_uniq_to_count.a FROM test_rewrite_uniq_to_count) t settings enable_analyzer=1;
 
 SELECT '4. test distinct with select expression alias';
-SELECT uniq(alias_of_a) FROM (SELECT DISTINCT a as alias_of_a FROM test_rewrite_uniq_to_count) t settings allow_experimental_analyzer=0;
-EXPLAIN SYNTAX SELECT uniq(alias_of_a) FROM (SELECT DISTINCT a as alias_of_a FROM test_rewrite_uniq_to_count) t settings allow_experimental_analyzer=0;
-SELECT uniq(alias_of_a) FROM (SELECT DISTINCT a as alias_of_a FROM test_rewrite_uniq_to_count) t settings allow_experimental_analyzer=1;
-EXPLAIN QUERY TREE SELECT uniq(alias_of_a) FROM (SELECT DISTINCT a as alias_of_a FROM test_rewrite_uniq_to_count) t settings allow_experimental_analyzer=1;
+SELECT uniq(alias_of_a) FROM (SELECT DISTINCT a as alias_of_a FROM test_rewrite_uniq_to_count) t settings enable_analyzer=0;
+EXPLAIN SYNTAX SELECT uniq(alias_of_a) FROM (SELECT DISTINCT a as alias_of_a FROM test_rewrite_uniq_to_count) t settings enable_analyzer=0;
+SELECT uniq(alias_of_a) FROM (SELECT DISTINCT a as alias_of_a FROM test_rewrite_uniq_to_count) t settings enable_analyzer=1;
+EXPLAIN QUERY TREE SELECT uniq(alias_of_a) FROM (SELECT DISTINCT a as alias_of_a FROM test_rewrite_uniq_to_count) t settings enable_analyzer=1;
 
 
 SELECT '5. test simple group by';
-SELECT uniq(a) FROM (SELECT a, sum(b) FROM test_rewrite_uniq_to_count GROUP BY a) settings allow_experimental_analyzer=0;
-EXPLAIN SYNTAX SELECT uniq(a) FROM (SELECT a, sum(b) FROM test_rewrite_uniq_to_count GROUP BY a) settings allow_experimental_analyzer=0;
-SELECT uniq(a) FROM (SELECT a, sum(b) FROM test_rewrite_uniq_to_count GROUP BY a) settings allow_experimental_analyzer=1;
-EXPLAIN QUERY TREE SELECT uniq(a) FROM (SELECT a, sum(b) FROM test_rewrite_uniq_to_count GROUP BY a) settings allow_experimental_analyzer=1;
+SELECT uniq(a) FROM (SELECT a, sum(b) FROM test_rewrite_uniq_to_count GROUP BY a) settings enable_analyzer=0;
+EXPLAIN SYNTAX SELECT uniq(a) FROM (SELECT a, sum(b) FROM test_rewrite_uniq_to_count GROUP BY a) settings enable_analyzer=0;
+SELECT uniq(a) FROM (SELECT a, sum(b) FROM test_rewrite_uniq_to_count GROUP BY a) settings enable_analyzer=1;
+EXPLAIN QUERY TREE SELECT uniq(a) FROM (SELECT a, sum(b) FROM test_rewrite_uniq_to_count GROUP BY a) settings enable_analyzer=1;
 
 SELECT '6. test group by with subquery alias';
-SELECT uniq(t.a) FROM (SELECT a, sum(b) FROM test_rewrite_uniq_to_count GROUP BY a) t settings allow_experimental_analyzer=0;
-EXPLAIN SYNTAX SELECT uniq(t.a) FROM (SELECT a, sum(b) FROM test_rewrite_uniq_to_count GROUP BY a) t settings allow_experimental_analyzer=0;
-SELECT uniq(t.a) FROM (SELECT a, sum(b) FROM test_rewrite_uniq_to_count GROUP BY a) t settings allow_experimental_analyzer=1;
-EXPLAIN QUERY TREE SELECT uniq(t.a) FROM (SELECT a, sum(b) FROM test_rewrite_uniq_to_count GROUP BY a) t settings allow_experimental_analyzer=1;
+SELECT uniq(t.a) FROM (SELECT a, sum(b) FROM test_rewrite_uniq_to_count GROUP BY a) t settings enable_analyzer=0;
+EXPLAIN SYNTAX SELECT uniq(t.a) FROM (SELECT a, sum(b) FROM test_rewrite_uniq_to_count GROUP BY a) t settings enable_analyzer=0;
+SELECT uniq(t.a) FROM (SELECT a, sum(b) FROM test_rewrite_uniq_to_count GROUP BY a) t settings enable_analyzer=1;
+EXPLAIN QUERY TREE SELECT uniq(t.a) FROM (SELECT a, sum(b) FROM test_rewrite_uniq_to_count GROUP BY a) t settings enable_analyzer=1;
 
 SELECT '7. test group by with compound column name';
-SELECT uniq(t.alias_of_a) FROM (SELECT a as alias_of_a, sum(b) FROM test_rewrite_uniq_to_count GROUP BY a) t settings allow_experimental_analyzer=0;
-EXPLAIN SYNTAX SELECT uniq(t.alias_of_a) FROM (SELECT a as alias_of_a, sum(b) FROM test_rewrite_uniq_to_count GROUP BY a) t settings allow_experimental_analyzer=0;
-SELECT uniq(t.alias_of_a) FROM (SELECT a as alias_of_a, sum(b) FROM test_rewrite_uniq_to_count GROUP BY a) t settings allow_experimental_analyzer=1;
-EXPLAIN QUERY TREE SELECT uniq(t.alias_of_a) FROM (SELECT a as alias_of_a, sum(b) FROM test_rewrite_uniq_to_count GROUP BY a) t settings allow_experimental_analyzer=1;
+SELECT uniq(t.alias_of_a) FROM (SELECT a as alias_of_a, sum(b) FROM test_rewrite_uniq_to_count GROUP BY a) t settings enable_analyzer=0;
+EXPLAIN SYNTAX SELECT uniq(t.alias_of_a) FROM (SELECT a as alias_of_a, sum(b) FROM test_rewrite_uniq_to_count GROUP BY a) t settings enable_analyzer=0;
+SELECT uniq(t.alias_of_a) FROM (SELECT a as alias_of_a, sum(b) FROM test_rewrite_uniq_to_count GROUP BY a) t settings enable_analyzer=1;
+EXPLAIN QUERY TREE SELECT uniq(t.alias_of_a) FROM (SELECT a as alias_of_a, sum(b) FROM test_rewrite_uniq_to_count GROUP BY a) t settings enable_analyzer=1;
 
 SELECT '8. test group by with select expression alias';
-SELECT uniq(t.alias_of_a) FROM (SELECT a as alias_of_a, sum(b) FROM test_rewrite_uniq_to_count GROUP BY alias_of_a) t settings allow_experimental_analyzer=0;
-EXPLAIN SYNTAX SELECT uniq(t.alias_of_a) FROM (SELECT a as alias_of_a, sum(b) FROM test_rewrite_uniq_to_count GROUP BY alias_of_a) t  settings allow_experimental_analyzer=0;
-SELECT uniq(t.alias_of_a) FROM (SELECT a as alias_of_a, sum(b) FROM test_rewrite_uniq_to_count GROUP BY alias_of_a) t  settings allow_experimental_analyzer=1;
-EXPLAIN QUERY TREE SELECT uniq(t.alias_of_a) FROM (SELECT a as alias_of_a, sum(b) FROM test_rewrite_uniq_to_count GROUP BY alias_of_a) t  settings allow_experimental_analyzer=1;
+SELECT uniq(t.alias_of_a) FROM (SELECT a as alias_of_a, sum(b) FROM test_rewrite_uniq_to_count GROUP BY alias_of_a) t settings enable_analyzer=0;
+EXPLAIN SYNTAX SELECT uniq(t.alias_of_a) FROM (SELECT a as alias_of_a, sum(b) FROM test_rewrite_uniq_to_count GROUP BY alias_of_a) t  settings enable_analyzer=0;
+SELECT uniq(t.alias_of_a) FROM (SELECT a as alias_of_a, sum(b) FROM test_rewrite_uniq_to_count GROUP BY alias_of_a) t  settings enable_analyzer=1;
+EXPLAIN QUERY TREE SELECT uniq(t.alias_of_a) FROM (SELECT a as alias_of_a, sum(b) FROM test_rewrite_uniq_to_count GROUP BY alias_of_a) t  settings enable_analyzer=1;
 
 drop table if exists test_rewrite_uniq_to_count;
diff --git a/tests/queries/0_stateless/02882_primary_key_index_in_function_different_types.sql b/tests/queries/0_stateless/02882_primary_key_index_in_function_different_types.sql
index 1b1a7607344..83b38955734 100644
--- a/tests/queries/0_stateless/02882_primary_key_index_in_function_different_types.sql
+++ b/tests/queries/0_stateless/02882_primary_key_index_in_function_different_types.sql
@@ -7,14 +7,14 @@ CREATE TABLE test_table
 
 INSERT INTO test_table SELECT number, number FROM numbers(10);
 
-set allow_experimental_analyzer = 0;
+set enable_analyzer = 0;
 
 EXPLAIN indexes = 1, description=0 SELECT id FROM test_table WHERE id <= 10 AND value IN (SELECT 5);
 EXPLAIN indexes = 1, description=0 SELECT id FROM test_table WHERE id <= 10 AND value IN (SELECT '5');
 EXPLAIN indexes = 1, description=0 SELECT id FROM test_table WHERE id <= 10 AND value IN (SELECT toUInt8(number) FROM numbers(5));
 EXPLAIN indexes = 1, description=0 SELECT id FROM test_table WHERE id <= 10 AND value IN (SELECT toString(number) FROM numbers(5));
 
-set allow_experimental_analyzer = 1;
+set enable_analyzer = 1;
 
 EXPLAIN indexes = 1, description=0 SELECT id FROM test_table WHERE id <= 10 AND value IN (SELECT 5);
 EXPLAIN indexes = 1, description=0 SELECT id FROM test_table WHERE id <= 10 AND value IN (SELECT '5');
diff --git a/tests/queries/0_stateless/02890_named_tuple_functions.sql b/tests/queries/0_stateless/02890_named_tuple_functions.sql
index 0033102bd53..6724462562a 100644
--- a/tests/queries/0_stateless/02890_named_tuple_functions.sql
+++ b/tests/queries/0_stateless/02890_named_tuple_functions.sql
@@ -1,5 +1,5 @@
 set enable_named_columns_in_function_tuple = 1;
-set allow_experimental_analyzer = 1;
+set enable_analyzer = 1;
 
 drop table if exists x;
 create table x (i int, j int) engine MergeTree order by i;
diff --git a/tests/queries/0_stateless/02890_untuple_column_names.reference b/tests/queries/0_stateless/02890_untuple_column_names.reference
index 13a85c70138..f91a63fa15f 100644
--- a/tests/queries/0_stateless/02890_untuple_column_names.reference
+++ b/tests/queries/0_stateless/02890_untuple_column_names.reference
@@ -57,7 +57,7 @@ t.1: 1
 Row 1:
 ──────
 t.1: 1
--- tuple() with enable_named_columns_in_function_tuple = 1 and allow_experimental_analyzer = 1 keeps the column names
+-- tuple() with enable_named_columns_in_function_tuple = 1 and enable_analyzer = 1 keeps the column names
 Row 1:
 ──────
 t.a: 1
diff --git a/tests/queries/0_stateless/02890_untuple_column_names.sql b/tests/queries/0_stateless/02890_untuple_column_names.sql
index cd490ca3522..9773e2e53a0 100644
--- a/tests/queries/0_stateless/02890_untuple_column_names.sql
+++ b/tests/queries/0_stateless/02890_untuple_column_names.sql
@@ -6,43 +6,43 @@
 
 SELECT '-- tuple element alias';
 
-SELECT untuple(tuple(1)::Tuple(a Int)), untuple(tuple('s')::Tuple(a String)) FORMAT Vertical SETTINGS allow_experimental_analyzer = 0;
-SELECT untuple(tuple(1)::Tuple(a Int)), untuple(tuple('s')::Tuple(a String)) FORMAT Vertical SETTINGS allow_experimental_analyzer = 1;
+SELECT untuple(tuple(1)::Tuple(a Int)), untuple(tuple('s')::Tuple(a String)) FORMAT Vertical SETTINGS enable_analyzer = 0;
+SELECT untuple(tuple(1)::Tuple(a Int)), untuple(tuple('s')::Tuple(a String)) FORMAT Vertical SETTINGS enable_analyzer = 1;
 
-SELECT untuple(tuple(1)::Tuple(a Int)), untuple(tuple(1)::Tuple(a Int)) FORMAT Vertical SETTINGS allow_experimental_analyzer = 0; -- { serverError DUPLICATE_COLUMN }
-SELECT untuple(tuple(1)::Tuple(a Int)), untuple(tuple(1)::Tuple(a Int)) FORMAT Vertical SETTINGS allow_experimental_analyzer = 1; -- Bug: doesn't throw an exception
+SELECT untuple(tuple(1)::Tuple(a Int)), untuple(tuple(1)::Tuple(a Int)) FORMAT Vertical SETTINGS enable_analyzer = 0; -- { serverError DUPLICATE_COLUMN }
+SELECT untuple(tuple(1)::Tuple(a Int)), untuple(tuple(1)::Tuple(a Int)) FORMAT Vertical SETTINGS enable_analyzer = 1; -- Bug: doesn't throw an exception
 
 SELECT '-- tuple element alias + untuple() alias';
 
-SELECT untuple(tuple(1)::Tuple(a Int)) x, untuple(tuple('s')::Tuple(a String)) y FORMAT Vertical SETTINGS allow_experimental_analyzer = 0;
-SELECT untuple(tuple(1)::Tuple(a Int)) x, untuple(tuple('s')::Tuple(a String)) y FORMAT Vertical SETTINGS allow_experimental_analyzer = 1;
+SELECT untuple(tuple(1)::Tuple(a Int)) x, untuple(tuple('s')::Tuple(a String)) y FORMAT Vertical SETTINGS enable_analyzer = 0;
+SELECT untuple(tuple(1)::Tuple(a Int)) x, untuple(tuple('s')::Tuple(a String)) y FORMAT Vertical SETTINGS enable_analyzer = 1;
 
-SELECT untuple(tuple(1)::Tuple(a Int)) x, untuple(tuple(1)::Tuple(a Int)) x FORMAT Vertical SETTINGS allow_experimental_analyzer = 0; -- { serverError DUPLICATE_COLUMN }
-SELECT untuple(tuple(1)::Tuple(a Int)) x, untuple(tuple(1)::Tuple(a Int)) x FORMAT Vertical SETTINGS allow_experimental_analyzer = 1; -- Bug: doesn't throw an exception
+SELECT untuple(tuple(1)::Tuple(a Int)) x, untuple(tuple(1)::Tuple(a Int)) x FORMAT Vertical SETTINGS enable_analyzer = 0; -- { serverError DUPLICATE_COLUMN }
+SELECT untuple(tuple(1)::Tuple(a Int)) x, untuple(tuple(1)::Tuple(a Int)) x FORMAT Vertical SETTINGS enable_analyzer = 1; -- Bug: doesn't throw an exception
 
 SELECT '-- untuple() alias';
 
-SELECT untuple(tuple(1)::Tuple(Int)) x, untuple(tuple('s')::Tuple(String)) y FORMAT Vertical SETTINGS allow_experimental_analyzer = 0;
-SELECT untuple(tuple(1)::Tuple(Int)) x, untuple(tuple('s')::Tuple(String)) y FORMAT Vertical SETTINGS allow_experimental_analyzer = 1;
+SELECT untuple(tuple(1)::Tuple(Int)) x, untuple(tuple('s')::Tuple(String)) y FORMAT Vertical SETTINGS enable_analyzer = 0;
+SELECT untuple(tuple(1)::Tuple(Int)) x, untuple(tuple('s')::Tuple(String)) y FORMAT Vertical SETTINGS enable_analyzer = 1;
 
-SELECT untuple(tuple(1)::Tuple(Int)) x, untuple(tuple(1)::Tuple(Int)) x FORMAT Vertical SETTINGS allow_experimental_analyzer = 0; -- { serverError DUPLICATE_COLUMN }
-SELECT untuple(tuple(1)::Tuple(Int)) x, untuple(tuple(1)::Tuple(Int)) x FORMAT Vertical SETTINGS allow_experimental_analyzer = 1; -- Bug: doesn't throw an exception
+SELECT untuple(tuple(1)::Tuple(Int)) x, untuple(tuple(1)::Tuple(Int)) x FORMAT Vertical SETTINGS enable_analyzer = 0; -- { serverError DUPLICATE_COLUMN }
+SELECT untuple(tuple(1)::Tuple(Int)) x, untuple(tuple(1)::Tuple(Int)) x FORMAT Vertical SETTINGS enable_analyzer = 1; -- Bug: doesn't throw an exception
 
 SELECT '-- no aliases';
 
-SELECT untuple(tuple(1)::Tuple(Int)), untuple(tuple('s')::Tuple(String)) FORMAT Vertical SETTINGS allow_experimental_analyzer = 0;
-SELECT untuple(tuple(1)::Tuple(Int)), untuple(tuple('s')::Tuple(String)) FORMAT Vertical SETTINGS allow_experimental_analyzer = 1;
+SELECT untuple(tuple(1)::Tuple(Int)), untuple(tuple('s')::Tuple(String)) FORMAT Vertical SETTINGS enable_analyzer = 0;
+SELECT untuple(tuple(1)::Tuple(Int)), untuple(tuple('s')::Tuple(String)) FORMAT Vertical SETTINGS enable_analyzer = 1;
 
-SELECT untuple(tuple(1)::Tuple(Int)), untuple(tuple(1)::Tuple(Int)) FORMAT Vertical SETTINGS allow_experimental_analyzer = 0; -- { serverError DUPLICATE_COLUMN }
-SELECT untuple(tuple(1)::Tuple(Int)), untuple(tuple(1)::Tuple(Int)) FORMAT Vertical SETTINGS allow_experimental_analyzer = 1; -- Bug: doesn't throw an exception
+SELECT untuple(tuple(1)::Tuple(Int)), untuple(tuple(1)::Tuple(Int)) FORMAT Vertical SETTINGS enable_analyzer = 0; -- { serverError DUPLICATE_COLUMN }
+SELECT untuple(tuple(1)::Tuple(Int)), untuple(tuple(1)::Tuple(Int)) FORMAT Vertical SETTINGS enable_analyzer = 1; -- Bug: doesn't throw an exception
 
 SELECT '-- tuple() loses the column names (would be good to fix, see #36773)';
-SELECT untuple(tuple(1 as a)) as t FORMAT Vertical SETTINGS allow_experimental_analyzer = 0, enable_named_columns_in_function_tuple = 0;
-SELECT untuple(tuple(1 as a)) as t FORMAT Vertical SETTINGS allow_experimental_analyzer = 1, enable_named_columns_in_function_tuple = 0;
+SELECT untuple(tuple(1 as a)) as t FORMAT Vertical SETTINGS enable_analyzer = 0, enable_named_columns_in_function_tuple = 0;
+SELECT untuple(tuple(1 as a)) as t FORMAT Vertical SETTINGS enable_analyzer = 1, enable_named_columns_in_function_tuple = 0;
 
-SELECT '-- tuple() with enable_named_columns_in_function_tuple = 1 and allow_experimental_analyzer = 1 keeps the column names';
-SELECT untuple(tuple(1 as a)) as t FORMAT Vertical SETTINGS allow_experimental_analyzer = 1, enable_named_columns_in_function_tuple = 1;
+SELECT '-- tuple() with enable_named_columns_in_function_tuple = 1 and enable_analyzer = 1 keeps the column names';
+SELECT untuple(tuple(1 as a)) as t FORMAT Vertical SETTINGS enable_analyzer = 1, enable_named_columns_in_function_tuple = 1;
 
 SELECT '-- thankfully JSONExtract() keeps them';
-SELECT untuple(JSONExtract('{"key": "value"}', 'Tuple(key String)')) x FORMAT Vertical SETTINGS allow_experimental_analyzer = 0;
-SELECT untuple(JSONExtract('{"key": "value"}', 'Tuple(key String)')) x FORMAT Vertical SETTINGS allow_experimental_analyzer = 1;
+SELECT untuple(JSONExtract('{"key": "value"}', 'Tuple(key String)')) x FORMAT Vertical SETTINGS enable_analyzer = 0;
+SELECT untuple(JSONExtract('{"key": "value"}', 'Tuple(key String)')) x FORMAT Vertical SETTINGS enable_analyzer = 1;
diff --git a/tests/queries/0_stateless/02911_analyzer_explain_estimate.sql b/tests/queries/0_stateless/02911_analyzer_explain_estimate.sql
index b082f2f33b2..77f30ba82ba 100644
--- a/tests/queries/0_stateless/02911_analyzer_explain_estimate.sql
+++ b/tests/queries/0_stateless/02911_analyzer_explain_estimate.sql
@@ -1,5 +1,5 @@
 -- Tags: distributed
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 EXPLAIN ESTIMATE SELECT 0 = 1048577, NULL, groupBitmapOr(bitmapBuild([toInt32(65537)])) FROM cluster(test_cluster_two_shards) WHERE NULL = 1048575;
diff --git a/tests/queries/0_stateless/02911_analyzer_order_by_read_in_order_query_plan.sql b/tests/queries/0_stateless/02911_analyzer_order_by_read_in_order_query_plan.sql
index 77a72c24f5a..907263168d0 100644
--- a/tests/queries/0_stateless/02911_analyzer_order_by_read_in_order_query_plan.sql
+++ b/tests/queries/0_stateless/02911_analyzer_order_by_read_in_order_query_plan.sql
@@ -1,4 +1,4 @@
-SET optimize_read_in_order = 1, query_plan_read_in_order = 1, allow_experimental_analyzer = 1;
+SET optimize_read_in_order = 1, query_plan_read_in_order = 1, enable_analyzer = 1;
 
 drop table if exists tab;
 drop table if exists tab2;
diff --git a/tests/queries/0_stateless/02911_analyzer_remove_unused_projection_columns.sql b/tests/queries/0_stateless/02911_analyzer_remove_unused_projection_columns.sql
index 70de63c592d..d567ac6c8f3 100644
--- a/tests/queries/0_stateless/02911_analyzer_remove_unused_projection_columns.sql
+++ b/tests/queries/0_stateless/02911_analyzer_remove_unused_projection_columns.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS test_table;
 CREATE TABLE test_table
diff --git a/tests/queries/0_stateless/02911_join_on_nullsafe_optimization.sql b/tests/queries/0_stateless/02911_join_on_nullsafe_optimization.sql
index ee5e1582015..e1e095a6eae 100644
--- a/tests/queries/0_stateless/02911_join_on_nullsafe_optimization.sql
+++ b/tests/queries/0_stateless/02911_join_on_nullsafe_optimization.sql
@@ -15,7 +15,7 @@ CREATE TABLE t2n (x Int64, y UInt64) ENGINE = TinyLog;
 INSERT INTO t1n VALUES (1,42), (2,2), (3,3);
 INSERT INTO t2n VALUES (2,2), (3,33), (4,42);
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 -- { echoOn }
 SELECT * FROM t1 JOIN t2 ON (t1.x <=> t2.x OR (t1.x IS NULL AND t2.x IS NULL)) ORDER BY t1.x NULLS LAST;
diff --git a/tests/queries/0_stateless/02911_support_alias_column_in_indices.sql b/tests/queries/0_stateless/02911_support_alias_column_in_indices.sql
index 4d68786d7db..5ab50044e64 100644
--- a/tests/queries/0_stateless/02911_support_alias_column_in_indices.sql
+++ b/tests/queries/0_stateless/02911_support_alias_column_in_indices.sql
@@ -17,8 +17,8 @@ settings index_granularity = 8192, min_index_granularity_bytes = 1024, index_gra
 insert into test1 select * from numbers(10);
 insert into test1 select * from numbers(11, 20);
 
-explain indexes = 1 select * from test1 where a > 10 settings allow_experimental_analyzer = 0;
-explain indexes = 1 select * from test1 where a > 10 settings allow_experimental_analyzer = 1;
+explain indexes = 1 select * from test1 where a > 10 settings enable_analyzer = 0;
+explain indexes = 1 select * from test1 where a > 10 settings enable_analyzer = 1;
 
 create table test2
 (
@@ -34,7 +34,7 @@ settings index_granularity = 8192, min_index_granularity_bytes = 1024, index_gra
 insert into test2 select * from numbers(10);
 insert into test2 select * from numbers(11, 20);
 
-explain indexes = 1 select * from test2 where a2 > 15 settings allow_experimental_analyzer = 0;
-explain indexes = 1 select * from test2 where a2 > 15 settings allow_experimental_analyzer = 1;
+explain indexes = 1 select * from test2 where a2 > 15 settings enable_analyzer = 0;
+explain indexes = 1 select * from test2 where a2 > 15 settings enable_analyzer = 1;
 
 drop database 02911_support_alias_column_in_indices;
diff --git a/tests/queries/0_stateless/02915_analyzer_fuzz_1.sql b/tests/queries/0_stateless/02915_analyzer_fuzz_1.sql
index 94849453063..f1d606ab1cd 100644
--- a/tests/queries/0_stateless/02915_analyzer_fuzz_1.sql
+++ b/tests/queries/0_stateless/02915_analyzer_fuzz_1.sql
@@ -1,2 +1,2 @@
-set allow_experimental_analyzer=1;
+set enable_analyzer=1;
 SELECT concat('With ', materialize(_CAST('ba\0', 'LowCardinality(FixedString(3))'))) AS `concat('With ', materialize(CAST('ba\\0', 'LowCardinality(FixedString(3))')))` FROM system.one GROUP BY 'With ';
diff --git a/tests/queries/0_stateless/02915_analyzer_fuzz_2.sql b/tests/queries/0_stateless/02915_analyzer_fuzz_2.sql
index ca9fff68446..8921d36c546 100644
--- a/tests/queries/0_stateless/02915_analyzer_fuzz_2.sql
+++ b/tests/queries/0_stateless/02915_analyzer_fuzz_2.sql
@@ -1,5 +1,4 @@
 SET aggregate_functions_null_for_empty = 1;
---set allow_experimental_analyzer=1;
+--set enable_analyzer=1;
 create table t_delete_projection (x UInt32, y UInt64, projection p (select sum(y))) engine = MergeTree order by tuple() SETTINGS index_granularity = 8192, index_granularity_bytes = '10Mi';
 insert into t_delete_projection select number, toString(number) from numbers(8192 * 10);
-
diff --git a/tests/queries/0_stateless/02915_analyzer_fuzz_5.sql b/tests/queries/0_stateless/02915_analyzer_fuzz_5.sql
index 29d06d2c315..d75d4f4eb72 100644
--- a/tests/queries/0_stateless/02915_analyzer_fuzz_5.sql
+++ b/tests/queries/0_stateless/02915_analyzer_fuzz_5.sql
@@ -1,4 +1,4 @@
-set allow_experimental_analyzer=1;
+set enable_analyzer=1;
 SET max_block_size = 1000;
 SET max_threads = 4;
 SET max_rows_to_group_by = 3000, group_by_overflow_mode = 'any';
diff --git a/tests/queries/0_stateless/02915_analyzer_fuzz_6.sql b/tests/queries/0_stateless/02915_analyzer_fuzz_6.sql
index b4eb1b4aff4..cc276ec4074 100644
--- a/tests/queries/0_stateless/02915_analyzer_fuzz_6.sql
+++ b/tests/queries/0_stateless/02915_analyzer_fuzz_6.sql
@@ -1,5 +1,5 @@
 set allow_suspicious_low_cardinality_types=1;
-set allow_experimental_analyzer=1;
+set enable_analyzer=1;
 
 create table tab (x LowCardinality(Nullable(Float64))) engine = MergeTree order by x settings allow_nullable_key=1;
 insert into tab select number from numbers(2);
diff --git a/tests/queries/0_stateless/02918_join_pm_lc_crash.sql b/tests/queries/0_stateless/02918_join_pm_lc_crash.sql
index 123208ee981..0326acff4a6 100644
--- a/tests/queries/0_stateless/02918_join_pm_lc_crash.sql
+++ b/tests/queries/0_stateless/02918_join_pm_lc_crash.sql
@@ -1,31 +1,30 @@
 
 SET joined_subquery_requires_alias = 0, join_algorithm = 'partial_merge';
 
-SET allow_experimental_analyzer = 0, join_use_nulls = 0;
+SET enable_analyzer = 0, join_use_nulls = 0;
 
 SELECT * FROM (SELECT dummy AS val FROM system.one)
 JOIN (SELECT toLowCardinality(toNullable(dummy)) AS val
 FROM system.one GROUP BY val WITH TOTALS)
 USING (val);
 
-SET allow_experimental_analyzer = 0, join_use_nulls = 1;
+SET enable_analyzer = 0, join_use_nulls = 1;
 
 SELECT * FROM (SELECT dummy AS val FROM system.one)
 JOIN (SELECT toLowCardinality(toNullable(dummy)) AS val
 FROM system.one GROUP BY val WITH TOTALS)
 USING (val);
 
-SET allow_experimental_analyzer = 1, join_use_nulls = 0;
+SET enable_analyzer = 1, join_use_nulls = 0;
 
 SELECT * FROM (SELECT dummy AS val FROM system.one)
 JOIN (SELECT toLowCardinality(toNullable(dummy)) AS val
 FROM system.one GROUP BY val WITH TOTALS)
 USING (val);
 
-SET allow_experimental_analyzer = 1, join_use_nulls = 1;
+SET enable_analyzer = 1, join_use_nulls = 1;
 
 SELECT * FROM (SELECT dummy AS val FROM system.one)
 JOIN (SELECT toLowCardinality(toNullable(dummy)) AS val
 FROM system.one GROUP BY val WITH TOTALS)
 USING (val);
-
diff --git a/tests/queries/0_stateless/02918_optimize_count_for_merge_tables.sql b/tests/queries/0_stateless/02918_optimize_count_for_merge_tables.sql
index 9feb2aa2ad6..f5647f01ab9 100644
--- a/tests/queries/0_stateless/02918_optimize_count_for_merge_tables.sql
+++ b/tests/queries/0_stateless/02918_optimize_count_for_merge_tables.sql
@@ -15,7 +15,7 @@ INSERT INTO mt2 VALUES (1);
 SELECT count() FROM merge;
 
 -- can use the trivial count optimization
-EXPLAIN SELECT count() FROM merge settings allow_experimental_analyzer=0;
+EXPLAIN SELECT count() FROM merge settings enable_analyzer=0;
 
 CREATE TABLE mt3 (id UInt64) ENGINE = TinyLog;
 
@@ -24,7 +24,7 @@ INSERT INTO mt2 VALUES (2);
 SELECT count() FROM merge;
 
 -- can't use the trivial count optimization as TinyLog doesn't support it
-EXPLAIN SELECT count() FROM merge settings allow_experimental_analyzer=0;
+EXPLAIN SELECT count() FROM merge settings enable_analyzer=0;
 
 DROP TABLE IF EXISTS mt1;
 DROP TABLE IF EXISTS mt2;
diff --git a/tests/queries/0_stateless/02922_respect_nulls_parser.sql b/tests/queries/0_stateless/02922_respect_nulls_parser.sql
index c9a17fdfbfe..ccd67dbe676 100644
--- a/tests/queries/0_stateless/02922_respect_nulls_parser.sql
+++ b/tests/queries/0_stateless/02922_respect_nulls_parser.sql
@@ -13,7 +13,7 @@ SELECT toDateTimeNonExistingFunction(now()) RESPECT NULLS b; -- { serverError UN
 SELECT toDateTime(now()) RESPECT NULLS b; -- { serverError SYNTAX_ERROR }
 SELECT count() from numbers(10) where in(number, (0)) RESPECT NULLS; -- { serverError SYNTAX_ERROR }
 SELECT if(number > 0, number, 0) respect nulls from numbers(0); -- { serverError SYNTAX_ERROR }
-WITH (x -> x + 1) AS lambda SELECT lambda(number) RESPECT NULLS FROM numbers(10) SETTINGS allow_experimental_analyzer = 1; -- { serverError SYNTAX_ERROR }
+WITH (x -> x + 1) AS lambda SELECT lambda(number) RESPECT NULLS FROM numbers(10) SETTINGS enable_analyzer = 1; -- { serverError SYNTAX_ERROR }
 SELECT * from system.one WHERE indexHint(dummy = 1) RESPECT NULLS; -- { serverError SYNTAX_ERROR }
 SELECT arrayJoin([[3,4,5], [6,7], [2], [1,1]]) IGNORE NULLS; -- { serverError SYNTAX_ERROR }
 SELECT number, grouping(number % 2, number) RESPECT NULLS AS gr FROM numbers(10) GROUP BY GROUPING SETS ((number), (number % 2)) SETTINGS force_grouping_standard_compatibility = 0; -- { serverError SYNTAX_ERROR }
diff --git a/tests/queries/0_stateless/02932_analyzer_rewrite_sum_column_and_constant.sql b/tests/queries/0_stateless/02932_analyzer_rewrite_sum_column_and_constant.sql
index 43dad8eb8e0..5492d061c12 100644
--- a/tests/queries/0_stateless/02932_analyzer_rewrite_sum_column_and_constant.sql
+++ b/tests/queries/0_stateless/02932_analyzer_rewrite_sum_column_and_constant.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 
 -- { echoOn }
 Select sum(number + 1) from numbers(10);
diff --git a/tests/queries/0_stateless/02932_parallel_replicas_fuzzer.sql b/tests/queries/0_stateless/02932_parallel_replicas_fuzzer.sql
index 3daaf36188a..e5cbe100a58 100644
--- a/tests/queries/0_stateless/02932_parallel_replicas_fuzzer.sql
+++ b/tests/queries/0_stateless/02932_parallel_replicas_fuzzer.sql
@@ -35,4 +35,4 @@ ORDER BY
     nan DESC,
     _CAST([0, NULL, NULL, NULL, NULL], 'Array(Nullable(UInt8))') DESC
 FORMAT Null
-SETTINGS receive_timeout = 10., receive_data_timeout_ms = 10000, use_hedged_requests = 0, allow_suspicious_low_cardinality_types = 1, max_parallel_replicas = 3, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_parallel_reading_from_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, log_queries = 1, table_function_remote_max_addresses = 200, allow_experimental_analyzer = 1;
+SETTINGS receive_timeout = 10., receive_data_timeout_ms = 10000, use_hedged_requests = 0, allow_suspicious_low_cardinality_types = 1, max_parallel_replicas = 3, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_parallel_reading_from_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, log_queries = 1, table_function_remote_max_addresses = 200, enable_analyzer = 1;
diff --git a/tests/queries/0_stateless/02933_paste_join.sql b/tests/queries/0_stateless/02933_paste_join.sql
index 604078d1c3a..6c5a923d028 100644
--- a/tests/queries/0_stateless/02933_paste_join.sql
+++ b/tests/queries/0_stateless/02933_paste_join.sql
@@ -41,10 +41,10 @@ INSERT INTO test SELECT number from numbers(6);
 SELECT * FROM (SELECT number FROM test) PASTE JOIN (SELECT number FROM numbers(6) ORDER BY number) SETTINGS joined_subquery_requires_alias = 0;
 SELECT * FROM (SELECT number FROM test PASTE JOIN (Select number FROM numbers(7))) PASTE JOIN (SELECT number FROM numbers(6) PASTE JOIN (SELECT number FROM test)) SETTINGS joined_subquery_requires_alias = 0;
 SELECT * FROM (SELECT number FROM test PASTE JOIN (SELECT number FROM test PASTE JOIN (Select number FROM numbers(7)))) PASTE JOIN (SELECT number FROM numbers(6) PASTE JOIN (SELECT number FROM test)) SETTINGS joined_subquery_requires_alias = 0;
-SELECT * FROM (SELECT 1 AS a) PASTE JOIN (SELECT 2 AS b) PASTE JOIN (SELECT 3 AS c) SETTINGS allow_experimental_analyzer = 1;
-SELECT * FROM (SELECT 1 AS a) PASTE JOIN (SELECT 2 AS b) PASTE JOIN (SELECT 3 AS a) SETTINGS allow_experimental_analyzer = 1; -- { serverError AMBIGUOUS_COLUMN_NAME }
+SELECT * FROM (SELECT 1 AS a) PASTE JOIN (SELECT 2 AS b) PASTE JOIN (SELECT 3 AS c) SETTINGS enable_analyzer = 1;
+SELECT * FROM (SELECT 1 AS a) PASTE JOIN (SELECT 2 AS b) PASTE JOIN (SELECT 3 AS a) SETTINGS enable_analyzer = 1; -- { serverError AMBIGUOUS_COLUMN_NAME }
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 CREATE TABLE test1 (a Int32) engine=MergeTree order by a;
 INSERT INTO test1 SELECT * FROM numbers(2);
 CREATE TABLE test2 (a Int32) engine=MergeTree order by a;
diff --git a/tests/queries/0_stateless/02943_order_by_all.sql b/tests/queries/0_stateless/02943_order_by_all.sql
index 0d5e0ea52e4..4ce59e84e6c 100644
--- a/tests/queries/0_stateless/02943_order_by_all.sql
+++ b/tests/queries/0_stateless/02943_order_by_all.sql
@@ -13,40 +13,40 @@ INSERT INTO order_by_all VALUES ('B', 3), ('C', NULL), ('D', 1), ('A', 2);
 
 SELECT '-- no modifiers';
 
-SET allow_experimental_analyzer = 0;
+SET enable_analyzer = 0;
 SELECT a, b FROM order_by_all ORDER BY ALL;
 SELECT b, a FROM order_by_all ORDER BY ALL;
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 SELECT a, b FROM order_by_all ORDER BY ALL;
 SELECT b, a FROM order_by_all ORDER BY ALL;
 
 SELECT '-- with ASC/DESC modifiers';
 
-SET allow_experimental_analyzer = 0;
+SET enable_analyzer = 0;
 SELECT a, b FROM order_by_all ORDER BY ALL ASC;
 SELECT a, b FROM order_by_all ORDER BY ALL DESC;
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 SELECT a, b FROM order_by_all ORDER BY ALL ASC;
 SELECT a, b FROM order_by_all ORDER BY ALL DESC;
 
 SELECT '-- with NULLS FIRST/LAST modifiers';
 
-SET allow_experimental_analyzer = 0;
+SET enable_analyzer = 0;
 SELECT b, a FROM order_by_all ORDER BY ALL NULLS FIRST;
 SELECT b, a FROM order_by_all ORDER BY ALL NULLS LAST;
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 SELECT b, a FROM order_by_all ORDER BY ALL NULLS FIRST;
 SELECT b, a FROM order_by_all ORDER BY ALL NULLS LAST;
 
 SELECT '-- SELECT *';
 
-SET allow_experimental_analyzer = 0;
+SET enable_analyzer = 0;
 SELECT * FROM order_by_all ORDER BY all;
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 SELECT * FROM order_by_all ORDER BY all;
 
 DROP TABLE order_by_all;
@@ -65,7 +65,7 @@ INSERT INTO order_by_all VALUES ('B', 3, 10), ('C', NULL, 40), ('D', 1, 20), ('A
 
 SELECT '  -- columns';
 
-SET allow_experimental_analyzer = 0;
+SET enable_analyzer = 0;
 SELECT a, b, all FROM order_by_all ORDER BY all;  -- { serverError UNEXPECTED_EXPRESSION }
 SELECT a, b, all FROM order_by_all ORDER BY all SETTINGS enable_order_by_all = false;
 SELECT a FROM order_by_all ORDER BY all;  -- { serverError UNEXPECTED_EXPRESSION }
@@ -73,7 +73,7 @@ SELECT a FROM order_by_all ORDER BY all SETTINGS enable_order_by_all = false;
 SELECT * FROM order_by_all ORDER BY all;  -- { serverError UNEXPECTED_EXPRESSION }
 SELECT * FROM order_by_all ORDER BY all SETTINGS enable_order_by_all = false;
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 SELECT a, b, all FROM order_by_all ORDER BY all;  -- { serverError UNEXPECTED_EXPRESSION }
 SELECT a, b, all FROM order_by_all ORDER BY all SETTINGS enable_order_by_all = false;
 SELECT a FROM order_by_all ORDER BY all SETTINGS enable_order_by_all = false;
@@ -86,30 +86,30 @@ SELECT * FROM order_by_all ORDER BY all SETTINGS enable_order_by_all = false;
 
 SELECT '  -- column aliases';
 
-SET allow_experimental_analyzer = 0;
+SET enable_analyzer = 0;
 SELECT a, b AS all FROM order_by_all ORDER BY all;  -- { serverError UNEXPECTED_EXPRESSION }
 SELECT a, b AS all FROM order_by_all ORDER BY all SETTINGS enable_order_by_all = false;
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 SELECT a, b AS all FROM order_by_all ORDER BY all;  -- { serverError UNEXPECTED_EXPRESSION }
 SELECT a, b AS all FROM order_by_all ORDER BY all SETTINGS enable_order_by_all = false;
 
 SELECT '  -- expressions';
 
-SET allow_experimental_analyzer = 0;
+SET enable_analyzer = 0;
 SELECT format('{} {}', a, b) AS all FROM order_by_all ORDER BY all;  -- { serverError UNEXPECTED_EXPRESSION }
 SELECT format('{} {}', a, b) AS all FROM order_by_all ORDER BY all SETTINGS enable_order_by_all = false;
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 SELECT format('{} {}', a, b) AS all FROM order_by_all ORDER BY all;  -- { serverError UNEXPECTED_EXPRESSION }
 SELECT format('{} {}', a, b) AS all FROM order_by_all ORDER BY all SETTINGS enable_order_by_all = false;
 
 SELECT '  -- ORDER BY ALL loses its special meaning when used in conjunction with other columns';
 
-SET allow_experimental_analyzer = 0;
+SET enable_analyzer = 0;
 SELECT a, b, all FROM order_by_all ORDER BY all, a;
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 SELECT a, b, all FROM order_by_all ORDER BY all, a;
 
 DROP TABLE order_by_all;
diff --git a/tests/queries/0_stateless/02943_tokenbf_and_ngrambf_indexes_support_match_function.sql b/tests/queries/0_stateless/02943_tokenbf_and_ngrambf_indexes_support_match_function.sql
index 42175cbb2c6..5ad54c872fa 100644
--- a/tests/queries/0_stateless/02943_tokenbf_and_ngrambf_indexes_support_match_function.sql
+++ b/tests/queries/0_stateless/02943_tokenbf_and_ngrambf_indexes_support_match_function.sql
@@ -41,7 +41,7 @@ FROM
 WHERE
     explain LIKE '%Granules: %'
 SETTINGS
-  allow_experimental_analyzer = 0;
+  enable_analyzer = 0;
 
 SELECT *
 FROM
@@ -52,7 +52,7 @@ FROM
 WHERE
     explain LIKE '%Granules: %'
 SETTINGS
-  allow_experimental_analyzer = 1;
+  enable_analyzer = 1;
 
 SELECT *
 FROM
@@ -63,7 +63,7 @@ FROM
 WHERE
     explain LIKE '%Granules: %'
 SETTINGS
-  allow_experimental_analyzer = 0;
+  enable_analyzer = 0;
 
 SELECT *
 FROM
@@ -74,7 +74,7 @@ FROM
 WHERE
     explain LIKE '%Granules: %'
 SETTINGS
-  allow_experimental_analyzer = 1;
+  enable_analyzer = 1;
 
 
 SELECT '---';
@@ -96,7 +96,7 @@ FROM
 WHERE
     explain LIKE '%Granules: %'
 SETTINGS
-  allow_experimental_analyzer = 0;
+  enable_analyzer = 0;
 
 SELECT *
 FROM
@@ -107,7 +107,7 @@ FROM
 WHERE
     explain LIKE '%Granules: %'
 SETTINGS
-  allow_experimental_analyzer = 1;
+  enable_analyzer = 1;
 
 SELECT *
 FROM
@@ -118,7 +118,7 @@ FROM
 WHERE
     explain LIKE '%Granules: %'
 SETTINGS
-  allow_experimental_analyzer = 0;
+  enable_analyzer = 0;
 
 SELECT *
 FROM
@@ -129,7 +129,7 @@ FROM
 WHERE
     explain LIKE '%Granules: %'
 SETTINGS
-  allow_experimental_analyzer = 1;
+  enable_analyzer = 1;
 
 SELECT '---';
 
@@ -150,7 +150,7 @@ FROM
 WHERE
     explain LIKE '%Granules: %'
 SETTINGS
-  allow_experimental_analyzer = 0;
+  enable_analyzer = 0;
 SELECT *
 FROM
 (
@@ -160,7 +160,7 @@ FROM
 WHERE
     explain LIKE '%Granules: %'
 SETTINGS
-  allow_experimental_analyzer = 1;
+  enable_analyzer = 1;
 
 SELECT *
 FROM
@@ -171,7 +171,7 @@ FROM
 WHERE
     explain LIKE '%Granules: %'
 SETTINGS
-  allow_experimental_analyzer = 0;
+  enable_analyzer = 0;
 
 SELECT *
 FROM
@@ -182,7 +182,7 @@ FROM
 WHERE
     explain LIKE '%Granules: %'
 SETTINGS
-  allow_experimental_analyzer = 1;
+  enable_analyzer = 1;
 
 DROP TABLE tokenbf_tab;
 DROP TABLE ngrambf_tab;
diff --git a/tests/queries/0_stateless/02944_variant_as_common_type_analyzer.sql b/tests/queries/0_stateless/02944_variant_as_common_type_analyzer.sql
index 77bed6cf796..7947c8a0ce1 100644
--- a/tests/queries/0_stateless/02944_variant_as_common_type_analyzer.sql
+++ b/tests/queries/0_stateless/02944_variant_as_common_type_analyzer.sql
@@ -1,7 +1,7 @@
 -- this test is just like 02944_variant_as_common_type, but with different expected output, because
 -- analyzer changes some return types. Specifically, if(c, x, y) always casts to the common type of
 -- x and y, even if c is constant.
-set allow_experimental_analyzer=1;
+set enable_analyzer=1;
 
 set allow_experimental_variant_type=1;
 set use_variant_as_common_type=1;
@@ -76,4 +76,3 @@ select toTypeName(res), array([1, 2, 3], [[1, 2, 3]]) as res;
 select toTypeName(res), map('a', 1, 'b', 'str_1') as res;
 select toTypeName(res), map('a', 1, 'b', map('c', 2, 'd', 'str_1')) as res;
 select toTypeName(res), map('a', 1, 'b', [1, 2, 3], 'c', [[4, 5, 6]]) as res;
-
diff --git a/tests/queries/0_stateless/02952_conjunction_optimization.sql b/tests/queries/0_stateless/02952_conjunction_optimization.sql
index 94bc352e4c5..fb6c26a2e1f 100644
--- a/tests/queries/0_stateless/02952_conjunction_optimization.sql
+++ b/tests/queries/0_stateless/02952_conjunction_optimization.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS 02952_disjunction_optimization;
 
diff --git a/tests/queries/0_stateless/02954_analyzer_fuzz_i57086.sql b/tests/queries/0_stateless/02954_analyzer_fuzz_i57086.sql
index a8029fdd3d6..886944e30c0 100644
--- a/tests/queries/0_stateless/02954_analyzer_fuzz_i57086.sql
+++ b/tests/queries/0_stateless/02954_analyzer_fuzz_i57086.sql
@@ -12,4 +12,4 @@ WITH ROLLUP
 ORDER BY
     count() ASC,
     number DESC NULLS LAST
-    SETTINGS limit = 2, allow_experimental_analyzer = 1;
+    SETTINGS limit = 2, enable_analyzer = 1;
diff --git a/tests/queries/0_stateless/02955_analyzer_using_functional_args.reference.j2 b/tests/queries/0_stateless/02955_analyzer_using_functional_args.reference.j2
index e0c6a439112..95a79c56194 100644
--- a/tests/queries/0_stateless/02955_analyzer_using_functional_args.reference.j2
+++ b/tests/queries/0_stateless/02955_analyzer_using_functional_args.reference.j2
@@ -10,7 +10,7 @@
 a
 -- { echoOn }
 -- USING alias column contains default in old analyzer (but both queries below should have the same result)
-SELECT y * 2, s || 'a' FROM t1 FULL JOIN t2 USING (y) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1;
+SELECT y * 2, s || 'a' FROM t1 FULL JOIN t2 USING (y) ORDER BY ALL SETTINGS enable_analyzer = 1;
 738	ba
 7386	aa
 13332	a
@@ -18,11 +18,11 @@ SELECT y * 2, s || 'a' FROM (SELECT s, y FROM t1) t1 FULL JOIN (SELECT y FROM t2
 738	ba
 7386	aa
 13332	a
-SELECT (1, *) FROM t1 FULL JOIN t2 USING (y) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1;
+SELECT (1, *) FROM t1 FULL JOIN t2 USING (y) ORDER BY ALL SETTINGS enable_analyzer = 1;
 (1,369,123,'b',124)
 (1,3693,1231,'a',0)
 (1,6666,0,'',48)
-SELECT (1, *) FROM (SELECT s, y FROM t1) t1 FULL JOIN (SELECT y FROM t2) t2 USING (y) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1;
+SELECT (1, *) FROM (SELECT s, y FROM t1) t1 FULL JOIN (SELECT y FROM t2) t2 USING (y) ORDER BY ALL SETTINGS enable_analyzer = 1;
 (1,369,'b')
 (1,3693,'a')
 (1,6666,'')
@@ -42,27 +42,27 @@ SELECT (1, t1.*, t2.*) FROM (SELECT s, y FROM t1) t1 FULL JOIN (SELECT y FROM t2
 (1,'',0,6666)
 (1,'a',3693,0)
 (1,'b',369,369)
-SELECT t1.z, t2.z, t3.z FROM t1 FULL JOIN t2 USING (y) FULL JOIN t3 USING (y) ORDER BY 1,2,3 SETTINGS allow_experimental_analyzer = 1;
+SELECT t1.z, t2.z, t3.z FROM t1 FULL JOIN t2 USING (y) FULL JOIN t3 USING (y) ORDER BY 1,2,3 SETTINGS enable_analyzer = 1;
 0	0	43
 0	48	0
 124	124	0
 1232	0	1232
-SELECT * FROM t1 FULL JOIN t2 USING (y) FULL JOIN t3 USING (y) ORDER BY 1,2,3 SETTINGS allow_experimental_analyzer = 1;
+SELECT * FROM t1 FULL JOIN t2 USING (y) FULL JOIN t3 USING (y) ORDER BY 1,2,3 SETTINGS enable_analyzer = 1;
 126	0		0	42
 369	123	b	124	0
 3693	1231	a	0	1231
 6666	0		48	0
-SELECT t1.*, t2.*, t3.* FROM t1 FULL JOIN t2 USING (y) FULL JOIN t3 USING (y) ORDER BY 1,2,3 SETTINGS allow_experimental_analyzer = 1;
+SELECT t1.*, t2.*, t3.* FROM t1 FULL JOIN t2 USING (y) FULL JOIN t3 USING (y) ORDER BY 1,2,3 SETTINGS enable_analyzer = 1;
 0		126	0	42
 0		6666	48	0
 123	b	369	124	0
 1231	a	3693	0	1231
-SELECT (1, t1.*, t2.*, t3.*) FROM t1 FULL JOIN t2 USING (y) FULL JOIN t3 USING (y) ORDER BY 1 SETTINGS allow_experimental_analyzer = 1;
+SELECT (1, t1.*, t2.*, t3.*) FROM t1 FULL JOIN t2 USING (y) FULL JOIN t3 USING (y) ORDER BY 1 SETTINGS enable_analyzer = 1;
 (1,0,'',126,0,42)
 (1,0,'',6666,48,0)
 (1,123,'b',369,124,0)
 (1,1231,'a',3693,0,1231)
-SELECT y FROM t1 FULL JOIN t2 USING (y) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1;
+SELECT y FROM t1 FULL JOIN t2 USING (y) ORDER BY ALL SETTINGS enable_analyzer = 1;
 369
 3693
 6666
@@ -78,11 +78,11 @@ SELECT s FROM (SELECT s, y FROM t1) t1 FULL JOIN (SELECT y FROM t2) t2 USING (y)
 
 a
 b
-SELECT y FROM t1 FULL JOIN t2 USING (y) PREWHERE y * 2 > 2 ORDER BY ALL SETTINGS allow_experimental_analyzer = 1, join_use_nulls = 0;
+SELECT y FROM t1 FULL JOIN t2 USING (y) PREWHERE y * 2 > 2 ORDER BY ALL SETTINGS enable_analyzer = 1, join_use_nulls = 0;
 369
 3693
 6666
-SELECT y FROM t1 FULL JOIN t2 USING (y) PREWHERE y * 2 > 2 ORDER BY ALL SETTINGS allow_experimental_analyzer = 1, join_use_nulls = 1;
+SELECT y FROM t1 FULL JOIN t2 USING (y) PREWHERE y * 2 > 2 ORDER BY ALL SETTINGS enable_analyzer = 1, join_use_nulls = 1;
 369
 3693
 6666
diff --git a/tests/queries/0_stateless/02955_analyzer_using_functional_args.sql.j2 b/tests/queries/0_stateless/02955_analyzer_using_functional_args.sql.j2
index f5b81231afe..1dde83149b6 100644
--- a/tests/queries/0_stateless/02955_analyzer_using_functional_args.sql.j2
+++ b/tests/queries/0_stateless/02955_analyzer_using_functional_args.sql.j2
@@ -24,21 +24,21 @@ SELECT count() FROM t1 INNER JOIN t2 USING (y);
 SELECT count() FROM t2 INNER JOIN t1 USING (y);
 
 -- `SELECT *` works differently for ALIAS columns with analyzer
-SELECT * FROM t1 INNER JOIN t2 USING (y, z) SETTINGS allow_experimental_analyzer = 1;
-SELECT * FROM t2 INNER JOIN t1 USING (y, z) SETTINGS allow_experimental_analyzer = 1;
+SELECT * FROM t1 INNER JOIN t2 USING (y, z) SETTINGS enable_analyzer = 1;
+SELECT * FROM t2 INNER JOIN t1 USING (y, z) SETTINGS enable_analyzer = 1;
 SELECT t2.z FROM t1 INNER JOIN t2 USING (y);
 
-SELECT * FROM t1 INNER JOIN t3 USING (y) SETTINGS allow_experimental_analyzer = 1;
-SELECT * FROM t3 INNER JOIN t1 USING (y, z) SETTINGS allow_experimental_analyzer = 1;
+SELECT * FROM t1 INNER JOIN t3 USING (y) SETTINGS enable_analyzer = 1;
+SELECT * FROM t3 INNER JOIN t1 USING (y, z) SETTINGS enable_analyzer = 1;
 SELECT s FROM t1 INNER JOIN t3 USING (y);
 
 -- { echoOn }
 -- USING alias column contains default in old analyzer (but both queries below should have the same result)
-SELECT y * 2, s || 'a' FROM t1 FULL JOIN t2 USING (y) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1;
+SELECT y * 2, s || 'a' FROM t1 FULL JOIN t2 USING (y) ORDER BY ALL SETTINGS enable_analyzer = 1;
 SELECT y * 2, s || 'a' FROM (SELECT s, y FROM t1) t1 FULL JOIN (SELECT y FROM t2) t2 USING (y) ORDER BY ALL;
 
-SELECT (1, *) FROM t1 FULL JOIN t2 USING (y) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1;
-SELECT (1, *) FROM (SELECT s, y FROM t1) t1 FULL JOIN (SELECT y FROM t2) t2 USING (y) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1;
+SELECT (1, *) FROM t1 FULL JOIN t2 USING (y) ORDER BY ALL SETTINGS enable_analyzer = 1;
+SELECT (1, *) FROM (SELECT s, y FROM t1) t1 FULL JOIN (SELECT y FROM t2) t2 USING (y) ORDER BY ALL SETTINGS enable_analyzer = 1;
 
 SELECT (1, t1.*) FROM t1 FULL JOIN t2 USING (y) ORDER BY ALL;
 SELECT (1, t1.*) FROM (SELECT s, y FROM t1) t1 FULL JOIN (SELECT y FROM t2) t2 USING (y) ORDER BY ALL;
@@ -46,19 +46,19 @@ SELECT (1, t1.*) FROM (SELECT s, y FROM t1) t1 FULL JOIN (SELECT y FROM t2) t2 U
 SELECT (1, t1.*, t2.*) FROM t1 FULL JOIN t2 USING (y) ORDER BY ALL;
 SELECT (1, t1.*, t2.*) FROM (SELECT s, y FROM t1) t1 FULL JOIN (SELECT y FROM t2) t2 USING (y) ORDER BY ALL;
 
-SELECT t1.z, t2.z, t3.z FROM t1 FULL JOIN t2 USING (y) FULL JOIN t3 USING (y) ORDER BY 1,2,3 SETTINGS allow_experimental_analyzer = 1;
-SELECT * FROM t1 FULL JOIN t2 USING (y) FULL JOIN t3 USING (y) ORDER BY 1,2,3 SETTINGS allow_experimental_analyzer = 1;
-SELECT t1.*, t2.*, t3.* FROM t1 FULL JOIN t2 USING (y) FULL JOIN t3 USING (y) ORDER BY 1,2,3 SETTINGS allow_experimental_analyzer = 1;
-SELECT (1, t1.*, t2.*, t3.*) FROM t1 FULL JOIN t2 USING (y) FULL JOIN t3 USING (y) ORDER BY 1 SETTINGS allow_experimental_analyzer = 1;
+SELECT t1.z, t2.z, t3.z FROM t1 FULL JOIN t2 USING (y) FULL JOIN t3 USING (y) ORDER BY 1,2,3 SETTINGS enable_analyzer = 1;
+SELECT * FROM t1 FULL JOIN t2 USING (y) FULL JOIN t3 USING (y) ORDER BY 1,2,3 SETTINGS enable_analyzer = 1;
+SELECT t1.*, t2.*, t3.* FROM t1 FULL JOIN t2 USING (y) FULL JOIN t3 USING (y) ORDER BY 1,2,3 SETTINGS enable_analyzer = 1;
+SELECT (1, t1.*, t2.*, t3.*) FROM t1 FULL JOIN t2 USING (y) FULL JOIN t3 USING (y) ORDER BY 1 SETTINGS enable_analyzer = 1;
 
-SELECT y FROM t1 FULL JOIN t2 USING (y) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1;
+SELECT y FROM t1 FULL JOIN t2 USING (y) ORDER BY ALL SETTINGS enable_analyzer = 1;
 SELECT y FROM (SELECT s, y FROM t1) t1 FULL JOIN (SELECT y FROM t2) t2 USING (y) ORDER BY ALL;
 
 SELECT s FROM t1 FULL JOIN t2 USING (y) ORDER BY ALL;
 SELECT s FROM (SELECT s, y FROM t1) t1 FULL JOIN (SELECT y FROM t2) t2 USING (y) ORDER BY ALL;
 
-SELECT y FROM t1 FULL JOIN t2 USING (y) PREWHERE y * 2 > 2 ORDER BY ALL SETTINGS allow_experimental_analyzer = 1, join_use_nulls = 0;
-SELECT y FROM t1 FULL JOIN t2 USING (y) PREWHERE y * 2 > 2 ORDER BY ALL SETTINGS allow_experimental_analyzer = 1, join_use_nulls = 1;
+SELECT y FROM t1 FULL JOIN t2 USING (y) PREWHERE y * 2 > 2 ORDER BY ALL SETTINGS enable_analyzer = 1, join_use_nulls = 0;
+SELECT y FROM t1 FULL JOIN t2 USING (y) PREWHERE y * 2 > 2 ORDER BY ALL SETTINGS enable_analyzer = 1, join_use_nulls = 1;
 
 DROP TABLE IF EXISTS t1;
 DROP TABLE IF EXISTS t2;
diff --git a/tests/queries/0_stateless/02955_sparkBar_alias_sparkbar.sql b/tests/queries/0_stateless/02955_sparkBar_alias_sparkbar.sql
index 98259fc8029..0f658379ff9 100644
--- a/tests/queries/0_stateless/02955_sparkBar_alias_sparkbar.sql
+++ b/tests/queries/0_stateless/02955_sparkBar_alias_sparkbar.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 DROP TABLE IF EXISTS spark_bar_test;
 
 CREATE TABLE spark_bar_test (`value` Int64, `event_date` Date) ENGINE = MergeTree ORDER BY event_date;
@@ -9,4 +9,3 @@ SELECT sparkbar(9)(event_date,cnt) FROM (SELECT sum(value) as cnt, event_date FR
 SELECT sparkBar(9)(event_date,cnt) FROM (SELECT sum(value) as cnt, event_date FROM spark_bar_test GROUP BY event_date);
 
 DROP TABLE IF EXISTS spark_bar_test;
-
diff --git a/tests/queries/0_stateless/02962_join_using_bug_57894.sql b/tests/queries/0_stateless/02962_join_using_bug_57894.sql
index 87aef8b1a71..96190241da5 100644
--- a/tests/queries/0_stateless/02962_join_using_bug_57894.sql
+++ b/tests/queries/0_stateless/02962_join_using_bug_57894.sql
@@ -9,7 +9,7 @@ CREATE TABLE r (`x` LowCardinality(Nullable(UInt32)), `s` Nullable(String)) ENGI
 INSERT INTO r SELECT number, toString(number) FROM numbers(2, 8);
 INSERT INTO r VALUES (NULL, NULL);
 
-SET allow_experimental_analyzer = 0;
+SET enable_analyzer = 0;
 
 SELECT x FROM t FULL JOIN r USING (x) ORDER BY ALL
 ;
@@ -21,7 +21,7 @@ SETTINGS join_algorithm = 'partial_merge';
 SELECT x FROM t FULL JOIN r USING (x) ORDER BY ALL
 SETTINGS join_algorithm = 'full_sorting_merge';
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 SELECT x FROM t FULL JOIN r USING (x) ORDER BY ALL
 ;
diff --git a/tests/queries/0_stateless/02967_analyzer_fuzz.sql b/tests/queries/0_stateless/02967_analyzer_fuzz.sql
index 7f2d9afcc71..dab6ec2af22 100644
--- a/tests/queries/0_stateless/02967_analyzer_fuzz.sql
+++ b/tests/queries/0_stateless/02967_analyzer_fuzz.sql
@@ -7,7 +7,7 @@ GROUP BY
     makeDateTime64(NULL, NULL, pow(NULL, '257') - '-1', '0.2147483647', 257),
     makeDateTime64(pow(pow(NULL, '21474836.46') - '0.0000065535', 1048577), '922337203685477580.6', NULL, NULL, pow(NULL, 1.0001) - 65536, NULL)
 WITH CUBE
-    SETTINGS allow_experimental_analyzer = 1;
+    SETTINGS enable_analyzer = 1;
 
 
 CREATE TABLE data_01223 (`key` Int) ENGINE = Memory;
@@ -17,4 +17,4 @@ SELECT count(round('92233720368547758.07', '-0.01', NULL, nan, '25.7', '-9223372
 FROM dist_01223
 WHERE round(NULL, 1025, 1.1754943508222875e-38, NULL)
 WITH TOTALS
-    SETTINGS allow_experimental_analyzer = 1;
+    SETTINGS enable_analyzer = 1;
diff --git a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_3.sh b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_3.sh
index e49a340ab67..e954cb0e78e 100755
--- a/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_3.sh
+++ b/tests/queries/0_stateless/02967_parallel_replicas_join_algo_and_analyzer_3.sh
@@ -26,7 +26,7 @@ select * from (select key, value from num_1) l
 inner join (select key, value from num_2 inner join
   (select number * 7 as key from numbers(1e5)) as nn on num_2.key = nn.key settings parallel_replicas_prefer_local_join=0) r
 on l.key = r.key order by l.key limit 10 offset 10000
-SETTINGS allow_experimental_analyzer=1,
+SETTINGS enable_analyzer=1,
 allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1,
 cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join=0"
 
@@ -35,7 +35,7 @@ select * from (select key, value from num_1) l
 inner join (select key, value from num_2 inner join
   (select number * 7 as key from numbers(1e5)) as nn on num_2.key = nn.key settings parallel_replicas_prefer_local_join=0) r
 on l.key = r.key order by l.key limit 10 offset 10000
-SETTINGS allow_experimental_analyzer=1, send_logs_level='trace',
+SETTINGS enable_analyzer=1, send_logs_level='trace',
 allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1,
 cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join=0" 2>&1 |
 grep "executeQuery\|<Debug>.*Coordinator: Coordination done" |
@@ -51,7 +51,7 @@ select * from (select key, value from num_1) l
 inner join (select key, value from num_2 inner join
   (select number * 7 as key from numbers(1e5)) as nn on num_2.key = nn.key settings parallel_replicas_prefer_local_join=1) r
 on l.key = r.key order by l.key limit 10 offset 10000
-SETTINGS allow_experimental_analyzer=1,
+SETTINGS enable_analyzer=1,
 allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1,
 cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join=0"
 
@@ -60,7 +60,7 @@ select * from (select key, value from num_1) l
 inner join (select key, value from num_2 inner join
   (select number * 7 as key from numbers(1e5)) as nn on num_2.key = nn.key settings parallel_replicas_prefer_local_join=1) r
 on l.key = r.key order by l.key limit 10 offset 10000
-SETTINGS allow_experimental_analyzer=1, send_logs_level='trace',
+SETTINGS enable_analyzer=1, send_logs_level='trace',
 allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1,
 cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_prefer_local_join=0" 2>&1 |
 grep "executeQuery\|<Debug>.*Coordinator: Coordination done" |
@@ -77,7 +77,7 @@ select * from (select key, value from num_1) l
 inner join (select key, value from num_2 inner join
   (select number * 7 as key from numbers(1e5)) as nn on num_2.key = nn.key settings join_algorithm='full_sorting_merge') r
 on l.key = r.key order by l.key limit 10 offset 10000
-SETTINGS allow_experimental_analyzer=1, parallel_replicas_prefer_local_join=0,
+SETTINGS enable_analyzer=1, parallel_replicas_prefer_local_join=0,
 allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1,
 cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', join_algorithm='full_sorting_merge'"
 
@@ -86,7 +86,7 @@ select * from (select key, value from num_1) l
 inner join (select key, value from num_2 inner join
   (select number * 7 as key from numbers(1e5)) as nn on num_2.key = nn.key settings join_algorithm='full_sorting_merge') r
 on l.key = r.key order by l.key limit 10 offset 10000
-SETTINGS allow_experimental_analyzer=1, parallel_replicas_prefer_local_join=0, send_logs_level='trace',
+SETTINGS enable_analyzer=1, parallel_replicas_prefer_local_join=0, send_logs_level='trace',
 allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1,
 cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', join_algorithm='full_sorting_merge'" 2>&1 |
 grep "executeQuery\|<Debug>.*Coordinator: Coordination done" |
@@ -102,7 +102,7 @@ select * from (select key, value from num_1) l
 inner join (select key, value from num_2 inner join
   (select number * 7 as key from numbers(1e5)) as nn on num_2.key = nn.key settings join_algorithm='hash') r
 on l.key = r.key order by l.key limit 10 offset 10000
-SETTINGS allow_experimental_analyzer=1, parallel_replicas_prefer_local_join=0,
+SETTINGS enable_analyzer=1, parallel_replicas_prefer_local_join=0,
 allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1,
 cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', join_algorithm='full_sorting_merge'"
 
@@ -111,7 +111,7 @@ select * from (select key, value from num_1) l
 inner join (select key, value from num_2 inner join
   (select number * 7 as key from numbers(1e5)) as nn on num_2.key = nn.key settings join_algorithm='hash') r
 on l.key = r.key order by l.key limit 10 offset 10000
-SETTINGS allow_experimental_analyzer=1, parallel_replicas_prefer_local_join=0, send_logs_level='trace',
+SETTINGS enable_analyzer=1, parallel_replicas_prefer_local_join=0, send_logs_level='trace',
 allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1,
 cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', join_algorithm='full_sorting_merge'" 2>&1 |
 grep "executeQuery\|<Debug>.*Coordinator: Coordination done" |
@@ -127,7 +127,7 @@ select * from (select key, value from num_1) l
 inner join (select key, value from num_2 inner join
   (select number * 7 as key from numbers(1e5)) as nn on num_2.key = nn.key settings join_algorithm='full_sorting_merge') r
 on l.key = r.key order by l.key limit 10 offset 10000
-SETTINGS allow_experimental_analyzer=1, parallel_replicas_prefer_local_join=0,
+SETTINGS enable_analyzer=1, parallel_replicas_prefer_local_join=0,
 allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1,
 cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', join_algorithm='hash'"
 
@@ -136,7 +136,7 @@ select * from (select key, value from num_1) l
 inner join (select key, value from num_2 inner join
   (select number * 7 as key from numbers(1e5)) as nn on num_2.key = nn.key settings join_algorithm='full_sorting_merge') r
 on l.key = r.key order by l.key limit 10 offset 10000
-SETTINGS allow_experimental_analyzer=1, parallel_replicas_prefer_local_join=0, send_logs_level='trace',
+SETTINGS enable_analyzer=1, parallel_replicas_prefer_local_join=0, send_logs_level='trace',
 allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1,
 cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', join_algorithm='hash'" 2>&1 |
 grep "executeQuery\|<Debug>.*Coordinator: Coordination done" |
diff --git a/tests/queries/0_stateless/02967_parallel_replicas_joins_and_analyzer.reference b/tests/queries/0_stateless/02967_parallel_replicas_joins_and_analyzer.reference
index 100e4e500cd..36f02b2f764 100644
--- a/tests/queries/0_stateless/02967_parallel_replicas_joins_and_analyzer.reference
+++ b/tests/queries/0_stateless/02967_parallel_replicas_joins_and_analyzer.reference
@@ -2,7 +2,7 @@
 
 set parallel_replicas_prefer_local_join = 0;
 -- A query with only INNER/LEFT joins is fully send to replicas. JOIN is executed in GLOBAL mode.
-select x, y, r.y, z, rr.z, a from (select l.x, l.y, r.y, r.z as z from (select x, y from tab1 where x != 2) l any left join (select y, z from tab2 where y != 4) r on l.y = r.y) ll any left join (select z, a from tab3 where z != 8) rr on ll.z = rr.z order by x SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1;
+select x, y, r.y, z, rr.z, a from (select l.x, l.y, r.y, r.z as z from (select x, y from tab1 where x != 2) l any left join (select y, z from tab2 where y != 4) r on l.y = r.y) ll any left join (select z, a from tab3 where z != 8) rr on ll.z = rr.z order by x SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
 0	0	0	0	0	0
 1	1	0	0	0	0
 3	3	0	0	0	0
@@ -18,7 +18,7 @@ select x, y, r.y, z, rr.z, a from (select l.x, l.y, r.y, r.z as z from (select x
 13	13	0	0	0	0
 14	14	14	14	0	0
 15	15	0	0	0	0
-explain description=0 select x, y, r.y, z, rr.z, a from (select l.x, l.y, r.y, r.z as z from (select x, y from tab1 where x != 2) l any left join (select y, z from tab2 where y != 4) r on l.y = r.y) ll any left join (select z, a from tab3 where z != 8) rr on ll.z = rr.z SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1;
+explain description=0 select x, y, r.y, z, rr.z, a from (select l.x, l.y, r.y, r.z as z from (select x, y from tab1 where x != 2) l any left join (select y, z from tab2 where y != 4) r on l.y = r.y) ll any left join (select z, a from tab3 where z != 8) rr on ll.z = rr.z SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
 Expression
   ReadFromRemoteParallelReplicas
 --
@@ -29,7 +29,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
 sub4 as (select z, a from tab3 where z != 8),
 sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
 select * from sub5 order by x
-SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1;
+SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
 0	0	0	0	0	0
 1	1	0	0	0	0
 3	3	0	0	0	0
@@ -52,7 +52,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
 sub4 as (select z, a from tab3 where z != 8),
 sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
 select * from sub5 order by x
-SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1;
+SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
 Expression
   Sorting
     Expression
@@ -65,7 +65,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
 sub4 as (select z, a from tab3 where z != 8),
 sub5 as (select sum(x), sum(y), sum(r.y), sum(z), sum(rr.z), sum(a), key from sub3 ll any left join sub4 rr on ll.z = rr.z group by x % 2 as key)
 select * from sub5 order by key
-SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1;
+SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
 54	54	50	50	12	12	0
 64	64	0	0	0	0	1
 explain description=0
@@ -75,7 +75,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
 sub4 as (select z, a from tab3 where z != 8),
 sub5 as (select sum(x), sum(y), sum(r.y), sum(z), sum(rr.z), sum(a), key from sub3 ll any left join sub4 rr on ll.z = rr.z group by x % 2 as key)
 select * from sub5 order by key
-SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1;
+SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
 Expression
   Sorting
     Expression
@@ -90,7 +90,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
 sub4 as (select z, a from tab3 where z != 8),
 sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
 select * from sub5 order by x
-SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1;
+SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
 0	0	0	0	0	0
 1	1	0	0	0	0
 3	3	0	0	0	0
@@ -113,7 +113,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
 sub4 as (select z, a from tab3 where z != 8),
 sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
 select * from sub5 order by x
-SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1;
+SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
 Expression
   Sorting
     Expression
@@ -130,7 +130,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
 sub4 as (select z, a from tab3 where z != 8),
 sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
 select * from sub5 order by x
-SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1;
+SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
 0	0	0	0	0	0
 1	1	0	0	0	0
 3	3	0	0	0	0
@@ -153,7 +153,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
 sub4 as (select z, a from tab3 where z != 8),
 sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
 select * from sub5 order by x
-SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1;
+SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
 Expression
   Sorting
     Expression
@@ -174,7 +174,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub2 r any right join sub1 l on l.y
 sub4 as (select z, a from tab3 where z != 8),
 sub5 as (select x, l.y, y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
 select * from sub5
-SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1;
+SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
 0	0	0	0	0	0
 6	6	6	6	0	0
 8	8	8	8	0	0
@@ -197,7 +197,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub2 r any right join sub1 l on l.y
 sub4 as (select z, a from tab3 where z != 8),
 sub5 as (select x, l.y, y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
 select * from sub5
-SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1;
+SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
 Expression
   Join
     Expression
@@ -215,7 +215,7 @@ sub2 as (select y, z from tab2 where y != 4),
 sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y = r.y),
 sub4 as (select z, a from tab3 where z != 8),
 sub5 as (select z, a, x, y, r.y, ll.z from sub4 rr any right join sub3 ll on ll.z = rr.z)
-select * from sub5 order by x SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1;
+select * from sub5 order by x SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
 0	0	0	0	0	0
 0	0	1	1	0	0
 0	0	3	3	0	0
@@ -237,7 +237,7 @@ sub2 as (select y, z from tab2 where y != 4),
 sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y = r.y),
 sub4 as (select z, a from tab3 where z != 8),
 sub5 as (select z, a, x, y, r.y, ll.z from sub4 rr any right join sub3 ll on ll.z = rr.z)
-select * from sub5 order by x SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1;
+select * from sub5 order by x SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
 Expression
   Sorting
     Expression
@@ -258,7 +258,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
 sub4 as (select z, a from tab3 where z != 8),
 sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
 select * from sub5 order by x
-SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1;
+SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
 0	0	0	0	0	0
 1	1	0	0	0	0
 3	3	0	0	0	0
@@ -281,7 +281,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
 sub4 as (select z, a from tab3 where z != 8),
 sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
 select * from sub5 order by x
-SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1;
+SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
 Expression
   Sorting
     Expression
@@ -294,7 +294,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
 sub4 as (select z, a from tab3 where z != 8),
 sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
 select * from sub5 order by x
-SETTINGS allow_experimental_parallel_reading_from_replicas = 1, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1, parallel_replicas_allow_in_with_subquery=0;
+SETTINGS allow_experimental_parallel_reading_from_replicas = 1, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1, parallel_replicas_allow_in_with_subquery=0;
 0	0	0	0	0	0
 1	1	0	0	0	0
 3	3	0	0	0	0
@@ -317,7 +317,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
 sub4 as (select z, a from tab3 where z != 8),
 sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
 select * from sub5 order by x
-SETTINGS allow_experimental_parallel_reading_from_replicas = 1, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1, parallel_replicas_allow_in_with_subquery=0;-- { echoOn }
+SETTINGS allow_experimental_parallel_reading_from_replicas = 1, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1, parallel_replicas_allow_in_with_subquery=0;-- { echoOn }
 Expression
   Sorting
     Expression
@@ -339,7 +339,7 @@ Expression
           ReadFromRemoteParallelReplicas
 set parallel_replicas_prefer_local_join = 1;
 -- A query with only INNER/LEFT joins is fully send to replicas. JOIN is executed in GLOBAL mode.
-select x, y, r.y, z, rr.z, a from (select l.x, l.y, r.y, r.z as z from (select x, y from tab1 where x != 2) l any left join (select y, z from tab2 where y != 4) r on l.y = r.y) ll any left join (select z, a from tab3 where z != 8) rr on ll.z = rr.z order by x SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1;
+select x, y, r.y, z, rr.z, a from (select l.x, l.y, r.y, r.z as z from (select x, y from tab1 where x != 2) l any left join (select y, z from tab2 where y != 4) r on l.y = r.y) ll any left join (select z, a from tab3 where z != 8) rr on ll.z = rr.z order by x SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
 0	0	0	0	0	0
 1	1	0	0	0	0
 3	3	0	0	0	0
@@ -355,7 +355,7 @@ select x, y, r.y, z, rr.z, a from (select l.x, l.y, r.y, r.z as z from (select x
 13	13	0	0	0	0
 14	14	14	14	0	0
 15	15	0	0	0	0
-explain description=0 select x, y, r.y, z, rr.z, a from (select l.x, l.y, r.y, r.z as z from (select x, y from tab1 where x != 2) l any left join (select y, z from tab2 where y != 4) r on l.y = r.y) ll any left join (select z, a from tab3 where z != 8) rr on ll.z = rr.z SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1;
+explain description=0 select x, y, r.y, z, rr.z, a from (select l.x, l.y, r.y, r.z as z from (select x, y from tab1 where x != 2) l any left join (select y, z from tab2 where y != 4) r on l.y = r.y) ll any left join (select z, a from tab3 where z != 8) rr on ll.z = rr.z SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
 Expression
   ReadFromRemoteParallelReplicas
 --
@@ -366,7 +366,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
 sub4 as (select z, a from tab3 where z != 8),
 sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
 select * from sub5 order by x
-SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1;
+SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
 0	0	0	0	0	0
 1	1	0	0	0	0
 3	3	0	0	0	0
@@ -389,7 +389,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
 sub4 as (select z, a from tab3 where z != 8),
 sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
 select * from sub5 order by x
-SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1;
+SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
 Expression
   Sorting
     Expression
@@ -402,7 +402,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
 sub4 as (select z, a from tab3 where z != 8),
 sub5 as (select sum(x), sum(y), sum(r.y), sum(z), sum(rr.z), sum(a), key from sub3 ll any left join sub4 rr on ll.z = rr.z group by x % 2 as key)
 select * from sub5 order by key
-SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1;
+SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
 54	54	50	50	12	12	0
 64	64	0	0	0	0	1
 explain description=0
@@ -412,7 +412,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
 sub4 as (select z, a from tab3 where z != 8),
 sub5 as (select sum(x), sum(y), sum(r.y), sum(z), sum(rr.z), sum(a), key from sub3 ll any left join sub4 rr on ll.z = rr.z group by x % 2 as key)
 select * from sub5 order by key
-SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1;
+SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
 Expression
   Sorting
     Expression
@@ -427,7 +427,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
 sub4 as (select z, a from tab3 where z != 8),
 sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
 select * from sub5 order by x
-SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1;
+SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
 0	0	0	0	0	0
 1	1	0	0	0	0
 3	3	0	0	0	0
@@ -450,7 +450,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
 sub4 as (select z, a from tab3 where z != 8),
 sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
 select * from sub5 order by x
-SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1;
+SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
 Expression
   Sorting
     Expression
@@ -467,7 +467,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
 sub4 as (select z, a from tab3 where z != 8),
 sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
 select * from sub5 order by x
-SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1;
+SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
 0	0	0	0	0	0
 1	1	0	0	0	0
 3	3	0	0	0	0
@@ -490,7 +490,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
 sub4 as (select z, a from tab3 where z != 8),
 sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
 select * from sub5 order by x
-SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1;
+SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
 Expression
   Sorting
     Expression
@@ -511,7 +511,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub2 r any right join sub1 l on l.y
 sub4 as (select z, a from tab3 where z != 8),
 sub5 as (select x, l.y, y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
 select * from sub5
-SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1;
+SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
 0	0	0	0	0	0
 6	6	6	6	0	0
 8	8	8	8	0	0
@@ -534,7 +534,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub2 r any right join sub1 l on l.y
 sub4 as (select z, a from tab3 where z != 8),
 sub5 as (select x, l.y, y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
 select * from sub5
-SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1;
+SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
 Expression
   Join
     Expression
@@ -552,7 +552,7 @@ sub2 as (select y, z from tab2 where y != 4),
 sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y = r.y),
 sub4 as (select z, a from tab3 where z != 8),
 sub5 as (select z, a, x, y, r.y, ll.z from sub4 rr any right join sub3 ll on ll.z = rr.z)
-select * from sub5 order by x SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1;
+select * from sub5 order by x SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
 0	0	0	0	0	0
 0	0	1	1	0	0
 0	0	3	3	0	0
@@ -574,7 +574,7 @@ sub2 as (select y, z from tab2 where y != 4),
 sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y = r.y),
 sub4 as (select z, a from tab3 where z != 8),
 sub5 as (select z, a, x, y, r.y, ll.z from sub4 rr any right join sub3 ll on ll.z = rr.z)
-select * from sub5 order by x SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1;
+select * from sub5 order by x SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
 Expression
   Sorting
     Expression
@@ -595,7 +595,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
 sub4 as (select z, a from tab3 where z != 8),
 sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
 select * from sub5 order by x
-SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1;
+SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
 0	0	0	0	0	0
 1	1	0	0	0	0
 3	3	0	0	0	0
@@ -618,7 +618,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
 sub4 as (select z, a from tab3 where z != 8),
 sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
 select * from sub5 order by x
-SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1;
+SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
 Expression
   Sorting
     Expression
@@ -631,7 +631,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
 sub4 as (select z, a from tab3 where z != 8),
 sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
 select * from sub5 order by x
-SETTINGS allow_experimental_parallel_reading_from_replicas = 1, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1, parallel_replicas_allow_in_with_subquery=0;
+SETTINGS allow_experimental_parallel_reading_from_replicas = 1, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1, parallel_replicas_allow_in_with_subquery=0;
 0	0	0	0	0	0
 1	1	0	0	0	0
 3	3	0	0	0	0
@@ -654,7 +654,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
 sub4 as (select z, a from tab3 where z != 8),
 sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
 select * from sub5 order by x
-SETTINGS allow_experimental_parallel_reading_from_replicas = 1, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1, parallel_replicas_allow_in_with_subquery=0;
+SETTINGS allow_experimental_parallel_reading_from_replicas = 1, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1, parallel_replicas_allow_in_with_subquery=0;
 Expression
   Sorting
     Expression
diff --git a/tests/queries/0_stateless/02967_parallel_replicas_joins_and_analyzer.sql.j2 b/tests/queries/0_stateless/02967_parallel_replicas_joins_and_analyzer.sql.j2
index 54505b147a3..775663768fe 100644
--- a/tests/queries/0_stateless/02967_parallel_replicas_joins_and_analyzer.sql.j2
+++ b/tests/queries/0_stateless/02967_parallel_replicas_joins_and_analyzer.sql.j2
@@ -17,8 +17,8 @@ insert into tab3 select number * 4, number * 4 from numbers(4);
 set parallel_replicas_prefer_local_join = {{use_global_in}};
 
 -- A query with only INNER/LEFT joins is fully send to replicas. JOIN is executed in GLOBAL mode.
-select x, y, r.y, z, rr.z, a from (select l.x, l.y, r.y, r.z as z from (select x, y from tab1 where x != 2) l any left join (select y, z from tab2 where y != 4) r on l.y = r.y) ll any left join (select z, a from tab3 where z != 8) rr on ll.z = rr.z order by x SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1;
-explain description=0 select x, y, r.y, z, rr.z, a from (select l.x, l.y, r.y, r.z as z from (select x, y from tab1 where x != 2) l any left join (select y, z from tab2 where y != 4) r on l.y = r.y) ll any left join (select z, a from tab3 where z != 8) rr on ll.z = rr.z SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1;
+select x, y, r.y, z, rr.z, a from (select l.x, l.y, r.y, r.z as z from (select x, y from tab1 where x != 2) l any left join (select y, z from tab2 where y != 4) r on l.y = r.y) ll any left join (select z, a from tab3 where z != 8) rr on ll.z = rr.z order by x SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
+explain description=0 select x, y, r.y, z, rr.z, a from (select l.x, l.y, r.y, r.z as z from (select x, y from tab1 where x != 2) l any left join (select y, z from tab2 where y != 4) r on l.y = r.y) ll any left join (select z, a from tab3 where z != 8) rr on ll.z = rr.z SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
 --
 -- The same query with cte;
 with sub1 as (select x, y from tab1 where x != 2),
@@ -27,7 +27,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
 sub4 as (select z, a from tab3 where z != 8),
 sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
 select * from sub5 order by x
-SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1;
+SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
 
 explain description=0
 with sub1 as (select x, y from tab1 where x != 2),
@@ -36,7 +36,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
 sub4 as (select z, a from tab3 where z != 8),
 sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
 select * from sub5 order by x
-SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1;
+SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
 --
 -- GROUP BY should work up to WithMergableStage
 with sub1 as (select x, y from tab1 where x != 2),
@@ -45,7 +45,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
 sub4 as (select z, a from tab3 where z != 8),
 sub5 as (select sum(x), sum(y), sum(r.y), sum(z), sum(rr.z), sum(a), key from sub3 ll any left join sub4 rr on ll.z = rr.z group by x % 2 as key)
 select * from sub5 order by key
-SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1;
+SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
 
 explain description=0
 with sub1 as (select x, y from tab1 where x != 2),
@@ -54,7 +54,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
 sub4 as (select z, a from tab3 where z != 8),
 sub5 as (select sum(x), sum(y), sum(r.y), sum(z), sum(rr.z), sum(a), key from sub3 ll any left join sub4 rr on ll.z = rr.z group by x % 2 as key)
 select * from sub5 order by key
-SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1;
+SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
 --
 -- ORDER BY in sub3 : sub1 is fully pushed, sub3 -> WithMergableStage
 with sub1 as (select x, y from tab1 where x != 2),
@@ -63,7 +63,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
 sub4 as (select z, a from tab3 where z != 8),
 sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
 select * from sub5 order by x
-SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1;
+SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
 
 explain description=0
 with sub1 as (select x, y from tab1 where x != 2),
@@ -72,7 +72,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
 sub4 as (select z, a from tab3 where z != 8),
 sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
 select * from sub5 order by x
-SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1;
+SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
 --
 -- ORDER BY in sub1 : sub1 -> WithMergableStage
 with sub1 as (select x, y from tab1 where x != 2 order by y),
@@ -81,7 +81,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
 sub4 as (select z, a from tab3 where z != 8),
 sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
 select * from sub5 order by x
-SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1;
+SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
 
 explain description=0
 with sub1 as (select x, y from tab1 where x != 2 order by y),
@@ -90,7 +90,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
 sub4 as (select z, a from tab3 where z != 8),
 sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
 select * from sub5 order by x
-SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1;
+SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
 --
 -- RIGHT JOIN in sub3: sub3 -> WithMergableStage
 with sub1 as (select x, y from tab1 where x != 2),
@@ -99,7 +99,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub2 r any right join sub1 l on l.y
 sub4 as (select z, a from tab3 where z != 8),
 sub5 as (select x, l.y, y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
 select * from sub5
-SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1;
+SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
 
 explain description=0
 with sub1 as (select x, y from tab1 where x != 2),
@@ -108,7 +108,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub2 r any right join sub1 l on l.y
 sub4 as (select z, a from tab3 where z != 8),
 sub5 as (select x, l.y, y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
 select * from sub5
-SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1;
+SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
 --
 -- RIGHT JOIN in sub5: sub5 -> WithMergableStage
 with sub1 as (select x, y from tab1 where x != 2),
@@ -116,7 +116,7 @@ sub2 as (select y, z from tab2 where y != 4),
 sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y = r.y),
 sub4 as (select z, a from tab3 where z != 8),
 sub5 as (select z, a, x, y, r.y, ll.z from sub4 rr any right join sub3 ll on ll.z = rr.z)
-select * from sub5 order by x SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1;
+select * from sub5 order by x SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
 
 explain description=0
 with sub1 as (select x, y from tab1 where x != 2),
@@ -124,7 +124,7 @@ sub2 as (select y, z from tab2 where y != 4),
 sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y = r.y),
 sub4 as (select z, a from tab3 where z != 8),
 sub5 as (select z, a, x, y, r.y, ll.z from sub4 rr any right join sub3 ll on ll.z = rr.z)
-select * from sub5 order by x SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1;
+select * from sub5 order by x SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
 
 --
 -- Subqueries for IN allowed
@@ -134,7 +134,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
 sub4 as (select z, a from tab3 where z != 8),
 sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
 select * from sub5 order by x
-SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1;
+SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
 
 explain description=0
 with sub1 as (select x, y from tab1 where x in (select number from numbers(16) where number != 2)),
@@ -143,7 +143,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
 sub4 as (select z, a from tab3 where z != 8),
 sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
 select * from sub5 order by x
-SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1;
+SETTINGS allow_experimental_parallel_reading_from_replicas = 2, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1;
 
 --
 -- Subqueries for IN are not allowed
@@ -153,7 +153,7 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
 sub4 as (select z, a from tab3 where z != 8),
 sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
 select * from sub5 order by x
-SETTINGS allow_experimental_parallel_reading_from_replicas = 1, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1, parallel_replicas_allow_in_with_subquery=0;
+SETTINGS allow_experimental_parallel_reading_from_replicas = 1, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1, parallel_replicas_allow_in_with_subquery=0;
 
 explain description=0
 with sub1 as (select x, y from tab1 where x in (select number from numbers(16) where number != 2)),
@@ -162,6 +162,6 @@ sub3 as (select l.x, l.y, r.y, r.z as z from sub1 l any left join sub2 r on l.y
 sub4 as (select z, a from tab3 where z != 8),
 sub5 as (select x, y, r.y, z, rr.z, a from sub3 ll any left join sub4 rr on ll.z = rr.z)
 select * from sub5 order by x
-SETTINGS allow_experimental_parallel_reading_from_replicas = 1, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', allow_experimental_analyzer=1, parallel_replicas_allow_in_with_subquery=0;
+SETTINGS allow_experimental_parallel_reading_from_replicas = 1, max_parallel_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, prefer_localhost_replica = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', enable_analyzer=1, parallel_replicas_allow_in_with_subquery=0;
 
 {%- endfor %}
diff --git a/tests/queries/0_stateless/02969_analyzer_eliminate_injective_functions.sql b/tests/queries/0_stateless/02969_analyzer_eliminate_injective_functions.sql
index 15f2550a63e..a7d0c7793db 100644
--- a/tests/queries/0_stateless/02969_analyzer_eliminate_injective_functions.sql
+++ b/tests/queries/0_stateless/02969_analyzer_eliminate_injective_functions.sql
@@ -1,4 +1,4 @@
-set allow_experimental_analyzer = 1;
+set enable_analyzer = 1;
 
 EXPLAIN QUERY TREE
 SELECT toString(toString(number + 1)) as val, count()
diff --git a/tests/queries/0_stateless/02969_functions_to_subcolumns_if_null.sql b/tests/queries/0_stateless/02969_functions_to_subcolumns_if_null.sql
index 361fd7c7a4e..859d26af1a1 100644
--- a/tests/queries/0_stateless/02969_functions_to_subcolumns_if_null.sql
+++ b/tests/queries/0_stateless/02969_functions_to_subcolumns_if_null.sql
@@ -7,21 +7,21 @@ INSERT INTO t_subcolumns_if SELECT number::Nullable(Int64) as number FROM number
 SELECT
     sum(multiIf(id IS NOT NULL, 1, 0))
 FROM t_subcolumns_if
-SETTINGS allow_experimental_analyzer = 1, optimize_functions_to_subcolumns = 1;
+SETTINGS enable_analyzer = 1, optimize_functions_to_subcolumns = 1;
 
 SELECT
     sum(multiIf(id IS NULL, 1, 0))
 FROM t_subcolumns_if
-SETTINGS allow_experimental_analyzer = 0, optimize_functions_to_subcolumns = 1;
+SETTINGS enable_analyzer = 0, optimize_functions_to_subcolumns = 1;
 
 SELECT
     sum(multiIf(id IS NULL, 1, 0))
 FROM t_subcolumns_if
-SETTINGS allow_experimental_analyzer = 1, optimize_functions_to_subcolumns = 0;
+SETTINGS enable_analyzer = 1, optimize_functions_to_subcolumns = 0;
 
 SELECT
     sum(multiIf(id IS NULL, 1, 0))
 FROM t_subcolumns_if
-SETTINGS allow_experimental_analyzer = 1, optimize_functions_to_subcolumns = 1;
+SETTINGS enable_analyzer = 1, optimize_functions_to_subcolumns = 1;
 
 DROP TABLE IF EXISTS t_subcolumns_if;
diff --git a/tests/queries/0_stateless/02971_analyzer_remote_id.sh b/tests/queries/0_stateless/02971_analyzer_remote_id.sh
index ab3c5292529..6d504ce3568 100755
--- a/tests/queries/0_stateless/02971_analyzer_remote_id.sh
+++ b/tests/queries/0_stateless/02971_analyzer_remote_id.sh
@@ -5,5 +5,5 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CURDIR"/../shell_config.sh
 
 ${CLICKHOUSE_CLIENT} --query="CREATE TABLE ${CLICKHOUSE_DATABASE}.x ENGINE = MergeTree() ORDER BY number AS SELECT * FROM numbers(2)"
-${CLICKHOUSE_LOCAL} --query="SELECT count() FROM remote('127.0.0.{2,3}', '${CLICKHOUSE_DATABASE}.x') SETTINGS allow_experimental_analyzer = 1" 2>&1 \
+${CLICKHOUSE_LOCAL} --query="SELECT count() FROM remote('127.0.0.{2,3}', '${CLICKHOUSE_DATABASE}.x') SETTINGS enable_analyzer = 1" 2>&1 \
         | grep -av "ASan doesn't fully support makecontext/swapcontext functions"
diff --git a/tests/queries/0_stateless/02971_functions_to_subcolumns_column_names.sql b/tests/queries/0_stateless/02971_functions_to_subcolumns_column_names.sql
index 48e5232d18b..6df2f27dbbf 100644
--- a/tests/queries/0_stateless/02971_functions_to_subcolumns_column_names.sql
+++ b/tests/queries/0_stateless/02971_functions_to_subcolumns_column_names.sql
@@ -5,7 +5,7 @@ CREATE TABLE t_column_names (arr Array(UInt64), n Nullable(String)) ENGINE = Mem
 INSERT INTO t_column_names VALUES ([1, 2, 3], 'foo');
 
 SET optimize_functions_to_subcolumns = 1;
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 EXPLAIN QUERY TREE dump_tree = 0, dump_ast = 1 SELECT length(arr), isNull(n) FROM t_column_names;
 SELECT length(arr), isNull(n) FROM t_column_names FORMAT JSONEachRow;
diff --git a/tests/queries/0_stateless/02971_functions_to_subcolumns_map.sql b/tests/queries/0_stateless/02971_functions_to_subcolumns_map.sql
index e8a752a82d5..c53a03b8ccd 100644
--- a/tests/queries/0_stateless/02971_functions_to_subcolumns_map.sql
+++ b/tests/queries/0_stateless/02971_functions_to_subcolumns_map.sql
@@ -5,7 +5,7 @@ CREATE TABLE t_func_to_subcolumns_map (id UInt64, m Map(String, UInt64)) ENGINE
 INSERT INTO t_func_to_subcolumns_map VALUES (1, map('aaa', 1, 'bbb', 2)) (2, map('ccc', 3));
 
 SET optimize_functions_to_subcolumns = 1;
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 EXPLAIN QUERY TREE dump_tree = 0, dump_ast = 1 SELECT length(m) FROM t_func_to_subcolumns_map;
 SELECT length(m) FROM t_func_to_subcolumns_map;
diff --git a/tests/queries/0_stateless/02971_functions_to_subcolumns_variant.sql b/tests/queries/0_stateless/02971_functions_to_subcolumns_variant.sql
index 511bcc44514..2612664a7b2 100644
--- a/tests/queries/0_stateless/02971_functions_to_subcolumns_variant.sql
+++ b/tests/queries/0_stateless/02971_functions_to_subcolumns_variant.sql
@@ -7,7 +7,7 @@ CREATE TABLE t_func_to_subcolumns_variant (id UInt64, v Variant(String, UInt64))
 INSERT INTO t_func_to_subcolumns_variant VALUES (1, 'foo') (2, 111);
 
 SET optimize_functions_to_subcolumns = 1;
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 EXPLAIN QUERY TREE dump_tree = 0, dump_ast = 1 SELECT variantElement(v, 'String') FROM t_func_to_subcolumns_variant;
 SELECT variantElement(v, 'String') FROM t_func_to_subcolumns_variant;
diff --git a/tests/queries/0_stateless/02972_parallel_replicas_cte.sql b/tests/queries/0_stateless/02972_parallel_replicas_cte.sql
index 083b0ecc5c9..767cd61216b 100644
--- a/tests/queries/0_stateless/02972_parallel_replicas_cte.sql
+++ b/tests/queries/0_stateless/02972_parallel_replicas_cte.sql
@@ -15,15 +15,15 @@ WITH filtered_groups AS (SELECT a FROM pr_1 WHERE a >= 100)
 SELECT count() FROM pr_2 INNER JOIN filtered_groups ON pr_2.a = filtered_groups.a
 SETTINGS allow_experimental_parallel_reading_from_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3;
 
--- Testing that it is disabled for allow_experimental_analyzer=0. With analyzer it will be supported (with correct result)
+-- Testing that it is disabled for enable_analyzer=0. With analyzer it will be supported (with correct result)
 WITH filtered_groups AS (SELECT a FROM pr_1 WHERE a >= 100)
 SELECT count() FROM pr_2 INNER JOIN filtered_groups ON pr_2.a = filtered_groups.a
-SETTINGS allow_experimental_analyzer = 0, allow_experimental_parallel_reading_from_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3; -- { serverError SUPPORT_IS_DISABLED }
+SETTINGS enable_analyzer = 0, allow_experimental_parallel_reading_from_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3; -- { serverError SUPPORT_IS_DISABLED }
 
 -- Disabled for any value of allow_experimental_parallel_reading_from_replicas != 1, not just 2
 WITH filtered_groups AS (SELECT a FROM pr_1 WHERE a >= 100)
 SELECT count() FROM pr_2 INNER JOIN filtered_groups ON pr_2.a = filtered_groups.a
-SETTINGS allow_experimental_analyzer = 0, allow_experimental_parallel_reading_from_replicas = 512, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3; -- { serverError SUPPORT_IS_DISABLED }
+SETTINGS enable_analyzer = 0, allow_experimental_parallel_reading_from_replicas = 512, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3; -- { serverError SUPPORT_IS_DISABLED }
 
 -- Sanitizer
 SELECT count() FROM pr_2 JOIN numbers(10) as pr_1 ON pr_2.a = pr_1.number
diff --git a/tests/queries/0_stateless/02974_analyzer_array_join_subcolumn.sql b/tests/queries/0_stateless/02974_analyzer_array_join_subcolumn.sql
index 30fb3c76c1f..1fd103d0bb0 100644
--- a/tests/queries/0_stateless/02974_analyzer_array_join_subcolumn.sql
+++ b/tests/queries/0_stateless/02974_analyzer_array_join_subcolumn.sql
@@ -7,16 +7,16 @@ INSERT INTO t2 VALUES (1, {'a': (1, 2), 'b': (2, 3)}),;
 CREATE TABLE t3 (id Int32, c Tuple(v String, pe Map(String, Tuple(a UInt64, b UInt64)))) ENGINE = MergeTree ORDER BY id;
 INSERT INTO t3 VALUES (1, ('A', {'a':(1, 2),'b':(2, 3)}));
 
-SELECT pe, pe.values.a FROM (SELECT * FROM t2) ARRAY JOIN pe SETTINGS allow_experimental_analyzer = 1;
-SELECT p, p.values.a FROM (SELECT * FROM t2) ARRAY JOIN pe AS p SETTINGS allow_experimental_analyzer = 1;
+SELECT pe, pe.values.a FROM (SELECT * FROM t2) ARRAY JOIN pe SETTINGS enable_analyzer = 1;
+SELECT p, p.values.a FROM (SELECT * FROM t2) ARRAY JOIN pe AS p SETTINGS enable_analyzer = 1;
 
 SELECT pe, pe.values.a FROM t2 ARRAY JOIN pe;
 SELECT p, p.values.a FROM t2 ARRAY JOIN pe AS p;
 
-SELECT c.pe, c.pe.values.a FROM (SELECT * FROM t3) ARRAY JOIN c.pe SETTINGS allow_experimental_analyzer = 1;
-SELECT p, p.values.a FROM (SELECT * FROM t3) ARRAY JOIN c.pe as p SETTINGS allow_experimental_analyzer = 1;
+SELECT c.pe, c.pe.values.a FROM (SELECT * FROM t3) ARRAY JOIN c.pe SETTINGS enable_analyzer = 1;
+SELECT p, p.values.a FROM (SELECT * FROM t3) ARRAY JOIN c.pe as p SETTINGS enable_analyzer = 1;
 
-SELECT c.pe, c.pe.values.a FROM t3 ARRAY JOIN c.pe SETTINGS allow_experimental_analyzer = 1;
+SELECT c.pe, c.pe.values.a FROM t3 ARRAY JOIN c.pe SETTINGS enable_analyzer = 1;
 SELECT p, p.values.a FROM t3 ARRAY JOIN c.pe as p;
 
 
diff --git a/tests/queries/0_stateless/02987_logical_optimizer_pass_lowcardinality.sql b/tests/queries/0_stateless/02987_logical_optimizer_pass_lowcardinality.sql
index be7689025b2..266270562cc 100644
--- a/tests/queries/0_stateless/02987_logical_optimizer_pass_lowcardinality.sql
+++ b/tests/queries/0_stateless/02987_logical_optimizer_pass_lowcardinality.sql
@@ -2,4 +2,4 @@ CREATE TABLE 02987_logical_optimizer_table (key Int, value Int) ENGINE=Memory();
 CREATE VIEW v1 AS SELECT * FROM 02987_logical_optimizer_table;
 CREATE TABLE 02987_logical_optimizer_merge AS v1 ENGINE=Merge(currentDatabase(), 'v1');
 
-SELECT _table, key FROM 02987_logical_optimizer_merge WHERE (_table = toFixedString(toFixedString(toFixedString('v1', toNullable(2)), 2), 2)) OR ((value = toLowCardinality(toNullable(10))) AND (_table = toFixedString(toNullable('v3'), 2))) OR ((value = 20) AND (_table = toFixedString(toFixedString(toFixedString('v1', 2), 2), 2)) AND (_table = toFixedString(toLowCardinality(toFixedString('v3', 2)), 2))) SETTINGS allow_experimental_analyzer = true, join_use_nulls = true, convert_query_to_cnf = true;
+SELECT _table, key FROM 02987_logical_optimizer_merge WHERE (_table = toFixedString(toFixedString(toFixedString('v1', toNullable(2)), 2), 2)) OR ((value = toLowCardinality(toNullable(10))) AND (_table = toFixedString(toNullable('v3'), 2))) OR ((value = 20) AND (_table = toFixedString(toFixedString(toFixedString('v1', 2), 2), 2)) AND (_table = toFixedString(toLowCardinality(toFixedString('v3', 2)), 2))) SETTINGS enable_analyzer = true, join_use_nulls = true, convert_query_to_cnf = true;
diff --git a/tests/queries/0_stateless/02989_join_using_parent_scope.reference b/tests/queries/0_stateless/02989_join_using_parent_scope.reference
index 965bfaf7c43..72678a1db4d 100644
--- a/tests/queries/0_stateless/02989_join_using_parent_scope.reference
+++ b/tests/queries/0_stateless/02989_join_using_parent_scope.reference
@@ -23,7 +23,7 @@ SELECT b AS a, a FROM tb JOIN tabc USING (a) ORDER BY ALL;
 3	3
 SELECT 1 AS b FROM tb JOIN ta USING (b); -- { serverError UNKNOWN_IDENTIFIER }
 -- SELECT * returns all columns from both tables in new analyzer
-SELECT 3 AS a, a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1;
+SELECT 3 AS a, a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL SETTINGS enable_analyzer = 1;
 3	3	0	3	abc3
 3	3	1	3	abc3
 3	3	2	3	abc3
@@ -31,57 +31,57 @@ SELECT 3 AS a, a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL SETTINGS allow
 3	3	\N	0	abc0
 3	3	\N	1	abc1
 3	3	\N	2	abc2
-SELECT b + 1 AS a, * FROM tb JOIN tabc USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1;
+SELECT b + 1 AS a, * FROM tb JOIN tabc USING (a) ORDER BY ALL SETTINGS enable_analyzer = 1;
 1	0	1	abc1
 2	1	2	abc2
 3	2	3	abc3
-SELECT b + 1 AS a, * FROM tb JOIN tabc USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1;
+SELECT b + 1 AS a, * FROM tb JOIN tabc USING (a) ORDER BY ALL SETTINGS enable_analyzer = 1;
 1	0	1	abc1
 2	1	2	abc2
 3	2	3	abc3
-SELECT b + 1 AS a, * FROM tb LEFT JOIN tabc USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1;
+SELECT b + 1 AS a, * FROM tb LEFT JOIN tabc USING (a) ORDER BY ALL SETTINGS enable_analyzer = 1;
 1	0	1	abc1
 2	1	2	abc2
 3	2	3	abc3
 4	3	\N	\N
-SELECT b + 1 AS a, * FROM tb RIGHT JOIN tabc USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1;
+SELECT b + 1 AS a, * FROM tb RIGHT JOIN tabc USING (a) ORDER BY ALL SETTINGS enable_analyzer = 1;
 1	0	1	abc1
 2	1	2	abc2
 3	2	3	abc3
 \N	\N	0	abc0
-SELECT b + 1 AS a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1;
+SELECT b + 1 AS a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL SETTINGS enable_analyzer = 1;
 1	0	1	abc1
 2	1	2	abc2
 3	2	3	abc3
 4	3	\N	\N
 \N	\N	0	abc0
-SELECT b + 1 AS a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL SETTINGS asterisk_include_alias_columns = 1, allow_experimental_analyzer = 1;
+SELECT b + 1 AS a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL SETTINGS asterisk_include_alias_columns = 1, enable_analyzer = 1;
 1	0	1	abc1	2	3
 2	1	2	abc2	3	4
 3	2	3	abc3	4	5
 4	3	\N	\N	\N	\N
 \N	\N	0	abc0	1	2
-SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1;
+SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL SETTINGS enable_analyzer = 1;
 1	0	1	2
 2	1	2	3
 3	2	3	4
-SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 LEFT JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1;
+SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 LEFT JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL SETTINGS enable_analyzer = 1;
 1	0	1	2
 2	1	2	3
 3	2	3	4
 4	3	\N	\N
-SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 RIGHT JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1;
+SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 RIGHT JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL SETTINGS enable_analyzer = 1;
 1	0	1	2
 2	1	2	3
 3	2	3	4
 \N	\N	0	1
-SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 FULL JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1;
+SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 FULL JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL SETTINGS enable_analyzer = 1;
 1	0	1	2
 2	1	2	3
 3	2	3	4
 4	3	\N	\N
 \N	\N	0	1
-SELECT b + 1 AS a, s FROM tb FULL OUTER JOIN tabc USING (a) PREWHERE a > 2 ORDER BY ALL SETTINGS allow_experimental_analyzer = 1;
+SELECT b + 1 AS a, s FROM tb FULL OUTER JOIN tabc USING (a) PREWHERE a > 2 ORDER BY ALL SETTINGS enable_analyzer = 1;
 3	abc3
 4	\N
 \N	abc0
@@ -94,7 +94,7 @@ SELECT a + 2 AS b FROM tb JOIN tabc USING (b) ORDER BY ALL
 SETTINGS analyzer_compatibility_join_using_top_level_identifier = 1; -- { serverError UNKNOWN_IDENTIFIER }
 -- In new analyzer with `analyzer_compatibility_join_using_top_level_identifier = 0` we get `b` from left table
 SELECT a + 2 AS b FROM tb JOIN tabc USING (b) ORDER BY ALL
-SETTINGS analyzer_compatibility_join_using_top_level_identifier = 0, allow_experimental_analyzer = 1;
+SETTINGS analyzer_compatibility_join_using_top_level_identifier = 0, enable_analyzer = 1;
 2
 3
 4
@@ -108,7 +108,7 @@ SELECT u1.uid, u1.spouse_name as name, u2.uid, u2.name
 FROM users u1 JOIN users u2 USING (name)
 ORDER BY u1.uid
 FORMAT TSVWithNamesAndTypes
-SETTINGS allow_experimental_analyzer = 1, analyzer_compatibility_join_using_top_level_identifier = 1;
+SETTINGS enable_analyzer = 1, analyzer_compatibility_join_using_top_level_identifier = 1;
 uid	name	u2.uid	u2.name
 Int16	String	Int16	String
 1231	Ksenia	6666	Ksenia
@@ -116,7 +116,7 @@ SELECT u1.uid, u1.spouse_name as name, u2.uid, u2.name
 FROM users u1 JOIN users u2 USING (name)
 ORDER BY u1.uid
 FORMAT TSVWithNamesAndTypes
-SETTINGS allow_experimental_analyzer = 1, analyzer_compatibility_join_using_top_level_identifier = 0;
+SETTINGS enable_analyzer = 1, analyzer_compatibility_join_using_top_level_identifier = 0;
 uid	name	u2.uid	u2.name
 Int16	String	Int16	String
 1231	Ksenia	1231	John
@@ -125,7 +125,7 @@ SELECT u1.uid, u1.spouse_name as name, u2.uid, u2.name
 FROM users u1 JOIN users u2 USING (name)
 ORDER BY u1.uid
 FORMAT TSVWithNamesAndTypes
-SETTINGS allow_experimental_analyzer = 0;
+SETTINGS enable_analyzer = 0;
 uid	name	u2.uid	u2.name
 Int16	String	Int16	String
 1231	Ksenia	6666	Ksenia
diff --git a/tests/queries/0_stateless/02989_join_using_parent_scope.sql b/tests/queries/0_stateless/02989_join_using_parent_scope.sql
index 2e4d9f097f7..4283d8b6eca 100644
--- a/tests/queries/0_stateless/02989_join_using_parent_scope.sql
+++ b/tests/queries/0_stateless/02989_join_using_parent_scope.sql
@@ -24,21 +24,21 @@ SELECT b AS a, a FROM tb JOIN tabc USING (a) ORDER BY ALL;
 SELECT 1 AS b FROM tb JOIN ta USING (b); -- { serverError UNKNOWN_IDENTIFIER }
 
 -- SELECT * returns all columns from both tables in new analyzer
-SELECT 3 AS a, a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1;
-SELECT b + 1 AS a, * FROM tb JOIN tabc USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1;
+SELECT 3 AS a, a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL SETTINGS enable_analyzer = 1;
+SELECT b + 1 AS a, * FROM tb JOIN tabc USING (a) ORDER BY ALL SETTINGS enable_analyzer = 1;
 
-SELECT b + 1 AS a, * FROM tb JOIN tabc USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1;
-SELECT b + 1 AS a, * FROM tb LEFT JOIN tabc USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1;
-SELECT b + 1 AS a, * FROM tb RIGHT JOIN tabc USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1;
-SELECT b + 1 AS a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1;
-SELECT b + 1 AS a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL SETTINGS asterisk_include_alias_columns = 1, allow_experimental_analyzer = 1;
+SELECT b + 1 AS a, * FROM tb JOIN tabc USING (a) ORDER BY ALL SETTINGS enable_analyzer = 1;
+SELECT b + 1 AS a, * FROM tb LEFT JOIN tabc USING (a) ORDER BY ALL SETTINGS enable_analyzer = 1;
+SELECT b + 1 AS a, * FROM tb RIGHT JOIN tabc USING (a) ORDER BY ALL SETTINGS enable_analyzer = 1;
+SELECT b + 1 AS a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL SETTINGS enable_analyzer = 1;
+SELECT b + 1 AS a, * FROM tb FULL JOIN tabc USING (a) ORDER BY ALL SETTINGS asterisk_include_alias_columns = 1, enable_analyzer = 1;
 
-SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1;
-SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 LEFT JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1;
-SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 RIGHT JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1;
-SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 FULL JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL SETTINGS allow_experimental_analyzer = 1;
+SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL SETTINGS enable_analyzer = 1;
+SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 LEFT JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL SETTINGS enable_analyzer = 1;
+SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 RIGHT JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL SETTINGS enable_analyzer = 1;
+SELECT b + 1 AS a, * FROM (SELECT b FROM tb) t1 FULL JOIN (SELECT a, b FROM tabc) t2 USING (a) ORDER BY ALL SETTINGS enable_analyzer = 1;
 
-SELECT b + 1 AS a, s FROM tb FULL OUTER JOIN tabc USING (a) PREWHERE a > 2 ORDER BY ALL SETTINGS allow_experimental_analyzer = 1;
+SELECT b + 1 AS a, s FROM tb FULL OUTER JOIN tabc USING (a) PREWHERE a > 2 ORDER BY ALL SETTINGS enable_analyzer = 1;
 
 
 -- It's a default behavior for old analyzer and new with analyzer_compatibility_join_using_top_level_identifier
@@ -49,7 +49,7 @@ SETTINGS analyzer_compatibility_join_using_top_level_identifier = 1; -- { server
 
 -- In new analyzer with `analyzer_compatibility_join_using_top_level_identifier = 0` we get `b` from left table
 SELECT a + 2 AS b FROM tb JOIN tabc USING (b) ORDER BY ALL
-SETTINGS analyzer_compatibility_join_using_top_level_identifier = 0, allow_experimental_analyzer = 1;
+SETTINGS analyzer_compatibility_join_using_top_level_identifier = 0, enable_analyzer = 1;
 
 -- This is example where query may return different results with different `analyzer_compatibility_join_using_top_level_identifier`
 
@@ -63,19 +63,19 @@ SELECT u1.uid, u1.spouse_name as name, u2.uid, u2.name
 FROM users u1 JOIN users u2 USING (name)
 ORDER BY u1.uid
 FORMAT TSVWithNamesAndTypes
-SETTINGS allow_experimental_analyzer = 1, analyzer_compatibility_join_using_top_level_identifier = 1;
+SETTINGS enable_analyzer = 1, analyzer_compatibility_join_using_top_level_identifier = 1;
 
 SELECT u1.uid, u1.spouse_name as name, u2.uid, u2.name
 FROM users u1 JOIN users u2 USING (name)
 ORDER BY u1.uid
 FORMAT TSVWithNamesAndTypes
-SETTINGS allow_experimental_analyzer = 1, analyzer_compatibility_join_using_top_level_identifier = 0;
+SETTINGS enable_analyzer = 1, analyzer_compatibility_join_using_top_level_identifier = 0;
 
 SELECT u1.uid, u1.spouse_name as name, u2.uid, u2.name
 FROM users u1 JOIN users u2 USING (name)
 ORDER BY u1.uid
 FORMAT TSVWithNamesAndTypes
-SETTINGS allow_experimental_analyzer = 0;
+SETTINGS enable_analyzer = 0;
 
 DROP TABLE IF EXISTS users;
 
diff --git a/tests/queries/0_stateless/02991_count_rewrite_analyzer.sql b/tests/queries/0_stateless/02991_count_rewrite_analyzer.sql
index b11aeedd225..bb0d3a1a9d1 100644
--- a/tests/queries/0_stateless/02991_count_rewrite_analyzer.sql
+++ b/tests/queries/0_stateless/02991_count_rewrite_analyzer.sql
@@ -1,5 +1,5 @@
 -- Regression test for https://github.com/ClickHouse/ClickHouse/issues/59919
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 
 SELECT toTypeName(sum(toNullable('a') IN toNullable('a'))) AS x;
 SELECT toTypeName(count(toNullable('a') IN toNullable('a'))) AS x;
diff --git a/tests/queries/0_stateless/02992_analyzer_group_by_const.sql b/tests/queries/0_stateless/02992_analyzer_group_by_const.sql
index 2a9e673d7bc..efe18918c93 100644
--- a/tests/queries/0_stateless/02992_analyzer_group_by_const.sql
+++ b/tests/queries/0_stateless/02992_analyzer_group_by_const.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 
 -- Illegal column String of first argument of function concatWithSeparator. Must be a constant String.
 SELECT concatWithSeparator('a', 'b') GROUP BY 'a';
diff --git a/tests/queries/0_stateless/02996_analyzer_prewhere_projection.sql b/tests/queries/0_stateless/02996_analyzer_prewhere_projection.sql
index 9d676001010..66f7c37a893 100644
--- a/tests/queries/0_stateless/02996_analyzer_prewhere_projection.sql
+++ b/tests/queries/0_stateless/02996_analyzer_prewhere_projection.sql
@@ -4,4 +4,4 @@ CREATE TABLE t__fuzz_0 (`i` LowCardinality(Int32), `j` Int32, `k` Int32, PROJECT
 INSERT INTO t__fuzz_0 Select number, number, number FROM numbers(100);
 
 SELECT * FROM t__fuzz_0 PREWHERE 7 AND (i < 2147483647) AND (j IN (2147483646, -2, 1))
-SETTINGS allow_experimental_analyzer = true;
+SETTINGS enable_analyzer = true;
diff --git a/tests/queries/0_stateless/02998_analyzer_prewhere_report.sql b/tests/queries/0_stateless/02998_analyzer_prewhere_report.sql
index b3027181901..5a7cab854e9 100644
--- a/tests/queries/0_stateless/02998_analyzer_prewhere_report.sql
+++ b/tests/queries/0_stateless/02998_analyzer_prewhere_report.sql
@@ -15,4 +15,4 @@ SELECT
     arrayFilter(x -> (x IN (2, 3)), data) AS filtered
 FROM hits
 WHERE arrayExists(x -> (x IN (2, 3)), data)
-SETTINGS allow_experimental_analyzer = 1;
+SETTINGS enable_analyzer = 1;
diff --git a/tests/queries/0_stateless/02998_analyzer_secret_args_tree_node.reference b/tests/queries/0_stateless/02998_analyzer_secret_args_tree_node.reference
index ffd2f68990b..ccd9540cb49 100644
--- a/tests/queries/0_stateless/02998_analyzer_secret_args_tree_node.reference
+++ b/tests/queries/0_stateless/02998_analyzer_secret_args_tree_node.reference
@@ -1,5 +1,5 @@
 -- { echoOn }
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 EXPLAIN QUERY TREE SELECT encrypt('aes-256-ofb', (SELECT 'qwerty'), '12345678901234567890123456789012'), encrypt('aes-256-ofb', (SELECT 'asdf'), '12345678901234567890123456789012');
 QUERY id: 0
   PROJECTION COLUMNS
@@ -7,7 +7,7 @@ QUERY id: 0
     encrypt(\'aes-256-ofb\', [HIDDEN id: 3], [HIDDEN id: 2]) Nullable(String)
   PROJECTION
     LIST id: 1, nodes: 2
-      CONSTANT id: 2, constant_value: \'\\n��&\', constant_value_type: Nullable(String)
+      CONSTANT id: 2, constant_value: \'\\n��&\', constant_value_type: Nullable(String)
         EXPRESSION
           FUNCTION id: 3, function_name: encrypt, function_type: ordinary, result_type: Nullable(String)
             ARGUMENTS
@@ -15,7 +15,7 @@ QUERY id: 0
                 CONSTANT id: 5, constant_value: \'aes-256-ofb\', constant_value_type: String
                 CONSTANT id: 6, constant_value: [HIDDEN id: 1], constant_value_type: Nullable(String)
                 CONSTANT id: 7, constant_value: [HIDDEN id: 2], constant_value_type: String
-      CONSTANT id: 8, constant_value: \'��\', constant_value_type: Nullable(String)
+      CONSTANT id: 8, constant_value: \'��\', constant_value_type: Nullable(String)
         EXPRESSION
           FUNCTION id: 9, function_name: encrypt, function_type: ordinary, result_type: Nullable(String)
             ARGUMENTS
@@ -33,7 +33,7 @@ QUERY id: 0
     encrypt(\'aes-256-ofb\', _subquery_2, \'12345678901234567890123456789012\') Nullable(String)
   PROJECTION
     LIST id: 1, nodes: 2
-      CONSTANT id: 2, constant_value: \'\\n��&\', constant_value_type: Nullable(String)
+      CONSTANT id: 2, constant_value: \'\\n��&\', constant_value_type: Nullable(String)
         EXPRESSION
           FUNCTION id: 3, function_name: encrypt, function_type: ordinary, result_type: Nullable(String)
             ARGUMENTS
@@ -50,7 +50,7 @@ QUERY id: 0
                       JOIN TREE
                         TABLE id: 10, table_name: system.one
                 CONSTANT id: 11, constant_value: \'12345678901234567890123456789012\', constant_value_type: String
-      CONSTANT id: 12, constant_value: \'��\', constant_value_type: Nullable(String)
+      CONSTANT id: 12, constant_value: \'��\', constant_value_type: Nullable(String)
         EXPRESSION
           FUNCTION id: 13, function_name: encrypt, function_type: ordinary, result_type: Nullable(String)
             ARGUMENTS
diff --git a/tests/queries/0_stateless/02998_analyzer_secret_args_tree_node.sql b/tests/queries/0_stateless/02998_analyzer_secret_args_tree_node.sql
index f40b40b6c8c..a216f886f8a 100644
--- a/tests/queries/0_stateless/02998_analyzer_secret_args_tree_node.sql
+++ b/tests/queries/0_stateless/02998_analyzer_secret_args_tree_node.sql
@@ -2,7 +2,7 @@
 -- encrypt function doesn't exist in the fastest build
 
 -- { echoOn }
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 EXPLAIN QUERY TREE SELECT encrypt('aes-256-ofb', (SELECT 'qwerty'), '12345678901234567890123456789012'), encrypt('aes-256-ofb', (SELECT 'asdf'), '12345678901234567890123456789012');
 
diff --git a/tests/queries/0_stateless/02998_projection_after_attach_partition.reference b/tests/queries/0_stateless/02998_projection_after_attach_partition.reference
index 1cb984f0f34..93cae129842 100644
--- a/tests/queries/0_stateless/02998_projection_after_attach_partition.reference
+++ b/tests/queries/0_stateless/02998_projection_after_attach_partition.reference
@@ -19,12 +19,12 @@ INSERT INTO visits_order SELECT 2, 'user2', number from numbers(1, 10);
 INSERT INTO visits_order SELECT 2, 'another_user2', number*2 from numbers(1, 10);
 INSERT INTO visits_order SELECT 2, 'yet_another_user2', number*3 from numbers(1, 10);
 ALTER TABLE visits_order_dst ATTACH PARTITION ID '2' FROM visits_order;
-SET allow_experimental_analyzer=0;
+SET enable_analyzer=0;
 EXPLAIN SELECT * FROM visits_order_dst WHERE user_name='another_user2';
 Expression ((Projection + Before ORDER BY))
   Filter
     ReadFromMergeTree (user_name_projection)
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 EXPLAIN SELECT * FROM visits_order_dst WHERE user_name='another_user2';
 Expression ((Project names + Projection))
   Filter
diff --git a/tests/queries/0_stateless/02998_projection_after_attach_partition.sql b/tests/queries/0_stateless/02998_projection_after_attach_partition.sql
index 4e0121dafe9..72ee4ad81e8 100644
--- a/tests/queries/0_stateless/02998_projection_after_attach_partition.sql
+++ b/tests/queries/0_stateless/02998_projection_after_attach_partition.sql
@@ -25,10 +25,10 @@ INSERT INTO visits_order SELECT 2, 'yet_another_user2', number*3 from numbers(1,
 
 ALTER TABLE visits_order_dst ATTACH PARTITION ID '2' FROM visits_order;
 
-SET allow_experimental_analyzer=0;
+SET enable_analyzer=0;
 
 EXPLAIN SELECT * FROM visits_order_dst WHERE user_name='another_user2';
 
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 
 EXPLAIN SELECT * FROM visits_order_dst WHERE user_name='another_user2';
diff --git a/tests/queries/0_stateless/02999_analyzer_preimage_null.sql b/tests/queries/0_stateless/02999_analyzer_preimage_null.sql
index 07d3a0f69c1..0fc61cf0836 100644
--- a/tests/queries/0_stateless/02999_analyzer_preimage_null.sql
+++ b/tests/queries/0_stateless/02999_analyzer_preimage_null.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 SET optimize_time_filter_with_preimage=1;
 
 CREATE TABLE date_t__fuzz_0 (`id` UInt32, `value1` String, `date1` Date) ENGINE = ReplacingMergeTree ORDER BY id SETTINGS allow_nullable_key=1;
diff --git a/tests/queries/0_stateless/03001_analyzer_nullable_nothing.sql b/tests/queries/0_stateless/03001_analyzer_nullable_nothing.sql
index 32c378ebf0a..c1c7ca87b5f 100644
--- a/tests/queries/0_stateless/03001_analyzer_nullable_nothing.sql
+++ b/tests/queries/0_stateless/03001_analyzer_nullable_nothing.sql
@@ -3,4 +3,4 @@ SELECT
     count(_CAST(NULL, 'Nullable(Nothing)')),
     round(avg(_CAST(NULL, 'Nullable(Nothing)'))) AS k
 FROM numbers(256)
-    SETTINGS allow_experimental_analyzer = 1;
+    SETTINGS enable_analyzer = 1;
diff --git a/tests/queries/0_stateless/03002_analyzer_prewhere.sql b/tests/queries/0_stateless/03002_analyzer_prewhere.sql
index 0edf16f1cbe..976e7cab73d 100644
--- a/tests/queries/0_stateless/03002_analyzer_prewhere.sql
+++ b/tests/queries/0_stateless/03002_analyzer_prewhere.sql
@@ -1,4 +1,4 @@
-SET max_threads = 16, receive_timeout = 10., receive_data_timeout_ms = 10000, allow_suspicious_low_cardinality_types = true, enable_positional_arguments = false, log_queries = true, table_function_remote_max_addresses = 200, any_join_distinct_right_table_keys = true, joined_subquery_requires_alias = false, allow_experimental_analyzer = true, max_execution_time = 10., max_memory_usage = 10000000000, log_comment = '/workspace/ch/tests/queries/0_stateless/01710_projection_in_index.sql', send_logs_level = 'fatal', enable_optimize_predicate_expression = false, prefer_localhost_replica = true, allow_introspection_functions = true, optimize_functions_to_subcolumns = false, transform_null_in = true, optimize_use_projections = true, allow_deprecated_syntax_for_merge_tree = true, parallelize_output_from_storages = false;
+SET max_threads = 16, receive_timeout = 10., receive_data_timeout_ms = 10000, allow_suspicious_low_cardinality_types = true, enable_positional_arguments = false, log_queries = true, table_function_remote_max_addresses = 200, any_join_distinct_right_table_keys = true, joined_subquery_requires_alias = false, enable_analyzer = true, max_execution_time = 10., max_memory_usage = 10000000000, log_comment = '/workspace/ch/tests/queries/0_stateless/01710_projection_in_index.sql', send_logs_level = 'fatal', enable_optimize_predicate_expression = false, prefer_localhost_replica = true, allow_introspection_functions = true, optimize_functions_to_subcolumns = false, transform_null_in = true, optimize_use_projections = true, allow_deprecated_syntax_for_merge_tree = true, parallelize_output_from_storages = false;
 
 CREATE TABLE t__fuzz_0 (`i` Int32, `j` Nullable(Int32), `k` Int32, PROJECTION p (SELECT * ORDER BY j)) ENGINE = MergeTree ORDER BY i SETTINGS index_granularity = 1, allow_nullable_key=1;
 
diff --git a/tests/queries/0_stateless/03003_analyzer_setting.sql b/tests/queries/0_stateless/03003_analyzer_setting.sql
index 2e5cab71277..3dbdaed4ad0 100644
--- a/tests/queries/0_stateless/03003_analyzer_setting.sql
+++ b/tests/queries/0_stateless/03003_analyzer_setting.sql
@@ -1,9 +1,9 @@
 CREATE TABLE test (dummy Int8) ENGINE = Distributed(test_cluster_two_shards, 'system', 'one');
 
-SET allow_experimental_analyzer = 0;
+SET enable_analyzer = 0;
 
-SELECT * FROM (SELECT * FROM test SETTINGS allow_experimental_analyzer = 1); -- { serverError INCORRECT_QUERY }
+SELECT * FROM (SELECT * FROM test SETTINGS enable_analyzer = 1); -- { serverError INCORRECT_QUERY }
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
-SELECT * FROM (SELECT * FROM test SETTINGS allow_experimental_analyzer = 0); -- { serverError INCORRECT_QUERY }
+SELECT * FROM (SELECT * FROM test SETTINGS enable_analyzer = 0); -- { serverError INCORRECT_QUERY }
diff --git a/tests/queries/0_stateless/03003_functions_to_subcolumns_final.sql b/tests/queries/0_stateless/03003_functions_to_subcolumns_final.sql
index 3fe29139c5f..b2ca478daa4 100644
--- a/tests/queries/0_stateless/03003_functions_to_subcolumns_final.sql
+++ b/tests/queries/0_stateless/03003_functions_to_subcolumns_final.sql
@@ -2,7 +2,7 @@ DROP TABLE IF EXISTS t_length_1;
 DROP TABLE IF EXISTS t_length_2;
 
 SET optimize_functions_to_subcolumns = 1;
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 SET optimize_on_insert = 0;
 
 CREATE TABLE t_length_1 (id UInt64, arr Array(UInt64)) ENGINE = ReplacingMergeTree ORDER BY id;
diff --git a/tests/queries/0_stateless/03006_join_on_inequal_expression_2.sql.j2 b/tests/queries/0_stateless/03006_join_on_inequal_expression_2.sql.j2
index f15fced161c..a09dc18739f 100644
--- a/tests/queries/0_stateless/03006_join_on_inequal_expression_2.sql.j2
+++ b/tests/queries/0_stateless/03006_join_on_inequal_expression_2.sql.j2
@@ -18,7 +18,7 @@ INSERT INTO t1 (key, a, attr) VALUES (1, 10, 'alpha'), (2, 15, 'beta'), (3, 20,
 INSERT INTO t2 (key, a, attr) VALUES (1, 5, 'ALPHA'), (2, 10, 'beta'), (4, 25, 'delta');
 
 
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 SET allow_experimental_join_condition=1;
 SET join_use_nulls=0;
 -- { echoOn }
diff --git a/tests/queries/0_stateless/03006_join_on_inequal_expression_3.sql.j2 b/tests/queries/0_stateless/03006_join_on_inequal_expression_3.sql.j2
index a97153ce3aa..009ae10e4ff 100644
--- a/tests/queries/0_stateless/03006_join_on_inequal_expression_3.sql.j2
+++ b/tests/queries/0_stateless/03006_join_on_inequal_expression_3.sql.j2
@@ -18,7 +18,7 @@ INSERT INTO t1 (key, a, attr) VALUES (1, 10, 'alpha'), (2, 15, 'beta'), (3, 20,
 INSERT INTO t2 (key, a, attr) VALUES (1, 5, 'ALPHA'), (2, 10, 'beta'), (4, 25, 'delta');
 
 
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 SET allow_experimental_join_condition=1;
 SET join_use_nulls=0;
 -- { echoOn }
diff --git a/tests/queries/0_stateless/03006_join_on_inequal_expression_4.sql.j2 b/tests/queries/0_stateless/03006_join_on_inequal_expression_4.sql.j2
index 3235019821b..37eaaa8ab5c 100644
--- a/tests/queries/0_stateless/03006_join_on_inequal_expression_4.sql.j2
+++ b/tests/queries/0_stateless/03006_join_on_inequal_expression_4.sql.j2
@@ -15,7 +15,7 @@ CREATE TABLE t2 (
 INSERT INTO t1 (key, a) VALUES (1, 10), (2, 15), (3, 20);
 INSERT INTO t2 (key, a) VALUES (1, 5), (2, 10), (4, 25);
 
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 SET allow_experimental_join_condition=1;
 SET join_algorithm='hash';
 -- { echoOn }
diff --git a/tests/queries/0_stateless/03006_join_on_inequal_expression_fast.sql.j2 b/tests/queries/0_stateless/03006_join_on_inequal_expression_fast.sql.j2
index a363101ca69..61ad5ec0bf1 100644
--- a/tests/queries/0_stateless/03006_join_on_inequal_expression_fast.sql.j2
+++ b/tests/queries/0_stateless/03006_join_on_inequal_expression_fast.sql.j2
@@ -6,7 +6,7 @@ INSERT INTO t1 VALUES ('key1', 'a', 1, 1, 2), ('key1', 'b', 2, 3, 2), ('key1', '
 CREATE TABLE t2 (key String, attr String, a UInt64, b UInt64, c Nullable(UInt64)) ENGINE = MergeTree ORDER BY key;
 INSERT INTO t2 VALUES ('key1', 'A', 1, 2, 1), ('key1', 'B', 2, 1, 2), ('key1', 'C', 3, 4, 5), ('key1', 'D', 4, 1, 6), ('key3', 'a3', 1, 1, 1), ('key4', 'F', 1,1,1);
 
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 SET allow_experimental_join_condition=1;
 SET join_use_nulls=0;
 -- { echoOn }
diff --git a/tests/queries/0_stateless/03006_parallel_replicas_cte_explain_syntax_crash.sql b/tests/queries/0_stateless/03006_parallel_replicas_cte_explain_syntax_crash.sql
index df4ec9d26a3..7c8d6dd9aff 100644
--- a/tests/queries/0_stateless/03006_parallel_replicas_cte_explain_syntax_crash.sql
+++ b/tests/queries/0_stateless/03006_parallel_replicas_cte_explain_syntax_crash.sql
@@ -20,7 +20,7 @@ ORDER BY n
 AS SELECT *
 FROM numbers(10);
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 SET allow_experimental_parallel_reading_from_replicas = 1, parallel_replicas_for_non_replicated_merge_tree = 1, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', max_parallel_replicas = 3, parallel_replicas_min_number_of_rows_per_replica=0;
 
 EXPLAIN SYNTAX
diff --git a/tests/queries/0_stateless/03007_column_nullable_uninitialzed_value.sql b/tests/queries/0_stateless/03007_column_nullable_uninitialzed_value.sql
index 9479044e0e0..44f6642d2a5 100644
--- a/tests/queries/0_stateless/03007_column_nullable_uninitialzed_value.sql
+++ b/tests/queries/0_stateless/03007_column_nullable_uninitialzed_value.sql
@@ -1 +1 @@
-SELECT count(NULL) IGNORE NULLS > avg(toDecimal32(NULL)) IGNORE NULLS, count() FROM numbers(1000) WITH TOTALS SETTINGS allow_experimental_analyzer = 1;
+SELECT count(NULL) IGNORE NULLS > avg(toDecimal32(NULL)) IGNORE NULLS, count() FROM numbers(1000) WITH TOTALS SETTINGS enable_analyzer = 1;
diff --git a/tests/queries/0_stateless/03010_sum_to_to_count_if_nullable.sql b/tests/queries/0_stateless/03010_sum_to_to_count_if_nullable.sql
index 71a175faac8..5ec6ee5a996 100644
--- a/tests/queries/0_stateless/03010_sum_to_to_count_if_nullable.sql
+++ b/tests/queries/0_stateless/03010_sum_to_to_count_if_nullable.sql
@@ -1,11 +1,11 @@
 SET optimize_rewrite_sum_if_to_count_if = 1;
 
-SET allow_experimental_analyzer = 0;
+SET enable_analyzer = 0;
 SELECT (sumIf(toNullable(1), (number % 2) = 0), NULL) FROM numbers(10);
 SELECT (sum(if((number % 2) = 0, toNullable(1), 0)), NULL) FROM numbers(10);
 SELECT (tuple(sum(if((number % 2) = 0, toNullable(0), 123)) IGNORE NULLS), toUInt8(3)) FROM numbers(100);
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 SELECT (sumIf(toNullable(1), (number % 2) = 0), NULL) FROM numbers(10);
 EXPLAIN QUERY TREE SELECT (sumIf(toNullable(1), (number % 2) = 0), NULL) FROM numbers(10);
 SELECT (sum(if((number % 2) = 0, toNullable(1), 0)), NULL) FROM numbers(10);
diff --git a/tests/queries/0_stateless/03013_group_by_use_nulls_with_materialize_and_analyzer.sql b/tests/queries/0_stateless/03013_group_by_use_nulls_with_materialize_and_analyzer.sql
index b15593b2abd..7b57dbd807d 100644
--- a/tests/queries/0_stateless/03013_group_by_use_nulls_with_materialize_and_analyzer.sql
+++ b/tests/queries/0_stateless/03013_group_by_use_nulls_with_materialize_and_analyzer.sql
@@ -1,4 +1,4 @@
-set allow_experimental_analyzer = 1;
+set enable_analyzer = 1;
 set group_by_use_nulls = 1;
 set optimize_group_by_function_keys = 1;
 set optimize_injective_functions_in_group_by = 1;
@@ -8,4 +8,3 @@ SELECT materialize(3) from numbers(10) GROUP BY GROUPING SETS (('str'), (materia
 SELECT ignore(3) from numbers(10) GROUP BY GROUPING SETS (('str'), (ignore(3))) order by all;
 SELECT materialize(ignore(3)) from numbers(10) GROUP BY GROUPING SETS (('str'), (materialize(ignore(3)))) order by all;
 SELECT ignore(materialize(3)) from numbers(10) GROUP BY GROUPING SETS (('str'), (ignore(materialize(3)))) order by all;
-
diff --git a/tests/queries/0_stateless/03014_analyzer_groupby_fuzz_60317.sql b/tests/queries/0_stateless/03014_analyzer_groupby_fuzz_60317.sql
index 094614cb78d..295f89c5a0a 100644
--- a/tests/queries/0_stateless/03014_analyzer_groupby_fuzz_60317.sql
+++ b/tests/queries/0_stateless/03014_analyzer_groupby_fuzz_60317.sql
@@ -6,7 +6,7 @@ SELECT
 FROM system.one
 GROUP BY
     _CAST(30, 'Nullable(UInt8)')
-SETTINGS allow_experimental_analyzer = 1;
+SETTINGS enable_analyzer = 1;
 
 -- WITH CUBE (note that result is different with the analyzer (analyzer is correct including all combinations)
 SELECT
@@ -24,4 +24,4 @@ GROUP BY
     _CAST(30, 'Nullable(UInt8)')
 WITH CUBE
 WITH TOTALS
-SETTINGS allow_experimental_analyzer = 1;
+SETTINGS enable_analyzer = 1;
diff --git a/tests/queries/0_stateless/03014_group_by_use_nulls_injective_functions_and_analyzer.sql b/tests/queries/0_stateless/03014_group_by_use_nulls_injective_functions_and_analyzer.sql
index e7ea964b876..d700f9ba3b1 100644
--- a/tests/queries/0_stateless/03014_group_by_use_nulls_injective_functions_and_analyzer.sql
+++ b/tests/queries/0_stateless/03014_group_by_use_nulls_injective_functions_and_analyzer.sql
@@ -1,5 +1,4 @@
-set allow_experimental_analyzer=1, group_by_use_nulls=1, optimize_injective_functions_in_group_by=1;
+set enable_analyzer=1, group_by_use_nulls=1, optimize_injective_functions_in_group_by=1;
 SELECT bitNot(bitNot(number)) + 3 FROM numbers(10) GROUP BY GROUPING SETS (('str', bitNot(bitNot(number))), ('str')) order by all;
 SELECT tuple(tuple(tuple(number))) FROM numbers(10) GROUP BY GROUPING SETS (('str', tuple(tuple(number))), ('str')) order by all;
 SELECT materialize(3) + 3 FROM numbers(10) GROUP BY GROUPING SETS (('str', materialize(materialize(3))), ('str')) order by all;
-
diff --git a/tests/queries/0_stateless/03015_analyzer_groupby_fuzz_60772.sql b/tests/queries/0_stateless/03015_analyzer_groupby_fuzz_60772.sql
index d3bd9ef0ce3..5190b8635d6 100644
--- a/tests/queries/0_stateless/03015_analyzer_groupby_fuzz_60772.sql
+++ b/tests/queries/0_stateless/03015_analyzer_groupby_fuzz_60772.sql
@@ -7,7 +7,7 @@ GROUP BY
     toFixedString(toFixedString('2018-01-02 22:33:44', 19), 19),
     'gr',
     '2018-01-02 22:33:44'
-SETTINGS allow_experimental_analyzer = 1;
+SETTINGS enable_analyzer = 1;
 
 -- WITH CUBE (note that result is different with the analyzer (analyzer is correct including all combinations)
 SELECT
@@ -20,4 +20,4 @@ GROUP BY
     'gr',
     '2018-01-02 22:33:44'
 WITH CUBE
-SETTINGS allow_experimental_analyzer = 1;
+SETTINGS enable_analyzer = 1;
diff --git a/tests/queries/0_stateless/03016_analyzer_groupby_fuzz_59796.sql b/tests/queries/0_stateless/03016_analyzer_groupby_fuzz_59796.sql
index c00a75a631c..6c926c32887 100644
--- a/tests/queries/0_stateless/03016_analyzer_groupby_fuzz_59796.sql
+++ b/tests/queries/0_stateless/03016_analyzer_groupby_fuzz_59796.sql
@@ -3,4 +3,4 @@ SELECT
 GROUP BY
     concat(unhex('00'), toFixedString(materialize(toFixedString(' key="v" ', 9)), 9), toFixedString(toFixedString('00', 2), toNullable(2)),  toFixedString(toFixedString(toFixedString(' key="v" ', 9), 9), 9)),
     concat(' key="v" ')
-SETTINGS allow_experimental_analyzer = 1;
+SETTINGS enable_analyzer = 1;
diff --git a/tests/queries/0_stateless/03017_analyzer_groupby_fuzz_61600.sql b/tests/queries/0_stateless/03017_analyzer_groupby_fuzz_61600.sql
index 53a5cfe9b1a..b22ea42b686 100644
--- a/tests/queries/0_stateless/03017_analyzer_groupby_fuzz_61600.sql
+++ b/tests/queries/0_stateless/03017_analyzer_groupby_fuzz_61600.sql
@@ -11,7 +11,7 @@ FROM set_index_not__fuzz_0
 GROUP BY
     toNullable(3),
     concat(concat(NULLIF(1, 1), toNullable(toNullable(3))))
-SETTINGS allow_experimental_analyzer = 1;
+SETTINGS enable_analyzer = 1;
 
 -- WITH ROLLUP (note that result is different with the analyzer (analyzer is correct including all combinations)
 SELECT
@@ -22,4 +22,4 @@ GROUP BY
     toNullable(3),
     concat(concat(NULLIF(1, 1), toNullable(toNullable(3))))
 WITH ROLLUP
-SETTINGS allow_experimental_analyzer = 1;
+SETTINGS enable_analyzer = 1;
diff --git a/tests/queries/0_stateless/03023_analyzer_optimize_group_by_function_keys_with_nulls.sql b/tests/queries/0_stateless/03023_analyzer_optimize_group_by_function_keys_with_nulls.sql
index f0b60caca36..0d0a81c9105 100644
--- a/tests/queries/0_stateless/03023_analyzer_optimize_group_by_function_keys_with_nulls.sql
+++ b/tests/queries/0_stateless/03023_analyzer_optimize_group_by_function_keys_with_nulls.sql
@@ -1,5 +1,4 @@
-set allow_experimental_analyzer=1;
+set enable_analyzer=1;
 set group_by_use_nulls=1;
 set optimize_group_by_function_keys=1;
 SELECT ignore(toLowCardinality(number)) FROM numbers(10) GROUP BY GROUPING SETS ((ignore(toLowCardinality(number)), toLowCardinality(number)));
-
diff --git a/tests/queries/0_stateless/03023_group_by_use_nulls_analyzer_crashes.sql b/tests/queries/0_stateless/03023_group_by_use_nulls_analyzer_crashes.sql
index b8c173520a9..d3d6ecaadaf 100644
--- a/tests/queries/0_stateless/03023_group_by_use_nulls_analyzer_crashes.sql
+++ b/tests/queries/0_stateless/03023_group_by_use_nulls_analyzer_crashes.sql
@@ -1,4 +1,4 @@
-set allow_experimental_analyzer = 1, group_by_use_nulls = 1;
+set enable_analyzer = 1, group_by_use_nulls = 1;
 
 SELECT tuple(tuple(number)) as x FROM numbers(10) GROUP BY (number, tuple(number)) with cube order by x;
 
@@ -56,4 +56,4 @@ SELECT arraySplit(number -> toUInt8(number), []) from numbers(1) GROUP BY toUInt
 
 SELECT count(arraySplit(number -> toUInt8(number), [arraySplit(x -> toUInt8(number), [])])) FROM numbers(10) GROUP BY number, [number] WITH ROLLUP settings group_by_use_nulls=1; -- {serverError ILLEGAL_TYPE_OF_ARGUMENT}
 
-SELECT count(arraySplit(x -> toUInt8(number), [])) FROM numbers(10) GROUP BY number, [number] WITH ROLLUP settings group_by_use_nulls=1;
\ No newline at end of file
+SELECT count(arraySplit(x -> toUInt8(number), [])) FROM numbers(10) GROUP BY number, [number] WITH ROLLUP settings group_by_use_nulls=1;
diff --git a/tests/queries/0_stateless/03023_remove_unused_column_distinct.sql b/tests/queries/0_stateless/03023_remove_unused_column_distinct.sql
index c2f32bfe3c1..af8756f1fbc 100644
--- a/tests/queries/0_stateless/03023_remove_unused_column_distinct.sql
+++ b/tests/queries/0_stateless/03023_remove_unused_column_distinct.sql
@@ -12,4 +12,4 @@ FROM
         FROM numbers(10)
     )
 )
-SETTINGS allow_experimental_analyzer = 1;
+SETTINGS enable_analyzer = 1;
diff --git a/tests/queries/0_stateless/03031_filter_float64_logical_error.sql b/tests/queries/0_stateless/03031_filter_float64_logical_error.sql
index 59d4a06c8f7..df35cc7b14a 100644
--- a/tests/queries/0_stateless/03031_filter_float64_logical_error.sql
+++ b/tests/queries/0_stateless/03031_filter_float64_logical_error.sql
@@ -26,7 +26,7 @@ PREWHERE (id = NULL) AND 1024
 WHERE 0.0001
 GROUP BY '0.03'
     WITH ROLLUP
-SETTINGS force_primary_key = 1, force_data_skipping_indices = 'value_1_idx, value_2_idx', allow_experimental_analyzer=0;
+SETTINGS force_primary_key = 1, force_data_skipping_indices = 'value_1_idx, value_2_idx', enable_analyzer=0;
 
 
 SELECT
@@ -37,4 +37,4 @@ PREWHERE (id = NULL) AND 1024
 WHERE 0.0001
 GROUP BY '0.03'
     WITH ROLLUP
-SETTINGS force_primary_key = 1, force_data_skipping_indices = 'value_1_idx, value_2_idx', allow_experimental_analyzer=1; -- { serverError ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER }
+SETTINGS force_primary_key = 1, force_data_skipping_indices = 'value_1_idx, value_2_idx', enable_analyzer=1; -- { serverError ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER }
diff --git a/tests/queries/0_stateless/03031_tuple_elimination_analyzer.sql b/tests/queries/0_stateless/03031_tuple_elimination_analyzer.sql
index 97a19cda7d3..42bd5004933 100644
--- a/tests/queries/0_stateless/03031_tuple_elimination_analyzer.sql
+++ b/tests/queries/0_stateless/03031_tuple_elimination_analyzer.sql
@@ -1,6 +1,6 @@
 DROP TABLE IF EXISTS test;
 
-SET allow_suspicious_low_cardinality_types = true, allow_experimental_analyzer = true;
+SET allow_suspicious_low_cardinality_types = true, enable_analyzer = true;
 
 CREATE TABLE test (`id` LowCardinality(UInt32)) ENGINE = MergeTree ORDER BY id AS SELECT 0;
 
diff --git a/tests/queries/0_stateless/03032_redundant_equals.sql b/tests/queries/0_stateless/03032_redundant_equals.sql
index de85ec5cf00..63073dbcefd 100644
--- a/tests/queries/0_stateless/03032_redundant_equals.sql
+++ b/tests/queries/0_stateless/03032_redundant_equals.sql
@@ -9,7 +9,7 @@ ORDER BY k;
 
 INSERT INTO test_table SELECT number FROM numbers(10000000);
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 SELECT * FROM test_table WHERE k in (100) = 1;
 SELECT * FROM test_table WHERE k = (100) = 1;
@@ -25,7 +25,7 @@ SELECT * FROM test_table WHERE (NOT ((k not in (100) = 0) OR (k in (100) = 1)))
 SELECT * FROM test_table WHERE (NOT ((k in (101) = 0) OR (k in (100) = 1))) = 1;
 SELECT * FROM test_table WHERE ((k not in (101) = 0) OR (k in (100) = 1)) = 1;
 SELECT * FROM test_table WHERE ((k not in (99) = 1) AND (k in (100) = 1)) = 1;
--- we skip optimizing queries with toNullable(0 or 1) but lets make sure they still work 
+-- we skip optimizing queries with toNullable(0 or 1) but lets make sure they still work
 SELECT * FROM test_table WHERE (k = 101) = toLowCardinality(toNullable(1));
 SELECT * FROM test_table WHERE (k = 101) = toNullable(1);
 SELECT * FROM test_table WHERE (k = 101) = toLowCardinality(1);
diff --git a/tests/queries/0_stateless/03033_analyzer_merge_engine_filter_push_down.sql b/tests/queries/0_stateless/03033_analyzer_merge_engine_filter_push_down.sql
index 9be1152bbbf..d01e458a544 100644
--- a/tests/queries/0_stateless/03033_analyzer_merge_engine_filter_push_down.sql
+++ b/tests/queries/0_stateless/03033_analyzer_merge_engine_filter_push_down.sql
@@ -3,6 +3,5 @@ drop table if exists test;
 create table test (`x` LowCardinality(Nullable(UInt32)), `y` String) engine = MergeTree order by tuple();
 insert into test values (1, 'a'), (2, 'bb'), (3, 'ccc'), (4, 'dddd');
 create table m_table (x UInt32, y String) engine = Merge(currentDatabase(), 'test*');
-select toTypeName(x), x FROM m_table SETTINGS additional_table_filters = {'m_table':'x != 4'}, optimize_move_to_prewhere=1, allow_experimental_analyzer=1;
+select toTypeName(x), x FROM m_table SETTINGS additional_table_filters = {'m_table':'x != 4'}, optimize_move_to_prewhere=1, enable_analyzer=1;
 drop table test;
-
diff --git a/tests/queries/0_stateless/03033_analyzer_query_parameters.sh b/tests/queries/0_stateless/03033_analyzer_query_parameters.sh
index cf46067df99..67ddf2449eb 100755
--- a/tests/queries/0_stateless/03033_analyzer_query_parameters.sh
+++ b/tests/queries/0_stateless/03033_analyzer_query_parameters.sh
@@ -4,5 +4,5 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
-${CLICKHOUSE_LOCAL} --param_rounding 1 --query "SELECT 1 AS x ORDER BY x WITH FILL STEP {rounding:UInt32} SETTINGS allow_experimental_analyzer = 1"
-${CLICKHOUSE_LOCAL} --param_rounding 1 --query "SELECT 1 AS x ORDER BY x WITH FILL STEP {rounding:UInt32} SETTINGS allow_experimental_analyzer = 0"
+${CLICKHOUSE_LOCAL} --param_rounding 1 --query "SELECT 1 AS x ORDER BY x WITH FILL STEP {rounding:UInt32} SETTINGS enable_analyzer = 1"
+${CLICKHOUSE_LOCAL} --param_rounding 1 --query "SELECT 1 AS x ORDER BY x WITH FILL STEP {rounding:UInt32} SETTINGS enable_analyzer = 0"
diff --git a/tests/queries/0_stateless/03033_cte_numbers_memory.sql b/tests/queries/0_stateless/03033_cte_numbers_memory.sql
index 66b11cbfaa5..b362f42f89a 100644
--- a/tests/queries/0_stateless/03033_cte_numbers_memory.sql
+++ b/tests/queries/0_stateless/03033_cte_numbers_memory.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/61238
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 
 WITH
 (SELECT number FROM system.numbers LIMIT 1) as w1,
diff --git a/tests/queries/0_stateless/03033_recursive_cte_basic.reference b/tests/queries/0_stateless/03033_recursive_cte_basic.reference
index a890fccef2b..dc6a9cd8f89 100644
--- a/tests/queries/0_stateless/03033_recursive_cte_basic.reference
+++ b/tests/queries/0_stateless/03033_recursive_cte_basic.reference
@@ -1,6 +1,6 @@
 -- { echoOn }
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 WITH RECURSIVE recursive_cte AS (SELECT 1 AS n UNION ALL SELECT n + 1 FROM recursive_cte WHERE n < 10)
 SELECT n FROM recursive_cte;
 1
diff --git a/tests/queries/0_stateless/03033_recursive_cte_basic.sql b/tests/queries/0_stateless/03033_recursive_cte_basic.sql
index f85e1ffe0bd..63014e9ccd4 100644
--- a/tests/queries/0_stateless/03033_recursive_cte_basic.sql
+++ b/tests/queries/0_stateless/03033_recursive_cte_basic.sql
@@ -1,6 +1,6 @@
 -- { echoOn }
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 WITH RECURSIVE recursive_cte AS (SELECT 1 AS n UNION ALL SELECT n + 1 FROM recursive_cte WHERE n < 10)
 SELECT n FROM recursive_cte;
diff --git a/tests/queries/0_stateless/03033_with_fill_interpolate.sql b/tests/queries/0_stateless/03033_with_fill_interpolate.sql
index 0ec0050a922..48457341e0d 100644
--- a/tests/queries/0_stateless/03033_with_fill_interpolate.sql
+++ b/tests/queries/0_stateless/03033_with_fill_interpolate.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/55794
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 DROP TABLE IF EXISTS 03033_example_table;
 
 CREATE TABLE 03033_example_table
diff --git a/tests/queries/0_stateless/03034_normalized_ast.sql b/tests/queries/0_stateless/03034_normalized_ast.sql
index 385af4e2c34..8b518d6d11b 100644
--- a/tests/queries/0_stateless/03034_normalized_ast.sql
+++ b/tests/queries/0_stateless/03034_normalized_ast.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/49472
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 SELECT
     concat(database, table) AS name,
     count()
diff --git a/tests/queries/0_stateless/03034_recursive_cte_tree.sql b/tests/queries/0_stateless/03034_recursive_cte_tree.sql
index a3ff43e435a..fa62298939a 100644
--- a/tests/queries/0_stateless/03034_recursive_cte_tree.sql
+++ b/tests/queries/0_stateless/03034_recursive_cte_tree.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS tree;
 CREATE TABLE tree
diff --git a/tests/queries/0_stateless/03034_recursive_cte_tree_fuzz_crash_fix.sql b/tests/queries/0_stateless/03034_recursive_cte_tree_fuzz_crash_fix.sql
index ae04fa8f377..1e26c53769b 100644
--- a/tests/queries/0_stateless/03034_recursive_cte_tree_fuzz_crash_fix.sql
+++ b/tests/queries/0_stateless/03034_recursive_cte_tree_fuzz_crash_fix.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 SET enable_global_with_statement=1;
 SET session_timezone = 'Etc/UTC';
 
diff --git a/tests/queries/0_stateless/03034_recursive_cte_tree_merge_tree.reference b/tests/queries/0_stateless/03034_recursive_cte_tree_merge_tree.reference
index 85f8c3267c6..f5b950db867 100644
--- a/tests/queries/0_stateless/03034_recursive_cte_tree_merge_tree.reference
+++ b/tests/queries/0_stateless/03034_recursive_cte_tree_merge_tree.reference
@@ -1,6 +1,6 @@
 -- { echoOn }
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 DROP TABLE IF EXISTS tree;
 CREATE TABLE tree
 (
diff --git a/tests/queries/0_stateless/03034_recursive_cte_tree_merge_tree.sql b/tests/queries/0_stateless/03034_recursive_cte_tree_merge_tree.sql
index 15acbaf6da9..231aae296e6 100644
--- a/tests/queries/0_stateless/03034_recursive_cte_tree_merge_tree.sql
+++ b/tests/queries/0_stateless/03034_recursive_cte_tree_merge_tree.sql
@@ -1,6 +1,6 @@
 -- { echoOn }
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS tree;
 CREATE TABLE tree
diff --git a/tests/queries/0_stateless/03035_alias_column_bug_distributed.sql b/tests/queries/0_stateless/03035_alias_column_bug_distributed.sql
index 74463743b01..8f60808d700 100644
--- a/tests/queries/0_stateless/03035_alias_column_bug_distributed.sql
+++ b/tests/queries/0_stateless/03035_alias_column_bug_distributed.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/44414
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 DROP TABLE IF EXISTS alias_bug;
 DROP TABLE IF EXISTS alias_bug_dist;
 CREATE TABLE alias_bug
diff --git a/tests/queries/0_stateless/03035_internal_functions_direct_call.sql b/tests/queries/0_stateless/03035_internal_functions_direct_call.sql
index 951e0733dbb..e358e498343 100644
--- a/tests/queries/0_stateless/03035_internal_functions_direct_call.sql
+++ b/tests/queries/0_stateless/03035_internal_functions_direct_call.sql
@@ -4,7 +4,7 @@
 
 SELECT __actionName(); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH }
 SELECT __actionName('aaa', 'aaa', 'aaa'); -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH }
-SELECT __actionName('aaa', '') SETTINGS allow_experimental_analyzer = 1; -- { serverError BAD_ARGUMENTS }
+SELECT __actionName('aaa', '') SETTINGS enable_analyzer = 1; -- { serverError BAD_ARGUMENTS }
 SELECT __actionName('aaa', materialize('aaa')); -- { serverError BAD_ARGUMENTS,ILLEGAL_COLUMN }
 SELECT __actionName(materialize('aaa'), 'aaa'); -- { serverError ILLEGAL_COLUMN }
 SELECT __actionName('aaa', 'aaa');
diff --git a/tests/queries/0_stateless/03035_recursive_cte_postgres_1.reference b/tests/queries/0_stateless/03035_recursive_cte_postgres_1.reference
index e6cbdc971eb..568421f3fba 100644
--- a/tests/queries/0_stateless/03035_recursive_cte_postgres_1.reference
+++ b/tests/queries/0_stateless/03035_recursive_cte_postgres_1.reference
@@ -32,7 +32,7 @@
 
 -- { echoOn }
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 -- WITH RECURSIVE
 
 -- sum of 1..100
diff --git a/tests/queries/0_stateless/03035_recursive_cte_postgres_1.sql b/tests/queries/0_stateless/03035_recursive_cte_postgres_1.sql
index 8026dadc331..9a4e313ce90 100644
--- a/tests/queries/0_stateless/03035_recursive_cte_postgres_1.sql
+++ b/tests/queries/0_stateless/03035_recursive_cte_postgres_1.sql
@@ -32,7 +32,7 @@
 
 -- { echoOn }
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 -- WITH RECURSIVE
 
diff --git a/tests/queries/0_stateless/03036_join_filter_push_down_equivalent_sets.sql b/tests/queries/0_stateless/03036_join_filter_push_down_equivalent_sets.sql
index d94a68aa4d8..e1a13d1ce71 100644
--- a/tests/queries/0_stateless/03036_join_filter_push_down_equivalent_sets.sql
+++ b/tests/queries/0_stateless/03036_join_filter_push_down_equivalent_sets.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 SET optimize_move_to_prewhere = 0;
 SET query_plan_convert_outer_join_to_inner_join = 0;
 
diff --git a/tests/queries/0_stateless/03036_recursive_cte_postgres_2.reference b/tests/queries/0_stateless/03036_recursive_cte_postgres_2.reference
index adc06d2edd2..a10e1da6907 100644
--- a/tests/queries/0_stateless/03036_recursive_cte_postgres_2.reference
+++ b/tests/queries/0_stateless/03036_recursive_cte_postgres_2.reference
@@ -32,7 +32,7 @@
 
 -- { echoOn }
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 --
 -- Some examples with a tree
 --
diff --git a/tests/queries/0_stateless/03036_recursive_cte_postgres_2.sql b/tests/queries/0_stateless/03036_recursive_cte_postgres_2.sql
index f06b1c62426..b8e850b4721 100644
--- a/tests/queries/0_stateless/03036_recursive_cte_postgres_2.sql
+++ b/tests/queries/0_stateless/03036_recursive_cte_postgres_2.sql
@@ -32,7 +32,7 @@
 
 -- { echoOn }
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 --
 -- Some examples with a tree
diff --git a/tests/queries/0_stateless/03036_with_numbers.sql b/tests/queries/0_stateless/03036_with_numbers.sql
index 3463ce826e2..bd0f6b6179c 100644
--- a/tests/queries/0_stateless/03036_with_numbers.sql
+++ b/tests/queries/0_stateless/03036_with_numbers.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/13843
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 WITH 10 AS n
 SELECT *
 FROM numbers(n);
diff --git a/tests/queries/0_stateless/03037_recursive_cte_postgres_3.reference b/tests/queries/0_stateless/03037_recursive_cte_postgres_3.reference
index 3f8f47ac23e..f0ff70a7c60 100644
--- a/tests/queries/0_stateless/03037_recursive_cte_postgres_3.reference
+++ b/tests/queries/0_stateless/03037_recursive_cte_postgres_3.reference
@@ -32,7 +32,7 @@
 
 -- { echoOn }
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 --
 -- different tree example
 --
diff --git a/tests/queries/0_stateless/03037_recursive_cte_postgres_3.sql b/tests/queries/0_stateless/03037_recursive_cte_postgres_3.sql
index 3c74ee889c4..213e8bc995d 100644
--- a/tests/queries/0_stateless/03037_recursive_cte_postgres_3.sql
+++ b/tests/queries/0_stateless/03037_recursive_cte_postgres_3.sql
@@ -32,7 +32,7 @@
 
 -- { echoOn }
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 --
 -- different tree example
diff --git a/tests/queries/0_stateless/03037_union_view.sql b/tests/queries/0_stateless/03037_union_view.sql
index 3ea81b829ba..d963444fd91 100644
--- a/tests/queries/0_stateless/03037_union_view.sql
+++ b/tests/queries/0_stateless/03037_union_view.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/55803
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 DROP TABLE IF EXISTS broken_table;
 DROP TABLE IF EXISTS broken_view;
 
@@ -28,4 +28,4 @@ FROM broken_view v FINAL
 WHERE v.start IS NOT NULL;
 
 DROP TABLE IF EXISTS broken_table;
-DROP TABLE IF EXISTS broken_view;
\ No newline at end of file
+DROP TABLE IF EXISTS broken_view;
diff --git a/tests/queries/0_stateless/03038_ambiguous_column.sql b/tests/queries/0_stateless/03038_ambiguous_column.sql
index 9df3cd9bc9b..131bc552f56 100644
--- a/tests/queries/0_stateless/03038_ambiguous_column.sql
+++ b/tests/queries/0_stateless/03038_ambiguous_column.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/48308
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 DROP TABLE IF EXISTS 03038_table;
 
 CREATE TABLE 03038_table
diff --git a/tests/queries/0_stateless/03038_recursive_cte_postgres_4.reference b/tests/queries/0_stateless/03038_recursive_cte_postgres_4.reference
index b920fc298b3..cf070eebc38 100644
--- a/tests/queries/0_stateless/03038_recursive_cte_postgres_4.reference
+++ b/tests/queries/0_stateless/03038_recursive_cte_postgres_4.reference
@@ -32,7 +32,7 @@
 
 -- { echoOn }
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 --
 -- test cycle detection
 --
diff --git a/tests/queries/0_stateless/03038_recursive_cte_postgres_4.sql b/tests/queries/0_stateless/03038_recursive_cte_postgres_4.sql
index a6fe3b1e55c..7dad74893b9 100644
--- a/tests/queries/0_stateless/03038_recursive_cte_postgres_4.sql
+++ b/tests/queries/0_stateless/03038_recursive_cte_postgres_4.sql
@@ -32,7 +32,7 @@
 
 -- { echoOn }
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 --
 -- test cycle detection
diff --git a/tests/queries/0_stateless/03039_recursive_cte_postgres_5.reference b/tests/queries/0_stateless/03039_recursive_cte_postgres_5.reference
index 2910ac174e7..43399a0ab95 100644
--- a/tests/queries/0_stateless/03039_recursive_cte_postgres_5.reference
+++ b/tests/queries/0_stateless/03039_recursive_cte_postgres_5.reference
@@ -32,7 +32,7 @@
 
 -- { echoOn }
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 --
 -- test multiple WITH queries
 --
diff --git a/tests/queries/0_stateless/03039_recursive_cte_postgres_5.sql b/tests/queries/0_stateless/03039_recursive_cte_postgres_5.sql
index e188a455434..eb4043cca42 100644
--- a/tests/queries/0_stateless/03039_recursive_cte_postgres_5.sql
+++ b/tests/queries/0_stateless/03039_recursive_cte_postgres_5.sql
@@ -33,7 +33,7 @@
 
 -- { echoOn }
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 --
 -- test multiple WITH queries
diff --git a/tests/queries/0_stateless/03039_unknown_identifier_window_function.sql b/tests/queries/0_stateless/03039_unknown_identifier_window_function.sql
index 640d217d2f9..652085d9f5a 100644
--- a/tests/queries/0_stateless/03039_unknown_identifier_window_function.sql
+++ b/tests/queries/0_stateless/03039_unknown_identifier_window_function.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/45535
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 
 SELECT
   *,
diff --git a/tests/queries/0_stateless/03040_alias_column_join.sql b/tests/queries/0_stateless/03040_alias_column_join.sql
index 54f579c0feb..6ffd749a6c4 100644
--- a/tests/queries/0_stateless/03040_alias_column_join.sql
+++ b/tests/queries/0_stateless/03040_alias_column_join.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/44365
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 DROP TABLE IF EXISTS 03040_test;
 
 CREATE TABLE 03040_test
diff --git a/tests/queries/0_stateless/03040_array_sum_and_join.sql b/tests/queries/0_stateless/03040_array_sum_and_join.sql
index 9aeddc9f765..90d3d83c9a2 100644
--- a/tests/queries/0_stateless/03040_array_sum_and_join.sql
+++ b/tests/queries/0_stateless/03040_array_sum_and_join.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 
 select t.1 as cnt,
        t.2 as name,
diff --git a/tests/queries/0_stateless/03040_recursive_cte_postgres_6.reference b/tests/queries/0_stateless/03040_recursive_cte_postgres_6.reference
index 2d9d7bb9f6c..7ccc3df9f0d 100644
--- a/tests/queries/0_stateless/03040_recursive_cte_postgres_6.reference
+++ b/tests/queries/0_stateless/03040_recursive_cte_postgres_6.reference
@@ -32,7 +32,7 @@
 
 -- { echoOn }
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 --
 -- error cases
 --
diff --git a/tests/queries/0_stateless/03040_recursive_cte_postgres_6.sql b/tests/queries/0_stateless/03040_recursive_cte_postgres_6.sql
index bff2ece6ece..6a6044198c8 100644
--- a/tests/queries/0_stateless/03040_recursive_cte_postgres_6.sql
+++ b/tests/queries/0_stateless/03040_recursive_cte_postgres_6.sql
@@ -32,7 +32,7 @@
 
 -- { echoOn }
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 --
 -- error cases
diff --git a/tests/queries/0_stateless/03041_analyzer_gigachad_join.sql b/tests/queries/0_stateless/03041_analyzer_gigachad_join.sql
index 7906e65f8b8..88f7fc562b1 100644
--- a/tests/queries/0_stateless/03041_analyzer_gigachad_join.sql
+++ b/tests/queries/0_stateless/03041_analyzer_gigachad_join.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 CREATE TABLE IF NOT EXISTS first engine = MergeTree PARTITION BY (inn, toYYYYMM(received)) ORDER BY (inn, sessionId)
 AS SELECT now() AS received, '123456789' AS inn, '42' AS sessionId;
 
diff --git a/tests/queries/0_stateless/03041_recursive_cte_postgres_7.reference b/tests/queries/0_stateless/03041_recursive_cte_postgres_7.reference
index 6bcc3f89dcb..b98b2ec7e89 100644
--- a/tests/queries/0_stateless/03041_recursive_cte_postgres_7.reference
+++ b/tests/queries/0_stateless/03041_recursive_cte_postgres_7.reference
@@ -32,7 +32,7 @@
 
 -- { echoOn }
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 WITH RECURSIVE foo AS
     (SELECT 1 AS i
     UNION ALL
diff --git a/tests/queries/0_stateless/03041_recursive_cte_postgres_7.sql b/tests/queries/0_stateless/03041_recursive_cte_postgres_7.sql
index 71c34b7361f..5f4455efcc6 100644
--- a/tests/queries/0_stateless/03041_recursive_cte_postgres_7.sql
+++ b/tests/queries/0_stateless/03041_recursive_cte_postgres_7.sql
@@ -32,7 +32,7 @@
 
 -- { echoOn }
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 WITH RECURSIVE foo AS
     (SELECT 1 AS i
diff --git a/tests/queries/0_stateless/03041_select_with_query_result.sql b/tests/queries/0_stateless/03041_select_with_query_result.sql
index 061223b43e1..e5897ea12cf 100644
--- a/tests/queries/0_stateless/03041_select_with_query_result.sql
+++ b/tests/queries/0_stateless/03041_select_with_query_result.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/44153
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 DROP TABLE IF EXISTS parent;
 DROP TABLE IF EXISTS join_table_1;
 DROP TABLE IF EXISTS join_table_2;
diff --git a/tests/queries/0_stateless/03042_analyzer_alias_join.sql b/tests/queries/0_stateless/03042_analyzer_alias_join.sql
index dac3b6a4983..d9a8d8b4c7b 100644
--- a/tests/queries/0_stateless/03042_analyzer_alias_join.sql
+++ b/tests/queries/0_stateless/03042_analyzer_alias_join.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/14978
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 CREATE TABLE test1(id UInt64, t1value UInt64) ENGINE=MergeTree ORDER BY tuple();
 CREATE TABLE test2(id UInt64, t2value String) ENGINE=MergeTree ORDER BY tuple();
 
diff --git a/tests/queries/0_stateless/03042_not_found_column_c1.sql b/tests/queries/0_stateless/03042_not_found_column_c1.sql
index b4dce2af489..08202dc0dca 100644
--- a/tests/queries/0_stateless/03042_not_found_column_c1.sql
+++ b/tests/queries/0_stateless/03042_not_found_column_c1.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/42399
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 
 CREATE TABLE IF NOT EXISTS t0 (c0 Int32) ENGINE = Memory() ;
 CREATE TABLE t1 (c0 Int32, c1 Int32, c2 Int32) ENGINE = Memory() ;
diff --git a/tests/queries/0_stateless/03043_group_array_result_is_expected.sql b/tests/queries/0_stateless/03043_group_array_result_is_expected.sql
index 5311927ae3c..e2c79e5c41e 100644
--- a/tests/queries/0_stateless/03043_group_array_result_is_expected.sql
+++ b/tests/queries/0_stateless/03043_group_array_result_is_expected.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/27115
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 drop table if exists fill_ex;
 
 create table fill_ex (
diff --git a/tests/queries/0_stateless/03044_analyzer_alias_join.sql b/tests/queries/0_stateless/03044_analyzer_alias_join.sql
index 3ab8edb005f..7636edbb411 100644
--- a/tests/queries/0_stateless/03044_analyzer_alias_join.sql
+++ b/tests/queries/0_stateless/03044_analyzer_alias_join.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/17319
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 CREATE TEMPORARY TABLE hits (date Date, data Float64) engine=Memory();
 
 SELECT
diff --git a/tests/queries/0_stateless/03044_array_join_columns_in_nested_table.sql b/tests/queries/0_stateless/03044_array_join_columns_in_nested_table.sql
index 0cf05763202..4885b7e3f81 100644
--- a/tests/queries/0_stateless/03044_array_join_columns_in_nested_table.sql
+++ b/tests/queries/0_stateless/03044_array_join_columns_in_nested_table.sql
@@ -1,3 +1,3 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/11813
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 select 1 from (select 1 x) l join (select 1 y, [1] a) r on l.x = r.y array join r.a;
diff --git a/tests/queries/0_stateless/03045_analyzer_alias_join_with_if.sql b/tests/queries/0_stateless/03045_analyzer_alias_join_with_if.sql
index ee8756b9460..cbc46726467 100644
--- a/tests/queries/0_stateless/03045_analyzer_alias_join_with_if.sql
+++ b/tests/queries/0_stateless/03045_analyzer_alias_join_with_if.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/13210
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 CREATE TABLE test_a_table (
     name String,
     a_col String
diff --git a/tests/queries/0_stateless/03045_unknown_identifier_alias_substitution.sql b/tests/queries/0_stateless/03045_unknown_identifier_alias_substitution.sql
index d97dfc880b3..967b7b24787 100644
--- a/tests/queries/0_stateless/03045_unknown_identifier_alias_substitution.sql
+++ b/tests/queries/0_stateless/03045_unknown_identifier_alias_substitution.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/23053
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 DROP TABLE IF EXISTS repl_tbl;
 
 CREATE TEMPORARY TABLE repl_tbl
diff --git a/tests/queries/0_stateless/03046_column_in_block_array_join.sql b/tests/queries/0_stateless/03046_column_in_block_array_join.sql
index c6b4613af3f..f91a18da831 100644
--- a/tests/queries/0_stateless/03046_column_in_block_array_join.sql
+++ b/tests/queries/0_stateless/03046_column_in_block_array_join.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/37729
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 
 DROP TABLE IF EXISTS nested_test;
 DROP TABLE IF EXISTS join_test;
diff --git a/tests/queries/0_stateless/03047_analyzer_alias_join.sql b/tests/queries/0_stateless/03047_analyzer_alias_join.sql
index 7d44c92b6f1..29fc711aaf4 100644
--- a/tests/queries/0_stateless/03047_analyzer_alias_join.sql
+++ b/tests/queries/0_stateless/03047_analyzer_alias_join.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 SELECT
     1 AS value,
     *
diff --git a/tests/queries/0_stateless/03047_group_by_field_identified_aggregation.sql b/tests/queries/0_stateless/03047_group_by_field_identified_aggregation.sql
index cfaf1df44bd..d12e85ca03d 100644
--- a/tests/queries/0_stateless/03047_group_by_field_identified_aggregation.sql
+++ b/tests/queries/0_stateless/03047_group_by_field_identified_aggregation.sql
@@ -1,4 +1,4 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/32639
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 
 SELECT 0 AND id ? 1 : 2 AS a, sum(id) FROM (SELECT 1 AS id) GROUP BY a;
diff --git a/tests/queries/0_stateless/03048_not_found_column_xxx_in_block.sql b/tests/queries/0_stateless/03048_not_found_column_xxx_in_block.sql
index 42fd581e142..f511ea81e26 100644
--- a/tests/queries/0_stateless/03048_not_found_column_xxx_in_block.sql
+++ b/tests/queries/0_stateless/03048_not_found_column_xxx_in_block.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/41964
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 
 DROP TABLE IF EXISTS ab_12_aaa;
 DROP TABLE IF EXISTS ab_12_bbb;
diff --git a/tests/queries/0_stateless/03049_analyzer_group_by_alias.sql b/tests/queries/0_stateless/03049_analyzer_group_by_alias.sql
index d25babe6788..712d1c27e6e 100644
--- a/tests/queries/0_stateless/03049_analyzer_group_by_alias.sql
+++ b/tests/queries/0_stateless/03049_analyzer_group_by_alias.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/7520
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 CREATE TABLE test (`a` UInt32, `b` UInt32) ENGINE = Memory;
 
 INSERT INTO test VALUES (1,2), (1,3), (2,4);
diff --git a/tests/queries/0_stateless/03049_unknown_identifier_materialized_column.sql b/tests/queries/0_stateless/03049_unknown_identifier_materialized_column.sql
index 938f270b9e4..0efe59a1f1c 100644
--- a/tests/queries/0_stateless/03049_unknown_identifier_materialized_column.sql
+++ b/tests/queries/0_stateless/03049_unknown_identifier_materialized_column.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/54317
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 DROP DATABASE IF EXISTS {CLICKHOUSE_DATABASE:Identifier};
 
 CREATE DATABASE {CLICKHOUSE_DATABASE:Identifier};
diff --git a/tests/queries/0_stateless/03050_select_one_one_one.sql b/tests/queries/0_stateless/03050_select_one_one_one.sql
index 28a55e0c471..09f3f20c35d 100644
--- a/tests/queries/0_stateless/03050_select_one_one_one.sql
+++ b/tests/queries/0_stateless/03050_select_one_one_one.sql
@@ -1,4 +1,4 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/36973
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 SELECT 1, 1, 1;
 SELECT * FROM (SELECT 1, 1, 1);
diff --git a/tests/queries/0_stateless/03051_many_ctes.sql b/tests/queries/0_stateless/03051_many_ctes.sql
index d4e613bd279..e442813b6a2 100644
--- a/tests/queries/0_stateless/03051_many_ctes.sql
+++ b/tests/queries/0_stateless/03051_many_ctes.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/40955
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 WITH toInt64(2) AS new_x SELECT new_x AS x FROM (SELECT 1 AS x) t;
 WITH toInt64(2) AS new_x SELECT * replace(new_x as x)  FROM (SELECT 1 AS x) t;
 SELECT 2 AS x FROM (SELECT 1 AS x) t;
diff --git a/tests/queries/0_stateless/03052_query_hash_includes_aliases.sql b/tests/queries/0_stateless/03052_query_hash_includes_aliases.sql
index 24e9ab0f36e..55993175bf3 100644
--- a/tests/queries/0_stateless/03052_query_hash_includes_aliases.sql
+++ b/tests/queries/0_stateless/03052_query_hash_includes_aliases.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/pull/40065
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 
 SELECT
 (
diff --git a/tests/queries/0_stateless/03053_analyzer_join_alias.sql b/tests/queries/0_stateless/03053_analyzer_join_alias.sql
index 894b8af7c6f..677cf9d4d5e 100644
--- a/tests/queries/0_stateless/03053_analyzer_join_alias.sql
+++ b/tests/queries/0_stateless/03053_analyzer_join_alias.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/23104
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 DROP DATABASE IF EXISTS {CLICKHOUSE_DATABASE:Identifier};
 CREATE DATABASE {CLICKHOUSE_DATABASE:Identifier};
 
diff --git a/tests/queries/0_stateless/03054_analyzer_join_alias.sql b/tests/queries/0_stateless/03054_analyzer_join_alias.sql
index e124aa33a9b..f018f57cc6f 100644
--- a/tests/queries/0_stateless/03054_analyzer_join_alias.sql
+++ b/tests/queries/0_stateless/03054_analyzer_join_alias.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/21584
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 SELECT count()
 FROM
 (
diff --git a/tests/queries/0_stateless/03055_analyzer_subquery_group_array.sql b/tests/queries/0_stateless/03055_analyzer_subquery_group_array.sql
index 25b6dcb3564..29ba1dd7c5b 100644
--- a/tests/queries/0_stateless/03055_analyzer_subquery_group_array.sql
+++ b/tests/queries/0_stateless/03055_analyzer_subquery_group_array.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/23344
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 SELECT logTrace(repeat('Hello', 100)), ignore(*)
 FROM (
     SELECT ignore((SELECT groupArrayState(([number], [number])) FROM numbers(19000)))
diff --git a/tests/queries/0_stateless/03057_analyzer_subquery_alias_join.sql b/tests/queries/0_stateless/03057_analyzer_subquery_alias_join.sql
index 2217af327fa..92f603ed595 100644
--- a/tests/queries/0_stateless/03057_analyzer_subquery_alias_join.sql
+++ b/tests/queries/0_stateless/03057_analyzer_subquery_alias_join.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/10276
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 SELECT
     sum(x.n) as n,
     sum(z.n) as n2
diff --git a/tests/queries/0_stateless/03058_analyzer_ambiguous_columns.sql b/tests/queries/0_stateless/03058_analyzer_ambiguous_columns.sql
index 3cce77f0240..ef3c0e5f63d 100644
--- a/tests/queries/0_stateless/03058_analyzer_ambiguous_columns.sql
+++ b/tests/queries/0_stateless/03058_analyzer_ambiguous_columns.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/4567
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 DROP TABLE IF EXISTS fact;
 DROP TABLE IF EXISTS animals;
 DROP TABLE IF EXISTS colors;
@@ -23,4 +23,3 @@ select  id, animal_name, animal_key, color_name, color_key
 from fact a
         left join (select toInt64(animal_key) animal_key, animal_name from animals) b on (a.animal_key = b.animal_key)
         left join (select toInt64(color_key) color_key, color_name from colors) c on (a.color_key = c.color_key);  -- { serverError AMBIGUOUS_IDENTIFIER }
-
diff --git a/tests/queries/0_stateless/03059_analyzer_join_engine_missing_column.sql b/tests/queries/0_stateless/03059_analyzer_join_engine_missing_column.sql
index 27782462075..164a42e5ba3 100644
--- a/tests/queries/0_stateless/03059_analyzer_join_engine_missing_column.sql
+++ b/tests/queries/0_stateless/03059_analyzer_join_engine_missing_column.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/17710
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 CREATE TABLE id_val(id UInt32, val UInt32) ENGINE = Memory;
 CREATE TABLE id_val_join0(id UInt32, val UInt8) ENGINE = Join(ANY, LEFT, id) SETTINGS join_use_nulls = 0;
 CREATE TABLE id_val_join1(id UInt32, val UInt8) ENGINE = Join(ANY, LEFT, id) SETTINGS join_use_nulls = 1;
diff --git a/tests/queries/0_stateless/03060_analyzer_regular_view_alias.sql b/tests/queries/0_stateless/03060_analyzer_regular_view_alias.sql
index f8cd8690ee5..0556683b97a 100644
--- a/tests/queries/0_stateless/03060_analyzer_regular_view_alias.sql
+++ b/tests/queries/0_stateless/03060_analyzer_regular_view_alias.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/11068
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 create table vt(datetime_value DateTime, value Float64) Engine=Memory;
 
 create view computed_datum_hours as
diff --git a/tests/queries/0_stateless/03061_analyzer_alias_as_right_key_in_join.sql b/tests/queries/0_stateless/03061_analyzer_alias_as_right_key_in_join.sql
index 6fee6d1f73d..a1b50967a46 100644
--- a/tests/queries/0_stateless/03061_analyzer_alias_as_right_key_in_join.sql
+++ b/tests/queries/0_stateless/03061_analyzer_alias_as_right_key_in_join.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/24395
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 CREATE TABLE xxxx_yyy (key UInt32, key_b ALIAS key) ENGINE=MergeTree() ORDER BY key;
 INSERT INTO xxxx_yyy SELECT number FROM numbers(10);
 
diff --git a/tests/queries/0_stateless/03062_analyzer_join_engine_missing_column.sql b/tests/queries/0_stateless/03062_analyzer_join_engine_missing_column.sql
index 9748175e4d4..487d74b3317 100644
--- a/tests/queries/0_stateless/03062_analyzer_join_engine_missing_column.sql
+++ b/tests/queries/0_stateless/03062_analyzer_join_engine_missing_column.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/23416
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 create table test (TOPIC String, PARTITION UInt64, OFFSET UInt64, ID UInt64) ENGINE ReplicatedMergeTree('/clickhouse/tables/{database}/test_03062', 'r2') ORDER BY (TOPIC, PARTITION, OFFSET);
 
 create table test_join (TOPIC String, PARTITION UInt64, OFFSET UInt64)  ENGINE = Join(ANY, LEFT, `TOPIC`, `PARTITION`) SETTINGS join_any_take_last_row = 1;
diff --git a/tests/queries/0_stateless/03063_analyzer_multi_join_wrong_table_specifier.sql b/tests/queries/0_stateless/03063_analyzer_multi_join_wrong_table_specifier.sql
index 7eab1fa846a..5655d4a0110 100644
--- a/tests/queries/0_stateless/03063_analyzer_multi_join_wrong_table_specifier.sql
+++ b/tests/queries/0_stateless/03063_analyzer_multi_join_wrong_table_specifier.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/23162
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 CREATE TABLE t1 ( k Int64, x Int64) ENGINE = Memory;
 
 CREATE TABLE t2( x Int64 ) ENGINE = Memory;
@@ -14,4 +14,3 @@ WHERE (t1.d >= now()); -- { serverError UNKNOWN_IDENTIFIER }
 SELECT * FROM t1
 INNER JOIN s ON t1.k = s.k
 WHERE (t1.d >= now()); -- { serverError UNKNOWN_IDENTIFIER }
-
diff --git a/tests/queries/0_stateless/03064_analyzer_named_subqueries.sql b/tests/queries/0_stateless/03064_analyzer_named_subqueries.sql
index 59ebb9d9af3..d5696411091 100644
--- a/tests/queries/0_stateless/03064_analyzer_named_subqueries.sql
+++ b/tests/queries/0_stateless/03064_analyzer_named_subqueries.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/25655
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 SELECT
   sum(t.b) / 1 a,
   sum(t.a)
diff --git a/tests/queries/0_stateless/03065_analyzer_cross_join_and_array_join.sql b/tests/queries/0_stateless/03065_analyzer_cross_join_and_array_join.sql
index 7e6befe181e..5034e2eed8f 100644
--- a/tests/queries/0_stateless/03065_analyzer_cross_join_and_array_join.sql
+++ b/tests/queries/0_stateless/03065_analyzer_cross_join_and_array_join.sql
@@ -1,3 +1,3 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/11757
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 select * from (select [1, 2] a) aa cross join (select [3, 4] b) bb array join aa.a, bb.b;
diff --git a/tests/queries/0_stateless/03066_analyzer_global_with_statement.sql b/tests/queries/0_stateless/03066_analyzer_global_with_statement.sql
index 8983be242c3..2b879ed73da 100644
--- a/tests/queries/0_stateless/03066_analyzer_global_with_statement.sql
+++ b/tests/queries/0_stateless/03066_analyzer_global_with_statement.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 WITH 0 AS test
 SELECT *
 FROM
diff --git a/tests/queries/0_stateless/03067_analyzer_complex_alias_join.sql b/tests/queries/0_stateless/03067_analyzer_complex_alias_join.sql
index 052a9eaf734..58845b93771 100644
--- a/tests/queries/0_stateless/03067_analyzer_complex_alias_join.sql
+++ b/tests/queries/0_stateless/03067_analyzer_complex_alias_join.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 with d as (select 'key'::Varchar(255) c, 'x'::Varchar(255) s)
 SELECT r1, c as r2
 FROM (
diff --git a/tests/queries/0_stateless/03068_analyzer_distributed_join.sql b/tests/queries/0_stateless/03068_analyzer_distributed_join.sql
index 542380feb7c..459c8f5e8ac 100644
--- a/tests/queries/0_stateless/03068_analyzer_distributed_join.sql
+++ b/tests/queries/0_stateless/03068_analyzer_distributed_join.sql
@@ -1,7 +1,7 @@
 -- Tags: no-replicated-database
 -- Closes: https://github.com/ClickHouse/ClickHouse/issues/6571
 
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 CREATE TABLE LINEITEM_shard ON CLUSTER test_shard_localhost
 (
     L_ORDERKEY UInt64,
diff --git a/tests/queries/0_stateless/03069_analyzer_with_alias_in_array_join.sql b/tests/queries/0_stateless/03069_analyzer_with_alias_in_array_join.sql
index 09d2985fe60..5ec04cbc0f5 100644
--- a/tests/queries/0_stateless/03069_analyzer_with_alias_in_array_join.sql
+++ b/tests/queries/0_stateless/03069_analyzer_with_alias_in_array_join.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/4432
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 WITH [1, 2] AS zz
 SELECT x
 FROM system.one
diff --git a/tests/queries/0_stateless/03070_analyzer_CTE_scalar_as_numbers.sql b/tests/queries/0_stateless/03070_analyzer_CTE_scalar_as_numbers.sql
index 7aadab2ca73..a94ae811476 100644
--- a/tests/queries/0_stateless/03070_analyzer_CTE_scalar_as_numbers.sql
+++ b/tests/queries/0_stateless/03070_analyzer_CTE_scalar_as_numbers.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/8259
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 with
     (select 25) as something
 select *, something
diff --git a/tests/queries/0_stateless/03071_analyzer_array_join_forbid_non_existing_columns.sql b/tests/queries/0_stateless/03071_analyzer_array_join_forbid_non_existing_columns.sql
index e2eb758d649..211fa2a3119 100644
--- a/tests/queries/0_stateless/03071_analyzer_array_join_forbid_non_existing_columns.sql
+++ b/tests/queries/0_stateless/03071_analyzer_array_join_forbid_non_existing_columns.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/9233
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 SELECT *
 FROM
 (
diff --git a/tests/queries/0_stateless/03071_fix_short_circuit_logic.sql b/tests/queries/0_stateless/03071_fix_short_circuit_logic.sql
index 7745bceca0b..dc5fb5085fb 100644
--- a/tests/queries/0_stateless/03071_fix_short_circuit_logic.sql
+++ b/tests/queries/0_stateless/03071_fix_short_circuit_logic.sql
@@ -59,4 +59,4 @@ FROM
     )
     WHERE (address = hex2bytes('0xd387a6e4e84a6c86bd90c158c6028a58cc8ac459')) AND (transfer_id NOT LIKE 'gas%') AND (value > 0) AND (dictGetOrDefault(token_data_map, 'is_blacklisted', (token_address_hex, 'zksync'), true))
 )
-SETTINGS max_threads = 1, short_circuit_function_evaluation = 'enable', allow_experimental_analyzer = 0;
\ No newline at end of file
+SETTINGS max_threads = 1, short_circuit_function_evaluation = 'enable', enable_analyzer = 0;
diff --git a/tests/queries/0_stateless/03072_analyzer_missing_columns_from_subquery.sql b/tests/queries/0_stateless/03072_analyzer_missing_columns_from_subquery.sql
index e2846033913..ec3b067cbdf 100644
--- a/tests/queries/0_stateless/03072_analyzer_missing_columns_from_subquery.sql
+++ b/tests/queries/0_stateless/03072_analyzer_missing_columns_from_subquery.sql
@@ -1,3 +1,3 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/14699
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 select * from (select number from numbers(1)) where not ignore(*);
diff --git a/tests/queries/0_stateless/03073_analyzer_alias_as_column_name.sql b/tests/queries/0_stateless/03073_analyzer_alias_as_column_name.sql
index 5599324c62b..bba51e28ba3 100644
--- a/tests/queries/0_stateless/03073_analyzer_alias_as_column_name.sql
+++ b/tests/queries/0_stateless/03073_analyzer_alias_as_column_name.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/27068
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 CREATE TABLE test ( id String, create_time DateTime ) ENGINE = MergeTree ORDER BY id;
 
 insert into test values(1,'1970-02-01 00:00:00');
diff --git a/tests/queries/0_stateless/03074_analyzer_alias_column_in_view.sql b/tests/queries/0_stateless/03074_analyzer_alias_column_in_view.sql
index 4df5f6f48e6..314b6c0e8d6 100644
--- a/tests/queries/0_stateless/03074_analyzer_alias_column_in_view.sql
+++ b/tests/queries/0_stateless/03074_analyzer_alias_column_in_view.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/28687
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 create view alias (dummy int, n alias dummy) as select * from system.one;
 
 select n from alias;
diff --git a/tests/queries/0_stateless/03075_analyzer_subquery_alias.sql b/tests/queries/0_stateless/03075_analyzer_subquery_alias.sql
index 416815e761b..4f097350da6 100644
--- a/tests/queries/0_stateless/03075_analyzer_subquery_alias.sql
+++ b/tests/queries/0_stateless/03075_analyzer_subquery_alias.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/28777
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 SELECT
     sum(q0.a2) AS a1,
     sum(q0.a1) AS a9
diff --git a/tests/queries/0_stateless/03076_analyzer_multiple_joins_alias.sql b/tests/queries/0_stateless/03076_analyzer_multiple_joins_alias.sql
index 7ac9fe6b446..894e3bc56ee 100644
--- a/tests/queries/0_stateless/03076_analyzer_multiple_joins_alias.sql
+++ b/tests/queries/0_stateless/03076_analyzer_multiple_joins_alias.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/29734
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 SELECT *
 FROM
 (
@@ -49,4 +49,3 @@ INNER JOIN
     SELECT number AS x
     FROM numbers(10)
 ) AS c ON a.x = c.x; -- { serverError UNKNOWN_IDENTIFIER }
-
diff --git a/tests/queries/0_stateless/03077_analyzer_multi_scalar_subquery_aliases.sql b/tests/queries/0_stateless/03077_analyzer_multi_scalar_subquery_aliases.sql
index 5a181023c57..d4335d35e51 100644
--- a/tests/queries/0_stateless/03077_analyzer_multi_scalar_subquery_aliases.sql
+++ b/tests/queries/0_stateless/03077_analyzer_multi_scalar_subquery_aliases.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/33825
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 CREATE TABLE t1 (i Int64, j Int64) ENGINE = Memory;
 INSERT INTO t1 SELECT number, number FROM system.numbers LIMIT 10;
 SELECT
diff --git a/tests/queries/0_stateless/03078_analyzer_multi_scalar_subquery_aliases.sql b/tests/queries/0_stateless/03078_analyzer_multi_scalar_subquery_aliases.sql
index d91a9ed106d..b9b850619ea 100644
--- a/tests/queries/0_stateless/03078_analyzer_multi_scalar_subquery_aliases.sql
+++ b/tests/queries/0_stateless/03078_analyzer_multi_scalar_subquery_aliases.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/33825
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 CREATE TABLE t2 (first_column Int64, second_column Int64) ENGINE = Memory;
 INSERT INTO t2 SELECT number, number FROM system.numbers LIMIT 10;
 
diff --git a/tests/queries/0_stateless/03079_analyzer_numeric_literals_as_column_names.sql b/tests/queries/0_stateless/03079_analyzer_numeric_literals_as_column_names.sql
index 955d3b49a00..80e681c0776 100644
--- a/tests/queries/0_stateless/03079_analyzer_numeric_literals_as_column_names.sql
+++ b/tests/queries/0_stateless/03079_analyzer_numeric_literals_as_column_names.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 CREATE TABLE testdata (`1` String) ENGINE=MergeTree ORDER BY tuple();
 INSERT INTO testdata VALUES ('testdata');
 
diff --git a/tests/queries/0_stateless/03080_analyzer_prefer_column_name_to_alias__virtual_columns.sql b/tests/queries/0_stateless/03080_analyzer_prefer_column_name_to_alias__virtual_columns.sql
index 01ab868f9ea..2138828cd27 100644
--- a/tests/queries/0_stateless/03080_analyzer_prefer_column_name_to_alias__virtual_columns.sql
+++ b/tests/queries/0_stateless/03080_analyzer_prefer_column_name_to_alias__virtual_columns.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/35652
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 CREATE TABLE test (
   id UInt64
 )
diff --git a/tests/queries/0_stateless/03080_incorrect_join_with_merge.sql b/tests/queries/0_stateless/03080_incorrect_join_with_merge.sql
index 7682e6ce866..a34c71a44e2 100644
--- a/tests/queries/0_stateless/03080_incorrect_join_with_merge.sql
+++ b/tests/queries/0_stateless/03080_incorrect_join_with_merge.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/29838
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 SET distributed_foreground_insert=1;
 
 CREATE TABLE first_table_lr
diff --git a/tests/queries/0_stateless/03081_analyzer_agg_func_CTE.sql b/tests/queries/0_stateless/03081_analyzer_agg_func_CTE.sql
index e6a540dc5df..3cb02512a7f 100644
--- a/tests/queries/0_stateless/03081_analyzer_agg_func_CTE.sql
+++ b/tests/queries/0_stateless/03081_analyzer_agg_func_CTE.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/36189
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 CREATE TABLE test
 (
     `dt` Date,
diff --git a/tests/queries/0_stateless/03082_analyzer_left_join_correct_column.sql b/tests/queries/0_stateless/03082_analyzer_left_join_correct_column.sql
index 8f17248ed0d..3b83f978326 100644
--- a/tests/queries/0_stateless/03082_analyzer_left_join_correct_column.sql
+++ b/tests/queries/0_stateless/03082_analyzer_left_join_correct_column.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/39634
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 CREATE TABLE test1
 (
     `pk` String,
diff --git a/tests/queries/0_stateless/03084_analyzer_join_column_alias.sql b/tests/queries/0_stateless/03084_analyzer_join_column_alias.sql
index 930726898b5..8a7258f5838 100644
--- a/tests/queries/0_stateless/03084_analyzer_join_column_alias.sql
+++ b/tests/queries/0_stateless/03084_analyzer_join_column_alias.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/47432
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 create table t1
 engine = MergeTree()
 order by tuple()
diff --git a/tests/queries/0_stateless/03085_analyzer_alias_column_group_by.sql b/tests/queries/0_stateless/03085_analyzer_alias_column_group_by.sql
index fd67194b08b..c360e86197f 100644
--- a/tests/queries/0_stateless/03085_analyzer_alias_column_group_by.sql
+++ b/tests/queries/0_stateless/03085_analyzer_alias_column_group_by.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/54910
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 SELECT toTypeName(stat_standard_id) AS stat_standard_id_1, count(1) AS value
 FROM ( SELECT 'string value' AS stat_standard_id )
 GROUP BY stat_standard_id_1 LIMIT 1
diff --git a/tests/queries/0_stateless/03086_analyzer_window_func_part_of_group_by.sql b/tests/queries/0_stateless/03086_analyzer_window_func_part_of_group_by.sql
index 31747328d1f..7e44b37f865 100644
--- a/tests/queries/0_stateless/03086_analyzer_window_func_part_of_group_by.sql
+++ b/tests/queries/0_stateless/03086_analyzer_window_func_part_of_group_by.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/57321
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 SELECT
     ver,
     max(ver) OVER () AS ver_max
diff --git a/tests/queries/0_stateless/03087_analyzer_subquery_with_alias.sql b/tests/queries/0_stateless/03087_analyzer_subquery_with_alias.sql
index 6546e50c99e..a00ca4960d7 100644
--- a/tests/queries/0_stateless/03087_analyzer_subquery_with_alias.sql
+++ b/tests/queries/0_stateless/03087_analyzer_subquery_with_alias.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/59154
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 SELECT *
 FROM
 (
diff --git a/tests/queries/0_stateless/03088_analyzer_ambiguous_column_multi_call.sql b/tests/queries/0_stateless/03088_analyzer_ambiguous_column_multi_call.sql
index e6f1ed81f91..3670404d124 100644
--- a/tests/queries/0_stateless/03088_analyzer_ambiguous_column_multi_call.sql
+++ b/tests/queries/0_stateless/03088_analyzer_ambiguous_column_multi_call.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/61014
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 
 DROP DATABASE IF EXISTS {CLICKHOUSE_DATABASE:Identifier};
 create database {CLICKHOUSE_DATABASE:Identifier};
diff --git a/tests/queries/0_stateless/03089_analyzer_alias_replacement.sql b/tests/queries/0_stateless/03089_analyzer_alias_replacement.sql
index 069da5fdd65..5526e1aaf7d 100644
--- a/tests/queries/0_stateless/03089_analyzer_alias_replacement.sql
+++ b/tests/queries/0_stateless/03089_analyzer_alias_replacement.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/61950
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 
 with dummy + 1 as dummy select dummy from system.one;
 
diff --git a/tests/queries/0_stateless/03090_analyzer_multiple_using_statements.sql b/tests/queries/0_stateless/03090_analyzer_multiple_using_statements.sql
index c35f33782ff..08ea103d3c9 100644
--- a/tests/queries/0_stateless/03090_analyzer_multiple_using_statements.sql
+++ b/tests/queries/0_stateless/03090_analyzer_multiple_using_statements.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/55647
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 
 SELECT
 *
diff --git a/tests/queries/0_stateless/03091_analyzer_same_table_name_in_different_databases.sql b/tests/queries/0_stateless/03091_analyzer_same_table_name_in_different_databases.sql
index 599275c66e8..11984aec496 100644
--- a/tests/queries/0_stateless/03091_analyzer_same_table_name_in_different_databases.sql
+++ b/tests/queries/0_stateless/03091_analyzer_same_table_name_in_different_databases.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/61947
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 
 DROP DATABASE IF EXISTS {CLICKHOUSE_DATABASE:Identifier};
 DROP DATABASE IF EXISTS {CLICKHOUSE_DATABASE_1:Identifier};
diff --git a/tests/queries/0_stateless/03092_analyzer_same_table_name_in_different_databases.sql b/tests/queries/0_stateless/03092_analyzer_same_table_name_in_different_databases.sql
index 10d18324c3c..83b1a902721 100644
--- a/tests/queries/0_stateless/03092_analyzer_same_table_name_in_different_databases.sql
+++ b/tests/queries/0_stateless/03092_analyzer_same_table_name_in_different_databases.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/61947
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 
 DROP DATABASE IF EXISTS {CLICKHOUSE_DATABASE:Identifier};
 DROP DATABASE IF EXISTS {CLICKHOUSE_DATABASE_1:Identifier};
diff --git a/tests/queries/0_stateless/03093_analyzer_column_alias.sql b/tests/queries/0_stateless/03093_analyzer_column_alias.sql
index 9ff0f78ba24..edf89108b56 100644
--- a/tests/queries/0_stateless/03093_analyzer_column_alias.sql
+++ b/tests/queries/0_stateless/03093_analyzer_column_alias.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/26674
-SET allow_experimental_analyzer = true;
+SET enable_analyzer = true;
 
 SELECT
     Carrier,
diff --git a/tests/queries/0_stateless/03093_analyzer_miel_test.sql b/tests/queries/0_stateless/03093_analyzer_miel_test.sql
index f408882dcd5..4915864bb51 100644
--- a/tests/queries/0_stateless/03093_analyzer_miel_test.sql
+++ b/tests/queries/0_stateless/03093_analyzer_miel_test.sql
@@ -8,7 +8,7 @@ INSERT INTO test_03093 VALUES ('x1', 123, {'k1': ''});
 INSERT INTO test_03093 VALUES ('x1', 123, {'k1': '', 'k11': ''});
 INSERT INTO test_03093 VALUES ('x1', 12,  {'k1': ''});
 
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 
 select app, arrayZip(untuple(sumMap(k.keys, replicate(1, k.keys)))) from test_03093 PREWHERE c > 1 group by app;
 select app, arrayZip(untuple(sumMap(k.keys, replicate(1, k.keys)))) from test_03093 WHERE c > 1 group by app;
diff --git a/tests/queries/0_stateless/03093_bug37909_query_does_not_finish.sql b/tests/queries/0_stateless/03093_bug37909_query_does_not_finish.sql
index 62fa3f437af..90f0a8a3b8e 100644
--- a/tests/queries/0_stateless/03093_bug37909_query_does_not_finish.sql
+++ b/tests/queries/0_stateless/03093_bug37909_query_does_not_finish.sql
@@ -75,4 +75,4 @@ FROM
 /* WHERE (v_date >= '2022-05-08') AND (v_date <= '2022-06-07') placing condition has same effect */
 GROUP BY vDate
 ORDER BY vDate ASC
-SETTINGS allow_experimental_analyzer = 1; -- the query times out if allow_experimental_analyzer = 0
+SETTINGS enable_analyzer = 1; -- the query times out if enable_analyzer = 0
diff --git a/tests/queries/0_stateless/03094_analyzer_fiddle_multiif.sql b/tests/queries/0_stateless/03094_analyzer_fiddle_multiif.sql
index 1b1603be18e..8426749917a 100644
--- a/tests/queries/0_stateless/03094_analyzer_fiddle_multiif.sql
+++ b/tests/queries/0_stateless/03094_analyzer_fiddle_multiif.sql
@@ -5,7 +5,7 @@ INSERT INTO users_03094 VALUES ('John', 33);
 INSERT INTO users_03094 VALUES ('Ksenia', 48);
 INSERT INTO users_03094 VALUES ('Alice', 50);
 
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 
 SELECT
     multiIf((age > 30) or (true), '1', '2') AS a,
diff --git a/tests/queries/0_stateless/03094_named_tuple_bug24607.sql b/tests/queries/0_stateless/03094_named_tuple_bug24607.sql
index e3c97f3fe41..698c339e53e 100644
--- a/tests/queries/0_stateless/03094_named_tuple_bug24607.sql
+++ b/tests/queries/0_stateless/03094_named_tuple_bug24607.sql
@@ -1,4 +1,4 @@
 SELECT
     JSONExtract('{"a":1, "b":"test"}', 'Tuple(a UInt8, b String)') AS x,
     x.a
-SETTINGS allow_experimental_analyzer = 1;
+SETTINGS enable_analyzer = 1;
diff --git a/tests/queries/0_stateless/03094_one_thousand_joins.sql b/tests/queries/0_stateless/03094_one_thousand_joins.sql
index 1f6bd99df7f..6ae4e4d4d3c 100644
--- a/tests/queries/0_stateless/03094_one_thousand_joins.sql
+++ b/tests/queries/0_stateless/03094_one_thousand_joins.sql
@@ -2,7 +2,7 @@
 -- (no-tsan because it has a small maximum stack size and the test would fail with TOO_DEEP_RECURSION)
 
 SET join_algorithm = 'default'; -- for 'full_sorting_merge' the query is 10x slower
-SET allow_experimental_analyzer = 1; -- old analyzer returns TOO_DEEP_SUBQUERIES
+SET enable_analyzer = 1; -- old analyzer returns TOO_DEEP_SUBQUERIES
 
 -- Bug 33446, marked as 'long' because it still runs around 10 sec
 SELECT * FROM (SELECT 1 AS x) t1 JOIN (SELECT 1 AS x) t2 ON t1.x = t2.x JOIN (SELECT 1 AS x) t3 ON t1.x = t3.x JOIN (SELECT 1 AS x) t4 ON t1.x = t4.x JOIN (SELECT 1 AS x) t5 ON t1.x = t5.x JOIN (SELECT 1 AS x) t6 ON t1.x = t6.x JOIN (SELECT 1 AS x) t7 ON t1.x = t7.x JOIN (SELECT 1 AS x) t8 ON t1.x = t8.x JOIN (SELECT 1 AS x) t9 ON t1.x = t9.x JOIN (SELECT 1 AS x) t10 ON t1.x = t10.x JOIN (SELECT 1 AS x) t11 ON t1.x = t11.x JOIN (SELECT 1 AS x) t12 ON t1.x = t12.x JOIN (SELECT 1 AS x) t13 ON t1.x = t13.x JOIN (SELECT 1 AS x) t14 ON t1.x = t14.x JOIN (SELECT 1 AS x) t15 ON t1.x = t15.x JOIN (SELECT 1 AS x) t16 ON t1.x = t16.x JOIN (SELECT 1 AS x) t17 ON t1.x = t17.x JOIN (SELECT 1 AS x) t18 ON t1.x = t18.x JOIN (SELECT 1 AS x) t19 ON t1.x = t19.x JOIN (SELECT 1 AS x) t20 ON t1.x = t20.x JOIN (SELECT 1 AS x) t21 ON t1.x = t21.x JOIN (SELECT 1 AS x) t22 ON t1.x = t22.x JOIN (SELECT 1 AS x) t23 ON t1.x = t23.x JOIN (SELECT 1 AS x) t24 ON t1.x = t24.x JOIN (SELECT 1 AS x) t25 ON t1.x = t25.x JOIN (SELECT 1 AS x) t26 ON t1.x = t26.x JOIN (SELECT 1 AS x) t27 ON t1.x = t27.x JOIN (SELECT 1 AS x) t28 ON t1.x = t28.x JOIN (SELECT 1 AS x) t29 ON t1.x = t29.x JOIN (SELECT 1 AS x) t30 ON t1.x = t30.x JOIN (SELECT 1 AS x) t31 ON t1.x = t31.x JOIN (SELECT 1 AS x) t32 ON t1.x = t32.x JOIN (SELECT 1 AS x) t33 ON t1.x = t33.x JOIN (SELECT 1 AS x) t34 ON t1.x = t34.x JOIN (SELECT 1 AS x) t35 ON t1.x = t35.x JOIN (SELECT 1 AS x) t36 ON t1.x = t36.x JOIN (SELECT 1 AS x) t37 ON t1.x = t37.x JOIN (SELECT 1 AS x) t38 ON t1.x = t38.x JOIN (SELECT 1 AS x) t39 ON t1.x = t39.x JOIN (SELECT 1 AS x) t40 ON t1.x = t40.x JOIN (SELECT 1 AS x) t41 ON t1.x = t41.x JOIN (SELECT 1 AS x) t42 ON t1.x = t42.x JOIN (SELECT 1 AS x) t43 ON t1.x = t43.x JOIN (SELECT 1 AS x) t44 ON t1.x = t44.x JOIN (SELECT 1 AS x) t45 ON t1.x = t45.x JOIN (SELECT 1 AS x) t46 ON t1.x = t46.x JOIN (SELECT 1 AS x) t47 ON t1.x = t47.x JOIN (SELECT 1 AS x) t48 ON t1.x = t48.x JOIN (SELECT 1 AS x) t49 ON t1.x = t49.x JOIN (SELECT 1 AS x) t50 ON t1.x = t50.x JOIN (SELECT 1 AS x) t51 ON t1.x = t51.x JOIN (SELECT 1 AS x) t52 ON t1.x = t52.x JOIN (SELECT 1 AS x) t53 ON t1.x = t53.x JOIN (SELECT 1 AS x) t54 ON t1.x = t54.x JOIN (SELECT 1 AS x) t55 ON t1.x = t55.x JOIN (SELECT 1 AS x) t56 ON t1.x = t56.x JOIN (SELECT 1 AS x) t57 ON t1.x = t57.x JOIN (SELECT 1 AS x) t58 ON t1.x = t58.x JOIN (SELECT 1 AS x) t59 ON t1.x = t59.x JOIN (SELECT 1 AS x) t60 ON t1.x = t60.x JOIN (SELECT 1 AS x) t61 ON t1.x = t61.x JOIN (SELECT 1 AS x) t62 ON t1.x = t62.x JOIN (SELECT 1 AS x) t63 ON t1.x = t63.x JOIN (SELECT 1 AS x) t64 ON t1.x = t64.x JOIN (SELECT 1 AS x) t65 ON t1.x = t65.x JOIN (SELECT 1 AS x) t66 ON t1.x = t66.x JOIN (SELECT 1 AS x) t67 ON t1.x = t67.x JOIN (SELECT 1 AS x) t68 ON t1.x = t68.x JOIN (SELECT 1 AS x) t69 ON t1.x = t69.x JOIN (SELECT 1 AS x) t70 ON t1.x = t70.x JOIN (SELECT 1 AS x) t71 ON t1.x = t71.x JOIN (SELECT 1 AS x) t72 ON t1.x = t72.x JOIN (SELECT 1 AS x) t73 ON t1.x = t73.x JOIN (SELECT 1 AS x) t74 ON t1.x = t74.x JOIN (SELECT 1 AS x) t75 ON t1.x = t75.x JOIN (SELECT 1 AS x) t76 ON t1.x = t76.x JOIN (SELECT 1 AS x) t77 ON t1.x = t77.x JOIN (SELECT 1 AS x) t78 ON t1.x = t78.x JOIN (SELECT 1 AS x) t79 ON t1.x = t79.x JOIN (SELECT 1 AS x) t80 ON t1.x = t80.x JOIN (SELECT 1 AS x) t81 ON t1.x = t81.x JOIN (SELECT 1 AS x) t82 ON t1.x = t82.x JOIN (SELECT 1 AS x) t83 ON t1.x = t83.x JOIN (SELECT 1 AS x) t84 ON t1.x = t84.x JOIN (SELECT 1 AS x) t85 ON t1.x = t85.x JOIN (SELECT 1 AS x) t86 ON t1.x = t86.x JOIN (SELECT 1 AS x) t87 ON t1.x = t87.x JOIN (SELECT 1 AS x) t88 ON t1.x = t88.x JOIN (SELECT 1 AS x) t89 ON t1.x = t89.x JOIN (SELECT 1 AS x) t90 ON t1.x = t90.x JOIN (SELECT 1 AS x) t91 ON t1.x = t91.x JOIN (SELECT 1 AS x) t92 ON t1.x = t92.x JOIN (SELECT 1 AS x) t93 ON t1.x = t93.x JOIN (SELECT 1 AS x) t94 ON t1.x = t94.x JOIN (SELECT 1 AS x) t95 ON t1.x = t95.x JOIN (SELECT 1 AS x) t96 ON t1.x = t96.x JOIN (SELECT 1 AS x) t97 ON t1.x = t97.x JOIN (SELECT 1 AS x) t98 ON t1.x = t98.x JOIN (SELECT 1 AS x) t99 ON t1.x = t99.x JOIN (SELECT 1 AS x) t100 ON t1.x = t100.x JOIN (SELECT 1 AS x) t101 ON t1.x = t101.x JOIN (SELECT 1 AS x) t102 ON t1.x = t102.x JOIN (SELECT 1 AS x) t103 ON t1.x = t103.x JOIN (SELECT 1 AS x) t104 ON t1.x = t104.x JOIN (SELECT 1 AS x) t105 ON t1.x = t105.x JOIN (SELECT 1 AS x) t106 ON t1.x = t106.x JOIN (SELECT 1 AS x) t107 ON t1.x = t107.x JOIN (SELECT 1 AS x) t108 ON t1.x = t108.x JOIN (SELECT 1 AS x) t109 ON t1.x = t109.x JOIN (SELECT 1 AS x) t110 ON t1.x = t110.x JOIN (SELECT 1 AS x) t111 ON t1.x = t111.x JOIN (SELECT 1 AS x) t112 ON t1.x = t112.x JOIN (SELECT 1 AS x) t113 ON t1.x = t113.x JOIN (SELECT 1 AS x) t114 ON t1.x = t114.x JOIN (SELECT 1 AS x) t115 ON t1.x = t115.x JOIN (SELECT 1 AS x) t116 ON t1.x = t116.x JOIN (SELECT 1 AS x) t117 ON t1.x = t117.x JOIN (SELECT 1 AS x) t118 ON t1.x = t118.x JOIN (SELECT 1 AS x) t119 ON t1.x = t119.x JOIN (SELECT 1 AS x) t120 ON t1.x = t120.x JOIN (SELECT 1 AS x) t121 ON t1.x = t121.x JOIN (SELECT 1 AS x) t122 ON t1.x = t122.x JOIN (SELECT 1 AS x) t123 ON t1.x = t123.x JOIN (SELECT 1 AS x) t124 ON t1.x = t124.x JOIN (SELECT 1 AS x) t125 ON t1.x = t125.x JOIN (SELECT 1 AS x) t126 ON t1.x = t126.x JOIN (SELECT 1 AS x) t127 ON t1.x = t127.x JOIN (SELECT 1 AS x) t128 ON t1.x = t128.x JOIN (SELECT 1 AS x) t129 ON t1.x = t129.x JOIN (SELECT 1 AS x) t130 ON t1.x = t130.x JOIN (SELECT 1 AS x) t131 ON t1.x = t131.x JOIN (SELECT 1 AS x) t132 ON t1.x = t132.x JOIN (SELECT 1 AS x) t133 ON t1.x = t133.x JOIN (SELECT 1 AS x) t134 ON t1.x = t134.x JOIN (SELECT 1 AS x) t135 ON t1.x = t135.x JOIN (SELECT 1 AS x) t136 ON t1.x = t136.x JOIN (SELECT 1 AS x) t137 ON t1.x = t137.x JOIN (SELECT 1 AS x) t138 ON t1.x = t138.x JOIN (SELECT 1 AS x) t139 ON t1.x = t139.x JOIN (SELECT 1 AS x) t140 ON t1.x = t140.x JOIN (SELECT 1 AS x) t141 ON t1.x = t141.x JOIN (SELECT 1 AS x) t142 ON t1.x = t142.x JOIN (SELECT 1 AS x) t143 ON t1.x = t143.x JOIN (SELECT 1 AS x) t144 ON t1.x = t144.x JOIN (SELECT 1 AS x) t145 ON t1.x = t145.x JOIN (SELECT 1 AS x) t146 ON t1.x = t146.x JOIN (SELECT 1 AS x) t147 ON t1.x = t147.x JOIN (SELECT 1 AS x) t148 ON t1.x = t148.x JOIN (SELECT 1 AS x) t149 ON t1.x = t149.x JOIN (SELECT 1 AS x) t150 ON t1.x = t150.x JOIN (SELECT 1 AS x) t151 ON t1.x = t151.x JOIN (SELECT 1 AS x) t152 ON t1.x = t152.x JOIN (SELECT 1 AS x) t153 ON t1.x = t153.x JOIN (SELECT 1 AS x) t154 ON t1.x = t154.x JOIN (SELECT 1 AS x) t155 ON t1.x = t155.x JOIN (SELECT 1 AS x) t156 ON t1.x = t156.x JOIN (SELECT 1 AS x) t157 ON t1.x = t157.x JOIN (SELECT 1 AS x) t158 ON t1.x = t158.x JOIN (SELECT 1 AS x) t159 ON t1.x = t159.x JOIN (SELECT 1 AS x) t160 ON t1.x = t160.x JOIN (SELECT 1 AS x) t161 ON t1.x = t161.x JOIN (SELECT 1 AS x) t162 ON t1.x = t162.x JOIN (SELECT 1 AS x) t163 ON t1.x = t163.x JOIN (SELECT 1 AS x) t164 ON t1.x = t164.x JOIN (SELECT 1 AS x) t165 ON t1.x = t165.x JOIN (SELECT 1 AS x) t166 ON t1.x = t166.x JOIN (SELECT 1 AS x) t167 ON t1.x = t167.x JOIN (SELECT 1 AS x) t168 ON t1.x = t168.x JOIN (SELECT 1 AS x) t169 ON t1.x = t169.x JOIN (SELECT 1 AS x) t170 ON t1.x = t170.x JOIN (SELECT 1 AS x) t171 ON t1.x = t171.x JOIN (SELECT 1 AS x) t172 ON t1.x = t172.x JOIN (SELECT 1 AS x) t173 ON t1.x = t173.x JOIN (SELECT 1 AS x) t174 ON t1.x = t174.x JOIN (SELECT 1 AS x) t175 ON t1.x = t175.x JOIN (SELECT 1 AS x) t176 ON t1.x = t176.x JOIN (SELECT 1 AS x) t177 ON t1.x = t177.x JOIN (SELECT 1 AS x) t178 ON t1.x = t178.x JOIN (SELECT 1 AS x) t179 ON t1.x = t179.x JOIN (SELECT 1 AS x) t180 ON t1.x = t180.x JOIN (SELECT 1 AS x) t181 ON t1.x = t181.x JOIN (SELECT 1 AS x) t182 ON t1.x = t182.x JOIN (SELECT 1 AS x) t183 ON t1.x = t183.x JOIN (SELECT 1 AS x) t184 ON t1.x = t184.x JOIN (SELECT 1 AS x) t185 ON t1.x = t185.x JOIN (SELECT 1 AS x) t186 ON t1.x = t186.x JOIN (SELECT 1 AS x) t187 ON t1.x = t187.x JOIN (SELECT 1 AS x) t188 ON t1.x = t188.x JOIN (SELECT 1 AS x) t189 ON t1.x = t189.x JOIN (SELECT 1 AS x) t190 ON t1.x = t190.x JOIN (SELECT 1 AS x) t191 ON t1.x = t191.x JOIN (SELECT 1 AS x) t192 ON t1.x = t192.x JOIN (SELECT 1 AS x) t193 ON t1.x = t193.x JOIN (SELECT 1 AS x) t194 ON t1.x = t194.x JOIN (SELECT 1 AS x) t195 ON t1.x = t195.x JOIN (SELECT 1 AS x) t196 ON t1.x = t196.x JOIN (SELECT 1 AS x) t197 ON t1.x = t197.x JOIN (SELECT 1 AS x) t198 ON t1.x = t198.x JOIN (SELECT 1 AS x) t199 ON t1.x = t199.x JOIN (SELECT 1 AS x) t200 ON t1.x = t200.x JOIN (SELECT 1 AS x) t201 ON t1.x = t201.x JOIN (SELECT 1 AS x) t202 ON t1.x = t202.x JOIN (SELECT 1 AS x) t203 ON t1.x = t203.x JOIN (SELECT 1 AS x) t204 ON t1.x = t204.x JOIN (SELECT 1 AS x) t205 ON t1.x = t205.x JOIN (SELECT 1 AS x) t206 ON t1.x = t206.x JOIN (SELECT 1 AS x) t207 ON t1.x = t207.x JOIN (SELECT 1 AS x) t208 ON t1.x = t208.x JOIN (SELECT 1 AS x) t209 ON t1.x = t209.x JOIN (SELECT 1 AS x) t210 ON t1.x = t210.x JOIN (SELECT 1 AS x) t211 ON t1.x = t211.x JOIN (SELECT 1 AS x) t212 ON t1.x = t212.x JOIN (SELECT 1 AS x) t213 ON t1.x = t213.x JOIN (SELECT 1 AS x) t214 ON t1.x = t214.x JOIN (SELECT 1 AS x) t215 ON t1.x = t215.x JOIN (SELECT 1 AS x) t216 ON t1.x = t216.x JOIN (SELECT 1 AS x) t217 ON t1.x = t217.x JOIN (SELECT 1 AS x) t218 ON t1.x = t218.x JOIN (SELECT 1 AS x) t219 ON t1.x = t219.x JOIN (SELECT 1 AS x) t220 ON t1.x = t220.x JOIN (SELECT 1 AS x) t221 ON t1.x = t221.x JOIN (SELECT 1 AS x) t222 ON t1.x = t222.x JOIN (SELECT 1 AS x) t223 ON t1.x = t223.x JOIN (SELECT 1 AS x) t224 ON t1.x = t224.x JOIN (SELECT 1 AS x) t225 ON t1.x = t225.x JOIN (SELECT 1 AS x) t226 ON t1.x = t226.x JOIN (SELECT 1 AS x) t227 ON t1.x = t227.x JOIN (SELECT 1 AS x) t228 ON t1.x = t228.x JOIN (SELECT 1 AS x) t229 ON t1.x = t229.x JOIN (SELECT 1 AS x) t230 ON t1.x = t230.x JOIN (SELECT 1 AS x) t231 ON t1.x = t231.x JOIN (SELECT 1 AS x) t232 ON t1.x = t232.x JOIN (SELECT 1 AS x) t233 ON t1.x = t233.x JOIN (SELECT 1 AS x) t234 ON t1.x = t234.x JOIN (SELECT 1 AS x) t235 ON t1.x = t235.x JOIN (SELECT 1 AS x) t236 ON t1.x = t236.x JOIN (SELECT 1 AS x) t237 ON t1.x = t237.x JOIN (SELECT 1 AS x) t238 ON t1.x = t238.x JOIN (SELECT 1 AS x) t239 ON t1.x = t239.x JOIN (SELECT 1 AS x) t240 ON t1.x = t240.x JOIN (SELECT 1 AS x) t241 ON t1.x = t241.x JOIN (SELECT 1 AS x) t242 ON t1.x = t242.x JOIN (SELECT 1 AS x) t243 ON t1.x = t243.x JOIN (SELECT 1 AS x) t244 ON t1.x = t244.x JOIN (SELECT 1 AS x) t245 ON t1.x = t245.x JOIN (SELECT 1 AS x) t246 ON t1.x = t246.x JOIN (SELECT 1 AS x) t247 ON t1.x = t247.x JOIN (SELECT 1 AS x) t248 ON t1.x = t248.x JOIN (SELECT 1 AS x) t249 ON t1.x = t249.x JOIN (SELECT 1 AS x) t250 ON t1.x = t250.x JOIN (SELECT 1 AS x) t251 ON t1.x = t251.x JOIN (SELECT 1 AS x) t252 ON t1.x = t252.x JOIN (SELECT 1 AS x) t253 ON t1.x = t253.x JOIN (SELECT 1 AS x) t254 ON t1.x = t254.x JOIN (SELECT 1 AS x) t255 ON t1.x = t255.x JOIN (SELECT 1 AS x) t256 ON t1.x = t256.x JOIN (SELECT 1 AS x) t257 ON t1.x = t257.x JOIN (SELECT 1 AS x) t258 ON t1.x = t258.x JOIN (SELECT 1 AS x) t259 ON t1.x = t259.x JOIN (SELECT 1 AS x) t260 ON t1.x = t260.x JOIN (SELECT 1 AS x) t261 ON t1.x = t261.x JOIN (SELECT 1 AS x) t262 ON t1.x = t262.x JOIN (SELECT 1 AS x) t263 ON t1.x = t263.x JOIN (SELECT 1 AS x) t264 ON t1.x = t264.x JOIN (SELECT 1 AS x) t265 ON t1.x = t265.x JOIN (SELECT 1 AS x) t266 ON t1.x = t266.x JOIN (SELECT 1 AS x) t267 ON t1.x = t267.x JOIN (SELECT 1 AS x) t268 ON t1.x = t268.x JOIN (SELECT 1 AS x) t269 ON t1.x = t269.x JOIN (SELECT 1 AS x) t270 ON t1.x = t270.x JOIN (SELECT 1 AS x) t271 ON t1.x = t271.x JOIN (SELECT 1 AS x) t272 ON t1.x = t272.x JOIN (SELECT 1 AS x) t273 ON t1.x = t273.x JOIN (SELECT 1 AS x) t274 ON t1.x = t274.x JOIN (SELECT 1 AS x) t275 ON t1.x = t275.x JOIN (SELECT 1 AS x) t276 ON t1.x = t276.x JOIN (SELECT 1 AS x) t277 ON t1.x = t277.x JOIN (SELECT 1 AS x) t278 ON t1.x = t278.x JOIN (SELECT 1 AS x) t279 ON t1.x = t279.x JOIN (SELECT 1 AS x) t280 ON t1.x = t280.x JOIN (SELECT 1 AS x) t281 ON t1.x = t281.x JOIN (SELECT 1 AS x) t282 ON t1.x = t282.x JOIN (SELECT 1 AS x) t283 ON t1.x = t283.x JOIN (SELECT 1 AS x) t284 ON t1.x = t284.x JOIN (SELECT 1 AS x) t285 ON t1.x = t285.x JOIN (SELECT 1 AS x) t286 ON t1.x = t286.x JOIN (SELECT 1 AS x) t287 ON t1.x = t287.x JOIN (SELECT 1 AS x) t288 ON t1.x = t288.x JOIN (SELECT 1 AS x) t289 ON t1.x = t289.x JOIN (SELECT 1 AS x) t290 ON t1.x = t290.x JOIN (SELECT 1 AS x) t291 ON t1.x = t291.x JOIN (SELECT 1 AS x) t292 ON t1.x = t292.x JOIN (SELECT 1 AS x) t293 ON t1.x = t293.x JOIN (SELECT 1 AS x) t294 ON t1.x = t294.x JOIN (SELECT 1 AS x) t295 ON t1.x = t295.x JOIN (SELECT 1 AS x) t296 ON t1.x = t296.x JOIN (SELECT 1 AS x) t297 ON t1.x = t297.x JOIN (SELECT 1 AS x) t298 ON t1.x = t298.x JOIN (SELECT 1 AS x) t299 ON t1.x = t299.x JOIN (SELECT 1 AS x) t300 ON t1.x = t300.x JOIN (SELECT 1 AS x) t301 ON t1.x = t301.x JOIN (SELECT 1 AS x) t302 ON t1.x = t302.x JOIN (SELECT 1 AS x) t303 ON t1.x = t303.x JOIN (SELECT 1 AS x) t304 ON t1.x = t304.x JOIN (SELECT 1 AS x) t305 ON t1.x = t305.x JOIN (SELECT 1 AS x) t306 ON t1.x = t306.x JOIN (SELECT 1 AS x) t307 ON t1.x = t307.x JOIN (SELECT 1 AS x) t308 ON t1.x = t308.x JOIN (SELECT 1 AS x) t309 ON t1.x = t309.x JOIN (SELECT 1 AS x) t310 ON t1.x = t310.x JOIN (SELECT 1 AS x) t311 ON t1.x = t311.x JOIN (SELECT 1 AS x) t312 ON t1.x = t312.x JOIN (SELECT 1 AS x) t313 ON t1.x = t313.x JOIN (SELECT 1 AS x) t314 ON t1.x = t314.x JOIN (SELECT 1 AS x) t315 ON t1.x = t315.x JOIN (SELECT 1 AS x) t316 ON t1.x = t316.x JOIN (SELECT 1 AS x) t317 ON t1.x = t317.x JOIN (SELECT 1 AS x) t318 ON t1.x = t318.x JOIN (SELECT 1 AS x) t319 ON t1.x = t319.x JOIN (SELECT 1 AS x) t320 ON t1.x = t320.x JOIN (SELECT 1 AS x) t321 ON t1.x = t321.x JOIN (SELECT 1 AS x) t322 ON t1.x = t322.x JOIN (SELECT 1 AS x) t323 ON t1.x = t323.x JOIN (SELECT 1 AS x) t324 ON t1.x = t324.x JOIN (SELECT 1 AS x) t325 ON t1.x = t325.x JOIN (SELECT 1 AS x) t326 ON t1.x = t326.x JOIN (SELECT 1 AS x) t327 ON t1.x = t327.x JOIN (SELECT 1 AS x) t328 ON t1.x = t328.x JOIN (SELECT 1 AS x) t329 ON t1.x = t329.x JOIN (SELECT 1 AS x) t330 ON t1.x = t330.x JOIN (SELECT 1 AS x) t331 ON t1.x = t331.x JOIN (SELECT 1 AS x) t332 ON t1.x = t332.x JOIN (SELECT 1 AS x) t333 ON t1.x = t333.x JOIN (SELECT 1 AS x) t334 ON t1.x = t334.x JOIN (SELECT 1 AS x) t335 ON t1.x = t335.x JOIN (SELECT 1 AS x) t336 ON t1.x = t336.x JOIN (SELECT 1 AS x) t337 ON t1.x = t337.x JOIN (SELECT 1 AS x) t338 ON t1.x = t338.x JOIN (SELECT 1 AS x) t339 ON t1.x = t339.x JOIN (SELECT 1 AS x) t340 ON t1.x = t340.x JOIN (SELECT 1 AS x) t341 ON t1.x = t341.x JOIN (SELECT 1 AS x) t342 ON t1.x = t342.x JOIN (SELECT 1 AS x) t343 ON t1.x = t343.x JOIN (SELECT 1 AS x) t344 ON t1.x = t344.x JOIN (SELECT 1 AS x) t345 ON t1.x = t345.x JOIN (SELECT 1 AS x) t346 ON t1.x = t346.x JOIN (SELECT 1 AS x) t347 ON t1.x = t347.x JOIN (SELECT 1 AS x) t348 ON t1.x = t348.x JOIN (SELECT 1 AS x) t349 ON t1.x = t349.x JOIN (SELECT 1 AS x) t350 ON t1.x = t350.x JOIN (SELECT 1 AS x) t351 ON t1.x = t351.x JOIN (SELECT 1 AS x) t352 ON t1.x = t352.x JOIN (SELECT 1 AS x) t353 ON t1.x = t353.x JOIN (SELECT 1 AS x) t354 ON t1.x = t354.x JOIN (SELECT 1 AS x) t355 ON t1.x = t355.x JOIN (SELECT 1 AS x) t356 ON t1.x = t356.x JOIN (SELECT 1 AS x) t357 ON t1.x = t357.x JOIN (SELECT 1 AS x) t358 ON t1.x = t358.x JOIN (SELECT 1 AS x) t359 ON t1.x = t359.x JOIN (SELECT 1 AS x) t360 ON t1.x = t360.x JOIN (SELECT 1 AS x) t361 ON t1.x = t361.x JOIN (SELECT 1 AS x) t362 ON t1.x = t362.x JOIN (SELECT 1 AS x) t363 ON t1.x = t363.x JOIN (SELECT 1 AS x) t364 ON t1.x = t364.x JOIN (SELECT 1 AS x) t365 ON t1.x = t365.x JOIN (SELECT 1 AS x) t366 ON t1.x = t366.x JOIN (SELECT 1 AS x) t367 ON t1.x = t367.x JOIN (SELECT 1 AS x) t368 ON t1.x = t368.x JOIN (SELECT 1 AS x) t369 ON t1.x = t369.x JOIN (SELECT 1 AS x) t370 ON t1.x = t370.x JOIN (SELECT 1 AS x) t371 ON t1.x = t371.x JOIN (SELECT 1 AS x) t372 ON t1.x = t372.x JOIN (SELECT 1 AS x) t373 ON t1.x = t373.x JOIN (SELECT 1 AS x) t374 ON t1.x = t374.x JOIN (SELECT 1 AS x) t375 ON t1.x = t375.x JOIN (SELECT 1 AS x) t376 ON t1.x = t376.x JOIN (SELECT 1 AS x) t377 ON t1.x = t377.x JOIN (SELECT 1 AS x) t378 ON t1.x = t378.x JOIN (SELECT 1 AS x) t379 ON t1.x = t379.x JOIN (SELECT 1 AS x) t380 ON t1.x = t380.x JOIN (SELECT 1 AS x) t381 ON t1.x = t381.x JOIN (SELECT 1 AS x) t382 ON t1.x = t382.x JOIN (SELECT 1 AS x) t383 ON t1.x = t383.x JOIN (SELECT 1 AS x) t384 ON t1.x = t384.x JOIN (SELECT 1 AS x) t385 ON t1.x = t385.x JOIN (SELECT 1 AS x) t386 ON t1.x = t386.x JOIN (SELECT 1 AS x) t387 ON t1.x = t387.x JOIN (SELECT 1 AS x) t388 ON t1.x = t388.x JOIN (SELECT 1 AS x) t389 ON t1.x = t389.x JOIN (SELECT 1 AS x) t390 ON t1.x = t390.x JOIN (SELECT 1 AS x) t391 ON t1.x = t391.x JOIN (SELECT 1 AS x) t392 ON t1.x = t392.x JOIN (SELECT 1 AS x) t393 ON t1.x = t393.x JOIN (SELECT 1 AS x) t394 ON t1.x = t394.x JOIN (SELECT 1 AS x) t395 ON t1.x = t395.x JOIN (SELECT 1 AS x) t396 ON t1.x = t396.x JOIN (SELECT 1 AS x) t397 ON t1.x = t397.x JOIN (SELECT 1 AS x) t398 ON t1.x = t398.x JOIN (SELECT 1 AS x) t399 ON t1.x = t399.x JOIN (SELECT 1 AS x) t400 ON t1.x = t400.x JOIN (SELECT 1 AS x) t401 ON t1.x = t401.x JOIN (SELECT 1 AS x) t402 ON t1.x = t402.x JOIN (SELECT 1 AS x) t403 ON t1.x = t403.x JOIN (SELECT 1 AS x) t404 ON t1.x = t404.x JOIN (SELECT 1 AS x) t405 ON t1.x = t405.x JOIN (SELECT 1 AS x) t406 ON t1.x = t406.x JOIN (SELECT 1 AS x) t407 ON t1.x = t407.x JOIN (SELECT 1 AS x) t408 ON t1.x = t408.x JOIN (SELECT 1 AS x) t409 ON t1.x = t409.x JOIN (SELECT 1 AS x) t410 ON t1.x = t410.x JOIN (SELECT 1 AS x) t411 ON t1.x = t411.x JOIN (SELECT 1 AS x) t412 ON t1.x = t412.x JOIN (SELECT 1 AS x) t413 ON t1.x = t413.x JOIN (SELECT 1 AS x) t414 ON t1.x = t414.x JOIN (SELECT 1 AS x) t415 ON t1.x = t415.x JOIN (SELECT 1 AS x) t416 ON t1.x = t416.x JOIN (SELECT 1 AS x) t417 ON t1.x = t417.x JOIN (SELECT 1 AS x) t418 ON t1.x = t418.x JOIN (SELECT 1 AS x) t419 ON t1.x = t419.x JOIN (SELECT 1 AS x) t420 ON t1.x = t420.x JOIN (SELECT 1 AS x) t421 ON t1.x = t421.x JOIN (SELECT 1 AS x) t422 ON t1.x = t422.x JOIN (SELECT 1 AS x) t423 ON t1.x = t423.x JOIN (SELECT 1 AS x) t424 ON t1.x = t424.x JOIN (SELECT 1 AS x) t425 ON t1.x = t425.x JOIN (SELECT 1 AS x) t426 ON t1.x = t426.x JOIN (SELECT 1 AS x) t427 ON t1.x = t427.x JOIN (SELECT 1 AS x) t428 ON t1.x = t428.x JOIN (SELECT 1 AS x) t429 ON t1.x = t429.x JOIN (SELECT 1 AS x) t430 ON t1.x = t430.x JOIN (SELECT 1 AS x) t431 ON t1.x = t431.x JOIN (SELECT 1 AS x) t432 ON t1.x = t432.x JOIN (SELECT 1 AS x) t433 ON t1.x = t433.x JOIN (SELECT 1 AS x) t434 ON t1.x = t434.x JOIN (SELECT 1 AS x) t435 ON t1.x = t435.x JOIN (SELECT 1 AS x) t436 ON t1.x = t436.x JOIN (SELECT 1 AS x) t437 ON t1.x = t437.x JOIN (SELECT 1 AS x) t438 ON t1.x = t438.x JOIN (SELECT 1 AS x) t439 ON t1.x = t439.x JOIN (SELECT 1 AS x) t440 ON t1.x = t440.x JOIN (SELECT 1 AS x) t441 ON t1.x = t441.x JOIN (SELECT 1 AS x) t442 ON t1.x = t442.x JOIN (SELECT 1 AS x) t443 ON t1.x = t443.x JOIN (SELECT 1 AS x) t444 ON t1.x = t444.x JOIN (SELECT 1 AS x) t445 ON t1.x = t445.x JOIN (SELECT 1 AS x) t446 ON t1.x = t446.x JOIN (SELECT 1 AS x) t447 ON t1.x = t447.x JOIN (SELECT 1 AS x) t448 ON t1.x = t448.x JOIN (SELECT 1 AS x) t449 ON t1.x = t449.x JOIN (SELECT 1 AS x) t450 ON t1.x = t450.x JOIN (SELECT 1 AS x) t451 ON t1.x = t451.x JOIN (SELECT 1 AS x) t452 ON t1.x = t452.x JOIN (SELECT 1 AS x) t453 ON t1.x = t453.x JOIN (SELECT 1 AS x) t454 ON t1.x = t454.x JOIN (SELECT 1 AS x) t455 ON t1.x = t455.x JOIN (SELECT 1 AS x) t456 ON t1.x = t456.x JOIN (SELECT 1 AS x) t457 ON t1.x = t457.x JOIN (SELECT 1 AS x) t458 ON t1.x = t458.x JOIN (SELECT 1 AS x) t459 ON t1.x = t459.x JOIN (SELECT 1 AS x) t460 ON t1.x = t460.x JOIN (SELECT 1 AS x) t461 ON t1.x = t461.x JOIN (SELECT 1 AS x) t462 ON t1.x = t462.x JOIN (SELECT 1 AS x) t463 ON t1.x = t463.x JOIN (SELECT 1 AS x) t464 ON t1.x = t464.x JOIN (SELECT 1 AS x) t465 ON t1.x = t465.x JOIN (SELECT 1 AS x) t466 ON t1.x = t466.x JOIN (SELECT 1 AS x) t467 ON t1.x = t467.x JOIN (SELECT 1 AS x) t468 ON t1.x = t468.x JOIN (SELECT 1 AS x) t469 ON t1.x = t469.x JOIN (SELECT 1 AS x) t470 ON t1.x = t470.x JOIN (SELECT 1 AS x) t471 ON t1.x = t471.x JOIN (SELECT 1 AS x) t472 ON t1.x = t472.x JOIN (SELECT 1 AS x) t473 ON t1.x = t473.x JOIN (SELECT 1 AS x) t474 ON t1.x = t474.x JOIN (SELECT 1 AS x) t475 ON t1.x = t475.x JOIN (SELECT 1 AS x) t476 ON t1.x = t476.x JOIN (SELECT 1 AS x) t477 ON t1.x = t477.x JOIN (SELECT 1 AS x) t478 ON t1.x = t478.x JOIN (SELECT 1 AS x) t479 ON t1.x = t479.x JOIN (SELECT 1 AS x) t480 ON t1.x = t480.x JOIN (SELECT 1 AS x) t481 ON t1.x = t481.x JOIN (SELECT 1 AS x) t482 ON t1.x = t482.x JOIN (SELECT 1 AS x) t483 ON t1.x = t483.x JOIN (SELECT 1 AS x) t484 ON t1.x = t484.x JOIN (SELECT 1 AS x) t485 ON t1.x = t485.x JOIN (SELECT 1 AS x) t486 ON t1.x = t486.x JOIN (SELECT 1 AS x) t487 ON t1.x = t487.x JOIN (SELECT 1 AS x) t488 ON t1.x = t488.x JOIN (SELECT 1 AS x) t489 ON t1.x = t489.x JOIN (SELECT 1 AS x) t490 ON t1.x = t490.x JOIN (SELECT 1 AS x) t491 ON t1.x = t491.x JOIN (SELECT 1 AS x) t492 ON t1.x = t492.x JOIN (SELECT 1 AS x) t493 ON t1.x = t493.x JOIN (SELECT 1 AS x) t494 ON t1.x = t494.x JOIN (SELECT 1 AS x) t495 ON t1.x = t495.x JOIN (SELECT 1 AS x) t496 ON t1.x = t496.x JOIN (SELECT 1 AS x) t497 ON t1.x = t497.x JOIN (SELECT 1 AS x) t498 ON t1.x = t498.x JOIN (SELECT 1 AS x) t499 ON t1.x = t499.x JOIN (SELECT 1 AS x) t500 ON t1.x = t500.x JOIN (SELECT 1 AS x) t501 ON t1.x = t501.x JOIN (SELECT 1 AS x) t502 ON t1.x = t502.x JOIN (SELECT 1 AS x) t503 ON t1.x = t503.x JOIN (SELECT 1 AS x) t504 ON t1.x = t504.x JOIN (SELECT 1 AS x) t505 ON t1.x = t505.x JOIN (SELECT 1 AS x) t506 ON t1.x = t506.x JOIN (SELECT 1 AS x) t507 ON t1.x = t507.x JOIN (SELECT 1 AS x) t508 ON t1.x = t508.x JOIN (SELECT 1 AS x) t509 ON t1.x = t509.x JOIN (SELECT 1 AS x) t510 ON t1.x = t510.x JOIN (SELECT 1 AS x) t511 ON t1.x = t511.x JOIN (SELECT 1 AS x) t512 ON t1.x = t512.x JOIN (SELECT 1 AS x) t513 ON t1.x = t513.x JOIN (SELECT 1 AS x) t514 ON t1.x = t514.x JOIN (SELECT 1 AS x) t515 ON t1.x = t515.x JOIN (SELECT 1 AS x) t516 ON t1.x = t516.x JOIN (SELECT 1 AS x) t517 ON t1.x = t517.x JOIN (SELECT 1 AS x) t518 ON t1.x = t518.x JOIN (SELECT 1 AS x) t519 ON t1.x = t519.x JOIN (SELECT 1 AS x) t520 ON t1.x = t520.x JOIN (SELECT 1 AS x) t521 ON t1.x = t521.x JOIN (SELECT 1 AS x) t522 ON t1.x = t522.x JOIN (SELECT 1 AS x) t523 ON t1.x = t523.x JOIN (SELECT 1 AS x) t524 ON t1.x = t524.x JOIN (SELECT 1 AS x) t525 ON t1.x = t525.x JOIN (SELECT 1 AS x) t526 ON t1.x = t526.x JOIN (SELECT 1 AS x) t527 ON t1.x = t527.x JOIN (SELECT 1 AS x) t528 ON t1.x = t528.x JOIN (SELECT 1 AS x) t529 ON t1.x = t529.x JOIN (SELECT 1 AS x) t530 ON t1.x = t530.x JOIN (SELECT 1 AS x) t531 ON t1.x = t531.x JOIN (SELECT 1 AS x) t532 ON t1.x = t532.x JOIN (SELECT 1 AS x) t533 ON t1.x = t533.x JOIN (SELECT 1 AS x) t534 ON t1.x = t534.x JOIN (SELECT 1 AS x) t535 ON t1.x = t535.x JOIN (SELECT 1 AS x) t536 ON t1.x = t536.x JOIN (SELECT 1 AS x) t537 ON t1.x = t537.x JOIN (SELECT 1 AS x) t538 ON t1.x = t538.x JOIN (SELECT 1 AS x) t539 ON t1.x = t539.x JOIN (SELECT 1 AS x) t540 ON t1.x = t540.x JOIN (SELECT 1 AS x) t541 ON t1.x = t541.x JOIN (SELECT 1 AS x) t542 ON t1.x = t542.x JOIN (SELECT 1 AS x) t543 ON t1.x = t543.x JOIN (SELECT 1 AS x) t544 ON t1.x = t544.x JOIN (SELECT 1 AS x) t545 ON t1.x = t545.x JOIN (SELECT 1 AS x) t546 ON t1.x = t546.x JOIN (SELECT 1 AS x) t547 ON t1.x = t547.x JOIN (SELECT 1 AS x) t548 ON t1.x = t548.x JOIN (SELECT 1 AS x) t549 ON t1.x = t549.x JOIN (SELECT 1 AS x) t550 ON t1.x = t550.x JOIN (SELECT 1 AS x) t551 ON t1.x = t551.x JOIN (SELECT 1 AS x) t552 ON t1.x = t552.x JOIN (SELECT 1 AS x) t553 ON t1.x = t553.x JOIN (SELECT 1 AS x) t554 ON t1.x = t554.x JOIN (SELECT 1 AS x) t555 ON t1.x = t555.x JOIN (SELECT 1 AS x) t556 ON t1.x = t556.x JOIN (SELECT 1 AS x) t557 ON t1.x = t557.x JOIN (SELECT 1 AS x) t558 ON t1.x = t558.x JOIN (SELECT 1 AS x) t559 ON t1.x = t559.x JOIN (SELECT 1 AS x) t560 ON t1.x = t560.x JOIN (SELECT 1 AS x) t561 ON t1.x = t561.x JOIN (SELECT 1 AS x) t562 ON t1.x = t562.x JOIN (SELECT 1 AS x) t563 ON t1.x = t563.x JOIN (SELECT 1 AS x) t564 ON t1.x = t564.x JOIN (SELECT 1 AS x) t565 ON t1.x = t565.x JOIN (SELECT 1 AS x) t566 ON t1.x = t566.x JOIN (SELECT 1 AS x) t567 ON t1.x = t567.x JOIN (SELECT 1 AS x) t568 ON t1.x = t568.x JOIN (SELECT 1 AS x) t569 ON t1.x = t569.x JOIN (SELECT 1 AS x) t570 ON t1.x = t570.x JOIN (SELECT 1 AS x) t571 ON t1.x = t571.x JOIN (SELECT 1 AS x) t572 ON t1.x = t572.x JOIN (SELECT 1 AS x) t573 ON t1.x = t573.x JOIN (SELECT 1 AS x) t574 ON t1.x = t574.x JOIN (SELECT 1 AS x) t575 ON t1.x = t575.x JOIN (SELECT 1 AS x) t576 ON t1.x = t576.x JOIN (SELECT 1 AS x) t577 ON t1.x = t577.x JOIN (SELECT 1 AS x) t578 ON t1.x = t578.x JOIN (SELECT 1 AS x) t579 ON t1.x = t579.x JOIN (SELECT 1 AS x) t580 ON t1.x = t580.x JOIN (SELECT 1 AS x) t581 ON t1.x = t581.x JOIN (SELECT 1 AS x) t582 ON t1.x = t582.x JOIN (SELECT 1 AS x) t583 ON t1.x = t583.x JOIN (SELECT 1 AS x) t584 ON t1.x = t584.x JOIN (SELECT 1 AS x) t585 ON t1.x = t585.x JOIN (SELECT 1 AS x) t586 ON t1.x = t586.x JOIN (SELECT 1 AS x) t587 ON t1.x = t587.x JOIN (SELECT 1 AS x) t588 ON t1.x = t588.x JOIN (SELECT 1 AS x) t589 ON t1.x = t589.x JOIN (SELECT 1 AS x) t590 ON t1.x = t590.x JOIN (SELECT 1 AS x) t591 ON t1.x = t591.x JOIN (SELECT 1 AS x) t592 ON t1.x = t592.x JOIN (SELECT 1 AS x) t593 ON t1.x = t593.x JOIN (SELECT 1 AS x) t594 ON t1.x = t594.x JOIN (SELECT 1 AS x) t595 ON t1.x = t595.x JOIN (SELECT 1 AS x) t596 ON t1.x = t596.x JOIN (SELECT 1 AS x) t597 ON t1.x = t597.x JOIN (SELECT 1 AS x) t598 ON t1.x = t598.x JOIN (SELECT 1 AS x) t599 ON t1.x = t599.x JOIN (SELECT 1 AS x) t600 ON t1.x = t600.x JOIN (SELECT 1 AS x) t601 ON t1.x = t601.x JOIN (SELECT 1 AS x) t602 ON t1.x = t602.x JOIN (SELECT 1 AS x) t603 ON t1.x = t603.x JOIN (SELECT 1 AS x) t604 ON t1.x = t604.x JOIN (SELECT 1 AS x) t605 ON t1.x = t605.x JOIN (SELECT 1 AS x) t606 ON t1.x = t606.x JOIN (SELECT 1 AS x) t607 ON t1.x = t607.x JOIN (SELECT 1 AS x) t608 ON t1.x = t608.x JOIN (SELECT 1 AS x) t609 ON t1.x = t609.x JOIN (SELECT 1 AS x) t610 ON t1.x = t610.x JOIN (SELECT 1 AS x) t611 ON t1.x = t611.x JOIN (SELECT 1 AS x) t612 ON t1.x = t612.x JOIN (SELECT 1 AS x) t613 ON t1.x = t613.x JOIN (SELECT 1 AS x) t614 ON t1.x = t614.x JOIN (SELECT 1 AS x) t615 ON t1.x = t615.x JOIN (SELECT 1 AS x) t616 ON t1.x = t616.x JOIN (SELECT 1 AS x) t617 ON t1.x = t617.x JOIN (SELECT 1 AS x) t618 ON t1.x = t618.x JOIN (SELECT 1 AS x) t619 ON t1.x = t619.x JOIN (SELECT 1 AS x) t620 ON t1.x = t620.x JOIN (SELECT 1 AS x) t621 ON t1.x = t621.x JOIN (SELECT 1 AS x) t622 ON t1.x = t622.x JOIN (SELECT 1 AS x) t623 ON t1.x = t623.x JOIN (SELECT 1 AS x) t624 ON t1.x = t624.x JOIN (SELECT 1 AS x) t625 ON t1.x = t625.x JOIN (SELECT 1 AS x) t626 ON t1.x = t626.x JOIN (SELECT 1 AS x) t627 ON t1.x = t627.x JOIN (SELECT 1 AS x) t628 ON t1.x = t628.x JOIN (SELECT 1 AS x) t629 ON t1.x = t629.x JOIN (SELECT 1 AS x) t630 ON t1.x = t630.x JOIN (SELECT 1 AS x) t631 ON t1.x = t631.x JOIN (SELECT 1 AS x) t632 ON t1.x = t632.x JOIN (SELECT 1 AS x) t633 ON t1.x = t633.x JOIN (SELECT 1 AS x) t634 ON t1.x = t634.x JOIN (SELECT 1 AS x) t635 ON t1.x = t635.x JOIN (SELECT 1 AS x) t636 ON t1.x = t636.x JOIN (SELECT 1 AS x) t637 ON t1.x = t637.x JOIN (SELECT 1 AS x) t638 ON t1.x = t638.x JOIN (SELECT 1 AS x) t639 ON t1.x = t639.x JOIN (SELECT 1 AS x) t640 ON t1.x = t640.x JOIN (SELECT 1 AS x) t641 ON t1.x = t641.x JOIN (SELECT 1 AS x) t642 ON t1.x = t642.x JOIN (SELECT 1 AS x) t643 ON t1.x = t643.x JOIN (SELECT 1 AS x) t644 ON t1.x = t644.x JOIN (SELECT 1 AS x) t645 ON t1.x = t645.x JOIN (SELECT 1 AS x) t646 ON t1.x = t646.x JOIN (SELECT 1 AS x) t647 ON t1.x = t647.x JOIN (SELECT 1 AS x) t648 ON t1.x = t648.x JOIN (SELECT 1 AS x) t649 ON t1.x = t649.x JOIN (SELECT 1 AS x) t650 ON t1.x = t650.x JOIN (SELECT 1 AS x) t651 ON t1.x = t651.x JOIN (SELECT 1 AS x) t652 ON t1.x = t652.x JOIN (SELECT 1 AS x) t653 ON t1.x = t653.x JOIN (SELECT 1 AS x) t654 ON t1.x = t654.x JOIN (SELECT 1 AS x) t655 ON t1.x = t655.x JOIN (SELECT 1 AS x) t656 ON t1.x = t656.x JOIN (SELECT 1 AS x) t657 ON t1.x = t657.x JOIN (SELECT 1 AS x) t658 ON t1.x = t658.x JOIN (SELECT 1 AS x) t659 ON t1.x = t659.x JOIN (SELECT 1 AS x) t660 ON t1.x = t660.x JOIN (SELECT 1 AS x) t661 ON t1.x = t661.x JOIN (SELECT 1 AS x) t662 ON t1.x = t662.x JOIN (SELECT 1 AS x) t663 ON t1.x = t663.x JOIN (SELECT 1 AS x) t664 ON t1.x = t664.x JOIN (SELECT 1 AS x) t665 ON t1.x = t665.x JOIN (SELECT 1 AS x) t666 ON t1.x = t666.x
diff --git a/tests/queries/0_stateless/03095_window_functions_qualify.sql b/tests/queries/0_stateless/03095_window_functions_qualify.sql
index adedff2e2cf..72903992745 100644
--- a/tests/queries/0_stateless/03095_window_functions_qualify.sql
+++ b/tests/queries/0_stateless/03095_window_functions_qualify.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 SELECT number, COUNT() OVER (PARTITION BY number % 3) AS partition_count FROM numbers(10) QUALIFY partition_count = 4 ORDER BY number;
 
diff --git a/tests/queries/0_stateless/03096_text_log_format_string_args_not_empty.sql b/tests/queries/0_stateless/03096_text_log_format_string_args_not_empty.sql
index b1ddd141e04..0e60ee77fe5 100644
--- a/tests/queries/0_stateless/03096_text_log_format_string_args_not_empty.sql
+++ b/tests/queries/0_stateless/03096_text_log_format_string_args_not_empty.sql
@@ -1,4 +1,4 @@
-set allow_experimental_analyzer = true;
+set enable_analyzer = true;
 
 select count; -- { serverError UNKNOWN_IDENTIFIER }
 
diff --git a/tests/queries/0_stateless/03097_query_log_join_processes.sql b/tests/queries/0_stateless/03097_query_log_join_processes.sql
index daf3136e3fe..135160f4709 100644
--- a/tests/queries/0_stateless/03097_query_log_join_processes.sql
+++ b/tests/queries/0_stateless/03097_query_log_join_processes.sql
@@ -2,6 +2,6 @@
 
 SYSTEM FLUSH LOGS;
 
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 
 SELECT count(1) as num, hostName() as hostName FROM system.query_log as a INNER JOIN system.processes as b on a.query_id = b.query_id and type = 'QueryStart' and dateDiff('second', event_time, now()) > 5 and current_database = currentDatabase() FORMAT Null;
diff --git a/tests/queries/0_stateless/03098_prefer_column_to_alias_subquery.sql b/tests/queries/0_stateless/03098_prefer_column_to_alias_subquery.sql
index cf3768e2f29..cb41151b9c9 100644
--- a/tests/queries/0_stateless/03098_prefer_column_to_alias_subquery.sql
+++ b/tests/queries/0_stateless/03098_prefer_column_to_alias_subquery.sql
@@ -13,7 +13,7 @@ CREATE TABLE clickhouse_alias_issue_2 (
     column_2 Nullable(Float32)
 ) Engine=Memory;
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 INSERT INTO `clickhouse_alias_issue_1`
 VALUES (1, 100), (2, 200), (3, 300);
@@ -28,7 +28,7 @@ VALUES (1, 10), (2, 20), (3, 30);
 -- \N	30	3
 -- \N	20	2
 -- \N	10	1
-SELECT * 
+SELECT *
 FROM
 (
 SELECT
diff --git a/tests/queries/0_stateless/03099_analyzer_multi_join.sql b/tests/queries/0_stateless/03099_analyzer_multi_join.sql
index 67985962ba8..dfdaeeea770 100644
--- a/tests/queries/0_stateless/03099_analyzer_multi_join.sql
+++ b/tests/queries/0_stateless/03099_analyzer_multi_join.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/56503
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 SELECT
     tb1.owner_id AS owner_id,
diff --git a/tests/queries/0_stateless/03100_analyzer_constants_in_multiif.sql b/tests/queries/0_stateless/03100_analyzer_constants_in_multiif.sql
index 04666411760..c9a4000d25e 100644
--- a/tests/queries/0_stateless/03100_analyzer_constants_in_multiif.sql
+++ b/tests/queries/0_stateless/03100_analyzer_constants_in_multiif.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/59101
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 CREATE TABLE users (name String, age Int16) ENGINE=Memory;
 INSERT INTO users VALUES ('John', 33);
diff --git a/tests/queries/0_stateless/03101_analyzer_identifiers_1.sql b/tests/queries/0_stateless/03101_analyzer_identifiers_1.sql
index 2e0ad7d8a5b..499f712e57a 100644
--- a/tests/queries/0_stateless/03101_analyzer_identifiers_1.sql
+++ b/tests/queries/0_stateless/03101_analyzer_identifiers_1.sql
@@ -1,6 +1,6 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/23194
 -- This test add query-templates for fuzzer
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP DATABASE IF EXISTS {CLICKHOUSE_DATABASE:Identifier};
 CREATE DATABASE {CLICKHOUSE_DATABASE:Identifier};
diff --git a/tests/queries/0_stateless/03101_analyzer_identifiers_2.sql b/tests/queries/0_stateless/03101_analyzer_identifiers_2.sql
index 35f34e33689..92c3e98265f 100644
--- a/tests/queries/0_stateless/03101_analyzer_identifiers_2.sql
+++ b/tests/queries/0_stateless/03101_analyzer_identifiers_2.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/23194
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 CREATE TEMPORARY TABLE test1 (a String, nest Nested(x String, y String));
 
diff --git a/tests/queries/0_stateless/03101_analyzer_identifiers_3.sql b/tests/queries/0_stateless/03101_analyzer_identifiers_3.sql
index 77a0f040e88..997fee91930 100644
--- a/tests/queries/0_stateless/03101_analyzer_identifiers_3.sql
+++ b/tests/queries/0_stateless/03101_analyzer_identifiers_3.sql
@@ -1,7 +1,7 @@
 -- Tags: no-parallel
 -- Looks like you cannot use the query parameter as a column name.
 -- https://github.com/ClickHouse/ClickHouse/issues/23194
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP DATABASE IF EXISTS db1_03101;
 DROP DATABASE IF EXISTS db2_03101;
diff --git a/tests/queries/0_stateless/03101_analyzer_identifiers_4.sql b/tests/queries/0_stateless/03101_analyzer_identifiers_4.sql
index eba6ad09a3f..869310aa181 100644
--- a/tests/queries/0_stateless/03101_analyzer_identifiers_4.sql
+++ b/tests/queries/0_stateless/03101_analyzer_identifiers_4.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/23194
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP DATABASE IF EXISTS {CLICKHOUSE_DATABASE:Identifier};
 CREATE DATABASE {CLICKHOUSE_DATABASE:Identifier};
diff --git a/tests/queries/0_stateless/03101_analyzer_invalid_join_on.sql b/tests/queries/0_stateless/03101_analyzer_invalid_join_on.sql
index 8539376e136..d7e26862d36 100644
--- a/tests/queries/0_stateless/03101_analyzer_invalid_join_on.sql
+++ b/tests/queries/0_stateless/03101_analyzer_invalid_join_on.sql
@@ -3,23 +3,23 @@
 drop table if exists t1;
 drop table if exists t2;
 
-set allow_experimental_analyzer=1;
+set enable_analyzer=1;
 
 create table t1 (c3 String, primary key(c3)) engine = MergeTree;
 create table t2 (c11 String, primary key(c11)) engine = MergeTree;
 insert into t1 values ('succeed');
 insert into t2 values ('succeed');
 
-select 
+select
     ref_0.c11 as c_2_c30_0
-  from 
+  from
     t2 as ref_0
-      cross join (select 
-            ref_1.c3 as c_6_c28_15 
-          from 
+      cross join (select
+            ref_1.c3 as c_6_c28_15
+          from
             t1 as ref_1
        ) as subq_0
   where subq_0.c_6_c28_15 = (select c11 from t2 order by c11 limit 1);
 
 drop table if exists t1;
-drop table if exists t2;
\ No newline at end of file
+drop table if exists t2;
diff --git a/tests/queries/0_stateless/03102_prefer_column_name_to_alias.sql b/tests/queries/0_stateless/03102_prefer_column_name_to_alias.sql
index 48e97fd0841..fafdb660e44 100644
--- a/tests/queries/0_stateless/03102_prefer_column_name_to_alias.sql
+++ b/tests/queries/0_stateless/03102_prefer_column_name_to_alias.sql
@@ -4,7 +4,7 @@ DROP TABLE IF EXISTS loans;
 
 CREATE TABLE loans (loan_number int, security_id text) ENGINE=Memory;
 
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 
 INSERT INTO loans VALUES (1, 'AAA');
 INSERT INTO loans VALUES (1, 'AAA');
diff --git a/tests/queries/0_stateless/03103_positional_arguments.sql b/tests/queries/0_stateless/03103_positional_arguments.sql
index ad30719dc3e..eecaa3f4ea6 100644
--- a/tests/queries/0_stateless/03103_positional_arguments.sql
+++ b/tests/queries/0_stateless/03103_positional_arguments.sql
@@ -1,6 +1,6 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/56466
 
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 
 DROP TABLE IF EXISTS users;
 
diff --git a/tests/queries/0_stateless/03104_create_view_join.sql b/tests/queries/0_stateless/03104_create_view_join.sql
index bed3d81c9a8..a39be92b1f1 100644
--- a/tests/queries/0_stateless/03104_create_view_join.sql
+++ b/tests/queries/0_stateless/03104_create_view_join.sql
@@ -4,7 +4,7 @@ DROP TABLE IF EXISTS test_table_01;
 DROP TABLE IF EXISTS test_table_02;
 DROP TABLE IF EXISTS test_view_01;
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 CREATE TABLE test_table_01 (
     column Int32
@@ -15,12 +15,12 @@ CREATE TABLE test_table_02 (
 ) ENGINE = Memory();
 
 CREATE VIEW test_view_01 AS
-SELECT 
+SELECT
     t1.column,
     t2.column
-FROM test_table_01 AS t1 
+FROM test_table_01 AS t1
     INNER JOIN test_table_02 AS t2 ON t1.column = t2.column;
 
 DROP TABLE IF EXISTS test_table_01;
 DROP TABLE IF EXISTS test_table_02;
-DROP TABLE IF EXISTS test_view_01;
\ No newline at end of file
+DROP TABLE IF EXISTS test_view_01;
diff --git a/tests/queries/0_stateless/03107_ill_formed_select_in_materialized_view.sql b/tests/queries/0_stateless/03107_ill_formed_select_in_materialized_view.sql
index d142bf1662e..5f6ec74bdeb 100644
--- a/tests/queries/0_stateless/03107_ill_formed_select_in_materialized_view.sql
+++ b/tests/queries/0_stateless/03107_ill_formed_select_in_materialized_view.sql
@@ -6,10 +6,10 @@ DROP TABLE iF EXISTS b;
 CREATE TABLE a ( a UInt64, b UInt64) ENGINE = Memory;
 CREATE TABLE b ( b UInt64) ENGINE = Memory;
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 SET joined_subquery_requires_alias = 0;
 
 CREATE MATERIALIZED VIEW view_4 ( bb UInt64, cnt UInt64) Engine=MergeTree ORDER BY bb POPULATE AS SELECT bb, count() AS cnt FROM (SELECT a, b AS j, b AS bb FROM a INNER JOIN (SELECT b AS j, b AS bb FROM b ) USING (j)) GROUP BY bb; -- { serverError UNKNOWN_IDENTIFIER }
 
 DROP TABLE IF EXISTS a;
-DROP TABLE iF EXISTS b;
\ No newline at end of file
+DROP TABLE iF EXISTS b;
diff --git a/tests/queries/0_stateless/03108_describe_union_all.sql b/tests/queries/0_stateless/03108_describe_union_all.sql
index c5172902a2b..7e207ae2854 100644
--- a/tests/queries/0_stateless/03108_describe_union_all.sql
+++ b/tests/queries/0_stateless/03108_describe_union_all.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/8030
 
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 
-DESCRIBE (SELECT 1, 1 UNION ALL SELECT 1, 2);
\ No newline at end of file
+DESCRIBE (SELECT 1, 1 UNION ALL SELECT 1, 2);
diff --git a/tests/queries/0_stateless/03109_ast_too_big.sql b/tests/queries/0_stateless/03109_ast_too_big.sql
index 3eedf305f44..1464f90fe83 100644
--- a/tests/queries/0_stateless/03109_ast_too_big.sql
+++ b/tests/queries/0_stateless/03109_ast_too_big.sql
@@ -1,6 +1,6 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/32139
 
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 
 WITH
   data AS (
@@ -50,4 +50,4 @@ SELECT
   value12 AS v12,
   value13 AS v13,
   value14 AS v14
-FORMAT Null;
\ No newline at end of file
+FORMAT Null;
diff --git a/tests/queries/0_stateless/03110_unicode_alias.sql b/tests/queries/0_stateless/03110_unicode_alias.sql
index b8cbe7390fa..aa33195ea51 100644
--- a/tests/queries/0_stateless/03110_unicode_alias.sql
+++ b/tests/queries/0_stateless/03110_unicode_alias.sql
@@ -1,6 +1,6 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/47288
 
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 
 select 1 as `c0`
 from (
diff --git a/tests/queries/0_stateless/03111_inner_join_group_by.sql b/tests/queries/0_stateless/03111_inner_join_group_by.sql
index fcc8c55a566..6ebaacfc3fe 100644
--- a/tests/queries/0_stateless/03111_inner_join_group_by.sql
+++ b/tests/queries/0_stateless/03111_inner_join_group_by.sql
@@ -1,6 +1,6 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/50705
 
-set allow_experimental_analyzer=1;
+set enable_analyzer=1;
 
 SELECT
     count(s0.number),
@@ -15,4 +15,4 @@ INNER JOIN
     LIMIT 10
 ) AS s1 ON s0.number = s1.number
 GROUP BY s0.number > 5
-LIMIT 10 -- {serverError NOT_AN_AGGREGATE}
\ No newline at end of file
+LIMIT 10 -- {serverError NOT_AN_AGGREGATE}
diff --git a/tests/queries/0_stateless/03112_analyzer_not_found_column_in_block.sql b/tests/queries/0_stateless/03112_analyzer_not_found_column_in_block.sql
index eb07ff1d837..cc734b2c49a 100644
--- a/tests/queries/0_stateless/03112_analyzer_not_found_column_in_block.sql
+++ b/tests/queries/0_stateless/03112_analyzer_not_found_column_in_block.sql
@@ -15,7 +15,7 @@ PRIMARY KEY (user_id, timestamp);
 INSERT INTO my_first_table (user_id, message, timestamp, metric) VALUES
     (101, 'Hello, ClickHouse!',                                 now(),       -1.0    ),     (102, 'Insert a lot of rows per batch',                     yesterday(), 1.41421 ),    (102, 'Sort your data based on your commonly-used queries', today(),     2.718   ),    (101, 'Granules are the smallest chunks of data read',      now() + 5,   3.14159 );
 
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 
 SELECT
     user_id
@@ -24,4 +24,4 @@ FROM my_first_table
 WHERE timestamp > 0 and user_id IN (101)
 LIMIT 2 BY user_id;
 
-DROP TABLE IF EXISTS my_first_table;
\ No newline at end of file
+DROP TABLE IF EXISTS my_first_table;
diff --git a/tests/queries/0_stateless/03113_analyzer_not_found_column_in_block_2.sql b/tests/queries/0_stateless/03113_analyzer_not_found_column_in_block_2.sql
index 7bcc6565ea0..4389bdf83fa 100644
--- a/tests/queries/0_stateless/03113_analyzer_not_found_column_in_block_2.sql
+++ b/tests/queries/0_stateless/03113_analyzer_not_found_column_in_block_2.sql
@@ -4,11 +4,11 @@ drop table if exists t;
 
 create table t  (ID String) Engine= Memory() ;
 insert into t values('a'),('b'),('c');
-     
+
 -- This optimization is disabled by default and even its description says that it could lead to
 -- inconsistencies for distributed queries.
 set optimize_if_transform_strings_to_enum=0;
-set allow_experimental_analyzer=1;
+set enable_analyzer=1;
 
 SELECT multiIf( ((multiIf(ID='' AND (ID = 'a' OR ID = 'c' OR ID = 'b'),'a','x') as y) = 'c') OR
                 (multiIf(ID='' AND (ID = 'a' OR ID = 'c' OR ID = 'b'),'a','x') = 'b') OR
@@ -18,4 +18,4 @@ SELECT multiIf( ((multiIf(ID='' AND (ID = 'a' OR ID = 'c' OR ID = 'b'),'a','x')
 FROM remote('127.0.0.{1,2}', currentDatabase(), t)
 GROUP BY alias;
 
-drop table if exists t;
\ No newline at end of file
+drop table if exists t;
diff --git a/tests/queries/0_stateless/03114_analyzer_cte_with_join.sql b/tests/queries/0_stateless/03114_analyzer_cte_with_join.sql
index 140197c7104..65dd3cb66b6 100644
--- a/tests/queries/0_stateless/03114_analyzer_cte_with_join.sql
+++ b/tests/queries/0_stateless/03114_analyzer_cte_with_join.sql
@@ -1,7 +1,7 @@
 -- Tags: no-replicated-database
 -- https://github.com/ClickHouse/ClickHouse/issues/58500
 
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 
 drop table if exists t;
 
diff --git a/tests/queries/0_stateless/03115_alias_exists_column.sql b/tests/queries/0_stateless/03115_alias_exists_column.sql
index 654cdd71175..65fc0d8f9a9 100644
--- a/tests/queries/0_stateless/03115_alias_exists_column.sql
+++ b/tests/queries/0_stateless/03115_alias_exists_column.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/44412
 
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 
 SELECT EXISTS(SELECT 1) AS mycheck FORMAT TSVWithNames;
diff --git a/tests/queries/0_stateless/03116_analyzer_explicit_alias_as_column_name.sql b/tests/queries/0_stateless/03116_analyzer_explicit_alias_as_column_name.sql
index 16035b8e72a..d3e3a29077a 100644
--- a/tests/queries/0_stateless/03116_analyzer_explicit_alias_as_column_name.sql
+++ b/tests/queries/0_stateless/03116_analyzer_explicit_alias_as_column_name.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/39923
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 
 SELECT
     errors.name AS labels,
diff --git a/tests/queries/0_stateless/03117_analyzer_same_column_name_as_func.sql b/tests/queries/0_stateless/03117_analyzer_same_column_name_as_func.sql
index 43f428a1bf9..a3f4da89525 100644
--- a/tests/queries/0_stateless/03117_analyzer_same_column_name_as_func.sql
+++ b/tests/queries/0_stateless/03117_analyzer_same_column_name_as_func.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/39855
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 
 create table x(
     a UInt64,
diff --git a/tests/queries/0_stateless/03118_analyzer_multi_join_prewhere.sql b/tests/queries/0_stateless/03118_analyzer_multi_join_prewhere.sql
index 8680e9215c3..84f89c2c647 100644
--- a/tests/queries/0_stateless/03118_analyzer_multi_join_prewhere.sql
+++ b/tests/queries/0_stateless/03118_analyzer_multi_join_prewhere.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/4596
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 
 CREATE TABLE a1 ( ANIMAL Nullable(String) ) engine = MergeTree order by tuple();
 insert into a1 values('CROCO');
diff --git a/tests/queries/0_stateless/03119_analyzer_window_function_in_CTE_alias.sql b/tests/queries/0_stateless/03119_analyzer_window_function_in_CTE_alias.sql
index 2eb2c66b551..edbb324bda2 100644
--- a/tests/queries/0_stateless/03119_analyzer_window_function_in_CTE_alias.sql
+++ b/tests/queries/0_stateless/03119_analyzer_window_function_in_CTE_alias.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/47422
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 
 DROP TEMPORARY TABLE IF EXISTS test;
 CREATE TEMPORARY TABLE test (a Float32, id UInt64);
diff --git a/tests/queries/0_stateless/03120_analyzer_dist_join.sql b/tests/queries/0_stateless/03120_analyzer_dist_join.sql
index 624da39c69b..e40df56c5ac 100644
--- a/tests/queries/0_stateless/03120_analyzer_dist_join.sql
+++ b/tests/queries/0_stateless/03120_analyzer_dist_join.sql
@@ -1,6 +1,6 @@
 -- Tags: no-replicated-database
 -- https://github.com/ClickHouse/ClickHouse/issues/8547
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 SET distributed_foreground_insert=1;
 
 CREATE TABLE a1_replicated ON CLUSTER test_shard_localhost (
diff --git a/tests/queries/0_stateless/03120_analyzer_param_in_CTE_alias.sql b/tests/queries/0_stateless/03120_analyzer_param_in_CTE_alias.sql
index 16fffae4737..d1e3d5a2ffa 100644
--- a/tests/queries/0_stateless/03120_analyzer_param_in_CTE_alias.sql
+++ b/tests/queries/0_stateless/03120_analyzer_param_in_CTE_alias.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/33000
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 
 SET param_test_a=30;
 
diff --git a/tests/queries/0_stateless/03121_analyzer_filed_redefenition_in_subquery.sql b/tests/queries/0_stateless/03121_analyzer_filed_redefenition_in_subquery.sql
index fe05259a320..891eb7aac1c 100644
--- a/tests/queries/0_stateless/03121_analyzer_filed_redefenition_in_subquery.sql
+++ b/tests/queries/0_stateless/03121_analyzer_filed_redefenition_in_subquery.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/14739
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 
 drop table if exists test_subquery;
 
diff --git a/tests/queries/0_stateless/03122_analyzer_collate_in_window_function.sql b/tests/queries/0_stateless/03122_analyzer_collate_in_window_function.sql
index 707c5d889f7..efd2e2fc873 100644
--- a/tests/queries/0_stateless/03122_analyzer_collate_in_window_function.sql
+++ b/tests/queries/0_stateless/03122_analyzer_collate_in_window_function.sql
@@ -1,6 +1,6 @@
 -- Tags: no-fasttest
 -- https://github.com/ClickHouse/ClickHouse/issues/44039
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 
 create table test_window_collate(c1 String, c2 String) engine=MergeTree order by c1;
 insert into test_window_collate values('1', '上海');
diff --git a/tests/queries/0_stateless/03123_analyzer_dist_join_CTE.sql b/tests/queries/0_stateless/03123_analyzer_dist_join_CTE.sql
index 4fb8e0b91c4..4d9f5e2971a 100644
--- a/tests/queries/0_stateless/03123_analyzer_dist_join_CTE.sql
+++ b/tests/queries/0_stateless/03123_analyzer_dist_join_CTE.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/22923
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 SET prefer_localhost_replica=0;
 
 create table "t0" (a Int64, b Int64) engine = MergeTree() partition by a order by a;
diff --git a/tests/queries/0_stateless/03124_analyzer_nested_CTE_dist_in.sql b/tests/queries/0_stateless/03124_analyzer_nested_CTE_dist_in.sql
index 406a50c6d16..be5346efa1c 100644
--- a/tests/queries/0_stateless/03124_analyzer_nested_CTE_dist_in.sql
+++ b/tests/queries/0_stateless/03124_analyzer_nested_CTE_dist_in.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/23865
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 
 create table table_local engine = Memory AS select * from numbers(10);
 
diff --git a/tests/queries/0_stateless/03125_analyzer_CTE_two_joins.sql b/tests/queries/0_stateless/03125_analyzer_CTE_two_joins.sql
index f7d5bb5f195..934e2bc3656 100644
--- a/tests/queries/0_stateless/03125_analyzer_CTE_two_joins.sql
+++ b/tests/queries/0_stateless/03125_analyzer_CTE_two_joins.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/29748
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 
 create table events ( distinct_id String ) engine = Memory;
 
diff --git a/tests/queries/0_stateless/03126_column_not_under_group_by.sql b/tests/queries/0_stateless/03126_column_not_under_group_by.sql
index 890a4e823a9..516126c899c 100644
--- a/tests/queries/0_stateless/03126_column_not_under_group_by.sql
+++ b/tests/queries/0_stateless/03126_column_not_under_group_by.sql
@@ -1,7 +1,6 @@
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 
 SELECT v.x, r.a, sum(c)
 FROM (select 1 x, 2 c) AS v
 ANY LEFT JOIN (SELECT 1 x, 2 a) AS r ON v.x = r.x
 GROUP BY v.x; -- { serverError NOT_AN_AGGREGATE}
-
diff --git a/tests/queries/0_stateless/03129_cte_with_final.sql b/tests/queries/0_stateless/03129_cte_with_final.sql
index 01e1ca6dfb0..2a0714ec571 100644
--- a/tests/queries/0_stateless/03129_cte_with_final.sql
+++ b/tests/queries/0_stateless/03129_cte_with_final.sql
@@ -12,7 +12,7 @@ ORDER BY key;
 INSERT INTO t Values (1, 'first', '2024-04-19 01:01:01');
 INSERT INTO t Values (1, 'first', '2024-04-19 01:01:01');
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 EXPLAIN QUERY TREE passes=1
 WITH merged_test AS(
diff --git a/tests/queries/0_stateless/03130_analyzer_self_join_group_by.sql b/tests/queries/0_stateless/03130_analyzer_self_join_group_by.sql
index 66b6b99981b..81af10c4a64 100644
--- a/tests/queries/0_stateless/03130_analyzer_self_join_group_by.sql
+++ b/tests/queries/0_stateless/03130_analyzer_self_join_group_by.sql
@@ -2,7 +2,7 @@ DROP TABLE IF EXISTS t1;
 CREATE TABLE t1 (x Int32) ENGINE = MergeTree ORDER BY x;
 INSERT INTO t1 VALUES (1), (2), (3);
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 SELECT t2.x FROM t1 JOIN t1 as t2 ON t1.x = t2.x GROUP BY t1.x; -- { serverError NOT_AN_AGGREGATE }
 SELECT t2.number FROM numbers(10) as t1 JOIN numbers(10) as t2 ON t1.number = t2.number GROUP BY t1.number; -- { serverError NOT_AN_AGGREGATE }
diff --git a/tests/queries/0_stateless/03130_convert_outer_join_to_inner_join.sql b/tests/queries/0_stateless/03130_convert_outer_join_to_inner_join.sql
index 28362f1f469..4e42d94fd79 100644
--- a/tests/queries/0_stateless/03130_convert_outer_join_to_inner_join.sql
+++ b/tests/queries/0_stateless/03130_convert_outer_join_to_inner_join.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 SET join_algorithm = 'hash';
 
 DROP TABLE IF EXISTS test_table_1;
diff --git a/tests/queries/0_stateless/03132_rewrite_aggregate_function_with_if_implicit_cast.sql b/tests/queries/0_stateless/03132_rewrite_aggregate_function_with_if_implicit_cast.sql
index 4e8096fbed1..7b7237ea7d1 100644
--- a/tests/queries/0_stateless/03132_rewrite_aggregate_function_with_if_implicit_cast.sql
+++ b/tests/queries/0_stateless/03132_rewrite_aggregate_function_with_if_implicit_cast.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 -- { echoOn }
 SELECT concat(1, sum(if(0, toUInt128(concat('%', toLowCardinality(toNullable(1)), toUInt256(1))), materialize(0))));
 SELECT any(if((number % 10) = 5, number, CAST(NULL, 'Nullable(Int128)'))) AS a, toTypeName(a) FROM numbers(100) AS a;
diff --git a/tests/queries/0_stateless/03132_sqlancer_union_all.sql b/tests/queries/0_stateless/03132_sqlancer_union_all.sql
index 576da9e53d5..2502ce31ec3 100644
--- a/tests/queries/0_stateless/03132_sqlancer_union_all.sql
+++ b/tests/queries/0_stateless/03132_sqlancer_union_all.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS t0;
 DROP TABLE IF EXISTS t1;
diff --git a/tests/queries/0_stateless/03142_untuple_crash.sql b/tests/queries/0_stateless/03142_untuple_crash.sql
index ac5dbba0de1..45106973271 100644
--- a/tests/queries/0_stateless/03142_untuple_crash.sql
+++ b/tests/queries/0_stateless/03142_untuple_crash.sql
@@ -1,2 +1,2 @@
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 SELECT untuple(x -> 0) -- { serverError ILLEGAL_TYPE_OF_ARGUMENT }
diff --git a/tests/queries/0_stateless/03142_window_function_limit_by.sql b/tests/queries/0_stateless/03142_window_function_limit_by.sql
index 44dd890db41..a3f6b56aee5 100644
--- a/tests/queries/0_stateless/03142_window_function_limit_by.sql
+++ b/tests/queries/0_stateless/03142_window_function_limit_by.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 
 -- https://github.com/ClickHouse/ClickHouse/issues/55965
@@ -8,7 +8,7 @@ CREATE TABLE error_win_func
     `k` String,
     `in` UInt64,
     `out` UInt64
-) 
+)
 ENGINE = MergeTree
 ORDER BY k AS
 SELECT * from VALUES (('a', 2, 4), ('a', 4, 2), ('a', 6, 3), ('a', 8, 4));
@@ -16,7 +16,7 @@ SELECT * from VALUES (('a', 2, 4), ('a', 4, 2), ('a', 6, 3), ('a', 8, 4));
 SELECT
     k,
     in / out AS ratio,
-    count(*) OVER w AS count_rows_w    
+    count(*) OVER w AS count_rows_w
 FROM error_win_func
 WINDOW
     w AS (ROWS BETWEEN CURRENT ROW AND 3 FOLLOWING)
@@ -38,4 +38,3 @@ WHERE st IN ('x', 'y')
 LIMIT 1 BY m;
 
 DROP TABLE t;
-
diff --git a/tests/queries/0_stateless/03143_cte_scope.sql b/tests/queries/0_stateless/03143_cte_scope.sql
index 1b1d9444651..cf4478e2ff9 100644
--- a/tests/queries/0_stateless/03143_cte_scope.sql
+++ b/tests/queries/0_stateless/03143_cte_scope.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/56287
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 DROP TABLE IF EXISTS tmp_a;
 DROP TABLE IF EXISTS tmp_b;
 
diff --git a/tests/queries/0_stateless/03143_group_by_constant_secondary.sql b/tests/queries/0_stateless/03143_group_by_constant_secondary.sql
index 030e3212748..099160dd522 100644
--- a/tests/queries/0_stateless/03143_group_by_constant_secondary.sql
+++ b/tests/queries/0_stateless/03143_group_by_constant_secondary.sql
@@ -3,6 +3,6 @@ SELECT count()
 FROM remote(test_cluster_two_shards, system, one)
 GROUP BY 'hi'
 SETTINGS
-    allow_experimental_analyzer = 1,
+    enable_analyzer = 1,
     group_by_two_level_threshold = 1,
     group_by_two_level_threshold_bytes = 33950592;
diff --git a/tests/queries/0_stateless/03143_parallel_replicas_mat_view_bug.sql b/tests/queries/0_stateless/03143_parallel_replicas_mat_view_bug.sql
index 97ed29802c7..02a8a2f3ce0 100644
--- a/tests/queries/0_stateless/03143_parallel_replicas_mat_view_bug.sql
+++ b/tests/queries/0_stateless/03143_parallel_replicas_mat_view_bug.sql
@@ -2,7 +2,7 @@ DROP TABLE IF EXISTS mv_table;
 DROP TABLE IF EXISTS null_table;
 
 SET cluster_for_parallel_replicas='parallel_replicas', max_parallel_replicas=4, allow_experimental_parallel_reading_from_replicas=1;
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 
 CREATE TABLE null_table (str String) ENGINE = Null;
 CREATE MATERIALIZED VIEW mv_table (str String) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/03143_parallel_replicas_mat_view_bug', '{replica}') ORDER BY str AS SELECT str AS str FROM null_table;
diff --git a/tests/queries/0_stateless/03144_aggregate_states_with_different_types.sql b/tests/queries/0_stateless/03144_aggregate_states_with_different_types.sql
index 5bb4a8c9c9c..4445d5b8449 100644
--- a/tests/queries/0_stateless/03144_aggregate_states_with_different_types.sql
+++ b/tests/queries/0_stateless/03144_aggregate_states_with_different_types.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 select * APPLY hex
 from (
diff --git a/tests/queries/0_stateless/03144_invalid_filter.sql b/tests/queries/0_stateless/03144_invalid_filter.sql
index deb8d7b96b3..5b434972cc7 100644
--- a/tests/queries/0_stateless/03144_invalid_filter.sql
+++ b/tests/queries/0_stateless/03144_invalid_filter.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/48049
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 CREATE TABLE test_table (`id` UInt64, `value` String) ENGINE = TinyLog() AS Select number, number::String from numbers(10);
 
diff --git a/tests/queries/0_stateless/03146_bug47862.sql b/tests/queries/0_stateless/03146_bug47862.sql
index 918f2316bea..0f411400618 100644
--- a/tests/queries/0_stateless/03146_bug47862.sql
+++ b/tests/queries/0_stateless/03146_bug47862.sql
@@ -9,4 +9,4 @@ FROM (
     WHERE val_idx != 0
 )
 WHERE cast_res > 0
-SETTINGS allow_experimental_analyzer = 1;
+SETTINGS enable_analyzer = 1;
diff --git a/tests/queries/0_stateless/03146_tpc_ds_grouping.sql b/tests/queries/0_stateless/03146_tpc_ds_grouping.sql
index f48c40e9bc4..cb290086b51 100644
--- a/tests/queries/0_stateless/03146_tpc_ds_grouping.sql
+++ b/tests/queries/0_stateless/03146_tpc_ds_grouping.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/46335
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 SELECT
     key_a + key_b AS d,
     rank() OVER () AS f
diff --git a/tests/queries/0_stateless/03148_query_log_used_dictionaries.sql b/tests/queries/0_stateless/03148_query_log_used_dictionaries.sql
index 1b647a7ee62..75b7489ae5b 100644
--- a/tests/queries/0_stateless/03148_query_log_used_dictionaries.sql
+++ b/tests/queries/0_stateless/03148_query_log_used_dictionaries.sql
@@ -15,7 +15,7 @@ SELECT
     dictGet('03148_dictionary', 'name', number) as dict_value
 FROM numbers(1)
 SETTINGS
-    allow_experimental_analyzer = 1,
+    enable_analyzer = 1,
     log_comment = 'simple_with_analyzer'
 FORMAT Null;
 
@@ -34,7 +34,7 @@ FROM (
     FROM numbers(1)
 ) t
 SETTINGS
-    allow_experimental_analyzer = 1,
+    enable_analyzer = 1,
     log_comment = 'nested_with_analyzer'
 FORMAT Null;
 
@@ -50,7 +50,7 @@ SELECT
     dictGet('03148_dictionary', 'name', number) as dict_value
 FROM numbers(1)
 SETTINGS
-    allow_experimental_analyzer = 0,
+    enable_analyzer = 0,
     log_comment = 'simple_without_analyzer'
 FORMAT Null;
 
@@ -69,7 +69,7 @@ FROM (
     FROM numbers(1)
 ) t
 SETTINGS
-    allow_experimental_analyzer = 0,
+    enable_analyzer = 0,
     log_comment = 'nested_without_analyzer'
 FORMAT Null;
 
diff --git a/tests/queries/0_stateless/03148_setting_max_streams_to_max_threads_ratio_overflow.sql b/tests/queries/0_stateless/03148_setting_max_streams_to_max_threads_ratio_overflow.sql
index af326c15bd8..38f25f60e09 100644
--- a/tests/queries/0_stateless/03148_setting_max_streams_to_max_threads_ratio_overflow.sql
+++ b/tests/queries/0_stateless/03148_setting_max_streams_to_max_threads_ratio_overflow.sql
@@ -7,8 +7,8 @@ CREATE TABLE test_table
 
 INSERT INTO test_table VALUES (0, 'Value_0');
 
-SELECT * FROM test_table SETTINGS max_threads = 1025, max_streams_to_max_threads_ratio = -9223372036854775808, allow_experimental_analyzer = 1; -- { serverError PARAMETER_OUT_OF_BOUND }
+SELECT * FROM test_table SETTINGS max_threads = 1025, max_streams_to_max_threads_ratio = -9223372036854775808, enable_analyzer = 1; -- { serverError PARAMETER_OUT_OF_BOUND }
 
-SELECT * FROM test_table SETTINGS max_threads = 1025, max_streams_to_max_threads_ratio = -9223372036854775808, allow_experimental_analyzer = 0; -- { serverError PARAMETER_OUT_OF_BOUND }
+SELECT * FROM test_table SETTINGS max_threads = 1025, max_streams_to_max_threads_ratio = -9223372036854775808, enable_analyzer = 0; -- { serverError PARAMETER_OUT_OF_BOUND }
 
 DROP TABLE test_table;
diff --git a/tests/queries/0_stateless/03150_grouping_sets_use_nulls_pushdown.sql b/tests/queries/0_stateless/03150_grouping_sets_use_nulls_pushdown.sql
index a0bd1381351..c39143216d4 100644
--- a/tests/queries/0_stateless/03150_grouping_sets_use_nulls_pushdown.sql
+++ b/tests/queries/0_stateless/03150_grouping_sets_use_nulls_pushdown.sql
@@ -21,21 +21,21 @@ SELECT *
 FROM ( SELECT day_ FROM test_grouping_sets_predicate GROUP BY GROUPING SETS ( (day_, type_1), (day_) ) )
 WHERE day_ = '2023-01-05'
 ORDER BY *
-SETTINGS allow_experimental_analyzer=1;
+SETTINGS enable_analyzer=1;
 
 SELECT *
 FROM ( SELECT * FROM test_grouping_sets_predicate GROUP BY GROUPING SETS ( (day_, type_1), (day_) ) )
 WHERE day_ = '2023-01-05'
 GROUP BY *
 ORDER BY ALL
-SETTINGS allow_experimental_analyzer=1;
+SETTINGS enable_analyzer=1;
 
 SELECT *
 FROM ( SELECT * FROM test_grouping_sets_predicate GROUP BY GROUPING SETS ( (*), (day_) ) )
 WHERE day_ = '2023-01-05'
 GROUP BY GROUPING SETS (*)
 ORDER BY type_1
-SETTINGS allow_experimental_analyzer=1;
+SETTINGS enable_analyzer=1;
 
 SELECT *
 FROM ( SELECT day_, COUNT(*) FROM test_grouping_sets_predicate GROUP BY GROUPING SETS ( (day_, type_1), (day_) ) )
diff --git a/tests/queries/0_stateless/03151_analyzer_view_read_only_necessary_columns.sql b/tests/queries/0_stateless/03151_analyzer_view_read_only_necessary_columns.sql
index 40204b5cd03..ac86a8705d5 100644
--- a/tests/queries/0_stateless/03151_analyzer_view_read_only_necessary_columns.sql
+++ b/tests/queries/0_stateless/03151_analyzer_view_read_only_necessary_columns.sql
@@ -8,7 +8,7 @@ CREATE TABLE test_table
 DROP VIEW IF EXISTS test_view;
 CREATE VIEW test_view AS SELECT id, value FROM test_table;
 
-EXPLAIN header = 1 SELECT sum(id) FROM test_view settings allow_experimental_analyzer=1;
+EXPLAIN header = 1 SELECT sum(id) FROM test_view settings enable_analyzer=1;
 
 DROP VIEW test_view;
 DROP TABLE test_table;
diff --git a/tests/queries/0_stateless/03152_join_filter_push_down_equivalent_columns.sql b/tests/queries/0_stateless/03152_join_filter_push_down_equivalent_columns.sql
index 645e89034d7..ea8a9e1f8d7 100644
--- a/tests/queries/0_stateless/03152_join_filter_push_down_equivalent_columns.sql
+++ b/tests/queries/0_stateless/03152_join_filter_push_down_equivalent_columns.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS users;
 CREATE TABLE users (uid Int16, name String, age Int16) ENGINE=MergeTree order by (uid, name);
diff --git a/tests/queries/0_stateless/03154_recursive_cte_distributed.sql b/tests/queries/0_stateless/03154_recursive_cte_distributed.sql
index b8c3356d5d6..47e0b9aad0b 100644
--- a/tests/queries/0_stateless/03154_recursive_cte_distributed.sql
+++ b/tests/queries/0_stateless/03154_recursive_cte_distributed.sql
@@ -1,6 +1,6 @@
 -- Tags: shard
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS test_table;
 CREATE TABLE test_table
diff --git a/tests/queries/0_stateless/03155_analyzer_interpolate.sql b/tests/queries/0_stateless/03155_analyzer_interpolate.sql
index 30423cb86ff..42c5f5ef65f 100644
--- a/tests/queries/0_stateless/03155_analyzer_interpolate.sql
+++ b/tests/queries/0_stateless/03155_analyzer_interpolate.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/62464
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 SELECT n, [number] AS inter FROM (
    SELECT toFloat32(number % 10) AS n, number
diff --git a/tests/queries/0_stateless/03155_in_nested_subselects.sql b/tests/queries/0_stateless/03155_in_nested_subselects.sql
index 4f5ccd30aa3..faecb73040d 100644
--- a/tests/queries/0_stateless/03155_in_nested_subselects.sql
+++ b/tests/queries/0_stateless/03155_in_nested_subselects.sql
@@ -1,5 +1,5 @@
 -- https://github.com/ClickHouse/ClickHouse/issues/63833
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 create table Example (id Int32) engine = MergeTree ORDER BY id;
 INSERT INTO Example SELECT number AS id FROM numbers(2);
@@ -8,10 +8,10 @@ create table Null engine=Null as Example ;
 --create table Null engine=MergeTree order by id as Example ;
 
 create materialized view Transform to Example as
-select * from Null 
-join ( select * FROM Example 
+select * from Null
+join ( select * FROM Example
        WHERE id IN (SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM (SELECT * FROM Null)))))
-     ) as old  
+     ) as old
 using id;
 
 INSERT INTO Null SELECT number AS id FROM numbers(2);
diff --git a/tests/queries/0_stateless/03161_cnf_reduction.reference b/tests/queries/0_stateless/03161_cnf_reduction.reference
index 5e39c0f3223..41051af362f 100644
--- a/tests/queries/0_stateless/03161_cnf_reduction.reference
+++ b/tests/queries/0_stateless/03161_cnf_reduction.reference
@@ -2,7 +2,7 @@
 SELECT id
 FROM `03161_table`
 WHERE f
-SETTINGS convert_query_to_cnf = 1, optimize_using_constraints = 1, allow_experimental_analyzer = 1
+SETTINGS convert_query_to_cnf = 1, optimize_using_constraints = 1, enable_analyzer = 1
 
 -- Expected result with analyzer:
 1
@@ -11,7 +11,7 @@ SETTINGS convert_query_to_cnf = 1, optimize_using_constraints = 1, allow_experim
 SELECT id
 FROM `03161_table`
 WHERE f
-SETTINGS convert_query_to_cnf = 1, optimize_using_constraints = 1, allow_experimental_analyzer = 0
+SETTINGS convert_query_to_cnf = 1, optimize_using_constraints = 1, enable_analyzer = 0
 
 -- Expected result w/o analyzer:
 1
diff --git a/tests/queries/0_stateless/03161_cnf_reduction.sql b/tests/queries/0_stateless/03161_cnf_reduction.sql
index b34e9171d45..c232823e9cd 100644
--- a/tests/queries/0_stateless/03161_cnf_reduction.sql
+++ b/tests/queries/0_stateless/03161_cnf_reduction.sql
@@ -10,7 +10,7 @@ EXPLAIN SYNTAX
 SELECT id
 FROM 03161_table
 WHERE f AND (NOT(f) OR f)
-SETTINGS convert_query_to_cnf = 1, optimize_using_constraints = 1, allow_experimental_analyzer = 1;
+SETTINGS convert_query_to_cnf = 1, optimize_using_constraints = 1, enable_analyzer = 1;
 
 SELECT '';
 
@@ -19,7 +19,7 @@ SELECT '-- Expected result with analyzer:';
 SELECT id
 FROM 03161_table
 WHERE f AND (NOT(f) OR f)
-SETTINGS convert_query_to_cnf = 1, optimize_using_constraints = 1, allow_experimental_analyzer = 1;
+SETTINGS convert_query_to_cnf = 1, optimize_using_constraints = 1, enable_analyzer = 1;
 
 SELECT '';
 
@@ -29,7 +29,7 @@ EXPLAIN SYNTAX
 SELECT id
 FROM 03161_table
 WHERE f AND (NOT(f) OR f)
-SETTINGS convert_query_to_cnf = 1, optimize_using_constraints = 1, allow_experimental_analyzer = 0;
+SETTINGS convert_query_to_cnf = 1, optimize_using_constraints = 1, enable_analyzer = 0;
 
 SELECT '';
 
@@ -38,7 +38,7 @@ SELECT '-- Expected result w/o analyzer:';
 SELECT id
 FROM 03161_table
 WHERE f AND (NOT(f) OR f)
-SETTINGS convert_query_to_cnf = 1, optimize_using_constraints = 1, allow_experimental_analyzer = 0;
+SETTINGS convert_query_to_cnf = 1, optimize_using_constraints = 1, enable_analyzer = 0;
 
 DROP TABLE IF EXISTS 03161_table;
 
@@ -58,7 +58,7 @@ SELECT '-- Reproducer from the issue with analyzer';
 SELECT count()
 FROM 03161_reproducer
 WHERE ((NOT c2) AND c2 AND (NOT c1)) OR ((NOT c2) AND c3 AND (NOT c5)) OR ((NOT c7) AND (NOT c8)) OR (c9 AND c6 AND c8 AND (NOT c8) AND (NOT c7))
-SETTINGS convert_query_to_cnf = 1, optimize_using_constraints = 1, allow_experimental_analyzer = 1;
+SETTINGS convert_query_to_cnf = 1, optimize_using_constraints = 1, enable_analyzer = 1;
 
 SELECT '';
 
@@ -67,6 +67,6 @@ SELECT '-- Reproducer from the issue w/o analyzer';
 SELECT count()
 FROM 03161_reproducer
 WHERE ((NOT c2) AND c2 AND (NOT c1)) OR ((NOT c2) AND c3 AND (NOT c5)) OR ((NOT c7) AND (NOT c8)) OR (c9 AND c6 AND c8 AND (NOT c8) AND (NOT c7))
-SETTINGS convert_query_to_cnf = 1, optimize_using_constraints = 1, allow_experimental_analyzer = 0;
+SETTINGS convert_query_to_cnf = 1, optimize_using_constraints = 1, enable_analyzer = 0;
 
 DROP TABLE IF EXISTS 03161_reproducer;
diff --git a/tests/queries/0_stateless/03164_analyzer_global_in_alias.sql b/tests/queries/0_stateless/03164_analyzer_global_in_alias.sql
index 00c293334ee..ccfacd12d98 100644
--- a/tests/queries/0_stateless/03164_analyzer_global_in_alias.sql
+++ b/tests/queries/0_stateless/03164_analyzer_global_in_alias.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 SELECT 1 GLOBAL IN (SELECT 1) AS s, s FROM remote('127.0.0.{2,3}', system.one) GROUP BY 1;
 SELECT 1 GLOBAL IN (SELECT 1) AS s FROM remote('127.0.0.{2,3}', system.one) GROUP BY 1;
 
diff --git a/tests/queries/0_stateless/03164_early_constant_folding_analyzer.sql b/tests/queries/0_stateless/03164_early_constant_folding_analyzer.sql
index dbffbc1af71..b1018d00082 100644
--- a/tests/queries/0_stateless/03164_early_constant_folding_analyzer.sql
+++ b/tests/queries/0_stateless/03164_early_constant_folding_analyzer.sql
@@ -27,4 +27,4 @@ ENGINE = MergeTree ORDER BY (date, pull_request_number, commit_sha, check_name,
 insert into checks select * from generateRandom() limit 1;
 
 
-select trimLeft(explain) from (explain SELECT count(1) FROM checks WHERE test_name IS NOT NULL) where explain like '%ReadFromPreparedSource%' SETTINGS allow_experimental_analyzer = 1, allow_experimental_parallel_reading_from_replicas = 0;
+select trimLeft(explain) from (explain SELECT count(1) FROM checks WHERE test_name IS NOT NULL) where explain like '%ReadFromPreparedSource%' SETTINGS enable_analyzer = 1, allow_experimental_parallel_reading_from_replicas = 0;
diff --git a/tests/queries/0_stateless/03164_materialize_skip_index.sql b/tests/queries/0_stateless/03164_materialize_skip_index.sql
index 4e59ef6b6cd..0443872596d 100644
--- a/tests/queries/0_stateless/03164_materialize_skip_index.sql
+++ b/tests/queries/0_stateless/03164_materialize_skip_index.sql
@@ -9,7 +9,7 @@ CREATE TABLE t_skip_index_insert
 )
 ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 4;
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 SET materialize_skip_indexes_on_insert = 0;
 
 SYSTEM STOP MERGES t_skip_index_insert;
diff --git a/tests/queries/0_stateless/03165_order_by_duplicate.reference b/tests/queries/0_stateless/03165_order_by_duplicate.reference
index 5d5e7a33f4a..93d312d5d02 100644
--- a/tests/queries/0_stateless/03165_order_by_duplicate.reference
+++ b/tests/queries/0_stateless/03165_order_by_duplicate.reference
@@ -36,4 +36,4 @@ QUERY id: 0
   LIMIT BY
     LIST id: 16, nodes: 1
       COLUMN id: 2, column_name: id, result_type: UInt64, source_id: 3
-  SETTINGS allow_experimental_analyzer=1
+  SETTINGS enable_analyzer=1
diff --git a/tests/queries/0_stateless/03165_order_by_duplicate.sql b/tests/queries/0_stateless/03165_order_by_duplicate.sql
index 0054cbc36a6..b8bcc10e119 100644
--- a/tests/queries/0_stateless/03165_order_by_duplicate.sql
+++ b/tests/queries/0_stateless/03165_order_by_duplicate.sql
@@ -13,4 +13,4 @@ WHERE id IN (
 )
 ORDER BY id ASC
 LIMIT 1 BY id
-SETTINGS allow_experimental_analyzer = 1;
+SETTINGS enable_analyzer = 1;
diff --git a/tests/queries/0_stateless/03166_mv_prewhere_duplicating_name_bug.sql b/tests/queries/0_stateless/03166_mv_prewhere_duplicating_name_bug.sql
index e32d23920dd..e27e8645466 100644
--- a/tests/queries/0_stateless/03166_mv_prewhere_duplicating_name_bug.sql
+++ b/tests/queries/0_stateless/03166_mv_prewhere_duplicating_name_bug.sql
@@ -3,5 +3,5 @@ create table dst (s String, lc LowCardinality(String)) engine MergeTree order by
 create materialized view mv to dst (s String, lc String) as select 'a' as s, toLowCardinality('b') as lc from src;
 insert into src values (1);
 
-select s, lc from mv where not ignore(lc) settings allow_experimental_analyzer=0;
-select s, lc from mv where not ignore(lc) settings allow_experimental_analyzer=1;
+select s, lc from mv where not ignore(lc) settings enable_analyzer=0;
+select s, lc from mv where not ignore(lc) settings enable_analyzer=1;
diff --git a/tests/queries/0_stateless/03166_skip_indexes_vertical_merge_1.sql b/tests/queries/0_stateless/03166_skip_indexes_vertical_merge_1.sql
index d3e3b38a3cb..96221f27e73 100644
--- a/tests/queries/0_stateless/03166_skip_indexes_vertical_merge_1.sql
+++ b/tests/queries/0_stateless/03166_skip_indexes_vertical_merge_1.sql
@@ -1,6 +1,6 @@
 DROP TABLE IF EXISTS t_ind_merge_1;
 
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 CREATE TABLE t_ind_merge_1 (a UInt64, b UInt64, c UInt64, d UInt64, INDEX idx_b b TYPE minmax)
 ENGINE = MergeTree
diff --git a/tests/queries/0_stateless/03167_parametrized_view_with_cte.sql b/tests/queries/0_stateless/03167_parametrized_view_with_cte.sql
index 1ac5540047a..ae6ab586415 100644
--- a/tests/queries/0_stateless/03167_parametrized_view_with_cte.sql
+++ b/tests/queries/0_stateless/03167_parametrized_view_with_cte.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 CREATE OR REPLACE VIEW param_test AS SELECT {test_str:String} as s_result;
 WITH 'OK' AS s SELECT * FROM param_test(test_str=s);
 WITH (SELECT 123) AS s SELECT * FROM param_test(test_str=s);
diff --git a/tests/queries/0_stateless/03169_optimize_injective_functions_inside_uniq_crash.sql b/tests/queries/0_stateless/03169_optimize_injective_functions_inside_uniq_crash.sql
index 50d99b851a6..5ab32415f1c 100644
--- a/tests/queries/0_stateless/03169_optimize_injective_functions_inside_uniq_crash.sql
+++ b/tests/queries/0_stateless/03169_optimize_injective_functions_inside_uniq_crash.sql
@@ -7,7 +7,7 @@ FROM
     FROM numbers(4096 * 100)
     GROUP BY k
 )
-SETTINGS allow_experimental_analyzer = 1, optimize_injective_functions_inside_uniq=0;
+SETTINGS enable_analyzer = 1, optimize_injective_functions_inside_uniq=0;
 
 SELECT sum(u)
 FROM
@@ -18,4 +18,4 @@ FROM
     FROM numbers(4096 * 100)
     GROUP BY k
 )
-SETTINGS allow_experimental_analyzer = 1, optimize_injective_functions_inside_uniq=1;
+SETTINGS enable_analyzer = 1, optimize_injective_functions_inside_uniq=1;
diff --git a/tests/queries/0_stateless/03170_part_offset_as_table_column.sql b/tests/queries/0_stateless/03170_part_offset_as_table_column.sql
index 36cbc156744..7711457f23f 100644
--- a/tests/queries/0_stateless/03170_part_offset_as_table_column.sql
+++ b/tests/queries/0_stateless/03170_part_offset_as_table_column.sql
@@ -9,12 +9,12 @@ ORDER BY key;
 INSERT INTO test_table (key) SELECT number
 FROM numbers(10);
 
-set allow_experimental_analyzer=0;
+set enable_analyzer=0;
 
 SELECT *
 FROM test_table;
 
-set allow_experimental_analyzer=1;
+set enable_analyzer=1;
 
 SELECT *
 FROM test_table;
diff --git a/tests/queries/0_stateless/03171_condition_pushdown.sql b/tests/queries/0_stateless/03171_condition_pushdown.sql
index 9cfe41ce921..fcf5db886e9 100644
--- a/tests/queries/0_stateless/03171_condition_pushdown.sql
+++ b/tests/queries/0_stateless/03171_condition_pushdown.sql
@@ -1,5 +1,5 @@
 -- This query succeeds only if it is correctly optimized.
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 SELECT * FROM (SELECT * FROM numbers(1e19)) AS t1, (SELECT * FROM numbers(1e19)) AS t2 WHERE t1.number IN (123, 456) AND t2.number = t1.number ORDER BY ALL;
 
 -- Still TODO:
diff --git a/tests/queries/0_stateless/03171_function_to_subcolumns_fuzzer.sql b/tests/queries/0_stateless/03171_function_to_subcolumns_fuzzer.sql
index f10019a78dd..53476c5bdd1 100644
--- a/tests/queries/0_stateless/03171_function_to_subcolumns_fuzzer.sql
+++ b/tests/queries/0_stateless/03171_function_to_subcolumns_fuzzer.sql
@@ -1,5 +1,5 @@
 SET optimize_functions_to_subcolumns = 1;
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS t_func_to_subcolumns_map_2;
 
diff --git a/tests/queries/0_stateless/03173_forbid_qualify.sql b/tests/queries/0_stateless/03173_forbid_qualify.sql
index d8cb2bad2ea..0a41385c52f 100644
--- a/tests/queries/0_stateless/03173_forbid_qualify.sql
+++ b/tests/queries/0_stateless/03173_forbid_qualify.sql
@@ -4,8 +4,8 @@ create table test_qualify (number Int64) ENGINE = MergeTree ORDER BY (number);
 insert into test_qualify SELECT * FROM numbers(100);
 
 select count() from test_qualify; -- 100
-select * from test_qualify qualify row_number() over (order by number) = 50 SETTINGS allow_experimental_analyzer = 1; -- 49
-select * from test_qualify qualify row_number() over (order by number) = 50 SETTINGS allow_experimental_analyzer = 0; -- { serverError NOT_IMPLEMENTED }
+select * from test_qualify qualify row_number() over (order by number) = 50 SETTINGS enable_analyzer = 1; -- 49
+select * from test_qualify qualify row_number() over (order by number) = 50 SETTINGS enable_analyzer = 0; -- { serverError NOT_IMPLEMENTED }
 
 delete from test_qualify where number in (select number from test_qualify qualify row_number() over (order by number) = 50); -- { serverError UNFINISHED }
 select count() from test_qualify; -- 100
diff --git a/tests/queries/0_stateless/03173_parallel_replicas_join_bug.sh b/tests/queries/0_stateless/03173_parallel_replicas_join_bug.sh
index 20a29e2734e..7c567c0f58f 100755
--- a/tests/queries/0_stateless/03173_parallel_replicas_join_bug.sh
+++ b/tests/queries/0_stateless/03173_parallel_replicas_join_bug.sh
@@ -17,7 +17,7 @@ $CLICKHOUSE_CLIENT -nq "
 "
 
 $CLICKHOUSE_CLIENT -nq "
-SET allow_experimental_analyzer = 1, cluster_for_parallel_replicas = 'parallel_replicas', max_parallel_replicas = 10, allow_experimental_parallel_reading_from_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, max_threads = 1;
+SET enable_analyzer = 1, cluster_for_parallel_replicas = 'parallel_replicas', max_parallel_replicas = 10, allow_experimental_parallel_reading_from_replicas = 2, parallel_replicas_for_non_replicated_merge_tree = 1, max_threads = 1;
 
 SELECT
     id,
diff --git a/tests/queries/0_stateless/03174_merge_join_bug.sql b/tests/queries/0_stateless/03174_merge_join_bug.sql
index ab4cb6cd4a9..ab3c384765d 100644
--- a/tests/queries/0_stateless/03174_merge_join_bug.sql
+++ b/tests/queries/0_stateless/03174_merge_join_bug.sql
@@ -1,6 +1,6 @@
 -- Tags: no-random-settings
 
-SET allow_experimental_analyzer=1, join_algorithm = 'full_sorting_merge';
+SET enable_analyzer=1, join_algorithm = 'full_sorting_merge';
 CREATE TABLE xxxx_yyy (key UInt32, key_b ALIAS key) ENGINE=MergeTree() ORDER BY key SETTINGS ratio_of_defaults_for_sparse_serialization=0.0;
 INSERT INTO xxxx_yyy SELECT number FROM numbers(10);
 
diff --git a/tests/queries/0_stateless/03199_join_with_materialized_column.sql b/tests/queries/0_stateless/03199_join_with_materialized_column.sql
index 8c53c5b3e66..fef171cb9d1 100644
--- a/tests/queries/0_stateless/03199_join_with_materialized_column.sql
+++ b/tests/queries/0_stateless/03199_join_with_materialized_column.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 DROP TABLE IF EXISTS table_with_materialized;
 CREATE TABLE table_with_materialized (col String MATERIALIZED 'A') ENGINE = Memory;
diff --git a/tests/queries/0_stateless/03199_queries_with_new_analyzer.sql b/tests/queries/0_stateless/03199_queries_with_new_analyzer.sql
index c32d7524492..d400a025f76 100644
--- a/tests/queries/0_stateless/03199_queries_with_new_analyzer.sql
+++ b/tests/queries/0_stateless/03199_queries_with_new_analyzer.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer=1;
+SET enable_analyzer=1;
 
 SELECT *, ngramMinHash(*) AS minhash, mortonEncode(untuple(ngramMinHash(*))) AS z
 FROM (SELECT toString(number) FROM numbers(10))
@@ -38,4 +38,3 @@ ORDER BY tuple();
 INSERT INTO seq VALUES (0), (6), (7);
 
 WITH (Select min(number), max(number) from seq) as range Select * from numbers(range.1, range.2);
-
diff --git a/tests/queries/0_stateless/03200_memory_engine_alter_dynamic.sql b/tests/queries/0_stateless/03200_memory_engine_alter_dynamic.sql
index a01a595dbb5..25f3bb0f4c8 100644
--- a/tests/queries/0_stateless/03200_memory_engine_alter_dynamic.sql
+++ b/tests/queries/0_stateless/03200_memory_engine_alter_dynamic.sql
@@ -2,6 +2,5 @@ set allow_experimental_dynamic_type=1;
 create table test (d Dynamic) engine=Memory;
 insert into table test select * from numbers(5);
 alter table test modify column d Dynamic(max_types=1);
-select d.UInt64 from test settings allow_experimental_analyzer=1;
-select d.UInt64 from test settings allow_experimental_analyzer=0;
-
+select d.UInt64 from test settings enable_analyzer=1;
+select d.UInt64 from test settings enable_analyzer=0;
diff --git a/tests/queries/0_stateless/03200_subcolumns_join_use_nulls.sql b/tests/queries/0_stateless/03200_subcolumns_join_use_nulls.sql
index 2dd0a37657d..6777a753490 100644
--- a/tests/queries/0_stateless/03200_subcolumns_join_use_nulls.sql
+++ b/tests/queries/0_stateless/03200_subcolumns_join_use_nulls.sql
@@ -8,6 +8,6 @@ SELECT
     count()
 FROM (SELECT number FROM numbers(10)) as tbl LEFT JOIN t_subcolumns_join ON number = id
 WHERE id is null
-SETTINGS allow_experimental_analyzer = 1, optimize_functions_to_subcolumns = 1, join_use_nulls = 1;
+SETTINGS enable_analyzer = 1, optimize_functions_to_subcolumns = 1, join_use_nulls = 1;
 
 DROP TABLE t_subcolumns_join;
diff --git a/tests/queries/0_stateless/03201_sumIf_to_countIf_return_type.sql b/tests/queries/0_stateless/03201_sumIf_to_countIf_return_type.sql
index 24369fd6497..b791f328da4 100644
--- a/tests/queries/0_stateless/03201_sumIf_to_countIf_return_type.sql
+++ b/tests/queries/0_stateless/03201_sumIf_to_countIf_return_type.sql
@@ -1,2 +1,2 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 EXPLAIN QUERY TREE SELECT tuple(sumIf(toInt64(1), 1)) FROM numbers(100) settings optimize_rewrite_sum_if_to_count_if=1;
diff --git a/tests/queries/1_stateful/00081_group_by_without_key_and_totals.sql b/tests/queries/1_stateful/00081_group_by_without_key_and_totals.sql
index 03e9e0feb40..1d10a2b9212 100644
--- a/tests/queries/1_stateful/00081_group_by_without_key_and_totals.sql
+++ b/tests/queries/1_stateful/00081_group_by_without_key_and_totals.sql
@@ -1,4 +1,4 @@
-SET allow_experimental_analyzer = 1;
+SET enable_analyzer = 1;
 
 SELECT count() AS c FROM test.hits WHERE CounterID = 1704509 WITH TOTALS SETTINGS totals_mode = 'before_having',          max_rows_to_group_by = 100000, group_by_overflow_mode = 'any';
 SELECT count() AS c FROM test.hits WHERE CounterID = 1704509 WITH TOTALS SETTINGS totals_mode = 'after_having_inclusive', max_rows_to_group_by = 100000, group_by_overflow_mode = 'any';
@@ -14,4 +14,3 @@ SELECT TraficSourceID AS k, count() AS c FROM test.hits WHERE CounterID = 170450
 SELECT TraficSourceID AS k, count() AS c FROM test.hits WHERE CounterID = 1704509 GROUP BY k WITH TOTALS ORDER BY k SETTINGS totals_mode = 'after_having_inclusive', max_rows_to_group_by = 100000, group_by_overflow_mode = 'any';
 SELECT TraficSourceID AS k, count() AS c FROM test.hits WHERE CounterID = 1704509 GROUP BY k WITH TOTALS ORDER BY k SETTINGS totals_mode = 'after_having_exclusive', max_rows_to_group_by = 100000, group_by_overflow_mode = 'any';
 SELECT TraficSourceID AS k, count() AS c FROM test.hits WHERE CounterID = 1704509 GROUP BY k WITH TOTALS ORDER BY k SETTINGS totals_mode = 'after_having_auto',      max_rows_to_group_by = 100000, group_by_overflow_mode = 'any';
-
diff --git a/tests/queries/1_stateful/00172_early_constant_folding.sql b/tests/queries/1_stateful/00172_early_constant_folding.sql
index b4b58ba3cb0..343e87a26d1 100644
--- a/tests/queries/1_stateful/00172_early_constant_folding.sql
+++ b/tests/queries/1_stateful/00172_early_constant_folding.sql
@@ -2,5 +2,5 @@
 
 set max_threads=10;
 set optimize_use_implicit_projections=1;
-EXPLAIN PIPELINE SELECT count(JavaEnable) FROM test.hits WHERE WatchID = 1 OR Title = 'next' OR URL = 'prev' OR URL = '???' OR 1 SETTINGS allow_experimental_analyzer = 0;
-EXPLAIN PIPELINE SELECT count(JavaEnable) FROM test.hits WHERE WatchID = 1 OR Title = 'next' OR URL = 'prev' OR URL = '???' OR 1 SETTINGS allow_experimental_analyzer = 1;
+EXPLAIN PIPELINE SELECT count(JavaEnable) FROM test.hits WHERE WatchID = 1 OR Title = 'next' OR URL = 'prev' OR URL = '???' OR 1 SETTINGS enable_analyzer = 0;
+EXPLAIN PIPELINE SELECT count(JavaEnable) FROM test.hits WHERE WatchID = 1 OR Title = 'next' OR URL = 'prev' OR URL = '???' OR 1 SETTINGS enable_analyzer = 1;
diff --git a/tests/queries/1_stateful/00173_group_by_use_nulls.reference b/tests/queries/1_stateful/00173_group_by_use_nulls.reference
index e82b996ad3c..4f8d10a9221 100644
--- a/tests/queries/1_stateful/00173_group_by_use_nulls.reference
+++ b/tests/queries/1_stateful/00173_group_by_use_nulls.reference
@@ -9,7 +9,7 @@
 33010362	1336
 800784	1336
 -- { echoOn }
-set allow_experimental_analyzer = 1;
+set enable_analyzer = 1;
 SELECT
     CounterID AS k,
     quantileBFloat16(0.5)(ResolutionWidth)
diff --git a/tests/queries/1_stateful/00173_group_by_use_nulls.sql b/tests/queries/1_stateful/00173_group_by_use_nulls.sql
index 8531e9efaf8..ed537bb289a 100644
--- a/tests/queries/1_stateful/00173_group_by_use_nulls.sql
+++ b/tests/queries/1_stateful/00173_group_by_use_nulls.sql
@@ -21,7 +21,7 @@ LIMIT 10
 SETTINGS group_by_use_nulls = 1 FORMAT Null;
 
 -- { echoOn }
-set allow_experimental_analyzer = 1;
+set enable_analyzer = 1;
 
 SELECT
     CounterID AS k,

From cc5dd9830e5063d5a6c71cd9c6e406961e78861d Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <mikhaylovnikitka@gmail.com>
Date: Fri, 12 Jul 2024 14:56:47 +0200
Subject: [PATCH 1390/1488] Modify the docs

---
 docs/en/development/architecture.md | 2 +-
 docs/ru/development/architecture.md | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/en/development/architecture.md b/docs/en/development/architecture.md
index c5d13ab63a5..23531f742c5 100644
--- a/docs/en/development/architecture.md
+++ b/docs/en/development/architecture.md
@@ -118,7 +118,7 @@ And the result of interpreting the `INSERT SELECT` query is a "completed" `Query
 
 `InterpreterSelectQuery` uses `ExpressionAnalyzer` and `ExpressionActions` machinery for query analysis and transformations. This is where most rule-based query optimizations are performed. `ExpressionAnalyzer` is quite messy and should be rewritten: various query transformations and optimizations should be extracted into separate classes to allow for modular transformations of the query.
 
-To address current problems that exist in interpreters, a new `InterpreterSelectQueryAnalyzer` is being developed. It is a new version of `InterpreterSelectQuery` that does not use `ExpressionAnalyzer` and introduces an additional abstraction level between `AST` and `QueryPipeline` called `QueryTree`. It is not production-ready yet, but it can be tested with the `enable_analyzer` flag.
+To address problems that exist in interpreters, a new `InterpreterSelectQueryAnalyzer` has been developed. IThis is a new version of the `Interpreter Select Query`, which does not use the `Expression Analyzer` and introduces an additional layer of abstraction between `AST` and `QueryPipeline`, called `QueryTree'. It is fully ready for use in production, but just in case it can be turned off by setting the value of the `enable_analyzer` setting to `false`.
 
 ## Functions {#functions}
 
diff --git a/docs/ru/development/architecture.md b/docs/ru/development/architecture.md
index 0701c8f4a51..0833120c34d 100644
--- a/docs/ru/development/architecture.md
+++ b/docs/ru/development/architecture.md
@@ -115,7 +115,7 @@ ClickHouse — полноценная столбцовая СУБД. Данны
 
 `InterpreterSelectQuery` использует `ExpressionAnalyzer` и `ExpressionActions` механизмы для анализа запросов и преобразований. Именно здесь выполняется большинство оптимизаций запросов на основе правил. `ExpressionAnalyzer` написан довольно грязно и должен быть переписан: различные преобразования запросов и оптимизации должны быть извлечены в отдельные классы, чтобы позволить модульные преобразования или запросы.
 
-Для решения текущих проблем, существующих в интерпретаторах, разрабатывается новый `InterpreterSelectQueryAnalyzer`. Это новая версия `InterpreterSelectQuery`, которая не использует `ExpressionAnalyzer` и вводит дополнительный уровень абстракции между `AST` и `QueryPipeline`, называемый `QueryTree`. Он еще не готов к использованию в продакшене, но его можно протестировать с помощью флага `enable_analyzer`.
+Для решения проблем, существующих в интерпретаторах, был разработан новый `InterpreterSelectQueryAnalyzer`. Это новая версия `InterpreterSelectQuery`, которая не использует `ExpressionAnalyzer` и вводит дополнительный уровень абстракции между `AST` и `QueryPipeline`, называемый `QueryTree`. Он полностью готов к использованию в продакшене, но на всякий случай его можно выключить, установив значение настройки `enable_analyzer` в  `false`.
 
 ## Функции {#functions}
 

From 62d5e83f2540d47487846da19d5297794126705e Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <mikhaylovnikitka@gmail.com>
Date: Fri, 12 Jul 2024 15:01:48 +0200
Subject: [PATCH 1391/1488] Double space...

---
 docs/ru/development/architecture.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/ru/development/architecture.md b/docs/ru/development/architecture.md
index 0833120c34d..d2afbf233b8 100644
--- a/docs/ru/development/architecture.md
+++ b/docs/ru/development/architecture.md
@@ -115,7 +115,7 @@ ClickHouse — полноценная столбцовая СУБД. Данны
 
 `InterpreterSelectQuery` использует `ExpressionAnalyzer` и `ExpressionActions` механизмы для анализа запросов и преобразований. Именно здесь выполняется большинство оптимизаций запросов на основе правил. `ExpressionAnalyzer` написан довольно грязно и должен быть переписан: различные преобразования запросов и оптимизации должны быть извлечены в отдельные классы, чтобы позволить модульные преобразования или запросы.
 
-Для решения проблем, существующих в интерпретаторах, был разработан новый `InterpreterSelectQueryAnalyzer`. Это новая версия `InterpreterSelectQuery`, которая не использует `ExpressionAnalyzer` и вводит дополнительный уровень абстракции между `AST` и `QueryPipeline`, называемый `QueryTree`. Он полностью готов к использованию в продакшене, но на всякий случай его можно выключить, установив значение настройки `enable_analyzer` в  `false`.
+Для решения проблем, существующих в интерпретаторах, был разработан новый `InterpreterSelectQueryAnalyzer`. Это новая версия `InterpreterSelectQuery`, которая не использует `ExpressionAnalyzer` и вводит дополнительный уровень абстракции между `AST` и `QueryPipeline`, называемый `QueryTree`. Он полностью готов к использованию в продакшене, но на всякий случай его можно выключить, установив значение настройки `enable_analyzer` в `false`.
 
 ## Функции {#functions}
 

From aa66203f1732f22f4a0d5ed4c4816e148ffd7861 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Sat, 13 Jul 2024 18:46:24 +0200
Subject: [PATCH 1392/1488] Update architecture.md

---
 docs/en/development/architecture.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/en/development/architecture.md b/docs/en/development/architecture.md
index 23531f742c5..a1a5901f859 100644
--- a/docs/en/development/architecture.md
+++ b/docs/en/development/architecture.md
@@ -118,7 +118,7 @@ And the result of interpreting the `INSERT SELECT` query is a "completed" `Query
 
 `InterpreterSelectQuery` uses `ExpressionAnalyzer` and `ExpressionActions` machinery for query analysis and transformations. This is where most rule-based query optimizations are performed. `ExpressionAnalyzer` is quite messy and should be rewritten: various query transformations and optimizations should be extracted into separate classes to allow for modular transformations of the query.
 
-To address problems that exist in interpreters, a new `InterpreterSelectQueryAnalyzer` has been developed. IThis is a new version of the `Interpreter Select Query`, which does not use the `Expression Analyzer` and introduces an additional layer of abstraction between `AST` and `QueryPipeline`, called `QueryTree'. It is fully ready for use in production, but just in case it can be turned off by setting the value of the `enable_analyzer` setting to `false`.
+To address problems that exist in interpreters, a new `InterpreterSelectQueryAnalyzer` has been developed. This is a new version of the `InterpreterSelectQuery`, which does not use the `ExpressionAnalyzer` and introduces an additional layer of abstraction between `AST` and `QueryPipeline`, called `QueryTree'. It is fully ready for use in production, but just in case it can be turned off by setting the value of the `enable_analyzer` setting to `false`.
 
 ## Functions {#functions}
 

From ee2589df597cd853259a18212ae979d7e6d65150 Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <nikitamikhaylov@clickhouse.com>
Date: Mon, 15 Jul 2024 23:06:03 +0200
Subject: [PATCH 1393/1488] Add a settings to the history changelog

---
 src/Core/SettingsChangesHistory.cpp | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp
index 08fb6dc3301..dc81932f923 100644
--- a/src/Core/SettingsChangesHistory.cpp
+++ b/src/Core/SettingsChangesHistory.cpp
@@ -524,6 +524,10 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
               {"backup_restore_s3_retry_attempts", 1000,1000, "Setting for Aws::Client::RetryStrategy, Aws::Client does retries itself, 0 means no retries. It takes place only for backup/restore."},
               {"postgresql_connection_attempt_timeout", 2, 2, "Allow to control 'connect_timeout' parameter of PostgreSQL connection."},
               {"postgresql_connection_pool_retries", 2, 2, "Allow to control the number of retries in PostgreSQL connection pool."}
+              {"postgresql_connection_pool_retries", 2, 2, "Allow to control the number of retries in PostgreSQL connection pool."},
+              {"restore_replace_external_table_functions_to_null", false, false, "New setting."},
+              {"restore_replace_external_engines_to_null", false, false, "New setting."}
+              {"enable_analyzer", false, true, "Move analyzer to the Beta stage."},
               }},
     {"24.6", {{"materialize_skip_indexes_on_insert", true, true, "Added new setting to allow to disable materialization of skip indexes on insert"},
               {"materialize_statistics_on_insert", true, true, "Added new setting to allow to disable materialization of statistics on insert"},
@@ -603,7 +607,7 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
               {"keeper_retry_initial_backoff_ms", 100, 100, "Initial backoff timeout for general keeper operations"},
               {"keeper_retry_max_backoff_ms", 5000, 5000, "Max backoff timeout for general keeper operations"},
               {"s3queue_allow_experimental_sharded_mode", false, false, "Enable experimental sharded mode of S3Queue table engine. It is experimental because it will be rewritten"},
-              {"enable_analyzer", false, true, "Enable analyzer and planner by default."},
+              {"allow_experimental_analyzer", false, true, "Enable analyzer and planner by default."},
               {"merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability", 0.0, 0.0, "For testing of `PartsSplitter` - split read ranges into intersecting and non intersecting every time you read from MergeTree with the specified probability."},
               {"allow_get_client_http_header", false, false, "Introduced a new function."},
               {"output_format_pretty_row_numbers", false, true, "It is better for usability."},

From 41bfdbe64ee871d595db875471c784b36893ac0b Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <nikitamikhaylov@clickhouse.com>
Date: Tue, 16 Jul 2024 13:37:34 +0000
Subject: [PATCH 1394/1488] Fix tests

---
 .../0_stateless/02995_baseline_24_7_2.tsv     | 26 +++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/tests/queries/0_stateless/02995_baseline_24_7_2.tsv b/tests/queries/0_stateless/02995_baseline_24_7_2.tsv
index 10b392f3e04..d3a07ecb644 100644
--- a/tests/queries/0_stateless/02995_baseline_24_7_2.tsv
+++ b/tests/queries/0_stateless/02995_baseline_24_7_2.tsv
@@ -18,7 +18,11 @@ allow_distributed_ddl	1
 allow_drop_detached	0
 allow_execute_multiif_columnar	1
 allow_experimental_alter_materialized_view_structure	1
+<<<<<<< HEAD:tests/queries/0_stateless/02995_baseline_24_7_2.tsv
 allow_experimental_analyzer	1
+=======
+allow_experimental_analyzer	0
+>>>>>>> Fix tests:tests/queries/0_stateless/02995_baseline_23_12_1.tsv
 allow_experimental_annoy_index	0
 allow_experimental_bigint_types	1
 allow_experimental_codecs	0
@@ -159,7 +163,10 @@ cloud_mode	0
 cloud_mode_engine	1
 cluster_for_parallel_replicas	
 collect_hash_table_stats_during_aggregation	1
+<<<<<<< HEAD:tests/queries/0_stateless/02995_baseline_24_7_2.tsv
 collect_hash_table_stats_during_joins	1
+=======
+>>>>>>> Fix tests:tests/queries/0_stateless/02995_baseline_23_12_1.tsv
 column_names_for_schema_inference	
 compatibility	
 compatibility_ignore_auto_increment_in_create_table	0
@@ -329,9 +336,13 @@ format_regexp_escaping_rule	Raw
 format_regexp_skip_unmatched	0
 format_schema	
 format_template_resultset	
+<<<<<<< HEAD:tests/queries/0_stateless/02995_baseline_24_7_2.tsv
 format_template_resultset_format	
 format_template_row	
 format_template_row_format	
+=======
+format_template_row	
+>>>>>>> Fix tests:tests/queries/0_stateless/02995_baseline_23_12_1.tsv
 format_template_rows_between_delimiter	\n
 format_tsv_null_representation	\\N
 formatdatetime_f_prints_single_zero	0
@@ -388,8 +399,11 @@ iceberg_engine_ignore_schema_evolution	0
 idle_connection_timeout	3600
 ignore_cold_parts_seconds	0
 ignore_data_skipping_indices	
+<<<<<<< HEAD:tests/queries/0_stateless/02995_baseline_24_7_2.tsv
 ignore_drop_queries_probability	0
 ignore_materialized_views_with_dropped_target_table	0
+=======
+>>>>>>> Fix tests:tests/queries/0_stateless/02995_baseline_23_12_1.tsv
 ignore_on_cluster_for_replicated_access_entities_queries	0
 ignore_on_cluster_for_replicated_named_collections_queries	0
 ignore_on_cluster_for_replicated_udf_queries	0
@@ -712,8 +726,13 @@ mutations_execute_subqueries_on_initiator	0
 mutations_max_literal_size_to_replace	16384
 mutations_sync	0
 mysql_datatypes_support_level	
+<<<<<<< HEAD:tests/queries/0_stateless/02995_baseline_24_7_2.tsv
 mysql_map_fixed_string_to_text_in_show_columns	1
 mysql_map_string_to_text_in_show_columns	1
+=======
+mysql_map_fixed_string_to_text_in_show_columns	0
+mysql_map_string_to_text_in_show_columns	0
+>>>>>>> Fix tests:tests/queries/0_stateless/02995_baseline_23_12_1.tsv
 mysql_max_rows_to_insert	65536
 network_compression_method	LZ4
 network_zstd_compression_level	1
@@ -780,9 +799,13 @@ os_thread_priority	0
 output_format_arrow_compression_method	lz4_frame
 output_format_arrow_fixed_string_as_fixed_byte_array	1
 output_format_arrow_low_cardinality_as_dictionary	0
+<<<<<<< HEAD:tests/queries/0_stateless/02995_baseline_24_7_2.tsv
 output_format_arrow_string_as_string	1
 output_format_arrow_use_64_bit_indexes_for_dictionary	0
 output_format_arrow_use_signed_indexes_for_dictionary	1
+=======
+output_format_arrow_string_as_string	0
+>>>>>>> Fix tests:tests/queries/0_stateless/02995_baseline_23_12_1.tsv
 output_format_avro_codec	
 output_format_avro_rows_in_file	1
 output_format_avro_string_column_pattern	
@@ -1045,7 +1068,10 @@ totals_mode	after_having_exclusive
 trace_profile_events	0
 transfer_overflow_mode	throw
 transform_null_in	0
+<<<<<<< HEAD:tests/queries/0_stateless/02995_baseline_24_7_2.tsv
 traverse_shadow_remote_data_paths	0
+=======
+>>>>>>> Fix tests:tests/queries/0_stateless/02995_baseline_23_12_1.tsv
 union_default_mode	
 unknown_packet_in_send_data	0
 update_insert_deduplication_token_in_dependent_materialized_views	0

From a8ca5ad50b8998d9e13d81b66bc89434c1364704 Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <nikitamikhaylov@clickhouse.com>
Date: Mon, 29 Jul 2024 17:02:11 +0000
Subject: [PATCH 1395/1488] Fixed build and made setting an alias

---
 src/Client/HedgedConnections.cpp              |  4 ++--
 src/Client/MultiplexedConnections.cpp         |  4 ++--
 src/Core/Settings.h                           |  4 ++--
 src/Core/SettingsChangesHistory.cpp           |  1 +
 src/Interpreters/ActionsVisitor.cpp           |  2 +-
 .../ClusterProxy/SelectStreamFactory.cpp      |  4 ++--
 .../ClusterProxy/executeQuery.cpp             |  4 ++--
 src/Interpreters/InterpreterCreateQuery.cpp   |  4 ++--
 src/Interpreters/InterpreterDescribeQuery.cpp |  2 +-
 src/Interpreters/InterpreterExplainQuery.cpp  | 10 +++++-----
 src/Interpreters/InterpreterFactory.cpp       |  6 +++---
 src/Interpreters/InterpreterInsertQuery.cpp   |  2 +-
 src/Interpreters/MutationsInterpreter.cpp     |  6 +++---
 src/Interpreters/executeQuery.cpp             |  6 +++---
 .../getHeaderForProcessingStage.cpp           |  2 +-
 .../QueryPlan/DistributedCreateLocalPlan.cpp  |  2 +-
 .../Transforms/buildPushingToViewsChain.cpp   |  4 ++--
 src/Server/TCPHandler.cpp                     |  6 +++---
 src/Storages/AlterCommands.cpp                |  2 +-
 src/Storages/IStorageCluster.cpp              |  2 +-
 src/Storages/LiveView/StorageLiveView.cpp     | 10 +++++-----
 src/Storages/MergeTree/MergeTreeData.cpp      |  2 +-
 src/Storages/MergeTree/RPNBuilder.cpp         | 18 ++++++++---------
 src/Storages/StorageBuffer.cpp                |  2 +-
 src/Storages/StorageDistributed.cpp           |  6 +++---
 src/Storages/StorageExecutable.cpp            |  2 +-
 src/Storages/StorageMerge.cpp                 | 20 +++++++++----------
 src/Storages/StorageMergeTree.cpp             |  6 +++---
 src/Storages/StorageReplicatedMergeTree.cpp   |  6 +++---
 src/Storages/StorageView.cpp                  |  2 +-
 src/Storages/TTLDescription.cpp               |  2 +-
 src/Storages/WindowView/StorageWindowView.cpp |  6 +++---
 src/TableFunctions/TableFunctionView.cpp      |  2 +-
 .../TableFunctionViewIfPermitted.cpp          |  2 +-
 34 files changed, 82 insertions(+), 81 deletions(-)

diff --git a/src/Client/HedgedConnections.cpp b/src/Client/HedgedConnections.cpp
index 1c7f222aa78..dd8348ea04f 100644
--- a/src/Client/HedgedConnections.cpp
+++ b/src/Client/HedgedConnections.cpp
@@ -196,11 +196,11 @@ void HedgedConnections::sendQuery(
             modified_settings.parallel_replica_offset = fd_to_replica_location[replica.packet_receiver->getFileDescriptor()].offset;
         }
 
-        /// FIXME: Remove once we will make `enable_analyzer` obsolete setting.
+        /// FIXME: Remove once we will make `allow_experimental_analyzer` obsolete setting.
         /// Make the analyzer being set, so it will be effectively applied on the remote server.
         /// In other words, the initiator always controls whether the analyzer enabled or not for
         /// all servers involved in the distributed query processing.
-        modified_settings.set("enable_analyzer", static_cast<bool>(modified_settings.enable_analyzer));
+        modified_settings.set("allow_experimental_analyzer", static_cast<bool>(modified_settings.allow_experimental_analyzer));
 
         replica.connection->sendQuery(
             timeouts, query, /* query_parameters */ {}, query_id, stage, &modified_settings, &client_info, with_pending_data, {});
diff --git a/src/Client/MultiplexedConnections.cpp b/src/Client/MultiplexedConnections.cpp
index 7ca22ae4c81..244eccf1ed9 100644
--- a/src/Client/MultiplexedConnections.cpp
+++ b/src/Client/MultiplexedConnections.cpp
@@ -150,11 +150,11 @@ void MultiplexedConnections::sendQuery(
         client_info.number_of_current_replica = replica_info->number_of_current_replica;
     }
 
-    /// FIXME: Remove once we will make `enable_analyzer` obsolete setting.
+    /// FIXME: Remove once we will make `allow_experimental_analyzer` obsolete setting.
     /// Make the analyzer being set, so it will be effectively applied on the remote server.
     /// In other words, the initiator always controls whether the analyzer enabled or not for
     /// all servers involved in the distributed query processing.
-    modified_settings.set("enable_analyzer", static_cast<bool>(modified_settings.enable_analyzer));
+    modified_settings.set("allow_experimental_analyzer", static_cast<bool>(modified_settings.allow_experimental_analyzer));
 
     const bool enable_offset_parallel_processing = context->canUseOffsetParallelReplicas();
 
diff --git a/src/Core/Settings.h b/src/Core/Settings.h
index d6c0dc223b2..ac24c087946 100644
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@@ -638,7 +638,7 @@ class IColumn;
     M(Bool, allow_non_metadata_alters, true, "Allow to execute alters which affects not only tables metadata, but also data on disk", 0) \
     M(Bool, enable_global_with_statement, true, "Propagate WITH statements to UNION queries and all subqueries", 0) \
     M(Bool, aggregate_functions_null_for_empty, false, "Rewrite all aggregate functions in a query, adding -OrNull suffix to them", 0) \
-    M(Bool, optimize_syntax_fuse_functions, false, "Allow apply fuse aggregating function. Available only with `enable_analyzer`", 0) \
+    M(Bool, optimize_syntax_fuse_functions, false, "Allow apply fuse aggregating function. Available only with `allow_experimental_analyzer`", 0) \
     M(Bool, flatten_nested, true, "If true, columns of type Nested will be flatten to separate array columns instead of one array of tuples", 0) \
     M(Bool, asterisk_include_materialized_columns, false, "Include MATERIALIZED columns for wildcard query", 0) \
     M(Bool, asterisk_include_alias_columns, false, "Include ALIAS columns for wildcard query", 0) \
@@ -943,7 +943,7 @@ class IColumn;
     \
     M(Bool, allow_experimental_join_condition, false, "Support join with inequal conditions which involve columns from both left and right table. e.g. t1.y < t2.y.", 0) \
     \
-    M(Bool, enable_analyzer, true, "Allow new query analyzer.", IMPORTANT) ALIAS(allow_experimental_analyzer) \
+    M(Bool, allow_experimental_analyzer, true, "Allow new query analyzer.", IMPORTANT) ALIAS(enable_analyzer) \
     M(Bool, analyzer_compatibility_join_using_top_level_identifier, false, "Force to resolve identifier in JOIN USING from projection (for example, in `SELECT a + 1 AS b FROM t1 JOIN t2 USING (b)` join will be performed by `t1.a + 1 = t2.b`, rather then `t1.b = t2.b`).", 0) \
     \
     M(Bool, allow_experimental_live_view, false, "Enable LIVE VIEW. Not mature enough.", 0) \
diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp
index dc81932f923..71f7c940e2c 100644
--- a/src/Core/SettingsChangesHistory.cpp
+++ b/src/Core/SettingsChangesHistory.cpp
@@ -528,6 +528,7 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
               {"restore_replace_external_table_functions_to_null", false, false, "New setting."},
               {"restore_replace_external_engines_to_null", false, false, "New setting."}
               {"enable_analyzer", false, true, "Move analyzer to the Beta stage."},
+              {"allow_experimental_analyzer", false, true, "Move analyzer to the Beta stage."},
               }},
     {"24.6", {{"materialize_skip_indexes_on_insert", true, true, "Added new setting to allow to disable materialization of skip indexes on insert"},
               {"materialize_statistics_on_insert", true, true, "Added new setting to allow to disable materialization of statistics on insert"},
diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp
index b8d70e5d5dd..e1b7e92ee5d 100644
--- a/src/Interpreters/ActionsVisitor.cpp
+++ b/src/Interpreters/ActionsVisitor.cpp
@@ -1411,7 +1411,7 @@ FutureSetPtr ActionsMatcher::makeSet(const ASTFunction & node, Data & data, bool
             return {};
 
         PreparedSets::Hash set_key;
-        if (data.getContext()->getSettingsRef().enable_analyzer && !identifier)
+        if (data.getContext()->getSettingsRef().allow_experimental_analyzer && !identifier)
         {
             /// Here we can be only from mutation interpreter. Normal selects with analyzed use other interpreter.
             /// This is a hacky way to allow reusing cache for prepared sets.
diff --git a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp
index 0948f24eca0..e35d31d2350 100644
--- a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp
+++ b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp
@@ -68,7 +68,7 @@ ASTPtr rewriteSelectQuery(
     // are written into the query context and will be sent by the query pipeline.
     select_query.setExpression(ASTSelectQuery::Expression::SETTINGS, {});
 
-    if (!context->getSettingsRef().enable_analyzer)
+    if (!context->getSettingsRef().allow_experimental_analyzer)
     {
         if (table_function_ptr)
             select_query.addTableFunction(table_function_ptr);
@@ -165,7 +165,7 @@ void SelectStreamFactory::createForShardImpl(
     auto emplace_remote_stream = [&](bool lazy = false, time_t local_delay = 0)
     {
         Block shard_header;
-        if (context->getSettingsRef().enable_analyzer)
+        if (context->getSettingsRef().allow_experimental_analyzer)
             shard_header = InterpreterSelectQueryAnalyzer::getSampleBlock(query_tree, context, SelectQueryOptions(processed_stage).analyze());
         else
             shard_header = header;
diff --git a/src/Interpreters/ClusterProxy/executeQuery.cpp b/src/Interpreters/ClusterProxy/executeQuery.cpp
index 6c8ab11bfc9..d04a73e384e 100644
--- a/src/Interpreters/ClusterProxy/executeQuery.cpp
+++ b/src/Interpreters/ClusterProxy/executeQuery.cpp
@@ -300,7 +300,7 @@ void executeQuery(
 
     const size_t shards = cluster->getShardCount();
 
-    if (context->getSettingsRef().enable_analyzer)
+    if (context->getSettingsRef().allow_experimental_analyzer)
     {
         for (size_t i = 0, s = cluster->getShardsInfo().size(); i < s; ++i)
         {
@@ -581,7 +581,7 @@ void executeQueryWithParallelReplicasCustomKey(
     /// Return directly (with correct header) if no shard to query.
     if (query_info.getCluster()->getShardsInfo().empty())
     {
-        if (context->getSettingsRef().enable_analyzer)
+        if (context->getSettingsRef().allow_experimental_analyzer)
             return;
 
         Pipe pipe(std::make_shared<NullSource>(header));
diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp
index ea631ef01d5..971f90bd3cd 100644
--- a/src/Interpreters/InterpreterCreateQuery.cpp
+++ b/src/Interpreters/InterpreterCreateQuery.cpp
@@ -834,7 +834,7 @@ InterpreterCreateQuery::TableProperties InterpreterCreateQuery::getTableProperti
 
         Block as_select_sample;
 
-        if (getContext()->getSettingsRef().enable_analyzer)
+        if (getContext()->getSettingsRef().allow_experimental_analyzer)
         {
             as_select_sample = InterpreterSelectQueryAnalyzer::getSampleBlock(create.select->clone(), getContext());
         }
@@ -1327,7 +1327,7 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create)
         {
             Block input_block;
 
-            if (getContext()->getSettingsRef().enable_analyzer)
+            if (getContext()->getSettingsRef().allow_experimental_analyzer)
             {
                 input_block = InterpreterSelectQueryAnalyzer::getSampleBlock(create.select->clone(), getContext());
             }
diff --git a/src/Interpreters/InterpreterDescribeQuery.cpp b/src/Interpreters/InterpreterDescribeQuery.cpp
index 4a061f02c2b..39fc85a5e23 100644
--- a/src/Interpreters/InterpreterDescribeQuery.cpp
+++ b/src/Interpreters/InterpreterDescribeQuery.cpp
@@ -129,7 +129,7 @@ void InterpreterDescribeQuery::fillColumnsFromSubquery(const ASTTableExpression
     auto select_query = table_expression.subquery->children.at(0);
     auto current_context = getContext();
 
-    if (settings.enable_analyzer)
+    if (settings.allow_experimental_analyzer)
     {
         SelectQueryOptions select_query_options;
         sample_block = InterpreterSelectQueryAnalyzer(select_query, current_context, select_query_options).getSampleBlock();
diff --git a/src/Interpreters/InterpreterExplainQuery.cpp b/src/Interpreters/InterpreterExplainQuery.cpp
index 2fbfbf3a809..bedd9cb4a80 100644
--- a/src/Interpreters/InterpreterExplainQuery.cpp
+++ b/src/Interpreters/InterpreterExplainQuery.cpp
@@ -394,9 +394,9 @@ QueryPipeline InterpreterExplainQuery::executeImpl()
         }
         case ASTExplainQuery::QueryTree:
         {
-            if (!getContext()->getSettingsRef().enable_analyzer)
+            if (!getContext()->getSettingsRef().allow_experimental_analyzer)
                 throw Exception(ErrorCodes::NOT_IMPLEMENTED,
-                    "EXPLAIN QUERY TREE is only supported with a new analyzer. Set enable_analyzer = 1.");
+                    "EXPLAIN QUERY TREE is only supported with a new analyzer. Set allow_experimental_analyzer = 1.");
 
             if (ast.getExplainedQuery()->as<ASTSelectWithUnionQuery>() == nullptr)
                 throw Exception(ErrorCodes::INCORRECT_QUERY, "Only SELECT is supported for EXPLAIN QUERY TREE query");
@@ -453,7 +453,7 @@ QueryPipeline InterpreterExplainQuery::executeImpl()
 
             ContextPtr context;
 
-            if (getContext()->getSettingsRef().enable_analyzer)
+            if (getContext()->getSettingsRef().allow_experimental_analyzer)
             {
                 InterpreterSelectQueryAnalyzer interpreter(ast.getExplainedQuery(), getContext(), options);
                 context = interpreter.getContext();
@@ -499,7 +499,7 @@ QueryPipeline InterpreterExplainQuery::executeImpl()
                 QueryPlan plan;
                 ContextPtr context;
 
-                if (getContext()->getSettingsRef().enable_analyzer)
+                if (getContext()->getSettingsRef().allow_experimental_analyzer)
                 {
                     InterpreterSelectQueryAnalyzer interpreter(ast.getExplainedQuery(), getContext(), options);
                     context = interpreter.getContext();
@@ -558,7 +558,7 @@ QueryPipeline InterpreterExplainQuery::executeImpl()
             QueryPlan plan;
             ContextPtr context = getContext();
 
-            if (context->getSettingsRef().enable_analyzer)
+            if (context->getSettingsRef().allow_experimental_analyzer)
             {
                 InterpreterSelectQueryAnalyzer interpreter(ast.getExplainedQuery(), getContext(), SelectQueryOptions());
                 context = interpreter.getContext();
diff --git a/src/Interpreters/InterpreterFactory.cpp b/src/Interpreters/InterpreterFactory.cpp
index a909c4e602d..12b3b510098 100644
--- a/src/Interpreters/InterpreterFactory.cpp
+++ b/src/Interpreters/InterpreterFactory.cpp
@@ -118,7 +118,7 @@ InterpreterFactory::InterpreterPtr InterpreterFactory::get(ASTPtr & query, Conte
 
     if (query->as<ASTSelectQuery>())
     {
-        if (context->getSettingsRef().enable_analyzer)
+        if (context->getSettingsRef().allow_experimental_analyzer)
             interpreter_name = "InterpreterSelectQueryAnalyzer";
         /// This is internal part of ASTSelectWithUnionQuery.
         /// Even if there is SELECT without union, it is represented by ASTSelectWithUnionQuery with single ASTSelectQuery as a child.
@@ -129,7 +129,7 @@ InterpreterFactory::InterpreterPtr InterpreterFactory::get(ASTPtr & query, Conte
     {
         ProfileEvents::increment(ProfileEvents::SelectQuery);
 
-        if (context->getSettingsRef().enable_analyzer)
+        if (context->getSettingsRef().allow_experimental_analyzer)
             interpreter_name = "InterpreterSelectQueryAnalyzer";
         else
             interpreter_name = "InterpreterSelectWithUnionQuery";
@@ -222,7 +222,7 @@ InterpreterFactory::InterpreterPtr InterpreterFactory::get(ASTPtr & query, Conte
     {
         const auto kind = query->as<ASTExplainQuery>()->getKind();
         if (kind == ASTExplainQuery::ParsedAST || kind == ASTExplainQuery::AnalyzedSyntax)
-            context->setSetting("enable_analyzer", false);
+            context->setSetting("allow_experimental_analyzer", false);
 
         interpreter_name = "InterpreterExplainQuery";
     }
diff --git a/src/Interpreters/InterpreterInsertQuery.cpp b/src/Interpreters/InterpreterInsertQuery.cpp
index 0213e2a2c42..c97593a1781 100644
--- a/src/Interpreters/InterpreterInsertQuery.cpp
+++ b/src/Interpreters/InterpreterInsertQuery.cpp
@@ -95,7 +95,7 @@ StoragePtr InterpreterInsertQuery::getTable(ASTInsertQuery & query)
             Block header_block;
             auto select_query_options = SelectQueryOptions(QueryProcessingStage::Complete, 1);
 
-            if (current_context->getSettingsRef().enable_analyzer)
+            if (current_context->getSettingsRef().allow_experimental_analyzer)
             {
                 InterpreterSelectQueryAnalyzer interpreter_select(query.select, current_context, select_query_options);
                 header_block = interpreter_select.getSampleBlock();
diff --git a/src/Interpreters/MutationsInterpreter.cpp b/src/Interpreters/MutationsInterpreter.cpp
index c049dbc9cc1..57ad5caa4c7 100644
--- a/src/Interpreters/MutationsInterpreter.cpp
+++ b/src/Interpreters/MutationsInterpreter.cpp
@@ -189,7 +189,7 @@ bool isStorageTouchedByMutations(
     std::optional<InterpreterSelectQuery> interpreter_select_query;
     BlockIO io;
 
-    if (context->getSettingsRef().enable_analyzer)
+    if (context->getSettingsRef().allow_experimental_analyzer)
     {
         auto select_query_tree = prepareQueryAffectedQueryTree(commands, storage.shared_from_this(), context);
         InterpreterSelectQueryAnalyzer interpreter(select_query_tree, context, SelectQueryOptions().ignoreLimits());
@@ -415,9 +415,9 @@ MutationsInterpreter::MutationsInterpreter(
     , logger(getLogger("MutationsInterpreter(" + source.getStorage()->getStorageID().getFullTableName() + ")"))
 {
     auto new_context = Context::createCopy(context_);
-    if (new_context->getSettingsRef().enable_analyzer)
+    if (new_context->getSettingsRef().allow_experimental_analyzer)
     {
-        new_context->setSetting("enable_analyzer", false);
+        new_context->setSetting("allow_experimental_analyzer", false);
         LOG_DEBUG(logger, "Will use old analyzer to prepare mutation");
     }
     context = std::move(new_context);
diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp
index 7476915ab8a..ce58f7f922c 100644
--- a/src/Interpreters/executeQuery.cpp
+++ b/src/Interpreters/executeQuery.cpp
@@ -685,10 +685,10 @@ void validateAnalyzerSettings(ASTPtr ast, bool context_value)
 
         if (auto * set_query = node->as<ASTSetQuery>())
         {
-            if (auto * value = set_query->changes.tryGet("enable_analyzer"))
+            if (auto * value = set_query->changes.tryGet("allow_experimental_analyzer"))
             {
                 if (top_level != value->safeGet<bool>())
-                    throw Exception(ErrorCodes::INCORRECT_QUERY, "Setting 'enable_analyzer' is changed in the subquery. Top level value: {}", top_level);
+                    throw Exception(ErrorCodes::INCORRECT_QUERY, "Setting 'allow_experimental_analyzer' is changed in the subquery. Top level value: {}", top_level);
             }
         }
 
@@ -912,7 +912,7 @@ static std::tuple<ASTPtr, BlockIO> executeQueryImpl(
         /// Interpret SETTINGS clauses as early as possible (before invoking the corresponding interpreter),
         /// to allow settings to take effect.
         InterpreterSetQuery::applySettingsFromQuery(ast, context);
-        validateAnalyzerSettings(ast, context->getSettingsRef().enable_analyzer);
+        validateAnalyzerSettings(ast, context->getSettingsRef().allow_experimental_analyzer);
 
         if (auto * insert_query = ast->as<ASTInsertQuery>())
             insert_query->tail = istr;
diff --git a/src/Interpreters/getHeaderForProcessingStage.cpp b/src/Interpreters/getHeaderForProcessingStage.cpp
index c4a791e85e1..cf18cbbb54a 100644
--- a/src/Interpreters/getHeaderForProcessingStage.cpp
+++ b/src/Interpreters/getHeaderForProcessingStage.cpp
@@ -141,7 +141,7 @@ Block getHeaderForProcessingStage(
 
             Block result;
 
-            if (context->getSettingsRef().enable_analyzer)
+            if (context->getSettingsRef().allow_experimental_analyzer)
             {
                 auto storage = std::make_shared<StorageDummy>(storage_snapshot->storage.getStorageID(),
                                                                                         storage_snapshot->getAllColumnsDescription(),
diff --git a/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp b/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp
index dc4b7fd733b..d8624a1c99b 100644
--- a/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp
+++ b/src/Processors/QueryPlan/DistributedCreateLocalPlan.cpp
@@ -65,7 +65,7 @@ std::unique_ptr<QueryPlan> createLocalPlan(
         .setShardInfo(static_cast<UInt32>(shard_num), static_cast<UInt32>(shard_count))
         .ignoreASTOptimizations();
 
-    if (context->getSettingsRef().enable_analyzer)
+    if (context->getSettingsRef().allow_experimental_analyzer)
     {
         /// For Analyzer, identifier in GROUP BY/ORDER BY/LIMIT BY lists has been resolved to
         /// ConstantNode in QueryTree if it is an alias of a constant, so we should not replace
diff --git a/src/Processors/Transforms/buildPushingToViewsChain.cpp b/src/Processors/Transforms/buildPushingToViewsChain.cpp
index a2d5ec5d1cb..98d66ed77c3 100644
--- a/src/Processors/Transforms/buildPushingToViewsChain.cpp
+++ b/src/Processors/Transforms/buildPushingToViewsChain.cpp
@@ -319,7 +319,7 @@ std::optional<Chain> generateViewChain(
         Block header;
 
         /// Get list of columns we get from select query.
-        if (select_context->getSettingsRef().enable_analyzer)
+        if (select_context->getSettingsRef().allow_experimental_analyzer)
             header = InterpreterSelectQueryAnalyzer::getSampleBlock(query, select_context);
         else
             header = InterpreterSelectQuery(query, select_context, SelectQueryOptions()).getSampleBlock();
@@ -613,7 +613,7 @@ static QueryPipeline process(Block block, ViewRuntimeData & view, const ViewsDat
 
     QueryPipelineBuilder pipeline;
 
-    if (local_context->getSettingsRef().enable_analyzer)
+    if (local_context->getSettingsRef().allow_experimental_analyzer)
     {
         InterpreterSelectQueryAnalyzer interpreter(view.query, local_context, local_context->getViewSource(), SelectQueryOptions().ignoreAccessCheck());
         pipeline = interpreter.buildQueryPipeline();
diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp
index 4262716b406..c5dfe3e6e5f 100644
--- a/src/Server/TCPHandler.cpp
+++ b/src/Server/TCPHandler.cpp
@@ -1904,14 +1904,14 @@ void TCPHandler::receiveQuery()
     /// Settings
     ///
 
-    /// FIXME: Remove when enable_analyzer will become obsolete.
+    /// FIXME: Remove when allow_experimental_analyzer will become obsolete.
     /// Analyzer became Beta in 24.3 and started to be enabled by default.
     /// We have to disable it for ourselves to make sure we don't have different settings on
     /// different servers.
     if (query_kind == ClientInfo::QueryKind::SECONDARY_QUERY
         && client_info.getVersionNumber() < VersionNumber(23, 3, 0)
-        && !passed_settings.enable_analyzer.changed)
-        passed_settings.set("enable_analyzer", false);
+        && !passed_settings.allow_experimental_analyzer.changed)
+        passed_settings.set("allow_experimental_analyzer", false);
 
     auto settings_changes = passed_settings.changes();
     query_kind = query_context->getClientInfo().query_kind;
diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp
index 2843ff5a14e..7891042bb96 100644
--- a/src/Storages/AlterCommands.cpp
+++ b/src/Storages/AlterCommands.cpp
@@ -806,7 +806,7 @@ void AlterCommand::apply(StorageInMemoryMetadata & metadata, ContextPtr context)
         metadata.select = SelectQueryDescription::getSelectQueryFromASTForMatView(select, metadata.refresh != nullptr, context);
         Block as_select_sample;
 
-        if (context->getSettingsRef().enable_analyzer)
+        if (context->getSettingsRef().allow_experimental_analyzer)
         {
             as_select_sample = InterpreterSelectQueryAnalyzer::getSampleBlock(select->clone(), context);
         }
diff --git a/src/Storages/IStorageCluster.cpp b/src/Storages/IStorageCluster.cpp
index b485ab9cbb5..63467603d16 100644
--- a/src/Storages/IStorageCluster.cpp
+++ b/src/Storages/IStorageCluster.cpp
@@ -125,7 +125,7 @@ void IStorageCluster::read(
     Block sample_block;
     ASTPtr query_to_send = query_info.query;
 
-    if (context->getSettingsRef().enable_analyzer)
+    if (context->getSettingsRef().allow_experimental_analyzer)
     {
         sample_block = InterpreterSelectQueryAnalyzer::getSampleBlock(query_info.query, context, SelectQueryOptions(processed_stage));
     }
diff --git a/src/Storages/LiveView/StorageLiveView.cpp b/src/Storages/LiveView/StorageLiveView.cpp
index c93da7ca512..71b1a0a73c9 100644
--- a/src/Storages/LiveView/StorageLiveView.cpp
+++ b/src/Storages/LiveView/StorageLiveView.cpp
@@ -378,7 +378,7 @@ void StorageLiveView::writeBlock(StorageLiveView & live_view, Block && block, Ch
 
         QueryPipelineBuilder builder;
 
-        if (local_context->getSettingsRef().enable_analyzer)
+        if (local_context->getSettingsRef().allow_experimental_analyzer)
         {
             auto select_description = buildSelectQueryTreeDescription(select_query_description.inner_query, local_context);
             if (select_description.dependent_table_node)
@@ -475,7 +475,7 @@ Block StorageLiveView::getHeader() const
 
     if (!sample_block)
     {
-        if (live_view_context->getSettingsRef().enable_analyzer)
+        if (live_view_context->getSettingsRef().allow_experimental_analyzer)
         {
             sample_block = InterpreterSelectQueryAnalyzer::getSampleBlock(select_query_description.select_query,
                 live_view_context,
@@ -519,7 +519,7 @@ ASTPtr StorageLiveView::getInnerBlocksQuery()
         auto & select_with_union_query = select_query_description.select_query->as<ASTSelectWithUnionQuery &>();
         auto blocks_query = select_with_union_query.list_of_selects->children.at(0)->clone();
 
-        if (!live_view_context->getSettingsRef().enable_analyzer)
+        if (!live_view_context->getSettingsRef().allow_experimental_analyzer)
         {
             /// Rewrite inner query with right aliases for JOIN.
             /// It cannot be done in constructor or startup() because InterpreterSelectQuery may access table,
@@ -543,7 +543,7 @@ MergeableBlocksPtr StorageLiveView::collectMergeableBlocks(ContextPtr local_cont
 
     QueryPipelineBuilder builder;
 
-    if (local_context->getSettingsRef().enable_analyzer)
+    if (local_context->getSettingsRef().allow_experimental_analyzer)
     {
         InterpreterSelectQueryAnalyzer interpreter(select_query_description.inner_query,
             local_context,
@@ -599,7 +599,7 @@ QueryPipelineBuilder StorageLiveView::completeQuery(Pipes pipes)
 
     QueryPipelineBuilder builder;
 
-    if (block_context->getSettingsRef().enable_analyzer)
+    if (block_context->getSettingsRef().allow_experimental_analyzer)
     {
         auto select_description = buildSelectQueryTreeDescription(select_query_description.select_query, block_context);
 
diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index 01ef0a409b0..ce27ad24e10 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -7097,7 +7097,7 @@ QueryProcessingStage::Enum MergeTreeData::getQueryProcessingStage(
     SelectQueryInfo &) const
 {
     /// with new analyzer, Planner make decision regarding parallel replicas usage, and so about processing stage on reading
-    if (!query_context->getSettingsRef().enable_analyzer)
+    if (!query_context->getSettingsRef().allow_experimental_analyzer)
     {
         const auto & settings = query_context->getSettingsRef();
         if (query_context->canUseParallelReplicasCustomKey())
diff --git a/src/Storages/MergeTree/RPNBuilder.cpp b/src/Storages/MergeTree/RPNBuilder.cpp
index fccb20c2b0a..6e963066f39 100644
--- a/src/Storages/MergeTree/RPNBuilder.cpp
+++ b/src/Storages/MergeTree/RPNBuilder.cpp
@@ -33,7 +33,7 @@ namespace ErrorCodes
 namespace
 {
 
-void appendColumnNameWithoutAlias(const ActionsDAG::Node & node, WriteBuffer & out, bool enable_analyzer, bool legacy = false)
+void appendColumnNameWithoutAlias(const ActionsDAG::Node & node, WriteBuffer & out, bool allow_experimental_analyzer, bool legacy = false)
 {
     switch (node.type)
     {
@@ -45,18 +45,18 @@ void appendColumnNameWithoutAlias(const ActionsDAG::Node & node, WriteBuffer & o
             /// If it was created from ASTLiteral, then result_name can be an alias.
             /// We need to convert value back to string here.
             const auto * column_const = typeid_cast<const ColumnConst *>(node.column.get());
-            if (column_const && !enable_analyzer)
+            if (column_const && !allow_experimental_analyzer)
                 writeString(applyVisitor(FieldVisitorToString(), column_const->getField()), out);
             else
                 writeString(node.result_name, out);
             break;
         }
         case ActionsDAG::ActionType::ALIAS:
-            appendColumnNameWithoutAlias(*node.children.front(), out, enable_analyzer, legacy);
+            appendColumnNameWithoutAlias(*node.children.front(), out, allow_experimental_analyzer, legacy);
             break;
         case ActionsDAG::ActionType::ARRAY_JOIN:
             writeCString("arrayJoin(", out);
-            appendColumnNameWithoutAlias(*node.children.front(), out, enable_analyzer, legacy);
+            appendColumnNameWithoutAlias(*node.children.front(), out, allow_experimental_analyzer, legacy);
             writeChar(')', out);
             break;
         case ActionsDAG::ActionType::FUNCTION:
@@ -75,17 +75,17 @@ void appendColumnNameWithoutAlias(const ActionsDAG::Node & node, WriteBuffer & o
                     writeCString(", ", out);
                 first = false;
 
-                appendColumnNameWithoutAlias(*arg, out, enable_analyzer, legacy);
+                appendColumnNameWithoutAlias(*arg, out, allow_experimental_analyzer, legacy);
             }
             writeChar(')', out);
         }
     }
 }
 
-String getColumnNameWithoutAlias(const ActionsDAG::Node & node, bool enable_analyzer, bool legacy = false)
+String getColumnNameWithoutAlias(const ActionsDAG::Node & node, bool allow_experimental_analyzer, bool legacy = false)
 {
     WriteBufferFromOwnString out;
-    appendColumnNameWithoutAlias(node, out, enable_analyzer, legacy);
+    appendColumnNameWithoutAlias(node, out, allow_experimental_analyzer, legacy);
 
     return std::move(out.str());
 }
@@ -131,7 +131,7 @@ std::string RPNBuilderTreeNode::getColumnName() const
     if (ast_node)
         return ast_node->getColumnNameWithoutAlias();
     else
-        return getColumnNameWithoutAlias(*dag_node, getTreeContext().getSettings().enable_analyzer);
+        return getColumnNameWithoutAlias(*dag_node, getTreeContext().getSettings().allow_experimental_analyzer);
 }
 
 std::string RPNBuilderTreeNode::getColumnNameWithModuloLegacy() const
@@ -144,7 +144,7 @@ std::string RPNBuilderTreeNode::getColumnNameWithModuloLegacy() const
     }
     else
     {
-        return getColumnNameWithoutAlias(*dag_node, getTreeContext().getSettings().enable_analyzer, true /*legacy*/);
+        return getColumnNameWithoutAlias(*dag_node, getTreeContext().getSettings().allow_experimental_analyzer, true /*legacy*/);
     }
 }
 
diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp
index da427ca4a6a..04e6d6676d1 100644
--- a/src/Storages/StorageBuffer.cpp
+++ b/src/Storages/StorageBuffer.cpp
@@ -397,7 +397,7 @@ void StorageBuffer::read(
     /// TODO: Find a way to support projections for StorageBuffer
     if (processed_stage > QueryProcessingStage::FetchColumns)
     {
-        if (local_context->getSettingsRef().enable_analyzer)
+        if (local_context->getSettingsRef().allow_experimental_analyzer)
         {
             auto storage = std::make_shared<StorageValues>(
                     getStorageID(),
diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp
index f1fe70b4594..3e38ddf830a 100644
--- a/src/Storages/StorageDistributed.cpp
+++ b/src/Storages/StorageDistributed.cpp
@@ -833,7 +833,7 @@ void StorageDistributed::read(
 
     const auto & settings = local_context->getSettingsRef();
 
-    if (settings.enable_analyzer)
+    if (settings.allow_experimental_analyzer)
     {
         StorageID remote_storage_id = StorageID::createEmpty();
         if (!remote_table_function_ptr)
@@ -1057,7 +1057,7 @@ static std::optional<ActionsDAG> getFilterFromQuery(const ASTPtr & ast, ContextP
     QueryPlan plan;
     SelectQueryOptions options;
     options.only_analyze = true;
-    if (context->getSettingsRef().enable_analyzer)
+    if (context->getSettingsRef().allow_experimental_analyzer)
     {
         InterpreterSelectQueryAnalyzer interpreter(ast, context, options);
         plan = std::move(interpreter).extractQueryPlan();
@@ -1611,7 +1611,7 @@ ClusterPtr StorageDistributed::skipUnusedShards(
     const StorageSnapshotPtr & storage_snapshot,
     ContextPtr local_context) const
 {
-    if (local_context->getSettingsRef().enable_analyzer)
+    if (local_context->getSettingsRef().allow_experimental_analyzer)
         return skipUnusedShardsWithAnalyzer(cluster, query_info, storage_snapshot, local_context);
 
     const auto & select = query_info.query->as<ASTSelectQuery &>();
diff --git a/src/Storages/StorageExecutable.cpp b/src/Storages/StorageExecutable.cpp
index 27bfa6f854c..0094723e3fd 100644
--- a/src/Storages/StorageExecutable.cpp
+++ b/src/Storages/StorageExecutable.cpp
@@ -150,7 +150,7 @@ void StorageExecutable::read(
     for (auto & input_query : input_queries)
     {
         QueryPipelineBuilder builder;
-        if (context->getSettingsRef().enable_analyzer)
+        if (context->getSettingsRef().allow_experimental_analyzer)
             builder = InterpreterSelectQueryAnalyzer(input_query, context, {}).buildQueryPipeline();
         else
             builder = InterpreterSelectWithUnionQuery(input_query, context, {}).buildQueryPipeline();
diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp
index 613317b2564..7c268d36a7b 100644
--- a/src/Storages/StorageMerge.cpp
+++ b/src/Storages/StorageMerge.cpp
@@ -590,7 +590,7 @@ std::vector<ReadFromMerge::ChildPlan> ReadFromMerge::createChildrenPlans(SelectQ
         auto modified_query_info
             = getModifiedQueryInfo(modified_context, table, nested_storage_snaphsot, real_column_names, column_names_as_aliases, aliases);
 
-        if (!context->getSettingsRef().enable_analyzer)
+        if (!context->getSettingsRef().allow_experimental_analyzer)
         {
             auto storage_columns = storage_metadata_snapshot->getColumns();
             auto syntax_result = TreeRewriter(context).analyzeSelect(
@@ -1047,13 +1047,13 @@ void ReadFromMerge::addVirtualColumns(
     const StorageWithLockAndName & storage_with_lock) const
 {
     const auto & [database_name, _, storage, table_name] = storage_with_lock;
-    bool enable_analyzer = context->getSettingsRef().enable_analyzer;
+    bool allow_experimental_analyzer = context->getSettingsRef().allow_experimental_analyzer;
 
     /// Add virtual columns if we don't already have them.
 
     Block plan_header = child.plan.getCurrentDataStream().header;
 
-    if (enable_analyzer)
+    if (allow_experimental_analyzer)
     {
         String table_alias = modified_query_info.query_tree->as<QueryNode>()->getJoinTree()->as<TableNode>()->getAlias();
 
@@ -1133,8 +1133,8 @@ QueryPipelineBuilderPtr ReadFromMerge::buildPipeline(
     if (!builder->initialized())
         return builder;
 
-    bool enable_analyzer = context->getSettingsRef().enable_analyzer;
-    if (processed_stage > child.stage || (enable_analyzer && processed_stage != QueryProcessingStage::FetchColumns))
+    bool allow_experimental_analyzer = context->getSettingsRef().allow_experimental_analyzer;
+    if (processed_stage > child.stage || (allow_experimental_analyzer && processed_stage != QueryProcessingStage::FetchColumns))
     {
         /** Materialization is needed, since from distributed storage the constants come materialized.
           * If you do not do this, different types (Const and non-Const) columns will be produced in different threads,
@@ -1168,7 +1168,7 @@ ReadFromMerge::ChildPlan ReadFromMerge::createPlanForTable(
         modified_select.setFinal();
     }
 
-    bool enable_analyzer = modified_context->getSettingsRef().enable_analyzer;
+    bool allow_experimental_analyzer = modified_context->getSettingsRef().allow_experimental_analyzer;
 
     auto storage_stage = storage->getQueryProcessingStage(modified_context,
         processed_stage,
@@ -1201,13 +1201,13 @@ ReadFromMerge::ChildPlan ReadFromMerge::createPlanForTable(
                 row_policy_data_opt->addStorageFilter(source_step_with_filter);
         }
     }
-    else if (processed_stage > storage_stage || enable_analyzer)
+    else if (processed_stage > storage_stage || allow_experimental_analyzer)
     {
         /// Maximum permissible parallelism is streams_num
         modified_context->setSetting("max_threads", streams_num);
         modified_context->setSetting("max_streams_to_max_threads_ratio", 1);
 
-        if (enable_analyzer)
+        if (allow_experimental_analyzer)
         {
             /// Converting query to AST because types might be different in the source table.
             /// Need to resolve types again.
@@ -1479,7 +1479,7 @@ void ReadFromMerge::convertAndFilterSourceStream(
     auto storage_sample_block = snapshot->metadata->getSampleBlock();
     auto pipe_columns = before_block_header.getNamesAndTypesList();
 
-    if (local_context->getSettingsRef().enable_analyzer)
+    if (local_context->getSettingsRef().allow_experimental_analyzer)
     {
         for (const auto & alias : aliases)
         {
@@ -1522,7 +1522,7 @@ void ReadFromMerge::convertAndFilterSourceStream(
 
     ActionsDAG::MatchColumnsMode convert_actions_match_columns_mode = ActionsDAG::MatchColumnsMode::Name;
 
-    if (local_context->getSettingsRef().enable_analyzer
+    if (local_context->getSettingsRef().allow_experimental_analyzer
         && (child.stage != QueryProcessingStage::FetchColumns || dynamic_cast<const StorageDistributed *>(&snapshot->storage) != nullptr))
         convert_actions_match_columns_mode = ActionsDAG::MatchColumnsMode::Position;
 
diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp
index ebc88993ee4..f55f672fe5e 100644
--- a/src/Storages/StorageMergeTree.cpp
+++ b/src/Storages/StorageMergeTree.cpp
@@ -208,7 +208,7 @@ void StorageMergeTree::read(
     const auto & settings = local_context->getSettingsRef();
     /// reading step for parallel replicas with new analyzer is built in Planner, so don't do it here
     if (local_context->canUseParallelReplicasOnInitiator() && settings.parallel_replicas_for_non_replicated_merge_tree
-        && !settings.enable_analyzer)
+        && !settings.allow_experimental_analyzer)
     {
         ClusterProxy::executeQueryWithParallelReplicas(
             query_plan, getStorageID(), processed_stage, query_info.query, local_context, query_info.storage_limits);
@@ -216,7 +216,7 @@ void StorageMergeTree::read(
     }
 
     if (local_context->canUseParallelReplicasCustomKey() && settings.parallel_replicas_for_non_replicated_merge_tree
-        && !settings.enable_analyzer && local_context->getClientInfo().distributed_depth == 0)
+        && !settings.allow_experimental_analyzer && local_context->getClientInfo().distributed_depth == 0)
     {
         if (auto cluster = local_context->getClusterForParallelReplicas();
             local_context->canUseParallelReplicasCustomKeyForCluster(*cluster))
@@ -244,7 +244,7 @@ void StorageMergeTree::read(
 
     const bool enable_parallel_reading = local_context->canUseParallelReplicasOnFollower()
         && local_context->getSettingsRef().parallel_replicas_for_non_replicated_merge_tree
-        && (!local_context->getSettingsRef().enable_analyzer || query_info.current_table_chosen_for_reading_with_parallel_replicas);
+        && (!local_context->getSettingsRef().allow_experimental_analyzer || query_info.current_table_chosen_for_reading_with_parallel_replicas);
 
     if (auto plan = reader.read(
             column_names,
diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp
index a3965e7a6d4..2d826c6c2df 100644
--- a/src/Storages/StorageReplicatedMergeTree.cpp
+++ b/src/Storages/StorageReplicatedMergeTree.cpp
@@ -5480,13 +5480,13 @@ void StorageReplicatedMergeTree::read(
         return;
     }
     /// reading step for parallel replicas with new analyzer is built in Planner, so don't do it here
-    if (local_context->canUseParallelReplicasOnInitiator() && !settings.enable_analyzer)
+    if (local_context->canUseParallelReplicasOnInitiator() && !settings.allow_experimental_analyzer)
     {
         readParallelReplicasImpl(query_plan, column_names, query_info, local_context, processed_stage);
         return;
     }
 
-    if (local_context->canUseParallelReplicasCustomKey() && !settings.enable_analyzer
+    if (local_context->canUseParallelReplicasCustomKey() && !settings.allow_experimental_analyzer
         && local_context->getClientInfo().distributed_depth == 0)
     {
         if (auto cluster = local_context->getClusterForParallelReplicas();
@@ -5555,7 +5555,7 @@ void StorageReplicatedMergeTree::readLocalImpl(
     const size_t num_streams)
 {
     const bool enable_parallel_reading = local_context->canUseParallelReplicasOnFollower()
-        && (!local_context->getSettingsRef().enable_analyzer
+        && (!local_context->getSettingsRef().allow_experimental_analyzer
             || query_info.current_table_chosen_for_reading_with_parallel_replicas);
 
     auto plan = reader.read(
diff --git a/src/Storages/StorageView.cpp b/src/Storages/StorageView.cpp
index dcb5ef2ae77..878998ebf12 100644
--- a/src/Storages/StorageView.cpp
+++ b/src/Storages/StorageView.cpp
@@ -164,7 +164,7 @@ void StorageView::read(
 
     auto options = SelectQueryOptions(QueryProcessingStage::Complete, 0, false, query_info.settings_limit_offset_done);
 
-    if (context->getSettingsRef().enable_analyzer)
+    if (context->getSettingsRef().allow_experimental_analyzer)
     {
         InterpreterSelectQueryAnalyzer interpreter(current_inner_query, getViewContext(context, storage_snapshot), options, column_names);
         interpreter.addStorageLimits(*query_info.storage_limits);
diff --git a/src/Storages/TTLDescription.cpp b/src/Storages/TTLDescription.cpp
index 16eccfd7343..d674f054632 100644
--- a/src/Storages/TTLDescription.cpp
+++ b/src/Storages/TTLDescription.cpp
@@ -172,7 +172,7 @@ static ExpressionAndSets buildExpressionAndSets(ASTPtr & ast, const NamesAndType
     /// with subqueries it's possible that new analyzer will be enabled in ::read method
     /// of underlying storage when all other parts of infra are not ready for it
     /// (built with old analyzer).
-    context_copy->setSetting("enable_analyzer", false);
+    context_copy->setSetting("allow_experimental_analyzer", false);
     auto syntax_analyzer_result = TreeRewriter(context_copy).analyze(ast, columns);
     ExpressionAnalyzer analyzer(ast, syntax_analyzer_result, context_copy);
     auto dag = analyzer.getActionsDAG(false);
diff --git a/src/Storages/WindowView/StorageWindowView.cpp b/src/Storages/WindowView/StorageWindowView.cpp
index a2b1704f24b..65bf6768b1b 100644
--- a/src/Storages/WindowView/StorageWindowView.cpp
+++ b/src/Storages/WindowView/StorageWindowView.cpp
@@ -1197,7 +1197,7 @@ StorageWindowView::StorageWindowView(
     , fire_signal_timeout_s(context_->getSettingsRef().wait_for_window_view_fire_signal_timeout.totalSeconds())
     , clean_interval_usec(context_->getSettingsRef().window_view_clean_interval.totalMicroseconds())
 {
-    if (context_->getSettingsRef().enable_analyzer)
+    if (context_->getSettingsRef().allow_experimental_analyzer)
         disabled_due_to_analyzer = true;
 
     if (mode <= LoadingStrictnessLevel::CREATE)
@@ -1753,9 +1753,9 @@ StoragePtr StorageWindowView::getTargetTable() const
 
 void StorageWindowView::throwIfWindowViewIsDisabled(ContextPtr local_context) const
 {
-    if (disabled_due_to_analyzer || (local_context && local_context->getSettingsRef().enable_analyzer))
+    if (disabled_due_to_analyzer || (local_context && local_context->getSettingsRef().allow_experimental_analyzer))
         throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "Experimental WINDOW VIEW feature is not supported "
-                        "in the current infrastructure for query analysis (the setting 'enable_analyzer')");
+                        "in the current infrastructure for query analysis (the setting 'allow_experimental_analyzer')");
 }
 
 void registerStorageWindowView(StorageFactory & factory)
diff --git a/src/TableFunctions/TableFunctionView.cpp b/src/TableFunctions/TableFunctionView.cpp
index 02a278cf590..57501df6d4d 100644
--- a/src/TableFunctions/TableFunctionView.cpp
+++ b/src/TableFunctions/TableFunctionView.cpp
@@ -50,7 +50,7 @@ ColumnsDescription TableFunctionView::getActualTableStructure(ContextPtr context
 
     Block sample_block;
 
-    if (context->getSettingsRef().enable_analyzer)
+    if (context->getSettingsRef().allow_experimental_analyzer)
         sample_block = InterpreterSelectQueryAnalyzer::getSampleBlock(create.children[0], context);
     else
         sample_block = InterpreterSelectWithUnionQuery::getSampleBlock(create.children[0], context);
diff --git a/src/TableFunctions/TableFunctionViewIfPermitted.cpp b/src/TableFunctions/TableFunctionViewIfPermitted.cpp
index 7bae2731525..935be6c1987 100644
--- a/src/TableFunctions/TableFunctionViewIfPermitted.cpp
+++ b/src/TableFunctions/TableFunctionViewIfPermitted.cpp
@@ -114,7 +114,7 @@ bool TableFunctionViewIfPermitted::isPermitted(const ContextPtr & context, const
 
     try
     {
-        if (context->getSettingsRef().enable_analyzer)
+        if (context->getSettingsRef().allow_experimental_analyzer)
         {
             sample_block = InterpreterSelectQueryAnalyzer::getSampleBlock(create.children[0], context);
         }

From ddcad048de8862392194d649788c56466a432b0e Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <nikitamikhaylov@clickhouse.com>
Date: Tue, 30 Jul 2024 14:44:56 +0000
Subject: [PATCH 1396/1488] Fix build

---
 src/Core/SettingsChangesHistory.cpp | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp
index 71f7c940e2c..bd7330ac6f8 100644
--- a/src/Core/SettingsChangesHistory.cpp
+++ b/src/Core/SettingsChangesHistory.cpp
@@ -500,6 +500,8 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
             {"enable_optimize_predicate_expression", 0, 1, "Optimize predicates to subqueries by default"}
         }
     },
+    {"24.8", {{"enable_analyzer", false, true, "Move analyzer to the Beta stage."},
+              }},
     {"24.7", {{"output_format_parquet_write_page_index", false, true, "Add a possibility to write page index into parquet files."},
               {"output_format_binary_encode_types_in_binary_format", false, false, "Added new setting to allow to write type names in binary format in RowBinaryWithNamesAndTypes output format"},
               {"input_format_binary_decode_types_in_binary_format", false, false, "Added new setting to allow to read type names in binary format in RowBinaryWithNamesAndTypes input format"},
@@ -526,7 +528,7 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
               {"postgresql_connection_pool_retries", 2, 2, "Allow to control the number of retries in PostgreSQL connection pool."}
               {"postgresql_connection_pool_retries", 2, 2, "Allow to control the number of retries in PostgreSQL connection pool."},
               {"restore_replace_external_table_functions_to_null", false, false, "New setting."},
-              {"restore_replace_external_engines_to_null", false, false, "New setting."}
+              {"restore_replace_external_engines_to_null", false, false, "New setting."},
               {"enable_analyzer", false, true, "Move analyzer to the Beta stage."},
               {"allow_experimental_analyzer", false, true, "Move analyzer to the Beta stage."},
               }},

From 194e7824de080d4ce86507c1f29c74a0d917b8a3 Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <nikitamikhaylov@clickhouse.com>
Date: Tue, 30 Jul 2024 21:29:07 +0000
Subject: [PATCH 1397/1488] Minor

---
 src/Core/SettingsChangesHistory.cpp            | 4 +---
 tests/queries/0_stateless/00202_cross_join.sql | 2 +-
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp
index bd7330ac6f8..84c0ef2b127 100644
--- a/src/Core/SettingsChangesHistory.cpp
+++ b/src/Core/SettingsChangesHistory.cpp
@@ -528,9 +528,7 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
               {"postgresql_connection_pool_retries", 2, 2, "Allow to control the number of retries in PostgreSQL connection pool."}
               {"postgresql_connection_pool_retries", 2, 2, "Allow to control the number of retries in PostgreSQL connection pool."},
               {"restore_replace_external_table_functions_to_null", false, false, "New setting."},
-              {"restore_replace_external_engines_to_null", false, false, "New setting."},
-              {"enable_analyzer", false, true, "Move analyzer to the Beta stage."},
-              {"allow_experimental_analyzer", false, true, "Move analyzer to the Beta stage."},
+              {"restore_replace_external_engines_to_null", false, false, "New setting."}
               }},
     {"24.6", {{"materialize_skip_indexes_on_insert", true, true, "Added new setting to allow to disable materialization of skip indexes on insert"},
               {"materialize_statistics_on_insert", true, true, "Added new setting to allow to disable materialization of statistics on insert"},
diff --git a/tests/queries/0_stateless/00202_cross_join.sql b/tests/queries/0_stateless/00202_cross_join.sql
index ea327817a58..301161b6583 100644
--- a/tests/queries/0_stateless/00202_cross_join.sql
+++ b/tests/queries/0_stateless/00202_cross_join.sql
@@ -4,5 +4,5 @@ SET join_algorithm = 'auto';
 SELECT x, y FROM (SELECT number AS x FROM system.numbers LIMIT 3) js1 CROSS JOIN (SELECT number AS y FROM system.numbers LIMIT 5) js2;
 
 -- Just to test that we preserved old setting name this we use `enable_analyzer` instead of `enable_analyzer` here.
-SET enable_analyzer = 1;
+SET allow_experimental_analyzer = 1;
 SELECT x, y FROM (SELECT number AS x FROM system.numbers LIMIT 3) js1 CROSS JOIN (SELECT number AS y FROM system.numbers LIMIT 5) js2;

From 735f1547575de4ea4a6a2a442f80ab159de37174 Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <nikitamikhaylov@clickhouse.com>
Date: Wed, 31 Jul 2024 14:21:12 +0000
Subject: [PATCH 1398/1488] Build

---
 src/Core/SettingsChangesHistory.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp
index 84c0ef2b127..c5d47fcdc4b 100644
--- a/src/Core/SettingsChangesHistory.cpp
+++ b/src/Core/SettingsChangesHistory.cpp
@@ -525,7 +525,6 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
               {"ignore_on_cluster_for_replicated_named_collections_queries", false, false, "Ignore ON CLUSTER clause for replicated named collections management queries."},
               {"backup_restore_s3_retry_attempts", 1000,1000, "Setting for Aws::Client::RetryStrategy, Aws::Client does retries itself, 0 means no retries. It takes place only for backup/restore."},
               {"postgresql_connection_attempt_timeout", 2, 2, "Allow to control 'connect_timeout' parameter of PostgreSQL connection."},
-              {"postgresql_connection_pool_retries", 2, 2, "Allow to control the number of retries in PostgreSQL connection pool."}
               {"postgresql_connection_pool_retries", 2, 2, "Allow to control the number of retries in PostgreSQL connection pool."},
               {"restore_replace_external_table_functions_to_null", false, false, "New setting."},
               {"restore_replace_external_engines_to_null", false, false, "New setting."}

From 7c729d397412b711310aa23e9b0d9152467ba2b5 Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <nikitamikhaylov@clickhouse.com>
Date: Wed, 31 Jul 2024 15:32:32 +0000
Subject: [PATCH 1399/1488] Remove all changes

---
 src/Core/SettingsChangesHistory.cpp | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp
index c5d47fcdc4b..75bc15358ab 100644
--- a/src/Core/SettingsChangesHistory.cpp
+++ b/src/Core/SettingsChangesHistory.cpp
@@ -525,9 +525,7 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
               {"ignore_on_cluster_for_replicated_named_collections_queries", false, false, "Ignore ON CLUSTER clause for replicated named collections management queries."},
               {"backup_restore_s3_retry_attempts", 1000,1000, "Setting for Aws::Client::RetryStrategy, Aws::Client does retries itself, 0 means no retries. It takes place only for backup/restore."},
               {"postgresql_connection_attempt_timeout", 2, 2, "Allow to control 'connect_timeout' parameter of PostgreSQL connection."},
-              {"postgresql_connection_pool_retries", 2, 2, "Allow to control the number of retries in PostgreSQL connection pool."},
-              {"restore_replace_external_table_functions_to_null", false, false, "New setting."},
-              {"restore_replace_external_engines_to_null", false, false, "New setting."}
+              {"postgresql_connection_pool_retries", 2, 2, "Allow to control the number of retries in PostgreSQL connection pool."}
               }},
     {"24.6", {{"materialize_skip_indexes_on_insert", true, true, "Added new setting to allow to disable materialization of skip indexes on insert"},
               {"materialize_statistics_on_insert", true, true, "Added new setting to allow to disable materialization of statistics on insert"},

From abf543cb2e7b1849d81c2cf51d5a2707c7bc0258 Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <nikitamikhaylov@clickhouse.com>
Date: Wed, 31 Jul 2024 16:00:04 +0000
Subject: [PATCH 1400/1488] Add to settings changes

---
 src/Core/SettingsChangesHistory.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp
index 75bc15358ab..893394adb61 100644
--- a/src/Core/SettingsChangesHistory.cpp
+++ b/src/Core/SettingsChangesHistory.cpp
@@ -501,6 +501,7 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
         }
     },
     {"24.8", {{"enable_analyzer", false, true, "Move analyzer to the Beta stage."},
+    {"24.8", {{"enable_analyzer", true, true, "Added an alias to the `allow_experimental_analyzer` setting."}
               }},
     {"24.7", {{"output_format_parquet_write_page_index", false, true, "Add a possibility to write page index into parquet files."},
               {"output_format_binary_encode_types_in_binary_format", false, false, "Added new setting to allow to write type names in binary format in RowBinaryWithNamesAndTypes output format"},

From 64c0903f541eb2028f506c69909785f9ca9e32e1 Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <nikitamikhaylov@clickhouse.com>
Date: Wed, 31 Jul 2024 22:45:18 +0200
Subject: [PATCH 1401/1488] Validate also alias

---
 src/Interpreters/executeQuery.cpp | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp
index ce58f7f922c..fe87eed5570 100644
--- a/src/Interpreters/executeQuery.cpp
+++ b/src/Interpreters/executeQuery.cpp
@@ -690,6 +690,12 @@ void validateAnalyzerSettings(ASTPtr ast, bool context_value)
                 if (top_level != value->safeGet<bool>())
                     throw Exception(ErrorCodes::INCORRECT_QUERY, "Setting 'allow_experimental_analyzer' is changed in the subquery. Top level value: {}", top_level);
             }
+
+            if (auto * value = set_query->changes.tryGet("enable_analyzer"))
+            {
+                if (top_level != value->safeGet<bool>())
+                    throw Exception(ErrorCodes::INCORRECT_QUERY, "Setting 'enable_analyzer' is changed in the subquery. Top level value: {}", top_level);
+            }
         }
 
         for (auto child : node->children)

From ef811fd25c36855e1c39029962999609ef6fb4de Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <nikitamikhaylov@clickhouse.com>
Date: Thu, 1 Aug 2024 12:50:56 +0000
Subject: [PATCH 1402/1488] Fix test

---
 .../01049_join_low_card_bug_long.reference.j2 | 60 +++++++++----------
 1 file changed, 30 insertions(+), 30 deletions(-)

diff --git a/tests/queries/0_stateless/01049_join_low_card_bug_long.reference.j2 b/tests/queries/0_stateless/01049_join_low_card_bug_long.reference.j2
index 872bb448027..341e77b1d78 100644
--- a/tests/queries/0_stateless/01049_join_low_card_bug_long.reference.j2
+++ b/tests/queries/0_stateless/01049_join_low_card_bug_long.reference.j2
@@ -19,17 +19,17 @@ str_r	LowCardinality(String)
 str_l	LowCardinality(String)
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l RIGHT JOIN r_lc AS r USING (x) ORDER BY x, r.lc, l.lc;
 LowCardinality(String)	LowCardinality(String)	str	str	LowCardinality(String)	LowCardinality(String)	str	str
-LowCardinality(String)	LowCardinality(String)	str_r	str_r	LowCardinality(String)	LowCardinality(String)
+LowCardinality(String)	LowCardinality(String)	str_r	str_r	LowCardinality(String)	LowCardinality(String)		
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l RIGHT JOIN r_lc AS r USING (lc) ORDER BY x, r.lc, l.lc;
 LowCardinality(String)	LowCardinality(String)	str	str	LowCardinality(String)	LowCardinality(String)	str	str
-LowCardinality(String)	LowCardinality(String)	str_r	str_r	LowCardinality(String)	LowCardinality(String)
+LowCardinality(String)	LowCardinality(String)	str_r	str_r	LowCardinality(String)	LowCardinality(String)		
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l FULL JOIN r_lc AS r USING (x) ORDER BY x, r.lc, l.lc;
 LowCardinality(String)	LowCardinality(String)	str	str	LowCardinality(String)	LowCardinality(String)	str	str
-LowCardinality(String)	LowCardinality(String)	str_r	str_r	LowCardinality(String)	LowCardinality(String)
+LowCardinality(String)	LowCardinality(String)	str_r	str_r	LowCardinality(String)	LowCardinality(String)		
 LowCardinality(String)	LowCardinality(String)			LowCardinality(String)	LowCardinality(String)	str_l	str_l
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l FULL JOIN r_lc AS r USING (lc) ORDER BY x, r.lc, l.lc;
 LowCardinality(String)	LowCardinality(String)	str	str	LowCardinality(String)	LowCardinality(String)	str	str
-LowCardinality(String)	LowCardinality(String)	str_r	str_r	LowCardinality(String)	LowCardinality(String)
+LowCardinality(String)	LowCardinality(String)	str_r	str_r	LowCardinality(String)	LowCardinality(String)		
 LowCardinality(String)	LowCardinality(String)			LowCardinality(String)	LowCardinality(String)	str_l	str_l
 --
 
@@ -49,17 +49,17 @@ str_r	LowCardinality(String)
 str_l	LowCardinality(String)
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l RIGHT JOIN r USING (x) ORDER BY x, r.lc, l.lc;
 String	String	str	str	LowCardinality(String)	LowCardinality(String)	str	str
-String	String	str_r	str_r	LowCardinality(String)	LowCardinality(String)
+String	String	str_r	str_r	LowCardinality(String)	LowCardinality(String)		
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l RIGHT JOIN r USING (lc) ORDER BY x, r.lc, l.lc;
 String	String	str	str	LowCardinality(String)	LowCardinality(String)	str	str
-String	String	str_r	str_r	LowCardinality(String)	LowCardinality(String)
+String	String	str_r	str_r	LowCardinality(String)	LowCardinality(String)		
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l FULL JOIN r USING (x) ORDER BY x, r.lc, l.lc;
 String	String	str	str	LowCardinality(String)	LowCardinality(String)	str	str
-String	String	str_r	str_r	LowCardinality(String)	LowCardinality(String)
+String	String	str_r	str_r	LowCardinality(String)	LowCardinality(String)		
 String	String			LowCardinality(String)	LowCardinality(String)	str_l	str_l
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l FULL JOIN r USING (lc) ORDER BY x, r.lc, l.lc;
 String	String	str	str	LowCardinality(String)	LowCardinality(String)	str	str
-String	String	str_r	str_r	LowCardinality(String)	LowCardinality(String)
+String	String	str_r	str_r	LowCardinality(String)	LowCardinality(String)		
 String	String			LowCardinality(String)	LowCardinality(String)	str_l	str_l
 --
 
@@ -79,17 +79,17 @@ str_r	String
 str_l	String
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l RIGHT JOIN r_lc AS r USING (x) ORDER BY x, r.lc, l.lc;
 LowCardinality(String)	LowCardinality(String)	str	str	String	String	str	str
-LowCardinality(String)	LowCardinality(String)	str_r	str_r	String	String
+LowCardinality(String)	LowCardinality(String)	str_r	str_r	String	String		
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l RIGHT JOIN r_lc AS r USING (lc) ORDER BY x, r.lc, l.lc;
 LowCardinality(String)	LowCardinality(String)	str	str	String	String	str	str
-LowCardinality(String)	LowCardinality(String)	str_r	str_r	String	String
+LowCardinality(String)	LowCardinality(String)	str_r	str_r	String	String		
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l FULL JOIN r_lc AS r USING (x) ORDER BY x, r.lc, l.lc;
 LowCardinality(String)	LowCardinality(String)	str	str	String	String	str	str
-LowCardinality(String)	LowCardinality(String)	str_r	str_r	String	String
+LowCardinality(String)	LowCardinality(String)	str_r	str_r	String	String		
 LowCardinality(String)	LowCardinality(String)			String	String	str_l	str_l
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l FULL JOIN r_lc AS r USING (lc) ORDER BY x, r.lc, l.lc;
 LowCardinality(String)	LowCardinality(String)	str	str	String	String	str	str
-LowCardinality(String)	LowCardinality(String)	str_r	str_r	String	String
+LowCardinality(String)	LowCardinality(String)	str_r	str_r	String	String		
 LowCardinality(String)	LowCardinality(String)			String	String	str_l	str_l
 --
 
@@ -109,17 +109,17 @@ str_r	LowCardinality(String)
 str_l	LowCardinality(String)
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l RIGHT JOIN nr AS r USING (x) ORDER BY x, r.lc, l.lc;
 Nullable(String)	Nullable(String)	str	str	LowCardinality(String)	LowCardinality(String)	str	str
-Nullable(String)	Nullable(String)	str_r	str_r	LowCardinality(String)	LowCardinality(String)
+Nullable(String)	Nullable(String)	str_r	str_r	LowCardinality(String)	LowCardinality(String)		
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l RIGHT JOIN nr AS r USING (lc) ORDER BY x, r.lc, l.lc;
 Nullable(String)	Nullable(String)	str	str	LowCardinality(String)	LowCardinality(String)	str	str
-Nullable(String)	Nullable(String)	str_r	str_r	LowCardinality(String)	LowCardinality(String)
+Nullable(String)	Nullable(String)	str_r	str_r	LowCardinality(String)	LowCardinality(String)		
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l FULL JOIN nr AS r USING (x) ORDER BY x, r.lc, l.lc;
 Nullable(String)	Nullable(String)	str	str	LowCardinality(String)	LowCardinality(String)	str	str
-Nullable(String)	Nullable(String)	str_r	str_r	LowCardinality(String)	LowCardinality(String)
+Nullable(String)	Nullable(String)	str_r	str_r	LowCardinality(String)	LowCardinality(String)		
 Nullable(String)	Nullable(String)	\N	\N	LowCardinality(String)	LowCardinality(String)	str_l	str_l
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l FULL JOIN nr AS r USING (lc) ORDER BY x, r.lc, l.lc;
 Nullable(String)	Nullable(String)	str	str	LowCardinality(String)	LowCardinality(String)	str	str
-Nullable(String)	Nullable(String)	str_r	str_r	LowCardinality(String)	LowCardinality(String)
+Nullable(String)	Nullable(String)	str_r	str_r	LowCardinality(String)	LowCardinality(String)		
 Nullable(String)	Nullable(String)	\N	\N	LowCardinality(String)	LowCardinality(String)	str_l	str_l
 --
 
@@ -333,17 +333,17 @@ str_r	LowCardinality(String)
 str_l	LowCardinality(String)
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l RIGHT JOIN r_lc AS r USING (x) ORDER BY x, r.lc, l.lc;
 LowCardinality(String)	LowCardinality(String)	str	str	LowCardinality(String)	LowCardinality(String)	str	str
-LowCardinality(String)	LowCardinality(String)	str_r	str_r	LowCardinality(String)	LowCardinality(String)
+LowCardinality(String)	LowCardinality(String)	str_r	str_r	LowCardinality(String)	LowCardinality(String)		
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l RIGHT JOIN r_lc AS r USING (lc) ORDER BY x, r.lc, l.lc;
 LowCardinality(String)	LowCardinality(String)	str	str	LowCardinality(String)	LowCardinality(String)	str	str
-LowCardinality(String)	LowCardinality(String)	str_r	str_r	LowCardinality(String)	LowCardinality(String)
+LowCardinality(String)	LowCardinality(String)	str_r	str_r	LowCardinality(String)	LowCardinality(String)		
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l FULL JOIN r_lc AS r USING (x) ORDER BY x, r.lc, l.lc;
 LowCardinality(String)	LowCardinality(String)	str	str	LowCardinality(String)	LowCardinality(String)	str	str
-LowCardinality(String)	LowCardinality(String)	str_r	str_r	LowCardinality(String)	LowCardinality(String)
+LowCardinality(String)	LowCardinality(String)	str_r	str_r	LowCardinality(String)	LowCardinality(String)		
 LowCardinality(String)	LowCardinality(String)			LowCardinality(String)	LowCardinality(String)	str_l	str_l
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l FULL JOIN r_lc AS r USING (lc) ORDER BY x, r.lc, l.lc;
 LowCardinality(String)	LowCardinality(String)	str	str	LowCardinality(String)	LowCardinality(String)	str	str
-LowCardinality(String)	LowCardinality(String)	str_r	str_r	LowCardinality(String)	LowCardinality(String)
+LowCardinality(String)	LowCardinality(String)	str_r	str_r	LowCardinality(String)	LowCardinality(String)		
 LowCardinality(String)	LowCardinality(String)			LowCardinality(String)	LowCardinality(String)	str_l	str_l
 --
 
@@ -363,17 +363,17 @@ str_r	String
 str_l	String
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l RIGHT JOIN r USING (x) ORDER BY x, r.lc, l.lc;
 String	String	str	str	LowCardinality(String)	LowCardinality(String)	str	str
-String	String	str_r	str_r	LowCardinality(String)	LowCardinality(String)
+String	String	str_r	str_r	LowCardinality(String)	LowCardinality(String)		
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l RIGHT JOIN r USING (lc) ORDER BY x, r.lc, l.lc;
 String	String	str	str	String	String	str	str
-String	String	str_r	str_r	String	String
+String	String	str_r	str_r	String	String		
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l FULL JOIN r USING (x) ORDER BY x, r.lc, l.lc;
 String	String	str	str	LowCardinality(String)	LowCardinality(String)	str	str
-String	String	str_r	str_r	LowCardinality(String)	LowCardinality(String)
+String	String	str_r	str_r	LowCardinality(String)	LowCardinality(String)		
 String	String			LowCardinality(String)	LowCardinality(String)	str_l	str_l
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l FULL JOIN r USING (lc) ORDER BY x, r.lc, l.lc;
 String	String	str	str	String	String	str	str
-String	String	str_r	str_r	String	String
+String	String	str_r	str_r	String	String		
 String	String			String	String	str_l	str_l
 --
 
@@ -393,17 +393,17 @@ str_r	String
 str_l	String
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l RIGHT JOIN r_lc AS r USING (x) ORDER BY x, r.lc, l.lc;
 LowCardinality(String)	LowCardinality(String)	str	str	String	String	str	str
-LowCardinality(String)	LowCardinality(String)	str_r	str_r	String	String
+LowCardinality(String)	LowCardinality(String)	str_r	str_r	String	String		
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l RIGHT JOIN r_lc AS r USING (lc) ORDER BY x, r.lc, l.lc;
 String	String	str	str	String	String	str	str
-String	String	str_r	str_r	String	String
+String	String	str_r	str_r	String	String		
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l FULL JOIN r_lc AS r USING (x) ORDER BY x, r.lc, l.lc;
 LowCardinality(String)	LowCardinality(String)	str	str	String	String	str	str
-LowCardinality(String)	LowCardinality(String)	str_r	str_r	String	String
+LowCardinality(String)	LowCardinality(String)	str_r	str_r	String	String		
 LowCardinality(String)	LowCardinality(String)			String	String	str_l	str_l
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l FULL JOIN r_lc AS r USING (lc) ORDER BY x, r.lc, l.lc;
 String	String	str	str	String	String	str	str
-String	String	str_r	str_r	String	String
+String	String	str_r	str_r	String	String		
 String	String			String	String	str_l	str_l
 --
 
@@ -423,13 +423,13 @@ str_r	Nullable(String)
 str_l	Nullable(String)
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l RIGHT JOIN nr AS r USING (x) ORDER BY x, r.lc, l.lc;
 Nullable(String)	Nullable(String)	str	str	LowCardinality(String)	LowCardinality(String)	str	str
-Nullable(String)	Nullable(String)	str_r	str_r	LowCardinality(String)	LowCardinality(String)
+Nullable(String)	Nullable(String)	str_r	str_r	LowCardinality(String)	LowCardinality(String)		
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l RIGHT JOIN nr AS r USING (lc) ORDER BY x, r.lc, l.lc;
 Nullable(String)	Nullable(String)	str	str	Nullable(String)	Nullable(String)	str	str
 Nullable(String)	Nullable(String)	str_r	str_r	Nullable(String)	Nullable(String)	\N	\N
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l FULL JOIN nr AS r USING (x) ORDER BY x, r.lc, l.lc;
 Nullable(String)	Nullable(String)	str	str	LowCardinality(String)	LowCardinality(String)	str	str
-Nullable(String)	Nullable(String)	str_r	str_r	LowCardinality(String)	LowCardinality(String)
+Nullable(String)	Nullable(String)	str_r	str_r	LowCardinality(String)	LowCardinality(String)		
 Nullable(String)	Nullable(String)	\N	\N	LowCardinality(String)	LowCardinality(String)	str_l	str_l
 SELECT toTypeName(r.lc), toTypeName(materialize(r.lc)), r.lc, materialize(r.lc), toTypeName(l.lc), toTypeName(materialize(l.lc)), l.lc, materialize(l.lc) FROM l_lc AS l FULL JOIN nr AS r USING (lc) ORDER BY x, r.lc, l.lc;
 Nullable(String)	Nullable(String)	str	str	Nullable(String)	Nullable(String)	str	str

From 0301af99e5c6eda72a379a0d048903a3ecb9e0e0 Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <nikitamikhaylov@clickhouse.com>
Date: Thu, 1 Aug 2024 15:53:08 +0000
Subject: [PATCH 1403/1488] Fixed a bunch of tests

---
 tests/config/users.d/analyzer.xml                         | 2 +-
 .../integration/helpers/0_common_enable_old_analyzer.xml  | 2 +-
 tests/integration/test_analyzer_compatibility/test.py     | 6 ++++--
 .../02998_analyzer_secret_args_tree_node.reference        | 8 ++++----
 4 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/tests/config/users.d/analyzer.xml b/tests/config/users.d/analyzer.xml
index edba8b8578e..4b9764526fa 100644
--- a/tests/config/users.d/analyzer.xml
+++ b/tests/config/users.d/analyzer.xml
@@ -1,7 +1,7 @@
 <clickhouse>
     <profiles>
         <default>
-            <enable_analyzer>0</enable_analyzer>
+            <allow_experimental_analyzer>0</allow_experimental_analyzer>
         </default>
     </profiles>
 </clickhouse>
diff --git a/tests/integration/helpers/0_common_enable_old_analyzer.xml b/tests/integration/helpers/0_common_enable_old_analyzer.xml
index edba8b8578e..4b9764526fa 100644
--- a/tests/integration/helpers/0_common_enable_old_analyzer.xml
+++ b/tests/integration/helpers/0_common_enable_old_analyzer.xml
@@ -1,7 +1,7 @@
 <clickhouse>
     <profiles>
         <default>
-            <enable_analyzer>0</enable_analyzer>
+            <allow_experimental_analyzer>0</allow_experimental_analyzer>
         </default>
     </profiles>
 </clickhouse>
diff --git a/tests/integration/test_analyzer_compatibility/test.py b/tests/integration/test_analyzer_compatibility/test.py
index 2c840154eb5..6eeba1f1274 100644
--- a/tests/integration/test_analyzer_compatibility/test.py
+++ b/tests/integration/test_analyzer_compatibility/test.py
@@ -78,10 +78,12 @@ WHERE initial_query_id = '{query_id}';"""
     current.query("SYSTEM FLUSH LOGS")
     backward.query("SYSTEM FLUSH LOGS")
 
+    # The old version doesn't know about the alias.
+    # For this we will ask about the old experimental name.
     assert (
         backward.query(
             """
-SELECT hostname() AS h, getSetting('enable_analyzer')
+SELECT hostname() AS h, getSetting('allow_experimental_analyzer')
 FROM clusterAllReplicas('test_cluster_mixed', system.one)
 ORDER BY h;"""
         )
@@ -92,7 +94,7 @@ ORDER BY h;"""
     analyzer_enabled = backward.query(
         f"""
 SELECT
-DISTINCT Settings['enable_analyzer']
+DISTINCT Settings['allow_experimental_analyzer']
 FROM clusterAllReplicas('test_cluster_mixed', system.query_log)
 WHERE initial_query_id = '{query_id}';"""
     )
diff --git a/tests/queries/0_stateless/02998_analyzer_secret_args_tree_node.reference b/tests/queries/0_stateless/02998_analyzer_secret_args_tree_node.reference
index ccd9540cb49..67ef38093d4 100644
--- a/tests/queries/0_stateless/02998_analyzer_secret_args_tree_node.reference
+++ b/tests/queries/0_stateless/02998_analyzer_secret_args_tree_node.reference
@@ -7,7 +7,7 @@ QUERY id: 0
     encrypt(\'aes-256-ofb\', [HIDDEN id: 3], [HIDDEN id: 2]) Nullable(String)
   PROJECTION
     LIST id: 1, nodes: 2
-      CONSTANT id: 2, constant_value: \'\\n��&\', constant_value_type: Nullable(String)
+      CONSTANT id: 2, constant_value: \'\\n��&\', constant_value_type: Nullable(String)
         EXPRESSION
           FUNCTION id: 3, function_name: encrypt, function_type: ordinary, result_type: Nullable(String)
             ARGUMENTS
@@ -15,7 +15,7 @@ QUERY id: 0
                 CONSTANT id: 5, constant_value: \'aes-256-ofb\', constant_value_type: String
                 CONSTANT id: 6, constant_value: [HIDDEN id: 1], constant_value_type: Nullable(String)
                 CONSTANT id: 7, constant_value: [HIDDEN id: 2], constant_value_type: String
-      CONSTANT id: 8, constant_value: \'��\', constant_value_type: Nullable(String)
+      CONSTANT id: 8, constant_value: \'��\', constant_value_type: Nullable(String)
         EXPRESSION
           FUNCTION id: 9, function_name: encrypt, function_type: ordinary, result_type: Nullable(String)
             ARGUMENTS
@@ -33,7 +33,7 @@ QUERY id: 0
     encrypt(\'aes-256-ofb\', _subquery_2, \'12345678901234567890123456789012\') Nullable(String)
   PROJECTION
     LIST id: 1, nodes: 2
-      CONSTANT id: 2, constant_value: \'\\n��&\', constant_value_type: Nullable(String)
+      CONSTANT id: 2, constant_value: \'\\n��&\', constant_value_type: Nullable(String)
         EXPRESSION
           FUNCTION id: 3, function_name: encrypt, function_type: ordinary, result_type: Nullable(String)
             ARGUMENTS
@@ -50,7 +50,7 @@ QUERY id: 0
                       JOIN TREE
                         TABLE id: 10, table_name: system.one
                 CONSTANT id: 11, constant_value: \'12345678901234567890123456789012\', constant_value_type: String
-      CONSTANT id: 12, constant_value: \'��\', constant_value_type: Nullable(String)
+      CONSTANT id: 12, constant_value: \'��\', constant_value_type: Nullable(String)
         EXPRESSION
           FUNCTION id: 13, function_name: encrypt, function_type: ordinary, result_type: Nullable(String)
             ARGUMENTS

From 330aae951706e4c3fd6ddada231d5d4bc23e37a0 Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <nikitamikhaylov@clickhouse.com>
Date: Thu, 1 Aug 2024 16:51:44 +0000
Subject: [PATCH 1404/1488] Even better test

---
 .../test_analyzer_compatibility/test.py       | 34 ++++++++++++++++---
 1 file changed, 29 insertions(+), 5 deletions(-)

diff --git a/tests/integration/test_analyzer_compatibility/test.py b/tests/integration/test_analyzer_compatibility/test.py
index 6eeba1f1274..505d1629cd2 100644
--- a/tests/integration/test_analyzer_compatibility/test.py
+++ b/tests/integration/test_analyzer_compatibility/test.py
@@ -1,4 +1,5 @@
 import uuid
+import time
 
 import pytest
 from helpers.cluster import ClickHouseCluster
@@ -51,19 +52,19 @@ def test_two_new_versions(start_cluster):
     assert (
         current.query(
             """
-SELECT hostname() AS h, getSetting('enable_analyzer')
+SELECT hostname() AS h, getSetting('allow_experimental_analyzer')
 FROM clusterAllReplicas('test_cluster_mixed', system.one)
 ORDER BY h;"""
         )
         == TSV([["backward", "true"], ["current", "true"]])
     )
 
-    # Should be enabled everywhere
-    analyzer_enabled = current.query(
+    # Should be enabled explicitly on the old instance.
+    analyzer_enabled = backward.query(
         f"""
 SELECT
-DISTINCT Settings['enable_analyzer']
-FROM clusterAllReplicas('test_cluster_mixed', system.query_log)
+DISTINCT Settings['allow_experimental_analyzer']
+FROM system.query_log
 WHERE initial_query_id = '{query_id}';"""
     )
 
@@ -100,3 +101,26 @@ WHERE initial_query_id = '{query_id}';"""
     )
 
     assert TSV(analyzer_enabled) == TSV("0")
+
+    # Only new version knows about the alias
+    # and it will send the old setting `allow_experimental_analyzer`
+    # to the remote server.
+    query_id = str(uuid.uuid4())
+    current.query(
+        "SELECT * FROM clusterAllReplicas('test_cluster_mixed', system.tables) SETTINGS enable_analyzer = 1;",
+        query_id=query_id,
+    )
+
+    current.query("SYSTEM FLUSH LOGS")
+    backward.query("SYSTEM FLUSH LOGS")
+
+    # Should be disabled explicitly everywhere.
+    analyzer_enabled = current.query(
+        f"""
+SELECT
+DISTINCT Settings['allow_experimental_analyzer']
+FROM system.query_log
+WHERE initial_query_id = '{query_id}';"""
+    )
+
+    assert TSV(analyzer_enabled) == TSV("1")

From a2020224750f9861f1571d4aa8e139560b3a1dfc Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <nikitamikhaylov@clickhouse.com>
Date: Fri, 2 Aug 2024 11:29:56 +0000
Subject: [PATCH 1405/1488] Fixed performance tests

---
 tests/performance/storage_join_direct_join.xml | 2 +-
 tests/performance/uniq_to_count.xml            | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/performance/storage_join_direct_join.xml b/tests/performance/storage_join_direct_join.xml
index 987500bb4f0..867108ac2b7 100644
--- a/tests/performance/storage_join_direct_join.xml
+++ b/tests/performance/storage_join_direct_join.xml
@@ -15,5 +15,5 @@
 
     <query>SELECT keys.key, value1 FROM keys ANY LEFT JOIN dict AS d ON (keys.key = d.key) FORMAT Null;</query>
     <query>SELECT keys.key, value1 FROM keys ANY LEFT JOIN dict AS d ON (keys.key = d.key) FORMAT Null SETTINGS
-        enable_analyzer=1</query>
+        allow_experimental_analyzer=1</query>
 </test>
diff --git a/tests/performance/uniq_to_count.xml b/tests/performance/uniq_to_count.xml
index 57b0085d8fa..64e4cf1cc0d 100644
--- a/tests/performance/uniq_to_count.xml
+++ b/tests/performance/uniq_to_count.xml
@@ -3,6 +3,6 @@
     <query>select uniq(number) from (select number from numbers(1000000) group by number)</query>
 
     <!--For new analyzer-->
-    <query>select uniq(number) from (select DISTINCT number from numbers(1000000)) SETTINGS enable_analyzer=1</query>
-    <query>select uniq(number) from (select number from numbers(1000000) group by number) SETTINGS enable_analyzer=1</query>
+    <query>select uniq(number) from (select DISTINCT number from numbers(1000000)) SETTINGS allow_experimental_analyzer=1</query>
+    <query>select uniq(number) from (select number from numbers(1000000) group by number) SETTINGS allow_experimental_analyzer=1</query>
 </test>

From a9204c5da3179ca7c5e78fe537f4e57bf129dbab Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <nikitamikhaylov@clickhouse.com>
Date: Fri, 2 Aug 2024 11:34:57 +0000
Subject: [PATCH 1406/1488] Delete changes

---
 .../0_stateless/02995_baseline_24_7_2.tsv     | 26 -------------------
 1 file changed, 26 deletions(-)

diff --git a/tests/queries/0_stateless/02995_baseline_24_7_2.tsv b/tests/queries/0_stateless/02995_baseline_24_7_2.tsv
index d3a07ecb644..10b392f3e04 100644
--- a/tests/queries/0_stateless/02995_baseline_24_7_2.tsv
+++ b/tests/queries/0_stateless/02995_baseline_24_7_2.tsv
@@ -18,11 +18,7 @@ allow_distributed_ddl	1
 allow_drop_detached	0
 allow_execute_multiif_columnar	1
 allow_experimental_alter_materialized_view_structure	1
-<<<<<<< HEAD:tests/queries/0_stateless/02995_baseline_24_7_2.tsv
 allow_experimental_analyzer	1
-=======
-allow_experimental_analyzer	0
->>>>>>> Fix tests:tests/queries/0_stateless/02995_baseline_23_12_1.tsv
 allow_experimental_annoy_index	0
 allow_experimental_bigint_types	1
 allow_experimental_codecs	0
@@ -163,10 +159,7 @@ cloud_mode	0
 cloud_mode_engine	1
 cluster_for_parallel_replicas	
 collect_hash_table_stats_during_aggregation	1
-<<<<<<< HEAD:tests/queries/0_stateless/02995_baseline_24_7_2.tsv
 collect_hash_table_stats_during_joins	1
-=======
->>>>>>> Fix tests:tests/queries/0_stateless/02995_baseline_23_12_1.tsv
 column_names_for_schema_inference	
 compatibility	
 compatibility_ignore_auto_increment_in_create_table	0
@@ -336,13 +329,9 @@ format_regexp_escaping_rule	Raw
 format_regexp_skip_unmatched	0
 format_schema	
 format_template_resultset	
-<<<<<<< HEAD:tests/queries/0_stateless/02995_baseline_24_7_2.tsv
 format_template_resultset_format	
 format_template_row	
 format_template_row_format	
-=======
-format_template_row	
->>>>>>> Fix tests:tests/queries/0_stateless/02995_baseline_23_12_1.tsv
 format_template_rows_between_delimiter	\n
 format_tsv_null_representation	\\N
 formatdatetime_f_prints_single_zero	0
@@ -399,11 +388,8 @@ iceberg_engine_ignore_schema_evolution	0
 idle_connection_timeout	3600
 ignore_cold_parts_seconds	0
 ignore_data_skipping_indices	
-<<<<<<< HEAD:tests/queries/0_stateless/02995_baseline_24_7_2.tsv
 ignore_drop_queries_probability	0
 ignore_materialized_views_with_dropped_target_table	0
-=======
->>>>>>> Fix tests:tests/queries/0_stateless/02995_baseline_23_12_1.tsv
 ignore_on_cluster_for_replicated_access_entities_queries	0
 ignore_on_cluster_for_replicated_named_collections_queries	0
 ignore_on_cluster_for_replicated_udf_queries	0
@@ -726,13 +712,8 @@ mutations_execute_subqueries_on_initiator	0
 mutations_max_literal_size_to_replace	16384
 mutations_sync	0
 mysql_datatypes_support_level	
-<<<<<<< HEAD:tests/queries/0_stateless/02995_baseline_24_7_2.tsv
 mysql_map_fixed_string_to_text_in_show_columns	1
 mysql_map_string_to_text_in_show_columns	1
-=======
-mysql_map_fixed_string_to_text_in_show_columns	0
-mysql_map_string_to_text_in_show_columns	0
->>>>>>> Fix tests:tests/queries/0_stateless/02995_baseline_23_12_1.tsv
 mysql_max_rows_to_insert	65536
 network_compression_method	LZ4
 network_zstd_compression_level	1
@@ -799,13 +780,9 @@ os_thread_priority	0
 output_format_arrow_compression_method	lz4_frame
 output_format_arrow_fixed_string_as_fixed_byte_array	1
 output_format_arrow_low_cardinality_as_dictionary	0
-<<<<<<< HEAD:tests/queries/0_stateless/02995_baseline_24_7_2.tsv
 output_format_arrow_string_as_string	1
 output_format_arrow_use_64_bit_indexes_for_dictionary	0
 output_format_arrow_use_signed_indexes_for_dictionary	1
-=======
-output_format_arrow_string_as_string	0
->>>>>>> Fix tests:tests/queries/0_stateless/02995_baseline_23_12_1.tsv
 output_format_avro_codec	
 output_format_avro_rows_in_file	1
 output_format_avro_string_column_pattern	
@@ -1068,10 +1045,7 @@ totals_mode	after_having_exclusive
 trace_profile_events	0
 transfer_overflow_mode	throw
 transform_null_in	0
-<<<<<<< HEAD:tests/queries/0_stateless/02995_baseline_24_7_2.tsv
 traverse_shadow_remote_data_paths	0
-=======
->>>>>>> Fix tests:tests/queries/0_stateless/02995_baseline_23_12_1.tsv
 union_default_mode	
 unknown_packet_in_send_data	0
 update_insert_deduplication_token_in_dependent_materialized_views	0

From d998bf55f1a592df93f6c839dd647ff437962076 Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <nikitamikhaylov@clickhouse.com>
Date: Fri, 2 Aug 2024 11:37:08 +0000
Subject: [PATCH 1407/1488] Fix build

---
 src/Core/SettingsChangesHistory.cpp | 263 +---------------------------
 1 file changed, 1 insertion(+), 262 deletions(-)

diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp
index 893394adb61..2ff392fcb84 100644
--- a/src/Core/SettingsChangesHistory.cpp
+++ b/src/Core/SettingsChangesHistory.cpp
@@ -78,6 +78,7 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
             {"input_format_json_max_depth", 1000000, 1000, "It was unlimited in previous versions, but that was unsafe."},
             {"merge_tree_min_bytes_per_task_for_remote_reading", 4194304, 2097152, "Value is unified with `filesystem_prefetch_min_bytes_for_single_read_task`"},
             {"allow_archive_path_syntax", true, true, "Added new setting to allow disabling archive path syntax."},
+            {"enable_analyzer", 1, 1, "Added an alias to a setting `allow_experimental_analyzer`."},
         }
     },
     {"24.7",
@@ -500,268 +501,6 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
             {"enable_optimize_predicate_expression", 0, 1, "Optimize predicates to subqueries by default"}
         }
     },
-    {"24.8", {{"enable_analyzer", false, true, "Move analyzer to the Beta stage."},
-    {"24.8", {{"enable_analyzer", true, true, "Added an alias to the `allow_experimental_analyzer` setting."}
-              }},
-    {"24.7", {{"output_format_parquet_write_page_index", false, true, "Add a possibility to write page index into parquet files."},
-              {"output_format_binary_encode_types_in_binary_format", false, false, "Added new setting to allow to write type names in binary format in RowBinaryWithNamesAndTypes output format"},
-              {"input_format_binary_decode_types_in_binary_format", false, false, "Added new setting to allow to read type names in binary format in RowBinaryWithNamesAndTypes input format"},
-              {"output_format_native_encode_types_in_binary_format", false, false, "Added new setting to allow to write type names in binary format in Native output format"},
-              {"input_format_native_decode_types_in_binary_format", false, false, "Added new setting to allow to read type names in binary format in Native output format"},
-              {"read_in_order_use_buffering", false, true, "Use buffering before merging while reading in order of primary key"},
-              {"enable_named_columns_in_function_tuple", false, true, "Generate named tuples in function tuple() when all names are unique and can be treated as unquoted identifiers."},
-              {"input_format_json_case_insensitive_column_matching", false, false, "Ignore case when matching JSON keys with CH columns."},
-              {"optimize_trivial_insert_select", true, false, "The optimization does not make sense in many cases."},
-              {"dictionary_validate_primary_key_type", false, false, "Validate primary key type for dictionaries. By default id type for simple layouts will be implicitly converted to UInt64."},
-              {"collect_hash_table_stats_during_joins", false, true, "New setting."},
-              {"max_size_to_preallocate_for_joins", 0, 100'000'000, "New setting."},
-              {"input_format_orc_reader_time_zone_name", "GMT", "GMT", "The time zone name for ORC row reader, the default ORC row reader's time zone is GMT."},
-              {"lightweight_mutation_projection_mode", "throw", "throw", "When lightweight delete happens on a table with projection(s), the possible operations include throw the exception as projection exists, or drop all projection related to this table then do lightweight delete."},
-              {"database_replicated_allow_heavy_create", true, false, "Long-running DDL queries (CREATE AS SELECT and POPULATE) for Replicated database engine was forbidden"},
-              {"query_plan_merge_filters", false, false, "Allow to merge filters in the query plan"},
-              {"azure_sdk_max_retries", 10, 10, "Maximum number of retries in azure sdk"},
-              {"azure_sdk_retry_initial_backoff_ms", 10, 10, "Minimal backoff between retries in azure sdk"},
-              {"azure_sdk_retry_max_backoff_ms", 1000, 1000, "Maximal backoff between retries in azure sdk"},
-              {"merge_tree_min_bytes_per_task_for_remote_reading", 4194304, 2097152, "Value is unified with `filesystem_prefetch_min_bytes_for_single_read_task`"},
-              {"ignore_on_cluster_for_replicated_named_collections_queries", false, false, "Ignore ON CLUSTER clause for replicated named collections management queries."},
-              {"backup_restore_s3_retry_attempts", 1000,1000, "Setting for Aws::Client::RetryStrategy, Aws::Client does retries itself, 0 means no retries. It takes place only for backup/restore."},
-              {"postgresql_connection_attempt_timeout", 2, 2, "Allow to control 'connect_timeout' parameter of PostgreSQL connection."},
-              {"postgresql_connection_pool_retries", 2, 2, "Allow to control the number of retries in PostgreSQL connection pool."}
-              }},
-    {"24.6", {{"materialize_skip_indexes_on_insert", true, true, "Added new setting to allow to disable materialization of skip indexes on insert"},
-              {"materialize_statistics_on_insert", true, true, "Added new setting to allow to disable materialization of statistics on insert"},
-              {"input_format_parquet_use_native_reader", false, false, "When reading Parquet files, to use native reader instead of arrow reader."},
-              {"hdfs_throw_on_zero_files_match", false, false, "Allow to throw an error when ListObjects request cannot match any files in HDFS engine instead of empty query result"},
-              {"azure_throw_on_zero_files_match", false, false, "Allow to throw an error when ListObjects request cannot match any files in AzureBlobStorage engine instead of empty query result"},
-              {"s3_validate_request_settings", true, true, "Allow to disable S3 request settings validation"},
-              {"allow_experimental_full_text_index", false, false, "Enable experimental full-text index"},
-              {"azure_skip_empty_files", false, false, "Allow to skip empty files in azure table engine"},
-              {"hdfs_ignore_file_doesnt_exist", false, false, "Allow to return 0 rows when the requested files don't exist instead of throwing an exception in HDFS table engine"},
-              {"azure_ignore_file_doesnt_exist", false, false, "Allow to return 0 rows when the requested files don't exist instead of throwing an exception in AzureBlobStorage table engine"},
-              {"s3_ignore_file_doesnt_exist", false, false, "Allow to return 0 rows when the requested files don't exist instead of throwing an exception in S3 table engine"},
-              {"s3_max_part_number", 10000, 10000, "Maximum part number number for s3 upload part"},
-              {"s3_max_single_operation_copy_size", 32 * 1024 * 1024, 32 * 1024 * 1024, "Maximum size for a single copy operation in s3"},
-              {"input_format_parquet_max_block_size", 8192, DEFAULT_BLOCK_SIZE, "Increase block size for parquet reader."},
-              {"input_format_parquet_prefer_block_bytes", 0, DEFAULT_BLOCK_SIZE * 256, "Average block bytes output by parquet reader."},
-              {"enable_blob_storage_log", true, true, "Write information about blob storage operations to system.blob_storage_log table"},
-              {"allow_deprecated_snowflake_conversion_functions", true, false, "Disabled deprecated functions snowflakeToDateTime[64] and dateTime[64]ToSnowflake."},
-              {"allow_statistic_optimize", false, false, "Old setting which popped up here being renamed."},
-              {"allow_experimental_statistic", false, false, "Old setting which popped up here being renamed."},
-              {"allow_statistics_optimize", false, false, "The setting was renamed. The previous name is `allow_statistic_optimize`."},
-              {"allow_experimental_statistics", false, false, "The setting was renamed. The previous name is `allow_experimental_statistic`."},
-              {"enable_vertical_final", false, true, "Enable vertical final by default again after fixing bug"},
-              {"parallel_replicas_custom_key_range_lower", 0, 0, "Add settings to control the range filter when using parallel replicas with dynamic shards"},
-              {"parallel_replicas_custom_key_range_upper", 0, 0, "Add settings to control the range filter when using parallel replicas with dynamic shards. A value of 0 disables the upper limit"},
-              {"output_format_pretty_display_footer_column_names", 0, 1, "Add a setting to display column names in the footer if there are many rows. Threshold value is controlled by output_format_pretty_display_footer_column_names_min_rows."},
-              {"output_format_pretty_display_footer_column_names_min_rows", 0, 50, "Add a setting to control the threshold value for setting output_format_pretty_display_footer_column_names_min_rows. Default 50."},
-              {"output_format_csv_serialize_tuple_into_separate_columns", true, true, "A new way of how interpret tuples in CSV format was added."},
-              {"input_format_csv_deserialize_separate_columns_into_tuple", true, true, "A new way of how interpret tuples in CSV format was added."},
-              {"input_format_csv_try_infer_strings_from_quoted_tuples", true, true, "A new way of how interpret tuples in CSV format was added."},
-              }},
-    {"24.5", {{"allow_deprecated_error_prone_window_functions", true, false, "Allow usage of deprecated error prone window functions (neighbor, runningAccumulate, runningDifferenceStartingWithFirstValue, runningDifference)"},
-              {"allow_experimental_join_condition", false, false, "Support join with inequal conditions which involve columns from both left and right table. e.g. t1.y < t2.y."},
-              {"input_format_tsv_crlf_end_of_line", false, false, "Enables reading of CRLF line endings with TSV formats"},
-              {"output_format_parquet_use_custom_encoder", false, true, "Enable custom Parquet encoder."},
-              {"cross_join_min_rows_to_compress", 0, 10000000, "Minimal count of rows to compress block in CROSS JOIN. Zero value means - disable this threshold. This block is compressed when any of the two thresholds (by rows or by bytes) are reached."},
-              {"cross_join_min_bytes_to_compress", 0, 1_GiB, "Minimal size of block to compress in CROSS JOIN. Zero value means - disable this threshold. This block is compressed when any of the two thresholds (by rows or by bytes) are reached."},
-              {"http_max_chunk_size", 0, 0, "Internal limitation"},
-              {"prefer_external_sort_block_bytes", 0, DEFAULT_BLOCK_SIZE * 256, "Prefer maximum block bytes for external sort, reduce the memory usage during merging."},
-              {"input_format_force_null_for_omitted_fields", false, false, "Disable type-defaults for omitted fields when needed"},
-              {"cast_string_to_dynamic_use_inference", false, false, "Add setting to allow converting String to Dynamic through parsing"},
-              {"allow_experimental_dynamic_type", false, false, "Add new experimental Dynamic type"},
-              {"azure_max_blocks_in_multipart_upload", 50000, 50000, "Maximum number of blocks in multipart upload for Azure."},
-              }},
-    {"24.4", {{"input_format_json_throw_on_bad_escape_sequence", true, true, "Allow to save JSON strings with bad escape sequences"},
-              {"max_parsing_threads", 0, 0, "Add a separate setting to control number of threads in parallel parsing from files"},
-              {"ignore_drop_queries_probability", 0, 0, "Allow to ignore drop queries in server with specified probability for testing purposes"},
-              {"lightweight_deletes_sync", 2, 2, "The same as 'mutation_sync', but controls only execution of lightweight deletes"},
-              {"query_cache_system_table_handling", "save", "throw", "The query cache no longer caches results of queries against system tables"},
-              {"input_format_json_ignore_unnecessary_fields", false, true, "Ignore unnecessary fields and not parse them. Enabling this may not throw exceptions on json strings of invalid format or with duplicated fields"},
-              {"input_format_hive_text_allow_variable_number_of_columns", false, true, "Ignore extra columns in Hive Text input (if file has more columns than expected) and treat missing fields in Hive Text input as default values."},
-              {"allow_experimental_database_replicated", false, true, "Database engine Replicated is now in Beta stage"},
-              {"temporary_data_in_cache_reserve_space_wait_lock_timeout_milliseconds", (10 * 60 * 1000), (10 * 60 * 1000), "Wait time to lock cache for sapce reservation in temporary data in filesystem cache"},
-              {"optimize_rewrite_sum_if_to_count_if", false, true, "Only available for the analyzer, where it works correctly"},
-              {"azure_allow_parallel_part_upload", "true", "true", "Use multiple threads for azure multipart upload."},
-              {"max_recursive_cte_evaluation_depth", DBMS_RECURSIVE_CTE_MAX_EVALUATION_DEPTH, DBMS_RECURSIVE_CTE_MAX_EVALUATION_DEPTH, "Maximum limit on recursive CTE evaluation depth"},
-              {"query_plan_convert_outer_join_to_inner_join", false, true, "Allow to convert OUTER JOIN to INNER JOIN if filter after JOIN always filters default values"},
-              }},
-    {"24.3", {{"s3_connect_timeout_ms", 1000, 1000, "Introduce new dedicated setting for s3 connection timeout"},
-              {"allow_experimental_shared_merge_tree", false, true, "The setting is obsolete"},
-              {"use_page_cache_for_disks_without_file_cache", false, false, "Added userspace page cache"},
-              {"read_from_page_cache_if_exists_otherwise_bypass_cache", false, false, "Added userspace page cache"},
-              {"page_cache_inject_eviction", false, false, "Added userspace page cache"},
-              {"default_table_engine", "None", "MergeTree", "Set default table engine to MergeTree for better usability"},
-              {"input_format_json_use_string_type_for_ambiguous_paths_in_named_tuples_inference_from_objects", false, false, "Allow to use String type for ambiguous paths during named tuple inference from JSON objects"},
-              {"traverse_shadow_remote_data_paths", false, false, "Traverse shadow directory when query system.remote_data_paths."},
-              {"throw_if_deduplication_in_dependent_materialized_views_enabled_with_async_insert", false, true, "Deduplication in dependent materialized view cannot work together with async inserts."},
-              {"parallel_replicas_allow_in_with_subquery", false, true, "If true, subquery for IN will be executed on every follower replica"},
-              {"log_processors_profiles", false, true, "Enable by default"},
-              {"function_locate_has_mysql_compatible_argument_order", false, true, "Increase compatibility with MySQL's locate function."},
-              {"allow_suspicious_primary_key", true, false, "Forbid suspicious PRIMARY KEY/ORDER BY for MergeTree (i.e. SimpleAggregateFunction)"},
-              {"filesystem_cache_reserve_space_wait_lock_timeout_milliseconds", 1000, 1000, "Wait time to lock cache for sapce reservation in filesystem cache"},
-              {"max_parser_backtracks", 0, 1000000, "Limiting the complexity of parsing"},
-              {"analyzer_compatibility_join_using_top_level_identifier", false, false, "Force to resolve identifier in JOIN USING from projection"},
-              {"distributed_insert_skip_read_only_replicas", false, false, "If true, INSERT into Distributed will skip read-only replicas"},
-              {"keeper_max_retries", 10, 10, "Max retries for general keeper operations"},
-              {"keeper_retry_initial_backoff_ms", 100, 100, "Initial backoff timeout for general keeper operations"},
-              {"keeper_retry_max_backoff_ms", 5000, 5000, "Max backoff timeout for general keeper operations"},
-              {"s3queue_allow_experimental_sharded_mode", false, false, "Enable experimental sharded mode of S3Queue table engine. It is experimental because it will be rewritten"},
-              {"allow_experimental_analyzer", false, true, "Enable analyzer and planner by default."},
-              {"merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability", 0.0, 0.0, "For testing of `PartsSplitter` - split read ranges into intersecting and non intersecting every time you read from MergeTree with the specified probability."},
-              {"allow_get_client_http_header", false, false, "Introduced a new function."},
-              {"output_format_pretty_row_numbers", false, true, "It is better for usability."},
-              {"output_format_pretty_max_value_width_apply_for_single_value", true, false, "Single values in Pretty formats won't be cut."},
-              {"output_format_parquet_string_as_string", false, true, "ClickHouse allows arbitrary binary data in the String data type, which is typically UTF-8. Parquet/ORC/Arrow Strings only support UTF-8. That's why you can choose which Arrow's data type to use for the ClickHouse String data type - String or Binary. While Binary would be more correct and compatible, using String by default will correspond to user expectations in most cases."},
-              {"output_format_orc_string_as_string", false, true, "ClickHouse allows arbitrary binary data in the String data type, which is typically UTF-8. Parquet/ORC/Arrow Strings only support UTF-8. That's why you can choose which Arrow's data type to use for the ClickHouse String data type - String or Binary. While Binary would be more correct and compatible, using String by default will correspond to user expectations in most cases."},
-              {"output_format_arrow_string_as_string", false, true, "ClickHouse allows arbitrary binary data in the String data type, which is typically UTF-8. Parquet/ORC/Arrow Strings only support UTF-8. That's why you can choose which Arrow's data type to use for the ClickHouse String data type - String or Binary. While Binary would be more correct and compatible, using String by default will correspond to user expectations in most cases."},
-              {"output_format_parquet_compression_method", "lz4", "zstd", "Parquet/ORC/Arrow support many compression methods, including lz4 and zstd. ClickHouse supports each and every compression method. Some inferior tools, such as 'duckdb', lack support for the faster `lz4` compression method, that's why we set zstd by default."},
-              {"output_format_orc_compression_method", "lz4", "zstd", "Parquet/ORC/Arrow support many compression methods, including lz4 and zstd. ClickHouse supports each and every compression method. Some inferior tools, such as 'duckdb', lack support for the faster `lz4` compression method, that's why we set zstd by default."},
-              {"output_format_pretty_highlight_digit_groups", false, true, "If enabled and if output is a terminal, highlight every digit corresponding to the number of thousands, millions, etc. with underline."},
-              {"geo_distance_returns_float64_on_float64_arguments", false, true, "Increase the default precision."},
-              {"azure_max_inflight_parts_for_one_file", 20, 20, "The maximum number of a concurrent loaded parts in multipart upload request. 0 means unlimited."},
-              {"azure_strict_upload_part_size", 0, 0, "The exact size of part to upload during multipart upload to Azure blob storage."},
-              {"azure_min_upload_part_size", 16*1024*1024, 16*1024*1024, "The minimum size of part to upload during multipart upload to Azure blob storage."},
-              {"azure_max_upload_part_size", 5ull*1024*1024*1024, 5ull*1024*1024*1024, "The maximum size of part to upload during multipart upload to Azure blob storage."},
-              {"azure_upload_part_size_multiply_factor", 2, 2, "Multiply azure_min_upload_part_size by this factor each time azure_multiply_parts_count_threshold parts were uploaded from a single write to Azure blob storage."},
-              {"azure_upload_part_size_multiply_parts_count_threshold", 500, 500, "Each time this number of parts was uploaded to Azure blob storage, azure_min_upload_part_size is multiplied by azure_upload_part_size_multiply_factor."},
-              {"output_format_csv_serialize_tuple_into_separate_columns", true, true, "A new way of how interpret tuples in CSV format was added."},
-              {"input_format_csv_deserialize_separate_columns_into_tuple", true, true, "A new way of how interpret tuples in CSV format was added."},
-              {"input_format_csv_try_infer_strings_from_quoted_tuples", true, true, "A new way of how interpret tuples in CSV format was added."},
-              }},
-    {"24.2", {{"allow_suspicious_variant_types", true, false, "Don't allow creating Variant type with suspicious variants by default"},
-              {"validate_experimental_and_suspicious_types_inside_nested_types", false, true, "Validate usage of experimental and suspicious types inside nested types"},
-              {"output_format_values_escape_quote_with_quote", false, false, "If true escape ' with '', otherwise quoted with \\'"},
-              {"output_format_pretty_single_large_number_tip_threshold", 0, 1'000'000, "Print a readable number tip on the right side of the table if the block consists of a single number which exceeds this value (except 0)"},
-              {"input_format_try_infer_exponent_floats", true, false, "Don't infer floats in exponential notation by default"},
-              {"query_plan_optimize_prewhere", true, true, "Allow to push down filter to PREWHERE expression for supported storages"},
-              {"async_insert_max_data_size", 1000000, 10485760, "The previous value appeared to be too small."},
-              {"async_insert_poll_timeout_ms", 10, 10, "Timeout in milliseconds for polling data from asynchronous insert queue"},
-              {"async_insert_use_adaptive_busy_timeout", false, true, "Use adaptive asynchronous insert timeout"},
-              {"async_insert_busy_timeout_min_ms", 50, 50, "The minimum value of the asynchronous insert timeout in milliseconds; it also serves as the initial value, which may be increased later by the adaptive algorithm"},
-              {"async_insert_busy_timeout_max_ms", 200, 200, "The minimum value of the asynchronous insert timeout in milliseconds; async_insert_busy_timeout_ms is aliased to async_insert_busy_timeout_max_ms"},
-              {"async_insert_busy_timeout_increase_rate", 0.2, 0.2, "The exponential growth rate at which the adaptive asynchronous insert timeout increases"},
-              {"async_insert_busy_timeout_decrease_rate", 0.2, 0.2, "The exponential growth rate at which the adaptive asynchronous insert timeout decreases"},
-              {"format_template_row_format", "", "", "Template row format string can be set directly in query"},
-              {"format_template_resultset_format", "", "", "Template result set format string can be set in query"},
-              {"split_parts_ranges_into_intersecting_and_non_intersecting_final", true, true, "Allow to split parts ranges into intersecting and non intersecting during FINAL optimization"},
-              {"split_intersecting_parts_ranges_into_layers_final", true, true, "Allow to split intersecting parts ranges into layers during FINAL optimization"},
-              {"azure_max_single_part_copy_size", 256*1024*1024, 256*1024*1024, "The maximum size of object to copy using single part copy to Azure blob storage."},
-              {"min_external_table_block_size_rows", DEFAULT_INSERT_BLOCK_SIZE, DEFAULT_INSERT_BLOCK_SIZE, "Squash blocks passed to external table to specified size in rows, if blocks are not big enough"},
-              {"min_external_table_block_size_bytes", DEFAULT_INSERT_BLOCK_SIZE * 256, DEFAULT_INSERT_BLOCK_SIZE * 256, "Squash blocks passed to external table to specified size in bytes, if blocks are not big enough."},
-              {"parallel_replicas_prefer_local_join", true, true, "If true, and JOIN can be executed with parallel replicas algorithm, and all storages of right JOIN part are *MergeTree, local JOIN will be used instead of GLOBAL JOIN."},
-              {"optimize_time_filter_with_preimage", true, true, "Optimize Date and DateTime predicates by converting functions into equivalent comparisons without conversions (e.g. toYear(col) = 2023 -> col >= '2023-01-01' AND col <= '2023-12-31')"},
-              {"extract_key_value_pairs_max_pairs_per_row", 0, 0, "Max number of pairs that can be produced by the `extractKeyValuePairs` function. Used as a safeguard against consuming too much memory."},
-              {"default_view_definer", "CURRENT_USER", "CURRENT_USER", "Allows to set default `DEFINER` option while creating a view"},
-              {"default_materialized_view_sql_security", "DEFINER", "DEFINER", "Allows to set a default value for SQL SECURITY option when creating a materialized view"},
-              {"default_normal_view_sql_security", "INVOKER", "INVOKER", "Allows to set default `SQL SECURITY` option while creating a normal view"},
-              {"mysql_map_string_to_text_in_show_columns", false, true, "Reduce the configuration effort to connect ClickHouse with BI tools."},
-              {"mysql_map_fixed_string_to_text_in_show_columns", false, true, "Reduce the configuration effort to connect ClickHouse with BI tools."},
-              }},
-    {"24.1", {{"print_pretty_type_names", false, true, "Better user experience."},
-              {"input_format_json_read_bools_as_strings", false, true, "Allow to read bools as strings in JSON formats by default"},
-              {"output_format_arrow_use_signed_indexes_for_dictionary", false, true, "Use signed indexes type for Arrow dictionaries by default as it's recommended"},
-              {"allow_experimental_variant_type", false, false, "Add new experimental Variant type"},
-              {"use_variant_as_common_type", false, false, "Allow to use Variant in if/multiIf if there is no common type"},
-              {"output_format_arrow_use_64_bit_indexes_for_dictionary", false, false, "Allow to use 64 bit indexes type in Arrow dictionaries"},
-              {"parallel_replicas_mark_segment_size", 128, 128, "Add new setting to control segment size in new parallel replicas coordinator implementation"},
-              {"ignore_materialized_views_with_dropped_target_table", false, false, "Add new setting to allow to ignore materialized views with dropped target table"},
-              {"output_format_compression_level", 3, 3, "Allow to change compression level in the query output"},
-              {"output_format_compression_zstd_window_log", 0, 0, "Allow to change zstd window log in the query output when zstd compression is used"},
-              {"enable_zstd_qat_codec", false, false, "Add new ZSTD_QAT codec"},
-              {"enable_vertical_final", false, true, "Use vertical final by default"},
-              {"output_format_arrow_use_64_bit_indexes_for_dictionary", false, false, "Allow to use 64 bit indexes type in Arrow dictionaries"},
-              {"max_rows_in_set_to_optimize_join", 100000, 0, "Disable join optimization as it prevents from read in order optimization"},
-              {"output_format_pretty_color", true, "auto", "Setting is changed to allow also for auto value, disabling ANSI escapes if output is not a tty"},
-              {"function_visible_width_behavior", 0, 1, "We changed the default behavior of `visibleWidth` to be more precise"},
-              {"max_estimated_execution_time", 0, 0, "Separate max_execution_time and max_estimated_execution_time"},
-              {"iceberg_engine_ignore_schema_evolution", false, false, "Allow to ignore schema evolution in Iceberg table engine"},
-              {"optimize_injective_functions_in_group_by", false, true, "Replace injective functions by it's arguments in GROUP BY section in analyzer"},
-              {"update_insert_deduplication_token_in_dependent_materialized_views", false, false, "Allow to update insert deduplication token with table identifier during insert in dependent materialized views"},
-              {"azure_max_unexpected_write_error_retries", 4, 4, "The maximum number of retries in case of unexpected errors during Azure blob storage write"},
-              {"split_parts_ranges_into_intersecting_and_non_intersecting_final", false, true, "Allow to split parts ranges into intersecting and non intersecting during FINAL optimization"},
-              {"split_intersecting_parts_ranges_into_layers_final", true, true, "Allow to split intersecting parts ranges into layers during FINAL optimization"}}},
-    {"23.12", {{"allow_suspicious_ttl_expressions", true, false, "It is a new setting, and in previous versions the behavior was equivalent to allowing."},
-              {"input_format_parquet_allow_missing_columns", false, true, "Allow missing columns in Parquet files by default"},
-              {"input_format_orc_allow_missing_columns", false, true, "Allow missing columns in ORC files by default"},
-              {"input_format_arrow_allow_missing_columns", false, true, "Allow missing columns in Arrow files by default"}}},
-    {"23.11", {{"parsedatetime_parse_without_leading_zeros", false, true, "Improved compatibility with MySQL DATE_FORMAT/STR_TO_DATE"}}},
-    {"23.9", {{"optimize_group_by_constant_keys", false, true, "Optimize group by constant keys by default"},
-              {"input_format_json_try_infer_named_tuples_from_objects", false, true, "Try to infer named Tuples from JSON objects by default"},
-              {"input_format_json_read_numbers_as_strings", false, true, "Allow to read numbers as strings in JSON formats by default"},
-              {"input_format_json_read_arrays_as_strings", false, true, "Allow to read arrays as strings in JSON formats by default"},
-              {"input_format_json_infer_incomplete_types_as_strings", false, true, "Allow to infer incomplete types as Strings in JSON formats by default"},
-              {"input_format_json_try_infer_numbers_from_strings", true, false, "Don't infer numbers from strings in JSON formats by default to prevent possible parsing errors"},
-              {"http_write_exception_in_output_format", false, true, "Output valid JSON/XML on exception in HTTP streaming."}}},
-    {"23.8", {{"rewrite_count_distinct_if_with_count_distinct_implementation", false, true, "Rewrite countDistinctIf with count_distinct_implementation configuration"}}},
-    {"23.7", {{"function_sleep_max_microseconds_per_block", 0, 3000000, "In previous versions, the maximum sleep time of 3 seconds was applied only for `sleep`, but not for `sleepEachRow` function. In the new version, we introduce this setting. If you set compatibility with the previous versions, we will disable the limit altogether."}}},
-    {"23.6", {{"http_send_timeout", 180, 30, "3 minutes seems crazy long. Note that this is timeout for a single network write call, not for the whole upload operation."},
-              {"http_receive_timeout", 180, 30, "See http_send_timeout."}}},
-    {"23.5", {{"input_format_parquet_preserve_order", true, false, "Allow Parquet reader to reorder rows for better parallelism."},
-              {"parallelize_output_from_storages", false, true, "Allow parallelism when executing queries that read from file/url/s3/etc. This may reorder rows."},
-              {"use_with_fill_by_sorting_prefix", false, true, "Columns preceding WITH FILL columns in ORDER BY clause form sorting prefix. Rows with different values in sorting prefix are filled independently"},
-              {"output_format_parquet_compliant_nested_types", false, true, "Change an internal field name in output Parquet file schema."}}},
-    {"23.4", {{"allow_suspicious_indices", true, false, "If true, index can defined with identical expressions"},
-              {"allow_nonconst_timezone_arguments", true, false, "Allow non-const timezone arguments in certain time-related functions like toTimeZone(), fromUnixTimestamp*(), snowflakeToDateTime*()."},
-              {"connect_timeout_with_failover_ms", 50, 1000, "Increase default connect timeout because of async connect"},
-              {"connect_timeout_with_failover_secure_ms", 100, 1000, "Increase default secure connect timeout because of async connect"},
-              {"hedged_connection_timeout_ms", 100, 50, "Start new connection in hedged requests after 50 ms instead of 100 to correspond with previous connect timeout"},
-              {"formatdatetime_f_prints_single_zero", true, false, "Improved compatibility with MySQL DATE_FORMAT()/STR_TO_DATE()"},
-              {"formatdatetime_parsedatetime_m_is_month_name", false, true, "Improved compatibility with MySQL DATE_FORMAT/STR_TO_DATE"}}},
-    {"23.3", {{"output_format_parquet_version", "1.0", "2.latest", "Use latest Parquet format version for output format"},
-              {"input_format_json_ignore_unknown_keys_in_named_tuple", false, true, "Improve parsing JSON objects as named tuples"},
-              {"input_format_native_allow_types_conversion", false, true, "Allow types conversion in Native input forma"},
-              {"output_format_arrow_compression_method", "none", "lz4_frame", "Use lz4 compression in Arrow output format by default"},
-              {"output_format_parquet_compression_method", "snappy", "lz4", "Use lz4 compression in Parquet output format by default"},
-              {"output_format_orc_compression_method", "none", "lz4_frame", "Use lz4 compression in ORC output format by default"},
-              {"async_query_sending_for_remote", false, true, "Create connections and send query async across shards"}}},
-    {"23.2", {{"output_format_parquet_fixed_string_as_fixed_byte_array", false, true, "Use Parquet FIXED_LENGTH_BYTE_ARRAY type for FixedString by default"},
-              {"output_format_arrow_fixed_string_as_fixed_byte_array", false, true, "Use Arrow FIXED_SIZE_BINARY type for FixedString by default"},
-              {"query_plan_remove_redundant_distinct", false, true, "Remove redundant Distinct step in query plan"},
-              {"optimize_duplicate_order_by_and_distinct", true, false, "Remove duplicate ORDER BY and DISTINCT if it's possible"},
-              {"insert_keeper_max_retries", 0, 20, "Enable reconnections to Keeper on INSERT, improve reliability"}}},
-    {"23.1", {{"input_format_json_read_objects_as_strings", 0, 1, "Enable reading nested json objects as strings while object type is experimental"},
-              {"input_format_json_defaults_for_missing_elements_in_named_tuple", false, true, "Allow missing elements in JSON objects while reading named tuples by default"},
-              {"input_format_csv_detect_header", false, true, "Detect header in CSV format by default"},
-              {"input_format_tsv_detect_header", false, true, "Detect header in TSV format by default"},
-              {"input_format_custom_detect_header", false, true, "Detect header in CustomSeparated format by default"},
-              {"query_plan_remove_redundant_sorting", false, true, "Remove redundant sorting in query plan. For example, sorting steps related to ORDER BY clauses in subqueries"}}},
-    {"22.12", {{"max_size_to_preallocate_for_aggregation", 10'000'000, 100'000'000, "This optimizes performance"},
-               {"query_plan_aggregation_in_order", 0, 1, "Enable some refactoring around query plan"},
-               {"format_binary_max_string_size", 0, 1_GiB, "Prevent allocating large amount of memory"}}},
-    {"22.11", {{"use_structure_from_insertion_table_in_table_functions", 0, 2, "Improve using structure from insertion table in table functions"}}},
-    {"22.9", {{"force_grouping_standard_compatibility", false, true, "Make GROUPING function output the same as in SQL standard and other DBMS"}}},
-    {"22.7", {{"cross_to_inner_join_rewrite", 1, 2, "Force rewrite comma join to inner"},
-              {"enable_positional_arguments", false, true, "Enable positional arguments feature by default"},
-              {"format_csv_allow_single_quotes", true, false, "Most tools don't treat single quote in CSV specially, don't do it by default too"}}},
-    {"22.6", {{"output_format_json_named_tuples_as_objects", false, true, "Allow to serialize named tuples as JSON objects in JSON formats by default"},
-              {"input_format_skip_unknown_fields", false, true, "Optimize reading subset of columns for some input formats"}}},
-    {"22.5", {{"memory_overcommit_ratio_denominator", 0, 1073741824, "Enable memory overcommit feature by default"},
-              {"memory_overcommit_ratio_denominator_for_user", 0, 1073741824, "Enable memory overcommit feature by default"}}},
-    {"22.4", {{"allow_settings_after_format_in_insert", true, false, "Do not allow SETTINGS after FORMAT for INSERT queries because ClickHouse interpret SETTINGS as some values, which is misleading"}}},
-    {"22.3", {{"cast_ipv4_ipv6_default_on_conversion_error", true, false, "Make functions cast(value, 'IPv4') and cast(value, 'IPv6') behave same as toIPv4 and toIPv6 functions"}}},
-    {"21.12", {{"stream_like_engine_allow_direct_select", true, false, "Do not allow direct select for Kafka/RabbitMQ/FileLog by default"}}},
-    {"21.9", {{"output_format_decimal_trailing_zeros", true, false, "Do not output trailing zeros in text representation of Decimal types by default for better looking output"},
-              {"use_hedged_requests", false, true, "Enable Hedged Requests feature by default"}}},
-    {"21.7", {{"legacy_column_name_of_tuple_literal", true, false, "Add this setting only for compatibility reasons. It makes sense to set to 'true', while doing rolling update of cluster from version lower than 21.7 to higher"}}},
-    {"21.5", {{"async_socket_for_remote", false, true, "Fix all problems and turn on asynchronous reads from socket for remote queries by default again"}}},
-    {"21.3", {{"async_socket_for_remote", true, false, "Turn off asynchronous reads from socket for remote queries because of some problems"},
-              {"optimize_normalize_count_variants", false, true, "Rewrite aggregate functions that semantically equals to count() as count() by default"},
-              {"normalize_function_names", false, true, "Normalize function names to their canonical names, this was needed for projection query routing"}}},
-    {"21.2", {{"enable_global_with_statement", false, true, "Propagate WITH statements to UNION queries and all subqueries by default"}}},
-    {"21.1", {{"insert_quorum_parallel", false, true, "Use parallel quorum inserts by default. It is significantly more convenient to use than sequential quorum inserts"},
-              {"input_format_null_as_default", false, true, "Allow to insert NULL as default for input formats by default"},
-              {"optimize_on_insert", false, true, "Enable data optimization on INSERT by default for better user experience"},
-              {"use_compact_format_in_distributed_parts_names", false, true, "Use compact format for async INSERT into Distributed tables by default"}}},
-    {"20.10", {{"format_regexp_escaping_rule", "Escaped", "Raw", "Use Raw as default escaping rule for Regexp format to male the behaviour more like to what users expect"}}},
-    {"20.7", {{"show_table_uuid_in_table_create_query_if_not_nil", true, false, "Stop showing  UID of the table in its CREATE query for Engine=Atomic"}}},
-    {"20.5", {{"input_format_with_names_use_header", false, true, "Enable using header with names for formats with WithNames/WithNamesAndTypes suffixes"},
-              {"allow_suspicious_codecs", true, false, "Don't allow to specify meaningless compression codecs"}}},
-    {"20.4", {{"validate_polygons", false, true, "Throw exception if polygon is invalid in function pointInPolygon by default instead of returning possibly wrong results"}}},
-    {"19.18", {{"enable_scalar_subquery_optimization", false, true, "Prevent scalar subqueries from (de)serializing large scalar values and possibly avoid running the same subquery more than once"}}},
-    {"19.14", {{"any_join_distinct_right_table_keys", true, false, "Disable ANY RIGHT and ANY FULL JOINs by default to avoid inconsistency"}}},
-    {"19.12", {{"input_format_defaults_for_omitted_fields", false, true, "Enable calculation of complex default expressions for omitted fields for some input formats, because it should be the expected behaviour"}}},
-    {"19.5", {{"max_partitions_per_insert_block", 0, 100, "Add a limit for the number of partitions in one block"}}},
-    {"18.12.17", {{"enable_optimize_predicate_expression", 0, 1, "Optimize predicates to subqueries by default"}}},
 };
 
 

From 011910a59409b3b2ec5430097d88c40e091c6b30 Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <nikitamikhaylov@clickhouse.com>
Date: Mon, 5 Aug 2024 15:28:43 +0000
Subject: [PATCH 1408/1488] Fixed the integration test

---
 ...allel_replicas_crash_after_refactoring.sql | 33 +++++++++++++++++++
 1 file changed, 33 insertions(+)
 create mode 100644 tests/queries/0_stateless/03215_parallel_replicas_crash_after_refactoring.sql

diff --git a/tests/queries/0_stateless/03215_parallel_replicas_crash_after_refactoring.sql b/tests/queries/0_stateless/03215_parallel_replicas_crash_after_refactoring.sql
new file mode 100644
index 00000000000..cae4fa0f0df
--- /dev/null
+++ b/tests/queries/0_stateless/03215_parallel_replicas_crash_after_refactoring.sql
@@ -0,0 +1,33 @@
+-- Tags: disabled
+
+DROP TABLE IF EXISTS t1__fuzz_5;
+
+CREATE TABLE t1__fuzz_5
+(
+    `k` Int16,
+    `v` Nullable(UInt8)
+)
+ENGINE = MergeTree
+ORDER BY k
+SETTINGS index_granularity = 10;
+
+INSERT INTO t1__fuzz_5 SELECT
+    number,
+    number
+FROM numbers(1000);
+
+INSERT INTO t1__fuzz_5 SELECT
+    number,
+    number
+FROM numbers(1000, 1000);
+
+INSERT INTO t1__fuzz_5 SELECT
+    number,
+    number
+FROM numbers(2000, 1000);
+
+SET receive_timeout = 10., receive_data_timeout_ms = 10000, allow_suspicious_low_cardinality_types = true, parallel_distributed_insert_select = 2, log_queries = true, table_function_remote_max_addresses = 200, max_execution_time = 10., max_memory_usage = 10000000000, log_comment = '/workspace/ch/tests/queries/0_stateless/02869_parallel_replicas_read_from_several.sql', send_logs_level = 'warning', prefer_localhost_replica = false, allow_introspection_functions = true, use_parallel_replicas = 257, max_parallel_replicas = 65535, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_single_task_marks_count_multiplier = -0., parallel_replicas_for_non_replicated_merge_tree = true;
+
+SELECT max(k) IGNORE NULLS FROM t1__fuzz_5 WITH TOTALS SETTINGS use_parallel_replicas = 257, max_parallel_replicas = 65535, prefer_localhost_replica = 0, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_single_task_marks_count_multiplier = -0;
+
+DROP TABLE IF EXISTS t1__fuzz_5;

From e4c71aec4dfa40d7e9b75c626b0229a33c59f315 Mon Sep 17 00:00:00 2001
From: robot-clickhouse <robot-clickhouse@users.noreply.github.com>
Date: Mon, 5 Aug 2024 16:12:25 +0000
Subject: [PATCH 1409/1488] Update version_date.tsv and changelogs after
 v24.5.5.78-stable

---
 docs/changelogs/v24.5.5.78-stable.md | 55 ++++++++++++++++++++++++++++
 utils/list-versions/version_date.tsv |  1 +
 2 files changed, 56 insertions(+)
 create mode 100644 docs/changelogs/v24.5.5.78-stable.md

diff --git a/docs/changelogs/v24.5.5.78-stable.md b/docs/changelogs/v24.5.5.78-stable.md
new file mode 100644
index 00000000000..415ea165101
--- /dev/null
+++ b/docs/changelogs/v24.5.5.78-stable.md
@@ -0,0 +1,55 @@
+---
+sidebar_position: 1
+sidebar_label: 2024
+---
+
+# 2024 Changelog
+
+### ClickHouse release v24.5.5.78-stable (0138248cb62) FIXME as compared to v24.5.4.49-stable (63b760955a0)
+
+#### Improvement
+* Backported in [#66768](https://github.com/ClickHouse/ClickHouse/issues/66768): Make allow_experimental_analyzer be controlled by the initiator for distributed queries. This ensures compatibility and correctness during operations in mixed version clusters. [#65777](https://github.com/ClickHouse/ClickHouse/pull/65777) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+
+#### Critical Bug Fix (crash, LOGICAL_ERROR, data loss, RBAC)
+* Backported in [#66884](https://github.com/ClickHouse/ClickHouse/issues/66884): Fix unexpeced size of low cardinality column in function calls. [#65298](https://github.com/ClickHouse/ClickHouse/pull/65298) ([Raúl Marín](https://github.com/Algunenano)).
+* Backported in [#66691](https://github.com/ClickHouse/ClickHouse/issues/66691): Fix the VALID UNTIL clause in the user definition resetting after a restart. Closes [#66405](https://github.com/ClickHouse/ClickHouse/issues/66405). [#66409](https://github.com/ClickHouse/ClickHouse/pull/66409) ([Nikolay Degterinsky](https://github.com/evillique)).
+* Backported in [#67814](https://github.com/ClickHouse/ClickHouse/issues/67814): Only relevant to the experimental Variant data type. Fix crash with Variant + AggregateFunction type. [#67122](https://github.com/ClickHouse/ClickHouse/pull/67122) ([Kruglov Pavel](https://github.com/Avogar)).
+* Backported in [#67501](https://github.com/ClickHouse/ClickHouse/issues/67501): Fix crash in DistributedAsyncInsert when connection is empty. [#67219](https://github.com/ClickHouse/ClickHouse/pull/67219) ([Pablo Marcos](https://github.com/pamarcos)).
+* Backported in [#67850](https://github.com/ClickHouse/ClickHouse/issues/67850): Fixes [#66026](https://github.com/ClickHouse/ClickHouse/issues/66026). Avoid unresolved table function arguments traversal in `ReplaceTableNodeToDummyVisitor`. [#67522](https://github.com/ClickHouse/ClickHouse/pull/67522) ([Dmitry Novik](https://github.com/novikd)).
+
+#### Bug Fix (user-visible misbehavior in an official stable release)
+* Backported in [#65350](https://github.com/ClickHouse/ClickHouse/issues/65350): Fix possible abort on uncaught exception in ~WriteBufferFromFileDescriptor in StatusFile. [#64206](https://github.com/ClickHouse/ClickHouse/pull/64206) ([Kruglov Pavel](https://github.com/Avogar)).
+* Backported in [#65621](https://github.com/ClickHouse/ClickHouse/issues/65621): Fix `Cannot find column` in distributed query with `ARRAY JOIN` by `Nested` column. Fixes [#64755](https://github.com/ClickHouse/ClickHouse/issues/64755). [#64801](https://github.com/ClickHouse/ClickHouse/pull/64801) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#65933](https://github.com/ClickHouse/ClickHouse/issues/65933): For queries that read from `PostgreSQL`, cancel the internal `PostgreSQL` query if the ClickHouse query is finished. Otherwise, `ClickHouse` query cannot be canceled until the internal `PostgreSQL` query is finished. [#65771](https://github.com/ClickHouse/ClickHouse/pull/65771) ([Maksim Kita](https://github.com/kitaisreal)).
+* Backported in [#66301](https://github.com/ClickHouse/ClickHouse/issues/66301): Better handling of join conditions involving `IS NULL` checks (for example `ON (a = b AND (a IS NOT NULL) AND (b IS NOT NULL) ) OR ( (a IS NULL) AND (b IS NULL) )` is rewritten to `ON a <=> b`), fix incorrect optimization when condition other then `IS NULL` are present. [#65835](https://github.com/ClickHouse/ClickHouse/pull/65835) ([vdimir](https://github.com/vdimir)).
+* Backported in [#66328](https://github.com/ClickHouse/ClickHouse/issues/66328): Add missing settings `input_format_csv_skip_first_lines/input_format_tsv_skip_first_lines/input_format_csv_try_infer_numbers_from_strings/input_format_csv_try_infer_strings_from_quoted_tuples` in schema inference cache because they can change the resulting schema. It prevents from incorrect result of schema inference with these settings changed. [#65980](https://github.com/ClickHouse/ClickHouse/pull/65980) ([Kruglov Pavel](https://github.com/Avogar)).
+* Backported in [#66155](https://github.com/ClickHouse/ClickHouse/issues/66155): Fixed buffer overflow bug in `unbin`/`unhex` implementation. [#66106](https://github.com/ClickHouse/ClickHouse/pull/66106) ([Nikita Taranov](https://github.com/nickitat)).
+* Backported in [#66454](https://github.com/ClickHouse/ClickHouse/issues/66454): Fixed a bug in ZooKeeper client: a session could get stuck in unusable state after receiving a hardware error from ZooKeeper. For example, this might happen due to "soft memory limit" in ClickHouse Keeper. [#66140](https://github.com/ClickHouse/ClickHouse/pull/66140) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Backported in [#66226](https://github.com/ClickHouse/ClickHouse/issues/66226): Fix issue in SumIfToCountIfVisitor and signed integers. [#66146](https://github.com/ClickHouse/ClickHouse/pull/66146) ([Raúl Marín](https://github.com/Algunenano)).
+* Backported in [#66680](https://github.com/ClickHouse/ClickHouse/issues/66680): Fix handling limit for `system.numbers_mt` when no index can be used. [#66231](https://github.com/ClickHouse/ClickHouse/pull/66231) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
+* Backported in [#66604](https://github.com/ClickHouse/ClickHouse/issues/66604): Fixed how the ClickHouse server detects the maximum number of usable CPU cores as specified by cgroups v2 if the server runs in a container such as Docker. In more detail, containers often run their process in the root cgroup which has an empty name. In that case, ClickHouse ignored the CPU limits set by cgroups v2. [#66237](https://github.com/ClickHouse/ClickHouse/pull/66237) ([filimonov](https://github.com/filimonov)).
+* Backported in [#66360](https://github.com/ClickHouse/ClickHouse/issues/66360): Fix the `Not-ready set` error when a subquery with `IN` is used in the constraint. [#66261](https://github.com/ClickHouse/ClickHouse/pull/66261) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#66972](https://github.com/ClickHouse/ClickHouse/issues/66972): Fix `Column identifier is already registered` error with `group_by_use_nulls=true` and new analyzer. [#66400](https://github.com/ClickHouse/ClickHouse/pull/66400) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#66969](https://github.com/ClickHouse/ClickHouse/issues/66969): Fix `Cannot find column` error for queries with constant expression in `GROUP BY` key and new analyzer enabled. [#66433](https://github.com/ClickHouse/ClickHouse/pull/66433) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#66720](https://github.com/ClickHouse/ClickHouse/issues/66720): Correctly track memory for `Allocator::realloc`. [#66548](https://github.com/ClickHouse/ClickHouse/pull/66548) ([Antonio Andelic](https://github.com/antonio2368)).
+* Backported in [#66951](https://github.com/ClickHouse/ClickHouse/issues/66951): Fix an invalid result for queries with `WINDOW`. This could happen when `PARTITION` columns have sparse serialization and window functions are executed in parallel. [#66579](https://github.com/ClickHouse/ClickHouse/pull/66579) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#66757](https://github.com/ClickHouse/ClickHouse/issues/66757): Fix `Unknown identifier` and `Column is not under aggregate function` errors for queries with the expression `(column IS NULL).` The bug was triggered by [#65088](https://github.com/ClickHouse/ClickHouse/issues/65088), with the disabled analyzer only. [#66654](https://github.com/ClickHouse/ClickHouse/pull/66654) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#66948](https://github.com/ClickHouse/ClickHouse/issues/66948): Fix `Method getResultType is not supported for QUERY query node` error when scalar subquery was used as the first argument of IN (with new analyzer). [#66655](https://github.com/ClickHouse/ClickHouse/pull/66655) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#67633](https://github.com/ClickHouse/ClickHouse/issues/67633): Fix for occasional deadlock in Context::getDDLWorker. [#66843](https://github.com/ClickHouse/ClickHouse/pull/66843) ([Alexander Gololobov](https://github.com/davenger)).
+* Backported in [#67481](https://github.com/ClickHouse/ClickHouse/issues/67481): In rare cases ClickHouse could consider parts as broken because of some unexpected projections on disk. Now it's fixed. [#66898](https://github.com/ClickHouse/ClickHouse/pull/66898) ([alesapin](https://github.com/alesapin)).
+* Backported in [#67197](https://github.com/ClickHouse/ClickHouse/issues/67197): TRUNCATE DATABASE used to stop replication as if it was a DROP DATABASE query, it's fixed. [#67129](https://github.com/ClickHouse/ClickHouse/pull/67129) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Backported in [#67379](https://github.com/ClickHouse/ClickHouse/issues/67379): Fix error `Cannot convert column because it is non constant in source stream but must be constant in result.` for a query that reads from the `Merge` table over the `Distriburted` table with one shard. [#67146](https://github.com/ClickHouse/ClickHouse/pull/67146) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#67576](https://github.com/ClickHouse/ClickHouse/issues/67576): Fix execution of nested short-circuit functions. [#67520](https://github.com/ClickHouse/ClickHouse/pull/67520) ([Kruglov Pavel](https://github.com/Avogar)).
+
+#### NOT FOR CHANGELOG / INSIGNIFICANT
+
+* Backported in [#66387](https://github.com/ClickHouse/ClickHouse/issues/66387): Disable broken cases from 02911_join_on_nullsafe_optimization. [#66310](https://github.com/ClickHouse/ClickHouse/pull/66310) ([vdimir](https://github.com/vdimir)).
+* Backported in [#66426](https://github.com/ClickHouse/ClickHouse/issues/66426): Ignore subquery for IN in DDLLoadingDependencyVisitor. [#66395](https://github.com/ClickHouse/ClickHouse/pull/66395) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#66544](https://github.com/ClickHouse/ClickHouse/issues/66544): Add additional log masking in CI. [#66523](https://github.com/ClickHouse/ClickHouse/pull/66523) ([Raúl Marín](https://github.com/Algunenano)).
+* Backported in [#66859](https://github.com/ClickHouse/ClickHouse/issues/66859): Fix data race in S3::ClientCache. [#66644](https://github.com/ClickHouse/ClickHouse/pull/66644) ([Konstantin Morozov](https://github.com/k-morozov)).
+* Backported in [#66875](https://github.com/ClickHouse/ClickHouse/issues/66875): Support one more case in JOIN ON ... IS NULL. [#66725](https://github.com/ClickHouse/ClickHouse/pull/66725) ([vdimir](https://github.com/vdimir)).
+* Backported in [#67059](https://github.com/ClickHouse/ClickHouse/issues/67059): Increase asio pool size in case the server is tiny. [#66761](https://github.com/ClickHouse/ClickHouse/pull/66761) ([alesapin](https://github.com/alesapin)).
+* Backported in [#66945](https://github.com/ClickHouse/ClickHouse/issues/66945): Small fix in realloc memory tracking. [#66820](https://github.com/ClickHouse/ClickHouse/pull/66820) ([Antonio Andelic](https://github.com/antonio2368)).
+* Backported in [#67252](https://github.com/ClickHouse/ClickHouse/issues/67252): Followup [#66725](https://github.com/ClickHouse/ClickHouse/issues/66725). [#66869](https://github.com/ClickHouse/ClickHouse/pull/66869) ([vdimir](https://github.com/vdimir)).
+* Backported in [#67412](https://github.com/ClickHouse/ClickHouse/issues/67412): CI: Fix build results for release branches. [#67402](https://github.com/ClickHouse/ClickHouse/pull/67402) ([Max K.](https://github.com/maxknv)).
+
diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv
index cb6b8f588da..75c10fa67b8 100644
--- a/utils/list-versions/version_date.tsv
+++ b/utils/list-versions/version_date.tsv
@@ -2,6 +2,7 @@ v24.7.2.13-stable	2024-08-01
 v24.7.1.2915-stable	2024-07-30
 v24.6.2.17-stable	2024-07-05
 v24.6.1.4423-stable	2024-07-01
+v24.5.5.78-stable	2024-08-05
 v24.5.4.49-stable	2024-07-01
 v24.5.3.5-stable	2024-06-13
 v24.5.2.34-stable	2024-06-13

From 4e8d11c48e9ac0107940d730ccf8d35b7ac3573e Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Mon, 5 Aug 2024 16:46:42 +0000
Subject: [PATCH 1410/1488] Add fuzzers to
 03208_buffer_over_distributed_type_mismatch

---
 ...208_buffer_over_distributed_type_mismatch.sql | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/tests/queries/0_stateless/03208_buffer_over_distributed_type_mismatch.sql b/tests/queries/0_stateless/03208_buffer_over_distributed_type_mismatch.sql
index 5a7c89074cf..333c445403d 100644
--- a/tests/queries/0_stateless/03208_buffer_over_distributed_type_mismatch.sql
+++ b/tests/queries/0_stateless/03208_buffer_over_distributed_type_mismatch.sql
@@ -58,3 +58,19 @@ SELECT amount FROM realtimebuff__fuzz_19 t1
 JOIN realtimebuff__fuzz_19 t2 ON t1.amount = t2.amount
 JOIN realtimebuff__fuzz_19 t3 ON t1.amount = t3.amount
 ; -- { serverError NOT_IMPLEMENTED,AMBIGUOUS_COLUMN_NAME }
+
+
+-- fuzzers:
+
+SELECT
+    toLowCardinality(1) + materialize(toLowCardinality(2))
+FROM realtimebuff__fuzz_19
+GROUP BY toLowCardinality(1)
+FORMAT Null
+;
+
+SELECT intDivOrZero(intDivOrZero(toLowCardinality(-128), toLowCardinality(-1)) = 0, materialize(toLowCardinality(4)))
+FROM realtimebuff__fuzz_19 GROUP BY materialize(toLowCardinality(-127)), intDivOrZero(0, 0) = toLowCardinality(toLowCardinality(0))
+WITH TOTALS ORDER BY ALL DESC NULLS FIRST
+FORMAT Null
+;

From 05395ac7bd93949d07f163bfc6cabf9c3e85adc5 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Mon, 5 Aug 2024 17:01:15 +0000
Subject: [PATCH 1411/1488] Un-flake 02524_fuzz_and_fuss_2 (hopefully)

https://s3.amazonaws.com/clickhouse-test-reports/0/3221b651da16cd868350a1aff022ba71a1a15f3c/stateless_tests__tsan__s3_storage__[2_5].html

https://s3.amazonaws.com/clickhouse-test-reports/0/1fde5b7bfa1b1a6a0d67258be5e7ef855b730559/stateless_tests__tsan__s3_storage__[2_4].html
---
 tests/queries/0_stateless/02524_fuzz_and_fuss_2.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/02524_fuzz_and_fuss_2.sql b/tests/queries/0_stateless/02524_fuzz_and_fuss_2.sql
index 7b49378d4da..a38fb0bd471 100644
--- a/tests/queries/0_stateless/02524_fuzz_and_fuss_2.sql
+++ b/tests/queries/0_stateless/02524_fuzz_and_fuss_2.sql
@@ -9,6 +9,6 @@ ENGINE = Memory;
 INSERT INTO data_a_02187
 SELECT *
 FROM system.one
-SETTINGS max_block_size = '1', min_insert_block_size_rows = '65536', min_insert_block_size_bytes = '0', max_insert_threads = '0', max_threads = '3', receive_timeout = '10', receive_data_timeout_ms = '10000', connections_with_failover_max_tries = '0', extremes = '1', use_uncompressed_cache = '0', optimize_move_to_prewhere = '1', optimize_move_to_prewhere_if_final = '0', replication_alter_partitions_sync = '2', totals_mode = 'before_having', allow_suspicious_low_cardinality_types = '1', compile_expressions = '1', min_count_to_compile_expression = '0', group_by_two_level_threshold = '100', distributed_aggregation_memory_efficient = '0', distributed_group_by_no_merge = '1', optimize_distributed_group_by_sharding_key = '1', optimize_skip_unused_shards = '1', optimize_skip_unused_shards_rewrite_in = '1', force_optimize_skip_unused_shards = '2', optimize_skip_unused_shards_nesting = '1', force_optimize_skip_unused_shards_nesting = '2', merge_tree_min_rows_for_concurrent_read = '10000', force_primary_key = '1', network_compression_method = 'ZSTD', network_zstd_compression_level = '7', log_queries = '0', log_queries_min_type = 'QUERY_FINISH', distributed_product_mode = 'local', insert_quorum = '2', insert_quorum_timeout = '0', insert_quorum_parallel = '0', select_sequential_consistency = '1', join_use_nulls = '1', any_join_distinct_right_table_keys = '1', preferred_max_column_in_block_size_bytes = '32', distributed_foreground_insert = '1', insert_allow_materialized_columns = '1', use_index_for_in_with_subqueries = '1', joined_subquery_requires_alias = '0', empty_result_for_aggregation_by_empty_set = '1', allow_suspicious_codecs = '1', query_profiler_real_time_period_ns = '0', query_profiler_cpu_time_period_ns = '0', opentelemetry_start_trace_probability = '1', max_rows_to_read = '1000000', read_overflow_mode = 'break', max_rows_to_group_by = '10', group_by_overflow_mode = 'any', max_rows_to_sort = '100', sort_overflow_mode = 'break', max_result_rows = '10', max_execution_time = '3', max_execution_speed = '1', max_bytes_in_join = '100', join_algorithm = 'partial_merge', max_memory_usage = '1099511627776', log_query_threads = '1', send_logs_level = 'fatal', enable_optimize_predicate_expression = '1', prefer_localhost_replica = '1', optimize_read_in_order = '1', optimize_aggregation_in_order = '1', read_in_order_two_level_merge_threshold = '1', allow_introspection_functions = '1', check_query_single_value_result = '1', allow_experimental_live_view = '1', default_table_engine = 'Memory', mutations_sync = '2', convert_query_to_cnf = '0', optimize_arithmetic_operations_in_aggregate_functions = '1', optimize_duplicate_order_by_and_distinct = '0', optimize_multiif_to_if = '0', optimize_functions_to_subcolumns = '1', optimize_using_constraints = '1', optimize_substitute_columns = '1', optimize_append_index = '1', transform_null_in = '1', data_type_default_nullable = '1', cast_keep_nullable = '1', cast_ipv4_ipv6_default_on_conversion_error = '0', system_events_show_zero_values = '1', enable_global_with_statement = '1', optimize_on_insert = '0', optimize_rewrite_sum_if_to_count_if = '1', distributed_ddl_output_mode = 'throw', union_default_mode = 'ALL', optimize_aggregators_of_group_by_keys = '1', optimize_group_by_function_keys = '1', short_circuit_function_evaluation = 'enable', async_insert = '1', enable_filesystem_cache = '0', allow_deprecated_database_ordinary = '1', allow_deprecated_syntax_for_merge_tree = '1', allow_experimental_nlp_functions = '1', allow_experimental_object_type = '1', optimize_use_projections = '1', input_format_null_as_default = '1', input_format_ipv4_default_on_conversion_error = '0', input_format_ipv6_default_on_conversion_error = '0', output_format_json_named_tuples_as_objects = '1', output_format_write_statistics = '0', output_format_pretty_row_numbers = '1';
+SETTINGS max_block_size = '1', min_insert_block_size_rows = '65536', min_insert_block_size_bytes = '0', max_insert_threads = '0', max_threads = '3', receive_timeout = '10', receive_data_timeout_ms = '10000', connections_with_failover_max_tries = '0', extremes = '1', use_uncompressed_cache = '0', optimize_move_to_prewhere = '1', optimize_move_to_prewhere_if_final = '0', replication_alter_partitions_sync = '2', totals_mode = 'before_having', allow_suspicious_low_cardinality_types = '1', compile_expressions = '1', min_count_to_compile_expression = '0', group_by_two_level_threshold = '100', distributed_aggregation_memory_efficient = '0', distributed_group_by_no_merge = '1', optimize_distributed_group_by_sharding_key = '1', optimize_skip_unused_shards = '1', optimize_skip_unused_shards_rewrite_in = '1', force_optimize_skip_unused_shards = '2', optimize_skip_unused_shards_nesting = '1', force_optimize_skip_unused_shards_nesting = '2', merge_tree_min_rows_for_concurrent_read = '10000', force_primary_key = '1', network_compression_method = 'ZSTD', network_zstd_compression_level = '7', log_queries = '0', log_queries_min_type = 'QUERY_FINISH', distributed_product_mode = 'local', insert_quorum = '2', insert_quorum_timeout = '0', insert_quorum_parallel = '0', select_sequential_consistency = '1', join_use_nulls = '1', any_join_distinct_right_table_keys = '1', preferred_max_column_in_block_size_bytes = '32', distributed_foreground_insert = '1', insert_allow_materialized_columns = '1', use_index_for_in_with_subqueries = '1', joined_subquery_requires_alias = '0', empty_result_for_aggregation_by_empty_set = '1', allow_suspicious_codecs = '1', query_profiler_real_time_period_ns = '0', query_profiler_cpu_time_period_ns = '0', opentelemetry_start_trace_probability = '1', max_rows_to_read = '1000000', read_overflow_mode = 'break', max_rows_to_group_by = '10', group_by_overflow_mode = 'any', max_rows_to_sort = '100', sort_overflow_mode = 'break', max_result_rows = '10', max_execution_time = '9', max_execution_speed = '1', max_bytes_in_join = '100', join_algorithm = 'partial_merge', max_memory_usage = '1099511627776', log_query_threads = '1', send_logs_level = 'fatal', enable_optimize_predicate_expression = '1', prefer_localhost_replica = '1', optimize_read_in_order = '1', optimize_aggregation_in_order = '1', read_in_order_two_level_merge_threshold = '1', allow_introspection_functions = '1', check_query_single_value_result = '1', allow_experimental_live_view = '1', default_table_engine = 'Memory', mutations_sync = '2', convert_query_to_cnf = '0', optimize_arithmetic_operations_in_aggregate_functions = '1', optimize_duplicate_order_by_and_distinct = '0', optimize_multiif_to_if = '0', optimize_functions_to_subcolumns = '1', optimize_using_constraints = '1', optimize_substitute_columns = '1', optimize_append_index = '1', transform_null_in = '1', data_type_default_nullable = '1', cast_keep_nullable = '1', cast_ipv4_ipv6_default_on_conversion_error = '0', system_events_show_zero_values = '1', enable_global_with_statement = '1', optimize_on_insert = '0', optimize_rewrite_sum_if_to_count_if = '1', distributed_ddl_output_mode = 'throw', union_default_mode = 'ALL', optimize_aggregators_of_group_by_keys = '1', optimize_group_by_function_keys = '1', short_circuit_function_evaluation = 'enable', async_insert = '1', enable_filesystem_cache = '0', allow_deprecated_database_ordinary = '1', allow_deprecated_syntax_for_merge_tree = '1', allow_experimental_nlp_functions = '1', allow_experimental_object_type = '1', optimize_use_projections = '1', input_format_null_as_default = '1', input_format_ipv4_default_on_conversion_error = '0', input_format_ipv6_default_on_conversion_error = '0', output_format_json_named_tuples_as_objects = '1', output_format_write_statistics = '0', output_format_pretty_row_numbers = '1';
 
 DROP TABLE data_a_02187;

From c8805fbcedcce226a82d78cff79d6afbdddca0a7 Mon Sep 17 00:00:00 2001
From: Alex Katsman <alex.katsman@clickhouse.com>
Date: Mon, 5 Aug 2024 17:41:47 +0000
Subject: [PATCH 1412/1488] Misc fixes

---
 docker/test/fasttest/run.sh           | 2 +-
 src/Functions/bitSlice.cpp            | 4 ++--
 src/Storages/MergeTree/KeyCondition.h | 3 ++-
 3 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/docker/test/fasttest/run.sh b/docker/test/fasttest/run.sh
index 26283afc86a..394d31addb1 100755
--- a/docker/test/fasttest/run.sh
+++ b/docker/test/fasttest/run.sh
@@ -41,7 +41,7 @@ export FASTTEST_WORKSPACE
 export FASTTEST_SOURCE
 export FASTTEST_BUILD
 export FASTTEST_DATA
-export FASTTEST_OUT
+export FASTTEST_OUTPUT
 export PATH
 
 function ccache_status
diff --git a/src/Functions/bitSlice.cpp b/src/Functions/bitSlice.cpp
index f24473351ae..908c534b228 100644
--- a/src/Functions/bitSlice.cpp
+++ b/src/Functions/bitSlice.cpp
@@ -42,11 +42,11 @@ public:
     {
         FunctionArgumentDescriptors mandatory_args{
             {"s", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isStringOrFixedString), nullptr, "String"},
-            {"offset", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNativeNumber), nullptr, "(U)Int8, (U)Int16, (U)Int32, (U)Int64 or Float"},
+            {"offset", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNativeNumber), nullptr, "(U)Int8/16/32/64 or Float"},
         };
 
         FunctionArgumentDescriptors optional_args{
-            {"length", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNativeNumber), nullptr, "(U)Int8, (U)Int16, (U)Int32, (U)Int64 or Float"},
+            {"length", static_cast<FunctionArgumentDescriptor::TypeValidator>(&isNativeNumber), nullptr, "(U)Int8/16/32/64 or Float"},
         };
 
         validateFunctionArguments(*this, arguments, mandatory_args, optional_args);
diff --git a/src/Storages/MergeTree/KeyCondition.h b/src/Storages/MergeTree/KeyCondition.h
index e9343ec08ea..8bbb86aba43 100644
--- a/src/Storages/MergeTree/KeyCondition.h
+++ b/src/Storages/MergeTree/KeyCondition.h
@@ -6,6 +6,8 @@
 #include <Core/Range.h>
 #include <Core/PlainRanges.h>
 
+#include <DataTypes/Serializations/ISerialization.h>
+
 #include <Parsers/ASTExpressionList.h>
 
 #include <Interpreters/Set.h>
@@ -14,7 +16,6 @@
 
 #include <Storages/SelectQueryInfo.h>
 #include <Storages/MergeTree/RPNBuilder.h>
-#include "DataTypes/Serializations/ISerialization.h"
 
 
 namespace DB

From 457686c379cb884a579b86cbfd6a9abb3ec16d1a Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Wed, 10 Jul 2024 19:39:58 +0200
Subject: [PATCH 1413/1488] Simplify StorageDistributed ctors

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 src/Storages/StorageDistributed.cpp        | 32 ----------------------
 src/Storages/StorageDistributed.h          | 14 ----------
 src/TableFunctions/TableFunctionRemote.cpp | 19 ++-----------
 3 files changed, 3 insertions(+), 62 deletions(-)

diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp
index 3e38ddf830a..eb9483127fc 100644
--- a/src/Storages/StorageDistributed.cpp
+++ b/src/Storages/StorageDistributed.cpp
@@ -373,38 +373,6 @@ StorageDistributed::StorageDistributed(
 }
 
 
-StorageDistributed::StorageDistributed(
-    const StorageID & id_,
-    const ColumnsDescription & columns_,
-    const ConstraintsDescription & constraints_,
-    ASTPtr remote_table_function_ptr_,
-    const String & cluster_name_,
-    ContextPtr context_,
-    const ASTPtr & sharding_key_,
-    const String & storage_policy_name_,
-    const String & relative_data_path_,
-    const DistributedSettings & distributed_settings_,
-    LoadingStrictnessLevel mode,
-    ClusterPtr owned_cluster_)
-    : StorageDistributed(
-        id_,
-        columns_,
-        constraints_,
-        String{},
-        String{},
-        String{},
-        cluster_name_,
-        context_,
-        sharding_key_,
-        storage_policy_name_,
-        relative_data_path_,
-        distributed_settings_,
-        mode,
-        std::move(owned_cluster_),
-        remote_table_function_ptr_)
-{
-}
-
 QueryProcessingStage::Enum StorageDistributed::getQueryProcessingStage(
     ContextPtr local_context,
     QueryProcessingStage::Enum to_stage,
diff --git a/src/Storages/StorageDistributed.h b/src/Storages/StorageDistributed.h
index 85a8de86953..a9e10dd14ba 100644
--- a/src/Storages/StorageDistributed.h
+++ b/src/Storages/StorageDistributed.h
@@ -63,20 +63,6 @@ public:
         ClusterPtr owned_cluster_ = {},
         ASTPtr remote_table_function_ptr_ = {});
 
-    StorageDistributed(
-        const StorageID & id_,
-        const ColumnsDescription & columns_,
-        const ConstraintsDescription & constraints_,
-        ASTPtr remote_table_function_ptr_,
-        const String & cluster_name_,
-        ContextPtr context_,
-        const ASTPtr & sharding_key_,
-        const String & storage_policy_name_,
-        const String & relative_data_path_,
-        const DistributedSettings & distributed_settings_,
-        LoadingStrictnessLevel mode,
-        ClusterPtr owned_cluster_ = {});
-
     ~StorageDistributed() override;
 
     std::string getName() const override { return "Distributed"; }
diff --git a/src/TableFunctions/TableFunctionRemote.cpp b/src/TableFunctions/TableFunctionRemote.cpp
index e60c31b2d77..5d136993018 100644
--- a/src/TableFunctions/TableFunctionRemote.cpp
+++ b/src/TableFunctions/TableFunctionRemote.cpp
@@ -306,21 +306,7 @@ StoragePtr TableFunctionRemote::executeImpl(const ASTPtr & /*ast_function*/, Con
         cached_columns = getActualTableStructure(context, is_insert_query);
 
     assert(cluster);
-    StoragePtr res = remote_table_function_ptr
-        ? std::make_shared<StorageDistributed>(
-            StorageID(getDatabaseName(), table_name),
-            cached_columns,
-            ConstraintsDescription{},
-            remote_table_function_ptr,
-            String{},
-            context,
-            sharding_key,
-            String{},
-            String{},
-            DistributedSettings{},
-            LoadingStrictnessLevel::CREATE,
-            cluster)
-        : std::make_shared<StorageDistributed>(
+    StoragePtr res = std::make_shared<StorageDistributed>(
             StorageID(getDatabaseName(), table_name),
             cached_columns,
             ConstraintsDescription{},
@@ -334,7 +320,8 @@ StoragePtr TableFunctionRemote::executeImpl(const ASTPtr & /*ast_function*/, Con
             String{},
             DistributedSettings{},
             LoadingStrictnessLevel::CREATE,
-            cluster);
+            cluster,
+            remote_table_function_ptr);
 
     res->startup();
     return res;

From 9d7710684b91a9a5b0d7c2a34b709232979ebe65 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Wed, 10 Jul 2024 19:57:09 +0200
Subject: [PATCH 1414/1488] tests/test_distributed_inter_server_secret:
 get_query_user_info return list

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 .../test_distributed_inter_server_secret/test.py | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/tests/integration/test_distributed_inter_server_secret/test.py b/tests/integration/test_distributed_inter_server_secret/test.py
index 7ecb2cda257..9389b901a30 100644
--- a/tests/integration/test_distributed_inter_server_secret/test.py
+++ b/tests/integration/test_distributed_inter_server_secret/test.py
@@ -116,10 +116,10 @@ def start_cluster():
         cluster.shutdown()
 
 
-# @return -- [user, initial_user]
+# @return -- [[user, initial_user]]
 def get_query_user_info(node, query_pattern):
     node.query("SYSTEM FLUSH LOGS")
-    return (
+    lines = (
         node.query(
             """
     SELECT user, initial_user
@@ -133,8 +133,10 @@ def get_query_user_info(node, query_pattern):
             )
         )
         .strip()
-        .split("\t")
+        .split("\n")
     )
+    lines = map(lambda x: x.split("\t"), lines)
+    return list(lines)
 
 
 # @return -- [user, initial_user]
@@ -331,19 +333,19 @@ def test_secure_disagree_insert():
 def test_user_insecure_cluster(user, password):
     id_ = "query-dist_insecure-" + user
     n1.query(f"SELECT *, '{id_}' FROM dist_insecure", user=user, password=password)
-    assert get_query_user_info(n1, id_) == [
+    assert get_query_user_info(n1, id_)[0] == [
         user,
         user,
     ]  # due to prefer_localhost_replica
-    assert get_query_user_info(n2, id_) == ["default", user]
+    assert get_query_user_info(n2, id_)[0] == ["default", user]
 
 
 @users
 def test_user_secure_cluster(user, password):
     id_ = "query-dist_secure-" + user
     n1.query(f"SELECT *, '{id_}' FROM dist_secure", user=user, password=password)
-    assert get_query_user_info(n1, id_) == [user, user]
-    assert get_query_user_info(n2, id_) == [user, user]
+    assert get_query_user_info(n1, id_)[0] == [user, user]
+    assert get_query_user_info(n2, id_)[0] == [user, user]
 
 
 @users

From 0dccc34a2004b075e290ebfd5ecc9fea3077a205 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Wed, 10 Jul 2024 19:23:34 +0200
Subject: [PATCH 1415/1488] Fix cluster() for inter-server secret (preserve
 initial user as before)

The behaviour of cluster() with inter-server secret had been changed in #63013,
after it started to use "default" user, and this introduces a
regression.

The intention of that patch was to adjust only remote(), since it only
it accept custom user, which should be ignored.

Fixes: https://github.com/ClickHouse/ClickHouse/issues/66287
Fixes: https://github.com/ClickHouse/ClickHouse/issues/66352
Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 src/Storages/StorageDistributed.cpp               |  6 ++++--
 src/Storages/StorageDistributed.h                 |  5 ++++-
 src/TableFunctions/TableFunctionRemote.cpp        |  3 ++-
 .../test_distributed_inter_server_secret/test.py  | 15 ++++++++++++++-
 4 files changed, 24 insertions(+), 5 deletions(-)

diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp
index eb9483127fc..e146e95f89f 100644
--- a/src/Storages/StorageDistributed.cpp
+++ b/src/Storages/StorageDistributed.cpp
@@ -308,7 +308,8 @@ StorageDistributed::StorageDistributed(
     const DistributedSettings & distributed_settings_,
     LoadingStrictnessLevel mode,
     ClusterPtr owned_cluster_,
-    ASTPtr remote_table_function_ptr_)
+    ASTPtr remote_table_function_ptr_,
+    bool is_remote_function_)
     : IStorage(id_)
     , WithContext(context_->getGlobalContext())
     , remote_database(remote_database_)
@@ -322,6 +323,7 @@ StorageDistributed::StorageDistributed(
     , relative_data_path(relative_data_path_)
     , distributed_settings(distributed_settings_)
     , rng(randomSeed())
+    , is_remote_function(is_remote_function_)
 {
     if (!distributed_settings.flush_on_detach && distributed_settings.background_insert_batch)
         throw Exception(ErrorCodes::BAD_ARGUMENTS, "Settings flush_on_detach=0 and background_insert_batch=1 are incompatible");
@@ -869,7 +871,7 @@ void StorageDistributed::read(
         sharding_key_column_name,
         distributed_settings,
         shard_filter_generator,
-        /* is_remote_function= */ static_cast<bool>(owned_cluster));
+        is_remote_function);
 
     /// This is a bug, it is possible only when there is no shards to query, and this is handled earlier.
     if (!query_plan.isInitialized())
diff --git a/src/Storages/StorageDistributed.h b/src/Storages/StorageDistributed.h
index a9e10dd14ba..8a5585e9fd0 100644
--- a/src/Storages/StorageDistributed.h
+++ b/src/Storages/StorageDistributed.h
@@ -61,7 +61,8 @@ public:
         const DistributedSettings & distributed_settings_,
         LoadingStrictnessLevel mode,
         ClusterPtr owned_cluster_ = {},
-        ASTPtr remote_table_function_ptr_ = {});
+        ASTPtr remote_table_function_ptr_ = {},
+        bool is_remote_function_ = false);
 
     ~StorageDistributed() override;
 
@@ -273,6 +274,8 @@ private:
     // For random shard index generation
     mutable std::mutex rng_mutex;
     pcg64 rng;
+
+    bool is_remote_function;
 };
 
 }
diff --git a/src/TableFunctions/TableFunctionRemote.cpp b/src/TableFunctions/TableFunctionRemote.cpp
index 5d136993018..8a877ff0802 100644
--- a/src/TableFunctions/TableFunctionRemote.cpp
+++ b/src/TableFunctions/TableFunctionRemote.cpp
@@ -321,7 +321,8 @@ StoragePtr TableFunctionRemote::executeImpl(const ASTPtr & /*ast_function*/, Con
             DistributedSettings{},
             LoadingStrictnessLevel::CREATE,
             cluster,
-            remote_table_function_ptr);
+            remote_table_function_ptr,
+            !is_cluster_function);
 
     res->startup();
     return res;
diff --git a/tests/integration/test_distributed_inter_server_secret/test.py b/tests/integration/test_distributed_inter_server_secret/test.py
index 9389b901a30..36d7e044f1c 100644
--- a/tests/integration/test_distributed_inter_server_secret/test.py
+++ b/tests/integration/test_distributed_inter_server_secret/test.py
@@ -418,7 +418,7 @@ def test_per_user_protocol_settings_secure_cluster(user, password):
     )
 
 
-def test_secure_cluster_distributed_over_distributed_different_users():
+def test_secure_cluster_distributed_over_distributed_different_users_remote():
     # This works because we will have initial_user='default'
     n1.query(
         "SELECT * FROM remote('n1', currentDatabase(), dist_secure)", user="new_user"
@@ -433,3 +433,16 @@ def test_secure_cluster_distributed_over_distributed_different_users():
     # and stuff).
     with pytest.raises(QueryRuntimeException):
         n1.query("SELECT * FROM dist_over_dist_secure", user="new_user")
+
+
+def test_secure_cluster_distributed_over_distributed_different_users_cluster():
+    id_ = "cluster-user"
+    n1.query(
+        f"SELECT *, '{id_}' FROM cluster(secure, currentDatabase(), dist_secure)",
+        user="nopass",
+        settings={
+            "prefer_localhost_replica": 0,
+        },
+    )
+    assert get_query_user_info(n1, id_) == [["nopass", "nopass"]] * 4
+    assert get_query_user_info(n2, id_) == [["nopass", "nopass"]] * 3

From 3593f740a8793339c5657a0bcc6f785e1198e510 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Thu, 25 Jul 2024 12:16:23 +0200
Subject: [PATCH 1416/1488] Revert "Remove bad tests @azat" (reverts #66823)

This reverts commit 0c2c027af63fcbababffbe3a39ed2631884e1938.

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 ...2_part_log_rmt_fetch_merge_error.reference | 10 +++++
 .../03002_part_log_rmt_fetch_merge_error.sql  | 35 ++++++++++++++++
 ..._part_log_rmt_fetch_mutate_error.reference | 10 +++++
 .../03002_part_log_rmt_fetch_mutate_error.sql | 41 +++++++++++++++++++
 4 files changed, 96 insertions(+)
 create mode 100644 tests/queries/0_stateless/03002_part_log_rmt_fetch_merge_error.reference
 create mode 100644 tests/queries/0_stateless/03002_part_log_rmt_fetch_merge_error.sql
 create mode 100644 tests/queries/0_stateless/03002_part_log_rmt_fetch_mutate_error.reference
 create mode 100644 tests/queries/0_stateless/03002_part_log_rmt_fetch_mutate_error.sql

diff --git a/tests/queries/0_stateless/03002_part_log_rmt_fetch_merge_error.reference b/tests/queries/0_stateless/03002_part_log_rmt_fetch_merge_error.reference
new file mode 100644
index 00000000000..b19d389d8d0
--- /dev/null
+++ b/tests/queries/0_stateless/03002_part_log_rmt_fetch_merge_error.reference
@@ -0,0 +1,10 @@
+before
+rmt_master	NewPart	0	1
+rmt_master	MergeParts	0	1
+rmt_slave	MergeParts	1	0
+rmt_slave	DownloadPart	0	1
+after
+rmt_master	NewPart	0	1
+rmt_master	MergeParts	0	1
+rmt_slave	MergeParts	1	0
+rmt_slave	DownloadPart	0	2
diff --git a/tests/queries/0_stateless/03002_part_log_rmt_fetch_merge_error.sql b/tests/queries/0_stateless/03002_part_log_rmt_fetch_merge_error.sql
new file mode 100644
index 00000000000..548a8e5570a
--- /dev/null
+++ b/tests/queries/0_stateless/03002_part_log_rmt_fetch_merge_error.sql
@@ -0,0 +1,35 @@
+-- Tags: no-replicated-database, no-parallel, no-shared-merge-tree
+-- SMT: The merge process is completely different from RMT
+
+drop table if exists rmt_master;
+drop table if exists rmt_slave;
+
+create table rmt_master (key Int) engine=ReplicatedMergeTree('/clickhouse/{database}', 'master') order by key settings always_fetch_merged_part=0;
+-- always_fetch_merged_part=1, consider this table as a "slave"
+create table rmt_slave (key Int) engine=ReplicatedMergeTree('/clickhouse/{database}', 'slave') order by key settings always_fetch_merged_part=1;
+
+insert into rmt_master values (1);
+
+system sync replica rmt_master;
+system sync replica rmt_slave;
+system stop replicated sends rmt_master;
+optimize table rmt_master final settings alter_sync=1, optimize_throw_if_noop=1;
+
+select sleep(3) format Null;
+
+system flush logs;
+select 'before';
+select table, event_type, error>0, countIf(error=0) from system.part_log where database = currentDatabase() group by 1, 2, 3 order by 1, 2, 3;
+
+system start replicated sends rmt_master;
+-- sleep few seconds to try rmt_slave to fetch the part and reflect this error
+-- in system.part_log
+select sleep(3) format Null;
+system sync replica rmt_slave;
+
+system flush logs;
+select 'after';
+select table, event_type, error>0, countIf(error=0) from system.part_log where database = currentDatabase() group by 1, 2, 3 order by 1, 2, 3;
+
+drop table rmt_master;
+drop table rmt_slave;
diff --git a/tests/queries/0_stateless/03002_part_log_rmt_fetch_mutate_error.reference b/tests/queries/0_stateless/03002_part_log_rmt_fetch_mutate_error.reference
new file mode 100644
index 00000000000..aac9e7527d1
--- /dev/null
+++ b/tests/queries/0_stateless/03002_part_log_rmt_fetch_mutate_error.reference
@@ -0,0 +1,10 @@
+before
+rmt_master	NewPart	0	1
+rmt_master	MutatePart	0	1
+rmt_slave	DownloadPart	0	1
+rmt_slave	MutatePart	1	0
+after
+rmt_master	NewPart	0	1
+rmt_master	MutatePart	0	1
+rmt_slave	DownloadPart	0	2
+rmt_slave	MutatePart	1	0
diff --git a/tests/queries/0_stateless/03002_part_log_rmt_fetch_mutate_error.sql b/tests/queries/0_stateless/03002_part_log_rmt_fetch_mutate_error.sql
new file mode 100644
index 00000000000..d8b5ebb3148
--- /dev/null
+++ b/tests/queries/0_stateless/03002_part_log_rmt_fetch_mutate_error.sql
@@ -0,0 +1,41 @@
+-- Tags: no-replicated-database, no-parallel, no-shared-merge-tree
+-- SMT: The merge process is completely different from RMT
+
+drop table if exists rmt_master;
+drop table if exists rmt_slave;
+
+create table rmt_master (key Int) engine=ReplicatedMergeTree('/clickhouse/{database}', 'master') order by tuple() settings always_fetch_merged_part=0, old_parts_lifetime=600;
+-- prefer_fetch_merged_part_*_threshold=0, consider this table as a "slave"
+create table rmt_slave (key Int) engine=ReplicatedMergeTree('/clickhouse/{database}', 'slave') order by tuple() settings prefer_fetch_merged_part_time_threshold=0, prefer_fetch_merged_part_size_threshold=0, old_parts_lifetime=600;
+
+insert into rmt_master values (1);
+
+system sync replica rmt_master;
+system sync replica rmt_slave;
+system stop replicated sends rmt_master;
+system stop pulling replication log rmt_slave;
+alter table rmt_master update key=key+100 where 1 settings alter_sync=1;
+
+-- first we need to make the rmt_master execute mutation so that it will have
+-- the part, and rmt_slave will consider it instead of performing mutation on
+-- it's own, otherwise prefer_fetch_merged_part_*_threshold will be simply ignored
+select sleep(3) format Null;
+system start pulling replication log rmt_slave;
+-- and sleep few more seconds to try rmt_slave to fetch the part and reflect
+-- this error in system.part_log
+select sleep(3) format Null;
+
+system flush logs;
+select 'before';
+select table, event_type, error>0, countIf(error=0) from system.part_log where database = currentDatabase() group by 1, 2, 3 order by 1, 2, 3;
+
+system start replicated sends rmt_master;
+select sleep(3) format Null;
+system sync replica rmt_slave;
+
+system flush logs;
+select 'after';
+select table, event_type, error>0, countIf(error=0) from system.part_log where database = currentDatabase() group by 1, 2, 3 order by 1, 2, 3;
+
+drop table rmt_master;
+drop table rmt_slave;

From 50aebcfc28c5fa98dca92d92c5233adea92801f4 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Thu, 25 Jul 2024 13:18:53 +0200
Subject: [PATCH 1417/1488] Rewrite 03002_part_log_rmt_fetch_*_error tests from
 sql to sh with retries

Except for this patch should fix the flakiness of this test it also
reduces the execution time from 15.4s to 3.3s (5x!)

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 .../03002_part_log_rmt_fetch_merge_error.sh   | 54 ++++++++++++++++
 .../03002_part_log_rmt_fetch_merge_error.sql  | 35 ----------
 .../03002_part_log_rmt_fetch_mutate_error.sh  | 64 +++++++++++++++++++
 .../03002_part_log_rmt_fetch_mutate_error.sql | 41 ------------
 4 files changed, 118 insertions(+), 76 deletions(-)
 create mode 100755 tests/queries/0_stateless/03002_part_log_rmt_fetch_merge_error.sh
 delete mode 100644 tests/queries/0_stateless/03002_part_log_rmt_fetch_merge_error.sql
 create mode 100755 tests/queries/0_stateless/03002_part_log_rmt_fetch_mutate_error.sh
 delete mode 100644 tests/queries/0_stateless/03002_part_log_rmt_fetch_mutate_error.sql

diff --git a/tests/queries/0_stateless/03002_part_log_rmt_fetch_merge_error.sh b/tests/queries/0_stateless/03002_part_log_rmt_fetch_merge_error.sh
new file mode 100755
index 00000000000..25d946b325d
--- /dev/null
+++ b/tests/queries/0_stateless/03002_part_log_rmt_fetch_merge_error.sh
@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+# Tags: no-replicated-database, no-parallel, no-shared-merge-tree
+# SMT: The merge process is completely different from RMT
+
+CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CUR_DIR"/../shell_config.sh
+
+set -e
+
+function wait_until()
+{
+    local q=$1 && shift
+    while [ "$($CLICKHOUSE_CLIENT -nm -q "$q")" != "1" ]; do
+        # too frequent FLUSH LOGS is too costly
+        sleep 2
+    done
+}
+
+$CLICKHOUSE_CLIENT -nm -q "
+    drop table if exists rmt_master;
+    drop table if exists rmt_slave;
+
+    create table rmt_master (key Int) engine=ReplicatedMergeTree('/clickhouse/{database}', 'master') order by key settings always_fetch_merged_part=0;
+    -- always_fetch_merged_part=1, consider this table as a 'slave'
+    create table rmt_slave (key Int) engine=ReplicatedMergeTree('/clickhouse/{database}', 'slave') order by key settings always_fetch_merged_part=1;
+
+    insert into rmt_master values (1);
+
+    system sync replica rmt_master;
+    system sync replica rmt_slave;
+    system stop replicated sends rmt_master;
+    optimize table rmt_master final settings alter_sync=1, optimize_throw_if_noop=1;
+"
+
+$CLICKHOUSE_CLIENT -nm -q "
+    system flush logs;
+    select 'before';
+    select table, event_type, error>0, countIf(error=0) from system.part_log where database = currentDatabase() group by 1, 2, 3 order by 1, 2, 3;
+
+    system start replicated sends rmt_master;
+"
+# wait until rmt_slave will fetch the part and reflect this error in system.part_log
+wait_until "system flush logs; select count()>0 from system.part_log where table = 'rmt_slave' and database = '$CLICKHOUSE_DATABASE' and error > 0"
+$CLICKHOUSE_CLIENT -nm -q "
+    system sync replica rmt_slave;
+
+    system flush logs;
+    select 'after';
+    select table, event_type, error>0, countIf(error=0) from system.part_log where database = currentDatabase() group by 1, 2, 3 order by 1, 2, 3;
+
+    drop table rmt_master;
+    drop table rmt_slave;
+"
diff --git a/tests/queries/0_stateless/03002_part_log_rmt_fetch_merge_error.sql b/tests/queries/0_stateless/03002_part_log_rmt_fetch_merge_error.sql
deleted file mode 100644
index 548a8e5570a..00000000000
--- a/tests/queries/0_stateless/03002_part_log_rmt_fetch_merge_error.sql
+++ /dev/null
@@ -1,35 +0,0 @@
--- Tags: no-replicated-database, no-parallel, no-shared-merge-tree
--- SMT: The merge process is completely different from RMT
-
-drop table if exists rmt_master;
-drop table if exists rmt_slave;
-
-create table rmt_master (key Int) engine=ReplicatedMergeTree('/clickhouse/{database}', 'master') order by key settings always_fetch_merged_part=0;
--- always_fetch_merged_part=1, consider this table as a "slave"
-create table rmt_slave (key Int) engine=ReplicatedMergeTree('/clickhouse/{database}', 'slave') order by key settings always_fetch_merged_part=1;
-
-insert into rmt_master values (1);
-
-system sync replica rmt_master;
-system sync replica rmt_slave;
-system stop replicated sends rmt_master;
-optimize table rmt_master final settings alter_sync=1, optimize_throw_if_noop=1;
-
-select sleep(3) format Null;
-
-system flush logs;
-select 'before';
-select table, event_type, error>0, countIf(error=0) from system.part_log where database = currentDatabase() group by 1, 2, 3 order by 1, 2, 3;
-
-system start replicated sends rmt_master;
--- sleep few seconds to try rmt_slave to fetch the part and reflect this error
--- in system.part_log
-select sleep(3) format Null;
-system sync replica rmt_slave;
-
-system flush logs;
-select 'after';
-select table, event_type, error>0, countIf(error=0) from system.part_log where database = currentDatabase() group by 1, 2, 3 order by 1, 2, 3;
-
-drop table rmt_master;
-drop table rmt_slave;
diff --git a/tests/queries/0_stateless/03002_part_log_rmt_fetch_mutate_error.sh b/tests/queries/0_stateless/03002_part_log_rmt_fetch_mutate_error.sh
new file mode 100755
index 00000000000..cc8f53aafb9
--- /dev/null
+++ b/tests/queries/0_stateless/03002_part_log_rmt_fetch_mutate_error.sh
@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+# Tags: no-replicated-database, no-parallel, no-shared-merge-tree
+# SMT: The merge process is completely different from RMT
+
+CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CUR_DIR"/../shell_config.sh
+
+# shellcheck source=./mergetree_mutations.lib
+. "$CUR_DIR"/mergetree_mutations.lib
+
+set -e
+
+function wait_until()
+{
+    local q=$1 && shift
+    while [ "$($CLICKHOUSE_CLIENT -nm -q "$q")" != "1" ]; do
+        # too frequent FLUSH LOGS is too costly
+        sleep 2
+    done
+}
+
+$CLICKHOUSE_CLIENT -nm -q "
+    drop table if exists rmt_master;
+    drop table if exists rmt_slave;
+
+    create table rmt_master (key Int) engine=ReplicatedMergeTree('/clickhouse/{database}', 'master') order by tuple() settings always_fetch_merged_part=0, old_parts_lifetime=600;
+    -- prefer_fetch_merged_part_*_threshold=0, consider this table as a 'slave'
+    create table rmt_slave (key Int) engine=ReplicatedMergeTree('/clickhouse/{database}', 'slave') order by tuple() settings prefer_fetch_merged_part_time_threshold=0, prefer_fetch_merged_part_size_threshold=0, old_parts_lifetime=600;
+
+    insert into rmt_master values (1);
+
+    system sync replica rmt_master;
+    system sync replica rmt_slave;
+    system stop replicated sends rmt_master;
+    system stop pulling replication log rmt_slave;
+    alter table rmt_master update key=key+100 where 1 settings alter_sync=1;
+"
+
+# first we need to make the rmt_master execute mutation so that it will have
+# the part, and rmt_slave will consider it instead of performing mutation on
+# it's own, otherwise prefer_fetch_merged_part_*_threshold will be simply ignored
+wait_for_mutation rmt_master 0000000000
+$CLICKHOUSE_CLIENT -nm -q "system start pulling replication log rmt_slave"
+# and wait until rmt_slave to fetch the part and reflect this error in system.part_log
+wait_until "system flush logs; select count()>0 from system.part_log where table = 'rmt_slave' and database = '$CLICKHOUSE_DATABASE' and error > 0"
+$CLICKHOUSE_CLIENT -nm -q "
+    system flush logs;
+    select 'before';
+    select table, event_type, error>0, countIf(error=0) from system.part_log where database = currentDatabase() group by 1, 2, 3 order by 1, 2, 3;
+
+    system start replicated sends rmt_master;
+"
+wait_for_mutation rmt_slave 0000000000
+$CLICKHOUSE_CLIENT -nm -q "
+    system sync replica rmt_slave;
+
+    system flush logs;
+    select 'after';
+    select table, event_type, error>0, countIf(error=0) from system.part_log where database = currentDatabase() group by 1, 2, 3 order by 1, 2, 3;
+
+    drop table rmt_master;
+    drop table rmt_slave;
+"
diff --git a/tests/queries/0_stateless/03002_part_log_rmt_fetch_mutate_error.sql b/tests/queries/0_stateless/03002_part_log_rmt_fetch_mutate_error.sql
deleted file mode 100644
index d8b5ebb3148..00000000000
--- a/tests/queries/0_stateless/03002_part_log_rmt_fetch_mutate_error.sql
+++ /dev/null
@@ -1,41 +0,0 @@
--- Tags: no-replicated-database, no-parallel, no-shared-merge-tree
--- SMT: The merge process is completely different from RMT
-
-drop table if exists rmt_master;
-drop table if exists rmt_slave;
-
-create table rmt_master (key Int) engine=ReplicatedMergeTree('/clickhouse/{database}', 'master') order by tuple() settings always_fetch_merged_part=0, old_parts_lifetime=600;
--- prefer_fetch_merged_part_*_threshold=0, consider this table as a "slave"
-create table rmt_slave (key Int) engine=ReplicatedMergeTree('/clickhouse/{database}', 'slave') order by tuple() settings prefer_fetch_merged_part_time_threshold=0, prefer_fetch_merged_part_size_threshold=0, old_parts_lifetime=600;
-
-insert into rmt_master values (1);
-
-system sync replica rmt_master;
-system sync replica rmt_slave;
-system stop replicated sends rmt_master;
-system stop pulling replication log rmt_slave;
-alter table rmt_master update key=key+100 where 1 settings alter_sync=1;
-
--- first we need to make the rmt_master execute mutation so that it will have
--- the part, and rmt_slave will consider it instead of performing mutation on
--- it's own, otherwise prefer_fetch_merged_part_*_threshold will be simply ignored
-select sleep(3) format Null;
-system start pulling replication log rmt_slave;
--- and sleep few more seconds to try rmt_slave to fetch the part and reflect
--- this error in system.part_log
-select sleep(3) format Null;
-
-system flush logs;
-select 'before';
-select table, event_type, error>0, countIf(error=0) from system.part_log where database = currentDatabase() group by 1, 2, 3 order by 1, 2, 3;
-
-system start replicated sends rmt_master;
-select sleep(3) format Null;
-system sync replica rmt_slave;
-
-system flush logs;
-select 'after';
-select table, event_type, error>0, countIf(error=0) from system.part_log where database = currentDatabase() group by 1, 2, 3 order by 1, 2, 3;
-
-drop table rmt_master;
-drop table rmt_slave;

From 0310f52d3a73fa3df61ef218ce643e448edfd012 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Thu, 1 Aug 2024 10:49:09 +0200
Subject: [PATCH 1418/1488] Fix 01811_storage_buffer_flush_parameters flakiness

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 .../01811_storage_buffer_flush_parameters.sh  | 47 +++++++++++++++++++
 .../01811_storage_buffer_flush_parameters.sql | 22 ---------
 2 files changed, 47 insertions(+), 22 deletions(-)
 create mode 100755 tests/queries/0_stateless/01811_storage_buffer_flush_parameters.sh
 delete mode 100644 tests/queries/0_stateless/01811_storage_buffer_flush_parameters.sql

diff --git a/tests/queries/0_stateless/01811_storage_buffer_flush_parameters.sh b/tests/queries/0_stateless/01811_storage_buffer_flush_parameters.sh
new file mode 100755
index 00000000000..6a5949741ab
--- /dev/null
+++ b/tests/queries/0_stateless/01811_storage_buffer_flush_parameters.sh
@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+
+CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CUR_DIR"/../shell_config.sh
+
+function wait_with_limit()
+{
+    local limit=$1 && shift
+    local expr=$1 && shift
+
+    for ((i = 0; i < limit; ++i)); do
+        if eval "$expr"; then
+            break
+        fi
+        sleep 1
+    done
+}
+
+$CLICKHOUSE_CLIENT -nm -q "
+    drop table if exists data_01811;
+    drop table if exists buffer_01811;
+
+
+    create table data_01811 (key Int) Engine=Memory();
+    /* Buffer with flush_rows=1000 */
+    create table buffer_01811 (key Int) Engine=Buffer(currentDatabase(), data_01811,
+        /* num_layers= */ 1,
+        /* min_time= */   1,     /* max_time= */  86400,
+        /* min_rows= */   1e9,   /* max_rows= */  1e6,
+        /* min_bytes= */  0,     /* max_bytes= */ 4e6,
+        /* flush_time= */ 86400, /* flush_rows= */ 10, /* flush_bytes= */0
+    );
+
+    insert into buffer_01811 select * from numbers(10);
+    insert into buffer_01811 select * from numbers(10);
+"
+
+# wait for background buffer flush
+wait_with_limit 30 '[[ $($CLICKHOUSE_CLIENT -q "select count() from data_01811") -gt 0 ]]'
+
+$CLICKHOUSE_CLIENT -nm -q "select count() from data_01811"
+
+$CLICKHOUSE_CLIENT -nm -q "
+    drop table buffer_01811;
+    drop table data_01811;
+"
diff --git a/tests/queries/0_stateless/01811_storage_buffer_flush_parameters.sql b/tests/queries/0_stateless/01811_storage_buffer_flush_parameters.sql
deleted file mode 100644
index dac68ad4ae8..00000000000
--- a/tests/queries/0_stateless/01811_storage_buffer_flush_parameters.sql
+++ /dev/null
@@ -1,22 +0,0 @@
-drop table if exists data_01811;
-drop table if exists buffer_01811;
-
-create table data_01811 (key Int) Engine=Memory();
-/* Buffer with flush_rows=1000 */
-create table buffer_01811 (key Int) Engine=Buffer(currentDatabase(), data_01811,
-    /* num_layers= */ 1,
-    /* min_time= */   1,     /* max_time= */  86400,
-    /* min_rows= */   1e9,   /* max_rows= */  1e6,
-    /* min_bytes= */  0,     /* max_bytes= */ 4e6,
-    /* flush_time= */ 86400, /* flush_rows= */ 10, /* flush_bytes= */0
-);
-
-insert into buffer_01811 select * from numbers(10);
-insert into buffer_01811 select * from numbers(10);
-
--- wait for background buffer flush
-select sleep(3) format Null;
-select count() from data_01811;
-
-drop table buffer_01811;
-drop table data_01811;

From 618789d1960182ff91d614545a16d1332768e008 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Sat, 22 Jun 2024 20:42:27 +0200
Subject: [PATCH 1419/1488] Do not pass logs for keeper explicitly in the image
 to allow overriding

Right now, if you will start keeper like this:

    $ cat /tmp/local.yaml
    ---
    logger:
        log:
            "@remove": remove
        errorlog:
            "@remove": remove
        console: 1
    $ docker run --name keeper -v /tmp/local.yaml:/etc/clickhouse-keeper/conf.d/local.yaml --rm -it clickhouse/clickhouse-keeper

This will still not work, because the --log-file/--errorlog-file passed
explicitly.

So this patch removes this in case config is valid, but, if keeper is
launched with embedded config, this had been left as-is, to keep
previous behavior.

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 docker/keeper/entrypoint.sh | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/docker/keeper/entrypoint.sh b/docker/keeper/entrypoint.sh
index 1390ad9ce74..68bd0ef9d87 100644
--- a/docker/keeper/entrypoint.sh
+++ b/docker/keeper/entrypoint.sh
@@ -40,8 +40,6 @@ fi
 
 DATA_DIR="${CLICKHOUSE_DATA_DIR:-/var/lib/clickhouse}"
 LOG_DIR="${LOG_DIR:-/var/log/clickhouse-keeper}"
-LOG_PATH="${LOG_DIR}/clickhouse-keeper.log"
-ERROR_LOG_PATH="${LOG_DIR}/clickhouse-keeper.err.log"
 COORDINATION_DIR="${DATA_DIR}/coordination"
 COORDINATION_LOG_DIR="${DATA_DIR}/coordination/log"
 COORDINATION_SNAPSHOT_DIR="${DATA_DIR}/coordination/snapshots"
@@ -84,7 +82,7 @@ if [[ $# -lt 1 ]] || [[ "$1" == "--"* ]]; then
 
     # There is a config file. It is already tested with gosu (if it is readably by keeper user)
     if [ -f "$KEEPER_CONFIG" ]; then
-        exec $gosu /usr/bin/clickhouse-keeper --config-file="$KEEPER_CONFIG" --log-file="$LOG_PATH" --errorlog-file="$ERROR_LOG_PATH" "$@"
+        exec $gosu /usr/bin/clickhouse-keeper --config-file="$KEEPER_CONFIG" "$@"
     fi
 
     # There is no config file. Will use embedded one

From 205303c2868645e2aebf76f9b49ab77960579c56 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Mon, 5 Aug 2024 20:37:43 +0200
Subject: [PATCH 1420/1488] Disable more bad tests

---
 .../Nodes/tests/gtest_resource_class_fair.cpp        | 12 +++++++-----
 .../Nodes/tests/gtest_resource_class_priority.cpp    | 10 ++++++----
 .../Nodes/tests/gtest_throttler_constraint.cpp       |  2 +-
 3 files changed, 14 insertions(+), 10 deletions(-)

diff --git a/src/Common/Scheduler/Nodes/tests/gtest_resource_class_fair.cpp b/src/Common/Scheduler/Nodes/tests/gtest_resource_class_fair.cpp
index 4f0e8c80734..16cce309c2a 100644
--- a/src/Common/Scheduler/Nodes/tests/gtest_resource_class_fair.cpp
+++ b/src/Common/Scheduler/Nodes/tests/gtest_resource_class_fair.cpp
@@ -8,7 +8,9 @@ using namespace DB;
 
 using ResourceTest = ResourceTestClass;
 
-TEST(SchedulerFairPolicy, Factory)
+/// Tests disabled because of leaks in the test themselves: https://github.com/ClickHouse/ClickHouse/issues/67678
+
+TEST(DISABLED_SchedulerFairPolicy, Factory)
 {
     ResourceTest t;
 
@@ -17,7 +19,7 @@ TEST(SchedulerFairPolicy, Factory)
     EXPECT_TRUE(dynamic_cast<FairPolicy *>(fair.get()) != nullptr);
 }
 
-TEST(SchedulerFairPolicy, FairnessWeights)
+TEST(DISABLED_SchedulerFairPolicy, FairnessWeights)
 {
     ResourceTest t;
 
@@ -41,7 +43,7 @@ TEST(SchedulerFairPolicy, FairnessWeights)
     t.consumed("B", 20);
 }
 
-TEST(SchedulerFairPolicy, Activation)
+TEST(DISABLED_SchedulerFairPolicy, Activation)
 {
     ResourceTest t;
 
@@ -77,7 +79,7 @@ TEST(SchedulerFairPolicy, Activation)
     t.consumed("B", 10);
 }
 
-TEST(SchedulerFairPolicy, FairnessMaxMin)
+TEST(DISABLED_SchedulerFairPolicy, FairnessMaxMin)
 {
     ResourceTest t;
 
@@ -101,7 +103,7 @@ TEST(SchedulerFairPolicy, FairnessMaxMin)
     t.consumed("A", 20);
 }
 
-TEST(SchedulerFairPolicy, HierarchicalFairness)
+TEST(DISABLED_SchedulerFairPolicy, HierarchicalFairness)
 {
     ResourceTest t;
 
diff --git a/src/Common/Scheduler/Nodes/tests/gtest_resource_class_priority.cpp b/src/Common/Scheduler/Nodes/tests/gtest_resource_class_priority.cpp
index a447b7f6780..d3d38aae048 100644
--- a/src/Common/Scheduler/Nodes/tests/gtest_resource_class_priority.cpp
+++ b/src/Common/Scheduler/Nodes/tests/gtest_resource_class_priority.cpp
@@ -8,7 +8,9 @@ using namespace DB;
 
 using ResourceTest = ResourceTestClass;
 
-TEST(SchedulerPriorityPolicy, Factory)
+/// Tests disabled because of leaks in the test themselves: https://github.com/ClickHouse/ClickHouse/issues/67678
+
+TEST(DISABLED_SchedulerPriorityPolicy, Factory)
 {
     ResourceTest t;
 
@@ -17,7 +19,7 @@ TEST(SchedulerPriorityPolicy, Factory)
     EXPECT_TRUE(dynamic_cast<PriorityPolicy *>(prio.get()) != nullptr);
 }
 
-TEST(SchedulerPriorityPolicy, Priorities)
+TEST(DISABLED_SchedulerPriorityPolicy, Priorities)
 {
     ResourceTest t;
 
@@ -51,7 +53,7 @@ TEST(SchedulerPriorityPolicy, Priorities)
     t.consumed("C", 0);
 }
 
-TEST(SchedulerPriorityPolicy, Activation)
+TEST(DISABLED_SchedulerPriorityPolicy, Activation)
 {
     ResourceTest t;
 
@@ -92,7 +94,7 @@ TEST(SchedulerPriorityPolicy, Activation)
     t.consumed("C", 0);
 }
 
-TEST(SchedulerPriorityPolicy, SinglePriority)
+TEST(DISABLED_SchedulerPriorityPolicy, SinglePriority)
 {
     ResourceTest t;
 
diff --git a/src/Common/Scheduler/Nodes/tests/gtest_throttler_constraint.cpp b/src/Common/Scheduler/Nodes/tests/gtest_throttler_constraint.cpp
index 363e286c91c..2bc24cdb292 100644
--- a/src/Common/Scheduler/Nodes/tests/gtest_throttler_constraint.cpp
+++ b/src/Common/Scheduler/Nodes/tests/gtest_throttler_constraint.cpp
@@ -10,7 +10,7 @@ using namespace DB;
 
 using ResourceTest = ResourceTestClass;
 
-/// Test disabled because of leaks in the test themselves: https://github.com/ClickHouse/ClickHouse/issues/67678
+/// Tests disabled because of leaks in the test themselves: https://github.com/ClickHouse/ClickHouse/issues/67678
 
 TEST(DISABLED_SchedulerThrottlerConstraint, LeakyBucketConstraint)
 {

From 529f21c6b8dac22235b92c8a443abfb4b95c0f52 Mon Sep 17 00:00:00 2001
From: Max K <maxkaynov@gmail.com>
Date: Thu, 25 Jul 2024 11:32:59 +0200
Subject: [PATCH 1421/1488] CI: CheckReadyForMerge fixes

---
 .github/workflows/backport_branches.yml |  7 ++++++-
 .github/workflows/jepsen.yml            |  1 +
 .github/workflows/merge_queue.yml       |  7 ++++++-
 .github/workflows/nightly.yml           |  1 +
 .github/workflows/pull_request.yml      |  6 +++++-
 .github/workflows/release_branches.yml  |  4 +---
 tests/ci/ci_config.py                   | 28 +++++++++++++++++++++++++
 tests/ci/ci_utils.py                    |  9 ++++----
 tests/ci/merge_pr.py                    | 24 ++++++++++++---------
 9 files changed, 67 insertions(+), 20 deletions(-)

diff --git a/.github/workflows/backport_branches.yml b/.github/workflows/backport_branches.yml
index 322946ac77b..23744dc7f8f 100644
--- a/.github/workflows/backport_branches.yml
+++ b/.github/workflows/backport_branches.yml
@@ -260,13 +260,18 @@ jobs:
       - name: Finish label
         if: ${{ !failure() }}
         run: |
+          export WORKFLOW_RESULT_FILE="/tmp/workflow_results.json"
+          cat > "$WORKFLOW_RESULT_FILE" << 'EOF'
+          ${{ toJson(needs) }}
+          EOF
           cd "$GITHUB_WORKSPACE/tests/ci"
           # update mergeable check
-          python3 merge_pr.py --set-ci-status --wf-status ${{ contains(needs.*.result, 'failure') && 'failure' || 'success' }}
+          python3 merge_pr.py --set-ci-status
           # update overall ci report
           python3 finish_check.py --wf-status ${{ contains(needs.*.result, 'failure') && 'failure' || 'success' }}
           python3 merge_pr.py
       - name: Check Workflow results
+        if: ${{ !cancelled() }}
         run: |
           export WORKFLOW_RESULT_FILE="/tmp/workflow_results.json"
           cat > "$WORKFLOW_RESULT_FILE" << 'EOF'
diff --git a/.github/workflows/jepsen.yml b/.github/workflows/jepsen.yml
index 035ba2e5b98..2280b1a7305 100644
--- a/.github/workflows/jepsen.yml
+++ b/.github/workflows/jepsen.yml
@@ -64,6 +64,7 @@ jobs:
       - name: Check out repository code
         uses: ClickHouse/checkout@v1
       - name: Check Workflow results
+        if: ${{ !cancelled() }}
         run: |
           export WORKFLOW_RESULT_FILE="/tmp/workflow_results.json"
           cat >> "$WORKFLOW_RESULT_FILE" << 'EOF'
diff --git a/.github/workflows/merge_queue.yml b/.github/workflows/merge_queue.yml
index 64083668719..629cf79770e 100644
--- a/.github/workflows/merge_queue.yml
+++ b/.github/workflows/merge_queue.yml
@@ -103,9 +103,14 @@ jobs:
       - name: Check and set merge status
         if: ${{ needs.StyleCheck.result == 'success' }}
         run: |
+          export WORKFLOW_RESULT_FILE="/tmp/workflow_results.json"
+          cat > "$WORKFLOW_RESULT_FILE" << 'EOF'
+          ${{ toJson(needs) }}
+          EOF
           cd "$GITHUB_WORKSPACE/tests/ci"
-          python3 merge_pr.py --set-ci-status --wf-status ${{ contains(needs.*.result, 'failure') && 'failure' || 'success' }}
+          python3 merge_pr.py --set-ci-status
       - name: Check Workflow results
+        if: ${{ !cancelled() }}
         run: |
           export WORKFLOW_RESULT_FILE="/tmp/workflow_results.json"
           cat > "$WORKFLOW_RESULT_FILE" << 'EOF'
diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
index ea9c125db70..36fea39686f 100644
--- a/.github/workflows/nightly.yml
+++ b/.github/workflows/nightly.yml
@@ -52,6 +52,7 @@ jobs:
       - name: Check out repository code
         uses: ClickHouse/checkout@v1
       - name: Check Workflow results
+        if: ${{ !cancelled() }}
         run: |
           export WORKFLOW_RESULT_FILE="/tmp/workflow_results.json"
           cat > "$WORKFLOW_RESULT_FILE" << 'EOF'
diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
index 071f0f1e20a..dbc740ebc1b 100644
--- a/.github/workflows/pull_request.yml
+++ b/.github/workflows/pull_request.yml
@@ -170,7 +170,11 @@ jobs:
         if: ${{ needs.StyleCheck.result == 'success' }}
         run: |
           cd "$GITHUB_WORKSPACE/tests/ci"
-          python3 merge_pr.py --set-ci-status --wf-status ${{ contains(needs.*.result, 'failure') && 'failure' || 'success' }}
+          export WORKFLOW_RESULT_FILE="/tmp/workflow_results.json"
+          cat > "$WORKFLOW_RESULT_FILE" << 'EOF'
+          ${{ toJson(needs) }}
+          EOF
+          python3 merge_pr.py --set-ci-status
       - name: Check Workflow results
         uses: ./.github/actions/check_workflow
         with:
diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml
index b79208b03a6..a5cd6321e8c 100644
--- a/.github/workflows/release_branches.yml
+++ b/.github/workflows/release_branches.yml
@@ -481,12 +481,10 @@ jobs:
       - name: Finish label
         if: ${{ !failure() }}
         run: |
-          cd "$GITHUB_WORKSPACE/tests/ci"
-          # update mergeable check
-          python3 merge_pr.py --set-ci-status --wf-status ${{ contains(needs.*.result, 'failure') && 'failure' || 'success' }}
           # update overall ci report
           python3 finish_check.py --wf-status ${{ contains(needs.*.result, 'failure') && 'failure' || 'success' }}
       - name: Check Workflow results
+        if: ${{ !cancelled() }}
         run: |
           export WORKFLOW_RESULT_FILE="/tmp/workflow_results.json"
           cat > "$WORKFLOW_RESULT_FILE" << 'EOF'
diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py
index c031ca9b805..d34405e7692 100644
--- a/tests/ci/ci_config.py
+++ b/tests/ci/ci_config.py
@@ -687,6 +687,34 @@ class CI:
         assert res, f"not a build [{build_name}] or invalid JobConfig"
         return res
 
+    @classmethod
+    def is_workflow_ok(cls) -> bool:
+        # TODO: temporary method to make Mergeable check working
+        res = cls.GH.get_workflow_results()
+        if not res:
+            print("ERROR: no workflow results found")
+            return False
+        for workflow_job, workflow_data in res.items():
+            status = workflow_data["result"]
+            if status in (
+                cls.GH.ActionStatuses.SUCCESS,
+                cls.GH.ActionStatuses.SKIPPED,
+            ):
+                print(f"Workflow status for [{workflow_job}] is [{status}] - continue")
+            elif status in (cls.GH.ActionStatuses.FAILURE,):
+                if workflow_job in (
+                    WorkflowStages.TESTS_2,
+                    WorkflowStages.TESTS_2_WW,
+                ):
+                    print(
+                        f"Failed Workflow status for [{workflow_job}], it's not required - continue"
+                    )
+                    continue
+
+                print(f"Failed Workflow status for [{workflow_job}]")
+                return False
+        return True
+
 
 if __name__ == "__main__":
     parser = ArgumentParser(
diff --git a/tests/ci/ci_utils.py b/tests/ci/ci_utils.py
index dae1520afb6..0a2dd5e35ba 100644
--- a/tests/ci/ci_utils.py
+++ b/tests/ci/ci_utils.py
@@ -94,9 +94,10 @@ class GH:
         FAILURE = "failure"
         PENDING = "pending"
         SUCCESS = "success"
+        SKIPPED = "skipped"
 
     @classmethod
-    def _get_workflow_results(cls):
+    def get_workflow_results(cls):
         if not Path(Envs.WORKFLOW_RESULT_FILE).exists():
             print(
                 f"ERROR: Failed to get workflow results from file [{Envs.WORKFLOW_RESULT_FILE}]"
@@ -115,13 +116,13 @@ class GH:
 
     @classmethod
     def print_workflow_results(cls):
-        res = cls._get_workflow_results()
+        res = cls.get_workflow_results()
         results = [f"{job}: {data['result']}" for job, data in res.items()]
         cls.print_in_group("Workflow results", results)
 
     @classmethod
     def is_workflow_ok(cls) -> bool:
-        res = cls._get_workflow_results()
+        res = cls.get_workflow_results()
         for _job, data in res.items():
             if data["result"] == "failure":
                 return False
@@ -129,7 +130,7 @@ class GH:
 
     @classmethod
     def get_workflow_job_result(cls, wf_job_name: str) -> Optional[str]:
-        res = cls._get_workflow_results()
+        res = cls.get_workflow_results()
         if wf_job_name in res:
             return res[wf_job_name]["result"]  # type: ignore
         else:
diff --git a/tests/ci/merge_pr.py b/tests/ci/merge_pr.py
index 13c7537a84b..ddeb76adf7e 100644
--- a/tests/ci/merge_pr.py
+++ b/tests/ci/merge_pr.py
@@ -23,7 +23,7 @@ from commit_status_helper import (
 from get_robot_token import get_best_robot_token
 from github_helper import GitHub, NamedUser, PullRequest, Repository
 from pr_info import PRInfo
-from report import SUCCESS, FAILURE
+from report import SUCCESS
 from env_helper import GITHUB_UPSTREAM_REPOSITORY, GITHUB_REPOSITORY
 from synchronizer_utils import SYNC_BRANCH_PREFIX
 from ci_config import CI
@@ -248,23 +248,27 @@ def main():
     repo = gh.get_repo(args.repo)
 
     if args.set_ci_status:
+        CI.GH.print_workflow_results()
         # set Mergeable check status and exit
-        assert args.wf_status in (FAILURE, SUCCESS)
         commit = get_commit(gh, args.pr_info.sha)
         statuses = get_commit_filtered_statuses(commit)
 
         has_failed_statuses = False
-        has_native_failed_status = False
         for status in statuses:
             print(f"Check status [{status.context}], [{status.state}]")
-            if CI.is_required(status.context) and status.state != SUCCESS:
-                print(f"WARNING: Failed status [{status.context}], [{status.state}]")
+            if (
+                CI.is_required(status.context)
+                and status.state != SUCCESS
+                and status.context != CI.StatusNames.SYNC
+            ):
+                print(
+                    f"WARNING: Not success status [{status.context}], [{status.state}]"
+                )
                 has_failed_statuses = True
-                if status.context != CI.StatusNames.SYNC:
-                    has_native_failed_status = True
 
-        if args.wf_status == SUCCESS or has_failed_statuses:
-            # set Mergeable check if workflow is successful (green)
+        workflow_ok = CI.is_workflow_ok()
+        if workflow_ok or has_failed_statuses:
+            # set Mergeable Check if workflow is successful (green)
             # or if we have GH statuses with failures (red)
             #    to avoid false-green on a died runner
             state = trigger_mergeable_check(
@@ -283,7 +287,7 @@ def main():
             print(
                 "Workflow failed but no failed statuses found (died runner?) - cannot set Mergeable Check status"
             )
-        if args.wf_status == SUCCESS and not has_native_failed_status:
+        if workflow_ok and not has_failed_statuses:
             sys.exit(0)
         else:
             sys.exit(1)

From bf2e679a51d843e03b52e0dc2853667c8b03978d Mon Sep 17 00:00:00 2001
From: Michael Kolupaev <michael.kolupaev@clickhouse.com>
Date: Mon, 5 Aug 2024 19:40:25 +0000
Subject: [PATCH 1422/1488] no-msan 00314_sample_factor_virtual_column

---
 .../0_stateless/00314_sample_factor_virtual_column.sql        | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tests/queries/0_stateless/00314_sample_factor_virtual_column.sql b/tests/queries/0_stateless/00314_sample_factor_virtual_column.sql
index 6e3dc019069..b8ac5e733ed 100644
--- a/tests/queries/0_stateless/00314_sample_factor_virtual_column.sql
+++ b/tests/queries/0_stateless/00314_sample_factor_virtual_column.sql
@@ -1,3 +1,7 @@
+-- Tags: no-msan
+--          ^
+--          makes SELECTs extremely slow sometimes for some reason: "Aggregated. 1000000 to 1 rows (from 7.63 MiB) in 242.829221645 sec."
+
 DROP TABLE IF EXISTS sample_00314_1;
 DROP TABLE IF EXISTS sample_00314_2;
 DROP TABLE IF EXISTS sample_merge_00314;

From 4829ba31d3e8f874beb9e626df801021673008fd Mon Sep 17 00:00:00 2001
From: Nikita Mikhaylov <nikitamikhaylov@clickhouse.com>
Date: Mon, 5 Aug 2024 21:18:53 +0000
Subject: [PATCH 1423/1488] Actually fix a test

---
 .../test_distributed_type_object/test.py      |  3 ++
 ...allel_replicas_crash_after_refactoring.sql | 33 -------------------
 2 files changed, 3 insertions(+), 33 deletions(-)
 delete mode 100644 tests/queries/0_stateless/03215_parallel_replicas_crash_after_refactoring.sql

diff --git a/tests/integration/test_distributed_type_object/test.py b/tests/integration/test_distributed_type_object/test.py
index e774876bc8b..e274bd6b774 100644
--- a/tests/integration/test_distributed_type_object/test.py
+++ b/tests/integration/test_distributed_type_object/test.py
@@ -31,6 +31,9 @@ def started_cluster():
 
 
 def test_distributed_type_object(started_cluster):
+    node1.query("TRUNCATE TABLE local_table")
+    node2.query("TRUNCATE TABLE local_table")
+
     node1.query(
         'INSERT INTO local_table FORMAT JSONEachRow {"id": 1, "data": {"k1": 10}}'
     )
diff --git a/tests/queries/0_stateless/03215_parallel_replicas_crash_after_refactoring.sql b/tests/queries/0_stateless/03215_parallel_replicas_crash_after_refactoring.sql
deleted file mode 100644
index cae4fa0f0df..00000000000
--- a/tests/queries/0_stateless/03215_parallel_replicas_crash_after_refactoring.sql
+++ /dev/null
@@ -1,33 +0,0 @@
--- Tags: disabled
-
-DROP TABLE IF EXISTS t1__fuzz_5;
-
-CREATE TABLE t1__fuzz_5
-(
-    `k` Int16,
-    `v` Nullable(UInt8)
-)
-ENGINE = MergeTree
-ORDER BY k
-SETTINGS index_granularity = 10;
-
-INSERT INTO t1__fuzz_5 SELECT
-    number,
-    number
-FROM numbers(1000);
-
-INSERT INTO t1__fuzz_5 SELECT
-    number,
-    number
-FROM numbers(1000, 1000);
-
-INSERT INTO t1__fuzz_5 SELECT
-    number,
-    number
-FROM numbers(2000, 1000);
-
-SET receive_timeout = 10., receive_data_timeout_ms = 10000, allow_suspicious_low_cardinality_types = true, parallel_distributed_insert_select = 2, log_queries = true, table_function_remote_max_addresses = 200, max_execution_time = 10., max_memory_usage = 10000000000, log_comment = '/workspace/ch/tests/queries/0_stateless/02869_parallel_replicas_read_from_several.sql', send_logs_level = 'warning', prefer_localhost_replica = false, allow_introspection_functions = true, use_parallel_replicas = 257, max_parallel_replicas = 65535, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_single_task_marks_count_multiplier = -0., parallel_replicas_for_non_replicated_merge_tree = true;
-
-SELECT max(k) IGNORE NULLS FROM t1__fuzz_5 WITH TOTALS SETTINGS use_parallel_replicas = 257, max_parallel_replicas = 65535, prefer_localhost_replica = 0, cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost', parallel_replicas_single_task_marks_count_multiplier = -0;
-
-DROP TABLE IF EXISTS t1__fuzz_5;

From 5cdf12a9277993f7adf97c5083c7e92f914a5ae7 Mon Sep 17 00:00:00 2001
From: robot-clickhouse <robot-clickhouse@users.noreply.github.com>
Date: Sun, 4 Aug 2024 22:31:17 +0000
Subject: [PATCH 1424/1488] Automatic style fix


From 29f06e8037316d61034700eff8e41c9b8730f317 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 5 Aug 2024 15:17:32 +0200
Subject: [PATCH 1425/1488] Update
 00002_log_and_exception_messages_formatting.sql

---
 .../0_stateless/00002_log_and_exception_messages_formatting.sql | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql b/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql
index b806aff3a00..efd961a0fda 100644
--- a/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql
+++ b/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql
@@ -1,5 +1,7 @@
 -- Tags: no-parallel, no-fasttest, no-ubsan, no-batch, no-flaky-check
 -- no-parallel because we want to run this test when most of the other tests already passed
+-- This is not a regular test. It is intended to run once after other tests to validate certain statistics about the whole test runs.
+-- TODO: I advice to put in inside clickhouse-test instead.
 
 -- If this test fails, see the "Top patterns of log messages" diagnostics in the end of run.log
 

From 61aff5efae873bde8631919386ba7f5c9db50af1 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Mon, 5 Aug 2024 15:17:48 +0200
Subject: [PATCH 1426/1488] Update
 00002_log_and_exception_messages_formatting.sql

---
 .../0_stateless/00002_log_and_exception_messages_formatting.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql b/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql
index efd961a0fda..53321afc94c 100644
--- a/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql
+++ b/tests/queries/0_stateless/00002_log_and_exception_messages_formatting.sql
@@ -1,7 +1,7 @@
 -- Tags: no-parallel, no-fasttest, no-ubsan, no-batch, no-flaky-check
 -- no-parallel because we want to run this test when most of the other tests already passed
 -- This is not a regular test. It is intended to run once after other tests to validate certain statistics about the whole test runs.
--- TODO: I advice to put in inside clickhouse-test instead.
+-- TODO: I advise to put in inside clickhouse-test instead.
 
 -- If this test fails, see the "Top patterns of log messages" diagnostics in the end of run.log
 

From cfcd9fe096de7e136093e20ac283569c23721714 Mon Sep 17 00:00:00 2001
From: Michael Kolupaev <michael.kolupaev@clickhouse.com>
Date: Mon, 5 Aug 2024 21:57:23 +0000
Subject: [PATCH 1427/1488] Conflicts

---
 src/Interpreters/ExpressionActions.h              |  2 +-
 src/Storages/StorageMergeTreeIndex.cpp            | 15 +++++++--------
 src/Storages/StorageMergeTreeIndex.h              |  2 +-
 src/Storages/System/IStorageSystemOneBlock.cpp    |  4 ++--
 src/Storages/System/StorageSystemColumns.cpp      |  6 +++---
 .../System/StorageSystemDataSkippingIndices.cpp   | 11 +++++------
 src/Storages/System/StorageSystemReplicas.cpp     | 11 +++++------
 src/Storages/VirtualColumnUtils.h                 |  7 +++----
 8 files changed, 27 insertions(+), 31 deletions(-)

diff --git a/src/Interpreters/ExpressionActions.h b/src/Interpreters/ExpressionActions.h
index 6ff39ee07f7..7652fe49eab 100644
--- a/src/Interpreters/ExpressionActions.h
+++ b/src/Interpreters/ExpressionActions.h
@@ -102,7 +102,7 @@ public:
     ///
     /// @param allow_duplicates_in_input - actions are allowed to have
     /// duplicated input (that will refer into the block). This is needed for
-    /// preliminary query filtering (filterBlockWithDAG()), because they just
+    /// preliminary query filtering (filterBlockWithExpression()), because they just
     /// pass available virtual columns, which cannot be moved in case they are
     /// used multiple times.
     void execute(Block & block, size_t & num_rows, bool dry_run = false, bool allow_duplicates_in_input = false) const;
diff --git a/src/Storages/StorageMergeTreeIndex.cpp b/src/Storages/StorageMergeTreeIndex.cpp
index 90d01d356e9..15728290f19 100644
--- a/src/Storages/StorageMergeTreeIndex.cpp
+++ b/src/Storages/StorageMergeTreeIndex.cpp
@@ -275,7 +275,7 @@ public:
 private:
     std::shared_ptr<StorageMergeTreeIndex> storage;
     Poco::Logger * log;
-    ActionsDAGPtr virtual_columns_filter;
+    ExpressionActionsPtr virtual_columns_filter;
 };
 
 void ReadFromMergeTreeIndex::applyFilters(ActionDAGNodes added_filter_nodes)
@@ -289,10 +289,9 @@ void ReadFromMergeTreeIndex::applyFilters(ActionDAGNodes added_filter_nodes)
             { {}, std::make_shared<DataTypeString>(), StorageMergeTreeIndex::part_name_column.name },
         };
 
-        virtual_columns_filter = VirtualColumnUtils::splitFilterDagForAllowedInputs(filter_actions_dag->getOutputs().at(0), &block_to_filter);
-
-        if (virtual_columns_filter)
-            VirtualColumnUtils::buildSetsForDAG(virtual_columns_filter, context);
+        auto dag = VirtualColumnUtils::splitFilterDagForAllowedInputs(filter_actions_dag->getOutputs().at(0), &block_to_filter);
+        if (dag)
+            virtual_columns_filter = VirtualColumnUtils::buildFilterExpression(std::move(*dag), context);
     }
 }
 
@@ -345,7 +344,7 @@ void StorageMergeTreeIndex::read(
 
 void ReadFromMergeTreeIndex::initializePipeline(QueryPipelineBuilder & pipeline, const BuildQueryPipelineSettings &)
 {
-    auto filtered_parts = storage->getFilteredDataParts(virtual_columns_filter, context);
+    auto filtered_parts = storage->getFilteredDataParts(virtual_columns_filter);
 
     LOG_DEBUG(log, "Reading index{}from {} parts of table {}",
         storage->with_marks ? " with marks " : " ",
@@ -355,7 +354,7 @@ void ReadFromMergeTreeIndex::initializePipeline(QueryPipelineBuilder & pipeline,
     pipeline.init(Pipe(std::make_shared<MergeTreeIndexSource>(getOutputStream().header, storage->key_sample_block, std::move(filtered_parts), context, storage->with_marks)));
 }
 
-MergeTreeData::DataPartsVector StorageMergeTreeIndex::getFilteredDataParts(ActionsDAGPtr virtual_columns_filter, const ContextPtr & context) const
+MergeTreeData::DataPartsVector StorageMergeTreeIndex::getFilteredDataParts(const ExpressionActionsPtr & virtual_columns_filter) const
 {
     if (!virtual_columns_filter)
         return data_parts;
@@ -365,7 +364,7 @@ MergeTreeData::DataPartsVector StorageMergeTreeIndex::getFilteredDataParts(Actio
         all_part_names->insert(part->name);
 
     Block filtered_block{{std::move(all_part_names), std::make_shared<DataTypeString>(), part_name_column.name}};
-    VirtualColumnUtils::filterBlockWithDAG(virtual_columns_filter, filtered_block, context);
+    VirtualColumnUtils::filterBlockWithExpression(virtual_columns_filter, filtered_block);
 
     if (!filtered_block.rows())
         return {};
diff --git a/src/Storages/StorageMergeTreeIndex.h b/src/Storages/StorageMergeTreeIndex.h
index 652a2d6eeaf..ed8274d7d92 100644
--- a/src/Storages/StorageMergeTreeIndex.h
+++ b/src/Storages/StorageMergeTreeIndex.h
@@ -36,7 +36,7 @@ public:
 private:
     friend class ReadFromMergeTreeIndex;
 
-    MergeTreeData::DataPartsVector getFilteredDataParts(ActionsDAGPtr virtual_columns_filter, const ContextPtr & context) const;
+    MergeTreeData::DataPartsVector getFilteredDataParts(const ExpressionActionsPtr & virtual_columns_filter) const;
 
     StoragePtr source_table;
     bool with_marks;
diff --git a/src/Storages/System/IStorageSystemOneBlock.cpp b/src/Storages/System/IStorageSystemOneBlock.cpp
index 308b34510ea..b8f32fcdb83 100644
--- a/src/Storages/System/IStorageSystemOneBlock.cpp
+++ b/src/Storages/System/IStorageSystemOneBlock.cpp
@@ -45,7 +45,7 @@ public:
 private:
     std::shared_ptr<IStorageSystemOneBlock> storage;
     std::vector<UInt8> columns_mask;
-    ActionsDAGPtr filter;
+    std::optional<ActionsDAG> filter;
 };
 
 void IStorageSystemOneBlock::read(
@@ -106,7 +106,7 @@ void ReadFromSystemOneBlock::applyFilters(ActionDAGNodes added_filter_nodes)
 
     /// Must prepare sets here, initializePipeline() would be too late, see comment on FutureSetFromSubquery.
     if (filter)
-        VirtualColumnUtils::buildSetsForDAG(filter, context);
+        VirtualColumnUtils::buildSetsForDAG(*filter, context);
 }
 
 }
diff --git a/src/Storages/System/StorageSystemColumns.cpp b/src/Storages/System/StorageSystemColumns.cpp
index 6b3e0094562..03c569303c5 100644
--- a/src/Storages/System/StorageSystemColumns.cpp
+++ b/src/Storages/System/StorageSystemColumns.cpp
@@ -338,7 +338,7 @@ private:
     std::shared_ptr<StorageSystemColumns> storage;
     std::vector<UInt8> columns_mask;
     const size_t max_block_size;
-    ActionsDAGPtr virtual_columns_filter;
+    std::optional<ActionsDAG> virtual_columns_filter;
 };
 
 void ReadFromSystemColumns::applyFilters(ActionDAGNodes added_filter_nodes)
@@ -355,7 +355,7 @@ void ReadFromSystemColumns::applyFilters(ActionDAGNodes added_filter_nodes)
 
         /// Must prepare sets here, initializePipeline() would be too late, see comment on FutureSetFromSubquery.
         if (virtual_columns_filter)
-            VirtualColumnUtils::buildSetsForDAG(virtual_columns_filter, context);
+            VirtualColumnUtils::buildSetsForDAG(*virtual_columns_filter, context);
     }
 }
 
@@ -468,7 +468,7 @@ void ReadFromSystemColumns::initializePipeline(QueryPipelineBuilder & pipeline,
 
     /// Filter block with `database` and `table` columns.
     if (virtual_columns_filter)
-        VirtualColumnUtils::filterBlockWithDAG(virtual_columns_filter, block_to_filter, context);
+        VirtualColumnUtils::filterBlockWithPredicate(virtual_columns_filter->getOutputs().at(0), block_to_filter, context);
 
     if (!block_to_filter.rows())
     {
diff --git a/src/Storages/System/StorageSystemDataSkippingIndices.cpp b/src/Storages/System/StorageSystemDataSkippingIndices.cpp
index a6bba44e257..a41771df406 100644
--- a/src/Storages/System/StorageSystemDataSkippingIndices.cpp
+++ b/src/Storages/System/StorageSystemDataSkippingIndices.cpp
@@ -214,7 +214,7 @@ private:
     std::shared_ptr<StorageSystemDataSkippingIndices> storage;
     std::vector<UInt8> columns_mask;
     const size_t max_block_size;
-    ActionsDAGPtr virtual_columns_filter;
+    ExpressionActionsPtr virtual_columns_filter;
 };
 
 void ReadFromSystemDataSkippingIndices::applyFilters(ActionDAGNodes added_filter_nodes)
@@ -228,10 +228,9 @@ void ReadFromSystemDataSkippingIndices::applyFilters(ActionDAGNodes added_filter
             { ColumnString::create(), std::make_shared<DataTypeString>(), "database" },
         };
 
-        virtual_columns_filter = VirtualColumnUtils::splitFilterDagForAllowedInputs(filter_actions_dag->getOutputs().at(0), &block_to_filter);
-
-        if (virtual_columns_filter)
-            VirtualColumnUtils::buildSetsForDAG(virtual_columns_filter, context);
+        auto dag = VirtualColumnUtils::splitFilterDagForAllowedInputs(filter_actions_dag->getOutputs().at(0), &block_to_filter);
+        if (dag)
+            virtual_columns_filter = VirtualColumnUtils::buildFilterExpression(std::move(*dag), context);
     }
 }
 
@@ -279,7 +278,7 @@ void ReadFromSystemDataSkippingIndices::initializePipeline(QueryPipelineBuilder
     /// Condition on "database" in a query acts like an index.
     Block block { ColumnWithTypeAndName(std::move(column), std::make_shared<DataTypeString>(), "database") };
     if (virtual_columns_filter)
-        VirtualColumnUtils::filterBlockWithDAG(virtual_columns_filter, block, context);
+        VirtualColumnUtils::filterBlockWithExpression(virtual_columns_filter, block);
 
     ColumnPtr & filtered_databases = block.getByPosition(0).column;
     pipeline.init(Pipe(std::make_shared<DataSkippingIndicesSource>(
diff --git a/src/Storages/System/StorageSystemReplicas.cpp b/src/Storages/System/StorageSystemReplicas.cpp
index 9fb4dc5ed6f..724e4bd3f77 100644
--- a/src/Storages/System/StorageSystemReplicas.cpp
+++ b/src/Storages/System/StorageSystemReplicas.cpp
@@ -285,7 +285,7 @@ private:
     const bool with_zk_fields;
     const size_t max_block_size;
     std::shared_ptr<StorageSystemReplicasImpl> impl;
-    ActionsDAGPtr virtual_columns_filter;
+    ExpressionActionsPtr virtual_columns_filter;
 };
 
 void ReadFromSystemReplicas::applyFilters(ActionDAGNodes added_filter_nodes)
@@ -301,10 +301,9 @@ void ReadFromSystemReplicas::applyFilters(ActionDAGNodes added_filter_nodes)
             { ColumnString::create(), std::make_shared<DataTypeString>(), "engine" },
         };
 
-        virtual_columns_filter = VirtualColumnUtils::splitFilterDagForAllowedInputs(filter_actions_dag->getOutputs().at(0), &block_to_filter);
-
-        if (virtual_columns_filter)
-            VirtualColumnUtils::buildSetsForDAG(virtual_columns_filter, context);
+        auto dag = VirtualColumnUtils::splitFilterDagForAllowedInputs(filter_actions_dag->getOutputs().at(0), &block_to_filter);
+        if (dag)
+            virtual_columns_filter = VirtualColumnUtils::buildFilterExpression(std::move(*dag), context);
     }
 }
 
@@ -443,7 +442,7 @@ void ReadFromSystemReplicas::initializePipeline(QueryPipelineBuilder & pipeline,
         };
 
         if (virtual_columns_filter)
-            VirtualColumnUtils::filterBlockWithDAG(virtual_columns_filter, filtered_block, context);
+            VirtualColumnUtils::filterBlockWithExpression(virtual_columns_filter, filtered_block);
 
         if (!filtered_block.rows())
         {
diff --git a/src/Storages/VirtualColumnUtils.h b/src/Storages/VirtualColumnUtils.h
index 3f3f93eccf7..abf46dc23a4 100644
--- a/src/Storages/VirtualColumnUtils.h
+++ b/src/Storages/VirtualColumnUtils.h
@@ -20,15 +20,14 @@ namespace VirtualColumnUtils
 
 /// The filtering functions are tricky to use correctly.
 /// There are 2 ways:
-///  1. Call filterBlockWithPredicate() or filterBlockWithDAG() inside SourceStepWithFilter::applyFilters().
+///  1. Call filterBlockWithPredicate() or filterBlockWithExpression() inside SourceStepWithFilter::applyFilters().
 ///  2. Call splitFilterDagForAllowedInputs() and buildSetsForDAG() inside SourceStepWithFilter::applyFilters().
-///     Then call filterBlockWithPredicate() or filterBlockWithDAG() in initializePipeline().
+///     Then call filterBlockWithPredicate() or filterBlockWithExpression() in initializePipeline().
 ///
 /// Otherwise calling filter*() outside applyFilters() will throw "Not-ready Set is passed"
 /// if there are subqueries.
 
-/// Similar to filterBlockWithQuery, but uses ActionsDAG as a predicate.
-/// Basically it is filterBlockWithDAG(splitFilterDagForAllowedInputs).
+/// Similar to filterBlockWithExpression(buildFilterExpression(splitFilterDagForAllowedInputs(...))).
 void filterBlockWithPredicate(const ActionsDAG::Node * predicate, Block & block, ContextPtr context);
 
 /// Just filters block. Block should contain all the required columns.

From a05ead3f77d52f6f168c0bb483633a63ffbb3997 Mon Sep 17 00:00:00 2001
From: Michael Kolupaev <michael.kolupaev@clickhouse.com>
Date: Mon, 5 Aug 2024 22:04:53 +0000
Subject: [PATCH 1428/1488] Conflict

---
 src/Core/SettingsChangesHistory.cpp | 260 ----------------------------
 1 file changed, 260 deletions(-)

diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp
index b80e4bf3086..5b94391bade 100644
--- a/src/Core/SettingsChangesHistory.cpp
+++ b/src/Core/SettingsChangesHistory.cpp
@@ -57,266 +57,6 @@ String ClickHouseVersion::toString() const
 /// Note: please check if the key already exists to prevent duplicate entries.
 static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory::SettingsChanges>> settings_changes_history_initializer =
 {
-    {"24.7", {{"output_format_parquet_write_page_index", false, true, "Add a possibility to write page index into parquet files."},
-              {"output_format_binary_encode_types_in_binary_format", false, false, "Added new setting to allow to write type names in binary format in RowBinaryWithNamesAndTypes output format"},
-              {"input_format_binary_decode_types_in_binary_format", false, false, "Added new setting to allow to read type names in binary format in RowBinaryWithNamesAndTypes input format"},
-              {"output_format_native_encode_types_in_binary_format", false, false, "Added new setting to allow to write type names in binary format in Native output format"},
-              {"input_format_native_decode_types_in_binary_format", false, false, "Added new setting to allow to read type names in binary format in Native output format"},
-              {"read_in_order_use_buffering", false, true, "Use buffering before merging while reading in order of primary key"},
-              {"enable_named_columns_in_function_tuple", false, true, "Generate named tuples in function tuple() when all names are unique and can be treated as unquoted identifiers."},
-              {"optimize_trivial_insert_select", true, false, "The optimization does not make sense in many cases."},
-              {"dictionary_validate_primary_key_type", false, false, "Validate primary key type for dictionaries. By default id type for simple layouts will be implicitly converted to UInt64."},
-              {"collect_hash_table_stats_during_joins", false, true, "New setting."},
-              {"max_size_to_preallocate_for_joins", 0, 100'000'000, "New setting."},
-              {"input_format_orc_read_use_writer_time_zone", false, false, "Whether use the writer's time zone in ORC stripe for ORC row reader, the default ORC row reader's time zone is GMT."},
-              {"lightweight_mutation_projection_mode", "throw", "throw", "When lightweight delete happens on a table with projection(s), the possible operations include throw the exception as projection exists, or drop all projection related to this table then do lightweight delete."},
-              {"database_replicated_allow_heavy_create", true, false, "Long-running DDL queries (CREATE AS SELECT and POPULATE) for Replicated database engine was forbidden"},
-              {"query_plan_merge_filters", false, false, "Allow to merge filters in the query plan"},
-              {"azure_sdk_max_retries", 10, 10, "Maximum number of retries in azure sdk"},
-              {"azure_sdk_retry_initial_backoff_ms", 10, 10, "Minimal backoff between retries in azure sdk"},
-              {"azure_sdk_retry_max_backoff_ms", 1000, 1000, "Maximal backoff between retries in azure sdk"},
-              {"merge_tree_min_bytes_per_task_for_remote_reading", 4194304, 2097152, "Value is unified with `filesystem_prefetch_min_bytes_for_single_read_task`"},
-              {"ignore_on_cluster_for_replicated_named_collections_queries", false, false, "Ignore ON CLUSTER clause for replicated named collections management queries."},
-              {"backup_restore_s3_retry_attempts", 1000,1000, "Setting for Aws::Client::RetryStrategy, Aws::Client does retries itself, 0 means no retries. It takes place only for backup/restore."},
-              {"postgresql_connection_attempt_timeout", 2, 2, "Allow to control 'connect_timeout' parameter of PostgreSQL connection."},
-              {"postgresql_connection_pool_retries", 2, 2, "Allow to control the number of retries in PostgreSQL connection pool."},
-              {"restore_replace_external_table_functions_to_null", false, false, "New setting."},
-              {"restore_replace_external_engines_to_null", false, false, "New setting."}
-              }},
-    {"24.6", {{"materialize_skip_indexes_on_insert", true, true, "Added new setting to allow to disable materialization of skip indexes on insert"},
-              {"materialize_statistics_on_insert", true, true, "Added new setting to allow to disable materialization of statistics on insert"},
-              {"input_format_parquet_use_native_reader", false, false, "When reading Parquet files, to use native reader instead of arrow reader."},
-              {"hdfs_throw_on_zero_files_match", false, false, "Allow to throw an error when ListObjects request cannot match any files in HDFS engine instead of empty query result"},
-              {"azure_throw_on_zero_files_match", false, false, "Allow to throw an error when ListObjects request cannot match any files in AzureBlobStorage engine instead of empty query result"},
-              {"s3_validate_request_settings", true, true, "Allow to disable S3 request settings validation"},
-              {"allow_experimental_full_text_index", false, false, "Enable experimental full-text index"},
-              {"azure_skip_empty_files", false, false, "Allow to skip empty files in azure table engine"},
-              {"hdfs_ignore_file_doesnt_exist", false, false, "Allow to return 0 rows when the requested files don't exist instead of throwing an exception in HDFS table engine"},
-              {"azure_ignore_file_doesnt_exist", false, false, "Allow to return 0 rows when the requested files don't exist instead of throwing an exception in AzureBlobStorage table engine"},
-              {"s3_ignore_file_doesnt_exist", false, false, "Allow to return 0 rows when the requested files don't exist instead of throwing an exception in S3 table engine"},
-              {"s3_max_part_number", 10000, 10000, "Maximum part number number for s3 upload part"},
-              {"s3_max_single_operation_copy_size", 32 * 1024 * 1024, 32 * 1024 * 1024, "Maximum size for a single copy operation in s3"},
-              {"input_format_parquet_max_block_size", 8192, DEFAULT_BLOCK_SIZE, "Increase block size for parquet reader."},
-              {"input_format_parquet_prefer_block_bytes", 0, DEFAULT_BLOCK_SIZE * 256, "Average block bytes output by parquet reader."},
-              {"enable_blob_storage_log", true, true, "Write information about blob storage operations to system.blob_storage_log table"},
-              {"allow_deprecated_snowflake_conversion_functions", true, false, "Disabled deprecated functions snowflakeToDateTime[64] and dateTime[64]ToSnowflake."},
-              {"allow_statistic_optimize", false, false, "Old setting which popped up here being renamed."},
-              {"allow_experimental_statistic", false, false, "Old setting which popped up here being renamed."},
-              {"allow_statistics_optimize", false, false, "The setting was renamed. The previous name is `allow_statistic_optimize`."},
-              {"allow_experimental_statistics", false, false, "The setting was renamed. The previous name is `allow_experimental_statistic`."},
-              {"enable_vertical_final", false, true, "Enable vertical final by default again after fixing bug"},
-              {"parallel_replicas_custom_key_range_lower", 0, 0, "Add settings to control the range filter when using parallel replicas with dynamic shards"},
-              {"parallel_replicas_custom_key_range_upper", 0, 0, "Add settings to control the range filter when using parallel replicas with dynamic shards. A value of 0 disables the upper limit"},
-              {"output_format_pretty_display_footer_column_names", 0, 1, "Add a setting to display column names in the footer if there are many rows. Threshold value is controlled by output_format_pretty_display_footer_column_names_min_rows."},
-              {"output_format_pretty_display_footer_column_names_min_rows", 0, 50, "Add a setting to control the threshold value for setting output_format_pretty_display_footer_column_names_min_rows. Default 50."},
-              {"output_format_csv_serialize_tuple_into_separate_columns", true, true, "A new way of how interpret tuples in CSV format was added."},
-              {"input_format_csv_deserialize_separate_columns_into_tuple", true, true, "A new way of how interpret tuples in CSV format was added."},
-              {"input_format_csv_try_infer_strings_from_quoted_tuples", true, true, "A new way of how interpret tuples in CSV format was added."},
-              }},
-    {"24.5", {{"allow_deprecated_error_prone_window_functions", true, false, "Allow usage of deprecated error prone window functions (neighbor, runningAccumulate, runningDifferenceStartingWithFirstValue, runningDifference)"},
-              {"allow_experimental_join_condition", false, false, "Support join with inequal conditions which involve columns from both left and right table. e.g. t1.y < t2.y."},
-              {"input_format_tsv_crlf_end_of_line", false, false, "Enables reading of CRLF line endings with TSV formats"},
-              {"output_format_parquet_use_custom_encoder", false, true, "Enable custom Parquet encoder."},
-              {"cross_join_min_rows_to_compress", 0, 10000000, "Minimal count of rows to compress block in CROSS JOIN. Zero value means - disable this threshold. This block is compressed when any of the two thresholds (by rows or by bytes) are reached."},
-              {"cross_join_min_bytes_to_compress", 0, 1_GiB, "Minimal size of block to compress in CROSS JOIN. Zero value means - disable this threshold. This block is compressed when any of the two thresholds (by rows or by bytes) are reached."},
-              {"http_max_chunk_size", 0, 0, "Internal limitation"},
-              {"prefer_external_sort_block_bytes", 0, DEFAULT_BLOCK_SIZE * 256, "Prefer maximum block bytes for external sort, reduce the memory usage during merging."},
-              {"input_format_force_null_for_omitted_fields", false, false, "Disable type-defaults for omitted fields when needed"},
-              {"cast_string_to_dynamic_use_inference", false, false, "Add setting to allow converting String to Dynamic through parsing"},
-              {"allow_experimental_dynamic_type", false, false, "Add new experimental Dynamic type"},
-              {"azure_max_blocks_in_multipart_upload", 50000, 50000, "Maximum number of blocks in multipart upload for Azure."},
-              }},
-    {"24.4", {{"input_format_json_throw_on_bad_escape_sequence", true, true, "Allow to save JSON strings with bad escape sequences"},
-              {"max_parsing_threads", 0, 0, "Add a separate setting to control number of threads in parallel parsing from files"},
-              {"ignore_drop_queries_probability", 0, 0, "Allow to ignore drop queries in server with specified probability for testing purposes"},
-              {"lightweight_deletes_sync", 2, 2, "The same as 'mutation_sync', but controls only execution of lightweight deletes"},
-              {"query_cache_system_table_handling", "save", "throw", "The query cache no longer caches results of queries against system tables"},
-              {"input_format_json_ignore_unnecessary_fields", false, true, "Ignore unnecessary fields and not parse them. Enabling this may not throw exceptions on json strings of invalid format or with duplicated fields"},
-              {"input_format_hive_text_allow_variable_number_of_columns", false, true, "Ignore extra columns in Hive Text input (if file has more columns than expected) and treat missing fields in Hive Text input as default values."},
-              {"allow_experimental_database_replicated", false, true, "Database engine Replicated is now in Beta stage"},
-              {"temporary_data_in_cache_reserve_space_wait_lock_timeout_milliseconds", (10 * 60 * 1000), (10 * 60 * 1000), "Wait time to lock cache for sapce reservation in temporary data in filesystem cache"},
-              {"optimize_rewrite_sum_if_to_count_if", false, true, "Only available for the analyzer, where it works correctly"},
-              {"azure_allow_parallel_part_upload", "true", "true", "Use multiple threads for azure multipart upload."},
-              {"max_recursive_cte_evaluation_depth", DBMS_RECURSIVE_CTE_MAX_EVALUATION_DEPTH, DBMS_RECURSIVE_CTE_MAX_EVALUATION_DEPTH, "Maximum limit on recursive CTE evaluation depth"},
-              {"query_plan_convert_outer_join_to_inner_join", false, true, "Allow to convert OUTER JOIN to INNER JOIN if filter after JOIN always filters default values"},
-              }},
-    {"24.3", {{"s3_connect_timeout_ms", 1000, 1000, "Introduce new dedicated setting for s3 connection timeout"},
-              {"allow_experimental_shared_merge_tree", false, true, "The setting is obsolete"},
-              {"use_page_cache_for_disks_without_file_cache", false, false, "Added userspace page cache"},
-              {"read_from_page_cache_if_exists_otherwise_bypass_cache", false, false, "Added userspace page cache"},
-              {"page_cache_inject_eviction", false, false, "Added userspace page cache"},
-              {"default_table_engine", "None", "MergeTree", "Set default table engine to MergeTree for better usability"},
-              {"input_format_json_use_string_type_for_ambiguous_paths_in_named_tuples_inference_from_objects", false, false, "Allow to use String type for ambiguous paths during named tuple inference from JSON objects"},
-              {"traverse_shadow_remote_data_paths", false, false, "Traverse shadow directory when query system.remote_data_paths."},
-              {"throw_if_deduplication_in_dependent_materialized_views_enabled_with_async_insert", false, true, "Deduplication in dependent materialized view cannot work together with async inserts."},
-              {"parallel_replicas_allow_in_with_subquery", false, true, "If true, subquery for IN will be executed on every follower replica"},
-              {"log_processors_profiles", false, true, "Enable by default"},
-              {"function_locate_has_mysql_compatible_argument_order", false, true, "Increase compatibility with MySQL's locate function."},
-              {"allow_suspicious_primary_key", true, false, "Forbid suspicious PRIMARY KEY/ORDER BY for MergeTree (i.e. SimpleAggregateFunction)"},
-              {"filesystem_cache_reserve_space_wait_lock_timeout_milliseconds", 1000, 1000, "Wait time to lock cache for sapce reservation in filesystem cache"},
-              {"max_parser_backtracks", 0, 1000000, "Limiting the complexity of parsing"},
-              {"analyzer_compatibility_join_using_top_level_identifier", false, false, "Force to resolve identifier in JOIN USING from projection"},
-              {"distributed_insert_skip_read_only_replicas", false, false, "If true, INSERT into Distributed will skip read-only replicas"},
-              {"keeper_max_retries", 10, 10, "Max retries for general keeper operations"},
-              {"keeper_retry_initial_backoff_ms", 100, 100, "Initial backoff timeout for general keeper operations"},
-              {"keeper_retry_max_backoff_ms", 5000, 5000, "Max backoff timeout for general keeper operations"},
-              {"s3queue_allow_experimental_sharded_mode", false, false, "Enable experimental sharded mode of S3Queue table engine. It is experimental because it will be rewritten"},
-              {"allow_experimental_analyzer", false, true, "Enable analyzer and planner by default."},
-              {"merge_tree_read_split_ranges_into_intersecting_and_non_intersecting_injection_probability", 0.0, 0.0, "For testing of `PartsSplitter` - split read ranges into intersecting and non intersecting every time you read from MergeTree with the specified probability."},
-              {"allow_get_client_http_header", false, false, "Introduced a new function."},
-              {"output_format_pretty_row_numbers", false, true, "It is better for usability."},
-              {"output_format_pretty_max_value_width_apply_for_single_value", true, false, "Single values in Pretty formats won't be cut."},
-              {"output_format_parquet_string_as_string", false, true, "ClickHouse allows arbitrary binary data in the String data type, which is typically UTF-8. Parquet/ORC/Arrow Strings only support UTF-8. That's why you can choose which Arrow's data type to use for the ClickHouse String data type - String or Binary. While Binary would be more correct and compatible, using String by default will correspond to user expectations in most cases."},
-              {"output_format_orc_string_as_string", false, true, "ClickHouse allows arbitrary binary data in the String data type, which is typically UTF-8. Parquet/ORC/Arrow Strings only support UTF-8. That's why you can choose which Arrow's data type to use for the ClickHouse String data type - String or Binary. While Binary would be more correct and compatible, using String by default will correspond to user expectations in most cases."},
-              {"output_format_arrow_string_as_string", false, true, "ClickHouse allows arbitrary binary data in the String data type, which is typically UTF-8. Parquet/ORC/Arrow Strings only support UTF-8. That's why you can choose which Arrow's data type to use for the ClickHouse String data type - String or Binary. While Binary would be more correct and compatible, using String by default will correspond to user expectations in most cases."},
-              {"output_format_parquet_compression_method", "lz4", "zstd", "Parquet/ORC/Arrow support many compression methods, including lz4 and zstd. ClickHouse supports each and every compression method. Some inferior tools, such as 'duckdb', lack support for the faster `lz4` compression method, that's why we set zstd by default."},
-              {"output_format_orc_compression_method", "lz4", "zstd", "Parquet/ORC/Arrow support many compression methods, including lz4 and zstd. ClickHouse supports each and every compression method. Some inferior tools, such as 'duckdb', lack support for the faster `lz4` compression method, that's why we set zstd by default."},
-              {"output_format_pretty_highlight_digit_groups", false, true, "If enabled and if output is a terminal, highlight every digit corresponding to the number of thousands, millions, etc. with underline."},
-              {"geo_distance_returns_float64_on_float64_arguments", false, true, "Increase the default precision."},
-              {"azure_max_inflight_parts_for_one_file", 20, 20, "The maximum number of a concurrent loaded parts in multipart upload request. 0 means unlimited."},
-              {"azure_strict_upload_part_size", 0, 0, "The exact size of part to upload during multipart upload to Azure blob storage."},
-              {"azure_min_upload_part_size", 16*1024*1024, 16*1024*1024, "The minimum size of part to upload during multipart upload to Azure blob storage."},
-              {"azure_max_upload_part_size", 5ull*1024*1024*1024, 5ull*1024*1024*1024, "The maximum size of part to upload during multipart upload to Azure blob storage."},
-              {"azure_upload_part_size_multiply_factor", 2, 2, "Multiply azure_min_upload_part_size by this factor each time azure_multiply_parts_count_threshold parts were uploaded from a single write to Azure blob storage."},
-              {"azure_upload_part_size_multiply_parts_count_threshold", 500, 500, "Each time this number of parts was uploaded to Azure blob storage, azure_min_upload_part_size is multiplied by azure_upload_part_size_multiply_factor."},
-              {"output_format_csv_serialize_tuple_into_separate_columns", true, true, "A new way of how interpret tuples in CSV format was added."},
-              {"input_format_csv_deserialize_separate_columns_into_tuple", true, true, "A new way of how interpret tuples in CSV format was added."},
-              {"input_format_csv_try_infer_strings_from_quoted_tuples", true, true, "A new way of how interpret tuples in CSV format was added."},
-              }},
-    {"24.2", {{"allow_suspicious_variant_types", true, false, "Don't allow creating Variant type with suspicious variants by default"},
-              {"validate_experimental_and_suspicious_types_inside_nested_types", false, true, "Validate usage of experimental and suspicious types inside nested types"},
-              {"output_format_values_escape_quote_with_quote", false, false, "If true escape ' with '', otherwise quoted with \\'"},
-              {"output_format_pretty_single_large_number_tip_threshold", 0, 1'000'000, "Print a readable number tip on the right side of the table if the block consists of a single number which exceeds this value (except 0)"},
-              {"input_format_try_infer_exponent_floats", true, false, "Don't infer floats in exponential notation by default"},
-              {"query_plan_optimize_prewhere", true, true, "Allow to push down filter to PREWHERE expression for supported storages"},
-              {"async_insert_max_data_size", 1000000, 10485760, "The previous value appeared to be too small."},
-              {"async_insert_poll_timeout_ms", 10, 10, "Timeout in milliseconds for polling data from asynchronous insert queue"},
-              {"async_insert_use_adaptive_busy_timeout", false, true, "Use adaptive asynchronous insert timeout"},
-              {"async_insert_busy_timeout_min_ms", 50, 50, "The minimum value of the asynchronous insert timeout in milliseconds; it also serves as the initial value, which may be increased later by the adaptive algorithm"},
-              {"async_insert_busy_timeout_max_ms", 200, 200, "The minimum value of the asynchronous insert timeout in milliseconds; async_insert_busy_timeout_ms is aliased to async_insert_busy_timeout_max_ms"},
-              {"async_insert_busy_timeout_increase_rate", 0.2, 0.2, "The exponential growth rate at which the adaptive asynchronous insert timeout increases"},
-              {"async_insert_busy_timeout_decrease_rate", 0.2, 0.2, "The exponential growth rate at which the adaptive asynchronous insert timeout decreases"},
-              {"format_template_row_format", "", "", "Template row format string can be set directly in query"},
-              {"format_template_resultset_format", "", "", "Template result set format string can be set in query"},
-              {"split_parts_ranges_into_intersecting_and_non_intersecting_final", true, true, "Allow to split parts ranges into intersecting and non intersecting during FINAL optimization"},
-              {"split_intersecting_parts_ranges_into_layers_final", true, true, "Allow to split intersecting parts ranges into layers during FINAL optimization"},
-              {"azure_max_single_part_copy_size", 256*1024*1024, 256*1024*1024, "The maximum size of object to copy using single part copy to Azure blob storage."},
-              {"min_external_table_block_size_rows", DEFAULT_INSERT_BLOCK_SIZE, DEFAULT_INSERT_BLOCK_SIZE, "Squash blocks passed to external table to specified size in rows, if blocks are not big enough"},
-              {"min_external_table_block_size_bytes", DEFAULT_INSERT_BLOCK_SIZE * 256, DEFAULT_INSERT_BLOCK_SIZE * 256, "Squash blocks passed to external table to specified size in bytes, if blocks are not big enough."},
-              {"parallel_replicas_prefer_local_join", true, true, "If true, and JOIN can be executed with parallel replicas algorithm, and all storages of right JOIN part are *MergeTree, local JOIN will be used instead of GLOBAL JOIN."},
-              {"optimize_time_filter_with_preimage", true, true, "Optimize Date and DateTime predicates by converting functions into equivalent comparisons without conversions (e.g. toYear(col) = 2023 -> col >= '2023-01-01' AND col <= '2023-12-31')"},
-              {"extract_key_value_pairs_max_pairs_per_row", 0, 0, "Max number of pairs that can be produced by the `extractKeyValuePairs` function. Used as a safeguard against consuming too much memory."},
-              {"default_view_definer", "CURRENT_USER", "CURRENT_USER", "Allows to set default `DEFINER` option while creating a view"},
-              {"default_materialized_view_sql_security", "DEFINER", "DEFINER", "Allows to set a default value for SQL SECURITY option when creating a materialized view"},
-              {"default_normal_view_sql_security", "INVOKER", "INVOKER", "Allows to set default `SQL SECURITY` option while creating a normal view"},
-              {"mysql_map_string_to_text_in_show_columns", false, true, "Reduce the configuration effort to connect ClickHouse with BI tools."},
-              {"mysql_map_fixed_string_to_text_in_show_columns", false, true, "Reduce the configuration effort to connect ClickHouse with BI tools."},
-              }},
-    {"24.1", {{"print_pretty_type_names", false, true, "Better user experience."},
-              {"input_format_json_read_bools_as_strings", false, true, "Allow to read bools as strings in JSON formats by default"},
-              {"output_format_arrow_use_signed_indexes_for_dictionary", false, true, "Use signed indexes type for Arrow dictionaries by default as it's recommended"},
-              {"allow_experimental_variant_type", false, false, "Add new experimental Variant type"},
-              {"use_variant_as_common_type", false, false, "Allow to use Variant in if/multiIf if there is no common type"},
-              {"output_format_arrow_use_64_bit_indexes_for_dictionary", false, false, "Allow to use 64 bit indexes type in Arrow dictionaries"},
-              {"parallel_replicas_mark_segment_size", 128, 128, "Add new setting to control segment size in new parallel replicas coordinator implementation"},
-              {"ignore_materialized_views_with_dropped_target_table", false, false, "Add new setting to allow to ignore materialized views with dropped target table"},
-              {"output_format_compression_level", 3, 3, "Allow to change compression level in the query output"},
-              {"output_format_compression_zstd_window_log", 0, 0, "Allow to change zstd window log in the query output when zstd compression is used"},
-              {"enable_zstd_qat_codec", false, false, "Add new ZSTD_QAT codec"},
-              {"enable_vertical_final", false, true, "Use vertical final by default"},
-              {"output_format_arrow_use_64_bit_indexes_for_dictionary", false, false, "Allow to use 64 bit indexes type in Arrow dictionaries"},
-              {"max_rows_in_set_to_optimize_join", 100000, 0, "Disable join optimization as it prevents from read in order optimization"},
-              {"output_format_pretty_color", true, "auto", "Setting is changed to allow also for auto value, disabling ANSI escapes if output is not a tty"},
-              {"function_visible_width_behavior", 0, 1, "We changed the default behavior of `visibleWidth` to be more precise"},
-              {"max_estimated_execution_time", 0, 0, "Separate max_execution_time and max_estimated_execution_time"},
-              {"iceberg_engine_ignore_schema_evolution", false, false, "Allow to ignore schema evolution in Iceberg table engine"},
-              {"optimize_injective_functions_in_group_by", false, true, "Replace injective functions by it's arguments in GROUP BY section in analyzer"},
-              {"update_insert_deduplication_token_in_dependent_materialized_views", false, false, "Allow to update insert deduplication token with table identifier during insert in dependent materialized views"},
-              {"azure_max_unexpected_write_error_retries", 4, 4, "The maximum number of retries in case of unexpected errors during Azure blob storage write"},
-              {"split_parts_ranges_into_intersecting_and_non_intersecting_final", false, true, "Allow to split parts ranges into intersecting and non intersecting during FINAL optimization"},
-              {"split_intersecting_parts_ranges_into_layers_final", true, true, "Allow to split intersecting parts ranges into layers during FINAL optimization"}}},
-    {"23.12", {{"allow_suspicious_ttl_expressions", true, false, "It is a new setting, and in previous versions the behavior was equivalent to allowing."},
-              {"input_format_parquet_allow_missing_columns", false, true, "Allow missing columns in Parquet files by default"},
-              {"input_format_orc_allow_missing_columns", false, true, "Allow missing columns in ORC files by default"},
-              {"input_format_arrow_allow_missing_columns", false, true, "Allow missing columns in Arrow files by default"}}},
-    {"23.11", {{"parsedatetime_parse_without_leading_zeros", false, true, "Improved compatibility with MySQL DATE_FORMAT/STR_TO_DATE"}}},
-    {"23.9", {{"optimize_group_by_constant_keys", false, true, "Optimize group by constant keys by default"},
-              {"input_format_json_try_infer_named_tuples_from_objects", false, true, "Try to infer named Tuples from JSON objects by default"},
-              {"input_format_json_read_numbers_as_strings", false, true, "Allow to read numbers as strings in JSON formats by default"},
-              {"input_format_json_read_arrays_as_strings", false, true, "Allow to read arrays as strings in JSON formats by default"},
-              {"input_format_json_infer_incomplete_types_as_strings", false, true, "Allow to infer incomplete types as Strings in JSON formats by default"},
-              {"input_format_json_try_infer_numbers_from_strings", true, false, "Don't infer numbers from strings in JSON formats by default to prevent possible parsing errors"},
-              {"http_write_exception_in_output_format", false, true, "Output valid JSON/XML on exception in HTTP streaming."}}},
-    {"23.8", {{"rewrite_count_distinct_if_with_count_distinct_implementation", false, true, "Rewrite countDistinctIf with count_distinct_implementation configuration"}}},
-    {"23.7", {{"function_sleep_max_microseconds_per_block", 0, 3000000, "In previous versions, the maximum sleep time of 3 seconds was applied only for `sleep`, but not for `sleepEachRow` function. In the new version, we introduce this setting. If you set compatibility with the previous versions, we will disable the limit altogether."}}},
-    {"23.6", {{"http_send_timeout", 180, 30, "3 minutes seems crazy long. Note that this is timeout for a single network write call, not for the whole upload operation."},
-              {"http_receive_timeout", 180, 30, "See http_send_timeout."}}},
-    {"23.5", {{"input_format_parquet_preserve_order", true, false, "Allow Parquet reader to reorder rows for better parallelism."},
-              {"parallelize_output_from_storages", false, true, "Allow parallelism when executing queries that read from file/url/s3/etc. This may reorder rows."},
-              {"use_with_fill_by_sorting_prefix", false, true, "Columns preceding WITH FILL columns in ORDER BY clause form sorting prefix. Rows with different values in sorting prefix are filled independently"},
-              {"output_format_parquet_compliant_nested_types", false, true, "Change an internal field name in output Parquet file schema."}}},
-    {"23.4", {{"allow_suspicious_indices", true, false, "If true, index can defined with identical expressions"},
-              {"allow_nonconst_timezone_arguments", true, false, "Allow non-const timezone arguments in certain time-related functions like toTimeZone(), fromUnixTimestamp*(), snowflakeToDateTime*()."},
-              {"connect_timeout_with_failover_ms", 50, 1000, "Increase default connect timeout because of async connect"},
-              {"connect_timeout_with_failover_secure_ms", 100, 1000, "Increase default secure connect timeout because of async connect"},
-              {"hedged_connection_timeout_ms", 100, 50, "Start new connection in hedged requests after 50 ms instead of 100 to correspond with previous connect timeout"},
-              {"formatdatetime_f_prints_single_zero", true, false, "Improved compatibility with MySQL DATE_FORMAT()/STR_TO_DATE()"},
-              {"formatdatetime_parsedatetime_m_is_month_name", false, true, "Improved compatibility with MySQL DATE_FORMAT/STR_TO_DATE"}}},
-    {"23.3", {{"output_format_parquet_version", "1.0", "2.latest", "Use latest Parquet format version for output format"},
-              {"input_format_json_ignore_unknown_keys_in_named_tuple", false, true, "Improve parsing JSON objects as named tuples"},
-              {"input_format_native_allow_types_conversion", false, true, "Allow types conversion in Native input forma"},
-              {"output_format_arrow_compression_method", "none", "lz4_frame", "Use lz4 compression in Arrow output format by default"},
-              {"output_format_parquet_compression_method", "snappy", "lz4", "Use lz4 compression in Parquet output format by default"},
-              {"output_format_orc_compression_method", "none", "lz4_frame", "Use lz4 compression in ORC output format by default"},
-              {"async_query_sending_for_remote", false, true, "Create connections and send query async across shards"}}},
-    {"23.2", {{"output_format_parquet_fixed_string_as_fixed_byte_array", false, true, "Use Parquet FIXED_LENGTH_BYTE_ARRAY type for FixedString by default"},
-              {"output_format_arrow_fixed_string_as_fixed_byte_array", false, true, "Use Arrow FIXED_SIZE_BINARY type for FixedString by default"},
-              {"query_plan_remove_redundant_distinct", false, true, "Remove redundant Distinct step in query plan"},
-              {"optimize_duplicate_order_by_and_distinct", true, false, "Remove duplicate ORDER BY and DISTINCT if it's possible"},
-              {"insert_keeper_max_retries", 0, 20, "Enable reconnections to Keeper on INSERT, improve reliability"}}},
-    {"23.1", {{"input_format_json_read_objects_as_strings", 0, 1, "Enable reading nested json objects as strings while object type is experimental"},
-              {"input_format_json_defaults_for_missing_elements_in_named_tuple", false, true, "Allow missing elements in JSON objects while reading named tuples by default"},
-              {"input_format_csv_detect_header", false, true, "Detect header in CSV format by default"},
-              {"input_format_tsv_detect_header", false, true, "Detect header in TSV format by default"},
-              {"input_format_custom_detect_header", false, true, "Detect header in CustomSeparated format by default"},
-              {"query_plan_remove_redundant_sorting", false, true, "Remove redundant sorting in query plan. For example, sorting steps related to ORDER BY clauses in subqueries"}}},
-    {"22.12", {{"max_size_to_preallocate_for_aggregation", 10'000'000, 100'000'000, "This optimizes performance"},
-               {"query_plan_aggregation_in_order", 0, 1, "Enable some refactoring around query plan"},
-               {"format_binary_max_string_size", 0, 1_GiB, "Prevent allocating large amount of memory"}}},
-    {"22.11", {{"use_structure_from_insertion_table_in_table_functions", 0, 2, "Improve using structure from insertion table in table functions"}}},
-    {"22.9", {{"force_grouping_standard_compatibility", false, true, "Make GROUPING function output the same as in SQL standard and other DBMS"}}},
-    {"22.7", {{"cross_to_inner_join_rewrite", 1, 2, "Force rewrite comma join to inner"},
-              {"enable_positional_arguments", false, true, "Enable positional arguments feature by default"},
-              {"format_csv_allow_single_quotes", true, false, "Most tools don't treat single quote in CSV specially, don't do it by default too"}}},
-    {"22.6", {{"output_format_json_named_tuples_as_objects", false, true, "Allow to serialize named tuples as JSON objects in JSON formats by default"},
-              {"input_format_skip_unknown_fields", false, true, "Optimize reading subset of columns for some input formats"}}},
-    {"22.5", {{"memory_overcommit_ratio_denominator", 0, 1073741824, "Enable memory overcommit feature by default"},
-              {"memory_overcommit_ratio_denominator_for_user", 0, 1073741824, "Enable memory overcommit feature by default"}}},
-    {"22.4", {{"allow_settings_after_format_in_insert", true, false, "Do not allow SETTINGS after FORMAT for INSERT queries because ClickHouse interpret SETTINGS as some values, which is misleading"}}},
-    {"22.3", {{"cast_ipv4_ipv6_default_on_conversion_error", true, false, "Make functions cast(value, 'IPv4') and cast(value, 'IPv6') behave same as toIPv4 and toIPv6 functions"}}},
-    {"21.12", {{"stream_like_engine_allow_direct_select", true, false, "Do not allow direct select for Kafka/RabbitMQ/FileLog by default"}}},
-    {"21.9", {{"output_format_decimal_trailing_zeros", true, false, "Do not output trailing zeros in text representation of Decimal types by default for better looking output"},
-              {"use_hedged_requests", false, true, "Enable Hedged Requests feature by default"}}},
-    {"21.7", {{"legacy_column_name_of_tuple_literal", true, false, "Add this setting only for compatibility reasons. It makes sense to set to 'true', while doing rolling update of cluster from version lower than 21.7 to higher"}}},
-    {"21.5", {{"async_socket_for_remote", false, true, "Fix all problems and turn on asynchronous reads from socket for remote queries by default again"}}},
-    {"21.3", {{"async_socket_for_remote", true, false, "Turn off asynchronous reads from socket for remote queries because of some problems"},
-              {"optimize_normalize_count_variants", false, true, "Rewrite aggregate functions that semantically equals to count() as count() by default"},
-              {"normalize_function_names", false, true, "Normalize function names to their canonical names, this was needed for projection query routing"}}},
-    {"21.2", {{"enable_global_with_statement", false, true, "Propagate WITH statements to UNION queries and all subqueries by default"}}},
-    {"21.1", {{"insert_quorum_parallel", false, true, "Use parallel quorum inserts by default. It is significantly more convenient to use than sequential quorum inserts"},
-              {"input_format_null_as_default", false, true, "Allow to insert NULL as default for input formats by default"},
-              {"optimize_on_insert", false, true, "Enable data optimization on INSERT by default for better user experience"},
-              {"use_compact_format_in_distributed_parts_names", false, true, "Use compact format for async INSERT into Distributed tables by default"}}},
-    {"20.10", {{"format_regexp_escaping_rule", "Escaped", "Raw", "Use Raw as default escaping rule for Regexp format to male the behaviour more like to what users expect"}}},
-    {"20.7", {{"show_table_uuid_in_table_create_query_if_not_nil", true, false, "Stop showing  UID of the table in its CREATE query for Engine=Atomic"}}},
-    {"20.5", {{"input_format_with_names_use_header", false, true, "Enable using header with names for formats with WithNames/WithNamesAndTypes suffixes"},
-              {"allow_suspicious_codecs", true, false, "Don't allow to specify meaningless compression codecs"}}},
-    {"20.4", {{"validate_polygons", false, true, "Throw exception if polygon is invalid in function pointInPolygon by default instead of returning possibly wrong results"}}},
-    {"19.18", {{"enable_scalar_subquery_optimization", false, true, "Prevent scalar subqueries from (de)serializing large scalar values and possibly avoid running the same subquery more than once"}}},
-    {"19.14", {{"any_join_distinct_right_table_keys", true, false, "Disable ANY RIGHT and ANY FULL JOINs by default to avoid inconsistency"}}},
-    {"19.12", {{"input_format_defaults_for_omitted_fields", false, true, "Enable calculation of complex default expressions for omitted fields for some input formats, because it should be the expected behaviour"}}},
-    {"19.5", {{"max_partitions_per_insert_block", 0, 100, "Add a limit for the number of partitions in one block"}}},
-    {"18.12.17", {{"enable_optimize_predicate_expression", 0, 1, "Optimize predicates to subqueries by default"}}},
     {"24.12",
         {
         }

From 4e8bd0654cd378fd03985170a4df8f49b583bd01 Mon Sep 17 00:00:00 2001
From: Michael Kolupaev <michael.kolupaev@clickhouse.com>
Date: Mon, 5 Aug 2024 22:06:36 +0000
Subject: [PATCH 1429/1488] Remove from history

---
 src/Core/SettingsChangesHistory.cpp | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp
index 5b94391bade..03de54f2780 100644
--- a/src/Core/SettingsChangesHistory.cpp
+++ b/src/Core/SettingsChangesHistory.cpp
@@ -89,7 +89,6 @@ static std::initializer_list<std::pair<ClickHouseVersion, SettingsChangesHistory
             {"input_format_native_decode_types_in_binary_format", false, false, "Added new setting to allow to read type names in binary format in Native output format"},
             {"read_in_order_use_buffering", false, true, "Use buffering before merging while reading in order of primary key"},
             {"enable_named_columns_in_function_tuple", false, true, "Generate named tuples in function tuple() when all names are unique and can be treated as unquoted identifiers."},
-            {"input_format_json_case_insensitive_column_matching", false, false, "Ignore case when matching JSON keys with CH columns."},
             {"optimize_trivial_insert_select", true, false, "The optimization does not make sense in many cases."},
             {"dictionary_validate_primary_key_type", false, false, "Validate primary key type for dictionaries. By default id type for simple layouts will be implicitly converted to UInt64."},
             {"collect_hash_table_stats_during_joins", false, true, "New setting."},

From 8703941a0f5933a8a2adbc4003942b5ba22496f1 Mon Sep 17 00:00:00 2001
From: Michael Kolupaev <michael.kolupaev@clickhouse.com>
Date: Mon, 5 Aug 2024 22:36:30 +0000
Subject: [PATCH 1430/1488] Apply https://github.com/ClickHouse/zlib-ng/pull/18

---
 contrib/zlib-ng | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contrib/zlib-ng b/contrib/zlib-ng
index c19ba056b7c..a2fbeffdc30 160000
--- a/contrib/zlib-ng
+++ b/contrib/zlib-ng
@@ -1 +1 @@
-Subproject commit c19ba056b7cc8029bb80f509956090c7ded58032
+Subproject commit a2fbeffdc30a8b0ce6d54ee31208e2688eac4c9f

From 087d014f2ae050b7a45f72a7cf87f5de5a8179cf Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Tue, 6 Aug 2024 03:17:17 +0200
Subject: [PATCH 1431/1488] Revert "Revert "Add a test for #47892""

---
 .../02968_full_sorting_join_fuzz.reference        |  3 +++
 .../0_stateless/02968_full_sorting_join_fuzz.sql  | 15 +++++++++++++++
 2 files changed, 18 insertions(+)
 create mode 100644 tests/queries/0_stateless/02968_full_sorting_join_fuzz.reference
 create mode 100644 tests/queries/0_stateless/02968_full_sorting_join_fuzz.sql

diff --git a/tests/queries/0_stateless/02968_full_sorting_join_fuzz.reference b/tests/queries/0_stateless/02968_full_sorting_join_fuzz.reference
new file mode 100644
index 00000000000..0e9f5cc1db1
--- /dev/null
+++ b/tests/queries/0_stateless/02968_full_sorting_join_fuzz.reference
@@ -0,0 +1,3 @@
+[NULL]	[]	100	0
+
+[NULL]	[]	100	0
diff --git a/tests/queries/0_stateless/02968_full_sorting_join_fuzz.sql b/tests/queries/0_stateless/02968_full_sorting_join_fuzz.sql
new file mode 100644
index 00000000000..802eda57df3
--- /dev/null
+++ b/tests/queries/0_stateless/02968_full_sorting_join_fuzz.sql
@@ -0,0 +1,15 @@
+SET max_bytes_in_join = 0, join_algorithm = 'full_sorting_merge', max_block_size = 10240;
+
+DROP TABLE IF EXISTS t1;
+DROP TABLE IF EXISTS t2;
+
+CREATE TABLE t1 (`key` UInt32, `s` String) ENGINE = MergeTree ORDER BY key;
+CREATE TABLE t2 (`key` UInt32, `s` String) ENGINE = MergeTree ORDER BY key;
+
+INSERT INTO t1 SELECT (sipHash64(number, 'x') % 10000000) + 1 AS key, concat('val', toString(number)) AS s FROM numbers_mt(10000000);
+INSERT INTO t2 SELECT (sipHash64(number, 'y') % 1000000) + 1 AS key, concat('val', toString(number)) AS s FROM numbers_mt(1000000);
+
+SELECT materialize([NULL]), [], 100, count(materialize(NULL)) FROM t1 ALL INNER JOIN t2 ON t1.key = t2.key PREWHERE 10 WHERE t2.key WITH TOTALS;
+
+DROP TABLE t1;
+DROP TABLE t2;

From 52f3ad97e86b485419f3f88c5c047d0868d0574a Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Tue, 6 Aug 2024 03:30:19 +0200
Subject: [PATCH 1432/1488] Fix MaterializedMySQL

---
 src/Databases/MySQL/DatabaseMaterializedMySQL.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/Databases/MySQL/DatabaseMaterializedMySQL.cpp b/src/Databases/MySQL/DatabaseMaterializedMySQL.cpp
index 2f5477a6b9d..8b3850c4e0c 100644
--- a/src/Databases/MySQL/DatabaseMaterializedMySQL.cpp
+++ b/src/Databases/MySQL/DatabaseMaterializedMySQL.cpp
@@ -46,6 +46,7 @@ DatabaseMaterializedMySQL::DatabaseMaterializedMySQL(
     , settings(std::move(settings_))
     , materialize_thread(context_, database_name_, mysql_database_name_, std::move(pool_), std::move(client_), binlog_client_, settings.get())
 {
+    createDirectories();
 }
 
 void DatabaseMaterializedMySQL::rethrowExceptionIfNeeded() const

From 15a4d13cdad3fe9181703d5c0a993b8dcb8cb263 Mon Sep 17 00:00:00 2001
From: Alexey Milovidov <milovidov@clickhouse.com>
Date: Tue, 6 Aug 2024 04:33:02 +0200
Subject: [PATCH 1433/1488] Update 02968_full_sorting_join_fuzz.sql

---
 tests/queries/0_stateless/02968_full_sorting_join_fuzz.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/02968_full_sorting_join_fuzz.sql b/tests/queries/0_stateless/02968_full_sorting_join_fuzz.sql
index 802eda57df3..85ca740cef2 100644
--- a/tests/queries/0_stateless/02968_full_sorting_join_fuzz.sql
+++ b/tests/queries/0_stateless/02968_full_sorting_join_fuzz.sql
@@ -9,7 +9,7 @@ CREATE TABLE t2 (`key` UInt32, `s` String) ENGINE = MergeTree ORDER BY key;
 INSERT INTO t1 SELECT (sipHash64(number, 'x') % 10000000) + 1 AS key, concat('val', toString(number)) AS s FROM numbers_mt(10000000);
 INSERT INTO t2 SELECT (sipHash64(number, 'y') % 1000000) + 1 AS key, concat('val', toString(number)) AS s FROM numbers_mt(1000000);
 
-SELECT materialize([NULL]), [], 100, count(materialize(NULL)) FROM t1 ALL INNER JOIN t2 ON t1.key = t2.key PREWHERE 10 WHERE t2.key WITH TOTALS;
+SELECT materialize([NULL]), [], 100, count(materialize(NULL)) FROM t1 ALL INNER JOIN t2 ON t1.key = t2.key PREWHERE 10 WHERE t2.key != 0 WITH TOTALS;
 
 DROP TABLE t1;
 DROP TABLE t2;

From 8e6de033355485a770f86b848e7574728a0eecf4 Mon Sep 17 00:00:00 2001
From: Antonio Andelic <antonio@clickhouse.com>
Date: Tue, 6 Aug 2024 09:39:00 +0200
Subject: [PATCH 1434/1488] Remove unsupported files

---
 .../queries/0_stateless/03215_parsing_archive_name_s3.sql | 8 ++++++++
 tests/queries/0_stateless/data_minio/::03215_archive.csv  | 1 -
 .../0_stateless/data_minio/test.zip::03215_archive.csv    | 1 -
 .../0_stateless/data_minio/test::03215_archive.csv        | 1 -
 4 files changed, 8 insertions(+), 3 deletions(-)
 delete mode 100644 tests/queries/0_stateless/data_minio/::03215_archive.csv
 delete mode 100644 tests/queries/0_stateless/data_minio/test.zip::03215_archive.csv
 delete mode 100644 tests/queries/0_stateless/data_minio/test::03215_archive.csv

diff --git a/tests/queries/0_stateless/03215_parsing_archive_name_s3.sql b/tests/queries/0_stateless/03215_parsing_archive_name_s3.sql
index e34be475c5a..e0d63d313fa 100644
--- a/tests/queries/0_stateless/03215_parsing_archive_name_s3.sql
+++ b/tests/queries/0_stateless/03215_parsing_archive_name_s3.sql
@@ -1,7 +1,15 @@
 -- Tags: no-fasttest
 -- Tag no-fasttest: Depends on AWS
 
+SET s3_truncate_on_insert=1;
+
+INSERT INTO FUNCTION s3(s3_conn, filename='::03215_archive.csv') SELECT 1;
 SELECT _file, _path FROM s3(s3_conn, filename='::03215_archive.csv') ORDER BY (_file, _path);
+
 SELECT _file, _path FROM s3(s3_conn, filename='test :: 03215_archive.csv') ORDER BY (_file, _path); -- { serverError S3_ERROR }
+
+INSERT INTO FUNCTION s3(s3_conn, filename='test::03215_archive.csv') SELECT 1;
 SELECT _file, _path FROM s3(s3_conn, filename='test::03215_archive.csv') ORDER BY (_file, _path);
+
+INSERT INTO FUNCTION s3(s3_conn, filename='test.zip::03215_archive.csv') SETTINGS allow_archive_path_syntax=0 SELECT 1;
 SELECT _file, _path FROM s3(s3_conn, filename='test.zip::03215_archive.csv') ORDER BY (_file, _path) SETTINGS allow_archive_path_syntax=0;
diff --git a/tests/queries/0_stateless/data_minio/::03215_archive.csv b/tests/queries/0_stateless/data_minio/::03215_archive.csv
deleted file mode 100644
index d00491fd7e5..00000000000
--- a/tests/queries/0_stateless/data_minio/::03215_archive.csv
+++ /dev/null
@@ -1 +0,0 @@
-1
diff --git a/tests/queries/0_stateless/data_minio/test.zip::03215_archive.csv b/tests/queries/0_stateless/data_minio/test.zip::03215_archive.csv
deleted file mode 100644
index d00491fd7e5..00000000000
--- a/tests/queries/0_stateless/data_minio/test.zip::03215_archive.csv
+++ /dev/null
@@ -1 +0,0 @@
-1
diff --git a/tests/queries/0_stateless/data_minio/test::03215_archive.csv b/tests/queries/0_stateless/data_minio/test::03215_archive.csv
deleted file mode 100644
index d00491fd7e5..00000000000
--- a/tests/queries/0_stateless/data_minio/test::03215_archive.csv
+++ /dev/null
@@ -1 +0,0 @@
-1

From 27569815d519c8ad70842798b989874cc04b3271 Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Mon, 5 Aug 2024 17:01:00 +0200
Subject: [PATCH 1435/1488] Fix query cache randomization in stress tests

Right now it fails with [1]:

    2024.08.05 12:53:53.659422 [ 3559 ] {123237a2-5f08-47e4-996e-21e20bc4a51a} <Error> executeQuery: Code: 704. DB::Exception: The query result was not cached because the query contains a non-deterministic function. Use setting `query_cache_nondeterministic_function_handling = 'save'` or `= 'ignore'` to cache the query result regardless or to omit caching. (QUERY_CACHE_USED_WITH_NONDETERMINISTIC_FUNCTIONS) (version 24.8.1.1356) (from [::1]:42534) (in query: SELECT version(), arrayStringConcat(groupArray(value), ' ') FROM system.build_options WHERE name IN ('GIT_HASH', 'GIT_BRANCH') ), Stack trace (when copying this message, always include the lines below):

    Uexpected exception, will not retry:  HTTPError :  Code: 500. Code: 704. DB::Exception: The query result was not cached because the query contains a non-deterministic function. Use setting `query_cache_nondeterministic_function_handling = 'save'` or `= 'ignore'` to cache the query result regardless or to omit caching. (QUERY_CACHE_USED_WITH_NONDETERMINISTIC_FUNCTIONS) (version 24.8.1.1356)

  [1]: https://s3.amazonaws.com/clickhouse-test-reports/67737/e68c9c8d16f37f6c25739076c9b071ed97952269/stress_test__asan_/stress_test_run_10.txt

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 tests/ci/stress.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/ci/stress.py b/tests/ci/stress.py
index 8b8b17dd2a9..a3cdd8b3d6f 100755
--- a/tests/ci/stress.py
+++ b/tests/ci/stress.py
@@ -47,6 +47,8 @@ def get_options(i: int, upgrade_check: bool) -> str:
 
     if i > 0 and random.random() < 1 / 3:
         client_options.append("use_query_cache=1")
+        client_options.append("query_cache_nondeterministic_function_handling='ignore'")
+        client_options.append("query_cache_system_table_handling='ignore'")
 
     if i % 5 == 1:
         client_options.append("memory_tracker_fault_probability=0.001")

From b2987e4f4dd267293b8b4f96f86f2202a7671b84 Mon Sep 17 00:00:00 2001
From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com>
Date: Tue, 6 Aug 2024 11:42:18 +0200
Subject: [PATCH 1436/1488] Update StorageWindowView.cpp

---
 src/Storages/WindowView/StorageWindowView.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Storages/WindowView/StorageWindowView.cpp b/src/Storages/WindowView/StorageWindowView.cpp
index bf934ed00d9..5830c844582 100644
--- a/src/Storages/WindowView/StorageWindowView.cpp
+++ b/src/Storages/WindowView/StorageWindowView.cpp
@@ -1450,7 +1450,7 @@ void StorageWindowView::writeIntoWindowView(
         LOG_TRACE(window_view.log, "New max watermark: {}", window_view.max_watermark);
     }
 
-    Pipe pipe(std::make_shared<SourceFromSingleChunk>(std::move(block)));
+    Pipe pipe(std::make_shared<SourceFromSingleChunk>(block));
 
     UInt32 lateness_bound = 0;
     UInt32 t_max_watermark = 0;

From 50ed7da27dc1bcdbb03f5cf7e3d80ae95ecf643f Mon Sep 17 00:00:00 2001
From: maxvostrikov <max.vostrikov@clickhouse.com>
Date: Tue, 6 Aug 2024 12:02:53 +0200
Subject: [PATCH 1437/1488] squash! fix for parallel execution

---
 tests/queries/0_stateless/03215_parquet_index.sql | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/queries/0_stateless/03215_parquet_index.sql b/tests/queries/0_stateless/03215_parquet_index.sql
index 5b176ff70ba..0afccdf6f3b 100644
--- a/tests/queries/0_stateless/03215_parquet_index.sql
+++ b/tests/queries/0_stateless/03215_parquet_index.sql
@@ -4,12 +4,14 @@ CREATE TABLE test_parquet (col1 int, col2 String) ENGINE=File(Parquet);
 INSERT INTO test_parquet SELECT number, toString(number) FROM numbers(100);
 SELECT col1, col2 FROM test_parquet;
 
+
 -- Parquet will have indexes in columns. We are not checking that indexes exist here, there is an integration test test_parquet_page_index for that. We just check that a setting doesn't break the SELECT
 DROP TABLE IF EXISTS test_parquet;
 CREATE TABLE test_parquet (col1 int, col2 String) ENGINE=File(Parquet)  SETTINGS output_format_parquet_use_custom_encoder=false, output_format_parquet_write_page_index=true;
 INSERT INTO test_parquet SELECT number, toString(number) FROM numbers(100);
 SELECT col1, col2 FROM test_parquet;
 
+
 -- Parquet will not have indexes in columns.
 DROP TABLE IF EXISTS test_parquet;
 CREATE TABLE test_parquet (col1 int, col2 String) ENGINE=File(Parquet)  SETTINGS output_format_parquet_use_custom_encoder=false, output_format_parquet_write_page_index=false;

From 4a8be15ca8bffc445a6c66cacb5226946531e105 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Tue, 6 Aug 2024 10:25:33 +0000
Subject: [PATCH 1438/1488] Bump rocksdb to v8.11.4

---
 contrib/rocksdb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contrib/rocksdb b/contrib/rocksdb
index 49ce8a1064d..5c2be544f55 160000
--- a/contrib/rocksdb
+++ b/contrib/rocksdb
@@ -1 +1 @@
-Subproject commit 49ce8a1064dd1ad89117899839bf136365e49e79
+Subproject commit 5c2be544f5509465957706c955b6d623e889ac4e

From 7c15ad3966a982a3597034f8a66d91f2750dd2d0 Mon Sep 17 00:00:00 2001
From: Anton Popov <anton@clickhouse.com>
Date: Tue, 6 Aug 2024 10:15:53 +0000
Subject: [PATCH 1439/1488] do not try to create azure container if not needed

---
 src/Common/ProfileEvents.cpp                  |  4 +-
 .../AzureBlobStorageCommon.cpp                | 45 ++++++++++++++++++-
 2 files changed, 46 insertions(+), 3 deletions(-)

diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp
index 2b090136e2e..c6cf6c04af3 100644
--- a/src/Common/ProfileEvents.cpp
+++ b/src/Common/ProfileEvents.cpp
@@ -457,6 +457,7 @@ The server successfully detected this situation and will download merged part fr
     M(AzureDeleteObjects, "Number of Azure blob storage API DeleteObject(s) calls.") \
     M(AzureListObjects, "Number of Azure blob storage API ListObjects calls.") \
     M(AzureGetProperties, "Number of Azure blob storage API GetProperties calls.") \
+    M(AzureCreateContainer, "Number of Azure blob storage API CreateContainer calls.") \
     \
     M(DiskAzureGetObject, "Number of Disk Azure API GetObject calls.") \
     M(DiskAzureUpload, "Number of Disk Azure blob storage API Upload calls") \
@@ -464,8 +465,9 @@ The server successfully detected this situation and will download merged part fr
     M(DiskAzureCommitBlockList, "Number of Disk Azure blob storage API CommitBlockList calls") \
     M(DiskAzureCopyObject, "Number of Disk Azure blob storage API CopyObject calls") \
     M(DiskAzureListObjects, "Number of Disk Azure blob storage API ListObjects calls.") \
-    M(DiskAzureDeleteObjects, "Number of Azure blob storage API DeleteObject(s) calls.") \
+    M(DiskAzureDeleteObjects, "Number of Disk Azure blob storage API DeleteObject(s) calls.") \
     M(DiskAzureGetProperties, "Number of Disk Azure blob storage API GetProperties calls.") \
+    M(DiskAzureCreateContainer, "Number of Disk Azure blob storage API CreateContainer calls.") \
     \
     M(ReadBufferFromAzureMicroseconds, "Time spent on reading from Azure.") \
     M(ReadBufferFromAzureInitMicroseconds, "Time spent initializing connection to Azure.") \
diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp b/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp
index 0aa3b9c40b5..9043edd66a0 100644
--- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp
+++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp
@@ -11,6 +11,14 @@
 #include <Poco/Util/AbstractConfiguration.h>
 #include <Interpreters/Context.h>
 
+namespace ProfileEvents
+{
+    extern const Event AzureGetProperties;
+    extern const Event DiskAzureGetProperties;
+    extern const Event AzureCreateContainer;
+    extern const Event DiskAzureCreateContainer;
+}
+
 namespace DB
 {
 
@@ -214,20 +222,53 @@ void processURL(const String & url, const String & container_name, Endpoint & en
     }
 }
 
+static bool containerExists(const ContainerClient & client)
+{
+    ProfileEvents::increment(ProfileEvents::AzureGetProperties);
+    if (client.GetClickhouseOptions().IsClientForDisk)
+        ProfileEvents::increment(ProfileEvents::DiskAzureGetProperties);
+
+    try
+    {
+        client.GetProperties();
+        return true;
+    }
+    catch (const Azure::Storage::StorageException & e)
+    {
+        if (e.StatusCode == Azure::Core::Http::HttpStatusCode::NotFound)
+            return false;
+        throw;
+    }
+}
+
 std::unique_ptr<ContainerClient> getContainerClient(const ConnectionParams & params, bool readonly)
 {
     if (params.endpoint.container_already_exists.value_or(false) || readonly)
+    {
         return params.createForContainer();
+    }
+
+    if (!params.endpoint.container_already_exists.has_value())
+    {
+        auto container_client = params.createForContainer();
+        if (containerExists(*container_client))
+            return container_client;
+    }
 
     try
     {
         auto service_client = params.createForService();
+
+        ProfileEvents::increment(ProfileEvents::AzureCreateContainer);
+        if (params.client_options.ClickhouseOptions.IsClientForDisk)
+            ProfileEvents::increment(ProfileEvents::DiskAzureCreateContainer);
+
         return std::make_unique<ContainerClient>(service_client->CreateBlobContainer(params.endpoint.container_name).Value);
     }
     catch (const Azure::Storage::StorageException & e)
     {
-        /// If container_already_exists is not set (in config), ignore already exists error.
-        /// (Conflict - The specified container already exists)
+        /// If container_already_exists is not set (in config), ignore already exists error. Conflict - The specified container already exists.
+        /// To avoid race with creation of container handle this error despite that we have already checked the existence of container.
         if (!params.endpoint.container_already_exists.has_value() && e.StatusCode == Azure::Core::Http::HttpStatusCode::Conflict)
             return params.createForContainer();
         throw;

From 7e87650a001279317ce79c36415160639a83a4da Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Tue, 6 Aug 2024 10:31:13 +0000
Subject: [PATCH 1440/1488] Bump rocksdb to v9.0.1

---
 contrib/rocksdb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contrib/rocksdb b/contrib/rocksdb
index 5c2be544f55..36540d43550 160000
--- a/contrib/rocksdb
+++ b/contrib/rocksdb
@@ -1 +1 @@
-Subproject commit 5c2be544f5509465957706c955b6d623e889ac4e
+Subproject commit 36540d43550da19e96515bd731ee416a8787672c

From f0a6ae220530fa9b34d23aa722ed8816842410ce Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Tue, 6 Aug 2024 10:32:40 +0000
Subject: [PATCH 1441/1488] Bump rocksdb to v9.1.1

---
 contrib/rocksdb                      | 2 +-
 contrib/rocksdb-cmake/CMakeLists.txt | 1 +
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/contrib/rocksdb b/contrib/rocksdb
index 36540d43550..6f7cabeac80 160000
--- a/contrib/rocksdb
+++ b/contrib/rocksdb
@@ -1 +1 @@
-Subproject commit 36540d43550da19e96515bd731ee416a8787672c
+Subproject commit 6f7cabeac80a3a6150be2c8a8369fcecb107bf43
diff --git a/contrib/rocksdb-cmake/CMakeLists.txt b/contrib/rocksdb-cmake/CMakeLists.txt
index 57c056532c6..0157f31d5d9 100644
--- a/contrib/rocksdb-cmake/CMakeLists.txt
+++ b/contrib/rocksdb-cmake/CMakeLists.txt
@@ -150,6 +150,7 @@ set(SOURCES
     ${ROCKSDB_SOURCE_DIR}/db/memtable_list.cc
     ${ROCKSDB_SOURCE_DIR}/db/merge_helper.cc
     ${ROCKSDB_SOURCE_DIR}/db/merge_operator.cc
+    ${ROCKSDB_SOURCE_DIR}/db/multi_cf_iterator.cc
     ${ROCKSDB_SOURCE_DIR}/db/output_validator.cc
     ${ROCKSDB_SOURCE_DIR}/db/periodic_task_scheduler.cc
     ${ROCKSDB_SOURCE_DIR}/db/range_del_aggregator.cc

From b3f5ddcd658efc8ad84b32481239750e8999ba0a Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Tue, 6 Aug 2024 10:37:35 +0000
Subject: [PATCH 1442/1488] Bump rocksdb to 9.2.1

---
 contrib/rocksdb                      |  2 +-
 contrib/rocksdb-cmake/CMakeLists.txt | 14 +++++++++-----
 2 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/contrib/rocksdb b/contrib/rocksdb
index 6f7cabeac80..08f93221f50 160000
--- a/contrib/rocksdb
+++ b/contrib/rocksdb
@@ -1 +1 @@
-Subproject commit 6f7cabeac80a3a6150be2c8a8369fcecb107bf43
+Subproject commit 08f93221f50700f19f11555fb46abfe708a716d1
diff --git a/contrib/rocksdb-cmake/CMakeLists.txt b/contrib/rocksdb-cmake/CMakeLists.txt
index 0157f31d5d9..cff8f832f23 100644
--- a/contrib/rocksdb-cmake/CMakeLists.txt
+++ b/contrib/rocksdb-cmake/CMakeLists.txt
@@ -88,6 +88,7 @@ set(SOURCES
     ${ROCKSDB_SOURCE_DIR}/cache/sharded_cache.cc
     ${ROCKSDB_SOURCE_DIR}/cache/tiered_secondary_cache.cc
     ${ROCKSDB_SOURCE_DIR}/db/arena_wrapped_db_iter.cc
+    ${ROCKSDB_SOURCE_DIR}/db/attribute_group_iterator_impl.cc
     ${ROCKSDB_SOURCE_DIR}/db/blob/blob_contents.cc
     ${ROCKSDB_SOURCE_DIR}/db/blob/blob_fetcher.cc
     ${ROCKSDB_SOURCE_DIR}/db/blob/blob_file_addition.cc
@@ -104,6 +105,7 @@ set(SOURCES
     ${ROCKSDB_SOURCE_DIR}/db/blob/prefetch_buffer_collection.cc
     ${ROCKSDB_SOURCE_DIR}/db/builder.cc
     ${ROCKSDB_SOURCE_DIR}/db/c.cc
+    ${ROCKSDB_SOURCE_DIR}/db/coalescing_iterator.cc
     ${ROCKSDB_SOURCE_DIR}/db/column_family.cc
     ${ROCKSDB_SOURCE_DIR}/db/compaction/compaction.cc
     ${ROCKSDB_SOURCE_DIR}/db/compaction/compaction_iterator.cc
@@ -150,7 +152,6 @@ set(SOURCES
     ${ROCKSDB_SOURCE_DIR}/db/memtable_list.cc
     ${ROCKSDB_SOURCE_DIR}/db/merge_helper.cc
     ${ROCKSDB_SOURCE_DIR}/db/merge_operator.cc
-    ${ROCKSDB_SOURCE_DIR}/db/multi_cf_iterator.cc
     ${ROCKSDB_SOURCE_DIR}/db/output_validator.cc
     ${ROCKSDB_SOURCE_DIR}/db/periodic_task_scheduler.cc
     ${ROCKSDB_SOURCE_DIR}/db/range_del_aggregator.cc
@@ -389,6 +390,7 @@ set(SOURCES
     ${ROCKSDB_SOURCE_DIR}/utilities/transactions/write_prepared_txn_db.cc
     ${ROCKSDB_SOURCE_DIR}/utilities/transactions/write_unprepared_txn.cc
     ${ROCKSDB_SOURCE_DIR}/utilities/transactions/write_unprepared_txn_db.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/types_util.cc
     ${ROCKSDB_SOURCE_DIR}/utilities/ttl/db_ttl_impl.cc
     ${ROCKSDB_SOURCE_DIR}/utilities/wal_filter.cc
     ${ROCKSDB_SOURCE_DIR}/utilities/write_batch_with_index/write_batch_with_index.cc
@@ -419,10 +421,12 @@ if(HAS_ARMV8_CRC)
 endif(HAS_ARMV8_CRC)
 
 list(APPEND SOURCES
-    "${ROCKSDB_SOURCE_DIR}/port/port_posix.cc"
-    "${ROCKSDB_SOURCE_DIR}/env/env_posix.cc"
-    "${ROCKSDB_SOURCE_DIR}/env/fs_posix.cc"
-    "${ROCKSDB_SOURCE_DIR}/env/io_posix.cc")
+    ${ROCKSDB_SOURCE_DIR}/db/db_impl/db_impl_follower.cc
+    ${ROCKSDB_SOURCE_DIR}/port/port_posix.cc
+    ${ROCKSDB_SOURCE_DIR}/env/env_posix.cc
+    ${ROCKSDB_SOURCE_DIR}/env/fs_on_demand.cc
+    ${ROCKSDB_SOURCE_DIR}/env/fs_posix.cc
+    ${ROCKSDB_SOURCE_DIR}/env/io_posix.cc)
 
 add_library(_rocksdb ${SOURCES})
 add_library(ch_contrib::rocksdb ALIAS _rocksdb)

From e7110be48c7b2b752cd51fd72a23fe956248077c Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Tue, 6 Aug 2024 10:43:21 +0000
Subject: [PATCH 1443/1488] Bump rocksdb to v9.3.1

---
 contrib/rocksdb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contrib/rocksdb b/contrib/rocksdb
index 08f93221f50..c5201abc4a9 160000
--- a/contrib/rocksdb
+++ b/contrib/rocksdb
@@ -1 +1 @@
-Subproject commit 08f93221f50700f19f11555fb46abfe708a716d1
+Subproject commit c5201abc4a983450f9423435a4405829be23d0a8

From 2a2ba4e685022d3687565586c06dc56dfc276f10 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Tue, 6 Aug 2024 10:47:03 +0000
Subject: [PATCH 1444/1488] Bump rocksdb to 9.4.0

---
 contrib/rocksdb                      | 2 +-
 contrib/rocksdb-cmake/CMakeLists.txt | 5 +++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/contrib/rocksdb b/contrib/rocksdb
index c5201abc4a9..5f003e4a22d 160000
--- a/contrib/rocksdb
+++ b/contrib/rocksdb
@@ -1 +1 @@
-Subproject commit c5201abc4a983450f9423435a4405829be23d0a8
+Subproject commit 5f003e4a22d2e48e37c98d9620241237cd30dd24
diff --git a/contrib/rocksdb-cmake/CMakeLists.txt b/contrib/rocksdb-cmake/CMakeLists.txt
index cff8f832f23..b39a739d04c 100644
--- a/contrib/rocksdb-cmake/CMakeLists.txt
+++ b/contrib/rocksdb-cmake/CMakeLists.txt
@@ -126,6 +126,7 @@ set(SOURCES
     ${ROCKSDB_SOURCE_DIR}/db/db_impl/db_impl_write.cc
     ${ROCKSDB_SOURCE_DIR}/db/db_impl/db_impl_compaction_flush.cc
     ${ROCKSDB_SOURCE_DIR}/db/db_impl/db_impl_files.cc
+    ${ROCKSDB_SOURCE_DIR}/db/db_impl/db_impl_follower.cc
     ${ROCKSDB_SOURCE_DIR}/db/db_impl/db_impl_open.cc
     ${ROCKSDB_SOURCE_DIR}/db/db_impl/db_impl_debug.cc
     ${ROCKSDB_SOURCE_DIR}/db/db_impl/db_impl_experimental.cc
@@ -183,6 +184,7 @@ set(SOURCES
     ${ROCKSDB_SOURCE_DIR}/env/env_encryption.cc
     ${ROCKSDB_SOURCE_DIR}/env/file_system.cc
     ${ROCKSDB_SOURCE_DIR}/env/file_system_tracer.cc
+    ${ROCKSDB_SOURCE_DIR}/env/fs_on_demand.cc
     ${ROCKSDB_SOURCE_DIR}/env/fs_remap.cc
     ${ROCKSDB_SOURCE_DIR}/env/mock_env.cc
     ${ROCKSDB_SOURCE_DIR}/env/unique_id_gen.cc
@@ -370,6 +372,7 @@ set(SOURCES
     ${ROCKSDB_SOURCE_DIR}/utilities/persistent_cache/volatile_tier_impl.cc
     ${ROCKSDB_SOURCE_DIR}/utilities/simulator_cache/cache_simulator.cc
     ${ROCKSDB_SOURCE_DIR}/utilities/simulator_cache/sim_cache.cc
+    ${ROCKSDB_SOURCE_DIR}/utilities/table_properties_collectors/compact_for_tiering_collector.cc
     ${ROCKSDB_SOURCE_DIR}/utilities/table_properties_collectors/compact_on_deletion_collector.cc
     ${ROCKSDB_SOURCE_DIR}/utilities/trace/file_trace_reader_writer.cc
     ${ROCKSDB_SOURCE_DIR}/utilities/trace/replayer_impl.cc
@@ -421,10 +424,8 @@ if(HAS_ARMV8_CRC)
 endif(HAS_ARMV8_CRC)
 
 list(APPEND SOURCES
-    ${ROCKSDB_SOURCE_DIR}/db/db_impl/db_impl_follower.cc
     ${ROCKSDB_SOURCE_DIR}/port/port_posix.cc
     ${ROCKSDB_SOURCE_DIR}/env/env_posix.cc
-    ${ROCKSDB_SOURCE_DIR}/env/fs_on_demand.cc
     ${ROCKSDB_SOURCE_DIR}/env/fs_posix.cc
     ${ROCKSDB_SOURCE_DIR}/env/io_posix.cc)
 

From df0dac2f5b509438cce28214b78765c46439aa8c Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Tue, 6 Aug 2024 11:17:41 +0000
Subject: [PATCH 1445/1488] Enable jemalloc and liburing in rocksdb

---
 contrib/CMakeLists.txt               |  2 +-
 contrib/rocksdb-cmake/CMakeLists.txt | 30 +++++++++++++++++-----------
 2 files changed, 19 insertions(+), 13 deletions(-)

diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt
index 977efda15ff..eb3afe0ccdf 100644
--- a/contrib/CMakeLists.txt
+++ b/contrib/CMakeLists.txt
@@ -71,7 +71,6 @@ add_contrib (zlib-ng-cmake zlib-ng)
 add_contrib (bzip2-cmake bzip2)
 add_contrib (minizip-ng-cmake minizip-ng)
 add_contrib (snappy-cmake snappy)
-add_contrib (rocksdb-cmake rocksdb)
 add_contrib (thrift-cmake thrift)
 # parquet/arrow/orc
 add_contrib (arrow-cmake arrow) # requires: snappy, thrift, double-conversion
@@ -148,6 +147,7 @@ add_contrib (hive-metastore-cmake hive-metastore) # requires: thrift, avro, arro
 add_contrib (cppkafka-cmake cppkafka)
 add_contrib (libpqxx-cmake libpqxx)
 add_contrib (libpq-cmake libpq)
+add_contrib (rocksdb-cmake rocksdb) # requires: jemalloc, snappy, zlib, lz4, zstd, liburing
 add_contrib (nuraft-cmake NuRaft)
 add_contrib (fast_float-cmake fast_float)
 add_contrib (idna-cmake idna)
diff --git a/contrib/rocksdb-cmake/CMakeLists.txt b/contrib/rocksdb-cmake/CMakeLists.txt
index b39a739d04c..7e5e9a28d0f 100644
--- a/contrib/rocksdb-cmake/CMakeLists.txt
+++ b/contrib/rocksdb-cmake/CMakeLists.txt
@@ -5,36 +5,38 @@ if (NOT ENABLE_ROCKSDB OR NO_SSE3_OR_HIGHER) # assumes SSE4.2 and PCLMUL
   return()
 endif()
 
-# not in original build system, otherwise xxHash.cc fails to compile with ClickHouse C++23 default
-set (CMAKE_CXX_STANDARD 20)
-
-# Always disable jemalloc for rocksdb by default because it introduces non-standard jemalloc APIs
-option(WITH_JEMALLOC "build with JeMalloc" OFF)
-
-option(WITH_LIBURING "build with liburing" OFF) # TODO could try to enable this conditionally, depending on ClickHouse's ENABLE_LIBURING
-
 # ClickHouse cannot be compiled without snappy, lz4, zlib, zstd
 option(WITH_SNAPPY "build with SNAPPY" ON)
 option(WITH_LZ4 "build with lz4" ON)
 option(WITH_ZLIB "build with zlib" ON)
 option(WITH_ZSTD "build with zstd" ON)
 
-if(WITH_SNAPPY)
+if (ENABLE_JEMALLOC)
+  add_definitions(-DROCKSDB_JEMALLOC -DJEMALLOC_NO_DEMANGLE)
+  list (APPEND THIRDPARTY_LIBS ch_contrib::jemalloc)
+endif ()
+
+if (ENABLE_LIBURING)
+  add_definitions(-DROCKSDB_IOURING_PRESENT)
+  list (APPEND THIRDPARTY_LIBS ch_contrib::liburing)
+endif ()
+
+if (WITH_SNAPPY)
   add_definitions(-DSNAPPY)
   list(APPEND THIRDPARTY_LIBS ch_contrib::snappy)
 endif()
 
-if(WITH_ZLIB)
+if (WITH_ZLIB)
   add_definitions(-DZLIB)
   list(APPEND THIRDPARTY_LIBS ch_contrib::zlib)
 endif()
 
-if(WITH_LZ4)
+if (WITH_LZ4)
   add_definitions(-DLZ4)
   list(APPEND THIRDPARTY_LIBS ch_contrib::lz4)
 endif()
 
-if(WITH_ZSTD)
+if (WITH_ZSTD)
   add_definitions(-DZSTD)
   list(APPEND THIRDPARTY_LIBS ch_contrib::zstd)
 endif()
@@ -433,5 +435,9 @@ add_library(_rocksdb ${SOURCES})
 add_library(ch_contrib::rocksdb ALIAS _rocksdb)
 target_link_libraries(_rocksdb PRIVATE ${THIRDPARTY_LIBS} ${SYSTEM_LIBS})
 
+# Not in the native build system but useful anyways:
+# Make all functions in xxHash.h inline. Beneficial for performance: https://github.com/Cyan4973/xxHash/tree/v0.8.2#build-modifiers
+target_compile_definitions (_rocksdb PRIVATE XXH_INLINE_ALL)
+
 # SYSTEM is required to overcome some issues
 target_include_directories(_rocksdb SYSTEM BEFORE INTERFACE "${ROCKSDB_SOURCE_DIR}/include")

From 523767c0c0725773c074e1ad00229b27865a34f2 Mon Sep 17 00:00:00 2001
From: Sema Checherinda <Sema.Checherinda@clickhouse.com>
Date: Tue, 30 Jul 2024 17:26:28 +0200
Subject: [PATCH 1446/1488] run 01171 test in parallel

---
 .../01171_mv_select_insert_isolation_long.sh       | 14 ++++----------
 1 file changed, 4 insertions(+), 10 deletions(-)

diff --git a/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh b/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh
index 2fb58e4cc57..27f063c34d8 100755
--- a/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh
+++ b/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh
@@ -1,6 +1,5 @@
 #!/usr/bin/env bash
-# Tags: long, no-parallel, no-ordinary-database
-# Test is too heavy, avoid parallel run in Flaky Check
+# Tags: long, no-ordinary-database
 # shellcheck disable=SC2119
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
@@ -22,11 +21,6 @@ $CLICKHOUSE_CLIENT --query "CREATE TABLE tmp (x UInt32, nm Int32) ENGINE=MergeTr
 
 $CLICKHOUSE_CLIENT --query "INSERT INTO src VALUES (0, 0)"
 
-function get_now()
-{
-    date +%s
-}
-
 is_pid_exist()
 {
     local pid=$1
@@ -42,14 +36,14 @@ function run_until_deadline_and_at_least_times()
     local function_to_run=$1; shift
 
     local started_time
-    started_time=$(get_now)
+    started_time=$SECONDS
     local i=0
 
     while true
     do
         $function_to_run $i "$@"
 
-        [[ $(get_now) -lt $deadline ]] || break
+        [[ $SECONDS -lt $deadline ]] || break
 
         i=$(($i + 1))
     done
@@ -165,7 +159,7 @@ if [[ $((MAIN_TIME_PART + SECOND_TIME_PART + WAIT_FINISH + LAST_TIME_GAP)) -ge
     exit 1
 fi
 
-START_TIME=$(get_now)
+START_TIME=$SECONDS
 STOP_TIME=$((START_TIME + MAIN_TIME_PART))
 SECOND_STOP_TIME=$((STOP_TIME + SECOND_TIME_PART))
 MIN_ITERATIONS=20

From 244da490138668882db10770679b940d2c4adadf Mon Sep 17 00:00:00 2001
From: Sema Checherinda <Sema.Checherinda@clickhouse.com>
Date: Tue, 30 Jul 2024 18:41:44 +0200
Subject: [PATCH 1447/1488] add max iteration

---
 .../01171_mv_select_insert_isolation_long.sh  | 29 ++++++++++---------
 1 file changed, 16 insertions(+), 13 deletions(-)

diff --git a/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh b/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh
index 27f063c34d8..fe5bfe529b7 100755
--- a/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh
+++ b/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh
@@ -27,28 +27,30 @@ is_pid_exist()
     ps -p $pid > /dev/null
 }
 
-function run_until_deadline_and_at_least_times()
+function run_until_deadline_with_min_max_iterations()
 {
     set -e
 
     local deadline=$1; shift
     local min_iterations=$1; shift
+    local max_iterations=$1; shift
     local function_to_run=$1; shift
 
     local started_time
     started_time=$SECONDS
-    local i=0
+    local iteration=0
 
     while true
     do
-        $function_to_run $i "$@"
+        $function_to_run $iteration "$@"
 
         [[ $SECONDS -lt $deadline ]] || break
+        [[ $iteration -lt $max_iterations ]] || break
 
-        i=$(($i + 1))
+        iteration=$(($iteration + 1))
     done
 
-    [[ $i -gt $min_iterations ]] || echo "$i/$min_iterations : not enough iterations of $function_to_run has been made from $started_time until $deadline" >&2
+    [[ $iteration -gt $min_iterations ]] || echo "$iteration/$min_iterations : not enough iterations of $function_to_run has been made from $started_time until $deadline" >&2
 }
 
 function insert_commit_action()
@@ -163,17 +165,18 @@ START_TIME=$SECONDS
 STOP_TIME=$((START_TIME + MAIN_TIME_PART))
 SECOND_STOP_TIME=$((STOP_TIME + SECOND_TIME_PART))
 MIN_ITERATIONS=20
+MAX_ITERATIONS=200
 
-run_until_deadline_and_at_least_times $STOP_TIME $MIN_ITERATIONS insert_commit_action 1   & PID_1=$!
-run_until_deadline_and_at_least_times $STOP_TIME $MIN_ITERATIONS insert_commit_action 2   & PID_2=$!
-run_until_deadline_and_at_least_times $STOP_TIME $MIN_ITERATIONS insert_rollback_action 3 & PID_3=$!
+run_until_deadline_with_min_max_iterations $STOP_TIME $MIN_ITERATIONS $MAX_ITERATIONS insert_commit_action 1   & PID_1=$!
+run_until_deadline_with_min_max_iterations $STOP_TIME $MIN_ITERATIONS $MAX_ITERATIONS insert_commit_action 2   & PID_2=$!
+run_until_deadline_with_min_max_iterations $STOP_TIME $MIN_ITERATIONS $MAX_ITERATIONS insert_rollback_action 3 & PID_3=$!
 
-run_until_deadline_and_at_least_times $SECOND_STOP_TIME $MIN_ITERATIONS optimize_action      & PID_4=$!
-run_until_deadline_and_at_least_times $SECOND_STOP_TIME $MIN_ITERATIONS select_action        & PID_5=$!
-run_until_deadline_and_at_least_times $SECOND_STOP_TIME $MIN_ITERATIONS select_insert_action & PID_6=$!
+run_until_deadline_with_min_max_iterations $SECOND_STOP_TIME $MIN_ITERATIONS $MAX_ITERATIONS optimize_action      & PID_4=$!
+run_until_deadline_with_min_max_iterations $SECOND_STOP_TIME $MIN_ITERATIONS $MAX_ITERATIONS select_action        & PID_5=$!
+run_until_deadline_with_min_max_iterations $SECOND_STOP_TIME $MIN_ITERATIONS $MAX_ITERATIONS select_insert_action & PID_6=$!
 sleep 0.$RANDOM
-run_until_deadline_and_at_least_times $SECOND_STOP_TIME $MIN_ITERATIONS select_action        & PID_7=$!
-run_until_deadline_and_at_least_times $SECOND_STOP_TIME $MIN_ITERATIONS select_insert_action & PID_8=$!
+run_until_deadline_with_min_max_iterations $SECOND_STOP_TIME $MIN_ITERATIONS $MAX_ITERATIONS select_action        & PID_7=$!
+run_until_deadline_with_min_max_iterations $SECOND_STOP_TIME $MIN_ITERATIONS $MAX_ITERATIONS select_insert_action & PID_8=$!
 
 wait $PID_1 || echo "insert_commit_action has failed with status $?" 2>&1
 wait $PID_2 || echo "second insert_commit_action has failed with status $?" 2>&1

From c896165e161b2267ae4d6bc255e028d7c0899bd9 Mon Sep 17 00:00:00 2001
From: Sema Checherinda <Sema.Checherinda@clickhouse.com>
Date: Wed, 31 Jul 2024 17:49:39 +0200
Subject: [PATCH 1448/1488] add min time

---
 .../01171_mv_select_insert_isolation_long.sh  | 53 +++++++++++--------
 1 file changed, 30 insertions(+), 23 deletions(-)

diff --git a/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh b/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh
index fe5bfe529b7..ff71d37cb32 100755
--- a/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh
+++ b/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh
@@ -27,15 +27,20 @@ is_pid_exist()
     ps -p $pid > /dev/null
 }
 
-function run_until_deadline_with_min_max_iterations()
+function run_with_time_and_iterations_limits()
 {
     set -e
 
-    local deadline=$1; shift
+    local min_time=$1; shift
+    local max_time=$1; shift
     local min_iterations=$1; shift
     local max_iterations=$1; shift
     local function_to_run=$1; shift
 
+    # if [ "${1:-X}" = "1" ]; then
+    #     set -x
+    # fi
+
     local started_time
     started_time=$SECONDS
     local iteration=0
@@ -44,13 +49,14 @@ function run_until_deadline_with_min_max_iterations()
     do
         $function_to_run $iteration "$@"
 
-        [[ $SECONDS -lt $deadline ]] || break
-        [[ $iteration -lt $max_iterations ]] || break
+        [[ $SECONDS -lt $max_time ]] || break
+        [[ $SECONDS -lt $min_time ]] || [[ $iteration -lt $max_iterations ]] || break
 
         iteration=$(($iteration + 1))
     done
 
-    [[ $iteration -gt $min_iterations ]] || echo "$iteration/$min_iterations : not enough iterations of $function_to_run has been made from $started_time until $deadline" >&2
+    [[ $iteration -gt $min_iterations ]] || echo "$iteration/$min_iterations : not enough iterations of $function_to_run has been made from $started_time until $max_time" >&2
+    set +x
 }
 
 function insert_commit_action()
@@ -151,43 +157,44 @@ function select_insert_action()
         ROLLBACK;"
 }
 
-MAIN_TIME_PART=400
-SECOND_TIME_PART=30
+MIN_SECONDS=5
+MAX_SECONDS=300
 WAIT_FINISH=60
-LAST_TIME_GAP=10
 
-if [[ $((MAIN_TIME_PART + SECOND_TIME_PART + WAIT_FINISH + LAST_TIME_GAP)) -ge  600 ]]; then
+if [[ $((MAX_SECONDS + WAIT_FINISH)) -ge  550 ]]; then
     echo "time sttings are wrong" 2>&1
     exit 1
 fi
 
 START_TIME=$SECONDS
-STOP_TIME=$((START_TIME + MAIN_TIME_PART))
-SECOND_STOP_TIME=$((STOP_TIME + SECOND_TIME_PART))
+MIN_TIME=$((START_TIME + MIN_SECONDS))
+MAX_TIME=$((START_TIME + MAX_SECONDS))
 MIN_ITERATIONS=20
 MAX_ITERATIONS=200
 
-run_until_deadline_with_min_max_iterations $STOP_TIME $MIN_ITERATIONS $MAX_ITERATIONS insert_commit_action 1   & PID_1=$!
-run_until_deadline_with_min_max_iterations $STOP_TIME $MIN_ITERATIONS $MAX_ITERATIONS insert_commit_action 2   & PID_2=$!
-run_until_deadline_with_min_max_iterations $STOP_TIME $MIN_ITERATIONS $MAX_ITERATIONS insert_rollback_action 3 & PID_3=$!
+run_with_time_and_iterations_limits $MIN_TIME $MAX_TIME $MIN_ITERATIONS $MAX_ITERATIONS insert_commit_action 1   & PID_1=$!
+run_with_time_and_iterations_limits $MIN_TIME $MAX_TIME $MIN_ITERATIONS $MAX_ITERATIONS insert_commit_action 2   & PID_2=$!
+run_with_time_and_iterations_limits $MIN_TIME $MAX_TIME $MIN_ITERATIONS $MAX_ITERATIONS insert_rollback_action 3 & PID_3=$!
 
-run_until_deadline_with_min_max_iterations $SECOND_STOP_TIME $MIN_ITERATIONS $MAX_ITERATIONS optimize_action      & PID_4=$!
-run_until_deadline_with_min_max_iterations $SECOND_STOP_TIME $MIN_ITERATIONS $MAX_ITERATIONS select_action        & PID_5=$!
-run_until_deadline_with_min_max_iterations $SECOND_STOP_TIME $MIN_ITERATIONS $MAX_ITERATIONS select_insert_action & PID_6=$!
+run_with_time_and_iterations_limits $MIN_TIME $MAX_TIME $MIN_ITERATIONS $MAX_ITERATIONS optimize_action      & PID_4=$!
+run_with_time_and_iterations_limits $MIN_TIME $MAX_TIME $MIN_ITERATIONS $MAX_ITERATIONS select_action        & PID_5=$!
+run_with_time_and_iterations_limits $MIN_TIME $MAX_TIME $MIN_ITERATIONS $MAX_ITERATIONS select_insert_action & PID_6=$!
 sleep 0.$RANDOM
-run_until_deadline_with_min_max_iterations $SECOND_STOP_TIME $MIN_ITERATIONS $MAX_ITERATIONS select_action        & PID_7=$!
-run_until_deadline_with_min_max_iterations $SECOND_STOP_TIME $MIN_ITERATIONS $MAX_ITERATIONS select_insert_action & PID_8=$!
-
-wait $PID_1 || echo "insert_commit_action has failed with status $?" 2>&1
-wait $PID_2 || echo "second insert_commit_action has failed with status $?" 2>&1
-wait $PID_3 || echo "insert_rollback_action has failed with status $?" 2>&1
+run_with_time_and_iterations_limits $MIN_TIME $MAX_TIME $MIN_ITERATIONS $MAX_ITERATIONS select_action        & PID_7=$!
+run_with_time_and_iterations_limits $MIN_TIME $MAX_TIME $MIN_ITERATIONS $MAX_ITERATIONS select_insert_action & PID_8=$!
 
+is_pid_exist $PID_1 || echo "insert_commit_action is not running" 2>&1
+is_pid_exist $PID_2 || echo "second insert_commit_action is not running" 2>&1
+is_pid_exist $PID_3 || echo "insert_rollback_action is not running" 2>&1
 is_pid_exist $PID_4 || echo "optimize_action is not running" 2>&1
 is_pid_exist $PID_5 || echo "select_action is not running" 2>&1
 is_pid_exist $PID_6 || echo "select_insert_action is not running" 2>&1
 is_pid_exist $PID_7 || echo "second select_action is not running" 2>&1
 is_pid_exist $PID_8 || echo "second select_insert_action is not running" 2>&1
 
+wait $PID_1 || echo "insert_commit_action has failed with status $?" 2>&1
+wait $PID_2 || echo "second insert_commit_action has failed with status $?" 2>&1
+wait $PID_3 || echo "insert_rollback_action has failed with status $?" 2>&1
 wait $PID_4 || echo "optimize_action has failed with status $?" 2>&1
 wait $PID_5 || echo "select_action has failed with status $?" 2>&1
 wait $PID_6 || echo "select_insert_action has failed with status $?" 2>&1

From 874a7dbe378e83331d6e4fab6aa82a0409fbfc5d Mon Sep 17 00:00:00 2001
From: Sema Checherinda <Sema.Checherinda@clickhouse.com>
Date: Thu, 1 Aug 2024 17:39:36 +0200
Subject: [PATCH 1449/1488] adjust settings

---
 .../0_stateless/01171_mv_select_insert_isolation_long.sh        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh b/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh
index ff71d37cb32..ba6fc85ca1a 100755
--- a/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh
+++ b/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh
@@ -158,7 +158,7 @@ function select_insert_action()
 }
 
 MIN_SECONDS=5
-MAX_SECONDS=300
+MAX_SECONDS=400
 WAIT_FINISH=60
 
 if [[ $((MAX_SECONDS + WAIT_FINISH)) -ge  550 ]]; then

From 860050eb3d15f26c371a1aeab92fd8c0d166ce09 Mon Sep 17 00:00:00 2001
From: Anton Popov <anton@clickhouse.com>
Date: Tue, 6 Aug 2024 13:30:23 +0200
Subject: [PATCH 1450/1488] Update
 src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp

Co-authored-by: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com>
---
 .../ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp b/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp
index 9043edd66a0..1a0b6157a86 100644
--- a/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp
+++ b/src/Disks/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp
@@ -268,7 +268,7 @@ std::unique_ptr<ContainerClient> getContainerClient(const ConnectionParams & par
     catch (const Azure::Storage::StorageException & e)
     {
         /// If container_already_exists is not set (in config), ignore already exists error. Conflict - The specified container already exists.
-        /// To avoid race with creation of container handle this error despite that we have already checked the existence of container.
+        /// To avoid race with creation of container, handle this error despite that we have already checked the existence of container.
         if (!params.endpoint.container_already_exists.has_value() && e.StatusCode == Azure::Core::Http::HttpStatusCode::Conflict)
             return params.createForContainer();
         throw;

From 851d4d3fdfdb682d0f9c9a8b1945608c6bc53086 Mon Sep 17 00:00:00 2001
From: Sema Checherinda <Sema.Checherinda@clickhouse.com>
Date: Tue, 6 Aug 2024 13:56:37 +0200
Subject: [PATCH 1451/1488] adjust MIN_ITERATIONS

---
 .../0_stateless/01171_mv_select_insert_isolation_long.sh        | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh b/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh
index ba6fc85ca1a..f6a88c205c1 100755
--- a/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh
+++ b/tests/queries/0_stateless/01171_mv_select_insert_isolation_long.sh
@@ -169,7 +169,7 @@ fi
 START_TIME=$SECONDS
 MIN_TIME=$((START_TIME + MIN_SECONDS))
 MAX_TIME=$((START_TIME + MAX_SECONDS))
-MIN_ITERATIONS=20
+MIN_ITERATIONS=15
 MAX_ITERATIONS=200
 
 run_with_time_and_iterations_limits $MIN_TIME $MAX_TIME $MIN_ITERATIONS $MAX_ITERATIONS insert_commit_action 1   & PID_1=$!

From 5313c9539ef8f988a607da87849eaa1f12e787e1 Mon Sep 17 00:00:00 2001
From: Blargian <shaunstruwig.ss@gmail.com>
Date: Tue, 6 Aug 2024 14:34:06 +0200
Subject: [PATCH 1452/1488] update documentation for Float32/64 and variants

---
 .../functions/type-conversion-functions.md    | 420 +++++++++++++++++-
 1 file changed, 416 insertions(+), 4 deletions(-)

diff --git a/docs/en/sql-reference/functions/type-conversion-functions.md b/docs/en/sql-reference/functions/type-conversion-functions.md
index 1e618b8cdab..77dd1628fe4 100644
--- a/docs/en/sql-reference/functions/type-conversion-functions.md
+++ b/docs/en/sql-reference/functions/type-conversion-functions.md
@@ -3045,13 +3045,425 @@ toUInt256OrDefault('abc', CAST('0', 'UInt256')):  0
 - [`toUInt256OrZero`](#touint256orzero).
 - [`toUInt256OrNull`](#touint256ornull).
 
-## toFloat(32\|64)
+## toFloat32
 
-## toFloat(32\|64)OrZero
+Converts an input value to a value of type [`Float32`](../data-types/float.md). Throws an exception in case of an error.
 
-## toFloat(32\|64)OrNull
+**Syntax**
 
-## toFloat(32\|64)OrDefault
+```sql
+toFloat32(expr)
+```
+
+**Arguments**
+
+- `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions).
+
+Supported arguments:
+- Values of type (U)Int8/16/32/64/128/256.
+- String representations of (U)Int8/16/32/128/256.
+- Values of type Float32/64, including `NaN` and `Inf`.
+- String representations of Float32/64, including `NaN` and `Inf`.
+
+Unsupported arguments:
+- String representations of binary and hexadecimal values, e.g. `SELECT toFloat32('0xc0fe');`.
+
+**Returned value**
+
+- 32-bit floating point value. [Float32](../data-types/float.md).
+
+**Example**
+
+Query:
+
+```sql
+SELECT
+    toFloat32(42.7),
+    toFloat32('42.7'),
+    toFloat32('NaN')
+FORMAT vertical;
+```
+
+Result:
+
+```response
+Row 1:
+──────
+toFloat32(42.7):   42.7
+toFloat32('42.7'): 42.7
+toFloat32('NaN'):  nan
+```
+
+**See also**
+
+- [`toFloat32OrZero`](#tofloat32orzero).
+- [`toFloat32OrNull`](#tofloat32ornull).
+- [`toFloat32OrDefault`](#tofloat32ordefault).
+
+## toFloat32OrZero
+
+Like [`toFloat32`](#tofloat32), this function converts an input value to a value of type [Float32](../data-types/float.md) but returns `0` in case of an error.
+
+**Syntax**
+
+```sql
+toFloat32OrZero(x)
+```
+
+**Arguments**
+
+- `x` — A String representation of a number. [String](../data-types/string.md).
+
+Supported arguments:
+- String representations of (U)Int8/16/32/128/256, Float32/64.
+
+Unsupported arguments (return `0`):
+- String representations of binary and hexadecimal values, e.g. `SELECT toFloat32OrZero('0xc0fe');`.
+
+**Returned value**
+
+- 32-bit Float value if successful, otherwise `0`. [Float32](../data-types/float.md).
+
+**Example**
+
+Query:
+
+``` sql
+SELECT
+    toFloat32OrZero('42.7'),
+    toFloat32OrZero('abc')
+FORMAT vertical;
+```
+
+Result:
+
+```response
+Row 1:
+──────
+toFloat32OrZero('42.7'): 42.7
+toFloat32OrZero('abc'):  0
+```
+
+**See also**
+
+- [`toFloat32`](#tofloat32).
+- [`toFloat32OrNull`](#tofloat32ornull).
+- [`toFloat32OrDefault`](#tofloat32ordefault).
+
+## toFloat32OrNull
+
+Like [`toFloat32`](#tofloat32), this function converts an input value to a value of type [Float32](../data-types/float.md) but returns `NULL` in case of an error.
+
+**Syntax**
+
+```sql
+toFloat32OrNull(x)
+```
+
+**Arguments**
+
+- `x` — A String representation of a number. [String](../data-types/string.md).
+
+Supported arguments:
+- String representations of (U)Int8/16/32/128/256, Float32/64.
+
+Unsupported arguments (return `\N`):
+- String representations of binary and hexadecimal values, e.g. `SELECT toFloat32OrNull('0xc0fe');`.
+
+**Returned value**
+
+- 32-bit Float value if successful, otherwise `\N`. [Float32](../data-types/float.md).
+
+**Example**
+
+Query:
+
+``` sql
+SELECT
+    toFloat32OrNull('42.7'),
+    toFloat32OrNull('abc')
+FORMAT vertical;
+```
+
+Result:
+
+```response
+Row 1:
+──────
+toFloat32OrNull('42.7'): 42.7
+toFloat32OrNull('abc'):  ᴺᵁᴸᴸ
+```
+
+**See also**
+
+- [`toFloat32`](#tofloat32).
+- [`toFloat32OrZero`](#tofloat32orzero).
+- [`toFloat32OrDefault`](#tofloat32ordefault).
+
+## toFloat32OrDefault
+
+Like [`toFloat32`](#tofloat32), this function converts an input value to a value of type [Float32](../data-types/float.md) but returns the default value in case of an error.
+If no `default` value is passed then `0` is returned in case of an error.
+
+**Syntax**
+
+```sql
+toFloat32OrDefault(expr[, default])
+```
+
+**Arguments**
+
+- `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
+- `default` (optional) — The default value to return if parsing to type `Float32` is unsuccessful. [Float32](../data-types/float.md).
+
+Supported arguments:
+- Values of type (U)Int8/16/32/64/128/256.
+- String representations of (U)Int8/16/32/128/256.
+- Values of type Float32/64.
+- String representations of Float32/64.
+
+Arguments for which the default value is returned:
+- String representations of binary and hexadecimal values, e.g. `SELECT toFloat32OrDefault('0xc0fe', CAST('0', 'Float32'));`.
+
+**Returned value**
+
+- 32-bit Float value if successful, otherwise returns the default value if passed or `0` if not. [Float32](../data-types/float.md).
+
+**Example**
+
+Query:
+
+``` sql
+SELECT
+    toFloat32OrDefault('8', CAST('0', 'Float32')),
+    toFloat32OrDefault('abc', CAST('0', 'Float32'))
+FORMAT vertical;
+```
+
+Result:
+
+```response
+Row 1:
+──────
+toFloat32OrDefault('8', CAST('0', 'Float32')):   8
+toFloat32OrDefault('abc', CAST('0', 'Float32')): 0
+```
+
+**See also**
+
+- [`toFloat32`](#tofloat32).
+- [`toFloat32OrZero`](#tofloat32orzero).
+- [`toFloat32OrNull`](#tofloat32ornull).
+
+## toFloat64
+
+Converts an input value to a value of type [`Float64`](../data-types/float.md). Throws an exception in case of an error.
+
+**Syntax**
+
+```sql
+toFloat64(expr)
+```
+
+**Arguments**
+
+- `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions).
+
+Supported arguments:
+- Values of type (U)Int8/16/32/64/128/256.
+- String representations of (U)Int8/16/32/128/256.
+- Values of type Float32/64, including `NaN` and `Inf`.
+- String representations of type Float32/64, including `NaN` and `Inf`.
+
+Unsupported arguments:
+- String representations of binary and hexadecimal values, e.g. `SELECT toFloat64('0xc0fe');`.
+
+**Returned value**
+
+- 64-bit floating point value. [Float64](../data-types/float.md).
+
+**Example**
+
+Query:
+
+```sql
+SELECT
+    toFloat64(42.7),
+    toFloat64('42.7'),
+    toFloat64('NaN')
+FORMAT vertical;
+```
+
+Result:
+
+```response
+Row 1:
+──────
+toFloat64(42.7):   42.7
+toFloat64('42.7'): 42.7
+toFloat64('NaN'):  nan
+```
+
+**See also**
+
+- [`toFloat64OrZero`](#tofloat64orzero).
+- [`toFloat64OrNull`](#tofloat64ornull).
+- [`toFloat64OrDefault`](#tofloat64ordefault).
+
+## toFloat64OrZero
+
+Like [`toFloat64`](#tofloat64), this function converts an input value to a value of type [Float64](../data-types/float.md) but returns `0` in case of an error.
+
+**Syntax**
+
+```sql
+toFloat64OrZero(x)
+```
+
+**Arguments**
+
+- `x` — A String representation of a number. [String](../data-types/string.md).
+
+Supported arguments:
+- String representations of (U)Int8/16/32/128/256, Float32/64.
+
+Unsupported arguments (return `0`):
+- String representations of binary and hexadecimal values, e.g. `SELECT toFloat64OrZero('0xc0fe');`.
+
+**Returned value**
+
+- 64-bit Float value if successful, otherwise `0`. [Float64](../data-types/float.md).
+
+**Example**
+
+Query:
+
+``` sql
+SELECT
+    toFloat64OrZero('42.7'),
+    toFloat64OrZero('abc')
+FORMAT vertical;
+```
+
+Result:
+
+```response
+Row 1:
+──────
+toFloat64OrZero('42.7'): 42.7
+toFloat64OrZero('abc'):  0
+```
+
+**See also**
+
+- [`toFloat64`](#tofloat64).
+- [`toFloat64OrNull`](#tofloat64ornull).
+- [`toFloat64OrDefault`](#tofloat64ordefault).
+
+## toFloat64OrNull
+
+Like [`toFloat64`](#tofloat64), this function converts an input value to a value of type [Float64](../data-types/float.md) but returns `NULL` in case of an error.
+
+**Syntax**
+
+```sql
+toFloat64OrNull(x)
+```
+
+**Arguments**
+
+- `x` — A String representation of a number. [String](../data-types/string.md).
+
+Supported arguments:
+- String representations of (U)Int8/16/32/128/256, Float32/64.
+
+Unsupported arguments (return `\N`):
+- String representations of binary and hexadecimal values, e.g. `SELECT toFloat64OrNull('0xc0fe');`.
+
+**Returned value**
+
+- 64-bit Float value if successful, otherwise `\N`. [Float64](../data-types/float.md).
+
+**Example**
+
+Query:
+
+``` sql
+SELECT
+    toFloat64OrNull('42.7'),
+    toFloat64OrNull('abc')
+FORMAT vertical;
+```
+
+Result:
+
+```response
+Row 1:
+──────
+toFloat64OrNull('42.7'): 42.7
+toFloat64OrNull('abc'):  ᴺᵁᴸᴸ
+```
+
+**See also**
+
+- [`toFloat64`](#tofloat64).
+- [`toFloat64OrZero`](#tofloat64orzero).
+- [`toFloat64OrDefault`](#tofloat64ordefault).
+
+## toFloat64OrDefault
+
+Like [`toFloat64`](#tofloat64), this function converts an input value to a value of type [Float64](../data-types/float.md) but returns the default value in case of an error.
+If no `default` value is passed then `0` is returned in case of an error.
+
+**Syntax**
+
+```sql
+toFloat64OrDefault(expr[, default])
+```
+
+**Arguments**
+
+- `expr` — Expression returning a number or a string representation of a number. [Expression](../syntax.md/#syntax-expressions) / [String](../data-types/string.md).
+- `default` (optional) — The default value to return if parsing to type `Float64` is unsuccessful. [Float64](../data-types/float.md).
+
+Supported arguments:
+- Values of type (U)Int8/16/32/64/128/256.
+- String representations of (U)Int8/16/32/128/256.
+- Values of type Float32/64.
+- String representations of Float32/64.
+
+Arguments for which the default value is returned:
+- String representations of binary and hexadecimal values, e.g. `SELECT toFloat64OrDefault('0xc0fe', CAST('0', 'Float64'));`.
+
+**Returned value**
+
+- 64-bit Float value if successful, otherwise returns the default value if passed or `0` if not. [Float64](../data-types/float.md).
+
+**Example**
+
+Query:
+
+``` sql
+SELECT
+    toFloat64OrDefault('8', CAST('0', 'Float64')),
+    toFloat64OrDefault('abc', CAST('0', 'Float64'))
+FORMAT vertical;
+```
+
+Result:
+
+```response
+Row 1:
+──────
+toFloat64OrDefault('8', CAST('0', 'Float64')):   8
+toFloat64OrDefault('abc', CAST('0', 'Float64')): 0
+```
+
+**See also**
+
+- [`toFloat64`](#tofloat64).
+- [`toFloat64OrZero`](#tofloat64orzero).
+- [`toFloat64OrNull`](#tofloat64ornull).
 
 ## toDate
 

From e2eeb6f1802090c29021977abdbc46d20f22c9a1 Mon Sep 17 00:00:00 2001
From: robot-clickhouse <robot-clickhouse@users.noreply.github.com>
Date: Tue, 6 Aug 2024 12:54:27 +0000
Subject: [PATCH 1453/1488] Update version_date.tsv and changelogs after
 v24.6.3.95-stable

---
 docs/changelogs/v24.6.3.95-stable.md | 67 ++++++++++++++++++++++++++++
 utils/list-versions/version_date.tsv |  1 +
 2 files changed, 68 insertions(+)
 create mode 100644 docs/changelogs/v24.6.3.95-stable.md

diff --git a/docs/changelogs/v24.6.3.95-stable.md b/docs/changelogs/v24.6.3.95-stable.md
new file mode 100644
index 00000000000..b90b7346e40
--- /dev/null
+++ b/docs/changelogs/v24.6.3.95-stable.md
@@ -0,0 +1,67 @@
+---
+sidebar_position: 1
+sidebar_label: 2024
+---
+
+# 2024 Changelog
+
+### ClickHouse release v24.6.3.95-stable (8325c920d11) FIXME as compared to v24.6.2.17-stable (5710a8b5c0c)
+
+#### Improvement
+* Backported in [#66770](https://github.com/ClickHouse/ClickHouse/issues/66770): Make allow_experimental_analyzer be controlled by the initiator for distributed queries. This ensures compatibility and correctness during operations in mixed version clusters. [#65777](https://github.com/ClickHouse/ClickHouse/pull/65777) ([Nikita Mikhaylov](https://github.com/nikitamikhaylov)).
+
+#### Critical Bug Fix (crash, LOGICAL_ERROR, data loss, RBAC)
+* Backported in [#66885](https://github.com/ClickHouse/ClickHouse/issues/66885): Fix unexpeced size of low cardinality column in function calls. [#65298](https://github.com/ClickHouse/ClickHouse/pull/65298) ([Raúl Marín](https://github.com/Algunenano)).
+* Backported in [#66693](https://github.com/ClickHouse/ClickHouse/issues/66693): Fix the VALID UNTIL clause in the user definition resetting after a restart. Closes [#66405](https://github.com/ClickHouse/ClickHouse/issues/66405). [#66409](https://github.com/ClickHouse/ClickHouse/pull/66409) ([Nikolay Degterinsky](https://github.com/evillique)).
+* Backported in [#67816](https://github.com/ClickHouse/ClickHouse/issues/67816): Only relevant to the experimental Variant data type. Fix crash with Variant + AggregateFunction type. [#67122](https://github.com/ClickHouse/ClickHouse/pull/67122) ([Kruglov Pavel](https://github.com/Avogar)).
+* Backported in [#67503](https://github.com/ClickHouse/ClickHouse/issues/67503): Fix crash in DistributedAsyncInsert when connection is empty. [#67219](https://github.com/ClickHouse/ClickHouse/pull/67219) ([Pablo Marcos](https://github.com/pamarcos)).
+* Backported in [#67852](https://github.com/ClickHouse/ClickHouse/issues/67852): Fixes [#66026](https://github.com/ClickHouse/ClickHouse/issues/66026). Avoid unresolved table function arguments traversal in `ReplaceTableNodeToDummyVisitor`. [#67522](https://github.com/ClickHouse/ClickHouse/pull/67522) ([Dmitry Novik](https://github.com/novikd)).
+* Backported in [#67838](https://github.com/ClickHouse/ClickHouse/issues/67838): Fix potential stack overflow in `JSONMergePatch` function. Renamed this function from `jsonMergePatch` to `JSONMergePatch` because the previous name was wrong. The previous name is still kept for compatibility. Improved diagnostic of errors in the function. This closes [#67304](https://github.com/ClickHouse/ClickHouse/issues/67304). [#67756](https://github.com/ClickHouse/ClickHouse/pull/67756) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+
+#### Bug Fix (user-visible misbehavior in an official stable release)
+* Backported in [#66303](https://github.com/ClickHouse/ClickHouse/issues/66303): Better handling of join conditions involving `IS NULL` checks (for example `ON (a = b AND (a IS NOT NULL) AND (b IS NOT NULL) ) OR ( (a IS NULL) AND (b IS NULL) )` is rewritten to `ON a <=> b`), fix incorrect optimization when condition other then `IS NULL` are present. [#65835](https://github.com/ClickHouse/ClickHouse/pull/65835) ([vdimir](https://github.com/vdimir)).
+* Backported in [#66330](https://github.com/ClickHouse/ClickHouse/issues/66330): Add missing settings `input_format_csv_skip_first_lines/input_format_tsv_skip_first_lines/input_format_csv_try_infer_numbers_from_strings/input_format_csv_try_infer_strings_from_quoted_tuples` in schema inference cache because they can change the resulting schema. It prevents from incorrect result of schema inference with these settings changed. [#65980](https://github.com/ClickHouse/ClickHouse/pull/65980) ([Kruglov Pavel](https://github.com/Avogar)).
+* Backported in [#66157](https://github.com/ClickHouse/ClickHouse/issues/66157): Fixed buffer overflow bug in `unbin`/`unhex` implementation. [#66106](https://github.com/ClickHouse/ClickHouse/pull/66106) ([Nikita Taranov](https://github.com/nickitat)).
+* Backported in [#66210](https://github.com/ClickHouse/ClickHouse/issues/66210): Disable the `merge-filters` optimization introduced in [#64760](https://github.com/ClickHouse/ClickHouse/issues/64760). It may cause an exception if optimization merges two filter expressions and does not apply a short-circuit evaluation. [#66126](https://github.com/ClickHouse/ClickHouse/pull/66126) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#66456](https://github.com/ClickHouse/ClickHouse/issues/66456): Fixed a bug in ZooKeeper client: a session could get stuck in unusable state after receiving a hardware error from ZooKeeper. For example, this might happen due to "soft memory limit" in ClickHouse Keeper. [#66140](https://github.com/ClickHouse/ClickHouse/pull/66140) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Backported in [#66228](https://github.com/ClickHouse/ClickHouse/issues/66228): Fix issue in SumIfToCountIfVisitor and signed integers. [#66146](https://github.com/ClickHouse/ClickHouse/pull/66146) ([Raúl Marín](https://github.com/Algunenano)).
+* Backported in [#66183](https://github.com/ClickHouse/ClickHouse/issues/66183): Fix rare case with missing data in the result of distributed query, close [#61432](https://github.com/ClickHouse/ClickHouse/issues/61432). [#66174](https://github.com/ClickHouse/ClickHouse/pull/66174) ([vdimir](https://github.com/vdimir)).
+* Backported in [#66271](https://github.com/ClickHouse/ClickHouse/issues/66271): Don't throw `TIMEOUT_EXCEEDED` for `none_only_active` mode of `distributed_ddl_output_mode`. [#66218](https://github.com/ClickHouse/ClickHouse/pull/66218) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Backported in [#66682](https://github.com/ClickHouse/ClickHouse/issues/66682): Fix handling limit for `system.numbers_mt` when no index can be used. [#66231](https://github.com/ClickHouse/ClickHouse/pull/66231) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
+* Backported in [#66587](https://github.com/ClickHouse/ClickHouse/issues/66587): Fixed how the ClickHouse server detects the maximum number of usable CPU cores as specified by cgroups v2 if the server runs in a container such as Docker. In more detail, containers often run their process in the root cgroup which has an empty name. In that case, ClickHouse ignored the CPU limits set by cgroups v2. [#66237](https://github.com/ClickHouse/ClickHouse/pull/66237) ([filimonov](https://github.com/filimonov)).
+* Backported in [#66362](https://github.com/ClickHouse/ClickHouse/issues/66362): Fix the `Not-ready set` error when a subquery with `IN` is used in the constraint. [#66261](https://github.com/ClickHouse/ClickHouse/pull/66261) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#66613](https://github.com/ClickHouse/ClickHouse/issues/66613): Fix `Column identifier is already registered` error with `group_by_use_nulls=true` and new analyzer. [#66400](https://github.com/ClickHouse/ClickHouse/pull/66400) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#66577](https://github.com/ClickHouse/ClickHouse/issues/66577): Fix `Cannot find column` error for queries with constant expression in `GROUP BY` key and new analyzer enabled. [#66433](https://github.com/ClickHouse/ClickHouse/pull/66433) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#66721](https://github.com/ClickHouse/ClickHouse/issues/66721): Correctly track memory for `Allocator::realloc`. [#66548](https://github.com/ClickHouse/ClickHouse/pull/66548) ([Antonio Andelic](https://github.com/antonio2368)).
+* Backported in [#66670](https://github.com/ClickHouse/ClickHouse/issues/66670): Fix reading of uninitialized memory when hashing empty tuples. This closes [#66559](https://github.com/ClickHouse/ClickHouse/issues/66559). [#66562](https://github.com/ClickHouse/ClickHouse/pull/66562) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
+* Backported in [#66952](https://github.com/ClickHouse/ClickHouse/issues/66952): Fix an invalid result for queries with `WINDOW`. This could happen when `PARTITION` columns have sparse serialization and window functions are executed in parallel. [#66579](https://github.com/ClickHouse/ClickHouse/pull/66579) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#66956](https://github.com/ClickHouse/ClickHouse/issues/66956): Fix removing named collections in local storage. [#66599](https://github.com/ClickHouse/ClickHouse/pull/66599) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
+* Backported in [#66716](https://github.com/ClickHouse/ClickHouse/issues/66716): Fix removing named collections in local storage. [#66599](https://github.com/ClickHouse/ClickHouse/pull/66599) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
+* Backported in [#66759](https://github.com/ClickHouse/ClickHouse/issues/66759): Fix `Unknown identifier` and `Column is not under aggregate function` errors for queries with the expression `(column IS NULL).` The bug was triggered by [#65088](https://github.com/ClickHouse/ClickHouse/issues/65088), with the disabled analyzer only. [#66654](https://github.com/ClickHouse/ClickHouse/pull/66654) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#66751](https://github.com/ClickHouse/ClickHouse/issues/66751): Fix `Method getResultType is not supported for QUERY query node` error when scalar subquery was used as the first argument of IN (with new analyzer). [#66655](https://github.com/ClickHouse/ClickHouse/pull/66655) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#67635](https://github.com/ClickHouse/ClickHouse/issues/67635): Fix for occasional deadlock in Context::getDDLWorker. [#66843](https://github.com/ClickHouse/ClickHouse/pull/66843) ([Alexander Gololobov](https://github.com/davenger)).
+* Backported in [#67482](https://github.com/ClickHouse/ClickHouse/issues/67482): In rare cases ClickHouse could consider parts as broken because of some unexpected projections on disk. Now it's fixed. [#66898](https://github.com/ClickHouse/ClickHouse/pull/66898) ([alesapin](https://github.com/alesapin)).
+* Backported in [#67199](https://github.com/ClickHouse/ClickHouse/issues/67199): TRUNCATE DATABASE used to stop replication as if it was a DROP DATABASE query, it's fixed. [#67129](https://github.com/ClickHouse/ClickHouse/pull/67129) ([Alexander Tokmakov](https://github.com/tavplubix)).
+* Backported in [#67381](https://github.com/ClickHouse/ClickHouse/issues/67381): Fix error `Cannot convert column because it is non constant in source stream but must be constant in result.` for a query that reads from the `Merge` table over the `Distriburted` table with one shard. [#67146](https://github.com/ClickHouse/ClickHouse/pull/67146) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#67244](https://github.com/ClickHouse/ClickHouse/issues/67244): This closes [#67156](https://github.com/ClickHouse/ClickHouse/issues/67156). This closes [#66447](https://github.com/ClickHouse/ClickHouse/issues/66447). The bug was introduced in https://github.com/ClickHouse/ClickHouse/pull/62907. [#67178](https://github.com/ClickHouse/ClickHouse/pull/67178) ([Maksim Kita](https://github.com/kitaisreal)).
+* Backported in [#67578](https://github.com/ClickHouse/ClickHouse/issues/67578): Fix execution of nested short-circuit functions. [#67520](https://github.com/ClickHouse/ClickHouse/pull/67520) ([Kruglov Pavel](https://github.com/Avogar)).
+* Backported in [#67808](https://github.com/ClickHouse/ClickHouse/issues/67808): Fix reloading SQL UDFs with UNION. Previously, restarting the server could make UDF invalid. [#67665](https://github.com/ClickHouse/ClickHouse/pull/67665) ([Antonio Andelic](https://github.com/antonio2368)).
+
+#### NO CL ENTRY
+
+* NO CL ENTRY:  'Revert "Backport [#66599](https://github.com/ClickHouse/ClickHouse/issues/66599) to 24.6: Fix dropping named collection in local storage"'. [#66922](https://github.com/ClickHouse/ClickHouse/pull/66922) ([János Benjamin Antal](https://github.com/antaljanosbenjamin)).
+
+#### NOT FOR CHANGELOG / INSIGNIFICANT
+
+* Backported in [#66332](https://github.com/ClickHouse/ClickHouse/issues/66332): Do not raise a NOT_IMPLEMENTED error when getting s3 metrics with a multiple disk configuration. [#65403](https://github.com/ClickHouse/ClickHouse/pull/65403) ([Elena Torró](https://github.com/elenatorro)).
+* Backported in [#66142](https://github.com/ClickHouse/ClickHouse/issues/66142): Fix flaky test_storage_s3_queue tests. [#66009](https://github.com/ClickHouse/ClickHouse/pull/66009) ([Kseniia Sumarokova](https://github.com/kssenii)).
+* Backported in [#66389](https://github.com/ClickHouse/ClickHouse/issues/66389): Disable broken cases from 02911_join_on_nullsafe_optimization. [#66310](https://github.com/ClickHouse/ClickHouse/pull/66310) ([vdimir](https://github.com/vdimir)).
+* Backported in [#66428](https://github.com/ClickHouse/ClickHouse/issues/66428): Ignore subquery for IN in DDLLoadingDependencyVisitor. [#66395](https://github.com/ClickHouse/ClickHouse/pull/66395) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
+* Backported in [#66546](https://github.com/ClickHouse/ClickHouse/issues/66546): Add additional log masking in CI. [#66523](https://github.com/ClickHouse/ClickHouse/pull/66523) ([Raúl Marín](https://github.com/Algunenano)).
+* Backported in [#66861](https://github.com/ClickHouse/ClickHouse/issues/66861): Fix data race in S3::ClientCache. [#66644](https://github.com/ClickHouse/ClickHouse/pull/66644) ([Konstantin Morozov](https://github.com/k-morozov)).
+* Backported in [#66877](https://github.com/ClickHouse/ClickHouse/issues/66877): Support one more case in JOIN ON ... IS NULL. [#66725](https://github.com/ClickHouse/ClickHouse/pull/66725) ([vdimir](https://github.com/vdimir)).
+* Backported in [#67061](https://github.com/ClickHouse/ClickHouse/issues/67061): Increase asio pool size in case the server is tiny. [#66761](https://github.com/ClickHouse/ClickHouse/pull/66761) ([alesapin](https://github.com/alesapin)).
+* Backported in [#66940](https://github.com/ClickHouse/ClickHouse/issues/66940): Small fix in realloc memory tracking. [#66820](https://github.com/ClickHouse/ClickHouse/pull/66820) ([Antonio Andelic](https://github.com/antonio2368)).
+* Backported in [#67254](https://github.com/ClickHouse/ClickHouse/issues/67254): Followup [#66725](https://github.com/ClickHouse/ClickHouse/issues/66725). [#66869](https://github.com/ClickHouse/ClickHouse/pull/66869) ([vdimir](https://github.com/vdimir)).
+* Backported in [#67414](https://github.com/ClickHouse/ClickHouse/issues/67414): CI: Fix build results for release branches. [#67402](https://github.com/ClickHouse/ClickHouse/pull/67402) ([Max K.](https://github.com/maxknv)).
+
diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv
index 75c10fa67b8..e410f31ca5a 100644
--- a/utils/list-versions/version_date.tsv
+++ b/utils/list-versions/version_date.tsv
@@ -1,5 +1,6 @@
 v24.7.2.13-stable	2024-08-01
 v24.7.1.2915-stable	2024-07-30
+v24.6.3.95-stable	2024-08-06
 v24.6.2.17-stable	2024-07-05
 v24.6.1.4423-stable	2024-07-01
 v24.5.5.78-stable	2024-08-05

From dbfba5ebc4d011d13ab78f91700fae12b46ce58b Mon Sep 17 00:00:00 2001
From: avogar <avogar@clickhouse.com>
Date: Tue, 6 Aug 2024 12:56:52 +0000
Subject: [PATCH 1454/1488] Validate data types in ALTER ADD/MODIFY COLUMN

---
 src/Storages/AlterCommands.cpp                |  5 +++++
 ..._type_in_alter_add_modify_column.reference |  0
 ...lidate_type_in_alter_add_modify_column.sql | 19 +++++++++++++++++++
 3 files changed, 24 insertions(+)
 create mode 100644 tests/queries/0_stateless/03215_validate_type_in_alter_add_modify_column.reference
 create mode 100644 tests/queries/0_stateless/03215_validate_type_in_alter_add_modify_column.sql

diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp
index 7891042bb96..dfb388ffdb2 100644
--- a/src/Storages/AlterCommands.cpp
+++ b/src/Storages/AlterCommands.cpp
@@ -19,6 +19,7 @@
 #include <Interpreters/inplaceBlockConversions.h>
 #include <Interpreters/InterpreterSelectWithUnionQuery.h>
 #include <Interpreters/InterpreterSelectQueryAnalyzer.h>
+#include <Interpreters/parseColumnsListForTableFunction.h>
 #include <Storages/StorageView.h>
 #include <Parsers/ASTAlterQuery.h>
 #include <Parsers/ASTColumnDeclaration.h>
@@ -1316,6 +1317,8 @@ void AlterCommands::validate(const StoragePtr & table, ContextPtr context) const
                 throw Exception(ErrorCodes::BAD_ARGUMENTS,
                                 "Data type have to be specified for column {} to add", backQuote(column_name));
 
+            validateDataType(command.data_type, DataTypeValidationSettings(context->getSettingsRef()));
+
             /// FIXME: Adding a new column of type Object(JSON) is broken.
             /// Looks like there is something around default expression for this column (method `getDefault` is not implemented for the data type Object).
             /// But after ALTER TABLE ADD COLUMN we need to fill existing rows with something (exactly the default value).
@@ -1395,6 +1398,8 @@ void AlterCommands::validate(const StoragePtr & table, ContextPtr context) const
             /// So we don't allow to do it for now.
             if (command.data_type)
             {
+                validateDataType(command.data_type, DataTypeValidationSettings(context->getSettingsRef()));
+
                 const GetColumnsOptions options(GetColumnsOptions::All);
                 const auto old_data_type = all_columns.getColumn(options, column_name).type;
 
diff --git a/tests/queries/0_stateless/03215_validate_type_in_alter_add_modify_column.reference b/tests/queries/0_stateless/03215_validate_type_in_alter_add_modify_column.reference
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/tests/queries/0_stateless/03215_validate_type_in_alter_add_modify_column.sql b/tests/queries/0_stateless/03215_validate_type_in_alter_add_modify_column.sql
new file mode 100644
index 00000000000..267bc7111f4
--- /dev/null
+++ b/tests/queries/0_stateless/03215_validate_type_in_alter_add_modify_column.sql
@@ -0,0 +1,19 @@
+set allow_experimental_variant_type = 0;
+set allow_experimental_dynamic_type = 0;
+set allow_suspicious_low_cardinality_types = 0;
+set allow_suspicious_fixed_string_types = 0;
+
+drop table if exists test;
+create table test (id UInt64) engine=MergeTree order by id;
+alter table test add column bad Variant(UInt32, String); -- {serverError ILLEGAL_COLUMN}
+alter table test add column bad Dynamic; -- {serverError ILLEGAL_COLUMN}
+alter table test add column bad LowCardinality(UInt8); -- {serverError SUSPICIOUS_TYPE_FOR_LOW_CARDINALITY}
+alter table test add column bad FixedString(10000); -- {serverError ILLEGAL_COLUMN}
+
+alter table test modify column id Variant(UInt32, String); -- {serverError ILLEGAL_COLUMN}
+alter table test modify column id Dynamic; -- {serverError ILLEGAL_COLUMN}
+alter table test modify column id LowCardinality(UInt8); -- {serverError SUSPICIOUS_TYPE_FOR_LOW_CARDINALITY}
+alter table test modify column id FixedString(10000); -- {serverError ILLEGAL_COLUMN}
+
+drop table test;
+

From e0e32b542560aeefa6863195d0207859ec5f338e Mon Sep 17 00:00:00 2001
From: avogar <avogar@clickhouse.com>
Date: Tue, 6 Aug 2024 13:26:12 +0000
Subject: [PATCH 1455/1488] Fix tests

---
 .../queries/0_stateless/02910_object-json-crash-add-column.sql  | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/queries/0_stateless/02910_object-json-crash-add-column.sql b/tests/queries/0_stateless/02910_object-json-crash-add-column.sql
index b2d64be1676..bda5e958453 100644
--- a/tests/queries/0_stateless/02910_object-json-crash-add-column.sql
+++ b/tests/queries/0_stateless/02910_object-json-crash-add-column.sql
@@ -1,3 +1,5 @@
+SET allow_experimental_object_type=1;
+
 DROP TABLE IF EXISTS test02910;
 
 CREATE TABLE test02910

From f37fcb776800cdb6bff4d89e5814c39bbbc46033 Mon Sep 17 00:00:00 2001
From: Nikita Taranov <nikita.taranov@clickhouse.com>
Date: Tue, 6 Aug 2024 14:32:20 +0100
Subject: [PATCH 1456/1488] impl

---
 src/Compression/fuzzers/CMakeLists.txt | 12 ++++++------
 src/Storages/fuzzers/CMakeLists.txt    |  2 +-
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/Compression/fuzzers/CMakeLists.txt b/src/Compression/fuzzers/CMakeLists.txt
index a693faecc14..311f1eb3d35 100644
--- a/src/Compression/fuzzers/CMakeLists.txt
+++ b/src/Compression/fuzzers/CMakeLists.txt
@@ -5,19 +5,19 @@
 # If you want really small size of the resulted binary, just link with fuzz_compression and clickhouse_common_io
 
 clickhouse_add_executable (compressed_buffer_fuzzer compressed_buffer_fuzzer.cpp)
-target_link_libraries (compressed_buffer_fuzzer PRIVATE dbms)
+target_link_libraries (compressed_buffer_fuzzer PRIVATE dbms clickhouse_functions)
 
 clickhouse_add_executable (lz4_decompress_fuzzer lz4_decompress_fuzzer.cpp)
-target_link_libraries (lz4_decompress_fuzzer PUBLIC dbms ch_contrib::lz4)
+target_link_libraries (lz4_decompress_fuzzer PUBLIC dbms ch_contrib::lz4 clickhouse_functions)
 
 clickhouse_add_executable (delta_decompress_fuzzer delta_decompress_fuzzer.cpp)
-target_link_libraries (delta_decompress_fuzzer PRIVATE dbms)
+target_link_libraries (delta_decompress_fuzzer PRIVATE dbms clickhouse_functions)
 
 clickhouse_add_executable (double_delta_decompress_fuzzer double_delta_decompress_fuzzer.cpp)
-target_link_libraries (double_delta_decompress_fuzzer PRIVATE dbms)
+target_link_libraries (double_delta_decompress_fuzzer PRIVATE dbms clickhouse_functions)
 
 clickhouse_add_executable (encrypted_decompress_fuzzer encrypted_decompress_fuzzer.cpp)
-target_link_libraries (encrypted_decompress_fuzzer PRIVATE dbms)
+target_link_libraries (encrypted_decompress_fuzzer PRIVATE dbms clickhouse_functions)
 
 clickhouse_add_executable (gcd_decompress_fuzzer gcd_decompress_fuzzer.cpp)
-target_link_libraries (gcd_decompress_fuzzer PRIVATE dbms)
+target_link_libraries (gcd_decompress_fuzzer PRIVATE dbms clickhouse_functions)
diff --git a/src/Storages/fuzzers/CMakeLists.txt b/src/Storages/fuzzers/CMakeLists.txt
index e36fccec8df..ec56b853666 100644
--- a/src/Storages/fuzzers/CMakeLists.txt
+++ b/src/Storages/fuzzers/CMakeLists.txt
@@ -1,7 +1,7 @@
 
 clickhouse_add_executable (mergetree_checksum_fuzzer mergetree_checksum_fuzzer.cpp)
 # Look at comment around fuzz_compression target declaration
-target_link_libraries (mergetree_checksum_fuzzer PRIVATE dbms)
+target_link_libraries (mergetree_checksum_fuzzer PRIVATE dbms clickhouse_functions)
 
 clickhouse_add_executable (columns_description_fuzzer columns_description_fuzzer.cpp)
 target_link_libraries (columns_description_fuzzer PRIVATE dbms clickhouse_functions)

From dbcc5cf1333c6a3d8f5f8f1b7dc67d038055be2e Mon Sep 17 00:00:00 2001
From: Max Kainov <maxkaynov@gmail.com>
Date: Tue, 6 Aug 2024 15:39:08 +0200
Subject: [PATCH 1457/1488] CI: Changelog: Critical Bug Fix to Bug Fix

---
 tests/ci/changelog.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tests/ci/changelog.py b/tests/ci/changelog.py
index 929f0f3523a..fe47fe53a9e 100755
--- a/tests/ci/changelog.py
+++ b/tests/ci/changelog.py
@@ -38,7 +38,7 @@ categories_preferred_order = (
     "Experimental Feature",
     "Performance Improvement",
     "Improvement",
-    "Critical Bug Fix (crash, LOGICAL_ERROR, data loss, RBAC)",
+    #"Critical Bug Fix (crash, LOGICAL_ERROR, data loss, RBAC)",
     "Bug Fix (user-visible misbehavior in an official stable release)",
     "Build/Testing/Packaging Improvement",
     "Other",
@@ -294,7 +294,8 @@ def generate_description(item: PullRequest, repo: Repository) -> Optional[Descri
             r"(?i)bug\Wfix",
             category,
         )
-        and "Critical Bug Fix" not in category
+        # Map "Critical Bug Fix" to "Bug fix" category for changelog
+        #and "Critical Bug Fix" not in category
     ):
         category = "Bug Fix (user-visible misbehavior in an official stable release)"
 

From b99c6c1153224b8d17dc663939b7c621dfe77ceb Mon Sep 17 00:00:00 2001
From: Anton Popov <anton@clickhouse.com>
Date: Tue, 6 Aug 2024 14:02:24 +0000
Subject: [PATCH 1458/1488] fix reading of size column from missed Nested in
 compact parts

---
 src/Storages/MergeTree/MergeTreeReaderCompact.cpp    | 12 +++++++++++-
 src/Storages/MergeTree/MergeTreeReaderCompact.h      |  3 ++-
 .../MergeTree/MergeTreeReaderCompactSingleBuffer.cpp |  7 ++++++-
 3 files changed, 19 insertions(+), 3 deletions(-)

diff --git a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp
index ff0311dc1ca..d49ad61feca 100644
--- a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp
+++ b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp
@@ -147,7 +147,8 @@ void MergeTreeReaderCompact::readData(
     const NameAndTypePair & name_and_type,
     ColumnPtr & column,
     size_t rows_to_read,
-    const InputStreamGetter & getter)
+    const InputStreamGetter & getter,
+    ISerialization::SubstreamsCache & cache)
 {
     try
     {
@@ -158,6 +159,13 @@ void MergeTreeReaderCompact::readData(
         deserialize_settings.getter = getter;
         deserialize_settings.avg_value_size_hint = avg_value_size_hints[name];
 
+        auto it = cache.find(name);
+        if (it != cache.end() && it->second != nullptr)
+        {
+            column = it->second;
+            return;
+        }
+
         if (name_and_type.isSubcolumn())
         {
             const auto & type_in_storage = name_and_type.getTypeInStorage();
@@ -181,6 +189,8 @@ void MergeTreeReaderCompact::readData(
             serialization->deserializeBinaryBulkWithMultipleStreams(column, rows_to_read, deserialize_settings, deserialize_binary_bulk_state_map[name], nullptr);
         }
 
+        cache.emplace(name, column);
+
         size_t read_rows_in_column = column->size() - column_size_before_reading;
         if (read_rows_in_column != rows_to_read)
             throw Exception(ErrorCodes::CANNOT_READ_ALL_DATA,
diff --git a/src/Storages/MergeTree/MergeTreeReaderCompact.h b/src/Storages/MergeTree/MergeTreeReaderCompact.h
index 22eabd47930..1c6bd1474e3 100644
--- a/src/Storages/MergeTree/MergeTreeReaderCompact.h
+++ b/src/Storages/MergeTree/MergeTreeReaderCompact.h
@@ -44,7 +44,8 @@ protected:
         const NameAndTypePair & name_and_type,
         ColumnPtr & column,
         size_t rows_to_read,
-        const InputStreamGetter & getter);
+        const InputStreamGetter & getter,
+        ISerialization::SubstreamsCache & cache);
 
     void readPrefix(
         const NameAndTypePair & name_and_type,
diff --git a/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp b/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp
index 2b2cf493bb5..004ba4db028 100644
--- a/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp
+++ b/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp
@@ -22,6 +22,10 @@ try
     checkNumberOfColumns(num_columns);
     createColumnsForReading(res_columns);
 
+    /// Use cache to avoid reading the column with the same name twice.
+    /// It may happen if there are empty array Nested in the part.
+    std::unordered_map<String, ISerialization::SubstreamsCache> caches;
+
     while (read_rows < max_rows_to_read)
     {
         size_t rows_to_read = data_part_info_for_read->getIndexGranularity().getMarkRows(from_mark);
@@ -32,6 +36,7 @@ try
                 continue;
 
             auto & column = res_columns[pos];
+            auto & cache = caches[columns_to_read[pos].name];
 
             stream->adjustRightMark(current_task_last_mark); /// Must go before seek.
             stream->seekToMarkAndColumn(from_mark, *column_positions[pos]);
@@ -52,7 +57,7 @@ try
             };
 
             readPrefix(columns_to_read[pos], buffer_getter, buffer_getter_for_prefix, columns_for_offsets[pos]);
-            readData(columns_to_read[pos], column, rows_to_read, buffer_getter);
+            readData(columns_to_read[pos], column, rows_to_read, buffer_getter, cache);
         }
 
         ++from_mark;

From 5c4f2c198573ba03b793e4610f4e9854a61a5543 Mon Sep 17 00:00:00 2001
From: vdimir <vdimir@clickhouse.com>
Date: Tue, 6 Aug 2024 14:33:28 +0000
Subject: [PATCH 1459/1488] Fix DateTime64 parsing after constant folding

---
 src/Analyzer/ConstantNode.cpp                 | 22 ++++++++++++++++++-
 ...03217_datetime64_constant_to_ast.reference |  2 ++
 .../03217_datetime64_constant_to_ast.sql      |  6 +++++
 3 files changed, 29 insertions(+), 1 deletion(-)
 create mode 100644 tests/queries/0_stateless/03217_datetime64_constant_to_ast.reference
 create mode 100644 tests/queries/0_stateless/03217_datetime64_constant_to_ast.sql

diff --git a/src/Analyzer/ConstantNode.cpp b/src/Analyzer/ConstantNode.cpp
index 46c1f7fb1ed..c65090f5b55 100644
--- a/src/Analyzer/ConstantNode.cpp
+++ b/src/Analyzer/ConstantNode.cpp
@@ -5,6 +5,7 @@
 #include <Common/assert_cast.h>
 #include <Common/FieldVisitorToString.h>
 #include <Common/SipHash.h>
+#include <DataTypes/DataTypeDateTime64.h>
 
 #include <IO/WriteBuffer.h>
 #include <IO/WriteHelpers.h>
@@ -162,6 +163,7 @@ QueryTreeNodePtr ConstantNode::cloneImpl() const
 ASTPtr ConstantNode::toASTImpl(const ConvertToASTOptions & options) const
 {
     const auto & constant_value_literal = constant_value->getValue();
+    const auto & constant_value_type = constant_value->getType();
     auto constant_value_ast = std::make_shared<ASTLiteral>(constant_value_literal);
 
     if (!options.add_cast_for_constants)
@@ -169,7 +171,25 @@ ASTPtr ConstantNode::toASTImpl(const ConvertToASTOptions & options) const
 
     if (requiresCastCall())
     {
-        auto constant_type_name_ast = std::make_shared<ASTLiteral>(constant_value->getType()->getName());
+        /** Value for DateTime64 is Decimal64, which is serialized as a string literal.
+          * If we serialize it as is, DateTime64 would be parsed from that string literal, which can be incorrect.
+          * For example, DateTime64 cannot be parsed from the short value, like '1', while it's a valid Decimal64 value.
+          * It could also lead to ambiguous parsing because we don't know if the string literal represents a date or a Decimal64 literal.
+          * For this reason, we use a string literal representing a date instead of a Decimal64 literal.
+          */
+        if (WhichDataType(constant_value_type->getTypeId()).isDateTime64())
+        {
+            const auto * date_time_type = typeid_cast<const DataTypeDateTime64 *>(constant_value_type.get());
+            DecimalField<Decimal64> decimal_value;
+            if (constant_value_literal.tryGet<DecimalField<Decimal64>>(decimal_value))
+            {
+                WriteBufferFromOwnString ostr;
+                writeDateTimeText(decimal_value.getValue(), date_time_type->getScale(), ostr, date_time_type->getTimeZone());
+                constant_value_ast = std::make_shared<ASTLiteral>(ostr.str());
+            }
+        }
+
+        auto constant_type_name_ast = std::make_shared<ASTLiteral>(constant_value_type->getName());
         return makeASTFunction("_CAST", std::move(constant_value_ast), std::move(constant_type_name_ast));
     }
 
diff --git a/tests/queries/0_stateless/03217_datetime64_constant_to_ast.reference b/tests/queries/0_stateless/03217_datetime64_constant_to_ast.reference
new file mode 100644
index 00000000000..c20baa0d261
--- /dev/null
+++ b/tests/queries/0_stateless/03217_datetime64_constant_to_ast.reference
@@ -0,0 +1,2 @@
+1970-01-01 00:00:01.000
+1970-01-01 00:00:01.000
diff --git a/tests/queries/0_stateless/03217_datetime64_constant_to_ast.sql b/tests/queries/0_stateless/03217_datetime64_constant_to_ast.sql
new file mode 100644
index 00000000000..63334a511c7
--- /dev/null
+++ b/tests/queries/0_stateless/03217_datetime64_constant_to_ast.sql
@@ -0,0 +1,6 @@
+
+SET session_timezone = 'UTC';
+
+SELECT toDateTime64('1970-01-01 00:00:01', 3)
+FROM remote('127.0.0.{1,2}', system, one)
+;

From f2591bd1a6401337de0e3bf8d5c1fa93fd1b9394 Mon Sep 17 00:00:00 2001
From: Max Kainov <maxkaynov@gmail.com>
Date: Tue, 6 Aug 2024 16:55:04 +0200
Subject: [PATCH 1460/1488] CI: CiBuddy with channel dispatcher

---
 tests/ci/ci_buddy.py       | 102 ++++++++++++++++++++++++++++---------
 tests/ci/create_release.py |   4 +-
 2 files changed, 81 insertions(+), 25 deletions(-)

diff --git a/tests/ci/ci_buddy.py b/tests/ci/ci_buddy.py
index f0e73e925fe..07f318207a4 100644
--- a/tests/ci/ci_buddy.py
+++ b/tests/ci/ci_buddy.py
@@ -1,7 +1,7 @@
 import argparse
 import json
 import os
-from typing import Union, Dict
+from typing import Union, Dict, List
 
 import boto3
 import requests
@@ -9,20 +9,44 @@ from botocore.exceptions import ClientError
 
 from pr_info import PRInfo
 from ci_config import CI
+from ci_utils import WithIter
+
+
+class Channels(metaclass=WithIter):
+    # Channel names must match json keys in ParameterStore
+    ALERTS = "alerts-channel"
+    INFO = "info-channel"
+    DRY_RUN = "dry-ryn-channel"
+    DEFAULT = "default"
 
 
 class CIBuddy:
+    Channels = Channels
     _HEADERS = {"Content-Type": "application/json"}
 
     def __init__(self, dry_run=False):
         self.repo = os.getenv("GITHUB_REPOSITORY", "")
         self.dry_run = dry_run
         res = self._get_webhooks()
-        self.test_channel = ""
-        self.dev_ci_channel = ""
+        self.channels = {}
         if res:
-            self.test_channel = json.loads(res)["test_channel"]
-            self.dev_ci_channel = json.loads(res)["ci_channel"]
+            channels = json.loads(res)
+            for channel in Channels:
+                if channel in channels:
+                    self.channels[channel] = channels[channel]
+
+            for channel in Channels:
+                if channel not in self.channels:
+                    if Channels.DEFAULT in self.channels:
+                        print(
+                            f"ERROR: missing config for channel [{channel}] - will use default channel instead"
+                        )
+                        self.channels[channel] = self.channels[Channels.DEFAULT]
+                    else:
+                        print(
+                            f"ERROR: missing config for channel [{channel}] - will disable notification"
+                        )
+                        self.channels[channel] = ""
         self.job_name = os.getenv("CHECK_NAME", "unknown")
         pr_info = PRInfo()
         self.pr_number = pr_info.number
@@ -63,22 +87,33 @@ class CIBuddy:
 
         return json_string
 
-    def post(self, message, dry_run=None):
-        if dry_run is None:
-            dry_run = self.dry_run
-        print(f"Posting slack message, dry_run [{dry_run}]")
-        if dry_run:
-            url = self.test_channel
+    def post(self, message: str, channels: List[str]) -> None:
+        print(f"Posting slack message, dry_run [{self.dry_run}]")
+        if self.dry_run:
+            urls = [self.channels[Channels.DRY_RUN]]
         else:
-            url = self.dev_ci_channel
+            urls = []
+            for channel in channels:
+                url = self.channels[channel]
+                if url:
+                    urls.append(url)
+                else:
+                    print(f"WARNING: no channel config for [{channel}] - skip")
         data = {"text": message}
         try:
-            requests.post(url, headers=self._HEADERS, data=json.dumps(data), timeout=10)
+            for url in urls:
+                requests.post(
+                    url, headers=self._HEADERS, data=json.dumps(data), timeout=10
+                )
         except Exception as e:
             print(f"ERROR: Failed to post message, ex {e}")
 
     def _post_formatted(
-        self, title: str, body: Union[Dict, str], with_wf_link: bool
+        self,
+        title: str,
+        body: Union[Dict, str],
+        with_wf_link: bool,
+        channels: Union[List[str], str],
     ) -> None:
         message = title
         if isinstance(body, dict):
@@ -96,31 +131,49 @@ class CIBuddy:
         run_id = os.getenv("GITHUB_RUN_ID", "")
         if with_wf_link and run_id:
             message += f"      *workflow*: <https://github.com/{self.repo}/actions/runs/{run_id}|{run_id}>\n"
-        self.post(message)
+        self.post(
+            message, channels=[channels] if isinstance(channels, str) else channels
+        )
 
     def post_info(
-        self, title: str, body: Union[Dict, str], with_wf_link: bool = True
+        self,
+        title: str,
+        body: Union[Dict, str],
+        with_wf_link: bool = True,
+        channels: Union[List[str], str] = Channels.INFO,
     ) -> None:
         title_extended = f":white_circle:    *{title}*\n\n"
-        self._post_formatted(title_extended, body, with_wf_link)
+        self._post_formatted(title_extended, body, with_wf_link, channels=channels)
 
     def post_done(
-        self, title: str, body: Union[Dict, str], with_wf_link: bool = True
+        self,
+        title: str,
+        body: Union[Dict, str],
+        with_wf_link: bool = True,
+        channels: Union[List[str], str] = Channels.INFO,
     ) -> None:
         title_extended = f":white_check_mark:    *{title}*\n\n"
-        self._post_formatted(title_extended, body, with_wf_link)
+        self._post_formatted(title_extended, body, with_wf_link, channels=channels)
 
     def post_warning(
-        self, title: str, body: Union[Dict, str], with_wf_link: bool = True
+        self,
+        title: str,
+        body: Union[Dict, str],
+        with_wf_link: bool = True,
+        channels: Union[List[str], str] = Channels.ALERTS,
     ) -> None:
         title_extended = f":warning:    *{title}*\n\n"
-        self._post_formatted(title_extended, body, with_wf_link)
+        self._post_formatted(title_extended, body, with_wf_link, channels=channels)
 
     def post_critical(
-        self, title: str, body: Union[Dict, str], with_wf_link: bool = True
+        self,
+        title: str,
+        body: Union[Dict, str],
+        with_wf_link: bool = True,
+        channels: Union[List[str], str] = Channels.ALERTS,
     ) -> None:
         title_extended = f":black_circle:    *{title}*\n\n"
-        self._post_formatted(title_extended, body, with_wf_link)
+        self._post_formatted(title_extended, body, with_wf_link, channels=channels)
 
     def post_job_error(
         self,
@@ -129,6 +182,7 @@ class CIBuddy:
         with_instance_info: bool = True,
         with_wf_link: bool = True,
         critical: bool = False,
+        channel: Union[List[str], str] = Channels.ALERTS,
     ) -> None:
         instance_id, instance_type = "unknown", "unknown"
         if with_instance_info:
@@ -159,7 +213,7 @@ class CIBuddy:
         run_id = os.getenv("GITHUB_RUN_ID", "")
         if with_wf_link and run_id:
             message += f"      *workflow*: <https://github.com/{self.repo}/actions/runs/{run_id}|{run_id}>\n"
-        self.post(message)
+        self.post(message, channels=[channel] if isinstance(channel, str) else channel)
 
 
 def parse_args():
diff --git a/tests/ci/create_release.py b/tests/ci/create_release.py
index b4e08f29dbe..27eba273ce0 100755
--- a/tests/ci/create_release.py
+++ b/tests/ci/create_release.py
@@ -821,7 +821,9 @@ if __name__ == "__main__":
         else:
             title = "Failed: " + title
             CIBuddy(dry_run=args.dry_run).post_critical(
-                title, dataclasses.asdict(release_info)
+                title,
+                dataclasses.asdict(release_info),
+                channels=[CIBuddy.Channels.ALERTS, CIBuddy.Channels.INFO],
             )
 
     if args.set_progress_started:

From e1e298f14e34a519d59844417e31b7a532b0a157 Mon Sep 17 00:00:00 2001
From: robot-clickhouse <robot-clickhouse@users.noreply.github.com>
Date: Tue, 6 Aug 2024 14:56:54 +0000
Subject: [PATCH 1461/1488] Automatic style fix

---
 tests/ci/changelog.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/ci/changelog.py b/tests/ci/changelog.py
index fe47fe53a9e..39e426945d3 100755
--- a/tests/ci/changelog.py
+++ b/tests/ci/changelog.py
@@ -38,7 +38,7 @@ categories_preferred_order = (
     "Experimental Feature",
     "Performance Improvement",
     "Improvement",
-    #"Critical Bug Fix (crash, LOGICAL_ERROR, data loss, RBAC)",
+    # "Critical Bug Fix (crash, LOGICAL_ERROR, data loss, RBAC)",
     "Bug Fix (user-visible misbehavior in an official stable release)",
     "Build/Testing/Packaging Improvement",
     "Other",
@@ -295,7 +295,7 @@ def generate_description(item: PullRequest, repo: Repository) -> Optional[Descri
             category,
         )
         # Map "Critical Bug Fix" to "Bug fix" category for changelog
-        #and "Critical Bug Fix" not in category
+        # and "Critical Bug Fix" not in category
     ):
         category = "Bug Fix (user-visible misbehavior in an official stable release)"
 

From bdfaffa9d74b0405ea619330215e6c71f0b6976a Mon Sep 17 00:00:00 2001
From: Azat Khuzhin <a3at.mail@gmail.com>
Date: Tue, 6 Aug 2024 16:59:23 +0200
Subject: [PATCH 1462/1488] tests: make test_distributed_inter_server_secret
 idempotent

Signed-off-by: Azat Khuzhin <a.khuzhin@semrush.com>
---
 .../test.py                                   | 22 +++++++++++--------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/tests/integration/test_distributed_inter_server_secret/test.py b/tests/integration/test_distributed_inter_server_secret/test.py
index 36d7e044f1c..d74cf97e5c6 100644
--- a/tests/integration/test_distributed_inter_server_secret/test.py
+++ b/tests/integration/test_distributed_inter_server_secret/test.py
@@ -46,6 +46,10 @@ users = pytest.mark.parametrize(
 )
 
 
+def generate_query_id():
+    return str(uuid.uuid4())
+
+
 def bootstrap():
     for n in list(cluster.instances.values()):
         n.query("DROP TABLE IF EXISTS data")
@@ -268,13 +272,13 @@ def test_secure_insert_buffer_async():
     n1.query("SYSTEM RELOAD CONFIG")
     # ensure that SELECT creates new connection (we need separate table for
     # this, so that separate distributed pool will be used)
-    query_id = uuid.uuid4().hex
+    query_id = generate_query_id()
     n1.query("SELECT * FROM dist_secure_from_buffer", user="ro", query_id=query_id)
     assert n1.contains_in_log(
         "{" + query_id + "} <Trace> Connection (n2:9000): Connecting."
     )
 
-    query_id = uuid.uuid4().hex
+    query_id = generate_query_id()
     n1.query(
         "INSERT INTO dist_secure_buffer SELECT * FROM numbers(2)", query_id=query_id
     )
@@ -331,7 +335,7 @@ def test_secure_disagree_insert():
 
 @users
 def test_user_insecure_cluster(user, password):
-    id_ = "query-dist_insecure-" + user
+    id_ = "query-dist_insecure-" + user + "-" + generate_query_id()
     n1.query(f"SELECT *, '{id_}' FROM dist_insecure", user=user, password=password)
     assert get_query_user_info(n1, id_)[0] == [
         user,
@@ -342,7 +346,7 @@ def test_user_insecure_cluster(user, password):
 
 @users
 def test_user_secure_cluster(user, password):
-    id_ = "query-dist_secure-" + user
+    id_ = "query-dist_secure-" + user + "-" + generate_query_id()
     n1.query(f"SELECT *, '{id_}' FROM dist_secure", user=user, password=password)
     assert get_query_user_info(n1, id_)[0] == [user, user]
     assert get_query_user_info(n2, id_)[0] == [user, user]
@@ -350,7 +354,7 @@ def test_user_secure_cluster(user, password):
 
 @users
 def test_per_user_inline_settings_insecure_cluster(user, password):
-    id_ = "query-ddl-settings-dist_insecure-" + user
+    id_ = "query-ddl-settings-dist_insecure-" + user + "-" + generate_query_id()
     n1.query(
         f"""
         SELECT *, '{id_}' FROM dist_insecure
@@ -367,7 +371,7 @@ def test_per_user_inline_settings_insecure_cluster(user, password):
 
 @users
 def test_per_user_inline_settings_secure_cluster(user, password):
-    id_ = "query-ddl-settings-dist_secure-" + user
+    id_ = "query-ddl-settings-dist_secure-" + user + "-" + generate_query_id()
     n1.query(
         f"""
         SELECT *, '{id_}' FROM dist_secure
@@ -386,7 +390,7 @@ def test_per_user_inline_settings_secure_cluster(user, password):
 
 @users
 def test_per_user_protocol_settings_insecure_cluster(user, password):
-    id_ = "query-protocol-settings-dist_insecure-" + user
+    id_ = "query-protocol-settings-dist_insecure-" + user + "-" + generate_query_id()
     n1.query(
         f"SELECT *, '{id_}' FROM dist_insecure",
         user=user,
@@ -402,7 +406,7 @@ def test_per_user_protocol_settings_insecure_cluster(user, password):
 
 @users
 def test_per_user_protocol_settings_secure_cluster(user, password):
-    id_ = "query-protocol-settings-dist_secure-" + user
+    id_ = "query-protocol-settings-dist_secure-" + user + "-" + generate_query_id()
     n1.query(
         f"SELECT *, '{id_}' FROM dist_secure",
         user=user,
@@ -436,7 +440,7 @@ def test_secure_cluster_distributed_over_distributed_different_users_remote():
 
 
 def test_secure_cluster_distributed_over_distributed_different_users_cluster():
-    id_ = "cluster-user"
+    id_ = "cluster-user" + "-" + generate_query_id()
     n1.query(
         f"SELECT *, '{id_}' FROM cluster(secure, currentDatabase(), dist_secure)",
         user="nopass",

From 47953da08dbc791d521b4e722c08af08b9072b89 Mon Sep 17 00:00:00 2001
From: Anton Popov <anton@clickhouse.com>
Date: Tue, 6 Aug 2024 15:08:40 +0000
Subject: [PATCH 1463/1488] fix reading of size column from missed Nested in
 compact parts

---
 src/Storages/MergeTree/MergeTreeReaderCompact.cpp        | 2 +-
 .../MergeTree/MergeTreeReaderCompactSingleBuffer.cpp     | 9 ++++-----
 2 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp
index d49ad61feca..69dc2e4b2bb 100644
--- a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp
+++ b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp
@@ -189,7 +189,7 @@ void MergeTreeReaderCompact::readData(
             serialization->deserializeBinaryBulkWithMultipleStreams(column, rows_to_read, deserialize_settings, deserialize_binary_bulk_state_map[name], nullptr);
         }
 
-        cache.emplace(name, column);
+        cache[name] = column;
 
         size_t read_rows_in_column = column->size() - column_size_before_reading;
         if (read_rows_in_column != rows_to_read)
diff --git a/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp b/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp
index 004ba4db028..649bcce1188 100644
--- a/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp
+++ b/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp
@@ -22,21 +22,20 @@ try
     checkNumberOfColumns(num_columns);
     createColumnsForReading(res_columns);
 
-    /// Use cache to avoid reading the column with the same name twice.
-    /// It may happen if there are empty array Nested in the part.
-    std::unordered_map<String, ISerialization::SubstreamsCache> caches;
-
     while (read_rows < max_rows_to_read)
     {
         size_t rows_to_read = data_part_info_for_read->getIndexGranularity().getMarkRows(from_mark);
 
+        /// Use cache to avoid reading the column with the same name twice.
+        /// It may happen if there are empty array Nested in the part.
+        ISerialization::SubstreamsCache cache;
+
         for (size_t pos = 0; pos < num_columns; ++pos)
         {
             if (!res_columns[pos])
                 continue;
 
             auto & column = res_columns[pos];
-            auto & cache = caches[columns_to_read[pos].name];
 
             stream->adjustRightMark(current_task_last_mark); /// Must go before seek.
             stream->seekToMarkAndColumn(from_mark, *column_positions[pos]);

From 0bb8d1a8ede4906a1cdc75af2dbf1e190ff355fc Mon Sep 17 00:00:00 2001
From: Blargian <shaunstruwig.ss@gmail.com>
Date: Tue, 6 Aug 2024 17:10:34 +0200
Subject: [PATCH 1464/1488] optimize formatting by 3%

---
 .../functions/type-conversion-functions.md    | 40 +++++++++----------
 1 file changed, 20 insertions(+), 20 deletions(-)

diff --git a/docs/en/sql-reference/functions/type-conversion-functions.md b/docs/en/sql-reference/functions/type-conversion-functions.md
index 77dd1628fe4..a2b6e496319 100644
--- a/docs/en/sql-reference/functions/type-conversion-functions.md
+++ b/docs/en/sql-reference/functions/type-conversion-functions.md
@@ -3063,7 +3063,7 @@ Supported arguments:
 - Values of type (U)Int8/16/32/64/128/256.
 - String representations of (U)Int8/16/32/128/256.
 - Values of type Float32/64, including `NaN` and `Inf`.
-- String representations of Float32/64, including `NaN` and `Inf`.
+- String representations of Float32/64, including `NaN` and `Inf` (case-insensitive).
 
 Unsupported arguments:
 - String representations of binary and hexadecimal values, e.g. `SELECT toFloat32('0xc0fe');`.
@@ -3081,7 +3081,7 @@ SELECT
     toFloat32(42.7),
     toFloat32('42.7'),
     toFloat32('NaN')
-FORMAT vertical;
+FORMAT Vertical;
 ```
 
 Result:
@@ -3128,11 +3128,11 @@ Unsupported arguments (return `0`):
 
 Query:
 
-``` sql
+```sql
 SELECT
     toFloat32OrZero('42.7'),
     toFloat32OrZero('abc')
-FORMAT vertical;
+FORMAT Vertical;
 ```
 
 Result:
@@ -3178,11 +3178,11 @@ Unsupported arguments (return `\N`):
 
 Query:
 
-``` sql
+```sql
 SELECT
     toFloat32OrNull('42.7'),
     toFloat32OrNull('abc')
-FORMAT vertical;
+FORMAT Vertical;
 ```
 
 Result:
@@ -3219,8 +3219,8 @@ toFloat32OrDefault(expr[, default])
 Supported arguments:
 - Values of type (U)Int8/16/32/64/128/256.
 - String representations of (U)Int8/16/32/128/256.
-- Values of type Float32/64.
-- String representations of Float32/64.
+- Values of type Float32/64, including `NaN` and `Inf`.
+- String representations of Float32/64, including `NaN` and `Inf` (case-insensitive).
 
 Arguments for which the default value is returned:
 - String representations of binary and hexadecimal values, e.g. `SELECT toFloat32OrDefault('0xc0fe', CAST('0', 'Float32'));`.
@@ -3233,11 +3233,11 @@ Arguments for which the default value is returned:
 
 Query:
 
-``` sql
+```sql
 SELECT
     toFloat32OrDefault('8', CAST('0', 'Float32')),
     toFloat32OrDefault('abc', CAST('0', 'Float32'))
-FORMAT vertical;
+FORMAT Vertical;
 ```
 
 Result:
@@ -3273,7 +3273,7 @@ Supported arguments:
 - Values of type (U)Int8/16/32/64/128/256.
 - String representations of (U)Int8/16/32/128/256.
 - Values of type Float32/64, including `NaN` and `Inf`.
-- String representations of type Float32/64, including `NaN` and `Inf`.
+- String representations of type Float32/64, including `NaN` and `Inf` (case-insensitive).
 
 Unsupported arguments:
 - String representations of binary and hexadecimal values, e.g. `SELECT toFloat64('0xc0fe');`.
@@ -3291,7 +3291,7 @@ SELECT
     toFloat64(42.7),
     toFloat64('42.7'),
     toFloat64('NaN')
-FORMAT vertical;
+FORMAT Vertical;
 ```
 
 Result:
@@ -3338,11 +3338,11 @@ Unsupported arguments (return `0`):
 
 Query:
 
-``` sql
+```sql
 SELECT
     toFloat64OrZero('42.7'),
     toFloat64OrZero('abc')
-FORMAT vertical;
+FORMAT Vertical;
 ```
 
 Result:
@@ -3388,11 +3388,11 @@ Unsupported arguments (return `\N`):
 
 Query:
 
-``` sql
+```sql
 SELECT
     toFloat64OrNull('42.7'),
     toFloat64OrNull('abc')
-FORMAT vertical;
+FORMAT Vertical;
 ```
 
 Result:
@@ -3429,8 +3429,8 @@ toFloat64OrDefault(expr[, default])
 Supported arguments:
 - Values of type (U)Int8/16/32/64/128/256.
 - String representations of (U)Int8/16/32/128/256.
-- Values of type Float32/64.
-- String representations of Float32/64.
+- Values of type Float32/64, including `NaN` and `Inf`.
+- String representations of Float32/64, including `NaN` and `Inf` (case-insensitive).
 
 Arguments for which the default value is returned:
 - String representations of binary and hexadecimal values, e.g. `SELECT toFloat64OrDefault('0xc0fe', CAST('0', 'Float64'));`.
@@ -3443,11 +3443,11 @@ Arguments for which the default value is returned:
 
 Query:
 
-``` sql
+```sql
 SELECT
     toFloat64OrDefault('8', CAST('0', 'Float64')),
     toFloat64OrDefault('abc', CAST('0', 'Float64'))
-FORMAT vertical;
+FORMAT Vertical;
 ```
 
 Result:

From 40e763dd8b79b20a56b9a5cbff9571d7d9ed8869 Mon Sep 17 00:00:00 2001
From: Han Fei <hanfei19910905@gmail.com>
Date: Tue, 6 Aug 2024 17:17:02 +0200
Subject: [PATCH 1465/1488] more logs to debug logical error from async inserts

---
 src/Storages/MergeTree/MergeTreeDataWriter.cpp | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp
index ee3ac4207cc..cb02f1cf5f2 100644
--- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp
+++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp
@@ -240,6 +240,14 @@ std::vector<AsyncInsertInfoPtr> scatterAsyncInsertInfoBySelector(AsyncInsertInfo
             ++offset_idx;
         }
     }
+    if (offset_idx != async_insert_info->offsets.size())
+    {
+        LOG_ERROR(
+            getLogger("MergeTreeDataWriter"),
+            "ChunkInfo of async insert offsets doesn't match the selector size {}. Offsets content is ({})",
+            selector.size(), fmt::join(async_insert_info->offsets.begin(), async_insert_info->offsets.end(), ","));
+        throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected error for async deduplicated insert, please check error logs");
+    }
     return result;
 }
 
@@ -317,8 +325,10 @@ BlocksWithPartition MergeTreeDataWriter::splitBlockIntoParts(
     {
         if (async_insert_info_with_partition[i] == nullptr)
         {
-            LOG_ERROR(getLogger("MergeTreeDataWriter"), "The {}th element in async_insert_info_with_partition is nullptr. There are totally {} partitions in the insert. Selector content is {}",
-                      i, partitions_count, fmt::join(selector.begin(), selector.end(), ","));
+            LOG_ERROR(
+                getLogger("MergeTreeDataWriter"),
+                "The {}th element in async_insert_info_with_partition is nullptr. There are totally {} partitions in the insert. Selector content is ({}). Offsets content is ({})",
+                i, partitions_count, fmt::join(selector.begin(), selector.end(), ","), fmt::join(async_insert_info->offsets.begin(), async_insert_info->offsets.end(), ","));
             throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected error for async deduplicated insert, please check error logs");
         }
         result[i].offsets = std::move(async_insert_info_with_partition[i]->offsets);

From 2d93910bd601f7a4f8cc9385a2d191ce806453e3 Mon Sep 17 00:00:00 2001
From: maxvostrikov <max.vostrikov@clickhouse.com>
Date: Tue, 6 Aug 2024 17:44:29 +0200
Subject: [PATCH 1466/1488] added tags no-fasttest

---
 tests/queries/0_stateless/03215_parquet_index.sql | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tests/queries/0_stateless/03215_parquet_index.sql b/tests/queries/0_stateless/03215_parquet_index.sql
index 0afccdf6f3b..5766f40f021 100644
--- a/tests/queries/0_stateless/03215_parquet_index.sql
+++ b/tests/queries/0_stateless/03215_parquet_index.sql
@@ -1,3 +1,5 @@
+-- Tags: no-fasttest
+
 -- default settings.
 DROP TABLE IF EXISTS test_parquet;
 CREATE TABLE test_parquet (col1 int, col2 String) ENGINE=File(Parquet);

From 1e631472f0aabcb1b856942fb7409c5c010315a1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Tue, 6 Aug 2024 18:07:49 +0200
Subject: [PATCH 1467/1488] Make 01685_ssd_cache_dictionary_complex_key
 parallelizable

---
 .../01685_ssd_cache_dictionary_complex_key.sh | 81 +++++++++----------
 1 file changed, 38 insertions(+), 43 deletions(-)

diff --git a/tests/queries/0_stateless/01685_ssd_cache_dictionary_complex_key.sh b/tests/queries/0_stateless/01685_ssd_cache_dictionary_complex_key.sh
index 5583a9dd5e7..55061b9a643 100755
--- a/tests/queries/0_stateless/01685_ssd_cache_dictionary_complex_key.sh
+++ b/tests/queries/0_stateless/01685_ssd_cache_dictionary_complex_key.sh
@@ -1,15 +1,11 @@
 #!/usr/bin/env bash
-# Tags: no-parallel
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
 $CLICKHOUSE_CLIENT -n --query="
-    DROP DATABASE IF EXISTS 01685_database_for_cache_dictionary;
-    CREATE DATABASE 01685_database_for_cache_dictionary;
-
-    CREATE TABLE 01685_database_for_cache_dictionary.complex_key_simple_attributes_source_table
+    CREATE TABLE complex_key_simple_attributes_source_table
     (
     id UInt64,
     id_key String,
@@ -18,11 +14,11 @@ $CLICKHOUSE_CLIENT -n --query="
     )
     ENGINE = TinyLog;
 
-    INSERT INTO 01685_database_for_cache_dictionary.complex_key_simple_attributes_source_table VALUES(0, 'id_key_0', 'value_0', 'value_second_0');
-    INSERT INTO 01685_database_for_cache_dictionary.complex_key_simple_attributes_source_table VALUES(1, 'id_key_1', 'value_1', 'value_second_1');
-    INSERT INTO 01685_database_for_cache_dictionary.complex_key_simple_attributes_source_table VALUES(2, 'id_key_2', 'value_2', 'value_second_2');
+    INSERT INTO complex_key_simple_attributes_source_table VALUES(0, 'id_key_0', 'value_0', 'value_second_0');
+    INSERT INTO complex_key_simple_attributes_source_table VALUES(1, 'id_key_1', 'value_1', 'value_second_1');
+    INSERT INTO complex_key_simple_attributes_source_table VALUES(2, 'id_key_2', 'value_2', 'value_second_2');
 
-    CREATE DICTIONARY 01685_database_for_cache_dictionary.cache_dictionary_complex_key_simple_attributes
+    CREATE DICTIONARY cache_dictionary_complex_key_simple_attributes
     (
     id UInt64,
     id_key String,
@@ -30,32 +26,32 @@ $CLICKHOUSE_CLIENT -n --query="
     value_second String DEFAULT 'value_second_default'
     )
     PRIMARY KEY id, id_key
-    SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'complex_key_simple_attributes_source_table' DB '01685_database_for_cache_dictionary'))
+    SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'complex_key_simple_attributes_source_table' DB '${CLICKHOUSE_DATABASE}'))
     LIFETIME(MIN 1 MAX 1000)
-    LAYOUT(COMPLEX_KEY_SSD_CACHE(BLOCK_SIZE 4096 FILE_SIZE 8192 PATH '$USER_FILES_PATH/0d'));
+    LAYOUT(COMPLEX_KEY_SSD_CACHE(BLOCK_SIZE 4096 FILE_SIZE 8192 PATH '$USER_FILES_PATH/${CLICKHOUSE_DATABASE}_dic'));
 
     SELECT 'Dictionary cache_dictionary_complex_key_simple_attributes';
     SELECT 'dictGet existing value';
-    SELECT dictGet('01685_database_for_cache_dictionary.cache_dictionary_complex_key_simple_attributes', 'value_first', (number, concat('id_key_', toString(number)))) as value_first,
-        dictGet('01685_database_for_cache_dictionary.cache_dictionary_complex_key_simple_attributes', 'value_second', (number, concat('id_key_', toString(number)))) as value_second FROM system.numbers LIMIT 3;
+    SELECT dictGet('cache_dictionary_complex_key_simple_attributes', 'value_first', (number, concat('id_key_', toString(number)))) as value_first,
+        dictGet('cache_dictionary_complex_key_simple_attributes', 'value_second', (number, concat('id_key_', toString(number)))) as value_second FROM system.numbers LIMIT 3;
     SELECT 'dictGet with non existing value';
-    SELECT dictGet('01685_database_for_cache_dictionary.cache_dictionary_complex_key_simple_attributes', 'value_first', (number, concat('id_key_', toString(number)))) as value_first,
-        dictGet('01685_database_for_cache_dictionary.cache_dictionary_complex_key_simple_attributes', 'value_second', (number, concat('id_key_', toString(number)))) as value_second FROM system.numbers LIMIT 4;
+    SELECT dictGet('cache_dictionary_complex_key_simple_attributes', 'value_first', (number, concat('id_key_', toString(number)))) as value_first,
+        dictGet('cache_dictionary_complex_key_simple_attributes', 'value_second', (number, concat('id_key_', toString(number)))) as value_second FROM system.numbers LIMIT 4;
     SELECT 'dictGetOrDefault existing value';
-    SELECT dictGetOrDefault('01685_database_for_cache_dictionary.cache_dictionary_complex_key_simple_attributes', 'value_first', (number, concat('id_key_', toString(number))), toString('default')) as value_first,
-        dictGetOrDefault('01685_database_for_cache_dictionary.cache_dictionary_complex_key_simple_attributes', 'value_second', (number, concat('id_key_', toString(number))), toString('default')) as value_second FROM system.numbers LIMIT 3;
+    SELECT dictGetOrDefault('cache_dictionary_complex_key_simple_attributes', 'value_first', (number, concat('id_key_', toString(number))), toString('default')) as value_first,
+        dictGetOrDefault('cache_dictionary_complex_key_simple_attributes', 'value_second', (number, concat('id_key_', toString(number))), toString('default')) as value_second FROM system.numbers LIMIT 3;
     SELECT 'dictGetOrDefault non existing value';
-    SELECT dictGetOrDefault('01685_database_for_cache_dictionary.cache_dictionary_complex_key_simple_attributes', 'value_first', (number, concat('id_key_', toString(number))), toString('default')) as value_first,
-        dictGetOrDefault('01685_database_for_cache_dictionary.cache_dictionary_complex_key_simple_attributes', 'value_second', (number, concat('id_key_', toString(number))), toString('default')) as value_second FROM system.numbers LIMIT 4;
+    SELECT dictGetOrDefault('cache_dictionary_complex_key_simple_attributes', 'value_first', (number, concat('id_key_', toString(number))), toString('default')) as value_first,
+        dictGetOrDefault('cache_dictionary_complex_key_simple_attributes', 'value_second', (number, concat('id_key_', toString(number))), toString('default')) as value_second FROM system.numbers LIMIT 4;
     SELECT 'dictHas';
-    SELECT dictHas('01685_database_for_cache_dictionary.cache_dictionary_complex_key_simple_attributes', (number, concat('id_key_', toString(number)))) FROM system.numbers LIMIT 4;
+    SELECT dictHas('cache_dictionary_complex_key_simple_attributes', (number, concat('id_key_', toString(number)))) FROM system.numbers LIMIT 4;
     SELECT 'select all values as input stream';
-    SELECT * FROM 01685_database_for_cache_dictionary.cache_dictionary_complex_key_simple_attributes ORDER BY id;
+    SELECT * FROM cache_dictionary_complex_key_simple_attributes ORDER BY id;
 
-    DROP DICTIONARY 01685_database_for_cache_dictionary.cache_dictionary_complex_key_simple_attributes;
-    DROP TABLE 01685_database_for_cache_dictionary.complex_key_simple_attributes_source_table;
+    DROP DICTIONARY cache_dictionary_complex_key_simple_attributes;
+    DROP TABLE complex_key_simple_attributes_source_table;
 
-    CREATE TABLE 01685_database_for_cache_dictionary.complex_key_complex_attributes_source_table
+    CREATE TABLE complex_key_complex_attributes_source_table
     (
     id UInt64,
     id_key String,
@@ -64,11 +60,11 @@ $CLICKHOUSE_CLIENT -n --query="
     )
     ENGINE = TinyLog;
 
-    INSERT INTO 01685_database_for_cache_dictionary.complex_key_complex_attributes_source_table VALUES(0, 'id_key_0', 'value_0', 'value_second_0');
-    INSERT INTO 01685_database_for_cache_dictionary.complex_key_complex_attributes_source_table VALUES(1, 'id_key_1', 'value_1', NULL);
-    INSERT INTO 01685_database_for_cache_dictionary.complex_key_complex_attributes_source_table VALUES(2, 'id_key_2', 'value_2', 'value_second_2');
+    INSERT INTO complex_key_complex_attributes_source_table VALUES(0, 'id_key_0', 'value_0', 'value_second_0');
+    INSERT INTO complex_key_complex_attributes_source_table VALUES(1, 'id_key_1', 'value_1', NULL);
+    INSERT INTO complex_key_complex_attributes_source_table VALUES(2, 'id_key_2', 'value_2', 'value_second_2');
 
-    CREATE DICTIONARY 01685_database_for_cache_dictionary.cache_dictionary_complex_key_complex_attributes
+    CREATE DICTIONARY cache_dictionary_complex_key_complex_attributes
     (
         id UInt64,
         id_key String,
@@ -77,29 +73,28 @@ $CLICKHOUSE_CLIENT -n --query="
         value_second Nullable(String) DEFAULT 'value_second_default'
     )
     PRIMARY KEY id, id_key
-    SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'complex_key_complex_attributes_source_table' DB '01685_database_for_cache_dictionary'))
+    SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'complex_key_complex_attributes_source_table' DB '${CLICKHOUSE_DATABASE}'))
     LIFETIME(MIN 1 MAX 1000)
     LAYOUT(COMPLEX_KEY_SSD_CACHE(BLOCK_SIZE 4096 FILE_SIZE 8192 PATH '$USER_FILES_PATH/1d'));
 
     SELECT 'Dictionary cache_dictionary_complex_key_complex_attributes';
     SELECT 'dictGet existing value';
-    SELECT dictGet('01685_database_for_cache_dictionary.cache_dictionary_complex_key_complex_attributes', 'value_first', (number, concat('id_key_', toString(number)))) as value_first,
-        dictGet('01685_database_for_cache_dictionary.cache_dictionary_complex_key_complex_attributes', 'value_second', (number, concat('id_key_', toString(number)))) as value_second FROM system.numbers LIMIT 3;
+    SELECT dictGet('cache_dictionary_complex_key_complex_attributes', 'value_first', (number, concat('id_key_', toString(number)))) as value_first,
+        dictGet('cache_dictionary_complex_key_complex_attributes', 'value_second', (number, concat('id_key_', toString(number)))) as value_second FROM system.numbers LIMIT 3;
     SELECT 'dictGet with non existing value';
-    SELECT dictGet('01685_database_for_cache_dictionary.cache_dictionary_complex_key_complex_attributes', 'value_first', (number, concat('id_key_', toString(number)))) as value_first,
-        dictGet('01685_database_for_cache_dictionary.cache_dictionary_complex_key_complex_attributes', 'value_second', (number, concat('id_key_', toString(number)))) as value_second FROM system.numbers LIMIT 4;
+    SELECT dictGet('cache_dictionary_complex_key_complex_attributes', 'value_first', (number, concat('id_key_', toString(number)))) as value_first,
+        dictGet('cache_dictionary_complex_key_complex_attributes', 'value_second', (number, concat('id_key_', toString(number)))) as value_second FROM system.numbers LIMIT 4;
     SELECT 'dictGetOrDefault existing value';
-    SELECT dictGetOrDefault('01685_database_for_cache_dictionary.cache_dictionary_complex_key_complex_attributes', 'value_first', (number, concat('id_key_', toString(number))), toString('default')) as value_first,
-        dictGetOrDefault('01685_database_for_cache_dictionary.cache_dictionary_complex_key_complex_attributes', 'value_second', (number, concat('id_key_', toString(number))), toString('default')) as value_second FROM system.numbers LIMIT 3;
+    SELECT dictGetOrDefault('cache_dictionary_complex_key_complex_attributes', 'value_first', (number, concat('id_key_', toString(number))), toString('default')) as value_first,
+        dictGetOrDefault('cache_dictionary_complex_key_complex_attributes', 'value_second', (number, concat('id_key_', toString(number))), toString('default')) as value_second FROM system.numbers LIMIT 3;
     SELECT 'dictGetOrDefault non existing value';
-    SELECT dictGetOrDefault('01685_database_for_cache_dictionary.cache_dictionary_complex_key_complex_attributes', 'value_first', (number, concat('id_key_', toString(number))), toString('default')) as value_first,
-        dictGetOrDefault('01685_database_for_cache_dictionary.cache_dictionary_complex_key_complex_attributes', 'value_second', (number, concat('id_key_', toString(number))), toString('default')) as value_second FROM system.numbers LIMIT 4;
+    SELECT dictGetOrDefault('cache_dictionary_complex_key_complex_attributes', 'value_first', (number, concat('id_key_', toString(number))), toString('default')) as value_first,
+        dictGetOrDefault('cache_dictionary_complex_key_complex_attributes', 'value_second', (number, concat('id_key_', toString(number))), toString('default')) as value_second FROM system.numbers LIMIT 4;
     SELECT 'dictHas';
-    SELECT dictHas('01685_database_for_cache_dictionary.cache_dictionary_complex_key_complex_attributes', (number, concat('id_key_', toString(number)))) FROM system.numbers LIMIT 4;
+    SELECT dictHas('cache_dictionary_complex_key_complex_attributes', (number, concat('id_key_', toString(number)))) FROM system.numbers LIMIT 4;
     SELECT 'select all values as input stream';
-    SELECT * FROM 01685_database_for_cache_dictionary.cache_dictionary_complex_key_complex_attributes ORDER BY id;
+    SELECT * FROM cache_dictionary_complex_key_complex_attributes ORDER BY id;
 
-    DROP DICTIONARY 01685_database_for_cache_dictionary.cache_dictionary_complex_key_complex_attributes;
-    DROP TABLE 01685_database_for_cache_dictionary.complex_key_complex_attributes_source_table;
-
-    DROP DATABASE 01685_database_for_cache_dictionary;"
+    DROP DICTIONARY cache_dictionary_complex_key_complex_attributes;
+    DROP TABLE complex_key_complex_attributes_source_table;
+"

From 38d891c6befdacfa835005b522f104e9428a270b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Tue, 6 Aug 2024 18:18:09 +0200
Subject: [PATCH 1468/1488] Make 02015_column_default_dict_get_identifier
 parallelizable

---
 ...015_column_default_dict_get_identifier.sql | 29 ++++++++-----------
 1 file changed, 12 insertions(+), 17 deletions(-)

diff --git a/tests/queries/0_stateless/02015_column_default_dict_get_identifier.sql b/tests/queries/0_stateless/02015_column_default_dict_get_identifier.sql
index 24a3b631388..046d0c42180 100644
--- a/tests/queries/0_stateless/02015_column_default_dict_get_identifier.sql
+++ b/tests/queries/0_stateless/02015_column_default_dict_get_identifier.sql
@@ -1,9 +1,6 @@
 -- Tags: no-parallel
 
-DROP DATABASE IF EXISTS 02015_db;
-CREATE DATABASE 02015_db;
-
-CREATE TABLE 02015_db.test_table
+CREATE TABLE test_table
 (
     key_column UInt64,
     data_column_1 UInt64,
@@ -12,9 +9,9 @@ CREATE TABLE 02015_db.test_table
 ENGINE = MergeTree
 ORDER BY key_column;
 
-INSERT INTO 02015_db.test_table VALUES (0, 0, 0);
+INSERT INTO test_table VALUES (0, 0, 0);
 
-CREATE DICTIONARY 02015_db.test_dictionary
+CREATE DICTIONARY test_dictionary
 (
     key_column UInt64 DEFAULT 0,
     data_column_1 UInt64 DEFAULT 1,
@@ -22,20 +19,18 @@ CREATE DICTIONARY 02015_db.test_dictionary
 )
 PRIMARY KEY key_column
 LAYOUT(DIRECT())
-SOURCE(CLICKHOUSE(DB '02015_db' TABLE 'test_table'));
+SOURCE(CLICKHOUSE(DB currentDatabase() TABLE 'test_table'));
 
-CREATE TABLE 02015_db.test_table_default
+CREATE TABLE test_table_default
 (
-    data_1 DEFAULT dictGetUInt64('02015_db.test_dictionary', 'data_column_1', toUInt64(0)),
-    data_2 DEFAULT dictGet(02015_db.test_dictionary, 'data_column_2', toUInt64(0))
+    data_1 DEFAULT dictGetUInt64('test_dictionary', 'data_column_1', toUInt64(0)),
+    data_2 DEFAULT dictGet(test_dictionary, 'data_column_2', toUInt64(0))
 )
 ENGINE=TinyLog;
 
-INSERT INTO 02015_db.test_table_default(data_1) VALUES (5);
-SELECT * FROM 02015_db.test_table_default;
+INSERT INTO test_table_default(data_1) VALUES (5);
+SELECT * FROM test_table_default;
 
-DROP TABLE 02015_db.test_table_default;
-DROP DICTIONARY 02015_db.test_dictionary;
-DROP TABLE 02015_db.test_table;
-
-DROP DATABASE 02015_db;
+DROP TABLE test_table_default;
+DROP DICTIONARY test_dictionary;
+DROP TABLE test_table;

From 425fb0f485b1936ebfa46f0ae9db1e7ea134272f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Tue, 6 Aug 2024 18:20:09 +0200
Subject: [PATCH 1469/1488] Make 01600_multiple_left_join_with_aliases
 parallelizable

---
 .../01600_multiple_left_join_with_aliases.sql | 25 +++++++------------
 1 file changed, 9 insertions(+), 16 deletions(-)

diff --git a/tests/queries/0_stateless/01600_multiple_left_join_with_aliases.sql b/tests/queries/0_stateless/01600_multiple_left_join_with_aliases.sql
index ae296e18560..2945622fbed 100644
--- a/tests/queries/0_stateless/01600_multiple_left_join_with_aliases.sql
+++ b/tests/queries/0_stateless/01600_multiple_left_join_with_aliases.sql
@@ -1,9 +1,4 @@
--- Tags: no-parallel
-
-drop database if exists test_01600;
-create database test_01600;
-
-CREATE TABLE test_01600.base
+CREATE TABLE base
 (
 `id` UInt64,
 `id2` UInt64,
@@ -14,7 +9,7 @@ ENGINE=MergeTree()
 PARTITION BY d
 ORDER BY (id,id2,d);
 
-CREATE TABLE test_01600.derived1
+CREATE TABLE derived1
 (
     `id1` UInt64,
     `d1` UInt64,
@@ -25,7 +20,7 @@ PARTITION BY d1
 ORDER BY (id1, d1)
 ;
 
-CREATE TABLE test_01600.derived2
+CREATE TABLE derived2
 (
     `id2` UInt64,
     `d2` UInt64,
@@ -36,19 +31,17 @@ PARTITION BY d2
 ORDER BY (id2, d2)
 ;
 
-select 
+select
 base.id as `base.id`,
 derived2.value2 as `derived2.value2`,
 derived1.value1 as `derived1.value1`
-from test_01600.base as base 
-left join test_01600.derived2 as derived2 on base.id2 = derived2.id2
-left join test_01600.derived1 as derived1 on base.id = derived1.id1;
+from base as base
+left join derived2 as derived2 on base.id2 = derived2.id2
+left join derived1 as derived1 on base.id = derived1.id1;
 
 
 SELECT
     base.id AS `base.id`,
     derived1.value1 AS `derived1.value1`
-FROM test_01600.base AS base
-LEFT JOIN test_01600.derived1 AS derived1 ON base.id = derived1.id1;
-
-drop database test_01600;
+FROM base AS base
+LEFT JOIN derived1 AS derived1 ON base.id = derived1.id1;

From 078e4ca36db15a4f3bab27141d24fe00f89a5a1a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Tue, 6 Aug 2024 18:21:58 +0200
Subject: [PATCH 1470/1488] Make 02950_dictionary_ssd_cache_short_circuit
 parallelizable

---
 ...2950_dictionary_ssd_cache_short_circuit.sh | 23 ++++++++-----------
 1 file changed, 9 insertions(+), 14 deletions(-)

diff --git a/tests/queries/0_stateless/02950_dictionary_ssd_cache_short_circuit.sh b/tests/queries/0_stateless/02950_dictionary_ssd_cache_short_circuit.sh
index 3d2fe5d664d..daa9c571a5d 100755
--- a/tests/queries/0_stateless/02950_dictionary_ssd_cache_short_circuit.sh
+++ b/tests/queries/0_stateless/02950_dictionary_ssd_cache_short_circuit.sh
@@ -1,15 +1,11 @@
 #!/usr/bin/env bash
-# Tags: no-parallel
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
 $CLICKHOUSE_CLIENT -n --query="
-    DROP DATABASE IF EXISTS 02950_database_for_ssd_cache_dictionary;
-    CREATE DATABASE 02950_database_for_ssd_cache_dictionary;
-
-    CREATE TABLE 02950_database_for_ssd_cache_dictionary.source_table
+    CREATE TABLE source_table
     (
     id UInt64,
     v1 String,
@@ -18,9 +14,9 @@ $CLICKHOUSE_CLIENT -n --query="
     )
     ENGINE = TinyLog;
 
-    INSERT INTO 02950_database_for_ssd_cache_dictionary.source_table VALUES (0, 'zero', 'zero', 0), (1, 'one', NULL, 1);
+    INSERT INTO source_table VALUES (0, 'zero', 'zero', 0), (1, 'one', NULL, 1);
 
-    CREATE DICTIONARY 02950_database_for_ssd_cache_dictionary.ssd_cache_dictionary
+    CREATE DICTIONARY ssd_cache_dictionary
     (
     id UInt64,
     v1 String,
@@ -30,12 +26,11 @@ $CLICKHOUSE_CLIENT -n --query="
     PRIMARY KEY id
     SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'source_table'))
     LIFETIME(MIN 1 MAX 1000)
-    LAYOUT(SSD_CACHE(BLOCK_SIZE 4096 FILE_SIZE 8192 PATH '$CLICKHOUSE_USER_FILES/0d'));
+    LAYOUT(SSD_CACHE(BLOCK_SIZE 4096 FILE_SIZE 8192 PATH '$CLICKHOUSE_USER_FILES/${CLICKHOUSE_DATABASE}_ssd_dic'));
 
-    SELECT dictGetOrDefault('02950_database_for_ssd_cache_dictionary.ssd_cache_dictionary', ('v1', 'v2'), 0, (intDiv(1, id), intDiv(1, id))) FROM 02950_database_for_ssd_cache_dictionary.source_table;
-    SELECT dictGetOrDefault('02950_database_for_ssd_cache_dictionary.ssd_cache_dictionary', 'v2', id+1, intDiv(NULL, id)) FROM 02950_database_for_ssd_cache_dictionary.source_table;
-    SELECT dictGetOrDefault('02950_database_for_ssd_cache_dictionary.ssd_cache_dictionary', 'v3', id+1, intDiv(NULL, id)) FROM 02950_database_for_ssd_cache_dictionary.source_table;
+    SELECT dictGetOrDefault('ssd_cache_dictionary', ('v1', 'v2'), 0, (intDiv(1, id), intDiv(1, id))) FROM source_table;
+    SELECT dictGetOrDefault('ssd_cache_dictionary', 'v2', id+1, intDiv(NULL, id)) FROM source_table;
+    SELECT dictGetOrDefault('ssd_cache_dictionary', 'v3', id+1, intDiv(NULL, id)) FROM source_table;
 
-    DROP DICTIONARY 02950_database_for_ssd_cache_dictionary.ssd_cache_dictionary;
-    DROP TABLE 02950_database_for_ssd_cache_dictionary.source_table;
-    DROP DATABASE 02950_database_for_ssd_cache_dictionary;"
+    DROP DICTIONARY ssd_cache_dictionary;
+    DROP TABLE source_table;"

From c58643897a95d1e58bf25d65dfaad52aa0eddefc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Tue, 6 Aug 2024 18:26:31 +0200
Subject: [PATCH 1471/1488] Make 02002_row_level_filter_bug parallelizable

---
 .../0_stateless/02002_row_level_filter_bug.sh | 31 +++++++++++--------
 1 file changed, 18 insertions(+), 13 deletions(-)

diff --git a/tests/queries/0_stateless/02002_row_level_filter_bug.sh b/tests/queries/0_stateless/02002_row_level_filter_bug.sh
index d15a26f48f5..557aa738217 100755
--- a/tests/queries/0_stateless/02002_row_level_filter_bug.sh
+++ b/tests/queries/0_stateless/02002_row_level_filter_bug.sh
@@ -1,11 +1,12 @@
 #!/usr/bin/env bash
-# Tags: no-parallel
-# Tag no-parallel: create user
 
 CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CURDIR"/../shell_config.sh
 
+TEST_ROLE="${CLICKHOUSE_DATABASE}_role"
+TEST_USER="${CLICKHOUSE_DATABASE}_user"
+TEST_POLICY="${CLICKHOUSE_DATABASE}_policy"
 
 $CLICKHOUSE_CLIENT --query "drop table if exists test_table"
 $CLICKHOUSE_CLIENT --query "CREATE TABLE test_table
@@ -37,26 +38,26 @@ arrayJoin(['AWD','ZZZ']) as team,
 arrayJoin([3183,3106,0,3130,3108,3126,3109,3107,3182,3180,3129,3128,3125,3266]) as a
 FROM numbers(600);"
 
-$CLICKHOUSE_CLIENT --query "DROP ROLE IF exists AWD;"
-$CLICKHOUSE_CLIENT --query "create role AWD;"
-$CLICKHOUSE_CLIENT --query "REVOKE ALL ON *.* FROM AWD;"
+$CLICKHOUSE_CLIENT --query "DROP ROLE IF EXISTS ${TEST_ROLE};"
+$CLICKHOUSE_CLIENT --query "create role ${TEST_ROLE};"
+$CLICKHOUSE_CLIENT --query "REVOKE ALL ON *.* FROM ${TEST_ROLE};"
 
-$CLICKHOUSE_CLIENT --query "DROP USER IF EXISTS AWD_user;"
-$CLICKHOUSE_CLIENT --query "CREATE USER AWD_user IDENTIFIED WITH plaintext_password BY 'AWD_pwd' DEFAULT ROLE AWD;"
+$CLICKHOUSE_CLIENT --query "DROP USER IF EXISTS ${TEST_USER};"
+$CLICKHOUSE_CLIENT --query "CREATE USER ${TEST_USER} IDENTIFIED WITH plaintext_password BY 'AWD_pwd' DEFAULT ROLE ${TEST_ROLE};"
 
-$CLICKHOUSE_CLIENT --query "GRANT SELECT ON test_table TO AWD;"
+$CLICKHOUSE_CLIENT --query "GRANT SELECT ON test_table TO ${TEST_ROLE};"
 
-$CLICKHOUSE_CLIENT --query "DROP ROW POLICY IF EXISTS ttt_bu_test_table_AWD ON test_table;"
-$CLICKHOUSE_CLIENT --query "CREATE ROW POLICY ttt_bu_test_table_AWD ON test_table FOR SELECT USING team = 'AWD' TO AWD;"
+$CLICKHOUSE_CLIENT --query "DROP ROW POLICY IF EXISTS ${TEST_POLICY} ON test_table;"
+$CLICKHOUSE_CLIENT --query "CREATE ROW POLICY ${TEST_POLICY} ON test_table FOR SELECT USING team = 'AWD' TO ${TEST_ROLE};"
 
-$CLICKHOUSE_CLIENT  --user=AWD_user --password=AWD_pwd  --query "
+$CLICKHOUSE_CLIENT  --user=${TEST_USER} --password=AWD_pwd  --query "
 SELECT count() AS count
  FROM test_table
 WHERE
  t_date = '2021-07-15' AND c = 'aur' AND a=3130;
 "
 
-$CLICKHOUSE_CLIENT  --user=AWD_user --password=AWD_pwd  --query "
+$CLICKHOUSE_CLIENT  --user=${TEST_USER} --password=AWD_pwd  --query "
 SELECT
     team,
     a,
@@ -70,8 +71,12 @@ GROUP BY
     t_date;
 "
 
-$CLICKHOUSE_CLIENT  --user=AWD_user --password=AWD_pwd  --query "
+$CLICKHOUSE_CLIENT  --user=${TEST_USER} --password=AWD_pwd  --query "
 SELECT count() AS count
 FROM test_table
 WHERE (t_date = '2021-07-15') AND (c = 'aur') AND (a = 313)
 "
+
+$CLICKHOUSE_CLIENT --query "DROP ROLE IF EXISTS ${TEST_ROLE};"
+$CLICKHOUSE_CLIENT --query "DROP USER IF EXISTS ${TEST_USER};"
+$CLICKHOUSE_CLIENT --query "DROP ROW POLICY IF EXISTS ${TEST_POLICY} ON test_table;"

From 57b6d461418d4ad8e2d5cb579d5671abc077117e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= <git@rmr.ninja>
Date: Tue, 6 Aug 2024 18:28:01 +0200
Subject: [PATCH 1472/1488] Make 01764_table_function_dictionary parallelizable

---
 tests/queries/0_stateless/01764_table_function_dictionary.sql | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/tests/queries/0_stateless/01764_table_function_dictionary.sql b/tests/queries/0_stateless/01764_table_function_dictionary.sql
index 76e7213b367..e37f8d2a290 100644
--- a/tests/queries/0_stateless/01764_table_function_dictionary.sql
+++ b/tests/queries/0_stateless/01764_table_function_dictionary.sql
@@ -1,5 +1,3 @@
--- Tags: no-parallel
-
 DROP TABLE IF EXISTS table_function_dictionary_source_table;
 CREATE TABLE table_function_dictionary_source_table
 (
@@ -18,7 +16,7 @@ CREATE DICTIONARY table_function_dictionary_test_dictionary
    value UInt64 DEFAULT 0
 )
 PRIMARY KEY id
-SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'table_function_dictionary_source_table'))
+SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' DATABASE currentDatabase() TABLE 'table_function_dictionary_source_table'))
 LAYOUT(DIRECT());
 
 SELECT * FROM dictionary('table_function_dictionary_test_dictionary');

From f0fdba3bbeb8687e38f89d8379e70cfd39db1252 Mon Sep 17 00:00:00 2001
From: Max Kainov <maxkaynov@gmail.com>
Date: Tue, 6 Aug 2024 19:02:49 +0200
Subject: [PATCH 1473/1488] CI: Strict job timeout 1.5h for tests, 2h for
 builds

---
 tests/ci/ci_config.py      | 1 -
 tests/ci/ci_definitions.py | 6 ++----
 2 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py
index 7a19eb6f827..f578cd8b559 100644
--- a/tests/ci/ci_config.py
+++ b/tests/ci/ci_config.py
@@ -510,7 +510,6 @@ class CI:
         JobNames.LIBFUZZER_TEST: JobConfig(
             required_builds=[BuildNames.FUZZERS],
             run_by_label=Tags.libFuzzer,
-            timeout=10800,
             run_command='libfuzzer_test_check.py "$CHECK_NAME"',
             runner_type=Runners.STYLE_CHECKER,
         ),
diff --git a/tests/ci/ci_definitions.py b/tests/ci/ci_definitions.py
index 48847b0d7a6..69e7ed259d5 100644
--- a/tests/ci/ci_definitions.py
+++ b/tests/ci/ci_definitions.py
@@ -331,7 +331,7 @@ class JobConfig:
     # will be triggered for the job if omitted in CI workflow yml
     run_command: str = ""
     # job timeout, seconds
-    timeout: Optional[int] = None
+    timeout: Optional[int] = 5400
     # sets number of batches for a multi-batch job
     num_batches: int = 1
     # label that enables job in CI, if set digest isn't used
@@ -420,7 +420,6 @@ class CommonJobConfigs:
         ),
         run_command='functional_test_check.py "$CHECK_NAME"',
         runner_type=Runners.FUNC_TESTER,
-        timeout=9000,
     )
     STATEFUL_TEST = JobConfig(
         job_name_keyword="stateful",
@@ -531,7 +530,6 @@ class CommonJobConfigs:
             docker=["clickhouse/sqllogic-test"],
         ),
         run_command="sqllogic_test.py",
-        timeout=10800,
         release_only=True,
         runner_type=Runners.FUNC_TESTER,
     )
@@ -543,7 +541,6 @@ class CommonJobConfigs:
             docker=["clickhouse/sqltest"],
         ),
         run_command="sqltest.py",
-        timeout=10800,
         release_only=True,
         runner_type=Runners.FUZZER_UNIT_TESTER,
     )
@@ -613,6 +610,7 @@ class CommonJobConfigs:
             docker=["clickhouse/binary-builder"],
             git_submodules=True,
         ),
+        timeout=7200,
         run_command="build_check.py $BUILD_NAME",
         runner_type=Runners.BUILDER,
     )

From 9c92c26edcb8ac83191f8a814196aa05a5600730 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Tue, 6 Aug 2024 17:51:26 +0000
Subject: [PATCH 1474/1488] Simplify code.

---
 src/Storages/MergeTree/MergeTreeData.cpp      |  9 +++---
 .../MergeTree/registerStorageMergeTree.cpp    | 13 +++++++++
 src/Storages/StorageFactory.cpp               | 28 -------------------
 3 files changed, 17 insertions(+), 33 deletions(-)

diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp
index 98e1cddcf4b..dbb3e39c12e 100644
--- a/src/Storages/MergeTree/MergeTreeData.cpp
+++ b/src/Storages/MergeTree/MergeTreeData.cpp
@@ -3216,13 +3216,12 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, Context
     /// Block the case of alter table add projection for special merge trees.
     if (std::any_of(commands.begin(), commands.end(), [](const AlterCommand & c) { return c.type == AlterCommand::ADD_PROJECTION; }))
     {
-        const std::unordered_set<String> allowed_storages{"MergeTree", "ReplicatedMergeTree", "SharedMergeTree"};
-        if (auto storage_name = getName(); !allowed_storages.contains(storage_name)
+        if (merging_params.mode != MergingParams::Mode::Ordinary
             && settings_from_storage->deduplicate_merge_projection_mode == DeduplicateMergeProjectionMode::THROW)
             throw Exception(ErrorCodes::SUPPORT_IS_DISABLED,
-                "Projection is fully supported in (Replictaed, Shared)MergeTree, but also allowed in non-throw mode with other"
-                " mergetree family members. Consider drop or rebuild option of deduplicate_merge_projection_mode."
-                " Current storage name is {}.", storage_name);
+                "Projection is fully supported in {} with deduplicate_merge_projection_mode = throw. "
+                "Use 'drop' or 'rebuild' option of deduplicate_merge_projection_mode.",
+                getName());
     }
 
     commands.apply(new_metadata, local_context);
diff --git a/src/Storages/MergeTree/registerStorageMergeTree.cpp b/src/Storages/MergeTree/registerStorageMergeTree.cpp
index 3f0603f6900..b7887c35590 100644
--- a/src/Storages/MergeTree/registerStorageMergeTree.cpp
+++ b/src/Storages/MergeTree/registerStorageMergeTree.cpp
@@ -34,6 +34,7 @@ namespace ErrorCodes
     extern const int UNKNOWN_STORAGE;
     extern const int NO_REPLICA_NAME_GIVEN;
     extern const int CANNOT_EXTRACT_TABLE_STRUCTURE;
+    extern const int SUPPORT_IS_DISABLED;
 }
 
 
@@ -829,6 +830,18 @@ static StoragePtr create(const StorageFactory::Arguments & args)
                     "Floating point partition key is not supported: {}", metadata.partition_key.column_names[i]);
     }
 
+    if (metadata.hasProjections() && args.mode == LoadingStrictnessLevel::CREATE)
+    {
+        /// Now let's handle the merge tree family. Note we only handle in the mode of CREATE due to backward compatibility.
+        /// Otherwise, it would fail to start in the case of existing projections with special mergetree.
+        if (merging_params.mode != MergeTreeData::MergingParams::Mode::Ordinary
+            && storage_settings->deduplicate_merge_projection_mode == DeduplicateMergeProjectionMode::THROW)
+            throw Exception(ErrorCodes::SUPPORT_IS_DISABLED,
+                "Projection is fully supported in {}MergeTree with deduplicate_merge_projection_mode = throw. "
+                "Use 'drop' or 'rebuild' option of deduplicate_merge_projection_mode.",
+                merging_params.getModeName());
+    }
+
     if (arg_num != arg_cnt)
         throw Exception(ErrorCodes::BAD_ARGUMENTS, "Wrong number of engine arguments.");
 
diff --git a/src/Storages/StorageFactory.cpp b/src/Storages/StorageFactory.cpp
index 557f53a9ada..25cb81fa5fa 100644
--- a/src/Storages/StorageFactory.cpp
+++ b/src/Storages/StorageFactory.cpp
@@ -20,7 +20,6 @@ namespace ErrorCodes
     extern const int FUNCTION_CANNOT_HAVE_PARAMETERS;
     extern const int BAD_ARGUMENTS;
     extern const int DATA_TYPE_CANNOT_BE_USED_IN_TABLES;
-    extern const int SUPPORT_IS_DISABLED;
 }
 
 
@@ -201,33 +200,6 @@ StoragePtr StorageFactory::get(
                 check_feature(
                     "projections",
                     [](StorageFeatures features) { return features.supports_projections; });
-
-                /// Now let's handle the merge tree family. Note we only handle in the mode of CREATE due to backward compatibility.
-                /// Otherwise, it would fail to start in the case of existing projections with special mergetree.
-                chassert(query.storage->engine);
-                const std::unordered_set<String> allowed_engines{"MergeTree", "ReplicatedMergeTree", "SharedMergeTree"};
-                if (auto engine_name(query.storage->engine->name); mode == LoadingStrictnessLevel::CREATE
-                    && !allowed_engines.contains(engine_name))
-                {
-                    /// default throw mode in deduplicate_merge_projection_mode
-                    bool projection_allowed = false;
-                    if (auto * setting = query.storage->settings; setting != nullptr)
-                    {
-                        for (const auto & change : setting->changes)
-                        {
-                            if (change.name == "deduplicate_merge_projection_mode" && change.value != Field("throw"))
-                            {
-                                projection_allowed = true;
-                                break;
-                            }
-                        }
-                    }
-                    if (!projection_allowed)
-                        throw Exception(ErrorCodes::SUPPORT_IS_DISABLED,
-                            "Projection is fully supported in (Replictaed, Shared)MergeTree, but also allowed in non-throw mode with other"
-                            " mergetree family members. Consider drop or rebuild option of deduplicate_merge_projection_mode."
-                            " Current storage name is {}.", engine_name);
-                }
             }
         }
     }

From c74460b47e23ced4562d6f5123d61042c42d2896 Mon Sep 17 00:00:00 2001
From: Nikolai Kochetov <nk@clickhouse.com>
Date: Tue, 6 Aug 2024 17:53:23 +0000
Subject: [PATCH 1475/1488] Cleanup.

---
 src/Storages/StorageFactory.cpp                                 | 2 --
 .../0_stateless/03206_projection_merge_special_mergetree.sql    | 2 +-
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/src/Storages/StorageFactory.cpp b/src/Storages/StorageFactory.cpp
index 25cb81fa5fa..060b271d8f4 100644
--- a/src/Storages/StorageFactory.cpp
+++ b/src/Storages/StorageFactory.cpp
@@ -196,11 +196,9 @@ StoragePtr StorageFactory::get(
                     [](StorageFeatures features) { return features.supports_skipping_indices; });
 
             if (query.columns_list && query.columns_list->projections && !query.columns_list->projections->children.empty())
-            {
                 check_feature(
                     "projections",
                     [](StorageFeatures features) { return features.supports_projections; });
-            }
         }
     }
 
diff --git a/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql b/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql
index d3448138396..82684f754b6 100644
--- a/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql
+++ b/tests/queries/0_stateless/03206_projection_merge_special_mergetree.sql
@@ -103,4 +103,4 @@ SELECT
 FROM system.projection_parts
 WHERE (database = currentDatabase()) AND (`table` = 'tp') AND (active = 1);
 
-DROP TABLE tp;
\ No newline at end of file
+DROP TABLE tp;

From 302bd5fdc6f077acf05a17c5e700cb44b2e609ab Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Tue, 6 Aug 2024 18:37:26 +0000
Subject: [PATCH 1476/1488] Remove usearch submodule

---
 .gitmodules     | 3 ---
 contrib/usearch | 1 -
 2 files changed, 4 deletions(-)
 delete mode 160000 contrib/usearch

diff --git a/.gitmodules b/.gitmodules
index 7e0b4df4ad1..c4c93822711 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -339,9 +339,6 @@
 [submodule "contrib/incbin"]
 	path = contrib/incbin
 	url = https://github.com/graphitemaster/incbin.git
-[submodule "contrib/usearch"]
-	path = contrib/usearch
-	url = https://github.com/unum-cloud/usearch.git
 [submodule "contrib/SimSIMD"]
 	path = contrib/SimSIMD
 	url = https://github.com/ashvardanian/SimSIMD.git
diff --git a/contrib/usearch b/contrib/usearch
deleted file mode 160000
index 955c6f9c11a..00000000000
--- a/contrib/usearch
+++ /dev/null
@@ -1 +0,0 @@
-Subproject commit 955c6f9c11adfd89c912e0d1643d160b4e9e543f

From 7a5b30d955b85a164097b4c7dd5ce957a47059e5 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Tue, 6 Aug 2024 18:39:18 +0000
Subject: [PATCH 1477/1488] Re-add forked usearch repo

---
 .gitmodules     | 3 +++
 contrib/usearch | 1 +
 2 files changed, 4 insertions(+)
 create mode 160000 contrib/usearch

diff --git a/.gitmodules b/.gitmodules
index c4c93822711..7fdfb1103c5 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -339,6 +339,9 @@
 [submodule "contrib/incbin"]
 	path = contrib/incbin
 	url = https://github.com/graphitemaster/incbin.git
+[submodule "contrib/usearch"]
+	path = contrib/usearch
+	url = https://github.com/ClickHouse/usearch.git
 [submodule "contrib/SimSIMD"]
 	path = contrib/SimSIMD
 	url = https://github.com/ashvardanian/SimSIMD.git
diff --git a/contrib/usearch b/contrib/usearch
new file mode 160000
index 00000000000..955c6f9c11a
--- /dev/null
+++ b/contrib/usearch
@@ -0,0 +1 @@
+Subproject commit 955c6f9c11adfd89c912e0d1643d160b4e9e543f

From 9594a9baffb125e21b28cd421b511d26febfc900 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Tue, 6 Aug 2024 18:43:44 +0000
Subject: [PATCH 1478/1488] Fix memory corruption in usearch

---
 contrib/usearch | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/contrib/usearch b/contrib/usearch
index 955c6f9c11a..30810452bec 160000
--- a/contrib/usearch
+++ b/contrib/usearch
@@ -1 +1 @@
-Subproject commit 955c6f9c11adfd89c912e0d1643d160b4e9e543f
+Subproject commit 30810452bec5d3d3aa0931bb5d761e2f09aa6356

From 36b6adbb30efdc0a96a274bdcbaeec2d961a8bed Mon Sep 17 00:00:00 2001
From: "Zhukova, Maria" <maria.zhukova@intel.com>
Date: Tue, 6 Aug 2024 12:15:23 -0700
Subject: [PATCH 1479/1488] qpl-cmake: Better wording on ISA-L copy and removed
 non-required linking

---
 contrib/qpl-cmake/CMakeLists.txt | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/contrib/qpl-cmake/CMakeLists.txt b/contrib/qpl-cmake/CMakeLists.txt
index b2f263252c2..e62612cff5a 100644
--- a/contrib/qpl-cmake/CMakeLists.txt
+++ b/contrib/qpl-cmake/CMakeLists.txt
@@ -24,7 +24,9 @@ message(STATUS "Intel QPL version: ${QPL_VERSION}")
 # which are then combined into static or shared qpl.
 # Output ch_contrib::qpl by linking with 8 library targets.
 
-# Note, qpl submodule comes with its own version of isal that is not compatible with upstream isal (e.g., ch_contrib::isal).
+# Note, QPL has integrated a customized version of ISA-L to meet specific needs.
+# This version has been significantly modified and there are no plans to maintain compatibility with the upstream version
+# or upgrade the current copy.
 
 ## cmake/CompileOptions.cmake and automatic wrappers generation
 
@@ -733,10 +735,6 @@ target_compile_definitions(_qpl
 target_link_libraries(_qpl
         PRIVATE ch_contrib::accel-config)
 
-# C++ filesystem library requires additional linking for older GNU/Clang
-target_link_libraries(_qpl PRIVATE $<$<AND:$<CXX_COMPILER_ID:GNU>,$<VERSION_LESS:$<CXX_COMPILER_VERSION>,9.1>>:stdc++fs>)
-target_link_libraries(_qpl PRIVATE $<$<AND:$<CXX_COMPILER_ID:Clang>,$<VERSION_LESS:$<CXX_COMPILER_VERSION>,9.0>>:c++fs>)
-
 target_include_directories(_qpl SYSTEM BEFORE
         PUBLIC "${QPL_PROJECT_DIR}/include"
         PUBLIC ${UUID_DIR})

From 139f5e55d34092969e49e4fd191404cac0ddd05b Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Tue, 6 Aug 2024 20:31:41 +0000
Subject: [PATCH 1480/1488] Unflake 02099_tsv_raw_format.sh

---
 ...rence => 02099_tsv_raw_format_1.reference} |  2 --
 ...aw_format.sh => 02099_tsv_raw_format_1.sh} | 13 -------------
 .../02099_tsv_raw_format_2.reference          |  2 ++
 .../0_stateless/02099_tsv_raw_format_2.sh     | 19 +++++++++++++++++++
 4 files changed, 21 insertions(+), 15 deletions(-)
 rename tests/queries/0_stateless/{02099_tsv_raw_format.reference => 02099_tsv_raw_format_1.reference} (94%)
 rename tests/queries/0_stateless/{02099_tsv_raw_format.sh => 02099_tsv_raw_format_1.sh} (69%)
 create mode 100644 tests/queries/0_stateless/02099_tsv_raw_format_2.reference
 create mode 100755 tests/queries/0_stateless/02099_tsv_raw_format_2.sh

diff --git a/tests/queries/0_stateless/02099_tsv_raw_format.reference b/tests/queries/0_stateless/02099_tsv_raw_format_1.reference
similarity index 94%
rename from tests/queries/0_stateless/02099_tsv_raw_format.reference
rename to tests/queries/0_stateless/02099_tsv_raw_format_1.reference
index de46cf8dff7..3ac175e51f6 100644
--- a/tests/queries/0_stateless/02099_tsv_raw_format.reference
+++ b/tests/queries/0_stateless/02099_tsv_raw_format_1.reference
@@ -109,5 +109,3 @@ UInt64	String	Date
 2
 \N
 nSome text
-b1cad4eb4be08a40387c9de70d02fcc2  -
-b1cad4eb4be08a40387c9de70d02fcc2  -
diff --git a/tests/queries/0_stateless/02099_tsv_raw_format.sh b/tests/queries/0_stateless/02099_tsv_raw_format_1.sh
similarity index 69%
rename from tests/queries/0_stateless/02099_tsv_raw_format.sh
rename to tests/queries/0_stateless/02099_tsv_raw_format_1.sh
index a69c96ab613..a3468f46ca0 100755
--- a/tests/queries/0_stateless/02099_tsv_raw_format.sh
+++ b/tests/queries/0_stateless/02099_tsv_raw_format_1.sh
@@ -46,16 +46,3 @@ echo 'nSome text' | $CLICKHOUSE_CLIENT -q "INSERT INTO test_nullable_string_0209
 
 $CLICKHOUSE_CLIENT -q "SELECT * FROM test_nullable_string_02099"
 $CLICKHOUSE_CLIENT -q "DROP TABLE test_nullable_string_02099"
-
-
-$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS test_parallel_parsing_02099"
-$CLICKHOUSE_CLIENT -q "CREATE TABLE test_parallel_parsing_02099 (x UInt64, a Array(UInt64), s String) ENGINE=Memory()";
-$CLICKHOUSE_CLIENT -q "SELECT number AS x, range(number % 50) AS a, toString(a) AS s FROM numbers(1000000) FORMAT TSVRaw" | $CLICKHOUSE_CLIENT --input_format_parallel_parsing=0 -q "INSERT INTO test_parallel_parsing_02099 FORMAT TSVRaw"
-$CLICKHOUSE_CLIENT -q "SELECT * FROM test_parallel_parsing_02099 ORDER BY x" | md5sum
-
-$CLICKHOUSE_CLIENT -q "TRUNCATE TABLE test_parallel_parsing_02099"
-
-$CLICKHOUSE_CLIENT -q "SELECT number AS x, range(number % 50) AS a, toString(a) AS s FROM numbers(1000000) FORMAT TSVRaw" | $CLICKHOUSE_CLIENT --input_format_parallel_parsing=1 -q "INSERT INTO test_parallel_parsing_02099 FORMAT TSVRaw"
-$CLICKHOUSE_CLIENT -q "SELECT * FROM test_parallel_parsing_02099 ORDER BY x" | md5sum
-
-$CLICKHOUSE_CLIENT -q "DROP TABLE test_parallel_parsing_02099"
diff --git a/tests/queries/0_stateless/02099_tsv_raw_format_2.reference b/tests/queries/0_stateless/02099_tsv_raw_format_2.reference
new file mode 100644
index 00000000000..4682749c21f
--- /dev/null
+++ b/tests/queries/0_stateless/02099_tsv_raw_format_2.reference
@@ -0,0 +1,2 @@
+c8ff17885084035ea1aebd95fee2efb6  -
+c8ff17885084035ea1aebd95fee2efb6  -
diff --git a/tests/queries/0_stateless/02099_tsv_raw_format_2.sh b/tests/queries/0_stateless/02099_tsv_raw_format_2.sh
new file mode 100755
index 00000000000..d6034a0616f
--- /dev/null
+++ b/tests/queries/0_stateless/02099_tsv_raw_format_2.sh
@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+# Tags: long
+
+CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CURDIR"/../shell_config.sh
+
+$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS test_parallel_parsing_02099"
+$CLICKHOUSE_CLIENT -q "CREATE TABLE test_parallel_parsing_02099 (x UInt64, a Array(UInt64), s String) ENGINE=Memory()";
+$CLICKHOUSE_CLIENT -q "SELECT number AS x, range(number % 50) AS a, toString(a) AS s FROM numbers(100000) FORMAT TSVRaw" | $CLICKHOUSE_CLIENT --input_format_parallel_parsing=0 -q "INSERT INTO test_parallel_parsing_02099 FORMAT TSVRaw"
+$CLICKHOUSE_CLIENT -q "SELECT * FROM test_parallel_parsing_02099 ORDER BY x" | md5sum
+
+$CLICKHOUSE_CLIENT -q "TRUNCATE TABLE test_parallel_parsing_02099"
+
+$CLICKHOUSE_CLIENT -q "SELECT number AS x, range(number % 50) AS a, toString(a) AS s FROM numbers(100000) FORMAT TSVRaw" | $CLICKHOUSE_CLIENT --input_format_parallel_parsing=1 -q "INSERT INTO test_parallel_parsing_02099 FORMAT TSVRaw"
+$CLICKHOUSE_CLIENT -q "SELECT * FROM test_parallel_parsing_02099 ORDER BY x" | md5sum
+
+$CLICKHOUSE_CLIENT -q "DROP TABLE test_parallel_parsing_02099"
+

From ead9dc42d8a75ac99a0e4f538764cba206ad59a7 Mon Sep 17 00:00:00 2001
From: Robert Schulze <robert@clickhouse.com>
Date: Wed, 7 Aug 2024 08:31:49 +0000
Subject: [PATCH 1481/1488] Make it worse

---
 ...02310_clickhouse_local_INSERT_progress_profile_events.expect | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/queries/0_stateless/02310_clickhouse_local_INSERT_progress_profile_events.expect b/tests/queries/0_stateless/02310_clickhouse_local_INSERT_progress_profile_events.expect
index d5b2a278180..5c731ea6d89 100755
--- a/tests/queries/0_stateless/02310_clickhouse_local_INSERT_progress_profile_events.expect
+++ b/tests/queries/0_stateless/02310_clickhouse_local_INSERT_progress_profile_events.expect
@@ -1,5 +1,5 @@
 #!/usr/bin/expect -f
-# Tags: no-debug, no-tsan, no-msan, no-asan, no-ubsan, no-s3-storage
+# Tags: no-debug, no-tsan, no-msan, no-asan, no-ubsan, no-s3-storage, no-cpu-aarch64
 # ^ it can be slower than 60 seconds
 
 # This is the regression for the concurrent access in ProgressIndication,

From de41ffa18f5c2d3533fab3222aa401c193a0baac Mon Sep 17 00:00:00 2001
From: maxvostrikov <max.vostrikov@clickhouse.com>
Date: Wed, 7 Aug 2024 12:06:14 +0200
Subject: [PATCH 1482/1488] fix docks for groupConcat function documentation
 had wrong usage of a function. Examples of usage -
 https://github.com/ClickHouse/ClickHouse/blob/763952bf36d0b55f1b33ff11c693267574aa9666/tests/queries/0_stateless/03156_group_concat.sql

---
 .../aggregate-functions/reference/groupconcat.md            | 6 +++---
 tests/instructions/easy_tasks_sorted_ru.md                  | 4 ----
 2 files changed, 3 insertions(+), 7 deletions(-)

diff --git a/docs/en/sql-reference/aggregate-functions/reference/groupconcat.md b/docs/en/sql-reference/aggregate-functions/reference/groupconcat.md
index 072252de8c9..bfa6160bbcc 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/groupconcat.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/groupconcat.md
@@ -10,7 +10,7 @@ Calculates a concatenated string from a group of strings, optionally separated b
 **Syntax**
 
 ``` sql
-groupConcat(expression [, delimiter] [, limit]);
+groupConcat[(delimiter [, limit])](expression);
 ```
 
 **Arguments**
@@ -20,7 +20,7 @@ groupConcat(expression [, delimiter] [, limit]);
 - `limit` — A positive [integer](../../../sql-reference/data-types/int-uint.md) specifying the maximum number of elements to concatenate. If more elements are present, excess elements are ignored. This parameter is optional.
 
 :::note
-If delimiter is specified without limit, it must be the first parameter following the expression. If both delimiter and limit are specified, delimiter must precede limit.
+If delimiter is specified without limit, it must be the first parameter. If both delimiter and limit are specified, delimiter must precede limit.
 :::
 
 **Returned value**
@@ -61,7 +61,7 @@ This concatenates all names into one continuous string without any separator.
 Query:
 
 ``` sql
-SELECT groupConcat(Name, ', ', 2) FROM Employees;
+SELECT groupConcat(', ')(Name)  FROM Employees;
 ```
 
 Result:
diff --git a/tests/instructions/easy_tasks_sorted_ru.md b/tests/instructions/easy_tasks_sorted_ru.md
index fbd86ebf08f..e0607126ecc 100644
--- a/tests/instructions/easy_tasks_sorted_ru.md
+++ b/tests/instructions/easy_tasks_sorted_ru.md
@@ -97,10 +97,6 @@ Upd: сделали по-другому: теперь всё безопасно.
 
 Возвращает инкрементальное число для повторно встречающихся значений key.
 
-## Агрегатная функция groupConcat.
-
-`groupConcat(x, ',')` - собрать из переданных значений x строку, разделённую запятыми.
-
 ## Функции DATE_ADD, DATE_SUB как синонимы для совместимости с SQL.
 
 https://dev.mysql.com/doc/refman/8.0/en/date-and-time-functions.html#function_date-add

From a9735f470c195dadbfffe02545b0979bfa9bd778 Mon Sep 17 00:00:00 2001
From: maxvostrikov <max.vostrikov@clickhouse.com>
Date: Wed, 7 Aug 2024 12:09:26 +0200
Subject: [PATCH 1483/1488] squash! fix docks for groupConcat function
 documentation had wrong usage of a function. Examples of usage -
 https://github.com/ClickHouse/ClickHouse/blob/763952bf36d0b55f1b33ff11c693267574aa9666/tests/queries/0_stateless/03156_group_concat.sql

---
 .../sql-reference/aggregate-functions/reference/groupconcat.md  | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/en/sql-reference/aggregate-functions/reference/groupconcat.md b/docs/en/sql-reference/aggregate-functions/reference/groupconcat.md
index bfa6160bbcc..6a24aa244bf 100644
--- a/docs/en/sql-reference/aggregate-functions/reference/groupconcat.md
+++ b/docs/en/sql-reference/aggregate-functions/reference/groupconcat.md
@@ -78,7 +78,7 @@ This output shows the names separated by a comma followed by a space.
 Query:
 
 ``` sql
-SELECT groupConcat(Name, ', ', 2) FROM Employees;
+SELECT groupConcat(', ', 2)(Name) FROM Employees;
 ```
 
 Result:

From e8bf5129c03dd88712829e9e187145e248ba4f04 Mon Sep 17 00:00:00 2001
From: maxvostrikov <max.vostrikov@clickhouse.com>
Date: Wed, 7 Aug 2024 12:44:31 +0200
Subject: [PATCH 1484/1488] fix docks for clickhouse-keeper-client starting
 24.7 paths are not accepted as bare strings, only as string literals -
 https://github.com/ClickHouse/ClickHouse/pull/65494

---
 .../utilities/clickhouse-keeper-client.md     | 36 +++++++++----------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/docs/en/operations/utilities/clickhouse-keeper-client.md b/docs/en/operations/utilities/clickhouse-keeper-client.md
index 6407c66783b..a66ecbc1372 100644
--- a/docs/en/operations/utilities/clickhouse-keeper-client.md
+++ b/docs/en/operations/utilities/clickhouse-keeper-client.md
@@ -28,39 +28,39 @@ A client application to interact with clickhouse-keeper by its native protocol.
 Connected to ZooKeeper at [::1]:9181 with session_id 137
 / :) ls
 keeper foo bar
-/ :) cd keeper
+/ :) cd 'keeper'
 /keeper :) ls
 api_version
-/keeper :) cd api_version
+/keeper :) cd 'api_version'
 /keeper/api_version :) ls
 
-/keeper/api_version :) cd xyz
+/keeper/api_version :) cd 'xyz'
 Path /keeper/api_version/xyz does not exist
 /keeper/api_version :) cd ../../
 / :) ls
 keeper foo bar
-/ :) get keeper/api_version
+/ :) get 'keeper/api_version'
 2
 ```
 
 ## Commands {#clickhouse-keeper-client-commands}
 
--   `ls [path]` -- Lists the nodes for the given path (default: cwd)
--   `cd [path]` -- Changes the working path (default `.`)
--   `exists <path>` -- Returns `1` if node exists, `0` otherwise
--   `set <path> <value> [version]` -- Updates the node's value. Only updates if version matches (default: -1)
--   `create <path> <value> [mode]` -- Creates new node with the set value
--   `touch <path>` -- Creates new node with an empty string as value. Doesn't throw an exception if the node already exists
--   `get <path>` -- Returns the node's value
--   `rm <path> [version]` -- Removes the node only if version matches (default: -1)
--   `rmr <path>` -- Recursively deletes path. Confirmation required
+-   `ls '[path]'` -- Lists the nodes for the given path (default: cwd)
+-   `cd '[path]'` -- Changes the working path (default `.`)
+-   `exists '<path>'` -- Returns `1` if node exists, `0` otherwise
+-   `set '<path>' <value> [version]` -- Updates the node's value. Only updates if version matches (default: -1)
+-   `create '<path>' <value> [mode]` -- Creates new node with the set value
+-   `touch '<path>'` -- Creates new node with an empty string as value. Doesn't throw an exception if the node already exists
+-   `get '<path>'` -- Returns the node's value
+-   `rm '<path>' [version]` -- Removes the node only if version matches (default: -1)
+-   `rmr '<path>'` -- Recursively deletes path. Confirmation required
 -   `flwc <command>` -- Executes four-letter-word command
 -   `help` -- Prints this message
--   `get_direct_children_number [path]` -- Get numbers of direct children nodes under a specific path
--   `get_all_children_number [path]` -- Get all numbers of children nodes under a specific path
--   `get_stat [path]` -- Returns the node's stat (default `.`)
--   `find_super_nodes <threshold> [path]` -- Finds nodes with number of children larger than some threshold for the given path (default `.`)
+-   `get_direct_children_number '[path]'` -- Get numbers of direct children nodes under a specific path
+-   `get_all_children_number '[path]'` -- Get all numbers of children nodes under a specific path
+-   `get_stat '[path]'` -- Returns the node's stat (default `.`)
+-   `find_super_nodes <threshold> '[path]'` -- Finds nodes with number of children larger than some threshold for the given path (default `.`)
 -   `delete_stale_backups` -- Deletes ClickHouse nodes used for backups that are now inactive
 -   `find_big_family [path] [n]` -- Returns the top n nodes with the biggest family in the subtree (default path = `.` and n = 10)
--   `sync <path>` -- Synchronizes node between processes and leader
+-   `sync '<path>'` -- Synchronizes node between processes and leader
 -   `reconfig <add|remove|set> "<arg>" [version]` -- Reconfigure Keeper cluster. See https://clickhouse.com/docs/en/guides/sre/keeper/clickhouse-keeper#reconfiguration

From c39cdcffaff5917510b27f457e35a63dddeed0b5 Mon Sep 17 00:00:00 2001
From: maxvostrikov <max.vostrikov@clickhouse.com>
Date: Wed, 7 Aug 2024 14:03:53 +0200
Subject: [PATCH 1485/1488] docs for lightweight_mutation_projection_mode
 option starting with 24.7 we can lightweight delete in tables with
 projections. Fixed docs stating it is not possible

---
 docs/en/sql-reference/statements/delete.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/docs/en/sql-reference/statements/delete.md b/docs/en/sql-reference/statements/delete.md
index a52b7204c30..8745a06c124 100644
--- a/docs/en/sql-reference/statements/delete.md
+++ b/docs/en/sql-reference/statements/delete.md
@@ -36,9 +36,10 @@ If you anticipate frequent deletes, consider using a [custom partitioning key](/
 
 ## Limitations of lightweight `DELETE`
 
-### Lightweight `DELETE`s do not work with projections
+### Lightweight `DELETE`s with projections
 
-Currently, `DELETE` does not work for tables with projections. This is because rows in a projection may be affected by a `DELETE` operation and may require the projection to be rebuilt, negatively affecting `DELETE` performance.
+By default, `DELETE` does not work for tables with projections. This is because rows in a projection may be affected by a `DELETE` operation and may require the projection to be rebuilt, negatively affecting `DELETE` performance.
+However, there is an option to change this behavior. By changing setting `lightweight_mutation_projection_mode = 'drop'` will work with projections.
 
 ## Performance considerations when using lightweight `DELETE`
 

From 37a6bd25f6d6dd2bfe3913639a7650c313642fb5 Mon Sep 17 00:00:00 2001
From: maxvostrikov <max.vostrikov@clickhouse.com>
Date: Wed, 7 Aug 2024 14:06:02 +0200
Subject: [PATCH 1486/1488] squash! docs for
 lightweight_mutation_projection_mode option starting with 24.7 we can
 lightweight delete in tables with projections. Fixed docs stating it is not
 possible

---
 docs/en/sql-reference/statements/delete.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/en/sql-reference/statements/delete.md b/docs/en/sql-reference/statements/delete.md
index 8745a06c124..88a9c933519 100644
--- a/docs/en/sql-reference/statements/delete.md
+++ b/docs/en/sql-reference/statements/delete.md
@@ -39,7 +39,7 @@ If you anticipate frequent deletes, consider using a [custom partitioning key](/
 ### Lightweight `DELETE`s with projections
 
 By default, `DELETE` does not work for tables with projections. This is because rows in a projection may be affected by a `DELETE` operation and may require the projection to be rebuilt, negatively affecting `DELETE` performance.
-However, there is an option to change this behavior. By changing setting `lightweight_mutation_projection_mode = 'drop'` will work with projections.
+However, there is an option to change this behavior. By changing setting `lightweight_mutation_projection_mode = 'drop'`, deletes will work with projections.
 
 ## Performance considerations when using lightweight `DELETE`
 

From ecba21bfe78fabf76eb7207f40e3fe53ecb70eaf Mon Sep 17 00:00:00 2001
From: Dmitry Novik <mrnovikd@gmail.com>
Date: Wed, 7 Aug 2024 15:23:19 +0200
Subject: [PATCH 1487/1488] Fix documentation for memory overcommit

---
 docs/en/operations/settings/settings.md | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md
index 2b11c836fc1..968481062e9 100644
--- a/docs/en/operations/settings/settings.md
+++ b/docs/en/operations/settings/settings.md
@@ -4629,8 +4629,8 @@ Default Value: 5.
 
 ## memory_overcommit_ratio_denominator {#memory_overcommit_ratio_denominator}
 
-It represents soft memory limit in case when hard limit is reached on user level.
-This value is used to compute overcommit ratio for the query.
+It represents the soft memory limit when the hard limit is reached on the global level.
+This value is used to compute the overcommit ratio for the query.
 Zero means skip the query.
 Read more about [memory overcommit](memory-overcommit.md).
 
@@ -4646,8 +4646,8 @@ Default value: `5000000`.
 
 ## memory_overcommit_ratio_denominator_for_user {#memory_overcommit_ratio_denominator_for_user}
 
-It represents soft memory limit in case when hard limit is reached on global level.
-This value is used to compute overcommit ratio for the query.
+It represents the soft memory limit when the hard limit is reached on the user level.
+This value is used to compute the overcommit ratio for the query.
 Zero means skip the query.
 Read more about [memory overcommit](memory-overcommit.md).
 

From 9a812fc53502cbd3bb69964d8e8b1afb186e6b46 Mon Sep 17 00:00:00 2001
From: "Max K." <max.kainov@clickhouse.com>
Date: Wed, 7 Aug 2024 15:55:03 +0200
Subject: [PATCH 1488/1488] Revert "CI: Strict job timeout 1.5h for tests, 2h
 for builds"

---
 tests/ci/ci_config.py      | 1 +
 tests/ci/ci_definitions.py | 6 ++++--
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py
index f578cd8b559..7a19eb6f827 100644
--- a/tests/ci/ci_config.py
+++ b/tests/ci/ci_config.py
@@ -510,6 +510,7 @@ class CI:
         JobNames.LIBFUZZER_TEST: JobConfig(
             required_builds=[BuildNames.FUZZERS],
             run_by_label=Tags.libFuzzer,
+            timeout=10800,
             run_command='libfuzzer_test_check.py "$CHECK_NAME"',
             runner_type=Runners.STYLE_CHECKER,
         ),
diff --git a/tests/ci/ci_definitions.py b/tests/ci/ci_definitions.py
index 69e7ed259d5..48847b0d7a6 100644
--- a/tests/ci/ci_definitions.py
+++ b/tests/ci/ci_definitions.py
@@ -331,7 +331,7 @@ class JobConfig:
     # will be triggered for the job if omitted in CI workflow yml
     run_command: str = ""
     # job timeout, seconds
-    timeout: Optional[int] = 5400
+    timeout: Optional[int] = None
     # sets number of batches for a multi-batch job
     num_batches: int = 1
     # label that enables job in CI, if set digest isn't used
@@ -420,6 +420,7 @@ class CommonJobConfigs:
         ),
         run_command='functional_test_check.py "$CHECK_NAME"',
         runner_type=Runners.FUNC_TESTER,
+        timeout=9000,
     )
     STATEFUL_TEST = JobConfig(
         job_name_keyword="stateful",
@@ -530,6 +531,7 @@ class CommonJobConfigs:
             docker=["clickhouse/sqllogic-test"],
         ),
         run_command="sqllogic_test.py",
+        timeout=10800,
         release_only=True,
         runner_type=Runners.FUNC_TESTER,
     )
@@ -541,6 +543,7 @@ class CommonJobConfigs:
             docker=["clickhouse/sqltest"],
         ),
         run_command="sqltest.py",
+        timeout=10800,
         release_only=True,
         runner_type=Runners.FUZZER_UNIT_TESTER,
     )
@@ -610,7 +613,6 @@ class CommonJobConfigs:
             docker=["clickhouse/binary-builder"],
             git_submodules=True,
         ),
-        timeout=7200,
         run_command="build_check.py $BUILD_NAME",
         runner_type=Runners.BUILDER,
     )